⬆️ @huggingface/inference v2.1.1 (#83)

package-lock.json

@@ -8,7 +8,7 @@
 			"name": "chat-ui",
 			"version": "0.0.1",
 			"dependencies": {
-				"@huggingface/inference": "^2.0.0-rc2",
+				"@huggingface/inference": "^2.1.2",
 				"autoprefixer": "^10.4.14",
 				"date-fns": "^2.29.3",
 				"dotenv": "^16.0.3",
@@ -473,9 +473,9 @@
 			}
 		},
 		"node_modules/@huggingface/inference": {
-			"version": "2.0.0-rc2",
-			"resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.0.0-rc2.tgz",
-			"integrity": "sha512-jCU+zl1fmbmaWbTa3P2KC1GQMwjMc9ZLcR8Nq5UeFmDbmdz0GiBHsK7F1hTHRAPeF1K3X5L3V3OQUNZ6nzEu/w==",
+			"version": "2.1.2",
+			"resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.1.2.tgz",
+			"integrity": "sha512-Qb+yVq13a1FVOLaQK0GOwDfp95VfIWP7uR83tAyluSkvpwAHaByVKw4zNGwMl3u3FxoDRn6snNGo54wTWApPzw==",
 			"engines": {
 				"node": ">=18"
 			}

package.json

@@ -33,7 +33,7 @@
 	},
 	"type": "module",
 	"dependencies": {
-		"@huggingface/inference": "^2.0.0-rc2",
+		"@huggingface/inference": "^2.1.2",
 		"autoprefixer": "^10.4.14",
 		"date-fns": "^2.29.3",
 		"dotenv": "^16.0.3",

src/routes/conversation/[id]/+page.svelte

@@ -4,7 +4,7 @@
 	import { onMount } from "svelte";
 	import type { PageData } from "./$types";
 	import { page } from "$app/stores";
-	import { HfInference } from "@huggingface/inference";
+	import { textGenerationStream } from "@huggingface/inference";
 	import { invalidate } from "$app/navigation";
 	import { base } from "$app/paths";
 	import { trimSuffix } from "$lib/utils/trimSuffix";
@@ -24,16 +24,15 @@
 		lastLoadedMessages = data.messages;
 	}
 
-	const hf = new HfInference();
-
 	let loading = false;
 	let pending = false;
 
 	async function getTextGenerationStream(inputs: string) {
 		let conversationId = $page.params.id;
 
-		const response = hf.endpoint($page.url.href).textGenerationStream(
+		const response = textGenerationStream(
 			{
+				model: $page.url.href,
 				inputs,
 				parameters: {
 					// Taken from https://huggingface.co/spaces/huggingface/open-assistant-private-testing/blob/main/app.py#L54

src/routes/conversation/[id]/summarize/+server.ts

@@ -2,6 +2,7 @@ import { HF_TOKEN } from "$env/static/private";
 import { PUBLIC_MODEL_ENDPOINT } from "$env/static/public";
 import { buildPrompt } from "$lib/buildPrompt";
 import { collections } from "$lib/server/database.js";
+import { textGeneration } from "@huggingface/inference";
 import { error } from "@sveltejs/kit";
 import { ObjectId } from "mongodb";
 
@@ -25,54 +26,52 @@ export async function POST({ params, locals, fetch }) {
 
 	const prompt = buildPrompt([{ from: "user", content: userPrompt }]);
 
-	const resp = await fetch(PUBLIC_MODEL_ENDPOINT, {
-		headers: {
-			"Content-Type": "application/json",
-			Authorization: `Basic ${HF_TOKEN}`,
-		},
-		method: "POST",
-		body: JSON.stringify({
-			inputs: prompt,
-			parameters: {
-				temperature: 0.9,
-				top_p: 0.95,
-				repetition_penalty: 1.2,
-				top_k: 50,
-				watermark: false,
-				max_new_tokens: 1024,
-				stop: ["<|endoftext|>"],
-				return_full_text: false,
-			},
-		}),
-	});
+	const parameters = {
+		temperature: 0.9,
+		top_p: 0.95,
+		repetition_penalty: 1.2,
+		top_k: 50,
+		watermark: false,
+		max_new_tokens: 1024,
+		stop: ["<|endoftext|>"],
+		return_full_text: false,
+	};
 
-	const response = await resp.json();
-	let generatedTitle: string | undefined;
-	try {
-		if (typeof response[0].generated_text === "string") {
-			generatedTitle = response[0].generated_text;
+	const { generated_text } = await textGeneration(
+		{
+			model: PUBLIC_MODEL_ENDPOINT,
+			inputs: prompt,
+			parameters,
+		},
+		{
+			fetch: (url, options) =>
+				fetch(url, {
+					...options,
+					headers: {
+						...options?.headers,
+						Authorization: `Basic ${HF_TOKEN}`,
+					},
+				}),
 		}
-	} catch {
-		console.error("summarization failed");
-	}
+	);
 
-	if (generatedTitle) {
+	if (generated_text) {
 		await collections.conversations.updateOne(
 			{
 				_id: convId,
 				sessionId: locals.sessionId,
 			},
 			{
-				$set: { title: generatedTitle },
+				$set: { title: generated_text },
 			}
 		);
 	}
 
 	return new Response(
 		JSON.stringify(
-			generatedTitle
+			generated_text
 				? {
-						title: generatedTitle,
+						title: generated_text,
 				  }
 				: {}
 		),