Models selector (#164)

Co-authored-by: Victor Mustar <[email protected]>
Co-authored-by: Julien Chaumond <[email protected]>
Co-authored-by: coyotte508 <[email protected]>

.env

@@ -5,25 +5,67 @@ MONGODB_URL=#your mongodb URL here
 MONGODB_DB_NAME=chat-ui
 COOKIE_NAME=hf-chat
 HF_ACCESS_TOKEN=#hf_<token> from from https://huggingface.co/settings/token
-MODELS=`["OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"]`
-# Alternative syntax (all fields are optional except 'name'):
-# MODELS=`[{
-#  "name": "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5", 
-#  "displayName": "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
-#  "endpoints": [{
-#    "url": "https://api-inference.huggingface.co/models/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
-#    "authorization": "Bearer hf_<token>",
-#    "weight": 1
-#  }]
-# }]
+# 'name', 'userMessageToken', 'assistantMessageToken', 'parameters' are required
+MODELS=`[
+  {
+    "name": "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
+    "datasetName": "OpenAssistant/oasst1",
+    "websiteUrl": "https://open-assistant.io",
+    "userMessageToken": "<|prompter|>",
+    "assistantMessageToken": "<|assistant|>",
+    "preprompt": "Below are a series of dialogues between various people and an AI assistant. The AI tries to be helpful, polite, honest, sophisticated, emotionally aware, and humble-but-knowledgeable. The assistant is happy to help with almost anything, and will do its best to understand exactly what is needed. It also tries to avoid giving false or misleading information, and it caveats when it isn't entirely sure about the right answer. That said, the assistant is practical and really does its best, and doesn't let caution get too much in the way of being useful.\n-----\n",
+    "promptExamples": [
+      {
+        "title": "Write an email from bullet list",
+        "prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
+      }, {
+        "title": "Code a snake game",
+        "prompt": "Code a basic snake game in python, give explanations for each step."
+      }, {
+        "title": "Assist in a task",
+        "prompt": "How do I make a delicious lemon cheesecake?"
+      }
+    ],
+    "parameters": {
+      "temperature": 0.9,
+      "top_p": 0.95,
+      "repetition_penalty": 1.2,
+      "top_k": 50,
+      "truncate": 1000,
+      "max_new_tokens": 1024,
+      "stop":["</s>"]
+    }
+  },
+  {
+    "name":"bigcode/starcoder",
+    "displayName":"BigCode/StarCoder",
+    "datasetName":"bigcode/the-stack-dedup",
+    "websiteUrl":"https://www.bigcode-project.org/",
+    "prepromptUrl": "https://huggingface.co/datasets/coyotte508/bigcodeprompt/raw/main/prompt.txt",
+    "promptExamples": [
+      {
+        "title": "Write a code snippet",
+        "prompt": "How to install pytorch with cuda?"
+      }, {
+        "title": "Explain a technical concept",
+        "prompt": "What is a Dockerfile?"
+      }, {
+        "title": "Solve a technical task",
+        "prompt": "Write a functions that loads a file and filters line starting with \"Star\"?"
+      }
+    ],
+    "userMessageToken": "\n\nHuman: ",
+    "assistantMessageToken": "\n\nAssistant: ",
+    "parameters": {
+      "temperature": 0.2,
+      "top_p": 0.9,
+      "repetition_penalty": 1.2,
+      "truncate": 8000,
+      "max_new_tokens": 2000
+    }
+  }
+]`
 
-# Increase depending on the model
-PUBLIC_MAX_INPUT_TOKENS=1000
 PUBLIC_ORIGIN=#https://hf.co
-PUBLIC_DISABLE_INTRO_TILES=false
-PUBLIC_USER_MESSAGE_TOKEN=<|prompter|>
-PUBLIC_ASSISTANT_MESSAGE_TOKEN=<|assistant|>
-PUBLIC_SEP_TOKEN=</s>
-PUBLIC_PREPROMPT="Below are a series of dialogues between various people and an AI assistant. The AI tries to be helpful, polite, honest, sophisticated, emotionally aware, and humble-but-knowledgeable. The assistant is happy to help with almost anything, and will do its best to understand exactly what is needed. It also tries to avoid giving false or misleading information, and it caveats when it isn't entirely sure about the right answer. That said, the assistant is practical and really does its best, and doesn't let caution get too much in the way of being useful."
 PUBLIC_GOOGLE_ANALYTICS_ID=#G-XXXXXXXX / Leave empty to disable
 PUBLIC_DEPRECATED_GOOGLE_ANALYTICS_ID=#UA-XXXXXXXX-X / Leave empty to disable

src/lib/buildPrompt.ts

@@ -1,10 +1,4 @@
-import {
-	PUBLIC_ASSISTANT_MESSAGE_TOKEN,
-	PUBLIC_MAX_INPUT_TOKENS,
-	PUBLIC_PREPROMPT,
-	PUBLIC_SEP_TOKEN,
-	PUBLIC_USER_MESSAGE_TOKEN,
-} from "$env/static/public";
+import type { BackendModel } from "./server/models";
 import type { Message } from "./types/Message";
 
 /**
@@ -12,22 +6,25 @@ import type { Message } from "./types/Message";
  *
  * <|assistant|>hi<|endoftext|><|prompter|>hello<|endoftext|><|assistant|>
  */
-export function buildPrompt(messages: Pick<Message, "from" | "content">[]): string {
+export function buildPrompt(
+	messages: Pick<Message, "from" | "content">[],
+	model: BackendModel
+): string {
 	const prompt =
 		messages
 			.map(
 				(m) =>
 					(m.from === "user"
-						? PUBLIC_USER_MESSAGE_TOKEN + m.content
-						: PUBLIC_ASSISTANT_MESSAGE_TOKEN + m.content) +
-					(m.content.endsWith(PUBLIC_SEP_TOKEN) ? "" : PUBLIC_SEP_TOKEN)
+						? model.userMessageToken + m.content
+						: model.assistantMessageToken + m.content) +
+					(model.parameters.stop
+						? m.content.endsWith(model.parameters.stop[0])
+							? ""
+							: model.parameters.stop[0]
+						: "")
 			)
-			.join("") + PUBLIC_ASSISTANT_MESSAGE_TOKEN;
+			.join("") + model.assistantMessageToken;
 
 	// Not super precise, but it's truncated in the model's backend anyway
-	return (
-		PUBLIC_PREPROMPT +
-		"\n-----\n" +
-		prompt.split(" ").slice(-parseInt(PUBLIC_MAX_INPUT_TOKENS)).join(" ")
-	);
+	return model.preprompt + prompt.split(" ").slice(-model.parameters.truncate).join(" ");
 }

src/lib/components/AnnouncementBanner.svelte

@@ -0,0 +1,15 @@
+<script lang="ts">
+	export let title = "";
+	export let classNames = "";
+</script>
+
+<div class="flex items-center rounded-xl bg-gray-100 p-1 text-sm dark:bg-gray-800 {classNames}">
+	<span
+		class="mr-2 inline-flex items-center rounded-lg bg-gradient-to-br from-yellow-300 px-2 py-1 text-xxs font-medium uppercase leading-3 text-yellow-700 dark:from-[#373010] dark:text-yellow-400"
+		>New</span
+	>
+	{title}
+	<div class="ml-auto shrink-0">
+		<slot />
+	</div>
+</div>

src/lib/components/EthicsModal.svelte

@@ -4,9 +4,9 @@
 	import { PUBLIC_VERSION } from "$env/static/public";
 	import Logo from "$lib/components/icons/Logo.svelte";
 	import Modal from "$lib/components/Modal.svelte";
-	import type { Settings } from "$lib/types/Settings";
+	import type { LayoutData } from "../../routes/$types";
 
-	export let settings: Omit<Settings, "sessionId" | "createdAt" | "updatedAt">;
+	export let settings: LayoutData["settings"];
 </script>
 
 <Modal>

src/lib/components/Modal.svelte

@@ -1,13 +1,15 @@
 <script lang="ts">
 	import { cubicOut } from "svelte/easing";
 	import { fade } from "svelte/transition";
+
+	export let width = "max-w-sm";
 </script>
 
 <div
 	transition:fade={{ easing: cubicOut, duration: 300 }}
 	class="fixed inset-0 z-40 flex items-center justify-center bg-black/80 p-8 backdrop-blur-sm dark:bg-black/50"
 >
-	<div class="-mt-10 max-w-sm overflow-hidden rounded-2xl bg-white shadow-2xl md:-mt-20">
+	<div class="-mt-10 overflow-hidden rounded-2xl bg-white shadow-2xl md:-mt-20 {width}">
 		<slot />
 	</div>
 </div>

src/lib/components/ModelCardMetadata.svelte

@@ -0,0 +1,48 @@
+<script lang="ts">
+	import CarbonEarth from "~icons/carbon/earth";
+	import CarbonArrowUpRight from "~icons/carbon/arrow-up-right";
+	import type { Model } from "$lib/types/Model";
+
+	export let model: Pick<Model, "name" | "datasetName" | "websiteUrl">;
+
+	export let variant: "light" | "dark" = "light";
+</script>
+
+<div
+	class="flex items-center gap-5 rounded-xl bg-gray-100 px-3 py-2 text-sm 
+	{variant === 'dark'
+		? 'text-gray-600 dark:bg-gray-800 dark:text-gray-300'
+		: 'text-gray-800 dark:bg-gray-100 dark:text-gray-600'}"
+>
+	<a
+		href="https://huggingface.co/datasets/{model.name}"
+		target="_blank"
+		rel="noreferrer"
+		class="flex items-center hover:underline"
+		><CarbonArrowUpRight class="mr-1.5 shrink-0 text-xs text-gray-400" />
+		Model
+		<div class="max-sm:hidden">&nbsp;page</div></a
+	>
+	{#if model.datasetName}
+		<a
+			href="https://huggingface.co/datasets/{model.datasetName}"
+			target="_blank"
+			rel="noreferrer"
+			class="flex items-center hover:underline"
+			><CarbonArrowUpRight class="mr-1.5 shrink-0 text-xs text-gray-400" />
+			Dataset
+			<div class="max-sm:hidden">&nbsp;page</div></a
+		>
+	{/if}
+	{#if model.websiteUrl}
+		<a
+			href={model.websiteUrl}
+			target="_blank"
+			class="ml-auto flex items-center hover:underline"
+			rel="noreferrer"
+		>
+			<CarbonEarth class="mr-1.5 shrink-0 text-xs text-gray-400" />
+			Website
+		</a>
+	{/if}
+</div>

src/lib/components/ModelsModal.svelte

@@ -0,0 +1,78 @@
+<script lang="ts">
+	import { createEventDispatcher } from "svelte";
+
+	import Modal from "$lib/components/Modal.svelte";
+	import CarbonClose from "~icons/carbon/close";
+	import CarbonCheckmark from "~icons/carbon/checkmark-filled";
+	import ModelCardMetadata from "./ModelCardMetadata.svelte";
+	import type { Model } from "$lib/types/Model";
+	import type { LayoutData } from "../../routes/$types";
+	import { enhance } from "$app/forms";
+	import { base } from "$app/paths";
+
+	export let settings: LayoutData["settings"];
+	export let models: Array<Model>;
+
+	let selectedModelName = settings.activeModel;
+
+	const dispatch = createEventDispatcher<{ close: void }>();
+</script>
+
+<Modal width="max-w-lg">
+	<form
+		action="{base}/settings"
+		method="post"
+		use:enhance={() => {
+			dispatch("close");
+		}}
+		class="flex w-full flex-col gap-5 p-6"
+	>
+		{#each Object.entries(settings).filter(([k]) => k !== "activeModel") as [key, val]}
+			<input type="hidden" name={key} value={val} />
+		{/each}
+		<div class="flex items-start justify-between text-xl font-semibold text-gray-800">
+			<h2>Models</h2>
+			<button type="button" class="group" on:click={() => dispatch("close")}>
+				<CarbonClose class="text-gray-900 group-hover:text-gray-500" />
+			</button>
+		</div>
+
+		<div class="space-y-4">
+			{#each models as model}
+				<div
+					class="rounded-xl border border-gray-100 {model.name === selectedModelName
+						? 'bg-gradient-to-r from-yellow-200/40 via-yellow-500/10'
+						: ''}"
+				>
+					<label class="group flex cursor-pointer p-3" on:change aria-label={model.displayName}>
+						<input
+							type="radio"
+							class="sr-only"
+							name="activeModel"
+							value={model.name}
+							bind:group={selectedModelName}
+						/>
+						<span>
+							<span class="text-md block font-semibold leading-tight text-gray-800"
+								>{model.displayName}</span
+							>
+							<span class="text-xs text-[#9FA8B5]">A good alternative to ChatGPT</span>
+						</span>
+						<CarbonCheckmark
+							class="-mr-1 -mt-1 ml-auto shrink-0 text-xl {model.name === selectedModelName
+								? 'text-yellow-400'
+								: 'text-transparent group-hover:text-gray-200'}"
+						/>
+					</label>
+					<ModelCardMetadata {model} />
+				</div>
+			{/each}
+		</div>
+		<button
+			type="submit"
+			class="mt-2 rounded-full bg-black px-5 py-2 text-lg font-semibold text-gray-100 ring-gray-400 ring-offset-1 transition-colors hover:ring"
+		>
+			Apply
+		</button>
+	</form>
+</Modal>

src/lib/components/SettingsModal.svelte

@@ -30,6 +30,9 @@
 		</div>
 
 		<label class="flex cursor-pointer select-none items-center gap-2 text-gray-500">
+			{#each Object.entries(settings).filter(([k]) => k !== "shareConversationsWithModelAuthors") as [key, val]}
+				<input type="hidden" name={key} value={val} />
+			{/each}
 			<Switch
 				name="shareConversationsWithModelAuthors"
 				bind:checked={settings.shareConversationsWithModelAuthors}

src/lib/components/chat/ChatIntroduction.svelte

@@ -1,11 +1,23 @@
 <script lang="ts">
-	import { PUBLIC_DISABLE_INTRO_TILES, PUBLIC_VERSION } from "$env/static/public";
+	import { PUBLIC_VERSION } from "$env/static/public";
 	import Logo from "$lib/components/icons/Logo.svelte";
-	import CarbonArrowUpRight from "~icons/carbon/arrow-up-right";
-	import CarbonEarth from "~icons/carbon/earth";
 	import { createEventDispatcher } from "svelte";
+	import IconChevron from "$lib/components/icons/IconChevron.svelte";
+	import AnnouncementBanner from "../AnnouncementBanner.svelte";
+	import ModelsModal from "../ModelsModal.svelte";
+	import type { Model } from "$lib/types/Model";
+	import ModelCardMetadata from "../ModelCardMetadata.svelte";
+	import type { LayoutData } from "../../../routes/$types";
+	import { findCurrentModel } from "$lib/utils/models";
+
+	export let currentModel: Model;
+	export let settings: LayoutData["settings"];
+	export let models: Model[];
+
+	let isModelsModalOpen = false;
+
+	$: currentModelMetadata = findCurrentModel(models, settings.activeModel);
 
-	export let currentModel: { name: string; displayName: string };
 	const dispatch = createEventDispatcher<{ message: string }>();
 </script>
 
@@ -27,80 +39,46 @@
 		</div>
 	</div>
 	<div class="lg:col-span-2 lg:pl-24">
-		<div class="overflow-hidden rounded-xl border dark:border-gray-800">
-			<div class="p-3">
-				<div class="text-sm text-gray-600 dark:text-gray-400">Current Model</div>
-				<div class="font-semibold">{currentModel.displayName}</div>
-			</div>
-			<div
-				class="flex items-center gap-5 rounded-xl bg-gray-100 px-3 py-2 text-sm text-gray-600 dark:bg-gray-800 dark:text-gray-300"
+		<AnnouncementBanner classNames="mb-4" title="BigCode/StarCoder is now available">
+			<button
+				type="button"
+				on:click={() => (isModelsModalOpen = true)}
+				class="mr-2 flex items-center underline hover:no-underline"
+				><IconChevron classNames="mr-1" /> Switch model</button
 			>
-				<a
-					href="https://huggingface.co/{currentModel.name}"
-					target="_blank"
-					rel="noreferrer"
-					class="flex items-center hover:underline"
-				>
-					<CarbonArrowUpRight class="mr-1.5 text-xs text-gray-400" />
-					Model
-					<div class="max-sm:hidden">&nbsp;page</div>
-				</a>
-				<a
-					href="https://huggingface.co/datasets/OpenAssistant/oasst1"
-					target="_blank"
-					rel="noreferrer"
-					class="flex items-center hover:underline"
-				>
-					<CarbonArrowUpRight class="mr-1.5 text-xs text-gray-400" />
-					Dataset
-					<div class="max-sm:hidden">&nbsp;page</div>
-				</a>
-				<a
-					href="https://open-assistant.io/"
-					target="_blank"
-					class="ml-auto flex items-center hover:underline"
-					rel="noreferrer"
+		</AnnouncementBanner>
+		{#if isModelsModalOpen}
+			<ModelsModal {settings} {models} on:close={() => (isModelsModalOpen = false)} />
+		{/if}
+		<div class="overflow-hidden rounded-xl border dark:border-gray-800">
+			<div class="flex p-3">
+				<div>
+					<div class="text-sm text-gray-600 dark:text-gray-400">Current Model</div>
+					<div class="font-semibold">{currentModel.displayName}</div>
+				</div>
+				<button
+					type="button"
+					on:click={() => (isModelsModalOpen = true)}
+					class="btn ml-auto flex h-7 w-7 self-start rounded-full bg-gray-100 p-1 text-xs hover:bg-gray-100 dark:border-gray-600 dark:bg-gray-800 dark:hover:bg-gray-600"
+					><IconChevron /></button
 				>
-					<CarbonEarth class="mr-1.5 text-xs text-gray-400" />
-					Open Assistant Website
-				</a>
 			</div>
+			<ModelCardMetadata variant="dark" model={currentModel} />
 		</div>
 	</div>
-	{#if PUBLIC_DISABLE_INTRO_TILES !== "true"}
+	{#if currentModelMetadata.promptExamples}
 		<div class="lg:col-span-3 lg:mt-12">
 			<p class="mb-3 text-gray-600 dark:text-gray-300">Examples</p>
 			<div class="grid gap-3 lg:grid-cols-3 lg:gap-5">
-				<button
-					type="button"
-					class="rounded-xl border bg-gray-50 p-2.5 text-gray-600 hover:bg-gray-100 dark:border-gray-800 dark:bg-gray-800 dark:text-gray-300 dark:hover:bg-gray-700 sm:p-4"
-					on:click={() =>
-						dispatch(
-							"message",
-							"As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
-						)}
-				>
-					"Write an email from bullet list"
-				</button>
-				<button
-					type="button"
-					class="rounded-xl border bg-gray-50 p-2.5 text-gray-600 hover:bg-gray-100 dark:border-gray-800 dark:bg-gray-800 dark:text-gray-300 dark:hover:bg-gray-700 sm:p-4"
-					on:click={() =>
-						dispatch(
-							"message",
-							"Code a basic snake game in python, give explanations for each step."
-						)}
-				>
-					"Code a snake game"
-				</button>
-				<button
-					type="button"
-					class="rounded-xl border bg-gray-50 p-2.5 text-gray-600 hover:bg-gray-100 dark:border-gray-800 dark:bg-gray-800 dark:text-gray-300 dark:hover:bg-gray-700 sm:p-4"
-					on:click={() => dispatch("message", "How do I make a delicious lemon cheesecake?")}
-				>
-					"Assist in a task"
-				</button>
+				{#each currentModelMetadata.promptExamples as example}
+					<button
+						type="button"
+						class="rounded-xl border bg-gray-50 p-2.5 text-gray-600 hover:bg-gray-100 dark:border-gray-800 dark:bg-gray-800 dark:text-gray-300 dark:hover:bg-gray-700 sm:p-4"
+						on:click={() => dispatch("message", example.prompt)}
+					>
+						{example.title}
+					</button>
+				{/each}
 			</div>
-		</div>
-	{/if}
+		</div>{/if}
 </div>

src/lib/components/chat/ChatMessage.svelte

@@ -7,7 +7,7 @@
 	import CodeBlock from "../CodeBlock.svelte";
 	import IconLoading from "../icons/IconLoading.svelte";
 	import CarbonRotate360 from "~icons/carbon/rotate-360";
-	import { PUBLIC_SEP_TOKEN } from "$env/static/public";
+	import { PUBLIC_SEP_TOKEN } from "$lib/constants/publicSepToken";
 
 	function sanitizeMd(md: string) {
 		return md

src/lib/components/chat/ChatMessages.svelte

@@ -7,13 +7,17 @@
 	import ChatIntroduction from "./ChatIntroduction.svelte";
 	import ChatMessage from "./ChatMessage.svelte";
 	import { randomUUID } from "$lib/utils/randomUuid";
+	import type { Model } from "$lib/types/Model";
+	import type { LayoutData } from "../../../routes/$types";
 
 	const dispatch = createEventDispatcher<{ retry: { id: Message["id"]; content: string } }>();
 
 	export let messages: Message[];
 	export let loading: boolean;
 	export let pending: boolean;
-	export let currentModel: { name: string; displayName: string };
+	export let currentModel: Model;
+	export let settings: LayoutData["settings"];
+	export let models: Model[] | undefined;
 
 	let chatContainer: HTMLElement;
 
@@ -41,7 +45,9 @@
 				on:retry={() => dispatch("retry", { id: message.id, content: message.content })}
 			/>
 		{:else}
-			<ChatIntroduction on:message {currentModel} />
+			{#if models}
+				<ChatIntroduction {settings} {models} {currentModel} on:message />
+			{/if}
 		{/each}
 		{#if pending}
 			<ChatMessage message={{ from: "assistant", content: "", id: randomUUID() }} />

src/lib/components/chat/ChatWindow.svelte

@@ -8,12 +8,16 @@
 	import ChatMessages from "./ChatMessages.svelte";
 	import ChatInput from "./ChatInput.svelte";
 	import StopGeneratingBtn from "../StopGeneratingBtn.svelte";
+	import type { Model } from "$lib/types/Model";
+	import type { LayoutData } from "../../../routes/$types";
 
 	export let messages: Message[] = [];
 	export let disabled = false;
 	export let loading = false;
 	export let pending = false;
-	export let currentModel: { name: string; displayName: string };
+	export let currentModel: Model;
+	export let models: Model[] | undefined = undefined;
+	export let settings: LayoutData["settings"];
 
 	let message: string;
 
@@ -35,7 +39,9 @@
 	<ChatMessages
 		{loading}
 		{pending}
+		{settings}
 		{currentModel}
+		{models}
 		{messages}
 		on:message
 		on:retry={(ev) => {

src/lib/components/icons/IconChevron.svelte

@@ -3,9 +3,9 @@
 </script>
 
 <svg
-	width="15"
-	height="8"
-	viewBox="0 0 15 8"
+	width="1em"
+	height="1em"
+	viewBox="0 0 15 6"
 	class={classNames}
 	fill="none"
 	xmlns="http://www.w3.org/2000/svg"

src/lib/constants/publicSepToken.ts

@@ -0,0 +1 @@
+export const PUBLIC_SEP_TOKEN = "</s>";

src/lib/server/modelEndpoint.ts

@@ -10,9 +10,7 @@ export function modelEndpoint(model: string): {
 	authorization: string;
 	weight: number;
 } {
-	const modelDefinition = models.find(
-		(m) => m === model || (typeof m === "object" && m.name === model)
-	);
+	const modelDefinition = models.find((m) => m.name === model);
 	if (!modelDefinition) {
 		throw new Error(`Invalid model: ${model}`);
 	}

src/lib/server/models.ts

@@ -1,30 +1,55 @@
 import { HF_ACCESS_TOKEN, MODELS } from "$env/static/private";
 import { z } from "zod";
 
-export const models = z
+const modelsRaw = z
 	.array(
-		z.union([
-			z.string().min(1),
-			z.object({
-				name: z.string().min(1),
-				displayName: z.string().min(1).optional(),
-				endpoints: z
-					.array(
-						z.object({
-							url: z.string().url(),
-							authorization: z.string().min(1).default(`Bearer ${HF_ACCESS_TOKEN}`),
-							weight: z.number().int().positive().default(1),
-						})
-					)
-					.optional(),
-			}),
-		])
+		z.object({
+			name: z.string().min(1),
+			displayName: z.string().min(1).optional(),
+			websiteUrl: z.string().url().optional(),
+			datasetName: z.string().min(1).optional(),
+			userMessageToken: z.string().min(1),
+			assistantMessageToken: z.string().min(1),
+			preprompt: z.string().default(""),
+			prepromptUrl: z.string().url().optional(),
+			promptExamples: z
+				.array(
+					z.object({
+						title: z.string().min(1),
+						prompt: z.string().min(1),
+					})
+				)
+				.optional(),
+			endpoints: z
+				.array(
+					z.object({
+						url: z.string().url(),
+						authorization: z.string().min(1).default(`Bearer ${HF_ACCESS_TOKEN}`),
+						weight: z.number().int().positive().default(1),
+					})
+				)
+				.optional(),
+			parameters: z
+				.object({
+					temperature: z.number().min(0).max(1),
+					truncate: z.number().int().positive(),
+					max_new_tokens: z.number().int().positive(),
+					stop: z.array(z.string()).min(1).optional(),
+				})
+				.passthrough(),
+		})
 	)
 	.parse(JSON.parse(MODELS));
 
-export const modelNames: Array<{ name: string; displayName: string }> = models.map((m) =>
-	typeof m === "string"
-		? { name: m, displayName: m }
-		: { name: m.name, displayName: m.displayName ?? m.name }
+export const models = await Promise.all(
+	modelsRaw.map(async (m) => ({
+		...m,
+		displayName: m.displayName || m.name,
+		preprompt: m.prepromptUrl ? await fetch(m.prepromptUrl).then((r) => r.text()) : m.preprompt,
+		promptExamples: m.promptExamples || [],
+	}))
 );
-export const defaultModel = modelNames[0];
+
+export type BackendModel = (typeof models)[0];
+
+export const defaultModel = models[0];

src/lib/types/Model.ts

@@ -0,0 +1,7 @@
+export interface Model {
+	name: string;
+	displayName?: string;
+	websiteUrl?: string;
+	datasetName?: string;
+	promptExamples?: Array<{ title: string; prompt: string }>;
+}

src/lib/types/Settings.ts

@@ -10,4 +10,5 @@ export interface Settings extends Timestamps {
 	 * */
 	shareConversationsWithModelAuthors: boolean;
 	ethicsModalAcceptedAt: Date | null;
+	activeModel: string;
 }

src/lib/utils/models.ts

@@ -0,0 +1,4 @@
+import type { Model } from "$lib/types/Model";
+
+export const findCurrentModel = (models: Model[], name?: string) =>
+	models.find((m) => m.name === name) ?? models[0];

src/routes/+layout.server.ts

@@ -2,9 +2,9 @@ import type { LayoutServerLoad } from "./$types";
 import { collections } from "$lib/server/database";
 import type { Conversation } from "$lib/types/Conversation";
 import { UrlDependency } from "$lib/types/UrlDependency";
-import { modelNames } from "$lib/server/models";
+import { defaultModel, models } from "$lib/server/models";
 
-export const load: LayoutServerLoad = async ({ locals, depends }) => {
+export const load: LayoutServerLoad = async ({ locals, depends, url }) => {
 	const { conversations } = collections;
 
 	depends(UrlDependency.ConversationList);
@@ -17,18 +17,32 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => {
 				sessionId: locals.sessionId,
 			})
 			.sort({ updatedAt: -1 })
-			.project<Pick<Conversation, "title" | "_id" | "updatedAt" | "createdAt">>({
+			.project<Pick<Conversation, "title" | "model" | "_id" | "updatedAt" | "createdAt">>({
 				title: 1,
+				model: 1,
 				_id: 1,
 				updatedAt: 1,
 				createdAt: 1,
 			})
-			.map((conv) => ({ id: conv._id.toString(), title: conv.title }))
+			.map((conv) => ({
+				id: conv._id.toString(),
+				title: conv.title,
+				model: conv.model ?? defaultModel,
+			}))
 			.toArray(),
 		settings: {
 			shareConversationsWithModelAuthors: settings?.shareConversationsWithModelAuthors ?? true,
 			ethicsModalAcceptedAt: settings?.ethicsModalAcceptedAt ?? null,
+			activeModel:
+				url.searchParams.get("model") ?? settings?.activeModel ?? defaultModel.name,
 		},
-		models: modelNames,
+		models: models.map((model) => ({
+			name: model.name,
+			websiteUrl: model.websiteUrl,
+			datasetName: model.datasetName,
+			displayName: model.displayName,
+			promptExamples: model.promptExamples,
+			parameters: model.parameters,
+		})),
 	};
 };

src/routes/+page.svelte

@@ -4,6 +4,7 @@
 	import ChatWindow from "$lib/components/chat/ChatWindow.svelte";
 	import { ERROR_MESSAGES, error } from "$lib/stores/errors";
 	import { pendingMessage } from "$lib/stores/pendingMessage";
+	import { findCurrentModel } from "$lib/utils/models.js";
 
 	export let data;
 	let loading = false;
@@ -16,7 +17,7 @@
 				headers: {
 					"Content-Type": "application/json",
 				},
-				body: JSON.stringify({ model: data.models[0].name }),
+				body: JSON.stringify({ model: data.settings.activeModel }),
 			});
 
 			if (!res.ok) {
@@ -44,5 +45,7 @@
 <ChatWindow
 	on:message={(ev) => createConversation(ev.detail)}
 	{loading}
-	currentModel={data.models[0]}
+	currentModel={findCurrentModel(data.models, data.settings.activeModel)}
+	models={data.models}
+	settings={data.settings}
 />

src/routes/conversation/+server.ts

@@ -5,7 +5,7 @@ import { error, redirect } from "@sveltejs/kit";
 import { base } from "$app/paths";
 import { z } from "zod";
 import type { Message } from "$lib/types/Message";
-import { defaultModel, modelNames } from "$lib/server/models";
+import { defaultModel, models } from "$lib/server/models";
 
 export const POST: RequestHandler = async (input) => {
 	const body = await input.request.text();
@@ -17,7 +17,7 @@ export const POST: RequestHandler = async (input) => {
 		.object({
 			fromShare: z.string().optional(),
 			model: z
-				.enum([modelNames[0].name, ...modelNames.slice(1).map((m) => m.name)])
+				.enum([models[0].name, ...models.slice(1).map((m) => m.name)])
 				.default(defaultModel.name),
 		})
 		.parse(JSON.parse(body));

src/routes/conversation/[id]/+page.server.ts

@@ -29,5 +29,6 @@ export const load: PageServerLoad = async (event) => {
 	return {
 		messages: conversation.messages,
 		title: conversation.title,
+		model: conversation.model,
 	};
 };

src/routes/conversation/[id]/+page.svelte

@@ -4,14 +4,14 @@
 	import { pendingMessageIdToRetry } from "$lib/stores/pendingMessageIdToRetry";
 	import { onMount } from "svelte";
 	import { page } from "$app/stores";
-	import { textGenerationStream } from "@huggingface/inference";
+	import { textGenerationStream, type Options } from "@huggingface/inference";
 	import { invalidate } from "$app/navigation";
 	import { base } from "$app/paths";
-	import { PUBLIC_MAX_INPUT_TOKENS, PUBLIC_SEP_TOKEN } from "$env/static/public";
 	import { shareConversation } from "$lib/shareConversation";
 	import { UrlDependency } from "$lib/types/UrlDependency";
 	import { ERROR_MESSAGES, error } from "$lib/stores/errors";
 	import { randomUUID } from "$lib/utils/randomUuid";
+	import { findCurrentModel } from "$lib/utils/models.js";
 
 	export let data;
 
@@ -36,16 +36,7 @@
 				model: $page.url.href,
 				inputs,
 				parameters: {
-					// Taken from https://huggingface.co/spaces/huggingface/open-assistant-private-testing/blob/main/app.py#L54
-					temperature: 0.9,
-					top_p: 0.95,
-					repetition_penalty: 1.2,
-					top_k: 50,
-					truncate: parseInt(PUBLIC_MAX_INPUT_TOKENS),
-					// @ts-expect-error this param is not available in @huggingface/inference
-					watermark: false,
-					max_new_tokens: 1024,
-					stop: [PUBLIC_SEP_TOKEN],
+					...data.models.find((m) => m.name === data.model)?.parameters,
 					return_full_text: false,
 				},
 			},
@@ -53,7 +44,7 @@
 				id: messageId,
 				is_retry: isRetry,
 				use_cache: false,
-			}
+			} as Options
 		);
 
 		for await (const output of response) {
@@ -181,5 +172,6 @@
 	on:retry={(message) => writeMessage(message.detail.content, message.detail.id)}
 	on:share={() => shareConversation($page.params.id, data.title)}
 	on:stop={() => (isAborted = true)}
-	currentModel={data.models[0]}
+	currentModel={findCurrentModel(data.models, data.model)}
+	settings={data.settings}
 />

src/routes/conversation/[id]/+server.ts

@@ -1,9 +1,9 @@
-import { PUBLIC_SEP_TOKEN } from "$env/static/public";
 import { buildPrompt } from "$lib/buildPrompt.js";
+import { PUBLIC_SEP_TOKEN } from "$lib/constants/publicSepToken.js";
 import { abortedGenerations } from "$lib/server/abortedGenerations.js";
 import { collections } from "$lib/server/database.js";
 import { modelEndpoint } from "$lib/server/modelEndpoint.js";
-import { defaultModel } from "$lib/server/models.js";
+import { defaultModel, models } from "$lib/server/models.js";
 import type { Message } from "$lib/types/Message.js";
 import { concatUint8Arrays } from "$lib/utils/concatUint8Arrays.js";
 import { streamToAsyncIterable } from "$lib/utils/streamToAsyncIterable";
@@ -67,7 +67,14 @@ export async function POST({ request, fetch, locals, params }) {
 			message.id = crypto.randomUUID();
 		}
 	}
-	const prompt = buildPrompt(messages);
+
+	const modelInfo = models.find((m) => m.name === model);
+
+	if (!modelInfo) {
+		throw error(400, "Model not availalbe anymore");
+	}
+
+	const prompt = buildPrompt(messages, modelInfo);
 
 	const randomEndpoint = modelEndpoint(model);
 
@@ -189,7 +196,7 @@ async function parseGeneratedText(
 	}
 
 	if (lastIndex === -1) {
-		console.error("Could not parse in last message");
+		console.error("Could not parse last message", message);
 	}
 
 	let lastMessage = message.slice(lastIndex).trim().slice("data:".length);

src/routes/conversation/[id]/summarize/+server.ts

@@ -1,5 +1,5 @@
-import { PUBLIC_MAX_INPUT_TOKENS, PUBLIC_SEP_TOKEN } from "$env/static/public";
 import { buildPrompt } from "$lib/buildPrompt";
+import { PUBLIC_SEP_TOKEN } from "$lib/constants/publicSepToken.js";
 import { collections } from "$lib/server/database.js";
 import { modelEndpoint } from "$lib/server/modelEndpoint.js";
 import { defaultModel } from "$lib/server/models.js";
@@ -27,17 +27,10 @@ export async function POST({ params, locals, fetch }) {
 		`Please summarize the following message as a single sentence of less than 5 words:\n` +
 		firstMessage?.content;
 
-	const prompt = buildPrompt([{ from: "user", content: userPrompt }]);
+	const prompt = buildPrompt([{ from: "user", content: userPrompt }], defaultModel);
 
 	const parameters = {
-		temperature: 0.9,
-		top_p: 0.95,
-		repetition_penalty: 1.2,
-		top_k: 50,
-		watermark: false,
-		max_new_tokens: 1024,
-		truncate: parseInt(PUBLIC_MAX_INPUT_TOKENS),
-		stop: [PUBLIC_SEP_TOKEN],
+		...defaultModel.parameters,
 		return_full_text: false,
 	};
 

src/routes/r/[id]/+page.server.ts

@@ -14,5 +14,6 @@ export const load: PageServerLoad = async ({ params }) => {
 	return {
 		messages: conversation.messages,
 		title: conversation.title,
+		model: conversation.model,
 	};
 };

src/routes/r/[id]/+page.svelte

@@ -6,6 +6,7 @@
 	import { ERROR_MESSAGES, error } from "$lib/stores/errors";
 	import { pendingMessage } from "$lib/stores/pendingMessage";
 	import { pendingMessageIdToRetry } from "$lib/stores/pendingMessageIdToRetry";
+	import { findCurrentModel } from "$lib/utils/models";
 	import { share } from "$lib/utils/share";
 	import type { PageData } from "./$types";
 
@@ -71,6 +72,7 @@
 			})
 			.finally(() => (loading = false))}
 	messages={data.messages}
-	currentModel={data.models[0]}
+	currentModel={findCurrentModel(data.models, data.model)}
+	settings={data.settings}
 	{loading}
 />

src/routes/settings/+page.server.ts

@@ -1,7 +1,8 @@
 import { base } from "$app/paths";
-import { collections } from "$lib/server/database.js";
+import { collections } from "$lib/server/database";
 import { redirect } from "@sveltejs/kit";
 import { z } from "zod";
+import { defaultModel, models } from "$lib/server/models";
 
 export const actions = {
 	default: async function ({ request, locals }) {
@@ -11,10 +12,12 @@ export const actions = {
 			.object({
 				shareConversationsWithModelAuthors: z.boolean({ coerce: true }).default(true),
 				ethicsModalAccepted: z.boolean({ coerce: true }).optional(),
+				activeModel: z.enum([models[0].name, ...models.slice(1).map((m) => m.name)]),
 			})
 			.parse({
 				shareConversationsWithModelAuthors: formData.get("shareConversationsWithModelAuthors"),
 				ethicsModalAccepted: formData.get("ethicsModalAccepted"),
+				activeModel: formData.get("activeModel") ?? defaultModel.name,
 			});
 
 		await collections.settings.updateOne(

tailwind.config.cjs

@@ -10,6 +10,7 @@ export default {
 			// 	sans: ['"Inter"', ...defaultTheme.fontFamily.sans]
 			// },
 			fontSize: {
+				xxs: "0.625rem",
 				smd: "0.94rem",
 			},
 		},