Disable models (#187)

* add ability for a model to be disabled

* fallback to default model if user has invalid/disabled model in settings

* set conversations to read-only if conv model is disabled

* add better disabled state on chat input

* fix inverted if statement

* remove bigcode from .env

* fix not updating settings

Co-authored-by: Eliott C. <[email protected]>

* refactor solution to use an OLD_MODELS env var instead

* OLD_MODELS needs to be in .env but can be empty

* fix typing issue + typo

* refactor models validation

* old models displayName is optional

* set read only prop on user message not assistant messages

* remove console.log

* ♻️ Refacto isReadOnly

* 🩹 Fluff: use 410 http code

---------

Co-authored-by: Eliott C. <[email protected]>

.env

@@ -41,38 +41,9 @@ MODELS=`[
       "truncate": 1000,
       "max_new_tokens": 1024
     }
-  },
-  {
-    "name":"bigcode/starcoderbase",
-    "displayName":"BigCode/StarCoderBase",
-    "datasetName":"bigcode/the-stack-dedup",
-    "description": "A good model for answering technical questions",
-    "websiteUrl":"https://huggingface.co/bigcode/",
-    "prepromptUrl": "https://huggingface.co/datasets/coyotte508/bigcodeprompt/raw/main/prompt.txt",
-    "promptExamples": [
-      {
-        "title": "Write a code snippet",
-        "prompt": "Write a function that loads a file and filters line starting with \"Star\"?"
-      }, {
-        "title": "Explain a technical concept",
-        "prompt": "What is a Dockerfile?"
-      }, {
-        "title": "Solve a technical task",
-        "prompt": "How to install pytorch with cuda?"
-      }
-    ],
-    "userMessageToken": "\n\nHuman: ",
-    "assistantMessageToken": "\n\nAssistant:",
-    "parameters": {
-      "temperature": 0.1,
-      "top_p": 0.9,
-      "repetition_penalty": 1.2,
-      "truncate": 8000,
-      "max_new_tokens": 2000,
-      "stop": ["Human:", "-----", "Assistant:"]
-    }
   }
 ]`
+OLD_MODELS=`[]`# any removed models, `{ name: string, displayName?: string, id?: string }`
 
 PUBLIC_ORIGIN=#https://hf.co
 PUBLIC_GOOGLE_ANALYTICS_ID=#G-XXXXXXXX / Leave empty to disable

src/lib/buildPrompt.ts

@@ -26,5 +26,11 @@ export function buildPrompt(
 			.join("") + model.assistantMessageToken;
 
 	// Not super precise, but it's truncated in the model's backend anyway
-	return model.preprompt + prompt.split(" ").slice(-model.parameters.truncate).join(" ");
+	return (
+		model.preprompt +
+		prompt
+			.split(" ")
+			.slice(-(model.parameters?.truncate ?? 0))
+			.join(" ")
+	);
 }

src/lib/components/chat/ChatIntroduction.svelte

@@ -3,7 +3,6 @@
 	import Logo from "$lib/components/icons/Logo.svelte";
 	import { createEventDispatcher } from "svelte";
 	import IconChevron from "$lib/components/icons/IconChevron.svelte";
-	import AnnouncementBanner from "../AnnouncementBanner.svelte";
 	import ModelsModal from "../ModelsModal.svelte";
 	import type { Model } from "$lib/types/Model";
 	import ModelCardMetadata from "../ModelCardMetadata.svelte";
@@ -39,14 +38,6 @@
 		</div>
 	</div>
 	<div class="lg:col-span-2 lg:pl-24">
-		<AnnouncementBanner classNames="mb-4" title="BigCode/StarCoderBase is now available">
-			<button
-				type="button"
-				on:click={() => (isModelsModalOpen = true)}
-				class="mr-2 flex items-center underline hover:no-underline"
-				><IconChevron classNames="mr-1" /> Switch model</button
-			>
-		</AnnouncementBanner>
 		{#if isModelsModalOpen}
 			<ModelsModal {settings} {models} on:close={() => (isModelsModalOpen = false)} />
 		{/if}
@@ -56,12 +47,14 @@
 					<div class="text-sm text-gray-600 dark:text-gray-400">Current Model</div>
 					<div class="font-semibold">{currentModel.displayName}</div>
 				</div>
-				<button
-					type="button"
-					on:click={() => (isModelsModalOpen = true)}
-					class="btn ml-auto flex h-7 w-7 self-start rounded-full bg-gray-100 p-1 text-xs hover:bg-gray-100 dark:border-gray-600 dark:bg-gray-800 dark:hover:bg-gray-600"
-					><IconChevron /></button
-				>
+				{#if models.length > 1}
+					<button
+						type="button"
+						on:click={() => (isModelsModalOpen = true)}
+						class="btn ml-auto flex h-7 w-7 self-start rounded-full bg-gray-100 p-1 text-xs hover:bg-gray-100 dark:border-gray-600 dark:bg-gray-800 dark:hover:bg-gray-600"
+						><IconChevron /></button
+					>
+				{/if}
 			</div>
 			<ModelCardMetadata variant="dark" model={currentModel} />
 		</div>

src/lib/components/chat/ChatMessage.svelte

@@ -23,7 +23,7 @@
 			.replaceAll("<", "&lt;")
 			.trim();
 
-		for (const stop of [...(model.parameters.stop ?? []), "<|endoftext|>"]) {
+		for (const stop of [...(model.parameters?.stop ?? []), "<|endoftext|>"]) {
 			if (ret.endsWith(stop)) {
 				ret = ret.slice(0, -stop.length).trim();
 			}
@@ -38,6 +38,7 @@
 	export let model: Model;
 	export let message: Message;
 	export let loading = false;
+	export let readOnly = false;
 
 	const dispatch = createEventDispatcher<{ retry: void }>();
 
@@ -131,14 +132,16 @@
 						<CarbonDownload />
 					</a>
 				{/if}
-				<button
-					class="cursor-pointer rounded-lg border border-gray-100 p-1 text-xs text-gray-400 group-hover:block hover:text-gray-500 dark:border-gray-800 dark:text-gray-400 dark:hover:text-gray-300 md:hidden lg:-right-2"
-					title="Retry"
-					type="button"
-					on:click={() => dispatch("retry")}
-				>
-					<CarbonRotate360 />
-				</button>
+				{#if !readOnly}
+					<button
+						class="cursor-pointer rounded-lg border border-gray-100 p-1 text-xs text-gray-400 group-hover:block hover:text-gray-500 dark:border-gray-800 dark:text-gray-400 dark:hover:text-gray-300 md:hidden lg:-right-2"
+						title="Retry"
+						type="button"
+						on:click={() => dispatch("retry")}
+					>
+						<CarbonRotate360 />
+					</button>
+				{/if}
 			</div>
 		{/if}
 	</div>

src/lib/components/chat/ChatMessages.svelte

@@ -17,7 +17,8 @@
 	export let pending: boolean;
 	export let currentModel: Model;
 	export let settings: LayoutData["settings"];
-	export let models: Model[] | undefined;
+	export let models: Model[];
+	export let readOnly: boolean;
 
 	let chatContainer: HTMLElement;
 
@@ -43,12 +44,11 @@
 				loading={loading && i === messages.length - 1}
 				{message}
 				model={currentModel}
+				{readOnly}
 				on:retry={() => dispatch("retry", { id: message.id, content: message.content })}
 			/>
 		{:else}
-			{#if models}
-				<ChatIntroduction {settings} {models} {currentModel} on:message />
-			{/if}
+			<ChatIntroduction {settings} {models} {currentModel} on:message />
 		{/each}
 		{#if pending}
 			<ChatMessage

src/lib/components/chat/ChatWindow.svelte

@@ -12,13 +12,14 @@
 	import type { LayoutData } from "../../../routes/$types";
 
 	export let messages: Message[] = [];
-	export let disabled = false;
 	export let loading = false;
 	export let pending = false;
 	export let currentModel: Model;
-	export let models: Model[] | undefined = undefined;
+	export let models: Model[];
 	export let settings: LayoutData["settings"];
 
+	$: isReadOnly = !models.some((model) => model.id === currentModel.id);
+
 	let message: string;
 
 	const dispatch = createEventDispatcher<{
@@ -43,6 +44,7 @@
 		{currentModel}
 		{models}
 		{messages}
+		readOnly={isReadOnly}
 		on:message
 		on:retry={(ev) => {
 			if (!loading) dispatch("retry", ev.detail);
@@ -58,7 +60,8 @@
 		/>
 		<form
 			on:submit|preventDefault={handleSubmit}
-			class="relative flex w-full max-w-4xl flex-1 items-center rounded-xl border bg-gray-100 focus-within:border-gray-300 dark:border-gray-600 dark:bg-gray-700 dark:focus-within:border-gray-500 "
+			class="relative flex w-full max-w-4xl flex-1 items-center rounded-xl border bg-gray-100 focus-within:border-gray-300 dark:border-gray-600 dark:bg-gray-700 dark:focus-within:border-gray-500 
+			{isReadOnly ? 'opacity-30' : ''}"
 		>
 			<div class="flex w-full flex-1 border-none bg-transparent">
 				<ChatInput
@@ -66,10 +69,11 @@
 					bind:value={message}
 					on:submit={handleSubmit}
 					maxRows={4}
+					disabled={isReadOnly}
 				/>
 				<button
 					class="btn mx-1 my-1 h-[2.4rem] self-end rounded-lg bg-transparent p-1 px-[0.7rem] text-gray-400 disabled:opacity-60 enabled:hover:text-gray-700 dark:disabled:opacity-40 enabled:dark:hover:text-gray-100"
-					disabled={!message || loading || disabled}
+					disabled={!message || loading || isReadOnly}
 					type="submit"
 				>
 					<CarbonSendAltFilled />

src/lib/server/models.ts

@@ -1,4 +1,4 @@
-import { HF_ACCESS_TOKEN, MODELS } from "$env/static/private";
+import { HF_ACCESS_TOKEN, MODELS, OLD_MODELS } from "$env/static/private";
 import { z } from "zod";
 
 const modelsRaw = z
@@ -41,7 +41,8 @@ const modelsRaw = z
 					max_new_tokens: z.number().int().positive(),
 					stop: z.array(z.string()).optional(),
 				})
-				.passthrough(),
+				.passthrough()
+				.optional(),
 		})
 	)
 	.parse(JSON.parse(MODELS));
@@ -55,6 +56,25 @@ export const models = await Promise.all(
 	}))
 );
 
+// Models that have been deprecated
+export const oldModels = OLD_MODELS
+	? z
+			.array(
+				z.object({
+					id: z.string().optional(),
+					name: z.string().min(1),
+					displayName: z.string().min(1).optional(),
+				})
+			)
+			.parse(JSON.parse(OLD_MODELS))
+			.map((m) => ({ ...m, id: m.id || m.name, displayName: m.displayName || m.name }))
+	: [];
+
 export type BackendModel = (typeof models)[0];
 
 export const defaultModel = models[0];
+
+export const validateModel = (_models: BackendModel[]) => {
+	// Zod enum function requires 2 parameters
+	return z.enum([_models[0].id, ..._models.slice(1).map((m) => m.id)]);
+};

src/lib/utils/models.ts

@@ -1,10 +1,4 @@
 import type { Model } from "$lib/types/Model";
-import { z } from "zod";
 
-export const findCurrentModel = (models: Model[], name?: string) =>
-	models.find((m) => m.id === name) ?? models[0];
-
-export const validateModel = (models: Model[]) => {
-	// Zod enum function requires 2 parameters
-	return z.enum([models[0].id, ...models.slice(1).map((m) => m.id)]);
-};
+export const findCurrentModel = (models: Model[], id?: string) =>
+	models.find((m) => m.id === id) ?? models[0];

src/routes/+layout.server.ts

@@ -3,8 +3,7 @@ import type { LayoutServerLoad } from "./$types";
 import { collections } from "$lib/server/database";
 import type { Conversation } from "$lib/types/Conversation";
 import { UrlDependency } from "$lib/types/UrlDependency";
-import { defaultModel, models } from "$lib/server/models";
-import { validateModel } from "$lib/utils/models";
+import { defaultModel, models, oldModels, validateModel } from "$lib/server/models";
 import { authCondition } from "$lib/server/auth";
 
 export const load: LayoutServerLoad = async ({ locals, depends, url }) => {
@@ -29,6 +28,15 @@ export const load: LayoutServerLoad = async ({ locals, depends, url }) => {
 
 	const settings = await collections.settings.findOne(authCondition(locals));
 
+	// If the active model in settings is not valid, set it to the default model. This can happen if model was disabled.
+	if (settings && !validateModel(models).safeParse(settings?.activeModel).success) {
+		settings.activeModel = defaultModel.id;
+		await collections.settings.updateOne(
+			{ sessionId: locals.sessionId },
+			{ $set: { activeModel: defaultModel.id } }
+		);
+	}
+
 	return {
 		conversations: await conversations
 			.find(authCondition(locals))
@@ -61,5 +69,6 @@ export const load: LayoutServerLoad = async ({ locals, depends, url }) => {
 			promptExamples: model.promptExamples,
 			parameters: model.parameters,
 		})),
+		oldModels,
 	};
 };

src/routes/+page.svelte

@@ -45,7 +45,7 @@
 <ChatWindow
 	on:message={(ev) => createConversation(ev.detail)}
 	{loading}
-	currentModel={findCurrentModel(data.models, data.settings.activeModel)}
+	currentModel={findCurrentModel([...data.models, ...data.oldModels], data.settings.activeModel)}
 	models={data.models}
 	settings={data.settings}
 />

src/routes/conversation/+server.ts

@@ -5,8 +5,7 @@ import { error, redirect } from "@sveltejs/kit";
 import { base } from "$app/paths";
 import { z } from "zod";
 import type { Message } from "$lib/types/Message";
-import { models } from "$lib/server/models";
-import { validateModel } from "$lib/utils/models";
+import { models, validateModel } from "$lib/server/models";
 import { authCondition } from "$lib/server/auth";
 
 export const POST: RequestHandler = async ({ locals, request }) => {

src/routes/conversation/[id]/+page.svelte

@@ -173,6 +173,7 @@
 	on:retry={(message) => writeMessage(message.detail.content, message.detail.id)}
 	on:share={() => shareConversation($page.params.id, data.title)}
 	on:stop={() => (isAborted = true)}
-	currentModel={findCurrentModel(data.models, data.model)}
+	models={data.models}
+	currentModel={findCurrentModel([...data.models, ...data.oldModels], data.model)}
 	settings={data.settings}
 />

src/routes/conversation/[id]/+server.ts

@@ -16,8 +16,8 @@ import { ObjectId } from "mongodb";
 import { z } from "zod";
 
 export async function POST({ request, fetch, locals, params }) {
-	// todo: add validation on params.id
-	const convId = new ObjectId(params.id);
+	const id = z.string().parse(params.id);
+	const convId = new ObjectId(id);
 	const date = new Date();
 
 	const conv = await collections.conversations.findOne({
@@ -32,7 +32,7 @@ export async function POST({ request, fetch, locals, params }) {
 	const model = models.find((m) => m.id === conv.model);
 
 	if (!model) {
-		throw error(400, "Model not availalbe anymore");
+		throw error(410, "Model not available anymore");
 	}
 
 	const json = await request.json();

src/routes/r/[id]/+page.svelte

@@ -72,6 +72,7 @@
 			})
 			.finally(() => (loading = false))}
 	messages={data.messages}
+	models={data.models}
 	currentModel={findCurrentModel(data.models, data.model)}
 	settings={data.settings}
 	{loading}

src/routes/settings/+page.server.ts

@@ -2,8 +2,7 @@ import { base } from "$app/paths";
 import { collections } from "$lib/server/database";
 import { redirect } from "@sveltejs/kit";
 import { z } from "zod";
-import { defaultModel, models } from "$lib/server/models";
-import { validateModel } from "$lib/utils/models";
+import { defaultModel, models, validateModel } from "$lib/server/models";
 import { authCondition } from "$lib/server/auth";
 
 export const actions = {