allow different user and assistant end-token (#375)

* allow different user and assistant end-token

For models like Llama2, the EndToken is not the same for a userMessage
and an assistantMessage. This implements `userMessageEndToken` and
`assistantMessageEndToken` which overwrites the messageEndToken
behavior.

This PR also allows empty strings as userMessageToken and
assistantMessageToken and makes this the default. This adds additional
flexibility, which is required in the case of Llama2 where the first
userMessage is effectively different because of the system message.

Note that because `userMessageEndToken` and `assistantMessageToken` are
nearly always concatenated, it is almost redundant to have both. The
exception is `generateQuery` for websearch which have several
consecutive user messages.

* Make model branding customizable based on env var (#345)

* rm open assistant branding

* Update SettingsModal.svelte

* make settings work with a dynamic list of models

* fixed types

---------

Co-authored-by: Nathan Sarrazin <[email protected]>

* trim and remove stop-suffixes from summary (#369)

The chat generation removes parameters.stop and <|endoftext|>
from the generated text. And additionally trims trailing whitespace.

This PR copies that behavior to the summarize functionality, when the
summary is produced by a the chat model.

* add a login button when users are logged out (#381)

* add fallback to message end token if there's no specified tokens for user & assistant

---------

Co-authored-by: Florian Zimmermeister <[email protected]>
Co-authored-by: Nathan Sarrazin <[email protected]>

README.md

@@ -121,6 +121,8 @@ MODELS=`[
     "userMessageToken": "<|prompter|>", # This does not need to be a token, can be any string
     "assistantMessageToken": "<|assistant|>", # This does not need to be a token, can be any string
     "messageEndToken": "<|endoftext|>", # This does not need to be a token, can be any string
+    # "userMessageEndToken": "", # Applies only to user messages, messageEndToken has no effect if specified. Can be any string.
+    # "assistantMessageEndToken": "", # Applies only to assistant messages, messageEndToken has no effect if specified. Can be any string.
     "preprompt": "Below are a series of dialogues between various people and an AI assistant. The AI tries to be helpful, polite, honest, sophisticated, emotionally aware, and humble-but-knowledgeable. The assistant is happy to help with almost anything, and will do its best to understand exactly what is needed. It also tries to avoid giving false or misleading information, and it caveats when it isn't entirely sure about the right answer. That said, the assistant is practical and really does its best, and doesn't let caution get too much in the way of being useful.\n-----\n",
     "promptExamples": [
       {

src/lib/buildPrompt.ts

@@ -13,18 +13,19 @@ export async function buildPrompt(
 	model: BackendModel,
 	webSearchId?: string
 ): Promise<string> {
+	const userEndToken = model.userMessageEndToken ?? model.messageEndToken;
+	const assistantEndToken = model.assistantMessageEndToken ?? model.messageEndToken;
+
 	const prompt =
 		messages
-			.map(
-				(m) =>
-					(m.from === "user"
-						? model.userMessageToken + m.content
-						: model.assistantMessageToken + m.content) +
-					(model.messageEndToken
-						? m.content.endsWith(model.messageEndToken)
-							? ""
-							: model.messageEndToken
-						: "")
+			.map((m) =>
+				m.from === "user"
+					? model.userMessageToken +
+					  m.content +
+					  (m.content.endsWith(userEndToken) ? "" : userEndToken)
+					: model.assistantMessageToken +
+					  m.content +
+					  (m.content.endsWith(assistantEndToken) ? "" : assistantEndToken)
 			)
 			.join("") + model.assistantMessageToken;
 
@@ -41,7 +42,7 @@ export async function buildPrompt(
 			webPrompt =
 				model.assistantMessageToken +
 				`The following context was found while searching the internet: ${webSearch.summary}` +
-				model.messageEndToken;
+				model.assistantMessageEndToken;
 		}
 	}
 	const finalPrompt =

src/lib/server/models.ts

@@ -14,9 +14,11 @@ const modelsRaw = z
 			modelUrl: z.string().url().optional(),
 			datasetName: z.string().min(1).optional(),
 			datasetUrl: z.string().url().optional(),
-			userMessageToken: z.string().min(1),
-			assistantMessageToken: z.string().min(1),
-			messageEndToken: z.string().min(1).optional(),
+			userMessageToken: z.string(),
+			userMessageEndToken: z.string().default(""),
+			assistantMessageToken: z.string(),
+			assistantMessageEndToken: z.string().default(""),
+			messageEndToken: z.string().default(""),
 			preprompt: z.string().default(""),
 			prepromptUrl: z.string().url().optional(),
 			promptExamples: z
@@ -52,6 +54,8 @@ const modelsRaw = z
 export const models = await Promise.all(
 	modelsRaw.map(async (m) => ({
 		...m,
+		userMessageEndToken: m?.userMessageEndToken || m?.messageEndToken,
+		assistantMessageEndToken: m?.assistantMessageEndToken || m?.messageEndToken,
 		id: m.id || m.name,
 		displayName: m.displayName || m.name,
 		preprompt: m.prepromptUrl ? await fetch(m.prepromptUrl).then((r) => r.text()) : m.preprompt,

src/lib/server/websearch/generateQuery.ts

@@ -6,13 +6,13 @@ export async function generateQuery(messages: Message[], model: BackendModel) {
 	const promptSearchQuery =
 		model.userMessageToken +
 		"The following messages were written by a user, trying to answer a question." +
-		model.messageEndToken +
+		model.userMessageEndToken +
 		messages
 			.filter((message) => message.from === "user")
-			.map((message) => model.userMessageToken + message.content + model.messageEndToken) +
+			.map((message) => model.userMessageToken + message.content + model.userMessageEndToken) +
 		model.userMessageToken +
 		"What plain-text english sentence would you input into Google to answer the last question? Answer with a short (10 words max) simple sentence." +
-		model.messageEndToken +
+		model.userMessageEndToken +
 		model.assistantMessageToken +
 		"Query: ";
 

src/lib/server/websearch/summarizeWeb.ts

@@ -29,10 +29,10 @@ export async function summarizeWeb(content: string, query: string, model: Backen
 			.split(" ")
 			.slice(0, model.parameters?.truncate ?? 0)
 			.join(" ") +
-		model.messageEndToken +
+		model.userMessageEndToken +
 		model.userMessageToken +
 		`The text above should be summarized to best answer the query: ${query}.` +
-		model.messageEndToken +
+		model.userMessageEndToken +
 		model.assistantMessageToken +
 		"Summary: ";