add you.com integration (#514)

* add you.com integration

* [Feat] Add types (#1)

* feat: add types

* feat: specify search provider

* [Feat] add return values (#3)

* feat: add types

* feat: specify search provider

* add values for ui

* add idea to gitignore

* lint and downgrade

* Revert "lint and downgrade"

This reverts commit fbfe012cf4d6aaf5ea00508ca0f99a408ce152e6.

* Updates to you.com integration (#4)

* `npm run format`

* enable search when `YDC_API_KEY` is available

* typing nit

* enum WebSearchProvider

* eslint-disable-next-line no-shadow

* downgrade

* You.com already provides text snippets (#5)

* Order desc you.com results by text length

* You.com already provides texts from webpages

* add to env

* Update .env

Co-authored-by: Mishig <[email protected]>

* Update src/lib/server/websearch/searchWeb.ts

---------

Co-authored-by: David Li <[email protected]>
Co-authored-by: Mishig <[email protected]>

.env

@@ -10,6 +10,7 @@ HF_ACCESS_TOKEN=#hf_<token> from from https://huggingface.co/settings/token
 HF_API_ROOT=https://api-inference.huggingface.co/models
 
 # used to activate search with web functionality. disabled if none are defined. choose one of the following:
+YDC_API_KEY=#your docs.you.com api key here
 SERPER_API_KEY=#your serper.dev api key here
 SERPAPI_KEY=#your serpapi key here
 

.gitignore

@@ -9,4 +9,5 @@ node_modules
 !.env.template
 vite.config.js.timestamp-*
 vite.config.ts.timestamp-*
-SECRET_CONFIG
+SECRET_CONFIG
+.idea

README.md

@@ -76,8 +76,8 @@ npm run dev
 
 Chat UI features a powerful Web Search feature. It works by:
 
-1. Generating an appropriate Google query from the user prompt.
-2. Performing Google search and extracting content from webpages.
+1. Generating an appropriate search query from the user prompt.
+2. Performing web search and extracting content from webpages.
 3. Creating embeddings from texts using [transformers.js](https://huggingface.co/docs/transformers.js). Specifically, using [Xenova/gte-small](https://huggingface.co/Xenova/gte-small) model.
 4. From these embeddings, find the ones that are closest to the user query using vector similarity search. Specifically, we use `inner product` distance.
 5. Get the corresponding texts to those closest embeddings and perform [Retrieval-Augmented Generation](https://huggingface.co/papers/2005.11401) (i.e. expand user prompt by adding those texts so that a LLM can use this information).
@@ -122,7 +122,7 @@ PUBLIC_APP_DISCLAIMER=
 
 ### Web Search config
 
-You can enable the web search by adding either `SERPER_API_KEY` ([serper.dev](https://serper.dev/)) or `SERPAPI_KEY` ([serpapi.com](https://serpapi.com/)) to your `.env.local`.
+You can enable the web search by adding any of `YDC_API_KEY` ([docs.you.com](https://docs.you.com)) or `SERPER_API_KEY` ([serper.dev](https://serper.dev/)) or `SERPAPI_KEY` ([serpapi.com](https://serpapi.com/)) to your `.env.local`.
 
 ### Custom models
 
@@ -209,7 +209,7 @@ The following is the default `webSearchQueryPromptTemplate`.
 ```prompt
 {{userMessageToken}}
   My question is: {{message.content}}.
-  Based on the conversation history (my previous questions are: {{previousMessages}}), give me an appropriate query to answer my question for google search. You should not say more than query. You should not say any words except the query. For the context, today is {{currentDate}}
+  Based on the conversation history (my previous questions are: {{previousMessages}}), give me an appropriate query to answer my question for web search. You should not say more than query. You should not say any words except the query. For the context, today is {{currentDate}}
 {{userMessageEndToken}}
 {{assistantMessageToken}}
 ```

src/lib/server/websearch/runWebSearch.ts

@@ -10,6 +10,7 @@ import {
 } from "$lib/server/websearch/sentenceSimilarity";
 import type { Conversation } from "$lib/types/Conversation";
 import type { MessageUpdate } from "$lib/types/MessageUpdate";
+import { getWebSearchProvider } from "./searchWeb";
 
 const MAX_N_PAGES_SCRAPE = 10 as const;
 const MAX_N_PAGES_EMBED = 5 as const;
@@ -39,14 +40,15 @@ export async function runWebSearch(
 
 	try {
 		webSearch.searchQuery = await generateQuery(messages);
-		appendUpdate("Searching Google", [webSearch.searchQuery]);
+		const searchProvider = getWebSearchProvider();
+		appendUpdate(`Searching ${searchProvider}`, [webSearch.searchQuery]);
 		const results = await searchWeb(webSearch.searchQuery);
 		webSearch.results =
 			(results.organic_results &&
-				results.organic_results.map((el: { title: string; link: string }) => {
-					const { title, link } = el;
+				results.organic_results.map((el: { title: string; link: string; text?: string }) => {
+					const { title, link, text } = el;
 					const { hostname } = new URL(link);
-					return { title, link, hostname };
+					return { title, link, hostname, text };
 				})) ??
 			[];
 		webSearch.results = webSearch.results
@@ -58,12 +60,14 @@ export async function runWebSearch(
 			appendUpdate("Browsing results");
 			const promises = webSearch.results.map(async (result) => {
 				const { link } = result;
-				let text = "";
-				try {
-					text = await parseWeb(link);
-					appendUpdate("Browsing webpage", [link]);
-				} catch (e) {
-					// ignore errors
+				let text = result.text ?? "";
+				if (!text) {
+					try {
+						text = await parseWeb(link);
+						appendUpdate("Browsing webpage", [link]);
+					} catch (e) {
+						// ignore errors
+					}
 				}
 				const MAX_N_CHUNKS = 100;
 				const texts = chunk(text, CHUNK_CAR_LEN).slice(0, MAX_N_CHUNKS);

src/lib/server/websearch/searchWeb.ts

@@ -1,17 +1,26 @@
-import { SERPAPI_KEY, SERPER_API_KEY } from "$env/static/private";
-
+import type { YouWebSearch } from "../../types/WebSearch";
+import { WebSearchProvider } from "../../types/WebSearch";
+import { SERPAPI_KEY, SERPER_API_KEY, YDC_API_KEY } from "$env/static/private";
 import { getJson } from "serpapi";
 import type { GoogleParameters } from "serpapi";
 
+// get which SERP api is providing web results
+export function getWebSearchProvider() {
+	return YDC_API_KEY ? WebSearchProvider.YOU : WebSearchProvider.GOOGLE;
+}
+
 // Show result as JSON
 export async function searchWeb(query: string) {
 	if (SERPER_API_KEY) {
 		return await searchWebSerper(query);
 	}
+	if (YDC_API_KEY) {
+		return await searchWebYouApi(query);
+	}
 	if (SERPAPI_KEY) {
 		return await searchWebSerpApi(query);
 	}
-	throw new Error("No Serper.dev or SerpAPI key found");
+	throw new Error("No You.com or Serper.dev or SerpAPI key found");
 }
 
 export async function searchWebSerper(query: string) {
@@ -59,3 +68,31 @@ export async function searchWebSerpApi(query: string) {
 
 	return response;
 }
+
+export async function searchWebYouApi(query: string) {
+	const response = await fetch(`https://api.ydc-index.io/search?query=${query}`, {
+		method: "GET",
+		headers: {
+			"X-API-Key": YDC_API_KEY,
+			"Content-type": "application/json; charset=UTF-8",
+		},
+	});
+
+	if (!response.ok) {
+		throw new Error(`You.com API returned error code ${response.status} - ${response.statusText}`);
+	}
+
+	const data = (await response.json()) as YouWebSearch;
+	const formattedResultsWithSnippets = data.hits
+		.map(({ title, url, snippets }) => ({
+			title,
+			link: url,
+			text: snippets?.join("\n") || "",
+			hostname: new URL(url).hostname,
+		}))
+		.sort((a, b) => b.text.length - a.text.length); // desc order by text length
+
+	return {
+		organic_results: formattedResultsWithSnippets,
+	};
+}

src/lib/types/WebSearch.ts

@@ -18,9 +18,28 @@ export interface WebSearchSource {
 	title: string;
 	link: string;
 	hostname: string;
+	text?: string; // You.com provides text of webpage right away
 }
 
 export type WebSearchMessageSources = {
 	type: "sources";
 	sources: WebSearchSource[];
 };
+
+export interface YouWebSearch {
+	hits: YouSearchHit[];
+	latency: number;
+}
+
+interface YouSearchHit {
+	url: string;
+	title: string;
+	description: string;
+	snippets: string[];
+}
+
+// eslint-disable-next-line no-shadow
+export enum WebSearchProvider {
+	GOOGLE = "Google",
+	YOU = "You.com",
+}

src/routes/+layout.server.ts

@@ -6,7 +6,12 @@ import { UrlDependency } from "$lib/types/UrlDependency";
 import { defaultModel, models, oldModels, validateModel } from "$lib/server/models";
 import { authCondition, requiresUser } from "$lib/server/auth";
 import { DEFAULT_SETTINGS } from "$lib/types/Settings";
-import { SERPAPI_KEY, SERPER_API_KEY, MESSAGES_BEFORE_LOGIN } from "$env/static/private";
+import {
+	SERPAPI_KEY,
+	SERPER_API_KEY,
+	MESSAGES_BEFORE_LOGIN,
+	YDC_API_KEY,
+} from "$env/static/private";
 
 export const load: LayoutServerLoad = async ({ locals, depends, url }) => {
 	const { conversations } = collections;
@@ -82,7 +87,7 @@ export const load: LayoutServerLoad = async ({ locals, depends, url }) => {
 			ethicsModalAcceptedAt: settings?.ethicsModalAcceptedAt ?? null,
 			activeModel: settings?.activeModel ?? DEFAULT_SETTINGS.activeModel,
 			hideEmojiOnSidebar: settings?.hideEmojiOnSidebar ?? false,
-			searchEnabled: !!(SERPAPI_KEY || SERPER_API_KEY),
+			searchEnabled: !!(SERPAPI_KEY || SERPER_API_KEY || YDC_API_KEY),
 			customPrompts: settings?.customPrompts ?? {},
 		},
 		models: models.map((model) => ({