Added support for SearXNG as a websearch source (#805)

* Created searchSearxng.ts for querying and parsing searxng instance.
Added searxng to the searchWeb.ts to ensure inclusion in the search initialisation script.
Added searxng to +layout.server.ts to ensure the search button is enabled if url provided in .env.local assignment
Updated .env to include searxng configuration
Updated README.md under Web Search Config to mention addition of searxng URL use in .env

* Changes requested
consolidated env variable to SEARXNG_QUERY_URL (felt considering the content the word QUERY was more relevant)
refactored the searchSearxng.ts to generate the query with the new variable
updated related references to variable (.env, searchWeb.ts and +layout.server.ts)
Updated readme and variable comment on .env to reflect the same

* urefectored to display SearXNG in search dialoge when used.

* lint

* types

---------

Co-authored-by: Nathan Sarrazin <[email protected]>

.env

@@ -18,6 +18,7 @@ SERPER_API_KEY=#your serper.dev api key here
 SERPAPI_KEY=#your serpapi key here
 SERPSTACK_API_KEY=#your serpstack api key here
 USE_LOCAL_WEBSEARCH=#set to true to parse google results yourself, overrides other API keys
+SEARXNG_QUERY_URL=# where '<query>' will be replaced with query keywords see https://docs.searxng.org/dev/search_api.html eg https://searxng.yourdomain.com/search?q=<query>&engines=duckduckgo,google&format=json
 
 WEBSEARCH_ALLOWLIST=`[]` # if it's defined, allow websites from only this list.
 WEBSEARCH_BLOCKLIST=`[]` # if it's defined, block websites from this list.

README.md

@@ -166,7 +166,7 @@ PUBLIC_APP_DISCLAIMER=
 
 You can enable the web search through an API by adding `YDC_API_KEY` ([docs.you.com](https://docs.you.com)) or `SERPER_API_KEY` ([serper.dev](https://serper.dev/)) or `SERPAPI_KEY` ([serpapi.com](https://serpapi.com/)) or `SERPSTACK_API_KEY` ([serpstack.com](https://serpstack.com/)) to your `.env.local`.
 
-You can also simply enable the local websearch by setting `USE_LOCAL_WEBSEARCH=true` in your `.env.local`.
+You can also simply enable the local google websearch by setting `USE_LOCAL_WEBSEARCH=true` in your `.env.local` or specify a SearXNG instance by adding the query URL to `SEARXNG_QUERY_URL`.
 
 ### Custom models
 

src/lib/server/websearch/searchSearxng.ts

@@ -0,0 +1,34 @@
+import { SEARXNG_QUERY_URL } from "$env/static/private";
+
+export async function searchSearxng(query: string) {
+	const abortController = new AbortController();
+	setTimeout(() => abortController.abort(), 10000);
+
+	// Insert the query into the URL template
+	let url = SEARXNG_QUERY_URL.replace("<query>", query);
+
+	// Check if "&format=json" already exists in the URL
+	if (!url.includes("&format=json")) {
+		url += "&format=json";
+	}
+
+	// Call the URL to return JSON data
+	const jsonResponse = await fetch(url, {
+		signal: abortController.signal,
+	})
+		.then((response) => response.json() as Promise<{ results: { url: string }[] }>)
+		.catch((error) => {
+			console.error("Failed to fetch or parse JSON", error);
+			throw new Error("Failed to fetch or parse JSON");
+		});
+
+	// Extract 'url' elements from the JSON response and trim to the top 5 URLs
+	const urls = jsonResponse.results.slice(0, 5).map((item) => item.url);
+
+	if (!urls.length) {
+		throw new Error(`Response doesn't contain any "url" elements`);
+	}
+
+	// Map URLs to the correct object shape
+	return { organic_results: urls.map((link) => ({ link })) };
+}

src/lib/server/websearch/searchWeb.ts

@@ -5,15 +5,23 @@ import {
 	SERPER_API_KEY,
 	SERPSTACK_API_KEY,
 	USE_LOCAL_WEBSEARCH,
+	SEARXNG_QUERY_URL,
 	YDC_API_KEY,
 } from "$env/static/private";
 import { getJson } from "serpapi";
 import type { GoogleParameters } from "serpapi";
 import { searchWebLocal } from "./searchWebLocal";
+import { searchSearxng } from "./searchSearxng";
 
 // get which SERP api is providing web results
 export function getWebSearchProvider() {
-	return YDC_API_KEY ? WebSearchProvider.YOU : WebSearchProvider.GOOGLE;
+	if (YDC_API_KEY) {
+		return WebSearchProvider.YOU;
+	} else if (SEARXNG_QUERY_URL) {
+		return WebSearchProvider.SEARXNG;
+	} else {
+		return WebSearchProvider.GOOGLE;
+	}
 }
 
 // Show result as JSON
@@ -21,6 +29,9 @@ export async function searchWeb(query: string) {
 	if (USE_LOCAL_WEBSEARCH) {
 		return await searchWebLocal(query);
 	}
+	if (SEARXNG_QUERY_URL) {
+		return await searchSearxng(query);
+	}
 	if (SERPER_API_KEY) {
 		return await searchWebSerper(query);
 	}

src/lib/types/WebSearch.ts

@@ -42,4 +42,5 @@ interface YouSearchHit {
 export enum WebSearchProvider {
 	GOOGLE = "Google",
 	YOU = "You.com",
+	SEARXNG = "SearXNG",
 }

src/routes/+layout.server.ts

@@ -12,6 +12,7 @@ import {
 	MESSAGES_BEFORE_LOGIN,
 	YDC_API_KEY,
 	USE_LOCAL_WEBSEARCH,
+	SEARXNG_QUERY_URL,
 	ENABLE_ASSISTANTS,
 } from "$env/static/private";
 import { ObjectId } from "mongodb";
@@ -126,7 +127,8 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => {
 				SERPER_API_KEY ||
 				SERPSTACK_API_KEY ||
 				YDC_API_KEY ||
-				USE_LOCAL_WEBSEARCH
+				USE_LOCAL_WEBSEARCH ||
+				SEARXNG_QUERY_URL
 			),
 			ethicsModalAccepted: !!settings?.ethicsModalAcceptedAt,
 			ethicsModalAcceptedAt: settings?.ethicsModalAcceptedAt ?? null,