gururise nsarrazin HF staff commited on
Commit
f02ffb2
1 Parent(s): 9b99c58

Add support for SerpStack API (#669)

Browse files

* Add support for SerpStack API

* update const to SERPSTACK_API_KEY

* update readme

* lint

---------

Co-authored-by: Nathan Sarrazin <[email protected]>

.env CHANGED
@@ -16,6 +16,7 @@ HF_ACCESS_TOKEN=#LEGACY! Use HF_TOKEN instead
16
  YDC_API_KEY=#your docs.you.com api key here
17
  SERPER_API_KEY=#your serper.dev api key here
18
  SERPAPI_KEY=#your serpapi key here
 
19
  USE_LOCAL_WEBSEARCH=#set to true to parse google results yourself, overrides other API keys
20
 
21
  WEBSEARCH_ALLOWLIST=`[]` # if it's defined, allow websites from only this list.
 
16
  YDC_API_KEY=#your docs.you.com api key here
17
  SERPER_API_KEY=#your serper.dev api key here
18
  SERPAPI_KEY=#your serpapi key here
19
+ SERPSTACK_API_KEY=#your serpstack api key here
20
  USE_LOCAL_WEBSEARCH=#set to true to parse google results yourself, overrides other API keys
21
 
22
  WEBSEARCH_ALLOWLIST=`[]` # if it's defined, allow websites from only this list.
README.md CHANGED
@@ -122,7 +122,7 @@ PUBLIC_APP_DISCLAIMER=
122
 
123
  ### Web Search config
124
 
125
- You can enable the web search through an API by adding `YDC_API_KEY` ([docs.you.com](https://docs.you.com)) or `SERPER_API_KEY` ([serper.dev](https://serper.dev/)) or `SERPAPI_KEY` ([serpapi.com](https://serpapi.com/)) to your `.env.local`.
126
 
127
  You can also simply enable the local websearch by setting `USE_LOCAL_WEBSEARCH=true` in your `.env.local`.
128
 
 
122
 
123
  ### Web Search config
124
 
125
+ You can enable the web search through an API by adding `YDC_API_KEY` ([docs.you.com](https://docs.you.com)) or `SERPER_API_KEY` ([serper.dev](https://serper.dev/)) or `SERPAPI_KEY` ([serpapi.com](https://serpapi.com/)) or `SERPSTACK_API_KEY` ([serpstack.com](https://serpstack.com/)) to your `.env.local`.
126
 
127
  You can also simply enable the local websearch by setting `USE_LOCAL_WEBSEARCH=true` in your `.env.local`.
128
 
src/lib/server/websearch/searchWeb.ts CHANGED
@@ -1,6 +1,12 @@
1
  import type { YouWebSearch } from "../../types/WebSearch";
2
  import { WebSearchProvider } from "../../types/WebSearch";
3
- import { SERPAPI_KEY, SERPER_API_KEY, USE_LOCAL_WEBSEARCH, YDC_API_KEY } from "$env/static/private";
 
 
 
 
 
 
4
  import { getJson } from "serpapi";
5
  import type { GoogleParameters } from "serpapi";
6
  import { searchWebLocal } from "./searchWebLocal";
@@ -24,6 +30,9 @@ export async function searchWeb(query: string) {
24
  if (SERPAPI_KEY) {
25
  return await searchWebSerpApi(query);
26
  }
 
 
 
27
  throw new Error("No You.com or Serper.dev or SerpAPI key found");
28
  }
29
 
@@ -100,3 +109,36 @@ export async function searchWebYouApi(query: string) {
100
  organic_results: formattedResultsWithSnippets,
101
  };
102
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import type { YouWebSearch } from "../../types/WebSearch";
2
  import { WebSearchProvider } from "../../types/WebSearch";
3
+ import {
4
+ SERPAPI_KEY,
5
+ SERPER_API_KEY,
6
+ SERPSTACK_API_KEY,
7
+ USE_LOCAL_WEBSEARCH,
8
+ YDC_API_KEY,
9
+ } from "$env/static/private";
10
  import { getJson } from "serpapi";
11
  import type { GoogleParameters } from "serpapi";
12
  import { searchWebLocal } from "./searchWebLocal";
 
30
  if (SERPAPI_KEY) {
31
  return await searchWebSerpApi(query);
32
  }
33
+ if (SERPSTACK_API_KEY) {
34
+ return await searchSerpStack(query);
35
+ }
36
  throw new Error("No You.com or Serper.dev or SerpAPI key found");
37
  }
38
 
 
109
  organic_results: formattedResultsWithSnippets,
110
  };
111
  }
112
+
113
+ export async function searchSerpStack(query: string) {
114
+ const response = await fetch(
115
+ `http://api.serpstack.com/search?access_key=${SERPSTACK_API_KEY}&query=${query}&hl=en&gl=us`,
116
+ {
117
+ method: "GET",
118
+ headers: {
119
+ "Content-type": "application/json; charset=UTF-8",
120
+ },
121
+ }
122
+ );
123
+
124
+ const data = (await response.json()) as Record<string, any>;
125
+
126
+ if (!response.ok) {
127
+ throw new Error(
128
+ data["error"] ??
129
+ `SerpStack API returned error code ${response.status} - ${response.statusText}`
130
+ );
131
+ }
132
+
133
+ const resultsWithSnippets = data["organic_results"].map(
134
+ ({ title, url, snippet }: { title: string; url: string; snippet: string | undefined }) => ({
135
+ title,
136
+ link: url,
137
+ text: snippet || "",
138
+ })
139
+ );
140
+
141
+ return {
142
+ organic_results: resultsWithSnippets ?? [],
143
+ };
144
+ }
src/routes/+layout.server.ts CHANGED
@@ -8,6 +8,7 @@ import { DEFAULT_SETTINGS } from "$lib/types/Settings";
8
  import {
9
  SERPAPI_KEY,
10
  SERPER_API_KEY,
 
11
  MESSAGES_BEFORE_LOGIN,
12
  YDC_API_KEY,
13
  USE_LOCAL_WEBSEARCH,
@@ -77,7 +78,13 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => {
77
  }))
78
  .toArray(),
79
  settings: {
80
- searchEnabled: !!(SERPAPI_KEY || SERPER_API_KEY || YDC_API_KEY || USE_LOCAL_WEBSEARCH),
 
 
 
 
 
 
81
  ethicsModalAccepted: !!settings?.ethicsModalAcceptedAt,
82
  ethicsModalAcceptedAt: settings?.ethicsModalAcceptedAt ?? null,
83
  activeModel: settings?.activeModel ?? DEFAULT_SETTINGS.activeModel,
 
8
  import {
9
  SERPAPI_KEY,
10
  SERPER_API_KEY,
11
+ SERPSTACK_API_KEY,
12
  MESSAGES_BEFORE_LOGIN,
13
  YDC_API_KEY,
14
  USE_LOCAL_WEBSEARCH,
 
78
  }))
79
  .toArray(),
80
  settings: {
81
+ searchEnabled: !!(
82
+ SERPAPI_KEY ||
83
+ SERPER_API_KEY ||
84
+ SERPSTACK_API_KEY ||
85
+ YDC_API_KEY ||
86
+ USE_LOCAL_WEBSEARCH
87
+ ),
88
  ethicsModalAccepted: !!settings?.ethicsModalAcceptedAt,
89
  ethicsModalAcceptedAt: settings?.ethicsModalAcceptedAt ?? null,
90
  activeModel: settings?.activeModel ?? DEFAULT_SETTINGS.activeModel,