Update src/lib/server/websearch/runWebSearch.ts

src/lib/server/websearch/runWebSearch.ts

@@ -10,6 +10,7 @@ import {
 } from "$lib/server/websearch/sentenceSimilarity";
 import type { Conversation } from "$lib/types/Conversation";
 import type { MessageUpdate } from "$lib/types/MessageUpdate";
+import { webSearchParameters } from "$lib/stores/webSearchParameters";
 
 const MAX_N_PAGES_SCRAPE = 10 as const;
 const MAX_N_PAGES_EMBED = 5 as const;
@@ -17,7 +18,8 @@ const MAX_N_PAGES_EMBED = 5 as const;
 export async function runWebSearch(
 	conv: Conversation,
 	prompt: string,
-	updatePad: (upd: MessageUpdate) => void
+	updatePad: (upd: MessageUpdate) => void,
+	domainFiltersStr: string
 ) {
 	const messages = (() => {
 		return [...conv.messages, { content: prompt, from: "user", id: crypto.randomUUID() }];
@@ -26,6 +28,7 @@ export async function runWebSearch(
 	const webSearch: WebSearch = {
 		prompt: prompt,
 		searchQuery: "",
+		domainFilters: [],
 		results: [],
 		context: "",
 		contextSources: [],
@@ -33,12 +36,40 @@ export async function runWebSearch(
 		updatedAt: new Date(),
 	};
 
+
+    function extractDomains(domainFiltersStr: string): string[] {
+        const parts = domainFiltersStr.split(/[ ,]+/);
+        const domains: string[] = [];
+        const domainPattern = /^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/;
+        for (const part of parts) {
+            if (domainPattern.test(part)) {
+                domains.push(part);
+            }
+        }
+
+        return domains;
+    }
+
+    webSearch.domainFilters = extractDomains(domainFiltersStr);
+
+	function formatSearchQuery(domainFilters: string[]): string {
+      if (domainFilters.length === 0) {
+        return "";
+      }
+
+      const filteredQueryPrefix = domainFilters.map((domain) => `site:${domain}`).join(" OR ") + " ";
+
+      return filteredQueryPrefix;
+    }
+
 	function appendUpdate(message: string, args?: string[], type?: "error" | "update") {
 		updatePad({ type: "webSearch", messageType: type ?? "update", message: message, args: args });
 	}
 
 	try {
 		webSearch.searchQuery = await generateQuery(messages);
+// 		limit the sources to certain sites
+		webSearch.searchQuery = formatSearchQuery(webSearch.domainFilters) + webSearch.searchQuery;
 		appendUpdate("Searching Google", [webSearch.searchQuery]);
 		const results = await searchWeb(webSearch.searchQuery);
 		webSearch.results =
@@ -63,7 +94,7 @@ export async function runWebSearch(
 					text = await parseWeb(link);
 					appendUpdate("Browsing webpage", [link]);
 				} catch (e) {
-					// ignore errors
+					console.error(`Error parsing webpage "${link}"`, e);
 				}
 				const MAX_N_CHUNKS = 100;
 				const texts = chunk(text, CHUNK_CAR_LEN).slice(0, MAX_N_CHUNKS);
@@ -111,4 +142,4 @@ export async function runWebSearch(
 	}
 
 	return webSearch;
-}
+}