Changes in search algorithm

chat-ui/src/lib/server/websearch/runWebSearch.ts

@@ -52,26 +52,27 @@ export async function runWebSearch(
 		appendUpdate("Ищем в яндексе по запросу: ", [webSearch.searchQuery]);
 
 		const results = await searchWeb(webSearch.searchQuery);
-
+		console.log('search results', results)
+		
 		webSearch.results =
-			(results.organic_results &&
+			(results && results.organic_results &&
 				results.organic_results.map((el: { title: string; link: string }) => {
 					const { title, link } = el;
 					const { hostname } = new URL(link);
 					return { title, link, hostname };
 				})) ??
 			[];
-		
-		
+
 		webSearch.results = webSearch.results
 			.filter(({ link }) => !link.includes("youtube.com") && !link.includes(".pdf")); // filter out youtube links
 			//slice(0, Number(MAX_N_PAGES_SCRAPE)); // limit to first 10 links only
 
 		// let paragraphChunks: { source: WebSearchSource; text: string }[] = [];
+
 		let texts : {source: any, text: string}[] = [];
 		if (webSearch.results.length > 0) {
 			appendUpdate("Обработка результатов");
-
+			let fullText = '';
 			for(const i in webSearch.results) {
 				if(texts.length > 30) break;
 				const { link, hostname, title } = webSearch.results[i];
@@ -96,9 +97,21 @@ export async function runWebSearch(
 				} catch (e) {
 					console.error(`Error parsing webpage "${link}"`, e);
 				}
-				// const MAX_N_CHUNKS = 20;
-				if(text.length > 0)
+				
+
+				
+				if(text.length > 0){
+					webSearch.contextSources.push({ link: link, hostname: hostname, title: title });
+					
 					texts.push({source: { link: link, hostname: hostname, title: title }, text})
+				}
+					
+				
+				fullText += text;
+				if(fullText.length >= Number(SAIGA_TRUNCATE_WEB_CONTEXT)){
+					break;
+				}
+				
 				// texts.push(...chunk(text, CHUNK_CAR_LEN).slice(0, MAX_N_CHUNKS));
 			}
 		} else {
@@ -107,28 +120,57 @@ export async function runWebSearch(
 			//throw new Error("No results found for this search query");
 		}		
 
-		if(texts && text.length > 0){
+		if(texts && texts.length > 0){
 			appendUpdate("Получение релевантной информации");
-
-			const indices = await findSimilarSentences(prompt, texts.map((t) => t.text));//, { topK: topKClosestParagraphs});
-			console.log('similarity check result:', indices);
-
-
-			webSearch.context = indices.filter((r) => r.score >= Number(SIMILARITY_THRESHOLD))
-									.sort((a, b) => b.score - a.score)
-									.slice(0, Number(MAX_N_PAGES_SCRAPE))
-									.map(({i}) => texts[i].text)
-									.join(" ")
-									.slice(0, Number(SAIGA_TRUNCATE_WEB_CONTEXT));
+			console.log('webSearch.contextSources', webSearch.contextSources)
+			// const allIndices = await findSimilarSentences(prompt, texts.map((t) => t.text));//, { topK: topKClosestParagraphs});
+			
+			// console.log('similarity check result:', allIndices);
+			// const indices = allIndices.filter((r) => r.score >= Number(SIMILARITY_THRESHOLD))
+			// 	.sort((a, b) => b.score - a.score)
+			// 	.slice(0, Number(MAX_N_PAGES_SCRAPE));
 			
-			console.log('web search context:', webSearch.context);
 
+			// webSearch.context = indices.map(({i}) => texts[i].text)
+			// 						.join("\n")
+			// 						.slice(0, Number(SAIGA_TRUNCATE_WEB_CONTEXT));
+
+			// webSearch.contextSources = [
+			// 	{
+			// 	  link: 'https://www.nalog.gov.ru/rn77/fl/interest/inn/calculation/',
+			// 	  hostname: 'www.nalog.gov.ru',
+			// 	  title: 'Я хочу встать на учёт в налоговый орган (получить...)'
+			// 	},
+			// 	{
+			// 	  link: 'https://www.nalog.gov.ru/rn71/news/activities_fts/13690572/',
+			// 	  hostname: 'www.nalog.gov.ru',
+			// 	  title: 'Как получить свидетельство ИНН | ФНС России'
+			// 	},
+			// 	{
+			// 	  link: 'https://www.nalog.gov.ru/rn10/news/activities_fts/13604016/',
+			// 	  hostname: 'www.nalog.gov.ru',
+			// 	  title: 'Как получить ИНН через электронные сервисы ФНС'
+			// 	}
+			//   ];
 			updatePad({
 				type: "webSearch",
 				messageType: "sources",
 				message: "sources",
-				sources: [],
+				sources: webSearch.contextSources,
 			});
+
+			webSearch.context = texts.map((t) => t.text)
+				.join("\n")
+				.slice(0, Number(SAIGA_TRUNCATE_WEB_CONTEXT));
+			
+			console.log('web search context:', webSearch.context);
+
+			// updatePad({
+			// 	type: "webSearch",
+			// 	messageType: "sources",
+			// 	message: "sources",
+			// 	sources: [],
+			// });
 			
 			// const usedSources = new Set<string>();
 			// for (const idx of indices) {

chat-ui/src/lib/server/websearch/searchWeb.ts

@@ -72,9 +72,13 @@ export async function searchWebSerpApi(query: string) {
 	} satisfies YandexParameters;
 
 	// Show result as JSON
-	const response = await getJson("yandex", params);
+	try {
+		const response = await getJson("yandex", params);
 
-	return response;
+		return response;
+	} catch (e) {
+		console.error('Error fetching yandex results:', e);
+	}
 }
 
 export async function searchYandex(query: string) {
@@ -103,7 +107,7 @@ export async function searchYandex(query: string) {
 		});
 	});
 
-	
+
 
 	console.log('Yandex search result', dataArray)
 	return { organic_results: dataArray };

chat-ui/src/routes/conversation/[id]/+server.ts

@@ -23,6 +23,10 @@ import { buildSearchPrompt } from "$lib/buildSearchPrompt.js";
 import PastebinAPI from 'pastebin-ts';
 
 function Log(data: any) {
+	if(!PASTEBIN_DEV_KEY || !PASTEBIN_USERNAME || !PASTEBIN_PASSWORD){
+		console.log('Skipping pastebin log')
+		return;
+	}
 	console.log('uploading data to pastebin');
 	try {
 		const pastebin = new PastebinAPI({