Spaces:

ruidiao
/

QuoteSearch

Running

App Files Files Community

ruidiao commited on Oct 19, 2025

Commit

fd5d5af

1 Parent(s): cf1c7cb

Upload 8 files

Browse files

Files changed (7) hide show

icons/linkedin.svg +1 -0
icons/substack.svg +1 -0
icons/x.svg +1 -0
index.html +17 -2
main.js +15 -2
offline_processing.py +52 -2
worker.js +17 -12

icons/linkedin.svg ADDED Viewed

icons/substack.svg ADDED Viewed

icons/x.svg ADDED Viewed

index.html CHANGED Viewed

@@ -8,8 +8,23 @@
 </head>
 <body class="bg-gray-100 flex items-center justify-center min-h-screen">
             <div class="container mx-auto p-8 bg-white rounded-lg shadow-lg w-full max-w-2xl relative">
-            <div id="delete-button-container" class="flex justify-end mb-4 hidden">
-                <button id="delete-data-button" class="bg-red-500 text-white px-3 py-1 rounded-lg text-sm hover:bg-red-600 focus:outline-none focus:ring-2 focus:ring-red-500">Delete Cached Data</button>
             </div>
             <h1 class="text-3xl font-bold text-center text-gray-800 mb-4">Client-Side Quote Search</h1>

 </head>
 <body class="bg-gray-100 flex items-center justify-center min-h-screen">
             <div class="container mx-auto p-8 bg-white rounded-lg shadow-lg w-full max-w-2xl relative">
+            <!-- Top controls row: icons on the left, delete button on the right. Row always visible; button visibility controlled by #delete-button-container -->
+            <div id="top-controls" class="flex items-center justify-between mb-4">
+                <div id="icons-row" class="flex items-center gap-3">
+                    <!-- Icons from icons/ folder (wrapped in links that open in a new tab) -->
+                    <a href="https://www.linkedin.com/in/ruidiao" target="_blank" rel="noopener noreferrer" aria-label="LinkedIn profile" title="LinkedIn">
+                        <img src="icons/linkedin.svg" alt="LinkedIn" class="h-6 w-6" />
+                    </a>
+                    <a href="https://ruidiao.substack.com/" target="_blank" rel="noopener noreferrer" aria-label="Substack" title="Substack">
+                        <img src="icons/substack.svg" alt="Substack" class="h-6 w-6" />
+                    </a>
+                    <a href="https://x.com/ruidiaox" target="_blank" rel="noopener noreferrer" aria-label="X (Twitter) profile" title="X">
+                        <img src="icons/x.svg" alt="X" class="h-6 w-6" />
+                    </a>
+                </div>
+                <div id="delete-button-container" class="flex justify-end hidden">
+                    <button id="delete-data-button" class="bg-red-500 text-white px-3 py-1 rounded-lg text-sm hover:bg-red-600 focus:outline-none focus:ring-2 focus:ring-red-500">Delete Cached Data</button>
+                </div>
             </div>
             <h1 class="text-3xl font-bold text-center text-gray-800 mb-4">Client-Side Quote Search</h1>

main.js CHANGED Viewed

@@ -62,7 +62,7 @@ worker.onmessage = (event) => {
             deleteButtonContainer.classList.add('hidden'); // Hide delete button if not cached
             searchInput.disabled = false; // Ensure input is enabled
             searchButton.disabled = false; // Ensure button is enabled
-            statusDiv.textContent = `To enable search, a one-time download of approximately ${formatBytesToMB(payload.size)} MB is required. This will happen when you perform your first search.`;
         }
     } else if (type === 'dataDeleted') {
         statusDiv.textContent = payload;
@@ -82,14 +82,27 @@ worker.onmessage = (event) => {
 // Send initial request for index size when the page loads
 worker.postMessage({ type: 'getIndexSize' });
-searchButton.addEventListener('click', () => {
     const query = searchInput.value;
     if (query) {
         worker.postMessage({ type: 'search', payload: query });
         statusDiv.textContent = 'Searching...';
         resultsDiv.innerHTML = ''; // Clear previous results
     } else {
         resultsDiv.innerHTML = '<p class="text-gray-500">Please enter a query.</p>';
     }
 });

             deleteButtonContainer.classList.add('hidden'); // Hide delete button if not cached
             searchInput.disabled = false; // Ensure input is enabled
             searchButton.disabled = false; // Ensure button is enabled
+            statusDiv.textContent = `To enable search, a one-time download of approximately ${formatBytesToMB(payload.size)} MB is required when you first search. You can easily remove the downloaded files later.`;
         }
     } else if (type === 'dataDeleted') {
         statusDiv.textContent = payload;
 // Send initial request for index size when the page loads
 worker.postMessage({ type: 'getIndexSize' });
+function doSearch() {
     const query = searchInput.value;
     if (query) {
         worker.postMessage({ type: 'search', payload: query });
         statusDiv.textContent = 'Searching...';
         resultsDiv.innerHTML = ''; // Clear previous results
+        return true;
     } else {
         resultsDiv.innerHTML = '<p class="text-gray-500">Please enter a query.</p>';
+        return false;
+    }
+}
+searchButton.addEventListener('click', doSearch);
+// Trigger search on Enter (without Shift). Shift+Enter inserts newline.
+searchInput.addEventListener('keydown', (e) => {
+    if (e.key === 'Enter' && !e.shiftKey) {
+        // Prevent newline from being inserted
+        e.preventDefault();
+        doSearch();
     }
 });

offline_processing.py CHANGED Viewed

@@ -4,6 +4,7 @@ import torch
 from sentence_transformers import SentenceTransformer
 import json
 import struct
 # 1. Constants
 MODEL_NAME = 'nomic-ai/nomic-embed-text-v1.5'
@@ -36,9 +37,58 @@ def load_quotes(file_path):
     if initial_rows - filtered_rows > 0:
         print(f"Ignored {initial_rows - filtered_rows} rows due to uppercase letters in category.")
     # Prepend the required prefix for retrieval tasks
-    df['quote_for_embedding'] = "search_query: " + df['quote']
-    return df.to_dict('records')
 # 3. Generate Embeddings
 def generate_embeddings(quotes, model_name, embedding_dim):

 from sentence_transformers import SentenceTransformer
 import json
 import struct
+import re
 # 1. Constants
 MODEL_NAME = 'nomic-ai/nomic-embed-text-v1.5'
     if initial_rows - filtered_rows > 0:
         print(f"Ignored {initial_rows - filtered_rows} rows due to uppercase letters in category.")
+    # Normalize categories and merge duplicate quotes (same quote + same author)
+    def normalize_category_string(cat_str):
+        if not isinstance(cat_str, str):
+            return []
+        # Split on common separators and strip whitespace
+        parts = re.split(r"[;,|/]", cat_str)
+        seen = set()
+        out = []
+        for p in parts:
+            p = p.strip()
+            if p == '':
+                continue
+            # collapse multiple spaces and lower
+            normalized = re.sub(r"\s+", " ", p).strip()
+            if normalized not in seen:
+                seen.add(normalized)
+                out.append(normalized)
+        return out
+    # Ensure author is a string for grouping (empty string for missing authors)
+    df['author'] = df['author'].fillna('').astype(str)
+    # Group by quote and author, merge categories
+    grouped = {}
+    for _, row in df.iterrows():
+        quote = row['quote']
+        author = row['author']
+        cat = row['category'] if isinstance(row['category'], str) else ''
+        key = (quote, author)
+        cats = normalize_category_string(cat)
+        if key not in grouped:
+            grouped[key] = []
+        # preserve order while merging
+        for c in cats:
+            if c not in grouped[key]:
+                grouped[key].append(c)
+    # Build records from grouped data
+    records = []
+    for (quote, author), cats in grouped.items():
+        category_combined = ', '.join(cats) if cats else ''
+        records.append({
+            'quote': quote,
+            'author': author if author != '' else None,
+            'category': category_combined
+        })
     # Prepend the required prefix for retrieval tasks
+    for r in records:
+        r['quote_for_embedding'] = "search_query: " + r['quote']
+    return records
 # 3. Generate Embeddings
 def generate_embeddings(quotes, model_name, embedding_dim):

worker.js CHANGED Viewed

@@ -91,13 +91,15 @@ async function deleteFromDB(key) {
 // 3. Load Model and Index
 async function loadModel() {
     try {
-        self.postMessage({ type: 'loading', payload: 'Loading model...' });
         // Load the model with a progress callback
         model = await pipeline('feature-extraction', MODEL_NAME, {
             progress_callback: (progress) => {
-                const detailMessage = `Model ${progress.status}: ${progress.file || ''} ${Math.floor(progress.progress || 0)}%`;
-                self.postMessage({ type: 'progress', payload: { ...progress, detail: detailMessage } });
             }
         });
     } catch (error) {
@@ -109,14 +111,15 @@ async function loadModel() {
 async function loadIndex() {
     try {
-        self.postMessage({ type: 'loading', payload: 'Checking cached index...' });
         const cachedIndex = await getFromDB('quoteIndexData');
         if (cachedIndex) {
             indexData = cachedIndex;
             self.postMessage({ type: 'loading', payload: 'Index loaded from cache.' });
-        } else {
-            self.postMessage({ type: 'loading', payload: 'Loading index...' });
             // Fetch and parse the index file with progress reporting
             const response = await fetch(INDEX_PATH);
@@ -139,7 +142,7 @@ async function loadIndex() {
                     status: 'Downloading Index',
                     progress: (loaded / total) * 100,
                     file: INDEX_PATH,
-                    detail: `Downloading index: ${Math.floor((loaded / total) * 100)}% (${(loaded / (1024 * 1024)).toFixed(2)}MB / ${(total / (1024 * 1024)).toFixed(2)}MB)`
                 };
                 self.postMessage({ type: 'progress', payload: progress });
             }
@@ -163,7 +166,7 @@ async function loadIndex() {
             const quantizedEmbeddings = new Int8Array(buffer.slice(offset));
-            // De-quantize embeddings with progress reporting
             const embeddings = new Float32Array(quantizedEmbeddings.length);
             const totalEmbeddings = quantizedEmbeddings.length;
             const updateInterval = Math.floor(totalEmbeddings / 100); // Update every 1%
@@ -171,11 +174,12 @@ async function loadIndex() {
             for (let i = 0; i < totalEmbeddings; i++) {
                 embeddings[i] = quantizedEmbeddings[i] / scale;
                 if (updateInterval > 0 && i % updateInterval === 0) {
                     const progress = {
-                        status: 'De-quantizing',
                         progress: (i / totalEmbeddings) * 100,
                         file: INDEX_PATH,
-                        detail: `De-quantizing embeddings: ${Math.floor((i / totalEmbeddings) * 100)}%`
                     };
                     self.postMessage({ type: 'progress', payload: progress });
                 }
@@ -211,11 +215,12 @@ self.onmessage = async (event) => {
     if (type === 'search') {
         if (!indexData) {
-            self.postMessage({ type: 'loading', payload: 'Loading index before search...' });
             await loadIndex();
         }
         if (!model) {
-            self.postMessage({ type: 'loading', payload: 'Loading model before search...' });
             await loadModel();
         }
         self.postMessage({ type: 'loading', payload: 'Searching...' });

 // 3. Load Model and Index
 async function loadModel() {
     try {
+        // Inform the UI that the model will be downloaded/loaded
+        self.postMessage({ type: 'loading', payload: 'Downloading model (this may take a while)...' });
         // Load the model with a progress callback
         model = await pipeline('feature-extraction', MODEL_NAME, {
             progress_callback: (progress) => {
+                // Make it explicit that this progress refers to model download or model file operations
+                const detailMessage = `Downloading model ${progress.status}: ${progress.file || ''} ${Math.floor(progress.progress || 0)}%`;
+                    self.postMessage({ type: 'progress', payload: { ...progress, detail: detailMessage } });
             }
         });
     } catch (error) {
 async function loadIndex() {
     try {
+    self.postMessage({ type: 'loading', payload: 'Checking for cached index file...' });
         const cachedIndex = await getFromDB('quoteIndexData');
         if (cachedIndex) {
             indexData = cachedIndex;
             self.postMessage({ type: 'loading', payload: 'Index loaded from cache.' });
+            } else {
+            // Inform UI that the index file will be downloaded
+            self.postMessage({ type: 'loading', payload: 'Downloading index file (this may take a while)...' });
             // Fetch and parse the index file with progress reporting
             const response = await fetch(INDEX_PATH);
                     status: 'Downloading Index',
                     progress: (loaded / total) * 100,
                     file: INDEX_PATH,
+                    detail: `Downloading index file: ${Math.floor((loaded / total) * 100)}% (${(loaded / (1024 * 1024)).toFixed(2)}MB / ${(total / (1024 * 1024)).toFixed(2)}MB)`
                 };
                 self.postMessage({ type: 'progress', payload: progress });
             }
             const quantizedEmbeddings = new Int8Array(buffer.slice(offset));
+            // De-quantize embeddings (processing step) with progress reporting
             const embeddings = new Float32Array(quantizedEmbeddings.length);
             const totalEmbeddings = quantizedEmbeddings.length;
             const updateInterval = Math.floor(totalEmbeddings / 100); // Update every 1%
             for (let i = 0; i < totalEmbeddings; i++) {
                 embeddings[i] = quantizedEmbeddings[i] / scale;
                 if (updateInterval > 0 && i % updateInterval === 0) {
+                    // This is processing (de-quantization), not download
                     const progress = {
+                        status: 'Processing index (de-quantizing)',
                         progress: (i / totalEmbeddings) * 100,
                         file: INDEX_PATH,
+                        detail: `Processing index: ${Math.floor((i / totalEmbeddings) * 100)}%`
                     };
                     self.postMessage({ type: 'progress', payload: progress });
                 }
     if (type === 'search') {
         if (!indexData) {
+            // Be explicit: index may be downloaded before search
+            self.postMessage({ type: 'loading', payload: 'Downloading index before running your search...' });
             await loadIndex();
         }
         if (!model) {
+            self.postMessage({ type: 'loading', payload: 'Downloading model before running your search...' });
             await loadModel();
         }
         self.postMessage({ type: 'loading', payload: 'Searching...' });