Update app.py

app.py

@@ -1,30 +1,20 @@
 import os
-import math
 from typing import List, Dict, Any, Tuple, Optional
 
 import requests
 import gradio as gr
 from openai import OpenAI
 
-# Firecrawl SDK (used for crawling URLs into markdown)
+# Firecrawl SDK (used for scraping URLs into markdown)
 try:
     from firecrawl import Firecrawl
 except ImportError:
-    Firecrawl = None  # we’ll handle this gracefully later
+    Firecrawl = None  # handled gracefully below
 
 # -------------------- CONFIG --------------------
 
 CHAT_MODEL = "gpt-5"  # main chat model
 
-# Candidate embedding models – we'll auto-select one your project has access to
-EMBED_MODEL_CANDIDATES = [
-    "text-embedding-3-small",
-    "text-embedding-3-large",
-    "text-embedding-ada-002",
-]
-
-SELECTED_EMBED_MODEL: Optional[str] = None  # set at runtime once discovered
-
 DEFAULT_SYSTEM_PROMPT = """You are a Retrieval-Augmented Generation (RAG) assistant.
 
 Rules:
@@ -64,7 +54,7 @@ PRESET_CONFIGS = {
     },
 }
 
-# -------------------- TEXT / EMBEDDING HELPERS --------------------
+# -------------------- TEXT HELPERS --------------------
 
 
 def chunk_text(text: str, max_chars: int = 2000, overlap: int = 200) -> List[str]:
@@ -85,45 +75,15 @@ def chunk_text(text: str, max_chars: int = 2000, overlap: int = 200) -> List[str
     return chunks
 
 
-def cosine_similarity(a: List[float], b: List[float]) -> float:
-    """Compute cosine similarity between two vectors."""
-    if not a or not b:
-        return 0.0
-    dot = sum(x * y for x, y in zip(a, b))
-    norm_a = math.sqrt(sum(x * x for x in a))
-    norm_b = math.sqrt(sum(y * y for y in b))
-    if norm_a == 0 or norm_b == 0:
-        return 0.0
-    return dot / (norm_a * norm_b)
-
-
-# -------------------- EMBEDDING MODEL SELECTION --------------------
-
-
-def pick_embedding_model(client: OpenAI) -> str:
-    """
-    Discover a usable embedding model for this project.
-    Tries candidates in order and caches the first that works.
-    """
-    global SELECTED_EMBED_MODEL
-    if SELECTED_EMBED_MODEL:
-        return SELECTED_EMBED_MODEL
-
-    last_error: Optional[Exception] = None
-    for model_name in EMBED_MODEL_CANDIDATES:
-        try:
-            # cheap sanity call
-            client.embeddings.create(model=model_name, input="test")
-            SELECTED_EMBED_MODEL = model_name
-            return model_name
-        except Exception as e:
-            last_error = e
-            continue
-
-    raise RuntimeError(
-        f"No usable embedding model found for this project. "
-        f"Tried: {EMBED_MODEL_CANDIDATES}. Last error: {last_error}"
-    )
+def tokenize(text: str) -> List[str]:
+    """Very simple tokenizer: lowercase, keep alphanumerics, split on spaces."""
+    cleaned = []
+    for ch in text.lower():
+        if ch.isalnum():
+            cleaned.append(ch)
+        else:
+            cleaned.append(" ")
+    return [tok for tok in "".join(cleaned).split() if tok]
 
 
 # -------------------- DATA SOURCE HELPERS --------------------
@@ -166,10 +126,7 @@ def read_file_text(path: str) -> str:
 
 def extract_markdown_from_firecrawl_result(result: Any) -> str:
     """
-    Firecrawl crawl(...) can return:
-    - A list of Document-like objects with .markdown
-    - An object with .data which is a list of Documents
-    - Dict structures with 'markdown' or 'data'
+    Firecrawl scrape(...) can return Document-like objects or dicts.
     We try to collect all markdown text into one big string.
     """
     texts: List[str] = []
@@ -206,10 +163,10 @@ def extract_markdown_from_firecrawl_result(result: Any) -> str:
     return str(result)
 
 
-def firecrawl_crawl_url(firecrawl_api_key: str, url: str) -> str:
+def firecrawl_scrape_url(firecrawl_api_key: str, url: str) -> str:
     """
-    Use Firecrawl to crawl a URL and return concatenated markdown for all pages.
-    If Firecrawl is not available or fails, return an error marker (caller can fallback).
+    Use Firecrawl to scrape a single URL and return markdown.
+    This is intentionally *not* a full crawl to keep it fast.
     """
     firecrawl_api_key = (firecrawl_api_key or "").strip()
     if not firecrawl_api_key:
@@ -220,31 +177,22 @@ def firecrawl_crawl_url(firecrawl_api_key: str, url: str) -> str:
 
     try:
         fc = Firecrawl(api_key=firecrawl_api_key)
-        # Crawl whole site; keep limit modest for speed
-        docs = fc.crawl(url=url, limit=50)
-        markdown = extract_markdown_from_firecrawl_result(docs)
+        # Fast single-page scrape → markdown suitable for RAG
+        doc = fc.scrape(url, formats=["markdown"])
+        markdown = extract_markdown_from_firecrawl_result(doc)
         return markdown
     except Exception as e:
         return f"[Firecrawl error for {url}: {e}]"
 
 
-# -------------------- EMBEDDING / KB BUILD --------------------
-
+# -------------------- LOCAL KB BUILD (NO OPENAI EMBEDDINGS) --------------------
 
-def build_embeddings(
-    api_key: str,
-    docs: List[Dict[str, Any]],
-) -> Tuple[List[Dict[str, Any]], str]:
-    """Embed all document chunks and return them as KB docs with embeddings."""
-    if not docs:
-        return [], "⚠️ No documents to index."
-
-    client = OpenAI(api_key=api_key)
-    try:
-        embed_model = pick_embedding_model(client)
-    except Exception as e:
-        return [], f"❌ Failed to select an embedding model: {e}"
 
+def build_local_kb(docs: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], str]:
+    """
+    Build a local KB with lexical features only (no OpenAI embeddings).
+    Each KB entry: {id, source, text, tokens}
+    """
     kb_chunks: List[Dict[str, Any]] = []
     total_chunks = 0
 
@@ -254,80 +202,54 @@ def build_embeddings(
         chunks = chunk_text(text, max_chars=2000, overlap=200)
 
         for idx, ch in enumerate(chunks):
-            try:
-                emb_resp = client.embeddings.create(
-                    model=embed_model,
-                    input=ch,
-                )
-                emb = emb_resp.data[0].embedding
-                kb_chunks.append(
-                    {
-                        "id": f"{source}_{idx}",
-                        "source": source,
-                        "text": ch,
-                        "embedding": emb,
-                    }
-                )
-                total_chunks += 1
-            except Exception as e:
-                kb_chunks.append(
-                    {
-                        "id": f"{source}_{idx}_error",
-                        "source": source,
-                        "text": f"[Error embedding chunk: {e}]",
-                        "embedding": [],
-                    }
-                )
-
-    status = (
-        f"✅ Knowledge base built with {len(docs)} documents and {total_chunks} chunks. "
-        f"Embedding model: `{SELECTED_EMBED_MODEL}`"
-    )
+            tokens = tokenize(ch)
+            kb_chunks.append(
+                {
+                    "id": f"{source}_{idx}",
+                    "source": source,
+                    "text": ch,
+                    "tokens": tokens,
+                }
+            )
+            total_chunks += 1
+
+    status = f"✅ Knowledge base built with {len(docs)} documents and {total_chunks} chunks (lexical retrieval)."
     return kb_chunks, status
 
 
-def retrieve_context(
-    api_key: str,
+def retrieve_context_local(
     kb: List[Dict[str, Any]],
     query: str,
     top_k: int = 5,
-    similarity_threshold: float = 0.25,
 ) -> Tuple[str, str]:
-    """Retrieve top-k relevant chunks from KB for the query."""
+    """
+    Retrieve top-k relevant chunks from KB for the query using simple lexical matching:
+    score = number of overlapping tokens between query and chunk.
+    """
     if not kb:
         return "", "ℹ️ No knowledge base yet. The model will answer from instructions only."
 
-    client = OpenAI(api_key=api_key)
-    try:
-        embed_model = pick_embedding_model(client)
-    except Exception as e:
-        return "", f"❌ Failed to select an embedding model: {e}"
+    q_tokens = tokenize(query)
+    if not q_tokens:
+        return "", "ℹ️ Query has no meaningful tokens; answering from instructions only."
 
-    try:
-        q_emb_resp = client.embeddings.create(
-            model=embed_model,
-            input=query,
-        )
-        q_emb = q_emb_resp.data[0].embedding
-    except Exception as e:
-        return "", f"⚠️ Error creating query embedding: {e}"
+    q_set = set(q_tokens)
 
-    scored: List[Tuple[float, Dict[str, Any]]] = []
+    scored: List[Tuple[int, Dict[str, Any]]] = []
     for d in kb:
-        emb = d.get("embedding") or []
-        if not emb:
+        tokens = d.get("tokens") or []
+        if not tokens:
             continue
-        sim = cosine_similarity(q_emb, emb)
-        scored.append((sim, d))
+        t_set = set(tokens)
+        overlap = len(q_set & t_set)
+        if overlap > 0:
+            scored.append((overlap, d))
 
     if not scored:
-        return "", "⚠️ No valid embeddings in KB; cannot retrieve context."
+        return "", "ℹ️ No lexical overlap with knowledge base; answering from instructions only."
 
     scored.sort(key=lambda x: x[0], reverse=True)
-    top = [d for (sim, d) in scored[:top_k] if sim >= similarity_threshold]
-
-    if not top:
-        return "", "ℹ️ No chunks passed the similarity threshold; answering from instructions only."
+    top = [d for (score, d) in scored[:top_k]]
 
     context_parts = []
     for idx, d in enumerate(top, start=1):
@@ -338,11 +260,7 @@ def retrieve_context(
         )
 
     context = "\n\n---\n\n".join(context_parts)
-    debug = (
-        f"📚 Retrieved {len(top)} chunks from KB "
-        f"(top_k={top_k}, threshold={similarity_threshold}). "
-        f"Embedding model: `{SELECTED_EMBED_MODEL}`"
-    )
+    debug = f"📚 Retrieved {len(top)} chunks from KB via lexical retrieval (no embeddings)."
     return context, debug
 
 
@@ -354,7 +272,7 @@ def save_api_key(api_key: str):
     if not api_key:
         return "❌ No API key provided.", ""
     masked = f"{api_key[:4]}...{api_key[-4:]}" if len(api_key) >= 8 else "******"
-    status = f"✅ API key saved for this session: `{masked}`"
+    status = f"✅ OpenAI key saved for this session: `{masked}`"
     return status, api_key
 
 
@@ -381,7 +299,7 @@ def build_knowledge_base(
 ):
     """
     Build knowledge base using:
-    - Firecrawl for URLs (if Firecrawl key provided and SDK available)
+    - Firecrawl scrape for URLs (if Firecrawl key provided and SDK available)
     - Fallback to simple HTTP fetch if Firecrawl not available
     - Raw text
     - Files
@@ -399,8 +317,8 @@ def build_knowledge_base(
     for u in urls:
         text_from_url = ""
         if firecrawl_api_key:
-            # Try Firecrawl first
-            fc_text = firecrawl_crawl_url(firecrawl_api_key, u)
+            # Try Firecrawl first (single-page scrape)
+            fc_text = firecrawl_scrape_url(firecrawl_api_key, u)
             if not fc_text.startswith("[Firecrawl error"):
                 text_from_url = fc_text
             else:
@@ -428,7 +346,7 @@ def build_knowledge_base(
     if not docs:
         return "⚠️ No knowledge sources provided (URLs, text, or files).", []
 
-    kb, status = build_embeddings(api_key, docs)
+    kb, status = build_local_kb(docs)
     return status, kb
 
 
@@ -452,8 +370,8 @@ def chat_with_rag(
     if not system_prompt:
         system_prompt = DEFAULT_SYSTEM_PROMPT
 
-    # Retrieve context from KB
-    context, debug_retrieval = retrieve_context(api_key, kb, user_message)
+    # Retrieve context from KB (local lexical retrieval)
+    context, debug_retrieval = retrieve_context_local(kb, user_message)
 
     client = OpenAI(api_key=api_key)
 
@@ -517,8 +435,8 @@ with gr.Blocks(title="RAG Chatbot — GPT-5 + URLs / Files / Text + Firecrawl")
 
 1. Enter your **OpenAI API key** and click **Save**.  
 2. (Optional) Enter your **Firecrawl API key** and save it.  
-3. Add knowledge via **URLs** (e.g. `https://zenai.world`), **uploaded files**, and/or **raw text**.  
-4. Click **Grab / Retrieve Knowledge (Firecrawl)** to crawl URLs + index everything.  
+3. Choose a preset (e.g. **ZEN Sites Deep QA**) — this auto-loads URLs like `https://zenai.world`.  
+4. Click **Grab / Retrieve Knowledge (Firecrawl + Lexical Index)** to scrape URLs + index everything.  
 5. Ask questions — the bot will answer **only** from your knowledge and system instructions.
 """
     )
@@ -528,6 +446,10 @@ with gr.Blocks(title="RAG Chatbot — GPT-5 + URLs / Files / Text + Firecrawl")
     kb_state = gr.State([])
     chat_state = gr.State([])
 
+    # default preset on load -> ZEN
+    default_preset_name = "ZEN Sites Deep QA (zenai.world + AI Arena)"
+    default_preset_cfg = PRESET_CONFIGS[default_preset_name]
+
     with gr.Row():
         with gr.Column(scale=1):
             gr.Markdown("### 🔑 API & System")
@@ -546,18 +468,20 @@ with gr.Blocks(title="RAG Chatbot — GPT-5 + URLs / Files / Text + Firecrawl")
                 type="password",
             )
             save_firecrawl_btn = gr.Button("Save Firecrawl Key")
-            firecrawl_status = gr.Markdown("Firecrawl key not set (fallback to simple URL fetch).")
+            firecrawl_status = gr.Markdown(
+                "Firecrawl key not set (will fall back to simple URL fetch)."
+            )
 
             preset_dropdown = gr.Dropdown(
                 label="Presets",
                 choices=list(PRESET_CONFIGS.keys()),
-                value="ZEN Sites Deep QA (zenai.world + AI Arena)",
+                value=default_preset_name,
             )
 
             system_box = gr.Textbox(
                 label="System Instructions",
                 lines=8,
-                value=DEFAULT_SYSTEM_PROMPT,
+                value=default_preset_cfg["system"],
             )
 
             gr.Markdown("### 📚 Knowledge Sources")
@@ -565,12 +489,14 @@ with gr.Blocks(title="RAG Chatbot — GPT-5 + URLs / Files / Text + Firecrawl")
             urls_box = gr.Textbox(
                 label="Knowledge URLs (one per line)",
                 lines=4,
+                value=default_preset_cfg["urls"],
                 placeholder="https://zenai.world\nhttps://us.zenai.biz",
             )
 
             raw_text_box = gr.Textbox(
                 label="Additional Knowledge Text",
                 lines=6,
+                value=default_preset_cfg["text"],
                 placeholder="Paste any notes, docs, or reference text here...",
             )
 
@@ -581,7 +507,7 @@ with gr.Blocks(title="RAG Chatbot — GPT-5 + URLs / Files / Text + Firecrawl")
             )
 
             grab_kb_btn = gr.Button(
-                "Grab / Retrieve Knowledge (Firecrawl + Embeddings)",
+                "Grab / Retrieve Knowledge (Firecrawl + Lexical Index)",
                 variant="secondary",
             )
             kb_status_md = gr.Markdown("ℹ️ No knowledge base built yet.")
@@ -598,7 +524,7 @@ with gr.Blocks(title="RAG Chatbot — GPT-5 + URLs / Files / Text + Firecrawl")
             user_input = gr.Textbox(
                 label="Ask a question",
                 lines=3,
-                placeholder="Ask about the content of zenai.world, AI Arena, or your uploaded docs...",
+                placeholder="Ask about zenai.world, AI Arena, or your uploaded docs...",
             )
 
             with gr.Row():
@@ -630,7 +556,7 @@ with gr.Blocks(title="RAG Chatbot — GPT-5 + URLs / Files / Text + Firecrawl")
         outputs=[system_box, urls_box, raw_text_box],
     )
 
-    # Wiring: build knowledge base (Firecrawl + embeddings)
+    # Wiring: build knowledge base (Firecrawl + lexical index)
     grab_kb_btn.click(
         fn=build_knowledge_base,
         inputs=[api_key_state, firecrawl_key_state, urls_box, raw_text_box, files_input],