CI: sync project3-document-qa to Space

README.md

@@ -24,6 +24,8 @@ A RAG (Retrieval-Augmented Generation) application using:
 ```bash
 cd project3-document-qa
 pip install -r requirements.txt
+export CHROMA_PERSIST_DIR="project3-document-qa/chroma_store"  # optional; defaults internally
+python ../scripts/refresh_embeddings.py --persist-dir "$CHROMA_PERSIST_DIR" --reset
 python app.py
 # Then open http://localhost:7860
 ```
@@ -92,3 +94,55 @@ Notes:
 - First build may take a while to download models.
 - You can switch to a lighter LLM in `app.py` (e.g., `sshleifer/tiny-gpt2`) if needed.
 - For production, consider larger models via Inference Endpoints or OpenAI with an API key.
+
+## Embedding Refresh (dbt-integrated)
+
+The RAG layer can ingest curated documents produced by the dbt data platform (`document_index` mart).
+
+### Pre-requisites
+1. Run dbt to build the `document_index` view:
+   ```bash
+   cd data-platform/dbt
+   dbt seed && dbt run
+   ```
+2. Ensure the DuckDB file exists at `data-platform/dbt/warehouse/data.duckdb`.
+
+### Build / Rebuild Vector Store
+```bash
+export CHROMA_PERSIST_DIR="project3-document-qa/chroma_store"  # choose any directory
+python scripts/refresh_embeddings.py --persist-dir "$CHROMA_PERSIST_DIR" --reset
+```
+
+Flags:
+- `--reset` (optional): clears existing collection before loading
+- `--limit N` (optional): ingest only first N rows for quick tests
+- `--duckdb PATH`: override DuckDB file location
+- `--collection NAME`: change Chroma collection name (default: `documents`)
+
+### Launch App Using Persisted Embeddings
+```bash
+cd project3-document-qa
+export CHROMA_PERSIST_DIR="project3-document-qa/chroma_store"
+python app.py
+```
+
+At startup the app detects a persistent store and **skips adding sample documents**, using the dbt-derived corpus instead.
+
+### Updating Embeddings After dbt Changes
+If you modify source seeds or transformation logic:
+```bash
+cd data-platform/dbt
+dbt seed && dbt run
+cd ../../
+python scripts/refresh_embeddings.py --persist-dir "$CHROMA_PERSIST_DIR" --reset
+```
+
+### Common Issues
+| Symptom | Cause | Fix |
+|---------|-------|-----|
+| `DuckDB file not found` | dbt not executed | Run `dbt seed && dbt run` |
+| Empty collection after refresh | `document_index` view missing | Confirm model name and rerun dbt |
+| App still shows sample docs | Persist dir not set or empty | Export `CHROMA_PERSIST_DIR` and rebuild embeddings |
+| Duplicate ID errors | Re-running without `--reset` and changed IDs | Use `--reset` for full rebuild |
+
+---

app.py

@@ -11,6 +11,7 @@ from transformers import pipeline, AutoTokenizer
 from typing import List, Dict, Tuple
 import os
 from datetime import datetime
+import socket
 import logging
 
 # Configure logging
@@ -39,20 +40,26 @@ class DocumentQASystem:
         self.embedding_model = SentenceTransformer(embedding_model_name)
         
         # Initialize ChromaDB
-        logger.info("Initializing ChromaDB...")
-        self.chroma_client = chromadb.Client(Settings(
-            anonymized_telemetry=False,
-            allow_reset=True
-        ))
+        logger.info("Initializing ChromaDB (persistent if available)...")
+        # Resolve persistent directory: env or local folder next to this file
+        persist_dir_env = os.getenv("CHROMA_PERSIST_DIR")
+        persist_dir = persist_dir_env if persist_dir_env else os.path.join(os.path.dirname(__file__), "chroma_store")
+        use_persistent = os.path.isdir(persist_dir) and any(os.scandir(persist_dir))
+        if use_persistent:
+            logger.info("Using persistent directory: %s", persist_dir)
+            self.chroma_client = chromadb.PersistentClient(path=persist_dir, settings=Settings(anonymized_telemetry=False))
+        else:
+            logger.info("Persistent dir not found (%s); falling back to in-memory client", persist_dir)
+            self.chroma_client = chromadb.Client(Settings(anonymized_telemetry=False, allow_reset=True))
         
         # Create or get collection
         try:
+            self.collection = self.chroma_client.get_collection(collection_name)
+        except Exception:
             self.collection = self.chroma_client.create_collection(
                 name=collection_name,
                 metadata={"description": "Document knowledge base"}
             )
-        except Exception:
-            self.collection = self.chroma_client.get_collection(name=collection_name)
         
         # Initialize LLM
         logger.info("Loading language model: %s", llm_model_name)
@@ -367,8 +374,16 @@ def _preload_sample_documents_if_empty():
         logger.warning("Could not preload sample documents: %s", e)
 
 
-# Execute preload at import/startup
-_preload_sample_documents_if_empty()
+def _persistent_store_present() -> bool:
+    persist_dir_env = os.getenv("CHROMA_PERSIST_DIR")
+    persist_dir = persist_dir_env if persist_dir_env else os.path.join(os.path.dirname(__file__), "chroma_store")
+    return os.path.isdir(persist_dir) and any(os.scandir(persist_dir))
+
+# Execute preload only if persistent store not already populated
+if not _persistent_store_present():
+    _preload_sample_documents_if_empty()
+else:
+    logger.info("🔒 Persistent Chroma store detected; skipping sample preload.")
 
 
 # Gradio Interface Functions
@@ -724,9 +739,22 @@ if __name__ == "__main__":
     print("🛠️ Tech: ChromaDB + SentenceTransformers + HuggingFace")
     print("=" * 60)
     
+    # Dynamic port selection fallback
+    desired_port = int(os.getenv("GRADIO_SERVER_PORT", "7860"))
+    port = desired_port
+    for _ in range(10):  # try up to 10 sequential ports
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            try:
+                s.bind(("0.0.0.0", port))
+                s.close()
+                break  # port is free
+            except OSError:
+                port += 1
+    if port != desired_port:
+        print(f"⚠️ Port {desired_port} busy; using fallback port {port}")
     demo.launch(
         server_name="0.0.0.0",
-        server_port=7860,
+        server_port=port,
         share=False,
         show_error=True
     )

requirements.txt

@@ -12,6 +12,7 @@ sentence-transformers
 transformers>=4.35.0
 torch
 huggingface_hub<1.0
+duckdb
 
 # Data Processing
 pandas