Upload folder using huggingface_hub

agents/crew.py

@@ -258,6 +258,7 @@ def run_query_crew(query: str, top_k: int = None, max_tokens: int = None, use_ve
     unique_chunks = list(all_chunks.values())
     
     t_retrieval = time.time() - t0
+    print(f"[Retrieval] Vector chunks: {len(vec_results)}, Graph chunks: {len(graph_results)}, BM25 chunks: {len(bm25_results)}")
     print(f"[Retrieval Phase] Done in {t_retrieval:.2f}s — "
           f"{len(unique_chunks)} unique chunks retrieved from Vector DB, Graph DB, and BM25.")
 

app.py

@@ -640,6 +640,58 @@ def process_document_pipeline(path: str, orig_name: str, tier: str, token: str,
             except OSError as e:
                 log.warning("Failed to delete %s: %s", path, e)
 
+def sync_he_data():
+    try:
+        import os
+        from huggingface_hub import snapshot_download
+        import keyring
+        
+        token = os.environ.get("HF_TOKEN")
+        if not token:
+            try:
+                token = keyring.get_password('huggingface', 'my_user') or keyring.get_password('huggingface', 'hf_token')
+            except Exception:
+                pass
+        
+        kbdocs_dir = Path(__file__).parent / "kbdocs"
+        kbdocs_dir.mkdir(exist_ok=True)
+        
+        log.info("Syncing fresh files from Sam-max1/he_data to local kbdocs...")
+        snapshot_download(
+            repo_id="Sam-max1/he_data",
+            repo_type="dataset",
+            local_dir=str(kbdocs_dir),
+            token=token,
+            ignore_patterns=[".git*"]
+        )
+        log.info("Successfully synced he_data repository.")
+        
+        from pipeline import vector_store
+        
+        existing_docs = vector_store.list_documents("admin")
+        existing_names = {d["source"] for d in existing_docs}
+        
+        ingested_count = 0
+        for file_path in kbdocs_dir.glob("*.*"):
+            if file_path.is_file() and file_path.name not in existing_names:
+                log.info(f"Auto-ingesting new file: {file_path.name}")
+                process_document_pipeline(str(file_path), file_path.name, "foundation", "admin", delete_after=False)
+                ingested_count += 1
+                
+        # show number of chunks and nodes
+        log.info("=== Full Data Ingestion Complete ===")
+        log.info(f"Vector DB Chunks: {vector_store.count()}")
+        
+        from pipeline import graph_store
+        if graph_store.is_available():
+            stats = graph_store.get_stats()
+            log.info(f"Kuzu DB Nodes: {stats.get('nodes', 0)}, Edges: {stats.get('edges', 0)}")
+            
+    except Exception as e:
+        log.error("Failed to sync he_data: %s", e)
+
+threading.Thread(target=sync_he_data, daemon=True).start()
+
 
 @app.route("/api/ingest", methods=["POST"])
 def ingest():

templates/index.html

@@ -167,7 +167,7 @@
   <header class="banner" role="banner">
     <div>
       <h1>Healthcare Policy Expert</h1>
-      <div style="font-size: 0.75rem; font-weight: normal; margin-top: 4px; opacity: 0.9;">Agentic AI . RAG System . Edge Compute . Sandboxed</div>
+      <div style="font-size: 0.75rem; font-weight: normal; margin-top: 4px; opacity: 0.9;">Agentic AI | RAG System | Edge Compute | Sandboxed</div>
     </div>
     <div id="status-pill" class="pill not-ready">Not Ready</div>
   </header>
@@ -344,8 +344,8 @@
           const payload = {
             query: q,
             use_vector: true,
-            use_graph: false,
-            use_bm25: false,
+            use_graph: true,
+            use_bm25: true,
             top_k: 10,
             max_tokens: 2048,
             cpu_threads: 16