Update app.py

app.py

@@ -57,25 +57,36 @@ UPLOADS_DIR.mkdir(parents=True, exist_ok=True)
 CONTENT_DIR.mkdir(exist_ok=True)
 init_db()
 
-_force_reseed = bool(os.getenv("FORCE_RESEED"))
+_force_reseed = os.getenv("FORCE_RESEED", "").lower() in (
+    "1", "true", "yes"
+)
+
 if _force_reseed and DB_PATH.exists():
     print("[startup] FORCE_RESEED=1 — deleting existing DB and reseeding")
     DB_PATH.unlink()
     init_db()
 
 _gc = get_captures
+
 if _force_reseed or len(_gc(limit=1)) == 0:
     try:
         import seed
         seed.run()
     except Exception as e:
         print(f"[seed] {e}")
+
 else:
     try:
         import sqlite3 as _sqlite3
+        import torch
+        import faiss
+        import numpy as np
 
         all_embs = get_all_embeddings()
 
+        # --------------------------------------------------
+        # Existing related_ids backfill
+        # --------------------------------------------------
         for cid, emb_row in all_embs:
             related = find_related(
                 emb_row,
@@ -86,7 +97,10 @@ else:
             with _sqlite3.connect(DB_PATH) as _conn:
                 _conn.execute(
                     "UPDATE captures SET related_ids=? WHERE id=?",
-                    (json.dumps(related), cid)
+                    (
+                        json.dumps(related),
+                        cid
+                    )
                 )
 
         print(
@@ -94,7 +108,9 @@ else:
             f"({len(all_embs)} captures)"
         )
 
-        # Export embeddings to .pt
+        # --------------------------------------------------
+        # Build knowledge.pt + knowledge.faiss
+        # --------------------------------------------------
         ids = []
         vectors = []
 
@@ -107,26 +123,98 @@ else:
             vectors.append(emb)
 
         if vectors:
-            pt_path = CONTENT_DIR / "knowledge.pt"
+
+            embeddings = torch.tensor(
+                vectors,
+                dtype=torch.float32
+            )
+
+            # ----------------------------------------------
+            # knowledge.pt
+            # ----------------------------------------------
+            pt_path = UPLOADS_DIR / "knowledge.pt"
 
             torch.save(
                 {
+                    "version": 1,
                     "embed_model": EMBED_MODEL,
-                    "ids": ids,
-                    "embeddings": torch.tensor(
-                        vectors,
-                        dtype=torch.float32
-                    ),
                     "count": len(ids),
+                    "ids": ids,
+                    "embeddings": embeddings,
                 },
                 pt_path,
             )
 
             print(
-                f"[startup] exported "
-                f"{len(ids)} embeddings -> {pt_path}"
+                f"[startup] knowledge.pt created "
+                f"({len(ids)} vectors)"
+            )
+
+            # ----------------------------------------------
+            # knowledge.faiss
+            # ----------------------------------------------
+            faiss_path = UPLOADS_DIR / "knowledge.faiss"
+
+            vectors_np = (
+                embeddings
+                .cpu()
+                .numpy()
+                .astype(np.float32)
+            )
+
+            # cosine similarity search
+            faiss.normalize_L2(vectors_np)
+
+            dimension = vectors_np.shape[1]
+
+            index = faiss.IndexFlatIP(dimension)
+
+            index.add(vectors_np)
+
+            faiss.write_index(
+                index,
+                str(faiss_path)
+            )
+
+            print(
+                f"[startup] knowledge.faiss created "
+                f"({index.ntotal} vectors)"
             )
 
+            # ----------------------------------------------
+            # Optional Hugging Face upload
+            # ----------------------------------------------
+            if os.getenv("HF_UPLOAD_EXPORTS", "").lower() in (
+                "1",
+                "true",
+                "yes",
+            ):
+                from huggingface_hub import upload_file
+
+                repo_id = os.getenv("HF_EXPORT_REPO")
+
+                if repo_id:
+                    upload_file(
+                        path_or_fileobj=str(pt_path),
+                        path_in_repo="knowledge.pt",
+                        repo_id=repo_id,
+                        repo_type="dataset",
+                        token=os.getenv("HF_TOKEN"),
+                    )
+
+                    upload_file(
+                        path_or_fileobj=str(faiss_path),
+                        path_in_repo="knowledge.faiss",
+                        repo_id=repo_id,
+                        repo_type="dataset",
+                        token=os.getenv("HF_TOKEN"),
+                    )
+
+                    print(
+                        f"[startup] uploaded exports "
+                        f"to {repo_id}"
+                    )
+
     except Exception as e:
         print(f"[startup backfill] {e}")