Create ingest/cluster.py

ingest/cluster.py

@@ -0,0 +1,33 @@
+from typing import List, Dict
+
+def build_embeddings(results: List[Dict]):
+    try:
+        from sentence_transformers import SentenceTransformer
+        model = SentenceTransformer("all-MiniLM-L6-v2")
+        texts = [r.get("snippet", "") for r in results]
+        return model.encode(texts, show_progress_bar=False)
+    except Exception:
+        return None
+
+
+def cluster_embeddings(results: List[Dict], embeddings):
+    if embeddings is None or len(results) == 0:
+        return {}
+
+    try:
+        import faiss
+        import numpy as np
+
+        dim = embeddings.shape[1]
+        index = faiss.IndexFlatL2(dim)
+        index.add(embeddings)
+
+        clusters = {}
+        for i, r in enumerate(results):
+            clusters.setdefault(r["source"], []).append({
+                "title": r["title"],
+                "index": i
+            })
+        return clusters
+    except Exception:
+        return {}