init project

app.py

@@ -27,6 +27,14 @@ async def ask_api(req: Request):
     answer = generate_answer(prompt)
     return {"answer": answer}
 
+@app.post("/rescan")
+async def rescan_api():
+    with open("data/raw_law.txt", "r", encoding="utf-8") as f:
+        text = f.read()
+    chunks = chunk_legal_text(text)
+    retriever.rescan_and_append(chunks, get_embedding)
+    return {"status": "Rescan & update thành công."}
+
 # Gradio UI
 iface = gr.Interface(
     fn=lambda q: generate_answer("\n\n".join(retriever.query(q, get_embedding)) + f"\n\nCâu hỏi: {q}\nTrả lời:"),

rag_core/embedder.py

@@ -10,13 +10,13 @@ def get_embedding(text: str, retries: int = 3):
             response = requests.post(
                 "https://vietcat-phobertnode.hf.space/embed",
                 json={"text": text},
-                timeout=30  # Tăng từ 10 lên 30 giây
+                timeout=30
             )
-            response.raise_for_status()  # Nếu không 200 -> raise exception
+            response.raise_for_status()
             return response.json()["embedding"]
         except requests.exceptions.RequestException as e:
             logging.warning(f"Lỗi embedding (lần {i+1}/{retries}): {e}")
             if i < retries - 1:
-                time.sleep(2)  # Đợi 2s rồi thử lại
+                time.sleep(2)
             else:
-                raise RuntimeError(f"Không thể lấy embedding sau {retries} lần thử.")
+                raise

rag_core/retriever.py

@@ -2,6 +2,7 @@ import faiss
 import numpy as np
 import os
 import pickle
+import logging
 from rag_core.utils import log_timed
 
 INDEX_PATH = "faiss_index/index.faiss"
@@ -19,17 +20,48 @@ class Retriever:
 
     @log_timed("xây FAISS index")
     def build(self, texts: list, embed_fn):
-        embeddings = [embed_fn(t) for t in texts]
+        embeddings = []
+        valid_texts = []
+        for i, t in enumerate(texts):
+            try:
+                emb = embed_fn(t)
+                embeddings.append(emb)
+                valid_texts.append(t)
+            except Exception as e:
+                logging.warning(f"❌ Lỗi embedding chunk {i}: {e}")
+        if not embeddings:
+            raise RuntimeError("Không có embedding nào thành công!")
         dim = len(embeddings[0])
         self.index = faiss.IndexFlatL2(dim)
         self.index.add(np.array(embeddings).astype("float32"))
         faiss.write_index(self.index, INDEX_PATH)
         with open(META_PATH, "wb") as f:
-            pickle.dump(texts, f)
-        self.texts = texts
+            pickle.dump(valid_texts, f)
+        self.texts = valid_texts
 
     @log_timed("truy vấn FAISS")
     def query(self, query_text, embed_fn, k=3):
         q_emb = np.array([embed_fn(query_text)]).astype("float32")
         D, I = self.index.search(q_emb, k)
-        return [self.texts[i] for i in I[0]]
+        return [self.texts[i] for i in I[0]]
+
+    @log_timed("bổ sung embedding bị thiếu")
+    def rescan_and_append(self, full_texts, embed_fn):
+        existing_set = set(self.texts)
+        new_texts = [t for t in full_texts if t not in existing_set]
+        if not new_texts:
+            logging.info("Không có chunk mới để thêm.")
+            return
+        new_embeddings = []
+        for i, t in enumerate(new_texts):
+            try:
+                emb = embed_fn(t)
+                new_embeddings.append(emb)
+                self.texts.append(t)
+            except Exception as e:
+                logging.warning(f"❌ Lỗi embedding chunk mới {i}: {e}")
+        if new_embeddings:
+            self.index.add(np.array(new_embeddings).astype("float32"))
+            faiss.write_index(self.index, INDEX_PATH)
+            with open(META_PATH, "wb") as f:
+                pickle.dump(self.texts, f)