Create build_vector_store.py

build_vector_store.py

@@ -0,0 +1,44 @@
+# ✅ build_vector_store.py
+# 將 web_data.txt 自動分段，嵌入向量並建立 FAISS 檢索庫
+
+import os
+import json
+import faiss
+from sentence_transformers import SentenceTransformer
+
+# ✅ 參數設定
+TEXT_FILE = "web_data.txt"
+VECTOR_FILE = "faiss_index.faiss"
+DOCS_FILE = "docs.json"
+MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+
+# ✅ 讀取並切段 web_data.txt
+print("📖 載入並分段 web_data.txt...")
+with open(TEXT_FILE, "r", encoding="utf-8") as f:
+    content = f.read()
+
+# 以空行為斷點分段（可根據實際資料再微調）
+docs = [chunk.strip() for chunk in content.split("\n\n") if chunk.strip()]
+
+# ✅ 載入嵌入模型
+print("🔤 載入嵌入模型...")
+model = SentenceTransformer(MODEL_NAME)
+
+print("🔍 轉換成嵌入向量...")
+embeddings = model.encode(docs, show_progress_bar=True)
+
+# ✅ 建立 FAISS 索引
+print("🧠 建立 FAISS 索引...")
+dimension = embeddings[0].shape[0]
+index = faiss.IndexFlatL2(dimension)
+index.add(embeddings)
+
+# ✅ 儲存向量與對應段落
+faiss.write_index(index, VECTOR_FILE)
+with open(DOCS_FILE, "w", encoding="utf-8") as f:
+    json.dump(docs, f, ensure_ascii=False, indent=2)
+
+print("✅ 向量資料庫建立完成：")
+print(f" - 向量檔：{VECTOR_FILE}")
+print(f" - 文件對應檔：{DOCS_FILE}")
+print(f" - 總段落數：{len(docs)}")