Create build_vector_store.py

build_vector_store.py

@@ -1,44 +1,31 @@
-# ✅ build_vector_store.py
-# 將 web_data.txt 自動分段，嵌入向量並建立 FAISS 檢索庫
-
-import os
-import json
+# build_vector_store.py
 import faiss
 from sentence_transformers import SentenceTransformer
+import json
 
-# ✅ 參數設定
-TEXT_FILE = "web_data.txt"
-VECTOR_FILE = "faiss_index.faiss"
-DOCS_FILE = "docs.json"
-MODEL_NAME = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
-
-# ✅ 讀取並切段 web_data.txt
-print("📖 載入並分段 web_data.txt...")
-with open(TEXT_FILE, "r", encoding="utf-8") as f:
-    content = f.read()
+# 載入爬蟲資料
+with open("web_data.txt", "r", encoding="utf-8") as f:
+    documents = f.readlines()
 
-# 以空行為斷點分段（可根據實際資料再微調）
-docs = [chunk.strip() for chunk in content.split("\n\n") if chunk.strip()]
+# 去除空行與整理
+documents = [line.strip() for line in documents if line.strip()]
 
-# ✅ 載入嵌入模型
-print("🔤 載入嵌入模型...")
-model = SentenceTransformer(MODEL_NAME)
+# 初始化向量模型（中文支援佳）
+encoder = SentenceTransformer("shibing624/text2vec-base-chinese")
 
-print("🔍 轉換成嵌入向量...")
-embeddings = model.encode(docs, show_progress_bar=True)
+# 建立向量資料
+embeddings = encoder.encode(documents, show_progress_bar=True)
 
-# ✅ 建立 FAISS 索引
-print("🧠 建立 FAISS 索引...")
+# 建立 FAISS Index
 dimension = embeddings[0].shape[0]
 index = faiss.IndexFlatL2(dimension)
 index.add(embeddings)
 
-# ✅ 儲存向量與對應段落
-faiss.write_index(index, VECTOR_FILE)
-with open(DOCS_FILE, "w", encoding="utf-8") as f:
-    json.dump(docs, f, ensure_ascii=False, indent=2)
+# 儲存
+faiss.write_index(index, "vector_store.faiss")
+
+# 儲存原始文件
+with open("documents.json", "w", encoding="utf-8") as f:
+    json.dump(documents, f, ensure_ascii=False, indent=2)
 
-print("✅ 向量資料庫建立完成：")
-print(f" - 向量檔：{VECTOR_FILE}")
-print(f" - 文件對應檔：{DOCS_FILE}")
-print(f" - 總段落數：{len(docs)}")
+print("✅ 向量資料庫建立完成")