Create build_index_from_txt.py

build_index_from_txt.py

@@ -0,0 +1,36 @@
+import glob, pickle, pathlib, re, faiss, tiktoken, numpy as np
+from sentence_transformers import SentenceTransformer
+
+TXT_DIR = pathlib.Path("docs_txt")
+CHUNK = 512
+MODEL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+
+enc = tiktoken.get_encoding("cl100k_base")
+model = SentenceTransformer(MODEL)
+
+docs, vecs = [], []
+
+for p in TXT_DIR.glob("*.txt"):
+    text = p.read_text(encoding="utf-8", errors="ignore")
+    parts = re.split(r"\n\s*Статья.+?\n", text, flags=re.I) or text.split("\n\n")
+    for part in parts:
+        if not part.strip():
+            continue
+        buf = []
+        for line in part.splitlines():
+            buf.append(line)
+            if len(enc.encode(" ".join(buf))) > CHUNK:
+                chunk = " ".join(buf); buf.clear()
+                docs.append({"src": p.name, "text": chunk})
+                vecs.append(model.encode(chunk, normalize_embeddings=True))
+        if buf:
+            chunk = " ".join(buf)
+            docs.append({"src": p.name, "text": chunk})
+            vecs.append(model.encode(chunk, normalize_embeddings=True))
+
+print("Фрагментов:", len(docs))
+index = faiss.IndexFlatIP(model.get_sentence_embedding_dimension())
+index.add(np.stack(vecs).astype("float32"))
+faiss.write_index(index, "vectorstore.faiss")
+pickle.dump(docs, open("docs.pkl", "wb"))
+print("✓ Индекс готов — файлы vectorstore.faiss и docs.pkl созданы")