ingest.py

"""
Loads data/sample_docs.txt, chunks it, embeds with multilingual HuggingFace
model, and saves a FAISS index to faiss_index/.
"""

from pathlib import Path

from dotenv import load_dotenv
from langchain_community.document_loaders import TextLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_huggingface import HuggingFaceEmbeddings
from langchain_community.vectorstores import FAISS

load_dotenv()

EMBEDDING_MODEL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
SOURCE_FILE = Path("data/sample_docs.txt")
INDEX_DIR = Path("faiss_index")
CHUNK_SIZE = 500
CHUNK_OVERLAP = 100


def main():
    if not SOURCE_FILE.exists():
        raise FileNotFoundError(f"{SOURCE_FILE} not found.")

    print(f"Loading {SOURCE_FILE}...")
    loader = TextLoader(str(SOURCE_FILE), encoding="utf-8")
    docs = loader.load()

    splitter = RecursiveCharacterTextSplitter(
        chunk_size=CHUNK_SIZE,
        chunk_overlap=CHUNK_OVERLAP,
        separators=["\n\n", "\n", ".", "،", " ", ""],
    )
    chunks = splitter.split_documents(docs)
    print(f"Created {len(chunks)} chunks.")

    print("Embedding chunks (this may take a moment)...")
    embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
    vectorstore = FAISS.from_documents(chunks, embeddings)

    INDEX_DIR.mkdir(exist_ok=True)
    vectorstore.save_local(str(INDEX_DIR))
    print(f"FAISS index saved to {INDEX_DIR}/")


if __name__ == "__main__":
    main()