build_index.py

"""
build_index.py
Run with: python build_index.py
"""
import sys
sys.path.insert(0, ".")

from src.ingestion.scientific_ingestor import ingest_corpus
from src.ingestion.scientific_chunker import chunk_corpus
from src.retrieval.embedder import Embedder
from src.retrieval.vector_store import ScientificChromaStore

papers = ingest_corpus("data/papers", "data/metadata.jsonl")
chunks = chunk_corpus(papers, strategy="section_aware")
embedder = Embedder("multilingual-e5-small")
embeddings = embedder.encode([c.text for c in chunks])
store = ScientificChromaStore()
store.index_chunks(chunks, embeddings)
print(f"Indexed {store.count()} chunks from {len(papers)} papers")