rebuilt index with new data

rebuild_index.py

@@ -5,35 +5,38 @@ from langchain_community.vectorstores import Chroma
 import shutil
 import os
 
-# Wipe old index
-if os.path.exists("./chroma_db"):
-    shutil.rmtree("./chroma_db")
-    print("Deleted old chroma_db/")
+CHROMA_DIR = "./chroma_db"
+DOCS_FILE = "./docs/fiscal_knowledge.md"
 
-# Load markdown
-loader = TextLoader("./docs/fiscal_knowledge.md", encoding="utf-8")
-docs = loader.load()
-text = docs[0].page_content
-print(f"Loaded {len(text):,} characters")
+# Only rebuild if chroma_db doesn't exist or docs changed
+if os.path.exists(CHROMA_DIR):
+    print("chroma_db/ exists, skipping rebuild. Delete chroma_db/ to force rebuild.")
+else:
+    print("Building new index...")
+    
+    loader = TextLoader(DOCS_FILE, encoding="utf-8")
+    docs = loader.load()
+    text = docs[0].page_content
+    print(f"Loaded {len(text):,} characters")
 
-splitter = MarkdownHeaderTextSplitter(
-    headers_to_split_on=[("##", "question")],
-    strip_headers=False,
-)
-chunks = splitter.split_text(text)
+    splitter = MarkdownHeaderTextSplitter(
+        headers_to_split_on=[("##", "question")],
+        strip_headers=False,
+    )
+    chunks = splitter.split_text(text)
+    print(f"Split into {len(chunks)} chunks")
 
-print(f"Split into {len(chunks)} chunks")
-print("Loading embedding model (first run downloads ~270MB)...")
-embeddings = HuggingFaceEmbeddings(
-    model_name="nomic-ai/nomic-embed-text-v1",
-    model_kwargs={"trust_remote_code": True},
-)
+    print("Loading embedding model...")
+    embeddings = HuggingFaceEmbeddings(
+        model_name="nomic-ai/nomic-embed-text-v1",
+        model_kwargs={"trust_remote_code": True},
+    )
 
-vectorstore = Chroma.from_documents(
-    chunks,
-    embedding=embeddings,
-    persist_directory="./chroma_db",
-)
+    vectorstore = Chroma.from_documents(
+        chunks,
+        embedding=embeddings,
+        persist_directory=CHROMA_DIR,
+    )
+    print(f"Indexed {len(chunks)} chunks into chroma_db/")
 
-print(f"Indexed {len(chunks)} chunks into chroma_db/")
-print("Done. Ready to deploy.")
+print("Ready.")