scripts/migrate_db.py

import sqlite3
import os
from langchain_chroma import Chroma
from langchain_huggingface import HuggingFaceEmbeddings
from langchain_core.documents import Document

DB_PATH = "data/dietary_guidelines.db"
CHROMA_DIR = "data/chroma_db"

def migrate():
    print("Connecting to sqlite DB...")
    conn = sqlite3.connect(DB_PATH)
    cursor = conn.cursor()
    cursor.execute("SELECT id, source, page, content FROM guidelines")
    rows = cursor.fetchall()
    
    docs = []
    for row in rows:
        row_id, source, page, content = row
        doc = Document(
            page_content=content,
            metadata={"source": source, "page": page, "id": row_id}
        )
        docs.append(doc)
        
    print(f"Loaded {len(docs)} documents from SQLite. Creating embeddings...")
    embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
    
    print("Inserting into Chroma DB...")
    vectorstore = Chroma.from_documents(
        documents=docs,
        embedding=embeddings,
        persist_directory=CHROMA_DIR,
        collection_name="guidelines"
    )
    print("Migration complete!")

if __name__ == "__main__":
    migrate()