File size: 1,152 Bytes
76962bf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | import sqlite3
import os
from langchain_chroma import Chroma
from langchain_huggingface import HuggingFaceEmbeddings
from langchain_core.documents import Document
DB_PATH = "data/dietary_guidelines.db"
CHROMA_DIR = "data/chroma_db"
def migrate():
print("Connecting to sqlite DB...")
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
cursor.execute("SELECT id, source, page, content FROM guidelines")
rows = cursor.fetchall()
docs = []
for row in rows:
row_id, source, page, content = row
doc = Document(
page_content=content,
metadata={"source": source, "page": page, "id": row_id}
)
docs.append(doc)
print(f"Loaded {len(docs)} documents from SQLite. Creating embeddings...")
embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
print("Inserting into Chroma DB...")
vectorstore = Chroma.from_documents(
documents=docs,
embedding=embeddings,
persist_directory=CHROMA_DIR,
collection_name="guidelines"
)
print("Migration complete!")
if __name__ == "__main__":
migrate()
|