Spaces:
Paused
Paused
| """Initialize RAG vector database with lesson content. | |
| Run this script once to populate the Pinecone vector database | |
| with content from data/rag_database/. | |
| """ | |
| import logging | |
| import os | |
| import sys | |
| from pathlib import Path | |
| from dotenv import load_dotenv | |
| from src.rag.markdown_parser import MarkdownParser | |
| from src.rag.pinecone_client import PineconeClient | |
| from src.rag.rag_ingestion import RAGIngestion | |
| from src.rag.sentence_transformer_client import SentenceTransformerClient | |
| logging.basicConfig(level=logging.INFO) | |
| logger = logging.getLogger(__name__) | |
| def main(): | |
| """Initialize RAG database with lesson content.""" | |
| load_dotenv() | |
| MODEL_NAME = os.getenv("EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2") | |
| EMBEDDING_DIM = int(os.getenv("EMBEDDING_DIMENSION", "384")) | |
| INDEX_NAME = os.getenv("PINECONE_INDEX", "arabic-teaching") | |
| PINECONE_CLOUD = os.getenv("PINECONE_CLOUD", "aws") | |
| PINECONE_REGION = os.getenv("PINECONE_REGION", "us-east-1") | |
| logger.info("🔵 Initializing RAG vector database...") | |
| logger.info(f"Loading embedding model ({MODEL_NAME})...") | |
| embedder = SentenceTransformerClient(model_name=MODEL_NAME, dimension=EMBEDDING_DIM) | |
| logger.info( | |
| f"Connecting to Pinecone (index: {INDEX_NAME}, {PINECONE_CLOUD}/{PINECONE_REGION})..." | |
| ) | |
| vector_db = PineconeClient( | |
| index_name=INDEX_NAME, | |
| dimension=EMBEDDING_DIM, | |
| cloud=PINECONE_CLOUD, | |
| region=PINECONE_REGION, | |
| ) | |
| if embedder.get_dimension() != vector_db.dimension: | |
| logger.error("❌ ERROR: Dimension mismatch detected!") | |
| logger.error(f" Embedder dimension: {embedder.get_dimension()}") | |
| logger.error(f" Vector DB dimension: {vector_db.dimension}") | |
| logger.error("Please ensure EMBEDDING_DIMENSION matches your Pinecone index configuration.") | |
| sys.exit(1) | |
| logger.info("Setting up ingestion pipeline...") | |
| parser = MarkdownParser() | |
| ingestion = RAGIngestion(parser, embedder, vector_db) | |
| data_dir = Path(__file__).parent.parent.parent / "data" / "rag_database" | |
| logger.info(f"Processing RAG database from: {data_dir}") | |
| if not data_dir.exists(): | |
| logger.error(f"Data directory not found: {data_dir}") | |
| sys.exit(1) | |
| result = ingestion.process_directory(data_dir, show_progress=True, batch_size=100) | |
| logger.info("✅ RAG initialization complete!") | |
| logger.info(f" Chunks parsed: {result['chunks_parsed']}") | |
| logger.info(f" Vectors created: {result['vectors_created']}") | |
| logger.info(f" Upserted to DB: {result.get('upserted_count', 0)}") | |
| if result.get("mismatch"): | |
| logger.warning("⚠️ Mismatch detected between vectors created and upserted") | |
| if __name__ == "__main__": | |
| main() | |