Download scripts/run_ingestion.py from recommender-system/llmproccrs: direct link, hf CLI and curl.
- Browser
- Download file 4.07 kB
-
https://huggingface.co/recommender-system/llmproccrs/resolve/main/scripts/run_ingestion.py
- Command line
-
hf download hf://recommender-system/llmproccrs/scripts/run_ingestion.py
-
curl -L -o run_ingestion.py https://huggingface.co/recommender-system/llmproccrs/resolve/main/scripts/run_ingestion.py
4.07 kB
| import sys | |
| import os | |
| import json | |
| import gc | |
| from tqdm import tqdm | |
| from sentence_transformers import SentenceTransformer | |
| # Hack path agar bisa import modules/config dari root | |
| sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) | |
| from config import Config | |
| from modules.domain_knowledge.vector_store import VectorStore | |
| from modules.domain_knowledge.data_processor import DataProcessor | |
| def run(): | |
| print("="*40) | |
| print("🚀 STARTING DATA INGESTION PROCESS") | |
| print("="*40) | |
| # 1. Initialize Components | |
| processor = DataProcessor( | |
| Config.PATHS["categories"], | |
| Config.PATHS["supplier"], | |
| Config.PATHS["kbli"], | |
| Config.PATHS["supplier_kbli"] | |
| ) | |
| vdb = VectorStore(Config.CHROMA_DB_DIR, Config.COLLECTION_NAME) | |
| print(f"[Model] Loading Embedding Model: {Config.EMBED_MODEL_NAME}...") | |
| model = SentenceTransformer(Config.EMBED_MODEL_NAME, device=Config.DEVICE) | |
| print(f"[Model] Loaded on {Config.DEVICE}") | |
| # 2. Count Total Rows (for progress bar) | |
| products_path = Config.PATHS["products"] | |
| if not os.path.exists(products_path): | |
| print(f"❌ Error: File not found {products_path}") | |
| return | |
| total_rows = sum(1 for line in open(products_path, 'r', encoding='utf-8') if line.strip()) | |
| print(f"[Data] Total raw products to process: {total_rows}") | |
| # 3. Processing Loop | |
| batch_ids = [] | |
| batch_texts = [] | |
| batch_metas = [] | |
| success_count = 0 | |
| skipped_count = 0 | |
| with open(products_path, 'r', encoding='utf-8') as f: | |
| # Gunakan TQDM untuk progress bar | |
| for line in tqdm(f, total=total_rows, desc="Ingesting", unit="item"): | |
| if not line.strip(): continue | |
| # A. Parse & Enrich | |
| try: | |
| raw_prod = json.loads(line) | |
| enriched_prod = processor.enrich_product(raw_prod) | |
| if not enriched_prod: | |
| skipped_count += 1 | |
| continue # Skip jika supplier tidak ditemukan | |
| # B. Create Text & Metadata | |
| text_repr = processor.create_text_representation(enriched_prod) | |
| metadata = processor.prepare_metadata(enriched_prod) | |
| # Gunakan product_id sebagai ID vector | |
| pid = str(enriched_prod.get("product_id")) | |
| # C. Add to Buffer | |
| batch_ids.append(pid) | |
| batch_texts.append(text_repr) | |
| batch_metas.append(metadata) | |
| except json.JSONDecodeError: | |
| continue | |
| # D. Flush Batch to DB | |
| if len(batch_ids) >= Config.BATCH_SIZE_DB: | |
| _flush_batch(vdb, model, batch_ids, batch_texts, batch_metas) | |
| success_count += len(batch_ids) | |
| # Reset Buffer & Clean Memory | |
| batch_ids, batch_texts, batch_metas = [], [], [] | |
| gc.collect() | |
| # E. Flush Sisa Data (Remaining Batch) | |
| if batch_ids: | |
| _flush_batch(vdb, model, batch_ids, batch_texts, batch_metas) | |
| success_count += len(batch_ids) | |
| print("\n" + "="*40) | |
| print("✅ INGESTION COMPLETED") | |
| print(f"Total Success Inserted : {success_count}") | |
| print(f"Total Skipped (No Data): {skipped_count}") | |
| print("="*40) | |
| def _flush_batch(vdb, model, ids, texts, metas): | |
| """Helper function to encode and save""" | |
| # Generate Embeddings | |
| # Perintah 'query' (e5 model) biasanya butuh prefix 'query: ' atau 'passage: ' | |
| # Untuk document storage di DB, E5 menyarankan prefix "passage: " | |
| processed_texts = [f"passage: {t}" for t in texts] | |
| embeddings = model.encode( | |
| processed_texts, | |
| batch_size=Config.BATCH_SIZE_EMB, | |
| convert_to_numpy=True, | |
| show_progress_bar=False, | |
| normalize_embeddings=True # Penting untuk cosine similarity | |
| ) | |
| # Save to Chroma | |
| vdb.add_documents(ids, texts, embeddings, metas) | |
| if __name__ == "__main__": | |
| run() |