File size: 4,069 Bytes
fbc0c1d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
import sys
import os
import json
import gc
from tqdm import tqdm
from sentence_transformers import SentenceTransformer

# Hack path agar bisa import modules/config dari root
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

from config import Config
from modules.domain_knowledge.vector_store import VectorStore
from modules.domain_knowledge.data_processor import DataProcessor

def run():
    print("="*40)
    print("🚀 STARTING DATA INGESTION PROCESS")
    print("="*40)

    # 1. Initialize Components
    processor = DataProcessor(
        Config.PATHS["categories"],
        Config.PATHS["supplier"],
        Config.PATHS["kbli"],
        Config.PATHS["supplier_kbli"]
    )
    
    vdb = VectorStore(Config.CHROMA_DB_DIR, Config.COLLECTION_NAME)

    print(f"[Model] Loading Embedding Model: {Config.EMBED_MODEL_NAME}...")
    model = SentenceTransformer(Config.EMBED_MODEL_NAME, device=Config.DEVICE)
    print(f"[Model] Loaded on {Config.DEVICE}")

    # 2. Count Total Rows (for progress bar)
    products_path = Config.PATHS["products"]
    if not os.path.exists(products_path):
        print(f"❌ Error: File not found {products_path}")
        return

    total_rows = sum(1 for line in open(products_path, 'r', encoding='utf-8') if line.strip())
    print(f"[Data] Total raw products to process: {total_rows}")

    # 3. Processing Loop
    batch_ids = []
    batch_texts = []
    batch_metas = []
    
    success_count = 0
    skipped_count = 0

    with open(products_path, 'r', encoding='utf-8') as f:
        # Gunakan TQDM untuk progress bar
        for line in tqdm(f, total=total_rows, desc="Ingesting", unit="item"):
            if not line.strip(): continue

            # A. Parse & Enrich
            try:
                raw_prod = json.loads(line)
                enriched_prod = processor.enrich_product(raw_prod)
                
                if not enriched_prod:
                    skipped_count += 1
                    continue # Skip jika supplier tidak ditemukan
                
                # B. Create Text & Metadata
                text_repr = processor.create_text_representation(enriched_prod)
                metadata = processor.prepare_metadata(enriched_prod)
                
                # Gunakan product_id sebagai ID vector
                pid = str(enriched_prod.get("product_id"))
                
                # C. Add to Buffer
                batch_ids.append(pid)
                batch_texts.append(text_repr)
                batch_metas.append(metadata)

            except json.JSONDecodeError:
                continue

            # D. Flush Batch to DB
            if len(batch_ids) >= Config.BATCH_SIZE_DB:
                _flush_batch(vdb, model, batch_ids, batch_texts, batch_metas)
                success_count += len(batch_ids)
                
                # Reset Buffer & Clean Memory
                batch_ids, batch_texts, batch_metas = [], [], []
                gc.collect()

        # E. Flush Sisa Data (Remaining Batch)
        if batch_ids:
            _flush_batch(vdb, model, batch_ids, batch_texts, batch_metas)
            success_count += len(batch_ids)

    print("\n" + "="*40)
    print("✅ INGESTION COMPLETED")
    print(f"Total Success Inserted : {success_count}")
    print(f"Total Skipped (No Data): {skipped_count}")
    print("="*40)

def _flush_batch(vdb, model, ids, texts, metas):
    """Helper function to encode and save"""
    # Generate Embeddings
    # Perintah 'query' (e5 model) biasanya butuh prefix 'query: ' atau 'passage: '
    # Untuk document storage di DB, E5 menyarankan prefix "passage: "
    processed_texts = [f"passage: {t}" for t in texts]
    
    embeddings = model.encode(
        processed_texts, 
        batch_size=Config.BATCH_SIZE_EMB, 
        convert_to_numpy=True, 
        show_progress_bar=False,
        normalize_embeddings=True # Penting untuk cosine similarity
    )
    
    # Save to Chroma
    vdb.add_documents(ids, texts, embeddings, metas)

if __name__ == "__main__":
    run()