aitek230telu commited on
Commit
fbc0c1d
·
verified ·
1 Parent(s): 2055a21

Upload 3 files

Browse files
scripts/.ipynb_checkpoints/run_ingestion-checkpoint.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import sys
2
+ import os
3
+ import json
4
+ import gc
5
+ from tqdm import tqdm
6
+ from sentence_transformers import SentenceTransformer
7
+
8
+ # Hack path agar bisa import modules/config dari root
9
+ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
10
+
11
+ from config import Config
12
+ from modules.domain_knowledge.vector_store import VectorStore
13
+ from modules.domain_knowledge.data_processor import DataProcessor
14
+
15
+ def run():
16
+ print("="*40)
17
+ print("🚀 STARTING DATA INGESTION PROCESS")
18
+ print("="*40)
19
+
20
+ # 1. Initialize Components
21
+ processor = DataProcessor(
22
+ Config.PATHS["categories"],
23
+ Config.PATHS["supplier"],
24
+ Config.PATHS["kbli"],
25
+ Config.PATHS["supplier_kbli"]
26
+ )
27
+
28
+ vdb = VectorStore(Config.CHROMA_DB_DIR, Config.COLLECTION_NAME)
29
+
30
+ print(f"[Model] Loading Embedding Model: {Config.EMBED_MODEL_NAME}...")
31
+ model = SentenceTransformer(Config.EMBED_MODEL_NAME, device=Config.DEVICE)
32
+ print(f"[Model] Loaded on {Config.DEVICE}")
33
+
34
+ # 2. Count Total Rows (for progress bar)
35
+ products_path = Config.PATHS["products"]
36
+ if not os.path.exists(products_path):
37
+ print(f"❌ Error: File not found {products_path}")
38
+ return
39
+
40
+ total_rows = sum(1 for line in open(products_path, 'r', encoding='utf-8') if line.strip())
41
+ print(f"[Data] Total raw products to process: {total_rows}")
42
+
43
+ # 3. Processing Loop
44
+ batch_ids = []
45
+ batch_texts = []
46
+ batch_metas = []
47
+
48
+ success_count = 0
49
+ skipped_count = 0
50
+
51
+ with open(products_path, 'r', encoding='utf-8') as f:
52
+ # Gunakan TQDM untuk progress bar
53
+ for line in tqdm(f, total=total_rows, desc="Ingesting", unit="item"):
54
+ if not line.strip(): continue
55
+
56
+ # A. Parse & Enrich
57
+ try:
58
+ raw_prod = json.loads(line)
59
+ enriched_prod = processor.enrich_product(raw_prod)
60
+
61
+ if not enriched_prod:
62
+ skipped_count += 1
63
+ continue # Skip jika supplier tidak ditemukan
64
+
65
+ # B. Create Text & Metadata
66
+ text_repr = processor.create_text_representation(enriched_prod)
67
+ metadata = processor.prepare_metadata(enriched_prod)
68
+
69
+ # Gunakan product_id sebagai ID vector
70
+ pid = str(enriched_prod.get("product_id"))
71
+
72
+ # C. Add to Buffer
73
+ batch_ids.append(pid)
74
+ batch_texts.append(text_repr)
75
+ batch_metas.append(metadata)
76
+
77
+ except json.JSONDecodeError:
78
+ continue
79
+
80
+ # D. Flush Batch to DB
81
+ if len(batch_ids) >= Config.BATCH_SIZE_DB:
82
+ _flush_batch(vdb, model, batch_ids, batch_texts, batch_metas)
83
+ success_count += len(batch_ids)
84
+
85
+ # Reset Buffer & Clean Memory
86
+ batch_ids, batch_texts, batch_metas = [], [], []
87
+ gc.collect()
88
+
89
+ # E. Flush Sisa Data (Remaining Batch)
90
+ if batch_ids:
91
+ _flush_batch(vdb, model, batch_ids, batch_texts, batch_metas)
92
+ success_count += len(batch_ids)
93
+
94
+ print("\n" + "="*40)
95
+ print("✅ INGESTION COMPLETED")
96
+ print(f"Total Success Inserted : {success_count}")
97
+ print(f"Total Skipped (No Data): {skipped_count}")
98
+ print("="*40)
99
+
100
+ def _flush_batch(vdb, model, ids, texts, metas):
101
+ """Helper function to encode and save"""
102
+ # Generate Embeddings
103
+ # Perintah 'query' (e5 model) biasanya butuh prefix 'query: ' atau 'passage: '
104
+ # Untuk document storage di DB, E5 menyarankan prefix "passage: "
105
+ processed_texts = [f"passage: {t}" for t in texts]
106
+
107
+ embeddings = model.encode(
108
+ processed_texts,
109
+ batch_size=Config.BATCH_SIZE_EMB,
110
+ convert_to_numpy=True,
111
+ show_progress_bar=False,
112
+ normalize_embeddings=True # Penting untuk cosine similarity
113
+ )
114
+
115
+ # Save to Chroma
116
+ vdb.add_documents(ids, texts, embeddings, metas)
117
+
118
+ if __name__ == "__main__":
119
+ run()
scripts/__init__.py ADDED
File without changes
scripts/run_ingestion.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import sys
2
+ import os
3
+ import json
4
+ import gc
5
+ from tqdm import tqdm
6
+ from sentence_transformers import SentenceTransformer
7
+
8
+ # Hack path agar bisa import modules/config dari root
9
+ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
10
+
11
+ from config import Config
12
+ from modules.domain_knowledge.vector_store import VectorStore
13
+ from modules.domain_knowledge.data_processor import DataProcessor
14
+
15
+ def run():
16
+ print("="*40)
17
+ print("🚀 STARTING DATA INGESTION PROCESS")
18
+ print("="*40)
19
+
20
+ # 1. Initialize Components
21
+ processor = DataProcessor(
22
+ Config.PATHS["categories"],
23
+ Config.PATHS["supplier"],
24
+ Config.PATHS["kbli"],
25
+ Config.PATHS["supplier_kbli"]
26
+ )
27
+
28
+ vdb = VectorStore(Config.CHROMA_DB_DIR, Config.COLLECTION_NAME)
29
+
30
+ print(f"[Model] Loading Embedding Model: {Config.EMBED_MODEL_NAME}...")
31
+ model = SentenceTransformer(Config.EMBED_MODEL_NAME, device=Config.DEVICE)
32
+ print(f"[Model] Loaded on {Config.DEVICE}")
33
+
34
+ # 2. Count Total Rows (for progress bar)
35
+ products_path = Config.PATHS["products"]
36
+ if not os.path.exists(products_path):
37
+ print(f"❌ Error: File not found {products_path}")
38
+ return
39
+
40
+ total_rows = sum(1 for line in open(products_path, 'r', encoding='utf-8') if line.strip())
41
+ print(f"[Data] Total raw products to process: {total_rows}")
42
+
43
+ # 3. Processing Loop
44
+ batch_ids = []
45
+ batch_texts = []
46
+ batch_metas = []
47
+
48
+ success_count = 0
49
+ skipped_count = 0
50
+
51
+ with open(products_path, 'r', encoding='utf-8') as f:
52
+ # Gunakan TQDM untuk progress bar
53
+ for line in tqdm(f, total=total_rows, desc="Ingesting", unit="item"):
54
+ if not line.strip(): continue
55
+
56
+ # A. Parse & Enrich
57
+ try:
58
+ raw_prod = json.loads(line)
59
+ enriched_prod = processor.enrich_product(raw_prod)
60
+
61
+ if not enriched_prod:
62
+ skipped_count += 1
63
+ continue # Skip jika supplier tidak ditemukan
64
+
65
+ # B. Create Text & Metadata
66
+ text_repr = processor.create_text_representation(enriched_prod)
67
+ metadata = processor.prepare_metadata(enriched_prod)
68
+
69
+ # Gunakan product_id sebagai ID vector
70
+ pid = str(enriched_prod.get("product_id"))
71
+
72
+ # C. Add to Buffer
73
+ batch_ids.append(pid)
74
+ batch_texts.append(text_repr)
75
+ batch_metas.append(metadata)
76
+
77
+ except json.JSONDecodeError:
78
+ continue
79
+
80
+ # D. Flush Batch to DB
81
+ if len(batch_ids) >= Config.BATCH_SIZE_DB:
82
+ _flush_batch(vdb, model, batch_ids, batch_texts, batch_metas)
83
+ success_count += len(batch_ids)
84
+
85
+ # Reset Buffer & Clean Memory
86
+ batch_ids, batch_texts, batch_metas = [], [], []
87
+ gc.collect()
88
+
89
+ # E. Flush Sisa Data (Remaining Batch)
90
+ if batch_ids:
91
+ _flush_batch(vdb, model, batch_ids, batch_texts, batch_metas)
92
+ success_count += len(batch_ids)
93
+
94
+ print("\n" + "="*40)
95
+ print("✅ INGESTION COMPLETED")
96
+ print(f"Total Success Inserted : {success_count}")
97
+ print(f"Total Skipped (No Data): {skipped_count}")
98
+ print("="*40)
99
+
100
+ def _flush_batch(vdb, model, ids, texts, metas):
101
+ """Helper function to encode and save"""
102
+ # Generate Embeddings
103
+ # Perintah 'query' (e5 model) biasanya butuh prefix 'query: ' atau 'passage: '
104
+ # Untuk document storage di DB, E5 menyarankan prefix "passage: "
105
+ processed_texts = [f"passage: {t}" for t in texts]
106
+
107
+ embeddings = model.encode(
108
+ processed_texts,
109
+ batch_size=Config.BATCH_SIZE_EMB,
110
+ convert_to_numpy=True,
111
+ show_progress_bar=False,
112
+ normalize_embeddings=True # Penting untuk cosine similarity
113
+ )
114
+
115
+ # Save to Chroma
116
+ vdb.add_documents(ids, texts, embeddings, metas)
117
+
118
+ if __name__ == "__main__":
119
+ run()