File size: 3,589 Bytes
17131ee | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | import json
class DataProcessor:
def __init__(self, categories_path, supplier_path, kbli_path, supplier_kbli_path):
"""
Load semua data referensi (lookup tables) ke memori saat inisialisasi.
"""
print("[D&K] Loading reference data (Categories, Supplier, KBLI)...")
self.categories = self._load_json(categories_path)
self.suppliers = self._load_json(supplier_path)
self.kbli = self._load_json(kbli_path)
self.supplier_kbli = self._load_json(supplier_kbli_path)
print("[D&K] Reference data loaded.")
def _load_json(self, path):
try:
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
except FileNotFoundError:
print(f"[ERROR] File not found: {path}")
return {}
def enrich_product(self, product_raw):
"""
Menggabungkan data produk mentah dengan detail Category, Supplier, dan KBLI.
"""
enriched = product_raw.copy()
# 1. Enrich Category
sub_sub_id = product_raw.get("sub_sub_category_id")
# Fallback jika ID tidak string/int yang sesuai
enriched["category_detail"] = self.categories.get(str(sub_sub_id), {})
# 2. Enrich Supplier
seller_name = product_raw.get("seller", "").strip()
supplier_info = self.suppliers.get(seller_name)
if not supplier_info:
return None # Skip jika supplier tidak terdaftar (Clean Data)
enriched["supplier_detail"] = supplier_info
# 3. Enrich KBLI
kbli_codes = self.supplier_kbli.get(seller_name, [])
kbli_descriptions = [self.kbli.get(code, code) for code in kbli_codes if code in self.kbli]
enriched["kbli_detail"] = kbli_descriptions
return enriched
def create_text_representation(self, prod):
"""
Membuat string deskriptif untuk di-embed.
Format: "JenisProduk - NamaProduk - Kategori - Harga - Lokasi - Supplier"
"""
try:
cat = prod.get("category_detail", {})
sup = prod.get("supplier_detail", {})
kbli_str = ", ".join(prod.get("kbli_detail", []))
# Mapping Product Type
p_type_map = {"PHYSICAL": "Barang Fisik", "SERVICE": "Jasa", "DIGITAL": "Produk Digital"}
p_type = p_type_map.get(cat.get("product_type"), "Produk")
# Construct Text
text = (
f"{p_type}: {prod.get('title', '')}. "
f"Kategori: {cat.get('category', '')} > {cat.get('sub_category', '')}. "
f"Harga: {prod.get('price', '')}. "
f"Lokasi: {sup.get('kota', '')}, {sup.get('provinsi', '')}. "
f"Supplier: {sup.get('nama_toko', prod.get('seller'))} ({sup.get('bentuk_usaha', '')}). "
f"Spesialisasi KBLI: {kbli_str}."
)
return text
except Exception as e:
return f"Produk: {prod.get('title', '')}"
def prepare_metadata(self, prod):
"""
Menyiapkan metadata untuk disimpan di ChromaDB.
ChromaDB preferensi metadatanya flat (str, int, float), bukan nested JSON.
"""
# Kita simpan JSON object sebagai string agar bisa diparsing balik nanti
return {
"product_id": str(prod.get("product_id")),
"title": str(prod.get("title")),
"price": str(prod.get("price")),
"original_json": json.dumps(prod, ensure_ascii=False) # Simpan full object untuk retrieval
} |