Download modules/domain_knowledge/data_processor.py from recommender-system/llmproccrs: direct link, hf CLI and curl.
- Browser
- Download file 3.59 kB
-
https://huggingface.co/recommender-system/llmproccrs/resolve/main/modules/domain_knowledge/data_processor.py
- Command line
-
hf download hf://recommender-system/llmproccrs/modules/domain_knowledge/data_processor.py
-
curl -L -o data_processor.py https://huggingface.co/recommender-system/llmproccrs/resolve/main/modules/domain_knowledge/data_processor.py
3.59 kB
| import json | |
| class DataProcessor: | |
| def __init__(self, categories_path, supplier_path, kbli_path, supplier_kbli_path): | |
| """ | |
| Load semua data referensi (lookup tables) ke memori saat inisialisasi. | |
| """ | |
| print("[D&K] Loading reference data (Categories, Supplier, KBLI)...") | |
| self.categories = self._load_json(categories_path) | |
| self.suppliers = self._load_json(supplier_path) | |
| self.kbli = self._load_json(kbli_path) | |
| self.supplier_kbli = self._load_json(supplier_kbli_path) | |
| print("[D&K] Reference data loaded.") | |
| def _load_json(self, path): | |
| try: | |
| with open(path, "r", encoding="utf-8") as f: | |
| return json.load(f) | |
| except FileNotFoundError: | |
| print(f"[ERROR] File not found: {path}") | |
| return {} | |
| def enrich_product(self, product_raw): | |
| """ | |
| Menggabungkan data produk mentah dengan detail Category, Supplier, dan KBLI. | |
| """ | |
| enriched = product_raw.copy() | |
| # 1. Enrich Category | |
| sub_sub_id = product_raw.get("sub_sub_category_id") | |
| # Fallback jika ID tidak string/int yang sesuai | |
| enriched["category_detail"] = self.categories.get(str(sub_sub_id), {}) | |
| # 2. Enrich Supplier | |
| seller_name = product_raw.get("seller", "").strip() | |
| supplier_info = self.suppliers.get(seller_name) | |
| if not supplier_info: | |
| return None # Skip jika supplier tidak terdaftar (Clean Data) | |
| enriched["supplier_detail"] = supplier_info | |
| # 3. Enrich KBLI | |
| kbli_codes = self.supplier_kbli.get(seller_name, []) | |
| kbli_descriptions = [self.kbli.get(code, code) for code in kbli_codes if code in self.kbli] | |
| enriched["kbli_detail"] = kbli_descriptions | |
| return enriched | |
| def create_text_representation(self, prod): | |
| """ | |
| Membuat string deskriptif untuk di-embed. | |
| Format: "JenisProduk - NamaProduk - Kategori - Harga - Lokasi - Supplier" | |
| """ | |
| try: | |
| cat = prod.get("category_detail", {}) | |
| sup = prod.get("supplier_detail", {}) | |
| kbli_str = ", ".join(prod.get("kbli_detail", [])) | |
| # Mapping Product Type | |
| p_type_map = {"PHYSICAL": "Barang Fisik", "SERVICE": "Jasa", "DIGITAL": "Produk Digital"} | |
| p_type = p_type_map.get(cat.get("product_type"), "Produk") | |
| # Construct Text | |
| text = ( | |
| f"{p_type}: {prod.get('title', '')}. " | |
| f"Kategori: {cat.get('category', '')} > {cat.get('sub_category', '')}. " | |
| f"Harga: {prod.get('price', '')}. " | |
| f"Lokasi: {sup.get('kota', '')}, {sup.get('provinsi', '')}. " | |
| f"Supplier: {sup.get('nama_toko', prod.get('seller'))} ({sup.get('bentuk_usaha', '')}). " | |
| f"Spesialisasi KBLI: {kbli_str}." | |
| ) | |
| return text | |
| except Exception as e: | |
| return f"Produk: {prod.get('title', '')}" | |
| def prepare_metadata(self, prod): | |
| """ | |
| Menyiapkan metadata untuk disimpan di ChromaDB. | |
| ChromaDB preferensi metadatanya flat (str, int, float), bukan nested JSON. | |
| """ | |
| # Kita simpan JSON object sebagai string agar bisa diparsing balik nanti | |
| return { | |
| "product_id": str(prod.get("product_id")), | |
| "title": str(prod.get("title")), | |
| "price": str(prod.get("price")), | |
| "original_json": json.dumps(prod, ensure_ascii=False) # Simpan full object untuk retrieval | |
| } |