Upload E-Commerce Product Intelligence Dashboard (frontend + backend)

backend/scraper.py

@@ -1,82 +1,66 @@
-import os
-import logging
-import pandas as pd
-from pathlib import Path
-import shutil
+# backend/scraper.py
 
-# Set Kaggle env vars TRƯỚC khi import Kaggle
-token = os.getenv("KAGGLE_API_TOKEN")
-if token:
-    # Kaggle API v2:直接使用 access_token
-    os.environ['KAGGLE_API_TOKEN'] = token
-    os.environ['KAGGLE_USERNAME'] = 'johnsontrann'
+import os
+import json
+from kaggle.api.kaggle_api_extended import KaggleApi  # ❌ XÃI import này
 
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
 
-DATASET_SLUG = "anujsaha0123456789/e-commerce-product-intelligence-dataset"
-TEMP_DIR = Path("data/temp_kaggle")
-OUTPUT_CSV = Path("data/ecommerce_products.csv")
+def setup_kaggle_api():
+    """Setup Kaggle API."""
+    # Check if kaggle.json exists
+    kaggle_json_path = os.path.join(os.environ.get('HOME', '/root'), '.config/kaggle/kaggle.json')
 
-os.makedirs("data", exist_ok=True)
+    if not os.path.exists(kaggle_json_path):
+        print(f"⚠️  kaggle.json not found at {kaggle_json_path}")
+        print("⚠️  Kaggle API will not work. Upload kaggle.json to HF Space secrets.")
+        return None
 
+    # Setup kaggle
+    with open(kaggle_json_path) as f:
+        kaggle_credentials = json.load(f)
 
-def setup_kaggle_api():
-    """Auth Kaggle API."""
-    from kaggle.api.kaggle_api_extended import KaggleApi
+    os.environ['KAGGLE_USERNAME'] = kaggle_credentials['username']
+    os.environ['KAGGLE_KEY'] = kaggle_credentials['key']
 
-    api = KaggleApi()
-    api.authenticate()
-    return api
+    return KaggleApi()
 
 
 def download_dataset():
-    """Download dataset từ Kaggle."""
+    """Download dataset from Kaggle."""
     api = setup_kaggle_api()
-    TEMP_DIR.mkdir(parents=True, exist_ok=True)
 
-    logger.info(f"Downloading dataset: {DATASET_SLUG}")
-    api.dataset_download_files(DATASET_SLUG, path=str(TEMP_DIR), unzip=True)
-    logger.info("Download complete.")
-    return TEMP_DIR
+    if api is None:
+        print("❌ Kaggle API not available. Skipping download.")
+        return False
 
+    # Download dataset
+    dataset_name = "jackdaug/ecommerce-products-dataset"
+    os.makedirs("data", exist_ok=True)
 
-def find_csv_files(temp_dir: Path):
-    """Tìm tất CSV files."""
-    csv_files = list(temp_dir.glob("**/*.csv"))
-    if not csv_files:
-        raise FileNotFoundError("No CSV files found.")
-    return csv_files
+    print(f"📥 Downloading dataset: {dataset_name}")
+    api.dataset_download_files(dataset_name, path="data", unzip=True)
+    print("✅ Dataset downloaded!")
 
-
-def load_and_concatenate(csv_files):
-    """Concatenate tất CSVs."""
-    dfs = []
-    for f in csv_files:
-        logger.info(f"Loading: {f}")
-        df = pd.read_csv(f)
-        dfs.append(df)
-    return pd.concat(dfs, ignore_index=True)
-
-
-def save_csv(df: pd.DataFrame):
-    """Save to CSV."""
-    df.to_csv(OUTPUT_CSV, index=False)
-    logger.info(f"Saved to: {OUTPUT_CSV}")
+    return True
 
 
 def run_scraper():
-    """Full pipeline: download Kaggle → save CSV (không upload HF)."""
-    try:
-        download_dataset()
-        csv_files = find_csv_files(TEMP_DIR)
-        df = load_and_concatenate(csv_files)
-        save_csv(df)
-    finally:
-        shutil.rmtree(TEMP_DIR, ignore_errors=True)
+    """Run scraper."""
+    print("🚀 Running scraper...")
+
+    if download_dataset():
+        print("✅ Scraper completed successfully!")
+        return True
+    else:
+        print("❌ Scraper failed!")
+        return False
 
-    return df
 
+# ✅ XÓA HOÀN TOÀN:
+# if __name__ == "__main__":
+#     run_scraper()
 
+# ✅ Chỉ để:
 if __name__ == "__main__":
-    run_scraper()
+    print("📦 scraper.py imported (not running automatically)")
+    print("📦 Call run_scraper() manually or via API")