Upload E-Commerce Product Intelligence Dashboard

app.py

@@ -1,66 +1,73 @@
 import logging
-import os
 import pandas as pd
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
 from fastapi.responses import HTMLResponse, JSONResponse
 from pathlib import Path
+from huggingface_hub import hf_hub_download
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 app = FastAPI(title="E-Commerce Product Intelligence Platform")
 
-# ==================== Load data (local CSV) ====================
+# ==================== HF Dataset Config ====================
+HF_DATASET_REPO = "Vincentran/ecommerce-dataset"
+HF_CSV_FILENAME = "ecommerce_products.csv"
 LOCAL_CSV_PATH = Path("data/ecommerce_products.csv")
 
-
+# ==================== Load data từ HF ====================
 def load_data():
-    """Load CSV từ local."""
+    """Load CSV từ HF Dataset (download nếu chưa có)."""
+    data_dir = Path("data")
+    data_dir.mkdir(parents=True, exist_ok=True)
+
     if not LOCAL_CSV_PATH.exists():
-        raise FileNotFoundError(f"CSV not found: {LOCAL_CSV_PATH}")
+        logger.info(f"Downloading CSV from HF Dataset: {HF_DATASET_REPO}")
+        try:
+            local_path = hf_hub_download(
+                repo_id=HF_DATASET_REPO,
+                filename=HF_CSV_FILENAME,
+                repo_type="dataset",
+                cache_dir=str(data_dir)
+            )
+            logger.info(f"Downloaded to: {local_path}")
+        except Exception as e:
+            logger.error(f"Failed to download CSV: {e}")
+            raise FileNotFoundError(f"CSV not found on HF: {e}")
 
     logger.info(f"Loading CSV from: {LOCAL_CSV_PATH}")
     return pd.read_csv(LOCAL_CSV_PATH)
 
-
 # ==================== API Routes ====================
 @app.get("/")
 def root():
     return {"status": "E-Commerce Product Intelligence API is running"}
 
-
 @app.get("/data")
 def get_data():
     df = load_data()
     return df.head(200).to_dict("records")
 
-
 @app.get("/stats/categories")
 def stats_categories():
     df = load_data()
     return df["category"].value_counts().head(10).to_dict()
 
-
 @app.get("/stats/brands")
 def stats_brands():
     df = load_data()
     return df["brand"].value_counts().head(10).to_dict()
 
-
 @app.get("/stats/price")
 def stats_price():
     df = load_data()
-    return df.groupby("category")["price"].agg(["mean", "median", "min", "max", "count"]).reset_index().to_dict(
-        "records")
-
+    return df.groupby("category")["price"].agg(["mean", "median", "min", "max", "count"]).reset_index().to_dict("records")
 
 @app.get("/stats/rating")
 def stats_rating():
     df = load_data()
-    return df.groupby("category")["rating"].agg(["mean", "median", "min", "max", "count"]).reset_index().to_dict(
-        "records")
-
+    return df.groupby("category")["rating"].agg(["mean", "median", "min", "max", "count"]).reset_index().to_dict("records")
 
 @app.get("/insights")
 def insights():
@@ -73,24 +80,32 @@ def insights():
         "avg_rating": df["rating"].mean(),
     })
 
-
 @app.get("/search")
 def search(query: str):
     df = load_data()
     q = query.lower()
     mask = (
-            df["title"].str.contains(q, case=False, na=False) |
-            df["description"].str.contains(q, case=False, na=False)
+        df["title"].str.contains(q, case=False, na=False) |
+        df["description"].str.contains(q, case=False, na=False)
     )
     return df[mask].head(100).to_dict("records")
 
-
 @app.get("/recommend")
 def recommend(category: str):
     df = load_data()
     subset = df[df["category"] == category]
     return subset.sort_values("rating", ascending=False).head(10).to_dict("records")
 
+# ==================== Scraper Trigger ====================
+@app.post("/run-scraper")
+def trigger_scraper():
+    """Trigger upload CSV lên HF Dataset."""
+    import subprocess
+    result = subprocess.run(["python", "backend/scraper.py"], capture_output=True, text=True)
+    if result.returncode == 0:
+        return {"status": "Scraper completed successfully", "output": result.stdout}
+    else:
+        return {"status": "Scraper failed", "error": result.stderr}
 
 # ==================== Frontend ====================
 frontend_dir = Path("frontend")

backend/scraper.py

@@ -3,6 +3,7 @@ import logging
 import pandas as pd
 from pathlib import Path
 import shutil
+from huggingface_hub import upload_file
 
 # Set Kaggle env vars TRƯỚC khi import Kaggle
 token = os.getenv("KAGGLE_API_TOKEN")
@@ -17,6 +18,8 @@ logger = logging.getLogger(__name__)
 DATASET_SLUG = "anujsaha0123456789/e-commerce-product-intelligence-dataset"
 TEMP_DIR = Path("data/temp_kaggle")
 OUTPUT_CSV = Path("data/ecommerce_products.csv")
+HF_DATASET_REPO = "Vincentran/ecommerce-dataset"
+HF_CSV_FILENAME = "ecommerce_products.csv"
 
 os.makedirs("data", exist_ok=True)
 
@@ -65,13 +68,29 @@ def save_csv(df: pd.DataFrame):
     logger.info(f"Saved to: {OUTPUT_CSV}")
 
 
+def upload_to_hf():
+    """Upload CSV lên HF Dataset."""
+    if not OUTPUT_CSV.exists():
+        raise FileNotFoundError(f"CSV not found: {OUTPUT_CSV}")
+
+    logger.info(f"Uploading CSV to HF Dataset: {HF_DATASET_REPO}")
+    upload_file(
+        path_or_fileobj=str(OUTPUT_CSV),
+        path_in_repo=HF_CSV_FILENAME,
+        repo_id=HF_DATASET_REPO,
+        repo_type="dataset"
+    )
+    logger.info("Upload completed successfully.")
+
+
 def run_scraper():
-    """Full pipeline: download Kaggle → save CSV."""
+    """Full pipeline: download Kaggle → save CSV → upload HF Dataset."""
     try:
         download_dataset()
         csv_files = find_csv_files(TEMP_DIR)
         df = load_and_concatenate(csv_files)
         save_csv(df)
+        upload_to_hf()
     finally:
         shutil.rmtree(TEMP_DIR, ignore_errors=True)
 

requirements.txt

@@ -2,4 +2,4 @@ fastapi==0.109.2
 uvicorn
 pandas
 kaggle
-pyarrow
+huggingface_hub