Upload E-Commerce Product Intelligence Dashboard

Dockerfile

@@ -7,6 +7,12 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 COPY . /app
 
+# Tạo data dir
+RUN mkdir -p /app/data
+
+# Script start: chạy scraper + uvicorn
+RUN echo '#!/bin/bash\nexport KAGGLE_API_TOKEN="$KAGGLE_API_TOKEN"\npython backend/scraper.py && uvicorn app:app --host 0.0.0.0 --port 8000' > /app/start.sh && chmod +x /app/start.sh
+
 EXPOSE 8000
 
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["/app/start.sh"]

app.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import pandas as pd
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
@@ -10,43 +11,56 @@ logger = logging.getLogger(__name__)
 
 app = FastAPI(title="E-Commerce Product Intelligence Platform")
 
-# ==================== Load data ====================
+# ==================== Load data (local parquet) ====================
+LOCAL_PARQUET_PATH = Path("data/ecommerce_products.parquet")
+
+
 def load_data():
-    """Load parquet."""
-    parquet_path = Path("data/ecommerce_products.parquet")
-    if not parquet_path.exists():
-        raise FileNotFoundError(f"Parquet not found: {parquet_path}")
-    return pd.read_parquet(parquet_path)
+    """Load parquet từ local."""
+    if not LOCAL_PARQUET_PATH.exists():
+        raise FileNotFoundError(f"Parquet not found: {LOCAL_PARQUET_PATH}")
+
+    logger.info(f"Loading parquet from: {LOCAL_PARQUET_PATH}")
+    return pd.read_parquet(LOCAL_PARQUET_PATH)
+
 
 # ==================== API Routes ====================
 @app.get("/")
 def root():
     return {"status": "E-Commerce Product Intelligence API is running"}
 
+
 @app.get("/data")
 def get_data():
     df = load_data()
     return df.head(200).to_dict("records")
 
+
 @app.get("/stats/categories")
 def stats_categories():
     df = load_data()
     return df["category"].value_counts().head(10).to_dict()
 
+
 @app.get("/stats/brands")
 def stats_brands():
     df = load_data()
     return df["brand"].value_counts().head(10).to_dict()
 
+
 @app.get("/stats/price")
 def stats_price():
     df = load_data()
-    return df.groupby("category")["price"].agg(["mean", "median", "min", "max", "count"]).reset_index().to_dict("records")
+    return df.groupby("category")["price"].agg(["mean", "median", "min", "max", "count"]).reset_index().to_dict(
+        "records")
+
 
 @app.get("/stats/rating")
 def stats_rating():
     df = load_data()
-    return df.groupby("category")["rating"].agg(["mean", "median", "min", "max", "count"]).reset_index().to_dict("records")
+    return df.groupby("category")["rating"].agg(["mean", "median", "min", "max", "count"]).reset_index().to_dict(
+        "records")
+
 
 @app.get("/insights")
 def insights():
@@ -59,22 +73,25 @@ def insights():
         "avg_rating": df["rating"].mean(),
     })
 
+
 @app.get("/search")
 def search(query: str):
     df = load_data()
     q = query.lower()
     mask = (
-        df["title"].str.contains(q, case=False, na=False) |
-        df["description"].str.contains(q, case=False, na=False)
+            df["title"].str.contains(q, case=False, na=False) |
+            df["description"].str.contains(q, case=False, na=False)
     )
     return df[mask].head(100).to_dict("records")
 
+
 @app.get("/recommend")
 def recommend(category: str):
     df = load_data()
     subset = df[df["category"] == category]
     return subset.sort_values("rating", ascending=False).head(10).to_dict("records")
 
+
 # ==================== Frontend ====================
 frontend_dir = Path("frontend")
 if frontend_dir.exists():

backend/scraper.py

@@ -1,9 +1,13 @@
 import os
-import json
-from pathlib import Path
+import logging
 import pandas as pd
+from pathlib import Path
+from kaggle.api.kaggle_api_extended import KaggleApi
 import shutil
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
 DATASET_SLUG = "anujsaha0123456789/e-commerce-product-intelligence-dataset"
 TEMP_DIR = Path("data/temp_kaggle")
 OUTPUT_PARQUET = Path("data/ecommerce_products.parquet")
@@ -12,32 +16,29 @@ os.makedirs("data", exist_ok=True)
 
 
 def setup_kaggle_api():
-    """Auth Kaggle API từ environment variable."""
-    from kaggle.api.kaggle_api_extended import KaggleApi
-
+    """Auth Kaggle API."""
     token = os.getenv("KAGGLE_API_TOKEN")
     if not token:
         raise ValueError("KAGGLE_API_TOKEN environment variable not set!")
 
     api = KaggleApi()
     api.api_token = token
-
     return api
 
 
 def download_dataset():
-    """Download full dataset."""
+    """Download dataset từ Kaggle."""
     api = setup_kaggle_api()
     TEMP_DIR.mkdir(parents=True, exist_ok=True)
 
-    print(f"Downloading dataset: {DATASET_SLUG}")
+    logger.info(f"Downloading dataset: {DATASET_SLUG}")
     api.dataset_download_files(DATASET_SLUG, path=str(TEMP_DIR), unzip=True)
-    print("Download complete.")
+    logger.info("Download complete.")
     return TEMP_DIR
 
 
 def find_csv_files(temp_dir: Path):
-    """Find all CSV files."""
+    """Tìm tất CSV files."""
     csv_files = list(temp_dir.glob("**/*.csv"))
     if not csv_files:
         raise FileNotFoundError("No CSV files found.")
@@ -45,23 +46,23 @@ def find_csv_files(temp_dir: Path):
 
 
 def load_and_concatenate(csv_files):
-    """Concatenate all CSVs."""
+    """Concatenate tất CSVs."""
     dfs = []
     for f in csv_files:
-        print(f"Loading: {f}")
+        logger.info(f"Loading: {f}")
         df = pd.read_csv(f)
         dfs.append(df)
     return pd.concat(dfs, ignore_index=True)
 
 
 def save_parquet(df: pd.DataFrame):
-    """Save to Parquet, overwrite."""
+    """Save to Parquet."""
     df.to_parquet(OUTPUT_PARQUET, index=False)
-    print(f"Saved to: {OUTPUT_PARQUET}")
+    logger.info(f"Saved to: {OUTPUT_PARQUET}")
 
 
 def run_scraper():
-    """Full pipeline."""
+    """Full pipeline: download Kaggle → save parquet."""
     try:
         download_dataset()
         csv_files = find_csv_files(TEMP_DIR)

requirements.txt

@@ -1,7 +1,4 @@
 fastapi==0.109.2
-uvicorn[standard]==0.27.1
-pandas==2.2.0
-kaggle==1.6.17
-pyarrow==18.0.0
-apscheduler==3.11.0
-httpx
+uvicorn
+pandas
+kaggle