Upload E-Commerce Product Intelligence Dashboard

.gitignore

@@ -0,0 +1,4 @@
+.env
+data/
+__pycache__/
+*.pyc

Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.10
+
+WORKDIR /app
+
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . /app
+
+EXPOSE 8000
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]

app.py

@@ -0,0 +1,33 @@
+import logging
+from contextlib import asynccontextmanager
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import HTMLResponse
+from pathlib import Path
+from backend.api import app as api_app
+from backend.scheduler import init_scheduler, shutdown_scheduler
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    logger.info("Starting app...")
+    init_scheduler()
+    yield
+    logger.info("Shutting down app...")
+    shutdown_scheduler()
+
+app = FastAPI(title="E-Commerce Product Intelligence Platform", lifespan=lifespan)
+
+app.include_router(api_app)
+
+frontend_dir = Path("frontend")
+if frontend_dir.exists():
+    app.mount("/", StaticFiles(directory=str(frontend), html=True), name="frontend")
+else:
+    @app.get("/")
+    def frontend_placeholder():
+        return HTMLResponse(
+            content="<h1>E-Commerce Product Intelligence Dashboard</h1><p>Frontend placeholder.</p>"
+        )

backend/agent.py

@@ -0,0 +1,22 @@
+import pandas as pd
+
+def generate_insights(df: pd.DataFrame):
+    return {
+        "total_products": len(df),
+        "categories": df["category"].nunique(),
+        "brands": df["brand"].nunique(),
+        "avg_price": df["price"].mean(),
+        "avg_rating": df["rating"].mean(),
+    }
+
+def semantic_search(query: str, df: pd.DataFrame):
+    q = query.lower()
+    mask = (
+        df["title"].str.contains(q, case=False, na=False) |
+        df["description"].str.contains(q, case=False, na=False)
+    )
+    return df[mask].head(100)
+
+def recommend_by_category(df: pd.DataFrame, category: str):
+    subset = df[df["category"] == category]
+    return subset.sort_values("rating", ascending=False).head(10)

backend/api.py

@@ -0,0 +1,50 @@
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import JSONResponse
+from backend.services import load_data, get_top_categories, get_top_brands, get_price_stats, get_rating_stats
+from backend.agent import generate_insights, semantic_search, recommend_by_category
+
+app = FastAPI(title="E-Commerce Product Intelligence API")
+
+@app.get("/")
+def root():
+    return {"status": "E-Commerce Product Intelligence API is running"}
+
+@app.get("/data")
+def get_data():
+    df = load_data()
+    return df.head(200).to_dict("records")
+
+@app.get("/stats/categories")
+def stats_categories():
+    df = load_data()
+    return get_top_categories(df, n=10).to_dict()
+
+@app.get("/stats/brands")
+def stats_brands():
+    df = load_data()
+    return get_top_brands(df, n=10).to_dict()
+
+@app.get("/stats/price")
+def stats_price():
+    df = load_data()
+    return get_price_stats(df).to_dict("records")
+
+@app.get("/stats/rating")
+def stats_rating():
+    df = load_data()
+    return get_rating_stats(df).to_dict("records")
+
+@app.get("/insights")
+def insights():
+    df = load_data()
+    return JSONResponse(content=generate_insights(df))
+
+@app.get("/search")
+def search(query: str):
+    df = load_data()
+    return semantic_search(query, df).head(100).to_dict("records")
+
+@app.get("/recommend")
+def recommend(category: str):
+    df = load_data()
+    return recommend_by_category(df, category).to_dict("records")

backend/scheduler.py

@@ -0,0 +1,30 @@
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers.cron import CronTrigger
+from backend.scraper import run_scraper
+import logging
+
+logger = logging.getLogger("scheduler")
+
+scheduler = BackgroundScheduler()
+
+def scheduled_scraper_job():
+    logger.info("Running scheduled scraper job...")
+    try:
+        run_scraper()
+        logger.info("Scheduled scraper job completed.")
+    except Exception as e:
+        logger.error(f"Scheduled scraper job failed: {e}")
+
+def init_scheduler():
+    """Schedule scraper to run daily at 02:00 AM."""
+    scheduler.add_job(
+        scheduled_scraper_job,
+        CronTrigger(hour=2, minute=0),
+        id="daily_scraper",
+        replace_existing=True
+    )
+    logger.info("Scheduled scraper job added: daily at 02:00 AM")
+    scheduler.start()
+
+def shutdown_scheduler():
+    scheduler.shutdown()

backend/scraper.py

@@ -0,0 +1,65 @@
+import os
+import json
+import zipfile
+from pathlib import Path
+from kaggle.api.kaggle_api_extended import KaggleApi
+import pandas as pd
+import shutil
+
+DATASET_SLUG = "anujsaha0123456789/e-commerce-product-intelligence-dataset"
+TEMP_DIR = Path("data/temp_kaggle")
+OUTPUT_PARQUET = Path("data/ecommerce_products.parquet")
+
+os.makedirs("data", exist_ok=True)
+
+def setup_kaggle_api():
+    """Auth Kaggle API from environment or file."""
+    api = KaggleApi()
+    api.authenticate()
+    return api
+
+def download_dataset():
+    """Download full dataset."""
+    api = setup_kaggle_api()
+    TEMP_DIR.mkdir(parents=True, exist_ok=True)
+
+    print(f"Downloading dataset: {DATASET_SLUG}")
+    api.dataset_download_files(DATASET_SLUG, path=str(TEMP_DIR), unzip=True)
+    print("Download complete.")
+    return TEMP_DIR
+
+def find_csv_files(temp_dir: Path):
+    """Find all CSV files."""
+    csv_files = list(temp_dir.glob("**/*.csv"))
+    if not csv_files:
+        raise FileNotFoundError("No CSV files found.")
+    return csv_files
+
+def load_and_concatenate(csv_files):
+    """Concatenate all CSVs."""
+    dfs = []
+    for f in csv_files:
+        print(f"Loading: {f}")
+        df = pd.read_csv(f)
+        dfs.append(df)
+    return pd.concat(dfs, ignore_index=True)
+
+def save_parquet(df: pd.DataFrame):
+    """Save to Parquet, overwrite."""
+    df.to_parquet(OUTPUT_PARQUET, index=False)
+    print(f"Saved to: {OUTPUT_PARQUET}")
+
+def run_scraper():
+    """Full pipeline."""
+    try:
+        download_dataset()
+        csv_files = find_csv_files(TEMP_DIR)
+        df = load_and_concatenate(csv_files)
+        save_parquet(df)
+    finally:
+        shutil.rmtree(TEMP_DIR, ignore_errors=True)
+
+    return df
+
+if __name__ == "__main__":
+    run_scraper()

backend/services.py

@@ -0,0 +1,22 @@
+import pandas as pd
+from pathlib import Path
+
+PARQUET_PATH = Path("data/ecommerce_products.parquet")
+
+def load_data():
+    """Load parquet."""
+    if not PARQUET_PATH.exists():
+        raise FileNotFoundError(f"Parquet not found: {PARQUET_PATH}")
+    return pd.read_parquet(PARQUET_PATH)
+
+def get_top_categories(df: pd.DataFrame, n: int = 10):
+    return df["category"].value_counts().head(n)
+
+def get_top_brands(df: pd.DataFrame, n: int = 10):
+    return df["brand"].value_counts().head(n)
+
+def get_price_stats(df: pd.DataFrame):
+    return df.groupby("category")["price"].agg(["mean", "median", "min", "max", "count"]).reset_index()
+
+def get_rating_stats(df: pd.DataFrame):
+    return df.groupby("category")["rating"].agg(["mean", "median", "min", "max", "count"]).reset_index()

requirements.txt

@@ -0,0 +1,7 @@
+fastapi==0.115.0
+uvicorn==0.34.0
+pandas==2.2.0
+kaggle==1.6.17
+pyarrow==18.0.0
+apscheduler==3.11.0
+httpx