Upload E-Commerce Product Intelligence Dashboard

backend/scheduler.py

@@ -1,20 +1,24 @@
 from apscheduler.schedulers.background import BackgroundScheduler
 from apscheduler.triggers.cron import CronTrigger
-from backend.scraper import run_scraper
 import logging
 
 logger = logging.getLogger("scheduler")
 
 scheduler = BackgroundScheduler()
 
+
 def scheduled_scraper_job():
+    """Chạy scraper trong job."""
     logger.info("Running scheduled scraper job...")
     try:
+        # Import dynamic khi cần
+        from backend.scraper import run_scraper
         run_scraper()
         logger.info("Scheduled scraper job completed.")
     except Exception as e:
         logger.error(f"Scheduled scraper job failed: {e}")
 
+
 def init_scheduler():
     """Schedule scraper to run daily at 02:00 AM."""
     scheduler.add_job(
@@ -26,5 +30,6 @@ def init_scheduler():
     logger.info("Scheduled scraper job added: daily at 02:00 AM")
     scheduler.start()
 
+
 def shutdown_scheduler():
     scheduler.shutdown()

backend/scraper.py

@@ -1,23 +1,37 @@
 import os
 import json
-import zipfile
 from pathlib import Path
-from kaggle.api.kaggle_api_extended import KaggleApi
 import pandas as pd
 import shutil
 
+# Không import KaggleApi ở đây!
+# Sẽ import và authenticate khi cần
+
 DATASET_SLUG = "anujsaha0123456789/e-commerce-product-intelligence-dataset"
 TEMP_DIR = Path("data/temp_kaggle")
 OUTPUT_PARQUET = Path("data/ecommerce_products.parquet")
 
 os.makedirs("data", exist_ok=True)
 
+
 def setup_kaggle_api():
-    """Auth Kaggle API from environment or file."""
+    """Auth Kaggle API từ environment variable."""
+    from kaggle.api.kaggle_api_extended import KaggleApi
+
+    token = os.getenv("KAGGLE_API_TOKEN")
+    if not token:
+        raise ValueError("KAGGLE_API_TOKEN environment variable not set!")
+
     api = KaggleApi()
-    api.authenticate()
+
+    # Tự authenticate từ token
+    # Token format: KGAT_xxxxx
+    # Kaggle cần: username + key
+    api.api_token = token
+
     return api
 
+
 def download_dataset():
     """Download full dataset."""
     api = setup_kaggle_api()
@@ -28,6 +42,7 @@ def download_dataset():
     print("Download complete.")
     return TEMP_DIR
 
+
 def find_csv_files(temp_dir: Path):
     """Find all CSV files."""
     csv_files = list(temp_dir.glob("**/*.csv"))
@@ -35,6 +50,7 @@ def find_csv_files(temp_dir: Path):
         raise FileNotFoundError("No CSV files found.")
     return csv_files
 
+
 def load_and_concatenate(csv_files):
     """Concatenate all CSVs."""
     dfs = []
@@ -44,11 +60,13 @@ def load_and_concatenate(csv_files):
         dfs.append(df)
     return pd.concat(dfs, ignore_index=True)
 
+
 def save_parquet(df: pd.DataFrame):
     """Save to Parquet, overwrite."""
     df.to_parquet(OUTPUT_PARQUET, index=False)
     print(f"Saved to: {OUTPUT_PARQUET}")
 
+
 def run_scraper():
     """Full pipeline."""
     try:
@@ -61,5 +79,6 @@ def run_scraper():
 
     return df
 
+
 if __name__ == "__main__":
     run_scraper()