Spaces:
No application file
No application file
File size: 2,151 Bytes
82e2272 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | import os
import logging
import pandas as pd
from pathlib import Path
import shutil
# Set Kaggle env vars TRƯỚC khi import Kaggle
token = os.getenv("KAGGLE_API_TOKEN")
if token:
token_value = token.split('_')[1] if '_' in token else token
os.environ['KAGGLE_KEY'] = token_value
os.environ['KAGGLE_USERNAME'] = 'johnsontrann'
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
DATASET_SLUG = "anujsaha0123456789/e-commerce-product-intelligence-dataset"
TEMP_DIR = Path("data/temp_kaggle")
OUTPUT_CSV = Path("data/ecommerce_products.csv")
os.makedirs("data", exist_ok=True)
def setup_kaggle_api():
"""Auth Kaggle API."""
from kaggle.api.kaggle_api_extended import KaggleApi
api = KaggleApi()
api.authenticate()
return api
def download_dataset():
"""Download dataset từ Kaggle."""
api = setup_kaggle_api()
TEMP_DIR.mkdir(parents=True, exist_ok=True)
logger.info(f"Downloading dataset: {DATASET_SLUG}")
api.dataset_download_files(DATASET_SLUG, path=str(TEMP_DIR), unzip=True)
logger.info("Download complete.")
return TEMP_DIR
def find_csv_files(temp_dir: Path):
"""Tìm tất CSV files."""
csv_files = list(temp_dir.glob("**/*.csv"))
if not csv_files:
raise FileNotFoundError("No CSV files found.")
return csv_files
def load_and_concatenate(csv_files):
"""Concatenate tất CSVs."""
dfs = []
for f in csv_files:
logger.info(f"Loading: {f}")
df = pd.read_csv(f)
dfs.append(df)
return pd.concat(dfs, ignore_index=True)
def save_csv(df: pd.DataFrame):
"""Save to CSV."""
df.to_csv(OUTPUT_CSV, index=False)
logger.info(f"Saved to: {OUTPUT_CSV}")
def run_scraper():
"""Full pipeline: download Kaggle → save CSV."""
try:
download_dataset()
csv_files = find_csv_files(TEMP_DIR)
df = load_and_concatenate(csv_files)
save_csv(df)
finally:
shutil.rmtree(TEMP_DIR, ignore_errors=True)
return df
if __name__ == "__main__":
run_scraper() |