scripts/download_data.py

from __future__ import annotations
import shutil
import sys
import zipfile
from pathlib import Path
import click
sys.path.insert(0, str(Path(__file__).parent.parent))
from src.utils.logging import setup_logging, get_logger
log = get_logger(__name__)

@click.command()
@click.option('--out-dir', default='data/raw', show_default=True)
@click.option('--sample', default=None, type=int, help='Optional: row sample size')
def main(out_dir: str, sample: int | None) -> None:
    setup_logging()
    out = Path(out_dir)
    out.mkdir(parents=True, exist_ok=True)
    if shutil.which('kaggle') is None:
        log.warning('kaggle CLI not found. Install with `pip install kaggle` and place your kaggle.json in ~/.kaggle/. Skipping download - the loader will fall back to the synthetic generator.')
        return
    import subprocess
    log.info('Downloading wordsforthewise/lending-club from Kaggle...')
    subprocess.run(['kaggle', 'datasets', 'download', '-d', 'wordsforthewise/lending-club', '-p', str(out)], check=True)
    zip_path = next(out.glob('*.zip'))
    log.info(f'Extracting {zip_path}...')
    with zipfile.ZipFile(zip_path) as zf:
        zf.extractall(out)
    zip_path.unlink()
    log.info(f'Done. Files in {out}: {[p.name for p in out.iterdir()]}')
if __name__ == '__main__':
    main()