scripts/download_data.sh

#!/usr/bin/env bash
# File Objective  : Fetch the HuffPost News Category dataset into ./data (gitignored).
# Scope           : Phase 2 data acquisition — reproducible, login only when needed.
# What it does    : 1. Skips download if file already present (idempotent).
#                   2. Tries $DATA_URL (no-auth direct download) first.
#                   3. Falls back to Kaggle CLI (uses ~/.kaggle/kaggle.json for auth).
#                   4. Prints clear manual instructions if both unavailable.
# What it does not: 1. Commit data to git (data/ is gitignored).
#                   2. Re-download if the output file already exists.
set -euo pipefail

DATA_DIR="data"
OUT="${DATA_DIR}/news_category.json"
KAGGLE_SLUG="rmisra/news-category-dataset"

mkdir -p "${DATA_DIR}"

# ── Early exit if already downloaded ──────────────────────────────────────────
if [[ -f "${OUT}" ]]; then
    echo "✅ Dataset already present: ${OUT} ($(wc -l < "${OUT}" | tr -d ' ') rows)"
    exit 0
fi

# ── Primary: direct URL (no auth) — set DATA_URL env var to enable ────────────
if [[ -n "${DATA_URL:-}" ]]; then
    echo "⬇️  Downloading from DATA_URL ..."
    curl -fSL "${DATA_URL}" -o "${OUT}"

# ── Fallback: Kaggle CLI (requires ~/.kaggle/kaggle.json) ─────────────────────
elif command -v kaggle >/dev/null 2>&1; then
    echo "⬇️  Downloading via Kaggle CLI ..."
    echo "    (login uses ~/.kaggle/kaggle.json — run 'kaggle auth' if not set up)"
    kaggle datasets download -d "${KAGGLE_SLUG}" -p "${DATA_DIR}" --unzip
    # Kaggle may extract as News_Category_Dataset_v3.json or similar — normalise
    extracted=$(find "${DATA_DIR}" -maxdepth 1 -name "*.json" ! -name "news_category.json" | head -1)
    if [[ -n "${extracted}" ]]; then
        mv "${extracted}" "${OUT}"
    fi

# ── No method available ────────────────────────────────────────────────────────
else
    echo "❌ Cannot download: no DATA_URL set and Kaggle CLI not found."
    echo ""
    echo "   Option A (no auth)  : export DATA_URL=<direct-link> && bash scripts/download_data.sh"
    echo "   Option B (Kaggle)   : pip install kaggle && kaggle auth && bash scripts/download_data.sh"
    echo "   Option C (manual)   : place the JSON file at ${OUT}"
    exit 1
fi

echo "✅ Done: ${OUT} ($(wc -l < "${OUT}" | tr -d ' ') rows)"