scripts/prepare_data.py

#!/usr/bin/env python3
"""
prepare_data.py
-----------------
ResearchPilot ships with a hand-curated corpus of 17 REAL arXiv papers on
low-resource / multilingual NLP (data/corpus.json), gathered directly from
arxiv.org so the Space works immediately with no download step and no
risk of a slow/failed dataset pull on first boot.

This script is provided so you (or a recruiter) can OPTIONALLY expand the
corpus using a real public Hugging Face dataset once the Space has
internet access (this sandbox used to build the project did not, per
docs/architecture.md).

It uses the `datasets` library to pull a slice of a public arXiv-abstracts
dataset from the Hugging Face Hub, filters it for low-resource/multilingual
NLP relevance via keyword matching, converts it into the same corpus.json
schema, and reports exactly how many real documents were added, their
source dataset name/version, and what preprocessing was applied -- per the
project's "no fabricated documents" requirement.

Usage:
    python scripts/prepare_data.py --limit 200 --out data/corpus_expanded.json
"""
import argparse
import json
import os
import re
import sys

KEYWORDS = [
    "low-resource", "low resource", "multilingual", "cross-lingual", "cross lingual",
    "under-resourced", "zero-shot translation", "few-shot", "code-switching",
    "language transfer", "endangered language",
]


def keyword_relevant(text: str) -> bool:
    t = text.lower()
    return any(k in t for k in KEYWORDS)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--dataset", default="gfissore/arxiv-abstracts-2021",
                         help="Hugging Face dataset repo id to pull real abstracts from.")
    parser.add_argument("--split", default="train")
    parser.add_argument("--limit", type=int, default=200,
                         help="Max raw rows to scan for keyword relevance (not the final doc count).")
    parser.add_argument("--max-docs", type=int, default=50, help="Cap on documents added.")
    parser.add_argument("--out", default=os.path.join(
        os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "corpus_expanded.json"))
    args = parser.parse_args()

    try:
        from datasets import load_dataset
    except ImportError:
        print("The `datasets` package is required. Install with: pip install datasets", file=sys.stderr)
        sys.exit(1)

    print(f"Loading {args.limit} rows from '{args.dataset}' split='{args.split}' (streaming)...")
    try:
        ds = load_dataset(args.dataset, split=args.split, streaming=True)
    except Exception as exc:  # noqa: BLE001
        print(f"Could not reach the Hugging Face Hub to load '{args.dataset}': {exc}", file=sys.stderr)
        print("This step requires internet access (works on a Hugging Face Space; may fail in an "
              "offline sandbox). The bundled data/corpus.json already contains 17 real papers and "
              "the app works fully without running this script.", file=sys.stderr)
        sys.exit(1)

    base_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "corpus.json")
    with open(base_path, "r", encoding="utf-8") as f:
        existing = json.load(f)
    existing_ids = {d["id"] for d in existing}

    added = []
    scanned = 0
    for row in ds:
        if scanned >= args.limit or len(added) >= args.max_docs:
            break
        scanned += 1
        title = row.get("title", "").strip()
        abstract = row.get("abstract", "").strip()
        arxiv_id = row.get("id", "") or row.get("arxiv_id", "")
        if not title or not abstract or not keyword_relevant(f"{title} {abstract}"):
            continue
        doc_id = re.sub(r"[^0-9A-Za-z.]", "", arxiv_id) or f"scraped-{len(added)}"
        if doc_id in existing_ids:
            continue
        added.append({
            "id": doc_id,
            "title": title,
            "authors": ", ".join(row.get("authors", [])) if isinstance(row.get("authors"), list) else str(row.get("authors", "")),
            "year": int(str(row.get("update_date", "0000"))[:4]) if row.get("update_date") else 0,
            "url": f"https://arxiv.org/abs/{doc_id}" if doc_id and doc_id[0].isdigit() else "",
            "topic": "auto-imported",
            "text": abstract,
        })

    combined = existing + added
    with open(args.out, "w", encoding="utf-8") as f:
        json.dump(combined, f, indent=2)

    print(f"Dataset: {args.dataset} (split={args.split})")
    print(f"Scanned {scanned} raw rows via streaming.")
    print(f"Preprocessing: kept rows with non-empty title/abstract matching low-resource/multilingual "
          f"keyword filter {KEYWORDS}; deduplicated against existing corpus ids.")
    print(f"Added {len(added)} new real documents (existing {len(existing)} preserved).")
    print(f"Wrote combined corpus of {len(combined)} documents to {args.out}")
    print("To use it, point RESEARCHPILOT_CORPUS_PATH or app.py's CORPUS_PATH at this file.")


if __name__ == "__main__":
    main()