scripts/_compute_new_event_ids.py

"""Diff disk-cached articles vs new train+test splits → list of event_ids needing 02 collection.

Used by the v0.3 dataset expansion (issue #57) to feed 02_collect_news --event-ids
without re-collecting events whose articles are already on disk from prior runs.

Usage (from project root):
    PYTHONPATH=. python scripts/_compute_new_event_ids.py
        > /tmp/new_event_ids.txt   # comma-separated single line
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent


def compute_new_event_ids(
    train_split: Path, test_split: Path, articles_dir: Path
) -> list[str]:
    """Return sorted list of event_ids in either split but not on disk."""
    split_ids: set[str] = set()
    for path in (train_split, test_split):
        if not path.exists():
            continue
        for ev in json.loads(path.read_text()):
            split_ids.add(ev["event_id"])

    cached: set[str] = set()
    if articles_dir.exists():
        for art in articles_dir.glob("*.json"):
            # Filename pattern: {event_id}_{md5_12}.json
            stem = art.stem
            cut = stem.rfind("_")
            if cut > 0:
                cached.add(stem[:cut])

    return sorted(split_ids - cached)


def main() -> int:
    train = ROOT / "data/splits/train_events.json"
    test = ROOT / "data/splits/test_events.json"
    articles = ROOT / "data/raw/articles"
    new_ids = compute_new_event_ids(train, test, articles)
    print(",".join(new_ids))
    return 0


if __name__ == "__main__":
    sys.exit(main())