"""Publish to HF Hub: the V dataset (deliverable) + a self-contained code+data snapshot the Colab notebook can pull with just an HF token (no GitHub needed). # after setting HF_TOKEN in .env: python scripts/publish_hf.py --dataset --code Creates: /mathcompose-verifier (dataset repo) — data/verifier/*.jsonl + card /mathcompose (model repo) — full code + data/verifier (excludes data/raw, runs, caches) The Colab notebook snapshot_downloads the model repo, `pip install -e .`, trains. """ import argparse import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from mathcompose.common.env import load_dotenv, first_env # noqa: E402 ROOT = Path(__file__).resolve().parents[1] DATASET_CARD = """--- license: mit task_categories: [text-generation] tags: [math, process-verification, ProcessBench, PRM800K, mathcompose] --- # mathcompose — Model V (process verifier) SFT data First-error-localization critiques for training a small generative math verifier. Each row: a problem + a step-indexed candidate solution (`prompt`) and a paragraph-by-paragraph critique ending in `\\boxed{{k}}` (`completion`), where `k` is the 0-based index of the first erroneous step (`-1` = all correct). - **Labels** are ground truth from **PRM800K** (OpenAI, MIT). - **Genuine-detection recipe:** **claude-opus-4-8** critiques each solution *blind* (never shown the answer); a critique is kept only if its predicted index matches the PRM800K gold (rejection sampling, ~77% keep-rate). So the critiques contain real error-finding reasoning, not rationalization. - **Deduped** against ProcessBench + MATH-500 (no eval contamination). - Naturally ~50/50 erroneous/all-correct (no rebalancing needed). Built by the mathcompose harness. Eval target: ProcessBench first-error F1. """ def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--dataset", action="store_true", help="publish the dataset repo") ap.add_argument("--code", action="store_true", help="publish the code+data snapshot repo") ap.add_argument("--user", default=None, help="HF username/org (default: from token)") ap.add_argument("--dataset-repo", default=None) ap.add_argument("--code-repo", default=None) ap.add_argument("--private", action="store_true") args = ap.parse_args() if not (args.dataset or args.code): args.dataset = args.code = True load_dotenv() token = first_env(["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HUGGINGFACE_TOKEN"]) if not token: print("No HF token — set HF_TOKEN in .env."); return 1 from huggingface_hub import HfApi api = HfApi(token=token) user = args.user or api.whoami()["name"] ds_repo = args.dataset_repo or f"{user}/mathcompose-verifier" code_repo = args.code_repo or f"{user}/mathcompose" if args.dataset: print(f"publishing dataset -> {ds_repo}") api.create_repo(ds_repo, repo_type="dataset", exist_ok=True, private=args.private) (ROOT / "data/verifier/README.md").write_text(DATASET_CARD) api.upload_folder( folder_path=str(ROOT / "data/verifier"), repo_id=ds_repo, repo_type="dataset", allow_patterns=["*.jsonl", "README.md"], ) print(f" https://huggingface.co/datasets/{ds_repo}") if args.code: print(f"publishing code+data snapshot -> {code_repo}") api.create_repo(code_repo, repo_type="model", exist_ok=True, private=args.private) api.upload_folder( folder_path=str(ROOT), repo_id=code_repo, repo_type="model", ignore_patterns=[ "data/raw/*", "**/__pycache__/*", "*.egg-info/*", "**/*.pyc", "runs/*", ".git/*", ".pytest_cache/*", "data/verifier/*_full.jsonl", ".env", ".env.*", ], ) print(f" https://huggingface.co/{code_repo}") print("\nColab: snapshot_download the model repo, `pip install -e .`, then train.") return 0 if __name__ == "__main__": raise SystemExit(main())