"""Command-line entry point for the KorByte pipeline.""" from __future__ import annotations import argparse import json from pathlib import Path from typing import Any from .benchmark import run_benchmark from .comparison import run_comparison from .config import DEFAULT_REPOSITORY_ID, DEFAULT_SEED, project_root from .corpus import prepare_corpus from .publish import authenticated_owner, publish_release from .report import render_reports from .train import train_tokenizer from .validate import validate_release def _print_json(value: Any) -> None: print(json.dumps(value, ensure_ascii=False, indent=2, default=str)) def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="korbyte", description="Build, benchmark, validate, and publish KorByte-128K" ) parser.add_argument( "--root", type=Path, default=project_root(), help="project root (defaults to the installed source tree)", ) subparsers = parser.add_subparsers(dest="command", required=True) prepare = subparsers.add_parser("prepare", help="prepare the deterministic corpus") prepare.add_argument("--scale", type=float, default=1.0) prepare.add_argument("--seed", type=int, default=DEFAULT_SEED) train = subparsers.add_parser("train", help="train and package the tokenizer") train.add_argument("--min-frequency", type=int, default=3) benchmark = subparsers.add_parser("benchmark", help="run held-out intrinsic benchmarks") benchmark.add_argument("--limit-per-domain", type=int, default=1_000) benchmark.add_argument("--repeats", type=int, default=3) benchmark.add_argument("--skip-morphological", action="store_true") compare = subparsers.add_parser("compare", help="run pinned public comparisons") compare.add_argument("--repeats", type=int, default=3) render = subparsers.add_parser("render", help="render the model card and reports") render.add_argument("--repo-id", default=DEFAULT_REPOSITORY_ID) validate = subparsers.add_parser("validate", help="validate release invariants") validate.add_argument("--no-require-benchmark", action="store_true") subparsers.add_parser("owner", help="print the authenticated Hugging Face username") publish = subparsers.add_parser("publish", help="publish and remotely verify the release") publish.add_argument("--repo-id") publish.add_argument("--allow-existing", action="store_true") return parser def main() -> None: args = _parser().parse_args() root = args.root.resolve() if args.command == "prepare": result = prepare_corpus(root, scale=args.scale, seed=args.seed) elif args.command == "train": result = train_tokenizer(root, min_frequency=args.min_frequency) elif args.command == "benchmark": result = run_benchmark( root, limit_per_domain=args.limit_per_domain, repeats=args.repeats, include_morphological=not args.skip_morphological, ) elif args.command == "compare": result = run_comparison(root, repeats=args.repeats) elif args.command == "render": result = render_reports(root, repository_id=args.repo_id) elif args.command == "validate": result = validate_release(root, require_benchmark_gate=not args.no_require_benchmark) elif args.command == "owner": result = {"owner": authenticated_owner()} elif args.command == "publish": result = publish_release( root, repo_id=args.repo_id, allow_existing=args.allow_existing, ) else: # pragma: no cover - argparse enforces subcommands raise AssertionError(args.command) _print_json(result) if __name__ == "__main__": main()