Download ocr-tech/src/ocrtech/cli.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 196 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/src/ocrtech/cli.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/src/ocrtech/cli.py
-
curl -L -o cli.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/src/ocrtech/cli.py
196 kB
"""Command-line interface for ocrtech."""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
from .ablation import audit_augmentation_ablation
from .admission import assert_recognizer_family_not_stopped, decide_recognizer_admission, review_recognizer_failures
from .benchmark import derive_table_postprocess_benchmark, rescore_benchmark_report, run_benchmark, summarize_benchmark_report
from .calibration import calibrate_quality_router, calibrate_script_router, calibrate_tesseract_psm_ensemble
from .claim_review import review_claim
from .capture import audit_limbu_capture, parse_crop_box, parse_perspective_quad, prepare_limbu_capture
from .crops import extract_crop_manifest
from .datasets import audit_manifest, audit_recognizer_corpus, build_recognizer_corpus, filter_manifest, merge_manifests, rebalance_manifest, split_manifest
from .error_analysis import analyze_benchmark_errors, export_failure_manifest
from .errors import OcrTechError
from .evalpack import audit_eval_pack, create_eval_pack
from .experiments import run_experiment
from .font_assets import audit_font_asset_readiness, prepare_font_assets
from .language_registry import audit_font_renderability, audit_language_readiness, audit_language_registry, audit_paddle_dictionary, inventory_fonts
from .limbu_legacy import convert_gorkhapatra_limbu_pdf_text, convert_limbu_legacy_text
from .limbu_runtime import preflight_limbu_pipeline_runtime
from .manifest import inspect_hf_dataset, load_manifest, normalize_manifest_images, prepare_hf_dataset, prepare_local_dataset
from .models import audit_model_card, write_model_card
from .oracle import build_benchmark_oracle
from .paper import export_paper_benchmark_table
from .pipeline import (
admit_limbu_post_correction_profile,
apply_limbu_review_corrections,
audit_limbu_correction_pair_pack,
audit_limbu_output,
audit_limbu_post_correction_profile,
build_limbu_correction_pair_pack,
derive_limbu_post_correction_profile,
parse_document,
parse_limbu_document,
score_limbu_post_correction_profile,
)
from .preflight import run_claim_preflight
from .public_benchmarks import audit_public_benchmark_manifest, list_public_benchmarks, prepare_public_benchmark
from .references import audit_references, materialize_references
from .remote import DEFAULT_PYTHON_CANDIDATES, audit_remote_host, bootstrap_remote_host, sync_remote_workspace
from .source_audit import (
DEFAULT_GORKHAPATRA_CATEGORY_URLS,
DEFAULT_GORKHAPATRA_EPAPER_URL,
DEFAULT_NAYA_NEPAL_PUBLICATION_URL,
apply_gorkhapatra_language_page_verification_bundle,
assign_gorkhapatra_language_page_verification_batches,
audit_gorkhapatra_language_page_verification_csv,
audit_gorkhapatra_language_pages,
audit_gorkhapatra_source,
extract_gorkhapatra_language_page_pdf_text,
audit_gorkhapatra_language_page_review,
finalize_gorkhapatra_language_page_review,
merge_gorkhapatra_language_page_verification_batches,
prepare_gorkhapatra_language_page_assisted_references,
prepare_gorkhapatra_language_page_ocr_sidecars,
prepare_gorkhapatra_language_page_pdf_native_references,
prepare_gorkhapatra_language_page_recognizer_eval,
prepare_gorkhapatra_language_page_reference_templates,
prepare_gorkhapatra_language_page_review,
prepare_gorkhapatra_language_page_reviewer_bundle,
prepare_gorkhapatra_language_page_transcription_work_order,
prepare_gorkhapatra_language_page_verification_bundle,
prepare_gorkhapatra_language_page_pack,
prepare_gorkhapatra_pack,
split_gorkhapatra_language_page_verification_csv,
)
from .synthesis_resources import (
audit_rendered_degradation_split,
audit_rendered_synthesis_lines,
audit_synthesis_text_manifest,
audit_synthesis_text_promotion,
audit_synthesis_resources,
prepare_bible_brain_text,
prepare_limbu_limdic_text,
prepare_limbu_unicode_text,
prepare_magar_text,
prepare_tamang_text,
prepare_toolkit_parallel_text,
render_synthesis_text_split,
render_synthesis_text_lines,
split_synthesis_text_manifest,
)
from .table_analysis import analyze_table_cells
from .training import (
PADDLE_DEVANAGARI_PPOCRV3_PRETRAINED,
PADDLE_PPOCRV5_MOBILE_REC_PRETRAINED,
bundle_recognizer_training,
finalize_hf_recognizer_run,
generate_correction_pairs,
package_hf_text_corrector_model,
package_hf_recognizer_model,
package_paddleocr_model,
write_corrector_recipe,
write_hf_text_corrector_recipe,
write_hf_recognizer_recipe,
write_recognizer_export_recipe,
write_recognizer_recipe,
)
from .validation import validate_claim
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="ocrtech", description="Nepali + English OCR document parser")
subparsers = parser.add_subparsers(dest="command", required=True)
prepare = subparsers.add_parser("prepare-data", help="download/verify/convert datasets into manifests")
prepare.add_argument("source", help="local CSV/JSONL/directory or Hugging Face dataset id")
prepare.add_argument("--out", default="data/manifests", help="output manifest directory")
prepare.add_argument("--dataset-name", help="manifest dataset name")
prepare.add_argument("--split", default="train", help="dataset split")
prepare.add_argument("--image-field", help="explicit image field")
prepare.add_argument("--text-field", help="explicit text field")
prepare.add_argument("--hf-subset", help="known HF dataset subset/language selector (for example: en)")
prepare.add_argument("--limit", type=int, help="max rows to convert")
prepare.add_argument("--hf", action="store_true", help="load source with Hugging Face datasets")
prepare.add_argument("--skip-invalid", action="store_true", help="skip invalid rows and write a reject report")
prepare.add_argument("--strict-chars", action="store_true", help="reject unsupported dictionary characters")
prepare.add_argument("--slice", action="append", default=[], help="slice label to stamp into manifest metadata; repeatable")
prepare.set_defaults(func=cmd_prepare_data)
inspect_hf = subparsers.add_parser("inspect-hf-dataset", help="inspect a Hugging Face dataset schema before manifest conversion")
inspect_hf.add_argument("dataset_id", help="Hugging Face dataset id")
inspect_hf.add_argument("--out", default="outputs/hf-dataset-inspection")
inspect_hf.add_argument("--split", default="train")
inspect_hf.add_argument("--hf-subset", help="dataset subset/language selector")
inspect_hf.add_argument("--image-field", help="explicit image field")
inspect_hf.add_argument("--text-field", help="explicit text field")
inspect_hf.add_argument("--limit", type=int, default=3)
inspect_hf.set_defaults(func=cmd_inspect_hf_dataset)
audit_manifest_parser = subparsers.add_parser("audit-manifest", help="audit a training or evaluation manifest")
audit_manifest_parser.add_argument("manifest")
audit_manifest_parser.add_argument("--out", default="outputs/manifest-audit")
audit_manifest_parser.add_argument("--no-verify-hashes", action="store_true")
audit_manifest_parser.add_argument("--require-slice", action="append", default=[])
audit_manifest_parser.add_argument("--strict-chars", action="store_true")
audit_manifest_parser.set_defaults(func=cmd_audit_manifest)
split = subparsers.add_parser("split-manifest", help="create deterministic leakage-safe train/eval manifests")
split.add_argument("manifest")
split.add_argument("--out", default="data/splits")
split.add_argument("--eval-ratio", type=float, default=0.15)
split.add_argument("--seed", type=int, default=13)
split.add_argument("--stratify-by", choices=["slices", "dataset", "document_type", "none"], default="slices")
split.add_argument("--group-by", choices=["sample_signature", "text_sha256", "image_sha256", "image_path", "sample_id"], default="sample_signature")
split.set_defaults(func=cmd_split_manifest)
rebalance = subparsers.add_parser("rebalance-manifest", help="upsample underrepresented slices in a manifest for training")
rebalance.add_argument("manifest")
rebalance.add_argument("--out", default="data/manifests/rebalanced.jsonl")
rebalance.add_argument("--slice", action="append", default=[], help="slice to rebalance; repeatable")
rebalance.add_argument("--target-count", type=int, help="target count per requested slice; defaults to the largest requested slice")
rebalance.add_argument("--seed", type=int, default=13)
rebalance.set_defaults(func=cmd_rebalance_manifest)
corpus_audit = subparsers.add_parser("audit-recognizer-corpus", help="audit train/eval manifests before recognizer training")
corpus_audit.add_argument("--train-manifest", required=True)
corpus_audit.add_argument("--eval-manifest", required=True)
corpus_audit.add_argument("--out", default="outputs/recognizer-corpus-audit")
corpus_audit.add_argument("--min-train-samples", type=int, default=1000)
corpus_audit.add_argument("--min-eval-samples", type=int, default=100)
corpus_audit.add_argument("--min-train-english", type=int, default=200)
corpus_audit.add_argument("--min-train-devanagari", type=int, default=200)
corpus_audit.add_argument("--min-train-mixed", type=int, default=25)
corpus_audit.add_argument("--min-eval-english", type=int, default=50)
corpus_audit.add_argument("--min-eval-devanagari", type=int, default=50)
corpus_audit.add_argument("--min-eval-mixed", type=int, default=10)
corpus_audit.add_argument("--min-train-latin-only", type=int, default=50)
corpus_audit.add_argument("--min-train-devanagari-only", type=int, default=50)
corpus_audit.add_argument("--min-eval-latin-only", type=int, default=10)
corpus_audit.add_argument("--min-eval-devanagari-only", type=int, default=10)
corpus_audit.add_argument("--min-train-real", type=int, default=100)
corpus_audit.add_argument("--min-train-synthetic", type=int, default=100)
corpus_audit.add_argument("--require-eval-real", action="store_true", help="fail if eval is not held-out real data or contains synthetic rows")
corpus_audit.set_defaults(func=cmd_audit_recognizer_corpus)
corpus_build = subparsers.add_parser("build-recognizer-corpus", help="assemble and upsample a recognizer training manifest by script buckets")
corpus_build.add_argument("manifests", nargs="+")
corpus_build.add_argument("--out", required=True)
corpus_build.add_argument("--target-latin-only", type=int, default=0)
corpus_build.add_argument("--target-devanagari-only", type=int, default=0)
corpus_build.add_argument("--target-mixed", type=int, default=0)
corpus_build.add_argument("--target-real", type=int, default=0)
corpus_build.add_argument("--target-synthetic", type=int, default=0)
corpus_build.add_argument("--seed", type=int, default=13)
corpus_build.set_defaults(func=cmd_build_recognizer_corpus)
merge = subparsers.add_parser("merge-manifests", help="combine multiple manifests into one auditable corpus")
merge.add_argument("manifests", nargs="+")
merge.add_argument("--out", default="data/manifests/combined.jsonl")
merge.set_defaults(func=cmd_merge_manifests)
filter_parser = subparsers.add_parser("filter-manifest", help="select a reproducible subset of a manifest")
filter_parser.add_argument("manifest")
filter_parser.add_argument("--out", default="data/manifests/filtered.jsonl")
filter_parser.add_argument("--sample-id", action="append", default=[], help="exact sample_id to include; repeatable")
filter_parser.add_argument("--slice", action="append", default=[], help="required slice label; repeatable")
filter_parser.add_argument("--document-type", action="append", default=[], help="required document_type metadata value; repeatable")
filter_parser.add_argument("--degradation", action="append", default=[], help="required degradation metadata value; repeatable")
filter_parser.add_argument("--limit", type=int)
filter_parser.set_defaults(func=cmd_filter_manifest)
refs = subparsers.add_parser("materialize-references", help="write benchmark reference files from a manifest and optionally rewrite manifest metadata")
refs.add_argument("manifest")
refs.add_argument("--out", default="data/references")
refs.add_argument("--rewritten-manifest", help="optional manifest output path with metadata.reference_path populated")
refs.set_defaults(func=cmd_materialize_references)
refs_audit = subparsers.add_parser("audit-references", help="audit benchmark reference files using the same parser as scoring")
refs_audit.add_argument("manifest")
refs_audit.add_argument("--out", default="outputs/reference-audit")
refs_audit.add_argument("--require-claim-ready", action="store_true")
refs_audit.set_defaults(func=cmd_audit_references)
normalize_images = subparsers.add_parser("normalize-manifest-images", help="copy/convert manifest images to OCR-supported suffixes")
normalize_images.add_argument("manifest")
normalize_images.add_argument("--out", default="data/normalized-images")
normalize_images.add_argument("--image-dir-name", default="images")
normalize_images.add_argument("--output-manifest-name", default="manifest-images-normalized.jsonl")
normalize_images.set_defaults(func=cmd_normalize_manifest_images)
create_eval = subparsers.add_parser("create-eval-pack", help="generate a held-out Nepali + English document eval pack")
create_eval.add_argument("--out", default="data/eval-pack", help="eval-pack output directory")
create_eval.add_argument("--count-per-template", type=int, default=1)
create_eval.add_argument("--input-format", choices=["text", "image"], default="text")
create_eval.add_argument("--degradation", action="append", help="degradation to generate; repeatable")
create_eval.add_argument("--seed", type=int, default=13)
create_eval.add_argument("--font-path", help="font path for image rendering")
create_eval.add_argument("--template", action="append", default=[], help="template name to generate; repeatable")
create_eval.add_argument("--variant-offset", type=int, default=0, help="starting copy index for generated template variants")
create_eval.set_defaults(func=cmd_create_eval_pack)
audit_eval = subparsers.add_parser("audit-eval-pack", help="audit eval-pack manifest integrity and claim readiness")
audit_eval.add_argument("manifest", help="eval-pack manifest JSONL")
audit_eval.add_argument("--out", default="outputs/eval-pack-audit")
audit_eval.add_argument("--require-claim-ready", action="store_true")
audit_eval.set_defaults(func=cmd_audit_eval_pack)
crops = subparsers.add_parser("extract-crops", help="extract line/cell/caption recognizer crops from rendered eval-pack layout metadata")
crops.add_argument("manifest", help="source eval-pack manifest JSONL")
crops.add_argument("--out", default="data/crops", help="crop manifest output directory")
crops.add_argument("--crop-type", action="append", default=[], choices=["line", "table-cell", "figure-caption"], help="crop type to extract; repeatable")
crops.add_argument("--split", default="train", help="split label for emitted crop manifest rows")
crops.add_argument("--dataset-name", default="ocrtech-crops")
crops.add_argument("--min-text-length", type=int, default=1)
crops.set_defaults(func=cmd_extract_crops)
train_rec = subparsers.add_parser("train-recognizer", help="write a PaddleOCR recognizer training recipe")
train_rec.add_argument("--train-manifest", required=True)
train_rec.add_argument("--eval-manifest", required=True)
train_rec.add_argument("--out", default="runs/recognizer")
train_rec.add_argument("--backend", choices=["paddleocr", "hf-vision-encoder-decoder"], default="paddleocr")
train_rec.add_argument("--base-model", default=PADDLE_PPOCRV5_MOBILE_REC_PRETRAINED)
train_rec.add_argument("--dictionary")
train_rec.add_argument("--max-target-length", type=int, default=128)
train_rec.add_argument("--train-batch-size", type=int, default=8)
train_rec.add_argument("--eval-batch-size", type=int, default=8)
train_rec.add_argument("--learning-rate", type=float, default=5e-5)
train_rec.add_argument("--epochs", type=float, default=3.0)
train_rec.add_argument("--warmup-epoch", type=float, default=5.0)
train_rec.add_argument("--use-gpu", action=argparse.BooleanOptionalAction, default=True)
train_rec.add_argument("--num-workers", type=int, default=2)
train_rec.add_argument("--train-drop-last", action=argparse.BooleanOptionalAction, default=True)
train_rec.add_argument("--main-indicator", choices=["acc", "norm_edit_dis"], default="acc")
train_rec.add_argument("--eval-batch-step", type=int, default=200)
train_rec.add_argument("--failure-review", help="recognizer-failure-review.json used to reject stopped model families")
train_rec.add_argument("--run", action="store_true", help="run PaddleOCR training command after writing config")
train_rec.set_defaults(func=cmd_train_recognizer)
export_rec = subparsers.add_parser("export-recognizer", help="write or run a PaddleOCR recognizer export recipe")
export_rec.add_argument("--training-config", required=True)
export_rec.add_argument("--checkpoint", required=True)
export_rec.add_argument("--out", default="runs/recognizer/export")
export_rec.add_argument("--inference-dir")
export_rec.add_argument("--paddleocr-dir", default="PaddleOCR")
export_rec.add_argument(
"--export-option",
action="append",
default=[],
help="additional PaddleOCR -o override such as Global.character_dict_path=/abs/dict.txt; repeatable",
)
export_rec.add_argument("--run", action="store_true")
export_rec.set_defaults(func=cmd_export_recognizer)
package_rec = subparsers.add_parser("package-recognizer", help="package PaddleOCR inference artifacts into a candidate model card")
package_rec.add_argument("--inference-dir", required=True)
package_rec.add_argument("--dictionary", required=True)
package_rec.add_argument("--out", default="runs/recognizer/package")
package_rec.add_argument("--model-id", required=True)
package_rec.add_argument("--base-model", default=PADDLE_DEVANAGARI_PPOCRV3_PRETRAINED)
package_rec.add_argument("--paddle-lang", default="ne")
package_rec.add_argument("--text-recognition-model-name", default="PP-OCRv5_mobile_rec")
package_rec.add_argument("--recognition-mode", choices=["full_page", "line", "auto", "recognition_only"], default="full_page")
package_rec.add_argument("--line-mode-max-height", type=int, default=256)
package_rec.add_argument("--line-mode-min-aspect-ratio", type=float, default=3.0)
package_rec.add_argument("--train-manifest")
package_rec.add_argument("--eval-manifest")
package_rec.add_argument("--training-summary")
package_rec.add_argument("--metrics-report")
package_rec.add_argument("--admission-validation-report")
package_rec.add_argument("--source-archive")
package_rec.add_argument("--source-checkpoint")
package_rec.add_argument("--source-training-config")
package_rec.add_argument("--export-recipe")
package_rec.set_defaults(func=cmd_package_recognizer)
bundle_rec = subparsers.add_parser("bundle-recognizer-training", help="package recognizer recipe, manifests, and images for transfer to a training host")
bundle_rec.add_argument("--recipe-dir", required=True)
bundle_rec.add_argument("--train-manifest", required=True)
bundle_rec.add_argument("--eval-manifest", required=True)
bundle_rec.add_argument("--out", default="runs/recognizer-training-bundle")
bundle_rec.add_argument("--archive-name", default="recognizer-training-bundle.tar.gz")
bundle_rec.add_argument("--base-dir", default=".")
bundle_rec.add_argument("--failure-review", help="recognizer-failure-review.json used to reject stopped model families before bundling")
bundle_rec.add_argument("--require-corpus-audit", action="store_true", help="fail unless train/eval manifests pass recognizer corpus coverage checks")
bundle_rec.add_argument("--min-train-samples", type=int, default=1000)
bundle_rec.add_argument("--min-eval-samples", type=int, default=100)
bundle_rec.add_argument("--min-train-english", type=int, default=200)
bundle_rec.add_argument("--min-train-devanagari", type=int, default=200)
bundle_rec.add_argument("--min-train-mixed", type=int, default=25)
bundle_rec.add_argument("--min-eval-english", type=int, default=50)
bundle_rec.add_argument("--min-eval-devanagari", type=int, default=50)
bundle_rec.add_argument("--min-eval-mixed", type=int, default=10)
bundle_rec.add_argument("--min-train-latin-only", type=int, default=50)
bundle_rec.add_argument("--min-train-devanagari-only", type=int, default=50)
bundle_rec.add_argument("--min-eval-latin-only", type=int, default=10)
bundle_rec.add_argument("--min-eval-devanagari-only", type=int, default=10)
bundle_rec.add_argument("--min-train-real", type=int, default=100)
bundle_rec.add_argument("--min-train-synthetic", type=int, default=100)
bundle_rec.add_argument("--require-eval-real", action="store_true", help="with --require-corpus-audit, require held-out real non-synthetic eval rows")
bundle_rec.set_defaults(func=cmd_bundle_recognizer_training)
package_hf = subparsers.add_parser("package-hf-recognizer", help="package a Hugging Face OCR model directory into a candidate model card")
package_hf.add_argument("--model-dir", required=True)
package_hf.add_argument("--out", default="runs/hf-recognizer/package")
package_hf.add_argument("--model-id", required=True)
package_hf.add_argument("--base-model", required=True)
package_hf.add_argument("--train-manifest")
package_hf.add_argument("--eval-manifest")
package_hf.add_argument("--training-summary")
package_hf.add_argument("--metrics-report")
package_hf.add_argument("--admission-validation-report")
package_hf.add_argument("--max-new-tokens", type=int, default=128)
package_hf.set_defaults(func=cmd_package_hf_recognizer)
finalize_hf = subparsers.add_parser("finalize-hf-recognizer-run", help="package and audit a completed Hugging Face recognizer training run")
finalize_hf.add_argument("--run-dir", required=True)
finalize_hf.add_argument("--out")
finalize_hf.add_argument("--model-id", required=True)
finalize_hf.add_argument("--base-model", required=True)
finalize_hf.add_argument("--train-manifest")
finalize_hf.add_argument("--eval-manifest")
finalize_hf.add_argument("--training-summary")
finalize_hf.add_argument("--admission-validation-report")
finalize_hf.add_argument("--max-new-tokens", type=int, default=128)
finalize_hf.set_defaults(func=cmd_finalize_hf_recognizer_run)
package_hf_corrector = subparsers.add_parser("package-hf-corrector", help="package a Hugging Face text corrector into a composite OCR candidate model card")
package_hf_corrector.add_argument("--model-dir", required=True)
package_hf_corrector.add_argument("--out", default="runs/hf-corrector/package")
package_hf_corrector.add_argument("--model-id", required=True)
package_hf_corrector.add_argument("--base-model", required=True)
package_hf_corrector.add_argument("--base-engine", default="tesseract")
package_hf_corrector.add_argument("--base-engine-kwarg", action="append", default=[], help="base OCR engine kwarg as key=value; repeatable")
package_hf_corrector.add_argument("--train-manifest")
package_hf_corrector.add_argument("--eval-manifest")
package_hf_corrector.add_argument("--metrics-report")
package_hf_corrector.add_argument("--max-new-tokens", type=int, default=256)
package_hf_corrector.set_defaults(func=cmd_package_hf_corrector)
pair_mining = subparsers.add_parser("generate-correction-pairs", help="run an OCR engine on a manifest and write noisy-clean correction pairs")
pair_mining.add_argument("--manifest", required=True)
pair_mining.add_argument("--out", required=True)
pair_mining.add_argument("--engine", default="tesseract", choices=["tesseract", "paddleocr", "surya", "candidate", "ours", "sidecar"])
pair_mining.add_argument("--model-config")
pair_mining.add_argument("--limit", type=int)
pair_mining.add_argument("--only-errors", action="store_true")
pair_mining.set_defaults(func=cmd_generate_correction_pairs)
train_corr = subparsers.add_parser("train-corrector", help="write an OCR correction training recipe")
train_corr.add_argument("--out", default="runs/corrector")
train_corr.add_argument("--clean-manifest", help="clean OCR manifest used to synthesize noisy pairs")
train_corr.add_argument("--pairs-source", help="JSONL with noisy_text and clean_text fields")
train_corr.add_argument("--hf-dataset", help="verified HF correction dataset id, for example cfilt/RoundTripOCR-nepali")
train_corr.add_argument("--hf-split", default="train")
train_corr.add_argument("--limit", type=int)
train_corr.add_argument("--seed", type=int, default=13)
train_corr.add_argument("--backend", choices=["recipe", "hf-seq2seq"], default="recipe")
train_corr.add_argument("--base-model", default="google/byt5-small")
train_corr.add_argument("--eval-ratio", type=float, default=0.1)
train_corr.add_argument("--max-source-length", type=int, default=256)
train_corr.add_argument("--max-target-length", type=int, default=256)
train_corr.add_argument("--train-batch-size", type=int, default=16)
train_corr.add_argument("--eval-batch-size", type=int, default=16)
train_corr.add_argument("--learning-rate", type=float, default=5e-5)
train_corr.add_argument("--epochs", type=float, default=3.0)
train_corr.add_argument("--run", action="store_true")
train_corr.set_defaults(func=cmd_train_corrector)
model_card = subparsers.add_parser("create-model-card", help="write a candidate model artifact card")
model_card.add_argument("--out", required=True)
model_card.add_argument("--model-id", required=True)
model_card.add_argument("--backend", choices=["paddleocr", "sidecar", "tesseract", "surya", "hf_vision_encoder_decoder", "text_correction_composite", "quality_select_composite", "quality_ranked_ensemble", "script_select_composite", "line_align_composite", "table_cell_refine_composite", "figure_caption_refine_composite"], default="paddleocr")
model_card.add_argument("--base-model", default=PADDLE_DEVANAGARI_PPOCRV3_PRETRAINED)
model_card.add_argument("--artifact", action="append", default=[], help="required model artifact path; repeatable")
model_card.add_argument("--backend-kwarg", action="append", default=[], help="backend kwarg as key=value; repeatable")
model_card.add_argument("--paddleocr-kwarg", action="append", default=[], help="deprecated alias for --backend-kwarg; repeatable")
model_card.add_argument("--train-manifest")
model_card.add_argument("--eval-manifest")
model_card.add_argument("--metrics-report")
model_card.add_argument("--admission-validation-report")
model_card.add_argument("--fallback-model-card", help="fallback component model-card path to record in provenance")
model_card.add_argument("--notes")
model_card.set_defaults(func=cmd_create_model_card)
model_audit = subparsers.add_parser("audit-model", help="audit a candidate model card and artifacts")
model_audit.add_argument("model_config")
model_audit.add_argument("--out", default="outputs/model-audit")
model_audit.set_defaults(func=cmd_audit_model)
parse = subparsers.add_parser("parse-doc", help="convert image/PDF/text sidecar to Markdown, JSON, tables, and figures")
parse.add_argument("input")
parse.add_argument("--out", default="outputs/document")
parse.add_argument("--engine", default="auto", choices=["auto", "sidecar", "tesseract", "paddleocr", "surya", "candidate", "ours"])
parse.add_argument("--model-config", help="candidate model card path when --engine candidate/ours")
parse.add_argument("--fallback-engine", choices=["sidecar", "tesseract", "paddleocr", "surya"])
parse.add_argument("--fallback-model-config", help="candidate model card path to use as a selective fallback engine")
parse.add_argument("--low-confidence-threshold", type=float, default=0.80)
parse.add_argument("--fallback-min-quality-score", type=float, help="trigger configured fallback when primary document quality score is below this value")
parse.set_defaults(func=cmd_parse_doc)
parse_limbu = subparsers.add_parser("parse-limbu-doc", help="parse a mixed Devanagari/Sirijonga Limbu document with review/audit artifacts")
parse_limbu.add_argument("input")
parse_limbu.add_argument("--out", default="outputs/limbu-document")
parse_limbu.add_argument("--engine", default="auto", choices=["auto", "sidecar", "tesseract", "paddleocr", "surya", "candidate", "ours"])
parse_limbu.add_argument("--model-config", help="candidate model card path when --engine candidate/ours")
parse_limbu.add_argument("--fallback-engine", choices=["sidecar", "tesseract", "paddleocr", "surya"])
parse_limbu.add_argument("--fallback-model-config", help="candidate model card path to use as a selective fallback engine")
parse_limbu.add_argument("--low-confidence-threshold", type=float, default=0.80)
parse_limbu.add_argument("--sirijonga-low-confidence-threshold", type=float, help="override review threshold for Sirijonga/Limbu-script lines")
parse_limbu.add_argument("--devanagari-low-confidence-threshold", type=float, help="override review threshold for Devanagari-written Limbu lines")
parse_limbu.add_argument("--mixed-script-low-confidence-threshold", type=float, help="override review threshold for mixed Sirijonga/Devanagari lines")
parse_limbu.add_argument("--other-script-low-confidence-threshold", type=float, help="override review threshold for lines without enough Limbu or Devanagari script")
parse_limbu.add_argument("--fallback-min-quality-score", type=float, help="trigger configured fallback when primary document quality score is below this value")
parse_limbu.add_argument("--script-ratio-threshold", type=float, default=0.20, help="minimum line-level codepoint ratio used to classify Devanagari or Sirijonga")
parse_limbu.add_argument("--capture-prep-metadata", help="limbu-capture-prep.json from prepare-limbu-capture to bind raw/prepared hashes into OCR output")
parse_limbu.add_argument("--post-correction-profile", help="guarded Limbu JSON correction profile to apply before review/audit export")
parse_limbu.add_argument("--line-detection-mode", default="engine", choices=["engine", "image-line"], help="use the OCR engine's native page detection or detect line crops from image pixels before recognition")
parse_limbu.add_argument("--image-line-threshold", default="otsu", help="image-line threshold: 'otsu' or an integer 0..255")
parse_limbu.add_argument("--image-line-bbox-source", default="ink", choices=["ink", "dilated"])
parse_limbu.add_argument(
"--image-line-reading-order",
default="auto_layout",
choices=[
"top_to_bottom",
"column_major",
"wide_column_right_to_left",
"wide_column_right_to_left_split_tall_last",
"auto_layout",
"auto_layout_split_tall_last",
],
)
parse_limbu.add_argument("--image-line-horizontal-kernel", type=int, default=23)
parse_limbu.add_argument("--image-line-vertical-kernel", type=int, default=3)
parse_limbu.add_argument("--image-line-dilation-iterations", type=int, default=1)
parse_limbu.add_argument("--image-line-min-width", type=int, default=35)
parse_limbu.add_argument("--image-line-min-height", type=int, default=10)
parse_limbu.add_argument("--image-line-min-area", type=int, default=100)
parse_limbu.add_argument("--image-line-max-height", type=int, default=140)
parse_limbu.add_argument("--image-line-min-aspect-ratio", type=float, default=0.0)
parse_limbu.add_argument("--image-line-max-aspect-ratio", type=float, default=0.0)
parse_limbu.add_argument("--image-line-detector-padding", type=int, default=2)
parse_limbu.add_argument("--image-line-crop-padding", type=int, default=12)
parse_limbu.add_argument("--image-line-split-tall-components", action="store_true")
parse_limbu.add_argument("--image-line-split-tall-row-min-ink", type=int, default=2)
parse_limbu.add_argument("--image-line-split-tall-max-row-gap", type=int, default=4)
parse_limbu.add_argument("--image-line-split-wide-components", action="store_true")
parse_limbu.add_argument("--image-line-split-wide-col-min-ink", type=int, default=2)
parse_limbu.add_argument("--image-line-split-wide-max-col-gap", type=int, default=24)
parse_limbu.add_argument("--image-line-split-wide-min-width", type=int, default=600)
parse_limbu.add_argument("--image-line-split-detected-row-components", action="store_true")
parse_limbu.add_argument("--image-line-split-detected-row-col-min-ink", type=int, default=2)
parse_limbu.add_argument("--image-line-split-detected-row-max-col-gap", type=int, default=24)
parse_limbu.add_argument("--image-line-split-detected-row-min-width", type=int, default=600)
parse_limbu.add_argument("--image-line-split-detected-row-min-segment-width", type=int, default=40)
parse_limbu.add_argument("--image-line-split-detected-tall-components", action="store_true")
parse_limbu.add_argument("--image-line-split-detected-tall-row-min-ink", type=int, default=20)
parse_limbu.add_argument("--image-line-split-detected-tall-max-row-gap", type=int, default=4)
parse_limbu.add_argument("--image-line-split-detected-tall-min-height", type=int, default=90)
parse_limbu.add_argument("--image-line-split-detected-tall-min-segment-height", type=int, default=24)
parse_limbu.add_argument("--image-line-merge-same-row-components", action="store_true")
parse_limbu.add_argument("--image-line-merge-same-row-y-tolerance", type=float, default=8.0)
parse_limbu.add_argument("--image-line-merge-same-row-max-gap", type=float, default=120.0)
parse_limbu.add_argument("--image-line-merge-same-row-max-center-delta", type=float, default=300.0)
parse_limbu.add_argument("--image-line-merge-same-row-max-width", type=float, default=420.0)
parse_limbu.add_argument("--image-line-merge-same-row-auto-fragmented-top-to-bottom", action="store_true")
parse_limbu.add_argument("--image-line-merge-same-row-auto-min-reduction-ratio", type=float, default=0.18)
parse_limbu.add_argument("--image-line-merge-same-row-auto-min-reduction-count", type=int, default=8)
parse_limbu.add_argument(
"--image-line-rescue-detector-pass",
action="append",
default=[],
help=(
"extra image-line detector pass as comma-separated key=value overrides; "
"repeatable, for example horizontal_kernel=15,vertical_kernel=1,min_width=20"
),
)
parse_limbu.add_argument("--image-line-merge-iou-threshold", type=float, default=0.80)
parse_limbu.add_argument("--image-line-allow-empty-lines", action="store_true")
parse_limbu.add_argument("--image-line-filter-drop-empty", action="store_true")
parse_limbu.add_argument("--image-line-filter-min-confidence", type=float)
parse_limbu.add_argument("--image-line-filter-require-script", default="any", choices=["any", "sirijonga", "devanagari", "limbu_or_devanagari"])
parse_limbu.add_argument("--image-line-filter-min-width-ratio", type=float)
parse_limbu.add_argument("--image-line-filter-max-width-ratio", type=float)
parse_limbu.add_argument("--image-line-filter-min-height-ratio", type=float)
parse_limbu.add_argument("--image-line-filter-max-height-ratio", type=float)
parse_limbu.set_defaults(func=cmd_parse_limbu_doc)
prep_limbu = subparsers.add_parser("prepare-limbu-capture", help="preserve and normalize an imperfect Limbu page capture before OCR")
prep_limbu.add_argument("input")
prep_limbu.add_argument("--out", default="outputs/limbu-capture-prep")
prep_limbu.add_argument("--rotate-degrees", type=float, default=0.0, help="manual deskew rotation in degrees; positive is counter-clockwise")
prep_limbu.add_argument("--crop-box", help="manual crop box as left,top,right,bottom after EXIF transpose")
prep_limbu.add_argument(
"--perspective-quad",
help=(
"manual page corners as top_left_x,top_left_y,top_right_x,top_right_y,"
"bottom_right_x,bottom_right_y,bottom_left_x,bottom_left_y after EXIF transpose/crop"
),
)
prep_limbu.add_argument("--auto-detect-page", action="store_true", help="detect a four-corner page contour and perspective-rectify it")
prep_limbu.add_argument("--auto-deskew", action="store_true", help="detect near-horizontal text skew and rotate the prepared image")
prep_limbu.add_argument("--max-auto-deskew-degrees", type=float, default=8.0, help="maximum absolute angle automatic deskew is allowed to apply")
prep_limbu.add_argument("--no-autocontrast", action="store_true")
prep_limbu.add_argument("--no-grayscale", action="store_true")
prep_limbu.set_defaults(func=cmd_prepare_limbu_capture)
audit_limbu_capture_parser = subparsers.add_parser("audit-limbu-capture", help="audit a prepared Limbu capture bundle for stale/missing raw and prepared images")
audit_limbu_capture_parser.add_argument("capture_metadata", help="limbu-capture-prep.json or its containing directory")
audit_limbu_capture_parser.add_argument("--out", help="audit report directory; defaults to the capture metadata directory")
audit_limbu_capture_parser.add_argument(
"--require-metadata-path-self",
action="store_true",
help="fail unless metadata_path points to the audited limbu-capture-prep.json file",
)
audit_limbu_capture_parser.add_argument("--min-prepared-width", type=int)
audit_limbu_capture_parser.add_argument("--min-prepared-height", type=int)
audit_limbu_capture_parser.add_argument("--min-prepared-entropy", type=float)
audit_limbu_capture_parser.add_argument("--min-prepared-luminance-stddev", type=float)
audit_limbu_capture_parser.add_argument("--min-prepared-edge-stddev", type=float)
audit_limbu_capture_parser.set_defaults(func=cmd_audit_limbu_capture)
limbu_preflight = subparsers.add_parser("preflight-limbu-pipeline", help="check runtime readiness for the packaged Limbu OCR pipeline")
limbu_preflight.add_argument("--model-config", default="outputs/limbu-first-router-v1/model-card.json")
limbu_preflight.add_argument("--out", default="outputs/limbu-first-router-v1/runtime-preflight")
limbu_preflight.add_argument("--required-script", default="limbu", help="required router script key, defaults to limbu")
limbu_preflight.add_argument(
"--required-tesseract-language",
action="append",
default=None,
help="required Tesseract language code; repeatable, defaults to nep and eng",
)
limbu_preflight.add_argument(
"--require-validated-components",
action="store_true",
help="fail if any recorded router component admission is experimental, pending, missing, or not validated/admitted",
)
limbu_preflight.add_argument(
"--required-component-role",
action="append",
default=None,
help="component admission role that must be present; repeatable, e.g. devanagari_primary and sirijonga_secondary",
)
limbu_preflight.set_defaults(func=cmd_preflight_limbu_pipeline)
limbu_apply_reviews = subparsers.add_parser("apply-limbu-review-corrections", help="apply accepted Limbu review-queue corrections to a parsed document")
limbu_apply_reviews.add_argument("--document", required=True, help="document.json emitted by parse-limbu-doc")
limbu_apply_reviews.add_argument("--review-queue", required=True, help="limbu-review-queue.tsv with accepted corrected_text rows")
limbu_apply_reviews.add_argument("--out", default="outputs/limbu-corrected-document")
limbu_apply_reviews.add_argument(
"--accepted-status",
action="append",
default=None,
help="review_status value to apply; repeatable, defaults to accepted/approved/corrected/verified/done",
)
limbu_apply_reviews.set_defaults(func=cmd_apply_limbu_review_corrections)
limbu_pair_pack = subparsers.add_parser(
"build-limbu-correction-pair-pack",
help="build a frozen train/heldout Limbu correction-pair pack from reviewed correction pairs",
)
limbu_pair_pack.add_argument(
"correction_pairs",
nargs="+",
help="limbu-correction-pairs.jsonl emitted by apply-limbu-review-corrections",
)
limbu_pair_pack.add_argument("--out", default="outputs/limbu-correction-pair-pack")
limbu_pair_pack.add_argument("--pack-id", required=True)
limbu_pair_pack.add_argument("--heldout-fraction", type=float, default=0.20)
limbu_pair_pack.add_argument("--min-heldout", type=int, default=1)
limbu_pair_pack.set_defaults(func=cmd_build_limbu_correction_pair_pack)
limbu_pair_pack_audit = subparsers.add_parser(
"audit-limbu-correction-pair-pack",
help="audit a frozen Limbu correction-pair pack for stale or tampered artifacts",
)
limbu_pair_pack_audit.add_argument("pack_dir")
limbu_pair_pack_audit.add_argument("--out", help="audit report directory; defaults to the pack directory")
limbu_pair_pack_audit.set_defaults(func=cmd_audit_limbu_correction_pair_pack)
limbu_derive_post_correction = subparsers.add_parser(
"derive-limbu-post-correction-profile",
help="derive an experimental deterministic Limbu post-correction profile from reviewed correction pairs",
)
limbu_derive_post_correction.add_argument(
"correction_pairs",
nargs="+",
help="limbu-correction-pairs.jsonl emitted by apply-limbu-review-corrections",
)
limbu_derive_post_correction.add_argument("--out", default="outputs/limbu-post-correction-profile-derived")
limbu_derive_post_correction.add_argument("--profile-id", default="limbu-derived-post-correction-profile-v1")
limbu_derive_post_correction.add_argument("--min-support", type=int, default=1)
limbu_derive_post_correction.set_defaults(func=cmd_derive_limbu_post_correction_profile)
limbu_score_post_correction = subparsers.add_parser(
"score-limbu-post-correction-profile",
help="score a Limbu post-correction profile on held-out correction pairs",
)
limbu_score_post_correction.add_argument("profile", help="limbu-post-correction-profile.json")
limbu_score_post_correction.add_argument(
"correction_pairs",
nargs="+",
help="held-out Limbu correction-pairs JSONL files",
)
limbu_score_post_correction.add_argument("--out", default="outputs/limbu-post-correction-profile-eval")
limbu_score_post_correction.add_argument("--frozen-eval-pack", help="identifier/path for the frozen held-out correction-pair pack")
limbu_score_post_correction.set_defaults(func=cmd_score_limbu_post_correction_profile)
limbu_admit_post_correction = subparsers.add_parser(
"admit-limbu-post-correction-profile",
help="write a validated Limbu post-correction profile from a passing held-out eval JSON",
)
limbu_admit_post_correction.add_argument("profile", help="source limbu-post-correction-profile.json")
limbu_admit_post_correction.add_argument("eval_run", help="limbu-post-correction-profile-eval.json from score-limbu-post-correction-profile")
limbu_admit_post_correction.add_argument("--out", default="outputs/limbu-post-correction-profile-admission")
limbu_admit_post_correction.add_argument("--output-filename", default="limbu-post-correction-profile-admitted.json")
limbu_admit_post_correction.set_defaults(func=cmd_admit_limbu_post_correction_profile)
limbu_audit_post_correction = subparsers.add_parser(
"audit-limbu-post-correction-profile",
help="audit a deterministic Limbu post-correction profile and its admission evidence",
)
limbu_audit_post_correction.add_argument("profile")
limbu_audit_post_correction.add_argument("--out", default="outputs/limbu-post-correction-profile-audit")
limbu_audit_post_correction.set_defaults(func=cmd_audit_limbu_post_correction_profile)
limbu_audit_output = subparsers.add_parser("audit-limbu-output", help="audit a Limbu OCR or correction output directory for stale/missing artifacts")
limbu_audit_output.add_argument("output_dir")
limbu_audit_output.add_argument("--out", help="audit report directory; defaults to the output directory")
limbu_audit_output.add_argument("--require-capture-prep", action="store_true", help="fail unless the output is bound to a passing prepare-limbu-capture audit")
limbu_audit_output.add_argument("--min-capture-prepared-width", type=int)
limbu_audit_output.add_argument("--min-capture-prepared-height", type=int)
limbu_audit_output.add_argument("--min-capture-prepared-entropy", type=float)
limbu_audit_output.add_argument("--min-capture-prepared-luminance-stddev", type=float)
limbu_audit_output.add_argument("--min-capture-prepared-edge-stddev", type=float)
limbu_audit_output.add_argument(
"--require-capture-metadata-path-self",
action="store_true",
help="fail capture-prep replay unless metadata_path points to the replayed metadata file",
)
limbu_audit_output.add_argument("--require-no-pending-review", action="store_true", help="fail when OCR/review output still has unresolved review rows")
limbu_audit_output.add_argument(
"--require-reviewer-for-corrections",
action="store_true",
help="fail review-correction outputs when accepted correction rows have no reviewer",
)
limbu_audit_output.add_argument(
"--require-no-dropped-image-lines",
action="store_true",
help="fail image-line OCR outputs when detected lines were filtered or dropped",
)
limbu_audit_output.add_argument("--min-line-count", type=int, help="fail unless the output document contains at least this many OCR text lines")
limbu_audit_output.add_argument("--min-average-line-confidence", type=float, help="fail unless quality.metrics.average_line_confidence is at least this value")
limbu_audit_output.add_argument("--min-quality-score", type=float, help="fail unless quality.quality_score is at least this value")
limbu_audit_output.add_argument(
"--require-script",
action="append",
choices=["limbu_sirijonga", "devanagari_limbu", "mixed_limbu_devanagari", "other"],
default=[],
help="fail unless at least one line of this Limbu script class is observed; repeatable",
)
limbu_audit_output.add_argument(
"--require-script-count",
action="append",
default=[],
metavar="SCRIPT=COUNT",
help="fail unless the observed Limbu script class has at least COUNT lines; repeatable",
)
limbu_audit_output.set_defaults(func=cmd_audit_limbu_output)
benchmark = subparsers.add_parser("benchmark", help="compare OCR baselines on the same files")
benchmark.add_argument("inputs", nargs="*")
benchmark.add_argument("--inputs-from-manifest", help="load benchmark input paths from a manifest image_path field")
benchmark.add_argument("--out", default="outputs/benchmark")
benchmark.add_argument("--baselines", default="ours,tesseract,stock-paddle,glm-ocr,paddleocr-vl")
benchmark.add_argument("--references-dir")
benchmark.add_argument("--eval-manifest")
benchmark.add_argument("--candidate-model-config")
benchmark.add_argument("--fallback-engine", choices=["sidecar", "tesseract", "paddleocr", "surya"])
benchmark.add_argument("--fallback-model-config", help="candidate model card path to use as a selective fallback engine")
benchmark.add_argument("--low-confidence-threshold", type=float, default=0.80)
benchmark.add_argument("--fallback-min-quality-score", type=float, help="trigger configured candidate fallback when primary document quality score is below this value")
benchmark.add_argument("--resume-existing", action="store_true", help="score existing document.json outputs and run only missing pages")
benchmark.add_argument("--sample-timeout-seconds", type=int, help="mark one page as error if a baseline exceeds this runtime")
benchmark.add_argument("--capture-gpu-metrics", action="store_true", help="sample nvidia-smi before and after each benchmark row when available")
benchmark.set_defaults(func=cmd_benchmark)
summarize = subparsers.add_parser("summarize-benchmark", help="summarize aggregate and paired benchmark metrics from report.json")
summarize.add_argument("--benchmark-report", required=True)
summarize.add_argument("--out", default="outputs/benchmark-summary")
summarize.add_argument("--candidate", default="candidate")
summarize.add_argument("--baseline", action="append", default=[], help="baseline to compare against candidate; repeatable")
summarize.add_argument("--metric", action="append", default=[], help="metric to summarize in paired comparisons; repeatable")
summarize.set_defaults(func=cmd_summarize_benchmark)
rescore = subparsers.add_parser("rescore-benchmark", help="recompute benchmark metrics from existing document.json outputs")
rescore.add_argument("--benchmark-report", required=True)
rescore.add_argument("--out", default="outputs/benchmark-rescored")
rescore.add_argument("--references-dir")
rescore.add_argument("--eval-manifest")
rescore.set_defaults(func=cmd_rescore_benchmark)
derive_table = subparsers.add_parser(
"derive-table-postprocess-benchmark",
help="derive a table-postprocessed candidate from an existing baseline document output",
)
derive_table.add_argument("--benchmark-report", required=True)
derive_table.add_argument("--out", default="outputs/benchmark-table-postprocess-derived")
derive_table.add_argument("--source-baseline", required=True)
derive_table.add_argument("--candidate-baseline", default="candidate")
derive_table.add_argument("--references-dir")
derive_table.add_argument("--eval-manifest")
derive_table.set_defaults(func=cmd_derive_table_postprocess_benchmark)
table_cells = subparsers.add_parser("analyze-table-cells", help="analyze cell-level table OCR errors from benchmark document outputs")
table_cells.add_argument("--benchmark-report", required=True)
table_cells.add_argument("--eval-manifest", required=True)
table_cells.add_argument("--out", default="outputs/table-cell-analysis")
table_cells.add_argument("--candidate", default="candidate")
table_cells.add_argument("--baseline", action="append", default=[], help="baseline to compare against candidate; repeatable")
table_cells.add_argument("--top-n", type=int, default=25)
table_cells.set_defaults(func=cmd_analyze_table_cells)
paper_table = subparsers.add_parser("paper-benchmark-table", help="merge benchmark summary artifacts into a paper-ready comparison table")
paper_table.add_argument("--summary", action="append", default=[], required=True, help="path to summarize-benchmark summary.json; repeatable")
paper_table.add_argument("--out", default="outputs/paper-table")
paper_table.add_argument("--slice", default="all")
paper_table.add_argument("--metric", action="append", default=[], help="aggregate metric column to include; repeatable")
paper_table.add_argument("--system", action="append", default=[], help="system row to include and order explicitly; repeatable")
paper_table.add_argument("--label", action="append", default=[], help="display label override as system=Label; repeatable")
paper_table.add_argument("--source-preference", action="append", default=[], help="pin a system to one summary artifact as system=path; repeatable")
paper_table.add_argument("--title", help="optional table title")
paper_table.set_defaults(func=cmd_paper_benchmark_table)
oracle = subparsers.add_parser("benchmark-oracle", help="compute per-sample oracle headroom from a benchmark report")
oracle.add_argument("--benchmark-report", required=True)
oracle.add_argument("--out", default="outputs/benchmark-oracle")
oracle.add_argument("--metric", default="cer")
oracle.add_argument("--direction", choices=["lower", "higher"])
oracle.add_argument("--system", action="append", default=[], help="system to include in oracle selection; repeatable")
oracle.add_argument("--oracle-name", default="oracle")
oracle.set_defaults(func=cmd_benchmark_oracle)
error_analysis = subparsers.add_parser("benchmark-error-analysis", help="rank hard samples and slice gaps from a benchmark report")
error_analysis.add_argument("--benchmark-report", required=True)
error_analysis.add_argument("--out", default="outputs/benchmark-error-analysis")
error_analysis.add_argument("--system", required=True)
error_analysis.add_argument("--baseline", action="append", default=[], help="baseline to compare against system; repeatable")
error_analysis.add_argument("--metric", default="cer")
error_analysis.add_argument("--direction", choices=["lower", "higher"])
error_analysis.add_argument("--top-n", type=int, default=20)
error_analysis.set_defaults(func=cmd_benchmark_error_analysis)
failure_manifest = subparsers.add_parser("export-failure-manifest", help="write a manifest from ranked benchmark failure rows")
failure_manifest.add_argument("--source-manifest", required=True)
failure_manifest.add_argument("--top-failures", required=True)
failure_manifest.add_argument("--out", required=True)
failure_manifest.add_argument("--split", help="optional split override for selected entries")
failure_manifest.set_defaults(func=cmd_export_failure_manifest)
admission_decision = subparsers.add_parser("recognizer-admission-decision", help="summarize whether a recognizer admission validation report promotes or rejects a model branch")
admission_decision.add_argument("--validation-report", required=True)
admission_decision.add_argument("--out", default="outputs/recognizer-admission-decision")
admission_decision.set_defaults(func=cmd_recognizer_admission_decision)
failure_review = subparsers.add_parser("recognizer-failure-review", help="aggregate recognizer admission failures across runs and flag exhausted model families")
failure_review.add_argument("--admission-decision", action="append", default=[], help="recognizer-admission-decision.json path; repeatable")
failure_review.add_argument("--validation-report", action="append", default=[], help="validation.json path to convert into a review input; repeatable")
failure_review.add_argument("--model-card", action="append", default=[], help="model-card.json path matched to inputs by order when provenance is unavailable; repeatable")
failure_review.add_argument("--min-rejections-to-stop", type=int, default=3)
failure_review.add_argument("--out", default="outputs/recognizer-failure-review")
failure_review.set_defaults(func=cmd_recognizer_failure_review)
calibrate = subparsers.add_parser("calibrate-quality-router", help="calibrate a quality-routed composite candidate from a calibration experiment")
calibrate.add_argument("--experiment-report", required=True)
calibrate.add_argument("--out", default="outputs/quality-router-calibration")
calibrate.add_argument("--model-id", required=True)
calibrate.add_argument("--base-model", default="tesseract-ocr")
calibrate.add_argument("--primary-engine", required=True)
calibrate.add_argument("--secondary-engine", required=True)
calibrate.add_argument("--primary-engine-kwarg", action="append", default=[], help="primary engine kwarg as key=value; repeatable")
calibrate.add_argument("--secondary-engine-kwarg", action="append", default=[], help="secondary engine kwarg as key=value; repeatable")
calibrate.add_argument("--candidate-baseline", default="candidate")
calibrate.add_argument("--primary-baseline", default="tesseract")
calibrate.add_argument("--metric", default="cer")
calibrate.add_argument("--direction", choices=["lower", "higher"])
calibrate.add_argument("--threshold-start", type=float, default=-0.20)
calibrate.add_argument("--threshold-end", type=float, default=0.20)
calibrate.add_argument("--threshold-step", type=float, default=0.01)
calibrate.add_argument("--notes")
calibrate.set_defaults(func=cmd_calibrate_quality_router)
calibrate_script = subparsers.add_parser("calibrate-script-router", help="calibrate a script-routed composite candidate from a calibration experiment")
calibrate_script.add_argument("--experiment-report", required=True)
calibrate_script.add_argument("--out", default="outputs/script-router-calibration")
calibrate_script.add_argument("--model-id", required=True)
calibrate_script.add_argument("--base-model", default="tesseract+surya")
calibrate_script.add_argument("--primary-engine", required=True)
calibrate_script.add_argument("--secondary-engine", required=True)
calibrate_script.add_argument("--primary-engine-kwarg", action="append", default=[], help="primary engine kwarg as key=value; repeatable")
calibrate_script.add_argument("--secondary-engine-kwarg", action="append", default=[], help="secondary engine kwarg as key=value; repeatable")
calibrate_script.add_argument("--primary-baseline", default="tesseract")
calibrate_script.add_argument("--secondary-baseline", default="surya")
calibrate_script.add_argument("--metric", default="cer")
calibrate_script.add_argument("--direction", choices=["lower", "higher"])
calibrate_script.add_argument("--script", default="devanagari")
calibrate_script.add_argument("--threshold-start", type=float, default=0.0)
calibrate_script.add_argument("--threshold-end", type=float, default=0.6)
calibrate_script.add_argument("--threshold-step", type=float, default=0.05)
calibrate_script.add_argument("--routing-granularity", choices=["document", "page"], default="document")
calibrate_script.add_argument("--secondary-structure-backfill", action="store_true")
calibrate_script.add_argument("--notes")
calibrate_script.set_defaults(func=cmd_calibrate_script_router)
calibrate_ensemble = subparsers.add_parser("calibrate-tesseract-psm-ensemble", help="calibrate a quality-ranked Tesseract PSM ensemble from separate variant experiments")
calibrate_ensemble.add_argument("--primary-experiment-report", required=True)
calibrate_ensemble.add_argument("--variant-experiment", action="append", default=[], help="variant label and experiment report as label=path; repeatable")
calibrate_ensemble.add_argument("--out", default="outputs/tesseract-psm-ensemble-calibration")
calibrate_ensemble.add_argument("--model-id", required=True)
calibrate_ensemble.add_argument("--metric", default="cer")
calibrate_ensemble.add_argument("--direction", choices=["lower", "higher"])
calibrate_ensemble.add_argument("--bias-start", type=float, default=-0.12)
calibrate_ensemble.add_argument("--bias-end", type=float, default=0.12)
calibrate_ensemble.add_argument("--bias-step", type=float, default=0.02)
calibrate_ensemble.add_argument("--language", default="nep+eng")
calibrate_ensemble.add_argument("--selection-margin", type=float, default=0.0)
calibrate_ensemble.add_argument("--notes")
calibrate_ensemble.set_defaults(func=cmd_calibrate_tesseract_psm_ensemble)
experiment = subparsers.add_parser("run-experiment", help="run benchmark and validation from an eval manifest")
experiment.add_argument("--eval-manifest", required=True)
experiment.add_argument("--out", default="outputs/experiment")
experiment.add_argument("--baselines", default="ours,tesseract,stock-paddle,glm-ocr,paddleocr-vl")
experiment.add_argument("--references-dir")
experiment.add_argument("--validation-config")
experiment.add_argument("--candidate-model-config")
experiment.add_argument("--fallback-engine", choices=["sidecar", "tesseract", "paddleocr", "surya"])
experiment.add_argument("--fallback-model-config", help="candidate model card path to use as a selective fallback engine")
experiment.add_argument("--low-confidence-threshold", type=float, default=0.80)
experiment.add_argument("--fallback-min-quality-score", type=float, help="trigger configured candidate fallback when primary document quality score is below this value")
experiment.add_argument("--train-manifest", action="append", default=[])
experiment.add_argument("--gorkhapatra-review-audit", action="append", default=[], help="passed Gorkhapatra language-page review-audit JSON; repeatable")
experiment.add_argument("--skip-validation", action="store_true")
experiment.add_argument("--require-validated", action="store_true")
experiment.add_argument("--preflight", action="store_true", help="run claim preflight before benchmark and stop on failure")
experiment.add_argument("--allow-smoke-eval-pack", action="store_true", help="allow smoke eval packs during run-experiment preflight")
experiment.add_argument("--require-trained-recognizer", action="store_true", help="require trained recognizer provenance during run-experiment preflight")
experiment.add_argument("--require-model-admission", action="store_true", help="require passing recognizer admission during run-experiment preflight")
experiment.set_defaults(func=cmd_run_experiment)
preflight = subparsers.add_parser("preflight-claim", help="fail early if claim benchmark inputs, models, references, or engines are not ready")
preflight.add_argument("--eval-manifest", required=True)
preflight.add_argument("--out", default="outputs/preflight")
preflight.add_argument("--baselines", default="candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl")
preflight.add_argument("--references-dir")
preflight.add_argument("--candidate-model-config")
preflight.add_argument("--fallback-model-config", help="fallback model card path to audit before claim benchmark runs")
preflight.add_argument("--validation-config")
preflight.add_argument("--train-manifest", action="append", default=[])
preflight.add_argument("--gorkhapatra-review-audit", action="append", default=[], help="passed Gorkhapatra language-page review-audit JSON; repeatable")
preflight.add_argument("--allow-smoke-eval-pack", action="store_true")
preflight.add_argument("--require-trained-recognizer", action="store_true")
preflight.add_argument("--require-model-admission", action="store_true")
preflight.set_defaults(func=cmd_preflight_claim)
public_list = subparsers.add_parser("list-public-benchmarks", help="list known public benchmark registry entries")
public_list.set_defaults(func=cmd_list_public_benchmarks)
public_prepare = subparsers.add_parser("prepare-public-benchmark", help="convert a verified public benchmark into an auditable manifest")
public_prepare.add_argument("benchmark", choices=["opendatalab/OmniDocBench"])
public_prepare.add_argument("--out", required=True)
public_prepare.add_argument("--limit", type=int)
public_prepare.add_argument("--language")
public_prepare.add_argument("--subset")
public_prepare.add_argument("--data-source")
public_prepare.add_argument("--require-tables", action="store_true")
public_prepare.add_argument("--annotation-path", help="local OmniDocBench.json path for offline/schema-fixture conversion")
public_prepare.add_argument("--local-image-root", help="local dataset root containing images/ for offline conversion")
public_prepare.set_defaults(func=cmd_prepare_public_benchmark)
public_audit = subparsers.add_parser("audit-public-benchmark", help="audit a public benchmark manifest for claim readiness")
public_audit.add_argument("manifest")
public_audit.add_argument("--out", default="outputs/public-benchmark-audit")
public_audit.add_argument("--benchmark", help="override benchmark name when manifest metadata is ambiguous")
public_audit.add_argument("--min-samples", type=int)
public_audit.add_argument("--allow-missing-reference-paths", action="store_true")
public_audit.set_defaults(func=cmd_audit_public_benchmark)
gorkhapatra_audit = subparsers.add_parser("audit-gorkhapatra-source", help="audit Gorkhapatra/Naya Nepal article images and epaper PDFs")
gorkhapatra_audit.add_argument("--out", default="outputs/gorkhapatra-source-audit")
gorkhapatra_audit.add_argument("--category-url", action="append", default=[], help="Naya Nepal category URL; repeatable")
gorkhapatra_audit.add_argument("--epaper-url", action="append", default=[], help="Gorkhapatra epaper list URL; repeatable")
gorkhapatra_audit.add_argument("--category-html", action="append", default=[], help="local category HTML fixture/path; repeatable")
gorkhapatra_audit.add_argument("--epaper-html", action="append", default=[], help="local epaper list HTML fixture/path; repeatable")
gorkhapatra_audit.add_argument("--no-default-urls", action="store_true", help="only use explicitly supplied URLs/HTML files")
gorkhapatra_audit.add_argument("--download-assets", action="store_true", help="download and hash discovered article images/PDFs")
gorkhapatra_audit.add_argument("--max-article-images", type=int, default=0, help="max article images to download; 0 means no limit")
gorkhapatra_audit.add_argument("--max-epaper-pdfs", type=int, default=0, help="max epaper PDFs to download; 0 means no limit")
gorkhapatra_audit.add_argument("--timeout-seconds", type=float, default=30.0)
gorkhapatra_audit.set_defaults(func=cmd_audit_gorkhapatra_source)
gorkhapatra_pack = subparsers.add_parser("prepare-gorkhapatra-pack", help="turn a Gorkhapatra source audit into a pending-reference validation manifest")
gorkhapatra_pack.add_argument("source_audit_json")
gorkhapatra_pack.add_argument("--out", default="data/gorkhapatra-pack")
gorkhapatra_pack.add_argument("--no-article-images", action="store_true")
gorkhapatra_pack.add_argument("--no-epaper-pages", action="store_true")
gorkhapatra_pack.add_argument("--max-article-images", type=int, default=0)
gorkhapatra_pack.add_argument("--max-epaper-pdfs", type=int, default=0)
gorkhapatra_pack.add_argument("--max-pages-per-pdf", type=int, default=2)
gorkhapatra_pack.add_argument("--split", default="eval")
gorkhapatra_pack.add_argument("--dataset-name", default="gorkhapatra-naya-nepal")
gorkhapatra_pack.set_defaults(func=cmd_prepare_gorkhapatra_pack)
language_pages = subparsers.add_parser("audit-gorkhapatra-language-pages", help="align Naya Nepal publication cards to same-date Gorkhapatra PDF language pages")
language_pages.add_argument("--out", default="outputs/gorkhapatra-language-pages")
language_pages.add_argument("--publication-url", action="append", default=[])
language_pages.add_argument("--publication-pages", type=int, default=1, help="expand each publication URL across this many ?page=N archive pages")
language_pages.add_argument("--epaper-url", action="append", default=[])
language_pages.add_argument("--publication-html", action="append", default=[])
language_pages.add_argument("--epaper-html", action="append", default=[])
language_pages.add_argument("--no-default-urls", action="store_true")
language_pages.add_argument("--download-pdfs", action="store_true")
language_pages.add_argument("--render-pages", action="store_true")
language_pages.add_argument("--max-publication-items", type=int, default=0)
language_pages.add_argument("--max-epaper-pdfs", type=int, default=0)
language_pages.add_argument("--timeout-seconds", type=float, default=30.0)
language_pages.set_defaults(func=cmd_audit_gorkhapatra_language_pages)
language_page_pack = subparsers.add_parser("prepare-gorkhapatra-language-page-pack", help="turn language-page alignment hits into a pending-reference manifest")
language_page_pack.add_argument("language_page_audit_json")
language_page_pack.add_argument("--out", default="data/gorkhapatra-language-page-pack")
language_page_pack.add_argument("--split", default="eval")
language_page_pack.add_argument("--dataset-name", default="gorkhapatra-language-pages")
language_page_pack.add_argument("--max-samples", type=int, default=0)
language_page_pack.set_defaults(func=cmd_prepare_gorkhapatra_language_page_pack)
language_page_review = subparsers.add_parser("prepare-gorkhapatra-language-page-review", help="write a manual review sheet for language-page candidates")
language_page_review.add_argument("language_page_pack_manifest")
language_page_review.add_argument("--out", default="data/gorkhapatra-language-page-review")
language_page_review.set_defaults(func=cmd_prepare_gorkhapatra_language_page_review)
language_page_review_audit = subparsers.add_parser(
"audit-gorkhapatra-language-page-review",
help="audit Gorkhapatra language-page manual decisions before reference template or finalization use",
)
language_page_review_audit.add_argument("language_page_pack_manifest")
language_page_review_audit.add_argument("review_csv")
language_page_review_audit.add_argument("--out", default="outputs/gorkhapatra-language-page-review-audit")
language_page_review_audit.add_argument("--require-verified-references", action="store_true")
language_page_review_audit.add_argument("--candidate-language", action="append", default=[], help="limit audit to accepted/reviewed rows for this candidate language; repeatable")
language_page_review_audit.set_defaults(func=cmd_audit_gorkhapatra_language_page_review)
language_page_templates = subparsers.add_parser(
"prepare-gorkhapatra-language-page-reference-templates",
help="write structured reference JSON drafts for Gorkhapatra language-page review rows",
)
language_page_templates.add_argument("language_page_pack_manifest")
language_page_templates.add_argument("--out", default="data/gorkhapatra-language-page-reference-templates")
language_page_templates.add_argument("--review-csv")
language_page_templates.add_argument("--accepted-only", action="store_true")
language_page_templates.set_defaults(func=cmd_prepare_gorkhapatra_language_page_reference_templates)
language_page_reviewer_bundle = subparsers.add_parser(
"prepare-gorkhapatra-language-page-reviewer-bundle",
help="bundle accepted language-page images and draft references for manual labeling",
)
language_page_reviewer_bundle.add_argument("language_page_pack_manifest")
language_page_reviewer_bundle.add_argument("review_csv")
language_page_reviewer_bundle.add_argument("--out", default="data/gorkhapatra-language-page-reviewer-bundle")
language_page_reviewer_bundle.add_argument("--reference-template-dir")
language_page_reviewer_bundle.set_defaults(func=cmd_prepare_gorkhapatra_language_page_reviewer_bundle)
language_page_work_order = subparsers.add_parser(
"prepare-gorkhapatra-language-page-transcription-work-order",
help="write a transcription work order for accepted Gorkhapatra language-page references",
)
language_page_work_order.add_argument("language_page_pack_manifest")
language_page_work_order.add_argument("review_csv")
language_page_work_order.add_argument("--out", default="data/gorkhapatra-language-page-transcription-work-order")
language_page_work_order.add_argument("--candidate-language", action="append", default=[], help="include only accepted rows for this candidate language; repeatable")
language_page_work_order.set_defaults(func=cmd_prepare_gorkhapatra_language_page_transcription_work_order)
language_page_pdf_text = subparsers.add_parser(
"extract-gorkhapatra-language-page-pdf-text",
help="extract PDF-native text spans, font metadata, and embedded fonts from Gorkhapatra language-page candidates",
)
language_page_pdf_text.add_argument("language_page_pack_manifest")
language_page_pdf_text.add_argument("--out", default="data/gorkhapatra-language-page-pdf-text")
language_page_pdf_text.add_argument("--candidate-language", action="append", default=[], help="include only manifest rows for this candidate language; repeatable")
language_page_pdf_text.add_argument("--target-font", action="append", default=[], help="extra target font substring to mark as language/script spans; repeatable")
language_page_pdf_text.set_defaults(func=cmd_extract_gorkhapatra_language_page_pdf_text)
limbu_legacy = subparsers.add_parser("convert-limbu-legacy", help="convert Namdhinggo/Sirijonga legacy text to Unicode Limbu")
limbu_legacy.add_argument("--text", help="legacy text string to convert")
limbu_legacy.add_argument("--pdf-text-json", help="Gorkhapatra PDF text sample JSON to convert target-font spans")
limbu_legacy.add_argument("--out", default="data/limbu-legacy-converted")
limbu_legacy.add_argument("--map", default="data/mappings/limbu-legacy/Limbu.map", help="SIL Limbu legacy TECkit map file")
limbu_legacy.set_defaults(func=cmd_convert_limbu_legacy)
language_page_assisted_refs = subparsers.add_parser(
"prepare-gorkhapatra-language-page-assisted-references",
help="write non-claim-ready machine-assisted reference drafts for accepted Gorkhapatra language pages",
)
language_page_assisted_refs.add_argument("language_page_pack_manifest")
language_page_assisted_refs.add_argument("review_csv")
language_page_assisted_refs.add_argument("--out", default="data/gorkhapatra-language-page-assisted-references")
language_page_assisted_refs.add_argument("--candidate-language", action="append", default=[], help="include only accepted rows for this candidate language; repeatable")
language_page_assisted_refs.add_argument("--engine", default="sidecar", choices=["sidecar", "tesseract", "auto"], help="OCR engine used only for draft suggestions")
language_page_assisted_refs.add_argument("--tesseract-language", default="nep+eng", help="Tesseract language code when --engine tesseract")
language_page_assisted_refs.add_argument("--overwrite", action="store_true", help="replace existing assisted draft reference files")
language_page_assisted_refs.set_defaults(func=cmd_prepare_gorkhapatra_language_page_assisted_references)
language_page_pdf_native_refs = subparsers.add_parser(
"prepare-gorkhapatra-language-page-pdf-native-references",
help="draft references from PDF-native legacy->Unicode conversion (pre-filled text for human review, not OCR)",
)
language_page_pdf_native_refs.add_argument("language_page_pack_manifest")
language_page_pdf_native_refs.add_argument("review_csv")
language_page_pdf_native_refs.add_argument("pdf_text_dir", help="extract-gorkhapatra-language-page-pdf-text output dir (contains samples/<id>.pdf-text.json)")
language_page_pdf_native_refs.add_argument("--out", default="data/gorkhapatra-language-page-pdf-native-references")
language_page_pdf_native_refs.add_argument("--candidate-language", action="append", default=[], help="include only accepted rows for this candidate language; repeatable")
language_page_pdf_native_refs.add_argument("--converter", default="limbu-namdhinggo", help="legacy converter id (e.g. limbu-namdhinggo)")
language_page_pdf_native_refs.add_argument("--map", dest="map_path", default=None, help="override the legacy TECkit map path for the converter")
language_page_pdf_native_refs.add_argument("--overwrite", action="store_true", help="replace existing pdf-native draft reference files")
language_page_pdf_native_refs.set_defaults(func=cmd_prepare_gorkhapatra_language_page_pdf_native_references)
language_page_recognizer_eval = subparsers.add_parser(
"prepare-gorkhapatra-language-page-recognizer-eval",
help="turn verified Gorkhapatra references into a line-level PaddleOCR recognizer eval manifest",
)
language_page_recognizer_eval.add_argument("language_page_pack_manifest")
language_page_recognizer_eval.add_argument("references_dir", help="directory of verified <sample_id>.ref.json files")
language_page_recognizer_eval.add_argument("--out", default="data/gorkhapatra-language-page-recognizer-eval")
language_page_recognizer_eval.add_argument("--candidate-language", action="append", default=[], help="include only this candidate language; repeatable")
language_page_recognizer_eval.add_argument("--split", default="eval", help="split label for emitted manifest rows")
language_page_recognizer_eval.add_argument("--dataset-name", default="gorkhapatra-language-pages-real")
language_page_recognizer_eval.add_argument("--min-text-length", type=int, default=1)
language_page_recognizer_eval.add_argument("--allow-draft", action="store_true", help="smoke-test against non-claim-eligible draft references (rows marked non-claim-eligible)")
language_page_recognizer_eval.set_defaults(func=cmd_prepare_gorkhapatra_language_page_recognizer_eval)
language_page_ocr_sidecars = subparsers.add_parser(
"prepare-gorkhapatra-language-page-ocr-sidecars",
help="run OCR on accepted Gorkhapatra language pages and write sidecar JSON for draft references",
)
language_page_ocr_sidecars.add_argument("language_page_pack_manifest")
language_page_ocr_sidecars.add_argument("review_csv")
language_page_ocr_sidecars.add_argument("--out", default="data/gorkhapatra-language-page-ocr-sidecars")
language_page_ocr_sidecars.add_argument("--candidate-language", action="append", default=[], help="include only accepted rows for this candidate language; repeatable")
language_page_ocr_sidecars.add_argument("--engine", default="paddleocr", choices=["sidecar", "tesseract", "paddleocr", "auto"], help="OCR engine used to create sidecars")
language_page_ocr_sidecars.add_argument("--tesseract-language", default="nep+eng", help="Tesseract language code when --engine tesseract")
language_page_ocr_sidecars.add_argument("--overwrite", action="store_true", help="replace existing OCR sidecar files")
language_page_ocr_sidecars.add_argument("--in-place", action="store_true", help="write sidecars beside each source image so SidecarEngine can discover them")
language_page_ocr_sidecars.set_defaults(func=cmd_prepare_gorkhapatra_language_page_ocr_sidecars)
language_page_verification_bundle = subparsers.add_parser(
"prepare-gorkhapatra-language-page-verification-bundle",
help="create line-crop review sheets from assisted Gorkhapatra reference drafts",
)
language_page_verification_bundle.add_argument("language_page_pack_manifest")
language_page_verification_bundle.add_argument("review_csv")
language_page_verification_bundle.add_argument("assisted_references_dir")
language_page_verification_bundle.add_argument("--out", default="data/gorkhapatra-language-page-verification-bundle")
language_page_verification_bundle.add_argument("--candidate-language", action="append", default=[], help="include only accepted rows for this candidate language; repeatable")
language_page_verification_bundle.add_argument("--overwrite", action="store_true", help="replace existing crop images")
language_page_verification_bundle.set_defaults(func=cmd_prepare_gorkhapatra_language_page_verification_bundle)
language_page_verification_audit = subparsers.add_parser(
"audit-gorkhapatra-language-page-verification-csv",
help="dry-run audit a completed line-level verification CSV before applying it",
)
language_page_verification_audit.add_argument("verification_csv")
language_page_verification_audit.add_argument("--out", default="outputs/gorkhapatra-language-page-verification-csv-audit")
language_page_verification_audit.add_argument("--candidate-language", action="append", default=[], help="include only rows for this candidate language; repeatable")
language_page_verification_audit.add_argument("--require-codepoint-range", action="append", default=[], help="require kept text to include at least one codepoint from this range, e.g. U+1900..U+194F; repeatable")
language_page_verification_audit.set_defaults(func=cmd_audit_gorkhapatra_language_page_verification_csv)
language_page_verification_split = subparsers.add_parser(
"split-gorkhapatra-language-page-verification-csv",
help="split a line-level verification CSV into reviewer batches with per-batch HTML contact sheets",
)
language_page_verification_split.add_argument("verification_csv")
language_page_verification_split.add_argument("--out", default="data/gorkhapatra-language-page-verification-batches")
language_page_verification_split.add_argument("--batch-size", type=int, default=50)
language_page_verification_split.add_argument("--candidate-language", action="append", default=[], help="include only rows for this candidate language; repeatable")
language_page_verification_split.set_defaults(func=cmd_split_gorkhapatra_language_page_verification_csv)
language_page_verification_merge = subparsers.add_parser(
"merge-gorkhapatra-language-page-verification-batches",
help="merge reviewed verification batch CSVs and validate coverage against the source CSV",
)
language_page_verification_merge.add_argument("source_verification_csv")
language_page_verification_merge.add_argument("batches_dir")
language_page_verification_merge.add_argument("--out", default="data/gorkhapatra-language-page-verification-merged")
language_page_verification_merge.set_defaults(func=cmd_merge_gorkhapatra_language_page_verification_batches)
language_page_verification_assign = subparsers.add_parser(
"assign-gorkhapatra-language-page-verification-batches",
help="create a reviewer assignment ledger from a verification batch split index",
)
language_page_verification_assign.add_argument("split_index_csv")
language_page_verification_assign.add_argument("--out", default="data/gorkhapatra-language-page-verification-assignments")
language_page_verification_assign.add_argument("--reviewer", action="append", default=[], help="reviewer id/name; repeatable, assigned round-robin")
language_page_verification_assign.add_argument("--due-date", help="optional due date recorded in the assignment ledger")
language_page_verification_assign.set_defaults(func=cmd_assign_gorkhapatra_language_page_verification_batches)
language_page_verification_apply = subparsers.add_parser(
"apply-gorkhapatra-language-page-verification-bundle",
help="convert completed line-level review CSVs into verified reference JSON and an updated review CSV",
)
language_page_verification_apply.add_argument("language_page_pack_manifest")
language_page_verification_apply.add_argument("review_csv")
language_page_verification_apply.add_argument("verification_csv")
language_page_verification_apply.add_argument("assisted_references_dir")
language_page_verification_apply.add_argument("--out", default="data/gorkhapatra-language-page-verified-references")
language_page_verification_apply.add_argument("--candidate-language", action="append", default=[], help="include only accepted rows for this candidate language; repeatable")
language_page_verification_apply.add_argument("--reviewer", help="human reviewer identifier to write into verified reference metadata")
language_page_verification_apply.add_argument("--reviewed-at", help="review completion timestamp/date to write into verified reference metadata")
language_page_verification_apply.add_argument("--tables-reviewed", action="store_true", help="declare that table presence/absence and table content were checked by a human")
language_page_verification_apply.add_argument("--figures-reviewed", action="store_true", help="declare that figure presence/absence and figure metadata were checked by a human")
language_page_verification_apply.add_argument("--captions-reviewed", action="store_true", help="declare that captions were checked by a human")
language_page_verification_apply.add_argument("--require-codepoint-range", action="append", default=[], help="require kept text to include at least one codepoint from this range, e.g. U+1900..U+194F; repeatable")
language_page_verification_apply.add_argument("--overwrite", action="store_true", help="replace existing verified reference JSON files")
language_page_verification_apply.set_defaults(func=cmd_apply_gorkhapatra_language_page_verification_bundle)
language_page_finalize = subparsers.add_parser(
"finalize-gorkhapatra-language-page-review",
help="promote accepted reviewed language-page candidates with verified references into claim-ready eval",
)
language_page_finalize.add_argument("language_page_pack_manifest")
language_page_finalize.add_argument("review_csv")
language_page_finalize.add_argument("--out", default="data/gorkhapatra-language-page-verified")
language_page_finalize.add_argument("--split", default="eval")
language_page_finalize.add_argument("--dataset-name", default="gorkhapatra-language-pages-verified")
language_page_finalize.add_argument("--candidate-language", action="append", default=[], help="finalize only accepted rows for this candidate language; repeatable")
language_page_finalize.set_defaults(func=cmd_finalize_gorkhapatra_language_page_review)
language_registry = subparsers.add_parser("audit-language-registry", help="audit a multilingual OCR language registry")
language_registry.add_argument("registry")
language_registry.add_argument("--out", default="outputs/language-registry-audit")
language_registry.add_argument("--min-nepal-languages", type=int, default=10)
language_registry.add_argument("--allow-non-limbu-first", action="store_true")
language_registry.add_argument("--no-verify-local-paths", action="store_true")
language_registry.set_defaults(func=cmd_audit_language_registry)
language_readiness = subparsers.add_parser(
"audit-language-readiness",
help="audit target-language readiness across registry, synthesis, and validation evidence",
)
language_readiness.add_argument("registry")
language_readiness.add_argument("--out", default="outputs/language-readiness-audit")
language_readiness.add_argument(
"--synthesis-text-audit",
action="append",
default=[],
help="passed synthesis-text-manifest-audit.json evidence; repeatable",
)
language_readiness.add_argument("--min-target-languages", type=int, default=10)
language_readiness.add_argument("--min-synthesis-ready-languages", type=int, default=1)
language_readiness.add_argument("--no-require-limbu-synthesis", action="store_true")
language_readiness.set_defaults(func=cmd_audit_language_readiness)
font_audit = subparsers.add_parser("audit-font-renderability", help="audit registry font coverage and optional rendered samples")
font_audit.add_argument("registry")
font_audit.add_argument("--out", default="outputs/font-renderability-audit")
font_audit.add_argument("--all-scripts", action="store_true", help="audit secondary/future scripts in addition to primary scripts")
font_audit.add_argument("--min-coverage-ratio", type=float, default=0.95)
font_audit.add_argument("--render-samples", action="store_true", help="write sample PNGs when Pillow is installed")
font_audit.set_defaults(func=cmd_audit_font_renderability)
font_inventory = subparsers.add_parser(
"inventory-fonts",
help="scan font directories and ZIP archives, hashing files and matching cmap coverage to registry scripts",
)
font_inventory.add_argument("registry")
font_inventory.add_argument("--root", action="append", required=True, help="font directory, font file, or ZIP archive to scan; repeatable")
font_inventory.add_argument("--out", default="outputs/font-inventory")
font_inventory.add_argument("--min-script-coverage-ratio", type=float, default=0.95)
font_inventory.set_defaults(func=cmd_inventory_fonts)
font_assets = subparsers.add_parser(
"prepare-font-assets",
help="download/reuse font assets from a manifest and verify expected SHA-256 values",
)
font_assets.add_argument("manifest")
font_assets.add_argument("--asset-root", default=".")
font_assets.add_argument("--out", default="outputs/font-assets-preparation")
font_assets.add_argument("--force", action="store_true", help="redownload files whose existing SHA-256 does not match")
font_assets.add_argument("--timeout-seconds", type=float, default=60)
font_assets.set_defaults(func=cmd_prepare_font_assets)
font_asset_readiness = subparsers.add_parser(
"audit-font-asset-readiness",
help="gate font assets against preparation, inventory, and renderability evidence",
)
font_asset_readiness.add_argument("manifest")
font_asset_readiness.add_argument("preparation_report")
font_asset_readiness.add_argument("inventory_report")
font_asset_readiness.add_argument("renderability_report")
font_asset_readiness.add_argument("--asset-root", default=".")
font_asset_readiness.add_argument("--out", default="outputs/font-asset-readiness")
font_asset_readiness.set_defaults(func=cmd_audit_font_asset_readiness)
dictionary_audit = subparsers.add_parser(
"audit-paddle-dictionary",
help="audit a PaddleOCR character dictionary against Unicode ranges and label files",
)
dictionary_audit.add_argument("dictionary")
dictionary_audit.add_argument("--out", default="outputs/paddle-dictionary-audit")
dictionary_audit.add_argument("--required-range", action="append", default=[], help="Unicode range such as U+1900..U+194F; repeatable")
dictionary_audit.add_argument("--label-file", action="append", default=[], help="PaddleOCR label file to check for character coverage; repeatable")
dictionary_audit.add_argument("--no-allow-space-char", action="store_true", help="require a literal space row instead of relying on use_space_char=true")
dictionary_audit.set_defaults(func=cmd_audit_paddle_dictionary)
synthesis_resources = subparsers.add_parser(
"audit-synthesis-resources",
help="audit real-text resource directories before synthetic OCR generation",
)
synthesis_resources.add_argument("--root", action="append", required=True, help="resource directory to audit; repeatable")
synthesis_resources.add_argument(
"--label",
action="append",
default=[],
help="label for each --root; repeatable and must match root count when used",
)
synthesis_resources.add_argument("--out", default="outputs/synthesis-resource-audit")
synthesis_resources.add_argument("--max-files-per-root", type=int, default=0)
synthesis_resources.add_argument("--max-text-samples-per-root", type=int, default=20)
synthesis_resources.add_argument("--sample-bytes", type=int, default=64000)
synthesis_resources.set_defaults(func=cmd_audit_synthesis_resources)
limbu_limdic = subparsers.add_parser("prepare-limbu-limdic-text", help="convert verified LTK Limdic JSON/TSV sources into synthesis text JSONL")
limbu_limdic.add_argument("--source", action="append", required=True, help="Limdic JSON/TSV source; repeatable")
limbu_limdic.add_argument("--out", default="data/synthesis-text/limbu-limdic")
limbu_limdic.add_argument("--split", default="train")
limbu_limdic.add_argument("--dataset", default="limbu-limdic")
limbu_limdic.add_argument("--min-text-chars", type=int, default=1)
limbu_limdic.set_defaults(func=cmd_prepare_limbu_limdic_text)
tamang_text = subparsers.add_parser("prepare-tamang-text", help="convert verified LTK Tamang CSV/TSV/JSON sources into synthesis text JSONL")
tamang_text.add_argument("--source", action="append", required=True, help="Tamang NepTam CSV or dictionary TSV/JSON source; repeatable")
tamang_text.add_argument("--out", default="data/synthesis-text/tamang")
tamang_text.add_argument("--split", default="train")
tamang_text.add_argument("--dataset", default="tamang-real-text")
tamang_text.add_argument("--min-text-chars", type=int, default=1)
tamang_text.add_argument("--limit-per-source", type=int, default=0)
tamang_text.set_defaults(func=cmd_prepare_tamang_text)
magar_text = subparsers.add_parser("prepare-magar-text", help="convert verified LTK Magar dictionary TSV/JSON sources into synthesis text JSONL")
magar_text.add_argument("--source", action="append", required=True, help="Magar dictionary TSV/JSON source; repeatable")
magar_text.add_argument("--out", default="data/synthesis-text/magar")
magar_text.add_argument("--split", default="train")
magar_text.add_argument("--dataset", default="magar-real-text")
magar_text.add_argument("--min-text-chars", type=int, default=1)
magar_text.add_argument("--limit-per-source", type=int, default=0)
magar_text.set_defaults(func=cmd_prepare_magar_text)
bible_brain = subparsers.add_parser("prepare-bible-brain-text", help="convert verified Bible Brain text manifests into synthesis text JSONL")
bible_brain.add_argument("--manifest", action="append", required=True, help="Bible Brain metadata/text_manifest.jsonl path; repeatable")
bible_brain.add_argument("--language", action="append", required=True, help="language id for each --manifest; repeatable and must match manifest count")
bible_brain.add_argument("--out", default="data/synthesis-text/bible-brain")
bible_brain.add_argument("--split", default="train")
bible_brain.add_argument("--dataset", default="bible-brain-nepal")
bible_brain.add_argument("--min-text-chars", type=int, default=1)
bible_brain.add_argument("--limit-per-manifest", type=int, default=0)
bible_brain.set_defaults(func=cmd_prepare_bible_brain_text)
limbu_unicode = subparsers.add_parser("prepare-limbu-unicode-text", help="convert verified Limbu/Sirijonga Unicode text sources into synthesis text JSONL")
limbu_unicode.add_argument("--source", action="append", required=True, help="Limbu Unicode .txt or tagged .tsv source; repeatable")
limbu_unicode.add_argument("--out", default="data/synthesis-text/limbu-unicode")
limbu_unicode.add_argument("--split", default="train")
limbu_unicode.add_argument("--dataset", default="limbu-unicode-real-text")
limbu_unicode.add_argument("--min-text-chars", type=int, default=1)
limbu_unicode.add_argument("--min-limbu-chars", type=int, default=1)
limbu_unicode.add_argument("--limit-per-source", type=int, default=0)
limbu_unicode.set_defaults(func=cmd_prepare_limbu_unicode_text)
toolkit_parallel = subparsers.add_parser(
"prepare-toolkit-parallel-text",
help="convert explicitly mapped toolkit JSONL/TSV/CSV text fields into synthesis text JSONL",
)
toolkit_parallel.add_argument("--source", action="append", required=True, help="toolkit JSONL/TSV/CSV source; repeatable")
toolkit_parallel.add_argument(
"--text-field",
action="append",
required=True,
help="text field mapping in FIELD=language form; repeatable",
)
toolkit_parallel.add_argument("--out", default="data/synthesis-text/toolkit-parallel")
toolkit_parallel.add_argument("--split", default="train")
toolkit_parallel.add_argument("--dataset", default="nepal-toolkit-parallel-text")
toolkit_parallel.add_argument("--min-text-chars", type=int, default=1)
toolkit_parallel.add_argument("--limit-per-source", type=int, default=0)
toolkit_parallel.add_argument("--row-id-field", default=None)
toolkit_parallel.add_argument("--metadata-field", action="append", default=[])
toolkit_parallel.add_argument("--license-status", default="pending_review")
toolkit_parallel.add_argument("--split-policy", default=None)
toolkit_parallel.set_defaults(func=cmd_prepare_toolkit_parallel_text)
audit_synthesis_text = subparsers.add_parser(
"audit-synthesis-text-manifest",
help="audit synthesis-text JSONL manifests before rendering or training",
)
audit_synthesis_text.add_argument("manifest")
audit_synthesis_text.add_argument("--out", default="outputs/synthesis-text-manifest-audit")
audit_synthesis_text.add_argument("--require-language", action="append", default=[], help="language id that must appear; repeatable")
audit_synthesis_text.add_argument("--require-script", action="append", default=[], help="script label that must appear; repeatable")
audit_synthesis_text.add_argument("--min-samples", type=int, default=1)
audit_synthesis_text.add_argument(
"--allow-claim-evidence",
action="store_true",
help="allow rows marked claim_evidence_eligible; default rejects them",
)
audit_synthesis_text.set_defaults(func=cmd_audit_synthesis_text_manifest)
audit_synthesis_promotion = subparsers.add_parser(
"audit-synthesis-text-promotion",
help="audit synthesis-text JSONL before rendering/training promotion",
)
audit_synthesis_promotion.add_argument("manifest")
audit_synthesis_promotion.add_argument("--out", default="outputs/synthesis-text-promotion-audit")
audit_synthesis_promotion.add_argument(
"--exclude",
action="append",
default=[],
help="manifest, reference file, text file, or directory whose text hashes must not overlap; repeatable",
)
audit_synthesis_promotion.add_argument(
"--require-reviewed-license",
action="store_true",
help="fail rows whose metadata.license_status is missing or pending_review",
)
audit_synthesis_promotion.set_defaults(func=cmd_audit_synthesis_text_promotion)
split_synthesis = subparsers.add_parser(
"split-synthesis-text",
help="split synthesis-text JSONL into deterministic train/eval files with promotion audits",
)
split_synthesis.add_argument("manifest")
split_synthesis.add_argument("--out", default="data/synthesis-text/splits")
split_synthesis.add_argument("--eval-ratio", type=float, default=0.15)
split_synthesis.add_argument("--seed", type=int, default=13)
split_synthesis.add_argument(
"--group-by",
choices=["text_sha256", "source_path", "source_row_id", "sample_id"],
default="text_sha256",
)
split_synthesis.add_argument(
"--exclude",
action="append",
default=[],
help="manifest, reference file, text file, or directory whose text hashes must not overlap; repeatable",
)
split_synthesis.add_argument("--require-reviewed-license", action="store_true")
split_synthesis.set_defaults(func=cmd_split_synthesis_text)
render_lines = subparsers.add_parser("render-synthesis-text-lines", help="render synthesis-text JSONL rows into OCR line images")
render_lines.add_argument("synthesis_text_manifest")
render_lines.add_argument("--out", default="data/rendered-lines")
render_lines.add_argument("--font-path")
render_lines.add_argument("--limit", type=int, default=0)
render_lines.add_argument("--script", action="append", default=[], help="script label to include; repeatable")
render_lines.add_argument("--font-size", type=int, default=36)
render_lines.add_argument("--padding", type=int, default=16)
render_lines.add_argument(
"--degradation-profile",
action="append",
default=[],
help="rendered-image degradation profile; repeatable: clean, scan, low_light, uneven_light, glare_light, camera_left, camera_right, camera_top, camera_bottom, phone_photo",
)
render_lines.add_argument("--degradation-seed", type=int, default=13)
render_lines.add_argument("--split", default="train")
render_lines.add_argument("--dataset", default="synthesis-rendered-lines")
render_lines.set_defaults(func=cmd_render_synthesis_text_lines)
render_split = subparsers.add_parser(
"render-synthesis-text-split",
help="render train/eval synthesis-text JSONL splits into audited OCR line bundles",
)
render_split.add_argument("--train-manifest", required=True)
render_split.add_argument("--eval-manifest", required=True)
render_split.add_argument("--out", default="data/rendered-line-split")
render_split.add_argument("--font-path")
render_split.add_argument("--limit-per-split", type=int, default=0)
render_split.add_argument("--script", action="append", default=[], help="script label to include; repeatable")
render_split.add_argument("--font-size", type=int, default=36)
render_split.add_argument("--padding", type=int, default=16)
render_split.add_argument(
"--degradation-profile",
action="append",
default=[],
help="rendered-image degradation profile; repeatable: clean, scan, low_light, uneven_light, glare_light, camera_left, camera_right, camera_top, camera_bottom, phone_photo",
)
render_split.add_argument("--degradation-seed", type=int, default=13)
render_split.add_argument("--dataset", default="synthesis-rendered-lines")
render_split.add_argument("--require-font", action="store_true", help="require font provenance in rendered audits")
render_split.add_argument("--font-readiness-report", help="passed font-asset-readiness.json proving font asset provenance")
render_split.set_defaults(func=cmd_render_synthesis_text_split)
audit_rendered_lines = subparsers.add_parser(
"audit-rendered-synthesis-lines",
help="audit rendered synthetic OCR line manifests for provenance and claim guardrails",
)
audit_rendered_lines.add_argument("manifest")
audit_rendered_lines.add_argument("--out", default="outputs/rendered-line-manifest-audit")
audit_rendered_lines.add_argument("--label-file", help="PaddleOCR label file to cross-check against the manifest")
audit_rendered_lines.add_argument("--require-font", action="store_true", help="require font path and font SHA provenance")
audit_rendered_lines.add_argument("--font-readiness-report", help="passed font-asset-readiness.json proving font asset provenance")
audit_rendered_lines.set_defaults(func=cmd_audit_rendered_synthesis_lines)
audit_degradation_split = subparsers.add_parser(
"audit-rendered-degradation-split",
help="audit augmented rendered train/eval manifests for degradation balance and text-hash leakage",
)
audit_degradation_split.add_argument("--train-manifest", required=True)
audit_degradation_split.add_argument("--eval-manifest", required=True)
audit_degradation_split.add_argument("--out", default="outputs/rendered-degradation-split-audit")
audit_degradation_split.add_argument(
"--expected-profile",
action="append",
default=[],
help="expected degradation profile for every text hash; repeatable",
)
audit_degradation_split.set_defaults(func=cmd_audit_rendered_degradation_split)
remote = subparsers.add_parser("audit-remote-host", help="audit a remote host for training and benchmark readiness")
remote.add_argument("--host", required=True)
remote.add_argument("--user")
remote.add_argument("--port", type=int, default=22)
remote.add_argument("--password-env")
remote.add_argument("--min-python", default="3.11")
remote.add_argument("--min-free-gb", type=float, default=20.0)
remote.add_argument("--workdir")
remote.add_argument("--require-gpu", action="store_true")
remote.add_argument("--require-paddle-training", action="store_true", help="fail if the host is not a valid official Paddle training target")
remote.add_argument("--python-candidate", action="append", default=[])
remote.add_argument("--require-command", action="append", default=[])
remote.add_argument("--optional-command", action="append", default=[])
remote.add_argument("--require-path", action="append", default=[])
remote.add_argument("--out", default="outputs/remote-audit")
remote.set_defaults(func=cmd_audit_remote_host)
bootstrap = subparsers.add_parser("bootstrap-remote-host", help="provision remote workdir and Python virtualenv for training")
bootstrap.add_argument("--host", required=True)
bootstrap.add_argument("--user")
bootstrap.add_argument("--port", type=int, default=22)
bootstrap.add_argument("--password-env")
bootstrap.add_argument("--workdir", required=True)
bootstrap.add_argument("--min-python", default="3.11")
bootstrap.add_argument("--python-candidate", action="append", default=[])
bootstrap.add_argument("--venv-name", default=".venv")
bootstrap.add_argument("--recreate-venv", action="store_true")
bootstrap.add_argument("--out", default="outputs/remote-bootstrap")
bootstrap.set_defaults(func=cmd_bootstrap_remote_host)
sync_remote = subparsers.add_parser("sync-remote-workspace", help="copy the current workspace to a remote host workdir")
sync_remote.add_argument("--host", required=True)
sync_remote.add_argument("--user")
sync_remote.add_argument("--port", type=int, default=22)
sync_remote.add_argument("--password-env")
sync_remote.add_argument("--local-root", default=".")
sync_remote.add_argument("--remote-workdir", required=True)
sync_remote.add_argument("--exclude", action="append", default=[])
sync_remote.add_argument("--out", default="outputs/remote-sync")
sync_remote.set_defaults(func=cmd_sync_remote_workspace)
review = subparsers.add_parser("review-claim", help="audit a full SOTA claim bundle across multiple experiment reports")
review.add_argument("--experiment", action="append", required=True, help="experiment.json path; repeatable")
review.add_argument("--config")
review.add_argument("--model-config")
review.add_argument("--out", default="outputs/claim-review")
review.set_defaults(func=cmd_review_claim)
ablation_audit = subparsers.add_parser("audit-augmentation-ablation", help="audit a synthetic augmentation ablation plan before model promotion")
ablation_audit.add_argument("config", help="augmentation ablation config JSON")
ablation_audit.add_argument("--real-review-audit", action="append", default=[], help="passed Gorkhapatra language-page review audit JSON; repeatable")
ablation_audit.add_argument("--out", default="outputs/augmentation-ablation-audit")
ablation_audit.set_defaults(func=cmd_audit_augmentation_ablation)
validate = subparsers.add_parser("validate-claim", help="validate whether benchmark evidence supports a SOTA claim")
validate.add_argument("--benchmark-report", required=True, help="benchmark report.json path")
validate.add_argument("--out", default="outputs/validation", help="validation report directory")
validate.add_argument("--config", help="validation gates JSON")
validate.add_argument("--eval-manifest", help="held-out evaluation manifest")
validate.add_argument("--train-manifest", action="append", default=[], help="training manifest to check for leakage; repeatable")
validate.add_argument("--candidate-model-config", help="candidate model card path to record in validation provenance")
validate.add_argument("--fallback-model-config", help="fallback model card path to record in validation provenance")
validate.set_defaults(func=cmd_validate_claim)
return parser
def cmd_prepare_data(args: argparse.Namespace) -> int:
if args.hf:
path = prepare_hf_dataset(
args.source,
args.out,
dataset=args.dataset_name,
split=args.split,
image_field=args.image_field,
text_field=args.text_field,
hf_subset=args.hf_subset,
limit=args.limit,
skip_invalid=args.skip_invalid,
strict_chars=args.strict_chars,
slices=args.slice,
)
else:
path = prepare_local_dataset(
args.source,
args.out,
dataset=args.dataset_name,
split=args.split,
image_field=args.image_field,
text_field=args.text_field,
limit=args.limit,
skip_invalid=args.skip_invalid,
strict_chars=args.strict_chars,
slices=args.slice,
)
print(path.manifest_path)
print(path.summary_json_path)
print(path.summary_md_path)
if path.rejects_path:
print(path.rejects_path)
print(f"prepared={path.sample_count} rejected={path.rejected_count}")
return 0
def cmd_inspect_hf_dataset(args: argparse.Namespace) -> int:
report = inspect_hf_dataset(
args.dataset_id,
args.out,
split=args.split,
hf_subset=args.hf_subset,
image_field=args.image_field,
text_field=args.text_field,
limit=args.limit,
)
print(report.output_json_path)
print(report.output_md_path)
print(f"passed={report.passed} inspected={report.inspected_count}")
return 0 if report.passed else 1
def cmd_audit_manifest(args: argparse.Namespace) -> int:
audit = audit_manifest(
args.manifest,
args.out,
verify_hashes=not args.no_verify_hashes,
require_slices=args.require_slice,
strict_chars=args.strict_chars,
)
print(Path(args.out) / "manifest-audit.json")
print(Path(args.out) / "manifest-audit.md")
print(f"passed={audit.passed} samples={audit.sample_count}")
return 0 if audit.passed else 1
def cmd_split_manifest(args: argparse.Namespace) -> int:
summary = split_manifest(
args.manifest,
args.out,
eval_ratio=args.eval_ratio,
seed=args.seed,
stratify_by=args.stratify_by,
group_by=args.group_by,
)
print(summary.train_manifest)
print(summary.eval_manifest)
print(f"train={summary.train_count} eval={summary.eval_count} leakage_passed={summary.leakage_passed}")
return 0
def cmd_rebalance_manifest(args: argparse.Namespace) -> int:
summary = rebalance_manifest(args.manifest, args.out, slices=args.slice, target_count=args.target_count, seed=args.seed)
out_path = Path(args.out)
print(summary.output_manifest)
print(out_path.with_name(f"{out_path.stem}-rebalance.json"))
print(out_path.with_name(f"{out_path.stem}-rebalance.md"))
print(f"original={summary.original_count} rebalanced={summary.rebalanced_count} target={summary.target_count}")
return 0
def cmd_audit_recognizer_corpus(args: argparse.Namespace) -> int:
audit = audit_recognizer_corpus(
args.train_manifest,
args.eval_manifest,
args.out,
min_train_samples=args.min_train_samples,
min_eval_samples=args.min_eval_samples,
min_train_english=args.min_train_english,
min_train_devanagari=args.min_train_devanagari,
min_train_mixed=args.min_train_mixed,
min_eval_english=args.min_eval_english,
min_eval_devanagari=args.min_eval_devanagari,
min_eval_mixed=args.min_eval_mixed,
min_train_latin_only=args.min_train_latin_only,
min_train_devanagari_only=args.min_train_devanagari_only,
min_eval_latin_only=args.min_eval_latin_only,
min_eval_devanagari_only=args.min_eval_devanagari_only,
min_train_real=args.min_train_real,
min_train_synthetic=args.min_train_synthetic,
require_eval_real=args.require_eval_real,
)
print(Path(args.out) / "recognizer-corpus-audit.json")
print(Path(args.out) / "recognizer-corpus-audit.md")
print(f"passed={audit.passed} train={audit.train_profile.sample_count} eval={audit.eval_profile.sample_count}")
return 0 if audit.passed else 1
def cmd_build_recognizer_corpus(args: argparse.Namespace) -> int:
summary = build_recognizer_corpus(
args.manifests,
args.out,
target_latin_only=args.target_latin_only,
target_devanagari_only=args.target_devanagari_only,
target_mixed=args.target_mixed,
target_real=args.target_real,
target_synthetic=args.target_synthetic,
seed=args.seed,
)
out_path = Path(args.out)
print(summary.output_manifest)
print(out_path.with_name(f"{out_path.stem}-recognizer-corpus-build.json"))
print(out_path.with_name(f"{out_path.stem}-recognizer-corpus-build.md"))
print(f"source={summary.source_count} output={summary.output_count}")
return 0
def cmd_merge_manifests(args: argparse.Namespace) -> int:
summary = merge_manifests(args.manifests, args.out)
out_path = Path(args.out)
print(summary.output_manifest)
print(out_path.with_name(f"{out_path.stem}-merge.json"))
print(out_path.with_name(f"{out_path.stem}-merge.md"))
print(f"samples={summary.sample_count} inputs={len(summary.input_manifests)} warnings={len(summary.warnings)}")
return 0
def cmd_filter_manifest(args: argparse.Namespace) -> int:
summary = filter_manifest(
args.manifest,
args.out,
sample_ids=args.sample_id,
slices=args.slice,
document_types=args.document_type,
degradations=args.degradation,
limit=args.limit,
)
out_path = Path(args.out)
print(summary.output_manifest)
print(out_path.with_name(f"{out_path.stem}-filter.json"))
print(out_path.with_name(f"{out_path.stem}-filter.md"))
print(f"selected={summary.selected_count} source={summary.source_count}")
return 0
def cmd_materialize_references(args: argparse.Namespace) -> int:
report = materialize_references(args.manifest, args.out, rewritten_manifest_path=args.rewritten_manifest)
print(report.references_dir)
if report.rewritten_manifest:
print(report.rewritten_manifest)
print(report.summary_json_path)
print(report.summary_md_path)
print(f"samples={report.sample_count}")
return 0
def cmd_audit_references(args: argparse.Namespace) -> int:
report = audit_references(
args.manifest,
args.out,
require_claim_ready=args.require_claim_ready,
)
print(report.summary_json_path)
print(report.summary_md_path)
print(
f"passed={report.passed} samples={report.sample_count} parsed={report.parsed_count} "
f"issues={len(report.issues)} warnings={len(report.warnings)}"
)
return 0 if report.passed else 1
def cmd_normalize_manifest_images(args: argparse.Namespace) -> int:
report = normalize_manifest_images(
args.manifest,
args.out,
image_dir_name=args.image_dir_name,
output_manifest_name=args.output_manifest_name,
)
print(report.manifest_path)
print(report.summary_json_path)
print(report.summary_md_path)
print(f"samples={report.sample_count} copied={report.copied_count} converted={report.converted_count}")
return 0
def cmd_create_eval_pack(args: argparse.Namespace) -> int:
summary = create_eval_pack(
args.out,
count_per_template=args.count_per_template,
input_format=args.input_format,
degradations=args.degradation,
seed=args.seed,
font_path=args.font_path,
templates=args.template or None,
variant_offset=args.variant_offset,
)
print(summary.manifest_path)
print(f"samples={summary.sample_count} claim_ready={summary.claim_ready}")
return 0
def cmd_audit_eval_pack(args: argparse.Namespace) -> int:
audit = audit_eval_pack(args.manifest, args.out)
print(Path(args.out) / "eval-pack-audit.json")
print(Path(args.out) / "eval-pack-audit.md")
print(f"passed={audit.passed} claim_ready={audit.claim_ready}")
if not audit.passed or (args.require_claim_ready and not audit.claim_ready):
return 1
return 0
def cmd_extract_crops(args: argparse.Namespace) -> int:
summary = extract_crop_manifest(
args.manifest,
args.out,
crop_types=args.crop_type or None,
split=args.split,
dataset_name=args.dataset_name,
min_text_length=args.min_text_length,
)
print(summary.manifest_path)
print(Path(args.out) / "crop-manifest-summary.json")
print(Path(args.out) / "crop-manifest-summary.md")
print(f"crops={summary.crop_count} skipped={summary.skipped_count}")
return 0
def cmd_train_recognizer(args: argparse.Namespace) -> int:
if args.backend == "hf-vision-encoder-decoder":
if args.failure_review:
assert_recognizer_family_not_stopped(args.failure_review, backend="hf_vision_encoder_decoder", base_model=args.base_model)
path = write_hf_recognizer_recipe(
args.train_manifest,
args.eval_manifest,
args.out,
base_model=args.base_model,
max_target_length=args.max_target_length,
per_device_train_batch_size=args.train_batch_size,
per_device_eval_batch_size=args.eval_batch_size,
learning_rate=args.learning_rate,
num_train_epochs=args.epochs,
run=args.run,
)
else:
path = write_recognizer_recipe(
args.train_manifest,
args.eval_manifest,
args.out,
base_model=args.base_model,
dictionary_path=args.dictionary,
train_batch_size=args.train_batch_size,
eval_batch_size=args.eval_batch_size,
learning_rate=args.learning_rate,
epochs=args.epochs,
warmup_epoch=args.warmup_epoch,
use_gpu=args.use_gpu,
train_num_workers=args.num_workers,
eval_num_workers=args.num_workers,
train_drop_last=args.train_drop_last,
main_indicator=args.main_indicator,
eval_batch_step=args.eval_batch_step,
failure_review=args.failure_review,
run=args.run,
)
print(path)
return 0
def cmd_export_recognizer(args: argparse.Namespace) -> int:
path = write_recognizer_export_recipe(
args.training_config,
args.checkpoint,
args.out,
inference_dir=args.inference_dir,
paddleocr_dir=args.paddleocr_dir,
export_options=args.export_option,
run=args.run,
)
print(path)
return 0
def cmd_package_recognizer(args: argparse.Namespace) -> int:
path = package_paddleocr_model(
args.inference_dir,
args.out,
model_id=args.model_id,
dictionary_path=args.dictionary,
base_model=args.base_model,
paddle_lang=args.paddle_lang,
text_recognition_model_name=args.text_recognition_model_name,
train_manifest=args.train_manifest,
eval_manifest=args.eval_manifest,
training_summary=args.training_summary,
metrics_report=args.metrics_report,
admission_validation_report=args.admission_validation_report,
recognition_mode=args.recognition_mode,
line_mode_max_height=args.line_mode_max_height,
line_mode_min_aspect_ratio=args.line_mode_min_aspect_ratio,
source_archive=args.source_archive,
source_checkpoint=args.source_checkpoint,
source_training_config=args.source_training_config,
export_recipe=args.export_recipe,
)
print(path)
return 0
def cmd_bundle_recognizer_training(args: argparse.Namespace) -> int:
bundle = bundle_recognizer_training(
args.recipe_dir,
args.out,
train_manifest=args.train_manifest,
eval_manifest=args.eval_manifest,
archive_name=args.archive_name,
base_dir=args.base_dir,
require_corpus_audit=args.require_corpus_audit,
min_train_samples=args.min_train_samples,
min_eval_samples=args.min_eval_samples,
min_train_english=args.min_train_english,
min_train_devanagari=args.min_train_devanagari,
min_train_mixed=args.min_train_mixed,
min_eval_english=args.min_eval_english,
min_eval_devanagari=args.min_eval_devanagari,
min_eval_mixed=args.min_eval_mixed,
min_train_latin_only=args.min_train_latin_only,
min_train_devanagari_only=args.min_train_devanagari_only,
min_eval_latin_only=args.min_eval_latin_only,
min_eval_devanagari_only=args.min_eval_devanagari_only,
min_train_real=args.min_train_real,
min_train_synthetic=args.min_train_synthetic,
require_eval_real=args.require_eval_real,
failure_review=args.failure_review,
)
print(bundle.archive_path)
print(bundle.manifest_path)
print(bundle.readme_path)
print(f"files={bundle.file_count} bytes={bundle.total_bytes}")
return 0
def cmd_package_hf_recognizer(args: argparse.Namespace) -> int:
path = package_hf_recognizer_model(
args.model_dir,
args.out,
model_id=args.model_id,
base_model=args.base_model,
train_manifest=args.train_manifest,
eval_manifest=args.eval_manifest,
training_summary=args.training_summary,
metrics_report=args.metrics_report,
admission_validation_report=args.admission_validation_report,
max_new_tokens=args.max_new_tokens,
)
print(path)
return 0
def cmd_finalize_hf_recognizer_run(args: argparse.Namespace) -> int:
result = finalize_hf_recognizer_run(
args.run_dir,
output_dir=args.out,
model_id=args.model_id,
base_model=args.base_model,
train_manifest=args.train_manifest,
eval_manifest=args.eval_manifest,
training_summary=args.training_summary,
admission_validation_report=args.admission_validation_report,
max_new_tokens=args.max_new_tokens,
)
print(result.model_card_path)
print(result.audit_path)
print(result.report_json_path)
print(result.report_md_path)
print(f"audit_passed={result.audit_passed}")
return 0 if result.audit_passed else 1
def cmd_package_hf_corrector(args: argparse.Namespace) -> int:
path = package_hf_text_corrector_model(
args.model_dir,
args.out,
model_id=args.model_id,
base_model=args.base_model,
base_engine=args.base_engine,
base_engine_kwargs=_parse_key_values(args.base_engine_kwarg),
train_manifest=args.train_manifest,
eval_manifest=args.eval_manifest,
metrics_report=args.metrics_report,
max_new_tokens=args.max_new_tokens,
)
print(path)
return 0
def cmd_generate_correction_pairs(args: argparse.Namespace) -> int:
path = generate_correction_pairs(
args.manifest,
args.out,
engine_name=args.engine,
model_config=args.model_config,
limit=args.limit,
only_errors=args.only_errors,
)
print(path)
return 0
def cmd_train_corrector(args: argparse.Namespace) -> int:
recipe_path = write_corrector_recipe(
args.out,
clean_manifest=args.clean_manifest,
pairs_source=args.pairs_source,
hf_dataset=args.hf_dataset,
hf_split=args.hf_split,
limit=args.limit,
seed=args.seed,
)
if args.backend == "recipe":
print(recipe_path)
return 0
path = write_hf_text_corrector_recipe(
Path(args.out) / "correction_pairs.jsonl",
args.out,
base_model=args.base_model,
max_source_length=args.max_source_length,
max_target_length=args.max_target_length,
per_device_train_batch_size=args.train_batch_size,
per_device_eval_batch_size=args.eval_batch_size,
learning_rate=args.learning_rate,
num_train_epochs=args.epochs,
eval_ratio=args.eval_ratio,
seed=args.seed,
run=args.run,
)
print(path)
return 0
def cmd_create_model_card(args: argparse.Namespace) -> int:
path = write_model_card(
args.out,
model_id=args.model_id,
backend=args.backend,
base_model=args.base_model,
artifact_paths=args.artifact,
backend_kwargs=_parse_key_values([*args.backend_kwarg, *args.paddleocr_kwarg]),
train_manifest=args.train_manifest,
eval_manifest=args.eval_manifest,
metrics_report=args.metrics_report,
admission_validation_report=args.admission_validation_report,
fallback_model_card=args.fallback_model_card,
notes=args.notes,
)
print(path)
return 0
def cmd_audit_model(args: argparse.Namespace) -> int:
audit = audit_model_card(args.model_config, args.out)
print(Path(args.out) / "model-audit.json")
print(Path(args.out) / "model-audit.md")
print(f"passed={audit.passed} model_id={audit.model_id}")
return 0 if audit.passed else 1
def cmd_parse_doc(args: argparse.Namespace) -> int:
document = parse_document(
args.input,
args.out,
engine_name=args.engine,
model_config=args.model_config,
fallback_engine=args.fallback_engine,
fallback_model_config=args.fallback_model_config,
low_confidence_threshold=args.low_confidence_threshold,
fallback_min_quality_score=args.fallback_min_quality_score,
)
print(Path(args.out) / "document.json")
print(Path(args.out) / "document.md")
print(f"pages={len(document.pages)} tables={len(document.tables)} figures={len(document.figures)}")
return 0
def cmd_parse_limbu_doc(args: argparse.Namespace) -> int:
document = parse_limbu_document(
args.input,
args.out,
engine_name=args.engine,
model_config=args.model_config,
fallback_engine=args.fallback_engine,
fallback_model_config=args.fallback_model_config,
low_confidence_threshold=args.low_confidence_threshold,
sirijonga_low_confidence_threshold=args.sirijonga_low_confidence_threshold,
devanagari_low_confidence_threshold=args.devanagari_low_confidence_threshold,
mixed_script_low_confidence_threshold=args.mixed_script_low_confidence_threshold,
other_script_low_confidence_threshold=args.other_script_low_confidence_threshold,
fallback_min_quality_score=args.fallback_min_quality_score,
script_ratio_threshold=args.script_ratio_threshold,
capture_prep_metadata=args.capture_prep_metadata,
post_correction_profile=args.post_correction_profile,
line_detection_mode=args.line_detection_mode,
image_line_threshold=args.image_line_threshold,
image_line_bbox_source=args.image_line_bbox_source,
image_line_reading_order=args.image_line_reading_order,
image_line_horizontal_kernel=args.image_line_horizontal_kernel,
image_line_vertical_kernel=args.image_line_vertical_kernel,
image_line_dilation_iterations=args.image_line_dilation_iterations,
image_line_min_width=args.image_line_min_width,
image_line_min_height=args.image_line_min_height,
image_line_min_area=args.image_line_min_area,
image_line_max_height=args.image_line_max_height,
image_line_min_aspect_ratio=args.image_line_min_aspect_ratio,
image_line_max_aspect_ratio=args.image_line_max_aspect_ratio,
image_line_detector_padding=args.image_line_detector_padding,
image_line_crop_padding=args.image_line_crop_padding,
image_line_rescue_detector_passes=_parse_image_line_rescue_detector_passes(args.image_line_rescue_detector_pass),
image_line_merge_iou_threshold=args.image_line_merge_iou_threshold,
image_line_split_tall_components=args.image_line_split_tall_components,
image_line_split_tall_row_min_ink=args.image_line_split_tall_row_min_ink,
image_line_split_tall_max_row_gap=args.image_line_split_tall_max_row_gap,
image_line_split_wide_components=args.image_line_split_wide_components,
image_line_split_wide_col_min_ink=args.image_line_split_wide_col_min_ink,
image_line_split_wide_max_col_gap=args.image_line_split_wide_max_col_gap,
image_line_split_wide_min_width=args.image_line_split_wide_min_width,
image_line_split_detected_row_components=args.image_line_split_detected_row_components,
image_line_split_detected_row_col_min_ink=args.image_line_split_detected_row_col_min_ink,
image_line_split_detected_row_max_col_gap=args.image_line_split_detected_row_max_col_gap,
image_line_split_detected_row_min_width=args.image_line_split_detected_row_min_width,
image_line_split_detected_row_min_segment_width=args.image_line_split_detected_row_min_segment_width,
image_line_split_detected_tall_components=args.image_line_split_detected_tall_components,
image_line_split_detected_tall_row_min_ink=args.image_line_split_detected_tall_row_min_ink,
image_line_split_detected_tall_max_row_gap=args.image_line_split_detected_tall_max_row_gap,
image_line_split_detected_tall_min_height=args.image_line_split_detected_tall_min_height,
image_line_split_detected_tall_min_segment_height=args.image_line_split_detected_tall_min_segment_height,
image_line_merge_same_row_components=args.image_line_merge_same_row_components,
image_line_merge_same_row_y_tolerance=args.image_line_merge_same_row_y_tolerance,
image_line_merge_same_row_max_gap=args.image_line_merge_same_row_max_gap,
image_line_merge_same_row_max_center_delta=args.image_line_merge_same_row_max_center_delta,
image_line_merge_same_row_max_width=args.image_line_merge_same_row_max_width,
image_line_merge_same_row_auto_fragmented_top_to_bottom=args.image_line_merge_same_row_auto_fragmented_top_to_bottom,
image_line_merge_same_row_auto_min_reduction_ratio=args.image_line_merge_same_row_auto_min_reduction_ratio,
image_line_merge_same_row_auto_min_reduction_count=args.image_line_merge_same_row_auto_min_reduction_count,
image_line_allow_empty_lines=args.image_line_allow_empty_lines,
image_line_filter_drop_empty=args.image_line_filter_drop_empty,
image_line_filter_min_confidence=args.image_line_filter_min_confidence,
image_line_filter_require_script=args.image_line_filter_require_script,
image_line_filter_min_width_ratio=args.image_line_filter_min_width_ratio,
image_line_filter_max_width_ratio=args.image_line_filter_max_width_ratio,
image_line_filter_min_height_ratio=args.image_line_filter_min_height_ratio,
image_line_filter_max_height_ratio=args.image_line_filter_max_height_ratio,
run_context=_cli_run_context(args),
)
audit = document.metadata.get("limbu_pipeline", {})
review_count = audit.get("review_line_count") if isinstance(audit, dict) else None
print(Path(args.out) / "document.json")
print(Path(args.out) / "document.md")
print(Path(args.out) / "limbu-pipeline-audit.json")
print(Path(args.out) / "limbu-post-correction-audit.json")
print(Path(args.out) / "limbu-review-queue.tsv")
print(Path(args.out) / "limbu-review-dashboard.html")
print(Path(args.out) / "limbu-output-manifest.json")
print(f"pages={len(document.pages)} tables={len(document.tables)} figures={len(document.figures)} review_lines={review_count}")
return 0
def cmd_prepare_limbu_capture(args: argparse.Namespace) -> int:
result = prepare_limbu_capture(
args.input,
args.out,
rotate_degrees=args.rotate_degrees,
crop_box=parse_crop_box(args.crop_box),
perspective_quad=parse_perspective_quad(args.perspective_quad),
auto_detect_page=args.auto_detect_page,
auto_deskew=args.auto_deskew,
max_auto_deskew_degrees=args.max_auto_deskew_degrees,
autocontrast=not args.no_autocontrast,
grayscale=not args.no_grayscale,
run_context=_cli_run_context(args),
)
print(result.prepared_image_path)
print(result.metadata_path)
return 0
def cmd_audit_limbu_capture(args: argparse.Namespace) -> int:
report = audit_limbu_capture(
args.capture_metadata,
args.out,
require_metadata_path_self=args.require_metadata_path_self,
min_prepared_width=args.min_prepared_width,
min_prepared_height=args.min_prepared_height,
min_prepared_entropy=args.min_prepared_entropy,
min_prepared_luminance_stddev=args.min_prepared_luminance_stddev,
min_prepared_edge_stddev=args.min_prepared_edge_stddev,
)
report_dir = Path(args.out) if args.out else (Path(args.capture_metadata) if Path(args.capture_metadata).is_dir() else Path(args.capture_metadata).parent)
print(report_dir / "limbu-capture-audit.json")
print(report_dir / "limbu-capture-audit.md")
print(f"passed={report['passed']}")
return 0 if report["passed"] else 1
def cmd_preflight_limbu_pipeline(args: argparse.Namespace) -> int:
report = preflight_limbu_pipeline_runtime(
args.model_config,
args.out,
required_script=args.required_script,
required_tesseract_languages=tuple(args.required_tesseract_language or ["nep", "eng"]),
require_validated_component_admissions=args.require_validated_components,
required_component_roles=tuple(args.required_component_role or ()),
)
print(Path(args.out) / "limbu-runtime-preflight.json")
print(Path(args.out) / "limbu-runtime-preflight.md")
print(f"passed={report.passed}")
return 0 if report.passed else 1
def cmd_apply_limbu_review_corrections(args: argparse.Namespace) -> int:
document = apply_limbu_review_corrections(
args.document,
args.review_queue,
args.out,
accepted_statuses=tuple(args.accepted_status or ("accepted", "approved", "corrected", "verified", "done")),
run_context=_cli_run_context(args),
)
audit = document.metadata.get("limbu_post_correction", {})
applied_count = audit.get("applied_count") if isinstance(audit, dict) else None
print(Path(args.out) / "document.json")
print(Path(args.out) / "document.md")
print(Path(args.out) / "limbu-correction-audit.json")
print(Path(args.out) / "limbu-correction-pairs.jsonl")
print(Path(args.out) / "limbu-output-manifest.json")
print(f"applied_corrections={applied_count}")
return 0
def cmd_build_limbu_correction_pair_pack(args: argparse.Namespace) -> int:
report = build_limbu_correction_pair_pack(
args.correction_pairs,
args.out,
pack_id=args.pack_id,
heldout_fraction=args.heldout_fraction,
min_heldout=args.min_heldout,
)
print(Path(args.out) / "limbu-correction-pair-pack.json")
print(Path(args.out) / "limbu-correction-pair-pack.md")
print(report["train_pairs"])
print(report["heldout_pairs"])
print(f"pack_content_sha256={report['pack_content_sha256']}")
print(f"train={report['train_count']} heldout={report['heldout_count']}")
return 0
def cmd_audit_limbu_correction_pair_pack(args: argparse.Namespace) -> int:
report = audit_limbu_correction_pair_pack(args.pack_dir, args.out)
audit_dir = Path(args.out) if args.out else Path(args.pack_dir)
print(audit_dir / "limbu-correction-pair-pack-audit.json")
print(audit_dir / "limbu-correction-pair-pack-audit.md")
print(f"passed={report['passed']}")
return 0 if report["passed"] else 1
def cmd_derive_limbu_post_correction_profile(args: argparse.Namespace) -> int:
result = derive_limbu_post_correction_profile(
args.correction_pairs,
args.out,
profile_id=args.profile_id,
min_support=args.min_support,
)
derivation = result.get("derivation", {})
rule_count = derivation.get("rule_count") if isinstance(derivation, dict) else None
print(Path(args.out) / "limbu-post-correction-profile.json")
print(Path(args.out) / "limbu-post-correction-profile-derivation.json")
print(Path(args.out) / "limbu-post-correction-profile-derivation.md")
print(f"rules={rule_count}")
return 0
def cmd_score_limbu_post_correction_profile(args: argparse.Namespace) -> int:
report = score_limbu_post_correction_profile(
args.profile,
args.correction_pairs,
args.out,
frozen_eval_pack=args.frozen_eval_pack,
)
print(Path(args.out) / "limbu-post-correction-profile-eval.json")
print(Path(args.out) / "limbu-post-correction-profile-eval.md")
print(Path(args.out) / "limbu-post-correction-profile-eval-lines.jsonl")
print(f"passed={report['passed']}")
print(f"metric={report['metric']} before={report['before']} after={report['after']}")
print(f"eval_run_sha256={report['eval_run_sha256']}")
return 0 if report["passed"] else 1
def cmd_admit_limbu_post_correction_profile(args: argparse.Namespace) -> int:
report = admit_limbu_post_correction_profile(
args.profile,
args.eval_run,
args.out,
output_filename=args.output_filename,
)
print(report["admitted_profile"])
print(Path(args.out) / "limbu-post-correction-profile-admission.json")
print(Path(args.out) / "limbu-post-correction-profile-admission.md")
print(Path(args.out) / "audit" / "limbu-post-correction-profile-audit.json")
print(f"audit_passed={report['audit_passed']} claim_ready={report['audit_claim_ready']}")
return 0 if report["audit_passed"] and report["audit_claim_ready"] else 1
def cmd_audit_limbu_post_correction_profile(args: argparse.Namespace) -> int:
report = audit_limbu_post_correction_profile(args.profile, args.out)
print(Path(args.out) / "limbu-post-correction-profile-audit.json")
print(Path(args.out) / "limbu-post-correction-profile-audit.md")
print(f"passed={report['passed']} claim_ready={report['claim_ready']}")
return 0 if report["passed"] else 1
def cmd_audit_limbu_output(args: argparse.Namespace) -> int:
required_script_counts = _parse_limbu_required_script_counts(args.require_script_count or ())
report = audit_limbu_output(
args.output_dir,
args.out,
require_capture_prep=args.require_capture_prep,
min_capture_prepared_width=args.min_capture_prepared_width,
min_capture_prepared_height=args.min_capture_prepared_height,
min_capture_prepared_entropy=args.min_capture_prepared_entropy,
min_capture_prepared_luminance_stddev=args.min_capture_prepared_luminance_stddev,
min_capture_prepared_edge_stddev=args.min_capture_prepared_edge_stddev,
require_capture_metadata_path_self=args.require_capture_metadata_path_self,
require_no_pending_review=args.require_no_pending_review,
require_reviewer_for_corrections=args.require_reviewer_for_corrections,
require_no_dropped_image_lines=args.require_no_dropped_image_lines,
min_line_count=args.min_line_count,
min_average_line_confidence=args.min_average_line_confidence,
min_quality_score=args.min_quality_score,
required_scripts=tuple(args.require_script or ()),
required_script_counts=required_script_counts,
)
report_dir = Path(args.out) if args.out else Path(args.output_dir)
print(report_dir / "limbu-output-audit.json")
print(report_dir / "limbu-output-audit.md")
print(f"passed={report['passed']}")
return 0 if report["passed"] else 1
def _parse_limbu_required_script_counts(items: list[str] | tuple[str, ...]) -> dict[str, int]:
allowed = {"limbu_sirijonga", "devanagari_limbu", "mixed_limbu_devanagari", "other"}
parsed: dict[str, int] = {}
for item in items:
if "=" not in item:
raise OcrTechError(f"--require-script-count must be SCRIPT=COUNT, got {item!r}")
script, raw_count = item.split("=", 1)
script = script.strip()
raw_count = raw_count.strip()
if script not in allowed:
raise OcrTechError(f"unsupported --require-script-count script {script!r}; allowed values are {sorted(allowed)}")
try:
count = int(raw_count)
except ValueError as exc:
raise OcrTechError(f"--require-script-count count must be an integer, got {raw_count!r}") from exc
if count < 1:
raise OcrTechError(f"--require-script-count count must be positive, got {count}")
parsed[script] = count
return dict(sorted(parsed.items()))
def cmd_benchmark(args: argparse.Namespace) -> int:
baselines = [item.strip() for item in args.baselines.split(",") if item.strip()]
inputs = list(args.inputs)
if args.inputs_from_manifest:
inputs.extend(_benchmark_inputs_from_manifest(args.inputs_from_manifest))
if not inputs:
raise OcrTechError("benchmark requires positional inputs or --inputs-from-manifest")
eval_manifest = args.eval_manifest or args.inputs_from_manifest
results = run_benchmark(
inputs,
args.out,
baselines=baselines,
references_dir=args.references_dir,
eval_manifest=eval_manifest,
candidate_model_config=args.candidate_model_config,
fallback_engine=args.fallback_engine,
fallback_model_config=args.fallback_model_config,
low_confidence_threshold=args.low_confidence_threshold,
fallback_min_quality_score=args.fallback_min_quality_score,
resume_existing=args.resume_existing,
sample_timeout_seconds=args.sample_timeout_seconds,
capture_gpu_metrics=args.capture_gpu_metrics,
)
print(Path(args.out) / "report.json")
print(Path(args.out) / "report.md")
print(f"runs={len(results)}")
return 0
def _benchmark_inputs_from_manifest(manifest_path: str | Path) -> list[str]:
manifest = Path(manifest_path)
entries = load_manifest(manifest)
if not entries:
raise OcrTechError(f"benchmark input manifest is empty: {manifest}")
inputs: list[str] = []
for entry in entries:
raw_path = Path(entry.image_path)
if raw_path.is_absolute():
resolved = raw_path
elif raw_path.exists():
resolved = raw_path
else:
resolved = manifest.parent / raw_path
if not resolved.exists():
raise OcrTechError(f"benchmark input does not exist for sample {entry.sample_id}: {resolved}")
if resolved.is_dir():
raise OcrTechError(f"benchmark input is a directory for sample {entry.sample_id}: {resolved}")
inputs.append(str(resolved))
return inputs
def cmd_summarize_benchmark(args: argparse.Namespace) -> int:
summary = summarize_benchmark_report(
args.benchmark_report,
args.out,
candidate=args.candidate,
baselines=args.baseline or None,
metrics=args.metric or None,
)
print(Path(args.out) / "summary.json")
print(Path(args.out) / "summary.md")
print(f"candidate={summary.candidate} paired={len(summary.paired_metrics)}")
return 0
def cmd_rescore_benchmark(args: argparse.Namespace) -> int:
results = rescore_benchmark_report(
args.benchmark_report,
args.out,
references_dir=args.references_dir,
eval_manifest=args.eval_manifest,
)
print(Path(args.out) / "report.json")
print(Path(args.out) / "report.md")
print(f"runs={len(results)}")
return 0
def cmd_derive_table_postprocess_benchmark(args: argparse.Namespace) -> int:
results = derive_table_postprocess_benchmark(
args.benchmark_report,
args.out,
source_baseline=args.source_baseline,
candidate_baseline=args.candidate_baseline,
references_dir=args.references_dir,
eval_manifest=args.eval_manifest,
)
print(Path(args.out) / "report.json")
print(Path(args.out) / "report.md")
print(f"runs={len(results)}")
return 0
def cmd_analyze_table_cells(args: argparse.Namespace) -> int:
analysis = analyze_table_cells(
args.benchmark_report,
args.eval_manifest,
args.out,
candidate=args.candidate,
baselines=args.baseline or None,
top_n=args.top_n,
)
print(Path(args.out) / "table-cell-analysis.json")
print(Path(args.out) / "table-cell-analysis.md")
print(f"systems={len(analysis.systems)} paired={len(analysis.paired)} failures={len(analysis.top_failures)}")
return 0
def cmd_paper_benchmark_table(args: argparse.Namespace) -> int:
display_names: dict[str, str] = {}
for item in args.label:
key, separator, value = str(item).partition("=")
if not separator or not key or not value:
raise OcrTechError(f"invalid --label value {item!r}; expected system=Label")
display_names[key] = value
source_preferences: dict[str, str] = {}
for item in args.source_preference:
key, separator, value = str(item).partition("=")
if not separator or not key or not value:
raise OcrTechError(f"invalid --source-preference value {item!r}; expected system=path")
source_preferences[key] = value
table = export_paper_benchmark_table(
args.summary,
args.out,
slice_name=args.slice,
metrics=args.metric or None,
systems=args.system or None,
display_names=display_names,
source_preferences=source_preferences,
title=args.title,
)
print(Path(args.out) / "paper-table.json")
print(Path(args.out) / "paper-table.md")
print(f"rows={len(table.rows)} slice={table.slice_name}")
return 0
def cmd_benchmark_oracle(args: argparse.Namespace) -> int:
report, summary = build_benchmark_oracle(
args.benchmark_report,
args.out,
metric=args.metric,
systems=args.system or None,
direction=args.direction,
oracle_name=args.oracle_name,
)
print(Path(args.out) / "oracle-analysis.json")
print(Path(args.out) / "oracle-analysis.md")
print(Path(args.out) / "oracle-choices.json")
print(Path(args.out) / "oracle-summary" / "summary.json")
print(f"oracle={report.oracle_name} samples={report.sample_count} paired={len(summary.paired_metrics)}")
return 0
def cmd_benchmark_error_analysis(args: argparse.Namespace) -> int:
analysis = analyze_benchmark_errors(
args.benchmark_report,
args.out,
system=args.system,
baselines=args.baseline or None,
metric=args.metric,
direction=args.direction,
top_n=args.top_n,
)
print(Path(args.out) / "error-analysis.json")
print(Path(args.out) / "error-analysis.md")
print(Path(args.out) / "top-failures.json")
print(f"system={analysis.system} samples={analysis.sample_count} metric={analysis.metric}")
return 0
def cmd_export_failure_manifest(args: argparse.Namespace) -> int:
export = export_failure_manifest(
args.source_manifest,
args.top_failures,
args.out,
split=args.split,
)
print(export.output_manifest)
print(export.summary_path)
print(f"selected={export.selected_count} split={export.split or ''}")
return 0
def cmd_calibrate_quality_router(args: argparse.Namespace) -> int:
calibration, model_path = calibrate_quality_router(
args.experiment_report,
args.out,
model_id=args.model_id,
base_model=args.base_model,
primary_engine=args.primary_engine,
secondary_engine=args.secondary_engine,
primary_engine_kwargs=_parse_key_values(args.primary_engine_kwarg),
secondary_engine_kwargs=_parse_key_values(args.secondary_engine_kwarg),
candidate_baseline=args.candidate_baseline,
primary_baseline=args.primary_baseline,
metric=args.metric,
direction=args.direction,
threshold_start=args.threshold_start,
threshold_end=args.threshold_end,
threshold_step=args.threshold_step,
notes=args.notes,
)
print(Path(args.out) / "calibration.json")
print(Path(args.out) / "calibration.md")
print(model_path)
print(f"selected_threshold={calibration.selected_threshold:.6f}")
return 0
def cmd_calibrate_script_router(args: argparse.Namespace) -> int:
calibration, model_path = calibrate_script_router(
args.experiment_report,
args.out,
model_id=args.model_id,
base_model=args.base_model,
primary_engine=args.primary_engine,
secondary_engine=args.secondary_engine,
primary_engine_kwargs=_parse_key_values(args.primary_engine_kwarg),
secondary_engine_kwargs=_parse_key_values(args.secondary_engine_kwarg),
primary_baseline=args.primary_baseline,
secondary_baseline=args.secondary_baseline,
metric=args.metric,
direction=args.direction,
script=args.script,
threshold_start=args.threshold_start,
threshold_end=args.threshold_end,
threshold_step=args.threshold_step,
routing_granularity=args.routing_granularity,
secondary_structure_backfill=args.secondary_structure_backfill,
notes=args.notes,
)
print(Path(args.out) / "calibration.json")
print(Path(args.out) / "calibration.md")
print(model_path)
print(f"selected_threshold={calibration.selected_threshold:.6f}")
return 0
def cmd_calibrate_tesseract_psm_ensemble(args: argparse.Namespace) -> int:
report, model_path = calibrate_tesseract_psm_ensemble(
args.primary_experiment_report,
args.out,
model_id=args.model_id,
variant_experiment_reports=_parse_labeled_paths(args.variant_experiment),
metric=args.metric,
direction=args.direction,
bias_start=args.bias_start,
bias_end=args.bias_end,
bias_step=args.bias_step,
language=args.language,
selection_margin=args.selection_margin,
notes=args.notes,
)
print(Path(args.out) / "calibration.json")
print(Path(args.out) / "calibration.md")
print(model_path)
print(f"selected_biases={report.selected_biases}")
return 0
def cmd_run_experiment(args: argparse.Namespace) -> int:
baselines = [item.strip() for item in args.baselines.split(",") if item.strip()]
report = run_experiment(
args.eval_manifest,
args.out,
baselines=baselines,
references_dir=args.references_dir,
validation_config=args.validation_config,
candidate_model_config=args.candidate_model_config,
fallback_engine=args.fallback_engine,
fallback_model_config=args.fallback_model_config,
low_confidence_threshold=args.low_confidence_threshold,
fallback_min_quality_score=args.fallback_min_quality_score,
train_manifests=args.train_manifest,
gorkhapatra_review_audits=args.gorkhapatra_review_audit,
run_validation=not args.skip_validation,
run_preflight=args.preflight,
require_claim_ready_eval_pack=not args.allow_smoke_eval_pack,
require_trained_recognizer=args.require_trained_recognizer,
require_model_admission=args.require_model_admission,
)
print(Path(args.out) / "experiment.json")
print(Path(args.out) / "experiment.md")
print(f"benchmark={report.benchmark_report}")
print(f"validation={report.validation_status or 'not_run'}")
if args.require_validated and report.validation_status != "validated":
return 1
return 0
def cmd_preflight_claim(args: argparse.Namespace) -> int:
baselines = [item.strip() for item in args.baselines.split(",") if item.strip()]
report = run_claim_preflight(
args.eval_manifest,
args.out,
baselines=baselines,
references_dir=args.references_dir,
candidate_model_config=args.candidate_model_config,
fallback_model_config=args.fallback_model_config,
train_manifests=args.train_manifest,
validation_config=args.validation_config,
gorkhapatra_review_audits=args.gorkhapatra_review_audit,
require_claim_ready_eval_pack=not args.allow_smoke_eval_pack,
require_trained_recognizer=args.require_trained_recognizer,
require_model_admission=args.require_model_admission,
)
print(Path(args.out) / "preflight.json")
print(Path(args.out) / "preflight.md")
print(f"passed={report.passed} samples={report.sample_count}")
return 0 if report.passed else 1
def cmd_audit_remote_host(args: argparse.Namespace) -> int:
min_python = _parse_python_version(args.min_python)
report = audit_remote_host(
args.host,
args.out,
user=args.user,
port=args.port,
password_env=args.password_env,
min_python=min_python,
min_free_gb=args.min_free_gb,
workdir=args.workdir,
require_gpu=args.require_gpu,
require_paddle_training=args.require_paddle_training,
python_candidates=args.python_candidate or DEFAULT_PYTHON_CANDIDATES,
require_commands=args.require_command or None,
optional_commands=args.optional_command or None,
require_paths=args.require_path,
)
print(Path(args.out) / "remote-audit.json")
print(Path(args.out) / "remote-audit.md")
print(f"passed={report.passed} host={report.host}")
return 0 if report.passed else 1
def cmd_bootstrap_remote_host(args: argparse.Namespace) -> int:
min_python = _parse_python_version(args.min_python)
report = bootstrap_remote_host(
args.host,
args.out,
user=args.user,
port=args.port,
password_env=args.password_env,
workdir=args.workdir,
min_python=min_python,
python_candidates=args.python_candidate or DEFAULT_PYTHON_CANDIDATES,
venv_name=args.venv_name,
recreate_venv=args.recreate_venv,
)
print(Path(args.out) / "remote-bootstrap.json")
print(Path(args.out) / "remote-bootstrap.md")
print(f"passed={report.passed} workdir={report.workdir} venv={report.venv_dir}")
return 0 if report.passed else 1
def cmd_sync_remote_workspace(args: argparse.Namespace) -> int:
report = sync_remote_workspace(
args.local_root,
args.out,
host=args.host,
user=args.user,
port=args.port,
password_env=args.password_env,
remote_workdir=args.remote_workdir,
exclude_patterns=args.exclude or None,
)
print(Path(args.out) / "remote-sync.json")
print(Path(args.out) / "remote-sync.md")
print(f"passed={report.passed} remote_workdir={report.remote_workdir}")
return 0 if report.passed else 1
def cmd_review_claim(args: argparse.Namespace) -> int:
report = review_claim(
args.experiment,
args.out,
config_path=args.config,
model_config=args.model_config,
)
print(Path(args.out) / "claim-review.json")
print(Path(args.out) / "claim-review.md")
print(f"passed={report.passed} experiments={len(report.experiments)}")
return 0 if report.passed else 1
def cmd_audit_augmentation_ablation(args: argparse.Namespace) -> int:
report = audit_augmentation_ablation(
args.config,
args.out,
real_review_audits=args.real_review_audit,
)
print(Path(args.out) / "augmentation-ablation-audit.json")
print(Path(args.out) / "augmentation-ablation-audit.md")
print(
f"passed={report.passed} arms={report.arm_count} "
f"real_evidence={report.real_evidence_count} verified_references={report.verified_reference_count} "
f"issues={len(report.issues)} warnings={len(report.warnings)}"
)
return 0 if report.passed else 1
def cmd_validate_claim(args: argparse.Namespace) -> int:
report = validate_claim(
args.benchmark_report,
args.out,
config_path=args.config,
eval_manifest=args.eval_manifest,
train_manifests=args.train_manifest,
candidate_model_config=args.candidate_model_config,
fallback_model_config=args.fallback_model_config,
)
print(Path(args.out) / "validation.json")
print(Path(args.out) / "validation.md")
print(f"status={report.claim_status}")
if not report.passed:
return 1
return 0
def cmd_recognizer_admission_decision(args: argparse.Namespace) -> int:
decision = decide_recognizer_admission(args.validation_report, args.out)
print(Path(args.out) / "recognizer-admission-decision.json")
print(Path(args.out) / "recognizer-admission-decision.md")
print(f"decision={decision.decision} status={decision.claim_status}")
return 0 if decision.decision == "admit" else 1
def cmd_recognizer_failure_review(args: argparse.Namespace) -> int:
review = review_recognizer_failures(
args.admission_decision,
args.validation_report,
args.out,
model_cards=args.model_card,
min_rejections_to_stop=args.min_rejections_to_stop,
)
stopped_count = sum(1 for family in review.families if family.recommendation == "stop_family")
print(Path(args.out) / "recognizer-failure-review.json")
print(Path(args.out) / "recognizer-failure-review.md")
print(f"families={len(review.families)} stopped={stopped_count}")
return 1 if stopped_count else 0
def cmd_list_public_benchmarks(args: argparse.Namespace) -> int:
_ = args
for spec in list_public_benchmarks():
print(f"{spec.dataset}\t{spec.title}\tmin_samples={spec.min_samples}\t{spec.url}")
return 0
def cmd_prepare_public_benchmark(args: argparse.Namespace) -> int:
report = prepare_public_benchmark(
args.benchmark,
args.out,
limit=args.limit,
language=args.language,
subset=args.subset,
data_source=args.data_source,
require_tables=args.require_tables,
annotation_path=args.annotation_path,
local_image_root=args.local_image_root,
)
print(report.summary_json_path)
print(report.summary_md_path)
print(f"manifest={report.manifest_path} samples={report.sample_count} benchmark={report.benchmark}")
return 0
def cmd_audit_public_benchmark(args: argparse.Namespace) -> int:
audit = audit_public_benchmark_manifest(
args.manifest,
args.out,
benchmark=args.benchmark,
min_samples=args.min_samples,
require_reference_paths=not args.allow_missing_reference_paths,
)
print(Path(args.out) / "public-benchmark-audit.json")
print(Path(args.out) / "public-benchmark-audit.md")
print(f"passed={audit.passed} samples={audit.sample_count} benchmark={audit.benchmark}")
return 0 if audit.passed else 1
def cmd_audit_gorkhapatra_source(args: argparse.Namespace) -> int:
category_urls = list(args.category_url)
epaper_urls = list(args.epaper_url)
if not args.no_default_urls:
if not category_urls and not args.category_html:
category_urls = list(DEFAULT_GORKHAPATRA_CATEGORY_URLS)
if not epaper_urls and not args.epaper_html:
epaper_urls = [DEFAULT_GORKHAPATRA_EPAPER_URL]
audit = audit_gorkhapatra_source(
args.out,
category_urls=category_urls,
epaper_urls=epaper_urls,
category_html_paths=args.category_html,
epaper_html_paths=args.epaper_html,
download_assets=args.download_assets,
max_article_images=args.max_article_images,
max_epaper_pdfs=args.max_epaper_pdfs,
timeout_seconds=args.timeout_seconds,
)
print(Path(args.out) / "gorkhapatra-source-audit.json")
print(Path(args.out) / "gorkhapatra-source-audit.md")
print(f"articles={audit.article_count} epapers={audit.epaper_count} downloaded={audit.downloaded_count} warnings={len(audit.warnings)}")
return 0
def cmd_prepare_gorkhapatra_pack(args: argparse.Namespace) -> int:
summary = prepare_gorkhapatra_pack(
args.source_audit_json,
args.out,
include_article_images=not args.no_article_images,
include_epaper_pages=not args.no_epaper_pages,
max_article_images=args.max_article_images,
max_epaper_pdfs=args.max_epaper_pdfs,
max_pages_per_pdf=args.max_pages_per_pdf,
split=args.split,
dataset_name=args.dataset_name,
)
print(summary.manifest_path)
print(Path(args.out) / "gorkhapatra-pack.json")
print(Path(args.out) / "gorkhapatra-pack.md")
print(f"samples={summary.sample_count} article_images={summary.article_image_count} epaper_pages={summary.epaper_page_count}")
return 0
def cmd_audit_gorkhapatra_language_pages(args: argparse.Namespace) -> int:
publication_urls = list(args.publication_url)
epaper_urls = list(args.epaper_url)
if not args.no_default_urls:
if not publication_urls and not args.publication_html:
publication_urls = [DEFAULT_NAYA_NEPAL_PUBLICATION_URL]
if not epaper_urls and not args.epaper_html:
epaper_urls = [DEFAULT_GORKHAPATRA_EPAPER_URL]
publication_urls = _expand_publication_page_urls(publication_urls, args.publication_pages)
audit = audit_gorkhapatra_language_pages(
args.out,
publication_urls=publication_urls,
epaper_urls=epaper_urls,
publication_html_paths=args.publication_html,
epaper_html_paths=args.epaper_html,
download_pdfs=args.download_pdfs,
render_pages=args.render_pages,
max_publication_items=args.max_publication_items,
max_epaper_pdfs=args.max_epaper_pdfs,
timeout_seconds=args.timeout_seconds,
)
print(Path(args.out) / "gorkhapatra-language-pages.json")
print(Path(args.out) / "gorkhapatra-language-pages.md")
print(
f"publication_items={audit.publication_item_count} "
f"aligned={audit.aligned_count} language_page_hits={audit.language_page_hit_count} warnings={len(audit.warnings)}"
)
return 0
def _expand_publication_page_urls(urls: list[str], page_count: int) -> list[str]:
if page_count < 1:
raise OcrTechError("--publication-pages must be >= 1")
if page_count == 1:
return urls
expanded: list[str] = []
for url in urls:
expanded.extend(_publication_page_url(url, page) for page in range(1, page_count + 1))
return expanded
def _publication_page_url(url: str, page: int) -> str:
parsed = urlparse(url)
query = dict(parse_qsl(parsed.query, keep_blank_values=True))
if page == 1 and "page" not in query:
return url
query["page"] = str(page)
return urlunparse(parsed._replace(query=urlencode(query)))
def cmd_prepare_gorkhapatra_language_page_pack(args: argparse.Namespace) -> int:
summary = prepare_gorkhapatra_language_page_pack(
args.language_page_audit_json,
args.out,
split=args.split,
dataset_name=args.dataset_name,
max_samples=args.max_samples,
)
print(summary.manifest_path)
print(Path(args.out) / "gorkhapatra-language-page-pack.json")
print(Path(args.out) / "gorkhapatra-language-page-pack.md")
print(f"samples={summary.sample_count} copied_pages={summary.copied_page_count} warnings={len(summary.warnings)}")
return 0
def cmd_prepare_gorkhapatra_language_page_review(args: argparse.Namespace) -> int:
summary = prepare_gorkhapatra_language_page_review(
args.language_page_pack_manifest,
args.out,
)
print(summary.review_json_path)
print(summary.review_csv_path)
print(summary.review_md_path)
print(
f"samples={summary.sample_count} pending_review={summary.pending_review_count} "
f"claim_eligible={summary.claim_evidence_eligible_count} warnings={len(summary.warnings)}"
)
return 0
def cmd_audit_gorkhapatra_language_page_review(args: argparse.Namespace) -> int:
summary = audit_gorkhapatra_language_page_review(
args.language_page_pack_manifest,
args.review_csv,
args.out,
require_verified_references=args.require_verified_references,
candidate_languages=args.candidate_language,
)
print(summary.summary_json_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} groups={summary.group_count} "
f"accepted={summary.accepted_count} unresolved={summary.unresolved_count} issues={len(summary.issues)}"
)
return 0 if summary.passed else 1
def cmd_prepare_gorkhapatra_language_page_reference_templates(args: argparse.Namespace) -> int:
summary = prepare_gorkhapatra_language_page_reference_templates(
args.language_page_pack_manifest,
args.out,
review_csv_path=args.review_csv,
accepted_only=args.accepted_only,
)
print(summary.output_dir)
print(summary.index_csv_path)
print(summary.summary_json_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} templates={summary.sample_count} "
f"skipped={summary.skipped_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_prepare_gorkhapatra_language_page_reviewer_bundle(args: argparse.Namespace) -> int:
summary = prepare_gorkhapatra_language_page_reviewer_bundle(
args.language_page_pack_manifest,
args.review_csv,
args.out,
reference_template_dir=args.reference_template_dir,
)
print(summary.output_dir)
print(summary.index_csv_path)
print(summary.summary_json_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} "
f"images={summary.copied_image_count} references={summary.copied_reference_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_prepare_gorkhapatra_language_page_transcription_work_order(args: argparse.Namespace) -> int:
summary = prepare_gorkhapatra_language_page_transcription_work_order(
args.language_page_pack_manifest,
args.review_csv,
args.out,
candidate_languages=args.candidate_language,
)
print(summary.summary_json_path)
print(summary.index_csv_path)
print(summary.transcription_html_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} "
f"blocked={summary.blocked_count} verified={summary.verified_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_extract_gorkhapatra_language_page_pdf_text(args: argparse.Namespace) -> int:
summary = extract_gorkhapatra_language_page_pdf_text(
args.language_page_pack_manifest,
args.out,
candidate_languages=args.candidate_language,
target_font=args.target_font,
)
print(summary.summary_json_path)
print(summary.index_csv_path)
print(summary.spans_csv_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.extracted_sample_count}/{summary.sample_count} "
f"spans={summary.span_count} target_spans={summary.target_span_count} fonts={summary.font_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_convert_limbu_legacy(args: argparse.Namespace) -> int:
if bool(args.text) == bool(args.pdf_text_json):
raise OcrTechError("convert-limbu-legacy requires exactly one of --text or --pdf-text-json")
if args.text:
conversion = convert_limbu_legacy_text(args.text, args.map)
print(conversion.unicode_text)
print(
f"limbu_chars={conversion.limbu_char_count} replacements={conversion.replacement_count} "
f"unmapped={len(conversion.unmapped_codepoints)}"
)
return 0 if conversion.limbu_char_count > 0 else 1
result = convert_gorkhapatra_limbu_pdf_text(args.pdf_text_json, args.out, map_path=args.map)
print(result["json_path"])
print(result["text_path"])
print(
f"sample={result['sample_id']} spans={result['target_span_count']} "
f"limbu_chars={result['limbu_char_count']}"
)
return 0 if int(result["limbu_char_count"]) > 0 else 1
def cmd_prepare_gorkhapatra_language_page_assisted_references(args: argparse.Namespace) -> int:
summary = prepare_gorkhapatra_language_page_assisted_references(
args.language_page_pack_manifest,
args.review_csv,
args.out,
candidate_languages=args.candidate_language,
engine=args.engine,
tesseract_language=args.tesseract_language,
overwrite=args.overwrite,
)
print(summary.summary_json_path)
print(summary.index_csv_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} assisted={summary.assisted_count} "
f"failed={summary.failed_count} skipped={summary.skipped_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_prepare_gorkhapatra_language_page_pdf_native_references(args: argparse.Namespace) -> int:
summary = prepare_gorkhapatra_language_page_pdf_native_references(
args.language_page_pack_manifest,
args.review_csv,
args.pdf_text_dir,
args.out,
candidate_languages=args.candidate_language,
converter_id=args.converter,
map_path=args.map_path,
overwrite=args.overwrite,
)
print(summary.summary_json_path)
print(summary.index_csv_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} converted={summary.assisted_count} "
f"failed={summary.failed_count} skipped={summary.skipped_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_prepare_gorkhapatra_language_page_recognizer_eval(args: argparse.Namespace) -> int:
summary = prepare_gorkhapatra_language_page_recognizer_eval(
args.language_page_pack_manifest,
args.references_dir,
args.out,
candidate_languages=args.candidate_language,
split=args.split,
dataset_name=args.dataset_name,
min_text_length=args.min_text_length,
allow_draft=args.allow_draft,
)
print(summary.summary_json_path)
print(summary.manifest_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} crops={summary.crop_count} "
f"claim_eligible={summary.claim_eligible_crop_count} skipped={summary.skipped_count} "
f"warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_prepare_gorkhapatra_language_page_ocr_sidecars(args: argparse.Namespace) -> int:
summary = prepare_gorkhapatra_language_page_ocr_sidecars(
args.language_page_pack_manifest,
args.review_csv,
args.out,
candidate_languages=args.candidate_language,
engine=args.engine,
tesseract_language=args.tesseract_language,
overwrite=args.overwrite,
in_place=args.in_place,
)
print(summary.summary_json_path)
print(summary.index_csv_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} sidecars={summary.sidecar_count} "
f"failed={summary.failed_count} skipped={summary.skipped_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_prepare_gorkhapatra_language_page_verification_bundle(args: argparse.Namespace) -> int:
summary = prepare_gorkhapatra_language_page_verification_bundle(
args.language_page_pack_manifest,
args.review_csv,
args.assisted_references_dir,
args.out,
candidate_languages=args.candidate_language,
overwrite=args.overwrite,
)
print(summary.summary_json_path)
print(summary.index_csv_path)
print(summary.review_html_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} lines={summary.line_count} "
f"crops={summary.crop_count} missing_bboxes={summary.missing_bbox_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_audit_gorkhapatra_language_page_verification_csv(args: argparse.Namespace) -> int:
summary = audit_gorkhapatra_language_page_verification_csv(
args.verification_csv,
args.out,
candidate_languages=args.candidate_language,
required_codepoint_ranges=args.require_codepoint_range,
)
print(summary.summary_json_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} lines={summary.line_count} "
f"ready={summary.ready_line_count} dropped={summary.dropped_line_count} "
f"blocked={summary.blocked_line_count} issues={len(summary.issues)} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_split_gorkhapatra_language_page_verification_csv(args: argparse.Namespace) -> int:
summary = split_gorkhapatra_language_page_verification_csv(
args.verification_csv,
args.out,
batch_size=args.batch_size,
candidate_languages=args.candidate_language,
)
print(summary.summary_json_path)
print(summary.index_csv_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} lines={summary.line_count} "
f"batches={summary.batch_count} batch_size={summary.batch_size} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_merge_gorkhapatra_language_page_verification_batches(args: argparse.Namespace) -> int:
summary = merge_gorkhapatra_language_page_verification_batches(
args.source_verification_csv,
args.batches_dir,
args.out,
)
print(summary.summary_json_path)
print(summary.merged_csv_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} source_lines={summary.source_line_count} merged_lines={summary.merged_line_count} "
f"batch_files={summary.batch_file_count} issues={len(summary.issues)} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_assign_gorkhapatra_language_page_verification_batches(args: argparse.Namespace) -> int:
summary = assign_gorkhapatra_language_page_verification_batches(
args.split_index_csv,
args.out,
reviewers=args.reviewer,
due_date=args.due_date,
)
print(summary.summary_json_path)
print(summary.assignment_csv_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} batches={summary.batch_count} assigned={summary.assigned_count} "
f"reviewers={len(summary.reviewer_counts)} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_apply_gorkhapatra_language_page_verification_bundle(args: argparse.Namespace) -> int:
summary = apply_gorkhapatra_language_page_verification_bundle(
args.language_page_pack_manifest,
args.review_csv,
args.verification_csv,
args.assisted_references_dir,
args.out,
candidate_languages=args.candidate_language,
reviewer=args.reviewer,
reviewed_at=args.reviewed_at,
tables_reviewed=args.tables_reviewed,
figures_reviewed=args.figures_reviewed,
captions_reviewed=args.captions_reviewed,
required_codepoint_ranges=args.require_codepoint_range,
overwrite=args.overwrite,
)
print(summary.summary_json_path)
print(summary.updated_review_csv_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} verified={summary.verified_reference_count} "
f"blocked={summary.blocked_count} reviewed_lines={summary.reviewed_line_count} "
f"dropped_lines={summary.dropped_line_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_finalize_gorkhapatra_language_page_review(args: argparse.Namespace) -> int:
summary = finalize_gorkhapatra_language_page_review(
args.language_page_pack_manifest,
args.review_csv,
args.out,
split=args.split,
dataset_name=args.dataset_name,
candidate_languages=args.candidate_language,
)
print(summary.manifest_path)
print(summary.summary_json_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} "
f"accepted={summary.accepted_review_count} skipped={summary.skipped_review_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_audit_language_registry(args: argparse.Namespace) -> int:
audit = audit_language_registry(
args.registry,
args.out,
min_nepal_languages=args.min_nepal_languages,
require_limbu_first=not args.allow_non_limbu_first,
verify_local_paths=not args.no_verify_local_paths,
)
print(Path(args.out) / "language-registry-audit.json")
print(Path(args.out) / "language-registry-audit.md")
print(
"passed="
f"{audit.passed} languages={audit.language_count} "
f"nepal_languages={audit.counted_nepal_language_count} first_targets={','.join(audit.first_target_ids)}"
)
return 0 if audit.passed else 1
def cmd_audit_language_readiness(args: argparse.Namespace) -> int:
audit = audit_language_readiness(
args.registry,
args.out,
synthesis_text_audit_paths=args.synthesis_text_audit,
min_target_languages=args.min_target_languages,
min_synthesis_ready_languages=args.min_synthesis_ready_languages,
require_limbu_synthesis=not args.no_require_limbu_synthesis,
)
print(Path(args.out) / "language-readiness-audit.json")
print(Path(args.out) / "language-readiness-audit.md")
print(
"passed="
f"{audit.passed} target_languages={audit.target_language_count} "
f"synthesis_ready={audit.synthesis_ready_language_count}/{audit.required_synthesis_ready_language_count}"
)
return 0 if audit.passed else 1
def cmd_audit_synthesis_resources(args: argparse.Namespace) -> int:
audit = audit_synthesis_resources(
args.root,
args.out,
labels=args.label,
max_files_per_root=args.max_files_per_root,
max_text_samples_per_root=args.max_text_samples_per_root,
sample_bytes=args.sample_bytes,
)
print(audit.output_json_path)
print(audit.output_md_path)
print(
f"passed={audit.passed} roots={audit.existing_root_count}/{audit.root_count} "
f"files={audit.total_files} warnings={len(audit.warnings)}"
)
return 0 if audit.passed else 1
def cmd_prepare_limbu_limdic_text(args: argparse.Namespace) -> int:
summary = prepare_limbu_limdic_text(
args.source,
args.out,
split=args.split,
dataset=args.dataset,
min_text_chars=args.min_text_chars,
)
print(summary.manifest_path)
print(summary.summary_json_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} "
f"duplicates={summary.duplicate_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_prepare_tamang_text(args: argparse.Namespace) -> int:
summary = prepare_tamang_text(
args.source,
args.out,
split=args.split,
dataset=args.dataset,
min_text_chars=args.min_text_chars,
limit_per_source=args.limit_per_source,
)
print(summary.manifest_path)
print(summary.summary_json_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} "
f"duplicates={summary.duplicate_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_prepare_magar_text(args: argparse.Namespace) -> int:
summary = prepare_magar_text(
args.source,
args.out,
split=args.split,
dataset=args.dataset,
min_text_chars=args.min_text_chars,
limit_per_source=args.limit_per_source,
)
print(summary.manifest_path)
print(summary.summary_json_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} "
f"duplicates={summary.duplicate_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_prepare_bible_brain_text(args: argparse.Namespace) -> int:
summary = prepare_bible_brain_text(
args.manifest,
args.out,
languages=args.language,
split=args.split,
dataset=args.dataset,
min_text_chars=args.min_text_chars,
limit_per_manifest=args.limit_per_manifest,
)
print(summary.manifest_path)
print(summary.summary_json_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} "
f"duplicates={summary.duplicate_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_prepare_limbu_unicode_text(args: argparse.Namespace) -> int:
summary = prepare_limbu_unicode_text(
args.source,
args.out,
split=args.split,
dataset=args.dataset,
min_text_chars=args.min_text_chars,
min_limbu_chars=args.min_limbu_chars,
limit_per_source=args.limit_per_source,
)
print(summary.manifest_path)
print(summary.summary_json_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} "
f"duplicates={summary.duplicate_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_prepare_toolkit_parallel_text(args: argparse.Namespace) -> int:
summary = prepare_toolkit_parallel_text(
args.source,
args.out,
text_fields=args.text_field,
split=args.split,
dataset=args.dataset,
min_text_chars=args.min_text_chars,
limit_per_source=args.limit_per_source,
row_id_field=args.row_id_field,
metadata_fields=args.metadata_field,
license_status=args.license_status,
split_policy=args.split_policy,
)
print(summary.manifest_path)
print(summary.summary_json_path)
print(summary.summary_md_path)
print(
f"passed={summary.passed} samples={summary.sample_count} "
f"duplicates={summary.duplicate_count} warnings={len(summary.warnings)}"
)
return 0 if summary.passed else 1
def cmd_audit_synthesis_text_promotion(args: argparse.Namespace) -> int:
audit = audit_synthesis_text_promotion(
args.manifest,
args.out,
exclude_paths=args.exclude,
require_reviewed_license=args.require_reviewed_license,
)
print(audit.output_json_path)
print(audit.output_md_path)
print(
f"passed={audit.passed} samples={audit.sample_count} overlaps={audit.overlap_count} "
f"issues={len(audit.issues)} warnings={len(audit.warnings)}"
)
return 0 if audit.passed else 1
def cmd_split_synthesis_text(args: argparse.Namespace) -> int:
summary = split_synthesis_text_manifest(
args.manifest,
args.out,
eval_ratio=args.eval_ratio,
seed=args.seed,
group_by=args.group_by,
exclude_paths=args.exclude,
require_reviewed_license=args.require_reviewed_license,
)
print(summary.train_manifest)
print(summary.eval_manifest)
print(
f"train={summary.train_count} eval={summary.eval_count} "
f"train_promotion={summary.train_promotion_passed} eval_promotion={summary.eval_promotion_passed}"
)
return 0 if summary.passed else 1
def cmd_audit_synthesis_text_manifest(args: argparse.Namespace) -> int:
audit = audit_synthesis_text_manifest(
args.manifest,
args.out,
require_languages=args.require_language,
require_scripts=args.require_script,
min_samples=args.min_samples,
allow_claim_evidence=args.allow_claim_evidence,
)
print(audit.output_json_path)
print(audit.output_md_path)
print(
f"passed={audit.passed} samples={audit.sample_count} "
f"claim_eligible={audit.claim_evidence_eligible_count} issues={len(audit.issues)} warnings={len(audit.warnings)}"
)
return 0 if audit.passed else 1
def cmd_render_synthesis_text_lines(args: argparse.Namespace) -> int:
summary = render_synthesis_text_lines(
args.synthesis_text_manifest,
args.out,
font_path=args.font_path,
limit=args.limit,
scripts=args.script,
font_size=args.font_size,
padding=args.padding,
degradation_profiles=args.degradation_profile,
degradation_seed=args.degradation_seed,
split=args.split,
dataset=args.dataset,
)
print(summary.manifest_path)
print(summary.label_path)
print(summary.summary_json_path)
print(summary.summary_md_path)
print(f"passed={summary.passed} samples={summary.sample_count} skipped={summary.skipped_count} warnings={len(summary.warnings)}")
return 0 if summary.passed else 1
def cmd_render_synthesis_text_split(args: argparse.Namespace) -> int:
summary = render_synthesis_text_split(
args.train_manifest,
args.eval_manifest,
args.out,
font_path=args.font_path,
limit_per_split=args.limit_per_split,
scripts=args.script,
font_size=args.font_size,
padding=args.padding,
degradation_profiles=args.degradation_profile,
degradation_seed=args.degradation_seed,
dataset=args.dataset,
require_font=args.require_font,
font_readiness_report=args.font_readiness_report,
)
print(summary.train_manifest)
print(summary.train_label_path)
print(summary.eval_manifest)
print(summary.eval_label_path)
print(
f"passed={summary.passed} train={summary.train_count} eval={summary.eval_count} "
f"train_audit={summary.train_audit_passed} eval_audit={summary.eval_audit_passed}"
)
return 0 if summary.passed else 1
def cmd_audit_rendered_synthesis_lines(args: argparse.Namespace) -> int:
audit = audit_rendered_synthesis_lines(
args.manifest,
args.out,
label_file=args.label_file,
require_font=args.require_font,
font_readiness_report=args.font_readiness_report,
)
print(audit.output_json_path)
print(audit.output_md_path)
print(
f"passed={audit.passed} samples={audit.sample_count} "
f"claim_eligible={audit.claim_evidence_eligible_count} issues={len(audit.issues)} warnings={len(audit.warnings)}"
)
return 0 if audit.passed else 1
def cmd_audit_rendered_degradation_split(args: argparse.Namespace) -> int:
audit = audit_rendered_degradation_split(
args.train_manifest,
args.eval_manifest,
args.out,
expected_profiles=args.expected_profile,
)
print(audit.output_json_path)
print(audit.output_md_path)
print(
f"passed={audit.passed} train={audit.train_count} eval={audit.eval_count} "
f"train_texts={audit.train_text_hash_count} eval_texts={audit.eval_text_hash_count} "
f"issues={len(audit.issues)} warnings={len(audit.warnings)}"
)
return 0 if audit.passed else 1
def cmd_audit_font_renderability(args: argparse.Namespace) -> int:
audit = audit_font_renderability(
args.registry,
args.out,
primary_only=not args.all_scripts,
min_coverage_ratio=args.min_coverage_ratio,
render_samples=args.render_samples,
)
print(Path(args.out) / "font-renderability-audit.json")
print(Path(args.out) / "font-renderability-audit.md")
print(
"passed="
f"{audit.passed} scripts={audit.script_count} candidates={audit.candidate_count} "
f"rendered={sum(1 for candidate in audit.candidates if candidate.render_status == 'rendered')}"
)
return 0 if audit.passed else 1
def cmd_inventory_fonts(args: argparse.Namespace) -> int:
audit = inventory_fonts(
args.registry,
args.root,
args.out,
min_script_coverage_ratio=args.min_script_coverage_ratio,
)
print(Path(args.out) / "font-inventory.json")
print(Path(args.out) / "font-inventory.csv")
print(Path(args.out) / "font-inventory.md")
print(
f"passed={audit.passed} fonts={audit.font_count} readable={audit.readable_font_count} "
f"archives={audit.archive_count} duplicates={audit.duplicate_sha_count} warnings={len(audit.warnings)}"
)
return 0 if audit.passed else 1
def cmd_prepare_font_assets(args: argparse.Namespace) -> int:
summary = prepare_font_assets(
args.manifest,
asset_root=args.asset_root,
out_dir=args.out,
force=args.force,
timeout_seconds=args.timeout_seconds,
)
print(Path(args.out) / "font-assets-preparation.json")
print(Path(args.out) / "font-assets-preparation.md")
print(
f"passed={summary.passed} assets={summary.asset_count} downloaded={summary.downloaded_count} "
f"reused={summary.reused_count} failed={summary.failed_count}"
)
return 0 if summary.passed else 1
def cmd_audit_font_asset_readiness(args: argparse.Namespace) -> int:
report = audit_font_asset_readiness(
args.manifest,
args.preparation_report,
args.inventory_report,
args.renderability_report,
args.out,
asset_root=args.asset_root,
)
print(Path(args.out) / "font-asset-readiness.json")
print(Path(args.out) / "font-asset-readiness.md")
print(
f"passed={report.passed} assets={report.asset_count} "
f"checked={report.checked_file_count} issues={len(report.issues)}"
)
return 0 if report.passed else 1
def cmd_audit_paddle_dictionary(args: argparse.Namespace) -> int:
audit = audit_paddle_dictionary(
args.dictionary,
args.out,
required_ranges=args.required_range,
label_file_paths=args.label_file,
allow_space_char=not args.no_allow_space_char,
)
print(Path(args.out) / "paddle-dictionary-audit.json")
print(Path(args.out) / "paddle-dictionary-audit.md")
print(
"passed="
f"{audit.passed} chars={audit.character_count} "
f"missing_required={len(audit.missing_required_characters)} "
f"missing_labels={len(audit.missing_label_characters)}"
)
return 0 if audit.passed else 1
def main(argv: list[str] | None = None) -> int:
parser = build_parser()
raw_argv = list(sys.argv[1:] if argv is None else argv)
args = parser.parse_args(raw_argv)
setattr(args, "_raw_argv", raw_argv)
try:
return int(args.func(args))
except OcrTechError as exc:
print(f"ocrtech: error: {exc}", file=sys.stderr)
return 2
def _cli_run_context(args: argparse.Namespace) -> dict[str, object]:
raw_argv = getattr(args, "_raw_argv", None)
if not isinstance(raw_argv, list):
raw_argv = []
return {
"command": "ocrtech " + " ".join(str(item) for item in raw_argv),
"argv": [str(item) for item in raw_argv],
}
def _parse_key_values(values: list[str]) -> dict[str, object]:
parsed: dict[str, object] = {}
for item in values:
if "=" not in item:
raise OcrTechError(f"expected key=value for --paddleocr-kwarg, got {item!r}")
key, value = item.split("=", 1)
if not key:
raise OcrTechError("paddleocr kwarg key cannot be empty")
parsed[key] = _coerce_scalar(value)
return parsed
def _parse_image_line_rescue_detector_passes(values: list[str]) -> tuple[dict[str, object], ...]:
allowed = {
"threshold",
"bbox_source",
"horizontal_kernel",
"vertical_kernel",
"dilation_iterations",
"min_width",
"min_height",
"min_area",
"max_height",
"min_aspect_ratio",
"max_aspect_ratio",
"detector_padding",
"split_tall_components",
"split_tall_row_min_ink",
"split_tall_max_row_gap",
"split_wide_components",
"split_wide_col_min_ink",
"split_wide_max_col_gap",
"split_wide_min_width",
"split_detected_row_components",
"split_detected_row_col_min_ink",
"split_detected_row_max_col_gap",
"split_detected_row_min_width",
"split_detected_row_min_segment_width",
"split_detected_tall_components",
"split_detected_tall_row_min_ink",
"split_detected_tall_max_row_gap",
"split_detected_tall_min_height",
"split_detected_tall_min_segment_height",
"region_min_x_ratio",
"region_max_x_ratio",
"region_min_y_ratio",
"region_max_y_ratio",
}
passes: list[dict[str, object]] = []
for raw_pass in values:
parsed: dict[str, object] = {}
for raw_item in raw_pass.split(","):
item = raw_item.strip()
if not item:
continue
if "=" not in item:
raise OcrTechError(f"expected key=value in --image-line-rescue-detector-pass, got {item!r}")
key, value = (part.strip() for part in item.split("=", 1))
if key not in allowed:
raise OcrTechError(
f"unsupported image-line rescue detector key {key!r}; allowed keys are {sorted(allowed)}"
)
if not value:
raise OcrTechError(f"image-line rescue detector key {key!r} cannot be empty")
parsed[key] = _coerce_scalar(value)
if not parsed:
raise OcrTechError("--image-line-rescue-detector-pass must contain at least one key=value override")
passes.append(parsed)
return tuple(passes)
def _parse_labeled_paths(values: list[str]) -> dict[str, str]:
parsed: dict[str, str] = {}
for item in values:
if "=" not in item:
raise OcrTechError(f"expected label=path, got {item!r}")
label, path = item.split("=", 1)
if not label or not path:
raise OcrTechError(f"expected non-empty label=path, got {item!r}")
parsed[label] = path
return parsed
def _coerce_scalar(value: str) -> object:
stripped = value.strip()
if stripped.startswith("{") or stripped.startswith("["):
try:
return json.loads(stripped)
except json.JSONDecodeError:
pass
lowered = value.lower()
if lowered == "true":
return True
if lowered == "false":
return False
if lowered == "none":
return None
try:
return int(value)
except ValueError:
try:
return float(value)
except ValueError:
return value
def _parse_python_version(value: str) -> tuple[int, int]:
parts = value.strip().split(".")
if len(parts) != 2:
raise OcrTechError(f"expected MINOR python version like 3.11, got {value!r}")
try:
major = int(parts[0])
minor = int(parts[1])
except ValueError as exc:
raise OcrTechError(f"python version must be numeric, got {value!r}") from exc
if major < 1 or minor < 0:
raise OcrTechError(f"invalid python version: {value!r}")
return major, minor
if __name__ == "__main__":
raise SystemExit(main())
"""Command-line interface for ocrtech."""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
from .ablation import audit_augmentation_ablation
from .admission import assert_recognizer_family_not_stopped, decide_recognizer_admission, review_recognizer_failures
from .benchmark import derive_table_postprocess_benchmark, rescore_benchmark_report, run_benchmark, summarize_benchmark_report
from .calibration import calibrate_quality_router, calibrate_script_router, calibrate_tesseract_psm_ensemble
from .claim_review import review_claim
from .capture import audit_limbu_capture, parse_crop_box, parse_perspective_quad, prepare_limbu_capture
from .crops import extract_crop_manifest
from .datasets import audit_manifest, audit_recognizer_corpus, build_recognizer_corpus, filter_manifest, merge_manifests, rebalance_manifest, split_manifest
from .error_analysis import analyze_benchmark_errors, export_failure_manifest
from .errors import OcrTechError
from .evalpack import audit_eval_pack, create_eval_pack
from .experiments import run_experiment
from .font_assets import audit_font_asset_readiness, prepare_font_assets
from .language_registry import audit_font_renderability, audit_language_readiness, audit_language_registry, audit_paddle_dictionary, inventory_fonts
from .limbu_legacy import convert_gorkhapatra_limbu_pdf_text, convert_limbu_legacy_text
from .limbu_runtime import preflight_limbu_pipeline_runtime
from .manifest import inspect_hf_dataset, load_manifest, normalize_manifest_images, prepare_hf_dataset, prepare_local_dataset
from .models import audit_model_card, write_model_card
from .oracle import build_benchmark_oracle
from .paper import export_paper_benchmark_table
from .pipeline import (
admit_limbu_post_correction_profile,
apply_limbu_review_corrections,
audit_limbu_correction_pair_pack,
audit_limbu_output,
audit_limbu_post_correction_profile,
build_limbu_correction_pair_pack,
derive_limbu_post_correction_profile,
parse_document,
parse_limbu_document,
score_limbu_post_correction_profile,
)
from .preflight import run_claim_preflight
from .public_benchmarks import audit_public_benchmark_manifest, list_public_benchmarks, prepare_public_benchmark
from .references import audit_references, materialize_references
from .remote import DEFAULT_PYTHON_CANDIDATES, audit_remote_host, bootstrap_remote_host, sync_remote_workspace
from .source_audit import (
DEFAULT_GORKHAPATRA_CATEGORY_URLS,
DEFAULT_GORKHAPATRA_EPAPER_URL,
DEFAULT_NAYA_NEPAL_PUBLICATION_URL,
apply_gorkhapatra_language_page_verification_bundle,
assign_gorkhapatra_language_page_verification_batches,
audit_gorkhapatra_language_page_verification_csv,
audit_gorkhapatra_language_pages,
audit_gorkhapatra_source,
extract_gorkhapatra_language_page_pdf_text,
audit_gorkhapatra_language_page_review,
finalize_gorkhapatra_language_page_review,
merge_gorkhapatra_language_page_verification_batches,
prepare_gorkhapatra_language_page_assisted_references,
prepare_gorkhapatra_language_page_ocr_sidecars,
prepare_gorkhapatra_language_page_pdf_native_references,
prepare_gorkhapatra_language_page_recognizer_eval,
prepare_gorkhapatra_language_page_reference_templates,
prepare_gorkhapatra_language_page_review,
prepare_gorkhapatra_language_page_reviewer_bundle,
prepare_gorkhapatra_language_page_transcription_work_order,
prepare_gorkhapatra_language_page_verification_bundle,
prepare_gorkhapatra_language_page_pack,
prepare_gorkhapatra_pack,
split_gorkhapatra_language_page_verification_csv,
)
from .synthesis_resources import (
audit_rendered_degradation_split,
audit_rendered_synthesis_lines,
audit_synthesis_text_manifest,
audit_synthesis_text_promotion,
audit_synthesis_resources,
prepare_bible_brain_text,
prepare_limbu_limdic_text,
prepare_limbu_unicode_text,
prepare_magar_text,
prepare_tamang_text,
prepare_toolkit_parallel_text,
render_synthesis_text_split,
render_synthesis_text_lines,
split_synthesis_text_manifest,
)
from .table_analysis import analyze_table_cells
from .training import (
PADDLE_DEVANAGARI_PPOCRV3_PRETRAINED,
PADDLE_PPOCRV5_MOBILE_REC_PRETRAINED,
bundle_recognizer_training,
finalize_hf_recognizer_run,
generate_correction_pairs,
package_hf_text_corrector_model,
package_hf_recognizer_model,
package_paddleocr_model,
write_corrector_recipe,
write_hf_text_corrector_recipe,