Add 51 files (.gitignore ...)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitignore +13 -0
- .ignore +9 -0
- configs/claim-review-sota-v1.json +111 -0
- configs/nepali-table-name-lexicon-v1.txt +10 -0
- configs/validation-complex-table-surya.json +57 -0
- configs/validation-hard15-surya.json +59 -0
- configs/validation-internal-sota-v1.json +120 -0
- configs/validation-paddle-hard50-tesseract.json +52 -0
- configs/validation-public-omnidocbench-sota-v1.json +60 -0
- configs/validation-public-omnidocbench-surya.json +46 -0
- configs/validation-public-real5-sota-v1.json +67 -0
- configs/validation-table-ledger-surya.json +39 -0
- docs/CLAIM_REVIEW.md +320 -0
- docs/DATASETS.md +381 -0
- docs/EVAL_PACK.md +226 -0
- docs/EXPERIMENTS.md +1034 -0
- docs/FONT_ASSETS.md +29 -0
- docs/GORKHAPATRA_LANGUAGE_PAGE_ALIGNMENT.md +172 -0
- docs/MODELS.md +490 -0
- docs/NEPAL_MULTILINGUAL_OCR_GOAL.md +371 -0
- docs/PAPER_NOTES.md +0 -0
- docs/PAPER_RESULTS.md +854 -0
- docs/PREFLIGHT.md +147 -0
- docs/REMOTE_TRAINING.md +1404 -0
- docs/VALIDATION.md +638 -0
- examples/sample-page.txt +7 -0
- languages/nepal-v1.json +370 -0
- pyproject.toml +61 -0
- runs/corrector-cfilt/README.md +5 -0
- runs/corrector-cfilt/correction_pairs.jsonl +5 -0
- runs/corrector-cfilt/corrector_config.json +10 -0
- runs/corrector-mixed-v1-rebalanced/tesseract_pairs.jsonl +0 -0
- runs/hf-corrector-mixed-v1-rebalanced/README.md +18 -0
- runs/hf-corrector-mixed-v1-rebalanced/correction_pairs.jsonl +0 -0
- runs/hf-corrector-mixed-v1-rebalanced/corrector_config.json +10 -0
- runs/hf-corrector-mixed-v1-rebalanced/eval_pairs.jsonl +170 -0
- runs/hf-corrector-mixed-v1-rebalanced/final-model/added_tokens.json +127 -0
- runs/hf-corrector-mixed-v1-rebalanced/final-model/config.json +34 -0
- runs/hf-corrector-mixed-v1-rebalanced/final-model/generation_config.json +37 -0
- runs/hf-corrector-mixed-v1-rebalanced/final-model/model.safetensors +3 -0
- runs/hf-corrector-mixed-v1-rebalanced/final-model/tokenizer_config.json +1291 -0
- runs/hf-corrector-mixed-v1-rebalanced/final-model/training_args.bin +3 -0
- runs/hf-corrector-mixed-v1-rebalanced/hf_text_corrector_config.json +17 -0
- runs/hf-corrector-mixed-v1-rebalanced/package/PACKAGE.md +10 -0
- runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model/added_tokens.json +127 -0
- runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model/config.json +34 -0
- runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model/generation_config.json +37 -0
- runs/hf-corrector-mixed-v1-rebalanced/package/model-card.json +38 -0
- runs/hf-corrector-mixed-v1-rebalanced/train_pairs.jsonl +0 -0
- runs/hf-corrector-mixed-v1-rebalanced/training_metrics.json +10 -0
.gitignore
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.venv/
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
.pytest_cache/
|
| 5 |
+
.ruff_cache/
|
| 6 |
+
.mypy_cache/
|
| 7 |
+
|
| 8 |
+
data/
|
| 9 |
+
outputs/
|
| 10 |
+
runs/
|
| 11 |
+
*.egg-info/
|
| 12 |
+
|
| 13 |
+
.DS_Store
|
.ignore
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.venv/
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
.pytest_cache/
|
| 5 |
+
|
| 6 |
+
data/
|
| 7 |
+
outputs/
|
| 8 |
+
runs/
|
| 9 |
+
*.egg-info/
|
configs/claim-review-sota-v1.json
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"require_shared_candidate_model": true,
|
| 3 |
+
"require_fallback_model_present": true,
|
| 4 |
+
"require_candidate_fallback_provenance": true,
|
| 5 |
+
"require_shared_fallback_model": true,
|
| 6 |
+
"require_model_audit_passed": true,
|
| 7 |
+
"require_fallback_model_audit_passed": true,
|
| 8 |
+
"require_trained_recognizer": true,
|
| 9 |
+
"require_model_admission": true,
|
| 10 |
+
"required_experiments": [
|
| 11 |
+
{
|
| 12 |
+
"name": "internal-nepali-english-document-pack",
|
| 13 |
+
"datasets": ["ocrtech-eval-pack"],
|
| 14 |
+
"require_validation_status": "validated",
|
| 15 |
+
"required_validation_config": "configs/validation-internal-sota-v1.json",
|
| 16 |
+
"require_baselines": ["candidate", "surya", "tesseract", "stock-paddle", "glm-ocr", "paddleocr-vl"],
|
| 17 |
+
"required_metrics": [
|
| 18 |
+
"cer",
|
| 19 |
+
"wer",
|
| 20 |
+
"document_reconstruction_similarity",
|
| 21 |
+
"table_cell_f1",
|
| 22 |
+
"reading_order_pair_accuracy",
|
| 23 |
+
"figure_detection_f1",
|
| 24 |
+
"figure_caption_cer"
|
| 25 |
+
],
|
| 26 |
+
"required_metric_slices": {
|
| 27 |
+
"cer": ["nepali", "english"],
|
| 28 |
+
"wer": ["nepali", "english"],
|
| 29 |
+
"document_reconstruction_similarity": ["table", "reading_order", "figure"],
|
| 30 |
+
"table_cell_f1": ["table"],
|
| 31 |
+
"reading_order_pair_accuracy": ["reading_order"],
|
| 32 |
+
"figure_detection_f1": ["figure"],
|
| 33 |
+
"figure_caption_cer": ["figure"]
|
| 34 |
+
},
|
| 35 |
+
"required_output_artifacts": ["document_json", "document_md", "quality_json"],
|
| 36 |
+
"required_output_artifact_slices": {
|
| 37 |
+
"table_csv_html": ["table"],
|
| 38 |
+
"figure_metadata": ["figure"],
|
| 39 |
+
"figure_images": ["figure"]
|
| 40 |
+
},
|
| 41 |
+
"required_validation_gates": [
|
| 42 |
+
{"baseline": "tesseract", "metric": "cer", "slices": ["nepali"]},
|
| 43 |
+
{"baseline": "tesseract", "metric": "wer", "slices": ["nepali"]},
|
| 44 |
+
{"baseline": "stock-paddle", "metric": "cer", "slices": ["nepali"]},
|
| 45 |
+
{"baseline": "stock-paddle", "metric": "cer", "slices": ["english"]},
|
| 46 |
+
{"baseline": "stock-paddle", "metric": "wer", "slices": ["english"]},
|
| 47 |
+
{"baseline": "glm-ocr", "metric": "cer", "slices": ["nepali"]},
|
| 48 |
+
{"baseline": "surya", "metric": "table_cell_f1", "slices": ["table"]},
|
| 49 |
+
{"baseline": "surya", "metric": "figure_detection_f1", "slices": ["figure"]},
|
| 50 |
+
{"baseline": "surya", "metric": "figure_caption_cer", "slices": ["figure"]},
|
| 51 |
+
{"baseline": "paddleocr-vl", "metric": "reading_order_pair_accuracy", "slices": ["reading_order"]},
|
| 52 |
+
{"baseline": "paddleocr-vl", "metric": "document_reconstruction_similarity", "slices": ["figure"]}
|
| 53 |
+
],
|
| 54 |
+
"require_candidate_fallback_metrics": true,
|
| 55 |
+
"max_candidate_fallback_trigger_rate": 0.35,
|
| 56 |
+
"max_candidate_fallback_failed_rate": 0.0,
|
| 57 |
+
"required_slices": ["nepali", "english", "table", "reading_order", "figure"]
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"name": "public-omnidocbench",
|
| 61 |
+
"datasets": ["opendatalab/OmniDocBench"],
|
| 62 |
+
"require_validation_status": "validated",
|
| 63 |
+
"required_validation_config": "configs/validation-public-omnidocbench-sota-v1.json",
|
| 64 |
+
"require_baselines": ["candidate", "surya", "paddleocr-vl"],
|
| 65 |
+
"required_metrics": ["cer", "document_reconstruction_similarity", "table_cell_f1", "table_structure_similarity", "reading_order_pair_accuracy"],
|
| 66 |
+
"required_metric_slices": {
|
| 67 |
+
"table_cell_f1": ["document_parsing"],
|
| 68 |
+
"table_structure_similarity": ["document_parsing"],
|
| 69 |
+
"reading_order_pair_accuracy": ["document_parsing"],
|
| 70 |
+
"document_reconstruction_similarity": ["document_parsing"]
|
| 71 |
+
},
|
| 72 |
+
"required_output_artifacts": ["document_json", "document_md", "quality_json"],
|
| 73 |
+
"required_output_artifact_slices": {
|
| 74 |
+
"table_csv_html": ["table"]
|
| 75 |
+
},
|
| 76 |
+
"required_validation_gates": [
|
| 77 |
+
{"baseline": "surya", "metric": "cer", "slices": ["public", "document_parsing"]},
|
| 78 |
+
{"baseline": "surya", "metric": "table_cell_f1", "slices": ["public", "table"]},
|
| 79 |
+
{"baseline": "paddleocr-vl", "metric": "document_reconstruction_similarity", "slices": ["public", "document_parsing"]}
|
| 80 |
+
],
|
| 81 |
+
"require_candidate_fallback_metrics": true,
|
| 82 |
+
"max_candidate_fallback_trigger_rate": 0.35,
|
| 83 |
+
"max_candidate_fallback_failed_rate": 0.0,
|
| 84 |
+
"required_slices": ["public", "document_parsing"]
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"name": "public-real5-omnidocbench-or-olmocr",
|
| 88 |
+
"eval_manifest_patterns": ["*Real5-OmniDocBench*", "*olmOCR-bench*"],
|
| 89 |
+
"require_validation_status": "validated",
|
| 90 |
+
"required_validation_config": "configs/validation-public-real5-sota-v1.json",
|
| 91 |
+
"require_baselines": ["candidate", "surya", "glm-ocr", "paddleocr-vl"],
|
| 92 |
+
"required_metrics": ["cer", "wer", "document_reconstruction_similarity", "reading_order_pair_accuracy"],
|
| 93 |
+
"required_metric_slices": {
|
| 94 |
+
"document_reconstruction_similarity": ["public"],
|
| 95 |
+
"reading_order_pair_accuracy": ["public"]
|
| 96 |
+
},
|
| 97 |
+
"required_output_artifacts": ["document_json", "document_md", "quality_json"],
|
| 98 |
+
"required_validation_gates": [
|
| 99 |
+
{"baseline": "surya", "metric": "cer", "slices": ["public"]},
|
| 100 |
+
{"baseline": "surya", "metric": "wer", "slices": ["public"]},
|
| 101 |
+
{"baseline": "glm-ocr", "metric": "cer", "slices": ["public"]},
|
| 102 |
+
{"baseline": "glm-ocr", "metric": "wer", "slices": ["public"]},
|
| 103 |
+
{"baseline": "paddleocr-vl", "metric": "document_reconstruction_similarity", "slices": ["public"]}
|
| 104 |
+
],
|
| 105 |
+
"require_candidate_fallback_metrics": true,
|
| 106 |
+
"max_candidate_fallback_trigger_rate": 0.35,
|
| 107 |
+
"max_candidate_fallback_failed_rate": 0.0,
|
| 108 |
+
"required_slices": ["public"]
|
| 109 |
+
}
|
| 110 |
+
]
|
| 111 |
+
}
|
configs/nepali-table-name-lexicon-v1.txt
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Small Nepali table-name lexicon for guarded OCR correction.
|
| 2 |
+
# Use only with provenance in the model card; this is not a general language model.
|
| 3 |
+
किरण
|
| 4 |
+
माया
|
| 5 |
+
राम
|
| 6 |
+
सीता
|
| 7 |
+
हरी
|
| 8 |
+
गीता
|
| 9 |
+
सोम
|
| 10 |
+
रीता
|
configs/validation-complex-table-surya.json
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"candidate": "candidate",
|
| 3 |
+
"min_samples": 40,
|
| 4 |
+
"required_slices": {
|
| 5 |
+
"complex_table": 40,
|
| 6 |
+
"table": 40,
|
| 7 |
+
"nepali": 40,
|
| 8 |
+
"english": 40
|
| 9 |
+
},
|
| 10 |
+
"bootstrap_iterations": 1000,
|
| 11 |
+
"bootstrap_confidence": 0.95,
|
| 12 |
+
"random_seed": 13,
|
| 13 |
+
"gates": [
|
| 14 |
+
{
|
| 15 |
+
"baseline": "surya",
|
| 16 |
+
"metric": "table_cell_f1",
|
| 17 |
+
"direction": "higher",
|
| 18 |
+
"slices": ["complex_table", "table"],
|
| 19 |
+
"min_pairs": 40,
|
| 20 |
+
"min_absolute_margin": 0.005
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"baseline": "surya",
|
| 24 |
+
"metric": "table_exact_match",
|
| 25 |
+
"direction": "higher",
|
| 26 |
+
"slices": ["complex_table", "table"],
|
| 27 |
+
"min_pairs": 40,
|
| 28 |
+
"min_absolute_margin": 0.0,
|
| 29 |
+
"require_ci_above_zero": false
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"baseline": "surya",
|
| 33 |
+
"metric": "table_structure_similarity",
|
| 34 |
+
"direction": "higher",
|
| 35 |
+
"slices": ["complex_table", "table"],
|
| 36 |
+
"min_pairs": 40,
|
| 37 |
+
"min_absolute_margin": 0.0,
|
| 38 |
+
"require_ci_above_zero": false
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"baseline": "surya",
|
| 42 |
+
"metric": "table_teds_structure",
|
| 43 |
+
"direction": "higher",
|
| 44 |
+
"slices": ["complex_table", "table"],
|
| 45 |
+
"min_pairs": 40,
|
| 46 |
+
"min_absolute_margin": 0.0
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"baseline": "surya",
|
| 50 |
+
"metric": "table_teds_text",
|
| 51 |
+
"direction": "higher",
|
| 52 |
+
"slices": ["complex_table", "table"],
|
| 53 |
+
"min_pairs": 40,
|
| 54 |
+
"min_absolute_margin": 0.005
|
| 55 |
+
}
|
| 56 |
+
]
|
| 57 |
+
}
|
configs/validation-hard15-surya.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"candidate": "candidate",
|
| 3 |
+
"min_samples": 15,
|
| 4 |
+
"required_slices": {
|
| 5 |
+
"hard_eval": 15,
|
| 6 |
+
"table": 8,
|
| 7 |
+
"figure": 7,
|
| 8 |
+
"nepali": 15,
|
| 9 |
+
"english": 15
|
| 10 |
+
},
|
| 11 |
+
"bootstrap_iterations": 1000,
|
| 12 |
+
"bootstrap_confidence": 0.95,
|
| 13 |
+
"random_seed": 13,
|
| 14 |
+
"gates": [
|
| 15 |
+
{
|
| 16 |
+
"baseline": "surya",
|
| 17 |
+
"metric": "cer",
|
| 18 |
+
"direction": "lower",
|
| 19 |
+
"slices": ["hard_eval"],
|
| 20 |
+
"min_pairs": 15,
|
| 21 |
+
"min_absolute_margin": 0.02,
|
| 22 |
+
"max_p_value": 0.05
|
| 23 |
+
},
|
| 24 |
+
{
|
| 25 |
+
"baseline": "surya",
|
| 26 |
+
"metric": "cer",
|
| 27 |
+
"direction": "lower",
|
| 28 |
+
"slices": ["hard_eval", "table"],
|
| 29 |
+
"min_pairs": 8,
|
| 30 |
+
"min_absolute_margin": 0.02,
|
| 31 |
+
"max_p_value": 0.05
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"baseline": "surya",
|
| 35 |
+
"metric": "cer",
|
| 36 |
+
"direction": "lower",
|
| 37 |
+
"slices": ["hard_eval", "figure"],
|
| 38 |
+
"min_pairs": 7,
|
| 39 |
+
"min_absolute_margin": 0.02,
|
| 40 |
+
"max_p_value": 0.05
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"baseline": "surya",
|
| 44 |
+
"metric": "table_cell_f1",
|
| 45 |
+
"direction": "higher",
|
| 46 |
+
"slices": ["hard_eval", "table"],
|
| 47 |
+
"min_pairs": 8,
|
| 48 |
+
"min_absolute_margin": 0.01
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"baseline": "surya",
|
| 52 |
+
"metric": "figure_caption_cer",
|
| 53 |
+
"direction": "lower",
|
| 54 |
+
"slices": ["hard_eval", "figure"],
|
| 55 |
+
"min_pairs": 7,
|
| 56 |
+
"min_absolute_margin": 0.01
|
| 57 |
+
}
|
| 58 |
+
]
|
| 59 |
+
}
|
configs/validation-internal-sota-v1.json
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"candidate": "candidate",
|
| 3 |
+
"min_samples": 100,
|
| 4 |
+
"required_slices": {
|
| 5 |
+
"nepali": 50,
|
| 6 |
+
"english": 50,
|
| 7 |
+
"table": 20,
|
| 8 |
+
"reading_order": 20,
|
| 9 |
+
"figure": 20
|
| 10 |
+
},
|
| 11 |
+
"bootstrap_iterations": 1000,
|
| 12 |
+
"bootstrap_confidence": 0.95,
|
| 13 |
+
"random_seed": 13,
|
| 14 |
+
"require_full_eval_manifest_coverage": true,
|
| 15 |
+
"gates": [
|
| 16 |
+
{
|
| 17 |
+
"baseline": "tesseract",
|
| 18 |
+
"metric": "cer",
|
| 19 |
+
"direction": "lower",
|
| 20 |
+
"slices": ["nepali"],
|
| 21 |
+
"min_pairs": 50,
|
| 22 |
+
"min_absolute_margin": 0.02,
|
| 23 |
+
"min_pair_win_rate": 0.55,
|
| 24 |
+
"max_p_value": 0.05
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"baseline": "tesseract",
|
| 28 |
+
"metric": "wer",
|
| 29 |
+
"direction": "lower",
|
| 30 |
+
"slices": ["nepali"],
|
| 31 |
+
"min_pairs": 50,
|
| 32 |
+
"min_absolute_margin": 0.01,
|
| 33 |
+
"min_pair_win_rate": 0.55,
|
| 34 |
+
"max_p_value": 0.05
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
"baseline": "stock-paddle",
|
| 38 |
+
"metric": "cer",
|
| 39 |
+
"direction": "lower",
|
| 40 |
+
"slices": ["nepali"],
|
| 41 |
+
"min_pairs": 50,
|
| 42 |
+
"min_absolute_margin": 0.01,
|
| 43 |
+
"min_pair_win_rate": 0.55,
|
| 44 |
+
"max_p_value": 0.05
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"baseline": "stock-paddle",
|
| 48 |
+
"metric": "cer",
|
| 49 |
+
"direction": "lower",
|
| 50 |
+
"slices": ["english"],
|
| 51 |
+
"min_pairs": 50,
|
| 52 |
+
"min_absolute_margin": 0.0,
|
| 53 |
+
"require_ci_above_zero": false
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"baseline": "stock-paddle",
|
| 57 |
+
"metric": "wer",
|
| 58 |
+
"direction": "lower",
|
| 59 |
+
"slices": ["english"],
|
| 60 |
+
"min_pairs": 50,
|
| 61 |
+
"min_absolute_margin": 0.0,
|
| 62 |
+
"require_ci_above_zero": false
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"baseline": "glm-ocr",
|
| 66 |
+
"metric": "cer",
|
| 67 |
+
"direction": "lower",
|
| 68 |
+
"slices": ["nepali"],
|
| 69 |
+
"min_pairs": 50,
|
| 70 |
+
"min_absolute_margin": 0.01,
|
| 71 |
+
"min_pair_win_rate": 0.55,
|
| 72 |
+
"max_p_value": 0.05
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"baseline": "surya",
|
| 76 |
+
"metric": "table_cell_f1",
|
| 77 |
+
"direction": "higher",
|
| 78 |
+
"slices": ["table"],
|
| 79 |
+
"min_pairs": 20,
|
| 80 |
+
"min_absolute_margin": 0.01,
|
| 81 |
+
"min_pair_win_rate": 0.55
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"baseline": "paddleocr-vl",
|
| 85 |
+
"metric": "reading_order_pair_accuracy",
|
| 86 |
+
"direction": "higher",
|
| 87 |
+
"slices": ["reading_order"],
|
| 88 |
+
"min_pairs": 20,
|
| 89 |
+
"min_absolute_margin": 0.01,
|
| 90 |
+
"min_pair_win_rate": 0.55
|
| 91 |
+
},
|
| 92 |
+
{
|
| 93 |
+
"baseline": "paddleocr-vl",
|
| 94 |
+
"metric": "document_reconstruction_similarity",
|
| 95 |
+
"direction": "higher",
|
| 96 |
+
"slices": ["figure"],
|
| 97 |
+
"min_pairs": 20,
|
| 98 |
+
"min_absolute_margin": 0.01,
|
| 99 |
+
"min_pair_win_rate": 0.55
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"baseline": "surya",
|
| 103 |
+
"metric": "figure_detection_f1",
|
| 104 |
+
"direction": "higher",
|
| 105 |
+
"slices": ["figure"],
|
| 106 |
+
"min_pairs": 20,
|
| 107 |
+
"min_absolute_margin": 0.0,
|
| 108 |
+
"require_ci_above_zero": false
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"baseline": "surya",
|
| 112 |
+
"metric": "figure_caption_cer",
|
| 113 |
+
"direction": "lower",
|
| 114 |
+
"slices": ["figure"],
|
| 115 |
+
"min_pairs": 20,
|
| 116 |
+
"min_absolute_margin": 0.01,
|
| 117 |
+
"min_pair_win_rate": 0.55
|
| 118 |
+
}
|
| 119 |
+
]
|
| 120 |
+
}
|
configs/validation-paddle-hard50-tesseract.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"candidate": "candidate",
|
| 3 |
+
"min_samples": 50,
|
| 4 |
+
"required_slices": {
|
| 5 |
+
"paddle_hard_eval": 50,
|
| 6 |
+
"synthetic": 50,
|
| 7 |
+
"english": 38,
|
| 8 |
+
"line_crop": 38,
|
| 9 |
+
"nepali": 12
|
| 10 |
+
},
|
| 11 |
+
"bootstrap_iterations": 1000,
|
| 12 |
+
"bootstrap_confidence": 0.95,
|
| 13 |
+
"random_seed": 13,
|
| 14 |
+
"gates": [
|
| 15 |
+
{
|
| 16 |
+
"baseline": "tesseract",
|
| 17 |
+
"metric": "cer",
|
| 18 |
+
"direction": "lower",
|
| 19 |
+
"slices": ["paddle_hard_eval"],
|
| 20 |
+
"min_pairs": 50,
|
| 21 |
+
"min_absolute_margin": 0.0,
|
| 22 |
+
"require_ci_above_zero": false
|
| 23 |
+
},
|
| 24 |
+
{
|
| 25 |
+
"baseline": "tesseract",
|
| 26 |
+
"metric": "cer",
|
| 27 |
+
"direction": "lower",
|
| 28 |
+
"slices": ["paddle_hard_eval", "english", "line_crop"],
|
| 29 |
+
"min_pairs": 38,
|
| 30 |
+
"min_absolute_margin": 0.0,
|
| 31 |
+
"require_ci_above_zero": false
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"baseline": "tesseract",
|
| 35 |
+
"metric": "wer",
|
| 36 |
+
"direction": "lower",
|
| 37 |
+
"slices": ["paddle_hard_eval", "english", "line_crop"],
|
| 38 |
+
"min_pairs": 38,
|
| 39 |
+
"min_absolute_margin": 0.0,
|
| 40 |
+
"require_ci_above_zero": false
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"baseline": "tesseract",
|
| 44 |
+
"metric": "wer",
|
| 45 |
+
"direction": "lower",
|
| 46 |
+
"slices": ["paddle_hard_eval", "nepali"],
|
| 47 |
+
"min_pairs": 12,
|
| 48 |
+
"min_absolute_margin": 0.05,
|
| 49 |
+
"require_ci_above_zero": false
|
| 50 |
+
}
|
| 51 |
+
]
|
| 52 |
+
}
|
configs/validation-public-omnidocbench-sota-v1.json
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"candidate": "candidate",
|
| 3 |
+
"min_samples": 100,
|
| 4 |
+
"required_slices": {
|
| 5 |
+
"public": 100,
|
| 6 |
+
"document_parsing": 100,
|
| 7 |
+
"table": 40
|
| 8 |
+
},
|
| 9 |
+
"bootstrap_iterations": 1000,
|
| 10 |
+
"bootstrap_confidence": 0.95,
|
| 11 |
+
"random_seed": 13,
|
| 12 |
+
"require_full_eval_manifest_coverage": true,
|
| 13 |
+
"gates": [
|
| 14 |
+
{
|
| 15 |
+
"baseline": "surya",
|
| 16 |
+
"metric": "cer",
|
| 17 |
+
"direction": "lower",
|
| 18 |
+
"slices": ["public", "document_parsing"],
|
| 19 |
+
"min_pairs": 100,
|
| 20 |
+
"min_absolute_margin": 0.005,
|
| 21 |
+
"min_pair_win_rate": 0.55
|
| 22 |
+
},
|
| 23 |
+
{
|
| 24 |
+
"baseline": "surya",
|
| 25 |
+
"metric": "table_cell_f1",
|
| 26 |
+
"direction": "higher",
|
| 27 |
+
"slices": ["public", "table"],
|
| 28 |
+
"min_pairs": 40,
|
| 29 |
+
"min_absolute_margin": 0.005,
|
| 30 |
+
"min_pair_win_rate": 0.55
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"baseline": "surya",
|
| 34 |
+
"metric": "table_structure_similarity",
|
| 35 |
+
"direction": "higher",
|
| 36 |
+
"slices": ["public", "document_parsing"],
|
| 37 |
+
"min_pairs": 40,
|
| 38 |
+
"min_absolute_margin": 0.0,
|
| 39 |
+
"require_ci_above_zero": false
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"baseline": "paddleocr-vl",
|
| 43 |
+
"metric": "document_reconstruction_similarity",
|
| 44 |
+
"direction": "higher",
|
| 45 |
+
"slices": ["public", "document_parsing"],
|
| 46 |
+
"min_pairs": 100,
|
| 47 |
+
"min_absolute_margin": 0.005,
|
| 48 |
+
"min_pair_win_rate": 0.55
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"baseline": "paddleocr-vl",
|
| 52 |
+
"metric": "reading_order_pair_accuracy",
|
| 53 |
+
"direction": "higher",
|
| 54 |
+
"slices": ["public", "document_parsing"],
|
| 55 |
+
"min_pairs": 100,
|
| 56 |
+
"min_absolute_margin": 0.005,
|
| 57 |
+
"min_pair_win_rate": 0.55
|
| 58 |
+
}
|
| 59 |
+
]
|
| 60 |
+
}
|
configs/validation-public-omnidocbench-surya.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"candidate": "candidate",
|
| 3 |
+
"min_samples": 100,
|
| 4 |
+
"required_slices": {
|
| 5 |
+
"public": 100,
|
| 6 |
+
"document_parsing": 100,
|
| 7 |
+
"table": 40
|
| 8 |
+
},
|
| 9 |
+
"bootstrap_iterations": 1000,
|
| 10 |
+
"bootstrap_confidence": 0.95,
|
| 11 |
+
"random_seed": 13,
|
| 12 |
+
"gates": [
|
| 13 |
+
{
|
| 14 |
+
"baseline": "surya",
|
| 15 |
+
"metric": "cer",
|
| 16 |
+
"direction": "lower",
|
| 17 |
+
"slices": ["public", "document_parsing"],
|
| 18 |
+
"min_pairs": 100,
|
| 19 |
+
"min_absolute_margin": 0.005
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"baseline": "surya",
|
| 23 |
+
"metric": "table_cell_f1",
|
| 24 |
+
"direction": "higher",
|
| 25 |
+
"slices": ["public", "table"],
|
| 26 |
+
"min_pairs": 40,
|
| 27 |
+
"min_absolute_margin": 0.005
|
| 28 |
+
},
|
| 29 |
+
{
|
| 30 |
+
"baseline": "surya",
|
| 31 |
+
"metric": "table_teds_structure",
|
| 32 |
+
"direction": "higher",
|
| 33 |
+
"slices": ["public", "table"],
|
| 34 |
+
"min_pairs": 40,
|
| 35 |
+
"min_absolute_margin": 0.005
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"baseline": "surya",
|
| 39 |
+
"metric": "table_teds_text",
|
| 40 |
+
"direction": "higher",
|
| 41 |
+
"slices": ["public", "table"],
|
| 42 |
+
"min_pairs": 40,
|
| 43 |
+
"min_absolute_margin": 0.005
|
| 44 |
+
}
|
| 45 |
+
]
|
| 46 |
+
}
|
configs/validation-public-real5-sota-v1.json
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"candidate": "candidate",
|
| 3 |
+
"min_samples": 100,
|
| 4 |
+
"required_slices": {
|
| 5 |
+
"public": 100
|
| 6 |
+
},
|
| 7 |
+
"bootstrap_iterations": 1000,
|
| 8 |
+
"bootstrap_confidence": 0.95,
|
| 9 |
+
"random_seed": 13,
|
| 10 |
+
"require_full_eval_manifest_coverage": true,
|
| 11 |
+
"gates": [
|
| 12 |
+
{
|
| 13 |
+
"baseline": "surya",
|
| 14 |
+
"metric": "cer",
|
| 15 |
+
"direction": "lower",
|
| 16 |
+
"slices": ["public"],
|
| 17 |
+
"min_pairs": 100,
|
| 18 |
+
"min_absolute_margin": 0.005,
|
| 19 |
+
"min_pair_win_rate": 0.55
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"baseline": "surya",
|
| 23 |
+
"metric": "wer",
|
| 24 |
+
"direction": "lower",
|
| 25 |
+
"slices": ["public"],
|
| 26 |
+
"min_pairs": 100,
|
| 27 |
+
"min_absolute_margin": 0.005,
|
| 28 |
+
"min_pair_win_rate": 0.55
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"baseline": "glm-ocr",
|
| 32 |
+
"metric": "cer",
|
| 33 |
+
"direction": "lower",
|
| 34 |
+
"slices": ["public"],
|
| 35 |
+
"min_pairs": 100,
|
| 36 |
+
"min_absolute_margin": 0.005,
|
| 37 |
+
"min_pair_win_rate": 0.55
|
| 38 |
+
},
|
| 39 |
+
{
|
| 40 |
+
"baseline": "glm-ocr",
|
| 41 |
+
"metric": "wer",
|
| 42 |
+
"direction": "lower",
|
| 43 |
+
"slices": ["public"],
|
| 44 |
+
"min_pairs": 100,
|
| 45 |
+
"min_absolute_margin": 0.005,
|
| 46 |
+
"min_pair_win_rate": 0.55
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"baseline": "paddleocr-vl",
|
| 50 |
+
"metric": "document_reconstruction_similarity",
|
| 51 |
+
"direction": "higher",
|
| 52 |
+
"slices": ["public"],
|
| 53 |
+
"min_pairs": 100,
|
| 54 |
+
"min_absolute_margin": 0.005,
|
| 55 |
+
"min_pair_win_rate": 0.55
|
| 56 |
+
},
|
| 57 |
+
{
|
| 58 |
+
"baseline": "paddleocr-vl",
|
| 59 |
+
"metric": "reading_order_pair_accuracy",
|
| 60 |
+
"direction": "higher",
|
| 61 |
+
"slices": ["public"],
|
| 62 |
+
"min_pairs": 100,
|
| 63 |
+
"min_absolute_margin": 0.005,
|
| 64 |
+
"min_pair_win_rate": 0.55
|
| 65 |
+
}
|
| 66 |
+
]
|
| 67 |
+
}
|
configs/validation-table-ledger-surya.json
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"candidate": "candidate",
|
| 3 |
+
"min_samples": 40,
|
| 4 |
+
"required_slices": {
|
| 5 |
+
"table": 40,
|
| 6 |
+
"nepali": 40,
|
| 7 |
+
"english": 40
|
| 8 |
+
},
|
| 9 |
+
"bootstrap_iterations": 1000,
|
| 10 |
+
"bootstrap_confidence": 0.95,
|
| 11 |
+
"random_seed": 13,
|
| 12 |
+
"gates": [
|
| 13 |
+
{
|
| 14 |
+
"baseline": "surya",
|
| 15 |
+
"metric": "table_cell_f1",
|
| 16 |
+
"direction": "higher",
|
| 17 |
+
"slices": ["table"],
|
| 18 |
+
"min_pairs": 40,
|
| 19 |
+
"min_absolute_margin": 0.01
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"baseline": "surya",
|
| 23 |
+
"metric": "table_exact_match",
|
| 24 |
+
"direction": "higher",
|
| 25 |
+
"slices": ["table"],
|
| 26 |
+
"min_pairs": 40,
|
| 27 |
+
"min_absolute_margin": 0.01
|
| 28 |
+
},
|
| 29 |
+
{
|
| 30 |
+
"baseline": "surya",
|
| 31 |
+
"metric": "table_structure_similarity",
|
| 32 |
+
"direction": "higher",
|
| 33 |
+
"slices": ["table"],
|
| 34 |
+
"min_pairs": 40,
|
| 35 |
+
"min_absolute_margin": 0.0,
|
| 36 |
+
"require_ci_above_zero": false
|
| 37 |
+
}
|
| 38 |
+
]
|
| 39 |
+
}
|
docs/CLAIM_REVIEW.md
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim Review
|
| 2 |
+
|
| 3 |
+
`review-claim` is the portfolio-level gate for a SOTA claim. `validate-claim` can tell you whether one experiment passed its configured gates. It cannot tell you whether you have enough evidence across the internal Nepali/English pack and public suites to say "better than what is out there." That is what `review-claim` is for.
|
| 4 |
+
|
| 5 |
+
## What It Audits
|
| 6 |
+
|
| 7 |
+
`review-claim` checks that:
|
| 8 |
+
|
| 9 |
+
- every required experiment slot is backed by a real `experiment.json`
|
| 10 |
+
- every referenced eval manifest and benchmark report still exists at the
|
| 11 |
+
recorded path, and the eval manifest matches the experiment's recorded
|
| 12 |
+
SHA-256
|
| 13 |
+
- successful benchmark rows are bound back to sample ids and input paths in the
|
| 14 |
+
reviewed eval manifest, and row slice labels must match the manifest entry
|
| 15 |
+
- the expected datasets or manifest patterns are present
|
| 16 |
+
- required baselines were actually run
|
| 17 |
+
- required metric families are present on successful rows for the required baselines
|
| 18 |
+
- required metric families are present on the slices they are supposed to
|
| 19 |
+
substantiate when `required_metric_slices` is configured
|
| 20 |
+
- benchmark reports do not contain error rows when the slot requires clean runs;
|
| 21 |
+
status counts are derived from `benchmark/report.json`, not trusted from the
|
| 22 |
+
copied `experiment.json` summary
|
| 23 |
+
- required validation status is present for internal claim-grade experiments,
|
| 24 |
+
and the referenced `validation.json` agrees with `experiment.json`
|
| 25 |
+
- required validation gates passed for the baseline/metric/slice comparisons
|
| 26 |
+
declared by `required_validation_gates`
|
| 27 |
+
- when configured, candidate fallback diagnostics are present and the aggregate
|
| 28 |
+
fallback trigger/failure rates stay under the slot limits
|
| 29 |
+
- required slices exist for the internal Nepali/English pack
|
| 30 |
+
- candidate experiments all point to the same candidate model config
|
| 31 |
+
- the candidate model card matches experiment hashes and passes `audit-model`,
|
| 32 |
+
with or without an explicit `--model-config`
|
| 33 |
+
- if candidate experiments record a fallback model card, that card exists,
|
| 34 |
+
matches the experiment SHA-256, and passes `audit-model`
|
| 35 |
+
- when configured, the candidate model card itself records the same fallback
|
| 36 |
+
component in `provenance.fallback_model_card`
|
| 37 |
+
- when configured, the candidate model card proves it includes a trained local
|
| 38 |
+
recognizer with artifacts, train/eval manifest hashes, and training-summary
|
| 39 |
+
provenance
|
| 40 |
+
- when configured, the candidate model card references a passing admission
|
| 41 |
+
validation report before it can enter portfolio review
|
| 42 |
+
- routed candidates such as `quality_select_composite`, `quality_ranked_ensemble`, and `script_select_composite` were calibrated on a separate eval manifest, not on one of the reviewed experiment manifests
|
| 43 |
+
- routed candidates also fail review if their selected calibration trial collapses to one engine on every calibration sample
|
| 44 |
+
- routed candidates also fail review if a reviewed benchmark shows they are exactly indistinguishable from one of their component baselines on paired holdout CER
|
| 45 |
+
|
| 46 |
+
Metric coverage is a hard gate because the target claim is a document-understanding
|
| 47 |
+
claim, not a plain OCR claim. A required experiment can now declare
|
| 48 |
+
`required_metrics`; each metric must appear as a finite value on at least one
|
| 49 |
+
successful row for every required baseline in that slot. For the internal
|
| 50 |
+
Nepali/English document pack, the SOTA config requires text OCR metrics
|
| 51 |
+
(`cer`, `wer`), document reconstruction
|
| 52 |
+
(`document_reconstruction_similarity`), table reconstruction
|
| 53 |
+
(`table_cell_f1`), reading order (`reading_order_pair_accuracy`), and figure/caption metrics
|
| 54 |
+
(`figure_detection_f1`, `figure_caption_cer`). Public table/document slots
|
| 55 |
+
declare their own metric set.
|
| 56 |
+
|
| 57 |
+
For claim-grade slots, also declare `required_metric_slices`. This prevents a
|
| 58 |
+
metric from satisfying the portfolio check on the wrong document type. For
|
| 59 |
+
example, `table_cell_f1` must be finite on a `table` or document-parsing row,
|
| 60 |
+
and `figure_caption_cer` must be finite on a `figure` row.
|
| 61 |
+
|
| 62 |
+
Metric presence is not enough. When a required slot includes `candidate` and
|
| 63 |
+
one or more external baselines, `review-claim` also requires paired metric
|
| 64 |
+
coverage on the same sample ids. If `required_metric_slices` is configured, the
|
| 65 |
+
pairing is checked on those slices; otherwise it is checked across all rows.
|
| 66 |
+
This prevents a SOTA review from passing because candidate metrics and baseline
|
| 67 |
+
metrics both exist somewhere in the report but never compare the same documents.
|
| 68 |
+
|
| 69 |
+
Sample coverage is also a hard gate. Every `require_baselines` entry must have a
|
| 70 |
+
successful benchmark row for every sample in the reviewed eval manifest. This
|
| 71 |
+
prevents a report from pointing at a hard eval pack while only running an easy
|
| 72 |
+
subset.
|
| 73 |
+
|
| 74 |
+
When a slot declares `require_validation_status`, `review-claim` reads the
|
| 75 |
+
referenced validation report and verifies `claim_status`. For `validated`, it
|
| 76 |
+
also requires `passed: true`. The copied status inside `experiment.json` is not
|
| 77 |
+
enough on its own. The validation artifact must also carry provenance linking it
|
| 78 |
+
to the same benchmark report, eval manifest, validation config, model cards,
|
| 79 |
+
and train manifest hashes referenced by the experiment.
|
| 80 |
+
|
| 81 |
+
Use `required_validation_gates` for claim-grade slots. Metric coverage proves
|
| 82 |
+
that a benchmark emitted a metric; passing validation gates prove the candidate
|
| 83 |
+
beat a named baseline under the configured sample count, slices, margins, and
|
| 84 |
+
statistical checks. Review requires each matched gate to carry quantitative
|
| 85 |
+
paired evidence, not just `passed: true`: positive `pairs`, finite candidate
|
| 86 |
+
and baseline means, finite absolute improvement, and win/loss/tie counts that
|
| 87 |
+
sum to the pair count. It also re-checks the threshold fields embedded in the
|
| 88 |
+
gate result, including `min_pairs`, margins, `min_pair_win_rate`, `max_p_value`,
|
| 89 |
+
and `require_ci_above_zero`, so a hand-edited report cannot mark a weak gate as
|
| 90 |
+
passed without satisfying its own configured test.
|
| 91 |
+
|
| 92 |
+
The SOTA validation configs also use `min_pair_win_rate` on improvement gates.
|
| 93 |
+
This is checked as wins divided by all paired samples, not only non-tied pairs.
|
| 94 |
+
It prevents an average-only improvement, or a single win with many ties, from
|
| 95 |
+
supporting a broad "better OCR/document engine" claim.
|
| 96 |
+
|
| 97 |
+
`required_validation_gates` is only valid on slots that also declare
|
| 98 |
+
`require_validation_status`. Each required gate baseline must also appear in
|
| 99 |
+
`require_baselines`, and each gate metric must appear in `required_metrics`.
|
| 100 |
+
|
| 101 |
+
For claim-grade slots, set `required_validation_config` to the checked-in
|
| 102 |
+
validation config path. `review-claim` hashes that file and requires
|
| 103 |
+
`experiment.json` to record the same `validation_config_sha256`. This prevents a
|
| 104 |
+
portfolio from passing with an easier local validation config that happens to
|
| 105 |
+
contain a subset of the expected gates.
|
| 106 |
+
|
| 107 |
+
For hybrid candidates, use `require_candidate_fallback_metrics`,
|
| 108 |
+
`max_candidate_fallback_trigger_rate`, and
|
| 109 |
+
`max_candidate_fallback_failed_rate`. These checks make fallback dependency
|
| 110 |
+
auditable. A candidate that wins mostly by routing pages to the fallback is a
|
| 111 |
+
different claim from a local-first engine with selective fallback, even if both
|
| 112 |
+
produce strong output metrics.
|
| 113 |
+
|
| 114 |
+
Fallback model cards are audited separately from the primary candidate card.
|
| 115 |
+
When `experiment.json` records `fallback_model_config` and
|
| 116 |
+
`fallback_model_config_sha256`, `review-claim` verifies the hash and runs
|
| 117 |
+
`audit-model` on that fallback card. The SOTA profile also sets
|
| 118 |
+
`require_fallback_model_present: true`,
|
| 119 |
+
`require_candidate_fallback_provenance: true`,
|
| 120 |
+
`require_shared_fallback_model: true` and
|
| 121 |
+
`require_fallback_model_audit_passed: true`, so a portfolio cannot silently mix
|
| 122 |
+
unaudited fallback components across required experiment slots or omit fallback
|
| 123 |
+
provenance while still claiming the selective-fallback engine.
|
| 124 |
+
|
| 125 |
+
The candidate card must also declare the fallback component it was built to use:
|
| 126 |
+
|
| 127 |
+
```json
|
| 128 |
+
{
|
| 129 |
+
"provenance": {
|
| 130 |
+
"fallback_model_card": {
|
| 131 |
+
"path": "runs/fallback/model-card.json",
|
| 132 |
+
"sha256": "..."
|
| 133 |
+
}
|
| 134 |
+
}
|
| 135 |
+
}
|
| 136 |
+
```
|
| 137 |
+
|
| 138 |
+
`review-claim` hashes that model card, compares it with every reviewed
|
| 139 |
+
`fallback_model_config_sha256`, and audits the fallback card. This closes the
|
| 140 |
+
gap where an experiment could record a fallback at benchmark time while the
|
| 141 |
+
candidate card itself remained ambiguous about which selective-fallback engine
|
| 142 |
+
was being claimed.
|
| 143 |
+
|
| 144 |
+
That second guardrail matters for exactly the case we hit on `k2`: a rendered-document `script_select_composite` calibrated on `claimpack-rendered-v2` chose `primary_script_threshold=0.0`, meaning the route degenerated to "always pick Surya." Separation from the final holdout is not enough there; the candidate is still not a meaningful new routed model.
|
| 145 |
+
|
| 146 |
+
The holdout-equivalence guardrail covers the next failure mode after calibration separation: even when calibration was done on a separate split, a routed candidate can still reduce to one component baseline on the reviewed holdout. If the paired benchmark rows show `0` wins, `0` losses, and all ties on CER against a component baseline, `review-claim` now fails `candidate-distinctness`.
|
| 147 |
+
|
| 148 |
+
For the SOTA track, set `require_trained_recognizer: true`. A direct
|
| 149 |
+
`paddleocr` or `hf_vision_encoder_decoder` candidate passes this check only when
|
| 150 |
+
the model card has recorded artifacts plus `provenance.train_manifest`,
|
| 151 |
+
`provenance.eval_manifest`, and `provenance.training_summary` with SHA-256
|
| 152 |
+
hashes. Review resolves those paths from the model-card location and verifies
|
| 153 |
+
the files still match the recorded hashes. Composite candidates must point to
|
| 154 |
+
the trained component with:
|
| 155 |
+
|
| 156 |
+
```json
|
| 157 |
+
{
|
| 158 |
+
"provenance": {
|
| 159 |
+
"trained_recognizer_model_card": {
|
| 160 |
+
"path": "runs/recognizer/package/model-card.json",
|
| 161 |
+
"sha256": "..."
|
| 162 |
+
}
|
| 163 |
+
}
|
| 164 |
+
}
|
| 165 |
+
```
|
| 166 |
+
|
| 167 |
+
This prevents a wrapper around Tesseract, Surya, or a VLM from satisfying the
|
| 168 |
+
paper objective by benchmark routing alone.
|
| 169 |
+
|
| 170 |
+
For recognizer branch admission, set `require_model_admission: true` and record
|
| 171 |
+
the admission validation report in the model card:
|
| 172 |
+
|
| 173 |
+
```json
|
| 174 |
+
{
|
| 175 |
+
"metrics": {
|
| 176 |
+
"admission_validation": {
|
| 177 |
+
"path": "outputs/validation-next-recognizer-hard50/validation.json",
|
| 178 |
+
"sha256": "...",
|
| 179 |
+
"required_status": "validated"
|
| 180 |
+
}
|
| 181 |
+
}
|
| 182 |
+
}
|
| 183 |
+
```
|
| 184 |
+
|
| 185 |
+
This catches the failure mode we hit with the PaddleOCR branches: the model can
|
| 186 |
+
have artifacts and train/eval provenance, but still be empirically rejected by
|
| 187 |
+
the hard recognizer gate. Rejected admission reports are summarized with their
|
| 188 |
+
failed requirements and gate deltas, so review output can show why a recognizer
|
| 189 |
+
was rejected without manually opening the raw validation JSON. Admission reports
|
| 190 |
+
are also bound back to `provenance.candidate_model_config`; review rejects a
|
| 191 |
+
passing admission artifact if it points at a different model id, backend, base
|
| 192 |
+
model, or artifact set.
|
| 193 |
+
|
| 194 |
+
For document-engine SOTA claims, metric presence is not enough. The
|
| 195 |
+
`configs/claim-review-sota-v1.json` portfolio requires passing validation gates
|
| 196 |
+
for the named claim dimensions:
|
| 197 |
+
|
| 198 |
+
- OCR: CER and WER gates on Nepali/English slices where applicable
|
| 199 |
+
- tables: `table_cell_f1`
|
| 200 |
+
- reading order: `reading_order_pair_accuracy`
|
| 201 |
+
- figures: `figure_detection_f1` and `figure_caption_cer`
|
| 202 |
+
- reconstruction: `document_reconstruction_similarity`
|
| 203 |
+
|
| 204 |
+
The corresponding validation configs must contain matching gates. The
|
| 205 |
+
`tests/test_claim_review_sota_config.py` checks pin this relationship so a
|
| 206 |
+
future config edit cannot silently downgrade figure/caption or WER from a
|
| 207 |
+
passing gate to a reported-only metric.
|
| 208 |
+
|
| 209 |
+
The SOTA profile also requires exported structured-output artifacts from the
|
| 210 |
+
candidate benchmark rows:
|
| 211 |
+
|
| 212 |
+
- every required slot: `document.json`, `document.md`, and `quality.json`
|
| 213 |
+
- table slices: table CSV and HTML exports under `tables/`
|
| 214 |
+
- figure slices: `figures/metadata.json` and exported figure image files
|
| 215 |
+
|
| 216 |
+
`review-claim` infers these paths from the benchmark report directory and the
|
| 217 |
+
candidate row input stem. It also performs minimal content checks: JSON
|
| 218 |
+
artifacts must parse with the expected top-level shape, Markdown/table/image
|
| 219 |
+
files must be nonempty, and figure metadata must contain entries. This means a
|
| 220 |
+
claim cannot be supported only by scalar metrics if the human-readable Markdown,
|
| 221 |
+
machine-readable JSON, tables, or figures were not actually produced as usable
|
| 222 |
+
artifacts.
|
| 223 |
+
|
| 224 |
+
## Example Config
|
| 225 |
+
|
| 226 |
+
```json
|
| 227 |
+
{
|
| 228 |
+
"require_trained_recognizer": true,
|
| 229 |
+
"require_model_admission": true,
|
| 230 |
+
"require_fallback_model_present": true,
|
| 231 |
+
"require_candidate_fallback_provenance": true,
|
| 232 |
+
"require_shared_fallback_model": true,
|
| 233 |
+
"require_fallback_model_audit_passed": true,
|
| 234 |
+
"required_experiments": [
|
| 235 |
+
{
|
| 236 |
+
"name": "internal-nepali-pack",
|
| 237 |
+
"datasets": ["ocrtech-eval-pack"],
|
| 238 |
+
"require_validation_status": "validated",
|
| 239 |
+
"require_baselines": ["candidate", "tesseract", "stock-paddle", "glm-ocr", "paddleocr-vl"],
|
| 240 |
+
"required_metrics": ["cer", "wer", "document_reconstruction_similarity", "table_cell_f1", "reading_order_pair_accuracy", "figure_detection_f1", "figure_caption_cer"],
|
| 241 |
+
"required_metric_slices": {
|
| 242 |
+
"cer": ["nepali", "english"],
|
| 243 |
+
"wer": ["nepali", "english"],
|
| 244 |
+
"document_reconstruction_similarity": ["table", "reading_order", "figure"],
|
| 245 |
+
"table_cell_f1": ["table"],
|
| 246 |
+
"reading_order_pair_accuracy": ["reading_order"],
|
| 247 |
+
"figure_detection_f1": ["figure"],
|
| 248 |
+
"figure_caption_cer": ["figure"]
|
| 249 |
+
},
|
| 250 |
+
"required_validation_gates": [
|
| 251 |
+
{"baseline": "tesseract", "metric": "cer", "slices": ["nepali"]},
|
| 252 |
+
{"baseline": "stock-paddle", "metric": "cer", "slices": ["nepali"]},
|
| 253 |
+
{"baseline": "glm-ocr", "metric": "cer", "slices": ["nepali"]},
|
| 254 |
+
{"baseline": "surya", "metric": "table_cell_f1", "slices": ["table"]},
|
| 255 |
+
{"baseline": "paddleocr-vl", "metric": "reading_order_pair_accuracy", "slices": ["reading_order"]}
|
| 256 |
+
],
|
| 257 |
+
"require_candidate_fallback_metrics": true,
|
| 258 |
+
"max_candidate_fallback_trigger_rate": 0.35,
|
| 259 |
+
"max_candidate_fallback_failed_rate": 0.0,
|
| 260 |
+
"required_slices": ["nepali", "english", "table", "reading_order", "figure", "scan"]
|
| 261 |
+
},
|
| 262 |
+
{
|
| 263 |
+
"name": "ocrbench-v2",
|
| 264 |
+
"datasets": ["ocrbench-v2"],
|
| 265 |
+
"require_validation_status": "validated",
|
| 266 |
+
"required_metrics": ["cer", "wer"],
|
| 267 |
+
"require_baselines": ["candidate", "glm-ocr", "paddleocr-vl"]
|
| 268 |
+
},
|
| 269 |
+
{
|
| 270 |
+
"name": "omnidocbench",
|
| 271 |
+
"datasets": ["omnidocbench"],
|
| 272 |
+
"require_validation_status": "validated",
|
| 273 |
+
"required_metrics": ["cer", "document_reconstruction_similarity", "table_cell_f1", "reading_order_pair_accuracy"],
|
| 274 |
+
"require_baselines": ["candidate", "paddleocr-vl"]
|
| 275 |
+
}
|
| 276 |
+
]
|
| 277 |
+
}
|
| 278 |
+
```
|
| 279 |
+
|
| 280 |
+
The config is intentionally explicit. If two experiments match one slot, the review fails. Tighten the config until every required slot maps to one concrete artifact.
|
| 281 |
+
`eval_manifest_patterns` use shell-style wildcards, so a slot such as
|
| 282 |
+
`"*Real5-OmniDocBench*"` matches a manifest path containing that benchmark name.
|
| 283 |
+
|
| 284 |
+
The checked-in SOTA profile is paired with slot-specific validation configs:
|
| 285 |
+
|
| 286 |
+
- `configs/validation-internal-sota-v1.json`
|
| 287 |
+
- `configs/validation-public-omnidocbench-sota-v1.json`
|
| 288 |
+
- `configs/validation-public-real5-sota-v1.json`
|
| 289 |
+
|
| 290 |
+
Those configs are expected to produce the passed gates referenced by
|
| 291 |
+
`required_validation_gates`.
|
| 292 |
+
|
| 293 |
+
## Run
|
| 294 |
+
|
| 295 |
+
```bash
|
| 296 |
+
ocrtech review-claim \
|
| 297 |
+
--experiment outputs/internal-claim/experiment.json \
|
| 298 |
+
--experiment outputs/ocrbench-v2/experiment.json \
|
| 299 |
+
--experiment outputs/omnidocbench/experiment.json \
|
| 300 |
+
--config claim-review.json \
|
| 301 |
+
--model-config runs/recognizer/package/model-card.json \
|
| 302 |
+
--out outputs/claim-review
|
| 303 |
+
```
|
| 304 |
+
|
| 305 |
+
## Outputs
|
| 306 |
+
|
| 307 |
+
```text
|
| 308 |
+
outputs/claim-review/
|
| 309 |
+
claim-review.json
|
| 310 |
+
claim-review.md
|
| 311 |
+
```
|
| 312 |
+
|
| 313 |
+
If this report fails, the claim is not review-ready.
|
| 314 |
+
|
| 315 |
+
Do not hand-edit copied `experiment.json` files to point at convenient local
|
| 316 |
+
paths. If an experiment was run remotely, sync the benchmark report, manifest,
|
| 317 |
+
references, model card, and validation report together, or regenerate the
|
| 318 |
+
experiment report from the synced paths. `review-claim` intentionally fails on
|
| 319 |
+
stale paths instead of guessing because path rewrites without hashes destroy
|
| 320 |
+
claim provenance.
|
docs/DATASETS.md
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Dataset Workflow
|
| 2 |
+
|
| 3 |
+
The dataset workflow is deliberately strict. If train/eval contamination gets in here, every later validation result is suspect.
|
| 4 |
+
|
| 5 |
+
## Prepare
|
| 6 |
+
|
| 7 |
+
Convert raw labels to JSONL:
|
| 8 |
+
|
| 9 |
+
```bash
|
| 10 |
+
ocrtech prepare-data labels.csv \
|
| 11 |
+
--out data/manifests \
|
| 12 |
+
--dataset-name nepali-lines \
|
| 13 |
+
--split all \
|
| 14 |
+
--image-field image_path \
|
| 15 |
+
--text-field text \
|
| 16 |
+
--strict-chars
|
| 17 |
+
```
|
| 18 |
+
|
| 19 |
+
If the source is noisy and you need an explicit reject list instead of failing at the first bad row:
|
| 20 |
+
|
| 21 |
+
```bash
|
| 22 |
+
ocrtech prepare-data labels.csv \
|
| 23 |
+
--out data/manifests \
|
| 24 |
+
--dataset-name nepali-lines \
|
| 25 |
+
--split all \
|
| 26 |
+
--image-field image_path \
|
| 27 |
+
--text-field text \
|
| 28 |
+
--skip-invalid
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
That writes:
|
| 32 |
+
|
| 33 |
+
- `data/manifests/nepali-lines-all.jsonl`
|
| 34 |
+
- `data/manifests/nepali-lines-all-prepare.json`
|
| 35 |
+
- `data/manifests/nepali-lines-all-prepare.md`
|
| 36 |
+
- `data/manifests/nepali-lines-all-rejects.jsonl` when any rows are skipped
|
| 37 |
+
|
| 38 |
+
Verified Hugging Face sources can be converted directly:
|
| 39 |
+
|
| 40 |
+
```bash
|
| 41 |
+
ocrtech prepare-data gauravgiri/nepali-ocr-dataset \
|
| 42 |
+
--hf \
|
| 43 |
+
--split train \
|
| 44 |
+
--limit 100 \
|
| 45 |
+
--skip-invalid \
|
| 46 |
+
--out data/manifests
|
| 47 |
+
|
| 48 |
+
ocrtech prepare-data prashant0919/nepali-synthetic-ocr-lines \
|
| 49 |
+
--hf \
|
| 50 |
+
--split train \
|
| 51 |
+
--limit 100 \
|
| 52 |
+
--out data/manifests
|
| 53 |
+
|
| 54 |
+
ocrtech prepare-data nvidia/OCR-Synthetic-Multilingual-v1 \
|
| 55 |
+
--hf \
|
| 56 |
+
--hf-subset en \
|
| 57 |
+
--split test \
|
| 58 |
+
--limit 1000 \
|
| 59 |
+
--out data/manifests-english-test
|
| 60 |
+
```
|
| 61 |
+
|
| 62 |
+
Before those conversion commands, record a schema inspection so the training
|
| 63 |
+
corpus has evidence that fields were verified from the actual source:
|
| 64 |
+
|
| 65 |
+
```bash
|
| 66 |
+
ocrtech inspect-hf-dataset gauravgiri/nepali-ocr-dataset \
|
| 67 |
+
--split train \
|
| 68 |
+
--limit 3 \
|
| 69 |
+
--out outputs/hf-inspect-training-v1
|
| 70 |
+
|
| 71 |
+
ocrtech inspect-hf-dataset prashant0919/nepali-synthetic-ocr-lines \
|
| 72 |
+
--split train \
|
| 73 |
+
--limit 3 \
|
| 74 |
+
--out outputs/hf-inspect-training-v1
|
| 75 |
+
|
| 76 |
+
ocrtech inspect-hf-dataset nvidia/OCR-Synthetic-Multilingual-v1 \
|
| 77 |
+
--hf-subset en \
|
| 78 |
+
--split test \
|
| 79 |
+
--limit 3 \
|
| 80 |
+
--out outputs/hf-inspect-training-v1
|
| 81 |
+
```
|
| 82 |
+
|
| 83 |
+
Verified on `k2` on `2026-06-15`:
|
| 84 |
+
|
| 85 |
+
| dataset | split/subset | mode | inferred image | inferred text | result |
|
| 86 |
+
| --- | --- | --- | --- | --- | --- |
|
| 87 |
+
| `gauravgiri/nepali-ocr-dataset` | `train` | `datasets.load_dataset` | `image` | `text` | pass |
|
| 88 |
+
| `prashant0919/nepali-synthetic-ocr-lines` | `train` | HF repo `metadata.jsonl` | `file_name` | `text` | pass |
|
| 89 |
+
| `nvidia/OCR-Synthetic-Multilingual-v1` | `test/en` | HDF5 `line_bboxes` | `images + annotations.line_bboxes[].bbox` | `annotations.line_bboxes[].text` | pass |
|
| 90 |
+
|
| 91 |
+
Artifacts:
|
| 92 |
+
|
| 93 |
+
- `outputs/hf-inspect-training-v1/gauravgiri__nepali-ocr-dataset-train-inspect.json`
|
| 94 |
+
- `outputs/hf-inspect-training-v1/prashant0919__nepali-synthetic-ocr-lines-train-inspect.json`
|
| 95 |
+
- `outputs/hf-inspect-training-v1/nvidia__OCR-Synthetic-Multilingual-v1__en-test-inspect.json`
|
| 96 |
+
|
| 97 |
+
## Public Benchmarks
|
| 98 |
+
|
| 99 |
+
Public document parsing benchmarks are tracked separately from training data.
|
| 100 |
+
Do not mix them into recognizer/corrector training manifests. They are final
|
| 101 |
+
claim evidence.
|
| 102 |
+
|
| 103 |
+
List the known registry:
|
| 104 |
+
|
| 105 |
+
```bash
|
| 106 |
+
ocrtech list-public-benchmarks
|
| 107 |
+
```
|
| 108 |
+
|
| 109 |
+
Current target order:
|
| 110 |
+
|
| 111 |
+
1. `opendatalab/OmniDocBench` for broad document parsing.
|
| 112 |
+
2. `PaddlePaddle/Real5-OmniDocBench` for real-world physical distortions.
|
| 113 |
+
3. `allenai/olmOCR-bench` for PDF-to-Markdown/unit-test style evaluation.
|
| 114 |
+
|
| 115 |
+
Prepare the verified OmniDocBench schema:
|
| 116 |
+
|
| 117 |
+
```bash
|
| 118 |
+
ocrtech prepare-public-benchmark opendatalab/OmniDocBench \
|
| 119 |
+
--out data/public/omnidocbench \
|
| 120 |
+
--limit 100 \
|
| 121 |
+
--language english \
|
| 122 |
+
--require-tables
|
| 123 |
+
```
|
| 124 |
+
|
| 125 |
+
This converter is schema-bound. It expects the verified Hugging Face layout:
|
| 126 |
+
`OmniDocBench.json` as a list of rows, each row with `page_info.image_path`,
|
| 127 |
+
`page_info.page_attribute`, and `layout_dets`. Table references come from
|
| 128 |
+
`layout_dets[].html`; text references come from ordered `layout_dets[].text`
|
| 129 |
+
plus table HTML text. Images are resolved under `images/`.
|
| 130 |
+
|
| 131 |
+
Before benchmarking a public pack, audit its manifest:
|
| 132 |
+
|
| 133 |
+
```bash
|
| 134 |
+
ocrtech audit-public-benchmark data/public/omnidocbench/manifests/omnidocbench-eval.jsonl \
|
| 135 |
+
--benchmark opendatalab/OmniDocBench \
|
| 136 |
+
--out outputs/public-omnidocbench-audit
|
| 137 |
+
```
|
| 138 |
+
|
| 139 |
+
Each public manifest row must carry provenance:
|
| 140 |
+
|
| 141 |
+
- `metadata.public_benchmark = true`
|
| 142 |
+
- `metadata.benchmark_name`
|
| 143 |
+
- `metadata.source_url`
|
| 144 |
+
- `metadata.reference_path`
|
| 145 |
+
- `metadata.slices` containing `public`
|
| 146 |
+
|
| 147 |
+
This is intentionally stricter than `prepare-data`. The agent must verify the
|
| 148 |
+
actual downloaded dataset schema before writing a converter for any public
|
| 149 |
+
benchmark. No schema guessing.
|
| 150 |
+
|
| 151 |
+
Current claim-sized public OmniDocBench table pack on `k2`:
|
| 152 |
+
|
| 153 |
+
- manifest: `data/public/omnidocbench-table100/manifests/omnidocbench-eval.jsonl`
|
| 154 |
+
- prepare report: `data/public/omnidocbench-table100/public-benchmark-prepare.json`
|
| 155 |
+
- audit: `outputs/public-omnidocbench-table100-audit/public-benchmark-audit.json`
|
| 156 |
+
- audit status: passed
|
| 157 |
+
- samples: `100`
|
| 158 |
+
- slices: `public=100`, `document_parsing=100`, `english=100`, `table=100`,
|
| 159 |
+
`table_hard=32`, `figure=22`
|
| 160 |
+
|
| 161 |
+
This is the first public OmniDocBench manifest in the repo that can satisfy the
|
| 162 |
+
configured `100`-page public document gate and the `40`-page public table gate.
|
| 163 |
+
|
| 164 |
+
For the current Nepali starter corpus, merge verified sources with the built-in command instead of shell concatenation:
|
| 165 |
+
|
| 166 |
+
```bash
|
| 167 |
+
ocrtech merge-manifests \
|
| 168 |
+
data/manifests/gauravgiri__nepali-ocr-dataset-train.jsonl \
|
| 169 |
+
data/manifests/prashant0919__nepali-synthetic-ocr-lines-train.jsonl \
|
| 170 |
+
--out data/manifests/nepali-mixed-all.jsonl
|
| 171 |
+
```
|
| 172 |
+
|
| 173 |
+
Do not pretend all HF datasets look the same. This repo only has verified adapters for the schemas we checked:
|
| 174 |
+
|
| 175 |
+
- `gauravgiri/nepali-ocr-dataset`: `image`, `text`
|
| 176 |
+
- `prashant0919/nepali-synthetic-ocr-lines`: `train/metadata.jsonl` plus image files
|
| 177 |
+
- `cfilt/RoundTripOCR-nepali`: `ocr`, `correct`, `font`
|
| 178 |
+
- `nvidia/OCR-Synthetic-Multilingual-v1`: HDF5 shards with `line_bboxes`
|
| 179 |
+
|
| 180 |
+
Known adapters also stamp verified slice metadata so later audits and experiments do not depend on hand-added tags:
|
| 181 |
+
|
| 182 |
+
- `gauravgiri/nepali-ocr-dataset`: `nepali`, `real`
|
| 183 |
+
- `prashant0919/nepali-synthetic-ocr-lines`: `nepali`, `synthetic`
|
| 184 |
+
- `nvidia/OCR-Synthetic-Multilingual-v1 --hf-subset en`: `english`, `synthetic`, `line_crop`, `en`
|
| 185 |
+
|
| 186 |
+
Verified on `k2` on June 14, 2026:
|
| 187 |
+
|
| 188 |
+
- `gauravgiri/nepali-ocr-dataset --limit 100 --skip-invalid`: `97` prepared, `3` rejected due to empty `text`
|
| 189 |
+
- `prashant0919/nepali-synthetic-ocr-lines --limit 100`: `100` prepared, `0` rejected
|
| 190 |
+
- merged starter corpus: `197` samples, no merge warnings
|
| 191 |
+
- deterministic split: `167` train / `30` eval, leakage check passed
|
| 192 |
+
|
| 193 |
+
Each manifest row contains:
|
| 194 |
+
|
| 195 |
+
- `sample_id`
|
| 196 |
+
- `dataset`
|
| 197 |
+
- `split`
|
| 198 |
+
- `image_path`
|
| 199 |
+
- `text`
|
| 200 |
+
- `sha256`
|
| 201 |
+
- `metadata.text_sha256`
|
| 202 |
+
- `metadata.sample_sha256`
|
| 203 |
+
|
| 204 |
+
## Audit
|
| 205 |
+
|
| 206 |
+
Before splitting or training:
|
| 207 |
+
|
| 208 |
+
```bash
|
| 209 |
+
ocrtech audit-manifest data/manifests/nepali-lines-all.jsonl \
|
| 210 |
+
--out outputs/manifest-audit \
|
| 211 |
+
--require-slice nepali \
|
| 212 |
+
--require-slice english \
|
| 213 |
+
--strict-chars
|
| 214 |
+
```
|
| 215 |
+
|
| 216 |
+
The audit checks:
|
| 217 |
+
|
| 218 |
+
- missing input files
|
| 219 |
+
- image SHA-256 mismatches
|
| 220 |
+
- duplicate sample IDs
|
| 221 |
+
- duplicate paths and hashes
|
| 222 |
+
- slice counts
|
| 223 |
+
- script counts
|
| 224 |
+
- unsupported characters when requested
|
| 225 |
+
|
| 226 |
+
## Split
|
| 227 |
+
|
| 228 |
+
Create train/eval manifests with deterministic grouping:
|
| 229 |
+
|
| 230 |
+
```bash
|
| 231 |
+
ocrtech split-manifest data/manifests/nepali-lines-all.jsonl \
|
| 232 |
+
--out data/splits \
|
| 233 |
+
--eval-ratio 0.15 \
|
| 234 |
+
--seed 13 \
|
| 235 |
+
--stratify-by slices
|
| 236 |
+
```
|
| 237 |
+
|
| 238 |
+
The splitter groups entries by `metadata.sample_sha256` before assigning splits by default. If the same image+text sample appears twice, it stays in one split.
|
| 239 |
+
|
| 240 |
+
For rendered packs with multiple degraded variants of the same underlying text, group by `text_sha256` instead. This keeps clean, blur, scan, low-contrast, and skew variants of the same target text on the same side of the train/eval boundary:
|
| 241 |
+
|
| 242 |
+
```bash
|
| 243 |
+
ocrtech split-manifest data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl \
|
| 244 |
+
--out data/splits-table-figure-dev-v1 \
|
| 245 |
+
--eval-ratio 0.2 \
|
| 246 |
+
--seed 17 \
|
| 247 |
+
--stratify-by document_type \
|
| 248 |
+
--group-by text_sha256
|
| 249 |
+
```
|
| 250 |
+
|
| 251 |
+
Supported grouping keys are:
|
| 252 |
+
|
| 253 |
+
- `sample_signature`
|
| 254 |
+
- `text_sha256`
|
| 255 |
+
- `image_sha256`
|
| 256 |
+
- `image_path`
|
| 257 |
+
- `sample_id`
|
| 258 |
+
|
| 259 |
+
Outputs:
|
| 260 |
+
|
| 261 |
+
```text
|
| 262 |
+
data/splits/
|
| 263 |
+
train.jsonl
|
| 264 |
+
eval.jsonl
|
| 265 |
+
split-summary.json
|
| 266 |
+
split-summary.md
|
| 267 |
+
```
|
| 268 |
+
|
| 269 |
+
The split summary records the grouping key. For the table/figure development pack, the verified `k2` split produced:
|
| 270 |
+
|
| 271 |
+
- `60` train samples
|
| 272 |
+
- `20` eval samples
|
| 273 |
+
- `16` text groups
|
| 274 |
+
- `group_by = text_sha256`
|
| 275 |
+
- leakage check passed
|
| 276 |
+
|
| 277 |
+
## Rebalance
|
| 278 |
+
|
| 279 |
+
Use `rebalance-manifest` only on the training split when a slice is obviously underrepresented and you want a reproducible upsampling step instead of an undocumented training-time sampler choice:
|
| 280 |
+
|
| 281 |
+
```bash
|
| 282 |
+
ocrtech rebalance-manifest data/splits/train.jsonl \
|
| 283 |
+
--out data/splits/train-rebalanced.jsonl \
|
| 284 |
+
--slice english \
|
| 285 |
+
--slice nepali \
|
| 286 |
+
--seed 13
|
| 287 |
+
```
|
| 288 |
+
|
| 289 |
+
Outputs:
|
| 290 |
+
|
| 291 |
+
```text
|
| 292 |
+
data/splits/
|
| 293 |
+
train-rebalanced.jsonl
|
| 294 |
+
train-rebalanced-rebalance.json
|
| 295 |
+
train-rebalanced-rebalance.md
|
| 296 |
+
```
|
| 297 |
+
|
| 298 |
+
This command duplicates rows with explicit provenance in metadata:
|
| 299 |
+
|
| 300 |
+
- `rebalance_duplicate`
|
| 301 |
+
- `rebalance_source_sample_id`
|
| 302 |
+
- `rebalance_slice`
|
| 303 |
+
|
| 304 |
+
Do not rebalance the unsplit combined manifest. That would duplicate samples before train/eval assignment and corrupt the benchmark.
|
| 305 |
+
|
| 306 |
+
## Train
|
| 307 |
+
|
| 308 |
+
Use only the split outputs:
|
| 309 |
+
|
| 310 |
+
```bash
|
| 311 |
+
ocrtech train-recognizer \
|
| 312 |
+
--train-manifest data/splits/train.jsonl \
|
| 313 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 314 |
+
--out runs/recognizer
|
| 315 |
+
```
|
| 316 |
+
|
| 317 |
+
For generated document packs, do not train a recognizer on whole rendered pages unless the model is explicitly document-level. The current full-page HF branch failed badly. Extract line/cell/caption crops first, then split those crop rows with text-level grouping:
|
| 318 |
+
|
| 319 |
+
```bash
|
| 320 |
+
ocrtech extract-crops \
|
| 321 |
+
data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl \
|
| 322 |
+
--out data/crops-table-figure-dev-v1 \
|
| 323 |
+
--crop-type line \
|
| 324 |
+
--crop-type table-cell \
|
| 325 |
+
--crop-type figure-caption
|
| 326 |
+
|
| 327 |
+
ocrtech split-manifest data/crops-table-figure-dev-v1/manifests/crops.jsonl \
|
| 328 |
+
--out data/splits-crops-table-figure-dev-v1 \
|
| 329 |
+
--eval-ratio 0.2 \
|
| 330 |
+
--seed 17 \
|
| 331 |
+
--stratify-by document_type \
|
| 332 |
+
--group-by text_sha256
|
| 333 |
+
```
|
| 334 |
+
|
| 335 |
+
The crop extractor only reads explicit renderer layout metadata from the reference JSON. If that metadata is absent, it fails instead of guessing bboxes.
|
| 336 |
+
|
| 337 |
+
Before training, build the recognizer corpus from verified manifests with
|
| 338 |
+
explicit bucket targets:
|
| 339 |
+
|
| 340 |
+
```bash
|
| 341 |
+
ocrtech build-recognizer-corpus \
|
| 342 |
+
data/manifests/english-lines.jsonl \
|
| 343 |
+
data/manifests/nepali-lines.jsonl \
|
| 344 |
+
data/splits-crops-table-figure-dev-v1/train.jsonl \
|
| 345 |
+
--out data/splits/recognizer-train-balanced.jsonl \
|
| 346 |
+
--target-latin-only 200 \
|
| 347 |
+
--target-devanagari-only 200 \
|
| 348 |
+
--target-mixed 100 \
|
| 349 |
+
--target-real 100 \
|
| 350 |
+
--target-synthetic 100
|
| 351 |
+
```
|
| 352 |
+
|
| 353 |
+
This command upsamples existing examples only. If a requested bucket is missing
|
| 354 |
+
from the inputs, it fails; add source data instead of weakening the audit gate.
|
| 355 |
+
|
| 356 |
+
For OCR correction from the verified CFILT dataset:
|
| 357 |
+
|
| 358 |
+
```bash
|
| 359 |
+
ocrtech train-corrector \
|
| 360 |
+
--hf-dataset cfilt/RoundTripOCR-nepali \
|
| 361 |
+
--hf-split train \
|
| 362 |
+
--limit 1000 \
|
| 363 |
+
--out runs/corrector-cfilt
|
| 364 |
+
```
|
| 365 |
+
|
| 366 |
+
Do not point training at the combined manifest.
|
| 367 |
+
|
| 368 |
+
## Validation
|
| 369 |
+
|
| 370 |
+
Pass the training manifests into claim validation:
|
| 371 |
+
|
| 372 |
+
```bash
|
| 373 |
+
ocrtech validate-claim \
|
| 374 |
+
--benchmark-report outputs/benchmark/report.json \
|
| 375 |
+
--eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
|
| 376 |
+
--train-manifest data/splits/train.jsonl \
|
| 377 |
+
--config validation.json \
|
| 378 |
+
--out outputs/validation
|
| 379 |
+
```
|
| 380 |
+
|
| 381 |
+
The validation step checks train/eval leakage again. That is intentional redundancy.
|
docs/EVAL_PACK.md
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Eval Pack Workflow
|
| 2 |
+
|
| 3 |
+
The eval pack is the internal held-out document set used to test the exact target domain:
|
| 4 |
+
|
| 5 |
+
- Devanagari/Nepali OCR
|
| 6 |
+
- English OCR
|
| 7 |
+
- mixed-script lines
|
| 8 |
+
- multi-page mixed-script documents
|
| 9 |
+
- tables
|
| 10 |
+
- forms
|
| 11 |
+
- figure captions
|
| 12 |
+
- reading order
|
| 13 |
+
- noisy scanned variants
|
| 14 |
+
|
| 15 |
+
This does not replace public benchmarks. It complements public suites with the Nepali + English documents we actually care about.
|
| 16 |
+
|
| 17 |
+
## Smoke Pack
|
| 18 |
+
|
| 19 |
+
A text pack exercises document reconstruction and scoring without needing OCR engines:
|
| 20 |
+
|
| 21 |
+
```bash
|
| 22 |
+
ocrtech create-eval-pack \
|
| 23 |
+
--out data/eval-pack-smoke \
|
| 24 |
+
--input-format text
|
| 25 |
+
|
| 26 |
+
ocrtech audit-eval-pack \
|
| 27 |
+
data/eval-pack-smoke/manifests/ocrtech-eval-pack.jsonl \
|
| 28 |
+
--out outputs/eval-pack-smoke-audit
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
Text packs are not OCR evidence. They are useful for validating Markdown/JSON/table/reading-order scoring.
|
| 32 |
+
|
| 33 |
+
## Claim Pack
|
| 34 |
+
|
| 35 |
+
An OCR claim pack must use rendered image inputs:
|
| 36 |
+
|
| 37 |
+
```bash
|
| 38 |
+
ocrtech create-eval-pack \
|
| 39 |
+
--out data/eval-pack \
|
| 40 |
+
--input-format image \
|
| 41 |
+
--degradation clean \
|
| 42 |
+
--degradation scan \
|
| 43 |
+
--degradation blur \
|
| 44 |
+
--degradation low_contrast \
|
| 45 |
+
--degradation skew \
|
| 46 |
+
--count-per-template 25
|
| 47 |
+
|
| 48 |
+
ocrtech audit-eval-pack \
|
| 49 |
+
data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
|
| 50 |
+
--out outputs/eval-pack-audit \
|
| 51 |
+
--require-claim-ready
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
Image packs require Pillow:
|
| 55 |
+
|
| 56 |
+
```bash
|
| 57 |
+
python -m pip install -e '.[eval]'
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
For Devanagari rendering quality, pass a real font if auto-discovery is not good enough:
|
| 61 |
+
|
| 62 |
+
```bash
|
| 63 |
+
ocrtech create-eval-pack \
|
| 64 |
+
--out data/eval-pack \
|
| 65 |
+
--input-format image \
|
| 66 |
+
--font-path /path/to/NotoSansDevanagari-Regular.ttf
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
## Targeted Development Packs
|
| 70 |
+
|
| 71 |
+
Use template selection for model-development packs that target a known failure pocket. These packs are useful for iteration, but they are not broad claim evidence unless they also satisfy the claim-pack coverage requirements.
|
| 72 |
+
|
| 73 |
+
```bash
|
| 74 |
+
ocrtech create-eval-pack \
|
| 75 |
+
--out data/eval-pack-table-figure-dev-v1 \
|
| 76 |
+
--input-format image \
|
| 77 |
+
--template table-ledger \
|
| 78 |
+
--template figure-caption \
|
| 79 |
+
--count-per-template 8 \
|
| 80 |
+
--variant-offset 100 \
|
| 81 |
+
--degradation clean \
|
| 82 |
+
--degradation scan \
|
| 83 |
+
--degradation blur \
|
| 84 |
+
--degradation low_contrast \
|
| 85 |
+
--degradation skew
|
| 86 |
+
```
|
| 87 |
+
|
| 88 |
+
Use `--variant-offset` when creating a development pack after a held-out pack, so sample IDs and visible reference text do not collide with the held-out evidence. Still check leakage explicitly before using a generated pack for training.
|
| 89 |
+
|
| 90 |
+
For span-aware table validation, use the `complex-table-ledger` template:
|
| 91 |
+
|
| 92 |
+
```bash
|
| 93 |
+
ocrtech create-eval-pack \
|
| 94 |
+
--out data/eval-pack-complex-table-dev-v1 \
|
| 95 |
+
--input-format image \
|
| 96 |
+
--template complex-table-ledger \
|
| 97 |
+
--count-per-template 8 \
|
| 98 |
+
--variant-offset 500 \
|
| 99 |
+
--degradation clean \
|
| 100 |
+
--degradation scan \
|
| 101 |
+
--degradation blur \
|
| 102 |
+
--degradation low_contrast
|
| 103 |
+
```
|
| 104 |
+
|
| 105 |
+
This template renders a visible rectangular table but writes structured reference cells with a merged header (`colspan=2`). It is useful for validating that scorers and candidate outputs preserve spans. It is still generated data, not a substitute for real scanned complex tables.
|
| 106 |
+
|
| 107 |
+
## Artifacts
|
| 108 |
+
|
| 109 |
+
Generated structure:
|
| 110 |
+
|
| 111 |
+
```text
|
| 112 |
+
data/eval-pack/
|
| 113 |
+
inputs/
|
| 114 |
+
references/
|
| 115 |
+
manifests/ocrtech-eval-pack.jsonl
|
| 116 |
+
eval-pack.json
|
| 117 |
+
eval-pack.md
|
| 118 |
+
```
|
| 119 |
+
|
| 120 |
+
Each manifest entry includes:
|
| 121 |
+
|
| 122 |
+
- `sample_id`
|
| 123 |
+
- `image_path`
|
| 124 |
+
- `sha256`
|
| 125 |
+
- `metadata.slices`
|
| 126 |
+
- `metadata.document_type`
|
| 127 |
+
- `metadata.degradation`
|
| 128 |
+
- `metadata.input_format`
|
| 129 |
+
- `metadata.reference_path`
|
| 130 |
+
- `metadata.claim_evidence_eligible`
|
| 131 |
+
|
| 132 |
+
For single-page samples, `image_path` points to one file. For multi-page samples, `image_path` points to a page-bundle directory such as:
|
| 133 |
+
|
| 134 |
+
```text
|
| 135 |
+
inputs/eval-multi-page-brief-000-clean/
|
| 136 |
+
page-0001.png
|
| 137 |
+
page-0002.png
|
| 138 |
+
```
|
| 139 |
+
|
| 140 |
+
Text smoke packs use the same shape for multi-page samples, but with a `document.ocr.json` sidecar inside the bundle directory so the deterministic sidecar engine can reconstruct the full document.
|
| 141 |
+
|
| 142 |
+
Structured references contain:
|
| 143 |
+
|
| 144 |
+
- `text`
|
| 145 |
+
- `reading_order`
|
| 146 |
+
- `tables` as either nested grids or structured cell objects with `row`, `col`, `rowspan`, and `colspan`
|
| 147 |
+
- `figures`
|
| 148 |
+
- metadata
|
| 149 |
+
|
| 150 |
+
For image packs generated after the crop-extraction update, reference metadata also contains renderer layout metadata:
|
| 151 |
+
|
| 152 |
+
- `metadata.layout.pages[].lines[]`
|
| 153 |
+
- `metadata.layout.pages[].table_cells[]`
|
| 154 |
+
- validated pixel bboxes for non-skew degradations
|
| 155 |
+
|
| 156 |
+
Skewed samples deliberately do not expose pre-rotation bboxes. Crop extraction skips those unavailable pages instead of using invalid coordinates.
|
| 157 |
+
|
| 158 |
+
## Crop Manifests
|
| 159 |
+
|
| 160 |
+
The failed full-page TrOCR table/figure branch showed that treating a whole page as one recognition target is the wrong training shape. Use crop manifests for the next recognizer branch:
|
| 161 |
+
|
| 162 |
+
```bash
|
| 163 |
+
ocrtech extract-crops \
|
| 164 |
+
data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl \
|
| 165 |
+
--out data/crops-table-figure-dev-v1 \
|
| 166 |
+
--crop-type line \
|
| 167 |
+
--crop-type table-cell \
|
| 168 |
+
--crop-type figure-caption
|
| 169 |
+
```
|
| 170 |
+
|
| 171 |
+
This writes:
|
| 172 |
+
|
| 173 |
+
```text
|
| 174 |
+
data/crops-table-figure-dev-v1/
|
| 175 |
+
images/
|
| 176 |
+
manifests/crops.jsonl
|
| 177 |
+
crop-manifest-summary.json
|
| 178 |
+
crop-manifest-summary.md
|
| 179 |
+
```
|
| 180 |
+
|
| 181 |
+
Each crop manifest row is a normal OCR training row with:
|
| 182 |
+
|
| 183 |
+
- crop image path
|
| 184 |
+
- crop text label
|
| 185 |
+
- source sample id
|
| 186 |
+
- source image path
|
| 187 |
+
- source reference path
|
| 188 |
+
- page index
|
| 189 |
+
- source bbox
|
| 190 |
+
- crop type
|
| 191 |
+
- `text_sha256`
|
| 192 |
+
- `sample_sha256`
|
| 193 |
+
|
| 194 |
+
If a reference lacks renderer layout metadata, `extract-crops` fails. That is intentional; do not infer crop coordinates from template names or visual guesses.
|
| 195 |
+
|
| 196 |
+
## Benchmark
|
| 197 |
+
|
| 198 |
+
Run the same inputs against every baseline:
|
| 199 |
+
|
| 200 |
+
```bash
|
| 201 |
+
inputs=$(python - <<'PY'
|
| 202 |
+
from ocrtech.manifest import load_manifest
|
| 203 |
+
for entry in load_manifest("data/eval-pack/manifests/ocrtech-eval-pack.jsonl"):
|
| 204 |
+
print(entry.image_path)
|
| 205 |
+
PY
|
| 206 |
+
)
|
| 207 |
+
|
| 208 |
+
ocrtech benchmark $inputs \
|
| 209 |
+
--out outputs/benchmark \
|
| 210 |
+
--eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
|
| 211 |
+
--references-dir data/eval-pack/references \
|
| 212 |
+
--baselines ours,tesseract,stock-paddle,glm-ocr,paddleocr-vl
|
| 213 |
+
```
|
| 214 |
+
|
| 215 |
+
Then validate the claim:
|
| 216 |
+
|
| 217 |
+
```bash
|
| 218 |
+
ocrtech validate-claim \
|
| 219 |
+
--benchmark-report outputs/benchmark/report.json \
|
| 220 |
+
--eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
|
| 221 |
+
--train-manifest data/manifests/train.jsonl \
|
| 222 |
+
--config validation.json \
|
| 223 |
+
--out outputs/validation
|
| 224 |
+
```
|
| 225 |
+
|
| 226 |
+
Do not use `--engine sidecar` or OCR sidecars for claim benchmarks.
|
docs/EXPERIMENTS.md
ADDED
|
@@ -0,0 +1,1034 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Experiment Workflow
|
| 2 |
+
|
| 3 |
+
If you are writing a paper from these runs, read [PAPER_NOTES.md](PAPER_NOTES.md) alongside this file. This document is the raw chronology; `PAPER_NOTES.md` is the stricter claim ledger.
|
| 4 |
+
|
| 5 |
+
`run-experiment` is the reproducible wrapper around benchmark and validation. It records the manifest hash, train manifest hashes, environment metadata, baseline list, benchmark report path, and validation status.
|
| 6 |
+
|
| 7 |
+
## Smoke Experiment
|
| 8 |
+
|
| 9 |
+
```bash
|
| 10 |
+
ocrtech create-eval-pack --out data/eval-pack-smoke --input-format text
|
| 11 |
+
|
| 12 |
+
ocrtech run-experiment \
|
| 13 |
+
--eval-manifest data/eval-pack-smoke/manifests/ocrtech-eval-pack.jsonl \
|
| 14 |
+
--references-dir data/eval-pack-smoke/references \
|
| 15 |
+
--baselines ours \
|
| 16 |
+
--skip-validation \
|
| 17 |
+
--out outputs/smoke-experiment
|
| 18 |
+
```
|
| 19 |
+
|
| 20 |
+
This proves the pipeline and scoring path works. It is not OCR evidence.
|
| 21 |
+
|
| 22 |
+
## Claim Experiment
|
| 23 |
+
|
| 24 |
+
Run preflight first:
|
| 25 |
+
|
| 26 |
+
```bash
|
| 27 |
+
ocrtech preflight-claim \
|
| 28 |
+
--eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
|
| 29 |
+
--references-dir data/eval-pack/references \
|
| 30 |
+
--baselines candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
|
| 31 |
+
--candidate-model-config runs/recognizer/package/model-card.json \
|
| 32 |
+
--validation-config validation.json \
|
| 33 |
+
--train-manifest data/splits/train.jsonl \
|
| 34 |
+
--out outputs/preflight
|
| 35 |
+
```
|
| 36 |
+
|
| 37 |
+
If the candidate is a routed composite, do not under-specify the baseline list. A claim-grade `script_select_composite` run must include its raw components in the same experiment, for example `candidate,tesseract,surya`, otherwise `preflight-claim` now fails `candidate-components`.
|
| 38 |
+
|
| 39 |
+
```bash
|
| 40 |
+
ocrtech run-experiment \
|
| 41 |
+
--eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
|
| 42 |
+
--references-dir data/eval-pack/references \
|
| 43 |
+
--baselines candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
|
| 44 |
+
--candidate-model-config runs/recognizer/package/model-card.json \
|
| 45 |
+
--validation-config validation.json \
|
| 46 |
+
--train-manifest data/splits/train.jsonl \
|
| 47 |
+
--preflight \
|
| 48 |
+
--require-trained-recognizer \
|
| 49 |
+
--require-model-admission \
|
| 50 |
+
--out outputs/claim-experiment \
|
| 51 |
+
--require-validated
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
`--preflight` runs the same claim-readiness checks inside `run-experiment` and
|
| 55 |
+
stops before benchmark output is produced if the setup cannot satisfy the
|
| 56 |
+
configured gates. `--require-validated` returns a non-zero exit code unless
|
| 57 |
+
validation status is `validated`.
|
| 58 |
+
|
| 59 |
+
After the internal claim experiment and the public suites are all complete, run `review-claim`; see [CLAIM_REVIEW.md](CLAIM_REVIEW.md).
|
| 60 |
+
|
| 61 |
+
Use `summarize-benchmark` when you need aggregate and paired slice comparisons from a raw `benchmark/report.json`:
|
| 62 |
+
|
| 63 |
+
```bash
|
| 64 |
+
ocrtech summarize-benchmark \
|
| 65 |
+
--benchmark-report outputs/claim-experiment/benchmark/report.json \
|
| 66 |
+
--out outputs/claim-experiment/benchmark-summary \
|
| 67 |
+
--candidate candidate \
|
| 68 |
+
--baseline tesseract \
|
| 69 |
+
--metric cer
|
| 70 |
+
```
|
| 71 |
+
|
| 72 |
+
Use `paper-benchmark-table` when you need one paper-facing table merged from multiple `summary.json` artifacts, for example separate candidate-vs-Tesseract, candidate-vs-Surya, and candidate-vs-stock-Paddle summaries from the same pack:
|
| 73 |
+
|
| 74 |
+
```bash
|
| 75 |
+
ocrtech paper-benchmark-table \
|
| 76 |
+
--summary outputs/paper-claimpack-summary-tesseract/summary.json \
|
| 77 |
+
--summary outputs/paper-claimpack-summary-surya/summary.json \
|
| 78 |
+
--summary outputs/paper-claimpack-summary-stock-paddle/summary.json \
|
| 79 |
+
--out outputs/paper-claimpack-table \
|
| 80 |
+
--system candidate \
|
| 81 |
+
--system surya \
|
| 82 |
+
--system tesseract \
|
| 83 |
+
--system stock-paddle \
|
| 84 |
+
--label 'candidate=Candidate' \
|
| 85 |
+
--label 'surya=Surya' \
|
| 86 |
+
--label 'tesseract=Tesseract' \
|
| 87 |
+
--label 'stock-paddle=Stock Paddle' \
|
| 88 |
+
--source-preference 'candidate=outputs/paper-claimpack-summary-tesseract/summary.json' \
|
| 89 |
+
--source-preference 'tesseract=outputs/paper-claimpack-summary-tesseract/summary.json' \
|
| 90 |
+
--source-preference 'surya=outputs/paper-claimpack-summary-surya/summary.json' \
|
| 91 |
+
--source-preference 'stock-paddle=outputs/paper-claimpack-summary-stock-paddle/summary.json' \
|
| 92 |
+
--metric cer \
|
| 93 |
+
--metric table_cell_f1 \
|
| 94 |
+
--metric reading_order_pair_accuracy \
|
| 95 |
+
--metric latency_seconds
|
| 96 |
+
```
|
| 97 |
+
|
| 98 |
+
The `--source-preference` pins each system to one summary artifact. That matters when the same candidate was rerun in separate comparisons and latency differs across those reruns.
|
| 99 |
+
|
| 100 |
+
## Outputs
|
| 101 |
+
|
| 102 |
+
```text
|
| 103 |
+
outputs/claim-experiment/
|
| 104 |
+
benchmark/report.json
|
| 105 |
+
benchmark/report.md
|
| 106 |
+
validation/validation.json
|
| 107 |
+
validation/validation.md
|
| 108 |
+
experiment.json
|
| 109 |
+
experiment.md
|
| 110 |
+
```
|
| 111 |
+
|
| 112 |
+
`experiment.json` is the artifact to attach to a claim review. It does not replace the full benchmark and validation reports.
|
| 113 |
+
|
| 114 |
+
For candidate model-card requirements, see [MODELS.md](MODELS.md).
|
| 115 |
+
|
| 116 |
+
## Verified k2 HF Runs
|
| 117 |
+
|
| 118 |
+
As of June 14, 2026, the most useful live experiment series on `k2` is a candidate-only TrOCR comparison over one held-out mixed Nepali/English eval manifest with `180` samples. This is workflow evidence, not claim evidence, because it does not yet include Tesseract, stock Paddle, GLM-OCR, PaddleOCR-VL, or public benchmark suites.
|
| 119 |
+
|
| 120 |
+
Corpus construction used:
|
| 121 |
+
|
| 122 |
+
- `97` valid `gauravgiri` Nepali real rows from the first `100`
|
| 123 |
+
- `100` `prashant0919` Nepali synthetic rows
|
| 124 |
+
- `1000` NVIDIA English synthetic line crops from `nvidia/OCR-Synthetic-Multilingual-v1 --hf-subset en --split test`
|
| 125 |
+
- deterministic split: `1017` train / `180` eval
|
| 126 |
+
- materialized references from the eval manifest
|
| 127 |
+
- rebalanced training variant: `english=850`, `nepali=850`
|
| 128 |
+
|
| 129 |
+
Artifacts pulled from `k2` into this workspace:
|
| 130 |
+
|
| 131 |
+
- `outputs/live-k2-baseline-bundle/outputs/experiment-hf-smoke-on-mixed-v1/benchmark/report.json`
|
| 132 |
+
- `outputs/live-k2-baseline-bundle/outputs/experiment-hf-mixed-v1/benchmark/report.json`
|
| 133 |
+
- `outputs/live-k2-rebalanced-bundle/outputs/experiment-hf-mixed-v1-rebalanced/benchmark/report.json`
|
| 134 |
+
- `outputs/live-k2-rebalanced-bundle/runs/hf-recognizer-mixed-v1-rebalanced/training_metrics.json`
|
| 135 |
+
- `outputs/live-k2-rebalanced-bundle/data/splits-mixed-v1/train-rebalanced-nep-eng-rebalance.json`
|
| 136 |
+
- `outputs/live-k2-vs-tesseract-bundle/outputs/experiment-hf-mixed-v1-rebalanced-vs-tesseract/benchmark/report.json`
|
| 137 |
+
|
| 138 |
+
Benchmark means on that same eval split:
|
| 139 |
+
|
| 140 |
+
| Run | Overall CER | Overall WER | English CER | Nepali CER | Real CER |
|
| 141 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 142 |
+
| smoke | 11.613341 | 2.637120 | 9.191231 | 23.723889 | 19.593537 |
|
| 143 |
+
| mixed | 1.628157 | 2.916153 | 0.379729 | 7.870295 | 12.702701 |
|
| 144 |
+
| rebalanced | 0.586139 | 0.801844 | 0.300454 | 2.014560 | 1.770016 |
|
| 145 |
+
|
| 146 |
+
The rebalanced run is the first one here that materially improves both English and Nepali on the same held-out split instead of trading one off against the other. It is still not a SOTA claim because:
|
| 147 |
+
|
| 148 |
+
- it is candidate-only
|
| 149 |
+
- it uses one internal eval manifest, not the full internal document pack plus public suites
|
| 150 |
+
- it has no paired validation against named baselines yet
|
| 151 |
+
- it does not measure document-structure leadership, only OCR metrics from the current report path
|
| 152 |
+
|
| 153 |
+
That last caveat is no longer theoretical. The first named baseline check on the same eval split is already negative for the project claim:
|
| 154 |
+
|
| 155 |
+
| Baseline | Overall CER | Overall WER | English CER | Nepali CER | Real CER |
|
| 156 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 157 |
+
| candidate | 0.586139 | 0.801844 | 0.300454 | 2.014560 | 1.770016 |
|
| 158 |
+
| tesseract | 0.329012 | 0.486485 | 0.243379 | 0.757175 | 1.162917 |
|
| 159 |
+
|
| 160 |
+
So the current state is precise:
|
| 161 |
+
|
| 162 |
+
- we fixed our own mixed-script HF candidate substantially
|
| 163 |
+
- we have a reproducible training and benchmark path on `k2`
|
| 164 |
+
- we still do not beat Tesseract on this held-out eval
|
| 165 |
+
|
| 166 |
+
We also tested a mined-correction fallback instead of pretending post-OCR cleanup would obviously help. A ByT5 corrector was trained on `1700` Tesseract noisy/clean pairs mined from the rebalanced training manifest, then packaged as a `text_correction_composite` candidate with raw Tesseract as the base engine. That result was much worse than raw Tesseract:
|
| 167 |
+
|
| 168 |
+
| Baseline | Overall CER | Overall WER | English CER | Nepali CER | Real CER |
|
| 169 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 170 |
+
| Tesseract + ByT5 corrector | 6.912065 | 3.312896 | 6.578699 | 8.578897 | 15.254935 |
|
| 171 |
+
| raw tesseract | 0.329012 | 0.486485 | 0.243379 | 0.757175 | 1.162917 |
|
| 172 |
+
|
| 173 |
+
Paired CER on the same `180` held-out samples was decisive: `0` wins, `170` losses, `10` ties. The corrector trainer metrics were already warning us that this was not a good model (`eval_cer=7.067461`, `eval_wer=3.334917`, `eval_exact_line_accuracy=0.0`), and the benchmark confirmed it.
|
| 174 |
+
|
| 175 |
+
Artifacts recorded locally:
|
| 176 |
+
|
| 177 |
+
- `outputs/live-k2-corrector-vs-tesseract-summary/summary.json`
|
| 178 |
+
- `outputs/live-k2-corrector-vs-tesseract-summary/summary.md`
|
| 179 |
+
|
| 180 |
+
We also tested whether Tesseract itself had enough headroom to matter before touching the recognizer again. Three direct variant checks on the same held-out `180`-sample eval all lost overall to raw default Tesseract (`psm=6`):
|
| 181 |
+
|
| 182 |
+
| Candidate | Overall CER | English CER | Nepali CER | Real CER |
|
| 183 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 184 |
+
| raw tesseract (`psm=6`) | 0.329012 | 0.243379 | 0.757175 | 1.162917 |
|
| 185 |
+
| tesseract (`psm=7`) | 0.353708 | 0.242640 | 0.909049 | 1.104155 |
|
| 186 |
+
| tesseract (`psm=11`) | 0.344319 | 0.258868 | 0.771572 | 1.073521 |
|
| 187 |
+
| tesseract (`psm=13`) | 0.364878 | 0.243850 | 0.970017 | 1.004421 |
|
| 188 |
+
|
| 189 |
+
That said, `psm=13` was not uniformly useless. Using the existing `quality_score` as a routing heuristic, a `quality_select_composite` candidate that keeps default Tesseract unless `psm=13` is better by more than `0.02` did improve the same eval pack slightly:
|
| 190 |
+
|
| 191 |
+
| Baseline | Overall CER | English CER | Nepali CER | Real CER |
|
| 192 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 193 |
+
| raw tesseract (`psm=6`) | 0.329012 | 0.243379 | 0.757175 | 1.162917 |
|
| 194 |
+
| routed tesseract (`psm=6` vs `psm=13`, margin `0.02`) | 0.324000 | 0.242003 | 0.733983 | 1.116534 |
|
| 195 |
+
|
| 196 |
+
Paired CER on that same pack was `5` wins, `3` losses, and `172` ties against raw default Tesseract. This is worth keeping as a baseline candidate backend, but it is not strong enough to confuse with model leadership, and the routing margin was selected on the same held-out pack.
|
| 197 |
+
|
| 198 |
+
We then used the new calibration workflow to stop cheating on that threshold. The original `180`-sample mixed eval was split into a calibration half (`89` samples) and untouched holdout half (`91` samples). `psm=13` was benchmarked against raw default Tesseract on the calibration half, `calibrate-quality-router` selected `secondary_quality_margin=0.03`, and that routed candidate was evaluated on the untouched holdout:
|
| 199 |
+
|
| 200 |
+
| Baseline | Overall CER | English CER | Nepali CER | Real CER |
|
| 201 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 202 |
+
| raw tesseract (`psm=6`) | 0.342234 | 0.236933 | 0.835834 | 1.269163 |
|
| 203 |
+
| routed tesseract (`psm=6` vs `psm=13`, calibrated on separate half) | 0.340608 | 0.235094 | 0.835207 | 1.267909 |
|
| 204 |
+
|
| 205 |
+
Paired CER on the untouched holdout was `2` wins, `1` loss, and `88` ties. That is a real out-of-sample gain, but the size is too small to matter for the main project objective. It tells us the routing idea is not pure leakage, not that we have a strong OCR engine.
|
| 206 |
+
|
| 207 |
+
We also tested a richer calibrated ensemble over `psm=6/7/11/13`. The calibration half liked it and selected biases:
|
| 208 |
+
|
| 209 |
+
- `psm7=-0.12`
|
| 210 |
+
- `psm11=0.02`
|
| 211 |
+
- `psm13=-0.02`
|
| 212 |
+
|
| 213 |
+
But the untouched holdout rejected it:
|
| 214 |
+
|
| 215 |
+
| Baseline | Overall CER | English CER | Nepali CER | Real CER |
|
| 216 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 217 |
+
| raw tesseract (`psm=6`) | 0.342234 | 0.236933 | 0.835834 | 1.269163 |
|
| 218 |
+
| calibrated `psm=6/7/11/13` ensemble | 0.358352 | 0.246311 | 0.883548 | 1.375632 |
|
| 219 |
+
|
| 220 |
+
Surya by itself was not competitive on the same `91`-sample holdout. As a raw baseline it landed at overall CER `2.198362` versus raw Tesseract `0.342234`, mainly because it was disastrous on the English-heavy line-crop slice. But Surya was materially better on the Nepali slice, so we tested a calibrated script-aware route instead of discarding it outright.
|
| 221 |
+
|
| 222 |
+
That routed candidate uses raw Tesseract output as the primary signal: if the Tesseract text looks sufficiently Devanagari-heavy, switch to Surya; otherwise keep Tesseract. The threshold was calibrated on the separate `89`-sample half with `ocrtech calibrate-script-router`, which selected `primary_script_threshold=0.4`. On the untouched `91`-sample holdout:
|
| 223 |
+
|
| 224 |
+
| Baseline | Overall CER | English CER | Nepali CER | Real CER |
|
| 225 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 226 |
+
| raw tesseract (`psm=6`) | 0.342234 | 0.236933 | 0.835834 | 1.269163 |
|
| 227 |
+
| calibrated script router (`tesseract` -> `surya`) | 0.313084 | 0.230513 | 0.700133 | 1.172699 |
|
| 228 |
+
|
| 229 |
+
Paired CER on that holdout was `9` wins, `0` losses, and `82` ties against raw Tesseract. This is the strongest out-of-sample baseline improvement in the repo so far. It is still not claim-grade evidence, because it is one internal line-level split rather than the full internal document pack plus public suites, but it is a real improvement and a better direction than the TrOCR branch.
|
| 230 |
+
|
| 231 |
+
We also ran an exploratory `routing_granularity=page` variant of the same Tesseract-to-Surya router on that holdout, reusing the calibrated threshold `0.4`. It produced the same summary numbers as the document-level router. That is not a hidden win or loss; it just means this holdout is mostly single-page line-crop material, so page routing has no additional room to differentiate behavior yet.
|
| 232 |
+
|
| 233 |
+
After that, the internal eval pack was upgraded so image-mode tables are rendered as actual grids and the pack now includes a real `multi_page` slice as bundled page images. On that rendered `5`-sample smoke pack, the plain script router reached `table_cell_f1 = 1.0` on the dedicated table page but still missed the table inside the multi-page report when it kept Tesseract text for the document. A follow-up exploratory candidate with `secondary_structure_backfill=true` fixed that exact gap: it kept the same CER while backfilling Surya table structure on the multi-page report, moving aggregate `table_cell_f1` from `0.5` to `1.0` on the rendered smoke pack. This is still smoke evidence, not claim evidence, because the pack is tiny and only covers the `clean` slice, but it is the first concrete sign that composition can improve document structure without paying a text penalty.
|
| 234 |
+
|
| 235 |
+
The next run made that path less hand-wavy. A broader rendered image pack was generated with `25` samples across `clean`, `scan`, `blur`, `low_contrast`, and `skew`, and `audit-eval-pack --require-claim-ready` passed on it. The backfill router was then calibrated through `ocrtech calibrate-script-router` with `secondary_structure_backfill=true`, producing `ocrtech-tesseract-surya-script-router-backfill-calibrated`, and `preflight-claim` passed on the final experiment pack.
|
| 236 |
+
|
| 237 |
+
Pulled artifacts for that run:
|
| 238 |
+
|
| 239 |
+
- `outputs/live-k2-claimpack-rendered-backfill/eval-pack-audit.json`
|
| 240 |
+
- `outputs/live-k2-claimpack-rendered-backfill/eval-pack-audit.md`
|
| 241 |
+
|
| 242 |
+
The next useful tightening step was not “run everything on all `100` rendered samples and hope.” That is too expensive for the current GLM and PaddleOCR-VL paths on `k2`. Instead, the repo now has `ocrtech filter-manifest`, which was used to create a reproducible `5`-sample blur-only subset covering:
|
| 243 |
+
|
| 244 |
+
- paragraph
|
| 245 |
+
- table
|
| 246 |
+
- form
|
| 247 |
+
- figure
|
| 248 |
+
- multi-page report
|
| 249 |
+
|
| 250 |
+
That subset was benchmarked on `k2` across the real baseline set:
|
| 251 |
+
|
| 252 |
+
- `candidate` using `outputs/script-router-backfill-calibration/model-card.json`
|
| 253 |
+
- `tesseract`
|
| 254 |
+
- `surya`
|
| 255 |
+
- `stock-paddle`
|
| 256 |
+
- `glm-ocr` via the isolated `.venv-glm` hook
|
| 257 |
+
- `paddleocr-vl`
|
| 258 |
+
|
| 259 |
+
All `30` runs completed successfully. The pulled local artifacts are:
|
| 260 |
+
|
| 261 |
+
- `outputs/live-k2-rendered-blur5/report.json`
|
| 262 |
+
- `outputs/live-k2-rendered-blur5/summary.json`
|
| 263 |
+
|
| 264 |
+
Aggregate OCR/text result on that subset:
|
| 265 |
+
|
| 266 |
+
- candidate CER: `0.223273`
|
| 267 |
+
- Surya CER: `0.197601`
|
| 268 |
+
- Tesseract CER: `0.732364`
|
| 269 |
+
- stock-Paddle CER: `2.810426`
|
| 270 |
+
- GLM-OCR CER: `2.820269`
|
| 271 |
+
- PaddleOCR-VL CER: `18.470663`
|
| 272 |
+
|
| 273 |
+
Aggregate structure result on that subset:
|
| 274 |
+
|
| 275 |
+
- candidate `table_cell_f1`: `1.0`
|
| 276 |
+
- Surya `table_cell_f1`: `1.0`
|
| 277 |
+
- Tesseract / stock-Paddle / GLM-OCR / PaddleOCR-VL `table_cell_f1`: `0.0`
|
| 278 |
+
|
| 279 |
+
Aggregate reading-order result on that subset:
|
| 280 |
+
|
| 281 |
+
- candidate: `0.783636`
|
| 282 |
+
- Tesseract: `0.823636`
|
| 283 |
+
- Surya: `0.750909`
|
| 284 |
+
- stock-Paddle: `1.0`
|
| 285 |
+
- GLM-OCR: `1.0`
|
| 286 |
+
- PaddleOCR-VL: `0.750909`
|
| 287 |
+
|
| 288 |
+
That subset is exploratory, not claim-grade. But it is useful because it removes more guesswork:
|
| 289 |
+
|
| 290 |
+
- the current candidate is much better than Tesseract, stock-Paddle, GLM-OCR, and PaddleOCR-VL on blur-heavy rendered OCR CER
|
| 291 |
+
- the current candidate still does not beat raw Surya on text CER
|
| 292 |
+
- the current candidate still trades away some reading-order quality relative to Tesseract and the layout-heavy baselines
|
| 293 |
+
|
| 294 |
+
For paper use, the right interpretation is narrower than "our model wins." This subset supports a more defensible statement: the current backfill candidate is a strong internal engineering baseline on degraded rendered documents, especially for table recovery, but raw Surya is still the stronger text OCR baseline on the same subset.
|
| 295 |
+
|
| 296 |
+
Additional pulled artifacts for that `25`-sample backfill run:
|
| 297 |
+
|
| 298 |
+
- `outputs/live-k2-claimpack-rendered-backfill/calibration.json`
|
| 299 |
+
- `outputs/live-k2-claimpack-rendered-backfill/model-card.json`
|
| 300 |
+
- `outputs/live-k2-claimpack-rendered-backfill/experiment.json`
|
| 301 |
+
- `outputs/live-k2-claimpack-rendered-backfill/report.json`
|
| 302 |
+
- `outputs/live-k2-claimpack-rendered-backfill/summary.json`
|
| 303 |
+
- `outputs/live-k2-claimpack-rendered-backfill/summary.md`
|
| 304 |
+
|
| 305 |
+
Key numbers versus raw Tesseract on that rendered claim-ready pack:
|
| 306 |
+
|
| 307 |
+
| Slice | Candidate CER | Tesseract CER | Notes |
|
| 308 |
+
| --- | ---: | ---: | --- |
|
| 309 |
+
| all | 0.248654 | 0.689214 | paired CER `9` wins, `1` loss, `15` ties |
|
| 310 |
+
| clean | 0.243224 | 0.540584 | same win pattern as smoke pack |
|
| 311 |
+
| blur | 0.260039 | 0.991613 | strongest degraded-slice text gain |
|
| 312 |
+
| scan | 0.237424 | 0.593631 | clear text win |
|
| 313 |
+
| low_contrast | 0.263265 | 0.554174 | smaller but still positive |
|
| 314 |
+
| skew | 0.239317 | 0.766067 | one loss, two wins, two ties |
|
| 315 |
+
| multi_page | 0.315789 | 0.315789 | CER tie |
|
| 316 |
+
| table | 0.339713 | 1.421531 | large text gain on table-heavy pages |
|
| 317 |
+
|
| 318 |
+
Structure metrics on the same pack matter more than the CER win:
|
| 319 |
+
|
| 320 |
+
- candidate aggregate `table_cell_f1`: `0.955556`
|
| 321 |
+
- Tesseract aggregate `table_cell_f1`: `0.0`
|
| 322 |
+
- candidate aggregate `table_exact_match`: `0.8`
|
| 323 |
+
- Tesseract aggregate `table_exact_match`: `0.0`
|
| 324 |
+
- candidate `multi_page` `table_cell_f1`: `0.911111`
|
| 325 |
+
- Tesseract `multi_page` `table_cell_f1`: `0.0`
|
| 326 |
+
|
| 327 |
+
This is better evidence than the tiny smoke pack, but it still is not a clean claim story. Two real weaknesses remain on this rendered pack:
|
| 328 |
+
|
| 329 |
+
- `form` CER regressed slightly: `0.231933` vs `0.230010`
|
| 330 |
+
- aggregate `reading_order_pair_accuracy` is lower than Tesseract: `0.764` vs `0.816`
|
| 331 |
+
|
| 332 |
+
So the honest reading is narrow: the calibrated Tesseract-plus-Surya backfill router is a strong internal rendered-pack baseline for OCR robustness and table extraction, but it is not yet a broad document-understanding winner.
|
| 333 |
+
|
| 334 |
+
We then ran two more comparisons on that same rendered claim-ready pack to stop overfitting the story to Tesseract.
|
| 335 |
+
|
| 336 |
+
First, raw Surya on the rendered pack was not the disaster it had been on the older line-crop holdout. On the `25`-sample rendered document pack it beat the calibrated backfill router on text CER across every checked slice:
|
| 337 |
+
|
| 338 |
+
| Slice | Candidate CER | Raw Surya CER |
|
| 339 |
+
| --- | ---: | ---: |
|
| 340 |
+
| all | 0.248654 | 0.195260 |
|
| 341 |
+
| blur | 0.260039 | 0.202626 |
|
| 342 |
+
| scan | 0.237424 | 0.191862 |
|
| 343 |
+
| low_contrast | 0.263265 | 0.191862 |
|
| 344 |
+
| skew | 0.239317 | 0.194733 |
|
| 345 |
+
| form | 0.231933 | 0.138462 |
|
| 346 |
+
| multi_page | 0.315789 | 0.200000 |
|
| 347 |
+
| table | 0.339713 | 0.281818 |
|
| 348 |
+
|
| 349 |
+
Paired CER was not close: `0` wins, `15` losses, `10` ties for the candidate on `all`.
|
| 350 |
+
|
| 351 |
+
That does not mean the router is useless. Structure on the same pack is more mixed:
|
| 352 |
+
|
| 353 |
+
- aggregate `table_cell_f1`: candidate `0.955556`, Surya `0.955556`
|
| 354 |
+
- aggregate `reading_order_pair_accuracy`: candidate `0.764000`, Surya `0.750909`
|
| 355 |
+
- paired reading-order wins on `all`: candidate `2`, losses `0`, ties `23`
|
| 356 |
+
|
| 357 |
+
So the rendered document-pack conclusion is stricter than the earlier line-crop conclusion: the current routed candidate is still behind raw Surya on text OCR, while only slightly ahead on reading-order behavior and roughly tied on table structure.
|
| 358 |
+
|
| 359 |
+
Second, `stock-paddle` stopped being a fake baseline. The original `k2` benchmark result showing `1.0` CER everywhere was an adapter bug in this repo, not a real PaddleOCR result: `paddleocr 3.7.0` on `k2` returns v3 `OCRResult` objects, and the old parser only understood the older tuple/list shape. After fixing that adapter and installing `paddlepaddle 3.3.1`, the stock-Paddle baseline became runnable on `k2`.
|
| 360 |
+
|
| 361 |
+
With that fix in place, the same rendered claim-ready pack produced:
|
| 362 |
+
|
| 363 |
+
| Slice | Candidate CER | Stock-Paddle CER |
|
| 364 |
+
| --- | ---: | ---: |
|
| 365 |
+
| all | 0.248654 | 1.749081 |
|
| 366 |
+
| blur | 0.260039 | 1.549743 |
|
| 367 |
+
| scan | 0.237424 | 1.896558 |
|
| 368 |
+
| low_contrast | 0.263265 | 1.854006 |
|
| 369 |
+
| skew | 0.239317 | 1.568643 |
|
| 370 |
+
| form | 0.231933 | 0.470694 |
|
| 371 |
+
| multi_page | 0.315789 | 0.391388 |
|
| 372 |
+
| table | 0.339713 | 3.395694 |
|
| 373 |
+
|
| 374 |
+
Paired CER on `all` was `21` wins, `4` losses, `0` ties for the candidate.
|
| 375 |
+
|
| 376 |
+
Structure was lopsided in the other direction:
|
| 377 |
+
|
| 378 |
+
- candidate aggregate `table_cell_f1`: `0.955556`
|
| 379 |
+
- stock-Paddle aggregate `table_cell_f1`: `0.0`
|
| 380 |
+
- candidate aggregate `reading_order_pair_accuracy`: `0.764000`
|
| 381 |
+
- stock-Paddle aggregate `reading_order_pair_accuracy`: `1.0`
|
| 382 |
+
|
| 383 |
+
So stock-Paddle, as currently configured here with `PaddleOCR(lang='ne')`, preserves reading order very aggressively but is not competitive on text accuracy or table extraction for this mixed rendered pack.
|
| 384 |
+
|
| 385 |
+
The stricter rendered-document result is now also explicit. On `k2`, a larger `claimpack-rendered-v2` split was calibrated separately and evaluated on an untouched `50`-sample holdout. The routed script candidate selected `primary_script_threshold=0.0` on calibration, which means it collapsed to "always pick Surya" there. On the holdout:
|
| 386 |
+
|
| 387 |
+
| Baseline | Overall CER | table_cell_f1 | reading_order_pair_accuracy |
|
| 388 |
+
| --- | ---: | ---: | ---: |
|
| 389 |
+
| candidate | 0.180893 | 0.947500 | 0.750909 |
|
| 390 |
+
| raw Surya | 0.180893 | 0.970833 | 0.750909 |
|
| 391 |
+
| raw Tesseract | 0.714207 | 0.000000 | 0.787818 |
|
| 392 |
+
|
| 393 |
+
Pulled local artifacts:
|
| 394 |
+
|
| 395 |
+
- `outputs/live-k2-claimpack-v2-script-router/calibration.json`
|
| 396 |
+
- `outputs/live-k2-claimpack-v2-script-router/model-card.json`
|
| 397 |
+
- `outputs/live-k2-claimpack-v2-script-router/experiment.json`
|
| 398 |
+
- `outputs/live-k2-claimpack-v2-script-router/summary.json`
|
| 399 |
+
- `outputs/live-k2-claimpack-v2-script-router/preflight.json`
|
| 400 |
+
- `outputs/paper-claimpack-v2-table/paper-table.md`
|
| 401 |
+
|
| 402 |
+
This is not a routed-model win. It is a branch ceiling. The candidate does not beat Surya on text, ties it exactly on CER, and is slightly worse on table structure. The new claim guardrail now rejects this case directly: `preflight-claim` fails `candidate-calibration` when a routed candidate collapses to one engine on every calibration sample.
|
| 403 |
+
|
| 404 |
+
We also tested the obvious opposite branch after adding Latin-script routing support to the repo: a Surya-primary page router with Tesseract fallback for English-heavy pages. The rendered-v2 calibration on `k2` selected `primary_script_threshold=0.7`, and the holdout result was unambiguously bad:
|
| 405 |
+
|
| 406 |
+
| Baseline | Overall CER | table_cell_f1 | reading_order_pair_accuracy |
|
| 407 |
+
| --- | ---: | ---: | ---: |
|
| 408 |
+
| candidate | 0.416854 | 0.337500 | 0.761818 |
|
| 409 |
+
| raw Surya | 0.180893 | 0.970833 | 0.750909 |
|
| 410 |
+
| raw Tesseract | 0.714207 | 0.000000 | 0.787818 |
|
| 411 |
+
|
| 412 |
+
Pulled local artifacts:
|
| 413 |
+
|
| 414 |
+
- `outputs/live-k2-claimpack-v2-surya-latin-router/calibration.json`
|
| 415 |
+
- `outputs/live-k2-claimpack-v2-surya-latin-router/experiment.json`
|
| 416 |
+
- `outputs/live-k2-claimpack-v2-surya-latin-router/report.json`
|
| 417 |
+
- `outputs/live-k2-claimpack-v2-surya-latin-router/summary.json`
|
| 418 |
+
- `outputs/paper-claimpack-v2-surya-latin-table/paper-table.md`
|
| 419 |
+
|
| 420 |
+
Paired CER versus raw Surya on that holdout was `0` wins, `12` losses, `38` ties. So the inverted script router is not a refinement. It is a disproved branch.
|
| 421 |
+
|
| 422 |
+
We then tested the narrower quality-based version of the same idea: keep Tesseract and Surya as components, but let the router pick based on the local `quality_score` signal instead of script ratio. The rendered-v2 calibration selected `selected_threshold=0.0`, which already tells you the calibration split did not find a real decision boundary beyond "prefer Surya whenever it is not clearly worse." On the untouched holdout:
|
| 423 |
+
|
| 424 |
+
| Baseline | Overall CER | table_cell_f1 | reading_order_pair_accuracy |
|
| 425 |
+
| --- | ---: | ---: | ---: |
|
| 426 |
+
| candidate | 0.181128 | 0.947500 | 0.750909 |
|
| 427 |
+
| raw Surya | 0.180893 | 0.947500 | 0.750909 |
|
| 428 |
+
| raw Tesseract | 0.714207 | 0.000000 | 0.787818 |
|
| 429 |
+
|
| 430 |
+
Pulled local artifacts:
|
| 431 |
+
|
| 432 |
+
- `outputs/live-k2-claimpack-v2-surya-quality-router/calibration.json`
|
| 433 |
+
- `outputs/live-k2-claimpack-v2-surya-quality-router/experiment.json`
|
| 434 |
+
- `outputs/live-k2-claimpack-v2-surya-quality-router/report.json`
|
| 435 |
+
- `outputs/live-k2-claimpack-v2-surya-quality-router/summary.json`
|
| 436 |
+
- `outputs/paper-claimpack-v2-surya-quality-router-table/paper-table.md`
|
| 437 |
+
|
| 438 |
+
Paired CER versus raw Surya on that holdout was `0` wins, `1` loss, `49` ties. So this is not the missing refinement either. It is nearly indistinguishable from Surya and still not better.
|
| 439 |
+
|
| 440 |
+
We also tested a non-routing composition on the same rendered-v2 holdout: `line_align_composite`. The idea was simple enough to be worth disproving. Keep Surya as the text engine, reuse Tesseract line scaffolding, and see whether better ordering helps enough to justify the merge. It did not:
|
| 441 |
+
|
| 442 |
+
| Baseline | Overall CER | table_cell_f1 | reading_order_pair_accuracy |
|
| 443 |
+
| --- | ---: | ---: | ---: |
|
| 444 |
+
| candidate | 0.310549 | 0.897500 | 0.792727 |
|
| 445 |
+
| raw Surya | 0.180893 | 0.970833 | 0.750909 |
|
| 446 |
+
| raw Tesseract | 0.714207 | 0.000000 | 0.787818 |
|
| 447 |
+
|
| 448 |
+
Pulled local artifacts:
|
| 449 |
+
|
| 450 |
+
- `outputs/live-k2-claimpack-v2-line-align-summary/summary.json`
|
| 451 |
+
- `outputs/paper-claimpack-v2-line-align-table/paper-table.md`
|
| 452 |
+
- `outputs/live-k2-claimpack-v2-line-align-model-card.json`
|
| 453 |
+
|
| 454 |
+
Paired CER versus raw Surya on that holdout was `0` wins, `7` losses, `43` ties. The candidate did improve reading-order accuracy relative to raw Surya, and even edged past Tesseract there, but the text and table regressions are too large to defend. So line alignment is another useful negative result, not a candidate answer.
|
| 455 |
+
|
| 456 |
+
Finally, we measured the sample-level upper bound for the same component family using `benchmark-oracle`. This is not a deployable candidate. It chooses the better of raw Surya and raw Tesseract per sample using ground-truth CER. That is the right diagnostic question before wasting more time on routing heuristics:
|
| 457 |
+
|
| 458 |
+
| System | Overall CER | table_cell_f1 | reading_order_pair_accuracy |
|
| 459 |
+
| --- | ---: | ---: | ---: |
|
| 460 |
+
| Surya/Tesseract oracle | 0.180529 | 0.970833 | 0.750909 |
|
| 461 |
+
| raw Surya | 0.180893 | 0.970833 | 0.750909 |
|
| 462 |
+
| raw Tesseract | 0.714207 | 0.000000 | 0.787818 |
|
| 463 |
+
|
| 464 |
+
Pulled/generated local artifacts:
|
| 465 |
+
|
| 466 |
+
- `outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-analysis.json`
|
| 467 |
+
- `outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-choices.json`
|
| 468 |
+
- `outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-summary/summary.json`
|
| 469 |
+
- `outputs/paper-claimpack-v2-oracle-table/paper-table.md`
|
| 470 |
+
|
| 471 |
+
The oracle picked raw Surya for `49` samples and raw Tesseract for `1`. Those per-sample source choices are stored in `oracle-choices.json` as provenance, not mixed into benchmark metrics. Paired CER versus raw Surya was `1` win, `0` losses, `49` ties. That means the current Surya/Tesseract routing space is effectively exhausted for text OCR on rendered-v2. The next useful model-side work needs a stronger recognizer or document model, not another selector over these two engines.
|
| 472 |
+
|
| 473 |
+
We then added `benchmark-error-analysis` to convert that failure into a training target instead of another routing guess. On the same rendered-v2 benchmark report, raw Surya's highest absolute CER slices are `figure` (`0.253157`) and `table` (`0.252781`). The hardest individual samples are dominated by table-ledger and figure-caption pages. The Surya-vs-oracle diagnostic confirms that Tesseract only improves one sample, `eval-form-notice-003-blur`, by `0.018182` CER. So the next serious model branch should target table/figure-heavy mixed-script recognition and document context, while treating Surya/Tesseract routing as exhausted.
|
| 474 |
+
|
| 475 |
+
We also exported those ranked failures back into a real manifest instead of a hand-maintained list. `data/splits-claimpack-rendered-v2/eval-surya-hard15.jsonl` is a `15`-sample `hard_eval` manifest built from the top Surya CER failures. It keeps the original reference metadata and adds `metadata.failure_analysis` with rank, metric, value, and comparison details. The local checkout does not include the rendered image assets, so the local audit reports missing image paths; the same manifest passes full file/reference audit on `k2` at `outputs/audit-claimpack-v2-surya-hard15/manifest-audit.json`, pulled locally to `outputs/audit-claimpack-v2-surya-hard15-k2/manifest-audit.json`.
|
| 476 |
+
|
| 477 |
+
The hard pocket now has an explicit validation config: `configs/validation-hard15-surya.json`. This uses conjunctive gate slices such as `["hard_eval", "table"]` and `["hard_eval", "figure"]`, so the table and figure gates cannot be satisfied by easier non-hard samples. As a negative control, the current line-align candidate fails this config on `outputs/live-k2-claimpack-v2-line-align/report.json`: hard-eval CER is `0.482039` versus Surya `0.348706`, hard-table CER is `0.613636` versus Surya `0.363636`, and the figure/table structure gates do not improve over Surya.
|
| 478 |
+
|
| 479 |
+
For model iteration, we generated a separate table/figure development pack on `k2`:
|
| 480 |
+
|
| 481 |
+
- manifest: `data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl`
|
| 482 |
+
- audit: `outputs/audit-eval-pack-table-figure-dev-v1-k2/eval-pack-audit.json`
|
| 483 |
+
- samples: `80`
|
| 484 |
+
- slices: `40` table, `40` figure, `80` Nepali/English/mixed-script/image/reading_order
|
| 485 |
+
- degradations: `clean`, `scan`, `blur`, `low_contrast`, `skew`
|
| 486 |
+
- hard15 overlap: `0` sample IDs and `0` text hashes
|
| 487 |
+
|
| 488 |
+
This pack was generated with `--template table-ledger --template figure-caption --variant-offset 100`, after updating the eval-pack generator so table OCR references include visible table rows and figure variants include copy-specific visible text. Treat this as a development/training target, not as held-out claim evidence.
|
| 489 |
+
|
| 490 |
+
The pack was then split with text-level grouping so degraded variants of the same target do not leak across train/eval:
|
| 491 |
+
|
| 492 |
+
```bash
|
| 493 |
+
ocrtech split-manifest data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl \
|
| 494 |
+
--out data/splits-table-figure-dev-v1 \
|
| 495 |
+
--eval-ratio 0.2 \
|
| 496 |
+
--seed 17 \
|
| 497 |
+
--stratify-by document_type \
|
| 498 |
+
--group-by text_sha256
|
| 499 |
+
```
|
| 500 |
+
|
| 501 |
+
Verified split:
|
| 502 |
+
|
| 503 |
+
- train: `60` samples
|
| 504 |
+
- eval: `20` samples
|
| 505 |
+
- text groups: `16`
|
| 506 |
+
- leakage check: `pass`
|
| 507 |
+
- train audit on `k2`: `pass`
|
| 508 |
+
- eval audit on `k2`: `pass`
|
| 509 |
+
|
| 510 |
+
Additional leakage checks on `k2` found no sample-id or text-hash overlap with `hard15` or rendered-v2 eval. That makes this a legitimate development pack for the next model branch, but it is still not claim evidence.
|
| 511 |
+
|
| 512 |
+
The first model-side attempt on this branch was a minimal Hugging Face VisionEncoderDecoder fine-tune using `microsoft/trocr-small-printed`:
|
| 513 |
+
|
| 514 |
+
```bash
|
| 515 |
+
ocrtech train-recognizer \
|
| 516 |
+
--backend hf-vision-encoder-decoder \
|
| 517 |
+
--base-model microsoft/trocr-small-printed \
|
| 518 |
+
--train-manifest data/splits-table-figure-dev-v1/train.jsonl \
|
| 519 |
+
--eval-manifest data/splits-table-figure-dev-v1/eval.jsonl \
|
| 520 |
+
--out runs/hf-table-figure-dev-v1 \
|
| 521 |
+
--epochs 1 \
|
| 522 |
+
--train-batch-size 4 \
|
| 523 |
+
--eval-batch-size 4 \
|
| 524 |
+
--learning-rate 5e-5 \
|
| 525 |
+
--run
|
| 526 |
+
```
|
| 527 |
+
|
| 528 |
+
Training completed and the package audited cleanly, but the result was a decisive negative:
|
| 529 |
+
|
| 530 |
+
| metric | value |
|
| 531 |
+
| --- | ---: |
|
| 532 |
+
| eval_loss | 7.884517 |
|
| 533 |
+
| eval_cer | 3.062119 |
|
| 534 |
+
| eval_wer | 3.562689 |
|
| 535 |
+
| eval_exact_line_accuracy | 0.000000 |
|
| 536 |
+
|
| 537 |
+
Benchmarking the packaged candidate against Surya and Tesseract on the `20`-sample dev eval confirmed the branch is not viable:
|
| 538 |
+
|
| 539 |
+
| system | CER | table_cell_f1 | reading_order_pair_accuracy |
|
| 540 |
+
| --- | ---: | ---: | ---: |
|
| 541 |
+
| HF table/figure candidate | 2.422001 | 0.000000 | 0.000000 |
|
| 542 |
+
| Surya | 0.563399 | 0.858333 | 0.550000 |
|
| 543 |
+
| Tesseract | 0.546227 | 0.000000 | 0.695000 |
|
| 544 |
+
|
| 545 |
+
Paired CER was `0` wins, `20` losses, and `0` ties against both Surya and Tesseract. The failure mode is not subtle: naive full-page TrOCR fine-tuning does not learn table/figure document recognition from this small pack. The next credible recognizer branch should train on line/cell crops or use a page/document model with structure-aware supervision.
|
| 546 |
+
|
| 547 |
+
We then implemented that crop-level path and tested it instead of leaving it as advice. Newly generated image-pack references now carry renderer layout metadata for non-skew pages, and `ocrtech extract-crops` turns those verified bboxes into ordinary recognizer manifests. Skew pages are skipped because pre-rotation bboxes are invalid.
|
| 548 |
+
|
| 549 |
+
Full crop development pack on `k2`:
|
| 550 |
+
|
| 551 |
+
- source pack: `data/eval-pack-table-figure-crops-dev-v1/manifests/ocrtech-eval-pack.jsonl`
|
| 552 |
+
- crop manifest: `data/crops-table-figure-dev-v1/manifests/crops.jsonl`
|
| 553 |
+
- source samples: `80`
|
| 554 |
+
- crops: `544`
|
| 555 |
+
- crop types: `384` table cells, `128` lines, `32` figure captions
|
| 556 |
+
- skipped candidates: `32` skew pages
|
| 557 |
+
- crop audit: `outputs/audit-crops-table-figure-dev-v1/manifest-audit.md`
|
| 558 |
+
- split: `480` train / `64` eval
|
| 559 |
+
- split grouping: `text_sha256`
|
| 560 |
+
- split leakage: pass
|
| 561 |
+
|
| 562 |
+
The first crop-level HF model used the same `microsoft/trocr-small-printed` backend for one epoch, this time on `480` crop images instead of full pages. Trainer metrics improved relative to the full-page branch, but still were not competitive:
|
| 563 |
+
|
| 564 |
+
| metric | value |
|
| 565 |
+
| --- | ---: |
|
| 566 |
+
| eval_loss | 3.087561 |
|
| 567 |
+
| eval_cer | 0.959831 |
|
| 568 |
+
| eval_wer | 1.363715 |
|
| 569 |
+
| eval_exact_line_accuracy | 0.078125 |
|
| 570 |
+
|
| 571 |
+
Packaged crop model audit passed:
|
| 572 |
+
|
| 573 |
+
- `runs/hf-crops-table-figure-dev-v1/training_metrics.json`
|
| 574 |
+
- `runs/hf-crops-table-figure-dev-v1/package/model-card.json`
|
| 575 |
+
- `outputs/audit-hf-crops-table-figure-dev-v1-model/model-audit.md`
|
| 576 |
+
|
| 577 |
+
Benchmark result on the `64`-sample crop eval:
|
| 578 |
+
|
| 579 |
+
| system | CER |
|
| 580 |
+
| --- | ---: |
|
| 581 |
+
| HF crop candidate | 1.043369 |
|
| 582 |
+
| Surya | 0.475309 |
|
| 583 |
+
| Tesseract | 0.077453 |
|
| 584 |
+
|
| 585 |
+
Paired CER:
|
| 586 |
+
|
| 587 |
+
- versus Surya: `7` wins, `50` losses, `7` ties
|
| 588 |
+
- versus Tesseract: `0` wins, `64` losses, `0` ties
|
| 589 |
+
|
| 590 |
+
This means crop-level training is the right data shape, but this tiny HF recognizer is still the wrong recognizer. The immediate technical takeaway is that Tesseract is extremely strong on these generated crop images; the next branch should use strong crop OCR behind better layout/cell/caption detection, or train a more suitable recognizer with substantially more real crop data.
|
| 591 |
+
|
| 592 |
+
The next composition test used that crop-OCR signal more directly: `table_cell_refine_composite` keeps Surya as the document/table structure engine and rereads detected table cells with Tesseract crops.
|
| 593 |
+
|
| 594 |
+
Two hard15 variants were tested:
|
| 595 |
+
|
| 596 |
+
| variant | hard_eval CER | table_cell_f1 | reading_order_pair_accuracy | verdict |
|
| 597 |
+
| --- | ---: | ---: | ---: | --- |
|
| 598 |
+
| Surya baseline | 0.348706 | 0.979167 | 0.520000 | current hard15 target |
|
| 599 |
+
| `replacement_policy=always` | 0.348706 | 0.593750 | 0.520000 | unsafe; damages good Surya cells |
|
| 600 |
+
| `replacement_policy=empty` | 0.348706 | 0.979167 | 0.520000 | safe tie; no improvement |
|
| 601 |
+
| `replacement_policy=empty`, table de-dup | 0.348706 | 0.979167 | 0.520000 | parser hygiene fix; still no improvement |
|
| 602 |
+
|
| 603 |
+
The safe policy also failed the hard15 validation gate with status `not_validated`. This is another useful negative result: Tesseract crop OCR is strong when the crop is exact, but naively applying it to Surya-inferred table cells does not improve Surya on this hard pocket.
|
| 604 |
+
|
| 605 |
+
Cell-level analysis explains why:
|
| 606 |
+
|
| 607 |
+
| system | reference_cells | exact_cells | exact_rate | mean_cer |
|
| 608 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 609 |
+
| Surya | 96 | 94 | 0.979167 | 0.010417 |
|
| 610 |
+
| table-cell refine, `always` | 96 | 57 | 0.593750 | 0.284309 |
|
| 611 |
+
| table-cell refine, `empty` | 96 | 94 | 0.979167 | 0.010417 |
|
| 612 |
+
|
| 613 |
+
Against Surya, the unsafe `always` policy improved `0` cells, worsened `37`, and tied `59`. The safe `empty` policy tied all `96` cells because Surya had no missing cells to fill in this hard pocket. The remaining Surya cell errors are only two occurrences of `किरण` predicted as `करिण`, both on the same table-ledger variant. The next table branch needs a targeted confidence/replacement signal for rare wrong-but-nonempty cells; blind crop rerecognition is the wrong direction.
|
| 614 |
+
|
| 615 |
+
The table de-duplication branch fixed a separate parser hygiene issue: line-based table inference no longer creates a second table when an engine already supplied an overlapping structured table. On the rerun artifact, both candidate and Surya now report `8` predicted tables for `8` reference tables and `0` duplicate overlapping tables. This is useful for cleaner exports, but because the fix applies at parser assembly time it also benefits the Surya baseline; hard15 validation remains `not_validated`.
|
| 616 |
+
|
| 617 |
+
Generated diagnostics:
|
| 618 |
+
|
| 619 |
+
- `outputs/paper-claimpack-v2-surya-error-analysis/error-analysis.md`
|
| 620 |
+
- `outputs/paper-claimpack-v2-surya-error-analysis/top-failures.json`
|
| 621 |
+
- `outputs/paper-claimpack-v2-surya-vs-tesseract-error-analysis/error-analysis.md`
|
| 622 |
+
- `outputs/paper-claimpack-v2-surya-vs-oracle-error-analysis/error-analysis.md`
|
| 623 |
+
- `data/splits-claimpack-rendered-v2/eval-surya-hard15.jsonl`
|
| 624 |
+
- `configs/validation-hard15-surya.json`
|
| 625 |
+
- `outputs/validation-claimpack-v2-line-align-hard15/validation.json`
|
| 626 |
+
- `outputs/audit-claimpack-v2-surya-hard15-k2/manifest-audit.json`
|
| 627 |
+
- `data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl`
|
| 628 |
+
- `outputs/audit-eval-pack-table-figure-dev-v1-k2/eval-pack-audit.json`
|
| 629 |
+
- `data/splits-table-figure-dev-v1/split-summary.md`
|
| 630 |
+
- `outputs/audit-table-figure-dev-v1-train-k2/manifest-audit.md`
|
| 631 |
+
- `outputs/audit-table-figure-dev-v1-eval-k2/manifest-audit.md`
|
| 632 |
+
- `runs/hf-table-figure-dev-v1/training_metrics.json`
|
| 633 |
+
- `runs/hf-table-figure-dev-v1/package/model-card.json`
|
| 634 |
+
- `outputs/audit-hf-table-figure-dev-v1-model/model-audit.md`
|
| 635 |
+
- `outputs/summary-hf-table-figure-dev-v1/summary.md`
|
| 636 |
+
- `outputs/experiment-hf-table-figure-dev-v1/benchmark/report.json`
|
| 637 |
+
- `outputs/crops-table-figure-dev-v1-k2/crop-manifest-summary.md`
|
| 638 |
+
- `outputs/crops-table-figure-dev-v1-k2/manifest-audit.md`
|
| 639 |
+
- `data/splits-crops-table-figure-dev-v1/split-summary.md`
|
| 640 |
+
- `runs/hf-crops-table-figure-dev-v1/training_metrics.json`
|
| 641 |
+
- `runs/hf-crops-table-figure-dev-v1/package/model-card.json`
|
| 642 |
+
- `outputs/audit-hf-crops-table-figure-dev-v1-model/model-audit.md`
|
| 643 |
+
- `outputs/summary-hf-crops-table-figure-dev-v1/summary.md`
|
| 644 |
+
- `outputs/experiment-hf-crops-table-figure-dev-v1/benchmark/report.json`
|
| 645 |
+
- `outputs/table-cell-refine-v1/model-card.json`
|
| 646 |
+
- `outputs/summary-table-cell-refine-hard15-v1/summary.md`
|
| 647 |
+
- `outputs/table-cell-refine-empty-v1/model-card.json`
|
| 648 |
+
- `outputs/summary-table-cell-refine-empty-hard15-v1/summary.md`
|
| 649 |
+
- `outputs/validation-table-cell-refine-empty-hard15-v1/validation.md`
|
| 650 |
+
- `outputs/table-cell-analysis-refine-always-hard15-v1/table-cell-analysis.md`
|
| 651 |
+
- `outputs/table-cell-analysis-refine-empty-hard15-v1/table-cell-analysis.md`
|
| 652 |
+
- `outputs/table-cell-refine-empty-dedup-v1/model-card.json`
|
| 653 |
+
- `outputs/summary-table-cell-refine-empty-dedup-hard15-v3/summary.md`
|
| 654 |
+
- `outputs/table-cell-analysis-refine-empty-dedup-hard15-v3/table-cell-analysis.md`
|
| 655 |
+
- `outputs/validation-table-cell-refine-empty-dedup-hard15-v3/validation.md`
|
| 656 |
+
|
| 657 |
+
When repeating this workflow, keep the comparison honest:
|
| 658 |
+
|
| 659 |
+
```bash
|
| 660 |
+
ocrtech prepare-data gauravgiri/nepali-ocr-dataset --hf --split train --limit 100 --skip-invalid --out data/manifests
|
| 661 |
+
ocrtech prepare-data prashant0919/nepali-synthetic-ocr-lines --hf --split train --limit 100 --out data/manifests
|
| 662 |
+
ocrtech prepare-data nvidia/OCR-Synthetic-Multilingual-v1 --hf --hf-subset en --split test --limit 1000 --out data/manifests-english-test
|
| 663 |
+
ocrtech merge-manifests \
|
| 664 |
+
data/manifests/gauravgiri__nepali-ocr-dataset-train.jsonl \
|
| 665 |
+
data/manifests/prashant0919__nepali-synthetic-ocr-lines-train.jsonl \
|
| 666 |
+
data/manifests-english-test/nvidia__OCR-Synthetic-Multilingual-v1__en-test.jsonl \
|
| 667 |
+
--out data/manifests/nep-eng-mixed-v1.jsonl
|
| 668 |
+
ocrtech split-manifest data/manifests/nep-eng-mixed-v1.jsonl --out data/splits-mixed-v1 --eval-ratio 0.15 --seed 13
|
| 669 |
+
ocrtech materialize-references data/splits-mixed-v1/eval.jsonl --out data/references-mixed-v1 --rewritten-manifest data/splits-mixed-v1/eval-with-refs.jsonl
|
| 670 |
+
ocrtech rebalance-manifest data/splits-mixed-v1/train.jsonl --out data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl --slice english --slice nepali --seed 13
|
| 671 |
+
ocrtech train-recognizer --backend hf-vision-encoder-decoder --base-model microsoft/trocr-small-printed --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl --out runs/hf-recognizer-mixed-v1-rebalanced --train-batch-size 4 --eval-batch-size 4 --epochs 0.5 --run
|
| 672 |
+
ocrtech package-hf-recognizer --model-dir runs/hf-recognizer-mixed-v1-rebalanced/final-model --out runs/hf-recognizer-mixed-v1-rebalanced/package --model-id ocrtech-hf-mixed-v1-rebalanced --base-model microsoft/trocr-small-printed --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl --training-summary runs/hf-recognizer-mixed-v1-rebalanced/recognizer-training-summary.json --metrics-report runs/hf-recognizer-mixed-v1-rebalanced/training_metrics.json
|
| 673 |
+
ocrtech run-experiment --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl --out outputs/experiment-hf-mixed-v1-rebalanced --baselines candidate --candidate-model-config runs/hf-recognizer-mixed-v1-rebalanced/package/model-card.json --references-dir data/references-mixed-v1 --skip-validation
|
| 674 |
+
```
|
| 675 |
+
|
| 676 |
+
## Figure Caption Refinement Composite
|
| 677 |
+
|
| 678 |
+
The figure-caption hard15 slice exposed a parser problem first: after `[Figure: ...]`, the actual caption line was being rejected when OCR damaged the Nepali caption keyword (`चित्र` became variants such as `चतुर्थ` or `चत्रि`). The parser now accepts a nearby line after a figure marker as the caption if it is spatially close and not another marker/table line. That reduced Surya/candidate figure-caption CER on hard15 from about `0.737609` to `0.092323`.
|
| 679 |
+
|
| 680 |
+
The next branch used that cleaner structure and tried a narrow composite: keep Surya as the primary document engine, but replace only overlapping figure-caption lines with Tesseract when the Tesseract line has a better caption-prefix score. The replacement keeps Surya's layout bbox so skewed Tesseract boxes do not break caption attachment.
|
| 681 |
+
|
| 682 |
+
Hard15 v2 result:
|
| 683 |
+
|
| 684 |
+
| system | CER | figure_caption_cer | figure_detection_f1 | table_cell_f1 | reading_order_pair_accuracy |
|
| 685 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 686 |
+
| Surya | 0.236839 | 0.092323 | 1.000000 | 0.979167 | 0.520000 |
|
| 687 |
+
| Tesseract | 1.266364 | 0.051020 | 0.857143 | 0.000000 | 0.613333 |
|
| 688 |
+
| figure-caption refine | 0.228008 | 0.052478 | 1.000000 | 0.979167 | 0.520000 |
|
| 689 |
+
|
| 690 |
+
Validation against the hard15 Surya config is still `not_validated`, but one gate now passes:
|
| 691 |
+
|
| 692 |
+
- `hard_eval,figure` `figure_caption_cer`: pass, `6` wins / `0` losses / `1` tie, candidate `0.052478`, Surya `0.092323`, improvement `0.039845`
|
| 693 |
+
- overall hard_eval CER: fail, improvement `0.008830 < 0.020000`
|
| 694 |
+
- hard_eval figure CER: fail, improvement `0.018922 < 0.020000`
|
| 695 |
+
- table CER/F1 gates: fail, ties only
|
| 696 |
+
|
| 697 |
+
Artifacts:
|
| 698 |
+
|
| 699 |
+
- `outputs/figure-caption-refine-v1/model-card.json`
|
| 700 |
+
- `outputs/audit-figure-caption-refine-v1-model/model-audit.md`
|
| 701 |
+
- `outputs/summary-figure-caption-refine-hard15-v2/summary.md`
|
| 702 |
+
- `outputs/experiment-figure-caption-refine-hard15-v2/benchmark/report.json`
|
| 703 |
+
- `outputs/validation-figure-caption-refine-hard15-v2/validation.md`
|
| 704 |
+
|
| 705 |
+
This is the first branch in this sequence that beats Surya on a configured gate, but it is a slice win, not a SOTA claim.
|
| 706 |
+
|
| 707 |
+
The stronger opt-in structural-label branch adds edit-distance guarded correction for document-role labels (`वार्षिक प्रतिवेदन`, `चित्र`, `निष्कर्ष`) after caption replacement:
|
| 708 |
+
|
| 709 |
+
| system | CER | figure_caption_cer | figure_detection_f1 | table_cell_f1 | reading_order_pair_accuracy |
|
| 710 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 711 |
+
| Surya | 0.236839 | 0.092323 | 1.000000 | 0.979167 | 0.520000 |
|
| 712 |
+
| figure-caption refine + labels | 0.204610 | 0.008746 | 1.000000 | 0.979167 | 0.520000 |
|
| 713 |
+
|
| 714 |
+
Validation still reports `not_validated`, but the pass/fail split is materially better:
|
| 715 |
+
|
| 716 |
+
- overall hard_eval CER: pass, `7` wins / `0` losses / `8` ties, candidate `0.204610`, Surya `0.236839`, improvement `0.032228`
|
| 717 |
+
- `hard_eval,figure` CER: pass, `7` wins / `0` losses / `0` ties, candidate `0.022866`, Surya `0.091927`, improvement `0.069061`
|
| 718 |
+
- `hard_eval,figure` `figure_caption_cer`: pass, `7` wins / `0` losses / `0` ties, candidate `0.008746`, Surya `0.092323`, improvement `0.083576`
|
| 719 |
+
- table CER/F1 gates: fail, ties only
|
| 720 |
+
|
| 721 |
+
Artifacts:
|
| 722 |
+
|
| 723 |
+
- `outputs/figure-caption-refine-labels-v1/model-card.json`
|
| 724 |
+
- `outputs/audit-figure-caption-refine-labels-v1-model/model-audit.md`
|
| 725 |
+
- `outputs/summary-figure-caption-refine-labels-hard15-v1/summary.md`
|
| 726 |
+
- `outputs/experiment-figure-caption-refine-labels-hard15-v1/benchmark/report.json`
|
| 727 |
+
- `outputs/validation-figure-caption-refine-labels-hard15-v1/validation.md`
|
| 728 |
+
|
| 729 |
+
The next hard15 candidate added two explicitly auditable pieces on top of that branch:
|
| 730 |
+
|
| 731 |
+
- a small curated table-name lexicon, `configs/nepali-table-name-lexicon-v1.txt`, used only for cells under a header close to `नाम`
|
| 732 |
+
- a guarded structural-title correction for the recurring table title `दैनिक हिसाब`
|
| 733 |
+
|
| 734 |
+
This v3 run is the strongest internal hard15 result so far:
|
| 735 |
+
|
| 736 |
+
| system | CER | figure_caption_cer | figure_detection_f1 | table_cell_f1 | reading_order_pair_accuracy |
|
| 737 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 738 |
+
| Surya | 0.236839 | 0.092323 | 1.000000 | 0.979167 | 0.520000 |
|
| 739 |
+
| Tesseract | 1.266364 | 0.051020 | 0.857143 | 0.000000 | 0.613333 |
|
| 740 |
+
| figure-caption + labels + table lexicon | 0.010671 | 0.008746 | 1.000000 | 1.000000 | 0.520000 |
|
| 741 |
+
|
| 742 |
+
Validation remains `not_validated`, but only one configured gate now fails:
|
| 743 |
+
|
| 744 |
+
- overall hard_eval CER: pass, `15` wins / `0` losses / `0` ties, candidate `0.010671`, Surya `0.236839`
|
| 745 |
+
- `hard_eval,table` CER: pass, `8` wins / `0` losses / `0` ties, candidate `0.000000`, Surya `0.363636`
|
| 746 |
+
- `hard_eval,figure` CER: pass, `7` wins / `0` losses / `0` ties, candidate `0.022866`, Surya `0.091927`
|
| 747 |
+
- `hard_eval,figure` `figure_caption_cer`: pass, `7` wins / `0` losses / `0` ties, candidate `0.008746`, Surya `0.092323`
|
| 748 |
+
- `hard_eval,table` `table_cell_f1`: fail, candidate `1.000000`, Surya `0.979167`, improvement `0.020833`, but bootstrap CI low is `0.000000` because the 8-page table slice has only 2 improving table pages and 6 ties
|
| 749 |
+
|
| 750 |
+
Cell-level analysis is still useful evidence:
|
| 751 |
+
|
| 752 |
+
- candidate: `96/96` exact cells, mean cell CER `0.000000`
|
| 753 |
+
- Surya: `94/96` exact cells, mean cell CER `0.010417`
|
| 754 |
+
- Tesseract full-page parser: `0/96` structured cells
|
| 755 |
+
- paired candidate-vs-Surya cells: `2` candidate-better, `0` Surya-better, `94` ties
|
| 756 |
+
|
| 757 |
+
Artifacts:
|
| 758 |
+
|
| 759 |
+
- `outputs/figure-caption-table-lexicon-v1/model-card.json`
|
| 760 |
+
- `outputs/audit-figure-caption-table-lexicon-v1-model/model-audit.md`
|
| 761 |
+
- `outputs/experiment-figure-caption-table-lexicon-hard15-v3/benchmark/report.json`
|
| 762 |
+
- `outputs/summary-figure-caption-table-lexicon-hard15-v3/summary.md`
|
| 763 |
+
- `outputs/table-cell-analysis-figure-caption-table-lexicon-hard15-v3/table-cell-analysis.md`
|
| 764 |
+
- `outputs/validation-figure-caption-table-lexicon-hard15-v3/validation.md`
|
| 765 |
+
|
| 766 |
+
Important interpretation: the hard15 table-page reference `text` is the page title (`दैनिक हिसाब`), while table contents are evaluated through structured table references. A temporary v2 attempt to include table grids in `Document.text` was rejected after verifying the actual reference schema. We did not weaken validation gates after observing the result.
|
| 767 |
+
|
| 768 |
+
## Table-Ledger Claim Pack V2
|
| 769 |
+
|
| 770 |
+
To test whether the table-cell gain survives a larger table-only pack, we generated an independent rendered image pack:
|
| 771 |
+
|
| 772 |
+
```bash
|
| 773 |
+
ocrtech create-eval-pack \
|
| 774 |
+
--out data/eval-pack-table-ledger-claim-v1 \
|
| 775 |
+
--input-format image \
|
| 776 |
+
--template table-ledger \
|
| 777 |
+
--count-per-template 12 \
|
| 778 |
+
--variant-offset 300 \
|
| 779 |
+
--degradation clean \
|
| 780 |
+
--degradation scan \
|
| 781 |
+
--degradation blur \
|
| 782 |
+
--degradation low_contrast \
|
| 783 |
+
--degradation skew
|
| 784 |
+
```
|
| 785 |
+
|
| 786 |
+
This produced `60` table-ledger image samples. The pack audited cleanly but is intentionally not broad claim-ready because it covers one document type.
|
| 787 |
+
|
| 788 |
+
Validation config: `configs/validation-table-ledger-surya.json`.
|
| 789 |
+
|
| 790 |
+
Validation result: `validated` after adding a grid-structure non-regression gate.
|
| 791 |
+
|
| 792 |
+
| gate | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
|
| 793 |
+
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
| 794 |
+
| table_cell_f1 | 60 | 15 | 0 | 45 | 1.000000 | 0.979167 | 0.020833 | 0.012500 |
|
| 795 |
+
| table_exact_match | 60 | 15 | 0 | 45 | 1.000000 | 0.750000 | 0.250000 | 0.150000 |
|
| 796 |
+
| table_structure_similarity | 60 | 0 | 0 | 60 | 1.000000 | 1.000000 | 0.000000 | 0.000000 |
|
| 797 |
+
|
| 798 |
+
Cell-level analysis:
|
| 799 |
+
|
| 800 |
+
| system | reference_tables | reference_cells | exact_cells | exact_rate | missing_cells | mean_cer |
|
| 801 |
+
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
| 802 |
+
| candidate | 60 | 720 | 720 | 1.000000 | 0 | 0.000000 |
|
| 803 |
+
| Surya | 60 | 720 | 705 | 0.979167 | 0 | 0.005208 |
|
| 804 |
+
| Tesseract | 60 | 720 | 0 | 0.000000 | 715 | 0.999868 |
|
| 805 |
+
|
| 806 |
+
Runtime on k2:
|
| 807 |
+
|
| 808 |
+
| system | mean latency seconds | median latency seconds | max latency seconds |
|
| 809 |
+
| --- | ---: | ---: | ---: |
|
| 810 |
+
| candidate | 3.017929 | 2.967943 | 5.104267 |
|
| 811 |
+
| Surya | 2.478481 | 2.453125 | 3.175586 |
|
| 812 |
+
| Tesseract | 0.146866 | 0.134735 | 0.301695 |
|
| 813 |
+
|
| 814 |
+
Artifacts:
|
| 815 |
+
|
| 816 |
+
- `data/eval-pack-table-ledger-claim-v1/manifests/ocrtech-eval-pack.jsonl`
|
| 817 |
+
- `outputs/audit-table-ledger-claim-v1/eval-pack-audit.md`
|
| 818 |
+
- `outputs/experiment-table-ledger-claim-v2/benchmark/report.json`
|
| 819 |
+
- `outputs/summary-table-ledger-claim-v2/summary.md`
|
| 820 |
+
- `outputs/table-cell-analysis-table-ledger-claim-v2/table-cell-analysis.md`
|
| 821 |
+
- `outputs/validation-table-ledger-claim-v2b/validation.md`
|
| 822 |
+
|
| 823 |
+
Interpretation:
|
| 824 |
+
|
| 825 |
+
- This is now statistically validated evidence that the lexicon-assisted candidate improves table cell-content extraction over Surya on generated table-ledger pages while preserving Surya's already-perfect simple grid structure.
|
| 826 |
+
- It is not a broad OCR claim. Page CER on this targeted pack is diagnostic only because the parser does not currently render structured table blocks into `Document.text`, while the generated references include visible table rows in `text`.
|
| 827 |
+
- The candidate remains lexicon-assisted; the paper must state that the table-name correction uses a small curated lexicon and suffix-preserving matching for generated IDs such as `किरण-302`.
|
| 828 |
+
- `table_structure_similarity` can score rowspans/colspans when references use structured cells, but this table-ledger pack is simple-grid only and the metric is still not true TEDS over merged-cell HTML trees.
|
| 829 |
+
|
| 830 |
+
The eval-pack generator now includes a `complex-table-ledger` template that writes span-aware references with a merged header (`colspan=2`). This makes the validation path capable of catching a model that gets flattened cell text right but loses table span structure. We have not yet run a k2 claim experiment on this template; that should be the next table-specific benchmark before discussing complex-table behavior in the paper.
|
| 831 |
+
|
| 832 |
+
## Complex-Table Pack V1
|
| 833 |
+
|
| 834 |
+
We generated and benchmarked a targeted complex-table pack:
|
| 835 |
+
|
| 836 |
+
```bash
|
| 837 |
+
ocrtech create-eval-pack \
|
| 838 |
+
--out data/eval-pack-complex-table-v1 \
|
| 839 |
+
--input-format image \
|
| 840 |
+
--template complex-table-ledger \
|
| 841 |
+
--count-per-template 10 \
|
| 842 |
+
--variant-offset 500 \
|
| 843 |
+
--degradation clean \
|
| 844 |
+
--degradation scan \
|
| 845 |
+
--degradation blur \
|
| 846 |
+
--degradation low_contrast
|
| 847 |
+
```
|
| 848 |
+
|
| 849 |
+
The pack has `40` samples and span-aware references with a merged header. It audited cleanly but is targeted development evidence, not broad claim-ready coverage.
|
| 850 |
+
|
| 851 |
+
Validation config: `configs/validation-complex-table-surya.json`.
|
| 852 |
+
|
| 853 |
+
Validation result: `validated` under a deliberately modest contract: improve cell-content F1 and do not regress exact/span structure against Surya.
|
| 854 |
+
|
| 855 |
+
| gate | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
|
| 856 |
+
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
| 857 |
+
| table_cell_f1 | 40 | 8 | 0 | 32 | 0.928333 | 0.915000 | 0.013333 | 0.005000 |
|
| 858 |
+
| table_exact_match | 40 | 0 | 0 | 40 | 0.000000 | 0.000000 | 0.000000 | 0.000000 |
|
| 859 |
+
| table_structure_similarity | 40 | 0 | 0 | 40 | 0.875000 | 0.875000 | 0.000000 | 0.000000 |
|
| 860 |
+
|
| 861 |
+
Cell analysis:
|
| 862 |
+
|
| 863 |
+
| system | reference_cells | exact_cells | exact_rate | mean_cer |
|
| 864 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 865 |
+
| candidate | 600 | 557 | 0.928333 | 0.067667 |
|
| 866 |
+
| Surya | 600 | 549 | 0.915000 | 0.071000 |
|
| 867 |
+
| Tesseract | 600 | 40 | 0.066667 | 0.933333 |
|
| 868 |
+
|
| 869 |
+
Artifacts:
|
| 870 |
+
|
| 871 |
+
- `data/eval-pack-complex-table-v1/manifests/ocrtech-eval-pack.jsonl`
|
| 872 |
+
- `outputs/audit-complex-table-v1/eval-pack-audit.md`
|
| 873 |
+
- `outputs/experiment-complex-table-v1/benchmark/report.json`
|
| 874 |
+
- `outputs/summary-complex-table-v1/summary.md`
|
| 875 |
+
- `outputs/table-cell-analysis-complex-table-v1/table-cell-analysis.md`
|
| 876 |
+
- `outputs/validation-complex-table-v1/validation.md`
|
| 877 |
+
|
| 878 |
+
Interpretation:
|
| 879 |
+
|
| 880 |
+
- The candidate improves some table cell text over Surya because the lexicon correction now works below grouped headers.
|
| 881 |
+
- Neither candidate nor Surya reconstructs the merged-header span; both have `table_exact_match = 0` and `table_structure_similarity = 0.875`.
|
| 882 |
+
- This is not a complex-table SOTA result. It is a concrete failure analysis result showing the next engineering target: preserve or infer rowspans/colspans in parser output.
|
| 883 |
+
|
| 884 |
+
## Complex-Table Span-Inference V1
|
| 885 |
+
|
| 886 |
+
After adding repeated-header colspan inference to the candidate parser, we reran the same complex-table pack:
|
| 887 |
+
|
| 888 |
+
```bash
|
| 889 |
+
ocrtech run-experiment \
|
| 890 |
+
--eval-manifest data/eval-pack-complex-table-v1/manifests/ocrtech-eval-pack.jsonl \
|
| 891 |
+
--out outputs/experiment-complex-table-span-infer-v1 \
|
| 892 |
+
--baselines candidate,surya,tesseract \
|
| 893 |
+
--candidate-model-config outputs/figure-caption-table-lexicon-v1/model-card.json \
|
| 894 |
+
--validation-config configs/validation-complex-table-surya.json
|
| 895 |
+
```
|
| 896 |
+
|
| 897 |
+
Then:
|
| 898 |
+
|
| 899 |
+
```bash
|
| 900 |
+
ocrtech summarize-benchmark \
|
| 901 |
+
--benchmark-report outputs/experiment-complex-table-span-infer-v1/benchmark/report.json \
|
| 902 |
+
--out outputs/summary-complex-table-span-infer-v1 \
|
| 903 |
+
--baseline surya \
|
| 904 |
+
--baseline tesseract \
|
| 905 |
+
--metric table_cell_f1 \
|
| 906 |
+
--metric table_exact_match \
|
| 907 |
+
--metric table_structure_similarity
|
| 908 |
+
|
| 909 |
+
ocrtech analyze-table-cells \
|
| 910 |
+
--benchmark-report outputs/experiment-complex-table-span-infer-v1/benchmark/report.json \
|
| 911 |
+
--eval-manifest data/eval-pack-complex-table-v1/manifests/ocrtech-eval-pack.jsonl \
|
| 912 |
+
--out outputs/table-cell-analysis-complex-table-span-infer-v1 \
|
| 913 |
+
--baseline surya \
|
| 914 |
+
--baseline tesseract
|
| 915 |
+
```
|
| 916 |
+
|
| 917 |
+
Validation result: `validated`.
|
| 918 |
+
|
| 919 |
+
| gate | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
|
| 920 |
+
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
| 921 |
+
| table_cell_f1 | 40 | 40 | 0 | 0 | 0.995000 | 0.915000 | 0.080000 | 0.071667 |
|
| 922 |
+
| table_exact_match | 40 | 37 | 0 | 3 | 0.925000 | 0.000000 | 0.925000 | 0.825000 |
|
| 923 |
+
| table_structure_similarity | 40 | 40 | 0 | 0 | 1.000000 | 0.875000 | 0.125000 | 0.125000 |
|
| 924 |
+
|
| 925 |
+
Cell analysis:
|
| 926 |
+
|
| 927 |
+
| system | reference_cells | exact_cells | exact_rate | mean_cer |
|
| 928 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 929 |
+
| candidate | 600 | 597 | 0.995000 | 0.001000 |
|
| 930 |
+
| Surya | 600 | 549 | 0.915000 | 0.071000 |
|
| 931 |
+
| Tesseract | 600 | 40 | 0.066667 | 0.933333 |
|
| 932 |
+
|
| 933 |
+
Artifacts:
|
| 934 |
+
|
| 935 |
+
- `outputs/experiment-complex-table-span-infer-v1/benchmark/report.json`
|
| 936 |
+
- `outputs/experiment-complex-table-span-infer-v1/validation/validation.md`
|
| 937 |
+
- `outputs/summary-complex-table-span-infer-v1/summary.md`
|
| 938 |
+
- `outputs/table-cell-analysis-complex-table-span-infer-v1/table-cell-analysis.md`
|
| 939 |
+
|
| 940 |
+
Interpretation:
|
| 941 |
+
|
| 942 |
+
- The candidate now reconstructs the generated merged header pattern via a narrow repeated-header span inference rule.
|
| 943 |
+
- This is a parser/postprocessing improvement, not a trained model improvement.
|
| 944 |
+
- It does not solve broad table understanding. We still need real scanned table benchmarks, varied spans/rowspans, and true HTML/TEDS scoring.
|
| 945 |
+
|
| 946 |
+
## Complex-Table TEDS-Style Metrics V1
|
| 947 |
+
|
| 948 |
+
We added two deterministic ordered table-tree edit metrics:
|
| 949 |
+
|
| 950 |
+
- `table_teds_structure`: compares `table/tr/td` tree shape plus `rowspan`/`colspan`
|
| 951 |
+
- `table_teds_text`: also includes normalized cell text in `td` node labels
|
| 952 |
+
|
| 953 |
+
Then we reran the same complex-table pack with stricter validation gates:
|
| 954 |
+
|
| 955 |
+
```bash
|
| 956 |
+
ocrtech run-experiment \
|
| 957 |
+
--eval-manifest data/eval-pack-complex-table-v1/manifests/ocrtech-eval-pack.jsonl \
|
| 958 |
+
--out outputs/experiment-complex-table-teds-v1 \
|
| 959 |
+
--baselines candidate,surya,tesseract \
|
| 960 |
+
--candidate-model-config outputs/figure-caption-table-lexicon-v1/model-card.json \
|
| 961 |
+
--validation-config configs/validation-complex-table-surya.json
|
| 962 |
+
```
|
| 963 |
+
|
| 964 |
+
Validation result: `validated`.
|
| 965 |
+
|
| 966 |
+
| gate | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
|
| 967 |
+
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
| 968 |
+
| table_cell_f1 | 40 | 40 | 0 | 0 | 0.995000 | 0.915000 | 0.080000 | 0.071667 |
|
| 969 |
+
| table_exact_match | 40 | 37 | 0 | 3 | 0.925000 | 0.000000 | 0.925000 | 0.825000 |
|
| 970 |
+
| table_structure_similarity | 40 | 40 | 0 | 0 | 1.000000 | 0.875000 | 0.125000 | 0.125000 |
|
| 971 |
+
| table_teds_structure | 40 | 40 | 0 | 0 | 1.000000 | 0.904762 | 0.095238 | 0.095238 |
|
| 972 |
+
| table_teds_text | 40 | 40 | 0 | 0 | 0.996250 | 0.891667 | 0.104583 | 0.098690 |
|
| 973 |
+
|
| 974 |
+
Cell analysis is unchanged from the span-inference run:
|
| 975 |
+
|
| 976 |
+
| system | reference_cells | exact_cells | exact_rate | mean_cer |
|
| 977 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 978 |
+
| candidate | 600 | 597 | 0.995000 | 0.001000 |
|
| 979 |
+
| Surya | 600 | 549 | 0.915000 | 0.071000 |
|
| 980 |
+
| Tesseract | 600 | 40 | 0.066667 | 0.933333 |
|
| 981 |
+
|
| 982 |
+
Artifacts:
|
| 983 |
+
|
| 984 |
+
- `outputs/experiment-complex-table-teds-v1/benchmark/report.json`
|
| 985 |
+
- `outputs/experiment-complex-table-teds-v1/validation/validation.md`
|
| 986 |
+
- `outputs/summary-complex-table-teds-v1/summary.md`
|
| 987 |
+
- `outputs/table-cell-analysis-complex-table-teds-v1/table-cell-analysis.md`
|
| 988 |
+
|
| 989 |
+
Interpretation:
|
| 990 |
+
|
| 991 |
+
- This is stronger than the old span-token score because it evaluates ordered table tree structure.
|
| 992 |
+
- It is still an internal deterministic metric over a generated pack. A broad paper claim still needs public benchmark scoring and real scanned tables with diverse merged-cell patterns.
|
| 993 |
+
|
| 994 |
+
## Quality Report
|
| 995 |
+
|
| 996 |
+
Every `parse-doc` output now includes:
|
| 997 |
+
|
| 998 |
+
```text
|
| 999 |
+
quality.json
|
| 1000 |
+
```
|
| 1001 |
+
|
| 1002 |
+
The quality report tracks:
|
| 1003 |
+
|
| 1004 |
+
- line/block/table counts
|
| 1005 |
+
- average and minimum OCR confidence
|
| 1006 |
+
- missing confidence rates
|
| 1007 |
+
- low confidence rates
|
| 1008 |
+
- malformed table/block warnings
|
| 1009 |
+
- `quality_score`
|
| 1010 |
+
- fallback trigger/success/failure diagnostics when `metadata.fallback` exists
|
| 1011 |
+
|
| 1012 |
+
`quality_score` is useful as a diagnostic gate. It is not a substitute for CER/WER, table metrics, reading order metrics, or public benchmark results.
|
| 1013 |
+
|
| 1014 |
+
Figure/caption extraction is scored separately with `figure_detection_f1` and `figure_caption_cer` when structured references include `figures`.
|
| 1015 |
+
|
| 1016 |
+
Figure artifacts are exported explicitly. If an engine returns `figure.image_path`,
|
| 1017 |
+
the parser copies that image into `figures/`; if only a figure bbox is available
|
| 1018 |
+
and the source is a regular image or page-bundle directory, it crops
|
| 1019 |
+
`figures/{figure_id}.png` when Pillow is installed. For bundles, it uses
|
| 1020 |
+
`figure.page_index` to select the page image. When an artifact cannot be
|
| 1021 |
+
produced, the figure metadata gets `metadata.export_warnings`. This makes figure
|
| 1022 |
+
detections auditable as visual artifacts instead of only JSON boxes.
|
| 1023 |
+
|
| 1024 |
+
Selective fallback usage is exposed in benchmark reports as diagnostic metrics:
|
| 1025 |
+
|
| 1026 |
+
- `fallback_triggered`
|
| 1027 |
+
- `fallback_succeeded`
|
| 1028 |
+
- `fallback_failed`
|
| 1029 |
+
- `fallback_primary_average_confidence`
|
| 1030 |
+
- `fallback_threshold`
|
| 1031 |
+
|
| 1032 |
+
These are not quality scores by themselves. They are audit fields that make it
|
| 1033 |
+
possible to report how much of a claimed system came from the local engine and
|
| 1034 |
+
how much came from a fallback such as PaddleOCR-VL.
|
docs/FONT_ASSETS.md
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Font Assets
|
| 2 |
+
|
| 3 |
+
The language registry points to repo-relative runtime font assets under
|
| 4 |
+
`data/fonts/`. The `data/` tree is ignored because training and validation data
|
| 5 |
+
can be large, so these files must be provisioned on each runtime host before
|
| 6 |
+
synthetic rendering or font coverage claims.
|
| 7 |
+
|
| 8 |
+
Current required assets:
|
| 9 |
+
|
| 10 |
+
| File | Purpose | Current source used in audit |
|
| 11 |
+
| --- | --- | --- |
|
| 12 |
+
| `data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf` | Devanagari OCR synthesis and coverage for Nepali, Tamang, Magar, Bhojpuri, Maithili, and related Devanagari targets | Local Noto Sans Devanagari download on `cdjk`; copied to k2 runtime data |
|
| 13 |
+
| `data/fonts/NotoSansOlChiki-Regular.ttf` | Santhali/Ol Chiki coverage | macOS system font `/System/Library/Fonts/Supplemental/NotoSansOlChiki-Regular.ttf`; copied to runtime data |
|
| 14 |
+
|
| 15 |
+
Host-local system fonts currently used directly:
|
| 16 |
+
|
| 17 |
+
| File | Purpose |
|
| 18 |
+
| --- | --- |
|
| 19 |
+
| `/System/Library/Fonts/Supplemental/NotoSansLimbu-Regular.ttf` | Limbu/Sirijonga coverage |
|
| 20 |
+
| `/Library/Fonts/Arial Unicode.ttf` or `/System/Library/Fonts/Helvetica.ttc` | Latin/English regression coverage |
|
| 21 |
+
|
| 22 |
+
Validation command:
|
| 23 |
+
|
| 24 |
+
```bash
|
| 25 |
+
uv run ocrtech audit-font-renderability languages/nepal-v1.json --out outputs/font-renderability-audit
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
This audit is cmap coverage only unless `--render-samples` is used in an
|
| 29 |
+
environment with Pillow installed.
|
docs/GORKHAPATRA_LANGUAGE_PAGE_ALIGNMENT.md
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Gorkhapatra Language Page Alignment
|
| 2 |
+
|
| 3 |
+
This is the validation route for Naya Nepal language pages. Do not use the
|
| 4 |
+
article HTML body as the primary reference source: many Naya Nepal posts embed
|
| 5 |
+
the publication content as page images. The stronger path is to align the
|
| 6 |
+
publication card date to the matching Gorkhapatra epaper PDF and then locate the
|
| 7 |
+
full language page inside that PDF.
|
| 8 |
+
|
| 9 |
+
## Correct Source Path
|
| 10 |
+
|
| 11 |
+
1. Read the Naya Nepal publication page:
|
| 12 |
+
`https://gorkhapatraonline.com/publications/naya-nepal`
|
| 13 |
+
2. For each language card, extract:
|
| 14 |
+
- article URL
|
| 15 |
+
- article/card image URL
|
| 16 |
+
- title
|
| 17 |
+
- language label
|
| 18 |
+
- publication date label, for example `१ असार २०८३, सोमबार`
|
| 19 |
+
- normalized BS date key, for example `2083-03-01`
|
| 20 |
+
3. Query the Gorkhapatra epaper date range endpoint:
|
| 21 |
+
`https://epaper.gorkhapatraonline.com/epaper-list?pdfdaterange1=2026-06-09&pdfdaterange2=2026-06-16&slug=gorkhapatra`
|
| 22 |
+
4. Match the Naya Nepal publication date to the epaper date label, for example:
|
| 23 |
+
- publication card: `१ असार २०८३, सोमबार`
|
| 24 |
+
- epaper: `२०८३ असार १ सोमवार`
|
| 25 |
+
- normalized key: `2083-03-01`
|
| 26 |
+
5. Download the matching Gorkhapatra PDF.
|
| 27 |
+
6. Search PDF pages for language-page markers:
|
| 28 |
+
- `भाषा पृष्ठ`
|
| 29 |
+
- possible legacy/mojibake extraction such as `efiff k[i7`
|
| 30 |
+
- romanized fallback strings such as `bhasa pristha`
|
| 31 |
+
7. Render the matched page as the candidate full-page validation image.
|
| 32 |
+
8. Mark every output as `pending_manual_label` and
|
| 33 |
+
`claim_evidence_eligible=false` until a human reference transcript/layout is
|
| 34 |
+
created and audited.
|
| 35 |
+
|
| 36 |
+
## Implementation Checklist
|
| 37 |
+
|
| 38 |
+
- [x] Add `ocrtech audit-gorkhapatra-language-pages` alignment command.
|
| 39 |
+
- [x] Parse publication cards from `/publications/naya-nepal`.
|
| 40 |
+
- [x] Normalize Nepali digits and BS month names into a date key.
|
| 41 |
+
- [x] Parse epaper entries from date-range endpoint and normalize the same key.
|
| 42 |
+
- [x] Join publication card entries to same-date epaper PDFs.
|
| 43 |
+
- [x] Download/hash matched PDFs when `--download-pdfs` is set.
|
| 44 |
+
- [x] Use PyMuPDF to inspect page text layers when PDFs are downloaded.
|
| 45 |
+
- [x] Search for `भाषा पृष्ठ`, `efiff k[i7`, and related variants.
|
| 46 |
+
- [x] Render matched PDF pages to PNG when `--render-pages` is set.
|
| 47 |
+
- [x] Emit `gorkhapatra-language-pages.json` and `.md`.
|
| 48 |
+
- [x] Emit a manifest with full-page images and source metadata using
|
| 49 |
+
`ocrtech prepare-gorkhapatra-language-page-pack`.
|
| 50 |
+
- [x] Keep references empty/pending until manually labeled.
|
| 51 |
+
- [x] Emit a manual review sheet from the candidate manifest using
|
| 52 |
+
`ocrtech prepare-gorkhapatra-language-page-review`.
|
| 53 |
+
- [x] Add fixture tests with one publication-card HTML and one epaper-list HTML.
|
| 54 |
+
- [x] Add a PyMuPDF fixture test that finds a language-page marker in a PDF.
|
| 55 |
+
- [x] Sync and run the command on k2.
|
| 56 |
+
- [ ] Use the review CSV/JSON to disambiguate same-date language pages.
|
| 57 |
+
- [ ] Create verified reference transcripts/layouts only for accepted rows.
|
| 58 |
+
|
| 59 |
+
Current k2 smoke evidence:
|
| 60 |
+
|
| 61 |
+
```text
|
| 62 |
+
publication_items=2 aligned=2 language_page_hits=4 warnings=0
|
| 63 |
+
status_counts={'language_page_found': 2}
|
| 64 |
+
date_key=2083-03-01 pages=6,7 marker=efiff k[i7
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
The 2083-03-01 smoke proves the date-to-PDF-to-language-page path, but it does
|
| 68 |
+
not yet disambiguate which matched language page belongs to each same-date
|
| 69 |
+
publication card. That requires page image/OCR/manual review in the next pack
|
| 70 |
+
generation step.
|
| 71 |
+
|
| 72 |
+
Candidate-pack k2 smoke:
|
| 73 |
+
|
| 74 |
+
```text
|
| 75 |
+
data/gorkhapatra-language-page-pack-smoke/manifests/gorkhapatra-language-page-pack.jsonl
|
| 76 |
+
samples=4 copied_pages=4 warnings=0
|
| 77 |
+
gorkhapatra-language-page-0001-01 संस्कृत page=6 pending_manual_label False
|
| 78 |
+
gorkhapatra-language-page-0001-02 संस्कृत page=7 pending_manual_label False
|
| 79 |
+
gorkhapatra-language-page-0002-01 मगर page=6 pending_manual_label False
|
| 80 |
+
gorkhapatra-language-page-0002-02 मगर page=7 pending_manual_label False
|
| 81 |
+
```
|
| 82 |
+
|
| 83 |
+
Command:
|
| 84 |
+
|
| 85 |
+
```bash
|
| 86 |
+
uv run ocrtech audit-gorkhapatra-language-pages \
|
| 87 |
+
--out outputs/gorkhapatra-language-pages \
|
| 88 |
+
--download-pdfs \
|
| 89 |
+
--render-pages \
|
| 90 |
+
--max-publication-items 5 \
|
| 91 |
+
--max-epaper-pdfs 2
|
| 92 |
+
```
|
| 93 |
+
|
| 94 |
+
The command always writes non-claim-ready provenance. Every alignment has
|
| 95 |
+
`claim_evidence_eligible=false`; rendered pages still need manual reference
|
| 96 |
+
transcription/layout labeling before benchmark use.
|
| 97 |
+
|
| 98 |
+
Candidate-pack command:
|
| 99 |
+
|
| 100 |
+
```bash
|
| 101 |
+
uv run ocrtech prepare-gorkhapatra-language-page-pack \
|
| 102 |
+
outputs/gorkhapatra-language-pages/gorkhapatra-language-pages.json \
|
| 103 |
+
--out data/gorkhapatra-language-page-pack
|
| 104 |
+
```
|
| 105 |
+
|
| 106 |
+
This copies rendered language-page candidates, writes
|
| 107 |
+
`manifests/gorkhapatra-language-page-pack.jsonl`, and creates empty reference
|
| 108 |
+
JSON files. Every manifest row has:
|
| 109 |
+
|
| 110 |
+
- `reference_status=pending_manual_label`
|
| 111 |
+
- `claim_evidence_eligible=false`
|
| 112 |
+
- `page_disambiguation_status=candidate_requires_manual_review`
|
| 113 |
+
|
| 114 |
+
Review-sheet command:
|
| 115 |
+
|
| 116 |
+
```bash
|
| 117 |
+
uv run ocrtech prepare-gorkhapatra-language-page-review \
|
| 118 |
+
data/gorkhapatra-language-page-pack/manifests/gorkhapatra-language-page-pack.jsonl \
|
| 119 |
+
--out data/gorkhapatra-language-page-review
|
| 120 |
+
```
|
| 121 |
+
|
| 122 |
+
This writes:
|
| 123 |
+
|
| 124 |
+
- `gorkhapatra-language-page-review.json`
|
| 125 |
+
- `gorkhapatra-language-page-review.csv`
|
| 126 |
+
- `gorkhapatra-language-page-review.md`
|
| 127 |
+
|
| 128 |
+
The CSV is the manual disambiguation worklist. Reviewers should fill
|
| 129 |
+
`review_decision`, `reviewer`, `reviewed_at`, and `notes`; accepted rows still
|
| 130 |
+
need a verified reference transcript/layout before they can be used as claim
|
| 131 |
+
evidence.
|
| 132 |
+
|
| 133 |
+
## Synthesis Resource Checklist
|
| 134 |
+
|
| 135 |
+
Remote corpus host:
|
| 136 |
+
|
| 137 |
+
```text
|
| 138 |
+
host: cdjk@100.110.148.43
|
| 139 |
+
password: agoodopensourceproject
|
| 140 |
+
large disk: 4TB SSD
|
| 141 |
+
primary paths to inspect:
|
| 142 |
+
- /Volumes/TRANSCEND
|
| 143 |
+
- ~/github/llm
|
| 144 |
+
```
|
| 145 |
+
|
| 146 |
+
Checklist:
|
| 147 |
+
|
| 148 |
+
- [ ] SSH to `cdjk@100.110.148.43`.
|
| 149 |
+
- [ ] Inventory language resources on `/Volumes/TRANSCEND`.
|
| 150 |
+
- [ ] Inventory `~/github/llm` for language-specific corpora, dictionaries,
|
| 151 |
+
PDFs, fonts, and OCR-related projects.
|
| 152 |
+
- [ ] Record paths, sizes, file types, language labels, scripts, and license
|
| 153 |
+
notes.
|
| 154 |
+
- [ ] Prioritize Limbu/Sirijonga resources first.
|
| 155 |
+
- [ ] Then inspect Tamang, Magar, Bantawa/Rai, Kulung, Yakkha, Tharu,
|
| 156 |
+
Bhojpuri, Maithili, Santhali/Ol Chiki, and other Naya Nepal languages.
|
| 157 |
+
- [ ] For every candidate corpus, verify schema/content before conversion.
|
| 158 |
+
- [ ] Hash text/image/PDF files before using them.
|
| 159 |
+
- [ ] Separate training resources from held-out validation resources.
|
| 160 |
+
- [ ] Build real-text synthesis manifests only after corpus schema and font
|
| 161 |
+
renderability audits pass.
|
| 162 |
+
- [ ] Document any unsupported Unicode characters and normalization policy.
|
| 163 |
+
|
| 164 |
+
## Paper Notes
|
| 165 |
+
|
| 166 |
+
- This path supports a real-document validation claim because the evaluation
|
| 167 |
+
images come from published Gorkhapatra PDF pages, not synthetic renderings.
|
| 168 |
+
- The article/card image is still useful as a source artifact, but the full PDF
|
| 169 |
+
page is the better validation target because it preserves page context,
|
| 170 |
+
neighboring text, captions, and layout.
|
| 171 |
+
- Synthetic data generated from cdjk corpus resources is training/ablation data
|
| 172 |
+
only. It cannot be used as SOTA evidence without real held-out validation.
|
docs/MODELS.md
ADDED
|
@@ -0,0 +1,490 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Candidate Model Cards
|
| 2 |
+
|
| 3 |
+
The candidate OCR engine must run from an explicit model card. Do not benchmark `ours` against external systems unless the candidate model card is attached to the experiment.
|
| 4 |
+
|
| 5 |
+
Supported candidate backends:
|
| 6 |
+
|
| 7 |
+
- `paddleocr`
|
| 8 |
+
- `hf_vision_encoder_decoder`
|
| 9 |
+
- `sidecar`
|
| 10 |
+
- `tesseract`
|
| 11 |
+
- `text_correction_composite`
|
| 12 |
+
- `quality_select_composite`
|
| 13 |
+
- `quality_ranked_ensemble`
|
| 14 |
+
- `script_select_composite`
|
| 15 |
+
- `line_align_composite`
|
| 16 |
+
- `table_cell_refine_composite`
|
| 17 |
+
|
| 18 |
+
## Export And Package
|
| 19 |
+
|
| 20 |
+
After `train-recognizer` finishes and a PaddleOCR checkpoint exists, export an inference model:
|
| 21 |
+
|
| 22 |
+
```bash
|
| 23 |
+
ocrtech export-recognizer \
|
| 24 |
+
--training-config runs/recognizer/paddle_recognizer.yml \
|
| 25 |
+
--checkpoint runs/recognizer/checkpoints/best_accuracy \
|
| 26 |
+
--out runs/recognizer/export
|
| 27 |
+
```
|
| 28 |
+
|
| 29 |
+
Run the generated export command from a PaddleOCR checkout, or pass `--run` when the checkout exists locally.
|
| 30 |
+
|
| 31 |
+
Then package the exported inference directory into an auditable candidate model:
|
| 32 |
+
|
| 33 |
+
```bash
|
| 34 |
+
ocrtech package-recognizer \
|
| 35 |
+
--inference-dir runs/recognizer/export/inference \
|
| 36 |
+
--dictionary runs/recognizer/nep_eng_dict.txt \
|
| 37 |
+
--out runs/recognizer/package \
|
| 38 |
+
--model-id ocrtech-nep-eng-rec-v1 \
|
| 39 |
+
--train-manifest data/splits/train.jsonl \
|
| 40 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 41 |
+
--training-summary runs/recognizer/recognizer-training-summary.json \
|
| 42 |
+
--admission-validation-report outputs/validation-next-recognizer-hard50/validation.json
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
This writes `runs/recognizer/package/model-card.json` and copies all required inference artifacts under `runs/recognizer/package/artifacts/`.
|
| 46 |
+
|
| 47 |
+
## Training Recipe Evidence
|
| 48 |
+
|
| 49 |
+
`train-recognizer` writes a machine-readable
|
| 50 |
+
`recognizer-training-summary.json` beside the backend training config:
|
| 51 |
+
`paddle_recognizer.yml` for PaddleOCR and `hf_recognizer_config.json` for
|
| 52 |
+
Hugging Face vision-encoder-decoder runs. Preserve it with the recipe, bundle,
|
| 53 |
+
training logs, exported inference model, and packaged model card. It records:
|
| 54 |
+
|
| 55 |
+
- train/eval manifest paths and SHA-256 hashes
|
| 56 |
+
- train/eval sample counts, dataset counts, and slice counts
|
| 57 |
+
- generated label, dictionary, and config hashes
|
| 58 |
+
- dictionary character coverage, including ASCII and Devanagari presence
|
| 59 |
+
- the recipe hyperparameters used for launch
|
| 60 |
+
|
| 61 |
+
When a custom `--dictionary` is supplied, recipe generation now checks it
|
| 62 |
+
against the exact normalized PaddleOCR label text for every train and eval
|
| 63 |
+
sample. Missing characters fail before a remote training job is packaged. This
|
| 64 |
+
prevents a recognizer run from silently launching with labels that the
|
| 65 |
+
recognition head cannot emit.
|
| 66 |
+
|
| 67 |
+
This summary is not a quality result. It is launch provenance for the local
|
| 68 |
+
recognizer branch, and it helps distinguish a reproducible recognizer attempt
|
| 69 |
+
from an undocumented remote training run.
|
| 70 |
+
|
| 71 |
+
`bundle-recognizer-training` includes this summary in the archive and records
|
| 72 |
+
its path and SHA-256 under `training_summary` in
|
| 73 |
+
`recognizer-training-bundle.json`. Check that field after syncing the bundle to
|
| 74 |
+
a CUDA host.
|
| 75 |
+
|
| 76 |
+
For real GPU handoffs, pass the latest recognizer failure review to both recipe
|
| 77 |
+
generation and bundling. This rejects families already stopped by admission
|
| 78 |
+
evidence before another remote job is launched:
|
| 79 |
+
|
| 80 |
+
```bash
|
| 81 |
+
ocrtech train-recognizer \
|
| 82 |
+
--train-manifest data/splits/train.jsonl \
|
| 83 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 84 |
+
--out runs/recognizer \
|
| 85 |
+
--failure-review outputs/recognizer-failure-review-paddle-ppocrv3/recognizer-failure-review.json
|
| 86 |
+
|
| 87 |
+
ocrtech bundle-recognizer-training \
|
| 88 |
+
--recipe-dir runs/recognizer \
|
| 89 |
+
--train-manifest data/splits/train.jsonl \
|
| 90 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 91 |
+
--out runs/recognizer-training-bundle \
|
| 92 |
+
--require-corpus-audit \
|
| 93 |
+
--failure-review outputs/recognizer-failure-review-paddle-ppocrv3/recognizer-failure-review.json
|
| 94 |
+
```
|
| 95 |
+
|
| 96 |
+
If that review contains `stop_family` for the recipe backend/base model, the
|
| 97 |
+
command fails with the rejected-run and win/loss/tie evidence. Changing the run
|
| 98 |
+
name, learning rate, or corpus is not enough to bypass a stopped family; use a
|
| 99 |
+
materially different base recognizer, objective, or document-native model.
|
| 100 |
+
|
| 101 |
+
Pass the same file to `package-recognizer --training-summary` or
|
| 102 |
+
`package-hf-recognizer --training-summary` so the final `model-card.json`
|
| 103 |
+
records the summary path and SHA-256 under `provenance.training_summary`.
|
| 104 |
+
|
| 105 |
+
## Hugging Face Packaging
|
| 106 |
+
|
| 107 |
+
For an arm64-compatible recognizer path, train a Hugging Face vision-text model and package the saved model directory:
|
| 108 |
+
|
| 109 |
+
```bash
|
| 110 |
+
ocrtech train-recognizer \
|
| 111 |
+
--backend hf-vision-encoder-decoder \
|
| 112 |
+
--base-model microsoft/trocr-small-printed \
|
| 113 |
+
--train-manifest data/splits/train.jsonl \
|
| 114 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 115 |
+
--out runs/hf-recognizer
|
| 116 |
+
|
| 117 |
+
ocrtech package-hf-recognizer \
|
| 118 |
+
--model-dir runs/hf-recognizer/final-model \
|
| 119 |
+
--out runs/hf-recognizer/package \
|
| 120 |
+
--model-id ocrtech-hf-nep-eng-rec-v1 \
|
| 121 |
+
--base-model microsoft/trocr-small-printed \
|
| 122 |
+
--train-manifest data/splits/train.jsonl \
|
| 123 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 124 |
+
--training-summary runs/hf-recognizer/recognizer-training-summary.json \
|
| 125 |
+
--admission-validation-report outputs/validation-next-recognizer-hard50/validation.json
|
| 126 |
+
```
|
| 127 |
+
|
| 128 |
+
The HF recipe writes `runs/hf-recognizer/recognizer-training-summary.json` at
|
| 129 |
+
recipe-generation time. That summary records train/eval manifest hashes,
|
| 130 |
+
script coverage, config hash, and hyperparameters. Packaging should include it
|
| 131 |
+
just like the PaddleOCR path; otherwise the trained-recognizer provenance gate
|
| 132 |
+
will have weaker launch evidence.
|
| 133 |
+
|
| 134 |
+
This writes `runs/hf-recognizer/package/model-card.json` and records backend kwargs for loading the saved processor and model directory.
|
| 135 |
+
|
| 136 |
+
## Create Manually
|
| 137 |
+
|
| 138 |
+
```bash
|
| 139 |
+
ocrtech create-model-card \
|
| 140 |
+
--out runs/recognizer/model-card.json \
|
| 141 |
+
--model-id ocrtech-nep-eng-rec-v1 \
|
| 142 |
+
--backend paddleocr \
|
| 143 |
+
--base-model PaddlePaddle/devanagari_PP-OCRv3_mobile_rec \
|
| 144 |
+
--artifact runs/recognizer/checkpoints/best_accuracy.pdparams \
|
| 145 |
+
--artifact runs/recognizer/nep_eng_dict.txt \
|
| 146 |
+
--paddleocr-kwarg lang=devanagari \
|
| 147 |
+
--paddleocr-kwarg rec_model_dir=runs/recognizer/inference \
|
| 148 |
+
--paddleocr-kwarg rec_char_dict_path=runs/recognizer/nep_eng_dict.txt \
|
| 149 |
+
--train-manifest data/splits/train.jsonl \
|
| 150 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 151 |
+
--metrics-report outputs/claim-experiment/validation/validation.json
|
| 152 |
+
```
|
| 153 |
+
|
| 154 |
+
The backend kwargs are passed through to the selected runtime backend. For `paddleocr`, keep them explicit in the model card because PaddleOCR constructor names can change by version.
|
| 155 |
+
|
| 156 |
+
## Routed Candidates
|
| 157 |
+
|
| 158 |
+
For routed or thresholded candidates, do not hand-edit the threshold into a model card and call that evidence. Calibrate it from a separate experiment artifact:
|
| 159 |
+
|
| 160 |
+
```bash
|
| 161 |
+
ocrtech calibrate-quality-router \
|
| 162 |
+
--experiment-report outputs/tesseract-psm13-calibration/experiment.json \
|
| 163 |
+
--out outputs/quality-router-calibration \
|
| 164 |
+
--model-id ocrtech-tesseract-quality-router \
|
| 165 |
+
--primary-engine tesseract \
|
| 166 |
+
--secondary-engine tesseract \
|
| 167 |
+
--primary-engine-kwarg 'language=nep+eng' \
|
| 168 |
+
--primary-engine-kwarg 'psm=6' \
|
| 169 |
+
--secondary-engine-kwarg 'language=nep+eng' \
|
| 170 |
+
--secondary-engine-kwarg 'psm=13'
|
| 171 |
+
```
|
| 172 |
+
|
| 173 |
+
This writes:
|
| 174 |
+
|
| 175 |
+
- `calibration.json`
|
| 176 |
+
- `calibration.md`
|
| 177 |
+
- `model-card.json`
|
| 178 |
+
|
| 179 |
+
The generated model card records calibration provenance, including the calibration experiment, benchmark report, and calibration eval manifest hash. `preflight-claim` and `review-claim` use that provenance to reject tuned-on-eval routed candidates.
|
| 180 |
+
|
| 181 |
+
## Multi-Variant Tesseract Ensemble
|
| 182 |
+
|
| 183 |
+
For a richer Tesseract-only ensemble, calibrate several PSM variants from separate experiment artifacts:
|
| 184 |
+
|
| 185 |
+
```bash
|
| 186 |
+
ocrtech calibrate-tesseract-psm-ensemble \
|
| 187 |
+
--primary-experiment-report outputs/psm13-calibration/experiment.json \
|
| 188 |
+
--variant-experiment psm7=outputs/psm7-calibration/experiment.json \
|
| 189 |
+
--variant-experiment psm11=outputs/psm11-calibration/experiment.json \
|
| 190 |
+
--variant-experiment psm13=outputs/psm13-calibration/experiment.json \
|
| 191 |
+
--out outputs/psm-ensemble-calibration \
|
| 192 |
+
--model-id ocrtech-tesseract-psm-ensemble
|
| 193 |
+
```
|
| 194 |
+
|
| 195 |
+
This writes a `quality_ranked_ensemble` model card with:
|
| 196 |
+
|
| 197 |
+
- `primary_engine`
|
| 198 |
+
- `alternative_engines`
|
| 199 |
+
- per-variant `quality_bias`
|
| 200 |
+
- calibration provenance
|
| 201 |
+
|
| 202 |
+
Current evidence says this path is not promising on the mixed Nepali/English line pack: it improved on the calibration half and regressed on the untouched holdout. Keep it as a research tool, not as a recommended candidate.
|
| 203 |
+
|
| 204 |
+
## Script-Routed Tesseract + Surya
|
| 205 |
+
|
| 206 |
+
For a script-aware routed candidate, calibrate a threshold over the primary engine output text:
|
| 207 |
+
|
| 208 |
+
```bash
|
| 209 |
+
ocrtech calibrate-script-router \
|
| 210 |
+
--experiment-report outputs/surya-calibration/experiment.json \
|
| 211 |
+
--out outputs/script-router-calibration \
|
| 212 |
+
--model-id ocrtech-tesseract-surya-script-router \
|
| 213 |
+
--primary-engine tesseract \
|
| 214 |
+
--secondary-engine surya \
|
| 215 |
+
--primary-engine-kwarg 'language=nep+eng' \
|
| 216 |
+
--primary-engine-kwarg 'psm=6' \
|
| 217 |
+
--secondary-engine-kwarg 'mode=auto'
|
| 218 |
+
```
|
| 219 |
+
|
| 220 |
+
This writes a `script_select_composite` model card with:
|
| 221 |
+
|
| 222 |
+
- `primary_engine`
|
| 223 |
+
- `secondary_engine`
|
| 224 |
+
- `script`
|
| 225 |
+
- `primary_script_threshold`
|
| 226 |
+
- optional `routing_granularity` (`document` by default, `page` for per-page routing)
|
| 227 |
+
- optional `secondary_structure_backfill` to keep primary text selection while filling missing tables/figures from the secondary engine
|
| 228 |
+
- calibration provenance
|
| 229 |
+
|
| 230 |
+
Current live evidence on `k2` is better than the quality-score-only Tesseract router: calibration on the `89`-sample split selected `primary_script_threshold=0.4`, and the untouched `91`-sample holdout improved overall CER from `0.342234` to `0.313084` with `9` wins, `0` losses, and `82` ties against raw Tesseract.
|
| 231 |
+
|
| 232 |
+
An exploratory `routing_granularity=page` run on that same holdout landed on the same numbers. That does not mean page routing is useless; it means this particular holdout is mostly single-page line-crop material. Page routing needs a true multi-page document eval pack before it can show a different outcome.
|
| 233 |
+
|
| 234 |
+
On the newer rendered multi-page image-pack smoke, `secondary_structure_backfill=true` turned out to be useful. The document-level router still kept Tesseract text on the multi-page report page, but it backfilled Surya's table structure on that page and moved `table_cell_f1` from `0.0` to `1.0` on the rendered smoke pack without changing CER.
|
| 235 |
+
|
| 236 |
+
That behavior survived on the broader rendered claim-ready image pack as well. The calibrated backfill model card `ocrtech-tesseract-surya-script-router-backfill-calibrated` kept `routing_granularity=document`, `secondary_structure_backfill=true`, and `primary_script_threshold=0.4`. On the `25`-sample rendered pack it still beat raw Tesseract on overall CER (`0.248654` vs `0.689214`) and kept a large structure advantage (`table_cell_f1` `0.955556` vs `0.0`). The weak spots are still visible, which is exactly why this should be treated as an engineering baseline rather than a claim candidate: `form` CER regressed slightly and aggregate reading-order accuracy on that pack is still lower than Tesseract.
|
| 237 |
+
|
| 238 |
+
## Table Cell Refinement Composite
|
| 239 |
+
|
| 240 |
+
`table_cell_refine_composite` uses one engine for document/table structure and a second engine for crop OCR over detected table cells:
|
| 241 |
+
|
| 242 |
+
```bash
|
| 243 |
+
ocrtech create-model-card \
|
| 244 |
+
--out outputs/table-cell-refine-empty-v1/model-card.json \
|
| 245 |
+
--model-id ocrtech-surya-tesseract-table-cell-refine-empty-v1 \
|
| 246 |
+
--backend table_cell_refine_composite \
|
| 247 |
+
--base-model surya+tesseract \
|
| 248 |
+
--backend-kwarg structure_engine=surya \
|
| 249 |
+
--backend-kwarg crop_engine=tesseract \
|
| 250 |
+
--backend-kwarg crop_padding=4 \
|
| 251 |
+
--backend-kwarg replacement_policy=empty
|
| 252 |
+
```
|
| 253 |
+
|
| 254 |
+
Policies:
|
| 255 |
+
|
| 256 |
+
- `replacement_policy=empty`: only OCR empty table cells. This is the safe default.
|
| 257 |
+
- `replacement_policy=always`: reread every detected table cell. This is experimental and can damage already-correct Surya cells.
|
| 258 |
+
- `deduplicate_tables=true`: drop overlapping duplicate tables after refinement. The parser also avoids re-inferring a table from text lines when an engine already supplied an overlapping structured table.
|
| 259 |
+
|
| 260 |
+
Current hard15 evidence says the idea is not enough:
|
| 261 |
+
|
| 262 |
+
- unsafe `always` policy tied Surya on text CER but regressed table-cell F1 from `0.979167` to `0.593750`
|
| 263 |
+
- safe `empty` policy tied Surya exactly on hard15 CER, table-cell F1, and reading order
|
| 264 |
+
- safe `empty` plus table de-duplication also tied Surya exactly on hard15 CER, table-cell F1, and reading order
|
| 265 |
+
- hard15 validation status for the safe policies is `not_validated`
|
| 266 |
+
|
| 267 |
+
So this backend is useful infrastructure for future missing-cell cases, not a claim candidate yet.
|
| 268 |
+
|
| 269 |
+
## Figure Caption Refinement Composite
|
| 270 |
+
|
| 271 |
+
`figure_caption_refine_composite` keeps a primary document engine for layout and OCR, then uses a second engine only to replace nearby figure-caption lines when the replacement overlaps spatially and has a better caption-prefix score:
|
| 272 |
+
|
| 273 |
+
```bash
|
| 274 |
+
ocrtech create-model-card \
|
| 275 |
+
--out outputs/figure-caption-refine-v1/model-card.json \
|
| 276 |
+
--model-id ocrtech-surya-tesseract-figure-caption-refine-v1 \
|
| 277 |
+
--backend figure_caption_refine_composite \
|
| 278 |
+
--base-model surya+tesseract \
|
| 279 |
+
--backend-kwarg primary_engine=surya \
|
| 280 |
+
--backend-kwarg caption_engine=tesseract \
|
| 281 |
+
--backend-kwarg min_caption_iou=0.5 \
|
| 282 |
+
--backend-kwarg min_score_delta=1
|
| 283 |
+
```
|
| 284 |
+
|
| 285 |
+
The replacement preserves the primary engine's bbox and layout ordering. Only the caption text/confidence changes. This avoids the skew failure where a secondary OCR bbox overlaps the figure marker and prevents caption attachment.
|
| 286 |
+
|
| 287 |
+
Current hard15 evidence:
|
| 288 |
+
|
| 289 |
+
- figure-caption CER improved over Surya from `0.092323` to `0.052478`
|
| 290 |
+
- figure detection F1 stayed `1.000000`
|
| 291 |
+
- table-cell F1 stayed `0.979167`
|
| 292 |
+
- reading-order pair accuracy stayed `0.520000`
|
| 293 |
+
- hard15 validation status is still `not_validated` because broad CER/table gates do not pass
|
| 294 |
+
|
| 295 |
+
An opt-in `correct_structural_labels=true` branch adds edit-distance guarded correction for common Nepali document-role labels such as `वार्षिक प्रतिवेदन`, `चित्र`, and `निष्कर्ष`. On hard15 this stronger branch improved:
|
| 296 |
+
|
| 297 |
+
- overall hard_eval CER over Surya from `0.236839` to `0.204610`
|
| 298 |
+
- figure CER from `0.091927` to `0.022866`
|
| 299 |
+
- figure-caption CER from `0.092323` to `0.008746`
|
| 300 |
+
|
| 301 |
+
Validation still reports `not_validated` because table CER and table-cell F1 tie Surya, but the overall CER gate and both figure gates pass.
|
| 302 |
+
|
| 303 |
+
A stronger auditable variant keeps the same backend but enables:
|
| 304 |
+
|
| 305 |
+
- `correct_structural_labels=true`, now covering `वार्षिक प्रतिवेदन`, `दैनिक हिसाब`, `चित्र`, and `निष्कर्ष`
|
| 306 |
+
- `table_name_lexicon_path=configs/nepali-table-name-lexicon-v1.txt`
|
| 307 |
+
- `table_name_max_distance=2`
|
| 308 |
+
|
| 309 |
+
The table-name lexicon correction is intentionally narrow: it only applies to cells under a header close to `नाम`, and only when there is a unique nearest lexicon entry within the configured edit-distance threshold. The current lexicon is tiny and curated, so paper claims must describe this as lexicon-assisted structured extraction, not a general language model.
|
| 310 |
+
|
| 311 |
+
The public-table variant also enables:
|
| 312 |
+
|
| 313 |
+
- `normalize_table_grids=true`
|
| 314 |
+
|
| 315 |
+
This removes empty columns, separator-only columns such as `—`, and exact
|
| 316 |
+
duplicate adjacent columns from predicted tables. The rule is runtime-only and
|
| 317 |
+
reference-free. It improved the 5-page public OmniDocBench smoke over raw Surya
|
| 318 |
+
on table metrics, but the public validation still remains `not_validated`
|
| 319 |
+
because the sample count and confidence gates are not met.
|
| 320 |
+
|
| 321 |
+
The later `normalize-rows-v3` public-table model card keeps the same option and
|
| 322 |
+
adds sparse bullet row-fragment merging inside the grid normalizer. This is also
|
| 323 |
+
runtime-only and reference-free. On the same 5-page public OmniDocBench smoke it
|
| 324 |
+
raises table metrics further (`table_cell_f1` `0.564335`,
|
| 325 |
+
`table_teds_structure` `0.942836`, `table_teds_text` `0.675633`) and changes the
|
| 326 |
+
`omnidocbench-00208...` fragmented table from `16x5` to `3x5`. It is not a
|
| 327 |
+
validated model: text CER is slightly worse than Surya on that smoke, and the
|
| 328 |
+
public validation gates still fail on sample count and bootstrap CI.
|
| 329 |
+
|
| 330 |
+
On the larger public OmniDocBench `table40` slice, the same model has higher
|
| 331 |
+
mean table metrics than Surya but does not validate:
|
| 332 |
+
|
| 333 |
+
- CER `0.713326` versus Surya `0.713960`
|
| 334 |
+
- table-cell F1 `0.323122` versus Surya `0.280062`, but `7` wins / `9` losses /
|
| 335 |
+
`24` ties and CI low `-0.019152`
|
| 336 |
+
- table TEDS structure `0.520959` versus Surya `0.503603`, but CI low
|
| 337 |
+
`-0.038381`
|
| 338 |
+
- table TEDS text `0.361666` versus Surya `0.358057`, below the configured
|
| 339 |
+
margin
|
| 340 |
+
- cell exactness `3440/10314` versus Surya `3421/10314`
|
| 341 |
+
|
| 342 |
+
This points to the next model requirement: better table segmentation and
|
| 343 |
+
large/wide-table handling. The current grid postprocessors are useful but too
|
| 344 |
+
local to support a public SOTA claim.
|
| 345 |
+
|
| 346 |
+
The `normalize-width-guard-v4` model card adds one guard to the same runtime
|
| 347 |
+
postprocessor: tables wider than `12` predicted columns skip grid normalization.
|
| 348 |
+
This directly addresses public table40 failures where v3 merged `15` sparse rows
|
| 349 |
+
or dropped `26` columns in wide tables. On full public table40, v4 improves over
|
| 350 |
+
v3 and passes two configured public table gates against Surya:
|
| 351 |
+
|
| 352 |
+
- CER `0.707864` versus Surya `0.713960`
|
| 353 |
+
- table-cell F1 `0.339319` versus Surya `0.280062`, CI low `0.001433`
|
| 354 |
+
- table TEDS structure `0.545375` versus Surya `0.503603`, CI low `0.004790`
|
| 355 |
+
- table TEDS text `0.386013` versus Surya `0.358057`, but CI low `-0.000220`
|
| 356 |
+
- cell exactness `3453/10314` versus Surya `3421/10314`
|
| 357 |
+
|
| 358 |
+
This is a better public table parser candidate than v3, but still not a SOTA
|
| 359 |
+
engine: the text-sensitive table gate and broad public document gate remain open.
|
| 360 |
+
|
| 361 |
+
On hard15 v3, this model reaches:
|
| 362 |
+
|
| 363 |
+
- overall CER `0.010671` versus Surya `0.236839`
|
| 364 |
+
- table-slice CER `0.000000` versus Surya `0.363636`
|
| 365 |
+
- figure-caption CER `0.008746` versus Surya `0.092323`
|
| 366 |
+
- table-cell F1 `1.000000` versus Surya `0.979167`
|
| 367 |
+
- table exact cells `96/96` versus Surya `94/96`
|
| 368 |
+
|
| 369 |
+
Configured validation is still `not_validated` because the table-cell F1 bootstrap CI low is `0.000000` on the tiny 8-page table slice. This is a sample-size limitation of the current gate, not permission to claim table SOTA. Build a larger independent table pack before making that claim.
|
| 370 |
+
|
| 371 |
+
The table-name lexicon matcher now also supports suffix-preserving generated IDs such as `करिण-302` -> `किरण-302`. The correction still only fires in columns whose header is close to `नाम`, and still requires a unique nearest base-name match within the configured edit distance.
|
| 372 |
+
|
| 373 |
+
On the larger `60`-sample table-ledger pack, this same model passes the targeted Surya table validation:
|
| 374 |
+
|
| 375 |
+
- table-cell F1: candidate `1.000000`, Surya `0.979167`, CI low `0.012500`
|
| 376 |
+
- table exact match: candidate `1.000000`, Surya `0.750000`, CI low `0.150000`
|
| 377 |
+
- table structure similarity: candidate `1.000000`, Surya `1.000000`, non-regression guard passes
|
| 378 |
+
- cell exactness: candidate `720/720`, Surya `705/720`
|
| 379 |
+
|
| 380 |
+
This is valid evidence for lexicon-assisted table-ledger cell-content extraction with grid-structure non-regression. It is not a general OCR SOTA result, and it does not prove improvement on complex merged-cell tables.
|
| 381 |
+
|
| 382 |
+
On the first `complex-table-ledger` run, the same candidate improved cell-content F1 over Surya (`0.928333` vs `0.915000`) after extending name correction below grouped headers, but it did not improve merged-header structure.
|
| 383 |
+
|
| 384 |
+
The later span-inference parser branch adds conservative repeated-header colspan inference. On the same generated complex-table pack, it reaches:
|
| 385 |
+
|
| 386 |
+
- table-cell F1 `0.995000` versus Surya `0.915000`
|
| 387 |
+
- table exact match `0.925000` versus Surya `0.000000`
|
| 388 |
+
- span-aware table structure similarity `1.000000` versus Surya `0.875000`
|
| 389 |
+
- TEDS-style table tree structure similarity `1.000000` versus Surya `0.904762`
|
| 390 |
+
- TEDS-style text-sensitive table tree similarity `0.996250` versus Surya `0.891667`
|
| 391 |
+
- cell exactness `597/600` versus Surya `549/600`
|
| 392 |
+
|
| 393 |
+
This is a parser/postprocessing improvement for a generated repeated-header pattern. It is not a trained merged-cell model and does not support a broad complex-table SOTA claim without real scanned tables and varied span structures.
|
| 394 |
+
|
| 395 |
+
The public OmniDocBench table40 width-guard model card is:
|
| 396 |
+
|
| 397 |
+
- `outputs/figure-caption-table-lexicon-normalize-width-guard-v4/model-card.json`
|
| 398 |
+
- model id: `ocrtech-surya-tesseract-figure-table-lexicon-normalize-width-guard-v4`
|
| 399 |
+
- audit: `outputs/audit-figure-caption-table-lexicon-normalize-width-guard-v4-model/model-audit.json`
|
| 400 |
+
|
| 401 |
+
This candidate keeps the normalized table-grid postprocessors but skips grid
|
| 402 |
+
normalization when predicted table width is greater than `12`. On the public
|
| 403 |
+
OmniDocBench table40 combined run, it improves over Surya on table-cell F1
|
| 404 |
+
(`0.339319` vs `0.280062`) and structure-only TEDS (`0.545375` vs `0.503603`),
|
| 405 |
+
but the run remains `not_validated` because the public document CER gate is
|
| 406 |
+
underpowered and table TEDS text is borderline in the separate-run comparison.
|
| 407 |
+
|
| 408 |
+
For postprocessor-only evidence, use
|
| 409 |
+
`outputs/experiment-public-omnidocbench-table40-derived-postprocess-v1/benchmark/report.json`.
|
| 410 |
+
That derived benchmark applies the table postprocessors to the exact same Surya
|
| 411 |
+
`document.json` outputs, so CER stays identical while table metrics isolate the
|
| 412 |
+
postprocessor contribution. It passes the configured table-cell F1,
|
| 413 |
+
structure-only TEDS, and text-sensitive TEDS gates on `40` pages, but it is a
|
| 414 |
+
validation lens rather than a deployable model card.
|
| 415 |
+
|
| 416 |
+
## Audit
|
| 417 |
+
|
| 418 |
+
```bash
|
| 419 |
+
ocrtech audit-model runs/recognizer/package/model-card.json \
|
| 420 |
+
--out outputs/model-audit
|
| 421 |
+
```
|
| 422 |
+
|
| 423 |
+
The audit verifies:
|
| 424 |
+
|
| 425 |
+
- required artifact paths exist
|
| 426 |
+
- recorded SHA-256 values match for file artifacts
|
| 427 |
+
- backend is supported
|
| 428 |
+
- the card shape is valid
|
| 429 |
+
|
| 430 |
+
`audit-model` does not by itself prove claim readiness. For `quality_select_composite`, `quality_ranked_ensemble`, and `script_select_composite`, the calibration provenance must also be separate from the final claim eval manifest.
|
| 431 |
+
|
| 432 |
+
For recognizer branch admission, attach the hard-gate validation report when
|
| 433 |
+
creating or packaging a model card:
|
| 434 |
+
|
| 435 |
+
```bash
|
| 436 |
+
ocrtech create-model-card \
|
| 437 |
+
--out runs/recognizer/package/model-card.json \
|
| 438 |
+
--model-id ocrtech-next-recognizer \
|
| 439 |
+
--backend paddleocr \
|
| 440 |
+
--artifact runs/recognizer/export/inference/inference.pdiparams \
|
| 441 |
+
--train-manifest data/splits/train.jsonl \
|
| 442 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 443 |
+
--admission-validation-report outputs/validation-next-recognizer-hard50/validation.json
|
| 444 |
+
```
|
| 445 |
+
|
| 446 |
+
Claim-grade preflight can then require the admission gate with
|
| 447 |
+
`--require-model-admission`. Admission reports must include quantitative
|
| 448 |
+
passing gate evidence; a bare `claim_status=validated` report is rejected.
|
| 449 |
+
|
| 450 |
+
## Parse With Candidate
|
| 451 |
+
|
| 452 |
+
```bash
|
| 453 |
+
ocrtech parse-doc page.png \
|
| 454 |
+
--engine candidate \
|
| 455 |
+
--model-config runs/recognizer/package/model-card.json \
|
| 456 |
+
--fallback-model-config runs/vlm-fallback/package/model-card.json \
|
| 457 |
+
--out outputs/page
|
| 458 |
+
```
|
| 459 |
+
|
| 460 |
+
When a fallback engine or fallback model card is configured, the parser does not
|
| 461 |
+
silently swap engines. It records `metadata.fallback` in `document.json` with
|
| 462 |
+
the primary engine, fallback engine/model config, average primary confidence,
|
| 463 |
+
threshold, trigger reason, and outcome. This is required for paper evidence: a
|
| 464 |
+
selective VLM/document fallback must report when it actually fired and whether
|
| 465 |
+
it succeeded.
|
| 466 |
+
|
| 467 |
+
## Benchmark With Candidate
|
| 468 |
+
|
| 469 |
+
```bash
|
| 470 |
+
ocrtech run-experiment \
|
| 471 |
+
--eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
|
| 472 |
+
--references-dir data/eval-pack/references \
|
| 473 |
+
--candidate-model-config runs/recognizer/package/model-card.json \
|
| 474 |
+
--fallback-model-config runs/vlm-fallback/package/model-card.json \
|
| 475 |
+
--low-confidence-threshold 0.80 \
|
| 476 |
+
--baselines candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
|
| 477 |
+
--validation-config validation.json \
|
| 478 |
+
--train-manifest data/splits/train.jsonl \
|
| 479 |
+
--out outputs/claim-experiment \
|
| 480 |
+
--require-validated
|
| 481 |
+
```
|
| 482 |
+
|
| 483 |
+
If `--candidate-model-config` is omitted, `ours` uses the local `auto` engine for smoke tests. That is not valid SOTA evidence.
|
| 484 |
+
|
| 485 |
+
Benchmark and experiment fallback flags apply only to the candidate/ours run.
|
| 486 |
+
They are meant to measure the deployed hybrid engine, not to contaminate
|
| 487 |
+
baseline rows. `experiment.json` records `fallback_engine`,
|
| 488 |
+
`fallback_model_config`, `fallback_model_config_sha256`, and
|
| 489 |
+
`low_confidence_threshold` so claim reviewers can tell which routing policy was
|
| 490 |
+
actually evaluated.
|
docs/NEPAL_MULTILINGUAL_OCR_GOAL.md
ADDED
|
@@ -0,0 +1,371 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Nepal Multilingual Document OCR Goal
|
| 2 |
+
|
| 3 |
+
This is the research target after the steering decision on 2026-06-16. The
|
| 4 |
+
project is now a PaddleOCR-centered multilingual document OCR system for
|
| 5 |
+
languages used in Nepal, not a narrow Nepali/English OCR experiment.
|
| 6 |
+
|
| 7 |
+
## Goal
|
| 8 |
+
|
| 9 |
+
Build and validate a PaddleOCR-based document OCR engine for at least ten
|
| 10 |
+
languages spoken in Nepal, with Limbu first, that supports:
|
| 11 |
+
|
| 12 |
+
- page/PDF/image parsing
|
| 13 |
+
- text OCR
|
| 14 |
+
- reading order
|
| 15 |
+
- table reconstruction
|
| 16 |
+
- figure and caption extraction
|
| 17 |
+
- Markdown export
|
| 18 |
+
- structured JSON export
|
| 19 |
+
- selective PaddleOCR-VL/VLM fallback for hard pages
|
| 20 |
+
|
| 21 |
+
The claim is valid only if the engine beats strong baselines on predefined real
|
| 22 |
+
and held-out evaluations. Synthetic-only results are not enough.
|
| 23 |
+
|
| 24 |
+
## Research Claim Shape
|
| 25 |
+
|
| 26 |
+
The paper should not claim "best OCR engine overall." The defensible claim is:
|
| 27 |
+
|
| 28 |
+
> A script-aware PaddleOCR adaptation framework can substantially improve
|
| 29 |
+
> document OCR and structured extraction for low-resource Nepal languages,
|
| 30 |
+
> including scripts not covered well by stock OCR systems, while preserving
|
| 31 |
+
> Nepali/English performance and using selective VLM fallback only when the
|
| 32 |
+
> local pipeline is uncertain.
|
| 33 |
+
|
| 34 |
+
The core contribution is the system and validation recipe:
|
| 35 |
+
|
| 36 |
+
1. Language/script registry for Nepal document OCR.
|
| 37 |
+
2. Gorkhapatra/Naya Nepal real-publication validation pack.
|
| 38 |
+
3. Unicode/font/renderability audit before synthesis.
|
| 39 |
+
4. Real-text synthetic data generation for lines, tables, captions, and pages.
|
| 40 |
+
5. PaddleOCR recognizer fine-tuning by script family.
|
| 41 |
+
6. PP-Structure-style document extraction.
|
| 42 |
+
7. Claim review that rejects weak SOTA statements.
|
| 43 |
+
|
| 44 |
+
## Primary Engine Direction
|
| 45 |
+
|
| 46 |
+
Use PaddleOCR as the engine family:
|
| 47 |
+
|
| 48 |
+
- **PP-OCRv6** for the modern small/local OCR baseline and fine-tuning target.
|
| 49 |
+
- **PP-StructureV3** for layout, reading order, tables, and document structure.
|
| 50 |
+
- **PaddleOCR-VL-1.6** as fallback/teacher for charts, figures, hard layouts,
|
| 51 |
+
seals, and low-confidence pages.
|
| 52 |
+
|
| 53 |
+
Do not route everything to a VLM and call that the engine. The local PaddleOCR
|
| 54 |
+
path must be the default path; fallback usage must be measured and reported.
|
| 55 |
+
|
| 56 |
+
## First Ten Language Targets
|
| 57 |
+
|
| 58 |
+
The first target set should be driven by actual Gorkhapatra/Naya Nepal
|
| 59 |
+
availability plus local corpus availability. OCR support is tracked by script,
|
| 60 |
+
Unicode, fonts, and evaluation data, not by language name alone.
|
| 61 |
+
|
| 62 |
+
| Priority | Language | First script target | Why first | Notes |
|
| 63 |
+
| --- | --- | --- | --- | --- |
|
| 64 |
+
| 1 | Limbu/Yakthung | Limbu/Sirijonga, U+1900..U+194F | Has own Unicode block and local Limbu resources | First non-Devanagari research target |
|
| 65 |
+
| 2 | Nepali | Devanagari | Core language and baseline | Must not regress |
|
| 66 |
+
| 3 | English | Latin | Mixed documents and forms | Must not regress |
|
| 67 |
+
| 4 | Tamang | Devanagari first; Tamyig audit later | Local NepTam corpus and dictionary | Start with Devanagari Tamang |
|
| 68 |
+
| 5 | Magar | Devanagari first | Gorkhapatra item confirmed; local dictionary | Own/community script requires separate audit |
|
| 69 |
+
| 6 | Wambule | Devanagari first | Gorkhapatra item confirmed | Rai-family validation candidate |
|
| 70 |
+
| 7 | Bantawa/Rai | Devanagari or Kirat Rai depending source | Gorkhapatra item confirmed; Rai resources on remote/local projects | Kirat Rai Unicode support is newer; font audit required |
|
| 71 |
+
| 8 | Kulung | Devanagari first | Gorkhapatra item confirmed | Rai-family validation candidate |
|
| 72 |
+
| 9 | Yakkha | Devanagari first | Gorkhapatra item confirmed | Rai-family validation candidate |
|
| 73 |
+
| 10 | Tharu/Rana Tharu | Devanagari first | Gorkhapatra item confirmed | Multiple varieties; keep labels explicit |
|
| 74 |
+
|
| 75 |
+
Additional Gorkhapatra-visible candidates include Sanskrit, Magahi, Bajjika,
|
| 76 |
+
Sunuwar, Danuwar, Majhi, Lhomi, Bhojpuri, Kisan, Thami, Sherpa, Achhami, Bote,
|
| 77 |
+
Rajbanshi, Hayu, Santhali, Dhimal, Bhote, Uraon, Kumal, Gurung, Khash Jumli,
|
| 78 |
+
and Meche. These should enter the registry after script/corpus/font checks.
|
| 79 |
+
|
| 80 |
+
## Source Findings So Far
|
| 81 |
+
|
| 82 |
+
### Gorkhapatra Naya Nepal
|
| 83 |
+
|
| 84 |
+
The category page is useful for language-specific validation:
|
| 85 |
+
|
| 86 |
+
- `https://gorkhapatraonline.com/categories/new-nepal`
|
| 87 |
+
- `https://gorkhapatraonline.com/categories/new-nepal?page=2`
|
| 88 |
+
|
| 89 |
+
Confirmed visible language-specific entries include Magar, Wambule, Rana Tharu,
|
| 90 |
+
Magahi, Bajjika, Sunuwar, Danuwar, Majhi, Lhomi, Bhojpuri, Kisan, Thami,
|
| 91 |
+
Sherpa, Achhami, Bote, Rajbanshi, Hayu, Santhali, Kulung, Yakkha, Meche, Khash
|
| 92 |
+
Jumli, Dhimal, Tharu, Bhote, Uraon, Bantawa, Kumal, Gurung, and others.
|
| 93 |
+
|
| 94 |
+
The blog/category pages expose image URLs and article URLs. They are useful for
|
| 95 |
+
real-world OCR samples, but they should not be trusted as clean text references
|
| 96 |
+
without manual or tool-assisted verification.
|
| 97 |
+
|
| 98 |
+
### Gorkhapatra Epaper
|
| 99 |
+
|
| 100 |
+
The epaper list endpoint exposes viewer links:
|
| 101 |
+
|
| 102 |
+
```text
|
| 103 |
+
https://epaper.gorkhapatraonline.com/epaper-list?pdfdaterange1=2026-06-09&pdfdaterange2=2026-06-16&slug=gorkhapatra
|
| 104 |
+
```
|
| 105 |
+
|
| 106 |
+
The visible links look like:
|
| 107 |
+
|
| 108 |
+
```text
|
| 109 |
+
https://epaper.gorkhapatraonline.com/pdf?file=/uploads/file/2026/6/gorkhapatra/2026-06-14-11-49-50-2083-03-01.pdf
|
| 110 |
+
```
|
| 111 |
+
|
| 112 |
+
That `pdf?...` URL returns a PDF.js viewer shell. The real PDF is downloaded by
|
| 113 |
+
fetching the file parameter directly:
|
| 114 |
+
|
| 115 |
+
```text
|
| 116 |
+
https://epaper.gorkhapatraonline.com/uploads/file/2026/6/gorkhapatra/2026-06-14-11-49-50-2083-03-01.pdf
|
| 117 |
+
```
|
| 118 |
+
|
| 119 |
+
On `k2`, this was verified as a real PDF:
|
| 120 |
+
|
| 121 |
+
```text
|
| 122 |
+
/Users/k2/ocr-tech/outputs/gorkhapatra-source-audit/sample.pdf
|
| 123 |
+
PDF document, version 1.6
|
| 124 |
+
size: 48M
|
| 125 |
+
```
|
| 126 |
+
|
| 127 |
+
The `k2` audit produced:
|
| 128 |
+
|
| 129 |
+
```text
|
| 130 |
+
/Users/k2/ocr-tech/outputs/gorkhapatra-source-audit/audit.txt
|
| 131 |
+
/Users/k2/ocr-tech/outputs/gorkhapatra-source-audit/new-nepal-language-items.txt
|
| 132 |
+
/Users/k2/ocr-tech/outputs/gorkhapatra-source-audit/epaper-assets.txt
|
| 133 |
+
/Users/k2/ocr-tech/outputs/gorkhapatra-source-audit/sample.pdf
|
| 134 |
+
```
|
| 135 |
+
|
| 136 |
+
`k2` does not currently expose `pdfinfo` or `pdftotext`. `pymupdf` was installed
|
| 137 |
+
into `/Users/k2/ocr-tech/.venv` on 2026-06-16 as the immediate PDF audit tool.
|
| 138 |
+
The sample epaper PDF opens successfully with PyMuPDF:
|
| 139 |
+
|
| 140 |
+
```text
|
| 141 |
+
page_count: 26
|
| 142 |
+
total extracted text chars: 309924
|
| 143 |
+
preview image: outputs/gorkhapatra-source-audit/sample-page-1-preview.jpg
|
| 144 |
+
audit: outputs/gorkhapatra-source-audit/sample-pymupdf-audit.json
|
| 145 |
+
text: outputs/gorkhapatra-source-audit/sample-pymupdf-text.txt
|
| 146 |
+
```
|
| 147 |
+
|
| 148 |
+
The extracted text layer is not reliable reference text. The first-page preview
|
| 149 |
+
contains mojibake/transliterated-looking output for Nepali content rather than
|
| 150 |
+
clean Unicode Devanagari. Therefore the epaper PDFs are useful as real page
|
| 151 |
+
images/layout sources, but their embedded text layer must be audited per file
|
| 152 |
+
and must not be treated as ground truth without verification.
|
| 153 |
+
|
| 154 |
+
### Local And Remote Corpus Hints
|
| 155 |
+
|
| 156 |
+
Local machine:
|
| 157 |
+
|
| 158 |
+
- `~/github/llm/ltk/tamang`: NepTam train/test, NepTam80K, dictionary, TTS
|
| 159 |
+
lexicon.
|
| 160 |
+
- `~/github/llm/ltk/limbu`: Limbu dictionary, Pangloss IGT, Limbu G2P.
|
| 161 |
+
- `~/github/llm/ltk/magar`: Western Magar dictionary and resource report.
|
| 162 |
+
|
| 163 |
+
Remote `cdjk@100.110.148.43`:
|
| 164 |
+
|
| 165 |
+
- `/Volumes/TRANSCEND` has about 2.0 TiB free.
|
| 166 |
+
- `~/github/llm` contains additional language projects:
|
| 167 |
+
`dhimal-toolkit`, `gurung-language-toolkit`, `limbu-speech-toolkit`,
|
| 168 |
+
`magar-toolkit`, `rai-speech-toolkit`, `nllb-better`, `nepa-newa-text-frontend`,
|
| 169 |
+
`new-place`, and `loksewa.zip`.
|
| 170 |
+
- `nllb-better` has Rai-family generated/validation artifacts and a language
|
| 171 |
+
registry that should be inspected before building the OCR registry.
|
| 172 |
+
|
| 173 |
+
## Data Plan
|
| 174 |
+
|
| 175 |
+
### Real Validation Data
|
| 176 |
+
|
| 177 |
+
Use Gorkhapatra/Naya Nepal as the central real-publication validation source.
|
| 178 |
+
|
| 179 |
+
Create a scraper that emits:
|
| 180 |
+
|
| 181 |
+
- article URL
|
| 182 |
+
- article title
|
| 183 |
+
- language label extracted from title
|
| 184 |
+
- article image URLs
|
| 185 |
+
- epaper date
|
| 186 |
+
- epaper PDF URL
|
| 187 |
+
- direct PDF file URL
|
| 188 |
+
- image/PDF hashes
|
| 189 |
+
- source timestamp
|
| 190 |
+
- license/terms note
|
| 191 |
+
- manual-review status
|
| 192 |
+
|
| 193 |
+
Then create a hand-labeled validation pack:
|
| 194 |
+
|
| 195 |
+
- at least 10 languages
|
| 196 |
+
- at least 20 pages or page crops per language for v1 if available
|
| 197 |
+
- Limbu set first
|
| 198 |
+
- no synthetic samples in the final real validation pack
|
| 199 |
+
- references include text, line boxes, reading order, table cells, captions, and
|
| 200 |
+
Markdown where applicable
|
| 201 |
+
|
| 202 |
+
### Synthetic Training Data
|
| 203 |
+
|
| 204 |
+
Use real text when synthesizing:
|
| 205 |
+
|
| 206 |
+
- Limbu dictionary/Pangloss/IGT text first.
|
| 207 |
+
- Tamang NepTam and dictionary text.
|
| 208 |
+
- Devanagari text from Gorkhapatra/Loksewa/government-like documents where
|
| 209 |
+
licensing permits.
|
| 210 |
+
- Rai-family text from verified local/remote corpora.
|
| 211 |
+
|
| 212 |
+
Synthetic generation must produce:
|
| 213 |
+
|
| 214 |
+
- line crops
|
| 215 |
+
- word crops
|
| 216 |
+
- table cell crops
|
| 217 |
+
- captions
|
| 218 |
+
- full pages
|
| 219 |
+
- degraded scan/photo variants
|
| 220 |
+
- PaddleOCR recognition labels
|
| 221 |
+
- document JSON/Markdown references
|
| 222 |
+
|
| 223 |
+
Every synthetic sample must carry:
|
| 224 |
+
|
| 225 |
+
- language
|
| 226 |
+
- script
|
| 227 |
+
- Unicode ranges observed
|
| 228 |
+
- font
|
| 229 |
+
- source text hash
|
| 230 |
+
- rendered image hash
|
| 231 |
+
- template id
|
| 232 |
+
- degradation id
|
| 233 |
+
- train/eval split id
|
| 234 |
+
|
| 235 |
+
### Handwriting Synthesis
|
| 236 |
+
|
| 237 |
+
Handwriting is not a v1 blocker. It can be a later ablation.
|
| 238 |
+
|
| 239 |
+
Possible direction:
|
| 240 |
+
|
| 241 |
+
- Use available fonts and handwritten-style fonts for cheap diversity.
|
| 242 |
+
- Explore diffusion/DM-based handwriting generation only after print OCR is
|
| 243 |
+
stable.
|
| 244 |
+
- For Limbu, do not synthesize handwriting until we have real handwritten or
|
| 245 |
+
community-reviewed glyph examples. Otherwise we risk generating plausible but
|
| 246 |
+
wrong orthography.
|
| 247 |
+
|
| 248 |
+
If we explore One-DM or related diffusion methods, it should be framed as
|
| 249 |
+
optional augmentation, not the main claim.
|
| 250 |
+
|
| 251 |
+
## Validation Gates
|
| 252 |
+
|
| 253 |
+
### Gate 0: Source Audit
|
| 254 |
+
|
| 255 |
+
Pass only if:
|
| 256 |
+
|
| 257 |
+
- Gorkhapatra article image URLs are downloadable.
|
| 258 |
+
- Epaper direct PDFs are downloadable.
|
| 259 |
+
- PDF text/image extraction path is implemented and audited.
|
| 260 |
+
- Every source artifact is hashed.
|
| 261 |
+
- Source license/terms risk is recorded.
|
| 262 |
+
|
| 263 |
+
### Gate 1: Language Registry
|
| 264 |
+
|
| 265 |
+
Pass only if every target language has:
|
| 266 |
+
|
| 267 |
+
- language name
|
| 268 |
+
- ISO code if known
|
| 269 |
+
- script name
|
| 270 |
+
- Unicode range
|
| 271 |
+
- font list
|
| 272 |
+
- corpus source
|
| 273 |
+
- renderability audit
|
| 274 |
+
- normalization policy
|
| 275 |
+
- PaddleOCR dictionary path
|
| 276 |
+
- evaluation availability
|
| 277 |
+
|
| 278 |
+
### Gate 2: Limbu First
|
| 279 |
+
|
| 280 |
+
Pass only if:
|
| 281 |
+
|
| 282 |
+
- Limbu Unicode rendering works with selected fonts.
|
| 283 |
+
- Limbu PaddleOCR dictionary exists.
|
| 284 |
+
- Limbu synthetic crop/page generator emits valid labels.
|
| 285 |
+
- Limbu held-out real or manually reviewed eval exists.
|
| 286 |
+
- Stock baselines are recorded before fine-tuning.
|
| 287 |
+
|
| 288 |
+
### Gate 3: PaddleOCR Baselines
|
| 289 |
+
|
| 290 |
+
Run before any claim:
|
| 291 |
+
|
| 292 |
+
- stock PP-OCRv6
|
| 293 |
+
- stock PP-OCRv5 if available
|
| 294 |
+
- Tesseract where traineddata exists
|
| 295 |
+
- Surya
|
| 296 |
+
- GLM-OCR
|
| 297 |
+
- PaddleOCR-VL-1.6
|
| 298 |
+
|
| 299 |
+
### Gate 4: Fine-Tuned Candidate
|
| 300 |
+
|
| 301 |
+
The candidate must beat baselines on:
|
| 302 |
+
|
| 303 |
+
- CER/WER per language
|
| 304 |
+
- strict and normalized Unicode CER
|
| 305 |
+
- exact-line accuracy
|
| 306 |
+
- reading order
|
| 307 |
+
- table text and structure
|
| 308 |
+
- figure/caption detection
|
| 309 |
+
- Markdown reconstruction
|
| 310 |
+
- JSON artifact completeness
|
| 311 |
+
|
| 312 |
+
English and Nepali must not regress versus the chosen stock baseline.
|
| 313 |
+
|
| 314 |
+
### Gate 5: Claim Review
|
| 315 |
+
|
| 316 |
+
The paper claim is allowed only if:
|
| 317 |
+
|
| 318 |
+
- all public/real validation rows have complete artifacts
|
| 319 |
+
- fallback usage is reported separately
|
| 320 |
+
- synthetic-only improvements are not used as SOTA evidence
|
| 321 |
+
- failures are included in error analysis
|
| 322 |
+
- a reproducibility bundle can rerun the benchmark
|
| 323 |
+
|
| 324 |
+
## Immediate Implementation Steps
|
| 325 |
+
|
| 326 |
+
1. Add `ocrtech audit-gorkhapatra-source`. **Implemented:** the command audits
|
| 327 |
+
Naya Nepal article images and Gorkhapatra epaper PDF links, resolves PDF.js
|
| 328 |
+
viewer URLs to direct `/uploads/file/...pdf` URLs, optionally downloads and
|
| 329 |
+
hashes assets, and records PyMuPDF text-layer metadata when available.
|
| 330 |
+
2. Add `ocrtech prepare-gorkhapatra-pack`. **Implemented:** the command turns a
|
| 331 |
+
source-audit JSON file into a pending-reference validation manifest, copies
|
| 332 |
+
audited article images, renders audited epaper PDF pages when PyMuPDF is
|
| 333 |
+
available, writes placeholder reference JSON files, and marks samples as
|
| 334 |
+
`reference_status=pending_manual_label` and
|
| 335 |
+
`claim_evidence_eligible=false` until references are verified.
|
| 336 |
+
3. Add `languages/nepal-v1.json` registry with Limbu first and at least ten
|
| 337 |
+
target languages. **Implemented:** the registry tracks Limbu/Sirijonga first,
|
| 338 |
+
more than ten Nepal-language targets, English as a non-counted regression
|
| 339 |
+
language, script/Unicode ranges, font candidates, corpus sources, Gorkhapatra
|
| 340 |
+
labels, validation status, and PaddleOCR dictionary targets. The
|
| 341 |
+
`ocrtech audit-language-registry` command validates the registry and writes
|
| 342 |
+
JSON/Markdown audit artifacts.
|
| 343 |
+
4. Add renderability audit for Unicode ranges and fonts. **Implemented:** the
|
| 344 |
+
`ocrtech audit-font-renderability` command reads TTF/OTF/TTC cmap tables,
|
| 345 |
+
checks registered font candidates against each script's Unicode ranges, and
|
| 346 |
+
writes JSON/Markdown evidence. Raster sample rendering is supported when
|
| 347 |
+
Pillow is installed. Current cmap evidence verifies Limbu, Devanagari,
|
| 348 |
+
Santhali/Ol Chiki, and Latin coverage when the registered runtime font assets
|
| 349 |
+
are present.
|
| 350 |
+
5. Add Limbu synthetic line/page generator.
|
| 351 |
+
6. Add PP-OCRv6 stock baseline runner.
|
| 352 |
+
7. Build the first Limbu validation pack from Gorkhapatra/Naya Nepal plus local
|
| 353 |
+
Limbu resources.
|
| 354 |
+
|
| 355 |
+
## External References
|
| 356 |
+
|
| 357 |
+
- PaddleOCR docs: <https://paddlepaddle.github.io/PaddleOCR/main/en/index.html>
|
| 358 |
+
- PaddleOCR repo: <https://github.com/PaddlePaddle/PaddleOCR>
|
| 359 |
+
- PP-StructureV3 docs:
|
| 360 |
+
<https://paddlepaddle.github.io/PaddleOCR/main/en/version3.x/pipeline_usage/PP-StructureV3.html>
|
| 361 |
+
- PaddleOCR-VL-1.6 docs:
|
| 362 |
+
<https://paddlepaddle.github.io/PaddleOCR/main/en/version3.x/algorithm/PaddleOCR-VL/PaddleOCR-VL-1.6.html>
|
| 363 |
+
- Gorkhapatra Naya Nepal category:
|
| 364 |
+
<https://gorkhapatraonline.com/categories/new-nepal>
|
| 365 |
+
- Gorkhapatra Naya Nepal publication:
|
| 366 |
+
<https://gorkhapatraonline.com/publications/naya-nepal>
|
| 367 |
+
- Gorkhapatra epaper list example:
|
| 368 |
+
<https://epaper.gorkhapatraonline.com/epaper-list?pdfdaterange1=2026-06-09&pdfdaterange2=2026-06-16&slug=gorkhapatra>
|
| 369 |
+
- Unicode Limbu chart: <https://www.unicode.org/charts/PDF/U1900.pdf>
|
| 370 |
+
- Kirat Rai Unicode proposal:
|
| 371 |
+
<https://www.unicode.org/L2/L2022/22043-kirat-rai.pdf>
|
docs/PAPER_NOTES.md
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
docs/PAPER_RESULTS.md
ADDED
|
@@ -0,0 +1,854 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Paper Results
|
| 2 |
+
|
| 3 |
+
This file is the shortest path from repo artifacts to a paper draft.
|
| 4 |
+
|
| 5 |
+
Use [PAPER_NOTES.md](PAPER_NOTES.md) for claim boundaries and framing.
|
| 6 |
+
Use this file for the concrete result tables and the branch chronology.
|
| 7 |
+
|
| 8 |
+
## Current Comparison Tables
|
| 9 |
+
|
| 10 |
+
### Rendered Claim Pack
|
| 11 |
+
|
| 12 |
+
Generated artifact:
|
| 13 |
+
|
| 14 |
+
- [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-table/paper-table.md)
|
| 15 |
+
- [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-table/paper-table.json)
|
| 16 |
+
|
| 17 |
+
Headline table:
|
| 18 |
+
|
| 19 |
+
| system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy | latency_seconds |
|
| 20 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 21 |
+
| Candidate | 25 | 0.248654 | 0.955556 | 0.764000 | 3.268937 |
|
| 22 |
+
| Surya | 25 | 0.195260 | 0.955556 | 0.750909 | 2.618872 |
|
| 23 |
+
| Tesseract | 25 | 0.689214 | 0.000000 | 0.816000 | 0.240963 |
|
| 24 |
+
| Stock Paddle | 25 | 1.749081 | 0.000000 | 1.000000 | 7.732034 |
|
| 25 |
+
|
| 26 |
+
Interpretation:
|
| 27 |
+
|
| 28 |
+
- Candidate beats Tesseract and stock-Paddle badly on text CER and table extraction.
|
| 29 |
+
- Candidate ties Surya on aggregate `table_cell_f1`.
|
| 30 |
+
- Candidate is slightly better than Surya on aggregate reading-order accuracy.
|
| 31 |
+
- Candidate still loses to raw Surya on text CER.
|
| 32 |
+
- Candidate is also slower than Tesseract and somewhat slower than Surya on this pack.
|
| 33 |
+
|
| 34 |
+
Latency caveat:
|
| 35 |
+
|
| 36 |
+
- these latency means come from separate benchmark reports on the same host and pack, then get pinned per system during table export
|
| 37 |
+
- treat them as comparable operational signals, not as a single synchronized runtime shootout
|
| 38 |
+
|
| 39 |
+
### Rendered Claim Pack V2 Holdout
|
| 40 |
+
|
| 41 |
+
Generated artifact:
|
| 42 |
+
|
| 43 |
+
- [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-table/paper-table.md)
|
| 44 |
+
- [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-table/paper-table.json)
|
| 45 |
+
- [preflight.md](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-script-router/preflight.md)
|
| 46 |
+
|
| 47 |
+
Headline table:
|
| 48 |
+
|
| 49 |
+
| system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy |
|
| 50 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 51 |
+
| Candidate | 50 | 0.180893 | 0.947500 | 0.750909 |
|
| 52 |
+
| Surya | 50 | 0.180893 | 0.970833 | 0.750909 |
|
| 53 |
+
| Tesseract | 50 | 0.714207 | 0.000000 | 0.787818 |
|
| 54 |
+
|
| 55 |
+
Interpretation:
|
| 56 |
+
|
| 57 |
+
- This is the stronger rendered-document holdout because calibration and holdout were separated at `50/50`.
|
| 58 |
+
- The routed candidate does not beat Surya here. It ties Surya exactly on text CER and reading-order accuracy.
|
| 59 |
+
- The routed candidate is slightly worse than raw Surya on aggregate `table_cell_f1`.
|
| 60 |
+
- The calibration artifact selected `primary_script_threshold=0.0`, which means the routed candidate collapsed to "always pick Surya" on the calibration split.
|
| 61 |
+
- That is why claim preflight now rejects this candidate as a routed model fit for a broad "better engine" claim.
|
| 62 |
+
|
| 63 |
+
### Rendered Claim Pack V2 Holdout - Surya Primary Latin Router
|
| 64 |
+
|
| 65 |
+
Generated artifact:
|
| 66 |
+
|
| 67 |
+
- [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-latin-table/paper-table.md)
|
| 68 |
+
- [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-latin-table/paper-table.json)
|
| 69 |
+
|
| 70 |
+
Headline table:
|
| 71 |
+
|
| 72 |
+
| system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy | latency_seconds |
|
| 73 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 74 |
+
| Candidate | 50 | 0.416854 | 0.337500 | 0.761818 | 6.067455 |
|
| 75 |
+
| Surya | 50 | 0.180893 | 0.970833 | 0.750909 | 6.699851 |
|
| 76 |
+
| Tesseract | 50 | 0.714207 | 0.000000 | 0.787818 | 0.238464 |
|
| 77 |
+
|
| 78 |
+
Interpretation:
|
| 79 |
+
|
| 80 |
+
- This candidate was a Surya-primary, Latin-triggered page router with Tesseract fallback.
|
| 81 |
+
- It was meant to preserve Surya's text strength while using Tesseract for English-heavy pages.
|
| 82 |
+
- It failed badly on the rendered-v2 holdout: worse than raw Surya on text CER and much worse on table structure.
|
| 83 |
+
- Paired CER against Surya was decisive enough to rule the branch out: `0` wins, `12` losses, `38` ties.
|
| 84 |
+
|
| 85 |
+
### Rendered Claim Pack V2 Holdout - Surya Quality Router
|
| 86 |
+
|
| 87 |
+
Generated artifact:
|
| 88 |
+
|
| 89 |
+
- [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-quality-router-table/paper-table.md)
|
| 90 |
+
- [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-quality-router-table/paper-table.json)
|
| 91 |
+
- [calibration.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-surya-quality-router/calibration.json)
|
| 92 |
+
|
| 93 |
+
Headline table:
|
| 94 |
+
|
| 95 |
+
| system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy | latency_seconds |
|
| 96 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 97 |
+
| Candidate | 50 | 0.181128 | 0.947500 | 0.750909 | 4.620400 |
|
| 98 |
+
| Surya | 50 | 0.180893 | 0.947500 | 0.750909 | 5.603801 |
|
| 99 |
+
| Tesseract | 50 | 0.714207 | 0.000000 | 0.787818 | 0.240773 |
|
| 100 |
+
|
| 101 |
+
Interpretation:
|
| 102 |
+
|
| 103 |
+
- This candidate was a Surya-vs-Tesseract quality selector calibrated on the rendered-v2 calibration split.
|
| 104 |
+
- Calibration selected `selected_threshold=0.0`, which means the quality gate kept Surya whenever it was not scored worse than Tesseract.
|
| 105 |
+
- On the untouched holdout, the candidate still did not beat raw Surya on text CER.
|
| 106 |
+
- Paired CER against Surya was effectively a tie with one bad miss: `0` wins, `1` loss, `49` ties.
|
| 107 |
+
- This is operationally closer to "Surya with a guardrail" than to a meaningfully new OCR engine.
|
| 108 |
+
|
| 109 |
+
### Rendered Claim Pack V2 Holdout - Line Align Composite
|
| 110 |
+
|
| 111 |
+
Generated artifact:
|
| 112 |
+
|
| 113 |
+
- [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-line-align-table/paper-table.md)
|
| 114 |
+
- [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-line-align-table/paper-table.json)
|
| 115 |
+
- [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-line-align-summary/summary.json)
|
| 116 |
+
|
| 117 |
+
Headline table:
|
| 118 |
+
|
| 119 |
+
| system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy | latency_seconds |
|
| 120 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 121 |
+
| Candidate | 50 | 0.310549 | 0.897500 | 0.792727 | 4.142242 |
|
| 122 |
+
| Surya | 50 | 0.180893 | 0.970833 | 0.750909 | 3.223913 |
|
| 123 |
+
| Tesseract | 50 | 0.714207 | 0.000000 | 0.787818 | 0.235074 |
|
| 124 |
+
|
| 125 |
+
Interpretation:
|
| 126 |
+
|
| 127 |
+
- This candidate kept Surya as the text engine and reused Tesseract line scaffolding through `line_align_composite`.
|
| 128 |
+
- It did improve reading-order accuracy relative to raw Surya, and even edged past raw Tesseract on this holdout.
|
| 129 |
+
- That gain is not enough. CER regressed badly versus Surya, and aggregate `table_cell_f1` also regressed.
|
| 130 |
+
- Paired CER against Surya was one-sided enough to kill the branch: `0` wins, `7` losses, `43` ties.
|
| 131 |
+
- This is a useful negative result for the paper because it shows that better line ordering alone does not rescue the Surya/Tesseract composite family.
|
| 132 |
+
|
| 133 |
+
### Rendered Claim Pack V2 Holdout - Sample-Level Oracle
|
| 134 |
+
|
| 135 |
+
Generated artifact:
|
| 136 |
+
|
| 137 |
+
- [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-oracle-table/paper-table.md)
|
| 138 |
+
- [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-oracle-table/paper-table.json)
|
| 139 |
+
- [oracle-analysis.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-analysis.json)
|
| 140 |
+
- [oracle-choices.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-choices.json)
|
| 141 |
+
|
| 142 |
+
Headline table:
|
| 143 |
+
|
| 144 |
+
| system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy | latency_seconds |
|
| 145 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 146 |
+
| Oracle Surya/Tesseract | 50 | 0.180529 | 0.970833 | 0.750909 | 3.193115 |
|
| 147 |
+
| Surya | 50 | 0.180893 | 0.970833 | 0.750909 | 3.223913 |
|
| 148 |
+
| Tesseract | 50 | 0.714207 | 0.000000 | 0.787818 | 0.235074 |
|
| 149 |
+
|
| 150 |
+
Interpretation:
|
| 151 |
+
|
| 152 |
+
- This is not a deployable system. It is an upper-bound analysis that picks the better of raw Surya and raw Tesseract per sample using ground-truth CER.
|
| 153 |
+
- The oracle picked Surya for `49` of `50` holdout samples and Tesseract for only `1`.
|
| 154 |
+
- Per-sample oracle selections are recorded separately in `oracle-choices.json`; they are provenance, not benchmark metrics.
|
| 155 |
+
- The absolute CER gain over raw Surya is only `0.000364`.
|
| 156 |
+
- That means the current Surya/Tesseract routing space has almost no text-OCR headroom left on rendered-v2.
|
| 157 |
+
- Future work should move to a stronger recognizer/document model, not more Surya/Tesseract routing heuristics.
|
| 158 |
+
|
| 159 |
+
### Rendered Claim Pack V2 Holdout - Surya Error Pockets
|
| 160 |
+
|
| 161 |
+
Generated artifact:
|
| 162 |
+
|
| 163 |
+
- [error-analysis.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-error-analysis/error-analysis.md)
|
| 164 |
+
- [top-failures.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-error-analysis/top-failures.json)
|
| 165 |
+
- [surya-vs-tesseract error-analysis.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-vs-tesseract-error-analysis/error-analysis.md)
|
| 166 |
+
- [surya-vs-oracle error-analysis.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-vs-oracle-error-analysis/error-analysis.md)
|
| 167 |
+
- [eval-surya-hard15.jsonl](/Users/cdjk/github/llm/ocr-tech/data/splits-claimpack-rendered-v2/eval-surya-hard15.jsonl)
|
| 168 |
+
- [validation-hard15-surya.json](/Users/cdjk/github/llm/ocr-tech/configs/validation-hard15-surya.json)
|
| 169 |
+
- [line-align hard15 validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-claimpack-v2-line-align-hard15/validation.md)
|
| 170 |
+
- [k2 manifest-audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/audit-claimpack-v2-surya-hard15-k2/manifest-audit.md)
|
| 171 |
+
- [table/figure dev pack manifest](/Users/cdjk/github/llm/ocr-tech/data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl)
|
| 172 |
+
- [table/figure dev pack audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/audit-eval-pack-table-figure-dev-v1-k2/eval-pack-audit.md)
|
| 173 |
+
|
| 174 |
+
Diagnostic summary:
|
| 175 |
+
|
| 176 |
+
| slice | sample_count | Surya CER |
|
| 177 |
+
| --- | ---: | ---: |
|
| 178 |
+
| figure | 20 | 0.253157 |
|
| 179 |
+
| table | 20 | 0.252781 |
|
| 180 |
+
| multi_page | 10 | 0.196472 |
|
| 181 |
+
| scan | 10 | 0.194861 |
|
| 182 |
+
| low_contrast | 10 | 0.193317 |
|
| 183 |
+
| blur | 10 | 0.190940 |
|
| 184 |
+
| clean | 10 | 0.182256 |
|
| 185 |
+
|
| 186 |
+
Interpretation:
|
| 187 |
+
|
| 188 |
+
- Surya's hardest absolute CER pockets on rendered-v2 are table-ledger and figure-caption samples, not generic blur/skew alone.
|
| 189 |
+
- Tesseract is worse than Surya on `49` of `50` samples and better on only `eval-form-notice-003-blur`; that single oracle win is too small to justify more Surya/Tesseract routing work.
|
| 190 |
+
- A ranked `15`-sample `hard_eval` manifest now exists for the next model branch; it contains `8` table samples and `7` figure samples, and its full file/reference audit passes on `k2`.
|
| 191 |
+
- `configs/validation-hard15-surya.json` defines the pocket gate a future candidate must clear against Surya, including intersection gates for `hard_eval,table` and `hard_eval,figure`.
|
| 192 |
+
- The current line-align candidate fails the hard15 gate: hard-eval CER is `0.482039` versus Surya `0.348706`, and hard table CER is `0.613636` versus Surya `0.363636`.
|
| 193 |
+
- A separate `80`-sample image development pack now exists for table/figure work: `40` table samples, `40` figure samples, all five degradations. It has zero sample-id overlap and zero text-hash overlap with hard15, and its `k2` audit passes.
|
| 194 |
+
- That development pack is not claim evidence; it is for model iteration before returning to the held-out hard15 and rendered-v2 gates.
|
| 195 |
+
- The next credible recognizer branch should target table/figure-heavy mixed-script pages and preserve Surya's already-strong behavior on the other `49` samples.
|
| 196 |
+
|
| 197 |
+
### Table/Figure Development Pack - HF TrOCR Negative Result
|
| 198 |
+
|
| 199 |
+
Generated artifact:
|
| 200 |
+
|
| 201 |
+
- [split-summary.md](/Users/cdjk/github/llm/ocr-tech/data/splits-table-figure-dev-v1/split-summary.md)
|
| 202 |
+
- [training_metrics.json](/Users/cdjk/github/llm/ocr-tech/runs/hf-table-figure-dev-v1/training_metrics.json)
|
| 203 |
+
- [model-card.json](/Users/cdjk/github/llm/ocr-tech/runs/hf-table-figure-dev-v1/package/model-card.json)
|
| 204 |
+
- [model-audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/audit-hf-table-figure-dev-v1-model/model-audit.md)
|
| 205 |
+
- [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-hf-table-figure-dev-v1/summary.md)
|
| 206 |
+
- [benchmark report.json](/Users/cdjk/github/llm/ocr-tech/outputs/experiment-hf-table-figure-dev-v1/benchmark/report.json)
|
| 207 |
+
- [paddle_recognizer.yml](/Users/cdjk/github/llm/ocr-tech/runs/paddle-table-figure-dev-v1/paddle_recognizer.yml)
|
| 208 |
+
|
| 209 |
+
Pack and split:
|
| 210 |
+
|
| 211 |
+
- `80` image samples: `40` table and `40` figure
|
| 212 |
+
- degradations: `clean`, `scan`, `blur`, `low_contrast`, `skew`
|
| 213 |
+
- split: `60` train / `20` eval
|
| 214 |
+
- grouping: `text_sha256`
|
| 215 |
+
- text groups: `16`
|
| 216 |
+
- split leakage: pass
|
| 217 |
+
- train/eval manifest audits on `k2`: pass
|
| 218 |
+
- hard15 overlap: `0` sample IDs and `0` text hashes
|
| 219 |
+
|
| 220 |
+
Training result for `microsoft/trocr-small-printed` after one epoch:
|
| 221 |
+
|
| 222 |
+
| metric | value |
|
| 223 |
+
| --- | ---: |
|
| 224 |
+
| eval_loss | 7.884517 |
|
| 225 |
+
| eval_cer | 3.062119 |
|
| 226 |
+
| eval_wer | 3.562689 |
|
| 227 |
+
| eval_exact_line_accuracy | 0.000000 |
|
| 228 |
+
|
| 229 |
+
Benchmark result on the `20`-sample dev eval:
|
| 230 |
+
|
| 231 |
+
| system | CER | table_cell_f1 | reading_order_pair_accuracy |
|
| 232 |
+
| --- | ---: | ---: | ---: |
|
| 233 |
+
| HF table/figure candidate | 2.422001 | 0.000000 | 0.000000 |
|
| 234 |
+
| Surya | 0.563399 | 0.858333 | 0.550000 |
|
| 235 |
+
| Tesseract | 0.546227 | 0.000000 | 0.695000 |
|
| 236 |
+
|
| 237 |
+
Interpretation:
|
| 238 |
+
|
| 239 |
+
- The model package audits cleanly, so this is a quality failure rather than a packaging failure.
|
| 240 |
+
- Paired CER was `0` wins, `20` losses, and `0` ties against Surya.
|
| 241 |
+
- Paired CER was also `0` wins, `20` losses, and `0` ties against Tesseract.
|
| 242 |
+
- Naive full-page TrOCR fine-tuning is not a serious next step for table/figure-heavy document pages.
|
| 243 |
+
- The next model branch should use line/cell crops or a structure-aware page/document model rather than treating full document images as single recognition targets.
|
| 244 |
+
|
| 245 |
+
### Table/Figure Crop Development Pack - HF TrOCR Negative Result
|
| 246 |
+
|
| 247 |
+
Generated artifact:
|
| 248 |
+
|
| 249 |
+
- [crop-manifest-summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/crops-table-figure-dev-v1-k2/crop-manifest-summary.md)
|
| 250 |
+
- [crop manifest audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/crops-table-figure-dev-v1-k2/manifest-audit.md)
|
| 251 |
+
- [crop split-summary.md](/Users/cdjk/github/llm/ocr-tech/data/splits-crops-table-figure-dev-v1/split-summary.md)
|
| 252 |
+
- [training_metrics.json](/Users/cdjk/github/llm/ocr-tech/runs/hf-crops-table-figure-dev-v1/training_metrics.json)
|
| 253 |
+
- [model-card.json](/Users/cdjk/github/llm/ocr-tech/runs/hf-crops-table-figure-dev-v1/package/model-card.json)
|
| 254 |
+
- [model-audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/audit-hf-crops-table-figure-dev-v1-model/model-audit.md)
|
| 255 |
+
- [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-hf-crops-table-figure-dev-v1/summary.md)
|
| 256 |
+
- [benchmark report.json](/Users/cdjk/github/llm/ocr-tech/outputs/experiment-hf-crops-table-figure-dev-v1/benchmark/report.json)
|
| 257 |
+
|
| 258 |
+
Pack and split:
|
| 259 |
+
|
| 260 |
+
- source rendered samples: `80`
|
| 261 |
+
- extracted crops: `544`
|
| 262 |
+
- crop types: `384` table cells, `128` lines, `32` figure captions
|
| 263 |
+
- skipped candidates: `32` skew pages because pre-rotation bboxes are invalid
|
| 264 |
+
- split: `480` train / `64` eval
|
| 265 |
+
- grouping: `text_sha256`
|
| 266 |
+
- text groups: `68`
|
| 267 |
+
- crop manifest audit on `k2`: pass
|
| 268 |
+
- split leakage: pass
|
| 269 |
+
|
| 270 |
+
Training result for `microsoft/trocr-small-printed` after one crop-level epoch:
|
| 271 |
+
|
| 272 |
+
| metric | value |
|
| 273 |
+
| --- | ---: |
|
| 274 |
+
| eval_loss | 3.087561 |
|
| 275 |
+
| eval_cer | 0.959831 |
|
| 276 |
+
| eval_wer | 1.363715 |
|
| 277 |
+
| eval_exact_line_accuracy | 0.078125 |
|
| 278 |
+
|
| 279 |
+
Benchmark result on the `64`-sample crop eval:
|
| 280 |
+
|
| 281 |
+
| system | CER |
|
| 282 |
+
| --- | ---: |
|
| 283 |
+
| HF crop candidate | 1.043369 |
|
| 284 |
+
| Surya | 0.475309 |
|
| 285 |
+
| Tesseract | 0.077453 |
|
| 286 |
+
|
| 287 |
+
Interpretation:
|
| 288 |
+
|
| 289 |
+
- Crop-level training is a better problem shape than full-page recognition, but this recognizer is still not competitive.
|
| 290 |
+
- Paired CER against Surya was `7` wins, `50` losses, and `7` ties.
|
| 291 |
+
- Paired CER against Tesseract was `0` wins, `64` losses, and `0` ties.
|
| 292 |
+
- Tesseract is extremely strong on these rendered crop images, so near-term gains are more likely from better crop/layout detection plus strong crop OCR than from this small synthetic TrOCR fine-tune.
|
| 293 |
+
|
| 294 |
+
### Hard15 - Table Cell Refinement Composite
|
| 295 |
+
|
| 296 |
+
Generated artifact:
|
| 297 |
+
|
| 298 |
+
- [unsafe model-card.json](/Users/cdjk/github/llm/ocr-tech/outputs/table-cell-refine-v1/model-card.json)
|
| 299 |
+
- [unsafe summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-table-cell-refine-hard15-v1/summary.md)
|
| 300 |
+
- [unsafe cell-analysis.md](/Users/cdjk/github/llm/ocr-tech/outputs/table-cell-analysis-refine-always-hard15-v1/table-cell-analysis.md)
|
| 301 |
+
- [safe model-card.json](/Users/cdjk/github/llm/ocr-tech/outputs/table-cell-refine-empty-v1/model-card.json)
|
| 302 |
+
- [safe summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-table-cell-refine-empty-hard15-v1/summary.md)
|
| 303 |
+
- [safe cell-analysis.md](/Users/cdjk/github/llm/ocr-tech/outputs/table-cell-analysis-refine-empty-hard15-v1/table-cell-analysis.md)
|
| 304 |
+
- [safe validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-table-cell-refine-empty-hard15-v1/validation.md)
|
| 305 |
+
- [safe+dedup model-card.json](/Users/cdjk/github/llm/ocr-tech/outputs/table-cell-refine-empty-dedup-v1/model-card.json)
|
| 306 |
+
- [safe+dedup summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-table-cell-refine-empty-dedup-hard15-v3/summary.md)
|
| 307 |
+
- [safe+dedup cell-analysis.md](/Users/cdjk/github/llm/ocr-tech/outputs/table-cell-analysis-refine-empty-dedup-hard15-v3/table-cell-analysis.md)
|
| 308 |
+
- [safe+dedup validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-table-cell-refine-empty-dedup-hard15-v3/validation.md)
|
| 309 |
+
|
| 310 |
+
Headline hard15 comparison:
|
| 311 |
+
|
| 312 |
+
| system | hard_eval CER | table_cell_f1 | reading_order_pair_accuracy |
|
| 313 |
+
| --- | ---: | ---: | ---: |
|
| 314 |
+
| Surya baseline | 0.348706 | 0.979167 | 0.520000 |
|
| 315 |
+
| Table-cell refine, `always` | 0.348706 | 0.593750 | 0.520000 |
|
| 316 |
+
| Table-cell refine, `empty` | 0.348706 | 0.979167 | 0.520000 |
|
| 317 |
+
| Table-cell refine, `empty` + parser de-dup | 0.348706 | 0.979167 | 0.520000 |
|
| 318 |
+
|
| 319 |
+
Interpretation:
|
| 320 |
+
|
| 321 |
+
- The unsafe table-cell crop replacement policy damaged Surya's already-good table cells.
|
| 322 |
+
- The conservative empty-cell-only policy avoided regression but did not improve any hard15 metric.
|
| 323 |
+
- The safe policy's hard15 validation status is `not_validated`.
|
| 324 |
+
- The parser de-dup branch removes duplicate overlapping table exports for both candidate and Surya, so it improves artifact cleanliness but does not create a candidate advantage.
|
| 325 |
+
- Exact crop OCR being strong is not enough; the deployment problem is reliable cell detection and knowing when a replacement is safer than the structure engine's existing cell text.
|
| 326 |
+
|
| 327 |
+
Cell-level result:
|
| 328 |
+
|
| 329 |
+
- Surya is already exact on `94/96` hard15 reference table cells.
|
| 330 |
+
- Unsafe replacement is exact on only `57/96` cells.
|
| 331 |
+
- Unsafe replacement improves `0` cells over Surya and worsens `37`.
|
| 332 |
+
- Safe+dedup is exact on `94/96` cells, with `8` predicted tables for `8` reference tables and `0` duplicate overlapping tables on the v3 rerun.
|
| 333 |
+
|
| 334 |
+
### Hard15 - Figure Caption Refinement Composite
|
| 335 |
+
|
| 336 |
+
Generated artifact:
|
| 337 |
+
|
| 338 |
+
- [model-card.json](/Users/cdjk/github/llm/ocr-tech/outputs/figure-caption-refine-v1/model-card.json)
|
| 339 |
+
- [model-audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/audit-figure-caption-refine-v1-model/model-audit.md)
|
| 340 |
+
- [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-figure-caption-refine-hard15-v2/summary.md)
|
| 341 |
+
- [benchmark report.json](/Users/cdjk/github/llm/ocr-tech/outputs/experiment-figure-caption-refine-hard15-v2/benchmark/report.json)
|
| 342 |
+
- [validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-figure-caption-refine-hard15-v2/validation.md)
|
| 343 |
+
- [labels model-card.json](/Users/cdjk/github/llm/ocr-tech/outputs/figure-caption-refine-labels-v1/model-card.json)
|
| 344 |
+
- [labels summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-figure-caption-refine-labels-hard15-v1/summary.md)
|
| 345 |
+
- [labels validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-figure-caption-refine-labels-hard15-v1/validation.md)
|
| 346 |
+
|
| 347 |
+
Headline hard15 comparison after the parser caption-attachment fix:
|
| 348 |
+
|
| 349 |
+
| system | CER | figure_caption_cer | figure_detection_f1 | table_cell_f1 | reading_order_pair_accuracy |
|
| 350 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 351 |
+
| Surya | 0.236839 | 0.092323 | 1.000000 | 0.979167 | 0.520000 |
|
| 352 |
+
| Tesseract | 1.266364 | 0.051020 | 0.857143 | 0.000000 | 0.613333 |
|
| 353 |
+
| Figure-caption refine | 0.228008 | 0.052478 | 1.000000 | 0.979167 | 0.520000 |
|
| 354 |
+
| Figure-caption refine + labels | 0.204610 | 0.008746 | 1.000000 | 0.979167 | 0.520000 |
|
| 355 |
+
|
| 356 |
+
Validation result:
|
| 357 |
+
|
| 358 |
+
- basic branch overall status: `not_validated`
|
| 359 |
+
- basic branch figure-caption gate: pass, `6` wins / `0` losses / `1` tie against Surya, improvement `0.039845`
|
| 360 |
+
- label branch overall status: `not_validated`
|
| 361 |
+
- label branch overall hard-eval CER gate: pass, improvement `0.032228`
|
| 362 |
+
- label branch figure CER gate: pass, improvement `0.069061`
|
| 363 |
+
- label branch figure-caption gate: pass, `7` wins / `0` losses / `0` ties against Surya, improvement `0.083576`
|
| 364 |
+
- label branch table gates: fail, ties against Surya
|
| 365 |
+
|
| 366 |
+
Interpretation:
|
| 367 |
+
|
| 368 |
+
- This is a real slice win over Surya on figure-caption extraction.
|
| 369 |
+
- It is not broad enough for a SOTA claim.
|
| 370 |
+
- Tesseract has better caption text but worse figure detection/table structure; the composite keeps Surya's structure and selectively borrows caption text.
|
| 371 |
+
- Safe empty-only replacement ties all `96` cells with Surya because there are no missing Surya cells in this pocket.
|
| 372 |
+
- The only remaining Surya cell errors are `किरण` predicted as `करिण` on `eval-table-ledger-002-clean` and `eval-table-ledger-002-scan`.
|
| 373 |
+
|
| 374 |
+
### Rendered Blur-Only Expensive-Baseline Subset
|
| 375 |
+
|
| 376 |
+
Generated artifact:
|
| 377 |
+
|
| 378 |
+
- [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-blur5-table/paper-table.md)
|
| 379 |
+
- [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-blur5-table/paper-table.json)
|
| 380 |
+
|
| 381 |
+
Headline table:
|
| 382 |
+
|
| 383 |
+
| system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy | latency_seconds |
|
| 384 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 385 |
+
| Candidate | 5 | 0.223273 | 1.000000 | 0.783636 | 3.784279 |
|
| 386 |
+
| Surya | 5 | 0.197601 | 1.000000 | 0.750909 | 2.943875 |
|
| 387 |
+
| Tesseract | 5 | 0.732364 | 0.000000 | 0.823636 | 0.220515 |
|
| 388 |
+
| Stock Paddle | 5 | 2.810426 | 0.000000 | 1.000000 | 7.440482 |
|
| 389 |
+
| GLM-OCR | 5 | 2.820269 | 0.000000 | 1.000000 | 16.846619 |
|
| 390 |
+
| PaddleOCR-VL | 5 | 18.470663 | 0.000000 | 0.750909 | 32.793166 |
|
| 391 |
+
|
| 392 |
+
Interpretation:
|
| 393 |
+
|
| 394 |
+
- Candidate is materially stronger than Tesseract, stock-Paddle, GLM-OCR, and PaddleOCR-VL on this degraded rendered subset.
|
| 395 |
+
- Candidate still does not beat raw Surya on text CER.
|
| 396 |
+
- The external VLM-style baselines are also much slower than the current candidate here.
|
| 397 |
+
|
| 398 |
+
## Branch Chronology
|
| 399 |
+
|
| 400 |
+
This is the shortest honest story of the project so far.
|
| 401 |
+
|
| 402 |
+
| branch | eval setting | candidate CER | baseline CER | paired result | verdict | evidence |
|
| 403 |
+
| --- | --- | ---: | ---: | --- | --- | --- |
|
| 404 |
+
| HF TrOCR recognizer | mixed Nepali/English line-style holdout, 180 samples | 0.586139 | 0.329012 vs Tesseract | 36 wins, 97 losses, 47 ties | runnable but not competitive | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-hf-vs-tesseract-summary/summary.json) |
|
| 405 |
+
| Tesseract + ByT5 corrector | same 180-sample mixed holdout | 6.912065 | 0.329012 vs Tesseract | 0 wins, 170 losses, 10 ties | failed badly | [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-corrector-vs-tesseract-summary/summary.md) |
|
| 406 |
+
| quality router (`psm=6` vs `psm=13`) | 91-sample mixed holdout | 0.340608 | 0.342234 vs Tesseract | 2 wins, 1 loss, 88 ties | tiny gain, plausible baseline only | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-quality-router-holdout/summary.json) |
|
| 407 |
+
| ranked PSM ensemble (`6/7/11/13`) | same 91-sample mixed holdout | 0.358352 | 0.342234 vs Tesseract | 9 wins, 16 losses, 66 ties | overfit, worse than baseline | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-psm-ensemble-holdout/summary.json) |
|
| 408 |
+
| script router (`tesseract -> surya`) | same 91-sample mixed holdout | 0.313084 | 0.342234 vs Tesseract | 9 wins, 0 losses, 82 ties | strongest older line-style result | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-script-router-holdout/summary.json) |
|
| 409 |
+
| script router + structure backfill | rendered claim-ready document pack, 25 samples | 0.248654 | 0.689214 vs Tesseract | 9 wins, 1 loss, 15 ties | strong document-pack improvement over Tesseract | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-rendered-backfill/summary.json) |
|
| 410 |
+
| raw Surya on rendered claim pack | rendered claim-ready document pack, 25 samples | 0.195260 | 0.248654 vs Candidate | candidate loses on text CER | current text leader on rendered pack | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-surya-compare/summary.json) |
|
| 411 |
+
| rendered-v2 script router | rendered claim-ready document holdout, 50 samples | 0.180893 | 0.180893 vs Surya | exact CER tie, worse table structure | routed candidate collapses to Surya on calibration; not a meaningful new engine | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-script-router/summary.json) |
|
| 412 |
+
| rendered-v2 Surya-primary Latin router | rendered claim-ready document holdout, 50 samples | 0.416854 | 0.180893 vs Surya | 0 wins, 12 losses, 38 ties on CER | failed badly; inverted script routing is not a viable answer | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-surya-latin-router/summary.json) |
|
| 413 |
+
| rendered-v2 Surya quality router | rendered claim-ready document holdout, 50 samples | 0.181128 | 0.180893 vs Surya | 0 wins, 1 loss, 49 ties on CER | nearly indistinguishable from Surya and still not better | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-surya-quality-router/summary.json) |
|
| 414 |
+
| rendered-v2 line-align composite | rendered claim-ready document holdout, 50 samples | 0.310549 | 0.180893 vs Surya | 0 wins, 7 losses, 43 ties on CER | better reading order, but text and table regressions are too large | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-line-align-summary/summary.json) |
|
| 415 |
+
| rendered-v2 Surya/Tesseract sample oracle | rendered claim-ready document holdout, 50 samples | 0.180529 | 0.180893 vs Surya | oracle picks Tesseract only 1/50 samples | upper bound shows routing headroom is essentially exhausted | [oracle-analysis.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-analysis.json) |
|
| 416 |
+
| HF TrOCR table/figure dev | table/figure development eval, 20 samples | 2.422001 | 0.563399 vs Surya; 0.546227 vs Tesseract | 0 wins, 20 losses, 0 ties vs both | failed badly; full-page TrOCR fine-tuning is not viable here | [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-hf-table-figure-dev-v1/summary.md) |
|
| 417 |
+
| HF TrOCR crop dev | table/figure crop eval, 64 samples | 1.043369 | 0.475309 vs Surya; 0.077453 vs Tesseract | 7 wins, 50 losses, 7 ties vs Surya; 0 wins, 64 losses vs Tesseract | crop shape helps but this recognizer still fails | [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-hf-crops-table-figure-dev-v1/summary.md) |
|
| 418 |
+
| table-cell refine, unsafe | hard15 rendered-v2 pocket, 15 samples | 0.348706 | 0.348706 vs Surya | text tie, table F1 regresses from 0.979167 to 0.593750 | replacement policy is unsafe | [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-table-cell-refine-hard15-v1/summary.md) |
|
| 419 |
+
| table-cell refine, empty-only | hard15 rendered-v2 pocket, 15 samples | 0.348706 | 0.348706 vs Surya | exact tie across CER/table/reading order; validation not_validated | safe but no improvement | [validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-table-cell-refine-empty-hard15-v1/validation.md) |
|
| 420 |
+
| normalized script router, hard-timeout rerun | normalized mixed Nepali/English line-style eval, 180 samples | 0.354840 | 0.329012 vs Tesseract; 3.157317 vs Surya on 178 scored Surya rows | 17 wins, 4 losses, 159 ties on CER vs Tesseract but worse aggregate mean; 2 Surya rows timed out at 45s | not a better engine; benchmark watchdog works and routing still loses overall CER | [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/k2-mixed-v1-normalized-script-router-hard-timeout/outputs/summary-mixed-v1-normalized-script-router-hard-timeout/summary.md) |
|
| 421 |
+
| PaddleOCR recognizer hard-50 failure slice | top 50 failures from AWS G6 trained PaddleOCR recognizer, remapped to normalized k2 eval paths | n/a | n/a | 50 failures: 38 English line-crop synthetic, 12 Nepali synthetic; k2 manifest audit passed | next recognizer gate; do not train on it | [manifest-audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/k2-paddle-mixed-v2-hard50-normalized/outputs/manifest-audit-paddle-mixed-v2-cuda-hard50-normalized/manifest-audit.md) |
|
| 422 |
+
| PaddleOCR hard-50 gate negative control | failed AWS G6 trained PaddleOCR recognizer evaluated on the hard-50 gate | 0.887557 | 0.080249 vs Tesseract | fails all four hard gates; English line-crop CER `0.852048` vs `0.031244` | validates the gate catches the known recognizer collapse | [validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-paddle-mixed-v2-cuda-hard50-tesseract-negative/validation.md) |
|
| 423 |
+
| line-level Tesseract/Paddle composite | hard-50 failure slice, Tesseract primary with PaddleOCR secondary only for Devanagari-looking lines | 0.276489 | 0.120106 vs Tesseract | English line-crop ties exactly (`0.060429` CER); Nepali collapses (`0.960679` vs `0.309083` CER, `1.000000` vs `0.711607` WER) | safe English preservation but failed Devanagari override; do not promote this recognizer | [validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-line-script-paddle-hard50-v2/validation.md) |
|
| 424 |
+
| PaddleOCR v3 CUDA recognizer | G6-trained lower-LR PaddleOCR branch on hard-50 admission gate | 0.921416 | 0.080249 vs Tesseract | 0 wins, 50 losses on CER; English line-crop CER `0.917511` vs `0.031244`; Nepali WER `1.055556` vs `0.716369` | failed worse than v2; stop branch before full mixed eval | [validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/aws-g6-paddle-mixed-v3-cuda/outputs/validation-paddle-mixed-v3-cuda-hard50-rescored/validation.md) |
|
| 425 |
+
| PaddleOCR v4 CUDA recognizer | G6-trained Devanagari PP-OCRv3 branch, LR `0.00005`, 20 epochs, evaluated on hard-50 admission gate | 0.940500 | 0.080249 vs Tesseract | 0 wins, 50 losses on CER; English line-crop CER `0.950730` vs `0.031244`; Nepali CER `0.908108` vs `0.235433` | failed worse than v2/v3; the same PaddleOCR mixed-script fine-tune branch should not be the engine core | [artifact bundle](/Users/cdjk/github/llm/ocr-tech/outputs/aws-g6-paddle-mixed-v4-cuda/ocrtech-paddle-mixed-v4-cuda-artifacts.tgz) |
|
| 426 |
+
|
| 427 |
+
## Current Best-Evidence Statement
|
| 428 |
+
|
| 429 |
+
The strongest evidence-backed statement available today is:
|
| 430 |
+
|
| 431 |
+
1. The current Tesseract-plus-Surya backfill candidate is a real document-OCR improvement over Tesseract and several weaker baselines on internal rendered Nepali/English document packs.
|
| 432 |
+
2. On the larger rendered-v2 holdout, every Tesseract/Surya composite tested so far either collapses to Surya, trails Surya, becomes effectively indistinguishable from Surya, or trades too much OCR accuracy for modest ordering gains.
|
| 433 |
+
3. On the cleaned 180-sample mixed line-style eval with hard per-sample timeouts, the older script-router candidate also loses to Tesseract overall (`0.354840` vs `0.329012` CER), despite a small overall WER improvement (`0.469944` vs `0.486485`) and a Nepali WER improvement.
|
| 434 |
+
4. The trained PaddleOCR recognizer failure profile is now materialized as a normalized hard-50 eval gate: all top failures are synthetic line crops, mostly English (`38/50`), so the next recognizer branch must prove English preservation instead of only improving Nepali.
|
| 435 |
+
5. The line-level Tesseract/Paddle composite proves the candidate shape can preserve English while routing Devanagari-looking lines, but the available trained PaddleOCR recognizer is too weak on the Nepali hard slice to use as an override (`0.960679` vs Tesseract `0.309083` CER).
|
| 436 |
+
6. The second AWS G6 PaddleOCR attempt fixed recipe/runtime issues but failed the hard-50 gate even worse than the first (`0.921416` vs Tesseract `0.080249` CER), so this PP-OCRv3 Devanagari recipe family is not the engine path.
|
| 437 |
+
7. A sample-level oracle over raw Surya and raw Tesseract has almost no text-OCR headroom over Surya (`0.180529` vs `0.180893` CER), so more routing in this component family is not a serious path to SOTA.
|
| 438 |
+
8. The repo now has enough validation discipline and artifact traceability to support a serious comparative paper, but not a SOTA claim.
|
| 439 |
+
|
| 440 |
+
## Regeneration Workflow
|
| 441 |
+
|
| 442 |
+
Rebuild the generated comparison tables with:
|
| 443 |
+
|
| 444 |
+
```bash
|
| 445 |
+
ocrtech summarize-benchmark \
|
| 446 |
+
--benchmark-report outputs/live-k2-claimpack-rendered-backfill/report.json \
|
| 447 |
+
--out outputs/paper-claimpack-summary-tesseract \
|
| 448 |
+
--candidate candidate \
|
| 449 |
+
--baseline tesseract \
|
| 450 |
+
--metric cer
|
| 451 |
+
```
|
| 452 |
+
|
| 453 |
+
```bash
|
| 454 |
+
ocrtech summarize-benchmark \
|
| 455 |
+
--benchmark-report outputs/live-k2-claimpack-surya-compare/report.json \
|
| 456 |
+
--out outputs/paper-claimpack-summary-surya \
|
| 457 |
+
--candidate candidate \
|
| 458 |
+
--baseline surya \
|
| 459 |
+
--metric cer
|
| 460 |
+
```
|
| 461 |
+
|
| 462 |
+
```bash
|
| 463 |
+
ocrtech summarize-benchmark \
|
| 464 |
+
--benchmark-report outputs/live-k2-claimpack-stock-paddle-fixed/report.json \
|
| 465 |
+
--out outputs/paper-claimpack-summary-stock-paddle \
|
| 466 |
+
--candidate candidate \
|
| 467 |
+
--baseline stock-paddle \
|
| 468 |
+
--metric cer
|
| 469 |
+
```
|
| 470 |
+
|
| 471 |
+
```bash
|
| 472 |
+
ocrtech paper-benchmark-table \
|
| 473 |
+
--summary outputs/paper-claimpack-summary-tesseract/summary.json \
|
| 474 |
+
--summary outputs/paper-claimpack-summary-surya/summary.json \
|
| 475 |
+
--summary outputs/paper-claimpack-summary-stock-paddle/summary.json \
|
| 476 |
+
--out outputs/paper-claimpack-table \
|
| 477 |
+
--system candidate \
|
| 478 |
+
--system surya \
|
| 479 |
+
--system tesseract \
|
| 480 |
+
--system stock-paddle \
|
| 481 |
+
--label 'candidate=Candidate' \
|
| 482 |
+
--label 'surya=Surya' \
|
| 483 |
+
--label 'tesseract=Tesseract' \
|
| 484 |
+
--label 'stock-paddle=Stock Paddle' \
|
| 485 |
+
--source-preference 'candidate=outputs/paper-claimpack-summary-tesseract/summary.json' \
|
| 486 |
+
--source-preference 'tesseract=outputs/paper-claimpack-summary-tesseract/summary.json' \
|
| 487 |
+
--source-preference 'surya=outputs/paper-claimpack-summary-surya/summary.json' \
|
| 488 |
+
--source-preference 'stock-paddle=outputs/paper-claimpack-summary-stock-paddle/summary.json' \
|
| 489 |
+
--metric cer \
|
| 490 |
+
--metric table_cell_f1 \
|
| 491 |
+
--metric reading_order_pair_accuracy \
|
| 492 |
+
--metric latency_seconds
|
| 493 |
+
```
|
| 494 |
+
|
| 495 |
+
```bash
|
| 496 |
+
ocrtech summarize-benchmark \
|
| 497 |
+
--benchmark-report outputs/live-k2-rendered-blur5/report.json \
|
| 498 |
+
--out outputs/paper-blur5-summary \
|
| 499 |
+
--candidate candidate \
|
| 500 |
+
--baseline tesseract \
|
| 501 |
+
--baseline surya \
|
| 502 |
+
--baseline stock-paddle \
|
| 503 |
+
--baseline glm-ocr \
|
| 504 |
+
--baseline paddleocr-vl \
|
| 505 |
+
--metric cer
|
| 506 |
+
```
|
| 507 |
+
|
| 508 |
+
```bash
|
| 509 |
+
ocrtech paper-benchmark-table \
|
| 510 |
+
--summary outputs/paper-blur5-summary/summary.json \
|
| 511 |
+
--out outputs/paper-blur5-table \
|
| 512 |
+
--system candidate \
|
| 513 |
+
--system surya \
|
| 514 |
+
--system tesseract \
|
| 515 |
+
--system stock-paddle \
|
| 516 |
+
--system glm-ocr \
|
| 517 |
+
--system paddleocr-vl \
|
| 518 |
+
--label 'candidate=Candidate' \
|
| 519 |
+
--label 'surya=Surya' \
|
| 520 |
+
--label 'tesseract=Tesseract' \
|
| 521 |
+
--label 'stock-paddle=Stock Paddle' \
|
| 522 |
+
--label 'glm-ocr=GLM-OCR' \
|
| 523 |
+
--label 'paddleocr-vl=PaddleOCR-VL' \
|
| 524 |
+
--metric cer \
|
| 525 |
+
--metric table_cell_f1 \
|
| 526 |
+
--metric reading_order_pair_accuracy \
|
| 527 |
+
--metric latency_seconds
|
| 528 |
+
```
|
| 529 |
+
|
| 530 |
+
```bash
|
| 531 |
+
ocrtech paper-benchmark-table \
|
| 532 |
+
--summary outputs/live-k2-claimpack-v2-script-router/summary.json \
|
| 533 |
+
--out outputs/paper-claimpack-v2-table \
|
| 534 |
+
--system candidate \
|
| 535 |
+
--system surya \
|
| 536 |
+
--system tesseract \
|
| 537 |
+
--label 'candidate=Candidate' \
|
| 538 |
+
--label 'surya=Surya' \
|
| 539 |
+
--label 'tesseract=Tesseract' \
|
| 540 |
+
--metric cer \
|
| 541 |
+
--metric table_cell_f1 \
|
| 542 |
+
--metric reading_order_pair_accuracy \
|
| 543 |
+
--title 'Rendered Claim Pack V2 Holdout Comparison'
|
| 544 |
+
```
|
| 545 |
+
|
| 546 |
+
```bash
|
| 547 |
+
ocrtech paper-benchmark-table \
|
| 548 |
+
--summary outputs/live-k2-claimpack-v2-surya-quality-router/summary.json \
|
| 549 |
+
--out outputs/paper-claimpack-v2-surya-quality-router-table \
|
| 550 |
+
--system candidate \
|
| 551 |
+
--system surya \
|
| 552 |
+
--system tesseract \
|
| 553 |
+
--label 'candidate=Candidate' \
|
| 554 |
+
--label 'surya=Surya' \
|
| 555 |
+
--label 'tesseract=Tesseract' \
|
| 556 |
+
--metric cer \
|
| 557 |
+
--metric table_cell_f1 \
|
| 558 |
+
--metric reading_order_pair_accuracy \
|
| 559 |
+
--metric latency_seconds \
|
| 560 |
+
--title 'Rendered Claim Pack V2 Holdout - Surya Quality Router'
|
| 561 |
+
```
|
| 562 |
+
|
| 563 |
+
```bash
|
| 564 |
+
ocrtech paper-benchmark-table \
|
| 565 |
+
--summary outputs/live-k2-claimpack-v2-line-align-summary/summary.json \
|
| 566 |
+
--out outputs/paper-claimpack-v2-line-align-table \
|
| 567 |
+
--system candidate \
|
| 568 |
+
--system surya \
|
| 569 |
+
--system tesseract \
|
| 570 |
+
--label 'candidate=Candidate' \
|
| 571 |
+
--label 'surya=Surya' \
|
| 572 |
+
--label 'tesseract=Tesseract' \
|
| 573 |
+
--metric cer \
|
| 574 |
+
--metric table_cell_f1 \
|
| 575 |
+
--metric reading_order_pair_accuracy \
|
| 576 |
+
--metric latency_seconds \
|
| 577 |
+
--title 'Rendered Claim Pack V2 Holdout - Line Align Composite'
|
| 578 |
+
```
|
| 579 |
+
|
| 580 |
+
```bash
|
| 581 |
+
ocrtech benchmark-oracle \
|
| 582 |
+
--benchmark-report outputs/live-k2-claimpack-v2-line-align/report.json \
|
| 583 |
+
--out outputs/paper-claimpack-v2-surya-tesseract-oracle \
|
| 584 |
+
--system surya \
|
| 585 |
+
--system tesseract \
|
| 586 |
+
--metric cer \
|
| 587 |
+
--oracle-name oracle-surya-tesseract
|
| 588 |
+
|
| 589 |
+
ocrtech paper-benchmark-table \
|
| 590 |
+
--summary outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-summary/summary.json \
|
| 591 |
+
--out outputs/paper-claimpack-v2-oracle-table \
|
| 592 |
+
--system oracle-surya-tesseract \
|
| 593 |
+
--system surya \
|
| 594 |
+
--system tesseract \
|
| 595 |
+
--label 'oracle-surya-tesseract=Oracle Surya/Tesseract' \
|
| 596 |
+
--label 'surya=Surya' \
|
| 597 |
+
--label 'tesseract=Tesseract' \
|
| 598 |
+
--metric cer \
|
| 599 |
+
--metric table_cell_f1 \
|
| 600 |
+
--metric reading_order_pair_accuracy \
|
| 601 |
+
--metric latency_seconds \
|
| 602 |
+
--title 'Rendered Claim Pack V2 Holdout - Sample-Level Oracle'
|
| 603 |
+
```
|
| 604 |
+
|
| 605 |
+
```bash
|
| 606 |
+
ocrtech benchmark-error-analysis \
|
| 607 |
+
--benchmark-report outputs/live-k2-claimpack-v2-line-align/report.json \
|
| 608 |
+
--out outputs/paper-claimpack-v2-surya-error-analysis \
|
| 609 |
+
--system surya \
|
| 610 |
+
--metric cer \
|
| 611 |
+
--top-n 15
|
| 612 |
+
|
| 613 |
+
ocrtech benchmark-error-analysis \
|
| 614 |
+
--benchmark-report outputs/live-k2-claimpack-v2-line-align/report.json \
|
| 615 |
+
--out outputs/paper-claimpack-v2-surya-vs-tesseract-error-analysis \
|
| 616 |
+
--system surya \
|
| 617 |
+
--baseline tesseract \
|
| 618 |
+
--metric cer \
|
| 619 |
+
--top-n 15
|
| 620 |
+
|
| 621 |
+
ocrtech benchmark-error-analysis \
|
| 622 |
+
--benchmark-report outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-report.json \
|
| 623 |
+
--out outputs/paper-claimpack-v2-surya-vs-oracle-error-analysis \
|
| 624 |
+
--system surya \
|
| 625 |
+
--baseline oracle-surya-tesseract \
|
| 626 |
+
--metric cer \
|
| 627 |
+
--top-n 10
|
| 628 |
+
|
| 629 |
+
ocrtech export-failure-manifest \
|
| 630 |
+
--source-manifest data/splits-claimpack-rendered-v2/eval.jsonl \
|
| 631 |
+
--top-failures outputs/paper-claimpack-v2-surya-error-analysis/top-failures.json \
|
| 632 |
+
--out data/splits-claimpack-rendered-v2/eval-surya-hard15.jsonl \
|
| 633 |
+
--split hard_eval
|
| 634 |
+
|
| 635 |
+
ocrtech validate-claim \
|
| 636 |
+
--benchmark-report outputs/live-k2-claimpack-v2-line-align/report.json \
|
| 637 |
+
--config configs/validation-hard15-surya.json \
|
| 638 |
+
--eval-manifest data/splits-claimpack-rendered-v2/eval-surya-hard15.jsonl \
|
| 639 |
+
--out outputs/validation-claimpack-v2-line-align-hard15
|
| 640 |
+
|
| 641 |
+
ocrtech create-eval-pack \
|
| 642 |
+
--out data/eval-pack-table-figure-dev-v1 \
|
| 643 |
+
--input-format image \
|
| 644 |
+
--template table-ledger \
|
| 645 |
+
--template figure-caption \
|
| 646 |
+
--count-per-template 8 \
|
| 647 |
+
--variant-offset 100 \
|
| 648 |
+
--degradation clean \
|
| 649 |
+
--degradation scan \
|
| 650 |
+
--degradation blur \
|
| 651 |
+
--degradation low_contrast \
|
| 652 |
+
--degradation skew
|
| 653 |
+
```
|
| 654 |
+
|
| 655 |
+
Memory still is not part of these generated tables. The current benchmark artifact format does not record it.
|
| 656 |
+
|
| 657 |
+
## Hard15 Figure-Caption + Table-Lexicon Candidate
|
| 658 |
+
|
| 659 |
+
Latest artifact set:
|
| 660 |
+
|
| 661 |
+
- `outputs/figure-caption-table-lexicon-v1/model-card.json`
|
| 662 |
+
- `outputs/experiment-figure-caption-table-lexicon-hard15-v3/benchmark/report.json`
|
| 663 |
+
- `outputs/summary-figure-caption-table-lexicon-hard15-v3/summary.md`
|
| 664 |
+
- `outputs/table-cell-analysis-figure-caption-table-lexicon-hard15-v3/table-cell-analysis.md`
|
| 665 |
+
- `outputs/validation-figure-caption-table-lexicon-hard15-v3/validation.md`
|
| 666 |
+
|
| 667 |
+
Headline table for internal hard15 only:
|
| 668 |
+
|
| 669 |
+
| system | CER | figure_caption_cer | figure_detection_f1 | table_cell_f1 | reading_order_pair_accuracy |
|
| 670 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 671 |
+
| candidate | 0.010671 | 0.008746 | 1.000000 | 1.000000 | 0.520000 |
|
| 672 |
+
| Surya | 0.236839 | 0.092323 | 1.000000 | 0.979167 | 0.520000 |
|
| 673 |
+
| Tesseract | 1.266364 | 0.051020 | 0.857143 | 0.000000 | 0.613333 |
|
| 674 |
+
|
| 675 |
+
Validation status is still `not_validated`.
|
| 676 |
+
|
| 677 |
+
Passed configured Surya gates:
|
| 678 |
+
|
| 679 |
+
- overall hard_eval CER: `15` wins / `0` losses / `0` ties
|
| 680 |
+
- table-slice CER: `8` wins / `0` losses / `0` ties
|
| 681 |
+
- figure-slice CER: `7` wins / `0` losses / `0` ties
|
| 682 |
+
- figure-caption CER: `7` wins / `0` losses / `0` ties
|
| 683 |
+
|
| 684 |
+
Remaining configured failure:
|
| 685 |
+
|
| 686 |
+
- table-cell F1: candidate `1.000000`, Surya `0.979167`, but bootstrap CI low is `0.000000` because there are only 2 improving table pages and 6 ties
|
| 687 |
+
|
| 688 |
+
Cell-level table evidence:
|
| 689 |
+
|
| 690 |
+
| system | reference_cells | exact_cells | exact_rate | mean_cer |
|
| 691 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 692 |
+
| candidate | 96 | 96 | 1.000000 | 0.000000 |
|
| 693 |
+
| Surya | 96 | 94 | 0.979167 | 0.010417 |
|
| 694 |
+
| Tesseract | 96 | 0 | 0.000000 | 1.000000 |
|
| 695 |
+
|
| 696 |
+
Use this result as an internal milestone, not a paper claim of broad SOTA. The candidate is lexicon-assisted for table names and uses guarded structural-title corrections (`वार्षिक प्रतिवेदन`, `दैनिक हिसाब`, `चित्र`, `निष्कर्ष`). A publishable table claim needs a larger independent table pack so confidence intervals are meaningful.
|
| 697 |
+
|
| 698 |
+
## Targeted Table-Ledger Validation Pack V2
|
| 699 |
+
|
| 700 |
+
Artifacts:
|
| 701 |
+
|
| 702 |
+
- `data/eval-pack-table-ledger-claim-v1/manifests/ocrtech-eval-pack.jsonl`
|
| 703 |
+
- `outputs/audit-table-ledger-claim-v1/eval-pack-audit.md`
|
| 704 |
+
- `outputs/experiment-table-ledger-claim-v2/benchmark/report.json`
|
| 705 |
+
- `outputs/summary-table-ledger-claim-v2/summary.md`
|
| 706 |
+
- `outputs/table-cell-analysis-table-ledger-claim-v2/table-cell-analysis.md`
|
| 707 |
+
- `outputs/validation-table-ledger-claim-v2b/validation.md`
|
| 708 |
+
- `configs/validation-table-ledger-surya.json`
|
| 709 |
+
|
| 710 |
+
Pack:
|
| 711 |
+
|
| 712 |
+
- template: `table-ledger`
|
| 713 |
+
- samples: `60`
|
| 714 |
+
- cells: `720`
|
| 715 |
+
- degradations: clean, scan, blur, low_contrast, skew
|
| 716 |
+
- variant offset: `300`
|
| 717 |
+
- claim-ready broad coverage: no, targeted table-only pack
|
| 718 |
+
|
| 719 |
+
Validation status: `validated`.
|
| 720 |
+
|
| 721 |
+
| metric | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
|
| 722 |
+
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
| 723 |
+
| table_cell_f1 | 60 | 15 | 0 | 45 | 1.000000 | 0.979167 | 0.020833 | 0.012500 |
|
| 724 |
+
| table_exact_match | 60 | 15 | 0 | 45 | 1.000000 | 0.750000 | 0.250000 | 0.150000 |
|
| 725 |
+
| table_structure_similarity | 60 | 0 | 0 | 60 | 1.000000 | 1.000000 | 0.000000 | 0.000000 |
|
| 726 |
+
|
| 727 |
+
Cell exactness:
|
| 728 |
+
|
| 729 |
+
| system | reference_cells | exact_cells | exact_rate | mean_cell_cer |
|
| 730 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 731 |
+
| candidate | 720 | 720 | 1.000000 | 0.000000 |
|
| 732 |
+
| Surya | 720 | 705 | 0.979167 | 0.005208 |
|
| 733 |
+
| Tesseract | 720 | 0 | 0.000000 | 0.999868 |
|
| 734 |
+
|
| 735 |
+
Runtime on k2:
|
| 736 |
+
|
| 737 |
+
| system | mean latency seconds | median latency seconds |
|
| 738 |
+
| --- | ---: | ---: |
|
| 739 |
+
| candidate | 3.017929 | 2.967943 |
|
| 740 |
+
| Surya | 2.478481 | 2.453125 |
|
| 741 |
+
| Tesseract | 0.146866 | 0.134735 |
|
| 742 |
+
|
| 743 |
+
Paper-safe claim text:
|
| 744 |
+
|
| 745 |
+
> On a targeted 60-page generated Nepali-English table-ledger validation pack, the lexicon-assisted candidate improved structured table extraction over Surya, reaching 720/720 exact cells versus Surya's 705/720, and passed bootstrap-gated validation for both table-cell F1 and table exact match.
|
| 746 |
+
|
| 747 |
+
More precise paper-safe wording:
|
| 748 |
+
|
| 749 |
+
> On a targeted 60-page generated Nepali-English table-ledger validation pack, the lexicon-assisted candidate improved table cell-content extraction over Surya, reaching 720/720 exact cells versus Surya's 705/720, while matching Surya's perfect grid-structure score.
|
| 750 |
+
|
| 751 |
+
Do not generalize this to all tables. The pack has no merged cells, no nested headers, and no real scanned government tables. The scoring path now supports span-aware structured references, but this pack does not exercise that capability and the metric is still not true HTML-tree TEDS.
|
| 752 |
+
|
| 753 |
+
## Targeted Complex-Table Validation Pack V1
|
| 754 |
+
|
| 755 |
+
Artifacts:
|
| 756 |
+
|
| 757 |
+
- `data/eval-pack-complex-table-v1/manifests/ocrtech-eval-pack.jsonl`
|
| 758 |
+
- `outputs/audit-complex-table-v1/eval-pack-audit.md`
|
| 759 |
+
- `outputs/experiment-complex-table-v1/benchmark/report.json`
|
| 760 |
+
- `outputs/summary-complex-table-v1/summary.md`
|
| 761 |
+
- `outputs/table-cell-analysis-complex-table-v1/table-cell-analysis.md`
|
| 762 |
+
- `outputs/validation-complex-table-v1/validation.md`
|
| 763 |
+
- `configs/validation-complex-table-surya.json`
|
| 764 |
+
|
| 765 |
+
Pack:
|
| 766 |
+
|
| 767 |
+
- template: `complex-table-ledger`
|
| 768 |
+
- samples: `40`
|
| 769 |
+
- reference cells: `600`
|
| 770 |
+
- span reference: merged header with `colspan=2`
|
| 771 |
+
- degradations: clean, scan, blur, low_contrast
|
| 772 |
+
- broad claim-ready coverage: no
|
| 773 |
+
|
| 774 |
+
Validation status: `validated` for cell-content improvement plus structure non-regression.
|
| 775 |
+
|
| 776 |
+
| metric | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
|
| 777 |
+
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
| 778 |
+
| table_cell_f1 | 40 | 8 | 0 | 32 | 0.928333 | 0.915000 | 0.013333 | 0.005000 |
|
| 779 |
+
| table_exact_match | 40 | 0 | 0 | 40 | 0.000000 | 0.000000 | 0.000000 | 0.000000 |
|
| 780 |
+
| table_structure_similarity | 40 | 0 | 0 | 40 | 0.875000 | 0.875000 | 0.000000 | 0.000000 |
|
| 781 |
+
|
| 782 |
+
Cell exactness:
|
| 783 |
+
|
| 784 |
+
| system | reference_cells | exact_cells | exact_rate | mean_cell_cer |
|
| 785 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 786 |
+
| candidate | 600 | 557 | 0.928333 | 0.067667 |
|
| 787 |
+
| Surya | 600 | 549 | 0.915000 | 0.071000 |
|
| 788 |
+
| Tesseract | 600 | 40 | 0.066667 | 0.933333 |
|
| 789 |
+
|
| 790 |
+
Paper-safe claim text:
|
| 791 |
+
|
| 792 |
+
> On a targeted generated complex-table pack with span-aware references, the lexicon-assisted candidate improved table cell-content F1 over Surya but did not improve merged-header reconstruction; both systems failed exact table match and tied on span-aware structure similarity.
|
| 793 |
+
|
| 794 |
+
This is a negative result for complex table structure. The next model/parser branch must preserve or infer rowspans/colspans before we can claim complex-table document understanding.
|
| 795 |
+
|
| 796 |
+
## Targeted Complex-Table Span-Inference Pack V1
|
| 797 |
+
|
| 798 |
+
After adding conservative repeated-header colspan inference, we reran the same `40`-sample generated complex-table pack on k2.
|
| 799 |
+
|
| 800 |
+
Artifacts:
|
| 801 |
+
|
| 802 |
+
- `outputs/experiment-complex-table-span-infer-v1/benchmark/report.json`
|
| 803 |
+
- `outputs/experiment-complex-table-span-infer-v1/validation/validation.md`
|
| 804 |
+
- `outputs/summary-complex-table-span-infer-v1/summary.md`
|
| 805 |
+
- `outputs/table-cell-analysis-complex-table-span-infer-v1/table-cell-analysis.md`
|
| 806 |
+
|
| 807 |
+
Validation status: `validated`.
|
| 808 |
+
|
| 809 |
+
| metric | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
|
| 810 |
+
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
| 811 |
+
| table_cell_f1 | 40 | 40 | 0 | 0 | 0.995000 | 0.915000 | 0.080000 | 0.071667 |
|
| 812 |
+
| table_exact_match | 40 | 37 | 0 | 3 | 0.925000 | 0.000000 | 0.925000 | 0.825000 |
|
| 813 |
+
| table_structure_similarity | 40 | 40 | 0 | 0 | 1.000000 | 0.875000 | 0.125000 | 0.125000 |
|
| 814 |
+
|
| 815 |
+
Cell exactness:
|
| 816 |
+
|
| 817 |
+
| system | reference_cells | exact_cells | exact_rate | mean_cell_cer |
|
| 818 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 819 |
+
| candidate | 600 | 597 | 0.995000 | 0.001000 |
|
| 820 |
+
| Surya | 600 | 549 | 0.915000 | 0.071000 |
|
| 821 |
+
| Tesseract | 600 | 40 | 0.066667 | 0.933333 |
|
| 822 |
+
|
| 823 |
+
Paper-safe claim text:
|
| 824 |
+
|
| 825 |
+
> On a targeted generated complex-table pack with one repeated merged-header pattern, a conservative span-inference postprocessor improved table structure extraction over Surya, raising span-aware structure similarity from 0.875 to 1.000 and table exact match from 0.000 to 0.925.
|
| 826 |
+
|
| 827 |
+
Do not generalize this to all complex tables. This is generated data with a single span pattern and a heuristic parser repair. The next paper-grade check needs real scanned tables with varied merged cells, rowspans, nested headers, and a true HTML/TEDS scorer.
|
| 828 |
+
|
| 829 |
+
## Targeted Complex-Table TEDS-Style Metrics V1
|
| 830 |
+
|
| 831 |
+
We added deterministic ordered table-tree edit metrics and reran the same complex-table pack with these gates included.
|
| 832 |
+
|
| 833 |
+
Artifacts:
|
| 834 |
+
|
| 835 |
+
- `outputs/experiment-complex-table-teds-v1/benchmark/report.json`
|
| 836 |
+
- `outputs/experiment-complex-table-teds-v1/validation/validation.md`
|
| 837 |
+
- `outputs/summary-complex-table-teds-v1/summary.md`
|
| 838 |
+
- `outputs/table-cell-analysis-complex-table-teds-v1/table-cell-analysis.md`
|
| 839 |
+
|
| 840 |
+
Validation status: `validated`.
|
| 841 |
+
|
| 842 |
+
| metric | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
|
| 843 |
+
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
| 844 |
+
| table_cell_f1 | 40 | 40 | 0 | 0 | 0.995000 | 0.915000 | 0.080000 | 0.071667 |
|
| 845 |
+
| table_exact_match | 40 | 37 | 0 | 3 | 0.925000 | 0.000000 | 0.925000 | 0.825000 |
|
| 846 |
+
| table_structure_similarity | 40 | 40 | 0 | 0 | 1.000000 | 0.875000 | 0.125000 | 0.125000 |
|
| 847 |
+
| table_teds_structure | 40 | 40 | 0 | 0 | 1.000000 | 0.904762 | 0.095238 | 0.095238 |
|
| 848 |
+
| table_teds_text | 40 | 40 | 0 | 0 | 0.996250 | 0.891667 | 0.104583 | 0.098690 |
|
| 849 |
+
|
| 850 |
+
Paper-safe claim text:
|
| 851 |
+
|
| 852 |
+
> On a targeted generated complex-table pack, the candidate improved ordered table-tree metrics over Surya, reaching 1.000 structure-only TEDS-style similarity and 0.996 text-sensitive TEDS-style similarity, compared with Surya's 0.905 and 0.892 respectively.
|
| 853 |
+
|
| 854 |
+
This is still not a public-benchmark SOTA claim. The metrics are deterministic and closer to table-structure scoring, but the pack is generated and covers one repeated-header merged-cell pattern.
|
docs/PREFLIGHT.md
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Claim Preflight
|
| 2 |
+
|
| 3 |
+
`preflight-claim` is the gate before any expensive benchmark or training-adjacent claim run. Use it to fail on missing references, bad eval packs, missing candidate model cards, missing baseline dependencies, invalid validation configs, train/eval leakage, and tuned-on-eval routed candidates before you burn remote machine time.
|
| 4 |
+
|
| 5 |
+
## Claim-Grade Run
|
| 6 |
+
|
| 7 |
+
```bash
|
| 8 |
+
ocrtech preflight-claim \
|
| 9 |
+
--eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
|
| 10 |
+
--references-dir data/eval-pack/references \
|
| 11 |
+
--baselines candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
|
| 12 |
+
--candidate-model-config runs/recognizer/package/model-card.json \
|
| 13 |
+
--fallback-model-config runs/vlm-fallback/package/model-card.json \
|
| 14 |
+
--require-trained-recognizer \
|
| 15 |
+
--require-model-admission \
|
| 16 |
+
--validation-config validation.json \
|
| 17 |
+
--train-manifest data/splits/train.jsonl \
|
| 18 |
+
--out outputs/preflight
|
| 19 |
+
```
|
| 20 |
+
|
| 21 |
+
This exits with `0` only when every required check passes.
|
| 22 |
+
|
| 23 |
+
## Smoke Pack Exception
|
| 24 |
+
|
| 25 |
+
```bash
|
| 26 |
+
ocrtech preflight-claim \
|
| 27 |
+
--eval-manifest data/eval-pack-smoke/manifests/ocrtech-eval-pack.jsonl \
|
| 28 |
+
--references-dir data/eval-pack-smoke/references \
|
| 29 |
+
--baselines ours \
|
| 30 |
+
--candidate-model-config runs/fixtures/sidecar-model.json \
|
| 31 |
+
--allow-smoke-eval-pack \
|
| 32 |
+
--out outputs/preflight-smoke
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
`--allow-smoke-eval-pack` only downgrades the non-image eval-pack check. It does not hide missing references, invalid model cards, missing baseline hooks, or leakage.
|
| 36 |
+
|
| 37 |
+
## Outputs
|
| 38 |
+
|
| 39 |
+
```text
|
| 40 |
+
outputs/preflight/
|
| 41 |
+
preflight.json
|
| 42 |
+
preflight.md
|
| 43 |
+
eval-pack-audit/eval-pack-audit.json
|
| 44 |
+
eval-pack-audit/eval-pack-audit.md
|
| 45 |
+
```
|
| 46 |
+
|
| 47 |
+
## Checks
|
| 48 |
+
|
| 49 |
+
`preflight-claim` currently verifies:
|
| 50 |
+
|
| 51 |
+
- eval manifest loads and every input file exists
|
| 52 |
+
- image/PDF claim inputs use OCR-supported suffixes; text sidecar smoke inputs
|
| 53 |
+
are allowed only when `metadata.input_format = text`
|
| 54 |
+
- references resolve and parse for every sample
|
| 55 |
+
- ocrtech eval packs are claim-ready unless `--allow-smoke-eval-pack` is set
|
| 56 |
+
- `ours` or `candidate` baselines have a passing candidate model card
|
| 57 |
+
- configured fallback model cards pass `audit-model` before the benchmark starts
|
| 58 |
+
- when a fallback model is configured, the candidate model card must also
|
| 59 |
+
declare the same fallback card in `provenance.fallback_model_card`; preflight
|
| 60 |
+
hashes both cards and fails if they differ
|
| 61 |
+
- claim runs can require trained-recognizer provenance with
|
| 62 |
+
`--require-trained-recognizer`; this checks for recorded trained artifacts and
|
| 63 |
+
hash-verified train/eval manifest and training-summary provenance, or a
|
| 64 |
+
nested `provenance.trained_recognizer_model_card` in composite candidates
|
| 65 |
+
- claim runs can require model admission validation with
|
| 66 |
+
`--require-model-admission`; this checks the candidate model card's
|
| 67 |
+
`metrics.admission_validation` report and requires `claim_status=validated`.
|
| 68 |
+
It also requires quantitative passing gate evidence in that validation JSON.
|
| 69 |
+
When the report is rejected, preflight now surfaces the failed requirements
|
| 70 |
+
and gate deltas from the validation JSON, including baseline, metric, slice,
|
| 71 |
+
paired means, and win/loss/tie counts. The admission report must also include
|
| 72 |
+
`provenance.candidate_model_config`; preflight compares the validated model
|
| 73 |
+
identity and artifact hashes against the current candidate model card so a
|
| 74 |
+
passing report from a different recognizer cannot be reused.
|
| 75 |
+
- routed candidates such as `quality_select_composite`, `quality_ranked_ensemble`, and `script_select_composite` have calibration provenance, and that calibration eval manifest is not the same as the current claim eval manifest
|
| 76 |
+
- routed candidates also fail if their selected calibration trial collapses to one engine for every calibration sample, because that is not a meaningful routed candidate for claim purposes
|
| 77 |
+
- routed candidates also fail unless the claim-grade benchmark includes their raw component baselines, so distinctness can be measured in the reviewed evidence
|
| 78 |
+
- `tesseract` is on `PATH` when requested
|
| 79 |
+
- `paddleocr` is importable when stock PaddleOCR is requested
|
| 80 |
+
- `OCRTECH_GLM_OCR_CMD` is set, shell-parseable, contains both `{input}` and `{out}`, and points at a runnable executable when `glm-ocr` is requested
|
| 81 |
+
- `OCRTECH_PADDLEOCR_VL_CMD` is set, shell-parseable, contains both `{input}` and `{out}`, and points at a runnable executable when `paddleocr-vl` is requested
|
| 82 |
+
- validation config loads, contains gates, names baselines that are scheduled
|
| 83 |
+
for the run, and can be satisfied by the eval manifest's sample and slice
|
| 84 |
+
counts
|
| 85 |
+
- train/eval leakage passes when train manifests are supplied
|
| 86 |
+
|
| 87 |
+
Use this before `run-experiment`. It is cheaper to fail here than after a half-day benchmark on a broken setup.
|
| 88 |
+
|
| 89 |
+
For a single guarded command, `run-experiment` can run the same preflight before
|
| 90 |
+
benchmarking:
|
| 91 |
+
|
| 92 |
+
```bash
|
| 93 |
+
ocrtech run-experiment \
|
| 94 |
+
--eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
|
| 95 |
+
--references-dir data/eval-pack/references \
|
| 96 |
+
--baselines candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
|
| 97 |
+
--candidate-model-config runs/recognizer/package/model-card.json \
|
| 98 |
+
--validation-config validation.json \
|
| 99 |
+
--train-manifest data/splits/train.jsonl \
|
| 100 |
+
--preflight \
|
| 101 |
+
--require-trained-recognizer \
|
| 102 |
+
--require-model-admission \
|
| 103 |
+
--out outputs/claim-experiment \
|
| 104 |
+
--require-validated
|
| 105 |
+
```
|
| 106 |
+
|
| 107 |
+
When embedded preflight fails, `run-experiment` writes
|
| 108 |
+
`outputs/claim-experiment/preflight/preflight.json` and stops before creating a
|
| 109 |
+
benchmark report.
|
| 110 |
+
|
| 111 |
+
When `--validation-config` is omitted, preflight emits a warning instead of
|
| 112 |
+
claiming the default gates are appropriate for the run. For claim-grade work,
|
| 113 |
+
always pass the slot-specific validation config. When a config is supplied,
|
| 114 |
+
preflight rejects impossible setups before benchmark time: `min_samples` larger
|
| 115 |
+
than the manifest, required slices absent from the manifest, gate `min_pairs`
|
| 116 |
+
larger than the matching slice intersection, or gate baselines that are not in
|
| 117 |
+
`--baselines`.
|
| 118 |
+
|
| 119 |
+
If the `eval-input-suffixes` check fails on files such as `.img`, normalize the
|
| 120 |
+
manifest before rerunning preflight:
|
| 121 |
+
|
| 122 |
+
```bash
|
| 123 |
+
ocrtech normalize-manifest-images \
|
| 124 |
+
data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 125 |
+
--out data/splits-mixed-v1/eval-normalized-images
|
| 126 |
+
|
| 127 |
+
ocrtech materialize-references \
|
| 128 |
+
data/splits-mixed-v1/eval-normalized-images/manifest-images-normalized.jsonl \
|
| 129 |
+
--out data/references-mixed-v1-normalized \
|
| 130 |
+
--rewritten-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl
|
| 131 |
+
```
|
| 132 |
+
|
| 133 |
+
For routed thresholded candidates, the intended behavior is now:
|
| 134 |
+
|
| 135 |
+
- calibrate on one experiment manifest with `ocrtech calibrate-quality-router`
|
| 136 |
+
- or `ocrtech calibrate-script-router`
|
| 137 |
+
- benchmark with `candidate` plus the candidate's raw component baselines in the same run
|
| 138 |
+
- benchmark on a different eval manifest
|
| 139 |
+
- expect `candidate-calibration=pass` only on that different eval manifest
|
| 140 |
+
|
| 141 |
+
On `k2`, the same-pack check was verified against a real routed Tesseract calibration artifact: preflight failed on the calibration manifest itself and passed the calibration-separation check on the untouched holdout manifest.
|
| 142 |
+
|
| 143 |
+
On the larger rendered `claimpack-rendered-v2` holdout, a second real case now exists: the calibrated script router selected `primary_script_threshold=0.0`, which meant `secondary_picks=50/50` on calibration. After syncing the new guardrail to `k2`, `preflight-claim` correctly failed that model with:
|
| 144 |
+
|
| 145 |
+
- `script_select_composite collapses to the secondary engine on every calibration sample`
|
| 146 |
+
|
| 147 |
+
That is the intended behavior. A routed candidate that reduces to "always use Surya" is not evidence of a novel better engine.
|
docs/REMOTE_TRAINING.md
ADDED
|
@@ -0,0 +1,1404 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Remote Training
|
| 2 |
+
|
| 3 |
+
Target host:
|
| 4 |
+
|
| 5 |
+
```text
|
| 6 |
+
k2@100.97.238.103
|
| 7 |
+
```
|
| 8 |
+
|
| 9 |
+
Do not store the SSH password in this repository. Use a local secret, `ssh-agent`, `sshpass -e`, or a one-off `expect` wrapper outside version control.
|
| 10 |
+
|
| 11 |
+
## Current Host Check
|
| 12 |
+
|
| 13 |
+
As of the last verified setup from this workspace:
|
| 14 |
+
|
| 15 |
+
- SSH login works.
|
| 16 |
+
- `~/ocr-tech` exists and is synced from the local workspace.
|
| 17 |
+
- `~/ocr-tech/.venv/bin/python` is `3.11.14`.
|
| 18 |
+
- `uv` is on `PATH` at `~/.local/bin/uv`.
|
| 19 |
+
- Home directory free space is about `352 GiB`.
|
| 20 |
+
- `nvidia-smi` is still not available, so do not claim NVIDIA-backed training there.
|
| 21 |
+
- The machine is `macOS 14.6 arm64` on Apple Silicon.
|
| 22 |
+
|
| 23 |
+
The host is valid for Python 3.11 CPU-side runs and repo automation. It is not an officially supported Paddle training target: the PaddlePaddle installation guide says macOS support requires `x86_64`, not `arm64`. Use `--require-paddle-training` in the audit if you need a host that can honestly be called a PaddleOCR training box.
|
| 24 |
+
|
| 25 |
+
That restriction is narrower than “no Paddle at all.” As of June 14, 2026, `paddlepaddle 3.3.1`, `paddleocr 3.7.0`, and `paddlex 3.7.1` do install and run for inference on this Apple Silicon host. The repo can benchmark a stock-Paddle baseline on `k2`. What is still not honest is calling `k2` a supported PaddleOCR training machine.
|
| 26 |
+
|
| 27 |
+
## Audit Command
|
| 28 |
+
|
| 29 |
+
Use the built-in audit before syncing data or starting training:
|
| 30 |
+
|
| 31 |
+
```bash
|
| 32 |
+
export OCRTECH_REMOTE_PASSWORD='...'
|
| 33 |
+
|
| 34 |
+
ocrtech audit-remote-host \
|
| 35 |
+
--host 100.97.238.103 \
|
| 36 |
+
--user k2 \
|
| 37 |
+
--password-env OCRTECH_REMOTE_PASSWORD \
|
| 38 |
+
--workdir '~/ocr-tech' \
|
| 39 |
+
--require-path '~/ocr-tech' \
|
| 40 |
+
--require-paddle-training \
|
| 41 |
+
--min-python 3.11 \
|
| 42 |
+
--min-free-gb 20 \
|
| 43 |
+
--out outputs/remote-audit
|
| 44 |
+
```
|
| 45 |
+
|
| 46 |
+
This writes:
|
| 47 |
+
|
| 48 |
+
```text
|
| 49 |
+
outputs/remote-audit/
|
| 50 |
+
remote-audit.json
|
| 51 |
+
remote-audit.md
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
If the audit fails on Python, disk, platform support, or required commands, fix the host first. Do not push large datasets to a host that already fails the audit.
|
| 55 |
+
|
| 56 |
+
The audit now checks multiple interpreter candidates such as `python3.11`, `python3.12`, `python3.13`, and `~/.local/bin/python3.11`. Do not assume `/usr/bin/python3` reflects the usable training interpreter on macOS hosts.
|
| 57 |
+
|
| 58 |
+
## Bootstrap Command
|
| 59 |
+
|
| 60 |
+
Once the audit passes on Python and disk, provision the remote workdir and virtualenv:
|
| 61 |
+
|
| 62 |
+
```bash
|
| 63 |
+
ocrtech bootstrap-remote-host \
|
| 64 |
+
--host 100.97.238.103 \
|
| 65 |
+
--user k2 \
|
| 66 |
+
--password-env OCRTECH_REMOTE_PASSWORD \
|
| 67 |
+
--workdir '~/ocr-tech' \
|
| 68 |
+
--out outputs/remote-bootstrap
|
| 69 |
+
```
|
| 70 |
+
|
| 71 |
+
This creates:
|
| 72 |
+
|
| 73 |
+
- `~/ocr-tech`
|
| 74 |
+
- `~/ocr-tech/.venv`
|
| 75 |
+
|
| 76 |
+
It does not sync the repository contents. It only prepares the target path and interpreter environment so the next sync and install step has a valid home.
|
| 77 |
+
|
| 78 |
+
If the host already has a `.venv` built with the wrong interpreter, recreate it explicitly:
|
| 79 |
+
|
| 80 |
+
```bash
|
| 81 |
+
ocrtech bootstrap-remote-host \
|
| 82 |
+
--host 100.97.238.103 \
|
| 83 |
+
--user k2 \
|
| 84 |
+
--password-env OCRTECH_REMOTE_PASSWORD \
|
| 85 |
+
--workdir '~/ocr-tech' \
|
| 86 |
+
--python-candidate '~/.local/bin/python3.11' \
|
| 87 |
+
--recreate-venv \
|
| 88 |
+
--out outputs/remote-bootstrap
|
| 89 |
+
```
|
| 90 |
+
|
| 91 |
+
## Sync Workspace
|
| 92 |
+
|
| 93 |
+
Copy the current workspace after bootstrap:
|
| 94 |
+
|
| 95 |
+
```bash
|
| 96 |
+
ocrtech sync-remote-workspace \
|
| 97 |
+
--host 100.97.238.103 \
|
| 98 |
+
--user k2 \
|
| 99 |
+
--password-env OCRTECH_REMOTE_PASSWORD \
|
| 100 |
+
--local-root . \
|
| 101 |
+
--remote-workdir '~/ocr-tech' \
|
| 102 |
+
--exclude .venv \
|
| 103 |
+
--exclude outputs \
|
| 104 |
+
--exclude runs \
|
| 105 |
+
--out outputs/remote-sync
|
| 106 |
+
```
|
| 107 |
+
|
| 108 |
+
This uses a temporary tar archive and remote extract. It is intentionally simple and does not delete remote files that no longer exist locally.
|
| 109 |
+
|
| 110 |
+
Keep `--remote-workdir` quoted when it contains `~`. If you pass `~/ocr-tech` unquoted, your local shell expands it before `ocrtech` sees the argument, which is the wrong machine.
|
| 111 |
+
|
| 112 |
+
## Expected Setup
|
| 113 |
+
|
| 114 |
+
On `k2`, provision a work directory with enough free disk:
|
| 115 |
+
|
| 116 |
+
```bash
|
| 117 |
+
mkdir -p ~/ocr-tech
|
| 118 |
+
cd ~/ocr-tech
|
| 119 |
+
python3 -m venv .venv
|
| 120 |
+
. .venv/bin/activate
|
| 121 |
+
python -m pip install --upgrade pip
|
| 122 |
+
python -m pip install -e '.[datasets,engines]'
|
| 123 |
+
```
|
| 124 |
+
|
| 125 |
+
If Python 3.11+ is not available, install a newer Python first or use a managed environment. The package targets Python 3.11+.
|
| 126 |
+
|
| 127 |
+
## Sync
|
| 128 |
+
|
| 129 |
+
From the local workspace:
|
| 130 |
+
|
| 131 |
+
```bash
|
| 132 |
+
rsync -az --delete \
|
| 133 |
+
--exclude .venv \
|
| 134 |
+
--exclude .pytest_cache \
|
| 135 |
+
--exclude outputs \
|
| 136 |
+
--exclude runs \
|
| 137 |
+
./ k2@100.97.238.103:~/ocr-tech/
|
| 138 |
+
```
|
| 139 |
+
|
| 140 |
+
Never sync a local `.venv` to a Linux training host. A macOS virtualenv can
|
| 141 |
+
overwrite the remote console scripts with bad interpreter paths and force a
|
| 142 |
+
full remote venv repair.
|
| 143 |
+
|
| 144 |
+
If password auth is required and `sshpass` is installed locally, prefer `SSHPASS` over putting the password in command history:
|
| 145 |
+
|
| 146 |
+
```bash
|
| 147 |
+
SSHPASS='...' sshpass -e rsync -az ./ k2@100.97.238.103:~/ocr-tech/
|
| 148 |
+
```
|
| 149 |
+
|
| 150 |
+
## Run Training Recipes
|
| 151 |
+
|
| 152 |
+
Generate manifests and configs locally or remotely:
|
| 153 |
+
|
| 154 |
+
```bash
|
| 155 |
+
ocrtech inspect-hf-dataset gauravgiri/nepali-ocr-dataset --split train --limit 3 --out outputs/hf-inspect-training-v1
|
| 156 |
+
ocrtech inspect-hf-dataset prashant0919/nepali-synthetic-ocr-lines --split train --limit 3 --out outputs/hf-inspect-training-v1
|
| 157 |
+
ocrtech inspect-hf-dataset nvidia/OCR-Synthetic-Multilingual-v1 --hf-subset en --split test --limit 3 --out outputs/hf-inspect-training-v1
|
| 158 |
+
|
| 159 |
+
ocrtech prepare-data gauravgiri/nepali-ocr-dataset --hf --split train --limit 100 --skip-invalid --out data/manifests
|
| 160 |
+
ocrtech prepare-data prashant0919/nepali-synthetic-ocr-lines --hf --split train --limit 100 --out data/manifests
|
| 161 |
+
ocrtech merge-manifests \
|
| 162 |
+
data/manifests/gauravgiri__nepali-ocr-dataset-train.jsonl \
|
| 163 |
+
data/manifests/prashant0919__nepali-synthetic-ocr-lines-train.jsonl \
|
| 164 |
+
--out data/manifests/nepali-mixed-all.jsonl
|
| 165 |
+
ocrtech audit-manifest data/manifests/nepali-mixed-all.jsonl --out outputs/manifest-audit
|
| 166 |
+
ocrtech split-manifest data/manifests/nepali-mixed-all.jsonl --out data/splits
|
| 167 |
+
ocrtech train-recognizer \
|
| 168 |
+
--train-manifest data/splits/train.jsonl \
|
| 169 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 170 |
+
--out runs/recognizer
|
| 171 |
+
```
|
| 172 |
+
|
| 173 |
+
For the current mixed Nepali/English line-recognition branch, the verified v2
|
| 174 |
+
recipe command on `k2` is:
|
| 175 |
+
|
| 176 |
+
```bash
|
| 177 |
+
ocrtech train-recognizer \
|
| 178 |
+
--backend paddleocr \
|
| 179 |
+
--train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
|
| 180 |
+
--eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 181 |
+
--out runs/paddle-recognizer-mixed-v2 \
|
| 182 |
+
--train-batch-size 16 \
|
| 183 |
+
--eval-batch-size 16 \
|
| 184 |
+
--learning-rate 0.0005 \
|
| 185 |
+
--epochs 10
|
| 186 |
+
```
|
| 187 |
+
|
| 188 |
+
After generating a PaddleOCR recipe, verify the invariant that each manifest
|
| 189 |
+
entry maps to exactly one physical label-file line:
|
| 190 |
+
|
| 191 |
+
```bash
|
| 192 |
+
uv run python - <<'PY'
|
| 193 |
+
from pathlib import Path
|
| 194 |
+
from ocrtech.manifest import load_manifest
|
| 195 |
+
|
| 196 |
+
train_count = len(load_manifest("data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl"))
|
| 197 |
+
eval_count = len(load_manifest("data/splits-mixed-v1/eval-with-refs.jsonl"))
|
| 198 |
+
train_lines = sum(1 for _ in Path("runs/paddle-recognizer-mixed-v2/labels/train.txt").open())
|
| 199 |
+
eval_lines = sum(1 for _ in Path("runs/paddle-recognizer-mixed-v2/labels/eval.txt").open())
|
| 200 |
+
assert train_count == train_lines, (train_count, train_lines)
|
| 201 |
+
assert eval_count == eval_lines, (eval_count, eval_lines)
|
| 202 |
+
PY
|
| 203 |
+
```
|
| 204 |
+
|
| 205 |
+
The verified v2 counts are `1700/1700` train and `180/180` eval. Earlier recipe
|
| 206 |
+
generation expanded embedded label newlines and produced too many label lines;
|
| 207 |
+
do not train from any recipe that fails this invariant.
|
| 208 |
+
|
| 209 |
+
Recipe generation also verifies dictionary coverage for the exact normalized
|
| 210 |
+
PaddleOCR labels. If you pass a custom `--dictionary`, generation fails when
|
| 211 |
+
any train/eval label contains a character missing from that dictionary. Do not
|
| 212 |
+
bypass that failure on the remote host; rebuild the dictionary or reject the
|
| 213 |
+
corpus.
|
| 214 |
+
|
| 215 |
+
To move the recipe to a supported PaddleOCR training host, create a portable
|
| 216 |
+
bundle that materializes all referenced images under `bundle_assets/images` and
|
| 217 |
+
rewrites manifests/label files to those portable paths:
|
| 218 |
+
|
| 219 |
+
```bash
|
| 220 |
+
ocrtech bundle-recognizer-training \
|
| 221 |
+
--recipe-dir runs/paddle-recognizer-mixed-v2 \
|
| 222 |
+
--train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
|
| 223 |
+
--eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 224 |
+
--out runs/paddle-recognizer-mixed-v2-bundle \
|
| 225 |
+
--archive-name paddle-recognizer-mixed-v2-training.tar.gz
|
| 226 |
+
```
|
| 227 |
+
|
| 228 |
+
Verified k2 bundle:
|
| 229 |
+
|
| 230 |
+
- archive:
|
| 231 |
+
`runs/paddle-recognizer-mixed-v2-bundle/paddle-recognizer-mixed-v2-training.tar.gz`
|
| 232 |
+
- bundle manifest:
|
| 233 |
+
`runs/paddle-recognizer-mixed-v2-bundle/recognizer-training-bundle.json`
|
| 234 |
+
- files in archive: `1887`
|
| 235 |
+
- materialized images: `1880`
|
| 236 |
+
- uncompressed bytes: `79133479`
|
| 237 |
+
- compressed archive size: `74061239`
|
| 238 |
+
|
| 239 |
+
After extraction on a supported training host, run PaddleOCR training from the
|
| 240 |
+
extraction root:
|
| 241 |
+
|
| 242 |
+
```bash
|
| 243 |
+
python PaddleOCR/tools/train.py -c runs/paddle-recognizer-mixed-v2/paddle_recognizer.yml
|
| 244 |
+
```
|
| 245 |
+
|
| 246 |
+
The verified `k2` starter run on June 14, 2026 produced:
|
| 247 |
+
|
| 248 |
+
- `97` valid `gauravgiri` samples and `3` explicit rejects
|
| 249 |
+
- `100` valid `prashant0919` samples
|
| 250 |
+
- a merged `197`-sample corpus
|
| 251 |
+
- a leakage-safe `167` train / `30` eval split
|
| 252 |
+
- `runs/recognizer-nepali-starter/paddle_recognizer.yml`
|
| 253 |
+
|
| 254 |
+
The verified `k2` schema inspection on June 15, 2026 produced:
|
| 255 |
+
|
| 256 |
+
- `outputs/hf-inspect-training-v1/gauravgiri__nepali-ocr-dataset-train-inspect.json`
|
| 257 |
+
- `outputs/hf-inspect-training-v1/prashant0919__nepali-synthetic-ocr-lines-train-inspect.json`
|
| 258 |
+
- `outputs/hf-inspect-training-v1/nvidia__OCR-Synthetic-Multilingual-v1__en-test-inspect.json`
|
| 259 |
+
|
| 260 |
+
All three passed. Keep this inspection step before conversion so dataset schema
|
| 261 |
+
drift is caught as a recorded artifact, not discovered halfway through training.
|
| 262 |
+
|
| 263 |
+
The verified mixed-script HF workflow on the same host later produced:
|
| 264 |
+
|
| 265 |
+
- `97` valid `gauravgiri` Nepali real rows and `3` explicit rejects
|
| 266 |
+
- `100` `prashant0919` Nepali synthetic rows
|
| 267 |
+
- `1000` NVIDIA English synthetic line crops from `--hf-subset en --split test`
|
| 268 |
+
- a merged `1197`-sample mixed corpus
|
| 269 |
+
- a leakage-safe `1017` train / `180` eval split
|
| 270 |
+
- materialized eval references at `data/references-mixed-v1`
|
| 271 |
+
- a rebalanced training manifest with `english=850` and `nepali=850`
|
| 272 |
+
- `runs/hf-recognizer-mixed-v1-rebalanced/final-model`
|
| 273 |
+
- `outputs/experiment-hf-mixed-v1-rebalanced/benchmark/report.json`
|
| 274 |
+
|
| 275 |
+
For `k2` specifically, prefer the Hugging Face recognizer backend over Paddle training when you need a real trainable path on Apple Silicon:
|
| 276 |
+
|
| 277 |
+
```bash
|
| 278 |
+
python -m pip install -e '.[hf-recognizer]'
|
| 279 |
+
|
| 280 |
+
ocrtech train-recognizer \
|
| 281 |
+
--backend hf-vision-encoder-decoder \
|
| 282 |
+
--base-model microsoft/trocr-small-printed \
|
| 283 |
+
--train-manifest data/splits/train.jsonl \
|
| 284 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 285 |
+
--out runs/hf-recognizer
|
| 286 |
+
```
|
| 287 |
+
|
| 288 |
+
This does not make `k2` a valid official Paddle training host. It gives the repo a separate arm64-compatible recognizer path.
|
| 289 |
+
|
| 290 |
+
Current active HF-compatible run on k2:
|
| 291 |
+
|
| 292 |
+
```text
|
| 293 |
+
runs/hf-recognizer-mixed-v2-full
|
| 294 |
+
```
|
| 295 |
+
|
| 296 |
+
Command:
|
| 297 |
+
|
| 298 |
+
```bash
|
| 299 |
+
uv run ocrtech train-recognizer \
|
| 300 |
+
--backend hf-vision-encoder-decoder \
|
| 301 |
+
--base-model microsoft/trocr-small-printed \
|
| 302 |
+
--train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
|
| 303 |
+
--eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 304 |
+
--out runs/hf-recognizer-mixed-v2-full \
|
| 305 |
+
--train-batch-size 4 \
|
| 306 |
+
--eval-batch-size 4 \
|
| 307 |
+
--learning-rate 0.00005 \
|
| 308 |
+
--epochs 3 \
|
| 309 |
+
--run
|
| 310 |
+
```
|
| 311 |
+
|
| 312 |
+
Monitor:
|
| 313 |
+
|
| 314 |
+
```bash
|
| 315 |
+
PID=$(cat runs/hf-recognizer-mixed-v2-full/train.pid)
|
| 316 |
+
ps -p "$PID" -o pid=,stat=,etime=,command=
|
| 317 |
+
tail -f runs/hf-recognizer-mixed-v2-full/train.log
|
| 318 |
+
```
|
| 319 |
+
|
| 320 |
+
Status artifact:
|
| 321 |
+
|
| 322 |
+
```text
|
| 323 |
+
runs/hf-recognizer-mixed-v2-full/train-status.json
|
| 324 |
+
```
|
| 325 |
+
|
| 326 |
+
Do not treat this run as a model result until
|
| 327 |
+
`runs/hf-recognizer-mixed-v2-full/training_metrics.json` and
|
| 328 |
+
`runs/hf-recognizer-mixed-v2-full/final-model/` exist, followed by packaging and
|
| 329 |
+
benchmarking.
|
| 330 |
+
|
| 331 |
+
Latest verified checkpoint:
|
| 332 |
+
|
| 333 |
+
- k2 time: `2026-06-15 03:35:55 +0545`
|
| 334 |
+
- PID: `24642`
|
| 335 |
+
- process: running
|
| 336 |
+
- latest observed log progress: `474/1275` trainer steps
|
| 337 |
+
- `training_metrics.json`: not present
|
| 338 |
+
- `final-model/`: not present
|
| 339 |
+
|
| 340 |
+
The run then completed and produced both `final-model/` and
|
| 341 |
+
`training_metrics.json`.
|
| 342 |
+
|
| 343 |
+
Training metrics:
|
| 344 |
+
|
| 345 |
+
- `eval_loss`: `2.672821283340454`
|
| 346 |
+
- `eval_cer`: `0.4532251781992946`
|
| 347 |
+
- `eval_wer`: `0.6495539645539645`
|
| 348 |
+
- `eval_exact_line_accuracy`: `0.12222222222222222`
|
| 349 |
+
- `epoch`: `3.0`
|
| 350 |
+
|
| 351 |
+
The package/audit step passed:
|
| 352 |
+
|
| 353 |
+
- model card:
|
| 354 |
+
`runs/hf-recognizer-mixed-v2-full/finalized/package/model-card.json`
|
| 355 |
+
- audit:
|
| 356 |
+
`runs/hf-recognizer-mixed-v2-full/finalized/audit/model-audit.json`
|
| 357 |
+
- audit status: pass
|
| 358 |
+
|
| 359 |
+
When `final-model/` and `training_metrics.json` exist, finalize the run before
|
| 360 |
+
benchmarking:
|
| 361 |
+
|
| 362 |
+
```bash
|
| 363 |
+
uv run ocrtech finalize-hf-recognizer-run \
|
| 364 |
+
--run-dir runs/hf-recognizer-mixed-v2-full \
|
| 365 |
+
--out runs/hf-recognizer-mixed-v2-full/finalized \
|
| 366 |
+
--model-id ocrtech-hf-mixed-v2-full \
|
| 367 |
+
--base-model microsoft/trocr-small-printed \
|
| 368 |
+
--train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
|
| 369 |
+
--eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 370 |
+
--training-summary runs/hf-recognizer-mixed-v2-full/recognizer-training-summary.json \
|
| 371 |
+
--admission-validation-report outputs/validation-next-recognizer-hard50/validation.json
|
| 372 |
+
```
|
| 373 |
+
|
| 374 |
+
This command intentionally fails if `final-model/` or `training_metrics.json`
|
| 375 |
+
is missing. If it passes, it writes a packaged model card, model audit, and a
|
| 376 |
+
post-training report with the next benchmark command.
|
| 377 |
+
|
| 378 |
+
The first held-out measurement run completed on k2:
|
| 379 |
+
|
| 380 |
+
- output:
|
| 381 |
+
`outputs/experiment-hf-recognizer-mixed-v2-full-vs-tesseract`
|
| 382 |
+
- baselines: `candidate,tesseract`
|
| 383 |
+
- rows: `360`
|
| 384 |
+
- status: `360` ok, `0` errors
|
| 385 |
+
- candidate CER: `0.4452926496941952`
|
| 386 |
+
- Tesseract CER: `0.3290117066116023`
|
| 387 |
+
- candidate WER: `0.6551095201095201`
|
| 388 |
+
- Tesseract WER: `0.48648528940195623`
|
| 389 |
+
- candidate mean latency: `0.4809882701633291`s
|
| 390 |
+
- Tesseract mean latency: `0.14839679651065832`s
|
| 391 |
+
- validation: skipped because this was a direct branch check, not a claim run
|
| 392 |
+
|
| 393 |
+
Conclusion: the HF TrOCR branch is a documented negative result. Use k2 for
|
| 394 |
+
orchestration and evaluation, but spend serious training time next on the
|
| 395 |
+
PaddleOCR CUDA path.
|
| 396 |
+
|
| 397 |
+
## CUDA PaddleOCR Handoff
|
| 398 |
+
|
| 399 |
+
The next serious recognizer training step needs a CUDA Linux host. k2 is macOS
|
| 400 |
+
arm64, so it should not be described as the claim-grade PaddleOCR training
|
| 401 |
+
machine.
|
| 402 |
+
|
| 403 |
+
Use the current admitted portable bundle as the handoff artifact:
|
| 404 |
+
|
| 405 |
+
```text
|
| 406 |
+
runs/paddle-recognizer-next-v1-bundle/paddle-recognizer-next-v1-training.tar.gz
|
| 407 |
+
```
|
| 408 |
+
|
| 409 |
+
After extracting it on the CUDA host and installing/checking out PaddleOCR, run:
|
| 410 |
+
|
| 411 |
+
```bash
|
| 412 |
+
python PaddleOCR/tools/train.py -c runs/paddle-recognizer-next-v1/paddle_recognizer.yml
|
| 413 |
+
```
|
| 414 |
+
|
| 415 |
+
Bring the exported inference model back into this repo, then package, audit,
|
| 416 |
+
and benchmark it with the same validation flow.
|
| 417 |
+
|
| 418 |
+
The `recognizer-next-v1` bundle was created on `k2` after the updated corpus
|
| 419 |
+
audit passed:
|
| 420 |
+
|
| 421 |
+
- train manifest: `data/splits-recognizer-next-v1/train.jsonl`
|
| 422 |
+
- eval manifest: `data/splits-recognizer-next-v1/eval.jsonl`
|
| 423 |
+
- train/eval samples: `2212` / `244`
|
| 424 |
+
- train Latin/Devanagari/mixed: `1084` / `1174` / `100`
|
| 425 |
+
- eval Latin/Devanagari/mixed: `172` / `90` / `28`
|
| 426 |
+
- train real/synthetic: `428` / `1272`
|
| 427 |
+
- leakage check: pass
|
| 428 |
+
- local mirrored audit:
|
| 429 |
+
`outputs/k2-recognizer-next-v1/recognizer-corpus-audit.json`
|
| 430 |
+
- local mirrored bundle manifest:
|
| 431 |
+
`outputs/k2-recognizer-next-v1/recognizer-training-bundle.json`
|
| 432 |
+
|
| 433 |
+
This is only a training handoff. It is not a quality result. A CUDA-trained
|
| 434 |
+
recognizer still has to clear export, packaging, benchmark, and admission gates
|
| 435 |
+
before it can replace the current document engine candidate.
|
| 436 |
+
|
| 437 |
+
Final outcome: `recognizer-next-v1` was trained on the restarted AWS G6 L4 host,
|
| 438 |
+
exported, packaged, benchmarked on hard-50, and rejected. The host was stopped
|
| 439 |
+
after artifact mirroring.
|
| 440 |
+
|
| 441 |
+
- instance: `i-0fa19dff7b233a353`
|
| 442 |
+
- final state after run: `stopped`
|
| 443 |
+
- model card: `runs/paddle-recognizer-next-v1/package/model-card.json`
|
| 444 |
+
- local slim artifact:
|
| 445 |
+
`outputs/aws-g6-paddle-recognizer-next-v1/ocrtech-paddle-recognizer-next-v1-slim-artifacts.tgz`
|
| 446 |
+
- slim artifact sha256:
|
| 447 |
+
`3558cb36fc63e463650d229cd21e1b91955c208cfcd483e419e89d3e86b309ce`
|
| 448 |
+
- hard-50 candidate CER/WER: `0.964227` / `1.000000`
|
| 449 |
+
- hard-50 Tesseract CER/WER: `0.080249` / `0.258921`
|
| 450 |
+
- paired CER wins/losses/ties: `0` / `50` / `0`
|
| 451 |
+
- admission decision: `reject`
|
| 452 |
+
|
| 453 |
+
Do not relaunch this branch as an engine candidate. The admitted corpus improved
|
| 454 |
+
the training setup but not recognizer quality.
|
| 455 |
+
|
| 456 |
+
Follow-up guardrail: `ocrtech recognizer-failure-review` now aggregates
|
| 457 |
+
recognizer admission failures across runs and exits nonzero when a model family
|
| 458 |
+
should be stopped. The current PaddleOCR Devanagari PP-OCRv3 mobile review is:
|
| 459 |
+
|
| 460 |
+
- artifact:
|
| 461 |
+
`outputs/recognizer-failure-review-paddle-ppocrv3/recognizer-failure-review.json`
|
| 462 |
+
- markdown:
|
| 463 |
+
`outputs/recognizer-failure-review-paddle-ppocrv3/recognizer-failure-review.md`
|
| 464 |
+
- included runs: PaddleOCR mixed v2, v3, v4, and `recognizer-next-v1`
|
| 465 |
+
- family recommendation: `stop_family`
|
| 466 |
+
- rejected runs: `4`
|
| 467 |
+
- paired failed-gate wins/losses/ties: `0` / `545` / `7`
|
| 468 |
+
|
| 469 |
+
Operational rule: do not start another AWS/G6 job for this Devanagari PP-OCRv3
|
| 470 |
+
mobile family unless the recognizer design changes materially. Use GPU time for
|
| 471 |
+
a different base recognizer, a different objective, or a document-native model
|
| 472 |
+
that still has to pass the same admission gate.
|
| 473 |
+
|
| 474 |
+
Enforcement rule for future handoffs: include the failure review when writing
|
| 475 |
+
and bundling recognizer training jobs:
|
| 476 |
+
|
| 477 |
+
```bash
|
| 478 |
+
ocrtech train-recognizer \
|
| 479 |
+
--train-manifest data/splits/train.jsonl \
|
| 480 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 481 |
+
--out runs/recognizer \
|
| 482 |
+
--failure-review outputs/recognizer-failure-review-paddle-ppocrv3/recognizer-failure-review.json
|
| 483 |
+
|
| 484 |
+
ocrtech bundle-recognizer-training \
|
| 485 |
+
--recipe-dir runs/recognizer \
|
| 486 |
+
--train-manifest data/splits/train.jsonl \
|
| 487 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 488 |
+
--out runs/recognizer-training-bundle \
|
| 489 |
+
--require-corpus-audit \
|
| 490 |
+
--failure-review outputs/recognizer-failure-review-paddle-ppocrv3/recognizer-failure-review.json
|
| 491 |
+
```
|
| 492 |
+
|
| 493 |
+
The bundle command reads `recognizer-training-summary.json` from the actual
|
| 494 |
+
recipe directory and refuses stopped backend/base-model families before archive
|
| 495 |
+
creation. That is the guard to run before copying anything to AWS or `k2`.
|
| 496 |
+
|
| 497 |
+
The first live TrOCR run on `k2` flushed out the actual dependency floor for this backend. Do not hand-install one package at a time. Use the repo extra:
|
| 498 |
+
|
| 499 |
+
```bash
|
| 500 |
+
python -m pip install -e '.[hf-recognizer]'
|
| 501 |
+
```
|
| 502 |
+
|
| 503 |
+
That extra now covers `torch`, `transformers`, `accelerate`, `sentencepiece`, `protobuf`, and `pillow`. On a cold cache, `microsoft/trocr-small-printed` also downloads about `246 MB` of model weights before training starts.
|
| 504 |
+
|
| 505 |
+
For the verified mixed-script run on `k2`, use this exact sequence:
|
| 506 |
+
|
| 507 |
+
```bash
|
| 508 |
+
ocrtech prepare-data gauravgiri/nepali-ocr-dataset --hf --split train --limit 100 --skip-invalid --out data/manifests
|
| 509 |
+
ocrtech prepare-data prashant0919/nepali-synthetic-ocr-lines --hf --split train --limit 100 --out data/manifests
|
| 510 |
+
ocrtech prepare-data nvidia/OCR-Synthetic-Multilingual-v1 --hf --hf-subset en --split test --limit 1000 --out data/manifests-english-test
|
| 511 |
+
ocrtech merge-manifests \
|
| 512 |
+
data/manifests/gauravgiri__nepali-ocr-dataset-train.jsonl \
|
| 513 |
+
data/manifests/prashant0919__nepali-synthetic-ocr-lines-train.jsonl \
|
| 514 |
+
data/manifests-english-test/nvidia__OCR-Synthetic-Multilingual-v1__en-test.jsonl \
|
| 515 |
+
--out data/manifests/nep-eng-mixed-v1.jsonl
|
| 516 |
+
ocrtech audit-manifest data/manifests/nep-eng-mixed-v1.jsonl --out outputs/manifest-audit-mixed-v1 --require-slice english --require-slice nepali
|
| 517 |
+
ocrtech split-manifest data/manifests/nep-eng-mixed-v1.jsonl --out data/splits-mixed-v1 --eval-ratio 0.15 --seed 13
|
| 518 |
+
ocrtech materialize-references data/splits-mixed-v1/eval.jsonl --out data/references-mixed-v1 --rewritten-manifest data/splits-mixed-v1/eval-with-refs.jsonl
|
| 519 |
+
ocrtech rebalance-manifest data/splits-mixed-v1/train.jsonl --out data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl --slice english --slice nepali --seed 13
|
| 520 |
+
ocrtech train-recognizer \
|
| 521 |
+
--backend hf-vision-encoder-decoder \
|
| 522 |
+
--base-model microsoft/trocr-small-printed \
|
| 523 |
+
--train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
|
| 524 |
+
--eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 525 |
+
--out runs/hf-recognizer-mixed-v1-rebalanced \
|
| 526 |
+
--train-batch-size 4 \
|
| 527 |
+
--eval-batch-size 4 \
|
| 528 |
+
--epochs 0.5 \
|
| 529 |
+
--run
|
| 530 |
+
ocrtech package-hf-recognizer \
|
| 531 |
+
--model-dir runs/hf-recognizer-mixed-v1-rebalanced/final-model \
|
| 532 |
+
--out runs/hf-recognizer-mixed-v1-rebalanced/package \
|
| 533 |
+
--model-id ocrtech-hf-mixed-v1-rebalanced \
|
| 534 |
+
--base-model microsoft/trocr-small-printed \
|
| 535 |
+
--train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
|
| 536 |
+
--eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 537 |
+
--training-summary runs/hf-recognizer-mixed-v1-rebalanced/recognizer-training-summary.json \
|
| 538 |
+
--metrics-report runs/hf-recognizer-mixed-v1-rebalanced/training_metrics.json \
|
| 539 |
+
--admission-validation-report outputs/validation-next-recognizer-hard50/validation.json
|
| 540 |
+
ocrtech run-experiment \
|
| 541 |
+
--eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 542 |
+
--out outputs/experiment-hf-mixed-v1-rebalanced \
|
| 543 |
+
--baselines candidate \
|
| 544 |
+
--candidate-model-config runs/hf-recognizer-mixed-v1-rebalanced/package/model-card.json \
|
| 545 |
+
--references-dir data/references-mixed-v1 \
|
| 546 |
+
--skip-validation
|
| 547 |
+
```
|
| 548 |
+
|
| 549 |
+
Trainer-side eval metrics for that rebalanced run:
|
| 550 |
+
|
| 551 |
+
- `eval_cer = 0.5983964991975725`
|
| 552 |
+
- `eval_wer = 0.825085223418557`
|
| 553 |
+
- `eval_exact_line_accuracy = 0.2611111111111111`
|
| 554 |
+
|
| 555 |
+
Candidate-only benchmark means on the same `180`-sample eval split:
|
| 556 |
+
|
| 557 |
+
- overall CER: `0.586139`
|
| 558 |
+
- English CER: `0.300454`
|
| 559 |
+
- Nepali CER: `2.014560`
|
| 560 |
+
- real-slice CER: `1.770016`
|
| 561 |
+
|
| 562 |
+
`k2` now also has a real `tesseract` baseline installed via Homebrew, with both `eng` and `nep` language data verified. The first direct comparison on that same eval split is still negative for the candidate:
|
| 563 |
+
|
| 564 |
+
- candidate overall CER: `0.586139`
|
| 565 |
+
- tesseract overall CER: `0.329012`
|
| 566 |
+
- candidate Nepali CER: `2.014560`
|
| 567 |
+
- tesseract Nepali CER: `0.757175`
|
| 568 |
+
|
| 569 |
+
One shell detail matters on `k2`: non-interactive SSH sessions do not reliably inherit the Homebrew path. For remote one-shot benchmark commands, export it explicitly before calling `ocrtech`:
|
| 570 |
+
|
| 571 |
+
```bash
|
| 572 |
+
export PATH=/opt/homebrew/bin:$PATH
|
| 573 |
+
```
|
| 574 |
+
|
| 575 |
+
For the external VLM baselines, the current live state on `k2` is now more concrete:
|
| 576 |
+
|
| 577 |
+
- `paddleocr-vl` is runnable through `scripts/run_paddleocr_vl.py` after installing `paddlex[ocr]==3.7.1`.
|
| 578 |
+
- `glm-ocr` does run through `scripts/run_glm_ocr.py`, but not in the main repo venv. It needed a dedicated Python 3.11 env with git `transformers` and `torchvision`.
|
| 579 |
+
- On the first rendered claim-pack blur sample, neither VLM beat raw Tesseract on text CER:
|
| 580 |
+
- `glm-ocr`: `0.436170`
|
| 581 |
+
- `paddleocr-vl`: `0.521277`
|
| 582 |
+
- `tesseract`: `0.372340`
|
| 583 |
+
|
| 584 |
+
That matters for `tesseract` baselines and for candidate model cards that instantiate Tesseract internally.
|
| 585 |
+
|
| 586 |
+
We also verified a small routed Tesseract candidate on `k2`. A `quality_select_composite` model card using default Tesseract (`psm=6`) as primary and `psm=13` as secondary, with `secondary_quality_margin=0.02`, slightly improved the held-out `180`-sample mixed eval:
|
| 587 |
+
|
| 588 |
+
- raw tesseract overall CER: `0.329012`
|
| 589 |
+
- routed tesseract overall CER: `0.324000`
|
| 590 |
+
- paired CER versus raw tesseract: `5` wins, `3` losses, `172` ties
|
| 591 |
+
|
| 592 |
+
That is an engineering improvement, not a frontier claim. The routing margin was tuned on the same pack, and the effect size is small.
|
| 593 |
+
|
| 594 |
+
We then reran it with proper separation on the same host:
|
| 595 |
+
|
| 596 |
+
1. split `data/splits-mixed-v1/eval-with-refs.jsonl` into a calibration half (`89` samples) and untouched holdout half (`91` samples)
|
| 597 |
+
2. ran `psm=13` vs raw default Tesseract on the calibration half
|
| 598 |
+
3. generated a routed model card with `ocrtech calibrate-quality-router`
|
| 599 |
+
4. ran that calibrated routed candidate on the untouched holdout half
|
| 600 |
+
|
| 601 |
+
Holdout result:
|
| 602 |
+
|
| 603 |
+
- raw tesseract overall CER: `0.342234`
|
| 604 |
+
- calibrated routed tesseract overall CER: `0.340608`
|
| 605 |
+
- paired CER on holdout: `2` wins, `1` loss, `88` ties
|
| 606 |
+
|
| 607 |
+
That is the first out-of-sample confirmation that the routed baseline is not just same-pack leakage. It is still a very small gain.
|
| 608 |
+
|
| 609 |
+
We also tested Surya on `k2` instead of assuming a VLM-style OCR stack would help by default. The host can run it on Apple Silicon after installing:
|
| 610 |
+
|
| 611 |
+
- `/opt/homebrew/bin/llama-server` from `llama.cpp`
|
| 612 |
+
- `surya-ocr` inside `~/ocr-tech/.venv`
|
| 613 |
+
|
| 614 |
+
Raw Surya was not a good general baseline on the `91`-sample holdout:
|
| 615 |
+
|
| 616 |
+
- raw Surya overall CER: `2.198362`
|
| 617 |
+
- raw Tesseract overall CER: `0.342234`
|
| 618 |
+
|
| 619 |
+
But Surya was better on the Nepali slice, so we calibrated a routed `script_select_composite` candidate on the separate `89`-sample half with:
|
| 620 |
+
|
| 621 |
+
```bash
|
| 622 |
+
python -m ocrtech.cli calibrate-script-router \
|
| 623 |
+
--experiment-report outputs/experiment-surya-calibration-vs-tesseract/experiment.json \
|
| 624 |
+
--out outputs/calibration-surya-script-router \
|
| 625 |
+
--model-id ocrtech-tesseract-surya-script-router \
|
| 626 |
+
--primary-engine tesseract \
|
| 627 |
+
--secondary-engine surya \
|
| 628 |
+
--primary-engine-kwarg language=nep+eng \
|
| 629 |
+
--primary-engine-kwarg psm=6 \
|
| 630 |
+
--secondary-engine-kwarg mode=auto
|
| 631 |
+
```
|
| 632 |
+
|
| 633 |
+
That selected `primary_script_threshold=0.4`. On the untouched `91`-sample holdout:
|
| 634 |
+
|
| 635 |
+
- raw tesseract overall CER: `0.342234`
|
| 636 |
+
- calibrated script router overall CER: `0.313084`
|
| 637 |
+
- paired CER on holdout: `9` wins, `0` losses, `82` ties
|
| 638 |
+
|
| 639 |
+
This is the strongest out-of-sample baseline improvement we currently have on `k2`.
|
| 640 |
+
|
| 641 |
+
So the host is now baseline-ready, but the model claim is not. The next useful work is not another README victory lap. It is either:
|
| 642 |
+
|
| 643 |
+
1. improve the candidate until it actually beats Tesseract on held-out Nepali and mixed-script slices, or
|
| 644 |
+
2. admit this TrOCR path is the wrong recognizer direction and switch to a stronger training backend.
|
| 645 |
+
|
| 646 |
+
We tested one more recovery path on the same host before stopping: mine Tesseract noisy/clean pairs from the rebalanced training manifest, train a ByT5 corrector, and package it as a `text_correction_composite` candidate with raw Tesseract as the base engine.
|
| 647 |
+
|
| 648 |
+
That experiment also failed badly. The corrector trainer finished with:
|
| 649 |
+
|
| 650 |
+
- `eval_cer = 7.067461343195775`
|
| 651 |
+
- `eval_wer = 3.334916864181571`
|
| 652 |
+
- `eval_exact_line_accuracy = 0.0`
|
| 653 |
+
|
| 654 |
+
And the held-out benchmark on the same `180`-sample mixed eval was nowhere close to viable:
|
| 655 |
+
|
| 656 |
+
- composite overall CER: `6.912065`
|
| 657 |
+
- raw Tesseract overall CER: `0.329012`
|
| 658 |
+
- paired CER: `0` wins / `170` losses / `10` ties
|
| 659 |
+
|
| 660 |
+
So `k2` remains useful for fast iteration, but the current Tesseract-plus-corrector branch is not a credible direction either.
|
| 661 |
+
|
| 662 |
+
Then run the generated PaddleOCR command from a checked-out PaddleOCR repo:
|
| 663 |
+
|
| 664 |
+
```bash
|
| 665 |
+
python PaddleOCR/tools/train.py -c runs/recognizer/paddle_recognizer.yml
|
| 666 |
+
```
|
| 667 |
+
|
| 668 |
+
After training, export and package the inference artifacts:
|
| 669 |
+
|
| 670 |
+
```bash
|
| 671 |
+
ocrtech export-recognizer \
|
| 672 |
+
--training-config runs/recognizer/paddle_recognizer.yml \
|
| 673 |
+
--checkpoint runs/recognizer/checkpoints/best_accuracy \
|
| 674 |
+
--out runs/recognizer/export \
|
| 675 |
+
--run
|
| 676 |
+
|
| 677 |
+
ocrtech package-recognizer \
|
| 678 |
+
--inference-dir runs/recognizer/export/inference \
|
| 679 |
+
--dictionary runs/recognizer/nep_eng_dict.txt \
|
| 680 |
+
--out runs/recognizer/package \
|
| 681 |
+
--model-id ocrtech-nep-eng-rec-v1 \
|
| 682 |
+
--train-manifest data/splits/train.jsonl \
|
| 683 |
+
--eval-manifest data/splits/eval.jsonl \
|
| 684 |
+
--training-summary runs/recognizer/recognizer-training-summary.json \
|
| 685 |
+
--admission-validation-report outputs/validation-next-recognizer-hard50/validation.json
|
| 686 |
+
|
| 687 |
+
ocrtech audit-model runs/recognizer/package/model-card.json
|
| 688 |
+
```
|
| 689 |
+
|
| 690 |
+
Keep evaluation manifests physically separate from training manifests. Do not use the same image or text hashes across train/eval.
|
| 691 |
+
|
| 692 |
+
## AWS G6 PaddleOCR Run, 2026-06-15
|
| 693 |
+
|
| 694 |
+
We launched a CUDA host through the `playaround` AWS profile before the login
|
| 695 |
+
window expired:
|
| 696 |
+
|
| 697 |
+
- instance: `i-044a50b7737e8a067`
|
| 698 |
+
- type: `g6.xlarge`
|
| 699 |
+
- GPU: NVIDIA L4, `23034 MiB`
|
| 700 |
+
- region: `us-east-1`
|
| 701 |
+
- AMI: Deep Learning Base OSS Nvidia Driver GPU AMI, Ubuntu 22.04
|
| 702 |
+
- SSH key: `ocrtech-g6-training-20260615`
|
| 703 |
+
|
| 704 |
+
The host passed `paddle.utils.run_check()` with `paddlepaddle-gpu==3.3.1`.
|
| 705 |
+
Training used the local PaddleOCR checkout at commit
|
| 706 |
+
`c166448875bcecb8d3b7628fd697ac1c28f8705b`.
|
| 707 |
+
|
| 708 |
+
The original portable recipe from k2 was not valid for actual PaddleOCR
|
| 709 |
+
training because it was a minimal YAML stub. On AWS we built a corrected config
|
| 710 |
+
from the upstream Devanagari PP-OCRv3 mobile recognizer config and then patched
|
| 711 |
+
the local recipe writer to generate that shape directly.
|
| 712 |
+
|
| 713 |
+
Training command:
|
| 714 |
+
|
| 715 |
+
```bash
|
| 716 |
+
cd /opt/ocrtech
|
| 717 |
+
source .venv/bin/activate
|
| 718 |
+
python PaddleOCR/tools/train.py \
|
| 719 |
+
-c runs/paddle-recognizer-mixed-v2-cuda/paddle_recognizer.yml
|
| 720 |
+
```
|
| 721 |
+
|
| 722 |
+
Export command:
|
| 723 |
+
|
| 724 |
+
```bash
|
| 725 |
+
python PaddleOCR/tools/export_model.py \
|
| 726 |
+
-c runs/paddle-recognizer-mixed-v2-cuda/paddle_recognizer.yml \
|
| 727 |
+
-o Global.pretrained_model=runs/paddle-recognizer-mixed-v2-cuda/checkpoints/best_accuracy \
|
| 728 |
+
Global.save_inference_dir=runs/paddle-recognizer-mixed-v2-cuda/export/inference
|
| 729 |
+
```
|
| 730 |
+
|
| 731 |
+
Runtime dependencies added for benchmarking:
|
| 732 |
+
|
| 733 |
+
```bash
|
| 734 |
+
pip install --no-deps -e /opt/ocrtech/PaddleOCR
|
| 735 |
+
pip install paddlex==3.7.1 "paddlex[ocr-core]==3.7.1" aiohttp
|
| 736 |
+
sudo apt-get install -y tesseract-ocr tesseract-ocr-eng tesseract-ocr-nep
|
| 737 |
+
```
|
| 738 |
+
|
| 739 |
+
Packaging caveat for PaddleOCR 3.x:
|
| 740 |
+
|
| 741 |
+
- use `text_recognition_model_dir`, not `rec_model_dir`
|
| 742 |
+
- set `text_recognition_model_name=devanagari_PP-OCRv3_mobile_rec`
|
| 743 |
+
- do not pass `rec_char_dict_path`; exported `inference.yml` embeds the
|
| 744 |
+
dictionary
|
| 745 |
+
|
| 746 |
+
Benchmark command, after materializing references on the AWS host:
|
| 747 |
+
|
| 748 |
+
```bash
|
| 749 |
+
cd /opt/ocrtech
|
| 750 |
+
source .venv/bin/activate
|
| 751 |
+
PYTHONPATH=/opt/ocrtech/ocr-tech/src python -m ocrtech.cli materialize-references \
|
| 752 |
+
data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 753 |
+
--out data/references-mixed-v1 \
|
| 754 |
+
--rewritten-manifest data/splits-mixed-v1/eval-with-refs-aws.jsonl
|
| 755 |
+
|
| 756 |
+
PYTHONPATH=/opt/ocrtech/ocr-tech/src python -m ocrtech.cli run-experiment \
|
| 757 |
+
--eval-manifest /opt/ocrtech/data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 758 |
+
--out /opt/ocrtech/outputs/experiment-paddle-mixed-v2-cuda-vs-tesseract-v2 \
|
| 759 |
+
--baselines candidate,tesseract \
|
| 760 |
+
--candidate-model-config /opt/ocrtech/runs/paddle-recognizer-mixed-v2-cuda/package/model-card.json \
|
| 761 |
+
--skip-validation
|
| 762 |
+
|
| 763 |
+
PYTHONPATH=/opt/ocrtech/ocr-tech/src python -m ocrtech.cli rescore-benchmark \
|
| 764 |
+
--benchmark-report outputs/experiment-paddle-mixed-v2-cuda-vs-tesseract-v2/benchmark/report.json \
|
| 765 |
+
--eval-manifest data/splits-mixed-v1/eval-with-refs-aws.jsonl \
|
| 766 |
+
--out outputs/experiment-paddle-mixed-v2-cuda-vs-tesseract-v2/rescored-with-refs
|
| 767 |
+
```
|
| 768 |
+
|
| 769 |
+
Local archive:
|
| 770 |
+
|
| 771 |
+
```text
|
| 772 |
+
outputs/aws-g6-paddle-mixed-v2-cuda/ocrtech-paddle-mixed-v2-cuda-artifacts.tar.gz
|
| 773 |
+
```
|
| 774 |
+
|
| 775 |
+
Result:
|
| 776 |
+
|
| 777 |
+
- rows: `360`, all `ok`
|
| 778 |
+
- candidate CER/WER: `0.690203` / `0.944234`
|
| 779 |
+
- Tesseract CER/WER: `0.364915` / `0.521763`
|
| 780 |
+
- candidate latency: `0.077315s`
|
| 781 |
+
- Tesseract latency: `0.177510s`
|
| 782 |
+
|
| 783 |
+
This is a real trained model and a real negative result. It proves the CUDA
|
| 784 |
+
PaddleOCR path works end to end, but it does not support a better-than-Tesseract
|
| 785 |
+
claim. Before any claim rerun, normalize `.img` synthetic inputs to supported
|
| 786 |
+
image suffixes because PaddleOCR logged unsupported-extension warnings for that
|
| 787 |
+
subset.
|
| 788 |
+
|
| 789 |
+
Use the manifest image normalizer before the next claim-grade rerun:
|
| 790 |
+
|
| 791 |
+
```bash
|
| 792 |
+
ocrtech normalize-manifest-images \
|
| 793 |
+
data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 794 |
+
--out data/splits-mixed-v1/eval-normalized-images
|
| 795 |
+
|
| 796 |
+
ocrtech materialize-references \
|
| 797 |
+
data/splits-mixed-v1/eval-normalized-images/manifest-images-normalized.jsonl \
|
| 798 |
+
--out data/references-mixed-v1-normalized \
|
| 799 |
+
--rewritten-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl
|
| 800 |
+
```
|
| 801 |
+
|
| 802 |
+
The normalizer copies already supported image files and converts unsupported
|
| 803 |
+
file suffixes to PNG with updated image hashes and provenance fields:
|
| 804 |
+
`metadata.source_image_path`, `metadata.source_image_sha256`, and
|
| 805 |
+
`metadata.image_normalization`.
|
| 806 |
+
|
| 807 |
+
On `k2`, the canonical mixed eval manifest already resolves to `.png` inputs.
|
| 808 |
+
After syncing the normalizer and preflight suffix guard, this command produced a
|
| 809 |
+
clean normalized eval artifact:
|
| 810 |
+
|
| 811 |
+
```bash
|
| 812 |
+
ocrtech normalize-manifest-images \
|
| 813 |
+
data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 814 |
+
--out data/splits-mixed-v1/eval-normalized-images
|
| 815 |
+
```
|
| 816 |
+
|
| 817 |
+
Result:
|
| 818 |
+
|
| 819 |
+
- samples: `180`
|
| 820 |
+
- copied: `180`
|
| 821 |
+
- converted: `0`
|
| 822 |
+
- unsupported suffixes: none
|
| 823 |
+
|
| 824 |
+
Then:
|
| 825 |
+
|
| 826 |
+
```bash
|
| 827 |
+
ocrtech materialize-references \
|
| 828 |
+
data/splits-mixed-v1/eval-normalized-images/manifest-images-normalized.jsonl \
|
| 829 |
+
--out data/references-mixed-v1-normalized \
|
| 830 |
+
--rewritten-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl
|
| 831 |
+
|
| 832 |
+
PATH=/opt/homebrew/bin:$PATH ocrtech preflight-claim \
|
| 833 |
+
--eval-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl \
|
| 834 |
+
--out outputs/preflight-mixed-v1-normalized-script-router-path \
|
| 835 |
+
--baselines candidate,tesseract,surya \
|
| 836 |
+
--candidate-model-config outputs/calibration-surya-script-router/model-card.json \
|
| 837 |
+
--train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
|
| 838 |
+
--allow-smoke-eval-pack
|
| 839 |
+
```
|
| 840 |
+
|
| 841 |
+
Preflight passed on runtime, references, suffixes, candidate audit, component
|
| 842 |
+
baselines, and calibration separation. It still warns that this is only an OCR
|
| 843 |
+
line-style eval, not a full document-understanding claim pack.
|
| 844 |
+
|
| 845 |
+
Local copy:
|
| 846 |
+
|
| 847 |
+
```text
|
| 848 |
+
outputs/k2-mixed-v1-normalized-preflight/k2-mixed-v1-normalized-preflight.tar.gz
|
| 849 |
+
```
|
| 850 |
+
|
| 851 |
+
## 2026-06-15 k2 Normalized Mixed Eval Benchmark
|
| 852 |
+
|
| 853 |
+
Synced the manifest-driven benchmark CLI option to `k2` and verified it there:
|
| 854 |
+
|
| 855 |
+
```bash
|
| 856 |
+
~/.local/bin/uv run pytest \
|
| 857 |
+
tests/test_cli.py::test_cli_benchmark_loads_inputs_from_manifest -q
|
| 858 |
+
```
|
| 859 |
+
|
| 860 |
+
Then ran the normalized mixed eval directly from the manifest:
|
| 861 |
+
|
| 862 |
+
```bash
|
| 863 |
+
PATH=/opt/homebrew/bin:$PATH ~/.local/bin/uv run ocrtech benchmark \
|
| 864 |
+
--inputs-from-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl \
|
| 865 |
+
--out outputs/live-k2-mixed-v1-normalized-script-router-benchmark \
|
| 866 |
+
--baselines candidate,tesseract,surya \
|
| 867 |
+
--candidate-model-config outputs/calibration-surya-script-router/model-card.json \
|
| 868 |
+
--resume-existing \
|
| 869 |
+
--sample-timeout-seconds 180
|
| 870 |
+
```
|
| 871 |
+
|
| 872 |
+
The run checkpointed `428` successful rows:
|
| 873 |
+
|
| 874 |
+
- candidate: `180/180`
|
| 875 |
+
- Tesseract: `180/180`
|
| 876 |
+
- Surya: `68/180`
|
| 877 |
+
|
| 878 |
+
The Surya phase stopped progressing after row `68`; the benchmark process was
|
| 879 |
+
terminated manually after preserving `report.json`. Treat the Surya numbers from
|
| 880 |
+
this run as partial diagnostics only.
|
| 881 |
+
|
| 882 |
+
Complete candidate-vs-Tesseract result:
|
| 883 |
+
|
| 884 |
+
- candidate CER/WER: `0.354222` / `0.493555`
|
| 885 |
+
- Tesseract CER/WER: `0.329012` / `0.486485`
|
| 886 |
+
- candidate English CER/WER: `0.272420` / `0.409552`
|
| 887 |
+
- Tesseract English CER/WER: `0.243379` / `0.370552`
|
| 888 |
+
- candidate Nepali CER/WER: `0.763234` / `0.913571`
|
| 889 |
+
- Tesseract Nepali CER/WER: `0.757175` / `1.066151`
|
| 890 |
+
|
| 891 |
+
Local copy:
|
| 892 |
+
|
| 893 |
+
```text
|
| 894 |
+
outputs/k2-mixed-v1-normalized-script-router-benchmark/k2-mixed-v1-normalized-script-router-benchmark.tgz
|
| 895 |
+
```
|
| 896 |
+
|
| 897 |
+
Interpretation: the normalized line-style rerun is a negative result for the
|
| 898 |
+
script-router candidate. It is worse than Tesseract overall and on English; its
|
| 899 |
+
only useful signal is Nepali WER. The next validation-infrastructure fix should
|
| 900 |
+
be a hard backend watchdog so stalled Surya/VLM calls become row-level errors
|
| 901 |
+
instead of manual run interruptions.
|
| 902 |
+
|
| 903 |
+
Follow-up: implemented the hard backend watchdog in `ocrtech.benchmark` and
|
| 904 |
+
reran the same normalized benchmark in a fresh output directory:
|
| 905 |
+
|
| 906 |
+
```bash
|
| 907 |
+
PATH=/opt/homebrew/bin:$PATH ~/.local/bin/uv run ocrtech benchmark \
|
| 908 |
+
--inputs-from-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl \
|
| 909 |
+
--out outputs/live-k2-mixed-v1-normalized-script-router-benchmark-hard-timeout \
|
| 910 |
+
--baselines candidate,tesseract,surya \
|
| 911 |
+
--candidate-model-config outputs/calibration-surya-script-router/model-card.json \
|
| 912 |
+
--resume-existing \
|
| 913 |
+
--sample-timeout-seconds 45
|
| 914 |
+
```
|
| 915 |
+
|
| 916 |
+
Completed hard-timeout result:
|
| 917 |
+
|
| 918 |
+
- rows: `540`
|
| 919 |
+
- candidate: `180/180`, all `ok`
|
| 920 |
+
- Tesseract: `180/180`, all `ok`
|
| 921 |
+
- Surya: `180/180` attempted, `178` ok, `2` timeout errors
|
| 922 |
+
- Surya timeout rows:
|
| 923 |
+
`nvidia__OCR-Synthetic-Multilingual-v1__en-test-000-000013-0000`,
|
| 924 |
+
`nvidia__OCR-Synthetic-Multilingual-v1__en-test-000-000028-0002`
|
| 925 |
+
|
| 926 |
+
Summary:
|
| 927 |
+
|
| 928 |
+
- candidate CER/WER: `0.354840` / `0.469944`
|
| 929 |
+
- Tesseract CER/WER: `0.329012` / `0.486485`
|
| 930 |
+
- Surya scored-row CER/WER: `3.157317` / `2.428888`
|
| 931 |
+
|
| 932 |
+
Local copy:
|
| 933 |
+
|
| 934 |
+
```text
|
| 935 |
+
outputs/k2-mixed-v1-normalized-script-router-hard-timeout/k2-mixed-v1-normalized-script-router-hard-timeout.tgz
|
| 936 |
+
```
|
| 937 |
+
|
| 938 |
+
Interpretation: the watchdog fix works. It converts the previous Surya hang into
|
| 939 |
+
two row-level timeouts and finishes the full three-system run. The result still
|
| 940 |
+
does not support the script-router candidate as a better OCR engine.
|
| 941 |
+
|
| 942 |
+
## 2026-06-15 k2 PaddleOCR Recognizer Hard-50 Eval Slice
|
| 943 |
+
|
| 944 |
+
The AWS G6 trained PaddleOCR recognizer lost overall. We ranked its top held-out
|
| 945 |
+
CER failures and remapped them onto the normalized k2 eval manifest so the next
|
| 946 |
+
recognizer branch has a runnable hard-eval gate.
|
| 947 |
+
|
| 948 |
+
Commands on k2:
|
| 949 |
+
|
| 950 |
+
```bash
|
| 951 |
+
~/.local/bin/uv run ocrtech export-failure-manifest \
|
| 952 |
+
--source-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl \
|
| 953 |
+
--top-failures outputs/paddle-mixed-v2-cuda-error-analysis/top-failures.json \
|
| 954 |
+
--out data/splits-mixed-v1/eval-paddle-mixed-v2-cuda-hard50-normalized.jsonl \
|
| 955 |
+
--split paddle_hard_eval
|
| 956 |
+
|
| 957 |
+
~/.local/bin/uv run ocrtech audit-manifest \
|
| 958 |
+
data/splits-mixed-v1/eval-paddle-mixed-v2-cuda-hard50-normalized.jsonl \
|
| 959 |
+
--out outputs/manifest-audit-paddle-mixed-v2-cuda-hard50-normalized \
|
| 960 |
+
--require-slice paddle_hard_eval \
|
| 961 |
+
--require-slice english \
|
| 962 |
+
--require-slice nepali
|
| 963 |
+
```
|
| 964 |
+
|
| 965 |
+
Result:
|
| 966 |
+
|
| 967 |
+
- selected hard failures: `50`
|
| 968 |
+
- slice counts: `synthetic=50`, `english=38`, `en=38`, `line_crop=38`,
|
| 969 |
+
`nepali=12`
|
| 970 |
+
- audit: pass
|
| 971 |
+
|
| 972 |
+
Local copy:
|
| 973 |
+
|
| 974 |
+
```text
|
| 975 |
+
outputs/k2-paddle-mixed-v2-hard50-normalized/k2-paddle-mixed-v2-hard50-normalized.tgz
|
| 976 |
+
```
|
| 977 |
+
|
| 978 |
+
Interpretation: this hard-50 manifest is held-out evaluation material, not
|
| 979 |
+
training data. It shows the trained PaddleOCR branch is failing mostly on
|
| 980 |
+
synthetic English line crops, despite the rebalanced training manifest containing
|
| 981 |
+
`850` English and `850` Nepali samples. The next recognizer experiment must pass
|
| 982 |
+
this hard slice while preserving the full normalized mixed eval and document
|
| 983 |
+
pack gates.
|
| 984 |
+
|
| 985 |
+
## 2026-06-15 k2 Line-Level PaddleOCR Composite Rerun
|
| 986 |
+
|
| 987 |
+
We tested whether the failed AWS G6 PaddleOCR recognizer could still be useful
|
| 988 |
+
as a selective Devanagari fallback behind a Tesseract primary engine.
|
| 989 |
+
|
| 990 |
+
The first hard-50 run was invalid because all candidate rows crashed with
|
| 991 |
+
`exit_code=-11`. Root cause: hard-timeout mode initialized PaddleOCR in the
|
| 992 |
+
parent process and then forked child workers. The benchmark now initializes the
|
| 993 |
+
engine inside each timeout worker. Focused tests passed locally and on k2.
|
| 994 |
+
|
| 995 |
+
k2 command:
|
| 996 |
+
|
| 997 |
+
```bash
|
| 998 |
+
PATH=/opt/homebrew/bin:$PATH ~/.local/bin/uv run ocrtech benchmark \
|
| 999 |
+
--inputs-from-manifest data/splits-mixed-v1/eval-paddle-mixed-v2-cuda-hard50-normalized.jsonl \
|
| 1000 |
+
--out outputs/experiment-line-script-paddle-hard50-v2 \
|
| 1001 |
+
--baselines candidate,tesseract \
|
| 1002 |
+
--candidate-model-config outputs/line-script-paddle-mixed-v2-cuda/model-card.json \
|
| 1003 |
+
--resume-existing \
|
| 1004 |
+
--sample-timeout-seconds 120
|
| 1005 |
+
```
|
| 1006 |
+
|
| 1007 |
+
Result:
|
| 1008 |
+
|
| 1009 |
+
- rows: `100`
|
| 1010 |
+
- candidate: `50/50`, all `ok`
|
| 1011 |
+
- Tesseract: `50/50`, all `ok`
|
| 1012 |
+
- validation: `not_validated`
|
| 1013 |
+
- hard-50 CER: candidate `0.276489`, Tesseract `0.120106`
|
| 1014 |
+
- English line-crop CER/WER: exact tie with Tesseract, `0.060429` / `0.132259`
|
| 1015 |
+
- Nepali hard CER/WER: candidate `0.960679` / `1.000000`, Tesseract
|
| 1016 |
+
`0.309083` / `0.711607`
|
| 1017 |
+
|
| 1018 |
+
Local artifacts:
|
| 1019 |
+
|
| 1020 |
+
```text
|
| 1021 |
+
outputs/experiment-line-script-paddle-hard50-v2/report.json
|
| 1022 |
+
outputs/summary-line-script-paddle-hard50-v2/summary.md
|
| 1023 |
+
outputs/validation-line-script-paddle-hard50-v2/validation.md
|
| 1024 |
+
```
|
| 1025 |
+
|
| 1026 |
+
Decision: do not spend AWS G6 time rerunning this exact composite. A CUDA/G6
|
| 1027 |
+
machine is useful only for the next recognizer training branch: either a better
|
| 1028 |
+
PaddleOCR recipe/base model or a different Devanagari recognizer that first
|
| 1029 |
+
passes the hard-50 admission gate.
|
| 1030 |
+
|
| 1031 |
+
## AWS G6 PaddleOCR V3 Run, 2026-06-15
|
| 1032 |
+
|
| 1033 |
+
We launched a fresh CUDA host while the `playaround` SSO window was active:
|
| 1034 |
+
|
| 1035 |
+
- instance: `i-0b8604ffbcbce25c7`
|
| 1036 |
+
- type: `g6.xlarge`
|
| 1037 |
+
- GPU: NVIDIA L4, `23034 MiB`
|
| 1038 |
+
- region: `us-east-1`
|
| 1039 |
+
- public IP at launch: `44.202.252.0`
|
| 1040 |
+
- AMI: `ami-01011b868ec560823`, Deep Learning Base OSS Nvidia Driver GPU AMI
|
| 1041 |
+
Ubuntu 22.04, `20260609`
|
| 1042 |
+
- SSH key: `ocrtech-g6-training-20260615`
|
| 1043 |
+
- security group: `sg-05ad6744a62dcabdd`, SSH restricted to the current
|
| 1044 |
+
workstation IP used at launch
|
| 1045 |
+
|
| 1046 |
+
Runtime setup:
|
| 1047 |
+
|
| 1048 |
+
- repo source snapshot extracted to `/opt/ocrtech`
|
| 1049 |
+
- k2 training bundle copied from
|
| 1050 |
+
`runs/paddle-recognizer-mixed-v2-bundle/paddle-recognizer-mixed-v2-training.tar.gz`
|
| 1051 |
+
- `uv` installed
|
| 1052 |
+
- Python `3.11` venv created
|
| 1053 |
+
- installed repo extras: `datasets`, `engines`, `dev`
|
| 1054 |
+
- installed `paddlepaddle-gpu==3.3.1` from the Paddle CUDA 12.6 index
|
| 1055 |
+
- checked out PaddleOCR at `c166448875bcecb8d3b7628fd697ac1c28f8705b`
|
| 1056 |
+
- installed PaddleOCR training requirements
|
| 1057 |
+
- `paddle.utils.run_check()` passed on one GPU
|
| 1058 |
+
|
| 1059 |
+
Two repo bugs were fixed before the run could start:
|
| 1060 |
+
|
| 1061 |
+
- integral `epoch_num` values must be emitted as integers, not floats. Paddle's
|
| 1062 |
+
cosine scheduler rejected `20.0` because `T_max` must be an `int`.
|
| 1063 |
+
- the PaddleOCR training default must be the official Paddle pretrained
|
| 1064 |
+
`.pdparams` URL, not the Hugging Face model id. PaddleOCR's training loader
|
| 1065 |
+
could not resolve `PaddlePaddle/devanagari_PP-OCRv3_mobile_rec`.
|
| 1066 |
+
|
| 1067 |
+
V3 training hypothesis:
|
| 1068 |
+
|
| 1069 |
+
- same verified train/eval split as v2: `1700` train, `180` eval
|
| 1070 |
+
- lower learning rate: `0.0001`
|
| 1071 |
+
- longer schedule: `20` epochs
|
| 1072 |
+
- official pretrained URL:
|
| 1073 |
+
`https://paddle-model-ecology.bj.bcebos.com/paddlex/official_pretrained_model/devanagari_PP-OCRv3_mobile_rec_pretrained.pdparams`
|
| 1074 |
+
- run directory: `/opt/ocrtech/runs/paddle-recognizer-mixed-v3-cuda`
|
| 1075 |
+
|
| 1076 |
+
Launch command on the G6:
|
| 1077 |
+
|
| 1078 |
+
```bash
|
| 1079 |
+
cd /opt/ocrtech
|
| 1080 |
+
. .venv/bin/activate
|
| 1081 |
+
ocrtech train-recognizer \
|
| 1082 |
+
--train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
|
| 1083 |
+
--eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 1084 |
+
--out runs/paddle-recognizer-mixed-v3-cuda \
|
| 1085 |
+
--train-batch-size 16 \
|
| 1086 |
+
--eval-batch-size 16 \
|
| 1087 |
+
--learning-rate 0.0001 \
|
| 1088 |
+
--epochs 20
|
| 1089 |
+
|
| 1090 |
+
nohup python PaddleOCR/tools/train.py \
|
| 1091 |
+
-c runs/paddle-recognizer-mixed-v3-cuda/paddle_recognizer.yml \
|
| 1092 |
+
> runs/paddle-recognizer-mixed-v3-cuda/train.log 2>&1 &
|
| 1093 |
+
```
|
| 1094 |
+
|
| 1095 |
+
Initial status:
|
| 1096 |
+
|
| 1097 |
+
- process started as PID `4071`
|
| 1098 |
+
- dataloaders built: `106` train iters, `12` eval iters
|
| 1099 |
+
- pretrained weights downloaded and loaded
|
| 1100 |
+
- classifier/head weights were shape-mismatched because the expanded dictionary
|
| 1101 |
+
has more classes; PaddleOCR skipped those layers and loaded the rest
|
| 1102 |
+
- first logged training step: epoch `1/20`, global step `10`, loss `137.718468`,
|
| 1103 |
+
ETA about `44` minutes
|
| 1104 |
+
|
| 1105 |
+
Final training status:
|
| 1106 |
+
|
| 1107 |
+
- completed all `20` epochs
|
| 1108 |
+
- best checkpoint: epoch `17`
|
| 1109 |
+
- best training-reported eval accuracy: `0.09444443919753115`
|
| 1110 |
+
- best norm edit distance: `0.5833825886055732`
|
| 1111 |
+
- export succeeded from `checkpoints/best_accuracy.pdparams`
|
| 1112 |
+
- package audit passed for `ocrtech-paddle-mixed-v3-cuda`
|
| 1113 |
+
|
| 1114 |
+
Hard-50 setup correction:
|
| 1115 |
+
|
| 1116 |
+
- the first G6 hard-50 benchmark produced no CER/WER because references had not
|
| 1117 |
+
been materialized on the host.
|
| 1118 |
+
- we copied all `50` normalized hard-50 PNGs from k2 and materialized references
|
| 1119 |
+
into `data/references-paddle-mixed-v3-hard50`.
|
| 1120 |
+
- the original hard-50 manifest then audited cleanly on G6.
|
| 1121 |
+
- the existing OCR outputs were rescored with
|
| 1122 |
+
`eval-paddle-mixed-v2-cuda-hard50-g6-with-refs.jsonl`.
|
| 1123 |
+
|
| 1124 |
+
Hard-50 result: `not_validated`.
|
| 1125 |
+
|
| 1126 |
+
| slice | metric | candidate | Tesseract | result |
|
| 1127 |
+
| --- | --- | ---: | ---: | --- |
|
| 1128 |
+
| all hard-50 | CER | 0.921416 | 0.080249 | fail |
|
| 1129 |
+
| all hard-50 | WER | 1.010476 | 0.258921 | fail |
|
| 1130 |
+
| English line-crop | CER | 0.917511 | 0.031244 | fail |
|
| 1131 |
+
| English line-crop | WER | 0.996241 | 0.114464 | fail |
|
| 1132 |
+
| Nepali hard | CER | 0.933782 | 0.235433 | fail |
|
| 1133 |
+
| Nepali hard | WER | 1.055556 | 0.716369 | fail |
|
| 1134 |
+
|
| 1135 |
+
Local artifact bundle:
|
| 1136 |
+
|
| 1137 |
+
```text
|
| 1138 |
+
outputs/aws-g6-paddle-mixed-v3-cuda/ocrtech-paddle-mixed-v3-cuda-artifacts.tgz
|
| 1139 |
+
```
|
| 1140 |
+
|
| 1141 |
+
Important interpretation: this v3 run fixed the infrastructure bugs but not the
|
| 1142 |
+
model problem. Lower learning rate plus longer schedule on the same PP-OCRv3
|
| 1143 |
+
Devanagari recipe still fails the hard-50 admission gate, especially English
|
| 1144 |
+
preservation. Do not run the full mixed eval for this branch.
|
| 1145 |
+
|
| 1146 |
+
## AWS G6 PaddleOCR V4 Run, 2026-06-15
|
| 1147 |
+
|
| 1148 |
+
Current live host:
|
| 1149 |
+
|
| 1150 |
+
- instance: `i-0fa19dff7b233a353`
|
| 1151 |
+
- type: `g6.xlarge`
|
| 1152 |
+
- region/AZ: `us-east-1` / `us-east-1a`
|
| 1153 |
+
- public IP: `3.231.208.121`
|
| 1154 |
+
- AMI: `ami-01011b868ec560823`
|
| 1155 |
+
- SSH key: `ocrtech-g6-training-20260615`
|
| 1156 |
+
- root volume: `200 GiB gp3`
|
| 1157 |
+
|
| 1158 |
+
Setup already completed on the host:
|
| 1159 |
+
|
| 1160 |
+
- source at `/opt/ocrtech`
|
| 1161 |
+
- mixed-v2 bundle extracted from
|
| 1162 |
+
`runs/paddle-recognizer-mixed-v2-bundle/paddle-recognizer-mixed-v2-training.tar.gz`
|
| 1163 |
+
- Python `3.11.15` venv created with `uv`
|
| 1164 |
+
- `paddlepaddle-gpu==3.3.1` installed
|
| 1165 |
+
- PaddleOCR checked out at `c166448875bcecb8d3b7628fd697ac1c28f8705b`
|
| 1166 |
+
- PaddleOCR requirements installed
|
| 1167 |
+
- `paddle.utils.run_check()` passed on one NVIDIA L4
|
| 1168 |
+
- train manifest audit passed: `1700` samples
|
| 1169 |
+
- eval manifest audit passed: `180` samples
|
| 1170 |
+
|
| 1171 |
+
V4 hypothesis:
|
| 1172 |
+
|
| 1173 |
+
- same verified mixed-v1 split as v2/v3
|
| 1174 |
+
- official Devanagari PP-OCRv3 `.pdparams` base model
|
| 1175 |
+
- lower learning rate than v3: `0.00005`
|
| 1176 |
+
- `20` epochs
|
| 1177 |
+
- batch size `16`
|
| 1178 |
+
- run directory: `/opt/ocrtech/runs/paddle-recognizer-mixed-v4-cuda`
|
| 1179 |
+
|
| 1180 |
+
Two operational notes matter:
|
| 1181 |
+
|
| 1182 |
+
- The first v4 launch failed before consuming training time because YAML parsed
|
| 1183 |
+
`learning_rate: 5e-05` as a string. The recipe generator now emits
|
| 1184 |
+
`learning_rate: 0.00005`, and local tests cover this.
|
| 1185 |
+
- A bad local sync once copied `.venv` to the host and replaced Linux console
|
| 1186 |
+
scripts with macOS interpreter paths. Repair was done by deleting `.venv`,
|
| 1187 |
+
recreating it with `uv venv --python 3.11`, reinstalling repo extras,
|
| 1188 |
+
`paddlepaddle-gpu==3.3.1`, PaddleOCR requirements, and rerunning
|
| 1189 |
+
`paddle.utils.run_check()`.
|
| 1190 |
+
|
| 1191 |
+
When SSH is available, first check whether training is already running:
|
| 1192 |
+
|
| 1193 |
+
```bash
|
| 1194 |
+
cd /opt/ocrtech
|
| 1195 |
+
pgrep -af 'PaddleOCR/tools/train.py|train-v4.sh' || true
|
| 1196 |
+
nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits
|
| 1197 |
+
tail -80 runs/paddle-recognizer-mixed-v4-cuda/train.log
|
| 1198 |
+
```
|
| 1199 |
+
|
| 1200 |
+
If it is not running, regenerate the recipe with the fixed local code and
|
| 1201 |
+
launch:
|
| 1202 |
+
|
| 1203 |
+
```bash
|
| 1204 |
+
cd /opt/ocrtech
|
| 1205 |
+
.venv/bin/ocrtech train-recognizer \
|
| 1206 |
+
--backend paddleocr \
|
| 1207 |
+
--train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
|
| 1208 |
+
--eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 1209 |
+
--out runs/paddle-recognizer-mixed-v4-cuda \
|
| 1210 |
+
--train-batch-size 16 \
|
| 1211 |
+
--eval-batch-size 16 \
|
| 1212 |
+
--learning-rate 0.00005 \
|
| 1213 |
+
--epochs 20
|
| 1214 |
+
|
| 1215 |
+
grep -E 'learning_rate|epoch_num' runs/paddle-recognizer-mixed-v4-cuda/paddle_recognizer.yml
|
| 1216 |
+
|
| 1217 |
+
cat > runs/paddle-recognizer-mixed-v4-cuda/train-v4.sh <<'SH'
|
| 1218 |
+
#!/usr/bin/env bash
|
| 1219 |
+
set -euo pipefail
|
| 1220 |
+
cd /opt/ocrtech
|
| 1221 |
+
source .venv/bin/activate
|
| 1222 |
+
export FLAGS_fraction_of_gpu_memory_to_use=0.92
|
| 1223 |
+
python PaddleOCR/tools/train.py -c runs/paddle-recognizer-mixed-v4-cuda/paddle_recognizer.yml
|
| 1224 |
+
SH
|
| 1225 |
+
|
| 1226 |
+
chmod +x runs/paddle-recognizer-mixed-v4-cuda/train-v4.sh
|
| 1227 |
+
nohup bash runs/paddle-recognizer-mixed-v4-cuda/train-v4.sh \
|
| 1228 |
+
> runs/paddle-recognizer-mixed-v4-cuda/train.log 2>&1 &
|
| 1229 |
+
echo $! > runs/paddle-recognizer-mixed-v4-cuda/train.pid
|
| 1230 |
+
```
|
| 1231 |
+
|
| 1232 |
+
After completion, export, package, and include training-summary provenance:
|
| 1233 |
+
|
| 1234 |
+
```bash
|
| 1235 |
+
cd /opt/ocrtech
|
| 1236 |
+
.venv/bin/ocrtech export-recognizer \
|
| 1237 |
+
--training-config runs/paddle-recognizer-mixed-v4-cuda/paddle_recognizer.yml \
|
| 1238 |
+
--checkpoint runs/paddle-recognizer-mixed-v4-cuda/checkpoints/best_accuracy \
|
| 1239 |
+
--out runs/paddle-recognizer-mixed-v4-cuda/export \
|
| 1240 |
+
--run
|
| 1241 |
+
|
| 1242 |
+
.venv/bin/ocrtech package-recognizer \
|
| 1243 |
+
--inference-dir runs/paddle-recognizer-mixed-v4-cuda/export/inference \
|
| 1244 |
+
--dictionary runs/paddle-recognizer-mixed-v4-cuda/nep_eng_dict.txt \
|
| 1245 |
+
--out runs/paddle-recognizer-mixed-v4-cuda/package \
|
| 1246 |
+
--model-id ocrtech-paddle-mixed-v4-cuda \
|
| 1247 |
+
--train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
|
| 1248 |
+
--eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
|
| 1249 |
+
--training-summary runs/paddle-recognizer-mixed-v4-cuda/recognizer-training-summary.json
|
| 1250 |
+
|
| 1251 |
+
.venv/bin/ocrtech audit-model runs/paddle-recognizer-mixed-v4-cuda/package/model-card.json \
|
| 1252 |
+
--out runs/paddle-recognizer-mixed-v4-cuda/audit
|
| 1253 |
+
```
|
| 1254 |
+
|
| 1255 |
+
V4 final status:
|
| 1256 |
+
|
| 1257 |
+
- training completed all `20` epochs on the G6 L4 instance.
|
| 1258 |
+
- best PaddleOCR eval metric from training: accuracy `0.016666665740740792`,
|
| 1259 |
+
norm edit distance `0.442565389555461`, best epoch `19`.
|
| 1260 |
+
- export succeeded from checkpoint prefix
|
| 1261 |
+
`runs/paddle-recognizer-mixed-v4-cuda/checkpoints/best_accuracy`.
|
| 1262 |
+
- package and model audit succeeded after model-card artifact hashes were fixed
|
| 1263 |
+
to resolve package-relative files.
|
| 1264 |
+
- hard-50 admission gate completed with status `not_validated`.
|
| 1265 |
+
- candidate all CER/WER: `0.940500` / `1.028667`.
|
| 1266 |
+
- Tesseract all CER/WER: `0.080249` / `0.258921`.
|
| 1267 |
+
- English line-crop CER: candidate `0.950730`, Tesseract `0.031244`.
|
| 1268 |
+
- Nepali CER: candidate `0.908108`, Tesseract `0.235433`.
|
| 1269 |
+
- paired CER wins/losses/ties vs Tesseract: `0` / `50` / `0`.
|
| 1270 |
+
- local artifact bundle:
|
| 1271 |
+
`outputs/aws-g6-paddle-mixed-v4-cuda/ocrtech-paddle-mixed-v4-cuda-artifacts.tgz`.
|
| 1272 |
+
- stop request completed for instance `i-0fa19dff7b233a353`; AWS reported
|
| 1273 |
+
final state `stopped`.
|
| 1274 |
+
|
| 1275 |
+
Do not relaunch this exact branch as the engine core. The v4 result is a third
|
| 1276 |
+
negative PaddleOCR mixed-script recognizer result after v2 and v3.
|
| 1277 |
+
|
| 1278 |
+
## AWS G6 HF Recognizer Next V1
|
| 1279 |
+
|
| 1280 |
+
The stopped G6 instance `i-0fa19dff7b233a353` was restarted on 2026-06-15 for a
|
| 1281 |
+
new HF recognizer run instead of launching a duplicate GPU host.
|
| 1282 |
+
|
| 1283 |
+
Host details at launch:
|
| 1284 |
+
|
| 1285 |
+
- instance type: `g6.xlarge`
|
| 1286 |
+
- public IP: `3.91.25.53`
|
| 1287 |
+
- SSH key: `~/.ssh/ocrtech-g6-training-20260615.pem`
|
| 1288 |
+
- remote workspace: `/opt/ocrtech`
|
| 1289 |
+
- GPU: NVIDIA L4, `23034` MiB
|
| 1290 |
+
- driver reported by `nvidia-smi`: `580.159.04`
|
| 1291 |
+
- torch runtime after install: `2.12.0+cu130`
|
| 1292 |
+
- CUDA visibility: `torch.cuda.is_available() == True`
|
| 1293 |
+
|
| 1294 |
+
The source files `training.py`, `admission.py`, `validation.py`, and `cli.py`
|
| 1295 |
+
were synced to `/opt/ocrtech/src/ocrtech/`. The verified bundle archive was
|
| 1296 |
+
uploaded to:
|
| 1297 |
+
|
| 1298 |
+
```text
|
| 1299 |
+
/opt/ocrtech/incoming/hf-recognizer-next-v1-training.tar.gz
|
| 1300 |
+
```
|
| 1301 |
+
|
| 1302 |
+
Archive verification:
|
| 1303 |
+
|
| 1304 |
+
```text
|
| 1305 |
+
aff0b277b9cfdd353ed82178c65586353c3511d6db06bd20099cbc62df113873
|
| 1306 |
+
```
|
| 1307 |
+
|
| 1308 |
+
Dependencies were installed with:
|
| 1309 |
+
|
| 1310 |
+
```bash
|
| 1311 |
+
cd /opt/ocrtech
|
| 1312 |
+
.venv/bin/python -m pip install -e ".[hf-recognizer]"
|
| 1313 |
+
```
|
| 1314 |
+
|
| 1315 |
+
The bundle was extracted at `/opt/ocrtech`, then the run was launched with
|
| 1316 |
+
`nohup` through:
|
| 1317 |
+
|
| 1318 |
+
```text
|
| 1319 |
+
/opt/ocrtech/runs/hf-recognizer-next-v1/train-hf-next-v1.sh
|
| 1320 |
+
```
|
| 1321 |
+
|
| 1322 |
+
The script runs:
|
| 1323 |
+
|
| 1324 |
+
```bash
|
| 1325 |
+
ocrtech train-recognizer \
|
| 1326 |
+
--backend hf-vision-encoder-decoder \
|
| 1327 |
+
--base-model microsoft/trocr-small-printed \
|
| 1328 |
+
--train-manifest data/splits-recognizer-next-v1/train.jsonl \
|
| 1329 |
+
--eval-manifest data/splits-recognizer-next-v1/eval.jsonl \
|
| 1330 |
+
--out runs/hf-recognizer-next-v1 \
|
| 1331 |
+
--max-target-length 128 \
|
| 1332 |
+
--train-batch-size 8 \
|
| 1333 |
+
--eval-batch-size 8 \
|
| 1334 |
+
--learning-rate 5e-05 \
|
| 1335 |
+
--epochs 5 \
|
| 1336 |
+
--run
|
| 1337 |
+
```
|
| 1338 |
+
|
| 1339 |
+
Live files:
|
| 1340 |
+
|
| 1341 |
+
- PID file:
|
| 1342 |
+
`/opt/ocrtech/runs/hf-recognizer-next-v1/train.pid`
|
| 1343 |
+
- launch PID: `1862`
|
| 1344 |
+
- log:
|
| 1345 |
+
`/opt/ocrtech/runs/hf-recognizer-next-v1/train.log`
|
| 1346 |
+
|
| 1347 |
+
Observed shortly after launch:
|
| 1348 |
+
|
| 1349 |
+
- process was alive
|
| 1350 |
+
- GPU memory used: about `6500` MiB
|
| 1351 |
+
- GPU utilization: `46%` to `98%`
|
| 1352 |
+
- training progress: at least `200` / `1385` trainer steps
|
| 1353 |
+
|
| 1354 |
+
Final status:
|
| 1355 |
+
|
| 1356 |
+
- training completed all `5` epochs.
|
| 1357 |
+
- final model copied locally to:
|
| 1358 |
+
`outputs/aws-g6-hf-recognizer-next-v1/runs-hf-recognizer-next-v1/final-model`
|
| 1359 |
+
- local package:
|
| 1360 |
+
`outputs/aws-g6-hf-recognizer-next-v1/package/model-card.json`
|
| 1361 |
+
- local audit:
|
| 1362 |
+
`outputs/aws-g6-hf-recognizer-next-v1/audit/model-audit.json`
|
| 1363 |
+
- audit status: passed
|
| 1364 |
+
- final `eval_cer`: `0.18840631911585276`
|
| 1365 |
+
- final `eval_wer`: `0.36464600972797695`
|
| 1366 |
+
- final exact-line accuracy: `0.4344262295081967`
|
| 1367 |
+
- final `eval_loss`: `1.0254589319229126`
|
| 1368 |
+
|
| 1369 |
+
The first attempt to stop the host interrupted a broad rsync before any local
|
| 1370 |
+
artifact copy landed. The same instance was restarted only to retrieve the
|
| 1371 |
+
completed run, then the copy was narrowed to final model files and metadata,
|
| 1372 |
+
excluding trainer checkpoints. The local artifact set intentionally omits
|
| 1373 |
+
checkpoint optimizer state.
|
| 1374 |
+
|
| 1375 |
+
After artifact retrieval, the local AWS SSO session had expired and blocked the
|
| 1376 |
+
API stop call. The instance was shut down through SSH instead:
|
| 1377 |
+
|
| 1378 |
+
```bash
|
| 1379 |
+
sudo shutdown -h now
|
| 1380 |
+
```
|
| 1381 |
+
|
| 1382 |
+
The SSH session closed and a later SSH check to the last public IP
|
| 1383 |
+
`3.236.132.219` timed out. Refresh AWS SSO and confirm the EC2 state before any
|
| 1384 |
+
future GPU work:
|
| 1385 |
+
|
| 1386 |
+
```bash
|
| 1387 |
+
aws sso login --profile playaround
|
| 1388 |
+
aws ec2 describe-instances \
|
| 1389 |
+
--profile playaround \
|
| 1390 |
+
--region us-east-1 \
|
| 1391 |
+
--instance-ids i-0fa19dff7b233a353 \
|
| 1392 |
+
--query 'Reservations[0].Instances[0].State.Name' \
|
| 1393 |
+
--output text
|
| 1394 |
+
```
|
| 1395 |
+
|
| 1396 |
+
For future runs, do not stop the instance while a PID is still making progress.
|
| 1397 |
+
Stop it as soon as the run completes, fails, or no longer needs the GPU:
|
| 1398 |
+
|
| 1399 |
+
```bash
|
| 1400 |
+
aws ec2 stop-instances \
|
| 1401 |
+
--profile playaround \
|
| 1402 |
+
--region us-east-1 \
|
| 1403 |
+
--instance-ids i-0fa19dff7b233a353
|
| 1404 |
+
```
|
docs/VALIDATION.md
ADDED
|
@@ -0,0 +1,638 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Validation Design
|
| 2 |
+
|
| 3 |
+
The claim target is not "we ran OCR and it looked good." The claim target is:
|
| 4 |
+
|
| 5 |
+
```text
|
| 6 |
+
ocrtech is better than named baselines on held-out Nepali + English OCR and document-structure extraction, under measured gates.
|
| 7 |
+
```
|
| 8 |
+
|
| 9 |
+
Until the validation report says `validated`, do not claim SOTA.
|
| 10 |
+
|
| 11 |
+
Claim-grade `validation.json` files record provenance for the benchmark report,
|
| 12 |
+
validation config, eval manifest, train manifests, candidate model card, and
|
| 13 |
+
fallback model card used by `validate-claim`. `review-claim` cross-checks
|
| 14 |
+
those hashes against `experiment.json`, so a validation artifact from a
|
| 15 |
+
different benchmark, eval split, validation config, model card, fallback card,
|
| 16 |
+
or training-manifest set cannot be reused as portfolio evidence.
|
| 17 |
+
|
| 18 |
+
The SOTA review profile is paired with these claim-grade validation configs:
|
| 19 |
+
|
| 20 |
+
- `configs/validation-internal-sota-v1.json`
|
| 21 |
+
- `configs/validation-public-omnidocbench-sota-v1.json`
|
| 22 |
+
- `configs/validation-public-real5-sota-v1.json`
|
| 23 |
+
|
| 24 |
+
Each one covers the `required_validation_gates` declared in
|
| 25 |
+
`configs/claim-review-sota-v1.json`. Use the slot-specific config when running
|
| 26 |
+
`run-experiment` or `validate-claim`; otherwise portfolio review may have
|
| 27 |
+
metrics and baselines but still lack the passed gates required for the paper
|
| 28 |
+
claim.
|
| 29 |
+
|
| 30 |
+
## Evidence Required
|
| 31 |
+
|
| 32 |
+
A defensible run needs all of this:
|
| 33 |
+
|
| 34 |
+
- A claim-ready image eval pack from `ocrtech create-eval-pack`, plus public benchmark runs.
|
| 35 |
+
- A candidate model card from `ocrtech create-model-card` and a passing `ocrtech audit-model` report.
|
| 36 |
+
- Training data prepared with `audit-manifest` and `split-manifest`.
|
| 37 |
+
- Held-out eval manifest that was never used for recognizer, corrector, threshold, or prompt tuning.
|
| 38 |
+
- Image/PDF eval inputs normalized to OCR-supported file suffixes before any
|
| 39 |
+
claim-grade PaddleOCR/Tesseract/Surya/VLM comparison.
|
| 40 |
+
- For routed or thresholded candidates, a separate calibration experiment artifact that records how the threshold was chosen.
|
| 41 |
+
- Train manifests for leakage checks.
|
| 42 |
+
- Paired benchmark outputs for every baseline on the same input files.
|
| 43 |
+
- Per-sample metrics, not only aggregate averages.
|
| 44 |
+
- For `glm-ocr` and `paddleocr-vl`, a real runnable command hook that emits Markdown or `document.md`, not a placeholder env var.
|
| 45 |
+
- Slice coverage for Nepali, English, multi-page documents, tables, reading order, figures/charts, forms, scans, photos, skew, blur, compression, and low contrast.
|
| 46 |
+
- Gates that name the baseline, metric, slice, direction, required margin, and confidence requirement.
|
| 47 |
+
- Paired win/loss evidence, a minimum paired win-rate requirement when the
|
| 48 |
+
claim needs broad per-document improvement, and a nonparametric significance
|
| 49 |
+
check when the claim requires it.
|
| 50 |
+
- Failure-case report, not just the winning average.
|
| 51 |
+
|
| 52 |
+
## Public Benchmark Gate
|
| 53 |
+
|
| 54 |
+
Generated eval packs are development evidence. They are not enough for SOTA
|
| 55 |
+
wording. Before a paper claim can say "state of the art," at least one public
|
| 56 |
+
document parsing benchmark must be converted into an auditable manifest and run
|
| 57 |
+
as a normal `ocrtech run-experiment`.
|
| 58 |
+
|
| 59 |
+
Known public benchmark registry:
|
| 60 |
+
|
| 61 |
+
```bash
|
| 62 |
+
ocrtech list-public-benchmarks
|
| 63 |
+
```
|
| 64 |
+
|
| 65 |
+
Current registry entries:
|
| 66 |
+
|
| 67 |
+
- `opendatalab/OmniDocBench`: real-world document parsing benchmark, dataset page `https://huggingface.co/datasets/opendatalab/OmniDocBench`
|
| 68 |
+
- `allenai/olmOCR-bench`: PDF-to-Markdown benchmark with unit-test style assertions, dataset page `https://huggingface.co/datasets/allenai/olmOCR-bench`
|
| 69 |
+
- `PaddlePaddle/Real5-OmniDocBench`: real-world physical reconstruction benchmark with scanning, warping, screen-photo, illumination, and skew scenarios, dataset page `https://huggingface.co/datasets/PaddlePaddle/Real5-OmniDocBench`
|
| 70 |
+
|
| 71 |
+
The first implemented public converter is:
|
| 72 |
+
|
| 73 |
+
```bash
|
| 74 |
+
ocrtech prepare-public-benchmark opendatalab/OmniDocBench \
|
| 75 |
+
--out data/public/omnidocbench \
|
| 76 |
+
--limit 100 \
|
| 77 |
+
--language english \
|
| 78 |
+
--require-tables
|
| 79 |
+
```
|
| 80 |
+
|
| 81 |
+
The converter was written only after checking the actual dataset structure:
|
| 82 |
+
`OmniDocBench.json` contains `page_info`, `page_info.page_attribute`, and
|
| 83 |
+
`layout_dets`; image files resolve as `images/{page_info.image_path}`; table
|
| 84 |
+
ground truth is stored in table-block `html`.
|
| 85 |
+
|
| 86 |
+
Every public benchmark manifest row must include:
|
| 87 |
+
|
| 88 |
+
- `metadata.public_benchmark = true`
|
| 89 |
+
- `metadata.benchmark_name`
|
| 90 |
+
- `metadata.source_url`
|
| 91 |
+
- `metadata.reference_path`, unless the benchmark has a separate official scorer integration
|
| 92 |
+
- `metadata.slices` including `public` plus benchmark-specific slices
|
| 93 |
+
|
| 94 |
+
Audit it before running:
|
| 95 |
+
|
| 96 |
+
```bash
|
| 97 |
+
ocrtech audit-public-benchmark data/public/omnidocbench/manifests/omnidocbench-eval.jsonl \
|
| 98 |
+
--benchmark opendatalab/OmniDocBench \
|
| 99 |
+
--out outputs/public-omnidocbench-audit
|
| 100 |
+
```
|
| 101 |
+
|
| 102 |
+
This audit intentionally fails if provenance or references are missing. Do not
|
| 103 |
+
use `--allow-missing-reference-paths` for claim runs unless an official external
|
| 104 |
+
scorer is wired and documented in the experiment report.
|
| 105 |
+
|
| 106 |
+
Portfolio review for SOTA claims should use:
|
| 107 |
+
|
| 108 |
+
```bash
|
| 109 |
+
ocrtech review-claim \
|
| 110 |
+
--experiment outputs/internal-experiment/experiment.json \
|
| 111 |
+
--experiment outputs/public-omnidocbench/experiment.json \
|
| 112 |
+
--config configs/claim-review-sota-v1.json \
|
| 113 |
+
--model-config outputs/figure-caption-table-lexicon-v1/model-card.json \
|
| 114 |
+
--out outputs/claim-review-sota-v1
|
| 115 |
+
```
|
| 116 |
+
|
| 117 |
+
The config requires internal Nepali-English evidence plus public benchmark
|
| 118 |
+
evidence. It should fail until those public experiments actually exist.
|
| 119 |
+
|
| 120 |
+
Claim-grade validation configs can also set `min_pair_win_rate` on a gate. This
|
| 121 |
+
is intentionally different from the reported `win_rate`: `win_rate` is computed
|
| 122 |
+
over non-tied pairs, while `min_pair_win_rate` is checked as wins divided by all
|
| 123 |
+
paired samples. That distinction matters for document OCR claims. A result with
|
| 124 |
+
one win and many ties can have `win_rate = 1.0`, but it has low pair coverage
|
| 125 |
+
and should not satisfy a broad "better than baseline" claim unless the gate is
|
| 126 |
+
explicitly a no-regression/tie-acceptable gate.
|
| 127 |
+
|
| 128 |
+
The public OmniDocBench-vs-Surya validation gate is:
|
| 129 |
+
|
| 130 |
+
```bash
|
| 131 |
+
ocrtech validate-claim \
|
| 132 |
+
--benchmark-report outputs/experiment-public-omnidocbench-smoke-v1/benchmark/report.json \
|
| 133 |
+
--config configs/validation-public-omnidocbench-surya.json \
|
| 134 |
+
--eval-manifest data/public/omnidocbench-smoke/manifests/omnidocbench-eval.jsonl \
|
| 135 |
+
--candidate-model-config runs/recognizer/package/model-card.json \
|
| 136 |
+
--fallback-model-config runs/vlm-fallback/package/model-card.json \
|
| 137 |
+
--out outputs/validation-public-omnidocbench-smoke-v1
|
| 138 |
+
```
|
| 139 |
+
|
| 140 |
+
It requires at least `100` public document-parsing samples and at least `40`
|
| 141 |
+
table samples, then gates candidate-vs-Surya CER, table cell F1, structure-only
|
| 142 |
+
TEDS, and text-sensitive TEDS. A 5-page smoke run is allowed as engineering
|
| 143 |
+
evidence, but it must remain `not_validated`.
|
| 144 |
+
|
| 145 |
+
The first audited manifest sized for this gate is:
|
| 146 |
+
|
| 147 |
+
```text
|
| 148 |
+
data/public/omnidocbench-table100/manifests/omnidocbench-eval.jsonl
|
| 149 |
+
```
|
| 150 |
+
|
| 151 |
+
Its audit artifact is
|
| 152 |
+
`outputs/public-omnidocbench-table100-audit/public-benchmark-audit.json`; the
|
| 153 |
+
audit passed with `100` public/document/table English samples. The corresponding
|
| 154 |
+
long benchmark run is tracked under
|
| 155 |
+
`outputs/experiment-public-omnidocbench-table100-width-guard-v4/`. Treat that
|
| 156 |
+
run as incomplete until `benchmark/report.json`, `summary/summary.json`,
|
| 157 |
+
`validation/validation.json`, and `table-cell-analysis/table-cell-analysis.json`
|
| 158 |
+
exist.
|
| 159 |
+
|
| 160 |
+
The completed width-guard-v4 Table100 run is an instructive non-passing result:
|
| 161 |
+
|
| 162 |
+
- rows: `200/200`, with one shared `300s` timeout for candidate and Surya
|
| 163 |
+
- paired scored samples: `99`
|
| 164 |
+
- candidate improves public table metrics over Surya:
|
| 165 |
+
`table_cell_f1` `0.484364` vs `0.406694`,
|
| 166 |
+
`table_teds_structure` `0.720160` vs `0.687550`,
|
| 167 |
+
`table_teds_text` `0.565284` vs `0.543987`
|
| 168 |
+
- candidate does not improve OCR CER:
|
| 169 |
+
`0.613201` vs Surya `0.612623`
|
| 170 |
+
- validation status: `not_validated`
|
| 171 |
+
|
| 172 |
+
For long-run status files, distinguish package readiness from claim readiness.
|
| 173 |
+
`evidence_package_ready` means the expected rows and final report artifacts are
|
| 174 |
+
present. `paper_claim_ready` must only be true when the validation report itself
|
| 175 |
+
passes. A completed benchmark package with `validation.claim_status =
|
| 176 |
+
not_validated` is useful evidence, but not a publishable SOTA claim.
|
| 177 |
+
|
| 178 |
+
Long public runs should use benchmark checkpointing and per-page timeouts:
|
| 179 |
+
|
| 180 |
+
```bash
|
| 181 |
+
ocrtech benchmark ... \
|
| 182 |
+
--inputs-from-manifest data/eval/manifest-with-refs.jsonl \
|
| 183 |
+
--resume-existing \
|
| 184 |
+
--sample-timeout-seconds 300
|
| 185 |
+
```
|
| 186 |
+
|
| 187 |
+
`--inputs-from-manifest` loads authoritative image paths from the manifest and
|
| 188 |
+
defaults `--eval-manifest` to the same file when it is not supplied. This avoids
|
| 189 |
+
shell-generated input lists drifting away from the manifest that contains
|
| 190 |
+
reference paths and slice labels. `--resume-existing` scores existing
|
| 191 |
+
`document.json` outputs and only runs OCR for missing pages.
|
| 192 |
+
`--sample-timeout-seconds` runs each OCR call behind a hard process watchdog on
|
| 193 |
+
platforms that support `fork`; a stalled backend is terminated and recorded as a
|
| 194 |
+
page-level `TimeoutError`, then the benchmark continues. Platforms without
|
| 195 |
+
`fork` fall back to the Python signal timer. A timeout is still a
|
| 196 |
+
candidate/baseline failure for validation; it is not a pass-through.
|
| 197 |
+
For timeout runs, the benchmark now constructs the OCR engine inside the
|
| 198 |
+
watchdog child process for each page. This avoids inheriting already-initialized
|
| 199 |
+
PaddleOCR/PaddlePaddle C++ runtime state across `fork`, which produced
|
| 200 |
+
`exit_code=-11` worker crashes in the first line-level Paddle composite run.
|
| 201 |
+
|
| 202 |
+
Table scoring matches each reference table to the best unused predicted table
|
| 203 |
+
before averaging table metrics. This matters on public document pages where an
|
| 204 |
+
engine may emit extra table fragments before the main table. Positional
|
| 205 |
+
table-index matching is too brittle for claim evidence.
|
| 206 |
+
|
| 207 |
+
When the evaluator changes, do not rerun slow OCR just to update metrics. Use:
|
| 208 |
+
|
| 209 |
+
```bash
|
| 210 |
+
ocrtech rescore-benchmark \
|
| 211 |
+
--benchmark-report outputs/experiment-public-omnidocbench-smoke-v1/benchmark/report.json \
|
| 212 |
+
--eval-manifest data/public/omnidocbench-smoke/manifests/omnidocbench-eval.jsonl \
|
| 213 |
+
--out outputs/experiment-public-omnidocbench-smoke-v2-rescored/benchmark
|
| 214 |
+
```
|
| 215 |
+
|
| 216 |
+
The rescore command copies the existing `document.json`, Markdown, table, and
|
| 217 |
+
figure outputs into the new benchmark directory so downstream diagnostics such
|
| 218 |
+
as `analyze-table-cells` remain usable.
|
| 219 |
+
|
| 220 |
+
When evaluating table postprocessors, do not rely only on separate candidate and
|
| 221 |
+
baseline OCR runs. Independent OCR/layout calls can introduce run-to-run variance
|
| 222 |
+
that makes a small table postprocessor look better or worse than it is. Use a
|
| 223 |
+
fixed-source derived comparison to isolate the postprocessor:
|
| 224 |
+
|
| 225 |
+
```bash
|
| 226 |
+
ocrtech derive-table-postprocess-benchmark \
|
| 227 |
+
--benchmark-report outputs/experiment-public-omnidocbench-table40-normalize-rows-v3/benchmark/report.json \
|
| 228 |
+
--source-baseline surya \
|
| 229 |
+
--candidate-baseline candidate \
|
| 230 |
+
--eval-manifest data/public/omnidocbench-table40/manifests/omnidocbench-eval.jsonl \
|
| 231 |
+
--out outputs/experiment-public-omnidocbench-table40-derived-postprocess-v1/benchmark
|
| 232 |
+
```
|
| 233 |
+
|
| 234 |
+
This command copies the chosen source baseline outputs, applies the current
|
| 235 |
+
reference-free table postprocessors to the copied candidate `document.json`, and
|
| 236 |
+
rescored both systems against the same references. The result is valid evidence
|
| 237 |
+
for the postprocessor contribution only. It is not an end-to-end OCR model card
|
| 238 |
+
and cannot support a broad OCR SOTA claim by itself.
|
| 239 |
+
|
| 240 |
+
The current normalized public candidate is
|
| 241 |
+
`ocrtech-surya-tesseract-figure-table-lexicon-normalize-v2`. Its table-grid
|
| 242 |
+
normalization is runtime-only and reference-free: empty columns, separator-only
|
| 243 |
+
columns, and exact duplicate adjacent columns are removed. On the 5-page
|
| 244 |
+
OmniDocBench smoke it improves table metrics over Surya, but the run remains
|
| 245 |
+
`not_validated` because the public sample count, table sample count, CER margin,
|
| 246 |
+
and bootstrap confidence requirements are not met.
|
| 247 |
+
|
| 248 |
+
## Reference Format
|
| 249 |
+
|
| 250 |
+
Text-only reference:
|
| 251 |
+
|
| 252 |
+
```text
|
| 253 |
+
page.png.ref.txt
|
| 254 |
+
```
|
| 255 |
+
|
| 256 |
+
Structured reference:
|
| 257 |
+
|
| 258 |
+
```json
|
| 259 |
+
{
|
| 260 |
+
"text": "नेपाल सरकार",
|
| 261 |
+
"reading_order": ["p0-l0", "p0-l1", "p0-l2"],
|
| 262 |
+
"tables": [
|
| 263 |
+
[
|
| 264 |
+
["नाम", "उमेर", "जिल्ला"],
|
| 265 |
+
["राम", "३२", "काठमाडौं"]
|
| 266 |
+
]
|
| 267 |
+
],
|
| 268 |
+
"metadata": {
|
| 269 |
+
"source": "hand_labeled"
|
| 270 |
+
}
|
| 271 |
+
}
|
| 272 |
+
```
|
| 273 |
+
|
| 274 |
+
Complex table references can also use cell objects with spans:
|
| 275 |
+
|
| 276 |
+
```json
|
| 277 |
+
{
|
| 278 |
+
"tables": [
|
| 279 |
+
{
|
| 280 |
+
"cells": [
|
| 281 |
+
{"row": 0, "col": 0, "text": "नाम", "rowspan": 1, "colspan": 2},
|
| 282 |
+
{"row": 1, "col": 0, "text": "राम"},
|
| 283 |
+
{"row": 1, "col": 1, "text": "१२"}
|
| 284 |
+
]
|
| 285 |
+
}
|
| 286 |
+
]
|
| 287 |
+
}
|
| 288 |
+
```
|
| 289 |
+
|
| 290 |
+
The nested-list grid form remains supported. Span-aware references make `table_exact_match` and `table_structure_similarity` sensitive to missing rowspans/colspans even when flattened table text is correct.
|
| 291 |
+
|
| 292 |
+
The candidate parser can apply a conservative repeated-header span postprocessor:
|
| 293 |
+
adjacent identical cells in the first header rows are collapsed into a single
|
| 294 |
+
`colspan` cell only when the following row has distinct child columns. This lets
|
| 295 |
+
`document.json` and exported table HTML represent the simple merged-header
|
| 296 |
+
pattern in `complex-table-ledger`. Treat this as a heuristic parser repair until
|
| 297 |
+
it is validated on real scanned tables with varied merged cells and rowspans.
|
| 298 |
+
|
| 299 |
+
Supported structured metrics:
|
| 300 |
+
|
| 301 |
+
- `quality_score`
|
| 302 |
+
- `cer`
|
| 303 |
+
- `wer`
|
| 304 |
+
- `text_cer`
|
| 305 |
+
- `text_wer`
|
| 306 |
+
- `document_reconstruction_cer`
|
| 307 |
+
- `document_reconstruction_wer`
|
| 308 |
+
- `document_reconstruction_similarity`
|
| 309 |
+
- `figure_detection_precision`
|
| 310 |
+
- `figure_detection_recall`
|
| 311 |
+
- `figure_detection_f1`
|
| 312 |
+
- `figure_caption_cer`
|
| 313 |
+
- `reading_order_pair_accuracy`
|
| 314 |
+
- `table_cell_precision`
|
| 315 |
+
- `table_cell_recall`
|
| 316 |
+
- `table_cell_f1`
|
| 317 |
+
- `table_exact_match`
|
| 318 |
+
- `table_structure_similarity`
|
| 319 |
+
- `table_teds_structure`
|
| 320 |
+
- `table_teds_text`
|
| 321 |
+
|
| 322 |
+
## Validation Config
|
| 323 |
+
|
| 324 |
+
Example:
|
| 325 |
+
|
| 326 |
+
```json
|
| 327 |
+
{
|
| 328 |
+
"candidate": "ours",
|
| 329 |
+
"min_samples": 300,
|
| 330 |
+
"required_slices": {
|
| 331 |
+
"nepali": 100,
|
| 332 |
+
"english": 100,
|
| 333 |
+
"multi_page": 25,
|
| 334 |
+
"table": 50,
|
| 335 |
+
"reading_order": 50,
|
| 336 |
+
"figure": 30,
|
| 337 |
+
"scan": 50,
|
| 338 |
+
"phone_photo": 50
|
| 339 |
+
},
|
| 340 |
+
"bootstrap_iterations": 1000,
|
| 341 |
+
"bootstrap_confidence": 0.95,
|
| 342 |
+
"require_full_eval_manifest_coverage": true,
|
| 343 |
+
"gates": [
|
| 344 |
+
{
|
| 345 |
+
"baseline": "tesseract",
|
| 346 |
+
"metric": "cer",
|
| 347 |
+
"direction": "lower",
|
| 348 |
+
"slice": "nepali",
|
| 349 |
+
"min_pairs": 50,
|
| 350 |
+
"min_relative_improvement": 0.05,
|
| 351 |
+
"max_p_value": 0.05
|
| 352 |
+
},
|
| 353 |
+
{
|
| 354 |
+
"baseline": "stock-paddle",
|
| 355 |
+
"metric": "cer",
|
| 356 |
+
"direction": "lower",
|
| 357 |
+
"slice": "english",
|
| 358 |
+
"min_absolute_margin": 0.0
|
| 359 |
+
},
|
| 360 |
+
{
|
| 361 |
+
"baseline": "glm-ocr",
|
| 362 |
+
"metric": "table_cell_f1",
|
| 363 |
+
"direction": "higher",
|
| 364 |
+
"slice": "table",
|
| 365 |
+
"min_absolute_margin": 0.01
|
| 366 |
+
},
|
| 367 |
+
{
|
| 368 |
+
"baseline": "surya",
|
| 369 |
+
"metric": "cer",
|
| 370 |
+
"direction": "lower",
|
| 371 |
+
"slices": ["hard_eval", "table"],
|
| 372 |
+
"min_pairs": 8,
|
| 373 |
+
"min_absolute_margin": 0.02
|
| 374 |
+
},
|
| 375 |
+
{
|
| 376 |
+
"baseline": "paddleocr-vl",
|
| 377 |
+
"metric": "reading_order_pair_accuracy",
|
| 378 |
+
"direction": "higher",
|
| 379 |
+
"slice": "reading_order",
|
| 380 |
+
"min_absolute_margin": 0.01
|
| 381 |
+
}
|
| 382 |
+
]
|
| 383 |
+
}
|
| 384 |
+
```
|
| 385 |
+
|
| 386 |
+
Use `slice` for a single required slice. Use `slices` when a gate must apply to the intersection of multiple labels, for example `["hard_eval", "table"]` for table samples inside a hard-eval pocket. Do not approximate this with separate `hard_eval` and `table` gates; that allows unrelated samples to satisfy the table gate.
|
| 387 |
+
|
| 388 |
+
For claim-grade runs, set `require_full_eval_manifest_coverage` to `true`.
|
| 389 |
+
When enabled, `validate-claim` fails unless the candidate and every baseline
|
| 390 |
+
named by the configured gates have a successful row for every sample in
|
| 391 |
+
`--eval-manifest`. This prevents a validation report from passing on an easy
|
| 392 |
+
subset while silently omitting hard pages from the held-out manifest.
|
| 393 |
+
|
| 394 |
+
Run:
|
| 395 |
+
|
| 396 |
+
```bash
|
| 397 |
+
ocrtech preflight-claim \
|
| 398 |
+
--eval-manifest data/manifests/eval.jsonl \
|
| 399 |
+
--references-dir data/references \
|
| 400 |
+
--baselines candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
|
| 401 |
+
--candidate-model-config runs/recognizer/package/model-card.json \
|
| 402 |
+
--validation-config validation.json \
|
| 403 |
+
--train-manifest data/manifests/train.jsonl \
|
| 404 |
+
--out outputs/preflight
|
| 405 |
+
|
| 406 |
+
ocrtech benchmark \
|
| 407 |
+
--eval-manifest data/manifests/eval.jsonl \
|
| 408 |
+
--references-dir data/references \
|
| 409 |
+
--baselines ours,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
|
| 410 |
+
--out outputs/benchmark \
|
| 411 |
+
$(cut -f4 data/manifests/eval.inputs)
|
| 412 |
+
|
| 413 |
+
ocrtech validate-claim \
|
| 414 |
+
--benchmark-report outputs/benchmark/report.json \
|
| 415 |
+
--config validation.json \
|
| 416 |
+
--eval-manifest data/manifests/eval.jsonl \
|
| 417 |
+
--train-manifest data/manifests/train.jsonl \
|
| 418 |
+
--out outputs/validation
|
| 419 |
+
```
|
| 420 |
+
|
| 421 |
+
`preflight-claim` should be the first command in a claim-grade workflow. If it fails, the benchmark run is not ready.
|
| 422 |
+
|
| 423 |
+
For explicit `--validation-config` runs, preflight now checks the config against
|
| 424 |
+
the requested baselines and eval manifest before any OCR starts. It fails when
|
| 425 |
+
the candidate label is absent from `--baselines`, a gate baseline is not
|
| 426 |
+
scheduled, `min_samples` exceeds the manifest size, required slices are missing,
|
| 427 |
+
or a gate's `min_pairs` exceeds the number of manifest rows matching the gate
|
| 428 |
+
slice intersection. Omitting `--validation-config` is only a warning; it is not
|
| 429 |
+
claim-grade evidence.
|
| 430 |
+
|
| 431 |
+
For hybrid runs with `--fallback-model-config`, preflight also checks that the
|
| 432 |
+
candidate model card declares the same fallback under
|
| 433 |
+
`provenance.fallback_model_card`. This catches a wrong runtime fallback flag
|
| 434 |
+
before a benchmark produces evidence for a different composite engine than the
|
| 435 |
+
one described by the candidate model card.
|
| 436 |
+
|
| 437 |
+
If the candidate model uses `quality_select_composite`, `preflight-claim` now checks that the model card contains calibration provenance and that the calibration eval manifest hash is different from the final claim eval manifest hash. Tuning on the same eval pack invalidates the claim path.
|
| 438 |
+
|
| 439 |
+
`validate-claim` exits with `0` only when the configured claim is validated. It exits with `1` when evidence is missing or a gate fails.
|
| 440 |
+
|
| 441 |
+
Each gate now reports:
|
| 442 |
+
|
| 443 |
+
- paired sample count
|
| 444 |
+
- win / loss / tie counts
|
| 445 |
+
- win rate on non-tied pairs
|
| 446 |
+
- bootstrap CI for mean paired improvement
|
| 447 |
+
- optional one-sided sign-test p-value
|
| 448 |
+
|
| 449 |
+
For claim-grade runs, use `min_pairs` and `max_p_value` instead of relying only on a favorable mean. A narrow win on a handful of pages is not SOTA evidence.
|
| 450 |
+
|
| 451 |
+
For a single reproducible command that runs benchmark and validation together, use `run-experiment`; see [EXPERIMENTS.md](EXPERIMENTS.md).
|
| 452 |
+
|
| 453 |
+
For the final portfolio-level claim review across the internal pack and public suites, use `review-claim`; see [CLAIM_REVIEW.md](CLAIM_REVIEW.md).
|
| 454 |
+
|
| 455 |
+
For model artifact provenance, see [MODELS.md](MODELS.md). Do not use an implicit `auto` engine for SOTA claims.
|
| 456 |
+
|
| 457 |
+
## Table Cell Analysis
|
| 458 |
+
|
| 459 |
+
Aggregate `table_cell_f1` is not enough to explain whether a table-oriented candidate is actually improving table OCR. Use `analyze-table-cells` after a benchmark has exported per-system `document.json` files:
|
| 460 |
+
|
| 461 |
+
```bash
|
| 462 |
+
ocrtech analyze-table-cells \
|
| 463 |
+
--benchmark-report outputs/experiment-table-cell-refine-empty-hard15-v1/benchmark/report.json \
|
| 464 |
+
--eval-manifest data/splits-claimpack-rendered-v2/eval-surya-hard15.jsonl \
|
| 465 |
+
--out outputs/table-cell-analysis-refine-empty-hard15-v1 \
|
| 466 |
+
--candidate candidate \
|
| 467 |
+
--baseline surya \
|
| 468 |
+
--baseline tesseract
|
| 469 |
+
```
|
| 470 |
+
|
| 471 |
+
The command requires:
|
| 472 |
+
|
| 473 |
+
- A benchmark report from `benchmark` or `run-experiment`.
|
| 474 |
+
- Exported benchmark documents at `benchmark/<system>/<input_stem>/document.json`.
|
| 475 |
+
- An eval manifest whose references contain structured `tables`.
|
| 476 |
+
|
| 477 |
+
It writes:
|
| 478 |
+
|
| 479 |
+
- `table-cell-analysis.json`
|
| 480 |
+
- `table-cell-analysis.md`
|
| 481 |
+
|
| 482 |
+
The report includes reference table count, predicted table count, duplicate overlapping table count, reference cell count, exact cell count, exact-cell rate, missing-cell count, mean cell CER, paired candidate-vs-baseline wins/losses/ties, and the highest-error candidate cells. This is the right artifact for identifying whether a table refiner is adding headroom, damaging already-correct cells, or producing structurally duplicated tables.
|
| 483 |
+
|
| 484 |
+
Interpret raw full-page Tesseract table results carefully. In the current v1 engine, Tesseract can be strong on exact image crops but still report zero structured table cells in full-page document parsing because it does not emit the table structure expected by `document.json`.
|
| 485 |
+
|
| 486 |
+
For the rendered hard15 table pages, reference `text` is the page title, while table contents live in structured `tables`. Do not mix these contracts: page-level `cer` answers whether the surrounding page text/title was recognized, and `table_cell_f1` plus cell exact analysis answers whether the table content was extracted. A benchmark change that inserts table grids into `Document.text` without also regenerating references changes the metric definition and is invalid for this pack.
|
| 487 |
+
|
| 488 |
+
The current `validation-hard15-surya.json` table-cell gate can fail even with a higher mean table-cell F1 when the bootstrap confidence interval includes zero. The figure-caption/table-lexicon v3 run is the concrete example: candidate `1.000000` vs Surya `0.979167`, but only 2 table pages improve and 6 tie, so the CI low is `0.000000`. Treat that as evidence that the table slice is too small for a strong claim. Do not remove the CI gate after observing the result; build a larger held-out table pack.
|
| 489 |
+
|
| 490 |
+
`configs/validation-table-ledger-surya.json` is the first targeted table-only validation config. It requires at least 40 table samples, gates `table_cell_f1` and `table_exact_match` against Surya with bootstrap CI above zero, and guards `table_structure_similarity` for non-regression. The v2 table-ledger pack (`60` samples, `720` cells) passes this config:
|
| 491 |
+
|
| 492 |
+
- `table_cell_f1`: candidate `1.000000`, Surya `0.979167`, CI low `0.012500`
|
| 493 |
+
- `table_exact_match`: candidate `1.000000`, Surya `0.750000`, CI low `0.150000`
|
| 494 |
+
- `table_structure_similarity`: candidate `1.000000`, Surya `1.000000`, non-regression guard passes
|
| 495 |
+
|
| 496 |
+
`table_structure_similarity` is a deterministic table-structure score over row count, column count, occupied cell coordinates, and rowspans/colspans when structured cell references are present. It is still not full HTML tree-edit TEDS. In the table-ledger v2 run it correctly shows no structure improvement over Surya because both systems already recover the simple grid structure perfectly. The candidate's validated gain is table cell-content accuracy with structure non-regression.
|
| 497 |
+
|
| 498 |
+
`table_teds_structure` and `table_teds_text` are deterministic TEDS-style ordered tree-edit scores over a table DOM made of `table`, `tr`, and `td` nodes. Structure mode compares row/cell tree shape plus `rowspan`/`colspan`; text mode also includes normalized cell text in `td` node labels. These are closer to paper-grade table metrics than the older token score, but they are still internal deterministic metrics, not a substitute for running an accepted public benchmark implementation when making broad SOTA claims.
|
| 499 |
+
|
| 500 |
+
After the span-inference postprocessor and tree metrics, the generated complex-table pack shows a structure gain over Surya (`table_structure_similarity` `1.000000` vs `0.875000`, `table_teds_structure` `1.000000` vs `0.904762`, `table_teds_text` `0.996250` vs `0.891667`, `table_exact_match` `0.925000` vs `0.000000`). This is evidence for the generated repeated-header pattern only. It does not replace a public or real scanned-table benchmark.
|
| 501 |
+
|
| 502 |
+
The public OmniDocBench smoke now also has a row-fragment normalization
|
| 503 |
+
candidate. It improves public table metrics over Surya on the same 5-page smoke
|
| 504 |
+
(`table_cell_f1` `0.564335` vs `0.296264`, `table_teds_structure` `0.942836`
|
| 505 |
+
vs `0.740128`, `table_teds_text` `0.675633` vs `0.526327`) and fixes one
|
| 506 |
+
`16x5` fragmented table into the expected `3x5` shape. It is still
|
| 507 |
+
`not_validated`: the sample count is far below the configured public gates,
|
| 508 |
+
bootstrap CI lower bounds are `0.000000` on table metrics, and text CER is
|
| 509 |
+
slightly worse than Surya on the smoke (`0.481629` vs `0.480531`).
|
| 510 |
+
|
| 511 |
+
The larger public OmniDocBench `table40` run is more important. It reaches the
|
| 512 |
+
configured table-pair count (`40` public table pages) but still fails validation:
|
| 513 |
+
`table_cell_f1` improves in mean (`0.323122` vs `0.280062`) while losing more
|
| 514 |
+
pages than it wins (`7` wins, `9` losses, `24` ties), and the bootstrap CI low is
|
| 515 |
+
negative (`-0.019152`). `table_teds_structure` also has a positive mean
|
| 516 |
+
(`0.520959` vs `0.503603`) but CI low is negative (`-0.038381`). `table_teds_text`
|
| 517 |
+
does not clear the margin (`0.003609` improvement). The document CER gate is
|
| 518 |
+
also not met: only `40/100` required public document pages, and CER improvement
|
| 519 |
+
is only `0.000634` against a `0.005` margin.
|
| 520 |
+
|
| 521 |
+
The width-guarded v4 rerun improves the public table40 evidence but still does
|
| 522 |
+
not validate the full claim. Skipping table-grid normalization on tables wider
|
| 523 |
+
than `12` columns makes the table-cell F1 gate pass (`0.339319` vs `0.280062`,
|
| 524 |
+
CI low `0.001433`) and the table TEDS-structure gate pass (`0.545375` vs
|
| 525 |
+
`0.503603`, CI low `0.004790`). The text-sensitive table gate still fails:
|
| 526 |
+
`table_teds_text` improves in mean (`0.386013` vs `0.358057`) but CI low is
|
| 527 |
+
`-0.000220`. The public document CER gate also still fails because the slice has
|
| 528 |
+
only `40/100` required public document pages, despite a mean CER improvement of
|
| 529 |
+
`0.006096`.
|
| 530 |
+
|
| 531 |
+
On `2026-06-14`, the `table100` public run was launched on `k2` using
|
| 532 |
+
`scripts/run_public_table100_claim.py` and the v4 width-guard model card. The
|
| 533 |
+
first all-or-nothing launch stalled after three completed candidate rows. We
|
| 534 |
+
restarted with an isolated per-page runner, `--resume-existing`, and
|
| 535 |
+
`--sample-timeout-seconds 300`, preserving completed outputs and forcing future
|
| 536 |
+
stalls to become explicit error rows. The first measured timeout row was
|
| 537 |
+
`omnidocbench-00216-page-dca64e05-1ce6-49ee-857a-9b5d05a87357`. Do not cite this
|
| 538 |
+
run as a result until the validation artifact exists and has been inspected.
|
| 539 |
+
|
| 540 |
+
This validates a targeted table-ledger extraction claim, not full document SOTA. Full SOTA validation still needs broad document coverage, real scanned tables, merged-cell/header cases, true TEDS/TEDS-IOU, and public benchmark evidence.
|
| 541 |
+
|
| 542 |
+
## Recognizer Hard-Failure Gate
|
| 543 |
+
|
| 544 |
+
The failed AWS G6 PaddleOCR recognizer produced a normalized `50`-sample
|
| 545 |
+
hard-eval slice:
|
| 546 |
+
|
| 547 |
+
```text
|
| 548 |
+
data/splits-mixed-v1/eval-paddle-mixed-v2-cuda-hard50-normalized.jsonl
|
| 549 |
+
```
|
| 550 |
+
|
| 551 |
+
Use this slice as a regression gate for the next trained recognizer branch. It
|
| 552 |
+
must remain held out; do not train on it.
|
| 553 |
+
|
| 554 |
+
Validation config:
|
| 555 |
+
|
| 556 |
+
```text
|
| 557 |
+
configs/validation-paddle-hard50-tesseract.json
|
| 558 |
+
```
|
| 559 |
+
|
| 560 |
+
Run it after benchmarking a candidate recognizer against Tesseract:
|
| 561 |
+
|
| 562 |
+
```bash
|
| 563 |
+
ocrtech validate-claim \
|
| 564 |
+
--benchmark-report outputs/next-recognizer-hard50/benchmark/report.json \
|
| 565 |
+
--config configs/validation-paddle-hard50-tesseract.json \
|
| 566 |
+
--eval-manifest data/splits-mixed-v1/eval-paddle-mixed-v2-cuda-hard50-normalized.jsonl \
|
| 567 |
+
--out outputs/validation-next-recognizer-hard50
|
| 568 |
+
```
|
| 569 |
+
|
| 570 |
+
This gate is intentionally framed as failure recovery:
|
| 571 |
+
|
| 572 |
+
- all `50` hard rows must be present
|
| 573 |
+
- the `38` English synthetic line-crop failures must not regress versus
|
| 574 |
+
Tesseract on CER or WER
|
| 575 |
+
- the `12` Nepali hard failures must improve WER over Tesseract by at least
|
| 576 |
+
`0.05`
|
| 577 |
+
|
| 578 |
+
Passing this gate alone is not a SOTA claim. It is a recognizer branch admission
|
| 579 |
+
test before rerunning the full mixed OCR eval, rendered document pack, and public
|
| 580 |
+
benchmark gates.
|
| 581 |
+
|
| 582 |
+
The first line-level Tesseract/Paddle composite run exposed a benchmark-lifecycle
|
| 583 |
+
bug, not a model result: every candidate row crashed with `exit_code=-11` because
|
| 584 |
+
PaddleOCR was initialized in the parent process and then inherited by forked
|
| 585 |
+
timeout workers. After changing timeout mode to initialize engines inside the
|
| 586 |
+
watchdog worker, the rerun completed all `100` rows:
|
| 587 |
+
|
| 588 |
+
```text
|
| 589 |
+
outputs/experiment-line-script-paddle-hard50-v2/report.json
|
| 590 |
+
outputs/summary-line-script-paddle-hard50-v2/summary.md
|
| 591 |
+
outputs/validation-line-script-paddle-hard50-v2/validation.md
|
| 592 |
+
```
|
| 593 |
+
|
| 594 |
+
Validation status: `not_validated`.
|
| 595 |
+
|
| 596 |
+
- hard-50 CER: candidate `0.276489`, Tesseract `0.120106`
|
| 597 |
+
- English line-crop CER: candidate `0.060429`, Tesseract `0.060429`
|
| 598 |
+
- English line-crop WER: candidate `0.132259`, Tesseract `0.132259`
|
| 599 |
+
- Nepali hard CER: candidate `0.960679`, Tesseract `0.309083`
|
| 600 |
+
- Nepali hard WER: candidate `1.000000`, Tesseract `0.711607`
|
| 601 |
+
|
| 602 |
+
Interpretation: line-level routing can preserve English by leaving English lines
|
| 603 |
+
on the Tesseract primary path, but the trained PaddleOCR secondary recognizer is
|
| 604 |
+
not admissible for Devanagari overrides.
|
| 605 |
+
|
| 606 |
+
## Leakage Rules
|
| 607 |
+
|
| 608 |
+
Fatal leakage:
|
| 609 |
+
|
| 610 |
+
- same `sample_id`
|
| 611 |
+
- same image SHA-256
|
| 612 |
+
- same image path
|
| 613 |
+
- same image+text sample signature
|
| 614 |
+
|
| 615 |
+
Warning only:
|
| 616 |
+
|
| 617 |
+
- same text SHA-256
|
| 618 |
+
|
| 619 |
+
Text-only overlap is not fatal because short OCR labels repeat naturally across real corpora. It is still reported because repeated long labels may indicate bad dataset splitting.
|
| 620 |
+
|
| 621 |
+
## External Benchmarks
|
| 622 |
+
|
| 623 |
+
Use public suites for broad claims and the internal Nepali/English pack for the actual target domain:
|
| 624 |
+
|
| 625 |
+
- OmniDocBench for general document parsing.
|
| 626 |
+
- olmOCR-bench for PDF-to-Markdown structure preservation.
|
| 627 |
+
- OCRBench / OCRBench v2 for VLM OCR coverage.
|
| 628 |
+
- Internal Nepali + English pages for Devanagari OCR, mixed-script pages, government forms, tables, captions, and noisy scans.
|
| 629 |
+
|
| 630 |
+
For table claims, cell exact match, `table_cell_f1`, and span-aware `table_structure_similarity` are v1 metrics. Add true TEDS/TEDS-IOU before claiming leadership on complex merged-cell tables.
|
| 631 |
+
|
| 632 |
+
## Internal Eval Pack
|
| 633 |
+
|
| 634 |
+
See [EVAL_PACK.md](EVAL_PACK.md). The internal pack is necessary because public OCR/document benchmarks do not sufficiently cover Nepali, mixed Devanagari-English pages, and local document layouts.
|
| 635 |
+
|
| 636 |
+
## Dataset Splitting
|
| 637 |
+
|
| 638 |
+
See [DATASETS.md](DATASETS.md). Do not manually split data when making benchmark claims; use `split-manifest` so duplicate sample signatures cannot cross train/eval boundaries.
|
examples/sample-page.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
नेपाल सरकार
|
| 2 |
+
|
| 3 |
+
नाम उमेर जिल्ला
|
| 4 |
+
राम ३२ काठमाडौं
|
| 5 |
+
Sita 29 Lalitpur
|
| 6 |
+
|
| 7 |
+
This is a mixed Nepali and English page.
|
languages/nepal-v1.json
ADDED
|
@@ -0,0 +1,370 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"registry_id": "nepal-v1",
|
| 3 |
+
"description": "Nepal multilingual OCR target registry. Limbu/Sirijonga is the first non-Devanagari target; English is included for preservation but does not count toward the Nepal-language target.",
|
| 4 |
+
"minimum_nepal_language_count": 10,
|
| 5 |
+
"created_for": "PaddleOCR script-aware fine-tuning, synthesis, and real-document validation",
|
| 6 |
+
"claim_policy": "Synthetic data is allowed for training and ablations only. SOTA claims require held-out real/public validation artifacts.",
|
| 7 |
+
"languages": [
|
| 8 |
+
{
|
| 9 |
+
"id": "limbu",
|
| 10 |
+
"name": "Limbu / Yakthung",
|
| 11 |
+
"endonym": "ᤕᤠᤰᤌᤢᤱ",
|
| 12 |
+
"iso_639_3": "lif",
|
| 13 |
+
"priority": 1,
|
| 14 |
+
"counts_toward_nepal_language_target": true,
|
| 15 |
+
"first_target": true,
|
| 16 |
+
"scripts": [
|
| 17 |
+
{
|
| 18 |
+
"script": "Limbu / Sirijonga",
|
| 19 |
+
"primary": true,
|
| 20 |
+
"status": "encoded",
|
| 21 |
+
"unicode_ranges": ["U+1900..U+194F"],
|
| 22 |
+
"notes": "First non-Devanagari OCR target. Requires renderability and dictionary audits before training admission."
|
| 23 |
+
},
|
| 24 |
+
{
|
| 25 |
+
"script": "Devanagari",
|
| 26 |
+
"primary": false,
|
| 27 |
+
"status": "fallback_transcription",
|
| 28 |
+
"unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"],
|
| 29 |
+
"notes": "Useful for parallel/transcribed resources, not the primary Limbu-script claim target."
|
| 30 |
+
}
|
| 31 |
+
],
|
| 32 |
+
"font_candidates": [
|
| 33 |
+
{"name": "Noto Sans Limbu", "path": "/System/Library/Fonts/Supplemental/NotoSansLimbu-Regular.ttf", "status": "pending_render_audit"},
|
| 34 |
+
{"name": "local-ltk-fonts", "path": "/Users/cdjk/github/llm/ltk/fonts.zip", "status": "legacy_archive_pending_coverage_audit"}
|
| 35 |
+
],
|
| 36 |
+
"paddle_dictionary": "data/dictionaries/limbu-sirijonga-v1.txt",
|
| 37 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_limbu_letters_and_signs"},
|
| 38 |
+
"corpora": [
|
| 39 |
+
{"name": "LTK Limbu resources", "path": "/Users/cdjk/github/llm/ltk/limbu", "host": "cdjk", "type": "local_verified_directory", "status": "schema_audit_pending"},
|
| 40 |
+
{"name": "Limdic dictionary", "path": "/Users/cdjk/github/llm/ltk/limbu/data/limdic_dictionary.json", "host": "cdjk", "type": "lexicon", "status": "exists_local"},
|
| 41 |
+
{"name": "Pangloss dictionary TSV", "path": "/Users/cdjk/github/llm/ltk/limbu/data/pangloss_dictionary.tsv", "host": "cdjk", "type": "lexicon", "status": "exists_local"}
|
| 42 |
+
],
|
| 43 |
+
"validation_sources": [
|
| 44 |
+
{"name": "Gorkhapatra/Naya Nepal article images", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "pending_manual_label"},
|
| 45 |
+
{"name": "Gorkhapatra epaper PDFs", "path": "data/gorkhapatra-pack-smoke", "type": "real_document_validation_pack", "status": "pending_manual_label"}
|
| 46 |
+
],
|
| 47 |
+
"gorkhapatra_language_labels": ["लिम्बु"],
|
| 48 |
+
"notes": "Primary research target for low-resource script-aware PaddleOCR adaptation."
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"id": "nepali",
|
| 52 |
+
"name": "Nepali",
|
| 53 |
+
"endonym": "नेपाली",
|
| 54 |
+
"iso_639_3": "nep",
|
| 55 |
+
"priority": 2,
|
| 56 |
+
"counts_toward_nepal_language_target": true,
|
| 57 |
+
"first_target": false,
|
| 58 |
+
"scripts": [
|
| 59 |
+
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Established base script and regression-protection target."}
|
| 60 |
+
],
|
| 61 |
+
"font_candidates": [
|
| 62 |
+
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
|
| 63 |
+
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
|
| 64 |
+
],
|
| 65 |
+
"paddle_dictionary": "data/dictionaries/nepali-devanagari-v1.txt",
|
| 66 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
|
| 67 |
+
"corpora": [
|
| 68 |
+
{"name": "Nepali hunspell dictionary", "path": "/Users/cdjk/github/llm/ltk/ne_NP.dic", "host": "cdjk", "type": "lexicon", "status": "exists_local"}
|
| 69 |
+
],
|
| 70 |
+
"validation_sources": [
|
| 71 |
+
{"name": "Gorkhapatra epaper PDFs", "path": "data/gorkhapatra-pack-smoke", "type": "real_document_validation_pack", "status": "pending_manual_label"}
|
| 72 |
+
],
|
| 73 |
+
"gorkhapatra_language_labels": ["नेपाली"],
|
| 74 |
+
"notes": "Must not regress versus stock PaddleOCR/Tesseract/Surya baselines."
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"id": "tamang",
|
| 78 |
+
"name": "Tamang",
|
| 79 |
+
"endonym": "तामाङ",
|
| 80 |
+
"iso_639_3": "taj",
|
| 81 |
+
"priority": 3,
|
| 82 |
+
"counts_toward_nepal_language_target": true,
|
| 83 |
+
"first_target": false,
|
| 84 |
+
"scripts": [
|
| 85 |
+
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari resources; dialect coverage must be tracked separately."}
|
| 86 |
+
],
|
| 87 |
+
"font_candidates": [
|
| 88 |
+
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
|
| 89 |
+
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
|
| 90 |
+
],
|
| 91 |
+
"paddle_dictionary": "data/dictionaries/tamang-devanagari-v1.txt",
|
| 92 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
|
| 93 |
+
"corpora": [
|
| 94 |
+
{"name": "LTK Tamang resources", "path": "/Users/cdjk/github/llm/ltk/tamang", "host": "cdjk", "type": "local_verified_directory", "status": "schema_audit_pending"},
|
| 95 |
+
{"name": "Western Tamang dictionary TSV", "path": "/Users/cdjk/github/llm/ltk/tamang/data/western_tamang_dictionary.tsv", "host": "cdjk", "type": "lexicon", "status": "exists_local"},
|
| 96 |
+
{"name": "NepTam80K train/test CSV", "path": "/Users/cdjk/github/llm/ltk/tamang/data", "host": "cdjk", "type": "parallel_or_synthetic_text", "status": "schema_audit_pending"}
|
| 97 |
+
],
|
| 98 |
+
"validation_sources": [
|
| 99 |
+
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_audit_pending"}
|
| 100 |
+
],
|
| 101 |
+
"gorkhapatra_language_labels": ["तामाङ"],
|
| 102 |
+
"notes": "High-priority Devanagari low-resource language with existing local resources."
|
| 103 |
+
},
|
| 104 |
+
{
|
| 105 |
+
"id": "magar",
|
| 106 |
+
"name": "Magar",
|
| 107 |
+
"endonym": "मगर",
|
| 108 |
+
"iso_639_3": "mrd",
|
| 109 |
+
"priority": 4,
|
| 110 |
+
"counts_toward_nepal_language_target": true,
|
| 111 |
+
"first_target": false,
|
| 112 |
+
"scripts": [
|
| 113 |
+
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
|
| 114 |
+
],
|
| 115 |
+
"font_candidates": [
|
| 116 |
+
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
|
| 117 |
+
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
|
| 118 |
+
],
|
| 119 |
+
"paddle_dictionary": "data/dictionaries/magar-devanagari-v1.txt",
|
| 120 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
|
| 121 |
+
"corpora": [
|
| 122 |
+
{"name": "LTK Magar resources", "path": "/Users/cdjk/github/llm/ltk/magar", "host": "cdjk", "type": "local_verified_directory", "status": "schema_audit_pending"},
|
| 123 |
+
{"name": "Western Magar dictionary TSV", "path": "/Users/cdjk/github/llm/ltk/magar/data/western_magar_dictionary.tsv", "host": "cdjk", "type": "lexicon", "status": "exists_local"}
|
| 124 |
+
],
|
| 125 |
+
"validation_sources": [
|
| 126 |
+
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
|
| 127 |
+
],
|
| 128 |
+
"gorkhapatra_language_labels": ["मगर"],
|
| 129 |
+
"notes": "Seen in Naya Nepal audit labels; needs manual reference pack."
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"id": "wambule",
|
| 133 |
+
"name": "Wambule",
|
| 134 |
+
"endonym": "वाम्बुले",
|
| 135 |
+
"iso_639_3": "wme",
|
| 136 |
+
"priority": 5,
|
| 137 |
+
"counts_toward_nepal_language_target": true,
|
| 138 |
+
"first_target": false,
|
| 139 |
+
"scripts": [
|
| 140 |
+
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
|
| 141 |
+
],
|
| 142 |
+
"font_candidates": [
|
| 143 |
+
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
|
| 144 |
+
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
|
| 145 |
+
],
|
| 146 |
+
"paddle_dictionary": "data/dictionaries/wambule-devanagari-v1.txt",
|
| 147 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
|
| 148 |
+
"corpora": [
|
| 149 |
+
{"name": "Gorkhapatra/Naya Nepal Wambule articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
|
| 150 |
+
],
|
| 151 |
+
"validation_sources": [
|
| 152 |
+
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
|
| 153 |
+
],
|
| 154 |
+
"gorkhapatra_language_labels": ["वाम्बुले"],
|
| 155 |
+
"notes": "Real-source first; no local training corpus audited yet."
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"id": "rana-tharu",
|
| 159 |
+
"name": "Rana Tharu",
|
| 160 |
+
"endonym": "राना थारु",
|
| 161 |
+
"iso_639_3": "thr",
|
| 162 |
+
"priority": 6,
|
| 163 |
+
"counts_toward_nepal_language_target": true,
|
| 164 |
+
"first_target": false,
|
| 165 |
+
"scripts": [
|
| 166 |
+
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
|
| 167 |
+
],
|
| 168 |
+
"font_candidates": [
|
| 169 |
+
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
|
| 170 |
+
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
|
| 171 |
+
],
|
| 172 |
+
"paddle_dictionary": "data/dictionaries/rana-tharu-devanagari-v1.txt",
|
| 173 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
|
| 174 |
+
"corpora": [
|
| 175 |
+
{"name": "Gorkhapatra/Naya Nepal Rana Tharu articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
|
| 176 |
+
],
|
| 177 |
+
"validation_sources": [
|
| 178 |
+
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
|
| 179 |
+
],
|
| 180 |
+
"gorkhapatra_language_labels": ["राना थारु", "थारु"],
|
| 181 |
+
"notes": "Track Rana Tharu separately from broader Tharu labels when references are created."
|
| 182 |
+
},
|
| 183 |
+
{
|
| 184 |
+
"id": "bantawa",
|
| 185 |
+
"name": "Bantawa / Rai",
|
| 186 |
+
"endonym": "बान्तावा",
|
| 187 |
+
"iso_639_3": "bap",
|
| 188 |
+
"priority": 7,
|
| 189 |
+
"counts_toward_nepal_language_target": true,
|
| 190 |
+
"first_target": false,
|
| 191 |
+
"scripts": [
|
| 192 |
+
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."},
|
| 193 |
+
{"script": "Kirat Rai", "primary": false, "status": "requires_unicode_font_audit", "unicode_ranges": ["U+16D40..U+16D7F"], "notes": "Potential future script target; do not train or claim until font/runtime support is verified."}
|
| 194 |
+
],
|
| 195 |
+
"font_candidates": [
|
| 196 |
+
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
|
| 197 |
+
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
|
| 198 |
+
],
|
| 199 |
+
"paddle_dictionary": "data/dictionaries/bantawa-devanagari-v1.txt",
|
| 200 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
|
| 201 |
+
"corpora": [
|
| 202 |
+
{"name": "Gorkhapatra/Naya Nepal Bantawa articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
|
| 203 |
+
],
|
| 204 |
+
"validation_sources": [
|
| 205 |
+
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
|
| 206 |
+
],
|
| 207 |
+
"gorkhapatra_language_labels": ["बान्तावा"],
|
| 208 |
+
"notes": "Rai-language coverage must stay language-specific; do not collapse all Rai languages into one metric."
|
| 209 |
+
},
|
| 210 |
+
{
|
| 211 |
+
"id": "kulung",
|
| 212 |
+
"name": "Kulung",
|
| 213 |
+
"endonym": "कुलुङ",
|
| 214 |
+
"iso_639_3": "kle",
|
| 215 |
+
"priority": 8,
|
| 216 |
+
"counts_toward_nepal_language_target": true,
|
| 217 |
+
"first_target": false,
|
| 218 |
+
"scripts": [
|
| 219 |
+
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
|
| 220 |
+
],
|
| 221 |
+
"font_candidates": [
|
| 222 |
+
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
|
| 223 |
+
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
|
| 224 |
+
],
|
| 225 |
+
"paddle_dictionary": "data/dictionaries/kulung-devanagari-v1.txt",
|
| 226 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
|
| 227 |
+
"corpora": [
|
| 228 |
+
{"name": "Gorkhapatra/Naya Nepal Kulung articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
|
| 229 |
+
],
|
| 230 |
+
"validation_sources": [
|
| 231 |
+
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
|
| 232 |
+
],
|
| 233 |
+
"gorkhapatra_language_labels": ["कुलुङ"],
|
| 234 |
+
"notes": "Real-source first; needs corpus expansion."
|
| 235 |
+
},
|
| 236 |
+
{
|
| 237 |
+
"id": "yakkha",
|
| 238 |
+
"name": "Yakkha",
|
| 239 |
+
"endonym": "याक्खा",
|
| 240 |
+
"iso_639_3": "ybh",
|
| 241 |
+
"priority": 9,
|
| 242 |
+
"counts_toward_nepal_language_target": true,
|
| 243 |
+
"first_target": false,
|
| 244 |
+
"scripts": [
|
| 245 |
+
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
|
| 246 |
+
],
|
| 247 |
+
"font_candidates": [
|
| 248 |
+
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
|
| 249 |
+
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
|
| 250 |
+
],
|
| 251 |
+
"paddle_dictionary": "data/dictionaries/yakkha-devanagari-v1.txt",
|
| 252 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
|
| 253 |
+
"corpora": [
|
| 254 |
+
{"name": "Gorkhapatra/Naya Nepal Yakkha articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
|
| 255 |
+
],
|
| 256 |
+
"validation_sources": [
|
| 257 |
+
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
|
| 258 |
+
],
|
| 259 |
+
"gorkhapatra_language_labels": ["याक्खा"],
|
| 260 |
+
"notes": "Real-source first; needs corpus expansion."
|
| 261 |
+
},
|
| 262 |
+
{
|
| 263 |
+
"id": "bhojpuri",
|
| 264 |
+
"name": "Bhojpuri",
|
| 265 |
+
"endonym": "भोजपुरी",
|
| 266 |
+
"iso_639_3": "bho",
|
| 267 |
+
"priority": 10,
|
| 268 |
+
"counts_toward_nepal_language_target": true,
|
| 269 |
+
"first_target": false,
|
| 270 |
+
"scripts": [
|
| 271 |
+
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
|
| 272 |
+
],
|
| 273 |
+
"font_candidates": [
|
| 274 |
+
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
|
| 275 |
+
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
|
| 276 |
+
],
|
| 277 |
+
"paddle_dictionary": "data/dictionaries/bhojpuri-devanagari-v1.txt",
|
| 278 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
|
| 279 |
+
"corpora": [
|
| 280 |
+
{"name": "Gorkhapatra/Naya Nepal Bhojpuri articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
|
| 281 |
+
],
|
| 282 |
+
"validation_sources": [
|
| 283 |
+
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
|
| 284 |
+
],
|
| 285 |
+
"gorkhapatra_language_labels": ["भोजपुरी"],
|
| 286 |
+
"notes": "Large-language baseline for Devanagari multilingual transfer."
|
| 287 |
+
},
|
| 288 |
+
{
|
| 289 |
+
"id": "santhali",
|
| 290 |
+
"name": "Santhali",
|
| 291 |
+
"endonym": "सन्थाली",
|
| 292 |
+
"iso_639_3": "sat",
|
| 293 |
+
"priority": 11,
|
| 294 |
+
"counts_toward_nepal_language_target": true,
|
| 295 |
+
"first_target": false,
|
| 296 |
+
"scripts": [
|
| 297 |
+
{"script": "Ol Chiki", "primary": true, "status": "encoded", "unicode_ranges": ["U+1C50..U+1C7F"], "notes": "Own-script target after Limbu; needs font/render audit."},
|
| 298 |
+
{"script": "Devanagari", "primary": false, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "May appear in Devanagari-transcribed sources."}
|
| 299 |
+
],
|
| 300 |
+
"font_candidates": [
|
| 301 |
+
{"name": "Noto Sans Ol Chiki", "path": "data/fonts/NotoSansOlChiki-Regular.ttf", "status": "pending_render_audit"},
|
| 302 |
+
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
|
| 303 |
+
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
|
| 304 |
+
],
|
| 305 |
+
"paddle_dictionary": "data/dictionaries/santhali-ol-chiki-v1.txt",
|
| 306 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_ol_chiki_letters"},
|
| 307 |
+
"corpora": [
|
| 308 |
+
{"name": "Gorkhapatra/Naya Nepal Santhali articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
|
| 309 |
+
],
|
| 310 |
+
"validation_sources": [
|
| 311 |
+
{"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
|
| 312 |
+
],
|
| 313 |
+
"gorkhapatra_language_labels": ["सन्थाली"],
|
| 314 |
+
"notes": "Second own-script candidate after Limbu if real rendered sources and fonts are available."
|
| 315 |
+
},
|
| 316 |
+
{
|
| 317 |
+
"id": "maithili",
|
| 318 |
+
"name": "Maithili",
|
| 319 |
+
"endonym": "मैथिली",
|
| 320 |
+
"iso_639_3": "mai",
|
| 321 |
+
"priority": 12,
|
| 322 |
+
"counts_toward_nepal_language_target": true,
|
| 323 |
+
"first_target": false,
|
| 324 |
+
"scripts": [
|
| 325 |
+
{"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."},
|
| 326 |
+
{"script": "Tirhuta", "primary": false, "status": "future_audit", "unicode_ranges": ["U+11480..U+114DF"], "notes": "Own-script support is a future target and not part of the v1 claim without real validation."}
|
| 327 |
+
],
|
| 328 |
+
"font_candidates": [
|
| 329 |
+
{"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
|
| 330 |
+
{"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
|
| 331 |
+
],
|
| 332 |
+
"paddle_dictionary": "data/dictionaries/maithili-devanagari-v1.txt",
|
| 333 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
|
| 334 |
+
"corpora": [
|
| 335 |
+
{"name": "Maithili corpus discovery", "path": "TBD", "type": "external_corpus_pending", "status": "not_audited"}
|
| 336 |
+
],
|
| 337 |
+
"validation_sources": [
|
| 338 |
+
{"name": "Gorkhapatra/Naya Nepal or public documents", "path": "TBD", "type": "real_document_discovery", "status": "not_audited"}
|
| 339 |
+
],
|
| 340 |
+
"gorkhapatra_language_labels": ["मैथिली"],
|
| 341 |
+
"notes": "Included because it is a major Nepal language; needs source discovery before training."
|
| 342 |
+
},
|
| 343 |
+
{
|
| 344 |
+
"id": "english",
|
| 345 |
+
"name": "English",
|
| 346 |
+
"endonym": "English",
|
| 347 |
+
"iso_639_3": "eng",
|
| 348 |
+
"priority": 99,
|
| 349 |
+
"counts_toward_nepal_language_target": false,
|
| 350 |
+
"first_target": false,
|
| 351 |
+
"scripts": [
|
| 352 |
+
{"script": "Latin", "primary": true, "status": "encoded", "unicode_ranges": ["U+0000..U+007F", "U+0080..U+00FF"], "notes": "Regression-protection target only."}
|
| 353 |
+
],
|
| 354 |
+
"font_candidates": [
|
| 355 |
+
{"name": "Arial Unicode MS", "path": "/Library/Fonts/Arial Unicode.ttf", "status": "pending_render_audit"},
|
| 356 |
+
{"name": "Helvetica", "path": "/System/Library/Fonts/Helvetica.ttc", "status": "pending_render_audit"}
|
| 357 |
+
],
|
| 358 |
+
"paddle_dictionary": "data/dictionaries/english-latin-v1.txt",
|
| 359 |
+
"normalization": {"unicode_form": "NFC", "policy": "preserve_ascii_and_common_latin"},
|
| 360 |
+
"corpora": [
|
| 361 |
+
{"name": "NVIDIA OCR Synthetic Multilingual English subset", "path": "huggingface:nvidia/OCR-Synthetic-Multilingual-v1", "type": "synthetic_line_crops", "status": "schema_audit_required_before_use"}
|
| 362 |
+
],
|
| 363 |
+
"validation_sources": [
|
| 364 |
+
{"name": "Stock English OCR eval subset", "path": "TBD", "type": "regression_eval", "status": "not_audited"}
|
| 365 |
+
],
|
| 366 |
+
"gorkhapatra_language_labels": [],
|
| 367 |
+
"notes": "Included to protect English accuracy in mixed documents; not counted as a Nepal-language target."
|
| 368 |
+
}
|
| 369 |
+
]
|
| 370 |
+
}
|
pyproject.toml
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=69", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "ocr-tech"
|
| 7 |
+
version = "0.1.0"
|
| 8 |
+
description = "Hybrid Nepali + English OCR and document-structure extraction toolkit."
|
| 9 |
+
readme = "README.md"
|
| 10 |
+
requires-python = ">=3.11"
|
| 11 |
+
license = { text = "Apache-2.0" }
|
| 12 |
+
authors = [
|
| 13 |
+
{ name = "Ampixa" }
|
| 14 |
+
]
|
| 15 |
+
dependencies = []
|
| 16 |
+
|
| 17 |
+
[project.optional-dependencies]
|
| 18 |
+
datasets = [
|
| 19 |
+
"datasets>=3.0.0",
|
| 20 |
+
"h5py>=3.11.0",
|
| 21 |
+
"huggingface-hub>=0.25.0",
|
| 22 |
+
"pillow>=10.0.0"
|
| 23 |
+
]
|
| 24 |
+
eval = [
|
| 25 |
+
"pillow>=10.0.0"
|
| 26 |
+
]
|
| 27 |
+
engines = [
|
| 28 |
+
"paddleocr>=3.0.0",
|
| 29 |
+
"pillow>=10.0.0"
|
| 30 |
+
]
|
| 31 |
+
surya = [
|
| 32 |
+
"surya-ocr>=0.20.0",
|
| 33 |
+
"pillow>=10.0.0"
|
| 34 |
+
]
|
| 35 |
+
hf-recognizer = [
|
| 36 |
+
"accelerate>=1.1.0",
|
| 37 |
+
"protobuf>=5.28.0",
|
| 38 |
+
"sentencepiece>=0.2.0",
|
| 39 |
+
"transformers>=4.57.0",
|
| 40 |
+
"torch>=2.5.0",
|
| 41 |
+
"pillow>=10.0.0"
|
| 42 |
+
]
|
| 43 |
+
vlm = [
|
| 44 |
+
"transformers>=4.50.0",
|
| 45 |
+
"torch>=2.5.0",
|
| 46 |
+
"pillow>=10.0.0"
|
| 47 |
+
]
|
| 48 |
+
dev = [
|
| 49 |
+
"pytest>=8.0.0"
|
| 50 |
+
]
|
| 51 |
+
|
| 52 |
+
[project.scripts]
|
| 53 |
+
ocrtech = "ocrtech.cli:main"
|
| 54 |
+
|
| 55 |
+
[tool.setuptools.packages.find]
|
| 56 |
+
where = ["src"]
|
| 57 |
+
|
| 58 |
+
[tool.pytest.ini_options]
|
| 59 |
+
pythonpath = ["src"]
|
| 60 |
+
testpaths = ["tests"]
|
| 61 |
+
addopts = "-q"
|
runs/corrector-cfilt/README.md
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Corrector Training Recipe
|
| 2 |
+
|
| 3 |
+
Pairs: `runs/corrector-cfilt/correction_pairs.jsonl`
|
| 4 |
+
|
| 5 |
+
Use these noisy-clean pairs with a compact text-to-text model. Keep Nepali and English validation sets separate.
|
runs/corrector-cfilt/correction_pairs.jsonl
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"noisy_text": "मधुमेहले विश्वमा एउटा महामारीको रूप लिइसकेको छ तथा यसका धेरै प्रकारका सामाजिक अनि आर्थिक दुष्परिणाम पनि अधि आइरहेका छन्।", "clean_text": "मधुमेहले विश्वमा एउटा महामारीको रूप लिइसकेको छ तथा यसका धेरै प्रकारका सामाजिक अनि आर्थिक दुष्परिणाम पनि अघि आइरहेका छन्।", "font": "Sura-Regular", "dataset": "cfilt/RoundTripOCR-nepali", "split": "train"}
|
| 2 |
+
{"noisy_text": "\"यस वठीले मुद्ु अनि ढिमागलाई शक्ति ढिने खाले, शुक्राणु अल्पता तथा शरीरमा स्फूर्ति बढाउने अनि वाजीकारक छ।\"", "clean_text": "\"यस वटीले मुटु अनि दिमागलाई शक्ति दिने खाले, शुक्राणु अल्पता तथा शरीरमा स्फूर्ति बढाउने अनि वाजीकारक छ।\"", "font": "Arya-Regular", "dataset": "cfilt/RoundTripOCR-nepali", "split": "train"}
|
| 3 |
+
{"noisy_text": "गाइडले हामीलाई सिंहका खुद्दाको चिह देखाए।", "clean_text": "गाइडले हामीलाई सिंहका खुट्टाको चिह्न देखाए।", "font": "Sanskrit2003", "dataset": "cfilt/RoundTripOCR-nepali", "split": "train"}
|
| 4 |
+
{"noisy_text": "\"भोजन, स्वास्थ्य, सौन्दर्य अनि धार्मिक सबै मामित्तामा यसको जवाफ छेैन|\"", "clean_text": "\"भोजन, स्वास्थ्य, सौन्दर्य अनि धार्मिक सबै मामिलामा यसको जवाफ छैन।\"", "font": "Asar-Regular", "dataset": "cfilt/RoundTripOCR-nepali", "split": "train"}
|
| 5 |
+
{"noisy_text": "\"डायबिटिजले तपाईँका आँखाका रेटिनालाई क्षति पुयाउन सक््छ, जसबाट ओपन हुनसक्छा'", "clean_text": "\"डायबिटिजले तपाईँका आँखाका रेटिनालाई क्षति पुर्याउन सक्छ, जसबाट अन्धोपन हुनसक्छ।\"", "font": "Halant-Regular", "dataset": "cfilt/RoundTripOCR-nepali", "split": "train"}
|
runs/corrector-cfilt/corrector_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"task": "ocr_text_correction",
|
| 3 |
+
"pairs": "runs/corrector-cfilt/correction_pairs.jsonl",
|
| 4 |
+
"recommended_model_family": "compact seq2seq or encoder-decoder text corrector",
|
| 5 |
+
"metrics": [
|
| 6 |
+
"cer",
|
| 7 |
+
"wer",
|
| 8 |
+
"exact_line_accuracy"
|
| 9 |
+
]
|
| 10 |
+
}
|
runs/corrector-mixed-v1-rebalanced/tesseract_pairs.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
runs/hf-corrector-mixed-v1-rebalanced/README.md
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Hugging Face Text Corrector Training Recipe
|
| 2 |
+
|
| 3 |
+
Base model: `google/byt5-small`
|
| 4 |
+
Pairs source: `runs/hf-corrector-mixed-v1-rebalanced/correction_pairs.jsonl` (1700 pairs)
|
| 5 |
+
Train pairs: `runs/hf-corrector-mixed-v1-rebalanced/train_pairs.jsonl` (1530 pairs)
|
| 6 |
+
Eval pairs: `runs/hf-corrector-mixed-v1-rebalanced/eval_pairs.jsonl` (170 pairs)
|
| 7 |
+
|
| 8 |
+
Install the backend dependencies first:
|
| 9 |
+
|
| 10 |
+
```bash
|
| 11 |
+
python -m pip install -e '.[hf-recognizer]'
|
| 12 |
+
```
|
| 13 |
+
|
| 14 |
+
Run in-process:
|
| 15 |
+
|
| 16 |
+
```bash
|
| 17 |
+
ocrtech train-corrector --backend hf-seq2seq --pairs-source runs/hf-corrector-mixed-v1-rebalanced/correction_pairs.jsonl --out runs/hf-corrector-mixed-v1-rebalanced --run
|
| 18 |
+
```
|
runs/hf-corrector-mixed-v1-rebalanced/correction_pairs.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
runs/hf-corrector-mixed-v1-rebalanced/corrector_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"task": "ocr_text_correction",
|
| 3 |
+
"pairs": "runs/hf-corrector-mixed-v1-rebalanced/correction_pairs.jsonl",
|
| 4 |
+
"recommended_model_family": "compact seq2seq or encoder-decoder text corrector",
|
| 5 |
+
"metrics": [
|
| 6 |
+
"cer",
|
| 7 |
+
"wer",
|
| 8 |
+
"exact_line_accuracy"
|
| 9 |
+
]
|
| 10 |
+
}
|
runs/hf-corrector-mixed-v1-rebalanced/eval_pairs.jsonl
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"noisy_text": "onto a gray plaid", "clean_text": "onto a gray plaid"}
|
| 2 |
+
{"noisy_text": "ae सादर अनुरोध eae, मुख्य aay बन", "clean_text": "सादर अनुरोध गर्दछु, मुख्य सचिव"}
|
| 3 |
+
{"noisy_text": ": क्षेत्रीय सहकार्य समूमेलनमा भांग after", "clean_text": "क्षेत्रीय सहकार्य सम्मेलनमा भाग लिनुहुन"}
|
| 4 |
+
{"noisy_text": "_ aturn for the worse with the", "clean_text": "- a turn for the worse with the"}
|
| 5 |
+
{"noisy_text": "——e छु od", "clean_text": "३ डिडि ढि"}
|
| 6 |
+
{"noisy_text": "has its", "clean_text": "has its"}
|
| 7 |
+
{"noisy_text": "known mutations and genetic", "clean_text": "known mutations and genetic"}
|
| 8 |
+
{"noisy_text": "a losing! battic", "clean_text": "fighting a (losing) battle"}
|
| 9 |
+
{"noisy_text": "९९७६ 1810 the wild, the bircle do nut", "clean_text": "had in the wild, the birds do not"}
|
| 10 |
+
{"noisy_text": "PANELS CAN MAKE A HUGE", "clean_text": "panels can make a huge"}
|
| 11 |
+
{"noisy_text": "", "clean_text": "monitoring of solar panel"}
|
| 12 |
+
{"noisy_text": "“सादर अनुरोध tea, प्रशासन safe : .", "clean_text": "सादर अनुरोध गर्दछु, प्रशासन अधिकृत"}
|
| 13 |
+
{"noisy_text": "जी: १०7० EEE Seale 7?\nee5\t1\t1\t1\t2\t2\t514\t131\t119\t113\t85.775848\tjot5\t1\t1\t1\t2\t3\t691\t89\t280\t146\t60.418049\t905\t1\t1\t1\t2\t4\t950\t123\t5\t61\t52.338493\t|5\t1\t1\t1\t2\t5\t987\t55\t195\t155\t64.715851\tBut5\t1\t1\t1\t2\t6\t1225\t86\t239\t89\t30.467482\t520685\t1\t1\t1\t2\t7\t1508\t51\t229\t102\t25.415092\twy5\t1\t1\t1\t2\t8\t1758\t81\t30\t37\t73.042397\t2०25\t1\t1\t1\t2\t9\t1795\t21\t211\t101\t35.075745\t(Wee", "clean_text": "मिति : २०१० वर्षको\nसम्यक गर्ने अन्तिम मिति : २०६४ चैत्र २० गतेसम्म"}
|
| 14 |
+
{"noisy_text": "prices. REAL SEO is your advantage.", "clean_text": "prices. REAL SEO is your advantage,"}
|
| 15 |
+
{"noisy_text": "“mare सहयोग: तथा कृषमता-अभविद्धाकिार्यक्रम", "clean_text": "प्राविधिक सहयोग तथा क्षमता अभिवृद्धि कार्यक्रम"}
|
| 16 |
+
{"noisy_text": "ल्कस्त्ग पाको IBS,", "clean_text": "कस्तो मैं दिऊ"}
|
| 17 |
+
{"noisy_text": "t\nh\n=", "clean_text": "themed"}
|
| 18 |
+
{"noisy_text": "रसूतुत वषियंमा आवश्यक PATI TTT लाँगपिठाइएको '", "clean_text": "प्रस्तुत विषयमा आवश्यक कार्यान्वयनका लागि पठाइएको छ"}
|
| 19 |
+
{"noisy_text": "odor of mold. Basements", "clean_text": "odor of mold. Basements"}
|
| 20 |
+
{"noisy_text": "cpp वबिरंण तथा बाँडफाँड समूबन्धमा ॥", "clean_text": "कार्य विवरण तथा बाँडफाँड सम्बन्धमा"}
|
| 21 |
+
{"noisy_text": "she learnt she had indigenous ancestors", "clean_text": "she learnt she had indigenous ancestors"}
|
| 22 |
+
{"noisy_text": "t\nh\ni\nr\ni\na\nत |\n|\nf]\na\n३\nUl}", "clean_text": "their manufacture."}
|
| 23 |
+
{"noisy_text": "Dm wore oS about, that hover!", "clean_text": "I’m worried about, that haven’t"}
|
| 24 |
+
{"noisy_text": "", "clean_text": "चलाउने नसकिएकाले"}
|
| 25 |
+
{"noisy_text": "€3326.02", "clean_text": "€3326.02"}
|
| 26 |
+
{"noisy_text": "2551S oe", "clean_text": "£5513.53"}
|
| 27 |
+
{"noisy_text": "€0101६61£:20100£ १ © 9%", "clean_text": "content to put down their roots"}
|
| 28 |
+
{"noisy_text": "handsome red fox", "clean_text": "handsome red fox"}
|
| 29 |
+
{"noisy_text": "oe मति22071. फागुन 21 गते 1414", "clean_text": "मितिः 2071 फागुन 21 गते"}
|
| 30 |
+
{"noisy_text": "टर ६८", "clean_text": "बन्धछे"}
|
| 31 |
+
{"noisy_text": "fe ir", "clean_text": "पुजि हो"}
|
| 32 |
+
{"noisy_text": "oneself. Usually in a", "clean_text": "oneself. Usually in a"}
|
| 33 |
+
{"noisy_text": "located about 244.6 million light-years away, This galaxy 0858 bright AGN, resulting .-......... 0.1", "clean_text": "located about 244.6 million light-years away. This galaxy has a bright AGN, resulting 6.1"}
|
| 34 |
+
{"noisy_text": "3790.57", "clean_text": "3790.57"}
|
| 35 |
+
{"noisy_text": "fantasy football", "clean_text": "fantasy football"}
|
| 36 |
+
{"noisy_text": "Yili wore ee Tun 000.\nora Bpeteta", "clean_text": "विद्यार्थी हिस्से समयमे"}
|
| 37 |
+
{"noisy_text": "from Google Ads to save time and get", "clean_text": "from Google Ads to save time and get"}
|
| 38 |
+
{"noisy_text": "> aro पपारटैलगा , Fa OCF २\npe 27?", "clean_text": "प्रश्नो परिक्षण, नित्य र"}
|
| 39 |
+
{"noisy_text": "ude", "clean_text": "यस"}
|
| 40 |
+
{"noisy_text": "~ yout nase There are.\n‘nd; Feviews yet,\n४०६ Ocak your manivans weed products onkine and provide proof that yeulrs", "clean_text": "- your needs. There are no reviews yet. Order your marijuana & weed products online and provide proof that you’re"}
|
| 41 |
+
{"noisy_text": "“ सादर अनुरोध गर्दछु, मनूत्रालय प्रवकृता...", "clean_text": "सादर अनुरोध गर्दछु, मन्त्रालय प्रवक्ता"}
|
| 42 |
+
{"noisy_text": "The hope was that", "clean_text": "The hope was that"}
|
| 43 |
+
{"noisy_text": "“ सादर अनुरोध गर्दछु, मनूत्रालय प्रवकृता...", "clean_text": "सादर अनुरोध गर्दछु, मन्त्रालय प्रवक्ता"}
|
| 44 |
+
{"noisy_text": "party on the issue, it is not a surprise", "clean_text": "party on the issue, it is not a surprise"}
|
| 45 |
+
{"noisy_text": "corresponds directly to the amount of dimming. Although scientists have long. . ---- 10", "clean_text": "corresponds directly to the amount of dimming. Although scientists have long 10"}
|
| 46 |
+
{"noisy_text": "", "clean_text": "damp and rainy, but oh my, the little gardens are"}
|
| 47 |
+
{"noisy_text": "Ol wtings.", "clean_text": "offerings."}
|
| 48 |
+
{"noisy_text": "is needle felted", "clean_text": "is needle felted"}
|
| 49 |
+
{"noisy_text": "", "clean_text": "चलाउने नसकिएकाले"}
|
| 50 |
+
{"noisy_text": "needs. if you're", "clean_text": "needs. if you’re"}
|
| 51 |
+
{"noisy_text": "did not respond. Please", "clean_text": "did not respond. Please"}
|
| 52 |
+
{"noisy_text": "upcycled off-white", "clean_text": "upcycled off-white"}
|
| 53 |
+
{"noisy_text": "शूरी पूरेशासने अधरकित; TATA तथा जनसंख्या मनूत्रालय", "clean_text": "श्री प्रशासन अधिकृत, स्वास्थ्य तथा जनसंख्या मन्त्रालय"}
|
| 54 |
+
{"noisy_text": "of three or four. We", "clean_text": "of three or four. We"}
|
| 55 |
+
{"noisy_text": "चा0कवितंशमा कस्ता मानिसको ryt a", "clean_text": "ग कवितांशमा कस्ता मानिसको शिर उचो हुन्छ भनि-"}
|
| 56 |
+
{"noisy_text": "हेट दर cae) त्यो oer oa) TOY ढ्किक्कि 2", "clean_text": "हेदै रहल लागदो त्यो नरम कमरे भातो भै दिऊ कि?"}
|
| 57 |
+
{"noisy_text": "photos, |<) an 80 तालले ९९,", "clean_text": "photos, Linda, and so glad to"}
|
| 58 |
+
{"noisy_text": "8344", "clean_text": "8344"}
|
| 59 |
+
{"noisy_text": "obvious reasons.", "clean_text": "obvious reasons."}
|
| 60 |
+
{"noisy_text": "‘oe feeders ondhave. -", "clean_text": "the feeders and have been"}
|
| 61 |
+
{"noisy_text": "|", "clean_text": "nature's goodness. The"}
|
| 62 |
+
{"noisy_text": "रस्तुत बंषियमा आवशूयक कार्यानूवयनका लाग पिठाइएको :", "clean_text": "प्रस्तुत विषयमा आवश्यक कार्यान्वयनका लागि पठाइएको छ"}
|
| 63 |
+
{"noisy_text": "not a type of belief", "clean_text": "not a type of belief"}
|
| 64 |
+
{"noisy_text": "we ; : “पत्र संख्या: 6315/075/076 20]", "clean_text": "पत्र संख्याः 5315/०75/०76"}
|
| 65 |
+
{"noisy_text": "checked at least two to three", "clean_text": "checked at least two to three"}
|
| 66 |
+
{"noisy_text": "oe कार्य afar तथा बाँडफीँड संमूबन्धमा: 1.7.", "clean_text": "कार्य विवरण तथा बाँडफाँड सम्बन्धमा"}
|
| 67 |
+
{"noisy_text": "or huzzah. Continue", "clean_text": "or huzzah. Continue"}
|
| 68 |
+
{"noisy_text": "Day 10:11 112. andI’ve", "clean_text": "Day around her and I’ve"}
|
| 69 |
+
{"noisy_text": "[नी SP", "clean_text": "नेपाली हस्तलेखन"}
|
| 70 |
+
{"noisy_text": "“सादर अनुरोध गर्दछु, महोनरिदिशक ..", "clean_text": "सादर अनुरोध गर्दछु, महानिर्देशक"}
|
| 71 |
+
{"noisy_text": "“नेपाल सरकारको afters ब्मोजमि तोकणिको ata", "clean_text": "नेपाल सरकारको निर्णय बमोजिम तोकिएको मिति"}
|
| 72 |
+
{"noisy_text": "21॥3न्ट्", "clean_text": "गारिन्दे"}
|
| 73 |
+
{"noisy_text": "[यवस्था, सहकारी तथा abet नविरणमन्त्रालय ALS, §", "clean_text": "भूमि व्यवस्था, सहकारी तथा गरिबी निवारण मन्त्रालय सुर्खेत, सुर्खेत"}
|
| 74 |
+
{"noisy_text": "11 est", "clean_text": "third of"}
|
| 75 |
+
{"noisy_text": "In", "clean_text": "\"In"}
|
| 76 |
+
{"noisy_text": "0714\n2101", "clean_text": "गर्न"}
|
| 77 |
+
{"noisy_text": "गेम ।वैनाग्रव HIEMIGE", "clean_text": "तेज विनायक माध्यमिक विद्यालय"}
|
| 78 |
+
{"noisy_text": "SB 11000000 DRODUCTION “10-1८01100110101010007 117 000? RODUMES 0401", "clean_text": "5. how the production and computation of hash products will"}
|
| 79 |
+
{"noisy_text": "( aft 2071 कातूतकि 19 गते...", "clean_text": "मितिः 2071 कात्तिक 19 गते"}
|
| 80 |
+
{"noisy_text": "you know. Microsoft Advertising has", "clean_text": "you know, Microsoft Advertising has"}
|
| 81 |
+
{"noisy_text": "ce tpt Feat TTS जनकपुर, घनुषा ७...", "clean_text": "प्रहरी प्रधान कार्यालय जनकपुर, धनुषा"}
|
| 82 |
+
{"noisy_text": "[. कृषेतूरीय सहकार्य समूमेलनमा भाग: लनिहन he", "clean_text": "क्षेत्रीय सहकार्य सम्मेलनमा भाग लिनुहुन"}
|
| 83 |
+
{"noisy_text": "“:. कुर्य afer तथा बाँडफाँड Hea 0", "clean_text": "कार्य विवरण तथा बाँडफाँड सम्बन्धमा"}
|
| 84 |
+
{"noisy_text": "Watching through", "clean_text": "Watching through"}
|
| 85 |
+
{"noisy_text": "र\nr\n०\nm", "clean_text": "from"}
|
| 86 |
+
{"noisy_text": "Na, ae ‘_\n(7/॥ >! ६", "clean_text": "लागी ६"}
|
| 87 |
+
{"noisy_text": "रस्तुत वषियमा आँवशूयक कार्यान्वयनका लाग पिठाइएकी :", "clean_text": "प्रस्तुत विषयमा आवश्यक कार्यान्वयनका लागि पठाइएको छ"}
|
| 88 |
+
{"noisy_text": "it’s important to", "clean_text": "it’s important to"}
|
| 89 |
+
{"noisy_text": "eo fan et न्युन तै ढिक र", "clean_text": "सबैले शिक्षा सजाउने सुत्र में दिऊं?"}
|
| 90 |
+
{"noisy_text": "", "clean_text": "you relax tend to be"}
|
| 91 |
+
{"noisy_text": "showers make bathrooms one of the most likely places for", "clean_text": "showers make bathrooms one of the most likely places for"}
|
| 92 |
+
{"noisy_text": "carbon", "clean_text": "carbon"}
|
| 93 |
+
{"noisy_text": "| SCHELL गरौ Hal THe", "clean_text": "होस अध्ययन गर्ने बानीले हामीलाई"}
|
| 94 |
+
{"noisy_text": "think ves al! feelso good jv:a!", "clean_text": "think we all feel so good just"}
|
| 95 |
+
{"noisy_text": "अफपसामबी Ace", "clean_text": "भ्रमणसम्बन्धी सूचना"}
|
| 96 |
+
{"noisy_text": "|\n॥\n\\\nb\ny\nt\nE\nu\nSs\n€\n0\nf\nt\nh\ne\nWwW\n0\nit\nd", "clean_text": "caused by the use of the word."}
|
| 97 |
+
{"noisy_text": "। 1111 0800 901", "clean_text": "your mom's backyard in your"}
|
| 98 |
+
{"noisy_text": "| नेपाल सरकारको नरिणय बमोजमि तोकषिको मति", "clean_text": "नेपाल सरकारको निर्णय बमोजिम तोकिएको मिति"}
|
| 99 |
+
{"noisy_text": "re wy ries\nNidk!s!", "clean_text": "काठमाडौँ"}
|
| 100 |
+
{"noisy_text": "TCG तथा जनसंख्या मन्त्रालय काठमाडौँ महानगरपालति", "clean_text": "स्वास्थ्य तथा जनसंख्या मन्त्रालय काठमाडौँ महानगरपालिका"}
|
| 101 |
+
{"noisy_text": "", "clean_text": "their powersso that they may"}
|
| 102 |
+
{"noisy_text": "“mare सहयोग: तथा कृषमता-अभविद्धाकिार्यक्रम", "clean_text": "प्राविधिक सहयोग तथा क्षमता अभिवृद्धि कार्यक्रम"}
|
| 103 |
+
{"noisy_text": "You will", "clean_text": "You will"}
|
| 104 |
+
{"noisy_text": "watching for his", "clean_text": "watching for his"}
|
| 105 |
+
{"noisy_text": "verifies you are of legal age to", "clean_text": "verifies you are of legal age to"}
|
| 106 |
+
{"noisy_text": "म तिम्रो erat) aa झै किक्न।", "clean_text": "म तिम्रो लौ कस्तो मैले।"}
|
| 107 |
+
{"noisy_text": "He's a reigning", "clean_text": "He's a reigning"}
|
| 108 |
+
{"noisy_text": "टर ६८", "clean_text": "बन्धछे"}
|
| 109 |
+
{"noisy_text": "वि", "clean_text": "and get rid of"}
|
| 110 |
+
{"noisy_text": "“जक", "clean_text": "ङ"}
|
| 111 |
+
{"noisy_text": "fry = the Covic. 1३ took awhii«", "clean_text": "from the Covid. It took a while"}
|
| 112 |
+
{"noisy_text": "at Northeast Advertising Solutions is to get your business in", "clean_text": "at Northeast Advertising Solutions is to get your business in"}
|
| 113 |
+
{"noisy_text": "i ee ee ee Oe ee WEF", "clean_text": "...यसले अनुसन्धान अधिकृतले मुद्दाको खोजतलास) कार्यको २०६० को\nवहाँ सहिदिको प्रतिवेदन ?"}
|
| 114 |
+
{"noisy_text": "| COB ICR गनर PIE ET", "clean_text": "तरीकाले उपभोग गर्ने सिफारिस । हानि"}
|
| 115 |
+
{"noisy_text": "“ सादर अनुरोध गर्दछु, मनूत्रा���य प्रवकृता...", "clean_text": "सादर अनुरोध गर्दछु, मन्त्रालय प्रवक्ता"}
|
| 116 |
+
{"noisy_text": "grape is notorious for its genetic", "clean_text": "grape is notorious for its genetic"}
|
| 117 |
+
{"noisy_text": "organic Google Listings! We combine the power of YouTube", "clean_text": "organic Google Listings! We combine the power of YouTube"}
|
| 118 |
+
{"noisy_text": "", "clean_text": "१४ १७ ५८ १२ १८ १३ -९ (अ)"}
|
| 119 |
+
{"noisy_text": "from a specialist if your unit is", "clean_text": "from a specialist if your unit is"}
|
| 120 |
+
{"noisy_text": "absorb", "clean_text": "absorb"}
|
| 121 |
+
{"noisy_text": "oo मति? 2077 area 14 गते... 7", "clean_text": "मितिः 2077 कात्तिक 14 गते"}
|
| 122 |
+
{"noisy_text": "ripping you off. We all have a part to play in creating", "clean_text": "ripping you off. We all have a part to play in creating"}
|
| 123 |
+
{"noisy_text": "and offers", "clean_text": "and offers"}
|
| 124 |
+
{"noisy_text": "own moment of “enlightenment”", "clean_text": "own moment of “enlightenment”"}
|
| 125 |
+
{"noisy_text": "wae वपियमा ओवशूयक कार्यानुवंयनेका arf eres", "clean_text": "प्रस्तुत विषयमा आवश्यक कार्यान्वयनका लागि पठाइएको छ"}
|
| 126 |
+
{"noisy_text": "surgeon.</p> <p>Earlier", "clean_text": "surgeon.</p> <p>Earlier"}
|
| 127 |
+
{"noisy_text": "technology Mik thinks we all need to read", "clean_text": "technology Mik thinks we all need to read"}
|
| 128 |
+
{"noisy_text": "——e छु od", "clean_text": "३ डिडि ढि"}
|
| 129 |
+
{"noisy_text": "~ ret ger Ree I झै किक किट", "clean_text": "बाटोमा दशा सुधार्ने गुलाम नै दिऊँ कि?"}
|
| 130 |
+
{"noisy_text": "“mare सहयोग: तथा कृषमता-अभविद्धाकिार्यक्रम", "clean_text": "प्राविधिक सहयोग तथा क्षमता अभिवृद्धि कार्यक्रम"}
|
| 131 |
+
{"noisy_text": "Pit -2068] 9/92", "clean_text": "मिति : २०६५/९२/९३"}
|
| 132 |
+
{"noisy_text": "media to promote", "clean_text": "media to promote"}
|
| 133 |
+
{"noisy_text": "jowers.creusing just 60:5४", "clean_text": "flowers,growing just out"}
|
| 134 |
+
{"noisy_text": "eo fan et न्युन तै ढिक र", "clean_text": "सबैले शिक्षा सजाउने सुत्र में दिऊं?"}
|
| 135 |
+
{"noisy_text": "experiences of", "clean_text": "experiences of"}
|
| 136 |
+
{"noisy_text": "pee en | / चन 6962/877/078 ee 7", "clean_text": "च.नं. 6962/०77/०78"}
|
| 137 |
+
{"noisy_text": "again, the price won't change. It does not matter", "clean_text": "again, the price won't change. It does not matter"}
|
| 138 |
+
{"noisy_text": "लपल chine ont the birdhouse.", "clean_text": "checking out the birdhouse."}
|
| 139 |
+
{"noisy_text": "नोम्पड VBP ut के", "clean_text": "माधव महा. च्याल्ने"}
|
| 140 |
+
{"noisy_text": "years. She assisted", "clean_text": "years. She assisted"}
|
| 141 |
+
{"noisy_text": "[यवस्था, सहकारी तथा abet नविरणमन्त्रालय ALS, §", "clean_text": "भूमि व्यवस्था, सहकारी तथा गरिबी निवारण मन्त्रालय सुर्खेत, सुर्खेत"}
|
| 142 |
+
{"noisy_text": "", "clean_text": "म कस्तो भै दिऊ?"}
|
| 143 |
+
{"noisy_text": "‘hearing things", "clean_text": "‘hearing things"}
|
| 144 |
+
{"noisy_text": "[र", "clean_text": "Pot."}
|
| 145 |
+
{"noisy_text": "१-संचवि, qatar, सहकारी तथा aft ATCT at", "clean_text": "श्री उप-सचिव, भूमि व्यवस्था, सहकारी तथा गरिबी निवारण मन्त्रालय"}
|
| 146 |
+
{"noisy_text": "", "clean_text": "म कस्तो भै दिऊ?"}
|
| 147 |
+
{"noisy_text": "“ सादर अनुरोध गर्दछु, मनूत्रालय प्रवकृता...", "clean_text": "सादर अनुरोध गर्दछु, मन्त्रालय प्रवक्ता"}
|
| 148 |
+
{"noisy_text": "द्र", "clean_text": "गुखु"}
|
| 149 |
+
{"noisy_text": "need new pillows. i want the cariloha gel pillow. | learned that they have bamboo mattresses! | would love ta", "clean_text": "need new pillows. i want the cariloha gel pillow. I learned that they have bamboo mattresses! I would love to"}
|
| 150 |
+
{"noisy_text": "— ee 7110.", "clean_text": "२ ७५० १५० १० १"}
|
| 151 |
+
{"noisy_text": "री महानरिदिशक, भौतकि qatar तथा aaa Aa", "clean_text": "श्री महानिर्देशक, भौतिक पूर्वाधार तथा यातायात मन्त्रालय"}
|
| 152 |
+
{"noisy_text": "कार्य बविरण तथा बाँडफाँड समूबन्धमा ॥", "clean_text": "कार्य विवरण तथा बाँडफ���ँड सम्बन्धमा"}
|
| 153 |
+
{"noisy_text": "means", "clean_text": "means"}
|
| 154 |
+
{"noisy_text": "mu value of such a reletionship is hard to quantify, the 1774 5/000511100 GENE", "clean_text": "the value of such a relationship is hard to quantify, the intangible benefits may be"}
|
| 155 |
+
{"noisy_text": "सैनकि मुख्यालय जङ्गी अड्डा जनकपुर, धनुषा", "clean_text": "सैनिक मुख्यालय जङ्गी अड्डा जनकपुर, धनुषा"}
|
| 156 |
+
{"noisy_text": "fe ir", "clean_text": "पुजि हो"}
|
| 157 |
+
{"noisy_text": "and", "clean_text": "and"}
|
| 158 |
+
{"noisy_text": "", "clean_text": "समर्थी तथा"}
|
| 159 |
+
{"noisy_text": "fe ir", "clean_text": "पुजि हो"}
|
| 160 |
+
{"noisy_text": "Se AS", "clean_text": "गे१११"}
|
| 161 |
+
{"noisy_text": "६ ast. 196/978/979", "clean_text": "च.नं. 196/०78/०79"}
|
| 162 |
+
{"noisy_text": "ce tpt Feat TTS जनकपुर, घनुषा ७...", "clean_text": "प्रहरी प्रधान कार्यालय जनकपुर, धनुषा"}
|
| 163 |
+
{"noisy_text": "", "clean_text": "Overhanging trees or high-rise"}
|
| 164 |
+
{"noisy_text": "mind, listening to", "clean_text": "mind, listening to"}
|
| 165 |
+
{"noisy_text": "warded बषियंमा HATH AAA AT लागपिडाइएको :", "clean_text": "प्रस्तुत विषयमा आवश्यक कार्यान्वयनका लागि पठाइएको छ"}
|
| 166 |
+
{"noisy_text": "clones we know of. However, ampslographers believe that the mutation rate for Pinot is actually on par with", "clean_text": "clones we know of. However, ampelographers believe that the mutation rate for Pinot is actually on par with"}
|
| 167 |
+
{"noisy_text": "e fe\nपद 4D", "clean_text": "1 1\n8 6\n९ ८"}
|
| 168 |
+
{"noisy_text": "e-commerce traffic is referred by search.” BI Intelligence Start", "clean_text": "e-commerce traffic is referred by search.” BI Intelligence Start"}
|
| 169 |
+
{"noisy_text": "jot", "clean_text": "गर्ने"}
|
| 170 |
+
{"noisy_text": "my selection of", "clean_text": "my selection of"}
|
runs/hf-corrector-mixed-v1-rebalanced/final-model/added_tokens.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<extra_id_0>": 259,
|
| 3 |
+
"<extra_id_100>": 359,
|
| 4 |
+
"<extra_id_101>": 360,
|
| 5 |
+
"<extra_id_102>": 361,
|
| 6 |
+
"<extra_id_103>": 362,
|
| 7 |
+
"<extra_id_104>": 363,
|
| 8 |
+
"<extra_id_105>": 364,
|
| 9 |
+
"<extra_id_106>": 365,
|
| 10 |
+
"<extra_id_107>": 366,
|
| 11 |
+
"<extra_id_108>": 367,
|
| 12 |
+
"<extra_id_109>": 368,
|
| 13 |
+
"<extra_id_10>": 269,
|
| 14 |
+
"<extra_id_110>": 369,
|
| 15 |
+
"<extra_id_111>": 370,
|
| 16 |
+
"<extra_id_112>": 371,
|
| 17 |
+
"<extra_id_113>": 372,
|
| 18 |
+
"<extra_id_114>": 373,
|
| 19 |
+
"<extra_id_115>": 374,
|
| 20 |
+
"<extra_id_116>": 375,
|
| 21 |
+
"<extra_id_117>": 376,
|
| 22 |
+
"<extra_id_118>": 377,
|
| 23 |
+
"<extra_id_119>": 378,
|
| 24 |
+
"<extra_id_11>": 270,
|
| 25 |
+
"<extra_id_120>": 379,
|
| 26 |
+
"<extra_id_121>": 380,
|
| 27 |
+
"<extra_id_122>": 381,
|
| 28 |
+
"<extra_id_123>": 382,
|
| 29 |
+
"<extra_id_124>": 383,
|
| 30 |
+
"<extra_id_12>": 271,
|
| 31 |
+
"<extra_id_13>": 272,
|
| 32 |
+
"<extra_id_14>": 273,
|
| 33 |
+
"<extra_id_15>": 274,
|
| 34 |
+
"<extra_id_16>": 275,
|
| 35 |
+
"<extra_id_17>": 276,
|
| 36 |
+
"<extra_id_18>": 277,
|
| 37 |
+
"<extra_id_19>": 278,
|
| 38 |
+
"<extra_id_1>": 260,
|
| 39 |
+
"<extra_id_20>": 279,
|
| 40 |
+
"<extra_id_21>": 280,
|
| 41 |
+
"<extra_id_22>": 281,
|
| 42 |
+
"<extra_id_23>": 282,
|
| 43 |
+
"<extra_id_24>": 283,
|
| 44 |
+
"<extra_id_25>": 284,
|
| 45 |
+
"<extra_id_26>": 285,
|
| 46 |
+
"<extra_id_27>": 286,
|
| 47 |
+
"<extra_id_28>": 287,
|
| 48 |
+
"<extra_id_29>": 288,
|
| 49 |
+
"<extra_id_2>": 261,
|
| 50 |
+
"<extra_id_30>": 289,
|
| 51 |
+
"<extra_id_31>": 290,
|
| 52 |
+
"<extra_id_32>": 291,
|
| 53 |
+
"<extra_id_33>": 292,
|
| 54 |
+
"<extra_id_34>": 293,
|
| 55 |
+
"<extra_id_35>": 294,
|
| 56 |
+
"<extra_id_36>": 295,
|
| 57 |
+
"<extra_id_37>": 296,
|
| 58 |
+
"<extra_id_38>": 297,
|
| 59 |
+
"<extra_id_39>": 298,
|
| 60 |
+
"<extra_id_3>": 262,
|
| 61 |
+
"<extra_id_40>": 299,
|
| 62 |
+
"<extra_id_41>": 300,
|
| 63 |
+
"<extra_id_42>": 301,
|
| 64 |
+
"<extra_id_43>": 302,
|
| 65 |
+
"<extra_id_44>": 303,
|
| 66 |
+
"<extra_id_45>": 304,
|
| 67 |
+
"<extra_id_46>": 305,
|
| 68 |
+
"<extra_id_47>": 306,
|
| 69 |
+
"<extra_id_48>": 307,
|
| 70 |
+
"<extra_id_49>": 308,
|
| 71 |
+
"<extra_id_4>": 263,
|
| 72 |
+
"<extra_id_50>": 309,
|
| 73 |
+
"<extra_id_51>": 310,
|
| 74 |
+
"<extra_id_52>": 311,
|
| 75 |
+
"<extra_id_53>": 312,
|
| 76 |
+
"<extra_id_54>": 313,
|
| 77 |
+
"<extra_id_55>": 314,
|
| 78 |
+
"<extra_id_56>": 315,
|
| 79 |
+
"<extra_id_57>": 316,
|
| 80 |
+
"<extra_id_58>": 317,
|
| 81 |
+
"<extra_id_59>": 318,
|
| 82 |
+
"<extra_id_5>": 264,
|
| 83 |
+
"<extra_id_60>": 319,
|
| 84 |
+
"<extra_id_61>": 320,
|
| 85 |
+
"<extra_id_62>": 321,
|
| 86 |
+
"<extra_id_63>": 322,
|
| 87 |
+
"<extra_id_64>": 323,
|
| 88 |
+
"<extra_id_65>": 324,
|
| 89 |
+
"<extra_id_66>": 325,
|
| 90 |
+
"<extra_id_67>": 326,
|
| 91 |
+
"<extra_id_68>": 327,
|
| 92 |
+
"<extra_id_69>": 328,
|
| 93 |
+
"<extra_id_6>": 265,
|
| 94 |
+
"<extra_id_70>": 329,
|
| 95 |
+
"<extra_id_71>": 330,
|
| 96 |
+
"<extra_id_72>": 331,
|
| 97 |
+
"<extra_id_73>": 332,
|
| 98 |
+
"<extra_id_74>": 333,
|
| 99 |
+
"<extra_id_75>": 334,
|
| 100 |
+
"<extra_id_76>": 335,
|
| 101 |
+
"<extra_id_77>": 336,
|
| 102 |
+
"<extra_id_78>": 337,
|
| 103 |
+
"<extra_id_79>": 338,
|
| 104 |
+
"<extra_id_7>": 266,
|
| 105 |
+
"<extra_id_80>": 339,
|
| 106 |
+
"<extra_id_81>": 340,
|
| 107 |
+
"<extra_id_82>": 341,
|
| 108 |
+
"<extra_id_83>": 342,
|
| 109 |
+
"<extra_id_84>": 343,
|
| 110 |
+
"<extra_id_85>": 344,
|
| 111 |
+
"<extra_id_86>": 345,
|
| 112 |
+
"<extra_id_87>": 346,
|
| 113 |
+
"<extra_id_88>": 347,
|
| 114 |
+
"<extra_id_89>": 348,
|
| 115 |
+
"<extra_id_8>": 267,
|
| 116 |
+
"<extra_id_90>": 349,
|
| 117 |
+
"<extra_id_91>": 350,
|
| 118 |
+
"<extra_id_92>": 351,
|
| 119 |
+
"<extra_id_93>": 352,
|
| 120 |
+
"<extra_id_94>": 353,
|
| 121 |
+
"<extra_id_95>": 354,
|
| 122 |
+
"<extra_id_96>": 355,
|
| 123 |
+
"<extra_id_97>": 356,
|
| 124 |
+
"<extra_id_98>": 357,
|
| 125 |
+
"<extra_id_99>": 358,
|
| 126 |
+
"<extra_id_9>": 268
|
| 127 |
+
}
|
runs/hf-corrector-mixed-v1-rebalanced/final-model/config.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"T5ForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"classifier_dropout": 0.0,
|
| 6 |
+
"d_ff": 3584,
|
| 7 |
+
"d_kv": 64,
|
| 8 |
+
"d_model": 1472,
|
| 9 |
+
"decoder_start_token_id": 0,
|
| 10 |
+
"dense_act_fn": "gelu_new",
|
| 11 |
+
"dropout_rate": 0.1,
|
| 12 |
+
"dtype": "float32",
|
| 13 |
+
"eos_token_id": 1,
|
| 14 |
+
"feed_forward_proj": "gated-gelu",
|
| 15 |
+
"gradient_checkpointing": false,
|
| 16 |
+
"initializer_factor": 1.0,
|
| 17 |
+
"is_decoder": false,
|
| 18 |
+
"is_encoder_decoder": true,
|
| 19 |
+
"is_gated_act": true,
|
| 20 |
+
"layer_norm_epsilon": 1e-06,
|
| 21 |
+
"model_type": "t5",
|
| 22 |
+
"num_decoder_layers": 4,
|
| 23 |
+
"num_heads": 6,
|
| 24 |
+
"num_layers": 12,
|
| 25 |
+
"pad_token_id": 0,
|
| 26 |
+
"relative_attention_max_distance": 128,
|
| 27 |
+
"relative_attention_num_buckets": 32,
|
| 28 |
+
"scale_decoder_outputs": false,
|
| 29 |
+
"tie_word_embeddings": true,
|
| 30 |
+
"tokenizer_class": "ByT5Tokenizer",
|
| 31 |
+
"transformers_version": "5.12.0",
|
| 32 |
+
"use_cache": false,
|
| 33 |
+
"vocab_size": 384
|
| 34 |
+
}
|
runs/hf-corrector-mixed-v1-rebalanced/final-model/generation_config.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": false,
|
| 3 |
+
"assistant_confidence_threshold": 0.4,
|
| 4 |
+
"assistant_lookbehind": 10,
|
| 5 |
+
"decoder_start_token_id": 0,
|
| 6 |
+
"diversity_penalty": 0.0,
|
| 7 |
+
"do_sample": false,
|
| 8 |
+
"early_stopping": false,
|
| 9 |
+
"encoder_no_repeat_ngram_size": 0,
|
| 10 |
+
"encoder_repetition_penalty": 1.0,
|
| 11 |
+
"eos_token_id": [
|
| 12 |
+
1
|
| 13 |
+
],
|
| 14 |
+
"epsilon_cutoff": 0.0,
|
| 15 |
+
"eta_cutoff": 0.0,
|
| 16 |
+
"length_penalty": 1.0,
|
| 17 |
+
"max_length": 256,
|
| 18 |
+
"min_length": 0,
|
| 19 |
+
"no_repeat_ngram_size": 0,
|
| 20 |
+
"num_assistant_tokens": 20,
|
| 21 |
+
"num_assistant_tokens_schedule": "constant",
|
| 22 |
+
"num_beam_groups": 1,
|
| 23 |
+
"num_beams": 1,
|
| 24 |
+
"num_return_sequences": 1,
|
| 25 |
+
"output_scores": false,
|
| 26 |
+
"pad_token_id": 0,
|
| 27 |
+
"remove_invalid_values": false,
|
| 28 |
+
"repetition_penalty": 1.0,
|
| 29 |
+
"return_dict_in_generate": false,
|
| 30 |
+
"target_lookbehind": 10,
|
| 31 |
+
"temperature": 1.0,
|
| 32 |
+
"top_k": 50,
|
| 33 |
+
"top_p": 1.0,
|
| 34 |
+
"transformers_version": "5.12.0",
|
| 35 |
+
"typical_p": 1.0,
|
| 36 |
+
"use_cache": true
|
| 37 |
+
}
|
runs/hf-corrector-mixed-v1-rebalanced/final-model/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:706c3eba193046762db698cfe95270fef8ebbf3cdae515839ccea75ccdf1079e
|
| 3 |
+
size 1198571496
|
runs/hf-corrector-mixed-v1-rebalanced/final-model/tokenizer_config.json
ADDED
|
@@ -0,0 +1,1291 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"added_tokens_decoder": {
|
| 3 |
+
"0": {
|
| 4 |
+
"content": "<pad>",
|
| 5 |
+
"lstrip": false,
|
| 6 |
+
"normalized": true,
|
| 7 |
+
"rstrip": false,
|
| 8 |
+
"single_word": false,
|
| 9 |
+
"special": true
|
| 10 |
+
},
|
| 11 |
+
"1": {
|
| 12 |
+
"content": "</s>",
|
| 13 |
+
"lstrip": false,
|
| 14 |
+
"normalized": true,
|
| 15 |
+
"rstrip": false,
|
| 16 |
+
"single_word": false,
|
| 17 |
+
"special": true
|
| 18 |
+
},
|
| 19 |
+
"2": {
|
| 20 |
+
"content": "<unk>",
|
| 21 |
+
"lstrip": false,
|
| 22 |
+
"normalized": true,
|
| 23 |
+
"rstrip": false,
|
| 24 |
+
"single_word": false,
|
| 25 |
+
"special": true
|
| 26 |
+
},
|
| 27 |
+
"259": {
|
| 28 |
+
"content": "<extra_id_0>",
|
| 29 |
+
"lstrip": false,
|
| 30 |
+
"normalized": false,
|
| 31 |
+
"rstrip": false,
|
| 32 |
+
"single_word": false,
|
| 33 |
+
"special": true
|
| 34 |
+
},
|
| 35 |
+
"260": {
|
| 36 |
+
"content": "<extra_id_1>",
|
| 37 |
+
"lstrip": false,
|
| 38 |
+
"normalized": false,
|
| 39 |
+
"rstrip": false,
|
| 40 |
+
"single_word": false,
|
| 41 |
+
"special": true
|
| 42 |
+
},
|
| 43 |
+
"261": {
|
| 44 |
+
"content": "<extra_id_2>",
|
| 45 |
+
"lstrip": false,
|
| 46 |
+
"normalized": false,
|
| 47 |
+
"rstrip": false,
|
| 48 |
+
"single_word": false,
|
| 49 |
+
"special": true
|
| 50 |
+
},
|
| 51 |
+
"262": {
|
| 52 |
+
"content": "<extra_id_3>",
|
| 53 |
+
"lstrip": false,
|
| 54 |
+
"normalized": false,
|
| 55 |
+
"rstrip": false,
|
| 56 |
+
"single_word": false,
|
| 57 |
+
"special": true
|
| 58 |
+
},
|
| 59 |
+
"263": {
|
| 60 |
+
"content": "<extra_id_4>",
|
| 61 |
+
"lstrip": false,
|
| 62 |
+
"normalized": false,
|
| 63 |
+
"rstrip": false,
|
| 64 |
+
"single_word": false,
|
| 65 |
+
"special": true
|
| 66 |
+
},
|
| 67 |
+
"264": {
|
| 68 |
+
"content": "<extra_id_5>",
|
| 69 |
+
"lstrip": false,
|
| 70 |
+
"normalized": false,
|
| 71 |
+
"rstrip": false,
|
| 72 |
+
"single_word": false,
|
| 73 |
+
"special": true
|
| 74 |
+
},
|
| 75 |
+
"265": {
|
| 76 |
+
"content": "<extra_id_6>",
|
| 77 |
+
"lstrip": false,
|
| 78 |
+
"normalized": false,
|
| 79 |
+
"rstrip": false,
|
| 80 |
+
"single_word": false,
|
| 81 |
+
"special": true
|
| 82 |
+
},
|
| 83 |
+
"266": {
|
| 84 |
+
"content": "<extra_id_7>",
|
| 85 |
+
"lstrip": false,
|
| 86 |
+
"normalized": false,
|
| 87 |
+
"rstrip": false,
|
| 88 |
+
"single_word": false,
|
| 89 |
+
"special": true
|
| 90 |
+
},
|
| 91 |
+
"267": {
|
| 92 |
+
"content": "<extra_id_8>",
|
| 93 |
+
"lstrip": false,
|
| 94 |
+
"normalized": false,
|
| 95 |
+
"rstrip": false,
|
| 96 |
+
"single_word": false,
|
| 97 |
+
"special": true
|
| 98 |
+
},
|
| 99 |
+
"268": {
|
| 100 |
+
"content": "<extra_id_9>",
|
| 101 |
+
"lstrip": false,
|
| 102 |
+
"normalized": false,
|
| 103 |
+
"rstrip": false,
|
| 104 |
+
"single_word": false,
|
| 105 |
+
"special": true
|
| 106 |
+
},
|
| 107 |
+
"269": {
|
| 108 |
+
"content": "<extra_id_10>",
|
| 109 |
+
"lstrip": false,
|
| 110 |
+
"normalized": false,
|
| 111 |
+
"rstrip": false,
|
| 112 |
+
"single_word": false,
|
| 113 |
+
"special": true
|
| 114 |
+
},
|
| 115 |
+
"270": {
|
| 116 |
+
"content": "<extra_id_11>",
|
| 117 |
+
"lstrip": false,
|
| 118 |
+
"normalized": false,
|
| 119 |
+
"rstrip": false,
|
| 120 |
+
"single_word": false,
|
| 121 |
+
"special": true
|
| 122 |
+
},
|
| 123 |
+
"271": {
|
| 124 |
+
"content": "<extra_id_12>",
|
| 125 |
+
"lstrip": false,
|
| 126 |
+
"normalized": false,
|
| 127 |
+
"rstrip": false,
|
| 128 |
+
"single_word": false,
|
| 129 |
+
"special": true
|
| 130 |
+
},
|
| 131 |
+
"272": {
|
| 132 |
+
"content": "<extra_id_13>",
|
| 133 |
+
"lstrip": false,
|
| 134 |
+
"normalized": false,
|
| 135 |
+
"rstrip": false,
|
| 136 |
+
"single_word": false,
|
| 137 |
+
"special": true
|
| 138 |
+
},
|
| 139 |
+
"273": {
|
| 140 |
+
"content": "<extra_id_14>",
|
| 141 |
+
"lstrip": false,
|
| 142 |
+
"normalized": false,
|
| 143 |
+
"rstrip": false,
|
| 144 |
+
"single_word": false,
|
| 145 |
+
"special": true
|
| 146 |
+
},
|
| 147 |
+
"274": {
|
| 148 |
+
"content": "<extra_id_15>",
|
| 149 |
+
"lstrip": false,
|
| 150 |
+
"normalized": false,
|
| 151 |
+
"rstrip": false,
|
| 152 |
+
"single_word": false,
|
| 153 |
+
"special": true
|
| 154 |
+
},
|
| 155 |
+
"275": {
|
| 156 |
+
"content": "<extra_id_16>",
|
| 157 |
+
"lstrip": false,
|
| 158 |
+
"normalized": false,
|
| 159 |
+
"rstrip": false,
|
| 160 |
+
"single_word": false,
|
| 161 |
+
"special": true
|
| 162 |
+
},
|
| 163 |
+
"276": {
|
| 164 |
+
"content": "<extra_id_17>",
|
| 165 |
+
"lstrip": false,
|
| 166 |
+
"normalized": false,
|
| 167 |
+
"rstrip": false,
|
| 168 |
+
"single_word": false,
|
| 169 |
+
"special": true
|
| 170 |
+
},
|
| 171 |
+
"277": {
|
| 172 |
+
"content": "<extra_id_18>",
|
| 173 |
+
"lstrip": false,
|
| 174 |
+
"normalized": false,
|
| 175 |
+
"rstrip": false,
|
| 176 |
+
"single_word": false,
|
| 177 |
+
"special": true
|
| 178 |
+
},
|
| 179 |
+
"278": {
|
| 180 |
+
"content": "<extra_id_19>",
|
| 181 |
+
"lstrip": false,
|
| 182 |
+
"normalized": false,
|
| 183 |
+
"rstrip": false,
|
| 184 |
+
"single_word": false,
|
| 185 |
+
"special": true
|
| 186 |
+
},
|
| 187 |
+
"279": {
|
| 188 |
+
"content": "<extra_id_20>",
|
| 189 |
+
"lstrip": false,
|
| 190 |
+
"normalized": false,
|
| 191 |
+
"rstrip": false,
|
| 192 |
+
"single_word": false,
|
| 193 |
+
"special": true
|
| 194 |
+
},
|
| 195 |
+
"280": {
|
| 196 |
+
"content": "<extra_id_21>",
|
| 197 |
+
"lstrip": false,
|
| 198 |
+
"normalized": false,
|
| 199 |
+
"rstrip": false,
|
| 200 |
+
"single_word": false,
|
| 201 |
+
"special": true
|
| 202 |
+
},
|
| 203 |
+
"281": {
|
| 204 |
+
"content": "<extra_id_22>",
|
| 205 |
+
"lstrip": false,
|
| 206 |
+
"normalized": false,
|
| 207 |
+
"rstrip": false,
|
| 208 |
+
"single_word": false,
|
| 209 |
+
"special": true
|
| 210 |
+
},
|
| 211 |
+
"282": {
|
| 212 |
+
"content": "<extra_id_23>",
|
| 213 |
+
"lstrip": false,
|
| 214 |
+
"normalized": false,
|
| 215 |
+
"rstrip": false,
|
| 216 |
+
"single_word": false,
|
| 217 |
+
"special": true
|
| 218 |
+
},
|
| 219 |
+
"283": {
|
| 220 |
+
"content": "<extra_id_24>",
|
| 221 |
+
"lstrip": false,
|
| 222 |
+
"normalized": false,
|
| 223 |
+
"rstrip": false,
|
| 224 |
+
"single_word": false,
|
| 225 |
+
"special": true
|
| 226 |
+
},
|
| 227 |
+
"284": {
|
| 228 |
+
"content": "<extra_id_25>",
|
| 229 |
+
"lstrip": false,
|
| 230 |
+
"normalized": false,
|
| 231 |
+
"rstrip": false,
|
| 232 |
+
"single_word": false,
|
| 233 |
+
"special": true
|
| 234 |
+
},
|
| 235 |
+
"285": {
|
| 236 |
+
"content": "<extra_id_26>",
|
| 237 |
+
"lstrip": false,
|
| 238 |
+
"normalized": false,
|
| 239 |
+
"rstrip": false,
|
| 240 |
+
"single_word": false,
|
| 241 |
+
"special": true
|
| 242 |
+
},
|
| 243 |
+
"286": {
|
| 244 |
+
"content": "<extra_id_27>",
|
| 245 |
+
"lstrip": false,
|
| 246 |
+
"normalized": false,
|
| 247 |
+
"rstrip": false,
|
| 248 |
+
"single_word": false,
|
| 249 |
+
"special": true
|
| 250 |
+
},
|
| 251 |
+
"287": {
|
| 252 |
+
"content": "<extra_id_28>",
|
| 253 |
+
"lstrip": false,
|
| 254 |
+
"normalized": false,
|
| 255 |
+
"rstrip": false,
|
| 256 |
+
"single_word": false,
|
| 257 |
+
"special": true
|
| 258 |
+
},
|
| 259 |
+
"288": {
|
| 260 |
+
"content": "<extra_id_29>",
|
| 261 |
+
"lstrip": false,
|
| 262 |
+
"normalized": false,
|
| 263 |
+
"rstrip": false,
|
| 264 |
+
"single_word": false,
|
| 265 |
+
"special": true
|
| 266 |
+
},
|
| 267 |
+
"289": {
|
| 268 |
+
"content": "<extra_id_30>",
|
| 269 |
+
"lstrip": false,
|
| 270 |
+
"normalized": false,
|
| 271 |
+
"rstrip": false,
|
| 272 |
+
"single_word": false,
|
| 273 |
+
"special": true
|
| 274 |
+
},
|
| 275 |
+
"290": {
|
| 276 |
+
"content": "<extra_id_31>",
|
| 277 |
+
"lstrip": false,
|
| 278 |
+
"normalized": false,
|
| 279 |
+
"rstrip": false,
|
| 280 |
+
"single_word": false,
|
| 281 |
+
"special": true
|
| 282 |
+
},
|
| 283 |
+
"291": {
|
| 284 |
+
"content": "<extra_id_32>",
|
| 285 |
+
"lstrip": false,
|
| 286 |
+
"normalized": false,
|
| 287 |
+
"rstrip": false,
|
| 288 |
+
"single_word": false,
|
| 289 |
+
"special": true
|
| 290 |
+
},
|
| 291 |
+
"292": {
|
| 292 |
+
"content": "<extra_id_33>",
|
| 293 |
+
"lstrip": false,
|
| 294 |
+
"normalized": false,
|
| 295 |
+
"rstrip": false,
|
| 296 |
+
"single_word": false,
|
| 297 |
+
"special": true
|
| 298 |
+
},
|
| 299 |
+
"293": {
|
| 300 |
+
"content": "<extra_id_34>",
|
| 301 |
+
"lstrip": false,
|
| 302 |
+
"normalized": false,
|
| 303 |
+
"rstrip": false,
|
| 304 |
+
"single_word": false,
|
| 305 |
+
"special": true
|
| 306 |
+
},
|
| 307 |
+
"294": {
|
| 308 |
+
"content": "<extra_id_35>",
|
| 309 |
+
"lstrip": false,
|
| 310 |
+
"normalized": false,
|
| 311 |
+
"rstrip": false,
|
| 312 |
+
"single_word": false,
|
| 313 |
+
"special": true
|
| 314 |
+
},
|
| 315 |
+
"295": {
|
| 316 |
+
"content": "<extra_id_36>",
|
| 317 |
+
"lstrip": false,
|
| 318 |
+
"normalized": false,
|
| 319 |
+
"rstrip": false,
|
| 320 |
+
"single_word": false,
|
| 321 |
+
"special": true
|
| 322 |
+
},
|
| 323 |
+
"296": {
|
| 324 |
+
"content": "<extra_id_37>",
|
| 325 |
+
"lstrip": false,
|
| 326 |
+
"normalized": false,
|
| 327 |
+
"rstrip": false,
|
| 328 |
+
"single_word": false,
|
| 329 |
+
"special": true
|
| 330 |
+
},
|
| 331 |
+
"297": {
|
| 332 |
+
"content": "<extra_id_38>",
|
| 333 |
+
"lstrip": false,
|
| 334 |
+
"normalized": false,
|
| 335 |
+
"rstrip": false,
|
| 336 |
+
"single_word": false,
|
| 337 |
+
"special": true
|
| 338 |
+
},
|
| 339 |
+
"298": {
|
| 340 |
+
"content": "<extra_id_39>",
|
| 341 |
+
"lstrip": false,
|
| 342 |
+
"normalized": false,
|
| 343 |
+
"rstrip": false,
|
| 344 |
+
"single_word": false,
|
| 345 |
+
"special": true
|
| 346 |
+
},
|
| 347 |
+
"299": {
|
| 348 |
+
"content": "<extra_id_40>",
|
| 349 |
+
"lstrip": false,
|
| 350 |
+
"normalized": false,
|
| 351 |
+
"rstrip": false,
|
| 352 |
+
"single_word": false,
|
| 353 |
+
"special": true
|
| 354 |
+
},
|
| 355 |
+
"300": {
|
| 356 |
+
"content": "<extra_id_41>",
|
| 357 |
+
"lstrip": false,
|
| 358 |
+
"normalized": false,
|
| 359 |
+
"rstrip": false,
|
| 360 |
+
"single_word": false,
|
| 361 |
+
"special": true
|
| 362 |
+
},
|
| 363 |
+
"301": {
|
| 364 |
+
"content": "<extra_id_42>",
|
| 365 |
+
"lstrip": false,
|
| 366 |
+
"normalized": false,
|
| 367 |
+
"rstrip": false,
|
| 368 |
+
"single_word": false,
|
| 369 |
+
"special": true
|
| 370 |
+
},
|
| 371 |
+
"302": {
|
| 372 |
+
"content": "<extra_id_43>",
|
| 373 |
+
"lstrip": false,
|
| 374 |
+
"normalized": false,
|
| 375 |
+
"rstrip": false,
|
| 376 |
+
"single_word": false,
|
| 377 |
+
"special": true
|
| 378 |
+
},
|
| 379 |
+
"303": {
|
| 380 |
+
"content": "<extra_id_44>",
|
| 381 |
+
"lstrip": false,
|
| 382 |
+
"normalized": false,
|
| 383 |
+
"rstrip": false,
|
| 384 |
+
"single_word": false,
|
| 385 |
+
"special": true
|
| 386 |
+
},
|
| 387 |
+
"304": {
|
| 388 |
+
"content": "<extra_id_45>",
|
| 389 |
+
"lstrip": false,
|
| 390 |
+
"normalized": false,
|
| 391 |
+
"rstrip": false,
|
| 392 |
+
"single_word": false,
|
| 393 |
+
"special": true
|
| 394 |
+
},
|
| 395 |
+
"305": {
|
| 396 |
+
"content": "<extra_id_46>",
|
| 397 |
+
"lstrip": false,
|
| 398 |
+
"normalized": false,
|
| 399 |
+
"rstrip": false,
|
| 400 |
+
"single_word": false,
|
| 401 |
+
"special": true
|
| 402 |
+
},
|
| 403 |
+
"306": {
|
| 404 |
+
"content": "<extra_id_47>",
|
| 405 |
+
"lstrip": false,
|
| 406 |
+
"normalized": false,
|
| 407 |
+
"rstrip": false,
|
| 408 |
+
"single_word": false,
|
| 409 |
+
"special": true
|
| 410 |
+
},
|
| 411 |
+
"307": {
|
| 412 |
+
"content": "<extra_id_48>",
|
| 413 |
+
"lstrip": false,
|
| 414 |
+
"normalized": false,
|
| 415 |
+
"rstrip": false,
|
| 416 |
+
"single_word": false,
|
| 417 |
+
"special": true
|
| 418 |
+
},
|
| 419 |
+
"308": {
|
| 420 |
+
"content": "<extra_id_49>",
|
| 421 |
+
"lstrip": false,
|
| 422 |
+
"normalized": false,
|
| 423 |
+
"rstrip": false,
|
| 424 |
+
"single_word": false,
|
| 425 |
+
"special": true
|
| 426 |
+
},
|
| 427 |
+
"309": {
|
| 428 |
+
"content": "<extra_id_50>",
|
| 429 |
+
"lstrip": false,
|
| 430 |
+
"normalized": false,
|
| 431 |
+
"rstrip": false,
|
| 432 |
+
"single_word": false,
|
| 433 |
+
"special": true
|
| 434 |
+
},
|
| 435 |
+
"310": {
|
| 436 |
+
"content": "<extra_id_51>",
|
| 437 |
+
"lstrip": false,
|
| 438 |
+
"normalized": false,
|
| 439 |
+
"rstrip": false,
|
| 440 |
+
"single_word": false,
|
| 441 |
+
"special": true
|
| 442 |
+
},
|
| 443 |
+
"311": {
|
| 444 |
+
"content": "<extra_id_52>",
|
| 445 |
+
"lstrip": false,
|
| 446 |
+
"normalized": false,
|
| 447 |
+
"rstrip": false,
|
| 448 |
+
"single_word": false,
|
| 449 |
+
"special": true
|
| 450 |
+
},
|
| 451 |
+
"312": {
|
| 452 |
+
"content": "<extra_id_53>",
|
| 453 |
+
"lstrip": false,
|
| 454 |
+
"normalized": false,
|
| 455 |
+
"rstrip": false,
|
| 456 |
+
"single_word": false,
|
| 457 |
+
"special": true
|
| 458 |
+
},
|
| 459 |
+
"313": {
|
| 460 |
+
"content": "<extra_id_54>",
|
| 461 |
+
"lstrip": false,
|
| 462 |
+
"normalized": false,
|
| 463 |
+
"rstrip": false,
|
| 464 |
+
"single_word": false,
|
| 465 |
+
"special": true
|
| 466 |
+
},
|
| 467 |
+
"314": {
|
| 468 |
+
"content": "<extra_id_55>",
|
| 469 |
+
"lstrip": false,
|
| 470 |
+
"normalized": false,
|
| 471 |
+
"rstrip": false,
|
| 472 |
+
"single_word": false,
|
| 473 |
+
"special": true
|
| 474 |
+
},
|
| 475 |
+
"315": {
|
| 476 |
+
"content": "<extra_id_56>",
|
| 477 |
+
"lstrip": false,
|
| 478 |
+
"normalized": false,
|
| 479 |
+
"rstrip": false,
|
| 480 |
+
"single_word": false,
|
| 481 |
+
"special": true
|
| 482 |
+
},
|
| 483 |
+
"316": {
|
| 484 |
+
"content": "<extra_id_57>",
|
| 485 |
+
"lstrip": false,
|
| 486 |
+
"normalized": false,
|
| 487 |
+
"rstrip": false,
|
| 488 |
+
"single_word": false,
|
| 489 |
+
"special": true
|
| 490 |
+
},
|
| 491 |
+
"317": {
|
| 492 |
+
"content": "<extra_id_58>",
|
| 493 |
+
"lstrip": false,
|
| 494 |
+
"normalized": false,
|
| 495 |
+
"rstrip": false,
|
| 496 |
+
"single_word": false,
|
| 497 |
+
"special": true
|
| 498 |
+
},
|
| 499 |
+
"318": {
|
| 500 |
+
"content": "<extra_id_59>",
|
| 501 |
+
"lstrip": false,
|
| 502 |
+
"normalized": false,
|
| 503 |
+
"rstrip": false,
|
| 504 |
+
"single_word": false,
|
| 505 |
+
"special": true
|
| 506 |
+
},
|
| 507 |
+
"319": {
|
| 508 |
+
"content": "<extra_id_60>",
|
| 509 |
+
"lstrip": false,
|
| 510 |
+
"normalized": false,
|
| 511 |
+
"rstrip": false,
|
| 512 |
+
"single_word": false,
|
| 513 |
+
"special": true
|
| 514 |
+
},
|
| 515 |
+
"320": {
|
| 516 |
+
"content": "<extra_id_61>",
|
| 517 |
+
"lstrip": false,
|
| 518 |
+
"normalized": false,
|
| 519 |
+
"rstrip": false,
|
| 520 |
+
"single_word": false,
|
| 521 |
+
"special": true
|
| 522 |
+
},
|
| 523 |
+
"321": {
|
| 524 |
+
"content": "<extra_id_62>",
|
| 525 |
+
"lstrip": false,
|
| 526 |
+
"normalized": false,
|
| 527 |
+
"rstrip": false,
|
| 528 |
+
"single_word": false,
|
| 529 |
+
"special": true
|
| 530 |
+
},
|
| 531 |
+
"322": {
|
| 532 |
+
"content": "<extra_id_63>",
|
| 533 |
+
"lstrip": false,
|
| 534 |
+
"normalized": false,
|
| 535 |
+
"rstrip": false,
|
| 536 |
+
"single_word": false,
|
| 537 |
+
"special": true
|
| 538 |
+
},
|
| 539 |
+
"323": {
|
| 540 |
+
"content": "<extra_id_64>",
|
| 541 |
+
"lstrip": false,
|
| 542 |
+
"normalized": false,
|
| 543 |
+
"rstrip": false,
|
| 544 |
+
"single_word": false,
|
| 545 |
+
"special": true
|
| 546 |
+
},
|
| 547 |
+
"324": {
|
| 548 |
+
"content": "<extra_id_65>",
|
| 549 |
+
"lstrip": false,
|
| 550 |
+
"normalized": false,
|
| 551 |
+
"rstrip": false,
|
| 552 |
+
"single_word": false,
|
| 553 |
+
"special": true
|
| 554 |
+
},
|
| 555 |
+
"325": {
|
| 556 |
+
"content": "<extra_id_66>",
|
| 557 |
+
"lstrip": false,
|
| 558 |
+
"normalized": false,
|
| 559 |
+
"rstrip": false,
|
| 560 |
+
"single_word": false,
|
| 561 |
+
"special": true
|
| 562 |
+
},
|
| 563 |
+
"326": {
|
| 564 |
+
"content": "<extra_id_67>",
|
| 565 |
+
"lstrip": false,
|
| 566 |
+
"normalized": false,
|
| 567 |
+
"rstrip": false,
|
| 568 |
+
"single_word": false,
|
| 569 |
+
"special": true
|
| 570 |
+
},
|
| 571 |
+
"327": {
|
| 572 |
+
"content": "<extra_id_68>",
|
| 573 |
+
"lstrip": false,
|
| 574 |
+
"normalized": false,
|
| 575 |
+
"rstrip": false,
|
| 576 |
+
"single_word": false,
|
| 577 |
+
"special": true
|
| 578 |
+
},
|
| 579 |
+
"328": {
|
| 580 |
+
"content": "<extra_id_69>",
|
| 581 |
+
"lstrip": false,
|
| 582 |
+
"normalized": false,
|
| 583 |
+
"rstrip": false,
|
| 584 |
+
"single_word": false,
|
| 585 |
+
"special": true
|
| 586 |
+
},
|
| 587 |
+
"329": {
|
| 588 |
+
"content": "<extra_id_70>",
|
| 589 |
+
"lstrip": false,
|
| 590 |
+
"normalized": false,
|
| 591 |
+
"rstrip": false,
|
| 592 |
+
"single_word": false,
|
| 593 |
+
"special": true
|
| 594 |
+
},
|
| 595 |
+
"330": {
|
| 596 |
+
"content": "<extra_id_71>",
|
| 597 |
+
"lstrip": false,
|
| 598 |
+
"normalized": false,
|
| 599 |
+
"rstrip": false,
|
| 600 |
+
"single_word": false,
|
| 601 |
+
"special": true
|
| 602 |
+
},
|
| 603 |
+
"331": {
|
| 604 |
+
"content": "<extra_id_72>",
|
| 605 |
+
"lstrip": false,
|
| 606 |
+
"normalized": false,
|
| 607 |
+
"rstrip": false,
|
| 608 |
+
"single_word": false,
|
| 609 |
+
"special": true
|
| 610 |
+
},
|
| 611 |
+
"332": {
|
| 612 |
+
"content": "<extra_id_73>",
|
| 613 |
+
"lstrip": false,
|
| 614 |
+
"normalized": false,
|
| 615 |
+
"rstrip": false,
|
| 616 |
+
"single_word": false,
|
| 617 |
+
"special": true
|
| 618 |
+
},
|
| 619 |
+
"333": {
|
| 620 |
+
"content": "<extra_id_74>",
|
| 621 |
+
"lstrip": false,
|
| 622 |
+
"normalized": false,
|
| 623 |
+
"rstrip": false,
|
| 624 |
+
"single_word": false,
|
| 625 |
+
"special": true
|
| 626 |
+
},
|
| 627 |
+
"334": {
|
| 628 |
+
"content": "<extra_id_75>",
|
| 629 |
+
"lstrip": false,
|
| 630 |
+
"normalized": false,
|
| 631 |
+
"rstrip": false,
|
| 632 |
+
"single_word": false,
|
| 633 |
+
"special": true
|
| 634 |
+
},
|
| 635 |
+
"335": {
|
| 636 |
+
"content": "<extra_id_76>",
|
| 637 |
+
"lstrip": false,
|
| 638 |
+
"normalized": false,
|
| 639 |
+
"rstrip": false,
|
| 640 |
+
"single_word": false,
|
| 641 |
+
"special": true
|
| 642 |
+
},
|
| 643 |
+
"336": {
|
| 644 |
+
"content": "<extra_id_77>",
|
| 645 |
+
"lstrip": false,
|
| 646 |
+
"normalized": false,
|
| 647 |
+
"rstrip": false,
|
| 648 |
+
"single_word": false,
|
| 649 |
+
"special": true
|
| 650 |
+
},
|
| 651 |
+
"337": {
|
| 652 |
+
"content": "<extra_id_78>",
|
| 653 |
+
"lstrip": false,
|
| 654 |
+
"normalized": false,
|
| 655 |
+
"rstrip": false,
|
| 656 |
+
"single_word": false,
|
| 657 |
+
"special": true
|
| 658 |
+
},
|
| 659 |
+
"338": {
|
| 660 |
+
"content": "<extra_id_79>",
|
| 661 |
+
"lstrip": false,
|
| 662 |
+
"normalized": false,
|
| 663 |
+
"rstrip": false,
|
| 664 |
+
"single_word": false,
|
| 665 |
+
"special": true
|
| 666 |
+
},
|
| 667 |
+
"339": {
|
| 668 |
+
"content": "<extra_id_80>",
|
| 669 |
+
"lstrip": false,
|
| 670 |
+
"normalized": false,
|
| 671 |
+
"rstrip": false,
|
| 672 |
+
"single_word": false,
|
| 673 |
+
"special": true
|
| 674 |
+
},
|
| 675 |
+
"340": {
|
| 676 |
+
"content": "<extra_id_81>",
|
| 677 |
+
"lstrip": false,
|
| 678 |
+
"normalized": false,
|
| 679 |
+
"rstrip": false,
|
| 680 |
+
"single_word": false,
|
| 681 |
+
"special": true
|
| 682 |
+
},
|
| 683 |
+
"341": {
|
| 684 |
+
"content": "<extra_id_82>",
|
| 685 |
+
"lstrip": false,
|
| 686 |
+
"normalized": false,
|
| 687 |
+
"rstrip": false,
|
| 688 |
+
"single_word": false,
|
| 689 |
+
"special": true
|
| 690 |
+
},
|
| 691 |
+
"342": {
|
| 692 |
+
"content": "<extra_id_83>",
|
| 693 |
+
"lstrip": false,
|
| 694 |
+
"normalized": false,
|
| 695 |
+
"rstrip": false,
|
| 696 |
+
"single_word": false,
|
| 697 |
+
"special": true
|
| 698 |
+
},
|
| 699 |
+
"343": {
|
| 700 |
+
"content": "<extra_id_84>",
|
| 701 |
+
"lstrip": false,
|
| 702 |
+
"normalized": false,
|
| 703 |
+
"rstrip": false,
|
| 704 |
+
"single_word": false,
|
| 705 |
+
"special": true
|
| 706 |
+
},
|
| 707 |
+
"344": {
|
| 708 |
+
"content": "<extra_id_85>",
|
| 709 |
+
"lstrip": false,
|
| 710 |
+
"normalized": false,
|
| 711 |
+
"rstrip": false,
|
| 712 |
+
"single_word": false,
|
| 713 |
+
"special": true
|
| 714 |
+
},
|
| 715 |
+
"345": {
|
| 716 |
+
"content": "<extra_id_86>",
|
| 717 |
+
"lstrip": false,
|
| 718 |
+
"normalized": false,
|
| 719 |
+
"rstrip": false,
|
| 720 |
+
"single_word": false,
|
| 721 |
+
"special": true
|
| 722 |
+
},
|
| 723 |
+
"346": {
|
| 724 |
+
"content": "<extra_id_87>",
|
| 725 |
+
"lstrip": false,
|
| 726 |
+
"normalized": false,
|
| 727 |
+
"rstrip": false,
|
| 728 |
+
"single_word": false,
|
| 729 |
+
"special": true
|
| 730 |
+
},
|
| 731 |
+
"347": {
|
| 732 |
+
"content": "<extra_id_88>",
|
| 733 |
+
"lstrip": false,
|
| 734 |
+
"normalized": false,
|
| 735 |
+
"rstrip": false,
|
| 736 |
+
"single_word": false,
|
| 737 |
+
"special": true
|
| 738 |
+
},
|
| 739 |
+
"348": {
|
| 740 |
+
"content": "<extra_id_89>",
|
| 741 |
+
"lstrip": false,
|
| 742 |
+
"normalized": false,
|
| 743 |
+
"rstrip": false,
|
| 744 |
+
"single_word": false,
|
| 745 |
+
"special": true
|
| 746 |
+
},
|
| 747 |
+
"349": {
|
| 748 |
+
"content": "<extra_id_90>",
|
| 749 |
+
"lstrip": false,
|
| 750 |
+
"normalized": false,
|
| 751 |
+
"rstrip": false,
|
| 752 |
+
"single_word": false,
|
| 753 |
+
"special": true
|
| 754 |
+
},
|
| 755 |
+
"350": {
|
| 756 |
+
"content": "<extra_id_91>",
|
| 757 |
+
"lstrip": false,
|
| 758 |
+
"normalized": false,
|
| 759 |
+
"rstrip": false,
|
| 760 |
+
"single_word": false,
|
| 761 |
+
"special": true
|
| 762 |
+
},
|
| 763 |
+
"351": {
|
| 764 |
+
"content": "<extra_id_92>",
|
| 765 |
+
"lstrip": false,
|
| 766 |
+
"normalized": false,
|
| 767 |
+
"rstrip": false,
|
| 768 |
+
"single_word": false,
|
| 769 |
+
"special": true
|
| 770 |
+
},
|
| 771 |
+
"352": {
|
| 772 |
+
"content": "<extra_id_93>",
|
| 773 |
+
"lstrip": false,
|
| 774 |
+
"normalized": false,
|
| 775 |
+
"rstrip": false,
|
| 776 |
+
"single_word": false,
|
| 777 |
+
"special": true
|
| 778 |
+
},
|
| 779 |
+
"353": {
|
| 780 |
+
"content": "<extra_id_94>",
|
| 781 |
+
"lstrip": false,
|
| 782 |
+
"normalized": false,
|
| 783 |
+
"rstrip": false,
|
| 784 |
+
"single_word": false,
|
| 785 |
+
"special": true
|
| 786 |
+
},
|
| 787 |
+
"354": {
|
| 788 |
+
"content": "<extra_id_95>",
|
| 789 |
+
"lstrip": false,
|
| 790 |
+
"normalized": false,
|
| 791 |
+
"rstrip": false,
|
| 792 |
+
"single_word": false,
|
| 793 |
+
"special": true
|
| 794 |
+
},
|
| 795 |
+
"355": {
|
| 796 |
+
"content": "<extra_id_96>",
|
| 797 |
+
"lstrip": false,
|
| 798 |
+
"normalized": false,
|
| 799 |
+
"rstrip": false,
|
| 800 |
+
"single_word": false,
|
| 801 |
+
"special": true
|
| 802 |
+
},
|
| 803 |
+
"356": {
|
| 804 |
+
"content": "<extra_id_97>",
|
| 805 |
+
"lstrip": false,
|
| 806 |
+
"normalized": false,
|
| 807 |
+
"rstrip": false,
|
| 808 |
+
"single_word": false,
|
| 809 |
+
"special": true
|
| 810 |
+
},
|
| 811 |
+
"357": {
|
| 812 |
+
"content": "<extra_id_98>",
|
| 813 |
+
"lstrip": false,
|
| 814 |
+
"normalized": false,
|
| 815 |
+
"rstrip": false,
|
| 816 |
+
"single_word": false,
|
| 817 |
+
"special": true
|
| 818 |
+
},
|
| 819 |
+
"358": {
|
| 820 |
+
"content": "<extra_id_99>",
|
| 821 |
+
"lstrip": false,
|
| 822 |
+
"normalized": false,
|
| 823 |
+
"rstrip": false,
|
| 824 |
+
"single_word": false,
|
| 825 |
+
"special": true
|
| 826 |
+
},
|
| 827 |
+
"359": {
|
| 828 |
+
"content": "<extra_id_100>",
|
| 829 |
+
"lstrip": false,
|
| 830 |
+
"normalized": false,
|
| 831 |
+
"rstrip": false,
|
| 832 |
+
"single_word": false,
|
| 833 |
+
"special": true
|
| 834 |
+
},
|
| 835 |
+
"360": {
|
| 836 |
+
"content": "<extra_id_101>",
|
| 837 |
+
"lstrip": false,
|
| 838 |
+
"normalized": false,
|
| 839 |
+
"rstrip": false,
|
| 840 |
+
"single_word": false,
|
| 841 |
+
"special": true
|
| 842 |
+
},
|
| 843 |
+
"361": {
|
| 844 |
+
"content": "<extra_id_102>",
|
| 845 |
+
"lstrip": false,
|
| 846 |
+
"normalized": false,
|
| 847 |
+
"rstrip": false,
|
| 848 |
+
"single_word": false,
|
| 849 |
+
"special": true
|
| 850 |
+
},
|
| 851 |
+
"362": {
|
| 852 |
+
"content": "<extra_id_103>",
|
| 853 |
+
"lstrip": false,
|
| 854 |
+
"normalized": false,
|
| 855 |
+
"rstrip": false,
|
| 856 |
+
"single_word": false,
|
| 857 |
+
"special": true
|
| 858 |
+
},
|
| 859 |
+
"363": {
|
| 860 |
+
"content": "<extra_id_104>",
|
| 861 |
+
"lstrip": false,
|
| 862 |
+
"normalized": false,
|
| 863 |
+
"rstrip": false,
|
| 864 |
+
"single_word": false,
|
| 865 |
+
"special": true
|
| 866 |
+
},
|
| 867 |
+
"364": {
|
| 868 |
+
"content": "<extra_id_105>",
|
| 869 |
+
"lstrip": false,
|
| 870 |
+
"normalized": false,
|
| 871 |
+
"rstrip": false,
|
| 872 |
+
"single_word": false,
|
| 873 |
+
"special": true
|
| 874 |
+
},
|
| 875 |
+
"365": {
|
| 876 |
+
"content": "<extra_id_106>",
|
| 877 |
+
"lstrip": false,
|
| 878 |
+
"normalized": false,
|
| 879 |
+
"rstrip": false,
|
| 880 |
+
"single_word": false,
|
| 881 |
+
"special": true
|
| 882 |
+
},
|
| 883 |
+
"366": {
|
| 884 |
+
"content": "<extra_id_107>",
|
| 885 |
+
"lstrip": false,
|
| 886 |
+
"normalized": false,
|
| 887 |
+
"rstrip": false,
|
| 888 |
+
"single_word": false,
|
| 889 |
+
"special": true
|
| 890 |
+
},
|
| 891 |
+
"367": {
|
| 892 |
+
"content": "<extra_id_108>",
|
| 893 |
+
"lstrip": false,
|
| 894 |
+
"normalized": false,
|
| 895 |
+
"rstrip": false,
|
| 896 |
+
"single_word": false,
|
| 897 |
+
"special": true
|
| 898 |
+
},
|
| 899 |
+
"368": {
|
| 900 |
+
"content": "<extra_id_109>",
|
| 901 |
+
"lstrip": false,
|
| 902 |
+
"normalized": false,
|
| 903 |
+
"rstrip": false,
|
| 904 |
+
"single_word": false,
|
| 905 |
+
"special": true
|
| 906 |
+
},
|
| 907 |
+
"369": {
|
| 908 |
+
"content": "<extra_id_110>",
|
| 909 |
+
"lstrip": false,
|
| 910 |
+
"normalized": false,
|
| 911 |
+
"rstrip": false,
|
| 912 |
+
"single_word": false,
|
| 913 |
+
"special": true
|
| 914 |
+
},
|
| 915 |
+
"370": {
|
| 916 |
+
"content": "<extra_id_111>",
|
| 917 |
+
"lstrip": false,
|
| 918 |
+
"normalized": false,
|
| 919 |
+
"rstrip": false,
|
| 920 |
+
"single_word": false,
|
| 921 |
+
"special": true
|
| 922 |
+
},
|
| 923 |
+
"371": {
|
| 924 |
+
"content": "<extra_id_112>",
|
| 925 |
+
"lstrip": false,
|
| 926 |
+
"normalized": false,
|
| 927 |
+
"rstrip": false,
|
| 928 |
+
"single_word": false,
|
| 929 |
+
"special": true
|
| 930 |
+
},
|
| 931 |
+
"372": {
|
| 932 |
+
"content": "<extra_id_113>",
|
| 933 |
+
"lstrip": false,
|
| 934 |
+
"normalized": false,
|
| 935 |
+
"rstrip": false,
|
| 936 |
+
"single_word": false,
|
| 937 |
+
"special": true
|
| 938 |
+
},
|
| 939 |
+
"373": {
|
| 940 |
+
"content": "<extra_id_114>",
|
| 941 |
+
"lstrip": false,
|
| 942 |
+
"normalized": false,
|
| 943 |
+
"rstrip": false,
|
| 944 |
+
"single_word": false,
|
| 945 |
+
"special": true
|
| 946 |
+
},
|
| 947 |
+
"374": {
|
| 948 |
+
"content": "<extra_id_115>",
|
| 949 |
+
"lstrip": false,
|
| 950 |
+
"normalized": false,
|
| 951 |
+
"rstrip": false,
|
| 952 |
+
"single_word": false,
|
| 953 |
+
"special": true
|
| 954 |
+
},
|
| 955 |
+
"375": {
|
| 956 |
+
"content": "<extra_id_116>",
|
| 957 |
+
"lstrip": false,
|
| 958 |
+
"normalized": false,
|
| 959 |
+
"rstrip": false,
|
| 960 |
+
"single_word": false,
|
| 961 |
+
"special": true
|
| 962 |
+
},
|
| 963 |
+
"376": {
|
| 964 |
+
"content": "<extra_id_117>",
|
| 965 |
+
"lstrip": false,
|
| 966 |
+
"normalized": false,
|
| 967 |
+
"rstrip": false,
|
| 968 |
+
"single_word": false,
|
| 969 |
+
"special": true
|
| 970 |
+
},
|
| 971 |
+
"377": {
|
| 972 |
+
"content": "<extra_id_118>",
|
| 973 |
+
"lstrip": false,
|
| 974 |
+
"normalized": false,
|
| 975 |
+
"rstrip": false,
|
| 976 |
+
"single_word": false,
|
| 977 |
+
"special": true
|
| 978 |
+
},
|
| 979 |
+
"378": {
|
| 980 |
+
"content": "<extra_id_119>",
|
| 981 |
+
"lstrip": false,
|
| 982 |
+
"normalized": false,
|
| 983 |
+
"rstrip": false,
|
| 984 |
+
"single_word": false,
|
| 985 |
+
"special": true
|
| 986 |
+
},
|
| 987 |
+
"379": {
|
| 988 |
+
"content": "<extra_id_120>",
|
| 989 |
+
"lstrip": false,
|
| 990 |
+
"normalized": false,
|
| 991 |
+
"rstrip": false,
|
| 992 |
+
"single_word": false,
|
| 993 |
+
"special": true
|
| 994 |
+
},
|
| 995 |
+
"380": {
|
| 996 |
+
"content": "<extra_id_121>",
|
| 997 |
+
"lstrip": false,
|
| 998 |
+
"normalized": false,
|
| 999 |
+
"rstrip": false,
|
| 1000 |
+
"single_word": false,
|
| 1001 |
+
"special": true
|
| 1002 |
+
},
|
| 1003 |
+
"381": {
|
| 1004 |
+
"content": "<extra_id_122>",
|
| 1005 |
+
"lstrip": false,
|
| 1006 |
+
"normalized": false,
|
| 1007 |
+
"rstrip": false,
|
| 1008 |
+
"single_word": false,
|
| 1009 |
+
"special": true
|
| 1010 |
+
},
|
| 1011 |
+
"382": {
|
| 1012 |
+
"content": "<extra_id_123>",
|
| 1013 |
+
"lstrip": false,
|
| 1014 |
+
"normalized": false,
|
| 1015 |
+
"rstrip": false,
|
| 1016 |
+
"single_word": false,
|
| 1017 |
+
"special": true
|
| 1018 |
+
},
|
| 1019 |
+
"383": {
|
| 1020 |
+
"content": "<extra_id_124>",
|
| 1021 |
+
"lstrip": false,
|
| 1022 |
+
"normalized": false,
|
| 1023 |
+
"rstrip": false,
|
| 1024 |
+
"single_word": false,
|
| 1025 |
+
"special": true
|
| 1026 |
+
}
|
| 1027 |
+
},
|
| 1028 |
+
"additional_special_tokens": [
|
| 1029 |
+
"<extra_id_0>",
|
| 1030 |
+
"<extra_id_1>",
|
| 1031 |
+
"<extra_id_2>",
|
| 1032 |
+
"<extra_id_3>",
|
| 1033 |
+
"<extra_id_4>",
|
| 1034 |
+
"<extra_id_5>",
|
| 1035 |
+
"<extra_id_6>",
|
| 1036 |
+
"<extra_id_7>",
|
| 1037 |
+
"<extra_id_8>",
|
| 1038 |
+
"<extra_id_9>",
|
| 1039 |
+
"<extra_id_10>",
|
| 1040 |
+
"<extra_id_11>",
|
| 1041 |
+
"<extra_id_12>",
|
| 1042 |
+
"<extra_id_13>",
|
| 1043 |
+
"<extra_id_14>",
|
| 1044 |
+
"<extra_id_15>",
|
| 1045 |
+
"<extra_id_16>",
|
| 1046 |
+
"<extra_id_17>",
|
| 1047 |
+
"<extra_id_18>",
|
| 1048 |
+
"<extra_id_19>",
|
| 1049 |
+
"<extra_id_20>",
|
| 1050 |
+
"<extra_id_21>",
|
| 1051 |
+
"<extra_id_22>",
|
| 1052 |
+
"<extra_id_23>",
|
| 1053 |
+
"<extra_id_24>",
|
| 1054 |
+
"<extra_id_25>",
|
| 1055 |
+
"<extra_id_26>",
|
| 1056 |
+
"<extra_id_27>",
|
| 1057 |
+
"<extra_id_28>",
|
| 1058 |
+
"<extra_id_29>",
|
| 1059 |
+
"<extra_id_30>",
|
| 1060 |
+
"<extra_id_31>",
|
| 1061 |
+
"<extra_id_32>",
|
| 1062 |
+
"<extra_id_33>",
|
| 1063 |
+
"<extra_id_34>",
|
| 1064 |
+
"<extra_id_35>",
|
| 1065 |
+
"<extra_id_36>",
|
| 1066 |
+
"<extra_id_37>",
|
| 1067 |
+
"<extra_id_38>",
|
| 1068 |
+
"<extra_id_39>",
|
| 1069 |
+
"<extra_id_40>",
|
| 1070 |
+
"<extra_id_41>",
|
| 1071 |
+
"<extra_id_42>",
|
| 1072 |
+
"<extra_id_43>",
|
| 1073 |
+
"<extra_id_44>",
|
| 1074 |
+
"<extra_id_45>",
|
| 1075 |
+
"<extra_id_46>",
|
| 1076 |
+
"<extra_id_47>",
|
| 1077 |
+
"<extra_id_48>",
|
| 1078 |
+
"<extra_id_49>",
|
| 1079 |
+
"<extra_id_50>",
|
| 1080 |
+
"<extra_id_51>",
|
| 1081 |
+
"<extra_id_52>",
|
| 1082 |
+
"<extra_id_53>",
|
| 1083 |
+
"<extra_id_54>",
|
| 1084 |
+
"<extra_id_55>",
|
| 1085 |
+
"<extra_id_56>",
|
| 1086 |
+
"<extra_id_57>",
|
| 1087 |
+
"<extra_id_58>",
|
| 1088 |
+
"<extra_id_59>",
|
| 1089 |
+
"<extra_id_60>",
|
| 1090 |
+
"<extra_id_61>",
|
| 1091 |
+
"<extra_id_62>",
|
| 1092 |
+
"<extra_id_63>",
|
| 1093 |
+
"<extra_id_64>",
|
| 1094 |
+
"<extra_id_65>",
|
| 1095 |
+
"<extra_id_66>",
|
| 1096 |
+
"<extra_id_67>",
|
| 1097 |
+
"<extra_id_68>",
|
| 1098 |
+
"<extra_id_69>",
|
| 1099 |
+
"<extra_id_70>",
|
| 1100 |
+
"<extra_id_71>",
|
| 1101 |
+
"<extra_id_72>",
|
| 1102 |
+
"<extra_id_73>",
|
| 1103 |
+
"<extra_id_74>",
|
| 1104 |
+
"<extra_id_75>",
|
| 1105 |
+
"<extra_id_76>",
|
| 1106 |
+
"<extra_id_77>",
|
| 1107 |
+
"<extra_id_78>",
|
| 1108 |
+
"<extra_id_79>",
|
| 1109 |
+
"<extra_id_80>",
|
| 1110 |
+
"<extra_id_81>",
|
| 1111 |
+
"<extra_id_82>",
|
| 1112 |
+
"<extra_id_83>",
|
| 1113 |
+
"<extra_id_84>",
|
| 1114 |
+
"<extra_id_85>",
|
| 1115 |
+
"<extra_id_86>",
|
| 1116 |
+
"<extra_id_87>",
|
| 1117 |
+
"<extra_id_88>",
|
| 1118 |
+
"<extra_id_89>",
|
| 1119 |
+
"<extra_id_90>",
|
| 1120 |
+
"<extra_id_91>",
|
| 1121 |
+
"<extra_id_92>",
|
| 1122 |
+
"<extra_id_93>",
|
| 1123 |
+
"<extra_id_94>",
|
| 1124 |
+
"<extra_id_95>",
|
| 1125 |
+
"<extra_id_96>",
|
| 1126 |
+
"<extra_id_97>",
|
| 1127 |
+
"<extra_id_98>",
|
| 1128 |
+
"<extra_id_99>",
|
| 1129 |
+
"<extra_id_100>",
|
| 1130 |
+
"<extra_id_101>",
|
| 1131 |
+
"<extra_id_102>",
|
| 1132 |
+
"<extra_id_103>",
|
| 1133 |
+
"<extra_id_104>",
|
| 1134 |
+
"<extra_id_105>",
|
| 1135 |
+
"<extra_id_106>",
|
| 1136 |
+
"<extra_id_107>",
|
| 1137 |
+
"<extra_id_108>",
|
| 1138 |
+
"<extra_id_109>",
|
| 1139 |
+
"<extra_id_110>",
|
| 1140 |
+
"<extra_id_111>",
|
| 1141 |
+
"<extra_id_112>",
|
| 1142 |
+
"<extra_id_113>",
|
| 1143 |
+
"<extra_id_114>",
|
| 1144 |
+
"<extra_id_115>",
|
| 1145 |
+
"<extra_id_116>",
|
| 1146 |
+
"<extra_id_117>",
|
| 1147 |
+
"<extra_id_118>",
|
| 1148 |
+
"<extra_id_119>",
|
| 1149 |
+
"<extra_id_120>",
|
| 1150 |
+
"<extra_id_121>",
|
| 1151 |
+
"<extra_id_122>",
|
| 1152 |
+
"<extra_id_123>",
|
| 1153 |
+
"<extra_id_124>"
|
| 1154 |
+
],
|
| 1155 |
+
"backend": "custom",
|
| 1156 |
+
"eos_token": "</s>",
|
| 1157 |
+
"extra_ids": 0,
|
| 1158 |
+
"extra_special_tokens": [
|
| 1159 |
+
"<extra_id_0>",
|
| 1160 |
+
"<extra_id_1>",
|
| 1161 |
+
"<extra_id_2>",
|
| 1162 |
+
"<extra_id_3>",
|
| 1163 |
+
"<extra_id_4>",
|
| 1164 |
+
"<extra_id_5>",
|
| 1165 |
+
"<extra_id_6>",
|
| 1166 |
+
"<extra_id_7>",
|
| 1167 |
+
"<extra_id_8>",
|
| 1168 |
+
"<extra_id_9>",
|
| 1169 |
+
"<extra_id_10>",
|
| 1170 |
+
"<extra_id_11>",
|
| 1171 |
+
"<extra_id_12>",
|
| 1172 |
+
"<extra_id_13>",
|
| 1173 |
+
"<extra_id_14>",
|
| 1174 |
+
"<extra_id_15>",
|
| 1175 |
+
"<extra_id_16>",
|
| 1176 |
+
"<extra_id_17>",
|
| 1177 |
+
"<extra_id_18>",
|
| 1178 |
+
"<extra_id_19>",
|
| 1179 |
+
"<extra_id_20>",
|
| 1180 |
+
"<extra_id_21>",
|
| 1181 |
+
"<extra_id_22>",
|
| 1182 |
+
"<extra_id_23>",
|
| 1183 |
+
"<extra_id_24>",
|
| 1184 |
+
"<extra_id_25>",
|
| 1185 |
+
"<extra_id_26>",
|
| 1186 |
+
"<extra_id_27>",
|
| 1187 |
+
"<extra_id_28>",
|
| 1188 |
+
"<extra_id_29>",
|
| 1189 |
+
"<extra_id_30>",
|
| 1190 |
+
"<extra_id_31>",
|
| 1191 |
+
"<extra_id_32>",
|
| 1192 |
+
"<extra_id_33>",
|
| 1193 |
+
"<extra_id_34>",
|
| 1194 |
+
"<extra_id_35>",
|
| 1195 |
+
"<extra_id_36>",
|
| 1196 |
+
"<extra_id_37>",
|
| 1197 |
+
"<extra_id_38>",
|
| 1198 |
+
"<extra_id_39>",
|
| 1199 |
+
"<extra_id_40>",
|
| 1200 |
+
"<extra_id_41>",
|
| 1201 |
+
"<extra_id_42>",
|
| 1202 |
+
"<extra_id_43>",
|
| 1203 |
+
"<extra_id_44>",
|
| 1204 |
+
"<extra_id_45>",
|
| 1205 |
+
"<extra_id_46>",
|
| 1206 |
+
"<extra_id_47>",
|
| 1207 |
+
"<extra_id_48>",
|
| 1208 |
+
"<extra_id_49>",
|
| 1209 |
+
"<extra_id_50>",
|
| 1210 |
+
"<extra_id_51>",
|
| 1211 |
+
"<extra_id_52>",
|
| 1212 |
+
"<extra_id_53>",
|
| 1213 |
+
"<extra_id_54>",
|
| 1214 |
+
"<extra_id_55>",
|
| 1215 |
+
"<extra_id_56>",
|
| 1216 |
+
"<extra_id_57>",
|
| 1217 |
+
"<extra_id_58>",
|
| 1218 |
+
"<extra_id_59>",
|
| 1219 |
+
"<extra_id_60>",
|
| 1220 |
+
"<extra_id_61>",
|
| 1221 |
+
"<extra_id_62>",
|
| 1222 |
+
"<extra_id_63>",
|
| 1223 |
+
"<extra_id_64>",
|
| 1224 |
+
"<extra_id_65>",
|
| 1225 |
+
"<extra_id_66>",
|
| 1226 |
+
"<extra_id_67>",
|
| 1227 |
+
"<extra_id_68>",
|
| 1228 |
+
"<extra_id_69>",
|
| 1229 |
+
"<extra_id_70>",
|
| 1230 |
+
"<extra_id_71>",
|
| 1231 |
+
"<extra_id_72>",
|
| 1232 |
+
"<extra_id_73>",
|
| 1233 |
+
"<extra_id_74>",
|
| 1234 |
+
"<extra_id_75>",
|
| 1235 |
+
"<extra_id_76>",
|
| 1236 |
+
"<extra_id_77>",
|
| 1237 |
+
"<extra_id_78>",
|
| 1238 |
+
"<extra_id_79>",
|
| 1239 |
+
"<extra_id_80>",
|
| 1240 |
+
"<extra_id_81>",
|
| 1241 |
+
"<extra_id_82>",
|
| 1242 |
+
"<extra_id_83>",
|
| 1243 |
+
"<extra_id_84>",
|
| 1244 |
+
"<extra_id_85>",
|
| 1245 |
+
"<extra_id_86>",
|
| 1246 |
+
"<extra_id_87>",
|
| 1247 |
+
"<extra_id_88>",
|
| 1248 |
+
"<extra_id_89>",
|
| 1249 |
+
"<extra_id_90>",
|
| 1250 |
+
"<extra_id_91>",
|
| 1251 |
+
"<extra_id_92>",
|
| 1252 |
+
"<extra_id_93>",
|
| 1253 |
+
"<extra_id_94>",
|
| 1254 |
+
"<extra_id_95>",
|
| 1255 |
+
"<extra_id_96>",
|
| 1256 |
+
"<extra_id_97>",
|
| 1257 |
+
"<extra_id_98>",
|
| 1258 |
+
"<extra_id_99>",
|
| 1259 |
+
"<extra_id_100>",
|
| 1260 |
+
"<extra_id_101>",
|
| 1261 |
+
"<extra_id_102>",
|
| 1262 |
+
"<extra_id_103>",
|
| 1263 |
+
"<extra_id_104>",
|
| 1264 |
+
"<extra_id_105>",
|
| 1265 |
+
"<extra_id_106>",
|
| 1266 |
+
"<extra_id_107>",
|
| 1267 |
+
"<extra_id_108>",
|
| 1268 |
+
"<extra_id_109>",
|
| 1269 |
+
"<extra_id_110>",
|
| 1270 |
+
"<extra_id_111>",
|
| 1271 |
+
"<extra_id_112>",
|
| 1272 |
+
"<extra_id_113>",
|
| 1273 |
+
"<extra_id_114>",
|
| 1274 |
+
"<extra_id_115>",
|
| 1275 |
+
"<extra_id_116>",
|
| 1276 |
+
"<extra_id_117>",
|
| 1277 |
+
"<extra_id_118>",
|
| 1278 |
+
"<extra_id_119>",
|
| 1279 |
+
"<extra_id_120>",
|
| 1280 |
+
"<extra_id_121>",
|
| 1281 |
+
"<extra_id_122>",
|
| 1282 |
+
"<extra_id_123>",
|
| 1283 |
+
"<extra_id_124>"
|
| 1284 |
+
],
|
| 1285 |
+
"is_local": false,
|
| 1286 |
+
"local_files_only": false,
|
| 1287 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 1288 |
+
"pad_token": "<pad>",
|
| 1289 |
+
"tokenizer_class": "ByT5Tokenizer",
|
| 1290 |
+
"unk_token": "<unk>"
|
| 1291 |
+
}
|
runs/hf-corrector-mixed-v1-rebalanced/final-model/training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9483641fdd68a02b027422a4d2d6a0662d0fe6e8d6eecbb844050d5e3fe86315
|
| 3 |
+
size 5329
|
runs/hf-corrector-mixed-v1-rebalanced/hf_text_corrector_config.json
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"train_pairs": "runs/hf-corrector-mixed-v1-rebalanced/train_pairs.jsonl",
|
| 3 |
+
"eval_pairs": "runs/hf-corrector-mixed-v1-rebalanced/eval_pairs.jsonl",
|
| 4 |
+
"output_dir": "runs/hf-corrector-mixed-v1-rebalanced",
|
| 5 |
+
"base_model": "google/byt5-small",
|
| 6 |
+
"max_source_length": 256,
|
| 7 |
+
"max_target_length": 256,
|
| 8 |
+
"per_device_train_batch_size": 16,
|
| 9 |
+
"per_device_eval_batch_size": 16,
|
| 10 |
+
"learning_rate": 5e-05,
|
| 11 |
+
"num_train_epochs": 2.0,
|
| 12 |
+
"warmup_steps": 0,
|
| 13 |
+
"logging_steps": 10,
|
| 14 |
+
"save_total_limit": 2,
|
| 15 |
+
"predict_with_generate": true,
|
| 16 |
+
"generation_max_length": 256
|
| 17 |
+
}
|
runs/hf-corrector-mixed-v1-rebalanced/package/PACKAGE.md
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Packaged Text Correction Composite Candidate
|
| 2 |
+
|
| 3 |
+
Model card: `runs/hf-corrector-mixed-v1-rebalanced/package/model-card.json`
|
| 4 |
+
Artifacts: `runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model`
|
| 5 |
+
|
| 6 |
+
Audit before benchmarking:
|
| 7 |
+
|
| 8 |
+
```bash
|
| 9 |
+
ocrtech audit-model runs/hf-corrector-mixed-v1-rebalanced/package/model-card.json
|
| 10 |
+
```
|
runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model/added_tokens.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<extra_id_0>": 259,
|
| 3 |
+
"<extra_id_100>": 359,
|
| 4 |
+
"<extra_id_101>": 360,
|
| 5 |
+
"<extra_id_102>": 361,
|
| 6 |
+
"<extra_id_103>": 362,
|
| 7 |
+
"<extra_id_104>": 363,
|
| 8 |
+
"<extra_id_105>": 364,
|
| 9 |
+
"<extra_id_106>": 365,
|
| 10 |
+
"<extra_id_107>": 366,
|
| 11 |
+
"<extra_id_108>": 367,
|
| 12 |
+
"<extra_id_109>": 368,
|
| 13 |
+
"<extra_id_10>": 269,
|
| 14 |
+
"<extra_id_110>": 369,
|
| 15 |
+
"<extra_id_111>": 370,
|
| 16 |
+
"<extra_id_112>": 371,
|
| 17 |
+
"<extra_id_113>": 372,
|
| 18 |
+
"<extra_id_114>": 373,
|
| 19 |
+
"<extra_id_115>": 374,
|
| 20 |
+
"<extra_id_116>": 375,
|
| 21 |
+
"<extra_id_117>": 376,
|
| 22 |
+
"<extra_id_118>": 377,
|
| 23 |
+
"<extra_id_119>": 378,
|
| 24 |
+
"<extra_id_11>": 270,
|
| 25 |
+
"<extra_id_120>": 379,
|
| 26 |
+
"<extra_id_121>": 380,
|
| 27 |
+
"<extra_id_122>": 381,
|
| 28 |
+
"<extra_id_123>": 382,
|
| 29 |
+
"<extra_id_124>": 383,
|
| 30 |
+
"<extra_id_12>": 271,
|
| 31 |
+
"<extra_id_13>": 272,
|
| 32 |
+
"<extra_id_14>": 273,
|
| 33 |
+
"<extra_id_15>": 274,
|
| 34 |
+
"<extra_id_16>": 275,
|
| 35 |
+
"<extra_id_17>": 276,
|
| 36 |
+
"<extra_id_18>": 277,
|
| 37 |
+
"<extra_id_19>": 278,
|
| 38 |
+
"<extra_id_1>": 260,
|
| 39 |
+
"<extra_id_20>": 279,
|
| 40 |
+
"<extra_id_21>": 280,
|
| 41 |
+
"<extra_id_22>": 281,
|
| 42 |
+
"<extra_id_23>": 282,
|
| 43 |
+
"<extra_id_24>": 283,
|
| 44 |
+
"<extra_id_25>": 284,
|
| 45 |
+
"<extra_id_26>": 285,
|
| 46 |
+
"<extra_id_27>": 286,
|
| 47 |
+
"<extra_id_28>": 287,
|
| 48 |
+
"<extra_id_29>": 288,
|
| 49 |
+
"<extra_id_2>": 261,
|
| 50 |
+
"<extra_id_30>": 289,
|
| 51 |
+
"<extra_id_31>": 290,
|
| 52 |
+
"<extra_id_32>": 291,
|
| 53 |
+
"<extra_id_33>": 292,
|
| 54 |
+
"<extra_id_34>": 293,
|
| 55 |
+
"<extra_id_35>": 294,
|
| 56 |
+
"<extra_id_36>": 295,
|
| 57 |
+
"<extra_id_37>": 296,
|
| 58 |
+
"<extra_id_38>": 297,
|
| 59 |
+
"<extra_id_39>": 298,
|
| 60 |
+
"<extra_id_3>": 262,
|
| 61 |
+
"<extra_id_40>": 299,
|
| 62 |
+
"<extra_id_41>": 300,
|
| 63 |
+
"<extra_id_42>": 301,
|
| 64 |
+
"<extra_id_43>": 302,
|
| 65 |
+
"<extra_id_44>": 303,
|
| 66 |
+
"<extra_id_45>": 304,
|
| 67 |
+
"<extra_id_46>": 305,
|
| 68 |
+
"<extra_id_47>": 306,
|
| 69 |
+
"<extra_id_48>": 307,
|
| 70 |
+
"<extra_id_49>": 308,
|
| 71 |
+
"<extra_id_4>": 263,
|
| 72 |
+
"<extra_id_50>": 309,
|
| 73 |
+
"<extra_id_51>": 310,
|
| 74 |
+
"<extra_id_52>": 311,
|
| 75 |
+
"<extra_id_53>": 312,
|
| 76 |
+
"<extra_id_54>": 313,
|
| 77 |
+
"<extra_id_55>": 314,
|
| 78 |
+
"<extra_id_56>": 315,
|
| 79 |
+
"<extra_id_57>": 316,
|
| 80 |
+
"<extra_id_58>": 317,
|
| 81 |
+
"<extra_id_59>": 318,
|
| 82 |
+
"<extra_id_5>": 264,
|
| 83 |
+
"<extra_id_60>": 319,
|
| 84 |
+
"<extra_id_61>": 320,
|
| 85 |
+
"<extra_id_62>": 321,
|
| 86 |
+
"<extra_id_63>": 322,
|
| 87 |
+
"<extra_id_64>": 323,
|
| 88 |
+
"<extra_id_65>": 324,
|
| 89 |
+
"<extra_id_66>": 325,
|
| 90 |
+
"<extra_id_67>": 326,
|
| 91 |
+
"<extra_id_68>": 327,
|
| 92 |
+
"<extra_id_69>": 328,
|
| 93 |
+
"<extra_id_6>": 265,
|
| 94 |
+
"<extra_id_70>": 329,
|
| 95 |
+
"<extra_id_71>": 330,
|
| 96 |
+
"<extra_id_72>": 331,
|
| 97 |
+
"<extra_id_73>": 332,
|
| 98 |
+
"<extra_id_74>": 333,
|
| 99 |
+
"<extra_id_75>": 334,
|
| 100 |
+
"<extra_id_76>": 335,
|
| 101 |
+
"<extra_id_77>": 336,
|
| 102 |
+
"<extra_id_78>": 337,
|
| 103 |
+
"<extra_id_79>": 338,
|
| 104 |
+
"<extra_id_7>": 266,
|
| 105 |
+
"<extra_id_80>": 339,
|
| 106 |
+
"<extra_id_81>": 340,
|
| 107 |
+
"<extra_id_82>": 341,
|
| 108 |
+
"<extra_id_83>": 342,
|
| 109 |
+
"<extra_id_84>": 343,
|
| 110 |
+
"<extra_id_85>": 344,
|
| 111 |
+
"<extra_id_86>": 345,
|
| 112 |
+
"<extra_id_87>": 346,
|
| 113 |
+
"<extra_id_88>": 347,
|
| 114 |
+
"<extra_id_89>": 348,
|
| 115 |
+
"<extra_id_8>": 267,
|
| 116 |
+
"<extra_id_90>": 349,
|
| 117 |
+
"<extra_id_91>": 350,
|
| 118 |
+
"<extra_id_92>": 351,
|
| 119 |
+
"<extra_id_93>": 352,
|
| 120 |
+
"<extra_id_94>": 353,
|
| 121 |
+
"<extra_id_95>": 354,
|
| 122 |
+
"<extra_id_96>": 355,
|
| 123 |
+
"<extra_id_97>": 356,
|
| 124 |
+
"<extra_id_98>": 357,
|
| 125 |
+
"<extra_id_99>": 358,
|
| 126 |
+
"<extra_id_9>": 268
|
| 127 |
+
}
|
runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model/config.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"T5ForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"classifier_dropout": 0.0,
|
| 6 |
+
"d_ff": 3584,
|
| 7 |
+
"d_kv": 64,
|
| 8 |
+
"d_model": 1472,
|
| 9 |
+
"decoder_start_token_id": 0,
|
| 10 |
+
"dense_act_fn": "gelu_new",
|
| 11 |
+
"dropout_rate": 0.1,
|
| 12 |
+
"dtype": "float32",
|
| 13 |
+
"eos_token_id": 1,
|
| 14 |
+
"feed_forward_proj": "gated-gelu",
|
| 15 |
+
"gradient_checkpointing": false,
|
| 16 |
+
"initializer_factor": 1.0,
|
| 17 |
+
"is_decoder": false,
|
| 18 |
+
"is_encoder_decoder": true,
|
| 19 |
+
"is_gated_act": true,
|
| 20 |
+
"layer_norm_epsilon": 1e-06,
|
| 21 |
+
"model_type": "t5",
|
| 22 |
+
"num_decoder_layers": 4,
|
| 23 |
+
"num_heads": 6,
|
| 24 |
+
"num_layers": 12,
|
| 25 |
+
"pad_token_id": 0,
|
| 26 |
+
"relative_attention_max_distance": 128,
|
| 27 |
+
"relative_attention_num_buckets": 32,
|
| 28 |
+
"scale_decoder_outputs": false,
|
| 29 |
+
"tie_word_embeddings": true,
|
| 30 |
+
"tokenizer_class": "ByT5Tokenizer",
|
| 31 |
+
"transformers_version": "5.12.0",
|
| 32 |
+
"use_cache": false,
|
| 33 |
+
"vocab_size": 384
|
| 34 |
+
}
|
runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model/generation_config.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": false,
|
| 3 |
+
"assistant_confidence_threshold": 0.4,
|
| 4 |
+
"assistant_lookbehind": 10,
|
| 5 |
+
"decoder_start_token_id": 0,
|
| 6 |
+
"diversity_penalty": 0.0,
|
| 7 |
+
"do_sample": false,
|
| 8 |
+
"early_stopping": false,
|
| 9 |
+
"encoder_no_repeat_ngram_size": 0,
|
| 10 |
+
"encoder_repetition_penalty": 1.0,
|
| 11 |
+
"eos_token_id": [
|
| 12 |
+
1
|
| 13 |
+
],
|
| 14 |
+
"epsilon_cutoff": 0.0,
|
| 15 |
+
"eta_cutoff": 0.0,
|
| 16 |
+
"length_penalty": 1.0,
|
| 17 |
+
"max_length": 256,
|
| 18 |
+
"min_length": 0,
|
| 19 |
+
"no_repeat_ngram_size": 0,
|
| 20 |
+
"num_assistant_tokens": 20,
|
| 21 |
+
"num_assistant_tokens_schedule": "constant",
|
| 22 |
+
"num_beam_groups": 1,
|
| 23 |
+
"num_beams": 1,
|
| 24 |
+
"num_return_sequences": 1,
|
| 25 |
+
"output_scores": false,
|
| 26 |
+
"pad_token_id": 0,
|
| 27 |
+
"remove_invalid_values": false,
|
| 28 |
+
"repetition_penalty": 1.0,
|
| 29 |
+
"return_dict_in_generate": false,
|
| 30 |
+
"target_lookbehind": 10,
|
| 31 |
+
"temperature": 1.0,
|
| 32 |
+
"top_k": 50,
|
| 33 |
+
"top_p": 1.0,
|
| 34 |
+
"transformers_version": "5.12.0",
|
| 35 |
+
"typical_p": 1.0,
|
| 36 |
+
"use_cache": true
|
| 37 |
+
}
|
runs/hf-corrector-mixed-v1-rebalanced/package/model-card.json
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_id": "ocrtech-tesseract-corrected-mixed-v1",
|
| 3 |
+
"backend": "text_correction_composite",
|
| 4 |
+
"base_model": "google/byt5-small",
|
| 5 |
+
"created_at_utc": "2026-06-14T03:08:28.115455+00:00",
|
| 6 |
+
"artifacts": [
|
| 7 |
+
{
|
| 8 |
+
"path": "artifacts/model",
|
| 9 |
+
"required": true,
|
| 10 |
+
"sha256": null,
|
| 11 |
+
"kind": "directory"
|
| 12 |
+
}
|
| 13 |
+
],
|
| 14 |
+
"backend_kwargs": {
|
| 15 |
+
"base_engine": "tesseract",
|
| 16 |
+
"base_engine_kwargs": {},
|
| 17 |
+
"model_dir": "artifacts/model",
|
| 18 |
+
"tokenizer_dir": "artifacts/model",
|
| 19 |
+
"max_new_tokens": 256
|
| 20 |
+
},
|
| 21 |
+
"provenance": {
|
| 22 |
+
"train_manifest": {
|
| 23 |
+
"path": "data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl",
|
| 24 |
+
"sha256": "86325bc94e5f707329e8735c68afee27d32f7b3ac6186386325f46dc296184c6"
|
| 25 |
+
},
|
| 26 |
+
"eval_manifest": {
|
| 27 |
+
"path": "data/splits-mixed-v1/eval-with-refs.jsonl",
|
| 28 |
+
"sha256": "314c11e25aa6caccb2a801a6c5a6a3f7222eeb59ae76e77f12d6dd72ff4a90ae"
|
| 29 |
+
}
|
| 30 |
+
},
|
| 31 |
+
"metrics": {
|
| 32 |
+
"report": {
|
| 33 |
+
"path": "runs/hf-corrector-mixed-v1-rebalanced/training_metrics.json",
|
| 34 |
+
"sha256": "491416430e31605548e3e07a9dc3634807e4efaa7e6fae11872b52a9af5bf873"
|
| 35 |
+
}
|
| 36 |
+
},
|
| 37 |
+
"notes": null
|
| 38 |
+
}
|
runs/hf-corrector-mixed-v1-rebalanced/train_pairs.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
runs/hf-corrector-mixed-v1-rebalanced/training_metrics.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"eval_loss": 0.8401811718940735,
|
| 3 |
+
"eval_cer": 7.067461343195775,
|
| 4 |
+
"eval_wer": 3.334916864181571,
|
| 5 |
+
"eval_exact_line_accuracy": 0.0,
|
| 6 |
+
"eval_runtime": 125.4858,
|
| 7 |
+
"eval_samples_per_second": 1.355,
|
| 8 |
+
"eval_steps_per_second": 0.088,
|
| 9 |
+
"epoch": 2.0
|
| 10 |
+
}
|