prosperious commited on
Commit
1c43956
·
verified ·
1 Parent(s): 5318548

Add 51 files (.gitignore ...)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitignore +13 -0
  2. .ignore +9 -0
  3. configs/claim-review-sota-v1.json +111 -0
  4. configs/nepali-table-name-lexicon-v1.txt +10 -0
  5. configs/validation-complex-table-surya.json +57 -0
  6. configs/validation-hard15-surya.json +59 -0
  7. configs/validation-internal-sota-v1.json +120 -0
  8. configs/validation-paddle-hard50-tesseract.json +52 -0
  9. configs/validation-public-omnidocbench-sota-v1.json +60 -0
  10. configs/validation-public-omnidocbench-surya.json +46 -0
  11. configs/validation-public-real5-sota-v1.json +67 -0
  12. configs/validation-table-ledger-surya.json +39 -0
  13. docs/CLAIM_REVIEW.md +320 -0
  14. docs/DATASETS.md +381 -0
  15. docs/EVAL_PACK.md +226 -0
  16. docs/EXPERIMENTS.md +1034 -0
  17. docs/FONT_ASSETS.md +29 -0
  18. docs/GORKHAPATRA_LANGUAGE_PAGE_ALIGNMENT.md +172 -0
  19. docs/MODELS.md +490 -0
  20. docs/NEPAL_MULTILINGUAL_OCR_GOAL.md +371 -0
  21. docs/PAPER_NOTES.md +0 -0
  22. docs/PAPER_RESULTS.md +854 -0
  23. docs/PREFLIGHT.md +147 -0
  24. docs/REMOTE_TRAINING.md +1404 -0
  25. docs/VALIDATION.md +638 -0
  26. examples/sample-page.txt +7 -0
  27. languages/nepal-v1.json +370 -0
  28. pyproject.toml +61 -0
  29. runs/corrector-cfilt/README.md +5 -0
  30. runs/corrector-cfilt/correction_pairs.jsonl +5 -0
  31. runs/corrector-cfilt/corrector_config.json +10 -0
  32. runs/corrector-mixed-v1-rebalanced/tesseract_pairs.jsonl +0 -0
  33. runs/hf-corrector-mixed-v1-rebalanced/README.md +18 -0
  34. runs/hf-corrector-mixed-v1-rebalanced/correction_pairs.jsonl +0 -0
  35. runs/hf-corrector-mixed-v1-rebalanced/corrector_config.json +10 -0
  36. runs/hf-corrector-mixed-v1-rebalanced/eval_pairs.jsonl +170 -0
  37. runs/hf-corrector-mixed-v1-rebalanced/final-model/added_tokens.json +127 -0
  38. runs/hf-corrector-mixed-v1-rebalanced/final-model/config.json +34 -0
  39. runs/hf-corrector-mixed-v1-rebalanced/final-model/generation_config.json +37 -0
  40. runs/hf-corrector-mixed-v1-rebalanced/final-model/model.safetensors +3 -0
  41. runs/hf-corrector-mixed-v1-rebalanced/final-model/tokenizer_config.json +1291 -0
  42. runs/hf-corrector-mixed-v1-rebalanced/final-model/training_args.bin +3 -0
  43. runs/hf-corrector-mixed-v1-rebalanced/hf_text_corrector_config.json +17 -0
  44. runs/hf-corrector-mixed-v1-rebalanced/package/PACKAGE.md +10 -0
  45. runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model/added_tokens.json +127 -0
  46. runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model/config.json +34 -0
  47. runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model/generation_config.json +37 -0
  48. runs/hf-corrector-mixed-v1-rebalanced/package/model-card.json +38 -0
  49. runs/hf-corrector-mixed-v1-rebalanced/train_pairs.jsonl +0 -0
  50. runs/hf-corrector-mixed-v1-rebalanced/training_metrics.json +10 -0
.gitignore ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ .venv/
2
+ __pycache__/
3
+ *.py[cod]
4
+ .pytest_cache/
5
+ .ruff_cache/
6
+ .mypy_cache/
7
+
8
+ data/
9
+ outputs/
10
+ runs/
11
+ *.egg-info/
12
+
13
+ .DS_Store
.ignore ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ .venv/
2
+ __pycache__/
3
+ *.py[cod]
4
+ .pytest_cache/
5
+
6
+ data/
7
+ outputs/
8
+ runs/
9
+ *.egg-info/
configs/claim-review-sota-v1.json ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "require_shared_candidate_model": true,
3
+ "require_fallback_model_present": true,
4
+ "require_candidate_fallback_provenance": true,
5
+ "require_shared_fallback_model": true,
6
+ "require_model_audit_passed": true,
7
+ "require_fallback_model_audit_passed": true,
8
+ "require_trained_recognizer": true,
9
+ "require_model_admission": true,
10
+ "required_experiments": [
11
+ {
12
+ "name": "internal-nepali-english-document-pack",
13
+ "datasets": ["ocrtech-eval-pack"],
14
+ "require_validation_status": "validated",
15
+ "required_validation_config": "configs/validation-internal-sota-v1.json",
16
+ "require_baselines": ["candidate", "surya", "tesseract", "stock-paddle", "glm-ocr", "paddleocr-vl"],
17
+ "required_metrics": [
18
+ "cer",
19
+ "wer",
20
+ "document_reconstruction_similarity",
21
+ "table_cell_f1",
22
+ "reading_order_pair_accuracy",
23
+ "figure_detection_f1",
24
+ "figure_caption_cer"
25
+ ],
26
+ "required_metric_slices": {
27
+ "cer": ["nepali", "english"],
28
+ "wer": ["nepali", "english"],
29
+ "document_reconstruction_similarity": ["table", "reading_order", "figure"],
30
+ "table_cell_f1": ["table"],
31
+ "reading_order_pair_accuracy": ["reading_order"],
32
+ "figure_detection_f1": ["figure"],
33
+ "figure_caption_cer": ["figure"]
34
+ },
35
+ "required_output_artifacts": ["document_json", "document_md", "quality_json"],
36
+ "required_output_artifact_slices": {
37
+ "table_csv_html": ["table"],
38
+ "figure_metadata": ["figure"],
39
+ "figure_images": ["figure"]
40
+ },
41
+ "required_validation_gates": [
42
+ {"baseline": "tesseract", "metric": "cer", "slices": ["nepali"]},
43
+ {"baseline": "tesseract", "metric": "wer", "slices": ["nepali"]},
44
+ {"baseline": "stock-paddle", "metric": "cer", "slices": ["nepali"]},
45
+ {"baseline": "stock-paddle", "metric": "cer", "slices": ["english"]},
46
+ {"baseline": "stock-paddle", "metric": "wer", "slices": ["english"]},
47
+ {"baseline": "glm-ocr", "metric": "cer", "slices": ["nepali"]},
48
+ {"baseline": "surya", "metric": "table_cell_f1", "slices": ["table"]},
49
+ {"baseline": "surya", "metric": "figure_detection_f1", "slices": ["figure"]},
50
+ {"baseline": "surya", "metric": "figure_caption_cer", "slices": ["figure"]},
51
+ {"baseline": "paddleocr-vl", "metric": "reading_order_pair_accuracy", "slices": ["reading_order"]},
52
+ {"baseline": "paddleocr-vl", "metric": "document_reconstruction_similarity", "slices": ["figure"]}
53
+ ],
54
+ "require_candidate_fallback_metrics": true,
55
+ "max_candidate_fallback_trigger_rate": 0.35,
56
+ "max_candidate_fallback_failed_rate": 0.0,
57
+ "required_slices": ["nepali", "english", "table", "reading_order", "figure"]
58
+ },
59
+ {
60
+ "name": "public-omnidocbench",
61
+ "datasets": ["opendatalab/OmniDocBench"],
62
+ "require_validation_status": "validated",
63
+ "required_validation_config": "configs/validation-public-omnidocbench-sota-v1.json",
64
+ "require_baselines": ["candidate", "surya", "paddleocr-vl"],
65
+ "required_metrics": ["cer", "document_reconstruction_similarity", "table_cell_f1", "table_structure_similarity", "reading_order_pair_accuracy"],
66
+ "required_metric_slices": {
67
+ "table_cell_f1": ["document_parsing"],
68
+ "table_structure_similarity": ["document_parsing"],
69
+ "reading_order_pair_accuracy": ["document_parsing"],
70
+ "document_reconstruction_similarity": ["document_parsing"]
71
+ },
72
+ "required_output_artifacts": ["document_json", "document_md", "quality_json"],
73
+ "required_output_artifact_slices": {
74
+ "table_csv_html": ["table"]
75
+ },
76
+ "required_validation_gates": [
77
+ {"baseline": "surya", "metric": "cer", "slices": ["public", "document_parsing"]},
78
+ {"baseline": "surya", "metric": "table_cell_f1", "slices": ["public", "table"]},
79
+ {"baseline": "paddleocr-vl", "metric": "document_reconstruction_similarity", "slices": ["public", "document_parsing"]}
80
+ ],
81
+ "require_candidate_fallback_metrics": true,
82
+ "max_candidate_fallback_trigger_rate": 0.35,
83
+ "max_candidate_fallback_failed_rate": 0.0,
84
+ "required_slices": ["public", "document_parsing"]
85
+ },
86
+ {
87
+ "name": "public-real5-omnidocbench-or-olmocr",
88
+ "eval_manifest_patterns": ["*Real5-OmniDocBench*", "*olmOCR-bench*"],
89
+ "require_validation_status": "validated",
90
+ "required_validation_config": "configs/validation-public-real5-sota-v1.json",
91
+ "require_baselines": ["candidate", "surya", "glm-ocr", "paddleocr-vl"],
92
+ "required_metrics": ["cer", "wer", "document_reconstruction_similarity", "reading_order_pair_accuracy"],
93
+ "required_metric_slices": {
94
+ "document_reconstruction_similarity": ["public"],
95
+ "reading_order_pair_accuracy": ["public"]
96
+ },
97
+ "required_output_artifacts": ["document_json", "document_md", "quality_json"],
98
+ "required_validation_gates": [
99
+ {"baseline": "surya", "metric": "cer", "slices": ["public"]},
100
+ {"baseline": "surya", "metric": "wer", "slices": ["public"]},
101
+ {"baseline": "glm-ocr", "metric": "cer", "slices": ["public"]},
102
+ {"baseline": "glm-ocr", "metric": "wer", "slices": ["public"]},
103
+ {"baseline": "paddleocr-vl", "metric": "document_reconstruction_similarity", "slices": ["public"]}
104
+ ],
105
+ "require_candidate_fallback_metrics": true,
106
+ "max_candidate_fallback_trigger_rate": 0.35,
107
+ "max_candidate_fallback_failed_rate": 0.0,
108
+ "required_slices": ["public"]
109
+ }
110
+ ]
111
+ }
configs/nepali-table-name-lexicon-v1.txt ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ # Small Nepali table-name lexicon for guarded OCR correction.
2
+ # Use only with provenance in the model card; this is not a general language model.
3
+ किरण
4
+ माया
5
+ राम
6
+ सीता
7
+ हरी
8
+ गीता
9
+ सोम
10
+ रीता
configs/validation-complex-table-surya.json ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "candidate": "candidate",
3
+ "min_samples": 40,
4
+ "required_slices": {
5
+ "complex_table": 40,
6
+ "table": 40,
7
+ "nepali": 40,
8
+ "english": 40
9
+ },
10
+ "bootstrap_iterations": 1000,
11
+ "bootstrap_confidence": 0.95,
12
+ "random_seed": 13,
13
+ "gates": [
14
+ {
15
+ "baseline": "surya",
16
+ "metric": "table_cell_f1",
17
+ "direction": "higher",
18
+ "slices": ["complex_table", "table"],
19
+ "min_pairs": 40,
20
+ "min_absolute_margin": 0.005
21
+ },
22
+ {
23
+ "baseline": "surya",
24
+ "metric": "table_exact_match",
25
+ "direction": "higher",
26
+ "slices": ["complex_table", "table"],
27
+ "min_pairs": 40,
28
+ "min_absolute_margin": 0.0,
29
+ "require_ci_above_zero": false
30
+ },
31
+ {
32
+ "baseline": "surya",
33
+ "metric": "table_structure_similarity",
34
+ "direction": "higher",
35
+ "slices": ["complex_table", "table"],
36
+ "min_pairs": 40,
37
+ "min_absolute_margin": 0.0,
38
+ "require_ci_above_zero": false
39
+ },
40
+ {
41
+ "baseline": "surya",
42
+ "metric": "table_teds_structure",
43
+ "direction": "higher",
44
+ "slices": ["complex_table", "table"],
45
+ "min_pairs": 40,
46
+ "min_absolute_margin": 0.0
47
+ },
48
+ {
49
+ "baseline": "surya",
50
+ "metric": "table_teds_text",
51
+ "direction": "higher",
52
+ "slices": ["complex_table", "table"],
53
+ "min_pairs": 40,
54
+ "min_absolute_margin": 0.005
55
+ }
56
+ ]
57
+ }
configs/validation-hard15-surya.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "candidate": "candidate",
3
+ "min_samples": 15,
4
+ "required_slices": {
5
+ "hard_eval": 15,
6
+ "table": 8,
7
+ "figure": 7,
8
+ "nepali": 15,
9
+ "english": 15
10
+ },
11
+ "bootstrap_iterations": 1000,
12
+ "bootstrap_confidence": 0.95,
13
+ "random_seed": 13,
14
+ "gates": [
15
+ {
16
+ "baseline": "surya",
17
+ "metric": "cer",
18
+ "direction": "lower",
19
+ "slices": ["hard_eval"],
20
+ "min_pairs": 15,
21
+ "min_absolute_margin": 0.02,
22
+ "max_p_value": 0.05
23
+ },
24
+ {
25
+ "baseline": "surya",
26
+ "metric": "cer",
27
+ "direction": "lower",
28
+ "slices": ["hard_eval", "table"],
29
+ "min_pairs": 8,
30
+ "min_absolute_margin": 0.02,
31
+ "max_p_value": 0.05
32
+ },
33
+ {
34
+ "baseline": "surya",
35
+ "metric": "cer",
36
+ "direction": "lower",
37
+ "slices": ["hard_eval", "figure"],
38
+ "min_pairs": 7,
39
+ "min_absolute_margin": 0.02,
40
+ "max_p_value": 0.05
41
+ },
42
+ {
43
+ "baseline": "surya",
44
+ "metric": "table_cell_f1",
45
+ "direction": "higher",
46
+ "slices": ["hard_eval", "table"],
47
+ "min_pairs": 8,
48
+ "min_absolute_margin": 0.01
49
+ },
50
+ {
51
+ "baseline": "surya",
52
+ "metric": "figure_caption_cer",
53
+ "direction": "lower",
54
+ "slices": ["hard_eval", "figure"],
55
+ "min_pairs": 7,
56
+ "min_absolute_margin": 0.01
57
+ }
58
+ ]
59
+ }
configs/validation-internal-sota-v1.json ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "candidate": "candidate",
3
+ "min_samples": 100,
4
+ "required_slices": {
5
+ "nepali": 50,
6
+ "english": 50,
7
+ "table": 20,
8
+ "reading_order": 20,
9
+ "figure": 20
10
+ },
11
+ "bootstrap_iterations": 1000,
12
+ "bootstrap_confidence": 0.95,
13
+ "random_seed": 13,
14
+ "require_full_eval_manifest_coverage": true,
15
+ "gates": [
16
+ {
17
+ "baseline": "tesseract",
18
+ "metric": "cer",
19
+ "direction": "lower",
20
+ "slices": ["nepali"],
21
+ "min_pairs": 50,
22
+ "min_absolute_margin": 0.02,
23
+ "min_pair_win_rate": 0.55,
24
+ "max_p_value": 0.05
25
+ },
26
+ {
27
+ "baseline": "tesseract",
28
+ "metric": "wer",
29
+ "direction": "lower",
30
+ "slices": ["nepali"],
31
+ "min_pairs": 50,
32
+ "min_absolute_margin": 0.01,
33
+ "min_pair_win_rate": 0.55,
34
+ "max_p_value": 0.05
35
+ },
36
+ {
37
+ "baseline": "stock-paddle",
38
+ "metric": "cer",
39
+ "direction": "lower",
40
+ "slices": ["nepali"],
41
+ "min_pairs": 50,
42
+ "min_absolute_margin": 0.01,
43
+ "min_pair_win_rate": 0.55,
44
+ "max_p_value": 0.05
45
+ },
46
+ {
47
+ "baseline": "stock-paddle",
48
+ "metric": "cer",
49
+ "direction": "lower",
50
+ "slices": ["english"],
51
+ "min_pairs": 50,
52
+ "min_absolute_margin": 0.0,
53
+ "require_ci_above_zero": false
54
+ },
55
+ {
56
+ "baseline": "stock-paddle",
57
+ "metric": "wer",
58
+ "direction": "lower",
59
+ "slices": ["english"],
60
+ "min_pairs": 50,
61
+ "min_absolute_margin": 0.0,
62
+ "require_ci_above_zero": false
63
+ },
64
+ {
65
+ "baseline": "glm-ocr",
66
+ "metric": "cer",
67
+ "direction": "lower",
68
+ "slices": ["nepali"],
69
+ "min_pairs": 50,
70
+ "min_absolute_margin": 0.01,
71
+ "min_pair_win_rate": 0.55,
72
+ "max_p_value": 0.05
73
+ },
74
+ {
75
+ "baseline": "surya",
76
+ "metric": "table_cell_f1",
77
+ "direction": "higher",
78
+ "slices": ["table"],
79
+ "min_pairs": 20,
80
+ "min_absolute_margin": 0.01,
81
+ "min_pair_win_rate": 0.55
82
+ },
83
+ {
84
+ "baseline": "paddleocr-vl",
85
+ "metric": "reading_order_pair_accuracy",
86
+ "direction": "higher",
87
+ "slices": ["reading_order"],
88
+ "min_pairs": 20,
89
+ "min_absolute_margin": 0.01,
90
+ "min_pair_win_rate": 0.55
91
+ },
92
+ {
93
+ "baseline": "paddleocr-vl",
94
+ "metric": "document_reconstruction_similarity",
95
+ "direction": "higher",
96
+ "slices": ["figure"],
97
+ "min_pairs": 20,
98
+ "min_absolute_margin": 0.01,
99
+ "min_pair_win_rate": 0.55
100
+ },
101
+ {
102
+ "baseline": "surya",
103
+ "metric": "figure_detection_f1",
104
+ "direction": "higher",
105
+ "slices": ["figure"],
106
+ "min_pairs": 20,
107
+ "min_absolute_margin": 0.0,
108
+ "require_ci_above_zero": false
109
+ },
110
+ {
111
+ "baseline": "surya",
112
+ "metric": "figure_caption_cer",
113
+ "direction": "lower",
114
+ "slices": ["figure"],
115
+ "min_pairs": 20,
116
+ "min_absolute_margin": 0.01,
117
+ "min_pair_win_rate": 0.55
118
+ }
119
+ ]
120
+ }
configs/validation-paddle-hard50-tesseract.json ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "candidate": "candidate",
3
+ "min_samples": 50,
4
+ "required_slices": {
5
+ "paddle_hard_eval": 50,
6
+ "synthetic": 50,
7
+ "english": 38,
8
+ "line_crop": 38,
9
+ "nepali": 12
10
+ },
11
+ "bootstrap_iterations": 1000,
12
+ "bootstrap_confidence": 0.95,
13
+ "random_seed": 13,
14
+ "gates": [
15
+ {
16
+ "baseline": "tesseract",
17
+ "metric": "cer",
18
+ "direction": "lower",
19
+ "slices": ["paddle_hard_eval"],
20
+ "min_pairs": 50,
21
+ "min_absolute_margin": 0.0,
22
+ "require_ci_above_zero": false
23
+ },
24
+ {
25
+ "baseline": "tesseract",
26
+ "metric": "cer",
27
+ "direction": "lower",
28
+ "slices": ["paddle_hard_eval", "english", "line_crop"],
29
+ "min_pairs": 38,
30
+ "min_absolute_margin": 0.0,
31
+ "require_ci_above_zero": false
32
+ },
33
+ {
34
+ "baseline": "tesseract",
35
+ "metric": "wer",
36
+ "direction": "lower",
37
+ "slices": ["paddle_hard_eval", "english", "line_crop"],
38
+ "min_pairs": 38,
39
+ "min_absolute_margin": 0.0,
40
+ "require_ci_above_zero": false
41
+ },
42
+ {
43
+ "baseline": "tesseract",
44
+ "metric": "wer",
45
+ "direction": "lower",
46
+ "slices": ["paddle_hard_eval", "nepali"],
47
+ "min_pairs": 12,
48
+ "min_absolute_margin": 0.05,
49
+ "require_ci_above_zero": false
50
+ }
51
+ ]
52
+ }
configs/validation-public-omnidocbench-sota-v1.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "candidate": "candidate",
3
+ "min_samples": 100,
4
+ "required_slices": {
5
+ "public": 100,
6
+ "document_parsing": 100,
7
+ "table": 40
8
+ },
9
+ "bootstrap_iterations": 1000,
10
+ "bootstrap_confidence": 0.95,
11
+ "random_seed": 13,
12
+ "require_full_eval_manifest_coverage": true,
13
+ "gates": [
14
+ {
15
+ "baseline": "surya",
16
+ "metric": "cer",
17
+ "direction": "lower",
18
+ "slices": ["public", "document_parsing"],
19
+ "min_pairs": 100,
20
+ "min_absolute_margin": 0.005,
21
+ "min_pair_win_rate": 0.55
22
+ },
23
+ {
24
+ "baseline": "surya",
25
+ "metric": "table_cell_f1",
26
+ "direction": "higher",
27
+ "slices": ["public", "table"],
28
+ "min_pairs": 40,
29
+ "min_absolute_margin": 0.005,
30
+ "min_pair_win_rate": 0.55
31
+ },
32
+ {
33
+ "baseline": "surya",
34
+ "metric": "table_structure_similarity",
35
+ "direction": "higher",
36
+ "slices": ["public", "document_parsing"],
37
+ "min_pairs": 40,
38
+ "min_absolute_margin": 0.0,
39
+ "require_ci_above_zero": false
40
+ },
41
+ {
42
+ "baseline": "paddleocr-vl",
43
+ "metric": "document_reconstruction_similarity",
44
+ "direction": "higher",
45
+ "slices": ["public", "document_parsing"],
46
+ "min_pairs": 100,
47
+ "min_absolute_margin": 0.005,
48
+ "min_pair_win_rate": 0.55
49
+ },
50
+ {
51
+ "baseline": "paddleocr-vl",
52
+ "metric": "reading_order_pair_accuracy",
53
+ "direction": "higher",
54
+ "slices": ["public", "document_parsing"],
55
+ "min_pairs": 100,
56
+ "min_absolute_margin": 0.005,
57
+ "min_pair_win_rate": 0.55
58
+ }
59
+ ]
60
+ }
configs/validation-public-omnidocbench-surya.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "candidate": "candidate",
3
+ "min_samples": 100,
4
+ "required_slices": {
5
+ "public": 100,
6
+ "document_parsing": 100,
7
+ "table": 40
8
+ },
9
+ "bootstrap_iterations": 1000,
10
+ "bootstrap_confidence": 0.95,
11
+ "random_seed": 13,
12
+ "gates": [
13
+ {
14
+ "baseline": "surya",
15
+ "metric": "cer",
16
+ "direction": "lower",
17
+ "slices": ["public", "document_parsing"],
18
+ "min_pairs": 100,
19
+ "min_absolute_margin": 0.005
20
+ },
21
+ {
22
+ "baseline": "surya",
23
+ "metric": "table_cell_f1",
24
+ "direction": "higher",
25
+ "slices": ["public", "table"],
26
+ "min_pairs": 40,
27
+ "min_absolute_margin": 0.005
28
+ },
29
+ {
30
+ "baseline": "surya",
31
+ "metric": "table_teds_structure",
32
+ "direction": "higher",
33
+ "slices": ["public", "table"],
34
+ "min_pairs": 40,
35
+ "min_absolute_margin": 0.005
36
+ },
37
+ {
38
+ "baseline": "surya",
39
+ "metric": "table_teds_text",
40
+ "direction": "higher",
41
+ "slices": ["public", "table"],
42
+ "min_pairs": 40,
43
+ "min_absolute_margin": 0.005
44
+ }
45
+ ]
46
+ }
configs/validation-public-real5-sota-v1.json ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "candidate": "candidate",
3
+ "min_samples": 100,
4
+ "required_slices": {
5
+ "public": 100
6
+ },
7
+ "bootstrap_iterations": 1000,
8
+ "bootstrap_confidence": 0.95,
9
+ "random_seed": 13,
10
+ "require_full_eval_manifest_coverage": true,
11
+ "gates": [
12
+ {
13
+ "baseline": "surya",
14
+ "metric": "cer",
15
+ "direction": "lower",
16
+ "slices": ["public"],
17
+ "min_pairs": 100,
18
+ "min_absolute_margin": 0.005,
19
+ "min_pair_win_rate": 0.55
20
+ },
21
+ {
22
+ "baseline": "surya",
23
+ "metric": "wer",
24
+ "direction": "lower",
25
+ "slices": ["public"],
26
+ "min_pairs": 100,
27
+ "min_absolute_margin": 0.005,
28
+ "min_pair_win_rate": 0.55
29
+ },
30
+ {
31
+ "baseline": "glm-ocr",
32
+ "metric": "cer",
33
+ "direction": "lower",
34
+ "slices": ["public"],
35
+ "min_pairs": 100,
36
+ "min_absolute_margin": 0.005,
37
+ "min_pair_win_rate": 0.55
38
+ },
39
+ {
40
+ "baseline": "glm-ocr",
41
+ "metric": "wer",
42
+ "direction": "lower",
43
+ "slices": ["public"],
44
+ "min_pairs": 100,
45
+ "min_absolute_margin": 0.005,
46
+ "min_pair_win_rate": 0.55
47
+ },
48
+ {
49
+ "baseline": "paddleocr-vl",
50
+ "metric": "document_reconstruction_similarity",
51
+ "direction": "higher",
52
+ "slices": ["public"],
53
+ "min_pairs": 100,
54
+ "min_absolute_margin": 0.005,
55
+ "min_pair_win_rate": 0.55
56
+ },
57
+ {
58
+ "baseline": "paddleocr-vl",
59
+ "metric": "reading_order_pair_accuracy",
60
+ "direction": "higher",
61
+ "slices": ["public"],
62
+ "min_pairs": 100,
63
+ "min_absolute_margin": 0.005,
64
+ "min_pair_win_rate": 0.55
65
+ }
66
+ ]
67
+ }
configs/validation-table-ledger-surya.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "candidate": "candidate",
3
+ "min_samples": 40,
4
+ "required_slices": {
5
+ "table": 40,
6
+ "nepali": 40,
7
+ "english": 40
8
+ },
9
+ "bootstrap_iterations": 1000,
10
+ "bootstrap_confidence": 0.95,
11
+ "random_seed": 13,
12
+ "gates": [
13
+ {
14
+ "baseline": "surya",
15
+ "metric": "table_cell_f1",
16
+ "direction": "higher",
17
+ "slices": ["table"],
18
+ "min_pairs": 40,
19
+ "min_absolute_margin": 0.01
20
+ },
21
+ {
22
+ "baseline": "surya",
23
+ "metric": "table_exact_match",
24
+ "direction": "higher",
25
+ "slices": ["table"],
26
+ "min_pairs": 40,
27
+ "min_absolute_margin": 0.01
28
+ },
29
+ {
30
+ "baseline": "surya",
31
+ "metric": "table_structure_similarity",
32
+ "direction": "higher",
33
+ "slices": ["table"],
34
+ "min_pairs": 40,
35
+ "min_absolute_margin": 0.0,
36
+ "require_ci_above_zero": false
37
+ }
38
+ ]
39
+ }
docs/CLAIM_REVIEW.md ADDED
@@ -0,0 +1,320 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Claim Review
2
+
3
+ `review-claim` is the portfolio-level gate for a SOTA claim. `validate-claim` can tell you whether one experiment passed its configured gates. It cannot tell you whether you have enough evidence across the internal Nepali/English pack and public suites to say "better than what is out there." That is what `review-claim` is for.
4
+
5
+ ## What It Audits
6
+
7
+ `review-claim` checks that:
8
+
9
+ - every required experiment slot is backed by a real `experiment.json`
10
+ - every referenced eval manifest and benchmark report still exists at the
11
+ recorded path, and the eval manifest matches the experiment's recorded
12
+ SHA-256
13
+ - successful benchmark rows are bound back to sample ids and input paths in the
14
+ reviewed eval manifest, and row slice labels must match the manifest entry
15
+ - the expected datasets or manifest patterns are present
16
+ - required baselines were actually run
17
+ - required metric families are present on successful rows for the required baselines
18
+ - required metric families are present on the slices they are supposed to
19
+ substantiate when `required_metric_slices` is configured
20
+ - benchmark reports do not contain error rows when the slot requires clean runs;
21
+ status counts are derived from `benchmark/report.json`, not trusted from the
22
+ copied `experiment.json` summary
23
+ - required validation status is present for internal claim-grade experiments,
24
+ and the referenced `validation.json` agrees with `experiment.json`
25
+ - required validation gates passed for the baseline/metric/slice comparisons
26
+ declared by `required_validation_gates`
27
+ - when configured, candidate fallback diagnostics are present and the aggregate
28
+ fallback trigger/failure rates stay under the slot limits
29
+ - required slices exist for the internal Nepali/English pack
30
+ - candidate experiments all point to the same candidate model config
31
+ - the candidate model card matches experiment hashes and passes `audit-model`,
32
+ with or without an explicit `--model-config`
33
+ - if candidate experiments record a fallback model card, that card exists,
34
+ matches the experiment SHA-256, and passes `audit-model`
35
+ - when configured, the candidate model card itself records the same fallback
36
+ component in `provenance.fallback_model_card`
37
+ - when configured, the candidate model card proves it includes a trained local
38
+ recognizer with artifacts, train/eval manifest hashes, and training-summary
39
+ provenance
40
+ - when configured, the candidate model card references a passing admission
41
+ validation report before it can enter portfolio review
42
+ - routed candidates such as `quality_select_composite`, `quality_ranked_ensemble`, and `script_select_composite` were calibrated on a separate eval manifest, not on one of the reviewed experiment manifests
43
+ - routed candidates also fail review if their selected calibration trial collapses to one engine on every calibration sample
44
+ - routed candidates also fail review if a reviewed benchmark shows they are exactly indistinguishable from one of their component baselines on paired holdout CER
45
+
46
+ Metric coverage is a hard gate because the target claim is a document-understanding
47
+ claim, not a plain OCR claim. A required experiment can now declare
48
+ `required_metrics`; each metric must appear as a finite value on at least one
49
+ successful row for every required baseline in that slot. For the internal
50
+ Nepali/English document pack, the SOTA config requires text OCR metrics
51
+ (`cer`, `wer`), document reconstruction
52
+ (`document_reconstruction_similarity`), table reconstruction
53
+ (`table_cell_f1`), reading order (`reading_order_pair_accuracy`), and figure/caption metrics
54
+ (`figure_detection_f1`, `figure_caption_cer`). Public table/document slots
55
+ declare their own metric set.
56
+
57
+ For claim-grade slots, also declare `required_metric_slices`. This prevents a
58
+ metric from satisfying the portfolio check on the wrong document type. For
59
+ example, `table_cell_f1` must be finite on a `table` or document-parsing row,
60
+ and `figure_caption_cer` must be finite on a `figure` row.
61
+
62
+ Metric presence is not enough. When a required slot includes `candidate` and
63
+ one or more external baselines, `review-claim` also requires paired metric
64
+ coverage on the same sample ids. If `required_metric_slices` is configured, the
65
+ pairing is checked on those slices; otherwise it is checked across all rows.
66
+ This prevents a SOTA review from passing because candidate metrics and baseline
67
+ metrics both exist somewhere in the report but never compare the same documents.
68
+
69
+ Sample coverage is also a hard gate. Every `require_baselines` entry must have a
70
+ successful benchmark row for every sample in the reviewed eval manifest. This
71
+ prevents a report from pointing at a hard eval pack while only running an easy
72
+ subset.
73
+
74
+ When a slot declares `require_validation_status`, `review-claim` reads the
75
+ referenced validation report and verifies `claim_status`. For `validated`, it
76
+ also requires `passed: true`. The copied status inside `experiment.json` is not
77
+ enough on its own. The validation artifact must also carry provenance linking it
78
+ to the same benchmark report, eval manifest, validation config, model cards,
79
+ and train manifest hashes referenced by the experiment.
80
+
81
+ Use `required_validation_gates` for claim-grade slots. Metric coverage proves
82
+ that a benchmark emitted a metric; passing validation gates prove the candidate
83
+ beat a named baseline under the configured sample count, slices, margins, and
84
+ statistical checks. Review requires each matched gate to carry quantitative
85
+ paired evidence, not just `passed: true`: positive `pairs`, finite candidate
86
+ and baseline means, finite absolute improvement, and win/loss/tie counts that
87
+ sum to the pair count. It also re-checks the threshold fields embedded in the
88
+ gate result, including `min_pairs`, margins, `min_pair_win_rate`, `max_p_value`,
89
+ and `require_ci_above_zero`, so a hand-edited report cannot mark a weak gate as
90
+ passed without satisfying its own configured test.
91
+
92
+ The SOTA validation configs also use `min_pair_win_rate` on improvement gates.
93
+ This is checked as wins divided by all paired samples, not only non-tied pairs.
94
+ It prevents an average-only improvement, or a single win with many ties, from
95
+ supporting a broad "better OCR/document engine" claim.
96
+
97
+ `required_validation_gates` is only valid on slots that also declare
98
+ `require_validation_status`. Each required gate baseline must also appear in
99
+ `require_baselines`, and each gate metric must appear in `required_metrics`.
100
+
101
+ For claim-grade slots, set `required_validation_config` to the checked-in
102
+ validation config path. `review-claim` hashes that file and requires
103
+ `experiment.json` to record the same `validation_config_sha256`. This prevents a
104
+ portfolio from passing with an easier local validation config that happens to
105
+ contain a subset of the expected gates.
106
+
107
+ For hybrid candidates, use `require_candidate_fallback_metrics`,
108
+ `max_candidate_fallback_trigger_rate`, and
109
+ `max_candidate_fallback_failed_rate`. These checks make fallback dependency
110
+ auditable. A candidate that wins mostly by routing pages to the fallback is a
111
+ different claim from a local-first engine with selective fallback, even if both
112
+ produce strong output metrics.
113
+
114
+ Fallback model cards are audited separately from the primary candidate card.
115
+ When `experiment.json` records `fallback_model_config` and
116
+ `fallback_model_config_sha256`, `review-claim` verifies the hash and runs
117
+ `audit-model` on that fallback card. The SOTA profile also sets
118
+ `require_fallback_model_present: true`,
119
+ `require_candidate_fallback_provenance: true`,
120
+ `require_shared_fallback_model: true` and
121
+ `require_fallback_model_audit_passed: true`, so a portfolio cannot silently mix
122
+ unaudited fallback components across required experiment slots or omit fallback
123
+ provenance while still claiming the selective-fallback engine.
124
+
125
+ The candidate card must also declare the fallback component it was built to use:
126
+
127
+ ```json
128
+ {
129
+ "provenance": {
130
+ "fallback_model_card": {
131
+ "path": "runs/fallback/model-card.json",
132
+ "sha256": "..."
133
+ }
134
+ }
135
+ }
136
+ ```
137
+
138
+ `review-claim` hashes that model card, compares it with every reviewed
139
+ `fallback_model_config_sha256`, and audits the fallback card. This closes the
140
+ gap where an experiment could record a fallback at benchmark time while the
141
+ candidate card itself remained ambiguous about which selective-fallback engine
142
+ was being claimed.
143
+
144
+ That second guardrail matters for exactly the case we hit on `k2`: a rendered-document `script_select_composite` calibrated on `claimpack-rendered-v2` chose `primary_script_threshold=0.0`, meaning the route degenerated to "always pick Surya." Separation from the final holdout is not enough there; the candidate is still not a meaningful new routed model.
145
+
146
+ The holdout-equivalence guardrail covers the next failure mode after calibration separation: even when calibration was done on a separate split, a routed candidate can still reduce to one component baseline on the reviewed holdout. If the paired benchmark rows show `0` wins, `0` losses, and all ties on CER against a component baseline, `review-claim` now fails `candidate-distinctness`.
147
+
148
+ For the SOTA track, set `require_trained_recognizer: true`. A direct
149
+ `paddleocr` or `hf_vision_encoder_decoder` candidate passes this check only when
150
+ the model card has recorded artifacts plus `provenance.train_manifest`,
151
+ `provenance.eval_manifest`, and `provenance.training_summary` with SHA-256
152
+ hashes. Review resolves those paths from the model-card location and verifies
153
+ the files still match the recorded hashes. Composite candidates must point to
154
+ the trained component with:
155
+
156
+ ```json
157
+ {
158
+ "provenance": {
159
+ "trained_recognizer_model_card": {
160
+ "path": "runs/recognizer/package/model-card.json",
161
+ "sha256": "..."
162
+ }
163
+ }
164
+ }
165
+ ```
166
+
167
+ This prevents a wrapper around Tesseract, Surya, or a VLM from satisfying the
168
+ paper objective by benchmark routing alone.
169
+
170
+ For recognizer branch admission, set `require_model_admission: true` and record
171
+ the admission validation report in the model card:
172
+
173
+ ```json
174
+ {
175
+ "metrics": {
176
+ "admission_validation": {
177
+ "path": "outputs/validation-next-recognizer-hard50/validation.json",
178
+ "sha256": "...",
179
+ "required_status": "validated"
180
+ }
181
+ }
182
+ }
183
+ ```
184
+
185
+ This catches the failure mode we hit with the PaddleOCR branches: the model can
186
+ have artifacts and train/eval provenance, but still be empirically rejected by
187
+ the hard recognizer gate. Rejected admission reports are summarized with their
188
+ failed requirements and gate deltas, so review output can show why a recognizer
189
+ was rejected without manually opening the raw validation JSON. Admission reports
190
+ are also bound back to `provenance.candidate_model_config`; review rejects a
191
+ passing admission artifact if it points at a different model id, backend, base
192
+ model, or artifact set.
193
+
194
+ For document-engine SOTA claims, metric presence is not enough. The
195
+ `configs/claim-review-sota-v1.json` portfolio requires passing validation gates
196
+ for the named claim dimensions:
197
+
198
+ - OCR: CER and WER gates on Nepali/English slices where applicable
199
+ - tables: `table_cell_f1`
200
+ - reading order: `reading_order_pair_accuracy`
201
+ - figures: `figure_detection_f1` and `figure_caption_cer`
202
+ - reconstruction: `document_reconstruction_similarity`
203
+
204
+ The corresponding validation configs must contain matching gates. The
205
+ `tests/test_claim_review_sota_config.py` checks pin this relationship so a
206
+ future config edit cannot silently downgrade figure/caption or WER from a
207
+ passing gate to a reported-only metric.
208
+
209
+ The SOTA profile also requires exported structured-output artifacts from the
210
+ candidate benchmark rows:
211
+
212
+ - every required slot: `document.json`, `document.md`, and `quality.json`
213
+ - table slices: table CSV and HTML exports under `tables/`
214
+ - figure slices: `figures/metadata.json` and exported figure image files
215
+
216
+ `review-claim` infers these paths from the benchmark report directory and the
217
+ candidate row input stem. It also performs minimal content checks: JSON
218
+ artifacts must parse with the expected top-level shape, Markdown/table/image
219
+ files must be nonempty, and figure metadata must contain entries. This means a
220
+ claim cannot be supported only by scalar metrics if the human-readable Markdown,
221
+ machine-readable JSON, tables, or figures were not actually produced as usable
222
+ artifacts.
223
+
224
+ ## Example Config
225
+
226
+ ```json
227
+ {
228
+ "require_trained_recognizer": true,
229
+ "require_model_admission": true,
230
+ "require_fallback_model_present": true,
231
+ "require_candidate_fallback_provenance": true,
232
+ "require_shared_fallback_model": true,
233
+ "require_fallback_model_audit_passed": true,
234
+ "required_experiments": [
235
+ {
236
+ "name": "internal-nepali-pack",
237
+ "datasets": ["ocrtech-eval-pack"],
238
+ "require_validation_status": "validated",
239
+ "require_baselines": ["candidate", "tesseract", "stock-paddle", "glm-ocr", "paddleocr-vl"],
240
+ "required_metrics": ["cer", "wer", "document_reconstruction_similarity", "table_cell_f1", "reading_order_pair_accuracy", "figure_detection_f1", "figure_caption_cer"],
241
+ "required_metric_slices": {
242
+ "cer": ["nepali", "english"],
243
+ "wer": ["nepali", "english"],
244
+ "document_reconstruction_similarity": ["table", "reading_order", "figure"],
245
+ "table_cell_f1": ["table"],
246
+ "reading_order_pair_accuracy": ["reading_order"],
247
+ "figure_detection_f1": ["figure"],
248
+ "figure_caption_cer": ["figure"]
249
+ },
250
+ "required_validation_gates": [
251
+ {"baseline": "tesseract", "metric": "cer", "slices": ["nepali"]},
252
+ {"baseline": "stock-paddle", "metric": "cer", "slices": ["nepali"]},
253
+ {"baseline": "glm-ocr", "metric": "cer", "slices": ["nepali"]},
254
+ {"baseline": "surya", "metric": "table_cell_f1", "slices": ["table"]},
255
+ {"baseline": "paddleocr-vl", "metric": "reading_order_pair_accuracy", "slices": ["reading_order"]}
256
+ ],
257
+ "require_candidate_fallback_metrics": true,
258
+ "max_candidate_fallback_trigger_rate": 0.35,
259
+ "max_candidate_fallback_failed_rate": 0.0,
260
+ "required_slices": ["nepali", "english", "table", "reading_order", "figure", "scan"]
261
+ },
262
+ {
263
+ "name": "ocrbench-v2",
264
+ "datasets": ["ocrbench-v2"],
265
+ "require_validation_status": "validated",
266
+ "required_metrics": ["cer", "wer"],
267
+ "require_baselines": ["candidate", "glm-ocr", "paddleocr-vl"]
268
+ },
269
+ {
270
+ "name": "omnidocbench",
271
+ "datasets": ["omnidocbench"],
272
+ "require_validation_status": "validated",
273
+ "required_metrics": ["cer", "document_reconstruction_similarity", "table_cell_f1", "reading_order_pair_accuracy"],
274
+ "require_baselines": ["candidate", "paddleocr-vl"]
275
+ }
276
+ ]
277
+ }
278
+ ```
279
+
280
+ The config is intentionally explicit. If two experiments match one slot, the review fails. Tighten the config until every required slot maps to one concrete artifact.
281
+ `eval_manifest_patterns` use shell-style wildcards, so a slot such as
282
+ `"*Real5-OmniDocBench*"` matches a manifest path containing that benchmark name.
283
+
284
+ The checked-in SOTA profile is paired with slot-specific validation configs:
285
+
286
+ - `configs/validation-internal-sota-v1.json`
287
+ - `configs/validation-public-omnidocbench-sota-v1.json`
288
+ - `configs/validation-public-real5-sota-v1.json`
289
+
290
+ Those configs are expected to produce the passed gates referenced by
291
+ `required_validation_gates`.
292
+
293
+ ## Run
294
+
295
+ ```bash
296
+ ocrtech review-claim \
297
+ --experiment outputs/internal-claim/experiment.json \
298
+ --experiment outputs/ocrbench-v2/experiment.json \
299
+ --experiment outputs/omnidocbench/experiment.json \
300
+ --config claim-review.json \
301
+ --model-config runs/recognizer/package/model-card.json \
302
+ --out outputs/claim-review
303
+ ```
304
+
305
+ ## Outputs
306
+
307
+ ```text
308
+ outputs/claim-review/
309
+ claim-review.json
310
+ claim-review.md
311
+ ```
312
+
313
+ If this report fails, the claim is not review-ready.
314
+
315
+ Do not hand-edit copied `experiment.json` files to point at convenient local
316
+ paths. If an experiment was run remotely, sync the benchmark report, manifest,
317
+ references, model card, and validation report together, or regenerate the
318
+ experiment report from the synced paths. `review-claim` intentionally fails on
319
+ stale paths instead of guessing because path rewrites without hashes destroy
320
+ claim provenance.
docs/DATASETS.md ADDED
@@ -0,0 +1,381 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Dataset Workflow
2
+
3
+ The dataset workflow is deliberately strict. If train/eval contamination gets in here, every later validation result is suspect.
4
+
5
+ ## Prepare
6
+
7
+ Convert raw labels to JSONL:
8
+
9
+ ```bash
10
+ ocrtech prepare-data labels.csv \
11
+ --out data/manifests \
12
+ --dataset-name nepali-lines \
13
+ --split all \
14
+ --image-field image_path \
15
+ --text-field text \
16
+ --strict-chars
17
+ ```
18
+
19
+ If the source is noisy and you need an explicit reject list instead of failing at the first bad row:
20
+
21
+ ```bash
22
+ ocrtech prepare-data labels.csv \
23
+ --out data/manifests \
24
+ --dataset-name nepali-lines \
25
+ --split all \
26
+ --image-field image_path \
27
+ --text-field text \
28
+ --skip-invalid
29
+ ```
30
+
31
+ That writes:
32
+
33
+ - `data/manifests/nepali-lines-all.jsonl`
34
+ - `data/manifests/nepali-lines-all-prepare.json`
35
+ - `data/manifests/nepali-lines-all-prepare.md`
36
+ - `data/manifests/nepali-lines-all-rejects.jsonl` when any rows are skipped
37
+
38
+ Verified Hugging Face sources can be converted directly:
39
+
40
+ ```bash
41
+ ocrtech prepare-data gauravgiri/nepali-ocr-dataset \
42
+ --hf \
43
+ --split train \
44
+ --limit 100 \
45
+ --skip-invalid \
46
+ --out data/manifests
47
+
48
+ ocrtech prepare-data prashant0919/nepali-synthetic-ocr-lines \
49
+ --hf \
50
+ --split train \
51
+ --limit 100 \
52
+ --out data/manifests
53
+
54
+ ocrtech prepare-data nvidia/OCR-Synthetic-Multilingual-v1 \
55
+ --hf \
56
+ --hf-subset en \
57
+ --split test \
58
+ --limit 1000 \
59
+ --out data/manifests-english-test
60
+ ```
61
+
62
+ Before those conversion commands, record a schema inspection so the training
63
+ corpus has evidence that fields were verified from the actual source:
64
+
65
+ ```bash
66
+ ocrtech inspect-hf-dataset gauravgiri/nepali-ocr-dataset \
67
+ --split train \
68
+ --limit 3 \
69
+ --out outputs/hf-inspect-training-v1
70
+
71
+ ocrtech inspect-hf-dataset prashant0919/nepali-synthetic-ocr-lines \
72
+ --split train \
73
+ --limit 3 \
74
+ --out outputs/hf-inspect-training-v1
75
+
76
+ ocrtech inspect-hf-dataset nvidia/OCR-Synthetic-Multilingual-v1 \
77
+ --hf-subset en \
78
+ --split test \
79
+ --limit 3 \
80
+ --out outputs/hf-inspect-training-v1
81
+ ```
82
+
83
+ Verified on `k2` on `2026-06-15`:
84
+
85
+ | dataset | split/subset | mode | inferred image | inferred text | result |
86
+ | --- | --- | --- | --- | --- | --- |
87
+ | `gauravgiri/nepali-ocr-dataset` | `train` | `datasets.load_dataset` | `image` | `text` | pass |
88
+ | `prashant0919/nepali-synthetic-ocr-lines` | `train` | HF repo `metadata.jsonl` | `file_name` | `text` | pass |
89
+ | `nvidia/OCR-Synthetic-Multilingual-v1` | `test/en` | HDF5 `line_bboxes` | `images + annotations.line_bboxes[].bbox` | `annotations.line_bboxes[].text` | pass |
90
+
91
+ Artifacts:
92
+
93
+ - `outputs/hf-inspect-training-v1/gauravgiri__nepali-ocr-dataset-train-inspect.json`
94
+ - `outputs/hf-inspect-training-v1/prashant0919__nepali-synthetic-ocr-lines-train-inspect.json`
95
+ - `outputs/hf-inspect-training-v1/nvidia__OCR-Synthetic-Multilingual-v1__en-test-inspect.json`
96
+
97
+ ## Public Benchmarks
98
+
99
+ Public document parsing benchmarks are tracked separately from training data.
100
+ Do not mix them into recognizer/corrector training manifests. They are final
101
+ claim evidence.
102
+
103
+ List the known registry:
104
+
105
+ ```bash
106
+ ocrtech list-public-benchmarks
107
+ ```
108
+
109
+ Current target order:
110
+
111
+ 1. `opendatalab/OmniDocBench` for broad document parsing.
112
+ 2. `PaddlePaddle/Real5-OmniDocBench` for real-world physical distortions.
113
+ 3. `allenai/olmOCR-bench` for PDF-to-Markdown/unit-test style evaluation.
114
+
115
+ Prepare the verified OmniDocBench schema:
116
+
117
+ ```bash
118
+ ocrtech prepare-public-benchmark opendatalab/OmniDocBench \
119
+ --out data/public/omnidocbench \
120
+ --limit 100 \
121
+ --language english \
122
+ --require-tables
123
+ ```
124
+
125
+ This converter is schema-bound. It expects the verified Hugging Face layout:
126
+ `OmniDocBench.json` as a list of rows, each row with `page_info.image_path`,
127
+ `page_info.page_attribute`, and `layout_dets`. Table references come from
128
+ `layout_dets[].html`; text references come from ordered `layout_dets[].text`
129
+ plus table HTML text. Images are resolved under `images/`.
130
+
131
+ Before benchmarking a public pack, audit its manifest:
132
+
133
+ ```bash
134
+ ocrtech audit-public-benchmark data/public/omnidocbench/manifests/omnidocbench-eval.jsonl \
135
+ --benchmark opendatalab/OmniDocBench \
136
+ --out outputs/public-omnidocbench-audit
137
+ ```
138
+
139
+ Each public manifest row must carry provenance:
140
+
141
+ - `metadata.public_benchmark = true`
142
+ - `metadata.benchmark_name`
143
+ - `metadata.source_url`
144
+ - `metadata.reference_path`
145
+ - `metadata.slices` containing `public`
146
+
147
+ This is intentionally stricter than `prepare-data`. The agent must verify the
148
+ actual downloaded dataset schema before writing a converter for any public
149
+ benchmark. No schema guessing.
150
+
151
+ Current claim-sized public OmniDocBench table pack on `k2`:
152
+
153
+ - manifest: `data/public/omnidocbench-table100/manifests/omnidocbench-eval.jsonl`
154
+ - prepare report: `data/public/omnidocbench-table100/public-benchmark-prepare.json`
155
+ - audit: `outputs/public-omnidocbench-table100-audit/public-benchmark-audit.json`
156
+ - audit status: passed
157
+ - samples: `100`
158
+ - slices: `public=100`, `document_parsing=100`, `english=100`, `table=100`,
159
+ `table_hard=32`, `figure=22`
160
+
161
+ This is the first public OmniDocBench manifest in the repo that can satisfy the
162
+ configured `100`-page public document gate and the `40`-page public table gate.
163
+
164
+ For the current Nepali starter corpus, merge verified sources with the built-in command instead of shell concatenation:
165
+
166
+ ```bash
167
+ ocrtech merge-manifests \
168
+ data/manifests/gauravgiri__nepali-ocr-dataset-train.jsonl \
169
+ data/manifests/prashant0919__nepali-synthetic-ocr-lines-train.jsonl \
170
+ --out data/manifests/nepali-mixed-all.jsonl
171
+ ```
172
+
173
+ Do not pretend all HF datasets look the same. This repo only has verified adapters for the schemas we checked:
174
+
175
+ - `gauravgiri/nepali-ocr-dataset`: `image`, `text`
176
+ - `prashant0919/nepali-synthetic-ocr-lines`: `train/metadata.jsonl` plus image files
177
+ - `cfilt/RoundTripOCR-nepali`: `ocr`, `correct`, `font`
178
+ - `nvidia/OCR-Synthetic-Multilingual-v1`: HDF5 shards with `line_bboxes`
179
+
180
+ Known adapters also stamp verified slice metadata so later audits and experiments do not depend on hand-added tags:
181
+
182
+ - `gauravgiri/nepali-ocr-dataset`: `nepali`, `real`
183
+ - `prashant0919/nepali-synthetic-ocr-lines`: `nepali`, `synthetic`
184
+ - `nvidia/OCR-Synthetic-Multilingual-v1 --hf-subset en`: `english`, `synthetic`, `line_crop`, `en`
185
+
186
+ Verified on `k2` on June 14, 2026:
187
+
188
+ - `gauravgiri/nepali-ocr-dataset --limit 100 --skip-invalid`: `97` prepared, `3` rejected due to empty `text`
189
+ - `prashant0919/nepali-synthetic-ocr-lines --limit 100`: `100` prepared, `0` rejected
190
+ - merged starter corpus: `197` samples, no merge warnings
191
+ - deterministic split: `167` train / `30` eval, leakage check passed
192
+
193
+ Each manifest row contains:
194
+
195
+ - `sample_id`
196
+ - `dataset`
197
+ - `split`
198
+ - `image_path`
199
+ - `text`
200
+ - `sha256`
201
+ - `metadata.text_sha256`
202
+ - `metadata.sample_sha256`
203
+
204
+ ## Audit
205
+
206
+ Before splitting or training:
207
+
208
+ ```bash
209
+ ocrtech audit-manifest data/manifests/nepali-lines-all.jsonl \
210
+ --out outputs/manifest-audit \
211
+ --require-slice nepali \
212
+ --require-slice english \
213
+ --strict-chars
214
+ ```
215
+
216
+ The audit checks:
217
+
218
+ - missing input files
219
+ - image SHA-256 mismatches
220
+ - duplicate sample IDs
221
+ - duplicate paths and hashes
222
+ - slice counts
223
+ - script counts
224
+ - unsupported characters when requested
225
+
226
+ ## Split
227
+
228
+ Create train/eval manifests with deterministic grouping:
229
+
230
+ ```bash
231
+ ocrtech split-manifest data/manifests/nepali-lines-all.jsonl \
232
+ --out data/splits \
233
+ --eval-ratio 0.15 \
234
+ --seed 13 \
235
+ --stratify-by slices
236
+ ```
237
+
238
+ The splitter groups entries by `metadata.sample_sha256` before assigning splits by default. If the same image+text sample appears twice, it stays in one split.
239
+
240
+ For rendered packs with multiple degraded variants of the same underlying text, group by `text_sha256` instead. This keeps clean, blur, scan, low-contrast, and skew variants of the same target text on the same side of the train/eval boundary:
241
+
242
+ ```bash
243
+ ocrtech split-manifest data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl \
244
+ --out data/splits-table-figure-dev-v1 \
245
+ --eval-ratio 0.2 \
246
+ --seed 17 \
247
+ --stratify-by document_type \
248
+ --group-by text_sha256
249
+ ```
250
+
251
+ Supported grouping keys are:
252
+
253
+ - `sample_signature`
254
+ - `text_sha256`
255
+ - `image_sha256`
256
+ - `image_path`
257
+ - `sample_id`
258
+
259
+ Outputs:
260
+
261
+ ```text
262
+ data/splits/
263
+ train.jsonl
264
+ eval.jsonl
265
+ split-summary.json
266
+ split-summary.md
267
+ ```
268
+
269
+ The split summary records the grouping key. For the table/figure development pack, the verified `k2` split produced:
270
+
271
+ - `60` train samples
272
+ - `20` eval samples
273
+ - `16` text groups
274
+ - `group_by = text_sha256`
275
+ - leakage check passed
276
+
277
+ ## Rebalance
278
+
279
+ Use `rebalance-manifest` only on the training split when a slice is obviously underrepresented and you want a reproducible upsampling step instead of an undocumented training-time sampler choice:
280
+
281
+ ```bash
282
+ ocrtech rebalance-manifest data/splits/train.jsonl \
283
+ --out data/splits/train-rebalanced.jsonl \
284
+ --slice english \
285
+ --slice nepali \
286
+ --seed 13
287
+ ```
288
+
289
+ Outputs:
290
+
291
+ ```text
292
+ data/splits/
293
+ train-rebalanced.jsonl
294
+ train-rebalanced-rebalance.json
295
+ train-rebalanced-rebalance.md
296
+ ```
297
+
298
+ This command duplicates rows with explicit provenance in metadata:
299
+
300
+ - `rebalance_duplicate`
301
+ - `rebalance_source_sample_id`
302
+ - `rebalance_slice`
303
+
304
+ Do not rebalance the unsplit combined manifest. That would duplicate samples before train/eval assignment and corrupt the benchmark.
305
+
306
+ ## Train
307
+
308
+ Use only the split outputs:
309
+
310
+ ```bash
311
+ ocrtech train-recognizer \
312
+ --train-manifest data/splits/train.jsonl \
313
+ --eval-manifest data/splits/eval.jsonl \
314
+ --out runs/recognizer
315
+ ```
316
+
317
+ For generated document packs, do not train a recognizer on whole rendered pages unless the model is explicitly document-level. The current full-page HF branch failed badly. Extract line/cell/caption crops first, then split those crop rows with text-level grouping:
318
+
319
+ ```bash
320
+ ocrtech extract-crops \
321
+ data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl \
322
+ --out data/crops-table-figure-dev-v1 \
323
+ --crop-type line \
324
+ --crop-type table-cell \
325
+ --crop-type figure-caption
326
+
327
+ ocrtech split-manifest data/crops-table-figure-dev-v1/manifests/crops.jsonl \
328
+ --out data/splits-crops-table-figure-dev-v1 \
329
+ --eval-ratio 0.2 \
330
+ --seed 17 \
331
+ --stratify-by document_type \
332
+ --group-by text_sha256
333
+ ```
334
+
335
+ The crop extractor only reads explicit renderer layout metadata from the reference JSON. If that metadata is absent, it fails instead of guessing bboxes.
336
+
337
+ Before training, build the recognizer corpus from verified manifests with
338
+ explicit bucket targets:
339
+
340
+ ```bash
341
+ ocrtech build-recognizer-corpus \
342
+ data/manifests/english-lines.jsonl \
343
+ data/manifests/nepali-lines.jsonl \
344
+ data/splits-crops-table-figure-dev-v1/train.jsonl \
345
+ --out data/splits/recognizer-train-balanced.jsonl \
346
+ --target-latin-only 200 \
347
+ --target-devanagari-only 200 \
348
+ --target-mixed 100 \
349
+ --target-real 100 \
350
+ --target-synthetic 100
351
+ ```
352
+
353
+ This command upsamples existing examples only. If a requested bucket is missing
354
+ from the inputs, it fails; add source data instead of weakening the audit gate.
355
+
356
+ For OCR correction from the verified CFILT dataset:
357
+
358
+ ```bash
359
+ ocrtech train-corrector \
360
+ --hf-dataset cfilt/RoundTripOCR-nepali \
361
+ --hf-split train \
362
+ --limit 1000 \
363
+ --out runs/corrector-cfilt
364
+ ```
365
+
366
+ Do not point training at the combined manifest.
367
+
368
+ ## Validation
369
+
370
+ Pass the training manifests into claim validation:
371
+
372
+ ```bash
373
+ ocrtech validate-claim \
374
+ --benchmark-report outputs/benchmark/report.json \
375
+ --eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
376
+ --train-manifest data/splits/train.jsonl \
377
+ --config validation.json \
378
+ --out outputs/validation
379
+ ```
380
+
381
+ The validation step checks train/eval leakage again. That is intentional redundancy.
docs/EVAL_PACK.md ADDED
@@ -0,0 +1,226 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Eval Pack Workflow
2
+
3
+ The eval pack is the internal held-out document set used to test the exact target domain:
4
+
5
+ - Devanagari/Nepali OCR
6
+ - English OCR
7
+ - mixed-script lines
8
+ - multi-page mixed-script documents
9
+ - tables
10
+ - forms
11
+ - figure captions
12
+ - reading order
13
+ - noisy scanned variants
14
+
15
+ This does not replace public benchmarks. It complements public suites with the Nepali + English documents we actually care about.
16
+
17
+ ## Smoke Pack
18
+
19
+ A text pack exercises document reconstruction and scoring without needing OCR engines:
20
+
21
+ ```bash
22
+ ocrtech create-eval-pack \
23
+ --out data/eval-pack-smoke \
24
+ --input-format text
25
+
26
+ ocrtech audit-eval-pack \
27
+ data/eval-pack-smoke/manifests/ocrtech-eval-pack.jsonl \
28
+ --out outputs/eval-pack-smoke-audit
29
+ ```
30
+
31
+ Text packs are not OCR evidence. They are useful for validating Markdown/JSON/table/reading-order scoring.
32
+
33
+ ## Claim Pack
34
+
35
+ An OCR claim pack must use rendered image inputs:
36
+
37
+ ```bash
38
+ ocrtech create-eval-pack \
39
+ --out data/eval-pack \
40
+ --input-format image \
41
+ --degradation clean \
42
+ --degradation scan \
43
+ --degradation blur \
44
+ --degradation low_contrast \
45
+ --degradation skew \
46
+ --count-per-template 25
47
+
48
+ ocrtech audit-eval-pack \
49
+ data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
50
+ --out outputs/eval-pack-audit \
51
+ --require-claim-ready
52
+ ```
53
+
54
+ Image packs require Pillow:
55
+
56
+ ```bash
57
+ python -m pip install -e '.[eval]'
58
+ ```
59
+
60
+ For Devanagari rendering quality, pass a real font if auto-discovery is not good enough:
61
+
62
+ ```bash
63
+ ocrtech create-eval-pack \
64
+ --out data/eval-pack \
65
+ --input-format image \
66
+ --font-path /path/to/NotoSansDevanagari-Regular.ttf
67
+ ```
68
+
69
+ ## Targeted Development Packs
70
+
71
+ Use template selection for model-development packs that target a known failure pocket. These packs are useful for iteration, but they are not broad claim evidence unless they also satisfy the claim-pack coverage requirements.
72
+
73
+ ```bash
74
+ ocrtech create-eval-pack \
75
+ --out data/eval-pack-table-figure-dev-v1 \
76
+ --input-format image \
77
+ --template table-ledger \
78
+ --template figure-caption \
79
+ --count-per-template 8 \
80
+ --variant-offset 100 \
81
+ --degradation clean \
82
+ --degradation scan \
83
+ --degradation blur \
84
+ --degradation low_contrast \
85
+ --degradation skew
86
+ ```
87
+
88
+ Use `--variant-offset` when creating a development pack after a held-out pack, so sample IDs and visible reference text do not collide with the held-out evidence. Still check leakage explicitly before using a generated pack for training.
89
+
90
+ For span-aware table validation, use the `complex-table-ledger` template:
91
+
92
+ ```bash
93
+ ocrtech create-eval-pack \
94
+ --out data/eval-pack-complex-table-dev-v1 \
95
+ --input-format image \
96
+ --template complex-table-ledger \
97
+ --count-per-template 8 \
98
+ --variant-offset 500 \
99
+ --degradation clean \
100
+ --degradation scan \
101
+ --degradation blur \
102
+ --degradation low_contrast
103
+ ```
104
+
105
+ This template renders a visible rectangular table but writes structured reference cells with a merged header (`colspan=2`). It is useful for validating that scorers and candidate outputs preserve spans. It is still generated data, not a substitute for real scanned complex tables.
106
+
107
+ ## Artifacts
108
+
109
+ Generated structure:
110
+
111
+ ```text
112
+ data/eval-pack/
113
+ inputs/
114
+ references/
115
+ manifests/ocrtech-eval-pack.jsonl
116
+ eval-pack.json
117
+ eval-pack.md
118
+ ```
119
+
120
+ Each manifest entry includes:
121
+
122
+ - `sample_id`
123
+ - `image_path`
124
+ - `sha256`
125
+ - `metadata.slices`
126
+ - `metadata.document_type`
127
+ - `metadata.degradation`
128
+ - `metadata.input_format`
129
+ - `metadata.reference_path`
130
+ - `metadata.claim_evidence_eligible`
131
+
132
+ For single-page samples, `image_path` points to one file. For multi-page samples, `image_path` points to a page-bundle directory such as:
133
+
134
+ ```text
135
+ inputs/eval-multi-page-brief-000-clean/
136
+ page-0001.png
137
+ page-0002.png
138
+ ```
139
+
140
+ Text smoke packs use the same shape for multi-page samples, but with a `document.ocr.json` sidecar inside the bundle directory so the deterministic sidecar engine can reconstruct the full document.
141
+
142
+ Structured references contain:
143
+
144
+ - `text`
145
+ - `reading_order`
146
+ - `tables` as either nested grids or structured cell objects with `row`, `col`, `rowspan`, and `colspan`
147
+ - `figures`
148
+ - metadata
149
+
150
+ For image packs generated after the crop-extraction update, reference metadata also contains renderer layout metadata:
151
+
152
+ - `metadata.layout.pages[].lines[]`
153
+ - `metadata.layout.pages[].table_cells[]`
154
+ - validated pixel bboxes for non-skew degradations
155
+
156
+ Skewed samples deliberately do not expose pre-rotation bboxes. Crop extraction skips those unavailable pages instead of using invalid coordinates.
157
+
158
+ ## Crop Manifests
159
+
160
+ The failed full-page TrOCR table/figure branch showed that treating a whole page as one recognition target is the wrong training shape. Use crop manifests for the next recognizer branch:
161
+
162
+ ```bash
163
+ ocrtech extract-crops \
164
+ data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl \
165
+ --out data/crops-table-figure-dev-v1 \
166
+ --crop-type line \
167
+ --crop-type table-cell \
168
+ --crop-type figure-caption
169
+ ```
170
+
171
+ This writes:
172
+
173
+ ```text
174
+ data/crops-table-figure-dev-v1/
175
+ images/
176
+ manifests/crops.jsonl
177
+ crop-manifest-summary.json
178
+ crop-manifest-summary.md
179
+ ```
180
+
181
+ Each crop manifest row is a normal OCR training row with:
182
+
183
+ - crop image path
184
+ - crop text label
185
+ - source sample id
186
+ - source image path
187
+ - source reference path
188
+ - page index
189
+ - source bbox
190
+ - crop type
191
+ - `text_sha256`
192
+ - `sample_sha256`
193
+
194
+ If a reference lacks renderer layout metadata, `extract-crops` fails. That is intentional; do not infer crop coordinates from template names or visual guesses.
195
+
196
+ ## Benchmark
197
+
198
+ Run the same inputs against every baseline:
199
+
200
+ ```bash
201
+ inputs=$(python - <<'PY'
202
+ from ocrtech.manifest import load_manifest
203
+ for entry in load_manifest("data/eval-pack/manifests/ocrtech-eval-pack.jsonl"):
204
+ print(entry.image_path)
205
+ PY
206
+ )
207
+
208
+ ocrtech benchmark $inputs \
209
+ --out outputs/benchmark \
210
+ --eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
211
+ --references-dir data/eval-pack/references \
212
+ --baselines ours,tesseract,stock-paddle,glm-ocr,paddleocr-vl
213
+ ```
214
+
215
+ Then validate the claim:
216
+
217
+ ```bash
218
+ ocrtech validate-claim \
219
+ --benchmark-report outputs/benchmark/report.json \
220
+ --eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
221
+ --train-manifest data/manifests/train.jsonl \
222
+ --config validation.json \
223
+ --out outputs/validation
224
+ ```
225
+
226
+ Do not use `--engine sidecar` or OCR sidecars for claim benchmarks.
docs/EXPERIMENTS.md ADDED
@@ -0,0 +1,1034 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Experiment Workflow
2
+
3
+ If you are writing a paper from these runs, read [PAPER_NOTES.md](PAPER_NOTES.md) alongside this file. This document is the raw chronology; `PAPER_NOTES.md` is the stricter claim ledger.
4
+
5
+ `run-experiment` is the reproducible wrapper around benchmark and validation. It records the manifest hash, train manifest hashes, environment metadata, baseline list, benchmark report path, and validation status.
6
+
7
+ ## Smoke Experiment
8
+
9
+ ```bash
10
+ ocrtech create-eval-pack --out data/eval-pack-smoke --input-format text
11
+
12
+ ocrtech run-experiment \
13
+ --eval-manifest data/eval-pack-smoke/manifests/ocrtech-eval-pack.jsonl \
14
+ --references-dir data/eval-pack-smoke/references \
15
+ --baselines ours \
16
+ --skip-validation \
17
+ --out outputs/smoke-experiment
18
+ ```
19
+
20
+ This proves the pipeline and scoring path works. It is not OCR evidence.
21
+
22
+ ## Claim Experiment
23
+
24
+ Run preflight first:
25
+
26
+ ```bash
27
+ ocrtech preflight-claim \
28
+ --eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
29
+ --references-dir data/eval-pack/references \
30
+ --baselines candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
31
+ --candidate-model-config runs/recognizer/package/model-card.json \
32
+ --validation-config validation.json \
33
+ --train-manifest data/splits/train.jsonl \
34
+ --out outputs/preflight
35
+ ```
36
+
37
+ If the candidate is a routed composite, do not under-specify the baseline list. A claim-grade `script_select_composite` run must include its raw components in the same experiment, for example `candidate,tesseract,surya`, otherwise `preflight-claim` now fails `candidate-components`.
38
+
39
+ ```bash
40
+ ocrtech run-experiment \
41
+ --eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
42
+ --references-dir data/eval-pack/references \
43
+ --baselines candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
44
+ --candidate-model-config runs/recognizer/package/model-card.json \
45
+ --validation-config validation.json \
46
+ --train-manifest data/splits/train.jsonl \
47
+ --preflight \
48
+ --require-trained-recognizer \
49
+ --require-model-admission \
50
+ --out outputs/claim-experiment \
51
+ --require-validated
52
+ ```
53
+
54
+ `--preflight` runs the same claim-readiness checks inside `run-experiment` and
55
+ stops before benchmark output is produced if the setup cannot satisfy the
56
+ configured gates. `--require-validated` returns a non-zero exit code unless
57
+ validation status is `validated`.
58
+
59
+ After the internal claim experiment and the public suites are all complete, run `review-claim`; see [CLAIM_REVIEW.md](CLAIM_REVIEW.md).
60
+
61
+ Use `summarize-benchmark` when you need aggregate and paired slice comparisons from a raw `benchmark/report.json`:
62
+
63
+ ```bash
64
+ ocrtech summarize-benchmark \
65
+ --benchmark-report outputs/claim-experiment/benchmark/report.json \
66
+ --out outputs/claim-experiment/benchmark-summary \
67
+ --candidate candidate \
68
+ --baseline tesseract \
69
+ --metric cer
70
+ ```
71
+
72
+ Use `paper-benchmark-table` when you need one paper-facing table merged from multiple `summary.json` artifacts, for example separate candidate-vs-Tesseract, candidate-vs-Surya, and candidate-vs-stock-Paddle summaries from the same pack:
73
+
74
+ ```bash
75
+ ocrtech paper-benchmark-table \
76
+ --summary outputs/paper-claimpack-summary-tesseract/summary.json \
77
+ --summary outputs/paper-claimpack-summary-surya/summary.json \
78
+ --summary outputs/paper-claimpack-summary-stock-paddle/summary.json \
79
+ --out outputs/paper-claimpack-table \
80
+ --system candidate \
81
+ --system surya \
82
+ --system tesseract \
83
+ --system stock-paddle \
84
+ --label 'candidate=Candidate' \
85
+ --label 'surya=Surya' \
86
+ --label 'tesseract=Tesseract' \
87
+ --label 'stock-paddle=Stock Paddle' \
88
+ --source-preference 'candidate=outputs/paper-claimpack-summary-tesseract/summary.json' \
89
+ --source-preference 'tesseract=outputs/paper-claimpack-summary-tesseract/summary.json' \
90
+ --source-preference 'surya=outputs/paper-claimpack-summary-surya/summary.json' \
91
+ --source-preference 'stock-paddle=outputs/paper-claimpack-summary-stock-paddle/summary.json' \
92
+ --metric cer \
93
+ --metric table_cell_f1 \
94
+ --metric reading_order_pair_accuracy \
95
+ --metric latency_seconds
96
+ ```
97
+
98
+ The `--source-preference` pins each system to one summary artifact. That matters when the same candidate was rerun in separate comparisons and latency differs across those reruns.
99
+
100
+ ## Outputs
101
+
102
+ ```text
103
+ outputs/claim-experiment/
104
+ benchmark/report.json
105
+ benchmark/report.md
106
+ validation/validation.json
107
+ validation/validation.md
108
+ experiment.json
109
+ experiment.md
110
+ ```
111
+
112
+ `experiment.json` is the artifact to attach to a claim review. It does not replace the full benchmark and validation reports.
113
+
114
+ For candidate model-card requirements, see [MODELS.md](MODELS.md).
115
+
116
+ ## Verified k2 HF Runs
117
+
118
+ As of June 14, 2026, the most useful live experiment series on `k2` is a candidate-only TrOCR comparison over one held-out mixed Nepali/English eval manifest with `180` samples. This is workflow evidence, not claim evidence, because it does not yet include Tesseract, stock Paddle, GLM-OCR, PaddleOCR-VL, or public benchmark suites.
119
+
120
+ Corpus construction used:
121
+
122
+ - `97` valid `gauravgiri` Nepali real rows from the first `100`
123
+ - `100` `prashant0919` Nepali synthetic rows
124
+ - `1000` NVIDIA English synthetic line crops from `nvidia/OCR-Synthetic-Multilingual-v1 --hf-subset en --split test`
125
+ - deterministic split: `1017` train / `180` eval
126
+ - materialized references from the eval manifest
127
+ - rebalanced training variant: `english=850`, `nepali=850`
128
+
129
+ Artifacts pulled from `k2` into this workspace:
130
+
131
+ - `outputs/live-k2-baseline-bundle/outputs/experiment-hf-smoke-on-mixed-v1/benchmark/report.json`
132
+ - `outputs/live-k2-baseline-bundle/outputs/experiment-hf-mixed-v1/benchmark/report.json`
133
+ - `outputs/live-k2-rebalanced-bundle/outputs/experiment-hf-mixed-v1-rebalanced/benchmark/report.json`
134
+ - `outputs/live-k2-rebalanced-bundle/runs/hf-recognizer-mixed-v1-rebalanced/training_metrics.json`
135
+ - `outputs/live-k2-rebalanced-bundle/data/splits-mixed-v1/train-rebalanced-nep-eng-rebalance.json`
136
+ - `outputs/live-k2-vs-tesseract-bundle/outputs/experiment-hf-mixed-v1-rebalanced-vs-tesseract/benchmark/report.json`
137
+
138
+ Benchmark means on that same eval split:
139
+
140
+ | Run | Overall CER | Overall WER | English CER | Nepali CER | Real CER |
141
+ | --- | ---: | ---: | ---: | ---: | ---: |
142
+ | smoke | 11.613341 | 2.637120 | 9.191231 | 23.723889 | 19.593537 |
143
+ | mixed | 1.628157 | 2.916153 | 0.379729 | 7.870295 | 12.702701 |
144
+ | rebalanced | 0.586139 | 0.801844 | 0.300454 | 2.014560 | 1.770016 |
145
+
146
+ The rebalanced run is the first one here that materially improves both English and Nepali on the same held-out split instead of trading one off against the other. It is still not a SOTA claim because:
147
+
148
+ - it is candidate-only
149
+ - it uses one internal eval manifest, not the full internal document pack plus public suites
150
+ - it has no paired validation against named baselines yet
151
+ - it does not measure document-structure leadership, only OCR metrics from the current report path
152
+
153
+ That last caveat is no longer theoretical. The first named baseline check on the same eval split is already negative for the project claim:
154
+
155
+ | Baseline | Overall CER | Overall WER | English CER | Nepali CER | Real CER |
156
+ | --- | ---: | ---: | ---: | ---: | ---: |
157
+ | candidate | 0.586139 | 0.801844 | 0.300454 | 2.014560 | 1.770016 |
158
+ | tesseract | 0.329012 | 0.486485 | 0.243379 | 0.757175 | 1.162917 |
159
+
160
+ So the current state is precise:
161
+
162
+ - we fixed our own mixed-script HF candidate substantially
163
+ - we have a reproducible training and benchmark path on `k2`
164
+ - we still do not beat Tesseract on this held-out eval
165
+
166
+ We also tested a mined-correction fallback instead of pretending post-OCR cleanup would obviously help. A ByT5 corrector was trained on `1700` Tesseract noisy/clean pairs mined from the rebalanced training manifest, then packaged as a `text_correction_composite` candidate with raw Tesseract as the base engine. That result was much worse than raw Tesseract:
167
+
168
+ | Baseline | Overall CER | Overall WER | English CER | Nepali CER | Real CER |
169
+ | --- | ---: | ---: | ---: | ---: | ---: |
170
+ | Tesseract + ByT5 corrector | 6.912065 | 3.312896 | 6.578699 | 8.578897 | 15.254935 |
171
+ | raw tesseract | 0.329012 | 0.486485 | 0.243379 | 0.757175 | 1.162917 |
172
+
173
+ Paired CER on the same `180` held-out samples was decisive: `0` wins, `170` losses, `10` ties. The corrector trainer metrics were already warning us that this was not a good model (`eval_cer=7.067461`, `eval_wer=3.334917`, `eval_exact_line_accuracy=0.0`), and the benchmark confirmed it.
174
+
175
+ Artifacts recorded locally:
176
+
177
+ - `outputs/live-k2-corrector-vs-tesseract-summary/summary.json`
178
+ - `outputs/live-k2-corrector-vs-tesseract-summary/summary.md`
179
+
180
+ We also tested whether Tesseract itself had enough headroom to matter before touching the recognizer again. Three direct variant checks on the same held-out `180`-sample eval all lost overall to raw default Tesseract (`psm=6`):
181
+
182
+ | Candidate | Overall CER | English CER | Nepali CER | Real CER |
183
+ | --- | ---: | ---: | ---: | ---: |
184
+ | raw tesseract (`psm=6`) | 0.329012 | 0.243379 | 0.757175 | 1.162917 |
185
+ | tesseract (`psm=7`) | 0.353708 | 0.242640 | 0.909049 | 1.104155 |
186
+ | tesseract (`psm=11`) | 0.344319 | 0.258868 | 0.771572 | 1.073521 |
187
+ | tesseract (`psm=13`) | 0.364878 | 0.243850 | 0.970017 | 1.004421 |
188
+
189
+ That said, `psm=13` was not uniformly useless. Using the existing `quality_score` as a routing heuristic, a `quality_select_composite` candidate that keeps default Tesseract unless `psm=13` is better by more than `0.02` did improve the same eval pack slightly:
190
+
191
+ | Baseline | Overall CER | English CER | Nepali CER | Real CER |
192
+ | --- | ---: | ---: | ---: | ---: |
193
+ | raw tesseract (`psm=6`) | 0.329012 | 0.243379 | 0.757175 | 1.162917 |
194
+ | routed tesseract (`psm=6` vs `psm=13`, margin `0.02`) | 0.324000 | 0.242003 | 0.733983 | 1.116534 |
195
+
196
+ Paired CER on that same pack was `5` wins, `3` losses, and `172` ties against raw default Tesseract. This is worth keeping as a baseline candidate backend, but it is not strong enough to confuse with model leadership, and the routing margin was selected on the same held-out pack.
197
+
198
+ We then used the new calibration workflow to stop cheating on that threshold. The original `180`-sample mixed eval was split into a calibration half (`89` samples) and untouched holdout half (`91` samples). `psm=13` was benchmarked against raw default Tesseract on the calibration half, `calibrate-quality-router` selected `secondary_quality_margin=0.03`, and that routed candidate was evaluated on the untouched holdout:
199
+
200
+ | Baseline | Overall CER | English CER | Nepali CER | Real CER |
201
+ | --- | ---: | ---: | ---: | ---: |
202
+ | raw tesseract (`psm=6`) | 0.342234 | 0.236933 | 0.835834 | 1.269163 |
203
+ | routed tesseract (`psm=6` vs `psm=13`, calibrated on separate half) | 0.340608 | 0.235094 | 0.835207 | 1.267909 |
204
+
205
+ Paired CER on the untouched holdout was `2` wins, `1` loss, and `88` ties. That is a real out-of-sample gain, but the size is too small to matter for the main project objective. It tells us the routing idea is not pure leakage, not that we have a strong OCR engine.
206
+
207
+ We also tested a richer calibrated ensemble over `psm=6/7/11/13`. The calibration half liked it and selected biases:
208
+
209
+ - `psm7=-0.12`
210
+ - `psm11=0.02`
211
+ - `psm13=-0.02`
212
+
213
+ But the untouched holdout rejected it:
214
+
215
+ | Baseline | Overall CER | English CER | Nepali CER | Real CER |
216
+ | --- | ---: | ---: | ---: | ---: |
217
+ | raw tesseract (`psm=6`) | 0.342234 | 0.236933 | 0.835834 | 1.269163 |
218
+ | calibrated `psm=6/7/11/13` ensemble | 0.358352 | 0.246311 | 0.883548 | 1.375632 |
219
+
220
+ Surya by itself was not competitive on the same `91`-sample holdout. As a raw baseline it landed at overall CER `2.198362` versus raw Tesseract `0.342234`, mainly because it was disastrous on the English-heavy line-crop slice. But Surya was materially better on the Nepali slice, so we tested a calibrated script-aware route instead of discarding it outright.
221
+
222
+ That routed candidate uses raw Tesseract output as the primary signal: if the Tesseract text looks sufficiently Devanagari-heavy, switch to Surya; otherwise keep Tesseract. The threshold was calibrated on the separate `89`-sample half with `ocrtech calibrate-script-router`, which selected `primary_script_threshold=0.4`. On the untouched `91`-sample holdout:
223
+
224
+ | Baseline | Overall CER | English CER | Nepali CER | Real CER |
225
+ | --- | ---: | ---: | ---: | ---: |
226
+ | raw tesseract (`psm=6`) | 0.342234 | 0.236933 | 0.835834 | 1.269163 |
227
+ | calibrated script router (`tesseract` -> `surya`) | 0.313084 | 0.230513 | 0.700133 | 1.172699 |
228
+
229
+ Paired CER on that holdout was `9` wins, `0` losses, and `82` ties against raw Tesseract. This is the strongest out-of-sample baseline improvement in the repo so far. It is still not claim-grade evidence, because it is one internal line-level split rather than the full internal document pack plus public suites, but it is a real improvement and a better direction than the TrOCR branch.
230
+
231
+ We also ran an exploratory `routing_granularity=page` variant of the same Tesseract-to-Surya router on that holdout, reusing the calibrated threshold `0.4`. It produced the same summary numbers as the document-level router. That is not a hidden win or loss; it just means this holdout is mostly single-page line-crop material, so page routing has no additional room to differentiate behavior yet.
232
+
233
+ After that, the internal eval pack was upgraded so image-mode tables are rendered as actual grids and the pack now includes a real `multi_page` slice as bundled page images. On that rendered `5`-sample smoke pack, the plain script router reached `table_cell_f1 = 1.0` on the dedicated table page but still missed the table inside the multi-page report when it kept Tesseract text for the document. A follow-up exploratory candidate with `secondary_structure_backfill=true` fixed that exact gap: it kept the same CER while backfilling Surya table structure on the multi-page report, moving aggregate `table_cell_f1` from `0.5` to `1.0` on the rendered smoke pack. This is still smoke evidence, not claim evidence, because the pack is tiny and only covers the `clean` slice, but it is the first concrete sign that composition can improve document structure without paying a text penalty.
234
+
235
+ The next run made that path less hand-wavy. A broader rendered image pack was generated with `25` samples across `clean`, `scan`, `blur`, `low_contrast`, and `skew`, and `audit-eval-pack --require-claim-ready` passed on it. The backfill router was then calibrated through `ocrtech calibrate-script-router` with `secondary_structure_backfill=true`, producing `ocrtech-tesseract-surya-script-router-backfill-calibrated`, and `preflight-claim` passed on the final experiment pack.
236
+
237
+ Pulled artifacts for that run:
238
+
239
+ - `outputs/live-k2-claimpack-rendered-backfill/eval-pack-audit.json`
240
+ - `outputs/live-k2-claimpack-rendered-backfill/eval-pack-audit.md`
241
+
242
+ The next useful tightening step was not “run everything on all `100` rendered samples and hope.” That is too expensive for the current GLM and PaddleOCR-VL paths on `k2`. Instead, the repo now has `ocrtech filter-manifest`, which was used to create a reproducible `5`-sample blur-only subset covering:
243
+
244
+ - paragraph
245
+ - table
246
+ - form
247
+ - figure
248
+ - multi-page report
249
+
250
+ That subset was benchmarked on `k2` across the real baseline set:
251
+
252
+ - `candidate` using `outputs/script-router-backfill-calibration/model-card.json`
253
+ - `tesseract`
254
+ - `surya`
255
+ - `stock-paddle`
256
+ - `glm-ocr` via the isolated `.venv-glm` hook
257
+ - `paddleocr-vl`
258
+
259
+ All `30` runs completed successfully. The pulled local artifacts are:
260
+
261
+ - `outputs/live-k2-rendered-blur5/report.json`
262
+ - `outputs/live-k2-rendered-blur5/summary.json`
263
+
264
+ Aggregate OCR/text result on that subset:
265
+
266
+ - candidate CER: `0.223273`
267
+ - Surya CER: `0.197601`
268
+ - Tesseract CER: `0.732364`
269
+ - stock-Paddle CER: `2.810426`
270
+ - GLM-OCR CER: `2.820269`
271
+ - PaddleOCR-VL CER: `18.470663`
272
+
273
+ Aggregate structure result on that subset:
274
+
275
+ - candidate `table_cell_f1`: `1.0`
276
+ - Surya `table_cell_f1`: `1.0`
277
+ - Tesseract / stock-Paddle / GLM-OCR / PaddleOCR-VL `table_cell_f1`: `0.0`
278
+
279
+ Aggregate reading-order result on that subset:
280
+
281
+ - candidate: `0.783636`
282
+ - Tesseract: `0.823636`
283
+ - Surya: `0.750909`
284
+ - stock-Paddle: `1.0`
285
+ - GLM-OCR: `1.0`
286
+ - PaddleOCR-VL: `0.750909`
287
+
288
+ That subset is exploratory, not claim-grade. But it is useful because it removes more guesswork:
289
+
290
+ - the current candidate is much better than Tesseract, stock-Paddle, GLM-OCR, and PaddleOCR-VL on blur-heavy rendered OCR CER
291
+ - the current candidate still does not beat raw Surya on text CER
292
+ - the current candidate still trades away some reading-order quality relative to Tesseract and the layout-heavy baselines
293
+
294
+ For paper use, the right interpretation is narrower than "our model wins." This subset supports a more defensible statement: the current backfill candidate is a strong internal engineering baseline on degraded rendered documents, especially for table recovery, but raw Surya is still the stronger text OCR baseline on the same subset.
295
+
296
+ Additional pulled artifacts for that `25`-sample backfill run:
297
+
298
+ - `outputs/live-k2-claimpack-rendered-backfill/calibration.json`
299
+ - `outputs/live-k2-claimpack-rendered-backfill/model-card.json`
300
+ - `outputs/live-k2-claimpack-rendered-backfill/experiment.json`
301
+ - `outputs/live-k2-claimpack-rendered-backfill/report.json`
302
+ - `outputs/live-k2-claimpack-rendered-backfill/summary.json`
303
+ - `outputs/live-k2-claimpack-rendered-backfill/summary.md`
304
+
305
+ Key numbers versus raw Tesseract on that rendered claim-ready pack:
306
+
307
+ | Slice | Candidate CER | Tesseract CER | Notes |
308
+ | --- | ---: | ---: | --- |
309
+ | all | 0.248654 | 0.689214 | paired CER `9` wins, `1` loss, `15` ties |
310
+ | clean | 0.243224 | 0.540584 | same win pattern as smoke pack |
311
+ | blur | 0.260039 | 0.991613 | strongest degraded-slice text gain |
312
+ | scan | 0.237424 | 0.593631 | clear text win |
313
+ | low_contrast | 0.263265 | 0.554174 | smaller but still positive |
314
+ | skew | 0.239317 | 0.766067 | one loss, two wins, two ties |
315
+ | multi_page | 0.315789 | 0.315789 | CER tie |
316
+ | table | 0.339713 | 1.421531 | large text gain on table-heavy pages |
317
+
318
+ Structure metrics on the same pack matter more than the CER win:
319
+
320
+ - candidate aggregate `table_cell_f1`: `0.955556`
321
+ - Tesseract aggregate `table_cell_f1`: `0.0`
322
+ - candidate aggregate `table_exact_match`: `0.8`
323
+ - Tesseract aggregate `table_exact_match`: `0.0`
324
+ - candidate `multi_page` `table_cell_f1`: `0.911111`
325
+ - Tesseract `multi_page` `table_cell_f1`: `0.0`
326
+
327
+ This is better evidence than the tiny smoke pack, but it still is not a clean claim story. Two real weaknesses remain on this rendered pack:
328
+
329
+ - `form` CER regressed slightly: `0.231933` vs `0.230010`
330
+ - aggregate `reading_order_pair_accuracy` is lower than Tesseract: `0.764` vs `0.816`
331
+
332
+ So the honest reading is narrow: the calibrated Tesseract-plus-Surya backfill router is a strong internal rendered-pack baseline for OCR robustness and table extraction, but it is not yet a broad document-understanding winner.
333
+
334
+ We then ran two more comparisons on that same rendered claim-ready pack to stop overfitting the story to Tesseract.
335
+
336
+ First, raw Surya on the rendered pack was not the disaster it had been on the older line-crop holdout. On the `25`-sample rendered document pack it beat the calibrated backfill router on text CER across every checked slice:
337
+
338
+ | Slice | Candidate CER | Raw Surya CER |
339
+ | --- | ---: | ---: |
340
+ | all | 0.248654 | 0.195260 |
341
+ | blur | 0.260039 | 0.202626 |
342
+ | scan | 0.237424 | 0.191862 |
343
+ | low_contrast | 0.263265 | 0.191862 |
344
+ | skew | 0.239317 | 0.194733 |
345
+ | form | 0.231933 | 0.138462 |
346
+ | multi_page | 0.315789 | 0.200000 |
347
+ | table | 0.339713 | 0.281818 |
348
+
349
+ Paired CER was not close: `0` wins, `15` losses, `10` ties for the candidate on `all`.
350
+
351
+ That does not mean the router is useless. Structure on the same pack is more mixed:
352
+
353
+ - aggregate `table_cell_f1`: candidate `0.955556`, Surya `0.955556`
354
+ - aggregate `reading_order_pair_accuracy`: candidate `0.764000`, Surya `0.750909`
355
+ - paired reading-order wins on `all`: candidate `2`, losses `0`, ties `23`
356
+
357
+ So the rendered document-pack conclusion is stricter than the earlier line-crop conclusion: the current routed candidate is still behind raw Surya on text OCR, while only slightly ahead on reading-order behavior and roughly tied on table structure.
358
+
359
+ Second, `stock-paddle` stopped being a fake baseline. The original `k2` benchmark result showing `1.0` CER everywhere was an adapter bug in this repo, not a real PaddleOCR result: `paddleocr 3.7.0` on `k2` returns v3 `OCRResult` objects, and the old parser only understood the older tuple/list shape. After fixing that adapter and installing `paddlepaddle 3.3.1`, the stock-Paddle baseline became runnable on `k2`.
360
+
361
+ With that fix in place, the same rendered claim-ready pack produced:
362
+
363
+ | Slice | Candidate CER | Stock-Paddle CER |
364
+ | --- | ---: | ---: |
365
+ | all | 0.248654 | 1.749081 |
366
+ | blur | 0.260039 | 1.549743 |
367
+ | scan | 0.237424 | 1.896558 |
368
+ | low_contrast | 0.263265 | 1.854006 |
369
+ | skew | 0.239317 | 1.568643 |
370
+ | form | 0.231933 | 0.470694 |
371
+ | multi_page | 0.315789 | 0.391388 |
372
+ | table | 0.339713 | 3.395694 |
373
+
374
+ Paired CER on `all` was `21` wins, `4` losses, `0` ties for the candidate.
375
+
376
+ Structure was lopsided in the other direction:
377
+
378
+ - candidate aggregate `table_cell_f1`: `0.955556`
379
+ - stock-Paddle aggregate `table_cell_f1`: `0.0`
380
+ - candidate aggregate `reading_order_pair_accuracy`: `0.764000`
381
+ - stock-Paddle aggregate `reading_order_pair_accuracy`: `1.0`
382
+
383
+ So stock-Paddle, as currently configured here with `PaddleOCR(lang='ne')`, preserves reading order very aggressively but is not competitive on text accuracy or table extraction for this mixed rendered pack.
384
+
385
+ The stricter rendered-document result is now also explicit. On `k2`, a larger `claimpack-rendered-v2` split was calibrated separately and evaluated on an untouched `50`-sample holdout. The routed script candidate selected `primary_script_threshold=0.0` on calibration, which means it collapsed to "always pick Surya" there. On the holdout:
386
+
387
+ | Baseline | Overall CER | table_cell_f1 | reading_order_pair_accuracy |
388
+ | --- | ---: | ---: | ---: |
389
+ | candidate | 0.180893 | 0.947500 | 0.750909 |
390
+ | raw Surya | 0.180893 | 0.970833 | 0.750909 |
391
+ | raw Tesseract | 0.714207 | 0.000000 | 0.787818 |
392
+
393
+ Pulled local artifacts:
394
+
395
+ - `outputs/live-k2-claimpack-v2-script-router/calibration.json`
396
+ - `outputs/live-k2-claimpack-v2-script-router/model-card.json`
397
+ - `outputs/live-k2-claimpack-v2-script-router/experiment.json`
398
+ - `outputs/live-k2-claimpack-v2-script-router/summary.json`
399
+ - `outputs/live-k2-claimpack-v2-script-router/preflight.json`
400
+ - `outputs/paper-claimpack-v2-table/paper-table.md`
401
+
402
+ This is not a routed-model win. It is a branch ceiling. The candidate does not beat Surya on text, ties it exactly on CER, and is slightly worse on table structure. The new claim guardrail now rejects this case directly: `preflight-claim` fails `candidate-calibration` when a routed candidate collapses to one engine on every calibration sample.
403
+
404
+ We also tested the obvious opposite branch after adding Latin-script routing support to the repo: a Surya-primary page router with Tesseract fallback for English-heavy pages. The rendered-v2 calibration on `k2` selected `primary_script_threshold=0.7`, and the holdout result was unambiguously bad:
405
+
406
+ | Baseline | Overall CER | table_cell_f1 | reading_order_pair_accuracy |
407
+ | --- | ---: | ---: | ---: |
408
+ | candidate | 0.416854 | 0.337500 | 0.761818 |
409
+ | raw Surya | 0.180893 | 0.970833 | 0.750909 |
410
+ | raw Tesseract | 0.714207 | 0.000000 | 0.787818 |
411
+
412
+ Pulled local artifacts:
413
+
414
+ - `outputs/live-k2-claimpack-v2-surya-latin-router/calibration.json`
415
+ - `outputs/live-k2-claimpack-v2-surya-latin-router/experiment.json`
416
+ - `outputs/live-k2-claimpack-v2-surya-latin-router/report.json`
417
+ - `outputs/live-k2-claimpack-v2-surya-latin-router/summary.json`
418
+ - `outputs/paper-claimpack-v2-surya-latin-table/paper-table.md`
419
+
420
+ Paired CER versus raw Surya on that holdout was `0` wins, `12` losses, `38` ties. So the inverted script router is not a refinement. It is a disproved branch.
421
+
422
+ We then tested the narrower quality-based version of the same idea: keep Tesseract and Surya as components, but let the router pick based on the local `quality_score` signal instead of script ratio. The rendered-v2 calibration selected `selected_threshold=0.0`, which already tells you the calibration split did not find a real decision boundary beyond "prefer Surya whenever it is not clearly worse." On the untouched holdout:
423
+
424
+ | Baseline | Overall CER | table_cell_f1 | reading_order_pair_accuracy |
425
+ | --- | ---: | ---: | ---: |
426
+ | candidate | 0.181128 | 0.947500 | 0.750909 |
427
+ | raw Surya | 0.180893 | 0.947500 | 0.750909 |
428
+ | raw Tesseract | 0.714207 | 0.000000 | 0.787818 |
429
+
430
+ Pulled local artifacts:
431
+
432
+ - `outputs/live-k2-claimpack-v2-surya-quality-router/calibration.json`
433
+ - `outputs/live-k2-claimpack-v2-surya-quality-router/experiment.json`
434
+ - `outputs/live-k2-claimpack-v2-surya-quality-router/report.json`
435
+ - `outputs/live-k2-claimpack-v2-surya-quality-router/summary.json`
436
+ - `outputs/paper-claimpack-v2-surya-quality-router-table/paper-table.md`
437
+
438
+ Paired CER versus raw Surya on that holdout was `0` wins, `1` loss, `49` ties. So this is not the missing refinement either. It is nearly indistinguishable from Surya and still not better.
439
+
440
+ We also tested a non-routing composition on the same rendered-v2 holdout: `line_align_composite`. The idea was simple enough to be worth disproving. Keep Surya as the text engine, reuse Tesseract line scaffolding, and see whether better ordering helps enough to justify the merge. It did not:
441
+
442
+ | Baseline | Overall CER | table_cell_f1 | reading_order_pair_accuracy |
443
+ | --- | ---: | ---: | ---: |
444
+ | candidate | 0.310549 | 0.897500 | 0.792727 |
445
+ | raw Surya | 0.180893 | 0.970833 | 0.750909 |
446
+ | raw Tesseract | 0.714207 | 0.000000 | 0.787818 |
447
+
448
+ Pulled local artifacts:
449
+
450
+ - `outputs/live-k2-claimpack-v2-line-align-summary/summary.json`
451
+ - `outputs/paper-claimpack-v2-line-align-table/paper-table.md`
452
+ - `outputs/live-k2-claimpack-v2-line-align-model-card.json`
453
+
454
+ Paired CER versus raw Surya on that holdout was `0` wins, `7` losses, `43` ties. The candidate did improve reading-order accuracy relative to raw Surya, and even edged past Tesseract there, but the text and table regressions are too large to defend. So line alignment is another useful negative result, not a candidate answer.
455
+
456
+ Finally, we measured the sample-level upper bound for the same component family using `benchmark-oracle`. This is not a deployable candidate. It chooses the better of raw Surya and raw Tesseract per sample using ground-truth CER. That is the right diagnostic question before wasting more time on routing heuristics:
457
+
458
+ | System | Overall CER | table_cell_f1 | reading_order_pair_accuracy |
459
+ | --- | ---: | ---: | ---: |
460
+ | Surya/Tesseract oracle | 0.180529 | 0.970833 | 0.750909 |
461
+ | raw Surya | 0.180893 | 0.970833 | 0.750909 |
462
+ | raw Tesseract | 0.714207 | 0.000000 | 0.787818 |
463
+
464
+ Pulled/generated local artifacts:
465
+
466
+ - `outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-analysis.json`
467
+ - `outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-choices.json`
468
+ - `outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-summary/summary.json`
469
+ - `outputs/paper-claimpack-v2-oracle-table/paper-table.md`
470
+
471
+ The oracle picked raw Surya for `49` samples and raw Tesseract for `1`. Those per-sample source choices are stored in `oracle-choices.json` as provenance, not mixed into benchmark metrics. Paired CER versus raw Surya was `1` win, `0` losses, `49` ties. That means the current Surya/Tesseract routing space is effectively exhausted for text OCR on rendered-v2. The next useful model-side work needs a stronger recognizer or document model, not another selector over these two engines.
472
+
473
+ We then added `benchmark-error-analysis` to convert that failure into a training target instead of another routing guess. On the same rendered-v2 benchmark report, raw Surya's highest absolute CER slices are `figure` (`0.253157`) and `table` (`0.252781`). The hardest individual samples are dominated by table-ledger and figure-caption pages. The Surya-vs-oracle diagnostic confirms that Tesseract only improves one sample, `eval-form-notice-003-blur`, by `0.018182` CER. So the next serious model branch should target table/figure-heavy mixed-script recognition and document context, while treating Surya/Tesseract routing as exhausted.
474
+
475
+ We also exported those ranked failures back into a real manifest instead of a hand-maintained list. `data/splits-claimpack-rendered-v2/eval-surya-hard15.jsonl` is a `15`-sample `hard_eval` manifest built from the top Surya CER failures. It keeps the original reference metadata and adds `metadata.failure_analysis` with rank, metric, value, and comparison details. The local checkout does not include the rendered image assets, so the local audit reports missing image paths; the same manifest passes full file/reference audit on `k2` at `outputs/audit-claimpack-v2-surya-hard15/manifest-audit.json`, pulled locally to `outputs/audit-claimpack-v2-surya-hard15-k2/manifest-audit.json`.
476
+
477
+ The hard pocket now has an explicit validation config: `configs/validation-hard15-surya.json`. This uses conjunctive gate slices such as `["hard_eval", "table"]` and `["hard_eval", "figure"]`, so the table and figure gates cannot be satisfied by easier non-hard samples. As a negative control, the current line-align candidate fails this config on `outputs/live-k2-claimpack-v2-line-align/report.json`: hard-eval CER is `0.482039` versus Surya `0.348706`, hard-table CER is `0.613636` versus Surya `0.363636`, and the figure/table structure gates do not improve over Surya.
478
+
479
+ For model iteration, we generated a separate table/figure development pack on `k2`:
480
+
481
+ - manifest: `data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl`
482
+ - audit: `outputs/audit-eval-pack-table-figure-dev-v1-k2/eval-pack-audit.json`
483
+ - samples: `80`
484
+ - slices: `40` table, `40` figure, `80` Nepali/English/mixed-script/image/reading_order
485
+ - degradations: `clean`, `scan`, `blur`, `low_contrast`, `skew`
486
+ - hard15 overlap: `0` sample IDs and `0` text hashes
487
+
488
+ This pack was generated with `--template table-ledger --template figure-caption --variant-offset 100`, after updating the eval-pack generator so table OCR references include visible table rows and figure variants include copy-specific visible text. Treat this as a development/training target, not as held-out claim evidence.
489
+
490
+ The pack was then split with text-level grouping so degraded variants of the same target do not leak across train/eval:
491
+
492
+ ```bash
493
+ ocrtech split-manifest data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl \
494
+ --out data/splits-table-figure-dev-v1 \
495
+ --eval-ratio 0.2 \
496
+ --seed 17 \
497
+ --stratify-by document_type \
498
+ --group-by text_sha256
499
+ ```
500
+
501
+ Verified split:
502
+
503
+ - train: `60` samples
504
+ - eval: `20` samples
505
+ - text groups: `16`
506
+ - leakage check: `pass`
507
+ - train audit on `k2`: `pass`
508
+ - eval audit on `k2`: `pass`
509
+
510
+ Additional leakage checks on `k2` found no sample-id or text-hash overlap with `hard15` or rendered-v2 eval. That makes this a legitimate development pack for the next model branch, but it is still not claim evidence.
511
+
512
+ The first model-side attempt on this branch was a minimal Hugging Face VisionEncoderDecoder fine-tune using `microsoft/trocr-small-printed`:
513
+
514
+ ```bash
515
+ ocrtech train-recognizer \
516
+ --backend hf-vision-encoder-decoder \
517
+ --base-model microsoft/trocr-small-printed \
518
+ --train-manifest data/splits-table-figure-dev-v1/train.jsonl \
519
+ --eval-manifest data/splits-table-figure-dev-v1/eval.jsonl \
520
+ --out runs/hf-table-figure-dev-v1 \
521
+ --epochs 1 \
522
+ --train-batch-size 4 \
523
+ --eval-batch-size 4 \
524
+ --learning-rate 5e-5 \
525
+ --run
526
+ ```
527
+
528
+ Training completed and the package audited cleanly, but the result was a decisive negative:
529
+
530
+ | metric | value |
531
+ | --- | ---: |
532
+ | eval_loss | 7.884517 |
533
+ | eval_cer | 3.062119 |
534
+ | eval_wer | 3.562689 |
535
+ | eval_exact_line_accuracy | 0.000000 |
536
+
537
+ Benchmarking the packaged candidate against Surya and Tesseract on the `20`-sample dev eval confirmed the branch is not viable:
538
+
539
+ | system | CER | table_cell_f1 | reading_order_pair_accuracy |
540
+ | --- | ---: | ---: | ---: |
541
+ | HF table/figure candidate | 2.422001 | 0.000000 | 0.000000 |
542
+ | Surya | 0.563399 | 0.858333 | 0.550000 |
543
+ | Tesseract | 0.546227 | 0.000000 | 0.695000 |
544
+
545
+ Paired CER was `0` wins, `20` losses, and `0` ties against both Surya and Tesseract. The failure mode is not subtle: naive full-page TrOCR fine-tuning does not learn table/figure document recognition from this small pack. The next credible recognizer branch should train on line/cell crops or use a page/document model with structure-aware supervision.
546
+
547
+ We then implemented that crop-level path and tested it instead of leaving it as advice. Newly generated image-pack references now carry renderer layout metadata for non-skew pages, and `ocrtech extract-crops` turns those verified bboxes into ordinary recognizer manifests. Skew pages are skipped because pre-rotation bboxes are invalid.
548
+
549
+ Full crop development pack on `k2`:
550
+
551
+ - source pack: `data/eval-pack-table-figure-crops-dev-v1/manifests/ocrtech-eval-pack.jsonl`
552
+ - crop manifest: `data/crops-table-figure-dev-v1/manifests/crops.jsonl`
553
+ - source samples: `80`
554
+ - crops: `544`
555
+ - crop types: `384` table cells, `128` lines, `32` figure captions
556
+ - skipped candidates: `32` skew pages
557
+ - crop audit: `outputs/audit-crops-table-figure-dev-v1/manifest-audit.md`
558
+ - split: `480` train / `64` eval
559
+ - split grouping: `text_sha256`
560
+ - split leakage: pass
561
+
562
+ The first crop-level HF model used the same `microsoft/trocr-small-printed` backend for one epoch, this time on `480` crop images instead of full pages. Trainer metrics improved relative to the full-page branch, but still were not competitive:
563
+
564
+ | metric | value |
565
+ | --- | ---: |
566
+ | eval_loss | 3.087561 |
567
+ | eval_cer | 0.959831 |
568
+ | eval_wer | 1.363715 |
569
+ | eval_exact_line_accuracy | 0.078125 |
570
+
571
+ Packaged crop model audit passed:
572
+
573
+ - `runs/hf-crops-table-figure-dev-v1/training_metrics.json`
574
+ - `runs/hf-crops-table-figure-dev-v1/package/model-card.json`
575
+ - `outputs/audit-hf-crops-table-figure-dev-v1-model/model-audit.md`
576
+
577
+ Benchmark result on the `64`-sample crop eval:
578
+
579
+ | system | CER |
580
+ | --- | ---: |
581
+ | HF crop candidate | 1.043369 |
582
+ | Surya | 0.475309 |
583
+ | Tesseract | 0.077453 |
584
+
585
+ Paired CER:
586
+
587
+ - versus Surya: `7` wins, `50` losses, `7` ties
588
+ - versus Tesseract: `0` wins, `64` losses, `0` ties
589
+
590
+ This means crop-level training is the right data shape, but this tiny HF recognizer is still the wrong recognizer. The immediate technical takeaway is that Tesseract is extremely strong on these generated crop images; the next branch should use strong crop OCR behind better layout/cell/caption detection, or train a more suitable recognizer with substantially more real crop data.
591
+
592
+ The next composition test used that crop-OCR signal more directly: `table_cell_refine_composite` keeps Surya as the document/table structure engine and rereads detected table cells with Tesseract crops.
593
+
594
+ Two hard15 variants were tested:
595
+
596
+ | variant | hard_eval CER | table_cell_f1 | reading_order_pair_accuracy | verdict |
597
+ | --- | ---: | ---: | ---: | --- |
598
+ | Surya baseline | 0.348706 | 0.979167 | 0.520000 | current hard15 target |
599
+ | `replacement_policy=always` | 0.348706 | 0.593750 | 0.520000 | unsafe; damages good Surya cells |
600
+ | `replacement_policy=empty` | 0.348706 | 0.979167 | 0.520000 | safe tie; no improvement |
601
+ | `replacement_policy=empty`, table de-dup | 0.348706 | 0.979167 | 0.520000 | parser hygiene fix; still no improvement |
602
+
603
+ The safe policy also failed the hard15 validation gate with status `not_validated`. This is another useful negative result: Tesseract crop OCR is strong when the crop is exact, but naively applying it to Surya-inferred table cells does not improve Surya on this hard pocket.
604
+
605
+ Cell-level analysis explains why:
606
+
607
+ | system | reference_cells | exact_cells | exact_rate | mean_cer |
608
+ | --- | ---: | ---: | ---: | ---: |
609
+ | Surya | 96 | 94 | 0.979167 | 0.010417 |
610
+ | table-cell refine, `always` | 96 | 57 | 0.593750 | 0.284309 |
611
+ | table-cell refine, `empty` | 96 | 94 | 0.979167 | 0.010417 |
612
+
613
+ Against Surya, the unsafe `always` policy improved `0` cells, worsened `37`, and tied `59`. The safe `empty` policy tied all `96` cells because Surya had no missing cells to fill in this hard pocket. The remaining Surya cell errors are only two occurrences of `किरण` predicted as `करिण`, both on the same table-ledger variant. The next table branch needs a targeted confidence/replacement signal for rare wrong-but-nonempty cells; blind crop rerecognition is the wrong direction.
614
+
615
+ The table de-duplication branch fixed a separate parser hygiene issue: line-based table inference no longer creates a second table when an engine already supplied an overlapping structured table. On the rerun artifact, both candidate and Surya now report `8` predicted tables for `8` reference tables and `0` duplicate overlapping tables. This is useful for cleaner exports, but because the fix applies at parser assembly time it also benefits the Surya baseline; hard15 validation remains `not_validated`.
616
+
617
+ Generated diagnostics:
618
+
619
+ - `outputs/paper-claimpack-v2-surya-error-analysis/error-analysis.md`
620
+ - `outputs/paper-claimpack-v2-surya-error-analysis/top-failures.json`
621
+ - `outputs/paper-claimpack-v2-surya-vs-tesseract-error-analysis/error-analysis.md`
622
+ - `outputs/paper-claimpack-v2-surya-vs-oracle-error-analysis/error-analysis.md`
623
+ - `data/splits-claimpack-rendered-v2/eval-surya-hard15.jsonl`
624
+ - `configs/validation-hard15-surya.json`
625
+ - `outputs/validation-claimpack-v2-line-align-hard15/validation.json`
626
+ - `outputs/audit-claimpack-v2-surya-hard15-k2/manifest-audit.json`
627
+ - `data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl`
628
+ - `outputs/audit-eval-pack-table-figure-dev-v1-k2/eval-pack-audit.json`
629
+ - `data/splits-table-figure-dev-v1/split-summary.md`
630
+ - `outputs/audit-table-figure-dev-v1-train-k2/manifest-audit.md`
631
+ - `outputs/audit-table-figure-dev-v1-eval-k2/manifest-audit.md`
632
+ - `runs/hf-table-figure-dev-v1/training_metrics.json`
633
+ - `runs/hf-table-figure-dev-v1/package/model-card.json`
634
+ - `outputs/audit-hf-table-figure-dev-v1-model/model-audit.md`
635
+ - `outputs/summary-hf-table-figure-dev-v1/summary.md`
636
+ - `outputs/experiment-hf-table-figure-dev-v1/benchmark/report.json`
637
+ - `outputs/crops-table-figure-dev-v1-k2/crop-manifest-summary.md`
638
+ - `outputs/crops-table-figure-dev-v1-k2/manifest-audit.md`
639
+ - `data/splits-crops-table-figure-dev-v1/split-summary.md`
640
+ - `runs/hf-crops-table-figure-dev-v1/training_metrics.json`
641
+ - `runs/hf-crops-table-figure-dev-v1/package/model-card.json`
642
+ - `outputs/audit-hf-crops-table-figure-dev-v1-model/model-audit.md`
643
+ - `outputs/summary-hf-crops-table-figure-dev-v1/summary.md`
644
+ - `outputs/experiment-hf-crops-table-figure-dev-v1/benchmark/report.json`
645
+ - `outputs/table-cell-refine-v1/model-card.json`
646
+ - `outputs/summary-table-cell-refine-hard15-v1/summary.md`
647
+ - `outputs/table-cell-refine-empty-v1/model-card.json`
648
+ - `outputs/summary-table-cell-refine-empty-hard15-v1/summary.md`
649
+ - `outputs/validation-table-cell-refine-empty-hard15-v1/validation.md`
650
+ - `outputs/table-cell-analysis-refine-always-hard15-v1/table-cell-analysis.md`
651
+ - `outputs/table-cell-analysis-refine-empty-hard15-v1/table-cell-analysis.md`
652
+ - `outputs/table-cell-refine-empty-dedup-v1/model-card.json`
653
+ - `outputs/summary-table-cell-refine-empty-dedup-hard15-v3/summary.md`
654
+ - `outputs/table-cell-analysis-refine-empty-dedup-hard15-v3/table-cell-analysis.md`
655
+ - `outputs/validation-table-cell-refine-empty-dedup-hard15-v3/validation.md`
656
+
657
+ When repeating this workflow, keep the comparison honest:
658
+
659
+ ```bash
660
+ ocrtech prepare-data gauravgiri/nepali-ocr-dataset --hf --split train --limit 100 --skip-invalid --out data/manifests
661
+ ocrtech prepare-data prashant0919/nepali-synthetic-ocr-lines --hf --split train --limit 100 --out data/manifests
662
+ ocrtech prepare-data nvidia/OCR-Synthetic-Multilingual-v1 --hf --hf-subset en --split test --limit 1000 --out data/manifests-english-test
663
+ ocrtech merge-manifests \
664
+ data/manifests/gauravgiri__nepali-ocr-dataset-train.jsonl \
665
+ data/manifests/prashant0919__nepali-synthetic-ocr-lines-train.jsonl \
666
+ data/manifests-english-test/nvidia__OCR-Synthetic-Multilingual-v1__en-test.jsonl \
667
+ --out data/manifests/nep-eng-mixed-v1.jsonl
668
+ ocrtech split-manifest data/manifests/nep-eng-mixed-v1.jsonl --out data/splits-mixed-v1 --eval-ratio 0.15 --seed 13
669
+ ocrtech materialize-references data/splits-mixed-v1/eval.jsonl --out data/references-mixed-v1 --rewritten-manifest data/splits-mixed-v1/eval-with-refs.jsonl
670
+ ocrtech rebalance-manifest data/splits-mixed-v1/train.jsonl --out data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl --slice english --slice nepali --seed 13
671
+ ocrtech train-recognizer --backend hf-vision-encoder-decoder --base-model microsoft/trocr-small-printed --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl --out runs/hf-recognizer-mixed-v1-rebalanced --train-batch-size 4 --eval-batch-size 4 --epochs 0.5 --run
672
+ ocrtech package-hf-recognizer --model-dir runs/hf-recognizer-mixed-v1-rebalanced/final-model --out runs/hf-recognizer-mixed-v1-rebalanced/package --model-id ocrtech-hf-mixed-v1-rebalanced --base-model microsoft/trocr-small-printed --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl --training-summary runs/hf-recognizer-mixed-v1-rebalanced/recognizer-training-summary.json --metrics-report runs/hf-recognizer-mixed-v1-rebalanced/training_metrics.json
673
+ ocrtech run-experiment --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl --out outputs/experiment-hf-mixed-v1-rebalanced --baselines candidate --candidate-model-config runs/hf-recognizer-mixed-v1-rebalanced/package/model-card.json --references-dir data/references-mixed-v1 --skip-validation
674
+ ```
675
+
676
+ ## Figure Caption Refinement Composite
677
+
678
+ The figure-caption hard15 slice exposed a parser problem first: after `[Figure: ...]`, the actual caption line was being rejected when OCR damaged the Nepali caption keyword (`चित्र` became variants such as `चतुर्थ` or `चत्रि`). The parser now accepts a nearby line after a figure marker as the caption if it is spatially close and not another marker/table line. That reduced Surya/candidate figure-caption CER on hard15 from about `0.737609` to `0.092323`.
679
+
680
+ The next branch used that cleaner structure and tried a narrow composite: keep Surya as the primary document engine, but replace only overlapping figure-caption lines with Tesseract when the Tesseract line has a better caption-prefix score. The replacement keeps Surya's layout bbox so skewed Tesseract boxes do not break caption attachment.
681
+
682
+ Hard15 v2 result:
683
+
684
+ | system | CER | figure_caption_cer | figure_detection_f1 | table_cell_f1 | reading_order_pair_accuracy |
685
+ | --- | ---: | ---: | ---: | ---: | ---: |
686
+ | Surya | 0.236839 | 0.092323 | 1.000000 | 0.979167 | 0.520000 |
687
+ | Tesseract | 1.266364 | 0.051020 | 0.857143 | 0.000000 | 0.613333 |
688
+ | figure-caption refine | 0.228008 | 0.052478 | 1.000000 | 0.979167 | 0.520000 |
689
+
690
+ Validation against the hard15 Surya config is still `not_validated`, but one gate now passes:
691
+
692
+ - `hard_eval,figure` `figure_caption_cer`: pass, `6` wins / `0` losses / `1` tie, candidate `0.052478`, Surya `0.092323`, improvement `0.039845`
693
+ - overall hard_eval CER: fail, improvement `0.008830 < 0.020000`
694
+ - hard_eval figure CER: fail, improvement `0.018922 < 0.020000`
695
+ - table CER/F1 gates: fail, ties only
696
+
697
+ Artifacts:
698
+
699
+ - `outputs/figure-caption-refine-v1/model-card.json`
700
+ - `outputs/audit-figure-caption-refine-v1-model/model-audit.md`
701
+ - `outputs/summary-figure-caption-refine-hard15-v2/summary.md`
702
+ - `outputs/experiment-figure-caption-refine-hard15-v2/benchmark/report.json`
703
+ - `outputs/validation-figure-caption-refine-hard15-v2/validation.md`
704
+
705
+ This is the first branch in this sequence that beats Surya on a configured gate, but it is a slice win, not a SOTA claim.
706
+
707
+ The stronger opt-in structural-label branch adds edit-distance guarded correction for document-role labels (`वार्षिक प्रतिवेदन`, `चित्र`, `निष्कर्ष`) after caption replacement:
708
+
709
+ | system | CER | figure_caption_cer | figure_detection_f1 | table_cell_f1 | reading_order_pair_accuracy |
710
+ | --- | ---: | ---: | ---: | ---: | ---: |
711
+ | Surya | 0.236839 | 0.092323 | 1.000000 | 0.979167 | 0.520000 |
712
+ | figure-caption refine + labels | 0.204610 | 0.008746 | 1.000000 | 0.979167 | 0.520000 |
713
+
714
+ Validation still reports `not_validated`, but the pass/fail split is materially better:
715
+
716
+ - overall hard_eval CER: pass, `7` wins / `0` losses / `8` ties, candidate `0.204610`, Surya `0.236839`, improvement `0.032228`
717
+ - `hard_eval,figure` CER: pass, `7` wins / `0` losses / `0` ties, candidate `0.022866`, Surya `0.091927`, improvement `0.069061`
718
+ - `hard_eval,figure` `figure_caption_cer`: pass, `7` wins / `0` losses / `0` ties, candidate `0.008746`, Surya `0.092323`, improvement `0.083576`
719
+ - table CER/F1 gates: fail, ties only
720
+
721
+ Artifacts:
722
+
723
+ - `outputs/figure-caption-refine-labels-v1/model-card.json`
724
+ - `outputs/audit-figure-caption-refine-labels-v1-model/model-audit.md`
725
+ - `outputs/summary-figure-caption-refine-labels-hard15-v1/summary.md`
726
+ - `outputs/experiment-figure-caption-refine-labels-hard15-v1/benchmark/report.json`
727
+ - `outputs/validation-figure-caption-refine-labels-hard15-v1/validation.md`
728
+
729
+ The next hard15 candidate added two explicitly auditable pieces on top of that branch:
730
+
731
+ - a small curated table-name lexicon, `configs/nepali-table-name-lexicon-v1.txt`, used only for cells under a header close to `नाम`
732
+ - a guarded structural-title correction for the recurring table title `दैनिक हिसाब`
733
+
734
+ This v3 run is the strongest internal hard15 result so far:
735
+
736
+ | system | CER | figure_caption_cer | figure_detection_f1 | table_cell_f1 | reading_order_pair_accuracy |
737
+ | --- | ---: | ---: | ---: | ---: | ---: |
738
+ | Surya | 0.236839 | 0.092323 | 1.000000 | 0.979167 | 0.520000 |
739
+ | Tesseract | 1.266364 | 0.051020 | 0.857143 | 0.000000 | 0.613333 |
740
+ | figure-caption + labels + table lexicon | 0.010671 | 0.008746 | 1.000000 | 1.000000 | 0.520000 |
741
+
742
+ Validation remains `not_validated`, but only one configured gate now fails:
743
+
744
+ - overall hard_eval CER: pass, `15` wins / `0` losses / `0` ties, candidate `0.010671`, Surya `0.236839`
745
+ - `hard_eval,table` CER: pass, `8` wins / `0` losses / `0` ties, candidate `0.000000`, Surya `0.363636`
746
+ - `hard_eval,figure` CER: pass, `7` wins / `0` losses / `0` ties, candidate `0.022866`, Surya `0.091927`
747
+ - `hard_eval,figure` `figure_caption_cer`: pass, `7` wins / `0` losses / `0` ties, candidate `0.008746`, Surya `0.092323`
748
+ - `hard_eval,table` `table_cell_f1`: fail, candidate `1.000000`, Surya `0.979167`, improvement `0.020833`, but bootstrap CI low is `0.000000` because the 8-page table slice has only 2 improving table pages and 6 ties
749
+
750
+ Cell-level analysis is still useful evidence:
751
+
752
+ - candidate: `96/96` exact cells, mean cell CER `0.000000`
753
+ - Surya: `94/96` exact cells, mean cell CER `0.010417`
754
+ - Tesseract full-page parser: `0/96` structured cells
755
+ - paired candidate-vs-Surya cells: `2` candidate-better, `0` Surya-better, `94` ties
756
+
757
+ Artifacts:
758
+
759
+ - `outputs/figure-caption-table-lexicon-v1/model-card.json`
760
+ - `outputs/audit-figure-caption-table-lexicon-v1-model/model-audit.md`
761
+ - `outputs/experiment-figure-caption-table-lexicon-hard15-v3/benchmark/report.json`
762
+ - `outputs/summary-figure-caption-table-lexicon-hard15-v3/summary.md`
763
+ - `outputs/table-cell-analysis-figure-caption-table-lexicon-hard15-v3/table-cell-analysis.md`
764
+ - `outputs/validation-figure-caption-table-lexicon-hard15-v3/validation.md`
765
+
766
+ Important interpretation: the hard15 table-page reference `text` is the page title (`दैनिक हिसाब`), while table contents are evaluated through structured table references. A temporary v2 attempt to include table grids in `Document.text` was rejected after verifying the actual reference schema. We did not weaken validation gates after observing the result.
767
+
768
+ ## Table-Ledger Claim Pack V2
769
+
770
+ To test whether the table-cell gain survives a larger table-only pack, we generated an independent rendered image pack:
771
+
772
+ ```bash
773
+ ocrtech create-eval-pack \
774
+ --out data/eval-pack-table-ledger-claim-v1 \
775
+ --input-format image \
776
+ --template table-ledger \
777
+ --count-per-template 12 \
778
+ --variant-offset 300 \
779
+ --degradation clean \
780
+ --degradation scan \
781
+ --degradation blur \
782
+ --degradation low_contrast \
783
+ --degradation skew
784
+ ```
785
+
786
+ This produced `60` table-ledger image samples. The pack audited cleanly but is intentionally not broad claim-ready because it covers one document type.
787
+
788
+ Validation config: `configs/validation-table-ledger-surya.json`.
789
+
790
+ Validation result: `validated` after adding a grid-structure non-regression gate.
791
+
792
+ | gate | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
793
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
794
+ | table_cell_f1 | 60 | 15 | 0 | 45 | 1.000000 | 0.979167 | 0.020833 | 0.012500 |
795
+ | table_exact_match | 60 | 15 | 0 | 45 | 1.000000 | 0.750000 | 0.250000 | 0.150000 |
796
+ | table_structure_similarity | 60 | 0 | 0 | 60 | 1.000000 | 1.000000 | 0.000000 | 0.000000 |
797
+
798
+ Cell-level analysis:
799
+
800
+ | system | reference_tables | reference_cells | exact_cells | exact_rate | missing_cells | mean_cer |
801
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: |
802
+ | candidate | 60 | 720 | 720 | 1.000000 | 0 | 0.000000 |
803
+ | Surya | 60 | 720 | 705 | 0.979167 | 0 | 0.005208 |
804
+ | Tesseract | 60 | 720 | 0 | 0.000000 | 715 | 0.999868 |
805
+
806
+ Runtime on k2:
807
+
808
+ | system | mean latency seconds | median latency seconds | max latency seconds |
809
+ | --- | ---: | ---: | ---: |
810
+ | candidate | 3.017929 | 2.967943 | 5.104267 |
811
+ | Surya | 2.478481 | 2.453125 | 3.175586 |
812
+ | Tesseract | 0.146866 | 0.134735 | 0.301695 |
813
+
814
+ Artifacts:
815
+
816
+ - `data/eval-pack-table-ledger-claim-v1/manifests/ocrtech-eval-pack.jsonl`
817
+ - `outputs/audit-table-ledger-claim-v1/eval-pack-audit.md`
818
+ - `outputs/experiment-table-ledger-claim-v2/benchmark/report.json`
819
+ - `outputs/summary-table-ledger-claim-v2/summary.md`
820
+ - `outputs/table-cell-analysis-table-ledger-claim-v2/table-cell-analysis.md`
821
+ - `outputs/validation-table-ledger-claim-v2b/validation.md`
822
+
823
+ Interpretation:
824
+
825
+ - This is now statistically validated evidence that the lexicon-assisted candidate improves table cell-content extraction over Surya on generated table-ledger pages while preserving Surya's already-perfect simple grid structure.
826
+ - It is not a broad OCR claim. Page CER on this targeted pack is diagnostic only because the parser does not currently render structured table blocks into `Document.text`, while the generated references include visible table rows in `text`.
827
+ - The candidate remains lexicon-assisted; the paper must state that the table-name correction uses a small curated lexicon and suffix-preserving matching for generated IDs such as `किरण-302`.
828
+ - `table_structure_similarity` can score rowspans/colspans when references use structured cells, but this table-ledger pack is simple-grid only and the metric is still not true TEDS over merged-cell HTML trees.
829
+
830
+ The eval-pack generator now includes a `complex-table-ledger` template that writes span-aware references with a merged header (`colspan=2`). This makes the validation path capable of catching a model that gets flattened cell text right but loses table span structure. We have not yet run a k2 claim experiment on this template; that should be the next table-specific benchmark before discussing complex-table behavior in the paper.
831
+
832
+ ## Complex-Table Pack V1
833
+
834
+ We generated and benchmarked a targeted complex-table pack:
835
+
836
+ ```bash
837
+ ocrtech create-eval-pack \
838
+ --out data/eval-pack-complex-table-v1 \
839
+ --input-format image \
840
+ --template complex-table-ledger \
841
+ --count-per-template 10 \
842
+ --variant-offset 500 \
843
+ --degradation clean \
844
+ --degradation scan \
845
+ --degradation blur \
846
+ --degradation low_contrast
847
+ ```
848
+
849
+ The pack has `40` samples and span-aware references with a merged header. It audited cleanly but is targeted development evidence, not broad claim-ready coverage.
850
+
851
+ Validation config: `configs/validation-complex-table-surya.json`.
852
+
853
+ Validation result: `validated` under a deliberately modest contract: improve cell-content F1 and do not regress exact/span structure against Surya.
854
+
855
+ | gate | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
856
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
857
+ | table_cell_f1 | 40 | 8 | 0 | 32 | 0.928333 | 0.915000 | 0.013333 | 0.005000 |
858
+ | table_exact_match | 40 | 0 | 0 | 40 | 0.000000 | 0.000000 | 0.000000 | 0.000000 |
859
+ | table_structure_similarity | 40 | 0 | 0 | 40 | 0.875000 | 0.875000 | 0.000000 | 0.000000 |
860
+
861
+ Cell analysis:
862
+
863
+ | system | reference_cells | exact_cells | exact_rate | mean_cer |
864
+ | --- | ---: | ---: | ---: | ---: |
865
+ | candidate | 600 | 557 | 0.928333 | 0.067667 |
866
+ | Surya | 600 | 549 | 0.915000 | 0.071000 |
867
+ | Tesseract | 600 | 40 | 0.066667 | 0.933333 |
868
+
869
+ Artifacts:
870
+
871
+ - `data/eval-pack-complex-table-v1/manifests/ocrtech-eval-pack.jsonl`
872
+ - `outputs/audit-complex-table-v1/eval-pack-audit.md`
873
+ - `outputs/experiment-complex-table-v1/benchmark/report.json`
874
+ - `outputs/summary-complex-table-v1/summary.md`
875
+ - `outputs/table-cell-analysis-complex-table-v1/table-cell-analysis.md`
876
+ - `outputs/validation-complex-table-v1/validation.md`
877
+
878
+ Interpretation:
879
+
880
+ - The candidate improves some table cell text over Surya because the lexicon correction now works below grouped headers.
881
+ - Neither candidate nor Surya reconstructs the merged-header span; both have `table_exact_match = 0` and `table_structure_similarity = 0.875`.
882
+ - This is not a complex-table SOTA result. It is a concrete failure analysis result showing the next engineering target: preserve or infer rowspans/colspans in parser output.
883
+
884
+ ## Complex-Table Span-Inference V1
885
+
886
+ After adding repeated-header colspan inference to the candidate parser, we reran the same complex-table pack:
887
+
888
+ ```bash
889
+ ocrtech run-experiment \
890
+ --eval-manifest data/eval-pack-complex-table-v1/manifests/ocrtech-eval-pack.jsonl \
891
+ --out outputs/experiment-complex-table-span-infer-v1 \
892
+ --baselines candidate,surya,tesseract \
893
+ --candidate-model-config outputs/figure-caption-table-lexicon-v1/model-card.json \
894
+ --validation-config configs/validation-complex-table-surya.json
895
+ ```
896
+
897
+ Then:
898
+
899
+ ```bash
900
+ ocrtech summarize-benchmark \
901
+ --benchmark-report outputs/experiment-complex-table-span-infer-v1/benchmark/report.json \
902
+ --out outputs/summary-complex-table-span-infer-v1 \
903
+ --baseline surya \
904
+ --baseline tesseract \
905
+ --metric table_cell_f1 \
906
+ --metric table_exact_match \
907
+ --metric table_structure_similarity
908
+
909
+ ocrtech analyze-table-cells \
910
+ --benchmark-report outputs/experiment-complex-table-span-infer-v1/benchmark/report.json \
911
+ --eval-manifest data/eval-pack-complex-table-v1/manifests/ocrtech-eval-pack.jsonl \
912
+ --out outputs/table-cell-analysis-complex-table-span-infer-v1 \
913
+ --baseline surya \
914
+ --baseline tesseract
915
+ ```
916
+
917
+ Validation result: `validated`.
918
+
919
+ | gate | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
920
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
921
+ | table_cell_f1 | 40 | 40 | 0 | 0 | 0.995000 | 0.915000 | 0.080000 | 0.071667 |
922
+ | table_exact_match | 40 | 37 | 0 | 3 | 0.925000 | 0.000000 | 0.925000 | 0.825000 |
923
+ | table_structure_similarity | 40 | 40 | 0 | 0 | 1.000000 | 0.875000 | 0.125000 | 0.125000 |
924
+
925
+ Cell analysis:
926
+
927
+ | system | reference_cells | exact_cells | exact_rate | mean_cer |
928
+ | --- | ---: | ---: | ---: | ---: |
929
+ | candidate | 600 | 597 | 0.995000 | 0.001000 |
930
+ | Surya | 600 | 549 | 0.915000 | 0.071000 |
931
+ | Tesseract | 600 | 40 | 0.066667 | 0.933333 |
932
+
933
+ Artifacts:
934
+
935
+ - `outputs/experiment-complex-table-span-infer-v1/benchmark/report.json`
936
+ - `outputs/experiment-complex-table-span-infer-v1/validation/validation.md`
937
+ - `outputs/summary-complex-table-span-infer-v1/summary.md`
938
+ - `outputs/table-cell-analysis-complex-table-span-infer-v1/table-cell-analysis.md`
939
+
940
+ Interpretation:
941
+
942
+ - The candidate now reconstructs the generated merged header pattern via a narrow repeated-header span inference rule.
943
+ - This is a parser/postprocessing improvement, not a trained model improvement.
944
+ - It does not solve broad table understanding. We still need real scanned table benchmarks, varied spans/rowspans, and true HTML/TEDS scoring.
945
+
946
+ ## Complex-Table TEDS-Style Metrics V1
947
+
948
+ We added two deterministic ordered table-tree edit metrics:
949
+
950
+ - `table_teds_structure`: compares `table/tr/td` tree shape plus `rowspan`/`colspan`
951
+ - `table_teds_text`: also includes normalized cell text in `td` node labels
952
+
953
+ Then we reran the same complex-table pack with stricter validation gates:
954
+
955
+ ```bash
956
+ ocrtech run-experiment \
957
+ --eval-manifest data/eval-pack-complex-table-v1/manifests/ocrtech-eval-pack.jsonl \
958
+ --out outputs/experiment-complex-table-teds-v1 \
959
+ --baselines candidate,surya,tesseract \
960
+ --candidate-model-config outputs/figure-caption-table-lexicon-v1/model-card.json \
961
+ --validation-config configs/validation-complex-table-surya.json
962
+ ```
963
+
964
+ Validation result: `validated`.
965
+
966
+ | gate | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
967
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
968
+ | table_cell_f1 | 40 | 40 | 0 | 0 | 0.995000 | 0.915000 | 0.080000 | 0.071667 |
969
+ | table_exact_match | 40 | 37 | 0 | 3 | 0.925000 | 0.000000 | 0.925000 | 0.825000 |
970
+ | table_structure_similarity | 40 | 40 | 0 | 0 | 1.000000 | 0.875000 | 0.125000 | 0.125000 |
971
+ | table_teds_structure | 40 | 40 | 0 | 0 | 1.000000 | 0.904762 | 0.095238 | 0.095238 |
972
+ | table_teds_text | 40 | 40 | 0 | 0 | 0.996250 | 0.891667 | 0.104583 | 0.098690 |
973
+
974
+ Cell analysis is unchanged from the span-inference run:
975
+
976
+ | system | reference_cells | exact_cells | exact_rate | mean_cer |
977
+ | --- | ---: | ---: | ---: | ---: |
978
+ | candidate | 600 | 597 | 0.995000 | 0.001000 |
979
+ | Surya | 600 | 549 | 0.915000 | 0.071000 |
980
+ | Tesseract | 600 | 40 | 0.066667 | 0.933333 |
981
+
982
+ Artifacts:
983
+
984
+ - `outputs/experiment-complex-table-teds-v1/benchmark/report.json`
985
+ - `outputs/experiment-complex-table-teds-v1/validation/validation.md`
986
+ - `outputs/summary-complex-table-teds-v1/summary.md`
987
+ - `outputs/table-cell-analysis-complex-table-teds-v1/table-cell-analysis.md`
988
+
989
+ Interpretation:
990
+
991
+ - This is stronger than the old span-token score because it evaluates ordered table tree structure.
992
+ - It is still an internal deterministic metric over a generated pack. A broad paper claim still needs public benchmark scoring and real scanned tables with diverse merged-cell patterns.
993
+
994
+ ## Quality Report
995
+
996
+ Every `parse-doc` output now includes:
997
+
998
+ ```text
999
+ quality.json
1000
+ ```
1001
+
1002
+ The quality report tracks:
1003
+
1004
+ - line/block/table counts
1005
+ - average and minimum OCR confidence
1006
+ - missing confidence rates
1007
+ - low confidence rates
1008
+ - malformed table/block warnings
1009
+ - `quality_score`
1010
+ - fallback trigger/success/failure diagnostics when `metadata.fallback` exists
1011
+
1012
+ `quality_score` is useful as a diagnostic gate. It is not a substitute for CER/WER, table metrics, reading order metrics, or public benchmark results.
1013
+
1014
+ Figure/caption extraction is scored separately with `figure_detection_f1` and `figure_caption_cer` when structured references include `figures`.
1015
+
1016
+ Figure artifacts are exported explicitly. If an engine returns `figure.image_path`,
1017
+ the parser copies that image into `figures/`; if only a figure bbox is available
1018
+ and the source is a regular image or page-bundle directory, it crops
1019
+ `figures/{figure_id}.png` when Pillow is installed. For bundles, it uses
1020
+ `figure.page_index` to select the page image. When an artifact cannot be
1021
+ produced, the figure metadata gets `metadata.export_warnings`. This makes figure
1022
+ detections auditable as visual artifacts instead of only JSON boxes.
1023
+
1024
+ Selective fallback usage is exposed in benchmark reports as diagnostic metrics:
1025
+
1026
+ - `fallback_triggered`
1027
+ - `fallback_succeeded`
1028
+ - `fallback_failed`
1029
+ - `fallback_primary_average_confidence`
1030
+ - `fallback_threshold`
1031
+
1032
+ These are not quality scores by themselves. They are audit fields that make it
1033
+ possible to report how much of a claimed system came from the local engine and
1034
+ how much came from a fallback such as PaddleOCR-VL.
docs/FONT_ASSETS.md ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Font Assets
2
+
3
+ The language registry points to repo-relative runtime font assets under
4
+ `data/fonts/`. The `data/` tree is ignored because training and validation data
5
+ can be large, so these files must be provisioned on each runtime host before
6
+ synthetic rendering or font coverage claims.
7
+
8
+ Current required assets:
9
+
10
+ | File | Purpose | Current source used in audit |
11
+ | --- | --- | --- |
12
+ | `data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf` | Devanagari OCR synthesis and coverage for Nepali, Tamang, Magar, Bhojpuri, Maithili, and related Devanagari targets | Local Noto Sans Devanagari download on `cdjk`; copied to k2 runtime data |
13
+ | `data/fonts/NotoSansOlChiki-Regular.ttf` | Santhali/Ol Chiki coverage | macOS system font `/System/Library/Fonts/Supplemental/NotoSansOlChiki-Regular.ttf`; copied to runtime data |
14
+
15
+ Host-local system fonts currently used directly:
16
+
17
+ | File | Purpose |
18
+ | --- | --- |
19
+ | `/System/Library/Fonts/Supplemental/NotoSansLimbu-Regular.ttf` | Limbu/Sirijonga coverage |
20
+ | `/Library/Fonts/Arial Unicode.ttf` or `/System/Library/Fonts/Helvetica.ttc` | Latin/English regression coverage |
21
+
22
+ Validation command:
23
+
24
+ ```bash
25
+ uv run ocrtech audit-font-renderability languages/nepal-v1.json --out outputs/font-renderability-audit
26
+ ```
27
+
28
+ This audit is cmap coverage only unless `--render-samples` is used in an
29
+ environment with Pillow installed.
docs/GORKHAPATRA_LANGUAGE_PAGE_ALIGNMENT.md ADDED
@@ -0,0 +1,172 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Gorkhapatra Language Page Alignment
2
+
3
+ This is the validation route for Naya Nepal language pages. Do not use the
4
+ article HTML body as the primary reference source: many Naya Nepal posts embed
5
+ the publication content as page images. The stronger path is to align the
6
+ publication card date to the matching Gorkhapatra epaper PDF and then locate the
7
+ full language page inside that PDF.
8
+
9
+ ## Correct Source Path
10
+
11
+ 1. Read the Naya Nepal publication page:
12
+ `https://gorkhapatraonline.com/publications/naya-nepal`
13
+ 2. For each language card, extract:
14
+ - article URL
15
+ - article/card image URL
16
+ - title
17
+ - language label
18
+ - publication date label, for example `१ असार २०८३, सोमबार`
19
+ - normalized BS date key, for example `2083-03-01`
20
+ 3. Query the Gorkhapatra epaper date range endpoint:
21
+ `https://epaper.gorkhapatraonline.com/epaper-list?pdfdaterange1=2026-06-09&pdfdaterange2=2026-06-16&slug=gorkhapatra`
22
+ 4. Match the Naya Nepal publication date to the epaper date label, for example:
23
+ - publication card: `१ असार २०८३, सोमबार`
24
+ - epaper: `२०८३ असार १ सोमवार`
25
+ - normalized key: `2083-03-01`
26
+ 5. Download the matching Gorkhapatra PDF.
27
+ 6. Search PDF pages for language-page markers:
28
+ - `भाषा पृष्ठ`
29
+ - possible legacy/mojibake extraction such as `efiff k[i7`
30
+ - romanized fallback strings such as `bhasa pristha`
31
+ 7. Render the matched page as the candidate full-page validation image.
32
+ 8. Mark every output as `pending_manual_label` and
33
+ `claim_evidence_eligible=false` until a human reference transcript/layout is
34
+ created and audited.
35
+
36
+ ## Implementation Checklist
37
+
38
+ - [x] Add `ocrtech audit-gorkhapatra-language-pages` alignment command.
39
+ - [x] Parse publication cards from `/publications/naya-nepal`.
40
+ - [x] Normalize Nepali digits and BS month names into a date key.
41
+ - [x] Parse epaper entries from date-range endpoint and normalize the same key.
42
+ - [x] Join publication card entries to same-date epaper PDFs.
43
+ - [x] Download/hash matched PDFs when `--download-pdfs` is set.
44
+ - [x] Use PyMuPDF to inspect page text layers when PDFs are downloaded.
45
+ - [x] Search for `भाषा पृष्ठ`, `efiff k[i7`, and related variants.
46
+ - [x] Render matched PDF pages to PNG when `--render-pages` is set.
47
+ - [x] Emit `gorkhapatra-language-pages.json` and `.md`.
48
+ - [x] Emit a manifest with full-page images and source metadata using
49
+ `ocrtech prepare-gorkhapatra-language-page-pack`.
50
+ - [x] Keep references empty/pending until manually labeled.
51
+ - [x] Emit a manual review sheet from the candidate manifest using
52
+ `ocrtech prepare-gorkhapatra-language-page-review`.
53
+ - [x] Add fixture tests with one publication-card HTML and one epaper-list HTML.
54
+ - [x] Add a PyMuPDF fixture test that finds a language-page marker in a PDF.
55
+ - [x] Sync and run the command on k2.
56
+ - [ ] Use the review CSV/JSON to disambiguate same-date language pages.
57
+ - [ ] Create verified reference transcripts/layouts only for accepted rows.
58
+
59
+ Current k2 smoke evidence:
60
+
61
+ ```text
62
+ publication_items=2 aligned=2 language_page_hits=4 warnings=0
63
+ status_counts={'language_page_found': 2}
64
+ date_key=2083-03-01 pages=6,7 marker=efiff k[i7
65
+ ```
66
+
67
+ The 2083-03-01 smoke proves the date-to-PDF-to-language-page path, but it does
68
+ not yet disambiguate which matched language page belongs to each same-date
69
+ publication card. That requires page image/OCR/manual review in the next pack
70
+ generation step.
71
+
72
+ Candidate-pack k2 smoke:
73
+
74
+ ```text
75
+ data/gorkhapatra-language-page-pack-smoke/manifests/gorkhapatra-language-page-pack.jsonl
76
+ samples=4 copied_pages=4 warnings=0
77
+ gorkhapatra-language-page-0001-01 संस्कृत page=6 pending_manual_label False
78
+ gorkhapatra-language-page-0001-02 संस्कृत page=7 pending_manual_label False
79
+ gorkhapatra-language-page-0002-01 मगर page=6 pending_manual_label False
80
+ gorkhapatra-language-page-0002-02 मगर page=7 pending_manual_label False
81
+ ```
82
+
83
+ Command:
84
+
85
+ ```bash
86
+ uv run ocrtech audit-gorkhapatra-language-pages \
87
+ --out outputs/gorkhapatra-language-pages \
88
+ --download-pdfs \
89
+ --render-pages \
90
+ --max-publication-items 5 \
91
+ --max-epaper-pdfs 2
92
+ ```
93
+
94
+ The command always writes non-claim-ready provenance. Every alignment has
95
+ `claim_evidence_eligible=false`; rendered pages still need manual reference
96
+ transcription/layout labeling before benchmark use.
97
+
98
+ Candidate-pack command:
99
+
100
+ ```bash
101
+ uv run ocrtech prepare-gorkhapatra-language-page-pack \
102
+ outputs/gorkhapatra-language-pages/gorkhapatra-language-pages.json \
103
+ --out data/gorkhapatra-language-page-pack
104
+ ```
105
+
106
+ This copies rendered language-page candidates, writes
107
+ `manifests/gorkhapatra-language-page-pack.jsonl`, and creates empty reference
108
+ JSON files. Every manifest row has:
109
+
110
+ - `reference_status=pending_manual_label`
111
+ - `claim_evidence_eligible=false`
112
+ - `page_disambiguation_status=candidate_requires_manual_review`
113
+
114
+ Review-sheet command:
115
+
116
+ ```bash
117
+ uv run ocrtech prepare-gorkhapatra-language-page-review \
118
+ data/gorkhapatra-language-page-pack/manifests/gorkhapatra-language-page-pack.jsonl \
119
+ --out data/gorkhapatra-language-page-review
120
+ ```
121
+
122
+ This writes:
123
+
124
+ - `gorkhapatra-language-page-review.json`
125
+ - `gorkhapatra-language-page-review.csv`
126
+ - `gorkhapatra-language-page-review.md`
127
+
128
+ The CSV is the manual disambiguation worklist. Reviewers should fill
129
+ `review_decision`, `reviewer`, `reviewed_at`, and `notes`; accepted rows still
130
+ need a verified reference transcript/layout before they can be used as claim
131
+ evidence.
132
+
133
+ ## Synthesis Resource Checklist
134
+
135
+ Remote corpus host:
136
+
137
+ ```text
138
+ host: cdjk@100.110.148.43
139
+ password: agoodopensourceproject
140
+ large disk: 4TB SSD
141
+ primary paths to inspect:
142
+ - /Volumes/TRANSCEND
143
+ - ~/github/llm
144
+ ```
145
+
146
+ Checklist:
147
+
148
+ - [ ] SSH to `cdjk@100.110.148.43`.
149
+ - [ ] Inventory language resources on `/Volumes/TRANSCEND`.
150
+ - [ ] Inventory `~/github/llm` for language-specific corpora, dictionaries,
151
+ PDFs, fonts, and OCR-related projects.
152
+ - [ ] Record paths, sizes, file types, language labels, scripts, and license
153
+ notes.
154
+ - [ ] Prioritize Limbu/Sirijonga resources first.
155
+ - [ ] Then inspect Tamang, Magar, Bantawa/Rai, Kulung, Yakkha, Tharu,
156
+ Bhojpuri, Maithili, Santhali/Ol Chiki, and other Naya Nepal languages.
157
+ - [ ] For every candidate corpus, verify schema/content before conversion.
158
+ - [ ] Hash text/image/PDF files before using them.
159
+ - [ ] Separate training resources from held-out validation resources.
160
+ - [ ] Build real-text synthesis manifests only after corpus schema and font
161
+ renderability audits pass.
162
+ - [ ] Document any unsupported Unicode characters and normalization policy.
163
+
164
+ ## Paper Notes
165
+
166
+ - This path supports a real-document validation claim because the evaluation
167
+ images come from published Gorkhapatra PDF pages, not synthetic renderings.
168
+ - The article/card image is still useful as a source artifact, but the full PDF
169
+ page is the better validation target because it preserves page context,
170
+ neighboring text, captions, and layout.
171
+ - Synthetic data generated from cdjk corpus resources is training/ablation data
172
+ only. It cannot be used as SOTA evidence without real held-out validation.
docs/MODELS.md ADDED
@@ -0,0 +1,490 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Candidate Model Cards
2
+
3
+ The candidate OCR engine must run from an explicit model card. Do not benchmark `ours` against external systems unless the candidate model card is attached to the experiment.
4
+
5
+ Supported candidate backends:
6
+
7
+ - `paddleocr`
8
+ - `hf_vision_encoder_decoder`
9
+ - `sidecar`
10
+ - `tesseract`
11
+ - `text_correction_composite`
12
+ - `quality_select_composite`
13
+ - `quality_ranked_ensemble`
14
+ - `script_select_composite`
15
+ - `line_align_composite`
16
+ - `table_cell_refine_composite`
17
+
18
+ ## Export And Package
19
+
20
+ After `train-recognizer` finishes and a PaddleOCR checkpoint exists, export an inference model:
21
+
22
+ ```bash
23
+ ocrtech export-recognizer \
24
+ --training-config runs/recognizer/paddle_recognizer.yml \
25
+ --checkpoint runs/recognizer/checkpoints/best_accuracy \
26
+ --out runs/recognizer/export
27
+ ```
28
+
29
+ Run the generated export command from a PaddleOCR checkout, or pass `--run` when the checkout exists locally.
30
+
31
+ Then package the exported inference directory into an auditable candidate model:
32
+
33
+ ```bash
34
+ ocrtech package-recognizer \
35
+ --inference-dir runs/recognizer/export/inference \
36
+ --dictionary runs/recognizer/nep_eng_dict.txt \
37
+ --out runs/recognizer/package \
38
+ --model-id ocrtech-nep-eng-rec-v1 \
39
+ --train-manifest data/splits/train.jsonl \
40
+ --eval-manifest data/splits/eval.jsonl \
41
+ --training-summary runs/recognizer/recognizer-training-summary.json \
42
+ --admission-validation-report outputs/validation-next-recognizer-hard50/validation.json
43
+ ```
44
+
45
+ This writes `runs/recognizer/package/model-card.json` and copies all required inference artifacts under `runs/recognizer/package/artifacts/`.
46
+
47
+ ## Training Recipe Evidence
48
+
49
+ `train-recognizer` writes a machine-readable
50
+ `recognizer-training-summary.json` beside the backend training config:
51
+ `paddle_recognizer.yml` for PaddleOCR and `hf_recognizer_config.json` for
52
+ Hugging Face vision-encoder-decoder runs. Preserve it with the recipe, bundle,
53
+ training logs, exported inference model, and packaged model card. It records:
54
+
55
+ - train/eval manifest paths and SHA-256 hashes
56
+ - train/eval sample counts, dataset counts, and slice counts
57
+ - generated label, dictionary, and config hashes
58
+ - dictionary character coverage, including ASCII and Devanagari presence
59
+ - the recipe hyperparameters used for launch
60
+
61
+ When a custom `--dictionary` is supplied, recipe generation now checks it
62
+ against the exact normalized PaddleOCR label text for every train and eval
63
+ sample. Missing characters fail before a remote training job is packaged. This
64
+ prevents a recognizer run from silently launching with labels that the
65
+ recognition head cannot emit.
66
+
67
+ This summary is not a quality result. It is launch provenance for the local
68
+ recognizer branch, and it helps distinguish a reproducible recognizer attempt
69
+ from an undocumented remote training run.
70
+
71
+ `bundle-recognizer-training` includes this summary in the archive and records
72
+ its path and SHA-256 under `training_summary` in
73
+ `recognizer-training-bundle.json`. Check that field after syncing the bundle to
74
+ a CUDA host.
75
+
76
+ For real GPU handoffs, pass the latest recognizer failure review to both recipe
77
+ generation and bundling. This rejects families already stopped by admission
78
+ evidence before another remote job is launched:
79
+
80
+ ```bash
81
+ ocrtech train-recognizer \
82
+ --train-manifest data/splits/train.jsonl \
83
+ --eval-manifest data/splits/eval.jsonl \
84
+ --out runs/recognizer \
85
+ --failure-review outputs/recognizer-failure-review-paddle-ppocrv3/recognizer-failure-review.json
86
+
87
+ ocrtech bundle-recognizer-training \
88
+ --recipe-dir runs/recognizer \
89
+ --train-manifest data/splits/train.jsonl \
90
+ --eval-manifest data/splits/eval.jsonl \
91
+ --out runs/recognizer-training-bundle \
92
+ --require-corpus-audit \
93
+ --failure-review outputs/recognizer-failure-review-paddle-ppocrv3/recognizer-failure-review.json
94
+ ```
95
+
96
+ If that review contains `stop_family` for the recipe backend/base model, the
97
+ command fails with the rejected-run and win/loss/tie evidence. Changing the run
98
+ name, learning rate, or corpus is not enough to bypass a stopped family; use a
99
+ materially different base recognizer, objective, or document-native model.
100
+
101
+ Pass the same file to `package-recognizer --training-summary` or
102
+ `package-hf-recognizer --training-summary` so the final `model-card.json`
103
+ records the summary path and SHA-256 under `provenance.training_summary`.
104
+
105
+ ## Hugging Face Packaging
106
+
107
+ For an arm64-compatible recognizer path, train a Hugging Face vision-text model and package the saved model directory:
108
+
109
+ ```bash
110
+ ocrtech train-recognizer \
111
+ --backend hf-vision-encoder-decoder \
112
+ --base-model microsoft/trocr-small-printed \
113
+ --train-manifest data/splits/train.jsonl \
114
+ --eval-manifest data/splits/eval.jsonl \
115
+ --out runs/hf-recognizer
116
+
117
+ ocrtech package-hf-recognizer \
118
+ --model-dir runs/hf-recognizer/final-model \
119
+ --out runs/hf-recognizer/package \
120
+ --model-id ocrtech-hf-nep-eng-rec-v1 \
121
+ --base-model microsoft/trocr-small-printed \
122
+ --train-manifest data/splits/train.jsonl \
123
+ --eval-manifest data/splits/eval.jsonl \
124
+ --training-summary runs/hf-recognizer/recognizer-training-summary.json \
125
+ --admission-validation-report outputs/validation-next-recognizer-hard50/validation.json
126
+ ```
127
+
128
+ The HF recipe writes `runs/hf-recognizer/recognizer-training-summary.json` at
129
+ recipe-generation time. That summary records train/eval manifest hashes,
130
+ script coverage, config hash, and hyperparameters. Packaging should include it
131
+ just like the PaddleOCR path; otherwise the trained-recognizer provenance gate
132
+ will have weaker launch evidence.
133
+
134
+ This writes `runs/hf-recognizer/package/model-card.json` and records backend kwargs for loading the saved processor and model directory.
135
+
136
+ ## Create Manually
137
+
138
+ ```bash
139
+ ocrtech create-model-card \
140
+ --out runs/recognizer/model-card.json \
141
+ --model-id ocrtech-nep-eng-rec-v1 \
142
+ --backend paddleocr \
143
+ --base-model PaddlePaddle/devanagari_PP-OCRv3_mobile_rec \
144
+ --artifact runs/recognizer/checkpoints/best_accuracy.pdparams \
145
+ --artifact runs/recognizer/nep_eng_dict.txt \
146
+ --paddleocr-kwarg lang=devanagari \
147
+ --paddleocr-kwarg rec_model_dir=runs/recognizer/inference \
148
+ --paddleocr-kwarg rec_char_dict_path=runs/recognizer/nep_eng_dict.txt \
149
+ --train-manifest data/splits/train.jsonl \
150
+ --eval-manifest data/splits/eval.jsonl \
151
+ --metrics-report outputs/claim-experiment/validation/validation.json
152
+ ```
153
+
154
+ The backend kwargs are passed through to the selected runtime backend. For `paddleocr`, keep them explicit in the model card because PaddleOCR constructor names can change by version.
155
+
156
+ ## Routed Candidates
157
+
158
+ For routed or thresholded candidates, do not hand-edit the threshold into a model card and call that evidence. Calibrate it from a separate experiment artifact:
159
+
160
+ ```bash
161
+ ocrtech calibrate-quality-router \
162
+ --experiment-report outputs/tesseract-psm13-calibration/experiment.json \
163
+ --out outputs/quality-router-calibration \
164
+ --model-id ocrtech-tesseract-quality-router \
165
+ --primary-engine tesseract \
166
+ --secondary-engine tesseract \
167
+ --primary-engine-kwarg 'language=nep+eng' \
168
+ --primary-engine-kwarg 'psm=6' \
169
+ --secondary-engine-kwarg 'language=nep+eng' \
170
+ --secondary-engine-kwarg 'psm=13'
171
+ ```
172
+
173
+ This writes:
174
+
175
+ - `calibration.json`
176
+ - `calibration.md`
177
+ - `model-card.json`
178
+
179
+ The generated model card records calibration provenance, including the calibration experiment, benchmark report, and calibration eval manifest hash. `preflight-claim` and `review-claim` use that provenance to reject tuned-on-eval routed candidates.
180
+
181
+ ## Multi-Variant Tesseract Ensemble
182
+
183
+ For a richer Tesseract-only ensemble, calibrate several PSM variants from separate experiment artifacts:
184
+
185
+ ```bash
186
+ ocrtech calibrate-tesseract-psm-ensemble \
187
+ --primary-experiment-report outputs/psm13-calibration/experiment.json \
188
+ --variant-experiment psm7=outputs/psm7-calibration/experiment.json \
189
+ --variant-experiment psm11=outputs/psm11-calibration/experiment.json \
190
+ --variant-experiment psm13=outputs/psm13-calibration/experiment.json \
191
+ --out outputs/psm-ensemble-calibration \
192
+ --model-id ocrtech-tesseract-psm-ensemble
193
+ ```
194
+
195
+ This writes a `quality_ranked_ensemble` model card with:
196
+
197
+ - `primary_engine`
198
+ - `alternative_engines`
199
+ - per-variant `quality_bias`
200
+ - calibration provenance
201
+
202
+ Current evidence says this path is not promising on the mixed Nepali/English line pack: it improved on the calibration half and regressed on the untouched holdout. Keep it as a research tool, not as a recommended candidate.
203
+
204
+ ## Script-Routed Tesseract + Surya
205
+
206
+ For a script-aware routed candidate, calibrate a threshold over the primary engine output text:
207
+
208
+ ```bash
209
+ ocrtech calibrate-script-router \
210
+ --experiment-report outputs/surya-calibration/experiment.json \
211
+ --out outputs/script-router-calibration \
212
+ --model-id ocrtech-tesseract-surya-script-router \
213
+ --primary-engine tesseract \
214
+ --secondary-engine surya \
215
+ --primary-engine-kwarg 'language=nep+eng' \
216
+ --primary-engine-kwarg 'psm=6' \
217
+ --secondary-engine-kwarg 'mode=auto'
218
+ ```
219
+
220
+ This writes a `script_select_composite` model card with:
221
+
222
+ - `primary_engine`
223
+ - `secondary_engine`
224
+ - `script`
225
+ - `primary_script_threshold`
226
+ - optional `routing_granularity` (`document` by default, `page` for per-page routing)
227
+ - optional `secondary_structure_backfill` to keep primary text selection while filling missing tables/figures from the secondary engine
228
+ - calibration provenance
229
+
230
+ Current live evidence on `k2` is better than the quality-score-only Tesseract router: calibration on the `89`-sample split selected `primary_script_threshold=0.4`, and the untouched `91`-sample holdout improved overall CER from `0.342234` to `0.313084` with `9` wins, `0` losses, and `82` ties against raw Tesseract.
231
+
232
+ An exploratory `routing_granularity=page` run on that same holdout landed on the same numbers. That does not mean page routing is useless; it means this particular holdout is mostly single-page line-crop material. Page routing needs a true multi-page document eval pack before it can show a different outcome.
233
+
234
+ On the newer rendered multi-page image-pack smoke, `secondary_structure_backfill=true` turned out to be useful. The document-level router still kept Tesseract text on the multi-page report page, but it backfilled Surya's table structure on that page and moved `table_cell_f1` from `0.0` to `1.0` on the rendered smoke pack without changing CER.
235
+
236
+ That behavior survived on the broader rendered claim-ready image pack as well. The calibrated backfill model card `ocrtech-tesseract-surya-script-router-backfill-calibrated` kept `routing_granularity=document`, `secondary_structure_backfill=true`, and `primary_script_threshold=0.4`. On the `25`-sample rendered pack it still beat raw Tesseract on overall CER (`0.248654` vs `0.689214`) and kept a large structure advantage (`table_cell_f1` `0.955556` vs `0.0`). The weak spots are still visible, which is exactly why this should be treated as an engineering baseline rather than a claim candidate: `form` CER regressed slightly and aggregate reading-order accuracy on that pack is still lower than Tesseract.
237
+
238
+ ## Table Cell Refinement Composite
239
+
240
+ `table_cell_refine_composite` uses one engine for document/table structure and a second engine for crop OCR over detected table cells:
241
+
242
+ ```bash
243
+ ocrtech create-model-card \
244
+ --out outputs/table-cell-refine-empty-v1/model-card.json \
245
+ --model-id ocrtech-surya-tesseract-table-cell-refine-empty-v1 \
246
+ --backend table_cell_refine_composite \
247
+ --base-model surya+tesseract \
248
+ --backend-kwarg structure_engine=surya \
249
+ --backend-kwarg crop_engine=tesseract \
250
+ --backend-kwarg crop_padding=4 \
251
+ --backend-kwarg replacement_policy=empty
252
+ ```
253
+
254
+ Policies:
255
+
256
+ - `replacement_policy=empty`: only OCR empty table cells. This is the safe default.
257
+ - `replacement_policy=always`: reread every detected table cell. This is experimental and can damage already-correct Surya cells.
258
+ - `deduplicate_tables=true`: drop overlapping duplicate tables after refinement. The parser also avoids re-inferring a table from text lines when an engine already supplied an overlapping structured table.
259
+
260
+ Current hard15 evidence says the idea is not enough:
261
+
262
+ - unsafe `always` policy tied Surya on text CER but regressed table-cell F1 from `0.979167` to `0.593750`
263
+ - safe `empty` policy tied Surya exactly on hard15 CER, table-cell F1, and reading order
264
+ - safe `empty` plus table de-duplication also tied Surya exactly on hard15 CER, table-cell F1, and reading order
265
+ - hard15 validation status for the safe policies is `not_validated`
266
+
267
+ So this backend is useful infrastructure for future missing-cell cases, not a claim candidate yet.
268
+
269
+ ## Figure Caption Refinement Composite
270
+
271
+ `figure_caption_refine_composite` keeps a primary document engine for layout and OCR, then uses a second engine only to replace nearby figure-caption lines when the replacement overlaps spatially and has a better caption-prefix score:
272
+
273
+ ```bash
274
+ ocrtech create-model-card \
275
+ --out outputs/figure-caption-refine-v1/model-card.json \
276
+ --model-id ocrtech-surya-tesseract-figure-caption-refine-v1 \
277
+ --backend figure_caption_refine_composite \
278
+ --base-model surya+tesseract \
279
+ --backend-kwarg primary_engine=surya \
280
+ --backend-kwarg caption_engine=tesseract \
281
+ --backend-kwarg min_caption_iou=0.5 \
282
+ --backend-kwarg min_score_delta=1
283
+ ```
284
+
285
+ The replacement preserves the primary engine's bbox and layout ordering. Only the caption text/confidence changes. This avoids the skew failure where a secondary OCR bbox overlaps the figure marker and prevents caption attachment.
286
+
287
+ Current hard15 evidence:
288
+
289
+ - figure-caption CER improved over Surya from `0.092323` to `0.052478`
290
+ - figure detection F1 stayed `1.000000`
291
+ - table-cell F1 stayed `0.979167`
292
+ - reading-order pair accuracy stayed `0.520000`
293
+ - hard15 validation status is still `not_validated` because broad CER/table gates do not pass
294
+
295
+ An opt-in `correct_structural_labels=true` branch adds edit-distance guarded correction for common Nepali document-role labels such as `वार्षिक प्रतिवेदन`, `चित्र`, and `निष्कर्ष`. On hard15 this stronger branch improved:
296
+
297
+ - overall hard_eval CER over Surya from `0.236839` to `0.204610`
298
+ - figure CER from `0.091927` to `0.022866`
299
+ - figure-caption CER from `0.092323` to `0.008746`
300
+
301
+ Validation still reports `not_validated` because table CER and table-cell F1 tie Surya, but the overall CER gate and both figure gates pass.
302
+
303
+ A stronger auditable variant keeps the same backend but enables:
304
+
305
+ - `correct_structural_labels=true`, now covering `वार्षिक प्रतिवेदन`, `दैनिक हिसाब`, `चित्र`, and `निष्कर्ष`
306
+ - `table_name_lexicon_path=configs/nepali-table-name-lexicon-v1.txt`
307
+ - `table_name_max_distance=2`
308
+
309
+ The table-name lexicon correction is intentionally narrow: it only applies to cells under a header close to `नाम`, and only when there is a unique nearest lexicon entry within the configured edit-distance threshold. The current lexicon is tiny and curated, so paper claims must describe this as lexicon-assisted structured extraction, not a general language model.
310
+
311
+ The public-table variant also enables:
312
+
313
+ - `normalize_table_grids=true`
314
+
315
+ This removes empty columns, separator-only columns such as `—`, and exact
316
+ duplicate adjacent columns from predicted tables. The rule is runtime-only and
317
+ reference-free. It improved the 5-page public OmniDocBench smoke over raw Surya
318
+ on table metrics, but the public validation still remains `not_validated`
319
+ because the sample count and confidence gates are not met.
320
+
321
+ The later `normalize-rows-v3` public-table model card keeps the same option and
322
+ adds sparse bullet row-fragment merging inside the grid normalizer. This is also
323
+ runtime-only and reference-free. On the same 5-page public OmniDocBench smoke it
324
+ raises table metrics further (`table_cell_f1` `0.564335`,
325
+ `table_teds_structure` `0.942836`, `table_teds_text` `0.675633`) and changes the
326
+ `omnidocbench-00208...` fragmented table from `16x5` to `3x5`. It is not a
327
+ validated model: text CER is slightly worse than Surya on that smoke, and the
328
+ public validation gates still fail on sample count and bootstrap CI.
329
+
330
+ On the larger public OmniDocBench `table40` slice, the same model has higher
331
+ mean table metrics than Surya but does not validate:
332
+
333
+ - CER `0.713326` versus Surya `0.713960`
334
+ - table-cell F1 `0.323122` versus Surya `0.280062`, but `7` wins / `9` losses /
335
+ `24` ties and CI low `-0.019152`
336
+ - table TEDS structure `0.520959` versus Surya `0.503603`, but CI low
337
+ `-0.038381`
338
+ - table TEDS text `0.361666` versus Surya `0.358057`, below the configured
339
+ margin
340
+ - cell exactness `3440/10314` versus Surya `3421/10314`
341
+
342
+ This points to the next model requirement: better table segmentation and
343
+ large/wide-table handling. The current grid postprocessors are useful but too
344
+ local to support a public SOTA claim.
345
+
346
+ The `normalize-width-guard-v4` model card adds one guard to the same runtime
347
+ postprocessor: tables wider than `12` predicted columns skip grid normalization.
348
+ This directly addresses public table40 failures where v3 merged `15` sparse rows
349
+ or dropped `26` columns in wide tables. On full public table40, v4 improves over
350
+ v3 and passes two configured public table gates against Surya:
351
+
352
+ - CER `0.707864` versus Surya `0.713960`
353
+ - table-cell F1 `0.339319` versus Surya `0.280062`, CI low `0.001433`
354
+ - table TEDS structure `0.545375` versus Surya `0.503603`, CI low `0.004790`
355
+ - table TEDS text `0.386013` versus Surya `0.358057`, but CI low `-0.000220`
356
+ - cell exactness `3453/10314` versus Surya `3421/10314`
357
+
358
+ This is a better public table parser candidate than v3, but still not a SOTA
359
+ engine: the text-sensitive table gate and broad public document gate remain open.
360
+
361
+ On hard15 v3, this model reaches:
362
+
363
+ - overall CER `0.010671` versus Surya `0.236839`
364
+ - table-slice CER `0.000000` versus Surya `0.363636`
365
+ - figure-caption CER `0.008746` versus Surya `0.092323`
366
+ - table-cell F1 `1.000000` versus Surya `0.979167`
367
+ - table exact cells `96/96` versus Surya `94/96`
368
+
369
+ Configured validation is still `not_validated` because the table-cell F1 bootstrap CI low is `0.000000` on the tiny 8-page table slice. This is a sample-size limitation of the current gate, not permission to claim table SOTA. Build a larger independent table pack before making that claim.
370
+
371
+ The table-name lexicon matcher now also supports suffix-preserving generated IDs such as `करिण-302` -> `किरण-302`. The correction still only fires in columns whose header is close to `नाम`, and still requires a unique nearest base-name match within the configured edit distance.
372
+
373
+ On the larger `60`-sample table-ledger pack, this same model passes the targeted Surya table validation:
374
+
375
+ - table-cell F1: candidate `1.000000`, Surya `0.979167`, CI low `0.012500`
376
+ - table exact match: candidate `1.000000`, Surya `0.750000`, CI low `0.150000`
377
+ - table structure similarity: candidate `1.000000`, Surya `1.000000`, non-regression guard passes
378
+ - cell exactness: candidate `720/720`, Surya `705/720`
379
+
380
+ This is valid evidence for lexicon-assisted table-ledger cell-content extraction with grid-structure non-regression. It is not a general OCR SOTA result, and it does not prove improvement on complex merged-cell tables.
381
+
382
+ On the first `complex-table-ledger` run, the same candidate improved cell-content F1 over Surya (`0.928333` vs `0.915000`) after extending name correction below grouped headers, but it did not improve merged-header structure.
383
+
384
+ The later span-inference parser branch adds conservative repeated-header colspan inference. On the same generated complex-table pack, it reaches:
385
+
386
+ - table-cell F1 `0.995000` versus Surya `0.915000`
387
+ - table exact match `0.925000` versus Surya `0.000000`
388
+ - span-aware table structure similarity `1.000000` versus Surya `0.875000`
389
+ - TEDS-style table tree structure similarity `1.000000` versus Surya `0.904762`
390
+ - TEDS-style text-sensitive table tree similarity `0.996250` versus Surya `0.891667`
391
+ - cell exactness `597/600` versus Surya `549/600`
392
+
393
+ This is a parser/postprocessing improvement for a generated repeated-header pattern. It is not a trained merged-cell model and does not support a broad complex-table SOTA claim without real scanned tables and varied span structures.
394
+
395
+ The public OmniDocBench table40 width-guard model card is:
396
+
397
+ - `outputs/figure-caption-table-lexicon-normalize-width-guard-v4/model-card.json`
398
+ - model id: `ocrtech-surya-tesseract-figure-table-lexicon-normalize-width-guard-v4`
399
+ - audit: `outputs/audit-figure-caption-table-lexicon-normalize-width-guard-v4-model/model-audit.json`
400
+
401
+ This candidate keeps the normalized table-grid postprocessors but skips grid
402
+ normalization when predicted table width is greater than `12`. On the public
403
+ OmniDocBench table40 combined run, it improves over Surya on table-cell F1
404
+ (`0.339319` vs `0.280062`) and structure-only TEDS (`0.545375` vs `0.503603`),
405
+ but the run remains `not_validated` because the public document CER gate is
406
+ underpowered and table TEDS text is borderline in the separate-run comparison.
407
+
408
+ For postprocessor-only evidence, use
409
+ `outputs/experiment-public-omnidocbench-table40-derived-postprocess-v1/benchmark/report.json`.
410
+ That derived benchmark applies the table postprocessors to the exact same Surya
411
+ `document.json` outputs, so CER stays identical while table metrics isolate the
412
+ postprocessor contribution. It passes the configured table-cell F1,
413
+ structure-only TEDS, and text-sensitive TEDS gates on `40` pages, but it is a
414
+ validation lens rather than a deployable model card.
415
+
416
+ ## Audit
417
+
418
+ ```bash
419
+ ocrtech audit-model runs/recognizer/package/model-card.json \
420
+ --out outputs/model-audit
421
+ ```
422
+
423
+ The audit verifies:
424
+
425
+ - required artifact paths exist
426
+ - recorded SHA-256 values match for file artifacts
427
+ - backend is supported
428
+ - the card shape is valid
429
+
430
+ `audit-model` does not by itself prove claim readiness. For `quality_select_composite`, `quality_ranked_ensemble`, and `script_select_composite`, the calibration provenance must also be separate from the final claim eval manifest.
431
+
432
+ For recognizer branch admission, attach the hard-gate validation report when
433
+ creating or packaging a model card:
434
+
435
+ ```bash
436
+ ocrtech create-model-card \
437
+ --out runs/recognizer/package/model-card.json \
438
+ --model-id ocrtech-next-recognizer \
439
+ --backend paddleocr \
440
+ --artifact runs/recognizer/export/inference/inference.pdiparams \
441
+ --train-manifest data/splits/train.jsonl \
442
+ --eval-manifest data/splits/eval.jsonl \
443
+ --admission-validation-report outputs/validation-next-recognizer-hard50/validation.json
444
+ ```
445
+
446
+ Claim-grade preflight can then require the admission gate with
447
+ `--require-model-admission`. Admission reports must include quantitative
448
+ passing gate evidence; a bare `claim_status=validated` report is rejected.
449
+
450
+ ## Parse With Candidate
451
+
452
+ ```bash
453
+ ocrtech parse-doc page.png \
454
+ --engine candidate \
455
+ --model-config runs/recognizer/package/model-card.json \
456
+ --fallback-model-config runs/vlm-fallback/package/model-card.json \
457
+ --out outputs/page
458
+ ```
459
+
460
+ When a fallback engine or fallback model card is configured, the parser does not
461
+ silently swap engines. It records `metadata.fallback` in `document.json` with
462
+ the primary engine, fallback engine/model config, average primary confidence,
463
+ threshold, trigger reason, and outcome. This is required for paper evidence: a
464
+ selective VLM/document fallback must report when it actually fired and whether
465
+ it succeeded.
466
+
467
+ ## Benchmark With Candidate
468
+
469
+ ```bash
470
+ ocrtech run-experiment \
471
+ --eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
472
+ --references-dir data/eval-pack/references \
473
+ --candidate-model-config runs/recognizer/package/model-card.json \
474
+ --fallback-model-config runs/vlm-fallback/package/model-card.json \
475
+ --low-confidence-threshold 0.80 \
476
+ --baselines candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
477
+ --validation-config validation.json \
478
+ --train-manifest data/splits/train.jsonl \
479
+ --out outputs/claim-experiment \
480
+ --require-validated
481
+ ```
482
+
483
+ If `--candidate-model-config` is omitted, `ours` uses the local `auto` engine for smoke tests. That is not valid SOTA evidence.
484
+
485
+ Benchmark and experiment fallback flags apply only to the candidate/ours run.
486
+ They are meant to measure the deployed hybrid engine, not to contaminate
487
+ baseline rows. `experiment.json` records `fallback_engine`,
488
+ `fallback_model_config`, `fallback_model_config_sha256`, and
489
+ `low_confidence_threshold` so claim reviewers can tell which routing policy was
490
+ actually evaluated.
docs/NEPAL_MULTILINGUAL_OCR_GOAL.md ADDED
@@ -0,0 +1,371 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Nepal Multilingual Document OCR Goal
2
+
3
+ This is the research target after the steering decision on 2026-06-16. The
4
+ project is now a PaddleOCR-centered multilingual document OCR system for
5
+ languages used in Nepal, not a narrow Nepali/English OCR experiment.
6
+
7
+ ## Goal
8
+
9
+ Build and validate a PaddleOCR-based document OCR engine for at least ten
10
+ languages spoken in Nepal, with Limbu first, that supports:
11
+
12
+ - page/PDF/image parsing
13
+ - text OCR
14
+ - reading order
15
+ - table reconstruction
16
+ - figure and caption extraction
17
+ - Markdown export
18
+ - structured JSON export
19
+ - selective PaddleOCR-VL/VLM fallback for hard pages
20
+
21
+ The claim is valid only if the engine beats strong baselines on predefined real
22
+ and held-out evaluations. Synthetic-only results are not enough.
23
+
24
+ ## Research Claim Shape
25
+
26
+ The paper should not claim "best OCR engine overall." The defensible claim is:
27
+
28
+ > A script-aware PaddleOCR adaptation framework can substantially improve
29
+ > document OCR and structured extraction for low-resource Nepal languages,
30
+ > including scripts not covered well by stock OCR systems, while preserving
31
+ > Nepali/English performance and using selective VLM fallback only when the
32
+ > local pipeline is uncertain.
33
+
34
+ The core contribution is the system and validation recipe:
35
+
36
+ 1. Language/script registry for Nepal document OCR.
37
+ 2. Gorkhapatra/Naya Nepal real-publication validation pack.
38
+ 3. Unicode/font/renderability audit before synthesis.
39
+ 4. Real-text synthetic data generation for lines, tables, captions, and pages.
40
+ 5. PaddleOCR recognizer fine-tuning by script family.
41
+ 6. PP-Structure-style document extraction.
42
+ 7. Claim review that rejects weak SOTA statements.
43
+
44
+ ## Primary Engine Direction
45
+
46
+ Use PaddleOCR as the engine family:
47
+
48
+ - **PP-OCRv6** for the modern small/local OCR baseline and fine-tuning target.
49
+ - **PP-StructureV3** for layout, reading order, tables, and document structure.
50
+ - **PaddleOCR-VL-1.6** as fallback/teacher for charts, figures, hard layouts,
51
+ seals, and low-confidence pages.
52
+
53
+ Do not route everything to a VLM and call that the engine. The local PaddleOCR
54
+ path must be the default path; fallback usage must be measured and reported.
55
+
56
+ ## First Ten Language Targets
57
+
58
+ The first target set should be driven by actual Gorkhapatra/Naya Nepal
59
+ availability plus local corpus availability. OCR support is tracked by script,
60
+ Unicode, fonts, and evaluation data, not by language name alone.
61
+
62
+ | Priority | Language | First script target | Why first | Notes |
63
+ | --- | --- | --- | --- | --- |
64
+ | 1 | Limbu/Yakthung | Limbu/Sirijonga, U+1900..U+194F | Has own Unicode block and local Limbu resources | First non-Devanagari research target |
65
+ | 2 | Nepali | Devanagari | Core language and baseline | Must not regress |
66
+ | 3 | English | Latin | Mixed documents and forms | Must not regress |
67
+ | 4 | Tamang | Devanagari first; Tamyig audit later | Local NepTam corpus and dictionary | Start with Devanagari Tamang |
68
+ | 5 | Magar | Devanagari first | Gorkhapatra item confirmed; local dictionary | Own/community script requires separate audit |
69
+ | 6 | Wambule | Devanagari first | Gorkhapatra item confirmed | Rai-family validation candidate |
70
+ | 7 | Bantawa/Rai | Devanagari or Kirat Rai depending source | Gorkhapatra item confirmed; Rai resources on remote/local projects | Kirat Rai Unicode support is newer; font audit required |
71
+ | 8 | Kulung | Devanagari first | Gorkhapatra item confirmed | Rai-family validation candidate |
72
+ | 9 | Yakkha | Devanagari first | Gorkhapatra item confirmed | Rai-family validation candidate |
73
+ | 10 | Tharu/Rana Tharu | Devanagari first | Gorkhapatra item confirmed | Multiple varieties; keep labels explicit |
74
+
75
+ Additional Gorkhapatra-visible candidates include Sanskrit, Magahi, Bajjika,
76
+ Sunuwar, Danuwar, Majhi, Lhomi, Bhojpuri, Kisan, Thami, Sherpa, Achhami, Bote,
77
+ Rajbanshi, Hayu, Santhali, Dhimal, Bhote, Uraon, Kumal, Gurung, Khash Jumli,
78
+ and Meche. These should enter the registry after script/corpus/font checks.
79
+
80
+ ## Source Findings So Far
81
+
82
+ ### Gorkhapatra Naya Nepal
83
+
84
+ The category page is useful for language-specific validation:
85
+
86
+ - `https://gorkhapatraonline.com/categories/new-nepal`
87
+ - `https://gorkhapatraonline.com/categories/new-nepal?page=2`
88
+
89
+ Confirmed visible language-specific entries include Magar, Wambule, Rana Tharu,
90
+ Magahi, Bajjika, Sunuwar, Danuwar, Majhi, Lhomi, Bhojpuri, Kisan, Thami,
91
+ Sherpa, Achhami, Bote, Rajbanshi, Hayu, Santhali, Kulung, Yakkha, Meche, Khash
92
+ Jumli, Dhimal, Tharu, Bhote, Uraon, Bantawa, Kumal, Gurung, and others.
93
+
94
+ The blog/category pages expose image URLs and article URLs. They are useful for
95
+ real-world OCR samples, but they should not be trusted as clean text references
96
+ without manual or tool-assisted verification.
97
+
98
+ ### Gorkhapatra Epaper
99
+
100
+ The epaper list endpoint exposes viewer links:
101
+
102
+ ```text
103
+ https://epaper.gorkhapatraonline.com/epaper-list?pdfdaterange1=2026-06-09&pdfdaterange2=2026-06-16&slug=gorkhapatra
104
+ ```
105
+
106
+ The visible links look like:
107
+
108
+ ```text
109
+ https://epaper.gorkhapatraonline.com/pdf?file=/uploads/file/2026/6/gorkhapatra/2026-06-14-11-49-50-2083-03-01.pdf
110
+ ```
111
+
112
+ That `pdf?...` URL returns a PDF.js viewer shell. The real PDF is downloaded by
113
+ fetching the file parameter directly:
114
+
115
+ ```text
116
+ https://epaper.gorkhapatraonline.com/uploads/file/2026/6/gorkhapatra/2026-06-14-11-49-50-2083-03-01.pdf
117
+ ```
118
+
119
+ On `k2`, this was verified as a real PDF:
120
+
121
+ ```text
122
+ /Users/k2/ocr-tech/outputs/gorkhapatra-source-audit/sample.pdf
123
+ PDF document, version 1.6
124
+ size: 48M
125
+ ```
126
+
127
+ The `k2` audit produced:
128
+
129
+ ```text
130
+ /Users/k2/ocr-tech/outputs/gorkhapatra-source-audit/audit.txt
131
+ /Users/k2/ocr-tech/outputs/gorkhapatra-source-audit/new-nepal-language-items.txt
132
+ /Users/k2/ocr-tech/outputs/gorkhapatra-source-audit/epaper-assets.txt
133
+ /Users/k2/ocr-tech/outputs/gorkhapatra-source-audit/sample.pdf
134
+ ```
135
+
136
+ `k2` does not currently expose `pdfinfo` or `pdftotext`. `pymupdf` was installed
137
+ into `/Users/k2/ocr-tech/.venv` on 2026-06-16 as the immediate PDF audit tool.
138
+ The sample epaper PDF opens successfully with PyMuPDF:
139
+
140
+ ```text
141
+ page_count: 26
142
+ total extracted text chars: 309924
143
+ preview image: outputs/gorkhapatra-source-audit/sample-page-1-preview.jpg
144
+ audit: outputs/gorkhapatra-source-audit/sample-pymupdf-audit.json
145
+ text: outputs/gorkhapatra-source-audit/sample-pymupdf-text.txt
146
+ ```
147
+
148
+ The extracted text layer is not reliable reference text. The first-page preview
149
+ contains mojibake/transliterated-looking output for Nepali content rather than
150
+ clean Unicode Devanagari. Therefore the epaper PDFs are useful as real page
151
+ images/layout sources, but their embedded text layer must be audited per file
152
+ and must not be treated as ground truth without verification.
153
+
154
+ ### Local And Remote Corpus Hints
155
+
156
+ Local machine:
157
+
158
+ - `~/github/llm/ltk/tamang`: NepTam train/test, NepTam80K, dictionary, TTS
159
+ lexicon.
160
+ - `~/github/llm/ltk/limbu`: Limbu dictionary, Pangloss IGT, Limbu G2P.
161
+ - `~/github/llm/ltk/magar`: Western Magar dictionary and resource report.
162
+
163
+ Remote `cdjk@100.110.148.43`:
164
+
165
+ - `/Volumes/TRANSCEND` has about 2.0 TiB free.
166
+ - `~/github/llm` contains additional language projects:
167
+ `dhimal-toolkit`, `gurung-language-toolkit`, `limbu-speech-toolkit`,
168
+ `magar-toolkit`, `rai-speech-toolkit`, `nllb-better`, `nepa-newa-text-frontend`,
169
+ `new-place`, and `loksewa.zip`.
170
+ - `nllb-better` has Rai-family generated/validation artifacts and a language
171
+ registry that should be inspected before building the OCR registry.
172
+
173
+ ## Data Plan
174
+
175
+ ### Real Validation Data
176
+
177
+ Use Gorkhapatra/Naya Nepal as the central real-publication validation source.
178
+
179
+ Create a scraper that emits:
180
+
181
+ - article URL
182
+ - article title
183
+ - language label extracted from title
184
+ - article image URLs
185
+ - epaper date
186
+ - epaper PDF URL
187
+ - direct PDF file URL
188
+ - image/PDF hashes
189
+ - source timestamp
190
+ - license/terms note
191
+ - manual-review status
192
+
193
+ Then create a hand-labeled validation pack:
194
+
195
+ - at least 10 languages
196
+ - at least 20 pages or page crops per language for v1 if available
197
+ - Limbu set first
198
+ - no synthetic samples in the final real validation pack
199
+ - references include text, line boxes, reading order, table cells, captions, and
200
+ Markdown where applicable
201
+
202
+ ### Synthetic Training Data
203
+
204
+ Use real text when synthesizing:
205
+
206
+ - Limbu dictionary/Pangloss/IGT text first.
207
+ - Tamang NepTam and dictionary text.
208
+ - Devanagari text from Gorkhapatra/Loksewa/government-like documents where
209
+ licensing permits.
210
+ - Rai-family text from verified local/remote corpora.
211
+
212
+ Synthetic generation must produce:
213
+
214
+ - line crops
215
+ - word crops
216
+ - table cell crops
217
+ - captions
218
+ - full pages
219
+ - degraded scan/photo variants
220
+ - PaddleOCR recognition labels
221
+ - document JSON/Markdown references
222
+
223
+ Every synthetic sample must carry:
224
+
225
+ - language
226
+ - script
227
+ - Unicode ranges observed
228
+ - font
229
+ - source text hash
230
+ - rendered image hash
231
+ - template id
232
+ - degradation id
233
+ - train/eval split id
234
+
235
+ ### Handwriting Synthesis
236
+
237
+ Handwriting is not a v1 blocker. It can be a later ablation.
238
+
239
+ Possible direction:
240
+
241
+ - Use available fonts and handwritten-style fonts for cheap diversity.
242
+ - Explore diffusion/DM-based handwriting generation only after print OCR is
243
+ stable.
244
+ - For Limbu, do not synthesize handwriting until we have real handwritten or
245
+ community-reviewed glyph examples. Otherwise we risk generating plausible but
246
+ wrong orthography.
247
+
248
+ If we explore One-DM or related diffusion methods, it should be framed as
249
+ optional augmentation, not the main claim.
250
+
251
+ ## Validation Gates
252
+
253
+ ### Gate 0: Source Audit
254
+
255
+ Pass only if:
256
+
257
+ - Gorkhapatra article image URLs are downloadable.
258
+ - Epaper direct PDFs are downloadable.
259
+ - PDF text/image extraction path is implemented and audited.
260
+ - Every source artifact is hashed.
261
+ - Source license/terms risk is recorded.
262
+
263
+ ### Gate 1: Language Registry
264
+
265
+ Pass only if every target language has:
266
+
267
+ - language name
268
+ - ISO code if known
269
+ - script name
270
+ - Unicode range
271
+ - font list
272
+ - corpus source
273
+ - renderability audit
274
+ - normalization policy
275
+ - PaddleOCR dictionary path
276
+ - evaluation availability
277
+
278
+ ### Gate 2: Limbu First
279
+
280
+ Pass only if:
281
+
282
+ - Limbu Unicode rendering works with selected fonts.
283
+ - Limbu PaddleOCR dictionary exists.
284
+ - Limbu synthetic crop/page generator emits valid labels.
285
+ - Limbu held-out real or manually reviewed eval exists.
286
+ - Stock baselines are recorded before fine-tuning.
287
+
288
+ ### Gate 3: PaddleOCR Baselines
289
+
290
+ Run before any claim:
291
+
292
+ - stock PP-OCRv6
293
+ - stock PP-OCRv5 if available
294
+ - Tesseract where traineddata exists
295
+ - Surya
296
+ - GLM-OCR
297
+ - PaddleOCR-VL-1.6
298
+
299
+ ### Gate 4: Fine-Tuned Candidate
300
+
301
+ The candidate must beat baselines on:
302
+
303
+ - CER/WER per language
304
+ - strict and normalized Unicode CER
305
+ - exact-line accuracy
306
+ - reading order
307
+ - table text and structure
308
+ - figure/caption detection
309
+ - Markdown reconstruction
310
+ - JSON artifact completeness
311
+
312
+ English and Nepali must not regress versus the chosen stock baseline.
313
+
314
+ ### Gate 5: Claim Review
315
+
316
+ The paper claim is allowed only if:
317
+
318
+ - all public/real validation rows have complete artifacts
319
+ - fallback usage is reported separately
320
+ - synthetic-only improvements are not used as SOTA evidence
321
+ - failures are included in error analysis
322
+ - a reproducibility bundle can rerun the benchmark
323
+
324
+ ## Immediate Implementation Steps
325
+
326
+ 1. Add `ocrtech audit-gorkhapatra-source`. **Implemented:** the command audits
327
+ Naya Nepal article images and Gorkhapatra epaper PDF links, resolves PDF.js
328
+ viewer URLs to direct `/uploads/file/...pdf` URLs, optionally downloads and
329
+ hashes assets, and records PyMuPDF text-layer metadata when available.
330
+ 2. Add `ocrtech prepare-gorkhapatra-pack`. **Implemented:** the command turns a
331
+ source-audit JSON file into a pending-reference validation manifest, copies
332
+ audited article images, renders audited epaper PDF pages when PyMuPDF is
333
+ available, writes placeholder reference JSON files, and marks samples as
334
+ `reference_status=pending_manual_label` and
335
+ `claim_evidence_eligible=false` until references are verified.
336
+ 3. Add `languages/nepal-v1.json` registry with Limbu first and at least ten
337
+ target languages. **Implemented:** the registry tracks Limbu/Sirijonga first,
338
+ more than ten Nepal-language targets, English as a non-counted regression
339
+ language, script/Unicode ranges, font candidates, corpus sources, Gorkhapatra
340
+ labels, validation status, and PaddleOCR dictionary targets. The
341
+ `ocrtech audit-language-registry` command validates the registry and writes
342
+ JSON/Markdown audit artifacts.
343
+ 4. Add renderability audit for Unicode ranges and fonts. **Implemented:** the
344
+ `ocrtech audit-font-renderability` command reads TTF/OTF/TTC cmap tables,
345
+ checks registered font candidates against each script's Unicode ranges, and
346
+ writes JSON/Markdown evidence. Raster sample rendering is supported when
347
+ Pillow is installed. Current cmap evidence verifies Limbu, Devanagari,
348
+ Santhali/Ol Chiki, and Latin coverage when the registered runtime font assets
349
+ are present.
350
+ 5. Add Limbu synthetic line/page generator.
351
+ 6. Add PP-OCRv6 stock baseline runner.
352
+ 7. Build the first Limbu validation pack from Gorkhapatra/Naya Nepal plus local
353
+ Limbu resources.
354
+
355
+ ## External References
356
+
357
+ - PaddleOCR docs: <https://paddlepaddle.github.io/PaddleOCR/main/en/index.html>
358
+ - PaddleOCR repo: <https://github.com/PaddlePaddle/PaddleOCR>
359
+ - PP-StructureV3 docs:
360
+ <https://paddlepaddle.github.io/PaddleOCR/main/en/version3.x/pipeline_usage/PP-StructureV3.html>
361
+ - PaddleOCR-VL-1.6 docs:
362
+ <https://paddlepaddle.github.io/PaddleOCR/main/en/version3.x/algorithm/PaddleOCR-VL/PaddleOCR-VL-1.6.html>
363
+ - Gorkhapatra Naya Nepal category:
364
+ <https://gorkhapatraonline.com/categories/new-nepal>
365
+ - Gorkhapatra Naya Nepal publication:
366
+ <https://gorkhapatraonline.com/publications/naya-nepal>
367
+ - Gorkhapatra epaper list example:
368
+ <https://epaper.gorkhapatraonline.com/epaper-list?pdfdaterange1=2026-06-09&pdfdaterange2=2026-06-16&slug=gorkhapatra>
369
+ - Unicode Limbu chart: <https://www.unicode.org/charts/PDF/U1900.pdf>
370
+ - Kirat Rai Unicode proposal:
371
+ <https://www.unicode.org/L2/L2022/22043-kirat-rai.pdf>
docs/PAPER_NOTES.md ADDED
The diff for this file is too large to render. See raw diff
 
docs/PAPER_RESULTS.md ADDED
@@ -0,0 +1,854 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Paper Results
2
+
3
+ This file is the shortest path from repo artifacts to a paper draft.
4
+
5
+ Use [PAPER_NOTES.md](PAPER_NOTES.md) for claim boundaries and framing.
6
+ Use this file for the concrete result tables and the branch chronology.
7
+
8
+ ## Current Comparison Tables
9
+
10
+ ### Rendered Claim Pack
11
+
12
+ Generated artifact:
13
+
14
+ - [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-table/paper-table.md)
15
+ - [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-table/paper-table.json)
16
+
17
+ Headline table:
18
+
19
+ | system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy | latency_seconds |
20
+ | --- | ---: | ---: | ---: | ---: | ---: |
21
+ | Candidate | 25 | 0.248654 | 0.955556 | 0.764000 | 3.268937 |
22
+ | Surya | 25 | 0.195260 | 0.955556 | 0.750909 | 2.618872 |
23
+ | Tesseract | 25 | 0.689214 | 0.000000 | 0.816000 | 0.240963 |
24
+ | Stock Paddle | 25 | 1.749081 | 0.000000 | 1.000000 | 7.732034 |
25
+
26
+ Interpretation:
27
+
28
+ - Candidate beats Tesseract and stock-Paddle badly on text CER and table extraction.
29
+ - Candidate ties Surya on aggregate `table_cell_f1`.
30
+ - Candidate is slightly better than Surya on aggregate reading-order accuracy.
31
+ - Candidate still loses to raw Surya on text CER.
32
+ - Candidate is also slower than Tesseract and somewhat slower than Surya on this pack.
33
+
34
+ Latency caveat:
35
+
36
+ - these latency means come from separate benchmark reports on the same host and pack, then get pinned per system during table export
37
+ - treat them as comparable operational signals, not as a single synchronized runtime shootout
38
+
39
+ ### Rendered Claim Pack V2 Holdout
40
+
41
+ Generated artifact:
42
+
43
+ - [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-table/paper-table.md)
44
+ - [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-table/paper-table.json)
45
+ - [preflight.md](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-script-router/preflight.md)
46
+
47
+ Headline table:
48
+
49
+ | system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy |
50
+ | --- | ---: | ---: | ---: | ---: |
51
+ | Candidate | 50 | 0.180893 | 0.947500 | 0.750909 |
52
+ | Surya | 50 | 0.180893 | 0.970833 | 0.750909 |
53
+ | Tesseract | 50 | 0.714207 | 0.000000 | 0.787818 |
54
+
55
+ Interpretation:
56
+
57
+ - This is the stronger rendered-document holdout because calibration and holdout were separated at `50/50`.
58
+ - The routed candidate does not beat Surya here. It ties Surya exactly on text CER and reading-order accuracy.
59
+ - The routed candidate is slightly worse than raw Surya on aggregate `table_cell_f1`.
60
+ - The calibration artifact selected `primary_script_threshold=0.0`, which means the routed candidate collapsed to "always pick Surya" on the calibration split.
61
+ - That is why claim preflight now rejects this candidate as a routed model fit for a broad "better engine" claim.
62
+
63
+ ### Rendered Claim Pack V2 Holdout - Surya Primary Latin Router
64
+
65
+ Generated artifact:
66
+
67
+ - [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-latin-table/paper-table.md)
68
+ - [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-latin-table/paper-table.json)
69
+
70
+ Headline table:
71
+
72
+ | system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy | latency_seconds |
73
+ | --- | ---: | ---: | ---: | ---: | ---: |
74
+ | Candidate | 50 | 0.416854 | 0.337500 | 0.761818 | 6.067455 |
75
+ | Surya | 50 | 0.180893 | 0.970833 | 0.750909 | 6.699851 |
76
+ | Tesseract | 50 | 0.714207 | 0.000000 | 0.787818 | 0.238464 |
77
+
78
+ Interpretation:
79
+
80
+ - This candidate was a Surya-primary, Latin-triggered page router with Tesseract fallback.
81
+ - It was meant to preserve Surya's text strength while using Tesseract for English-heavy pages.
82
+ - It failed badly on the rendered-v2 holdout: worse than raw Surya on text CER and much worse on table structure.
83
+ - Paired CER against Surya was decisive enough to rule the branch out: `0` wins, `12` losses, `38` ties.
84
+
85
+ ### Rendered Claim Pack V2 Holdout - Surya Quality Router
86
+
87
+ Generated artifact:
88
+
89
+ - [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-quality-router-table/paper-table.md)
90
+ - [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-quality-router-table/paper-table.json)
91
+ - [calibration.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-surya-quality-router/calibration.json)
92
+
93
+ Headline table:
94
+
95
+ | system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy | latency_seconds |
96
+ | --- | ---: | ---: | ---: | ---: | ---: |
97
+ | Candidate | 50 | 0.181128 | 0.947500 | 0.750909 | 4.620400 |
98
+ | Surya | 50 | 0.180893 | 0.947500 | 0.750909 | 5.603801 |
99
+ | Tesseract | 50 | 0.714207 | 0.000000 | 0.787818 | 0.240773 |
100
+
101
+ Interpretation:
102
+
103
+ - This candidate was a Surya-vs-Tesseract quality selector calibrated on the rendered-v2 calibration split.
104
+ - Calibration selected `selected_threshold=0.0`, which means the quality gate kept Surya whenever it was not scored worse than Tesseract.
105
+ - On the untouched holdout, the candidate still did not beat raw Surya on text CER.
106
+ - Paired CER against Surya was effectively a tie with one bad miss: `0` wins, `1` loss, `49` ties.
107
+ - This is operationally closer to "Surya with a guardrail" than to a meaningfully new OCR engine.
108
+
109
+ ### Rendered Claim Pack V2 Holdout - Line Align Composite
110
+
111
+ Generated artifact:
112
+
113
+ - [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-line-align-table/paper-table.md)
114
+ - [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-line-align-table/paper-table.json)
115
+ - [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-line-align-summary/summary.json)
116
+
117
+ Headline table:
118
+
119
+ | system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy | latency_seconds |
120
+ | --- | ---: | ---: | ---: | ---: | ---: |
121
+ | Candidate | 50 | 0.310549 | 0.897500 | 0.792727 | 4.142242 |
122
+ | Surya | 50 | 0.180893 | 0.970833 | 0.750909 | 3.223913 |
123
+ | Tesseract | 50 | 0.714207 | 0.000000 | 0.787818 | 0.235074 |
124
+
125
+ Interpretation:
126
+
127
+ - This candidate kept Surya as the text engine and reused Tesseract line scaffolding through `line_align_composite`.
128
+ - It did improve reading-order accuracy relative to raw Surya, and even edged past raw Tesseract on this holdout.
129
+ - That gain is not enough. CER regressed badly versus Surya, and aggregate `table_cell_f1` also regressed.
130
+ - Paired CER against Surya was one-sided enough to kill the branch: `0` wins, `7` losses, `43` ties.
131
+ - This is a useful negative result for the paper because it shows that better line ordering alone does not rescue the Surya/Tesseract composite family.
132
+
133
+ ### Rendered Claim Pack V2 Holdout - Sample-Level Oracle
134
+
135
+ Generated artifact:
136
+
137
+ - [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-oracle-table/paper-table.md)
138
+ - [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-oracle-table/paper-table.json)
139
+ - [oracle-analysis.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-analysis.json)
140
+ - [oracle-choices.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-choices.json)
141
+
142
+ Headline table:
143
+
144
+ | system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy | latency_seconds |
145
+ | --- | ---: | ---: | ---: | ---: | ---: |
146
+ | Oracle Surya/Tesseract | 50 | 0.180529 | 0.970833 | 0.750909 | 3.193115 |
147
+ | Surya | 50 | 0.180893 | 0.970833 | 0.750909 | 3.223913 |
148
+ | Tesseract | 50 | 0.714207 | 0.000000 | 0.787818 | 0.235074 |
149
+
150
+ Interpretation:
151
+
152
+ - This is not a deployable system. It is an upper-bound analysis that picks the better of raw Surya and raw Tesseract per sample using ground-truth CER.
153
+ - The oracle picked Surya for `49` of `50` holdout samples and Tesseract for only `1`.
154
+ - Per-sample oracle selections are recorded separately in `oracle-choices.json`; they are provenance, not benchmark metrics.
155
+ - The absolute CER gain over raw Surya is only `0.000364`.
156
+ - That means the current Surya/Tesseract routing space has almost no text-OCR headroom left on rendered-v2.
157
+ - Future work should move to a stronger recognizer/document model, not more Surya/Tesseract routing heuristics.
158
+
159
+ ### Rendered Claim Pack V2 Holdout - Surya Error Pockets
160
+
161
+ Generated artifact:
162
+
163
+ - [error-analysis.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-error-analysis/error-analysis.md)
164
+ - [top-failures.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-error-analysis/top-failures.json)
165
+ - [surya-vs-tesseract error-analysis.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-vs-tesseract-error-analysis/error-analysis.md)
166
+ - [surya-vs-oracle error-analysis.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-vs-oracle-error-analysis/error-analysis.md)
167
+ - [eval-surya-hard15.jsonl](/Users/cdjk/github/llm/ocr-tech/data/splits-claimpack-rendered-v2/eval-surya-hard15.jsonl)
168
+ - [validation-hard15-surya.json](/Users/cdjk/github/llm/ocr-tech/configs/validation-hard15-surya.json)
169
+ - [line-align hard15 validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-claimpack-v2-line-align-hard15/validation.md)
170
+ - [k2 manifest-audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/audit-claimpack-v2-surya-hard15-k2/manifest-audit.md)
171
+ - [table/figure dev pack manifest](/Users/cdjk/github/llm/ocr-tech/data/eval-pack-table-figure-dev-v1/manifests/ocrtech-eval-pack.jsonl)
172
+ - [table/figure dev pack audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/audit-eval-pack-table-figure-dev-v1-k2/eval-pack-audit.md)
173
+
174
+ Diagnostic summary:
175
+
176
+ | slice | sample_count | Surya CER |
177
+ | --- | ---: | ---: |
178
+ | figure | 20 | 0.253157 |
179
+ | table | 20 | 0.252781 |
180
+ | multi_page | 10 | 0.196472 |
181
+ | scan | 10 | 0.194861 |
182
+ | low_contrast | 10 | 0.193317 |
183
+ | blur | 10 | 0.190940 |
184
+ | clean | 10 | 0.182256 |
185
+
186
+ Interpretation:
187
+
188
+ - Surya's hardest absolute CER pockets on rendered-v2 are table-ledger and figure-caption samples, not generic blur/skew alone.
189
+ - Tesseract is worse than Surya on `49` of `50` samples and better on only `eval-form-notice-003-blur`; that single oracle win is too small to justify more Surya/Tesseract routing work.
190
+ - A ranked `15`-sample `hard_eval` manifest now exists for the next model branch; it contains `8` table samples and `7` figure samples, and its full file/reference audit passes on `k2`.
191
+ - `configs/validation-hard15-surya.json` defines the pocket gate a future candidate must clear against Surya, including intersection gates for `hard_eval,table` and `hard_eval,figure`.
192
+ - The current line-align candidate fails the hard15 gate: hard-eval CER is `0.482039` versus Surya `0.348706`, and hard table CER is `0.613636` versus Surya `0.363636`.
193
+ - A separate `80`-sample image development pack now exists for table/figure work: `40` table samples, `40` figure samples, all five degradations. It has zero sample-id overlap and zero text-hash overlap with hard15, and its `k2` audit passes.
194
+ - That development pack is not claim evidence; it is for model iteration before returning to the held-out hard15 and rendered-v2 gates.
195
+ - The next credible recognizer branch should target table/figure-heavy mixed-script pages and preserve Surya's already-strong behavior on the other `49` samples.
196
+
197
+ ### Table/Figure Development Pack - HF TrOCR Negative Result
198
+
199
+ Generated artifact:
200
+
201
+ - [split-summary.md](/Users/cdjk/github/llm/ocr-tech/data/splits-table-figure-dev-v1/split-summary.md)
202
+ - [training_metrics.json](/Users/cdjk/github/llm/ocr-tech/runs/hf-table-figure-dev-v1/training_metrics.json)
203
+ - [model-card.json](/Users/cdjk/github/llm/ocr-tech/runs/hf-table-figure-dev-v1/package/model-card.json)
204
+ - [model-audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/audit-hf-table-figure-dev-v1-model/model-audit.md)
205
+ - [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-hf-table-figure-dev-v1/summary.md)
206
+ - [benchmark report.json](/Users/cdjk/github/llm/ocr-tech/outputs/experiment-hf-table-figure-dev-v1/benchmark/report.json)
207
+ - [paddle_recognizer.yml](/Users/cdjk/github/llm/ocr-tech/runs/paddle-table-figure-dev-v1/paddle_recognizer.yml)
208
+
209
+ Pack and split:
210
+
211
+ - `80` image samples: `40` table and `40` figure
212
+ - degradations: `clean`, `scan`, `blur`, `low_contrast`, `skew`
213
+ - split: `60` train / `20` eval
214
+ - grouping: `text_sha256`
215
+ - text groups: `16`
216
+ - split leakage: pass
217
+ - train/eval manifest audits on `k2`: pass
218
+ - hard15 overlap: `0` sample IDs and `0` text hashes
219
+
220
+ Training result for `microsoft/trocr-small-printed` after one epoch:
221
+
222
+ | metric | value |
223
+ | --- | ---: |
224
+ | eval_loss | 7.884517 |
225
+ | eval_cer | 3.062119 |
226
+ | eval_wer | 3.562689 |
227
+ | eval_exact_line_accuracy | 0.000000 |
228
+
229
+ Benchmark result on the `20`-sample dev eval:
230
+
231
+ | system | CER | table_cell_f1 | reading_order_pair_accuracy |
232
+ | --- | ---: | ---: | ---: |
233
+ | HF table/figure candidate | 2.422001 | 0.000000 | 0.000000 |
234
+ | Surya | 0.563399 | 0.858333 | 0.550000 |
235
+ | Tesseract | 0.546227 | 0.000000 | 0.695000 |
236
+
237
+ Interpretation:
238
+
239
+ - The model package audits cleanly, so this is a quality failure rather than a packaging failure.
240
+ - Paired CER was `0` wins, `20` losses, and `0` ties against Surya.
241
+ - Paired CER was also `0` wins, `20` losses, and `0` ties against Tesseract.
242
+ - Naive full-page TrOCR fine-tuning is not a serious next step for table/figure-heavy document pages.
243
+ - The next model branch should use line/cell crops or a structure-aware page/document model rather than treating full document images as single recognition targets.
244
+
245
+ ### Table/Figure Crop Development Pack - HF TrOCR Negative Result
246
+
247
+ Generated artifact:
248
+
249
+ - [crop-manifest-summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/crops-table-figure-dev-v1-k2/crop-manifest-summary.md)
250
+ - [crop manifest audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/crops-table-figure-dev-v1-k2/manifest-audit.md)
251
+ - [crop split-summary.md](/Users/cdjk/github/llm/ocr-tech/data/splits-crops-table-figure-dev-v1/split-summary.md)
252
+ - [training_metrics.json](/Users/cdjk/github/llm/ocr-tech/runs/hf-crops-table-figure-dev-v1/training_metrics.json)
253
+ - [model-card.json](/Users/cdjk/github/llm/ocr-tech/runs/hf-crops-table-figure-dev-v1/package/model-card.json)
254
+ - [model-audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/audit-hf-crops-table-figure-dev-v1-model/model-audit.md)
255
+ - [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-hf-crops-table-figure-dev-v1/summary.md)
256
+ - [benchmark report.json](/Users/cdjk/github/llm/ocr-tech/outputs/experiment-hf-crops-table-figure-dev-v1/benchmark/report.json)
257
+
258
+ Pack and split:
259
+
260
+ - source rendered samples: `80`
261
+ - extracted crops: `544`
262
+ - crop types: `384` table cells, `128` lines, `32` figure captions
263
+ - skipped candidates: `32` skew pages because pre-rotation bboxes are invalid
264
+ - split: `480` train / `64` eval
265
+ - grouping: `text_sha256`
266
+ - text groups: `68`
267
+ - crop manifest audit on `k2`: pass
268
+ - split leakage: pass
269
+
270
+ Training result for `microsoft/trocr-small-printed` after one crop-level epoch:
271
+
272
+ | metric | value |
273
+ | --- | ---: |
274
+ | eval_loss | 3.087561 |
275
+ | eval_cer | 0.959831 |
276
+ | eval_wer | 1.363715 |
277
+ | eval_exact_line_accuracy | 0.078125 |
278
+
279
+ Benchmark result on the `64`-sample crop eval:
280
+
281
+ | system | CER |
282
+ | --- | ---: |
283
+ | HF crop candidate | 1.043369 |
284
+ | Surya | 0.475309 |
285
+ | Tesseract | 0.077453 |
286
+
287
+ Interpretation:
288
+
289
+ - Crop-level training is a better problem shape than full-page recognition, but this recognizer is still not competitive.
290
+ - Paired CER against Surya was `7` wins, `50` losses, and `7` ties.
291
+ - Paired CER against Tesseract was `0` wins, `64` losses, and `0` ties.
292
+ - Tesseract is extremely strong on these rendered crop images, so near-term gains are more likely from better crop/layout detection plus strong crop OCR than from this small synthetic TrOCR fine-tune.
293
+
294
+ ### Hard15 - Table Cell Refinement Composite
295
+
296
+ Generated artifact:
297
+
298
+ - [unsafe model-card.json](/Users/cdjk/github/llm/ocr-tech/outputs/table-cell-refine-v1/model-card.json)
299
+ - [unsafe summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-table-cell-refine-hard15-v1/summary.md)
300
+ - [unsafe cell-analysis.md](/Users/cdjk/github/llm/ocr-tech/outputs/table-cell-analysis-refine-always-hard15-v1/table-cell-analysis.md)
301
+ - [safe model-card.json](/Users/cdjk/github/llm/ocr-tech/outputs/table-cell-refine-empty-v1/model-card.json)
302
+ - [safe summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-table-cell-refine-empty-hard15-v1/summary.md)
303
+ - [safe cell-analysis.md](/Users/cdjk/github/llm/ocr-tech/outputs/table-cell-analysis-refine-empty-hard15-v1/table-cell-analysis.md)
304
+ - [safe validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-table-cell-refine-empty-hard15-v1/validation.md)
305
+ - [safe+dedup model-card.json](/Users/cdjk/github/llm/ocr-tech/outputs/table-cell-refine-empty-dedup-v1/model-card.json)
306
+ - [safe+dedup summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-table-cell-refine-empty-dedup-hard15-v3/summary.md)
307
+ - [safe+dedup cell-analysis.md](/Users/cdjk/github/llm/ocr-tech/outputs/table-cell-analysis-refine-empty-dedup-hard15-v3/table-cell-analysis.md)
308
+ - [safe+dedup validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-table-cell-refine-empty-dedup-hard15-v3/validation.md)
309
+
310
+ Headline hard15 comparison:
311
+
312
+ | system | hard_eval CER | table_cell_f1 | reading_order_pair_accuracy |
313
+ | --- | ---: | ---: | ---: |
314
+ | Surya baseline | 0.348706 | 0.979167 | 0.520000 |
315
+ | Table-cell refine, `always` | 0.348706 | 0.593750 | 0.520000 |
316
+ | Table-cell refine, `empty` | 0.348706 | 0.979167 | 0.520000 |
317
+ | Table-cell refine, `empty` + parser de-dup | 0.348706 | 0.979167 | 0.520000 |
318
+
319
+ Interpretation:
320
+
321
+ - The unsafe table-cell crop replacement policy damaged Surya's already-good table cells.
322
+ - The conservative empty-cell-only policy avoided regression but did not improve any hard15 metric.
323
+ - The safe policy's hard15 validation status is `not_validated`.
324
+ - The parser de-dup branch removes duplicate overlapping table exports for both candidate and Surya, so it improves artifact cleanliness but does not create a candidate advantage.
325
+ - Exact crop OCR being strong is not enough; the deployment problem is reliable cell detection and knowing when a replacement is safer than the structure engine's existing cell text.
326
+
327
+ Cell-level result:
328
+
329
+ - Surya is already exact on `94/96` hard15 reference table cells.
330
+ - Unsafe replacement is exact on only `57/96` cells.
331
+ - Unsafe replacement improves `0` cells over Surya and worsens `37`.
332
+ - Safe+dedup is exact on `94/96` cells, with `8` predicted tables for `8` reference tables and `0` duplicate overlapping tables on the v3 rerun.
333
+
334
+ ### Hard15 - Figure Caption Refinement Composite
335
+
336
+ Generated artifact:
337
+
338
+ - [model-card.json](/Users/cdjk/github/llm/ocr-tech/outputs/figure-caption-refine-v1/model-card.json)
339
+ - [model-audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/audit-figure-caption-refine-v1-model/model-audit.md)
340
+ - [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-figure-caption-refine-hard15-v2/summary.md)
341
+ - [benchmark report.json](/Users/cdjk/github/llm/ocr-tech/outputs/experiment-figure-caption-refine-hard15-v2/benchmark/report.json)
342
+ - [validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-figure-caption-refine-hard15-v2/validation.md)
343
+ - [labels model-card.json](/Users/cdjk/github/llm/ocr-tech/outputs/figure-caption-refine-labels-v1/model-card.json)
344
+ - [labels summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-figure-caption-refine-labels-hard15-v1/summary.md)
345
+ - [labels validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-figure-caption-refine-labels-hard15-v1/validation.md)
346
+
347
+ Headline hard15 comparison after the parser caption-attachment fix:
348
+
349
+ | system | CER | figure_caption_cer | figure_detection_f1 | table_cell_f1 | reading_order_pair_accuracy |
350
+ | --- | ---: | ---: | ---: | ---: | ---: |
351
+ | Surya | 0.236839 | 0.092323 | 1.000000 | 0.979167 | 0.520000 |
352
+ | Tesseract | 1.266364 | 0.051020 | 0.857143 | 0.000000 | 0.613333 |
353
+ | Figure-caption refine | 0.228008 | 0.052478 | 1.000000 | 0.979167 | 0.520000 |
354
+ | Figure-caption refine + labels | 0.204610 | 0.008746 | 1.000000 | 0.979167 | 0.520000 |
355
+
356
+ Validation result:
357
+
358
+ - basic branch overall status: `not_validated`
359
+ - basic branch figure-caption gate: pass, `6` wins / `0` losses / `1` tie against Surya, improvement `0.039845`
360
+ - label branch overall status: `not_validated`
361
+ - label branch overall hard-eval CER gate: pass, improvement `0.032228`
362
+ - label branch figure CER gate: pass, improvement `0.069061`
363
+ - label branch figure-caption gate: pass, `7` wins / `0` losses / `0` ties against Surya, improvement `0.083576`
364
+ - label branch table gates: fail, ties against Surya
365
+
366
+ Interpretation:
367
+
368
+ - This is a real slice win over Surya on figure-caption extraction.
369
+ - It is not broad enough for a SOTA claim.
370
+ - Tesseract has better caption text but worse figure detection/table structure; the composite keeps Surya's structure and selectively borrows caption text.
371
+ - Safe empty-only replacement ties all `96` cells with Surya because there are no missing Surya cells in this pocket.
372
+ - The only remaining Surya cell errors are `किरण` predicted as `करिण` on `eval-table-ledger-002-clean` and `eval-table-ledger-002-scan`.
373
+
374
+ ### Rendered Blur-Only Expensive-Baseline Subset
375
+
376
+ Generated artifact:
377
+
378
+ - [paper-table.md](/Users/cdjk/github/llm/ocr-tech/outputs/paper-blur5-table/paper-table.md)
379
+ - [paper-table.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-blur5-table/paper-table.json)
380
+
381
+ Headline table:
382
+
383
+ | system | sample_count | cer | table_cell_f1 | reading_order_pair_accuracy | latency_seconds |
384
+ | --- | ---: | ---: | ---: | ---: | ---: |
385
+ | Candidate | 5 | 0.223273 | 1.000000 | 0.783636 | 3.784279 |
386
+ | Surya | 5 | 0.197601 | 1.000000 | 0.750909 | 2.943875 |
387
+ | Tesseract | 5 | 0.732364 | 0.000000 | 0.823636 | 0.220515 |
388
+ | Stock Paddle | 5 | 2.810426 | 0.000000 | 1.000000 | 7.440482 |
389
+ | GLM-OCR | 5 | 2.820269 | 0.000000 | 1.000000 | 16.846619 |
390
+ | PaddleOCR-VL | 5 | 18.470663 | 0.000000 | 0.750909 | 32.793166 |
391
+
392
+ Interpretation:
393
+
394
+ - Candidate is materially stronger than Tesseract, stock-Paddle, GLM-OCR, and PaddleOCR-VL on this degraded rendered subset.
395
+ - Candidate still does not beat raw Surya on text CER.
396
+ - The external VLM-style baselines are also much slower than the current candidate here.
397
+
398
+ ## Branch Chronology
399
+
400
+ This is the shortest honest story of the project so far.
401
+
402
+ | branch | eval setting | candidate CER | baseline CER | paired result | verdict | evidence |
403
+ | --- | --- | ---: | ---: | --- | --- | --- |
404
+ | HF TrOCR recognizer | mixed Nepali/English line-style holdout, 180 samples | 0.586139 | 0.329012 vs Tesseract | 36 wins, 97 losses, 47 ties | runnable but not competitive | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-hf-vs-tesseract-summary/summary.json) |
405
+ | Tesseract + ByT5 corrector | same 180-sample mixed holdout | 6.912065 | 0.329012 vs Tesseract | 0 wins, 170 losses, 10 ties | failed badly | [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-corrector-vs-tesseract-summary/summary.md) |
406
+ | quality router (`psm=6` vs `psm=13`) | 91-sample mixed holdout | 0.340608 | 0.342234 vs Tesseract | 2 wins, 1 loss, 88 ties | tiny gain, plausible baseline only | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-quality-router-holdout/summary.json) |
407
+ | ranked PSM ensemble (`6/7/11/13`) | same 91-sample mixed holdout | 0.358352 | 0.342234 vs Tesseract | 9 wins, 16 losses, 66 ties | overfit, worse than baseline | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-psm-ensemble-holdout/summary.json) |
408
+ | script router (`tesseract -> surya`) | same 91-sample mixed holdout | 0.313084 | 0.342234 vs Tesseract | 9 wins, 0 losses, 82 ties | strongest older line-style result | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-script-router-holdout/summary.json) |
409
+ | script router + structure backfill | rendered claim-ready document pack, 25 samples | 0.248654 | 0.689214 vs Tesseract | 9 wins, 1 loss, 15 ties | strong document-pack improvement over Tesseract | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-rendered-backfill/summary.json) |
410
+ | raw Surya on rendered claim pack | rendered claim-ready document pack, 25 samples | 0.195260 | 0.248654 vs Candidate | candidate loses on text CER | current text leader on rendered pack | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-surya-compare/summary.json) |
411
+ | rendered-v2 script router | rendered claim-ready document holdout, 50 samples | 0.180893 | 0.180893 vs Surya | exact CER tie, worse table structure | routed candidate collapses to Surya on calibration; not a meaningful new engine | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-script-router/summary.json) |
412
+ | rendered-v2 Surya-primary Latin router | rendered claim-ready document holdout, 50 samples | 0.416854 | 0.180893 vs Surya | 0 wins, 12 losses, 38 ties on CER | failed badly; inverted script routing is not a viable answer | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-surya-latin-router/summary.json) |
413
+ | rendered-v2 Surya quality router | rendered claim-ready document holdout, 50 samples | 0.181128 | 0.180893 vs Surya | 0 wins, 1 loss, 49 ties on CER | nearly indistinguishable from Surya and still not better | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-surya-quality-router/summary.json) |
414
+ | rendered-v2 line-align composite | rendered claim-ready document holdout, 50 samples | 0.310549 | 0.180893 vs Surya | 0 wins, 7 losses, 43 ties on CER | better reading order, but text and table regressions are too large | [summary.json](/Users/cdjk/github/llm/ocr-tech/outputs/live-k2-claimpack-v2-line-align-summary/summary.json) |
415
+ | rendered-v2 Surya/Tesseract sample oracle | rendered claim-ready document holdout, 50 samples | 0.180529 | 0.180893 vs Surya | oracle picks Tesseract only 1/50 samples | upper bound shows routing headroom is essentially exhausted | [oracle-analysis.json](/Users/cdjk/github/llm/ocr-tech/outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-analysis.json) |
416
+ | HF TrOCR table/figure dev | table/figure development eval, 20 samples | 2.422001 | 0.563399 vs Surya; 0.546227 vs Tesseract | 0 wins, 20 losses, 0 ties vs both | failed badly; full-page TrOCR fine-tuning is not viable here | [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-hf-table-figure-dev-v1/summary.md) |
417
+ | HF TrOCR crop dev | table/figure crop eval, 64 samples | 1.043369 | 0.475309 vs Surya; 0.077453 vs Tesseract | 7 wins, 50 losses, 7 ties vs Surya; 0 wins, 64 losses vs Tesseract | crop shape helps but this recognizer still fails | [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-hf-crops-table-figure-dev-v1/summary.md) |
418
+ | table-cell refine, unsafe | hard15 rendered-v2 pocket, 15 samples | 0.348706 | 0.348706 vs Surya | text tie, table F1 regresses from 0.979167 to 0.593750 | replacement policy is unsafe | [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/summary-table-cell-refine-hard15-v1/summary.md) |
419
+ | table-cell refine, empty-only | hard15 rendered-v2 pocket, 15 samples | 0.348706 | 0.348706 vs Surya | exact tie across CER/table/reading order; validation not_validated | safe but no improvement | [validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-table-cell-refine-empty-hard15-v1/validation.md) |
420
+ | normalized script router, hard-timeout rerun | normalized mixed Nepali/English line-style eval, 180 samples | 0.354840 | 0.329012 vs Tesseract; 3.157317 vs Surya on 178 scored Surya rows | 17 wins, 4 losses, 159 ties on CER vs Tesseract but worse aggregate mean; 2 Surya rows timed out at 45s | not a better engine; benchmark watchdog works and routing still loses overall CER | [summary.md](/Users/cdjk/github/llm/ocr-tech/outputs/k2-mixed-v1-normalized-script-router-hard-timeout/outputs/summary-mixed-v1-normalized-script-router-hard-timeout/summary.md) |
421
+ | PaddleOCR recognizer hard-50 failure slice | top 50 failures from AWS G6 trained PaddleOCR recognizer, remapped to normalized k2 eval paths | n/a | n/a | 50 failures: 38 English line-crop synthetic, 12 Nepali synthetic; k2 manifest audit passed | next recognizer gate; do not train on it | [manifest-audit.md](/Users/cdjk/github/llm/ocr-tech/outputs/k2-paddle-mixed-v2-hard50-normalized/outputs/manifest-audit-paddle-mixed-v2-cuda-hard50-normalized/manifest-audit.md) |
422
+ | PaddleOCR hard-50 gate negative control | failed AWS G6 trained PaddleOCR recognizer evaluated on the hard-50 gate | 0.887557 | 0.080249 vs Tesseract | fails all four hard gates; English line-crop CER `0.852048` vs `0.031244` | validates the gate catches the known recognizer collapse | [validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-paddle-mixed-v2-cuda-hard50-tesseract-negative/validation.md) |
423
+ | line-level Tesseract/Paddle composite | hard-50 failure slice, Tesseract primary with PaddleOCR secondary only for Devanagari-looking lines | 0.276489 | 0.120106 vs Tesseract | English line-crop ties exactly (`0.060429` CER); Nepali collapses (`0.960679` vs `0.309083` CER, `1.000000` vs `0.711607` WER) | safe English preservation but failed Devanagari override; do not promote this recognizer | [validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/validation-line-script-paddle-hard50-v2/validation.md) |
424
+ | PaddleOCR v3 CUDA recognizer | G6-trained lower-LR PaddleOCR branch on hard-50 admission gate | 0.921416 | 0.080249 vs Tesseract | 0 wins, 50 losses on CER; English line-crop CER `0.917511` vs `0.031244`; Nepali WER `1.055556` vs `0.716369` | failed worse than v2; stop branch before full mixed eval | [validation.md](/Users/cdjk/github/llm/ocr-tech/outputs/aws-g6-paddle-mixed-v3-cuda/outputs/validation-paddle-mixed-v3-cuda-hard50-rescored/validation.md) |
425
+ | PaddleOCR v4 CUDA recognizer | G6-trained Devanagari PP-OCRv3 branch, LR `0.00005`, 20 epochs, evaluated on hard-50 admission gate | 0.940500 | 0.080249 vs Tesseract | 0 wins, 50 losses on CER; English line-crop CER `0.950730` vs `0.031244`; Nepali CER `0.908108` vs `0.235433` | failed worse than v2/v3; the same PaddleOCR mixed-script fine-tune branch should not be the engine core | [artifact bundle](/Users/cdjk/github/llm/ocr-tech/outputs/aws-g6-paddle-mixed-v4-cuda/ocrtech-paddle-mixed-v4-cuda-artifacts.tgz) |
426
+
427
+ ## Current Best-Evidence Statement
428
+
429
+ The strongest evidence-backed statement available today is:
430
+
431
+ 1. The current Tesseract-plus-Surya backfill candidate is a real document-OCR improvement over Tesseract and several weaker baselines on internal rendered Nepali/English document packs.
432
+ 2. On the larger rendered-v2 holdout, every Tesseract/Surya composite tested so far either collapses to Surya, trails Surya, becomes effectively indistinguishable from Surya, or trades too much OCR accuracy for modest ordering gains.
433
+ 3. On the cleaned 180-sample mixed line-style eval with hard per-sample timeouts, the older script-router candidate also loses to Tesseract overall (`0.354840` vs `0.329012` CER), despite a small overall WER improvement (`0.469944` vs `0.486485`) and a Nepali WER improvement.
434
+ 4. The trained PaddleOCR recognizer failure profile is now materialized as a normalized hard-50 eval gate: all top failures are synthetic line crops, mostly English (`38/50`), so the next recognizer branch must prove English preservation instead of only improving Nepali.
435
+ 5. The line-level Tesseract/Paddle composite proves the candidate shape can preserve English while routing Devanagari-looking lines, but the available trained PaddleOCR recognizer is too weak on the Nepali hard slice to use as an override (`0.960679` vs Tesseract `0.309083` CER).
436
+ 6. The second AWS G6 PaddleOCR attempt fixed recipe/runtime issues but failed the hard-50 gate even worse than the first (`0.921416` vs Tesseract `0.080249` CER), so this PP-OCRv3 Devanagari recipe family is not the engine path.
437
+ 7. A sample-level oracle over raw Surya and raw Tesseract has almost no text-OCR headroom over Surya (`0.180529` vs `0.180893` CER), so more routing in this component family is not a serious path to SOTA.
438
+ 8. The repo now has enough validation discipline and artifact traceability to support a serious comparative paper, but not a SOTA claim.
439
+
440
+ ## Regeneration Workflow
441
+
442
+ Rebuild the generated comparison tables with:
443
+
444
+ ```bash
445
+ ocrtech summarize-benchmark \
446
+ --benchmark-report outputs/live-k2-claimpack-rendered-backfill/report.json \
447
+ --out outputs/paper-claimpack-summary-tesseract \
448
+ --candidate candidate \
449
+ --baseline tesseract \
450
+ --metric cer
451
+ ```
452
+
453
+ ```bash
454
+ ocrtech summarize-benchmark \
455
+ --benchmark-report outputs/live-k2-claimpack-surya-compare/report.json \
456
+ --out outputs/paper-claimpack-summary-surya \
457
+ --candidate candidate \
458
+ --baseline surya \
459
+ --metric cer
460
+ ```
461
+
462
+ ```bash
463
+ ocrtech summarize-benchmark \
464
+ --benchmark-report outputs/live-k2-claimpack-stock-paddle-fixed/report.json \
465
+ --out outputs/paper-claimpack-summary-stock-paddle \
466
+ --candidate candidate \
467
+ --baseline stock-paddle \
468
+ --metric cer
469
+ ```
470
+
471
+ ```bash
472
+ ocrtech paper-benchmark-table \
473
+ --summary outputs/paper-claimpack-summary-tesseract/summary.json \
474
+ --summary outputs/paper-claimpack-summary-surya/summary.json \
475
+ --summary outputs/paper-claimpack-summary-stock-paddle/summary.json \
476
+ --out outputs/paper-claimpack-table \
477
+ --system candidate \
478
+ --system surya \
479
+ --system tesseract \
480
+ --system stock-paddle \
481
+ --label 'candidate=Candidate' \
482
+ --label 'surya=Surya' \
483
+ --label 'tesseract=Tesseract' \
484
+ --label 'stock-paddle=Stock Paddle' \
485
+ --source-preference 'candidate=outputs/paper-claimpack-summary-tesseract/summary.json' \
486
+ --source-preference 'tesseract=outputs/paper-claimpack-summary-tesseract/summary.json' \
487
+ --source-preference 'surya=outputs/paper-claimpack-summary-surya/summary.json' \
488
+ --source-preference 'stock-paddle=outputs/paper-claimpack-summary-stock-paddle/summary.json' \
489
+ --metric cer \
490
+ --metric table_cell_f1 \
491
+ --metric reading_order_pair_accuracy \
492
+ --metric latency_seconds
493
+ ```
494
+
495
+ ```bash
496
+ ocrtech summarize-benchmark \
497
+ --benchmark-report outputs/live-k2-rendered-blur5/report.json \
498
+ --out outputs/paper-blur5-summary \
499
+ --candidate candidate \
500
+ --baseline tesseract \
501
+ --baseline surya \
502
+ --baseline stock-paddle \
503
+ --baseline glm-ocr \
504
+ --baseline paddleocr-vl \
505
+ --metric cer
506
+ ```
507
+
508
+ ```bash
509
+ ocrtech paper-benchmark-table \
510
+ --summary outputs/paper-blur5-summary/summary.json \
511
+ --out outputs/paper-blur5-table \
512
+ --system candidate \
513
+ --system surya \
514
+ --system tesseract \
515
+ --system stock-paddle \
516
+ --system glm-ocr \
517
+ --system paddleocr-vl \
518
+ --label 'candidate=Candidate' \
519
+ --label 'surya=Surya' \
520
+ --label 'tesseract=Tesseract' \
521
+ --label 'stock-paddle=Stock Paddle' \
522
+ --label 'glm-ocr=GLM-OCR' \
523
+ --label 'paddleocr-vl=PaddleOCR-VL' \
524
+ --metric cer \
525
+ --metric table_cell_f1 \
526
+ --metric reading_order_pair_accuracy \
527
+ --metric latency_seconds
528
+ ```
529
+
530
+ ```bash
531
+ ocrtech paper-benchmark-table \
532
+ --summary outputs/live-k2-claimpack-v2-script-router/summary.json \
533
+ --out outputs/paper-claimpack-v2-table \
534
+ --system candidate \
535
+ --system surya \
536
+ --system tesseract \
537
+ --label 'candidate=Candidate' \
538
+ --label 'surya=Surya' \
539
+ --label 'tesseract=Tesseract' \
540
+ --metric cer \
541
+ --metric table_cell_f1 \
542
+ --metric reading_order_pair_accuracy \
543
+ --title 'Rendered Claim Pack V2 Holdout Comparison'
544
+ ```
545
+
546
+ ```bash
547
+ ocrtech paper-benchmark-table \
548
+ --summary outputs/live-k2-claimpack-v2-surya-quality-router/summary.json \
549
+ --out outputs/paper-claimpack-v2-surya-quality-router-table \
550
+ --system candidate \
551
+ --system surya \
552
+ --system tesseract \
553
+ --label 'candidate=Candidate' \
554
+ --label 'surya=Surya' \
555
+ --label 'tesseract=Tesseract' \
556
+ --metric cer \
557
+ --metric table_cell_f1 \
558
+ --metric reading_order_pair_accuracy \
559
+ --metric latency_seconds \
560
+ --title 'Rendered Claim Pack V2 Holdout - Surya Quality Router'
561
+ ```
562
+
563
+ ```bash
564
+ ocrtech paper-benchmark-table \
565
+ --summary outputs/live-k2-claimpack-v2-line-align-summary/summary.json \
566
+ --out outputs/paper-claimpack-v2-line-align-table \
567
+ --system candidate \
568
+ --system surya \
569
+ --system tesseract \
570
+ --label 'candidate=Candidate' \
571
+ --label 'surya=Surya' \
572
+ --label 'tesseract=Tesseract' \
573
+ --metric cer \
574
+ --metric table_cell_f1 \
575
+ --metric reading_order_pair_accuracy \
576
+ --metric latency_seconds \
577
+ --title 'Rendered Claim Pack V2 Holdout - Line Align Composite'
578
+ ```
579
+
580
+ ```bash
581
+ ocrtech benchmark-oracle \
582
+ --benchmark-report outputs/live-k2-claimpack-v2-line-align/report.json \
583
+ --out outputs/paper-claimpack-v2-surya-tesseract-oracle \
584
+ --system surya \
585
+ --system tesseract \
586
+ --metric cer \
587
+ --oracle-name oracle-surya-tesseract
588
+
589
+ ocrtech paper-benchmark-table \
590
+ --summary outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-summary/summary.json \
591
+ --out outputs/paper-claimpack-v2-oracle-table \
592
+ --system oracle-surya-tesseract \
593
+ --system surya \
594
+ --system tesseract \
595
+ --label 'oracle-surya-tesseract=Oracle Surya/Tesseract' \
596
+ --label 'surya=Surya' \
597
+ --label 'tesseract=Tesseract' \
598
+ --metric cer \
599
+ --metric table_cell_f1 \
600
+ --metric reading_order_pair_accuracy \
601
+ --metric latency_seconds \
602
+ --title 'Rendered Claim Pack V2 Holdout - Sample-Level Oracle'
603
+ ```
604
+
605
+ ```bash
606
+ ocrtech benchmark-error-analysis \
607
+ --benchmark-report outputs/live-k2-claimpack-v2-line-align/report.json \
608
+ --out outputs/paper-claimpack-v2-surya-error-analysis \
609
+ --system surya \
610
+ --metric cer \
611
+ --top-n 15
612
+
613
+ ocrtech benchmark-error-analysis \
614
+ --benchmark-report outputs/live-k2-claimpack-v2-line-align/report.json \
615
+ --out outputs/paper-claimpack-v2-surya-vs-tesseract-error-analysis \
616
+ --system surya \
617
+ --baseline tesseract \
618
+ --metric cer \
619
+ --top-n 15
620
+
621
+ ocrtech benchmark-error-analysis \
622
+ --benchmark-report outputs/paper-claimpack-v2-surya-tesseract-oracle/oracle-report.json \
623
+ --out outputs/paper-claimpack-v2-surya-vs-oracle-error-analysis \
624
+ --system surya \
625
+ --baseline oracle-surya-tesseract \
626
+ --metric cer \
627
+ --top-n 10
628
+
629
+ ocrtech export-failure-manifest \
630
+ --source-manifest data/splits-claimpack-rendered-v2/eval.jsonl \
631
+ --top-failures outputs/paper-claimpack-v2-surya-error-analysis/top-failures.json \
632
+ --out data/splits-claimpack-rendered-v2/eval-surya-hard15.jsonl \
633
+ --split hard_eval
634
+
635
+ ocrtech validate-claim \
636
+ --benchmark-report outputs/live-k2-claimpack-v2-line-align/report.json \
637
+ --config configs/validation-hard15-surya.json \
638
+ --eval-manifest data/splits-claimpack-rendered-v2/eval-surya-hard15.jsonl \
639
+ --out outputs/validation-claimpack-v2-line-align-hard15
640
+
641
+ ocrtech create-eval-pack \
642
+ --out data/eval-pack-table-figure-dev-v1 \
643
+ --input-format image \
644
+ --template table-ledger \
645
+ --template figure-caption \
646
+ --count-per-template 8 \
647
+ --variant-offset 100 \
648
+ --degradation clean \
649
+ --degradation scan \
650
+ --degradation blur \
651
+ --degradation low_contrast \
652
+ --degradation skew
653
+ ```
654
+
655
+ Memory still is not part of these generated tables. The current benchmark artifact format does not record it.
656
+
657
+ ## Hard15 Figure-Caption + Table-Lexicon Candidate
658
+
659
+ Latest artifact set:
660
+
661
+ - `outputs/figure-caption-table-lexicon-v1/model-card.json`
662
+ - `outputs/experiment-figure-caption-table-lexicon-hard15-v3/benchmark/report.json`
663
+ - `outputs/summary-figure-caption-table-lexicon-hard15-v3/summary.md`
664
+ - `outputs/table-cell-analysis-figure-caption-table-lexicon-hard15-v3/table-cell-analysis.md`
665
+ - `outputs/validation-figure-caption-table-lexicon-hard15-v3/validation.md`
666
+
667
+ Headline table for internal hard15 only:
668
+
669
+ | system | CER | figure_caption_cer | figure_detection_f1 | table_cell_f1 | reading_order_pair_accuracy |
670
+ | --- | ---: | ---: | ---: | ---: | ---: |
671
+ | candidate | 0.010671 | 0.008746 | 1.000000 | 1.000000 | 0.520000 |
672
+ | Surya | 0.236839 | 0.092323 | 1.000000 | 0.979167 | 0.520000 |
673
+ | Tesseract | 1.266364 | 0.051020 | 0.857143 | 0.000000 | 0.613333 |
674
+
675
+ Validation status is still `not_validated`.
676
+
677
+ Passed configured Surya gates:
678
+
679
+ - overall hard_eval CER: `15` wins / `0` losses / `0` ties
680
+ - table-slice CER: `8` wins / `0` losses / `0` ties
681
+ - figure-slice CER: `7` wins / `0` losses / `0` ties
682
+ - figure-caption CER: `7` wins / `0` losses / `0` ties
683
+
684
+ Remaining configured failure:
685
+
686
+ - table-cell F1: candidate `1.000000`, Surya `0.979167`, but bootstrap CI low is `0.000000` because there are only 2 improving table pages and 6 ties
687
+
688
+ Cell-level table evidence:
689
+
690
+ | system | reference_cells | exact_cells | exact_rate | mean_cer |
691
+ | --- | ---: | ---: | ---: | ---: |
692
+ | candidate | 96 | 96 | 1.000000 | 0.000000 |
693
+ | Surya | 96 | 94 | 0.979167 | 0.010417 |
694
+ | Tesseract | 96 | 0 | 0.000000 | 1.000000 |
695
+
696
+ Use this result as an internal milestone, not a paper claim of broad SOTA. The candidate is lexicon-assisted for table names and uses guarded structural-title corrections (`वार्षिक प्रतिवेदन`, `दैनिक हिसाब`, `चित्र`, `निष्कर्ष`). A publishable table claim needs a larger independent table pack so confidence intervals are meaningful.
697
+
698
+ ## Targeted Table-Ledger Validation Pack V2
699
+
700
+ Artifacts:
701
+
702
+ - `data/eval-pack-table-ledger-claim-v1/manifests/ocrtech-eval-pack.jsonl`
703
+ - `outputs/audit-table-ledger-claim-v1/eval-pack-audit.md`
704
+ - `outputs/experiment-table-ledger-claim-v2/benchmark/report.json`
705
+ - `outputs/summary-table-ledger-claim-v2/summary.md`
706
+ - `outputs/table-cell-analysis-table-ledger-claim-v2/table-cell-analysis.md`
707
+ - `outputs/validation-table-ledger-claim-v2b/validation.md`
708
+ - `configs/validation-table-ledger-surya.json`
709
+
710
+ Pack:
711
+
712
+ - template: `table-ledger`
713
+ - samples: `60`
714
+ - cells: `720`
715
+ - degradations: clean, scan, blur, low_contrast, skew
716
+ - variant offset: `300`
717
+ - claim-ready broad coverage: no, targeted table-only pack
718
+
719
+ Validation status: `validated`.
720
+
721
+ | metric | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
722
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
723
+ | table_cell_f1 | 60 | 15 | 0 | 45 | 1.000000 | 0.979167 | 0.020833 | 0.012500 |
724
+ | table_exact_match | 60 | 15 | 0 | 45 | 1.000000 | 0.750000 | 0.250000 | 0.150000 |
725
+ | table_structure_similarity | 60 | 0 | 0 | 60 | 1.000000 | 1.000000 | 0.000000 | 0.000000 |
726
+
727
+ Cell exactness:
728
+
729
+ | system | reference_cells | exact_cells | exact_rate | mean_cell_cer |
730
+ | --- | ---: | ---: | ---: | ---: |
731
+ | candidate | 720 | 720 | 1.000000 | 0.000000 |
732
+ | Surya | 720 | 705 | 0.979167 | 0.005208 |
733
+ | Tesseract | 720 | 0 | 0.000000 | 0.999868 |
734
+
735
+ Runtime on k2:
736
+
737
+ | system | mean latency seconds | median latency seconds |
738
+ | --- | ---: | ---: |
739
+ | candidate | 3.017929 | 2.967943 |
740
+ | Surya | 2.478481 | 2.453125 |
741
+ | Tesseract | 0.146866 | 0.134735 |
742
+
743
+ Paper-safe claim text:
744
+
745
+ > On a targeted 60-page generated Nepali-English table-ledger validation pack, the lexicon-assisted candidate improved structured table extraction over Surya, reaching 720/720 exact cells versus Surya's 705/720, and passed bootstrap-gated validation for both table-cell F1 and table exact match.
746
+
747
+ More precise paper-safe wording:
748
+
749
+ > On a targeted 60-page generated Nepali-English table-ledger validation pack, the lexicon-assisted candidate improved table cell-content extraction over Surya, reaching 720/720 exact cells versus Surya's 705/720, while matching Surya's perfect grid-structure score.
750
+
751
+ Do not generalize this to all tables. The pack has no merged cells, no nested headers, and no real scanned government tables. The scoring path now supports span-aware structured references, but this pack does not exercise that capability and the metric is still not true HTML-tree TEDS.
752
+
753
+ ## Targeted Complex-Table Validation Pack V1
754
+
755
+ Artifacts:
756
+
757
+ - `data/eval-pack-complex-table-v1/manifests/ocrtech-eval-pack.jsonl`
758
+ - `outputs/audit-complex-table-v1/eval-pack-audit.md`
759
+ - `outputs/experiment-complex-table-v1/benchmark/report.json`
760
+ - `outputs/summary-complex-table-v1/summary.md`
761
+ - `outputs/table-cell-analysis-complex-table-v1/table-cell-analysis.md`
762
+ - `outputs/validation-complex-table-v1/validation.md`
763
+ - `configs/validation-complex-table-surya.json`
764
+
765
+ Pack:
766
+
767
+ - template: `complex-table-ledger`
768
+ - samples: `40`
769
+ - reference cells: `600`
770
+ - span reference: merged header with `colspan=2`
771
+ - degradations: clean, scan, blur, low_contrast
772
+ - broad claim-ready coverage: no
773
+
774
+ Validation status: `validated` for cell-content improvement plus structure non-regression.
775
+
776
+ | metric | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
777
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
778
+ | table_cell_f1 | 40 | 8 | 0 | 32 | 0.928333 | 0.915000 | 0.013333 | 0.005000 |
779
+ | table_exact_match | 40 | 0 | 0 | 40 | 0.000000 | 0.000000 | 0.000000 | 0.000000 |
780
+ | table_structure_similarity | 40 | 0 | 0 | 40 | 0.875000 | 0.875000 | 0.000000 | 0.000000 |
781
+
782
+ Cell exactness:
783
+
784
+ | system | reference_cells | exact_cells | exact_rate | mean_cell_cer |
785
+ | --- | ---: | ---: | ---: | ---: |
786
+ | candidate | 600 | 557 | 0.928333 | 0.067667 |
787
+ | Surya | 600 | 549 | 0.915000 | 0.071000 |
788
+ | Tesseract | 600 | 40 | 0.066667 | 0.933333 |
789
+
790
+ Paper-safe claim text:
791
+
792
+ > On a targeted generated complex-table pack with span-aware references, the lexicon-assisted candidate improved table cell-content F1 over Surya but did not improve merged-header reconstruction; both systems failed exact table match and tied on span-aware structure similarity.
793
+
794
+ This is a negative result for complex table structure. The next model/parser branch must preserve or infer rowspans/colspans before we can claim complex-table document understanding.
795
+
796
+ ## Targeted Complex-Table Span-Inference Pack V1
797
+
798
+ After adding conservative repeated-header colspan inference, we reran the same `40`-sample generated complex-table pack on k2.
799
+
800
+ Artifacts:
801
+
802
+ - `outputs/experiment-complex-table-span-infer-v1/benchmark/report.json`
803
+ - `outputs/experiment-complex-table-span-infer-v1/validation/validation.md`
804
+ - `outputs/summary-complex-table-span-infer-v1/summary.md`
805
+ - `outputs/table-cell-analysis-complex-table-span-infer-v1/table-cell-analysis.md`
806
+
807
+ Validation status: `validated`.
808
+
809
+ | metric | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
810
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
811
+ | table_cell_f1 | 40 | 40 | 0 | 0 | 0.995000 | 0.915000 | 0.080000 | 0.071667 |
812
+ | table_exact_match | 40 | 37 | 0 | 3 | 0.925000 | 0.000000 | 0.925000 | 0.825000 |
813
+ | table_structure_similarity | 40 | 40 | 0 | 0 | 1.000000 | 0.875000 | 0.125000 | 0.125000 |
814
+
815
+ Cell exactness:
816
+
817
+ | system | reference_cells | exact_cells | exact_rate | mean_cell_cer |
818
+ | --- | ---: | ---: | ---: | ---: |
819
+ | candidate | 600 | 597 | 0.995000 | 0.001000 |
820
+ | Surya | 600 | 549 | 0.915000 | 0.071000 |
821
+ | Tesseract | 600 | 40 | 0.066667 | 0.933333 |
822
+
823
+ Paper-safe claim text:
824
+
825
+ > On a targeted generated complex-table pack with one repeated merged-header pattern, a conservative span-inference postprocessor improved table structure extraction over Surya, raising span-aware structure similarity from 0.875 to 1.000 and table exact match from 0.000 to 0.925.
826
+
827
+ Do not generalize this to all complex tables. This is generated data with a single span pattern and a heuristic parser repair. The next paper-grade check needs real scanned tables with varied merged cells, rowspans, nested headers, and a true HTML/TEDS scorer.
828
+
829
+ ## Targeted Complex-Table TEDS-Style Metrics V1
830
+
831
+ We added deterministic ordered table-tree edit metrics and reran the same complex-table pack with these gates included.
832
+
833
+ Artifacts:
834
+
835
+ - `outputs/experiment-complex-table-teds-v1/benchmark/report.json`
836
+ - `outputs/experiment-complex-table-teds-v1/validation/validation.md`
837
+ - `outputs/summary-complex-table-teds-v1/summary.md`
838
+ - `outputs/table-cell-analysis-complex-table-teds-v1/table-cell-analysis.md`
839
+
840
+ Validation status: `validated`.
841
+
842
+ | metric | pairs | wins | losses | ties | candidate | Surya | improvement | ci_low |
843
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
844
+ | table_cell_f1 | 40 | 40 | 0 | 0 | 0.995000 | 0.915000 | 0.080000 | 0.071667 |
845
+ | table_exact_match | 40 | 37 | 0 | 3 | 0.925000 | 0.000000 | 0.925000 | 0.825000 |
846
+ | table_structure_similarity | 40 | 40 | 0 | 0 | 1.000000 | 0.875000 | 0.125000 | 0.125000 |
847
+ | table_teds_structure | 40 | 40 | 0 | 0 | 1.000000 | 0.904762 | 0.095238 | 0.095238 |
848
+ | table_teds_text | 40 | 40 | 0 | 0 | 0.996250 | 0.891667 | 0.104583 | 0.098690 |
849
+
850
+ Paper-safe claim text:
851
+
852
+ > On a targeted generated complex-table pack, the candidate improved ordered table-tree metrics over Surya, reaching 1.000 structure-only TEDS-style similarity and 0.996 text-sensitive TEDS-style similarity, compared with Surya's 0.905 and 0.892 respectively.
853
+
854
+ This is still not a public-benchmark SOTA claim. The metrics are deterministic and closer to table-structure scoring, but the pack is generated and covers one repeated-header merged-cell pattern.
docs/PREFLIGHT.md ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Claim Preflight
2
+
3
+ `preflight-claim` is the gate before any expensive benchmark or training-adjacent claim run. Use it to fail on missing references, bad eval packs, missing candidate model cards, missing baseline dependencies, invalid validation configs, train/eval leakage, and tuned-on-eval routed candidates before you burn remote machine time.
4
+
5
+ ## Claim-Grade Run
6
+
7
+ ```bash
8
+ ocrtech preflight-claim \
9
+ --eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
10
+ --references-dir data/eval-pack/references \
11
+ --baselines candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
12
+ --candidate-model-config runs/recognizer/package/model-card.json \
13
+ --fallback-model-config runs/vlm-fallback/package/model-card.json \
14
+ --require-trained-recognizer \
15
+ --require-model-admission \
16
+ --validation-config validation.json \
17
+ --train-manifest data/splits/train.jsonl \
18
+ --out outputs/preflight
19
+ ```
20
+
21
+ This exits with `0` only when every required check passes.
22
+
23
+ ## Smoke Pack Exception
24
+
25
+ ```bash
26
+ ocrtech preflight-claim \
27
+ --eval-manifest data/eval-pack-smoke/manifests/ocrtech-eval-pack.jsonl \
28
+ --references-dir data/eval-pack-smoke/references \
29
+ --baselines ours \
30
+ --candidate-model-config runs/fixtures/sidecar-model.json \
31
+ --allow-smoke-eval-pack \
32
+ --out outputs/preflight-smoke
33
+ ```
34
+
35
+ `--allow-smoke-eval-pack` only downgrades the non-image eval-pack check. It does not hide missing references, invalid model cards, missing baseline hooks, or leakage.
36
+
37
+ ## Outputs
38
+
39
+ ```text
40
+ outputs/preflight/
41
+ preflight.json
42
+ preflight.md
43
+ eval-pack-audit/eval-pack-audit.json
44
+ eval-pack-audit/eval-pack-audit.md
45
+ ```
46
+
47
+ ## Checks
48
+
49
+ `preflight-claim` currently verifies:
50
+
51
+ - eval manifest loads and every input file exists
52
+ - image/PDF claim inputs use OCR-supported suffixes; text sidecar smoke inputs
53
+ are allowed only when `metadata.input_format = text`
54
+ - references resolve and parse for every sample
55
+ - ocrtech eval packs are claim-ready unless `--allow-smoke-eval-pack` is set
56
+ - `ours` or `candidate` baselines have a passing candidate model card
57
+ - configured fallback model cards pass `audit-model` before the benchmark starts
58
+ - when a fallback model is configured, the candidate model card must also
59
+ declare the same fallback card in `provenance.fallback_model_card`; preflight
60
+ hashes both cards and fails if they differ
61
+ - claim runs can require trained-recognizer provenance with
62
+ `--require-trained-recognizer`; this checks for recorded trained artifacts and
63
+ hash-verified train/eval manifest and training-summary provenance, or a
64
+ nested `provenance.trained_recognizer_model_card` in composite candidates
65
+ - claim runs can require model admission validation with
66
+ `--require-model-admission`; this checks the candidate model card's
67
+ `metrics.admission_validation` report and requires `claim_status=validated`.
68
+ It also requires quantitative passing gate evidence in that validation JSON.
69
+ When the report is rejected, preflight now surfaces the failed requirements
70
+ and gate deltas from the validation JSON, including baseline, metric, slice,
71
+ paired means, and win/loss/tie counts. The admission report must also include
72
+ `provenance.candidate_model_config`; preflight compares the validated model
73
+ identity and artifact hashes against the current candidate model card so a
74
+ passing report from a different recognizer cannot be reused.
75
+ - routed candidates such as `quality_select_composite`, `quality_ranked_ensemble`, and `script_select_composite` have calibration provenance, and that calibration eval manifest is not the same as the current claim eval manifest
76
+ - routed candidates also fail if their selected calibration trial collapses to one engine for every calibration sample, because that is not a meaningful routed candidate for claim purposes
77
+ - routed candidates also fail unless the claim-grade benchmark includes their raw component baselines, so distinctness can be measured in the reviewed evidence
78
+ - `tesseract` is on `PATH` when requested
79
+ - `paddleocr` is importable when stock PaddleOCR is requested
80
+ - `OCRTECH_GLM_OCR_CMD` is set, shell-parseable, contains both `{input}` and `{out}`, and points at a runnable executable when `glm-ocr` is requested
81
+ - `OCRTECH_PADDLEOCR_VL_CMD` is set, shell-parseable, contains both `{input}` and `{out}`, and points at a runnable executable when `paddleocr-vl` is requested
82
+ - validation config loads, contains gates, names baselines that are scheduled
83
+ for the run, and can be satisfied by the eval manifest's sample and slice
84
+ counts
85
+ - train/eval leakage passes when train manifests are supplied
86
+
87
+ Use this before `run-experiment`. It is cheaper to fail here than after a half-day benchmark on a broken setup.
88
+
89
+ For a single guarded command, `run-experiment` can run the same preflight before
90
+ benchmarking:
91
+
92
+ ```bash
93
+ ocrtech run-experiment \
94
+ --eval-manifest data/eval-pack/manifests/ocrtech-eval-pack.jsonl \
95
+ --references-dir data/eval-pack/references \
96
+ --baselines candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
97
+ --candidate-model-config runs/recognizer/package/model-card.json \
98
+ --validation-config validation.json \
99
+ --train-manifest data/splits/train.jsonl \
100
+ --preflight \
101
+ --require-trained-recognizer \
102
+ --require-model-admission \
103
+ --out outputs/claim-experiment \
104
+ --require-validated
105
+ ```
106
+
107
+ When embedded preflight fails, `run-experiment` writes
108
+ `outputs/claim-experiment/preflight/preflight.json` and stops before creating a
109
+ benchmark report.
110
+
111
+ When `--validation-config` is omitted, preflight emits a warning instead of
112
+ claiming the default gates are appropriate for the run. For claim-grade work,
113
+ always pass the slot-specific validation config. When a config is supplied,
114
+ preflight rejects impossible setups before benchmark time: `min_samples` larger
115
+ than the manifest, required slices absent from the manifest, gate `min_pairs`
116
+ larger than the matching slice intersection, or gate baselines that are not in
117
+ `--baselines`.
118
+
119
+ If the `eval-input-suffixes` check fails on files such as `.img`, normalize the
120
+ manifest before rerunning preflight:
121
+
122
+ ```bash
123
+ ocrtech normalize-manifest-images \
124
+ data/splits-mixed-v1/eval-with-refs.jsonl \
125
+ --out data/splits-mixed-v1/eval-normalized-images
126
+
127
+ ocrtech materialize-references \
128
+ data/splits-mixed-v1/eval-normalized-images/manifest-images-normalized.jsonl \
129
+ --out data/references-mixed-v1-normalized \
130
+ --rewritten-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl
131
+ ```
132
+
133
+ For routed thresholded candidates, the intended behavior is now:
134
+
135
+ - calibrate on one experiment manifest with `ocrtech calibrate-quality-router`
136
+ - or `ocrtech calibrate-script-router`
137
+ - benchmark with `candidate` plus the candidate's raw component baselines in the same run
138
+ - benchmark on a different eval manifest
139
+ - expect `candidate-calibration=pass` only on that different eval manifest
140
+
141
+ On `k2`, the same-pack check was verified against a real routed Tesseract calibration artifact: preflight failed on the calibration manifest itself and passed the calibration-separation check on the untouched holdout manifest.
142
+
143
+ On the larger rendered `claimpack-rendered-v2` holdout, a second real case now exists: the calibrated script router selected `primary_script_threshold=0.0`, which meant `secondary_picks=50/50` on calibration. After syncing the new guardrail to `k2`, `preflight-claim` correctly failed that model with:
144
+
145
+ - `script_select_composite collapses to the secondary engine on every calibration sample`
146
+
147
+ That is the intended behavior. A routed candidate that reduces to "always use Surya" is not evidence of a novel better engine.
docs/REMOTE_TRAINING.md ADDED
@@ -0,0 +1,1404 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Remote Training
2
+
3
+ Target host:
4
+
5
+ ```text
6
+ k2@100.97.238.103
7
+ ```
8
+
9
+ Do not store the SSH password in this repository. Use a local secret, `ssh-agent`, `sshpass -e`, or a one-off `expect` wrapper outside version control.
10
+
11
+ ## Current Host Check
12
+
13
+ As of the last verified setup from this workspace:
14
+
15
+ - SSH login works.
16
+ - `~/ocr-tech` exists and is synced from the local workspace.
17
+ - `~/ocr-tech/.venv/bin/python` is `3.11.14`.
18
+ - `uv` is on `PATH` at `~/.local/bin/uv`.
19
+ - Home directory free space is about `352 GiB`.
20
+ - `nvidia-smi` is still not available, so do not claim NVIDIA-backed training there.
21
+ - The machine is `macOS 14.6 arm64` on Apple Silicon.
22
+
23
+ The host is valid for Python 3.11 CPU-side runs and repo automation. It is not an officially supported Paddle training target: the PaddlePaddle installation guide says macOS support requires `x86_64`, not `arm64`. Use `--require-paddle-training` in the audit if you need a host that can honestly be called a PaddleOCR training box.
24
+
25
+ That restriction is narrower than “no Paddle at all.” As of June 14, 2026, `paddlepaddle 3.3.1`, `paddleocr 3.7.0`, and `paddlex 3.7.1` do install and run for inference on this Apple Silicon host. The repo can benchmark a stock-Paddle baseline on `k2`. What is still not honest is calling `k2` a supported PaddleOCR training machine.
26
+
27
+ ## Audit Command
28
+
29
+ Use the built-in audit before syncing data or starting training:
30
+
31
+ ```bash
32
+ export OCRTECH_REMOTE_PASSWORD='...'
33
+
34
+ ocrtech audit-remote-host \
35
+ --host 100.97.238.103 \
36
+ --user k2 \
37
+ --password-env OCRTECH_REMOTE_PASSWORD \
38
+ --workdir '~/ocr-tech' \
39
+ --require-path '~/ocr-tech' \
40
+ --require-paddle-training \
41
+ --min-python 3.11 \
42
+ --min-free-gb 20 \
43
+ --out outputs/remote-audit
44
+ ```
45
+
46
+ This writes:
47
+
48
+ ```text
49
+ outputs/remote-audit/
50
+ remote-audit.json
51
+ remote-audit.md
52
+ ```
53
+
54
+ If the audit fails on Python, disk, platform support, or required commands, fix the host first. Do not push large datasets to a host that already fails the audit.
55
+
56
+ The audit now checks multiple interpreter candidates such as `python3.11`, `python3.12`, `python3.13`, and `~/.local/bin/python3.11`. Do not assume `/usr/bin/python3` reflects the usable training interpreter on macOS hosts.
57
+
58
+ ## Bootstrap Command
59
+
60
+ Once the audit passes on Python and disk, provision the remote workdir and virtualenv:
61
+
62
+ ```bash
63
+ ocrtech bootstrap-remote-host \
64
+ --host 100.97.238.103 \
65
+ --user k2 \
66
+ --password-env OCRTECH_REMOTE_PASSWORD \
67
+ --workdir '~/ocr-tech' \
68
+ --out outputs/remote-bootstrap
69
+ ```
70
+
71
+ This creates:
72
+
73
+ - `~/ocr-tech`
74
+ - `~/ocr-tech/.venv`
75
+
76
+ It does not sync the repository contents. It only prepares the target path and interpreter environment so the next sync and install step has a valid home.
77
+
78
+ If the host already has a `.venv` built with the wrong interpreter, recreate it explicitly:
79
+
80
+ ```bash
81
+ ocrtech bootstrap-remote-host \
82
+ --host 100.97.238.103 \
83
+ --user k2 \
84
+ --password-env OCRTECH_REMOTE_PASSWORD \
85
+ --workdir '~/ocr-tech' \
86
+ --python-candidate '~/.local/bin/python3.11' \
87
+ --recreate-venv \
88
+ --out outputs/remote-bootstrap
89
+ ```
90
+
91
+ ## Sync Workspace
92
+
93
+ Copy the current workspace after bootstrap:
94
+
95
+ ```bash
96
+ ocrtech sync-remote-workspace \
97
+ --host 100.97.238.103 \
98
+ --user k2 \
99
+ --password-env OCRTECH_REMOTE_PASSWORD \
100
+ --local-root . \
101
+ --remote-workdir '~/ocr-tech' \
102
+ --exclude .venv \
103
+ --exclude outputs \
104
+ --exclude runs \
105
+ --out outputs/remote-sync
106
+ ```
107
+
108
+ This uses a temporary tar archive and remote extract. It is intentionally simple and does not delete remote files that no longer exist locally.
109
+
110
+ Keep `--remote-workdir` quoted when it contains `~`. If you pass `~/ocr-tech` unquoted, your local shell expands it before `ocrtech` sees the argument, which is the wrong machine.
111
+
112
+ ## Expected Setup
113
+
114
+ On `k2`, provision a work directory with enough free disk:
115
+
116
+ ```bash
117
+ mkdir -p ~/ocr-tech
118
+ cd ~/ocr-tech
119
+ python3 -m venv .venv
120
+ . .venv/bin/activate
121
+ python -m pip install --upgrade pip
122
+ python -m pip install -e '.[datasets,engines]'
123
+ ```
124
+
125
+ If Python 3.11+ is not available, install a newer Python first or use a managed environment. The package targets Python 3.11+.
126
+
127
+ ## Sync
128
+
129
+ From the local workspace:
130
+
131
+ ```bash
132
+ rsync -az --delete \
133
+ --exclude .venv \
134
+ --exclude .pytest_cache \
135
+ --exclude outputs \
136
+ --exclude runs \
137
+ ./ k2@100.97.238.103:~/ocr-tech/
138
+ ```
139
+
140
+ Never sync a local `.venv` to a Linux training host. A macOS virtualenv can
141
+ overwrite the remote console scripts with bad interpreter paths and force a
142
+ full remote venv repair.
143
+
144
+ If password auth is required and `sshpass` is installed locally, prefer `SSHPASS` over putting the password in command history:
145
+
146
+ ```bash
147
+ SSHPASS='...' sshpass -e rsync -az ./ k2@100.97.238.103:~/ocr-tech/
148
+ ```
149
+
150
+ ## Run Training Recipes
151
+
152
+ Generate manifests and configs locally or remotely:
153
+
154
+ ```bash
155
+ ocrtech inspect-hf-dataset gauravgiri/nepali-ocr-dataset --split train --limit 3 --out outputs/hf-inspect-training-v1
156
+ ocrtech inspect-hf-dataset prashant0919/nepali-synthetic-ocr-lines --split train --limit 3 --out outputs/hf-inspect-training-v1
157
+ ocrtech inspect-hf-dataset nvidia/OCR-Synthetic-Multilingual-v1 --hf-subset en --split test --limit 3 --out outputs/hf-inspect-training-v1
158
+
159
+ ocrtech prepare-data gauravgiri/nepali-ocr-dataset --hf --split train --limit 100 --skip-invalid --out data/manifests
160
+ ocrtech prepare-data prashant0919/nepali-synthetic-ocr-lines --hf --split train --limit 100 --out data/manifests
161
+ ocrtech merge-manifests \
162
+ data/manifests/gauravgiri__nepali-ocr-dataset-train.jsonl \
163
+ data/manifests/prashant0919__nepali-synthetic-ocr-lines-train.jsonl \
164
+ --out data/manifests/nepali-mixed-all.jsonl
165
+ ocrtech audit-manifest data/manifests/nepali-mixed-all.jsonl --out outputs/manifest-audit
166
+ ocrtech split-manifest data/manifests/nepali-mixed-all.jsonl --out data/splits
167
+ ocrtech train-recognizer \
168
+ --train-manifest data/splits/train.jsonl \
169
+ --eval-manifest data/splits/eval.jsonl \
170
+ --out runs/recognizer
171
+ ```
172
+
173
+ For the current mixed Nepali/English line-recognition branch, the verified v2
174
+ recipe command on `k2` is:
175
+
176
+ ```bash
177
+ ocrtech train-recognizer \
178
+ --backend paddleocr \
179
+ --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
180
+ --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
181
+ --out runs/paddle-recognizer-mixed-v2 \
182
+ --train-batch-size 16 \
183
+ --eval-batch-size 16 \
184
+ --learning-rate 0.0005 \
185
+ --epochs 10
186
+ ```
187
+
188
+ After generating a PaddleOCR recipe, verify the invariant that each manifest
189
+ entry maps to exactly one physical label-file line:
190
+
191
+ ```bash
192
+ uv run python - <<'PY'
193
+ from pathlib import Path
194
+ from ocrtech.manifest import load_manifest
195
+
196
+ train_count = len(load_manifest("data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl"))
197
+ eval_count = len(load_manifest("data/splits-mixed-v1/eval-with-refs.jsonl"))
198
+ train_lines = sum(1 for _ in Path("runs/paddle-recognizer-mixed-v2/labels/train.txt").open())
199
+ eval_lines = sum(1 for _ in Path("runs/paddle-recognizer-mixed-v2/labels/eval.txt").open())
200
+ assert train_count == train_lines, (train_count, train_lines)
201
+ assert eval_count == eval_lines, (eval_count, eval_lines)
202
+ PY
203
+ ```
204
+
205
+ The verified v2 counts are `1700/1700` train and `180/180` eval. Earlier recipe
206
+ generation expanded embedded label newlines and produced too many label lines;
207
+ do not train from any recipe that fails this invariant.
208
+
209
+ Recipe generation also verifies dictionary coverage for the exact normalized
210
+ PaddleOCR labels. If you pass a custom `--dictionary`, generation fails when
211
+ any train/eval label contains a character missing from that dictionary. Do not
212
+ bypass that failure on the remote host; rebuild the dictionary or reject the
213
+ corpus.
214
+
215
+ To move the recipe to a supported PaddleOCR training host, create a portable
216
+ bundle that materializes all referenced images under `bundle_assets/images` and
217
+ rewrites manifests/label files to those portable paths:
218
+
219
+ ```bash
220
+ ocrtech bundle-recognizer-training \
221
+ --recipe-dir runs/paddle-recognizer-mixed-v2 \
222
+ --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
223
+ --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
224
+ --out runs/paddle-recognizer-mixed-v2-bundle \
225
+ --archive-name paddle-recognizer-mixed-v2-training.tar.gz
226
+ ```
227
+
228
+ Verified k2 bundle:
229
+
230
+ - archive:
231
+ `runs/paddle-recognizer-mixed-v2-bundle/paddle-recognizer-mixed-v2-training.tar.gz`
232
+ - bundle manifest:
233
+ `runs/paddle-recognizer-mixed-v2-bundle/recognizer-training-bundle.json`
234
+ - files in archive: `1887`
235
+ - materialized images: `1880`
236
+ - uncompressed bytes: `79133479`
237
+ - compressed archive size: `74061239`
238
+
239
+ After extraction on a supported training host, run PaddleOCR training from the
240
+ extraction root:
241
+
242
+ ```bash
243
+ python PaddleOCR/tools/train.py -c runs/paddle-recognizer-mixed-v2/paddle_recognizer.yml
244
+ ```
245
+
246
+ The verified `k2` starter run on June 14, 2026 produced:
247
+
248
+ - `97` valid `gauravgiri` samples and `3` explicit rejects
249
+ - `100` valid `prashant0919` samples
250
+ - a merged `197`-sample corpus
251
+ - a leakage-safe `167` train / `30` eval split
252
+ - `runs/recognizer-nepali-starter/paddle_recognizer.yml`
253
+
254
+ The verified `k2` schema inspection on June 15, 2026 produced:
255
+
256
+ - `outputs/hf-inspect-training-v1/gauravgiri__nepali-ocr-dataset-train-inspect.json`
257
+ - `outputs/hf-inspect-training-v1/prashant0919__nepali-synthetic-ocr-lines-train-inspect.json`
258
+ - `outputs/hf-inspect-training-v1/nvidia__OCR-Synthetic-Multilingual-v1__en-test-inspect.json`
259
+
260
+ All three passed. Keep this inspection step before conversion so dataset schema
261
+ drift is caught as a recorded artifact, not discovered halfway through training.
262
+
263
+ The verified mixed-script HF workflow on the same host later produced:
264
+
265
+ - `97` valid `gauravgiri` Nepali real rows and `3` explicit rejects
266
+ - `100` `prashant0919` Nepali synthetic rows
267
+ - `1000` NVIDIA English synthetic line crops from `--hf-subset en --split test`
268
+ - a merged `1197`-sample mixed corpus
269
+ - a leakage-safe `1017` train / `180` eval split
270
+ - materialized eval references at `data/references-mixed-v1`
271
+ - a rebalanced training manifest with `english=850` and `nepali=850`
272
+ - `runs/hf-recognizer-mixed-v1-rebalanced/final-model`
273
+ - `outputs/experiment-hf-mixed-v1-rebalanced/benchmark/report.json`
274
+
275
+ For `k2` specifically, prefer the Hugging Face recognizer backend over Paddle training when you need a real trainable path on Apple Silicon:
276
+
277
+ ```bash
278
+ python -m pip install -e '.[hf-recognizer]'
279
+
280
+ ocrtech train-recognizer \
281
+ --backend hf-vision-encoder-decoder \
282
+ --base-model microsoft/trocr-small-printed \
283
+ --train-manifest data/splits/train.jsonl \
284
+ --eval-manifest data/splits/eval.jsonl \
285
+ --out runs/hf-recognizer
286
+ ```
287
+
288
+ This does not make `k2` a valid official Paddle training host. It gives the repo a separate arm64-compatible recognizer path.
289
+
290
+ Current active HF-compatible run on k2:
291
+
292
+ ```text
293
+ runs/hf-recognizer-mixed-v2-full
294
+ ```
295
+
296
+ Command:
297
+
298
+ ```bash
299
+ uv run ocrtech train-recognizer \
300
+ --backend hf-vision-encoder-decoder \
301
+ --base-model microsoft/trocr-small-printed \
302
+ --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
303
+ --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
304
+ --out runs/hf-recognizer-mixed-v2-full \
305
+ --train-batch-size 4 \
306
+ --eval-batch-size 4 \
307
+ --learning-rate 0.00005 \
308
+ --epochs 3 \
309
+ --run
310
+ ```
311
+
312
+ Monitor:
313
+
314
+ ```bash
315
+ PID=$(cat runs/hf-recognizer-mixed-v2-full/train.pid)
316
+ ps -p "$PID" -o pid=,stat=,etime=,command=
317
+ tail -f runs/hf-recognizer-mixed-v2-full/train.log
318
+ ```
319
+
320
+ Status artifact:
321
+
322
+ ```text
323
+ runs/hf-recognizer-mixed-v2-full/train-status.json
324
+ ```
325
+
326
+ Do not treat this run as a model result until
327
+ `runs/hf-recognizer-mixed-v2-full/training_metrics.json` and
328
+ `runs/hf-recognizer-mixed-v2-full/final-model/` exist, followed by packaging and
329
+ benchmarking.
330
+
331
+ Latest verified checkpoint:
332
+
333
+ - k2 time: `2026-06-15 03:35:55 +0545`
334
+ - PID: `24642`
335
+ - process: running
336
+ - latest observed log progress: `474/1275` trainer steps
337
+ - `training_metrics.json`: not present
338
+ - `final-model/`: not present
339
+
340
+ The run then completed and produced both `final-model/` and
341
+ `training_metrics.json`.
342
+
343
+ Training metrics:
344
+
345
+ - `eval_loss`: `2.672821283340454`
346
+ - `eval_cer`: `0.4532251781992946`
347
+ - `eval_wer`: `0.6495539645539645`
348
+ - `eval_exact_line_accuracy`: `0.12222222222222222`
349
+ - `epoch`: `3.0`
350
+
351
+ The package/audit step passed:
352
+
353
+ - model card:
354
+ `runs/hf-recognizer-mixed-v2-full/finalized/package/model-card.json`
355
+ - audit:
356
+ `runs/hf-recognizer-mixed-v2-full/finalized/audit/model-audit.json`
357
+ - audit status: pass
358
+
359
+ When `final-model/` and `training_metrics.json` exist, finalize the run before
360
+ benchmarking:
361
+
362
+ ```bash
363
+ uv run ocrtech finalize-hf-recognizer-run \
364
+ --run-dir runs/hf-recognizer-mixed-v2-full \
365
+ --out runs/hf-recognizer-mixed-v2-full/finalized \
366
+ --model-id ocrtech-hf-mixed-v2-full \
367
+ --base-model microsoft/trocr-small-printed \
368
+ --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
369
+ --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
370
+ --training-summary runs/hf-recognizer-mixed-v2-full/recognizer-training-summary.json \
371
+ --admission-validation-report outputs/validation-next-recognizer-hard50/validation.json
372
+ ```
373
+
374
+ This command intentionally fails if `final-model/` or `training_metrics.json`
375
+ is missing. If it passes, it writes a packaged model card, model audit, and a
376
+ post-training report with the next benchmark command.
377
+
378
+ The first held-out measurement run completed on k2:
379
+
380
+ - output:
381
+ `outputs/experiment-hf-recognizer-mixed-v2-full-vs-tesseract`
382
+ - baselines: `candidate,tesseract`
383
+ - rows: `360`
384
+ - status: `360` ok, `0` errors
385
+ - candidate CER: `0.4452926496941952`
386
+ - Tesseract CER: `0.3290117066116023`
387
+ - candidate WER: `0.6551095201095201`
388
+ - Tesseract WER: `0.48648528940195623`
389
+ - candidate mean latency: `0.4809882701633291`s
390
+ - Tesseract mean latency: `0.14839679651065832`s
391
+ - validation: skipped because this was a direct branch check, not a claim run
392
+
393
+ Conclusion: the HF TrOCR branch is a documented negative result. Use k2 for
394
+ orchestration and evaluation, but spend serious training time next on the
395
+ PaddleOCR CUDA path.
396
+
397
+ ## CUDA PaddleOCR Handoff
398
+
399
+ The next serious recognizer training step needs a CUDA Linux host. k2 is macOS
400
+ arm64, so it should not be described as the claim-grade PaddleOCR training
401
+ machine.
402
+
403
+ Use the current admitted portable bundle as the handoff artifact:
404
+
405
+ ```text
406
+ runs/paddle-recognizer-next-v1-bundle/paddle-recognizer-next-v1-training.tar.gz
407
+ ```
408
+
409
+ After extracting it on the CUDA host and installing/checking out PaddleOCR, run:
410
+
411
+ ```bash
412
+ python PaddleOCR/tools/train.py -c runs/paddle-recognizer-next-v1/paddle_recognizer.yml
413
+ ```
414
+
415
+ Bring the exported inference model back into this repo, then package, audit,
416
+ and benchmark it with the same validation flow.
417
+
418
+ The `recognizer-next-v1` bundle was created on `k2` after the updated corpus
419
+ audit passed:
420
+
421
+ - train manifest: `data/splits-recognizer-next-v1/train.jsonl`
422
+ - eval manifest: `data/splits-recognizer-next-v1/eval.jsonl`
423
+ - train/eval samples: `2212` / `244`
424
+ - train Latin/Devanagari/mixed: `1084` / `1174` / `100`
425
+ - eval Latin/Devanagari/mixed: `172` / `90` / `28`
426
+ - train real/synthetic: `428` / `1272`
427
+ - leakage check: pass
428
+ - local mirrored audit:
429
+ `outputs/k2-recognizer-next-v1/recognizer-corpus-audit.json`
430
+ - local mirrored bundle manifest:
431
+ `outputs/k2-recognizer-next-v1/recognizer-training-bundle.json`
432
+
433
+ This is only a training handoff. It is not a quality result. A CUDA-trained
434
+ recognizer still has to clear export, packaging, benchmark, and admission gates
435
+ before it can replace the current document engine candidate.
436
+
437
+ Final outcome: `recognizer-next-v1` was trained on the restarted AWS G6 L4 host,
438
+ exported, packaged, benchmarked on hard-50, and rejected. The host was stopped
439
+ after artifact mirroring.
440
+
441
+ - instance: `i-0fa19dff7b233a353`
442
+ - final state after run: `stopped`
443
+ - model card: `runs/paddle-recognizer-next-v1/package/model-card.json`
444
+ - local slim artifact:
445
+ `outputs/aws-g6-paddle-recognizer-next-v1/ocrtech-paddle-recognizer-next-v1-slim-artifacts.tgz`
446
+ - slim artifact sha256:
447
+ `3558cb36fc63e463650d229cd21e1b91955c208cfcd483e419e89d3e86b309ce`
448
+ - hard-50 candidate CER/WER: `0.964227` / `1.000000`
449
+ - hard-50 Tesseract CER/WER: `0.080249` / `0.258921`
450
+ - paired CER wins/losses/ties: `0` / `50` / `0`
451
+ - admission decision: `reject`
452
+
453
+ Do not relaunch this branch as an engine candidate. The admitted corpus improved
454
+ the training setup but not recognizer quality.
455
+
456
+ Follow-up guardrail: `ocrtech recognizer-failure-review` now aggregates
457
+ recognizer admission failures across runs and exits nonzero when a model family
458
+ should be stopped. The current PaddleOCR Devanagari PP-OCRv3 mobile review is:
459
+
460
+ - artifact:
461
+ `outputs/recognizer-failure-review-paddle-ppocrv3/recognizer-failure-review.json`
462
+ - markdown:
463
+ `outputs/recognizer-failure-review-paddle-ppocrv3/recognizer-failure-review.md`
464
+ - included runs: PaddleOCR mixed v2, v3, v4, and `recognizer-next-v1`
465
+ - family recommendation: `stop_family`
466
+ - rejected runs: `4`
467
+ - paired failed-gate wins/losses/ties: `0` / `545` / `7`
468
+
469
+ Operational rule: do not start another AWS/G6 job for this Devanagari PP-OCRv3
470
+ mobile family unless the recognizer design changes materially. Use GPU time for
471
+ a different base recognizer, a different objective, or a document-native model
472
+ that still has to pass the same admission gate.
473
+
474
+ Enforcement rule for future handoffs: include the failure review when writing
475
+ and bundling recognizer training jobs:
476
+
477
+ ```bash
478
+ ocrtech train-recognizer \
479
+ --train-manifest data/splits/train.jsonl \
480
+ --eval-manifest data/splits/eval.jsonl \
481
+ --out runs/recognizer \
482
+ --failure-review outputs/recognizer-failure-review-paddle-ppocrv3/recognizer-failure-review.json
483
+
484
+ ocrtech bundle-recognizer-training \
485
+ --recipe-dir runs/recognizer \
486
+ --train-manifest data/splits/train.jsonl \
487
+ --eval-manifest data/splits/eval.jsonl \
488
+ --out runs/recognizer-training-bundle \
489
+ --require-corpus-audit \
490
+ --failure-review outputs/recognizer-failure-review-paddle-ppocrv3/recognizer-failure-review.json
491
+ ```
492
+
493
+ The bundle command reads `recognizer-training-summary.json` from the actual
494
+ recipe directory and refuses stopped backend/base-model families before archive
495
+ creation. That is the guard to run before copying anything to AWS or `k2`.
496
+
497
+ The first live TrOCR run on `k2` flushed out the actual dependency floor for this backend. Do not hand-install one package at a time. Use the repo extra:
498
+
499
+ ```bash
500
+ python -m pip install -e '.[hf-recognizer]'
501
+ ```
502
+
503
+ That extra now covers `torch`, `transformers`, `accelerate`, `sentencepiece`, `protobuf`, and `pillow`. On a cold cache, `microsoft/trocr-small-printed` also downloads about `246 MB` of model weights before training starts.
504
+
505
+ For the verified mixed-script run on `k2`, use this exact sequence:
506
+
507
+ ```bash
508
+ ocrtech prepare-data gauravgiri/nepali-ocr-dataset --hf --split train --limit 100 --skip-invalid --out data/manifests
509
+ ocrtech prepare-data prashant0919/nepali-synthetic-ocr-lines --hf --split train --limit 100 --out data/manifests
510
+ ocrtech prepare-data nvidia/OCR-Synthetic-Multilingual-v1 --hf --hf-subset en --split test --limit 1000 --out data/manifests-english-test
511
+ ocrtech merge-manifests \
512
+ data/manifests/gauravgiri__nepali-ocr-dataset-train.jsonl \
513
+ data/manifests/prashant0919__nepali-synthetic-ocr-lines-train.jsonl \
514
+ data/manifests-english-test/nvidia__OCR-Synthetic-Multilingual-v1__en-test.jsonl \
515
+ --out data/manifests/nep-eng-mixed-v1.jsonl
516
+ ocrtech audit-manifest data/manifests/nep-eng-mixed-v1.jsonl --out outputs/manifest-audit-mixed-v1 --require-slice english --require-slice nepali
517
+ ocrtech split-manifest data/manifests/nep-eng-mixed-v1.jsonl --out data/splits-mixed-v1 --eval-ratio 0.15 --seed 13
518
+ ocrtech materialize-references data/splits-mixed-v1/eval.jsonl --out data/references-mixed-v1 --rewritten-manifest data/splits-mixed-v1/eval-with-refs.jsonl
519
+ ocrtech rebalance-manifest data/splits-mixed-v1/train.jsonl --out data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl --slice english --slice nepali --seed 13
520
+ ocrtech train-recognizer \
521
+ --backend hf-vision-encoder-decoder \
522
+ --base-model microsoft/trocr-small-printed \
523
+ --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
524
+ --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
525
+ --out runs/hf-recognizer-mixed-v1-rebalanced \
526
+ --train-batch-size 4 \
527
+ --eval-batch-size 4 \
528
+ --epochs 0.5 \
529
+ --run
530
+ ocrtech package-hf-recognizer \
531
+ --model-dir runs/hf-recognizer-mixed-v1-rebalanced/final-model \
532
+ --out runs/hf-recognizer-mixed-v1-rebalanced/package \
533
+ --model-id ocrtech-hf-mixed-v1-rebalanced \
534
+ --base-model microsoft/trocr-small-printed \
535
+ --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
536
+ --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
537
+ --training-summary runs/hf-recognizer-mixed-v1-rebalanced/recognizer-training-summary.json \
538
+ --metrics-report runs/hf-recognizer-mixed-v1-rebalanced/training_metrics.json \
539
+ --admission-validation-report outputs/validation-next-recognizer-hard50/validation.json
540
+ ocrtech run-experiment \
541
+ --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
542
+ --out outputs/experiment-hf-mixed-v1-rebalanced \
543
+ --baselines candidate \
544
+ --candidate-model-config runs/hf-recognizer-mixed-v1-rebalanced/package/model-card.json \
545
+ --references-dir data/references-mixed-v1 \
546
+ --skip-validation
547
+ ```
548
+
549
+ Trainer-side eval metrics for that rebalanced run:
550
+
551
+ - `eval_cer = 0.5983964991975725`
552
+ - `eval_wer = 0.825085223418557`
553
+ - `eval_exact_line_accuracy = 0.2611111111111111`
554
+
555
+ Candidate-only benchmark means on the same `180`-sample eval split:
556
+
557
+ - overall CER: `0.586139`
558
+ - English CER: `0.300454`
559
+ - Nepali CER: `2.014560`
560
+ - real-slice CER: `1.770016`
561
+
562
+ `k2` now also has a real `tesseract` baseline installed via Homebrew, with both `eng` and `nep` language data verified. The first direct comparison on that same eval split is still negative for the candidate:
563
+
564
+ - candidate overall CER: `0.586139`
565
+ - tesseract overall CER: `0.329012`
566
+ - candidate Nepali CER: `2.014560`
567
+ - tesseract Nepali CER: `0.757175`
568
+
569
+ One shell detail matters on `k2`: non-interactive SSH sessions do not reliably inherit the Homebrew path. For remote one-shot benchmark commands, export it explicitly before calling `ocrtech`:
570
+
571
+ ```bash
572
+ export PATH=/opt/homebrew/bin:$PATH
573
+ ```
574
+
575
+ For the external VLM baselines, the current live state on `k2` is now more concrete:
576
+
577
+ - `paddleocr-vl` is runnable through `scripts/run_paddleocr_vl.py` after installing `paddlex[ocr]==3.7.1`.
578
+ - `glm-ocr` does run through `scripts/run_glm_ocr.py`, but not in the main repo venv. It needed a dedicated Python 3.11 env with git `transformers` and `torchvision`.
579
+ - On the first rendered claim-pack blur sample, neither VLM beat raw Tesseract on text CER:
580
+ - `glm-ocr`: `0.436170`
581
+ - `paddleocr-vl`: `0.521277`
582
+ - `tesseract`: `0.372340`
583
+
584
+ That matters for `tesseract` baselines and for candidate model cards that instantiate Tesseract internally.
585
+
586
+ We also verified a small routed Tesseract candidate on `k2`. A `quality_select_composite` model card using default Tesseract (`psm=6`) as primary and `psm=13` as secondary, with `secondary_quality_margin=0.02`, slightly improved the held-out `180`-sample mixed eval:
587
+
588
+ - raw tesseract overall CER: `0.329012`
589
+ - routed tesseract overall CER: `0.324000`
590
+ - paired CER versus raw tesseract: `5` wins, `3` losses, `172` ties
591
+
592
+ That is an engineering improvement, not a frontier claim. The routing margin was tuned on the same pack, and the effect size is small.
593
+
594
+ We then reran it with proper separation on the same host:
595
+
596
+ 1. split `data/splits-mixed-v1/eval-with-refs.jsonl` into a calibration half (`89` samples) and untouched holdout half (`91` samples)
597
+ 2. ran `psm=13` vs raw default Tesseract on the calibration half
598
+ 3. generated a routed model card with `ocrtech calibrate-quality-router`
599
+ 4. ran that calibrated routed candidate on the untouched holdout half
600
+
601
+ Holdout result:
602
+
603
+ - raw tesseract overall CER: `0.342234`
604
+ - calibrated routed tesseract overall CER: `0.340608`
605
+ - paired CER on holdout: `2` wins, `1` loss, `88` ties
606
+
607
+ That is the first out-of-sample confirmation that the routed baseline is not just same-pack leakage. It is still a very small gain.
608
+
609
+ We also tested Surya on `k2` instead of assuming a VLM-style OCR stack would help by default. The host can run it on Apple Silicon after installing:
610
+
611
+ - `/opt/homebrew/bin/llama-server` from `llama.cpp`
612
+ - `surya-ocr` inside `~/ocr-tech/.venv`
613
+
614
+ Raw Surya was not a good general baseline on the `91`-sample holdout:
615
+
616
+ - raw Surya overall CER: `2.198362`
617
+ - raw Tesseract overall CER: `0.342234`
618
+
619
+ But Surya was better on the Nepali slice, so we calibrated a routed `script_select_composite` candidate on the separate `89`-sample half with:
620
+
621
+ ```bash
622
+ python -m ocrtech.cli calibrate-script-router \
623
+ --experiment-report outputs/experiment-surya-calibration-vs-tesseract/experiment.json \
624
+ --out outputs/calibration-surya-script-router \
625
+ --model-id ocrtech-tesseract-surya-script-router \
626
+ --primary-engine tesseract \
627
+ --secondary-engine surya \
628
+ --primary-engine-kwarg language=nep+eng \
629
+ --primary-engine-kwarg psm=6 \
630
+ --secondary-engine-kwarg mode=auto
631
+ ```
632
+
633
+ That selected `primary_script_threshold=0.4`. On the untouched `91`-sample holdout:
634
+
635
+ - raw tesseract overall CER: `0.342234`
636
+ - calibrated script router overall CER: `0.313084`
637
+ - paired CER on holdout: `9` wins, `0` losses, `82` ties
638
+
639
+ This is the strongest out-of-sample baseline improvement we currently have on `k2`.
640
+
641
+ So the host is now baseline-ready, but the model claim is not. The next useful work is not another README victory lap. It is either:
642
+
643
+ 1. improve the candidate until it actually beats Tesseract on held-out Nepali and mixed-script slices, or
644
+ 2. admit this TrOCR path is the wrong recognizer direction and switch to a stronger training backend.
645
+
646
+ We tested one more recovery path on the same host before stopping: mine Tesseract noisy/clean pairs from the rebalanced training manifest, train a ByT5 corrector, and package it as a `text_correction_composite` candidate with raw Tesseract as the base engine.
647
+
648
+ That experiment also failed badly. The corrector trainer finished with:
649
+
650
+ - `eval_cer = 7.067461343195775`
651
+ - `eval_wer = 3.334916864181571`
652
+ - `eval_exact_line_accuracy = 0.0`
653
+
654
+ And the held-out benchmark on the same `180`-sample mixed eval was nowhere close to viable:
655
+
656
+ - composite overall CER: `6.912065`
657
+ - raw Tesseract overall CER: `0.329012`
658
+ - paired CER: `0` wins / `170` losses / `10` ties
659
+
660
+ So `k2` remains useful for fast iteration, but the current Tesseract-plus-corrector branch is not a credible direction either.
661
+
662
+ Then run the generated PaddleOCR command from a checked-out PaddleOCR repo:
663
+
664
+ ```bash
665
+ python PaddleOCR/tools/train.py -c runs/recognizer/paddle_recognizer.yml
666
+ ```
667
+
668
+ After training, export and package the inference artifacts:
669
+
670
+ ```bash
671
+ ocrtech export-recognizer \
672
+ --training-config runs/recognizer/paddle_recognizer.yml \
673
+ --checkpoint runs/recognizer/checkpoints/best_accuracy \
674
+ --out runs/recognizer/export \
675
+ --run
676
+
677
+ ocrtech package-recognizer \
678
+ --inference-dir runs/recognizer/export/inference \
679
+ --dictionary runs/recognizer/nep_eng_dict.txt \
680
+ --out runs/recognizer/package \
681
+ --model-id ocrtech-nep-eng-rec-v1 \
682
+ --train-manifest data/splits/train.jsonl \
683
+ --eval-manifest data/splits/eval.jsonl \
684
+ --training-summary runs/recognizer/recognizer-training-summary.json \
685
+ --admission-validation-report outputs/validation-next-recognizer-hard50/validation.json
686
+
687
+ ocrtech audit-model runs/recognizer/package/model-card.json
688
+ ```
689
+
690
+ Keep evaluation manifests physically separate from training manifests. Do not use the same image or text hashes across train/eval.
691
+
692
+ ## AWS G6 PaddleOCR Run, 2026-06-15
693
+
694
+ We launched a CUDA host through the `playaround` AWS profile before the login
695
+ window expired:
696
+
697
+ - instance: `i-044a50b7737e8a067`
698
+ - type: `g6.xlarge`
699
+ - GPU: NVIDIA L4, `23034 MiB`
700
+ - region: `us-east-1`
701
+ - AMI: Deep Learning Base OSS Nvidia Driver GPU AMI, Ubuntu 22.04
702
+ - SSH key: `ocrtech-g6-training-20260615`
703
+
704
+ The host passed `paddle.utils.run_check()` with `paddlepaddle-gpu==3.3.1`.
705
+ Training used the local PaddleOCR checkout at commit
706
+ `c166448875bcecb8d3b7628fd697ac1c28f8705b`.
707
+
708
+ The original portable recipe from k2 was not valid for actual PaddleOCR
709
+ training because it was a minimal YAML stub. On AWS we built a corrected config
710
+ from the upstream Devanagari PP-OCRv3 mobile recognizer config and then patched
711
+ the local recipe writer to generate that shape directly.
712
+
713
+ Training command:
714
+
715
+ ```bash
716
+ cd /opt/ocrtech
717
+ source .venv/bin/activate
718
+ python PaddleOCR/tools/train.py \
719
+ -c runs/paddle-recognizer-mixed-v2-cuda/paddle_recognizer.yml
720
+ ```
721
+
722
+ Export command:
723
+
724
+ ```bash
725
+ python PaddleOCR/tools/export_model.py \
726
+ -c runs/paddle-recognizer-mixed-v2-cuda/paddle_recognizer.yml \
727
+ -o Global.pretrained_model=runs/paddle-recognizer-mixed-v2-cuda/checkpoints/best_accuracy \
728
+ Global.save_inference_dir=runs/paddle-recognizer-mixed-v2-cuda/export/inference
729
+ ```
730
+
731
+ Runtime dependencies added for benchmarking:
732
+
733
+ ```bash
734
+ pip install --no-deps -e /opt/ocrtech/PaddleOCR
735
+ pip install paddlex==3.7.1 "paddlex[ocr-core]==3.7.1" aiohttp
736
+ sudo apt-get install -y tesseract-ocr tesseract-ocr-eng tesseract-ocr-nep
737
+ ```
738
+
739
+ Packaging caveat for PaddleOCR 3.x:
740
+
741
+ - use `text_recognition_model_dir`, not `rec_model_dir`
742
+ - set `text_recognition_model_name=devanagari_PP-OCRv3_mobile_rec`
743
+ - do not pass `rec_char_dict_path`; exported `inference.yml` embeds the
744
+ dictionary
745
+
746
+ Benchmark command, after materializing references on the AWS host:
747
+
748
+ ```bash
749
+ cd /opt/ocrtech
750
+ source .venv/bin/activate
751
+ PYTHONPATH=/opt/ocrtech/ocr-tech/src python -m ocrtech.cli materialize-references \
752
+ data/splits-mixed-v1/eval-with-refs.jsonl \
753
+ --out data/references-mixed-v1 \
754
+ --rewritten-manifest data/splits-mixed-v1/eval-with-refs-aws.jsonl
755
+
756
+ PYTHONPATH=/opt/ocrtech/ocr-tech/src python -m ocrtech.cli run-experiment \
757
+ --eval-manifest /opt/ocrtech/data/splits-mixed-v1/eval-with-refs.jsonl \
758
+ --out /opt/ocrtech/outputs/experiment-paddle-mixed-v2-cuda-vs-tesseract-v2 \
759
+ --baselines candidate,tesseract \
760
+ --candidate-model-config /opt/ocrtech/runs/paddle-recognizer-mixed-v2-cuda/package/model-card.json \
761
+ --skip-validation
762
+
763
+ PYTHONPATH=/opt/ocrtech/ocr-tech/src python -m ocrtech.cli rescore-benchmark \
764
+ --benchmark-report outputs/experiment-paddle-mixed-v2-cuda-vs-tesseract-v2/benchmark/report.json \
765
+ --eval-manifest data/splits-mixed-v1/eval-with-refs-aws.jsonl \
766
+ --out outputs/experiment-paddle-mixed-v2-cuda-vs-tesseract-v2/rescored-with-refs
767
+ ```
768
+
769
+ Local archive:
770
+
771
+ ```text
772
+ outputs/aws-g6-paddle-mixed-v2-cuda/ocrtech-paddle-mixed-v2-cuda-artifacts.tar.gz
773
+ ```
774
+
775
+ Result:
776
+
777
+ - rows: `360`, all `ok`
778
+ - candidate CER/WER: `0.690203` / `0.944234`
779
+ - Tesseract CER/WER: `0.364915` / `0.521763`
780
+ - candidate latency: `0.077315s`
781
+ - Tesseract latency: `0.177510s`
782
+
783
+ This is a real trained model and a real negative result. It proves the CUDA
784
+ PaddleOCR path works end to end, but it does not support a better-than-Tesseract
785
+ claim. Before any claim rerun, normalize `.img` synthetic inputs to supported
786
+ image suffixes because PaddleOCR logged unsupported-extension warnings for that
787
+ subset.
788
+
789
+ Use the manifest image normalizer before the next claim-grade rerun:
790
+
791
+ ```bash
792
+ ocrtech normalize-manifest-images \
793
+ data/splits-mixed-v1/eval-with-refs.jsonl \
794
+ --out data/splits-mixed-v1/eval-normalized-images
795
+
796
+ ocrtech materialize-references \
797
+ data/splits-mixed-v1/eval-normalized-images/manifest-images-normalized.jsonl \
798
+ --out data/references-mixed-v1-normalized \
799
+ --rewritten-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl
800
+ ```
801
+
802
+ The normalizer copies already supported image files and converts unsupported
803
+ file suffixes to PNG with updated image hashes and provenance fields:
804
+ `metadata.source_image_path`, `metadata.source_image_sha256`, and
805
+ `metadata.image_normalization`.
806
+
807
+ On `k2`, the canonical mixed eval manifest already resolves to `.png` inputs.
808
+ After syncing the normalizer and preflight suffix guard, this command produced a
809
+ clean normalized eval artifact:
810
+
811
+ ```bash
812
+ ocrtech normalize-manifest-images \
813
+ data/splits-mixed-v1/eval-with-refs.jsonl \
814
+ --out data/splits-mixed-v1/eval-normalized-images
815
+ ```
816
+
817
+ Result:
818
+
819
+ - samples: `180`
820
+ - copied: `180`
821
+ - converted: `0`
822
+ - unsupported suffixes: none
823
+
824
+ Then:
825
+
826
+ ```bash
827
+ ocrtech materialize-references \
828
+ data/splits-mixed-v1/eval-normalized-images/manifest-images-normalized.jsonl \
829
+ --out data/references-mixed-v1-normalized \
830
+ --rewritten-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl
831
+
832
+ PATH=/opt/homebrew/bin:$PATH ocrtech preflight-claim \
833
+ --eval-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl \
834
+ --out outputs/preflight-mixed-v1-normalized-script-router-path \
835
+ --baselines candidate,tesseract,surya \
836
+ --candidate-model-config outputs/calibration-surya-script-router/model-card.json \
837
+ --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
838
+ --allow-smoke-eval-pack
839
+ ```
840
+
841
+ Preflight passed on runtime, references, suffixes, candidate audit, component
842
+ baselines, and calibration separation. It still warns that this is only an OCR
843
+ line-style eval, not a full document-understanding claim pack.
844
+
845
+ Local copy:
846
+
847
+ ```text
848
+ outputs/k2-mixed-v1-normalized-preflight/k2-mixed-v1-normalized-preflight.tar.gz
849
+ ```
850
+
851
+ ## 2026-06-15 k2 Normalized Mixed Eval Benchmark
852
+
853
+ Synced the manifest-driven benchmark CLI option to `k2` and verified it there:
854
+
855
+ ```bash
856
+ ~/.local/bin/uv run pytest \
857
+ tests/test_cli.py::test_cli_benchmark_loads_inputs_from_manifest -q
858
+ ```
859
+
860
+ Then ran the normalized mixed eval directly from the manifest:
861
+
862
+ ```bash
863
+ PATH=/opt/homebrew/bin:$PATH ~/.local/bin/uv run ocrtech benchmark \
864
+ --inputs-from-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl \
865
+ --out outputs/live-k2-mixed-v1-normalized-script-router-benchmark \
866
+ --baselines candidate,tesseract,surya \
867
+ --candidate-model-config outputs/calibration-surya-script-router/model-card.json \
868
+ --resume-existing \
869
+ --sample-timeout-seconds 180
870
+ ```
871
+
872
+ The run checkpointed `428` successful rows:
873
+
874
+ - candidate: `180/180`
875
+ - Tesseract: `180/180`
876
+ - Surya: `68/180`
877
+
878
+ The Surya phase stopped progressing after row `68`; the benchmark process was
879
+ terminated manually after preserving `report.json`. Treat the Surya numbers from
880
+ this run as partial diagnostics only.
881
+
882
+ Complete candidate-vs-Tesseract result:
883
+
884
+ - candidate CER/WER: `0.354222` / `0.493555`
885
+ - Tesseract CER/WER: `0.329012` / `0.486485`
886
+ - candidate English CER/WER: `0.272420` / `0.409552`
887
+ - Tesseract English CER/WER: `0.243379` / `0.370552`
888
+ - candidate Nepali CER/WER: `0.763234` / `0.913571`
889
+ - Tesseract Nepali CER/WER: `0.757175` / `1.066151`
890
+
891
+ Local copy:
892
+
893
+ ```text
894
+ outputs/k2-mixed-v1-normalized-script-router-benchmark/k2-mixed-v1-normalized-script-router-benchmark.tgz
895
+ ```
896
+
897
+ Interpretation: the normalized line-style rerun is a negative result for the
898
+ script-router candidate. It is worse than Tesseract overall and on English; its
899
+ only useful signal is Nepali WER. The next validation-infrastructure fix should
900
+ be a hard backend watchdog so stalled Surya/VLM calls become row-level errors
901
+ instead of manual run interruptions.
902
+
903
+ Follow-up: implemented the hard backend watchdog in `ocrtech.benchmark` and
904
+ reran the same normalized benchmark in a fresh output directory:
905
+
906
+ ```bash
907
+ PATH=/opt/homebrew/bin:$PATH ~/.local/bin/uv run ocrtech benchmark \
908
+ --inputs-from-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl \
909
+ --out outputs/live-k2-mixed-v1-normalized-script-router-benchmark-hard-timeout \
910
+ --baselines candidate,tesseract,surya \
911
+ --candidate-model-config outputs/calibration-surya-script-router/model-card.json \
912
+ --resume-existing \
913
+ --sample-timeout-seconds 45
914
+ ```
915
+
916
+ Completed hard-timeout result:
917
+
918
+ - rows: `540`
919
+ - candidate: `180/180`, all `ok`
920
+ - Tesseract: `180/180`, all `ok`
921
+ - Surya: `180/180` attempted, `178` ok, `2` timeout errors
922
+ - Surya timeout rows:
923
+ `nvidia__OCR-Synthetic-Multilingual-v1__en-test-000-000013-0000`,
924
+ `nvidia__OCR-Synthetic-Multilingual-v1__en-test-000-000028-0002`
925
+
926
+ Summary:
927
+
928
+ - candidate CER/WER: `0.354840` / `0.469944`
929
+ - Tesseract CER/WER: `0.329012` / `0.486485`
930
+ - Surya scored-row CER/WER: `3.157317` / `2.428888`
931
+
932
+ Local copy:
933
+
934
+ ```text
935
+ outputs/k2-mixed-v1-normalized-script-router-hard-timeout/k2-mixed-v1-normalized-script-router-hard-timeout.tgz
936
+ ```
937
+
938
+ Interpretation: the watchdog fix works. It converts the previous Surya hang into
939
+ two row-level timeouts and finishes the full three-system run. The result still
940
+ does not support the script-router candidate as a better OCR engine.
941
+
942
+ ## 2026-06-15 k2 PaddleOCR Recognizer Hard-50 Eval Slice
943
+
944
+ The AWS G6 trained PaddleOCR recognizer lost overall. We ranked its top held-out
945
+ CER failures and remapped them onto the normalized k2 eval manifest so the next
946
+ recognizer branch has a runnable hard-eval gate.
947
+
948
+ Commands on k2:
949
+
950
+ ```bash
951
+ ~/.local/bin/uv run ocrtech export-failure-manifest \
952
+ --source-manifest data/splits-mixed-v1/eval-normalized-images-with-refs.jsonl \
953
+ --top-failures outputs/paddle-mixed-v2-cuda-error-analysis/top-failures.json \
954
+ --out data/splits-mixed-v1/eval-paddle-mixed-v2-cuda-hard50-normalized.jsonl \
955
+ --split paddle_hard_eval
956
+
957
+ ~/.local/bin/uv run ocrtech audit-manifest \
958
+ data/splits-mixed-v1/eval-paddle-mixed-v2-cuda-hard50-normalized.jsonl \
959
+ --out outputs/manifest-audit-paddle-mixed-v2-cuda-hard50-normalized \
960
+ --require-slice paddle_hard_eval \
961
+ --require-slice english \
962
+ --require-slice nepali
963
+ ```
964
+
965
+ Result:
966
+
967
+ - selected hard failures: `50`
968
+ - slice counts: `synthetic=50`, `english=38`, `en=38`, `line_crop=38`,
969
+ `nepali=12`
970
+ - audit: pass
971
+
972
+ Local copy:
973
+
974
+ ```text
975
+ outputs/k2-paddle-mixed-v2-hard50-normalized/k2-paddle-mixed-v2-hard50-normalized.tgz
976
+ ```
977
+
978
+ Interpretation: this hard-50 manifest is held-out evaluation material, not
979
+ training data. It shows the trained PaddleOCR branch is failing mostly on
980
+ synthetic English line crops, despite the rebalanced training manifest containing
981
+ `850` English and `850` Nepali samples. The next recognizer experiment must pass
982
+ this hard slice while preserving the full normalized mixed eval and document
983
+ pack gates.
984
+
985
+ ## 2026-06-15 k2 Line-Level PaddleOCR Composite Rerun
986
+
987
+ We tested whether the failed AWS G6 PaddleOCR recognizer could still be useful
988
+ as a selective Devanagari fallback behind a Tesseract primary engine.
989
+
990
+ The first hard-50 run was invalid because all candidate rows crashed with
991
+ `exit_code=-11`. Root cause: hard-timeout mode initialized PaddleOCR in the
992
+ parent process and then forked child workers. The benchmark now initializes the
993
+ engine inside each timeout worker. Focused tests passed locally and on k2.
994
+
995
+ k2 command:
996
+
997
+ ```bash
998
+ PATH=/opt/homebrew/bin:$PATH ~/.local/bin/uv run ocrtech benchmark \
999
+ --inputs-from-manifest data/splits-mixed-v1/eval-paddle-mixed-v2-cuda-hard50-normalized.jsonl \
1000
+ --out outputs/experiment-line-script-paddle-hard50-v2 \
1001
+ --baselines candidate,tesseract \
1002
+ --candidate-model-config outputs/line-script-paddle-mixed-v2-cuda/model-card.json \
1003
+ --resume-existing \
1004
+ --sample-timeout-seconds 120
1005
+ ```
1006
+
1007
+ Result:
1008
+
1009
+ - rows: `100`
1010
+ - candidate: `50/50`, all `ok`
1011
+ - Tesseract: `50/50`, all `ok`
1012
+ - validation: `not_validated`
1013
+ - hard-50 CER: candidate `0.276489`, Tesseract `0.120106`
1014
+ - English line-crop CER/WER: exact tie with Tesseract, `0.060429` / `0.132259`
1015
+ - Nepali hard CER/WER: candidate `0.960679` / `1.000000`, Tesseract
1016
+ `0.309083` / `0.711607`
1017
+
1018
+ Local artifacts:
1019
+
1020
+ ```text
1021
+ outputs/experiment-line-script-paddle-hard50-v2/report.json
1022
+ outputs/summary-line-script-paddle-hard50-v2/summary.md
1023
+ outputs/validation-line-script-paddle-hard50-v2/validation.md
1024
+ ```
1025
+
1026
+ Decision: do not spend AWS G6 time rerunning this exact composite. A CUDA/G6
1027
+ machine is useful only for the next recognizer training branch: either a better
1028
+ PaddleOCR recipe/base model or a different Devanagari recognizer that first
1029
+ passes the hard-50 admission gate.
1030
+
1031
+ ## AWS G6 PaddleOCR V3 Run, 2026-06-15
1032
+
1033
+ We launched a fresh CUDA host while the `playaround` SSO window was active:
1034
+
1035
+ - instance: `i-0b8604ffbcbce25c7`
1036
+ - type: `g6.xlarge`
1037
+ - GPU: NVIDIA L4, `23034 MiB`
1038
+ - region: `us-east-1`
1039
+ - public IP at launch: `44.202.252.0`
1040
+ - AMI: `ami-01011b868ec560823`, Deep Learning Base OSS Nvidia Driver GPU AMI
1041
+ Ubuntu 22.04, `20260609`
1042
+ - SSH key: `ocrtech-g6-training-20260615`
1043
+ - security group: `sg-05ad6744a62dcabdd`, SSH restricted to the current
1044
+ workstation IP used at launch
1045
+
1046
+ Runtime setup:
1047
+
1048
+ - repo source snapshot extracted to `/opt/ocrtech`
1049
+ - k2 training bundle copied from
1050
+ `runs/paddle-recognizer-mixed-v2-bundle/paddle-recognizer-mixed-v2-training.tar.gz`
1051
+ - `uv` installed
1052
+ - Python `3.11` venv created
1053
+ - installed repo extras: `datasets`, `engines`, `dev`
1054
+ - installed `paddlepaddle-gpu==3.3.1` from the Paddle CUDA 12.6 index
1055
+ - checked out PaddleOCR at `c166448875bcecb8d3b7628fd697ac1c28f8705b`
1056
+ - installed PaddleOCR training requirements
1057
+ - `paddle.utils.run_check()` passed on one GPU
1058
+
1059
+ Two repo bugs were fixed before the run could start:
1060
+
1061
+ - integral `epoch_num` values must be emitted as integers, not floats. Paddle's
1062
+ cosine scheduler rejected `20.0` because `T_max` must be an `int`.
1063
+ - the PaddleOCR training default must be the official Paddle pretrained
1064
+ `.pdparams` URL, not the Hugging Face model id. PaddleOCR's training loader
1065
+ could not resolve `PaddlePaddle/devanagari_PP-OCRv3_mobile_rec`.
1066
+
1067
+ V3 training hypothesis:
1068
+
1069
+ - same verified train/eval split as v2: `1700` train, `180` eval
1070
+ - lower learning rate: `0.0001`
1071
+ - longer schedule: `20` epochs
1072
+ - official pretrained URL:
1073
+ `https://paddle-model-ecology.bj.bcebos.com/paddlex/official_pretrained_model/devanagari_PP-OCRv3_mobile_rec_pretrained.pdparams`
1074
+ - run directory: `/opt/ocrtech/runs/paddle-recognizer-mixed-v3-cuda`
1075
+
1076
+ Launch command on the G6:
1077
+
1078
+ ```bash
1079
+ cd /opt/ocrtech
1080
+ . .venv/bin/activate
1081
+ ocrtech train-recognizer \
1082
+ --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
1083
+ --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
1084
+ --out runs/paddle-recognizer-mixed-v3-cuda \
1085
+ --train-batch-size 16 \
1086
+ --eval-batch-size 16 \
1087
+ --learning-rate 0.0001 \
1088
+ --epochs 20
1089
+
1090
+ nohup python PaddleOCR/tools/train.py \
1091
+ -c runs/paddle-recognizer-mixed-v3-cuda/paddle_recognizer.yml \
1092
+ > runs/paddle-recognizer-mixed-v3-cuda/train.log 2>&1 &
1093
+ ```
1094
+
1095
+ Initial status:
1096
+
1097
+ - process started as PID `4071`
1098
+ - dataloaders built: `106` train iters, `12` eval iters
1099
+ - pretrained weights downloaded and loaded
1100
+ - classifier/head weights were shape-mismatched because the expanded dictionary
1101
+ has more classes; PaddleOCR skipped those layers and loaded the rest
1102
+ - first logged training step: epoch `1/20`, global step `10`, loss `137.718468`,
1103
+ ETA about `44` minutes
1104
+
1105
+ Final training status:
1106
+
1107
+ - completed all `20` epochs
1108
+ - best checkpoint: epoch `17`
1109
+ - best training-reported eval accuracy: `0.09444443919753115`
1110
+ - best norm edit distance: `0.5833825886055732`
1111
+ - export succeeded from `checkpoints/best_accuracy.pdparams`
1112
+ - package audit passed for `ocrtech-paddle-mixed-v3-cuda`
1113
+
1114
+ Hard-50 setup correction:
1115
+
1116
+ - the first G6 hard-50 benchmark produced no CER/WER because references had not
1117
+ been materialized on the host.
1118
+ - we copied all `50` normalized hard-50 PNGs from k2 and materialized references
1119
+ into `data/references-paddle-mixed-v3-hard50`.
1120
+ - the original hard-50 manifest then audited cleanly on G6.
1121
+ - the existing OCR outputs were rescored with
1122
+ `eval-paddle-mixed-v2-cuda-hard50-g6-with-refs.jsonl`.
1123
+
1124
+ Hard-50 result: `not_validated`.
1125
+
1126
+ | slice | metric | candidate | Tesseract | result |
1127
+ | --- | --- | ---: | ---: | --- |
1128
+ | all hard-50 | CER | 0.921416 | 0.080249 | fail |
1129
+ | all hard-50 | WER | 1.010476 | 0.258921 | fail |
1130
+ | English line-crop | CER | 0.917511 | 0.031244 | fail |
1131
+ | English line-crop | WER | 0.996241 | 0.114464 | fail |
1132
+ | Nepali hard | CER | 0.933782 | 0.235433 | fail |
1133
+ | Nepali hard | WER | 1.055556 | 0.716369 | fail |
1134
+
1135
+ Local artifact bundle:
1136
+
1137
+ ```text
1138
+ outputs/aws-g6-paddle-mixed-v3-cuda/ocrtech-paddle-mixed-v3-cuda-artifacts.tgz
1139
+ ```
1140
+
1141
+ Important interpretation: this v3 run fixed the infrastructure bugs but not the
1142
+ model problem. Lower learning rate plus longer schedule on the same PP-OCRv3
1143
+ Devanagari recipe still fails the hard-50 admission gate, especially English
1144
+ preservation. Do not run the full mixed eval for this branch.
1145
+
1146
+ ## AWS G6 PaddleOCR V4 Run, 2026-06-15
1147
+
1148
+ Current live host:
1149
+
1150
+ - instance: `i-0fa19dff7b233a353`
1151
+ - type: `g6.xlarge`
1152
+ - region/AZ: `us-east-1` / `us-east-1a`
1153
+ - public IP: `3.231.208.121`
1154
+ - AMI: `ami-01011b868ec560823`
1155
+ - SSH key: `ocrtech-g6-training-20260615`
1156
+ - root volume: `200 GiB gp3`
1157
+
1158
+ Setup already completed on the host:
1159
+
1160
+ - source at `/opt/ocrtech`
1161
+ - mixed-v2 bundle extracted from
1162
+ `runs/paddle-recognizer-mixed-v2-bundle/paddle-recognizer-mixed-v2-training.tar.gz`
1163
+ - Python `3.11.15` venv created with `uv`
1164
+ - `paddlepaddle-gpu==3.3.1` installed
1165
+ - PaddleOCR checked out at `c166448875bcecb8d3b7628fd697ac1c28f8705b`
1166
+ - PaddleOCR requirements installed
1167
+ - `paddle.utils.run_check()` passed on one NVIDIA L4
1168
+ - train manifest audit passed: `1700` samples
1169
+ - eval manifest audit passed: `180` samples
1170
+
1171
+ V4 hypothesis:
1172
+
1173
+ - same verified mixed-v1 split as v2/v3
1174
+ - official Devanagari PP-OCRv3 `.pdparams` base model
1175
+ - lower learning rate than v3: `0.00005`
1176
+ - `20` epochs
1177
+ - batch size `16`
1178
+ - run directory: `/opt/ocrtech/runs/paddle-recognizer-mixed-v4-cuda`
1179
+
1180
+ Two operational notes matter:
1181
+
1182
+ - The first v4 launch failed before consuming training time because YAML parsed
1183
+ `learning_rate: 5e-05` as a string. The recipe generator now emits
1184
+ `learning_rate: 0.00005`, and local tests cover this.
1185
+ - A bad local sync once copied `.venv` to the host and replaced Linux console
1186
+ scripts with macOS interpreter paths. Repair was done by deleting `.venv`,
1187
+ recreating it with `uv venv --python 3.11`, reinstalling repo extras,
1188
+ `paddlepaddle-gpu==3.3.1`, PaddleOCR requirements, and rerunning
1189
+ `paddle.utils.run_check()`.
1190
+
1191
+ When SSH is available, first check whether training is already running:
1192
+
1193
+ ```bash
1194
+ cd /opt/ocrtech
1195
+ pgrep -af 'PaddleOCR/tools/train.py|train-v4.sh' || true
1196
+ nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits
1197
+ tail -80 runs/paddle-recognizer-mixed-v4-cuda/train.log
1198
+ ```
1199
+
1200
+ If it is not running, regenerate the recipe with the fixed local code and
1201
+ launch:
1202
+
1203
+ ```bash
1204
+ cd /opt/ocrtech
1205
+ .venv/bin/ocrtech train-recognizer \
1206
+ --backend paddleocr \
1207
+ --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
1208
+ --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
1209
+ --out runs/paddle-recognizer-mixed-v4-cuda \
1210
+ --train-batch-size 16 \
1211
+ --eval-batch-size 16 \
1212
+ --learning-rate 0.00005 \
1213
+ --epochs 20
1214
+
1215
+ grep -E 'learning_rate|epoch_num' runs/paddle-recognizer-mixed-v4-cuda/paddle_recognizer.yml
1216
+
1217
+ cat > runs/paddle-recognizer-mixed-v4-cuda/train-v4.sh <<'SH'
1218
+ #!/usr/bin/env bash
1219
+ set -euo pipefail
1220
+ cd /opt/ocrtech
1221
+ source .venv/bin/activate
1222
+ export FLAGS_fraction_of_gpu_memory_to_use=0.92
1223
+ python PaddleOCR/tools/train.py -c runs/paddle-recognizer-mixed-v4-cuda/paddle_recognizer.yml
1224
+ SH
1225
+
1226
+ chmod +x runs/paddle-recognizer-mixed-v4-cuda/train-v4.sh
1227
+ nohup bash runs/paddle-recognizer-mixed-v4-cuda/train-v4.sh \
1228
+ > runs/paddle-recognizer-mixed-v4-cuda/train.log 2>&1 &
1229
+ echo $! > runs/paddle-recognizer-mixed-v4-cuda/train.pid
1230
+ ```
1231
+
1232
+ After completion, export, package, and include training-summary provenance:
1233
+
1234
+ ```bash
1235
+ cd /opt/ocrtech
1236
+ .venv/bin/ocrtech export-recognizer \
1237
+ --training-config runs/paddle-recognizer-mixed-v4-cuda/paddle_recognizer.yml \
1238
+ --checkpoint runs/paddle-recognizer-mixed-v4-cuda/checkpoints/best_accuracy \
1239
+ --out runs/paddle-recognizer-mixed-v4-cuda/export \
1240
+ --run
1241
+
1242
+ .venv/bin/ocrtech package-recognizer \
1243
+ --inference-dir runs/paddle-recognizer-mixed-v4-cuda/export/inference \
1244
+ --dictionary runs/paddle-recognizer-mixed-v4-cuda/nep_eng_dict.txt \
1245
+ --out runs/paddle-recognizer-mixed-v4-cuda/package \
1246
+ --model-id ocrtech-paddle-mixed-v4-cuda \
1247
+ --train-manifest data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl \
1248
+ --eval-manifest data/splits-mixed-v1/eval-with-refs.jsonl \
1249
+ --training-summary runs/paddle-recognizer-mixed-v4-cuda/recognizer-training-summary.json
1250
+
1251
+ .venv/bin/ocrtech audit-model runs/paddle-recognizer-mixed-v4-cuda/package/model-card.json \
1252
+ --out runs/paddle-recognizer-mixed-v4-cuda/audit
1253
+ ```
1254
+
1255
+ V4 final status:
1256
+
1257
+ - training completed all `20` epochs on the G6 L4 instance.
1258
+ - best PaddleOCR eval metric from training: accuracy `0.016666665740740792`,
1259
+ norm edit distance `0.442565389555461`, best epoch `19`.
1260
+ - export succeeded from checkpoint prefix
1261
+ `runs/paddle-recognizer-mixed-v4-cuda/checkpoints/best_accuracy`.
1262
+ - package and model audit succeeded after model-card artifact hashes were fixed
1263
+ to resolve package-relative files.
1264
+ - hard-50 admission gate completed with status `not_validated`.
1265
+ - candidate all CER/WER: `0.940500` / `1.028667`.
1266
+ - Tesseract all CER/WER: `0.080249` / `0.258921`.
1267
+ - English line-crop CER: candidate `0.950730`, Tesseract `0.031244`.
1268
+ - Nepali CER: candidate `0.908108`, Tesseract `0.235433`.
1269
+ - paired CER wins/losses/ties vs Tesseract: `0` / `50` / `0`.
1270
+ - local artifact bundle:
1271
+ `outputs/aws-g6-paddle-mixed-v4-cuda/ocrtech-paddle-mixed-v4-cuda-artifacts.tgz`.
1272
+ - stop request completed for instance `i-0fa19dff7b233a353`; AWS reported
1273
+ final state `stopped`.
1274
+
1275
+ Do not relaunch this exact branch as the engine core. The v4 result is a third
1276
+ negative PaddleOCR mixed-script recognizer result after v2 and v3.
1277
+
1278
+ ## AWS G6 HF Recognizer Next V1
1279
+
1280
+ The stopped G6 instance `i-0fa19dff7b233a353` was restarted on 2026-06-15 for a
1281
+ new HF recognizer run instead of launching a duplicate GPU host.
1282
+
1283
+ Host details at launch:
1284
+
1285
+ - instance type: `g6.xlarge`
1286
+ - public IP: `3.91.25.53`
1287
+ - SSH key: `~/.ssh/ocrtech-g6-training-20260615.pem`
1288
+ - remote workspace: `/opt/ocrtech`
1289
+ - GPU: NVIDIA L4, `23034` MiB
1290
+ - driver reported by `nvidia-smi`: `580.159.04`
1291
+ - torch runtime after install: `2.12.0+cu130`
1292
+ - CUDA visibility: `torch.cuda.is_available() == True`
1293
+
1294
+ The source files `training.py`, `admission.py`, `validation.py`, and `cli.py`
1295
+ were synced to `/opt/ocrtech/src/ocrtech/`. The verified bundle archive was
1296
+ uploaded to:
1297
+
1298
+ ```text
1299
+ /opt/ocrtech/incoming/hf-recognizer-next-v1-training.tar.gz
1300
+ ```
1301
+
1302
+ Archive verification:
1303
+
1304
+ ```text
1305
+ aff0b277b9cfdd353ed82178c65586353c3511d6db06bd20099cbc62df113873
1306
+ ```
1307
+
1308
+ Dependencies were installed with:
1309
+
1310
+ ```bash
1311
+ cd /opt/ocrtech
1312
+ .venv/bin/python -m pip install -e ".[hf-recognizer]"
1313
+ ```
1314
+
1315
+ The bundle was extracted at `/opt/ocrtech`, then the run was launched with
1316
+ `nohup` through:
1317
+
1318
+ ```text
1319
+ /opt/ocrtech/runs/hf-recognizer-next-v1/train-hf-next-v1.sh
1320
+ ```
1321
+
1322
+ The script runs:
1323
+
1324
+ ```bash
1325
+ ocrtech train-recognizer \
1326
+ --backend hf-vision-encoder-decoder \
1327
+ --base-model microsoft/trocr-small-printed \
1328
+ --train-manifest data/splits-recognizer-next-v1/train.jsonl \
1329
+ --eval-manifest data/splits-recognizer-next-v1/eval.jsonl \
1330
+ --out runs/hf-recognizer-next-v1 \
1331
+ --max-target-length 128 \
1332
+ --train-batch-size 8 \
1333
+ --eval-batch-size 8 \
1334
+ --learning-rate 5e-05 \
1335
+ --epochs 5 \
1336
+ --run
1337
+ ```
1338
+
1339
+ Live files:
1340
+
1341
+ - PID file:
1342
+ `/opt/ocrtech/runs/hf-recognizer-next-v1/train.pid`
1343
+ - launch PID: `1862`
1344
+ - log:
1345
+ `/opt/ocrtech/runs/hf-recognizer-next-v1/train.log`
1346
+
1347
+ Observed shortly after launch:
1348
+
1349
+ - process was alive
1350
+ - GPU memory used: about `6500` MiB
1351
+ - GPU utilization: `46%` to `98%`
1352
+ - training progress: at least `200` / `1385` trainer steps
1353
+
1354
+ Final status:
1355
+
1356
+ - training completed all `5` epochs.
1357
+ - final model copied locally to:
1358
+ `outputs/aws-g6-hf-recognizer-next-v1/runs-hf-recognizer-next-v1/final-model`
1359
+ - local package:
1360
+ `outputs/aws-g6-hf-recognizer-next-v1/package/model-card.json`
1361
+ - local audit:
1362
+ `outputs/aws-g6-hf-recognizer-next-v1/audit/model-audit.json`
1363
+ - audit status: passed
1364
+ - final `eval_cer`: `0.18840631911585276`
1365
+ - final `eval_wer`: `0.36464600972797695`
1366
+ - final exact-line accuracy: `0.4344262295081967`
1367
+ - final `eval_loss`: `1.0254589319229126`
1368
+
1369
+ The first attempt to stop the host interrupted a broad rsync before any local
1370
+ artifact copy landed. The same instance was restarted only to retrieve the
1371
+ completed run, then the copy was narrowed to final model files and metadata,
1372
+ excluding trainer checkpoints. The local artifact set intentionally omits
1373
+ checkpoint optimizer state.
1374
+
1375
+ After artifact retrieval, the local AWS SSO session had expired and blocked the
1376
+ API stop call. The instance was shut down through SSH instead:
1377
+
1378
+ ```bash
1379
+ sudo shutdown -h now
1380
+ ```
1381
+
1382
+ The SSH session closed and a later SSH check to the last public IP
1383
+ `3.236.132.219` timed out. Refresh AWS SSO and confirm the EC2 state before any
1384
+ future GPU work:
1385
+
1386
+ ```bash
1387
+ aws sso login --profile playaround
1388
+ aws ec2 describe-instances \
1389
+ --profile playaround \
1390
+ --region us-east-1 \
1391
+ --instance-ids i-0fa19dff7b233a353 \
1392
+ --query 'Reservations[0].Instances[0].State.Name' \
1393
+ --output text
1394
+ ```
1395
+
1396
+ For future runs, do not stop the instance while a PID is still making progress.
1397
+ Stop it as soon as the run completes, fails, or no longer needs the GPU:
1398
+
1399
+ ```bash
1400
+ aws ec2 stop-instances \
1401
+ --profile playaround \
1402
+ --region us-east-1 \
1403
+ --instance-ids i-0fa19dff7b233a353
1404
+ ```
docs/VALIDATION.md ADDED
@@ -0,0 +1,638 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Validation Design
2
+
3
+ The claim target is not "we ran OCR and it looked good." The claim target is:
4
+
5
+ ```text
6
+ ocrtech is better than named baselines on held-out Nepali + English OCR and document-structure extraction, under measured gates.
7
+ ```
8
+
9
+ Until the validation report says `validated`, do not claim SOTA.
10
+
11
+ Claim-grade `validation.json` files record provenance for the benchmark report,
12
+ validation config, eval manifest, train manifests, candidate model card, and
13
+ fallback model card used by `validate-claim`. `review-claim` cross-checks
14
+ those hashes against `experiment.json`, so a validation artifact from a
15
+ different benchmark, eval split, validation config, model card, fallback card,
16
+ or training-manifest set cannot be reused as portfolio evidence.
17
+
18
+ The SOTA review profile is paired with these claim-grade validation configs:
19
+
20
+ - `configs/validation-internal-sota-v1.json`
21
+ - `configs/validation-public-omnidocbench-sota-v1.json`
22
+ - `configs/validation-public-real5-sota-v1.json`
23
+
24
+ Each one covers the `required_validation_gates` declared in
25
+ `configs/claim-review-sota-v1.json`. Use the slot-specific config when running
26
+ `run-experiment` or `validate-claim`; otherwise portfolio review may have
27
+ metrics and baselines but still lack the passed gates required for the paper
28
+ claim.
29
+
30
+ ## Evidence Required
31
+
32
+ A defensible run needs all of this:
33
+
34
+ - A claim-ready image eval pack from `ocrtech create-eval-pack`, plus public benchmark runs.
35
+ - A candidate model card from `ocrtech create-model-card` and a passing `ocrtech audit-model` report.
36
+ - Training data prepared with `audit-manifest` and `split-manifest`.
37
+ - Held-out eval manifest that was never used for recognizer, corrector, threshold, or prompt tuning.
38
+ - Image/PDF eval inputs normalized to OCR-supported file suffixes before any
39
+ claim-grade PaddleOCR/Tesseract/Surya/VLM comparison.
40
+ - For routed or thresholded candidates, a separate calibration experiment artifact that records how the threshold was chosen.
41
+ - Train manifests for leakage checks.
42
+ - Paired benchmark outputs for every baseline on the same input files.
43
+ - Per-sample metrics, not only aggregate averages.
44
+ - For `glm-ocr` and `paddleocr-vl`, a real runnable command hook that emits Markdown or `document.md`, not a placeholder env var.
45
+ - Slice coverage for Nepali, English, multi-page documents, tables, reading order, figures/charts, forms, scans, photos, skew, blur, compression, and low contrast.
46
+ - Gates that name the baseline, metric, slice, direction, required margin, and confidence requirement.
47
+ - Paired win/loss evidence, a minimum paired win-rate requirement when the
48
+ claim needs broad per-document improvement, and a nonparametric significance
49
+ check when the claim requires it.
50
+ - Failure-case report, not just the winning average.
51
+
52
+ ## Public Benchmark Gate
53
+
54
+ Generated eval packs are development evidence. They are not enough for SOTA
55
+ wording. Before a paper claim can say "state of the art," at least one public
56
+ document parsing benchmark must be converted into an auditable manifest and run
57
+ as a normal `ocrtech run-experiment`.
58
+
59
+ Known public benchmark registry:
60
+
61
+ ```bash
62
+ ocrtech list-public-benchmarks
63
+ ```
64
+
65
+ Current registry entries:
66
+
67
+ - `opendatalab/OmniDocBench`: real-world document parsing benchmark, dataset page `https://huggingface.co/datasets/opendatalab/OmniDocBench`
68
+ - `allenai/olmOCR-bench`: PDF-to-Markdown benchmark with unit-test style assertions, dataset page `https://huggingface.co/datasets/allenai/olmOCR-bench`
69
+ - `PaddlePaddle/Real5-OmniDocBench`: real-world physical reconstruction benchmark with scanning, warping, screen-photo, illumination, and skew scenarios, dataset page `https://huggingface.co/datasets/PaddlePaddle/Real5-OmniDocBench`
70
+
71
+ The first implemented public converter is:
72
+
73
+ ```bash
74
+ ocrtech prepare-public-benchmark opendatalab/OmniDocBench \
75
+ --out data/public/omnidocbench \
76
+ --limit 100 \
77
+ --language english \
78
+ --require-tables
79
+ ```
80
+
81
+ The converter was written only after checking the actual dataset structure:
82
+ `OmniDocBench.json` contains `page_info`, `page_info.page_attribute`, and
83
+ `layout_dets`; image files resolve as `images/{page_info.image_path}`; table
84
+ ground truth is stored in table-block `html`.
85
+
86
+ Every public benchmark manifest row must include:
87
+
88
+ - `metadata.public_benchmark = true`
89
+ - `metadata.benchmark_name`
90
+ - `metadata.source_url`
91
+ - `metadata.reference_path`, unless the benchmark has a separate official scorer integration
92
+ - `metadata.slices` including `public` plus benchmark-specific slices
93
+
94
+ Audit it before running:
95
+
96
+ ```bash
97
+ ocrtech audit-public-benchmark data/public/omnidocbench/manifests/omnidocbench-eval.jsonl \
98
+ --benchmark opendatalab/OmniDocBench \
99
+ --out outputs/public-omnidocbench-audit
100
+ ```
101
+
102
+ This audit intentionally fails if provenance or references are missing. Do not
103
+ use `--allow-missing-reference-paths` for claim runs unless an official external
104
+ scorer is wired and documented in the experiment report.
105
+
106
+ Portfolio review for SOTA claims should use:
107
+
108
+ ```bash
109
+ ocrtech review-claim \
110
+ --experiment outputs/internal-experiment/experiment.json \
111
+ --experiment outputs/public-omnidocbench/experiment.json \
112
+ --config configs/claim-review-sota-v1.json \
113
+ --model-config outputs/figure-caption-table-lexicon-v1/model-card.json \
114
+ --out outputs/claim-review-sota-v1
115
+ ```
116
+
117
+ The config requires internal Nepali-English evidence plus public benchmark
118
+ evidence. It should fail until those public experiments actually exist.
119
+
120
+ Claim-grade validation configs can also set `min_pair_win_rate` on a gate. This
121
+ is intentionally different from the reported `win_rate`: `win_rate` is computed
122
+ over non-tied pairs, while `min_pair_win_rate` is checked as wins divided by all
123
+ paired samples. That distinction matters for document OCR claims. A result with
124
+ one win and many ties can have `win_rate = 1.0`, but it has low pair coverage
125
+ and should not satisfy a broad "better than baseline" claim unless the gate is
126
+ explicitly a no-regression/tie-acceptable gate.
127
+
128
+ The public OmniDocBench-vs-Surya validation gate is:
129
+
130
+ ```bash
131
+ ocrtech validate-claim \
132
+ --benchmark-report outputs/experiment-public-omnidocbench-smoke-v1/benchmark/report.json \
133
+ --config configs/validation-public-omnidocbench-surya.json \
134
+ --eval-manifest data/public/omnidocbench-smoke/manifests/omnidocbench-eval.jsonl \
135
+ --candidate-model-config runs/recognizer/package/model-card.json \
136
+ --fallback-model-config runs/vlm-fallback/package/model-card.json \
137
+ --out outputs/validation-public-omnidocbench-smoke-v1
138
+ ```
139
+
140
+ It requires at least `100` public document-parsing samples and at least `40`
141
+ table samples, then gates candidate-vs-Surya CER, table cell F1, structure-only
142
+ TEDS, and text-sensitive TEDS. A 5-page smoke run is allowed as engineering
143
+ evidence, but it must remain `not_validated`.
144
+
145
+ The first audited manifest sized for this gate is:
146
+
147
+ ```text
148
+ data/public/omnidocbench-table100/manifests/omnidocbench-eval.jsonl
149
+ ```
150
+
151
+ Its audit artifact is
152
+ `outputs/public-omnidocbench-table100-audit/public-benchmark-audit.json`; the
153
+ audit passed with `100` public/document/table English samples. The corresponding
154
+ long benchmark run is tracked under
155
+ `outputs/experiment-public-omnidocbench-table100-width-guard-v4/`. Treat that
156
+ run as incomplete until `benchmark/report.json`, `summary/summary.json`,
157
+ `validation/validation.json`, and `table-cell-analysis/table-cell-analysis.json`
158
+ exist.
159
+
160
+ The completed width-guard-v4 Table100 run is an instructive non-passing result:
161
+
162
+ - rows: `200/200`, with one shared `300s` timeout for candidate and Surya
163
+ - paired scored samples: `99`
164
+ - candidate improves public table metrics over Surya:
165
+ `table_cell_f1` `0.484364` vs `0.406694`,
166
+ `table_teds_structure` `0.720160` vs `0.687550`,
167
+ `table_teds_text` `0.565284` vs `0.543987`
168
+ - candidate does not improve OCR CER:
169
+ `0.613201` vs Surya `0.612623`
170
+ - validation status: `not_validated`
171
+
172
+ For long-run status files, distinguish package readiness from claim readiness.
173
+ `evidence_package_ready` means the expected rows and final report artifacts are
174
+ present. `paper_claim_ready` must only be true when the validation report itself
175
+ passes. A completed benchmark package with `validation.claim_status =
176
+ not_validated` is useful evidence, but not a publishable SOTA claim.
177
+
178
+ Long public runs should use benchmark checkpointing and per-page timeouts:
179
+
180
+ ```bash
181
+ ocrtech benchmark ... \
182
+ --inputs-from-manifest data/eval/manifest-with-refs.jsonl \
183
+ --resume-existing \
184
+ --sample-timeout-seconds 300
185
+ ```
186
+
187
+ `--inputs-from-manifest` loads authoritative image paths from the manifest and
188
+ defaults `--eval-manifest` to the same file when it is not supplied. This avoids
189
+ shell-generated input lists drifting away from the manifest that contains
190
+ reference paths and slice labels. `--resume-existing` scores existing
191
+ `document.json` outputs and only runs OCR for missing pages.
192
+ `--sample-timeout-seconds` runs each OCR call behind a hard process watchdog on
193
+ platforms that support `fork`; a stalled backend is terminated and recorded as a
194
+ page-level `TimeoutError`, then the benchmark continues. Platforms without
195
+ `fork` fall back to the Python signal timer. A timeout is still a
196
+ candidate/baseline failure for validation; it is not a pass-through.
197
+ For timeout runs, the benchmark now constructs the OCR engine inside the
198
+ watchdog child process for each page. This avoids inheriting already-initialized
199
+ PaddleOCR/PaddlePaddle C++ runtime state across `fork`, which produced
200
+ `exit_code=-11` worker crashes in the first line-level Paddle composite run.
201
+
202
+ Table scoring matches each reference table to the best unused predicted table
203
+ before averaging table metrics. This matters on public document pages where an
204
+ engine may emit extra table fragments before the main table. Positional
205
+ table-index matching is too brittle for claim evidence.
206
+
207
+ When the evaluator changes, do not rerun slow OCR just to update metrics. Use:
208
+
209
+ ```bash
210
+ ocrtech rescore-benchmark \
211
+ --benchmark-report outputs/experiment-public-omnidocbench-smoke-v1/benchmark/report.json \
212
+ --eval-manifest data/public/omnidocbench-smoke/manifests/omnidocbench-eval.jsonl \
213
+ --out outputs/experiment-public-omnidocbench-smoke-v2-rescored/benchmark
214
+ ```
215
+
216
+ The rescore command copies the existing `document.json`, Markdown, table, and
217
+ figure outputs into the new benchmark directory so downstream diagnostics such
218
+ as `analyze-table-cells` remain usable.
219
+
220
+ When evaluating table postprocessors, do not rely only on separate candidate and
221
+ baseline OCR runs. Independent OCR/layout calls can introduce run-to-run variance
222
+ that makes a small table postprocessor look better or worse than it is. Use a
223
+ fixed-source derived comparison to isolate the postprocessor:
224
+
225
+ ```bash
226
+ ocrtech derive-table-postprocess-benchmark \
227
+ --benchmark-report outputs/experiment-public-omnidocbench-table40-normalize-rows-v3/benchmark/report.json \
228
+ --source-baseline surya \
229
+ --candidate-baseline candidate \
230
+ --eval-manifest data/public/omnidocbench-table40/manifests/omnidocbench-eval.jsonl \
231
+ --out outputs/experiment-public-omnidocbench-table40-derived-postprocess-v1/benchmark
232
+ ```
233
+
234
+ This command copies the chosen source baseline outputs, applies the current
235
+ reference-free table postprocessors to the copied candidate `document.json`, and
236
+ rescored both systems against the same references. The result is valid evidence
237
+ for the postprocessor contribution only. It is not an end-to-end OCR model card
238
+ and cannot support a broad OCR SOTA claim by itself.
239
+
240
+ The current normalized public candidate is
241
+ `ocrtech-surya-tesseract-figure-table-lexicon-normalize-v2`. Its table-grid
242
+ normalization is runtime-only and reference-free: empty columns, separator-only
243
+ columns, and exact duplicate adjacent columns are removed. On the 5-page
244
+ OmniDocBench smoke it improves table metrics over Surya, but the run remains
245
+ `not_validated` because the public sample count, table sample count, CER margin,
246
+ and bootstrap confidence requirements are not met.
247
+
248
+ ## Reference Format
249
+
250
+ Text-only reference:
251
+
252
+ ```text
253
+ page.png.ref.txt
254
+ ```
255
+
256
+ Structured reference:
257
+
258
+ ```json
259
+ {
260
+ "text": "नेपाल सरकार",
261
+ "reading_order": ["p0-l0", "p0-l1", "p0-l2"],
262
+ "tables": [
263
+ [
264
+ ["नाम", "उमेर", "जिल्ला"],
265
+ ["राम", "३२", "काठमाडौं"]
266
+ ]
267
+ ],
268
+ "metadata": {
269
+ "source": "hand_labeled"
270
+ }
271
+ }
272
+ ```
273
+
274
+ Complex table references can also use cell objects with spans:
275
+
276
+ ```json
277
+ {
278
+ "tables": [
279
+ {
280
+ "cells": [
281
+ {"row": 0, "col": 0, "text": "नाम", "rowspan": 1, "colspan": 2},
282
+ {"row": 1, "col": 0, "text": "राम"},
283
+ {"row": 1, "col": 1, "text": "१२"}
284
+ ]
285
+ }
286
+ ]
287
+ }
288
+ ```
289
+
290
+ The nested-list grid form remains supported. Span-aware references make `table_exact_match` and `table_structure_similarity` sensitive to missing rowspans/colspans even when flattened table text is correct.
291
+
292
+ The candidate parser can apply a conservative repeated-header span postprocessor:
293
+ adjacent identical cells in the first header rows are collapsed into a single
294
+ `colspan` cell only when the following row has distinct child columns. This lets
295
+ `document.json` and exported table HTML represent the simple merged-header
296
+ pattern in `complex-table-ledger`. Treat this as a heuristic parser repair until
297
+ it is validated on real scanned tables with varied merged cells and rowspans.
298
+
299
+ Supported structured metrics:
300
+
301
+ - `quality_score`
302
+ - `cer`
303
+ - `wer`
304
+ - `text_cer`
305
+ - `text_wer`
306
+ - `document_reconstruction_cer`
307
+ - `document_reconstruction_wer`
308
+ - `document_reconstruction_similarity`
309
+ - `figure_detection_precision`
310
+ - `figure_detection_recall`
311
+ - `figure_detection_f1`
312
+ - `figure_caption_cer`
313
+ - `reading_order_pair_accuracy`
314
+ - `table_cell_precision`
315
+ - `table_cell_recall`
316
+ - `table_cell_f1`
317
+ - `table_exact_match`
318
+ - `table_structure_similarity`
319
+ - `table_teds_structure`
320
+ - `table_teds_text`
321
+
322
+ ## Validation Config
323
+
324
+ Example:
325
+
326
+ ```json
327
+ {
328
+ "candidate": "ours",
329
+ "min_samples": 300,
330
+ "required_slices": {
331
+ "nepali": 100,
332
+ "english": 100,
333
+ "multi_page": 25,
334
+ "table": 50,
335
+ "reading_order": 50,
336
+ "figure": 30,
337
+ "scan": 50,
338
+ "phone_photo": 50
339
+ },
340
+ "bootstrap_iterations": 1000,
341
+ "bootstrap_confidence": 0.95,
342
+ "require_full_eval_manifest_coverage": true,
343
+ "gates": [
344
+ {
345
+ "baseline": "tesseract",
346
+ "metric": "cer",
347
+ "direction": "lower",
348
+ "slice": "nepali",
349
+ "min_pairs": 50,
350
+ "min_relative_improvement": 0.05,
351
+ "max_p_value": 0.05
352
+ },
353
+ {
354
+ "baseline": "stock-paddle",
355
+ "metric": "cer",
356
+ "direction": "lower",
357
+ "slice": "english",
358
+ "min_absolute_margin": 0.0
359
+ },
360
+ {
361
+ "baseline": "glm-ocr",
362
+ "metric": "table_cell_f1",
363
+ "direction": "higher",
364
+ "slice": "table",
365
+ "min_absolute_margin": 0.01
366
+ },
367
+ {
368
+ "baseline": "surya",
369
+ "metric": "cer",
370
+ "direction": "lower",
371
+ "slices": ["hard_eval", "table"],
372
+ "min_pairs": 8,
373
+ "min_absolute_margin": 0.02
374
+ },
375
+ {
376
+ "baseline": "paddleocr-vl",
377
+ "metric": "reading_order_pair_accuracy",
378
+ "direction": "higher",
379
+ "slice": "reading_order",
380
+ "min_absolute_margin": 0.01
381
+ }
382
+ ]
383
+ }
384
+ ```
385
+
386
+ Use `slice` for a single required slice. Use `slices` when a gate must apply to the intersection of multiple labels, for example `["hard_eval", "table"]` for table samples inside a hard-eval pocket. Do not approximate this with separate `hard_eval` and `table` gates; that allows unrelated samples to satisfy the table gate.
387
+
388
+ For claim-grade runs, set `require_full_eval_manifest_coverage` to `true`.
389
+ When enabled, `validate-claim` fails unless the candidate and every baseline
390
+ named by the configured gates have a successful row for every sample in
391
+ `--eval-manifest`. This prevents a validation report from passing on an easy
392
+ subset while silently omitting hard pages from the held-out manifest.
393
+
394
+ Run:
395
+
396
+ ```bash
397
+ ocrtech preflight-claim \
398
+ --eval-manifest data/manifests/eval.jsonl \
399
+ --references-dir data/references \
400
+ --baselines candidate,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
401
+ --candidate-model-config runs/recognizer/package/model-card.json \
402
+ --validation-config validation.json \
403
+ --train-manifest data/manifests/train.jsonl \
404
+ --out outputs/preflight
405
+
406
+ ocrtech benchmark \
407
+ --eval-manifest data/manifests/eval.jsonl \
408
+ --references-dir data/references \
409
+ --baselines ours,tesseract,stock-paddle,glm-ocr,paddleocr-vl \
410
+ --out outputs/benchmark \
411
+ $(cut -f4 data/manifests/eval.inputs)
412
+
413
+ ocrtech validate-claim \
414
+ --benchmark-report outputs/benchmark/report.json \
415
+ --config validation.json \
416
+ --eval-manifest data/manifests/eval.jsonl \
417
+ --train-manifest data/manifests/train.jsonl \
418
+ --out outputs/validation
419
+ ```
420
+
421
+ `preflight-claim` should be the first command in a claim-grade workflow. If it fails, the benchmark run is not ready.
422
+
423
+ For explicit `--validation-config` runs, preflight now checks the config against
424
+ the requested baselines and eval manifest before any OCR starts. It fails when
425
+ the candidate label is absent from `--baselines`, a gate baseline is not
426
+ scheduled, `min_samples` exceeds the manifest size, required slices are missing,
427
+ or a gate's `min_pairs` exceeds the number of manifest rows matching the gate
428
+ slice intersection. Omitting `--validation-config` is only a warning; it is not
429
+ claim-grade evidence.
430
+
431
+ For hybrid runs with `--fallback-model-config`, preflight also checks that the
432
+ candidate model card declares the same fallback under
433
+ `provenance.fallback_model_card`. This catches a wrong runtime fallback flag
434
+ before a benchmark produces evidence for a different composite engine than the
435
+ one described by the candidate model card.
436
+
437
+ If the candidate model uses `quality_select_composite`, `preflight-claim` now checks that the model card contains calibration provenance and that the calibration eval manifest hash is different from the final claim eval manifest hash. Tuning on the same eval pack invalidates the claim path.
438
+
439
+ `validate-claim` exits with `0` only when the configured claim is validated. It exits with `1` when evidence is missing or a gate fails.
440
+
441
+ Each gate now reports:
442
+
443
+ - paired sample count
444
+ - win / loss / tie counts
445
+ - win rate on non-tied pairs
446
+ - bootstrap CI for mean paired improvement
447
+ - optional one-sided sign-test p-value
448
+
449
+ For claim-grade runs, use `min_pairs` and `max_p_value` instead of relying only on a favorable mean. A narrow win on a handful of pages is not SOTA evidence.
450
+
451
+ For a single reproducible command that runs benchmark and validation together, use `run-experiment`; see [EXPERIMENTS.md](EXPERIMENTS.md).
452
+
453
+ For the final portfolio-level claim review across the internal pack and public suites, use `review-claim`; see [CLAIM_REVIEW.md](CLAIM_REVIEW.md).
454
+
455
+ For model artifact provenance, see [MODELS.md](MODELS.md). Do not use an implicit `auto` engine for SOTA claims.
456
+
457
+ ## Table Cell Analysis
458
+
459
+ Aggregate `table_cell_f1` is not enough to explain whether a table-oriented candidate is actually improving table OCR. Use `analyze-table-cells` after a benchmark has exported per-system `document.json` files:
460
+
461
+ ```bash
462
+ ocrtech analyze-table-cells \
463
+ --benchmark-report outputs/experiment-table-cell-refine-empty-hard15-v1/benchmark/report.json \
464
+ --eval-manifest data/splits-claimpack-rendered-v2/eval-surya-hard15.jsonl \
465
+ --out outputs/table-cell-analysis-refine-empty-hard15-v1 \
466
+ --candidate candidate \
467
+ --baseline surya \
468
+ --baseline tesseract
469
+ ```
470
+
471
+ The command requires:
472
+
473
+ - A benchmark report from `benchmark` or `run-experiment`.
474
+ - Exported benchmark documents at `benchmark/<system>/<input_stem>/document.json`.
475
+ - An eval manifest whose references contain structured `tables`.
476
+
477
+ It writes:
478
+
479
+ - `table-cell-analysis.json`
480
+ - `table-cell-analysis.md`
481
+
482
+ The report includes reference table count, predicted table count, duplicate overlapping table count, reference cell count, exact cell count, exact-cell rate, missing-cell count, mean cell CER, paired candidate-vs-baseline wins/losses/ties, and the highest-error candidate cells. This is the right artifact for identifying whether a table refiner is adding headroom, damaging already-correct cells, or producing structurally duplicated tables.
483
+
484
+ Interpret raw full-page Tesseract table results carefully. In the current v1 engine, Tesseract can be strong on exact image crops but still report zero structured table cells in full-page document parsing because it does not emit the table structure expected by `document.json`.
485
+
486
+ For the rendered hard15 table pages, reference `text` is the page title, while table contents live in structured `tables`. Do not mix these contracts: page-level `cer` answers whether the surrounding page text/title was recognized, and `table_cell_f1` plus cell exact analysis answers whether the table content was extracted. A benchmark change that inserts table grids into `Document.text` without also regenerating references changes the metric definition and is invalid for this pack.
487
+
488
+ The current `validation-hard15-surya.json` table-cell gate can fail even with a higher mean table-cell F1 when the bootstrap confidence interval includes zero. The figure-caption/table-lexicon v3 run is the concrete example: candidate `1.000000` vs Surya `0.979167`, but only 2 table pages improve and 6 tie, so the CI low is `0.000000`. Treat that as evidence that the table slice is too small for a strong claim. Do not remove the CI gate after observing the result; build a larger held-out table pack.
489
+
490
+ `configs/validation-table-ledger-surya.json` is the first targeted table-only validation config. It requires at least 40 table samples, gates `table_cell_f1` and `table_exact_match` against Surya with bootstrap CI above zero, and guards `table_structure_similarity` for non-regression. The v2 table-ledger pack (`60` samples, `720` cells) passes this config:
491
+
492
+ - `table_cell_f1`: candidate `1.000000`, Surya `0.979167`, CI low `0.012500`
493
+ - `table_exact_match`: candidate `1.000000`, Surya `0.750000`, CI low `0.150000`
494
+ - `table_structure_similarity`: candidate `1.000000`, Surya `1.000000`, non-regression guard passes
495
+
496
+ `table_structure_similarity` is a deterministic table-structure score over row count, column count, occupied cell coordinates, and rowspans/colspans when structured cell references are present. It is still not full HTML tree-edit TEDS. In the table-ledger v2 run it correctly shows no structure improvement over Surya because both systems already recover the simple grid structure perfectly. The candidate's validated gain is table cell-content accuracy with structure non-regression.
497
+
498
+ `table_teds_structure` and `table_teds_text` are deterministic TEDS-style ordered tree-edit scores over a table DOM made of `table`, `tr`, and `td` nodes. Structure mode compares row/cell tree shape plus `rowspan`/`colspan`; text mode also includes normalized cell text in `td` node labels. These are closer to paper-grade table metrics than the older token score, but they are still internal deterministic metrics, not a substitute for running an accepted public benchmark implementation when making broad SOTA claims.
499
+
500
+ After the span-inference postprocessor and tree metrics, the generated complex-table pack shows a structure gain over Surya (`table_structure_similarity` `1.000000` vs `0.875000`, `table_teds_structure` `1.000000` vs `0.904762`, `table_teds_text` `0.996250` vs `0.891667`, `table_exact_match` `0.925000` vs `0.000000`). This is evidence for the generated repeated-header pattern only. It does not replace a public or real scanned-table benchmark.
501
+
502
+ The public OmniDocBench smoke now also has a row-fragment normalization
503
+ candidate. It improves public table metrics over Surya on the same 5-page smoke
504
+ (`table_cell_f1` `0.564335` vs `0.296264`, `table_teds_structure` `0.942836`
505
+ vs `0.740128`, `table_teds_text` `0.675633` vs `0.526327`) and fixes one
506
+ `16x5` fragmented table into the expected `3x5` shape. It is still
507
+ `not_validated`: the sample count is far below the configured public gates,
508
+ bootstrap CI lower bounds are `0.000000` on table metrics, and text CER is
509
+ slightly worse than Surya on the smoke (`0.481629` vs `0.480531`).
510
+
511
+ The larger public OmniDocBench `table40` run is more important. It reaches the
512
+ configured table-pair count (`40` public table pages) but still fails validation:
513
+ `table_cell_f1` improves in mean (`0.323122` vs `0.280062`) while losing more
514
+ pages than it wins (`7` wins, `9` losses, `24` ties), and the bootstrap CI low is
515
+ negative (`-0.019152`). `table_teds_structure` also has a positive mean
516
+ (`0.520959` vs `0.503603`) but CI low is negative (`-0.038381`). `table_teds_text`
517
+ does not clear the margin (`0.003609` improvement). The document CER gate is
518
+ also not met: only `40/100` required public document pages, and CER improvement
519
+ is only `0.000634` against a `0.005` margin.
520
+
521
+ The width-guarded v4 rerun improves the public table40 evidence but still does
522
+ not validate the full claim. Skipping table-grid normalization on tables wider
523
+ than `12` columns makes the table-cell F1 gate pass (`0.339319` vs `0.280062`,
524
+ CI low `0.001433`) and the table TEDS-structure gate pass (`0.545375` vs
525
+ `0.503603`, CI low `0.004790`). The text-sensitive table gate still fails:
526
+ `table_teds_text` improves in mean (`0.386013` vs `0.358057`) but CI low is
527
+ `-0.000220`. The public document CER gate also still fails because the slice has
528
+ only `40/100` required public document pages, despite a mean CER improvement of
529
+ `0.006096`.
530
+
531
+ On `2026-06-14`, the `table100` public run was launched on `k2` using
532
+ `scripts/run_public_table100_claim.py` and the v4 width-guard model card. The
533
+ first all-or-nothing launch stalled after three completed candidate rows. We
534
+ restarted with an isolated per-page runner, `--resume-existing`, and
535
+ `--sample-timeout-seconds 300`, preserving completed outputs and forcing future
536
+ stalls to become explicit error rows. The first measured timeout row was
537
+ `omnidocbench-00216-page-dca64e05-1ce6-49ee-857a-9b5d05a87357`. Do not cite this
538
+ run as a result until the validation artifact exists and has been inspected.
539
+
540
+ This validates a targeted table-ledger extraction claim, not full document SOTA. Full SOTA validation still needs broad document coverage, real scanned tables, merged-cell/header cases, true TEDS/TEDS-IOU, and public benchmark evidence.
541
+
542
+ ## Recognizer Hard-Failure Gate
543
+
544
+ The failed AWS G6 PaddleOCR recognizer produced a normalized `50`-sample
545
+ hard-eval slice:
546
+
547
+ ```text
548
+ data/splits-mixed-v1/eval-paddle-mixed-v2-cuda-hard50-normalized.jsonl
549
+ ```
550
+
551
+ Use this slice as a regression gate for the next trained recognizer branch. It
552
+ must remain held out; do not train on it.
553
+
554
+ Validation config:
555
+
556
+ ```text
557
+ configs/validation-paddle-hard50-tesseract.json
558
+ ```
559
+
560
+ Run it after benchmarking a candidate recognizer against Tesseract:
561
+
562
+ ```bash
563
+ ocrtech validate-claim \
564
+ --benchmark-report outputs/next-recognizer-hard50/benchmark/report.json \
565
+ --config configs/validation-paddle-hard50-tesseract.json \
566
+ --eval-manifest data/splits-mixed-v1/eval-paddle-mixed-v2-cuda-hard50-normalized.jsonl \
567
+ --out outputs/validation-next-recognizer-hard50
568
+ ```
569
+
570
+ This gate is intentionally framed as failure recovery:
571
+
572
+ - all `50` hard rows must be present
573
+ - the `38` English synthetic line-crop failures must not regress versus
574
+ Tesseract on CER or WER
575
+ - the `12` Nepali hard failures must improve WER over Tesseract by at least
576
+ `0.05`
577
+
578
+ Passing this gate alone is not a SOTA claim. It is a recognizer branch admission
579
+ test before rerunning the full mixed OCR eval, rendered document pack, and public
580
+ benchmark gates.
581
+
582
+ The first line-level Tesseract/Paddle composite run exposed a benchmark-lifecycle
583
+ bug, not a model result: every candidate row crashed with `exit_code=-11` because
584
+ PaddleOCR was initialized in the parent process and then inherited by forked
585
+ timeout workers. After changing timeout mode to initialize engines inside the
586
+ watchdog worker, the rerun completed all `100` rows:
587
+
588
+ ```text
589
+ outputs/experiment-line-script-paddle-hard50-v2/report.json
590
+ outputs/summary-line-script-paddle-hard50-v2/summary.md
591
+ outputs/validation-line-script-paddle-hard50-v2/validation.md
592
+ ```
593
+
594
+ Validation status: `not_validated`.
595
+
596
+ - hard-50 CER: candidate `0.276489`, Tesseract `0.120106`
597
+ - English line-crop CER: candidate `0.060429`, Tesseract `0.060429`
598
+ - English line-crop WER: candidate `0.132259`, Tesseract `0.132259`
599
+ - Nepali hard CER: candidate `0.960679`, Tesseract `0.309083`
600
+ - Nepali hard WER: candidate `1.000000`, Tesseract `0.711607`
601
+
602
+ Interpretation: line-level routing can preserve English by leaving English lines
603
+ on the Tesseract primary path, but the trained PaddleOCR secondary recognizer is
604
+ not admissible for Devanagari overrides.
605
+
606
+ ## Leakage Rules
607
+
608
+ Fatal leakage:
609
+
610
+ - same `sample_id`
611
+ - same image SHA-256
612
+ - same image path
613
+ - same image+text sample signature
614
+
615
+ Warning only:
616
+
617
+ - same text SHA-256
618
+
619
+ Text-only overlap is not fatal because short OCR labels repeat naturally across real corpora. It is still reported because repeated long labels may indicate bad dataset splitting.
620
+
621
+ ## External Benchmarks
622
+
623
+ Use public suites for broad claims and the internal Nepali/English pack for the actual target domain:
624
+
625
+ - OmniDocBench for general document parsing.
626
+ - olmOCR-bench for PDF-to-Markdown structure preservation.
627
+ - OCRBench / OCRBench v2 for VLM OCR coverage.
628
+ - Internal Nepali + English pages for Devanagari OCR, mixed-script pages, government forms, tables, captions, and noisy scans.
629
+
630
+ For table claims, cell exact match, `table_cell_f1`, and span-aware `table_structure_similarity` are v1 metrics. Add true TEDS/TEDS-IOU before claiming leadership on complex merged-cell tables.
631
+
632
+ ## Internal Eval Pack
633
+
634
+ See [EVAL_PACK.md](EVAL_PACK.md). The internal pack is necessary because public OCR/document benchmarks do not sufficiently cover Nepali, mixed Devanagari-English pages, and local document layouts.
635
+
636
+ ## Dataset Splitting
637
+
638
+ See [DATASETS.md](DATASETS.md). Do not manually split data when making benchmark claims; use `split-manifest` so duplicate sample signatures cannot cross train/eval boundaries.
examples/sample-page.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ नेपाल सरकार
2
+
3
+ नाम उमेर जिल्ला
4
+ राम ३२ काठमाडौं
5
+ Sita 29 Lalitpur
6
+
7
+ This is a mixed Nepali and English page.
languages/nepal-v1.json ADDED
@@ -0,0 +1,370 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "registry_id": "nepal-v1",
3
+ "description": "Nepal multilingual OCR target registry. Limbu/Sirijonga is the first non-Devanagari target; English is included for preservation but does not count toward the Nepal-language target.",
4
+ "minimum_nepal_language_count": 10,
5
+ "created_for": "PaddleOCR script-aware fine-tuning, synthesis, and real-document validation",
6
+ "claim_policy": "Synthetic data is allowed for training and ablations only. SOTA claims require held-out real/public validation artifacts.",
7
+ "languages": [
8
+ {
9
+ "id": "limbu",
10
+ "name": "Limbu / Yakthung",
11
+ "endonym": "ᤕᤠᤰᤌᤢᤱ",
12
+ "iso_639_3": "lif",
13
+ "priority": 1,
14
+ "counts_toward_nepal_language_target": true,
15
+ "first_target": true,
16
+ "scripts": [
17
+ {
18
+ "script": "Limbu / Sirijonga",
19
+ "primary": true,
20
+ "status": "encoded",
21
+ "unicode_ranges": ["U+1900..U+194F"],
22
+ "notes": "First non-Devanagari OCR target. Requires renderability and dictionary audits before training admission."
23
+ },
24
+ {
25
+ "script": "Devanagari",
26
+ "primary": false,
27
+ "status": "fallback_transcription",
28
+ "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"],
29
+ "notes": "Useful for parallel/transcribed resources, not the primary Limbu-script claim target."
30
+ }
31
+ ],
32
+ "font_candidates": [
33
+ {"name": "Noto Sans Limbu", "path": "/System/Library/Fonts/Supplemental/NotoSansLimbu-Regular.ttf", "status": "pending_render_audit"},
34
+ {"name": "local-ltk-fonts", "path": "/Users/cdjk/github/llm/ltk/fonts.zip", "status": "legacy_archive_pending_coverage_audit"}
35
+ ],
36
+ "paddle_dictionary": "data/dictionaries/limbu-sirijonga-v1.txt",
37
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_limbu_letters_and_signs"},
38
+ "corpora": [
39
+ {"name": "LTK Limbu resources", "path": "/Users/cdjk/github/llm/ltk/limbu", "host": "cdjk", "type": "local_verified_directory", "status": "schema_audit_pending"},
40
+ {"name": "Limdic dictionary", "path": "/Users/cdjk/github/llm/ltk/limbu/data/limdic_dictionary.json", "host": "cdjk", "type": "lexicon", "status": "exists_local"},
41
+ {"name": "Pangloss dictionary TSV", "path": "/Users/cdjk/github/llm/ltk/limbu/data/pangloss_dictionary.tsv", "host": "cdjk", "type": "lexicon", "status": "exists_local"}
42
+ ],
43
+ "validation_sources": [
44
+ {"name": "Gorkhapatra/Naya Nepal article images", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "pending_manual_label"},
45
+ {"name": "Gorkhapatra epaper PDFs", "path": "data/gorkhapatra-pack-smoke", "type": "real_document_validation_pack", "status": "pending_manual_label"}
46
+ ],
47
+ "gorkhapatra_language_labels": ["लिम्बु"],
48
+ "notes": "Primary research target for low-resource script-aware PaddleOCR adaptation."
49
+ },
50
+ {
51
+ "id": "nepali",
52
+ "name": "Nepali",
53
+ "endonym": "नेपाली",
54
+ "iso_639_3": "nep",
55
+ "priority": 2,
56
+ "counts_toward_nepal_language_target": true,
57
+ "first_target": false,
58
+ "scripts": [
59
+ {"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Established base script and regression-protection target."}
60
+ ],
61
+ "font_candidates": [
62
+ {"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
63
+ {"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
64
+ ],
65
+ "paddle_dictionary": "data/dictionaries/nepali-devanagari-v1.txt",
66
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
67
+ "corpora": [
68
+ {"name": "Nepali hunspell dictionary", "path": "/Users/cdjk/github/llm/ltk/ne_NP.dic", "host": "cdjk", "type": "lexicon", "status": "exists_local"}
69
+ ],
70
+ "validation_sources": [
71
+ {"name": "Gorkhapatra epaper PDFs", "path": "data/gorkhapatra-pack-smoke", "type": "real_document_validation_pack", "status": "pending_manual_label"}
72
+ ],
73
+ "gorkhapatra_language_labels": ["नेपाली"],
74
+ "notes": "Must not regress versus stock PaddleOCR/Tesseract/Surya baselines."
75
+ },
76
+ {
77
+ "id": "tamang",
78
+ "name": "Tamang",
79
+ "endonym": "तामाङ",
80
+ "iso_639_3": "taj",
81
+ "priority": 3,
82
+ "counts_toward_nepal_language_target": true,
83
+ "first_target": false,
84
+ "scripts": [
85
+ {"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari resources; dialect coverage must be tracked separately."}
86
+ ],
87
+ "font_candidates": [
88
+ {"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
89
+ {"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
90
+ ],
91
+ "paddle_dictionary": "data/dictionaries/tamang-devanagari-v1.txt",
92
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
93
+ "corpora": [
94
+ {"name": "LTK Tamang resources", "path": "/Users/cdjk/github/llm/ltk/tamang", "host": "cdjk", "type": "local_verified_directory", "status": "schema_audit_pending"},
95
+ {"name": "Western Tamang dictionary TSV", "path": "/Users/cdjk/github/llm/ltk/tamang/data/western_tamang_dictionary.tsv", "host": "cdjk", "type": "lexicon", "status": "exists_local"},
96
+ {"name": "NepTam80K train/test CSV", "path": "/Users/cdjk/github/llm/ltk/tamang/data", "host": "cdjk", "type": "parallel_or_synthetic_text", "status": "schema_audit_pending"}
97
+ ],
98
+ "validation_sources": [
99
+ {"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_audit_pending"}
100
+ ],
101
+ "gorkhapatra_language_labels": ["तामाङ"],
102
+ "notes": "High-priority Devanagari low-resource language with existing local resources."
103
+ },
104
+ {
105
+ "id": "magar",
106
+ "name": "Magar",
107
+ "endonym": "मगर",
108
+ "iso_639_3": "mrd",
109
+ "priority": 4,
110
+ "counts_toward_nepal_language_target": true,
111
+ "first_target": false,
112
+ "scripts": [
113
+ {"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
114
+ ],
115
+ "font_candidates": [
116
+ {"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
117
+ {"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
118
+ ],
119
+ "paddle_dictionary": "data/dictionaries/magar-devanagari-v1.txt",
120
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
121
+ "corpora": [
122
+ {"name": "LTK Magar resources", "path": "/Users/cdjk/github/llm/ltk/magar", "host": "cdjk", "type": "local_verified_directory", "status": "schema_audit_pending"},
123
+ {"name": "Western Magar dictionary TSV", "path": "/Users/cdjk/github/llm/ltk/magar/data/western_magar_dictionary.tsv", "host": "cdjk", "type": "lexicon", "status": "exists_local"}
124
+ ],
125
+ "validation_sources": [
126
+ {"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
127
+ ],
128
+ "gorkhapatra_language_labels": ["मगर"],
129
+ "notes": "Seen in Naya Nepal audit labels; needs manual reference pack."
130
+ },
131
+ {
132
+ "id": "wambule",
133
+ "name": "Wambule",
134
+ "endonym": "वाम्बुले",
135
+ "iso_639_3": "wme",
136
+ "priority": 5,
137
+ "counts_toward_nepal_language_target": true,
138
+ "first_target": false,
139
+ "scripts": [
140
+ {"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
141
+ ],
142
+ "font_candidates": [
143
+ {"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
144
+ {"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
145
+ ],
146
+ "paddle_dictionary": "data/dictionaries/wambule-devanagari-v1.txt",
147
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
148
+ "corpora": [
149
+ {"name": "Gorkhapatra/Naya Nepal Wambule articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
150
+ ],
151
+ "validation_sources": [
152
+ {"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
153
+ ],
154
+ "gorkhapatra_language_labels": ["वाम्बुले"],
155
+ "notes": "Real-source first; no local training corpus audited yet."
156
+ },
157
+ {
158
+ "id": "rana-tharu",
159
+ "name": "Rana Tharu",
160
+ "endonym": "राना थारु",
161
+ "iso_639_3": "thr",
162
+ "priority": 6,
163
+ "counts_toward_nepal_language_target": true,
164
+ "first_target": false,
165
+ "scripts": [
166
+ {"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
167
+ ],
168
+ "font_candidates": [
169
+ {"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
170
+ {"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
171
+ ],
172
+ "paddle_dictionary": "data/dictionaries/rana-tharu-devanagari-v1.txt",
173
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
174
+ "corpora": [
175
+ {"name": "Gorkhapatra/Naya Nepal Rana Tharu articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
176
+ ],
177
+ "validation_sources": [
178
+ {"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
179
+ ],
180
+ "gorkhapatra_language_labels": ["राना थारु", "थारु"],
181
+ "notes": "Track Rana Tharu separately from broader Tharu labels when references are created."
182
+ },
183
+ {
184
+ "id": "bantawa",
185
+ "name": "Bantawa / Rai",
186
+ "endonym": "बान्तावा",
187
+ "iso_639_3": "bap",
188
+ "priority": 7,
189
+ "counts_toward_nepal_language_target": true,
190
+ "first_target": false,
191
+ "scripts": [
192
+ {"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."},
193
+ {"script": "Kirat Rai", "primary": false, "status": "requires_unicode_font_audit", "unicode_ranges": ["U+16D40..U+16D7F"], "notes": "Potential future script target; do not train or claim until font/runtime support is verified."}
194
+ ],
195
+ "font_candidates": [
196
+ {"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
197
+ {"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
198
+ ],
199
+ "paddle_dictionary": "data/dictionaries/bantawa-devanagari-v1.txt",
200
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
201
+ "corpora": [
202
+ {"name": "Gorkhapatra/Naya Nepal Bantawa articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
203
+ ],
204
+ "validation_sources": [
205
+ {"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
206
+ ],
207
+ "gorkhapatra_language_labels": ["बान्तावा"],
208
+ "notes": "Rai-language coverage must stay language-specific; do not collapse all Rai languages into one metric."
209
+ },
210
+ {
211
+ "id": "kulung",
212
+ "name": "Kulung",
213
+ "endonym": "कुलुङ",
214
+ "iso_639_3": "kle",
215
+ "priority": 8,
216
+ "counts_toward_nepal_language_target": true,
217
+ "first_target": false,
218
+ "scripts": [
219
+ {"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
220
+ ],
221
+ "font_candidates": [
222
+ {"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
223
+ {"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
224
+ ],
225
+ "paddle_dictionary": "data/dictionaries/kulung-devanagari-v1.txt",
226
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
227
+ "corpora": [
228
+ {"name": "Gorkhapatra/Naya Nepal Kulung articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
229
+ ],
230
+ "validation_sources": [
231
+ {"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
232
+ ],
233
+ "gorkhapatra_language_labels": ["कुलुङ"],
234
+ "notes": "Real-source first; needs corpus expansion."
235
+ },
236
+ {
237
+ "id": "yakkha",
238
+ "name": "Yakkha",
239
+ "endonym": "याक्खा",
240
+ "iso_639_3": "ybh",
241
+ "priority": 9,
242
+ "counts_toward_nepal_language_target": true,
243
+ "first_target": false,
244
+ "scripts": [
245
+ {"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
246
+ ],
247
+ "font_candidates": [
248
+ {"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
249
+ {"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
250
+ ],
251
+ "paddle_dictionary": "data/dictionaries/yakkha-devanagari-v1.txt",
252
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
253
+ "corpora": [
254
+ {"name": "Gorkhapatra/Naya Nepal Yakkha articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
255
+ ],
256
+ "validation_sources": [
257
+ {"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
258
+ ],
259
+ "gorkhapatra_language_labels": ["याक्खा"],
260
+ "notes": "Real-source first; needs corpus expansion."
261
+ },
262
+ {
263
+ "id": "bhojpuri",
264
+ "name": "Bhojpuri",
265
+ "endonym": "भोजपुरी",
266
+ "iso_639_3": "bho",
267
+ "priority": 10,
268
+ "counts_toward_nepal_language_target": true,
269
+ "first_target": false,
270
+ "scripts": [
271
+ {"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."}
272
+ ],
273
+ "font_candidates": [
274
+ {"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
275
+ {"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
276
+ ],
277
+ "paddle_dictionary": "data/dictionaries/bhojpuri-devanagari-v1.txt",
278
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
279
+ "corpora": [
280
+ {"name": "Gorkhapatra/Naya Nepal Bhojpuri articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
281
+ ],
282
+ "validation_sources": [
283
+ {"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
284
+ ],
285
+ "gorkhapatra_language_labels": ["भोजपुरी"],
286
+ "notes": "Large-language baseline for Devanagari multilingual transfer."
287
+ },
288
+ {
289
+ "id": "santhali",
290
+ "name": "Santhali",
291
+ "endonym": "सन्थाली",
292
+ "iso_639_3": "sat",
293
+ "priority": 11,
294
+ "counts_toward_nepal_language_target": true,
295
+ "first_target": false,
296
+ "scripts": [
297
+ {"script": "Ol Chiki", "primary": true, "status": "encoded", "unicode_ranges": ["U+1C50..U+1C7F"], "notes": "Own-script target after Limbu; needs font/render audit."},
298
+ {"script": "Devanagari", "primary": false, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "May appear in Devanagari-transcribed sources."}
299
+ ],
300
+ "font_candidates": [
301
+ {"name": "Noto Sans Ol Chiki", "path": "data/fonts/NotoSansOlChiki-Regular.ttf", "status": "pending_render_audit"},
302
+ {"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
303
+ {"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
304
+ ],
305
+ "paddle_dictionary": "data/dictionaries/santhali-ol-chiki-v1.txt",
306
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_ol_chiki_letters"},
307
+ "corpora": [
308
+ {"name": "Gorkhapatra/Naya Nepal Santhali articles", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_text_pending", "status": "needs_extraction"}
309
+ ],
310
+ "validation_sources": [
311
+ {"name": "Gorkhapatra/Naya Nepal", "path": "outputs/gorkhapatra-source-audit", "type": "real_document_discovery", "status": "language_label_seen"}
312
+ ],
313
+ "gorkhapatra_language_labels": ["सन्थाली"],
314
+ "notes": "Second own-script candidate after Limbu if real rendered sources and fonts are available."
315
+ },
316
+ {
317
+ "id": "maithili",
318
+ "name": "Maithili",
319
+ "endonym": "मैथिली",
320
+ "iso_639_3": "mai",
321
+ "priority": 12,
322
+ "counts_toward_nepal_language_target": true,
323
+ "first_target": false,
324
+ "scripts": [
325
+ {"script": "Devanagari", "primary": true, "status": "encoded", "unicode_ranges": ["U+0900..U+097F", "U+A8E0..U+A8FF"], "notes": "Initial OCR route uses Devanagari."},
326
+ {"script": "Tirhuta", "primary": false, "status": "future_audit", "unicode_ranges": ["U+11480..U+114DF"], "notes": "Own-script support is a future target and not part of the v1 claim without real validation."}
327
+ ],
328
+ "font_candidates": [
329
+ {"name": "Noto Sans Devanagari", "path": "data/fonts/NotoSansDevanagari-VariableFont_wdth,wght.ttf", "status": "pending_render_audit"},
330
+ {"name": "Devanagari MT", "path": "/System/Library/Fonts/Supplemental/DevanagariMT.ttc", "status": "pending_render_audit"}
331
+ ],
332
+ "paddle_dictionary": "data/dictionaries/maithili-devanagari-v1.txt",
333
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_devanagari_graphemes"},
334
+ "corpora": [
335
+ {"name": "Maithili corpus discovery", "path": "TBD", "type": "external_corpus_pending", "status": "not_audited"}
336
+ ],
337
+ "validation_sources": [
338
+ {"name": "Gorkhapatra/Naya Nepal or public documents", "path": "TBD", "type": "real_document_discovery", "status": "not_audited"}
339
+ ],
340
+ "gorkhapatra_language_labels": ["मैथिली"],
341
+ "notes": "Included because it is a major Nepal language; needs source discovery before training."
342
+ },
343
+ {
344
+ "id": "english",
345
+ "name": "English",
346
+ "endonym": "English",
347
+ "iso_639_3": "eng",
348
+ "priority": 99,
349
+ "counts_toward_nepal_language_target": false,
350
+ "first_target": false,
351
+ "scripts": [
352
+ {"script": "Latin", "primary": true, "status": "encoded", "unicode_ranges": ["U+0000..U+007F", "U+0080..U+00FF"], "notes": "Regression-protection target only."}
353
+ ],
354
+ "font_candidates": [
355
+ {"name": "Arial Unicode MS", "path": "/Library/Fonts/Arial Unicode.ttf", "status": "pending_render_audit"},
356
+ {"name": "Helvetica", "path": "/System/Library/Fonts/Helvetica.ttc", "status": "pending_render_audit"}
357
+ ],
358
+ "paddle_dictionary": "data/dictionaries/english-latin-v1.txt",
359
+ "normalization": {"unicode_form": "NFC", "policy": "preserve_ascii_and_common_latin"},
360
+ "corpora": [
361
+ {"name": "NVIDIA OCR Synthetic Multilingual English subset", "path": "huggingface:nvidia/OCR-Synthetic-Multilingual-v1", "type": "synthetic_line_crops", "status": "schema_audit_required_before_use"}
362
+ ],
363
+ "validation_sources": [
364
+ {"name": "Stock English OCR eval subset", "path": "TBD", "type": "regression_eval", "status": "not_audited"}
365
+ ],
366
+ "gorkhapatra_language_labels": [],
367
+ "notes": "Included to protect English accuracy in mixed documents; not counted as a Nepal-language target."
368
+ }
369
+ ]
370
+ }
pyproject.toml ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [build-system]
2
+ requires = ["setuptools>=69", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "ocr-tech"
7
+ version = "0.1.0"
8
+ description = "Hybrid Nepali + English OCR and document-structure extraction toolkit."
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ license = { text = "Apache-2.0" }
12
+ authors = [
13
+ { name = "Ampixa" }
14
+ ]
15
+ dependencies = []
16
+
17
+ [project.optional-dependencies]
18
+ datasets = [
19
+ "datasets>=3.0.0",
20
+ "h5py>=3.11.0",
21
+ "huggingface-hub>=0.25.0",
22
+ "pillow>=10.0.0"
23
+ ]
24
+ eval = [
25
+ "pillow>=10.0.0"
26
+ ]
27
+ engines = [
28
+ "paddleocr>=3.0.0",
29
+ "pillow>=10.0.0"
30
+ ]
31
+ surya = [
32
+ "surya-ocr>=0.20.0",
33
+ "pillow>=10.0.0"
34
+ ]
35
+ hf-recognizer = [
36
+ "accelerate>=1.1.0",
37
+ "protobuf>=5.28.0",
38
+ "sentencepiece>=0.2.0",
39
+ "transformers>=4.57.0",
40
+ "torch>=2.5.0",
41
+ "pillow>=10.0.0"
42
+ ]
43
+ vlm = [
44
+ "transformers>=4.50.0",
45
+ "torch>=2.5.0",
46
+ "pillow>=10.0.0"
47
+ ]
48
+ dev = [
49
+ "pytest>=8.0.0"
50
+ ]
51
+
52
+ [project.scripts]
53
+ ocrtech = "ocrtech.cli:main"
54
+
55
+ [tool.setuptools.packages.find]
56
+ where = ["src"]
57
+
58
+ [tool.pytest.ini_options]
59
+ pythonpath = ["src"]
60
+ testpaths = ["tests"]
61
+ addopts = "-q"
runs/corrector-cfilt/README.md ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # Corrector Training Recipe
2
+
3
+ Pairs: `runs/corrector-cfilt/correction_pairs.jsonl`
4
+
5
+ Use these noisy-clean pairs with a compact text-to-text model. Keep Nepali and English validation sets separate.
runs/corrector-cfilt/correction_pairs.jsonl ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {"noisy_text": "मधुमेहले विश्वमा एउटा महामारीको रूप लिइसकेको छ तथा यसका धेरै प्रकारका सामाजिक अनि आर्थिक दुष्परिणाम पनि अधि आइरहेका छन्‌।", "clean_text": "मधुमेहले विश्‍वमा एउटा महामारीको रूप लिइसकेको छ तथा यसका धेरै प्रकारका सामाजिक अनि आर्थिक दुष्‍परिणाम पनि अघि आइरहेका छन्।", "font": "Sura-Regular", "dataset": "cfilt/RoundTripOCR-nepali", "split": "train"}
2
+ {"noisy_text": "\"यस वठीले मुद्ु अनि ढिमागलाई शक्ति ढिने खाले, शुक्राणु अल्पता तथा शरीरमा स्फूर्ति बढाउने अनि वाजीकारक छ।\"", "clean_text": "\"यस वटीले मुटु अनि दिमागलाई शक्ति दिने खाले, शुक्राणु अल्पता तथा शरीरमा स्फूर्ति बढाउने अनि वाजीकारक छ।\"", "font": "Arya-Regular", "dataset": "cfilt/RoundTripOCR-nepali", "split": "train"}
3
+ {"noisy_text": "गाइडले हामीलाई सिंहका खुद्दाको चिह देखाए।", "clean_text": "गाइडले हामीलाई सिंहका खुट्टाको चिह्न देखाए।", "font": "Sanskrit2003", "dataset": "cfilt/RoundTripOCR-nepali", "split": "train"}
4
+ {"noisy_text": "\"भोजन, स्वास्थ्य, सौन्दर्य अनि धार्मिक सबै मामित्तामा यसको जवाफ छेैन|\"", "clean_text": "\"भोजन, स्वास्थ्य, सौन्दर्य अनि धार्मिक सबै मामिलामा यसको जवाफ छैन।\"", "font": "Asar-Regular", "dataset": "cfilt/RoundTripOCR-nepali", "split": "train"}
5
+ {"noisy_text": "\"डायबिटिजले तपाईँका आँखाका रेटिनालाई क्षति पुयाउन सक्‍्छ, जसबाट ओपन हुनसक्छा'", "clean_text": "\"डायबिटिजले तपाईँका आँखाका रेटिनालाई क्षति पुर्‍याउन सक्छ, जसबाट अन्धोपन हुनसक्छ।\"", "font": "Halant-Regular", "dataset": "cfilt/RoundTripOCR-nepali", "split": "train"}
runs/corrector-cfilt/corrector_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": "ocr_text_correction",
3
+ "pairs": "runs/corrector-cfilt/correction_pairs.jsonl",
4
+ "recommended_model_family": "compact seq2seq or encoder-decoder text corrector",
5
+ "metrics": [
6
+ "cer",
7
+ "wer",
8
+ "exact_line_accuracy"
9
+ ]
10
+ }
runs/corrector-mixed-v1-rebalanced/tesseract_pairs.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/hf-corrector-mixed-v1-rebalanced/README.md ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Hugging Face Text Corrector Training Recipe
2
+
3
+ Base model: `google/byt5-small`
4
+ Pairs source: `runs/hf-corrector-mixed-v1-rebalanced/correction_pairs.jsonl` (1700 pairs)
5
+ Train pairs: `runs/hf-corrector-mixed-v1-rebalanced/train_pairs.jsonl` (1530 pairs)
6
+ Eval pairs: `runs/hf-corrector-mixed-v1-rebalanced/eval_pairs.jsonl` (170 pairs)
7
+
8
+ Install the backend dependencies first:
9
+
10
+ ```bash
11
+ python -m pip install -e '.[hf-recognizer]'
12
+ ```
13
+
14
+ Run in-process:
15
+
16
+ ```bash
17
+ ocrtech train-corrector --backend hf-seq2seq --pairs-source runs/hf-corrector-mixed-v1-rebalanced/correction_pairs.jsonl --out runs/hf-corrector-mixed-v1-rebalanced --run
18
+ ```
runs/hf-corrector-mixed-v1-rebalanced/correction_pairs.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/hf-corrector-mixed-v1-rebalanced/corrector_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": "ocr_text_correction",
3
+ "pairs": "runs/hf-corrector-mixed-v1-rebalanced/correction_pairs.jsonl",
4
+ "recommended_model_family": "compact seq2seq or encoder-decoder text corrector",
5
+ "metrics": [
6
+ "cer",
7
+ "wer",
8
+ "exact_line_accuracy"
9
+ ]
10
+ }
runs/hf-corrector-mixed-v1-rebalanced/eval_pairs.jsonl ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"noisy_text": "onto a gray plaid", "clean_text": "onto a gray plaid"}
2
+ {"noisy_text": "ae सादर अनुरोध eae, मुख्य aay बन", "clean_text": "सादर अनुरोध गर्दछु, मुख्य सचिव"}
3
+ {"noisy_text": ": क्षेत्रीय सहकार्‌य समूमेलनमा भांग after", "clean_text": "क्षेत्रीय सहकार्य सम्मेलनमा भाग लिनुहुन"}
4
+ {"noisy_text": "_ aturn for the worse with the", "clean_text": "- a turn for the worse with the"}
5
+ {"noisy_text": "——e छु od", "clean_text": "३ डिडि ढि"}
6
+ {"noisy_text": "has its", "clean_text": "has its"}
7
+ {"noisy_text": "known mutations and genetic", "clean_text": "known mutations and genetic"}
8
+ {"noisy_text": "a losing! battic", "clean_text": "fighting a (losing) battle"}
9
+ {"noisy_text": "९९७६ 1810 the wild, the bircle do nut", "clean_text": "had in the wild, the birds do not"}
10
+ {"noisy_text": "PANELS CAN MAKE A HUGE", "clean_text": "panels can make a huge"}
11
+ {"noisy_text": "", "clean_text": "monitoring of solar panel"}
12
+ {"noisy_text": "“सादर अनुरोध tea, प्रशासन safe : .", "clean_text": "सादर अनुरोध गर्दछु, प्रशासन अधिकृत"}
13
+ {"noisy_text": "जी: १०7० EEE Seale 7?\nee5\t1\t1\t1\t2\t2\t514\t131\t119\t113\t85.775848\tjot5\t1\t1\t1\t2\t3\t691\t89\t280\t146\t60.418049\t905\t1\t1\t1\t2\t4\t950\t123\t5\t61\t52.338493\t|5\t1\t1\t1\t2\t5\t987\t55\t195\t155\t64.715851\tBut5\t1\t1\t1\t2\t6\t1225\t86\t239\t89\t30.467482\t520685\t1\t1\t1\t2\t7\t1508\t51\t229\t102\t25.415092\twy5\t1\t1\t1\t2\t8\t1758\t81\t30\t37\t73.042397\t2०25\t1\t1\t1\t2\t9\t1795\t21\t211\t101\t35.075745\t(Wee", "clean_text": "मिति : २०१० वर्षको\nसम्यक गर्ने अन्तिम मिति : २०६४ चैत्र २० गतेसम्म"}
14
+ {"noisy_text": "prices. REAL SEO is your advantage.", "clean_text": "prices. REAL SEO is your advantage,"}
15
+ {"noisy_text": "“mare सहयोग: तथा कृषमता-अभविद्धाकिार्‌यक्‌रम", "clean_text": "प्राविधिक सहयोग तथा क्षमता अभिवृद्धि कार्यक्रम"}
16
+ {"noisy_text": "ल्कस्त्ग पाको IBS,", "clean_text": "कस्तो मैं दिऊ"}
17
+ {"noisy_text": "t\nh\n=", "clean_text": "themed"}
18
+ {"noisy_text": "रसूतुत वषियंमा आवश्यक PATI TTT लाँगपिठाइएको '", "clean_text": "प्रस्तुत विषयमा आवश्यक कार्यान्वयनका लागि पठाइएको छ"}
19
+ {"noisy_text": "odor of mold. Basements", "clean_text": "odor of mold. Basements"}
20
+ {"noisy_text": "cpp वबिरंण तथा बाँडफाँड समूबन्‌धमा ॥", "clean_text": "कार्य विवरण तथा बाँडफाँड सम्बन्धमा"}
21
+ {"noisy_text": "she learnt she had indigenous ancestors", "clean_text": "she learnt she had indigenous ancestors"}
22
+ {"noisy_text": "t\nh\ni\nr\ni\na\nत |\n|\nf]\na\n३\nUl}", "clean_text": "their manufacture."}
23
+ {"noisy_text": "Dm wore oS about, that hover!", "clean_text": "I’m worried about, that haven’t"}
24
+ {"noisy_text": "", "clean_text": "चलाउने नसकिएकाले"}
25
+ {"noisy_text": "€3326.02", "clean_text": "€3326.02"}
26
+ {"noisy_text": "2551S oe", "clean_text": "£5513.53"}
27
+ {"noisy_text": "€0101६61£:20100£ १ © 9%", "clean_text": "content to put down their roots"}
28
+ {"noisy_text": "handsome red fox", "clean_text": "handsome red fox"}
29
+ {"noisy_text": "oe मति22071. फागुन 21 गते 1414", "clean_text": "मितिः 2071 फागुन 21 गते"}
30
+ {"noisy_text": "टर ६८", "clean_text": "बन्धछे"}
31
+ {"noisy_text": "fe ir", "clean_text": "पुजि हो"}
32
+ {"noisy_text": "oneself. Usually in a", "clean_text": "oneself. Usually in a"}
33
+ {"noisy_text": "located about 244.6 million light-years away, This galaxy 0858 bright AGN, resulting .-......... 0.1", "clean_text": "located about 244.6 million light-years away. This galaxy has a bright AGN, resulting 6.1"}
34
+ {"noisy_text": "3790.57", "clean_text": "3790.57"}
35
+ {"noisy_text": "fantasy football", "clean_text": "fantasy football"}
36
+ {"noisy_text": "Yili wore ee Tun 000.\nora Bpeteta", "clean_text": "विद्यार्थी हिस्से समयमे"}
37
+ {"noisy_text": "from Google Ads to save time and get", "clean_text": "from Google Ads to save time and get"}
38
+ {"noisy_text": "> aro पपारटैलगा , Fa OCF २\npe 27?", "clean_text": "प्रश्नो परिक्षण, नित्य र"}
39
+ {"noisy_text": "ude", "clean_text": "यस"}
40
+ {"noisy_text": "~ yout nase There are.\n‘nd; Feviews yet,\n४०६ Ocak your manivans weed products onkine and provide proof that yeulrs", "clean_text": "- your needs. There are no reviews yet. Order your marijuana & weed products online and provide proof that you’re"}
41
+ {"noisy_text": "“ सादर अनुरोध गर्दछु, मनूत्‌रालय प्रवकृता...", "clean_text": "सादर अनुरोध गर्दछु, मन्त्रालय प्रवक्ता"}
42
+ {"noisy_text": "The hope was that", "clean_text": "The hope was that"}
43
+ {"noisy_text": "“ सादर अनुरोध गर्दछु, मनूत्‌रालय प्रवकृता...", "clean_text": "सादर अनुरोध गर्दछु, मन्त्रालय प्रवक्ता"}
44
+ {"noisy_text": "party on the issue, it is not a surprise", "clean_text": "party on the issue, it is not a surprise"}
45
+ {"noisy_text": "corresponds directly to the amount of dimming. Although scientists have long. . ---- 10", "clean_text": "corresponds directly to the amount of dimming. Although scientists have long 10"}
46
+ {"noisy_text": "", "clean_text": "damp and rainy, but oh my, the little gardens are"}
47
+ {"noisy_text": "Ol wtings.", "clean_text": "offerings."}
48
+ {"noisy_text": "is needle felted", "clean_text": "is needle felted"}
49
+ {"noisy_text": "", "clean_text": "चलाउने नसकिएकाले"}
50
+ {"noisy_text": "needs. if you're", "clean_text": "needs. if you’re"}
51
+ {"noisy_text": "did not respond. Please", "clean_text": "did not respond. Please"}
52
+ {"noisy_text": "upcycled off-white", "clean_text": "upcycled off-white"}
53
+ {"noisy_text": "शूरी पूरेशासने अधरकित; TATA तथा जनसंख्या मनूत्रालय", "clean_text": "श्री प्रशासन अधिकृत, स्वास्थ्य तथा जनसंख्या मन्त्रालय"}
54
+ {"noisy_text": "of three or four. We", "clean_text": "of three or four. We"}
55
+ {"noisy_text": "चा0कवितंशमा कस्ता मानिसको ryt a", "clean_text": "ग कवितांशमा कस्ता मानिसको शिर उचो हुन्छ भनि-"}
56
+ {"noisy_text": "हेट दर cae) त्यो oer oa) TOY ढ्किक्कि 2", "clean_text": "हेदै रहल लागदो त्यो नरम कमरे भातो भै दिऊ कि?"}
57
+ {"noisy_text": "photos, |<) an 80 तालले ९९,", "clean_text": "photos, Linda, and so glad to"}
58
+ {"noisy_text": "8344", "clean_text": "8344"}
59
+ {"noisy_text": "obvious reasons.", "clean_text": "obvious reasons."}
60
+ {"noisy_text": "‘oe feeders ondhave. -", "clean_text": "the feeders and have been"}
61
+ {"noisy_text": "|", "clean_text": "nature's goodness. The"}
62
+ {"noisy_text": "रस्‌तुत बंषियमा आवशूयक कार्‌यानूवयनका लाग पिठाइएको :", "clean_text": "प्रस्तुत विषयमा आवश्यक कार्यान्वयनका लागि पठाइएको छ"}
63
+ {"noisy_text": "not a type of belief", "clean_text": "not a type of belief"}
64
+ {"noisy_text": "we ; : “पत्र संख्या: 6315/075/076 20]", "clean_text": "पत्र संख्याः 5315/०75/०76"}
65
+ {"noisy_text": "checked at least two to three", "clean_text": "checked at least two to three"}
66
+ {"noisy_text": "oe कार्‌य afar तथा बाँडफीँड संमूबन्‌धमा: 1.7.", "clean_text": "कार्य विवरण तथा बाँडफाँड सम्बन्धमा"}
67
+ {"noisy_text": "or huzzah. Continue", "clean_text": "or huzzah. Continue"}
68
+ {"noisy_text": "Day 10:11 112. andI’ve", "clean_text": "Day around her and I’ve"}
69
+ {"noisy_text": "[नी SP", "clean_text": "नेपाली हस्तलेखन"}
70
+ {"noisy_text": "“सादर अनुरोध गर्दछु, महोनरिदिशक ..", "clean_text": "सादर अनुरोध गर्दछु, महानिर्देशक"}
71
+ {"noisy_text": "“नेपाल सरकारको afters ब्मोजमि तोकणिको ata", "clean_text": "नेपाल सरकारको निर्णय बमोजिम तोकिएको मिति"}
72
+ {"noisy_text": "21॥3न्ट्", "clean_text": "गारिन्दे"}
73
+ {"noisy_text": "[यवस्‌था, सहकारी तथा abet नविरणमन्‌त्रालय ALS, §", "clean_text": "भूमि व्यवस्था, सहकारी तथा गरिबी निवारण मन्त्रालय सुर्खेत, सुर्खेत"}
74
+ {"noisy_text": "11 est", "clean_text": "third of"}
75
+ {"noisy_text": "In", "clean_text": "\"In"}
76
+ {"noisy_text": "0714\n2101", "clean_text": "गर्न"}
77
+ {"noisy_text": "गेम ।वैनाग्रव HIEMIGE", "clean_text": "तेज विनायक माध्यमिक विद्यालय"}
78
+ {"noisy_text": "SB 11000000 DRODUCTION “10-1८01100110101010007 117 000? RODUMES 0401", "clean_text": "5. how the production and computation of hash products will"}
79
+ {"noisy_text": "( aft 2071 कातूतकि 19 गते...", "clean_text": "मितिः 2071 कात्तिक 19 गते"}
80
+ {"noisy_text": "you know. Microsoft Advertising has", "clean_text": "you know, Microsoft Advertising has"}
81
+ {"noisy_text": "ce tpt Feat TTS जनकपुर, घनुषा ७...", "clean_text": "प्रहरी प्रधान कार्यालय जनकपुर, धनुषा"}
82
+ {"noisy_text": "[. कृषेतूरीय सहकार्य समूमेलनमा भाग: लनिहन he", "clean_text": "क्षेत्रीय सहकार्य सम्मेलनमा भाग लिनुहुन"}
83
+ {"noisy_text": "“:. कुर्य afer तथा बाँडफाँड Hea 0", "clean_text": "कार्य विवरण तथा बाँडफाँड सम्बन्धमा"}
84
+ {"noisy_text": "Watching through", "clean_text": "Watching through"}
85
+ {"noisy_text": "र\nr\n०\nm", "clean_text": "from"}
86
+ {"noisy_text": "Na, ae ‘_\n(7/॥ >! ६", "clean_text": "लागी ६"}
87
+ {"noisy_text": "रस्‌तुत वषियमा आँवशूयक कार्यान्वयनका लाग पिठाइएकी :", "clean_text": "प्रस्तुत विषयमा आवश्यक कार्यान्वयनका लागि पठाइएको छ"}
88
+ {"noisy_text": "it’s important to", "clean_text": "it’s important to"}
89
+ {"noisy_text": "eo fan et न्युन तै ढिक र", "clean_text": "सबैले शिक्षा सजाउने सुत्र में दिऊं?"}
90
+ {"noisy_text": "", "clean_text": "you relax tend to be"}
91
+ {"noisy_text": "showers make bathrooms one of the most likely places for", "clean_text": "showers make bathrooms one of the most likely places for"}
92
+ {"noisy_text": "carbon", "clean_text": "carbon"}
93
+ {"noisy_text": "| SCHELL गरौ Hal THe", "clean_text": "होस अध्ययन गर्ने बानीले हामीलाई"}
94
+ {"noisy_text": "think ves al! feelso good jv:a!", "clean_text": "think we all feel so good just"}
95
+ {"noisy_text": "अफपसामबी Ace", "clean_text": "भ्रमणसम्बन्धी सूचना"}
96
+ {"noisy_text": "|\n॥\n\\\nb\ny\nt\nE\nu\nSs\n€\n0\nf\nt\nh\ne\nWwW\n0\nit\nd", "clean_text": "caused by the use of the word."}
97
+ {"noisy_text": "। 1111 0800 901", "clean_text": "your mom's backyard in your"}
98
+ {"noisy_text": "| नेपाल सरकारको नरिणय बमोजमि तोकषिको मति", "clean_text": "नेपाल सरकारको निर्णय बमोजिम तोकिएको मिति"}
99
+ {"noisy_text": "re wy ries\nNidk!s!", "clean_text": "काठमाडौँ"}
100
+ {"noisy_text": "TCG तथा जनसंख्या मन्‌त्रालय काठमाडौँ महानगरपालति", "clean_text": "स्वास्थ्य तथा जनसंख्या मन्त्रालय काठमाडौँ महानगरपालिका"}
101
+ {"noisy_text": "", "clean_text": "their powersso that they may"}
102
+ {"noisy_text": "“mare सहयोग: तथा कृषमता-अभविद्धाकिार्‌यक्‌रम", "clean_text": "प्राविधिक सहयोग तथा क्षमता अभिवृद्धि कार्यक्रम"}
103
+ {"noisy_text": "You will", "clean_text": "You will"}
104
+ {"noisy_text": "watching for his", "clean_text": "watching for his"}
105
+ {"noisy_text": "verifies you are of legal age to", "clean_text": "verifies you are of legal age to"}
106
+ {"noisy_text": "म तिम्रो erat) aa झै किक्न।", "clean_text": "म तिम्रो लौ कस्तो मैले।"}
107
+ {"noisy_text": "He's a reigning", "clean_text": "He's a reigning"}
108
+ {"noisy_text": "टर ६८", "clean_text": "बन्धछे"}
109
+ {"noisy_text": "वि", "clean_text": "and get rid of"}
110
+ {"noisy_text": "“जक", "clean_text": "ङ"}
111
+ {"noisy_text": "fry = the Covic. 1३ took awhii«", "clean_text": "from the Covid. It took a while"}
112
+ {"noisy_text": "at Northeast Advertising Solutions is to get your business in", "clean_text": "at Northeast Advertising Solutions is to get your business in"}
113
+ {"noisy_text": "i ee ee ee Oe ee WEF", "clean_text": "...यसले अनुसन्धान अधिकृतले मुद्दाको खोजतलास) कार्यको २०६० को\nवहाँ सहिदिको प्रतिवेदन ?"}
114
+ {"noisy_text": "| COB ICR गनर PIE ET", "clean_text": "तरीकाले उपभोग गर्ने सिफारिस । हानि"}
115
+ {"noisy_text": "“ सादर अनुरोध गर्दछु, मनूत्‌रा���य प्रवकृता...", "clean_text": "सादर अनुरोध गर्दछु, मन्त्रालय प्रवक्ता"}
116
+ {"noisy_text": "grape is notorious for its genetic", "clean_text": "grape is notorious for its genetic"}
117
+ {"noisy_text": "organic Google Listings! We combine the power of YouTube", "clean_text": "organic Google Listings! We combine the power of YouTube"}
118
+ {"noisy_text": "", "clean_text": "१४ १७ ५८ १२ १८ १३ -९ (अ)"}
119
+ {"noisy_text": "from a specialist if your unit is", "clean_text": "from a specialist if your unit is"}
120
+ {"noisy_text": "absorb", "clean_text": "absorb"}
121
+ {"noisy_text": "oo मति? 2077 area 14 गते... 7", "clean_text": "मितिः 2077 कात्तिक 14 गते"}
122
+ {"noisy_text": "ripping you off. We all have a part to play in creating", "clean_text": "ripping you off. We all have a part to play in creating"}
123
+ {"noisy_text": "and offers", "clean_text": "and offers"}
124
+ {"noisy_text": "own moment of “enlightenment”", "clean_text": "own moment of “enlightenment”"}
125
+ {"noisy_text": "wae वपियमा ओवशूयक कार्यानुवंयनेका arf eres", "clean_text": "प्रस्तुत विषयमा आवश्यक कार्यान्वयनका लागि पठाइएको छ"}
126
+ {"noisy_text": "surgeon.</p> <p>Earlier", "clean_text": "surgeon.</p> <p>Earlier"}
127
+ {"noisy_text": "technology Mik thinks we all need to read", "clean_text": "technology Mik thinks we all need to read"}
128
+ {"noisy_text": "——e छु od", "clean_text": "३ डिडि ढि"}
129
+ {"noisy_text": "~ ret ger Ree I झै किक किट", "clean_text": "बाटोमा दशा सुधार्ने गुलाम नै दिऊँ कि?"}
130
+ {"noisy_text": "“mare सहयोग: तथा कृषमता-अभविद्धाकिार्‌यक्‌रम", "clean_text": "प्राविधिक सहयोग तथा क्षमता अभिवृद्धि कार्यक्रम"}
131
+ {"noisy_text": "Pit -2068] 9/92", "clean_text": "मिति : २०६५/९२/९३"}
132
+ {"noisy_text": "media to promote", "clean_text": "media to promote"}
133
+ {"noisy_text": "jowers.creusing just 60:5४", "clean_text": "flowers,growing just out"}
134
+ {"noisy_text": "eo fan et न्युन तै ढिक र", "clean_text": "सबैले शिक्षा सजाउने सुत्र में दिऊं?"}
135
+ {"noisy_text": "experiences of", "clean_text": "experiences of"}
136
+ {"noisy_text": "pee en | / चन 6962/877/078 ee 7", "clean_text": "च.नं. 6962/०77/०78"}
137
+ {"noisy_text": "again, the price won't change. It does not matter", "clean_text": "again, the price won't change. It does not matter"}
138
+ {"noisy_text": "लपल chine ont the birdhouse.", "clean_text": "checking out the birdhouse."}
139
+ {"noisy_text": "नोम्पड VBP ut के", "clean_text": "माधव महा. च्याल्ने"}
140
+ {"noisy_text": "years. She assisted", "clean_text": "years. She assisted"}
141
+ {"noisy_text": "[यवस्‌था, सहकारी तथा abet नविरणमन्‌त्रालय ALS, §", "clean_text": "भूमि व्यवस्था, सहकारी तथा गरिबी निवारण मन्त्रालय सुर्खेत, सुर्खेत"}
142
+ {"noisy_text": "", "clean_text": "म कस्तो भै दिऊ?"}
143
+ {"noisy_text": "‘hearing things", "clean_text": "‘hearing things"}
144
+ {"noisy_text": "[र", "clean_text": "Pot."}
145
+ {"noisy_text": "१-संचवि, qatar, सहकारी तथा aft ATCT at", "clean_text": "श्री उप-सचिव, भूमि व्यवस्था, सहकारी तथा गरिबी निवारण मन्त्रालय"}
146
+ {"noisy_text": "", "clean_text": "म कस्तो भै दिऊ?"}
147
+ {"noisy_text": "“ सादर अनुरोध गर्दछु, मनूत्‌रालय प्रवकृता...", "clean_text": "सादर अनुरोध गर्दछु, मन्त्रालय प्रवक्ता"}
148
+ {"noisy_text": "द्र", "clean_text": "गुखु"}
149
+ {"noisy_text": "need new pillows. i want the cariloha gel pillow. | learned that they have bamboo mattresses! | would love ta", "clean_text": "need new pillows. i want the cariloha gel pillow. I learned that they have bamboo mattresses! I would love to"}
150
+ {"noisy_text": "— ee 7110.", "clean_text": "२ ७५० १५० १० १"}
151
+ {"noisy_text": "री महानरिदिशक, भौतकि qatar तथा aaa Aa", "clean_text": "श्री महानिर्देशक, भौतिक पूर्वाधार तथा यातायात मन्त्रालय"}
152
+ {"noisy_text": "कार्य बविरण तथा बाँडफाँड समूबन्‌धमा ॥", "clean_text": "कार्य विवरण तथा बाँडफ���ँड सम्बन्धमा"}
153
+ {"noisy_text": "means", "clean_text": "means"}
154
+ {"noisy_text": "mu value of such a reletionship is hard to quantify, the 1774 5/000511100 GENE", "clean_text": "the value of such a relationship is hard to quantify, the intangible benefits may be"}
155
+ {"noisy_text": "सैनकि मुख्यालय जङ्गी अड्डा जनकपुर, धनुषा", "clean_text": "सैनिक मुख्यालय जङ्गी अड्डा जनकपुर, धनुषा"}
156
+ {"noisy_text": "fe ir", "clean_text": "पुजि हो"}
157
+ {"noisy_text": "and", "clean_text": "and"}
158
+ {"noisy_text": "", "clean_text": "समर्थी तथा"}
159
+ {"noisy_text": "fe ir", "clean_text": "पुजि हो"}
160
+ {"noisy_text": "Se AS", "clean_text": "गे१११"}
161
+ {"noisy_text": "६ ast. 196/978/979", "clean_text": "च.नं. 196/०78/०79"}
162
+ {"noisy_text": "ce tpt Feat TTS जनकपुर, घनुषा ७...", "clean_text": "प्रहरी प्रधान कार्यालय जनकपुर, धनुषा"}
163
+ {"noisy_text": "", "clean_text": "Overhanging trees or high-rise"}
164
+ {"noisy_text": "mind, listening to", "clean_text": "mind, listening to"}
165
+ {"noisy_text": "warded बषियंमा HATH AAA AT लागपिडाइएको :", "clean_text": "प्रस्तुत विषयमा आवश्यक कार्यान्वयनका लागि पठाइएको छ"}
166
+ {"noisy_text": "clones we know of. However, ampslographers believe that the mutation rate for Pinot is actually on par with", "clean_text": "clones we know of. However, ampelographers believe that the mutation rate for Pinot is actually on par with"}
167
+ {"noisy_text": "e fe\nपद 4D", "clean_text": "1 1\n8 6\n९ ८"}
168
+ {"noisy_text": "e-commerce traffic is referred by search.” BI Intelligence Start", "clean_text": "e-commerce traffic is referred by search.” BI Intelligence Start"}
169
+ {"noisy_text": "jot", "clean_text": "गर्ने"}
170
+ {"noisy_text": "my selection of", "clean_text": "my selection of"}
runs/hf-corrector-mixed-v1-rebalanced/final-model/added_tokens.json ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<extra_id_0>": 259,
3
+ "<extra_id_100>": 359,
4
+ "<extra_id_101>": 360,
5
+ "<extra_id_102>": 361,
6
+ "<extra_id_103>": 362,
7
+ "<extra_id_104>": 363,
8
+ "<extra_id_105>": 364,
9
+ "<extra_id_106>": 365,
10
+ "<extra_id_107>": 366,
11
+ "<extra_id_108>": 367,
12
+ "<extra_id_109>": 368,
13
+ "<extra_id_10>": 269,
14
+ "<extra_id_110>": 369,
15
+ "<extra_id_111>": 370,
16
+ "<extra_id_112>": 371,
17
+ "<extra_id_113>": 372,
18
+ "<extra_id_114>": 373,
19
+ "<extra_id_115>": 374,
20
+ "<extra_id_116>": 375,
21
+ "<extra_id_117>": 376,
22
+ "<extra_id_118>": 377,
23
+ "<extra_id_119>": 378,
24
+ "<extra_id_11>": 270,
25
+ "<extra_id_120>": 379,
26
+ "<extra_id_121>": 380,
27
+ "<extra_id_122>": 381,
28
+ "<extra_id_123>": 382,
29
+ "<extra_id_124>": 383,
30
+ "<extra_id_12>": 271,
31
+ "<extra_id_13>": 272,
32
+ "<extra_id_14>": 273,
33
+ "<extra_id_15>": 274,
34
+ "<extra_id_16>": 275,
35
+ "<extra_id_17>": 276,
36
+ "<extra_id_18>": 277,
37
+ "<extra_id_19>": 278,
38
+ "<extra_id_1>": 260,
39
+ "<extra_id_20>": 279,
40
+ "<extra_id_21>": 280,
41
+ "<extra_id_22>": 281,
42
+ "<extra_id_23>": 282,
43
+ "<extra_id_24>": 283,
44
+ "<extra_id_25>": 284,
45
+ "<extra_id_26>": 285,
46
+ "<extra_id_27>": 286,
47
+ "<extra_id_28>": 287,
48
+ "<extra_id_29>": 288,
49
+ "<extra_id_2>": 261,
50
+ "<extra_id_30>": 289,
51
+ "<extra_id_31>": 290,
52
+ "<extra_id_32>": 291,
53
+ "<extra_id_33>": 292,
54
+ "<extra_id_34>": 293,
55
+ "<extra_id_35>": 294,
56
+ "<extra_id_36>": 295,
57
+ "<extra_id_37>": 296,
58
+ "<extra_id_38>": 297,
59
+ "<extra_id_39>": 298,
60
+ "<extra_id_3>": 262,
61
+ "<extra_id_40>": 299,
62
+ "<extra_id_41>": 300,
63
+ "<extra_id_42>": 301,
64
+ "<extra_id_43>": 302,
65
+ "<extra_id_44>": 303,
66
+ "<extra_id_45>": 304,
67
+ "<extra_id_46>": 305,
68
+ "<extra_id_47>": 306,
69
+ "<extra_id_48>": 307,
70
+ "<extra_id_49>": 308,
71
+ "<extra_id_4>": 263,
72
+ "<extra_id_50>": 309,
73
+ "<extra_id_51>": 310,
74
+ "<extra_id_52>": 311,
75
+ "<extra_id_53>": 312,
76
+ "<extra_id_54>": 313,
77
+ "<extra_id_55>": 314,
78
+ "<extra_id_56>": 315,
79
+ "<extra_id_57>": 316,
80
+ "<extra_id_58>": 317,
81
+ "<extra_id_59>": 318,
82
+ "<extra_id_5>": 264,
83
+ "<extra_id_60>": 319,
84
+ "<extra_id_61>": 320,
85
+ "<extra_id_62>": 321,
86
+ "<extra_id_63>": 322,
87
+ "<extra_id_64>": 323,
88
+ "<extra_id_65>": 324,
89
+ "<extra_id_66>": 325,
90
+ "<extra_id_67>": 326,
91
+ "<extra_id_68>": 327,
92
+ "<extra_id_69>": 328,
93
+ "<extra_id_6>": 265,
94
+ "<extra_id_70>": 329,
95
+ "<extra_id_71>": 330,
96
+ "<extra_id_72>": 331,
97
+ "<extra_id_73>": 332,
98
+ "<extra_id_74>": 333,
99
+ "<extra_id_75>": 334,
100
+ "<extra_id_76>": 335,
101
+ "<extra_id_77>": 336,
102
+ "<extra_id_78>": 337,
103
+ "<extra_id_79>": 338,
104
+ "<extra_id_7>": 266,
105
+ "<extra_id_80>": 339,
106
+ "<extra_id_81>": 340,
107
+ "<extra_id_82>": 341,
108
+ "<extra_id_83>": 342,
109
+ "<extra_id_84>": 343,
110
+ "<extra_id_85>": 344,
111
+ "<extra_id_86>": 345,
112
+ "<extra_id_87>": 346,
113
+ "<extra_id_88>": 347,
114
+ "<extra_id_89>": 348,
115
+ "<extra_id_8>": 267,
116
+ "<extra_id_90>": 349,
117
+ "<extra_id_91>": 350,
118
+ "<extra_id_92>": 351,
119
+ "<extra_id_93>": 352,
120
+ "<extra_id_94>": 353,
121
+ "<extra_id_95>": 354,
122
+ "<extra_id_96>": 355,
123
+ "<extra_id_97>": 356,
124
+ "<extra_id_98>": 357,
125
+ "<extra_id_99>": 358,
126
+ "<extra_id_9>": 268
127
+ }
runs/hf-corrector-mixed-v1-rebalanced/final-model/config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "T5ForConditionalGeneration"
4
+ ],
5
+ "classifier_dropout": 0.0,
6
+ "d_ff": 3584,
7
+ "d_kv": 64,
8
+ "d_model": 1472,
9
+ "decoder_start_token_id": 0,
10
+ "dense_act_fn": "gelu_new",
11
+ "dropout_rate": 0.1,
12
+ "dtype": "float32",
13
+ "eos_token_id": 1,
14
+ "feed_forward_proj": "gated-gelu",
15
+ "gradient_checkpointing": false,
16
+ "initializer_factor": 1.0,
17
+ "is_decoder": false,
18
+ "is_encoder_decoder": true,
19
+ "is_gated_act": true,
20
+ "layer_norm_epsilon": 1e-06,
21
+ "model_type": "t5",
22
+ "num_decoder_layers": 4,
23
+ "num_heads": 6,
24
+ "num_layers": 12,
25
+ "pad_token_id": 0,
26
+ "relative_attention_max_distance": 128,
27
+ "relative_attention_num_buckets": 32,
28
+ "scale_decoder_outputs": false,
29
+ "tie_word_embeddings": true,
30
+ "tokenizer_class": "ByT5Tokenizer",
31
+ "transformers_version": "5.12.0",
32
+ "use_cache": false,
33
+ "vocab_size": 384
34
+ }
runs/hf-corrector-mixed-v1-rebalanced/final-model/generation_config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": false,
3
+ "assistant_confidence_threshold": 0.4,
4
+ "assistant_lookbehind": 10,
5
+ "decoder_start_token_id": 0,
6
+ "diversity_penalty": 0.0,
7
+ "do_sample": false,
8
+ "early_stopping": false,
9
+ "encoder_no_repeat_ngram_size": 0,
10
+ "encoder_repetition_penalty": 1.0,
11
+ "eos_token_id": [
12
+ 1
13
+ ],
14
+ "epsilon_cutoff": 0.0,
15
+ "eta_cutoff": 0.0,
16
+ "length_penalty": 1.0,
17
+ "max_length": 256,
18
+ "min_length": 0,
19
+ "no_repeat_ngram_size": 0,
20
+ "num_assistant_tokens": 20,
21
+ "num_assistant_tokens_schedule": "constant",
22
+ "num_beam_groups": 1,
23
+ "num_beams": 1,
24
+ "num_return_sequences": 1,
25
+ "output_scores": false,
26
+ "pad_token_id": 0,
27
+ "remove_invalid_values": false,
28
+ "repetition_penalty": 1.0,
29
+ "return_dict_in_generate": false,
30
+ "target_lookbehind": 10,
31
+ "temperature": 1.0,
32
+ "top_k": 50,
33
+ "top_p": 1.0,
34
+ "transformers_version": "5.12.0",
35
+ "typical_p": 1.0,
36
+ "use_cache": true
37
+ }
runs/hf-corrector-mixed-v1-rebalanced/final-model/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:706c3eba193046762db698cfe95270fef8ebbf3cdae515839ccea75ccdf1079e
3
+ size 1198571496
runs/hf-corrector-mixed-v1-rebalanced/final-model/tokenizer_config.json ADDED
@@ -0,0 +1,1291 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<pad>",
5
+ "lstrip": false,
6
+ "normalized": true,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "</s>",
13
+ "lstrip": false,
14
+ "normalized": true,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<unk>",
21
+ "lstrip": false,
22
+ "normalized": true,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "259": {
28
+ "content": "<extra_id_0>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "260": {
36
+ "content": "<extra_id_1>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "261": {
44
+ "content": "<extra_id_2>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "262": {
52
+ "content": "<extra_id_3>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "263": {
60
+ "content": "<extra_id_4>",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ },
67
+ "264": {
68
+ "content": "<extra_id_5>",
69
+ "lstrip": false,
70
+ "normalized": false,
71
+ "rstrip": false,
72
+ "single_word": false,
73
+ "special": true
74
+ },
75
+ "265": {
76
+ "content": "<extra_id_6>",
77
+ "lstrip": false,
78
+ "normalized": false,
79
+ "rstrip": false,
80
+ "single_word": false,
81
+ "special": true
82
+ },
83
+ "266": {
84
+ "content": "<extra_id_7>",
85
+ "lstrip": false,
86
+ "normalized": false,
87
+ "rstrip": false,
88
+ "single_word": false,
89
+ "special": true
90
+ },
91
+ "267": {
92
+ "content": "<extra_id_8>",
93
+ "lstrip": false,
94
+ "normalized": false,
95
+ "rstrip": false,
96
+ "single_word": false,
97
+ "special": true
98
+ },
99
+ "268": {
100
+ "content": "<extra_id_9>",
101
+ "lstrip": false,
102
+ "normalized": false,
103
+ "rstrip": false,
104
+ "single_word": false,
105
+ "special": true
106
+ },
107
+ "269": {
108
+ "content": "<extra_id_10>",
109
+ "lstrip": false,
110
+ "normalized": false,
111
+ "rstrip": false,
112
+ "single_word": false,
113
+ "special": true
114
+ },
115
+ "270": {
116
+ "content": "<extra_id_11>",
117
+ "lstrip": false,
118
+ "normalized": false,
119
+ "rstrip": false,
120
+ "single_word": false,
121
+ "special": true
122
+ },
123
+ "271": {
124
+ "content": "<extra_id_12>",
125
+ "lstrip": false,
126
+ "normalized": false,
127
+ "rstrip": false,
128
+ "single_word": false,
129
+ "special": true
130
+ },
131
+ "272": {
132
+ "content": "<extra_id_13>",
133
+ "lstrip": false,
134
+ "normalized": false,
135
+ "rstrip": false,
136
+ "single_word": false,
137
+ "special": true
138
+ },
139
+ "273": {
140
+ "content": "<extra_id_14>",
141
+ "lstrip": false,
142
+ "normalized": false,
143
+ "rstrip": false,
144
+ "single_word": false,
145
+ "special": true
146
+ },
147
+ "274": {
148
+ "content": "<extra_id_15>",
149
+ "lstrip": false,
150
+ "normalized": false,
151
+ "rstrip": false,
152
+ "single_word": false,
153
+ "special": true
154
+ },
155
+ "275": {
156
+ "content": "<extra_id_16>",
157
+ "lstrip": false,
158
+ "normalized": false,
159
+ "rstrip": false,
160
+ "single_word": false,
161
+ "special": true
162
+ },
163
+ "276": {
164
+ "content": "<extra_id_17>",
165
+ "lstrip": false,
166
+ "normalized": false,
167
+ "rstrip": false,
168
+ "single_word": false,
169
+ "special": true
170
+ },
171
+ "277": {
172
+ "content": "<extra_id_18>",
173
+ "lstrip": false,
174
+ "normalized": false,
175
+ "rstrip": false,
176
+ "single_word": false,
177
+ "special": true
178
+ },
179
+ "278": {
180
+ "content": "<extra_id_19>",
181
+ "lstrip": false,
182
+ "normalized": false,
183
+ "rstrip": false,
184
+ "single_word": false,
185
+ "special": true
186
+ },
187
+ "279": {
188
+ "content": "<extra_id_20>",
189
+ "lstrip": false,
190
+ "normalized": false,
191
+ "rstrip": false,
192
+ "single_word": false,
193
+ "special": true
194
+ },
195
+ "280": {
196
+ "content": "<extra_id_21>",
197
+ "lstrip": false,
198
+ "normalized": false,
199
+ "rstrip": false,
200
+ "single_word": false,
201
+ "special": true
202
+ },
203
+ "281": {
204
+ "content": "<extra_id_22>",
205
+ "lstrip": false,
206
+ "normalized": false,
207
+ "rstrip": false,
208
+ "single_word": false,
209
+ "special": true
210
+ },
211
+ "282": {
212
+ "content": "<extra_id_23>",
213
+ "lstrip": false,
214
+ "normalized": false,
215
+ "rstrip": false,
216
+ "single_word": false,
217
+ "special": true
218
+ },
219
+ "283": {
220
+ "content": "<extra_id_24>",
221
+ "lstrip": false,
222
+ "normalized": false,
223
+ "rstrip": false,
224
+ "single_word": false,
225
+ "special": true
226
+ },
227
+ "284": {
228
+ "content": "<extra_id_25>",
229
+ "lstrip": false,
230
+ "normalized": false,
231
+ "rstrip": false,
232
+ "single_word": false,
233
+ "special": true
234
+ },
235
+ "285": {
236
+ "content": "<extra_id_26>",
237
+ "lstrip": false,
238
+ "normalized": false,
239
+ "rstrip": false,
240
+ "single_word": false,
241
+ "special": true
242
+ },
243
+ "286": {
244
+ "content": "<extra_id_27>",
245
+ "lstrip": false,
246
+ "normalized": false,
247
+ "rstrip": false,
248
+ "single_word": false,
249
+ "special": true
250
+ },
251
+ "287": {
252
+ "content": "<extra_id_28>",
253
+ "lstrip": false,
254
+ "normalized": false,
255
+ "rstrip": false,
256
+ "single_word": false,
257
+ "special": true
258
+ },
259
+ "288": {
260
+ "content": "<extra_id_29>",
261
+ "lstrip": false,
262
+ "normalized": false,
263
+ "rstrip": false,
264
+ "single_word": false,
265
+ "special": true
266
+ },
267
+ "289": {
268
+ "content": "<extra_id_30>",
269
+ "lstrip": false,
270
+ "normalized": false,
271
+ "rstrip": false,
272
+ "single_word": false,
273
+ "special": true
274
+ },
275
+ "290": {
276
+ "content": "<extra_id_31>",
277
+ "lstrip": false,
278
+ "normalized": false,
279
+ "rstrip": false,
280
+ "single_word": false,
281
+ "special": true
282
+ },
283
+ "291": {
284
+ "content": "<extra_id_32>",
285
+ "lstrip": false,
286
+ "normalized": false,
287
+ "rstrip": false,
288
+ "single_word": false,
289
+ "special": true
290
+ },
291
+ "292": {
292
+ "content": "<extra_id_33>",
293
+ "lstrip": false,
294
+ "normalized": false,
295
+ "rstrip": false,
296
+ "single_word": false,
297
+ "special": true
298
+ },
299
+ "293": {
300
+ "content": "<extra_id_34>",
301
+ "lstrip": false,
302
+ "normalized": false,
303
+ "rstrip": false,
304
+ "single_word": false,
305
+ "special": true
306
+ },
307
+ "294": {
308
+ "content": "<extra_id_35>",
309
+ "lstrip": false,
310
+ "normalized": false,
311
+ "rstrip": false,
312
+ "single_word": false,
313
+ "special": true
314
+ },
315
+ "295": {
316
+ "content": "<extra_id_36>",
317
+ "lstrip": false,
318
+ "normalized": false,
319
+ "rstrip": false,
320
+ "single_word": false,
321
+ "special": true
322
+ },
323
+ "296": {
324
+ "content": "<extra_id_37>",
325
+ "lstrip": false,
326
+ "normalized": false,
327
+ "rstrip": false,
328
+ "single_word": false,
329
+ "special": true
330
+ },
331
+ "297": {
332
+ "content": "<extra_id_38>",
333
+ "lstrip": false,
334
+ "normalized": false,
335
+ "rstrip": false,
336
+ "single_word": false,
337
+ "special": true
338
+ },
339
+ "298": {
340
+ "content": "<extra_id_39>",
341
+ "lstrip": false,
342
+ "normalized": false,
343
+ "rstrip": false,
344
+ "single_word": false,
345
+ "special": true
346
+ },
347
+ "299": {
348
+ "content": "<extra_id_40>",
349
+ "lstrip": false,
350
+ "normalized": false,
351
+ "rstrip": false,
352
+ "single_word": false,
353
+ "special": true
354
+ },
355
+ "300": {
356
+ "content": "<extra_id_41>",
357
+ "lstrip": false,
358
+ "normalized": false,
359
+ "rstrip": false,
360
+ "single_word": false,
361
+ "special": true
362
+ },
363
+ "301": {
364
+ "content": "<extra_id_42>",
365
+ "lstrip": false,
366
+ "normalized": false,
367
+ "rstrip": false,
368
+ "single_word": false,
369
+ "special": true
370
+ },
371
+ "302": {
372
+ "content": "<extra_id_43>",
373
+ "lstrip": false,
374
+ "normalized": false,
375
+ "rstrip": false,
376
+ "single_word": false,
377
+ "special": true
378
+ },
379
+ "303": {
380
+ "content": "<extra_id_44>",
381
+ "lstrip": false,
382
+ "normalized": false,
383
+ "rstrip": false,
384
+ "single_word": false,
385
+ "special": true
386
+ },
387
+ "304": {
388
+ "content": "<extra_id_45>",
389
+ "lstrip": false,
390
+ "normalized": false,
391
+ "rstrip": false,
392
+ "single_word": false,
393
+ "special": true
394
+ },
395
+ "305": {
396
+ "content": "<extra_id_46>",
397
+ "lstrip": false,
398
+ "normalized": false,
399
+ "rstrip": false,
400
+ "single_word": false,
401
+ "special": true
402
+ },
403
+ "306": {
404
+ "content": "<extra_id_47>",
405
+ "lstrip": false,
406
+ "normalized": false,
407
+ "rstrip": false,
408
+ "single_word": false,
409
+ "special": true
410
+ },
411
+ "307": {
412
+ "content": "<extra_id_48>",
413
+ "lstrip": false,
414
+ "normalized": false,
415
+ "rstrip": false,
416
+ "single_word": false,
417
+ "special": true
418
+ },
419
+ "308": {
420
+ "content": "<extra_id_49>",
421
+ "lstrip": false,
422
+ "normalized": false,
423
+ "rstrip": false,
424
+ "single_word": false,
425
+ "special": true
426
+ },
427
+ "309": {
428
+ "content": "<extra_id_50>",
429
+ "lstrip": false,
430
+ "normalized": false,
431
+ "rstrip": false,
432
+ "single_word": false,
433
+ "special": true
434
+ },
435
+ "310": {
436
+ "content": "<extra_id_51>",
437
+ "lstrip": false,
438
+ "normalized": false,
439
+ "rstrip": false,
440
+ "single_word": false,
441
+ "special": true
442
+ },
443
+ "311": {
444
+ "content": "<extra_id_52>",
445
+ "lstrip": false,
446
+ "normalized": false,
447
+ "rstrip": false,
448
+ "single_word": false,
449
+ "special": true
450
+ },
451
+ "312": {
452
+ "content": "<extra_id_53>",
453
+ "lstrip": false,
454
+ "normalized": false,
455
+ "rstrip": false,
456
+ "single_word": false,
457
+ "special": true
458
+ },
459
+ "313": {
460
+ "content": "<extra_id_54>",
461
+ "lstrip": false,
462
+ "normalized": false,
463
+ "rstrip": false,
464
+ "single_word": false,
465
+ "special": true
466
+ },
467
+ "314": {
468
+ "content": "<extra_id_55>",
469
+ "lstrip": false,
470
+ "normalized": false,
471
+ "rstrip": false,
472
+ "single_word": false,
473
+ "special": true
474
+ },
475
+ "315": {
476
+ "content": "<extra_id_56>",
477
+ "lstrip": false,
478
+ "normalized": false,
479
+ "rstrip": false,
480
+ "single_word": false,
481
+ "special": true
482
+ },
483
+ "316": {
484
+ "content": "<extra_id_57>",
485
+ "lstrip": false,
486
+ "normalized": false,
487
+ "rstrip": false,
488
+ "single_word": false,
489
+ "special": true
490
+ },
491
+ "317": {
492
+ "content": "<extra_id_58>",
493
+ "lstrip": false,
494
+ "normalized": false,
495
+ "rstrip": false,
496
+ "single_word": false,
497
+ "special": true
498
+ },
499
+ "318": {
500
+ "content": "<extra_id_59>",
501
+ "lstrip": false,
502
+ "normalized": false,
503
+ "rstrip": false,
504
+ "single_word": false,
505
+ "special": true
506
+ },
507
+ "319": {
508
+ "content": "<extra_id_60>",
509
+ "lstrip": false,
510
+ "normalized": false,
511
+ "rstrip": false,
512
+ "single_word": false,
513
+ "special": true
514
+ },
515
+ "320": {
516
+ "content": "<extra_id_61>",
517
+ "lstrip": false,
518
+ "normalized": false,
519
+ "rstrip": false,
520
+ "single_word": false,
521
+ "special": true
522
+ },
523
+ "321": {
524
+ "content": "<extra_id_62>",
525
+ "lstrip": false,
526
+ "normalized": false,
527
+ "rstrip": false,
528
+ "single_word": false,
529
+ "special": true
530
+ },
531
+ "322": {
532
+ "content": "<extra_id_63>",
533
+ "lstrip": false,
534
+ "normalized": false,
535
+ "rstrip": false,
536
+ "single_word": false,
537
+ "special": true
538
+ },
539
+ "323": {
540
+ "content": "<extra_id_64>",
541
+ "lstrip": false,
542
+ "normalized": false,
543
+ "rstrip": false,
544
+ "single_word": false,
545
+ "special": true
546
+ },
547
+ "324": {
548
+ "content": "<extra_id_65>",
549
+ "lstrip": false,
550
+ "normalized": false,
551
+ "rstrip": false,
552
+ "single_word": false,
553
+ "special": true
554
+ },
555
+ "325": {
556
+ "content": "<extra_id_66>",
557
+ "lstrip": false,
558
+ "normalized": false,
559
+ "rstrip": false,
560
+ "single_word": false,
561
+ "special": true
562
+ },
563
+ "326": {
564
+ "content": "<extra_id_67>",
565
+ "lstrip": false,
566
+ "normalized": false,
567
+ "rstrip": false,
568
+ "single_word": false,
569
+ "special": true
570
+ },
571
+ "327": {
572
+ "content": "<extra_id_68>",
573
+ "lstrip": false,
574
+ "normalized": false,
575
+ "rstrip": false,
576
+ "single_word": false,
577
+ "special": true
578
+ },
579
+ "328": {
580
+ "content": "<extra_id_69>",
581
+ "lstrip": false,
582
+ "normalized": false,
583
+ "rstrip": false,
584
+ "single_word": false,
585
+ "special": true
586
+ },
587
+ "329": {
588
+ "content": "<extra_id_70>",
589
+ "lstrip": false,
590
+ "normalized": false,
591
+ "rstrip": false,
592
+ "single_word": false,
593
+ "special": true
594
+ },
595
+ "330": {
596
+ "content": "<extra_id_71>",
597
+ "lstrip": false,
598
+ "normalized": false,
599
+ "rstrip": false,
600
+ "single_word": false,
601
+ "special": true
602
+ },
603
+ "331": {
604
+ "content": "<extra_id_72>",
605
+ "lstrip": false,
606
+ "normalized": false,
607
+ "rstrip": false,
608
+ "single_word": false,
609
+ "special": true
610
+ },
611
+ "332": {
612
+ "content": "<extra_id_73>",
613
+ "lstrip": false,
614
+ "normalized": false,
615
+ "rstrip": false,
616
+ "single_word": false,
617
+ "special": true
618
+ },
619
+ "333": {
620
+ "content": "<extra_id_74>",
621
+ "lstrip": false,
622
+ "normalized": false,
623
+ "rstrip": false,
624
+ "single_word": false,
625
+ "special": true
626
+ },
627
+ "334": {
628
+ "content": "<extra_id_75>",
629
+ "lstrip": false,
630
+ "normalized": false,
631
+ "rstrip": false,
632
+ "single_word": false,
633
+ "special": true
634
+ },
635
+ "335": {
636
+ "content": "<extra_id_76>",
637
+ "lstrip": false,
638
+ "normalized": false,
639
+ "rstrip": false,
640
+ "single_word": false,
641
+ "special": true
642
+ },
643
+ "336": {
644
+ "content": "<extra_id_77>",
645
+ "lstrip": false,
646
+ "normalized": false,
647
+ "rstrip": false,
648
+ "single_word": false,
649
+ "special": true
650
+ },
651
+ "337": {
652
+ "content": "<extra_id_78>",
653
+ "lstrip": false,
654
+ "normalized": false,
655
+ "rstrip": false,
656
+ "single_word": false,
657
+ "special": true
658
+ },
659
+ "338": {
660
+ "content": "<extra_id_79>",
661
+ "lstrip": false,
662
+ "normalized": false,
663
+ "rstrip": false,
664
+ "single_word": false,
665
+ "special": true
666
+ },
667
+ "339": {
668
+ "content": "<extra_id_80>",
669
+ "lstrip": false,
670
+ "normalized": false,
671
+ "rstrip": false,
672
+ "single_word": false,
673
+ "special": true
674
+ },
675
+ "340": {
676
+ "content": "<extra_id_81>",
677
+ "lstrip": false,
678
+ "normalized": false,
679
+ "rstrip": false,
680
+ "single_word": false,
681
+ "special": true
682
+ },
683
+ "341": {
684
+ "content": "<extra_id_82>",
685
+ "lstrip": false,
686
+ "normalized": false,
687
+ "rstrip": false,
688
+ "single_word": false,
689
+ "special": true
690
+ },
691
+ "342": {
692
+ "content": "<extra_id_83>",
693
+ "lstrip": false,
694
+ "normalized": false,
695
+ "rstrip": false,
696
+ "single_word": false,
697
+ "special": true
698
+ },
699
+ "343": {
700
+ "content": "<extra_id_84>",
701
+ "lstrip": false,
702
+ "normalized": false,
703
+ "rstrip": false,
704
+ "single_word": false,
705
+ "special": true
706
+ },
707
+ "344": {
708
+ "content": "<extra_id_85>",
709
+ "lstrip": false,
710
+ "normalized": false,
711
+ "rstrip": false,
712
+ "single_word": false,
713
+ "special": true
714
+ },
715
+ "345": {
716
+ "content": "<extra_id_86>",
717
+ "lstrip": false,
718
+ "normalized": false,
719
+ "rstrip": false,
720
+ "single_word": false,
721
+ "special": true
722
+ },
723
+ "346": {
724
+ "content": "<extra_id_87>",
725
+ "lstrip": false,
726
+ "normalized": false,
727
+ "rstrip": false,
728
+ "single_word": false,
729
+ "special": true
730
+ },
731
+ "347": {
732
+ "content": "<extra_id_88>",
733
+ "lstrip": false,
734
+ "normalized": false,
735
+ "rstrip": false,
736
+ "single_word": false,
737
+ "special": true
738
+ },
739
+ "348": {
740
+ "content": "<extra_id_89>",
741
+ "lstrip": false,
742
+ "normalized": false,
743
+ "rstrip": false,
744
+ "single_word": false,
745
+ "special": true
746
+ },
747
+ "349": {
748
+ "content": "<extra_id_90>",
749
+ "lstrip": false,
750
+ "normalized": false,
751
+ "rstrip": false,
752
+ "single_word": false,
753
+ "special": true
754
+ },
755
+ "350": {
756
+ "content": "<extra_id_91>",
757
+ "lstrip": false,
758
+ "normalized": false,
759
+ "rstrip": false,
760
+ "single_word": false,
761
+ "special": true
762
+ },
763
+ "351": {
764
+ "content": "<extra_id_92>",
765
+ "lstrip": false,
766
+ "normalized": false,
767
+ "rstrip": false,
768
+ "single_word": false,
769
+ "special": true
770
+ },
771
+ "352": {
772
+ "content": "<extra_id_93>",
773
+ "lstrip": false,
774
+ "normalized": false,
775
+ "rstrip": false,
776
+ "single_word": false,
777
+ "special": true
778
+ },
779
+ "353": {
780
+ "content": "<extra_id_94>",
781
+ "lstrip": false,
782
+ "normalized": false,
783
+ "rstrip": false,
784
+ "single_word": false,
785
+ "special": true
786
+ },
787
+ "354": {
788
+ "content": "<extra_id_95>",
789
+ "lstrip": false,
790
+ "normalized": false,
791
+ "rstrip": false,
792
+ "single_word": false,
793
+ "special": true
794
+ },
795
+ "355": {
796
+ "content": "<extra_id_96>",
797
+ "lstrip": false,
798
+ "normalized": false,
799
+ "rstrip": false,
800
+ "single_word": false,
801
+ "special": true
802
+ },
803
+ "356": {
804
+ "content": "<extra_id_97>",
805
+ "lstrip": false,
806
+ "normalized": false,
807
+ "rstrip": false,
808
+ "single_word": false,
809
+ "special": true
810
+ },
811
+ "357": {
812
+ "content": "<extra_id_98>",
813
+ "lstrip": false,
814
+ "normalized": false,
815
+ "rstrip": false,
816
+ "single_word": false,
817
+ "special": true
818
+ },
819
+ "358": {
820
+ "content": "<extra_id_99>",
821
+ "lstrip": false,
822
+ "normalized": false,
823
+ "rstrip": false,
824
+ "single_word": false,
825
+ "special": true
826
+ },
827
+ "359": {
828
+ "content": "<extra_id_100>",
829
+ "lstrip": false,
830
+ "normalized": false,
831
+ "rstrip": false,
832
+ "single_word": false,
833
+ "special": true
834
+ },
835
+ "360": {
836
+ "content": "<extra_id_101>",
837
+ "lstrip": false,
838
+ "normalized": false,
839
+ "rstrip": false,
840
+ "single_word": false,
841
+ "special": true
842
+ },
843
+ "361": {
844
+ "content": "<extra_id_102>",
845
+ "lstrip": false,
846
+ "normalized": false,
847
+ "rstrip": false,
848
+ "single_word": false,
849
+ "special": true
850
+ },
851
+ "362": {
852
+ "content": "<extra_id_103>",
853
+ "lstrip": false,
854
+ "normalized": false,
855
+ "rstrip": false,
856
+ "single_word": false,
857
+ "special": true
858
+ },
859
+ "363": {
860
+ "content": "<extra_id_104>",
861
+ "lstrip": false,
862
+ "normalized": false,
863
+ "rstrip": false,
864
+ "single_word": false,
865
+ "special": true
866
+ },
867
+ "364": {
868
+ "content": "<extra_id_105>",
869
+ "lstrip": false,
870
+ "normalized": false,
871
+ "rstrip": false,
872
+ "single_word": false,
873
+ "special": true
874
+ },
875
+ "365": {
876
+ "content": "<extra_id_106>",
877
+ "lstrip": false,
878
+ "normalized": false,
879
+ "rstrip": false,
880
+ "single_word": false,
881
+ "special": true
882
+ },
883
+ "366": {
884
+ "content": "<extra_id_107>",
885
+ "lstrip": false,
886
+ "normalized": false,
887
+ "rstrip": false,
888
+ "single_word": false,
889
+ "special": true
890
+ },
891
+ "367": {
892
+ "content": "<extra_id_108>",
893
+ "lstrip": false,
894
+ "normalized": false,
895
+ "rstrip": false,
896
+ "single_word": false,
897
+ "special": true
898
+ },
899
+ "368": {
900
+ "content": "<extra_id_109>",
901
+ "lstrip": false,
902
+ "normalized": false,
903
+ "rstrip": false,
904
+ "single_word": false,
905
+ "special": true
906
+ },
907
+ "369": {
908
+ "content": "<extra_id_110>",
909
+ "lstrip": false,
910
+ "normalized": false,
911
+ "rstrip": false,
912
+ "single_word": false,
913
+ "special": true
914
+ },
915
+ "370": {
916
+ "content": "<extra_id_111>",
917
+ "lstrip": false,
918
+ "normalized": false,
919
+ "rstrip": false,
920
+ "single_word": false,
921
+ "special": true
922
+ },
923
+ "371": {
924
+ "content": "<extra_id_112>",
925
+ "lstrip": false,
926
+ "normalized": false,
927
+ "rstrip": false,
928
+ "single_word": false,
929
+ "special": true
930
+ },
931
+ "372": {
932
+ "content": "<extra_id_113>",
933
+ "lstrip": false,
934
+ "normalized": false,
935
+ "rstrip": false,
936
+ "single_word": false,
937
+ "special": true
938
+ },
939
+ "373": {
940
+ "content": "<extra_id_114>",
941
+ "lstrip": false,
942
+ "normalized": false,
943
+ "rstrip": false,
944
+ "single_word": false,
945
+ "special": true
946
+ },
947
+ "374": {
948
+ "content": "<extra_id_115>",
949
+ "lstrip": false,
950
+ "normalized": false,
951
+ "rstrip": false,
952
+ "single_word": false,
953
+ "special": true
954
+ },
955
+ "375": {
956
+ "content": "<extra_id_116>",
957
+ "lstrip": false,
958
+ "normalized": false,
959
+ "rstrip": false,
960
+ "single_word": false,
961
+ "special": true
962
+ },
963
+ "376": {
964
+ "content": "<extra_id_117>",
965
+ "lstrip": false,
966
+ "normalized": false,
967
+ "rstrip": false,
968
+ "single_word": false,
969
+ "special": true
970
+ },
971
+ "377": {
972
+ "content": "<extra_id_118>",
973
+ "lstrip": false,
974
+ "normalized": false,
975
+ "rstrip": false,
976
+ "single_word": false,
977
+ "special": true
978
+ },
979
+ "378": {
980
+ "content": "<extra_id_119>",
981
+ "lstrip": false,
982
+ "normalized": false,
983
+ "rstrip": false,
984
+ "single_word": false,
985
+ "special": true
986
+ },
987
+ "379": {
988
+ "content": "<extra_id_120>",
989
+ "lstrip": false,
990
+ "normalized": false,
991
+ "rstrip": false,
992
+ "single_word": false,
993
+ "special": true
994
+ },
995
+ "380": {
996
+ "content": "<extra_id_121>",
997
+ "lstrip": false,
998
+ "normalized": false,
999
+ "rstrip": false,
1000
+ "single_word": false,
1001
+ "special": true
1002
+ },
1003
+ "381": {
1004
+ "content": "<extra_id_122>",
1005
+ "lstrip": false,
1006
+ "normalized": false,
1007
+ "rstrip": false,
1008
+ "single_word": false,
1009
+ "special": true
1010
+ },
1011
+ "382": {
1012
+ "content": "<extra_id_123>",
1013
+ "lstrip": false,
1014
+ "normalized": false,
1015
+ "rstrip": false,
1016
+ "single_word": false,
1017
+ "special": true
1018
+ },
1019
+ "383": {
1020
+ "content": "<extra_id_124>",
1021
+ "lstrip": false,
1022
+ "normalized": false,
1023
+ "rstrip": false,
1024
+ "single_word": false,
1025
+ "special": true
1026
+ }
1027
+ },
1028
+ "additional_special_tokens": [
1029
+ "<extra_id_0>",
1030
+ "<extra_id_1>",
1031
+ "<extra_id_2>",
1032
+ "<extra_id_3>",
1033
+ "<extra_id_4>",
1034
+ "<extra_id_5>",
1035
+ "<extra_id_6>",
1036
+ "<extra_id_7>",
1037
+ "<extra_id_8>",
1038
+ "<extra_id_9>",
1039
+ "<extra_id_10>",
1040
+ "<extra_id_11>",
1041
+ "<extra_id_12>",
1042
+ "<extra_id_13>",
1043
+ "<extra_id_14>",
1044
+ "<extra_id_15>",
1045
+ "<extra_id_16>",
1046
+ "<extra_id_17>",
1047
+ "<extra_id_18>",
1048
+ "<extra_id_19>",
1049
+ "<extra_id_20>",
1050
+ "<extra_id_21>",
1051
+ "<extra_id_22>",
1052
+ "<extra_id_23>",
1053
+ "<extra_id_24>",
1054
+ "<extra_id_25>",
1055
+ "<extra_id_26>",
1056
+ "<extra_id_27>",
1057
+ "<extra_id_28>",
1058
+ "<extra_id_29>",
1059
+ "<extra_id_30>",
1060
+ "<extra_id_31>",
1061
+ "<extra_id_32>",
1062
+ "<extra_id_33>",
1063
+ "<extra_id_34>",
1064
+ "<extra_id_35>",
1065
+ "<extra_id_36>",
1066
+ "<extra_id_37>",
1067
+ "<extra_id_38>",
1068
+ "<extra_id_39>",
1069
+ "<extra_id_40>",
1070
+ "<extra_id_41>",
1071
+ "<extra_id_42>",
1072
+ "<extra_id_43>",
1073
+ "<extra_id_44>",
1074
+ "<extra_id_45>",
1075
+ "<extra_id_46>",
1076
+ "<extra_id_47>",
1077
+ "<extra_id_48>",
1078
+ "<extra_id_49>",
1079
+ "<extra_id_50>",
1080
+ "<extra_id_51>",
1081
+ "<extra_id_52>",
1082
+ "<extra_id_53>",
1083
+ "<extra_id_54>",
1084
+ "<extra_id_55>",
1085
+ "<extra_id_56>",
1086
+ "<extra_id_57>",
1087
+ "<extra_id_58>",
1088
+ "<extra_id_59>",
1089
+ "<extra_id_60>",
1090
+ "<extra_id_61>",
1091
+ "<extra_id_62>",
1092
+ "<extra_id_63>",
1093
+ "<extra_id_64>",
1094
+ "<extra_id_65>",
1095
+ "<extra_id_66>",
1096
+ "<extra_id_67>",
1097
+ "<extra_id_68>",
1098
+ "<extra_id_69>",
1099
+ "<extra_id_70>",
1100
+ "<extra_id_71>",
1101
+ "<extra_id_72>",
1102
+ "<extra_id_73>",
1103
+ "<extra_id_74>",
1104
+ "<extra_id_75>",
1105
+ "<extra_id_76>",
1106
+ "<extra_id_77>",
1107
+ "<extra_id_78>",
1108
+ "<extra_id_79>",
1109
+ "<extra_id_80>",
1110
+ "<extra_id_81>",
1111
+ "<extra_id_82>",
1112
+ "<extra_id_83>",
1113
+ "<extra_id_84>",
1114
+ "<extra_id_85>",
1115
+ "<extra_id_86>",
1116
+ "<extra_id_87>",
1117
+ "<extra_id_88>",
1118
+ "<extra_id_89>",
1119
+ "<extra_id_90>",
1120
+ "<extra_id_91>",
1121
+ "<extra_id_92>",
1122
+ "<extra_id_93>",
1123
+ "<extra_id_94>",
1124
+ "<extra_id_95>",
1125
+ "<extra_id_96>",
1126
+ "<extra_id_97>",
1127
+ "<extra_id_98>",
1128
+ "<extra_id_99>",
1129
+ "<extra_id_100>",
1130
+ "<extra_id_101>",
1131
+ "<extra_id_102>",
1132
+ "<extra_id_103>",
1133
+ "<extra_id_104>",
1134
+ "<extra_id_105>",
1135
+ "<extra_id_106>",
1136
+ "<extra_id_107>",
1137
+ "<extra_id_108>",
1138
+ "<extra_id_109>",
1139
+ "<extra_id_110>",
1140
+ "<extra_id_111>",
1141
+ "<extra_id_112>",
1142
+ "<extra_id_113>",
1143
+ "<extra_id_114>",
1144
+ "<extra_id_115>",
1145
+ "<extra_id_116>",
1146
+ "<extra_id_117>",
1147
+ "<extra_id_118>",
1148
+ "<extra_id_119>",
1149
+ "<extra_id_120>",
1150
+ "<extra_id_121>",
1151
+ "<extra_id_122>",
1152
+ "<extra_id_123>",
1153
+ "<extra_id_124>"
1154
+ ],
1155
+ "backend": "custom",
1156
+ "eos_token": "</s>",
1157
+ "extra_ids": 0,
1158
+ "extra_special_tokens": [
1159
+ "<extra_id_0>",
1160
+ "<extra_id_1>",
1161
+ "<extra_id_2>",
1162
+ "<extra_id_3>",
1163
+ "<extra_id_4>",
1164
+ "<extra_id_5>",
1165
+ "<extra_id_6>",
1166
+ "<extra_id_7>",
1167
+ "<extra_id_8>",
1168
+ "<extra_id_9>",
1169
+ "<extra_id_10>",
1170
+ "<extra_id_11>",
1171
+ "<extra_id_12>",
1172
+ "<extra_id_13>",
1173
+ "<extra_id_14>",
1174
+ "<extra_id_15>",
1175
+ "<extra_id_16>",
1176
+ "<extra_id_17>",
1177
+ "<extra_id_18>",
1178
+ "<extra_id_19>",
1179
+ "<extra_id_20>",
1180
+ "<extra_id_21>",
1181
+ "<extra_id_22>",
1182
+ "<extra_id_23>",
1183
+ "<extra_id_24>",
1184
+ "<extra_id_25>",
1185
+ "<extra_id_26>",
1186
+ "<extra_id_27>",
1187
+ "<extra_id_28>",
1188
+ "<extra_id_29>",
1189
+ "<extra_id_30>",
1190
+ "<extra_id_31>",
1191
+ "<extra_id_32>",
1192
+ "<extra_id_33>",
1193
+ "<extra_id_34>",
1194
+ "<extra_id_35>",
1195
+ "<extra_id_36>",
1196
+ "<extra_id_37>",
1197
+ "<extra_id_38>",
1198
+ "<extra_id_39>",
1199
+ "<extra_id_40>",
1200
+ "<extra_id_41>",
1201
+ "<extra_id_42>",
1202
+ "<extra_id_43>",
1203
+ "<extra_id_44>",
1204
+ "<extra_id_45>",
1205
+ "<extra_id_46>",
1206
+ "<extra_id_47>",
1207
+ "<extra_id_48>",
1208
+ "<extra_id_49>",
1209
+ "<extra_id_50>",
1210
+ "<extra_id_51>",
1211
+ "<extra_id_52>",
1212
+ "<extra_id_53>",
1213
+ "<extra_id_54>",
1214
+ "<extra_id_55>",
1215
+ "<extra_id_56>",
1216
+ "<extra_id_57>",
1217
+ "<extra_id_58>",
1218
+ "<extra_id_59>",
1219
+ "<extra_id_60>",
1220
+ "<extra_id_61>",
1221
+ "<extra_id_62>",
1222
+ "<extra_id_63>",
1223
+ "<extra_id_64>",
1224
+ "<extra_id_65>",
1225
+ "<extra_id_66>",
1226
+ "<extra_id_67>",
1227
+ "<extra_id_68>",
1228
+ "<extra_id_69>",
1229
+ "<extra_id_70>",
1230
+ "<extra_id_71>",
1231
+ "<extra_id_72>",
1232
+ "<extra_id_73>",
1233
+ "<extra_id_74>",
1234
+ "<extra_id_75>",
1235
+ "<extra_id_76>",
1236
+ "<extra_id_77>",
1237
+ "<extra_id_78>",
1238
+ "<extra_id_79>",
1239
+ "<extra_id_80>",
1240
+ "<extra_id_81>",
1241
+ "<extra_id_82>",
1242
+ "<extra_id_83>",
1243
+ "<extra_id_84>",
1244
+ "<extra_id_85>",
1245
+ "<extra_id_86>",
1246
+ "<extra_id_87>",
1247
+ "<extra_id_88>",
1248
+ "<extra_id_89>",
1249
+ "<extra_id_90>",
1250
+ "<extra_id_91>",
1251
+ "<extra_id_92>",
1252
+ "<extra_id_93>",
1253
+ "<extra_id_94>",
1254
+ "<extra_id_95>",
1255
+ "<extra_id_96>",
1256
+ "<extra_id_97>",
1257
+ "<extra_id_98>",
1258
+ "<extra_id_99>",
1259
+ "<extra_id_100>",
1260
+ "<extra_id_101>",
1261
+ "<extra_id_102>",
1262
+ "<extra_id_103>",
1263
+ "<extra_id_104>",
1264
+ "<extra_id_105>",
1265
+ "<extra_id_106>",
1266
+ "<extra_id_107>",
1267
+ "<extra_id_108>",
1268
+ "<extra_id_109>",
1269
+ "<extra_id_110>",
1270
+ "<extra_id_111>",
1271
+ "<extra_id_112>",
1272
+ "<extra_id_113>",
1273
+ "<extra_id_114>",
1274
+ "<extra_id_115>",
1275
+ "<extra_id_116>",
1276
+ "<extra_id_117>",
1277
+ "<extra_id_118>",
1278
+ "<extra_id_119>",
1279
+ "<extra_id_120>",
1280
+ "<extra_id_121>",
1281
+ "<extra_id_122>",
1282
+ "<extra_id_123>",
1283
+ "<extra_id_124>"
1284
+ ],
1285
+ "is_local": false,
1286
+ "local_files_only": false,
1287
+ "model_max_length": 1000000000000000019884624838656,
1288
+ "pad_token": "<pad>",
1289
+ "tokenizer_class": "ByT5Tokenizer",
1290
+ "unk_token": "<unk>"
1291
+ }
runs/hf-corrector-mixed-v1-rebalanced/final-model/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9483641fdd68a02b027422a4d2d6a0662d0fe6e8d6eecbb844050d5e3fe86315
3
+ size 5329
runs/hf-corrector-mixed-v1-rebalanced/hf_text_corrector_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train_pairs": "runs/hf-corrector-mixed-v1-rebalanced/train_pairs.jsonl",
3
+ "eval_pairs": "runs/hf-corrector-mixed-v1-rebalanced/eval_pairs.jsonl",
4
+ "output_dir": "runs/hf-corrector-mixed-v1-rebalanced",
5
+ "base_model": "google/byt5-small",
6
+ "max_source_length": 256,
7
+ "max_target_length": 256,
8
+ "per_device_train_batch_size": 16,
9
+ "per_device_eval_batch_size": 16,
10
+ "learning_rate": 5e-05,
11
+ "num_train_epochs": 2.0,
12
+ "warmup_steps": 0,
13
+ "logging_steps": 10,
14
+ "save_total_limit": 2,
15
+ "predict_with_generate": true,
16
+ "generation_max_length": 256
17
+ }
runs/hf-corrector-mixed-v1-rebalanced/package/PACKAGE.md ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ # Packaged Text Correction Composite Candidate
2
+
3
+ Model card: `runs/hf-corrector-mixed-v1-rebalanced/package/model-card.json`
4
+ Artifacts: `runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model`
5
+
6
+ Audit before benchmarking:
7
+
8
+ ```bash
9
+ ocrtech audit-model runs/hf-corrector-mixed-v1-rebalanced/package/model-card.json
10
+ ```
runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model/added_tokens.json ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<extra_id_0>": 259,
3
+ "<extra_id_100>": 359,
4
+ "<extra_id_101>": 360,
5
+ "<extra_id_102>": 361,
6
+ "<extra_id_103>": 362,
7
+ "<extra_id_104>": 363,
8
+ "<extra_id_105>": 364,
9
+ "<extra_id_106>": 365,
10
+ "<extra_id_107>": 366,
11
+ "<extra_id_108>": 367,
12
+ "<extra_id_109>": 368,
13
+ "<extra_id_10>": 269,
14
+ "<extra_id_110>": 369,
15
+ "<extra_id_111>": 370,
16
+ "<extra_id_112>": 371,
17
+ "<extra_id_113>": 372,
18
+ "<extra_id_114>": 373,
19
+ "<extra_id_115>": 374,
20
+ "<extra_id_116>": 375,
21
+ "<extra_id_117>": 376,
22
+ "<extra_id_118>": 377,
23
+ "<extra_id_119>": 378,
24
+ "<extra_id_11>": 270,
25
+ "<extra_id_120>": 379,
26
+ "<extra_id_121>": 380,
27
+ "<extra_id_122>": 381,
28
+ "<extra_id_123>": 382,
29
+ "<extra_id_124>": 383,
30
+ "<extra_id_12>": 271,
31
+ "<extra_id_13>": 272,
32
+ "<extra_id_14>": 273,
33
+ "<extra_id_15>": 274,
34
+ "<extra_id_16>": 275,
35
+ "<extra_id_17>": 276,
36
+ "<extra_id_18>": 277,
37
+ "<extra_id_19>": 278,
38
+ "<extra_id_1>": 260,
39
+ "<extra_id_20>": 279,
40
+ "<extra_id_21>": 280,
41
+ "<extra_id_22>": 281,
42
+ "<extra_id_23>": 282,
43
+ "<extra_id_24>": 283,
44
+ "<extra_id_25>": 284,
45
+ "<extra_id_26>": 285,
46
+ "<extra_id_27>": 286,
47
+ "<extra_id_28>": 287,
48
+ "<extra_id_29>": 288,
49
+ "<extra_id_2>": 261,
50
+ "<extra_id_30>": 289,
51
+ "<extra_id_31>": 290,
52
+ "<extra_id_32>": 291,
53
+ "<extra_id_33>": 292,
54
+ "<extra_id_34>": 293,
55
+ "<extra_id_35>": 294,
56
+ "<extra_id_36>": 295,
57
+ "<extra_id_37>": 296,
58
+ "<extra_id_38>": 297,
59
+ "<extra_id_39>": 298,
60
+ "<extra_id_3>": 262,
61
+ "<extra_id_40>": 299,
62
+ "<extra_id_41>": 300,
63
+ "<extra_id_42>": 301,
64
+ "<extra_id_43>": 302,
65
+ "<extra_id_44>": 303,
66
+ "<extra_id_45>": 304,
67
+ "<extra_id_46>": 305,
68
+ "<extra_id_47>": 306,
69
+ "<extra_id_48>": 307,
70
+ "<extra_id_49>": 308,
71
+ "<extra_id_4>": 263,
72
+ "<extra_id_50>": 309,
73
+ "<extra_id_51>": 310,
74
+ "<extra_id_52>": 311,
75
+ "<extra_id_53>": 312,
76
+ "<extra_id_54>": 313,
77
+ "<extra_id_55>": 314,
78
+ "<extra_id_56>": 315,
79
+ "<extra_id_57>": 316,
80
+ "<extra_id_58>": 317,
81
+ "<extra_id_59>": 318,
82
+ "<extra_id_5>": 264,
83
+ "<extra_id_60>": 319,
84
+ "<extra_id_61>": 320,
85
+ "<extra_id_62>": 321,
86
+ "<extra_id_63>": 322,
87
+ "<extra_id_64>": 323,
88
+ "<extra_id_65>": 324,
89
+ "<extra_id_66>": 325,
90
+ "<extra_id_67>": 326,
91
+ "<extra_id_68>": 327,
92
+ "<extra_id_69>": 328,
93
+ "<extra_id_6>": 265,
94
+ "<extra_id_70>": 329,
95
+ "<extra_id_71>": 330,
96
+ "<extra_id_72>": 331,
97
+ "<extra_id_73>": 332,
98
+ "<extra_id_74>": 333,
99
+ "<extra_id_75>": 334,
100
+ "<extra_id_76>": 335,
101
+ "<extra_id_77>": 336,
102
+ "<extra_id_78>": 337,
103
+ "<extra_id_79>": 338,
104
+ "<extra_id_7>": 266,
105
+ "<extra_id_80>": 339,
106
+ "<extra_id_81>": 340,
107
+ "<extra_id_82>": 341,
108
+ "<extra_id_83>": 342,
109
+ "<extra_id_84>": 343,
110
+ "<extra_id_85>": 344,
111
+ "<extra_id_86>": 345,
112
+ "<extra_id_87>": 346,
113
+ "<extra_id_88>": 347,
114
+ "<extra_id_89>": 348,
115
+ "<extra_id_8>": 267,
116
+ "<extra_id_90>": 349,
117
+ "<extra_id_91>": 350,
118
+ "<extra_id_92>": 351,
119
+ "<extra_id_93>": 352,
120
+ "<extra_id_94>": 353,
121
+ "<extra_id_95>": 354,
122
+ "<extra_id_96>": 355,
123
+ "<extra_id_97>": 356,
124
+ "<extra_id_98>": 357,
125
+ "<extra_id_99>": 358,
126
+ "<extra_id_9>": 268
127
+ }
runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model/config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "T5ForConditionalGeneration"
4
+ ],
5
+ "classifier_dropout": 0.0,
6
+ "d_ff": 3584,
7
+ "d_kv": 64,
8
+ "d_model": 1472,
9
+ "decoder_start_token_id": 0,
10
+ "dense_act_fn": "gelu_new",
11
+ "dropout_rate": 0.1,
12
+ "dtype": "float32",
13
+ "eos_token_id": 1,
14
+ "feed_forward_proj": "gated-gelu",
15
+ "gradient_checkpointing": false,
16
+ "initializer_factor": 1.0,
17
+ "is_decoder": false,
18
+ "is_encoder_decoder": true,
19
+ "is_gated_act": true,
20
+ "layer_norm_epsilon": 1e-06,
21
+ "model_type": "t5",
22
+ "num_decoder_layers": 4,
23
+ "num_heads": 6,
24
+ "num_layers": 12,
25
+ "pad_token_id": 0,
26
+ "relative_attention_max_distance": 128,
27
+ "relative_attention_num_buckets": 32,
28
+ "scale_decoder_outputs": false,
29
+ "tie_word_embeddings": true,
30
+ "tokenizer_class": "ByT5Tokenizer",
31
+ "transformers_version": "5.12.0",
32
+ "use_cache": false,
33
+ "vocab_size": 384
34
+ }
runs/hf-corrector-mixed-v1-rebalanced/package/artifacts/model/generation_config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": false,
3
+ "assistant_confidence_threshold": 0.4,
4
+ "assistant_lookbehind": 10,
5
+ "decoder_start_token_id": 0,
6
+ "diversity_penalty": 0.0,
7
+ "do_sample": false,
8
+ "early_stopping": false,
9
+ "encoder_no_repeat_ngram_size": 0,
10
+ "encoder_repetition_penalty": 1.0,
11
+ "eos_token_id": [
12
+ 1
13
+ ],
14
+ "epsilon_cutoff": 0.0,
15
+ "eta_cutoff": 0.0,
16
+ "length_penalty": 1.0,
17
+ "max_length": 256,
18
+ "min_length": 0,
19
+ "no_repeat_ngram_size": 0,
20
+ "num_assistant_tokens": 20,
21
+ "num_assistant_tokens_schedule": "constant",
22
+ "num_beam_groups": 1,
23
+ "num_beams": 1,
24
+ "num_return_sequences": 1,
25
+ "output_scores": false,
26
+ "pad_token_id": 0,
27
+ "remove_invalid_values": false,
28
+ "repetition_penalty": 1.0,
29
+ "return_dict_in_generate": false,
30
+ "target_lookbehind": 10,
31
+ "temperature": 1.0,
32
+ "top_k": 50,
33
+ "top_p": 1.0,
34
+ "transformers_version": "5.12.0",
35
+ "typical_p": 1.0,
36
+ "use_cache": true
37
+ }
runs/hf-corrector-mixed-v1-rebalanced/package/model-card.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_id": "ocrtech-tesseract-corrected-mixed-v1",
3
+ "backend": "text_correction_composite",
4
+ "base_model": "google/byt5-small",
5
+ "created_at_utc": "2026-06-14T03:08:28.115455+00:00",
6
+ "artifacts": [
7
+ {
8
+ "path": "artifacts/model",
9
+ "required": true,
10
+ "sha256": null,
11
+ "kind": "directory"
12
+ }
13
+ ],
14
+ "backend_kwargs": {
15
+ "base_engine": "tesseract",
16
+ "base_engine_kwargs": {},
17
+ "model_dir": "artifacts/model",
18
+ "tokenizer_dir": "artifacts/model",
19
+ "max_new_tokens": 256
20
+ },
21
+ "provenance": {
22
+ "train_manifest": {
23
+ "path": "data/splits-mixed-v1/train-rebalanced-nep-eng.jsonl",
24
+ "sha256": "86325bc94e5f707329e8735c68afee27d32f7b3ac6186386325f46dc296184c6"
25
+ },
26
+ "eval_manifest": {
27
+ "path": "data/splits-mixed-v1/eval-with-refs.jsonl",
28
+ "sha256": "314c11e25aa6caccb2a801a6c5a6a3f7222eeb59ae76e77f12d6dd72ff4a90ae"
29
+ }
30
+ },
31
+ "metrics": {
32
+ "report": {
33
+ "path": "runs/hf-corrector-mixed-v1-rebalanced/training_metrics.json",
34
+ "sha256": "491416430e31605548e3e07a9dc3634807e4efaa7e6fae11872b52a9af5bf873"
35
+ }
36
+ },
37
+ "notes": null
38
+ }
runs/hf-corrector-mixed-v1-rebalanced/train_pairs.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/hf-corrector-mixed-v1-rebalanced/training_metrics.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eval_loss": 0.8401811718940735,
3
+ "eval_cer": 7.067461343195775,
4
+ "eval_wer": 3.334916864181571,
5
+ "eval_exact_line_accuracy": 0.0,
6
+ "eval_runtime": 125.4858,
7
+ "eval_samples_per_second": 1.355,
8
+ "eval_steps_per_second": 0.088,
9
+ "epoch": 2.0
10
+ }