Bina 0.4 Rizeh: full 95k production pipeline

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

MANIFEST.json

@@ -0,0 +1,653 @@
+{
+ "files": [
+  {
+   "path": "stallion/bina04-assembly-audit-95k-20261104/graphic_blocks.py",
+   "source": "/home/rezo/bina04-assembly-audit-95k-20261104/graphic_blocks.py",
+   "bytes": 3780
+  },
+  {
+   "path": "stallion/bina04-assembly-audit-95k-20261104/production.py",
+   "source": "/home/rezo/bina04-assembly-audit-95k-20261104/production.py",
+   "bytes": 616
+  },
+  {
+   "path": "stallion/bina04-block-fields-95k-20261103/assembly_fix.py",
+   "source": "/home/rezo/bina04-block-fields-95k-20261103/assembly_fix.py",
+   "bytes": 4466
+  },
+  {
+   "path": "stallion/bina04-block-fields-95k-20261103/production.py",
+   "source": "/home/rezo/bina04-block-fields-95k-20261103/production.py",
+   "bytes": 872
+  },
+  {
+   "path": "stallion/bina04-cell-rowcores-95k-20261103/candidate.py",
+   "source": "/home/rezo/bina04-cell-rowcores-95k-20261103/candidate.py",
+   "bytes": 2858
+  },
+  {
+   "path": "stallion/bina04-cell-rowcores-95k-20261103/production.py",
+   "source": "/home/rezo/bina04-cell-rowcores-95k-20261103/production.py",
+   "bytes": 1229
+  },
+  {
+   "path": "stallion/bina04-column-assembly-95k-20261103/column_order.py",
+   "source": "/home/rezo/bina04-column-assembly-95k-20261103/column_order.py",
+   "bytes": 2812
+  },
+  {
+   "path": "stallion/bina04-column-assembly-95k-20261103/production.py",
+   "source": "/home/rezo/bina04-column-assembly-95k-20261103/production.py",
+   "bytes": 665
+  },
+  {
+   "path": "stallion/bina04-crop-bandseg-95k-20261102/bandseg.py",
+   "source": "/home/rezo/bina04-crop-bandseg-95k-20261102/bandseg.py",
+   "bytes": 14573
+  },
+  {
+   "path": "stallion/bina04-crop-bandseg-95k-20261102/consumer_runtime.py",
+   "source": "/home/rezo/bina04-crop-bandseg-95k-20261102/consumer_runtime.py",
+   "bytes": 2357
+  },
+  {
+   "path": "stallion/bina04-crop-picture-cells-95k-20261102/CALIBRATION.json",
+   "source": "/home/rezo/bina04-crop-picture-cells-95k-20261102/CALIBRATION.json",
+   "bytes": 14990
+  },
+  {
+   "path": "stallion/bina04-crop-picture-cells-95k-20261102/candidate.py",
+   "source": "/home/rezo/bina04-crop-picture-cells-95k-20261102/candidate.py",
+   "bytes": 3443
+  },
+  {
+   "path": "stallion/bina04-crop-picture-cells-95k-20261102/cell_reader.py",
+   "source": "/home/rezo/bina04-crop-picture-cells-95k-20261102/cell_reader.py",
+   "bytes": 5178
+  },
+  {
+   "path": "stallion/bina04-crop-picture-cells-95k-20261102/junk_filter.py",
+   "source": "/home/rezo/bina04-crop-picture-cells-95k-20261102/junk_filter.py",
+   "bytes": 3251
+  },
+  {
+   "path": "stallion/bina04-crop-picture-cells-95k-20261102/pen-classifier.joblib",
+   "source": "/home/rezo/bina04-crop-picture-cells-95k-20261102/pen-classifier.joblib",
+   "bytes": 13905
+  },
+  {
+   "path": "stallion/bina04-crop-picture-cells-95k-20261102/pen_classifier.py",
+   "source": "/home/rezo/bina04-crop-picture-cells-95k-20261102/pen_classifier.py",
+   "bytes": 1065
+  },
+  {
+   "path": "stallion/bina04-detector-role-isolation-95k-20261103/production.py",
+   "source": "/home/rezo/bina04-detector-role-isolation-95k-20261103/production.py",
+   "bytes": 1676
+  },
+  {
+   "path": "stallion/bina04-document-handoff-95k-20261103/document.py",
+   "source": "/home/rezo/bina04-document-handoff-95k-20261103/document.py",
+   "bytes": 5647
+  },
+  {
+   "path": "stallion/bina04-document-handoff-95k-20261103/numeric_repairs.py",
+   "source": "/home/rezo/bina04-document-handoff-95k-20261103/numeric_repairs.py",
+   "bytes": 7574
+  },
+  {
+   "path": "stallion/bina04-document-handoff-95k-20261103/production.py",
+   "source": "/home/rezo/bina04-document-handoff-95k-20261103/production.py",
+   "bytes": 2693
+  },
+  {
+   "path": "stallion/bina04-document-handoff-95k-20261103/region_reads.py",
+   "source": "/home/rezo/bina04-document-handoff-95k-20261103/region_reads.py",
+   "bytes": 3952
+  },
+  {
+   "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/learned_backend.py",
+   "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/learned_backend.py",
+   "bytes": 1737
+  },
+  {
+   "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/run/checkpoints/step-001000.pt",
+   "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/run/checkpoints/step-001000.pt",
+   "bytes": 153922431
+  },
+  {
+   "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/crop_line_guard.py",
+   "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/crop_line_guard.py",
+   "bytes": 2397
+  },
+  {
+   "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/input_policy_v3.py",
+   "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/input_policy_v3.py",
+   "bytes": 5112
+  },
+  {
+   "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/pipeline_v3.py",
+   "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/pipeline_v3.py",
+   "bytes": 18321
+  },
+  {
+   "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/table_client.py",
+   "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/table_client.py",
+   "bytes": 1440
+  },
+  {
+   "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/line_contract_reader.py",
+   "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/line_contract_reader.py",
+   "bytes": 5263
+  },
+  {
+   "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/pipeline_crop_fix.py",
+   "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/pipeline_crop_fix.py",
+   "bytes": 6508
+  },
+  {
+   "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/table_cells.py",
+   "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/table_cells.py",
+   "bytes": 7008
+  },
+  {
+   "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_builder.py",
+   "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_builder.py",
+   "bytes": 10084
+  },
+  {
+   "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_pipeline.py",
+   "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_pipeline.py",
+   "bytes": 4315
+  },
+  {
+   "path": "stallion/bina04-header-marks-95k-20261103/CALIBRATION.json",
+   "source": "/home/rezo/bina04-header-marks-95k-20261103/CALIBRATION.json",
+   "bytes": 15542
+  },
+  {
+   "path": "stallion/bina04-header-marks-95k-20261103/candidate.py",
+   "source": "/home/rezo/bina04-header-marks-95k-20261103/candidate.py",
+   "bytes": 6479
+  },
+  {
+   "path": "stallion/bina04-header-marks-95k-20261103/production.py",
+   "source": "/home/rezo/bina04-header-marks-95k-20261103/production.py",
+   "bytes": 961
+  },
+  {
+   "path": "stallion/bina04-header-marks-95k-20261103/serializer.py",
+   "source": "/home/rezo/bina04-header-marks-95k-20261103/serializer.py",
+   "bytes": 7021
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/models/det.onnx",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/models/det.onnx",
+   "bytes": 389948
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/__init__.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/__init__.py",
+   "bytes": 1263
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/det_post.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/det_post.py",
+   "bytes": 6030
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/document.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/document.py",
+   "bytes": 18586
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/layout_post.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/layout_post.py",
+   "bytes": 14411
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/line_mask.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/line_mask.py",
+   "bytes": 5600
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/math.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/math.py",
+   "bytes": 6338
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/models.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/models.py",
+   "bytes": 2702
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/order_post.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/order_post.py",
+   "bytes": 12544
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/parallel.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/parallel.py",
+   "bytes": 7766
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/pipeline.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/pipeline.py",
+   "bytes": 45649
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/rec_preproc.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/rec_preproc.py",
+   "bytes": 5500
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/rtl.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/rtl.py",
+   "bytes": 6097
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/textlayer.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/textlayer.py",
+   "bytes": 14861
+  },
+  {
+   "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/vocab.py",
+   "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/vocab.py",
+   "bytes": 11395
+  },
+  {
+   "path": "stallion/bina04-ink-loss-guard-95k-20261104/production.py",
+   "source": "/home/rezo/bina04-ink-loss-guard-95k-20261104/production.py",
+   "bytes": 1590
+  },
+  {
+   "path": "stallion/bina04-line-detector-v2-20261030/detector-run/checkpoints/step-000500.pt",
+   "source": "/home/rezo/bina04-line-detector-v2-20261030/detector-run/checkpoints/step-000500.pt",
+   "bytes": 153922431
+  },
+  {
+   "path": "stallion/bina04-line-geometry-95k-20261103/assembly.py",
+   "source": "/home/rezo/bina04-line-geometry-95k-20261103/assembly.py",
+   "bytes": 2786
+  },
+  {
+   "path": "stallion/bina04-line-geometry-95k-20261103/form_rows.py",
+   "source": "/home/rezo/bina04-line-geometry-95k-20261103/form_rows.py",
+   "bytes": 2236
+  },
+  {
+   "path": "stallion/bina04-line-geometry-95k-20261103/geometry_fix.py",
+   "source": "/home/rezo/bina04-line-geometry-95k-20261103/geometry_fix.py",
+   "bytes": 5655
+  },
+  {
+   "path": "stallion/bina04-line-geometry-95k-20261103/production.py",
+   "source": "/home/rezo/bina04-line-geometry-95k-20261103/production.py",
+   "bytes": 1241
+  },
+  {
+   "path": "stallion/bina04-numeric-math-recognizer-95k-20261102/evaluate.py",
+   "source": "/home/rezo/bina04-numeric-math-recognizer-95k-20261102/evaluate.py",
+   "bytes": 15324
+  },
+  {
+   "path": "stallion/bina04-numeric-math-recognizer-95k-20261102/recognizer/selected.pt",
+   "source": "/home/rezo/bina04-numeric-math-recognizer-95k-20261102/recognizer/selected.pt",
+   "bytes": 131255668
+  },
+  {
+   "path": "stallion/bina04-numeric-routing-95k-20261103/consumer.py",
+   "source": "/home/rezo/bina04-numeric-routing-95k-20261103/consumer.py",
+   "bytes": 4335
+  },
+  {
+   "path": "stallion/bina04-numeric-routing-95k-20261103/formula.py",
+   "source": "/home/rezo/bina04-numeric-routing-95k-20261103/formula.py",
+   "bytes": 4391
+  },
+  {
+   "path": "stallion/bina04-numeric-routing-95k-20261103/formula_private.py",
+   "source": "/home/rezo/bina04-numeric-routing-95k-20261103/formula_private.py",
+   "bytes": 5993
+  },
+  {
+   "path": "stallion/bina04-numeric-routing-95k-20261103/route.py",
+   "source": "/home/rezo/bina04-numeric-routing-95k-20261103/route.py",
+   "bytes": 7778
+  },
+  {
+   "path": "stallion/bina04-order-application-debug-20261102/fixed_order.py",
+   "source": "/home/rezo/bina04-order-application-debug-20261102/fixed_order.py",
+   "bytes": 5790
+  },
+  {
+   "path": "stallion/bina04-order-form-batch-20261101/recognizer/weights-001000.pt",
+   "source": "/home/rezo/bina04-order-form-batch-20261101/recognizer/weights-001000.pt",
+   "bytes": 131256958
+  },
+  {
+   "path": "stallion/bina04-order-labels-current-95k-20261102/CALIBRATION.json",
+   "source": "/home/rezo/bina04-order-labels-current-95k-20261102/CALIBRATION.json",
+   "bytes": 187
+  },
+  {
+   "path": "stallion/bina04-order-labels-current-95k-20261102/order_only.py",
+   "source": "/home/rezo/bina04-order-labels-current-95k-20261102/order_only.py",
+   "bytes": 1888
+  },
+  {
+   "path": "stallion/bina04-order-labels-current-95k-20261102/production.py",
+   "source": "/home/rezo/bina04-order-labels-current-95k-20261102/production.py",
+   "bytes": 1740
+  },
+  {
+   "path": "stallion/bina04-order-predicted-20261101/order_nodes.py",
+   "source": "/home/rezo/bina04-order-predicted-20261101/order_nodes.py",
+   "bytes": 4931
+  },
+  {
+   "path": "stallion/bina04-order-predicted-20261101/pred_order_model.py",
+   "source": "/home/rezo/bina04-order-predicted-20261101/pred_order_model.py",
+   "bytes": 2771
+  },
+  {
+   "path": "stallion/bina04-order-salvage-20261101/evaluate_batch.py",
+   "source": "/home/rezo/bina04-order-salvage-20261101/evaluate_batch.py",
+   "bytes": 8459
+  },
+  {
+   "path": "stallion/bina04-order-salvage-20261101/row_builder.py",
+   "source": "/home/rezo/bina04-order-salvage-20261101/row_builder.py",
+   "bytes": 10084
+  },
+  {
+   "path": "stallion/bina04-page-assembly-95k-20261103/CALIBRATION.json",
+   "source": "/home/rezo/bina04-page-assembly-95k-20261103/CALIBRATION.json",
+   "bytes": 649
+  },
+  {
+   "path": "stallion/bina04-page-assembly-95k-20261103/assembly.py",
+   "source": "/home/rezo/bina04-page-assembly-95k-20261103/assembly.py",
+   "bytes": 2809
+  },
+  {
+   "path": "stallion/bina04-page-assembly-95k-20261103/production.py",
+   "source": "/home/rezo/bina04-page-assembly-95k-20261103/production.py",
+   "bytes": 905
+  },
+  {
+   "path": "stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json",
+   "source": "/home/rezo/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json",
+   "bytes": 20007
+  },
+  {
+   "path": "stallion/bina04-pagepath-v3-round3-clean-20261031/fusion_v3.py",
+   "source": "/home/rezo/bina04-pagepath-v3-round3-clean-20261031/fusion_v3.py",
+   "bytes": 4521
+  },
+  {
+   "path": "stallion/bina04-pagepath-v3-round3-clean-20261031/input_policy_v3.py",
+   "source": "/home/rezo/bina04-pagepath-v3-round3-clean-20261031/input_policy_v3.py",
+   "bytes": 5112
+  },
+  {
+   "path": "stallion/bina04-pagepath-v3-round3-clean-20261031/pipeline_v3.py",
+   "source": "/home/rezo/bina04-pagepath-v3-round3-clean-20261031/pipeline_v3.py",
+   "bytes": 18321
+  },
+  {
+   "path": "stallion/bina04-pagepath-v3-round3-clean-20261031/run/weights-095000.pt",
+   "source": "/home/rezo/bina04-pagepath-v3-round3-clean-20261031/run/weights-095000.pt",
+   "bytes": 131256958
+  },
+  {
+   "path": "stallion/bina04-pagepath-v3-round3-clean-20261031/text_contract.py",
+   "source": "/home/rezo/bina04-pagepath-v3-round3-clean-20261031/text_contract.py",
+   "bytes": 1030
+  },
+  {
+   "path": "stallion/bina04-print-textgate-20261102/CALIBRATION.json",
+   "source": "/home/rezo/bina04-print-textgate-20261102/CALIBRATION.json",
+   "bytes": 2509
+  },
+  {
+   "path": "stallion/bina04-print-textgate-20261102/include-model.pkl",
+   "source": "/home/rezo/bina04-print-textgate-20261102/include-model.pkl",
+   "bytes": 7982847
+  },
+  {
+   "path": "stallion/bina04-print-textgate-20261102/include_model.py",
+   "source": "/home/rezo/bina04-print-textgate-20261102/include_model.py",
+   "bytes": 1878
+  },
+  {
+   "path": "stallion/bina04-print-textgate-20261102/layer_runtime.py",
+   "source": "/home/rezo/bina04-print-textgate-20261102/layer_runtime.py",
+   "bytes": 4969
+  },
+  {
+   "path": "stallion/bina04-print-textgate-20261102/order_adapter.py",
+   "source": "/home/rezo/bina04-print-textgate-20261102/order_adapter.py",
+   "bytes": 940
+  },
+  {
+   "path": "stallion/bina04-print-textgate-20261102/production.py",
+   "source": "/home/rezo/bina04-print-textgate-20261102/production.py",
+   "bytes": 1372
+  },
+  {
+   "path": "stallion/bina04-print-textgate-20261102/run/best.json",
+   "source": "/home/rezo/bina04-print-textgate-20261102/run/best.json",
+   "bytes": 2157
+  },
+  {
+   "path": "stallion/bina04-print-textgate-20261102/run/selected-printed-recognizer.pt",
+   "source": "/home/rezo/bina04-print-textgate-20261102/run/selected-printed-recognizer.pt",
+   "bytes": 131256937
+  },
+  {
+   "path": "stallion/bina04-printed-form-fix-20261102/CALIBRATION.json",
+   "source": "/home/rezo/bina04-printed-form-fix-20261102/CALIBRATION.json",
+   "bytes": 1591
+  },
+  {
+   "path": "stallion/bina04-printed-form-fix-20261102/SERIALIZED-LEXICON.json",
+   "source": "/home/rezo/bina04-printed-form-fix-20261102/SERIALIZED-LEXICON.json",
+   "bytes": 2644
+  },
+  {
+   "path": "stallion/bina04-printed-form-fix-20261102/candidate_apply.py",
+   "source": "/home/rezo/bina04-printed-form-fix-20261102/candidate_apply.py",
+   "bytes": 4498
+  },
+  {
+   "path": "stallion/bina04-printed-form-fix-20261102/label_model.py",
+   "source": "/home/rezo/bina04-printed-form-fix-20261102/label_model.py",
+   "bytes": 1684
+  },
+  {
+   "path": "stallion/bina04-printed-form-fix-20261102/printed-label-model.pkl",
+   "source": "/home/rezo/bina04-printed-form-fix-20261102/printed-label-model.pkl",
+   "bytes": 12109247
+  },
+  {
+   "path": "stallion/bina04-printed-form-fix-20261102/production.py",
+   "source": "/home/rezo/bina04-printed-form-fix-20261102/production.py",
+   "bytes": 4248
+  },
+  {
+   "path": "stallion/bina04-printed-layer-training-20261102/run/weights-002000.pt",
+   "source": "/home/rezo/bina04-printed-layer-training-20261102/run/weights-002000.pt",
+   "bytes": 132814787
+  },
+  {
+   "path": "stallion/bina04-quality-route-printscope-95k-20261103/QUALITY-POLICY.json",
+   "source": "/home/rezo/bina04-quality-route-printscope-95k-20261103/QUALITY-POLICY.json",
+   "bytes": 1590
+  },
+  {
+   "path": "stallion/bina04-quality-route-printscope-95k-20261103/print_scope.py",
+   "source": "/home/rezo/bina04-quality-route-printscope-95k-20261103/print_scope.py",
+   "bytes": 1087
+  },
+  {
+   "path": "stallion/bina04-quality-route-printscope-95k-20261103/production.py",
+   "source": "/home/rezo/bina04-quality-route-printscope-95k-20261103/production.py",
+   "bytes": 3428
+  },
+  {
+   "path": "stallion/bina04-quality-route-printscope-95k-20261103/quality.py",
+   "source": "/home/rezo/bina04-quality-route-printscope-95k-20261103/quality.py",
+   "bytes": 1947
+  },
+  {
+   "path": "stallion/bina04-real-degraded-95k-20261103/recognizer/selected.pt",
+   "source": "/home/rezo/bina04-real-degraded-95k-20261103/recognizer/selected.pt",
+   "bytes": 131254078
+  },
+  {
+   "path": "stallion/bina04-small-crossing-rows-20261102/crossing_rows.py",
+   "source": "/home/rezo/bina04-small-crossing-rows-20261102/crossing_rows.py",
+   "bytes": 4043
+  },
+  {
+   "path": "stallion/bina04-small-crossing-rows-20261102/runtime.py",
+   "source": "/home/rezo/bina04-small-crossing-rows-20261102/runtime.py",
+   "bytes": 2983
+  },
+  {
+   "path": "stallion/bina04-small-crossing-rows-20261102/runtime_release.py",
+   "source": "/home/rezo/bina04-small-crossing-rows-20261102/runtime_release.py",
+   "bytes": 1162
+  },
+  {
+   "path": "stallion/bina04-small-crossing-rows-20261102/scale_utils.py",
+   "source": "/home/rezo/bina04-small-crossing-rows-20261102/scale_utils.py",
+   "bytes": 1741
+  },
+  {
+   "path": "stallion/bina04-smallmarks-celllabels-95k-20261102/decompose.py",
+   "source": "/home/rezo/bina04-smallmarks-celllabels-95k-20261102/decompose.py",
+   "bytes": 903
+  },
+  {
+   "path": "stallion/bina04-surya25-order-20261101/role-model/weights-003000.pt",
+   "source": "/home/rezo/bina04-surya25-order-20261101/role-model/weights-003000.pt",
+   "bytes": 5093599
+  },
+  {
+   "path": "stallion/bina04-surya25-order-20261101/role_model.py",
+   "source": "/home/rezo/bina04-surya25-order-20261101/role_model.py",
+   "bytes": 4264
+  },
+  {
+   "path": "stallion/bina04-surya25-order-20261101/role_nodes.py",
+   "source": "/home/rezo/bina04-surya25-order-20261101/role_nodes.py",
+   "bytes": 1697
+  },
+  {
+   "path": "stallion/bina04-table-asides-95k-20261104/production.py",
+   "source": "/home/rezo/bina04-table-asides-95k-20261104/production.py",
+   "bytes": 616
+  },
+  {
+   "path": "stallion/bina04-table-asides-95k-20261104/table_asides.py",
+   "source": "/home/rezo/bina04-table-asides-95k-20261104/table_asides.py",
+   "bytes": 3046
+  },
+  {
+   "path": "stallion/indra-scale/geometry.py",
+   "source": "/tmp/bina04-indra-scale-20261029/geometry.py",
+   "bytes": 1935
+  },
+  {
+   "path": "stallion/indra-scale/gpu_eval_runtime.py",
+   "source": "/tmp/bina04-indra-scale-20261029/gpu_eval_runtime.py",
+   "bytes": 2267
+  },
+  {
+   "path": "stallion/indra-scale/recognizer.py",
+   "source": "/tmp/bina04-indra-scale-20261029/recognizer.py",
+   "bytes": 3455
+  },
+  {
+   "path": "stallion/indra-scale/regional_pipeline.py",
+   "source": "/tmp/bina04-indra-scale-20261029/regional_pipeline.py",
+   "bytes": 7967
+  },
+  {
+   "path": "stallion/indra-scale/regional_pipeline_v2.py",
+   "source": "/tmp/bina04-indra-scale-20261029/regional_pipeline_v2.py",
+   "bytes": 8415
+  },
+  {
+   "path": "stallion/indra-scale/routing-rule.json",
+   "source": "/tmp/bina04-indra-scale-20261029/routing-rule.json",
+   "bytes": 17922
+  },
+  {
+   "path": "stallion/indra-scale/routing_modes.py",
+   "source": "/tmp/bina04-indra-scale-20261029/routing_modes.py",
+   "bytes": 2548
+  },
+  {
+   "path": "stallion/indra-scale/rtl_reader.py",
+   "source": "/tmp/bina04-indra-scale-20261029/rtl_reader.py",
+   "bytes": 2892
+  },
+  {
+   "path": "stallion/indra-scale/scoringTT/shenavasanj.py",
+   "source": "/tmp/bina04-indra-scale-20261029/scoringTT/shenavasanj.py",
+   "bytes": 5126
+  },
+  {
+   "path": "stallion/indra-scale/visual_token.py",
+   "source": "/tmp/bina04-indra-scale-20261029/visual_token.py",
+   "bytes": 491
+  }
+ ],
+ "rewritten_text_files": [
+  "stallion/bina04-assembly-audit-95k-20261104/production.py",
+  "stallion/bina04-block-fields-95k-20261103/production.py",
+  "stallion/bina04-cell-rowcores-95k-20261103/candidate.py",
+  "stallion/bina04-cell-rowcores-95k-20261103/production.py",
+  "stallion/bina04-column-assembly-95k-20261103/production.py",
+  "stallion/bina04-detector-role-isolation-95k-20261103/production.py",
+  "stallion/bina04-document-handoff-95k-20261103/production.py",
+  "stallion/bina04-document-handoff-95k-20261103/region_reads.py",
+  "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/crop_line_guard.py",
+  "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/pipeline_v3.py",
+  "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/line_contract_reader.py",
+  "stallion/bina04-header-marks-95k-20261103/candidate.py",
+  "stallion/bina04-header-marks-95k-20261103/production.py",
+  "stallion/bina04-header-marks-95k-20261103/serializer.py",
+  "stallion/bina04-ink-loss-guard-95k-20261104/production.py",
+  "stallion/bina04-line-geometry-95k-20261103/production.py",
+  "stallion/bina04-numeric-math-recognizer-95k-20261102/evaluate.py",
+  "stallion/bina04-numeric-routing-95k-20261103/consumer.py",
+  "stallion/bina04-numeric-routing-95k-20261103/formula_private.py",
+  "stallion/bina04-order-application-debug-20261102/fixed_order.py",
+  "stallion/bina04-order-labels-current-95k-20261102/order_only.py",
+  "stallion/bina04-order-labels-current-95k-20261102/production.py",
+  "stallion/bina04-order-salvage-20261101/evaluate_batch.py",
+  "stallion/bina04-page-assembly-95k-20261103/production.py",
+  "stallion/bina04-pagepath-v3-round3-clean-20261031/pipeline_v3.py",
+  "stallion/bina04-pagepath-v3-round3-clean-20261031/text_contract.py",
+  "stallion/bina04-print-textgate-20261102/include_model.py",
+  "stallion/bina04-print-textgate-20261102/layer_runtime.py",
+  "stallion/bina04-print-textgate-20261102/order_adapter.py",
+  "stallion/bina04-print-textgate-20261102/production.py",
+  "stallion/bina04-print-textgate-20261102/run/best.json",
+  "stallion/bina04-printed-form-fix-20261102/CALIBRATION.json",
+  "stallion/bina04-printed-form-fix-20261102/production.py",
+  "stallion/bina04-quality-route-printscope-95k-20261103/print_scope.py",
+  "stallion/bina04-quality-route-printscope-95k-20261103/production.py",
+  "stallion/bina04-small-crossing-rows-20261102/runtime.py",
+  "stallion/bina04-table-asides-95k-20261104/production.py"
+ ],
+ "placeholder": "__BINA_ROOT__",
+ "layout_model": {
+  "repo": "datalab-to/surya_layout2",
+  "revision": "0aee81d5fd9275c0582e545bf3a56944b1e75679"
+ }
+}

README.md

@@ -0,0 +1,111 @@
+---
+language:
+- fa
+license: other
+license_name: bina-0.4-rizeh
+pipeline_tag: image-to-text
+tags:
+- bina
+- ocr
+- persian
+- farsi
+- handwriting
+- printed-text
+- ctc
+- document-understanding
+datasets:
+- Reza2kn/persian-handwriting-pages-3.69m
+- Reza2kn/persian-printed-ocr-3.5m
+---
+
+# Bina 0.4 Rizeh
+
+Bina 0.4 Rizeh is the small, fast member of the Bina 0.4 Persian OCR family. It reads whole pages (printed text, handwriting, forms, screenshots, letters, newspapers and simple math). Its core is about 71M parameters: a 38.5M line detector (fine-tuned Surya text detector) and a 32.8M CTC line recognizer distilled from [Bina 0.4 Koochik](https://huggingface.co/Reza2kn/Bina-0.4-Koochik). They run inside a page pipeline:
+
+1. **Layout**: Surya layout (`datalab-to/surya_layout2`, pinned revision, downloaded on first run).
+2. **Lines**: fine-tuned Surya-architecture line detectors (body text, printed layer, handwriting page path).
+3. **Reading**: the 95k-step Indra CTC student for body text, plus specialist readers with the same architecture: printed/form labels, short form cells, numeric and math, and degraded scans.
+4. **Assembly**: row consolidation, column and table ordering, printed-label admission and page-level ordering.
+
+This repository is the complete production pipeline exactly as it scored below, not just a checkpoint: all weights, calibrations and code are bundled.
+
+## Quick run
+
+```bash
+hf download Reza2kn/Bina-0.4-Rizeh --local-dir Bina-0.4-Rizeh
+cd Bina-0.4-Rizeh
+pip install -r requirements.txt          # Python 3.10, NVIDIA GPU recommended
+python rizeh.py examples/demo.png --out results
+python rizeh.py scan1.jpg scan2.png document.pdf --out results
+```
+
+The first run binds the bundled paths to your checkout (it edits the bundled text files in place, and reruns automatically if you move the folder) and downloads the pinned Surya layout model. For each page you get `<name>.txt` with the page text in reading order, and `<name>.json` with every line's text, box, layer (`printed` / handwriting / body) and confidence. PDFs are rendered at 200 dpi.
+
+From Python:
+
+```python
+from PIL import Image
+from rizeh import Rizeh
+ocr = Rizeh("cuda")
+result = ocr(Image.open("page.jpg").convert("RGB"))
+print(result["text"])
+```
+
+## Scores
+
+Character error rate (CER, %) on the internal gates this release was frozen on. Lower is better.
+
+| Gate | CER |
+|---|---:|
+| 193 cropped regions, overall | 9.39 |
+| · forms | 3.80 |
+| · screenshots | 4.65 |
+| · prose | 9.15 |
+| · math | 12.09 |
+| · newspaper | 13.49 |
+| · tables | 19.63 |
+| 50 handwriting pages, handwritten body only | 3.97 |
+| 50 handwriting pages, full page incl. printed form text | 10.89 |
+| 338 printed lines | 1.76 |
+| 7 real scanned documents (printed regions) | 10.61 |
+| Scanned official letter | 2.12 |
+
+[Persian OCR Triple Threat](https://huggingface.co/spaces/Reza2kn/PersianOCR-TripleThreat), same 95k recognizer, measured 2026-11-03 before the final assembly fixes (lower is better):
+
+| Split | Rows | WER | CER | S³ | Triple Threat |
+|---|---:|---:|---:|---:|---:|
+| Printed | 6,669 | 4.70 | 1.37 | 5.88 | 4.74 |
+| Handwriting | 6,669 | 18.48 | 13.01 | 15.19 | 15.41 |
+| Equal-split overall | 13,338 | 11.59 | 7.19 | 10.54 | **10.08** |
+
+## Limitations
+
+- **Historical manuscripts do not work.** On HPHD (historical nastaliq and shekasteh manuscripts) the recognizer is at 67% CER even on oracle line crops, and full pages come out as fluent-looking nonsense. Do not use it for manuscripts.
+- **Tables are weak** (about 20% CER), especially dense or low-resolution tables with handwriting over the cells.
+- **Reading order** on busy layouts (multi-column newspapers, exam sheets with many labels) is the largest remaining error source; the characters are usually right but lines can come out in a different order.
+- **Faint or coloured printed labels** (light-blue Nastaliq on forms) are often missed.
+- The pipeline has many stages tuned together. Swapping one component (for example a newer recognizer) without retuning the stages after it makes results worse; this was measured, not assumed.
+- Persian-first. Latin text and digits are read, but other languages were not a goal.
+- Confidences are model scores, not calibrated probabilities.
+
+## Training and provenance
+
+- Recognizer: Indra CTC, 32,796,250 parameters, 20 blocks, 1,617-symbol vocabulary; 95,000 training steps, distilled from Bina 0.4 Koochik checkpoint 18,404.
+- Specialist readers (printed labels, form cells, numeric/math, degraded scans) are 32.8M fine-tunes of the same recognizer; the two line detectors are 38.5M fine-tunes of Surya's text detector, and the printed-text head adds 0.39M.
+- Checkpoint selection used internal development sets only; the gates above were never used to pick weights.
+- `MANIFEST.json` lists every bundled file with its original location and size. Under `stallion/` the files are organized by experiment directory exactly as they ran in production; `rizeh.py` is the entry point.
+
+## License
+
+Bina recognizer weights and code: CC BY-NC 4.0, matching the non-commercial training data. Line-detector weights are fine-tunes of Surya (datalab) detection weights, and the Surya layout model is downloaded from datalab; both stay under [Surya's model license](https://github.com/datalab-to/surya), which limits commercial use.
+
+## فارسی
+
+بینا ۰٫۴ ریزه عضو کوچک و سریع خانواده OCR فارسی بینا ۰٫۴ است. یک مدل CTC با ۳۲٫۸ میلیون پارامتر که از بینا ۰٫۴ کوچیک تقطیر شده، درون یک خط‌لوله کامل صفحه کار می‌کند: چیدمان صفحه، تشخیص خط، خواندن متن چاپی و دست‌نویس، فرم‌ها، اعداد و چیدن متن به ترتیب خواندن.
+
+```bash
+pip install -r requirements.txt
+python rizeh.py page.jpg --out results
+```
+
+محدودیت مهم: روی نسخه‌های خطی تاریخی (نستعلیق و شکسته قدیمی) قابل استفاده نیست، و جدول‌ها و ترتیب خواندن صفحه‌های شلوغ هنوز ضعیف‌اند.

requirements.txt

@@ -0,0 +1,17 @@
+# Tested with Python 3.10 on an NVIDIA RTX 5080.
+torch==2.10.0
+torchvision==0.25.0
+surya-ocr==0.22.1
+transformers==5.14.1
+numpy==1.26.4
+scipy==1.15.3
+scikit-learn==1.7.2
+joblib==1.5.3
+rapidfuzz==3.14.5
+jiwer==4.0.0
+opencv-python-headless==4.11.0.86
+onnxruntime==1.23.2
+pillow==10.4.0
+pyarrow==23.0.1
+huggingface_hub>=1.0
+pypdfium2

rizeh.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Bina 0.4 Rizeh: Persian page OCR (Surya layout + 95k Indra CTC student + printed/table/numeric experts and assembly).
+
+usage: python rizeh.py page.jpg [more.png doc.pdf ...] [--out results] [--device cuda|cpu]
+"""
+import argparse,json,os,sys,time,importlib.util
+from pathlib import Path
+ROOT=Path(__file__).resolve().parent;ST=ROOT/'stallion';MARK=ROOT/'.bina-root'
+LAYOUT_REPO,LAYOUT_REV='datalab-to/surya_layout2','0aee81d5fd9275c0582e545bf3a56944b1e75679'
+TEXT=('.py','.json','.yml','.yaml','.txt','.jsonl')
+
+def setup():
+ """Point every bundled path at this checkout (idempotent; reruns after a move) and fetch the pinned layout model."""
+ here=str(ROOT);old=MARK.read_text().strip()if MARK.exists()else'__BINA_ROOT__'
+ if old!=here:
+  n=0
+  for p in ST.rglob('*'):
+   if p.is_file()and p.suffix in TEXT:
+    s=p.read_text(encoding='utf-8')
+    if old in s:p.write_text(s.replace(old,here),encoding='utf-8');n+=1
+  MARK.write_text(here);print(f'[rizeh] bound {n} files to {here}',file=sys.stderr)
+ snap=ST/'indra-scale/hf/hub/models--datalab-to--surya_layout2/snapshots'/LAYOUT_REV
+ if not(snap/'rfdetr_layout.pth').exists():
+  from huggingface_hub import snapshot_download
+  print('[rizeh] downloading layout model',LAYOUT_REPO,LAYOUT_REV,file=sys.stderr)
+  snapshot_download(LAYOUT_REPO,revision=LAYOUT_REV,cache_dir=str(ST/'indra-scale/hf/hub'))
+ return snap
+
+def pages(path):
+ p=Path(path)
+ if p.suffix.lower()=='.pdf':
+  import pypdfium2 as pdfium
+  doc=pdfium.PdfDocument(str(p))
+  for i in range(len(doc)):yield f'{p.stem}-p{i+1:03d}',doc[i].render(scale=200/72).to_pil().convert('RGB')
+ else:
+  from PIL import Image
+  yield p.stem,Image.open(p).convert('RGB')
+
+class Rizeh:
+ def __init__(self,device='cuda'):
+  snap=setup()
+  os.environ.setdefault('GATE_DEVICE',device);os.environ.setdefault('FAST_DETECTOR_DEVICE','cpu');os.environ.setdefault('DETECTOR_IMAGE_CHUNK_HEIGHT','1400');os.environ['FAST_ORDER_MODEL_CHECKPOINT']=str(snap/'order')
+  P=ST/'indra-scale';sys.path[:0]=[str(P),str(ST/'bina04-pagepath-v3-round3-clean-20261031'),str(P/'scoringTT')]
+  spec=importlib.util.spec_from_file_location('bina04_rizeh_runtime',ST/'bina04-ink-loss-guard-95k-20261104/production.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
+  self.engine=m.Runtime(device)
+  from surya.fast_layout.server import LayoutEngine
+  self.layout=LayoutEngine(checkpoint=str(snap))
+ def __call__(self,image,name='page'):
+  raw=self.layout.run_batch([image],[dict(threshold=None,use_order=True)])[0].model_dump(mode='json')
+  out=self.engine(image,dict(raw_layout=raw),'page',name)
+  lines=[dict(text=l.get('text',''),layer=l.get('layer'),box=[round(float(v),1)for v in l['box']],confidence=l.get('confidence'))for l in out.get('lines',[])if l.get('box')]
+  return dict(text=out['text'],lines=lines)
+ def close(self):self.engine.close()
+
+def main():
+ ap=argparse.ArgumentParser(description=__doc__,formatter_class=argparse.RawDescriptionHelpFormatter);ap.add_argument('inputs',nargs='+');ap.add_argument('--out',default='rizeh-out');ap.add_argument('--device',default='cuda',choices=['cuda','cpu']);a=ap.parse_args()
+ O=Path(a.out);O.mkdir(parents=True,exist_ok=True);r=Rizeh(a.device)
+ for src in a.inputs:
+  for name,im in pages(src):
+   t=time.perf_counter();res=r(im,name);res.update(source=str(src),seconds=round(time.perf_counter()-t,2),model='Bina-0.4-Rizeh')
+   (O/f'{name}.json').write_text(json.dumps(res,ensure_ascii=False,indent=1),encoding='utf-8');(O/f'{name}.txt').write_text(res['text'],encoding='utf-8')
+   print(f'{name}: {len(res["lines"])} lines, {res["seconds"]}s -> {O/name}.txt',flush=True)
+ r.close()
+if __name__=='__main__':main()

stallion/bina04-assembly-audit-95k-20261104/graphic_blocks.py

@@ -0,0 +1,51 @@
+"""Atomic supported text blocks in illustrated Latin documents; frozen reader."""
+import copy,unicodedata
+import numpy as np
+
+def area(b):return max(0,b[2]-b[0])*max(0,b[3]-b[1])
+def ov(a,b):return area([max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])])/max(1,area(a))
+def union(bs):return[min(b[0]for b in bs),min(b[1]for b in bs),max(b[2]for b in bs),max(b[3]for b in bs)]
+def apply(old,image):
+ out=copy.deepcopy(old);ls=out.get('lines',[])
+ if any(l.get('layer')=='handwriting'for l in ls)or any(r.get('table_trace',{}).get('grid')for r in out.get('regions',[])):return old
+ text=''.join(l['text']for l in ls if l.get('confidence',0)>=.9);letters=[c for c in text if c.isalpha()];latin=sum('LATIN'in unicodedata.name(c,'')for c in letters)
+ if len(letters)<60 or latin/max(1,len(letters))<.8:return old
+ W,H=image.size;a=np.asarray(image.convert('RGB'));sat=(a.max(2).astype(float)-a.min(2));colour=float((sat>30).mean())
+ layouts=[r['bbox']for r in out.get('raw_layout',{}).get('bboxes',[])if r.get('label')in('Text','SectionHeader','Caption','ListItem')and'bbox'in r]
+ if colour<.08 or len(layouts)<3:return old
+ if ''.join(out['text'].split())!=''.join(''.join(l['text'].split())for l in ls):return old
+ med=float(np.median([l['box'][3]-l['box'][1]for l in ls if l.get('confidence',0)>.94 and len(l['text'].strip())>=8]));assigned={};groups={}
+ for i,l in enumerate(ls):
+  if not l['text'].strip():continue
+  scores=[ov(l['box'],b)for b in layouts]
+  if scores and max(scores)>=.5:k=int(np.argmax(scores));groups.setdefault(k,[]).append(i);assigned[i]=k
+ unsupported=[i for i,l in enumerate(ls)if i not in assigned and l['text'].strip()and l.get('confidence',0)>=.9]
+ # Infer omitted text blocks from high-confidence aligned physical baselines.
+ for i in unsupported:
+  b=ls[i]['box'];found=None
+  for k,ids in groups.items():
+   if k<len(layouts):continue
+   bb=union([ls[j]['box']for j in ids]);x=max(0,min(b[2],bb[2])-max(b[0],bb[0]));gap=max(0,max(b[1],bb[1])-min(b[3],bb[3]))
+   if x>.55*min(b[2]-b[0],bb[2]-bb[0])and abs(b[0]-bb[0])<1.5*med and gap<2*med:found=k;break
+  k=found if found is not None else len(layouts)+i+1;groups.setdefault(k,[]).append(i);assigned[i]=k
+ drop=[]
+ for i,l in enumerate(ls):
+  if not l['text'].strip()or i in assigned:continue
+  if l.get('confidence',0)<.55 and len(''.join(l['text'].split()))<=8:drop.append(i)
+  else:groups[len(layouts)+len(ls)+i+1]=[i]
+ if not groups:return old
+ blocks=[dict(ids=sorted(ids,key=lambda i:(ls[i]['box'][1],ls[i]['box'][0])),box=union([ls[i]['box']for i in ids]))for ids in groups.values()]
+ def cut(blocks):
+  if len(blocks)<=1:return blocks
+  for axis in [1,0]:
+   intervals=sorted((z['box'][axis],z['box'][axis+2])for z in blocks);end=intervals[0][1];gaps=[]
+   for a,b in intervals[1:]:
+    if a-end>.25*med:gaps.append((a-end,(a+end)/2))
+    end=max(end,b)
+   if gaps:
+    _,mid=max(gaps);left=[z for z in blocks if z['box'][axis+2]<=mid];right=[z for z in blocks if z['box'][axis]>=mid]
+    if left and right and len(left)+len(right)==len(blocks):return cut(left)+cut(right)
+  return sorted(blocks,key=lambda z:(z['box'][1],z['box'][0]))
+ ordered=cut(blocks);ids=[i for z in ordered for i in z['ids']];newtext='\n\n'.join(ls[i]['text']for i in ids)
+ if newtext==out['text']:return old
+ out['lines']=[ls[i]for i in ids];out['text']=newtext;out['regions']=[dict(path='atomic_graphic_text_blocks',lines=[ls[i]for i in z['ids']],text='\n'.join(ls[i]['text']for i in z['ids']),box=z['box'])for z in ordered];out['graphic_block_assembly']=dict(colour_fraction=colour,latin_fraction=latin/len(letters),removed=[ls[i]for i in drop],blocks=ordered,confidence_threshold=.55,threshold_source='Existing frozen private-dev graphic-junk threshold; no193/TT tuning',reader_changed=False);return out

stallion/bina04-assembly-audit-95k-20261104/production.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+import sys,importlib.util
+R=Path(__file__).resolve().parent;sys.path.insert(0,str(R));from graphic_blocks import apply
+class Runtime:
+ def __init__(self,device='auto'):
+  s=importlib.util.spec_from_file_location('graphic_frozen_parent','__BINA_ROOT__/stallion/bina04-block-fields-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);self.raw=self.parent.raw;self.core=self.parent.core
+ def __call__(self,image,old,kind,name='',**kwargs):return apply(self.parent(image,old,kind,name,**kwargs),image)
+ def close(self):self.parent.close()

stallion/bina04-block-fields-95k-20261103/assembly_fix.py

@@ -0,0 +1,56 @@
+import copy
+import numpy as np,cv2
+
+def norm(s):return ''.join(s.split())
+def rule_bands(gray):
+ H,W=gray.shape;bw=cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY_INV,31,12);r=cv2.morphologyEx(bw,cv2.MORPH_OPEN,np.ones((1,max(25,int(W*.35))),np.uint8));ys=np.flatnonzero((r>0).sum(1)>.65*W);return [float(np.median(g))for g in np.split(ys,np.flatnonzero(np.diff(ys)>1)+1)if len(g)]if len(ys)else[]
+def apply(old,image,reader=None):
+ out=copy.deepcopy(old);ls=out.get('lines',[]);W,H=image.size
+ if len(ls)<3 or any(l.get('layer')=='handwriting'for l in ls)or any(r.get('table_trace',{}).get('grid')for r in out.get('regions',[])):return out
+ if norm(''.join(l.get('text','')for l in ls))!=norm(out['text']):return out
+ gray=np.asarray(image.convert('L'));bounds=rule_bands(gray);hs=[l['box'][3]-l['box'][1]for l in ls if len(norm(l.get('text','')))>8 and l['confidence']>.8];med=float(np.median(hs))if hs else 0
+ if med<=0:return out
+ trace=dict(header_blocks=[],form_rows=[],split_boxes=[])
+ # Split a vertically merged ruling-band heading and content. Horizontal expansion
+ # is bounded by actual foreground in that row, not the detected partial x-extent.
+ if reader is not None and len(bounds)>=4:
+  for i,l in enumerate(ls[:]):
+   x,y,X,Y=l['box'];hh=Y-y
+   if hh<2.5*med or hh>.12*H or X-x<.4*W:continue
+   cells=[(a,b)for a,b in zip(bounds,bounds[1:])if a>=y-2 and b<=Y+2 and .5*med<b-a<3*med]
+   if not 2<=len(cells)<=4:continue
+   boxes=[]
+   for a,b in cells:
+    yy=max(0,int(a)+2);YY=min(H,int(b)-1);ink=(gray[yy:YY]<170).astype(np.uint8);hr=cv2.morphologyEx(ink,cv2.MORPH_OPEN,np.ones((1,max(30,int(.3*W))),np.uint8));vr=cv2.morphologyEx(ink,cv2.MORPH_OPEN,np.ones((max(8,int(.8*(YY-yy))),1),np.uint8));ink[(hr>0)|(vr>0)]=0;ys=np.flatnonzero(ink.sum(1)>3);xs=np.flatnonzero(ink.sum(0)>1)
+    if len(ys)<4 or len(xs)<8:break
+    boxes.append([max(0,int(xs[0])-2),max(0,yy+int(ys[0])-2),min(W,int(xs[-1])+3),min(H,yy+int(ys[-1])+3)])
+   if len(boxes)!=len(cells):continue
+   rr=reader(gray,boxes)
+   if any(r['confidence']<.65 or len(norm(r['text']))<4 for r in rr):continue
+   ls[i:i+1]=[dict(r,layer=l.get('layer'),source='ruling_band_multiline_split')for r in rr];trace['split_boxes'].append(dict(original=l['box'],boxes=boxes,original_text=l['text'],new_text=[r['text']for r in rr]));break
+ # Restore recovered printed fields to physical rows, regardless of admission layer.
+ if len(bounds)>=4:
+  assignment={}
+  for a,b in zip(bounds,bounds[1:]):
+   if not .6*med<b-a<3*med:continue
+   ids=[i for i,l in enumerate(ls)if norm(l.get('text','')) and a<=(l['box'][1]+l['box'][3])/2<=b and l['box'][3]-l['box'][1]<1.6*(b-a)]
+   if len(ids)<2:continue
+   if any(min(ls[i]['box'][2],ls[j]['box'][2])-max(ls[i]['box'][0],ls[j]['box'][0])>.2*min(ls[i]['box'][2]-ls[i]['box'][0],ls[j]['box'][2]-ls[j]['box'][0])for k,i in enumerate(ids)for j in ids[k+1:]):continue
+   order=sorted(ids,key=lambda i:-ls[i]['box'][2]);assignment.update({i:order for i in ids});trace['form_rows'].append(dict(band=[a,b],before=ids,after=order))
+  done=set();order=[]
+  for i in range(len(ls)):
+   if i in done:continue
+   ids=assignment.get(i,[i]);order+=ids;done.update(ids)
+  ls=[ls[i]for i in order]
+ # Side-by-side headers remain atomic blocks, read RTL. The reading convention
+ # is geometric; no document names, reference strings or page-specific IDs.
+ left=[i for i,l in enumerate(ls)if l['box'][2]<.42*W and l['box'][3]<.17*H and len(norm(l['text']))>=8 and l['confidence']>.75 and l['box'][3]-l['box'][1]<2*med]
+ right=[i for i,l in enumerate(ls)if l['box'][0]>.58*W and l['box'][3]<.17*H and len(norm(l['text']))>=8 and l['confidence']>.75 and l['box'][3]-l['box'][1]<2*med]
+ if len(left)>=2 and len(right)>=2:
+  ids=set(left+right);ordered=sorted(right,key=lambda i:(ls[i]['box'][1],-ls[i]['box'][2]))+sorted(left,key=lambda i:(ls[i]['box'][1],-ls[i]['box'][2]));new=[]
+  for i in range(len(ls)):
+   if i==min(ids):new+=ordered
+   if i not in ids:new.append(i)
+  trace['header_blocks']=dict(left_ids=left,right_ids=right,after=ordered,convention='right block then left block; each top-to-bottom');ls=[ls[i]for i in new]
+ if not trace['split_boxes'] and [l['text']for l in ls]==[l['text']for l in old['lines']]:return old
+ out['lines']=ls;out['regions']=[dict(path='physical_header_and_form_rows',layout=None,lines=[l],text=l['text'])for l in ls];out['text']='\n\n'.join(l['text']for l in ls);out['physical_block_fields']=trace;return out

stallion/bina04-block-fields-95k-20261103/production.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+import importlib.util,sys
+R=Path(__file__).resolve().parent;sys.path.insert(0,str(R))
+from assembly_fix import apply
+class Runtime:
+ def __init__(self,device='auto'):
+  s=importlib.util.spec_from_file_location('block_fields_parent','__BINA_ROOT__/stallion/bina04-quality-route-printscope-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);self.raw=self.parent.raw;self.core=self.parent.core
+  from pipeline_v3 import UnifiedReader
+  V=Path('__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031');self.split_reader=UnifiedReader(V/'run/weights-095000.pt',V/'data/vocab.json',device='cpu')
+ def __call__(self,image,old,kind,name='',**kwargs):
+  out=self.parent(image,old,kind,name,**kwargs);return apply(out,image,self.split_reader)if kind=='page'else out
+ def close(self):self.parent.close()

stallion/bina04-cell-rowcores-95k-20261103/candidate.py

@@ -0,0 +1,41 @@
+"""Attach detached small bands inside cells; preserve two full text rows."""
+import copy,sys,numpy as np
+from pathlib import Path
+sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-crop-picture-cells-95k-20261102');sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-crop-bandseg-95k-20261102')
+from cell_reader import read_cell,owned_hand_cells
+from bandseg import segment
+
+def rowcores(image):
+ boxes,trace=segment(image);boxes=sorted(boxes,key=lambda b:b[1]);merged=[];events=[]
+ for b in boxes:
+  if merged:
+   a=merged[-1];ha=a[3]-a[1];hb=b[3]-b[1];gap=max(0,b[1]-a[3]);horizontal=min(a[2],b[2])-max(a[0],b[0])
+   # A small detached dot/ascender band cannot be a second full writing row.
+   if min(ha,hb)<.6*max(ha,hb)and gap<=.5*max(ha,hb)and horizontal>0:
+    merged[-1]=[min(a[0],b[0]),min(a[1],b[1]),max(a[2],b[2]),max(a[3],b[3])];events.append(dict(first=a,second=b,reason='unequal core heights, near and horizontally overlapping'));continue
+  merged.append(list(b))
+ trace=dict(trace,cell_fragment_attachments=events);return merged,trace
+
+class Patch:
+ def __init__(self,reader,flip):self.reader=reader;self.flip=flip
+ def __call__(self,image,base,kind='crop'):
+  if kind!='crop':return base
+  out=copy.deepcopy(base);rgb=np.asarray(image.convert('RGB'));changed=[]
+  for region in out['regions']:
+   t=region.get('table_trace',{});cells=t.get('cell_debug',[])
+   if not t.get('grid')or not cells:continue
+   pen=region.get('table_cell_contract',{}).get('table_pen',False);owned,owner_trace=owned_hand_cells(rgb,cells)if pen else(None,None)
+   for index,c in enumerate(cells):
+    if len(c.get('lines',[]))<2:continue
+    heights=[l['box'][3]-l['box'][1]for l in c['lines']]
+    if min(heights)>=.6*max(heights):continue
+    pixels,box=owned[index]if pen else(rgb,c['bbox']);text,ll,trace=read_cell(self.reader,self.flip,pixels,box,rowcores,pen=False)
+    if not trace.get('proposal_trace',{}).get('cell_fragment_attachments'):continue
+    old=c['text'];c.update(text=text,lines=ll,cell_rowcores=trace);changed.append(dict(row=c['row'],col=c['col'],old=old,new=text,old_lines=len(heights),new_lines=len(ll)))
+   values={(c['row'],c['col']):c['text']for c in cells};rendered=[]
+   for i,row in enumerate(t['kept_rows']):
+    vals=[' '.join(values.get((row,col),'').split()).replace('|',r'\|')for col in t['kept_columns']];rendered.append('| '+' | '.join(vals)+' |')
+    if i==0:rendered.append('| '+' | '.join('---'for _ in vals)+' |')
+   region['text']='\n'.join(rendered);region['lines']=[l for c in cells for l in c['lines']]
+  if not changed:return base
+  out.update(text='\n\n'.join(r['text']for r in out['regions']),lines=[l for r in out['regions']for l in r['lines']],cell_rowcore_changes=changed);out['emitted_boxes']=[l['box']for l in out['lines']];out['recognition_extents']=[l.get('geometry',{}).get('source_pixels_extent',l['box'])for l in out['lines']];return out

stallion/bina04-cell-rowcores-95k-20261103/production.py

@@ -0,0 +1,17 @@
+import importlib.util
+from pathlib import Path
+R=Path(__file__).resolve().parent
+class Runtime:
+ def __init__(self,device='auto'):
+  s=importlib.util.spec_from_file_location('rowcore_frozen_parent','__BINA_ROOT__/stallion/bina04-header-marks-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device)
+  s=importlib.util.spec_from_file_location('rowcore_header_apply','__BINA_ROOT__/stallion/bina04-header-marks-95k-20261103/candidate.py');self.header=importlib.util.module_from_spec(s);s.loader.exec_module(self.header)
+  s=importlib.util.spec_from_file_location('rowcore_candidate',R/'candidate.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m)
+  from indra_ocr.rtl import flip
+  self.patch=m.Patch(self.parent.parent.numeric,lambda t:flip(t,base_rtl=True))
+ def __call__(self,image,old,kind,name='',**kwargs):
+  out=self.parent.parent(image,old,kind,name,**kwargs)
+  assert self.parent.params['header_confidence']>1,'Frozen accepted header mode required'
+  data=dict(new=[],old_support=[1.]*len(out['lines']),WH=list(image.size),header_regions=[])
+  out=self.header.apply(out,data,self.parent.params)
+  return self.patch(image,out,kind)
+ def close(self):self.parent.close()

stallion/bina04-column-assembly-95k-20261103/column_order.py

@@ -0,0 +1,39 @@
+"""Conservative physical-gutter order for dense, two-column RTL text; tables atomic."""
+import copy,statistics
+from assembly import norm
+
+def apply_columns(old,W,H):
+ out=copy.deepcopy(old);lines=out.get('lines',[])
+ if any(r.get('table_trace',{}).get('grid')for r in out.get('regions',[])):return out
+ useful=[i for i,l in enumerate(lines)if len(norm(l.get('text','')))>=8 and .12*W<l['box'][2]-l['box'][0]<.65*W]
+ if len(useful)<10:return out
+ candidates=[]
+ for k in range(20,81):
+  x=W*k/100;left=[i for i in useful if lines[i]['box'][2]<=x];right=[i for i in useful if lines[i]['box'][0]>=x];cross=[i for i in useful if i not in left+right]
+  if min(len(left),len(right))<4 or len(cross)>.05*len(useful):continue
+  span=lambda ids:max(lines[i]['box'][3]for i in ids)-min(lines[i]['box'][1]for i in ids)
+  if min(span(left),span(right))<.4*H:continue
+  gap=min(lines[i]['box'][0]for i in right)-max(lines[i]['box'][2]for i in left)
+  if gap<.012*W:continue
+  candidates.append((len(left)+len(right),gap,x,left,right))
+ if not candidates:return out
+ _,gap,x,left,right=max(candidates,key=lambda v:(v[0],v[1]));med=statistics.median(lines[i]['box'][3]-lines[i]['box'][1]for i in useful);start=max(min(lines[i]['box'][1]for i in left),min(lines[i]['box'][1]for i in right));end=min(max(lines[i]['box'][3]for i in left),max(lines[i]['box'][3]for i in right))
+ # A shared spanning heading distinguishes one multi-column document from
+ # two independent exam/form panels with their own headers. Ambiguous pages retain baseline order.
+ if not any(l['box'][2]-l['box'][0]>.65*W and l['box'][1]<start and len(norm(l.get('text','')))>=8 for l in lines):return out
+ headers=[];footers=[];lc=[];rc=[]
+ for i,l in enumerate(lines):
+  if not l.get('text','').strip():continue
+  b=l['box'];cy=(b[1]+b[3])/2
+  if b[3]<start-.25*med or b[0]<x<b[2]and cy<start+med:headers.append(i)
+  elif b[0]<x<b[2]and b[1]>end-med:footers.append(i)
+  elif b[0]<x<b[2]:return old # A genuine spanning middle block makes a simple cut ambiguous.
+  elif (b[0]+b[2])/2>x:rc.append(i)
+  else:lc.append(i)
+ key=lambda i:(lines[i]['box'][1],-lines[i]['box'][2]);ids=sorted(headers,key=key)+sorted(rc,key=key)+sorted(lc,key=key)+sorted(footers,key=key)
+ # Reorder reads, never edit them. Only apply when plain-line emission exactly accounts for output.
+ if norm(''.join(lines[i]['text']for i in range(len(lines))))!=norm(out['text']):return out
+ out['text']='\n\n'.join(lines[i]['text']for i in ids);out['lines']=[lines[i]for i in ids]
+ out['regions']=[dict(path='physical_column_order',layout=None,source_line_ids=[i],lines=[lines[i]],text=lines[i]['text'])for i in ids]
+ out['physical_column_order']=dict(applied=True,gutter_x=x,gutter_width=gap,header_ids=headers,right_ids=rc,left_ids=lc,footer_ids=footers,read_text_changed=False)
+ return out

stallion/bina04-column-assembly-95k-20261103/production.py

@@ -0,0 +1,11 @@
+"""Accepted page assembly with conservative shared-heading RTL column order."""
+import importlib.util,sys
+from pathlib import Path
+R=Path(__file__).resolve().parent
+sys.path.insert(0,str(R))
+from column_order import apply_columns
+class Runtime:
+ def __init__(self,device='auto'):
+  s=importlib.util.spec_from_file_location('columns_frozen_parent','__BINA_ROOT__/stallion/bina04-page-assembly-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device)
+ def __call__(self,image,old,kind,name='',**kwargs):return apply_columns(self.parent(image,old,kind,name,**kwargs),*image.size)
+ def close(self):self.parent.close()

stallion/bina04-crop-bandseg-95k-20261102/bandseg.py

@@ -0,0 +1,277 @@
+"""Pixel-only crop line-count guard; pages and atomic tables stay unchanged."""
+import copy
+import cv2
+import numpy as np
+from scipy.ndimage import gaussian_filter1d
+from scipy.signal import find_peaks, peak_widths
+
+
+def glyph_height(m, fallback=3.):
+    _,_,stats,_=cv2.connectedComponentsWithStats(m.astype(np.uint8),8)
+    values=[(int(h),int(a)) for x,y,w,h,a in stats[1:] if h>=3 and w>=2 and h<=m.shape[0]]
+    if not values:return fallback
+    values.sort();hs,areas=zip(*values)
+    return float(hs[np.searchsorted(np.cumsum(areas),.70*sum(areas))])
+
+
+def mask(gray):
+    _, m = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
+    if (m > 0).mean() > .5:
+        m = 255 - m
+    n, labels, stats, _ = cv2.connectedComponentsWithStats(m, 8)
+    H, W = gray.shape
+    good = []
+    heights = []
+    for i, (x, y, w, h, area) in enumerate(stats[1:], 1):
+        if area < 2 or (w > .8 * W and h > .65 * H):
+            continue  # connected frames/decorations, not a text-line core
+        if (w > .7 * W and h < max(3, .035 * H)) or (h > .7 * H and w < max(2,.035*W)):
+            continue  # ruling line
+        good.append(i)
+        if h >= 3 and w >= 2 and h < .9 * H:
+            heights.append(h)
+    clean = np.isin(labels, good)
+    glyph = float(np.percentile(heights, 70)) if heights else max(3., H / 2)
+    # Remove long peripheral scan rules without deleting connected text.
+    hline=cv2.morphologyEx(clean.astype(np.uint8),cv2.MORPH_OPEN,np.ones((1,max(8,int(.65*W))),np.uint8))
+    vline=cv2.morphologyEx(clean.astype(np.uint8),cv2.MORPH_OPEN,np.ones((max(8,int(.65*H)),1),np.uint8))
+    hline[int(.12*H):int(.88*H),:]=0
+    vline[:,int(.12*W):int(.88*W)]=0
+    clean &= ~((hline>0)|(vline>0))
+    framed=any(w>.8*W and h>.65*H for x,y,w,h,area in stats[1:])
+    if framed:
+        edge_rules=cv2.morphologyEx(clean.astype(np.uint8),cv2.MORPH_OPEN,np.ones((1,max(8,int(.35*W))),np.uint8))
+        edge_rules[int(.25*H):int(.75*H),:]=0
+        clean &= ~(edge_rules>0)
+    if H<160:
+        for i,(x,y,w,h,area) in enumerate(stats[1:],1):
+            if h>1.8*glyph and .7*h<w<1.5*h and (x<.12*W or x+w>.88*W) and w<.2*W:
+                clean[labels==i]=False
+    return clean, glyph_height(clean,glyph)
+
+
+def horizontal(m, glyph, scale=1):
+    H, W = m.shape
+    prof = gaussian_filter1d(m.sum(1).astype(float), max(.6, .055 * glyph))
+    if prof.max() < 1:
+        return [], dict(peaks=[], valleys=[])
+    peaks, props = find_peaks(np.r_[0., prof, 0.],
+                             distance=max(3, int(.35 * glyph),int(min(9*scale,H/3))),
+                             prominence=max(1.5, .10 * prof.max()))
+    peaks = peaks - 1
+    peaks = [int(p) for p in peaks if 0 <= p < H]
+    _, _, comps, _ = cv2.connectedComponentsWithStats(m.astype(np.uint8), 8)
+    def local_height(y):
+        rows=[(h,area) for x,cy,w,h,area in comps[1:] if cy <= y < cy+h and h>=3 and w>=2 and h<.7*H]
+        if not rows:return glyph
+        rows.sort();hs,areas=zip(*rows)
+        return float(hs[np.searchsorted(np.cumsum(areas),.7*sum(areas))])
+    # A detached dot or descender is not another baseline. Only a deep valley
+    # between substantial core peaks can create another physical line.
+    while len(peaks) > 1:
+        remove = None
+        for i, (a, b) in enumerate(zip(peaks, peaks[1:])):
+            valley = float(prof[a:b+1].min())
+            if valley > .55 * min(prof[a], prof[b]) or b-a < .85*max(local_height(a),local_height(b)):
+                remove = i if prof[a] < prof[b] else i + 1
+                break
+        if remove is None:
+            break
+        peaks.pop(remove)
+    if not peaks:
+        peaks = [int(prof.argmax())]
+    cuts = [0]
+    for a, b in zip(peaks, peaks[1:]):
+        q = prof[a:b+1]
+        low = np.flatnonzero(q <= q.min() + .015 * min(prof[a], prof[b]))
+        cuts.append(a + int(np.median(low)))
+    cuts.append(H)
+    bands = []
+    for i, (a, b) in enumerate(zip(cuts, cuts[1:])):
+        ys = np.flatnonzero(m[a:b].sum(1) >= max(1, .015 * prof.max()))
+        if not len(ys):
+            continue
+        y0, y1 = a + int(ys[0]), a + int(ys[-1]) + 1
+        xs = np.flatnonzero(m[y0:y1].any(0))
+        if not len(xs) or y1-y0 < max(2,.4*local_height(peaks[i])):
+            continue
+        # Keep the separating valley as a hard recognition crop boundary.
+        bands.append([int(xs[0]), y0, int(xs[-1])+1, y1])
+    if len(bands)>1:
+        tallest=max(b[3]-b[1] for b in bands)
+        bands=[b for b in bands if not (b[3]-b[1]<.45*tallest and (b[1]<=scale or b[3]>=H-scale))]
+    return bands, dict(peaks=peaks, valleys=cuts[1:-1], glyph=glyph)
+
+
+def runs(v):
+    edges = np.diff(np.r_[False, v, False].astype(np.int8))
+    return list(zip(np.flatnonzero(edges == 1), np.flatnonzero(edges == -1)))
+
+
+def segment(gray):
+    H, W = gray.shape
+    scale = 3 if H < 40 else 2 if H < 64 else 1
+    big = cv2.resize(gray, (W*scale, H*scale), interpolation=cv2.INTER_CUBIC) if scale != 1 else gray
+    events = []
+
+    def recurse(g, ox=0, oy=0, depth=0, allow_horizontal=True, supplied_mask=None):
+        if supplied_mask is None:m,glyph=mask(g)
+        else:m=supplied_mask;glyph=glyph_height(m,3.)
+        hh, ww = g.shape
+        bands, ht = horizontal(m, glyph,scale)
+        ys=np.flatnonzero(m.any(1))
+        mass=m.sum(1);total=mass.sum()
+        if total:
+            lo,hi=np.searchsorted(np.cumsum(mass),[.01*total,.99*total])
+        else:lo,hi=0,0
+        if depth==0 and (len(bands)<=1 or hi-lo<=1.5*glyph):
+            return [[0,int(ys[0]),ww,int(ys[-1])+1]] if len(ys) else []
+        if depth < 3 and len(bands):
+            # True gutter candidates, with multiple lines on at least one side.
+            # A narrow gutter is allowed only between different type scales
+            # (e.g. a small side column alongside a giant headline).
+            col = m.mean(0)
+            gaps = [(a,b) for a,b in runs(col <= .012)
+                    if a > .12*ww and b < .88*ww and b-a >= 2*scale]
+            for a,b in sorted(gaps, key=lambda q: q[1]-q[0], reverse=True):
+                left,right=m[:,:a],m[:,b:]
+                gl,gr=glyph_height(left,3.),glyph_height(right,3.)
+                if min(gl,gr)<(.25 if max(gl,gr)<64 else .12)*max(gl,gr):continue
+                lb, _ = horizontal(left, gl,scale); rb, _ = horizontal(right, gr,scale)
+                area = max(1,m.sum())
+                disparate = max(gl,gr) >= 3.*max(1,min(gl,gr))
+                true_columns = min(len(lb),len(rb)) >= 2 and b-a >= 1.5*min(gl,gr)
+                if max(len(lb),len(rb)) < 2 or min(left.sum(),right.sum()) < .025*area or a < 3*gl or ww-b < 3*gr:
+                    continue
+                if not disparate or max(len(lb),len(rb))<3:
+                    continue
+                L = recurse(g[:, :a], ox, oy, depth+1, supplied_mask=left)
+                R = recurse(g[:, b:], ox+b, oy, depth+1, supplied_mask=right)
+                events.append(dict(kind='vertical_gutter', box=[ox,oy,ox+ww,oy+hh], gap=[int(a),int(b)], glyphs=[gl,gr]))
+                # Primary large headline before its small side column; ordinary
+                # columns use RTL. Within each column retain top-to-bottom.
+                return (L+R) if gl >= 3.*gr else (R+L)
+        if len(bands) >= 2 and depth < 3 and allow_horizontal:
+            out = []
+            # Inspect each horizontal band for a side column. Whitespace is
+            # bounded by the valley, so adjacent lines never enter the crop.
+            for x0,y0,x1,y1 in bands:
+                out += recurse(g[y0:y1, :], ox, oy+y0, depth+1, allow_horizontal=False, supplied_mask=m[y0:y1,:])
+            events.append(dict(kind='horizontal_valleys',box=[ox,oy,ox+ww,oy+hh],**ht))
+            return out
+        if not allow_horizontal and len(bands)>1:
+            ys,xs=np.where(m)
+            bands=[[int(xs.min()),int(ys.min()),int(xs.max())+1,int(ys.max())+1]] if len(xs) else []
+        return [[ox,oy+y0,ox+ww,oy+y1] for x0,y0,x1,y1 in bands]
+
+    root_mask, root_glyph=mask(big)
+    _,labs,stats,_=cv2.connectedComponentsWithStats(root_mask.astype(np.uint8),8)
+    for i,(x,y,w,h,area) in enumerate(stats[1:],1):
+        if h<.10*root_glyph and area<.15*root_glyph**2:
+            root_mask[labs==i]=False
+    boxes = recurse(big,supplied_mask=root_mask)
+    # A compact framed badge has ornamental upper/lower strokes, not extra
+    # text baselines. Do not turn its border into recognizer inputs.
+    _,_,raw_stats,_=cv2.connectedComponentsWithStats((cv2.threshold(big,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)[1]>0).astype(np.uint8),8)
+    framed=any(w>.8*big.shape[1] and h>.65*big.shape[0] for x,y,w,h,a in raw_stats[1:])
+    if framed and W/max(1,H)>2.5 and len(boxes)>=2:
+        dominant=max(boxes,key=lambda b:b[3]-b[1])
+        if dominant[3]-dominant[1]>.45*big.shape[0] and all(b is dominant or b[3]-b[1]<.25*big.shape[0] for b in boxes):
+            boxes=[dominant]
+            events.append(dict(kind='framed_badge_ornaments_not_lines'))
+    # Recognition margins are real pixels, not a larger coverage-only receipt.
+    # Limit vertical expansion at the midpoint between adjacent physical rows.
+    padded=[]
+    for b in boxes:
+        h=b[3]-b[1];xp=max(2*scale,.15*h);yp=max(2*scale,.2*h)
+        lo,hi=max(0,b[1]-yp),min(big.shape[0],b[3]+yp)
+        for q in boxes:
+            overlap=min(b[2],q[2])-max(b[0],q[0])
+            if q is b or overlap<.2*min(b[2]-b[0],q[2]-q[0]):continue
+            if q[3]<=b[1]:lo=max(lo,(q[3]+b[1])/2)
+            if q[1]>=b[3]:hi=min(hi,(q[1]+b[3])/2)
+        padded.append([max(0,b[0]-xp),lo,min(big.shape[1],b[2]+xp),hi])
+    boxes = [[v/scale for v in b] for b in padded]
+    return boxes, dict(scale=scale, boxes=boxes, events=events, image_only=True)
+
+
+def should_replace(baseline, boxes):
+    if len(boxes) < 2:
+        return False, 'one_or_zero_supported_band'
+    lines = baseline.get('lines', [])
+    if any('Surya_grid' in r.get('path','') or (r.get('layout') or {}).get('label','').lower()=='table' for r in baseline.get('regions',[])) or any(r.get('label','').lower()=='table' for r in baseline.get('raw_layout',{}).get('bboxes',[])):
+        return False, 'atomic_table'
+    old = [l['box'] for l in lines]
+    # Keep successful existing multiline reads; repair a box spanning two or
+    # more separately supported cores, or a genuine shortage of physical rows.
+    if len(old) < len(boxes):
+        return True, 'more_physical_lines_than_reads'
+    for b in old:
+        contained = []
+        for q in boxes:
+            iy=max(0,min(b[3],q[3])-max(b[1],q[1]))
+            ix=max(0,min(b[2],q[2])-max(b[0],q[0]))
+            if iy >= .8*(q[3]-q[1]) and ix >= .6*(q[2]-q[0]):
+                contained.append(q)
+        if len(contained)>=2:
+            return True, 'existing_read_spans_multiple_cores'
+    return False, 'existing_multiline_reads_preserved'
+
+
+class CropBandPatch:
+    def __init__(self, reader, flip):
+        self.reader, self.flip = reader, flip
+
+    def __call__(self, image, baseline, *, input_type='crop'):
+        if input_type != 'crop':
+            return copy.deepcopy(baseline)
+        gray = np.asarray(image.convert('L'))
+        boxes, trace = segment(gray)
+        use, reason = should_replace(baseline, boxes)
+        trace.update(applied=use,reason=reason,original_reads=len(baseline.get('lines',[])))
+        result = copy.deepcopy(baseline)
+        if not use:
+            result['crop_band_guard'] = trace
+            return result
+        lines=[]
+        scale=trace['scale']
+        big=cv2.resize(gray,(gray.shape[1]*scale,gray.shape[0]*scale),interpolation=cv2.INTER_CUBIC) if scale!=1 else gray
+        used=set()
+        for b in boxes:
+            # Keep an existing correctly localized row, including its actual
+            # training-contract margins; discard only merged/full-crop reads.
+            matches=[]
+            for j,old in enumerate(baseline.get('lines',[])):
+                o=old['box'];h=b[3]-b[1];oh=o[3]-o[1]
+                iy=max(0,min(b[3],o[3])-max(b[1],o[1]));ix=max(0,min(b[2],o[2])-max(b[0],o[0]))
+                if j not in used and .7*h<=oh<=1.6*h and iy>=.8*h and ix>=.65*(b[2]-b[0]):
+                    overlaps=sum(q is not b and max(0,min(q[3],o[3])-max(q[1],o[1]))>=.7*(q[3]-q[1]) for q in boxes)
+                    if not overlaps:matches.append((iy*ix,j))
+            if matches:
+                j=max(matches)[1];lines.append(copy.deepcopy(baseline['lines'][j]));used.add(j);continue
+            x0,y0,x1,y1=[int(round(v*scale)) for v in b]
+            local=big[y0:y1,x0:x1]
+            # Clip the local canvas before adding the exact training margins:
+            # neighbouring lines cannot bleed into crop_line64's 20% pad.
+            _, receipts=self.reader(local,np.asarray([[0,0,local.shape[1],local.shape[0]]],np.float32))
+            r=receipts[0]
+            text=self.flip(r['raw_visual_text'],base_rtl=True)
+            r['source_pixels_extent']=b
+            r['band_local_original_box']=b
+            lines.append(dict(box=b,text=text,visual_text=r['raw_visual_text'],confidence=r['confidence'],geometry=r,recognizer='unified32M_95k',confidence_filter_applied=False))
+        # Retain small existing reads that projection cannot see (footnotes,
+        # signatures), rather than losing them when repairing a merged box.
+        median=float(np.median([b[3]-b[1] for b in boxes]))
+        for j,old in enumerate(baseline.get('lines',[])):
+            if j in used:continue
+            o=old['box'];oh=o[3]-o[1]
+            if oh>1.5*median:continue
+            hits=sum(max(0,min(o[3],q[3])-max(o[1],q[1])) >= .5*min(oh,q[3]-q[1]) and min(o[2],q[2])-max(o[0],q[0]) >= .5*min(o[2]-o[0],q[2]-q[0]) for q in boxes)
+            if hits>=2:continue
+            overlaps=[max(0,min(o[3],q[3])-max(o[1],q[1]))/max(1,oh) for q in boxes if min(o[2],q[2])>max(o[0],q[0])]
+            if overlaps and max(overlaps)>=.65:continue
+            at=next((k for k,l in enumerate(lines) if (l['box'][1]+l['box'][3])/2>(o[1]+o[3])/2),len(lines))
+            lines.insert(at,copy.deepcopy(old))
+        region=dict(layout=None,path='crop_projection_physical_lines' ,text='\n'.join(l['text']for l in lines),lines=lines,source_line_ids=list(range(len(lines))))
+        result.update(text=region['text'],lines=lines,global_readings=lines,regions=[region],detected_boxes=boxes,emitted_boxes=[l['box'] for l in lines],recognition_extents=[l.get('geometry',{}).get('source_pixels_extent',l['box']) for l in lines],crop_band_guard=trace)
+        return result

stallion/bina04-crop-bandseg-95k-20261102/consumer_runtime.py

@@ -0,0 +1,51 @@
+"""Crop-only preflight around the accepted 95k engine; no page-path edits.
+
+The parent supplies its normal image-only layout receipt in `old`. This wrapper
+does not need reference text, gold line boxes, benchmark IDs or categories.
+"""
+import hashlib
+import numpy as np
+from bandseg import CropBandPatch, segment, should_replace
+
+
+class BandGuardRuntime:
+    def __init__(self, parent_engine, flip):
+        self.parent = parent_engine
+        reader = parent_engine.patch.reader
+        self.patch = CropBandPatch(reader.reader, flip)
+        self.context = context = {}
+        original = type(reader)
+
+        class PreflightReader(original):
+            def __call__(self, gray, boxes):
+                H,W = gray.shape
+                if (context.get('enabled') and len(boxes)==1
+                    and np.allclose(boxes[0],[0,0,W,H])
+                    and gray.shape == context['shape']
+                    and hashlib.sha256(gray.tobytes()).hexdigest()==context['pixels']):
+                    context['full_strip_reads_skipped'] += 1
+                    return [dict(box=[0.,0.,float(W),float(H)],text='',visual_text='',
+                                 confidence=0.,recognizer='skipped_by_multiline_preflight',
+                                 geometry=dict(source_pixels_extent=[0.,0.,float(W),float(H)],padded_source_extent=[0.,0.,float(W),float(H)],raw_visual_text=''))]
+                return super().__call__(gray, boxes)
+
+        reader.__class__ = PreflightReader
+
+    def __call__(self,image,old,kind,name,**kwargs):
+        self.context.clear()
+        if kind=='crop':
+            gray=np.asarray(image.convert('L'))
+            boxes,trace=segment(gray)
+            use,reason=should_replace(old,boxes)
+            self.context.update(enabled=use,shape=gray.shape,
+                                pixels=hashlib.sha256(gray.tobytes()).hexdigest(),
+                                full_strip_reads_skipped=0)
+        baseline=self.parent(image,old,kind,name,**kwargs)
+        result=self.patch(image,baseline,input_type=kind)
+        if kind=='crop':
+            result['crop_band_preflight']=dict(full_strip_reads_skipped=self.context['full_strip_reads_skipped'],
+                                               before_rule_C=True,reference_or_label_used=False)
+        return result
+
+    def close(self):
+        self.parent.close()

stallion/bina04-crop-picture-cells-95k-20261102/CALIBRATION.json

@@ -0,0 +1,612 @@
+{
+  "source": "Internal synthetic dev80 and frozen calib25 only; no193/check25/publicTT calibration",
+  "pen": {
+    "train_samples": 12589,
+    "dev_samples": 3180,
+    "train_pages": 160,
+    "dev_pages": 40,
+    "dev": {
+      "threshold": 0.12000000000000001,
+      "printed_FPR": 0.0029154518950437317,
+      "handwriting_recall": 0.9996475149806133
+    },
+    "seconds": 3.9756869729608297,
+    "parameters": 1623,
+    "source": "Existing synthetic train only; threshold on synthetic dev only; no193/dev50/TT training",
+    "options": [
+      {
+        "threshold": 0.1,
+        "printed_FPR": 0.011661807580174927,
+        "handwriting_recall": 0.9996475149806133
+      },
+      {
+        "threshold": 0.11,
+        "printed_FPR": 0.011661807580174927,
+        "handwriting_recall": 0.9996475149806133
+      },
+      {
+        "threshold": 0.12000000000000001,
+        "printed_FPR": 0.0029154518950437317,
+        "handwriting_recall": 0.9996475149806133
+      },
+      {
+        "threshold": 0.13,
+        "printed_FPR": 0.0029154518950437317,
+        "handwriting_recall": 0.9996475149806133
+      },
+      {
+        "threshold": 0.14,
+        "printed_FPR": 0.0029154518950437317,
+        "handwriting_recall": 0.9996475149806133
+      },
+      {
+        "threshold": 0.15000000000000002,
+        "printed_FPR": 0.0029154518950437317,
+        "handwriting_recall": 0.9996475149806133
+      },
+      {
+        "threshold": 0.16,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9996475149806133
+      },
+      {
+        "threshold": 0.17,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9996475149806133
+      },
+      {
+        "threshold": 0.18,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9992950299612267
+      },
+      {
+        "threshold": 0.19,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9992950299612267
+      },
+      {
+        "threshold": 0.2,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9985900599224533
+      },
+      {
+        "threshold": 0.21000000000000002,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9985900599224533
+      },
+      {
+        "threshold": 0.22,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9985900599224533
+      },
+      {
+        "threshold": 0.23,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9985900599224533
+      },
+      {
+        "threshold": 0.24000000000000002,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9985900599224533
+      },
+      {
+        "threshold": 0.25,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9982375749030666
+      },
+      {
+        "threshold": 0.26,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9982375749030666
+      },
+      {
+        "threshold": 0.27,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9982375749030666
+      },
+      {
+        "threshold": 0.28,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9982375749030666
+      },
+      {
+        "threshold": 0.29000000000000004,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.99788508988368
+      },
+      {
+        "threshold": 0.30000000000000004,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.99788508988368
+      },
+      {
+        "threshold": 0.31,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.99788508988368
+      },
+      {
+        "threshold": 0.32,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.99788508988368
+      },
+      {
+        "threshold": 0.33,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9975326048642933
+      },
+      {
+        "threshold": 0.33999999999999997,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9975326048642933
+      },
+      {
+        "threshold": 0.35,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9971801198449066
+      },
+      {
+        "threshold": 0.36,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9971801198449066
+      },
+      {
+        "threshold": 0.37,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9971801198449066
+      },
+      {
+        "threshold": 0.38,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9964751498061333
+      },
+      {
+        "threshold": 0.39,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9964751498061333
+      },
+      {
+        "threshold": 0.4,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9964751498061333
+      },
+      {
+        "threshold": 0.41000000000000003,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9964751498061333
+      },
+      {
+        "threshold": 0.42000000000000004,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9964751498061333
+      },
+      {
+        "threshold": 0.43000000000000005,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9961226647867466
+      },
+      {
+        "threshold": 0.44000000000000006,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9961226647867466
+      },
+      {
+        "threshold": 0.45000000000000007,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9961226647867466
+      },
+      {
+        "threshold": 0.45999999999999996,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9961226647867466
+      },
+      {
+        "threshold": 0.47,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9954176947479733
+      },
+      {
+        "threshold": 0.48,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9947127247091999
+      },
+      {
+        "threshold": 0.49,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9947127247091999
+      },
+      {
+        "threshold": 0.5,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9947127247091999
+      },
+      {
+        "threshold": 0.51,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9947127247091999
+      },
+      {
+        "threshold": 0.52,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9940077546704265
+      },
+      {
+        "threshold": 0.53,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9940077546704265
+      },
+      {
+        "threshold": 0.54,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9936552696510398
+      },
+      {
+        "threshold": 0.55,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9936552696510398
+      },
+      {
+        "threshold": 0.56,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9936552696510398
+      },
+      {
+        "threshold": 0.5700000000000001,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9933027846316531
+      },
+      {
+        "threshold": 0.58,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9925978145928798
+      },
+      {
+        "threshold": 0.59,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9925978145928798
+      },
+      {
+        "threshold": 0.6,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9918928445541064
+      },
+      {
+        "threshold": 0.61,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9918928445541064
+      },
+      {
+        "threshold": 0.62,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9918928445541064
+      },
+      {
+        "threshold": 0.63,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9918928445541064
+      },
+      {
+        "threshold": 0.64,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9918928445541064
+      },
+      {
+        "threshold": 0.65,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9915403595347198
+      },
+      {
+        "threshold": 0.66,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9904829044765597
+      },
+      {
+        "threshold": 0.67,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9897779344377864
+      },
+      {
+        "threshold": 0.6799999999999999,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9897779344377864
+      },
+      {
+        "threshold": 0.69,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9894254494183997
+      },
+      {
+        "threshold": 0.7,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9883679943602397
+      },
+      {
+        "threshold": 0.71,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.988015509340853
+      },
+      {
+        "threshold": 0.72,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9876630243214664
+      },
+      {
+        "threshold": 0.73,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.986958054282693
+      },
+      {
+        "threshold": 0.74,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9866055692633063
+      },
+      {
+        "threshold": 0.75,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.985900599224533
+      },
+      {
+        "threshold": 0.76,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9851956291857596
+      },
+      {
+        "threshold": 0.77,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9844906591469863
+      },
+      {
+        "threshold": 0.78,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9837856891082128
+      },
+      {
+        "threshold": 0.79,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9820232640112795
+      },
+      {
+        "threshold": 0.8,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9806133239337328
+      },
+      {
+        "threshold": 0.8099999999999999,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9792033838561861
+      },
+      {
+        "threshold": 0.82,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9777934437786394
+      },
+      {
+        "threshold": 0.83,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9767359887204794
+      },
+      {
+        "threshold": 0.84,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9756785336623194
+      },
+      {
+        "threshold": 0.85,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9725061684878392
+      },
+      {
+        "threshold": 0.86,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9700387733521325
+      },
+      {
+        "threshold": 0.87,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9682763482551991
+      },
+      {
+        "threshold": 0.88,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9658089531194924
+      },
+      {
+        "threshold": 0.89,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9629890729643991
+      },
+      {
+        "threshold": 0.9,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9591117377511456
+      },
+      {
+        "threshold": 0.91,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9548819175185055
+      },
+      {
+        "threshold": 0.92,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9502996122664786
+      },
+      {
+        "threshold": 0.93,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9436023968981319
+      },
+      {
+        "threshold": 0.94,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9344377863940783
+      },
+      {
+        "threshold": 0.95,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9213958406767713
+      },
+      {
+        "threshold": 0.96,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.9041240747268241
+      },
+      {
+        "threshold": 0.97,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.8790976383503701
+      },
+      {
+        "threshold": 0.98,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.8403242862178357
+      },
+      {
+        "threshold": 0.99,
+        "printed_FPR": 0.0,
+        "handwriting_recall": 0.7673598872047938
+      }
+    ]
+  },
+  "junk": {
+    "threshold": 0.55,
+    "false_positive_controls": 0,
+    "dev_junk_removed": 0
+  },
+  "pen_trials": [
+    {
+      "threshold": 0.05,
+      "printed_FPR": 0.9416666666666667,
+      "coloured_handwriting_recall": 1.0,
+      "printed": 120,
+      "handwriting": 80
+    },
+    {
+      "threshold": 0.1,
+      "printed_FPR": 0.8833333333333333,
+      "coloured_handwriting_recall": 0.9625,
+      "printed": 120,
+      "handwriting": 80
+    },
+    {
+      "threshold": 0.15,
+      "printed_FPR": 0.775,
+      "coloured_handwriting_recall": 0.8375,
+      "printed": 120,
+      "handwriting": 80
+    },
+    {
+      "threshold": 0.2,
+      "printed_FPR": 0.675,
+      "coloured_handwriting_recall": 0.6875,
+      "printed": 120,
+      "handwriting": 80
+    },
+    {
+      "threshold": 0.25,
+      "printed_FPR": 0.4666666666666667,
+      "coloured_handwriting_recall": 0.525,
+      "printed": 120,
+      "handwriting": 80
+    },
+    {
+      "threshold": 0.3,
+      "printed_FPR": 0.2,
+      "coloured_handwriting_recall": 0.45,
+      "printed": 120,
+      "handwriting": 80
+    },
+    {
+      "threshold": 0.4,
+      "printed_FPR": 0.1,
+      "coloured_handwriting_recall": 0.4375,
+      "printed": 120,
+      "handwriting": 80
+    },
+    {
+      "threshold": 0.5,
+      "printed_FPR": 0.06666666666666667,
+      "coloured_handwriting_recall": 0.3875,
+      "printed": 120,
+      "handwriting": 80
+    },
+    {
+      "threshold": 0.6,
+      "printed_FPR": 0.05,
+      "coloured_handwriting_recall": 0.375,
+      "printed": 120,
+      "handwriting": 80
+    },
+    {
+      "threshold": 0.7,
+      "printed_FPR": 0.03333333333333333,
+      "coloured_handwriting_recall": 0.2125,
+      "printed": 120,
+      "handwriting": 80
+    },
+    {
+      "threshold": 0.8,
+      "printed_FPR": 0.008333333333333333,
+      "coloured_handwriting_recall": 0.0375,
+      "printed": 120,
+      "handwriting": 80
+    },
+    {
+      "threshold": 0.9,
+      "printed_FPR": 0.0,
+      "coloured_handwriting_recall": 0.0,
+      "printed": 120,
+      "handwriting": 80
+    }
+  ],
+  "junk_trials": [
+    {
+      "threshold": 0.35,
+      "false_positive_controls": 0,
+      "dev_junk_removed": 0
+    },
+    {
+      "threshold": 0.45,
+      "false_positive_controls": 0,
+      "dev_junk_removed": 0
+    },
+    {
+      "threshold": 0.55,
+      "false_positive_controls": 0,
+      "dev_junk_removed": 0
+    },
+    {
+      "threshold": 0.65,
+      "false_positive_controls": 1,
+      "dev_junk_removed": 0
+    },
+    {
+      "threshold": 0.75,
+      "false_positive_controls": 1,
+      "dev_junk_removed": 1
+    },
+    {
+      "threshold": 0.8,
+      "false_positive_controls": 2,
+      "dev_junk_removed": 2
+    },
+    {
+      "threshold": 0.85,
+      "false_positive_controls": 2,
+      "dev_junk_removed": 2
+    },
+    {
+      "threshold": 0.9,
+      "false_positive_controls": 2,
+      "dev_junk_removed": 2
+    },
+    {
+      "threshold": 0.95,
+      "false_positive_controls": 2,
+      "dev_junk_removed": 2
+    }
+  ],
+  "synthetic_short_control_cases": 0,
+  "synthetic_short_correct": 0,
+  "calib25_short_cases": 37,
+  "manifest_sha256": "959ee4f442705c44dea8557c850a1004af3fdfb8d7c0f1fdfe90a8291f773940",
+  "pen_colour_abstention": 0.05
+}

stallion/bina04-crop-picture-cells-95k-20261102/candidate.py

@@ -0,0 +1,62 @@
+"""Isolated picture/cell candidate on the accepted bandseg path."""
+import copy,json
+from pathlib import Path
+import numpy as np
+from cell_reader import pen_score,read_cell,owned_hand_cells
+from junk_filter import apply as filter_junk
+from pen_classifier import PenClassifier
+R=Path(__file__).resolve().parent
+
+class CropPatch:
+ def __init__(self,reader,flip,segment):
+  self.reader,self.flip,self.segment=reader,flip,segment
+  self.cal=json.loads((R/'CALIBRATION.json').read_text())
+  self.router=PenClassifier(R/'pen-classifier.joblib')
+ def __call__(self,image,baseline,*,input_type='crop'):
+  if input_type!='crop':return copy.deepcopy(baseline)
+  result=copy.deepcopy(baseline);rgb=np.asarray(image.convert('RGB'));trace=[]
+  page_colour_score=pen_score(rgb)
+  for region in result.get('regions',[]):
+   t=region.get('table_trace',{});cells=t.get('cell_debug',[])
+   if not t.get('grid')or not cells:continue
+   probabilities=[]
+   for cell in cells:
+    x,y,X,Y=[int(round(v))for v in cell['bbox']]
+    view=rgb[max(0,y):min(rgb.shape[0],Y),max(0,x):min(rgb.shape[1],X)]
+    probabilities.append(self.router.probability(view)if view.size and page_colour_score>.05 else 0.)
+   nonempty=[i for i,c in enumerate(cells)if c['text'].strip()]
+   table_pen=bool(nonempty)and sum(probabilities[i]>=self.router.threshold for i in nonempty)>len(nonempty)/2
+   ownership,owner_trace=owned_hand_cells(rgb,cells)if table_pen else (None,None)
+   values={};lines=[]
+   for index,cell in enumerate(cells):
+    probability=probabilities[index];pen=table_pen
+    pixels,box=ownership[index]if table_pen else (rgb,cell['bbox'])
+    text,ll,info=read_cell(self.reader,self.flip,pixels,box,self.segment,pen=False)
+    if table_pen:info['physical_component_ownership']=owner_trace
+    trace.append(dict(row=cell['row'],col=cell['col'],old=cell['text'],new=text,old_lines=len(cell['lines']),new_lines=len(ll),pen_route=pen,pen_probability=probability,preprocessing=info))
+    cell.update(text=text,lines=ll,cell_contract=info)
+    values[(cell['row'],cell['col'])]=text;lines+=ll
+   rendered=[]
+   for i,row in enumerate(t['kept_rows']):
+    vals=[' '.join(values.get((row,col),'').split()).replace('|',r'\|')for col in t['kept_columns']]
+    rendered.append('| '+' | '.join(vals)+' |')
+    if i==0:rendered.append('| '+' | '.join('---'for _ in vals)+' |')
+   region['text']='\n'.join(rendered);region['lines']=lines
+   region['table_cell_contract']=dict(pen_cells=sum(q['pen_route']for q in trace),pen_score=page_colour_score,table_pen=table_pen,component_ownership=owner_trace,checkpoint=95000,no_date_text_substitution=True)
+  if trace:
+   result['text']='\n\n'.join(r['text']for r in result['regions'])
+   result['lines']=[l for r in result['regions']for l in r['lines']]
+   result['emitted_boxes']=[l['box']for l in result['lines']]
+   result['recognition_extents']=[l.get('geometry',{}).get('source_pixels_extent',l['box'])for l in result['lines']]
+   result['cell_contract_changes']=trace
+  result,removed=filter_junk(result,self.cal['junk']['threshold'])
+  return result
+
+class PipelineRuntime:
+ def __init__(self,parent,flip,segment):
+  self.parent=parent
+  # The accepted BandGuardRuntime owns the raw95k RTLReader here.
+  self.patch=CropPatch(parent.patch.reader,flip,segment)
+ def __call__(self,image,old,kind,name,**kwargs):
+  return self.patch(image,self.parent(image,old,kind,name,**kwargs),input_type=kind)
+ def close(self):self.parent.close()

stallion/bina04-crop-picture-cells-95k-20261102/cell_reader.py

@@ -0,0 +1,95 @@
+"""Physical-line cell reads; original95k, no date repair or text substitution."""
+import math
+import cv2
+import numpy as np
+
+def owned_hand_cells(rgb,cells):
+    """Assign whole connected ink components once, rather than cut at virtual grids.
+
+    Pale paper rulings do not participate. Recognition keeps original stroke
+    pixels plus their antialiasing fringe on a white, row-isolated canvas.
+    """
+    g=cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY)
+    value,_=cv2.threshold(g,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
+    fg=(g<min(160,max(60,value+1))).astype(np.uint8)
+    n,labels,stats,_=cv2.connectedComponentsWithStats(fg,8)
+    owners=[[]for _ in cells];crossings=0
+    for i,(x,y,w,h,area)in enumerate(stats[1:],1):
+        if area<2:continue
+        scores=[]
+        for c in cells:
+            a,b,A,B=c['bbox']
+            scores.append(max(0,min(x+w,A)-max(x,a))*max(0,min(y+h,B)-max(y,b)))
+        if not scores or max(scores)<=0:continue
+        owner=int(np.argmax(scores));owners[owner].append(i)
+        a,b,A,B=cells[owner]['bbox']
+        crossings+=int(x<a or x+w>A or y<b or y+h>B)
+    out=[]
+    for ids,c in zip(owners,cells):
+        m=np.isin(labels,ids)
+        yy,xx=np.where(m)
+        if not len(xx):out.append((np.full_like(rgb,255),c['bbox']));continue
+        halo=cv2.dilate(m.astype(np.uint8),np.ones((3,3),np.uint8))>0
+        canvas=np.full_like(rgb,255);canvas[halo]=rgb[halo]
+        out.append((canvas,[float(xx.min()),float(yy.min()),float(xx.max()+1),float(yy.max()+1)]))
+    return out,dict(connected_components_assigned=sum(map(len,owners)),components_crossing_inferred_cell_edges=crossings,ink_components_never_split=True,original_stroke_pixels_preserved=True)
+
+def pen_score(rgb):
+    a=rgb.astype(float);g=cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY)
+    dark=g<130
+    return float(((a[:,:,2]-a[:,:,0]>12)&(a[:,:,1]-a[:,:,0]>3)&dark).sum()/max(1,dark.sum()))
+
+def prepare(rgb,pen=False):
+    g=cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY)
+    work=g.copy()
+    if pen:
+        a=rgb.astype(float)
+        # Pale cyan ruled-paper lines differ from the darker blue/black pen.
+        # Erase only long light rulings, retaining strokes crossing the rule.
+        cyan=((a[:,:,1]-a[:,:,0])>12)&((a[:,:,1]-a[:,:,2])>0)&(g>130)
+        work[cyan]=255
+    val,_=cv2.threshold(work,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
+    ink=work<min(190,max(50,val+1))
+    H,W=ink.shape
+    # Gemini's useful border check: only peripheral, long ruling segments;
+    # punctuation and strokes in the interior are retained.
+    hh=cv2.morphologyEx(ink.astype(np.uint8),cv2.MORPH_OPEN,np.ones((1,max(10,round(.75*W))),np.uint8))
+    vv=cv2.morphologyEx(ink.astype(np.uint8),cv2.MORPH_OPEN,np.ones((max(10,round(.75*H)),1),np.uint8))
+    hh[3:max(3,H-3),:]=0;vv[:,3:max(3,W-3)]=0
+    rules=(hh|vv)>0;work[rules]=255;ink[rules]=False
+    return work,ink,dict(pen_mask=pen,peripheral_rule_pixels=int(rules.sum()),interior_separators_not_removed=True)
+
+
+def read_cell(reader,flip,rgb,bbox,segment,*,pen=False):
+    H,W=rgb.shape[:2];x,y,X,Y=bbox
+    a,b,A,B=max(0,math.floor(x)-3),max(0,math.floor(y)-3),min(W,math.ceil(X)+3),min(H,math.ceil(Y)+3)
+    local=rgb[b:B,a:A]
+    if not local.size:return '',[],dict(empty=True)
+    work,ink,trace=prepare(local,pen)
+    if ink.sum()<3:return '',[],dict(trace,empty=True)
+    binary=np.where(ink,0,255).astype(np.uint8)
+    proposals,pt=segment(binary)
+    if not proposals:proposals=[[0,0,work.shape[1],work.shape[0]]]
+    proposals.sort(key=lambda q:(q[1]+q[3])/2)
+    # A cell's columns are atomic; only physical horizontal rows are split.
+    centres=[(q[1]+q[3])/2 for q in proposals]
+    cuts=[0]+[round((c+d)/2)for c,d in zip(centres,centres[1:])]+[work.shape[0]]
+    lines=[]
+    for top,bottom in zip(cuts,cuts[1:]):
+        ys,xs=np.where(ink[top:bottom])
+        if len(ys)<3:continue
+        # Tight on ALL ink, including one-pixel colon dots and slash tips;
+        # crop_line64 supplies the unchanged20/30percent training margins.
+        core=[float(xs.min()),float(ys.min()),float(xs.max()+1),float(ys.max()+1)]
+        plane=work[top:bottom].copy()
+        _,receipts=reader(plane,np.asarray([core],np.float32))
+        r=receipts[0];offset=[a,b+top,a,b+top]
+        for key in ['ink_box','padded_source_extent']:
+            r[key]=[float(v)+o for v,o in zip(r[key],offset)]
+        r['source_pixels_extent']=[max(a,r['padded_source_extent'][0]),max(b+top,r['padded_source_extent'][1]),min(A,r['padded_source_extent'][2]),min(b+bottom,r['padded_source_extent'][3])]
+        r['original_WH']=[W,H]
+        r['frame_centers']=[float(v)+a for v in r['frame_centers']]
+        r['row_mask_isolation']=True;r['pen_mask_applied']=pen
+        line=dict(box=[v+o for v,o in zip(core,offset)],text=flip(r['raw_visual_text']),visual_text=r['raw_visual_text'],confidence=r['confidence'],geometry=r,recognizer='handwriting95k_pen_mask'if pen else'printed95k_cell_contract')
+        lines.append(line)
+    return ' '.join(q['text'].strip()for q in lines if q['text'].strip()),lines,dict(trace,empty=not bool(lines),proposal_trace=pt,padded_bbox=[a,b,A,B],physical_line_count=len(lines),same_training_contract=True)

stallion/bina04-crop-picture-cells-95k-20261102/junk_filter.py

@@ -0,0 +1,54 @@
+"""Conservative crop-only graphic junk filter; threshold from dev only."""
+import copy
+
+def clean(s):return ''.join(s.split()).replace('\u200c','').replace('\u200d','')
+def area(b):return max(0,b[2]-b[0])*max(0,b[3]-b[1])
+def overlap(a,b):return area([max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])])/max(1,area(a))
+def rect(r):
+ if 'bbox'in r:return r['bbox']
+ p=r['polygon'];return[min(q[0]for q in p),min(q[1]for q in p),max(q[0]for q in p),max(q[1]for q in p)]
+
+def context(line,pipeline):
+ b=line['box'];h=max(2,b[3]-b[1]);cy=(b[1]+b[3])/2
+ layouts=pipeline.get('raw_layout',{}).get('bboxes',[])
+ text_support=any(overlap(b,rect(r))>=.5 for r in layouts if r.get('label','').lower()in['text','sectionheader','pageheader','caption','listitem','formula','table'])
+ picture=any(overlap(b,rect(r))>=.8 for r in layouts if r.get('label','').lower()in['picture','figure','image'])
+ neighbours=[]
+ for q in pipeline.get('lines',[]):
+  if q is line or len(clean(q['text']))<4 or q.get('confidence',0)<.92:continue
+  z=q['box'];hh=max(2,z[3]-z[1]);d=abs(cy-(z[1]+z[3])/2);scale=max(h,hh)
+  xgap=max(0,max(b[0],z[0])-min(b[2],z[2]));xo=max(0,min(b[2],z[2])-max(b[0],z[0]))
+  same=d<.6*scale and xgap<3*scale
+  stacked=d<1.25*scale and xo>.7*min(b[2]-b[0],z[2]-z[0])
+  if same or stacked:neighbours.append(z)
+ return dict(text_layout_support=text_support,picture=picture,neighbouring_text=bool(neighbours))
+
+def decision(line,pipeline,threshold):
+ t=clean(line['text']);c=context(line,pipeline)
+ # Tables have their own cell path; never suppress a table digit by this rule.
+ table=any('grid' in r.get('path','').lower() for r in pipeline.get('regions',[]))
+ short=len(t)<=3;weak=line.get('confidence',0)<threshold
+ use=bool(t)and weak and not table and not c['text_layout_support']and not c['neighbouring_text']and(short or c['picture']and len(t)<=8)
+ return use,dict(c,short=short,confidence=line.get('confidence',0),threshold=threshold)
+
+def apply(pipeline,threshold):
+ result=copy.deepcopy(pipeline);removed=[];indices=[]
+ for i,l in enumerate(result.get('lines',[])):
+  use,trace=decision(l,result,threshold)
+  if use:removed.append(dict(line=l,reason=trace));indices.append(i)
+ if not removed:return result,[]
+ ids={id(result['lines'][i])for i in indices}
+ # deepcopy preserves shared references where present, but saved JSON repeats
+ # line dictionaries. Match box+text so every representation is consistent.
+ keys={(tuple(r['line']['box']),r['line']['text'])for r in removed}
+ keep=lambda l:(tuple(l['box']),l['text'])not in keys
+ result['lines']=[l for l in result['lines']if keep(l)]
+ for r in result.get('regions',[]):
+  if 'grid'in r.get('path','').lower():continue
+  r['lines']=[l for l in r.get('lines',[])if keep(l)];r['text']='\n'.join(l['text']for l in r['lines'])
+ result['regions']=[r for r in result.get('regions',[])if r.get('lines')or r.get('text','').strip()]
+ result['text']='\n\n'.join(r['text']for r in result['regions'])
+ result['emitted_boxes']=[l['box']for l in result['lines']]
+ result['recognition_extents']=[l.get('geometry',{}).get('source_pixels_extent',l['box'])for l in result['lines']]
+ result['graphic_junk_filter']=dict(threshold=threshold,removed=removed,dev_only_calibration=True)
+ return result,removed

stallion/bina04-crop-picture-cells-95k-20261102/pen-classifier.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7c520fefc9de3021459580a20dcb848854e25f1d1a72282e4c4caf55fc26490
+size 13905

stallion/bina04-crop-picture-cells-95k-20261102/pen_classifier.py

@@ -0,0 +1,22 @@
+"""Small HOG/logistic pen-vs-print router. Never changes OCR pixels/aspect."""
+import cv2,numpy as np
+def feature(rgb):
+ g=cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY)if rgb.ndim==3 else rgb
+ g=cv2.resize(g,(128,32),interpolation=cv2.INTER_AREA)
+ g=cv2.normalize(g,None,0,255,cv2.NORM_MINMAX)
+ hog=cv2.HOGDescriptor((128,32),(16,16),(8,8),(8,8),9)
+ v=hog.compute(g).ravel()
+ return np.r_[v,(g<128).mean(),(g.std()/255)].astype(np.float32)
+class PenClassifier:
+ def __init__(self,path):
+  import joblib
+  d=joblib.load(path);self.model=d['model'];self.threshold=d['threshold']
+ def probability(self,rgb):
+  g=cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY)
+  t,_=cv2.threshold(g,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
+  yy,xx=np.where(g<min(180,max(50,t+1)))
+  if not len(xx):return 0.
+  im=rgb[yy.min():yy.max()+1,xx.min():xx.max()+1]
+  width=min(im.shape[1],max(32,round(5*im.shape[0])))
+  views=[feature(im[:,x:x+width])for x in sorted(set([0,max(0,(im.shape[1]-width)//2),max(0,im.shape[1]-width)]))]
+  return float(np.median(self.model.predict_proba(np.array(views))[:,1]))

stallion/bina04-detector-role-isolation-95k-20261103/production.py

@@ -0,0 +1,21 @@
+"""Match frozen crop protocol without leaking the page detector into crop guard."""
+import importlib.util
+from pathlib import Path
+R=Path(__file__).resolve().parent
+class Runtime:
+ def __init__(self,device='auto'):
+  s=importlib.util.spec_from_file_location('guard_frozen_parent','__BINA_ROOT__/stallion/bina04-cell-rowcores-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime('cpu');numeric=self.parent.parent.parent;self.numeric=numeric;self.core=numeric.engine.parent.parent;self.raw=self.core.patch.reader.reader
+  import baseline.crop_line_guard as g
+  original=next(c for c in g.CropLineGuard.__mro__ if c.__name__=='CropLineGuard'and c.__module__=='baseline.crop_line_guard');self.crop_guard=original();self.page_guard=g.CropLineGuard();self.guard_assignments=[]
+  import torch
+  if device=='auto':device='cuda'if torch.cuda.is_available()else'cpu'
+  if device=='cuda':
+   from gpu_eval_runtime import configure_fp32
+   configure_fp32()
+   for reader in [self.raw,self.numeric.numeric,self.core.patch.overlay.layer.reader]:reader.model.to('cuda');reader.device=torch.device('cuda')
+ def __call__(self,image,old,kind,name='',**kwargs):
+  with self.numeric.lock:
+   self.core.patch.base._crop_line_guard=self.crop_guard if kind=='crop'else self.page_guard
+   out=self.parent(image,old,kind,name,**kwargs)
+  out['detector_role_isolation']=dict(main_page_detector='Frozen combined Surya step1000',crop_guard='Original Indra LineNet'if kind=='crop'else'Frozen learned page guard',detector_device='cpu',recognizer_device=str(self.raw.device),wiring_does_not_change_weights=True);return out
+ def close(self):self.parent.close()

stallion/bina04-document-handoff-95k-20261103/document.py

@@ -0,0 +1,80 @@
+"""General handoff invariants and document serialization, no document identities."""
+import copy,re,unicodedata
+import numpy as np,cv2
+
+def box(r):
+ if 'bbox'in r:return list(r['bbox'])
+ p=r['polygon'];return[min(a[0]for a in p),min(a[1]for a in p),max(a[0]for a in p),max(a[1]for a in p)]
+def area(b):return max(0,b[2]-b[0])*max(0,b[3]-b[1])
+def overlap(a,b):return area([max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])])/max(1,area(a))
+def lost_main_text(image,pipeline):
+ gray=np.asarray(image.convert('L'));H,W=gray.shape;coverage=np.zeros((H,W),np.uint8)
+ for l in pipeline['lines']:
+  b=l.get('geometry',{}).get('source_pixels_extent',l['box']);x,y,X,Y=map(round,b);x=max(0,x);y=max(0,y);X=min(W,X);Y=min(H,Y);coverage[y:Y,x:X]=1
+ failed=[];checked=[]
+ raw=pipeline.get('raw_line_detection',{}).get('full_image_SDK',{}).get('bboxes',[])
+ for a in raw:
+  b=box(a);x,y,X,Y=map(round,b)
+  if X-x<.5*W or Y-y>.06*H or Y-y<3:continue
+  x=max(0,x);y=max(0,y);X=min(W,X);Y=min(H,Y);g=gray[y:Y,x:X]
+  if not g.size:continue
+  val,_=cv2.threshold(g,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU);ink=g<min(190,max(60,val+1));n=int(ink.sum());fraction=float((ink*(coverage[y:Y,x:X]>0)).sum()/max(1,n));rec=dict(box=b,source_ink=n,emitted_ink_fraction=fraction);checked.append(rec)
+  if n>=20 and fraction<.5:failed.append(rec)
+ return dict(fallback_required=len(failed)>=3,failed_primary_lines=failed,checked=checked,criterion='At least3 wide primary detector lines have <50% ink covered by emitted crops; same pixels, no text labels used')
+
+def ltr_decode(line):
+ from indra_ocr.rtl import flip,has_rtl
+ if line.get('ltr_repair'):return line
+ old=line['text'];g=line.get('geometry',{});raw=g.get('raw_visual_text',line.get('visual_text',''))
+ # Tatweel is a layout extender. Remove it before choosing runs, never after reordering.
+ if raw and any(c.isascii()and c.isalpha()for c in raw):
+  v=raw.replace('\u0640','');t=flip(v,base_rtl=True)
+  # Literal email punctuation can be joined without inventing letters or domains.
+  t=re.sub(r'(?<=[A-Za-z0-9])\s*([@.])\s*(?=[A-Za-z0-9])',r'\1',t)
+  t=re.sub(r'(?<=[A-Za-z0-9])\s+([.])\s*(?=[A-Za-z0-9])',r'\1',t)
+  if t!=old:return dict(line,text=t,document_decode=dict(original=old,raw_visual_used=True,tatweel_removed_before_bidi=True))
+ return line
+
+def row_order(lines):
+ rows=[]
+ for l in sorted(lines,key=lambda l:((l['box'][1]+l['box'][3])/2,-l['box'][0])):
+  b=l['box'];found=False
+  for group in rows[-3:]:
+   g=group[0]['box'];vy=min(b[3],g[3])-max(b[1],g[1]);vx=min(b[2],g[2])-max(b[0],g[0])
+   if vy>=.5*min(b[3]-b[1],g[3]-g[1])and vx<=0:group.append(l);found=True;break
+  if not found:rows.append([l])
+ return [l for group in rows for l in sorted(group,key=lambda l:-l['box'][0])]
+def join_fields(lines,W):
+ result=copy.deepcopy(lines);used=set()
+ for i,key in enumerate(result):
+  b=key['box'];t=key['text'].strip()
+  if not t or t[-1]not in ':،' or len(t)>20 or b[2]-b[0]>.25*W:continue
+  options=[]
+  for j,v in enumerate(result):
+   if j==i or j in used:continue
+   z=v['box'];s=''.join(v['text'].split());value=bool(s)and(all(c.isdigit()or c in '/:.-٫،'for c in s)or bool(re.search(r'[@.]',s)and any(c.isascii()and c.isalpha()for c in s)))
+   h=max(b[3]-b[1],z[3]-z[1]);distance=abs((b[1]+b[3]-z[1]-z[3])/2);gap=b[0]-z[2]
+   if value and 0<=gap<=3*h and distance<=1.5*h:options.append((distance,j))
+  if options:
+   _,j=min(options);v=result[j];key['text']=t+' '+v['text'].strip();key['joined_field_evidence']=[copy.deepcopy(lines[i]),copy.deepcopy(lines[j])];used.add(j)
+ return [l for i,l in enumerate(result)if i not in used]
+
+def serialize(image,pipeline,graphic_threshold=.75):
+ out=copy.deepcopy(pipeline);W,H=image.size;layout=out.get('raw_layout',{}).get('bboxes',[]);graphics=[box(r)for r in layout if str(r.get('label','')).lower()in ['picture','figure','image','diagram']];removed=[];lines=[]
+ for l in out['lines']:
+  if l.get('uncertain_graphic')or any(overlap(l['box'],r)>=.6 for r in graphics)and l.get('confidence',0)<graphic_threshold:removed.append(l);continue
+  lines.append(ltr_decode(l))
+ # Separate a narrow edge column from a wider main column; preserve all surviving reads.
+ b=[l['box']for l in lines];candidates=[]
+ for start,end,side in [(0,.25*W,'left'),(.75*W,W,'right')]:
+  narrow=[i for i,a in enumerate(b)if start<=(a[0]+a[2])/2<=end and a[2]-a[0]<.3*W];wide=[i for i,a in enumerate(b)if a[2]-a[0]>.4*W and i not in narrow]
+  if len(narrow)<3 or len(wide)<3:continue
+  if side=='right':gap=min(b[i][0]for i in narrow)-max(b[i][2]for i in wide)
+  else:gap=min(b[i][0]for i in wide)-max(b[i][2]for i in narrow)
+  if gap>max(5,.005*W):candidates.append((len(narrow),side,narrow))
+ if candidates:
+  _,side,ids=max(candidates);ids=set(ids);main=[l for i,l in enumerate(lines)if i not in ids];sidebar=[l for i,l in enumerate(lines)if i in ids];main=row_order(join_fields(main,W));sidebar=row_order(join_fields(sidebar,W));lines=main+sidebar
+ else:side=None
+ # Keep generic grid serialization atomic instead of flattening cells.
+ if any(r.get('table_trace',{}).get('grid')for r in pipeline['regions']):return copy.deepcopy(pipeline)
+ out.update(lines=lines,regions=[dict(path='document_columns',lines=lines,text='\n'.join(l['text']for l in lines))],text='\n'.join(l['text']for l in lines),global_readings=lines,emitted_boxes=[l['box']for l in lines],recognition_extents=[l.get('geometry',{}).get('source_pixels_extent',l['box'])for l in lines]);out['document_handoff']=dict(graphic_confidence_threshold=graphic_threshold,graphic_reads_removed=removed,narrow_sidebar=side,main_column_before_sidebar=bool(side),no_document_identity_used=True);return out

stallion/bina04-document-handoff-95k-20261103/numeric_repairs.py

@@ -0,0 +1,106 @@
+"""Image-grounded numeric groups and LTR runs; no values, hosts or words hardcoded."""
+import re,copy,collections,unicodedata
+import cv2,numpy as np
+from PIL import Image
+MAP=str.maketrans('٠١٢٣٤٥٦٧٨٩','۰۱۲۳۴۵۶۷۸۹')
+def slash_pixels(gray):
+ _,m=cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU);yy,xx=np.where(m>0)
+ if len(xx)<8:return False
+ w=xx.max()-xx.min()+1;h=yy.max()-yy.min()+1
+ if not(.12<w/max(1,h)<.85)or h<.25*gray.shape[0]:return False
+ return bool(np.corrcoef(xx,yy)[0,1]<-.7)
+def compress_extenders(gray,ratio=.75):
+ _,m=cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU);ink=m>0;yy,xx=np.where(ink)
+ if not len(xx):return gray,[]
+ h=yy.max()-yy.min()+1;counts=ink.sum(0);thin=(counts>0)&(counts<=max(2,.12*h));e=np.diff(np.r_[False,thin,False].astype(int));remove=[];trace=[]
+ for a,b in zip(np.flatnonzero(e==1),np.flatnonzero(e==-1)):
+  if b-a<ratio*h or a<.25*h or b>gray.shape[1]-.25*h:continue
+  ys=np.where(ink[:,a:b])[0]
+  if not len(ys)or ys.max()-ys.min()+1>max(3,.2*h):continue
+  left=np.where(ink[:,max(0,a-round(h)):a])[0];right=np.where(ink[:,b:min(gray.shape[1],b+round(h))])[0]
+  if not len(left)or not len(right)or np.ptp(left)<.4*h or np.ptp(right)<.4*h:continue
+  keep=max(2,round(.12*h));remove+=list(range(a+keep,b));trace.append(dict(x0=int(a),x1=int(b),kept=keep))
+ if not remove:return gray,[]
+ return np.delete(gray,remove,axis=1),trace
+
+def url_normal(s):
+ s=s.replace('\u0640','');s=re.sub(r'(?<=[A-Za-z0-9])\s*([@.٫])\s*(?=[A-Za-z0-9])',lambda m:'.'if m[1]=='٫'else m[1],s)
+ return s.strip()
+def url_valid(s):return bool(re.fullmatch(r'(?:https?://)?(?:[A-Za-z0-9._%+\-]+@)?(?:[A-Za-z0-9\-]+\.)+[A-Za-z]{2,16}(?:[/:?#][\x21-\x7e]*)?',s))
+class Repairs:
+ def __init__(self,primary,numeric,short):self.readers=dict(primary=primary,numeric=numeric,short=short)
+ def read(self,gray,boxes,scales=(3,)):
+  from text_contract import decode_prediction
+  rows=[[]for b in boxes]
+  for name,reader in self.readers.items():
+   for scale in scales:
+    work=cv2.resize(gray,None,fx=scale,fy=scale,interpolation=cv2.INTER_CUBIC)if scale!=1 else gray
+    for start in range(0,len(boxes),24):
+     bb=boxes[start:start+24];_,gg=reader(work,np.asarray([[v*scale for v in b]for b in bb],np.float32))
+     for j,g in enumerate(gg):rows[start+j].append(dict(reader=name,scale=scale,text=decode_prediction(g['raw_visual_text']),raw=g['raw_visual_text'],confidence=float(g['confidence'])))
+  return rows
+ def dates(self,image,line):
+  raw=line.get('geometry',{}).get('raw_visual_text',line.get('visual_text',''));xs=line.get('geometry',{}).get('frame_centers',[]);b=line['box'];h=b[3]-b[1]
+  if len(xs)!=len(raw)or not any(unicodedata.bidirectional(c)in('R','AL')for c in raw):return line
+  spans=[];self.date_diagnostics=[];gray=np.asarray(image.convert('L'));H,W=gray.shape
+  for m in re.finditer(r'[\d\s/]{3,}',raw):
+   if m.group().count('/')<2 or not re.search(r'\d\s+/|/\s+\d',m.group()):continue
+   xx=xs[m.start():m.end()];a=max(0,round(min(xx)-.2*h));A=min(W,round(max(xx)+.2*h));y=max(0,round(b[1]-2));Y=min(H,round(b[3]+2));g=gray[y:Y,a:A];_,ink=cv2.threshold(g,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU);active=(ink>0).sum(0)>1;e=np.diff(np.r_[False,active,False].astype(int));groups=[]
+   for x,X in zip(np.flatnonzero(e==1),np.flatnonzero(e==-1)):
+    if groups and x-groups[-1][1]<=.3*h:groups[-1][1]=int(X)
+    else:groups.append([int(x),int(X)])
+   boxes=[[max(0,a+x-2),y,min(W,a+X+2),Y]for x,X in groups];reads=self.read(gray,boxes);tokens=[];evidence=[]
+   for bb,rr in zip(boxes,reads):
+    votes=collections.defaultdict(dict)
+    for r in rr:
+     t=''.join(r['text'].split()).translate(MAP)
+     if re.fullmatch(r'[\d/]+',t)and any(c.isdigit()for c in t):votes[t][r['reader']]=r['confidence']
+    values=[t for t,models in votes.items()if len(models)>=2 and np.mean(sorted(models.values(),reverse=True)[:2])>=.9]
+    if len(values)==1:tokens.extend(re.findall(r'\d+|/',values[0]));evidence.append(dict(box=bb,value=values[0],reads=rr))
+    elif slash_pixels(gray[bb[1]:bb[3],bb[0]:bb[2]]):tokens.append('/');evidence.append(dict(box=bb,value='/',source='diagonal source ink'))
+    else:tokens.append(None);evidence.append(dict(box=bb,value=None,reads=rr))
+   self.date_diagnostics.append(dict(raw_span=m.group(),tokens=tokens,evidence=evidence))
+   # Ignore flanking letters; require every internal group to have source evidence.
+   good=[i for i,t in enumerate(tokens)if t is not None]
+   if not good:continue
+   ts=tokens[min(good):max(good)+1]
+   if None in ts:continue
+   nums=[t for t in ts if t!='/'];rev=list(reversed(ts));valid=False
+   if len(nums)==3 and len(nums[0])==4 and int(nums[0])>=1000 and 1<=int(nums[1])<=12 and 1<=int(nums[2])<=31 and ts.count('/')==2:valid=True
+   elif len(nums)==2 and ts.count('/')==2 and ts[0]=='/' and 1<=int(nums[0])<=12 and 1<=int(nums[1])<=31:valid=True
+   if valid:spans.append((m.start(),m.end(),' '.join(rev),evidence,m.group()))
+  if not spans:return line
+  result=copy.deepcopy(line);text=line['text'];replacements=[]
+  for start,end,new,evidence,old_run in spans:
+   digits=lambda t:''.join(c for c in t.translate(MAP)if c.isdigit())
+   matches=[m for m in re.finditer(r'[\d\s/]{3,}',text)if m.group().count('/')>=2 and digits(m.group())==digits(old_run)]
+   if len(matches)!=1:continue
+   m=matches[0];a=m.start();A=m.end();leading=' 'if m.group()[:1].isspace()else'';trailing=' 'if m.group()[-1:].isspace()else'';text=text[:a]+leading+new+trailing+text[A:];replacements.append(dict(original=m.group(),new=new,evidence=evidence))
+  if not replacements:return line
+  result['text']=text;result['numeric_group_reboxing']=dict(original=line['text'],source_groups=replacements,values_require_two_model_agreement=True,words_outside_numeric_runs_preserved=True);return result
+ def latin(self,image,line):
+  old=line['text'];raw=line.get('geometry',{}).get('raw_visual_text',line.get('visual_text',''));base=url_normal(raw)
+  if not any(k in raw.lower()for k in ['www','http','@']):return line
+  if url_valid(base):return dict(line,text=base,ltr_repair=dict(source='original visual glyph run',original=old))
+  gray=np.asarray(image.convert('L'));rr=self.read(gray,[line['box']],scales=(1,3))[0];candidates=[]
+  for r in rr:
+   t=url_normal(r['raw'])
+   if not url_valid(t)or r['confidence']<.85:continue
+   key=re.sub('[^A-Za-z0-9]','',t).lower();models={s['reader']for s in rr if re.sub('[^A-Za-z0-9]','',s['raw']).lower()==key}
+   same=[s for s in rr if url_normal(s['raw'])==t]
+   if len(models)>=2 and len({s['scale']for s in same})==2:candidates.append((r['confidence'],t,r))
+  if not candidates:return line
+  _,t,r=max(candidates);return dict(line,text=t,ltr_repair=dict(source='two-model alphanumeric agreement; two-scale punctuation agreement',original=old,reader=r['reader']))
+ def extenders(self,image,line):
+  t=line['text']
+  if len(t)>30 or any(c.isdigit()or c.isascii()and c.isalpha()for c in t):return line
+  x,y,X,Y=map(round,line['box']);gray=np.asarray(image.convert('L'))[y:Y,x:X]
+  if not gray.size:return line
+  work,trace=compress_extenders(gray)
+  if not trace:return line
+  rr=self.read(work,[[0,0,work.shape[1],work.shape[0]]])[0];votes=collections.defaultdict(set)
+  for r in rr:
+   if r['confidence']>=.9:votes[r['text'].strip()].add(r['reader'])
+  options=[s for s,v in votes.items()if len(v)>=2 and s]
+  if len(options)!=1:return line
+  return dict(line,text=options[0],extender_repair=dict(source='source-pixel horizontal connector compression; two-model agreement',original=t,compressed_spans=trace))

stallion/bina04-document-handoff-95k-20261103/production.py

@@ -0,0 +1,38 @@
+"""A page handoff cannot discard primary text ink. OCR remains image-grounded."""
+import importlib.util,json,sys,copy
+from pathlib import Path
+R=Path(__file__).resolve().parent;V=Path('__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031')
+class Runtime:
+ def __init__(self,device='auto'):
+  s=importlib.util.spec_from_file_location('frozen_header_parent','__BINA_ROOT__/stallion/bina04-header-marks-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-header-marks-95k-20261103');sys.path.insert(0,str(R));from numeric_repairs import Repairs
+  numeric=self.parent.parent;numeric.capture.reset();self.core=numeric.engine.parent.parent;self.repair=Repairs(self.core.patch.reader.reader,numeric.numeric,self.core.patch.overlay.layer.reader);self.generic=None
+ def generic_page(self,image,old):
+  if self.generic is None:
+   sys.path.insert(0,str(V));from pipeline_v3 import PagePipeline,UnifiedReader
+   import numpy as np
+   class WholeLinePipeline(PagePipeline):
+    def detect(self,image):
+     boxes,trace=super().detect(image);heights=[b[3]-b[1]for b in boxes if b[3]>b[1]];median=float(np.median(heights))if heights else 1.;keep=[];dropped=[]
+     from document import overlap
+     for i,b in enumerate(boxes):
+      if any(i!=j and overlap(b,a)>=.8 and b[3]-b[1]<.6*(a[3]-a[1])and a[3]-a[1]<=2*median for j,a in enumerate(boxes)):dropped.append(b)
+      else:keep.append(b)
+     trace['contained_partial_height_slivers_removed']=dropped;return keep,trace
+   self.generic=WholeLinePipeline(self.core.patch.detector,self.core.patch.base.layout,UnifiedReader(V/'run/weights-095000.pt',V/'data/vocab.json',device=str(self.core.patch.reader.reader.device)),self.core.patch.table)
+  self.core.context['layout']=copy.deepcopy(old['raw_layout']);return self.generic(image,input_type='page')
+ def __call__(self,image,old,kind,name='',**kwargs):
+  base=self.parent(image,old,kind,name,**kwargs)
+  return self.finish(image,base,old,kind)
+ def finish(self,image,base,old,kind):
+  from document import lost_main_text,serialize
+  if kind!='page':return base
+  guard=lost_main_text(image,base)
+  if not guard['fallback_required']:return base
+  out=self.generic_page(image,old);out['ink_handoff_guard']=guard
+  from region_reads import supplement,display_evidence
+  out=supplement(image,out,self.repair)
+  for i,line in enumerate(out['lines']):
+   line=self.repair.dates(image,line);line=self.repair.latin(image,line);line=self.repair.extenders(image,line);out['lines'][i]=line
+  out=display_evidence(image,out,self.repair)
+  return serialize(image,out,graphic_threshold=.75)
+ def close(self):self.parent.close()

stallion/bina04-document-handoff-95k-20261103/region_reads.py

@@ -0,0 +1,59 @@
+"""General small text-box reads and independent-reader evidence on display fonts."""
+import copy,re,unicodedata
+from difflib import SequenceMatcher
+import cv2,numpy as np
+from PIL import Image
+from document import box,overlap
+
+def ns(s):return ''.join(s.split()).replace('\u0640','')
+def consensus(rows,minimum=.85):
+ groups={}
+ for r in rows:
+  key=ns(r['text'])
+  if not key:continue
+  groups.setdefault(key,[]).append(r)
+ good=[]
+ for k,rr in groups.items():
+  models={r['reader']for r in rr}
+  if len(models)>=2 and np.mean(sorted([r['confidence']for r in rr],reverse=True)[:2])>=minimum:good.append((len(models),max(r['confidence']for r in rr),rr))
+ if not good:return None
+ _,_,rr=max(good,key=lambda a:a[:2]);return max(rr,key=lambda r:r['confidence'])
+
+def supplement(image,pipeline,repairs):
+ import sys
+ sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-crop-bandseg-95k-20261102')
+ from bandseg import mask,horizontal
+ W,H=image.size;result=copy.deepcopy(pipeline);new=[];debug=[]
+ for region in result.get('raw_layout',{}).get('bboxes',[]):
+  if region.get('label','').lower()!='text':continue
+  b=box(region);x,y,X,Y=map(round,b)
+  if X-x>.3*W or Y-y>.12*H or Y-y<6:continue
+  rgb=np.asarray(image.crop((x,y,X,Y)).convert('RGB'));g=rgb.min(2).astype('float32');bg=cv2.GaussianBlur(g,(0,0),max(7,(Y-y)/8));norm=np.clip(g/(bg+1e-4)*245,0,255).astype('uint8');work=cv2.resize(norm,None,fx=3,fy=3,interpolation=cv2.INTER_CUBIC);ink,glyph=mask(work);boxes,trace=horizontal(ink,glyph,scale=3)
+  if not boxes:continue
+  reads=repairs.read(work,boxes,scales=(1,))
+  for bb,rows in zip(boxes,reads):
+   target=[bb[0]/3+x,bb[1]/3+y,bb[2]/3+x,bb[3]/3+y]
+   if any(l.get('text','').strip()and overlap(target,l['box'])>=.6 and l['box'][3]-l['box'][1]<=1.8*(target[3]-target[1])for l in result['lines']):
+    debug.append(dict(box=target,covered=True,reads=rows));continue
+   chosen=consensus(rows,.9);debug.append(dict(box=target,chosen=chosen,reads=rows))
+   if chosen is None:continue
+   # A tighter reread of the same glyphs is not a second text line.
+   if any(l.get('text','').strip() and overlap(target,l['box'])>=.8 and SequenceMatcher(None,ns(chosen['text']),ns(l['text'])).ratio()>=.7 for l in result['lines']):continue
+   if any(overlap(target,l['box'])>.5 and ns(chosen['text'])==ns(l['text'])for l in new):continue
+   new.append(dict(box=target,text=chosen['text'],visual_text=chosen['raw'],confidence=chosen['confidence'],recognizer='existing_reader_consensus_small_textbox',geometry=dict(source_pixels_extent=target,padded_source_extent=target,raw_visual_text=chosen['raw'],frame_centers=[],frame_centers_source_valid=False,source_pixels_modified=True),source_region=region,small_textbox_read=dict(upscale=3,background_normalized=True,readers=rows)))
+ result['lines']+=new;result['small_textbox_supplement']=dict(added=new,template_text_used=False,diagnostics=debug);return result
+
+def display_evidence(image,pipeline,repairs):
+ W,H=image.size;result=copy.deepcopy(pipeline);gray=np.asarray(image.convert('L'));graphic=[r for r in result.get('raw_layout',{}).get('bboxes',[])if r.get('label','').lower()in['picture','figure','image','diagram']]
+ for i,line in enumerate(result['lines']):
+  b=line['box'];g=max([r.get('confidence',0)for r in graphic if overlap(b,box(r))>=.6]or[0.]);edge=(b[0]+b[2])/2>.75*W or(b[0]+b[2])/2<.25*W;display=edge and b[3]-b[1]>.018*H and b[2]-b[0]<.3*W
+  if not(g or display):continue
+  if g:
+   score=line.get('confidence',0)*(1-g);line['document_text_confidence']=score
+   # The raw read is retained in provenance even when not emitted as document text.
+   if score<.75:line['uncertain_graphic']=True;continue
+  if display:
+   rr=repairs.read(gray,[b],scales=(1,3))[0];chosen=consensus(rr,.9);models={r['reader']for r in rr if chosen and ns(r['text'])==ns(chosen['text'])};line['display_font_evidence']=rr
+   if chosen and len(models)==3:line['text']=chosen['text']
+   else:line['uncertain_graphic']=True
+ return result

stallion/bina04-e2e-batch-95k-20261101/candidates/combined/learned_backend.py

@@ -0,0 +1,30 @@
+"""Substitute the trained detector in existing newspaper/cell line roles.
+
+Keep the frozen guard, margins, NMS, grid, ordering and recognizer code intact.
+The SDK preprocessing used here is also used for synthetic page/cell training.
+"""
+import hashlib
+import numpy as np
+from PIL import Image
+def install(detector,model_sha256,yield_check=lambda:None):
+    import baseline.crop_line_guard as module
+    original=module.CropLineGuard
+    cache={}
+    class Engine:
+        long_side=0
+        def detect(self,gray):
+            key=hashlib.sha256(str(gray.shape).encode()+gray.tobytes()).hexdigest()
+            if key not in cache:
+                yield_check();raw=detector([Image.fromarray(gray).convert('RGB')],batch_size=1)[0].model_dump(mode='json');rows=[]
+                for item in raw['bboxes']:
+                    p=np.asarray(item['polygon']);rows.append([float(p[:,0].min()),float(p[:,1].min()),float(p[:,0].max()),float(p[:,1].max()),float(item.get('confidence',0.))])
+                cache[key]=np.asarray(rows,np.float32).reshape(-1,5)
+            return cache[key].copy()
+    class LearnedGuard(original):
+        def __init__(self):self.engine=Engine();self.sha256=model_sha256
+        def __call__(self,*args,**kwargs):
+            boxes,trace=super().__call__(*args,**kwargs)
+            trace.update(detector='Selected learned Surya physical-line model',model_sha256=model_sha256,providers=['CUDA FP32 SDK'],legacy_long_side_not_used_by_SDK=True)
+            return boxes,trace
+    module.CropLineGuard=LearnedGuard
+    return dict(model_sha256=model_sha256,roles=['whole_page','native_context','newspaper_guard_and_rescan','table_cell_lines'],recognizer_preprocessing_changed=False,heuristic_rules_changed=False)

stallion/bina04-e2e-batch-95k-20261101/candidates/combined/run/checkpoints/step-001000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3208fac304f7c8a64735587f1e46024acb51cc0b0ef2d2288944eadd8992d4c9
+size 153922431

stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/crop_line_guard.py

@@ -0,0 +1,39 @@
+"""Crop-only anti-merge detector. Original Indra LineNet; no recognizer banks."""
+import hashlib,json
+from pathlib import Path
+import numpy as np
+MODELS=Path('__BINA_ROOT__/stallion/bina04-indra-pilot-20261027/models')
+
+class CropLineGuard:
+    def __init__(self):
+        import onnxruntime as ort
+        from indra_ocr.pipeline import OrtOCR
+        so=ort.SessionOptions();so.intra_op_num_threads=1;so.inter_op_num_threads=1;so.enable_cpu_mem_arena=False
+        self.engine=OrtOCR.__new__(OrtOCR)
+        self.engine.det=ort.InferenceSession(str(MODELS/'det.onnx'),so,providers=['CPUExecutionProvider'])
+        self.engine.adaptive=False
+        self.sha256=hashlib.sha256((MODELS/'det.onnx').read_bytes()).hexdigest()
+    def __call__(self,gray,boxes,line):
+        long_side=max(32,round(max(gray.shape)*12/max(1,line)/32)*32)
+        self.engine.long_side=long_side
+        native=self.engine.detect(gray)[:,:4];out=[];used=set();events=[]
+        for box in boxes:
+            x,y,X,Y=box;candidate=[]
+            if Y-y>1.8*line:
+                for i,b in enumerate(native):
+                    a,t,A,T=map(float,b);h=T-t
+                    overlap=max(0,min(A,X)-max(a,x))*max(0,min(T,Y)-max(t,y))/max(1.,(A-a)*h)
+                    if i not in used and overlap>=.7 and .5*line<=h<=2.*line:candidate.append(i)
+            groups=[]
+            for i in sorted(candidate,key=lambda i:(native[i,1]+native[i,3])/2):
+                center=float((native[i,1]+native[i,3])/2)
+                if not groups or center-groups[-1][-1][0]>.6*line:groups.append([])
+                groups[-1].append((center,i))
+            if len(groups)>=2:
+                replacements=[]
+                for group in groups:
+                    members=[native[i]for _,i in group]
+                    replacements.append([min(float(b[0])for b in members),min(float(b[1])for b in members),max(float(b[2])for b in members),max(float(b[3])for b in members)])
+                out+=replacements;used.update(candidate);events.append(dict(original_box=box,replacement_boxes=replacements,native_indices=candidate))
+            else:out.append(list(box))
+        return out,dict(detector='Original Indra LineNet only; no OCR/script bank',model_sha256=self.sha256,providers=['CPUExecutionProvider'],long_side=long_side,target_input_line_height=12,native_boxes=native.tolist(),replacements=events,reference_or_layout_used=False)

stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/input_policy_v3.py

@@ -0,0 +1,96 @@
+"""Frozen rule C estimator and same-line sliver consolidation; image geometry only."""
+import numpy as np
+
+def input_kind(width,height,requested='auto'):
+    if requested not in {'auto','crop','page'}:raise ValueError('input_type must be auto/crop/page')
+    if requested!='auto':return requested
+    return 'crop' if min(width,height)<=384 or height<=128 or(height<=768 and width/max(1,height)>=2.5) else 'page'
+
+
+def estimated_line_height(gray):
+    # Exact estimator from the previously frozen rule C. No labels/categories.
+    import cv2
+    H, W = gray.shape
+    _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+    ink = (binary == 0).astype(np.uint8)
+    if ink.mean() > .5:
+        ink = 1 - ink
+    _, labels, stats, _ = cv2.connectedComponentsWithStats(ink, 8)
+    heights = [];glyph_ids=[]
+    for label,(x, y, w, h, area) in enumerate(stats[1:],1):
+        aspect = float(w) / max(1, int(h))
+        if area >= 6 and h >= 3 and w >= 2 and h <= .95 * H and .05 <= aspect <= 20.:
+            heights.append(int(h));glyph_ids.append(label)
+    glyph = float(np.percentile(heights, 75)) if heights else None
+    line = min(float(H), max(1., 1.5 * glyph)) if glyph is not None else float(H)
+    original_line=line
+    projection_glyph=glyph
+    noise_robust=False
+    projection_ids=glyph_ids
+    if H<=96 and glyph_ids:
+        selected=stats[glyph_ids];order=np.argsort(selected[:,3]);ordered=selected[order]
+        weighted=float(ordered[np.searchsorted(np.cumsum(ordered[:,4]),.75*ordered[:,4].sum()),3])
+        if weighted>=2.*max(1.,glyph):
+            # Speckle can outnumber letters. Weight height by ink area and
+            # remove tiny speckles from this routing-only projection.
+            noise_robust=True;projection_glyph=weighted
+            projection_ids=[i for i in glyph_ids if stats[i,4]>=max(6,.05*selected[:,4].max())]
+            line=max(line,min(float(H),1.5*weighted))
+    # Padded one-line strips have small glyphs inside a much taller canvas.
+    # Do not count their blank margins as extra lines. This is confined to
+    # short/wide inputs and a single compact ink band, never handwriting pages.
+    # Exclude full-height scan borders/rules from the projection, using the
+    # same glyph-component eligibility as the height estimator.
+    glyph_ink=np.isin(labels,projection_ids) if projection_ids else ink
+    active=glyph_ink.sum(axis=1)>=max(2,.0015*W)
+    gap=max(2,round(.35*(projection_glyph or 1.)))
+    edges=np.diff(np.r_[False,active,False].astype(np.int8))
+    bands=[[int(a),int(b)] for a,b in zip(np.where(edges==1)[0],np.where(edges==-1)[0])]
+    joined=[]
+    for a,b in bands:
+        if joined and a-joined[-1][1]<=gap:joined[-1][1]=b
+        else:joined.append([a,b])
+    padded_single=(H<=96 or W/max(1,H)>=3.) and len(joined)==1 and joined[0][1]-joined[0][0]<=max(6,3.5*(projection_glyph or H))
+    if padded_single:line=float(H)
+    return dict(estimated_line_height=line, estimated_glyph_height=glyph,
+                valid_components=len(heights), crop_height=H,
+                height_to_line_height=H / max(1., line), segment=H > 1.8 * line,
+                original_glyph_based_line_height=original_line,ink_bands=joined,
+                single_ink_band_with_blank_margins=bool(padded_single),
+                noise_robust_projection=noise_robust,projection_glyph_height=projection_glyph)
+
+
+def same_line(a, b):
+    overlap_y = min(a[3], b[3]) - max(a[1], b[1])
+    overlap_x = min(a[2], b[2]) - max(a[0], b[0])
+    return overlap_x > 0 and overlap_y > .5 * min(a[3] - a[1], b[3] - b[1])
+
+
+def merge_slivers(boxes, line_height=None):
+    """Union overlapping observations; complete-link groups avoid bridge chaining.
+
+    A grown union is never used to decide subsequent matches. Every pair of
+    original members must overlap, or a plausible full-height anchor must
+    contain the sliver. A grown union never recruits a neighbouring row.
+    """
+    groups = []
+    for i in sorted(range(len(boxes)), key=lambda i: (-(boxes[i][3]-boxes[i][1]), i)):
+        compatible = []
+        for j, group in enumerate(groups):
+            anchor=boxes[group[0]];box=boxes[i]
+            inside=(anchor[0]<=box[0] and anchor[1]<=box[1] and
+                    anchor[2]>=box[2] and anchor[3]>=box[3])
+            plausible=line_height is None or anchor[3]-anchor[1]<=1.8*line_height
+            if (inside and plausible) or all(same_line(box,boxes[k]) for k in group):
+                compatible.append(j)
+        if compatible:
+            groups[compatible[0]].append(i)
+        else:
+            groups.append([i])
+    groups.sort(key=min)
+    merged = [[min(boxes[i][0] for i in g), min(boxes[i][1] for i in g),
+               max(boxes[i][2] for i in g), max(boxes[i][3] for i in g)] for g in groups]
+    return merged, dict(input_boxes=len(boxes), output_boxes=len(merged),
+                        original_member_indices=groups,
+                        criterion='Vertical overlap >50% smaller height and positive horizontal overlap; complete-link unions',
+                        reference_text_or_boxes_used=False)

stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/pipeline_v3.py

@@ -0,0 +1,216 @@
+"""Layout cannot filter detection. Exactly one recognizer reads every script."""
+import math, sys, time
+from pathlib import Path
+import numpy as np
+from PIL import Image
+P=Path('__BINA_ROOT__/stallion/indra-scale')
+sys.path[:0]=[str(P),'__BINA_ROOT__/stallion/bina04-indra-pilot-20261027/upstream']
+from geometry import crop_line64
+from indra_ocr.rtl import flip,rtl_base
+
+def rectangle(polygon):
+    p=np.asarray(polygon,float)
+    return [float(p[:,0].min()),float(p[:,1].min()),float(p[:,0].max()),float(p[:,1].max())]
+
+def area(b):return max(0,b[2]-b[0])*max(0,b[3]-b[1])
+def intersection(a,b):return [max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])]
+def clipped(b,w,h):return [max(0,b[0]),max(0,b[1]),min(w,b[2]),min(h,b[3])]
+def reading_extent(line):return line['geometry'].get('source_pixels_extent',line['geometry']['padded_source_extent'])
+
+def xycut_order(boxes,width):
+    """RTL column order; spanning headings divide vertical bands."""
+    n=len(boxes)
+    raster=lambda ids:sorted(ids,key=lambda i:((boxes[i][1]+boxes[i][3])/2,-boxes[i][0]))
+    core=[b for b in boxes if b[2]-b[0]<.7*width]
+    if len(core)<4:return raster(range(n))
+    profile=np.zeros(max(1,int(math.ceil(width))),int)
+    for b in core:profile[max(0,int(b[0])):min(len(profile),int(math.ceil(b[2])))]+=1
+    low=profile<=max(1,int(.05*len(core)));low[:int(.25*width)]=False;low[int(.75*width):]=False
+    e=np.diff(np.r_[False,low,False].astype(int));candidates=[]
+    for a,b in zip(np.where(e==1)[0],np.where(e==-1)[0]):
+        if b-a<max(8,.01*width):continue
+        cut=(a+b)/2;left=[i for i,v in enumerate(boxes)if v[2]<=cut];right=[i for i,v in enumerate(boxes)if v[0]>=cut]
+        if len(left)>=2 and len(right)>=2:candidates.append((b-a,cut,left,right))
+    if not candidates:return raster(range(n))
+    _,cut,left,right=max(candidates);middle=raster(set(range(n))-set(left)-set(right));remaining=set(left+right);out=[]
+    for stop in middle+[None]:
+        boundary=float('inf')if stop is None else(boxes[stop][1]+boxes[stop][3])/2
+        for group in[right,left]:
+            take=raster(i for i in group if i in remaining and(boxes[i][1]+boxes[i][3])/2<boundary);out+=take;remaining.difference_update(take)
+        if stop is not None:out.append(stop)
+    assert sorted(out)==list(range(n));return out
+
+def order_blocks(boxes,layout,width):
+    regions=layout.get('bboxes',[]);blocks={};assign=[]
+    for i,box in enumerate(boxes):
+        scores=[area(intersection(box,rectangle(r['polygon'])))for r in regions]
+        j=int(np.argmax(scores))if scores and max(scores)>0 else None
+        key=('region',j)if j is not None else('outside',i)
+        if key not in blocks:blocks[key]=dict(key=list(key),region=regions[j]if j is not None else None,line_ids=[])
+        blocks[key]['line_ids'].append(i);assign.append(list(key))
+    blocks=list(blocks.values());bounds=[]
+    for b in blocks:
+        members=[boxes[i]for i in b['line_ids']];bounds.append([min(x[0]for x in members),min(x[1]for x in members),max(x[2]for x in members),max(x[3]for x in members)])
+        b['line_ids'].sort(key=lambda i:((boxes[i][1]+boxes[i][3])/2,-boxes[i][0]))
+    positions=[b['region'].get('position')if b['region']else None for b in blocks]
+    reasons=[]
+    if any(not isinstance(p,int)for p in positions)or len(set(positions))!=len(positions):reasons.append('missing_or_duplicate_region_order_or_outside_lines')
+    for i,a in enumerate(bounds):
+        for b in bounds[i+1:]:
+            if area(intersection(a,b))>.1*max(1,min(area(a),area(b))):reasons.append('overlapping_blocks');break
+    for block in blocks:
+        if block['region']and block['region'].get('label','').lower()=='table':continue
+        group=block['line_ids'];local=[boxes[i]for i in group];raster=sorted(range(len(group)),key=lambda i:((local[i][1]+local[i][3])/2,-local[i][0]))
+        if xycut_order(local,width)!=raster:reasons.append('multiple_columns_inside_layout_block')
+    if reasons:
+        # Apply the column fallback to physical lines, then form contiguous
+        # column chunks. Sorting the enclosing blocks alone interleaves any
+        # columns that Surya put inside one large Picture/Text region.
+        physical=xycut_order(boxes,width);owner={i:j for j,b in enumerate(blocks)for i in b['line_ids']};result=[];used_tables=set()
+        for i in physical:
+            j=owner[i];block=blocks[j];is_table=block['region']and block['region'].get('label','').lower()=='table'
+            if is_table:
+                if j not in used_tables:result.append(dict(block));used_tables.add(j)
+            elif result and result[-1].get('column_chunk_owner')==j and(boxes[i][1]+boxes[i][3])/2>=(boxes[result[-1]['line_ids'][-1]][1]+boxes[result[-1]['line_ids'][-1]][3])/2-.25*(boxes[i][3]-boxes[i][1]):result[-1]['line_ids'].append(i)
+            else:result.append(dict(block,line_ids=[i],column_chunk_owner=j))
+        mode='RTL_physical_line_columns_with_atomic_tables'
+    else:result=[blocks[i]for i in sorted(range(len(blocks)),key=lambda i:positions[i])];mode='Surya_block_order'
+    assert sorted(i for b in result for i in b['line_ids'])==list(range(len(boxes)))
+    return result,dict(mode=mode,reasons=sorted(set(reasons)),assignments=assign,region_type_line_losses=0)
+
+class UnifiedReader:
+    """One model, unfiltered CTC, logical-order output. No script classifier."""
+    def __init__(self,checkpoint,vocabulary,device='cpu'):
+        from rtl_reader import RTLReader
+        self.reader=RTLReader(checkpoint,vocabulary,device=device);self.source_is_crop=False
+    def __call__(self,gray,boxes):
+        boxes=np.asarray(boxes,np.float32).reshape(-1,4).tolist();normalization=None
+        if self.source_is_crop and len(boxes)==1 and np.allclose(boxes[0],[0.,0.,gray.shape[1],gray.shape[0]]):
+            from input_policy_v3 import estimated_line_height
+            from single_line_normalization import ink_band_box
+            estimate=estimated_line_height(gray)
+            if not estimate['segment']:
+                original=boxes[0];boxes=[ink_band_box(gray,estimate)]
+                normalization=dict(original_box=original,ink_band_box=boxes[0],one_full_width_line=True,no_detection_or_line_splitting=True)
+        if self.source_is_crop and boxes:
+            from line_isolation import atlas
+            canvas,shifted,maps=atlas(gray,boxes);_,receipts=self.reader(canvas,np.asarray(shifted,np.float32).reshape(-1,4))
+            for receipt,mapping in zip(receipts,maps):
+                dx,dy=mapping['dx'],mapping['dy'];x,y,X,Y=receipt['padded_source_extent'];receipt.update(original_WH=[gray.shape[1],gray.shape[0]],padded_source_extent=[x-dx,y-dy,X-dx,Y-dy],source_pixels_extent=mapping['visible'],neighbor_line_pixels_excluded=True)
+                receipt['frame_centers']=[float(x-dx)for x in receipt['frame_centers']]
+                x,y,X,Y=receipt['ink_box'];receipt['ink_box']=[x-dx,y-dy,X-dx,Y-dy]
+        else:_,receipts=self.reader(gray,np.asarray(boxes,np.float32).reshape(-1,4))
+        return[dict(box=list(map(float,b)),text=flip(r['raw_visual_text'],base_rtl=True),visual_text=r['raw_visual_text'],confidence=r['confidence'],recognizer='unified32M',geometry=r,confidence_filter_applied=False,whole_input_normalization=normalization)for b,r in zip(boxes,receipts)]
+
+def markdown_cell(text):return ' '.join(text.split()).replace('|',r'\|')
+
+class PagePipeline:
+    def __init__(self,detector,layout,reader,table=None):
+        self.detector=detector;self.layout=layout;self.reader=reader;self.table=table;self._crop_line_guard=None
+    def detect(self,image):
+        # The SDK performs its own resize/pad/tiling on the WHOLE input.
+        raw=self.detector([image],batch_size=1)[0].model_dump(mode='json');w,h=image.size
+        boxes=[clipped(rectangle(r['polygon']),w,h)for r in raw['bboxes']]
+        boxes=[b for b in boxes if area(b)>0]
+        # Fixed native-resolution windows are independent of every layout label.
+        # Full-width SDK downsampling otherwise makes dense columns/cells tiny.
+        def starts(size,tile,overlap):
+            if size<=tile:return [0]
+            out=list(range(0,size-tile+1,tile-overlap))
+            if out[-1]!=size-tile:out.append(size-tile)
+            return out
+        traces=[];supplements=[];background=tuple(int(v)for v in np.median(np.asarray(image).reshape(-1,3),axis=0));size=max(self.detector.processor.size.values())
+        def context(piece,x,y,scale):
+            # Padding prevents prepare_image's final square stretch from
+            # deforming glyphs. Recognition never uses these resized pixels.
+            pw,ph=piece.size;rw=max(1,round(pw*scale));rh=max(1,round(ph*scale));canvas=Image.new('RGB',(size,size),background);canvas.paste(piece.resize((rw,rh),Image.Resampling.LANCZOS),(0,0));part=self.detector([canvas],batch_size=1)[0].model_dump(mode='json');out=[]
+            for item in part['bboxes']:
+                a,b,A,B=rectangle(item['polygon']);a=max(0,min(rw,a));A=max(0,min(rw,A));b=max(0,min(rh,b));B=max(0,min(rh,B));v=clipped([a*pw/rw+x,b*ph/rh+y,A*pw/rw+x,B*ph/rh+y],w,h)
+                if area(v)>0:out.append(v)
+            traces.append(dict(tile=[x,y,x+pw,y+ph],scale=scale,canvas_WH=[size,size],boxes=out));supplements.extend(out)
+        for y in starts(h,size,240):
+            for x in starts(w,size,240):context(image.crop((x,y,min(w,x+size),min(h,y+size))),x,y,1.)
+        if max(w,h)<=1400:context(image,0,0,min(.5,size/max(w,h)))
+        from fusion_v3 import fuse
+        merged,fusion=fuse(np.asarray(image.convert('L')),boxes,supplements)
+        return merged,dict(full_image_SDK=raw,native_resolution_tiles=traces,fusion=fusion,layout_used_for_detection=False,recognizer_pixels='Original image only; detector context resize is inverted geometrically')
+
+    def table_block(self,image,gray,block,boxes,readings):
+        region=block['region'];w,h=image.size;b=clipped(rectangle(region['polygon']),w,h);x,y,X,Y=map(int,[math.floor(b[0]),math.floor(b[1]),math.ceil(b[2]),math.ceil(b[3])]);crop=image.crop((x,y,X,Y))
+        if self.table is None:return None,dict(error='Surya_table_engine_not_ready',all_original_lines_retained=True)
+        try:result=self.table(crop)
+        except Exception as e:return None,dict(error=type(e).__name__+': '+str(e),all_original_lines_retained=True)
+        cells=result.get('cells',[])
+        if result.get('error')or not cells:return None,dict(error='Surya_table_grid_empty',grid=result,all_original_lines_retained=True)
+        cells=sorted(cells,key=lambda c:(c['row_id'],c.get('col_id')if c.get('col_id')is not None else c.get('within_row_id',0)));pieces=[];meta=[];represented=set();cell_fragments={i:[]for i in block['line_ids']}
+        for cell in cells:
+            cb=cell.get('bbox')or rectangle(cell['polygon']);cb=[cb[0]+x,cb[1]+y,cb[2]+x,cb[3]+y];parts=[]
+            for i in block['line_ids']:
+                part=intersection(boxes[i],cb)
+                if part[2]-part[0]>=2 and part[3]-part[1]>=.5*(boxes[i][3]-boxes[i][1]):
+                    # A physical row inside a cell keeps the cell's horizontal
+                    # context; only its vertical line band is cropped.
+                    part=[cb[0],part[1],cb[2],part[3]];parts.append(part);represented.add(i);cell_fragments[i].append(part)
+            if not parts:parts=[cb]
+            parts.sort(key=lambda p:((p[1]+p[3])/2,-p[0]));bands=[]
+            for part in parts:
+                hit=next((j for j,b in enumerate(bands)if min(part[3],b[3])-max(part[1],b[1])>=.65*min(part[3]-part[1],b[3]-b[1])),None)
+                if hit is None:bands.append(part)
+                else:bands[hit]=[cb[0],min(bands[hit][1],part[1]),cb[2],max(bands[hit][3],part[3])]
+            begin=len(pieces);pieces+=bands;meta.append((cell,cb,begin,len(pieces)))
+        # A grid may not silently erase part of a detected line. A line entirely
+        # outside cells is a retained note; a partly clipped line fails back to
+        # the original block rather than emitting a misleading partial table.
+        import cv2
+        coverage=[]
+        for i in represented:
+            b=boxes[i];a,c,A,C=map(int,[math.floor(b[0]),math.floor(b[1]),math.ceil(b[2]),math.ceil(b[3])]);patch=gray[c:C,a:A];_,ink=cv2.threshold(patch,0,255,(cv2.THRESH_BINARY if np.median(patch)<128 else cv2.THRESH_BINARY_INV)|cv2.THRESH_OTSU);mask=np.zeros(patch.shape,bool)
+            for q in cell_fragments[i]:
+                height=q[3]-q[1];u,v,U,V=map(int,[math.floor(q[0]-.3*height),math.floor(q[1]-.2*height),math.ceil(q[2]+.3*height),math.ceil(q[3]+.2*height)]);mask[max(0,v-c):min(C-c,V-c),max(0,u-a):min(A-a,U-a)]=True
+            n=int((ink>0).sum());fraction=float((mask&(ink>0)).sum())/n if n else 1.;coverage.append(dict(source_line=i,ink_fraction=fraction))
+        if any(r['ink_fraction']<.98 for r in coverage):return None,dict(error='Table_grid_clips_detected_line_ink',grid=result,coverage=coverage,all_original_lines_retained=True)
+        lines=self.reader(gray,pieces);rows={};rtl=rtl_base(' '.join(l['text']for l in lines))
+        for cell,cb,a,b in meta:
+            rows.setdefault(cell['row_id'],[]).append(dict(text=' '.join(line['text']for line in lines[a:b]),box=cb,colspan=cell.get('colspan',1)))
+        rendered=[]
+        for j,(_,row)in enumerate(sorted(rows.items())):
+            row.sort(key=lambda c:(-1 if rtl else 1)*(c['box'][0]+c['box'][2])/2);values=[]
+            for cell in row:values+=[cell['text']]+['']*(max(1,cell['colspan'])-1)
+            rendered.append('| '+' | '.join(markdown_cell(s)for s in values)+' |')
+            if j==0:rendered.append('| '+' | '.join('---'for _ in values)+' |')
+        # A line not represented by any cell remains in output after the table.
+        remainder=[i for i in block['line_ids']if i not in represented]
+        rendered +=[readings[i]['text']for i in remainder]
+        emitted=lines+[readings[i]for i in remainder]
+        return dict(text='\n'.join(rendered),lines=emitted),dict(grid=result,represented_line_ids=sorted(represented),unassigned_line_ids=remainder,grid_text_never_used=True,Markdown_first_row_separator=True,column_direction='RTL'if rtl else'LTR',cell_coverage=coverage)
+    def __call__(self,image,*,input_type='auto',**_):
+        from input_policy_v3 import estimated_line_height,merge_slivers,input_kind
+        tick=time.monotonic();image=image.convert('RGB');gray=np.asarray(image.convert('L'));estimate=estimated_line_height(gray);kind=input_kind(image.width,image.height,input_type);self.reader.source_is_crop=kind=='crop'
+        if kind=='crop'and not estimate['segment']:
+            boxes=[[0.,0.,float(image.width),float(image.height)]];readings=self.reader(gray,boxes);assert len(readings)==1
+            region=dict(layout=None,source_line_ids=[0],path='rule_C_whole_input_unified_reader',text=readings[0]['text'],lines=readings)
+            return dict(text=readings[0]['text'],regions=[region],lines=readings,global_readings=readings,detected_boxes=[],emitted_boxes=[r['box']for r in readings],recognition_extents=[reading_extent(r)for r in readings],raw_line_detection=dict(skipped=True,reason='Rule C: height <=1.8 estimated line height'),raw_layout=dict(bboxes=[],skipped=True),order=dict(mode='single_input_rule_C',region_type_line_losses=0),fallbacks=[],crop_routing_policy=dict(mode='c',input_type=kind,estimate=estimate,whole_crop_by_policy=True,native_detector_calls=0),region_type_line_losses=0,recognizer_bank='unified32M_only',whole_image_detection=False,seconds=time.monotonic()-tick,main_Bina_training_changes=False)
+        boxes,detection=self.detect(image);fallbacks=[]
+        if kind=='crop':
+            line=estimate['estimated_line_height']
+            if boxes and any(b[3]-b[1]>1.8*line for b in boxes):
+                if self._crop_line_guard is None:
+                    from crop_line_guard import CropLineGuard
+                    self._crop_line_guard=CropLineGuard()
+                boxes,guard=self._crop_line_guard(gray,boxes,line);detection['crop_anti_merge_guard']=guard
+            boxes,consolidation=merge_slivers(boxes,line);detection['same_line_box_consolidation']=consolidation
+        if not boxes:boxes=[[0.,0.,float(image.width),float(image.height)]];fallbacks.append('zero_detector_lines_whole_image_attempt_not_valid_for_coverage_gate')
+        try:layout=self.layout.run_batch([image],[dict(threshold=None,use_order=True)])[0].model_dump(mode='json')
+        except Exception as e:layout=dict(bboxes=[],error=repr(e));fallbacks.append('layout_error_RTL_column_order_all_lines_retained')
+        blocks,order=order_blocks(boxes,layout,image.width);readings=self.reader(gray,boxes);assert len(readings)==len(boxes)
+        regions=[];emitted=[]
+        for block in blocks:
+            ids=block['line_ids'];lines=[readings[i]for i in ids];record=dict(layout=block['region'],source_line_ids=ids,path='whole_image_lines',text='\n'.join(l['text']for l in lines),lines=lines)
+            if block['region']and block['region'].get('label','').lower()=='table':
+                assembled,trace=self.table_block(image,gray,block,boxes,readings);record['table_trace']=trace
+                if assembled:record.update(assembled,path='Surya_grid_unified_recognizer_cells')
+                else:
+                    fallback_order=xycut_order([boxes[i]for i in ids],image.width);record['lines']=[readings[ids[j]]for j in fallback_order];record['text']='\n'.join(l['text']for l in record['lines']);record['source_line_ids']=[ids[j]for j in fallback_order];fallbacks.append('table_grid_failed_original_lines_retained_in_RTL_columns')
+            regions.append(record);emitted+=record['lines']
+        assert sorted(i for r in regions for i in r['source_line_ids'])==list(range(len(boxes)))
+        return dict(text='\n\n'.join(r['text']for r in regions),regions=regions,lines=emitted,global_readings=readings,detected_boxes=boxes,emitted_boxes=[l['box']for l in emitted],recognition_extents=[reading_extent(l)for l in emitted],raw_line_detection=detection,raw_layout=layout,order=order,fallbacks=fallbacks,crop_routing_policy=dict(mode='c',input_type=kind,estimate=estimate,whole_crop_by_policy=False),region_type_line_losses=0,recognizer_bank='unified32M_only',whole_image_detection=True,seconds=time.monotonic()-tick,main_Bina_training_changes=False)

stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/table_client.py

@@ -0,0 +1,24 @@
+import base64,io,json,subprocess,select,time
+from pathlib import Path
+R=Path(__file__).resolve().parent
+class SuryaTableGrid:
+ def __init__(self,yield_check=None):self.process=None;self.log=None;self.yield_check=yield_check
+ def __call__(self,image):
+  if self.process is None:
+   self.log=(R/'table-worker.log').open('a');self.process=subprocess.Popen([str(R/'table-runtime/bin/python'),'-u',str(R/'table_worker.py')],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=self.log,text=True)
+  stream=io.BytesIO();image.save(stream,format='PNG');self.process.stdin.write(json.dumps(dict(image=base64.b64encode(stream.getvalue()).decode()))+'\n');self.process.stdin.flush();tick=time.monotonic()
+  while not select.select([self.process.stdout],[],[],.5)[0]:
+   if self.yield_check:self.yield_check()
+   if time.monotonic()-tick>300:self.close();raise TimeoutError('Owned table worker timeout; original line output retained')
+  line=self.process.stdout.readline()
+  if not line:raise RuntimeError('Isolated Surya table worker exited; original line text retained')
+  reply=json.loads(line)
+  if not reply['ok']:raise RuntimeError(reply['error'])
+  return reply['result']
+ def close(self):
+  if self.process is not None:
+   self.process.stdin.close()
+   try:self.process.wait(timeout=15)
+   except subprocess.TimeoutExpired:self.process.terminate();self.process.wait(timeout=10)
+   self.process=None
+  if self.log:self.log.close();self.log=None

stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/line_contract_reader.py

@@ -0,0 +1,69 @@
+"""Single-line shadow candidate: exact round-2 detector and crop_line64 contract.
+Always one recognition input for a rule-C line; no router and no extra strips.
+"""
+import hashlib,json,math
+from pathlib import Path
+import numpy as np
+from baseline.pipeline_v3 import UnifiedReader,rectangle,clipped,flip
+from baseline.input_policy_v3 import estimated_line_height
+from pipeline_crop_fix import CropPipeline
+DETECTOR=Path('__BINA_ROOT__/stallion/bina04-line-detector-v2-20261030/detector-run/checkpoints/step-000500.pt')
+DETECTOR_SHA='0320b9d1d9e7464c4eae6f35b93fdd835913e785eae552eb5f04b58091b32ae6'
+
+def round2_estimate(gray):
+    # Execute only the exact frozen estimator function, without creating any
+    # of the old recognizer/script banks. This is a boxing decision only.
+    import ast
+    p=Path('__BINA_ROOT__/stallion/indra-scale/routing_modes.py')
+    rule=json.loads((p.parent/'routing-rule.json').read_text())
+    assert hashlib.sha256(p.read_bytes()).hexdigest()==rule['source_sha256'][p.name]
+    node=next(n for n in ast.parse(p.read_text()).body if isinstance(n,ast.FunctionDef)and n.name=='estimated_line_height')
+    scope={'np':np};exec(compile(ast.Module(body=[node],type_ignores=[]),str(p),'exec'),scope)
+    return scope['estimated_line_height'](gray)
+
+class ContractReader(UnifiedReader):
+    def __init__(self,*args,cache,**kwargs):
+        super().__init__(*args,**kwargs);self.rule_c_active=False;self.detector=None;self.cache=Path(cache);self.cache.mkdir(parents=True,exist_ok=True)
+    def locate(self,gray):
+        old=round2_estimate(gray)
+        if not old['segment']:
+            H,W=gray.shape
+            return dict(box=[0.,0.,float(W),float(H)],round2_rule_C=old,round2_skipped_detection=True,one_recognition_crop=True,no_reference_or_label_used=True)
+        key=hashlib.sha256(str(gray.shape).encode()+gray.tobytes()).hexdigest();target=self.cache/(key+'.json')
+        if target.exists():return json.loads(target.read_text())
+        if self.detector is None:
+            import torch
+            from surya.detection import DetectionPredictor
+            assert hashlib.sha256(DETECTOR.read_bytes()).hexdigest()==DETECTOR_SHA
+            self.detector=DetectionPredictor.local(device='cpu',dtype=torch.float32);self.detector._disable_tqdm=True
+            self.detector.model.load_state_dict(torch.load(DETECTOR,map_location='cpu',weights_only=True)['model']);self.detector.model.eval()
+        from PIL import Image
+        H,W=gray.shape;raw=self.detector([Image.fromarray(gray).convert('RGB')],batch_size=1)[0].model_dump(mode='json');boxes=[clipped(rectangle(b['polygon']),W,H)for b in raw['bboxes']];boxes=[b for b in boxes if b[2]>b[0]and b[3]>b[1]]
+        box=[min(b[0]for b in boxes),min(b[1]for b in boxes),max(b[2]for b in boxes),max(b[3]for b in boxes)]if boxes else[0.,0.,float(W),float(H)]
+        d=dict(box=box,raw_detector_boxes=boxes,raw=raw,detector_sha256=DETECTOR_SHA,device='cpu',one_recognition_crop=True,fallback=not bool(boxes),no_reference_or_label_used=True)
+        tmp=target.with_suffix('.partial');tmp.write_text(json.dumps(d)+'\n');tmp.replace(target);return d
+    def __call__(self,gray,boxes):
+        H,W=gray.shape
+        if not(self.rule_c_active and self.source_is_crop and len(boxes)==1 and np.allclose(boxes[0],[0.,0.,float(W),float(H)])):
+            return super().__call__(gray,boxes)
+        trace=self.locate(gray);box=trace['box']
+        # Same direct original grayscale -> crop_line64 -> pad_batch -> model
+        # path as round2 RTLReader, bypassing v3 projection and the atlas.
+        _,receipts=self.reader(gray,np.asarray([box],np.float32));r=receipts[0]
+        if not r['raw_visual_text'].strip():
+            recovered=super().__call__(gray,boxes)
+            for line in recovered:line['single_line_contract_fallback']=dict(reason='Tight crop decoded empty; original single-line read retained',attempt_box=box,attempt_confidence=r['confidence'],reference_or_label_used=False)
+            return recovered
+        a,b,A,B=r['padded_source_extent'];r['source_pixels_extent']=[max(0,a),max(0,b),min(W,A),min(H,B)]
+        r['preprocessing_contract']='Exact shared geometry.crop_line64 and RTLReader, direct original uint8 pixels'
+        return[dict(box=list(map(float,box)),text=flip(r['raw_visual_text'],base_rtl=True),visual_text=r['raw_visual_text'],confidence=r['confidence'],recognizer='unified32M',geometry=r,confidence_filter_applied=False,whole_input_normalization=dict(rule_c_single_line=True,round2_detector_reboxing=trace,one_recognition_crop=True))]
+
+class ContractPipeline(CropPipeline):
+    """Apply the new reader to generic rule-C crops only.
+    The existing specialized newspaper/table and whole-page paths stay frozen.
+    """
+    def __call__(self,image,*,input_type='auto',source_name='',content_type='',**kw):
+        from baseline.input_policy_v3 import input_kind
+        gray=np.asarray(image.convert('L'));generic=content_type.lower()not in ['newspaper','table']and'table'not in source_name.lower()
+        self.reader.rule_c_active=generic and input_kind(*image.size,input_type)=='crop'and not estimated_line_height(gray)['segment']
+        return super().__call__(image,input_type=input_type,source_name=source_name,content_type=content_type,**kw)

stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/pipeline_crop_fix.py

@@ -0,0 +1,76 @@
+"""New crop path only. Full pages delegate to the exact frozen v3 implementation."""
+import copy,time
+import numpy as np
+from baseline import pipeline_v3 as base
+from baseline.input_policy_v3 import input_kind,estimated_line_height,merge_slivers
+from table_cells import CellDetector,assemble
+
+class CropPipeline(base.PagePipeline):
+    def __init__(self,*args,variant='tables-v1',**kwargs):
+        super().__init__(*args,**kwargs);self.variant=variant;self._crop_active=False;self._column_order='auto';self._cell_detector=None
+    def table_block(self,image,gray,block,boxes,readings):
+        if not self._crop_active:return super().table_block(image,gray,block,boxes,readings)
+        if self._cell_detector is None:self._cell_detector=CellDetector()
+        return assemble(image,gray,block['region'],self.reader,self.table,self._cell_detector,self._column_order)
+    def __call__(self,image,*,input_type='auto',source_name='',content_type='',table_column_order='auto',**kwargs):
+        kind=input_kind(*image.size,input_type);self._crop_active=kind=='crop';self._column_order=table_column_order
+        if kind!='crop':return super().__call__(image,input_type=input_type,**kwargs)
+        image=image.convert('RGB');gray=np.asarray(image.convert('L'));self.reader.source_is_crop=True
+        if content_type.lower()=='newspaper'and ('news' in self.variant or 'combined' in self.variant):
+            stage=next((n for n in [4,3,2,1]if f'news{n}'in self.variant),2)
+            return self.newspaper_call(image,gray,stage)
+        table_hint=content_type.lower()=='table'or'table'in source_name.lower()
+        if table_hint:
+            tick=time.monotonic();w,h=image.size;region=dict(label='Table',position=0,polygon=[[0,0],[w,0],[w,h],[0,h]])
+            block=dict(region=region,line_ids=[]);value,trace=self.table_block(image,gray,block,[],[])
+            if value is not None:
+                record=dict(layout=region,path='Surya_grid_unified_recognizer_cells',source_line_ids=[],table_trace=trace,**value)
+                return dict(text=value['text'],regions=[record],lines=value['lines'],global_readings=[],detected_boxes=[],emitted_boxes=[q['box']for q in value['lines']],recognition_extents=[base.reading_extent(q)for q in value['lines']],raw_line_detection=dict(skipped=True,reason='Explicit table crop: each cell is segmented independently'),raw_layout=dict(bboxes=[region],source='Explicit table metadata/name'),order=dict(mode='table_grid'),fallbacks=[],crop_routing_policy=dict(input_type='crop',whole_crop_by_policy=False),region_type_line_losses=0,recognizer_bank='unified32M_only',whole_image_detection=False,seconds=time.monotonic()-tick,main_Bina_training_changes=False)
+        original=self.layout
+        class PreferTableOnTie:
+            def run_batch(_,images,*args,**kw):
+                results=original.run_batch(images,*args,**kw);out=[]
+                for result in results:
+                    data=copy.deepcopy(result.model_dump(mode='json'));data['bboxes'].sort(key=lambda r:r.get('label','').lower()!='table')
+                    class Result:
+                        def __init__(self,d):self.data=d
+                        def model_dump(self,**kw):return self.data
+                    out.append(Result(data))
+                return out
+        self.layout=PreferTableOnTie()
+        try:
+            result=super().__call__(image,input_type='crop',**kwargs)
+            result['crop_table_policy']=dict(explicit_hint=table_hint,layout_table_wins_exact_overlap_tie=True,column_order=table_column_order)
+            return result
+        finally:self.layout=original
+
+    def newspaper_call(self,image,gray,stage):
+        tick=time.monotonic();estimate=estimated_line_height(gray)
+        if not estimate['segment']:
+            result=super().__call__(image,input_type='crop');result['newspaper_policy']=dict(stage=stage,already_one_full_width_line=True);return result
+        boxes,detection=self.detect(image);fallbacks=[];line=estimate['estimated_line_height']
+        if self._crop_line_guard is None:
+            from baseline.crop_line_guard import CropLineGuard
+            self._crop_line_guard=CropLineGuard()
+        if boxes and any(b[3]-b[1]>1.8*line for b in boxes):
+            boxes,guard=self._crop_line_guard(gray,boxes,line);detection['crop_anti_merge_guard']=guard
+        boxes,consolidation=merge_slivers(boxes,line);detection['same_line_box_consolidation']=consolidation
+        from newspaper_crop import apply
+        boxes,trace=apply(gray,boxes,stage,self._crop_line_guard);detection['newspaper_policy']=trace
+        if not boxes:boxes=[[0.,0.,float(image.width),float(image.height)]];fallbacks.append('zero_detector_lines_whole_image_attempt_not_valid_for_coverage_gate')
+        try:layout=self.layout.run_batch([image],[dict(threshold=None,use_order=True)])[0].model_dump(mode='json')
+        except Exception as e:layout=dict(bboxes=[],error=repr(e));fallbacks.append('layout_error_RTL_column_order_all_lines_retained')
+        blocks,order=base.order_blocks(boxes,layout,image.width)
+        if stage>=4:
+            from newspaper_crop import read_lines
+            readings=read_lines(self.reader,gray,boxes)
+        else:readings=self.reader(gray,boxes)
+        regions=[];emitted=[]
+        for block in blocks:
+            ids=block['line_ids'];lines=[readings[i]for i in ids];record=dict(layout=block['region'],source_line_ids=ids,path='whole_image_lines',text='\n'.join(l['text']for l in lines),lines=lines)
+            if block['region']and block['region'].get('label','').lower()=='table':
+                assembled,t=self.table_block(image,gray,block,boxes,readings);record['table_trace']=t
+                if assembled:record.update(assembled,path='Surya_grid_unified_recognizer_cells')
+            regions.append(record);emitted+=record['lines']
+        assert sorted(i for r in regions for i in r['source_line_ids'])==list(range(len(boxes)))
+        return dict(text='\n\n'.join(r['text']for r in regions),regions=regions,lines=emitted,global_readings=readings,detected_boxes=boxes,emitted_boxes=[l['box']for l in emitted],recognition_extents=[base.reading_extent(l)for l in emitted],raw_line_detection=detection,raw_layout=layout,order=order,fallbacks=fallbacks,crop_routing_policy=dict(mode='c',input_type='crop',estimate=estimate,whole_crop_by_policy=False),region_type_line_losses=0,recognizer_bank='unified32M_only',whole_image_detection=True,seconds=time.monotonic()-tick,main_Bina_training_changes=False)

stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/table_cells.py

@@ -0,0 +1,111 @@
+"""Crop-only table cells: native line splitting, cell padding, explicit serialization."""
+import math
+from pathlib import Path
+import numpy as np
+from baseline.pipeline_v3 import rectangle,clipped,markdown_cell,reading_extent
+from baseline.input_policy_v3 import estimated_line_height
+from indra_ocr.rtl import rtl_base
+
+class CellDetector:
+    def __init__(self):
+        from baseline.crop_line_guard import CropLineGuard
+        self.detector=CropLineGuard()
+    def lines(self,gray,line):
+        engine=self.detector.engine
+        engine.long_side=max(32,min(2048,round(max(gray.shape)*12/max(3,line)/32)*32))
+        return engine.detect(gray)[:,:4].tolist()
+
+def glyph_presence(gray):
+    import cv2
+    H,W=gray.shape;_,v=cv2.threshold(gray,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU);ink=(v==0).astype(np.uint8)
+    if ink.mean()>.5:ink=1-ink
+    _,_,stats,_=cv2.connectedComponentsWithStats(ink,8);valid=[]
+    for x,y,w,h,n in stats[1:]:
+        if n<6 or h<3 or w<2 or not .05<=w/max(1,h)<=20:continue
+        frame=(h>=.7*H and w>=.7*W and n/max(1,w*h)<.15)
+        if frame or h>=.98*H or(w>=.95*W and h<=3):continue
+        valid.append([int(x),int(y),int(w),int(h),int(n)])
+    return bool(valid),valid
+
+def tighten_ink_band(gray,box,components):
+    """Remove native detector margin while retaining >=98% observed glyph ink
+    inside the unchanged reader's .2H padding. No decoded labels enter this.
+    """
+    import cv2
+    H,W=gray.shape;lo=max(0,math.floor(box[1]));hi=min(H,math.ceil(box[3]))
+    _,v=cv2.threshold(gray,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU);ink=v==0
+    if ink.mean()>.5:ink=~ink
+    mask=np.zeros(gray.shape,bool)
+    for x,y,w,h,n in components:mask[y:y+h,x:x+w]=ink[y:y+h,x:x+w]
+    counts=mask[lo:hi].sum(axis=1);total=int(counts.sum())
+    if not total:return box,dict(changed=False,reason='No glyph ink')
+    active=np.where(counts>=max(1,.2*counts.max()))[0]
+    if not len(active):return box,dict(changed=False)
+    top=lo+int(active[0]);bottom=lo+int(active[-1])+1
+    while True:
+        h=bottom-top;a=max(lo,math.floor(top-.2*h));b=min(hi,math.ceil(bottom+.2*h));kept=int(mask[a:b].sum())
+        if kept/total>=.98 or(top<=lo and bottom>=hi):break
+        top=max(lo,top-1);bottom=min(hi,bottom+1)
+    return [box[0],float(top),box[2],float(bottom)],dict(changed=True,original=box,core=[top,bottom],glyph_ink_fraction_in_padded_reader=kept/total)
+
+def remap(line,x,y,W,H):
+    line=dict(line);line['box']=[line['box'][0]+x,line['box'][1]+y,line['box'][2]+x,line['box'][3]+y]
+    g=dict(line['geometry'])
+    for key in ['ink_box','padded_source_extent','source_pixels_extent']:
+        if key in g:a,b,A,B=g[key];g[key]=[a+x,b+y,A+x,B+y]
+    g['original_WH']=[W,H]
+    if 'frame_centers'in g:g['frame_centers']=[a+x for a in g['frame_centers']]
+    line['geometry']=g;return line
+
+def recognize_cell(image,gray,cb,reader,detector,padding=3):
+    W,H=image.size;x,y,X,Y=clipped(cb,W,H)
+    a=max(0,math.floor(x)-padding);b=max(0,math.floor(y)-padding);A=min(W,math.ceil(X)+padding);B=min(H,math.ceil(Y)+padding)
+    if A<=a or B<=b:return '',[],dict(empty=True,reason='invalid_cell')
+    local=gray[b:B,a:A].copy();present,components=glyph_presence(local)
+    if not present:return '',[],dict(empty=True,padded_bbox=[a,b,A,B],glyph_components=[])
+    estimate=estimated_line_height(local);line=max(3.,estimate['original_glyph_based_line_height']);tall=local.shape[0]>1.8*line
+    native=detector.lines(local,line)if tall else[]
+    # Padding may include a neighbour's descender; its centre lies outside this cell.
+    native=[q for q in native if y-b<=(q[1]+q[3])/2<=Y-b and q[3]-q[1]>=2]
+    groups=[]
+    for q in sorted(native,key=lambda q:(q[1]+q[3])/2):
+        cy=(q[1]+q[3])/2
+        if not groups or cy-groups[-1][-1][0]>.6*line:groups.append([])
+        groups[-1].append((cy,q))
+    bands=[[0.,min(q[1]for _,q in g),float(local.shape[1]),max(q[3]for _,q in g)]for g in groups]
+    projected=estimate['ink_bands']
+    fallback=None
+    if tall and len(bands)<=1 and len(projected)>=2:
+        bands=[[0.,float(t),float(local.shape[1]),float(T)]for t,T in projected if T-t>=2];fallback='projection_after_native_merged_or_empty'
+    if not bands:bands=[[0.,0.,float(local.shape[1]),float(local.shape[0])]]
+    tight=[];normalization=[]
+    for band in bands:
+        q,t=tighten_ink_band(local,band,components);tight.append(q);normalization.append(t)
+    lines=reader(local,tight);lines=[remap(q,a,b,W,H)for q in lines]
+    return ' '.join(q['text'].strip()for q in lines if q['text'].strip()),lines,dict(empty=False,padded_bbox=[a,b,A,B],glyph_components=components,line_estimate=estimate,native_called=tall,native_boxes=native,line_boxes=tight,ink_normalization=normalization,fallback=fallback)
+
+def assemble(image,gray,region,reader,grid,detector,column_order='auto'):
+    W,H=image.size;b=clipped(rectangle(region['polygon']),W,H);x,y,X,Y=[math.floor(b[0]),math.floor(b[1]),math.ceil(b[2]),math.ceil(b[3])]
+    result=grid(image.crop((x,y,X,Y)));cells=result.get('cells',[])
+    if result.get('error')or not cells:return None,dict(error='Surya_table_grid_empty',grid=result,grid_invoked=True)
+    values={};row_y={};col_x={};debug=[];emitted=[];presence={}
+    for cell in cells:
+        cb=cell.get('bbox')or rectangle(cell['polygon']);cb=[cb[0]+x,cb[1]+y,cb[2]+x,cb[3]+y]
+        row=cell['row_id'];col=cell.get('col_id')
+        if col is None:col=cell.get('within_row_id',0)
+        text,lines,trace=recognize_cell(image,gray,cb,reader,detector)
+        key=(row,col);values[key]=text;presence[key]=not trace['empty'];row_y.setdefault(row,[]).append((cb[1]+cb[3])/2);col_x.setdefault(col,[]).append((cb[0]+cb[2])/2)
+        emitted+=lines;debug.append(dict(row=row,col=col,bbox=cb,text=text,lines=lines,trace=trace,colspan=cell.get('colspan',1),rowspan=cell.get('rowspan',1)))
+    rows=sorted(row_y,key=lambda r:float(np.median(row_y[r])))
+    cols=sorted(col_x,key=lambda c:float(np.median(col_x[c])))
+    empty_rows=[r for r in rows if not any(values.get((r,c),'').strip()or presence.get((r,c),False)for c in cols)]
+    empty_cols=[c for c in cols if not any(values.get((r,c),'').strip()or presence.get((r,c),False)for r in rows)]
+    rows=[r for r in rows if r not in empty_rows];cols=[c for c in cols if c not in empty_cols]
+    rtl=rtl_base(' '.join(values.values()));chosen=('rtl'if rtl else'ltr')if column_order=='auto'else column_order
+    assert chosen in ['rtl','ltr']
+    if chosen=='rtl':cols.reverse()
+    rendered=[]
+    for i,r in enumerate(rows):
+        rendered.append('| '+' | '.join(markdown_cell(values.get((r,c),''))for c in cols)+' |')
+        if i==0:rendered.append('| '+' | '.join('---'for _ in cols)+' |')
+    return dict(text='\n'.join(rendered),lines=emitted),dict(grid=result,grid_invoked=True,cell_debug=debug,kept_rows=rows,kept_columns=cols,dropped_empty_rows=empty_rows,dropped_empty_columns=empty_cols,content_RTL=bool(rtl),physical_column_serialization=chosen,cell_padding=3,grid_text_never_used=True)

stallion/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_builder.py

@@ -0,0 +1,102 @@
+"""Image-only pen/print separation and physical row construction; no gold inputs."""
+import math
+import numpy as np
+import cv2
+
+DEFAULT=dict(core_overlap=.5,baseline_distance=.4,max_gap=3.,nominal_pad=.06,color_angle_degrees=35,color_depth=.055,color_chroma=.045,color_relative_chroma=.16,min_pen_components=15,min_pen_pixels=80)
+def rect_intersection(a,b):return[max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])]
+def rect_area(a):return max(0,a[2]-a[0])*max(0,a[3]-a[1])
+def estimate_height(proposals,H):
+    h=[b[3]-b[1]for b in proposals if b[3]>b[1]and b[2]-b[0]>=2*(b[3]-b[1])and 4<=b[3]-b[1]<=min(128,.12*H)]
+    return max(6.,min(100.,float(np.percentile(h,60))if h else H/40))
+def remove_rules(mask,line):
+    H,W=mask.shape;u=mask.astype(np.uint8);horizontal=cv2.morphologyEx(u,cv2.MORPH_OPEN,np.ones((1,max(30,round(.12*W))),np.uint8));vertical=cv2.morphologyEx(u,cv2.MORPH_OPEN,np.ones((max(30,round(.1*H)),1),np.uint8));rules=cv2.dilate(horizontal|vertical,np.ones((3,3),np.uint8));return(u>0)&~(rules>0),rules>0
+def components(mask,min_area=2):
+    n,labels,stats,centers=cv2.connectedComponentsWithStats(mask.astype(np.uint8),8);return labels,[dict(label=i,x=int(x),y=int(y),w=int(w),h=int(h),area=int(a),cx=float(centers[i][0]),cy=float(centers[i][1]))for i,(x,y,w,h,a)in enumerate(stats)if i and a>=min_area]
+def columns(mask,line,proposals=()):
+    H,W=mask.shape;profile=mask.sum(axis=0);low=profile<=max(1,.002*H);edges=np.diff(np.r_[False,low,False].astype(int));gaps=[]
+    for a,b in zip(np.where(edges==1)[0],np.where(edges==-1)[0]):
+        if b-a<max(round(.4*line),.005*W)or a<.15*W or b>.85*W:continue
+        cut=(a+b)//2
+        crossing=[p for p in proposals if p[0]<a-line and p[2]>b+line and p[2]-p[0]>.55*W and p[3]-p[1]<2*line]
+        if len(crossing)>=3:continue
+        left=int(mask[:,:a].sum());right=int(mask[:,b:].sum());total=max(1,int(mask.sum()))
+        if min(left,right)>=.15*total and min(a,W-b)>=5*line:gaps.append((b-a,(a+b)//2))
+    if not gaps:return[(0,W)]
+    cut=max(gaps)[1];return[(cut,W),(0,cut)]
+def proposal_columns(boxes,W,H,gray):
+    vertical=cv2.morphologyEx((gray<100).astype(np.uint8),cv2.MORPH_OPEN,np.ones((max(30,round(.18*H)),1),np.uint8));occupied=vertical.sum(0)>.18*H;edges=np.diff(np.r_[False,occupied,False].astype(int));rulings=[]
+    for a,b in zip(np.where(edges==1)[0],np.where(edges==-1)[0]):
+        if not rulings or a-rulings[-1][1]>.01*W:rulings.append([a,b])
+        else:rulings[-1][1]=b
+    # Count physical rulings once, including antialiased/double-stroke borders.
+    if len(rulings)>=5:return[(0,W)]
+    anchors=[b for b in boxes if .2*W<=b[2]-b[0]<=.7*W and b[3]-b[1]<=.15*H];centers=sorted(((b[0]+b[2])/2,b)for b in anchors)
+    if len(centers)<6:return None
+    for gap,k in sorted(((centers[k+1][0]-centers[k][0],k)for k in range(2,len(centers)-3)),reverse=True):
+        if gap<.15*W:break
+        left=[b for _,b in centers[:k+1]];right=[b for _,b in centers[k+1:]];a=float(np.median([b[2]for b in left]));z=float(np.median([b[0]for b in right]))
+        if z-a>=.02*W:
+            cut=round((a+z)/2);return[(cut,W),(0,cut)]
+    return None
+
+def layer_masks(rgb,proposals,params):
+    H,W=rgb.shape[:2];line=estimate_height(proposals,H);k=max(15,min(71,round(1.2*line)));k+=1-k%2
+    background=cv2.morphologyEx(rgb,cv2.MORPH_CLOSE,cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(k,k))).astype(np.float32)
+    optical=np.maximum(0,np.log((background+2)/(rgb.astype(np.float32)+2)));depth=optical.mean(2);chroma=optical-depth[:,:,None];strength=np.linalg.norm(chroma,axis=2);relative=strength/(depth+.05)
+    direction=optical/(optical.sum(2)[:,:,None]+1e-5);valid=depth>.30;points=direction[valid].astype(np.float32)
+    if len(points)<100:return np.zeros((H,W),bool),np.zeros((H,W),bool),cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY),dict(reliable=False)
+    cv2.setRNGSeed(1729);sample=points[::max(1,len(points)//30000)].copy();K=4;_,_,centers=cv2.kmeans(sample,K,None,(cv2.TERM_CRITERIA_EPS+cv2.TERM_CRITERIA_MAX_ITER,60,.0002),3,cv2.KMEANS_PP_CENTERS)
+    distances=np.stack([np.linalg.norm(direction-c,axis=2)for c in centers]);assigned=distances.argmin(0);scores=[];clean=[];traces=[]
+    for i in range(K):
+        # Include neighbouring optical directions belonging to the same pen colour.
+        compatible=np.linalg.norm(centers-centers[i],axis=1)<.035
+        mask=(depth>.22)&compatible[assigned];mask,rules=remove_rules(mask,line);lab,cc=components(mask,max(3,round(.004*line*line)))
+        good=[c for c in cc if c['h']<3*line and c['w']<.85*W and c['area']<.04*W*H and not(c['w']<.3*line and c['h']>.9*line and(c['x']<.04*W or c['x']+c['w']>.96*W))]
+        mask=np.isin(lab,[c['label']for c in good]);long=[c for c in good if c['w']>=1.5*c['h']and c['h']>=.15*line and c['w']>=.5*line]
+        bins_y=len(set(int(c['cy']/max(1,2*line))for c in long));mass=sum(c['area']for c in long);score=mass*min(1,bins_y/8)*max(.008,float(np.linalg.norm(centers[i]-np.ones(3)/3)))
+        scores.append(score);clean.append(mask);traces.append(dict(class_id=i,center=centers[i].tolist(),pixels=int(mask.sum()),long_components=len(long),vertical_bins=bins_y,score=score))
+    best=int(np.argmax(scores));pen=clean[best];lab,cc=components(pen,2);span=max((c['y']+c['h']for c in cc),default=0)-min((c['y']for c in cc),default=0)
+    other=[scores[j]for j in range(K)if np.linalg.norm(centers[j]-centers[best])>=.04];secondary=max(other,default=0)
+    reliable=traces[best]['long_components']>=8 and traces[best]['vertical_bins']>=5 and secondary>max(.5,.015*scores[best])
+    neutral=centers[int(np.argmin(np.linalg.norm(centers-np.ones(3)/3,axis=1)))];radius=max(.025,.65*float(np.linalg.norm(centers[best]-neutral)));compatible=np.linalg.norm(centers-centers[best],axis=1)<radius
+    _,all_rules=remove_rules(depth>.16,line);pen=(depth>.07)&compatible[assigned]&~all_rules;labels,allcc=components(pen,2)
+    ids=[c['label']for c in allcc if c['h']<3*line and c['w']<.85*W and not(c['w']<.3*line and c['h']>.9*line and(c['x']<.04*W or c['x']+c['w']>.96*W))];pen=np.isin(labels,ids)
+    printed=(depth>.22)&~(cv2.dilate(pen.astype(np.uint8),np.ones((3,3),np.uint8))>0);printed,_=remove_rules(printed,line)
+    dilated=cv2.dilate(pen.astype(np.uint8),np.ones((3,3),np.uint8))>0
+    gray=cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY);bg=cv2.cvtColor(background.clip(0,255).astype(np.uint8),cv2.COLOR_RGB2GRAY);print_gray=gray.copy();print_gray[dilated]=bg[dilated]
+    trace_core=(depth>.22)&compatible[assigned]&~all_rules
+    return pen,printed,print_gray,dict(_core_mask=trace_core,reliable=bool(reliable),estimated_line_height=line,selected_class=best,classes=traces,pen_components=len(cc),pen_pixels=int(pen.sum()),color_span_y=span,background_kernel=k,no_gold_or_template_used=True)
+
+def build_rows(mask,proposals,params,regions=None,core=None):
+    from scipy.ndimage import gaussian_filter1d
+    from scipy.signal import find_peaks
+    H,W=mask.shape;line=estimate_height(proposals,H);cols=regions or columns(mask,line,proposals);rows=[];row_trace=[]
+    for column,(left,right)in enumerate(cols):
+        m=mask[:,left:right];cm=(core if core is not None else mask)[:,left:right];profile=cm.sum(1).astype(float);smooth=gaussian_filter1d(profile,max(1,.08*line));corr=np.correlate(smooth-smooth.mean(),smooth-smooth.mean(),mode='full')[H-1:];lo=max(5,round(.5*line));hi=min(H//3,round(3.5*line));period=line*1.7
+        if hi>lo:
+            peaks,_=find_peaks(corr[lo:hi]);valid=[lo+int(k)for k in peaks if corr[lo+k]>.12*corr[0]]
+            if valid:period=min(valid)
+        period=line;peaks,_=find_peaks(smooth,distance=max(3,round(.65*line)),prominence=max(.8,.025*smooth.max()))
+        peaks=[int(y)for y in peaks if smooth[y]>=max(1,.025*smooth.max())]
+        for j,peak in enumerate(peaks):
+            low=0 if not j else peaks[j-1]+int(np.argmin(smooth[peaks[j-1]:peak+1]));high=H if j+1==len(peaks)else peak+int(np.argmin(smooth[peak:peaks[j+1]+1]))
+            # Limit isolated/noisy bands, retaining full connected strokes and dots.
+            low=max(low,round(peak-.8*period));high=min(high,round(peak+.8*period));xs=np.where(m[low:high].any(0))[0]
+            segments=[]
+            for x in xs:
+                if not segments or x-segments[-1][-1]>max(params['max_gap'],8.)*line:segments.append([])
+                segments[-1].append(int(x))
+            for segment in segments:
+                x=left+segment[0];X=left+segment[-1]+1;ys,xx=np.where(mask[low:high,x:X])
+                if len(ys)<max(5,.01*line*line):continue
+                coreys=np.where(cm[low:high,x-left:X-left])[0];tight=coreys if len(coreys)>=max(4,.05*len(ys))else ys
+                y=low+int(tight.min());Y=low+int(tight.max())+1;pad=max(1.,.04*line);box=[max(0.,x-pad),max(0.,y-pad),min(float(W),X+pad),min(float(H),Y+pad)];support=[i for i,p in enumerate(proposals)if rect_area(rect_intersection(box,p))>0]
+                rows.append(dict(box=box,baseline=float(np.quantile(ys,.8)+low),column=column));row_trace.append(dict(source_proposals=support,ink_pixels=len(ys),read_band=[low,high],peak=peak,period=period))
+    return rows,dict(estimated_line_height=line,columns=cols,rows=row_trace)
+
+def build(rgb,proposals,params=None,primary=None):
+    params=dict(DEFAULT,**(params or {}));pen,printed,print_gray,trace=layer_masks(rgb,proposals,params);core=trace.pop('_core_mask',None)
+    if not trace['reliable']:return None,dict(layer_mask=trace,reason='No confident separated handwriting and printed layers')
+    cols=proposal_columns(primary or proposals,rgb.shape[1],rgb.shape[0],cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY));hand,ht=build_rows(pen,proposals,params,core=core,regions=cols);printed_rows,pt=build_rows(printed,proposals,params,regions=ht['columns'])
+    return dict(handwriting=hand,printed=printed_rows,printed_gray=print_gray,pen_mask=pen,printed_mask=printed),dict(parameters=params,layer_mask=trace,handwriting_rows=ht,printed_rows=pt,source_proposal_count=len(proposals),handwriting_readings=len(hand),printed_readings=len(printed_rows),no_gold_or_template_used=True)

stallion/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_pipeline.py

@@ -0,0 +1,42 @@
+import time
+import numpy as np
+from line_contract_reader import ContractPipeline
+from baseline.pipeline_v3 import rectangle,clipped,order_blocks,reading_extent
+from baseline.input_policy_v3 import input_kind
+from row_builder import build
+
+class RowPipeline(ContractPipeline):
+    def __init__(self,*args,row_parameters=None,**kw):
+        super().__init__(*args,**kw);self.row_parameters=row_parameters or{}
+    def __call__(self,image,*,input_type='auto',source_name='',content_type='',**kw):
+        if input_kind(*image.size,input_type)!='page':return super().__call__(image,input_type=input_type,source_name=source_name,content_type=content_type,**kw)
+        tick=time.monotonic();image=image.convert('RGB');rgb=np.asarray(image);gray=np.asarray(image.convert('L'));boxes,detection=self.detect(image);proposals=[]
+        if detection.get('full_image_SDK'):
+            proposals +=[clipped(rectangle(b['polygon']),image.width,image.height)for b in detection['full_image_SDK']['bboxes']]
+            proposals +=[b for t in detection.get('native_resolution_tiles',[])for b in t['boxes']]
+        if not proposals:proposals=boxes
+        layers,trace=build(rgb,proposals,self.row_parameters,primary=boxes)
+        if layers is None or not layers['handwriting']:
+            result=super().__call__(image,input_type='page',source_name=source_name,content_type=content_type,**kw);result['row_consolidation']=dict(applied=False,**trace);return result
+        self.reader.rule_c_active=False;self.reader.source_is_crop=False
+        hand=self.reader(gray,[r['box']for r in layers['handwriting']])
+        for line,meta in zip(hand,layers['handwriting']):line.update(layer='handwriting',baseline=meta['baseline'],column=meta['column'],source_pixels_modified=False)
+        # Retain the established printed reads outside consolidated pen rows.
+        # This avoids turning weak paper texture or printed rules into new reads.
+        original=super().__call__(image,input_type='page',source_name=source_name,content_type=content_type,**kw);printed=[]
+        for line in original['lines']:
+            b=line['box'];overlap=False
+            for row in layers['handwriting']:
+                r=row['box'];vy=min(b[3],r[3])-max(b[1],r[1]);vx=min(b[2],r[2])-max(b[0],r[0])
+                if vy>.35*min(b[3]-b[1],r[3]-r[1])and vx>0:overlap=True;break
+            if overlap:continue
+            x=dict(line);x.update(layer='printed',baseline=float(b[3]),column=next((i for i,(a,z)in enumerate(trace['handwriting_rows']['columns'])if a<=(b[0]+b[2])/2<z),0),source_pixels_modified=False);printed.append(x)
+        all_attempts=hand+printed;hand=[l for l in hand if l['text'].strip()];printed=[l for l in printed if l['text'].strip()]
+        lines=hand+printed;newboxes=[l['box']for l in lines];layout=self.layout.run_batch([image],[dict(threshold=None,use_order=True)])[0].model_dump(mode='json');blocks=[];ordering={'method':'image_column_then_baseline'}
+        for col in sorted(set(l['column']for l in lines)):
+            blocks.append(dict(line_ids=[i for i,l in enumerate(lines)if l['column']==col],region={'column':col,'polygon':[[0,0],[image.width,0],[image.width,image.height],[0,image.height]]}))
+        regions=[];emitted=[]
+        for block in blocks:
+            ids=block['line_ids'];ids.sort(key=lambda i:(lines[i]['baseline'],-lines[i]['box'][0]));rr=[lines[i]for i in ids];regions.append(dict(layout=block['region'],source_line_ids=ids,path='consolidated_rows_printed_layer',lines=rr,text='\n'.join(l['text']for l in rr)));emitted+=rr
+        assert sorted(i for r in regions for i in r['source_line_ids'])==list(range(len(lines)))
+        return dict(text='\n\n'.join(r['text']for r in regions),regions=regions,lines=emitted,global_readings=lines,detected_boxes=newboxes,emitted_boxes=[l['box']for l in emitted],recognition_extents=[reading_extent(l)for l in emitted],handwriting_extents=[reading_extent(l)for l in hand],raw_line_detection=detection,raw_layout=layout,order=ordering,fallbacks=[],region_type_line_losses=0,recognizer_bank='unified32M_only',whole_image_detection=True,seconds=time.monotonic()-tick,row_consolidation=dict(applied=True,read_attempts=len(all_attempts),blank_attempts_not_emitted=sum(not l['text'].strip()for l in all_attempts),**trace),main_Bina_training_changes=False)

stallion/bina04-header-marks-95k-20261103/CALIBRATION.json

@@ -0,0 +1 @@
+{"selected": {"parameters": {"max_chars": 1, "radius": 2.0, "header_confidence": 1.1}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.040464916056823, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.3517003874300473, "ordered": 8.949634093844168}, "full_CER": 11.74942771614721, "reading_gate_pass": true}, "trials": [{"parameters": {"max_chars": 0, "radius": 2.0, "header_confidence": 0.7}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.40206629358588, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.7692638829100302, "ordered": 9.328454584588894}, "full_CER": 12.11040676175383, "reading_gate_pass": true}, {"parameters": {"max_chars": 0, "radius": 2.0, "header_confidence": 0.85}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.303056392595781, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.6616444253120963, "ordered": 9.229444683598794}, "full_CER": 12.013558725127663, "reading_gate_pass": true}, {"parameters": {"max_chars": 0, "radius": 2.0, "header_confidence": 0.95}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.277227722772277, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.6358157554885924, "ordered": 9.20361601377529}, "full_CER": 11.987145624229617, "reading_gate_pass": true}, {"parameters": {"max_chars": 0, "radius": 2.0, "header_confidence": 1.1}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.04476969436074, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.3560051657339647, "ordered": 8.953938872148084}, "full_CER": 11.753829899630217, "reading_gate_pass": true}, {"parameters": {"max_chars": 0, "radius": 3.0, "header_confidence": 0.7}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.40206629358588, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.7692638829100302, "ordered": 9.328454584588894}, "full_CER": 12.11040676175383, "reading_gate_pass": true}, {"parameters": {"max_chars": 0, "radius": 3.0, "header_confidence": 0.85}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.303056392595781, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.6616444253120963, "ordered": 9.229444683598794}, "full_CER": 12.013558725127663, "reading_gate_pass": true}, {"parameters": {"max_chars": 0, "radius": 3.0, "header_confidence": 0.95}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.277227722772277, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.6358157554885924, "ordered": 9.20361601377529}, "full_CER": 11.987145624229617, "reading_gate_pass": true}, {"parameters": {"max_chars": 0, "radius": 3.0, "header_confidence": 1.1}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.04476969436074, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.3560051657339647, "ordered": 8.953938872148084}, "full_CER": 11.753829899630217, "reading_gate_pass": true}, {"parameters": {"max_chars": 0, "radius": 4.0, "header_confidence": 0.7}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.40206629358588, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.7692638829100302, "ordered": 9.328454584588894}, "full_CER": 12.11040676175383, "reading_gate_pass": true}, {"parameters": {"max_chars": 0, "radius": 4.0, "header_confidence": 0.85}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.303056392595781, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.6616444253120963, "ordered": 9.229444683598794}, "full_CER": 12.013558725127663, "reading_gate_pass": true}, {"parameters": {"max_chars": 0, "radius": 4.0, "header_confidence": 0.95}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.277227722772277, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.6358157554885924, "ordered": 9.20361601377529}, "full_CER": 11.987145624229617, "reading_gate_pass": true}, {"parameters": {"max_chars": 0, "radius": 4.0, "header_confidence": 1.1}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.04476969436074, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.3560051657339647, "ordered": 8.953938872148084}, "full_CER": 11.753829899630217, "reading_gate_pass": true}, {"parameters": {"max_chars": 1, "radius": 2.0, "header_confidence": 0.7}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.397761515281964, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.7649591046061128, "ordered": 9.324149806284977}, "full_CER": 12.106004578270822, "reading_gate_pass": true}, {"parameters": {"max_chars": 1, "radius": 2.0, "header_confidence": 0.85}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.298751614291865, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.6573396470081792, "ordered": 9.225139905294878}, "full_CER": 12.009156541644655, "reading_gate_pass": true}, {"parameters": {"max_chars": 1, "radius": 2.0, "header_confidence": 0.95}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.27292294446836, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.631510977184675, "ordered": 9.199311235471374}, "full_CER": 11.98274344074661, "reading_gate_pass": true}, {"parameters": {"max_chars": 1, "radius": 2.0, "header_confidence": 1.1}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.040464916056823, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.3517003874300473, "ordered": 8.949634093844168}, "full_CER": 11.74942771614721, "reading_gate_pass": true}, {"parameters": {"max_chars": 1, "radius": 3.0, "header_confidence": 0.7}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.397761515281964, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.7649591046061128, "ordered": 9.324149806284977}, "full_CER": 12.106004578270822, "reading_gate_pass": true}, {"parameters": {"max_chars": 1, "radius": 3.0, "header_confidence": 0.85}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.298751614291865, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.6573396470081792, "ordered": 9.225139905294878}, "full_CER": 12.009156541644655, "reading_gate_pass": true}, {"parameters": {"max_chars": 1, "radius": 3.0, "header_confidence": 0.95}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.27292294446836, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.631510977184675, "ordered": 9.199311235471374}, "full_CER": 11.98274344074661, "reading_gate_pass": true}, {"parameters": {"max_chars": 1, "radius": 3.0, "header_confidence": 1.1}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.040464916056823, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.3517003874300473, "ordered": 8.949634093844168}, "full_CER": 11.74942771614721, "reading_gate_pass": true}, {"parameters": {"max_chars": 1, "radius": 4.0, "header_confidence": 0.7}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.397761515281964, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.7649591046061128, "ordered": 9.324149806284977}, "full_CER": 12.106004578270822, "reading_gate_pass": true}, {"parameters": {"max_chars": 1, "radius": 4.0, "header_confidence": 0.85}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.298751614291865, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.6573396470081792, "ordered": 9.225139905294878}, "full_CER": 12.009156541644655, "reading_gate_pass": true}, {"parameters": {"max_chars": 1, "radius": 4.0, "header_confidence": 0.95}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.27292294446836, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.631510977184675, "ordered": 9.199311235471374}, "full_CER": 11.98274344074661, "reading_gate_pass": true}, {"parameters": {"max_chars": 1, "radius": 4.0, "header_confidence": 1.1}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.040464916056823, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.3517003874300473, "ordered": 8.949634093844168}, "full_CER": 11.74942771614721, "reading_gate_pass": true}, {"parameters": {"max_chars": 2, "radius": 2.0, "header_confidence": 0.7}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.397761515281964, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.7649591046061128, "ordered": 9.324149806284977}, "full_CER": 12.106004578270822, "reading_gate_pass": true}, {"parameters": {"max_chars": 2, "radius": 2.0, "header_confidence": 0.85}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.298751614291865, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.6573396470081792, "ordered": 9.225139905294878}, "full_CER": 12.009156541644655, "reading_gate_pass": true}, {"parameters": {"max_chars": 2, "radius": 2.0, "header_confidence": 0.95}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.27292294446836, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.631510977184675, "ordered": 9.199311235471374}, "full_CER": 11.98274344074661, "reading_gate_pass": true}, {"parameters": {"max_chars": 2, "radius": 2.0, "header_confidence": 1.1}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.040464916056823, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.3517003874300473, "ordered": 8.949634093844168}, "full_CER": 11.74942771614721, "reading_gate_pass": true}, {"parameters": {"max_chars": 2, "radius": 3.0, "header_confidence": 0.7}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.397761515281964, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.7649591046061128, "ordered": 9.324149806284977}, "full_CER": 12.106004578270822, "reading_gate_pass": true}, {"parameters": {"max_chars": 2, "radius": 3.0, "header_confidence": 0.85}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.298751614291865, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.6573396470081792, "ordered": 9.225139905294878}, "full_CER": 12.009156541644655, "reading_gate_pass": true}, {"parameters": {"max_chars": 2, "radius": 3.0, "header_confidence": 0.95}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.27292294446836, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.631510977184675, "ordered": 9.199311235471374}, "full_CER": 11.98274344074661, "reading_gate_pass": true}, {"parameters": {"max_chars": 2, "radius": 3.0, "header_confidence": 1.1}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.040464916056823, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.3517003874300473, "ordered": 8.949634093844168}, "full_CER": 11.74942771614721, "reading_gate_pass": true}, {"parameters": {"max_chars": 2, "radius": 4.0, "header_confidence": 0.7}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.397761515281964, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.7649591046061128, "ordered": 9.324149806284977}, "full_CER": 12.106004578270822, "reading_gate_pass": true}, {"parameters": {"max_chars": 2, "radius": 4.0, "header_confidence": 0.85}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.298751614291865, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.6573396470081792, "ordered": 9.225139905294878}, "full_CER": 12.009156541644655, "reading_gate_pass": true}, {"parameters": {"max_chars": 2, "radius": 4.0, "header_confidence": 0.95}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.27292294446836, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.631510977184675, "ordered": 9.199311235471374}, "full_CER": 11.98274344074661, "reading_gate_pass": true}, {"parameters": {"max_chars": 2, "radius": 4.0, "header_confidence": 1.1}, "reading": {"pages": 25, "reference_chars": 23230, "total_CER": 12.040464916056823, "match_short": 0.4476969436074042, "match_long": 2.0705983641842445, "missing_short": 5.368058544984933, "missing_long": 0.594059405940594, "extra": 1.3517003874300473, "ordered": 8.949634093844168}, "full_CER": 11.74942771614721, "reading_gate_pass": true}], "check25_used_for_thresholds": false, "no_template_classifier": true}

stallion/bina04-header-marks-95k-20261103/candidate.py

@@ -0,0 +1,76 @@
+"""Image-only header enhancement plus isolated-mark removal; no templates."""
+import sys,copy,json
+from pathlib import Path
+import cv2,numpy as np
+from PIL import Image
+R=Path(__file__).resolve().parent;P=Path('__BINA_ROOT__/stallion/indra-scale');sys.path[:0]=[str(R),str(P),'__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031','__BINA_ROOT__/stallion/bina04-indra-pilot-20261027/upstream']
+from serializer import apply as add_printed
+
+def header_boxes(image):
+ rgb=np.asarray(image.convert('RGB'));H,W=rgb.shape[:2];h=max(1,round(.2*H));top=rgb[:h];gray=cv2.cvtColor(top,cv2.COLOR_RGB2GRAY);hsv=cv2.cvtColor(top,cv2.COLOR_RGB2HSV);tint=((hsv[:,:,1]>12)&(hsv[:,:,2]>135)).astype('uint8')*255;tint=cv2.morphologyEx(tint,cv2.MORPH_CLOSE,np.ones((9,25),np.uint8));tint=cv2.morphologyEx(tint,cv2.MORPH_OPEN,np.ones((5,9),np.uint8))
+ ink=cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY_INV,31,5);hor=cv2.morphologyEx(ink,cv2.MORPH_OPEN,np.ones((1,max(30,W//8)),np.uint8));ver=cv2.morphologyEx(ink,cv2.MORPH_OPEN,np.ones((max(12,h//5),1),np.uint8));rules=cv2.dilate(hor|ver,np.ones((5,5),np.uint8));out=[]
+ for kind,mask in [('tinted',tint),('bordered',rules)]:
+  nc,_,stats,_=cv2.connectedComponentsWithStats(mask)
+  for x,y,w,hh,a in stats[1:]:
+   if w<.18*W or hh<max(8,.012*H)or a<.0008*H*W:continue
+   b=[max(0,int(x-3)),max(0,int(y-3)),min(W,int(x+w+3)),min(h,int(y+hh+3))]
+   if not any(b[0]>=q['box'][0]and b[1]>=q['box'][1]and b[2]<=q['box'][2]and b[3]<=q['box'][3]for q in out):out.append(dict(box=b,source=kind))
+ return out
+
+def runs(mask):
+ e=np.diff(np.r_[False,mask,False].astype(int));return list(zip(np.flatnonzero(e==1),np.flatnonzero(e==-1)))
+def split_labels(gray):
+ ink=cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY_INV,31,3);H,W=gray.shape;hr=cv2.morphologyEx(ink,cv2.MORPH_OPEN,np.ones((1,max(60,W//8)),np.uint8));vr=cv2.morphologyEx(ink,cv2.MORPH_OPEN,np.ones((max(40,H//3),1),np.uint8));ink=cv2.subtract(ink,hr|vr);nc,_,stats,_=cv2.connectedComponentsWithStats(ink)
+ heights=[a[3]for a in stats[1:]if 4<=a[3]<=H*.3 and a[4]>=5];charh=float(np.median(heights))if heights else 18.;profile=ink.sum(1)/255;bands=runs(profile>=max(2,.001*W));merged=[]
+ for a,b in bands:
+  if merged and a-merged[-1][1]<=max(2,.25*charh):merged[-1][1]=b
+  else:merged.append([a,b])
+ out=[]
+ for y,Y in merged:
+  if Y-y<3:continue
+  xbands=runs((ink[y:Y]>0).any(0));groups=[]
+  for x,X in xbands:
+   if groups and x-groups[-1][1]<=2*charh:groups[-1][1]=X
+   else:groups.append([x,X])
+  for x,X in groups:
+   if X-x>=2 and int((ink[y:Y,x:X]>0).sum())>=5:out.append([max(0,int(x-3)),max(0,int(y-3)),min(W,int(X+3)),min(H,int(Y+3))])
+ return out,dict(character_height_upscaled=charh,rows=len(merged),labels=len(out),split_gap_character_heights=2.)
+class Collector:
+ def __init__(self,device='cpu'):
+  from rtl_reader import RTLReader
+  self.reader=RTLReader('__BINA_ROOT__/stallion/bina04-print-textgate-20261102/run/selected-printed-recognizer.pt','__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json',device=device)
+ def __call__(self,image,base):
+  from text_contract import decode_prediction
+  new=[];traces=[]
+  for region in header_boxes(image):
+   x,y,X,Y=region['box'];rgb=np.asarray(image.crop((x,y,X,Y)).convert('RGB'));gray=rgb.min(2).astype('float32');bg=cv2.GaussianBlur(gray,(0,0),max(9,(Y-y)/12));norm=np.clip(gray/(bg+1e-4)*245,0,255).astype('uint8');norm=cv2.createCLAHE(clipLimit=2.,tileGridSize=(8,8)).apply(norm);up=cv2.resize(norm,None,fx=3,fy=3,interpolation=cv2.INTER_CUBIC);boxes,trace=split_labels(up);traces.append(dict(**region,segmentation=trace))
+   for start in range(0,len(boxes),24):
+    bb=boxes[start:start+24];_,receipts=self.reader(up,np.asarray(bb,np.float32))
+    for b,g in zip(bb,receipts):
+     text=decode_prediction(g['raw_visual_text']);box=[b[0]/3+x,b[1]/3+y,b[2]/3+x,b[3]/3+y];extent=g['padded_source_extent'];geom=dict(raw_visual_text=g['raw_visual_text'],raw_token_ids=g.get('raw_token_ids',[]),confidence=g['confidence'],frame_centers=[v/3+x for v in g.get('frame_centers',[])],source_pixels_extent=[extent[0]/3+x,extent[1]/3+y,extent[2]/3+x,extent[3]/3+y],padded_source_extent=[extent[0]/3+x,extent[1]/3+y,extent[2]/3+x,extent[3]/3+y],preprocessing_contract='3x bicubic, min-RGB background division and CLAHE; fixed crop_line64',source_pixels_modified=True)
+     new.append(dict(box=box,text=text,visual_text=g['raw_visual_text'],confidence=g['confidence'],geometry=geom,layer='printed',recognizer='frozen_overlap_short_expert',printed_probability=1.,enhanced_header=True,header_region=region))
+  return dict(new=new,old_support=[1.]*len(base['lines']),WH=list(image.size),header_regions=traces)
+
+def clean(s):return ''.join(s.split()).replace('\u200c','')
+def isolated(line,lines,maxlen,radius,W):
+ t=clean(line['text']);b=line['box'];h=max(2,b[3]-b[1]);cx=(b[0]+b[2])/2
+ if not t or len(t)>maxlen:return False
+ if all(c.isdigit()for c in t)and(cx<.16*W or cx>.84*W):return False
+ for q in lines:
+  if q is line or len(clean(q['text']))<=2:continue
+  z=q['box'];hh=max(2,z[3]-z[1]);over=min(b[3],z[3])-max(b[1],z[1]);gap=max(0,max(b[0],z[0])-min(b[2],z[2]))
+  if over>=.5*min(h,hh)and gap<=radius*max(h,hh):return False
+ return True
+
+def apply(base,prepared,params):
+ p=dict(printed_probability=.5,read_confidence=params['header_confidence'],old_support=.5);result=add_printed(base,prepared,p);original=copy.deepcopy(result);removed=[]
+ if params['max_chars']:
+  removed=[l for l in result['lines']if isolated(l,result['lines'],params['max_chars'],params['radius'],prepared['WH'][0])]
+ if not removed:return result
+ # Preserve grids as atomic structures; their punctuation/digit cells have a dedicated path.
+ if any(r.get('table_trace',{}).get('grid')for r in result['regions']):return result
+ keys={(tuple(l['box']),l['text'])for l in removed};ll=[l for l in result['lines']if(tuple(l['box']),l['text'])not in keys];result['lines']=ll;regions=[]
+ for r in result['regions']:
+  r['lines']=[l for l in r.get('lines',[])if(tuple(l['box']),l['text'])not in keys];r['text']='\n'.join(l['text']for l in r['lines'])
+  if r['lines']:regions.append(r)
+ result.update(regions=regions,text='\n\n'.join(r['text']for r in regions),global_readings=ll,emitted_boxes=[l['box']for l in ll],recognition_extents=[l.get('geometry',{}).get('source_pixels_extent',l['box'])for l in ll]);result['isolated_mark_filter']=dict(removed=removed,parameters=params,margin_digits_exempt=True);return result

stallion/bina04-header-marks-95k-20261103/production.py

@@ -0,0 +1,16 @@
+import json,importlib.util,sys
+from pathlib import Path
+R=Path(__file__).resolve().parent
+class Runtime:
+ def __init__(self,device='auto'):
+  s=importlib.util.spec_from_file_location('frozen_parent_header_marks','__BINA_ROOT__/stallion/bina04-numeric-routing-95k-20261103/consumer.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);self.params=json.loads((R/'CALIBRATION.json').read_text())['selected']['parameters'];self.collector=None;sys.path.insert(0,str(R))
+ def __call__(self,image,old,kind,name='',**kwargs):
+  out=self.parent(image,old,kind,name,**kwargs)
+  if self.params['header_confidence']>1:p=dict(new=[],old_support=[1.]*len(out['lines']),WH=list(image.size),header_regions=[])
+  else:
+   from candidate import Collector
+   if self.collector is None:self.collector=Collector('cpu')
+   p=self.collector(image,out)
+  from candidate import apply
+  return apply(out,p,self.params)
+ def close(self):self.parent.close()

stallion/bina04-header-marks-95k-20261103/serializer.py

@@ -0,0 +1,82 @@
+"""Learned visual printed layer; body records and order remain immutable."""
+import copy,json,sys
+from pathlib import Path
+import numpy as np
+from PIL import Image
+R=Path(__file__).resolve().parent;P=Path('__BINA_ROOT__/stallion/indra-scale');sys.path[:0]=[str(P),'__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031','__BINA_ROOT__/stallion/bina04-indra-pilot-20261027/upstream']
+def body(p):return json.dumps([l for l in p['lines']if l.get('layer')!='printed'],sort_keys=True,ensure_ascii=False)
+def hit(a,b):
+ z=max(0,min(a[2],b[2])-max(a[0],b[0]))*max(0,min(a[3],b[3])-max(a[1],b[1]));return z/max(1,min((a[2]-a[0])*(a[3]-a[1]),(b[2]-b[0])*(b[3]-b[1])))
+def same_glyph_evidence(new,old):
+ """Independent heads must not serialize the same CTC glyphs twice.
+ Different text at the same location is preserved, including print under pen.
+ """
+ if old.get('layer')=='printed':return False
+ a=new['box'];b=old['box'];ah=max(1,a[3]-a[1]);bh=max(1,b[3]-b[1])
+ if hit(a,b)<.5 or abs((a[1]+a[3]-b[1]-b[3])/2)>.5*min(ah,bh):return False
+ ng=new.get('geometry',{});og=old.get('geometry',{})
+ if og.get('frame_centers_source_valid',True)is False:return False
+ nv=ng.get('raw_visual_text',new.get('visual_text',''));ov=og.get('raw_visual_text',old.get('visual_text',''));nc=ng.get('frame_centers',[]);oc=og.get('frame_centers',[])
+ if not nv or len(nc)!=len(nv) or len(oc)!=len(ov):return False
+ start=0
+ while True:
+  at=ov.find(nv,start)
+  if at<0:return False
+  delta=np.abs(np.asarray(nc)-np.asarray(oc[at:at+len(nv)]))
+  if float(np.quantile(delta,.9))<=max(2.,.5*min(ah,bh)):return True
+  start=at+1
+
+class Layer:
+ def __init__(self,device='cuda'):
+  import torch
+  from surya.detection import DetectionPredictor
+  from rtl_reader import RTLReader
+  self.det=DetectionPredictor.local(device=device,dtype=torch.float32);q=torch.load(R/'run/selected.pt',map_location='cpu',weights_only=True);assert q['classes']==['printed','handwriting'];self.det.model.load_state_dict(q['model']);self.det.model.eval();self.det._disable_tqdm=True;self.reader=RTLReader('__BINA_ROOT__/stallion/bina04-print-textgate-20261102/run/selected-printed-recognizer.pt','__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json',device=device)
+ def infer(self,image,base):
+  from surya.detection.heatmap import parallel_get_boxes
+  from text_contract import decode_prediction
+  W,H=image.size;preds,sizes=next(self.det.batch_detection([image],batch_size=1));p,h=preds[0];raw=parallel_get_boxes([np.minimum(1,p*1.5),h],sizes[0]).model_dump(mode='json');boxes=[]
+  def support(b):
+   x,y,X,Y=b;hh,ww=p.shape;a=max(0,round(x*ww/W));A=min(ww,round(X*ww/W));b0=max(0,round(y*hh/H));B=min(hh,round(Y*hh/H));z=p[b0:B,a:A];return float(np.quantile(z,.65))if z.size else 0.
+  atomic=[r for r in base['regions']if r.get('table_trace',{}).get('grid')];gridboxes=[]
+  for r in atomic:
+   ll=r.get('lines',[])
+   if ll:gridboxes.append([min(l['box'][0]for l in ll),min(l['box'][1]for l in ll),max(l['box'][2]for l in ll),max(l['box'][3]for l in ll)])
+  for a in raw['bboxes']:
+   q=np.asarray(a['polygon']);b=[float(q[:,0].min()),float(q[:,1].min()),float(q[:,0].max()),float(q[:,1].max())]
+   if not any(hit(b,g)>.8 for g in gridboxes):boxes.append((b,support(b)))
+  up=np.asarray(image.resize((2*W,2*H),Image.Resampling.BICUBIC).convert('L'));new=[]
+  for i in range(0,len(boxes),24):
+   bb=boxes[i:i+24];_,receipts=self.reader(up,np.asarray([[2*v for v in b]for b,_ in bb],np.float32))
+   for (b,s),g in zip(bb,receipts):
+    g={k:v for k,v in g.items()if k!='_numeric_capture_id'};text=decode_prediction(g['raw_visual_text']);geometry=dict(original_WH=[W,H],ink_box=[v/2 for v in g['ink_box']],source_pixels_extent=[v/2 for v in g['padded_source_extent']],padded_source_extent=[v/2 for v in g['padded_source_extent']],raw_visual_text=g['raw_visual_text'],raw_token_ids=g.get('raw_token_ids',[]),confidence=g['confidence'],frame_centers=[v/2 for v in g.get('frame_centers',[])],upscale=2,upscaled_input_receipt=g,preprocessing_contract='2x bicubic then unchanged crop_line64 and overlap-trained short-label expert')
+    new.append(dict(box=b,text=text,visual_text=g['raw_visual_text'],confidence=g['confidence'],geometry=geometry,layer='printed',recognizer='frozen_overlap_short_expert',printed_probability=s,new_two_class_printed=True))
+  return dict(new=new,old_support=[support(l['box'])if l.get('layer')=='printed'else None for l in base['lines']],raw=raw,WH=[W,H],no_template_or_order_model=True)
+def apply(base,prepared,params):
+ if params.get('disabled'):return copy.deepcopy(base)
+ result=copy.deepcopy(base);original=body(base);new=[l for l in prepared['new']if l['text'].strip()and l['confidence']>=params['read_confidence']and l['printed_probability']>=params['printed_probability']];duplicates=[l for l in new if any(same_glyph_evidence(l,b)for b in base['lines'])];new=[l for l in new if not any(same_glyph_evidence(l,b)for b in base['lines'])];new.sort(key=lambda l:(-l['printed_probability'],-l['confidence']));unique=[]
+ for l in new:
+  if not any(hit(l['box'],q['box'])>.6 for q in unique):unique.append(l)
+ keep=[];removed=[]
+ for i,l in enumerate(base['lines']):
+  if l.get('layer')!='printed':keep.append(copy.deepcopy(l));continue
+  hits=[n for n in unique if hit(l['box'],n['box'])>.5];b=l['box'];covered=bool(hits)and max(0,min(b[2],max(n['box'][2]for n in hits))-max(b[0],min(n['box'][0]for n in hits)))/max(1,b[2]-b[0])>=.8
+  if not covered and prepared['old_support'][i]>=params['old_support']:keep.append(copy.deepcopy(l))
+  else:removed.append(l)
+ # Never re-sort existing nodes. Insert new print by the nearest physical anchor.
+ slots={i:[]for i in range(len(keep)+1)}
+ for l in unique:
+  cy=(l['box'][1]+l['box'][3])/2;cx=(l['box'][0]+l['box'][2])/2
+  if keep:
+   k=min(range(len(keep)),key=lambda i:abs(cy-(keep[i]['box'][1]+keep[i]['box'][3])/2)+.15*abs(cx-(keep[i]['box'][0]+keep[i]['box'][2])/2));slot=k+int(cy>=(keep[k]['box'][1]+keep[k]['box'][3])/2)
+  else:slot=0
+  slots[slot].append(l)
+ lines=[]
+ for i in range(len(keep)+1):
+  lines+=sorted(slots[i],key=lambda l:((l['box'][1]+l['box'][3])/2,-l['box'][0]))
+  if i<len(keep):lines.append(keep[i])
+ if not unique and not removed:return result
+ # Atomic grids remain byte-identical; page row paths serialize line records.
+ grids=[r for r in base['regions']if r.get('table_trace',{}).get('grid')]
+ if grids:return copy.deepcopy(base)
+ result.update(global_readings=lines,detected_boxes=[l['box']for l in lines],lines=lines,text='\n'.join(l['text']for l in lines),regions=[dict(path='two_class_printed_overlay',lines=lines,text='\n'.join(l['text']for l in lines))],emitted_boxes=[l['box']for l in lines],recognition_extents=[l.get('geometry',{}).get('source_pixels_extent',l['box'])for l in lines]);assert body(result)==original;result['two_class_printed_layer']=dict(new_printed=len(unique),duplicate_read_evidence_removed=len(duplicates),old_printed_removed=len(removed),parameters=params,body_records_byte_identical=True,template_text_used=False,learned_order_changed=False);return result

stallion/bina04-indra-pilot-20261027/models/det.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2bd4cba23e9c328cf7ec824634e423e191a2c6aacbbf35c62bd79dd70f62f59
+size 389948

stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/__init__.py

@@ -0,0 +1,49 @@
+"""Indra OCR: fast CPU OCR (line detection, CTC recognition, reading order, markdown)."""
+
+from .document import Converter
+from .models import model_dir
+from .parallel import convert_many, usable_cores
+from .models import ModelError
+from .pipeline import InputError, Line, OrtOCR, Page, load_pages
+
+__all__ = [
+    "Converter",
+    "InputError",
+    "ModelError",
+    "Line",
+    "OrtOCR",
+    "Page",
+    "convert_many",
+    "usable_cores",
+    "load_pages",
+    "load",
+    "model_dir",
+]
+__version__ = "1.1.2"
+
+
+def load(
+    path=None,
+    threads: int = 4,
+    dpi: float = 200.0,
+    mode: str = "auto",
+    image_ocr: bool = True,
+    providers: list | None = None,
+    furniture: bool = False,
+    math: bool = False,
+) -> Converter:
+    """A ready Converter (see ``Converter`` for the options).  mode="ocr" never
+    reads the PDF text layer.  furniture: keep page headers and footers in the
+    markdown.  math: write math as LaTeX (slower on math pages; off by default)."""
+    return Converter(
+        OrtOCR(
+            model_dir(path, math=math),
+            threads=threads,
+            providers=providers,
+            furniture=furniture,
+            math=math,
+        ),
+        dpi=dpi,
+        mode=mode,
+        image_ocr=image_ocr,
+    )

stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/det_post.py

@@ -0,0 +1,155 @@
+"""Torch-free pre- and post-processing for LineNet (shared by training and inference).
+
+Inference with ONNX Runtime needs only numpy and cv2, so these functions live
+apart from the PyTorch model in det_model.py, which re-exports them.
+"""
+
+from __future__ import annotations
+
+import math
+
+import numpy as np
+
+# Target geometry, shared by training (encode) and inference (decode).
+CORE_FRAC = 0.4  # core band height as a fraction of the line ink height
+CORE_X_SHRINK = 0.15  # the core is shorter than the line by this fraction of its height, at each end
+OUT_STRIDE = 2
+INPUT_MULTIPLE = 32  # H and W must divide by this
+
+
+# ---------------------------------------------------------------------------
+# Input preprocessing (shared by training and inference)
+# ---------------------------------------------------------------------------
+
+
+def page_to_input(gray: np.ndarray, long_side: int) -> tuple[np.ndarray, float]:
+    """Resize a uint8 grayscale page so its long side is ``long_side`` and pad
+    to a multiple of 32 with white.  Returns (float32 [H, W] ink map, scale)."""
+    import cv2
+
+    h, w = gray.shape
+    scale = long_side / max(h, w)
+    nh, nw = max(1, round(h * scale)), max(1, round(w * scale))
+    if (nh, nw) != (h, w):
+        interp = cv2.INTER_AREA if scale < 1 else cv2.INTER_LINEAR
+        gray = cv2.resize(gray, (nw, nh), interpolation=interp)
+    ph = math.ceil(nh / INPUT_MULTIPLE) * INPUT_MULTIPLE
+    pw = math.ceil(nw / INPUT_MULTIPLE) * INPUT_MULTIPLE
+    out = np.zeros((ph, pw), np.float32)  # ink map: 0 = white
+    out[:nh, :nw] = 1.0 - gray.astype(np.float32) / 255.0
+    return out, scale
+
+
+# ---------------------------------------------------------------------------
+# Targets
+# ---------------------------------------------------------------------------
+
+
+def encode_targets(
+    boxes: np.ndarray,
+    in_hw: tuple[int, int],
+    valid_hw: tuple[int, int],
+    ignore: np.ndarray | None = None,
+):
+    """Build training targets at 1/2 resolution.
+
+    boxes:   [N, 4] line ink boxes in input pixels (x0, y0, x1, y1)
+    in_hw:   padded input size; valid_hw: size of the real page inside it
+    ignore:  [M, 4] regions with no loss, in input pixels
+
+    Returns core [h, w] float32, dist [2, h, w] float32, weight [h, w] float32
+    (0 on padding and ignore regions), and reg_mask [h, w] bool (core pixels).
+    """
+    s = OUT_STRIDE
+    h, w = in_hw[0] // s, in_hw[1] // s
+    core = np.zeros((h, w), np.float32)
+    dist = np.zeros((2, h, w), np.float32)
+    weight = np.zeros((h, w), np.float32)
+    weight[: math.ceil(valid_hw[0] / s), : math.ceil(valid_hw[1] / s)] = 1.0
+    if ignore is not None:
+        for x0, y0, x1, y1 in ignore:
+            weight[
+                max(0, int(y0 // s)) : max(0, math.ceil(y1 / s)),
+                max(0, int(x0 // s)) : max(0, math.ceil(x1 / s)),
+            ] = 0.0
+    # Larger lines first, so a small line drawn later keeps its own band.
+    order = np.argsort(-(boxes[:, 3] - boxes[:, 1])) if len(boxes) else []
+    ys = (np.arange(h, dtype=np.float32) + 0.5) * s  # pixel centres in input px
+    for i in order:
+        x0, y0, x1, y1 = boxes[i]
+        lh = y1 - y0
+        if lh < 2 or x1 - x0 < 2:
+            continue
+        cy = 0.5 * (y0 + y1)
+        half = 0.5 * CORE_FRAC * lh
+        r0 = max(0, int(math.floor((cy - half) / s)))
+        r1 = min(h, max(r0 + 1, int(math.ceil((cy + half) / s))))
+        sx = min(CORE_X_SHRINK * lh, 0.25 * (x1 - x0))
+        c0 = max(0, int(math.floor((x0 + sx) / s)))
+        c1 = min(w, max(c0 + 1, int(math.ceil((x1 - sx) / s))))
+        if r0 >= h or c0 >= w:
+            continue
+        core[r0:r1, c0:c1] = 1.0
+        dist[0, r0:r1, c0:c1] = (ys[r0:r1] - y0)[:, None]
+        dist[1, r0:r1, c0:c1] = (y1 - ys[r0:r1])[:, None]
+        # A line inside an ignore region is still a positive.
+        weight[r0:r1, c0:c1] = np.maximum(weight[r0:r1, c0:c1], 1.0)
+    return core, dist, weight, core > 0.5
+
+
+# ---------------------------------------------------------------------------
+# Decoding
+# ---------------------------------------------------------------------------
+
+
+def decode(
+    out: np.ndarray,
+    thresh: float = 0.5,
+    min_area: int = 2,
+    grow_y: float = 0.0,
+    grow_x: float = 0.0,
+) -> np.ndarray:
+    """Decode one page.  out: [3, h, w] (core logit, top dist, bottom dist).
+
+    ``grow_y`` / ``grow_x`` widen every box by that fraction of its height
+    (each side), to absorb the +-1 px error of the 1/2-resolution output.
+
+    Returns [N, 5] boxes in input pixels: x0, y0, x1, y1, score."""
+    import cv2
+
+    prob = 1.0 / (1.0 + np.exp(-out[0]))
+    mask = (prob > thresh).astype(np.uint8)
+    n, lab, stats, _ = cv2.connectedComponentsWithStats(mask, connectivity=4)
+    if n <= 1:
+        return np.zeros((0, 5), np.float32)
+    s = OUT_STRIDE
+    flat = lab.ravel()
+    keep = flat > 0
+    idx = flat[keep]
+    ys = (np.indices(lab.shape)[0].ravel()[keep].astype(np.float32) + 0.5) * s
+    top = ys - out[1].ravel()[keep]
+    bot = ys + out[2].ravel()[keep]
+    sc = prob.ravel()[keep]
+    cnt = np.bincount(idx, minlength=n).astype(np.float32)
+    # Mean is a robust enough summary here and is far cheaper than a median.
+    y0 = np.bincount(idx, weights=top, minlength=n) / np.maximum(cnt, 1)
+    y1 = np.bincount(idx, weights=bot, minlength=n) / np.maximum(cnt, 1)
+    score = np.bincount(idx, weights=sc, minlength=n) / np.maximum(cnt, 1)
+    boxes = []
+    for i in range(1, n):
+        if stats[i, cv2.CC_STAT_AREA] < min_area:
+            continue
+        lh = max(1.0, y1[i] - y0[i])
+        ex = min(CORE_X_SHRINK * lh, 0.5 * lh)
+        x0 = stats[i, cv2.CC_STAT_LEFT] * s - ex
+        x1 = (stats[i, cv2.CC_STAT_LEFT] + stats[i, cv2.CC_STAT_WIDTH]) * s + ex
+        boxes.append(
+            (
+                x0 - grow_x * lh,
+                y0[i] - grow_y * lh,
+                x1 + grow_x * lh,
+                y1[i] + grow_y * lh,
+                score[i],
+            )
+        )
+    return np.asarray(boxes, np.float32).reshape(-1, 5)

stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/document.py

@@ -0,0 +1,426 @@
+"""Whole documents: the PDF text layer where it can be trusted, OCR elsewhere.
+
+For each PDF page (``mode="auto"``):
+
+1. Read the text layer with pdfium and check it (``textlayer.read_text_layer``).  A page
+   without a sound layer (a scan, an OCR layer, a broken font map) goes to OCR.
+2. The detector finds the visual lines, as in OCR.  Each text-layer character
+   goes to the detected line that contains its center.  Inside a line, the
+   characters of each text-layer run (pdfium's segment between line breaks) keep the PDF order
+   and spaces, and runs are ordered by x, right to left in a right-to-left line.  So inline math, sub- and
+   superscripts and footnote marks land in their line, and a run across two
+   table cells splits between them.  A detected line with no text (text inside
+   an image) is OCR'd when the recognizer reads the page's script.
+3. The rest is the OCR path: lines split at wide gaps, reading order, layout
+   classes and markdown.  The PDF content order is kept when it is coherent
+   (few upward jumps that are not column changes, and the order model agrees
+   with at least 85% of consecutive pairs).
+
+``mode="ocr"`` never reads the text layer; ``mode="text"`` uses it even when
+the checks fail (and OCR only on pages with no text at all).
+"""
+
+from __future__ import annotations
+
+import time
+import unicodedata
+from pathlib import Path
+
+import numpy as np
+
+from .pipeline import InputError, OrtOCR, Page, is_pdf, read_images
+from .textlayer import BROKEN_FONT_MAP, page_turn, read_text_layer, vertical_layout
+
+CHECK_DPI = 72.0  # resolution of the render for the text-layer ink checks
+MAX_PAGE_PIXELS = 100e6  # a page larger than this at the set dpi renders at a lower dpi (large scans reach ~65 MP)
+
+READABLE_MIN = 0.9  # share of the layer's letters the recognizer must know
+AGREE_LINES = 6  # text-layer lines whose script is checked against the page image
+AGREE_SHARE = 0.25  # under this share of agreeing lines, the layer is not the page's text
+LATIN_CODES = 0.9  # a layer line this Latin while its image is another script: legacy font codes
+NOT_THE_PAGE = "text layer does not match the page"  # Page.note when the check sends a page to OCR
+
+
+def latin_share(text: str) -> float:
+    """Share of the letters in text that are basic Latin (a-z, A-Z)."""
+    letters = [c for c in text if c.isalpha()]
+    return sum(c.isascii() for c in letters) / max(len(letters), 1)
+
+
+def layer_agrees(agree: list) -> bool:
+    """Whether a text layer matches its page: at least AGREE_SHARE of the checked
+    lines agree.  A legacy font (Kruti Dev and DevLys for Hindi, Bijoy for Bangla)
+    stores Latin codes for another script, so almost every line disagrees."""
+    return not agree or sum(agree) >= AGREE_SHARE * len(agree)
+
+
+def readable(ocr: OrtOCR, runs: list) -> bool:
+    """Whether the recognizer can read the script of these text-layer runs."""
+    letters = [c for r in runs for c in r.text if c.isalpha()]
+    if not letters:
+        return True
+    vocabs = [ocr.vocab.index] + [v.index for _, v in ocr.groups.values()]
+    # A recognizer that reads Hangul as jamo knows a syllable by its jamo.
+    known = sum(
+        any(
+            c in v or all(j in v for j in unicodedata.normalize("NFD", c))
+            for v in vocabs
+        )
+        for c in letters
+    )
+    return known >= READABLE_MIN * len(letters)
+
+
+def right_to_left(tokens: list) -> bool:
+    """Unicode bidi base direction: most strong characters are right-to-left."""
+    rtl = ltr = 0
+    for t in tokens:
+        b = unicodedata.bidirectional(t[0]) if len(t[0]) == 1 else ""
+        rtl += b in ("R", "AL")
+        ltr += b == "L"
+    return rtl > ltr
+
+
+def _covered(box, boxes: np.ndarray, frac: float = 0.5) -> bool:
+    """Whether more than ``frac`` of ``box`` lies inside one of ``boxes``."""
+    ix = np.clip(
+        np.minimum(box[2], boxes[:, 2]) - np.maximum(box[0], boxes[:, 0]), 0, None
+    )
+    iy = np.clip(
+        np.minimum(box[3], boxes[:, 3]) - np.maximum(box[1], boxes[:, 1]), 0, None
+    )
+    area = max((box[2] - box[0]) * (box[3] - box[1]), 1e-6)
+    return bool(((ix * iy) / area > frac).any())
+
+
+def _render(page, scale: float) -> np.ndarray:
+    arr = page.render(scale=scale, grayscale=True).to_numpy()
+    return np.ascontiguousarray(arr[..., 0] if arr.ndim == 3 else arr)
+
+
+def assign_runs(runs: list, boxes: np.ndarray, k: float) -> tuple[list[list], int]:
+    """Tokens (char, x, content index) per detected line.
+
+    A run is one text-layer line piece (``textlayer.TextLine``), in content
+    order.  Each character goes to the smallest detected box that contains its
+    center (a space follows the character before it), so a run that spans two
+    table cells splits between them.  Inside a detected line, each run's part
+    keeps the PDF character order, and the parts are ordered by x.  ``boxes``
+    are pixels, ``k`` pixels per point.  Returns (tokens per detected line,
+    characters outside every box)."""
+    parts: list[dict] = [{} for _ in range(len(boxes))]  # line -> run idx -> tokens
+    lost = 0
+    if not len(boxes):
+        return [], sum(not c.isspace() for r in runs for c, _ in r.chars)
+    area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
+    for pos, run in enumerate(runs):
+        idx = run.seq if run.seq is not None else pos
+        j = None
+        for c, b in run.chars:
+            if not c.isspace():
+                cx, cy = (b[0] + b[2]) / 2 * k, (b[1] + b[3]) / 2 * k
+                inside = (
+                    (boxes[:, 0] <= cx)
+                    & (cx <= boxes[:, 2])
+                    & (boxes[:, 1] <= cy)
+                    & (cy <= boxes[:, 3])
+                )
+                if not inside.any():
+                    lost += 1
+                    j = None
+                    continue
+                j = int(np.flatnonzero(inside)[np.argmin(area[inside])])
+                parts[j].setdefault(idx, []).append((c, cx, idx))
+            elif j is not None:
+                prev = parts[j][idx][-1]
+                parts[j][idx].append((c, prev[1], idx))
+    toks: list[list] = []
+    for line in parts:
+        tk: list = []
+        rtl = right_to_left([t for part in line.values() for t in part])
+        order = (
+            sorted(line, key=lambda i: -max(t[1] for t in line[i]))
+            if rtl
+            else sorted(line, key=lambda i: min(t[1] for t in line[i]))
+        )
+        for idx in order:
+            if tk:
+                tk.append((" ", tk[-1][1], idx))  # parts of different runs
+            tk += line[idx]
+        toks.append(tk)
+    return toks, lost
+
+
+class Converter:
+    def __init__(
+        self,
+        ocr: OrtOCR,
+        dpi: float = 200.0,
+        mode: str = "auto",
+        image_ocr: bool = True,
+    ):
+        """mode: auto (text layer where sound, OCR elsewhere), ocr (always OCR,
+        never read the text layer), text (text layer even when the checks fail).
+        image_ocr: OCR detected lines that
+        hold no text-layer characters (False: drop them)."""
+        if mode not in ("auto", "ocr", "text"):
+            raise ValueError(f"mode must be auto, ocr or text, not {mode!r}")
+        self.ocr, self.dpi, self.mode = ocr, dpi, mode
+        self.image_ocr = image_ocr
+
+    def page_scale(self, page) -> float:
+        """Pixels per point for this page: the set dpi, lowered for pages so
+        large that the render would pass MAX_PAGE_PIXELS (posters, plans)."""
+        w, h = page.get_size()
+        k = self.dpi / 72.0
+        return min(k, (MAX_PAGE_PIXELS / max(w * h, 1.0)) ** 0.5)
+
+    def pdf_page(self, pdf_path: str, doc, index: int) -> Page:
+        page = doc[index]
+        t: dict = {}
+        note = "mode=ocr"
+        if self.mode != "ocr":
+            t0 = time.perf_counter()
+            s = CHECK_DPI / 72.0
+            check = _render(page, s)
+            tl = read_text_layer(page, check, s)
+            # A table set sideways: most characters share one quarter turn.  Read
+            # the page turned upright, as the OCR path does with orient.onnx.
+            k = page_turn(tl.stats)
+            rot = page.get_rotation()
+            if k and (rot + 90 * k) % 360 == 0:
+                # The page's /Rotate alone turns the text: read the page without
+                # it.  pdfium builds its text runs in the displayed frame, and on
+                # a turned page it can join a line's font runs out of order.
+                page.set_rotation(0)
+                upright = _render(page, s)
+                tl = read_text_layer(page, upright, s)
+                if tl.lines and (tl.trusted or self.mode == "text"):
+                    tl.stats["turn"] = k
+                    t["text_ms"] = 1000 * (time.perf_counter() - t0)
+                    out = self._text_page(page, tl, t, check=self.mode == "auto")
+                    if out is not None:
+                        return out
+                page.set_rotation(rot)
+                tl = read_text_layer(
+                    page, np.ascontiguousarray(np.rot90(check, -k)), s, k
+                )
+            elif k:
+                tl = read_text_layer(
+                    page, np.ascontiguousarray(np.rot90(check, -k)), s, k
+                )
+            elif vertical_layout(tl.stats):
+                # Vertical writing: upright characters, read in the page turned a
+                # quarter turn counter-clockwise (columns become lines).
+                tl = read_text_layer(
+                    page, np.ascontiguousarray(np.rot90(check, 1)), s, 0, 3
+                )
+                tl.stats["vertical"] = True
+            t["text_ms"] = 1000 * (time.perf_counter() - t0)
+            # A layer the recognizer cannot read beats OCR, unless its codes are junk.
+            unreadable = tl.reason != BROKEN_FONT_MAP and not readable(
+                self.ocr, tl.lines
+            )
+            note = tl.reason
+            if tl.lines and (tl.trusted or self.mode == "text" or unreadable):
+                # A trusted layer in a script the recognizer reads is checked
+                # against what the page shows (legacy fonts store other codes).
+                out = self._text_page(
+                    page, tl, t, check=self.mode == "auto" and tl.trusted and not unreadable
+                )
+                if out is not None:
+                    return out
+                note = NOT_THE_PAGE
+            # A layer with a broken font map still places its glyphs: when most
+            # run upright, the page is upright (orient.onnx is not asked).
+            st = tl.stats
+            turn = not (
+                tl.reason == BROKEN_FONT_MAP and st["rotated"] < 0.5 * st["chars"]
+            )
+        else:
+            turn = True
+        t0 = time.perf_counter()
+        gray = _render(page, self.page_scale(page))
+        t["render_ms"] = 1000 * (time.perf_counter() - t0)
+        out = self.ocr(gray, turn)
+        out.timings = {**t, **out.timings}
+        out.timings["total_ms"] = sum(
+            v for k, v in out.timings.items() if k != "total_ms"
+        )
+        out.note = f"{note}; {out.note}"
+        return out
+
+    def _text_page(self, page, tl, t: dict, check: bool = False) -> Page | None:
+        """The page from its text layer.  check: first compare the script of up to
+        AGREE_LINES of the layer's lines with the script the line classifier sees
+        in their images; when they do not match (layer_agrees), return None, so OCR
+        reads the page."""
+        k = self.page_scale(page)
+        turn = tl.stats.get("turn", 0)
+        frame = tl.stats.get("frame", turn)
+        vertical = bool(tl.stats.get("vertical"))
+        t0 = time.perf_counter()
+        gray = _render(page, k)
+        if frame:
+            gray = np.ascontiguousarray(np.rot90(gray, -frame))
+        t["render_ms"] = 1000 * (time.perf_counter() - t0)
+        t0 = time.perf_counter()
+        boxes = self.ocr.detect(gray)
+        t["det_ms"] = 1000 * (time.perf_counter() - t0)
+        toks, lost = assign_runs(tl.lines, boxes[:, :4], k)
+        rec: list = [None] * len(boxes)
+        empty = []
+        for i, tk in enumerate(toks):
+            text = self.ocr._decode(tk) if tk else ""
+            if text:
+                rec[i] = (text, 1.0, tk)
+            else:
+                empty.append(i)
+        if check:
+            t0 = time.perf_counter()
+            full = [i for i, r in enumerate(rec) if r and sum(c.isalpha() for c in r[0]) >= 4]
+            pick = full[:: max(1, len(full) // AGREE_LINES)][:AGREE_LINES]
+            if pick:
+                # A line agrees unless its layer text is Latin while the router is
+                # sure its image is another script.
+                groups, sure = self.ocr.box_groups(gray, boxes[pick])
+                agree = [
+                    not (latin_share(rec[i][0]) >= LATIN_CODES and g != "base" and s)
+                    for i, g, s in zip(pick, groups, sure)
+                ]
+                t["check_ms"] = 1000 * (time.perf_counter() - t0)
+                if not layer_agrees(agree):
+                    return None
+        source = "text"
+        # Sideways runs (a rotated table header, a chart's axis label) come from
+        # the layer as their own lines; detected lines on them are not OCR'd.
+        side = [r for r in tl.rotated if r.text.strip()]
+        side_boxes = np.array([[v * k for v in r.bbox] for r in side], np.float32)
+        if len(side) and empty:
+            empty = [i for i in empty if not _covered(boxes[i, :4], side_boxes)]
+        if empty and self.image_ocr and readable(self.ocr, tl.lines):
+            t0 = time.perf_counter()
+            for i, r in zip(
+                empty, self.ocr.recognize(gray, boxes[empty], vertical=vertical)
+            ):
+                if r[0].strip():
+                    rec[i] = r
+                    source = "hybrid"
+            t["rec_ms"] = 1000 * (time.perf_counter() - t0)
+        if getattr(self.ocr, "math", None) is not None:
+            t0 = time.perf_counter()
+            self._read_math(gray, boxes, rec)
+            t["math_ms"] = 1000 * (time.perf_counter() - t0)
+        keep = [i for i, r in enumerate(rec) if r is not None]
+        kept_boxes = boxes[keep].reshape(-1, boxes.shape[1])
+        kept_rec = [rec[i] for i in keep]
+        if len(side):
+            extra = np.zeros((len(side), boxes.shape[1]), np.float32)
+            extra[:, :4] = side_boxes
+            if boxes.shape[1] > 4:
+                extra[:, 4] = 1.0
+            kept_boxes = np.concatenate([kept_boxes, extra])
+            for r, b in zip(side, side_boxes):
+                cx = float(b[0] + b[2]) / 2  # one x: a sideways line is never cut
+                kept_rec.append((r.text, 1.0, [(c, cx, r.seq) for c in r.text]))
+        out = self.ocr.assemble(
+            (gray.shape[1], gray.shape[0]),
+            kept_boxes,
+            kept_rec,
+            t,
+            vertical=vertical,
+        )
+        out.source = source
+        out.rotation = 90 * turn
+        if vertical:  # back on the page as it is: (x, y) -> (W - y, x)
+            W = gray.shape[0]
+
+            def back(b):
+                return [W - b[3], b[0], W - b[1], b[2]]
+
+            for ln in out.lines:
+                ln.box = back(ln.box)
+            for b in out.blocks:
+                b["bbox"] = back(b["bbox"])
+            out.vertical = True
+        out.note += f"; {lost} chars outside detected lines" if lost else ""
+        return out
+
+    def _read_math(self, gray, boxes, rec: list) -> None:
+        """Text-layer lines that hold math (the classifier, on their crops) are read
+        by MathRec instead: the layer has the symbols but not the LaTeX.  The line
+        keeps its place in the PDF content order (its first character's index)."""
+        from .line_mask import crop_lines
+
+        idx = [
+            i
+            for i, r in enumerate(rec)
+            if r is not None
+            and r[2]
+            and isinstance(r[2][0][0], str)
+            and len(r[2][0]) > 2
+        ]
+        if not idx:
+            return
+        crops = crop_lines(gray, boxes[idx, :4], self.ocr.height, None)
+        math = [i for i, f in zip(idx, self.ocr.math.flags(crops)) if f]
+        if not math:
+            return
+        from .math import MATH_MIN_CONF
+
+        for i, (text, conf) in zip(math, self.ocr.math.read(gray, boxes[math])):
+            # A reading without math, or not confident, leaves the layer's text.
+            if "$" in text and conf >= MATH_MIN_CONF:
+                first = min(tk[2] for tk in rec[i][2])
+                xc = float(boxes[i, 0] + boxes[i, 2]) / 2
+                rec[i] = (text, 1.0, [(text, xc, first)])
+
+    def convert(
+        self, path: str | Path, max_pages: int = 0, password: str | None = None
+    ):
+        """Yield (page index, Page) for a PDF or an image file (the file's bytes
+        decide which).  A page that fails yields an empty Page whose ``source``
+        is "error" and whose ``note`` says why; a file that cannot be opened
+        raises InputError.  max_pages: 0 reads every page; a negative value
+        raises ValueError."""
+        if max_pages < 0:
+            raise ValueError(
+                f"max_pages must be 0 (all pages) or more, not {max_pages}"
+            )
+        path = Path(path)
+        if not path.is_file():
+            raise InputError(f"{path}: no such file")
+        if not is_pdf(path):
+            for i, gray in enumerate(read_images(path)[: max_pages or None]):
+                yield i, self._safe(lambda: self.ocr(gray))
+            return
+        import pypdfium2 as pdfium
+
+        try:
+            doc = pdfium.PdfDocument(str(path), password=password)
+        except pdfium.PdfiumError as e:
+            locked = "password" in str(e).lower()
+            raise InputError(
+                f"{path}: password-protected PDF (pass password=, or --password)"
+                if locked
+                else f"{path}: cannot open the PDF ({e})"
+            ) from e
+        try:
+            n = len(doc) if not max_pages else min(len(doc), max_pages)
+            for i in range(n):
+                yield i, self._safe(lambda: self.pdf_page(str(path), doc, i))
+        finally:
+            doc.close()
+
+    @staticmethod
+    def _safe(read) -> Page:
+        try:
+            return read()
+        except Exception as e:  # noqa: BLE001  (one bad page must not end the document)
+            return Page(
+                [],
+                {"total_ms": 0.0},
+                markdown="",
+                source="error",
+                note=f"{type(e).__name__}: {e}",
+            )

stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/layout_post.py

@@ -0,0 +1,381 @@
+"""Page text from the ordered lines and the OrderNet layout heads (torch-free).
+
+OrderNet with ``layout`` gives every line a block class and every pair of lines
+a "same block" logit.  Walking the reading order, a new block starts wherever
+the same-block logit between consecutive lines is negative.  Each block is then
+rendered by its class:
+
+  Section-Header        "## " + joined text
+  Text, Caption, ...    one paragraph; a word split by a hyphen at a line end is joined
+  List-Group            one line per item (a new item starts at a bullet or number)
+  Table                 an HTML table assembled from the cell lines (rows by vertical
+                        centre, columns anchored on the fullest rows)
+  Page-Header / Footer  left out unless ``keep_furniture``
+"""
+
+from __future__ import annotations
+
+import html
+import re
+
+import numpy as np
+
+from .order_post import BLOCK_CLASSES, _cluster
+
+HEADER, FOOTER = BLOCK_CLASSES.index("Page-Header"), BLOCK_CLASSES.index("Page-Footer")
+SECTION = BLOCK_CLASSES.index("Section-Header")
+LIST = BLOCK_CLASSES.index("List-Group")
+TABLE = BLOCK_CLASSES.index("Table")
+
+_WHOLE_MATH = re.compile(r"^\$[^$]+\$$")  # a line that is one formula
+# A list marker, followed by text or alone on its line (the detector often cuts
+# a bullet off as a line of its own; the next line then joins its item).
+_ITEM = re.compile(r"^\s*([•\-–▪◦*·→]|\(?\d{1,3}[.)]|\(?[a-zA-Z][.)])(\s|$)")
+
+
+def _cjk(c: str) -> bool:
+    """Han, kana, CJK punctuation and fullwidth forms: written without spaces
+    (Korean Hangul is not: Korean separates words with spaces)."""
+    o = ord(c)
+    return (
+        0x3000 <= o <= 0x30FF
+        or 0x3400 <= o <= 0x9FFF
+        or 0xF900 <= o <= 0xFAFF
+        or 0xFF00 <= o <= 0xFFEF
+        or 0x20000 <= o <= 0x2FA1F
+    )
+
+
+def join_lines(texts: list[str]) -> str:
+    """Join the lines of one paragraph.  'tumori-' + 'genesis' -> 'tumorigenesis';
+    a line break between two CJK characters takes no space."""
+    out = ""
+    for t in texts:
+        t = t.strip()
+        if not t:
+            continue
+        if (
+            len(out) > 1
+            and out.endswith("-")
+            and out[-2].isalpha()
+            and t[0].isalpha()
+            and t[0].islower()
+        ):
+            out = out[:-1] + t
+        elif out and _cjk(out[-1]) and _cjk(t[0]):
+            out += t
+        else:
+            out = f"{out} {t}" if out else t
+    return out
+
+
+def split_blocks(order: list[int], same: np.ndarray) -> list[list[int]]:
+    """Runs of the reading order with a positive same-block logit between neighbours."""
+    blocks: list[list[int]] = []
+    for i in order:
+        if blocks and same[blocks[-1][-1], i] > 0:
+            blocks[-1].append(i)
+        else:
+            blocks.append([i])
+    return blocks
+
+
+def _rows(boxes: np.ndarray) -> list[list[int]]:
+    """Group cell lines into rows by vertical centre, top to bottom, left to right."""
+    cy = (boxes[:, 1] + boxes[:, 3]) / 2
+    h = float(np.median(boxes[:, 3] - boxes[:, 1]))
+    rows: list[list[int]] = []
+    for i in np.argsort(cy):
+        if rows and abs(cy[i] - np.mean(cy[rows[-1]])) < 0.5 * h:
+            rows[-1].append(int(i))
+        else:
+            rows.append([int(i)])
+    return [sorted(r, key=lambda k: boxes[k, 0]) for r in rows]
+
+
+def _merge_wrapped(rows: list[list[int]], boxes: np.ndarray) -> list[list[int]]:
+    """Join the line rows of wrapped cells into one table row.
+
+    Inside a wrapped cell the gap to the next line is the font's line gap; a new
+    table row adds cell padding.  A row joins the row above when its gap is under
+    half the table's median row gap and under 0.6 line heights, and it has fewer
+    lines than the row above (a continuation fills only the wrapped columns)."""
+    if len(rows) < 3:
+        return rows
+    h = float(np.median(boxes[:, 3] - boxes[:, 1]))
+    gaps = [
+        float(boxes[nxt, 1].min() - boxes[cur, 3].max())
+        for cur, nxt in zip(rows[:-1], rows[1:])
+    ]
+    med = float(np.median(gaps))
+    out = [list(rows[0])]
+    prev = rows[0]
+    for g, r in zip(gaps, rows[1:]):
+        # A continuation fills only the wrapped columns: fewer lines than the row above.
+        if g < 0.5 * med and g < 0.6 * h and len(r) < len(prev):
+            out[-1] += r
+        else:
+            out.append(list(r))
+        prev = r
+    return out
+
+
+def table_html(boxes: np.ndarray, texts: list[str]) -> str:
+    """An HTML table from cell lines (one text line per cell, as born-digital PDFs give).
+
+    A first row that is one line across most of the table is a title: it goes
+    before the table as text.  The first remaining row is the header (<th>)."""
+    rows = _rows(boxes)
+    span_x = boxes[:, 2].max() - boxes[:, 0].min()
+    if (
+        len(rows) > 2
+        and len(rows[0]) == 1
+        and max(len(r) for r in rows) > 1
+        and boxes[rows[0][0], 2] - boxes[rows[0][0], 0] > 0.6 * span_x
+    ):
+        keep = [k for k in range(len(texts)) if k != rows[0][0]]
+        return (
+            texts[rows[0][0]]
+            + "\n\n"
+            + table_html(boxes[keep], [texts[k] for k in keep])
+        )
+    width = max(len(r) for r in rows)
+    full = [r for r in rows if len(r) == width]
+    rows = _merge_wrapped(rows, boxes)
+    # Column bands from the rows with the most cells: those rows rarely span.
+    bands = np.array(
+        [
+            [
+                np.median([boxes[r[c], 0] for r in full]),
+                np.median([boxes[r[c], 2] for r in full]),
+            ]
+            for c in range(width)
+        ]
+    )
+    # Widen each band to the midpoints between neighbours, so every x falls in one band.
+    edges = (
+        [-np.inf]
+        + [(bands[c, 1] + bands[c + 1, 0]) / 2 for c in range(width - 1)]
+        + [np.inf]
+    )
+    out = ["<table>"]
+    for ri, r in enumerate(rows):
+        tag = "th" if ri == 0 and len(rows) > 1 else "td"
+        cells: dict[int, tuple[int, str]] = {}
+        for k in sorted(r, key=lambda k: (boxes[k, 1], boxes[k, 0])):
+            x0, x1 = boxes[k, 0], boxes[k, 2]
+            first = int(np.searchsorted(edges, x0, side="right") - 1)
+            last = int(np.searchsorted(edges, x1, side="right") - 1)
+            # A cell spans every band it covers by more than half of the band.
+            span = [
+                c
+                for c in range(max(first, 0), min(last, width - 1) + 1)
+                if min(x1, bands[c, 1]) - max(x0, bands[c, 0])
+                > 0.5 * (bands[c, 1] - bands[c, 0])
+            ] or [min(max(first, 0), width - 1)]
+            c0 = span[0]
+            prev = cells.get(c0)
+            text = texts[k] if prev is None else prev[1] + " " + texts[k]
+            cells[c0] = (len(span) if prev is None else max(prev[0], len(span)), text)
+        tds, c = [], 0
+        while c < width:
+            if c in cells:
+                n, text = cells[c]
+                attr = f' colspan="{n}"' if n > 1 else ""
+                tds.append(f"<{tag}{attr}>{html.escape(text, quote=False)}</{tag}>")
+                c += n
+            else:
+                tds.append(f"<{tag}></{tag}>")
+                c += 1
+        out.append("<tr>" + "".join(tds) + "</tr>")
+    out.append("</table>")
+    return "\n".join(out)
+
+
+def heads_plausible(boxes: np.ndarray, tab: np.ndarray) -> bool:
+    """The lines of one cell are stacked (they overlap horizontally).  When the
+    cell head joins side-by-side lines in more than 10% of its cells, it has
+    failed on this table and the geometric assembly is used instead."""
+    cells = _cluster(tab[2])
+    bad = 0
+    for c in cells:
+        if len(c) < 2:
+            continue
+        x0, x1 = boxes[c, 0], boxes[c, 2]
+        if (np.minimum.outer(x1, x1) - np.maximum.outer(x0, x0)).min() < 0:
+            bad += 1
+    return bad <= 0.1 * len(cells)
+
+
+def table_html_heads(boxes: np.ndarray, texts: list[str], tab: np.ndarray) -> str:
+    """An HTML table from the OrderNet table heads.  tab [3, N, N] = logits
+    same row, same column, same cell, for the N lines of one table block."""
+    cells = _cluster(tab[2])
+    cb = np.array(
+        [
+            [boxes[c, 0].min(), boxes[c, 1].min(), boxes[c, 2].max(), boxes[c, 3].max()]
+            for c in cells
+        ]
+    )
+    ctext = [
+        " ".join(texts[k] for k in sorted(c, key=lambda k: (boxes[k, 1], boxes[k, 0])))
+        for c in cells
+    ]
+    m = len(cells)
+
+    def cell_logits(k: int) -> np.ndarray:
+        out = np.zeros((m, m), np.float32)
+        for a in range(m):
+            for b in range(a + 1, m):
+                out[a, b] = out[b, a] = tab[k][np.ix_(cells[a], cells[b])].mean()
+        return out
+
+    rows = sorted(_cluster(cell_logits(0)), key=lambda g: cb[g, 1].min())
+    cols = sorted(_cluster(cell_logits(1)), key=lambda g: cb[g, 0].min())
+    row_of = {c: r for r, g in enumerate(rows) for c in g}
+    col_of = {c: k for k, g in enumerate(cols) for c in g}
+    centers = [float(np.median((cb[g, 0] + cb[g, 2]) / 2)) for g in cols]
+    h = float(np.median(boxes[:, 3] - boxes[:, 1]))
+    grid: dict[tuple[int, int], tuple[int, str]] = {}
+    for c in range(m):
+        c0 = col_of[c]
+        span = 1
+        while (
+            c0 + span < len(cols) and cb[c, 0] - h <= centers[c0 + span] <= cb[c, 2] + h
+        ):
+            span += 1
+        key = (row_of[c], c0)
+        if key in grid:  # two cells on one slot: keep both texts
+            grid[key] = (max(grid[key][0], span), grid[key][1] + " " + ctext[c])
+        else:
+            grid[key] = (span, ctext[c])
+    out = ["<table>"]
+    for r in range(len(rows)):
+        tag = "th" if r == 0 and len(rows) > 1 else "td"
+        tds, c = [], 0
+        while c < len(cols):
+            if (r, c) in grid:
+                span, text = grid[(r, c)]
+                attr = f' colspan="{span}"' if span > 1 else ""
+                tds.append(f"<{tag}{attr}>{html.escape(text, quote=False)}</{tag}>")
+                c += span
+            else:
+                tds.append(f"<{tag}></{tag}>")
+                c += 1
+        out.append("<tr>" + "".join(tds) + "</tr>")
+    out.append("</table>")
+    return "\n".join(out)
+
+
+def page_blocks(
+    boxes: np.ndarray,
+    texts: list[str],
+    order: list[int],
+    cls_logits: np.ndarray,
+    same: np.ndarray,
+) -> list[dict]:
+    """The layout blocks behind ``page_markdown``, in reading order:
+    {"label" (a BLOCK_CLASSES name), "bbox" (pixels, the union of its lines),
+    "text" (its lines, newline-joined)}."""
+    out = []
+    for blk in split_blocks(order, same):
+        b = boxes[blk, :4]
+        out.append(
+            {
+                "label": BLOCK_CLASSES[int(cls_logits[blk].sum(0).argmax())],
+                "bbox": [
+                    round(float(b[:, 0].min()), 1),
+                    round(float(b[:, 1].min()), 1),
+                    round(float(b[:, 2].max()), 1),
+                    round(float(b[:, 3].max()), 1),
+                ],
+                "text": "\n".join(texts[i] for i in blk),
+            }
+        )
+    return out
+
+
+def page_markdown(
+    boxes: np.ndarray,
+    texts: list[str],
+    order: list[int],
+    cls_logits: np.ndarray,
+    same: np.ndarray,
+    keep_furniture: bool = False,
+    cells: dict | None = None,
+    tab: np.ndarray | None = None,
+    pictures: np.ndarray | None = None,
+) -> str:
+    """Markdown for one page.  boxes [N, 4], texts [N], order = reading order,
+    cls_logits [N, C], same [N, N] (logit "same block").  ``cells`` maps a line
+    to its table cells [(box, text)] when the line holds several.  ``tab``
+    [3, N, N] (same row / column / cell logits) builds tables from the heads.
+    ``pictures`` [P, 4]: picture regions, each written as ``![](picture-<k>)``
+    before the first block below it that it overlaps horizontally."""
+    parts: list[str] = []
+    part_boxes: list = []
+    for blk in split_blocks(order, same):
+        n_parts = len(parts)
+        _block_part(parts, blk, boxes, texts, cls_logits, keep_furniture, cells, tab)
+        if len(parts) > n_parts:
+            b = boxes[blk, :4]
+            part_boxes.append(
+                (b[:, 0].min(), b[:, 1].min(), b[:, 2].max(), b[:, 3].max())
+            )
+    if pictures is not None and len(pictures):
+        slots: dict[int, list[str]] = {}
+        for k, (x0, y0, x1, y1) in enumerate(np.asarray(pictures)[:, :4]):
+            cy = (y0 + y1) / 2
+            at = next(
+                (
+                    j
+                    for j, (bx0, by0, bx1, by1) in enumerate(part_boxes)
+                    if (by0 + by1) / 2 > cy and min(x1, bx1) > max(x0, bx0)
+                ),
+                len(parts),
+            )
+            slots.setdefault(at, []).append(f"![](picture-{k})")
+        merged = []
+        for j in range(len(parts) + 1):
+            merged.extend(slots.get(j, []))
+            if j < len(parts):
+                merged.append(parts[j])
+        parts = merged
+    return "\n\n".join(p for p in parts if p)
+
+
+def _block_part(parts, blk, boxes, texts, cls_logits, keep_furniture, cells, tab):
+    """Append one block's markdown to ``parts`` (nothing for dropped furniture)."""
+    cls = int(cls_logits[blk].sum(0).argmax())
+    texts_b = [texts[i] for i in blk]
+    if cls in (HEADER, FOOTER) and not keep_furniture:
+        return
+    if all(_WHOLE_MATH.match(t.strip()) for t in texts_b):  # display equations
+        parts.append("\n".join("$$" + t.strip()[1:-1] + "$$" for t in texts_b))
+        return
+    if cls == SECTION:
+        parts.append("## " + join_lines(texts_b))
+    elif (
+        cls == TABLE
+        and len(blk) > 1
+        and tab is not None
+        and heads_plausible(boxes[blk, :4], tab[:, blk][:, :, blk])
+    ):
+        parts.append(table_html_heads(boxes[blk, :4], texts_b, tab[:, blk][:, :, blk]))
+    elif cls == TABLE and len(blk) > 1:
+        cb, ct = [], []
+        for i in blk:
+            for bx, tx in (cells or {}).get(i) or [(boxes[i, :4], texts[i])]:
+                cb.append(bx)
+                ct.append(tx)
+        parts.append(table_html(np.array(cb, np.float32).reshape(-1, 4), ct))
+    elif cls == LIST:
+        items: list[list[str]] = []
+        for t in texts_b:
+            if not items or _ITEM.match(t):
+                items.append([t])
+            else:
+                items[-1].append(t)
+        parts.append("\n".join(join_lines(it) for it in items))
+    else:
+        parts.append(join_lines(texts_b))

stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/line_mask.py

@@ -0,0 +1,153 @@
+"""Mask a line window with CenterMask before the recognizer crops the line.
+
+One function for training and inference: the recognizer must only ever see
+crops that went through the same exported mask model (mask.onnx).
+
+    sess = load_mask("mask.onnx")
+    win, box_in_win = masked_window(gray, box, sess)
+    img = crop_line(win, box_in_win, height)
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+WINDOW_H = 96  # window height in pixels; the centre line is about a third of it
+WINDOW_ABOVE = 1.0  # box heights above and below the line box
+WINDOW_SIDE = 0.5  # box heights left and right
+
+
+def window_box(box, w_page: float, h_page: float) -> tuple[float, float, float, float]:
+    """The window around a line box (page pixels), clipped to the page."""
+    x0, y0, x1, y1 = (float(v) for v in box[:4])
+    h = max(1.0, y1 - y0)
+    return (
+        max(0.0, x0 - WINDOW_SIDE * h),
+        max(0.0, y0 - WINDOW_ABOVE * h),
+        min(w_page, x1 + WINDOW_SIDE * h),
+        min(h_page, y1 + WINDOW_ABOVE * h),
+    )
+
+
+KEEP = 0.5  # pixels with a lower centre-line probability become paper
+
+
+def resize_window(a: np.ndarray, size: tuple[int, int]) -> np.ndarray:
+    """Scale a window to ``size`` (w, h): linear down to half scale, area below
+    (anti-aliasing).  INTER_AREA at a non-integer scale costs 20x more."""
+    import cv2
+
+    shrink = size[1] / max(1, a.shape[0])
+    interp = cv2.INTER_LINEAR if shrink >= 0.5 else cv2.INTER_AREA
+    return cv2.resize(a, size, interpolation=interp)
+
+
+def load_mask(path: str, threads: int = 1):
+    import onnxruntime as ort
+
+    so = ort.SessionOptions()
+    so.intra_op_num_threads = threads
+    so.inter_op_num_threads = 1
+    return ort.InferenceSession(str(path), so, providers=["CPUExecutionProvider"])
+
+
+def _window(gray: np.ndarray, box) -> tuple[np.ndarray, tuple[float, float]]:
+    H, W = gray.shape
+    x0, y0, x1, y1 = (int(round(v)) for v in window_box(box, W, H))
+    return gray[y0:y1, x0:x1], (x0, y0)
+
+
+def mask_windows(sess, wins: list[np.ndarray]) -> list[np.ndarray]:
+    """Centre-line keep masks (bool, window resolution) for grayscale windows.
+
+    The probabilities are scaled back bilinearly before the threshold: a
+    threshold at mask resolution with a nearest-neighbour upscale breaks the
+    stroke edges (surya OCR 2 CER on M804 lines 0.30 -> 0.36)."""
+    import cv2
+
+    if not wins:
+        return []
+    xs, sizes = [], []
+    for w in wins:
+        h, wd = w.shape
+        W = max(32, int(round(wd * WINDOW_H / max(h, 1) / 8)) * 8)
+        ink = 1.0 - resize_window(w, (W, WINDOW_H)).astype(np.float32) / 255.0
+        xs.append(ink)
+        sizes.append((wd, h))
+    out: list = [None] * len(wins)
+    order = np.argsort([x.shape[1] for x in xs])
+    for k in range(0, len(order), 16):  # similar widths together: less padding
+        idx = order[k : k + 16]
+        Wm = max(xs[i].shape[1] for i in idx)
+        batch = np.zeros((len(idx), 1, WINDOW_H, Wm), np.float32)
+        for j, i in enumerate(idx):
+            batch[j, 0, :, : xs[i].shape[1]] = xs[i]
+        p = sess.run(None, {"x": batch})[0]
+        for j, i in enumerate(idx):
+            pm = p[j, 0, :, : xs[i].shape[1]]
+            out[i] = cv2.resize(pm, sizes[i], interpolation=cv2.INTER_LINEAR) >= KEEP
+    return out
+
+
+def masked_windows(
+    gray: np.ndarray, boxes, sess
+) -> list[tuple[np.ndarray, list[float]]]:
+    """For each line box: (window with foreign ink whitened, the box inside it)."""
+    wins, offs = zip(*[_window(gray, b) for b in boxes]) if len(boxes) else ((), ())
+    out = []
+    for w, (ox, oy), keep, b in zip(wins, offs, mask_windows(sess, list(wins)), boxes):
+        m = w.copy()
+        # paper level: 90th percentile grey on a 1/16 subsample of the window
+        m[~keep] = np.uint8(np.percentile(w[::4, ::4], 90)) if w.size else 255
+        out.append((m, [b[0] - ox, b[1] - oy, b[2] - ox, b[3] - oy]))
+    return out
+
+
+def masked_window(gray: np.ndarray, box, sess) -> tuple[np.ndarray, list[float]]:
+    return masked_windows(gray, [box], sess)[0]
+
+
+def crop_lines(
+    gray: np.ndarray,
+    boxes,
+    height: int,
+    sess=None,
+    window: int = 1,
+    pool=None,
+    dark: list | None = None,
+) -> list[np.ndarray]:
+    """Recognizer inputs for the line boxes of one page: ``crop_line`` of each
+    line, after CenterMask when ``sess`` is given (one batched call per page).
+    A tall window reads the page around the line, so it is never masked.
+    pool: a thread pool for the crops; dark: ``line_darkness`` of the boxes,
+    when already known (not used with CenterMask)."""
+    from .rec_preproc import crop_line
+
+    run = pool.map if pool is not None else map
+    if sess is None or window > 1:
+        d = dark if dark is not None else [None] * len(boxes)
+        return list(
+            run(lambda bd: crop_line(gray, bd[0], height, window=window, dark=bd[1]),
+                list(zip(boxes, d)))
+        )
+    return [crop_line(w, b, height) for w, b in masked_windows(gray, list(boxes), sess)]
+
+
+def line_darkness(gray: np.ndarray, boxes, pool=None) -> list[bool]:
+    """``is_dark_line`` of each line box (light text on a dark ground)."""
+    from .rec_preproc import is_dark_line
+
+    run = pool.map if pool is not None else map
+    return list(run(lambda b: is_dark_line(gray, b), list(boxes)))
+
+
+_SESSIONS: dict = {}
+
+
+def session_for(path: str | None):
+    """One CPU session per process (dataloader workers call this lazily)."""
+    if not path:
+        return None
+    if path not in _SESSIONS:
+        _SESSIONS[path] = load_mask(path, threads=1)
+    return _SESSIONS[path]

stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/math.py

@@ -0,0 +1,145 @@
+"""Math lines: a classifier finds lines that hold math, and MathRec reads them as
+text with inline LaTeX ("we have $x _ { i } \\leq 1$").
+
+The CTC recognizer reads one symbol per column, so it cannot write a fraction or
+a sum with limits.  MathRec reads a taller crop of the line with an attention
+decoder, one token per step, with the decoder's keys and values cached.
+
+Models (all optional; the pipeline skips math without them): mathcls.onnx,
+math_enc.onnx, math_dec_init.onnx, math_dec_step.onnx and math_meta.json.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+
+from .line_mask import crop_lines
+from .rec_preproc import pad_batch, width_batches
+
+MATH_MIN = 0.5  # classifier probability above which a line is read as math
+# A math reading replaces a line only at this mean token probability or more: on
+# print the model has not seen (old scans) it writes confident-looking LaTeX
+# that is wrong, at low probability.
+MATH_MIN_CONF = 0.9
+# A line whose mean token probability is this far below MATH_MIN_CONF after 8
+# tokens stops decoding: it would be rejected anyway (garbage on unseen print).
+EARLY_STOP_MARGIN = 0.1
+BOS, EOS, N_SPECIAL = 1, 2, 4
+FILES = ("mathcls.onnx", "math_enc.onnx", "math_dec_init.onnx", "math_dec_step.onnx")
+
+
+def detokenize(tokens: list[str]) -> str:
+    """Tokens -> text; inside math (between "$" tokens) they are joined by spaces."""
+    out, math, in_math = [], [], False
+    for t in tokens:
+        if t == "$":
+            if in_math:
+                out.append("$" + " ".join(math) + "$")
+                math = []
+            in_math = not in_math
+        elif in_math:
+            math.append(t)
+        else:
+            out.append(t)
+    if in_math and math:
+        out.append("$" + " ".join(math) + "$")
+    return " ".join("".join(out).split())
+
+
+class MathReader:
+    def __init__(self, onnx_dir: Path, so, providers):
+        import onnxruntime as ort
+
+        s = {
+            f: ort.InferenceSession(str(onnx_dir / f), so, providers=providers)
+            for f in FILES
+        }
+        self.cls, self.enc = s["mathcls.onnx"], s["math_enc.onnx"]
+        self.init, self.step = s["math_dec_init.onnx"], s["math_dec_step.onnx"]
+        meta = json.loads((onnx_dir / "math_meta.json").read_text())
+        cfg = meta["config"]
+        self.height = cfg["height"]
+        self.max_tokens = cfg["max_tokens"]
+        self.layers, self.heads = cfg["dec_layers"], cfg["dec_heads"]
+        self.head_dim = cfg["channels"] // cfg["dec_heads"]
+        self.itos = ["", "", "", ""] + meta["tokens"]
+        # Exports drop unused inputs (the init graph does not read the mask).
+        self.inputs = {f: {i.name for i in v.get_inputs()} for f, v in s.items()}
+
+    @classmethod
+    def load(cls, onnx_dir: Path, so, providers) -> "MathReader | None":
+        names = (*FILES, "math_meta.json")
+        if all((onnx_dir / f).exists() for f in names):
+            return cls(onnx_dir, so, providers)
+        return None
+
+    def flags(self, crops: list, batch: int = 64) -> list[bool]:
+        """Per line crop (the recognizer's crop): does it hold math?"""
+        out = [False] * len(crops)
+        for idx in width_batches([c.shape[1] for c in crops], batch):
+            arr, widths = pad_batch([crops[j] for j in idx])
+            p = self.cls.run(None, {"x": arr, "widths": widths})[0]
+            for k, j in enumerate(idx):
+                out[j] = bool(p[k] > MATH_MIN)
+        return out
+
+    def read(
+        self, gray: np.ndarray, boxes: np.ndarray, batch: int = 64
+    ) -> list[tuple[str, float]]:
+        """(text with inline LaTeX, mean token probability) for each box."""
+        crops = crop_lines(gray, boxes[:, :4], self.height, None)
+        out = [("", 0.0)] * len(crops)
+        # Few, wide batches: each batch is one decoding loop of up to max_tokens
+        # steps, and a step costs about the same for 8 lines as for 40.
+        widths = [c.shape[1] for c in crops]
+        for idx in width_batches(widths, batch, max_ratio=4.0, max_area=16 * 64 * 640):
+            arr, widths = pad_batch([crops[j] for j in idx])
+            mem, mask = self.enc.run(None, {"x": arr, "widths": widths})
+            feed = {"memory": mem, "mask": mask}
+            names = self.inputs["math_dec_init.onnx"]
+            ck, cv = self.init.run(None, {k: v for k, v in feed.items() if k in names})
+            B = len(idx)
+            sk = np.zeros((self.layers, B, self.heads, 0, self.head_dim), np.float32)
+            sv = sk
+            tok = np.full(B, BOS, np.int64)
+            step_in = self.inputs["math_dec_step.onnx"]
+            seqs: list[list[int]] = [[] for _ in range(B)]
+            probs: list[list[float]] = [[] for _ in range(B)]
+            done = np.zeros(B, bool)
+            for t in range(self.max_tokens - 1):
+                feed = {
+                    "token": tok,
+                    "pos": np.array(t, np.int64),
+                    "self_k": sk,
+                    "self_v": sv,
+                    "cross_k": ck,
+                    "cross_v": cv,
+                    "mask": mask,
+                }
+                logits, sk, sv = self.step.run(
+                    None, {k: v for k, v in feed.items() if k in step_in}
+                )
+                tok = logits.argmax(-1).astype(np.int64)
+                e = np.exp(logits - logits.max(-1, keepdims=True))
+                top = 1.0 / e.sum(-1)  # softmax probability of the argmax token
+                for b in range(B):
+                    if not done[b]:
+                        probs[b].append(float(top[b]))
+                        if tok[b] == EOS:
+                            done[b] = True
+                        elif (
+                            len(probs[b]) >= 8
+                            and np.mean(probs[b]) < MATH_MIN_CONF - EARLY_STOP_MARGIN
+                        ):
+                            done[b] = True  # rejected: keeps a low mean probability
+                        elif tok[b] >= N_SPECIAL:
+                            seqs[b].append(int(tok[b]))
+                if done.all():
+                    break
+            for k, j in enumerate(idx):
+                text = detokenize([self.itos[i] for i in seqs[k]])
+                out[j] = (text, float(np.mean(probs[k])) if probs[k] else 0.0)
+        return out

stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/models.py

@@ -0,0 +1,75 @@
+"""Model files: a local folder, or a download from the Hugging Face repo."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+HF_REPO = "datalab-to/indra-ocr"
+# Model set this package version was built and tested against (a tag in HF_REPO).
+HF_REVISION = "v1.1.0"
+FILES = ("det.onnx", "rec.onnx", "order.onnx", "meta.json")
+
+
+class ModelError(OSError):
+    """The models are not available: a download failed (no network, no access)
+    or a local model folder lacks files."""
+
+
+MATH_FILES = (
+    "mathcls.onnx",
+    "math_enc.onnx",
+    "math_dec_init.onnx",
+    "math_dec_step.onnx",
+    "math_meta.json",
+)
+
+
+def model_dir(
+    path: str | Path | None = None, revision: str | None = None, math: bool = False
+) -> Path:
+    """Folder with det.onnx, rec.onnx, meta.json and usually order.onnx (and,
+    with ``math``, the math models, which are downloaded only then).  A local
+    folder without order.onnx works: reading order then falls back to
+    XY-cut, with no layout labels.
+
+    Order of lookup: ``path``, the INDRA_OCR_MODEL_DIR environment variable, then
+    a download of ``HF_REPO``.  Downloads are cached by huggingface_hub."""
+    path = path or os.environ.get("INDRA_OCR_MODEL_DIR")
+    if path:
+        p = Path(path).expanduser()
+        missing = [f for f in FILES[:2] + FILES[3:] if not (p / f).exists()]
+        if math:
+            missing += [f for f in MATH_FILES if not (p / f).exists()]
+        if missing:
+            raise ModelError(f"the model folder {p} lacks {', '.join(missing)}")
+        return p
+    from huggingface_hub import snapshot_download
+
+    rev = revision or HF_REVISION
+    kw = dict(
+        revision=rev,
+        allow_patterns=[
+            "*.onnx",
+            "meta.json",
+            "lm.npz",
+            "lm_*.npz",
+            *(MATH_FILES if math else ()),
+        ],
+        ignore_patterns=[] if math else list(MATH_FILES),
+    )
+    try:
+        # A tag's files never change: a cached copy needs no network (and no
+        # progress bar on every run).
+        return Path(snapshot_download(HF_REPO, local_files_only=True, **kw))
+    except Exception:  # noqa: BLE001  (not cached yet)
+        pass
+    try:
+        return Path(snapshot_download(HF_REPO, **kw))
+    except Exception as e:  # noqa: BLE001  (network, access, disk: one clear message)
+        reason = (str(e).strip().splitlines() or [type(e).__name__])[0]
+        raise ModelError(
+            f"could not download the models from Hugging Face ({HF_REPO}, {rev}): "
+            f"{type(e).__name__}: {reason[:200]}. Check the network connection, or "
+            "point at a local model folder with --model-dir or INDRA_OCR_MODEL_DIR."
+        ) from e