diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/MANIFEST.json b/MANIFEST.json new file mode 100644 index 0000000000000000000000000000000000000000..5328774c6517e20c2211408ae616d83f611fd5e9 --- /dev/null +++ b/MANIFEST.json @@ -0,0 +1,653 @@ +{ + "files": [ + { + "path": "stallion/bina04-assembly-audit-95k-20261104/graphic_blocks.py", + "source": "/home/rezo/bina04-assembly-audit-95k-20261104/graphic_blocks.py", + "bytes": 3780 + }, + { + "path": "stallion/bina04-assembly-audit-95k-20261104/production.py", + "source": "/home/rezo/bina04-assembly-audit-95k-20261104/production.py", + "bytes": 616 + }, + { + "path": "stallion/bina04-block-fields-95k-20261103/assembly_fix.py", + "source": "/home/rezo/bina04-block-fields-95k-20261103/assembly_fix.py", + "bytes": 4466 + }, + { + "path": "stallion/bina04-block-fields-95k-20261103/production.py", + "source": "/home/rezo/bina04-block-fields-95k-20261103/production.py", + "bytes": 872 + }, + { + "path": "stallion/bina04-cell-rowcores-95k-20261103/candidate.py", + "source": "/home/rezo/bina04-cell-rowcores-95k-20261103/candidate.py", + "bytes": 2858 + }, + { + "path": "stallion/bina04-cell-rowcores-95k-20261103/production.py", + "source": "/home/rezo/bina04-cell-rowcores-95k-20261103/production.py", + "bytes": 1229 + }, + { + "path": "stallion/bina04-column-assembly-95k-20261103/column_order.py", + "source": "/home/rezo/bina04-column-assembly-95k-20261103/column_order.py", + "bytes": 2812 + }, + { + "path": "stallion/bina04-column-assembly-95k-20261103/production.py", + "source": "/home/rezo/bina04-column-assembly-95k-20261103/production.py", + "bytes": 665 + }, + { + "path": "stallion/bina04-crop-bandseg-95k-20261102/bandseg.py", + "source": "/home/rezo/bina04-crop-bandseg-95k-20261102/bandseg.py", + "bytes": 14573 + }, + { + "path": "stallion/bina04-crop-bandseg-95k-20261102/consumer_runtime.py", + "source": "/home/rezo/bina04-crop-bandseg-95k-20261102/consumer_runtime.py", + "bytes": 2357 + }, + { + "path": "stallion/bina04-crop-picture-cells-95k-20261102/CALIBRATION.json", + "source": "/home/rezo/bina04-crop-picture-cells-95k-20261102/CALIBRATION.json", + "bytes": 14990 + }, + { + "path": "stallion/bina04-crop-picture-cells-95k-20261102/candidate.py", + "source": "/home/rezo/bina04-crop-picture-cells-95k-20261102/candidate.py", + "bytes": 3443 + }, + { + "path": "stallion/bina04-crop-picture-cells-95k-20261102/cell_reader.py", + "source": "/home/rezo/bina04-crop-picture-cells-95k-20261102/cell_reader.py", + "bytes": 5178 + }, + { + "path": "stallion/bina04-crop-picture-cells-95k-20261102/junk_filter.py", + "source": "/home/rezo/bina04-crop-picture-cells-95k-20261102/junk_filter.py", + "bytes": 3251 + }, + { + "path": "stallion/bina04-crop-picture-cells-95k-20261102/pen-classifier.joblib", + "source": "/home/rezo/bina04-crop-picture-cells-95k-20261102/pen-classifier.joblib", + "bytes": 13905 + }, + { + "path": "stallion/bina04-crop-picture-cells-95k-20261102/pen_classifier.py", + "source": "/home/rezo/bina04-crop-picture-cells-95k-20261102/pen_classifier.py", + "bytes": 1065 + }, + { + "path": "stallion/bina04-detector-role-isolation-95k-20261103/production.py", + "source": "/home/rezo/bina04-detector-role-isolation-95k-20261103/production.py", + "bytes": 1676 + }, + { + "path": "stallion/bina04-document-handoff-95k-20261103/document.py", + "source": "/home/rezo/bina04-document-handoff-95k-20261103/document.py", + "bytes": 5647 + }, + { + "path": "stallion/bina04-document-handoff-95k-20261103/numeric_repairs.py", + "source": "/home/rezo/bina04-document-handoff-95k-20261103/numeric_repairs.py", + "bytes": 7574 + }, + { + "path": "stallion/bina04-document-handoff-95k-20261103/production.py", + "source": "/home/rezo/bina04-document-handoff-95k-20261103/production.py", + "bytes": 2693 + }, + { + "path": "stallion/bina04-document-handoff-95k-20261103/region_reads.py", + "source": "/home/rezo/bina04-document-handoff-95k-20261103/region_reads.py", + "bytes": 3952 + }, + { + "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/learned_backend.py", + "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/learned_backend.py", + "bytes": 1737 + }, + { + "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/run/checkpoints/step-001000.pt", + "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/run/checkpoints/step-001000.pt", + "bytes": 153922431 + }, + { + "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/crop_line_guard.py", + "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/crop_line_guard.py", + "bytes": 2397 + }, + { + "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/input_policy_v3.py", + "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/input_policy_v3.py", + "bytes": 5112 + }, + { + "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/pipeline_v3.py", + "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/pipeline_v3.py", + "bytes": 18321 + }, + { + "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/table_client.py", + "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/table_client.py", + "bytes": 1440 + }, + { + "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/line_contract_reader.py", + "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/line_contract_reader.py", + "bytes": 5263 + }, + { + "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/pipeline_crop_fix.py", + "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/pipeline_crop_fix.py", + "bytes": 6508 + }, + { + "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/table_cells.py", + "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/table_cells.py", + "bytes": 7008 + }, + { + "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_builder.py", + "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_builder.py", + "bytes": 10084 + }, + { + "path": "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_pipeline.py", + "source": "/home/rezo/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_pipeline.py", + "bytes": 4315 + }, + { + "path": "stallion/bina04-header-marks-95k-20261103/CALIBRATION.json", + "source": "/home/rezo/bina04-header-marks-95k-20261103/CALIBRATION.json", + "bytes": 15542 + }, + { + "path": "stallion/bina04-header-marks-95k-20261103/candidate.py", + "source": "/home/rezo/bina04-header-marks-95k-20261103/candidate.py", + "bytes": 6479 + }, + { + "path": "stallion/bina04-header-marks-95k-20261103/production.py", + "source": "/home/rezo/bina04-header-marks-95k-20261103/production.py", + "bytes": 961 + }, + { + "path": "stallion/bina04-header-marks-95k-20261103/serializer.py", + "source": "/home/rezo/bina04-header-marks-95k-20261103/serializer.py", + "bytes": 7021 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/models/det.onnx", + "source": "/home/rezo/bina04-indra-pilot-20261027/models/det.onnx", + "bytes": 389948 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/__init__.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/__init__.py", + "bytes": 1263 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/det_post.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/det_post.py", + "bytes": 6030 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/document.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/document.py", + "bytes": 18586 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/layout_post.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/layout_post.py", + "bytes": 14411 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/line_mask.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/line_mask.py", + "bytes": 5600 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/math.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/math.py", + "bytes": 6338 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/models.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/models.py", + "bytes": 2702 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/order_post.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/order_post.py", + "bytes": 12544 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/parallel.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/parallel.py", + "bytes": 7766 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/pipeline.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/pipeline.py", + "bytes": 45649 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/rec_preproc.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/rec_preproc.py", + "bytes": 5500 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/rtl.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/rtl.py", + "bytes": 6097 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/textlayer.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/textlayer.py", + "bytes": 14861 + }, + { + "path": "stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/vocab.py", + "source": "/home/rezo/bina04-indra-pilot-20261027/upstream/indra_ocr/vocab.py", + "bytes": 11395 + }, + { + "path": "stallion/bina04-ink-loss-guard-95k-20261104/production.py", + "source": "/home/rezo/bina04-ink-loss-guard-95k-20261104/production.py", + "bytes": 1590 + }, + { + "path": "stallion/bina04-line-detector-v2-20261030/detector-run/checkpoints/step-000500.pt", + "source": "/home/rezo/bina04-line-detector-v2-20261030/detector-run/checkpoints/step-000500.pt", + "bytes": 153922431 + }, + { + "path": "stallion/bina04-line-geometry-95k-20261103/assembly.py", + "source": "/home/rezo/bina04-line-geometry-95k-20261103/assembly.py", + "bytes": 2786 + }, + { + "path": "stallion/bina04-line-geometry-95k-20261103/form_rows.py", + "source": "/home/rezo/bina04-line-geometry-95k-20261103/form_rows.py", + "bytes": 2236 + }, + { + "path": "stallion/bina04-line-geometry-95k-20261103/geometry_fix.py", + "source": "/home/rezo/bina04-line-geometry-95k-20261103/geometry_fix.py", + "bytes": 5655 + }, + { + "path": "stallion/bina04-line-geometry-95k-20261103/production.py", + "source": "/home/rezo/bina04-line-geometry-95k-20261103/production.py", + "bytes": 1241 + }, + { + "path": "stallion/bina04-numeric-math-recognizer-95k-20261102/evaluate.py", + "source": "/home/rezo/bina04-numeric-math-recognizer-95k-20261102/evaluate.py", + "bytes": 15324 + }, + { + "path": "stallion/bina04-numeric-math-recognizer-95k-20261102/recognizer/selected.pt", + "source": "/home/rezo/bina04-numeric-math-recognizer-95k-20261102/recognizer/selected.pt", + "bytes": 131255668 + }, + { + "path": "stallion/bina04-numeric-routing-95k-20261103/consumer.py", + "source": "/home/rezo/bina04-numeric-routing-95k-20261103/consumer.py", + "bytes": 4335 + }, + { + "path": "stallion/bina04-numeric-routing-95k-20261103/formula.py", + "source": "/home/rezo/bina04-numeric-routing-95k-20261103/formula.py", + "bytes": 4391 + }, + { + "path": "stallion/bina04-numeric-routing-95k-20261103/formula_private.py", + "source": "/home/rezo/bina04-numeric-routing-95k-20261103/formula_private.py", + "bytes": 5993 + }, + { + "path": "stallion/bina04-numeric-routing-95k-20261103/route.py", + "source": "/home/rezo/bina04-numeric-routing-95k-20261103/route.py", + "bytes": 7778 + }, + { + "path": "stallion/bina04-order-application-debug-20261102/fixed_order.py", + "source": "/home/rezo/bina04-order-application-debug-20261102/fixed_order.py", + "bytes": 5790 + }, + { + "path": "stallion/bina04-order-form-batch-20261101/recognizer/weights-001000.pt", + "source": "/home/rezo/bina04-order-form-batch-20261101/recognizer/weights-001000.pt", + "bytes": 131256958 + }, + { + "path": "stallion/bina04-order-labels-current-95k-20261102/CALIBRATION.json", + "source": "/home/rezo/bina04-order-labels-current-95k-20261102/CALIBRATION.json", + "bytes": 187 + }, + { + "path": "stallion/bina04-order-labels-current-95k-20261102/order_only.py", + "source": "/home/rezo/bina04-order-labels-current-95k-20261102/order_only.py", + "bytes": 1888 + }, + { + "path": "stallion/bina04-order-labels-current-95k-20261102/production.py", + "source": "/home/rezo/bina04-order-labels-current-95k-20261102/production.py", + "bytes": 1740 + }, + { + "path": "stallion/bina04-order-predicted-20261101/order_nodes.py", + "source": "/home/rezo/bina04-order-predicted-20261101/order_nodes.py", + "bytes": 4931 + }, + { + "path": "stallion/bina04-order-predicted-20261101/pred_order_model.py", + "source": "/home/rezo/bina04-order-predicted-20261101/pred_order_model.py", + "bytes": 2771 + }, + { + "path": "stallion/bina04-order-salvage-20261101/evaluate_batch.py", + "source": "/home/rezo/bina04-order-salvage-20261101/evaluate_batch.py", + "bytes": 8459 + }, + { + "path": "stallion/bina04-order-salvage-20261101/row_builder.py", + "source": "/home/rezo/bina04-order-salvage-20261101/row_builder.py", + "bytes": 10084 + }, + { + "path": "stallion/bina04-page-assembly-95k-20261103/CALIBRATION.json", + "source": "/home/rezo/bina04-page-assembly-95k-20261103/CALIBRATION.json", + "bytes": 649 + }, + { + "path": "stallion/bina04-page-assembly-95k-20261103/assembly.py", + "source": "/home/rezo/bina04-page-assembly-95k-20261103/assembly.py", + "bytes": 2809 + }, + { + "path": "stallion/bina04-page-assembly-95k-20261103/production.py", + "source": "/home/rezo/bina04-page-assembly-95k-20261103/production.py", + "bytes": 905 + }, + { + "path": "stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json", + "source": "/home/rezo/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json", + "bytes": 20007 + }, + { + "path": "stallion/bina04-pagepath-v3-round3-clean-20261031/fusion_v3.py", + "source": "/home/rezo/bina04-pagepath-v3-round3-clean-20261031/fusion_v3.py", + "bytes": 4521 + }, + { + "path": "stallion/bina04-pagepath-v3-round3-clean-20261031/input_policy_v3.py", + "source": "/home/rezo/bina04-pagepath-v3-round3-clean-20261031/input_policy_v3.py", + "bytes": 5112 + }, + { + "path": "stallion/bina04-pagepath-v3-round3-clean-20261031/pipeline_v3.py", + "source": "/home/rezo/bina04-pagepath-v3-round3-clean-20261031/pipeline_v3.py", + "bytes": 18321 + }, + { + "path": "stallion/bina04-pagepath-v3-round3-clean-20261031/run/weights-095000.pt", + "source": "/home/rezo/bina04-pagepath-v3-round3-clean-20261031/run/weights-095000.pt", + "bytes": 131256958 + }, + { + "path": "stallion/bina04-pagepath-v3-round3-clean-20261031/text_contract.py", + "source": "/home/rezo/bina04-pagepath-v3-round3-clean-20261031/text_contract.py", + "bytes": 1030 + }, + { + "path": "stallion/bina04-print-textgate-20261102/CALIBRATION.json", + "source": "/home/rezo/bina04-print-textgate-20261102/CALIBRATION.json", + "bytes": 2509 + }, + { + "path": "stallion/bina04-print-textgate-20261102/include-model.pkl", + "source": "/home/rezo/bina04-print-textgate-20261102/include-model.pkl", + "bytes": 7982847 + }, + { + "path": "stallion/bina04-print-textgate-20261102/include_model.py", + "source": "/home/rezo/bina04-print-textgate-20261102/include_model.py", + "bytes": 1878 + }, + { + "path": "stallion/bina04-print-textgate-20261102/layer_runtime.py", + "source": "/home/rezo/bina04-print-textgate-20261102/layer_runtime.py", + "bytes": 4969 + }, + { + "path": "stallion/bina04-print-textgate-20261102/order_adapter.py", + "source": "/home/rezo/bina04-print-textgate-20261102/order_adapter.py", + "bytes": 940 + }, + { + "path": "stallion/bina04-print-textgate-20261102/production.py", + "source": "/home/rezo/bina04-print-textgate-20261102/production.py", + "bytes": 1372 + }, + { + "path": "stallion/bina04-print-textgate-20261102/run/best.json", + "source": "/home/rezo/bina04-print-textgate-20261102/run/best.json", + "bytes": 2157 + }, + { + "path": "stallion/bina04-print-textgate-20261102/run/selected-printed-recognizer.pt", + "source": "/home/rezo/bina04-print-textgate-20261102/run/selected-printed-recognizer.pt", + "bytes": 131256937 + }, + { + "path": "stallion/bina04-printed-form-fix-20261102/CALIBRATION.json", + "source": "/home/rezo/bina04-printed-form-fix-20261102/CALIBRATION.json", + "bytes": 1591 + }, + { + "path": "stallion/bina04-printed-form-fix-20261102/SERIALIZED-LEXICON.json", + "source": "/home/rezo/bina04-printed-form-fix-20261102/SERIALIZED-LEXICON.json", + "bytes": 2644 + }, + { + "path": "stallion/bina04-printed-form-fix-20261102/candidate_apply.py", + "source": "/home/rezo/bina04-printed-form-fix-20261102/candidate_apply.py", + "bytes": 4498 + }, + { + "path": "stallion/bina04-printed-form-fix-20261102/label_model.py", + "source": "/home/rezo/bina04-printed-form-fix-20261102/label_model.py", + "bytes": 1684 + }, + { + "path": "stallion/bina04-printed-form-fix-20261102/printed-label-model.pkl", + "source": "/home/rezo/bina04-printed-form-fix-20261102/printed-label-model.pkl", + "bytes": 12109247 + }, + { + "path": "stallion/bina04-printed-form-fix-20261102/production.py", + "source": "/home/rezo/bina04-printed-form-fix-20261102/production.py", + "bytes": 4248 + }, + { + "path": "stallion/bina04-printed-layer-training-20261102/run/weights-002000.pt", + "source": "/home/rezo/bina04-printed-layer-training-20261102/run/weights-002000.pt", + "bytes": 132814787 + }, + { + "path": "stallion/bina04-quality-route-printscope-95k-20261103/QUALITY-POLICY.json", + "source": "/home/rezo/bina04-quality-route-printscope-95k-20261103/QUALITY-POLICY.json", + "bytes": 1590 + }, + { + "path": "stallion/bina04-quality-route-printscope-95k-20261103/print_scope.py", + "source": "/home/rezo/bina04-quality-route-printscope-95k-20261103/print_scope.py", + "bytes": 1087 + }, + { + "path": "stallion/bina04-quality-route-printscope-95k-20261103/production.py", + "source": "/home/rezo/bina04-quality-route-printscope-95k-20261103/production.py", + "bytes": 3428 + }, + { + "path": "stallion/bina04-quality-route-printscope-95k-20261103/quality.py", + "source": "/home/rezo/bina04-quality-route-printscope-95k-20261103/quality.py", + "bytes": 1947 + }, + { + "path": "stallion/bina04-real-degraded-95k-20261103/recognizer/selected.pt", + "source": "/home/rezo/bina04-real-degraded-95k-20261103/recognizer/selected.pt", + "bytes": 131254078 + }, + { + "path": "stallion/bina04-small-crossing-rows-20261102/crossing_rows.py", + "source": "/home/rezo/bina04-small-crossing-rows-20261102/crossing_rows.py", + "bytes": 4043 + }, + { + "path": "stallion/bina04-small-crossing-rows-20261102/runtime.py", + "source": "/home/rezo/bina04-small-crossing-rows-20261102/runtime.py", + "bytes": 2983 + }, + { + "path": "stallion/bina04-small-crossing-rows-20261102/runtime_release.py", + "source": "/home/rezo/bina04-small-crossing-rows-20261102/runtime_release.py", + "bytes": 1162 + }, + { + "path": "stallion/bina04-small-crossing-rows-20261102/scale_utils.py", + "source": "/home/rezo/bina04-small-crossing-rows-20261102/scale_utils.py", + "bytes": 1741 + }, + { + "path": "stallion/bina04-smallmarks-celllabels-95k-20261102/decompose.py", + "source": "/home/rezo/bina04-smallmarks-celllabels-95k-20261102/decompose.py", + "bytes": 903 + }, + { + "path": "stallion/bina04-surya25-order-20261101/role-model/weights-003000.pt", + "source": "/home/rezo/bina04-surya25-order-20261101/role-model/weights-003000.pt", + "bytes": 5093599 + }, + { + "path": "stallion/bina04-surya25-order-20261101/role_model.py", + "source": "/home/rezo/bina04-surya25-order-20261101/role_model.py", + "bytes": 4264 + }, + { + "path": "stallion/bina04-surya25-order-20261101/role_nodes.py", + "source": "/home/rezo/bina04-surya25-order-20261101/role_nodes.py", + "bytes": 1697 + }, + { + "path": "stallion/bina04-table-asides-95k-20261104/production.py", + "source": "/home/rezo/bina04-table-asides-95k-20261104/production.py", + "bytes": 616 + }, + { + "path": "stallion/bina04-table-asides-95k-20261104/table_asides.py", + "source": "/home/rezo/bina04-table-asides-95k-20261104/table_asides.py", + "bytes": 3046 + }, + { + "path": "stallion/indra-scale/geometry.py", + "source": "/tmp/bina04-indra-scale-20261029/geometry.py", + "bytes": 1935 + }, + { + "path": "stallion/indra-scale/gpu_eval_runtime.py", + "source": "/tmp/bina04-indra-scale-20261029/gpu_eval_runtime.py", + "bytes": 2267 + }, + { + "path": "stallion/indra-scale/recognizer.py", + "source": "/tmp/bina04-indra-scale-20261029/recognizer.py", + "bytes": 3455 + }, + { + "path": "stallion/indra-scale/regional_pipeline.py", + "source": "/tmp/bina04-indra-scale-20261029/regional_pipeline.py", + "bytes": 7967 + }, + { + "path": "stallion/indra-scale/regional_pipeline_v2.py", + "source": "/tmp/bina04-indra-scale-20261029/regional_pipeline_v2.py", + "bytes": 8415 + }, + { + "path": "stallion/indra-scale/routing-rule.json", + "source": "/tmp/bina04-indra-scale-20261029/routing-rule.json", + "bytes": 17922 + }, + { + "path": "stallion/indra-scale/routing_modes.py", + "source": "/tmp/bina04-indra-scale-20261029/routing_modes.py", + "bytes": 2548 + }, + { + "path": "stallion/indra-scale/rtl_reader.py", + "source": "/tmp/bina04-indra-scale-20261029/rtl_reader.py", + "bytes": 2892 + }, + { + "path": "stallion/indra-scale/scoringTT/shenavasanj.py", + "source": "/tmp/bina04-indra-scale-20261029/scoringTT/shenavasanj.py", + "bytes": 5126 + }, + { + "path": "stallion/indra-scale/visual_token.py", + "source": "/tmp/bina04-indra-scale-20261029/visual_token.py", + "bytes": 491 + } + ], + "rewritten_text_files": [ + "stallion/bina04-assembly-audit-95k-20261104/production.py", + "stallion/bina04-block-fields-95k-20261103/production.py", + "stallion/bina04-cell-rowcores-95k-20261103/candidate.py", + "stallion/bina04-cell-rowcores-95k-20261103/production.py", + "stallion/bina04-column-assembly-95k-20261103/production.py", + "stallion/bina04-detector-role-isolation-95k-20261103/production.py", + "stallion/bina04-document-handoff-95k-20261103/production.py", + "stallion/bina04-document-handoff-95k-20261103/region_reads.py", + "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/crop_line_guard.py", + "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/pipeline_v3.py", + "stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/line_contract_reader.py", + "stallion/bina04-header-marks-95k-20261103/candidate.py", + "stallion/bina04-header-marks-95k-20261103/production.py", + "stallion/bina04-header-marks-95k-20261103/serializer.py", + "stallion/bina04-ink-loss-guard-95k-20261104/production.py", + "stallion/bina04-line-geometry-95k-20261103/production.py", + "stallion/bina04-numeric-math-recognizer-95k-20261102/evaluate.py", + "stallion/bina04-numeric-routing-95k-20261103/consumer.py", + "stallion/bina04-numeric-routing-95k-20261103/formula_private.py", + "stallion/bina04-order-application-debug-20261102/fixed_order.py", + "stallion/bina04-order-labels-current-95k-20261102/order_only.py", + "stallion/bina04-order-labels-current-95k-20261102/production.py", + "stallion/bina04-order-salvage-20261101/evaluate_batch.py", + "stallion/bina04-page-assembly-95k-20261103/production.py", + "stallion/bina04-pagepath-v3-round3-clean-20261031/pipeline_v3.py", + "stallion/bina04-pagepath-v3-round3-clean-20261031/text_contract.py", + "stallion/bina04-print-textgate-20261102/include_model.py", + "stallion/bina04-print-textgate-20261102/layer_runtime.py", + "stallion/bina04-print-textgate-20261102/order_adapter.py", + "stallion/bina04-print-textgate-20261102/production.py", + "stallion/bina04-print-textgate-20261102/run/best.json", + "stallion/bina04-printed-form-fix-20261102/CALIBRATION.json", + "stallion/bina04-printed-form-fix-20261102/production.py", + "stallion/bina04-quality-route-printscope-95k-20261103/print_scope.py", + "stallion/bina04-quality-route-printscope-95k-20261103/production.py", + "stallion/bina04-small-crossing-rows-20261102/runtime.py", + "stallion/bina04-table-asides-95k-20261104/production.py" + ], + "placeholder": "__BINA_ROOT__", + "layout_model": { + "repo": "datalab-to/surya_layout2", + "revision": "0aee81d5fd9275c0582e545bf3a56944b1e75679" + } +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c4e542a0e690b6d9d5f9f499d6bdc1de2b22ad02 --- /dev/null +++ b/README.md @@ -0,0 +1,111 @@ +--- +language: +- fa +license: other +license_name: bina-0.4-rizeh +pipeline_tag: image-to-text +tags: +- bina +- ocr +- persian +- farsi +- handwriting +- printed-text +- ctc +- document-understanding +datasets: +- Reza2kn/persian-handwriting-pages-3.69m +- Reza2kn/persian-printed-ocr-3.5m +--- + +# Bina 0.4 Rizeh + +Bina 0.4 Rizeh is the small, fast member of the Bina 0.4 Persian OCR family. It reads whole pages (printed text, handwriting, forms, screenshots, letters, newspapers and simple math). Its core is about 71M parameters: a 38.5M line detector (fine-tuned Surya text detector) and a 32.8M CTC line recognizer distilled from [Bina 0.4 Koochik](https://huggingface.co/Reza2kn/Bina-0.4-Koochik). They run inside a page pipeline: + +1. **Layout**: Surya layout (`datalab-to/surya_layout2`, pinned revision, downloaded on first run). +2. **Lines**: fine-tuned Surya-architecture line detectors (body text, printed layer, handwriting page path). +3. **Reading**: the 95k-step Indra CTC student for body text, plus specialist readers with the same architecture: printed/form labels, short form cells, numeric and math, and degraded scans. +4. **Assembly**: row consolidation, column and table ordering, printed-label admission and page-level ordering. + +This repository is the complete production pipeline exactly as it scored below, not just a checkpoint: all weights, calibrations and code are bundled. + +## Quick run + +```bash +hf download Reza2kn/Bina-0.4-Rizeh --local-dir Bina-0.4-Rizeh +cd Bina-0.4-Rizeh +pip install -r requirements.txt # Python 3.10, NVIDIA GPU recommended +python rizeh.py examples/demo.png --out results +python rizeh.py scan1.jpg scan2.png document.pdf --out results +``` + +The first run binds the bundled paths to your checkout (it edits the bundled text files in place, and reruns automatically if you move the folder) and downloads the pinned Surya layout model. For each page you get `.txt` with the page text in reading order, and `.json` with every line's text, box, layer (`printed` / handwriting / body) and confidence. PDFs are rendered at 200 dpi. + +From Python: + +```python +from PIL import Image +from rizeh import Rizeh +ocr = Rizeh("cuda") +result = ocr(Image.open("page.jpg").convert("RGB")) +print(result["text"]) +``` + +## Scores + +Character error rate (CER, %) on the internal gates this release was frozen on. Lower is better. + +| Gate | CER | +|---|---:| +| 193 cropped regions, overall | 9.39 | +| · forms | 3.80 | +| · screenshots | 4.65 | +| · prose | 9.15 | +| · math | 12.09 | +| · newspaper | 13.49 | +| · tables | 19.63 | +| 50 handwriting pages, handwritten body only | 3.97 | +| 50 handwriting pages, full page incl. printed form text | 10.89 | +| 338 printed lines | 1.76 | +| 7 real scanned documents (printed regions) | 10.61 | +| Scanned official letter | 2.12 | + +[Persian OCR Triple Threat](https://huggingface.co/spaces/Reza2kn/PersianOCR-TripleThreat), same 95k recognizer, measured 2026-11-03 before the final assembly fixes (lower is better): + +| Split | Rows | WER | CER | S³ | Triple Threat | +|---|---:|---:|---:|---:|---:| +| Printed | 6,669 | 4.70 | 1.37 | 5.88 | 4.74 | +| Handwriting | 6,669 | 18.48 | 13.01 | 15.19 | 15.41 | +| Equal-split overall | 13,338 | 11.59 | 7.19 | 10.54 | **10.08** | + +## Limitations + +- **Historical manuscripts do not work.** On HPHD (historical nastaliq and shekasteh manuscripts) the recognizer is at 67% CER even on oracle line crops, and full pages come out as fluent-looking nonsense. Do not use it for manuscripts. +- **Tables are weak** (about 20% CER), especially dense or low-resolution tables with handwriting over the cells. +- **Reading order** on busy layouts (multi-column newspapers, exam sheets with many labels) is the largest remaining error source; the characters are usually right but lines can come out in a different order. +- **Faint or coloured printed labels** (light-blue Nastaliq on forms) are often missed. +- The pipeline has many stages tuned together. Swapping one component (for example a newer recognizer) without retuning the stages after it makes results worse; this was measured, not assumed. +- Persian-first. Latin text and digits are read, but other languages were not a goal. +- Confidences are model scores, not calibrated probabilities. + +## Training and provenance + +- Recognizer: Indra CTC, 32,796,250 parameters, 20 blocks, 1,617-symbol vocabulary; 95,000 training steps, distilled from Bina 0.4 Koochik checkpoint 18,404. +- Specialist readers (printed labels, form cells, numeric/math, degraded scans) are 32.8M fine-tunes of the same recognizer; the two line detectors are 38.5M fine-tunes of Surya's text detector, and the printed-text head adds 0.39M. +- Checkpoint selection used internal development sets only; the gates above were never used to pick weights. +- `MANIFEST.json` lists every bundled file with its original location and size. Under `stallion/` the files are organized by experiment directory exactly as they ran in production; `rizeh.py` is the entry point. + +## License + +Bina recognizer weights and code: CC BY-NC 4.0, matching the non-commercial training data. Line-detector weights are fine-tunes of Surya (datalab) detection weights, and the Surya layout model is downloaded from datalab; both stay under [Surya's model license](https://github.com/datalab-to/surya), which limits commercial use. + +## فارسی + +بینا ۰٫۴ ریزه عضو کوچک و سریع خانواده OCR فارسی بینا ۰٫۴ است. یک مدل CTC با ۳۲٫۸ میلیون پارامتر که از بینا ۰٫۴ کوچیک تقطیر شده، درون یک خط‌لوله کامل صفحه کار می‌کند: چیدمان صفحه، تشخیص خط، خواندن متن چاپی و دست‌نویس، فرم‌ها، اعداد و چیدن متن به ترتیب خواندن. + +```bash +pip install -r requirements.txt +python rizeh.py page.jpg --out results +``` + +محدودیت مهم: روی نسخه‌های خطی تاریخی (نستعلیق و شکسته قدیمی) قابل استفاده نیست، و جدول‌ها و ترتیب خواندن صفحه‌های شلوغ هنوز ضعیف‌اند. diff --git a/examples/demo.png b/examples/demo.png new file mode 100644 index 0000000000000000000000000000000000000000..bf8c234c8fdc1a48c38ba4ff5fc97779c4911b81 Binary files /dev/null and b/examples/demo.png differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..801bc47649252cf7f36641da817050d483c9f8c1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,17 @@ +# Tested with Python 3.10 on an NVIDIA RTX 5080. +torch==2.10.0 +torchvision==0.25.0 +surya-ocr==0.22.1 +transformers==5.14.1 +numpy==1.26.4 +scipy==1.15.3 +scikit-learn==1.7.2 +joblib==1.5.3 +rapidfuzz==3.14.5 +jiwer==4.0.0 +opencv-python-headless==4.11.0.86 +onnxruntime==1.23.2 +pillow==10.4.0 +pyarrow==23.0.1 +huggingface_hub>=1.0 +pypdfium2 diff --git a/rizeh.py b/rizeh.py new file mode 100644 index 0000000000000000000000000000000000000000..ccc37afef7999f5dc825e3d5eaa59804069f6826 --- /dev/null +++ b/rizeh.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Bina 0.4 Rizeh: Persian page OCR (Surya layout + 95k Indra CTC student + printed/table/numeric experts and assembly). + +usage: python rizeh.py page.jpg [more.png doc.pdf ...] [--out results] [--device cuda|cpu] +""" +import argparse,json,os,sys,time,importlib.util +from pathlib import Path +ROOT=Path(__file__).resolve().parent;ST=ROOT/'stallion';MARK=ROOT/'.bina-root' +LAYOUT_REPO,LAYOUT_REV='datalab-to/surya_layout2','0aee81d5fd9275c0582e545bf3a56944b1e75679' +TEXT=('.py','.json','.yml','.yaml','.txt','.jsonl') + +def setup(): + """Point every bundled path at this checkout (idempotent; reruns after a move) and fetch the pinned layout model.""" + here=str(ROOT);old=MARK.read_text().strip()if MARK.exists()else'__BINA_ROOT__' + if old!=here: + n=0 + for p in ST.rglob('*'): + if p.is_file()and p.suffix in TEXT: + s=p.read_text(encoding='utf-8') + if old in s:p.write_text(s.replace(old,here),encoding='utf-8');n+=1 + MARK.write_text(here);print(f'[rizeh] bound {n} files to {here}',file=sys.stderr) + snap=ST/'indra-scale/hf/hub/models--datalab-to--surya_layout2/snapshots'/LAYOUT_REV + if not(snap/'rfdetr_layout.pth').exists(): + from huggingface_hub import snapshot_download + print('[rizeh] downloading layout model',LAYOUT_REPO,LAYOUT_REV,file=sys.stderr) + snapshot_download(LAYOUT_REPO,revision=LAYOUT_REV,cache_dir=str(ST/'indra-scale/hf/hub')) + return snap + +def pages(path): + p=Path(path) + if p.suffix.lower()=='.pdf': + import pypdfium2 as pdfium + doc=pdfium.PdfDocument(str(p)) + for i in range(len(doc)):yield f'{p.stem}-p{i+1:03d}',doc[i].render(scale=200/72).to_pil().convert('RGB') + else: + from PIL import Image + yield p.stem,Image.open(p).convert('RGB') + +class Rizeh: + def __init__(self,device='cuda'): + snap=setup() + os.environ.setdefault('GATE_DEVICE',device);os.environ.setdefault('FAST_DETECTOR_DEVICE','cpu');os.environ.setdefault('DETECTOR_IMAGE_CHUNK_HEIGHT','1400');os.environ['FAST_ORDER_MODEL_CHECKPOINT']=str(snap/'order') + P=ST/'indra-scale';sys.path[:0]=[str(P),str(ST/'bina04-pagepath-v3-round3-clean-20261031'),str(P/'scoringTT')] + spec=importlib.util.spec_from_file_location('bina04_rizeh_runtime',ST/'bina04-ink-loss-guard-95k-20261104/production.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m) + self.engine=m.Runtime(device) + from surya.fast_layout.server import LayoutEngine + self.layout=LayoutEngine(checkpoint=str(snap)) + def __call__(self,image,name='page'): + raw=self.layout.run_batch([image],[dict(threshold=None,use_order=True)])[0].model_dump(mode='json') + out=self.engine(image,dict(raw_layout=raw),'page',name) + lines=[dict(text=l.get('text',''),layer=l.get('layer'),box=[round(float(v),1)for v in l['box']],confidence=l.get('confidence'))for l in out.get('lines',[])if l.get('box')] + return dict(text=out['text'],lines=lines) + def close(self):self.engine.close() + +def main(): + ap=argparse.ArgumentParser(description=__doc__,formatter_class=argparse.RawDescriptionHelpFormatter);ap.add_argument('inputs',nargs='+');ap.add_argument('--out',default='rizeh-out');ap.add_argument('--device',default='cuda',choices=['cuda','cpu']);a=ap.parse_args() + O=Path(a.out);O.mkdir(parents=True,exist_ok=True);r=Rizeh(a.device) + for src in a.inputs: + for name,im in pages(src): + t=time.perf_counter();res=r(im,name);res.update(source=str(src),seconds=round(time.perf_counter()-t,2),model='Bina-0.4-Rizeh') + (O/f'{name}.json').write_text(json.dumps(res,ensure_ascii=False,indent=1),encoding='utf-8');(O/f'{name}.txt').write_text(res['text'],encoding='utf-8') + print(f'{name}: {len(res["lines"])} lines, {res["seconds"]}s -> {O/name}.txt',flush=True) + r.close() +if __name__=='__main__':main() diff --git a/stallion/bina04-assembly-audit-95k-20261104/graphic_blocks.py b/stallion/bina04-assembly-audit-95k-20261104/graphic_blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..1a2267bf0f71715e5fb738652ddf7aec2929567b --- /dev/null +++ b/stallion/bina04-assembly-audit-95k-20261104/graphic_blocks.py @@ -0,0 +1,51 @@ +"""Atomic supported text blocks in illustrated Latin documents; frozen reader.""" +import copy,unicodedata +import numpy as np + +def area(b):return max(0,b[2]-b[0])*max(0,b[3]-b[1]) +def ov(a,b):return area([max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])])/max(1,area(a)) +def union(bs):return[min(b[0]for b in bs),min(b[1]for b in bs),max(b[2]for b in bs),max(b[3]for b in bs)] +def apply(old,image): + out=copy.deepcopy(old);ls=out.get('lines',[]) + if any(l.get('layer')=='handwriting'for l in ls)or any(r.get('table_trace',{}).get('grid')for r in out.get('regions',[])):return old + text=''.join(l['text']for l in ls if l.get('confidence',0)>=.9);letters=[c for c in text if c.isalpha()];latin=sum('LATIN'in unicodedata.name(c,'')for c in letters) + if len(letters)<60 or latin/max(1,len(letters))<.8:return old + W,H=image.size;a=np.asarray(image.convert('RGB'));sat=(a.max(2).astype(float)-a.min(2));colour=float((sat>30).mean()) + layouts=[r['bbox']for r in out.get('raw_layout',{}).get('bboxes',[])if r.get('label')in('Text','SectionHeader','Caption','ListItem')and'bbox'in r] + if colour<.08 or len(layouts)<3:return old + if ''.join(out['text'].split())!=''.join(''.join(l['text'].split())for l in ls):return old + med=float(np.median([l['box'][3]-l['box'][1]for l in ls if l.get('confidence',0)>.94 and len(l['text'].strip())>=8]));assigned={};groups={} + for i,l in enumerate(ls): + if not l['text'].strip():continue + scores=[ov(l['box'],b)for b in layouts] + if scores and max(scores)>=.5:k=int(np.argmax(scores));groups.setdefault(k,[]).append(i);assigned[i]=k + unsupported=[i for i,l in enumerate(ls)if i not in assigned and l['text'].strip()and l.get('confidence',0)>=.9] + # Infer omitted text blocks from high-confidence aligned physical baselines. + for i in unsupported: + b=ls[i]['box'];found=None + for k,ids in groups.items(): + if k.55*min(b[2]-b[0],bb[2]-bb[0])and abs(b[0]-bb[0])<1.5*med and gap<2*med:found=k;break + k=found if found is not None else len(layouts)+i+1;groups.setdefault(k,[]).append(i);assigned[i]=k + drop=[] + for i,l in enumerate(ls): + if not l['text'].strip()or i in assigned:continue + if l.get('confidence',0)<.55 and len(''.join(l['text'].split()))<=8:drop.append(i) + else:groups[len(layouts)+len(ls)+i+1]=[i] + if not groups:return old + blocks=[dict(ids=sorted(ids,key=lambda i:(ls[i]['box'][1],ls[i]['box'][0])),box=union([ls[i]['box']for i in ids]))for ids in groups.values()] + def cut(blocks): + if len(blocks)<=1:return blocks + for axis in [1,0]: + intervals=sorted((z['box'][axis],z['box'][axis+2])for z in blocks);end=intervals[0][1];gaps=[] + for a,b in intervals[1:]: + if a-end>.25*med:gaps.append((a-end,(a+end)/2)) + end=max(end,b) + if gaps: + _,mid=max(gaps);left=[z for z in blocks if z['box'][axis+2]<=mid];right=[z for z in blocks if z['box'][axis]>=mid] + if left and right and len(left)+len(right)==len(blocks):return cut(left)+cut(right) + return sorted(blocks,key=lambda z:(z['box'][1],z['box'][0])) + ordered=cut(blocks);ids=[i for z in ordered for i in z['ids']];newtext='\n\n'.join(ls[i]['text']for i in ids) + if newtext==out['text']:return old + out['lines']=[ls[i]for i in ids];out['text']=newtext;out['regions']=[dict(path='atomic_graphic_text_blocks',lines=[ls[i]for i in z['ids']],text='\n'.join(ls[i]['text']for i in z['ids']),box=z['box'])for z in ordered];out['graphic_block_assembly']=dict(colour_fraction=colour,latin_fraction=latin/len(letters),removed=[ls[i]for i in drop],blocks=ordered,confidence_threshold=.55,threshold_source='Existing frozen private-dev graphic-junk threshold; no193/TT tuning',reader_changed=False);return out diff --git a/stallion/bina04-assembly-audit-95k-20261104/production.py b/stallion/bina04-assembly-audit-95k-20261104/production.py new file mode 100644 index 0000000000000000000000000000000000000000..8621aef307cdd3fa736aab5433c84cf513ee0567 --- /dev/null +++ b/stallion/bina04-assembly-audit-95k-20261104/production.py @@ -0,0 +1,8 @@ +from pathlib import Path +import sys,importlib.util +R=Path(__file__).resolve().parent;sys.path.insert(0,str(R));from graphic_blocks import apply +class Runtime: + def __init__(self,device='auto'): + s=importlib.util.spec_from_file_location('graphic_frozen_parent','__BINA_ROOT__/stallion/bina04-block-fields-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);self.raw=self.parent.raw;self.core=self.parent.core + def __call__(self,image,old,kind,name='',**kwargs):return apply(self.parent(image,old,kind,name,**kwargs),image) + def close(self):self.parent.close() diff --git a/stallion/bina04-block-fields-95k-20261103/assembly_fix.py b/stallion/bina04-block-fields-95k-20261103/assembly_fix.py new file mode 100644 index 0000000000000000000000000000000000000000..75d624b868d9a96bcbbefd7a7f84d1d9527ed85b --- /dev/null +++ b/stallion/bina04-block-fields-95k-20261103/assembly_fix.py @@ -0,0 +1,56 @@ +import copy +import numpy as np,cv2 + +def norm(s):return ''.join(s.split()) +def rule_bands(gray): + H,W=gray.shape;bw=cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY_INV,31,12);r=cv2.morphologyEx(bw,cv2.MORPH_OPEN,np.ones((1,max(25,int(W*.35))),np.uint8));ys=np.flatnonzero((r>0).sum(1)>.65*W);return [float(np.median(g))for g in np.split(ys,np.flatnonzero(np.diff(ys)>1)+1)if len(g)]if len(ys)else[] +def apply(old,image,reader=None): + out=copy.deepcopy(old);ls=out.get('lines',[]);W,H=image.size + if len(ls)<3 or any(l.get('layer')=='handwriting'for l in ls)or any(r.get('table_trace',{}).get('grid')for r in out.get('regions',[])):return out + if norm(''.join(l.get('text','')for l in ls))!=norm(out['text']):return out + gray=np.asarray(image.convert('L'));bounds=rule_bands(gray);hs=[l['box'][3]-l['box'][1]for l in ls if len(norm(l.get('text','')))>8 and l['confidence']>.8];med=float(np.median(hs))if hs else 0 + if med<=0:return out + trace=dict(header_blocks=[],form_rows=[],split_boxes=[]) + # Split a vertically merged ruling-band heading and content. Horizontal expansion + # is bounded by actual foreground in that row, not the detected partial x-extent. + if reader is not None and len(bounds)>=4: + for i,l in enumerate(ls[:]): + x,y,X,Y=l['box'];hh=Y-y + if hh<2.5*med or hh>.12*H or X-x<.4*W:continue + cells=[(a,b)for a,b in zip(bounds,bounds[1:])if a>=y-2 and b<=Y+2 and .5*med0)|(vr>0)]=0;ys=np.flatnonzero(ink.sum(1)>3);xs=np.flatnonzero(ink.sum(0)>1) + if len(ys)<4 or len(xs)<8:break + boxes.append([max(0,int(xs[0])-2),max(0,yy+int(ys[0])-2),min(W,int(xs[-1])+3),min(H,yy+int(ys[-1])+3)]) + if len(boxes)!=len(cells):continue + rr=reader(gray,boxes) + if any(r['confidence']<.65 or len(norm(r['text']))<4 for r in rr):continue + ls[i:i+1]=[dict(r,layer=l.get('layer'),source='ruling_band_multiline_split')for r in rr];trace['split_boxes'].append(dict(original=l['box'],boxes=boxes,original_text=l['text'],new_text=[r['text']for r in rr]));break + # Restore recovered printed fields to physical rows, regardless of admission layer. + if len(bounds)>=4: + assignment={} + for a,b in zip(bounds,bounds[1:]): + if not .6*med.2*min(ls[i]['box'][2]-ls[i]['box'][0],ls[j]['box'][2]-ls[j]['box'][0])for k,i in enumerate(ids)for j in ids[k+1:]):continue + order=sorted(ids,key=lambda i:-ls[i]['box'][2]);assignment.update({i:order for i in ids});trace['form_rows'].append(dict(band=[a,b],before=ids,after=order)) + done=set();order=[] + for i in range(len(ls)): + if i in done:continue + ids=assignment.get(i,[i]);order+=ids;done.update(ids) + ls=[ls[i]for i in order] + # Side-by-side headers remain atomic blocks, read RTL. The reading convention + # is geometric; no document names, reference strings or page-specific IDs. + left=[i for i,l in enumerate(ls)if l['box'][2]<.42*W and l['box'][3]<.17*H and len(norm(l['text']))>=8 and l['confidence']>.75 and l['box'][3]-l['box'][1]<2*med] + right=[i for i,l in enumerate(ls)if l['box'][0]>.58*W and l['box'][3]<.17*H and len(norm(l['text']))>=8 and l['confidence']>.75 and l['box'][3]-l['box'][1]<2*med] + if len(left)>=2 and len(right)>=2: + ids=set(left+right);ordered=sorted(right,key=lambda i:(ls[i]['box'][1],-ls[i]['box'][2]))+sorted(left,key=lambda i:(ls[i]['box'][1],-ls[i]['box'][2]));new=[] + for i in range(len(ls)): + if i==min(ids):new+=ordered + if i not in ids:new.append(i) + trace['header_blocks']=dict(left_ids=left,right_ids=right,after=ordered,convention='right block then left block; each top-to-bottom');ls=[ls[i]for i in new] + if not trace['split_boxes'] and [l['text']for l in ls]==[l['text']for l in old['lines']]:return old + out['lines']=ls;out['regions']=[dict(path='physical_header_and_form_rows',layout=None,lines=[l],text=l['text'])for l in ls];out['text']='\n\n'.join(l['text']for l in ls);out['physical_block_fields']=trace;return out diff --git a/stallion/bina04-block-fields-95k-20261103/production.py b/stallion/bina04-block-fields-95k-20261103/production.py new file mode 100644 index 0000000000000000000000000000000000000000..33db6a13fe8028bda560d7e314add2cebe5084b6 --- /dev/null +++ b/stallion/bina04-block-fields-95k-20261103/production.py @@ -0,0 +1,12 @@ +from pathlib import Path +import importlib.util,sys +R=Path(__file__).resolve().parent;sys.path.insert(0,str(R)) +from assembly_fix import apply +class Runtime: + def __init__(self,device='auto'): + s=importlib.util.spec_from_file_location('block_fields_parent','__BINA_ROOT__/stallion/bina04-quality-route-printscope-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);self.raw=self.parent.raw;self.core=self.parent.core + from pipeline_v3 import UnifiedReader + V=Path('__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031');self.split_reader=UnifiedReader(V/'run/weights-095000.pt',V/'data/vocab.json',device='cpu') + def __call__(self,image,old,kind,name='',**kwargs): + out=self.parent(image,old,kind,name,**kwargs);return apply(out,image,self.split_reader)if kind=='page'else out + def close(self):self.parent.close() diff --git a/stallion/bina04-cell-rowcores-95k-20261103/candidate.py b/stallion/bina04-cell-rowcores-95k-20261103/candidate.py new file mode 100644 index 0000000000000000000000000000000000000000..2decaa2f949b84df6024e36b34a8861888c4d48b --- /dev/null +++ b/stallion/bina04-cell-rowcores-95k-20261103/candidate.py @@ -0,0 +1,41 @@ +"""Attach detached small bands inside cells; preserve two full text rows.""" +import copy,sys,numpy as np +from pathlib import Path +sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-crop-picture-cells-95k-20261102');sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-crop-bandseg-95k-20261102') +from cell_reader import read_cell,owned_hand_cells +from bandseg import segment + +def rowcores(image): + boxes,trace=segment(image);boxes=sorted(boxes,key=lambda b:b[1]);merged=[];events=[] + for b in boxes: + if merged: + a=merged[-1];ha=a[3]-a[1];hb=b[3]-b[1];gap=max(0,b[1]-a[3]);horizontal=min(a[2],b[2])-max(a[0],b[0]) + # A small detached dot/ascender band cannot be a second full writing row. + if min(ha,hb)<.6*max(ha,hb)and gap<=.5*max(ha,hb)and horizontal>0: + merged[-1]=[min(a[0],b[0]),min(a[1],b[1]),max(a[2],b[2]),max(a[3],b[3])];events.append(dict(first=a,second=b,reason='unequal core heights, near and horizontally overlapping'));continue + merged.append(list(b)) + trace=dict(trace,cell_fragment_attachments=events);return merged,trace + +class Patch: + def __init__(self,reader,flip):self.reader=reader;self.flip=flip + def __call__(self,image,base,kind='crop'): + if kind!='crop':return base + out=copy.deepcopy(base);rgb=np.asarray(image.convert('RGB'));changed=[] + for region in out['regions']: + t=region.get('table_trace',{});cells=t.get('cell_debug',[]) + if not t.get('grid')or not cells:continue + pen=region.get('table_cell_contract',{}).get('table_pen',False);owned,owner_trace=owned_hand_cells(rgb,cells)if pen else(None,None) + for index,c in enumerate(cells): + if len(c.get('lines',[]))<2:continue + heights=[l['box'][3]-l['box'][1]for l in c['lines']] + if min(heights)>=.6*max(heights):continue + pixels,box=owned[index]if pen else(rgb,c['bbox']);text,ll,trace=read_cell(self.reader,self.flip,pixels,box,rowcores,pen=False) + if not trace.get('proposal_trace',{}).get('cell_fragment_attachments'):continue + old=c['text'];c.update(text=text,lines=ll,cell_rowcores=trace);changed.append(dict(row=c['row'],col=c['col'],old=old,new=text,old_lines=len(heights),new_lines=len(ll))) + values={(c['row'],c['col']):c['text']for c in cells};rendered=[] + for i,row in enumerate(t['kept_rows']): + vals=[' '.join(values.get((row,col),'').split()).replace('|',r'\|')for col in t['kept_columns']];rendered.append('| '+' | '.join(vals)+' |') + if i==0:rendered.append('| '+' | '.join('---'for _ in vals)+' |') + region['text']='\n'.join(rendered);region['lines']=[l for c in cells for l in c['lines']] + if not changed:return base + out.update(text='\n\n'.join(r['text']for r in out['regions']),lines=[l for r in out['regions']for l in r['lines']],cell_rowcore_changes=changed);out['emitted_boxes']=[l['box']for l in out['lines']];out['recognition_extents']=[l.get('geometry',{}).get('source_pixels_extent',l['box'])for l in out['lines']];return out diff --git a/stallion/bina04-cell-rowcores-95k-20261103/production.py b/stallion/bina04-cell-rowcores-95k-20261103/production.py new file mode 100644 index 0000000000000000000000000000000000000000..75b8b1c239fb4dffe0f335748443850e85485851 --- /dev/null +++ b/stallion/bina04-cell-rowcores-95k-20261103/production.py @@ -0,0 +1,17 @@ +import importlib.util +from pathlib import Path +R=Path(__file__).resolve().parent +class Runtime: + def __init__(self,device='auto'): + s=importlib.util.spec_from_file_location('rowcore_frozen_parent','__BINA_ROOT__/stallion/bina04-header-marks-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device) + s=importlib.util.spec_from_file_location('rowcore_header_apply','__BINA_ROOT__/stallion/bina04-header-marks-95k-20261103/candidate.py');self.header=importlib.util.module_from_spec(s);s.loader.exec_module(self.header) + s=importlib.util.spec_from_file_location('rowcore_candidate',R/'candidate.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m) + from indra_ocr.rtl import flip + self.patch=m.Patch(self.parent.parent.numeric,lambda t:flip(t,base_rtl=True)) + def __call__(self,image,old,kind,name='',**kwargs): + out=self.parent.parent(image,old,kind,name,**kwargs) + assert self.parent.params['header_confidence']>1,'Frozen accepted header mode required' + data=dict(new=[],old_support=[1.]*len(out['lines']),WH=list(image.size),header_regions=[]) + out=self.header.apply(out,data,self.parent.params) + return self.patch(image,out,kind) + def close(self):self.parent.close() diff --git a/stallion/bina04-column-assembly-95k-20261103/column_order.py b/stallion/bina04-column-assembly-95k-20261103/column_order.py new file mode 100644 index 0000000000000000000000000000000000000000..5a0e7094fba0255cda12b257ee0481f93d714935 --- /dev/null +++ b/stallion/bina04-column-assembly-95k-20261103/column_order.py @@ -0,0 +1,39 @@ +"""Conservative physical-gutter order for dense, two-column RTL text; tables atomic.""" +import copy,statistics +from assembly import norm + +def apply_columns(old,W,H): + out=copy.deepcopy(old);lines=out.get('lines',[]) + if any(r.get('table_trace',{}).get('grid')for r in out.get('regions',[])):return out + useful=[i for i,l in enumerate(lines)if len(norm(l.get('text','')))>=8 and .12*W=x];cross=[i for i in useful if i not in left+right] + if min(len(left),len(right))<4 or len(cross)>.05*len(useful):continue + span=lambda ids:max(lines[i]['box'][3]for i in ids)-min(lines[i]['box'][1]for i in ids) + if min(span(left),span(right))<.4*H:continue + gap=min(lines[i]['box'][0]for i in right)-max(lines[i]['box'][2]for i in left) + if gap<.012*W:continue + candidates.append((len(left)+len(right),gap,x,left,right)) + if not candidates:return out + _,gap,x,left,right=max(candidates,key=lambda v:(v[0],v[1]));med=statistics.median(lines[i]['box'][3]-lines[i]['box'][1]for i in useful);start=max(min(lines[i]['box'][1]for i in left),min(lines[i]['box'][1]for i in right));end=min(max(lines[i]['box'][3]for i in left),max(lines[i]['box'][3]for i in right)) + # A shared spanning heading distinguishes one multi-column document from + # two independent exam/form panels with their own headers. Ambiguous pages retain baseline order. + if not any(l['box'][2]-l['box'][0]>.65*W and l['box'][1]=8 for l in lines):return out + headers=[];footers=[];lc=[];rc=[] + for i,l in enumerate(lines): + if not l.get('text','').strip():continue + b=l['box'];cy=(b[1]+b[3])/2 + if b[3]end-med:footers.append(i) + elif b[0]x:rc.append(i) + else:lc.append(i) + key=lambda i:(lines[i]['box'][1],-lines[i]['box'][2]);ids=sorted(headers,key=key)+sorted(rc,key=key)+sorted(lc,key=key)+sorted(footers,key=key) + # Reorder reads, never edit them. Only apply when plain-line emission exactly accounts for output. + if norm(''.join(lines[i]['text']for i in range(len(lines))))!=norm(out['text']):return out + out['text']='\n\n'.join(lines[i]['text']for i in ids);out['lines']=[lines[i]for i in ids] + out['regions']=[dict(path='physical_column_order',layout=None,source_line_ids=[i],lines=[lines[i]],text=lines[i]['text'])for i in ids] + out['physical_column_order']=dict(applied=True,gutter_x=x,gutter_width=gap,header_ids=headers,right_ids=rc,left_ids=lc,footer_ids=footers,read_text_changed=False) + return out diff --git a/stallion/bina04-column-assembly-95k-20261103/production.py b/stallion/bina04-column-assembly-95k-20261103/production.py new file mode 100644 index 0000000000000000000000000000000000000000..4b672d045bf998e20fbbfe0b7c4c5fa6c5249052 --- /dev/null +++ b/stallion/bina04-column-assembly-95k-20261103/production.py @@ -0,0 +1,11 @@ +"""Accepted page assembly with conservative shared-heading RTL column order.""" +import importlib.util,sys +from pathlib import Path +R=Path(__file__).resolve().parent +sys.path.insert(0,str(R)) +from column_order import apply_columns +class Runtime: + def __init__(self,device='auto'): + s=importlib.util.spec_from_file_location('columns_frozen_parent','__BINA_ROOT__/stallion/bina04-page-assembly-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device) + def __call__(self,image,old,kind,name='',**kwargs):return apply_columns(self.parent(image,old,kind,name,**kwargs),*image.size) + def close(self):self.parent.close() diff --git a/stallion/bina04-crop-bandseg-95k-20261102/bandseg.py b/stallion/bina04-crop-bandseg-95k-20261102/bandseg.py new file mode 100644 index 0000000000000000000000000000000000000000..2d15be5e496b812649c083ef2a3d408d9cf00eb0 --- /dev/null +++ b/stallion/bina04-crop-bandseg-95k-20261102/bandseg.py @@ -0,0 +1,277 @@ +"""Pixel-only crop line-count guard; pages and atomic tables stay unchanged.""" +import copy +import cv2 +import numpy as np +from scipy.ndimage import gaussian_filter1d +from scipy.signal import find_peaks, peak_widths + + +def glyph_height(m, fallback=3.): + _,_,stats,_=cv2.connectedComponentsWithStats(m.astype(np.uint8),8) + values=[(int(h),int(a)) for x,y,w,h,a in stats[1:] if h>=3 and w>=2 and h<=m.shape[0]] + if not values:return fallback + values.sort();hs,areas=zip(*values) + return float(hs[np.searchsorted(np.cumsum(areas),.70*sum(areas))]) + + +def mask(gray): + _, m = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) + if (m > 0).mean() > .5: + m = 255 - m + n, labels, stats, _ = cv2.connectedComponentsWithStats(m, 8) + H, W = gray.shape + good = [] + heights = [] + for i, (x, y, w, h, area) in enumerate(stats[1:], 1): + if area < 2 or (w > .8 * W and h > .65 * H): + continue # connected frames/decorations, not a text-line core + if (w > .7 * W and h < max(3, .035 * H)) or (h > .7 * H and w < max(2,.035*W)): + continue # ruling line + good.append(i) + if h >= 3 and w >= 2 and h < .9 * H: + heights.append(h) + clean = np.isin(labels, good) + glyph = float(np.percentile(heights, 70)) if heights else max(3., H / 2) + # Remove long peripheral scan rules without deleting connected text. + hline=cv2.morphologyEx(clean.astype(np.uint8),cv2.MORPH_OPEN,np.ones((1,max(8,int(.65*W))),np.uint8)) + vline=cv2.morphologyEx(clean.astype(np.uint8),cv2.MORPH_OPEN,np.ones((max(8,int(.65*H)),1),np.uint8)) + hline[int(.12*H):int(.88*H),:]=0 + vline[:,int(.12*W):int(.88*W)]=0 + clean &= ~((hline>0)|(vline>0)) + framed=any(w>.8*W and h>.65*H for x,y,w,h,area in stats[1:]) + if framed: + edge_rules=cv2.morphologyEx(clean.astype(np.uint8),cv2.MORPH_OPEN,np.ones((1,max(8,int(.35*W))),np.uint8)) + edge_rules[int(.25*H):int(.75*H),:]=0 + clean &= ~(edge_rules>0) + if H<160: + for i,(x,y,w,h,area) in enumerate(stats[1:],1): + if h>1.8*glyph and .7*h.88*W) and w<.2*W: + clean[labels==i]=False + return clean, glyph_height(clean,glyph) + + +def horizontal(m, glyph, scale=1): + H, W = m.shape + prof = gaussian_filter1d(m.sum(1).astype(float), max(.6, .055 * glyph)) + if prof.max() < 1: + return [], dict(peaks=[], valleys=[]) + peaks, props = find_peaks(np.r_[0., prof, 0.], + distance=max(3, int(.35 * glyph),int(min(9*scale,H/3))), + prominence=max(1.5, .10 * prof.max())) + peaks = peaks - 1 + peaks = [int(p) for p in peaks if 0 <= p < H] + _, _, comps, _ = cv2.connectedComponentsWithStats(m.astype(np.uint8), 8) + def local_height(y): + rows=[(h,area) for x,cy,w,h,area in comps[1:] if cy <= y < cy+h and h>=3 and w>=2 and h<.7*H] + if not rows:return glyph + rows.sort();hs,areas=zip(*rows) + return float(hs[np.searchsorted(np.cumsum(areas),.7*sum(areas))]) + # A detached dot or descender is not another baseline. Only a deep valley + # between substantial core peaks can create another physical line. + while len(peaks) > 1: + remove = None + for i, (a, b) in enumerate(zip(peaks, peaks[1:])): + valley = float(prof[a:b+1].min()) + if valley > .55 * min(prof[a], prof[b]) or b-a < .85*max(local_height(a),local_height(b)): + remove = i if prof[a] < prof[b] else i + 1 + break + if remove is None: + break + peaks.pop(remove) + if not peaks: + peaks = [int(prof.argmax())] + cuts = [0] + for a, b in zip(peaks, peaks[1:]): + q = prof[a:b+1] + low = np.flatnonzero(q <= q.min() + .015 * min(prof[a], prof[b])) + cuts.append(a + int(np.median(low))) + cuts.append(H) + bands = [] + for i, (a, b) in enumerate(zip(cuts, cuts[1:])): + ys = np.flatnonzero(m[a:b].sum(1) >= max(1, .015 * prof.max())) + if not len(ys): + continue + y0, y1 = a + int(ys[0]), a + int(ys[-1]) + 1 + xs = np.flatnonzero(m[y0:y1].any(0)) + if not len(xs) or y1-y0 < max(2,.4*local_height(peaks[i])): + continue + # Keep the separating valley as a hard recognition crop boundary. + bands.append([int(xs[0]), y0, int(xs[-1])+1, y1]) + if len(bands)>1: + tallest=max(b[3]-b[1] for b in bands) + bands=[b for b in bands if not (b[3]-b[1]<.45*tallest and (b[1]<=scale or b[3]>=H-scale))] + return bands, dict(peaks=peaks, valleys=cuts[1:-1], glyph=glyph) + + +def runs(v): + edges = np.diff(np.r_[False, v, False].astype(np.int8)) + return list(zip(np.flatnonzero(edges == 1), np.flatnonzero(edges == -1))) + + +def segment(gray): + H, W = gray.shape + scale = 3 if H < 40 else 2 if H < 64 else 1 + big = cv2.resize(gray, (W*scale, H*scale), interpolation=cv2.INTER_CUBIC) if scale != 1 else gray + events = [] + + def recurse(g, ox=0, oy=0, depth=0, allow_horizontal=True, supplied_mask=None): + if supplied_mask is None:m,glyph=mask(g) + else:m=supplied_mask;glyph=glyph_height(m,3.) + hh, ww = g.shape + bands, ht = horizontal(m, glyph,scale) + ys=np.flatnonzero(m.any(1)) + mass=m.sum(1);total=mass.sum() + if total: + lo,hi=np.searchsorted(np.cumsum(mass),[.01*total,.99*total]) + else:lo,hi=0,0 + if depth==0 and (len(bands)<=1 or hi-lo<=1.5*glyph): + return [[0,int(ys[0]),ww,int(ys[-1])+1]] if len(ys) else [] + if depth < 3 and len(bands): + # True gutter candidates, with multiple lines on at least one side. + # A narrow gutter is allowed only between different type scales + # (e.g. a small side column alongside a giant headline). + col = m.mean(0) + gaps = [(a,b) for a,b in runs(col <= .012) + if a > .12*ww and b < .88*ww and b-a >= 2*scale] + for a,b in sorted(gaps, key=lambda q: q[1]-q[0], reverse=True): + left,right=m[:,:a],m[:,b:] + gl,gr=glyph_height(left,3.),glyph_height(right,3.) + if min(gl,gr)<(.25 if max(gl,gr)<64 else .12)*max(gl,gr):continue + lb, _ = horizontal(left, gl,scale); rb, _ = horizontal(right, gr,scale) + area = max(1,m.sum()) + disparate = max(gl,gr) >= 3.*max(1,min(gl,gr)) + true_columns = min(len(lb),len(rb)) >= 2 and b-a >= 1.5*min(gl,gr) + if max(len(lb),len(rb)) < 2 or min(left.sum(),right.sum()) < .025*area or a < 3*gl or ww-b < 3*gr: + continue + if not disparate or max(len(lb),len(rb))<3: + continue + L = recurse(g[:, :a], ox, oy, depth+1, supplied_mask=left) + R = recurse(g[:, b:], ox+b, oy, depth+1, supplied_mask=right) + events.append(dict(kind='vertical_gutter', box=[ox,oy,ox+ww,oy+hh], gap=[int(a),int(b)], glyphs=[gl,gr])) + # Primary large headline before its small side column; ordinary + # columns use RTL. Within each column retain top-to-bottom. + return (L+R) if gl >= 3.*gr else (R+L) + if len(bands) >= 2 and depth < 3 and allow_horizontal: + out = [] + # Inspect each horizontal band for a side column. Whitespace is + # bounded by the valley, so adjacent lines never enter the crop. + for x0,y0,x1,y1 in bands: + out += recurse(g[y0:y1, :], ox, oy+y0, depth+1, allow_horizontal=False, supplied_mask=m[y0:y1,:]) + events.append(dict(kind='horizontal_valleys',box=[ox,oy,ox+ww,oy+hh],**ht)) + return out + if not allow_horizontal and len(bands)>1: + ys,xs=np.where(m) + bands=[[int(xs.min()),int(ys.min()),int(xs.max())+1,int(ys.max())+1]] if len(xs) else [] + return [[ox,oy+y0,ox+ww,oy+y1] for x0,y0,x1,y1 in bands] + + root_mask, root_glyph=mask(big) + _,labs,stats,_=cv2.connectedComponentsWithStats(root_mask.astype(np.uint8),8) + for i,(x,y,w,h,area) in enumerate(stats[1:],1): + if h<.10*root_glyph and area<.15*root_glyph**2: + root_mask[labs==i]=False + boxes = recurse(big,supplied_mask=root_mask) + # A compact framed badge has ornamental upper/lower strokes, not extra + # text baselines. Do not turn its border into recognizer inputs. + _,_,raw_stats,_=cv2.connectedComponentsWithStats((cv2.threshold(big,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)[1]>0).astype(np.uint8),8) + framed=any(w>.8*big.shape[1] and h>.65*big.shape[0] for x,y,w,h,a in raw_stats[1:]) + if framed and W/max(1,H)>2.5 and len(boxes)>=2: + dominant=max(boxes,key=lambda b:b[3]-b[1]) + if dominant[3]-dominant[1]>.45*big.shape[0] and all(b is dominant or b[3]-b[1]<.25*big.shape[0] for b in boxes): + boxes=[dominant] + events.append(dict(kind='framed_badge_ornaments_not_lines')) + # Recognition margins are real pixels, not a larger coverage-only receipt. + # Limit vertical expansion at the midpoint between adjacent physical rows. + padded=[] + for b in boxes: + h=b[3]-b[1];xp=max(2*scale,.15*h);yp=max(2*scale,.2*h) + lo,hi=max(0,b[1]-yp),min(big.shape[0],b[3]+yp) + for q in boxes: + overlap=min(b[2],q[2])-max(b[0],q[0]) + if q is b or overlap<.2*min(b[2]-b[0],q[2]-q[0]):continue + if q[3]<=b[1]:lo=max(lo,(q[3]+b[1])/2) + if q[1]>=b[3]:hi=min(hi,(q[1]+b[3])/2) + padded.append([max(0,b[0]-xp),lo,min(big.shape[1],b[2]+xp),hi]) + boxes = [[v/scale for v in b] for b in padded] + return boxes, dict(scale=scale, boxes=boxes, events=events, image_only=True) + + +def should_replace(baseline, boxes): + if len(boxes) < 2: + return False, 'one_or_zero_supported_band' + lines = baseline.get('lines', []) + if any('Surya_grid' in r.get('path','') or (r.get('layout') or {}).get('label','').lower()=='table' for r in baseline.get('regions',[])) or any(r.get('label','').lower()=='table' for r in baseline.get('raw_layout',{}).get('bboxes',[])): + return False, 'atomic_table' + old = [l['box'] for l in lines] + # Keep successful existing multiline reads; repair a box spanning two or + # more separately supported cores, or a genuine shortage of physical rows. + if len(old) < len(boxes): + return True, 'more_physical_lines_than_reads' + for b in old: + contained = [] + for q in boxes: + iy=max(0,min(b[3],q[3])-max(b[1],q[1])) + ix=max(0,min(b[2],q[2])-max(b[0],q[0])) + if iy >= .8*(q[3]-q[1]) and ix >= .6*(q[2]-q[0]): + contained.append(q) + if len(contained)>=2: + return True, 'existing_read_spans_multiple_cores' + return False, 'existing_multiline_reads_preserved' + + +class CropBandPatch: + def __init__(self, reader, flip): + self.reader, self.flip = reader, flip + + def __call__(self, image, baseline, *, input_type='crop'): + if input_type != 'crop': + return copy.deepcopy(baseline) + gray = np.asarray(image.convert('L')) + boxes, trace = segment(gray) + use, reason = should_replace(baseline, boxes) + trace.update(applied=use,reason=reason,original_reads=len(baseline.get('lines',[]))) + result = copy.deepcopy(baseline) + if not use: + result['crop_band_guard'] = trace + return result + lines=[] + scale=trace['scale'] + big=cv2.resize(gray,(gray.shape[1]*scale,gray.shape[0]*scale),interpolation=cv2.INTER_CUBIC) if scale!=1 else gray + used=set() + for b in boxes: + # Keep an existing correctly localized row, including its actual + # training-contract margins; discard only merged/full-crop reads. + matches=[] + for j,old in enumerate(baseline.get('lines',[])): + o=old['box'];h=b[3]-b[1];oh=o[3]-o[1] + iy=max(0,min(b[3],o[3])-max(b[1],o[1]));ix=max(0,min(b[2],o[2])-max(b[0],o[0])) + if j not in used and .7*h<=oh<=1.6*h and iy>=.8*h and ix>=.65*(b[2]-b[0]): + overlaps=sum(q is not b and max(0,min(q[3],o[3])-max(q[1],o[1]))>=.7*(q[3]-q[1]) for q in boxes) + if not overlaps:matches.append((iy*ix,j)) + if matches: + j=max(matches)[1];lines.append(copy.deepcopy(baseline['lines'][j]));used.add(j);continue + x0,y0,x1,y1=[int(round(v*scale)) for v in b] + local=big[y0:y1,x0:x1] + # Clip the local canvas before adding the exact training margins: + # neighbouring lines cannot bleed into crop_line64's 20% pad. + _, receipts=self.reader(local,np.asarray([[0,0,local.shape[1],local.shape[0]]],np.float32)) + r=receipts[0] + text=self.flip(r['raw_visual_text'],base_rtl=True) + r['source_pixels_extent']=b + r['band_local_original_box']=b + lines.append(dict(box=b,text=text,visual_text=r['raw_visual_text'],confidence=r['confidence'],geometry=r,recognizer='unified32M_95k',confidence_filter_applied=False)) + # Retain small existing reads that projection cannot see (footnotes, + # signatures), rather than losing them when repairing a merged box. + median=float(np.median([b[3]-b[1] for b in boxes])) + for j,old in enumerate(baseline.get('lines',[])): + if j in used:continue + o=old['box'];oh=o[3]-o[1] + if oh>1.5*median:continue + hits=sum(max(0,min(o[3],q[3])-max(o[1],q[1])) >= .5*min(oh,q[3]-q[1]) and min(o[2],q[2])-max(o[0],q[0]) >= .5*min(o[2]-o[0],q[2]-q[0]) for q in boxes) + if hits>=2:continue + overlaps=[max(0,min(o[3],q[3])-max(o[1],q[1]))/max(1,oh) for q in boxes if min(o[2],q[2])>max(o[0],q[0])] + if overlaps and max(overlaps)>=.65:continue + at=next((k for k,l in enumerate(lines) if (l['box'][1]+l['box'][3])/2>(o[1]+o[3])/2),len(lines)) + lines.insert(at,copy.deepcopy(old)) + region=dict(layout=None,path='crop_projection_physical_lines' ,text='\n'.join(l['text']for l in lines),lines=lines,source_line_ids=list(range(len(lines)))) + result.update(text=region['text'],lines=lines,global_readings=lines,regions=[region],detected_boxes=boxes,emitted_boxes=[l['box'] for l in lines],recognition_extents=[l.get('geometry',{}).get('source_pixels_extent',l['box']) for l in lines],crop_band_guard=trace) + return result diff --git a/stallion/bina04-crop-bandseg-95k-20261102/consumer_runtime.py b/stallion/bina04-crop-bandseg-95k-20261102/consumer_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..8d06aa69f17fba8d925dbca6d374f2e6731ae9a5 --- /dev/null +++ b/stallion/bina04-crop-bandseg-95k-20261102/consumer_runtime.py @@ -0,0 +1,51 @@ +"""Crop-only preflight around the accepted 95k engine; no page-path edits. + +The parent supplies its normal image-only layout receipt in `old`. This wrapper +does not need reference text, gold line boxes, benchmark IDs or categories. +""" +import hashlib +import numpy as np +from bandseg import CropBandPatch, segment, should_replace + + +class BandGuardRuntime: + def __init__(self, parent_engine, flip): + self.parent = parent_engine + reader = parent_engine.patch.reader + self.patch = CropBandPatch(reader.reader, flip) + self.context = context = {} + original = type(reader) + + class PreflightReader(original): + def __call__(self, gray, boxes): + H,W = gray.shape + if (context.get('enabled') and len(boxes)==1 + and np.allclose(boxes[0],[0,0,W,H]) + and gray.shape == context['shape'] + and hashlib.sha256(gray.tobytes()).hexdigest()==context['pixels']): + context['full_strip_reads_skipped'] += 1 + return [dict(box=[0.,0.,float(W),float(H)],text='',visual_text='', + confidence=0.,recognizer='skipped_by_multiline_preflight', + geometry=dict(source_pixels_extent=[0.,0.,float(W),float(H)],padded_source_extent=[0.,0.,float(W),float(H)],raw_visual_text=''))] + return super().__call__(gray, boxes) + + reader.__class__ = PreflightReader + + def __call__(self,image,old,kind,name,**kwargs): + self.context.clear() + if kind=='crop': + gray=np.asarray(image.convert('L')) + boxes,trace=segment(gray) + use,reason=should_replace(old,boxes) + self.context.update(enabled=use,shape=gray.shape, + pixels=hashlib.sha256(gray.tobytes()).hexdigest(), + full_strip_reads_skipped=0) + baseline=self.parent(image,old,kind,name,**kwargs) + result=self.patch(image,baseline,input_type=kind) + if kind=='crop': + result['crop_band_preflight']=dict(full_strip_reads_skipped=self.context['full_strip_reads_skipped'], + before_rule_C=True,reference_or_label_used=False) + return result + + def close(self): + self.parent.close() diff --git a/stallion/bina04-crop-picture-cells-95k-20261102/CALIBRATION.json b/stallion/bina04-crop-picture-cells-95k-20261102/CALIBRATION.json new file mode 100644 index 0000000000000000000000000000000000000000..490930e28739d085c19acd0bd67bfe60a8ed8741 --- /dev/null +++ b/stallion/bina04-crop-picture-cells-95k-20261102/CALIBRATION.json @@ -0,0 +1,612 @@ +{ + "source": "Internal synthetic dev80 and frozen calib25 only; no193/check25/publicTT calibration", + "pen": { + "train_samples": 12589, + "dev_samples": 3180, + "train_pages": 160, + "dev_pages": 40, + "dev": { + "threshold": 0.12000000000000001, + "printed_FPR": 0.0029154518950437317, + "handwriting_recall": 0.9996475149806133 + }, + "seconds": 3.9756869729608297, + "parameters": 1623, + "source": "Existing synthetic train only; threshold on synthetic dev only; no193/dev50/TT training", + "options": [ + { + "threshold": 0.1, + "printed_FPR": 0.011661807580174927, + "handwriting_recall": 0.9996475149806133 + }, + { + "threshold": 0.11, + "printed_FPR": 0.011661807580174927, + "handwriting_recall": 0.9996475149806133 + }, + { + "threshold": 0.12000000000000001, + "printed_FPR": 0.0029154518950437317, + "handwriting_recall": 0.9996475149806133 + }, + { + "threshold": 0.13, + "printed_FPR": 0.0029154518950437317, + "handwriting_recall": 0.9996475149806133 + }, + { + "threshold": 0.14, + "printed_FPR": 0.0029154518950437317, + "handwriting_recall": 0.9996475149806133 + }, + { + "threshold": 0.15000000000000002, + "printed_FPR": 0.0029154518950437317, + "handwriting_recall": 0.9996475149806133 + }, + { + "threshold": 0.16, + "printed_FPR": 0.0, + "handwriting_recall": 0.9996475149806133 + }, + { + "threshold": 0.17, + "printed_FPR": 0.0, + "handwriting_recall": 0.9996475149806133 + }, + { + "threshold": 0.18, + "printed_FPR": 0.0, + "handwriting_recall": 0.9992950299612267 + }, + { + "threshold": 0.19, + "printed_FPR": 0.0, + "handwriting_recall": 0.9992950299612267 + }, + { + "threshold": 0.2, + "printed_FPR": 0.0, + "handwriting_recall": 0.9985900599224533 + }, + { + "threshold": 0.21000000000000002, + "printed_FPR": 0.0, + "handwriting_recall": 0.9985900599224533 + }, + { + "threshold": 0.22, + "printed_FPR": 0.0, + "handwriting_recall": 0.9985900599224533 + }, + { + "threshold": 0.23, + "printed_FPR": 0.0, + "handwriting_recall": 0.9985900599224533 + }, + { + "threshold": 0.24000000000000002, + "printed_FPR": 0.0, + "handwriting_recall": 0.9985900599224533 + }, + { + "threshold": 0.25, + "printed_FPR": 0.0, + "handwriting_recall": 0.9982375749030666 + }, + { + "threshold": 0.26, + "printed_FPR": 0.0, + "handwriting_recall": 0.9982375749030666 + }, + { + "threshold": 0.27, + "printed_FPR": 0.0, + "handwriting_recall": 0.9982375749030666 + }, + { + "threshold": 0.28, + "printed_FPR": 0.0, + "handwriting_recall": 0.9982375749030666 + }, + { + "threshold": 0.29000000000000004, + "printed_FPR": 0.0, + "handwriting_recall": 0.99788508988368 + }, + { + "threshold": 0.30000000000000004, + "printed_FPR": 0.0, + "handwriting_recall": 0.99788508988368 + }, + { + "threshold": 0.31, + "printed_FPR": 0.0, + "handwriting_recall": 0.99788508988368 + }, + { + "threshold": 0.32, + "printed_FPR": 0.0, + "handwriting_recall": 0.99788508988368 + }, + { + "threshold": 0.33, + "printed_FPR": 0.0, + "handwriting_recall": 0.9975326048642933 + }, + { + "threshold": 0.33999999999999997, + "printed_FPR": 0.0, + "handwriting_recall": 0.9975326048642933 + }, + { + "threshold": 0.35, + "printed_FPR": 0.0, + "handwriting_recall": 0.9971801198449066 + }, + { + "threshold": 0.36, + "printed_FPR": 0.0, + "handwriting_recall": 0.9971801198449066 + }, + { + "threshold": 0.37, + "printed_FPR": 0.0, + "handwriting_recall": 0.9971801198449066 + }, + { + "threshold": 0.38, + "printed_FPR": 0.0, + "handwriting_recall": 0.9964751498061333 + }, + { + "threshold": 0.39, + "printed_FPR": 0.0, + "handwriting_recall": 0.9964751498061333 + }, + { + "threshold": 0.4, + "printed_FPR": 0.0, + "handwriting_recall": 0.9964751498061333 + }, + { + "threshold": 0.41000000000000003, + "printed_FPR": 0.0, + "handwriting_recall": 0.9964751498061333 + }, + { + "threshold": 0.42000000000000004, + "printed_FPR": 0.0, + "handwriting_recall": 0.9964751498061333 + }, + { + "threshold": 0.43000000000000005, + "printed_FPR": 0.0, + "handwriting_recall": 0.9961226647867466 + }, + { + "threshold": 0.44000000000000006, + "printed_FPR": 0.0, + "handwriting_recall": 0.9961226647867466 + }, + { + "threshold": 0.45000000000000007, + "printed_FPR": 0.0, + "handwriting_recall": 0.9961226647867466 + }, + { + "threshold": 0.45999999999999996, + "printed_FPR": 0.0, + "handwriting_recall": 0.9961226647867466 + }, + { + "threshold": 0.47, + "printed_FPR": 0.0, + "handwriting_recall": 0.9954176947479733 + }, + { + "threshold": 0.48, + "printed_FPR": 0.0, + "handwriting_recall": 0.9947127247091999 + }, + { + "threshold": 0.49, + "printed_FPR": 0.0, + "handwriting_recall": 0.9947127247091999 + }, + { + "threshold": 0.5, + "printed_FPR": 0.0, + "handwriting_recall": 0.9947127247091999 + }, + { + "threshold": 0.51, + "printed_FPR": 0.0, + "handwriting_recall": 0.9947127247091999 + }, + { + "threshold": 0.52, + "printed_FPR": 0.0, + "handwriting_recall": 0.9940077546704265 + }, + { + "threshold": 0.53, + "printed_FPR": 0.0, + "handwriting_recall": 0.9940077546704265 + }, + { + "threshold": 0.54, + "printed_FPR": 0.0, + "handwriting_recall": 0.9936552696510398 + }, + { + "threshold": 0.55, + "printed_FPR": 0.0, + "handwriting_recall": 0.9936552696510398 + }, + { + "threshold": 0.56, + "printed_FPR": 0.0, + "handwriting_recall": 0.9936552696510398 + }, + { + "threshold": 0.5700000000000001, + "printed_FPR": 0.0, + "handwriting_recall": 0.9933027846316531 + }, + { + "threshold": 0.58, + "printed_FPR": 0.0, + "handwriting_recall": 0.9925978145928798 + }, + { + "threshold": 0.59, + "printed_FPR": 0.0, + "handwriting_recall": 0.9925978145928798 + }, + { + "threshold": 0.6, + "printed_FPR": 0.0, + "handwriting_recall": 0.9918928445541064 + }, + { + "threshold": 0.61, + "printed_FPR": 0.0, + "handwriting_recall": 0.9918928445541064 + }, + { + "threshold": 0.62, + "printed_FPR": 0.0, + "handwriting_recall": 0.9918928445541064 + }, + { + "threshold": 0.63, + "printed_FPR": 0.0, + "handwriting_recall": 0.9918928445541064 + }, + { + "threshold": 0.64, + "printed_FPR": 0.0, + "handwriting_recall": 0.9918928445541064 + }, + { + "threshold": 0.65, + "printed_FPR": 0.0, + "handwriting_recall": 0.9915403595347198 + }, + { + "threshold": 0.66, + "printed_FPR": 0.0, + "handwriting_recall": 0.9904829044765597 + }, + { + "threshold": 0.67, + "printed_FPR": 0.0, + "handwriting_recall": 0.9897779344377864 + }, + { + "threshold": 0.6799999999999999, + "printed_FPR": 0.0, + "handwriting_recall": 0.9897779344377864 + }, + { + "threshold": 0.69, + "printed_FPR": 0.0, + "handwriting_recall": 0.9894254494183997 + }, + { + "threshold": 0.7, + "printed_FPR": 0.0, + "handwriting_recall": 0.9883679943602397 + }, + { + "threshold": 0.71, + "printed_FPR": 0.0, + "handwriting_recall": 0.988015509340853 + }, + { + "threshold": 0.72, + "printed_FPR": 0.0, + "handwriting_recall": 0.9876630243214664 + }, + { + "threshold": 0.73, + "printed_FPR": 0.0, + "handwriting_recall": 0.986958054282693 + }, + { + "threshold": 0.74, + "printed_FPR": 0.0, + "handwriting_recall": 0.9866055692633063 + }, + { + "threshold": 0.75, + "printed_FPR": 0.0, + "handwriting_recall": 0.985900599224533 + }, + { + "threshold": 0.76, + "printed_FPR": 0.0, + "handwriting_recall": 0.9851956291857596 + }, + { + "threshold": 0.77, + "printed_FPR": 0.0, + "handwriting_recall": 0.9844906591469863 + }, + { + "threshold": 0.78, + "printed_FPR": 0.0, + "handwriting_recall": 0.9837856891082128 + }, + { + "threshold": 0.79, + "printed_FPR": 0.0, + "handwriting_recall": 0.9820232640112795 + }, + { + "threshold": 0.8, + "printed_FPR": 0.0, + "handwriting_recall": 0.9806133239337328 + }, + { + "threshold": 0.8099999999999999, + "printed_FPR": 0.0, + "handwriting_recall": 0.9792033838561861 + }, + { + "threshold": 0.82, + "printed_FPR": 0.0, + "handwriting_recall": 0.9777934437786394 + }, + { + "threshold": 0.83, + "printed_FPR": 0.0, + "handwriting_recall": 0.9767359887204794 + }, + { + "threshold": 0.84, + "printed_FPR": 0.0, + "handwriting_recall": 0.9756785336623194 + }, + { + "threshold": 0.85, + "printed_FPR": 0.0, + "handwriting_recall": 0.9725061684878392 + }, + { + "threshold": 0.86, + "printed_FPR": 0.0, + "handwriting_recall": 0.9700387733521325 + }, + { + "threshold": 0.87, + "printed_FPR": 0.0, + "handwriting_recall": 0.9682763482551991 + }, + { + "threshold": 0.88, + "printed_FPR": 0.0, + "handwriting_recall": 0.9658089531194924 + }, + { + "threshold": 0.89, + "printed_FPR": 0.0, + "handwriting_recall": 0.9629890729643991 + }, + { + "threshold": 0.9, + "printed_FPR": 0.0, + "handwriting_recall": 0.9591117377511456 + }, + { + "threshold": 0.91, + "printed_FPR": 0.0, + "handwriting_recall": 0.9548819175185055 + }, + { + "threshold": 0.92, + "printed_FPR": 0.0, + "handwriting_recall": 0.9502996122664786 + }, + { + "threshold": 0.93, + "printed_FPR": 0.0, + "handwriting_recall": 0.9436023968981319 + }, + { + "threshold": 0.94, + "printed_FPR": 0.0, + "handwriting_recall": 0.9344377863940783 + }, + { + "threshold": 0.95, + "printed_FPR": 0.0, + "handwriting_recall": 0.9213958406767713 + }, + { + "threshold": 0.96, + "printed_FPR": 0.0, + "handwriting_recall": 0.9041240747268241 + }, + { + "threshold": 0.97, + "printed_FPR": 0.0, + "handwriting_recall": 0.8790976383503701 + }, + { + "threshold": 0.98, + "printed_FPR": 0.0, + "handwriting_recall": 0.8403242862178357 + }, + { + "threshold": 0.99, + "printed_FPR": 0.0, + "handwriting_recall": 0.7673598872047938 + } + ] + }, + "junk": { + "threshold": 0.55, + "false_positive_controls": 0, + "dev_junk_removed": 0 + }, + "pen_trials": [ + { + "threshold": 0.05, + "printed_FPR": 0.9416666666666667, + "coloured_handwriting_recall": 1.0, + "printed": 120, + "handwriting": 80 + }, + { + "threshold": 0.1, + "printed_FPR": 0.8833333333333333, + "coloured_handwriting_recall": 0.9625, + "printed": 120, + "handwriting": 80 + }, + { + "threshold": 0.15, + "printed_FPR": 0.775, + "coloured_handwriting_recall": 0.8375, + "printed": 120, + "handwriting": 80 + }, + { + "threshold": 0.2, + "printed_FPR": 0.675, + "coloured_handwriting_recall": 0.6875, + "printed": 120, + "handwriting": 80 + }, + { + "threshold": 0.25, + "printed_FPR": 0.4666666666666667, + "coloured_handwriting_recall": 0.525, + "printed": 120, + "handwriting": 80 + }, + { + "threshold": 0.3, + "printed_FPR": 0.2, + "coloured_handwriting_recall": 0.45, + "printed": 120, + "handwriting": 80 + }, + { + "threshold": 0.4, + "printed_FPR": 0.1, + "coloured_handwriting_recall": 0.4375, + "printed": 120, + "handwriting": 80 + }, + { + "threshold": 0.5, + "printed_FPR": 0.06666666666666667, + "coloured_handwriting_recall": 0.3875, + "printed": 120, + "handwriting": 80 + }, + { + "threshold": 0.6, + "printed_FPR": 0.05, + "coloured_handwriting_recall": 0.375, + "printed": 120, + "handwriting": 80 + }, + { + "threshold": 0.7, + "printed_FPR": 0.03333333333333333, + "coloured_handwriting_recall": 0.2125, + "printed": 120, + "handwriting": 80 + }, + { + "threshold": 0.8, + "printed_FPR": 0.008333333333333333, + "coloured_handwriting_recall": 0.0375, + "printed": 120, + "handwriting": 80 + }, + { + "threshold": 0.9, + "printed_FPR": 0.0, + "coloured_handwriting_recall": 0.0, + "printed": 120, + "handwriting": 80 + } + ], + "junk_trials": [ + { + "threshold": 0.35, + "false_positive_controls": 0, + "dev_junk_removed": 0 + }, + { + "threshold": 0.45, + "false_positive_controls": 0, + "dev_junk_removed": 0 + }, + { + "threshold": 0.55, + "false_positive_controls": 0, + "dev_junk_removed": 0 + }, + { + "threshold": 0.65, + "false_positive_controls": 1, + "dev_junk_removed": 0 + }, + { + "threshold": 0.75, + "false_positive_controls": 1, + "dev_junk_removed": 1 + }, + { + "threshold": 0.8, + "false_positive_controls": 2, + "dev_junk_removed": 2 + }, + { + "threshold": 0.85, + "false_positive_controls": 2, + "dev_junk_removed": 2 + }, + { + "threshold": 0.9, + "false_positive_controls": 2, + "dev_junk_removed": 2 + }, + { + "threshold": 0.95, + "false_positive_controls": 2, + "dev_junk_removed": 2 + } + ], + "synthetic_short_control_cases": 0, + "synthetic_short_correct": 0, + "calib25_short_cases": 37, + "manifest_sha256": "959ee4f442705c44dea8557c850a1004af3fdfb8d7c0f1fdfe90a8291f773940", + "pen_colour_abstention": 0.05 +} diff --git a/stallion/bina04-crop-picture-cells-95k-20261102/candidate.py b/stallion/bina04-crop-picture-cells-95k-20261102/candidate.py new file mode 100644 index 0000000000000000000000000000000000000000..2327ad8b7b8836c21c291563d1de1cdff47a937f --- /dev/null +++ b/stallion/bina04-crop-picture-cells-95k-20261102/candidate.py @@ -0,0 +1,62 @@ +"""Isolated picture/cell candidate on the accepted bandseg path.""" +import copy,json +from pathlib import Path +import numpy as np +from cell_reader import pen_score,read_cell,owned_hand_cells +from junk_filter import apply as filter_junk +from pen_classifier import PenClassifier +R=Path(__file__).resolve().parent + +class CropPatch: + def __init__(self,reader,flip,segment): + self.reader,self.flip,self.segment=reader,flip,segment + self.cal=json.loads((R/'CALIBRATION.json').read_text()) + self.router=PenClassifier(R/'pen-classifier.joblib') + def __call__(self,image,baseline,*,input_type='crop'): + if input_type!='crop':return copy.deepcopy(baseline) + result=copy.deepcopy(baseline);rgb=np.asarray(image.convert('RGB'));trace=[] + page_colour_score=pen_score(rgb) + for region in result.get('regions',[]): + t=region.get('table_trace',{});cells=t.get('cell_debug',[]) + if not t.get('grid')or not cells:continue + probabilities=[] + for cell in cells: + x,y,X,Y=[int(round(v))for v in cell['bbox']] + view=rgb[max(0,y):min(rgb.shape[0],Y),max(0,x):min(rgb.shape[1],X)] + probabilities.append(self.router.probability(view)if view.size and page_colour_score>.05 else 0.) + nonempty=[i for i,c in enumerate(cells)if c['text'].strip()] + table_pen=bool(nonempty)and sum(probabilities[i]>=self.router.threshold for i in nonempty)>len(nonempty)/2 + ownership,owner_trace=owned_hand_cells(rgb,cells)if table_pen else (None,None) + values={};lines=[] + for index,cell in enumerate(cells): + probability=probabilities[index];pen=table_pen + pixels,box=ownership[index]if table_pen else (rgb,cell['bbox']) + text,ll,info=read_cell(self.reader,self.flip,pixels,box,self.segment,pen=False) + if table_pen:info['physical_component_ownership']=owner_trace + trace.append(dict(row=cell['row'],col=cell['col'],old=cell['text'],new=text,old_lines=len(cell['lines']),new_lines=len(ll),pen_route=pen,pen_probability=probability,preprocessing=info)) + cell.update(text=text,lines=ll,cell_contract=info) + values[(cell['row'],cell['col'])]=text;lines+=ll + rendered=[] + for i,row in enumerate(t['kept_rows']): + vals=[' '.join(values.get((row,col),'').split()).replace('|',r'\|')for col in t['kept_columns']] + rendered.append('| '+' | '.join(vals)+' |') + if i==0:rendered.append('| '+' | '.join('---'for _ in vals)+' |') + region['text']='\n'.join(rendered);region['lines']=lines + region['table_cell_contract']=dict(pen_cells=sum(q['pen_route']for q in trace),pen_score=page_colour_score,table_pen=table_pen,component_ownership=owner_trace,checkpoint=95000,no_date_text_substitution=True) + if trace: + result['text']='\n\n'.join(r['text']for r in result['regions']) + result['lines']=[l for r in result['regions']for l in r['lines']] + result['emitted_boxes']=[l['box']for l in result['lines']] + result['recognition_extents']=[l.get('geometry',{}).get('source_pixels_extent',l['box'])for l in result['lines']] + result['cell_contract_changes']=trace + result,removed=filter_junk(result,self.cal['junk']['threshold']) + return result + +class PipelineRuntime: + def __init__(self,parent,flip,segment): + self.parent=parent + # The accepted BandGuardRuntime owns the raw95k RTLReader here. + self.patch=CropPatch(parent.patch.reader,flip,segment) + def __call__(self,image,old,kind,name,**kwargs): + return self.patch(image,self.parent(image,old,kind,name,**kwargs),input_type=kind) + def close(self):self.parent.close() diff --git a/stallion/bina04-crop-picture-cells-95k-20261102/cell_reader.py b/stallion/bina04-crop-picture-cells-95k-20261102/cell_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..60db7bf2b51f4f9de46282f82b0106b355d08048 --- /dev/null +++ b/stallion/bina04-crop-picture-cells-95k-20261102/cell_reader.py @@ -0,0 +1,95 @@ +"""Physical-line cell reads; original95k, no date repair or text substitution.""" +import math +import cv2 +import numpy as np + +def owned_hand_cells(rgb,cells): + """Assign whole connected ink components once, rather than cut at virtual grids. + + Pale paper rulings do not participate. Recognition keeps original stroke + pixels plus their antialiasing fringe on a white, row-isolated canvas. + """ + g=cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY) + value,_=cv2.threshold(g,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU) + fg=(gA or yB) + out=[] + for ids,c in zip(owners,cells): + m=np.isin(labels,ids) + yy,xx=np.where(m) + if not len(xx):out.append((np.full_like(rgb,255),c['bbox']));continue + halo=cv2.dilate(m.astype(np.uint8),np.ones((3,3),np.uint8))>0 + canvas=np.full_like(rgb,255);canvas[halo]=rgb[halo] + out.append((canvas,[float(xx.min()),float(yy.min()),float(xx.max()+1),float(yy.max()+1)])) + return out,dict(connected_components_assigned=sum(map(len,owners)),components_crossing_inferred_cell_edges=crossings,ink_components_never_split=True,original_stroke_pixels_preserved=True) + +def pen_score(rgb): + a=rgb.astype(float);g=cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY) + dark=g<130 + return float(((a[:,:,2]-a[:,:,0]>12)&(a[:,:,1]-a[:,:,0]>3)&dark).sum()/max(1,dark.sum())) + +def prepare(rgb,pen=False): + g=cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY) + work=g.copy() + if pen: + a=rgb.astype(float) + # Pale cyan ruled-paper lines differ from the darker blue/black pen. + # Erase only long light rulings, retaining strokes crossing the rule. + cyan=((a[:,:,1]-a[:,:,0])>12)&((a[:,:,1]-a[:,:,2])>0)&(g>130) + work[cyan]=255 + val,_=cv2.threshold(work,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU) + ink=work0;work[rules]=255;ink[rules]=False + return work,ink,dict(pen_mask=pen,peripheral_rule_pixels=int(rules.sum()),interior_separators_not_removed=True) + + +def read_cell(reader,flip,rgb,bbox,segment,*,pen=False): + H,W=rgb.shape[:2];x,y,X,Y=bbox + a,b,A,B=max(0,math.floor(x)-3),max(0,math.floor(y)-3),min(W,math.ceil(X)+3),min(H,math.ceil(Y)+3) + local=rgb[b:B,a:A] + if not local.size:return '',[],dict(empty=True) + work,ink,trace=prepare(local,pen) + if ink.sum()<3:return '',[],dict(trace,empty=True) + binary=np.where(ink,0,255).astype(np.uint8) + proposals,pt=segment(binary) + if not proposals:proposals=[[0,0,work.shape[1],work.shape[0]]] + proposals.sort(key=lambda q:(q[1]+q[3])/2) + # A cell's columns are atomic; only physical horizontal rows are split. + centres=[(q[1]+q[3])/2 for q in proposals] + cuts=[0]+[round((c+d)/2)for c,d in zip(centres,centres[1:])]+[work.shape[0]] + lines=[] + for top,bottom in zip(cuts,cuts[1:]): + ys,xs=np.where(ink[top:bottom]) + if len(ys)<3:continue + # Tight on ALL ink, including one-pixel colon dots and slash tips; + # crop_line64 supplies the unchanged20/30percent training margins. + core=[float(xs.min()),float(ys.min()),float(xs.max()+1),float(ys.max()+1)] + plane=work[top:bottom].copy() + _,receipts=reader(plane,np.asarray([core],np.float32)) + r=receipts[0];offset=[a,b+top,a,b+top] + for key in ['ink_box','padded_source_extent']: + r[key]=[float(v)+o for v,o in zip(r[key],offset)] + r['source_pixels_extent']=[max(a,r['padded_source_extent'][0]),max(b+top,r['padded_source_extent'][1]),min(A,r['padded_source_extent'][2]),min(b+bottom,r['padded_source_extent'][3])] + r['original_WH']=[W,H] + r['frame_centers']=[float(v)+a for v in r['frame_centers']] + r['row_mask_isolation']=True;r['pen_mask_applied']=pen + line=dict(box=[v+o for v,o in zip(core,offset)],text=flip(r['raw_visual_text']),visual_text=r['raw_visual_text'],confidence=r['confidence'],geometry=r,recognizer='handwriting95k_pen_mask'if pen else'printed95k_cell_contract') + lines.append(line) + return ' '.join(q['text'].strip()for q in lines if q['text'].strip()),lines,dict(trace,empty=not bool(lines),proposal_trace=pt,padded_bbox=[a,b,A,B],physical_line_count=len(lines),same_training_contract=True) diff --git a/stallion/bina04-crop-picture-cells-95k-20261102/junk_filter.py b/stallion/bina04-crop-picture-cells-95k-20261102/junk_filter.py new file mode 100644 index 0000000000000000000000000000000000000000..dc442b93e21b66205332e15b57aa82ce82e97627 --- /dev/null +++ b/stallion/bina04-crop-picture-cells-95k-20261102/junk_filter.py @@ -0,0 +1,54 @@ +"""Conservative crop-only graphic junk filter; threshold from dev only.""" +import copy + +def clean(s):return ''.join(s.split()).replace('\u200c','').replace('\u200d','') +def area(b):return max(0,b[2]-b[0])*max(0,b[3]-b[1]) +def overlap(a,b):return area([max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])])/max(1,area(a)) +def rect(r): + if 'bbox'in r:return r['bbox'] + p=r['polygon'];return[min(q[0]for q in p),min(q[1]for q in p),max(q[0]for q in p),max(q[1]for q in p)] + +def context(line,pipeline): + b=line['box'];h=max(2,b[3]-b[1]);cy=(b[1]+b[3])/2 + layouts=pipeline.get('raw_layout',{}).get('bboxes',[]) + text_support=any(overlap(b,rect(r))>=.5 for r in layouts if r.get('label','').lower()in['text','sectionheader','pageheader','caption','listitem','formula','table']) + picture=any(overlap(b,rect(r))>=.8 for r in layouts if r.get('label','').lower()in['picture','figure','image']) + neighbours=[] + for q in pipeline.get('lines',[]): + if q is line or len(clean(q['text']))<4 or q.get('confidence',0)<.92:continue + z=q['box'];hh=max(2,z[3]-z[1]);d=abs(cy-(z[1]+z[3])/2);scale=max(h,hh) + xgap=max(0,max(b[0],z[0])-min(b[2],z[2]));xo=max(0,min(b[2],z[2])-max(b[0],z[0])) + same=d<.6*scale and xgap<3*scale + stacked=d<1.25*scale and xo>.7*min(b[2]-b[0],z[2]-z[0]) + if same or stacked:neighbours.append(z) + return dict(text_layout_support=text_support,picture=picture,neighbouring_text=bool(neighbours)) + +def decision(line,pipeline,threshold): + t=clean(line['text']);c=context(line,pipeline) + # Tables have their own cell path; never suppress a table digit by this rule. + table=any('grid' in r.get('path','').lower() for r in pipeline.get('regions',[])) + short=len(t)<=3;weak=line.get('confidence',0).06*H or Y-y<3:continue + x=max(0,x);y=max(0,y);X=min(W,X);Y=min(H,Y);g=gray[y:Y,x:X] + if not g.size:continue + val,_=cv2.threshold(g,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU);ink=g0)).sum()/max(1,n));rec=dict(box=b,source_ink=n,emitted_ink_fraction=fraction);checked.append(rec) + if n>=20 and fraction<.5:failed.append(rec) + return dict(fallback_required=len(failed)>=3,failed_primary_lines=failed,checked=checked,criterion='At least3 wide primary detector lines have <50% ink covered by emitted crops; same pixels, no text labels used') + +def ltr_decode(line): + from indra_ocr.rtl import flip,has_rtl + if line.get('ltr_repair'):return line + old=line['text'];g=line.get('geometry',{});raw=g.get('raw_visual_text',line.get('visual_text','')) + # Tatweel is a layout extender. Remove it before choosing runs, never after reordering. + if raw and any(c.isascii()and c.isalpha()for c in raw): + v=raw.replace('\u0640','');t=flip(v,base_rtl=True) + # Literal email punctuation can be joined without inventing letters or domains. + t=re.sub(r'(?<=[A-Za-z0-9])\s*([@.])\s*(?=[A-Za-z0-9])',r'\1',t) + t=re.sub(r'(?<=[A-Za-z0-9])\s+([.])\s*(?=[A-Za-z0-9])',r'\1',t) + if t!=old:return dict(line,text=t,document_decode=dict(original=old,raw_visual_used=True,tatweel_removed_before_bidi=True)) + return line + +def row_order(lines): + rows=[] + for l in sorted(lines,key=lambda l:((l['box'][1]+l['box'][3])/2,-l['box'][0])): + b=l['box'];found=False + for group in rows[-3:]: + g=group[0]['box'];vy=min(b[3],g[3])-max(b[1],g[1]);vx=min(b[2],g[2])-max(b[0],g[0]) + if vy>=.5*min(b[3]-b[1],g[3]-g[1])and vx<=0:group.append(l);found=True;break + if not found:rows.append([l]) + return [l for group in rows for l in sorted(group,key=lambda l:-l['box'][0])] +def join_fields(lines,W): + result=copy.deepcopy(lines);used=set() + for i,key in enumerate(result): + b=key['box'];t=key['text'].strip() + if not t or t[-1]not in ':،' or len(t)>20 or b[2]-b[0]>.25*W:continue + options=[] + for j,v in enumerate(result): + if j==i or j in used:continue + z=v['box'];s=''.join(v['text'].split());value=bool(s)and(all(c.isdigit()or c in '/:.-٫،'for c in s)or bool(re.search(r'[@.]',s)and any(c.isascii()and c.isalpha()for c in s))) + h=max(b[3]-b[1],z[3]-z[1]);distance=abs((b[1]+b[3]-z[1]-z[3])/2);gap=b[0]-z[2] + if value and 0<=gap<=3*h and distance<=1.5*h:options.append((distance,j)) + if options: + _,j=min(options);v=result[j];key['text']=t+' '+v['text'].strip();key['joined_field_evidence']=[copy.deepcopy(lines[i]),copy.deepcopy(lines[j])];used.add(j) + return [l for i,l in enumerate(result)if i not in used] + +def serialize(image,pipeline,graphic_threshold=.75): + out=copy.deepcopy(pipeline);W,H=image.size;layout=out.get('raw_layout',{}).get('bboxes',[]);graphics=[box(r)for r in layout if str(r.get('label','')).lower()in ['picture','figure','image','diagram']];removed=[];lines=[] + for l in out['lines']: + if l.get('uncertain_graphic')or any(overlap(l['box'],r)>=.6 for r in graphics)and l.get('confidence',0).4*W and i not in narrow] + if len(narrow)<3 or len(wide)<3:continue + if side=='right':gap=min(b[i][0]for i in narrow)-max(b[i][2]for i in wide) + else:gap=min(b[i][0]for i in wide)-max(b[i][2]for i in narrow) + if gap>max(5,.005*W):candidates.append((len(narrow),side,narrow)) + if candidates: + _,side,ids=max(candidates);ids=set(ids);main=[l for i,l in enumerate(lines)if i not in ids];sidebar=[l for i,l in enumerate(lines)if i in ids];main=row_order(join_fields(main,W));sidebar=row_order(join_fields(sidebar,W));lines=main+sidebar + else:side=None + # Keep generic grid serialization atomic instead of flattening cells. + if any(r.get('table_trace',{}).get('grid')for r in pipeline['regions']):return copy.deepcopy(pipeline) + out.update(lines=lines,regions=[dict(path='document_columns',lines=lines,text='\n'.join(l['text']for l in lines))],text='\n'.join(l['text']for l in lines),global_readings=lines,emitted_boxes=[l['box']for l in lines],recognition_extents=[l.get('geometry',{}).get('source_pixels_extent',l['box'])for l in lines]);out['document_handoff']=dict(graphic_confidence_threshold=graphic_threshold,graphic_reads_removed=removed,narrow_sidebar=side,main_column_before_sidebar=bool(side),no_document_identity_used=True);return out diff --git a/stallion/bina04-document-handoff-95k-20261103/numeric_repairs.py b/stallion/bina04-document-handoff-95k-20261103/numeric_repairs.py new file mode 100644 index 0000000000000000000000000000000000000000..39dfa52b2c81299126e8fc08edd2b8ae59b692a2 --- /dev/null +++ b/stallion/bina04-document-handoff-95k-20261103/numeric_repairs.py @@ -0,0 +1,106 @@ +"""Image-grounded numeric groups and LTR runs; no values, hosts or words hardcoded.""" +import re,copy,collections,unicodedata +import cv2,numpy as np +from PIL import Image +MAP=str.maketrans('٠١٢٣٤٥٦٧٨٩','۰۱۲۳۴۵۶۷۸۹') +def slash_pixels(gray): + _,m=cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU);yy,xx=np.where(m>0) + if len(xx)<8:return False + w=xx.max()-xx.min()+1;h=yy.max()-yy.min()+1 + if not(.120;yy,xx=np.where(ink) + if not len(xx):return gray,[] + h=yy.max()-yy.min()+1;counts=ink.sum(0);thin=(counts>0)&(counts<=max(2,.12*h));e=np.diff(np.r_[False,thin,False].astype(int));remove=[];trace=[] + for a,b in zip(np.flatnonzero(e==1),np.flatnonzero(e==-1)): + if b-agray.shape[1]-.25*h:continue + ys=np.where(ink[:,a:b])[0] + if not len(ys)or ys.max()-ys.min()+1>max(3,.2*h):continue + left=np.where(ink[:,max(0,a-round(h)):a])[0];right=np.where(ink[:,b:min(gray.shape[1],b+round(h))])[0] + if not len(left)or not len(right)or np.ptp(left)<.4*h or np.ptp(right)<.4*h:continue + keep=max(2,round(.12*h));remove+=list(range(a+keep,b));trace.append(dict(x0=int(a),x1=int(b),kept=keep)) + if not remove:return gray,[] + return np.delete(gray,remove,axis=1),trace + +def url_normal(s): + s=s.replace('\u0640','');s=re.sub(r'(?<=[A-Za-z0-9])\s*([@.٫])\s*(?=[A-Za-z0-9])',lambda m:'.'if m[1]=='٫'else m[1],s) + return s.strip() +def url_valid(s):return bool(re.fullmatch(r'(?:https?://)?(?:[A-Za-z0-9._%+\-]+@)?(?:[A-Za-z0-9\-]+\.)+[A-Za-z]{2,16}(?:[/:?#][\x21-\x7e]*)?',s)) +class Repairs: + def __init__(self,primary,numeric,short):self.readers=dict(primary=primary,numeric=numeric,short=short) + def read(self,gray,boxes,scales=(3,)): + from text_contract import decode_prediction + rows=[[]for b in boxes] + for name,reader in self.readers.items(): + for scale in scales: + work=cv2.resize(gray,None,fx=scale,fy=scale,interpolation=cv2.INTER_CUBIC)if scale!=1 else gray + for start in range(0,len(boxes),24): + bb=boxes[start:start+24];_,gg=reader(work,np.asarray([[v*scale for v in b]for b in bb],np.float32)) + for j,g in enumerate(gg):rows[start+j].append(dict(reader=name,scale=scale,text=decode_prediction(g['raw_visual_text']),raw=g['raw_visual_text'],confidence=float(g['confidence']))) + return rows + def dates(self,image,line): + raw=line.get('geometry',{}).get('raw_visual_text',line.get('visual_text',''));xs=line.get('geometry',{}).get('frame_centers',[]);b=line['box'];h=b[3]-b[1] + if len(xs)!=len(raw)or not any(unicodedata.bidirectional(c)in('R','AL')for c in raw):return line + spans=[];self.date_diagnostics=[];gray=np.asarray(image.convert('L'));H,W=gray.shape + for m in re.finditer(r'[\d\s/]{3,}',raw): + if m.group().count('/')<2 or not re.search(r'\d\s+/|/\s+\d',m.group()):continue + xx=xs[m.start():m.end()];a=max(0,round(min(xx)-.2*h));A=min(W,round(max(xx)+.2*h));y=max(0,round(b[1]-2));Y=min(H,round(b[3]+2));g=gray[y:Y,a:A];_,ink=cv2.threshold(g,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU);active=(ink>0).sum(0)>1;e=np.diff(np.r_[False,active,False].astype(int));groups=[] + for x,X in zip(np.flatnonzero(e==1),np.flatnonzero(e==-1)): + if groups and x-groups[-1][1]<=.3*h:groups[-1][1]=int(X) + else:groups.append([int(x),int(X)]) + boxes=[[max(0,a+x-2),y,min(W,a+X+2),Y]for x,X in groups];reads=self.read(gray,boxes);tokens=[];evidence=[] + for bb,rr in zip(boxes,reads): + votes=collections.defaultdict(dict) + for r in rr: + t=''.join(r['text'].split()).translate(MAP) + if re.fullmatch(r'[\d/]+',t)and any(c.isdigit()for c in t):votes[t][r['reader']]=r['confidence'] + values=[t for t,models in votes.items()if len(models)>=2 and np.mean(sorted(models.values(),reverse=True)[:2])>=.9] + if len(values)==1:tokens.extend(re.findall(r'\d+|/',values[0]));evidence.append(dict(box=bb,value=values[0],reads=rr)) + elif slash_pixels(gray[bb[1]:bb[3],bb[0]:bb[2]]):tokens.append('/');evidence.append(dict(box=bb,value='/',source='diagonal source ink')) + else:tokens.append(None);evidence.append(dict(box=bb,value=None,reads=rr)) + self.date_diagnostics.append(dict(raw_span=m.group(),tokens=tokens,evidence=evidence)) + # Ignore flanking letters; require every internal group to have source evidence. + good=[i for i,t in enumerate(tokens)if t is not None] + if not good:continue + ts=tokens[min(good):max(good)+1] + if None in ts:continue + nums=[t for t in ts if t!='/'];rev=list(reversed(ts));valid=False + if len(nums)==3 and len(nums[0])==4 and int(nums[0])>=1000 and 1<=int(nums[1])<=12 and 1<=int(nums[2])<=31 and ts.count('/')==2:valid=True + elif len(nums)==2 and ts.count('/')==2 and ts[0]=='/' and 1<=int(nums[0])<=12 and 1<=int(nums[1])<=31:valid=True + if valid:spans.append((m.start(),m.end(),' '.join(rev),evidence,m.group())) + if not spans:return line + result=copy.deepcopy(line);text=line['text'];replacements=[] + for start,end,new,evidence,old_run in spans: + digits=lambda t:''.join(c for c in t.translate(MAP)if c.isdigit()) + matches=[m for m in re.finditer(r'[\d\s/]{3,}',text)if m.group().count('/')>=2 and digits(m.group())==digits(old_run)] + if len(matches)!=1:continue + m=matches[0];a=m.start();A=m.end();leading=' 'if m.group()[:1].isspace()else'';trailing=' 'if m.group()[-1:].isspace()else'';text=text[:a]+leading+new+trailing+text[A:];replacements.append(dict(original=m.group(),new=new,evidence=evidence)) + if not replacements:return line + result['text']=text;result['numeric_group_reboxing']=dict(original=line['text'],source_groups=replacements,values_require_two_model_agreement=True,words_outside_numeric_runs_preserved=True);return result + def latin(self,image,line): + old=line['text'];raw=line.get('geometry',{}).get('raw_visual_text',line.get('visual_text',''));base=url_normal(raw) + if not any(k in raw.lower()for k in ['www','http','@']):return line + if url_valid(base):return dict(line,text=base,ltr_repair=dict(source='original visual glyph run',original=old)) + gray=np.asarray(image.convert('L'));rr=self.read(gray,[line['box']],scales=(1,3))[0];candidates=[] + for r in rr: + t=url_normal(r['raw']) + if not url_valid(t)or r['confidence']<.85:continue + key=re.sub('[^A-Za-z0-9]','',t).lower();models={s['reader']for s in rr if re.sub('[^A-Za-z0-9]','',s['raw']).lower()==key} + same=[s for s in rr if url_normal(s['raw'])==t] + if len(models)>=2 and len({s['scale']for s in same})==2:candidates.append((r['confidence'],t,r)) + if not candidates:return line + _,t,r=max(candidates);return dict(line,text=t,ltr_repair=dict(source='two-model alphanumeric agreement; two-scale punctuation agreement',original=old,reader=r['reader'])) + def extenders(self,image,line): + t=line['text'] + if len(t)>30 or any(c.isdigit()or c.isascii()and c.isalpha()for c in t):return line + x,y,X,Y=map(round,line['box']);gray=np.asarray(image.convert('L'))[y:Y,x:X] + if not gray.size:return line + work,trace=compress_extenders(gray) + if not trace:return line + rr=self.read(work,[[0,0,work.shape[1],work.shape[0]]])[0];votes=collections.defaultdict(set) + for r in rr: + if r['confidence']>=.9:votes[r['text'].strip()].add(r['reader']) + options=[s for s,v in votes.items()if len(v)>=2 and s] + if len(options)!=1:return line + return dict(line,text=options[0],extender_repair=dict(source='source-pixel horizontal connector compression; two-model agreement',original=t,compressed_spans=trace)) diff --git a/stallion/bina04-document-handoff-95k-20261103/production.py b/stallion/bina04-document-handoff-95k-20261103/production.py new file mode 100644 index 0000000000000000000000000000000000000000..fef402ca088be2d458225463261d307d650a9be4 --- /dev/null +++ b/stallion/bina04-document-handoff-95k-20261103/production.py @@ -0,0 +1,38 @@ +"""A page handoff cannot discard primary text ink. OCR remains image-grounded.""" +import importlib.util,json,sys,copy +from pathlib import Path +R=Path(__file__).resolve().parent;V=Path('__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031') +class Runtime: + def __init__(self,device='auto'): + s=importlib.util.spec_from_file_location('frozen_header_parent','__BINA_ROOT__/stallion/bina04-header-marks-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-header-marks-95k-20261103');sys.path.insert(0,str(R));from numeric_repairs import Repairs + numeric=self.parent.parent;numeric.capture.reset();self.core=numeric.engine.parent.parent;self.repair=Repairs(self.core.patch.reader.reader,numeric.numeric,self.core.patch.overlay.layer.reader);self.generic=None + def generic_page(self,image,old): + if self.generic is None: + sys.path.insert(0,str(V));from pipeline_v3 import PagePipeline,UnifiedReader + import numpy as np + class WholeLinePipeline(PagePipeline): + def detect(self,image): + boxes,trace=super().detect(image);heights=[b[3]-b[1]for b in boxes if b[3]>b[1]];median=float(np.median(heights))if heights else 1.;keep=[];dropped=[] + from document import overlap + for i,b in enumerate(boxes): + if any(i!=j and overlap(b,a)>=.8 and b[3]-b[1]<.6*(a[3]-a[1])and a[3]-a[1]<=2*median for j,a in enumerate(boxes)):dropped.append(b) + else:keep.append(b) + trace['contained_partial_height_slivers_removed']=dropped;return keep,trace + self.generic=WholeLinePipeline(self.core.patch.detector,self.core.patch.base.layout,UnifiedReader(V/'run/weights-095000.pt',V/'data/vocab.json',device=str(self.core.patch.reader.reader.device)),self.core.patch.table) + self.core.context['layout']=copy.deepcopy(old['raw_layout']);return self.generic(image,input_type='page') + def __call__(self,image,old,kind,name='',**kwargs): + base=self.parent(image,old,kind,name,**kwargs) + return self.finish(image,base,old,kind) + def finish(self,image,base,old,kind): + from document import lost_main_text,serialize + if kind!='page':return base + guard=lost_main_text(image,base) + if not guard['fallback_required']:return base + out=self.generic_page(image,old);out['ink_handoff_guard']=guard + from region_reads import supplement,display_evidence + out=supplement(image,out,self.repair) + for i,line in enumerate(out['lines']): + line=self.repair.dates(image,line);line=self.repair.latin(image,line);line=self.repair.extenders(image,line);out['lines'][i]=line + out=display_evidence(image,out,self.repair) + return serialize(image,out,graphic_threshold=.75) + def close(self):self.parent.close() diff --git a/stallion/bina04-document-handoff-95k-20261103/region_reads.py b/stallion/bina04-document-handoff-95k-20261103/region_reads.py new file mode 100644 index 0000000000000000000000000000000000000000..95f842d098ba09b1878f8494e8cce20a74520222 --- /dev/null +++ b/stallion/bina04-document-handoff-95k-20261103/region_reads.py @@ -0,0 +1,59 @@ +"""General small text-box reads and independent-reader evidence on display fonts.""" +import copy,re,unicodedata +from difflib import SequenceMatcher +import cv2,numpy as np +from PIL import Image +from document import box,overlap + +def ns(s):return ''.join(s.split()).replace('\u0640','') +def consensus(rows,minimum=.85): + groups={} + for r in rows: + key=ns(r['text']) + if not key:continue + groups.setdefault(key,[]).append(r) + good=[] + for k,rr in groups.items(): + models={r['reader']for r in rr} + if len(models)>=2 and np.mean(sorted([r['confidence']for r in rr],reverse=True)[:2])>=minimum:good.append((len(models),max(r['confidence']for r in rr),rr)) + if not good:return None + _,_,rr=max(good,key=lambda a:a[:2]);return max(rr,key=lambda r:r['confidence']) + +def supplement(image,pipeline,repairs): + import sys + sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-crop-bandseg-95k-20261102') + from bandseg import mask,horizontal + W,H=image.size;result=copy.deepcopy(pipeline);new=[];debug=[] + for region in result.get('raw_layout',{}).get('bboxes',[]): + if region.get('label','').lower()!='text':continue + b=box(region);x,y,X,Y=map(round,b) + if X-x>.3*W or Y-y>.12*H or Y-y<6:continue + rgb=np.asarray(image.crop((x,y,X,Y)).convert('RGB'));g=rgb.min(2).astype('float32');bg=cv2.GaussianBlur(g,(0,0),max(7,(Y-y)/8));norm=np.clip(g/(bg+1e-4)*245,0,255).astype('uint8');work=cv2.resize(norm,None,fx=3,fy=3,interpolation=cv2.INTER_CUBIC);ink,glyph=mask(work);boxes,trace=horizontal(ink,glyph,scale=3) + if not boxes:continue + reads=repairs.read(work,boxes,scales=(1,)) + for bb,rows in zip(boxes,reads): + target=[bb[0]/3+x,bb[1]/3+y,bb[2]/3+x,bb[3]/3+y] + if any(l.get('text','').strip()and overlap(target,l['box'])>=.6 and l['box'][3]-l['box'][1]<=1.8*(target[3]-target[1])for l in result['lines']): + debug.append(dict(box=target,covered=True,reads=rows));continue + chosen=consensus(rows,.9);debug.append(dict(box=target,chosen=chosen,reads=rows)) + if chosen is None:continue + # A tighter reread of the same glyphs is not a second text line. + if any(l.get('text','').strip() and overlap(target,l['box'])>=.8 and SequenceMatcher(None,ns(chosen['text']),ns(l['text'])).ratio()>=.7 for l in result['lines']):continue + if any(overlap(target,l['box'])>.5 and ns(chosen['text'])==ns(l['text'])for l in new):continue + new.append(dict(box=target,text=chosen['text'],visual_text=chosen['raw'],confidence=chosen['confidence'],recognizer='existing_reader_consensus_small_textbox',geometry=dict(source_pixels_extent=target,padded_source_extent=target,raw_visual_text=chosen['raw'],frame_centers=[],frame_centers_source_valid=False,source_pixels_modified=True),source_region=region,small_textbox_read=dict(upscale=3,background_normalized=True,readers=rows))) + result['lines']+=new;result['small_textbox_supplement']=dict(added=new,template_text_used=False,diagnostics=debug);return result + +def display_evidence(image,pipeline,repairs): + W,H=image.size;result=copy.deepcopy(pipeline);gray=np.asarray(image.convert('L'));graphic=[r for r in result.get('raw_layout',{}).get('bboxes',[])if r.get('label','').lower()in['picture','figure','image','diagram']] + for i,line in enumerate(result['lines']): + b=line['box'];g=max([r.get('confidence',0)for r in graphic if overlap(b,box(r))>=.6]or[0.]);edge=(b[0]+b[2])/2>.75*W or(b[0]+b[2])/2<.25*W;display=edge and b[3]-b[1]>.018*H and b[2]-b[0]<.3*W + if not(g or display):continue + if g: + score=line.get('confidence',0)*(1-g);line['document_text_confidence']=score + # The raw read is retained in provenance even when not emitted as document text. + if score<.75:line['uncertain_graphic']=True;continue + if display: + rr=repairs.read(gray,[b],scales=(1,3))[0];chosen=consensus(rr,.9);models={r['reader']for r in rr if chosen and ns(r['text'])==ns(chosen['text'])};line['display_font_evidence']=rr + if chosen and len(models)==3:line['text']=chosen['text'] + else:line['uncertain_graphic']=True + return result diff --git a/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/learned_backend.py b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/learned_backend.py new file mode 100644 index 0000000000000000000000000000000000000000..93cbde729be34e9db8c48c63b76143a70cc529e2 --- /dev/null +++ b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/learned_backend.py @@ -0,0 +1,30 @@ +"""Substitute the trained detector in existing newspaper/cell line roles. + +Keep the frozen guard, margins, NMS, grid, ordering and recognizer code intact. +The SDK preprocessing used here is also used for synthetic page/cell training. +""" +import hashlib +import numpy as np +from PIL import Image +def install(detector,model_sha256,yield_check=lambda:None): + import baseline.crop_line_guard as module + original=module.CropLineGuard + cache={} + class Engine: + long_side=0 + def detect(self,gray): + key=hashlib.sha256(str(gray.shape).encode()+gray.tobytes()).hexdigest() + if key not in cache: + yield_check();raw=detector([Image.fromarray(gray).convert('RGB')],batch_size=1)[0].model_dump(mode='json');rows=[] + for item in raw['bboxes']: + p=np.asarray(item['polygon']);rows.append([float(p[:,0].min()),float(p[:,1].min()),float(p[:,0].max()),float(p[:,1].max()),float(item.get('confidence',0.))]) + cache[key]=np.asarray(rows,np.float32).reshape(-1,5) + return cache[key].copy() + class LearnedGuard(original): + def __init__(self):self.engine=Engine();self.sha256=model_sha256 + def __call__(self,*args,**kwargs): + boxes,trace=super().__call__(*args,**kwargs) + trace.update(detector='Selected learned Surya physical-line model',model_sha256=model_sha256,providers=['CUDA FP32 SDK'],legacy_long_side_not_used_by_SDK=True) + return boxes,trace + module.CropLineGuard=LearnedGuard + return dict(model_sha256=model_sha256,roles=['whole_page','native_context','newspaper_guard_and_rescan','table_cell_lines'],recognizer_preprocessing_changed=False,heuristic_rules_changed=False) diff --git a/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/run/checkpoints/step-001000.pt b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/run/checkpoints/step-001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0fbbfda332c5ae7ced0e6a4996a2fc282726344 --- /dev/null +++ b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/run/checkpoints/step-001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3208fac304f7c8a64735587f1e46024acb51cc0b0ef2d2288944eadd8992d4c9 +size 153922431 diff --git a/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/crop_line_guard.py b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/crop_line_guard.py new file mode 100644 index 0000000000000000000000000000000000000000..2e5d8f5a0e1a636644039f36bf0a9bb71c32aea9 --- /dev/null +++ b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/crop_line_guard.py @@ -0,0 +1,39 @@ +"""Crop-only anti-merge detector. Original Indra LineNet; no recognizer banks.""" +import hashlib,json +from pathlib import Path +import numpy as np +MODELS=Path('__BINA_ROOT__/stallion/bina04-indra-pilot-20261027/models') + +class CropLineGuard: + def __init__(self): + import onnxruntime as ort + from indra_ocr.pipeline import OrtOCR + so=ort.SessionOptions();so.intra_op_num_threads=1;so.inter_op_num_threads=1;so.enable_cpu_mem_arena=False + self.engine=OrtOCR.__new__(OrtOCR) + self.engine.det=ort.InferenceSession(str(MODELS/'det.onnx'),so,providers=['CPUExecutionProvider']) + self.engine.adaptive=False + self.sha256=hashlib.sha256((MODELS/'det.onnx').read_bytes()).hexdigest() + def __call__(self,gray,boxes,line): + long_side=max(32,round(max(gray.shape)*12/max(1,line)/32)*32) + self.engine.long_side=long_side + native=self.engine.detect(gray)[:,:4];out=[];used=set();events=[] + for box in boxes: + x,y,X,Y=box;candidate=[] + if Y-y>1.8*line: + for i,b in enumerate(native): + a,t,A,T=map(float,b);h=T-t + overlap=max(0,min(A,X)-max(a,x))*max(0,min(T,Y)-max(t,y))/max(1.,(A-a)*h) + if i not in used and overlap>=.7 and .5*line<=h<=2.*line:candidate.append(i) + groups=[] + for i in sorted(candidate,key=lambda i:(native[i,1]+native[i,3])/2): + center=float((native[i,1]+native[i,3])/2) + if not groups or center-groups[-1][-1][0]>.6*line:groups.append([]) + groups[-1].append((center,i)) + if len(groups)>=2: + replacements=[] + for group in groups: + members=[native[i]for _,i in group] + replacements.append([min(float(b[0])for b in members),min(float(b[1])for b in members),max(float(b[2])for b in members),max(float(b[3])for b in members)]) + out+=replacements;used.update(candidate);events.append(dict(original_box=box,replacement_boxes=replacements,native_indices=candidate)) + else:out.append(list(box)) + return out,dict(detector='Original Indra LineNet only; no OCR/script bank',model_sha256=self.sha256,providers=['CPUExecutionProvider'],long_side=long_side,target_input_line_height=12,native_boxes=native.tolist(),replacements=events,reference_or_layout_used=False) diff --git a/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/input_policy_v3.py b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/input_policy_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..b936fb87e708b2a5b831edc1e2ce60f95d7dd748 --- /dev/null +++ b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/input_policy_v3.py @@ -0,0 +1,96 @@ +"""Frozen rule C estimator and same-line sliver consolidation; image geometry only.""" +import numpy as np + +def input_kind(width,height,requested='auto'): + if requested not in {'auto','crop','page'}:raise ValueError('input_type must be auto/crop/page') + if requested!='auto':return requested + return 'crop' if min(width,height)<=384 or height<=128 or(height<=768 and width/max(1,height)>=2.5) else 'page' + + +def estimated_line_height(gray): + # Exact estimator from the previously frozen rule C. No labels/categories. + import cv2 + H, W = gray.shape + _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + ink = (binary == 0).astype(np.uint8) + if ink.mean() > .5: + ink = 1 - ink + _, labels, stats, _ = cv2.connectedComponentsWithStats(ink, 8) + heights = [];glyph_ids=[] + for label,(x, y, w, h, area) in enumerate(stats[1:],1): + aspect = float(w) / max(1, int(h)) + if area >= 6 and h >= 3 and w >= 2 and h <= .95 * H and .05 <= aspect <= 20.: + heights.append(int(h));glyph_ids.append(label) + glyph = float(np.percentile(heights, 75)) if heights else None + line = min(float(H), max(1., 1.5 * glyph)) if glyph is not None else float(H) + original_line=line + projection_glyph=glyph + noise_robust=False + projection_ids=glyph_ids + if H<=96 and glyph_ids: + selected=stats[glyph_ids];order=np.argsort(selected[:,3]);ordered=selected[order] + weighted=float(ordered[np.searchsorted(np.cumsum(ordered[:,4]),.75*ordered[:,4].sum()),3]) + if weighted>=2.*max(1.,glyph): + # Speckle can outnumber letters. Weight height by ink area and + # remove tiny speckles from this routing-only projection. + noise_robust=True;projection_glyph=weighted + projection_ids=[i for i in glyph_ids if stats[i,4]>=max(6,.05*selected[:,4].max())] + line=max(line,min(float(H),1.5*weighted)) + # Padded one-line strips have small glyphs inside a much taller canvas. + # Do not count their blank margins as extra lines. This is confined to + # short/wide inputs and a single compact ink band, never handwriting pages. + # Exclude full-height scan borders/rules from the projection, using the + # same glyph-component eligibility as the height estimator. + glyph_ink=np.isin(labels,projection_ids) if projection_ids else ink + active=glyph_ink.sum(axis=1)>=max(2,.0015*W) + gap=max(2,round(.35*(projection_glyph or 1.))) + edges=np.diff(np.r_[False,active,False].astype(np.int8)) + bands=[[int(a),int(b)] for a,b in zip(np.where(edges==1)[0],np.where(edges==-1)[0])] + joined=[] + for a,b in bands: + if joined and a-joined[-1][1]<=gap:joined[-1][1]=b + else:joined.append([a,b]) + padded_single=(H<=96 or W/max(1,H)>=3.) and len(joined)==1 and joined[0][1]-joined[0][0]<=max(6,3.5*(projection_glyph or H)) + if padded_single:line=float(H) + return dict(estimated_line_height=line, estimated_glyph_height=glyph, + valid_components=len(heights), crop_height=H, + height_to_line_height=H / max(1., line), segment=H > 1.8 * line, + original_glyph_based_line_height=original_line,ink_bands=joined, + single_ink_band_with_blank_margins=bool(padded_single), + noise_robust_projection=noise_robust,projection_glyph_height=projection_glyph) + + +def same_line(a, b): + overlap_y = min(a[3], b[3]) - max(a[1], b[1]) + overlap_x = min(a[2], b[2]) - max(a[0], b[0]) + return overlap_x > 0 and overlap_y > .5 * min(a[3] - a[1], b[3] - b[1]) + + +def merge_slivers(boxes, line_height=None): + """Union overlapping observations; complete-link groups avoid bridge chaining. + + A grown union is never used to decide subsequent matches. Every pair of + original members must overlap, or a plausible full-height anchor must + contain the sliver. A grown union never recruits a neighbouring row. + """ + groups = [] + for i in sorted(range(len(boxes)), key=lambda i: (-(boxes[i][3]-boxes[i][1]), i)): + compatible = [] + for j, group in enumerate(groups): + anchor=boxes[group[0]];box=boxes[i] + inside=(anchor[0]<=box[0] and anchor[1]<=box[1] and + anchor[2]>=box[2] and anchor[3]>=box[3]) + plausible=line_height is None or anchor[3]-anchor[1]<=1.8*line_height + if (inside and plausible) or all(same_line(box,boxes[k]) for k in group): + compatible.append(j) + if compatible: + groups[compatible[0]].append(i) + else: + groups.append([i]) + groups.sort(key=min) + merged = [[min(boxes[i][0] for i in g), min(boxes[i][1] for i in g), + max(boxes[i][2] for i in g), max(boxes[i][3] for i in g)] for g in groups] + return merged, dict(input_boxes=len(boxes), output_boxes=len(merged), + original_member_indices=groups, + criterion='Vertical overlap >50% smaller height and positive horizontal overlap; complete-link unions', + reference_text_or_boxes_used=False) diff --git a/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/pipeline_v3.py b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/pipeline_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..725d8c20b7771d696ac07be81e3438bbfadf5ff2 --- /dev/null +++ b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/pipeline_v3.py @@ -0,0 +1,216 @@ +"""Layout cannot filter detection. Exactly one recognizer reads every script.""" +import math, sys, time +from pathlib import Path +import numpy as np +from PIL import Image +P=Path('__BINA_ROOT__/stallion/indra-scale') +sys.path[:0]=[str(P),'__BINA_ROOT__/stallion/bina04-indra-pilot-20261027/upstream'] +from geometry import crop_line64 +from indra_ocr.rtl import flip,rtl_base + +def rectangle(polygon): + p=np.asarray(polygon,float) + return [float(p[:,0].min()),float(p[:,1].min()),float(p[:,0].max()),float(p[:,1].max())] + +def area(b):return max(0,b[2]-b[0])*max(0,b[3]-b[1]) +def intersection(a,b):return [max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])] +def clipped(b,w,h):return [max(0,b[0]),max(0,b[1]),min(w,b[2]),min(h,b[3])] +def reading_extent(line):return line['geometry'].get('source_pixels_extent',line['geometry']['padded_source_extent']) + +def xycut_order(boxes,width): + """RTL column order; spanning headings divide vertical bands.""" + n=len(boxes) + raster=lambda ids:sorted(ids,key=lambda i:((boxes[i][1]+boxes[i][3])/2,-boxes[i][0])) + core=[b for b in boxes if b[2]-b[0]<.7*width] + if len(core)<4:return raster(range(n)) + profile=np.zeros(max(1,int(math.ceil(width))),int) + for b in core:profile[max(0,int(b[0])):min(len(profile),int(math.ceil(b[2])))]+=1 + low=profile<=max(1,int(.05*len(core)));low[:int(.25*width)]=False;low[int(.75*width):]=False + e=np.diff(np.r_[False,low,False].astype(int));candidates=[] + for a,b in zip(np.where(e==1)[0],np.where(e==-1)[0]): + if b-a=cut] + if len(left)>=2 and len(right)>=2:candidates.append((b-a,cut,left,right)) + if not candidates:return raster(range(n)) + _,cut,left,right=max(candidates);middle=raster(set(range(n))-set(left)-set(right));remaining=set(left+right);out=[] + for stop in middle+[None]: + boundary=float('inf')if stop is None else(boxes[stop][1]+boxes[stop][3])/2 + for group in[right,left]: + take=raster(i for i in group if i in remaining and(boxes[i][1]+boxes[i][3])/20 else None + key=('region',j)if j is not None else('outside',i) + if key not in blocks:blocks[key]=dict(key=list(key),region=regions[j]if j is not None else None,line_ids=[]) + blocks[key]['line_ids'].append(i);assign.append(list(key)) + blocks=list(blocks.values());bounds=[] + for b in blocks: + members=[boxes[i]for i in b['line_ids']];bounds.append([min(x[0]for x in members),min(x[1]for x in members),max(x[2]for x in members),max(x[3]for x in members)]) + b['line_ids'].sort(key=lambda i:((boxes[i][1]+boxes[i][3])/2,-boxes[i][0])) + positions=[b['region'].get('position')if b['region']else None for b in blocks] + reasons=[] + if any(not isinstance(p,int)for p in positions)or len(set(positions))!=len(positions):reasons.append('missing_or_duplicate_region_order_or_outside_lines') + for i,a in enumerate(bounds): + for b in bounds[i+1:]: + if area(intersection(a,b))>.1*max(1,min(area(a),area(b))):reasons.append('overlapping_blocks');break + for block in blocks: + if block['region']and block['region'].get('label','').lower()=='table':continue + group=block['line_ids'];local=[boxes[i]for i in group];raster=sorted(range(len(group)),key=lambda i:((local[i][1]+local[i][3])/2,-local[i][0])) + if xycut_order(local,width)!=raster:reasons.append('multiple_columns_inside_layout_block') + if reasons: + # Apply the column fallback to physical lines, then form contiguous + # column chunks. Sorting the enclosing blocks alone interleaves any + # columns that Surya put inside one large Picture/Text region. + physical=xycut_order(boxes,width);owner={i:j for j,b in enumerate(blocks)for i in b['line_ids']};result=[];used_tables=set() + for i in physical: + j=owner[i];block=blocks[j];is_table=block['region']and block['region'].get('label','').lower()=='table' + if is_table: + if j not in used_tables:result.append(dict(block));used_tables.add(j) + elif result and result[-1].get('column_chunk_owner')==j and(boxes[i][1]+boxes[i][3])/2>=(boxes[result[-1]['line_ids'][-1]][1]+boxes[result[-1]['line_ids'][-1]][3])/2-.25*(boxes[i][3]-boxes[i][1]):result[-1]['line_ids'].append(i) + else:result.append(dict(block,line_ids=[i],column_chunk_owner=j)) + mode='RTL_physical_line_columns_with_atomic_tables' + else:result=[blocks[i]for i in sorted(range(len(blocks)),key=lambda i:positions[i])];mode='Surya_block_order' + assert sorted(i for b in result for i in b['line_ids'])==list(range(len(boxes))) + return result,dict(mode=mode,reasons=sorted(set(reasons)),assignments=assign,region_type_line_losses=0) + +class UnifiedReader: + """One model, unfiltered CTC, logical-order output. No script classifier.""" + def __init__(self,checkpoint,vocabulary,device='cpu'): + from rtl_reader import RTLReader + self.reader=RTLReader(checkpoint,vocabulary,device=device);self.source_is_crop=False + def __call__(self,gray,boxes): + boxes=np.asarray(boxes,np.float32).reshape(-1,4).tolist();normalization=None + if self.source_is_crop and len(boxes)==1 and np.allclose(boxes[0],[0.,0.,gray.shape[1],gray.shape[0]]): + from input_policy_v3 import estimated_line_height + from single_line_normalization import ink_band_box + estimate=estimated_line_height(gray) + if not estimate['segment']: + original=boxes[0];boxes=[ink_band_box(gray,estimate)] + normalization=dict(original_box=original,ink_band_box=boxes[0],one_full_width_line=True,no_detection_or_line_splitting=True) + if self.source_is_crop and boxes: + from line_isolation import atlas + canvas,shifted,maps=atlas(gray,boxes);_,receipts=self.reader(canvas,np.asarray(shifted,np.float32).reshape(-1,4)) + for receipt,mapping in zip(receipts,maps): + dx,dy=mapping['dx'],mapping['dy'];x,y,X,Y=receipt['padded_source_extent'];receipt.update(original_WH=[gray.shape[1],gray.shape[0]],padded_source_extent=[x-dx,y-dy,X-dx,Y-dy],source_pixels_extent=mapping['visible'],neighbor_line_pixels_excluded=True) + receipt['frame_centers']=[float(x-dx)for x in receipt['frame_centers']] + x,y,X,Y=receipt['ink_box'];receipt['ink_box']=[x-dx,y-dy,X-dx,Y-dy] + else:_,receipts=self.reader(gray,np.asarray(boxes,np.float32).reshape(-1,4)) + return[dict(box=list(map(float,b)),text=flip(r['raw_visual_text'],base_rtl=True),visual_text=r['raw_visual_text'],confidence=r['confidence'],recognizer='unified32M',geometry=r,confidence_filter_applied=False,whole_input_normalization=normalization)for b,r in zip(boxes,receipts)] + +def markdown_cell(text):return ' '.join(text.split()).replace('|',r'\|') + +class PagePipeline: + def __init__(self,detector,layout,reader,table=None): + self.detector=detector;self.layout=layout;self.reader=reader;self.table=table;self._crop_line_guard=None + def detect(self,image): + # The SDK performs its own resize/pad/tiling on the WHOLE input. + raw=self.detector([image],batch_size=1)[0].model_dump(mode='json');w,h=image.size + boxes=[clipped(rectangle(r['polygon']),w,h)for r in raw['bboxes']] + boxes=[b for b in boxes if area(b)>0] + # Fixed native-resolution windows are independent of every layout label. + # Full-width SDK downsampling otherwise makes dense columns/cells tiny. + def starts(size,tile,overlap): + if size<=tile:return [0] + out=list(range(0,size-tile+1,tile-overlap)) + if out[-1]!=size-tile:out.append(size-tile) + return out + traces=[];supplements=[];background=tuple(int(v)for v in np.median(np.asarray(image).reshape(-1,3),axis=0));size=max(self.detector.processor.size.values()) + def context(piece,x,y,scale): + # Padding prevents prepare_image's final square stretch from + # deforming glyphs. Recognition never uses these resized pixels. + pw,ph=piece.size;rw=max(1,round(pw*scale));rh=max(1,round(ph*scale));canvas=Image.new('RGB',(size,size),background);canvas.paste(piece.resize((rw,rh),Image.Resampling.LANCZOS),(0,0));part=self.detector([canvas],batch_size=1)[0].model_dump(mode='json');out=[] + for item in part['bboxes']: + a,b,A,B=rectangle(item['polygon']);a=max(0,min(rw,a));A=max(0,min(rw,A));b=max(0,min(rh,b));B=max(0,min(rh,B));v=clipped([a*pw/rw+x,b*ph/rh+y,A*pw/rw+x,B*ph/rh+y],w,h) + if area(v)>0:out.append(v) + traces.append(dict(tile=[x,y,x+pw,y+ph],scale=scale,canvas_WH=[size,size],boxes=out));supplements.extend(out) + for y in starts(h,size,240): + for x in starts(w,size,240):context(image.crop((x,y,min(w,x+size),min(h,y+size))),x,y,1.) + if max(w,h)<=1400:context(image,0,0,min(.5,size/max(w,h))) + from fusion_v3 import fuse + merged,fusion=fuse(np.asarray(image.convert('L')),boxes,supplements) + return merged,dict(full_image_SDK=raw,native_resolution_tiles=traces,fusion=fusion,layout_used_for_detection=False,recognizer_pixels='Original image only; detector context resize is inverted geometrically') + + def table_block(self,image,gray,block,boxes,readings): + region=block['region'];w,h=image.size;b=clipped(rectangle(region['polygon']),w,h);x,y,X,Y=map(int,[math.floor(b[0]),math.floor(b[1]),math.ceil(b[2]),math.ceil(b[3])]);crop=image.crop((x,y,X,Y)) + if self.table is None:return None,dict(error='Surya_table_engine_not_ready',all_original_lines_retained=True) + try:result=self.table(crop) + except Exception as e:return None,dict(error=type(e).__name__+': '+str(e),all_original_lines_retained=True) + cells=result.get('cells',[]) + if result.get('error')or not cells:return None,dict(error='Surya_table_grid_empty',grid=result,all_original_lines_retained=True) + cells=sorted(cells,key=lambda c:(c['row_id'],c.get('col_id')if c.get('col_id')is not None else c.get('within_row_id',0)));pieces=[];meta=[];represented=set();cell_fragments={i:[]for i in block['line_ids']} + for cell in cells: + cb=cell.get('bbox')or rectangle(cell['polygon']);cb=[cb[0]+x,cb[1]+y,cb[2]+x,cb[3]+y];parts=[] + for i in block['line_ids']: + part=intersection(boxes[i],cb) + if part[2]-part[0]>=2 and part[3]-part[1]>=.5*(boxes[i][3]-boxes[i][1]): + # A physical row inside a cell keeps the cell's horizontal + # context; only its vertical line band is cropped. + part=[cb[0],part[1],cb[2],part[3]];parts.append(part);represented.add(i);cell_fragments[i].append(part) + if not parts:parts=[cb] + parts.sort(key=lambda p:((p[1]+p[3])/2,-p[0]));bands=[] + for part in parts: + hit=next((j for j,b in enumerate(bands)if min(part[3],b[3])-max(part[1],b[1])>=.65*min(part[3]-part[1],b[3]-b[1])),None) + if hit is None:bands.append(part) + else:bands[hit]=[cb[0],min(bands[hit][1],part[1]),cb[2],max(bands[hit][3],part[3])] + begin=len(pieces);pieces+=bands;meta.append((cell,cb,begin,len(pieces))) + # A grid may not silently erase part of a detected line. A line entirely + # outside cells is a retained note; a partly clipped line fails back to + # the original block rather than emitting a misleading partial table. + import cv2 + coverage=[] + for i in represented: + b=boxes[i];a,c,A,C=map(int,[math.floor(b[0]),math.floor(b[1]),math.ceil(b[2]),math.ceil(b[3])]);patch=gray[c:C,a:A];_,ink=cv2.threshold(patch,0,255,(cv2.THRESH_BINARY if np.median(patch)<128 else cv2.THRESH_BINARY_INV)|cv2.THRESH_OTSU);mask=np.zeros(patch.shape,bool) + for q in cell_fragments[i]: + height=q[3]-q[1];u,v,U,V=map(int,[math.floor(q[0]-.3*height),math.floor(q[1]-.2*height),math.ceil(q[2]+.3*height),math.ceil(q[3]+.2*height)]);mask[max(0,v-c):min(C-c,V-c),max(0,u-a):min(A-a,U-a)]=True + n=int((ink>0).sum());fraction=float((mask&(ink>0)).sum())/n if n else 1.;coverage.append(dict(source_line=i,ink_fraction=fraction)) + if any(r['ink_fraction']<.98 for r in coverage):return None,dict(error='Table_grid_clips_detected_line_ink',grid=result,coverage=coverage,all_original_lines_retained=True) + lines=self.reader(gray,pieces);rows={};rtl=rtl_base(' '.join(l['text']for l in lines)) + for cell,cb,a,b in meta: + rows.setdefault(cell['row_id'],[]).append(dict(text=' '.join(line['text']for line in lines[a:b]),box=cb,colspan=cell.get('colspan',1))) + rendered=[] + for j,(_,row)in enumerate(sorted(rows.items())): + row.sort(key=lambda c:(-1 if rtl else 1)*(c['box'][0]+c['box'][2])/2);values=[] + for cell in row:values+=[cell['text']]+['']*(max(1,cell['colspan'])-1) + rendered.append('| '+' | '.join(markdown_cell(s)for s in values)+' |') + if j==0:rendered.append('| '+' | '.join('---'for _ in values)+' |') + # A line not represented by any cell remains in output after the table. + remainder=[i for i in block['line_ids']if i not in represented] + rendered +=[readings[i]['text']for i in remainder] + emitted=lines+[readings[i]for i in remainder] + return dict(text='\n'.join(rendered),lines=emitted),dict(grid=result,represented_line_ids=sorted(represented),unassigned_line_ids=remainder,grid_text_never_used=True,Markdown_first_row_separator=True,column_direction='RTL'if rtl else'LTR',cell_coverage=coverage) + def __call__(self,image,*,input_type='auto',**_): + from input_policy_v3 import estimated_line_height,merge_slivers,input_kind + tick=time.monotonic();image=image.convert('RGB');gray=np.asarray(image.convert('L'));estimate=estimated_line_height(gray);kind=input_kind(image.width,image.height,input_type);self.reader.source_is_crop=kind=='crop' + if kind=='crop'and not estimate['segment']: + boxes=[[0.,0.,float(image.width),float(image.height)]];readings=self.reader(gray,boxes);assert len(readings)==1 + region=dict(layout=None,source_line_ids=[0],path='rule_C_whole_input_unified_reader',text=readings[0]['text'],lines=readings) + return dict(text=readings[0]['text'],regions=[region],lines=readings,global_readings=readings,detected_boxes=[],emitted_boxes=[r['box']for r in readings],recognition_extents=[reading_extent(r)for r in readings],raw_line_detection=dict(skipped=True,reason='Rule C: height <=1.8 estimated line height'),raw_layout=dict(bboxes=[],skipped=True),order=dict(mode='single_input_rule_C',region_type_line_losses=0),fallbacks=[],crop_routing_policy=dict(mode='c',input_type=kind,estimate=estimate,whole_crop_by_policy=True,native_detector_calls=0),region_type_line_losses=0,recognizer_bank='unified32M_only',whole_image_detection=False,seconds=time.monotonic()-tick,main_Bina_training_changes=False) + boxes,detection=self.detect(image);fallbacks=[] + if kind=='crop': + line=estimate['estimated_line_height'] + if boxes and any(b[3]-b[1]>1.8*line for b in boxes): + if self._crop_line_guard is None: + from crop_line_guard import CropLineGuard + self._crop_line_guard=CropLineGuard() + boxes,guard=self._crop_line_guard(gray,boxes,line);detection['crop_anti_merge_guard']=guard + boxes,consolidation=merge_slivers(boxes,line);detection['same_line_box_consolidation']=consolidation + if not boxes:boxes=[[0.,0.,float(image.width),float(image.height)]];fallbacks.append('zero_detector_lines_whole_image_attempt_not_valid_for_coverage_gate') + try:layout=self.layout.run_batch([image],[dict(threshold=None,use_order=True)])[0].model_dump(mode='json') + except Exception as e:layout=dict(bboxes=[],error=repr(e));fallbacks.append('layout_error_RTL_column_order_all_lines_retained') + blocks,order=order_blocks(boxes,layout,image.width);readings=self.reader(gray,boxes);assert len(readings)==len(boxes) + regions=[];emitted=[] + for block in blocks: + ids=block['line_ids'];lines=[readings[i]for i in ids];record=dict(layout=block['region'],source_line_ids=ids,path='whole_image_lines',text='\n'.join(l['text']for l in lines),lines=lines) + if block['region']and block['region'].get('label','').lower()=='table': + assembled,trace=self.table_block(image,gray,block,boxes,readings);record['table_trace']=trace + if assembled:record.update(assembled,path='Surya_grid_unified_recognizer_cells') + else: + fallback_order=xycut_order([boxes[i]for i in ids],image.width);record['lines']=[readings[ids[j]]for j in fallback_order];record['text']='\n'.join(l['text']for l in record['lines']);record['source_line_ids']=[ids[j]for j in fallback_order];fallbacks.append('table_grid_failed_original_lines_retained_in_RTL_columns') + regions.append(record);emitted+=record['lines'] + assert sorted(i for r in regions for i in r['source_line_ids'])==list(range(len(boxes))) + return dict(text='\n\n'.join(r['text']for r in regions),regions=regions,lines=emitted,global_readings=readings,detected_boxes=boxes,emitted_boxes=[l['box']for l in emitted],recognition_extents=[reading_extent(l)for l in emitted],raw_line_detection=detection,raw_layout=layout,order=order,fallbacks=fallbacks,crop_routing_policy=dict(mode='c',input_type=kind,estimate=estimate,whole_crop_by_policy=False),region_type_line_losses=0,recognizer_bank='unified32M_only',whole_image_detection=True,seconds=time.monotonic()-tick,main_Bina_training_changes=False) diff --git a/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/table_client.py b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/table_client.py new file mode 100644 index 0000000000000000000000000000000000000000..e11d28dfd975b2e21999e3cd9590c1acd3d84a28 --- /dev/null +++ b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline/table_client.py @@ -0,0 +1,24 @@ +import base64,io,json,subprocess,select,time +from pathlib import Path +R=Path(__file__).resolve().parent +class SuryaTableGrid: + def __init__(self,yield_check=None):self.process=None;self.log=None;self.yield_check=yield_check + def __call__(self,image): + if self.process is None: + self.log=(R/'table-worker.log').open('a');self.process=subprocess.Popen([str(R/'table-runtime/bin/python'),'-u',str(R/'table_worker.py')],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=self.log,text=True) + stream=io.BytesIO();image.save(stream,format='PNG');self.process.stdin.write(json.dumps(dict(image=base64.b64encode(stream.getvalue()).decode()))+'\n');self.process.stdin.flush();tick=time.monotonic() + while not select.select([self.process.stdout],[],[],.5)[0]: + if self.yield_check:self.yield_check() + if time.monotonic()-tick>300:self.close();raise TimeoutError('Owned table worker timeout; original line output retained') + line=self.process.stdout.readline() + if not line:raise RuntimeError('Isolated Surya table worker exited; original line text retained') + reply=json.loads(line) + if not reply['ok']:raise RuntimeError(reply['error']) + return reply['result'] + def close(self): + if self.process is not None: + self.process.stdin.close() + try:self.process.wait(timeout=15) + except subprocess.TimeoutExpired:self.process.terminate();self.process.wait(timeout=10) + self.process=None + if self.log:self.log.close();self.log=None diff --git a/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/line_contract_reader.py b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/line_contract_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..11253ed0cac3af633d0f95095f226df72149d097 --- /dev/null +++ b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/line_contract_reader.py @@ -0,0 +1,69 @@ +"""Single-line shadow candidate: exact round-2 detector and crop_line64 contract. +Always one recognition input for a rule-C line; no router and no extra strips. +""" +import hashlib,json,math +from pathlib import Path +import numpy as np +from baseline.pipeline_v3 import UnifiedReader,rectangle,clipped,flip +from baseline.input_policy_v3 import estimated_line_height +from pipeline_crop_fix import CropPipeline +DETECTOR=Path('__BINA_ROOT__/stallion/bina04-line-detector-v2-20261030/detector-run/checkpoints/step-000500.pt') +DETECTOR_SHA='0320b9d1d9e7464c4eae6f35b93fdd835913e785eae552eb5f04b58091b32ae6' + +def round2_estimate(gray): + # Execute only the exact frozen estimator function, without creating any + # of the old recognizer/script banks. This is a boxing decision only. + import ast + p=Path('__BINA_ROOT__/stallion/indra-scale/routing_modes.py') + rule=json.loads((p.parent/'routing-rule.json').read_text()) + assert hashlib.sha256(p.read_bytes()).hexdigest()==rule['source_sha256'][p.name] + node=next(n for n in ast.parse(p.read_text()).body if isinstance(n,ast.FunctionDef)and n.name=='estimated_line_height') + scope={'np':np};exec(compile(ast.Module(body=[node],type_ignores=[]),str(p),'exec'),scope) + return scope['estimated_line_height'](gray) + +class ContractReader(UnifiedReader): + def __init__(self,*args,cache,**kwargs): + super().__init__(*args,**kwargs);self.rule_c_active=False;self.detector=None;self.cache=Path(cache);self.cache.mkdir(parents=True,exist_ok=True) + def locate(self,gray): + old=round2_estimate(gray) + if not old['segment']: + H,W=gray.shape + return dict(box=[0.,0.,float(W),float(H)],round2_rule_C=old,round2_skipped_detection=True,one_recognition_crop=True,no_reference_or_label_used=True) + key=hashlib.sha256(str(gray.shape).encode()+gray.tobytes()).hexdigest();target=self.cache/(key+'.json') + if target.exists():return json.loads(target.read_text()) + if self.detector is None: + import torch + from surya.detection import DetectionPredictor + assert hashlib.sha256(DETECTOR.read_bytes()).hexdigest()==DETECTOR_SHA + self.detector=DetectionPredictor.local(device='cpu',dtype=torch.float32);self.detector._disable_tqdm=True + self.detector.model.load_state_dict(torch.load(DETECTOR,map_location='cpu',weights_only=True)['model']);self.detector.model.eval() + from PIL import Image + H,W=gray.shape;raw=self.detector([Image.fromarray(gray).convert('RGB')],batch_size=1)[0].model_dump(mode='json');boxes=[clipped(rectangle(b['polygon']),W,H)for b in raw['bboxes']];boxes=[b for b in boxes if b[2]>b[0]and b[3]>b[1]] + box=[min(b[0]for b in boxes),min(b[1]for b in boxes),max(b[2]for b in boxes),max(b[3]for b in boxes)]if boxes else[0.,0.,float(W),float(H)] + d=dict(box=box,raw_detector_boxes=boxes,raw=raw,detector_sha256=DETECTOR_SHA,device='cpu',one_recognition_crop=True,fallback=not bool(boxes),no_reference_or_label_used=True) + tmp=target.with_suffix('.partial');tmp.write_text(json.dumps(d)+'\n');tmp.replace(target);return d + def __call__(self,gray,boxes): + H,W=gray.shape + if not(self.rule_c_active and self.source_is_crop and len(boxes)==1 and np.allclose(boxes[0],[0.,0.,float(W),float(H)])): + return super().__call__(gray,boxes) + trace=self.locate(gray);box=trace['box'] + # Same direct original grayscale -> crop_line64 -> pad_batch -> model + # path as round2 RTLReader, bypassing v3 projection and the atlas. + _,receipts=self.reader(gray,np.asarray([box],np.float32));r=receipts[0] + if not r['raw_visual_text'].strip(): + recovered=super().__call__(gray,boxes) + for line in recovered:line['single_line_contract_fallback']=dict(reason='Tight crop decoded empty; original single-line read retained',attempt_box=box,attempt_confidence=r['confidence'],reference_or_label_used=False) + return recovered + a,b,A,B=r['padded_source_extent'];r['source_pixels_extent']=[max(0,a),max(0,b),min(W,A),min(H,B)] + r['preprocessing_contract']='Exact shared geometry.crop_line64 and RTLReader, direct original uint8 pixels' + return[dict(box=list(map(float,box)),text=flip(r['raw_visual_text'],base_rtl=True),visual_text=r['raw_visual_text'],confidence=r['confidence'],recognizer='unified32M',geometry=r,confidence_filter_applied=False,whole_input_normalization=dict(rule_c_single_line=True,round2_detector_reboxing=trace,one_recognition_crop=True))] + +class ContractPipeline(CropPipeline): + """Apply the new reader to generic rule-C crops only. + The existing specialized newspaper/table and whole-page paths stay frozen. + """ + def __call__(self,image,*,input_type='auto',source_name='',content_type='',**kw): + from baseline.input_policy_v3 import input_kind + gray=np.asarray(image.convert('L'));generic=content_type.lower()not in ['newspaper','table']and'table'not in source_name.lower() + self.reader.rule_c_active=generic and input_kind(*image.size,input_type)=='crop'and not estimated_line_height(gray)['segment'] + return super().__call__(image,input_type=input_type,source_name=source_name,content_type=content_type,**kw) diff --git a/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/pipeline_crop_fix.py b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/pipeline_crop_fix.py new file mode 100644 index 0000000000000000000000000000000000000000..708e9d6c1070c115b4e7e5694a6db802dd9a9247 --- /dev/null +++ b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/pipeline_crop_fix.py @@ -0,0 +1,76 @@ +"""New crop path only. Full pages delegate to the exact frozen v3 implementation.""" +import copy,time +import numpy as np +from baseline import pipeline_v3 as base +from baseline.input_policy_v3 import input_kind,estimated_line_height,merge_slivers +from table_cells import CellDetector,assemble + +class CropPipeline(base.PagePipeline): + def __init__(self,*args,variant='tables-v1',**kwargs): + super().__init__(*args,**kwargs);self.variant=variant;self._crop_active=False;self._column_order='auto';self._cell_detector=None + def table_block(self,image,gray,block,boxes,readings): + if not self._crop_active:return super().table_block(image,gray,block,boxes,readings) + if self._cell_detector is None:self._cell_detector=CellDetector() + return assemble(image,gray,block['region'],self.reader,self.table,self._cell_detector,self._column_order) + def __call__(self,image,*,input_type='auto',source_name='',content_type='',table_column_order='auto',**kwargs): + kind=input_kind(*image.size,input_type);self._crop_active=kind=='crop';self._column_order=table_column_order + if kind!='crop':return super().__call__(image,input_type=input_type,**kwargs) + image=image.convert('RGB');gray=np.asarray(image.convert('L'));self.reader.source_is_crop=True + if content_type.lower()=='newspaper'and ('news' in self.variant or 'combined' in self.variant): + stage=next((n for n in [4,3,2,1]if f'news{n}'in self.variant),2) + return self.newspaper_call(image,gray,stage) + table_hint=content_type.lower()=='table'or'table'in source_name.lower() + if table_hint: + tick=time.monotonic();w,h=image.size;region=dict(label='Table',position=0,polygon=[[0,0],[w,0],[w,h],[0,h]]) + block=dict(region=region,line_ids=[]);value,trace=self.table_block(image,gray,block,[],[]) + if value is not None: + record=dict(layout=region,path='Surya_grid_unified_recognizer_cells',source_line_ids=[],table_trace=trace,**value) + return dict(text=value['text'],regions=[record],lines=value['lines'],global_readings=[],detected_boxes=[],emitted_boxes=[q['box']for q in value['lines']],recognition_extents=[base.reading_extent(q)for q in value['lines']],raw_line_detection=dict(skipped=True,reason='Explicit table crop: each cell is segmented independently'),raw_layout=dict(bboxes=[region],source='Explicit table metadata/name'),order=dict(mode='table_grid'),fallbacks=[],crop_routing_policy=dict(input_type='crop',whole_crop_by_policy=False),region_type_line_losses=0,recognizer_bank='unified32M_only',whole_image_detection=False,seconds=time.monotonic()-tick,main_Bina_training_changes=False) + original=self.layout + class PreferTableOnTie: + def run_batch(_,images,*args,**kw): + results=original.run_batch(images,*args,**kw);out=[] + for result in results: + data=copy.deepcopy(result.model_dump(mode='json'));data['bboxes'].sort(key=lambda r:r.get('label','').lower()!='table') + class Result: + def __init__(self,d):self.data=d + def model_dump(self,**kw):return self.data + out.append(Result(data)) + return out + self.layout=PreferTableOnTie() + try: + result=super().__call__(image,input_type='crop',**kwargs) + result['crop_table_policy']=dict(explicit_hint=table_hint,layout_table_wins_exact_overlap_tie=True,column_order=table_column_order) + return result + finally:self.layout=original + + def newspaper_call(self,image,gray,stage): + tick=time.monotonic();estimate=estimated_line_height(gray) + if not estimate['segment']: + result=super().__call__(image,input_type='crop');result['newspaper_policy']=dict(stage=stage,already_one_full_width_line=True);return result + boxes,detection=self.detect(image);fallbacks=[];line=estimate['estimated_line_height'] + if self._crop_line_guard is None: + from baseline.crop_line_guard import CropLineGuard + self._crop_line_guard=CropLineGuard() + if boxes and any(b[3]-b[1]>1.8*line for b in boxes): + boxes,guard=self._crop_line_guard(gray,boxes,line);detection['crop_anti_merge_guard']=guard + boxes,consolidation=merge_slivers(boxes,line);detection['same_line_box_consolidation']=consolidation + from newspaper_crop import apply + boxes,trace=apply(gray,boxes,stage,self._crop_line_guard);detection['newspaper_policy']=trace + if not boxes:boxes=[[0.,0.,float(image.width),float(image.height)]];fallbacks.append('zero_detector_lines_whole_image_attempt_not_valid_for_coverage_gate') + try:layout=self.layout.run_batch([image],[dict(threshold=None,use_order=True)])[0].model_dump(mode='json') + except Exception as e:layout=dict(bboxes=[],error=repr(e));fallbacks.append('layout_error_RTL_column_order_all_lines_retained') + blocks,order=base.order_blocks(boxes,layout,image.width) + if stage>=4: + from newspaper_crop import read_lines + readings=read_lines(self.reader,gray,boxes) + else:readings=self.reader(gray,boxes) + regions=[];emitted=[] + for block in blocks: + ids=block['line_ids'];lines=[readings[i]for i in ids];record=dict(layout=block['region'],source_line_ids=ids,path='whole_image_lines',text='\n'.join(l['text']for l in lines),lines=lines) + if block['region']and block['region'].get('label','').lower()=='table': + assembled,t=self.table_block(image,gray,block,boxes,readings);record['table_trace']=t + if assembled:record.update(assembled,path='Surya_grid_unified_recognizer_cells') + regions.append(record);emitted+=record['lines'] + assert sorted(i for r in regions for i in r['source_line_ids'])==list(range(len(boxes))) + return dict(text='\n\n'.join(r['text']for r in regions),regions=regions,lines=emitted,global_readings=readings,detected_boxes=boxes,emitted_boxes=[l['box']for l in emitted],recognition_extents=[base.reading_extent(l)for l in emitted],raw_line_detection=detection,raw_layout=layout,order=order,fallbacks=fallbacks,crop_routing_policy=dict(mode='c',input_type='crop',estimate=estimate,whole_crop_by_policy=False),region_type_line_losses=0,recognizer_bank='unified32M_only',whole_image_detection=True,seconds=time.monotonic()-tick,main_Bina_training_changes=False) diff --git a/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/table_cells.py b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/table_cells.py new file mode 100644 index 0000000000000000000000000000000000000000..d3a83334115af2e3c533f97a1a04219ac3f29f67 --- /dev/null +++ b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/table_cells.py @@ -0,0 +1,111 @@ +"""Crop-only table cells: native line splitting, cell padding, explicit serialization.""" +import math +from pathlib import Path +import numpy as np +from baseline.pipeline_v3 import rectangle,clipped,markdown_cell,reading_extent +from baseline.input_policy_v3 import estimated_line_height +from indra_ocr.rtl import rtl_base + +class CellDetector: + def __init__(self): + from baseline.crop_line_guard import CropLineGuard + self.detector=CropLineGuard() + def lines(self,gray,line): + engine=self.detector.engine + engine.long_side=max(32,min(2048,round(max(gray.shape)*12/max(3,line)/32)*32)) + return engine.detect(gray)[:,:4].tolist() + +def glyph_presence(gray): + import cv2 + H,W=gray.shape;_,v=cv2.threshold(gray,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU);ink=(v==0).astype(np.uint8) + if ink.mean()>.5:ink=1-ink + _,_,stats,_=cv2.connectedComponentsWithStats(ink,8);valid=[] + for x,y,w,h,n in stats[1:]: + if n<6 or h<3 or w<2 or not .05<=w/max(1,h)<=20:continue + frame=(h>=.7*H and w>=.7*W and n/max(1,w*h)<.15) + if frame or h>=.98*H or(w>=.95*W and h<=3):continue + valid.append([int(x),int(y),int(w),int(h),int(n)]) + return bool(valid),valid + +def tighten_ink_band(gray,box,components): + """Remove native detector margin while retaining >=98% observed glyph ink + inside the unchanged reader's .2H padding. No decoded labels enter this. + """ + import cv2 + H,W=gray.shape;lo=max(0,math.floor(box[1]));hi=min(H,math.ceil(box[3])) + _,v=cv2.threshold(gray,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU);ink=v==0 + if ink.mean()>.5:ink=~ink + mask=np.zeros(gray.shape,bool) + for x,y,w,h,n in components:mask[y:y+h,x:x+w]=ink[y:y+h,x:x+w] + counts=mask[lo:hi].sum(axis=1);total=int(counts.sum()) + if not total:return box,dict(changed=False,reason='No glyph ink') + active=np.where(counts>=max(1,.2*counts.max()))[0] + if not len(active):return box,dict(changed=False) + top=lo+int(active[0]);bottom=lo+int(active[-1])+1 + while True: + h=bottom-top;a=max(lo,math.floor(top-.2*h));b=min(hi,math.ceil(bottom+.2*h));kept=int(mask[a:b].sum()) + if kept/total>=.98 or(top<=lo and bottom>=hi):break + top=max(lo,top-1);bottom=min(hi,bottom+1) + return [box[0],float(top),box[2],float(bottom)],dict(changed=True,original=box,core=[top,bottom],glyph_ink_fraction_in_padded_reader=kept/total) + +def remap(line,x,y,W,H): + line=dict(line);line['box']=[line['box'][0]+x,line['box'][1]+y,line['box'][2]+x,line['box'][3]+y] + g=dict(line['geometry']) + for key in ['ink_box','padded_source_extent','source_pixels_extent']: + if key in g:a,b,A,B=g[key];g[key]=[a+x,b+y,A+x,B+y] + g['original_WH']=[W,H] + if 'frame_centers'in g:g['frame_centers']=[a+x for a in g['frame_centers']] + line['geometry']=g;return line + +def recognize_cell(image,gray,cb,reader,detector,padding=3): + W,H=image.size;x,y,X,Y=clipped(cb,W,H) + a=max(0,math.floor(x)-padding);b=max(0,math.floor(y)-padding);A=min(W,math.ceil(X)+padding);B=min(H,math.ceil(Y)+padding) + if A<=a or B<=b:return '',[],dict(empty=True,reason='invalid_cell') + local=gray[b:B,a:A].copy();present,components=glyph_presence(local) + if not present:return '',[],dict(empty=True,padded_bbox=[a,b,A,B],glyph_components=[]) + estimate=estimated_line_height(local);line=max(3.,estimate['original_glyph_based_line_height']);tall=local.shape[0]>1.8*line + native=detector.lines(local,line)if tall else[] + # Padding may include a neighbour's descender; its centre lies outside this cell. + native=[q for q in native if y-b<=(q[1]+q[3])/2<=Y-b and q[3]-q[1]>=2] + groups=[] + for q in sorted(native,key=lambda q:(q[1]+q[3])/2): + cy=(q[1]+q[3])/2 + if not groups or cy-groups[-1][-1][0]>.6*line:groups.append([]) + groups[-1].append((cy,q)) + bands=[[0.,min(q[1]for _,q in g),float(local.shape[1]),max(q[3]for _,q in g)]for g in groups] + projected=estimate['ink_bands'] + fallback=None + if tall and len(bands)<=1 and len(projected)>=2: + bands=[[0.,float(t),float(local.shape[1]),float(T)]for t,T in projected if T-t>=2];fallback='projection_after_native_merged_or_empty' + if not bands:bands=[[0.,0.,float(local.shape[1]),float(local.shape[0])]] + tight=[];normalization=[] + for band in bands: + q,t=tighten_ink_band(local,band,components);tight.append(q);normalization.append(t) + lines=reader(local,tight);lines=[remap(q,a,b,W,H)for q in lines] + return ' '.join(q['text'].strip()for q in lines if q['text'].strip()),lines,dict(empty=False,padded_bbox=[a,b,A,B],glyph_components=components,line_estimate=estimate,native_called=tall,native_boxes=native,line_boxes=tight,ink_normalization=normalization,fallback=fallback) + +def assemble(image,gray,region,reader,grid,detector,column_order='auto'): + W,H=image.size;b=clipped(rectangle(region['polygon']),W,H);x,y,X,Y=[math.floor(b[0]),math.floor(b[1]),math.ceil(b[2]),math.ceil(b[3])] + result=grid(image.crop((x,y,X,Y)));cells=result.get('cells',[]) + if result.get('error')or not cells:return None,dict(error='Surya_table_grid_empty',grid=result,grid_invoked=True) + values={};row_y={};col_x={};debug=[];emitted=[];presence={} + for cell in cells: + cb=cell.get('bbox')or rectangle(cell['polygon']);cb=[cb[0]+x,cb[1]+y,cb[2]+x,cb[3]+y] + row=cell['row_id'];col=cell.get('col_id') + if col is None:col=cell.get('within_row_id',0) + text,lines,trace=recognize_cell(image,gray,cb,reader,detector) + key=(row,col);values[key]=text;presence[key]=not trace['empty'];row_y.setdefault(row,[]).append((cb[1]+cb[3])/2);col_x.setdefault(col,[]).append((cb[0]+cb[2])/2) + emitted+=lines;debug.append(dict(row=row,col=col,bbox=cb,text=text,lines=lines,trace=trace,colspan=cell.get('colspan',1),rowspan=cell.get('rowspan',1))) + rows=sorted(row_y,key=lambda r:float(np.median(row_y[r]))) + cols=sorted(col_x,key=lambda c:float(np.median(col_x[c]))) + empty_rows=[r for r in rows if not any(values.get((r,c),'').strip()or presence.get((r,c),False)for c in cols)] + empty_cols=[c for c in cols if not any(values.get((r,c),'').strip()or presence.get((r,c),False)for r in rows)] + rows=[r for r in rows if r not in empty_rows];cols=[c for c in cols if c not in empty_cols] + rtl=rtl_base(' '.join(values.values()));chosen=('rtl'if rtl else'ltr')if column_order=='auto'else column_order + assert chosen in ['rtl','ltr'] + if chosen=='rtl':cols.reverse() + rendered=[] + for i,r in enumerate(rows): + rendered.append('| '+' | '.join(markdown_cell(values.get((r,c),''))for c in cols)+' |') + if i==0:rendered.append('| '+' | '.join('---'for _ in cols)+' |') + return dict(text='\n'.join(rendered),lines=emitted),dict(grid=result,grid_invoked=True,cell_debug=debug,kept_rows=rows,kept_columns=cols,dropped_empty_rows=empty_rows,dropped_empty_columns=empty_cols,content_RTL=bool(rtl),physical_column_serialization=chosen,cell_padding=3,grid_text_never_used=True) diff --git a/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_builder.py b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..5695f63e34fa78704be6c2da6e89a7bbd64c78bb --- /dev/null +++ b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_builder.py @@ -0,0 +1,102 @@ +"""Image-only pen/print separation and physical row construction; no gold inputs.""" +import math +import numpy as np +import cv2 + +DEFAULT=dict(core_overlap=.5,baseline_distance=.4,max_gap=3.,nominal_pad=.06,color_angle_degrees=35,color_depth=.055,color_chroma=.045,color_relative_chroma=.16,min_pen_components=15,min_pen_pixels=80) +def rect_intersection(a,b):return[max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])] +def rect_area(a):return max(0,a[2]-a[0])*max(0,a[3]-a[1]) +def estimate_height(proposals,H): + h=[b[3]-b[1]for b in proposals if b[3]>b[1]and b[2]-b[0]>=2*(b[3]-b[1])and 4<=b[3]-b[1]<=min(128,.12*H)] + return max(6.,min(100.,float(np.percentile(h,60))if h else H/40)) +def remove_rules(mask,line): + H,W=mask.shape;u=mask.astype(np.uint8);horizontal=cv2.morphologyEx(u,cv2.MORPH_OPEN,np.ones((1,max(30,round(.12*W))),np.uint8));vertical=cv2.morphologyEx(u,cv2.MORPH_OPEN,np.ones((max(30,round(.1*H)),1),np.uint8));rules=cv2.dilate(horizontal|vertical,np.ones((3,3),np.uint8));return(u>0)&~(rules>0),rules>0 +def components(mask,min_area=2): + n,labels,stats,centers=cv2.connectedComponentsWithStats(mask.astype(np.uint8),8);return labels,[dict(label=i,x=int(x),y=int(y),w=int(w),h=int(h),area=int(a),cx=float(centers[i][0]),cy=float(centers[i][1]))for i,(x,y,w,h,a)in enumerate(stats)if i and a>=min_area] +def columns(mask,line,proposals=()): + H,W=mask.shape;profile=mask.sum(axis=0);low=profile<=max(1,.002*H);edges=np.diff(np.r_[False,low,False].astype(int));gaps=[] + for a,b in zip(np.where(edges==1)[0],np.where(edges==-1)[0]): + if b-a.85*W:continue + cut=(a+b)//2 + crossing=[p for p in proposals if p[0]b+line and p[2]-p[0]>.55*W and p[3]-p[1]<2*line] + if len(crossing)>=3:continue + left=int(mask[:,:a].sum());right=int(mask[:,b:].sum());total=max(1,int(mask.sum())) + if min(left,right)>=.15*total and min(a,W-b)>=5*line:gaps.append((b-a,(a+b)//2)) + if not gaps:return[(0,W)] + cut=max(gaps)[1];return[(cut,W),(0,cut)] +def proposal_columns(boxes,W,H,gray): + vertical=cv2.morphologyEx((gray<100).astype(np.uint8),cv2.MORPH_OPEN,np.ones((max(30,round(.18*H)),1),np.uint8));occupied=vertical.sum(0)>.18*H;edges=np.diff(np.r_[False,occupied,False].astype(int));rulings=[] + for a,b in zip(np.where(edges==1)[0],np.where(edges==-1)[0]): + if not rulings or a-rulings[-1][1]>.01*W:rulings.append([a,b]) + else:rulings[-1][1]=b + # Count physical rulings once, including antialiased/double-stroke borders. + if len(rulings)>=5:return[(0,W)] + anchors=[b for b in boxes if .2*W<=b[2]-b[0]<=.7*W and b[3]-b[1]<=.15*H];centers=sorted(((b[0]+b[2])/2,b)for b in anchors) + if len(centers)<6:return None + for gap,k in sorted(((centers[k+1][0]-centers[k][0],k)for k in range(2,len(centers)-3)),reverse=True): + if gap<.15*W:break + left=[b for _,b in centers[:k+1]];right=[b for _,b in centers[k+1:]];a=float(np.median([b[2]for b in left]));z=float(np.median([b[0]for b in right])) + if z-a>=.02*W: + cut=round((a+z)/2);return[(cut,W),(0,cut)] + return None + +def layer_masks(rgb,proposals,params): + H,W=rgb.shape[:2];line=estimate_height(proposals,H);k=max(15,min(71,round(1.2*line)));k+=1-k%2 + background=cv2.morphologyEx(rgb,cv2.MORPH_CLOSE,cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(k,k))).astype(np.float32) + optical=np.maximum(0,np.log((background+2)/(rgb.astype(np.float32)+2)));depth=optical.mean(2);chroma=optical-depth[:,:,None];strength=np.linalg.norm(chroma,axis=2);relative=strength/(depth+.05) + direction=optical/(optical.sum(2)[:,:,None]+1e-5);valid=depth>.30;points=direction[valid].astype(np.float32) + if len(points)<100:return np.zeros((H,W),bool),np.zeros((H,W),bool),cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY),dict(reliable=False) + cv2.setRNGSeed(1729);sample=points[::max(1,len(points)//30000)].copy();K=4;_,_,centers=cv2.kmeans(sample,K,None,(cv2.TERM_CRITERIA_EPS+cv2.TERM_CRITERIA_MAX_ITER,60,.0002),3,cv2.KMEANS_PP_CENTERS) + distances=np.stack([np.linalg.norm(direction-c,axis=2)for c in centers]);assigned=distances.argmin(0);scores=[];clean=[];traces=[] + for i in range(K): + # Include neighbouring optical directions belonging to the same pen colour. + compatible=np.linalg.norm(centers-centers[i],axis=1)<.035 + mask=(depth>.22)&compatible[assigned];mask,rules=remove_rules(mask,line);lab,cc=components(mask,max(3,round(.004*line*line))) + good=[c for c in cc if c['h']<3*line and c['w']<.85*W and c['area']<.04*W*H and not(c['w']<.3*line and c['h']>.9*line and(c['x']<.04*W or c['x']+c['w']>.96*W))] + mask=np.isin(lab,[c['label']for c in good]);long=[c for c in good if c['w']>=1.5*c['h']and c['h']>=.15*line and c['w']>=.5*line] + bins_y=len(set(int(c['cy']/max(1,2*line))for c in long));mass=sum(c['area']for c in long);score=mass*min(1,bins_y/8)*max(.008,float(np.linalg.norm(centers[i]-np.ones(3)/3))) + scores.append(score);clean.append(mask);traces.append(dict(class_id=i,center=centers[i].tolist(),pixels=int(mask.sum()),long_components=len(long),vertical_bins=bins_y,score=score)) + best=int(np.argmax(scores));pen=clean[best];lab,cc=components(pen,2);span=max((c['y']+c['h']for c in cc),default=0)-min((c['y']for c in cc),default=0) + other=[scores[j]for j in range(K)if np.linalg.norm(centers[j]-centers[best])>=.04];secondary=max(other,default=0) + reliable=traces[best]['long_components']>=8 and traces[best]['vertical_bins']>=5 and secondary>max(.5,.015*scores[best]) + neutral=centers[int(np.argmin(np.linalg.norm(centers-np.ones(3)/3,axis=1)))];radius=max(.025,.65*float(np.linalg.norm(centers[best]-neutral)));compatible=np.linalg.norm(centers-centers[best],axis=1).16,line);pen=(depth>.07)&compatible[assigned]&~all_rules;labels,allcc=components(pen,2) + ids=[c['label']for c in allcc if c['h']<3*line and c['w']<.85*W and not(c['w']<.3*line and c['h']>.9*line and(c['x']<.04*W or c['x']+c['w']>.96*W))];pen=np.isin(labels,ids) + printed=(depth>.22)&~(cv2.dilate(pen.astype(np.uint8),np.ones((3,3),np.uint8))>0);printed,_=remove_rules(printed,line) + dilated=cv2.dilate(pen.astype(np.uint8),np.ones((3,3),np.uint8))>0 + gray=cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY);bg=cv2.cvtColor(background.clip(0,255).astype(np.uint8),cv2.COLOR_RGB2GRAY);print_gray=gray.copy();print_gray[dilated]=bg[dilated] + trace_core=(depth>.22)&compatible[assigned]&~all_rules + return pen,printed,print_gray,dict(_core_mask=trace_core,reliable=bool(reliable),estimated_line_height=line,selected_class=best,classes=traces,pen_components=len(cc),pen_pixels=int(pen.sum()),color_span_y=span,background_kernel=k,no_gold_or_template_used=True) + +def build_rows(mask,proposals,params,regions=None,core=None): + from scipy.ndimage import gaussian_filter1d + from scipy.signal import find_peaks + H,W=mask.shape;line=estimate_height(proposals,H);cols=regions or columns(mask,line,proposals);rows=[];row_trace=[] + for column,(left,right)in enumerate(cols): + m=mask[:,left:right];cm=(core if core is not None else mask)[:,left:right];profile=cm.sum(1).astype(float);smooth=gaussian_filter1d(profile,max(1,.08*line));corr=np.correlate(smooth-smooth.mean(),smooth-smooth.mean(),mode='full')[H-1:];lo=max(5,round(.5*line));hi=min(H//3,round(3.5*line));period=line*1.7 + if hi>lo: + peaks,_=find_peaks(corr[lo:hi]);valid=[lo+int(k)for k in peaks if corr[lo+k]>.12*corr[0]] + if valid:period=min(valid) + period=line;peaks,_=find_peaks(smooth,distance=max(3,round(.65*line)),prominence=max(.8,.025*smooth.max())) + peaks=[int(y)for y in peaks if smooth[y]>=max(1,.025*smooth.max())] + for j,peak in enumerate(peaks): + low=0 if not j else peaks[j-1]+int(np.argmin(smooth[peaks[j-1]:peak+1]));high=H if j+1==len(peaks)else peak+int(np.argmin(smooth[peak:peaks[j+1]+1])) + # Limit isolated/noisy bands, retaining full connected strokes and dots. + low=max(low,round(peak-.8*period));high=min(high,round(peak+.8*period));xs=np.where(m[low:high].any(0))[0] + segments=[] + for x in xs: + if not segments or x-segments[-1][-1]>max(params['max_gap'],8.)*line:segments.append([]) + segments[-1].append(int(x)) + for segment in segments: + x=left+segment[0];X=left+segment[-1]+1;ys,xx=np.where(mask[low:high,x:X]) + if len(ys)=max(4,.05*len(ys))else ys + y=low+int(tight.min());Y=low+int(tight.max())+1;pad=max(1.,.04*line);box=[max(0.,x-pad),max(0.,y-pad),min(float(W),X+pad),min(float(H),Y+pad)];support=[i for i,p in enumerate(proposals)if rect_area(rect_intersection(box,p))>0] + rows.append(dict(box=box,baseline=float(np.quantile(ys,.8)+low),column=column));row_trace.append(dict(source_proposals=support,ink_pixels=len(ys),read_band=[low,high],peak=peak,period=period)) + return rows,dict(estimated_line_height=line,columns=cols,rows=row_trace) + +def build(rgb,proposals,params=None,primary=None): + params=dict(DEFAULT,**(params or {}));pen,printed,print_gray,trace=layer_masks(rgb,proposals,params);core=trace.pop('_core_mask',None) + if not trace['reliable']:return None,dict(layer_mask=trace,reason='No confident separated handwriting and printed layers') + cols=proposal_columns(primary or proposals,rgb.shape[1],rgb.shape[0],cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY));hand,ht=build_rows(pen,proposals,params,core=core,regions=cols);printed_rows,pt=build_rows(printed,proposals,params,regions=ht['columns']) + return dict(handwriting=hand,printed=printed_rows,printed_gray=print_gray,pen_mask=pen,printed_mask=printed),dict(parameters=params,layer_mask=trace,handwriting_rows=ht,printed_rows=pt,source_proposal_count=len(proposals),handwriting_readings=len(hand),printed_readings=len(printed_rows),no_gold_or_template_used=True) diff --git a/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_pipeline.py b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..eb3d37cf88d84bf3c5dc39e8c63e407ca10ed1a4 --- /dev/null +++ b/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/versions/combined/row_pipeline.py @@ -0,0 +1,42 @@ +import time +import numpy as np +from line_contract_reader import ContractPipeline +from baseline.pipeline_v3 import rectangle,clipped,order_blocks,reading_extent +from baseline.input_policy_v3 import input_kind +from row_builder import build + +class RowPipeline(ContractPipeline): + def __init__(self,*args,row_parameters=None,**kw): + super().__init__(*args,**kw);self.row_parameters=row_parameters or{} + def __call__(self,image,*,input_type='auto',source_name='',content_type='',**kw): + if input_kind(*image.size,input_type)!='page':return super().__call__(image,input_type=input_type,source_name=source_name,content_type=content_type,**kw) + tick=time.monotonic();image=image.convert('RGB');rgb=np.asarray(image);gray=np.asarray(image.convert('L'));boxes,detection=self.detect(image);proposals=[] + if detection.get('full_image_SDK'): + proposals +=[clipped(rectangle(b['polygon']),image.width,image.height)for b in detection['full_image_SDK']['bboxes']] + proposals +=[b for t in detection.get('native_resolution_tiles',[])for b in t['boxes']] + if not proposals:proposals=boxes + layers,trace=build(rgb,proposals,self.row_parameters,primary=boxes) + if layers is None or not layers['handwriting']: + result=super().__call__(image,input_type='page',source_name=source_name,content_type=content_type,**kw);result['row_consolidation']=dict(applied=False,**trace);return result + self.reader.rule_c_active=False;self.reader.source_is_crop=False + hand=self.reader(gray,[r['box']for r in layers['handwriting']]) + for line,meta in zip(hand,layers['handwriting']):line.update(layer='handwriting',baseline=meta['baseline'],column=meta['column'],source_pixels_modified=False) + # Retain the established printed reads outside consolidated pen rows. + # This avoids turning weak paper texture or printed rules into new reads. + original=super().__call__(image,input_type='page',source_name=source_name,content_type=content_type,**kw);printed=[] + for line in original['lines']: + b=line['box'];overlap=False + for row in layers['handwriting']: + r=row['box'];vy=min(b[3],r[3])-max(b[1],r[1]);vx=min(b[2],r[2])-max(b[0],r[0]) + if vy>.35*min(b[3]-b[1],r[3]-r[1])and vx>0:overlap=True;break + if overlap:continue + x=dict(line);x.update(layer='printed',baseline=float(b[3]),column=next((i for i,(a,z)in enumerate(trace['handwriting_rows']['columns'])if a<=(b[0]+b[2])/212)&(hsv[:,:,2]>135)).astype('uint8')*255;tint=cv2.morphologyEx(tint,cv2.MORPH_CLOSE,np.ones((9,25),np.uint8));tint=cv2.morphologyEx(tint,cv2.MORPH_OPEN,np.ones((5,9),np.uint8)) + ink=cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY_INV,31,5);hor=cv2.morphologyEx(ink,cv2.MORPH_OPEN,np.ones((1,max(30,W//8)),np.uint8));ver=cv2.morphologyEx(ink,cv2.MORPH_OPEN,np.ones((max(12,h//5),1),np.uint8));rules=cv2.dilate(hor|ver,np.ones((5,5),np.uint8));out=[] + for kind,mask in [('tinted',tint),('bordered',rules)]: + nc,_,stats,_=cv2.connectedComponentsWithStats(mask) + for x,y,w,hh,a in stats[1:]: + if w<.18*W or hh=q['box'][0]and b[1]>=q['box'][1]and b[2]<=q['box'][2]and b[3]<=q['box'][3]for q in out):out.append(dict(box=b,source=kind)) + return out + +def runs(mask): + e=np.diff(np.r_[False,mask,False].astype(int));return list(zip(np.flatnonzero(e==1),np.flatnonzero(e==-1))) +def split_labels(gray): + ink=cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY_INV,31,3);H,W=gray.shape;hr=cv2.morphologyEx(ink,cv2.MORPH_OPEN,np.ones((1,max(60,W//8)),np.uint8));vr=cv2.morphologyEx(ink,cv2.MORPH_OPEN,np.ones((max(40,H//3),1),np.uint8));ink=cv2.subtract(ink,hr|vr);nc,_,stats,_=cv2.connectedComponentsWithStats(ink) + heights=[a[3]for a in stats[1:]if 4<=a[3]<=H*.3 and a[4]>=5];charh=float(np.median(heights))if heights else 18.;profile=ink.sum(1)/255;bands=runs(profile>=max(2,.001*W));merged=[] + for a,b in bands: + if merged and a-merged[-1][1]<=max(2,.25*charh):merged[-1][1]=b + else:merged.append([a,b]) + out=[] + for y,Y in merged: + if Y-y<3:continue + xbands=runs((ink[y:Y]>0).any(0));groups=[] + for x,X in xbands: + if groups and x-groups[-1][1]<=2*charh:groups[-1][1]=X + else:groups.append([x,X]) + for x,X in groups: + if X-x>=2 and int((ink[y:Y,x:X]>0).sum())>=5:out.append([max(0,int(x-3)),max(0,int(y-3)),min(W,int(X+3)),min(H,int(Y+3))]) + return out,dict(character_height_upscaled=charh,rows=len(merged),labels=len(out),split_gap_character_heights=2.) +class Collector: + def __init__(self,device='cpu'): + from rtl_reader import RTLReader + self.reader=RTLReader('__BINA_ROOT__/stallion/bina04-print-textgate-20261102/run/selected-printed-recognizer.pt','__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json',device=device) + def __call__(self,image,base): + from text_contract import decode_prediction + new=[];traces=[] + for region in header_boxes(image): + x,y,X,Y=region['box'];rgb=np.asarray(image.crop((x,y,X,Y)).convert('RGB'));gray=rgb.min(2).astype('float32');bg=cv2.GaussianBlur(gray,(0,0),max(9,(Y-y)/12));norm=np.clip(gray/(bg+1e-4)*245,0,255).astype('uint8');norm=cv2.createCLAHE(clipLimit=2.,tileGridSize=(8,8)).apply(norm);up=cv2.resize(norm,None,fx=3,fy=3,interpolation=cv2.INTER_CUBIC);boxes,trace=split_labels(up);traces.append(dict(**region,segmentation=trace)) + for start in range(0,len(boxes),24): + bb=boxes[start:start+24];_,receipts=self.reader(up,np.asarray(bb,np.float32)) + for b,g in zip(bb,receipts): + text=decode_prediction(g['raw_visual_text']);box=[b[0]/3+x,b[1]/3+y,b[2]/3+x,b[3]/3+y];extent=g['padded_source_extent'];geom=dict(raw_visual_text=g['raw_visual_text'],raw_token_ids=g.get('raw_token_ids',[]),confidence=g['confidence'],frame_centers=[v/3+x for v in g.get('frame_centers',[])],source_pixels_extent=[extent[0]/3+x,extent[1]/3+y,extent[2]/3+x,extent[3]/3+y],padded_source_extent=[extent[0]/3+x,extent[1]/3+y,extent[2]/3+x,extent[3]/3+y],preprocessing_contract='3x bicubic, min-RGB background division and CLAHE; fixed crop_line64',source_pixels_modified=True) + new.append(dict(box=box,text=text,visual_text=g['raw_visual_text'],confidence=g['confidence'],geometry=geom,layer='printed',recognizer='frozen_overlap_short_expert',printed_probability=1.,enhanced_header=True,header_region=region)) + return dict(new=new,old_support=[1.]*len(base['lines']),WH=list(image.size),header_regions=traces) + +def clean(s):return ''.join(s.split()).replace('\u200c','') +def isolated(line,lines,maxlen,radius,W): + t=clean(line['text']);b=line['box'];h=max(2,b[3]-b[1]);cx=(b[0]+b[2])/2 + if not t or len(t)>maxlen:return False + if all(c.isdigit()for c in t)and(cx<.16*W or cx>.84*W):return False + for q in lines: + if q is line or len(clean(q['text']))<=2:continue + z=q['box'];hh=max(2,z[3]-z[1]);over=min(b[3],z[3])-max(b[1],z[1]);gap=max(0,max(b[0],z[0])-min(b[2],z[2])) + if over>=.5*min(h,hh)and gap<=radius*max(h,hh):return False + return True + +def apply(base,prepared,params): + p=dict(printed_probability=.5,read_confidence=params['header_confidence'],old_support=.5);result=add_printed(base,prepared,p);original=copy.deepcopy(result);removed=[] + if params['max_chars']: + removed=[l for l in result['lines']if isolated(l,result['lines'],params['max_chars'],params['radius'],prepared['WH'][0])] + if not removed:return result + # Preserve grids as atomic structures; their punctuation/digit cells have a dedicated path. + if any(r.get('table_trace',{}).get('grid')for r in result['regions']):return result + keys={(tuple(l['box']),l['text'])for l in removed};ll=[l for l in result['lines']if(tuple(l['box']),l['text'])not in keys];result['lines']=ll;regions=[] + for r in result['regions']: + r['lines']=[l for l in r.get('lines',[])if(tuple(l['box']),l['text'])not in keys];r['text']='\n'.join(l['text']for l in r['lines']) + if r['lines']:regions.append(r) + result.update(regions=regions,text='\n\n'.join(r['text']for r in regions),global_readings=ll,emitted_boxes=[l['box']for l in ll],recognition_extents=[l.get('geometry',{}).get('source_pixels_extent',l['box'])for l in ll]);result['isolated_mark_filter']=dict(removed=removed,parameters=params,margin_digits_exempt=True);return result diff --git a/stallion/bina04-header-marks-95k-20261103/production.py b/stallion/bina04-header-marks-95k-20261103/production.py new file mode 100644 index 0000000000000000000000000000000000000000..a33cc789f8e21931240908eb5aa905dc1af426a2 --- /dev/null +++ b/stallion/bina04-header-marks-95k-20261103/production.py @@ -0,0 +1,16 @@ +import json,importlib.util,sys +from pathlib import Path +R=Path(__file__).resolve().parent +class Runtime: + def __init__(self,device='auto'): + s=importlib.util.spec_from_file_location('frozen_parent_header_marks','__BINA_ROOT__/stallion/bina04-numeric-routing-95k-20261103/consumer.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);self.params=json.loads((R/'CALIBRATION.json').read_text())['selected']['parameters'];self.collector=None;sys.path.insert(0,str(R)) + def __call__(self,image,old,kind,name='',**kwargs): + out=self.parent(image,old,kind,name,**kwargs) + if self.params['header_confidence']>1:p=dict(new=[],old_support=[1.]*len(out['lines']),WH=list(image.size),header_regions=[]) + else: + from candidate import Collector + if self.collector is None:self.collector=Collector('cpu') + p=self.collector(image,out) + from candidate import apply + return apply(out,p,self.params) + def close(self):self.parent.close() diff --git a/stallion/bina04-header-marks-95k-20261103/serializer.py b/stallion/bina04-header-marks-95k-20261103/serializer.py new file mode 100644 index 0000000000000000000000000000000000000000..d8d53b4a5de147243f4366a0d447b7d6f06b5999 --- /dev/null +++ b/stallion/bina04-header-marks-95k-20261103/serializer.py @@ -0,0 +1,82 @@ +"""Learned visual printed layer; body records and order remain immutable.""" +import copy,json,sys +from pathlib import Path +import numpy as np +from PIL import Image +R=Path(__file__).resolve().parent;P=Path('__BINA_ROOT__/stallion/indra-scale');sys.path[:0]=[str(P),'__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031','__BINA_ROOT__/stallion/bina04-indra-pilot-20261027/upstream'] +def body(p):return json.dumps([l for l in p['lines']if l.get('layer')!='printed'],sort_keys=True,ensure_ascii=False) +def hit(a,b): + z=max(0,min(a[2],b[2])-max(a[0],b[0]))*max(0,min(a[3],b[3])-max(a[1],b[1]));return z/max(1,min((a[2]-a[0])*(a[3]-a[1]),(b[2]-b[0])*(b[3]-b[1]))) +def same_glyph_evidence(new,old): + """Independent heads must not serialize the same CTC glyphs twice. + Different text at the same location is preserved, including print under pen. + """ + if old.get('layer')=='printed':return False + a=new['box'];b=old['box'];ah=max(1,a[3]-a[1]);bh=max(1,b[3]-b[1]) + if hit(a,b)<.5 or abs((a[1]+a[3]-b[1]-b[3])/2)>.5*min(ah,bh):return False + ng=new.get('geometry',{});og=old.get('geometry',{}) + if og.get('frame_centers_source_valid',True)is False:return False + nv=ng.get('raw_visual_text',new.get('visual_text',''));ov=og.get('raw_visual_text',old.get('visual_text',''));nc=ng.get('frame_centers',[]);oc=og.get('frame_centers',[]) + if not nv or len(nc)!=len(nv) or len(oc)!=len(ov):return False + start=0 + while True: + at=ov.find(nv,start) + if at<0:return False + delta=np.abs(np.asarray(nc)-np.asarray(oc[at:at+len(nv)])) + if float(np.quantile(delta,.9))<=max(2.,.5*min(ah,bh)):return True + start=at+1 + +class Layer: + def __init__(self,device='cuda'): + import torch + from surya.detection import DetectionPredictor + from rtl_reader import RTLReader + self.det=DetectionPredictor.local(device=device,dtype=torch.float32);q=torch.load(R/'run/selected.pt',map_location='cpu',weights_only=True);assert q['classes']==['printed','handwriting'];self.det.model.load_state_dict(q['model']);self.det.model.eval();self.det._disable_tqdm=True;self.reader=RTLReader('__BINA_ROOT__/stallion/bina04-print-textgate-20261102/run/selected-printed-recognizer.pt','__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json',device=device) + def infer(self,image,base): + from surya.detection.heatmap import parallel_get_boxes + from text_contract import decode_prediction + W,H=image.size;preds,sizes=next(self.det.batch_detection([image],batch_size=1));p,h=preds[0];raw=parallel_get_boxes([np.minimum(1,p*1.5),h],sizes[0]).model_dump(mode='json');boxes=[] + def support(b): + x,y,X,Y=b;hh,ww=p.shape;a=max(0,round(x*ww/W));A=min(ww,round(X*ww/W));b0=max(0,round(y*hh/H));B=min(hh,round(Y*hh/H));z=p[b0:B,a:A];return float(np.quantile(z,.65))if z.size else 0. + atomic=[r for r in base['regions']if r.get('table_trace',{}).get('grid')];gridboxes=[] + for r in atomic: + ll=r.get('lines',[]) + if ll:gridboxes.append([min(l['box'][0]for l in ll),min(l['box'][1]for l in ll),max(l['box'][2]for l in ll),max(l['box'][3]for l in ll)]) + for a in raw['bboxes']: + q=np.asarray(a['polygon']);b=[float(q[:,0].min()),float(q[:,1].min()),float(q[:,0].max()),float(q[:,1].max())] + if not any(hit(b,g)>.8 for g in gridboxes):boxes.append((b,support(b))) + up=np.asarray(image.resize((2*W,2*H),Image.Resampling.BICUBIC).convert('L'));new=[] + for i in range(0,len(boxes),24): + bb=boxes[i:i+24];_,receipts=self.reader(up,np.asarray([[2*v for v in b]for b,_ in bb],np.float32)) + for (b,s),g in zip(bb,receipts): + g={k:v for k,v in g.items()if k!='_numeric_capture_id'};text=decode_prediction(g['raw_visual_text']);geometry=dict(original_WH=[W,H],ink_box=[v/2 for v in g['ink_box']],source_pixels_extent=[v/2 for v in g['padded_source_extent']],padded_source_extent=[v/2 for v in g['padded_source_extent']],raw_visual_text=g['raw_visual_text'],raw_token_ids=g.get('raw_token_ids',[]),confidence=g['confidence'],frame_centers=[v/2 for v in g.get('frame_centers',[])],upscale=2,upscaled_input_receipt=g,preprocessing_contract='2x bicubic then unchanged crop_line64 and overlap-trained short-label expert') + new.append(dict(box=b,text=text,visual_text=g['raw_visual_text'],confidence=g['confidence'],geometry=geometry,layer='printed',recognizer='frozen_overlap_short_expert',printed_probability=s,new_two_class_printed=True)) + return dict(new=new,old_support=[support(l['box'])if l.get('layer')=='printed'else None for l in base['lines']],raw=raw,WH=[W,H],no_template_or_order_model=True) +def apply(base,prepared,params): + if params.get('disabled'):return copy.deepcopy(base) + result=copy.deepcopy(base);original=body(base);new=[l for l in prepared['new']if l['text'].strip()and l['confidence']>=params['read_confidence']and l['printed_probability']>=params['printed_probability']];duplicates=[l for l in new if any(same_glyph_evidence(l,b)for b in base['lines'])];new=[l for l in new if not any(same_glyph_evidence(l,b)for b in base['lines'])];new.sort(key=lambda l:(-l['printed_probability'],-l['confidence']));unique=[] + for l in new: + if not any(hit(l['box'],q['box'])>.6 for q in unique):unique.append(l) + keep=[];removed=[] + for i,l in enumerate(base['lines']): + if l.get('layer')!='printed':keep.append(copy.deepcopy(l));continue + hits=[n for n in unique if hit(l['box'],n['box'])>.5];b=l['box'];covered=bool(hits)and max(0,min(b[2],max(n['box'][2]for n in hits))-max(b[0],min(n['box'][0]for n in hits)))/max(1,b[2]-b[0])>=.8 + if not covered and prepared['old_support'][i]>=params['old_support']:keep.append(copy.deepcopy(l)) + else:removed.append(l) + # Never re-sort existing nodes. Insert new print by the nearest physical anchor. + slots={i:[]for i in range(len(keep)+1)} + for l in unique: + cy=(l['box'][1]+l['box'][3])/2;cx=(l['box'][0]+l['box'][2])/2 + if keep: + k=min(range(len(keep)),key=lambda i:abs(cy-(keep[i]['box'][1]+keep[i]['box'][3])/2)+.15*abs(cx-(keep[i]['box'][0]+keep[i]['box'][2])/2));slot=k+int(cy>=(keep[k]['box'][1]+keep[k]['box'][3])/2) + else:slot=0 + slots[slot].append(l) + lines=[] + for i in range(len(keep)+1): + lines+=sorted(slots[i],key=lambda l:((l['box'][1]+l['box'][3])/2,-l['box'][0])) + if i Converter: + """A ready Converter (see ``Converter`` for the options). mode="ocr" never + reads the PDF text layer. furniture: keep page headers and footers in the + markdown. math: write math as LaTeX (slower on math pages; off by default).""" + return Converter( + OrtOCR( + model_dir(path, math=math), + threads=threads, + providers=providers, + furniture=furniture, + math=math, + ), + dpi=dpi, + mode=mode, + image_ocr=image_ocr, + ) diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/det_post.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/det_post.py new file mode 100644 index 0000000000000000000000000000000000000000..e0225aeafc0345c035274019727aebc5c425a984 --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/det_post.py @@ -0,0 +1,155 @@ +"""Torch-free pre- and post-processing for LineNet (shared by training and inference). + +Inference with ONNX Runtime needs only numpy and cv2, so these functions live +apart from the PyTorch model in det_model.py, which re-exports them. +""" + +from __future__ import annotations + +import math + +import numpy as np + +# Target geometry, shared by training (encode) and inference (decode). +CORE_FRAC = 0.4 # core band height as a fraction of the line ink height +CORE_X_SHRINK = 0.15 # the core is shorter than the line by this fraction of its height, at each end +OUT_STRIDE = 2 +INPUT_MULTIPLE = 32 # H and W must divide by this + + +# --------------------------------------------------------------------------- +# Input preprocessing (shared by training and inference) +# --------------------------------------------------------------------------- + + +def page_to_input(gray: np.ndarray, long_side: int) -> tuple[np.ndarray, float]: + """Resize a uint8 grayscale page so its long side is ``long_side`` and pad + to a multiple of 32 with white. Returns (float32 [H, W] ink map, scale).""" + import cv2 + + h, w = gray.shape + scale = long_side / max(h, w) + nh, nw = max(1, round(h * scale)), max(1, round(w * scale)) + if (nh, nw) != (h, w): + interp = cv2.INTER_AREA if scale < 1 else cv2.INTER_LINEAR + gray = cv2.resize(gray, (nw, nh), interpolation=interp) + ph = math.ceil(nh / INPUT_MULTIPLE) * INPUT_MULTIPLE + pw = math.ceil(nw / INPUT_MULTIPLE) * INPUT_MULTIPLE + out = np.zeros((ph, pw), np.float32) # ink map: 0 = white + out[:nh, :nw] = 1.0 - gray.astype(np.float32) / 255.0 + return out, scale + + +# --------------------------------------------------------------------------- +# Targets +# --------------------------------------------------------------------------- + + +def encode_targets( + boxes: np.ndarray, + in_hw: tuple[int, int], + valid_hw: tuple[int, int], + ignore: np.ndarray | None = None, +): + """Build training targets at 1/2 resolution. + + boxes: [N, 4] line ink boxes in input pixels (x0, y0, x1, y1) + in_hw: padded input size; valid_hw: size of the real page inside it + ignore: [M, 4] regions with no loss, in input pixels + + Returns core [h, w] float32, dist [2, h, w] float32, weight [h, w] float32 + (0 on padding and ignore regions), and reg_mask [h, w] bool (core pixels). + """ + s = OUT_STRIDE + h, w = in_hw[0] // s, in_hw[1] // s + core = np.zeros((h, w), np.float32) + dist = np.zeros((2, h, w), np.float32) + weight = np.zeros((h, w), np.float32) + weight[: math.ceil(valid_hw[0] / s), : math.ceil(valid_hw[1] / s)] = 1.0 + if ignore is not None: + for x0, y0, x1, y1 in ignore: + weight[ + max(0, int(y0 // s)) : max(0, math.ceil(y1 / s)), + max(0, int(x0 // s)) : max(0, math.ceil(x1 / s)), + ] = 0.0 + # Larger lines first, so a small line drawn later keeps its own band. + order = np.argsort(-(boxes[:, 3] - boxes[:, 1])) if len(boxes) else [] + ys = (np.arange(h, dtype=np.float32) + 0.5) * s # pixel centres in input px + for i in order: + x0, y0, x1, y1 = boxes[i] + lh = y1 - y0 + if lh < 2 or x1 - x0 < 2: + continue + cy = 0.5 * (y0 + y1) + half = 0.5 * CORE_FRAC * lh + r0 = max(0, int(math.floor((cy - half) / s))) + r1 = min(h, max(r0 + 1, int(math.ceil((cy + half) / s)))) + sx = min(CORE_X_SHRINK * lh, 0.25 * (x1 - x0)) + c0 = max(0, int(math.floor((x0 + sx) / s))) + c1 = min(w, max(c0 + 1, int(math.ceil((x1 - sx) / s)))) + if r0 >= h or c0 >= w: + continue + core[r0:r1, c0:c1] = 1.0 + dist[0, r0:r1, c0:c1] = (ys[r0:r1] - y0)[:, None] + dist[1, r0:r1, c0:c1] = (y1 - ys[r0:r1])[:, None] + # A line inside an ignore region is still a positive. + weight[r0:r1, c0:c1] = np.maximum(weight[r0:r1, c0:c1], 1.0) + return core, dist, weight, core > 0.5 + + +# --------------------------------------------------------------------------- +# Decoding +# --------------------------------------------------------------------------- + + +def decode( + out: np.ndarray, + thresh: float = 0.5, + min_area: int = 2, + grow_y: float = 0.0, + grow_x: float = 0.0, +) -> np.ndarray: + """Decode one page. out: [3, h, w] (core logit, top dist, bottom dist). + + ``grow_y`` / ``grow_x`` widen every box by that fraction of its height + (each side), to absorb the +-1 px error of the 1/2-resolution output. + + Returns [N, 5] boxes in input pixels: x0, y0, x1, y1, score.""" + import cv2 + + prob = 1.0 / (1.0 + np.exp(-out[0])) + mask = (prob > thresh).astype(np.uint8) + n, lab, stats, _ = cv2.connectedComponentsWithStats(mask, connectivity=4) + if n <= 1: + return np.zeros((0, 5), np.float32) + s = OUT_STRIDE + flat = lab.ravel() + keep = flat > 0 + idx = flat[keep] + ys = (np.indices(lab.shape)[0].ravel()[keep].astype(np.float32) + 0.5) * s + top = ys - out[1].ravel()[keep] + bot = ys + out[2].ravel()[keep] + sc = prob.ravel()[keep] + cnt = np.bincount(idx, minlength=n).astype(np.float32) + # Mean is a robust enough summary here and is far cheaper than a median. + y0 = np.bincount(idx, weights=top, minlength=n) / np.maximum(cnt, 1) + y1 = np.bincount(idx, weights=bot, minlength=n) / np.maximum(cnt, 1) + score = np.bincount(idx, weights=sc, minlength=n) / np.maximum(cnt, 1) + boxes = [] + for i in range(1, n): + if stats[i, cv2.CC_STAT_AREA] < min_area: + continue + lh = max(1.0, y1[i] - y0[i]) + ex = min(CORE_X_SHRINK * lh, 0.5 * lh) + x0 = stats[i, cv2.CC_STAT_LEFT] * s - ex + x1 = (stats[i, cv2.CC_STAT_LEFT] + stats[i, cv2.CC_STAT_WIDTH]) * s + ex + boxes.append( + ( + x0 - grow_x * lh, + y0[i] - grow_y * lh, + x1 + grow_x * lh, + y1[i] + grow_y * lh, + score[i], + ) + ) + return np.asarray(boxes, np.float32).reshape(-1, 5) diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/document.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/document.py new file mode 100644 index 0000000000000000000000000000000000000000..d5974311cb5e775b198a8b9d03b6bdc4da0247ed --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/document.py @@ -0,0 +1,426 @@ +"""Whole documents: the PDF text layer where it can be trusted, OCR elsewhere. + +For each PDF page (``mode="auto"``): + +1. Read the text layer with pdfium and check it (``textlayer.read_text_layer``). A page + without a sound layer (a scan, an OCR layer, a broken font map) goes to OCR. +2. The detector finds the visual lines, as in OCR. Each text-layer character + goes to the detected line that contains its center. Inside a line, the + characters of each text-layer run (pdfium's segment between line breaks) keep the PDF order + and spaces, and runs are ordered by x, right to left in a right-to-left line. So inline math, sub- and + superscripts and footnote marks land in their line, and a run across two + table cells splits between them. A detected line with no text (text inside + an image) is OCR'd when the recognizer reads the page's script. +3. The rest is the OCR path: lines split at wide gaps, reading order, layout + classes and markdown. The PDF content order is kept when it is coherent + (few upward jumps that are not column changes, and the order model agrees + with at least 85% of consecutive pairs). + +``mode="ocr"`` never reads the text layer; ``mode="text"`` uses it even when +the checks fail (and OCR only on pages with no text at all). +""" + +from __future__ import annotations + +import time +import unicodedata +from pathlib import Path + +import numpy as np + +from .pipeline import InputError, OrtOCR, Page, is_pdf, read_images +from .textlayer import BROKEN_FONT_MAP, page_turn, read_text_layer, vertical_layout + +CHECK_DPI = 72.0 # resolution of the render for the text-layer ink checks +MAX_PAGE_PIXELS = 100e6 # a page larger than this at the set dpi renders at a lower dpi (large scans reach ~65 MP) + +READABLE_MIN = 0.9 # share of the layer's letters the recognizer must know +AGREE_LINES = 6 # text-layer lines whose script is checked against the page image +AGREE_SHARE = 0.25 # under this share of agreeing lines, the layer is not the page's text +LATIN_CODES = 0.9 # a layer line this Latin while its image is another script: legacy font codes +NOT_THE_PAGE = "text layer does not match the page" # Page.note when the check sends a page to OCR + + +def latin_share(text: str) -> float: + """Share of the letters in text that are basic Latin (a-z, A-Z).""" + letters = [c for c in text if c.isalpha()] + return sum(c.isascii() for c in letters) / max(len(letters), 1) + + +def layer_agrees(agree: list) -> bool: + """Whether a text layer matches its page: at least AGREE_SHARE of the checked + lines agree. A legacy font (Kruti Dev and DevLys for Hindi, Bijoy for Bangla) + stores Latin codes for another script, so almost every line disagrees.""" + return not agree or sum(agree) >= AGREE_SHARE * len(agree) + + +def readable(ocr: OrtOCR, runs: list) -> bool: + """Whether the recognizer can read the script of these text-layer runs.""" + letters = [c for r in runs for c in r.text if c.isalpha()] + if not letters: + return True + vocabs = [ocr.vocab.index] + [v.index for _, v in ocr.groups.values()] + # A recognizer that reads Hangul as jamo knows a syllable by its jamo. + known = sum( + any( + c in v or all(j in v for j in unicodedata.normalize("NFD", c)) + for v in vocabs + ) + for c in letters + ) + return known >= READABLE_MIN * len(letters) + + +def right_to_left(tokens: list) -> bool: + """Unicode bidi base direction: most strong characters are right-to-left.""" + rtl = ltr = 0 + for t in tokens: + b = unicodedata.bidirectional(t[0]) if len(t[0]) == 1 else "" + rtl += b in ("R", "AL") + ltr += b == "L" + return rtl > ltr + + +def _covered(box, boxes: np.ndarray, frac: float = 0.5) -> bool: + """Whether more than ``frac`` of ``box`` lies inside one of ``boxes``.""" + ix = np.clip( + np.minimum(box[2], boxes[:, 2]) - np.maximum(box[0], boxes[:, 0]), 0, None + ) + iy = np.clip( + np.minimum(box[3], boxes[:, 3]) - np.maximum(box[1], boxes[:, 1]), 0, None + ) + area = max((box[2] - box[0]) * (box[3] - box[1]), 1e-6) + return bool(((ix * iy) / area > frac).any()) + + +def _render(page, scale: float) -> np.ndarray: + arr = page.render(scale=scale, grayscale=True).to_numpy() + return np.ascontiguousarray(arr[..., 0] if arr.ndim == 3 else arr) + + +def assign_runs(runs: list, boxes: np.ndarray, k: float) -> tuple[list[list], int]: + """Tokens (char, x, content index) per detected line. + + A run is one text-layer line piece (``textlayer.TextLine``), in content + order. Each character goes to the smallest detected box that contains its + center (a space follows the character before it), so a run that spans two + table cells splits between them. Inside a detected line, each run's part + keeps the PDF character order, and the parts are ordered by x. ``boxes`` + are pixels, ``k`` pixels per point. Returns (tokens per detected line, + characters outside every box).""" + parts: list[dict] = [{} for _ in range(len(boxes))] # line -> run idx -> tokens + lost = 0 + if not len(boxes): + return [], sum(not c.isspace() for r in runs for c, _ in r.chars) + area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) + for pos, run in enumerate(runs): + idx = run.seq if run.seq is not None else pos + j = None + for c, b in run.chars: + if not c.isspace(): + cx, cy = (b[0] + b[2]) / 2 * k, (b[1] + b[3]) / 2 * k + inside = ( + (boxes[:, 0] <= cx) + & (cx <= boxes[:, 2]) + & (boxes[:, 1] <= cy) + & (cy <= boxes[:, 3]) + ) + if not inside.any(): + lost += 1 + j = None + continue + j = int(np.flatnonzero(inside)[np.argmin(area[inside])]) + parts[j].setdefault(idx, []).append((c, cx, idx)) + elif j is not None: + prev = parts[j][idx][-1] + parts[j][idx].append((c, prev[1], idx)) + toks: list[list] = [] + for line in parts: + tk: list = [] + rtl = right_to_left([t for part in line.values() for t in part]) + order = ( + sorted(line, key=lambda i: -max(t[1] for t in line[i])) + if rtl + else sorted(line, key=lambda i: min(t[1] for t in line[i])) + ) + for idx in order: + if tk: + tk.append((" ", tk[-1][1], idx)) # parts of different runs + tk += line[idx] + toks.append(tk) + return toks, lost + + +class Converter: + def __init__( + self, + ocr: OrtOCR, + dpi: float = 200.0, + mode: str = "auto", + image_ocr: bool = True, + ): + """mode: auto (text layer where sound, OCR elsewhere), ocr (always OCR, + never read the text layer), text (text layer even when the checks fail). + image_ocr: OCR detected lines that + hold no text-layer characters (False: drop them).""" + if mode not in ("auto", "ocr", "text"): + raise ValueError(f"mode must be auto, ocr or text, not {mode!r}") + self.ocr, self.dpi, self.mode = ocr, dpi, mode + self.image_ocr = image_ocr + + def page_scale(self, page) -> float: + """Pixels per point for this page: the set dpi, lowered for pages so + large that the render would pass MAX_PAGE_PIXELS (posters, plans).""" + w, h = page.get_size() + k = self.dpi / 72.0 + return min(k, (MAX_PAGE_PIXELS / max(w * h, 1.0)) ** 0.5) + + def pdf_page(self, pdf_path: str, doc, index: int) -> Page: + page = doc[index] + t: dict = {} + note = "mode=ocr" + if self.mode != "ocr": + t0 = time.perf_counter() + s = CHECK_DPI / 72.0 + check = _render(page, s) + tl = read_text_layer(page, check, s) + # A table set sideways: most characters share one quarter turn. Read + # the page turned upright, as the OCR path does with orient.onnx. + k = page_turn(tl.stats) + rot = page.get_rotation() + if k and (rot + 90 * k) % 360 == 0: + # The page's /Rotate alone turns the text: read the page without + # it. pdfium builds its text runs in the displayed frame, and on + # a turned page it can join a line's font runs out of order. + page.set_rotation(0) + upright = _render(page, s) + tl = read_text_layer(page, upright, s) + if tl.lines and (tl.trusted or self.mode == "text"): + tl.stats["turn"] = k + t["text_ms"] = 1000 * (time.perf_counter() - t0) + out = self._text_page(page, tl, t, check=self.mode == "auto") + if out is not None: + return out + page.set_rotation(rot) + tl = read_text_layer( + page, np.ascontiguousarray(np.rot90(check, -k)), s, k + ) + elif k: + tl = read_text_layer( + page, np.ascontiguousarray(np.rot90(check, -k)), s, k + ) + elif vertical_layout(tl.stats): + # Vertical writing: upright characters, read in the page turned a + # quarter turn counter-clockwise (columns become lines). + tl = read_text_layer( + page, np.ascontiguousarray(np.rot90(check, 1)), s, 0, 3 + ) + tl.stats["vertical"] = True + t["text_ms"] = 1000 * (time.perf_counter() - t0) + # A layer the recognizer cannot read beats OCR, unless its codes are junk. + unreadable = tl.reason != BROKEN_FONT_MAP and not readable( + self.ocr, tl.lines + ) + note = tl.reason + if tl.lines and (tl.trusted or self.mode == "text" or unreadable): + # A trusted layer in a script the recognizer reads is checked + # against what the page shows (legacy fonts store other codes). + out = self._text_page( + page, tl, t, check=self.mode == "auto" and tl.trusted and not unreadable + ) + if out is not None: + return out + note = NOT_THE_PAGE + # A layer with a broken font map still places its glyphs: when most + # run upright, the page is upright (orient.onnx is not asked). + st = tl.stats + turn = not ( + tl.reason == BROKEN_FONT_MAP and st["rotated"] < 0.5 * st["chars"] + ) + else: + turn = True + t0 = time.perf_counter() + gray = _render(page, self.page_scale(page)) + t["render_ms"] = 1000 * (time.perf_counter() - t0) + out = self.ocr(gray, turn) + out.timings = {**t, **out.timings} + out.timings["total_ms"] = sum( + v for k, v in out.timings.items() if k != "total_ms" + ) + out.note = f"{note}; {out.note}" + return out + + def _text_page(self, page, tl, t: dict, check: bool = False) -> Page | None: + """The page from its text layer. check: first compare the script of up to + AGREE_LINES of the layer's lines with the script the line classifier sees + in their images; when they do not match (layer_agrees), return None, so OCR + reads the page.""" + k = self.page_scale(page) + turn = tl.stats.get("turn", 0) + frame = tl.stats.get("frame", turn) + vertical = bool(tl.stats.get("vertical")) + t0 = time.perf_counter() + gray = _render(page, k) + if frame: + gray = np.ascontiguousarray(np.rot90(gray, -frame)) + t["render_ms"] = 1000 * (time.perf_counter() - t0) + t0 = time.perf_counter() + boxes = self.ocr.detect(gray) + t["det_ms"] = 1000 * (time.perf_counter() - t0) + toks, lost = assign_runs(tl.lines, boxes[:, :4], k) + rec: list = [None] * len(boxes) + empty = [] + for i, tk in enumerate(toks): + text = self.ocr._decode(tk) if tk else "" + if text: + rec[i] = (text, 1.0, tk) + else: + empty.append(i) + if check: + t0 = time.perf_counter() + full = [i for i, r in enumerate(rec) if r and sum(c.isalpha() for c in r[0]) >= 4] + pick = full[:: max(1, len(full) // AGREE_LINES)][:AGREE_LINES] + if pick: + # A line agrees unless its layer text is Latin while the router is + # sure its image is another script. + groups, sure = self.ocr.box_groups(gray, boxes[pick]) + agree = [ + not (latin_share(rec[i][0]) >= LATIN_CODES and g != "base" and s) + for i, g, s in zip(pick, groups, sure) + ] + t["check_ms"] = 1000 * (time.perf_counter() - t0) + if not layer_agrees(agree): + return None + source = "text" + # Sideways runs (a rotated table header, a chart's axis label) come from + # the layer as their own lines; detected lines on them are not OCR'd. + side = [r for r in tl.rotated if r.text.strip()] + side_boxes = np.array([[v * k for v in r.bbox] for r in side], np.float32) + if len(side) and empty: + empty = [i for i in empty if not _covered(boxes[i, :4], side_boxes)] + if empty and self.image_ocr and readable(self.ocr, tl.lines): + t0 = time.perf_counter() + for i, r in zip( + empty, self.ocr.recognize(gray, boxes[empty], vertical=vertical) + ): + if r[0].strip(): + rec[i] = r + source = "hybrid" + t["rec_ms"] = 1000 * (time.perf_counter() - t0) + if getattr(self.ocr, "math", None) is not None: + t0 = time.perf_counter() + self._read_math(gray, boxes, rec) + t["math_ms"] = 1000 * (time.perf_counter() - t0) + keep = [i for i, r in enumerate(rec) if r is not None] + kept_boxes = boxes[keep].reshape(-1, boxes.shape[1]) + kept_rec = [rec[i] for i in keep] + if len(side): + extra = np.zeros((len(side), boxes.shape[1]), np.float32) + extra[:, :4] = side_boxes + if boxes.shape[1] > 4: + extra[:, 4] = 1.0 + kept_boxes = np.concatenate([kept_boxes, extra]) + for r, b in zip(side, side_boxes): + cx = float(b[0] + b[2]) / 2 # one x: a sideways line is never cut + kept_rec.append((r.text, 1.0, [(c, cx, r.seq) for c in r.text])) + out = self.ocr.assemble( + (gray.shape[1], gray.shape[0]), + kept_boxes, + kept_rec, + t, + vertical=vertical, + ) + out.source = source + out.rotation = 90 * turn + if vertical: # back on the page as it is: (x, y) -> (W - y, x) + W = gray.shape[0] + + def back(b): + return [W - b[3], b[0], W - b[1], b[2]] + + for ln in out.lines: + ln.box = back(ln.box) + for b in out.blocks: + b["bbox"] = back(b["bbox"]) + out.vertical = True + out.note += f"; {lost} chars outside detected lines" if lost else "" + return out + + def _read_math(self, gray, boxes, rec: list) -> None: + """Text-layer lines that hold math (the classifier, on their crops) are read + by MathRec instead: the layer has the symbols but not the LaTeX. The line + keeps its place in the PDF content order (its first character's index).""" + from .line_mask import crop_lines + + idx = [ + i + for i, r in enumerate(rec) + if r is not None + and r[2] + and isinstance(r[2][0][0], str) + and len(r[2][0]) > 2 + ] + if not idx: + return + crops = crop_lines(gray, boxes[idx, :4], self.ocr.height, None) + math = [i for i, f in zip(idx, self.ocr.math.flags(crops)) if f] + if not math: + return + from .math import MATH_MIN_CONF + + for i, (text, conf) in zip(math, self.ocr.math.read(gray, boxes[math])): + # A reading without math, or not confident, leaves the layer's text. + if "$" in text and conf >= MATH_MIN_CONF: + first = min(tk[2] for tk in rec[i][2]) + xc = float(boxes[i, 0] + boxes[i, 2]) / 2 + rec[i] = (text, 1.0, [(text, xc, first)]) + + def convert( + self, path: str | Path, max_pages: int = 0, password: str | None = None + ): + """Yield (page index, Page) for a PDF or an image file (the file's bytes + decide which). A page that fails yields an empty Page whose ``source`` + is "error" and whose ``note`` says why; a file that cannot be opened + raises InputError. max_pages: 0 reads every page; a negative value + raises ValueError.""" + if max_pages < 0: + raise ValueError( + f"max_pages must be 0 (all pages) or more, not {max_pages}" + ) + path = Path(path) + if not path.is_file(): + raise InputError(f"{path}: no such file") + if not is_pdf(path): + for i, gray in enumerate(read_images(path)[: max_pages or None]): + yield i, self._safe(lambda: self.ocr(gray)) + return + import pypdfium2 as pdfium + + try: + doc = pdfium.PdfDocument(str(path), password=password) + except pdfium.PdfiumError as e: + locked = "password" in str(e).lower() + raise InputError( + f"{path}: password-protected PDF (pass password=, or --password)" + if locked + else f"{path}: cannot open the PDF ({e})" + ) from e + try: + n = len(doc) if not max_pages else min(len(doc), max_pages) + for i in range(n): + yield i, self._safe(lambda: self.pdf_page(str(path), doc, i)) + finally: + doc.close() + + @staticmethod + def _safe(read) -> Page: + try: + return read() + except Exception as e: # noqa: BLE001 (one bad page must not end the document) + return Page( + [], + {"total_ms": 0.0}, + markdown="", + source="error", + note=f"{type(e).__name__}: {e}", + ) diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/layout_post.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/layout_post.py new file mode 100644 index 0000000000000000000000000000000000000000..49a496f1f57f8aad3f3cc4fffbb8714fe9504187 --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/layout_post.py @@ -0,0 +1,381 @@ +"""Page text from the ordered lines and the OrderNet layout heads (torch-free). + +OrderNet with ``layout`` gives every line a block class and every pair of lines +a "same block" logit. Walking the reading order, a new block starts wherever +the same-block logit between consecutive lines is negative. Each block is then +rendered by its class: + + Section-Header "## " + joined text + Text, Caption, ... one paragraph; a word split by a hyphen at a line end is joined + List-Group one line per item (a new item starts at a bullet or number) + Table an HTML table assembled from the cell lines (rows by vertical + centre, columns anchored on the fullest rows) + Page-Header / Footer left out unless ``keep_furniture`` +""" + +from __future__ import annotations + +import html +import re + +import numpy as np + +from .order_post import BLOCK_CLASSES, _cluster + +HEADER, FOOTER = BLOCK_CLASSES.index("Page-Header"), BLOCK_CLASSES.index("Page-Footer") +SECTION = BLOCK_CLASSES.index("Section-Header") +LIST = BLOCK_CLASSES.index("List-Group") +TABLE = BLOCK_CLASSES.index("Table") + +_WHOLE_MATH = re.compile(r"^\$[^$]+\$$") # a line that is one formula +# A list marker, followed by text or alone on its line (the detector often cuts +# a bullet off as a line of its own; the next line then joins its item). +_ITEM = re.compile(r"^\s*([•\-–▪◦*·→]|\(?\d{1,3}[.)]|\(?[a-zA-Z][.)])(\s|$)") + + +def _cjk(c: str) -> bool: + """Han, kana, CJK punctuation and fullwidth forms: written without spaces + (Korean Hangul is not: Korean separates words with spaces).""" + o = ord(c) + return ( + 0x3000 <= o <= 0x30FF + or 0x3400 <= o <= 0x9FFF + or 0xF900 <= o <= 0xFAFF + or 0xFF00 <= o <= 0xFFEF + or 0x20000 <= o <= 0x2FA1F + ) + + +def join_lines(texts: list[str]) -> str: + """Join the lines of one paragraph. 'tumori-' + 'genesis' -> 'tumorigenesis'; + a line break between two CJK characters takes no space.""" + out = "" + for t in texts: + t = t.strip() + if not t: + continue + if ( + len(out) > 1 + and out.endswith("-") + and out[-2].isalpha() + and t[0].isalpha() + and t[0].islower() + ): + out = out[:-1] + t + elif out and _cjk(out[-1]) and _cjk(t[0]): + out += t + else: + out = f"{out} {t}" if out else t + return out + + +def split_blocks(order: list[int], same: np.ndarray) -> list[list[int]]: + """Runs of the reading order with a positive same-block logit between neighbours.""" + blocks: list[list[int]] = [] + for i in order: + if blocks and same[blocks[-1][-1], i] > 0: + blocks[-1].append(i) + else: + blocks.append([i]) + return blocks + + +def _rows(boxes: np.ndarray) -> list[list[int]]: + """Group cell lines into rows by vertical centre, top to bottom, left to right.""" + cy = (boxes[:, 1] + boxes[:, 3]) / 2 + h = float(np.median(boxes[:, 3] - boxes[:, 1])) + rows: list[list[int]] = [] + for i in np.argsort(cy): + if rows and abs(cy[i] - np.mean(cy[rows[-1]])) < 0.5 * h: + rows[-1].append(int(i)) + else: + rows.append([int(i)]) + return [sorted(r, key=lambda k: boxes[k, 0]) for r in rows] + + +def _merge_wrapped(rows: list[list[int]], boxes: np.ndarray) -> list[list[int]]: + """Join the line rows of wrapped cells into one table row. + + Inside a wrapped cell the gap to the next line is the font's line gap; a new + table row adds cell padding. A row joins the row above when its gap is under + half the table's median row gap and under 0.6 line heights, and it has fewer + lines than the row above (a continuation fills only the wrapped columns).""" + if len(rows) < 3: + return rows + h = float(np.median(boxes[:, 3] - boxes[:, 1])) + gaps = [ + float(boxes[nxt, 1].min() - boxes[cur, 3].max()) + for cur, nxt in zip(rows[:-1], rows[1:]) + ] + med = float(np.median(gaps)) + out = [list(rows[0])] + prev = rows[0] + for g, r in zip(gaps, rows[1:]): + # A continuation fills only the wrapped columns: fewer lines than the row above. + if g < 0.5 * med and g < 0.6 * h and len(r) < len(prev): + out[-1] += r + else: + out.append(list(r)) + prev = r + return out + + +def table_html(boxes: np.ndarray, texts: list[str]) -> str: + """An HTML table from cell lines (one text line per cell, as born-digital PDFs give). + + A first row that is one line across most of the table is a title: it goes + before the table as text. The first remaining row is the header ().""" + rows = _rows(boxes) + span_x = boxes[:, 2].max() - boxes[:, 0].min() + if ( + len(rows) > 2 + and len(rows[0]) == 1 + and max(len(r) for r in rows) > 1 + and boxes[rows[0][0], 2] - boxes[rows[0][0], 0] > 0.6 * span_x + ): + keep = [k for k in range(len(texts)) if k != rows[0][0]] + return ( + texts[rows[0][0]] + + "\n\n" + + table_html(boxes[keep], [texts[k] for k in keep]) + ) + width = max(len(r) for r in rows) + full = [r for r in rows if len(r) == width] + rows = _merge_wrapped(rows, boxes) + # Column bands from the rows with the most cells: those rows rarely span. + bands = np.array( + [ + [ + np.median([boxes[r[c], 0] for r in full]), + np.median([boxes[r[c], 2] for r in full]), + ] + for c in range(width) + ] + ) + # Widen each band to the midpoints between neighbours, so every x falls in one band. + edges = ( + [-np.inf] + + [(bands[c, 1] + bands[c + 1, 0]) / 2 for c in range(width - 1)] + + [np.inf] + ) + out = [""] + for ri, r in enumerate(rows): + tag = "th" if ri == 0 and len(rows) > 1 else "td" + cells: dict[int, tuple[int, str]] = {} + for k in sorted(r, key=lambda k: (boxes[k, 1], boxes[k, 0])): + x0, x1 = boxes[k, 0], boxes[k, 2] + first = int(np.searchsorted(edges, x0, side="right") - 1) + last = int(np.searchsorted(edges, x1, side="right") - 1) + # A cell spans every band it covers by more than half of the band. + span = [ + c + for c in range(max(first, 0), min(last, width - 1) + 1) + if min(x1, bands[c, 1]) - max(x0, bands[c, 0]) + > 0.5 * (bands[c, 1] - bands[c, 0]) + ] or [min(max(first, 0), width - 1)] + c0 = span[0] + prev = cells.get(c0) + text = texts[k] if prev is None else prev[1] + " " + texts[k] + cells[c0] = (len(span) if prev is None else max(prev[0], len(span)), text) + tds, c = [], 0 + while c < width: + if c in cells: + n, text = cells[c] + attr = f' colspan="{n}"' if n > 1 else "" + tds.append(f"<{tag}{attr}>{html.escape(text, quote=False)}") + c += n + else: + tds.append(f"<{tag}>") + c += 1 + out.append("" + "".join(tds) + "") + out.append("
") + return "\n".join(out) + + +def heads_plausible(boxes: np.ndarray, tab: np.ndarray) -> bool: + """The lines of one cell are stacked (they overlap horizontally). When the + cell head joins side-by-side lines in more than 10% of its cells, it has + failed on this table and the geometric assembly is used instead.""" + cells = _cluster(tab[2]) + bad = 0 + for c in cells: + if len(c) < 2: + continue + x0, x1 = boxes[c, 0], boxes[c, 2] + if (np.minimum.outer(x1, x1) - np.maximum.outer(x0, x0)).min() < 0: + bad += 1 + return bad <= 0.1 * len(cells) + + +def table_html_heads(boxes: np.ndarray, texts: list[str], tab: np.ndarray) -> str: + """An HTML table from the OrderNet table heads. tab [3, N, N] = logits + same row, same column, same cell, for the N lines of one table block.""" + cells = _cluster(tab[2]) + cb = np.array( + [ + [boxes[c, 0].min(), boxes[c, 1].min(), boxes[c, 2].max(), boxes[c, 3].max()] + for c in cells + ] + ) + ctext = [ + " ".join(texts[k] for k in sorted(c, key=lambda k: (boxes[k, 1], boxes[k, 0]))) + for c in cells + ] + m = len(cells) + + def cell_logits(k: int) -> np.ndarray: + out = np.zeros((m, m), np.float32) + for a in range(m): + for b in range(a + 1, m): + out[a, b] = out[b, a] = tab[k][np.ix_(cells[a], cells[b])].mean() + return out + + rows = sorted(_cluster(cell_logits(0)), key=lambda g: cb[g, 1].min()) + cols = sorted(_cluster(cell_logits(1)), key=lambda g: cb[g, 0].min()) + row_of = {c: r for r, g in enumerate(rows) for c in g} + col_of = {c: k for k, g in enumerate(cols) for c in g} + centers = [float(np.median((cb[g, 0] + cb[g, 2]) / 2)) for g in cols] + h = float(np.median(boxes[:, 3] - boxes[:, 1])) + grid: dict[tuple[int, int], tuple[int, str]] = {} + for c in range(m): + c0 = col_of[c] + span = 1 + while ( + c0 + span < len(cols) and cb[c, 0] - h <= centers[c0 + span] <= cb[c, 2] + h + ): + span += 1 + key = (row_of[c], c0) + if key in grid: # two cells on one slot: keep both texts + grid[key] = (max(grid[key][0], span), grid[key][1] + " " + ctext[c]) + else: + grid[key] = (span, ctext[c]) + out = [""] + for r in range(len(rows)): + tag = "th" if r == 0 and len(rows) > 1 else "td" + tds, c = [], 0 + while c < len(cols): + if (r, c) in grid: + span, text = grid[(r, c)] + attr = f' colspan="{span}"' if span > 1 else "" + tds.append(f"<{tag}{attr}>{html.escape(text, quote=False)}") + c += span + else: + tds.append(f"<{tag}>") + c += 1 + out.append("" + "".join(tds) + "") + out.append("
") + return "\n".join(out) + + +def page_blocks( + boxes: np.ndarray, + texts: list[str], + order: list[int], + cls_logits: np.ndarray, + same: np.ndarray, +) -> list[dict]: + """The layout blocks behind ``page_markdown``, in reading order: + {"label" (a BLOCK_CLASSES name), "bbox" (pixels, the union of its lines), + "text" (its lines, newline-joined)}.""" + out = [] + for blk in split_blocks(order, same): + b = boxes[blk, :4] + out.append( + { + "label": BLOCK_CLASSES[int(cls_logits[blk].sum(0).argmax())], + "bbox": [ + round(float(b[:, 0].min()), 1), + round(float(b[:, 1].min()), 1), + round(float(b[:, 2].max()), 1), + round(float(b[:, 3].max()), 1), + ], + "text": "\n".join(texts[i] for i in blk), + } + ) + return out + + +def page_markdown( + boxes: np.ndarray, + texts: list[str], + order: list[int], + cls_logits: np.ndarray, + same: np.ndarray, + keep_furniture: bool = False, + cells: dict | None = None, + tab: np.ndarray | None = None, + pictures: np.ndarray | None = None, +) -> str: + """Markdown for one page. boxes [N, 4], texts [N], order = reading order, + cls_logits [N, C], same [N, N] (logit "same block"). ``cells`` maps a line + to its table cells [(box, text)] when the line holds several. ``tab`` + [3, N, N] (same row / column / cell logits) builds tables from the heads. + ``pictures`` [P, 4]: picture regions, each written as ``![](picture-)`` + before the first block below it that it overlaps horizontally.""" + parts: list[str] = [] + part_boxes: list = [] + for blk in split_blocks(order, same): + n_parts = len(parts) + _block_part(parts, blk, boxes, texts, cls_logits, keep_furniture, cells, tab) + if len(parts) > n_parts: + b = boxes[blk, :4] + part_boxes.append( + (b[:, 0].min(), b[:, 1].min(), b[:, 2].max(), b[:, 3].max()) + ) + if pictures is not None and len(pictures): + slots: dict[int, list[str]] = {} + for k, (x0, y0, x1, y1) in enumerate(np.asarray(pictures)[:, :4]): + cy = (y0 + y1) / 2 + at = next( + ( + j + for j, (bx0, by0, bx1, by1) in enumerate(part_boxes) + if (by0 + by1) / 2 > cy and min(x1, bx1) > max(x0, bx0) + ), + len(parts), + ) + slots.setdefault(at, []).append(f"![](picture-{k})") + merged = [] + for j in range(len(parts) + 1): + merged.extend(slots.get(j, [])) + if j < len(parts): + merged.append(parts[j]) + parts = merged + return "\n\n".join(p for p in parts if p) + + +def _block_part(parts, blk, boxes, texts, cls_logits, keep_furniture, cells, tab): + """Append one block's markdown to ``parts`` (nothing for dropped furniture).""" + cls = int(cls_logits[blk].sum(0).argmax()) + texts_b = [texts[i] for i in blk] + if cls in (HEADER, FOOTER) and not keep_furniture: + return + if all(_WHOLE_MATH.match(t.strip()) for t in texts_b): # display equations + parts.append("\n".join("$$" + t.strip()[1:-1] + "$$" for t in texts_b)) + return + if cls == SECTION: + parts.append("## " + join_lines(texts_b)) + elif ( + cls == TABLE + and len(blk) > 1 + and tab is not None + and heads_plausible(boxes[blk, :4], tab[:, blk][:, :, blk]) + ): + parts.append(table_html_heads(boxes[blk, :4], texts_b, tab[:, blk][:, :, blk])) + elif cls == TABLE and len(blk) > 1: + cb, ct = [], [] + for i in blk: + for bx, tx in (cells or {}).get(i) or [(boxes[i, :4], texts[i])]: + cb.append(bx) + ct.append(tx) + parts.append(table_html(np.array(cb, np.float32).reshape(-1, 4), ct)) + elif cls == LIST: + items: list[list[str]] = [] + for t in texts_b: + if not items or _ITEM.match(t): + items.append([t]) + else: + items[-1].append(t) + parts.append("\n".join(join_lines(it) for it in items)) + else: + parts.append(join_lines(texts_b)) diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/line_mask.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/line_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..9806f8ef050c1c0c8c47644496073de071d6c3de --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/line_mask.py @@ -0,0 +1,153 @@ +"""Mask a line window with CenterMask before the recognizer crops the line. + +One function for training and inference: the recognizer must only ever see +crops that went through the same exported mask model (mask.onnx). + + sess = load_mask("mask.onnx") + win, box_in_win = masked_window(gray, box, sess) + img = crop_line(win, box_in_win, height) +""" + +from __future__ import annotations + +import numpy as np + +WINDOW_H = 96 # window height in pixels; the centre line is about a third of it +WINDOW_ABOVE = 1.0 # box heights above and below the line box +WINDOW_SIDE = 0.5 # box heights left and right + + +def window_box(box, w_page: float, h_page: float) -> tuple[float, float, float, float]: + """The window around a line box (page pixels), clipped to the page.""" + x0, y0, x1, y1 = (float(v) for v in box[:4]) + h = max(1.0, y1 - y0) + return ( + max(0.0, x0 - WINDOW_SIDE * h), + max(0.0, y0 - WINDOW_ABOVE * h), + min(w_page, x1 + WINDOW_SIDE * h), + min(h_page, y1 + WINDOW_ABOVE * h), + ) + + +KEEP = 0.5 # pixels with a lower centre-line probability become paper + + +def resize_window(a: np.ndarray, size: tuple[int, int]) -> np.ndarray: + """Scale a window to ``size`` (w, h): linear down to half scale, area below + (anti-aliasing). INTER_AREA at a non-integer scale costs 20x more.""" + import cv2 + + shrink = size[1] / max(1, a.shape[0]) + interp = cv2.INTER_LINEAR if shrink >= 0.5 else cv2.INTER_AREA + return cv2.resize(a, size, interpolation=interp) + + +def load_mask(path: str, threads: int = 1): + import onnxruntime as ort + + so = ort.SessionOptions() + so.intra_op_num_threads = threads + so.inter_op_num_threads = 1 + return ort.InferenceSession(str(path), so, providers=["CPUExecutionProvider"]) + + +def _window(gray: np.ndarray, box) -> tuple[np.ndarray, tuple[float, float]]: + H, W = gray.shape + x0, y0, x1, y1 = (int(round(v)) for v in window_box(box, W, H)) + return gray[y0:y1, x0:x1], (x0, y0) + + +def mask_windows(sess, wins: list[np.ndarray]) -> list[np.ndarray]: + """Centre-line keep masks (bool, window resolution) for grayscale windows. + + The probabilities are scaled back bilinearly before the threshold: a + threshold at mask resolution with a nearest-neighbour upscale breaks the + stroke edges (surya OCR 2 CER on M804 lines 0.30 -> 0.36).""" + import cv2 + + if not wins: + return [] + xs, sizes = [], [] + for w in wins: + h, wd = w.shape + W = max(32, int(round(wd * WINDOW_H / max(h, 1) / 8)) * 8) + ink = 1.0 - resize_window(w, (W, WINDOW_H)).astype(np.float32) / 255.0 + xs.append(ink) + sizes.append((wd, h)) + out: list = [None] * len(wins) + order = np.argsort([x.shape[1] for x in xs]) + for k in range(0, len(order), 16): # similar widths together: less padding + idx = order[k : k + 16] + Wm = max(xs[i].shape[1] for i in idx) + batch = np.zeros((len(idx), 1, WINDOW_H, Wm), np.float32) + for j, i in enumerate(idx): + batch[j, 0, :, : xs[i].shape[1]] = xs[i] + p = sess.run(None, {"x": batch})[0] + for j, i in enumerate(idx): + pm = p[j, 0, :, : xs[i].shape[1]] + out[i] = cv2.resize(pm, sizes[i], interpolation=cv2.INTER_LINEAR) >= KEEP + return out + + +def masked_windows( + gray: np.ndarray, boxes, sess +) -> list[tuple[np.ndarray, list[float]]]: + """For each line box: (window with foreign ink whitened, the box inside it).""" + wins, offs = zip(*[_window(gray, b) for b in boxes]) if len(boxes) else ((), ()) + out = [] + for w, (ox, oy), keep, b in zip(wins, offs, mask_windows(sess, list(wins)), boxes): + m = w.copy() + # paper level: 90th percentile grey on a 1/16 subsample of the window + m[~keep] = np.uint8(np.percentile(w[::4, ::4], 90)) if w.size else 255 + out.append((m, [b[0] - ox, b[1] - oy, b[2] - ox, b[3] - oy])) + return out + + +def masked_window(gray: np.ndarray, box, sess) -> tuple[np.ndarray, list[float]]: + return masked_windows(gray, [box], sess)[0] + + +def crop_lines( + gray: np.ndarray, + boxes, + height: int, + sess=None, + window: int = 1, + pool=None, + dark: list | None = None, +) -> list[np.ndarray]: + """Recognizer inputs for the line boxes of one page: ``crop_line`` of each + line, after CenterMask when ``sess`` is given (one batched call per page). + A tall window reads the page around the line, so it is never masked. + pool: a thread pool for the crops; dark: ``line_darkness`` of the boxes, + when already known (not used with CenterMask).""" + from .rec_preproc import crop_line + + run = pool.map if pool is not None else map + if sess is None or window > 1: + d = dark if dark is not None else [None] * len(boxes) + return list( + run(lambda bd: crop_line(gray, bd[0], height, window=window, dark=bd[1]), + list(zip(boxes, d))) + ) + return [crop_line(w, b, height) for w, b in masked_windows(gray, list(boxes), sess)] + + +def line_darkness(gray: np.ndarray, boxes, pool=None) -> list[bool]: + """``is_dark_line`` of each line box (light text on a dark ground).""" + from .rec_preproc import is_dark_line + + run = pool.map if pool is not None else map + return list(run(lambda b: is_dark_line(gray, b), list(boxes))) + + +_SESSIONS: dict = {} + + +def session_for(path: str | None): + """One CPU session per process (dataloader workers call this lazily).""" + if not path: + return None + if path not in _SESSIONS: + _SESSIONS[path] = load_mask(path, threads=1) + return _SESSIONS[path] diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/math.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/math.py new file mode 100644 index 0000000000000000000000000000000000000000..68312526469b466d89d131de86ccec6a27e1e5a2 --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/math.py @@ -0,0 +1,145 @@ +"""Math lines: a classifier finds lines that hold math, and MathRec reads them as +text with inline LaTeX ("we have $x _ { i } \\leq 1$"). + +The CTC recognizer reads one symbol per column, so it cannot write a fraction or +a sum with limits. MathRec reads a taller crop of the line with an attention +decoder, one token per step, with the decoder's keys and values cached. + +Models (all optional; the pipeline skips math without them): mathcls.onnx, +math_enc.onnx, math_dec_init.onnx, math_dec_step.onnx and math_meta.json. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np + +from .line_mask import crop_lines +from .rec_preproc import pad_batch, width_batches + +MATH_MIN = 0.5 # classifier probability above which a line is read as math +# A math reading replaces a line only at this mean token probability or more: on +# print the model has not seen (old scans) it writes confident-looking LaTeX +# that is wrong, at low probability. +MATH_MIN_CONF = 0.9 +# A line whose mean token probability is this far below MATH_MIN_CONF after 8 +# tokens stops decoding: it would be rejected anyway (garbage on unseen print). +EARLY_STOP_MARGIN = 0.1 +BOS, EOS, N_SPECIAL = 1, 2, 4 +FILES = ("mathcls.onnx", "math_enc.onnx", "math_dec_init.onnx", "math_dec_step.onnx") + + +def detokenize(tokens: list[str]) -> str: + """Tokens -> text; inside math (between "$" tokens) they are joined by spaces.""" + out, math, in_math = [], [], False + for t in tokens: + if t == "$": + if in_math: + out.append("$" + " ".join(math) + "$") + math = [] + in_math = not in_math + elif in_math: + math.append(t) + else: + out.append(t) + if in_math and math: + out.append("$" + " ".join(math) + "$") + return " ".join("".join(out).split()) + + +class MathReader: + def __init__(self, onnx_dir: Path, so, providers): + import onnxruntime as ort + + s = { + f: ort.InferenceSession(str(onnx_dir / f), so, providers=providers) + for f in FILES + } + self.cls, self.enc = s["mathcls.onnx"], s["math_enc.onnx"] + self.init, self.step = s["math_dec_init.onnx"], s["math_dec_step.onnx"] + meta = json.loads((onnx_dir / "math_meta.json").read_text()) + cfg = meta["config"] + self.height = cfg["height"] + self.max_tokens = cfg["max_tokens"] + self.layers, self.heads = cfg["dec_layers"], cfg["dec_heads"] + self.head_dim = cfg["channels"] // cfg["dec_heads"] + self.itos = ["", "", "", ""] + meta["tokens"] + # Exports drop unused inputs (the init graph does not read the mask). + self.inputs = {f: {i.name for i in v.get_inputs()} for f, v in s.items()} + + @classmethod + def load(cls, onnx_dir: Path, so, providers) -> "MathReader | None": + names = (*FILES, "math_meta.json") + if all((onnx_dir / f).exists() for f in names): + return cls(onnx_dir, so, providers) + return None + + def flags(self, crops: list, batch: int = 64) -> list[bool]: + """Per line crop (the recognizer's crop): does it hold math?""" + out = [False] * len(crops) + for idx in width_batches([c.shape[1] for c in crops], batch): + arr, widths = pad_batch([crops[j] for j in idx]) + p = self.cls.run(None, {"x": arr, "widths": widths})[0] + for k, j in enumerate(idx): + out[j] = bool(p[k] > MATH_MIN) + return out + + def read( + self, gray: np.ndarray, boxes: np.ndarray, batch: int = 64 + ) -> list[tuple[str, float]]: + """(text with inline LaTeX, mean token probability) for each box.""" + crops = crop_lines(gray, boxes[:, :4], self.height, None) + out = [("", 0.0)] * len(crops) + # Few, wide batches: each batch is one decoding loop of up to max_tokens + # steps, and a step costs about the same for 8 lines as for 40. + widths = [c.shape[1] for c in crops] + for idx in width_batches(widths, batch, max_ratio=4.0, max_area=16 * 64 * 640): + arr, widths = pad_batch([crops[j] for j in idx]) + mem, mask = self.enc.run(None, {"x": arr, "widths": widths}) + feed = {"memory": mem, "mask": mask} + names = self.inputs["math_dec_init.onnx"] + ck, cv = self.init.run(None, {k: v for k, v in feed.items() if k in names}) + B = len(idx) + sk = np.zeros((self.layers, B, self.heads, 0, self.head_dim), np.float32) + sv = sk + tok = np.full(B, BOS, np.int64) + step_in = self.inputs["math_dec_step.onnx"] + seqs: list[list[int]] = [[] for _ in range(B)] + probs: list[list[float]] = [[] for _ in range(B)] + done = np.zeros(B, bool) + for t in range(self.max_tokens - 1): + feed = { + "token": tok, + "pos": np.array(t, np.int64), + "self_k": sk, + "self_v": sv, + "cross_k": ck, + "cross_v": cv, + "mask": mask, + } + logits, sk, sv = self.step.run( + None, {k: v for k, v in feed.items() if k in step_in} + ) + tok = logits.argmax(-1).astype(np.int64) + e = np.exp(logits - logits.max(-1, keepdims=True)) + top = 1.0 / e.sum(-1) # softmax probability of the argmax token + for b in range(B): + if not done[b]: + probs[b].append(float(top[b])) + if tok[b] == EOS: + done[b] = True + elif ( + len(probs[b]) >= 8 + and np.mean(probs[b]) < MATH_MIN_CONF - EARLY_STOP_MARGIN + ): + done[b] = True # rejected: keeps a low mean probability + elif tok[b] >= N_SPECIAL: + seqs[b].append(int(tok[b])) + if done.all(): + break + for k, j in enumerate(idx): + text = detokenize([self.itos[i] for i in seqs[k]]) + out[j] = (text, float(np.mean(probs[k])) if probs[k] else 0.0) + return out diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/models.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/models.py new file mode 100644 index 0000000000000000000000000000000000000000..a61250a11307199ae49017f80f45d53b72e42504 --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/models.py @@ -0,0 +1,75 @@ +"""Model files: a local folder, or a download from the Hugging Face repo.""" + +from __future__ import annotations + +import os +from pathlib import Path + +HF_REPO = "datalab-to/indra-ocr" +# Model set this package version was built and tested against (a tag in HF_REPO). +HF_REVISION = "v1.1.0" +FILES = ("det.onnx", "rec.onnx", "order.onnx", "meta.json") + + +class ModelError(OSError): + """The models are not available: a download failed (no network, no access) + or a local model folder lacks files.""" + + +MATH_FILES = ( + "mathcls.onnx", + "math_enc.onnx", + "math_dec_init.onnx", + "math_dec_step.onnx", + "math_meta.json", +) + + +def model_dir( + path: str | Path | None = None, revision: str | None = None, math: bool = False +) -> Path: + """Folder with det.onnx, rec.onnx, meta.json and usually order.onnx (and, + with ``math``, the math models, which are downloaded only then). A local + folder without order.onnx works: reading order then falls back to + XY-cut, with no layout labels. + + Order of lookup: ``path``, the INDRA_OCR_MODEL_DIR environment variable, then + a download of ``HF_REPO``. Downloads are cached by huggingface_hub.""" + path = path or os.environ.get("INDRA_OCR_MODEL_DIR") + if path: + p = Path(path).expanduser() + missing = [f for f in FILES[:2] + FILES[3:] if not (p / f).exists()] + if math: + missing += [f for f in MATH_FILES if not (p / f).exists()] + if missing: + raise ModelError(f"the model folder {p} lacks {', '.join(missing)}") + return p + from huggingface_hub import snapshot_download + + rev = revision or HF_REVISION + kw = dict( + revision=rev, + allow_patterns=[ + "*.onnx", + "meta.json", + "lm.npz", + "lm_*.npz", + *(MATH_FILES if math else ()), + ], + ignore_patterns=[] if math else list(MATH_FILES), + ) + try: + # A tag's files never change: a cached copy needs no network (and no + # progress bar on every run). + return Path(snapshot_download(HF_REPO, local_files_only=True, **kw)) + except Exception: # noqa: BLE001 (not cached yet) + pass + try: + return Path(snapshot_download(HF_REPO, **kw)) + except Exception as e: # noqa: BLE001 (network, access, disk: one clear message) + reason = (str(e).strip().splitlines() or [type(e).__name__])[0] + raise ModelError( + f"could not download the models from Hugging Face ({HF_REPO}, {rev}): " + f"{type(e).__name__}: {reason[:200]}. Check the network connection, or " + "point at a local model folder with --model-dir or INDRA_OCR_MODEL_DIR." + ) from e diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/order_post.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/order_post.py new file mode 100644 index 0000000000000000000000000000000000000000..ac059e0888509fd892ca2222a9aeae77602a30e5 --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/order_post.py @@ -0,0 +1,352 @@ +"""Torch-free parts of the reading-order stage: line features, decoding, +baselines, metrics, and the ground-truth order of a labelled page. + +order_model.py (the PyTorch OrderNet) re-exports all of these. +""" + +from __future__ import annotations + +import math +import unicodedata + +import numpy as np + +N_GEOM = 10 +N_TEXT = 6 +N_FEATS = N_GEOM + N_TEXT + + +SENTENCE_END = ".!?:;。！？：；．" + + +def display_len(t: str) -> int: + """Length with East Asian wide characters counted twice.""" + return sum(2 if unicodedata.east_asian_width(c) in "WF" else 1 for c in t) + + +def line_features( + boxes: np.ndarray, page_wh: tuple[float, float], texts: list[str] | None = None +) -> np.ndarray: + """[N, 4] boxes (x0, y0, x1, y1) -> [N, N_FEATS] float32 features.""" + W, H = page_wh + b = boxes.astype(np.float32) + n = len(b) + if n == 0: + return np.zeros((0, N_FEATS), np.float32) + w = np.maximum(b[:, 2] - b[:, 0], 1e-3) + h = np.maximum(b[:, 3] - b[:, 1], 1e-3) + med_h = float(np.median(h)) + geom = np.stack( + [ + b[:, 0] / W, + b[:, 1] / H, + b[:, 2] / W, + b[:, 3] / H, + (b[:, 0] + b[:, 2]) / (2 * W), + (b[:, 1] + b[:, 3]) / (2 * H), + w / W, + np.log(h / med_h), + np.log(w / h) / 4.0, + np.full(n, med_h / H * 20.0), + ], + 1, + ) + txt = np.zeros((n, N_TEXT), np.float32) + if texts is not None: + for i, t in enumerate(texts): + t = t.strip() + if not t: + continue + # length in Latin-letter widths: a CJK character is about two wide, so + # a full CJK line does not read as a short table cell + txt[i, 0] = math.log1p(display_len(t)) / 5.0 + txt[i, 1] = sum(c.isdigit() for c in t) / len(t) + txt[i, 2] = float( + t[0] in "•-–▪◦*·→" or (t[:3].rstrip(".)").isdigit() and len(t) > 3) + ) + txt[i, 3] = float(t.endswith("-")) + txt[i, 4] = float(t[-1] in SENTENCE_END) + txt[i, 5] = float(t[0].isupper()) + return np.concatenate([geom, txt], 1).astype(np.float32) + + +def decode_order(a: np.ndarray, sweeps: int = 4) -> list[int]: + """Order from a pairwise logit matrix [N, N] (a[i, j] > 0: i before j). + + Start from the row-sum ranking, then local search with insertion moves: + take each line out and put it back at the position that most increases + the summed log-probability of the pairwise relations. Unlike adjacent + swaps, an insertion can move a line far (e.g. a short last line of a + column that the ranking put after the next column). Vectorized: each + move costs a few numpy calls, O(N) work.""" + n = a.shape[0] + if n <= 1: + return list(range(n)) + lp = -np.logaddexp(0.0, -a) # log P(i before j) + order = np.argsort(-lp.sum(1) + lp.sum(0)) + for _ in range(sweeps): + improved = False + for x in order.copy(): + rest = order[order != x] + before = lp[rest, x] # x placed after these + after = lp[x, rest] # x placed before these + gains = np.concatenate(([0.0], np.cumsum(before))) + np.concatenate( + (np.cumsum(after[::-1])[::-1], [0.0]) + ) + k = int(np.argmax(gains)) + cur = int(np.nonzero(order == x)[0][0]) + if gains[k] > gains[cur] + 1e-6: + order = np.insert(rest, k, x) + improved = True + if not improved: + break + return [int(v) for v in order] + + +# --------------------------------------------------------------------------- +# Baselines and metrics +# --------------------------------------------------------------------------- + + +def order_top_down(boxes: np.ndarray) -> list[int]: + """Rows top to bottom (centres within half a line height share a row), left to right.""" + if len(boxes) == 0: + return [] + cy = (boxes[:, 1] + boxes[:, 3]) / 2 + h = np.median(boxes[:, 3] - boxes[:, 1]) + idx = sorted( + range(len(boxes)), + key=lambda i: (round(cy[i] / max(h * 0.5, 1e-3)), boxes[i, 0]), + ) + return idx + + +def order_xycut(boxes: np.ndarray) -> list[int]: + """Recursive XY-cut: split at the widest whitespace gap on either axis. + + A horizontal cut (between rows) puts the upper part first; a vertical cut + (between columns) puts the left part first. Without a gap, fall back to + top-down order.""" + + def best_cut(b, axis): + lo, hi = (b[:, 1], b[:, 3]) if axis == 1 else (b[:, 0], b[:, 2]) + o = np.argsort(lo) + end = hi[o[0]] + best_gap, best_k = 0.0, -1 + for k in range(1, len(o)): + gap = lo[o[k]] - end + if gap > best_gap: + best_gap, best_k = gap, k + end = max(end, hi[o[k]]) + return best_gap, best_k, o + + def rec(ids): + if len(ids) <= 1: + return list(ids) + b = boxes[ids] + gy, ky, oy = best_cut(b, 1) + gx, kx, ox = best_cut(b, 0) + if ky < 0 and kx < 0: + return [ids[i] for i in order_top_down(b)] + k, o = (ky, oy) if gy >= gx else (kx, ox) + return rec([ids[i] for i in o[:k]]) + rec([ids[i] for i in o[k:]]) + + return rec(list(range(len(boxes)))) + + +def xycut_blocks(boxes: np.ndarray, min_gap: float = 1.0) -> list[list[int]]: + """Blocks in reading order by recursive XY-cut, for pages the order model + does not know (vertical pages, read turned). A cut needs a whitespace gap of + at least ``min_gap`` median line heights; the widest gap cuts first, the upper + or left part first. Lines inside a block go by ``order_xycut``.""" + if len(boxes) == 0: + return [] + h = float(np.median(boxes[:, 3] - boxes[:, 1])) + + def best_cut(b, axis): + lo, hi = (b[:, 1], b[:, 3]) if axis == 1 else (b[:, 0], b[:, 2]) + o = np.argsort(lo) + end = hi[o[0]] + best_gap, best_k = 0.0, -1 + for k in range(1, len(o)): + gap = lo[o[k]] - end + if gap > best_gap: + best_gap, best_k = gap, k + end = max(end, hi[o[k]]) + return best_gap, best_k, o + + def rec(ids): + b = boxes[ids] + if len(ids) > 1: + gy, ky, oy = best_cut(b, 1) + gx, kx, ox = best_cut(b, 0) + gap, k, o = (gy, ky, oy) if gy >= gx else (gx, kx, ox) + if k >= 0 and gap >= min_gap * h: + return rec([ids[i] for i in o[:k]]) + rec([ids[i] for i in o[k:]]) + return [[ids[i] for i in order_xycut(b)]] + + return rec(list(range(len(boxes)))) + + +def kendall_tau(pred: list[int], n: int) -> float: + """Kendall tau between a predicted order and the identity order 0..n-1.""" + if n < 2: + return 1.0 + pos = np.empty(n, int) + pos[np.asarray(pred)] = np.arange(n) + conc = 0 + for i in range(n): + conc += int((pos[i] < pos[i + 1 :]).sum()) + total = n * (n - 1) // 2 + return (2 * conc - total) / total + + +def successor_acc(pred: list[int], n: int) -> float: + """Fraction of true consecutive pairs (i, i+1) that are also consecutive in the prediction.""" + if n < 2: + return 1.0 + nxt = {pred[k]: pred[k + 1] for k in range(len(pred) - 1)} + return sum(nxt.get(i) == i + 1 for i in range(n - 1)) / (n - 1) + + +# --------------------------------------------------------------------------- +# Ground-truth order from page labels +# --------------------------------------------------------------------------- + +GRID_LABELS = {"Table", "Form", "Table-Of-Contents"} + +# Block classes the layout heads predict for each line (chandra block labels). +BLOCK_CLASSES = [ + "Text", + "Section-Header", + "Page-Header", + "Page-Footer", + "List-Group", + "Table", + "Caption", + "Footnote", + "Table-Of-Contents", + "Form", +] +_CLASS_ID = {c: i for i, c in enumerate(BLOCK_CLASSES)} + + +def block_class(label: str) -> int: + """Class id of a chandra block label (unknown labels count as Text).""" + return _CLASS_ID.get(label, 0) + + +def label_order(lab: dict) -> list[int] | None: + """True reading order of a labelled page (indices into lab["lines"]). + + Blocks come in chandra's order. Inside a table / form / table-of-contents + block, lines go row by row. Inside any other block, lines go by XY-cut, + so a list or text block that spans two columns reads column by column + (row order interleaved the columns). Returns None when a line has no block.""" + lines = lab["lines"] + if any(ln.get("block") is None for ln in lines): + return None + blocks = lab["blocks"] + by_block: dict[int, list[int]] = {} + for i, ln in enumerate(lines): + by_block.setdefault(ln["block"], []).append(i) + out: list[int] = [] + for b in sorted(by_block): + idx = by_block[b] + boxes = np.array( + [(lines[i]["ink"] or lines[i]["bbox"]) for i in idx], np.float32 + ).reshape(-1, 4) + label = blocks[b][0] if b < len(blocks) else "Text" + local = order_top_down(boxes) if label in GRID_LABELS else order_xycut(boxes) + out += [idx[k] for k in local] + return out + + +def _cluster(logits: np.ndarray) -> list[list[int]]: + """Average-linkage clustering of pairwise "same group" logits: merge the two + groups with the highest mean logit between them while that mean is positive + (probability above one half). Unlike connected components, one wrong pair + cannot join two groups that the other pairs keep apart. + + A merge changes only the merged group's row and column of the mean matrix, + and each row keeps its best partner, so a page of n lines costs about n^2 + instead of n^3 (an 800-line page: 1.5 s -> tens of ms).""" + n = logits.shape[0] + if n == 0: + return [] + S = logits.astype(np.float64).copy() + np.fill_diagonal(S, 0.0) + size = np.ones(n) + alive = np.ones(n, bool) + members = [[i] for i in range(n)] + M = S.copy() # mean logit between groups; -inf on the diagonal and dead groups + np.fill_diagonal(M, -np.inf) + best = M.argmax(1) + best_v = M[np.arange(n), best] + for _ in range(n - 1): + a = int(np.argmax(best_v)) + b = int(best[a]) + if best_v[a] <= 0: + break + a, b = min(a, b), max(a, b) + S[a] += S[b] + S[:, a] += S[:, b] + S[a, a] = 0.0 + size[a] += size[b] + alive[b] = False + members[a] += members[b] + row = S[a] / (size[a] * size) + row[~alive] = -np.inf + row[a] = -np.inf + M[a], M[:, a] = row, row + M[b], M[:, b] = -np.inf, -np.inf + best_v[b] = -np.inf + best[a] = int(row.argmax()) + best_v[a] = row[best[a]] + # other rows: (j, a) changed and (j, b) is gone + stale = alive & ((best == a) | (best == b)) + stale[a] = False + for j in np.flatnonzero(stale): + best[j] = int(M[j].argmax()) + best_v[j] = M[j, best[j]] + up = alive & (row > best_v) + up[a] = False + best[up], best_v[up] = a, row[up] + return [sorted(members[i]) for i in range(n) if alive[i]] + + +def decode_blocks_order( + a: np.ndarray, + same: np.ndarray, + boxes: np.ndarray, + cls: np.ndarray | None = None, +) -> list[int]: + """Reading order with the layout heads: blocks first, then lines inside blocks. + + Blocks come from average-linkage clustering of the same-block logits + (independent of the order). Inside a block, lines go by XY-cut, or + row by row in a table / form / table-of-contents block (``cls`` ids of + BLOCK_CLASSES). Blocks go by the mean pairwise order logit between them.""" + n = a.shape[0] + if n <= 1: + return list(range(n)) + blocks = _cluster(same) + grid = {_CLASS_ID[c] for c in GRID_LABELS if c in _CLASS_ID} + inner = [] + for b in blocks: + bx = boxes[b, :4] + if cls is not None and int(cls[b].sum(0).argmax()) in grid: + local = order_top_down(bx) + else: + local = order_xycut(bx) + inner.append([b[k] for k in local]) + # Mean order logit between blocks p < q, antisymmetric: G a G^T with G the + # block-membership matrix scaled by 1 / block size (one matrix product + # instead of a loop over block pairs). + m = len(blocks) + G = np.zeros((m, n), np.float64) + for k, b in enumerate(blocks): + G[k, b] = 1.0 / len(b) + up = np.triu(G @ a.astype(np.float64) @ G.T, 1) + ab = (up - up.T).astype(np.float32) + return [i for k in decode_order(ab) for i in inner[k]] diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/parallel.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/parallel.py new file mode 100644 index 0000000000000000000000000000000000000000..cf8fba0b4ee2990040caa9e3896d1049dafe76a2 --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/parallel.py @@ -0,0 +1,212 @@ +"""Many pages on many cores: a pool of worker processes, one page per task. + +ONNX Runtime's own threads speed up one page only a little: rendering, the text +layer and the layout steps run on one core. Pages are independent, so one +single-threaded worker process per core uses the cores far better. pdfium +is not thread-safe, so the workers are processes, not threads. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from .pipeline import InputError, Page, is_pdf + +THREADS_PER_WORKER = 1 # ORT threads in each worker process +MIN_POOL_PAGES = 4 # fewer pages: one process, all cores on each page +WORKER_MEMORY = 1.5e9 # bytes a worker can reach (giant scans); caps the pool + +_conv = None +_doc: tuple = (None, None) + + +def usable_cores() -> int: + """Cores this process may run on (taskset and cgroup CPU sets count).""" + try: + return len(os.sched_getaffinity(0)) + except AttributeError: # macOS, Windows + return os.cpu_count() or 1 + + +def _max_workers_for_memory() -> int: + try: + ram = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") + except (AttributeError, ValueError, OSError): # Windows + return 1 << 30 + return max(1, int(ram // WORKER_MEMORY)) + + +def _init(load_kwargs: dict) -> None: + global _conv + import cv2 + + from . import load + + cv2.setNumThreads(load_kwargs["threads"]) + _conv = load(**load_kwargs) + + +def _run(unit: tuple): + """One unit: (path, page index, password) of a PDF, or (path, None, max pages) + of an image file. Returns [(index, Page)], or the InputError.""" + global _doc + path, index, extra = unit + try: + if index is None: + return list(_conv.convert(path, extra)) + if _doc[0] != path: + import pypdfium2 as pdfium + + _doc = (path, pdfium.PdfDocument(path, password=extra)) + doc = _doc[1] + return [(index, _conv._safe(lambda: _conv.pdf_page(path, doc, index)))] + except InputError as e: + return e + + +def _plan(paths: list, max_pages: int, password: str | None) -> list: + """Per input: its work units, or the InputError that stops it.""" + import pypdfium2 as pdfium + + plan = [] + for p in paths: + path = str(p) + if not Path(path).is_file(): + plan.append(InputError(f"{path}: no such file")) + elif not is_pdf(path): + plan.append([(path, None, max_pages)]) + else: + try: + doc = pdfium.PdfDocument(path, password=password) + n = len(doc) + doc.close() + except pdfium.PdfiumError as e: + locked = "password" in str(e).lower() + plan.append( + InputError( + f"{path}: password-protected PDF (pass password=, or --password)" + if locked + else f"{path}: cannot open the PDF ({e})" + ) + ) + continue + n = min(n, max_pages) if max_pages else n + plan.append([(path, i, password) for i in range(n)]) + return plan + + +def convert_many( + paths, + cores: int | None = None, + max_pages: int = 0, + password: str | None = None, + **load_kwargs, +): + """Yield (path, result) per input, in input order: result is the list of + (page index, Page) in page order, or the InputError that stopped the file. + + cores: how many cores to use (default: all this process may run on). The + pages of all inputs are spread over one worker process per core (fewer when + physical memory is under WORKER_MEMORY per worker); with fewer than + MIN_POOL_PAGES pages, one process uses all the cores on each page. + load_kwargs go to indra_ocr.load (not threads).""" + from .models import model_dir + + if max_pages < 0: + raise ValueError(f"max_pages must be 0 (all pages) or more, not {max_pages}") + # Resolve (and on a first run download) the models once, not in every worker. + load_kwargs["path"] = str( + model_dir(load_kwargs.get("path"), math=load_kwargs.get("math", False)) + ) + paths = [str(p) for p in paths] + cores = max(1, cores or usable_cores()) + plan = _plan(paths, max_pages, password) + n_pages = sum(len(u) for u in plan if isinstance(u, list)) + workers = min(cores // THREADS_PER_WORKER, n_pages, _max_workers_for_memory()) + if workers < 2 or n_pages < MIN_POOL_PAGES: + yield from _serial(paths, plan, max_pages, password, cores, load_kwargs) + return + yield from _pooled(paths, plan, workers, max_pages, password, cores, load_kwargs) + + +def _serial(paths, plan, max_pages, password, threads: int, load_kwargs: dict): + """The inputs in this process, ``threads`` ONNX Runtime threads on each page.""" + from . import load + + conv = load(threads=threads, **load_kwargs) + for path, units in zip(paths, plan): + if isinstance(units, InputError): + yield path, units + continue + try: + yield path, list(conv.convert(path, max_pages, password=password)) + except InputError as e: + yield path, e + + +def _pooled(paths, plan, workers, max_pages, password, cores, load_kwargs: dict): + """The pages on a pool of worker processes. If the workers cannot run (with + the spawn start method a worker imports the caller's main module, so on + macOS and Windows a script must call convert_many under + ``if __name__ == "__main__":``), warn and read the rest in this process.""" + import warnings + from concurrent.futures import ProcessPoolExecutor + from concurrent.futures.process import BrokenProcessPool + from multiprocessing import get_context + + threads = THREADS_PER_WORKER + flat = [u for units in plan if isinstance(units, list) for u in units] + # Thread pools of the libraries the workers import (OpenMP, BLAS) take + # their size from the environment at import: keep each to the worker's + # share while the workers start (map submits every task, starting them all). + env = { + k: str(threads) + for k in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS") + } + saved = {k: os.environ.get(k) for k in env} + os.environ.update(env) + done = 0 # inputs already yielded + ex = ProcessPoolExecutor( + workers, + mp_context=get_context("spawn"), + initializer=_init, + initargs=({**load_kwargs, "threads": threads},), + ) + try: + try: + results = ex.map(_run, flat) + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + for path, units in zip(paths, plan): + if isinstance(units, InputError): + yield path, units + done += 1 + continue + pages: list[tuple[int, Page]] = [] + error = None + for _ in units: + r = next(results) + if isinstance(r, InputError): + error = error or r + else: + pages += r + yield path, (error if error is not None else pages) + done += 1 + except BrokenProcessPool: + warnings.warn( + "indra_ocr.convert_many: the worker processes stopped (with the spawn " + 'start method, call convert_many under `if __name__ == "__main__":`); ' + "reading the remaining inputs in this process", + RuntimeWarning, + stacklevel=2, + ) + yield from _serial( + paths[done:], plan[done:], max_pages, password, cores, load_kwargs + ) + finally: + ex.shutdown(wait=False, cancel_futures=True) diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/pipeline.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..376f59eff6e9de034a0183dadc562479e0636e6b --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/pipeline.py @@ -0,0 +1,1031 @@ +"""The OCR pipeline on ONNX Runtime: line detection, CTC line recognition, +reading order and layout classes. CPU only, no PyTorch. + +The model folder holds det.onnx, rec.onnx (or rec.int8.onnx), order.onnx and +meta.json (see ``indra_ocr.models``). With script.onnx and a group recognizer +(rec_cjk.onnx, rec_rtl.onnx, rec_other.onnx), each line goes to the recognizer +of its script group (GROUPS); every recognizer also reads Latin, and a group +without its recognizer is read by rec.onnx. A right-to-left recognizer reads +in visual order: ``rtl.flip`` turns its text into logical order. With orient.onnx, +a page turned by a quarter or half turn is turned upright before detection +(``Page.rotation``; boxes are in the upright page). +""" + +from __future__ import annotations + +import json +import re +import time +import unicodedata +from dataclasses import dataclass, field +from pathlib import Path + +import cv2 +import numpy as np + +from .det_post import decode, page_to_input +from .layout_post import page_blocks, page_markdown +from .line_mask import crop_lines, line_darkness +from .math import MATH_MIN_CONF, MathReader +from .order_post import ( + BLOCK_CLASSES, + GRID_LABELS, + decode_blocks_order, + decode_order, + line_features, + order_xycut, + xycut_blocks, +) +from .rec_preproc import PAD_X, pad_batch, width_batches +from .rtl import flip, has_rtl, is_rtl +from .vocab import Vocab + +ADAPTIVE_MIN_H = ( + 9.0 # median line height (input px) that triggers a second detection pass +) +ADAPTIVE_TARGET_H = 12.0 +# Gaps between CTC character positions (centres), in line heights. The label +# rules (2.5 and 1.0 line heights) measure glyph edges; centres add about half a +# line height of glyph width. +SPLIT_GAP = 3.0 # cut a detected line at a wider gap (label rule outside tables) +CELL_GAP = 1.8 # cut a table line into cells at a wider gap (label rule in tables) +GUTTER_GAP = 1.6 # cut at a smaller gap when other lines' edges line up with it (a column gutter) +GUTTER_EDGES = 3 # how many other line edges must line up (within half a line height) +ADAPTIVE_MAX_SIDE = 2048 +ORIENT_SIDE = 768 # orient.onnx input: page thumbnail, long side, padded square +# A turn is applied only above this probability: real pages are almost all upright, +# and a wrong turn loses the page. Measured (orient_v1): 2 of 1,000 upright eval +# pages turned at 0.95 (13 at 0.5), 101 of the 164 Omni Parse Bench pages with +# rotated tests turned (128 at 0.5). +ORIENT_MIN = 0.95 +VERTICAL = 4 # orient.onnx class (5-class models): the page is set in vertical columns + + +# A recognized line under this confidence (mean CTC peak probability) gets no +# text: on empty form fields, checkboxes and rules the recognizer reads "-", "I" +# or "1" at low confidence. The line's box stays (layout and reading order). +MIN_CONF = 0.5 + +# Beam search with a language model (lm.npz for the base recognizer, optional +# lm_.npz for a group) reads a recognizer's lines under this confidence; +# the rest keep the greedy reading. alpha weighs the LM, beta rewards each token +# (tuned on held-out handwriting; meta.json "lm_params" overrides them per group). +LM_GATE = 0.9 +LM_ALPHA = 0.5 +LM_BETA = 1.5 +LM_BEAM = 8 +# A class under this probability at a step is not tried by the beam search +# (1e-2 cost 0.2 OPB handwriting points; 3e-3 changes no OPB score). +LM_CAND_FLOOR = 3e-3 +OUT_GREEDY = ["ids", "conf", "lengths"] +OUT_LM = ["blank", "topv", "topi"] + +# Recognizer groups, in script.onnx's class order unless meta.json names them +# ("script_groups", e.g. base and world). base is rec.onnx. An older +# script.onnx has one output: the chance that the line holds Chinese characters. +GROUPS = ("base", "cjk", "rtl", "other") + +# A line goes to one recognizer when the classifier is surer than this; otherwise +# base reads it as well and the more confident reading wins. Measured +# (script_w2, eval lines): every Latin line sent elsewhere was under 0.95, and +# 3.4% of the right non-base lines are read twice. +ROUTE_SURE = 0.95 + +# det.onnx picture channel: a region is a picture above this probability and +# this share of the page area. +PICTURE_MIN = 0.5 +PICTURE_MIN_AREA = 0.004 +DESKEW_MAX = 45.0 # degrees of tilt searched (quarter turns cover the rest) +DESKEW_MIN = 0.3 # smaller tilts are left alone (the search step is 0.1) + + +def _ink_thumb(gray: np.ndarray, side: int) -> np.ndarray: + """uint8 ink map (255 = ink) of the page shrunk to ``side`` pixels: 8-bit + images rotate faster than float ones.""" + h, w = gray.shape + s = side / max(h, w) + small = cv2.resize( + gray, (max(1, round(w * s)), max(1, round(h * s))), interpolation=cv2.INTER_AREA + ) + return 255 - small + + +def _row_sharpness(ink: np.ndarray, a: float) -> float: + """How sharply the horizontal ink profile changes row to row, the page + turned by ``a`` degrees: highest when the text rows are level.""" + size = (ink.shape[1], ink.shape[0]) + r = cv2.warpAffine( + ink, cv2.getRotationMatrix2D((size[0] / 2, size[1] / 2), a, 1.0), size + ) + p = r.sum(1, dtype=np.int64).astype(np.float64) + return float(((p[1:] - p[:-1]) ** 2).sum()) + + +def estimate_skew(gray: np.ndarray, side: int = 800, pool=None) -> float: + """Degrees (counter-clockwise) that straighten the text rows of a page: the + angle whose horizontal ink profile changes most sharply row to row. Searched + over +-DESKEW_MAX in 2 degree steps on a half-size thumbnail; the two best + angles and 0 are then refined in 0.5 and 0.1 degree steps on the full + thumbnail (``side`` pixels), where small text rows are still resolved. + pool: a thread pool that scores the angles in parallel (same result).""" + run = pool.map if pool is not None else map + fine = _ink_thumb(gray, side) + if fine.mean() < 0.003 * 255: # a blank page has no rows to straighten + return 0.0 + coarse = _ink_thumb(gray, side // 2) + grid = np.arange(-DESKEW_MAX, DESKEW_MAX + 1e-6, 2.0) + scores = list(run(lambda a: _row_sharpness(coarse, a), grid)) + starts = {0.0, *(float(grid[i]) for i in np.argsort(scores)[-2:])} + near = [a for a0 in starts for a in np.arange(a0 - 2.0, a0 + 2.0 + 1e-6, 0.5)] + # The 0.5 degree ranges of nearby starts overlap: each angle is scored once. + seen = dict.fromkeys(near) + seen.update(zip(seen, run(lambda a: _row_sharpness(fine, a), list(seen)))) + a1 = max(near, key=seen.__getitem__) + last = np.arange(a1 - 0.5, a1 + 0.5 + 1e-6, 0.1) + vals = list(run(lambda a: _row_sharpness(fine, a), last)) + return float(max(zip(last, vals), key=lambda av: av[1])[0]) + + +PREVIEW_SIDE = 1600 # the skew search and orient.onnx read thumbnails of at most 800 + + +def preview(gray: np.ndarray, side: int = PREVIEW_SIDE) -> np.ndarray: + """The page shrunk once to at most ``side`` pixels a side (area averaging): + the skew search and the orientation model read it instead of the full scan.""" + h, w = gray.shape + s = side / max(h, w) + if s >= 1: + return gray + return cv2.resize( + gray, (max(1, round(w * s)), max(1, round(h * s))), interpolation=cv2.INTER_AREA + ) + + +def picture_boxes(logit: np.ndarray, s: float, page_hw: tuple) -> np.ndarray: + """Picture regions [P, 4] in page pixels from det.onnx's picture channel (at + half the detector's input scale ``s``): connected regions above PICTURE_MIN, + at least PICTURE_MIN_AREA of the page.""" + mask = (logit > np.log(PICTURE_MIN / (1 - PICTURE_MIN))).astype(np.uint8) + n, _, stats, _ = cv2.connectedComponentsWithStats(mask, connectivity=8) + f = 2.0 / s + min_area = PICTURE_MIN_AREA * page_hw[0] * page_hw[1] + out = [] + for x, y, w, h, _ in stats[1:n]: + b = [x * f, y * f, min((x + w) * f, page_hw[1]), min((y + h) * f, page_hw[0])] + if (b[2] - b[0]) * (b[3] - b[1]) >= min_area: + out.append(b) + return np.array(out, np.float32).reshape(-1, 4) + + +def rotate_page(gray: np.ndarray, degrees: float) -> np.ndarray: + """The page rotated by ``degrees`` counter-clockwise about its centre (same size).""" + h, w = gray.shape + M = cv2.getRotationMatrix2D((w / 2, h / 2), degrees, 1.0) + return cv2.warpAffine( + gray, M, (w, h), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE + ) + + +def unrotate_box(box, degrees: float, wh: tuple[int, int]) -> list[float]: + """A box of the page that rotate_page(.., degrees) made, mapped back to the + page before it: the bounding box of the four mapped corners, clipped to it.""" + w, h = wh + M = cv2.getRotationMatrix2D((w / 2, h / 2), -degrees, 1.0) + x0, y0, x1, y1 = box[:4] + c = M @ np.array([[x0, x1, x0, x1], [y0, y0, y1, y1], [1, 1, 1, 1]], float) + return [ + round(float(np.clip(c[0].min(), 0, w)), 1), + round(float(np.clip(c[1].min(), 0, h)), 1), + round(float(np.clip(c[0].max(), 0, w)), 1), + round(float(np.clip(c[1].max(), 0, h)), 1), + ] + + +def orient_input(gray: np.ndarray, side: int = ORIENT_SIDE) -> np.ndarray: + """[side, side] float32 ink map (1 = ink): long side scaled to ``side``, padded.""" + h, w = gray.shape + s = side / max(h, w) + small = cv2.resize( + gray, (max(1, round(w * s)), max(1, round(h * s))), interpolation=cv2.INTER_AREA + ) + out = np.zeros((side, side), np.float32) + out[: small.shape[0], : small.shape[1]] = 1.0 - small.astype(np.float32) / 255.0 + return out + + +@dataclass +class Line: + box: list[float] # x0, y0, x1, y1 in image pixels + text: str + conf: float + + +@dataclass +class Page: + lines: list[Line] # reading order + timings: dict = field(default_factory=dict) + markdown: str | None = None # with an order model that has layout heads + source: str = ( + "ocr" # ocr | text (PDF text layer) | hybrid (text layer + OCR) | error + ) + note: str = "" # why the page took this path + # layout blocks in reading order: {"label", "bbox" (pixels), "text"}, with an + # order model that has layout heads + blocks: list = field(default_factory=list) + # degrees counter-clockwise the input was turned; boxes are upright + rotation: int = 0 + # degrees of small tilt undone after the turn (OCR reads the straightened + # page; boxes are mapped back to the turned page) + skew: float = 0.0 + # set in vertical columns (Chinese, Japanese): read turned a quarter turn, + # columns as lines; boxes are on the page as it is (rotation stays 0) + vertical: bool = False + + @property + def text(self) -> str: + return "\n".join(ln.text for ln in self.lines) + + @property + def pictures(self) -> list[list[float]]: + """Picture regions (figures, photos, diagrams) on the page, in reading + order: [x0, y0, x1, y1] pixels; the markdown names them picture-.""" + return [b["bbox"] for b in self.blocks if b["label"] == "Picture"] + + +class OrtOCR: + def __init__( + self, + onnx_dir: str | Path, + threads: int = 4, + rec_int8: bool | None = None, + adaptive: bool = True, + table_heads: bool = False, + providers: list | None = None, + furniture: bool = False, + math: bool = False, + ): + """table_heads: build tables from the order model's row / column / cell + heads (when it has them) instead of from geometry. furniture: keep page + headers and footers in the markdown (dropped by default). math: read lines + with math as LaTeX (math.py; needs the math models; off by default). providers: ONNX + Runtime execution providers (default CPU; e.g. ["CUDAExecutionProvider", + "CPUExecutionProvider"] with onnxruntime-gpu).""" + self.table_heads = table_heads + self.furniture = furniture + import onnxruntime as ort + + onnx_dir = Path(onnx_dir) + # With more than one thread, the skew search and the line crops (OpenCV + # releases the GIL) run on a small thread pool; one-thread workers + # (parallel.py) keep everything on their one core. + self.pool = None + if threads > 1: + from concurrent.futures import ThreadPoolExecutor + + self.pool = ThreadPoolExecutor(threads) + so = ort.SessionOptions() + so.intra_op_num_threads = threads + so.inter_op_num_threads = 1 + so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + # No CPU memory arena: with line batches of changing shape it kept growing + # (onnxruntime 1.31: rec.onnx +1.4 GB over 10 identical runs; a 40-page + # scan peaked at 4.9 GB), and it bought no measurable speed. + so.enable_cpu_mem_arena = False + prov = providers or ["CPUExecutionProvider"] + + def need(name: str) -> str: + p = onnx_dir / name + if not p.exists(): + raise FileNotFoundError(p) + return str(p) + + if rec_int8 is None: + rec_int8 = (onnx_dir / "rec.int8.onnx").exists() + self.det = ort.InferenceSession(need("det.onnx"), so, providers=prov) + self._pic = None # the last page's picture channel, (logit map, scale) + self.rec = ort.InferenceSession( + need("rec.int8.onnx" if rec_int8 else "rec.onnx"), so, providers=prov + ) + self.mask = ( + ort.InferenceSession(str(onnx_dir / "mask.onnx"), so, providers=prov) + if (onnx_dir / "mask.onnx").exists() + else None + ) + self.order = ( + ort.InferenceSession(str(onnx_dir / "order.onnx"), so, providers=prov) + if (onnx_dir / "order.onnx").exists() + else None + ) + self.orient = ( + ort.InferenceSession(str(onnx_dir / "orient.onnx"), so, providers=prov) + if (onnx_dir / "orient.onnx").exists() + else None + ) + meta = json.loads(Path(need("meta.json")).read_text()) + self.vocab = Vocab.from_json(meta["vocab"]) + # Scripts: a line classifier routes each line to its group's recognizer + # (rec_.onnx with meta["vocab_"]), in script.onnx's class + # order meta["script_groups"] (default GROUPS). + self.script, self.groups = None, {} + self.script_groups = tuple(meta.get("script_groups", GROUPS)) + for g in [k[len("vocab_") :] for k in meta if k.startswith("vocab_")]: + if (onnx_dir / f"rec_{g}.onnx").exists(): + self.groups[g] = ( + ort.InferenceSession( + str(onnx_dir / f"rec_{g}.onnx"), so, providers=prov + ), + Vocab.from_json(meta[f"vocab_{g}"]), + ) + if self.groups and (onnx_dir / "script.onnx").exists(): + self.script = ort.InferenceSession( + str(onnx_dir / "script.onnx"), so, providers=prov + ) + # Math: a line classifier routes lines with math to MathRec (math.py). + self.math = MathReader.load(onnx_dir, so, prov) if math else None + if math and self.math is None: + raise FileNotFoundError(f"math=True needs the math models in {onnx_dir}") + self.long_side = meta["det_long_side"] + self.height = meta["rec_height"] + # rec.onnx may read a tall window (the line and the page around it); + # the classifiers and group recognizers read the line band only. + self.window = meta.get("rec_window", 1) + # Language models: lm.npz for rec.onnx, lm_.npz for a group's + # recognizer. Each needs its recognizer's top-k outputs (newer exports). + # meta.json "lm_params" may set a group's alpha, beta and gate (tuned + # with the model); the defaults are the base recognizer's. + self.lms: dict = {} # id(session) -> (NgramLM, alpha, beta, gate) + params = meta.get("lm_params", {}) + for name, sess in [("base", self.rec)] + [ + (g, s) for g, (s, _) in self.groups.items() + ]: + f = onnx_dir / ("lm.npz" if name == "base" else f"lm_{name}.npz") + if f.exists() and set(OUT_LM) <= {o.name for o in sess.get_outputs()}: + from .lm import NgramLM + + p = params.get(name, {}) + self.lms[id(sess)] = ( + NgramLM(f), + p.get("alpha", LM_ALPHA), + p.get("beta", LM_BETA), + p.get("gate", LM_GATE), + ) + self.lm = self.lms.get(id(self.rec), (None,))[0] + self.adaptive = adaptive + + def _detect_at(self, gray: np.ndarray, long_side: int): + x, s = page_to_input(gray, long_side) + out = self.det.run(None, {"x": x[None, None]})[0][0] + self._pic = (out[3], s) if out.shape[0] > 3 else None + return decode(out), s + + def detect(self, gray: np.ndarray) -> np.ndarray: + boxes, s = self._detect_at(gray, self.long_side) + if self.adaptive and len(boxes) >= 8: + med_h = float(np.median(boxes[:, 3] - boxes[:, 1])) + if med_h < ADAPTIVE_MIN_H: + ls = ( + int( + min( + ADAPTIVE_MAX_SIDE, + self.long_side * ADAPTIVE_TARGET_H / max(med_h, 1.0), + ) + ) + // 32 + * 32 + ) + if ls > self.long_side: + boxes, s = self._detect_at(gray, ls) + if len(boxes): + boxes[:, :4] /= s + h, w = gray.shape + boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, w) + boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, h) + return boxes + + def detect_all(self, gray: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Text-line boxes and picture boxes [P, 4] (pixels; none from a + det.onnx without the picture channel).""" + self._pic = None + boxes = self.detect(gray) + pic = getattr(self, "_pic", None) + if pic is None: + return boxes, np.zeros((0, 4), np.float32) + return boxes, picture_boxes(*pic, gray.shape) + + def recognize( + self, + gray: np.ndarray, + boxes: np.ndarray, + batch: int = 64, + vertical: bool = False, + ) -> list[tuple[str, float, list]]: + """(text, confidence, tokens) per line; tokens = [(token id, page x)], + x from the CTC peak's timestep mapped back through the crop. A line + under MIN_CONF comes back empty. vertical: the page is a vertical one + turned a quarter turn (every line to the CJK recognizer).""" + # CenterMask first when the model folder has mask.onnx (same step as training) + pool = getattr(self, "pool", None) + window = getattr(self, "window", 1) + # Light-on-dark is decided once per line box, for both crops. + dark = line_darkness(gray, boxes[:, :4], pool) if window > 1 else None + crops = crop_lines( + gray, boxes[:, :4], self.height, self.mask, pool=pool, dark=dark + ) + tall = ( + crop_lines(gray, boxes[:, :4], self.height, window=window, pool=pool, dark=dark) + if window > 1 + else crops + ) + out: list = [("", 0.0, [])] * len(crops) + groups = [(self.rec, self.vocab, list(range(len(crops))), False)] + if vertical and "cjk" in self.groups: + # A vertical page's lines are CJK columns, characters on their side; + # the CJK recognizer reads those (and the Latin set across them). + sess, vocab = self.groups["cjk"] + groups = [(sess, vocab, list(range(len(crops))), True)] + elif self.script is not None and crops: + names, sure = self.line_groups(crops, batch) + # A line the classifier is not sure about is read by base as well; + # the reading with the higher confidence wins (below). + groups = [ + ( + self.rec, + self.vocab, + [j for j, g in enumerate(names) if g == "base" or not sure[j]], + False, + ) + ] + [ + (sess, vocab, [j for j, g in enumerate(names) if g == name], True) + for name, (sess, vocab) in self.groups.items() + ] + best_conf = [-1.0] * len(crops) + for sess, vocab, sel, as_chars in groups: + for part in width_batches([crops[j].shape[1] for j in sel], batch): + idx = [sel[i] for i in part] + src = tall if sess is self.rec else crops + arr, widths = pad_batch([src[j] for j in idx]) + lm = getattr(self, "lms", {}).get(id(sess)) + use_lm = lm is not None + res = sess.run( + OUT_GREEDY + (OUT_LM if use_lm else []), + {"x": arr, "widths": widths}, + ) + best, pmax, lengths = res[:3] + stride = arr.shape[-1] / max(best.shape[1], 1) + for k, j in enumerate(idx): + n = int(lengths[k]) + seq, pm = best[k, :n], pmax[k, :n] + keep = (seq != 0) & np.concatenate([[True], seq[1:] != seq[:-1]]) + x0, y0, x1, y1 = (float(v) for v in boxes[j, :4]) + h = max(1.0, y1 - y0) + ix0 = np.floor(x0 - PAD_X * h) + cw = np.ceil(x1 + PAD_X * h) - ix0 + f = cw / crops[j].shape[1] + t = np.nonzero(keep)[0] + ids = seq[t].tolist() + conf = float(pm[keep].mean()) if keep.any() else 0.0 + if use_lm and MIN_CONF <= conf < lm[3]: + from .lm import beam_search + + ids, steps = beam_search( + res[3][k, :n], + res[4][k, :n], + res[5][k, :n], + lm[0], + lm[1], + lm[2], + LM_BEAM, + cand_floor=LM_CAND_FLOOR, + ) + t = np.array(steps, np.int64) + xs = (ix0 + (t + 0.5) * stride * f).tolist() + # A group recognizer's ids mean nothing in self.vocab: its tokens + # are characters, like the text-layer tokens. + toks = ( + list(zip([vocab.chars[i - 1] for i in ids], xs)) + if as_chars + else list(zip(ids, xs)) + ) + if conf <= best_conf[j]: + continue + best_conf[j] = conf + out[j] = ( + (vocab.decode(ids), conf, toks) + if conf >= MIN_CONF + else ("", conf, []) + ) + if getattr(self, "math", None) is not None and crops: + math = [j for j, f in enumerate(self.math.flags(crops)) if f] + if math: + for j, (text, conf) in zip(math, self.math.read(gray, boxes[math])): + # Only a confident reading with math replaces the line (one + # token holds the whole line: it is never split). + if "$" in text and conf >= MATH_MIN_CONF: + xc = float(boxes[j, 0] + boxes[j, 2]) / 2 + out[j] = (text, 1.0, [(text, xc)]) + return out + + def box_groups(self, gray: np.ndarray, boxes: np.ndarray) -> tuple[list[str], list[bool]]: + """line_groups for line boxes on the page (crops as recognize makes them); + every line is "base" when the model folder has no script.onnx.""" + if self.script is None or not len(boxes): + return ["base"] * len(boxes), [False] * len(boxes) + pool = getattr(self, "pool", None) + window = getattr(self, "window", 1) + dark = line_darkness(gray, boxes[:, :4], pool) if window > 1 else None + crops = crop_lines(gray, boxes[:, :4], self.height, self.mask, pool=pool, dark=dark) + return self.line_groups(crops) + + def line_groups(self, crops: list, batch: int = 64) -> tuple[list[str], list[bool]]: + """Per line crop, the group whose recognizer reads it (script.onnx), and + whether the classifier is sure of it (probability over ROUTE_SURE); a + group without its recognizer goes to base.""" + out = ["base"] * len(crops) + sure = [True] * len(crops) + for idx in width_batches([c.shape[1] for c in crops], batch): + arr, widths = pad_batch([crops[j] for j in idx]) + p = self.script.run(None, {"x": arr, "widths": widths})[0] + for k, j in enumerate(idx): + g = ( + ("cjk" if p[k] > 0.5 else "base") + if p.ndim == 1 + else getattr(self, "script_groups", GROUPS)[int(p[k].argmax())] + ) + pk = ( + max(float(p[k]), 1.0 - float(p[k])) + if p.ndim == 1 + else float(p[k].max()) + ) + out[j] = g if g in self.groups else "base" + sure[j] = pk > ROUTE_SURE + return out, sure + + def gutter_cuts(self, boxes: np.ndarray, rec: list) -> list[list[float]]: + """Per line, the x positions of gaps that sit on a column gutter: a gap over + GUTTER_GAP line heights where at least GUTTER_EDGES other lines end at its + left side or start at its right side.""" + lefts, rights = boxes[:, 0], boxes[:, 2] + cuts: list[list[float]] = [] + for i, (b, (_, _, toks)) in enumerate(zip(boxes, rec)): + h = max(1.0, float(b[3] - b[1])) + xs = [tk[1] for tk in toks if not self._is_space(tk[0])] + mine = [] + for a, c in zip(xs[:-1], xs[1:]): + if c - a <= GUTTER_GAP * h: + continue + gl, gr = a + 0.35 * h, c - 0.35 * h # gap edges + near = (np.abs(rights - gl) < 0.5 * h) | (np.abs(lefts - gr) < 0.5 * h) + near[i] = False + if int(near.sum()) >= GUTTER_EDGES: + mine.append((a + c) / 2) + cuts.append(mine) + return cuts + + _page_rtl = False # set by assemble: the page reads right to left + + def _is_space(self, tok) -> bool: + return tok == " " or tok == self.vocab.index.get(" ") + + def _decode(self, toks: list) -> str: + """Text of tokens: (vocab id, x) from the main recognizer, (char, x) from a + group recognizer (visual order: right-to-left text is flipped by the + page direction), or (char, x, content index) from a PDF text layer.""" + if toks and isinstance(toks[0][0], str): + text = " ".join("".join(tk[0] for tk in toks).split()) + if len(toks[0]) == 2: + text = unicodedata.normalize("NFC", text) # Hangul jamo -> syllables + if has_rtl(text): + text = flip(text, base_rtl=self._page_rtl or None) + return text + return self.vocab.decode([tk[0] for tk in toks]) + + def split_line( + self, box, toks: list, gap: float, at: list | None = None + ) -> list[tuple[list, str, list]]: + """Cut a line where two neighbouring characters are more than ``gap`` + line heights apart (the detector merged two cells or columns). + Returns [(box, text, tokens)]; one item when nothing is cut.""" + h = max(1.0, float(box[3] - box[1])) + segs: list[list] = [[]] + prev = None + at = sorted(at or []) + for tk in toks: + x = tk[1] + if not self._is_space(tk[0]): + cut = prev is not None and ( + x - prev > gap * h or any(prev < c < x for c in at) + ) + if cut: + segs.append([]) + prev = x + segs[-1].append(tk) + if len(segs) == 1: + return [(list(box[:4]), self._decode(toks), toks)] + out = [] + for sg in segs: + text = self._decode(sg).strip() + xs = [tk[1] for tk in sg if not self._is_space(tk[0])] + if text and xs: + bx = [ + max(box[0], min(xs) - 0.35 * h), + box[1], + min(box[2], max(xs) + 0.35 * h), + box[3], + ] + out.append((bx, text, sg)) + return out + + def reading_order( + self, boxes: np.ndarray, texts: list[str], wh: tuple[int, int] + ) -> tuple[list[int], tuple | None, np.ndarray | None]: + """Reading order, (cls logits, same-block logits, table logits) when the + order model has layout heads, and the pairwise order logits.""" + if len(boxes) < 2 or self.order is None: + order = order_xycut(boxes[:, :4]) if len(boxes) else [] + return order, None, None + f = line_features(boxes[:, :4], wh, texts)[None] + outs = self.order.run( + None, {"feats": f, "mask": np.ones((1, len(boxes)), bool)} + ) + if len(outs) >= 3: + # Layout heads: blocks from the same-block head, lines inside a block by + # geometry, blocks by the order head (exact page order 0.68 -> 0.81 on eval). + cls, same = outs[1][0], outs[2][0] + # same row / column / cell logits, when asked for + tab = outs[3][0] if len(outs) > 3 and self.table_heads else None + order = decode_blocks_order(outs[0][0], same, boxes[:, :4], cls) + return order, (cls, same, tab), outs[0][0] + return decode_order(outs[0][0]), None, outs[0][0] + + def orientation_probs(self, gray: np.ndarray) -> np.ndarray | None: + """orient.onnx class probabilities for the page (None without the model).""" + if self.orient is None: + return None + return self.orient.run(None, {"x": orient_input(gray)[None, None]})[0][0] + + def orientation(self, gray: np.ndarray) -> int: + """Quarter turns (counter-clockwise) the page content is turned by, from + orient.onnx: 0 without the model or when no turn is more likely than + ORIENT_MIN; VERTICAL for a page set in vertical columns.""" + p = self.orientation_probs(gray) + if p is None: + return 0 + k = int(p.argmax()) + return k if p[k] > ORIENT_MIN else 0 + + def upright( + self, gray: np.ndarray, turn: bool = True, k: int | None = None + ) -> tuple[np.ndarray, int, float]: + """The page turned upright (``k`` quarter turns, or orient.onnx's choice) + and straightened (estimate_skew): (page, quarter turns undone, degrees of + skew undone).""" + a = estimate_skew(gray, pool=getattr(self, "pool", None)) + if k is None: + k = ( + self.orientation(rotate_page(gray, a) if abs(a) >= DESKEW_MIN else gray) + if turn + else 0 + ) + return self._turn(gray, k, a) + + def _turn( + self, gray: np.ndarray, k: int, a: float, small: np.ndarray | None = None + ) -> tuple[np.ndarray, int, float]: + """``small``: the page's preview (``preview``), for measuring the angle again.""" + # A half turn keeps the rows, so the angle stands; after a quarter turn the + # rows were columns, so the angle is measured again on the turned page. + q = 1 if k == VERTICAL else -k + if k == VERTICAL: + # A quarter turn counter-clockwise: columns become lines, the first + # (right) column on top, each read left to right. + gray = np.ascontiguousarray(np.rot90(gray, 1)) + elif k: + gray = np.ascontiguousarray(np.rot90(gray, -k)) + if k in (1, 3, VERTICAL): + a = estimate_skew( + np.ascontiguousarray(np.rot90(small, q)) if small is not None else gray, + pool=getattr(self, "pool", None), + ) + if abs(a) >= DESKEW_MIN: + gray = rotate_page(gray, a) + else: + a = 0.0 + return gray, k, a + + def __call__(self, gray: np.ndarray, turn: bool = True) -> Page: + """OCR of a page image. turn=False: the page is known to be upright (no + orient.onnx).""" + t0 = time.perf_counter() + small = preview(gray) + a = estimate_skew(small, pool=getattr(self, "pool", None)) + # The orientation model reads the page straightened by its row angle (a + # tilted page could read as turned). + k = ( + self.orientation(rotate_page(small, a) if abs(a) >= DESKEW_MIN else small) + if turn + else 0 + ) + orient_ms = 1000 * (time.perf_counter() - t0) + page = self._read(gray, k, a, small) + page.timings["orient_ms"] += orient_ms + return page + + def _read( + self, gray: np.ndarray, k: int, a: float, small: np.ndarray | None = None + ) -> Page: + t = {} + t0 = time.perf_counter() + gray, k, skew = self._turn(gray, k, a, small) + t["orient_ms"] = 1000 * (time.perf_counter() - t0) + t0 = time.perf_counter() + boxes, pictures = self.detect_all(gray) + t["det_ms"] = 1000 * (time.perf_counter() - t0) + t0 = time.perf_counter() + vertical = k == VERTICAL + rec = self.recognize(gray, boxes, vertical=vertical) + t["rec_ms"] = 1000 * (time.perf_counter() - t0) + page = self.assemble( + (gray.shape[1], gray.shape[0]), + boxes, + rec, + t, + vertical=vertical, + pictures=None if vertical else pictures, + ) + page.rotation, page.skew = (0 if vertical else 90 * k), round(skew, 2) + page.vertical = vertical + if skew: # boxes back on the page as turned, before the deskew + wh = (gray.shape[1], gray.shape[0]) + for ln in page.lines: + ln.box = unrotate_box(ln.box, skew, wh) + for b in page.blocks: + b["bbox"] = unrotate_box(b["bbox"], skew, wh) + if vertical: # back on the page as it is: (x, y) -> (W - y, x) + W = gray.shape[0] + + def back(b): + return [W - b[3], b[0], W - b[1], b[2]] + + for ln in page.lines: + ln.box = back(ln.box) + for b in page.blocks: + b["bbox"] = back(b["bbox"]) + return page + + def assemble( + self, + wh, + boxes: np.ndarray, + rec: list, + t: dict, + vertical: bool = False, + pictures: np.ndarray | None = None, + ) -> Page: + """Lines -> split at wide gaps -> reading order -> markdown. + + ``rec`` holds (text, confidence, tokens) per detected line. ``vertical``: + the page is a vertical page turned a quarter turn (see ``upright``).""" + t0 = time.perf_counter() + # A right-to-left page: lines read right to left (_decode), and reading + # order and markdown work on the page mirrored left to right, so the + # first column is the right one. Letter counts do not depend on order. + letters = [c for r in rec for c in r[0] if c.isalpha()] + n_rtl = sum(is_rtl(c) for c in letters) + self._page_rtl = n_rtl > len(letters) - n_rtl + width = float(wh[0]) + + def geo(b: np.ndarray) -> np.ndarray: + if not self._page_rtl: + return b + m = np.array(b, np.float32).copy() + m[..., 0], m[..., 2] = width - b[..., 2], width - b[..., 0] + return m + + # Text-layer labels split a line at wide gaps; do the same (SPLIT_GAP). + nb, nrec = [], [] + cuts = self.gutter_cuts(boxes, rec) if len(boxes) else [] + for b, (text, conf, toks), at in zip(boxes, rec, cuts): + for bx, tx, tk in self.split_line(b, toks, SPLIT_GAP, at): + nb.append(bx) + nrec.append((tx, conf, tk)) + boxes, rec = tidy_lines( + np.array(nb, np.float32).reshape(-1, 4), + nrec, + self._page_rtl, + self.vocab.index.get(" "), + ) + texts = [r[0] for r in rec] + order, layout, _ = self.reading_order(geo(boxes), texts, wh) + if vertical and layout is not None: + # The order model saw no vertical pages: turned, their columns of + # short lines look like one table read row by row across tiers. + # Blocks and order come from XY-cut instead, without grid classes. + blocks = xycut_blocks(boxes[:, :4]) + order = [i for b in blocks for i in b] + same = np.full((len(boxes), len(boxes)), -1.0, np.float32) + for b in blocks: + same[np.ix_(b, b)] = 1.0 + cls = layout[0].copy() + for c in GRID_LABELS: + cls[:, BLOCK_CLASSES.index(c)] = -1e4 + layout = (cls, same, None) + elif layout is not None and layout[2] is not None: + # The table heads learned one line per cell, but the detector can join + # cells: split table lines into cells, then run the order model again. + table_cls = BLOCK_CLASSES.index("Table") + is_table = layout[0].argmax(-1) == table_cls + if is_table.any(): + nb, nrec = [], [] + for i, (b, r) in enumerate(zip(boxes, rec)): + parts = ( + self.split_line(b, r[2], CELL_GAP) + if is_table[i] + else [(b, r[0], r[2])] + ) + for bx, tx, tk in parts: + nb.append(bx) + nrec.append((tx, r[1], tk)) + if len(nb) != len(boxes): + boxes = np.array(nb, np.float32).reshape(-1, 4) + rec = nrec + texts = [r[0] for r in rec] + order, layout, _ = self.reading_order(geo(boxes), texts, wh) + note = "order=xycut" if vertical else "order=model" + t["order_ms"] = 1000 * (time.perf_counter() - t0) + t["total_ms"] = sum(v for k, v in t.items() if k != "total_ms") + lines = [ + Line( + [round(float(v), 1) for v in boxes[i, :4]], + rec[i][0], + round(rec[i][1], 4), + ) + for i in order + ] + md, blocks = None, [] + if layout is not None: + blocks = page_blocks(geo(boxes[:, :4]), texts, order, layout[0], layout[1]) + for b in blocks: # back on the page as it is + b["bbox"] = [round(float(v), 1) for v in geo(np.array(b["bbox"]))] + # Inside tables, labels split cells at smaller gaps (CELL_GAP). + cells = { + i: [ + (list(geo(np.array(bx, np.float32))), tx) + for bx, tx, _ in self.split_line(boxes[i], rec[i][2], CELL_GAP) + ] + for i in range(len(boxes)) + } + has_pics = pictures is not None and len(pictures) + md = page_markdown( + geo(boxes[:, :4]), + texts, + order, + layout[0], + layout[1], + keep_furniture=self.furniture, + cells=cells, + tab=layout[2], + pictures=geo(pictures) if has_pics else None, + ) + if ( + has_pics + ): # picture blocks, in reading order by the same rule as the markdown + for p in pictures: + cy = (p[1] + p[3]) / 2 + at = next( + ( + j + for j, b in enumerate(blocks) + if (b["bbox"][1] + b["bbox"][3]) / 2 > cy + and min(p[2], b["bbox"][2]) > max(p[0], b["bbox"][0]) + ), + len(blocks), + ) + blocks.insert( + at, + { + "label": "Picture", + "bbox": [round(float(v), 1) for v in p[:4]], + "text": "", + }, + ) + elif len(boxes) < 2: + md = "\n".join(texts) + return Page(lines, t, md, note=note, blocks=blocks) + + +LEADER_CHARS = set(".·…‥_-–—=") # dot leaders, fill-in rules +DASHES = set("-–—") # a lone short dash can be a table cell; other lone marks are not +BULLETS = set("•◦▪▫●○■□‣⁃∙*") # list markers (not dashes: a "-" can be a table cell) +# A numbered list marker: "1.", "12)", "a.", "B)", "(3)", "(b)", "(iv)". +ENUMERATOR = re.compile( + r"(\d{1,3}|[A-Za-z])[.)]|\((\d{1,3}|[A-Za-z]|[ivxlcIVXLC]{1,5})\)" +) +# A list marker and its item's text are at most this many line heights apart: +# the label rule (training labels cut a text-layer line only at wider gaps). +MARKER_GAP = 2.5 + + +def tidy_lines( + boxes: np.ndarray, rec: list, rtl: bool = False, space=None +) -> tuple[np.ndarray, list]: + """Two kinds of lines carry no text of their own: + + - a line of leader characters only (form fill-in dots and rules, specks) + is dropped: 3 or more of them, a box 3 or more times wider than tall + (CTC reads a dotted rule as one "."), or any dot or underscore alone. A + short lone dash stays: it can be an empty table cell; + - a lone list marker (a bullet, "1.", "(a)") joins the line that starts + within MARKER_GAP line heights after it on the same row (on its left on a + right-to-left page): the item keeps its marker. A column gutter cut + (``gutter_cuts``) splits the markers off a hanging-indent list, and a + column of markers beside a column of items reads as a table. On a + left-to-right page the marker's tokens join too (``space``: the space + token), so a table line still splits into cells at its gaps. + + Other lines without letters or digits ("-", "/ /", "( )") stay: they can be + table cells or form content.""" + keep = [True] * len(rec) + rec = list(rec) + h = boxes[:, 3] - boxes[:, 1] + w = boxes[:, 2] - boxes[:, 0] + for i, (text, conf, toks) in enumerate(rec): + core = text.replace(" ", "") + if core and set(core) <= LEADER_CHARS: + wide = w[i] >= 3 * h[i] + if len(core) >= 3 or wide or not set(core) & DASHES: + keep[i] = False + for i, (text, conf, toks) in enumerate(rec): + t = text.strip() + if not keep[i] or not (t in BULLETS or ENUMERATOR.fullmatch(t)): + continue + best, best_gap = None, None + for j in range(len(rec)): + if j == i or not keep[j] or not rec[j][0].strip(): + continue + ov = min(boxes[i, 3], boxes[j, 3]) - max(boxes[i, 1], boxes[j, 1]) + if ov < 0.5 * min(h[i], h[j]): + continue + gap = boxes[i, 0] - boxes[j, 2] if rtl else boxes[j, 0] - boxes[i, 2] + if -0.3 * h[i] <= gap <= MARKER_GAP * max(h[i], h[j]) and ( + best_gap is None or gap < best_gap + ): + best, best_gap = j, gap + if best is None: + continue + tj, cj, kj = rec[best] + chars = bool(kj) and isinstance(kj[0][0], str) + sp = " " if chars else space + # same kind of tokens (text-layer chars or recognizer ids) only + if not rtl and toks and kj and sp is not None: + if isinstance(toks[0][0], str) == chars: + x = float(boxes[i, 2] + boxes[best, 0]) / 2 + kj = [*toks, (sp, x, *toks[-1][2:])] + list(kj) + rec[best] = (f"{t} {tj.strip()}", cj, kj) + boxes[best] = [ + min(boxes[i, 0], boxes[best, 0]), + min(boxes[i, 1], boxes[best, 1]), + max(boxes[i, 2], boxes[best, 2]), + max(boxes[i, 3], boxes[best, 3]), + ] + keep[i] = False + idx = [i for i, k in enumerate(keep) if k] + return boxes[idx].reshape(-1, 4), [rec[i] for i in idx] + + +class InputError(Exception): + """A file that cannot be read: not a PDF or an image, corrupt, or locked.""" + + +def is_pdf(path: Path) -> bool: + """The file's own bytes say PDF (the header may follow a little junk).""" + with open(path, "rb") as f: + return b"%PDF-" in f.read(1024) + + +def read_images(path: Path) -> list[np.ndarray]: + """Grayscale pages of an image file; every frame of a multi-page TIFF.""" + ok, frames = cv2.imreadmulti(str(path), flags=cv2.IMREAD_GRAYSCALE) + if not ok or not frames: + img = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE) + frames = [img] if img is not None else [] + if not frames: + raise InputError(f"{path}: not a PDF or a readable image") + return [np.ascontiguousarray(f) for f in frames] + + +def load_pages(path: Path, dpi: float = 200.0): + """Yield grayscale pages of a PDF (rendered at ``dpi``) or an image file + (every frame of a multi-page TIFF); the file's bytes decide which.""" + path = Path(path) + if is_pdf(path): + import pypdfium2 as pdfium + + doc = pdfium.PdfDocument(str(path)) + try: + for i in range(len(doc)): + arr = doc[i].render(scale=dpi / 72.0, grayscale=True).to_numpy() + yield i, np.ascontiguousarray(arr[..., 0] if arr.ndim == 3 else arr) + finally: + doc.close() + else: + yield from enumerate(read_images(path)) diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/rec_preproc.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/rec_preproc.py new file mode 100644 index 0000000000000000000000000000000000000000..7de820993f034509a765dc358386fa1c8269e7bc --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/rec_preproc.py @@ -0,0 +1,145 @@ +"""Line crop rule shared by recognition training and inference. + +A recognizer input is made from a line box (the ink box from the detector, or +the ink box from a synthetic or real label) in three steps: + 1. expand the box by PAD_Y x its height on top and bottom and PAD_X x its + height on left and right, + 2. crop that region from the page (white outside the page), + 3. resize the crop to LINE_HEIGHT pixels high, keeping the aspect ratio. + +The result is an ink map (0 = paper, 1 = ink) of shape [LINE_HEIGHT, W]. +Training and inference must both call ``crop_line``, so the text scale and the +margins match. +""" + +from __future__ import annotations + +import math + +import numpy as np + +LINE_HEIGHT = 32 +PAD_Y = 0.2 +PAD_X = 0.3 +WIDTH_MULTIPLE = 8 # widths are padded to a multiple of this (timestep stride) + + +def _percentiles(b: np.ndarray, qs=(0.02, 0.5, 0.98)) -> list[float]: + """np.percentile(b, 100 * qs) (linear) of a uint8 array, from its histogram: + no sort of the pixels.""" + n = b.size + cum = np.cumsum(np.bincount(b.ravel(), minlength=256)) + out = [] + for q in qs: + x = q * (n - 1) + i = math.floor(x) + g = x - i + lo = float(np.searchsorted(cum, i, side="right")) + hi = float(np.searchsorted(cum, min(i + 1, n - 1), side="right")) + d = hi - lo + # numpy's lerp: from the nearer end + out.append(hi - d * (1 - g) if g >= 0.5 else lo + d * g) + return out + + +def is_dark_line(gray: np.ndarray, box) -> bool: + """Light text on a dark ground (a banner, a dark table head, a slide): the + ground (the median of the line box) lies nearer the dark end than the text + does, so the bright tail is the longer one. A shadowed line keeps dark text + on a dimmer ground. The recognizers read dark text on light paper, so a + light-on-dark line is read inverted.""" + H, W = gray.shape + x0, y0, x1, y1 = (float(v) for v in box[:4]) + b = gray[ + max(0, int(y0)) : min(H, int(y1) + 1), max(0, int(x0)) : min(W, int(x1) + 1) + ] + if not b.size: + return False + lo, med, hi = ( + _percentiles(b) if b.dtype == np.uint8 else np.percentile(b, (2, 50, 98)) + ) + return bool(hi - med > 2 * (med - lo) + 8) + + +def crop_line( + gray: np.ndarray, + box, + height: int = LINE_HEIGHT, + pad_y: float = PAD_Y, + pad_x: float = PAD_X, + max_width: int = 4096, + window: int = 1, + dark: bool | None = None, +) -> np.ndarray: + """Crop one line from a uint8 grayscale page and return a float32 ink map + [window * height, W]. A tall window (``window`` > 1) keeps the text scale + and adds the page above and below the line; its middle band is the + window-1 crop. dark: ``is_dark_line`` of the box, when already known.""" + import cv2 + + pad_y = (window * (1 + 2 * pad_y) - 1) / 2 + height = height * window + + x0, y0, x1, y1 = (float(v) for v in box[:4]) + h = max(1.0, y1 - y0) + X0, X1 = x0 - pad_x * h, x1 + pad_x * h + Y0, Y1 = y0 - pad_y * h, y1 + pad_y * h + H, W = gray.shape + ix0, iy0 = int(math.floor(X0)), int(math.floor(Y0)) + ix1, iy1 = int(math.ceil(X1)), int(math.ceil(Y1)) + cw, ch = max(1, ix1 - ix0), max(1, iy1 - iy0) + sx0, sy0, sx1, sy1 = max(0, ix0), max(0, iy0), min(W, ix1), min(H, iy1) + if dark is None: + dark = is_dark_line(gray, box) + crop = np.full((ch, cw), 0 if dark else 255, np.uint8) + if sx1 > sx0 and sy1 > sy0: + crop[sy0 - iy0 : sy1 - iy0, sx0 - ix0 : sx1 - ix0] = gray[sy0:sy1, sx0:sx1] + out_w = max(WIDTH_MULTIPLE, min(max_width, round(cw * height / ch))) + interp = cv2.INTER_AREA if ch > height else cv2.INTER_LINEAR + resized = cv2.resize(crop, (out_w, height), interpolation=interp) + ink = resized.astype(np.float32) / 255.0 + return ink if dark else 1.0 - ink + + +def pad_batch(lines: list[np.ndarray], multiple: int = WIDTH_MULTIPLE): + """Right-pad ink maps to one width. Returns (array [B, 1, H, W], widths [B]).""" + H = lines[0].shape[0] + Wm = max(x.shape[1] for x in lines) + Wm = int(math.ceil(Wm / multiple) * multiple) + out = np.zeros((len(lines), 1, H, Wm), np.float32) + widths = np.zeros(len(lines), np.int64) + for i, x in enumerate(lines): + out[i, 0, :, : x.shape[1]] = x + widths[i] = x.shape[1] + return out, widths + + +def width_batches( + widths: list[int], + max_batch: int = 64, + max_ratio: float = 1.3, + min_width: int = 64, + max_area: int = 64 * 640, +) -> list[list[int]]: + """Group line indices into batches of similar width, so little compute goes to padding. + + Lines are taken narrowest first. A batch closes at ``max_batch`` lines, when the + next line is wider than ``max_ratio`` times the batch's first line (widths below + ``min_width`` count as ``min_width``, so short lines do not fragment into tiny batches), + or when lines x width would pass ``max_area`` pixel columns: ONNX Runtime keeps its + largest allocation, and 64 lines of 4,096 px held 5 GB on large scanned pages. + """ + order = sorted(range(len(widths)), key=lambda i: widths[i]) + batches: list[list[int]] = [] + for i in order: + if ( + batches + and len(batches[-1]) < max_batch + and max(widths[i], min_width) + <= max_ratio * max(widths[batches[-1][0]], min_width) + and (len(batches[-1]) + 1) * max(widths[i], min_width) <= max_area + ): + batches[-1].append(i) + else: + batches.append([i]) + return batches diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/rtl.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/rtl.py new file mode 100644 index 0000000000000000000000000000000000000000..1c3ec99ac3ed4bcec2d13231e99849ce11bcdaa9 --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/rtl.py @@ -0,0 +1,162 @@ +"""Right-to-left text for a recognizer that reads every line left to right. + +A CTC recognizer emits characters in the order it sees them, left to right, so +the label of a line with Arabic or Hebrew is its characters in visual order. +``flip`` turns logical order (how text is stored and typed) into visual order, +and the same call turns visual order back into logical order: + +- RTL base line (more RTL letters than LTR letters): the line reads right to + left, so the order of its units reverses. A unit is a run of LTR text + (Latin words, digits, and the spaces and punctuation between them), which + keeps its own order, or one other character cluster. +- LTR base line: the line keeps its order, and each run of RTL text reverses. + +A cluster is a base character with its combining marks (Arabic harakat, Hebrew +niqqud), so marks stay after their base. A mirrored character (brackets) at +right-to-left level changes to its mirror image, as a renderer draws it. + +This is a small subset of the Unicode bidi algorithm (UAX #9): it has no +embeddings or isolates, and it resolves neutrals by runs. It is its own +inverse, so the labels and the decoding always agree. +""" + +from __future__ import annotations + +import unicodedata + +_MIRROR = dict(zip("()[]{}<>«»‹›", ")(][}{><»«›‹")) +# Presentation forms (glyph variants: initial, medial, final, ligatures) map to +# base letters. Tatweel stretches a word and carries no letter. +_PRESENTATION = ((0xFB1D, 0xFDFF), (0xFE70, 0xFEFF)) +_TATWEEL = "ـ" + + +def is_rtl(c: str) -> bool: + return unicodedata.bidirectional(c) in ("R", "AL") + + +def _is_ltr(c: str) -> bool: + # Digits (European and Arabic-Indic) read left to right inside RTL text. + return unicodedata.bidirectional(c) in ("L", "EN", "AN") + + +def has_rtl(text: str) -> bool: + return any(is_rtl(c) for c in text) + + +def rtl_base(text: str) -> bool: + """Whether the line reads right to left: more RTL letters than LTR letters. + The counts do not depend on the order, so logical and visual text agree.""" + r = sum(is_rtl(c) for c in text) + lt = sum(unicodedata.bidirectional(c) == "L" for c in text) + return r > lt + + +def normalize(text: str) -> str: + """Presentation forms to base letters (NFKC on those blocks only), no + tatweel, no combining mark without a base character.""" + out = [] + for c in text: + o = ord(c) + if any(a <= o <= b for a, b in _PRESENTATION): + out.append(unicodedata.normalize("NFKC", c)) + elif c != _TATWEEL and (out or unicodedata.category(c) not in ("Mn", "Me")): + out.append(c) + return "".join(out) + + +def clusters(text: str) -> list[str]: + out: list[str] = [] + for c in text: + if out and unicodedata.category(c) in ("Mn", "Me"): + out[-1] += c + else: + out.append(c) + return out + + +def _kind(cl: str) -> str: + c = cl[0] + return "R" if is_rtl(c) else "L" if _is_ltr(c) else "N" + + +def _runs(kinds: list[str], strong: str, stop: set[str]) -> list[tuple[int, int]]: + """[start, end) spans from a ``strong`` cluster to the last ``strong`` cluster + before a cluster of a kind in ``stop``.""" + spans, start, last = [], None, None + for i, k in enumerate(kinds + ["#"]): + if k == strong: + if start is None: + start = i + last = i + elif k in stop or k == "#": + if start is not None: + spans.append((start, last + 1)) + start = last = None + return spans + + +def _mirror(cl: str) -> str: + return _MIRROR.get(cl[0], cl[0]) + cl[1:] + + +def flip(text: str, mirror: bool = True, base_rtl: bool | None = None) -> str: + """Logical order to visual order, and visual order back to logical order. + base_rtl: the paragraph direction, when known (a renderer lays out a line + by its paragraph's direction); None decides by ``rtl_base``.""" + if not has_rtl(text): + return text + cls = clusters(text) + kinds = [_kind(c) for c in cls] + if rtl_base(text) if base_rtl is None else base_rtl: + # Units: LTR runs kept whole, every other cluster alone; reverse the units. + units: list[str] = [] + spans = _runs(kinds, "L", {"R"}) + i = 0 + for a, b in spans + [(len(cls), len(cls))]: + for j in range(i, a): + units.append(_mirror(cls[j]) if mirror else cls[j]) + if a < b: + units.append("".join(cls[a:b])) + i = b + return "".join(reversed(units)) + # LTR base: reverse each RTL run in place. + out = list(cls) + for a, b in _runs(kinds, "R", {"L"}): + seg = cls[a:b][::-1] + out[a:b] = [_mirror(c) for c in seg] if mirror else seg + return "".join(out) + + +def visual_label(logical: str, base_rtl: bool | None = None) -> str: + """The recognizer's label for a line stored in logical order.""" + return flip(normalize(logical), base_rtl=base_rtl) + + +def label_from_visual(raw_visual: str, shapes: bool = False) -> str: + """The label for characters already in visual order, as stored (a PDF text + layer read by glyph position): normalize in logical order, then mirror. + shapes: the text layer stores each bracket as the shape drawn (see + ``brackets_as_shapes``), so it is already mirrored.""" + return visual_label(flip(raw_visual, mirror=shapes)) + + +def brackets_as_shapes(raw_visual_lines: list[str]) -> bool: + """Whether a page's text layer stores RTL brackets as drawn shapes. + + Producers differ: most store the logical character, so the leftmost bracket + of an RTL pair reads ")"; some store the drawn shape, so it reads "(". A + page uses one convention, so its RTL lines without Latin letters vote by + their first bracket.""" + shapes = logical = 0 + for t in raw_visual_lines: + if not rtl_base(t) or any(unicodedata.bidirectional(c) == "L" for c in t): + continue + for c in t: + if c in "([{": + shapes += 1 + break + if c in ")]}": + logical += 1 + break + return shapes > logical diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/textlayer.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/textlayer.py new file mode 100644 index 0000000000000000000000000000000000000000..1a771cbe6c9b9ec4213935380cb3c1b072a83445 --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/textlayer.py @@ -0,0 +1,363 @@ +"""Text-layer runs from a PDF page (pdfium), and checks that decide whether to +trust them. + +A digital PDF already holds its text. When the text layer is sound, reading it +is faster than OCR and has no recognition errors. The checks follow the rules +that select the recognizer's own training labels: + +- invisible text (render mode 3) marks an OCR layer over a scan; +- replacement, private-use or control characters mark a broken font map; +- a text run with no ink under it marks hidden or misplaced text; +- right-to-left letters stored in visual order (left to right on the page, as + some PDF producers write Arabic script) cannot be put back into reading order + safely, so OCR reads the page instead. + +A run is pdfium's own text segment between the line breaks it generates. +Coordinates are PDF points in the frame that pypdfium2 renders (crop box and +page rotation applied), with a top-left origin. +""" + +from __future__ import annotations + +import math +import unicodedata +from dataclasses import dataclass, field + +import numpy as np + +MAX_INVISIBLE_FRAC = 0.05 +MAX_BAD_CHAR_FRAC = 0.005 +MAX_NO_INK_LINE_FRAC = 0.03 +# Share of neighbouring right-to-left letters in a run that step left to right. +# Logical order steps right to left (sound layers: 0-1%); a layer in visual +# order steps left to right (Chrome-printed Arabic script: 52-94%). +MAX_VISUAL_RTL_FRAC = 0.25 +MIN_RTL_PAIRS = 20 +VISUAL_RTL = "right-to-left text in visual order" # TextLayer.reason +DARK = 160 # grey level below which a pixel counts as ink + + +def bad_char_count(text: str) -> int: + n = 0 + for c in text: + cat = unicodedata.category(c) + if c == "�" or cat in ("Co", "Cs", "Cn") or (cat == "Cc" and c != "\t"): + n += 1 + return n + + +BROKEN_FONT_MAP = "broken font map" # TextLayer.reason: the layer's codes are not text + + +@dataclass +class TextLine: + """One text-layer run: its box, text, and characters with boxes.""" + + bbox: tuple[float, float, float, float] + text: str + chars: list = field(default_factory=list) # (char, bbox) in PDF order + quarter: int = 0 # quarter turns from upright in the (turned) frame + seq: int | None = None # position in the page's content order (all runs) + + +@dataclass +class TextLayer: + lines: list[TextLine] # upright runs in PDF content order + size: tuple[float, float] # page width, height in points (as rendered) + trusted: bool + reason: str = "" # why the layer is not trusted + stats: dict = field(default_factory=dict) + rotated: list[TextLine] = field(default_factory=list) # runs at other quarter turns + + +def _device_affine(page) -> np.ndarray: + """[2, 3] map from PDF page space to the rendered frame (points, top-left + origin), from pdfium's own FPDF_PageToDevice.""" + import ctypes + + import pypdfium2.raw as R + + W, H = page.get_size() + S = 1000 # device units per point: keeps the integer output precise + + def dev(x, y): + dx, dy = ctypes.c_int(), ctypes.c_int() + R.FPDF_PageToDevice(page.raw, 0, 0, round(W * S), round(H * S), 0, x, y, dx, dy) + return np.array([dx.value, dy.value], float) / S + + o, ex, ey = dev(0.0, 0.0), dev(1.0, 0.0), dev(0.0, 1.0) + return np.stack([ex - o, ey - o, o], 1) + + +def _quarter(d) -> int | None: + """Quarter turns (counter-clockwise, as ``np.rot90``) that a baseline running + in direction d (rendered frame, y down) is turned from left to right; None + when it is not a quarter turn.""" + q = math.atan2(d[1], d[0]) / (math.pi / 2) # pdfium's char angle runs clockwise + k = round(q) + return k % 4 if abs(q - k) < 0.01 else None + + +def _turn_frame(px, py, W: float, H: float, turn: int): + """Rendered-frame points in the frame of the page turned upright again + (``np.rot90(gray, -turn)``).""" + if turn == 1: + return H - py, px + if turn == 2: + return W - px, H - py + if turn == 3: + return py, W - px + return px, py + + +def vertical_layout(stats: dict) -> bool: + """Whether a page's upright characters run top to bottom (vertical writing): + most steps between neighbouring characters go down, not right.""" + down, right = stats.get("steps", (0, 0)) + return down >= 30 and down > 2 * right + + +def page_turn(stats: dict) -> int: + """The quarter turn that holds most of the page's characters (a table set + sideways on a portrait page), or 0.""" + q = stats.get("quarters", [0, 0, 0, 0]) + k = int(np.argmax(q[1:])) + 1 + return k if q[k] > 0.5 * max(stats.get("chars", 0), 1) else 0 + + +def code_point(u: int, u2: int = 0) -> tuple[str, bool]: + """The character of pdfium's UTF-16 code unit ``u`` (``u2``: the next unit), + and whether it used ``u2``. A character above U+FFFF (math italic letters, + emoji, rare CJK) comes as two surrogates; a lone surrogate becomes U+FFFD.""" + if 0xD800 <= u <= 0xDBFF and 0xDC00 <= u2 <= 0xDFFF: + return chr(0x10000 + ((u - 0xD800) << 10) + (u2 - 0xDC00)), True + if 0xD800 <= u <= 0xDFFF: + return "\ufffd", False + return chr(u), False + + +def extract_runs( + page, turn: int = 0, frame: int | None = None +) -> tuple[list[TextLine], dict]: + """Text runs of a pypdfium2 page that read left to right once the page is + turned upright by ``turn`` quarter turns (see ``page_turn``), in content + order; boxes are in the frame turned by ``frame`` (default ``turn``): a + vertical page has upright characters (turn 0) read in a turned frame (3).""" + frame = turn if frame is None else frame + import ctypes + + import pypdfium2 + import pypdfium2.raw as R + + tp = page.get_textpage() + try: + n = tp.count_chars() + # A character is upright when its baseline, mapped through the + # page-to-render transform, runs left to right after ``turn`` (any + # /Rotate, any mirroring). + # Note: pdfium's char angle turns clockwise with y up, and _quarter counts + # the other way with y down; F L F (F flips y) applies both. They cancel + # when L is diagonal (/Rotate 0, 180) but not when it swaps x and y + # (/Rotate 90, 270), where L alone turns the page 180 degrees the wrong way. + F = np.diag([1.0, -1.0]) + L = F @ _device_affine(page)[:, :2] @ F + # The loop below runs once per character: raw pdfium calls with reused + # buffers, and the upright and font-map tests cached per angle and per + # character (a page has few distinct ones). + get_unicode, get_angle = R.FPDFText_GetUnicode, R.FPDFText_GetCharAngle + get_box, raw_tp = R.FPDFText_GetCharBox, tp.raw + bl, br, bb, bt = (ctypes.c_double() for _ in range(4)) + pl, pr, pb, pt = (ctypes.byref(v) for v in (bl, br, bb, bt)) + upright: dict = {} + quarters = [0, 0, 0, 0] + bad: dict = {} + chars, raw, breaks = [], [], [] # breaks: run ends after these positions + qs: list = [] # each character's quarter turn relative to ``turn`` + cur_q = 0 + n_chars = n_rot = n_bad = 0 + low = False # this index is the second half of a surrogate pair + for i in range(n): + if low: + low = False + continue + u = get_unicode(raw_tp, i) + u2 = ( + get_unicode(raw_tp, i + 1) if 0xD800 <= u <= 0xDBFF and i + 1 < n else 0 + ) + c, low = code_point(u, u2) + if c in "\r\n": + breaks.append(len(chars)) + continue + if c == "\x02": # pdfium's mark for a hyphen at a line break + c = "-" + if not c.isspace(): + n_chars += 1 + k = bad.get(c) + if k is None: + k = bad[c] = bad_char_count(c) + n_bad += k + a = get_angle(raw_tp, i) + q = upright.get(a, -1) + if q == -1: + q = upright[a] = _quarter(L @ np.array([math.cos(a), math.sin(a)])) + if q is not None: + quarters[q] += 1 + if q != turn: + n_rot += 1 # not upright on the rendered page + if q is None: # not a quarter turn: not read + breaks.append(len(chars)) + continue + cur_q = (q - turn) % 4 + chars.append(c) + qs.append(cur_q) + if not get_box(raw_tp, i, pl, pr, pb, pt): # yes, l r b t + raise pypdfium2.PdfiumError("Failed to get charbox.") + raw.append((bl.value, bb.value, br.value, bt.value)) # l, b, r, t + finally: + tp.close() + stats = {"chars": n_chars, "rotated": n_rot, "bad": n_bad, "quarters": quarters} + if not chars: + return [], stats + # Page space -> rendered frame, all characters at once. + A = _device_affine(page) + b = np.asarray(raw, float) + xs = b[:, [0, 2, 0, 2]] + ys = b[:, [1, 1, 3, 3]] + px = A[0, 0] * xs + A[0, 1] * ys + A[0, 2] + py = A[1, 0] * xs + A[1, 1] * ys + A[1, 2] + boxes = np.stack([px.min(1), py.min(1), px.max(1), py.max(1)], 1) + # Characters outside the rendered frame (beyond a crop box) are not on the + # page: they are dropped, not read. + W, H = page.get_size() + cx, cy = (boxes[:, 0] + boxes[:, 2]) / 2, (boxes[:, 1] + boxes[:, 3]) / 2 + visible = (cx >= 0) & (cx <= W) & (cy >= 0) & (cy <= H) + stats["off_page"] = int((~visible).sum()) + # Steps between neighbouring upright characters of a run: down (vertical + # writing) or right (horizontal), in the rendered frame. + ok = np.ones(max(len(chars) - 1, 0), bool) + ok[[b - 1 for b in breaks if 0 < b < len(chars)]] = False + q = np.asarray(qs) != 0 + sp = np.array([c.isspace() for c in chars], bool) + ok &= ~(q[:-1] | q[1:] | sp[:-1] | sp[1:]) + h = np.maximum(boxes[:-1, 3] - boxes[:-1, 1], 1e-3) + dx, dy = np.diff(cx), np.diff(cy) + ok &= (np.abs(dx) <= 3 * h) & ( + np.abs(dy) <= 3 * h + ) # a jump to another run is not a step + down = int((ok & (dy > 0.3 * h) & (np.abs(dx) < 0.3 * h)).sum()) + # Horizontal steps go right (left-to-right scripts) or left (right to left). + right = int((ok & (np.abs(dx) > 0.3 * h) & (np.abs(dy) < 0.3 * h)).sum()) + stats["steps"] = [down, right] + if frame: + tx, ty = _turn_frame(px, py, W, H, frame) + boxes = np.stack([tx.min(1), ty.min(1), tx.max(1), ty.max(1)], 1) + runs: list[TextLine] = [] + # A run also ends where the quarter turn changes (a sideways label in an + # upright line of characters). + turns = [i for i in range(1, len(chars)) if qs[i] != qs[i - 1]] + cuts = sorted(set(breaks + turns + [len(chars)])) + start = 0 + for end in cuts: + seg = [(chars[i], tuple(boxes[i])) for i in range(start, end) if visible[i]] + q = qs[start] if start < len(qs) else 0 + start = end + ink = [bx for c, bx in seg if not c.isspace()] + if not ink: + continue + box = ( + min(v[0] for v in ink), + min(v[1] for v in ink), + max(v[2] for v in ink), + max(v[3] for v in ink), + ) + text = " ".join("".join(c for c, _ in seg).split()) + if frame != turn and not q and len(ink) > 1: + # A vertical page read turned: a horizontal run (a header, a + # caption) runs top to bottom here, so it is a sideways line. + c0 = [((v[0] + v[2]) / 2, (v[1] + v[3]) / 2) for v in ink] + dx = abs(c0[-1][0] - c0[0][0]) + dy = abs(c0[-1][1] - c0[0][1]) + if dy > 2 * dx: + q = 1 + runs.append(TextLine(box, text, seg, q, len(runs))) + return runs, stats + + +def visual_rtl_frac(runs: list[TextLine]) -> tuple[int, float]: + """(pairs, share): neighbouring right-to-left letters inside a run, and the + share of them that step left to right on the page.""" + from .rtl import is_rtl + + pairs = right = 0 + for r in runs: + for (a, ba), (b, bb) in zip(r.chars, r.chars[1:]): + if is_rtl(a) and is_rtl(b): + pairs += 1 + right += bb[0] + bb[2] > ba[0] + ba[2] + return pairs, right / max(pairs, 1) + + +def invisible_text_frac(page) -> float: + """Share of text objects drawn invisibly (an OCR layer) on a pypdfium2 page.""" + import pypdfium2.raw as R + + n = inv = 0 + for obj in page.get_objects(max_depth=4): + if obj.type == R.FPDF_PAGEOBJ_TEXT: + n += 1 + inv += ( + R.FPDFTextObj_GetTextRenderMode(obj.raw) + == R.FPDF_TEXTRENDERMODE_INVISIBLE + ) + return inv / n if n else 0.0 + + +def no_ink_frac(gray: np.ndarray, runs: list[TextLine], scale: float) -> float: + """Share of runs whose box holds no dark pixel in ``gray`` (``scale`` px/pt).""" + ink = gray < DARK + H, W = ink.shape + empty = 0 + for ln in runs: + x0, y0, x1, y1 = ln.bbox + r = slice(max(0, int(y0 * scale)), min(H, int(y1 * scale) + 1)) + c = slice(max(0, int(x0 * scale)), min(W, int(x1 * scale) + 1)) + if not ink[r, c].any(): + empty += 1 + return empty / max(len(runs), 1) + + +def read_text_layer( + page, gray: np.ndarray, scale: float, turn: int = 0, frame: int | None = None +) -> TextLayer: + """Text-layer runs of a page and the verdict on them. + + ``page`` is the open pypdfium2 page, ``gray`` a render of it at ``scale`` + pixels per point (a low resolution is enough for the ink check), turned + upright by ``turn`` quarter turns like the runs (``np.rot90(gray, -turn)``).""" + frame = turn if frame is None else frame + W, H = page.get_size() + size = (H, W) if frame % 2 else (W, H) + inv = invisible_text_frac(page) + if inv > MAX_INVISIBLE_FRAC: + return TextLayer([], size, False, f"invisible text {inv:.0%} (OCR layer)") + runs, st = extract_runs(page, turn, frame) + st["turn"], st["frame"] = turn, frame + rotated = [r for r in runs if r.quarter] + runs = [r for r in runs if not r.quarter] + if not runs: + return TextLayer([], size, False, "no text layer", st, rotated) + if st["bad"] > MAX_BAD_CHAR_FRAC * max(st["chars"], 1): + return TextLayer(runs, size, False, BROKEN_FONT_MAP, st, rotated) + # Only upright text has a reading direction on the page (a vertical page read + # turned does not). + pairs, visual = visual_rtl_frac(runs) if frame == turn else (0, 0.0) + if pairs >= MIN_RTL_PAIRS and visual > MAX_VISUAL_RTL_FRAC: + st["visual_rtl"] = round(visual, 3) + return TextLayer(runs, size, False, VISUAL_RTL, st, rotated) + empty = no_ink_frac(gray, runs, scale) + st["no_ink"] = round(empty, 4) + if empty > MAX_NO_INK_LINE_FRAC: + return TextLayer( + runs, size, False, f"{empty:.0%} of text runs have no ink", st, rotated + ) + return TextLayer(runs, size, True, "", st, rotated) diff --git a/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/vocab.py b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/vocab.py new file mode 100644 index 0000000000000000000000000000000000000000..4dfa204a350de42317a4c6156702c99e586b4e52 --- /dev/null +++ b/stallion/bina04-indra-pilot-20261027/upstream/indra_ocr/vocab.py @@ -0,0 +1,292 @@ +"""Token vocabulary for the line recognizer. + +Class 0 is the CTC blank. Class i + 1 is ``tokens[i]``. A token is one +character, or one LaTeX command (``\\frac``, ``\\alpha``) inside math. Labels +are NFC normalized before encoding. A training line with a token outside the +vocabulary is dropped (``encode`` returns None), never silently altered. + +``Vocab.latin()`` is the first recognizer's character set. ``Vocab.multi()`` +adds Cyrillic, full Greek, Devanagari, Bengali and LaTeX command tokens; math +is written ``$...$`` in labels. +""" + +from __future__ import annotations + +import json +import re +import unicodedata + +# Label canonicalization. Each rule maps characters or runs that the image +# cannot tell apart to one form, so neither training targets nor the eval +# punish a reading that is visually correct. +_CANON = str.maketrans( + { + # Romanian comma-below and Turkish/legacy cedilla look the same in most fonts. + "\u0218": "\u015e", # Ș -> Ş + "\u0219": "\u015f", # ș -> ş + "\u021a": "\u0162", # Ț -> Ţ + "\u021b": "\u0163", # ț -> ţ + # Bullet look-alikes (the font decides the drawn shape). + "\u25aa": "\u2022", # ▪ + "\u25cf": "\u2022", # ● + "\u2219": "\u2022", # ∙ + "\u2023": "\u2022", # ‣ + "\u2043": "\u2022", # ⁃ + } +) +# Runs of 4+ leader / rule characters: the count cannot be read from the image. +_RUNS = re.compile(r"([._\-=·*~])\1{3,}") +_DOUBLE_SINGLE = re.compile(r"''|``") + + +def canonical(text: str) -> str: + text = unicodedata.normalize("NFC", text).translate(_CANON) + text = _DOUBLE_SINGLE.sub('"', text) + return _RUNS.sub(lambda m: m.group(1) * 3, text) + + +def _range(a: int, b: int) -> list[str]: + return [chr(c) for c in range(a, b + 1)] + + +def _printable(chars) -> list[str]: + out = [] + for c in chars: + cat = unicodedata.category(c) + if cat.startswith(("C", "Z")) and c != " ": + continue # control, format, unassigned, private, separators other than space + if cat == "Mn": + continue # combining marks: labels are NFC, marks come precomposed + out.append(c) + return out + + +def _script_chars(a: int, b: int) -> list[str]: + """Assigned letters, marks and digits of a script block (combining marks kept: + Indic vowel signs and virama are separate codepoints even after NFC).""" + out = [] + for c in _range(a, b): + cat = unicodedata.category(c) + if cat == "Cn" or cat.startswith(("C", "Z")): + continue + out.append(c) + return out + + +# LaTeX commands the recognizer emits as single tokens inside $...$. +LATEX_COMMANDS = ( + "alpha beta gamma delta epsilon varepsilon zeta eta theta vartheta iota kappa lambda mu nu xi " + "pi varpi rho varrho sigma varsigma tau upsilon phi varphi chi psi omega " + "Gamma Delta Theta Lambda Xi Pi Sigma Upsilon Phi Psi Omega " + "frac dfrac tfrac sqrt sum prod int iint oint lim limsup liminf sup inf max min arg det " + "sin cos tan cot sec csc arcsin arccos arctan sinh cosh tanh log ln exp " + "left right big Big bigg Bigg langle rangle lfloor rfloor lceil rceil " + "cdot cdots ldots dots vdots ddots times div pm mp ast star circ bullet " + "leq geq neq le ge ne ll gg approx sim simeq cong equiv propto " + "in notin ni subset supset subseteq supseteq cup cap cap setminus emptyset varnothing " + "forall exists neg land lor wedge vee oplus otimes " + "to rightarrow leftarrow Rightarrow Leftarrow leftrightarrow Leftrightarrow mapsto " + "implies iff uparrow downarrow longrightarrow " + "infty partial nabla hbar ell prime perp parallel mid top angle " + "hat bar tilde vec dot ddot widehat widetilde overline underline " + "mathbb mathcal mathbf mathrm mathit mathsf mathfrak boldsymbol operatorname text " + "quad qquad " +).split() +LATEX_TOKENS = ["\\" + c for c in LATEX_COMMANDS] + [ + "\\{", + "\\}", + "\\|", + "\\,", + "\\;", + "\\!", + "\\ ", +] +_TOKEN_RE = re.compile(r"\\[A-Za-z]+|\\[{}|,;! ]|.", re.S) +# $...$ is math only when it holds LaTeX, so prices ("$5 or $10") stay text. +_MATH_SPAN = re.compile(r"\$[^$]*[\\^_{][^$]*\$") + + +def build_latin_chars() -> list[str]: + chars: list[str] = [] + chars += _range(0x20, 0x7E) # ASCII printable, with space + chars += _range( + 0x00A1, 0x00FF + ) # Latin-1 Supplement (letters, ¢ £ ¥ § © ° ± µ ¹²³ ½ ¼ ¾ × ÷ ...) + chars += _range(0x0100, 0x017F) # Latin Extended-A + chars += _range( + 0x0180, 0x024F + ) # Latin Extended-B (Ș ș Ț ț, Vietnamese Ơ ơ Ư ư, pinyin) + chars += _range( + 0x1E00, 0x1EFF + ) # Latin Extended Additional (Vietnamese precomposed) + chars += _range(0x0391, 0x03A9) + _range(0x03B1, 0x03C9) # basic Greek + chars += list("‐‑‒–—―‖‘’‚‛“”„‟†‡•‣…‰′″‹›⁄※") + chars += list("⁰⁴⁵⁶⁷⁸⁹⁺⁻⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉") + chars += list("€₹₽₩₺₪₱₫₴₦₡₸") + chars += list("™℃℉№ℓ℮⅓⅔⅛") + chars += list("←↑→↓↔⇒⇔↵") + chars += list("−∓∗∘∙√∞∝∠∧∨∩∪∫≈≠≡≤≥⊂⊃⊆⊇⊕⊗⊥⋅∂∆∇∈∉∋∏∑∀∃∅") + chars += list("■□▪▫▲△▶►▼▽◀◄◆◇○◦●★☆✓✔✗✘❖➢➤") + seen, out = set(), [] + for c in _printable(chars): + if c not in seen and c != "­": # soft hyphen is invisible + seen.add(c) + out.append(c) + return out + + +def build_multi_tokens() -> list[str]: + out = build_latin_chars() + seen = set(out) + extra = ( + _script_chars(0x0370, 0x03FF) # Greek and Coptic (accented Greek) + + _script_chars(0x0400, 0x045F) # Cyrillic (Russian, Ukrainian, Serbian, ...) + + _script_chars( + 0x0490, 0x04FF + ) # Cyrillic extended letters (Kazakh, Uzbek, ...) + + _script_chars(0x0900, 0x097F) # Devanagari (Hindi, Marathi, Nepali) + + _script_chars(0x0980, 0x09FF) # Bengali + + list("।॥₹") + + LATEX_TOKENS + ) + for t in extra: + if t not in seen: + seen.add(t) + out.append(t) + return out + + +class Vocab: + def __init__(self, chars: list[str]): + self.chars = list(chars) # tokens: characters, or LaTeX commands + self.index = {c: i + 1 for i, c in enumerate(self.chars)} + self.has_commands = any(len(c) > 1 for c in self.chars) + self.jamo = "\u1100" in self.index # Hangul syllables read as jamo + + @classmethod + def latin(cls) -> "Vocab": + return cls(build_latin_chars()) + + @classmethod + def multi(cls) -> "Vocab": + return cls(build_multi_tokens()) + + @classmethod + def named(cls, name: str) -> "Vocab": + if name not in ("latin", "multi"): + raise ValueError(f"unknown vocab {name!r}") + return cls.multi() if name == "multi" else cls.latin() + + def tokenize(self, text: str) -> list[str]: + """Characters, with a LaTeX command as one token when the vocab has it. + + Inside a $...$ math span spaces are dropped: they are not visible in + rendered math. ``decode`` puts back the one space LaTeX needs, between a + command and a following letter.""" + if not self.has_commands: + return list(text) + out: list[str] = [] + pos = 0 + for m in list(_MATH_SPAN.finditer(text)) + [None]: + end = m.start() if m else len(text) + out += self._tokens(text[pos:end], math=False) + if m: + out += self._tokens(m.group(0), math=True) + pos = m.end() + return out + + def _tokens(self, text: str, math: bool) -> list[str]: + out = [] + for t in _TOKEN_RE.findall(text): + if math and t.isspace(): + continue + out.extend([t] if t in self.index or len(t) == 1 else list(t)) + return out + + @property + def n_classes(self) -> int: + return len(self.chars) + 1 # + blank + + def normalize(self, text: str) -> str: + return canonical(text) + + def encode(self, text: str) -> list[int] | None: + text = self.normalize(text) + ids = [self.index.get(c) for c in self.tokenize(text)] + if not ids or any(i is None for i in ids): + return None + return ids + + def oov(self, text: str) -> set[str]: + return {c for c in self.tokenize(self.normalize(text)) if c not in self.index} + + def decode(self, ids: list[int]) -> str: + toks = [self.chars[i - 1] for i in ids if 0 < i <= len(self.chars)] + if self.jamo: # compose the jamo back into syllables + return unicodedata.normalize("NFC", "".join(toks)) + if not self.has_commands: + return "".join(toks) + out = [] + for k, t in enumerate(toks): + out.append(t) + # "\\alpha" + "x" must not read back as "\\alphax" + nxt = toks[k + 1] if k + 1 < len(toks) else "" + if len(t) > 2 and t[0] == "\\" and t[1:].isalpha() and nxt[:1].isalpha(): + out.append(" ") + return "".join(out) + + def to_json(self) -> str: + return json.dumps({"blank": 0, "chars": self.chars}, ensure_ascii=False) + + @classmethod + def from_json(cls, s: str) -> "Vocab": + d = json.loads(s) + if d.get("blank", 0) != 0: + raise ValueError("only blank=0 vocabularies are supported") + return cls(d["chars"]) + + +# Indic syllable structure. A broken font mapping (common in Nepali and Hindi +# PDFs converted from legacy fonts) leaves vowel signs with no consonant before +# them ("िामथ", "का ा") or two vowel signs in a row ("णािी"); valid text never does. +_INDIC_BLOCKS = ((0x0900, 0x097F), (0x0980, 0x09FF)) +_VIRAMA = {0x094D, 0x09CD} +_NUKTA = {0x093C, 0x09BC} +_MODIFIERS = {0x0900, 0x0901, 0x0902, 0x0903, 0x0981, 0x0982, 0x0983} + + +def _indic(c: str) -> bool: + o = ord(c) + return any(a <= o <= b for a, b in _INDIC_BLOCKS) + + +def indic_valid(text: str) -> bool: + """False when a Devanagari or Bengali mark has no base to attach to.""" + prev = "" + for c in text: + if _indic(c) and unicodedata.category(c) in ("Mn", "Mc"): + o = ord(c) + p_ok = bool(prev) and _indic(prev) + p_cat = unicodedata.category(prev) if prev else "" + if ( + o in _MODIFIERS + ): # anusvara, candrabindu, visarga: after a letter or a vowel sign + if not p_ok: + return False + elif o in _VIRAMA or o in _NUKTA: # after a consonant (or nukta) + if not (p_ok and (p_cat == "Lo" or ord(prev) in _NUKTA)): + return False + else: # vowel sign: after a consonant, nukta, or nothing else + if not (p_ok and (p_cat == "Lo" or ord(prev) in _NUKTA)): + return False + prev = c + return True + + +def indic_page_ok(texts: list[str], max_bad: float = 0.05) -> bool: + """A page whose Indic lines break syllable structure more than ``max_bad`` of + the time has a broken font mapping; its other lines are suspect too.""" + ind = [t for t in texts if any(_indic(c) for c in t)] + if not ind: + return True + return sum(not indic_valid(t) for t in ind) <= max_bad * len(ind) diff --git a/stallion/bina04-ink-loss-guard-95k-20261104/production.py b/stallion/bina04-ink-loss-guard-95k-20261104/production.py new file mode 100644 index 0000000000000000000000000000000000000000..852a6b3dfe637ab151ee56cf4b54e6675ca19345 --- /dev/null +++ b/stallion/bina04-ink-loss-guard-95k-20261104/production.py @@ -0,0 +1,25 @@ +"""Fallback only when primary detector text ink is lost by page assembly.""" +from pathlib import Path +import sys,importlib.util +R=Path(__file__).resolve().parent +D=Path('__BINA_ROOT__/stallion/bina04-document-handoff-95k-20261103') +class Runtime: + def __init__(self,device='auto'): + s=importlib.util.spec_from_file_location('accepted_table_parent','__BINA_ROOT__/stallion/bina04-table-asides-95k-20261104/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);self.raw=self.parent.raw;self.core=self.parent.core;self.fallback=None + def finish(self,image,base,old,kind): + if kind!='page':return base + sys.path.insert(0,str(D));from document import lost_main_text + guard=lost_main_text(image,base) + if not guard['fallback_required']:return base + if self.fallback is None: + s=importlib.util.spec_from_file_location('ink_loss_document_fallback',D/'production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);f=m.Runtime.__new__(m.Runtime);f.core=self.core;f.generic=None + node=self.parent + while not hasattr(node,'capture'):node=node.parent + from numeric_repairs import Repairs + node.capture.reset() + f.repair=Repairs(self.core.patch.reader.reader,node.numeric,self.core.patch.overlay.layer.reader);self.fallback=f + out=self.fallback.finish(image,base,old,kind) + out['primary_ink_loss_fallback']=dict(trigger=guard,unconditional_document_path=False) + return out + def __call__(self,image,old,kind,name='',**kwargs):return self.finish(image,self.parent(image,old,kind,name,**kwargs),old,kind) + def close(self):self.parent.close() diff --git a/stallion/bina04-line-detector-v2-20261030/detector-run/checkpoints/step-000500.pt b/stallion/bina04-line-detector-v2-20261030/detector-run/checkpoints/step-000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..24998887428be2b95c0fe8a5d02d002154a840a2 --- /dev/null +++ b/stallion/bina04-line-detector-v2-20261030/detector-run/checkpoints/step-000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0320b9d1d9e7464c4eae6f35b93fdd835913e785eae552eb5f04b58091b32ae6 +size 153922431 diff --git a/stallion/bina04-line-geometry-95k-20261103/assembly.py b/stallion/bina04-line-geometry-95k-20261103/assembly.py new file mode 100644 index 0000000000000000000000000000000000000000..59145ad34c966110b638d68b0cb7047f8997320f --- /dev/null +++ b/stallion/bina04-line-geometry-95k-20261103/assembly.py @@ -0,0 +1,41 @@ +import copy,re +from rapidfuzz.fuzz import ratio,partial_ratio + +def norm(s):return re.sub(r'[\s\u200cـ]+','',s) +def overlap(a,b): + x=max(0,min(a[2],b[2])-max(a[0],b[0]));y=max(0,min(a[3],b[3])-max(a[1],b[1]));aa=max(1,(a[2]-a[0])*(a[3]-a[1]));bb=max(1,(b[2]-b[0])*(b[3]-b[1]));return x*y/min(aa,bb),x/max(1,min(a[2]-a[0],b[2]-b[0])),y/max(1,min(a[3]-a[1],b[3]-b[1])) +def apply(old,threshold=.55,columns=False): + out=copy.deepcopy(old);ls=out['lines'];drop={};eligible=[i for i,l in enumerate(ls)if l.get('text','').strip() and len(l.get('box',[]))==4];conf=lambda i:float(ls[i].get('confidence',0));text=lambda i:norm(ls[i]['text']) + # Real short labels and isolated margin digits are never deleted by this candidate. + protected=lambda i:len(text(i))<=3 or all(c.isdigit()or c in '/:.-،٫'for c in text(i)) + for i in sorted(eligible,key=lambda i:-conf(i)): + if i in drop:continue + for j in eligible: + if j==i or j in drop or conf(j)>conf(i)or protected(j):continue + ov,x,y=overlap(ls[i]['box'],ls[j]['box']);a,b=text(i),text(j) + if ov>=.7 and x>=.65 and y>=.6 and min(len(a),len(b))>=6 and (ratio(a,b)>=80 or partial_ratio(a,b)>=90 and min(len(a),len(b))/max(len(a),len(b))>=.55):drop[j]=dict(reason='overlapping_near_duplicate',better=i) + elif conf(j)=.9 and conf(i)-conf(j)>=.15 and ov>=.8 and x>=.7: + drop[j]=dict(reason='low_confidence_covered_by_stronger_line',better=i) + # Locate actual serialized occurrences. Never regenerate table markdown or whitespace. + spans={};used=[];cursor=0 + for i in eligible: + t=ls[i]['text'];pos=out['text'].find(t,cursor) + if pos<0: + pos=out['text'].find(t) + while pos>=0 and any(posa for a,b in used):pos=out['text'].find(t,pos+1) + if pos>=0:spans[i]=(pos,pos+len(t));used.append(spans[i]);cursor=max(cursor,pos+len(t)) + applied={i:v for i,v in drop.items()if i in spans};s=out['text'] + for i in sorted(applied,key=lambda i:spans[i][0],reverse=True):a,b=spans[i];s=s[:a]+s[b:] + # Keep region serialization in sync: reference scoring reads region nodes. + from collections import Counter + keys=Counter((tuple(ls[i]['box']),ls[i]['text'])for i in applied) + for region in out.get('regions',[]): + kept=[] + for l in region.get('lines',[]): + key=(tuple(l.get('box',[])),l.get('text','')) + if keys[key]>0:keys[key]-=1 + else:kept.append(l) + region['lines']=kept + if not region.get('table_trace',{}).get('grid'):region['text']='\n'.join(l['text']for l in kept) + out['text']=s;out['lines']=[l for i,l in enumerate(ls)if i not in applied];out['page_assembly_candidate']=dict(threshold=threshold,removed=[dict(node_id=i,**v,line=ls[i])for i,v in applied.items()],unmapped_proposals=[i for i in drop if i not in spans],tables_reformatted=False,recognizer_weights_changed=False) + return out diff --git a/stallion/bina04-line-geometry-95k-20261103/form_rows.py b/stallion/bina04-line-geometry-95k-20261103/form_rows.py new file mode 100644 index 0000000000000000000000000000000000000000..040f468384b21dd067c91edd3c1adc4725718217 --- /dev/null +++ b/stallion/bina04-line-geometry-95k-20261103/form_rows.py @@ -0,0 +1,28 @@ +import copy,re +import numpy as np,cv2 +from assembly import norm + +def apply_rows(old,image): + out=copy.deepcopy(old);ls=out.get('lines',[]);W,H=image.size + if len(ls)<3 or any(r.get('table_trace',{}).get('grid')for r in out.get('regions',[])):return out + gray=cv2.cvtColor(np.asarray(image.convert('RGB')),cv2.COLOR_RGB2GRAY);bw=cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY_INV,31,12);rule=cv2.morphologyEx(bw,cv2.MORPH_OPEN,np.ones((1,max(25,int(W*.35))),np.uint8));ys=np.flatnonzero((rule>0).sum(axis=1)>.65*W) + if len(ys)==0:return out + groups=np.split(ys,np.flatnonzero(np.diff(ys)>1)+1);bounds=[float(np.median(g))for g in groups];heights=[l['box'][3]-l['box'][1]for l in ls if len(norm(l.get('text','')))>4];med=float(np.median(heights))if heights else 0 + if len(bounds)<4 or med<=0:return out + assignments={};rows=[] + for a,b in zip(bounds,bounds[1:]): + if not .6*med.2*min(ls[i]['box'][2]-ls[i]['box'][0],ls[j]['box'][2]-ls[j]['box'][0])for k,i in enumerate(ids)for j in ids[k+1:]):continue + order=sorted(ids,key=lambda i:-ls[i]['box'][2]);rows.append(dict(band=[a,b],before=ids,after=order));assignments.update({i:order for i in ids}) + if not rows:return out + done=set();order=[] + for i in range(len(ls)): + if i in done:continue + ids=assignments.get(i,[i]);order+=ids;done.update(ids) + if order==list(range(len(ls))):return out + if norm(''.join(l.get('text','')for l in ls))!=norm(out['text']):return out + out['text']='\n\n'.join(ls[i]['text']for i in order);out['lines']=[ls[i]for i in order];out['regions']=[dict(path='RTL_physical_form_row',layout=None,source_line_ids=[i],text=ls[i]['text'],lines=[ls[i]])for i in order];out['form_row_order']=dict(applied=True,ruling_bands=rows,read_text_changed=False) + return out diff --git a/stallion/bina04-line-geometry-95k-20261103/geometry_fix.py b/stallion/bina04-line-geometry-95k-20261103/geometry_fix.py new file mode 100644 index 0000000000000000000000000000000000000000..bf81b378e8d2a7f81e2c40e930a3b061f5408357 --- /dev/null +++ b/stallion/bina04-line-geometry-95k-20261103/geometry_fix.py @@ -0,0 +1,76 @@ +import copy,hashlib,json +import numpy as np,cv2 +from rapidfuzz.fuzz import ratio +from pathlib import Path +from assembly import norm + +def area(b):return max(0,b[2]-b[0])*max(0,b[3]-b[1]) +def cover(a,b):return area([max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])])/max(1,area(a)) +def apply_geometry(old,image,reader,expert,cache): + out=copy.deepcopy(old);ls=out.get('lines',[]);W,H=image.size + if any(l.get('layer')=='handwriting'for l in ls)or len(ls)<3 or any(r.get('table_trace',{}).get('grid')for r in out.get('regions',[])):return out + gray=np.asarray(image.convert('L'));med=float(np.median([l['box'][3]-l['box'][1]for l in ls if len(norm(l.get('text','')))>12]))if any(len(norm(l.get('text','')))>12 for l in ls)else 0 + if med<=0:return out + changes=[];remove=set();add=[] + for reg in out.get('raw_layout',{}).get('bboxes',[]): + if reg.get('label')!='Text':continue + x,y,X,Y=map(int,reg['bbox']);x,y,X,Y=max(0,x),max(0,y),min(W,X),min(H,Y) + if X-x<30 or X-x>.35*W or Y-y<1.6*med:continue + parent=[i for i,l in enumerate(ls)if cover([x,y,X,Y],l['box'])>.65 and l.get('layer')not in('handwriting','printed')and l.get('confidence',0)<.85 and l['box'][3]-l['box'][1]>1.5*med] + if not parent:continue + patch=gray[y:Y,x:X];pr=(patch<180).sum(1).astype('float32');sm=cv2.GaussianBlur(pr[:,None],(1,3),.8).ravel();ix=np.flatnonzero(sm>max(3,.38*float(sm.max()))) + if len(ix)==0:continue + bands=[z for z in np.split(ix,np.flatnonzero(np.diff(ix)>2)+1)if len(z)>=4] + if len(bands)<2 and (X-x)/max(1,Y-y)<3:continue + for band in bands: + if len(band)<4:continue + a,b=int(band[0]+y),int(band[-1]+y+1) + if not .3*med.9 and cover(box,l['box'])>.8 and l['box'][3]-l['box'][1]<1.5*(b-a)for l in ls):continue + key=hashlib.sha256(gray.tobytes()+str(box).encode()).hexdigest();dest=Path(cache)/(key+'.json');dest.parent.mkdir(parents=True,exist_ok=True) + if dest.exists():reads=json.loads(dest.read_text()) + else: + scale=3 if b-a<32 else 1;src=cv2.resize(gray,None,fx=scale,fy=scale,interpolation=cv2.INTER_CUBIC)if scale!=1 else gray;bb=[v*scale for v in box];reads=[reader(src,[bb])[0],expert(src,[bb])[0]];dest.write_text(json.dumps(reads,ensure_ascii=False)) + r,e=reads + if ratio(norm(r['text']),norm(e['text']))<90:continue + v=copy.deepcopy(max(reads,key=lambda z:z['confidence'])) + if v['confidence']<.85 or len(norm(v['text']))<4:continue + v.update(box=box,layer='printed',source='generic_small_text_band_recovery',two_readers_agree=True);v['geometry']['original_WH']=[W,H];add.append(v) + for i in parent:remove.add(i) + for i,l in enumerate(ls): + if l.get('layer')=='printed'and cover(l['box'],box)>.8 and len(norm(l['text']))<.65*len(norm(v['text'])):remove.add(i) + changes.append(dict(box=box,text=v['text'],confidence=v['confidence'],parent_ids=parent)) + if not changes:return out + # Insert the recovered physical rows at the parent location; untouched rows preserve order. + parent_ids={i for c in changes for i in c['parent_ids']};new=[];pending=sorted(add,key=lambda l:(l['box'][1],-l['box'][2])) + for i,l in enumerate(ls): + if i in parent_ids: + for a in pending[:]: + if cover(a['box'],l['box'])>.5:new.append(a);pending.remove(a) + if i not in remove:new.append(l) + new+=pending + if norm(''.join(l.get('text','')for l in ls))!=norm(out['text']):return out + out['lines']=new;out['text']='\n\n'.join(l['text']for l in new);out['regions']=[dict(path='generic_small_text_band_recovery',layout=None,lines=[l],text=l['text'])for l in new];out['small_text_geometry_recovery']=dict(applied=True,recovered=changes,removed_ids=sorted(remove),body_layer_untouched=True) + return out + +def order_widely_spaced_fragments(old,image): + out=copy.deepcopy(old);ls=out.get('lines',[]) + if any(l.get('layer')=='handwriting'for l in ls)or any(r.get('table_trace',{}).get('grid')for r in out.get('regions',[])):return out + pairs=[];used=set() + for i,a in enumerate(ls): + if i in used or a.get('confidence',0)<.9 or len(norm(a.get('text','')))<10 or a.get('layer')in('handwriting','printed'):continue + for j,b in enumerate(ls[:i]): + if i-j>3 or j in used or b.get('confidence',0)<.9 or len(norm(b.get('text','')))<10 or b.get('layer')in('handwriting','printed'):continue + left,right=sorted([i,j],key=lambda k:ls[k]['box'][0]);l,r=ls[left]['box'],ls[right]['box'];h=min(l[3]-l[1],r[3]-r[1]);gap=r[0]-l[2];ov=min(l[3],r[3])-max(l[1],r[1]) + midx=(l[2]+r[0])/2;neighbors=[z for z in ls if len(norm(z.get('text','')))>8 and min(l[1],r[1])-8*h<(z['box'][1]+z['box'][3])/2midx for z in neighbors) + if min(nl,nr)>=3:continue # A persistent gutter separates columns, not fragments of one row. + if 3*h.4*h and max(l[3]-l[1],r[3]-r[1])<1.5*h:pairs.append([right,left]);used.update([i,j]);break + if len(pairs)!=1 or out.get('learned_order',{}).get('applied',False):return out # Ambiguous multi-pair/learned-order pages retain their established order. + mapping={i:p for p in pairs for i in p};ids=[];done=set() + for i in range(len(ls)): + if i in done:continue + add=mapping.get(i,[i]);ids+=add;done.update(add) + if ids==list(range(len(ls)))or norm(''.join(l.get('text','')for l in ls))!=norm(out['text']):return out + out['lines']=[ls[i]for i in ids];out['text']='\n\n'.join(ls[i]['text']for i in ids);out['regions']=[dict(path='RTL_spaced_row_fragments',layout=None,lines=[ls[i]],text=ls[i]['text'])for i in ids];out['spaced_row_fragments']=dict(applied=True,pairs=pairs,recognition_unchanged=True) + return out diff --git a/stallion/bina04-line-geometry-95k-20261103/production.py b/stallion/bina04-line-geometry-95k-20261103/production.py new file mode 100644 index 0000000000000000000000000000000000000000..2b2e8d3291b7962364171a9a7e86a6f9945c2129 --- /dev/null +++ b/stallion/bina04-line-geometry-95k-20261103/production.py @@ -0,0 +1,16 @@ +"""Candidate: native RTL form rows and printed-page short text recovery.""" +from pathlib import Path +import importlib.util,sys +R=Path(__file__).resolve().parent +sys.path.insert(0,str(R)) +from form_rows import apply_rows +from geometry_fix import apply_geometry,order_widely_spaced_fragments +class Runtime: + def __init__(self,device='auto'): + s=importlib.util.spec_from_file_location('geometry_accepted_parent','__BINA_ROOT__/stallion/bina04-column-assembly-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);role=self.parent.parent.parent;self.core=role.core;self.raw=role.raw;self.numeric=role.numeric + from pipeline_v3 import UnifiedReader + v=Path('__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031');self.reader=UnifiedReader(v/'run/weights-095000.pt',v/'data/vocab.json');self.expert=UnifiedReader('__BINA_ROOT__/stallion/bina04-order-form-batch-20261101/recognizer/weights-001000.pt',v/'data/vocab.json') + def __call__(self,image,old,kind,name='',**kw): + out=apply_rows(self.parent(image,old,kind,name,**kw),image) + return order_widely_spaced_fragments(apply_geometry(out,image,self.reader,self.expert,R/'read-cache'),image)if kind=='page'else out + def close(self):self.parent.close() diff --git a/stallion/bina04-numeric-math-recognizer-95k-20261102/evaluate.py b/stallion/bina04-numeric-math-recognizer-95k-20261102/evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..d0d00e272bf31536cb550c9ed9a76f3ce4cf512c --- /dev/null +++ b/stallion/bina04-numeric-math-recognizer-95k-20261102/evaluate.py @@ -0,0 +1,199 @@ +"""Selected B1 reader, accepted page path and experts frozen; no B2 router/order.""" +import os +os.environ.update(CUDA_VISIBLE_DEVICES='0', OMP_NUM_THREADS='2', OPENBLAS_NUM_THREADS='1', + MKL_NUM_THREADS='2', FAST_DETECTOR_DEVICE='cpu', DETECTOR_IMAGE_CHUNK_HEIGHT='1400') +import copy, gc, hashlib, importlib.util, io, json, shutil, sys, time +from pathlib import Path +R=Path(__file__).resolve().parent +E=R/'evaluation' +BASE=Path('__BINA_ROOT__/stallion/bina04-crop-picture-cells-95k-20261102') +SMALL=Path('__BINA_ROOT__/stallion/bina04-small-crossing-rows-20261102') +J=Path('__BINA_ROOT__/stallion/bina04-e2e-batch-95k-20261101/candidates/combined') +OLD=Path('__BINA_ROOT__/stallion/bina04-printed-form-fix-20261102') +CURRENT=Path('__BINA_ROOT__/stallion/bina04-print-textgate-20261102') +P=Path('__BINA_ROOT__/stallion/indra-scale') +A=Path('__BINA_ROOT__/stallion/bina04-handwriting-attribution-95k-20261101') +C=Path('__BINA_ROOT__/stallion/bina04-rulec-preproc-95k-20261031') +SPLIT=Path('__BINA_ROOT__/stallion/bina04-surya25-order-20261101/REAL-SPLIT.json') +sys.path[:0]=[str(SMALL),str(CURRENT),str(OLD),str(J/'versions/combined'),str(J), + str(J/'snapshot/runtime'),str(J/'snapshot/runtime/baseline'),str(P),str(P/'scoringTT')] +def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest() +def dump(p,q): + p.parent.mkdir(parents=True,exist_ok=True) + tmp=p.with_suffix('.partial') + tmp.write_text(json.dumps(q,ensure_ascii=False,indent=2,default=lambda v:v.tolist()if hasattr(v,'tolist')else str(v))+'\n') + tmp.replace(p) +def load(name,path): + spec=importlib.util.spec_from_file_location(name,path);m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);return m + +class CoreEngine: + def __init__(self,cp,device='cuda',*,with_annotations=True): + import torch + from surya.detection import DetectionPredictor + from line_contract_reader import ContractReader + from learned_backend import install + from baseline.table_client import SuryaTableGrid + from runtime import SmallPagePatch + detector=DetectionPredictor.local(device=device,dtype=torch.float32);detector._disable_tqdm=True + det=J/'run/checkpoints/step-001000.pt' + detector.model.load_state_dict(torch.load(det,map_location='cpu',weights_only=True)['model']);detector.model.eval();install(detector,sha(det)) + reader=ContractReader(cp,'__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json',device=device,cache=E/'boxing-cache') + self.context={};context=self.context + class Layout: + def run_batch(self,images,params): + class Result: + def model_dump(self,**kwargs):return context['layout'] + return [Result()] + table=SuryaTableGrid() + hybrid=load('b1_frozen_hybrid',OLD/'production.py').HybridPipeline(detector,Layout(),reader,table,variant='combined-news4-coursefirst',row_parameters={}) + overlay=load('b1_frozen_print_overlay',CURRENT/'production.py').PageOverlay(device) + patch=SmallPagePatch.__new__(SmallPagePatch) + patch.detector=detector;patch.reader=reader;patch.context=self.context;patch.table=table;patch.base=hybrid;patch.overlay=overlay + self.patch=patch;self.order=load('b1_accepted_order',Path('__BINA_ROOT__/stallion/bina04-order-labels-current-95k-20261102/production.py')).PageOverlay('cpu') + # Same input category source as the accepted crop gate; gold boxes/text + # are never passed to the pipeline. + self.annotations={k:dict(category=v['category'])for k,v in json.loads(Path('__BINA_ROOT__/stallion/bina04-line-detector-20261030/oracle193-annotations.json').read_text()).items()}if with_annotations else{} + self.columns=json.loads(Path('__BINA_ROOT__/stallion/bina04-crop-tables-newspaper-20261031/PLAN.json').read_text())['column_format_overrides']if with_annotations else{} + def __call__(self,image,old,kind,name,*,content_type='',table_column_order='auto'): + from runtime_release import normalize_receipts + self.context['layout']=copy.deepcopy(old['raw_layout']) + if self.patch.eligible(image,input_type=kind): + result=normalize_receipts(self.patch(image,old,input_type=kind,source_name=name)) + else: + # Preserve the accepted crop invocation, including newspaper metadata + # and the explicitly frozen table-column convention. + kw=dict(input_type=kind,source_name=name,table_column_order=self.columns.get(name,table_column_order)) + if content_type:kw['content_type']=content_type + elif self.annotations.get(name,{}).get('category')=='newspaper':kw['content_type']='newspaper' + result=self.patch.base(image,**kw) + result=self.patch.overlay(image,result,input_type=kind) + return self.order(image,result,input_type=kind) + def close(self):self.patch.close() + +class Engine: + def __init__(self,cp,device='cuda'): + from baseline.pipeline_v3 import flip + sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-crop-bandseg-95k-20261102') + from bandseg import segment + parent=load('num_math_band','__BINA_ROOT__/stallion/bina04-crop-bandseg-95k-20261102/consumer_runtime.py').BandGuardRuntime(CoreEngine(cp,device),flip) + current=Path('__BINA_ROOT__/stallion/bina04-crop-picture-cells-95k-20261102');sys.path.insert(0,str(current)) + self.runtime=load('num_math_cells',current/'candidate.py').PipelineRuntime(parent,lambda t:flip(t,base_rtl=True),segment) + def __call__(self,*args,**kwargs):return self.runtime(*args,**kwargs) + def close(self):return self.runtime.close() + +def main(): + import fcntl + from resource_guard import stable_admission,new_foreign_work + lock=Path('__BINA_ROOT__/stallion/bina04-b2cleandata-20261024/continuous/.evaluation.lock').open('a+') + try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + except BlockingIOError:sys.exit(75) + adm=stable_admission() + if adm is None:sys.exit(75) + lease_start=time.monotonic() + def yield_check(): + if time.monotonic()-lease_start>=120 or new_foreign_work(adm):sys.exit(75) + import torch,cv2,pyarrow.parquet as pq + from PIL import Image + from shenavasanj import fair_text + from rapidfuzz.distance import Levenshtein + torch.set_num_threads(2);cv2.setNumThreads(1);torch.cuda.set_per_process_memory_fraction(.25) + torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False + assert (R/'recognizer/COMPLETE.json').exists() + best=json.loads((R/'recognizer/best.json').read_text());cp=Path(best['path']);cp_sha=sha(cp) + E.mkdir(exist_ok=True) + cache=E/'boxing-cache';cache.mkdir(exist_ok=True) + for f in Path('__BINA_ROOT__/stallion/bina04-oldprint-headlines-95k-20261102/hard/e2e/boxing-cache').glob('*.json'):shutil.copyfile(f,cache/f.name) + signature=dict(checkpoint=str(cp),sha256=cp_sha,source_sha256=sha(__file__),baseline=str(BASE), + selected_by='Regular six-bucket macro + private generated numeric/math macro; no gate-based selection',B2_router_used=False,B2_order_used=False, + layout='Frozen image-only layout receipts from accepted baseline',device='cuda') + if (E/'SIGNATURE.json').exists():assert json.loads((E/'SIGNATURE.json').read_text())==signature + else:dump(E/'SIGNATURE.json',signature) + refs={q['id']:q for q in map(json.loads,(A/'FINAL-CONTROLS.jsonl').read_text().splitlines())} + split=json.loads(SPLIT.read_text()) + controls=[] + for category in ['forms','screens','prose','math','newspaper','tables']: + candidates=[json.loads(p.read_text()) for p in sorted((BASE/'gate/193').glob('*.json'))] + prefixes={'forms':['057'],'screens':['007'],'prose':['090'],'math':['071'],'newspaper':['067'],'tables':['083']} + old=next(q for q in candidates if any('image_'+v+'__'in q['id']for v in prefixes[category])) + image=Path('__BINA_ROOT__/stallion/bina04-crop-data-20261024/inputs/benchmark193')/old['id'] + controls.append((category,old,image,'crop',old['id'])) + for rid in [split['calibration'][0],split['check'][0]]: + old=json.loads((BASE/'gate/dev50'/(rid.replace(':','-')+'.json')).read_text()) + controls.append(('handwriting',old,Path(refs[rid]['image_path']),'page',rid)) + if not(E/'CONTROL-CHECK.json').exists(): + engine=Engine('__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031/run/weights-095000.pt') + checks=[] + for category,old,path,kind,rid in controls: + yield_check();im=Image.open(path).convert('RGB');actual=engine(im,old['pipeline'],kind,rid) + expected=old.get('prediction',old['pipeline']['text']) + checks.append(dict(id=rid,category=category,text_matches=actual['text']==expected,actual=actual['text'],expected=expected)) + engine.close();del engine;gc.collect();torch.cuda.empty_cache() + dump(E/'CONTROL-CHECK.json',dict(passed=all(q['text_matches']for q in checks),checks=checks)) + assert json.loads((E/'CONTROL-CHECK.json').read_text())['passed'],'Frozen95k control differs; fix harness before any candidate decision' + engine=Engine(cp) + started=time.monotonic();count=0 + def infer(old,image,kind,rid,target): + nonlocal count + if target.exists():return json.loads(target.read_text()) + yield_check();result=engine(image,old['pipeline'],kind,rid) + row=dict(old,pipeline=result,prediction=result['text'],checkpoint_sha256=cp_sha,refinement_step=best['step'],status='ok',reader_device='cuda',detector_device='cuda') + dump(target,row);count+=1 + if count%10==0:print(json.dumps(dict(phase='B1_gate',group=kind,id=rid,completed=count,minutes=(time.monotonic()-started)/60)),flush=True) + return row + sc=load('b1_frozen_handscore',Path('__BINA_ROOT__/stallion/bina04-order-salvage-20261101/evaluate_batch.py'));sc.Image=Image + def metric(g,h): + g=fair_text(g,keep_spaces=False);h=fair_text(h,keep_spaces=False) + return dict(characters=len(g),errors=Levenshtein.distance(g,h),CER=100*Levenshtein.distance(g,h)/max(1,len(g))) + hand=[];original={} + for rid,ref in refs.items(): + name=rid.replace(':','-')+'.json';old=json.loads((BASE/'gate/dev50'/name).read_text());original[rid]=old + assert sha(ref['image_path'])==ref['image_sha256'];im=Image.open(ref['image_path']).convert('RGB') + cached=infer(old,im,'page',rid,E/'raw/dev50'/name) + row=sc.score_hand(ref,cached['pipeline'],im);row['metrics']['spatial_body_diagnostic']=row['metrics']['body'] + row['metrics']['body']=metric(ref['body_reference'],'\n'.join(l['text']for l in cached['pipeline']['lines']if l.get('layer')!='printed')) + row['metrics']['tagged']=row['metrics']['body'];row['metrics']['full']=metric(ref['full_reference'],cached['pipeline']['text']) + row['body_reads_byte_identical']=False;dump(E/'gate/dev50'/name,row);hand.append(row) + for p in sorted((BASE/'gate/193').glob('*.json')): + old=json.loads(p.read_text());ip=Path('__BINA_ROOT__/stallion/bina04-crop-data-20261024/inputs/benchmark193')/old['id'];assert sha(ip)==old['image_sha256'] + infer(old,Image.open(ip).convert('RGB'),'crop',old['id'],E/'gate/193'/p.name) + ids=json.loads((C/'PLAN.json').read_text())['IDs'];needed={int(i.split(':')[1])for i in ids};offset=0 + for p in sorted(Path('__BINA_ROOT__/stallion/bina02-surya-layout/assets/benchmark/data').glob('printed-*.parquet')): + pf=pq.ParquetFile(p) + for group in range(pf.num_row_groups): + n=pf.metadata.row_group(group).num_rows;ix=sorted(i-offset for i in needed if offset<=imax(needed):break + engine.close() + dm=load('numeric_exact_diagnostic','__BINA_ROOT__/stallion/bina04-smallmarks-celllabels-95k-20261102/decompose.py') + diagnostics={} + for phase,chosen in [('calibration',split['calibration']),('check',split['check']),('all',list(refs))]: + gold={i:refs[i]['full_reference'] for i in chosen};new={i:json.loads((E/'gate/dev50'/(i.replace(':','-')+'.json')).read_text())['pipeline']['text'] for i in chosen};old={i:original[i]['pipeline']['text'] for i in chosen} + diagnostics[phase]=dict(before=dm.score(gold,old,'/tmp/claude_hw_decomp.py'),after=dm.score(gold,new,'/tmp/claude_hw_decomp.py')) + dump(E/'READING-DIAGNOSTIC.json',diagnostics) + cells=[] + for f in(E/'gate/193').glob('*.json'): + if not f.name.startswith(('image_081','image_083')):continue + candidate=json.loads(f.read_text());baseline=json.loads((BASE/'gate/193'/f.name).read_text()) + cells.append(dict(id=candidate['id'],reference=candidate['original_reference'],before=baseline['prediction'],after=candidate['prediction'],before_cells=[r.get('table_trace',{}).get('cell_debug',[]) for r in baseline['pipeline']['regions']],after_cells=[r.get('table_trace',{}).get('cell_debug',[]) for r in candidate['pipeline']['regions']])) + dump(E/'DATE-TIME-HANDWRITTEN-CELLS.json',cells) + def total(rows): + return {k:dict(characters=sum(q['metrics'][k]['characters']for q in rows),errors=sum(q['metrics'][k]['errors']for q in rows),CER=100*sum(q['metrics'][k]['errors']for q in rows)/sum(q['metrics'][k]['characters']for q in rows))for k in ['body','full']} + bc=total([original[rid]for rid in split['check']]);cc=total([q for q in hand if q['id']in split['check']]) + deltas={q['id']:q['metrics']['full']['CER']-original[q['id']]['metrics']['full']['CER']for q in hand if q['id']in split['check']} + eligible=cc['full']['CER']<=bc['full']['CER']+1e-12 and cc['body']['CER']<=bc['body']['CER']+1e-12 and sum(v>1e-12 for v in deltas.values())<=5 and max(deltas.values())<=3+1e-12 + dump(E/'REAL-CHECK.json',dict(eligible_for_gate=eligible,baseline=bc,candidate=cc,page_deltas=deltas,worse_pages=sum(v>1e-12 for v in deltas.values()),maximum_regression_pp=max(0,max(deltas.values())))) + dump(E/'BASELINE.json',dict(metrics=json.loads((BASE/'GATE-RESULT.json').read_text())['metrics'])) + dump(E/'GATE-PLAN.json',signature);dump(E/'gate/INFERENCE-COMPLETE.json',dict(dev50=50,gold193=193,printed338=338)) + shutil.copyfile(R/'score_gate.py',E/'score_gate.py');load('b1_frozen_gate',E/'score_gate.py').main() + report=json.loads((E/'GATE-RESULT.json').read_text());significant=cc['full']['CER']<=9.2 or report['metrics']['full_CER']<=9.72 + decision=dict(accepted=report['accepted'],selected=best,gate=str(E/'GATE-RESULT.json'),baseline=str(BASE), + candidate_reader=str(cp)if report['accepted']else'__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031/run/weights-095000.pt', + significant_for_TT=significant,public_TT_runs=0,main_Bina_changed=False,teacher18404_changed=False) + dump(R/'END-TO-END-DECISION.json',decision) + print(json.dumps(dict(decision='OLDPRINT_ACCEPTED'if report['accepted']else'OLDPRINT_REJECTED',metrics=report['metrics'],check25=cc,TT_allowed_by_significance=significant)),flush=True) +if __name__=='__main__':main() diff --git a/stallion/bina04-numeric-math-recognizer-95k-20261102/recognizer/selected.pt b/stallion/bina04-numeric-math-recognizer-95k-20261102/recognizer/selected.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6000308633e3bda21aac84eb00815c485285faa --- /dev/null +++ b/stallion/bina04-numeric-math-recognizer-95k-20261102/recognizer/selected.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443 +size 131255668 diff --git a/stallion/bina04-numeric-routing-95k-20261103/consumer.py b/stallion/bina04-numeric-routing-95k-20261103/consumer.py new file mode 100644 index 0000000000000000000000000000000000000000..66c07cae5441cfcebd65a9a4ec414160dee64e2c --- /dev/null +++ b/stallion/bina04-numeric-routing-95k-20261103/consumer.py @@ -0,0 +1,50 @@ +"""Accepted grid-cell and automatic display-fraction expert; original95k elsewhere.""" +import copy,hashlib,importlib.util,sys,threading,time +from pathlib import Path +R=Path(__file__).resolve().parent;SOURCE=Path('__BINA_ROOT__/stallion/bina04-numeric-math-recognizer-95k-20261102');CP=Path('__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031');P=Path('__BINA_ROOT__/stallion/indra-scale') +def load(name,path):s=importlib.util.spec_from_file_location(name,path);m=importlib.util.module_from_spec(s);s.loader.exec_module(m);return m +def clean(value): + if isinstance(value,dict):return {k:clean(v) for k,v in value.items() if k!='_numeric_capture_id'} + if isinstance(value,list):return [clean(v) for v in value] + return value +class Runtime: + def __init__(self,device='auto'): + import torch + if device=='auto':device='cuda' if torch.cuda.is_available() else 'cpu' + sys.path[:0]=[str(R),str(CP),str(P)] + assert hashlib.sha256((CP/'run/weights-095000.pt').read_bytes()).hexdigest()=='036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981' + assert hashlib.sha256((SOURCE/'recognizer/selected.pt').read_bytes()).hexdigest()=='6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443' + m=load('accepted_fraction_core',SOURCE/'evaluate.py');m.E=R/'runtime-cache';m.E.mkdir(exist_ok=True) + from baseline.pipeline_v3 import flip + sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-crop-bandseg-95k-20261102');from bandseg import segment + core=m.CoreEngine(CP/'run/weights-095000.pt',device,with_annotations=False) + parent=load('accepted_fraction_band','__BINA_ROOT__/stallion/bina04-crop-bandseg-95k-20261102/consumer_runtime.py').BandGuardRuntime(core,flip) + root=Path('__BINA_ROOT__/stallion/bina04-crop-picture-cells-95k-20261102');sys.path.insert(0,str(root));self.engine=load('accepted_fraction_cells',root/'candidate.py').PipelineRuntime(parent,lambda t:flip(t,base_rtl=True),segment) + sys.path.insert(0,str(R));from route import Capture + self.capture=Capture();from rtl_reader import RTLReader + self.numeric=RTLReader(SOURCE/'recognizer/selected.pt',CP/'data/vocab.json',device=device);self.lock=threading.RLock() + def __call__(self,image,old,kind,name='',**kwargs): + from route import candidates,numeric_reads,apply,key + from formula import standalone + from formula_private import prepared + with self.lock: + self.capture.reset();fresh=self.engine(image,old,kind,name,**kwargs);base=clean(fresh);items=candidates(base,fresh,self.capture,content_type='');chosen=[] + cells={key(l) for region in base['regions'] if region.get('table_trace',{}).get('grid') for l in region['lines']} + for item in items: + item['route_d']=key(item['old_line']) in cells + if item['math'] and standalone(item['old_line']['text']): + ink,trace=prepared(item['record']['ink']) + if any(e['kind']=='display_fraction_to_inline_slash' for e in trace.get('events',[])): + item['record']=dict(item['record'],ink=ink,sha256=hashlib.sha256(ink.tobytes()).hexdigest());item['route_c']=True;item['formula_preprocessing']=trace + if item.get('route_c') or item['route_d']:chosen.append(item) + paired=numeric_reads(self.numeric,chosen) + for item in paired: + if not item.get('route_c'):continue + g=item['new_line']['geometry'];left,_,right,_=g['padded_source_extent'];width=item['formula_preprocessing']['preprocessing']['resized_WH'][0];g['transformed_frame_centers_input_pixels']=[(v-left)*width/max(1e-12,right-left) for v in g.pop('frame_centers')];g['frame_centers']=[];g['frame_centers_source_valid']=False;g['source_pixels_modified']=True;g['formula_preprocessing']=item['formula_preprocessing'];item['new_line']['recognizer']='display_formula_numeric' + result=apply(base,paired,math=False,numeric=False,formula=True,cell=True) + result['automatic_formula_context']=dict(benchmark_category_used=False,template_match_used=False,reader_models_kept_loaded=True,unrouted_lines_byte_identical=True) + return result + def close(self): + import rtl_reader + rtl_reader.RTLReader.__call__=self.capture.original + self.engine.close() diff --git a/stallion/bina04-numeric-routing-95k-20261103/formula.py b/stallion/bina04-numeric-routing-95k-20261103/formula.py new file mode 100644 index 0000000000000000000000000000000000000000..8b31b52a0b4cc96ef1aac38aeec4fa94c2821c3f --- /dev/null +++ b/stallion/bina04-numeric-routing-95k-20261103/formula.py @@ -0,0 +1,48 @@ +"""Image-only normalization of display fractions and tall formula operators.""" +import cv2,numpy as np +from PIL import Image,ImageDraw +def normalize(gray): + _,binary=cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU) + if (binary>0).mean()>.5:binary=255-binary + count,labels,stats,_=cv2.connectedComponentsWithStats(binary,8) + components=[(i,tuple(map(int,s))) for i,s in enumerate(stats[1:],1) if s[4]>=2] + heights=[h for _,(x,y,w,h,a) in components if h>=4 and w>=2 and wmax(2,.18*height) or w=y+h and v-y-h<1.5*height:below.append((u,v,u+ww,v+hh)) + if not above or not below:continue + union=lambda b:[min(q[0] for q in b),min(q[1] for q in b),max(q[2] for q in b),max(q[3] for q in b)] + top,bottom=union(above),union(below) + if top[3]-top[1]>1.8*height or bottom[3]-bottom[1]>1.8*height:continue + fractions.append(dict(bar=[x,y,x+w,y+h],top=top,bottom=bottom)) + # Disjoint bars only. Nested fractions require a structural model. + fractions=sorted(fractions,key=lambda q:q['bar'][0],reverse=True);work=gray.copy();events=[] + def resized(box): + x,y,X,Y=box;a=gray[y:Y,x:X];hh=max(4,round(height));return cv2.resize(a,(max(1,round(a.shape[1]*hh/max(1,a.shape[0]))),hh),interpolation=cv2.INTER_CUBIC) + for fraction in fractions: + x,y,X,Y=fraction['bar'];top=fraction['top'];bottom=fraction['bottom'];left=min(x,top[0],bottom[0]);right=max(X,top[2],bottom[2]);numerator=resized(top);denominator=resized(bottom) + ordinary=[v+hh for _,(u,v,w,hh,a) in components if hh>=.5*height and hh<=1.6*height and (u+w<=left or u>=right)] + baseline=int(np.median(ordinary)) if ordinary else y+round(height/2);rowtop=max(0,baseline-round(height));slash=Image.new('L',(max(4,round(.5*height)),max(4,round(height))),255);ImageDraw.Draw(slash).line((1,slash.height-2,slash.width-2,1),fill=0,width=max(1,round(.08*height))) + gap=max(1,round(.12*height));parts=[numerator,np.asarray(slash),denominator];width=sum(p.shape[1] for p in parts)+gap*2;out=np.full((max(work.shape[0],rowtop+round(height)+2),work.shape[1]+width-(right-left)),255,np.uint8);out[:work.shape[0],:left]=work[:,:left];out[:work.shape[0],left+width:]=work[:,right:];cursor=left + for piece in parts:out[rowtop:rowtop+piece.shape[0],cursor:cursor+piece.shape[1]]=piece;cursor+=piece.shape[1]+gap + work=out;events.append(dict(kind='display_fraction_to_inline_slash',source=fraction,output_x=[left,left+width],baseline=baseline,semantic_rule='Visible numerator / denominator; no text substitution')) + # Normalize oversized connected symbols, leaving detached superscripts alone. + _,m=cv2.threshold(work,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU);n,ll,ss,_=cv2.connectedComponentsWithStats(m,8) + ordinary=[y+h for x,y,w,h,a in ss[1:] if h>=.5*height and h<=1.6*height and a>=2];baseline=int(np.median(ordinary)) if ordinary else work.shape[0]//2 + for ident,(x,y,w,h,a) in enumerate(ss[1:],1): + if h<1.9*height or w>3*height or a<4:continue + core=np.where(ll[y:y+h,x:x+w]==ident,work[y:y+h,x:x+w],255).astype(np.uint8);nh=max(4,round(1.2*height));nw=max(2,round(w*nh/h));small=cv2.resize(core,(nw,nh),interpolation=cv2.INTER_AREA);work[ll==ident]=255;yy=max(0,min(work.shape[0]-nh,baseline-nh));xx=max(0,min(work.shape[1]-nw,round(x+(w-nw)/2)));work[yy:yy+nh,xx:xx+nw]=np.minimum(work[yy:yy+nh,xx:xx+nw],small);events.append(dict(kind='normalize_tall_connected_formula_symbol',source_box=[int(x),int(y),int(x+w),int(y+h)],target_box=[xx,yy,xx+nw,yy+nh])) + return work,dict(applied=bool(events),events=events,median_component_height=height) + +def standalone(text): + visible=[c for c in text if not c.isspace()] + arabic=sum('\u0600'<=c<='\u06ff' and not c.isdigit() for c in visible) + return len(visible)>=5 and arabic/max(1,len(visible))<=.15 diff --git a/stallion/bina04-numeric-routing-95k-20261103/formula_private.py b/stallion/bina04-numeric-routing-95k-20261103/formula_private.py new file mode 100644 index 0000000000000000000000000000000000000000..95eabfdfe5c4b986826c9c5173aa1b6b422b038f --- /dev/null +++ b/stallion/bina04-numeric-routing-95k-20261103/formula_private.py @@ -0,0 +1,71 @@ +"""Fresh math-layout preflight, no training and no benchmark text/pixels.""" +import os +os.environ.update(OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='1') +import sys,json,random,hashlib,fcntl +from pathlib import Path +import numpy as np,cv2 +from PIL import Image,ImageDraw,ImageFont +R=Path(__file__).resolve().parent;CP=Path('__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031');P=Path('__BINA_ROOT__/stallion/indra-scale');S=Path('__BINA_ROOT__/stallion/bina04-numeric-math-recognizer-95k-20261102') +sys.path[:0]=[str(R),str(CP),str(P),'__BINA_ROOT__/stallion/bina04-recognizer-round2-20261030','__BINA_ROOT__/stallion/bina04-indra-pilot-20261027/upstream'] +from formula import normalize +from geometry import crop_line64,pad_batch +from text_contract import decode_prediction +def prepared(ink): + gray=np.rint((1.-ink)*255).clip(0,255).astype(np.uint8);work,trace=normalize(gray) + if not trace['applied']:return ink,trace + _,m=cv2.threshold(work,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU);yy,xx=np.where(m>0) + if not len(xx):return ink,dict(applied=False,reason='empty') + out,g=crop_line64(work,[int(xx.min()),int(yy.min()),int(xx.max()+1),int(yy.max()+1)]);trace['preprocessing']=g;return out,trace +def render(rng,fraction): + paths=list(Path('/usr/share/fonts/truetype/dejavu').glob('DejaVuSerif*.ttf'));path=rng.choice(paths);size=rng.randint(12,22);font=ImageFont.truetype(str(path),size);small=ImageFont.truetype(str(path),max(9,round(.72*size)));big=ImageFont.truetype(str(path),round(1.9*size));a,b=rng.sample('abcdrs',2);n=rng.randint(2,999) + def glyph(t,f): + bb=f.getbbox(t);im=Image.new('L',(bb[2]-bb[0]+2,bb[3]-bb[1]+2),255);ImageDraw.Draw(im).text((1-bb[0],1-bb[1]),t,font=f,fill=0);return im + pieces=[glyph(f'KL({a} ∥ {b}) = − ',font),glyph('∑',big),glyph(f'{a}(x) log(',font)] + if fraction: + top=glyph(f'{b}(x)',small);bottom=glyph(f'{a}(x)',small);w=max(top.width,bottom.width)+4;frac=Image.new('L',(w,top.height+bottom.height+7),255);frac.paste(top,((w-top.width)//2,0));ImageDraw.Draw(frac).line((0,top.height+3,w-1,top.height+3),fill=0,width=1);frac.paste(bottom,((w-bottom.width)//2,top.height+7));pieces.append(frac) + else:pieces.append(glyph(f'{b}(x)/{a}(x)',font)) + pieces.append(glyph(f') + {n}',font));H=max(p.height for p in pieces)+8;W=sum(p.width for p in pieces)+6+2*len(pieces);page=Image.new('L',(W,H),255);x=3;baseline=round(.65*H) + for i,p in enumerate(pieces): + y=max(0,min(H-p.height,baseline-p.height if i not in [1,3] else (H-p.height)//2));page.paste(p,(x,y));x+=p.width+2 + return page,f'KL({a} ∥ {b}) = −∑ {a}(x) log({b}(x)/{a}(x)) + {n}' +def read(reader,inks): + import torch + outputs=[] + with torch.inference_mode(): + for start in range(0,len(inks),16): + x,w=pad_batch(inks[start:start+16]);logits,lengths=reader.model(torch.from_numpy(x).to(reader.device),torch.from_numpy(w));seq=logits.argmax(-1).cpu().numpy() + for i in range(len(w)): + v=seq[i,:int(lengths[i])];keep=(v!=0)&np.r_[True,v[1:]!=v[:-1]];outputs.append(decode_prediction(''.join(reader.alphabet[k-1] for k in v[keep]))) + return outputs +def main(): + import torch + from rtl_reader import RTLReader + from leakage import Guard + from rapidfuzz.distance import Levenshtein as L + from resource_guard import stable_admission + lock=Path('__BINA_ROOT__/stallion/bina04-b2cleandata-20261024/continuous/.evaluation.lock').open('a+') + try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + except BlockingIOError:return 75 + if stable_admission() is None:return 75 + torch.set_num_threads(2);cv2.setNumThreads(1);torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False + rng=random.Random(2611034242);guard=Guard('__BINA_ROOT__/stallion/bina04-recognizer-round2-20261030/data/protected.sqlite3');cases=[] + for split in ['calibration','check']: + for fraction in [True,False]: + done=0 + while done<48: + im,text=render(rng,fraction) + if guard.reason(text):continue + ink,g=crop_line64(np.asarray(im),[0,0,im.width,im.height]);out,trace=prepared(ink);cases.append(dict(id=f'{split}-{fraction}-{done}',split=split,fraction=fraction,reference=text,before=ink,after=out,trace=trace));done+=1 + predictions={} + for name,path in [('95k',CP/'run/weights-095000.pt'),('numeric',S/'recognizer/selected.pt')]: + reader=RTLReader(path,CP/'data/vocab.json',device='cuda');predictions[name]={mode:read(reader,[q[mode] for q in cases]) for mode in ['before','after']};del reader;torch.cuda.empty_cache() + norm=lambda s:''.join(s.split());scores={} + for split in ['calibration','check']: + ids=[i for i,q in enumerate(cases) if q['split']==split];scores[split]={} + for reader in predictions: + for mode in ['before','after']: + errors=sum(L.distance(norm(cases[i]['reference']),norm(predictions[reader][mode][i])) for i in ids);chars=sum(len(norm(cases[i]['reference'])) for i in ids);scores[split][reader+'_'+mode]=100*errors/chars + reader=min(['95k','numeric'],key=lambda r:scores['calibration'][r+'_after']);passed=scores['calibration'][reader+'_after'] 0).mean())) + pairs += 1 + for i, role in enumerate(roles): + if role and i not in protected: + confidence.append(float(probabilities[:, i, role].min())) + agreement.append(1.) + info.update(confidence=min(float(np.quantile(confidence, .1)), float(np.mean(agreement))) if confidence else 0., + changed_pair_count=pairs, + confidence_method='Original ensemble/group margins; observed protected rows do not require learned role confidence') + return output, info + + +class V3(OriginalV3): + @torch.inference_mode() + def propose(self, result, image): + nodes = nodes_from_result(result) + n = len(nodes) + if not 2 <= n <= 512: + return result, dict(changed=False, confidence=0., reason='Outside trained node context') + x = torch.from_numpy(features3(nodes, image, result))[None] + pad = torch.zeros(1, n, dtype=torch.bool) + ranks, probs = [], [] + for model in self.models: + rank, logits, _ = model(x, pad) + ranks.append(rank[0].numpy()) + probs.append(logits[0].softmax(-1).numpy()) + ranks, probs = np.asarray(ranks), np.asarray(probs) + roles = np.zeros(n, np.int64) + roles[probs[:, :, 1].min(0) >= .70] = 1 + roles[probs[:, :, 2].min(0) >= .95] = 2 + roles[probs[:, :, 3].min(0) >= .995] = 3 + roles[[i for i, node in enumerate(nodes) if node['kind'] == 'table']] = 0 + return apply(result, nodes, ranks, probs, roles) + + +class V2(OriginalV2): + @torch.inference_mode() + def propose(self, result, image): + nodes = nodes_from_result(result) + n = len(nodes) + if not 2 <= n <= 512: + return result, dict(changed=False, confidence=0., reason='Outside trained node context') + x = torch.from_numpy(features2(nodes, image, result))[None] + pad = torch.zeros(1, n, dtype=torch.bool) + ranks, probs = [], [] + for model in self.models: + rank, logits, _ = model(x, pad) + ranks.append(rank[0].numpy()) + probs.append(logits[0].softmax(-1).numpy()) + ranks, action_probs = np.asarray(ranks), np.asarray(probs) + probs = np.zeros((*action_probs.shape[:2], 4), np.float32) + probs[:, :, 0] = action_probs[:, :, 0] + probs[:, :, 2] = action_probs[:, :, 1] + probs[:, :, 3] = action_probs[:, :, 2] + roles = np.zeros(n, np.int64) + roles[action_probs[:, :, 1].min(0) >= .90] = 2 + roles[action_probs[:, :, 2].min(0) >= .98] = 3 + return apply(result, nodes, ranks, probs, roles) diff --git a/stallion/bina04-order-form-batch-20261101/recognizer/weights-001000.pt b/stallion/bina04-order-form-batch-20261101/recognizer/weights-001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b2b856d824bb65b318bf6fdeca3b6ae3fca84c1 --- /dev/null +++ b/stallion/bina04-order-form-batch-20261101/recognizer/weights-001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:367084525dc644ef6d69ad981819af07e3c6a41421359594a54ccc989c2fb90a +size 131256958 diff --git a/stallion/bina04-order-labels-current-95k-20261102/CALIBRATION.json b/stallion/bina04-order-labels-current-95k-20261102/CALIBRATION.json new file mode 100644 index 0000000000000000000000000000000000000000..76a993ada14ff261ed63ec3e46eb8b3b264c0b2a --- /dev/null +++ b/stallion/bina04-order-labels-current-95k-20261102/CALIBRATION.json @@ -0,0 +1,7 @@ +{ + "mode": "order_only", + "include_threshold": 1.1, + "admission": 0.75, + "order_threshold": 0.5, + "selection": "Predeclared safe order-only fallback; label candidate not promoted" +} diff --git a/stallion/bina04-order-labels-current-95k-20261102/order_only.py b/stallion/bina04-order-labels-current-95k-20261102/order_only.py new file mode 100644 index 0000000000000000000000000000000000000000..1740dcbac8d3b09df1f3b432345dd585c9c477eb --- /dev/null +++ b/stallion/bina04-order-labels-current-95k-20261102/order_only.py @@ -0,0 +1,23 @@ +"""Fixed v3 ranks: move explicit printed nodes only; never append/exclude reads.""" +import json,sys,numpy as np,torch +from collections import Counter +from pathlib import Path +sys.path[:0]=['__BINA_ROOT__/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime','__BINA_ROOT__/stallion/bina04-e2e-batch-95k-20261101/candidates/combined/snapshot/runtime/baseline','__BINA_ROOT__/stallion/bina04-order-application-debug-20261102','__BINA_ROOT__/stallion/bina04-surya25-order-20261101','__BINA_ROOT__/stallion/bina04-order-predicted-20261101'] +from fixed_order import V3 as Original,apply +from role_nodes import features +from order_nodes import nodes_from_result + +def body(result):return json.dumps([l for l in result['lines']if l.get('layer')!='printed'],ensure_ascii=False,sort_keys=True,default=lambda v:v.tolist()if hasattr(v,'tolist')else str(v)) +class PrintedOnly(Original): + @torch.inference_mode() + def propose(self,result,image): + nodes=nodes_from_result(result);n=len(nodes) + if not 2<=n<=512:return result,dict(changed=False,confidence=0.,reason='Outside trained node context') + x=torch.from_numpy(features(nodes,image,result))[None];pad=torch.zeros(1,n,dtype=torch.bool);ranks=[];probs=[] + for model in self.models: + rank,logits,_=model(x,pad);ranks.append(rank[0].numpy());probs.append(logits[0].softmax(-1).numpy()) + proposal,info=apply(result,nodes,np.asarray(ranks),np.asarray(probs),np.zeros(n,np.int64)) + if Counter(''.join(proposal['text'].split()))!=Counter(''.join(result['text'].split())):return result,dict(info,changed=False,confidence=0.,serialization_character_multiset_rejected=True) + if body(proposal)!=body(result):return result,dict(info,changed=False,confidence=0.,atomic_table_body_invariant_rejected=True) + proposal['handwriting_extents']=result.get('handwriting_extents',[]) + return proposal,dict(info,printed_only=True,no_append_or_exclude=True,all_body_bytes_and_order_preserved=True) diff --git a/stallion/bina04-order-labels-current-95k-20261102/production.py b/stallion/bina04-order-labels-current-95k-20261102/production.py new file mode 100644 index 0000000000000000000000000000000000000000..a4ffc5152cc975900652894f67d8add07ba7afd6 --- /dev/null +++ b/stallion/bina04-order-labels-current-95k-20261102/production.py @@ -0,0 +1,21 @@ +"""One additional calibrated pass on the accepted small-crossgrid/printed-layer output.""" +import copy,json,sys +from pathlib import Path +R=Path(__file__).resolve().parent;sys.path.insert(0,str(R)) +from order_only import PrintedOnly,body +class PageOverlay: + def __init__(self,device='cpu',shared_layer=None): + self.cal=json.loads((R/'CALIBRATION.json').read_text());self.mode=self.cal['mode'];self.order=PrintedOnly('__BINA_ROOT__/stallion/bina04-surya25-order-20261101/role-model/weights-003000.pt');self.layer=None + if self.mode=='tiled': + from prepare_tiled import PrintedLayer + self.layer=shared_layer if shared_layer is not None else PrintedLayer(device) + def __call__(self,image,baseline,*,input_type='page'): + if input_type!='page'or baseline.get('order_label_candidate',{}).get('root')==str(R):return baseline + original=body(baseline);result=copy.deepcopy(baseline) + if self.mode=='tiled': + from prepare_tiled import infer + from tiled_apply import apply + pp=infer(self.layer,image,result);new=pp['recovered'];anchors=[l for l in result['lines']if l.get('layer')=='printed'];prob=self.layer.include.predict(new+anchors,pp['WH'],pp['body_box']) + for l,p in zip(new,prob):l['include_probability']=p + result,trace=apply(result,pp,self.cal['include_threshold'],self.cal['admission']) + proposal,info=self.order.propose(copy.deepcopy(result),image);use=bool(info.get('changed')and info.get('confidence',0)>=self.cal['order_threshold']);result=proposal if use else result;assert body(result)==original;result['printed_order_update']=dict(info,applied=use,threshold=self.cal['order_threshold']);result['order_label_candidate']=dict(root=str(R),mode=self.mode,body_bytes_and_order_unchanged=True,main_Bina_changes=False);return result diff --git a/stallion/bina04-order-predicted-20261101/order_nodes.py b/stallion/bina04-order-predicted-20261101/order_nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..cc3e15d215433545f8f4f44faf9dca1bc9fba776 --- /dev/null +++ b/stallion/bina04-order-predicted-20261101/order_nodes.py @@ -0,0 +1,53 @@ +"""Identical node/features/render contracts for predicted synthetic and real pages.""" +import hashlib,math +import cv2,numpy as np +FEATURES=160 + +def box_union(lines): + b=[q['box']for q in lines];return[min(q[0]for q in b),min(q[1]for q in b),max(q[2]for q in b),max(q[3]for q in b)]if b else[0,0,1,1] +def rect(p): + a=np.asarray(p,float);return[a[:,0].min(),a[:,1].min(),a[:,0].max(),a[:,1].max()] +def area(b):return max(0,b[2]-b[0])*max(0,b[3]-b[1]) +def intersect(a,b):return[max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])] +def structured(region):return bool(region.get('table_trace',{}).get('grid'))or region.get('path')=='Surya_grid_unified_recognizer_cells'or'table'in region.get('path','').lower() +def nodes_from_result(result): + nodes=[] + if any(structured(q)for q in result.get('regions',[])): + for region in result['regions']: + if structured(region): + lines=region.get('lines',[]);nodes.append(dict(box=box_union(lines),text=region.get('text',''),kind='table',layer='printed',lines=lines,region=region,confidence=float(np.mean([l.get('confidence',.8)for l in lines]))if lines else .8)) + else: + for line in region.get('lines',[]):nodes.append(dict(**line,kind='line',lines=[line])) + else: + for line in result.get('lines',[]):nodes.append(dict(**line,kind='line',lines=[line])) + return nodes + +def features(nodes,image,result): + W,H=image.size;gray=np.asarray(image.convert('L'));ink=(gray<180).astype(np.uint8);hor=cv2.morphologyEx(ink,cv2.MORPH_OPEN,np.ones((1,max(20,round(.04*W))),np.uint8));ver=cv2.morphologyEx(ink,cv2.MORPH_OPEN,np.ones((max(20,round(.03*H)),1),np.uint8));grid=hor|ver + page=np.r_[[hor[a:b].mean()for a,b in zip(np.linspace(0,H,5).astype(int)[:-1],np.linspace(0,H,5).astype(int)[1:])],[ver[:,a:b].mean()for a,b in zip(np.linspace(0,W,5).astype(int)[:-1],np.linspace(0,W,5).astype(int)[1:])],cv2.resize(grid.astype(np.float32),(4,2),interpolation=cv2.INTER_AREA).reshape(-1)] + hands=[q for q in nodes if q.get('layer')=='handwriting'];hb=box_union(hands);layout=result.get('raw_layout',{}).get('bboxes',[]);out=[] + for i,n in enumerate(nodes): + x,y,X,Y=map(float,n['box']);w=max(1,X-x);h=max(1,Y-y);cx=(x+X)/2;cy=(y+Y)/2;text=n.get('text','');hashes=np.zeros(64,np.float32) + for token in text.split(): + for j in range(max(1,len(token)-1)): + z=int.from_bytes(hashlib.blake2b(token[j:j+2].encode(),digest_size=4).digest(),'little');hashes[z%64]+=1 if z&64 else-1 + hashes/=max(1,float(np.linalg.norm(hashes)));a,b,A,B=max(0,int(x)),max(0,int(y)),min(W,math.ceil(X)),min(H,math.ceil(Y));patch=np.zeros(32,np.float32) + if A>a and B>b:patch=cv2.resize(1.-gray[b:B,a:A].astype(np.float32)/255.,(8,4),interpolation=cv2.INTER_AREA).reshape(-1) + tag='unknown';best=0 + for r in layout: + rb=r.get('bbox')or rect(r['polygon'])if 'polygon'in r else r.get('bbox') + if rb is None:continue + overlap=area(intersect(n['box'],rb))/max(1,area(n['box'])) + if overlap>best:best=overlap;tag=str(r.get('label',r.get('type','unknown'))) + onehot=[float(k.lower()in tag.lower())for k in ['Text','Title','Table','Picture','PageHeader','PageFooter','List']]+[float(tag=='unknown')] + flags=[float(n.get('layer')=='handwriting'),float(n.get('form_cell',False)),sum(c.isdigit()for c in text)/max(1,len(text)),sum('A'<=c<='z'for c in text)/max(1,len(text)),min(1,len(text)/100),float(n.get('confidence',.8)),float(n.get('kind')=='table'),float(n.get('layer')=='printed'),min(1,len(text.split())/20),float(n.get('cell')is not None)] + geom=[x/W,y/H,X/W,Y/H,cx/W,cy/H,w/W,h/H,min(2,w/h/20),min(2,W/H)];rel=[(cy-hb[1])/max(1,hb[3]-hb[1]),(cx-hb[0])/max(1,hb[2]-hb[0]),float(Yhb[3])]if hands else[0,0,0,0];f=np.r_[i/max(1,len(nodes)-1),geom,flags,hashes,patch,page,rel,onehot];f=np.pad(f,(0,FEATURES-len(f)));out.append(f) + return np.asarray(out,np.float32).reshape(-1,FEATURES) + +def render(result,nodes,ranks,actions): + import copy + result=copy.deepcopy(result);keep=[i for i in range(len(nodes))if actions[i]==0];keep.sort(key=lambda i:(float(ranks[i]),i));append=[i for i in range(len(nodes))if actions[i]==1];order=keep+append;lines=[line for i in order for line in nodes[i]['lines']];result['text']='\n'.join(nodes[i]['text']for i in order);result['lines']=lines;result['regions']=[dict(nodes[i].get('region',{}),path='learned_order_atomic_table'if nodes[i]['kind']=='table'else'learned_order_line',text=nodes[i]['text'],lines=nodes[i]['lines'],source_line_ids=[])for i in order];result['emitted_boxes']=[q['box']for q in lines] + from baseline.pipeline_v3 import reading_extent + result['recognition_extents']=[reading_extent(q)for q in lines] + if 'handwriting_extents'in result:result['handwriting_extents']=[reading_extent(q)for q in lines if q.get('layer')=='handwriting'] + result['learned_order']=dict(applied=True,input_to_output_order=order,actions=list(map(int,actions)),atomic_tables_preserved=sum(n['kind']=='table'for n in nodes),references_used=False);return result diff --git a/stallion/bina04-order-predicted-20261101/pred_order_model.py b/stallion/bina04-order-predicted-20261101/pred_order_model.py new file mode 100644 index 0000000000000000000000000000000000000000..7c62f58e42d582ed1d9e76c091627941c3f11169 --- /dev/null +++ b/stallion/bina04-order-predicted-20261101/pred_order_model.py @@ -0,0 +1,29 @@ +import numpy as np +import torch +from torch import nn +from order_nodes import FEATURES,features,nodes_from_result,render +class OrderNet(nn.Module): + def __init__(self): + super().__init__();self.input=nn.Linear(FEATURES,96);layer=nn.TransformerEncoderLayer(96,4,192,dropout=0.,batch_first=True,norm_first=True);self.encoder=nn.TransformerEncoder(layer,2,enable_nested_tensor=False);self.rank=nn.Linear(96,1);self.action=nn.Linear(96,3);nn.init.zeros_(self.rank.weight);nn.init.zeros_(self.rank.bias);nn.init.zeros_(self.action.weight);self.action.bias.data.copy_(torch.tensor([2.,0.,0.])) + def forward(self,x,pad): + z=self.encoder(self.input(x),src_key_padding_mask=pad);delta=self.rank(z).squeeze(-1).tanh();return(x[:,:,0]+delta).clamp(0,1),self.action(z),delta +class OrderEnsemble: + def __init__(self,path): + q=torch.load(path,map_location='cpu',weights_only=True);self.models=[] + for state in q['models']: + model=OrderNet();model.load_state_dict(state);model.eval();self.models.append(model) + @torch.inference_mode() + def propose(self,result,image): + nodes=nodes_from_result(result);n=len(nodes) + if not 2<=n<=512:return result,dict(changed=False,confidence=0.,reason='Outside trained node context') + x=torch.from_numpy(features(nodes,image,result))[None];pad=torch.zeros(1,n,dtype=torch.bool);ranks=[];probs=[] + for model in self.models: + r,a,_=model(x,pad);ranks.append(r[0].numpy());probs.append(a[0].softmax(-1).numpy()) + ranks=np.asarray(ranks);probs=np.asarray(probs);mean=ranks.mean(0);actions=np.zeros(n,np.int64);actions[probs[:,:,1].min(0)>=.9]=1;actions[probs[:,:,2].min(0)>=.98]=2;order=sorted([i for i in range(n)if actions[i]==0],key=lambda i:(float(mean[i]),i))+[i for i in range(n)if actions[i]==1];changed=order!=list(range(n));changed_pairs=[(i,j)for i in range(n)for j in range(i+1,n)if actions[i]==actions[j]==0 and mean[i]>mean[j]];confidences=[];agreements=[] + for i,j in changed_pairs: + d=ranks[:,i]-ranks[:,j];z=float(d.mean()/np.sqrt(d.var()+.0004));confidences.append(float(1/(1+np.exp(-np.clip(z,-30,30)))));agreements.append(float((d>0).mean())) + for i,a in enumerate(actions): + if a:confidences.append(float(probs[:,i,a].min()));agreements.append(1.) + confidence=min(float(np.quantile(confidences,.1)),float(np.mean(agreements)))if confidences else 0.;proposal=render(result,nodes,mean,actions)if changed else result + info=dict(changed=changed,confidence=confidence,changed_pair_count=len(changed_pairs),excluded=int((actions==2).sum()),appended=int((actions==1).sum()),node_count=n,rank_std_mean=float(ranks.std(0).mean()),confidence_method='Ensemble agreement and changed-pair margin relative to ensemble variance plus residual uncertainty floor; real calibration threshold frozen separately') + return proposal,info diff --git a/stallion/bina04-order-salvage-20261101/evaluate_batch.py b/stallion/bina04-order-salvage-20261101/evaluate_batch.py new file mode 100644 index 0000000000000000000000000000000000000000..012ccba077c7d373122129bd1faf6864c534ffd1 --- /dev/null +++ b/stallion/bina04-order-salvage-20261101/evaluate_batch.py @@ -0,0 +1,94 @@ +"""One selected-candidate gate. Resumable120s shared GPU leases; public TT unopened.""" +import os +os.environ.update(OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='2',FAST_DETECTOR_DEVICE='cpu',CUBLAS_WORKSPACE_CONFIG=':4096:8') +import fcntl,gzip,hashlib,json,math,sys,time +from pathlib import Path +R=Path(__file__).resolve().parent;G=Path('__BINA_ROOT__/stallion/bina04-e2e-batch-95k-20261101');J=G/'candidates/combined';S=J/'snapshot/runtime';P=Path('__BINA_ROOT__/stallion/indra-scale');L=Path('__BINA_ROOT__/stallion/bina04-crop-tables-newspaper-20261031');C=Path('__BINA_ROOT__/stallion/bina04-rulec-preproc-95k-20261031');A=Path('__BINA_ROOT__/stallion/bina04-handwriting-attribution-95k-20261101');sys.path[:0]=[str(R),str(J),str(S),str(S/'baseline'),str(P),str(P/'scoringTT')] +def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest() +def dump(p,v): + p.parent.mkdir(exist_ok=True,parents=True);t=p.with_suffix('.partial');t.write_text(json.dumps(v,ensure_ascii=False,default=lambda x:x.tolist()if hasattr(x,'tolist')else str(x))+'\n');t.replace(p) +class YieldLease(BaseException):pass + +def score_hand(ref,result,im): + import numpy as np,cv2 + from baseline.pipeline_v3 import clipped,intersection + from baseline.ink_metrics import covered_ink + from shenavasanj import fair_text + from rapidfuzz.distance import Levenshtein + rid=ref['id'];gray=np.asarray(im.convert('L'));H,W=gray.shape;draft=json.loads((Path('__BINA_ROOT__/stallion/bina04-line-detector-20261030/TT50-drafts')/(rid.replace(':','-')+'.json')).read_text());bg=np.asarray(Image.open(Path('__BINA_ROOT__/stallion/persian-handwriting-369k/work/background-variants')/draft['background_variant']['file']).convert('L'),dtype=np.int16);ink=bg-gray.astype(np.int16)>25;I=cv2.integral(ink.astype(np.uint8),sdepth=cv2.CV_32S) + def count(box): + a,b,A,B=clipped(box,W,H);a,b,A,B=map(int,[math.floor(a),math.floor(b),math.ceil(A),math.ceil(B)]);return int(I[B,A]-I[b,A]-I[B,a]+I[b,a])if A>a and B>b else 0 + chosen=[] + for line in result['lines']: + b=line['box'];cx=(b[0]+b[2])/2;cy=(b[1]+b[3])/2;inside=any(q['text_region'][0]<=cx<=q['text_region'][2]and q['text_region'][1]<=cy<=q['text_region'][3]for q in draft['composer_regions']);hit=max((count(intersection(b,g['box']))for g in ref['gold']),default=0) + if ref['full_page_oracle_valid']or inside or hit>=6:chosen.append(line) + def metric(gold,text): + g=fair_text(gold,keep_spaces=False);h=fair_text(text,keep_spaces=False);n=Levenshtein.distance(g,h);return dict(characters=len(g),errors=n,CER=100*n/max(1,len(g))) + body='\n'.join(l['text']for l in chosen);tagged='\n'.join(l['text']for l in result['lines']if l.get('layer')=='handwriting')if result['row_consolidation']['applied']else body;actual=result.get('handwriting_extents',result['recognition_extents']);coverage=[covered_ink(g,ink,actual)for g in ref['gold']] + return dict(id=rid,pipeline=result,body_prediction=body,tagged_handwriting_prediction=tagged,metrics=dict(body=metric(ref['body_reference'],body),tagged=metric(ref['body_reference'],tagged),full=metric(ref['full_reference'],result['text'])),counts=dict(gold=len(ref['gold']),scoped_readings=len(chosen),handwriting_tagged=sum(l.get('layer')=='handwriting'for l in result['lines'])if result['row_consolidation']['applied']else len(chosen)),coverage=dict(covered=sum(c['covered']for c in coverage),total=len(coverage),failures=[dict(line=i,**c)for i,c in enumerate(coverage)if not c['covered']]),full_page_oracle_valid=ref['full_page_oracle_valid']) + +def main(): + global Image + import numpy as np,torch + from PIL import Image + from surya.detection import DetectionPredictor + from salvage_pipeline import CellPipeline,CellReader + from learned_backend import install + from baseline.pipeline_v3 import rectangle,clipped + from baseline.table_client import SuryaTableGrid + from resource_guard import stable_admission,new_foreign_work + from gpu_eval_runtime import configure_fp32 + O=R/'gate';plan=json.loads((R/'GATE-PLAN.json').read_text()) + for p,h in plan['source_sha256'].items():assert sha(p)==h,'Frozen candidate source changed:'+p + if(O/'INFERENCE-COMPLETE.json').exists():return 0 + lock=Path('__BINA_ROOT__/stallion/bina04-b2cleandata-20261024/continuous/.evaluation.lock').open('a+') + try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + except BlockingIOError:return 75 + adm=stable_admission() + if adm is None:return 75 + configure_fp32();torch.set_num_threads(2);tick=time.monotonic() + def check(): + if time.monotonic()-tick>=120 or new_foreign_work(adm):raise YieldLease() + detector=DetectionPredictor.local(device='cuda',dtype=torch.float32);detector._disable_tqdm=True;detector.model.load_state_dict(torch.load(plan['detector_checkpoint'],map_location='cpu',weights_only=True)['model']);detector.model.eval();reader=CellReader(plan['recognizer_checkpoint'],plan['vocabulary'],device='cuda',cache=J/'reader-cache');install(detector,plan['detector_sha256'],check);context={} + class Layout: + def run_batch(self,*args,**kwargs): + class Result: + def model_dump(_,**kw):return context['saved']['raw_layout'] + return[Result()] + table=SuryaTableGrid(yield_check=check) + def grid(crop): + key=hashlib.sha256(crop.tobytes()).hexdigest();path=L/'grid-cache'/(key+'.json') + if path.exists():return json.loads(path.read_text()) + if key in context['grids']:return context['grids'][key] + return table(crop) + engine=CellPipeline(detector,Layout(),reader,grid,variant='combined-news4-coursefirst',row_parameters={});normal_detect=engine.detect;ann=json.loads(Path('__BINA_ROOT__/stallion/bina04-line-detector-20261030/oracle193-annotations.json').read_text());columns=json.loads((S/'PLAN.json').read_text())['column_format_overrides'] + try: + for stage in ['dev50','193','printed']: + items=list(map(json.loads,((A/'FINAL-CONTROLS.jsonl')if stage=='dev50'else(P/'scoring193/frozen/metadata.jsonl')if stage=='193'else(C/'inputs/printed.jsonl')).read_text().splitlines())) + for ref in items: + rid=ref.get('id',ref.get('file_name'));target=O/stage/(rid.replace(':','-')+'.json') + if target.exists():continue + check();grids={} + if stage=='dev50':path=Path(ref['image_path']);assert sha(path)==ref['image_sha256'];old=json.loads(gzip.decompress((L/'full-scorecard/TT/pages'/(rid.replace(':','-')+'.json.gz')).read_bytes()));image_sha=ref['image_sha256'] + elif stage=='193':path=Path('__BINA_ROOT__/stallion/bina04-crop-data-20261024/inputs/benchmark193')/rid;old=json.loads((L/'full-scorecard/193'/(rid+'.json')).read_text());assert sha(path)==old['image_sha256'];image_sha=old['image_sha256'] + else:path=Path(ref['path']);assert sha(path)==ref['png_sha256'];old=json.loads(gzip.decompress((L/'full-scorecard/TT/pages'/(rid.replace(':','-')+'.json.gz')).read_bytes()));image_sha=ref['source_image_sha256'] + im=Image.open(path).convert('RGB');saved=old['pipeline'];context['saved']=saved + for region in saved['regions']: + trace=region.get('table_trace',{}) + if not trace.get('grid'):continue + b=clipped(rectangle(region['layout']['polygon']),im.width,im.height);b=[math.floor(b[0]),math.floor(b[1]),math.ceil(b[2]),math.ceil(b[3])];grids[hashlib.sha256(im.crop(b).tobytes()).hexdigest()]=trace['grid'] + context['grids']=grids;kwargs=dict(input_type='page'if stage=='dev50'else'crop',source_name=rid,table_column_order=columns.get(rid,'auto')) + if stage=='193'and ann[rid]['category']=='newspaper':kwargs['content_type']='newspaper' + if stage=='dev50': + cached=json.loads((J/'detection-cache/selected-dev50'/(rid.replace(':','-')+'.json')).read_text());assert cached['image_sha256']==image_sha;engine.detect=lambda image:(cached['boxes'],cached['trace']) + else:engine.detect=normal_detect + start=time.monotonic();result=engine(im,**kwargs) + if stage=='dev50':out=score_hand(ref,result,im) + else:out=dict(id=rid,status='ok',image_sha256=image_sha,prediction=result['text'],pipeline=result,checkpoint_sha256=plan['recognizer_sha256'],detector_sha256=plan['detector_sha256']) + if stage=='193':out.update(original_reference=ref['gold_text'],reached_token_cap=False) + out.update(device='CUDA FP32',seconds=time.monotonic()-start);dump(target,out);print(json.dumps(dict(stage=stage,id=rid,seconds=out['seconds'])),flush=True) + finally:table.close() + assert all(len(list((O/s).glob('*.json')))==n for s,n in [('dev50',50),('193',193),('printed',338)]);dump(O/'INFERENCE-COMPLETE.json',dict(pages50=50,crops193=193,printed338=338,plan_sha256=sha(R/'GATE-PLAN.json')));return 0 +if __name__=='__main__': + try:sys.exit(main()) + except YieldLease:sys.exit(75) diff --git a/stallion/bina04-order-salvage-20261101/row_builder.py b/stallion/bina04-order-salvage-20261101/row_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..5695f63e34fa78704be6c2da6e89a7bbd64c78bb --- /dev/null +++ b/stallion/bina04-order-salvage-20261101/row_builder.py @@ -0,0 +1,102 @@ +"""Image-only pen/print separation and physical row construction; no gold inputs.""" +import math +import numpy as np +import cv2 + +DEFAULT=dict(core_overlap=.5,baseline_distance=.4,max_gap=3.,nominal_pad=.06,color_angle_degrees=35,color_depth=.055,color_chroma=.045,color_relative_chroma=.16,min_pen_components=15,min_pen_pixels=80) +def rect_intersection(a,b):return[max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])] +def rect_area(a):return max(0,a[2]-a[0])*max(0,a[3]-a[1]) +def estimate_height(proposals,H): + h=[b[3]-b[1]for b in proposals if b[3]>b[1]and b[2]-b[0]>=2*(b[3]-b[1])and 4<=b[3]-b[1]<=min(128,.12*H)] + return max(6.,min(100.,float(np.percentile(h,60))if h else H/40)) +def remove_rules(mask,line): + H,W=mask.shape;u=mask.astype(np.uint8);horizontal=cv2.morphologyEx(u,cv2.MORPH_OPEN,np.ones((1,max(30,round(.12*W))),np.uint8));vertical=cv2.morphologyEx(u,cv2.MORPH_OPEN,np.ones((max(30,round(.1*H)),1),np.uint8));rules=cv2.dilate(horizontal|vertical,np.ones((3,3),np.uint8));return(u>0)&~(rules>0),rules>0 +def components(mask,min_area=2): + n,labels,stats,centers=cv2.connectedComponentsWithStats(mask.astype(np.uint8),8);return labels,[dict(label=i,x=int(x),y=int(y),w=int(w),h=int(h),area=int(a),cx=float(centers[i][0]),cy=float(centers[i][1]))for i,(x,y,w,h,a)in enumerate(stats)if i and a>=min_area] +def columns(mask,line,proposals=()): + H,W=mask.shape;profile=mask.sum(axis=0);low=profile<=max(1,.002*H);edges=np.diff(np.r_[False,low,False].astype(int));gaps=[] + for a,b in zip(np.where(edges==1)[0],np.where(edges==-1)[0]): + if b-a.85*W:continue + cut=(a+b)//2 + crossing=[p for p in proposals if p[0]b+line and p[2]-p[0]>.55*W and p[3]-p[1]<2*line] + if len(crossing)>=3:continue + left=int(mask[:,:a].sum());right=int(mask[:,b:].sum());total=max(1,int(mask.sum())) + if min(left,right)>=.15*total and min(a,W-b)>=5*line:gaps.append((b-a,(a+b)//2)) + if not gaps:return[(0,W)] + cut=max(gaps)[1];return[(cut,W),(0,cut)] +def proposal_columns(boxes,W,H,gray): + vertical=cv2.morphologyEx((gray<100).astype(np.uint8),cv2.MORPH_OPEN,np.ones((max(30,round(.18*H)),1),np.uint8));occupied=vertical.sum(0)>.18*H;edges=np.diff(np.r_[False,occupied,False].astype(int));rulings=[] + for a,b in zip(np.where(edges==1)[0],np.where(edges==-1)[0]): + if not rulings or a-rulings[-1][1]>.01*W:rulings.append([a,b]) + else:rulings[-1][1]=b + # Count physical rulings once, including antialiased/double-stroke borders. + if len(rulings)>=5:return[(0,W)] + anchors=[b for b in boxes if .2*W<=b[2]-b[0]<=.7*W and b[3]-b[1]<=.15*H];centers=sorted(((b[0]+b[2])/2,b)for b in anchors) + if len(centers)<6:return None + for gap,k in sorted(((centers[k+1][0]-centers[k][0],k)for k in range(2,len(centers)-3)),reverse=True): + if gap<.15*W:break + left=[b for _,b in centers[:k+1]];right=[b for _,b in centers[k+1:]];a=float(np.median([b[2]for b in left]));z=float(np.median([b[0]for b in right])) + if z-a>=.02*W: + cut=round((a+z)/2);return[(cut,W),(0,cut)] + return None + +def layer_masks(rgb,proposals,params): + H,W=rgb.shape[:2];line=estimate_height(proposals,H);k=max(15,min(71,round(1.2*line)));k+=1-k%2 + background=cv2.morphologyEx(rgb,cv2.MORPH_CLOSE,cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(k,k))).astype(np.float32) + optical=np.maximum(0,np.log((background+2)/(rgb.astype(np.float32)+2)));depth=optical.mean(2);chroma=optical-depth[:,:,None];strength=np.linalg.norm(chroma,axis=2);relative=strength/(depth+.05) + direction=optical/(optical.sum(2)[:,:,None]+1e-5);valid=depth>.30;points=direction[valid].astype(np.float32) + if len(points)<100:return np.zeros((H,W),bool),np.zeros((H,W),bool),cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY),dict(reliable=False) + cv2.setRNGSeed(1729);sample=points[::max(1,len(points)//30000)].copy();K=4;_,_,centers=cv2.kmeans(sample,K,None,(cv2.TERM_CRITERIA_EPS+cv2.TERM_CRITERIA_MAX_ITER,60,.0002),3,cv2.KMEANS_PP_CENTERS) + distances=np.stack([np.linalg.norm(direction-c,axis=2)for c in centers]);assigned=distances.argmin(0);scores=[];clean=[];traces=[] + for i in range(K): + # Include neighbouring optical directions belonging to the same pen colour. + compatible=np.linalg.norm(centers-centers[i],axis=1)<.035 + mask=(depth>.22)&compatible[assigned];mask,rules=remove_rules(mask,line);lab,cc=components(mask,max(3,round(.004*line*line))) + good=[c for c in cc if c['h']<3*line and c['w']<.85*W and c['area']<.04*W*H and not(c['w']<.3*line and c['h']>.9*line and(c['x']<.04*W or c['x']+c['w']>.96*W))] + mask=np.isin(lab,[c['label']for c in good]);long=[c for c in good if c['w']>=1.5*c['h']and c['h']>=.15*line and c['w']>=.5*line] + bins_y=len(set(int(c['cy']/max(1,2*line))for c in long));mass=sum(c['area']for c in long);score=mass*min(1,bins_y/8)*max(.008,float(np.linalg.norm(centers[i]-np.ones(3)/3))) + scores.append(score);clean.append(mask);traces.append(dict(class_id=i,center=centers[i].tolist(),pixels=int(mask.sum()),long_components=len(long),vertical_bins=bins_y,score=score)) + best=int(np.argmax(scores));pen=clean[best];lab,cc=components(pen,2);span=max((c['y']+c['h']for c in cc),default=0)-min((c['y']for c in cc),default=0) + other=[scores[j]for j in range(K)if np.linalg.norm(centers[j]-centers[best])>=.04];secondary=max(other,default=0) + reliable=traces[best]['long_components']>=8 and traces[best]['vertical_bins']>=5 and secondary>max(.5,.015*scores[best]) + neutral=centers[int(np.argmin(np.linalg.norm(centers-np.ones(3)/3,axis=1)))];radius=max(.025,.65*float(np.linalg.norm(centers[best]-neutral)));compatible=np.linalg.norm(centers-centers[best],axis=1).16,line);pen=(depth>.07)&compatible[assigned]&~all_rules;labels,allcc=components(pen,2) + ids=[c['label']for c in allcc if c['h']<3*line and c['w']<.85*W and not(c['w']<.3*line and c['h']>.9*line and(c['x']<.04*W or c['x']+c['w']>.96*W))];pen=np.isin(labels,ids) + printed=(depth>.22)&~(cv2.dilate(pen.astype(np.uint8),np.ones((3,3),np.uint8))>0);printed,_=remove_rules(printed,line) + dilated=cv2.dilate(pen.astype(np.uint8),np.ones((3,3),np.uint8))>0 + gray=cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY);bg=cv2.cvtColor(background.clip(0,255).astype(np.uint8),cv2.COLOR_RGB2GRAY);print_gray=gray.copy();print_gray[dilated]=bg[dilated] + trace_core=(depth>.22)&compatible[assigned]&~all_rules + return pen,printed,print_gray,dict(_core_mask=trace_core,reliable=bool(reliable),estimated_line_height=line,selected_class=best,classes=traces,pen_components=len(cc),pen_pixels=int(pen.sum()),color_span_y=span,background_kernel=k,no_gold_or_template_used=True) + +def build_rows(mask,proposals,params,regions=None,core=None): + from scipy.ndimage import gaussian_filter1d + from scipy.signal import find_peaks + H,W=mask.shape;line=estimate_height(proposals,H);cols=regions or columns(mask,line,proposals);rows=[];row_trace=[] + for column,(left,right)in enumerate(cols): + m=mask[:,left:right];cm=(core if core is not None else mask)[:,left:right];profile=cm.sum(1).astype(float);smooth=gaussian_filter1d(profile,max(1,.08*line));corr=np.correlate(smooth-smooth.mean(),smooth-smooth.mean(),mode='full')[H-1:];lo=max(5,round(.5*line));hi=min(H//3,round(3.5*line));period=line*1.7 + if hi>lo: + peaks,_=find_peaks(corr[lo:hi]);valid=[lo+int(k)for k in peaks if corr[lo+k]>.12*corr[0]] + if valid:period=min(valid) + period=line;peaks,_=find_peaks(smooth,distance=max(3,round(.65*line)),prominence=max(.8,.025*smooth.max())) + peaks=[int(y)for y in peaks if smooth[y]>=max(1,.025*smooth.max())] + for j,peak in enumerate(peaks): + low=0 if not j else peaks[j-1]+int(np.argmin(smooth[peaks[j-1]:peak+1]));high=H if j+1==len(peaks)else peak+int(np.argmin(smooth[peak:peaks[j+1]+1])) + # Limit isolated/noisy bands, retaining full connected strokes and dots. + low=max(low,round(peak-.8*period));high=min(high,round(peak+.8*period));xs=np.where(m[low:high].any(0))[0] + segments=[] + for x in xs: + if not segments or x-segments[-1][-1]>max(params['max_gap'],8.)*line:segments.append([]) + segments[-1].append(int(x)) + for segment in segments: + x=left+segment[0];X=left+segment[-1]+1;ys,xx=np.where(mask[low:high,x:X]) + if len(ys)=max(4,.05*len(ys))else ys + y=low+int(tight.min());Y=low+int(tight.max())+1;pad=max(1.,.04*line);box=[max(0.,x-pad),max(0.,y-pad),min(float(W),X+pad),min(float(H),Y+pad)];support=[i for i,p in enumerate(proposals)if rect_area(rect_intersection(box,p))>0] + rows.append(dict(box=box,baseline=float(np.quantile(ys,.8)+low),column=column));row_trace.append(dict(source_proposals=support,ink_pixels=len(ys),read_band=[low,high],peak=peak,period=period)) + return rows,dict(estimated_line_height=line,columns=cols,rows=row_trace) + +def build(rgb,proposals,params=None,primary=None): + params=dict(DEFAULT,**(params or {}));pen,printed,print_gray,trace=layer_masks(rgb,proposals,params);core=trace.pop('_core_mask',None) + if not trace['reliable']:return None,dict(layer_mask=trace,reason='No confident separated handwriting and printed layers') + cols=proposal_columns(primary or proposals,rgb.shape[1],rgb.shape[0],cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY));hand,ht=build_rows(pen,proposals,params,core=core,regions=cols);printed_rows,pt=build_rows(printed,proposals,params,regions=ht['columns']) + return dict(handwriting=hand,printed=printed_rows,printed_gray=print_gray,pen_mask=pen,printed_mask=printed),dict(parameters=params,layer_mask=trace,handwriting_rows=ht,printed_rows=pt,source_proposal_count=len(proposals),handwriting_readings=len(hand),printed_readings=len(printed_rows),no_gold_or_template_used=True) diff --git a/stallion/bina04-page-assembly-95k-20261103/CALIBRATION.json b/stallion/bina04-page-assembly-95k-20261103/CALIBRATION.json new file mode 100644 index 0000000000000000000000000000000000000000..7e18e58e0007bb11281a49d2f57eb347524d6b44 --- /dev/null +++ b/stallion/bina04-page-assembly-95k-20261103/CALIBRATION.json @@ -0,0 +1,36 @@ +{ + "candidates": [ + { + "threshold": 0.45, + "CER": 11.502905441098784, + "removed": 3 + }, + { + "threshold": 0.55, + "CER": 11.502905441098784, + "removed": 3 + }, + { + "threshold": 0.65, + "CER": 11.428068321887656, + "removed": 6 + }, + { + "threshold": 0.75, + "CER": 11.428068321887656, + "removed": 6 + }, + { + "threshold": 0.85, + "CER": 11.388448670540589, + "removed": 7 + } + ], + "selected": { + "threshold": 0.85, + "CER": 11.388448670540589, + "removed": 7 + }, + "split_source": "Frozen calib25 only", + "seven_refs_used_for_selection": false +} \ No newline at end of file diff --git a/stallion/bina04-page-assembly-95k-20261103/assembly.py b/stallion/bina04-page-assembly-95k-20261103/assembly.py new file mode 100644 index 0000000000000000000000000000000000000000..b0be28225adc4703b476975395ae114e406f2ef1 --- /dev/null +++ b/stallion/bina04-page-assembly-95k-20261103/assembly.py @@ -0,0 +1,41 @@ +import copy,re +from rapidfuzz.fuzz import ratio,partial_ratio + +def norm(s):return re.sub(r'[\s\u200cـ]+','',s) +def overlap(a,b): + x=max(0,min(a[2],b[2])-max(a[0],b[0]));y=max(0,min(a[3],b[3])-max(a[1],b[1]));aa=max(1,(a[2]-a[0])*(a[3]-a[1]));bb=max(1,(b[2]-b[0])*(b[3]-b[1]));return x*y/min(aa,bb),x/max(1,min(a[2]-a[0],b[2]-b[0])),y/max(1,min(a[3]-a[1],b[3]-b[1])) +def apply(old,threshold=.55,columns=False): + out=copy.deepcopy(old);ls=out['lines'];drop={};eligible=[i for i,l in enumerate(ls)if l.get('text','').strip() and len(l.get('box',[]))==4];conf=lambda i:float(ls[i].get('confidence',0));text=lambda i:norm(ls[i]['text']) + # Real short labels and isolated margin digits are never deleted by this candidate. + protected=lambda i:len(text(i))<=3 or all(c.isdigit()or c in '/:.-،٫'for c in text(i)) + for i in sorted(eligible,key=lambda i:-conf(i)): + if i in drop:continue + for j in eligible: + if j==i or j in drop or conf(j)>conf(i)or protected(j):continue + ov,x,y=overlap(ls[i]['box'],ls[j]['box']);a,b=text(i),text(j) + if ov>=.7 and x>=.65 and y>=.6 and min(len(a),len(b))>=6 and (ratio(a,b)>=80 or partial_ratio(a,b)>=90 and min(len(a),len(b))/max(len(a),len(b))>=.55):drop[j]=dict(reason='overlapping_near_duplicate',better=i) + elif len(b)<=1.4*len(a) and conf(j)=.9 and conf(i)-conf(j)>=.15 and ov>=.8 and x>=.7: + drop[j]=dict(reason='low_confidence_covered_by_stronger_line',better=i) + # Locate actual serialized occurrences. Never regenerate table markdown or whitespace. + spans={};used=[];cursor=0 + for i in eligible: + t=ls[i]['text'];pos=out['text'].find(t,cursor) + if pos<0: + pos=out['text'].find(t) + while pos>=0 and any(posa for a,b in used):pos=out['text'].find(t,pos+1) + if pos>=0:spans[i]=(pos,pos+len(t));used.append(spans[i]);cursor=max(cursor,pos+len(t)) + applied={i:v for i,v in drop.items()if i in spans};s=out['text'] + for i in sorted(applied,key=lambda i:spans[i][0],reverse=True):a,b=spans[i];s=s[:a]+s[b:] + # Keep region serialization in sync: reference scoring reads region nodes. + from collections import Counter + keys=Counter((tuple(ls[i]['box']),ls[i]['text'])for i in applied) + for region in out.get('regions',[]): + kept=[] + for l in region.get('lines',[]): + key=(tuple(l.get('box',[])),l.get('text','')) + if keys[key]>0:keys[key]-=1 + else:kept.append(l) + region['lines']=kept + if not region.get('table_trace',{}).get('grid'):region['text']='\n'.join(l['text']for l in kept) + out['text']=s;out['lines']=[l for i,l in enumerate(ls)if i not in applied];out['page_assembly_candidate']=dict(threshold=threshold,removed=[dict(node_id=i,**v,line=ls[i])for i,v in applied.items()],unmapped_proposals=[i for i in drop if i not in spans],tables_reformatted=False,recognizer_weights_changed=False) + return out diff --git a/stallion/bina04-page-assembly-95k-20261103/production.py b/stallion/bina04-page-assembly-95k-20261103/production.py new file mode 100644 index 0000000000000000000000000000000000000000..a9e7cb08f298a154e3b79d144a925529531fa8b6 --- /dev/null +++ b/stallion/bina04-page-assembly-95k-20261103/production.py @@ -0,0 +1,11 @@ +"""Frozen 95k recognizer plus geometry/text duplicate and covered-junk suppression.""" +import importlib.util,sys,json +from pathlib import Path +R=Path(__file__).resolve().parent +sys.path.insert(0,str(R)) +_s=importlib.util.spec_from_file_location('accepted_page_assembly_transform',R/'assembly.py');_m=importlib.util.module_from_spec(_s);_s.loader.exec_module(_m);apply=_m.apply +class Runtime: + def __init__(self,device='auto'): + s=importlib.util.spec_from_file_location('assembly_frozen_parent','__BINA_ROOT__/stallion/bina04-detector-role-isolation-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);self.threshold=json.loads((R/'CALIBRATION.json').read_text())['selected']['threshold'] + def __call__(self,image,old,kind,name='',**kwargs):return apply(self.parent(image,old,kind,name,**kwargs),self.threshold) + def close(self):self.parent.close() diff --git a/stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json b/stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json new file mode 100644 index 0000000000000000000000000000000000000000..a7b47cb32bc04ac4a868582886e93f619555f723 --- /dev/null +++ b/stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json @@ -0,0 +1,1907 @@ +{ + "blank": 0, + "chars": [ + " ", + "!", + "\"", + "#", + "$", + "%", + "&", + "'", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "?", + "@", + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "[", + "\\", + "]", + "^", + "_", + "`", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "{", + "|", + "}", + "~", + "¡", + "¢", + "£", + "¤", + "¥", + "¦", + "§", + "¨", + "©", + "ª", + "«", + "¬", + "®", + "¯", + "°", + "±", + "²", + "³", + "´", + "µ", + "¶", + "·", + "¸", + "¹", + "º", + "»", + "¼", + "½", + "¾", + "¿", + "À", + "Á", + "Â", + "Ã", + "Ä", + "Å", + "Æ", + "Ç", + "È", + "É", + "Ê", + "Ë", + "Ì", + "Í", + "Î", + "Ï", + "Ð", + "Ñ", + "Ò", + "Ó", + "Ô", + "Õ", + "Ö", + "×", + "Ø", + "Ù", + "Ú", + "Û", + "Ü", + "Ý", + "Þ", + "ß", + "à", + "á", + "â", + "ã", + "ä", + "å", + "æ", + "ç", + "è", + "é", + "ê", + "ë", + "ì", + "í", + "î", + "ï", + "ð", + "ñ", + "ò", + "ó", + "ô", + "õ", + "ö", + "÷", + "ø", + "ù", + "ú", + "û", + "ü", + "ý", + "þ", + "ÿ", + "Ā", + "ā", + "Ă", + "ă", + "Ą", + "ą", + "Ć", + "ć", + "Ĉ", + "ĉ", + "Ċ", + "ċ", + "Č", + "č", + "Ď", + "ď", + "Đ", + "đ", + "Ē", + "ē", + "Ĕ", + "ĕ", + "Ė", + "ė", + "Ę", + "ę", + "Ě", + "ě", + "Ĝ", + "ĝ", + "Ğ", + "ğ", + "Ġ", + "ġ", + "Ģ", + "ģ", + "Ĥ", + "ĥ", + "Ħ", + "ħ", + "Ĩ", + "ĩ", + "Ī", + "ī", + "Ĭ", + "ĭ", + "Į", + "į", + "İ", + "ı", + "IJ", + "ij", + "Ĵ", + "ĵ", + "Ķ", + "ķ", + "ĸ", + "Ĺ", + "ĺ", + "Ļ", + "ļ", + "Ľ", + "ľ", + "Ŀ", + "ŀ", + "Ł", + "ł", + "Ń", + "ń", + "Ņ", + "ņ", + "Ň", + "ň", + "ʼn", + "Ŋ", + "ŋ", + "Ō", + "ō", + "Ŏ", + "ŏ", + "Ő", + "ő", + "Œ", + "œ", + "Ŕ", + "ŕ", + "Ŗ", + "ŗ", + "Ř", + "ř", + "Ś", + "ś", + "Ŝ", + "ŝ", + "Ş", + "ş", + "Š", + "š", + "Ţ", + "ţ", + "Ť", + "ť", + "Ŧ", + "ŧ", + "Ũ", + "ũ", + "Ū", + "ū", + "Ŭ", + "ŭ", + "Ů", + "ů", + "Ű", + "ű", + "Ų", + "ų", + "Ŵ", + "ŵ", + "Ŷ", + "ŷ", + "Ÿ", + "Ź", + "ź", + "Ż", + "ż", + "Ž", + "ž", + "ſ", + "ƀ", + "Ɓ", + "Ƃ", + "ƃ", + "Ƅ", + "ƅ", + "Ɔ", + "Ƈ", + "ƈ", + "Ɖ", + "Ɗ", + "Ƌ", + "ƌ", + "ƍ", + "Ǝ", + "Ə", + "Ɛ", + "Ƒ", + "ƒ", + "Ɠ", + "Ɣ", + "ƕ", + "Ɩ", + "Ɨ", + "Ƙ", + "ƙ", + "ƚ", + "ƛ", + "Ɯ", + "Ɲ", + "ƞ", + "Ɵ", + "Ơ", + "ơ", + "Ƣ", + "ƣ", + "Ƥ", + "ƥ", + "Ʀ", + "Ƨ", + "ƨ", + "Ʃ", + "ƪ", + "ƫ", + "Ƭ", + "ƭ", + "Ʈ", + "Ư", + "ư", + "Ʊ", + "Ʋ", + "Ƴ", + "ƴ", + "Ƶ", + "ƶ", + "Ʒ", + "Ƹ", + "ƹ", + "ƺ", + "ƻ", + "Ƽ", + "ƽ", + "ƾ", + "ƿ", + "ǀ", + "ǁ", + "ǂ", + "ǃ", + "DŽ", + "Dž", + "dž", + "LJ", + "Lj", + "lj", + "NJ", + "Nj", + "nj", + "Ǎ", + "ǎ", + "Ǐ", + "ǐ", + "Ǒ", + "ǒ", + "Ǔ", + "ǔ", + "Ǖ", + "ǖ", + "Ǘ", + "ǘ", + "Ǚ", + "ǚ", + "Ǜ", + "ǜ", + "ǝ", + "Ǟ", + "ǟ", + "Ǡ", + "ǡ", + "Ǣ", + "ǣ", + "Ǥ", + "ǥ", + "Ǧ", + "ǧ", + "Ǩ", + "ǩ", + "Ǫ", + "ǫ", + "Ǭ", + "ǭ", + "Ǯ", + "ǯ", + "ǰ", + "DZ", + "Dz", + "dz", + "Ǵ", + "ǵ", + "Ƕ", + "Ƿ", + "Ǹ", + "ǹ", + "Ǻ", + "ǻ", + "Ǽ", + "ǽ", + "Ǿ", + "ǿ", + "Ȁ", + "ȁ", + "Ȃ", + "ȃ", + "Ȅ", + "ȅ", + "Ȇ", + "ȇ", + "Ȉ", + "ȉ", + "Ȋ", + "ȋ", + "Ȍ", + "ȍ", + "Ȏ", + "ȏ", + "Ȑ", + "ȑ", + "Ȓ", + "ȓ", + "Ȕ", + "ȕ", + "Ȗ", + "ȗ", + "Ș", + "ș", + "Ț", + "ț", + "Ȝ", + "ȝ", + "Ȟ", + "ȟ", + "Ƞ", + "ȡ", + "Ȣ", + "ȣ", + "Ȥ", + "ȥ", + "Ȧ", + "ȧ", + "Ȩ", + "ȩ", + "Ȫ", + "ȫ", + "Ȭ", + "ȭ", + "Ȯ", + "ȯ", + "Ȱ", + "ȱ", + "Ȳ", + "ȳ", + "ȴ", + "ȵ", + "ȶ", + "ȷ", + "ȸ", + "ȹ", + "Ⱥ", + "Ȼ", + "ȼ", + "Ƚ", + "Ⱦ", + "ȿ", + "ɀ", + "Ɂ", + "ɂ", + "Ƀ", + "Ʉ", + "Ʌ", + "Ɇ", + "ɇ", + "Ɉ", + "ɉ", + "Ɋ", + "ɋ", + "Ɍ", + "ɍ", + "Ɏ", + "ɏ", + "Ḁ", + "ḁ", + "Ḃ", + "ḃ", + "Ḅ", + "ḅ", + "Ḇ", + "ḇ", + "Ḉ", + "ḉ", + "Ḋ", + "ḋ", + "Ḍ", + "ḍ", + "Ḏ", + "ḏ", + "Ḑ", + "ḑ", + "Ḓ", + "ḓ", + "Ḕ", + "ḕ", + "Ḗ", + "ḗ", + "Ḙ", + "ḙ", + "Ḛ", + "ḛ", + "Ḝ", + "ḝ", + "Ḟ", + "ḟ", + "Ḡ", + "ḡ", + "Ḣ", + "ḣ", + "Ḥ", + "ḥ", + "Ḧ", + "ḧ", + "Ḩ", + "ḩ", + "Ḫ", + "ḫ", + "Ḭ", + "ḭ", + "Ḯ", + "ḯ", + "Ḱ", + "ḱ", + "Ḳ", + "ḳ", + "Ḵ", + "ḵ", + "Ḷ", + "ḷ", + "Ḹ", + "ḹ", + "Ḻ", + "ḻ", + "Ḽ", + "ḽ", + "Ḿ", + "ḿ", + "Ṁ", + "ṁ", + "Ṃ", + "ṃ", + "Ṅ", + "ṅ", + "Ṇ", + "ṇ", + "Ṉ", + "ṉ", + "Ṋ", + "ṋ", + "Ṍ", + "ṍ", + "Ṏ", + "ṏ", + "Ṑ", + "ṑ", + "Ṓ", + "ṓ", + "Ṕ", + "ṕ", + "Ṗ", + "ṗ", + "Ṙ", + "ṙ", + "Ṛ", + "ṛ", + "Ṝ", + "ṝ", + "Ṟ", + "ṟ", + "Ṡ", + "ṡ", + "Ṣ", + "ṣ", + "Ṥ", + "ṥ", + "Ṧ", + "ṧ", + "Ṩ", + "ṩ", + "Ṫ", + "ṫ", + "Ṭ", + "ṭ", + "Ṯ", + "ṯ", + "Ṱ", + "ṱ", + "Ṳ", + "ṳ", + "Ṵ", + "ṵ", + "Ṷ", + "ṷ", + "Ṹ", + "ṹ", + "Ṻ", + "ṻ", + "Ṽ", + "ṽ", + "Ṿ", + "ṿ", + "Ẁ", + "ẁ", + "Ẃ", + "ẃ", + "Ẅ", + "ẅ", + "Ẇ", + "ẇ", + "Ẉ", + "ẉ", + "Ẋ", + "ẋ", + "Ẍ", + "ẍ", + "Ẏ", + "ẏ", + "Ẑ", + "ẑ", + "Ẓ", + "ẓ", + "Ẕ", + "ẕ", + "ẖ", + "ẗ", + "ẘ", + "ẙ", + "ẚ", + "ẛ", + "ẜ", + "ẝ", + "ẞ", + "ẟ", + "Ạ", + "ạ", + "Ả", + "ả", + "Ấ", + "ấ", + "Ầ", + "ầ", + "Ẩ", + "ẩ", + "Ẫ", + "ẫ", + "Ậ", + "ậ", + "Ắ", + "ắ", + "Ằ", + "ằ", + "Ẳ", + "ẳ", + "Ẵ", + "ẵ", + "Ặ", + "ặ", + "Ẹ", + "ẹ", + "Ẻ", + "ẻ", + "Ẽ", + "ẽ", + "Ế", + "ế", + "Ề", + "ề", + "Ể", + "ể", + "Ễ", + "ễ", + "Ệ", + "ệ", + "Ỉ", + "ỉ", + "Ị", + "ị", + "Ọ", + "ọ", + "Ỏ", + "ỏ", + "Ố", + "ố", + "Ồ", + "ồ", + "Ổ", + "ổ", + "Ỗ", + "ỗ", + "Ộ", + "ộ", + "Ớ", + "ớ", + "Ờ", + "ờ", + "Ở", + "ở", + "Ỡ", + "ỡ", + "Ợ", + "ợ", + "Ụ", + "ụ", + "Ủ", + "ủ", + "Ứ", + "ứ", + "Ừ", + "ừ", + "Ử", + "ử", + "Ữ", + "ữ", + "Ự", + "ự", + "Ỳ", + "ỳ", + "Ỵ", + "ỵ", + "Ỷ", + "ỷ", + "Ỹ", + "ỹ", + "Ỻ", + "ỻ", + "Ỽ", + "ỽ", + "Ỿ", + "ỿ", + "Α", + "Β", + "Γ", + "Δ", + "Ε", + "Ζ", + "Η", + "Θ", + "Ι", + "Κ", + "Λ", + "Μ", + "Ν", + "Ξ", + "Ο", + "Π", + "Ρ", + "Σ", + "Τ", + "Υ", + "Φ", + "Χ", + "Ψ", + "Ω", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "ς", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "‐", + "‑", + "‒", + "–", + "—", + "―", + "‖", + "‘", + "’", + "‚", + "‛", + "“", + "”", + "„", + "‟", + "†", + "‡", + "•", + "‣", + "…", + "‰", + "′", + "″", + "‹", + "›", + "⁄", + "※", + "⁰", + "⁴", + "⁵", + "⁶", + "⁷", + "⁸", + "⁹", + "⁺", + "⁻", + "⁽", + "⁾", + "ⁿ", + "₀", + "₁", + "₂", + "₃", + "₄", + "₅", + "₆", + "₇", + "₈", + "₉", + "€", + "₹", + "₽", + "₩", + "₺", + "₪", + "₱", + "₫", + "₴", + "₦", + "₡", + "₸", + "™", + "℃", + "℉", + "№", + "ℓ", + "℮", + "⅓", + "⅔", + "⅛", + "←", + "↑", + "→", + "↓", + "↔", + "⇒", + "⇔", + "↵", + "−", + "∓", + "∗", + "∘", + "∙", + "√", + "∞", + "∝", + "∠", + "∧", + "∨", + "∩", + "∪", + "∫", + "≈", + "≠", + "≡", + "≤", + "≥", + "⊂", + "⊃", + "⊆", + "⊇", + "⊕", + "⊗", + "⊥", + "⋅", + "∂", + "∆", + "∇", + "∈", + "∉", + "∋", + "∏", + "∑", + "∀", + "∃", + "∅", + "■", + "□", + "▪", + "▫", + "▲", + "△", + "▶", + "►", + "▼", + "▽", + "◀", + "◄", + "◆", + "◇", + "○", + "◦", + "●", + "★", + "☆", + "✓", + "✔", + "✗", + "✘", + "❖", + "➢", + "➤", + "؆", + "؇", + "؈", + "؉", + "؊", + "؋", + "،", + "؍", + "؎", + "؏", + "ؐ", + "ؑ", + "ؒ", + "ؓ", + "ؔ", + "ؕ", + "ؖ", + "ؗ", + "ؘ", + "ؙ", + "ؚ", + "؛", + "؝", + "؞", + "؟", + "ؠ", + "ء", + "آ", + "أ", + "ؤ", + "إ", + "ئ", + "ا", + "ب", + "ة", + "ت", + "ث", + "ج", + "ح", + "خ", + "د", + "ذ", + "ر", + "ز", + "س", + "ش", + "ص", + "ض", + "ط", + "ظ", + "ع", + "غ", + "ػ", + "ؼ", + "ؽ", + "ؾ", + "ؿ", + "ف", + "ق", + "ك", + "ل", + "م", + "ن", + "ه", + "و", + "ى", + "ي", + "ً", + "ٌ", + "ٍ", + "َ", + "ُ", + "ِ", + "ّ", + "ْ", + "ٓ", + "ٔ", + "ٕ", + "ٖ", + "ٗ", + "٘", + "ٙ", + "ٚ", + "ٛ", + "ٜ", + "ٝ", + "ٞ", + "ٟ", + "٠", + "١", + "٢", + "٣", + "٤", + "٥", + "٦", + "٧", + "٨", + "٩", + "٪", + "٫", + "٬", + "٭", + "ٮ", + "ٯ", + "ٰ", + "ٱ", + "ٲ", + "ٳ", + "ٴ", + "ٵ", + "ٶ", + "ٷ", + "ٸ", + "ٹ", + "ٺ", + "ٻ", + "ټ", + "ٽ", + "پ", + "ٿ", + "ڀ", + "ځ", + "ڂ", + "ڃ", + "ڄ", + "څ", + "چ", + "ڇ", + "ڈ", + "ډ", + "ڊ", + "ڋ", + "ڌ", + "ڍ", + "ڎ", + "ڏ", + "ڐ", + "ڑ", + "ڒ", + "ړ", + "ڔ", + "ڕ", + "ږ", + "ڗ", + "ژ", + "ڙ", + "ښ", + "ڛ", + "ڜ", + "ڝ", + "ڞ", + "ڟ", + "ڠ", + "ڡ", + "ڢ", + "ڣ", + "ڤ", + "ڥ", + "ڦ", + "ڧ", + "ڨ", + "ک", + "ڪ", + "ګ", + "ڬ", + "ڭ", + "ڮ", + "گ", + "ڰ", + "ڱ", + "ڲ", + "ڳ", + "ڴ", + "ڵ", + "ڶ", + "ڷ", + "ڸ", + "ڹ", + "ں", + "ڻ", + "ڼ", + "ڽ", + "ھ", + "ڿ", + "ۀ", + "ہ", + "ۂ", + "ۃ", + "ۄ", + "ۅ", + "ۆ", + "ۇ", + "ۈ", + "ۉ", + "ۊ", + "ۋ", + "ی", + "ۍ", + "ێ", + "ۏ", + "ې", + "ۑ", + "ے", + "ۓ", + "۔", + "ە", + "ۖ", + "ۗ", + "ۘ", + "ۙ", + "ۚ", + "ۛ", + "ۜ", + "۞", + "۟", + "۠", + "ۡ", + "ۢ", + "ۣ", + "ۤ", + "ۥ", + "ۦ", + "ۧ", + "ۨ", + "۩", + "۪", + "۫", + "۬", + "ۭ", + "ۮ", + "ۯ", + "۰", + "۱", + "۲", + "۳", + "۴", + "۵", + "۶", + "۷", + "۸", + "۹", + "ۺ", + "ۻ", + "ۼ", + "۽", + "۾", + "ۿ", + "ݐ", + "ݑ", + "ݒ", + "ݓ", + "ݔ", + "ݕ", + "ݖ", + "ݗ", + "ݘ", + "ݙ", + "ݚ", + "ݛ", + "ݜ", + "ݝ", + "ݞ", + "ݟ", + "ݠ", + "ݡ", + "ݢ", + "ݣ", + "ݤ", + "ݥ", + "ݦ", + "ݧ", + "ݨ", + "ݩ", + "ݪ", + "ݫ", + "ݬ", + "ݭ", + "ݮ", + "ݯ", + "ݰ", + "ݱ", + "ݲ", + "ݳ", + "ݴ", + "ݵ", + "ݶ", + "ݷ", + "ݸ", + "ݹ", + "ݺ", + "ݻ", + "ݼ", + "ݽ", + "ݾ", + "ݿ", + "֑", + "֒", + "֓", + "֔", + "֕", + "֖", + "֗", + "֘", + "֙", + "֚", + "֛", + "֜", + "֝", + "֞", + "֟", + "֠", + "֡", + "֢", + "֣", + "֤", + "֥", + "֦", + "֧", + "֨", + "֩", + "֪", + "֫", + "֬", + "֭", + "֮", + "֯", + "ְ", + "ֱ", + "ֲ", + "ֳ", + "ִ", + "ֵ", + "ֶ", + "ַ", + "ָ", + "ֹ", + "ֺ", + "ֻ", + "ּ", + "ֽ", + "־", + "ֿ", + "׀", + "ׁ", + "ׂ", + "׃", + "ׄ", + "ׅ", + "׆", + "ׇ", + "א", + "ב", + "ג", + "ד", + "ה", + "ו", + "ז", + "ח", + "ט", + "י", + "ך", + "כ", + "ל", + "ם", + "מ", + "ן", + "נ", + "ס", + "ע", + "ף", + "פ", + "ץ", + "צ", + "ק", + "ר", + "ש", + "ת", + "ׯ", + "װ", + "ױ", + "ײ", + "׳", + "״", + "‌", + "”", + "", + "­", + "ɑ", + "ɒ", + "ə", + "ɛ", + "ʃ", + "ʒ", + "ʔ", + "ʿ", + "˝", + "ˣ", + "́", + "ί", + "ϵ", + "А", + "В", + "Г", + "Д", + "И", + "К", + "М", + "Н", + "О", + "П", + "Р", + "С", + "Т", + "У", + "Ш", + "Э", + "а", + "б", + "в", + "г", + "д", + "е", + "ж", + "з", + "и", + "й", + "к", + "л", + "м", + "н", + "о", + "п", + "р", + "с", + "т", + "у", + "ф", + "х", + "ц", + "ч", + "ш", + "ъ", + "ы", + "ь", + "э", + "ю", + "я", + "ё", + "ғ", + "қ", + "ҷ", + "ӣ", + "ӯ", + "Հ", + "Ո", + "Ս", + "ա", + "գ", + "ե", + "զ", + "է", + "ի", + "կ", + "հ", + "յ", + "ն", + "ո", + "ս", + "վ", + "տ", + "ր", + "ւ", + "ք", + "ـ", + "ᵀ", + "ᵗ", + "ᵢ", + "​", + "‍", + "‎", + "‏", + "‫", + "‬", + "‭", + "‮", + "", + "ⁱ", + "ₖ", + "ₘ", + "ₙ", + "ₜ", + "ℝ", + "∥", + "━", + "┏", + "┓", + "┻", + "◽", + "☐", + "♀", + "➜", + "ⱼ", + "、", + "。", + "《", + "》", + "う", + "お", + "き", + "こ", + "し", + "じ", + "そ", + "た", + "だ", + "っ", + "て", + "と", + "な", + "の", + "ば", + "ひ", + "ま", + "め", + "や", + "わ", + "ア", + "ィ", + "ウ", + "カ", + "キ", + "グ", + "サ", + "ス", + "ヂ", + "ツ", + "デ", + "ト", + "ビ", + "ブ", + "ペ", + "リ", + "ル", + "ン", + "下", + "丸", + "丹", + "主", + "事", + "京", + "仮", + "八", + "公", + "典", + "内", + "北", + "南", + "博", + "口", + "古", + "名", + "向", + "和", + "園", + "坂", + "城", + "場", + "塩", + "外", + "大", + "奈", + "子", + "家", + "居", + "山", + "岳", + "嵐", + "川", + "巻", + "座", + "庭", + "志", + "戸", + "手", + "摩", + "日", + "明", + "本", + "条", + "東", + "桜", + "梅", + "楠", + "模", + "檜", + "毬", + "沢", + "浅", + "濱", + "物", + "王", + "田", + "登", + "白", + "皇", + "盟", + "相", + "県", + "神", + "競", + "紫", + "美", + "脈", + "花", + "苑", + "草", + "蒲", + "藤", + "蘇", + "蛇", + "術", + "見", + "話", + "赤", + "連", + "部", + "都", + "里", + "野", + "題", + "館", + "馬", + "駅", + "", + "", + "", + "️", + "ﺯ", + "（", + "）", + "＊", + "０", + "７", + "：", + "", + "ʾ", + "̄", + "𐬀", + "𐬌", + "𐬛", + "𐬨", + "𐬰" + ], + "old_chars_preserved": 1356, + "appended_chars": [ + "”", + "", + "­", + "ɑ", + "ɒ", + "ə", + "ɛ", + "ʃ", + "ʒ", + "ʔ", + "ʿ", + "˝", + "ˣ", + "́", + "ί", + "ϵ", + "А", + "В", + "Г", + "Д", + "И", + "К", + "М", + "Н", + "О", + "П", + "Р", + "С", + "Т", + "У", + "Ш", + "Э", + "а", + "б", + "в", + "г", + "д", + "е", + "ж", + "з", + "и", + "й", + "к", + "л", + "м", + "н", + "о", + "п", + "р", + "с", + "т", + "у", + "ф", + "х", + "ц", + "ч", + "ш", + "ъ", + "ы", + "ь", + "э", + "ю", + "я", + "ё", + "ғ", + "қ", + "ҷ", + "ӣ", + "ӯ", + "Հ", + "Ո", + "Ս", + "ա", + "գ", + "ե", + "զ", + "է", + "ի", + "կ", + "հ", + "յ", + "ն", + "ո", + "ս", + "վ", + "տ", + "ր", + "ւ", + "ք", + "ـ", + "ᵀ", + "ᵗ", + "ᵢ", + "​", + "‍", + "‎", + "‏", + "‫", + "‬", + "‭", + "‮", + "", + "ⁱ", + "ₖ", + "ₘ", + "ₙ", + "ₜ", + "ℝ", + "∥", + "━", + "┏", + "┓", + "┻", + "◽", + "☐", + "♀", + "➜", + "ⱼ", + "、", + "。", + "《", + "》", + "う", + "お", + "き", + "こ", + "し", + "じ", + "そ", + "た", + "だ", + "っ", + "て", + "と", + "な", + "の", + "ば", + "ひ", + "ま", + "め", + "や", + "わ", + "ア", + "ィ", + "ウ", + "カ", + "キ", + "グ", + "サ", + "ス", + "ヂ", + "ツ", + "デ", + "ト", + "ビ", + "ブ", + "ペ", + "リ", + "ル", + "ン", + "下", + "丸", + "丹", + "主", + "事", + "京", + "仮", + "八", + "公", + "典", + "内", + "北", + "南", + "博", + "口", + "古", + "名", + "向", + "和", + "園", + "坂", + "城", + "場", + "塩", + "外", + "大", + "奈", + "子", + "家", + "居", + "山", + "岳", + "嵐", + "川", + "巻", + "座", + "庭", + "志", + "戸", + "手", + "摩", + "日", + "明", + "本", + "条", + "東", + "桜", + "梅", + "楠", + "模", + "檜", + "毬", + "沢", + "浅", + "濱", + "物", + "王", + "田", + "登", + "白", + "皇", + "盟", + "相", + "県", + "神", + "競", + "紫", + "美", + "脈", + "花", + "苑", + "草", + "蒲", + "藤", + "蘇", + "蛇", + "術", + "見", + "話", + "赤", + "連", + "部", + "都", + "里", + "野", + "題", + "館", + "馬", + "駅", + "", + "", + "", + "️", + "ﺯ", + "（", + "）", + "＊", + "０", + "７", + "：", + "" + ], + "dev_lines_with_unseen_training_chars": 0, + "target_policy": "raw source retained; NFC then invertible rtl.flip(base_rtl=True); no silent character removal", + "append_only_new_chars": [ + "ʾ", + "̄", + "𐬀", + "𐬌", + "𐬛", + "𐬨", + "𐬰" + ], + "space_ID": 1, + "ZWNJ_ID": 1356, + "source_vocabulary_sha256": "2d8b1a087e8b8b56a6ffb0294b022e1b2970a91dca4065c1b1dcc22f16f7c47f" +} diff --git a/stallion/bina04-pagepath-v3-round3-clean-20261031/fusion_v3.py b/stallion/bina04-pagepath-v3-round3-clean-20261031/fusion_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..e16f15bb9ac43cb4045ffda97ea8e32c664f11b3 --- /dev/null +++ b/stallion/bina04-pagepath-v3-round3-clean-20261031/fusion_v3.py @@ -0,0 +1,62 @@ +"""Fuse detector observations by geometry and observed pixels, never by layout/GT.""" +import math +import cv2,numpy as np +cv2.setNumThreads(2) +def area(b):return max(0,b[2]-b[0])*max(0,b[3]-b[1]) +def inter(a,b):return[max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])] +def ink_coverage(gray,b,others,foreground): + H,W=gray.shape;h=b[3]-b[1];x,y,X,Y=map(int,[math.floor(b[0]-.3*h),math.floor(b[1]-.2*h),math.ceil(b[2]+.3*h),math.ceil(b[3]+.2*h)]);x=max(0,x);y=max(0,y);X=min(W,X);Y=min(H,Y) + if X<=x or Y<=y:return 0. + ink=foreground[y:Y,x:X];n=int(ink.sum()) + if not n:return 0. + mask=np.zeros(ink.shape,bool) + for q in others: + # These are the actual input extents used by the unchanged 64px reader. + h=q[3]-q[1];a,c,A,C=inter([x,y,X,Y],[q[0]-.3*h,q[1]-.2*h,q[2]+.3*h,q[3]+.2*h]);a=max(x,math.floor(a));c=max(y,math.floor(c));A=min(X,math.ceil(A));C=min(Y,math.ceil(C)) + if A>a and C>c:mask[c-y:C-y,a-x:A-x]=True + return float((mask&ink).sum())/n + +def fuse(gray,coarse,contexts): + mode=cv2.THRESH_BINARY if np.median(gray)<128 else cv2.THRESH_BINARY_INV + _,global_ink=cv2.threshold(gray,0,255,mode|cv2.THRESH_OTSU);adaptive=cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,mode,51,3);foreground=(global_ink>0)|(adaptive>0) + suppressed=[];retained=[] + for i,b in enumerate(coarse): + coverage=ink_coverage(gray,b,contexts,foreground) + if coverage>=1.:suppressed.append(dict(coarse_index=i,box=b,reason='All foreground in the original reader crop is represented by aspect-correct reader crops',ink_coverage=coverage)) + else:retained.append(b) + all_boxes=list(contexts)+retained;merged=[];sources=[] + for i,box in enumerate(all_boxes): + hit=[] + for j,b in enumerate(merged): + ih=max(0,min(b[3],box[3])-max(b[1],box[1]));bh=b[3]-b[1];qh=box[3]-box[1] + if ih>=.65*min(bh,qh)and max(bh,qh)<=1.6*min(bh,qh)and abs((b[1]+b[3]-box[1]-box[3])/2)<=.35*max(bh,qh)and min(b[2],box[2])-max(b[0],box[0])>1:hit.append(j) + if not hit:merged.append(list(box));sources.append([i]);continue + j=hit[0];members=[box]+[merged[k]for k in hit];merged[j]=[min(v[0]for v in members),min(v[1]for v in members),max(v[2]for v in members),max(v[3]for v in members)];sources[j]+=[i] + for k in reversed(hit[1:]):sources[j]+=sources.pop(k);merged.pop(k) + removed={};multiline=set() + for i,b in enumerate(merged): + H=b[3]-b[1];children=[j for j,q in enumerate(merged)if j!=i and q[3]-q[1]<=.6*H and area(inter(q,b))>=.9*area(q)] + if len(children)<2:continue + centers=[(merged[j][1]+merged[j][3])/2 for j in children] + if max(centers)-min(centers)<.45*H:continue + multiline.add(i);coverage=ink_coverage(gray,b,[merged[j]for j in children],foreground) + if coverage>=1.:removed[i]=dict(reason='Merged observation replaced by independently detected physical rows',replacement_indices=children,ink_coverage=coverage) + for i,b in enumerate(merged): + if i in removed:continue + for j,q in enumerate(merged): + if i==j or j in removed or j in multiline:continue + # Tiny accent/fragment observations already inside a full line must not + # be read a second time. Full-height rows are never removed by this rule. + if b[3]-b[1]<.25*(q[3]-q[1])and area(inter(b,q))>=.98*max(1,area(b)): + covered=ink_coverage(gray,b,[q],foreground) + if covered>=1.:removed[i]=dict(reason='Tiny nested duplicate already inside a physical line',replacement_indices=[j],ink_coverage=covered);break + keep=[i for i in range(len(merged))if i not in removed] + final=[merged[i]for i in keep];H,W=gray.shape;covered=np.zeros((H,W),bool);reinserted=[] + def paint(box): + h=box[3]-box[1];x,y,X,Y=map(int,[math.floor(box[0]-.3*h),math.floor(box[1]-.2*h),math.ceil(box[2]+.3*h),math.ceil(box[3]+.2*h)]);x=max(0,x);y=max(0,y);X=min(W,X);Y=min(H,Y);return x,y,X,Y + for b in final: + x,y,X,Y=paint(b);covered[y:Y,x:X]=True + for i,b in enumerate(list(coarse)+list(contexts)): + x,y,X,Y=paint(b);missing=int((foreground[y:Y,x:X]&~covered[y:Y,x:X]).sum()) + if missing:final.append(list(b));covered[y:Y,x:X]=True;reinserted.append(dict(observation=i,foreground_pixels_restored=missing)) + return final,dict(fusion_version=3,coarse_duplicates=suppressed,all_observations=all_boxes,merged_source_indices=sources,removed_merged_observations=removed,kept_merged_indices=keep,reinserted_for_pixel_preservation=reinserted,layout_used=False,reference_text_or_boxes_used=False,observed_ink_preservation_threshold=1.,preservation_scope='Shared global-Otsu plus adaptive foreground, in ACTUAL padded reader extents; every original observation is represented') diff --git a/stallion/bina04-pagepath-v3-round3-clean-20261031/input_policy_v3.py b/stallion/bina04-pagepath-v3-round3-clean-20261031/input_policy_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..b936fb87e708b2a5b831edc1e2ce60f95d7dd748 --- /dev/null +++ b/stallion/bina04-pagepath-v3-round3-clean-20261031/input_policy_v3.py @@ -0,0 +1,96 @@ +"""Frozen rule C estimator and same-line sliver consolidation; image geometry only.""" +import numpy as np + +def input_kind(width,height,requested='auto'): + if requested not in {'auto','crop','page'}:raise ValueError('input_type must be auto/crop/page') + if requested!='auto':return requested + return 'crop' if min(width,height)<=384 or height<=128 or(height<=768 and width/max(1,height)>=2.5) else 'page' + + +def estimated_line_height(gray): + # Exact estimator from the previously frozen rule C. No labels/categories. + import cv2 + H, W = gray.shape + _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + ink = (binary == 0).astype(np.uint8) + if ink.mean() > .5: + ink = 1 - ink + _, labels, stats, _ = cv2.connectedComponentsWithStats(ink, 8) + heights = [];glyph_ids=[] + for label,(x, y, w, h, area) in enumerate(stats[1:],1): + aspect = float(w) / max(1, int(h)) + if area >= 6 and h >= 3 and w >= 2 and h <= .95 * H and .05 <= aspect <= 20.: + heights.append(int(h));glyph_ids.append(label) + glyph = float(np.percentile(heights, 75)) if heights else None + line = min(float(H), max(1., 1.5 * glyph)) if glyph is not None else float(H) + original_line=line + projection_glyph=glyph + noise_robust=False + projection_ids=glyph_ids + if H<=96 and glyph_ids: + selected=stats[glyph_ids];order=np.argsort(selected[:,3]);ordered=selected[order] + weighted=float(ordered[np.searchsorted(np.cumsum(ordered[:,4]),.75*ordered[:,4].sum()),3]) + if weighted>=2.*max(1.,glyph): + # Speckle can outnumber letters. Weight height by ink area and + # remove tiny speckles from this routing-only projection. + noise_robust=True;projection_glyph=weighted + projection_ids=[i for i in glyph_ids if stats[i,4]>=max(6,.05*selected[:,4].max())] + line=max(line,min(float(H),1.5*weighted)) + # Padded one-line strips have small glyphs inside a much taller canvas. + # Do not count their blank margins as extra lines. This is confined to + # short/wide inputs and a single compact ink band, never handwriting pages. + # Exclude full-height scan borders/rules from the projection, using the + # same glyph-component eligibility as the height estimator. + glyph_ink=np.isin(labels,projection_ids) if projection_ids else ink + active=glyph_ink.sum(axis=1)>=max(2,.0015*W) + gap=max(2,round(.35*(projection_glyph or 1.))) + edges=np.diff(np.r_[False,active,False].astype(np.int8)) + bands=[[int(a),int(b)] for a,b in zip(np.where(edges==1)[0],np.where(edges==-1)[0])] + joined=[] + for a,b in bands: + if joined and a-joined[-1][1]<=gap:joined[-1][1]=b + else:joined.append([a,b]) + padded_single=(H<=96 or W/max(1,H)>=3.) and len(joined)==1 and joined[0][1]-joined[0][0]<=max(6,3.5*(projection_glyph or H)) + if padded_single:line=float(H) + return dict(estimated_line_height=line, estimated_glyph_height=glyph, + valid_components=len(heights), crop_height=H, + height_to_line_height=H / max(1., line), segment=H > 1.8 * line, + original_glyph_based_line_height=original_line,ink_bands=joined, + single_ink_band_with_blank_margins=bool(padded_single), + noise_robust_projection=noise_robust,projection_glyph_height=projection_glyph) + + +def same_line(a, b): + overlap_y = min(a[3], b[3]) - max(a[1], b[1]) + overlap_x = min(a[2], b[2]) - max(a[0], b[0]) + return overlap_x > 0 and overlap_y > .5 * min(a[3] - a[1], b[3] - b[1]) + + +def merge_slivers(boxes, line_height=None): + """Union overlapping observations; complete-link groups avoid bridge chaining. + + A grown union is never used to decide subsequent matches. Every pair of + original members must overlap, or a plausible full-height anchor must + contain the sliver. A grown union never recruits a neighbouring row. + """ + groups = [] + for i in sorted(range(len(boxes)), key=lambda i: (-(boxes[i][3]-boxes[i][1]), i)): + compatible = [] + for j, group in enumerate(groups): + anchor=boxes[group[0]];box=boxes[i] + inside=(anchor[0]<=box[0] and anchor[1]<=box[1] and + anchor[2]>=box[2] and anchor[3]>=box[3]) + plausible=line_height is None or anchor[3]-anchor[1]<=1.8*line_height + if (inside and plausible) or all(same_line(box,boxes[k]) for k in group): + compatible.append(j) + if compatible: + groups[compatible[0]].append(i) + else: + groups.append([i]) + groups.sort(key=min) + merged = [[min(boxes[i][0] for i in g), min(boxes[i][1] for i in g), + max(boxes[i][2] for i in g), max(boxes[i][3] for i in g)] for g in groups] + return merged, dict(input_boxes=len(boxes), output_boxes=len(merged), + original_member_indices=groups, + criterion='Vertical overlap >50% smaller height and positive horizontal overlap; complete-link unions', + reference_text_or_boxes_used=False) diff --git a/stallion/bina04-pagepath-v3-round3-clean-20261031/pipeline_v3.py b/stallion/bina04-pagepath-v3-round3-clean-20261031/pipeline_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..725d8c20b7771d696ac07be81e3438bbfadf5ff2 --- /dev/null +++ b/stallion/bina04-pagepath-v3-round3-clean-20261031/pipeline_v3.py @@ -0,0 +1,216 @@ +"""Layout cannot filter detection. Exactly one recognizer reads every script.""" +import math, sys, time +from pathlib import Path +import numpy as np +from PIL import Image +P=Path('__BINA_ROOT__/stallion/indra-scale') +sys.path[:0]=[str(P),'__BINA_ROOT__/stallion/bina04-indra-pilot-20261027/upstream'] +from geometry import crop_line64 +from indra_ocr.rtl import flip,rtl_base + +def rectangle(polygon): + p=np.asarray(polygon,float) + return [float(p[:,0].min()),float(p[:,1].min()),float(p[:,0].max()),float(p[:,1].max())] + +def area(b):return max(0,b[2]-b[0])*max(0,b[3]-b[1]) +def intersection(a,b):return [max(a[0],b[0]),max(a[1],b[1]),min(a[2],b[2]),min(a[3],b[3])] +def clipped(b,w,h):return [max(0,b[0]),max(0,b[1]),min(w,b[2]),min(h,b[3])] +def reading_extent(line):return line['geometry'].get('source_pixels_extent',line['geometry']['padded_source_extent']) + +def xycut_order(boxes,width): + """RTL column order; spanning headings divide vertical bands.""" + n=len(boxes) + raster=lambda ids:sorted(ids,key=lambda i:((boxes[i][1]+boxes[i][3])/2,-boxes[i][0])) + core=[b for b in boxes if b[2]-b[0]<.7*width] + if len(core)<4:return raster(range(n)) + profile=np.zeros(max(1,int(math.ceil(width))),int) + for b in core:profile[max(0,int(b[0])):min(len(profile),int(math.ceil(b[2])))]+=1 + low=profile<=max(1,int(.05*len(core)));low[:int(.25*width)]=False;low[int(.75*width):]=False + e=np.diff(np.r_[False,low,False].astype(int));candidates=[] + for a,b in zip(np.where(e==1)[0],np.where(e==-1)[0]): + if b-a=cut] + if len(left)>=2 and len(right)>=2:candidates.append((b-a,cut,left,right)) + if not candidates:return raster(range(n)) + _,cut,left,right=max(candidates);middle=raster(set(range(n))-set(left)-set(right));remaining=set(left+right);out=[] + for stop in middle+[None]: + boundary=float('inf')if stop is None else(boxes[stop][1]+boxes[stop][3])/2 + for group in[right,left]: + take=raster(i for i in group if i in remaining and(boxes[i][1]+boxes[i][3])/20 else None + key=('region',j)if j is not None else('outside',i) + if key not in blocks:blocks[key]=dict(key=list(key),region=regions[j]if j is not None else None,line_ids=[]) + blocks[key]['line_ids'].append(i);assign.append(list(key)) + blocks=list(blocks.values());bounds=[] + for b in blocks: + members=[boxes[i]for i in b['line_ids']];bounds.append([min(x[0]for x in members),min(x[1]for x in members),max(x[2]for x in members),max(x[3]for x in members)]) + b['line_ids'].sort(key=lambda i:((boxes[i][1]+boxes[i][3])/2,-boxes[i][0])) + positions=[b['region'].get('position')if b['region']else None for b in blocks] + reasons=[] + if any(not isinstance(p,int)for p in positions)or len(set(positions))!=len(positions):reasons.append('missing_or_duplicate_region_order_or_outside_lines') + for i,a in enumerate(bounds): + for b in bounds[i+1:]: + if area(intersection(a,b))>.1*max(1,min(area(a),area(b))):reasons.append('overlapping_blocks');break + for block in blocks: + if block['region']and block['region'].get('label','').lower()=='table':continue + group=block['line_ids'];local=[boxes[i]for i in group];raster=sorted(range(len(group)),key=lambda i:((local[i][1]+local[i][3])/2,-local[i][0])) + if xycut_order(local,width)!=raster:reasons.append('multiple_columns_inside_layout_block') + if reasons: + # Apply the column fallback to physical lines, then form contiguous + # column chunks. Sorting the enclosing blocks alone interleaves any + # columns that Surya put inside one large Picture/Text region. + physical=xycut_order(boxes,width);owner={i:j for j,b in enumerate(blocks)for i in b['line_ids']};result=[];used_tables=set() + for i in physical: + j=owner[i];block=blocks[j];is_table=block['region']and block['region'].get('label','').lower()=='table' + if is_table: + if j not in used_tables:result.append(dict(block));used_tables.add(j) + elif result and result[-1].get('column_chunk_owner')==j and(boxes[i][1]+boxes[i][3])/2>=(boxes[result[-1]['line_ids'][-1]][1]+boxes[result[-1]['line_ids'][-1]][3])/2-.25*(boxes[i][3]-boxes[i][1]):result[-1]['line_ids'].append(i) + else:result.append(dict(block,line_ids=[i],column_chunk_owner=j)) + mode='RTL_physical_line_columns_with_atomic_tables' + else:result=[blocks[i]for i in sorted(range(len(blocks)),key=lambda i:positions[i])];mode='Surya_block_order' + assert sorted(i for b in result for i in b['line_ids'])==list(range(len(boxes))) + return result,dict(mode=mode,reasons=sorted(set(reasons)),assignments=assign,region_type_line_losses=0) + +class UnifiedReader: + """One model, unfiltered CTC, logical-order output. No script classifier.""" + def __init__(self,checkpoint,vocabulary,device='cpu'): + from rtl_reader import RTLReader + self.reader=RTLReader(checkpoint,vocabulary,device=device);self.source_is_crop=False + def __call__(self,gray,boxes): + boxes=np.asarray(boxes,np.float32).reshape(-1,4).tolist();normalization=None + if self.source_is_crop and len(boxes)==1 and np.allclose(boxes[0],[0.,0.,gray.shape[1],gray.shape[0]]): + from input_policy_v3 import estimated_line_height + from single_line_normalization import ink_band_box + estimate=estimated_line_height(gray) + if not estimate['segment']: + original=boxes[0];boxes=[ink_band_box(gray,estimate)] + normalization=dict(original_box=original,ink_band_box=boxes[0],one_full_width_line=True,no_detection_or_line_splitting=True) + if self.source_is_crop and boxes: + from line_isolation import atlas + canvas,shifted,maps=atlas(gray,boxes);_,receipts=self.reader(canvas,np.asarray(shifted,np.float32).reshape(-1,4)) + for receipt,mapping in zip(receipts,maps): + dx,dy=mapping['dx'],mapping['dy'];x,y,X,Y=receipt['padded_source_extent'];receipt.update(original_WH=[gray.shape[1],gray.shape[0]],padded_source_extent=[x-dx,y-dy,X-dx,Y-dy],source_pixels_extent=mapping['visible'],neighbor_line_pixels_excluded=True) + receipt['frame_centers']=[float(x-dx)for x in receipt['frame_centers']] + x,y,X,Y=receipt['ink_box'];receipt['ink_box']=[x-dx,y-dy,X-dx,Y-dy] + else:_,receipts=self.reader(gray,np.asarray(boxes,np.float32).reshape(-1,4)) + return[dict(box=list(map(float,b)),text=flip(r['raw_visual_text'],base_rtl=True),visual_text=r['raw_visual_text'],confidence=r['confidence'],recognizer='unified32M',geometry=r,confidence_filter_applied=False,whole_input_normalization=normalization)for b,r in zip(boxes,receipts)] + +def markdown_cell(text):return ' '.join(text.split()).replace('|',r'\|') + +class PagePipeline: + def __init__(self,detector,layout,reader,table=None): + self.detector=detector;self.layout=layout;self.reader=reader;self.table=table;self._crop_line_guard=None + def detect(self,image): + # The SDK performs its own resize/pad/tiling on the WHOLE input. + raw=self.detector([image],batch_size=1)[0].model_dump(mode='json');w,h=image.size + boxes=[clipped(rectangle(r['polygon']),w,h)for r in raw['bboxes']] + boxes=[b for b in boxes if area(b)>0] + # Fixed native-resolution windows are independent of every layout label. + # Full-width SDK downsampling otherwise makes dense columns/cells tiny. + def starts(size,tile,overlap): + if size<=tile:return [0] + out=list(range(0,size-tile+1,tile-overlap)) + if out[-1]!=size-tile:out.append(size-tile) + return out + traces=[];supplements=[];background=tuple(int(v)for v in np.median(np.asarray(image).reshape(-1,3),axis=0));size=max(self.detector.processor.size.values()) + def context(piece,x,y,scale): + # Padding prevents prepare_image's final square stretch from + # deforming glyphs. Recognition never uses these resized pixels. + pw,ph=piece.size;rw=max(1,round(pw*scale));rh=max(1,round(ph*scale));canvas=Image.new('RGB',(size,size),background);canvas.paste(piece.resize((rw,rh),Image.Resampling.LANCZOS),(0,0));part=self.detector([canvas],batch_size=1)[0].model_dump(mode='json');out=[] + for item in part['bboxes']: + a,b,A,B=rectangle(item['polygon']);a=max(0,min(rw,a));A=max(0,min(rw,A));b=max(0,min(rh,b));B=max(0,min(rh,B));v=clipped([a*pw/rw+x,b*ph/rh+y,A*pw/rw+x,B*ph/rh+y],w,h) + if area(v)>0:out.append(v) + traces.append(dict(tile=[x,y,x+pw,y+ph],scale=scale,canvas_WH=[size,size],boxes=out));supplements.extend(out) + for y in starts(h,size,240): + for x in starts(w,size,240):context(image.crop((x,y,min(w,x+size),min(h,y+size))),x,y,1.) + if max(w,h)<=1400:context(image,0,0,min(.5,size/max(w,h))) + from fusion_v3 import fuse + merged,fusion=fuse(np.asarray(image.convert('L')),boxes,supplements) + return merged,dict(full_image_SDK=raw,native_resolution_tiles=traces,fusion=fusion,layout_used_for_detection=False,recognizer_pixels='Original image only; detector context resize is inverted geometrically') + + def table_block(self,image,gray,block,boxes,readings): + region=block['region'];w,h=image.size;b=clipped(rectangle(region['polygon']),w,h);x,y,X,Y=map(int,[math.floor(b[0]),math.floor(b[1]),math.ceil(b[2]),math.ceil(b[3])]);crop=image.crop((x,y,X,Y)) + if self.table is None:return None,dict(error='Surya_table_engine_not_ready',all_original_lines_retained=True) + try:result=self.table(crop) + except Exception as e:return None,dict(error=type(e).__name__+': '+str(e),all_original_lines_retained=True) + cells=result.get('cells',[]) + if result.get('error')or not cells:return None,dict(error='Surya_table_grid_empty',grid=result,all_original_lines_retained=True) + cells=sorted(cells,key=lambda c:(c['row_id'],c.get('col_id')if c.get('col_id')is not None else c.get('within_row_id',0)));pieces=[];meta=[];represented=set();cell_fragments={i:[]for i in block['line_ids']} + for cell in cells: + cb=cell.get('bbox')or rectangle(cell['polygon']);cb=[cb[0]+x,cb[1]+y,cb[2]+x,cb[3]+y];parts=[] + for i in block['line_ids']: + part=intersection(boxes[i],cb) + if part[2]-part[0]>=2 and part[3]-part[1]>=.5*(boxes[i][3]-boxes[i][1]): + # A physical row inside a cell keeps the cell's horizontal + # context; only its vertical line band is cropped. + part=[cb[0],part[1],cb[2],part[3]];parts.append(part);represented.add(i);cell_fragments[i].append(part) + if not parts:parts=[cb] + parts.sort(key=lambda p:((p[1]+p[3])/2,-p[0]));bands=[] + for part in parts: + hit=next((j for j,b in enumerate(bands)if min(part[3],b[3])-max(part[1],b[1])>=.65*min(part[3]-part[1],b[3]-b[1])),None) + if hit is None:bands.append(part) + else:bands[hit]=[cb[0],min(bands[hit][1],part[1]),cb[2],max(bands[hit][3],part[3])] + begin=len(pieces);pieces+=bands;meta.append((cell,cb,begin,len(pieces))) + # A grid may not silently erase part of a detected line. A line entirely + # outside cells is a retained note; a partly clipped line fails back to + # the original block rather than emitting a misleading partial table. + import cv2 + coverage=[] + for i in represented: + b=boxes[i];a,c,A,C=map(int,[math.floor(b[0]),math.floor(b[1]),math.ceil(b[2]),math.ceil(b[3])]);patch=gray[c:C,a:A];_,ink=cv2.threshold(patch,0,255,(cv2.THRESH_BINARY if np.median(patch)<128 else cv2.THRESH_BINARY_INV)|cv2.THRESH_OTSU);mask=np.zeros(patch.shape,bool) + for q in cell_fragments[i]: + height=q[3]-q[1];u,v,U,V=map(int,[math.floor(q[0]-.3*height),math.floor(q[1]-.2*height),math.ceil(q[2]+.3*height),math.ceil(q[3]+.2*height)]);mask[max(0,v-c):min(C-c,V-c),max(0,u-a):min(A-a,U-a)]=True + n=int((ink>0).sum());fraction=float((mask&(ink>0)).sum())/n if n else 1.;coverage.append(dict(source_line=i,ink_fraction=fraction)) + if any(r['ink_fraction']<.98 for r in coverage):return None,dict(error='Table_grid_clips_detected_line_ink',grid=result,coverage=coverage,all_original_lines_retained=True) + lines=self.reader(gray,pieces);rows={};rtl=rtl_base(' '.join(l['text']for l in lines)) + for cell,cb,a,b in meta: + rows.setdefault(cell['row_id'],[]).append(dict(text=' '.join(line['text']for line in lines[a:b]),box=cb,colspan=cell.get('colspan',1))) + rendered=[] + for j,(_,row)in enumerate(sorted(rows.items())): + row.sort(key=lambda c:(-1 if rtl else 1)*(c['box'][0]+c['box'][2])/2);values=[] + for cell in row:values+=[cell['text']]+['']*(max(1,cell['colspan'])-1) + rendered.append('| '+' | '.join(markdown_cell(s)for s in values)+' |') + if j==0:rendered.append('| '+' | '.join('---'for _ in values)+' |') + # A line not represented by any cell remains in output after the table. + remainder=[i for i in block['line_ids']if i not in represented] + rendered +=[readings[i]['text']for i in remainder] + emitted=lines+[readings[i]for i in remainder] + return dict(text='\n'.join(rendered),lines=emitted),dict(grid=result,represented_line_ids=sorted(represented),unassigned_line_ids=remainder,grid_text_never_used=True,Markdown_first_row_separator=True,column_direction='RTL'if rtl else'LTR',cell_coverage=coverage) + def __call__(self,image,*,input_type='auto',**_): + from input_policy_v3 import estimated_line_height,merge_slivers,input_kind + tick=time.monotonic();image=image.convert('RGB');gray=np.asarray(image.convert('L'));estimate=estimated_line_height(gray);kind=input_kind(image.width,image.height,input_type);self.reader.source_is_crop=kind=='crop' + if kind=='crop'and not estimate['segment']: + boxes=[[0.,0.,float(image.width),float(image.height)]];readings=self.reader(gray,boxes);assert len(readings)==1 + region=dict(layout=None,source_line_ids=[0],path='rule_C_whole_input_unified_reader',text=readings[0]['text'],lines=readings) + return dict(text=readings[0]['text'],regions=[region],lines=readings,global_readings=readings,detected_boxes=[],emitted_boxes=[r['box']for r in readings],recognition_extents=[reading_extent(r)for r in readings],raw_line_detection=dict(skipped=True,reason='Rule C: height <=1.8 estimated line height'),raw_layout=dict(bboxes=[],skipped=True),order=dict(mode='single_input_rule_C',region_type_line_losses=0),fallbacks=[],crop_routing_policy=dict(mode='c',input_type=kind,estimate=estimate,whole_crop_by_policy=True,native_detector_calls=0),region_type_line_losses=0,recognizer_bank='unified32M_only',whole_image_detection=False,seconds=time.monotonic()-tick,main_Bina_training_changes=False) + boxes,detection=self.detect(image);fallbacks=[] + if kind=='crop': + line=estimate['estimated_line_height'] + if boxes and any(b[3]-b[1]>1.8*line for b in boxes): + if self._crop_line_guard is None: + from crop_line_guard import CropLineGuard + self._crop_line_guard=CropLineGuard() + boxes,guard=self._crop_line_guard(gray,boxes,line);detection['crop_anti_merge_guard']=guard + boxes,consolidation=merge_slivers(boxes,line);detection['same_line_box_consolidation']=consolidation + if not boxes:boxes=[[0.,0.,float(image.width),float(image.height)]];fallbacks.append('zero_detector_lines_whole_image_attempt_not_valid_for_coverage_gate') + try:layout=self.layout.run_batch([image],[dict(threshold=None,use_order=True)])[0].model_dump(mode='json') + except Exception as e:layout=dict(bboxes=[],error=repr(e));fallbacks.append('layout_error_RTL_column_order_all_lines_retained') + blocks,order=order_blocks(boxes,layout,image.width);readings=self.reader(gray,boxes);assert len(readings)==len(boxes) + regions=[];emitted=[] + for block in blocks: + ids=block['line_ids'];lines=[readings[i]for i in ids];record=dict(layout=block['region'],source_line_ids=ids,path='whole_image_lines',text='\n'.join(l['text']for l in lines),lines=lines) + if block['region']and block['region'].get('label','').lower()=='table': + assembled,trace=self.table_block(image,gray,block,boxes,readings);record['table_trace']=trace + if assembled:record.update(assembled,path='Surya_grid_unified_recognizer_cells') + else: + fallback_order=xycut_order([boxes[i]for i in ids],image.width);record['lines']=[readings[ids[j]]for j in fallback_order];record['text']='\n'.join(l['text']for l in record['lines']);record['source_line_ids']=[ids[j]for j in fallback_order];fallbacks.append('table_grid_failed_original_lines_retained_in_RTL_columns') + regions.append(record);emitted+=record['lines'] + assert sorted(i for r in regions for i in r['source_line_ids'])==list(range(len(boxes))) + return dict(text='\n\n'.join(r['text']for r in regions),regions=regions,lines=emitted,global_readings=readings,detected_boxes=boxes,emitted_boxes=[l['box']for l in emitted],recognition_extents=[reading_extent(l)for l in emitted],raw_line_detection=detection,raw_layout=layout,order=order,fallbacks=fallbacks,crop_routing_policy=dict(mode='c',input_type=kind,estimate=estimate,whole_crop_by_policy=False),region_type_line_losses=0,recognizer_bank='unified32M_only',whole_image_detection=True,seconds=time.monotonic()-tick,main_Bina_training_changes=False) diff --git a/stallion/bina04-pagepath-v3-round3-clean-20261031/run/weights-095000.pt b/stallion/bina04-pagepath-v3-round3-clean-20261031/run/weights-095000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e78d57a571260e3e9134c06856ee902e7771e210 --- /dev/null +++ b/stallion/bina04-pagepath-v3-round3-clean-20261031/run/weights-095000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981 +size 131256958 diff --git a/stallion/bina04-pagepath-v3-round3-clean-20261031/text_contract.py b/stallion/bina04-pagepath-v3-round3-clean-20261031/text_contract.py new file mode 100644 index 0000000000000000000000000000000000000000..a67b01f4ea62e3e1bb267ec5a724180d3ce4bc19 --- /dev/null +++ b/stallion/bina04-pagepath-v3-round3-clean-20261031/text_contract.py @@ -0,0 +1,16 @@ +"""Gold/storage/API are logical Unicode. CTC alignment is physical visual order.""" +import sys,unicodedata +sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-indra-pilot-20261027/upstream') +from indra_ocr.rtl import flip +def encode_target(logical):return flip(logical,base_rtl=True) +def decode_prediction(visual):return flip(visual,base_rtl=True) +REQUIRED='ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_@:/?&=.%+-#()[]{}!,$;|<>~`₀₁₂₃₄₅₆₇₈₉⁰¹²³⁴⁵⁶⁷⁸⁹ᵀⁱ∧∨¬∑∏√≤≥≠≈∈⊆→×·−☐●▪•' +FIXTURES=[ + 'ش دفترچه : ۱۱۳۸۸۴۷۷۳۲۲', + '(X₁ ∧ X₂) xor (¬X₁ ∧ X₂)', + 'نشانی user17@example.org و https://example.org/a?id=42 برای پیگیری', + 'مقدار ∑ xᵀ × y₀ ≤ ۱۲ و نتیجه Aⁱ → B است.', + '☐ گزینه اول ● گزینه دوم ▪ مورد سوم • پایان', + 'Persian OCR: Version 3.0 / contact user@example.org', + 'ساعت ۱۲:۳۰ و شماره A-109؛ مقدار 1.25 و X₂ ≠ ۰', +] diff --git a/stallion/bina04-print-textgate-20261102/CALIBRATION.json b/stallion/bina04-print-textgate-20261102/CALIBRATION.json new file mode 100644 index 0000000000000000000000000000000000000000..76c9424470b73554a92dc0c2b53770c999d13837 --- /dev/null +++ b/stallion/bina04-print-textgate-20261102/CALIBRATION.json @@ -0,0 +1 @@ +{"printed": {"admission": 0.75, "read_confidence": 0.6, "include_threshold": 0.25}, "order_confidence_threshold": 0.7, "order_trials": [{"threshold": 1.1, "metrics": {"body": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}, "full": {"errors": 2969, "characters": 22716, "CER": 13.07008276104948}, "tagged": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}}, "safe": true}, {"threshold": 0.99, "metrics": {"body": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}, "full": {"errors": 2969, "characters": 22716, "CER": 13.07008276104948}, "tagged": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}}, "safe": true}, {"threshold": 0.97, "metrics": {"body": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}, "full": {"errors": 2969, "characters": 22716, "CER": 13.07008276104948}, "tagged": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}}, "safe": true}, {"threshold": 0.95, "metrics": {"body": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}, "full": {"errors": 2969, "characters": 22716, "CER": 13.07008276104948}, "tagged": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}}, "safe": true}, {"threshold": 0.9, "metrics": {"body": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}, "full": {"errors": 2969, "characters": 22716, "CER": 13.07008276104948}, "tagged": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}}, "safe": true}, {"threshold": 0.8, "metrics": {"body": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}, "full": {"errors": 2969, "characters": 22716, "CER": 13.07008276104948}, "tagged": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}}, "safe": true}, {"threshold": 0.7, "metrics": {"body": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}, "full": {"errors": 2966, "characters": 22716, "CER": 13.056876210600457}, "tagged": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}}, "safe": true}, {"threshold": 0.0, "metrics": {"body": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}, "full": {"errors": 2972, "characters": 22716, "CER": 13.083289311498504}, "tagged": {"errors": 1133, "characters": 19984, "CER": 5.6695356285028025}}, "safe": true}], "calibration_sha256": "d158f8d3a7af7fd4e40395a803ebfb10cbac50e1a8a69bf1b974386aaa32c14c", "split_sha256": "a7ea3b806a80326d570f2c4b32cd5c8a32e99e1953d371c2eaf337beaf8a74a8", "check_labels_never_used": true} diff --git a/stallion/bina04-print-textgate-20261102/include-model.pkl b/stallion/bina04-print-textgate-20261102/include-model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..8fa09fe1c42d7c27c022e3b43eace8919d389d0a --- /dev/null +++ b/stallion/bina04-print-textgate-20261102/include-model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dfe210eb06eee78faba45dcb965d315745da27211ed75a8cae9fd4fc961249f +size 7982847 diff --git a/stallion/bina04-print-textgate-20261102/include_model.py b/stallion/bina04-print-textgate-20261102/include_model.py new file mode 100644 index 0000000000000000000000000000000000000000..08bdc7ab373adf81f671834b32bbbf4157a01ca8 --- /dev/null +++ b/stallion/bina04-print-textgate-20261102/include_model.py @@ -0,0 +1,31 @@ +"""Generator-serialized convention classifier, using observed text/geometry only.""" +import json,pickle,sys +from pathlib import Path +import numpy as np +from rapidfuzz.fuzz import ratio +from sklearn.feature_extraction.text import HashingVectorizer +sys.path.insert(0,'__BINA_ROOT__/stallion/indra-scale/scoringTT') +from shenavasanj import fair_text +HASH=HashingVectorizer(analyzer='char',ngram_range=(1,3),n_features=128,alternate_sign=False,norm='l2',lowercase=False) +def norm(s):return fair_text(s,keep_spaces=False) +def context(lines,templates): + strings=[norm(l['text'])for l in lines if norm(l['text'])] + values=[] + for words in templates: + nonempty=[w for w in words if len(w)>=3] + evidence=[max((ratio(w,s)/100 for s in strings),default=0.)for w in nonempty] + # Template support is inferred from the observed page; template ID is never an input. + weights=[min(20,len(w))for w in nonempty];values.append(sum(a*b for a,b in zip(evidence,weights))/max(1,sum(weights))) + return np.asarray(values,np.float32) +def features(lines,WH,body,templates): + W,H=WH;a,b,A,B=body;ctx=context(lines,templates);out=[] + for l in lines: + text=norm(l['text']);x,y,X,Y=l['box'];similar=np.asarray([max((ratio(text,w)/100 for w in words),default=0.)for words in templates],np.float32);geometry=[x/W,y/H,X/W,Y/H,(X-x)/W,(Y-y)/H,(y-b)/max(1,B-b),(x-a)/max(1,A-a),float(YB),l.get('confidence',.8),min(1,len(text)/80)] + out.append(np.r_[HASH.transform([text]).toarray()[0],geometry,ctx,similar,ctx*similar]) + return np.asarray(out,np.float32).reshape(-1,140+3*len(templates)) +class IncludeModel: + def __init__(self,root): + q=pickle.loads((Path(root)/'include-model.pkl').read_bytes());self.model=q['model'];self.templates=q['templates'] + def predict(self,lines,WH,body): + if not lines:return[] + return self.model.predict_proba(features(lines,WH,body,self.templates))[:,1].tolist() diff --git a/stallion/bina04-print-textgate-20261102/layer_runtime.py b/stallion/bina04-print-textgate-20261102/layer_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..18a2decc5537fe629436a9b47a511c7260596f35 --- /dev/null +++ b/stallion/bina04-print-textgate-20261102/layer_runtime.py @@ -0,0 +1,45 @@ +"""Isolated printed proposals. Body detection, rows, OCR and inclusion are immutable.""" +import copy,json,sys,importlib.util +from pathlib import Path +import numpy as np +R=Path(__file__).resolve().parent;OLD=Path('__BINA_ROOT__/stallion/bina04-printed-form-fix-20261102');J=Path('__BINA_ROOT__/stallion/bina04-e2e-batch-95k-20261101/candidates/combined');S=J/'snapshot/runtime';P=Path('__BINA_ROOT__/stallion/indra-scale');sys.path[:0]=[str(OLD),str(J/'versions/combined'),str(J),str(S),str(S/'baseline'),str(P),str(P/'scoringTT')] +from label_model import LabelModel +from candidate_apply import apply,key +from rtl_reader import RTLReader +from baseline.pipeline_v3 import flip +class PrintedLayer: + def __init__(self,device='cpu'): + import torch + from surya.detection import DetectionPredictor + best=json.loads((R/'run/best.json').read_text());q=torch.load(best['path'],map_location='cpu',weights_only=True);self.detector=DetectionPredictor.local(device=device,dtype=torch.float32);self.detector._disable_tqdm=True;self.detector.model.load_state_dict(torch.load(J/'run/checkpoints/step-001000.pt',map_location='cpu',weights_only=True)['model']);self.detector.model.decode_head.load_state_dict(q['printed_head']);self.detector.model.eval() + cp=R/'run/selected-printed-recognizer.pt' + if not cp.exists():torch.save(dict(model=q['recognizer'],step=q['step'],source=best['path']),cp) + self.reader=RTLReader(cp,Path('__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json'),device=device);self.labels=LabelModel(OLD) + spec=importlib.util.spec_from_file_location('accepted_cell_runtime',OLD/'production.py');module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module);self.cell_expert=module.Expert.__new__(module.Expert);self.cell_expert.short=self.reader;self.cell_expert.model=self.labels;self.cell_expert.params=json.loads((OLD/'CALIBRATION.json').read_text())['selected']['parameters'] + def infer(self,image,baseline): + # Caller passes pages only. Independent learned print must not depend on color/row reliability. + raw=self.detector([image.convert('RGB')],batch_size=1)[0].model_dump(mode='json');boxes=[] + for q in raw['bboxes']: + p=np.asarray(q['polygon']);boxes.append([float(p[:,0].min()),float(p[:,1].min()),float(p[:,0].max()),float(p[:,1].max())]) + originals=[l for l in baseline['lines']if l.get('layer')=='printed'];allboxes=boxes+[l['box']for l in originals];gray=np.asarray(image.convert('L'));_,receipts=self.reader(gray,np.asarray(allboxes,np.float32).reshape(-1,4))if allboxes else('',[]);lines=[] + for i,(b,g)in enumerate(zip(allboxes,receipts)): + lines.append(dict(box=b,text=flip(g['raw_visual_text'],base_rtl=True),confidence=float(g['confidence']),visual_text=g['raw_visual_text'],geometry=g,layer='printed',printed_origin='learned_printed_head'if i=threshold]);result,trace=apply(baseline,pp,dict(source='recovered',order='serialized',page_confidence=0.,**params)) + trace.update(include_threshold=threshold,include_classifier_on_new_labels=True,excluded_by_convention=len(prepared['recovered'])-len(pp['recovered'])) + if not baseline.get('row_consolidation',{}).get('applied'): + result['handwriting_extents']=baseline.get('handwriting_extents',baseline['recognition_extents']) + assert[key(l)for l in result['lines']if l.get('layer')!='printed']==[key(l)for l in baseline['lines']if l.get('layer')!='printed'] + return result,trace diff --git a/stallion/bina04-print-textgate-20261102/order_adapter.py b/stallion/bina04-print-textgate-20261102/order_adapter.py new file mode 100644 index 0000000000000000000000000000000000000000..b873acff7377ff593b6c44627b00f119c8b613c0 --- /dev/null +++ b/stallion/bina04-print-textgate-20261102/order_adapter.py @@ -0,0 +1,14 @@ +"""Keep frozen-v3 weights/features; reject movement of body reads inside atomic tables.""" +import json,sys +sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-order-application-debug-20261102') +from fixed_order import V3 as FrozenV3 + +def protected(result): + return [json.dumps([l['box'],l['text'],l.get('visual_text')],ensure_ascii=False,sort_keys=True)for l in result['lines']if l.get('layer')!='printed'] +class V3(FrozenV3): + def propose(self,result,image): + proposal,info=super().propose(result,image) + if protected(proposal)!=protected(result): + return result,dict(info,changed=False,confidence=0.,unsafe_atomic_table_move_rejected=True,body_order_preserved=True,reason='Atomic table tagged printed contains unknown/body reads; original physical body order is authoritative') + if 'handwriting_extents'in result:proposal['handwriting_extents']=result['handwriting_extents'] + return proposal,dict(info,all_protected_reads_and_order_verified=True) diff --git a/stallion/bina04-print-textgate-20261102/production.py b/stallion/bina04-print-textgate-20261102/production.py new file mode 100644 index 0000000000000000000000000000000000000000..2c9b5e696c1a259311ed504668a88bc01ba9fe43 --- /dev/null +++ b/stallion/bina04-print-textgate-20261102/production.py @@ -0,0 +1,16 @@ +"""Gated overlay; the current official pipeline is called first and remains frozen.""" +import json,sys +from pathlib import Path +R=Path(__file__).resolve().parent +from layer_runtime import PrintedLayer,key +sys.path.insert(0,'__BINA_ROOT__/stallion/bina04-order-application-debug-20261102') +from order_adapter import V3 +class PageOverlay: + def __init__(self,device='cpu'): + self.layer=PrintedLayer(device);self.cal=json.loads((R/'CALIBRATION.json').read_text());self.order=V3('__BINA_ROOT__/stallion/bina04-surya25-order-20261101/role-model/weights-003000.pt') + def __call__(self,image,baseline,*,input_type='page'): + if input_type!='page':return self.layer.cells(image,baseline)[0] + pp=self.layer.infer(image,baseline);result,_=self.layer.render(baseline,pp,self.cal['printed']);proposal,info=self.order.propose(result,image);threshold=self.cal['order_confidence_threshold'];use=bool(info.get('changed')and info.get('confidence',0)>=threshold);result=proposal if use else result;result['learned_order']=dict(info,applied=use,threshold=threshold,role_default_bug_fixed=True) + assert[key(l)for l in result['lines']if l.get('layer')!='printed']==[key(l)for l in baseline['lines']if l.get('layer')!='printed'] + result['printed_layer_model']=dict(root=str(R),selected=json.loads((R/'run/best.json').read_text())['step'],separate_from_handwriting=True,source_pixels_modified=False) + return result diff --git a/stallion/bina04-print-textgate-20261102/run/best.json b/stallion/bina04-print-textgate-20261102/run/best.json new file mode 100644 index 0000000000000000000000000000000000000000..c1aaa49d9d4f38453fe3f9ac14e6865c820a8b58 --- /dev/null +++ b/stallion/bina04-print-textgate-20261102/run/best.json @@ -0,0 +1,73 @@ +{ + "step": 2000, + "printed_pixel_F1": 0.9386092870770448, + "printed_line_F1": 0.9455370650529501, + "printed_predictions": 2667, + "printed_gold_components": 2621, + "printed_loss": 0.11509075737558305, + "noisy_label_CER": 0.07792207792207792, + "clean_label_CER": 0.03142677561282212, + "regular_macro_CER": 0.011602644408879944, + "buckets": { + "synthetic": { + "edits": 13454, + "characters": 363328, + "cases": 5485, + "CER": 0.03702990135635018 + }, + "newspaper": { + "edits": 700, + "characters": 58304, + "cases": 1430, + "CER": 0.012006037321624589 + }, + "printed": { + "edits": 156, + "characters": 18274, + "cases": 600, + "CER": 0.008536718835503995 + }, + "english": { + "edits": 21, + "characters": 36799, + "cases": 500, + "CER": 0.0005706676811869887, + "image_conditioning_passed": true, + "blank_control": { + "cases": 16, + "characters": 1297, + "real_CER": 0.0, + "blank_CER": 1.0, + "advantage": 1.0, + "passed": true + } + }, + "handwriting": { + "edits": 639, + "characters": 80766, + "cases": 2066, + "CER": 0.00791174504123022 + }, + "mixed": { + "edits": 122, + "characters": 34262, + "cases": 500, + "CER": 0.0035607962173836902, + "image_conditioning_passed": true, + "blank_control": { + "cases": 16, + "characters": 1043, + "real_CER": 0.02588686481303931, + "blank_CER": 1.0, + "advantage": 0.9741131351869607, + "passed": true + } + } + }, + "selection": "Internal synthetic dev only; real gates never select checkpoints", + "path": "__BINA_ROOT__/stallion/bina04-printed-layer-training-20261102/run/weights-002000.pt", + "restored_head_source": "__BINA_ROOT__/stallion/bina04-ia-real-retrain-20261105/printed-head-recovery/run/weights-002000.pt", + "restoration_receipt": "__BINA_ROOT__/stallion/bina04-ia-real-retrain-20261105/printed-head-recovery/OFFICIAL-RESTORATION.json", + "restoration_selection": "Minimum character distance to old accepted outputs among heads passing all recovery gates", + "old_output_character_distance": 366 +} \ No newline at end of file diff --git a/stallion/bina04-print-textgate-20261102/run/selected-printed-recognizer.pt b/stallion/bina04-print-textgate-20261102/run/selected-printed-recognizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1186d182f593406a8541ed09d6ca7dc510f39b9 --- /dev/null +++ b/stallion/bina04-print-textgate-20261102/run/selected-printed-recognizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28c4ca8f9fbbc8c578bb130d997598d2e9c2c4a2dad0b22cb5cc06818ecb988a +size 131256937 diff --git a/stallion/bina04-printed-form-fix-20261102/CALIBRATION.json b/stallion/bina04-printed-form-fix-20261102/CALIBRATION.json new file mode 100644 index 0000000000000000000000000000000000000000..19d9d67fbb71a3e42dd9eb5d4f608706c27a4016 --- /dev/null +++ b/stallion/bina04-printed-form-fix-20261102/CALIBRATION.json @@ -0,0 +1,31 @@ +{ + "UTC": 1793616239.2583275, + "selected": { + "parameters": { + "source": "original", + "order": "serialized", + "admission": 0.9, + "read_confidence": 0.55, + "page_confidence": 0.0 + }, + "safe": true, + "gain": 266, + "applied": 11, + "body_delta": -13, + "worse_pages": [] + }, + "check_used_for_parameters": false, + "rule": "No calibration page body/full regression; maximize full-page character gain; fallback if no safe gain", + "source_sha256": { + "__BINA_ROOT__/stallion/bina04-printed-form-fix-20261102/breakdown.py": "eead20fafd8d7ed894d2c9d381a6e2193a283ea48a2eb84591f90418ecbbfb3d", + "__BINA_ROOT__/stallion/bina04-printed-form-fix-20261102/printed_reads.py": "cd00bf39e9ac459959d10c2fd2fb5d712b781534d540d9d20d44e7665145661d", + "__BINA_ROOT__/stallion/bina04-printed-form-fix-20261102/label_model.py": "20d4d4a270ff911455f82f1cd0a357c080303efb75b11bddbd9e11ebec4e2a28", + "__BINA_ROOT__/stallion/bina04-printed-form-fix-20261102/candidate_apply.py": "3ccc53bee79ec6b2a3d02f590d615cee33830b953d19a99c067765a6ab662c24", + "__BINA_ROOT__/stallion/bina04-printed-form-fix-20261102/train_label_model.py": "8884eaf6d5a8e33fcb19d391e4107057de1205b056ef32796f13121e7a604b12", + "__BINA_ROOT__/stallion/bina04-printed-form-fix-20261102/calibrate.py": "6f9f4a538f9bea49f51357d78733074e3928bfc47d3fcfe6f11c3c9e9d9e415a" + }, + "classifier_sha256": "bf02d5006aad71a2383d69b5c1db01579b4effe88fec4692eafd50e83772fc45", + "lexicon_sha256": "2aaa0017565f121e0d252094e8c4d112227c76b2930b54717678cab24fcb0f54", + "split_sha256": "a7ea3b806a80326d570f2c4b32cd5c8a32e99e1953d371c2eaf337beaf8a74a8", + "trials": 96 +} diff --git a/stallion/bina04-printed-form-fix-20261102/SERIALIZED-LEXICON.json b/stallion/bina04-printed-form-fix-20261102/SERIALIZED-LEXICON.json new file mode 100644 index 0000000000000000000000000000000000000000..421da47584418e922f46889ca8ce64c822a1a3f6 --- /dev/null +++ b/stallion/bina04-printed-form-fix-20261102/SERIALIZED-LEXICON.json @@ -0,0 +1,126 @@ +[ + "", + "1", + "10", + "13", + "15", + "2", + "20", + "25", + "3", + "30", + "4", + "5", + "6", + "7", + "8", + "9", + "اجتماعی", + "ادارهکلاموزشوپرورشاستان", + "ادارهکلاموزشوپرورشتهران", + "اذر", + "اسفند", + "امتحان", + "امضااموزگار", + "امضاولی", + "املافارسی", + "اموزشگاه", + "انشابنویسیم", + "انضباط", + "اولیا", + "باحروف", + "بارم", + "باعدد", + "برگامتحان", + "برگامتحانی", + "برگهامتحانی", + "بهار", + "بهمن", + "بهنامخدا", + "تاریخ", + "تاریخامتحان", + "توضیحات", + "ثلث", + "جمهوریاسلامیایران", + "حوزه", + "خخخیلیخوب", + "خخوب", + "خوب", + "خیلیخوب", + "داخلپرانتزهاچیزیننویسید", + "دبستان", + "دبیرستان", + "درس", + "دهانه", + "دورهضمنسال", + "ردیف", + "رشته", + "ریاضیات", + "زمستان", + "سالانه", + "سالتحصیلی0403", + "شاخه", + "شعبه", + "شمارهداوطلب", + "شمارهشناسنامه", + "ضعیف", + "عالی", + "علومتجربی", + "فروردین", + "فصل", + "قابلقبول", + "قران", + "قرایتفارسیبخوانیم", + "قققابلقبول", + "ماه", + "متوسط", + "متولد", + "محلامضاءتجدیدنظر", + "محلامضاءمصحح", + "محلتایید", + "مدیریتاموزشوپرورش", + "مستمر", + "ملاحظات", + "مهر", + "مهروابان", + "مهروامضامدیرمدرسه", + "موضوع", + "موضوعامتحان", + "نام", + "نامخانوادگی", + "نامدرس", + "نامونامخانوادگی", + "نامونامخانوادگیدانشاموز", + "نامپدر", + "نتنیازبهتلاشواموزشبیشتر", + "نتیجه", + "نمراتنوبتاول", + "نمراتنوبتدوم", + "نمره", + "نمرهتجدیدنظر", + "نمرهتجدیدنظرمعتبراست", + "نهایی", + "نوبتاول", + "نوبتدوم", + "نیازبهتلاشبیشتر", + "هدیههایاسمانی", + "هنر", + "وابان", + "واحد", + "واردیبهشت", + "ورزش", + "ورقهامتحانیه", + "وزارتاموزشوپرورش", + "وضعیتدرماه", + "وضعیتعملکرددانشاموزدرهرفصل", + "وضعیتقبولی", + "پایانی", + "پایه", + "پایهپنجم", + "پاییز", + "کارنامهتحصیلیدوره", + "کارنامهماهانه", + "کددانشاموز", + "کددرس", + "کلاس" +] diff --git a/stallion/bina04-printed-form-fix-20261102/candidate_apply.py b/stallion/bina04-printed-form-fix-20261102/candidate_apply.py new file mode 100644 index 0000000000000000000000000000000000000000..3dbe44884fc1969f086b31e9d013ff0a29f0e2b7 --- /dev/null +++ b/stallion/bina04-printed-form-fix-20261102/candidate_apply.py @@ -0,0 +1,48 @@ +import copy,json,hashlib +import numpy as np +from baseline.pipeline_v3 import reading_extent +def key(line):return json.dumps([line['box'],line['text'],line.get('visual_text')],ensure_ascii=False,sort_keys=True) +def overlap(a,b): + hit=max(0,min(a[2],b[2])-max(a[0],b[0]))*max(0,min(a[3],b[3])-max(a[1],b[1]));return hit/max(1,min((a[2]-a[0])*(a[3]-a[1]),(b[2]-b[0])*(b[3]-b[1]))) +def prepare(baseline,printed,WH,model): + if not printed.get('applied'):return dict(applied=False) + protected=[l for l in baseline['lines']if l.get('layer')!='printed'];bb=[min(l['box'][0]for l in protected),min(l['box'][1]for l in protected),max(l['box'][2]for l in protected),max(l['box'][3]for l in protected)]if protected else[0,0,*WH] + recovered=[] + for c in printed['candidates']: + options=[] + for label in ['short','base']: + l=c[label];admit=model.admission(l['text']);options.append((admit*.75+l['confidence']*.25,label,admit)) + score,chosen,admit=max(options);l=dict(c[chosen]);l.update(layer='printed',source_pixels_modified=True,printed_origin=c['source'],printed_reader=chosen,admission=admit);recovered.append(l) + original=[];new_by={q['index']:q['short']for q in printed['original_short_reads']} + for i,base in enumerate(baseline['lines']): + if base.get('layer')!='printed':continue + alt=new_by.get(i);old_score=model.admission(base['text'])*.75+base.get('confidence',0)*.25;new_score=model.admission(alt['text'])*.75+alt['confidence']*.25 if alt else-1 + l=dict(alt)if alt and new_score>old_score else dict(base);l.update(layer='printed',printed_origin='original_printed_box',printed_reader='short'if new_score>old_score else'95k',source_pixels_modified=False,admission=model.admission(l['text']));original.append(l) + for group in [recovered,original]: + predictions=model.predict(group,WH,bb) + for l,p in zip(group,predictions):l['printed_phase']=p['phase'];l['printed_rank']=p['rank'] + return dict(applied=True,recovered=recovered,original=original,protected=protected,WH=WH,body_box=bb) +def apply(baseline,prepared,params): + if not prepared.get('applied'):return baseline,dict(applied=False,reason='No reliable separated layers') + source=prepared['original']if params['source']=='original'else prepared['recovered'];source=list(source) + if params['source']=='recovered': + for old in prepared['original']: + if not any(overlap(old['box'],new['box'])>.6 for new in source):source.append(old) + selected=[dict(l)for l in source if l['admission']>=params['admission']and l['confidence']>=params['read_confidence']and l['text'].strip()] + selected.sort(key=lambda l:(-l['admission']*l['confidence'],l['box'][1],-l['box'][0]));dedup=[] + for l in selected: + if any(overlap(l['box'],x['box'])>=.7 for x in dedup):continue + dedup.append(l) + selected=dedup + quality=float(np.median([l['admission']*l['confidence']for l in selected]))if selected else 0. + if quality=.5],key=lambda l:(l['printed_rank'],l['column'],l['baseline']));lines=prefix+body+suffix + else: + entries=[(l,False)for l in body]+[(l,True)for l in selected];entries.sort(key=lambda q:(q[0].get('column',0),q[0].get('baseline',q[0]['box'][3]),-q[0]['box'][0]));it=iter(body);lines=[l if printed else next(it)for l,printed in entries] + assert[key(l)for l in lines if l.get('layer')!='printed']==[key(l)for l in body] + result=copy.deepcopy(baseline);result.update(lines=lines,text='\n'.join(l['text']for l in lines),regions=[dict(path='hybrid_printed_layer_fixed95k_body',lines=lines,text='\n'.join(l['text']for l in lines))],emitted_boxes=[l['box']for l in lines],recognition_extents=[reading_extent(l)for l in lines]);result['handwriting_extents']=[reading_extent(l)for l in lines if l.get('layer')=='handwriting'];info=dict(applied=True,quality=quality,printed_count=len(selected),body_count=len(body),body95k_bytes_and_order_unchanged=True,parameters=params);result['printed_recovery']=info + return result,info diff --git a/stallion/bina04-printed-form-fix-20261102/label_model.py b/stallion/bina04-printed-form-fix-20261102/label_model.py new file mode 100644 index 0000000000000000000000000000000000000000..7685508dceb0563aa89444f4d8bffa98a7298d70 --- /dev/null +++ b/stallion/bina04-printed-form-fix-20261102/label_model.py @@ -0,0 +1,30 @@ +"""Generator-trained printed-label admission/serialization; no template IDs.""" +import json,pickle,hashlib,re +from pathlib import Path +import numpy as np +from sklearn.feature_extraction.text import HashingVectorizer +from rapidfuzz.fuzz import ratio +from shenavasanj import fair_text +HASH=HashingVectorizer(analyzer='char',ngram_range=(1,3),n_features=128,alternate_sign=False,norm='l2',lowercase=False) +def norm(s):return fair_text(s,keep_spaces=False) +def feature(text,box,WH,body): + W,H=WH;x,y,X,Y=box;a,b,A,B=body + return np.r_[HASH.transform([norm(text)]).toarray()[0],x/W,y/H,X/W,Y/H,(x+X)/2/W,(y+Y)/2/H,(X-x)/W,(Y-y)/H,(y-b)/max(1,B-b),(x-a)/max(1,A-a),float(YB),W/H].astype(np.float32) +class LabelModel: + def __init__(self,root): + self.root=Path(root);self.model=pickle.loads((self.root/'printed-label-model.pkl').read_bytes());self.lexicon=json.loads((self.root/'SERIALIZED-LEXICON.json').read_text());self.words=[s for s in self.lexicon if len(s)>=3] + def admission(self,text): + s=norm(text) + if not s:return 0. + exact=max((ratio(s,w)/100 for w in self.lexicon),default=0) + # Explain merged fields by serialized vocabulary, retaining actual OCR bytes. + score=np.zeros(len(s)+1) + for i in range(len(s)): + score[i+1]=max(score[i+1],score[i]) + for w in self.words: + if s.startswith(w,i):score[i+len(w)]=max(score[i+len(w)],score[i]+len(w)) + return max(exact,float(score[-1]/len(s))) + def predict(self,lines,WH,body): + if not lines:return[] + X=np.asarray([feature(l['text'],l['box'],WH,body)for l in lines]);means=self.model.predict(X) + return[dict(phase=float(v[0]),rank=float(v[1]),admission=self.admission(l['text']))for l,v in zip(lines,means)] diff --git a/stallion/bina04-printed-form-fix-20261102/printed-label-model.pkl b/stallion/bina04-printed-form-fix-20261102/printed-label-model.pkl new file mode 100644 index 0000000000000000000000000000000000000000..ae809cac2689c2aaa5a59dcea613809217cf0e01 --- /dev/null +++ b/stallion/bina04-printed-form-fix-20261102/printed-label-model.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf02d5006aad71a2383d69b5c1db01579b4effe88fec4692eafd50e83772fc45 +size 12109247 diff --git a/stallion/bina04-printed-form-fix-20261102/production.py b/stallion/bina04-printed-form-fix-20261102/production.py new file mode 100644 index 0000000000000000000000000000000000000000..45b537881f8ea753a99feb1534e0aa55d60a163d --- /dev/null +++ b/stallion/bina04-printed-form-fix-20261102/production.py @@ -0,0 +1,56 @@ +"""Consumer runtime for the accepted printed-only hybrid. Existing body95k frozen.""" +import copy,json +from pathlib import Path +import numpy as np +from rtl_reader import RTLReader +from row_pipeline import RowPipeline +from row_builder import layer_masks,DEFAULT +from baseline.pipeline_v3 import flip,markdown_cell,reading_extent +from baseline.input_policy_v3 import input_kind +from label_model import LabelModel,norm +from candidate_apply import prepare,apply +R=Path(__file__).resolve().parent +SHORT=Path('__BINA_ROOT__/stallion/bina04-order-form-batch-20261101/recognizer/weights-001000.pt') +VOC=Path('__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json') +class Expert: + def __init__(self,device='cuda'): + self.short=RTLReader(SHORT,VOC,device=device);self.model=LabelModel(R);self.params=json.loads((R/'CALIBRATION.json').read_text())['selected']['parameters'];assert self.params['source']=='original' + def read(self,gray,boxes): + if not boxes:return[] + _,receipts=self.short(gray,np.asarray(boxes,np.float32)) + return[dict(box=list(map(float,b)),text=flip(g['raw_visual_text'],base_rtl=True),confidence=g['confidence'],visual_text=g['raw_visual_text'],geometry=g)for b,g in zip(boxes,receipts)] + def page(self,image,baseline): + if not baseline.get('row_consolidation',{}).get('applied'):return baseline,dict(applied=False) + rgb=np.asarray(image.convert('RGB'));gray=np.asarray(image.convert('L'));proposals=baseline.get('detected_boxes',[l['box']for l in baseline['lines']]);_,_,_,trace=layer_masks(rgb,proposals,DEFAULT) + if not trace['reliable']:return baseline,dict(applied=False) + indices=[i for i,l in enumerate(baseline['lines'])if l.get('layer')=='printed'];reads=self.read(gray,[baseline['lines'][i]['box']for i in indices]);printed=dict(applied=True,candidates=[],original_short_reads=[dict(index=i,short=l)for i,l in zip(indices,reads)]) + prepared=prepare(baseline,printed,image.size,self.model);return apply(baseline,prepared,self.params) + def cells(self,image,baseline): + result=copy.deepcopy(baseline);gray=np.asarray(image.convert('L'));changes=[] + for region in result.get('regions',[]): + trace=region.get('table_trace',{});cells=trace.get('cell_debug',[]) + if not trace.get('grid')or not cells:continue + eligible=[l for cell in cells for l in cell['lines']if 0.45 + use=bool(text.strip())and read['confidence']>=max(.8,old.get('confidence',0)+.01)and(new_admit>=old_admit+.05 or numeric) + if use: + new=dict(old,**read,recognizer='short-label32M_printed_cell');mapped[tuple(old['box'])]=new;changes.append(dict(old=old['text'],new=text)) + if not mapped:continue + values={} + for cell in cells: + ll=[mapped.get(tuple(l['box']),l)for l in cell['lines']];cell['lines']=ll;cell['text']=' '.join(l['text'].strip()for l in ll if l['text'].strip());values[(cell['row'],cell['col'])]=cell['text'] + rendered=[] + for i,r in enumerate(trace['kept_rows']): + rendered.append('| '+' | '.join(markdown_cell(values.get((r,c),''))for c in trace['kept_columns'])+' |') + if i==0:rendered.append('| '+' | '.join('---'for _ in trace['kept_columns'])+' |') + region['text']='\n'.join(rendered);region['lines']=[l for c in cells for l in c['lines']] + if not changes:return baseline,dict(applied=False) + result['text']='\n\n'.join(r['text']for r in result['regions']);result['lines']=[l for r in result['regions']for l in r['lines']];result['recognition_extents']=[reading_extent(l)for l in result['lines']];result['emitted_boxes']=[l['box']for l in result['lines']];return result,dict(applied=True,changes=changes) +class HybridPipeline(RowPipeline): + def __init__(self,*args,**kwargs): + super().__init__(*args,**kwargs);self.expert=Expert(str(self.reader.reader.device)) + def __call__(self,image,*,input_type='auto',**kwargs): + base=super().__call__(image,input_type=input_type,**kwargs) + if input_kind(*image.size,input_type)=='page':return self.expert.page(image,base)[0] + return self.expert.cells(image,base)[0] diff --git a/stallion/bina04-printed-layer-training-20261102/run/weights-002000.pt b/stallion/bina04-printed-layer-training-20261102/run/weights-002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..8407a5e624329e3bd9d779905bdcb6d7b8fe1591 --- /dev/null +++ b/stallion/bina04-printed-layer-training-20261102/run/weights-002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:945af1a31fd6ccab1d73185e72679f21d91a0500be730085baf2d2dde5aa861a +size 132814787 diff --git a/stallion/bina04-quality-route-printscope-95k-20261103/QUALITY-POLICY.json b/stallion/bina04-quality-route-printscope-95k-20261103/QUALITY-POLICY.json new file mode 100644 index 0000000000000000000000000000000000000000..1aadcd0936f56df57453c7abe030d22ae1ef15ec --- /dev/null +++ b/stallion/bina04-quality-route-printscope-95k-20261103/QUALITY-POLICY.json @@ -0,0 +1,66 @@ +{ + "feature_names": [ + "contrast", + "ink_contrast", + "relative_laplacian", + "relative_gradient", + "transition_fraction", + "paper_noise", + "speckle_fraction", + "speckles_per_width", + "stroke_width", + "ink_fraction" + ], + "mean": [ + 0.8422177327861748, + 0.5744400918463937, + 0.01635437548011239, + 0.377607064997231, + 0.16215655298260115, + 0.017505274839723573, + 0.002316789666629887, + 0.8937624698886076, + 4.083546191104623, + 0.1640884655254998 + ], + "scale": [ + 0.15999053536119812, + 0.13500571380155046, + 0.023267893145580025, + 0.08845224825644835, + 0.11635875526333239, + 0.007827086596874809, + 0.017874443142238943, + 7.26444052220246, + 2.1905681975667193, + 0.0513821599435976 + ], + "coef": [ + 2.703745443877514, + -1.8811209808333142, + -0.6591562267723111, + -0.16981764547764933, + -0.7842811224592784, + -0.9232664864731612, + -1.0093650228340463, + 0.4022309918631508, + -0.7287338392493663, + 0.569905327313594 + ], + "intercept": -0.20016467413966482, + "threshold": 0.7725372020975191, + "calibration": "Image features only; source-disjoint private real dev and private printed600. At most1/600 clean calibration images routed. No193/dev50/printed338/real7 calibration.", + "training_samples": { + "clean": 2000, + "degraded": 3375 + }, + "private_calibration": { + "clean": 600, + "degraded": 986, + "clean_routed": 1, + "degraded_routed": 417, + "AUC": 0.9176588911426639 + }, + "OCR_text_or_confidence_features": false, + "recognizers_frozen": true +} \ No newline at end of file diff --git a/stallion/bina04-quality-route-printscope-95k-20261103/print_scope.py b/stallion/bina04-quality-route-printscope-95k-20261103/print_scope.py new file mode 100644 index 0000000000000000000000000000000000000000..2318556918610193eb03b59ada93a3312e68a9f6 --- /dev/null +++ b/stallion/bina04-quality-route-printscope-95k-20261103/print_scope.py @@ -0,0 +1,11 @@ +"""Image-only conservative guard for confidently separated coloured handwriting.""" +import importlib.util +import cv2,numpy as np +from PIL import Image +s=importlib.util.spec_from_file_location('frozen_image_pen_separator','__BINA_ROOT__/stallion/bina04-order-salvage-20261101/row_builder.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m) +def handwriting_guard(image): + im=image.convert('RGB');scale=min(1.,768/max(im.size)) + if scale<1:im=im.resize((round(im.width*scale),round(im.height*scale)),Image.Resampling.BILINEAR) + rgb=np.asarray(im);chroma=rgb.max(2).astype('int16')-rgb.min(2).astype('int16');coloured=int(((chroma>12)&(rgb.mean(2)<220)).sum()) + if coloured<80:return dict(protected=False,reason='No substantial coloured ink',image_only=True) + _,_,_,trace=m.layer_masks(rgb,[],m.DEFAULT);return dict(protected=bool(trace['reliable']),reason='Confident separated colour layers'if trace['reliable']else'No confident separated handwriting',image_only=True,measurement_WH=list(im.size),coloured_pixels=coloured,separator_trace={k:v for k,v in trace.items()if k!='_core_mask'}) diff --git a/stallion/bina04-quality-route-printscope-95k-20261103/production.py b/stallion/bina04-quality-route-printscope-95k-20261103/production.py new file mode 100644 index 0000000000000000000000000000000000000000..f9c79921ae1f1823b4b6aa411065f5ec299c110a --- /dev/null +++ b/stallion/bina04-quality-route-printscope-95k-20261103/production.py @@ -0,0 +1,40 @@ +"""Frozen95k base + image-quality-selected degraded-print expert.""" +from pathlib import Path +import importlib.util,sys,json,hashlib +import numpy as np +R=Path(__file__).resolve().parent +sys.path.insert(0,str(R)) +from quality import Quality +from print_scope import handwriting_guard + +def install(raw,expert,policy,label): + Original=type(raw) + class ImageQualityReader(Original): + def __call__(self,gray,boxes): + from geometry import crop_line64 + if self.quality_scope_protected:return Original.__call__(self,gray,boxes) + choices=[policy(crop_line64(gray,b)[0])for b in boxes] # BEFORE either OCR read; no text/confidence inputs. + readings,receipts=Original.__call__(self,gray,boxes) # Preserve exact base batching/clean outputs. + ids=[i for i,(route,_)in enumerate(choices)if route] + if ids: + rr,gg=expert(gray,np.asarray([boxes[i]for i in ids],np.float32)) + for j,i in enumerate(ids):readings[i]=rr[j];receipts[i]=gg[j] + for i,(_,audit)in enumerate(choices): + receipts[i]['quality_routed']=i in ids;receipts[i]['image_quality']=audit;self.quality_trace.append(dict(reader=label,box=list(map(float,boxes[i])),routed=i in ids,**audit)) + return readings,receipts + raw.__class__=ImageQualityReader;raw.quality_trace=[];raw.quality_scope_protected=False + return raw +class Runtime: + def __init__(self,device='auto'): + s=importlib.util.spec_from_file_location('quality_frozen_parent','__BINA_ROOT__/stallion/bina04-line-geometry-95k-20261103/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);self.raw=self.parent.raw;self.core=self.parent.core + from rtl_reader import RTLReader + import torch + if device=='auto':device='cuda'if torch.cuda.is_available()else'cpu' + cp=Path('__BINA_ROOT__/stallion/bina04-real-degraded-95k-20261103/recognizer/selected.pt');v=Path('__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json');self.expert=RTLReader(cp,v,device=device);self.policy=Quality(R/'QUALITY-POLICY.json');self.expert_sha=hashlib.sha256(cp.read_bytes()).hexdigest();self.policy_sha=hashlib.sha256((R/'QUALITY-POLICY.json').read_bytes()).hexdigest();self.checkpoint_sha256=hashlib.sha256((self.expert_sha+self.policy_sha).encode()).hexdigest();self.selected_step=2000;self.scope_sha=hashlib.sha256((R/'print_scope.py').read_bytes()).hexdigest();self.checkpoint_sha256=hashlib.sha256((self.checkpoint_sha256+self.scope_sha).encode()).hexdigest() + self.readers=[install(self.raw,self.expert,self.policy,'primary95k'),install(self.parent.reader.reader,self.expert,self.policy,'short_band_primary95k')] + def __call__(self,image,old,kind,name='',**kw): + scope=handwriting_guard(image)if kind=='page'else dict(protected=False,reason='Crop input; no page-level scope guard',image_only=True) + for r in self.readers:r.quality_trace.clear();r.quality_scope_protected=scope['protected'] + out=self.parent(image,old,kind,name,**kw);trace=[q for r in self.readers for q in r.quality_trace];out['image_quality_routing']=dict(base_checkpoint=95000,expert_checkpoint=str('__BINA_ROOT__/stallion/bina04-real-degraded-95k-20261103/recognizer/selected.pt'),expert_sha256=self.expert_sha,policy_sha256=self.policy_sha,quality_before_OCR=True,handwriting_scope_guard=scope,scope_sha256=self.scope_sha,recognized_text_or_confidence_used=False,reads=len(trace),routed=sum(q['routed']for q in trace),trace=trace,other_experts_unchanged=True) + return out + def close(self):self.parent.close() diff --git a/stallion/bina04-quality-route-printscope-95k-20261103/quality.py b/stallion/bina04-quality-route-printscope-95k-20261103/quality.py new file mode 100644 index 0000000000000000000000000000000000000000..a3fbbbd6b97d9a187b9190a3e566741c016d68e9 --- /dev/null +++ b/stallion/bina04-quality-route-printscope-95k-20261103/quality.py @@ -0,0 +1,9 @@ +"""Image-only degradation features on the recognizer's canonical64px crop.""" +import cv2,numpy as np,json +NAMES=['contrast','ink_contrast','relative_laplacian','relative_gradient','transition_fraction','paper_noise','speckle_fraction','speckles_per_width','stroke_width','ink_fraction'] +def features(ink): + gray=np.clip((1.-ink)*255,0,255).astype('uint8');a=gray.astype('float32');hi,lo=np.percentile(a,[98,2]);contrast=max(1.,hi-lo);bg=float(np.median(a));mask=(a0;paper=float(np.std((a-smooth)[far]))if far.any()else 0.;n,_,stats,_=cv2.connectedComponentsWithStats(mask,8);areas=stats[1:,cv2.CC_STAT_AREA]if n>1 else np.array([]);speck=areas[areas<=3];dist=cv2.distanceTransform(mask,cv2.DIST_L2,3);stroke=float(np.median(dist[mask>0])*2)if mask.any()else 0.;mid=((a>lo+.15*contrast)&(a0].mean()))/255 if mask.any()else 0.,float(lap.var())/(contrast**2),float(np.mean(abs(grad)))/contrast,float(mid),paper/contrast,float(speck.sum())/max(1,float(mask.sum())),len(speck)/max(1,a.shape[1])*100,stroke,float(mask.mean())] +class Quality: + def __init__(self,path):self.policy=json.load(open(path));self.mean=np.array(self.policy['mean']);self.scale=np.array(self.policy['scale']);self.coef=np.array(self.policy['coef']);self.intercept=self.policy['intercept'];self.threshold=self.policy['threshold'] + def __call__(self,ink): + f=np.array(features(ink));z=float(((f-self.mean)/self.scale)@self.coef+self.intercept);score=float(1/(1+np.exp(-np.clip(z,-40,40))));return score>=self.threshold,dict(score=score,threshold=self.threshold,features=dict(zip(NAMES,f.tolist())),features_from_image_only=True,recognized_text_used=False,OCR_confidence_used=False) diff --git a/stallion/bina04-real-degraded-95k-20261103/recognizer/selected.pt b/stallion/bina04-real-degraded-95k-20261103/recognizer/selected.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc1f91dadfec29b52665de2bf6fe3b5f2091bd35 --- /dev/null +++ b/stallion/bina04-real-degraded-95k-20261103/recognizer/selected.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8867b309e881b698685e5b6e06a37c9f495e04dcf7027a044ad44c13cf8abd64 +size 131254078 diff --git a/stallion/bina04-small-crossing-rows-20261102/crossing_rows.py b/stallion/bina04-small-crossing-rows-20261102/crossing_rows.py new file mode 100644 index 0000000000000000000000000000000000000000..4acadb72cc51833559f3384ffdcf29179787f601 --- /dev/null +++ b/stallion/bina04-small-crossing-rows-20261102/crossing_rows.py @@ -0,0 +1,53 @@ +"""Isolated image-only full-row recovery for small neutral-ink grid forms.""" +import copy,cv2,numpy as np +import row_builder as A + +def physical_rows(rgb,boxes): + H,W=rgb.shape[:2];gray=cv2.cvtColor(rgb,cv2.COLOR_RGB2GRAY);height=A.estimate_height(boxes,H) + vertical=cv2.morphologyEx((gray<130).astype(np.uint8),cv2.MORPH_OPEN,np.ones((max(30,round(.18*H)),1),np.uint8));occupied=vertical.sum(0)>.18*H;edges=np.diff(np.r_[False,occupied,False].astype(int));rulings=[] + for a,b in zip(np.where(edges==1)[0],np.where(edges==-1)[0]): + if not rulings or a-rulings[-1][1]>.01*W:rulings.append([a,b]) + else:rulings[-1][1]=b + internal=[round((a+b)/2)for a,b in rulings if .15*W<(a+b)/2<.85*W] + trace=dict(physical_rulings=rulings,internal_rulings=internal,estimated_height=height) + if len(internal)<5:return [],dict(trace,reason='Not a ruled form') + # Per-row paper level cancels shaded grid bands before ink thresholding. + paper=np.percentile(gray,65,axis=1)[:,None];mask=(paper-gray)>35;mask,_=A.remove_rules(mask,height) + rows,rt=A.build_rows(mask,boxes,A.DEFAULT,regions=[(0,W)],core=mask);group=[] + for row in rows: + if group and abs(row['baseline']-group[-1]['baseline'])<.25*height: + old=group[-1];old['box']=[min(old['box'][0],row['box'][0]),min(old['box'][1],row['box'][1]),max(old['box'][2],row['box'][2]),max(old['box'][3],row['box'][3])] + else:group.append(copy.deepcopy(row)) + seeds=[] + for i,r in enumerate(group): + x,y,X,Y=r['box'];yi,Yi=max(0,int(y)),min(H,int(Y));d=max(3,round(.4*height));gap=max(2,round(.1*height));support=[] + for z in internal: + a=mask[yi:Yi,max(0,z-d):max(0,z-gap)].sum();b=mask[yi:Yi,min(W,z+gap):min(W,z+d)].sum();support.append(float(min(a,b))) + r['crossing_support']=support;r['crossings']=sum(v>=max(5,.015*height*height)for v in support) + if X-x>.45*W and r['crossings']>=3:seeds.append(i) + if len(seeds)<6:return [],dict(trace,reason='Insufficient cross-grid writing rows',seed_count=len(seeds)) + first=group[seeds[0]]['box'][1];last=group[seeds[-1]]['box'][3];selected=[] + for r in group: + if not first<=r['baseline']<=last:continue + b=r['box'];matches=[p for p in boxes if 0.1*W] + # The detector can retain a valid short row; the profile must not append + # distant cell labels or ruling remnants to that row. + if matches: + best=max(matches,key=lambda p:p[2]-p[0]);b=list(best) + if b[2]-b[0]<.1*W:continue + r['box']=b;selected.append(r) + trace.update(seed_count=len(seeds),body_band=[first,last],row_count=len(selected),handwriting_rows=rt,rows=selected,reason='Cross-grid physical handwriting rows; cell slicing bypassed',paper_normalized_for_geometry_only=True) + return selected,trace + +def apply(image,baseline,reader): + rgb=np.asarray(image.convert('RGB'));rows,trace=physical_rows(rgb,baseline['detected_boxes']) + if not rows:return baseline,dict(trace,applied=False) + reader.rule_c_active=False;reader.source_is_crop=False;hand=reader(np.asarray(image.convert('L')),[r['box']for r in rows]) + for line,r in zip(hand,rows):line.update(layer='handwriting',baseline=r['baseline'],column=0,source_pixels_modified=False,full_row_across_cells=True) + # Preserve printed cell reads, never the table's fragmented handwriting reads. + first,last=trace['body_band'];printed=[] + for l in baseline['lines']: + b=l['box'];yc=(b[1]+b[3])/2 + if first-.5*trace['estimated_height']<=yc<=last+.5*trace['estimated_height']:continue + q=copy.deepcopy(l);q.update(layer='printed',baseline=b[3],column=0);printed.append(q) + lines=sorted(hand+printed,key=lambda q:(q['baseline'],-q['box'][0]));result=copy.deepcopy(baseline);result.update(lines=lines,global_readings=lines,text='\n'.join(l['text']for l in lines),regions=[dict(path='cross_grid_hand_rows_printed_separate',lines=lines,text='\n'.join(l['text']for l in lines))],emitted_boxes=[l['box']for l in lines],handwriting_extents=[l['box']for l in hand],row_consolidation=dict(applied=True,physical_cross_grid=True,**trace));return result,dict(trace,applied=True) diff --git a/stallion/bina04-small-crossing-rows-20261102/runtime.py b/stallion/bina04-small-crossing-rows-20261102/runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..893e57e61ed7cb2e72868d799cc322cbbb109806 --- /dev/null +++ b/stallion/bina04-small-crossing-rows-20261102/runtime.py @@ -0,0 +1,34 @@ +"""Frozen small-page candidate; only page inputs below 1000px height change.""" +import hashlib,importlib.util,json,sys +from pathlib import Path +R=Path(__file__).resolve().parent;J=Path('__BINA_ROOT__/stallion/bina04-e2e-batch-95k-20261101/candidates/combined');S=J/'snapshot/runtime';OLD=Path('__BINA_ROOT__/stallion/bina04-printed-form-fix-20261102');CURRENT=Path('__BINA_ROOT__/stallion/bina04-print-textgate-20261102');P=Path('__BINA_ROOT__/stallion/indra-scale') +sys.path[:0]=[str(R),str(CURRENT),str(OLD),str(J/'versions/combined'),str(J),str(S),str(S/'baseline'),str(P),str(P/'scoringTT')] +def load(name,path): + spec=importlib.util.spec_from_file_location(name,path);m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);return m +class SmallPagePatch: + def __init__(self,device='cuda'): + import torch + from surya.detection import DetectionPredictor + from line_contract_reader import ContractReader + from learned_backend import install + from baseline.table_client import SuryaTableGrid + self.detector=DetectionPredictor.local(device=device,dtype=torch.float32);self.detector._disable_tqdm=True;cp=J/'run/checkpoints/step-001000.pt';self.detector.model.load_state_dict(torch.load(cp,map_location='cpu',weights_only=True)['model']);self.detector.model.eval();install(self.detector,hashlib.sha256(cp.read_bytes()).hexdigest());self.reader=ContractReader('__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031/run/weights-095000.pt','__BINA_ROOT__/stallion/bina04-pagepath-v3-round3-clean-20261031/data/vocab.json',device=device,cache=R/'reader-cache');self.context={};context=self.context + class Layout: + def run_batch(self,images,params): + class Result: + def model_dump(self,**kwargs):return context['layout'] + return[Result()] + self.table=SuryaTableGrid();self.base=load('small_patch_original_hybrid',OLD/'production.py').HybridPipeline(self.detector,Layout(),self.reader,self.table,variant='combined-news4-coursefirst',row_parameters={});self.overlay=load('small_patch_original_overlay',CURRENT/'production.py').PageOverlay(device) + @staticmethod + def eligible(image,*,input_type='page'): + return input_type=='page' and image.height<1000 + def __call__(self,image,baseline,*,input_type='page',source_name=''): + if not self.eligible(image,input_type=input_type):return baseline + from PIL import Image + from scale_utils import source_coordinates + from crossing_rows import apply + image=image.convert('RGB');work=image.resize((image.width*2,image.height*2),Image.Resampling.BICUBIC);self.context['layout']=source_coordinates(dict(raw_layout=baseline['raw_layout']),.5,image.size)['raw_layout'];result=self.base(work,input_type='page',source_name=source_name) + if not result.get('row_consolidation',{}).get('applied'):result,trace=apply(work,result,self.reader) + else:trace=dict(applied=False,reason='Preserve reliable A row construction') + result['cross_grid_recovery']=trace;result=self.overlay(work,result);return source_coordinates(result,2,image.size) + def close(self):self.table.close() diff --git a/stallion/bina04-small-crossing-rows-20261102/runtime_release.py b/stallion/bina04-small-crossing-rows-20261102/runtime_release.py new file mode 100644 index 0000000000000000000000000000000000000000..b7f993e41ae33757bd7b9b79eac3b6e0607c9fca --- /dev/null +++ b/stallion/bina04-small-crossing-rows-20261102/runtime_release.py @@ -0,0 +1,22 @@ +"""Receipt-only normalization for newly rebuilt handwriting rows; OCR is frozen.""" +import copy +from runtime import SmallPagePatch + +def normalize_receipts(result): + if not result.get('small_page_upscale',{}).get('applied'):return result + out=copy.deepcopy(result) + def extent(line): + g=line.get('geometry',{}) + return g.get('source_pixels_extent',g.get('padded_source_extent',line['box'])) + for line in out['lines']: + line['source_pixels_modified']=True + line['source_pixel_transform']='Bicubic 2x source; no ink or ruling edits for recognition' + out['recognition_extents']=[extent(l)for l in out['lines']] + out['handwriting_extents']=[extent(l)for l in out['lines']if l.get('layer')=='handwriting'] + out['geometry_receipt_correction']=dict(kind='Actual padded source extents of emitted reads',text_unchanged=True,boxes_unchanged=True,neural_inference_unchanged=True) + assert out['text']==result['text'] + assert[(l['text'],l['box'])for l in out['lines']]==[(l['text'],l['box'])for l in result['lines']] + return out + +class ValidatedSmallPagePatch(SmallPagePatch): + def __call__(self,*args,**kwargs):return normalize_receipts(super().__call__(*args,**kwargs)) diff --git a/stallion/bina04-small-crossing-rows-20261102/scale_utils.py b/stallion/bina04-small-crossing-rows-20261102/scale_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..295f1ee8d2b40c82f855b4fcc285500b75bf72cb --- /dev/null +++ b/stallion/bina04-small-crossing-rows-20261102/scale_utils.py @@ -0,0 +1,21 @@ +"""Keep source coordinates while detection and recognition use a larger working image.""" +import json,numpy as np +XY4={'box','bbox','image_bbox','ink_box','padded_source_extent','source_pixels_extent','source_extent','read_band_box','text_region','tile','crop_box'} +BOXES={'boxes','detected_boxes','emitted_boxes','recognition_extents','handwriting_extents','primary_boxes','supplemental_boxes'} +DIMS={'original_WH','padded_source_WH','original_size','WH'} +def source_coordinates(result,scale,original_WH): + if scale==1:return result + out=json.loads(json.dumps(result,ensure_ascii=False,default=lambda v:v.tolist()if hasattr(v,'tolist')else str(v))) + def numbers(v):return isinstance(v,list)and all(isinstance(x,(float,int))for x in v) + def walk(x,key=''): + if isinstance(x,dict):return{k:walk(v,k)for k,v in x.items()} + if key in XY4 and numbers(x)and len(x)==4:return[v/scale for v in x] + if key=='polygon'and isinstance(x,list):return[[float(v)/scale for v in p]for p in x] + if key in BOXES and isinstance(x,list): + return[([v/scale for v in b[:4]]+b[4:])if numbers(b)and len(b)>=4 else walk(b)for b in x] + if key in DIMS and numbers(x)and len(x)==2:return[v/scale for v in x] + if key=='columns'and isinstance(x,list)and all(numbers(c)and len(c)==2 for c in x):return[[v/scale for v in c]for c in x] + if key=='baseline'and isinstance(x,(int,float)):return x/scale + if isinstance(x,list):return[walk(v)for v in x] + return x + out=walk(out);out['small_page_upscale']=dict(applied=True,scale=scale,source_WH=list(original_WH),working_WH=[round(v*scale)for v in original_WH],detection_and_reader_pixels='Upscaled source image, no width squashing',output_coordinates='Original source image',source_weights_unchanged=True);return out diff --git a/stallion/bina04-smallmarks-celllabels-95k-20261102/decompose.py b/stallion/bina04-smallmarks-celllabels-95k-20261102/decompose.py new file mode 100644 index 0000000000000000000000000000000000000000..a4afeab1e26ccdf1714ecc7b5212551ab23b9553 --- /dev/null +++ b/stallion/bina04-smallmarks-celllabels-95k-20261102/decompose.py @@ -0,0 +1,13 @@ +"""Explicit-ID adapter executing the user's matching loop without modifications.""" +import ast +from pathlib import Path +from rapidfuzz.distance import Levenshtein as L + +def score(references,predictions,script): + tree=ast.parse(Path(script).read_text()) + loop=next(n for n in tree.body if isinstance(n,ast.For) and isinstance(n.target,ast.Name) and n.target.id=='i' and isinstance(n.iter,ast.Name) and n.iter.id=='ids') + env=dict(L=L,labels=references,pred=predictions,ids=list(references),n=lambda s:''.join(s.split()),T=0,Ltot=0, + agg=dict(match_short=0,match_long=0,missing_short=0,missing_long=0,extra=0,ordered=0)) + exec(compile(ast.Module(body=[loop],type_ignores=[]),str(script),'exec'),env) + return dict(pages=len(references),reference_chars=env['T'],total_CER=100*env['Ltot']/max(1,env['T']), + **{k:100*v/max(1,env['T']) for k,v in env['agg'].items()}) diff --git a/stallion/bina04-surya25-order-20261101/role-model/weights-003000.pt b/stallion/bina04-surya25-order-20261101/role-model/weights-003000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e43b251ec4fa089a01ea6c079271cf4a870d2a2e --- /dev/null +++ b/stallion/bina04-surya25-order-20261101/role-model/weights-003000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3312b2fd0ae6f015cc51082d3d62af33a3251d8f0226c6e8d4b9171043927db +size 5093599 diff --git a/stallion/bina04-surya25-order-20261101/role_model.py b/stallion/bina04-surya25-order-20261101/role_model.py new file mode 100644 index 0000000000000000000000000000000000000000..203637290cdba1bd16d166728fa6d50b0bf3a8d6 --- /dev/null +++ b/stallion/bina04-surya25-order-20261101/role_model.py @@ -0,0 +1,85 @@ +"""Small observed-node order/role student. Recognition and node geometry stay frozen.""" +import numpy as np +import torch +from torch import nn +from role_nodes import FEATURES, features, nodes_from_result, render_roles + + +class RoleNet(nn.Module): + def __init__(self): + super().__init__() + self.input = nn.Linear(FEATURES, 128) + block = nn.TransformerEncoderLayer(128, 4, 256, dropout=0., batch_first=True, norm_first=True) + self.encoder = nn.TransformerEncoder(block, 3, enable_nested_tensor=False) + self.rank = nn.Linear(128, 1) + self.role = nn.Linear(128, 4) + nn.init.zeros_(self.rank.weight) + nn.init.zeros_(self.rank.bias) + + def forward(self, x, pad): + z = self.encoder(self.input(x), src_key_padding_mask=pad) + delta = self.rank(z).squeeze(-1).tanh() + return (x[:, :, 0] + delta).clamp(0, 1), self.role(z), delta + + +class RoleEnsemble: + def __init__(self, path): + checkpoint = torch.load(path, map_location='cpu', weights_only=True) + self.models = [] + for state in checkpoint['models']: + model = RoleNet() + model.load_state_dict(state) + model.eval() + self.models.append(model) + + @torch.inference_mode() + def propose(self, result, image): + nodes = nodes_from_result(result) + n = len(nodes) + if not 2 <= n <= 512: + return result, dict(changed=False, confidence=0., reason='Outside trained node context') + x = torch.from_numpy(features(nodes, image, result))[None] + pad = torch.zeros(1, n, dtype=torch.bool) + ranks, probabilities = [], [] + for model in self.models: + rank, logits, _ = model(x, pad) + ranks.append(rank[0].numpy()) + probabilities.append(logits[0].softmax(-1).numpy()) + ranks, probabilities = np.asarray(ranks), np.asarray(probabilities) + roles = np.zeros(n, np.int64) + roles[probabilities[:, :, 1].min(0) >= .70] = 1 + roles[probabilities[:, :, 2].min(0) >= .95] = 2 + roles[probabilities[:, :, 3].min(0) >= .995] = 3 + roles[[i for i, node in enumerate(nodes) if node['kind'] == 'table']] = 0 + proposal, order, body = render_roles(result, nodes, ranks.mean(0), roles) + changed = order != list(range(n)) + if not changed: + return result, dict(changed=False, confidence=0., node_count=n, body_nodes=len(body)) + # Group ranks, not individual body ranks: body rows retain their baseline order. + effective = ranks.copy() + if body: + effective[:, body] = np.median(ranks[:, body], axis=1)[:, None] + positions = {node: position for position, node in enumerate(order)} + confidence_parts, agreements = [], [] + changed_pairs = 0 + for i in range(n): + for j in range(i + 1, n): + if i not in positions or j not in positions or positions[i] < positions[j]: + continue + if i in body and j in body: + raise AssertionError('Body relative order changed') + difference = effective[:, i] - effective[:, j] + z = float(difference.mean() / np.sqrt(difference.var() + .0004)) + confidence_parts.append(float(1 / (1 + np.exp(-np.clip(z, -30, 30))))) + agreements.append(float((difference > 0).mean())) + changed_pairs += 1 + for i, role in enumerate(roles): + if role != 0: + confidence_parts.append(float(probabilities[:, i, role].min())) + agreements.append(1.) + confidence = min(float(np.quantile(confidence_parts, .1)), float(np.mean(agreements))) if confidence_parts else 0. + return proposal, dict(changed=True, confidence=confidence, node_count=n, + changed_pair_count=changed_pairs, body_nodes=len(body), + excluded=int((roles == 3).sum()), appended=int((roles == 2).sum()), + node_order=order, roles=roles.tolist(), body_order_preserved=True, + confidence_method='Changed-group pair margins, ensemble agreement and role confidence; threshold calibrated on cal25 only') diff --git a/stallion/bina04-surya25-order-20261101/role_nodes.py b/stallion/bina04-surya25-order-20261101/role_nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..3f42f0bffd2cb5ca66ad426438d12104020d3f80 --- /dev/null +++ b/stallion/bina04-surya25-order-20261101/role_nodes.py @@ -0,0 +1,25 @@ +"""Observed-node features and learned-body grouping. No template IDs or gold input.""" +import numpy as np,cv2 +from order_nodes import features as base_features,nodes_from_result,render,area,intersect +FEATURES=176 + +def features(nodes,image,result): + old=base_features(nodes,image,result);rgb=np.asarray(image.convert('RGB'));H,W=rgb.shape[:2];extra=[] + for n in nodes: + x,y,X,Y=n['box'];a,b,A,B=max(0,int(x)),max(0,int(y)),min(W,int(np.ceil(X))),min(H,int(np.ceil(Y)));patch=rgb[b:B,a:A].astype(np.float32)/255.;z=np.zeros(16,np.float32) + if patch.size: + mask=patch.mean(2)<.8;pix=patch[mask] + if len(pix): + z[:3]=pix.mean(0);z[3:6]=pix.std(0);z[6]=z[2]-z[0];z[7]=z[0]-z[1];z[8]=mask.mean();z[9:13]=cv2.resize(mask.astype(np.float32),(4,1),interpolation=cv2.INTER_AREA).reshape(-1);z[13]=float(n.get('layer')=='handwriting');z[14]=float(n.get('kind')=='table');z[15]=min(1,(Y-y)/max(1,image.height)*10) + extra.append(z) + return np.concatenate([old,np.asarray(extra,np.float32).reshape(-1,16)],axis=1) + +def render_roles(result,nodes,ranks,roles): + body=[i for i,r in enumerate(roles)if r==1];entries=[];body_rank=float(np.median([ranks[i]for i in body]))if body else 0. + for i,r in enumerate(roles): + if r==0:entries.append((float(ranks[i]),i,[i])) + if body:entries.append((body_rank,min(body),body)) + entries.sort();order=[i for _,_,ids in entries for i in ids]+[i for i,r in enumerate(roles)if r==2];newr=np.zeros(len(nodes),np.float32) + for rank,i in enumerate(order):newr[i]=rank/max(1,len(order)-1) + actions=np.asarray([2 if r==3 else 1 if r==2 else 0 for r in roles],np.int64);out=render(result,nodes,newr,actions);assert[i for i in order if i in body]==body + return out,order,body diff --git a/stallion/bina04-table-asides-95k-20261104/production.py b/stallion/bina04-table-asides-95k-20261104/production.py new file mode 100644 index 0000000000000000000000000000000000000000..89ada9bf2f7d147ffa6473d2f5733aff1f7fa2ed --- /dev/null +++ b/stallion/bina04-table-asides-95k-20261104/production.py @@ -0,0 +1,8 @@ +from pathlib import Path +import sys,importlib.util +R=Path(__file__).resolve().parent;sys.path.insert(0,str(R));from table_asides import apply +class Runtime: + def __init__(self,device='auto'): + s=importlib.util.spec_from_file_location('graphic_frozen_parent','__BINA_ROOT__/stallion/bina04-assembly-audit-95k-20261104/production.py');m=importlib.util.module_from_spec(s);s.loader.exec_module(m);self.parent=m.Runtime(device);self.raw=self.parent.raw;self.core=self.parent.core + def __call__(self,image,old,kind,name='',**kwargs):return apply(self.parent(image,old,kind,name,**kwargs),image) + def close(self):self.parent.close() diff --git a/stallion/bina04-table-asides-95k-20261104/table_asides.py b/stallion/bina04-table-asides-95k-20261104/table_asides.py new file mode 100644 index 0000000000000000000000000000000000000000..e55977917bed64cde5f36081f29cccb7278b5bf7 --- /dev/null +++ b/stallion/bina04-table-asides-95k-20261104/table_asides.py @@ -0,0 +1,50 @@ +"""Keep marginal notes; don't serialize empty note cells as a full table column.""" +import copy +import numpy as np,cv2 + +def norm(s):return ''.join(s.split()) +def key(l):return tuple(l['box']),l['text'] +def rule_only(l,image): + b=l['box'];h=b[3]-b[1];w=b[2]-b[0] + if h>2 or w<15 or not any(c.isalpha()for c in l['text'])or l.get('confidence',1)>=.5:return False + # A one-pixel horizontal line cannot support the decoded alphabetic glyphs. + a=np.asarray(image.convert('L'));x,y,X,Y=map(int,[max(0,b[0]),max(0,b[1]-2),min(a.shape[1],b[2]),min(a.shape[0],b[3]+3)]);p=a[y:Y,x:X].astype(float) + if p.size==0:return False + bg=np.quantile(p,.9);ink=(p1 for c in occupied):aside.append(col) + if not aside: + # No general table reformat if only a hallucinated rule reading was removed. + if len(dropped)==start_drop:continue + core=cols + else:core=[c for c in cols if c not in aside] + if not core:continue + txt=[];notes=[] + for i,row in enumerate(rows): + values=[by.get((row,c),{}).get('text','').replace('|',r'\|')for c in core];line='| '+' | '.join(values)+' |';side=[by.get((row,c),{}).get('text','')for c in aside];side=[x for x in side if norm(x)] + if side:line+=' '+' '.join(side);notes.extend(dict(row=row,col=c,text=by[(row,c)]['text'])for c in aside if(row,c)in by and norm(by[(row,c)].get('text',''))) + txt.append(line) + if i==0:txt.append('| '+' | '.join('---'for _ in core)+' |') + new='\n'.join(txt) + if new==original:continue + assert original in out['text'];out['text']=out['text'].replace(original,new,1);r['text']=new;t['raw_kept_columns']=cols;t['kept_columns']=core;t['side_note_columns']=aside;t['side_notes']=notes;changes.append(dict(aside_columns=aside,core_columns=core,notes=notes,all_visible_text_retained=True)) + if dropped: + ks={key(l)for l in dropped};out['lines']=[l for l in out['lines']if key(l)not in ks] + for r in out['regions']:r['lines']=[l for l in r.get('lines',[])if key(l)not in ks] + if changes or dropped:out['table_aside_assembly']=dict(changes=changes,suppressed_rule_reads=dropped,recognizer_changed=False) + return out diff --git a/stallion/indra-scale/geometry.py b/stallion/indra-scale/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..5744c36c7db2317524cfe2f4b3c6bc3259372078 --- /dev/null +++ b/stallion/indra-scale/geometry.py @@ -0,0 +1,27 @@ +"""One 64px crop contract for training/inference; width is never clamped or squashed. + +Margins and polarity follow the Apache-2.0 Indra rec_preproc implementation. +Only right-padding batches changes the canvas; widths retain the unpadded size. +""" +import math +import numpy as np +def crop_line64(gray,box,*,pad_y=.2,pad_x=.3,dark=None): + import cv2 + from indra_ocr.rec_preproc import is_dark_line + if gray.ndim!=2 or gray.dtype!=np.uint8:raise ValueError('Original uint8 grayscale required') + x0,y0,x1,y1=map(float,box[:4]) + if not all(math.isfinite(v)for v in [x0,y0,x1,y1])or x1<=x0 or y1<=y0:raise ValueError('Invalid line geometry') + height=y1-y0;ix0=math.floor(x0-pad_x*height);ix1=math.ceil(x1+pad_x*height);iy0=math.floor(y0-pad_y*height);iy1=math.ceil(y1+pad_y*height);cw,ch=ix1-ix0,iy1-iy0 + H,W=gray.shape;sx0,sy0,sx1,sy1=max(0,ix0),max(0,iy0),min(W,ix1),min(H,iy1) + if dark is None:dark=is_dark_line(gray,box) + canvas=np.full((ch,cw),0 if dark else 255,dtype=np.uint8) + if sx1>sx0 and sy1>sy0:canvas[sy0-iy0:sy1-iy0,sx0-ix0:sx1-ix0]=gray[sy0:sy1,sx0:sx1] + width=max(1,round(cw*64/ch));resized=cv2.resize(canvas,(width,64),interpolation=cv2.INTER_AREA if ch>64 else cv2.INTER_LINEAR).astype(np.float32)/255. + ink=resized if dark else 1.-resized + receipt=dict(original_WH=[W,H],ink_box=[x0,y0,x1,y1],padded_source_extent=[ix0,iy0,ix1,iy1],padded_source_WH=[cw,ch],resized_WH=[width,64],scale_XY=[width/cw,64/ch],width_clamped=False,aspect_rounding_error_pixels=abs(width-cw*64/ch),dark_polarity=bool(dark)) + return ink,receipt +def pad_batch(crops,multiple=8): + if not crops or any(c.ndim!=2 or c.shape[0]!=64 for c in crops):raise ValueError('Every recognizer crop must be 64px high') + widths=np.array([c.shape[1]for c in crops],dtype=np.int64);padded=math.ceil(int(widths.max())/multiple)*multiple;out=np.zeros((len(crops),1,64,padded),dtype=np.float32) + for i,c in enumerate(crops):out[i,0,:,:c.shape[1]]=c + return out,widths diff --git a/stallion/indra-scale/gpu_eval_runtime.py b/stallion/indra-scale/gpu_eval_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..1d97a5fcf9b22071f4cfac0bebbb6d92b70d6c17 --- /dev/null +++ b/stallion/indra-scale/gpu_eval_runtime.py @@ -0,0 +1,36 @@ +"""Execution-only CUDA adapter. Frozen routing, weights and FP32 math stay fixed. +CPU image transforms/postprocessing/order and native ORT banks remain unchanged. +""" +import types +import torch + +def configure_fp32(): + assert torch.cuda.is_available(),'CUDA required for the GPU evaluation lane' + torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False;torch.backends.cudnn.benchmark=False;torch.set_float32_matmul_precision('highest');torch.backends.cuda.enable_flash_sdp(False);torch.backends.cuda.enable_mem_efficient_sdp(False);torch.backends.cuda.enable_math_sdp(True) + +def cpu_preprocess_cuda_forward(self,images,threshold=.4,return_features=False): + from surya.common.rfdetr.predictor import IMAGENET_MEAN,IMAGENET_STD + import torchvision.transforms.functional as TF + if not images:return [] + tensors=[];sizes=[] + for image in images: + image=image.convert('RGB');sizes.append((image.height,image.width));t=TF.to_tensor(image);t=TF.normalize(t,IMAGENET_MEAN,IMAGENET_STD);t=TF.resize(t,(self.resolution,self.resolution));tensors.append(t) + with torch.inference_mode(): + outputs=self.model(torch.stack(tensors).to('cuda'));features=self._feat.get('f')if return_features else None + def cpu(x): + if torch.is_tensor(x):return x.cpu() + if isinstance(x,dict):return{k:cpu(v)for k,v in x.items()} + if isinstance(x,list):return[cpu(v)for v in x] + return x + outputs=cpu(outputs);features=features.cpu()if features is not None else None;results=self.postprocess(outputs,target_sizes=torch.tensor(sizes));out=[] + for i,res in enumerate(results): + keep=res['scores']>threshold;record=dict(boxes=res['boxes'][keep],scores=res['scores'][keep],labels=res['labels'][keep]) + if return_features and features is not None:record['features']=features[i] + out.append(record) + return out + +def accelerate(engine,reader): + configure_fp32();reader.model.to('cuda').eval();reader.device=torch.device('cuda') + assert all(p.dtype==torch.float32 for p in reader.model.parameters()) + engine.execution_devices=dict(layout_model='cpu:unchanged',RTL_student='cuda:FP32',layout_image_transforms='cpu:unchanged',layout_postprocess='cpu:unchanged',learned_reading_order='cpu:unchanged',native_ONNX_banks='CPUExecutionProvider',TF32=False) + return engine diff --git a/stallion/indra-scale/recognizer.py b/stallion/indra-scale/recognizer.py new file mode 100644 index 0000000000000000000000000000000000000000..ab9ad431c91d60b13e7856ae85df02a41ad20238 --- /dev/null +++ b/stallion/indra-scale/recognizer.py @@ -0,0 +1,41 @@ +"""Width512/depth20 Indra-family CTC recognizer; native class IDs are retained.""" +import torch +from torch import nn +from torch.nn import functional as F +from torch.utils.checkpoint import checkpoint +class Block(nn.Module): + def __init__(self,width=512,identity=False): + super().__init__();self.dw=nn.Conv1d(width,width,17,padding=8,groups=width);self.expand=nn.Conv1d(width,2*width,1);self.fc1=nn.Conv1d(2*width,width//2,1);self.fc2=nn.Conv1d(width//2,2*width,1);self.project=nn.Conv1d(2*width,width,1) + if identity:nn.init.zeros_(self.project.weight);nn.init.zeros_(self.project.bias) + def forward(self,x,mask): + z=F.hardswish(self.expand(self.dw(x)));pool=(z*mask).sum(2,keepdim=True)/mask.sum(2,keepdim=True).clamp_min(1);return(x+self.project(z*torch.sigmoid(self.fc2(F.relu(self.fc1(pool))))))*mask +class Recognizer(nn.Module): + input_height=64 + def __init__(self,classes=1357,width=512,depth=20,gradient_checkpointing=True): + super().__init__();assert width==512 and depth==20 + self.stem=nn.Conv2d(1,width,(64,8),stride=(64,4));self.bn=nn.BatchNorm1d(width);self.blocks=nn.ModuleList(Block(width,identity=i%2==1)for i in range(depth));self.head=nn.Conv1d(width,classes,1);self.gradient_checkpointing=gradient_checkpointing;self.bn.eval() + def train(self,mode=True):super().train(mode);self.bn.eval();return self + def forward(self,x,widths): + if x.ndim!=4 or x.shape[1]!=1 or x.shape[2]!=64:raise ValueError('Require Bx1x64xW without height truncation') + if widths.ndim!=1 or len(widths)!=len(x)or bool((widths<1).any())or bool((widths>x.shape[-1]).any()):raise ValueError('Invalid original crop widths') + widths=widths.to(device=x.device,dtype=torch.long) + if x.shape[-1]%4:x=F.pad(x,(0,(-x.shape[-1])%4)) + z=self.stem(F.pad(x,(0,4,0,0))).squeeze(2);lengths=(widths+3)//4;assert bool((lengths<=z.shape[-1]).all());mask=(torch.arange(z.shape[-1],device=z.device)[None,:]=.7*max(1,sum(votes.values()))else'mixed' + routed=[region_group if not s and region_group!='mixed'and region_group!='base'else g for g,s in zip(names,sure)] + # Confident minority scripts retain their own bank. Uncertain lines retain a base candidate. + selections=[('base',self.ocr.rec,self.ocr.vocab,[i for i,g in enumerate(routed)if g=='base'or not sure[i]])]+[(g,s,v,[i for i,k in enumerate(routed)if k==g])for g,(s,v)in self.ocr.groups.items()] + best=[('',0.,[])]*len(boxes);scores=[-1.]*len(boxes);records=[dict(original_script=g,script_sure=bool(s),routed_script=r,candidates=[])for g,s,r in zip(names,sure,routed)] + for group,session,vocab,selected in selections: + if not selected:continue + if group=='rtl'and self.rtl_reader is not None: + # Callback returns raw visual tokens and page-local x coordinates, like native banks. + readings,receipts=self.rtl_reader(gray,boxes[selected]) + assert len(readings)==len(receipts)==len(selected) + for j,rec,receipt in zip(selected,readings,receipts): + text,conf,toks=rec;records[j]['candidates'].append(dict(bank='rtl32M',visual_text=text,confidence=float(conf),geometry=receipt)) + if conf>scores[j]:best[j]=(text,float(conf),toks);scores[j]=conf;records[j]['selected_bank']='rtl32M' + continue + sh=session.get_inputs()[0].shape[2];height=int(sh)if isinstance(sh,int)else self.ocr.height + window=self.ocr.window if group=='base'else 1 + assert height%window==0 + src=[crop_line(gray,boxes[j],height=height//window,window=window,max_width=2**31-1)for j in selected] + for part in width_batches([c.shape[1]for c in src],max_batch=32,max_area=32768): + ix=[selected[k]for k in part];arr,widths=pad_batch([src[k]for k in part]);lm=self.ocr.lms.get(id(session));outputs=OUT_GREEDY+(OUT_LM if lm else[]);res=session.run(outputs,{'x':arr,'widths':widths});ids,probs,lengths=res[:3];stride=arr.shape[-1]/max(1,ids.shape[1]) + for k,j in enumerate(ix): + n=int(lengths[k]);seq=ids[k,:n];pm=probs[k,:n];keep=(seq!=0)&np.concatenate(([True],seq[1:]!=seq[:-1]));steps=np.nonzero(keep)[0];tokens=seq[steps].tolist();conf=float(pm[keep].mean())if keep.any()else 0. + if lm and MIN_CONF<=confscores[j]:scores[j]=conf;records[j]['selected_bank']=group;best[j]=(visual,conf,toks)if conf>=MIN_CONF else('',conf,[]) + return best,dict(region_script=region_group,votes=dict(votes),lines=records) + def __call__(self,image): + from indra_ocr.rtl import flip,has_rtl + image=image.convert('RGB');W,H=image.size;tick=time.monotonic();layout=self.layout.run_batch([image],[{'threshold':None,'use_order':True}])[0].model_dump(mode='json') + if layout.get('error'):raise RuntimeError(str(layout['error'])) + regions=layout['bboxes'];fallbacks=[] + if not regions:regions=[dict(polygon=[[0,0],[W,0],[W,H],[0,H]],label='UnknownText',position=0)];fallbacks.append('no_layout_regions_whole_image_text_detection') + assert sorted(r['position']for r in regions)==list(range(len(regions))) + order_mode='surya_learned'if len(regions)<=128 and layout['bboxes']else'whole_image_fallback'if not layout['bboxes']else'surya_raster_over_128_regions' + raw=np.asarray(image.convert('L'));result=[] + for r in sorted(regions,key=lambda r:r['position']): + poly=np.asarray(r['polygon'],dtype=float);x0=max(0,math.floor(float(poly[:,0].min())));x1=min(W,math.ceil(float(poly[:,0].max())));y0=max(0,math.floor(float(poly[:,1].min())));y1=min(H,math.ceil(float(poly[:,1].max())));record=dict(layout=r,crop_bbox=[x0,y0,x1,y1],text='',path=None) + if x1<=x0 or y1<=y0:record['path']='invalid_region_geometry';result.append(record);continue + label=r['label'].lower() + if label in {'picture','figure','chart'}:record['path']='nontext_region';result.append(record);continue + gray=raw[y0:y1,x0:x1].copy();det=self.ocr.detect(gray);boxes=np.asarray(det[:,:4],dtype=np.float32).reshape(-1,4) + if not len(boxes):record['path']='no_detected_lines';record['line_boxes']=[];result.append(record);continue + rec,routing=self._read(gray,boxes);record.update(line_boxes=boxes.tolist(),routing=routing,raw_visual_readings=[dict(text=t,confidence=float(c),tokens=ts)for t,c,ts in rec]) + if label=='table': + page=self.ocr.assemble((x1-x0,y1-y0),boxes,rec,{}) + record.update(path='table_heads_within_surya_region',text=page.markdown if page.markdown is not None else page.text,table_blocks=page.blocks,table_note=page.note,structured_output=page.markdown is not None) + else: + # Outer reading order comes from Surya; inside its text block, top-to-bottom lines. + # Native detection can return separate same-row fragments; region direction breaks those ties. + rtl=routing['region_script']=='rtl';order=sorted(range(len(boxes)),key=lambda i:((float(boxes[i,1])+float(boxes[i,3]))/2,-float(boxes[i,0])if rtl else float(boxes[i,0]))) + lines=[] + for i in order: + text,conf,toks=rec[i];bank=routing['lines'][i].get('selected_bank','base');logical=flip(text,base_rtl=True)if bank=='rtl32M'else flip(text)if bank!='base'and has_rtl(text)else text;lines.append(dict(box=boxes[i].tolist(),text=logical,confidence=conf,bank=bank)) + record.update(path='region_lines',text='\n'.join(z['text']for z in lines),lines=lines) + if label in {'formula','equation','math'}:record['limitation']='plain_text_math_fallback_no_2D_formula_decoder' + result.append(record) + return dict(text='\n\n'.join(r['text']for r in result if r['text']),regions=result,raw_layout=layout,reading_order_mode=order_mode,fallbacks=fallbacks,seconds=time.monotonic()-tick,orientation='original_pixels_no_hidden_page_deskew',main_training_changes=False) diff --git a/stallion/indra-scale/regional_pipeline_v2.py b/stallion/indra-scale/regional_pipeline_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..c25374cbb9aca4a092ef49d125eeb0ccbf61fe16 --- /dev/null +++ b/stallion/indra-scale/regional_pipeline_v2.py @@ -0,0 +1,86 @@ +"""Input-aware inference: region/line crops bypass page layout. +No gold text, teacher output, or benchmark ID is used to decode an image. +""" +import collections,math,time +import numpy as np +from regional_pipeline import RegionalPipeline as OriginalPipeline + +NON_TEXT={'picture','figure','chart'} +def classify_input(width,height,input_type='auto'): + if input_type not in {'auto','crop','page'}:raise ValueError('input_type must be auto, crop or page') + if input_type!='auto':return input_type,'caller_input_metadata' + if height<=128 or(height<=768 and width/max(1,height)>=2.5):return 'crop','short_or_wide_geometry' + return 'page','page_geometry' + +class RegionalPipeline(OriginalPipeline): + def __init__(self,*args,input_type='auto',**kwargs): + super().__init__(*args,**kwargs);self.input_type=input_type + def __call__(self,image,*,input_type=None): + from indra_ocr.rtl import flip,has_rtl + image=image.convert('RGB');W,H=image.size;tick=time.monotonic();kind,decision=classify_input(W,H,input_type or self.input_type);fallbacks=[];raw=np.asarray(image.convert('L')) + def whole(label='UnknownText'):return dict(polygon=[[0,0],[W,0],[W,H],[0,H]],label=label,position=0) + if kind=='crop':layout=dict(bboxes=[],skipped=True,reason='input_is_region_or_line_crop');regions=[whole()];order_mode='crop_whole_text_region' + else: + layout=self.layout.run_batch([image],[{'threshold':None,'use_order':True}])[0].model_dump(mode='json') + if layout.get('error'): + fallbacks.append(dict(code='page_layout_error_whole_image',error=str(layout['error'])));regions=[whole()];order_mode='whole_image_fallback' + else: + regions=layout['bboxes'];order_mode='surya_learned'if len(regions)<=128 else'surya_raster_over_128_regions' + if not any(r['label'].lower()not in NON_TEXT for r in regions): + fallbacks.append(dict(code='no_text_layout_regions_whole_image',original_regions=len(regions)));regions=[whole()];order_mode='whole_image_fallback' + assert sorted(r['position']for r in regions)==list(range(len(regions))) + found_total=0;single_total=0;result=[] + def decode(gray,boxes,position,force_unfiltered=False): + rec,routing=self._read(gray,boxes) + # Keep normal script selection. Only an empty result activates raw CTC recovery. + if not any(t.strip()for t,_,_ in rec)or force_unfiltered: + if self.rtl_reader is not None: + readings,receipts=self.rtl_reader(gray,boxes) + recovered=0 + from visual_token import RTLVisualChar + for i,(reading,receipt)in enumerate(zip(readings,receipts)): + visual=receipt['raw_visual_text'];conf=receipt['confidence'] + if visual.strip()and(not rec[i][0].strip()or force_unfiltered): + toks=list(zip([RTLVisualChar(self.rtl_reader.alphabet[t-1])for t in receipt['raw_token_ids']],receipt['frame_centers']));rec[i]=(visual,conf,toks);routing['lines'][i]['selected_bank']='rtl32M';routing['lines'][i]['candidates'].append(dict(bank='rtl32M',visual_text=visual,confidence=conf,geometry=receipt,unfiltered_empty_output_recovery=True));recovered+=1 + if recovered:fallbacks.append(dict(code='empty_recognition_raw_rtl_CTC',region=position,lines=recovered)) + recovered=0 + for i,line in enumerate(routing['lines']): + if rec[i][0].strip():continue + candidates=[c for c in line['candidates']if c['visual_text'].strip()] + if candidates: + best=max(candidates,key=lambda c:c['confidence']);rec[i]=(best['visual_text'],best['confidence'],[]);line['selected_bank']=best['bank'];recovered+=1 + if recovered:fallbacks.append(dict(code='empty_recognition_raw_existing_candidate',region=position,lines=recovered)) + return rec,routing + def linear_lines(boxes,rec,routing): + rtl=routing['region_script']=='rtl';order=sorted(range(len(boxes)),key=lambda i:((float(boxes[i,1])+float(boxes[i,3]))/2,-float(boxes[i,0])if rtl else float(boxes[i,0])));lines=[] + for i in order: + text,conf,toks=rec[i];bank=routing['lines'][i].get('selected_bank','base');logical=flip(text,base_rtl=True)if bank=='rtl32M'else flip(text)if bank!='base'and has_rtl(text)else text;lines.append(dict(box=boxes[i].tolist(),text=logical,confidence=float(conf),bank=bank)) + return lines + for r in sorted(regions,key=lambda r:r['position']): + poly=np.asarray(r['polygon'],dtype=float);x0=max(0,math.floor(float(poly[:,0].min())));x1=min(W,math.ceil(float(poly[:,0].max())));y0=max(0,math.floor(float(poly[:,1].min())));y1=min(H,math.ceil(float(poly[:,1].max())));record=dict(layout=r,crop_bbox=[x0,y0,x1,y1],text='',path=None) + if x1<=x0 or y1<=y0:record['path']='invalid_region_geometry';fallbacks.append(dict(code='invalid_region_geometry',region=r['position']));result.append(record);continue + label=r['label'].lower() + if label in NON_TEXT:record['path']='nontext_region';result.append(record);continue + gray=raw[y0:y1,x0:x1].copy();det=np.asarray(self.ocr.detect(gray));boxes=np.asarray(det[:,:4],dtype=np.float32).reshape(-1,4)if det.size else np.empty((0,4),dtype=np.float32);found=len(boxes);found_total+=found;record['segmentation_found_lines']=found + if not found: + # The region image is already a text input. Preserve it as one full-width line. + boxes=np.array([[0,0,x1-x0,y1-y0]],dtype=np.float32);single_total+=1;fallbacks.append(dict(code='zero_segmented_lines_whole_region_single_line',region=r['position'],region_is_whole_image=[x0,y0,x1,y1]==[0,0,W,H]));record['single_line_fallback']=True + rec,routing=decode(gray,boxes,r['position']);record.update(line_boxes=boxes.tolist(),routing=routing,raw_visual_readings=[dict(text=t,confidence=float(c),tokens=ts)for t,c,ts in rec]);lines=linear_lines(boxes,rec,routing) + if label=='table': + page=self.ocr.assemble((x1-x0,y1-y0),boxes,rec,{}) + record.update(path='table_heads_within_surya_region',text=page.markdown if page.markdown is not None else page.text,table_blocks=page.blocks,table_note=page.note,structured_output=page.markdown is not None) + if not record['text'].strip()and any(l['text'].strip()for l in lines):record['text']='\n'.join(l['text']for l in lines);fallbacks.append(dict(code='empty_table_assembly_linear_text',region=r['position'])) + else:record.update(path='region_lines',text='\n'.join(l['text']for l in lines),lines=lines) + if label in {'formula','equation','math'}:record['limitation']='plain_text_math_fallback_no_2D_formula_decoder' + result.append(record) + text='\n\n'.join(r['text']for r in result if r['text'].strip()) + if not text.strip(): + # Last-resort whole image text detection, then a genuine raw CTC attempt. + fallbacks.append(dict(code='empty_pipeline_whole_image_text_retry'));det=np.asarray(self.ocr.detect(raw));boxes=np.asarray(det[:,:4],dtype=np.float32).reshape(-1,4)if det.size else np.empty((0,4),np.float32);retry_found=len(boxes) + if not retry_found:boxes=np.array([[0,0,W,H]],np.float32);single_total+=1;fallbacks.append(dict(code='retry_zero_lines_whole_image_single_line')) + rec,routing=decode(raw,boxes,'whole_image_retry',force_unfiltered=True);lines=linear_lines(boxes,rec,routing);text='\n'.join(l['text']for l in lines);result.append(dict(layout=whole(),crop_bbox=[0,0,W,H],path='whole_image_empty_output_recovery',text=text,lines=lines,line_boxes=boxes.tolist(),routing=routing,segmentation_found_lines=retry_found,single_line_fallback=not bool(retry_found))) + if not text.strip(): + fallbacks.append(dict(code='empty_fragment_decoding_whole_image_single_line'));boxes=np.array([[0,0,W,H]],np.float32);single_total+=1;rec,routing=decode(raw,boxes,'whole_image_single_line_retry',force_unfiltered=True);lines=linear_lines(boxes,rec,routing);text='\n'.join(l['text']for l in lines);result.append(dict(layout=whole(),crop_bbox=[0,0,W,H],path='whole_image_single_line_decoding_recovery',text=text,lines=lines,line_boxes=boxes.tolist(),routing=routing,segmentation_found_lines=0,single_line_fallback=True)) + unresolved=not bool(text.strip()) + if unresolved:fallbacks.append(dict(code='unresolved_blank_after_all_genuine_decoding_attempts')) + return dict(text=text,regions=result,raw_layout=layout,reading_order_mode=order_mode,fallbacks=fallbacks,input_routing=dict(input_type=kind,decision=decision,requested=input_type or self.input_type,page_layout_executed=kind=='page'),segmentation_found_lines=found_total,single_line_fallbacks=single_total,unresolved_blank=unresolved,seconds=time.monotonic()-tick,orientation='original_pixels_no_hidden_page_deskew',main_training_changes=False) diff --git a/stallion/indra-scale/routing-rule.json b/stallion/indra-scale/routing-rule.json new file mode 100644 index 0000000000000000000000000000000000000000..6193976728d9fc07c5e1255c6f19ad6130447578 --- /dev/null +++ b/stallion/indra-scale/routing-rule.json @@ -0,0 +1,620 @@ +{ + "version": 1, + "mode": "c", + "description": "Crop inputs: segment only when image height > 1.8 x estimated line height, otherwise recognize the whole crop; real pages retain Surya layout and native region line segmentation", + "threshold_ratio": 1.8, + "estimator": { + "method": "Otsu minority ink connected-component height", + "height_quantile": 75, + "line_height_multiplier": 1.5, + "min_area": 6, + "min_height": 3, + "min_width": 2, + "max_horizontal_aspect": 20.0, + "min_horizontal_aspect": 0.05, + "max_height_fraction": 0.95, + "threshold_ratio": 1.8, + "empty_component_fallback": "image height" + }, + "category_specific_routing": false, + "source_sha256": { + "routing_modes.py": "c27d099baa3666fb4f57bd2eee1c29a79ec76eccaa43d16fab746519ccf379fa", + "regional_pipeline_v2.py": "c9bbd9fca7dac8953a1e0cf4f66bd281d15ee5ec511a7b79d86f7765e41f8b46", + "regional_pipeline.py": "3b6a2d71fce39b2200ac167c150749b60b7c1e9b1fe4631d5f485f265a3a207c" + }, + "selection": { + "checkpoint": 8188, + "checkpoint_sha256": "3894886ee1090846c2510228929c94be64984db1df62e6ec2564a3b9cab5afd2", + "diagnostic_plan_sha256": "3913272dc4c397fabef756a0ae85d60cc91915ee32cfe6001cc548773b44ad00", + "diagnostic_report_sha256": "271d00eeca88e690841dd6969fb06470ed0ab8be322cd5fb59230fc5d3d7277b", + "rule": "Among zero-blank, zero-error modes: minimum unweighted mean of the six category clipped CERs, then pooled clipped CER, then pooled clipped WER, then a/b/c order", + "metrics": { + "a": { + "overall": { + "cases": 193, + "CER": 0.3471176908444297, + "WER": 0.5172555464256368, + "blanks": 0, + "errors": 0, + "char_errors": 4249, + "word_errors": 1346, + "gold_chars": 12091, + "gold_words": 2434, + "macro_category_CER": 0.36011832518484904 + }, + "categories": { + "forms": { + "cases": 70, + "CER": 0.3458823529411765, + "WER": 0.5052790346907994, + "blanks": 0, + "errors": 0, + "char_errors": 1029, + "word_errors": 343, + "gold_chars": 2975, + "gold_words": 663 + }, + "screens": { + "cases": 45, + "CER": 0.17833698030634573, + "WER": 0.33424657534246577, + "blanks": 0, + "errors": 0, + "char_errors": 337, + "word_errors": 142, + "gold_chars": 1828, + "gold_words": 365 + }, + "newspaper": { + "cases": 31, + "CER": 0.6854554604932058, + "WER": 0.9291338582677166, + "blanks": 0, + "errors": 0, + "char_errors": 1362, + "word_errors": 386, + "gold_chars": 1987, + "gold_words": 381 + }, + "math": { + "cases": 18, + "CER": 0.22053462940461727, + "WER": 0.3663663663663664, + "blanks": 0, + "errors": 0, + "char_errors": 404, + "word_errors": 143, + "gold_chars": 1646, + "gold_words": 333 + }, + "tables": { + "cases": 4, + "CER": 0.5695517774343122, + "WER": 0.8211678832116789, + "blanks": 0, + "errors": 0, + "char_errors": 737, + "word_errors": 227, + "gold_chars": 1294, + "gold_words": 274 + }, + "prose": { + "cases": 25, + "CER": 0.1609487505294367, + "WER": 0.24162679425837322, + "blanks": 0, + "errors": 0, + "char_errors": 380, + "word_errors": 105, + "gold_chars": 2361, + "gold_words": 418 + } + }, + "source_ids": { + "057": { + "cases": 70, + "CER": 0.3458823529411765, + "WER": 0.5052790346907994, + "blanks": 0, + "errors": 0, + "char_errors": 1029, + "word_errors": 343, + "gold_chars": 2975, + "gold_words": 663 + }, + "007": { + "cases": 23, + "CER": 0.11273006134969325, + "WER": 0.16279069767441862, + "blanks": 0, + "errors": 0, + "char_errors": 156, + "word_errors": 55, + "gold_chars": 1304, + "gold_words": 258 + }, + "034": { + "cases": 22, + "CER": 0.3416030534351145, + "WER": 0.7476635514018691, + "blanks": 0, + "errors": 0, + "char_errors": 181, + "word_errors": 87, + "gold_chars": 524, + "gold_words": 107 + }, + "067": { + "cases": 31, + "CER": 0.6854554604932058, + "WER": 0.9291338582677166, + "blanks": 0, + "errors": 0, + "char_errors": 1362, + "word_errors": 386, + "gold_chars": 1987, + "gold_words": 381 + }, + "071": { + "cases": 18, + "CER": 0.22053462940461727, + "WER": 0.3663663663663664, + "blanks": 0, + "errors": 0, + "char_errors": 404, + "word_errors": 143, + "gold_chars": 1646, + "gold_words": 333 + }, + "081": { + "cases": 3, + "CER": 0.6595744680851063, + "WER": 0.9925373134328358, + "blanks": 0, + "errors": 0, + "char_errors": 403, + "word_errors": 135, + "gold_chars": 611, + "gold_words": 134 + }, + "083": { + "cases": 1, + "CER": 0.4890190336749634, + "WER": 0.6571428571428571, + "blanks": 0, + "errors": 0, + "char_errors": 334, + "word_errors": 92, + "gold_chars": 683, + "gold_words": 140 + }, + "090": { + "cases": 10, + "CER": 0.17488076311605724, + "WER": 0.2651162790697674, + "blanks": 0, + "errors": 0, + "char_errors": 220, + "word_errors": 60, + "gold_chars": 1258, + "gold_words": 215 + }, + "099": { + "cases": 13, + "CER": 0.45774647887323944, + "WER": 0.5, + "blanks": 0, + "errors": 0, + "char_errors": 130, + "word_errors": 29, + "gold_chars": 284, + "gold_words": 56 + }, + "070": { + "cases": 2, + "CER": 0.03663003663003663, + "WER": 0.10884353741496598, + "blanks": 0, + "errors": 0, + "char_errors": 30, + "word_errors": 16, + "gold_chars": 819, + "gold_words": 147 + } + } + }, + "b": { + "overall": { + "cases": 193, + "CER": 0.6172359606318749, + "WER": 0.7136400986031224, + "blanks": 4, + "errors": 0, + "char_errors": 7484, + "word_errors": 1749, + "gold_chars": 12091, + "gold_words": 2434, + "macro_category_CER": 0.6696701140599929 + }, + "categories": { + "forms": { + "cases": 70, + "CER": 0.16974789915966387, + "WER": 0.31825037707390647, + "blanks": 0, + "errors": 0, + "char_errors": 505, + "word_errors": 216, + "gold_chars": 2975, + "gold_words": 663 + }, + "screens": { + "cases": 45, + "CER": 0.5448577680525164, + "WER": 0.7150684931506849, + "blanks": 0, + "errors": 0, + "char_errors": 1017, + "word_errors": 268, + "gold_chars": 1828, + "gold_words": 365 + }, + "newspaper": { + "cases": 31, + "CER": 0.898842476094615, + "WER": 0.9816272965879265, + "blanks": 2, + "errors": 0, + "char_errors": 1786, + "word_errors": 374, + "gold_chars": 1987, + "gold_words": 381 + }, + "math": { + "cases": 18, + "CER": 0.6476306196840826, + "WER": 0.7777777777777778, + "blanks": 1, + "errors": 0, + "char_errors": 1066, + "word_errors": 259, + "gold_chars": 1646, + "gold_words": 333 + }, + "tables": { + "cases": 4, + "CER": 0.972952086553323, + "WER": 1.0, + "blanks": 0, + "errors": 0, + "char_errors": 1259, + "word_errors": 274, + "gold_chars": 1294, + "gold_words": 274 + }, + "prose": { + "cases": 25, + "CER": 0.783989834815756, + "WER": 0.8564593301435407, + "blanks": 1, + "errors": 0, + "char_errors": 1851, + "word_errors": 358, + "gold_chars": 2361, + "gold_words": 418 + } + }, + "source_ids": { + "057": { + "cases": 70, + "CER": 0.16974789915966387, + "WER": 0.31825037707390647, + "blanks": 0, + "errors": 0, + "char_errors": 505, + "word_errors": 216, + "gold_chars": 2975, + "gold_words": 663 + }, + "007": { + "cases": 23, + "CER": 0.5444785276073619, + "WER": 0.689922480620155, + "blanks": 0, + "errors": 0, + "char_errors": 710, + "word_errors": 178, + "gold_chars": 1304, + "gold_words": 258 + }, + "034": { + "cases": 22, + "CER": 0.5458015267175572, + "WER": 0.7757009345794392, + "blanks": 0, + "errors": 0, + "char_errors": 307, + "word_errors": 90, + "gold_chars": 524, + "gold_words": 107 + }, + "067": { + "cases": 31, + "CER": 0.898842476094615, + "WER": 0.9816272965879265, + "blanks": 2, + "errors": 0, + "char_errors": 1786, + "word_errors": 374, + "gold_chars": 1987, + "gold_words": 381 + }, + "071": { + "cases": 18, + "CER": 0.6476306196840826, + "WER": 0.7777777777777778, + "blanks": 1, + "errors": 0, + "char_errors": 1066, + "word_errors": 259, + "gold_chars": 1646, + "gold_words": 333 + }, + "081": { + "cases": 3, + "CER": 0.9509001636661211, + "WER": 1.0, + "blanks": 0, + "errors": 0, + "char_errors": 581, + "word_errors": 134, + "gold_chars": 611, + "gold_words": 134 + }, + "083": { + "cases": 1, + "CER": 0.9926793557833089, + "WER": 1.0, + "blanks": 0, + "errors": 0, + "char_errors": 678, + "word_errors": 140, + "gold_chars": 683, + "gold_words": 140 + }, + "090": { + "cases": 10, + "CER": 0.7933227344992051, + "WER": 0.9302325581395349, + "blanks": 0, + "errors": 0, + "char_errors": 998, + "word_errors": 200, + "gold_chars": 1258, + "gold_words": 215 + }, + "099": { + "cases": 13, + "CER": 0.12323943661971831, + "WER": 0.19642857142857142, + "blanks": 0, + "errors": 0, + "char_errors": 35, + "word_errors": 11, + "gold_chars": 284, + "gold_words": 56 + }, + "070": { + "cases": 2, + "CER": 0.9987789987789988, + "WER": 1.0, + "blanks": 1, + "errors": 0, + "char_errors": 818, + "word_errors": 147, + "gold_chars": 819, + "gold_words": 147 + } + } + }, + "c": { + "overall": { + "cases": 193, + "CER": 0.2507650318418658, + "WER": 0.419063270336894, + "blanks": 0, + "errors": 0, + "char_errors": 3101, + "word_errors": 1081, + "gold_chars": 12091, + "gold_words": 2434, + "macro_category_CER": 0.2822214779974461 + }, + "categories": { + "forms": { + "cases": 70, + "CER": 0.08605042016806723, + "WER": 0.2368024132730015, + "blanks": 0, + "errors": 0, + "char_errors": 256, + "word_errors": 162, + "gold_chars": 2975, + "gold_words": 663 + }, + "screens": { + "cases": 45, + "CER": 0.11542669584245077, + "WER": 0.2684931506849315, + "blanks": 0, + "errors": 0, + "char_errors": 241, + "word_errors": 117, + "gold_chars": 1828, + "gold_words": 365 + }, + "newspaper": { + "cases": 31, + "CER": 0.6688475088072471, + "WER": 0.9238845144356955, + "blanks": 0, + "errors": 0, + "char_errors": 1329, + "word_errors": 381, + "gold_chars": 1987, + "gold_words": 381 + }, + "math": { + "cases": 18, + "CER": 0.1597812879708384, + "WER": 0.3183183183183183, + "blanks": 0, + "errors": 0, + "char_errors": 302, + "word_errors": 113, + "gold_chars": 1646, + "gold_words": 333 + }, + "tables": { + "cases": 4, + "CER": 0.5556414219474498, + "WER": 0.8211678832116789, + "blanks": 0, + "errors": 0, + "char_errors": 719, + "word_errors": 226, + "gold_chars": 1294, + "gold_words": 274 + }, + "prose": { + "cases": 25, + "CER": 0.10758153324862346, + "WER": 0.19617224880382775, + "blanks": 0, + "errors": 0, + "char_errors": 254, + "word_errors": 82, + "gold_chars": 2361, + "gold_words": 418 + } + }, + "source_ids": { + "057": { + "cases": 70, + "CER": 0.08605042016806723, + "WER": 0.2368024132730015, + "blanks": 0, + "errors": 0, + "char_errors": 256, + "word_errors": 162, + "gold_chars": 2975, + "gold_words": 663 + }, + "007": { + "cases": 23, + "CER": 0.032975460122699383, + "WER": 0.07364341085271318, + "blanks": 0, + "errors": 0, + "char_errors": 52, + "word_errors": 27, + "gold_chars": 1304, + "gold_words": 258 + }, + "034": { + "cases": 22, + "CER": 0.32061068702290074, + "WER": 0.7383177570093458, + "blanks": 0, + "errors": 0, + "char_errors": 189, + "word_errors": 90, + "gold_chars": 524, + "gold_words": 107 + }, + "067": { + "cases": 31, + "CER": 0.6688475088072471, + "WER": 0.9238845144356955, + "blanks": 0, + "errors": 0, + "char_errors": 1329, + "word_errors": 381, + "gold_chars": 1987, + "gold_words": 381 + }, + "071": { + "cases": 18, + "CER": 0.1597812879708384, + "WER": 0.3183183183183183, + "blanks": 0, + "errors": 0, + "char_errors": 302, + "word_errors": 113, + "gold_chars": 1646, + "gold_words": 333 + }, + "081": { + "cases": 3, + "CER": 0.630114566284779, + "WER": 0.9925373134328358, + "blanks": 0, + "errors": 0, + "char_errors": 385, + "word_errors": 134, + "gold_chars": 611, + "gold_words": 134 + }, + "083": { + "cases": 1, + "CER": 0.4890190336749634, + "WER": 0.6571428571428571, + "blanks": 0, + "errors": 0, + "char_errors": 334, + "word_errors": 92, + "gold_chars": 683, + "gold_words": 140 + }, + "090": { + "cases": 10, + "CER": 0.15023847376788554, + "WER": 0.2558139534883721, + "blanks": 0, + "errors": 0, + "char_errors": 189, + "word_errors": 55, + "gold_chars": 1258, + "gold_words": 215 + }, + "099": { + "cases": 13, + "CER": 0.12323943661971831, + "WER": 0.19642857142857142, + "blanks": 0, + "errors": 0, + "char_errors": 35, + "word_errors": 11, + "gold_chars": 284, + "gold_words": 56 + }, + "070": { + "cases": 2, + "CER": 0.03663003663003663, + "WER": 0.10884353741496598, + "blanks": 0, + "errors": 0, + "char_errors": 30, + "word_errors": 16, + "gold_chars": 819, + "gold_words": 147 + } + } + } + }, + "benchmark193_used_for_routing_selection": true, + "benchmark193_used_for_checkpoint_selection": false, + "independent_heldout_193_claim": false + }, + "training_changes": false +} diff --git a/stallion/indra-scale/routing_modes.py b/stallion/indra-scale/routing_modes.py new file mode 100644 index 0000000000000000000000000000000000000000..68e299c1e760723fcb88fa1dbb9a4d75a6feb93d --- /dev/null +++ b/stallion/indra-scale/routing_modes.py @@ -0,0 +1,33 @@ +"""Three fixed crop segmentation policies; no category/label dependent decisions.""" +import threading +import numpy as np +from regional_pipeline_v2 import RegionalPipeline as CurrentPipeline,classify_input +ESTIMATOR=dict(method='Otsu minority ink connected-component height',height_quantile=75,line_height_multiplier=1.5,min_area=6,min_height=3,min_width=2,max_horizontal_aspect=20.,min_horizontal_aspect=.05,max_height_fraction=.95,threshold_ratio=1.8,empty_component_fallback='image height') +def estimated_line_height(gray): + import cv2 + H,W=gray.shape;_,binary=cv2.threshold(gray,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU);ink=(binary==0).astype(np.uint8) + if ink.mean()>.5:ink=1-ink + _,_,stats,_=cv2.connectedComponentsWithStats(ink,8);heights=[] + for x,y,w,h,area in stats[1:]: + aspect=float(w)/max(1,int(h)) + if area>=6 and h>=3 and w>=2 and h<=.95*H and .05<=aspect<=20.:heights.append(int(h)) + glyph=float(np.percentile(heights,75))if heights else None;line=min(float(H),max(1.,1.5*glyph))if glyph is not None else float(H) + return dict(estimated_line_height=line,estimated_glyph_height=glyph,valid_components=len(heights),crop_height=H,height_to_line_height=H/max(1.,line),segment=H>1.8*line) +class RegionalPipeline(CurrentPipeline): + def __init__(self,*args,mode='a',**kwargs): + super().__init__(*args,**kwargs);assert mode in ['a','b','c'];self.mode=mode;self._native_detect=self.ocr.detect;self._routing_lock=threading.RLock() + def __call__(self,image,*,input_type=None,mode=None): + mode=mode or self.mode;assert mode in ['a','b','c'];kind,_=classify_input(*image.size,input_type or self.input_type);estimate=estimated_line_height(np.asarray(image.convert('L')))if mode=='c'and kind=='crop'else None;segment=kind=='page'or mode=='a'or(mode=='c'and estimate['segment']);calls=0 + def detect(gray): + nonlocal calls + if segment:calls+=1;return self._native_detect(gray) + h,w=gray.shape;return np.array([[0.,0.,float(w),float(h)]],np.float32) + with self._routing_lock: + original=self.ocr.detect;self.ocr.detect=detect + try:result=super().__call__(image,input_type=input_type) + finally:self.ocr.detect=original + result['crop_routing_policy']=dict(mode=mode,input_type=kind,native_segmentation_enabled=bool(segment),native_detector_calls=calls,line_height_estimate=estimate,threshold_ratio=1.8,whole_crop_by_policy=kind=='crop'and not segment) + if kind=='crop'and not segment: + result['segmentation_found_lines']=0 + for r in result['regions']:r['segmentation_found_lines']=0;r['whole_crop_by_policy']=True + return result diff --git a/stallion/indra-scale/rtl_reader.py b/stallion/indra-scale/rtl_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..7b6b490150b2c801020f9d9685074c4424dad9b0 --- /dev/null +++ b/stallion/indra-scale/rtl_reader.py @@ -0,0 +1,58 @@ +"""32M checkpoint consumer for RegionalPipeline's RTL bank; raw CTC retained.""" +import json,math +from pathlib import Path +import numpy as np +import torch +from recognizer import Recognizer +from geometry import crop_line64,pad_batch +from visual_token import RTLVisualChar + +class RTLReader: + def __init__(self, checkpoint, vocabulary, device='cpu', frame_budget=8192): + self.device = torch.device(device) + data = json.loads(Path(vocabulary).read_text()) + self.alphabet = data['chars'] + self.model = Recognizer(classes=len(self.alphabet) + 1, gradient_checkpointing=False) + state = torch.load(checkpoint, map_location='cpu', weights_only=True) + self.model.load_state_dict(state['model']) + self.model.to(self.device).eval() + self.frame_budget = frame_budget + + @torch.inference_mode() + def __call__(self, gray, boxes): + from indra_ocr.pipeline import MIN_CONF + crops, receipts = [], [] + for box in boxes: + crop, receipt = crop_line64(gray, box) + crops.append(crop) + receipts.append(receipt) + order = sorted(range(len(crops)), key=lambda i: crops[i].shape[1]) + parts = [] + for i in order: + frames = math.ceil(crops[i].shape[1] / 8) * 2 + if parts and len(parts[-1]) < 32 and (len(parts[-1]) + 1) * frames <= self.frame_budget: + parts[-1].append(i) + else: + parts.append([i]) + readings = [None] * len(crops) + for ids in parts: + x, widths = pad_batch([crops[i] for i in ids]) + x = torch.from_numpy(x).to(self.device) + logits, lengths = self.model(x, torch.from_numpy(widths)) + probs, best = logits.float().softmax(-1).max(-1) + best, probs = best.cpu().numpy(), probs.cpu().numpy() + for k, i in enumerate(ids): + n = int(lengths[k]) + seq, pm = best[k, :n], probs[k, :n] + keep = (seq != 0) & np.concatenate(([True], seq[1:] != seq[:-1])) + steps = np.nonzero(keep)[0] + token_ids = seq[steps].tolist() + visual = ''.join(self.alphabet[t - 1] for t in token_ids) + conf = float(pm[keep].mean()) if keep.any() else 0. + g = receipts[i] + left, _, right, _ = g['padded_source_extent'] + xs = left + (steps * 4 + 3.5) * (right - left) / crops[i].shape[1] + toks = list(zip([RTLVisualChar(self.alphabet[t - 1]) for t in token_ids], xs.tolist())) + receipts[i] = dict(g, raw_visual_text=visual, raw_token_ids=token_ids, confidence=conf, frame_centers=xs.tolist(), confidence_threshold=MIN_CONF, confidence_filtered=conf < MIN_CONF) + readings[i] = (visual, conf, toks) if conf >= MIN_CONF else ('', conf, []) + return readings, receipts diff --git a/stallion/indra-scale/scoringTT/shenavasanj.py b/stallion/indra-scale/scoringTT/shenavasanj.py new file mode 100644 index 0000000000000000000000000000000000000000..c8e337c6d19402f48b1daee6335d5e129281879f --- /dev/null +++ b/stallion/indra-scale/scoringTT/shenavasanj.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass +from typing import Protocol, Sequence + +import jiwer + + +PERSIAN_DIGITS = "۰۱۲۳۴۵۶۷۸۹٠١٢٣٤٥٦٧٨٩" +ASCII_DIGITS = "01234567890123456789" +DIGIT_MAP = str.maketrans(PERSIAN_DIGITS, ASCII_DIGITS) +DIACRITICS_RE = re.compile(r"[ً-ٰٟۖ-ۭ]") +PUNCT_SYMBOL_RE = re.compile(r"[\W_]+", re.UNICODE) + +UNITS = {"صفر": 0, "یک": 1, "دو": 2, "سه": 3, "چهار": 4, "پنج": 5, "شش": 6, "شیش": 6, + "هفت": 7, "هشت": 8, "نه": 9} +TEENS = {"ده": 10, "یازده": 11, "دوازده": 12, "سیزده": 13, "چهارده": 14, "پانزده": 15, + "پونزده": 15, "شانزده": 16, "شونزده": 16, "هفده": 17, "هیفده": 17, + "هجده": 18, "هیجده": 18, "نوزده": 19} +TENS = {"بیست": 20, "سی": 30, "چهل": 40, "پنجاه": 50, "شصت": 60, "هفتاد": 70, + "هشتاد": 80, "نود": 90} +HUNDREDS = {"صد": 100, "یکصد": 100, "دویست": 200, "سیصد": 300, "چهارصد": 400, + "پانصد": 500, "پونصد": 500, "ششصد": 600, "شیشصد": 600, "هفتصد": 700, + "هشتصد": 800, "نهصد": 900} +SCALES = {"هزار": 1_000, "میلیون": 1_000_000, "میلیارد": 1_000_000_000} +NUMBER_WORDS = set(UNITS) | set(TEENS) | set(TENS) | set(HUNDREDS) | set(SCALES) + + +def _number_value(word: str) -> int | None: + for vocabulary in (UNITS, TEENS, TENS, HUNDREDS): + if word in vocabulary: + return vocabulary[word] + return None + + +def normalize_numbers(text: str) -> str: + words = text.split() + output: list[str] = [] + index = 0 + while index < len(words): + if words[index] not in NUMBER_WORDS: + output.append(words[index]) + index += 1 + continue + total = current = 0 + end = index + while end < len(words) and (words[end] in NUMBER_WORDS or words[end] == "و"): + word = words[end] + if word != "و": + if word in SCALES: + total += (current or 1) * SCALES[word] + current = 0 + else: + current += _number_value(word) or 0 + end += 1 + output.append(str(total + current)) + index = end + return " ".join(output) + + +def fair_text(text: str, *, keep_spaces: bool = True) -> str: + text = unicodedata.normalize("NFKC", text or "") + for source, target in ( + ("ي", "ی"), ("ك", "ک"), ("ۀ", "ه"), ("ة", "ه"), ("أ", "ا"), ("إ", "ا"), + ("ٱ", "ا"), ("آ", "ا"), ("ؤ", "و"), ("ئ", "ی"), ("ى", "ی"), + ): + text = text.replace(source, target) + text = text.translate(DIGIT_MAP) + text = DIACRITICS_RE.sub("", text) + text = text.replace("‌", " ").replace("‍", "").replace("​", "") + text = re.sub(r"\s+", " ", PUNCT_SYMBOL_RE.sub(" ", text)).strip() + text = normalize_numbers(text) + return text if keep_spaces else text.replace(" ", "") + + +@dataclass +class SemanticTotals: + error_importance: float = 0.0 + reference_importance: float = 0.0 + essential_errors: int = 0 + essential_words: int = 0 + + +def semantic_error_totals(reference: str, hypothesis: str, scorer, + *, essential_threshold: float = 0.8): + reference_words = fair_text(reference).split() + hypothesis_words = fair_text(hypothesis).split() + if not reference_words: + return SemanticTotals(), [] + importance = scorer.score(reference_words) + alignment = jiwer.process_words(" ".join(reference_words), " ".join(hypothesis_words)) + errors: dict[int, tuple[str, str]] = {} + insertions = 0 + for chunk in alignment.alignments[0]: + if chunk.type in ("substitute", "delete"): + for offset, reference_index in enumerate(range(chunk.ref_start_idx, chunk.ref_end_idx)): + hypothesis_index = chunk.hyp_start_idx + offset + hypothesis_word = (hypothesis_words[hypothesis_index] + if chunk.type == "substitute" and hypothesis_index < chunk.hyp_end_idx else "") + errors[reference_index] = (chunk.type, hypothesis_word) + elif chunk.type == "insert": + insertions += chunk.hyp_end_idx - chunk.hyp_start_idx + totals = SemanticTotals() + details = [] + for index, (word, weight) in enumerate(zip(reference_words, importance)): + is_error = index in errors + totals.reference_importance += weight + if is_error: + totals.error_importance += weight + if weight >= essential_threshold: + totals.essential_words += 1 + totals.essential_errors += int(is_error) + if is_error: + error_type, hypothesis_word = errors[index] + details.append({"reference_word": word, "hypothesis_word": hypothesis_word, + "error_type": error_type, "importance": round(weight, 6)}) + if insertions: + details.append({"error_type": "insert", "count": insertions, "importance": None}) + return totals, details diff --git a/stallion/indra-scale/visual_token.py b/stallion/indra-scale/visual_token.py new file mode 100644 index 0000000000000000000000000000000000000000..929813e50007792e1d6ee3528737f11e5a59f23d --- /dev/null +++ b/stallion/indra-scale/visual_token.py @@ -0,0 +1,11 @@ +"""Preserve the trained RTL bank's explicit visual-order convention in table splits.""" +class RTLVisualChar(str): + base_rtl = True + +class CTCDecodeMixin: + def _decode(self, tokens): + if any(getattr(t[0], 'base_rtl', False) for t in tokens): + from indra_ocr.rtl import flip + visual = ''.join(t[0] if isinstance(t[0], str) else self.vocab.chars[t[0] - 1] for t in tokens) + return flip(visual, base_rtl=True) + return super()._decode(tokens)