| { |
| "task": "Bashkir person named entity recognition", |
| "metric": "exact person-span micro F1 with orphan I-PER treated as a new span", |
| "evaluation_protocol": "public runtime.py, one sentence per ONNX invocation, CPU", |
| "gold": { |
| "description": "human-verified held-out Bashkir person-entity benchmark", |
| "sentences": 1013, |
| "gold_entities": 3027, |
| "fp32": {"tp": 2542, "fp": 347, "fn": 485, "f1": 0.8593644354293442, "perfect_sentences": 746}, |
| "int8": {"tp": 2539, "fp": 346, "fn": 488, "f1": 0.8589309878213802, "perfect_sentences": 745} |
| }, |
| "authored_stress": { |
| "description": "100 authored diagnostic sentences in ten categories; not a representative population sample", |
| "sentences": 100, |
| "gold_entities": 85, |
| "exact_sentence_overlap_with_checked_training_and_gold_files": 0, |
| "fp32": {"tp": 79, "fp": 6, "fn": 6, "f1": 0.9294117647058824, "perfect_sentences": 92}, |
| "int8": {"tp": 78, "fp": 6, "fn": 7, "f1": 0.9230769230769231, "perfect_sentences": 92} |
| }, |
| "inference": { |
| "backend": "ONNX Runtime CPU", |
| "export": "FP32 ONNX opset 17 and dynamic MatMul/Gemm INT8 quantization", |
| "input": "int64 input_ids with dynamic batch and sequence dimensions", |
| "output": "float logits [batch, sequence, 3]", |
| "tested_max_length": 256, |
| "note": "Tokenization and first-subword label alignment are identical across formats. Dynamic INT8 scores can vary with batch composition; this release summary measures the public single-sentence runtime. A separate batch-16 experiment measured INT8 gold F1 0.8582051715396316. Timings vary by host and are not a portable speed guarantee." |
| } |
| } |
|
|
