File size: 1,646 Bytes
16409df | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | {
"task": "Bashkir person named entity recognition",
"metric": "exact person-span micro F1 with orphan I-PER treated as a new span",
"evaluation_protocol": "public runtime.py, one sentence per ONNX invocation, CPU",
"gold": {
"description": "human-verified held-out Bashkir person-entity benchmark",
"sentences": 1013,
"gold_entities": 3027,
"fp32": {"tp": 2542, "fp": 347, "fn": 485, "f1": 0.8593644354293442, "perfect_sentences": 746},
"int8": {"tp": 2539, "fp": 346, "fn": 488, "f1": 0.8589309878213802, "perfect_sentences": 745}
},
"authored_stress": {
"description": "100 authored diagnostic sentences in ten categories; not a representative population sample",
"sentences": 100,
"gold_entities": 85,
"exact_sentence_overlap_with_checked_training_and_gold_files": 0,
"fp32": {"tp": 79, "fp": 6, "fn": 6, "f1": 0.9294117647058824, "perfect_sentences": 92},
"int8": {"tp": 78, "fp": 6, "fn": 7, "f1": 0.9230769230769231, "perfect_sentences": 92}
},
"inference": {
"backend": "ONNX Runtime CPU",
"export": "FP32 ONNX opset 17 and dynamic MatMul/Gemm INT8 quantization",
"input": "int64 input_ids with dynamic batch and sequence dimensions",
"output": "float logits [batch, sequence, 3]",
"tested_max_length": 256,
"note": "Tokenization and first-subword label alignment are identical across formats. Dynamic INT8 scores can vary with batch composition; this release summary measures the public single-sentence runtime. A separate batch-16 experiment measured INT8 gold F1 0.8582051715396316. Timings vary by host and are not a portable speed guarantee."
}
}
|