IrishCore-DiffMask-135M-v1-rc5 / eval /benchmark_summary.json
temsa's picture
Release IrishCore-DiffMask-135M-v1-rc5
0086dde verified
{
"release": "IrishCore-DiffMask-135M-v1-rc5",
"repo_id": "temsa/IrishCore-DiffMask-135M-v1-rc5",
"architecture": {
"family": "DistilBERT-size token-span extractor",
"diffusion_style_training": true,
"runtime_diffusion": false,
"scanner_free": true,
"validator_free": true,
"heads": [
"token_presence_head",
"typed_start_boundary_head",
"typed_end_boundary_head"
]
},
"base_model": "OpenMed/OpenMed-PII-mLiteClinical-Base-135M-v1",
"notes": [
"DiffMask uses a masked denoising training schedule, not a generative diffusion runtime.",
"ONNX q8 is the recommended CPU deployment artifact.",
"The release inference scripts emit [PII:LABEL] placeholders.",
"rc5 selection was driven by the post-rc3 QA feedback suite and a stricter exact-boundary check on the full-checkpoint path.",
"The current remaining UAT-style misses are a second phone in a long support sentence, one multiline postcode block, and EPStamp4@enterprise.gov.ie in a longer employment-permit message."
],
"full": {
"core_f1": 0.9733333333333333,
"edge_f1": null,
"multilingual_f1": null,
"hardening_f1": null,
"qa_feedback_exact_strict_boundary_f1": 0.9444444444444444
},
"onnx_q8": {
"core_f1": 0.9733333333333333,
"edge_f1": 0.9500000000000001,
"finance_f1": 1.0,
"finance_boundary_f1": 1.0,
"user_ppsn_f1": null,
"gaelic_weak_ppsn_f1": 1.0,
"multilingual_f1": 0.9378531073446327,
"hardening_f1": 1.0,
"core_examples_per_second": 249.22496929603884,
"multilingual_examples_per_second": 247.43129312915065,
"runtime_profile_examples_per_second": 157.4093520813721,
"qa_feedback_exact_f1": 1.0,
"uat_replay_exact_f1": 0.9032258064516129
},
"comparison": {
"public_rc5_onnx_q8": {
"core": 0.9668874172185431,
"edge": 0.9743589743589743,
"remaining_gaps": 0.888888888888889,
"finance": 0.9361702127659575,
"finance_boundary": 0.8750000000000001,
"multilingual_ppsn": 0.9333333333333333,
"user_ppsn": 1.0,
"gaelic_weak_ppsn": 1.0,
"overlap_ppsn": 1.0
},
"public_rc8_onnx_q8": {
"min_score": 0.5,
"irish_core_manual_f1": 0.9736842105263158,
"irish_edge_f1": 1.0,
"finance_suite_f1": 1.0,
"finance_boundary_f1": 1.0,
"user_raw_ppsn_f1": 1.0,
"gaelic_weak_ppsn_f1": 1.0,
"multilingual_ppsn_f1": 0.9176470588235294,
"core_examples_per_second": 46.14201741375802,
"multilingual_examples_per_second": 99.71655616732895
}
}
}