| { |
| "repo_id": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2", |
| "thresholds": { |
| "ppsn_min_score": 0.5, |
| "other_min_score": 0.35 |
| }, |
| "artifacts": { |
| "transformers": true, |
| "onnx_dynamic_int8": "onnx/model_quantized.onnx", |
| "transformers_inference_script": "inference_mask.py", |
| "onnx_inference_script": "inference_mask_onnx.py" |
| }, |
| "exact_qa": { |
| "numeric_v2": { |
| "v1": { |
| "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v1", |
| "overall_f1": 0.3, |
| "passport_f1": 0.5, |
| "routing_f1": 0.33333333333333337, |
| "phone_f1": 0.0, |
| "cpu_examples_per_second": 30.847927604113178 |
| }, |
| "v2_rc1": { |
| "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc1", |
| "overall_f1": 0.21052631578947364, |
| "passport_f1": 0.0, |
| "routing_f1": 0.0, |
| "phone_f1": 0.5, |
| "cpu_examples_per_second": 31.344761140137095 |
| }, |
| "v2_rc2": { |
| "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2", |
| "overall_f1": 0.896551724137931, |
| "passport_f1": 0.9090909090909091, |
| "routing_f1": 1.0, |
| "phone_f1": 0.75, |
| "cpu_examples_per_second": 30.321280764124715 |
| }, |
| "v2_rc2_onnx_q8": { |
| "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2#onnx_q8", |
| "overall_f1": 0.8387096774193549, |
| "passport_f1": 0.7692307692307693, |
| "routing_f1": 1.0, |
| "phone_f1": 0.75, |
| "cpu_examples_per_second": 28.129100872864512 |
| } |
| }, |
| "gap_feedback_v1": { |
| "v1": { |
| "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v1", |
| "overall_f1": 0.26666666666666666, |
| "passport_f1": 0.3333333333333333, |
| "routing_f1": 0.4, |
| "phone_f1": 0.0, |
| "cpu_examples_per_second": 30.3635921284123 |
| }, |
| "v2_rc1": { |
| "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc1", |
| "overall_f1": 0.13333333333333333, |
| "passport_f1": 0.0, |
| "routing_f1": 0.0, |
| "phone_f1": 0.3333333333333333, |
| "cpu_examples_per_second": 31.045050418345404 |
| }, |
| "v2_rc2": { |
| "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2", |
| "overall_f1": 0.8695652173913043, |
| "passport_f1": 0.888888888888889, |
| "routing_f1": 1.0, |
| "phone_f1": 0.6666666666666666, |
| "cpu_examples_per_second": 32.037976756673686 |
| }, |
| "v2_rc2_onnx_q8": { |
| "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2#onnx_q8", |
| "overall_f1": 0.8333333333333333, |
| "passport_f1": 0.8, |
| "routing_f1": 1.0, |
| "phone_f1": 0.6666666666666666, |
| "cpu_examples_per_second": 41.070526382683134 |
| } |
| } |
| }, |
| "broader": { |
| "base_openmed": { |
| "model": "OpenMed/OpenMed-PII-mLiteClinical-Base-135M-v1", |
| "user_ppsn_f1": 0.0, |
| "core_overall_f1": 0.5408805031446541, |
| "edge_overall_f1": 0.05128205128205128, |
| "multilingual_ppsn_f1": 0.0, |
| "core_ppsn_f1": 0.0, |
| "edge_ppsn_f1": 0.0, |
| "edge_phone_f1": 0.14285714285714288, |
| "core_cpu_examples_per_second": 33.550658935826185, |
| "edge_cpu_examples_per_second": 30.571458913889202, |
| "multilingual_cpu_examples_per_second": 116.83828742231812 |
| }, |
| "v1": { |
| "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v1", |
| "user_ppsn_f1": 0.5, |
| "core_overall_f1": 0.9530201342281879, |
| "edge_overall_f1": 0.5714285714285714, |
| "multilingual_ppsn_f1": 0.9940119760479043, |
| "core_ppsn_f1": 0.8, |
| "edge_ppsn_f1": 0.5, |
| "edge_phone_f1": 0.631578947368421, |
| "core_cpu_examples_per_second": 27.18654336491933, |
| "edge_cpu_examples_per_second": 29.208022233908082, |
| "multilingual_cpu_examples_per_second": 127.0333531280346 |
| }, |
| "v2_rc1": { |
| "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc1", |
| "user_ppsn_f1": 1.0, |
| "core_overall_f1": 0.9487179487179489, |
| "edge_overall_f1": 0.8205128205128205, |
| "multilingual_ppsn_f1": 0.7567567567567568, |
| "core_ppsn_f1": 0.8571428571428571, |
| "edge_ppsn_f1": 0.8571428571428571, |
| "edge_phone_f1": 0.8, |
| "core_cpu_examples_per_second": 27.619704960037197, |
| "edge_cpu_examples_per_second": 28.996839164339264, |
| "multilingual_cpu_examples_per_second": 111.33829609559976 |
| }, |
| "v2_rc2": { |
| "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2", |
| "user_ppsn_f1": 1.0, |
| "core_overall_f1": 0.9554140127388535, |
| "edge_overall_f1": 0.9500000000000001, |
| "multilingual_ppsn_f1": 0.8038277511961722, |
| "core_ppsn_f1": 0.8571428571428571, |
| "edge_ppsn_f1": 0.8571428571428571, |
| "edge_phone_f1": 1.0, |
| "core_cpu_examples_per_second": 27.479599368654775, |
| "edge_cpu_examples_per_second": 30.1890248608861, |
| "multilingual_cpu_examples_per_second": 131.09557525150234 |
| }, |
| "v2_rc2_onnx_q8": { |
| "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2#onnx_q8", |
| "user_ppsn_f1": 1.0, |
| "core_overall_f1": 0.9615384615384615, |
| "edge_overall_f1": 0.9500000000000001, |
| "multilingual_ppsn_f1": 0.7887323943661972, |
| "core_ppsn_f1": 0.8571428571428571, |
| "edge_ppsn_f1": 0.8571428571428571, |
| "edge_phone_f1": 1.0, |
| "core_cpu_examples_per_second": 119.06137431000792, |
| "edge_cpu_examples_per_second": 69.96690039628702, |
| "multilingual_cpu_examples_per_second": 86.94930904250197 |
| } |
| } |
| } |
|
|
