| { |
| "full": { |
| "current_release": { |
| "user": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/user_raw_regression_cases_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 7, |
| "batch_size": 8, |
| "min_score": 0.35, |
| "ppsn_min_score": 0.5, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.11388953996356577, |
| "examples_per_second": 61.46306326497903, |
| "overall": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 3, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PHONE_NUMBER": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "tp": 0, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "multilingual": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/multilingual_ppsn_v1_all.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 168, |
| "batch_size": 16, |
| "ppsn_decoder": "word_aligned", |
| "elapsed_seconds": 1.310564298008103, |
| "examples_per_second": 128.18905585581678, |
| "label_filter": "PPSN", |
| "overall": { |
| "precision": 0.672, |
| "recall": 1.0, |
| "f1": 0.8038277511961722, |
| "tp": 84, |
| "fp": 41, |
| "fn": 0 |
| }, |
| "by_domain": { |
| "citizen_chat": { |
| "precision": 0.6511627906976745, |
| "recall": 1.0, |
| "f1": 0.7887323943661972, |
| "tp": 28, |
| "fp": 15, |
| "fn": 0 |
| }, |
| "gov_data": { |
| "precision": 0.7, |
| "recall": 1.0, |
| "f1": 0.8235294117647058, |
| "tp": 28, |
| "fp": 12, |
| "fn": 0 |
| }, |
| "hse_medical": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 28, |
| "fp": 14, |
| "fn": 0 |
| } |
| }, |
| "by_language": { |
| "de": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "en": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "es": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "fr": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "tp": 6, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "ga": { |
| "precision": 0.5, |
| "recall": 1.0, |
| "f1": 0.6666666666666666, |
| "tp": 6, |
| "fp": 6, |
| "fn": 0 |
| }, |
| "it": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 6, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "ja": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "nl": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "pl": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "pt": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "ro": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 6, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "ru": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "uk": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "zh": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| } |
| } |
| } |
| }, |
| "candidate_release": { |
| "user": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/user_raw_regression_cases_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 7, |
| "batch_size": 8, |
| "min_score": 0.4, |
| "ppsn_min_score": 0.55, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.11162209301255643, |
| "examples_per_second": 62.711599568488346, |
| "overall": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PPSN": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "multilingual": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/multilingual_ppsn_v1_all.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 168, |
| "batch_size": 16, |
| "ppsn_decoder": "word_aligned", |
| "elapsed_seconds": 1.3280671570100822, |
| "examples_per_second": 126.49962700547727, |
| "label_filter": "PPSN", |
| "overall": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 84, |
| "fp": 12, |
| "fn": 0 |
| }, |
| "by_domain": { |
| "citizen_chat": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 28, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "gov_data": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 28, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "hse_medical": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 28, |
| "fp": 4, |
| "fn": 0 |
| } |
| }, |
| "by_language": { |
| "de": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "en": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "es": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "fr": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ga": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "it": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ja": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "nl": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "pl": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "pt": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ro": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ru": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "uk": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "zh": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| } |
| } |
| } |
| } |
| }, |
| "onnx_q8": { |
| "current_release": { |
| "user": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/user_raw_regression_cases_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 7, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.15658493799855933, |
| "examples_per_second": 44.70417199427192, |
| "overall": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 3, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PHONE_NUMBER": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "tp": 0, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "multilingual": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/multilingual_ppsn_v1_all.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 168, |
| "batch_size": 16, |
| "ppsn_decoder": "word_aligned", |
| "elapsed_seconds": 0.5646560749737546, |
| "examples_per_second": 297.5262419833324, |
| "label_filter": "PPSN", |
| "overall": { |
| "precision": 0.6774193548387096, |
| "recall": 1.0, |
| "f1": 0.8076923076923077, |
| "tp": 84, |
| "fp": 40, |
| "fn": 0 |
| }, |
| "by_domain": { |
| "citizen_chat": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 28, |
| "fp": 14, |
| "fn": 0 |
| }, |
| "gov_data": { |
| "precision": 0.6829268292682927, |
| "recall": 1.0, |
| "f1": 0.8115942028985507, |
| "tp": 28, |
| "fp": 13, |
| "fn": 0 |
| }, |
| "hse_medical": { |
| "precision": 0.6829268292682927, |
| "recall": 1.0, |
| "f1": 0.8115942028985507, |
| "tp": 28, |
| "fp": 13, |
| "fn": 0 |
| } |
| }, |
| "by_language": { |
| "de": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "en": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "es": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "fr": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "tp": 6, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "ga": { |
| "precision": 0.5454545454545454, |
| "recall": 1.0, |
| "f1": 0.7058823529411764, |
| "tp": 6, |
| "fp": 5, |
| "fn": 0 |
| }, |
| "it": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "ja": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 6, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "nl": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "pl": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "pt": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "ro": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "ru": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "uk": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "zh": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 6, |
| "fp": 4, |
| "fn": 0 |
| } |
| }, |
| "onnx_file": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8/model_quantized.onnx" |
| } |
| }, |
| "candidate_release": { |
| "user": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/user_raw_regression_cases_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 7, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.13422382500721142, |
| "examples_per_second": 52.1516951228585, |
| "overall": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PPSN": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "multilingual": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/multilingual_ppsn_v1_all.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 168, |
| "batch_size": 16, |
| "ppsn_decoder": "word_aligned", |
| "elapsed_seconds": 0.8803652579663321, |
| "examples_per_second": 190.8298839371338, |
| "label_filter": "PPSN", |
| "overall": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 84, |
| "fp": 12, |
| "fn": 0 |
| }, |
| "by_domain": { |
| "citizen_chat": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 28, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "gov_data": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 28, |
| "fp": 4, |
| "fn": 0 |
| }, |
| "hse_medical": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 28, |
| "fp": 4, |
| "fn": 0 |
| } |
| }, |
| "by_language": { |
| "de": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "tp": 6, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "en": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "es": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "fr": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ga": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "it": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ja": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "nl": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "pl": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "pt": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ro": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "ru": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "uk": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "zh": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 6, |
| "fp": 3, |
| "fn": 0 |
| } |
| }, |
| "onnx_file": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8/model_quantized.onnx" |
| } |
| } |
| } |
| } |
|
|
