| { |
| "full": { |
| "recommended_thresholds": { |
| "other_min_score": 0.4, |
| "ppsn_min_score": 0.55 |
| }, |
| "current_release": { |
| "core": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/irish_core_pii_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 37, |
| "batch_size": 8, |
| "min_score": 0.35, |
| "ppsn_min_score": 0.5, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "ACCOUNT_NUMBER", |
| "BANK_ROUTING_NUMBER", |
| "CREDIT_DEBIT_CARD", |
| "EMAIL", |
| "FIRST_NAME", |
| "LAST_NAME", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER", |
| "POSTCODE", |
| "PPSN", |
| "SWIFT_BIC" |
| ], |
| "elapsed_seconds": 0.6340816880110651, |
| "examples_per_second": 58.35210304851814, |
| "overall": { |
| "precision": 0.9259259259259259, |
| "recall": 0.9868421052631579, |
| "f1": 0.9554140127388535, |
| "tp": 75, |
| "fp": 6, |
| "fn": 1 |
| }, |
| "by_label": { |
| "ACCOUNT_NUMBER": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 3, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 1, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "CREDIT_DEBIT_CARD": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "EMAIL": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "FIRST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "LAST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "tp": 2, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 12, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "POSTCODE": { |
| "precision": 0.75, |
| "recall": 0.75, |
| "f1": 0.75, |
| "tp": 3, |
| "fp": 1, |
| "fn": 1 |
| }, |
| "PPSN": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "SWIFT_BIC": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "edge": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/irish_ppsn_phone_edge_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 22, |
| "batch_size": 8, |
| "min_score": 0.35, |
| "ppsn_min_score": 0.5, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PHONE_NUMBER", |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.34832625201670453, |
| "examples_per_second": 63.15917870854292, |
| "overall": { |
| "precision": 0.9047619047619048, |
| "recall": 1.0, |
| "f1": 0.9500000000000001, |
| "tp": 19, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 13, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| } |
| } |
| }, |
| "numeric_v2": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/irish_numeric_qafix_v2.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 19, |
| "batch_size": 8, |
| "min_score": 0.35, |
| "ppsn_min_score": 0.5, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.29271358298137784, |
| "examples_per_second": 64.90986788682356, |
| "overall": { |
| "precision": 0.8666666666666667, |
| "recall": 0.9285714285714286, |
| "f1": 0.896551724137931, |
| "tp": 13, |
| "fp": 2, |
| "fn": 1 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 5, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8333333333333334, |
| "recall": 1.0, |
| "f1": 0.9090909090909091, |
| "tp": 5, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.75, |
| "recall": 0.75, |
| "f1": 0.75, |
| "tp": 3, |
| "fp": 1, |
| "fn": 1 |
| } |
| } |
| }, |
| "gap_feedback": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/irish_core_gap_feedback_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 14, |
| "batch_size": 8, |
| "min_score": 0.35, |
| "ppsn_min_score": 0.5, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.2173796920105815, |
| "examples_per_second": 64.40344022255086, |
| "overall": { |
| "precision": 0.8333333333333334, |
| "recall": 0.9090909090909091, |
| "f1": 0.8695652173913043, |
| "tp": 10, |
| "fp": 2, |
| "fn": 1 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 4, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8, |
| "recall": 1.0, |
| "f1": 0.888888888888889, |
| "tp": 4, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.6666666666666666, |
| "recall": 0.6666666666666666, |
| "f1": 0.6666666666666666, |
| "tp": 2, |
| "fp": 1, |
| "fn": 1 |
| } |
| } |
| }, |
| "strict_remaining_iou1": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180", |
| "input": "eval/irish_remaining_gaps_v2.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 6, |
| "batch_size": 8, |
| "min_score": 0.35, |
| "ppsn_min_score": 0.5, |
| "iou_threshold": 1.0, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.10174437699606642, |
| "examples_per_second": 58.97131789633906, |
| "overall": { |
| "precision": 0.4, |
| "recall": 0.4, |
| "f1": 0.4000000000000001, |
| "tp": 2, |
| "fp": 3, |
| "fn": 3 |
| }, |
| "by_label": { |
| "PASSPORT_NUMBER": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "tp": 0, |
| "fp": 2, |
| "fn": 2 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.6666666666666666, |
| "recall": 0.6666666666666666, |
| "f1": 0.6666666666666666, |
| "tp": 2, |
| "fp": 1, |
| "fn": 1 |
| } |
| } |
| } |
| }, |
| "candidate_release": { |
| "core": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/irish_core_pii_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 37, |
| "batch_size": 8, |
| "min_score": 0.4, |
| "ppsn_min_score": 0.55, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "ACCOUNT_NUMBER", |
| "BANK_ROUTING_NUMBER", |
| "CREDIT_DEBIT_CARD", |
| "EMAIL", |
| "FIRST_NAME", |
| "LAST_NAME", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER", |
| "POSTCODE", |
| "PPSN", |
| "SWIFT_BIC" |
| ], |
| "elapsed_seconds": 0.5703115339856595, |
| "examples_per_second": 64.87682221929316, |
| "overall": { |
| "precision": 0.9620253164556962, |
| "recall": 1.0, |
| "f1": 0.9806451612903226, |
| "tp": 76, |
| "fp": 3, |
| "fn": 0 |
| }, |
| "by_label": { |
| "ACCOUNT_NUMBER": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 3, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 1, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "CREDIT_DEBIT_CARD": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "EMAIL": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "FIRST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "LAST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 12, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "POSTCODE": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 4, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "tp": 6, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "SWIFT_BIC": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "edge": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/irish_ppsn_phone_edge_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 22, |
| "batch_size": 8, |
| "min_score": 0.4, |
| "ppsn_min_score": 0.55, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PHONE_NUMBER", |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.35221877897856757, |
| "examples_per_second": 62.46117842949735, |
| "overall": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 13, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "numeric_v2": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/irish_numeric_qafix_v2.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 19, |
| "batch_size": 8, |
| "min_score": 0.4, |
| "ppsn_min_score": 0.55, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.2983117959811352, |
| "examples_per_second": 63.69174888813827, |
| "overall": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 14, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 5, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8333333333333334, |
| "recall": 1.0, |
| "f1": 0.9090909090909091, |
| "tp": 5, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.8, |
| "recall": 1.0, |
| "f1": 0.888888888888889, |
| "tp": 4, |
| "fp": 1, |
| "fn": 0 |
| } |
| } |
| }, |
| "gap_feedback": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/irish_core_gap_feedback_v1.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 14, |
| "batch_size": 8, |
| "min_score": 0.4, |
| "ppsn_min_score": 0.55, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.229215633997228, |
| "examples_per_second": 61.07785824142043, |
| "overall": { |
| "precision": 0.8461538461538461, |
| "recall": 1.0, |
| "f1": 0.9166666666666666, |
| "tp": 11, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 4, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8, |
| "recall": 1.0, |
| "f1": 0.888888888888889, |
| "tp": 4, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 3, |
| "fp": 1, |
| "fn": 0 |
| } |
| } |
| }, |
| "strict_remaining_iou1": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220", |
| "input": "eval/irish_remaining_gaps_v2.jsonl", |
| "loader_type": "standard", |
| "device": "cpu", |
| "examples": 6, |
| "batch_size": 8, |
| "min_score": 0.4, |
| "ppsn_min_score": 0.55, |
| "iou_threshold": 1.0, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.09866602800320834, |
| "examples_per_second": 60.8112044381162, |
| "overall": { |
| "precision": 0.5, |
| "recall": 0.4, |
| "f1": 0.4444444444444445, |
| "tp": 2, |
| "fp": 2, |
| "fn": 3 |
| }, |
| "by_label": { |
| "PASSPORT_NUMBER": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "tp": 0, |
| "fp": 1, |
| "fn": 2 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.6666666666666666, |
| "recall": 0.6666666666666666, |
| "f1": 0.6666666666666666, |
| "tp": 2, |
| "fp": 1, |
| "fn": 1 |
| } |
| } |
| } |
| } |
| }, |
| "onnx_q8": { |
| "recommended_thresholds": { |
| "other_min_score": 0.45, |
| "ppsn_min_score": 0.6 |
| }, |
| "current_release": { |
| "core": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/irish_core_pii_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 37, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "ACCOUNT_NUMBER", |
| "BANK_ROUTING_NUMBER", |
| "CREDIT_DEBIT_CARD", |
| "EMAIL", |
| "FIRST_NAME", |
| "LAST_NAME", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER", |
| "POSTCODE", |
| "PPSN", |
| "SWIFT_BIC" |
| ], |
| "elapsed_seconds": 0.8229939279844984, |
| "examples_per_second": 44.957804355388774, |
| "overall": { |
| "precision": 0.9493670886075949, |
| "recall": 0.9868421052631579, |
| "f1": 0.967741935483871, |
| "tp": 75, |
| "fp": 4, |
| "fn": 1 |
| }, |
| "by_label": { |
| "ACCOUNT_NUMBER": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 3, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 1, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "CREDIT_DEBIT_CARD": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "EMAIL": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "FIRST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "LAST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 12, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "POSTCODE": { |
| "precision": 1.0, |
| "recall": 0.75, |
| "f1": 0.8571428571428571, |
| "tp": 3, |
| "fp": 0, |
| "fn": 1 |
| }, |
| "PPSN": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "SWIFT_BIC": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "edge": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/irish_ppsn_phone_edge_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 22, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PHONE_NUMBER", |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.4826828690129332, |
| "examples_per_second": 45.57858049735451, |
| "overall": { |
| "precision": 0.9047619047619048, |
| "recall": 1.0, |
| "f1": 0.9500000000000001, |
| "tp": 19, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 13, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 6, |
| "fp": 2, |
| "fn": 0 |
| } |
| } |
| }, |
| "numeric_v2": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/irish_numeric_qafix_v2.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 19, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.48927704500965774, |
| "examples_per_second": 38.8328048368281, |
| "overall": { |
| "precision": 0.8125, |
| "recall": 0.9285714285714286, |
| "f1": 0.8666666666666666, |
| "tp": 13, |
| "fp": 3, |
| "fn": 1 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 5, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.7142857142857143, |
| "recall": 1.0, |
| "f1": 0.8333333333333333, |
| "tp": 5, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.75, |
| "recall": 0.75, |
| "f1": 0.75, |
| "tp": 3, |
| "fp": 1, |
| "fn": 1 |
| } |
| } |
| }, |
| "gap_feedback": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/irish_core_gap_feedback_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 14, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.25282940798206255, |
| "examples_per_second": 55.37330531183009, |
| "overall": { |
| "precision": 0.8333333333333334, |
| "recall": 0.9090909090909091, |
| "f1": 0.8695652173913043, |
| "tp": 10, |
| "fp": 2, |
| "fn": 1 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 4, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8, |
| "recall": 1.0, |
| "f1": 0.888888888888889, |
| "tp": 4, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.6666666666666666, |
| "recall": 0.6666666666666666, |
| "f1": 0.6666666666666666, |
| "tp": 2, |
| "fp": 1, |
| "fn": 1 |
| } |
| } |
| }, |
| "strict_remaining_iou1": { |
| "model": "models/openmed-mliteclinical-irish-core-v17_full_s180_onnx_q8", |
| "input": "eval/irish_remaining_gaps_v2.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 6, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 1.0, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.1308825800078921, |
| "examples_per_second": 45.84261709723483, |
| "overall": { |
| "precision": 0.6, |
| "recall": 0.6, |
| "f1": 0.6, |
| "tp": 3, |
| "fp": 2, |
| "fn": 2 |
| }, |
| "by_label": { |
| "PASSPORT_NUMBER": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "tp": 0, |
| "fp": 2, |
| "fn": 2 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| } |
| }, |
| "candidate_release": { |
| "core": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/irish_core_pii_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 37, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "ACCOUNT_NUMBER", |
| "BANK_ROUTING_NUMBER", |
| "CREDIT_DEBIT_CARD", |
| "EMAIL", |
| "FIRST_NAME", |
| "LAST_NAME", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER", |
| "POSTCODE", |
| "PPSN", |
| "SWIFT_BIC" |
| ], |
| "elapsed_seconds": 0.8233243200229481, |
| "examples_per_second": 44.93976322595295, |
| "overall": { |
| "precision": 0.9493670886075949, |
| "recall": 0.9868421052631579, |
| "f1": 0.967741935483871, |
| "tp": 75, |
| "fp": 4, |
| "fn": 1 |
| }, |
| "by_label": { |
| "ACCOUNT_NUMBER": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "tp": 3, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 1, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "CREDIT_DEBIT_CARD": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "EMAIL": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "FIRST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "LAST_NAME": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 12, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "POSTCODE": { |
| "precision": 0.75, |
| "recall": 0.75, |
| "f1": 0.75, |
| "tp": 3, |
| "fp": 1, |
| "fn": 1 |
| }, |
| "PPSN": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "tp": 6, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "SWIFT_BIC": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 2, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "edge": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/irish_ppsn_phone_edge_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 22, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PHONE_NUMBER", |
| "PPSN" |
| ], |
| "elapsed_seconds": 0.5273098170291632, |
| "examples_per_second": 41.72120315139757, |
| "overall": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 19, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "by_label": { |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 13, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PPSN": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 6, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| }, |
| "numeric_v2": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/irish_numeric_qafix_v2.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 19, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.3350751459947787, |
| "examples_per_second": 56.70369834083745, |
| "overall": { |
| "precision": 0.875, |
| "recall": 1.0, |
| "f1": 0.9333333333333333, |
| "tp": 14, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 5, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8333333333333334, |
| "recall": 1.0, |
| "f1": 0.9090909090909091, |
| "tp": 5, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.8, |
| "recall": 1.0, |
| "f1": 0.888888888888889, |
| "tp": 4, |
| "fp": 1, |
| "fn": 0 |
| } |
| } |
| }, |
| "gap_feedback": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/irish_core_gap_feedback_v1.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 14, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 0.5, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "BANK_ROUTING_NUMBER", |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.1985326989670284, |
| "examples_per_second": 70.51735090915713, |
| "overall": { |
| "precision": 0.8461538461538461, |
| "recall": 1.0, |
| "f1": 0.9166666666666666, |
| "tp": 11, |
| "fp": 2, |
| "fn": 0 |
| }, |
| "by_label": { |
| "BANK_ROUTING_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 4, |
| "fp": 0, |
| "fn": 0 |
| }, |
| "PASSPORT_NUMBER": { |
| "precision": 0.8, |
| "recall": 1.0, |
| "f1": 0.888888888888889, |
| "tp": 4, |
| "fp": 1, |
| "fn": 0 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "tp": 3, |
| "fp": 1, |
| "fn": 0 |
| } |
| } |
| }, |
| "strict_remaining_iou1": { |
| "model": "models/openmed-mliteclinical-irish-core-v19_boundaryplus_full_s220_onnx_q8", |
| "input": "eval/irish_remaining_gaps_v2.jsonl", |
| "loader_type": "onnx", |
| "device": "cpu", |
| "examples": 6, |
| "batch_size": 8, |
| "min_score": 0.45, |
| "ppsn_min_score": 0.6, |
| "iou_threshold": 1.0, |
| "ppsn_decoder": "word_aligned", |
| "labels_evaluated": [ |
| "PASSPORT_NUMBER", |
| "PHONE_NUMBER" |
| ], |
| "elapsed_seconds": 0.142813170037698, |
| "examples_per_second": 42.01293198950906, |
| "overall": { |
| "precision": 0.75, |
| "recall": 0.6, |
| "f1": 0.6666666666666665, |
| "tp": 3, |
| "fp": 1, |
| "fn": 2 |
| }, |
| "by_label": { |
| "PASSPORT_NUMBER": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "tp": 0, |
| "fp": 1, |
| "fn": 2 |
| }, |
| "PHONE_NUMBER": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "tp": 3, |
| "fp": 0, |
| "fn": 0 |
| } |
| } |
| } |
| } |
| } |
| } |
|
|
