| { |
| "model": "townboy/kpfbert-ner", |
| "holdout": "private synthetic final holdout (1,320 documents)", |
| "threshold_policy": "threshold_policy.json", |
| "one_time_final_evaluation": true, |
| "raw_exact_span": { |
| "macro_precision": 0.8435092886798229, |
| "macro_recall": 0.9445454545454546, |
| "macro_f1": 0.883369786048345, |
| "micro_precision": 0.7965755175057501, |
| "micro_recall": 0.9445454545454546, |
| "micro_f1": 0.8642728407042839, |
| "tp": 3117, |
| "fp": 796, |
| "fn": 183 |
| }, |
| "initial_safe_candidate_routing": { |
| "auto_tp": 216, |
| "auto_fp": 0, |
| "sllm_tp_candidates": 2901, |
| "sllm_fp_candidates": 796, |
| "auto_precision": 1.0, |
| "auto_candidate_coverage": 0.055200613340148225, |
| "sllm_candidates": 3697 |
| }, |
| "document_fallback_analysis": { |
| "documents": 1320, |
| "candidate_documents": 813, |
| "zero_candidate_documents": 507, |
| "zero_candidate_positive_documents": 0, |
| "zero_candidate_negative_documents": 507, |
| "documents_with_sllm_candidate": 813, |
| "fn_total": 183, |
| "fn_in_documents_with_sllm_candidate": 183, |
| "fallback_rule_zero_candidate_and_tokens_gt_120_docs": 473, |
| "fn_potentially_exposed_to_sllm_with_candidate_or_zero_fallback": 183, |
| "strict_rule_all_documents_tokens_gt_256_docs": 758, |
| "fn_potentially_exposed_to_sllm_with_candidate_or_strict_long_fallback": 183, |
| "important": "Potential exposure is not recovery; actual recovery depends on SLLM recall. Candidate thresholds cannot recover FN by themselves." |
| }, |
| "per_label": { |
| "ACCOUNT_NUMBER": { |
| "low": 0.6, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 87, |
| "raw_fp": 19, |
| "raw_fn": 13, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 87, |
| "sllm_fp_candidates": 19 |
| }, |
| "ADDRESS": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 90, |
| "raw_fp": 13, |
| "raw_fn": 10, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 90, |
| "sllm_fp_candidates": 13 |
| }, |
| "AGE": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 100, |
| "raw_fp": 12, |
| "raw_fn": 0, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 100, |
| "sllm_fp_candidates": 12 |
| }, |
| "ALIEN_NUMBER": { |
| "low": 0.618, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 97, |
| "raw_fp": 10, |
| "raw_fn": 3, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 97, |
| "sllm_fp_candidates": 10 |
| }, |
| "BIRTHDATE": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 100, |
| "raw_fp": 2, |
| "raw_fn": 0, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 100, |
| "sllm_fp_candidates": 2 |
| }, |
| "BLOOD_TYPE": { |
| "low": 0.6, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 86, |
| "raw_fp": 42, |
| "raw_fn": 14, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 86, |
| "sllm_fp_candidates": 42 |
| }, |
| "CARD_NUMBER": { |
| "low": 0.756, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 100, |
| "raw_fp": 15, |
| "raw_fn": 0, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 100, |
| "sllm_fp_candidates": 15 |
| }, |
| "CITY": { |
| "low": 0.84, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 100, |
| "raw_fp": 0, |
| "raw_fn": 0, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 100, |
| "sllm_fp_candidates": 0 |
| }, |
| "DEPARTMENT": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 92, |
| "raw_fp": 4, |
| "raw_fn": 8, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 92, |
| "sllm_fp_candidates": 4 |
| }, |
| "DRIVER_LICENSE": { |
| "low": 0.6, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 90, |
| "raw_fp": 18, |
| "raw_fn": 10, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 90, |
| "sllm_fp_candidates": 18 |
| }, |
| "EMAIL": { |
| "low": 0.85, |
| "high": 0.999, |
| "auto_accept_enabled": true, |
| "raw_tp": 100, |
| "raw_fp": 8, |
| "raw_fn": 0, |
| "auto_tp": 82, |
| "auto_fp": 0, |
| "auto_precision": 1.0, |
| "sllm_tp_candidates": 18, |
| "sllm_fp_candidates": 8 |
| }, |
| "EMPLOYEE_ID": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 100, |
| "raw_fp": 58, |
| "raw_fn": 0, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 100, |
| "sllm_fp_candidates": 58 |
| }, |
| "GENDER": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 99, |
| "raw_fp": 10, |
| "raw_fn": 1, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 99, |
| "sllm_fp_candidates": 10 |
| }, |
| "HEIGHT": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 97, |
| "raw_fp": 13, |
| "raw_fn": 3, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 97, |
| "sllm_fp_candidates": 13 |
| }, |
| "IP_ADDRESS": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 94, |
| "raw_fp": 24, |
| "raw_fn": 6, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 94, |
| "sllm_fp_candidates": 24 |
| }, |
| "MAJOR": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 100, |
| "raw_fp": 12, |
| "raw_fn": 0, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 100, |
| "sllm_fp_candidates": 12 |
| }, |
| "MEMBER_ID": { |
| "low": 0.6, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 94, |
| "raw_fp": 12, |
| "raw_fn": 6, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 94, |
| "sllm_fp_candidates": 12 |
| }, |
| "NAME": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 99, |
| "raw_fp": 25, |
| "raw_fn": 1, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 99, |
| "sllm_fp_candidates": 25 |
| }, |
| "NATIONALITY": { |
| "low": 0.85, |
| "high": 0.999, |
| "auto_accept_enabled": true, |
| "raw_tp": 100, |
| "raw_fp": 0, |
| "raw_fn": 0, |
| "auto_tp": 41, |
| "auto_fp": 0, |
| "auto_precision": 1.0, |
| "sllm_tp_candidates": 59, |
| "sllm_fp_candidates": 0 |
| }, |
| "NICKNAME": { |
| "low": 0.6, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 71, |
| "raw_fp": 200, |
| "raw_fn": 29, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 71, |
| "sllm_fp_candidates": 200 |
| }, |
| "PARTICIPANT_ID": { |
| "low": 0.767, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 98, |
| "raw_fp": 144, |
| "raw_fn": 2, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 98, |
| "sllm_fp_candidates": 144 |
| }, |
| "PASSPORTNUM": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 99, |
| "raw_fp": 1, |
| "raw_fn": 1, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 99, |
| "sllm_fp_candidates": 1 |
| }, |
| "PHONE": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 100, |
| "raw_fp": 1, |
| "raw_fn": 0, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 100, |
| "sllm_fp_candidates": 1 |
| }, |
| "POSITION": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 99, |
| "raw_fp": 27, |
| "raw_fn": 1, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 99, |
| "sllm_fp_candidates": 27 |
| }, |
| "RELIGION": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 80, |
| "raw_fp": 0, |
| "raw_fn": 20, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 80, |
| "sllm_fp_candidates": 0 |
| }, |
| "RRN": { |
| "low": 0.6, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 93, |
| "raw_fp": 9, |
| "raw_fn": 7, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 93, |
| "sllm_fp_candidates": 9 |
| }, |
| "SCHOOL": { |
| "low": 0.85, |
| "high": 0.999, |
| "auto_accept_enabled": true, |
| "raw_tp": 100, |
| "raw_fp": 2, |
| "raw_fn": 0, |
| "auto_tp": 93, |
| "auto_fp": 0, |
| "auto_precision": 1.0, |
| "sllm_tp_candidates": 7, |
| "sllm_fp_candidates": 2 |
| }, |
| "URL": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 90, |
| "raw_fp": 19, |
| "raw_fn": 10, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 90, |
| "sllm_fp_candidates": 19 |
| }, |
| "USER_ID": { |
| "low": 0.6, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 88, |
| "raw_fp": 35, |
| "raw_fn": 12, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 88, |
| "sllm_fp_candidates": 35 |
| }, |
| "VEHICLE_NUMBER": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 88, |
| "raw_fp": 21, |
| "raw_fn": 12, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 88, |
| "sllm_fp_candidates": 21 |
| }, |
| "WEIGHT": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 98, |
| "raw_fp": 13, |
| "raw_fn": 2, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 98, |
| "sllm_fp_candidates": 13 |
| }, |
| "WORKPLACE": { |
| "low": 0.6, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 98, |
| "raw_fp": 10, |
| "raw_fn": 2, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 98, |
| "sllm_fp_candidates": 10 |
| }, |
| "ZIPCODE": { |
| "low": 0.85, |
| "high": 1.0, |
| "auto_accept_enabled": false, |
| "raw_tp": 90, |
| "raw_fp": 17, |
| "raw_fn": 10, |
| "auto_tp": 0, |
| "auto_fp": 0, |
| "auto_precision": null, |
| "sllm_tp_candidates": 90, |
| "sllm_fp_candidates": 17 |
| } |
| } |
| } |
|
|
