| { |
| "config": { |
| "glossary_sizes": [ |
| 10, |
| 30, |
| 100, |
| 300, |
| 1000, |
| 3000, |
| 10000 |
| ], |
| "control_sizes": [ |
| 100, |
| 1000, |
| 9000 |
| ], |
| "n_target_terms": 100, |
| "n_sentences": 400, |
| "n_train_sentences": 700, |
| "n_train_glossary": 400, |
| "tau": 0.28, |
| "gate_threshold": 0.5, |
| "max_damage_rate": 0.01, |
| "gate_train_tau": 0.45, |
| "gate_train_max_raw": 1.2, |
| "seed": 20260823, |
| "gate_path": null, |
| "include_cloud": false, |
| "quantized_model": null, |
| "reader": "auto", |
| "hallucination_patterns": [ |
| "ご視聴ありがとうございました", |
| "ご視聴ありがとうございます", |
| "ご清聴ありがとうございました", |
| "最後までご視聴いただきありがとうございました", |
| "チャンネル登録お願いします", |
| "チャンネル登録よろしくお願いします", |
| "高評価とチャンネル登録をお願いします", |
| "次回もお楽しみに", |
| "また次の動画でお会いしましょう", |
| "本日はご覧いただきありがとうございます", |
| "字幕視聴ありがとうございました", |
| "この動画は自動生成された字幕です", |
| "エンディング", |
| "おわり", |
| "終わり", |
| "スタッフのみなさん、ありがとうございました", |
| "皆さんこんにちは", |
| "はい", |
| "うん", |
| "ありがとうございました" |
| ], |
| "simulation": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| } |
| }, |
| "machine": { |
| "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "processor": "arm", |
| "python": "3.13.5", |
| "cpu_count": 8, |
| "cpu_brand": "Apple M2", |
| "memory_gb": 16.0 |
| }, |
| "provenance": "simulated", |
| "provenance_note": "Conditions marked 'simulated' come from mondegreen.simulate / mondegreen.baselines, whose stated assumptions are in config.simulation. Re-run scripts/harvest_errors.py --mode real and mondegreen bench --cloud to replace them with measurements.", |
| "separation": { |
| "train_glossary_terms": 400, |
| "eval_vocabulary_terms": 10000, |
| "eval_target_terms": 987, |
| "glossary_overlap": 0, |
| "train_sentences": 700, |
| "eval_sentences": 400, |
| "sentence_overlap": 0, |
| "eval_glossary_seen_in_training": false |
| }, |
| "gate": { |
| "n_train": 623, |
| "n_positive": 730, |
| "accuracy": 0.9566265060240964, |
| "auc": 0.9847945205479453, |
| "log_loss": 0.12524717221001125, |
| "ece": 0.05253578460489156, |
| "platt": [ |
| 0.8113781764914337, |
| 0.7707739984152188 |
| ], |
| "feature_weights": { |
| "norm_distance": -1.236312684512737, |
| "margin": 0.48721441497362894, |
| "exact_reading_match": 0.3662493098626638, |
| "span_morae": 0.12309106100638274, |
| "term_morae": 0.05894192373282612, |
| "mora_ratio": 0.5167039483274594, |
| "span_chars": 0.09026522247401973, |
| "n_candidates": -0.19252786761995272, |
| "unknown_ratio": -0.3270157178704295, |
| "reading_ambiguity": 0.0, |
| "log_term_weight": -0.7349845615275823, |
| "span_has_kanji": -0.02512550845331862, |
| "span_has_katakana": 0.7002502572081454, |
| "span_all_hiragana": 0.0, |
| "span_is_glossary_surface": 0.0, |
| "span_is_common_word": 0.4644144469027838, |
| "span_is_proper_noun": 0.6173376220372168, |
| "boundary_left": -0.008701139439452502, |
| "boundary_right": 0.0, |
| "lm_delta": 0.0 |
| }, |
| "chosen_threshold": 0.82, |
| "train_sweep": [ |
| { |
| "threshold": 0.0, |
| "correction_rate": 0.7063492063492064, |
| "damage_rate": 0.05693950177935943, |
| "edit_damage_rate": 0.09615384615384616, |
| "accepted": 728.0, |
| "repairs": 640.0, |
| "damages": 70.0, |
| "neutrals": 18.0 |
| }, |
| { |
| "threshold": 0.01, |
| "correction_rate": 0.7045855379188712, |
| "damage_rate": 0.05693950177935943, |
| "edit_damage_rate": 0.09641873278236915, |
| "accepted": 726.0, |
| "repairs": 638.0, |
| "damages": 70.0, |
| "neutrals": 18.0 |
| }, |
| { |
| "threshold": 0.02, |
| "correction_rate": 0.7045855379188712, |
| "damage_rate": 0.05693950177935943, |
| "edit_damage_rate": 0.09655172413793103, |
| "accepted": 725.0, |
| "repairs": 638.0, |
| "damages": 70.0, |
| "neutrals": 17.0 |
| }, |
| { |
| "threshold": 0.03, |
| "correction_rate": 0.7045855379188712, |
| "damage_rate": 0.05693950177935943, |
| "edit_damage_rate": 0.09668508287292818, |
| "accepted": 724.0, |
| "repairs": 638.0, |
| "damages": 70.0, |
| "neutrals": 16.0 |
| }, |
| { |
| "threshold": 0.04, |
| "correction_rate": 0.7045855379188712, |
| "damage_rate": 0.05693950177935943, |
| "edit_damage_rate": 0.09668508287292818, |
| "accepted": 724.0, |
| "repairs": 638.0, |
| "damages": 70.0, |
| "neutrals": 16.0 |
| }, |
| { |
| "threshold": 0.05, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.05693950177935943, |
| "edit_damage_rate": 0.09681881051175657, |
| "accepted": 723.0, |
| "repairs": 637.0, |
| "damages": 70.0, |
| "neutrals": 16.0 |
| }, |
| { |
| "threshold": 0.06, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.05693950177935943, |
| "edit_damage_rate": 0.09681881051175657, |
| "accepted": 723.0, |
| "repairs": 637.0, |
| "damages": 70.0, |
| "neutrals": 16.0 |
| }, |
| { |
| "threshold": 0.07, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.05516014234875445, |
| "edit_damage_rate": 0.09583333333333334, |
| "accepted": 720.0, |
| "repairs": 637.0, |
| "damages": 69.0, |
| "neutrals": 14.0 |
| }, |
| { |
| "threshold": 0.08, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.05516014234875445, |
| "edit_damage_rate": 0.09583333333333334, |
| "accepted": 720.0, |
| "repairs": 637.0, |
| "damages": 69.0, |
| "neutrals": 14.0 |
| }, |
| { |
| "threshold": 0.09, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.05516014234875445, |
| "edit_damage_rate": 0.09583333333333334, |
| "accepted": 720.0, |
| "repairs": 637.0, |
| "damages": 69.0, |
| "neutrals": 14.0 |
| }, |
| { |
| "threshold": 0.1, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.05516014234875445, |
| "edit_damage_rate": 0.09583333333333334, |
| "accepted": 720.0, |
| "repairs": 637.0, |
| "damages": 69.0, |
| "neutrals": 14.0 |
| }, |
| { |
| "threshold": 0.11, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.05516014234875445, |
| "edit_damage_rate": 0.09596662030598054, |
| "accepted": 719.0, |
| "repairs": 637.0, |
| "damages": 69.0, |
| "neutrals": 13.0 |
| }, |
| { |
| "threshold": 0.12, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.05516014234875445, |
| "edit_damage_rate": 0.09636871508379888, |
| "accepted": 716.0, |
| "repairs": 637.0, |
| "damages": 69.0, |
| "neutrals": 10.0 |
| }, |
| { |
| "threshold": 0.13, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.04804270462633452, |
| "edit_damage_rate": 0.09129213483146068, |
| "accepted": 712.0, |
| "repairs": 637.0, |
| "damages": 65.0, |
| "neutrals": 10.0 |
| }, |
| { |
| "threshold": 0.14, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.046263345195729534, |
| "edit_damage_rate": 0.08873239436619719, |
| "accepted": 710.0, |
| "repairs": 637.0, |
| "damages": 63.0, |
| "neutrals": 10.0 |
| }, |
| { |
| "threshold": 0.15, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.04448398576512456, |
| "edit_damage_rate": 0.0876944837340877, |
| "accepted": 707.0, |
| "repairs": 637.0, |
| "damages": 62.0, |
| "neutrals": 8.0 |
| }, |
| { |
| "threshold": 0.16, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.04448398576512456, |
| "edit_damage_rate": 0.0876944837340877, |
| "accepted": 707.0, |
| "repairs": 637.0, |
| "damages": 62.0, |
| "neutrals": 8.0 |
| }, |
| { |
| "threshold": 0.17, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.033807829181494664, |
| "edit_damage_rate": 0.08, |
| "accepted": 700.0, |
| "repairs": 637.0, |
| "damages": 56.0, |
| "neutrals": 7.0 |
| }, |
| { |
| "threshold": 0.18, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.033807829181494664, |
| "edit_damage_rate": 0.07879656160458452, |
| "accepted": 698.0, |
| "repairs": 637.0, |
| "damages": 55.0, |
| "neutrals": 6.0 |
| }, |
| { |
| "threshold": 0.19, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.033807829181494664, |
| "edit_damage_rate": 0.028700906344410877, |
| "accepted": 662.0, |
| "repairs": 637.0, |
| "damages": 19.0, |
| "neutrals": 6.0 |
| }, |
| { |
| "threshold": 0.2, |
| "correction_rate": 0.7037037037037037, |
| "damage_rate": 0.033807829181494664, |
| "edit_damage_rate": 0.028700906344410877, |
| "accepted": 662.0, |
| "repairs": 637.0, |
| "damages": 19.0, |
| "neutrals": 6.0 |
| }, |
| { |
| "threshold": 0.21, |
| "correction_rate": 0.7028218694885362, |
| "damage_rate": 0.033807829181494664, |
| "edit_damage_rate": 0.028744326777609682, |
| "accepted": 661.0, |
| "repairs": 636.0, |
| "damages": 19.0, |
| "neutrals": 6.0 |
| }, |
| { |
| "threshold": 0.22, |
| "correction_rate": 0.7028218694885362, |
| "damage_rate": 0.033807829181494664, |
| "edit_damage_rate": 0.028744326777609682, |
| "accepted": 661.0, |
| "repairs": 636.0, |
| "damages": 19.0, |
| "neutrals": 6.0 |
| }, |
| { |
| "threshold": 0.23, |
| "correction_rate": 0.7028218694885362, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.024390243902439025, |
| "accepted": 656.0, |
| "repairs": 635.0, |
| "damages": 16.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.24, |
| "correction_rate": 0.7028218694885362, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.024390243902439025, |
| "accepted": 656.0, |
| "repairs": 635.0, |
| "damages": 16.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.25, |
| "correction_rate": 0.7028218694885362, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.024390243902439025, |
| "accepted": 656.0, |
| "repairs": 635.0, |
| "damages": 16.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.26, |
| "correction_rate": 0.7028218694885362, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.024390243902439025, |
| "accepted": 656.0, |
| "repairs": 635.0, |
| "damages": 16.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.27, |
| "correction_rate": 0.7028218694885362, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.024390243902439025, |
| "accepted": 656.0, |
| "repairs": 635.0, |
| "damages": 16.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.28, |
| "correction_rate": 0.7019400352733686, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.024427480916030534, |
| "accepted": 655.0, |
| "repairs": 634.0, |
| "damages": 16.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.29, |
| "correction_rate": 0.7019400352733686, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.024427480916030534, |
| "accepted": 655.0, |
| "repairs": 634.0, |
| "damages": 16.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.3, |
| "correction_rate": 0.7019400352733686, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.024427480916030534, |
| "accepted": 655.0, |
| "repairs": 634.0, |
| "damages": 16.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.31, |
| "correction_rate": 0.7001763668430335, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.024464831804281346, |
| "accepted": 654.0, |
| "repairs": 633.0, |
| "damages": 16.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.32, |
| "correction_rate": 0.7001763668430335, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.024464831804281346, |
| "accepted": 654.0, |
| "repairs": 633.0, |
| "damages": 16.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.33, |
| "correction_rate": 0.7001763668430335, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.024464831804281346, |
| "accepted": 654.0, |
| "repairs": 633.0, |
| "damages": 16.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.34, |
| "correction_rate": 0.7001763668430335, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.024464831804281346, |
| "accepted": 654.0, |
| "repairs": 633.0, |
| "damages": 16.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.35, |
| "correction_rate": 0.7001763668430335, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.02450229709035222, |
| "accepted": 653.0, |
| "repairs": 633.0, |
| "damages": 16.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.36, |
| "correction_rate": 0.7001763668430335, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.02450229709035222, |
| "accepted": 653.0, |
| "repairs": 633.0, |
| "damages": 16.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.37, |
| "correction_rate": 0.7001763668430335, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.02450229709035222, |
| "accepted": 653.0, |
| "repairs": 633.0, |
| "damages": 16.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.38, |
| "correction_rate": 0.7001763668430335, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.02450229709035222, |
| "accepted": 653.0, |
| "repairs": 633.0, |
| "damages": 16.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.39, |
| "correction_rate": 0.7001763668430335, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.02450229709035222, |
| "accepted": 653.0, |
| "repairs": 633.0, |
| "damages": 16.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.4, |
| "correction_rate": 0.7001763668430335, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.02450229709035222, |
| "accepted": 653.0, |
| "repairs": 633.0, |
| "damages": 16.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.41, |
| "correction_rate": 0.7001763668430335, |
| "damage_rate": 0.030249110320284697, |
| "edit_damage_rate": 0.02450229709035222, |
| "accepted": 653.0, |
| "repairs": 633.0, |
| "damages": 16.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.42, |
| "correction_rate": 0.7001763668430335, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.01848998459167951, |
| "accepted": 649.0, |
| "repairs": 633.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.43, |
| "correction_rate": 0.699294532627866, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018518518518518517, |
| "accepted": 648.0, |
| "repairs": 632.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.44, |
| "correction_rate": 0.699294532627866, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018518518518518517, |
| "accepted": 648.0, |
| "repairs": 632.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.45, |
| "correction_rate": 0.699294532627866, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018518518518518517, |
| "accepted": 648.0, |
| "repairs": 632.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.46, |
| "correction_rate": 0.699294532627866, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018518518518518517, |
| "accepted": 648.0, |
| "repairs": 632.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.47, |
| "correction_rate": 0.699294532627866, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018518518518518517, |
| "accepted": 648.0, |
| "repairs": 632.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.48, |
| "correction_rate": 0.699294532627866, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018518518518518517, |
| "accepted": 648.0, |
| "repairs": 632.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.49, |
| "correction_rate": 0.6957671957671958, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018604651162790697, |
| "accepted": 645.0, |
| "repairs": 629.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.5, |
| "correction_rate": 0.6957671957671958, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018604651162790697, |
| "accepted": 645.0, |
| "repairs": 629.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.51, |
| "correction_rate": 0.6957671957671958, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018604651162790697, |
| "accepted": 645.0, |
| "repairs": 629.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.52, |
| "correction_rate": 0.6957671957671958, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018604651162790697, |
| "accepted": 645.0, |
| "repairs": 629.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.53, |
| "correction_rate": 0.6957671957671958, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018633540372670808, |
| "accepted": 644.0, |
| "repairs": 628.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.54, |
| "correction_rate": 0.6957671957671958, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018633540372670808, |
| "accepted": 644.0, |
| "repairs": 628.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.55, |
| "correction_rate": 0.6940035273368607, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.01866251944012442, |
| "accepted": 643.0, |
| "repairs": 627.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.56, |
| "correction_rate": 0.691358024691358, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.0187207488299532, |
| "accepted": 641.0, |
| "repairs": 625.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.57, |
| "correction_rate": 0.689594356261023, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.01875, |
| "accepted": 640.0, |
| "repairs": 624.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.58, |
| "correction_rate": 0.689594356261023, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.01875, |
| "accepted": 640.0, |
| "repairs": 624.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.59, |
| "correction_rate": 0.689594356261023, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.01875, |
| "accepted": 640.0, |
| "repairs": 624.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.6, |
| "correction_rate": 0.689594356261023, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.01875, |
| "accepted": 640.0, |
| "repairs": 624.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.61, |
| "correction_rate": 0.689594356261023, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.01875, |
| "accepted": 640.0, |
| "repairs": 624.0, |
| "damages": 12.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.62, |
| "correction_rate": 0.689594356261023, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018779342723004695, |
| "accepted": 639.0, |
| "repairs": 624.0, |
| "damages": 12.0, |
| "neutrals": 3.0 |
| }, |
| { |
| "threshold": 0.63, |
| "correction_rate": 0.689594356261023, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018779342723004695, |
| "accepted": 639.0, |
| "repairs": 624.0, |
| "damages": 12.0, |
| "neutrals": 3.0 |
| }, |
| { |
| "threshold": 0.64, |
| "correction_rate": 0.6887125220458554, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018808777429467086, |
| "accepted": 638.0, |
| "repairs": 623.0, |
| "damages": 12.0, |
| "neutrals": 3.0 |
| }, |
| { |
| "threshold": 0.65, |
| "correction_rate": 0.6887125220458554, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.018838304552590265, |
| "accepted": 637.0, |
| "repairs": 623.0, |
| "damages": 12.0, |
| "neutrals": 2.0 |
| }, |
| { |
| "threshold": 0.66, |
| "correction_rate": 0.6869488536155203, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.01889763779527559, |
| "accepted": 635.0, |
| "repairs": 621.0, |
| "damages": 12.0, |
| "neutrals": 2.0 |
| }, |
| { |
| "threshold": 0.67, |
| "correction_rate": 0.6869488536155203, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.017350157728706624, |
| "accepted": 634.0, |
| "repairs": 621.0, |
| "damages": 11.0, |
| "neutrals": 2.0 |
| }, |
| { |
| "threshold": 0.68, |
| "correction_rate": 0.6860670194003528, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.017377567140600316, |
| "accepted": 633.0, |
| "repairs": 620.0, |
| "damages": 11.0, |
| "neutrals": 2.0 |
| }, |
| { |
| "threshold": 0.69, |
| "correction_rate": 0.6860670194003528, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.017432646592709985, |
| "accepted": 631.0, |
| "repairs": 620.0, |
| "damages": 11.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "correction_rate": 0.6834215167548501, |
| "damage_rate": 0.023131672597864767, |
| "edit_damage_rate": 0.017488076311605722, |
| "accepted": 629.0, |
| "repairs": 618.0, |
| "damages": 11.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.71, |
| "correction_rate": 0.681657848324515, |
| "damage_rate": 0.017793594306049824, |
| "edit_damage_rate": 0.01282051282051282, |
| "accepted": 624.0, |
| "repairs": 616.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.72, |
| "correction_rate": 0.6807760141093474, |
| "damage_rate": 0.0017793594306049821, |
| "edit_damage_rate": 0.0016233766233766235, |
| "accepted": 616.0, |
| "repairs": 615.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.73, |
| "correction_rate": 0.6798941798941799, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 614.0, |
| "repairs": 614.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.74, |
| "correction_rate": 0.6798941798941799, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 614.0, |
| "repairs": 614.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.75, |
| "correction_rate": 0.6781305114638448, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 612.0, |
| "repairs": 612.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.76, |
| "correction_rate": 0.6763668430335097, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 611.0, |
| "repairs": 611.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.77, |
| "correction_rate": 0.6763668430335097, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 611.0, |
| "repairs": 611.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.78, |
| "correction_rate": 0.673721340388007, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 609.0, |
| "repairs": 609.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.79, |
| "correction_rate": 0.671957671957672, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 608.0, |
| "repairs": 608.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "correction_rate": 0.6701940035273368, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 607.0, |
| "repairs": 607.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.81, |
| "correction_rate": 0.6701940035273368, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 607.0, |
| "repairs": 607.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.82, |
| "correction_rate": 0.6675485008818343, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 604.0, |
| "repairs": 604.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.83, |
| "correction_rate": 0.6666666666666666, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 603.0, |
| "repairs": 603.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.84, |
| "correction_rate": 0.6657848324514991, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 602.0, |
| "repairs": 602.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.85, |
| "correction_rate": 0.6649029982363316, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 600.0, |
| "repairs": 600.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.86, |
| "correction_rate": 0.6631393298059964, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 599.0, |
| "repairs": 599.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.87, |
| "correction_rate": 0.6631393298059964, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 599.0, |
| "repairs": 599.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.88, |
| "correction_rate": 0.6604938271604939, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 597.0, |
| "repairs": 597.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.89, |
| "correction_rate": 0.6578483245149912, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 595.0, |
| "repairs": 595.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "correction_rate": 0.6481481481481481, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 588.0, |
| "repairs": 588.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.91, |
| "correction_rate": 0.6419753086419753, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 583.0, |
| "repairs": 583.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.92, |
| "correction_rate": 0.626984126984127, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 571.0, |
| "repairs": 571.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.93, |
| "correction_rate": 0.6225749559082893, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 567.0, |
| "repairs": 567.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.94, |
| "correction_rate": 0.6128747795414462, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 557.0, |
| "repairs": 557.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "correction_rate": 0.5970017636684304, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 542.0, |
| "repairs": 542.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.96, |
| "correction_rate": 0.5908289241622575, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 535.0, |
| "repairs": 535.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.97, |
| "correction_rate": 0.562610229276896, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 508.0, |
| "repairs": 508.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.98, |
| "correction_rate": 0.4656084656084656, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 423.0, |
| "repairs": 423.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "correction_rate": 0.2927689594356261, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 261.0, |
| "repairs": 261.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 1.0, |
| "correction_rate": 0.0, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 0.0, |
| "repairs": 0.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| } |
| ], |
| "n_decisions": 830 |
| }, |
| "rows": [ |
| { |
| "condition": "A", |
| "label": "raw Whisper (no correction)", |
| "glossary_size": 10, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2842323651452282, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 0.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.0, |
| "cer_improvement_rel": 0.0, |
| "term_recall_gain": 0.0, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| } |
| }, |
| { |
| "condition": "A", |
| "label": "raw Whisper (no correction)", |
| "glossary_size": 30, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2842323651452282, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 0.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.0, |
| "cer_improvement_rel": 0.0, |
| "term_recall_gain": 0.0, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| } |
| }, |
| { |
| "condition": "A", |
| "label": "raw Whisper (no correction)", |
| "glossary_size": 100, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2842323651452282, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 0.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.0, |
| "cer_improvement_rel": 0.0, |
| "term_recall_gain": 0.0, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| } |
| }, |
| { |
| "condition": "A", |
| "label": "raw Whisper (no correction)", |
| "glossary_size": 300, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2842323651452282, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 0.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.0, |
| "cer_improvement_rel": 0.0, |
| "term_recall_gain": 0.0, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| } |
| }, |
| { |
| "condition": "A", |
| "label": "raw Whisper (no correction)", |
| "glossary_size": 1000, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2842323651452282, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 0.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.0, |
| "cer_improvement_rel": 0.0, |
| "term_recall_gain": 0.0, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| } |
| }, |
| { |
| "condition": "A", |
| "label": "raw Whisper (no correction)", |
| "glossary_size": 3000, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2842323651452282, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 0.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.0, |
| "cer_improvement_rel": 0.0, |
| "term_recall_gain": 0.0, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| } |
| }, |
| { |
| "condition": "A", |
| "label": "raw Whisper (no correction)", |
| "glossary_size": 10000, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2842323651452282, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 0.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.0, |
| "cer_improvement_rel": 0.0, |
| "term_recall_gain": 0.0, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| } |
| }, |
| { |
| "condition": "B", |
| "label": "Whisper initial_prompt stuffed with the glossary (244-token ceiling)", |
| "glossary_size": 10, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.28670603255665494, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.2549501570394647, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 315, |
| "total": 1239, |
| "recall": 0.2542372881355932, |
| "by_category": { |
| "product": { |
| "recovered": 135, |
| "total": 487, |
| "recall": 0.27720739219712526 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 2.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": -0.0024736674114267387, |
| "cer_improvement_rel": -0.008702975856260496, |
| "term_recall_gain": 0.0016142050040354738, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "B", |
| "provenance": "simulated", |
| "capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 131, |
| "terms_included": 10, |
| "terms_dropped": 0, |
| "coverage": 1.0 |
| }, |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 1, |
| "intrusions": 7, |
| "prompt_capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 131, |
| "terms_included": 10, |
| "terms_dropped": 0, |
| "coverage": 1.0 |
| }, |
| "glossary_covers_spoken_terms": 2, |
| "targets_in_prompt": 2 |
| } |
| }, |
| { |
| "condition": "C", |
| "label": "cloud LLM post-processing", |
| "glossary_size": 10, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.24736674114267476, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.2264099412808958, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 3.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 0.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.00028142589118198874, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.03686562400255344, |
| "cer_improvement_rel": 0.12970241437394717, |
| "term_recall_gain": 0.0, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 57.0, |
| "removal_rate": 0.8142857142857143, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "C", |
| "provenance": "simulated", |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 0, |
| "gratuitous_rewrites": 1, |
| "hallucinations_removed": 43, |
| "note": "requires shipping the transcript to a third party", |
| "glossary_covers_spoken_terms": 2 |
| } |
| }, |
| { |
| "condition": "D", |
| "label": "Mondegreen (phonetic constraint + conservative gate)", |
| "glossary_size": 10, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.23723268432812, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.21371022804861395, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 315, |
| "total": 1239, |
| "recall": 0.2542372881355932, |
| "by_category": { |
| "product": { |
| "recovered": 135, |
| "total": 487, |
| "recall": 0.27720739219712526 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 2.0, |
| "edits": 1.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 1.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 1.0 |
| }, |
| "cer_improvement": 0.0469996808171082, |
| "cer_improvement_rel": 0.16535654126895002, |
| "term_recall_gain": 0.0016142050040354738, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "seconds": 2.434504917000595, |
| "index": { |
| "entries": 10, |
| "indexed_readings": 10, |
| "reachable_terms": 10, |
| "grams": 213, |
| "mean_posting": 1.2910798122065728, |
| "max_posting": 4, |
| "ngram": 2, |
| "cache_hits": 1761, |
| "cache_misses": 5157 |
| }, |
| "tau": 0.28, |
| "gate_threshold": 0.82, |
| "glossary_covers_spoken_terms": 2, |
| "edits": 1, |
| "hallucinations_removed": 36 |
| } |
| }, |
| { |
| "condition": "E", |
| "label": "Mondegreen (after Q4_K_M quantisation)", |
| "glossary_size": 10, |
| "sweep": "coverage", |
| "provenance": "not-measured", |
| "metrics": { |
| "n": 400, |
| "cer": 0.23723268432812, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.21371022804861395, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 315, |
| "total": 1239, |
| "recall": 0.2542372881355932, |
| "by_category": { |
| "product": { |
| "recovered": 135, |
| "total": 487, |
| "recall": 0.27720739219712526 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 2.0, |
| "edits": 1.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 1.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 1.0 |
| }, |
| "cer_improvement": 0.0469996808171082, |
| "cer_improvement_rel": 0.16535654126895002, |
| "term_recall_gain": 0.0016142050040354738, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "note": "no quantised checkpoint supplied; (E) reported as (D) with the LM re-ranking term absent -- which is quantisation's worst case, bounded by tests/test_quantization.py. Pass --quantized-model to measure the real number." |
| } |
| }, |
| { |
| "condition": "B", |
| "label": "Whisper initial_prompt stuffed with the glossary (244-token ceiling)", |
| "glossary_size": 30, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2861474624960102, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.2549501570394647, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 315, |
| "total": 1239, |
| "recall": 0.2542372881355932, |
| "by_category": { |
| "product": { |
| "recovered": 135, |
| "total": 487, |
| "recall": 0.27720739219712526 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 2.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": -0.0019150973507819913, |
| "cer_improvement_rel": -0.0067377877596855465, |
| "term_recall_gain": 0.0016142050040354738, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "B", |
| "provenance": "simulated", |
| "capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 242, |
| "terms_included": 19, |
| "terms_dropped": 11, |
| "coverage": 0.6333333333333333 |
| }, |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 1, |
| "intrusions": 7, |
| "prompt_capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 242, |
| "terms_included": 19, |
| "terms_dropped": 11, |
| "coverage": 0.6333333333333333 |
| }, |
| "glossary_covers_spoken_terms": 5, |
| "targets_in_prompt": 3 |
| } |
| }, |
| { |
| "condition": "C", |
| "label": "cloud LLM post-processing", |
| "glossary_size": 30, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.247127353973827, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.22586371705585143, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 317, |
| "total": 1239, |
| "recall": 0.25585149313962874, |
| "by_category": { |
| "product": { |
| "recovered": 137, |
| "total": 487, |
| "recall": 0.2813141683778234 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 3.0, |
| "fixed_chars": 2.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 4.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.00028142589118198874, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.03710501117140119, |
| "cer_improvement_rel": 0.13054463784390785, |
| "term_recall_gain": 0.003228410008071003, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 57.0, |
| "removal_rate": 0.8142857142857143, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "C", |
| "provenance": "simulated", |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 2, |
| "gratuitous_rewrites": 1, |
| "hallucinations_removed": 43, |
| "note": "requires shipping the transcript to a third party", |
| "glossary_covers_spoken_terms": 5 |
| } |
| }, |
| { |
| "condition": "D", |
| "label": "Mondegreen (phonetic constraint + conservative gate)", |
| "glossary_size": 30, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.23715288860517075, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.21343711593609177, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 317, |
| "total": 1239, |
| "recall": 0.25585149313962874, |
| "by_category": { |
| "product": { |
| "recovered": 137, |
| "total": 487, |
| "recall": 0.2813141683778234 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 1.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 4.0, |
| "edits": 2.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 2.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 1.0 |
| }, |
| "cer_improvement": 0.04707947654005745, |
| "cer_improvement_rel": 0.1656372824256036, |
| "term_recall_gain": 0.003228410008071003, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "seconds": 2.607052791001479, |
| "index": { |
| "entries": 30, |
| "indexed_readings": 30, |
| "reachable_terms": 30, |
| "grams": 429, |
| "mean_posting": 1.8181818181818181, |
| "max_posting": 11, |
| "ngram": 2, |
| "cache_hits": 3292, |
| "cache_misses": 5642 |
| }, |
| "tau": 0.28, |
| "gate_threshold": 0.82, |
| "glossary_covers_spoken_terms": 5, |
| "edits": 2, |
| "hallucinations_removed": 36 |
| } |
| }, |
| { |
| "condition": "E", |
| "label": "Mondegreen (after Q4_K_M quantisation)", |
| "glossary_size": 30, |
| "sweep": "coverage", |
| "provenance": "not-measured", |
| "metrics": { |
| "n": 400, |
| "cer": 0.23715288860517075, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.21343711593609177, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 317, |
| "total": 1239, |
| "recall": 0.25585149313962874, |
| "by_category": { |
| "product": { |
| "recovered": 137, |
| "total": 487, |
| "recall": 0.2813141683778234 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 1.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 4.0, |
| "edits": 2.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 2.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 1.0 |
| }, |
| "cer_improvement": 0.04707947654005745, |
| "cer_improvement_rel": 0.1656372824256036, |
| "term_recall_gain": 0.003228410008071003, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "note": "no quantised checkpoint supplied; (E) reported as (D) with the LM re-ranking term absent -- which is quantisation's worst case, bounded by tests/test_quantization.py. Pass --quantized-model to measure the real number." |
| } |
| }, |
| { |
| "condition": "B", |
| "label": "Whisper initial_prompt stuffed with the glossary (244-token ceiling)", |
| "glossary_size": 100, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.28542930098946695, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.2545404888706814, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 317, |
| "total": 1239, |
| "recall": 0.25585149313962874, |
| "by_category": { |
| "product": { |
| "recovered": 135, |
| "total": 487, |
| "recall": 0.27720739219712526 |
| }, |
| "person": { |
| "recovered": 136, |
| "total": 586, |
| "recall": 0.23208191126279865 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 4.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 4.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": -0.0011969358442387446, |
| "cer_improvement_rel": -0.004211117349803466, |
| "term_recall_gain": 0.003228410008071003, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "B", |
| "provenance": "simulated", |
| "capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 242, |
| "terms_included": 19, |
| "terms_dropped": 81, |
| "coverage": 0.19 |
| }, |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 2, |
| "intrusions": 6, |
| "prompt_capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 242, |
| "terms_included": 19, |
| "terms_dropped": 81, |
| "coverage": 0.19 |
| }, |
| "glossary_covers_spoken_terms": 10, |
| "targets_in_prompt": 3 |
| } |
| }, |
| { |
| "condition": "C", |
| "label": "cloud LLM post-processing", |
| "glossary_size": 100, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.24561123523779124, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.2243616004369794, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 328, |
| "total": 1239, |
| "recall": 0.26472962066182404, |
| "by_category": { |
| "product": { |
| "recovered": 137, |
| "total": 487, |
| "recall": 0.2813141683778234 |
| }, |
| "person": { |
| "recovered": 144, |
| "total": 586, |
| "recall": 0.24573378839590443 |
| }, |
| "org": { |
| "recovered": 30, |
| "total": 106, |
| "recall": 0.2830188679245283 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 4.0, |
| "fixed_chars": 18.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 15.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.000375234521575985, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.03862112990743696, |
| "cer_improvement_rel": 0.13587871982032568, |
| "term_recall_gain": 0.012106537530266304, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 57.0, |
| "removal_rate": 0.8142857142857143, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "C", |
| "provenance": "simulated", |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 9, |
| "gratuitous_rewrites": 1, |
| "hallucinations_removed": 43, |
| "note": "requires shipping the transcript to a third party", |
| "glossary_covers_spoken_terms": 10 |
| } |
| }, |
| { |
| "condition": "D", |
| "label": "Mondegreen (phonetic constraint + conservative gate)", |
| "glossary_size": 100, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.236594318544526, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.21302744776730848, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 320, |
| "total": 1239, |
| "recall": 0.258272800645682, |
| "by_category": { |
| "product": { |
| "recovered": 137, |
| "total": 487, |
| "recall": 0.2813141683778234 |
| }, |
| "person": { |
| "recovered": 137, |
| "total": 586, |
| "recall": 0.23378839590443687 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 2.0, |
| "fixed_chars": 7.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 1.0, |
| "fixed_terms": 8.0, |
| "edits": 5.0, |
| "damaging_edits": 1.0, |
| "repairing_edits": 4.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0001876172607879925, |
| "damage_rate_terms": 0.003194888178913738, |
| "damage_rate_edits": 0.2, |
| "repair_rate_edits": 0.8 |
| }, |
| "cer_improvement": 0.0476380466007022, |
| "cer_improvement_rel": 0.16760247052217855, |
| "term_recall_gain": 0.005649717514124242, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "seconds": 3.2017689580025035, |
| "index": { |
| "entries": 100, |
| "indexed_readings": 100, |
| "reachable_terms": 100, |
| "grams": 765, |
| "mean_posting": 3.406535947712418, |
| "max_posting": 41, |
| "ngram": 2, |
| "cache_hits": 3681, |
| "cache_misses": 6095 |
| }, |
| "tau": 0.28, |
| "gate_threshold": 0.82, |
| "glossary_covers_spoken_terms": 10, |
| "edits": 5, |
| "hallucinations_removed": 36 |
| } |
| }, |
| { |
| "condition": "E", |
| "label": "Mondegreen (after Q4_K_M quantisation)", |
| "glossary_size": 100, |
| "sweep": "coverage", |
| "provenance": "not-measured", |
| "metrics": { |
| "n": 400, |
| "cer": 0.236594318544526, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.21302744776730848, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 320, |
| "total": 1239, |
| "recall": 0.258272800645682, |
| "by_category": { |
| "product": { |
| "recovered": 137, |
| "total": 487, |
| "recall": 0.2813141683778234 |
| }, |
| "person": { |
| "recovered": 137, |
| "total": 586, |
| "recall": 0.23378839590443687 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 2.0, |
| "fixed_chars": 7.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 1.0, |
| "fixed_terms": 8.0, |
| "edits": 5.0, |
| "damaging_edits": 1.0, |
| "repairing_edits": 4.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0001876172607879925, |
| "damage_rate_terms": 0.003194888178913738, |
| "damage_rate_edits": 0.2, |
| "repair_rate_edits": 0.8 |
| }, |
| "cer_improvement": 0.0476380466007022, |
| "cer_improvement_rel": 0.16760247052217855, |
| "term_recall_gain": 0.005649717514124242, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "note": "no quantised checkpoint supplied; (E) reported as (D) with the LM re-ranking term absent -- which is quantisation's worst case, bounded by tests/test_quantization.py. Pass --quantized-model to measure the real number." |
| } |
| }, |
| { |
| "condition": "B", |
| "label": "Whisper initial_prompt stuffed with the glossary (244-token ceiling)", |
| "glossary_size": 300, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.28542930098946695, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.2545404888706814, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 317, |
| "total": 1239, |
| "recall": 0.25585149313962874, |
| "by_category": { |
| "product": { |
| "recovered": 135, |
| "total": 487, |
| "recall": 0.27720739219712526 |
| }, |
| "person": { |
| "recovered": 136, |
| "total": 586, |
| "recall": 0.23208191126279865 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 4.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 4.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": -0.0011969358442387446, |
| "cer_improvement_rel": -0.004211117349803466, |
| "term_recall_gain": 0.003228410008071003, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "B", |
| "provenance": "simulated", |
| "capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 242, |
| "terms_included": 19, |
| "terms_dropped": 281, |
| "coverage": 0.06333333333333334 |
| }, |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 2, |
| "intrusions": 6, |
| "prompt_capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 242, |
| "terms_included": 19, |
| "terms_dropped": 281, |
| "coverage": 0.06333333333333334 |
| }, |
| "glossary_covers_spoken_terms": 27, |
| "targets_in_prompt": 3 |
| } |
| }, |
| { |
| "condition": "C", |
| "label": "cloud LLM post-processing", |
| "glossary_size": 300, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.244254707947654, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.22299603987436842, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 336, |
| "total": 1239, |
| "recall": 0.2711864406779661, |
| "by_category": { |
| "product": { |
| "recovered": 139, |
| "total": 487, |
| "recall": 0.28542094455852157 |
| }, |
| "person": { |
| "recovered": 150, |
| "total": 586, |
| "recall": 0.25597269624573377 |
| }, |
| "org": { |
| "recovered": 30, |
| "total": 106, |
| "recall": 0.2830188679245283 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 7.0, |
| "fixed_chars": 31.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 23.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0006566604127579737, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.03997765719757421, |
| "cer_improvement_rel": 0.14065131948343626, |
| "term_recall_gain": 0.018563357546408366, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 57.0, |
| "removal_rate": 0.8142857142857143, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "C", |
| "provenance": "simulated", |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 14, |
| "gratuitous_rewrites": 1, |
| "hallucinations_removed": 43, |
| "note": "requires shipping the transcript to a third party", |
| "glossary_covers_spoken_terms": 27 |
| } |
| }, |
| { |
| "condition": "D", |
| "label": "Mondegreen (phonetic constraint + conservative gate)", |
| "glossary_size": 300, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.23507819980849026, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.21152533114843644, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 330, |
| "total": 1239, |
| "recall": 0.26634382566585957, |
| "by_category": { |
| "product": { |
| "recovered": 141, |
| "total": 487, |
| "recall": 0.28952772073921973 |
| }, |
| "person": { |
| "recovered": 142, |
| "total": 586, |
| "recall": 0.24232081911262798 |
| }, |
| "org": { |
| "recovered": 30, |
| "total": 106, |
| "recall": 0.2830188679245283 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 2.0, |
| "fixed_chars": 21.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 1.0, |
| "fixed_terms": 18.0, |
| "edits": 10.0, |
| "damaging_edits": 1.0, |
| "repairing_edits": 9.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0001876172607879925, |
| "damage_rate_terms": 0.003194888178913738, |
| "damage_rate_edits": 0.1, |
| "repair_rate_edits": 0.9 |
| }, |
| "cer_improvement": 0.04915416533673794, |
| "cer_improvement_rel": 0.17293655249859627, |
| "term_recall_gain": 0.013720742534301833, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "seconds": 4.8765948330001265, |
| "index": { |
| "entries": 300, |
| "indexed_readings": 300, |
| "reachable_terms": 300, |
| "grams": 1103, |
| "mean_posting": 7.049864007252946, |
| "max_posting": 98, |
| "ngram": 2, |
| "cache_hits": 3681, |
| "cache_misses": 6090 |
| }, |
| "tau": 0.28, |
| "gate_threshold": 0.82, |
| "glossary_covers_spoken_terms": 27, |
| "edits": 10, |
| "hallucinations_removed": 36 |
| } |
| }, |
| { |
| "condition": "E", |
| "label": "Mondegreen (after Q4_K_M quantisation)", |
| "glossary_size": 300, |
| "sweep": "coverage", |
| "provenance": "not-measured", |
| "metrics": { |
| "n": 400, |
| "cer": 0.23507819980849026, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.21152533114843644, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 330, |
| "total": 1239, |
| "recall": 0.26634382566585957, |
| "by_category": { |
| "product": { |
| "recovered": 141, |
| "total": 487, |
| "recall": 0.28952772073921973 |
| }, |
| "person": { |
| "recovered": 142, |
| "total": 586, |
| "recall": 0.24232081911262798 |
| }, |
| "org": { |
| "recovered": 30, |
| "total": 106, |
| "recall": 0.2830188679245283 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 2.0, |
| "fixed_chars": 21.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 1.0, |
| "fixed_terms": 18.0, |
| "edits": 10.0, |
| "damaging_edits": 1.0, |
| "repairing_edits": 9.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0001876172607879925, |
| "damage_rate_terms": 0.003194888178913738, |
| "damage_rate_edits": 0.1, |
| "repair_rate_edits": 0.9 |
| }, |
| "cer_improvement": 0.04915416533673794, |
| "cer_improvement_rel": 0.17293655249859627, |
| "term_recall_gain": 0.013720742534301833, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "note": "no quantised checkpoint supplied; (E) reported as (D) with the LM re-ranking term absent -- which is quantisation's worst case, bounded by tests/test_quantization.py. Pass --quantized-model to measure the real number." |
| } |
| }, |
| { |
| "condition": "B", |
| "label": "Whisper initial_prompt stuffed with the glossary (244-token ceiling)", |
| "glossary_size": 1000, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2843919565911267, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 318, |
| "total": 1239, |
| "recall": 0.2566585956416465, |
| "by_category": { |
| "product": { |
| "recovered": 135, |
| "total": 487, |
| "recall": 0.27720739219712526 |
| }, |
| "person": { |
| "recovered": 136, |
| "total": 586, |
| "recall": 0.23208191126279865 |
| }, |
| "org": { |
| "recovered": 30, |
| "total": 106, |
| "recall": 0.2830188679245283 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 8.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 5.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": -0.00015959144589849927, |
| "cer_improvement_rel": -0.0005614823133071288, |
| "term_recall_gain": 0.004035512510088768, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "B", |
| "provenance": "simulated", |
| "capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 242, |
| "terms_included": 19, |
| "terms_dropped": 981, |
| "coverage": 0.019 |
| }, |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 3, |
| "intrusions": 5, |
| "prompt_capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 242, |
| "terms_included": 19, |
| "terms_dropped": 981, |
| "coverage": 0.019 |
| }, |
| "glossary_covers_spoken_terms": 97, |
| "targets_in_prompt": 3 |
| } |
| }, |
| { |
| "condition": "C", |
| "label": "cloud LLM post-processing", |
| "glossary_size": 1000, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.22917331631024576, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.20906732213573673, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 417, |
| "total": 1239, |
| "recall": 0.3365617433414044, |
| "by_category": { |
| "product": { |
| "recovered": 169, |
| "total": 487, |
| "recall": 0.3470225872689938 |
| }, |
| "person": { |
| "recovered": 196, |
| "total": 586, |
| "recall": 0.33447098976109213 |
| }, |
| "org": { |
| "recovered": 34, |
| "total": 106, |
| "recall": 0.32075471698113206 |
| }, |
| "jargon": { |
| "recovered": 18, |
| "total": 60, |
| "recall": 0.3 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 10.0, |
| "fixed_chars": 142.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 104.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0009380863039399625, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.055059048834982444, |
| "cer_improvement_rel": 0.19371139809096014, |
| "term_recall_gain": 0.08393866020984664, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 57.0, |
| "removal_rate": 0.8142857142857143, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "C", |
| "provenance": "simulated", |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 61, |
| "gratuitous_rewrites": 1, |
| "hallucinations_removed": 43, |
| "note": "requires shipping the transcript to a third party", |
| "glossary_covers_spoken_terms": 97 |
| } |
| }, |
| { |
| "condition": "D", |
| "label": "Mondegreen (phonetic constraint + conservative gate)", |
| "glossary_size": 1000, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.22334822853495054, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.20182985115389868, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 390, |
| "total": 1239, |
| "recall": 0.31476997578692495, |
| "by_category": { |
| "product": { |
| "recovered": 165, |
| "total": 487, |
| "recall": 0.33880903490759756 |
| }, |
| "person": { |
| "recovered": 173, |
| "total": 586, |
| "recall": 0.295221843003413 |
| }, |
| "org": { |
| "recovered": 35, |
| "total": 106, |
| "recall": 0.330188679245283 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 8.0, |
| "fixed_chars": 104.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 5.0, |
| "fixed_terms": 82.0, |
| "edits": 50.0, |
| "damaging_edits": 5.0, |
| "repairing_edits": 45.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.00075046904315197, |
| "damage_rate_terms": 0.01597444089456869, |
| "damage_rate_edits": 0.1, |
| "repair_rate_edits": 0.9 |
| }, |
| "cer_improvement": 0.06088413661027767, |
| "cer_improvement_rel": 0.21420550252667034, |
| "term_recall_gain": 0.062146892655367214, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "seconds": 7.922351083998365, |
| "index": { |
| "entries": 1000, |
| "indexed_readings": 1000, |
| "reachable_terms": 1000, |
| "grams": 1432, |
| "mean_posting": 18.372206703910614, |
| "max_posting": 341, |
| "ngram": 2, |
| "cache_hits": 3701, |
| "cache_misses": 6175 |
| }, |
| "tau": 0.28, |
| "gate_threshold": 0.82, |
| "glossary_covers_spoken_terms": 97, |
| "edits": 50, |
| "hallucinations_removed": 36 |
| } |
| }, |
| { |
| "condition": "E", |
| "label": "Mondegreen (after Q4_K_M quantisation)", |
| "glossary_size": 1000, |
| "sweep": "coverage", |
| "provenance": "not-measured", |
| "metrics": { |
| "n": 400, |
| "cer": 0.22334822853495054, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.20182985115389868, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 390, |
| "total": 1239, |
| "recall": 0.31476997578692495, |
| "by_category": { |
| "product": { |
| "recovered": 165, |
| "total": 487, |
| "recall": 0.33880903490759756 |
| }, |
| "person": { |
| "recovered": 173, |
| "total": 586, |
| "recall": 0.295221843003413 |
| }, |
| "org": { |
| "recovered": 35, |
| "total": 106, |
| "recall": 0.330188679245283 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 8.0, |
| "fixed_chars": 104.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 5.0, |
| "fixed_terms": 82.0, |
| "edits": 50.0, |
| "damaging_edits": 5.0, |
| "repairing_edits": 45.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.00075046904315197, |
| "damage_rate_terms": 0.01597444089456869, |
| "damage_rate_edits": 0.1, |
| "repair_rate_edits": 0.9 |
| }, |
| "cer_improvement": 0.06088413661027767, |
| "cer_improvement_rel": 0.21420550252667034, |
| "term_recall_gain": 0.062146892655367214, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "note": "no quantised checkpoint supplied; (E) reported as (D) with the LM re-ranking term absent -- which is quantisation's worst case, bounded by tests/test_quantization.py. Pass --quantized-model to measure the real number." |
| } |
| }, |
| { |
| "condition": "B", |
| "label": "Whisper initial_prompt stuffed with the glossary (244-token ceiling)", |
| "glossary_size": 3000, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2837535908075327, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.2535845964768538, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 322, |
| "total": 1239, |
| "recall": 0.2598870056497175, |
| "by_category": { |
| "product": { |
| "recovered": 137, |
| "total": 487, |
| "recall": 0.2813141683778234 |
| }, |
| "person": { |
| "recovered": 137, |
| "total": 586, |
| "recall": 0.23378839590443687 |
| }, |
| "org": { |
| "recovered": 30, |
| "total": 106, |
| "recall": 0.2830188679245283 |
| }, |
| "jargon": { |
| "recovered": 18, |
| "total": 60, |
| "recall": 0.3 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 15.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 9.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.0004787743376954978, |
| "cer_improvement_rel": 0.0016844469399213866, |
| "term_recall_gain": 0.007263922518159771, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "B", |
| "provenance": "simulated", |
| "capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 242, |
| "terms_included": 19, |
| "terms_dropped": 2981, |
| "coverage": 0.006333333333333333 |
| }, |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 6, |
| "intrusions": 6, |
| "prompt_capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 242, |
| "terms_included": 19, |
| "terms_dropped": 2981, |
| "coverage": 0.006333333333333333 |
| }, |
| "glossary_covers_spoken_terms": 298, |
| "targets_in_prompt": 3 |
| } |
| }, |
| { |
| "condition": "C", |
| "label": "cloud LLM post-processing", |
| "glossary_size": 3000, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.1881583147143313, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.17656698074559607, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 589, |
| "total": 1239, |
| "recall": 0.47538337368845845, |
| "by_category": { |
| "product": { |
| "recovered": 231, |
| "total": 487, |
| "recall": 0.47433264887063653 |
| }, |
| "person": { |
| "recovered": 287, |
| "total": 586, |
| "recall": 0.48976109215017066 |
| }, |
| "org": { |
| "recovered": 50, |
| "total": 106, |
| "recall": 0.4716981132075472 |
| }, |
| "jargon": { |
| "recovered": 21, |
| "total": 60, |
| "recall": 0.35 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 34.0, |
| "fixed_chars": 451.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 1.0, |
| "fixed_terms": 277.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0031894934333958724, |
| "damage_rate_terms": 0.003194888178913738, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.0960740504308969, |
| "cer_improvement_rel": 0.33801235261089274, |
| "term_recall_gain": 0.22276029055690072, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 60.0, |
| "removal_rate": 0.8571428571428571, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "C", |
| "provenance": "simulated", |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 175, |
| "gratuitous_rewrites": 1, |
| "hallucinations_removed": 47, |
| "note": "requires shipping the transcript to a third party", |
| "glossary_covers_spoken_terms": 298 |
| } |
| }, |
| { |
| "condition": "D", |
| "label": "Mondegreen (phonetic constraint + conservative gate)", |
| "glossary_size": 3000, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.19222789658474307, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.18161955482725659, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 508, |
| "total": 1239, |
| "recall": 0.4100080710250202, |
| "by_category": { |
| "product": { |
| "recovered": 201, |
| "total": 487, |
| "recall": 0.4127310061601643 |
| }, |
| "person": { |
| "recovered": 245, |
| "total": 586, |
| "recall": 0.4180887372013652 |
| }, |
| "org": { |
| "recovered": 42, |
| "total": 106, |
| "recall": 0.39622641509433965 |
| }, |
| "jargon": { |
| "recovered": 20, |
| "total": 60, |
| "recall": 0.3333333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 13.0, |
| "fixed_chars": 324.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 9.0, |
| "fixed_terms": 204.0, |
| "edits": 134.0, |
| "damaging_edits": 10.0, |
| "repairing_edits": 124.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0012195121951219512, |
| "damage_rate_terms": 0.02875399361022364, |
| "damage_rate_edits": 0.07462686567164178, |
| "repair_rate_edits": 0.9253731343283582 |
| }, |
| "cer_improvement": 0.09200446856048514, |
| "cer_improvement_rel": 0.32369455362156085, |
| "term_recall_gain": 0.15738498789346245, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "seconds": 13.009008041997731, |
| "index": { |
| "entries": 3000, |
| "indexed_readings": 3000, |
| "reachable_terms": 3000, |
| "grams": 1663, |
| "mean_posting": 47.09140108238124, |
| "max_posting": 974, |
| "ngram": 2, |
| "cache_hits": 3734, |
| "cache_misses": 6444 |
| }, |
| "tau": 0.28, |
| "gate_threshold": 0.82, |
| "glossary_covers_spoken_terms": 298, |
| "edits": 134, |
| "hallucinations_removed": 36 |
| } |
| }, |
| { |
| "condition": "E", |
| "label": "Mondegreen (after Q4_K_M quantisation)", |
| "glossary_size": 3000, |
| "sweep": "coverage", |
| "provenance": "not-measured", |
| "metrics": { |
| "n": 400, |
| "cer": 0.19222789658474307, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.18161955482725659, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 508, |
| "total": 1239, |
| "recall": 0.4100080710250202, |
| "by_category": { |
| "product": { |
| "recovered": 201, |
| "total": 487, |
| "recall": 0.4127310061601643 |
| }, |
| "person": { |
| "recovered": 245, |
| "total": 586, |
| "recall": 0.4180887372013652 |
| }, |
| "org": { |
| "recovered": 42, |
| "total": 106, |
| "recall": 0.39622641509433965 |
| }, |
| "jargon": { |
| "recovered": 20, |
| "total": 60, |
| "recall": 0.3333333333333333 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 13.0, |
| "fixed_chars": 324.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 9.0, |
| "fixed_terms": 204.0, |
| "edits": 134.0, |
| "damaging_edits": 10.0, |
| "repairing_edits": 124.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0012195121951219512, |
| "damage_rate_terms": 0.02875399361022364, |
| "damage_rate_edits": 0.07462686567164178, |
| "repair_rate_edits": 0.9253731343283582 |
| }, |
| "cer_improvement": 0.09200446856048514, |
| "cer_improvement_rel": 0.32369455362156085, |
| "term_recall_gain": 0.15738498789346245, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "note": "no quantised checkpoint supplied; (E) reported as (D) with the LM re-ranking term absent -- which is quantisation's worst case, bounded by tests/test_quantization.py. Pass --quantized-model to measure the real number." |
| } |
| }, |
| { |
| "condition": "B", |
| "label": "Whisper initial_prompt stuffed with the glossary (244-token ceiling)", |
| "glossary_size": 10000, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2796042132141717, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.2508534753516318, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 337, |
| "total": 1239, |
| "recall": 0.27199354317998387, |
| "by_category": { |
| "product": { |
| "recovered": 141, |
| "total": 487, |
| "recall": 0.28952772073921973 |
| }, |
| "person": { |
| "recovered": 148, |
| "total": 586, |
| "recall": 0.2525597269624573 |
| }, |
| "org": { |
| "recovered": 30, |
| "total": 106, |
| "recall": 0.2830188679245283 |
| }, |
| "jargon": { |
| "recovered": 18, |
| "total": 60, |
| "recall": 0.3 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 1.0, |
| "fixed_chars": 40.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 24.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 9.380863039399625e-05, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.004628151931056479, |
| "cer_improvement_rel": 0.016282987085906735, |
| "term_recall_gain": 0.01937046004842613, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "B", |
| "provenance": "simulated", |
| "capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 242, |
| "terms_included": 19, |
| "terms_dropped": 9981, |
| "coverage": 0.0019 |
| }, |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 15, |
| "intrusions": 3, |
| "prompt_capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 242, |
| "terms_included": 19, |
| "terms_dropped": 9981, |
| "coverage": 0.0019 |
| }, |
| "glossary_covers_spoken_terms": 987, |
| "targets_in_prompt": 3 |
| } |
| }, |
| { |
| "condition": "C", |
| "label": "cloud LLM post-processing", |
| "glossary_size": 10000, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.08929141398021066, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.09436023487641677, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 1028, |
| "total": 1239, |
| "recall": 0.8297013720742534, |
| "by_category": { |
| "product": { |
| "recovered": 414, |
| "total": 487, |
| "recall": 0.8501026694045175 |
| }, |
| "person": { |
| "recovered": 486, |
| "total": 586, |
| "recall": 0.8293515358361775 |
| }, |
| "org": { |
| "recovered": 86, |
| "total": 106, |
| "recall": 0.8113207547169812 |
| }, |
| "jargon": { |
| "recovered": 42, |
| "total": 60, |
| "recall": 0.7 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 70.0, |
| "fixed_chars": 1268.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 715.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.006566604127579738, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.19494095116501753, |
| "cer_improvement_rel": 0.6858506457046603, |
| "term_recall_gain": 0.5770782889426957, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 61.0, |
| "removal_rate": 0.8714285714285714, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "C", |
| "provenance": "simulated", |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 476, |
| "gratuitous_rewrites": 2, |
| "hallucinations_removed": 46, |
| "note": "requires shipping the transcript to a third party", |
| "glossary_covers_spoken_terms": 987 |
| } |
| }, |
| { |
| "condition": "D", |
| "label": "Mondegreen (phonetic constraint + conservative gate)", |
| "glossary_size": 10000, |
| "sweep": "coverage", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.11051707628471114, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.1216714461286358, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 822, |
| "total": 1239, |
| "recall": 0.6634382566585957, |
| "by_category": { |
| "product": { |
| "recovered": 327, |
| "total": 487, |
| "recall": 0.6714579055441479 |
| }, |
| "person": { |
| "recovered": 388, |
| "total": 586, |
| "recall": 0.6621160409556314 |
| }, |
| "org": { |
| "recovered": 68, |
| "total": 106, |
| "recall": 0.6415094339622641 |
| }, |
| "jargon": { |
| "recovered": 39, |
| "total": 60, |
| "recall": 0.65 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 1.0, |
| "fixed_chars": 906.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 509.0, |
| "edits": 326.0, |
| "damaging_edits": 1.0, |
| "repairing_edits": 325.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 9.380863039399625e-05, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.003067484662576687, |
| "repair_rate_edits": 0.9969325153374233 |
| }, |
| "cer_improvement": 0.17371528886051707, |
| "cer_improvement_rel": 0.6111734980348119, |
| "term_recall_gain": 0.41081517352703795, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "seconds": 29.62237279100009, |
| "index": { |
| "entries": 10000, |
| "indexed_readings": 10000, |
| "reachable_terms": 10000, |
| "grams": 1837, |
| "mean_posting": 142.00544365813826, |
| "max_posting": 3078, |
| "ngram": 2, |
| "cache_hits": 3716, |
| "cache_misses": 6260 |
| }, |
| "tau": 0.28, |
| "gate_threshold": 0.82, |
| "glossary_covers_spoken_terms": 987, |
| "edits": 326, |
| "hallucinations_removed": 36 |
| } |
| }, |
| { |
| "condition": "E", |
| "label": "Mondegreen (after Q4_K_M quantisation)", |
| "glossary_size": 10000, |
| "sweep": "coverage", |
| "provenance": "not-measured", |
| "metrics": { |
| "n": 400, |
| "cer": 0.11051707628471114, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.1216714461286358, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 822, |
| "total": 1239, |
| "recall": 0.6634382566585957, |
| "by_category": { |
| "product": { |
| "recovered": 327, |
| "total": 487, |
| "recall": 0.6714579055441479 |
| }, |
| "person": { |
| "recovered": 388, |
| "total": 586, |
| "recall": 0.6621160409556314 |
| }, |
| "org": { |
| "recovered": 68, |
| "total": 106, |
| "recall": 0.6415094339622641 |
| }, |
| "jargon": { |
| "recovered": 39, |
| "total": 60, |
| "recall": 0.65 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 313, |
| "total": 1239, |
| "recall": 0.25262308313155774, |
| "by_category": { |
| "product": { |
| "recovered": 133, |
| "total": 487, |
| "recall": 0.2731006160164271 |
| }, |
| "person": { |
| "recovered": 134, |
| "total": 586, |
| "recall": 0.22866894197952217 |
| }, |
| "org": { |
| "recovered": 29, |
| "total": 106, |
| "recall": 0.27358490566037735 |
| }, |
| "jargon": { |
| "recovered": 17, |
| "total": 60, |
| "recall": 0.2833333333333333 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 1.0, |
| "fixed_chars": 906.0, |
| "correct_terms_before": 313.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 509.0, |
| "edits": 326.0, |
| "damaging_edits": 1.0, |
| "repairing_edits": 325.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 9.380863039399625e-05, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.003067484662576687, |
| "repair_rate_edits": 0.9969325153374233 |
| }, |
| "cer_improvement": 0.17371528886051707, |
| "cer_improvement_rel": 0.6111734980348119, |
| "term_recall_gain": 0.41081517352703795, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "note": "no quantised checkpoint supplied; (E) reported as (D) with the LM re-ranking term absent -- which is quantisation's worst case, bounded by tests/test_quantization.py. Pass --quantized-model to measure the real number." |
| } |
| }, |
| { |
| "condition": "A", |
| "label": "raw Whisper (no correction)", |
| "glossary_size": 100, |
| "sweep": "distractor", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2842323651452282, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 0.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.0, |
| "cer_improvement_rel": 0.0, |
| "term_recall_gain": 0.0, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| } |
| }, |
| { |
| "condition": "A", |
| "label": "raw Whisper (no correction)", |
| "glossary_size": 1000, |
| "sweep": "distractor", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2842323651452282, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 0.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.0, |
| "cer_improvement_rel": 0.0, |
| "term_recall_gain": 0.0, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| } |
| }, |
| { |
| "condition": "A", |
| "label": "raw Whisper (no correction)", |
| "glossary_size": 9000, |
| "sweep": "distractor", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2842323651452282, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 0.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 0.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.0, |
| "cer_improvement_rel": 0.0, |
| "term_recall_gain": 0.0, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| } |
| }, |
| { |
| "condition": "B", |
| "label": "Whisper initial_prompt stuffed with the glossary (244-token ceiling)", |
| "glossary_size": 100, |
| "sweep": "distractor", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2813597191190552, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25139969957667624, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 43, |
| "total": 130, |
| "recall": 0.33076923076923076, |
| "by_category": { |
| "person": { |
| "recovered": 21, |
| "total": 71, |
| "recall": 0.29577464788732394 |
| }, |
| "product": { |
| "recovered": 12, |
| "total": 43, |
| "recall": 0.27906976744186046 |
| }, |
| "org": { |
| "recovered": 9, |
| "total": 14, |
| "recall": 0.6428571428571429 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 29.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 13.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.002872646026172987, |
| "cer_improvement_rel": 0.010106681639528318, |
| "term_recall_gain": 0.09999999999999998, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "B", |
| "provenance": "simulated", |
| "capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 239, |
| "terms_included": 19, |
| "terms_dropped": 81, |
| "coverage": 0.19 |
| }, |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 11, |
| "intrusions": 5, |
| "prompt_capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 239, |
| "terms_included": 19, |
| "terms_dropped": 81, |
| "coverage": 0.19 |
| }, |
| "glossary_covers_spoken_terms": 100, |
| "targets_in_prompt": 19 |
| } |
| }, |
| { |
| "condition": "C", |
| "label": "cloud LLM post-processing", |
| "glossary_size": 100, |
| "sweep": "distractor", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.22917331631024576, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.20906732213573673, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 94, |
| "total": 130, |
| "recall": 0.7230769230769231, |
| "by_category": { |
| "person": { |
| "recovered": 51, |
| "total": 71, |
| "recall": 0.7183098591549296 |
| }, |
| "product": { |
| "recovered": 31, |
| "total": 43, |
| "recall": 0.7209302325581395 |
| }, |
| "org": { |
| "recovered": 10, |
| "total": 14, |
| "recall": 0.7142857142857143 |
| }, |
| "jargon": { |
| "recovered": 2, |
| "total": 2, |
| "recall": 1.0 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 10.0, |
| "fixed_chars": 142.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 64.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0009380863039399625, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.055059048834982444, |
| "cer_improvement_rel": 0.19371139809096014, |
| "term_recall_gain": 0.4923076923076923, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 57.0, |
| "removal_rate": 0.8142857142857143, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "C", |
| "provenance": "simulated", |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 61, |
| "gratuitous_rewrites": 1, |
| "hallucinations_removed": 43, |
| "note": "requires shipping the transcript to a third party", |
| "glossary_covers_spoken_terms": 100 |
| } |
| }, |
| { |
| "condition": "D", |
| "label": "Mondegreen (phonetic constraint + conservative gate)", |
| "glossary_size": 100, |
| "sweep": "distractor", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.22255027130545801, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.20101051481633211, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 79, |
| "total": 130, |
| "recall": 0.6076923076923076, |
| "by_category": { |
| "person": { |
| "recovered": 38, |
| "total": 71, |
| "recall": 0.5352112676056338 |
| }, |
| "product": { |
| "recovered": 29, |
| "total": 43, |
| "recall": 0.6744186046511628 |
| }, |
| "org": { |
| "recovered": 11, |
| "total": 14, |
| "recall": 0.7857142857142857 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 4.0, |
| "fixed_chars": 109.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 49.0, |
| "edits": 48.0, |
| "damaging_edits": 2.0, |
| "repairing_edits": 46.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.000375234521575985, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.041666666666666664, |
| "repair_rate_edits": 0.9583333333333334 |
| }, |
| "cer_improvement": 0.06168209383977019, |
| "cer_improvement_rel": 0.21701291409320608, |
| "term_recall_gain": 0.37692307692307686, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "seconds": 3.104672167002718, |
| "index": { |
| "entries": 100, |
| "indexed_readings": 100, |
| "reachable_terms": 100, |
| "grams": 748, |
| "mean_posting": 3.2580213903743314, |
| "max_posting": 29, |
| "ngram": 2, |
| "cache_hits": 3679, |
| "cache_misses": 5990 |
| }, |
| "tau": 0.28, |
| "gate_threshold": 0.82, |
| "glossary_covers_spoken_terms": 100, |
| "edits": 48, |
| "hallucinations_removed": 36 |
| } |
| }, |
| { |
| "condition": "E", |
| "label": "Mondegreen (after Q4_K_M quantisation)", |
| "glossary_size": 100, |
| "sweep": "distractor", |
| "provenance": "not-measured", |
| "metrics": { |
| "n": 400, |
| "cer": 0.22255027130545801, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.20101051481633211, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 79, |
| "total": 130, |
| "recall": 0.6076923076923076, |
| "by_category": { |
| "person": { |
| "recovered": 38, |
| "total": 71, |
| "recall": 0.5352112676056338 |
| }, |
| "product": { |
| "recovered": 29, |
| "total": 43, |
| "recall": 0.6744186046511628 |
| }, |
| "org": { |
| "recovered": 11, |
| "total": 14, |
| "recall": 0.7857142857142857 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 4.0, |
| "fixed_chars": 109.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 49.0, |
| "edits": 48.0, |
| "damaging_edits": 2.0, |
| "repairing_edits": 46.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.000375234521575985, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.041666666666666664, |
| "repair_rate_edits": 0.9583333333333334 |
| }, |
| "cer_improvement": 0.06168209383977019, |
| "cer_improvement_rel": 0.21701291409320608, |
| "term_recall_gain": 0.37692307692307686, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "note": "no quantised checkpoint supplied; (E) reported as (D) with the LM re-ranking term absent -- which is quantisation's worst case, bounded by tests/test_quantization.py. Pass --quantized-model to measure the real number." |
| } |
| }, |
| { |
| "condition": "B", |
| "label": "Whisper initial_prompt stuffed with the glossary (244-token ceiling)", |
| "glossary_size": 1000, |
| "sweep": "distractor", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2847111394829237, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.253994264645637, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 33, |
| "total": 130, |
| "recall": 0.25384615384615383, |
| "by_category": { |
| "person": { |
| "recovered": 15, |
| "total": 71, |
| "recall": 0.2112676056338028 |
| }, |
| "product": { |
| "recovered": 9, |
| "total": 43, |
| "recall": 0.20930232558139536 |
| }, |
| "org": { |
| "recovered": 8, |
| "total": 14, |
| "recall": 0.5714285714285714 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 9.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 3.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": -0.0004787743376954978, |
| "cer_improvement_rel": -0.0016844469399213866, |
| "term_recall_gain": 0.02307692307692305, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "B", |
| "provenance": "simulated", |
| "capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 241, |
| "terms_included": 18, |
| "terms_dropped": 982, |
| "coverage": 0.018 |
| }, |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 3, |
| "intrusions": 5, |
| "prompt_capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 241, |
| "terms_included": 18, |
| "terms_dropped": 982, |
| "coverage": 0.018 |
| }, |
| "glossary_covers_spoken_terms": 100, |
| "targets_in_prompt": 1 |
| } |
| }, |
| { |
| "condition": "C", |
| "label": "cloud LLM post-processing", |
| "glossary_size": 1000, |
| "sweep": "distractor", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.22917331631024576, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.20906732213573673, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 94, |
| "total": 130, |
| "recall": 0.7230769230769231, |
| "by_category": { |
| "person": { |
| "recovered": 51, |
| "total": 71, |
| "recall": 0.7183098591549296 |
| }, |
| "product": { |
| "recovered": 31, |
| "total": 43, |
| "recall": 0.7209302325581395 |
| }, |
| "org": { |
| "recovered": 10, |
| "total": 14, |
| "recall": 0.7142857142857143 |
| }, |
| "jargon": { |
| "recovered": 2, |
| "total": 2, |
| "recall": 1.0 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 10.0, |
| "fixed_chars": 142.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 64.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0009380863039399625, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.055059048834982444, |
| "cer_improvement_rel": 0.19371139809096014, |
| "term_recall_gain": 0.4923076923076923, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 57.0, |
| "removal_rate": 0.8142857142857143, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "C", |
| "provenance": "simulated", |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 61, |
| "gratuitous_rewrites": 1, |
| "hallucinations_removed": 43, |
| "note": "requires shipping the transcript to a third party", |
| "glossary_covers_spoken_terms": 100 |
| } |
| }, |
| { |
| "condition": "D", |
| "label": "Mondegreen (phonetic constraint + conservative gate)", |
| "glossary_size": 1000, |
| "sweep": "distractor", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2229492499202043, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.20142018298511538, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 79, |
| "total": 130, |
| "recall": 0.6076923076923076, |
| "by_category": { |
| "person": { |
| "recovered": 38, |
| "total": 71, |
| "recall": 0.5352112676056338 |
| }, |
| "product": { |
| "recovered": 29, |
| "total": 43, |
| "recall": 0.6744186046511628 |
| }, |
| "org": { |
| "recovered": 11, |
| "total": 14, |
| "recall": 0.7857142857142857 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 8.0, |
| "fixed_chars": 109.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 49.0, |
| "edits": 51.0, |
| "damaging_edits": 5.0, |
| "repairing_edits": 46.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.00075046904315197, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.09803921568627451, |
| "repair_rate_edits": 0.9019607843137255 |
| }, |
| "cer_improvement": 0.061283115225023915, |
| "cer_improvement_rel": 0.21560920830993816, |
| "term_recall_gain": 0.37692307692307686, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "seconds": 7.920975792003446, |
| "index": { |
| "entries": 1000, |
| "indexed_readings": 1000, |
| "reachable_terms": 1000, |
| "grams": 1434, |
| "mean_posting": 18.350069735006972, |
| "max_posting": 342, |
| "ngram": 2, |
| "cache_hits": 3701, |
| "cache_misses": 6174 |
| }, |
| "tau": 0.28, |
| "gate_threshold": 0.82, |
| "glossary_covers_spoken_terms": 100, |
| "edits": 51, |
| "hallucinations_removed": 36 |
| } |
| }, |
| { |
| "condition": "E", |
| "label": "Mondegreen (after Q4_K_M quantisation)", |
| "glossary_size": 1000, |
| "sweep": "distractor", |
| "provenance": "not-measured", |
| "metrics": { |
| "n": 400, |
| "cer": 0.2229492499202043, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.20142018298511538, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 79, |
| "total": 130, |
| "recall": 0.6076923076923076, |
| "by_category": { |
| "person": { |
| "recovered": 38, |
| "total": 71, |
| "recall": 0.5352112676056338 |
| }, |
| "product": { |
| "recovered": 29, |
| "total": 43, |
| "recall": 0.6744186046511628 |
| }, |
| "org": { |
| "recovered": 11, |
| "total": 14, |
| "recall": 0.7857142857142857 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 8.0, |
| "fixed_chars": 109.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 49.0, |
| "edits": 51.0, |
| "damaging_edits": 5.0, |
| "repairing_edits": 46.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.00075046904315197, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.09803921568627451, |
| "repair_rate_edits": 0.9019607843137255 |
| }, |
| "cer_improvement": 0.061283115225023915, |
| "cer_improvement_rel": 0.21560920830993816, |
| "term_recall_gain": 0.37692307692307686, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "note": "no quantised checkpoint supplied; (E) reported as (D) with the LM re-ranking term absent -- which is quantisation's worst case, bounded by tests/test_quantization.py. Pass --quantized-model to measure the real number." |
| } |
| }, |
| { |
| "condition": "B", |
| "label": "Whisper initial_prompt stuffed with the glossary (244-token ceiling)", |
| "glossary_size": 9000, |
| "sweep": "distractor", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.28447175231407595, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.25426737675815925, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 32, |
| "total": 130, |
| "recall": 0.24615384615384617, |
| "by_category": { |
| "person": { |
| "recovered": 15, |
| "total": 71, |
| "recall": 0.2112676056338028 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 8, |
| "total": 14, |
| "recall": 0.5714285714285714 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 0.0, |
| "fixed_chars": 8.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 2.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": -0.0002393871688477489, |
| "cer_improvement_rel": -0.0008422234699606933, |
| "term_recall_gain": 0.015384615384615385, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 0.0, |
| "removal_rate": 0.0, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "B", |
| "provenance": "simulated", |
| "capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 240, |
| "terms_included": 18, |
| "terms_dropped": 8982, |
| "coverage": 0.002 |
| }, |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 2, |
| "intrusions": 5, |
| "prompt_capacity": { |
| "token_limit": 244, |
| "tokenizer": "heuristic", |
| "tokens_used": 240, |
| "terms_included": 18, |
| "terms_dropped": 8982, |
| "coverage": 0.002 |
| }, |
| "glossary_covers_spoken_terms": 100, |
| "targets_in_prompt": 0 |
| } |
| }, |
| { |
| "condition": "C", |
| "label": "cloud LLM post-processing", |
| "glossary_size": 9000, |
| "sweep": "distractor", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.22917331631024576, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.20906732213573673, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 94, |
| "total": 130, |
| "recall": 0.7230769230769231, |
| "by_category": { |
| "person": { |
| "recovered": 51, |
| "total": 71, |
| "recall": 0.7183098591549296 |
| }, |
| "product": { |
| "recovered": 31, |
| "total": 43, |
| "recall": 0.7209302325581395 |
| }, |
| "org": { |
| "recovered": 10, |
| "total": 14, |
| "recall": 0.7142857142857143 |
| }, |
| "jargon": { |
| "recovered": 2, |
| "total": 2, |
| "recall": 1.0 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 10.0, |
| "fixed_chars": 142.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 64.0, |
| "edits": 0.0, |
| "damaging_edits": 0.0, |
| "repairing_edits": 0.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0009380863039399625, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "repair_rate_edits": 0.0 |
| }, |
| "cer_improvement": 0.055059048834982444, |
| "cer_improvement_rel": 0.19371139809096014, |
| "term_recall_gain": 0.4923076923076923, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 57.0, |
| "removal_rate": 0.8142857142857143, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "condition": "C", |
| "provenance": "simulated", |
| "params": { |
| "prompt_recovery_max": 0.55, |
| "prompt_recovery_decay": 4.0, |
| "prompt_recovery_offprompt": 0.02, |
| "prompt_intrusion": 0.015, |
| "cloud_recovery_max": 0.78, |
| "cloud_recovery_decay": 2.2, |
| "cloud_rewrite_rate": 0.08, |
| "cloud_hallucination_removal": 0.8, |
| "seed": 20260823.0 |
| }, |
| "recovered_spans": 61, |
| "gratuitous_rewrites": 1, |
| "hallucinations_removed": 43, |
| "note": "requires shipping the transcript to a third party", |
| "glossary_covers_spoken_terms": 100 |
| } |
| }, |
| { |
| "condition": "D", |
| "label": "Mondegreen (phonetic constraint + conservative gate)", |
| "glossary_size": 9000, |
| "sweep": "distractor", |
| "provenance": "simulated", |
| "metrics": { |
| "n": 400, |
| "cer": 0.22095435684647302, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.20196640721015977, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 76, |
| "total": 130, |
| "recall": 0.5846153846153846, |
| "by_category": { |
| "person": { |
| "recovered": 36, |
| "total": 71, |
| "recall": 0.5070422535211268 |
| }, |
| "product": { |
| "recovered": 29, |
| "total": 43, |
| "recall": 0.6744186046511628 |
| }, |
| "org": { |
| "recovered": 10, |
| "total": 14, |
| "recall": 0.7142857142857143 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 25.0, |
| "fixed_chars": 128.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 46.0, |
| "edits": 76.0, |
| "damaging_edits": 17.0, |
| "repairing_edits": 59.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0023452157598499064, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.2236842105263158, |
| "repair_rate_edits": 0.7763157894736842 |
| }, |
| "cer_improvement": 0.06327800829875518, |
| "cer_improvement_rel": 0.22262773722627738, |
| "term_recall_gain": 0.35384615384615387, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "seconds": 28.093405957999494, |
| "index": { |
| "entries": 9000, |
| "indexed_readings": 9000, |
| "reachable_terms": 9000, |
| "grams": 1823, |
| "mean_posting": 129.40702139330773, |
| "max_posting": 2772, |
| "ngram": 2, |
| "cache_hits": 3739, |
| "cache_misses": 6496 |
| }, |
| "tau": 0.28, |
| "gate_threshold": 0.82, |
| "glossary_covers_spoken_terms": 100, |
| "edits": 76, |
| "hallucinations_removed": 36 |
| } |
| }, |
| { |
| "condition": "E", |
| "label": "Mondegreen (after Q4_K_M quantisation)", |
| "glossary_size": 9000, |
| "sweep": "distractor", |
| "provenance": "not-measured", |
| "metrics": { |
| "n": 400, |
| "cer": 0.22095435684647302, |
| "cer_baseline": 0.2842323651452282, |
| "wer": 0.20196640721015977, |
| "wer_baseline": 0.25413082070189813, |
| "term_recall": { |
| "recovered": 76, |
| "total": 130, |
| "recall": 0.5846153846153846, |
| "by_category": { |
| "person": { |
| "recovered": 36, |
| "total": 71, |
| "recall": 0.5070422535211268 |
| }, |
| "product": { |
| "recovered": 29, |
| "total": 43, |
| "recall": 0.6744186046511628 |
| }, |
| "org": { |
| "recovered": 10, |
| "total": 14, |
| "recall": 0.7142857142857143 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "term_recall_baseline": { |
| "recovered": 30, |
| "total": 130, |
| "recall": 0.23076923076923078, |
| "by_category": { |
| "person": { |
| "recovered": 14, |
| "total": 71, |
| "recall": 0.19718309859154928 |
| }, |
| "product": { |
| "recovered": 8, |
| "total": 43, |
| "recall": 0.18604651162790697 |
| }, |
| "org": { |
| "recovered": 7, |
| "total": 14, |
| "recall": 0.5 |
| }, |
| "jargon": { |
| "recovered": 1, |
| "total": 2, |
| "recall": 0.5 |
| } |
| } |
| }, |
| "damage": { |
| "correct_chars_before": 10660.0, |
| "broken_chars": 25.0, |
| "fixed_chars": 128.0, |
| "correct_terms_before": 30.0, |
| "broken_terms": 0.0, |
| "fixed_terms": 46.0, |
| "edits": 76.0, |
| "damaging_edits": 17.0, |
| "repairing_edits": 59.0, |
| "neutral_edits": 0.0, |
| "damage_rate_chars": 0.0023452157598499064, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.2236842105263158, |
| "repair_rate_edits": 0.7763157894736842 |
| }, |
| "cer_improvement": 0.06327800829875518, |
| "cer_improvement_rel": 0.22262773722627738, |
| "term_recall_gain": 0.35384615384615387, |
| "hallucination": { |
| "hallucinations": 70.0, |
| "removed": 52.0, |
| "removal_rate": 0.7428571428571429, |
| "legitimate_occurrences": 0.0, |
| "false_removals": 0.0, |
| "false_removal_rate": 0.0 |
| } |
| }, |
| "extra": { |
| "note": "no quantised checkpoint supplied; (E) reported as (D) with the LM re-ranking term absent -- which is quantisation's worst case, bounded by tests/test_quantization.py. Pass --quantized-model to measure the real number." |
| } |
| } |
| ], |
| "eval_sweep": [ |
| { |
| "threshold": 0.0, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.01, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.02, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.03, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.04, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.05, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.06, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.07, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.08, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.09, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.1, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.11, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.12, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.13, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.14, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.15, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.16, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.17, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.18, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.19, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.2, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.21, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.22, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.23, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.24, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.25, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.26, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.27, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.28, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.29, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.3, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.31, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.32, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.33, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.34, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.35, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.36, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.37, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.38, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.39, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.4, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.41, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.42, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.43, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.44, |
| "correction_rate": 0.5518358531317494, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.023809523809523808, |
| "accepted": 336.0, |
| "repairs": 328.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.45, |
| "correction_rate": 0.5518358531317494, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.023809523809523808, |
| "accepted": 336.0, |
| "repairs": 328.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.46, |
| "correction_rate": 0.5518358531317494, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.023809523809523808, |
| "accepted": 336.0, |
| "repairs": 328.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.47, |
| "correction_rate": 0.5518358531317494, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.023809523809523808, |
| "accepted": 336.0, |
| "repairs": 328.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.48, |
| "correction_rate": 0.5518358531317494, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.023809523809523808, |
| "accepted": 336.0, |
| "repairs": 328.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.49, |
| "correction_rate": 0.5518358531317494, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.023809523809523808, |
| "accepted": 336.0, |
| "repairs": 328.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.5, |
| "correction_rate": 0.5518358531317494, |
| "damage_rate": 0.009584664536741214, |
| "edit_damage_rate": 0.015015015015015015, |
| "accepted": 333.0, |
| "repairs": 328.0, |
| "damages": 5.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.51, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012084592145015106, |
| "accepted": 331.0, |
| "repairs": 327.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.52, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012084592145015106, |
| "accepted": 331.0, |
| "repairs": 327.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.53, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012084592145015106, |
| "accepted": 331.0, |
| "repairs": 327.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.54, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012084592145015106, |
| "accepted": 331.0, |
| "repairs": 327.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.55, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012084592145015106, |
| "accepted": 331.0, |
| "repairs": 327.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.56, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012084592145015106, |
| "accepted": 331.0, |
| "repairs": 327.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.57, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012084592145015106, |
| "accepted": 331.0, |
| "repairs": 327.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.58, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012121212121212121, |
| "accepted": 330.0, |
| "repairs": 326.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.59, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012121212121212121, |
| "accepted": 330.0, |
| "repairs": 326.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.6, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012121212121212121, |
| "accepted": 330.0, |
| "repairs": 326.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.61, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012121212121212121, |
| "accepted": 330.0, |
| "repairs": 326.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.62, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012121212121212121, |
| "accepted": 330.0, |
| "repairs": 326.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.63, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012121212121212121, |
| "accepted": 330.0, |
| "repairs": 326.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.64, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012121212121212121, |
| "accepted": 330.0, |
| "repairs": 326.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.65, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.66, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.67, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.68, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.69, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.71, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.72, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.73, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.74, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.75, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.76, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.77, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.78, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0030581039755351682, |
| "accepted": 327.0, |
| "repairs": 326.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.79, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0030581039755351682, |
| "accepted": 327.0, |
| "repairs": 326.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0030581039755351682, |
| "accepted": 327.0, |
| "repairs": 326.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.81, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0030581039755351682, |
| "accepted": 327.0, |
| "repairs": 326.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.82, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003067484662576687, |
| "accepted": 326.0, |
| "repairs": 325.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.83, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003067484662576687, |
| "accepted": 326.0, |
| "repairs": 325.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.84, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003067484662576687, |
| "accepted": 326.0, |
| "repairs": 325.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.85, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003067484662576687, |
| "accepted": 326.0, |
| "repairs": 325.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.86, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003067484662576687, |
| "accepted": 326.0, |
| "repairs": 325.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.87, |
| "correction_rate": 0.5475161987041036, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003076923076923077, |
| "accepted": 325.0, |
| "repairs": 324.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.88, |
| "correction_rate": 0.5442764578833693, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0030959752321981426, |
| "accepted": 323.0, |
| "repairs": 322.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.89, |
| "correction_rate": 0.5442764578833693, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0030959752321981426, |
| "accepted": 323.0, |
| "repairs": 322.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "correction_rate": 0.5431965442764579, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003105590062111801, |
| "accepted": 322.0, |
| "repairs": 321.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.91, |
| "correction_rate": 0.5431965442764579, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003105590062111801, |
| "accepted": 322.0, |
| "repairs": 321.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.92, |
| "correction_rate": 0.5377969762419006, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0031446540880503146, |
| "accepted": 318.0, |
| "repairs": 317.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.93, |
| "correction_rate": 0.5356371490280778, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0031545741324921135, |
| "accepted": 317.0, |
| "repairs": 316.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.94, |
| "correction_rate": 0.5291576673866091, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0031847133757961785, |
| "accepted": 314.0, |
| "repairs": 313.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "correction_rate": 0.519438444924406, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003246753246753247, |
| "accepted": 308.0, |
| "repairs": 307.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.96, |
| "correction_rate": 0.5161987041036717, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0032679738562091504, |
| "accepted": 306.0, |
| "repairs": 305.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.97, |
| "correction_rate": 0.5, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003355704697986577, |
| "accepted": 298.0, |
| "repairs": 297.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.98, |
| "correction_rate": 0.4427645788336933, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0037593984962406013, |
| "accepted": 266.0, |
| "repairs": 265.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "correction_rate": 0.28509719222462204, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 172.0, |
| "repairs": 172.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 1.0, |
| "correction_rate": 0.0, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 0.0, |
| "repairs": 0.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| } |
| ], |
| "eval_grid": { |
| "0.2": [ |
| { |
| "threshold": 0.0, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.05, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.1, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.15, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.2, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.25, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.3, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.35, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.4, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.02373887240356083, |
| "accepted": 337.0, |
| "repairs": 329.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.45, |
| "correction_rate": 0.5518358531317494, |
| "damage_rate": 0.019169329073482427, |
| "edit_damage_rate": 0.023809523809523808, |
| "accepted": 336.0, |
| "repairs": 328.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.5, |
| "correction_rate": 0.5518358531317494, |
| "damage_rate": 0.009584664536741214, |
| "edit_damage_rate": 0.015015015015015015, |
| "accepted": 333.0, |
| "repairs": 328.0, |
| "damages": 5.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.55, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012084592145015106, |
| "accepted": 331.0, |
| "repairs": 327.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.6, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012121212121212121, |
| "accepted": 330.0, |
| "repairs": 326.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.65, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.75, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0030581039755351682, |
| "accepted": 327.0, |
| "repairs": 326.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.85, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003067484662576687, |
| "accepted": 326.0, |
| "repairs": 325.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "correction_rate": 0.5431965442764579, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003105590062111801, |
| "accepted": 322.0, |
| "repairs": 321.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "correction_rate": 0.5205183585313174, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003236245954692557, |
| "accepted": 309.0, |
| "repairs": 308.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 1.0, |
| "correction_rate": 0.0, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 0.0, |
| "repairs": 0.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| } |
| ], |
| "0.4": [ |
| { |
| "threshold": 0.0, |
| "correction_rate": 0.5572354211663066, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.011869436201780416, |
| "accepted": 337.0, |
| "repairs": 332.0, |
| "damages": 4.0, |
| "neutrals": 1.0 |
| }, |
| { |
| "threshold": 0.05, |
| "correction_rate": 0.5572354211663066, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.011869436201780416, |
| "accepted": 337.0, |
| "repairs": 332.0, |
| "damages": 4.0, |
| "neutrals": 1.0 |
| }, |
| { |
| "threshold": 0.1, |
| "correction_rate": 0.5572354211663066, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.011869436201780416, |
| "accepted": 337.0, |
| "repairs": 332.0, |
| "damages": 4.0, |
| "neutrals": 1.0 |
| }, |
| { |
| "threshold": 0.15, |
| "correction_rate": 0.5572354211663066, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.011904761904761904, |
| "accepted": 336.0, |
| "repairs": 332.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.2, |
| "correction_rate": 0.5572354211663066, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.011904761904761904, |
| "accepted": 336.0, |
| "repairs": 332.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.25, |
| "correction_rate": 0.5572354211663066, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.011904761904761904, |
| "accepted": 336.0, |
| "repairs": 332.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.3, |
| "correction_rate": 0.5572354211663066, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.011904761904761904, |
| "accepted": 336.0, |
| "repairs": 332.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.35, |
| "correction_rate": 0.5572354211663066, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.011904761904761904, |
| "accepted": 336.0, |
| "repairs": 332.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.4, |
| "correction_rate": 0.5572354211663066, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.011904761904761904, |
| "accepted": 336.0, |
| "repairs": 332.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.45, |
| "correction_rate": 0.5550755939524838, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.011940298507462687, |
| "accepted": 335.0, |
| "repairs": 331.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.5, |
| "correction_rate": 0.5550755939524838, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.011940298507462687, |
| "accepted": 335.0, |
| "repairs": 331.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.55, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.011976047904191617, |
| "accepted": 334.0, |
| "repairs": 330.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.6, |
| "correction_rate": 0.5529157667386609, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.012012012012012012, |
| "accepted": 333.0, |
| "repairs": 329.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.65, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006060606060606061, |
| "accepted": 330.0, |
| "repairs": 328.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006060606060606061, |
| "accepted": 330.0, |
| "repairs": 328.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.75, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0060790273556231, |
| "accepted": 329.0, |
| "repairs": 327.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0030581039755351682, |
| "accepted": 327.0, |
| "repairs": 326.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.85, |
| "correction_rate": 0.550755939524838, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0030581039755351682, |
| "accepted": 327.0, |
| "repairs": 326.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "correction_rate": 0.5399568034557235, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003115264797507788, |
| "accepted": 321.0, |
| "repairs": 320.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "correction_rate": 0.5140388768898488, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.003278688524590164, |
| "accepted": 305.0, |
| "repairs": 304.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 1.0, |
| "correction_rate": 0.0, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 0.0, |
| "repairs": 0.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| } |
| ], |
| "0.8": [ |
| { |
| "threshold": 0.0, |
| "correction_rate": 0.572354211663067, |
| "damage_rate": 0.022364217252396165, |
| "edit_damage_rate": 0.027932960893854747, |
| "accepted": 358.0, |
| "repairs": 341.0, |
| "damages": 10.0, |
| "neutrals": 7.0 |
| }, |
| { |
| "threshold": 0.05, |
| "correction_rate": 0.5712742980561555, |
| "damage_rate": 0.022364217252396165, |
| "edit_damage_rate": 0.028011204481792718, |
| "accepted": 357.0, |
| "repairs": 341.0, |
| "damages": 10.0, |
| "neutrals": 6.0 |
| }, |
| { |
| "threshold": 0.1, |
| "correction_rate": 0.5712742980561555, |
| "damage_rate": 0.022364217252396165, |
| "edit_damage_rate": 0.028011204481792718, |
| "accepted": 357.0, |
| "repairs": 341.0, |
| "damages": 10.0, |
| "neutrals": 6.0 |
| }, |
| { |
| "threshold": 0.15, |
| "correction_rate": 0.5712742980561555, |
| "damage_rate": 0.022364217252396165, |
| "edit_damage_rate": 0.028089887640449437, |
| "accepted": 356.0, |
| "repairs": 341.0, |
| "damages": 10.0, |
| "neutrals": 5.0 |
| }, |
| { |
| "threshold": 0.2, |
| "correction_rate": 0.5712742980561555, |
| "damage_rate": 0.022364217252396165, |
| "edit_damage_rate": 0.028169014084507043, |
| "accepted": 355.0, |
| "repairs": 341.0, |
| "damages": 10.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.25, |
| "correction_rate": 0.5701943844492441, |
| "damage_rate": 0.022364217252396165, |
| "edit_damage_rate": 0.025495750708215296, |
| "accepted": 353.0, |
| "repairs": 341.0, |
| "damages": 9.0, |
| "neutrals": 3.0 |
| }, |
| { |
| "threshold": 0.3, |
| "correction_rate": 0.5680345572354212, |
| "damage_rate": 0.009584664536741214, |
| "edit_damage_rate": 0.011527377521613832, |
| "accepted": 347.0, |
| "repairs": 340.0, |
| "damages": 4.0, |
| "neutrals": 3.0 |
| }, |
| { |
| "threshold": 0.35, |
| "correction_rate": 0.5637149028077754, |
| "damage_rate": 0.009584664536741214, |
| "edit_damage_rate": 0.011594202898550725, |
| "accepted": 345.0, |
| "repairs": 338.0, |
| "damages": 4.0, |
| "neutrals": 3.0 |
| }, |
| { |
| "threshold": 0.4, |
| "correction_rate": 0.5637149028077754, |
| "damage_rate": 0.009584664536741214, |
| "edit_damage_rate": 0.011627906976744186, |
| "accepted": 344.0, |
| "repairs": 337.0, |
| "damages": 4.0, |
| "neutrals": 3.0 |
| }, |
| { |
| "threshold": 0.45, |
| "correction_rate": 0.5637149028077754, |
| "damage_rate": 0.009584664536741214, |
| "edit_damage_rate": 0.011627906976744186, |
| "accepted": 344.0, |
| "repairs": 337.0, |
| "damages": 4.0, |
| "neutrals": 3.0 |
| }, |
| { |
| "threshold": 0.5, |
| "correction_rate": 0.5615550755939525, |
| "damage_rate": 0.009584664536741214, |
| "edit_damage_rate": 0.011695906432748537, |
| "accepted": 342.0, |
| "repairs": 335.0, |
| "damages": 4.0, |
| "neutrals": 3.0 |
| }, |
| { |
| "threshold": 0.55, |
| "correction_rate": 0.5593952483801296, |
| "damage_rate": 0.009584664536741214, |
| "edit_damage_rate": 0.011764705882352941, |
| "accepted": 340.0, |
| "repairs": 333.0, |
| "damages": 4.0, |
| "neutrals": 3.0 |
| }, |
| { |
| "threshold": 0.6, |
| "correction_rate": 0.5550755939524838, |
| "damage_rate": 0.009584664536741214, |
| "edit_damage_rate": 0.011869436201780416, |
| "accepted": 337.0, |
| "repairs": 331.0, |
| "damages": 4.0, |
| "neutrals": 2.0 |
| }, |
| { |
| "threshold": 0.65, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.003194888178913738, |
| "edit_damage_rate": 0.005988023952095809, |
| "accepted": 334.0, |
| "repairs": 330.0, |
| "damages": 2.0, |
| "neutrals": 2.0 |
| }, |
| { |
| "threshold": 0.7, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.003194888178913738, |
| "edit_damage_rate": 0.006006006006006006, |
| "accepted": 333.0, |
| "repairs": 330.0, |
| "damages": 2.0, |
| "neutrals": 1.0 |
| }, |
| { |
| "threshold": 0.75, |
| "correction_rate": 0.5485961123110151, |
| "damage_rate": 0.003194888178913738, |
| "edit_damage_rate": 0.0060790273556231, |
| "accepted": 329.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 1.0 |
| }, |
| { |
| "threshold": 0.8, |
| "correction_rate": 0.5453563714902808, |
| "damage_rate": 0.003194888178913738, |
| "edit_damage_rate": 0.006153846153846154, |
| "accepted": 325.0, |
| "repairs": 323.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.85, |
| "correction_rate": 0.5431965442764579, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0030959752321981426, |
| "accepted": 323.0, |
| "repairs": 322.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "correction_rate": 0.5280777537796977, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 313.0, |
| "repairs": 313.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "correction_rate": 0.4838012958963283, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 289.0, |
| "repairs": 289.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 1.0, |
| "correction_rate": 0.0, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 0.0, |
| "repairs": 0.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| } |
| ], |
| "1.6": [ |
| { |
| "threshold": 0.0, |
| "correction_rate": 0.5691144708423326, |
| "damage_rate": 0.03194888178913738, |
| "edit_damage_rate": 0.13238770685579196, |
| "accepted": 423.0, |
| "repairs": 353.0, |
| "damages": 56.0, |
| "neutrals": 14.0 |
| }, |
| { |
| "threshold": 0.05, |
| "correction_rate": 0.5637149028077754, |
| "damage_rate": 0.03194888178913738, |
| "edit_damage_rate": 0.12745098039215685, |
| "accepted": 408.0, |
| "repairs": 348.0, |
| "damages": 52.0, |
| "neutrals": 8.0 |
| }, |
| { |
| "threshold": 0.1, |
| "correction_rate": 0.562634989200864, |
| "damage_rate": 0.02875399361022364, |
| "edit_damage_rate": 0.11838790931989925, |
| "accepted": 397.0, |
| "repairs": 346.0, |
| "damages": 47.0, |
| "neutrals": 4.0 |
| }, |
| { |
| "threshold": 0.15, |
| "correction_rate": 0.562634989200864, |
| "damage_rate": 0.02875399361022364, |
| "edit_damage_rate": 0.11450381679389313, |
| "accepted": 393.0, |
| "repairs": 345.0, |
| "damages": 45.0, |
| "neutrals": 3.0 |
| }, |
| { |
| "threshold": 0.2, |
| "correction_rate": 0.562634989200864, |
| "damage_rate": 0.02875399361022364, |
| "edit_damage_rate": 0.0521978021978022, |
| "accepted": 364.0, |
| "repairs": 344.0, |
| "damages": 19.0, |
| "neutrals": 1.0 |
| }, |
| { |
| "threshold": 0.25, |
| "correction_rate": 0.5593952483801296, |
| "damage_rate": 0.02875399361022364, |
| "edit_damage_rate": 0.052486187845303865, |
| "accepted": 362.0, |
| "repairs": 342.0, |
| "damages": 19.0, |
| "neutrals": 1.0 |
| }, |
| { |
| "threshold": 0.3, |
| "correction_rate": 0.5572354211663066, |
| "damage_rate": 0.012779552715654952, |
| "edit_damage_rate": 0.03389830508474576, |
| "accepted": 354.0, |
| "repairs": 341.0, |
| "damages": 12.0, |
| "neutrals": 1.0 |
| }, |
| { |
| "threshold": 0.35, |
| "correction_rate": 0.5529157667386609, |
| "damage_rate": 0.012779552715654952, |
| "edit_damage_rate": 0.03151862464183381, |
| "accepted": 349.0, |
| "repairs": 338.0, |
| "damages": 11.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.4, |
| "correction_rate": 0.5475161987041036, |
| "damage_rate": 0.012779552715654952, |
| "edit_damage_rate": 0.029069767441860465, |
| "accepted": 344.0, |
| "repairs": 334.0, |
| "damages": 10.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.45, |
| "correction_rate": 0.5442764578833693, |
| "damage_rate": 0.012779552715654952, |
| "edit_damage_rate": 0.02932551319648094, |
| "accepted": 341.0, |
| "repairs": 331.0, |
| "damages": 10.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.5, |
| "correction_rate": 0.5431965442764579, |
| "damage_rate": 0.012779552715654952, |
| "edit_damage_rate": 0.029411764705882353, |
| "accepted": 340.0, |
| "repairs": 330.0, |
| "damages": 10.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.55, |
| "correction_rate": 0.5388768898488121, |
| "damage_rate": 0.012779552715654952, |
| "edit_damage_rate": 0.026865671641791045, |
| "accepted": 335.0, |
| "repairs": 326.0, |
| "damages": 9.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.6, |
| "correction_rate": 0.5334773218142549, |
| "damage_rate": 0.009584664536741214, |
| "edit_damage_rate": 0.021212121212121213, |
| "accepted": 330.0, |
| "repairs": 323.0, |
| "damages": 7.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.65, |
| "correction_rate": 0.521598272138229, |
| "damage_rate": 0.003194888178913738, |
| "edit_damage_rate": 0.012578616352201259, |
| "accepted": 318.0, |
| "repairs": 314.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "correction_rate": 0.509719222462203, |
| "damage_rate": 0.003194888178913738, |
| "edit_damage_rate": 0.012861736334405145, |
| "accepted": 311.0, |
| "repairs": 307.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.75, |
| "correction_rate": 0.49136069114470843, |
| "damage_rate": 0.003194888178913738, |
| "edit_damage_rate": 0.013245033112582781, |
| "accepted": 302.0, |
| "repairs": 298.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "correction_rate": 0.4838012958963283, |
| "damage_rate": 0.003194888178913738, |
| "edit_damage_rate": 0.013559322033898305, |
| "accepted": 295.0, |
| "repairs": 291.0, |
| "damages": 4.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.85, |
| "correction_rate": 0.4524838012958963, |
| "damage_rate": 0.003194888178913738, |
| "edit_damage_rate": 0.01079136690647482, |
| "accepted": 278.0, |
| "repairs": 275.0, |
| "damages": 3.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "correction_rate": 0.4136069114470842, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.00398406374501992, |
| "accepted": 251.0, |
| "repairs": 250.0, |
| "damages": 1.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "correction_rate": 0.3714902807775378, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 227.0, |
| "repairs": 227.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 1.0, |
| "correction_rate": 0.0, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 0.0, |
| "repairs": 0.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| } |
| ], |
| "3.2": [ |
| { |
| "threshold": 0.0, |
| "correction_rate": 0.5539956803455723, |
| "damage_rate": 0.2108626198083067, |
| "edit_damage_rate": 0.28867924528301886, |
| "accepted": 530.0, |
| "repairs": 360.0, |
| "damages": 153.0, |
| "neutrals": 17.0 |
| }, |
| { |
| "threshold": 0.05, |
| "correction_rate": 0.5550755939524838, |
| "damage_rate": 0.12460063897763578, |
| "edit_damage_rate": 0.21806167400881057, |
| "accepted": 454.0, |
| "repairs": 348.0, |
| "damages": 99.0, |
| "neutrals": 7.0 |
| }, |
| { |
| "threshold": 0.1, |
| "correction_rate": 0.5485961123110151, |
| "damage_rate": 0.10223642172523961, |
| "edit_damage_rate": 0.20093457943925233, |
| "accepted": 428.0, |
| "repairs": 339.0, |
| "damages": 86.0, |
| "neutrals": 3.0 |
| }, |
| { |
| "threshold": 0.15, |
| "correction_rate": 0.5485961123110151, |
| "damage_rate": 0.0926517571884984, |
| "edit_damage_rate": 0.18159806295399517, |
| "accepted": 413.0, |
| "repairs": 336.0, |
| "damages": 75.0, |
| "neutrals": 2.0 |
| }, |
| { |
| "threshold": 0.2, |
| "correction_rate": 0.5475161987041036, |
| "damage_rate": 0.07348242811501597, |
| "edit_damage_rate": 0.11375661375661375, |
| "accepted": 378.0, |
| "repairs": 333.0, |
| "damages": 43.0, |
| "neutrals": 2.0 |
| }, |
| { |
| "threshold": 0.25, |
| "correction_rate": 0.5453563714902808, |
| "damage_rate": 0.054313099041533544, |
| "edit_damage_rate": 0.09536784741144415, |
| "accepted": 367.0, |
| "repairs": 330.0, |
| "damages": 35.0, |
| "neutrals": 2.0 |
| }, |
| { |
| "threshold": 0.3, |
| "correction_rate": 0.5453563714902808, |
| "damage_rate": 0.04153354632587859, |
| "edit_damage_rate": 0.08055555555555556, |
| "accepted": 360.0, |
| "repairs": 329.0, |
| "damages": 29.0, |
| "neutrals": 2.0 |
| }, |
| { |
| "threshold": 0.35, |
| "correction_rate": 0.5388768898488121, |
| "damage_rate": 0.04153354632587859, |
| "edit_damage_rate": 0.07386363636363637, |
| "accepted": 352.0, |
| "repairs": 324.0, |
| "damages": 26.0, |
| "neutrals": 2.0 |
| }, |
| { |
| "threshold": 0.4, |
| "correction_rate": 0.5356371490280778, |
| "damage_rate": 0.03514376996805112, |
| "edit_damage_rate": 0.06666666666666667, |
| "accepted": 345.0, |
| "repairs": 321.0, |
| "damages": 23.0, |
| "neutrals": 1.0 |
| }, |
| { |
| "threshold": 0.45, |
| "correction_rate": 0.5334773218142549, |
| "damage_rate": 0.03514376996805112, |
| "edit_damage_rate": 0.06140350877192982, |
| "accepted": 342.0, |
| "repairs": 320.0, |
| "damages": 21.0, |
| "neutrals": 1.0 |
| }, |
| { |
| "threshold": 0.5, |
| "correction_rate": 0.531317494600432, |
| "damage_rate": 0.022364217252396165, |
| "edit_damage_rate": 0.05325443786982249, |
| "accepted": 338.0, |
| "repairs": 319.0, |
| "damages": 18.0, |
| "neutrals": 1.0 |
| }, |
| { |
| "threshold": 0.55, |
| "correction_rate": 0.5291576673866091, |
| "damage_rate": 0.012779552715654952, |
| "edit_damage_rate": 0.045180722891566265, |
| "accepted": 332.0, |
| "repairs": 317.0, |
| "damages": 15.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.6, |
| "correction_rate": 0.5205183585313174, |
| "damage_rate": 0.006389776357827476, |
| "edit_damage_rate": 0.04, |
| "accepted": 325.0, |
| "repairs": 312.0, |
| "damages": 13.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.65, |
| "correction_rate": 0.4946004319654428, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.032679738562091505, |
| "accepted": 306.0, |
| "repairs": 296.0, |
| "damages": 10.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "correction_rate": 0.48056155507559395, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.030508474576271188, |
| "accepted": 295.0, |
| "repairs": 286.0, |
| "damages": 9.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.75, |
| "correction_rate": 0.45680345572354214, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.028268551236749116, |
| "accepted": 283.0, |
| "repairs": 275.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "correction_rate": 0.44600431965442766, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.02909090909090909, |
| "accepted": 275.0, |
| "repairs": 267.0, |
| "damages": 8.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.85, |
| "correction_rate": 0.41252699784017277, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.027237354085603113, |
| "accepted": 257.0, |
| "repairs": 250.0, |
| "damages": 7.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "correction_rate": 0.37688984881209503, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.02127659574468085, |
| "accepted": 235.0, |
| "repairs": 230.0, |
| "damages": 5.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "correction_rate": 0.3509719222462203, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 216.0, |
| "repairs": 216.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| }, |
| { |
| "threshold": 1.0, |
| "correction_rate": 0.0, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.0, |
| "accepted": 0.0, |
| "repairs": 0.0, |
| "damages": 0.0, |
| "neutrals": 0.0 |
| } |
| ] |
| }, |
| "throughput": [ |
| { |
| "label": "D@10", |
| "n_texts": 150, |
| "chars": 5254, |
| "seconds": 0.9162743329980003, |
| "chars_per_second": 5734.090556492175, |
| "hours_of_audio_per_second": 0.27305193126153215, |
| "seconds_per_hour_of_audio": 3.662307002846975, |
| "peak_rss_mb": 162.046875, |
| "corrections": 0, |
| "glossary_terms": 10, |
| "machine": { |
| "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "processor": "arm", |
| "python": "3.13.5", |
| "cpu_count": 8, |
| "cpu_brand": "Apple M2", |
| "memory_gb": 16.0 |
| }, |
| "notes": "cold_start=0.003s (index build + reader init, excluded from throughput); 1h of audio assumed to be 21000 characters" |
| }, |
| { |
| "label": "D@30", |
| "n_texts": 150, |
| "chars": 5254, |
| "seconds": 0.9786489579964837, |
| "chars_per_second": 5368.625753974264, |
| "hours_of_audio_per_second": 0.2556488454273459, |
| "seconds_per_hour_of_audio": 3.911615553469006, |
| "peak_rss_mb": 163.40625, |
| "corrections": 0, |
| "glossary_terms": 30, |
| "machine": { |
| "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "processor": "arm", |
| "python": "3.13.5", |
| "cpu_count": 8, |
| "cpu_brand": "Apple M2", |
| "memory_gb": 16.0 |
| }, |
| "notes": "cold_start=0.004s (index build + reader init, excluded from throughput); 1h of audio assumed to be 21000 characters" |
| }, |
| { |
| "label": "D@100", |
| "n_texts": 150, |
| "chars": 5254, |
| "seconds": 1.2076335409983585, |
| "chars_per_second": 4350.6575642611615, |
| "hours_of_audio_per_second": 0.20717416972672198, |
| "seconds_per_hour_of_audio": 4.826856558996104, |
| "peak_rss_mb": 164.875, |
| "corrections": 0, |
| "glossary_terms": 100, |
| "machine": { |
| "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "processor": "arm", |
| "python": "3.13.5", |
| "cpu_count": 8, |
| "cpu_brand": "Apple M2", |
| "memory_gb": 16.0 |
| }, |
| "notes": "cold_start=0.008s (index build + reader init, excluded from throughput); 1h of audio assumed to be 21000 characters" |
| }, |
| { |
| "label": "D@300", |
| "n_texts": 150, |
| "chars": 5254, |
| "seconds": 1.811501791999035, |
| "chars_per_second": 2900.3559495252207, |
| "hours_of_audio_per_second": 0.13811218807262957, |
| "seconds_per_hour_of_audio": 7.240490603726634, |
| "peak_rss_mb": 166.09375, |
| "corrections": 2, |
| "glossary_terms": 300, |
| "machine": { |
| "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "processor": "arm", |
| "python": "3.13.5", |
| "cpu_count": 8, |
| "cpu_brand": "Apple M2", |
| "memory_gb": 16.0 |
| }, |
| "notes": "cold_start=0.018s (index build + reader init, excluded from throughput); 1h of audio assumed to be 21000 characters" |
| }, |
| { |
| "label": "D@1000", |
| "n_texts": 150, |
| "chars": 5254, |
| "seconds": 3.0188671670002805, |
| "chars_per_second": 1740.3879367175587, |
| "hours_of_audio_per_second": 0.08287561603416946, |
| "seconds_per_hour_of_audio": 12.066275315379881, |
| "peak_rss_mb": 168.515625, |
| "corrections": 16, |
| "glossary_terms": 1000, |
| "machine": { |
| "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "processor": "arm", |
| "python": "3.13.5", |
| "cpu_count": 8, |
| "cpu_brand": "Apple M2", |
| "memory_gb": 16.0 |
| }, |
| "notes": "cold_start=0.037s (index build + reader init, excluded from throughput); 1h of audio assumed to be 21000 characters" |
| }, |
| { |
| "label": "D@3000", |
| "n_texts": 150, |
| "chars": 5254, |
| "seconds": 4.964868040999136, |
| "chars_per_second": 1058.2355777864095, |
| "hours_of_audio_per_second": 0.050392170370781406, |
| "seconds_per_hour_of_audio": 19.8443526572101, |
| "peak_rss_mb": 173.171875, |
| "corrections": 46, |
| "glossary_terms": 3000, |
| "machine": { |
| "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "processor": "arm", |
| "python": "3.13.5", |
| "cpu_count": 8, |
| "cpu_brand": "Apple M2", |
| "memory_gb": 16.0 |
| }, |
| "notes": "cold_start=0.065s (index build + reader init, excluded from throughput); 1h of audio assumed to be 21000 characters" |
| }, |
| { |
| "label": "D@10000", |
| "n_texts": 150, |
| "chars": 5254, |
| "seconds": 11.320105165999848, |
| "chars_per_second": 464.129963719815, |
| "hours_of_audio_per_second": 0.022101426843800714, |
| "seconds_per_hour_of_audio": 45.245947561095704, |
| "peak_rss_mb": 195.71875, |
| "corrections": 115, |
| "glossary_terms": 10000, |
| "machine": { |
| "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "processor": "arm", |
| "python": "3.13.5", |
| "cpu_count": 8, |
| "cpu_brand": "Apple M2", |
| "memory_gb": 16.0 |
| }, |
| "notes": "cold_start=0.156s (index build + reader init, excluded from throughput); 1h of audio assumed to be 21000 characters" |
| } |
| ], |
| "retrieval_recall": { |
| "queries": 60.0, |
| "exhaustive_hits": 300.0, |
| "recovered": 299.0, |
| "missed": 1.0, |
| "recall": 0.9966666666666667 |
| }, |
| "elapsed_seconds": 416.9022165419992, |
| "summary": { |
| "by_condition": { |
| "A": { |
| "10": { |
| "cer": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "term_recall": 0.25262308313155774, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| }, |
| "30": { |
| "cer": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "term_recall": 0.25262308313155774, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| }, |
| "100": { |
| "cer": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "term_recall": 0.25262308313155774, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| }, |
| "300": { |
| "cer": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "term_recall": 0.25262308313155774, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| }, |
| "1000": { |
| "cer": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "term_recall": 0.25262308313155774, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| }, |
| "3000": { |
| "cer": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "term_recall": 0.25262308313155774, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| }, |
| "10000": { |
| "cer": 0.2842323651452282, |
| "wer": 0.25413082070189813, |
| "term_recall": 0.25262308313155774, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| } |
| }, |
| "B": { |
| "10": { |
| "cer": 0.28670603255665494, |
| "wer": 0.2549501570394647, |
| "term_recall": 0.2542372881355932, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| }, |
| "30": { |
| "cer": 0.2861474624960102, |
| "wer": 0.2549501570394647, |
| "term_recall": 0.2542372881355932, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| }, |
| "100": { |
| "cer": 0.28542930098946695, |
| "wer": 0.2545404888706814, |
| "term_recall": 0.25585149313962874, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| }, |
| "300": { |
| "cer": 0.28542930098946695, |
| "wer": 0.2545404888706814, |
| "term_recall": 0.25585149313962874, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| }, |
| "1000": { |
| "cer": 0.2843919565911267, |
| "wer": 0.25413082070189813, |
| "term_recall": 0.2566585956416465, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| }, |
| "3000": { |
| "cer": 0.2837535908075327, |
| "wer": 0.2535845964768538, |
| "term_recall": 0.2598870056497175, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| }, |
| "10000": { |
| "cer": 0.2796042132141717, |
| "wer": 0.2508534753516318, |
| "term_recall": 0.27199354317998387, |
| "damage_rate_chars": 9.380863039399625e-05, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.0, |
| "provenance": "simulated" |
| } |
| }, |
| "C": { |
| "10": { |
| "cer": 0.24736674114267476, |
| "wer": 0.2264099412808958, |
| "term_recall": 0.25262308313155774, |
| "damage_rate_chars": 0.00028142589118198874, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.8142857142857143, |
| "provenance": "simulated" |
| }, |
| "30": { |
| "cer": 0.247127353973827, |
| "wer": 0.22586371705585143, |
| "term_recall": 0.25585149313962874, |
| "damage_rate_chars": 0.00028142589118198874, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.8142857142857143, |
| "provenance": "simulated" |
| }, |
| "100": { |
| "cer": 0.24561123523779124, |
| "wer": 0.2243616004369794, |
| "term_recall": 0.26472962066182404, |
| "damage_rate_chars": 0.000375234521575985, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.8142857142857143, |
| "provenance": "simulated" |
| }, |
| "300": { |
| "cer": 0.244254707947654, |
| "wer": 0.22299603987436842, |
| "term_recall": 0.2711864406779661, |
| "damage_rate_chars": 0.0006566604127579737, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.8142857142857143, |
| "provenance": "simulated" |
| }, |
| "1000": { |
| "cer": 0.22917331631024576, |
| "wer": 0.20906732213573673, |
| "term_recall": 0.3365617433414044, |
| "damage_rate_chars": 0.0009380863039399625, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.8142857142857143, |
| "provenance": "simulated" |
| }, |
| "3000": { |
| "cer": 0.1881583147143313, |
| "wer": 0.17656698074559607, |
| "term_recall": 0.47538337368845845, |
| "damage_rate_chars": 0.0031894934333958724, |
| "damage_rate_terms": 0.003194888178913738, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.8571428571428571, |
| "provenance": "simulated" |
| }, |
| "10000": { |
| "cer": 0.08929141398021066, |
| "wer": 0.09436023487641677, |
| "term_recall": 0.8297013720742534, |
| "damage_rate_chars": 0.006566604127579738, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.8714285714285714, |
| "provenance": "simulated" |
| } |
| }, |
| "D": { |
| "10": { |
| "cer": 0.23723268432812, |
| "wer": 0.21371022804861395, |
| "term_recall": 0.2542372881355932, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "simulated" |
| }, |
| "30": { |
| "cer": 0.23715288860517075, |
| "wer": 0.21343711593609177, |
| "term_recall": 0.25585149313962874, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "simulated" |
| }, |
| "100": { |
| "cer": 0.236594318544526, |
| "wer": 0.21302744776730848, |
| "term_recall": 0.258272800645682, |
| "damage_rate_chars": 0.0001876172607879925, |
| "damage_rate_terms": 0.003194888178913738, |
| "damage_rate_edits": 0.2, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "simulated" |
| }, |
| "300": { |
| "cer": 0.23507819980849026, |
| "wer": 0.21152533114843644, |
| "term_recall": 0.26634382566585957, |
| "damage_rate_chars": 0.0001876172607879925, |
| "damage_rate_terms": 0.003194888178913738, |
| "damage_rate_edits": 0.1, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "simulated" |
| }, |
| "1000": { |
| "cer": 0.22334822853495054, |
| "wer": 0.20182985115389868, |
| "term_recall": 0.31476997578692495, |
| "damage_rate_chars": 0.00075046904315197, |
| "damage_rate_terms": 0.01597444089456869, |
| "damage_rate_edits": 0.1, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "simulated" |
| }, |
| "3000": { |
| "cer": 0.19222789658474307, |
| "wer": 0.18161955482725659, |
| "term_recall": 0.4100080710250202, |
| "damage_rate_chars": 0.0012195121951219512, |
| "damage_rate_terms": 0.02875399361022364, |
| "damage_rate_edits": 0.07462686567164178, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "simulated" |
| }, |
| "10000": { |
| "cer": 0.11051707628471114, |
| "wer": 0.1216714461286358, |
| "term_recall": 0.6634382566585957, |
| "damage_rate_chars": 9.380863039399625e-05, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.003067484662576687, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "simulated" |
| } |
| }, |
| "E": { |
| "10": { |
| "cer": 0.23723268432812, |
| "wer": 0.21371022804861395, |
| "term_recall": 0.2542372881355932, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "not-measured" |
| }, |
| "30": { |
| "cer": 0.23715288860517075, |
| "wer": 0.21343711593609177, |
| "term_recall": 0.25585149313962874, |
| "damage_rate_chars": 0.0, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.0, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "not-measured" |
| }, |
| "100": { |
| "cer": 0.236594318544526, |
| "wer": 0.21302744776730848, |
| "term_recall": 0.258272800645682, |
| "damage_rate_chars": 0.0001876172607879925, |
| "damage_rate_terms": 0.003194888178913738, |
| "damage_rate_edits": 0.2, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "not-measured" |
| }, |
| "300": { |
| "cer": 0.23507819980849026, |
| "wer": 0.21152533114843644, |
| "term_recall": 0.26634382566585957, |
| "damage_rate_chars": 0.0001876172607879925, |
| "damage_rate_terms": 0.003194888178913738, |
| "damage_rate_edits": 0.1, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "not-measured" |
| }, |
| "1000": { |
| "cer": 0.22334822853495054, |
| "wer": 0.20182985115389868, |
| "term_recall": 0.31476997578692495, |
| "damage_rate_chars": 0.00075046904315197, |
| "damage_rate_terms": 0.01597444089456869, |
| "damage_rate_edits": 0.1, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "not-measured" |
| }, |
| "3000": { |
| "cer": 0.19222789658474307, |
| "wer": 0.18161955482725659, |
| "term_recall": 0.4100080710250202, |
| "damage_rate_chars": 0.0012195121951219512, |
| "damage_rate_terms": 0.02875399361022364, |
| "damage_rate_edits": 0.07462686567164178, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "not-measured" |
| }, |
| "10000": { |
| "cer": 0.11051707628471114, |
| "wer": 0.1216714461286358, |
| "term_recall": 0.6634382566585957, |
| "damage_rate_chars": 9.380863039399625e-05, |
| "damage_rate_terms": 0.0, |
| "damage_rate_edits": 0.003067484662576687, |
| "hallucination_removal": 0.7428571428571429, |
| "provenance": "not-measured" |
| } |
| } |
| }, |
| "operating_point": { |
| "threshold": 0.65, |
| "correction_rate": 0.5496760259179265, |
| "damage_rate": 0.0, |
| "edit_damage_rate": 0.006097560975609756, |
| "accepted": 328.0, |
| "repairs": 326.0, |
| "damages": 2.0, |
| "neutrals": 0.0 |
| }, |
| "throughput": { |
| "label": "D@10000", |
| "n_texts": 150, |
| "chars": 5254, |
| "seconds": 11.320105165999848, |
| "chars_per_second": 464.129963719815, |
| "hours_of_audio_per_second": 0.022101426843800714, |
| "seconds_per_hour_of_audio": 45.245947561095704, |
| "peak_rss_mb": 195.71875, |
| "corrections": 115, |
| "glossary_terms": 10000, |
| "machine": { |
| "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", |
| "machine": "arm64", |
| "processor": "arm", |
| "python": "3.13.5", |
| "cpu_count": 8, |
| "cpu_brand": "Apple M2", |
| "memory_gb": 16.0 |
| }, |
| "notes": "cold_start=0.156s (index build + reader init, excluded from throughput); 1h of audio assumed to be 21000 characters" |
| } |
| } |
| } |
