| { |
| "schema_version": 1, |
| "model": "nineninesix/tyger-s2s", |
| "language": "en", |
| "evaluation_type": "offline_controller_replay", |
| "evaluation_policy": { |
| "ep_thr": 0.91, |
| "ep_deb": 2, |
| "barge_thr": 0.93, |
| "barge_deb": 2, |
| "bc_thr": 0.8, |
| "bc_deb": 2, |
| "bc_refractory_frames": 15, |
| "fallback_sil_frames": 15, |
| "vad_db": -50.0, |
| "min_speech_ms": 350.0, |
| "warmup_frames": 6 |
| }, |
| "default_policy_evaluated": false, |
| "datasets": { |
| "open_yap_1k": { |
| "name": "Open Yap 1K", |
| "version": "1.0", |
| "url": "https://theagenticdatacompany.com/open-yap-1k", |
| "train_conversations": 1395, |
| "dev_conversations": 107, |
| "test_conversations": 100, |
| "split_unit": "conversation", |
| "channel_assignments_grouped": true, |
| "test_frame_windows": 23936 |
| }, |
| "full_duplex_bench": { |
| "name": "Full-Duplex-Bench", |
| "version": "1.0", |
| "url": "https://huggingface.co/datasets/Ssshangfu/Full-Duplex-Bench-Data", |
| "revision": "a3579cb9ebbe164204b196e33d4d0e91860eb90e", |
| "subsets": [ |
| "candor_turn_taking", |
| "candor_pause_handling" |
| ], |
| "scope": "Adapted endpoint replay; not the official full voice-agent benchmark or TOR." |
| }, |
| "internal": { |
| "name": "Internal evaluation", |
| "access": "non-public", |
| "scope": "Additional out-of-domain endpoint probes; source identifiers are not distributed." |
| } |
| }, |
| "open_yap_test_frames": { |
| "yield_auc": 0.9796688041466441, |
| "yield_ap": 0.6657115340151823, |
| "yield_positive_frames": 78882, |
| "yield_masked_frames": 2522031, |
| "bc_auc_freerun": 0.8975721507406702, |
| "bc_ap": 0.30040181934186144, |
| "bc_positive_frames": 31352, |
| "bc_masked_frames": 3030374, |
| "ep_auc_freerun": 0.9838886216113089, |
| "endpoint_ap": 0.8149145794136884, |
| "endpoint_positive_frames": 163373, |
| "endpoint_masked_frames": 2651747, |
| "state_macro_f1": 0.9436201457545479, |
| "state_confusion": [ |
| [ |
| 2987221, |
| 9814, |
| 17284, |
| 5521 |
| ], |
| [ |
| 7876, |
| 314958, |
| 24061, |
| 34 |
| ], |
| [ |
| 17528, |
| 27440, |
| 2464707, |
| 1748 |
| ], |
| [ |
| 6912, |
| 11, |
| 1385, |
| 71910 |
| ] |
| ], |
| "hazard_accuracy": 0.9315253565968102 |
| }, |
| "open_yap_test_events": { |
| "yield_detect": 0.2531545741324921, |
| "yield_latency_ms": 400.0, |
| "yield_false_per_h": 0.0, |
| "bc_recall": 0.14856081708449395, |
| "bc_latency_ms": -160.0, |
| "bc_acks_per_h": 135.0786866135613, |
| "yield_onsets": 1268, |
| "backchannel_onsets": 1077, |
| "clean_speech_hours": 4.722822222222222, |
| "listen_hours_approx": 6.86, |
| "conversation_prefix_seconds": 240 |
| }, |
| "endpoints": { |
| "open_yap_dev": { |
| "fallback_on": { |
| "mean_latency_s": 0.4745854700854754, |
| "median_latency_s": 0.33249999999994584, |
| "p90_latency_s": 0.9725000000001192, |
| "head_share": 0.8316666666666667, |
| "fallback_share": 0.14333333333333334, |
| "no_endpoint_share": 0.025, |
| "miss_within_1p2s": 0.058333333333333334, |
| "false_cutins_per_floor_h": 4.498953731690305, |
| "turns_with_early_cutin_share": 0.285, |
| "n_turn": 600, |
| "n_pause": 600, |
| "deadline_eligible_turns": 600, |
| "deadline_censored_turns": 0, |
| "floor_h": 57.346666666666664 |
| }, |
| "head_only": { |
| "mean_latency_s": 0.4530970149253823, |
| "median_latency_s": 0.2525000000001185, |
| "p90_latency_s": 0.8924999999999983, |
| "head_share": 0.8933333333333333, |
| "fallback_share": 0.0, |
| "no_endpoint_share": 0.10666666666666667, |
| "miss_within_1p2s": 0.16166666666666665, |
| "false_cutins_per_floor_h": 3.487561032318066, |
| "turns_with_early_cutin_share": 0.25833333333333336, |
| "n_turn": 600, |
| "n_pause": 600, |
| "deadline_eligible_turns": 600, |
| "deadline_censored_turns": 0, |
| "floor_h": 57.346666666666664 |
| } |
| }, |
| "open_yap_test": { |
| "fallback_on": { |
| "mean_latency_s": 0.46497668393782077, |
| "median_latency_s": 0.2525000000001185, |
| "p90_latency_s": 0.9725000000001908, |
| "head_share": 0.843, |
| "fallback_share": 0.122, |
| "no_endpoint_share": 0.035, |
| "miss_within_1p2s": 0.074, |
| "false_cutins_per_floor_h": 6.537794226996417, |
| "turns_with_early_cutin_share": 0.299, |
| "n_turn": 1000, |
| "n_pause": 1000, |
| "deadline_eligible_turns": 1000, |
| "deadline_censored_turns": 0, |
| "floor_h": 63.477066666666666 |
| }, |
| "head_only": { |
| "mean_latency_s": 0.4364999999999966, |
| "median_latency_s": 0.25250000000003947, |
| "p90_latency_s": 0.972499999999957, |
| "head_share": 0.9, |
| "fallback_share": 0.0, |
| "no_endpoint_share": 0.1, |
| "miss_within_1p2s": 0.16, |
| "false_cutins_per_floor_h": 5.009683287192435, |
| "turns_with_early_cutin_share": 0.275, |
| "n_turn": 1000, |
| "n_pause": 1000, |
| "deadline_eligible_turns": 1000, |
| "deadline_censored_turns": 0, |
| "floor_h": 63.477066666666666 |
| } |
| }, |
| "internal": { |
| "fallback_on": { |
| "mean_latency_s": 0.2865495867768678, |
| "median_latency_s": 0.2525000000000013, |
| "p90_latency_s": 0.4924999999999995, |
| "head_share": 0.9834710743801653, |
| "fallback_share": 0.01652892561983471, |
| "no_endpoint_share": 0.0, |
| "miss_within_1p2s": 0.0, |
| "false_cutins_per_floor_h": 28.316057268126336, |
| "turns_with_early_cutin_share": 0.12396694214876033, |
| "n_turn": 242, |
| "n_pause": 539, |
| "deadline_eligible_turns": 242, |
| "deadline_censored_turns": 0, |
| "floor_h": 6.568711111111111 |
| }, |
| "head_only": { |
| "mean_latency_s": 0.2893464730290536, |
| "median_latency_s": 0.2525000000000013, |
| "p90_latency_s": 0.4924999999999975, |
| "head_share": 0.9958677685950413, |
| "fallback_share": 0.0, |
| "no_endpoint_share": 0.004132231404958678, |
| "miss_within_1p2s": 0.012396694214876033, |
| "false_cutins_per_floor_h": 25.575793661533467, |
| "turns_with_early_cutin_share": 0.12396694214876033, |
| "n_turn": 242, |
| "n_pause": 539, |
| "deadline_eligible_turns": 242, |
| "deadline_censored_turns": 0, |
| "floor_h": 6.568711111111111 |
| } |
| }, |
| "fdb_turn_taking": { |
| "fallback_on": { |
| "mean_latency_s": 0.3830172413793251, |
| "median_latency_s": 0.3074999999999104, |
| "p90_latency_s": 0.6324999999999386, |
| "head_share": 0.9243697478991597, |
| "fallback_share": 0.05042016806722689, |
| "no_endpoint_share": 0.025210084033613446, |
| "miss_within_1p2s": 0.05042016806722689, |
| "false_cutins_per_floor_h": 0.0, |
| "turns_with_early_cutin_share": 0.25210084033613445, |
| "n_turn": 119, |
| "n_pause": 0, |
| "deadline_eligible_turns": 119, |
| "deadline_censored_turns": 0, |
| "floor_h": 0.4636100868055557 |
| }, |
| "head_only": { |
| "mean_latency_s": 0.4245175438596653, |
| "median_latency_s": 0.30249999999994737, |
| "p90_latency_s": 0.602500000000009, |
| "head_share": 0.957983193277311, |
| "fallback_share": 0.0, |
| "no_endpoint_share": 0.04201680672268908, |
| "miss_within_1p2s": 0.07563025210084033, |
| "false_cutins_per_floor_h": 0.0, |
| "turns_with_early_cutin_share": 0.24369747899159663, |
| "n_turn": 119, |
| "n_pause": 0, |
| "deadline_eligible_turns": 119, |
| "deadline_censored_turns": 0, |
| "floor_h": 0.4636100868055557 |
| } |
| }, |
| "fdb_pause_handling": { |
| "fallback_on": { |
| "mean_latency_s": null, |
| "median_latency_s": null, |
| "p90_latency_s": null, |
| "head_share": null, |
| "fallback_share": null, |
| "no_endpoint_share": null, |
| "miss_within_1p2s": null, |
| "false_cutins_per_floor_h": 173.25028170777514, |
| "turns_with_early_cutin_share": null, |
| "n_turn": 0, |
| "n_pause": 277, |
| "deadline_eligible_turns": 0, |
| "deadline_censored_turns": 0, |
| "floor_h": 0.7099555555555556 |
| }, |
| "head_only": { |
| "mean_latency_s": null, |
| "median_latency_s": null, |
| "p90_latency_s": null, |
| "head_share": null, |
| "fallback_share": null, |
| "no_endpoint_share": null, |
| "miss_within_1p2s": null, |
| "false_cutins_per_floor_h": 147.8965819456617, |
| "turns_with_early_cutin_share": null, |
| "n_turn": 0, |
| "n_pause": 277, |
| "deadline_eligible_turns": 0, |
| "deadline_censored_turns": 0, |
| "floor_h": 0.7099555555555556 |
| } |
| } |
| }, |
| "open_yap_dev_filler_proxy": { |
| "threshold": 0.91, |
| "recall": 0.6199889301810706, |
| "n_turn_ends": 12647, |
| "median_latency_ms": 80.0, |
| "false_endpoints_per_floor_h": 1004.5060622336916, |
| "filler_floor_takings": 390, |
| "user_floor_h": 57.38442222222223 |
| }, |
| "controller_compute": { |
| "device": "NVIDIA GeForce RTX 4090", |
| "batch_size": 1, |
| "prefill_frames": 64, |
| "measured_frames_per_repeat": 200, |
| "repeats": 3, |
| "parameter_count": 13991432, |
| "parameter_mib": 53.373077392578125, |
| "peak_allocated_mib": 108.75830078125, |
| "wall_ms": { |
| "p50": 0.805371964816004, |
| "p90": 0.8225860772654414 |
| }, |
| "cuda_event_ms": { |
| "p50": 0.7813119888305664, |
| "p90": 0.7976959943771362 |
| }, |
| "streaming_state_bytes_batch1": 9216, |
| "precision": "fp32", |
| "tf32": false, |
| "includes_encoder": false, |
| "includes_asr": false, |
| "includes_tts": false, |
| "includes_audio_wait": false |
| }, |
| "limitations": [ |
| "One training seed; evaluation sets informed development and are diagnostic.", |
| "State and yield frame metrics use teacher-forced previous state; endpoint and backchannel use forced LISTEN.", |
| "Event metrics use frozen development-set thresholds, not the lower deployment yield/backchannel defaults.", |
| "Endpoint latency is conditional on firing; report misses and early cut-ins alongside latency.", |
| "Full-Duplex-Bench results cover adapted v1.0 CANDOR probes only, not FDB-v3 or official full-agent scores.", |
| "Internal evaluation data are non-public, so that subset is not independently reproducible from this release.", |
| "Controller compute excludes Nemotron, ASR, audio buffering and all downstream voice-agent components.", |
| "This release does not add a new live conversation benchmark or a new ASR quality benchmark." |
| ] |
| } |
|
|
