| { |
| "device": "NVIDIA A100-SXM4-40GB", |
| "dtype": "bfloat16", |
| "attn": "chunked_eager", |
| "max_tokens": 49152, |
| "note": "harness --stage1 union refuses in bf16 (pointwise selftest |route-solo|>tol); comparison uses --stage1 retriever. No CLI flag skips/relaxes the pointwise selftest (--selftest sets record COUNT, min 1). picker argmax == harness(retriever) on all 5 rows for BOTH bundles.", |
| "bundles": { |
| "utod-picker-12b-v1": { |
| "temps": { |
| "T1": 1.7237, |
| "T2": 1.4778, |
| "T1_ret": 0.05 |
| }, |
| "load_test": [ |
| { |
| "id": "gpqa-94", |
| "N": 4, |
| "picker_union_argmax": "canthaxanthin_biosynthesis", |
| "harness_retriever_pred": "canthaxanthin_biosynthesis", |
| "match": true, |
| "sum_p": 1.0 |
| }, |
| { |
| "id": "glaive-98691", |
| "N": 2, |
| "picker_union_argmax": "yes", |
| "harness_retriever_pred": "yes", |
| "match": true, |
| "sum_p": 1.0 |
| }, |
| { |
| "id": "tabfact-10626", |
| "N": 2, |
| "picker_union_argmax": "yes", |
| "harness_retriever_pred": "yes", |
| "match": true, |
| "sum_p": 1.0 |
| }, |
| { |
| "id": "banking77-train-17", |
| "N": 50, |
| "picker_union_argmax": "card_delivery_estimate", |
| "harness_retriever_pred": "card_delivery_estimate", |
| "match": true, |
| "sum_p": 1.0 |
| }, |
| { |
| "id": "banking77-train-23", |
| "N": 50, |
| "picker_union_argmax": "card_arrival", |
| "harness_retriever_pred": "card_arrival", |
| "match": true, |
| "sum_p": 1.0 |
| } |
| ], |
| "latency_small_bf16": { |
| "n": 50, |
| "stage1": "union", |
| "p50_ms": 541.8913, |
| "p95_ms": 3846.8851, |
| "mean_ms": 968.4763, |
| "mean_prompt_tokens": 942.16 |
| }, |
| "latency_highN_bf16": { |
| "n": 20, |
| "stage1": "retriever", |
| "p50_ms": 182.12849099654704, |
| "p95_ms": 199.31893999455497, |
| "mean_ms": 183.2959999483137, |
| "mean_prompt_tokens": 315.95 |
| } |
| }, |
| "utod-picker-12b-v1-armC": { |
| "temps": { |
| "T1": 5.4679, |
| "T2": 1.267, |
| "T1_ret": 0.05 |
| }, |
| "load_test": [ |
| { |
| "id": "gpqa-94", |
| "N": 4, |
| "picker_union_argmax": "canthaxanthin_biosynthesis", |
| "harness_retriever_pred": "canthaxanthin_biosynthesis", |
| "match": true, |
| "sum_p": 1.0 |
| }, |
| { |
| "id": "glaive-98691", |
| "N": 2, |
| "picker_union_argmax": "yes", |
| "harness_retriever_pred": "yes", |
| "match": true, |
| "sum_p": 1.0 |
| }, |
| { |
| "id": "tabfact-10626", |
| "N": 2, |
| "picker_union_argmax": "yes", |
| "harness_retriever_pred": "yes", |
| "match": true, |
| "sum_p": 1.0 |
| }, |
| { |
| "id": "banking77-train-17", |
| "N": 50, |
| "picker_union_argmax": "card_delivery_estimate", |
| "harness_retriever_pred": "card_delivery_estimate", |
| "match": true, |
| "sum_p": 1.0 |
| }, |
| { |
| "id": "banking77-train-23", |
| "N": 50, |
| "picker_union_argmax": "card_arrival", |
| "harness_retriever_pred": "card_arrival", |
| "match": true, |
| "sum_p": 1.0 |
| } |
| ], |
| "latency_small_bf16": { |
| "n": 50, |
| "stage1": "union", |
| "p50_ms": 543.0618170066737, |
| "p95_ms": 3847.9274609999266, |
| "mean_ms": 970.9316438387032, |
| "mean_prompt_tokens": 942.16 |
| }, |
| "latency_highN_bf16": { |
| "n": 20, |
| "stage1": "retriever", |
| "p50_ms": 181.16832099622115, |
| "p95_ms": 198.3372819959186, |
| "mean_ms": 182.499660000758, |
| "mean_prompt_tokens": 315.95 |
| }, |
| "latency_small_8bit": { |
| "n": 50, |
| "p50_ms": 537.7577670005849, |
| "p95_ms": 2232.4729199899593, |
| "mean_ms": 758.5707962393644, |
| "mean_prompt_tokens": 942.16 |
| }, |
| "eightbit_vs_bf16": { |
| "argmax_agreement": 0.92, |
| "mean_abs_dtopp": 0.028711071880883027, |
| "n": 50 |
| }, |
| "eightbit_note": "8-bit attn=eager (chunked_eager compute-identical for short prompts); stage1=retriever letter-read path" |
| } |
| } |
| } |
