| { |
| "complete": true, |
| "device": "AMD Instinct MI355X", |
| "torch": "2.14.0+rocm7.2", |
| "hip": "7.2.53211", |
| "same_16_inputs": true, |
| "includes_prompt_tokenization_forward_softmax_CPU_probabilities": true, |
| "input_ids": [ |
| "language-focus/ACOS/laptop/official-dev/39/201", |
| "domain-retention/When2Call/3379", |
| "domain-retention/When2Call/8402", |
| "candidates-v3:57:2480", |
| "domain-retention/VAST/8376", |
| "candidates-v3:57:11139", |
| "domain-retention/ANLI/f9dc5f0c-ca9b-4cdb-8554-85ee9e2f2d2a", |
| "domain-retention/HoVer/b68c5781-66fc-4596-96ea-0033c56327a9", |
| "domain-retention/RAGTruth/16293", |
| "domain-retention/VAST/17487", |
| "retrieval-arts/ESCI/1585008", |
| "repair-science/public-bulk-v1/MegaScience/MegaScience/574552", |
| "domain-retention/ANLI/4f4b1fc9-99af-40ea-beb3-d40ff74ab948", |
| "reasoning-gym-batch2-0c14ebb1-3c4f-42e8-80dc-454f3074a36b/order1", |
| "domain-retention/RAGTruth/13079", |
| "chess-r4:05:003020" |
| ], |
| "backend": "FP4 W4A16 portable: Triton dequantization followed by BF16 GEMM; not native FP4 GEMM", |
| "results": { |
| "bf16": { |
| "load_seconds": 57.19352705893107, |
| "resident_allocated_GiB": 48.60280990600586, |
| "batches": { |
| "1": { |
| "batch_latency_p50_ms": 54.19635958969593, |
| "batch_latency_p95_ms": 71.69750705361366, |
| "requests_per_second": 17.545324611772127, |
| "peak_allocated_GiB": 48.818500995635986, |
| "measurements": [ |
| { |
| "seconds": 0.09753149002790451, |
| "requests": 1, |
| "tokens": 176 |
| }, |
| { |
| "seconds": 0.05726980906911194, |
| "requests": 1, |
| "tokens": 201 |
| }, |
| { |
| "seconds": 0.057499883929267526, |
| "requests": 1, |
| "tokens": 411 |
| }, |
| { |
| "seconds": 0.05371670797467232, |
| "requests": 1, |
| "tokens": 195 |
| }, |
| { |
| "seconds": 0.05342476908117533, |
| "requests": 1, |
| "tokens": 167 |
| }, |
| { |
| "seconds": 0.05325274309143424, |
| "requests": 1, |
| "tokens": 157 |
| }, |
| { |
| "seconds": 0.053995239082723856, |
| "requests": 1, |
| "tokens": 229 |
| }, |
| { |
| "seconds": 0.05385898286476731, |
| "requests": 1, |
| "tokens": 311 |
| }, |
| { |
| "seconds": 0.062414316926151514, |
| "requests": 1, |
| "tokens": 551 |
| }, |
| { |
| "seconds": 0.05333736911416054, |
| "requests": 1, |
| "tokens": 241 |
| }, |
| { |
| "seconds": 0.05362454196438193, |
| "requests": 1, |
| "tokens": 196 |
| }, |
| { |
| "seconds": 0.05355773097835481, |
| "requests": 1, |
| "tokens": 213 |
| }, |
| { |
| "seconds": 0.05317683983594179, |
| "requests": 1, |
| "tokens": 261 |
| }, |
| { |
| "seconds": 0.05316391191445291, |
| "requests": 1, |
| "tokens": 301 |
| }, |
| { |
| "seconds": 0.07271995604969561, |
| "requests": 1, |
| "tokens": 726 |
| }, |
| { |
| "seconds": 0.057841310976073146, |
| "requests": 1, |
| "tokens": 510 |
| }, |
| { |
| "seconds": 0.0531228450126946, |
| "requests": 1, |
| "tokens": 176 |
| }, |
| { |
| "seconds": 0.05372273619286716, |
| "requests": 1, |
| "tokens": 201 |
| }, |
| { |
| "seconds": 0.05708522698841989, |
| "requests": 1, |
| "tokens": 411 |
| }, |
| { |
| "seconds": 0.053957157069817185, |
| "requests": 1, |
| "tokens": 195 |
| }, |
| { |
| "seconds": 0.05323703191243112, |
| "requests": 1, |
| "tokens": 167 |
| }, |
| { |
| "seconds": 0.05320191406644881, |
| "requests": 1, |
| "tokens": 157 |
| }, |
| { |
| "seconds": 0.05381570407189429, |
| "requests": 1, |
| "tokens": 229 |
| }, |
| { |
| "seconds": 0.053544011898338795, |
| "requests": 1, |
| "tokens": 311 |
| }, |
| { |
| "seconds": 0.062205130932852626, |
| "requests": 1, |
| "tokens": 551 |
| }, |
| { |
| "seconds": 0.053216683911159635, |
| "requests": 1, |
| "tokens": 241 |
| }, |
| { |
| "seconds": 0.05345195601694286, |
| "requests": 1, |
| "tokens": 196 |
| }, |
| { |
| "seconds": 0.053699644980952144, |
| "requests": 1, |
| "tokens": 213 |
| }, |
| { |
| "seconds": 0.056666122982278466, |
| "requests": 1, |
| "tokens": 261 |
| }, |
| { |
| "seconds": 0.05431815097108483, |
| "requests": 1, |
| "tokens": 301 |
| }, |
| { |
| "seconds": 0.07177247991785407, |
| "requests": 1, |
| "tokens": 726 |
| }, |
| { |
| "seconds": 0.05799283902160823, |
| "requests": 1, |
| "tokens": 510 |
| }, |
| { |
| "seconds": 0.05421221605502069, |
| "requests": 1, |
| "tokens": 176 |
| }, |
| { |
| "seconds": 0.054013164015486836, |
| "requests": 1, |
| "tokens": 201 |
| }, |
| { |
| "seconds": 0.05697794910520315, |
| "requests": 1, |
| "tokens": 411 |
| }, |
| { |
| "seconds": 0.05430118809454143, |
| "requests": 1, |
| "tokens": 195 |
| }, |
| { |
| "seconds": 0.05409477511420846, |
| "requests": 1, |
| "tokens": 167 |
| }, |
| { |
| "seconds": 0.05392967886291444, |
| "requests": 1, |
| "tokens": 157 |
| }, |
| { |
| "seconds": 0.05439830990508199, |
| "requests": 1, |
| "tokens": 229 |
| }, |
| { |
| "seconds": 0.05478060385212302, |
| "requests": 1, |
| "tokens": 311 |
| }, |
| { |
| "seconds": 0.06227720994502306, |
| "requests": 1, |
| "tokens": 551 |
| }, |
| { |
| "seconds": 0.054351587081328034, |
| "requests": 1, |
| "tokens": 241 |
| }, |
| { |
| "seconds": 0.05442318506538868, |
| "requests": 1, |
| "tokens": 196 |
| }, |
| { |
| "seconds": 0.054551840061321855, |
| "requests": 1, |
| "tokens": 213 |
| }, |
| { |
| "seconds": 0.05429398803971708, |
| "requests": 1, |
| "tokens": 261 |
| }, |
| { |
| "seconds": 0.05418050312437117, |
| "requests": 1, |
| "tokens": 301 |
| }, |
| { |
| "seconds": 0.07169750705361366, |
| "requests": 1, |
| "tokens": 726 |
| }, |
| { |
| "seconds": 0.05789261101745069, |
| "requests": 1, |
| "tokens": 510 |
| } |
| ] |
| }, |
| "8": { |
| "batch_latency_p50_ms": 295.54353503044695, |
| "batch_latency_p95_ms": 374.23561001196504, |
| "requests_per_second": 27.134567546742975, |
| "peak_allocated_GiB": 49.85915184020996, |
| "measurements": [ |
| { |
| "seconds": 0.21804877696558833, |
| "requests": 8, |
| "tokens": 1847 |
| }, |
| { |
| "seconds": 0.37578478013165295, |
| "requests": 8, |
| "tokens": 2999 |
| }, |
| { |
| "seconds": 0.21404898399487138, |
| "requests": 8, |
| "tokens": 1847 |
| }, |
| { |
| "seconds": 0.37423561001196504, |
| "requests": 8, |
| "tokens": 2999 |
| }, |
| { |
| "seconds": 0.21380486013367772, |
| "requests": 8, |
| "tokens": 1847 |
| }, |
| { |
| "seconds": 0.37303829309530556, |
| "requests": 8, |
| "tokens": 2999 |
| } |
| ] |
| } |
| } |
| }, |
| "fp4": { |
| "load_seconds": 37.296120763989165, |
| "resident_allocated_GiB": 16.26628541946411, |
| "batches": { |
| "1": { |
| "batch_latency_p50_ms": 66.10575248487294, |
| "batch_latency_p95_ms": 76.5055047813803, |
| "requests_per_second": 14.478255012045283, |
| "peak_allocated_GiB": 16.507136344909668, |
| "measurements": [ |
| { |
| "seconds": 0.06496857199817896, |
| "requests": 1, |
| "tokens": 176 |
| }, |
| { |
| "seconds": 0.06544231995940208, |
| "requests": 1, |
| "tokens": 201 |
| }, |
| { |
| "seconds": 0.07402830617502332, |
| "requests": 1, |
| "tokens": 411 |
| }, |
| { |
| "seconds": 0.06589341699145734, |
| "requests": 1, |
| "tokens": 195 |
| }, |
| { |
| "seconds": 0.06557931704446673, |
| "requests": 1, |
| "tokens": 167 |
| }, |
| { |
| "seconds": 0.06496005994267762, |
| "requests": 1, |
| "tokens": 157 |
| }, |
| { |
| "seconds": 0.06581282708793879, |
| "requests": 1, |
| "tokens": 229 |
| }, |
| { |
| "seconds": 0.0685572309885174, |
| "requests": 1, |
| "tokens": 311 |
| }, |
| { |
| "seconds": 0.0765055047813803, |
| "requests": 1, |
| "tokens": 551 |
| }, |
| { |
| "seconds": 0.06611774396151304, |
| "requests": 1, |
| "tokens": 241 |
| }, |
| { |
| "seconds": 0.06552175804972649, |
| "requests": 1, |
| "tokens": 196 |
| }, |
| { |
| "seconds": 0.06568572297692299, |
| "requests": 1, |
| "tokens": 213 |
| }, |
| { |
| "seconds": 0.06540890503674746, |
| "requests": 1, |
| "tokens": 261 |
| }, |
| { |
| "seconds": 0.06838645786046982, |
| "requests": 1, |
| "tokens": 301 |
| }, |
| { |
| "seconds": 0.0863226349465549, |
| "requests": 1, |
| "tokens": 726 |
| }, |
| { |
| "seconds": 0.07383130700327456, |
| "requests": 1, |
| "tokens": 510 |
| }, |
| { |
| "seconds": 0.06545696989633143, |
| "requests": 1, |
| "tokens": 176 |
| }, |
| { |
| "seconds": 0.06609197799116373, |
| "requests": 1, |
| "tokens": 201 |
| }, |
| { |
| "seconds": 0.07416753610596061, |
| "requests": 1, |
| "tokens": 411 |
| }, |
| { |
| "seconds": 0.06622499111108482, |
| "requests": 1, |
| "tokens": 195 |
| }, |
| { |
| "seconds": 0.06530973804183304, |
| "requests": 1, |
| "tokens": 167 |
| }, |
| { |
| "seconds": 0.06533637479878962, |
| "requests": 1, |
| "tokens": 157 |
| }, |
| { |
| "seconds": 0.06579005508683622, |
| "requests": 1, |
| "tokens": 229 |
| }, |
| { |
| "seconds": 0.06860739993862808, |
| "requests": 1, |
| "tokens": 311 |
| }, |
| { |
| "seconds": 0.07632632995955646, |
| "requests": 1, |
| "tokens": 551 |
| }, |
| { |
| "seconds": 0.0657245060428977, |
| "requests": 1, |
| "tokens": 241 |
| }, |
| { |
| "seconds": 0.06579803698696196, |
| "requests": 1, |
| "tokens": 196 |
| }, |
| { |
| "seconds": 0.06622335803695023, |
| "requests": 1, |
| "tokens": 213 |
| }, |
| { |
| "seconds": 0.0655344061087817, |
| "requests": 1, |
| "tokens": 261 |
| }, |
| { |
| "seconds": 0.06830077082850039, |
| "requests": 1, |
| "tokens": 301 |
| }, |
| { |
| "seconds": 0.08639194094575942, |
| "requests": 1, |
| "tokens": 726 |
| }, |
| { |
| "seconds": 0.07383402087725699, |
| "requests": 1, |
| "tokens": 510 |
| }, |
| { |
| "seconds": 0.065410046139732, |
| "requests": 1, |
| "tokens": 176 |
| }, |
| { |
| "seconds": 0.06643647002056241, |
| "requests": 1, |
| "tokens": 201 |
| }, |
| { |
| "seconds": 0.07415988598950207, |
| "requests": 1, |
| "tokens": 411 |
| }, |
| { |
| "seconds": 0.06597798294387758, |
| "requests": 1, |
| "tokens": 195 |
| }, |
| { |
| "seconds": 0.06547786900773644, |
| "requests": 1, |
| "tokens": 167 |
| }, |
| { |
| "seconds": 0.06554997689090669, |
| "requests": 1, |
| "tokens": 157 |
| }, |
| { |
| "seconds": 0.06611520005390048, |
| "requests": 1, |
| "tokens": 229 |
| }, |
| { |
| "seconds": 0.06863704114221036, |
| "requests": 1, |
| "tokens": 311 |
| }, |
| { |
| "seconds": 0.07638701307587326, |
| "requests": 1, |
| "tokens": 551 |
| }, |
| { |
| "seconds": 0.06636205804534256, |
| "requests": 1, |
| "tokens": 241 |
| }, |
| { |
| "seconds": 0.0660417799372226, |
| "requests": 1, |
| "tokens": 196 |
| }, |
| { |
| "seconds": 0.0660963049158454, |
| "requests": 1, |
| "tokens": 213 |
| }, |
| { |
| "seconds": 0.06588856992311776, |
| "requests": 1, |
| "tokens": 261 |
| }, |
| { |
| "seconds": 0.06834727409295738, |
| "requests": 1, |
| "tokens": 301 |
| }, |
| { |
| "seconds": 0.08631210192106664, |
| "requests": 1, |
| "tokens": 726 |
| }, |
| { |
| "seconds": 0.0739765849430114, |
| "requests": 1, |
| "tokens": 510 |
| } |
| ] |
| }, |
| "8": { |
| "batch_latency_p50_ms": 310.49984705168754, |
| "batch_latency_p95_ms": 392.74976099841297, |
| "requests_per_second": 25.762332831510818, |
| "peak_allocated_GiB": 17.448820114135742, |
| "measurements": [ |
| { |
| "seconds": 0.2306268650572747, |
| "requests": 8, |
| "tokens": 1847 |
| }, |
| { |
| "seconds": 0.3929026599507779, |
| "requests": 8, |
| "tokens": 2999 |
| }, |
| { |
| "seconds": 0.23050653981044888, |
| "requests": 8, |
| "tokens": 1847 |
| }, |
| { |
| "seconds": 0.39274976099841297, |
| "requests": 8, |
| "tokens": 2999 |
| }, |
| { |
| "seconds": 0.2260266519151628, |
| "requests": 8, |
| "tokens": 1847 |
| }, |
| { |
| "seconds": 0.3903728290461004, |
| "requests": 8, |
| "tokens": 2999 |
| } |
| ] |
| } |
| } |
| } |
| } |
| } |
|
|
