{
  "complete": true,
  "device": "AMD Instinct MI355X",
  "torch": "2.14.0+rocm7.2",
  "hip": "7.2.53211",
  "same_16_inputs": true,
  "includes_prompt_tokenization_forward_softmax_CPU_probabilities": true,
  "input_ids": [
    "language-focus/ACOS/laptop/official-dev/39/201",
    "domain-retention/When2Call/3379",
    "domain-retention/When2Call/8402",
    "candidates-v3:57:2480",
    "domain-retention/VAST/8376",
    "candidates-v3:57:11139",
    "domain-retention/ANLI/f9dc5f0c-ca9b-4cdb-8554-85ee9e2f2d2a",
    "domain-retention/HoVer/b68c5781-66fc-4596-96ea-0033c56327a9",
    "domain-retention/RAGTruth/16293",
    "domain-retention/VAST/17487",
    "retrieval-arts/ESCI/1585008",
    "repair-science/public-bulk-v1/MegaScience/MegaScience/574552",
    "domain-retention/ANLI/4f4b1fc9-99af-40ea-beb3-d40ff74ab948",
    "reasoning-gym-batch2-0c14ebb1-3c4f-42e8-80dc-454f3074a36b/order1",
    "domain-retention/RAGTruth/13079",
    "chess-r4:05:003020"
  ],
  "backend": "FP4 W4A16 portable: Triton dequantization followed by BF16 GEMM; not native FP4 GEMM",
  "results": {
    "bf16": {
      "load_seconds": 57.19352705893107,
      "resident_allocated_GiB": 48.60280990600586,
      "batches": {
        "1": {
          "batch_latency_p50_ms": 54.19635958969593,
          "batch_latency_p95_ms": 71.69750705361366,
          "requests_per_second": 17.545324611772127,
          "peak_allocated_GiB": 48.818500995635986,
          "measurements": [
            {
              "seconds": 0.09753149002790451,
              "requests": 1,
              "tokens": 176
            },
            {
              "seconds": 0.05726980906911194,
              "requests": 1,
              "tokens": 201
            },
            {
              "seconds": 0.057499883929267526,
              "requests": 1,
              "tokens": 411
            },
            {
              "seconds": 0.05371670797467232,
              "requests": 1,
              "tokens": 195
            },
            {
              "seconds": 0.05342476908117533,
              "requests": 1,
              "tokens": 167
            },
            {
              "seconds": 0.05325274309143424,
              "requests": 1,
              "tokens": 157
            },
            {
              "seconds": 0.053995239082723856,
              "requests": 1,
              "tokens": 229
            },
            {
              "seconds": 0.05385898286476731,
              "requests": 1,
              "tokens": 311
            },
            {
              "seconds": 0.062414316926151514,
              "requests": 1,
              "tokens": 551
            },
            {
              "seconds": 0.05333736911416054,
              "requests": 1,
              "tokens": 241
            },
            {
              "seconds": 0.05362454196438193,
              "requests": 1,
              "tokens": 196
            },
            {
              "seconds": 0.05355773097835481,
              "requests": 1,
              "tokens": 213
            },
            {
              "seconds": 0.05317683983594179,
              "requests": 1,
              "tokens": 261
            },
            {
              "seconds": 0.05316391191445291,
              "requests": 1,
              "tokens": 301
            },
            {
              "seconds": 0.07271995604969561,
              "requests": 1,
              "tokens": 726
            },
            {
              "seconds": 0.057841310976073146,
              "requests": 1,
              "tokens": 510
            },
            {
              "seconds": 0.0531228450126946,
              "requests": 1,
              "tokens": 176
            },
            {
              "seconds": 0.05372273619286716,
              "requests": 1,
              "tokens": 201
            },
            {
              "seconds": 0.05708522698841989,
              "requests": 1,
              "tokens": 411
            },
            {
              "seconds": 0.053957157069817185,
              "requests": 1,
              "tokens": 195
            },
            {
              "seconds": 0.05323703191243112,
              "requests": 1,
              "tokens": 167
            },
            {
              "seconds": 0.05320191406644881,
              "requests": 1,
              "tokens": 157
            },
            {
              "seconds": 0.05381570407189429,
              "requests": 1,
              "tokens": 229
            },
            {
              "seconds": 0.053544011898338795,
              "requests": 1,
              "tokens": 311
            },
            {
              "seconds": 0.062205130932852626,
              "requests": 1,
              "tokens": 551
            },
            {
              "seconds": 0.053216683911159635,
              "requests": 1,
              "tokens": 241
            },
            {
              "seconds": 0.05345195601694286,
              "requests": 1,
              "tokens": 196
            },
            {
              "seconds": 0.053699644980952144,
              "requests": 1,
              "tokens": 213
            },
            {
              "seconds": 0.056666122982278466,
              "requests": 1,
              "tokens": 261
            },
            {
              "seconds": 0.05431815097108483,
              "requests": 1,
              "tokens": 301
            },
            {
              "seconds": 0.07177247991785407,
              "requests": 1,
              "tokens": 726
            },
            {
              "seconds": 0.05799283902160823,
              "requests": 1,
              "tokens": 510
            },
            {
              "seconds": 0.05421221605502069,
              "requests": 1,
              "tokens": 176
            },
            {
              "seconds": 0.054013164015486836,
              "requests": 1,
              "tokens": 201
            },
            {
              "seconds": 0.05697794910520315,
              "requests": 1,
              "tokens": 411
            },
            {
              "seconds": 0.05430118809454143,
              "requests": 1,
              "tokens": 195
            },
            {
              "seconds": 0.05409477511420846,
              "requests": 1,
              "tokens": 167
            },
            {
              "seconds": 0.05392967886291444,
              "requests": 1,
              "tokens": 157
            },
            {
              "seconds": 0.05439830990508199,
              "requests": 1,
              "tokens": 229
            },
            {
              "seconds": 0.05478060385212302,
              "requests": 1,
              "tokens": 311
            },
            {
              "seconds": 0.06227720994502306,
              "requests": 1,
              "tokens": 551
            },
            {
              "seconds": 0.054351587081328034,
              "requests": 1,
              "tokens": 241
            },
            {
              "seconds": 0.05442318506538868,
              "requests": 1,
              "tokens": 196
            },
            {
              "seconds": 0.054551840061321855,
              "requests": 1,
              "tokens": 213
            },
            {
              "seconds": 0.05429398803971708,
              "requests": 1,
              "tokens": 261
            },
            {
              "seconds": 0.05418050312437117,
              "requests": 1,
              "tokens": 301
            },
            {
              "seconds": 0.07169750705361366,
              "requests": 1,
              "tokens": 726
            },
            {
              "seconds": 0.05789261101745069,
              "requests": 1,
              "tokens": 510
            }
          ]
        },
        "8": {
          "batch_latency_p50_ms": 295.54353503044695,
          "batch_latency_p95_ms": 374.23561001196504,
          "requests_per_second": 27.134567546742975,
          "peak_allocated_GiB": 49.85915184020996,
          "measurements": [
            {
              "seconds": 0.21804877696558833,
              "requests": 8,
              "tokens": 1847
            },
            {
              "seconds": 0.37578478013165295,
              "requests": 8,
              "tokens": 2999
            },
            {
              "seconds": 0.21404898399487138,
              "requests": 8,
              "tokens": 1847
            },
            {
              "seconds": 0.37423561001196504,
              "requests": 8,
              "tokens": 2999
            },
            {
              "seconds": 0.21380486013367772,
              "requests": 8,
              "tokens": 1847
            },
            {
              "seconds": 0.37303829309530556,
              "requests": 8,
              "tokens": 2999
            }
          ]
        }
      }
    },
    "fp4": {
      "load_seconds": 37.296120763989165,
      "resident_allocated_GiB": 16.26628541946411,
      "batches": {
        "1": {
          "batch_latency_p50_ms": 66.10575248487294,
          "batch_latency_p95_ms": 76.5055047813803,
          "requests_per_second": 14.478255012045283,
          "peak_allocated_GiB": 16.507136344909668,
          "measurements": [
            {
              "seconds": 0.06496857199817896,
              "requests": 1,
              "tokens": 176
            },
            {
              "seconds": 0.06544231995940208,
              "requests": 1,
              "tokens": 201
            },
            {
              "seconds": 0.07402830617502332,
              "requests": 1,
              "tokens": 411
            },
            {
              "seconds": 0.06589341699145734,
              "requests": 1,
              "tokens": 195
            },
            {
              "seconds": 0.06557931704446673,
              "requests": 1,
              "tokens": 167
            },
            {
              "seconds": 0.06496005994267762,
              "requests": 1,
              "tokens": 157
            },
            {
              "seconds": 0.06581282708793879,
              "requests": 1,
              "tokens": 229
            },
            {
              "seconds": 0.0685572309885174,
              "requests": 1,
              "tokens": 311
            },
            {
              "seconds": 0.0765055047813803,
              "requests": 1,
              "tokens": 551
            },
            {
              "seconds": 0.06611774396151304,
              "requests": 1,
              "tokens": 241
            },
            {
              "seconds": 0.06552175804972649,
              "requests": 1,
              "tokens": 196
            },
            {
              "seconds": 0.06568572297692299,
              "requests": 1,
              "tokens": 213
            },
            {
              "seconds": 0.06540890503674746,
              "requests": 1,
              "tokens": 261
            },
            {
              "seconds": 0.06838645786046982,
              "requests": 1,
              "tokens": 301
            },
            {
              "seconds": 0.0863226349465549,
              "requests": 1,
              "tokens": 726
            },
            {
              "seconds": 0.07383130700327456,
              "requests": 1,
              "tokens": 510
            },
            {
              "seconds": 0.06545696989633143,
              "requests": 1,
              "tokens": 176
            },
            {
              "seconds": 0.06609197799116373,
              "requests": 1,
              "tokens": 201
            },
            {
              "seconds": 0.07416753610596061,
              "requests": 1,
              "tokens": 411
            },
            {
              "seconds": 0.06622499111108482,
              "requests": 1,
              "tokens": 195
            },
            {
              "seconds": 0.06530973804183304,
              "requests": 1,
              "tokens": 167
            },
            {
              "seconds": 0.06533637479878962,
              "requests": 1,
              "tokens": 157
            },
            {
              "seconds": 0.06579005508683622,
              "requests": 1,
              "tokens": 229
            },
            {
              "seconds": 0.06860739993862808,
              "requests": 1,
              "tokens": 311
            },
            {
              "seconds": 0.07632632995955646,
              "requests": 1,
              "tokens": 551
            },
            {
              "seconds": 0.0657245060428977,
              "requests": 1,
              "tokens": 241
            },
            {
              "seconds": 0.06579803698696196,
              "requests": 1,
              "tokens": 196
            },
            {
              "seconds": 0.06622335803695023,
              "requests": 1,
              "tokens": 213
            },
            {
              "seconds": 0.0655344061087817,
              "requests": 1,
              "tokens": 261
            },
            {
              "seconds": 0.06830077082850039,
              "requests": 1,
              "tokens": 301
            },
            {
              "seconds": 0.08639194094575942,
              "requests": 1,
              "tokens": 726
            },
            {
              "seconds": 0.07383402087725699,
              "requests": 1,
              "tokens": 510
            },
            {
              "seconds": 0.065410046139732,
              "requests": 1,
              "tokens": 176
            },
            {
              "seconds": 0.06643647002056241,
              "requests": 1,
              "tokens": 201
            },
            {
              "seconds": 0.07415988598950207,
              "requests": 1,
              "tokens": 411
            },
            {
              "seconds": 0.06597798294387758,
              "requests": 1,
              "tokens": 195
            },
            {
              "seconds": 0.06547786900773644,
              "requests": 1,
              "tokens": 167
            },
            {
              "seconds": 0.06554997689090669,
              "requests": 1,
              "tokens": 157
            },
            {
              "seconds": 0.06611520005390048,
              "requests": 1,
              "tokens": 229
            },
            {
              "seconds": 0.06863704114221036,
              "requests": 1,
              "tokens": 311
            },
            {
              "seconds": 0.07638701307587326,
              "requests": 1,
              "tokens": 551
            },
            {
              "seconds": 0.06636205804534256,
              "requests": 1,
              "tokens": 241
            },
            {
              "seconds": 0.0660417799372226,
              "requests": 1,
              "tokens": 196
            },
            {
              "seconds": 0.0660963049158454,
              "requests": 1,
              "tokens": 213
            },
            {
              "seconds": 0.06588856992311776,
              "requests": 1,
              "tokens": 261
            },
            {
              "seconds": 0.06834727409295738,
              "requests": 1,
              "tokens": 301
            },
            {
              "seconds": 0.08631210192106664,
              "requests": 1,
              "tokens": 726
            },
            {
              "seconds": 0.0739765849430114,
              "requests": 1,
              "tokens": 510
            }
          ]
        },
        "8": {
          "batch_latency_p50_ms": 310.49984705168754,
          "batch_latency_p95_ms": 392.74976099841297,
          "requests_per_second": 25.762332831510818,
          "peak_allocated_GiB": 17.448820114135742,
          "measurements": [
            {
              "seconds": 0.2306268650572747,
              "requests": 8,
              "tokens": 1847
            },
            {
              "seconds": 0.3929026599507779,
              "requests": 8,
              "tokens": 2999
            },
            {
              "seconds": 0.23050653981044888,
              "requests": 8,
              "tokens": 1847
            },
            {
              "seconds": 0.39274976099841297,
              "requests": 8,
              "tokens": 2999
            },
            {
              "seconds": 0.2260266519151628,
              "requests": 8,
              "tokens": 1847
            },
            {
              "seconds": 0.3903728290461004,
              "requests": 8,
              "tokens": 2999
            }
          ]
        }
      }
    }
  }
}