| const NAME_RE = /^[A-Za-z0-9_.-]{1,128}$/; |
| const PLATFORMS = new Set(["onnxruntime_onnx", "tensorrt_plan"]); |
| const DTYPES = new Set(["TYPE_FP32", "TYPE_FP16", "TYPE_INT32", "TYPE_INT64", "TYPE_UINT8", "TYPE_BOOL"]); |
| const NUMBER_RE = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/i; |
| const MAX_REPORT_ROWS = 1000; |
| const HAS_OWN = (object, key) => Object.prototype.hasOwnProperty.call(object, key); |
|
|
| function validName(value, label) { |
| const name = String(value ?? "").trim(); |
| if (!NAME_RE.test(name) || name.includes("..")) { |
| throw new Error(`${label} must use letters, numbers, _, - or . (no '..').`); |
| } |
| return name; |
| } |
|
|
| function integer(value, label, minimum, maximum) { |
| const number = Number(value); |
| if (!Number.isInteger(number) || number < minimum || number > maximum) { |
| throw new Error(`${label} must be an integer between ${minimum} and ${maximum}.`); |
| } |
| return number; |
| } |
|
|
| function dimensions(value, label) { |
| const parts = String(value ?? "").replace(/[\[\]]/g, "").split(",").map((x) => x.trim()).filter(Boolean); |
| const dims = parts.map(Number); |
| if (!dims.length || dims.some((d) => !Number.isInteger(d) || d === 0 || d < -1)) { |
| throw new Error(`${label} needs one or more positive dimensions; use -1 for a variable dimension.`); |
| } |
| return dims; |
| } |
|
|
| export function generateTritonConfig(payload) { |
| const model = validName(payload.model_name || "resnet18", "Model name"); |
| const inputName = validName(payload.input_name || "input", "Input name"); |
| const outputName = validName(payload.output_name || "output", "Output name"); |
| const platform = String(payload.platform || "onnxruntime_onnx"); |
| if (!PLATFORMS.has(platform)) throw new Error("Choose ONNX Runtime or TensorRT plan as the model platform."); |
| const dtype = String(payload.data_type || "TYPE_FP32"); |
| if (!DTYPES.has(dtype)) throw new Error("Unsupported tensor data type."); |
| const inputDims = dimensions(payload.input_dims || "3,224,224", "Input dimensions"); |
| const outputDims = dimensions(payload.output_dims || "1000", "Output dimensions"); |
| const maxBatch = integer(payload.max_batch_size ?? 32, "Maximum batch size", 0, 4096); |
| const instances = integer(payload.instances ?? 1, "Instance count", 1, 64); |
| const queueDelay = integer(payload.queue_delay ?? 100, "Queue delay", 0, 100000); |
| const kind = String(payload.instance_kind || "KIND_GPU"); |
| if (!new Set(["KIND_GPU", "KIND_CPU"]).has(kind)) throw new Error("Instance kind must be KIND_GPU or KIND_CPU."); |
|
|
| const lines = [ |
| `name: "${model}"`, |
| `platform: "${platform}"`, |
| `max_batch_size: ${maxBatch}`, |
| "input [", " {", ` name: "${inputName}"`, ` data_type: ${dtype}`, |
| ` dims: [ ${inputDims.join(", ")} ]`, " }", "]", |
| "output [", " {", ` name: "${outputName}"`, ` data_type: ${dtype}`, |
| ` dims: [ ${outputDims.join(", ")} ]`, " }", "]", |
| "instance_group [", " {", ` count: ${instances}`, ` kind: ${kind}`, " }", "]", |
| ]; |
| if (maxBatch > 0) { |
| const preferred = [...new Set(String(payload.preferred_batch_sizes || "4,8,16,32").split(",").map((x) => Number(x.trim())).filter((x) => Number.isFinite(x)))].sort((a, b) => a - b); |
| if (!preferred.length || preferred.some((x) => !Number.isInteger(x) || x < 1 || x > maxBatch)) { |
| throw new Error("Preferred batch sizes must be integers between 1 and max_batch_size."); |
| } |
| lines.push("dynamic_batching {", ` preferred_batch_size: [ ${preferred.join(", ")} ]`, ` max_queue_delay_microseconds: ${queueDelay}`, "}"); |
| } |
| return `${lines.join("\n")}\n`; |
| } |
|
|
| function checkedNumber(value, label, { minimum = 0, maximum = Infinity, integerOnly = false } = {}) { |
| if (value === undefined || value === null || value === "") return null; |
| let number = value; |
| if (typeof value === "string") { |
| const trimmed = value.trim(); |
| if (!NUMBER_RE.test(trimmed)) throw new Error(`${label} must be a finite number.`); |
| number = Number(trimmed); |
| } else if (typeof value !== "number") { |
| throw new Error(`${label} must be a finite number.`); |
| } |
| if (!Number.isFinite(number)) throw new Error(`${label} must be a finite number.`); |
| if (number < minimum || number > maximum) { |
| const range = Number.isFinite(maximum) ? `between ${minimum} and ${maximum}` : `at least ${minimum}`; |
| throw new Error(`${label} must be ${range}.`); |
| } |
| if (integerOnly && !Number.isInteger(number)) throw new Error(`${label} must be an integer.`); |
| return number; |
| } |
|
|
| function isObject(value) { |
| return value !== null && typeof value === "object" && !Array.isArray(value); |
| } |
|
|
| function normalizeTimeUnit(value, unit, key, label) { |
| const number = checkedNumber(value, label, { minimum: 0 }); |
| if (number === null) return null; |
| let normalizedUnit = String(unit || "").trim().toLowerCase(); |
| if (!normalizedUnit) { |
| const name = String(key || "").toLowerCase(); |
| if (name.endsWith("_seconds") || name.endsWith("_second")) normalizedUnit = "seconds"; |
| else if (name.endsWith("_ns") || name.endsWith("_nanoseconds")) normalizedUnit = "nanoseconds"; |
| else if (name.endsWith("_us") || name.endsWith("_microseconds")) normalizedUnit = "microseconds"; |
| else normalizedUnit = "milliseconds"; |
| } |
| if (["ms", "millisecond", "milliseconds"].includes(normalizedUnit)) return number; |
| if (["s", "sec", "second", "seconds"].includes(normalizedUnit)) return number * 1000; |
| if (["ns", "nanosecond", "nanoseconds"].includes(normalizedUnit)) return number / 1e6; |
| if (["us", "µs", "microsecond", "microseconds"].includes(normalizedUnit)) return number / 1000; |
| throw new Error(`${label} uses an unsupported time unit: ${normalizedUnit}.`); |
| } |
|
|
| const STAT_KEYS = { |
| mean: ["mean", "avg", "average", "mean_ms", "avg_ms"], |
| p50: ["p50", "median", "p50_ms", "median_ms"], |
| p99: ["p99", "p99_ms", "p99_estimate", "99th_percentile"], |
| }; |
|
|
| function statFromEntry(entry, statistic) { |
| if (!isObject(entry)) return { found: false }; |
| const source = isObject(entry.stats) ? entry.stats : entry; |
| for (const key of STAT_KEYS[statistic]) { |
| if (HAS_OWN(source, key)) return { found: true, value: source[key], unit: entry.unit || source.unit }; |
| } |
| if (HAS_OWN(source, "value")) return { found: true, value: source.value, unit: entry.unit || source.unit }; |
| return { found: false }; |
| } |
|
|
| function findNumber(containers, keys, label, { statistic = null, time = false, integerOnly = false, maximum = Infinity } = {}) { |
| for (const container of containers) { |
| if (!isObject(container)) continue; |
| for (const key of keys) { |
| if (!HAS_OWN(container, key) || container[key] === null || container[key] === undefined || container[key] === "") continue; |
| const entry = container[key]; |
| let value = entry; |
| let unit = ""; |
| if (isObject(entry)) { |
| if (statistic) { |
| const nested = statFromEntry(entry, statistic); |
| if (!nested.found) continue; |
| value = nested.value; |
| unit = nested.unit || ""; |
| } else if (HAS_OWN(entry, "value")) { |
| value = entry.value; |
| unit = entry.unit || ""; |
| } else { |
| const nested = statFromEntry(entry, "mean"); |
| if (!nested.found) continue; |
| value = nested.value; |
| unit = nested.unit || ""; |
| } |
| } |
| if (time) return normalizeTimeUnit(value, unit, key, label); |
| return checkedNumber(value, label, { minimum: 0, maximum, integerOnly }); |
| } |
| } |
| return null; |
| } |
|
|
| function metricGroup(containers, definition, label) { |
| return { |
| mean_ms: findNumber(containers, definition.mean, `${label} mean`, { statistic: "mean", time: true }), |
| p50_ms: findNumber(containers, definition.p50, `${label} p50`, { statistic: "p50", time: true }), |
| p99_ms: findNumber(containers, definition.p99, `${label} p99`, { statistic: "p99", time: true }), |
| }; |
| } |
|
|
| const LLM_GROUPS = { |
| ttft: { |
| label: "TTFT", |
| mean: ["mean_ttft_ms", "avg_ttft_ms", "ttft_mean_ms", "ttft_avg_ms", "time_to_first_token_mean_ms", "time_to_first_token_avg_ms", "ttft_ms", "time_to_first_token_ms", "time_to_first_token"], |
| p50: ["median_ttft_ms", "p50_ttft_ms", "ttft_median_ms", "ttft_p50_ms", "time_to_first_token_median_ms", "time_to_first_token_p50_ms", "time_to_first_token"], |
| p99: ["p99_ttft_ms", "ttft_p99_ms", "time_to_first_token_p99_ms", "time_to_first_token"], |
| }, |
| tpot: { |
| label: "TPOT", |
| mean: ["mean_tpot_ms", "avg_tpot_ms", "tpot_mean_ms", "tpot_avg_ms", "time_per_output_token_mean_ms", "tpot_ms", "time_per_output_token_ms", "time_per_output_token", "tpot"], |
| p50: ["median_tpot_ms", "p50_tpot_ms", "tpot_median_ms", "tpot_p50_ms", "time_per_output_token_p50_ms", "time_per_output_token", "tpot"], |
| p99: ["p99_tpot_ms", "tpot_p99_ms", "time_per_output_token_p99_ms", "time_per_output_token", "tpot"], |
| }, |
| itl: { |
| label: "ITL", |
| mean: ["mean_itl_ms", "avg_itl_ms", "itl_mean_ms", "itl_avg_ms", "inter_token_latency_mean_ms", "itl_ms", "inter_token_latency_ms", "itl", "inter_token_latency"], |
| p50: ["median_itl_ms", "p50_itl_ms", "itl_median_ms", "itl_p50_ms", "inter_token_latency_p50_ms", "itl", "inter_token_latency"], |
| p99: ["p99_itl_ms", "itl_p99_ms", "inter_token_latency_p99_ms", "itl", "inter_token_latency"], |
| }, |
| request_latency: { |
| label: "Request latency", |
| mean: ["mean_request_latency_ms", "avg_request_latency_ms", "request_latency_mean_ms", "mean_e2el_ms", "mean_e2e_latency_ms", "request_latency_ms", "e2el_ms", "request_latency", "e2el"], |
| p50: ["median_request_latency_ms", "p50_request_latency_ms", "request_latency_p50_ms", "median_e2el_ms", "p50_e2el_ms", "p50_e2e_latency_ms", "request_latency", "e2el"], |
| p99: ["p99_request_latency_ms", "request_latency_p99_ms", "p99_e2el_ms", "p99_e2e_latency_ms", "request_latency", "e2el"], |
| }, |
| }; |
|
|
| function getLlmMetrics(containers) { |
| const llm = {}; |
| for (const [key, definition] of Object.entries(LLM_GROUPS)) { |
| llm[key] = metricGroup(containers, definition, definition.label); |
| validatePercentiles(llm[key], definition.label); |
| } |
| llm.request_throughput_rps = findNumber(containers, |
| ["request_throughput", "request_throughput_rps", "requests_per_second", "req_per_sec"], |
| "Request throughput"); |
| llm.output_token_throughput_tps = findNumber(containers, |
| ["output_token_throughput", "output_token_throughput_tps", "output_throughput", "output_tokens_per_second", "tokens_per_second"], |
| "Output token throughput"); |
| llm.total_input_tokens = findNumber(containers, |
| ["total_input_tokens", "total_prompt_tokens", "input_tokens", "prompt_tokens", "num_input_tokens"], |
| "Total input tokens", { integerOnly: true }); |
| llm.total_output_tokens = findNumber(containers, |
| ["total_output_tokens", "total_generated_tokens", "generated_tokens", "output_tokens", "completion_tokens", "num_output_tokens"], |
| "Total output tokens", { integerOnly: true }); |
| return llm; |
| } |
|
|
| function hasLlmData(llm) { |
| for (const [key, value] of Object.entries(llm)) { |
| if (key === "total_input_tokens" || key === "total_output_tokens" || key.endsWith("_rps") || key.endsWith("_tps")) { |
| if (value !== null) return true; |
| } else if (isObject(value) && Object.values(value).some((number) => number !== null)) { |
| return true; |
| } |
| } |
| return false; |
| } |
|
|
| function validatePercentiles(group, label) { |
| if (group.p50_ms !== null && group.p99_ms !== null && group.p99_ms < group.p50_ms) { |
| throw new Error(`${label} p99 must be greater than or equal to p50.`); |
| } |
| } |
|
|
| function normalizeRow(result, payload, flat = false) { |
| const containers = [result.metrics, result.latency, result.stats, result, payload.metrics, payload.stats, payload]; |
| const llm = getLlmMetrics(containers); |
| const legacyLatency = [result.latency, result.metrics, result, payload.metrics, payload]; |
| const p50 = findNumber(legacyLatency, ["p50_ms", "median_latency_ms", "latency_p50_ms"], "p50 latency", { time: true }); |
| const p99 = findNumber(legacyLatency, ["p99_ms", "p99_latency_ms", "latency_p99_ms"], "p99 latency", { time: true }); |
| validatePercentiles({ p50_ms: p50, p99_ms: p99 }, "Latency"); |
| const throughput = findNumber(legacyLatency, ["throughput_ips", "inferences_per_second", "inference_throughput_ips"], "Inference throughput"); |
| const measured = [p50, p99, throughput].some((value) => value !== null) || hasLlmData(llm); |
|
|
| let available; |
| if (HAS_OWN(result, "available")) { |
| if (typeof result.available !== "boolean") throw new Error("Backend availability must be a boolean."); |
| available = result.available; |
| } else if (HAS_OWN(result, "success")) { |
| if (typeof result.success !== "boolean") throw new Error("Backend success status must be a boolean."); |
| available = result.success; |
| } else { |
| available = measured; |
| } |
|
|
| const backendValue = result.backend || result.engine || result.framework || result.runtime || payload.backend || payload.engine || payload.framework; |
| const backend = String(backendValue || (flat ? "LLM benchmark" : "unknown")).slice(0, 120); |
| const reasonValue = result.reason || result.error || result.message || ""; |
| return { |
| backend, |
| available, |
| reason: String(reasonValue).slice(0, 500), |
| p50_ms: p50, |
| p99_ms: p99, |
| throughput_ips: throughput, |
| llm, |
| measured, |
| }; |
| } |
|
|
| export function summarizeReport(payload) { |
| if (!isObject(payload)) throw new Error("The report must be a JSON object."); |
| const hasResultsArray = Array.isArray(payload.results); |
| if (hasResultsArray && payload.results.length > MAX_REPORT_ROWS) { |
| throw new Error(`This report has more than ${MAX_REPORT_ROWS.toLocaleString()} result rows. Load an aggregate benchmark summary instead of per-request details.`); |
| } |
|
|
| let rows; |
| let format; |
| if (hasResultsArray && payload.results.some(isObject)) { |
| rows = payload.results.filter(isObject).map((result) => normalizeRow(result, payload)); |
| format = "results-array"; |
| } else { |
| const flatRow = normalizeRow(payload, payload, true); |
| if (!flatRow.measured) { |
| throw new Error("Unsupported report. Provide an inferbench 'results' array or a flat LLM summary with recognized metrics."); |
| } |
| rows = [flatRow]; |
| format = "flat-llm-summary"; |
| } |
|
|
| if (!rows.length) throw new Error("The report contains no valid result rows."); |
| const accuracySource = isObject(payload.accuracy) ? Object.values(payload.accuracy) : []; |
| if (accuracySource.length > MAX_REPORT_ROWS) throw new Error(`This report has more than ${MAX_REPORT_ROWS.toLocaleString()} accuracy rows.`); |
| const accuracyRows = accuracySource.filter(isObject).map((row) => ({ |
| backend: String(row.backend || "unknown").slice(0, 120), |
| reference: String(row.reference || payload.reference || "unknown").slice(0, 120), |
| samples: checkedNumber(row.samples, "Accuracy samples", { minimum: 0, integerOnly: true }), |
| max_abs_diff: checkedNumber(row.max_abs_diff, "Maximum absolute difference", { minimum: 0 }), |
| mean_abs_diff: checkedNumber(row.mean_abs_diff, "Mean absolute difference", { minimum: 0 }), |
| top1_agreement: checkedNumber(row.top1_agreement, "Top-1 agreement", { minimum: 0, maximum: 1 }), |
| })); |
|
|
| const modelValue = payload.model || payload.model_name || payload.model_id || payload.served_model_name || "unknown"; |
| const metadataNumber = (key, minimum = 0) => checkedNumber(payload[key], key, { minimum, integerOnly: key !== "batch" }); |
| const host = isObject(payload.host) ? payload.host |
| : isObject(payload.hardware) ? payload.hardware |
| : isObject(payload.system) ? payload.system |
| : typeof payload.host === "string" ? { platform: payload.host } : {}; |
| return { |
| format, |
| schemaDemo: payload.schema_demo === true, |
| model: String(modelValue).slice(0, 120), |
| batch: metadataNumber("batch"), |
| iterations: metadataNumber("iterations"), |
| warmup: metadataNumber("warmup"), |
| reference: String(payload.reference || "unknown").slice(0, 120), |
| host, |
| rows, |
| accuracyRows, |
| hasLlmMetrics: rows.some((row) => hasLlmData(row.llm)), |
| source: "Analyzed locally in your browser. This Space did not run the benchmark or upload your report.", |
| }; |
| } |
|
|
