const NAME_RE = /^[A-Za-z0-9_.-]{1,128}$/; const PLATFORMS = new Set(["onnxruntime_onnx", "tensorrt_plan"]); const DTYPES = new Set(["TYPE_FP32", "TYPE_FP16", "TYPE_INT32", "TYPE_INT64", "TYPE_UINT8", "TYPE_BOOL"]); const NUMBER_RE = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/i; const MAX_REPORT_ROWS = 1000; const HAS_OWN = (object, key) => Object.prototype.hasOwnProperty.call(object, key); function validName(value, label) { const name = String(value ?? "").trim(); if (!NAME_RE.test(name) || name.includes("..")) { throw new Error(`${label} must use letters, numbers, _, - or . (no '..').`); } return name; } function integer(value, label, minimum, maximum) { const number = Number(value); if (!Number.isInteger(number) || number < minimum || number > maximum) { throw new Error(`${label} must be an integer between ${minimum} and ${maximum}.`); } return number; } function dimensions(value, label) { const parts = String(value ?? "").replace(/[\[\]]/g, "").split(",").map((x) => x.trim()).filter(Boolean); const dims = parts.map(Number); if (!dims.length || dims.some((d) => !Number.isInteger(d) || d === 0 || d < -1)) { throw new Error(`${label} needs one or more positive dimensions; use -1 for a variable dimension.`); } return dims; } export function generateTritonConfig(payload) { const model = validName(payload.model_name || "resnet18", "Model name"); const inputName = validName(payload.input_name || "input", "Input name"); const outputName = validName(payload.output_name || "output", "Output name"); const platform = String(payload.platform || "onnxruntime_onnx"); if (!PLATFORMS.has(platform)) throw new Error("Choose ONNX Runtime or TensorRT plan as the model platform."); const dtype = String(payload.data_type || "TYPE_FP32"); if (!DTYPES.has(dtype)) throw new Error("Unsupported tensor data type."); const inputDims = dimensions(payload.input_dims || "3,224,224", "Input dimensions"); const outputDims = dimensions(payload.output_dims || "1000", "Output dimensions"); const maxBatch = integer(payload.max_batch_size ?? 32, "Maximum batch size", 0, 4096); const instances = integer(payload.instances ?? 1, "Instance count", 1, 64); const queueDelay = integer(payload.queue_delay ?? 100, "Queue delay", 0, 100000); const kind = String(payload.instance_kind || "KIND_GPU"); if (!new Set(["KIND_GPU", "KIND_CPU"]).has(kind)) throw new Error("Instance kind must be KIND_GPU or KIND_CPU."); const lines = [ `name: "${model}"`, `platform: "${platform}"`, `max_batch_size: ${maxBatch}`, "input [", " {", ` name: "${inputName}"`, ` data_type: ${dtype}`, ` dims: [ ${inputDims.join(", ")} ]`, " }", "]", "output [", " {", ` name: "${outputName}"`, ` data_type: ${dtype}`, ` dims: [ ${outputDims.join(", ")} ]`, " }", "]", "instance_group [", " {", ` count: ${instances}`, ` kind: ${kind}`, " }", "]", ]; if (maxBatch > 0) { const preferred = [...new Set(String(payload.preferred_batch_sizes || "4,8,16,32").split(",").map((x) => Number(x.trim())).filter((x) => Number.isFinite(x)))].sort((a, b) => a - b); if (!preferred.length || preferred.some((x) => !Number.isInteger(x) || x < 1 || x > maxBatch)) { throw new Error("Preferred batch sizes must be integers between 1 and max_batch_size."); } lines.push("dynamic_batching {", ` preferred_batch_size: [ ${preferred.join(", ")} ]`, ` max_queue_delay_microseconds: ${queueDelay}`, "}"); } return `${lines.join("\n")}\n`; } function checkedNumber(value, label, { minimum = 0, maximum = Infinity, integerOnly = false } = {}) { if (value === undefined || value === null || value === "") return null; let number = value; if (typeof value === "string") { const trimmed = value.trim(); if (!NUMBER_RE.test(trimmed)) throw new Error(`${label} must be a finite number.`); number = Number(trimmed); } else if (typeof value !== "number") { throw new Error(`${label} must be a finite number.`); } if (!Number.isFinite(number)) throw new Error(`${label} must be a finite number.`); if (number < minimum || number > maximum) { const range = Number.isFinite(maximum) ? `between ${minimum} and ${maximum}` : `at least ${minimum}`; throw new Error(`${label} must be ${range}.`); } if (integerOnly && !Number.isInteger(number)) throw new Error(`${label} must be an integer.`); return number; } function isObject(value) { return value !== null && typeof value === "object" && !Array.isArray(value); } function normalizeTimeUnit(value, unit, key, label) { const number = checkedNumber(value, label, { minimum: 0 }); if (number === null) return null; let normalizedUnit = String(unit || "").trim().toLowerCase(); if (!normalizedUnit) { const name = String(key || "").toLowerCase(); if (name.endsWith("_seconds") || name.endsWith("_second")) normalizedUnit = "seconds"; else if (name.endsWith("_ns") || name.endsWith("_nanoseconds")) normalizedUnit = "nanoseconds"; else if (name.endsWith("_us") || name.endsWith("_microseconds")) normalizedUnit = "microseconds"; else normalizedUnit = "milliseconds"; } if (["ms", "millisecond", "milliseconds"].includes(normalizedUnit)) return number; if (["s", "sec", "second", "seconds"].includes(normalizedUnit)) return number * 1000; if (["ns", "nanosecond", "nanoseconds"].includes(normalizedUnit)) return number / 1e6; if (["us", "µs", "microsecond", "microseconds"].includes(normalizedUnit)) return number / 1000; throw new Error(`${label} uses an unsupported time unit: ${normalizedUnit}.`); } const STAT_KEYS = { mean: ["mean", "avg", "average", "mean_ms", "avg_ms"], p50: ["p50", "median", "p50_ms", "median_ms"], p99: ["p99", "p99_ms", "p99_estimate", "99th_percentile"], }; function statFromEntry(entry, statistic) { if (!isObject(entry)) return { found: false }; const source = isObject(entry.stats) ? entry.stats : entry; for (const key of STAT_KEYS[statistic]) { if (HAS_OWN(source, key)) return { found: true, value: source[key], unit: entry.unit || source.unit }; } if (HAS_OWN(source, "value")) return { found: true, value: source.value, unit: entry.unit || source.unit }; return { found: false }; } function findNumber(containers, keys, label, { statistic = null, time = false, integerOnly = false, maximum = Infinity } = {}) { for (const container of containers) { if (!isObject(container)) continue; for (const key of keys) { if (!HAS_OWN(container, key) || container[key] === null || container[key] === undefined || container[key] === "") continue; const entry = container[key]; let value = entry; let unit = ""; if (isObject(entry)) { if (statistic) { const nested = statFromEntry(entry, statistic); if (!nested.found) continue; value = nested.value; unit = nested.unit || ""; } else if (HAS_OWN(entry, "value")) { value = entry.value; unit = entry.unit || ""; } else { const nested = statFromEntry(entry, "mean"); if (!nested.found) continue; value = nested.value; unit = nested.unit || ""; } } if (time) return normalizeTimeUnit(value, unit, key, label); return checkedNumber(value, label, { minimum: 0, maximum, integerOnly }); } } return null; } function metricGroup(containers, definition, label) { return { mean_ms: findNumber(containers, definition.mean, `${label} mean`, { statistic: "mean", time: true }), p50_ms: findNumber(containers, definition.p50, `${label} p50`, { statistic: "p50", time: true }), p99_ms: findNumber(containers, definition.p99, `${label} p99`, { statistic: "p99", time: true }), }; } const LLM_GROUPS = { ttft: { label: "TTFT", mean: ["mean_ttft_ms", "avg_ttft_ms", "ttft_mean_ms", "ttft_avg_ms", "time_to_first_token_mean_ms", "time_to_first_token_avg_ms", "ttft_ms", "time_to_first_token_ms", "time_to_first_token"], p50: ["median_ttft_ms", "p50_ttft_ms", "ttft_median_ms", "ttft_p50_ms", "time_to_first_token_median_ms", "time_to_first_token_p50_ms", "time_to_first_token"], p99: ["p99_ttft_ms", "ttft_p99_ms", "time_to_first_token_p99_ms", "time_to_first_token"], }, tpot: { label: "TPOT", mean: ["mean_tpot_ms", "avg_tpot_ms", "tpot_mean_ms", "tpot_avg_ms", "time_per_output_token_mean_ms", "tpot_ms", "time_per_output_token_ms", "time_per_output_token", "tpot"], p50: ["median_tpot_ms", "p50_tpot_ms", "tpot_median_ms", "tpot_p50_ms", "time_per_output_token_p50_ms", "time_per_output_token", "tpot"], p99: ["p99_tpot_ms", "tpot_p99_ms", "time_per_output_token_p99_ms", "time_per_output_token", "tpot"], }, itl: { label: "ITL", mean: ["mean_itl_ms", "avg_itl_ms", "itl_mean_ms", "itl_avg_ms", "inter_token_latency_mean_ms", "itl_ms", "inter_token_latency_ms", "itl", "inter_token_latency"], p50: ["median_itl_ms", "p50_itl_ms", "itl_median_ms", "itl_p50_ms", "inter_token_latency_p50_ms", "itl", "inter_token_latency"], p99: ["p99_itl_ms", "itl_p99_ms", "inter_token_latency_p99_ms", "itl", "inter_token_latency"], }, request_latency: { label: "Request latency", mean: ["mean_request_latency_ms", "avg_request_latency_ms", "request_latency_mean_ms", "mean_e2el_ms", "mean_e2e_latency_ms", "request_latency_ms", "e2el_ms", "request_latency", "e2el"], p50: ["median_request_latency_ms", "p50_request_latency_ms", "request_latency_p50_ms", "median_e2el_ms", "p50_e2el_ms", "p50_e2e_latency_ms", "request_latency", "e2el"], p99: ["p99_request_latency_ms", "request_latency_p99_ms", "p99_e2el_ms", "p99_e2e_latency_ms", "request_latency", "e2el"], }, }; function getLlmMetrics(containers) { const llm = {}; for (const [key, definition] of Object.entries(LLM_GROUPS)) { llm[key] = metricGroup(containers, definition, definition.label); validatePercentiles(llm[key], definition.label); } llm.request_throughput_rps = findNumber(containers, ["request_throughput", "request_throughput_rps", "requests_per_second", "req_per_sec"], "Request throughput"); llm.output_token_throughput_tps = findNumber(containers, ["output_token_throughput", "output_token_throughput_tps", "output_throughput", "output_tokens_per_second", "tokens_per_second"], "Output token throughput"); llm.total_input_tokens = findNumber(containers, ["total_input_tokens", "total_prompt_tokens", "input_tokens", "prompt_tokens", "num_input_tokens"], "Total input tokens", { integerOnly: true }); llm.total_output_tokens = findNumber(containers, ["total_output_tokens", "total_generated_tokens", "generated_tokens", "output_tokens", "completion_tokens", "num_output_tokens"], "Total output tokens", { integerOnly: true }); return llm; } function hasLlmData(llm) { for (const [key, value] of Object.entries(llm)) { if (key === "total_input_tokens" || key === "total_output_tokens" || key.endsWith("_rps") || key.endsWith("_tps")) { if (value !== null) return true; } else if (isObject(value) && Object.values(value).some((number) => number !== null)) { return true; } } return false; } function validatePercentiles(group, label) { if (group.p50_ms !== null && group.p99_ms !== null && group.p99_ms < group.p50_ms) { throw new Error(`${label} p99 must be greater than or equal to p50.`); } } function normalizeRow(result, payload, flat = false) { const containers = [result.metrics, result.latency, result.stats, result, payload.metrics, payload.stats, payload]; const llm = getLlmMetrics(containers); const legacyLatency = [result.latency, result.metrics, result, payload.metrics, payload]; const p50 = findNumber(legacyLatency, ["p50_ms", "median_latency_ms", "latency_p50_ms"], "p50 latency", { time: true }); const p99 = findNumber(legacyLatency, ["p99_ms", "p99_latency_ms", "latency_p99_ms"], "p99 latency", { time: true }); validatePercentiles({ p50_ms: p50, p99_ms: p99 }, "Latency"); const throughput = findNumber(legacyLatency, ["throughput_ips", "inferences_per_second", "inference_throughput_ips"], "Inference throughput"); const measured = [p50, p99, throughput].some((value) => value !== null) || hasLlmData(llm); let available; if (HAS_OWN(result, "available")) { if (typeof result.available !== "boolean") throw new Error("Backend availability must be a boolean."); available = result.available; } else if (HAS_OWN(result, "success")) { if (typeof result.success !== "boolean") throw new Error("Backend success status must be a boolean."); available = result.success; } else { available = measured; } const backendValue = result.backend || result.engine || result.framework || result.runtime || payload.backend || payload.engine || payload.framework; const backend = String(backendValue || (flat ? "LLM benchmark" : "unknown")).slice(0, 120); const reasonValue = result.reason || result.error || result.message || ""; return { backend, available, reason: String(reasonValue).slice(0, 500), p50_ms: p50, p99_ms: p99, throughput_ips: throughput, llm, measured, }; } export function summarizeReport(payload) { if (!isObject(payload)) throw new Error("The report must be a JSON object."); const hasResultsArray = Array.isArray(payload.results); if (hasResultsArray && payload.results.length > MAX_REPORT_ROWS) { throw new Error(`This report has more than ${MAX_REPORT_ROWS.toLocaleString()} result rows. Load an aggregate benchmark summary instead of per-request details.`); } let rows; let format; if (hasResultsArray && payload.results.some(isObject)) { rows = payload.results.filter(isObject).map((result) => normalizeRow(result, payload)); format = "results-array"; } else { const flatRow = normalizeRow(payload, payload, true); if (!flatRow.measured) { throw new Error("Unsupported report. Provide an inferbench 'results' array or a flat LLM summary with recognized metrics."); } rows = [flatRow]; format = "flat-llm-summary"; } if (!rows.length) throw new Error("The report contains no valid result rows."); const accuracySource = isObject(payload.accuracy) ? Object.values(payload.accuracy) : []; if (accuracySource.length > MAX_REPORT_ROWS) throw new Error(`This report has more than ${MAX_REPORT_ROWS.toLocaleString()} accuracy rows.`); const accuracyRows = accuracySource.filter(isObject).map((row) => ({ backend: String(row.backend || "unknown").slice(0, 120), reference: String(row.reference || payload.reference || "unknown").slice(0, 120), samples: checkedNumber(row.samples, "Accuracy samples", { minimum: 0, integerOnly: true }), max_abs_diff: checkedNumber(row.max_abs_diff, "Maximum absolute difference", { minimum: 0 }), mean_abs_diff: checkedNumber(row.mean_abs_diff, "Mean absolute difference", { minimum: 0 }), top1_agreement: checkedNumber(row.top1_agreement, "Top-1 agreement", { minimum: 0, maximum: 1 }), })); const modelValue = payload.model || payload.model_name || payload.model_id || payload.served_model_name || "unknown"; const metadataNumber = (key, minimum = 0) => checkedNumber(payload[key], key, { minimum, integerOnly: key !== "batch" }); const host = isObject(payload.host) ? payload.host : isObject(payload.hardware) ? payload.hardware : isObject(payload.system) ? payload.system : typeof payload.host === "string" ? { platform: payload.host } : {}; return { format, schemaDemo: payload.schema_demo === true, model: String(modelValue).slice(0, 120), batch: metadataNumber("batch"), iterations: metadataNumber("iterations"), warmup: metadataNumber("warmup"), reference: String(payload.reference || "unknown").slice(0, 120), host, rows, accuracyRows, hasLlmMetrics: rows.some((row) => hasLlmData(row.llm)), source: "Analyzed locally in your browser. This Space did not run the benchmark or upload your report.", }; }