diff --git a/zig/pkg/inference/build/benches.zig b/zig/pkg/inference/build/benches.zig index e2eb2f8972..3fb42f3fcb 100644 --- a/zig/pkg/inference/build/benches.zig +++ b/zig/pkg/inference/build/benches.zig @@ -223,6 +223,24 @@ pub fn addGliner25(ctx: Context) *std.Build.Module { decide_bench.root_module.addImport("inference_linalg", ctx.graph.inference_linalg_mod); decide_bench.root_module.link_libc = true; ctx.step("bench-gliner25-decide-build", "Build the loaded-model GLiNER2.5-Decide request benchmark").dependOn(&b.addInstallArtifact(decide_bench, .{}).step); + const decide_quant_bench = b.addExecutable(.{ + .name = "antfly-inference-gliner-decide-quant-bench", + .max_rss = 8 * 1024 * 1024 * 1024, + .root_module = b.createModule(.{ + .root_source_file = ctx.path("src/bench/gliner_decide_quant.zig"), + .target = ctx.target, + .optimize = ctx.optimize, + }), + }); + decide_quant_bench.root_module.addImport("inference_internal", ctx.graph.inference_internal_mod); + decide_quant_bench.root_module.link_libc = true; + ctx.step("bench-gliner-decide-quant-build", "Build the offline ModernBERT GLiNER precision comparison").dependOn(&b.addInstallArtifact(decide_quant_bench, .{}).step); + const decide_quant_tests = b.addTest(.{ + .max_rss = 6 * 1024 * 1024 * 1024, + .root_module = ctx.graph.inference_internal_mod, + .filters = &.{ "GLiNER Decide", "open split gliner gguf" }, + }); + ctx.step("test-gliner-decide-quant", "Test exact GLiNER Decide artifact identity and split GGUF snapshots").dependOn(&ctx.addRunArtifact(decide_quant_tests).step); // Use the shared optimize value for the entire dependency graph. The // worker refuses to compile unless that graph is ReleaseFast; forcing // only this executable root would leave imported kernels unoptimized. diff --git a/zig/pkg/inference/src/architectures/modern_bert.zig b/zig/pkg/inference/src/architectures/modern_bert.zig index f9063f286a..0c136fe494 100644 --- a/zig/pkg/inference/src/architectures/modern_bert.zig +++ b/zig/pkg/inference/src/architectures/modern_bert.zig @@ -211,6 +211,15 @@ fn preplanMetalModernBertEncoder( // first it avoids even loading the 88 projection weights from safetensors. if (metalModernBertEncoderSlotsPrepared(cb, config)) return true; + // The measured 1B encoder path keeps packed checkpoint weights, + // but prepare reusable F16 MPS matrices for encoder-shaped GEMMs. Bound + // the shape and per-matrix staging (largest matrix is 52.5 MiB in F32; + // all 112 resident F16 matrices total 1,875,378,176 bytes). The override + // retains the direct-quant path for controlled comparisons. + const prefer_f16_mps = config.metal_f16_weight_mirrors and hidden == 1792 and intermediate == 3840 and + layer_count == 28 and heads == 28 and + platform.env.getenvBoolDefault("TERMITE_METAL_MODERNBERT_1B_F16_MIRRORS", true); + const qkv_zero_bias = try makeZeroBias(cb, allocator, hidden * 3); defer cb.free(qkv_zero_bias); const ffn_in_zero_bias = try makeZeroBias(cb, allocator, intermediate * 2); @@ -239,11 +248,14 @@ fn preplanMetalModernBertEncoder( .out_dim = output_dim, // Native F16 safetensors reach Metal directly through the // prepare path. No F32 mirror is required for this layout. - .retain_dense_fallback = false, + .retain_dense_fallback = prefer_f16_mps, + .dense_fallback_max_bytes = if (prefer_f16_mps) 64 * 1024 * 1024 else null, + .allow_direct_quant_fallback = prefer_f16_mps, + .prefer_f16_mps_fallback = prefer_f16_mps, // MPS GEMM outruns the hand-written dense kernels here // (Laya-large 3.1x, OpenDecider-nano 1.4x on M4 Max); BF16 // weights are expanded to F32 for it. - .prefer_f32_mps_fallback = !platform.env.getenvBool("TERMITE_METAL_DISABLE_MODERNBERT_F32_MPS"), + .prefer_f32_mps_fallback = !prefer_f16_mps and !platform.env.getenvBool("TERMITE_METAL_DISABLE_MODERNBERT_F32_MPS"), }))) return false; } } @@ -273,6 +285,9 @@ fn preplanMetalModernBertEncoder( // --------------------------------------------------------------------------- pub const Config = struct { + /// Execution preference enabled by the measured GLiNER decision path. + /// Checkpoint parsing leaves other ModernBERT callers unchanged. + metal_f16_weight_mirrors: bool = false, laya: ?@import("../models/laya.zig").Config = null, vocab_size: u32 = 50368, hidden_size: u32 = 768, diff --git a/zig/pkg/inference/src/architectures/session_factory.zig b/zig/pkg/inference/src/architectures/session_factory.zig index 2e4c3b287b..e2b2c24da1 100644 --- a/zig/pkg/inference/src/architectures/session_factory.zig +++ b/zig/pkg/inference/src/architectures/session_factory.zig @@ -380,13 +380,22 @@ fn captureGlinerDecisionIdentity(mf: *const manifest_mod.ModelManifest, store: t const encoder = mf.gliner_span_encoder_digest orelse return null; const tokenizer = mf.gliner_span_tokenizer_digest orelse return null; const tokenizer_config = mf.gliner_span_tokenizer_config_digest orelse return null; - const reader = store.singleSafetensorsReader() orelse return null; - var all_f32 = true; - var entries = reader.header.tensors.iterator(); - while (entries.next()) |entry| if (entry.value_ptr.dtype != .f32) { - all_f32 = false; - break; - }; + var inventory: gliner_decide_qualification.TensorInventory = undefined; + var weight: gliner_decide_qualification.Digest = undefined; + var companion: ?gliner_decide_qualification.Digest = null; + if (store.singleSafetensorsReader()) |reader| { + inventory = .{ .count = reader.header.tensors.count(), .all_f32 = true }; + var entries = reader.header.tensors.iterator(); + while (entries.next()) |entry| if (entry.value_ptr.dtype != .f32) { + inventory.all_f32 = false; + break; + }; + weight = gliner_decide_qualification.Digest.of(reader.file_bytes); + } else if (store.glinerGgufSnapshot()) |snapshot| { + inventory = .{ .count = snapshot.tensor_count, .all_f32 = snapshot.all_f32 }; + weight = gliner_decide_qualification.Digest.of(snapshot.encoder); + companion = gliner_decide_qualification.Digest.of(snapshot.head); + } else return null; return .{ .encoder_family = family, .geometry = .{ @@ -406,8 +415,9 @@ fn captureGlinerDecisionIdentity(mf: *const manifest_mod.ModelManifest, store: t .sep_struct = mf.gliner_token_sep_struct, .sep_text = mf.gliner_token_sep_text, }, - .inventory = .{ .count = reader.header.tensors.count(), .all_f32 = all_f32 }, - .weight = gliner_decide_qualification.Digest.of(reader.file_bytes), + .inventory = inventory, + .weight = weight, + .weight_companion = companion, .sidecars = .{ .config = config, .encoder_config = encoder, diff --git a/zig/pkg/inference/src/extractors/gliner_span_v2_executor.zig b/zig/pkg/inference/src/extractors/gliner_span_v2_executor.zig index a55ef76363..dcad321622 100644 --- a/zig/pkg/inference/src/extractors/gliner_span_v2_executor.zig +++ b/zig/pkg/inference/src/extractors/gliner_span_v2_executor.zig @@ -171,7 +171,11 @@ pub fn classificationLogitsProfiled( cb.preferEagerQuantMirrors(true); break :blk try deberta_arch.forwardCtProfiled(cb, allocator, deberta, sample.input_ids, attention_mask, 1, seq_len, deberta_mod.glinerPrefersWeightMirrors(deberta), if (profile) |p| &p.encoder else null); }, - .modern_bert => |modern| try modern_bert_arch.forwardCT(cb, allocator, modern, sample.input_ids, attention_mask, 1, seq_len), + .modern_bert => |modern| blk: { + var execution_config = modern; + execution_config.metal_f16_weight_mirrors = true; + break :blk try modern_bert_arch.forwardCT(cb, allocator, execution_config, sample.input_ids, attention_mask, 1, seq_len); + }, }; defer cb.free(hidden); if (profile) |p| { diff --git a/zig/pkg/inference/src/inference_internal.zig b/zig/pkg/inference/src/inference_internal.zig index 13db702e4f..511e5e1f49 100644 --- a/zig/pkg/inference/src/inference_internal.zig +++ b/zig/pkg/inference/src/inference_internal.zig @@ -15,6 +15,7 @@ const build_options = @import("build_options"); pub const gliner_boundary_export = @import("gliner_boundary_export.zig"); +pub const gliner_span_v2_executor = @import("extractors/gliner_span_v2_executor.zig"); pub const platform = @import("antfly_platform"); pub const backends = @import("backends/backends.zig"); @@ -119,6 +120,8 @@ pub const architectures = struct { pub const gliner_head_graph = @import("architectures/gliner_head_graph.zig"); }; pub const models = struct { + pub const gliner_decide_qualification = @import("models/gliner_decide_qualification.zig"); + pub const tensor_store = @import("models/tensor_store.zig"); pub const manifest = @import("models/manifest.zig"); pub const gliner_boundary_bundle = @import("models/gliner_boundary_bundle.zig"); pub const gliner_boundary = @import("models/gliner_boundary.zig"); @@ -141,3 +144,8 @@ pub const gguf = struct { pub const quant_codec = @import("gguf/quant_codec.zig"); pub const tensor_types = @import("gguf/tensor_types.zig"); }; + +test "GLiNER Decide artifact test reachability" { + _ = models.gliner_decide_qualification; + _ = models.tensor_store; +} diff --git a/zig/pkg/inference/src/models/gliner_decide_qualification.zig b/zig/pkg/inference/src/models/gliner_decide_qualification.zig index afa5b98206..f90732c344 100644 --- a/zig/pkg/inference/src/models/gliner_decide_qualification.zig +++ b/zig/pkg/inference/src/models/gliner_decide_qualification.zig @@ -1,5 +1,17 @@ // Copyright 2026 Antfly, Inc. // SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. //! Immutable loaded-artifact qualification for public GLiNER Decide serving. //! Registry publication and request execution consume this same contract. An @@ -52,6 +64,7 @@ pub const Identity = struct { markers: Markers, inventory: TensorInventory, weight: Digest, + weight_companion: ?Digest = null, sidecars: Sidecars, }; @@ -69,6 +82,8 @@ const legacy_tokenizer_config = FilePin{ .path = "tokenizer_config.json", .size_ // capture. Runtime request admission remains bounded separately below. pub const decide_1b_production_qualified = true; const decide_1b_weight = FilePin{ .path = "model.safetensors", .size_bytes = 4_755_208_228, .sha256 = "02c567d791aed26550d300064c7f0c0094fd65291503c65969b45b30786e33b3" }; +const decide_1b_q8_encoder = FilePin{ .path = "gliner2-encoder.Q8_0.gguf", .size_bytes = 1_118_356_576, .sha256 = "9ba4aea751bc8ce3586e70c4dfb3db93d0a850da5023b39d1bb68d21247f1694" }; +const decide_1b_q8_head = FilePin{ .path = "gliner_head.gguf", .size_bytes = 617_195_872, .sha256 = "9a6c1a8ae0f58158404ea6875012e0c27e5923188906492ec7d948968741f558" }; const decide_1b_config = FilePin{ .path = "config.json", .size_bytes = 464, .sha256 = "d2732928820b95649bc87f051345b2394fff87fc28c01f945610f943d6f402b2" }; const decide_1b_encoder_config = FilePin{ .path = "encoder_config/config.json", .size_bytes = 2160, .sha256 = "c2cc4c15c7504b9e15651f2a32dff82b84fa9a99060a3218671a930da0a9b50d" }; const decide_1b_tokenizer = FilePin{ .path = "tokenizer.json", .size_bytes = 3_585_055, .sha256 = "ddb379b6a4679ee16646bf0b726de9ec538d7c80ea9d4d4e33b069f19c9e1efb" }; @@ -107,7 +122,7 @@ fn isLegacyDecide(identity: Identity) bool { .sep_struct = 128001, .sep_text = 128002, }) and - identity.inventory.count == 419 and identity.inventory.all_f32 and + identity.inventory.count == 419 and identity.inventory.all_f32 and identity.weight_companion == null and digestMatches(identity.weight, legacy_weight) and sidecarsMatch(identity.sidecars, legacy_config, legacy_encoder_config, legacy_tokenizer, legacy_tokenizer_config, legacy_special_tokens); } @@ -131,8 +146,11 @@ fn isDecide1B(identity: Identity) bool { .sep_struct = 50368, .sep_text = 50369, }) and - identity.inventory.count == 199 and identity.inventory.all_f32 and - digestMatches(identity.weight, decide_1b_weight) and + identity.inventory.count == 199 and + (if (identity.weight_companion) |head| + !identity.inventory.all_f32 and digestMatches(identity.weight, decide_1b_q8_encoder) and digestMatches(head, decide_1b_q8_head) + else + identity.inventory.all_f32 and digestMatches(identity.weight, decide_1b_weight)) and sidecarsMatch(identity.sidecars, decide_1b_config, decide_1b_encoder_config, decide_1b_tokenizer, decide_1b_tokenizer_config, null); } @@ -221,6 +239,32 @@ test "GLiNER Decide 1B qualification binds exact loaded identity" { identity.inventory.all_f32 = false; try std.testing.expect(!isDecide1B(identity)); try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); + + identity.weight = digestFromPin(decide_1b_q8_encoder); + identity.weight_companion = digestFromPin(decide_1b_q8_head); + try std.testing.expectEqual(QualifiedVariant.decide_1b, try require(identity)); + const quantized = identity; + identity.weight_companion = null; + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); + identity = quantized; + identity.weight_companion.?.sha256[0] ^= 1; + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); + identity = quantized; + identity.weight.sha256[0] ^= 1; + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); + identity = quantized; + identity.sidecars.tokenizer.sha256[0] ^= 1; + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); + identity = quantized; + identity.inventory.count -= 1; + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); + identity = quantized; + identity.inventory.all_f32 = true; + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); + identity = quantized; + identity.weight = digestFromPin(decide_1b_weight); + identity.inventory.all_f32 = true; + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); } test "GLiNER Decide 1B serving policy stays within measured backend and geometry" { diff --git a/zig/pkg/inference/src/models/tensor_store.zig b/zig/pkg/inference/src/models/tensor_store.zig index 214ebb8aa0..6c6f95ed1c 100644 --- a/zig/pkg/inference/src/models/tensor_store.zig +++ b/zig/pkg/inference/src/models/tensor_store.zig @@ -168,6 +168,39 @@ pub const TensorStore = struct { return store.rawData(); } + pub const GlinerGgufSnapshot = struct { + encoder: []const u8, + head: []const u8, + tensor_count: usize, + all_f32: bool, + }; + + /// Borrow both exact open GGUF artifacts. Never reopen a pathname to + /// establish the identity of an already loaded split GLiNER session. + pub fn glinerGgufSnapshot(self: TensorStore) ?GlinerGgufSnapshot { + if (self.vtable != &CompositeGlinerStore.vtable) return null; + const store: *const CompositeGlinerStore = @ptrCast(@alignCast(self.ptr)); + const head = store.head_gguf orelse return null; + if (store.head_safetensors != null) return null; + var all_f32 = true; + for (store.encoder.parsed.tensors) |tensor| { + if (tensor.tensor_type != .known or tensor.tensor_type.known != .F32) all_f32 = false; + // Reject shadowing across the encoder and auxiliary head. + for (head.parsed.tensors) |other| { + if (std.mem.eql(u8, tensor.name, other.name)) return null; + } + } + for (head.parsed.tensors) |tensor| { + if (tensor.tensor_type != .known or tensor.tensor_type.known != .F32) all_f32 = false; + } + return .{ + .encoder = store.encoder.rawData(), + .head = head.rawData(), + .tensor_count = store.encoder.parsed.tensors.len + head.parsed.tensors.len, + .all_f32 = all_f32, + }; + } + pub fn singleSafetensorsReader(self: TensorStore) ?*const @import("safetensors.zig").MMapReader { if (self.vtable != &SafetensorsStore.vtable) return null; const store: *const SafetensorsStore = @ptrCast(@alignCast(self.ptr)); @@ -1522,6 +1555,7 @@ test "open gguf tensor store from manifest" { defer store.deinit(); try std.testing.expectEqual(StoreKind.gguf, store.kind()); try std.testing.expect(store.ggufFile() != null); + try std.testing.expect((try store.weightSource()) != null); var tensor_ref = try store.describeTensor(allocator, "tok_embeddings.weight"); try std.testing.expectEqual(@as(usize, 8), tensor_ref.byte_len); @@ -1739,6 +1773,20 @@ test "open split gliner gguf bundle from manifest" { try std.testing.expectEqual(StoreKind.gguf, store.kind()); try std.testing.expect(store.ggufFile() != null); + const snapshot = store.glinerGgufSnapshot() orelse return error.TestUnexpectedResult; + try std.testing.expectEqualSlices(u8, gguf_data.items, snapshot.encoder); + try std.testing.expectEqualSlices(u8, head_data.items, snapshot.head); + try std.testing.expectEqual(@as(usize, 3), snapshot.tensor_count); + try std.testing.expect(!snapshot.all_f32); + try std.testing.expect(store.ggufArtifactBytes() == null); + + const composite: *CompositeGlinerStore = @ptrCast(@alignCast(store.ptr)); + const original_name = composite.head_gguf.?.parsed.tensors[0].name; + composite.head_gguf.?.parsed.tensors[0].name = composite.encoder.parsed.tensors[0].name; + const shadowed = store.glinerGgufSnapshot(); + composite.head_gguf.?.parsed.tensors[0].name = original_name; + try std.testing.expect(shadowed == null); + const source = (try store.weightSource()) orelse return error.TestUnexpectedResult; const names = try source.listNames(allocator); defer allocator.free(names); diff --git a/zig/pkg/inference/src/bench/gliner_decide_quant.zig b/zig/pkg/inference/src/bench/gliner_decide_quant.zig new file mode 100644 index 0000000000..4f183b36e1 --- /dev/null +++ b/zig/pkg/inference/src/bench/gliner_decide_quant.zig @@ -0,0 +1,128 @@ +// Copyright 2026 Antfly, Inc. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Offline artifact comparison. Never grants public serving qualification. +const std = @import("std"); +const inference = @import("inference_internal"); +const factory = inference.architectures.session_factory; +const processor = inference.pipelines.gliner_boundary_processor; +const schema = inference.pipelines.extraction_schema; +const executor = inference.gliner_span_v2_executor; +const Case = struct { + id: []const u8, + text: []const u8, + schema_json: []const u8, + input_ids: []const i64, + logits: []const []const f64, +}; +const Report = struct { id: []const u8, tokens: usize, median_ms: f64, p95_ms: f64, max_logit_error: f64, logits: [][]f64, samples_ms: []f64 }; + +pub fn main(init: std.process.Init) !void { + const a = inference.platform.allocator.processAllocator(std.heap.smp_allocator); + var args = std.process.Args.Iterator.init(init.minimal.args); + _ = args.next(); + var path: ?[]const u8 = null; + var capture: ?[]const u8 = null; + var backend: []const u8 = "native"; + var warmups: usize = 3; + var reps: usize = 20; + var tolerance: f64 = 0.05; + while (args.next()) |key| { + const value = args.next() orelse return error.MissingArgument; + if (std.mem.eql(u8, key, "--model-dir")) path = value else if (std.mem.eql(u8, key, "--capture")) capture = value else if (std.mem.eql(u8, key, "--backend")) backend = value else if (std.mem.eql(u8, key, "--warmups")) warmups = try std.fmt.parseInt(usize, value, 10) else if (std.mem.eql(u8, key, "--reps")) reps = try std.fmt.parseInt(usize, value, 10) else if (std.mem.eql(u8, key, "--tolerance")) tolerance = try std.fmt.parseFloat(f64, value) else return error.InvalidArgument; + } + if (reps < 3 or reps > 1000 or warmups > 100 or !std.math.isFinite(tolerance) or tolerance <= 0) return error.InvalidArgument; + const bytes = try inference.util.c_file.readFile(a, capture orelse return error.MissingArgument); + defer a.free(bytes); + const cases = try std.json.parseFromSlice([]const Case, a, bytes, .{}); + defer cases.deinit(); + const directory = path orelse return error.MissingArgument; + const start_load = inference.platform.time.monotonicNs(); + const session = if (std.mem.eql(u8, backend, "metal")) try factory.createMetalSession(a, directory) else if (std.mem.eql(u8, backend, "native")) try factory.createNativeSession(a, directory) else return error.InvalidBackend; + defer session.close(); + if (std.mem.eql(u8, backend, "metal") and session.backend() != .metal) return error.InvalidBackend; + const config = try factory.getGlinerSpanConfig(session); + if (config != .modern_bert) return error.InvalidModel; + const tokenizer_path = try std.fs.path.join(a, &.{ directory, "tokenizer.json" }); + defer a.free(tokenizer_path); + const tok_bytes = try inference.util.c_file.readFile(a, tokenizer_path); + defer a.free(tok_bytes); + const tok = try inference.hf_tokenizer.HfTokenizer.loadFromBytes(a, tok_bytes); + defer tok.tokenizer().deinitTokenizer(); + const load_ms = @as(f64, @floatFromInt(inference.platform.time.monotonicNs() - start_load)) / std.time.ns_per_ms; + const watchdog = if (std.mem.eql(u8, backend, "metal")) try inference.HardCancellationWatchdog.create(a) else null; + defer if (watchdog) |owner| owner.destroy(); + if (watchdog) |owner| try owner.start(init.io); + var report_arena = std.heap.ArenaAllocator.init(a); + defer report_arena.deinit(); + const out = report_arena.allocator(); + const reports = try out.alloc(Report, cases.value.len); + for (cases.value, reports) |case, *report| { + const samples = try out.alloc(f64, reps); + var max_error: f64 = 0; + var last_rows: [][]f64 = &.{}; + for (0..1 + warmups + reps) |iteration| { + var request_arena = std.heap.ArenaAllocator.init(a); + defer request_arena.deinit(); + const ra = request_arena.allocator(); + const control = inference.InferenceExecutionControl{ + .hard_cancellation = if (watchdog) |owner| owner.boundary() else null, + .deadline_ns = inference.platform.time.monotonicNs() + 120 * std.time.ns_per_s, + }; + const start = inference.platform.time.monotonicNs(); + var compiled = try schema.compile(ra, case.schema_json, .{}); + defer compiled.deinit(); + var prepared = try processor.prepare(ra, tok.tokenizer(), &.{.{ .text = case.text, .schema = &compiled }}, .{ .max_sequence_tokens = 7999, .max_batch_tokens = 7999 }); + defer prepared.deinit(); + if (prepared.samples.len != 1) return error.InvalidCase; + const counts = try ra.alloc(usize, compiled.schema.classifications.len); + for (compiled.schema.classifications, counts) |classification, *count| count.* = classification.task.labels.len; + var managed = try factory.getManagedComputeBackend(session, ra, null, control); + defer managed.deinit(); + const rows = try executor.classificationLogits(&managed.backend, ra, .{ .modern_bert = config.modern_bert }, prepared.samples[0], counts); + const elapsed = @as(f64, @floatFromInt(inference.platform.time.monotonicNs() - start)) / std.time.ns_per_ms; + if (!std.mem.eql(i64, prepared.input_ids, case.input_ids) or rows.len != case.logits.len) return error.PreprocessingMismatch; + for (rows, case.logits) |got, expected| { + if (got.len != expected.len) return error.LogitShapeMismatch; + for (got, expected) |actual, want| { + if (!std.math.isFinite(actual)) return error.NonFiniteLogit; + const delta = @abs(actual - want); + max_error = @max(max_error, delta); + if (delta > tolerance) { + std.debug.print("{s}: logit error {d} exceeds {d}\n", .{ case.id, delta, tolerance }); + return error.LogitMismatch; + } + } + } + if (iteration > warmups) samples[iteration - warmups - 1] = elapsed; + if (iteration == warmups + reps) { + last_rows = try out.alloc([]f64, rows.len); + for (rows, last_rows) |row, *saved| saved.* = try out.dupe(f64, row); + } + } + const sorted = try out.dupe(f64, samples); + std.mem.sort(f64, sorted, {}, struct { + fn less(_: void, x: f64, y: f64) bool { + return x < y; + } + }.less); + report.* = .{ .id = case.id, .tokens = case.input_ids.len, .median_ms = if (reps % 2 == 0) (sorted[reps / 2 - 1] + sorted[reps / 2]) / 2 else sorted[reps / 2], .p95_ms = sorted[@min(reps - 1, (reps * 95 + 99) / 100 - 1)], .max_logit_error = max_error, .logits = last_rows, .samples_ms = samples }; + std.debug.print("{s} {s}: {d:.3} ms, error {d:.6}\n", .{ backend, case.id, report.median_ms, max_error }); + } + const result = try std.json.Stringify.valueAlloc(a, .{ .scope = "offline loaded pipeline: schema compile, tokenization, encoder, classifier and completed readback; excludes HTTP, model load, and output presentation", .backend = backend, .model_dir = directory, .load_ms = load_ms, .warmups = warmups, .reps = reps, .validation_preflights = 1, .tolerance = tolerance, .reports = reports }, .{}); + defer a.free(result); + try std.Io.File.stdout().writeStreamingAll(init.io, result); + try std.Io.File.stdout().writeStreamingAll(init.io, "\n"); +}