| |
| |
| |
| |
| @@ -223,6 +223,24 @@ pub fn addGliner25(ctx: Context) *std.Build.Module { |
| decide_bench.root_module.addImport("inference_linalg", ctx.graph.inference_linalg_mod); |
| decide_bench.root_module.link_libc = true; |
| ctx.step("bench-gliner25-decide-build", "Build the loaded-model GLiNER2.5-Decide request benchmark").dependOn(&b.addInstallArtifact(decide_bench, .{}).step); |
| + const decide_quant_bench = b.addExecutable(.{ |
| + .name = "antfly-inference-gliner-decide-quant-bench", |
| + .max_rss = 8 * 1024 * 1024 * 1024, |
| + .root_module = b.createModule(.{ |
| + .root_source_file = ctx.path("src/bench/gliner_decide_quant.zig"), |
| + .target = ctx.target, |
| + .optimize = ctx.optimize, |
| + }), |
| + }); |
| + decide_quant_bench.root_module.addImport("inference_internal", ctx.graph.inference_internal_mod); |
| + decide_quant_bench.root_module.link_libc = true; |
| + ctx.step("bench-gliner-decide-quant-build", "Build the offline ModernBERT GLiNER precision comparison").dependOn(&b.addInstallArtifact(decide_quant_bench, .{}).step); |
| + const decide_quant_tests = b.addTest(.{ |
| + .max_rss = 6 * 1024 * 1024 * 1024, |
| + .root_module = ctx.graph.inference_internal_mod, |
| + .filters = &.{ "GLiNER Decide", "open split gliner gguf" }, |
| + }); |
| + ctx.step("test-gliner-decide-quant", "Test exact GLiNER Decide artifact identity and split GGUF snapshots").dependOn(&ctx.addRunArtifact(decide_quant_tests).step); |
| // Use the shared optimize value for the entire dependency graph. The |
| // worker refuses to compile unless that graph is ReleaseFast; forcing |
| // only this executable root would leave imported kernels unoptimized. |
| |
| |
| |
| |
| @@ -211,6 +211,15 @@ fn preplanMetalModernBertEncoder( |
| // first it avoids even loading the 88 projection weights from safetensors. |
| if (metalModernBertEncoderSlotsPrepared(cb, config)) return true; |
| |
| + // The measured 1B encoder path keeps packed checkpoint weights, |
| + // but prepare reusable F16 MPS matrices for encoder-shaped GEMMs. Bound |
| + // the shape and per-matrix staging (largest matrix is 52.5 MiB in F32; |
| + // all 112 resident F16 matrices total 1,875,378,176 bytes). The override |
| + // retains the direct-quant path for controlled comparisons. |
| + const prefer_f16_mps = config.metal_f16_weight_mirrors and hidden == 1792 and intermediate == 3840 and |
| + layer_count == 28 and heads == 28 and |
| + platform.env.getenvBoolDefault("TERMITE_METAL_MODERNBERT_1B_F16_MIRRORS", true); |
| + |
| const qkv_zero_bias = try makeZeroBias(cb, allocator, hidden * 3); |
| defer cb.free(qkv_zero_bias); |
| const ffn_in_zero_bias = try makeZeroBias(cb, allocator, intermediate * 2); |
| @@ -239,11 +248,14 @@ fn preplanMetalModernBertEncoder( |
| .out_dim = output_dim, |
| // Native F16 safetensors reach Metal directly through the |
| // prepare path. No F32 mirror is required for this layout. |
| - .retain_dense_fallback = false, |
| + .retain_dense_fallback = prefer_f16_mps, |
| + .dense_fallback_max_bytes = if (prefer_f16_mps) 64 * 1024 * 1024 else null, |
| + .allow_direct_quant_fallback = prefer_f16_mps, |
| + .prefer_f16_mps_fallback = prefer_f16_mps, |
| // MPS GEMM outruns the hand-written dense kernels here |
| // (Laya-large 3.1x, OpenDecider-nano 1.4x on M4 Max); BF16 |
| // weights are expanded to F32 for it. |
| - .prefer_f32_mps_fallback = !platform.env.getenvBool("TERMITE_METAL_DISABLE_MODERNBERT_F32_MPS"), |
| + .prefer_f32_mps_fallback = !prefer_f16_mps and !platform.env.getenvBool("TERMITE_METAL_DISABLE_MODERNBERT_F32_MPS"), |
| }))) return false; |
| } |
| } |
| @@ -273,6 +285,9 @@ fn preplanMetalModernBertEncoder( |
| // --------------------------------------------------------------------------- |
| |
| pub const Config = struct { |
| + /// Execution preference enabled by the measured GLiNER decision path. |
| + /// Checkpoint parsing leaves other ModernBERT callers unchanged. |
| + metal_f16_weight_mirrors: bool = false, |
| laya: ?@import("../models/laya.zig").Config = null, |
| vocab_size: u32 = 50368, |
| hidden_size: u32 = 768, |
| |
| |
| |
| |
| @@ -380,13 +380,22 @@ fn captureGlinerDecisionIdentity(mf: *const manifest_mod.ModelManifest, store: t |
| const encoder = mf.gliner_span_encoder_digest orelse return null; |
| const tokenizer = mf.gliner_span_tokenizer_digest orelse return null; |
| const tokenizer_config = mf.gliner_span_tokenizer_config_digest orelse return null; |
| - const reader = store.singleSafetensorsReader() orelse return null; |
| - var all_f32 = true; |
| - var entries = reader.header.tensors.iterator(); |
| - while (entries.next()) |entry| if (entry.value_ptr.dtype != .f32) { |
| - all_f32 = false; |
| - break; |
| - }; |
| + var inventory: gliner_decide_qualification.TensorInventory = undefined; |
| + var weight: gliner_decide_qualification.Digest = undefined; |
| + var companion: ?gliner_decide_qualification.Digest = null; |
| + if (store.singleSafetensorsReader()) |reader| { |
| + inventory = .{ .count = reader.header.tensors.count(), .all_f32 = true }; |
| + var entries = reader.header.tensors.iterator(); |
| + while (entries.next()) |entry| if (entry.value_ptr.dtype != .f32) { |
| + inventory.all_f32 = false; |
| + break; |
| + }; |
| + weight = gliner_decide_qualification.Digest.of(reader.file_bytes); |
| + } else if (store.glinerGgufSnapshot()) |snapshot| { |
| + inventory = .{ .count = snapshot.tensor_count, .all_f32 = snapshot.all_f32 }; |
| + weight = gliner_decide_qualification.Digest.of(snapshot.encoder); |
| + companion = gliner_decide_qualification.Digest.of(snapshot.head); |
| + } else return null; |
| return .{ |
| .encoder_family = family, |
| .geometry = .{ |
| @@ -406,8 +415,9 @@ fn captureGlinerDecisionIdentity(mf: *const manifest_mod.ModelManifest, store: t |
| .sep_struct = mf.gliner_token_sep_struct, |
| .sep_text = mf.gliner_token_sep_text, |
| }, |
| - .inventory = .{ .count = reader.header.tensors.count(), .all_f32 = all_f32 }, |
| - .weight = gliner_decide_qualification.Digest.of(reader.file_bytes), |
| + .inventory = inventory, |
| + .weight = weight, |
| + .weight_companion = companion, |
| .sidecars = .{ |
| .config = config, |
| .encoder_config = encoder, |
| |
| |
| |
| |
| @@ -171,7 +171,11 @@ pub fn classificationLogitsProfiled( |
| cb.preferEagerQuantMirrors(true); |
| break :blk try deberta_arch.forwardCtProfiled(cb, allocator, deberta, sample.input_ids, attention_mask, 1, seq_len, deberta_mod.glinerPrefersWeightMirrors(deberta), if (profile) |p| &p.encoder else null); |
| }, |
| - .modern_bert => |modern| try modern_bert_arch.forwardCT(cb, allocator, modern, sample.input_ids, attention_mask, 1, seq_len), |
| + .modern_bert => |modern| blk: { |
| + var execution_config = modern; |
| + execution_config.metal_f16_weight_mirrors = true; |
| + break :blk try modern_bert_arch.forwardCT(cb, allocator, execution_config, sample.input_ids, attention_mask, 1, seq_len); |
| + }, |
| }; |
| defer cb.free(hidden); |
| if (profile) |p| { |
| |
| |
| |
| |
| @@ -15,6 +15,7 @@ |
| |
| const build_options = @import("build_options"); |
| pub const gliner_boundary_export = @import("gliner_boundary_export.zig"); |
| +pub const gliner_span_v2_executor = @import("extractors/gliner_span_v2_executor.zig"); |
| |
| pub const platform = @import("antfly_platform"); |
| pub const backends = @import("backends/backends.zig"); |
| @@ -119,6 +120,8 @@ pub const architectures = struct { |
| pub const gliner_head_graph = @import("architectures/gliner_head_graph.zig"); |
| }; |
| pub const models = struct { |
| + pub const gliner_decide_qualification = @import("models/gliner_decide_qualification.zig"); |
| + pub const tensor_store = @import("models/tensor_store.zig"); |
| pub const manifest = @import("models/manifest.zig"); |
| pub const gliner_boundary_bundle = @import("models/gliner_boundary_bundle.zig"); |
| pub const gliner_boundary = @import("models/gliner_boundary.zig"); |
| @@ -141,3 +144,8 @@ pub const gguf = struct { |
| pub const quant_codec = @import("gguf/quant_codec.zig"); |
| pub const tensor_types = @import("gguf/tensor_types.zig"); |
| }; |
| + |
| +test "GLiNER Decide artifact test reachability" { |
| + _ = models.gliner_decide_qualification; |
| + _ = models.tensor_store; |
| +} |
| |
| |
| |
| |
| @@ -1,5 +1,17 @@ |
| // Copyright 2026 Antfly, Inc. |
| // SPDX-License-Identifier: Apache-2.0 |
| +// |
| +// Licensed under the Apache License, Version 2.0 (the "License"); |
| +// you may not use this file except in compliance with the License. |
| +// You may obtain a copy of the License at |
| +// |
| +// http://www.apache.org/licenses/LICENSE-2.0 |
| +// |
| +// Unless required by applicable law or agreed to in writing, software |
| +// distributed under the License is distributed on an "AS IS" BASIS, |
| +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| +// See the License for the specific language governing permissions and |
| +// limitations under the License. |
| |
| //! Immutable loaded-artifact qualification for public GLiNER Decide serving. |
| //! Registry publication and request execution consume this same contract. An |
| @@ -52,6 +64,7 @@ pub const Identity = struct { |
| markers: Markers, |
| inventory: TensorInventory, |
| weight: Digest, |
| + weight_companion: ?Digest = null, |
| sidecars: Sidecars, |
| }; |
| |
| @@ -69,6 +82,8 @@ const legacy_tokenizer_config = FilePin{ .path = "tokenizer_config.json", .size_ |
| // capture. Runtime request admission remains bounded separately below. |
| pub const decide_1b_production_qualified = true; |
| const decide_1b_weight = FilePin{ .path = "model.safetensors", .size_bytes = 4_755_208_228, .sha256 = "02c567d791aed26550d300064c7f0c0094fd65291503c65969b45b30786e33b3" }; |
| +const decide_1b_q8_encoder = FilePin{ .path = "gliner2-encoder.Q8_0.gguf", .size_bytes = 1_118_356_576, .sha256 = "9ba4aea751bc8ce3586e70c4dfb3db93d0a850da5023b39d1bb68d21247f1694" }; |
| +const decide_1b_q8_head = FilePin{ .path = "gliner_head.gguf", .size_bytes = 617_195_872, .sha256 = "9a6c1a8ae0f58158404ea6875012e0c27e5923188906492ec7d948968741f558" }; |
| const decide_1b_config = FilePin{ .path = "config.json", .size_bytes = 464, .sha256 = "d2732928820b95649bc87f051345b2394fff87fc28c01f945610f943d6f402b2" }; |
| const decide_1b_encoder_config = FilePin{ .path = "encoder_config/config.json", .size_bytes = 2160, .sha256 = "c2cc4c15c7504b9e15651f2a32dff82b84fa9a99060a3218671a930da0a9b50d" }; |
| const decide_1b_tokenizer = FilePin{ .path = "tokenizer.json", .size_bytes = 3_585_055, .sha256 = "ddb379b6a4679ee16646bf0b726de9ec538d7c80ea9d4d4e33b069f19c9e1efb" }; |
| @@ -107,7 +122,7 @@ fn isLegacyDecide(identity: Identity) bool { |
| .sep_struct = 128001, |
| .sep_text = 128002, |
| }) and |
| - identity.inventory.count == 419 and identity.inventory.all_f32 and |
| + identity.inventory.count == 419 and identity.inventory.all_f32 and identity.weight_companion == null and |
| digestMatches(identity.weight, legacy_weight) and |
| sidecarsMatch(identity.sidecars, legacy_config, legacy_encoder_config, legacy_tokenizer, legacy_tokenizer_config, legacy_special_tokens); |
| } |
| @@ -131,8 +146,11 @@ fn isDecide1B(identity: Identity) bool { |
| .sep_struct = 50368, |
| .sep_text = 50369, |
| }) and |
| - identity.inventory.count == 199 and identity.inventory.all_f32 and |
| - digestMatches(identity.weight, decide_1b_weight) and |
| + identity.inventory.count == 199 and |
| + (if (identity.weight_companion) |head| |
| + !identity.inventory.all_f32 and digestMatches(identity.weight, decide_1b_q8_encoder) and digestMatches(head, decide_1b_q8_head) |
| + else |
| + identity.inventory.all_f32 and digestMatches(identity.weight, decide_1b_weight)) and |
| sidecarsMatch(identity.sidecars, decide_1b_config, decide_1b_encoder_config, decide_1b_tokenizer, decide_1b_tokenizer_config, null); |
| } |
| |
| @@ -221,6 +239,32 @@ test "GLiNER Decide 1B qualification binds exact loaded identity" { |
| identity.inventory.all_f32 = false; |
| try std.testing.expect(!isDecide1B(identity)); |
| try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); |
| + |
| + identity.weight = digestFromPin(decide_1b_q8_encoder); |
| + identity.weight_companion = digestFromPin(decide_1b_q8_head); |
| + try std.testing.expectEqual(QualifiedVariant.decide_1b, try require(identity)); |
| + const quantized = identity; |
| + identity.weight_companion = null; |
| + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); |
| + identity = quantized; |
| + identity.weight_companion.?.sha256[0] ^= 1; |
| + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); |
| + identity = quantized; |
| + identity.weight.sha256[0] ^= 1; |
| + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); |
| + identity = quantized; |
| + identity.sidecars.tokenizer.sha256[0] ^= 1; |
| + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); |
| + identity = quantized; |
| + identity.inventory.count -= 1; |
| + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); |
| + identity = quantized; |
| + identity.inventory.all_f32 = true; |
| + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); |
| + identity = quantized; |
| + identity.weight = digestFromPin(decide_1b_weight); |
| + identity.inventory.all_f32 = true; |
| + try std.testing.expectError(error.UnsupportedGlinerDecisionArtifact, require(identity)); |
| } |
| |
| test "GLiNER Decide 1B serving policy stays within measured backend and geometry" { |
| |
| |
| |
| |
| @@ -168,6 +168,39 @@ pub const TensorStore = struct { |
| return store.rawData(); |
| } |
| |
| + pub const GlinerGgufSnapshot = struct { |
| + encoder: []const u8, |
| + head: []const u8, |
| + tensor_count: usize, |
| + all_f32: bool, |
| + }; |
| + |
| + /// Borrow both exact open GGUF artifacts. Never reopen a pathname to |
| + /// establish the identity of an already loaded split GLiNER session. |
| + pub fn glinerGgufSnapshot(self: TensorStore) ?GlinerGgufSnapshot { |
| + if (self.vtable != &CompositeGlinerStore.vtable) return null; |
| + const store: *const CompositeGlinerStore = @ptrCast(@alignCast(self.ptr)); |
| + const head = store.head_gguf orelse return null; |
| + if (store.head_safetensors != null) return null; |
| + var all_f32 = true; |
| + for (store.encoder.parsed.tensors) |tensor| { |
| + if (tensor.tensor_type != .known or tensor.tensor_type.known != .F32) all_f32 = false; |
| + // Reject shadowing across the encoder and auxiliary head. |
| + for (head.parsed.tensors) |other| { |
| + if (std.mem.eql(u8, tensor.name, other.name)) return null; |
| + } |
| + } |
| + for (head.parsed.tensors) |tensor| { |
| + if (tensor.tensor_type != .known or tensor.tensor_type.known != .F32) all_f32 = false; |
| + } |
| + return .{ |
| + .encoder = store.encoder.rawData(), |
| + .head = head.rawData(), |
| + .tensor_count = store.encoder.parsed.tensors.len + head.parsed.tensors.len, |
| + .all_f32 = all_f32, |
| + }; |
| + } |
| + |
| pub fn singleSafetensorsReader(self: TensorStore) ?*const @import("safetensors.zig").MMapReader { |
| if (self.vtable != &SafetensorsStore.vtable) return null; |
| const store: *const SafetensorsStore = @ptrCast(@alignCast(self.ptr)); |
| @@ -1522,6 +1555,7 @@ test "open gguf tensor store from manifest" { |
| defer store.deinit(); |
| try std.testing.expectEqual(StoreKind.gguf, store.kind()); |
| try std.testing.expect(store.ggufFile() != null); |
| + |
| try std.testing.expect((try store.weightSource()) != null); |
| var tensor_ref = try store.describeTensor(allocator, "tok_embeddings.weight"); |
| try std.testing.expectEqual(@as(usize, 8), tensor_ref.byte_len); |
| @@ -1739,6 +1773,20 @@ test "open split gliner gguf bundle from manifest" { |
| try std.testing.expectEqual(StoreKind.gguf, store.kind()); |
| try std.testing.expect(store.ggufFile() != null); |
| |
| + const snapshot = store.glinerGgufSnapshot() orelse return error.TestUnexpectedResult; |
| + try std.testing.expectEqualSlices(u8, gguf_data.items, snapshot.encoder); |
| + try std.testing.expectEqualSlices(u8, head_data.items, snapshot.head); |
| + try std.testing.expectEqual(@as(usize, 3), snapshot.tensor_count); |
| + try std.testing.expect(!snapshot.all_f32); |
| + try std.testing.expect(store.ggufArtifactBytes() == null); |
| + |
| + const composite: *CompositeGlinerStore = @ptrCast(@alignCast(store.ptr)); |
| + const original_name = composite.head_gguf.?.parsed.tensors[0].name; |
| + composite.head_gguf.?.parsed.tensors[0].name = composite.encoder.parsed.tensors[0].name; |
| + const shadowed = store.glinerGgufSnapshot(); |
| + composite.head_gguf.?.parsed.tensors[0].name = original_name; |
| + try std.testing.expect(shadowed == null); |
| + |
| const source = (try store.weightSource()) orelse return error.TestUnexpectedResult; |
| const names = try source.listNames(allocator); |
| defer allocator.free(names); |
| |
| new file mode 100644 |
| |
| |
| |
| @@ -0,0 +1,128 @@ |
| +// Copyright 2026 Antfly, Inc. |
| +// SPDX-License-Identifier: Apache-2.0 |
| +// |
| +// Licensed under the Apache License, Version 2.0 (the "License"); |
| +// you may not use this file except in compliance with the License. |
| +// You may obtain a copy of the License at |
| +// |
| +// http://www.apache.org/licenses/LICENSE-2.0 |
| +// |
| +// Unless required by applicable law or agreed to in writing, software |
| +// distributed under the License is distributed on an "AS IS" BASIS, |
| +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| +// See the License for the specific language governing permissions and |
| +// limitations under the License. |
| + |
| +//! Offline artifact comparison. Never grants public serving qualification. |
| +const std = @import("std"); |
| +const inference = @import("inference_internal"); |
| +const factory = inference.architectures.session_factory; |
| +const processor = inference.pipelines.gliner_boundary_processor; |
| +const schema = inference.pipelines.extraction_schema; |
| +const executor = inference.gliner_span_v2_executor; |
| +const Case = struct { |
| + id: []const u8, |
| + text: []const u8, |
| + schema_json: []const u8, |
| + input_ids: []const i64, |
| + logits: []const []const f64, |
| +}; |
| +const Report = struct { id: []const u8, tokens: usize, median_ms: f64, p95_ms: f64, max_logit_error: f64, logits: [][]f64, samples_ms: []f64 }; |
| + |
| +pub fn main(init: std.process.Init) !void { |
| + const a = inference.platform.allocator.processAllocator(std.heap.smp_allocator); |
| + var args = std.process.Args.Iterator.init(init.minimal.args); |
| + _ = args.next(); |
| + var path: ?[]const u8 = null; |
| + var capture: ?[]const u8 = null; |
| + var backend: []const u8 = "native"; |
| + var warmups: usize = 3; |
| + var reps: usize = 20; |
| + var tolerance: f64 = 0.05; |
| + while (args.next()) |key| { |
| + const value = args.next() orelse return error.MissingArgument; |
| + if (std.mem.eql(u8, key, "--model-dir")) path = value else if (std.mem.eql(u8, key, "--capture")) capture = value else if (std.mem.eql(u8, key, "--backend")) backend = value else if (std.mem.eql(u8, key, "--warmups")) warmups = try std.fmt.parseInt(usize, value, 10) else if (std.mem.eql(u8, key, "--reps")) reps = try std.fmt.parseInt(usize, value, 10) else if (std.mem.eql(u8, key, "--tolerance")) tolerance = try std.fmt.parseFloat(f64, value) else return error.InvalidArgument; |
| + } |
| + if (reps < 3 or reps > 1000 or warmups > 100 or !std.math.isFinite(tolerance) or tolerance <= 0) return error.InvalidArgument; |
| + const bytes = try inference.util.c_file.readFile(a, capture orelse return error.MissingArgument); |
| + defer a.free(bytes); |
| + const cases = try std.json.parseFromSlice([]const Case, a, bytes, .{}); |
| + defer cases.deinit(); |
| + const directory = path orelse return error.MissingArgument; |
| + const start_load = inference.platform.time.monotonicNs(); |
| + const session = if (std.mem.eql(u8, backend, "metal")) try factory.createMetalSession(a, directory) else if (std.mem.eql(u8, backend, "native")) try factory.createNativeSession(a, directory) else return error.InvalidBackend; |
| + defer session.close(); |
| + if (std.mem.eql(u8, backend, "metal") and session.backend() != .metal) return error.InvalidBackend; |
| + const config = try factory.getGlinerSpanConfig(session); |
| + if (config != .modern_bert) return error.InvalidModel; |
| + const tokenizer_path = try std.fs.path.join(a, &.{ directory, "tokenizer.json" }); |
| + defer a.free(tokenizer_path); |
| + const tok_bytes = try inference.util.c_file.readFile(a, tokenizer_path); |
| + defer a.free(tok_bytes); |
| + const tok = try inference.hf_tokenizer.HfTokenizer.loadFromBytes(a, tok_bytes); |
| + defer tok.tokenizer().deinitTokenizer(); |
| + const load_ms = @as(f64, @floatFromInt(inference.platform.time.monotonicNs() - start_load)) / std.time.ns_per_ms; |
| + const watchdog = if (std.mem.eql(u8, backend, "metal")) try inference.HardCancellationWatchdog.create(a) else null; |
| + defer if (watchdog) |owner| owner.destroy(); |
| + if (watchdog) |owner| try owner.start(init.io); |
| + var report_arena = std.heap.ArenaAllocator.init(a); |
| + defer report_arena.deinit(); |
| + const out = report_arena.allocator(); |
| + const reports = try out.alloc(Report, cases.value.len); |
| + for (cases.value, reports) |case, *report| { |
| + const samples = try out.alloc(f64, reps); |
| + var max_error: f64 = 0; |
| + var last_rows: [][]f64 = &.{}; |
| + for (0..1 + warmups + reps) |iteration| { |
| + var request_arena = std.heap.ArenaAllocator.init(a); |
| + defer request_arena.deinit(); |
| + const ra = request_arena.allocator(); |
| + const control = inference.InferenceExecutionControl{ |
| + .hard_cancellation = if (watchdog) |owner| owner.boundary() else null, |
| + .deadline_ns = inference.platform.time.monotonicNs() + 120 * std.time.ns_per_s, |
| + }; |
| + const start = inference.platform.time.monotonicNs(); |
| + var compiled = try schema.compile(ra, case.schema_json, .{}); |
| + defer compiled.deinit(); |
| + var prepared = try processor.prepare(ra, tok.tokenizer(), &.{.{ .text = case.text, .schema = &compiled }}, .{ .max_sequence_tokens = 7999, .max_batch_tokens = 7999 }); |
| + defer prepared.deinit(); |
| + if (prepared.samples.len != 1) return error.InvalidCase; |
| + const counts = try ra.alloc(usize, compiled.schema.classifications.len); |
| + for (compiled.schema.classifications, counts) |classification, *count| count.* = classification.task.labels.len; |
| + var managed = try factory.getManagedComputeBackend(session, ra, null, control); |
| + defer managed.deinit(); |
| + const rows = try executor.classificationLogits(&managed.backend, ra, .{ .modern_bert = config.modern_bert }, prepared.samples[0], counts); |
| + const elapsed = @as(f64, @floatFromInt(inference.platform.time.monotonicNs() - start)) / std.time.ns_per_ms; |
| + if (!std.mem.eql(i64, prepared.input_ids, case.input_ids) or rows.len != case.logits.len) return error.PreprocessingMismatch; |
| + for (rows, case.logits) |got, expected| { |
| + if (got.len != expected.len) return error.LogitShapeMismatch; |
| + for (got, expected) |actual, want| { |
| + if (!std.math.isFinite(actual)) return error.NonFiniteLogit; |
| + const delta = @abs(actual - want); |
| + max_error = @max(max_error, delta); |
| + if (delta > tolerance) { |
| + std.debug.print("{s}: logit error {d} exceeds {d}\n", .{ case.id, delta, tolerance }); |
| + return error.LogitMismatch; |
| + } |
| + } |
| + } |
| + if (iteration > warmups) samples[iteration - warmups - 1] = elapsed; |
| + if (iteration == warmups + reps) { |
| + last_rows = try out.alloc([]f64, rows.len); |
| + for (rows, last_rows) |row, *saved| saved.* = try out.dupe(f64, row); |
| + } |
| + } |
| + const sorted = try out.dupe(f64, samples); |
| + std.mem.sort(f64, sorted, {}, struct { |
| + fn less(_: void, x: f64, y: f64) bool { |
| + return x < y; |
| + } |
| + }.less); |
| + report.* = .{ .id = case.id, .tokens = case.input_ids.len, .median_ms = if (reps % 2 == 0) (sorted[reps / 2 - 1] + sorted[reps / 2]) / 2 else sorted[reps / 2], .p95_ms = sorted[@min(reps - 1, (reps * 95 + 99) / 100 - 1)], .max_logit_error = max_error, .logits = last_rows, .samples_ms = samples }; |
| + std.debug.print("{s} {s}: {d:.3} ms, error {d:.6}\n", .{ backend, case.id, report.median_ms, max_error }); |
| + } |
| + const result = try std.json.Stringify.valueAlloc(a, .{ .scope = "offline loaded pipeline: schema compile, tokenization, encoder, classifier and completed readback; excludes HTTP, model load, and output presentation", .backend = backend, .model_dir = directory, .load_ms = load_ms, .warmups = warmups, .reps = reps, .validation_preflights = 1, .tolerance = tolerance, .reports = reports }, .{}); |
| + defer a.free(result); |
| + try std.Io.File.stdout().writeStreamingAll(init.io, result); |
| + try std.Io.File.stdout().writeStreamingAll(init.io, "\n"); |
| +} |
|
|
