Add validated embedding W8 classification option

README.md

@@ -41,4 +41,6 @@ The feature graph compute plan assigned 53.13% of operations to ANE and 46.87% t
 
 The original L128/K8 classification packages remain available, with a separate `runtime.py` entry point. Up to eight labels fit that bucket. 100/100 selected choices matched in FP16. See the classification report JSONs for the exact selected samples and limits. The full checkpoint's task and dataset scores have not been reproduced here.
 
+An optional **embedding-only W8** classifier keeps the encoder and heads FP16. It reduces the package from **388,981,604 to 291,053,994 bytes** (25.2%). On an M5 Pro plugged into AC, it agreed with FP16 on all 14 bucket-fitting requests in a fixed 20-request manifest (largest probability difference 0.00992), and matched the native classifier on 100/100 selected eligible application requests (largest confidence difference 0.01370). The 14-request paired full-request p50 was 3.662 ms FP16 versus 3.632 ms W8 with automatic device selection; this is a size option, not a demonstrated speedup. Forced CPU+ANE was slower (9.168 versus 9.666 ms). The W8 compute plan put 496/516 executable ops on ANE under forced CPU+ANE, with 20 CPU boundary/constant ops. Use `runtime.py --precision embedding_w8` for this classification package; extraction continues to use its separate FP32 or FP16 packages.
+
 The Core ML deployment target is iOS 17/macOS 14. Conversion scripts, pinned dependencies, asset hashes, and selected verification reports are included. The source revision is a current pinned snapshot; identity with the historical Decision Index evaluation checkpoint has not been established.

build/gliner2_base_classification_embedding_w8_L128_K8.mlpackage/Data/com.apple.CoreML/model.mlmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11c56d5d686afb3812181024125c8102756d616230606e849a0c97ebd9b34dd9
+size 990593

build/gliner2_base_classification_embedding_w8_L128_K8.mlpackage/Data/com.apple.CoreML/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:037f91cb176d31a38bc1795272d99c47531f3ac235b6f4533bab0df9ae4de327
+size 290062784

build/gliner2_base_classification_embedding_w8_L128_K8.mlpackage/Manifest.json

@@ -0,0 +1,18 @@
+{
+    "fileFormatVersion": "1.0.0",
+    "itemInfoEntries": {
+        "1D1E4BE6-7753-4709-9319-081055B47FD9": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Specification",
+            "name": "model.mlmodel",
+            "path": "com.apple.CoreML/model.mlmodel"
+        },
+        "8F904B6A-73EB-47FB-B055-2D2B3240F1A0": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Weights",
+            "name": "weights",
+            "path": "com.apple.CoreML/weights"
+        }
+    },
+    "rootModelIdentifier": "1D1E4BE6-7753-4709-9319-081055B47FD9"
+}

compress-coreml.py

@@ -5,7 +5,14 @@ import time
 from pathlib import Path
 
 import coremltools as ct
-from coremltools.optimize.coreml import OpPalettizerConfig, OptimizationConfig, palettize_weights
+from coremltools.optimize.coreml import (
+    OpLinearQuantizerConfig,
+    OpPalettizerConfig,
+    OptimizationConfig,
+    get_weights_metadata,
+    linear_quantize_weights,
+    palettize_weights,
+)
 
 
 def package_bytes(path):
@@ -17,25 +24,58 @@ def main():
     parser.add_argument("--source", required=True)
     parser.add_argument("--output", required=True)
     parser.add_argument("--bits", type=int, choices=[4, 6, 8], default=8)
+    parser.add_argument("--method", choices=["lut", "linear", "embedding"], default="lut")
+    parser.add_argument("--linear-mode", choices=["linear_symmetric", "linear"], default="linear_symmetric")
     parser.add_argument("--granularity", choices=["per_tensor", "per_grouped_channel"], default="per_tensor")
     parser.add_argument("--group-size", type=int, default=32)
     args = parser.parse_args()
     start = time.perf_counter()
     model = ct.models.MLModel(args.source, skip_model_load=True)
-    config = OptimizationConfig(global_config=OpPalettizerConfig(
-        mode="kmeans", nbits=args.bits, granularity=args.granularity,
-        group_size=args.group_size,
-        enable_per_channel_scale=args.granularity == "per_grouped_channel",
-        num_kmeans_workers=4,
-    ))
-    compressed = palettize_weights(model, config=config)
-    compressed.short_description = f"{model.short_description}; LUT{args.bits} weights"
+    selected_weights = None
+    if args.method in ("linear", "embedding"):
+        if args.bits != 8:
+            parser.error("linear quantization here supports only 8-bit weights")
+        quantizer = OpLinearQuantizerConfig(
+            mode=args.linear_mode, dtype="int8", granularity="per_channel", weight_threshold=2048,
+        )
+        if args.method == "embedding":
+            metadata = get_weights_metadata(model, weight_threshold=2048)
+            names = [
+                name for name, weight in metadata.items()
+                if "encoder_embeddings_word_embeddings" in name
+                and len(weight.val.shape) == 2
+                and any(op.op_type == "gather" for op in weight.child_ops)
+            ]
+            if len(names) != 1:
+                raise ValueError(f"expected one token embedding weight, found {names}")
+            config = OptimizationConfig(op_name_configs={names[0]: quantizer})
+            selected_weights = names
+        else:
+            config = OptimizationConfig(global_config=quantizer)
+        compressed = linear_quantize_weights(model, config=config)
+        variant = (
+            f"W8 {args.linear_mode} token embedding only"
+            if args.method == "embedding" else f"W8 {args.linear_mode} per-channel"
+        )
+    else:
+        config = OptimizationConfig(global_config=OpPalettizerConfig(
+            mode="kmeans", nbits=args.bits, granularity=args.granularity,
+            group_size=args.group_size,
+            enable_per_channel_scale=args.granularity == "per_grouped_channel",
+            num_kmeans_workers=4,
+        ))
+        compressed = palettize_weights(model, config=config)
+        variant = f"LUT{args.bits} {args.granularity}"
+    compressed.short_description = f"{model.short_description}; {variant} weights"
     compressed.author = model.author
     compressed.license = model.license
     compressed.user_defined_metadata.update(model.user_defined_metadata)
-    compressed.user_defined_metadata["weight_compression"] = f"LUT{args.bits} kmeans"
+    compressed.user_defined_metadata["weight_compression"] = variant
     compressed.save(args.output)
-    report = {"source": args.source, "output": args.output, "granularity": args.granularity,
+    report = {"source": args.source, "output": args.output, "method": args.method,
+              "linear_mode": args.linear_mode if args.method in ("linear", "embedding") else None,
+              "granularity": "per_channel" if args.method in ("linear", "embedding") else args.granularity,
+              "selected_weights": selected_weights,
               "source_bytes": package_bytes(Path(args.source)),
               "output_bytes": package_bytes(Path(args.output)), "compression_seconds": time.perf_counter() - start}
     Path(args.output).with_suffix(".json").write_text(json.dumps(report, indent=2) + "\n")

reports/classification-embedding-w8.json

@@ -0,0 +1,149 @@
+{
+  "source_revision": "1a8bc24e00dc7300b9017c81d63e3dcdabb26596",
+  "shape": "L128/K8 classification",
+  "candidate_package": "gliner2_base_classification_embedding_w8_L128_K8.mlpackage",
+  "weight_sha256": "037f91cb176d31a38bc1795272d99c47531f3ac235b6f4533bab0df9ae4de327",
+  "compression": {
+    "source": "build/gliner2_base_classification_fp16_L128_K8.mlpackage",
+    "output": "build/gliner2_base_classification_embedding_w8_L128_K8.mlpackage",
+    "method": "embedding",
+    "granularity": "per_channel",
+    "selected_weights": [
+      "encoder_embeddings_word_embeddings_weight_to_fp16"
+    ],
+    "source_bytes": 388981604,
+    "output_bytes": 291053994,
+    "compression_seconds": 5.819930917001329
+  },
+  "paired_first_manifest_all": {
+    "reference": "build/gliner2_base_classification_fp16_L128_K8.mlpackage",
+    "candidate": "build/gliner2_base_classification_embedding_w8_L128_K8.mlpackage",
+    "manifest": "../../gliner2-small/coreml/fixtures/optimization-selected20.jsonl",
+    "compute_units": "all",
+    "processor": "arm",
+    "machine": "arm64",
+    "macos": "27.0",
+    "power_audit_after_run": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=35651683)\t50%; charging; 1:25 remaining present: true",
+    "reference_load_ms": 2044.181875011418,
+    "candidate_load_ms": 4582.357916980982,
+    "warmup_calls_per_model": 1,
+    "timed_order": "alternating reference-first/candidate-first by request",
+    "checked": 14,
+    "choice_agreement": 14,
+    "max_probability_error": 0.00992196798324585,
+    "probability_error_limit": 0.02,
+    "reference_p50_ms": 3.334374981932342,
+    "candidate_p50_ms": 3.261812496930361,
+    "reference_p95_ms": 9.934750036336482,
+    "candidate_p95_ms": 5.232541996520013,
+    "reference_total_p50_ms": 3.6611664691008627,
+    "candidate_total_p50_ms": 3.6319584760349244
+  },
+  "paired_first_manifest_cpu_ane": {
+    "reference": "build/gliner2_base_classification_fp16_L128_K8.mlpackage",
+    "candidate": "build/gliner2_base_classification_embedding_w8_L128_K8.mlpackage",
+    "manifest": "../../gliner2-small/coreml/fixtures/optimization-selected20.jsonl",
+    "compute_units": "cpu-ane",
+    "processor": "arm",
+    "machine": "arm64",
+    "macos": "27.0",
+    "power_audit_after_run": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=35651683)\t51%; charging; 1:33 remaining present: true",
+    "reference_load_ms": 3068.8863749965094,
+    "candidate_load_ms": 5926.175208995119,
+    "warmup_calls_per_model": 1,
+    "timed_order": "alternating reference-first/candidate-first by request",
+    "checked": 14,
+    "choice_agreement": 14,
+    "max_probability_error": 0.0068359375,
+    "probability_error_limit": 0.02,
+    "reference_p50_ms": 7.578062970424071,
+    "candidate_p50_ms": 7.721708010649309,
+    "reference_p95_ms": 8.475541020743549,
+    "candidate_p95_ms": 9.883625025395304,
+    "reference_total_p50_ms": 9.168104501441121,
+    "candidate_total_p50_ms": 9.666354482760653
+  },
+  "native_selected100": {
+    "model": "fastino/gliner2.5-base-v1",
+    "revision": "1a8bc24e00dc7300b9017c81d63e3dcdabb26596",
+    "package": "build/gliner2_base_classification_embedding_w8_L128_K8.mlpackage",
+    "selected_manifest": "first eligible rows in source suite order; no gold labels used",
+    "counts": {
+      "checked": 100,
+      "too_many_options": 300,
+      "too_long": 7,
+      "duplicate_labels": 0,
+      "mismatches": 0
+    },
+    "maximum_confidence_error": 0.013702988624572754,
+    "mean_confidence_error": 0.001015886664390564,
+    "model_call_p50_ms": 10.512874985579401,
+    "model_call_p95_ms": 14.00475000264123,
+    "failures": []
+  },
+  "compute_plan_forced_cpu_ane": {
+    "total_ops": 516,
+    "ane_ops": 496,
+    "cpu_ops": 20,
+    "ane_percent": 96.1,
+    "reasons": [
+      {
+        "reason": "Unsupported tensor data type: int32",
+        "count": 17,
+        "estimated_cpu_runtime_ms": 2.8975,
+        "op_types": {
+          "ios17.expand_dims": 5,
+          "ios17.cast": 5,
+          "ios17.add": 2,
+          "select": 2,
+          "ios17.gather": 1,
+          "ios17.mul": 1,
+          "tile": 1
+        },
+        "ops": [
+          "add_0",
+          "select_0",
+          "add_0_1",
+          "select_0_1",
+          "inputs_embeds_cast_fp16",
+          "mask_3",
+          "mask_3_to_fp16",
+          "var_86",
+          "extended_attention_mask",
+          "extended_attention_mask_to_uint16",
+          "var_88_cast_int16_to_int32",
+          "var_89",
+          "attention_mask_1",
+          "var_272",
+          "var_2537",
+          "indices",
+          "indices_to_int16"
+        ]
+      },
+      {
+        "reason": "Unable to resolve operation input \"y\".",
+        "count": 2,
+        "estimated_cpu_runtime_ms": 0.002,
+        "op_types": {
+          "ios17.greater_equal": 2
+        },
+        "ops": [
+          "greater_equal_0",
+          "greater_equal_0_1"
+        ]
+      },
+      {
+        "reason": "ANE not available for this op",
+        "count": 1,
+        "estimated_cpu_runtime_ms": 0,
+        "op_types": {
+          "ios16.constexpr_affine_dequantize": 1
+        },
+        "ops": [
+          "encoder_embeddings_word_embeddings_weight_to_fp16_quantized"
+        ]
+      }
+    ]
+  },
+  "scope": "Selected real classification requests only; no Decision Index or 2048 score; extraction packages are separate."
+}

runtime.py

@@ -9,9 +9,12 @@ import numpy as np
 from preprocessing import load_processor, prepare_with_processor
 
 
-def classify(model_dir: str, text: str, task: str, labels: list[str], length: int = 128, max_options: int = 8):
+def classify(model_dir: str, text: str, task: str, labels: list[str], length: int = 128, max_options: int = 8,
+             precision: str = "fp16"):
     model_dir = Path(model_dir)
-    package = model_dir / f"gliner2_base_classification_fp16_L{length}_K{max_options}.mlpackage"
+    if precision not in ("fp16", "embedding_w8"):
+        raise ValueError(f"unsupported classification precision: {precision}")
+    package = model_dir / f"gliner2_base_classification_{precision}_L{length}_K{max_options}.mlpackage"
     processor = load_processor(str(model_dir / "tokenizer"))
     arrays = prepare_with_processor(processor, text, task, labels, length, max_options)
     model = ct.models.MLModel(str(package), compute_units=ct.ComputeUnit.ALL)
@@ -28,9 +31,10 @@ def main():
     parser.add_argument("--labels", required=True, help="JSON list of label strings")
     parser.add_argument("--length", type=int, default=128)
     parser.add_argument("--max-options", type=int, default=8)
+    parser.add_argument("--precision", choices=["fp16", "embedding_w8"], default="fp16")
     args = parser.parse_args()
     print(json.dumps(classify(args.model_dir, args.text, args.task, json.loads(args.labels),
-                              args.length, args.max_options), indent=2))
+                              args.length, args.max_options, args.precision), indent=2))
 
 
 if __name__ == "__main__":

verify-compression.py

@@ -0,0 +1,135 @@
+"""Compare two GLiNER2 Core ML packages on selected real classification requests."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import platform
+import statistics
+import subprocess
+import time
+from pathlib import Path
+
+import coremltools as ct
+import numpy as np
+
+from preprocessing import load_processor, prepare_with_processor
+
+
+def percentile(values: list[float], p: float) -> float | None:
+    if not values:
+        return None
+    return sorted(values)[max(0, min(len(values) - 1, math.ceil(p * len(values)) - 1))]
+
+
+def median(cases: list[dict], field: str) -> float | None:
+    return statistics.median(row[field] for row in cases) if cases else None
+
+
+def run() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--reference", type=Path, required=True)
+    parser.add_argument("--candidate", type=Path, required=True)
+    parser.add_argument("--tokenizer", type=Path, required=True)
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--compute-units", choices=["all", "cpu-ane"], default="all")
+    parser.add_argument("--length", type=int, default=128)
+    parser.add_argument("--max-options", type=int, default=8)
+    parser.add_argument("--limit", type=int, default=None, help="stop after this many eligible requests")
+    parser.add_argument("--max-probability-error", type=float, default=0.02)
+    args = parser.parse_args()
+
+    units = ct.ComputeUnit.ALL if args.compute_units == "all" else ct.ComputeUnit.CPU_AND_NE
+    processor = load_processor(str(args.tokenizer))
+    load_start = time.perf_counter()
+    reference = ct.models.MLModel(str(args.reference), compute_units=units)
+    reference_load_ms = (time.perf_counter() - load_start) * 1000
+    load_start = time.perf_counter()
+    candidate = ct.models.MLModel(str(args.candidate), compute_units=units)
+    candidate_load_ms = (time.perf_counter() - load_start) * 1000
+    rows = [json.loads(line) for line in args.manifest.read_text().splitlines() if line]
+    prepared = []
+    skipped = []
+    for row in rows:
+        if args.limit is not None and len(prepared) >= args.limit:
+            break
+        labels = [(description or key).strip() for key, description in row["options"]]
+        prepare_start = time.perf_counter()
+        try:
+            arrays = prepare_with_processor(processor, row["state"], "decision", labels, args.length, args.max_options)
+        except ValueError as error:
+            skipped.append({"suite": row["suite"], "index": row["index"], "reason": str(error)})
+            continue
+        prepare_ms = (time.perf_counter() - prepare_start) * 1000
+
+        prepared.append((row, labels, arrays, prepare_ms))
+
+    if prepared:
+        for model in (reference, candidate):
+            model.predict(prepared[0][2])
+
+    checked = []
+    for index, (row, labels, arrays, prepare_ms) in enumerate(prepared):
+        outputs = {}
+        order = (("reference", reference), ("candidate", candidate))
+        if index % 2:
+            order = tuple(reversed(order))
+        for name, model in order:
+            start = time.perf_counter()
+            probabilities = np.asarray(model.predict(arrays)["probabilities"])[0, : len(labels)]
+            outputs[name] = (probabilities, (time.perf_counter() - start) * 1000)
+        expected, reference_ms = outputs["reference"]
+        actual, candidate_ms = outputs["candidate"]
+        checked.append({
+            "suite": row["suite"],
+            "index": row["index"],
+            "choice_agrees": int(expected.argmax()) == int(actual.argmax()),
+            "max_probability_error": float(np.max(np.abs(expected - actual))),
+            "prepare_ms": prepare_ms,
+            "reference_ms": reference_ms,
+            "candidate_ms": candidate_ms,
+            "reference_total_ms": prepare_ms + reference_ms,
+            "candidate_total_ms": prepare_ms + candidate_ms,
+        })
+
+    report = {
+        "reference": str(args.reference),
+        "candidate": str(args.candidate),
+        "manifest": str(args.manifest),
+        "compute_units": args.compute_units,
+        "processor": platform.processor(),
+        "machine": platform.machine(),
+        "macos": platform.mac_ver()[0],
+        "power_audit_after_run": subprocess.run(["pmset", "-g", "batt"], capture_output=True, text=True).stdout.strip(),
+        "reference_load_ms": reference_load_ms,
+        "candidate_load_ms": candidate_load_ms,
+        "warmup_calls_per_model": 1 if prepared else 0,
+        "timed_order": "alternating reference-first/candidate-first by request",
+        "checked": len(checked),
+        "skipped": skipped,
+        "choice_agreement": sum(row["choice_agrees"] for row in checked),
+        "max_probability_error": max((row["max_probability_error"] for row in checked), default=None),
+        "probability_error_limit": args.max_probability_error,
+        "reference_p50_ms": median(checked, "reference_ms"),
+        "candidate_p50_ms": median(checked, "candidate_ms"),
+        "reference_p95_ms": percentile([row["reference_ms"] for row in checked], 0.95),
+        "candidate_p95_ms": percentile([row["candidate_ms"] for row in checked], 0.95),
+        "reference_total_p50_ms": median(checked, "reference_total_ms"),
+        "candidate_total_p50_ms": median(checked, "candidate_total_ms"),
+        "cases": checked,
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(report, indent=2) + "\n")
+    summary = {key: value for key, value in report.items() if key not in ("cases", "skipped")}
+    summary["skipped_count"] = len(skipped)
+    print(json.dumps(summary, indent=2))
+    if not checked or report["choice_agreement"] != len(checked):
+        raise SystemExit("candidate changed a checked decision")
+    if report["max_probability_error"] > args.max_probability_error:
+        raise SystemExit("candidate exceeded the selected-request probability error limit")
+
+
+if __name__ == "__main__":
+    run()