| """Selected end-to-end native/Core ML checks across GLiNER2 schema paths.""" |
|
|
| import argparse |
| import json |
| from pathlib import Path |
|
|
| import torch |
| from gliner2 import AttributeGroup, AutoExtractor, Schema |
| from huggingface_hub import snapshot_download |
|
|
| from extraction_runtime import CoreMLBoundaryExtractor |
|
|
| MODEL_ID = "fastino/gliner2.5-base-v1" |
| MODEL_REVISION = "1a8bc24e00dc7300b9017c81d63e3dcdabb26596" |
|
|
|
|
| def structure(mode): |
| schema = Schema() |
| builder = schema.structure("employment", mode=mode, anchor="person" if mode == "natural" else None) |
| builder.field("person", dtype="str") |
| builder.field("company", dtype="str") |
| return schema |
|
|
|
|
| def fixtures(): |
| attribute = Schema().entities(["person", "organization"]) |
| attribute.entity_attributes({"role": AttributeGroup(labels=["founder", "employee"])}) |
| mixed = Schema().entities(["person", "organization"]) |
| mixed.classification("sentiment", ["positive", "negative"]) |
| choice = Schema() |
| choice.structure("product").field("category", dtype="str", choices=["electronics", "clothing"]) |
| return [ |
| ( |
| "entities", |
| "Alice founded Acme in Toronto in 2020.", |
| Schema().entities(["person", "organization", "location"]), |
| ), |
| ( |
| "entities_multi", |
| "Marie Curie was born in Warsaw and worked in Paris.", |
| Schema().entities(["person", "city"]), |
| ), |
| ("relations", "Alice founded Acme in Toronto.", Schema().relations(["founded"])), |
| ("relations_two", "Steve Jobs founded Apple.", Schema().relations(["founded"])), |
| ("record_natural", "Alice works at Acme. Bob works at Beta.", structure("natural")), |
| ("record_latent", "Alice works at Acme. Bob works at Beta.", structure("latent")), |
| ("record_anchorless", "Alice works at Acme. Bob works at Beta.", structure("anchorless")), |
| ("attributes", "Alice founded Acme.", attribute), |
| ("mixed_classification", "Alice founded Acme.", mixed), |
| ( |
| "classification_only", |
| "The rocket launched successfully.", |
| Schema().classification("topic", ["science", "sports", "politics"]), |
| ), |
| ("enum_choice", "The item is electronics.", choice), |
| ] |
|
|
|
|
| def without_confidence(value): |
| if isinstance(value, dict): |
| return {key: without_confidence(item) for key, item in value.items() if key != "confidence"} |
| if isinstance(value, list): |
| return [without_confidence(item) for item in value] |
| return value |
|
|
|
|
| def confidence_errors(reference, actual): |
| if isinstance(reference, dict) and isinstance(actual, dict): |
| errors = [] |
| for key, value in reference.items(): |
| if key == "confidence" and key in actual: |
| errors.append(abs(float(value) - float(actual[key]))) |
| elif key in actual: |
| errors.extend(confidence_errors(value, actual[key])) |
| return errors |
| if isinstance(reference, list) and isinstance(actual, list): |
| return [error for a, b in zip(reference, actual) for error in confidence_errors(a, b)] |
| return [] |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--model-dir", required=True) |
| parser.add_argument("--precision", choices=["fp16", "fp32"], default="fp32") |
| parser.add_argument("--allow-mismatch", action="store_true") |
| args = parser.parse_args() |
| torch.set_num_threads(4) |
| source = snapshot_download(MODEL_ID, revision=MODEL_REVISION) |
| native = AutoExtractor.from_pretrained(str(source), map_location="cpu").eval() |
| runtime = CoreMLBoundaryExtractor(args.model_dir, precision=args.precision) |
| cases = [] |
| for name, text, schema in fixtures(): |
| expected = native.extract(text, schema, include_confidence=True, include_spans=True) |
| actual = runtime.extract(text, schema, include_confidence=True, include_spans=True) |
| errors = confidence_errors(expected, actual) |
| cases.append( |
| { |
| "name": name, |
| "text": text, |
| "native": expected, |
| "coreml": actual, |
| "structure_match": without_confidence(expected) == without_confidence(actual), |
| "maximum_confidence_error": max(errors, default=None), |
| } |
| ) |
| result = { |
| "source_model": MODEL_ID, |
| "source_revision": MODEL_REVISION, |
| "precision": args.precision, |
| "selected_manifest": "eleven fixed real-text schema fixtures, not a Decision Index score", |
| "matched": sum(case["structure_match"] for case in cases), |
| "total": len(cases), |
| "cases": cases, |
| } |
| path = Path(args.model_dir) / f"verify-full-{args.precision}.json" |
| path.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n") |
| print( |
| json.dumps( |
| { |
| "matched": result["matched"], |
| "total": result["total"], |
| "failed": [case["name"] for case in cases if not case["structure_match"]], |
| }, |
| indent=2, |
| ) |
| ) |
| if result["matched"] != result["total"] and not args.allow_mismatch: |
| raise SystemExit(1) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
