GLiNER2.5-Decide-1B / validation.json
timkaye's picture
Add validated Q8_0 Decide-1B with optimized Apple Silicon runtime
5c61a1a verified
Raw History Blame Contribute Delete
26.2 kB
{
"scope": "FP32 CPU inference after decoding each exact GGUF tensor; not native quantized-kernel validation or a labeled accuracy benchmark",
"runtime": {
"torch": "2.9.1",
"transformers": "5.17.0",
"gliner2": "2.0.0"
},
"rope": {
"head_dim": 64,
"layers": {
"full_attention": {
"rope_theta": 160000.0,
"inv_freq_count": 32,
"inv_freq_sha256_f32le": "adb41194debe8e6c185d47754f68cc2dd59960a03cb4ccbc71acd14b860c4604",
"first": 1.0,
"last": 9.088847036764491e-06
},
"sliding_attention": {
"rope_theta": 160000.0,
"inv_freq_count": 32,
"inv_freq_sha256_f32le": "adb41194debe8e6c185d47754f68cc2dd59960a03cb4ccbc71acd14b860c4604",
"first": 1.0,
"last": 9.088847036764491e-06
}
}
},
"fixtures": {
"decide_1b_capture.json": "de9fe72fd2038a2c62290a2f8f90aeb8081c5007b82f82a47ef137c33d3c5d61",
"decide_1b_short_holdout.json": "e429219899d3e3d470113f06756c9e807a286a6ba65cc210fd90b5c628628cff"
},
"cases": [
{
"id": "english_intent",
"tokens": 25,
"exact_selected": true,
"max_logit_delta": 0.005266427993774414,
"max_probability_delta": 2.880075461164644e-05
},
{
"id": "spanish_intent",
"tokens": 37,
"exact_selected": true,
"max_logit_delta": 0.007391929626464844,
"max_probability_delta": 4.651299505775697e-06
},
{
"id": "japanese_intent",
"tokens": 45,
"exact_selected": true,
"max_logit_delta": 0.009930849075317383,
"max_probability_delta": 7.443083455760391e-05
},
{
"id": "arabic_intent",
"tokens": 61,
"exact_selected": true,
"max_logit_delta": 0.005479097366333008,
"max_probability_delta": 5.220071690322306e-05
},
{
"id": "described_labels",
"tokens": 83,
"exact_selected": true,
"max_logit_delta": 0.005062580108642578,
"max_probability_delta": 0.0013643558643515186
},
{
"id": "instruction_prompt",
"tokens": 35,
"exact_selected": true,
"max_logit_delta": 0.02295064926147461,
"max_probability_delta": 0.0012888251085210467
},
{
"id": "multi_label_aspects",
"tokens": 41,
"exact_selected": true,
"max_logit_delta": 0.014750480651855469,
"max_probability_delta": 0.0011633577001082251
},
{
"id": "long_context_cutoff",
"tokens": 198,
"exact_selected": true,
"max_logit_delta": 0.01198965311050415,
"max_probability_delta": 0.0034002745542122126
},
{
"id": "described_prompt_choice",
"tokens": 87,
"exact_selected": true,
"max_logit_delta": 0.003972172737121582,
"max_probability_delta": 0.0014366224671735761
},
{
"id": "choice_score_noul",
"tokens": 158,
"exact_selected": true,
"max_logit_delta": 0.0024559497833251953,
"max_probability_delta": 0.0005955669786238404
},
{
"id": "binary_short",
"tokens": 40,
"exact_selected": true,
"max_logit_delta": 0.0016401037573814392,
"max_probability_delta": 0.0008083928258438844
},
{
"id": "described_four_labels",
"tokens": 79,
"exact_selected": true,
"max_logit_delta": 0.004905819892883301,
"max_probability_delta": 0.001215037704017874
},
{
"id": "score_and_noul",
"tokens": 93,
"exact_selected": true,
"max_logit_delta": 0.0012126564979553223,
"max_probability_delta": 0.0005590087507172936
},
{
"id": "mixed_longer",
"tokens": 174,
"exact_selected": true,
"max_logit_delta": 0.00739365816116333,
"max_probability_delta": 0.002287856129400523
}
],
"outputs": {
"fp32": [
{
"id": "english_intent",
"tokens": 25,
"selected": {
"intent": [
"refund"
]
},
"logits": {
"intent": {
"refund": 2.7878055572509766,
"technical_support": -3.442857265472412,
"sales": -3.554849863052368
}
},
"probabilities": {
"intent": {
"refund": 0.9962860742432521,
"technical_support": 0.0019608373801152756,
"sales": 0.0017530883766326147
}
}
},
{
"id": "spanish_intent",
"tokens": 37,
"selected": {
"intenci\u00f3n": [
"cancelar"
]
},
"logits": {
"intenci\u00f3n": {
"reservar": -4.536904811859131,
"cambiar": -4.376052379608154,
"cancelar": 3.6670308113098145,
"consultar": -5.33974552154541
}
},
"probabilities": {
"intenci\u00f3n": {
"reservar": 0.00027337862092737604,
"cambiar": 0.0003210863713091987,
"cancelar": 0.9992830465241191,
"consultar": 0.0001224884836443723
}
}
},
{
"id": "japanese_intent",
"tokens": 45,
"selected": {
"\u4f9d\u983c": [
"\u30ad\u30e3\u30f3\u30bb\u30eb"
]
},
"logits": {
"\u4f9d\u983c": {
"\u6ce8\u6587": -2.0116703510284424,
"\u5909\u66f4": -4.246890544891357,
"\u30ad\u30e3\u30f3\u30bb\u30eb": 2.7734789848327637,
"\u72b6\u6cc1\u78ba\u8a8d": -4.4611077308654785
}
},
"probabilities": {
"\u4f9d\u983c": {
"\u6ce8\u6587": 0.008270440024456262,
"\u5909\u66f4": 0.000884677171106771,
"\u30ad\u30e3\u30f3\u30bb\u30eb": 0.9901307952257268,
"\u72b6\u6cc1\u78ba\u8a8d": 0.0007140875787102114
}
}
},
{
"id": "arabic_intent",
"tokens": 61,
"selected": {
"\u0627\u0644\u0637\u0644\u0628": [
"\u062a\u063a\u064a\u064a\u0631"
]
},
"logits": {
"\u0627\u0644\u0637\u0644\u0628": {
"\u062d\u062c\u0632": -2.4143543243408203,
"\u062a\u063a\u064a\u064a\u0631": 1.5446805953979492,
"\u0625\u0644\u063a\u0627\u0621": -3.041379928588867,
"\u0627\u0633\u062a\u0639\u0644\u0627\u0645": -2.5336153507232666
}
},
"probabilities": {
"\u0627\u0644\u0637\u0644\u0628": {
"\u062d\u062c\u0632": 0.018238696642310485,
"\u062a\u063a\u064a\u064a\u0631": 0.9558303563785708,
"\u0625\u0644\u063a\u0627\u0621": 0.009742715931272027,
"\u0627\u0633\u062a\u0639\u0644\u0627\u0645": 0.016188231047846707
}
}
},
{
"id": "described_labels",
"tokens": 83,
"selected": {
"intent": [
"card_pin_change"
]
},
"logits": {
"intent": {
"card_pin_change": -0.44867444038391113,
"card_lost": -0.6882049441337585,
"balance_inquiry": -2.027876138687134
}
},
"probabilities": {
"intent": {
"card_pin_change": 0.5017216929258406,
"card_lost": 0.3948536009306061,
"balance_inquiry": 0.10342470614355333
}
}
},
{
"id": "instruction_prompt",
"tokens": 35,
"selected": {
"answer": [
"yes"
]
},
"logits": {
"answer": {
"yes": 1.8183664083480835,
"no": -1.6395200490951538
}
},
"probabilities": {
"answer": {
"yes": 0.9694654633121459,
"no": 0.030534536687854136
}
}
},
{
"id": "multi_label_aspects",
"tokens": 41,
"selected": {
"aspects": [
"battery",
"keyboard",
"screen"
]
},
"logits": {
"aspects": {
"battery": 2.02371883392334,
"keyboard": 1.7001844644546509,
"screen": 1.531786322593689,
"camera": -5.12205696105957,
"price": -3.9481678009033203,
"support": -4.606496810913086
}
},
"probabilities": {
"aspects": {
"battery": 0.8832649972915061,
"keyboard": 0.8455588254987787,
"screen": 0.8222675242071651,
"camera": 0.005928387701353327,
"price": 0.01892495003697551,
"support": 0.009887993699153375
}
}
},
{
"id": "long_context_cutoff",
"tokens": 198,
"selected": {
"topic": [
"employment"
]
},
"logits": {
"topic": {
"employment": 0.5384681820869446,
"product": -1.0756205320358276,
"other": -1.902353048324585
}
},
"probabilities": {
"topic": {
"employment": 0.7775074551015492,
"product": 0.15477996356438692,
"other": 0.0677125813340639
}
}
},
{
"id": "described_prompt_choice",
"tokens": 87,
"selected": {
"action": [
"send_pin"
]
},
"logits": {
"action": {
"send_pin": 0.28340578079223633,
"replace_card": -0.5972828269004822,
"balance_help": -2.3520114421844482
}
},
"probabilities": {
"action": {
"send_pin": 0.6728630897096067,
"replace_card": 0.2788999926926913,
"balance_help": 0.048236917597701996
}
}
},
{
"id": "choice_score_noul",
"tokens": 158,
"selected": {
"response": [
"rollback"
],
"severity": [
"3"
],
"page_on_call": [
"true"
]
},
"logits": {
"response": {
"rollback": 0.8552014827728271,
"retry": -1.3567795753479004,
"observe": -1.485878586769104
},
"severity": {
"0": -1.539130449295044,
"1": -1.8733935356140137,
"2": -1.546050786972046,
"3": -1.0720943212509155
},
"page_on_call": {
"false": -0.7481071949005127,
"true": 0.3949044644832611
}
},
"probabilities": {
"response": {
"rollback": 0.8293887644298192,
"retry": 0.09080441819652658,
"observe": 0.07980681737365411
},
"severity": {
"0": 0.23232969937318687,
"1": 0.16631679852930845,
"2": 0.23072744984945065,
"3": 0.37062605224805406
},
"page_on_call": {
"false": 0.2417678459017241,
"true": 0.7582321540982758
}
}
},
{
"id": "binary_short",
"tokens": 40,
"selected": {
"refund": [
"refund"
]
},
"logits": {
"refund": {
"refund": -0.08197160065174103,
"keep": -0.16259559988975525
}
},
"probabilities": {
"refund": {
"refund": 0.5201450886836316,
"keep": 0.4798549113163683
}
}
},
{
"id": "described_four_labels",
"tokens": 79,
"selected": {
"action": [
"reset"
]
},
"logits": {
"action": {
"reset": 0.6374183893203735,
"refund": -1.7385791540145874,
"cancel": -1.5831961631774902,
"ship": -1.8632981777191162
}
},
"probabilities": {
"action": {
"reset": 0.779125462968518,
"refund": 0.07239770109165528,
"cancel": 0.08456813466454835,
"ship": 0.0639087012752783
}
}
},
{
"id": "score_and_noul",
"tokens": 93,
"selected": {
"severity": [
"0"
],
"page": [
"false"
]
},
"logits": {
"severity": {
"0": -0.5489766001701355,
"1": -0.77288818359375,
"2": -1.283555269241333
},
"page": {
"false": 0.47178131341934204,
"true": -0.7986574172973633
}
},
"probabilities": {
"severity": {
"0": 0.4387709715409824,
"1": 0.3507472885781483,
"2": 0.21048173988086938
},
"page": {
"false": 0.7808178421253431,
"true": 0.2191821578746568
}
}
},
{
"id": "mixed_longer",
"tokens": 174,
"selected": {
"action": [
"rollback"
],
"severity": [
"3"
],
"page": [
"true"
]
},
"logits": {
"action": {
"rollback": 0.3483457863330841,
"retry": -1.256446123123169,
"wait": -0.9032326936721802
},
"severity": {
"0": -1.5509973764419556,
"1": -1.5161590576171875,
"2": -1.3640714883804321,
"3": -1.2701959609985352
},
"page": {
"false": -1.021315336227417,
"true": 0.587446391582489
}
},
"probabilities": {
"action": {
"rollback": 0.672502068445642,
"retry": 0.1351267568811651,
"wait": 0.1923711746731929
},
"severity": {
"0": 0.2190493731910276,
"1": 0.22681517336831628,
"2": 0.2640723294547154,
"3": 0.2900631239859407
},
"page": {
"false": 0.166760602366851,
"true": 0.8332393976331489
}
}
}
],
"q8_0": [
{
"id": "english_intent",
"tokens": 25,
"selected": {
"intent": [
"refund"
]
},
"logits": {
"intent": {
"refund": 2.782539129257202,
"technical_support": -3.441603183746338,
"sales": -3.5509848594665527
}
},
"probabilities": {
"intent": {
"refund": 0.9962572734886405,
"technical_support": 0.00197360775963439,
"sales": 0.0017691187517252117
}
}
},
{
"id": "spanish_intent",
"tokens": 37,
"selected": {
"intenci\u00f3n": [
"cancelar"
]
},
"logits": {
"intenci\u00f3n": {
"reservar": -4.5386786460876465,
"cambiar": -4.374134063720703,
"cancelar": 3.6620101928710938,
"consultar": -5.332353591918945
}
},
"probabilities": {
"intenci\u00f3n": {
"reservar": 0.00027426638818669536,
"cambiar": 0.00032332061147933536,
"cancelar": 0.9992783952246134,
"consultar": 0.00012401777572063524
}
}
},
{
"id": "japanese_intent",
"tokens": 45,
"selected": {
"\u4f9d\u983c": [
"\u30ad\u30e3\u30f3\u30bb\u30eb"
]
},
"logits": {
"\u4f9d\u983c": {
"\u6ce8\u6587": -2.0136139392852783,
"\u5909\u66f4": -4.24935245513916,
"\u30ad\u30e3\u30f3\u30bb\u30eb": 2.7635481357574463,
"\u72b6\u6cc1\u78ba\u8a8d": -4.467929840087891
}
},
"probabilities": {
"\u4f9d\u983c": {
"\u6ce8\u6587": 0.008336136004472373,
"\u5909\u66f4": 0.0008912425060391694,
"\u30ad\u30e3\u30f3\u30bb\u30eb": 0.9900563643911692,
"\u72b6\u6cc1\u78ba\u8a8d": 0.0007162570983192617
}
}
},
{
"id": "arabic_intent",
"tokens": 61,
"selected": {
"\u0627\u0644\u0637\u0644\u0628": [
"\u062a\u063a\u064a\u064a\u0631"
]
},
"logits": {
"\u0627\u0644\u0637\u0644\u0628": {
"\u062d\u062c\u0632": -2.4140942096710205,
"\u062a\u063a\u064a\u064a\u0631": 1.546945571899414,
"\u0625\u0644\u063a\u0627\u0621": -3.0412609577178955,
"\u0627\u0633\u062a\u0639\u0644\u0627\u0645": -2.5281362533569336
}
},
"probabilities": {
"\u0627\u0644\u0637\u0644\u0628": {
"\u062d\u062c\u0632": 0.018202263683549207,
"\u062a\u063a\u064a\u064a\u0631": 0.9558354225965263,
"\u0625\u0644\u063a\u0627\u0621": 0.009721881955174486,
"\u0627\u0633\u062a\u0639\u0644\u0627\u0645": 0.01624043176474993
}
}
},
{
"id": "described_labels",
"tokens": 83,
"selected": {
"intent": [
"card_pin_change"
]
},
"logits": {
"intent": {
"card_pin_change": -0.44942909479141235,
"card_lost": -0.683142364025116,
"balance_inquiry": -2.0279831886291504
}
},
"probabilities": {
"intent": {
"card_pin_change": 0.5005350994275541,
"card_lost": 0.39621795679495764,
"balance_inquiry": 0.10324694377748818
}
}
},
{
"id": "instruction_prompt",
"tokens": 35,
"selected": {
"answer": [
"yes"
]
},
"logits": {
"answer": {
"yes": 1.841317057609558,
"no": -1.6610233783721924
}
},
"probabilities": {
"answer": {
"yes": 0.9707542884206668,
"no": 0.02924571157933309
}
}
},
{
"id": "multi_label_aspects",
"tokens": 41,
"selected": {
"aspects": [
"battery",
"keyboard",
"screen"
]
},
"logits": {
"aspects": {
"battery": 2.035050868988037,
"keyboard": 1.7003912925720215,
"screen": 1.5374376773834229,
"camera": -5.107306480407715,
"price": -3.954435348510742,
"support": -4.606987476348877
}
},
"probabilities": {
"aspects": {
"battery": 0.8844283549916143,
"keyboard": 0.8455858330657302,
"screen": 0.823091930141229,
"camera": 0.006015952418212739,
"price": 0.018808931950199958,
"support": 0.009883191130985501
}
}
},
{
"id": "long_context_cutoff",
"tokens": 198,
"selected": {
"topic": [
"employment"
]
},
"logits": {
"topic": {
"employment": 0.5264785289764404,
"product": -1.0682833194732666,
"other": -1.8942830562591553
}
},
"probabilities": {
"topic": {
"employment": 0.774107180547337,
"product": 0.15711036052904584,
"other": 0.06878245892361716
}
}
},
{
"id": "described_prompt_choice",
"tokens": 87,
"selected": {
"action": [
"send_pin"
]
},
"logits": {
"action": {
"send_pin": 0.2804247736930847,
"replace_card": -0.5933106541633606,
"balance_help": -2.3509857654571533
}
},
"probabilities": {
"action": {
"send_pin": 0.6714264672424332,
"replace_card": 0.28024636095914995,
"balance_help": 0.04832717179841683
}
}
},
{
"id": "choice_score_noul",
"tokens": 158,
"selected": {
"response": [
"rollback"
],
"severity": [
"3"
],
"page_on_call": [
"true"
]
},
"logits": {
"response": {
"rollback": 0.8533177375793457,
"retry": -1.3543236255645752,
"observe": -1.4837919473648071
},
"severity": {
"0": -1.5404956340789795,
"1": -1.8746217489242554,
"2": -1.5476183891296387,
"3": -1.072235107421875
},
"page_on_call": {
"false": -0.7505422830581665,
"true": 0.3957209587097168
}
},
"probabilities": {
"response": {
"rollback": 0.828798317402001,
"retry": 0.09113441259313358,
"observe": 0.08006727000486542
},
"severity": {
"0": 0.23222980441087135,
"1": 0.1662680596013798,
"2": 0.23058156536849664,
"3": 0.3709205706192522
},
"page_on_call": {
"false": 0.24117227892310028,
"true": 0.7588277210768997
}
}
},
{
"id": "binary_short",
"tokens": 40,
"selected": {
"refund": [
"refund"
]
},
"logits": {
"refund": {
"refund": -0.08033149689435959,
"keep": -0.16419453918933868
}
},
"probabilities": {
"refund": {
"refund": 0.5209534815094755,
"keep": 0.4790465184905245
}
}
},
{
"id": "described_four_labels",
"tokens": 79,
"selected": {
"action": [
"reset"
]
},
"logits": {
"action": {
"reset": 0.6337000727653503,
"refund": -1.733673334121704,
"cancel": -1.5801693201065063,
"ship": -1.8613603115081787
}
},
"probabilities": {
"action": {
"reset": 0.7779104252645002,
"refund": 0.0729108874821827,
"cancel": 0.08500771270302684,
"ship": 0.06417097455029025
}
}
},
{
"id": "score_and_noul",
"tokens": 93,
"selected": {
"severity": [
"0"
],
"page": [
"false"
]
},
"logits": {
"severity": {
"0": -0.5477639436721802,
"1": -0.7740756273269653,
"2": -1.2843952178955078
},
"page": {
"false": 0.4722679555416107,
"true": -0.7998051643371582
}
},
"probabilities": {
"severity": {
"0": 0.4393299802916997,
"1": 0.3503522617901444,
"2": 0.21031775791815593
},
"page": {
"false": 0.7810974252945717,
"true": 0.21890257470542834
}
}
},
{
"id": "mixed_longer",
"tokens": 174,
"selected": {
"action": [
"rollback"
],
"severity": [
"3"
],
"page": [
"true"
]
},
"logits": {
"action": {
"rollback": 0.35573944449424744,
"retry": -1.260330080986023,
"wait": -0.9056342840194702
},
"severity": {
"0": -1.5539894104003906,
"1": -1.518075704574585,
"2": -1.3633182048797607,
"3": -1.2683053016662598
},
"page": {
"false": -1.0249624252319336,
"true": 0.5893797874450684
}
},
"probabilities": {
"action": {
"rollback": 0.6747899245750425,
"retry": 0.13406595684321404,
"wait": 0.19114411858174343
},
"severity": {
"0": 0.218469405303305,
"1": 0.2264580434340135,
"2": 0.26436142177285954,
"3": 0.2907111294898219
},
"page": {
"false": 0.16598662692454494,
"true": 0.8340133730754551
}
}
}
]
},
"all_selected_identical": true,
"max_probability_delta": 0.0034002745542122126,
"max_logit_delta": 0.02295064926147461,
"container_repack": {
"tensor_payloads_bit_identical": true,
"tensor_count": 199,
"change": "Moved the four unchanged FP32 classifier tensors from auxiliary head container to encoder container for Antfly ModernBERT metadata inspection; no tensor bytes changed."
},
"artifacts": {
"antfly_inference_bundle.json": {
"size_bytes": 128,
"sha256": "49a5c40b088c1bd9f33842054b950ff206f52be4c7c60379422e69d506da4828"
},
"config.json": {
"size_bytes": 464,
"sha256": "d2732928820b95649bc87f051345b2394fff87fc28c01f945610f943d6f402b2"
},
"encoder_config/config.json": {
"size_bytes": 2160,
"sha256": "c2cc4c15c7504b9e15651f2a32dff82b84fa9a99060a3218671a930da0a9b50d"
},
"gliner2-encoder.Q8_0.gguf": {
"size_bytes": 1118356576,
"sha256": "9ba4aea751bc8ce3586e70c4dfb3db93d0a850da5023b39d1bb68d21247f1694"
},
"gliner_head.gguf": {
"size_bytes": 617195872,
"sha256": "9a6c1a8ae0f58158404ea6875012e0c27e5923188906492ec7d948968741f558"
},
"tokenizer.json": {
"size_bytes": 3585055,
"sha256": "ddb379b6a4679ee16646bf0b726de9ec538d7c80ea9d4d4e33b069f19c9e1efb"
},
"tokenizer_config.json": {
"size_bytes": 559,
"sha256": "8bb8d094c7cde84942866ef98fb5b24f7387245573a7f707c8349729484c27b3"
}
}
}