Spaces:
Running
Running
| { | |
| "contrasts": [ | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.004975124378109541, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.08457711442786098 | |
| }, | |
| "delta": 0.039800995024875774, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.21757824217578242, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T01", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.31499999999999995, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.16999999999999998 | |
| }, | |
| "delta": -0.24, | |
| "family_size": 98, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.009799020097990201, | |
| "p_raw": 9.999000099990002e-05, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T02", | |
| "unit": "score", | |
| "verdict": "worse" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 7.5, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 17.5 | |
| }, | |
| "delta": 12.5, | |
| "family_size": 98, | |
| "metric": "accuracy", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.009799020097990201, | |
| "p_raw": 9.999000099990002e-05, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T04", | |
| "unit": "percent", | |
| "verdict": "better" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.325, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.195 | |
| }, | |
| "delta": -0.26, | |
| "family_size": 98, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.009799020097990201, | |
| "p_raw": 9.999000099990002e-05, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T05", | |
| "unit": "score", | |
| "verdict": "worse" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.17499999999999993, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.0050000000000000044 | |
| }, | |
| "delta": -0.08499999999999996, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.07949205079492051, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T07", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.09000000000000002, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.010000000000000009 | |
| }, | |
| "delta": -0.03999999999999998, | |
| "family_size": 98, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.1653834616538346, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T08", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.25, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.135 | |
| }, | |
| "delta": -0.19000000000000006, | |
| "family_size": 98, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.009799020097990201, | |
| "p_raw": 9.999000099990002e-05, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T09", | |
| "unit": "score", | |
| "verdict": "worse" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.16999999999999993, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.010000000000000009 | |
| }, | |
| "delta": -0.08000000000000007, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.10278972102789721, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T10", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.19999999999999996, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.04999999999999993 | |
| }, | |
| "delta": -0.12, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 100, | |
| "p_holm": 0.34283447265625, | |
| "p_raw": 0.004180908203125, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T11", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.32142857142857145, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.017857142857142794 | |
| }, | |
| "delta": -0.1607142857142857, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 56, | |
| "p_holm": 1.0, | |
| "p_raw": 0.10468953104689531, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T12", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.12, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.05999999999999994 | |
| }, | |
| "delta": -0.030000000000000027, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.5862413758624138, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T15", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.135, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.025000000000000022 | |
| }, | |
| "delta": -0.05499999999999999, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.22567743225677434, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T16", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.12, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.02999999999999997 | |
| }, | |
| "delta": -0.044999999999999984, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.28847115288471153, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T17", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.065, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.08499999999999996 | |
| }, | |
| "delta": 0.010000000000000009, | |
| "family_size": 98, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.8964103589641036, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T20", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.37500000000000006, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.18000000000000005 | |
| }, | |
| "delta": -0.27999999999999997, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.009799020097990201, | |
| "p_raw": 9.999000099990002e-05, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T22", | |
| "unit": "score", | |
| "verdict": "worse" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.19499999999999995, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.04500000000000004 | |
| }, | |
| "delta": -0.11999999999999994, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.19318068193180682, | |
| "p_raw": 0.0022997700229977, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T23", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.09500000000000003, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.09499999999999997 | |
| }, | |
| "delta": 0.0, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 1.0, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T24", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.06895846122365462, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.10913930789707185 | |
| }, | |
| "delta": 0.022458628841607597, | |
| "family_size": 98, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": null, | |
| "p_raw": 0.5989401059894011, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": null, | |
| "artifact_sha256": null, | |
| "status": "pending" | |
| }, | |
| "subset": "full", | |
| "task_id": "T25", | |
| "unit": "score", | |
| "verdict": "withheld" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.13883677298311442, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.4227642276422764 | |
| }, | |
| "delta": 0.15609756097560978, | |
| "family_size": 98, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 40, | |
| "p_holm": 1.0, | |
| "p_raw": 0.14908509149085092, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "hit_free_answer_fields", | |
| "task_id": "T25", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.275, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.05499999999999999 | |
| }, | |
| "delta": -0.16499999999999998, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.4079592040795921, | |
| "p_raw": 0.005099490050994901, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T26", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.185, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.015000000000000013 | |
| }, | |
| "delta": -0.10000000000000003, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.029197080291970802, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T29", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.10000000000000003, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.06000000000000005 | |
| }, | |
| "delta": -0.020000000000000018, | |
| "family_size": 98, | |
| "metric": "levenshtein,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": null, | |
| "p_raw": 0.7123287671232876, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": null, | |
| "artifact_sha256": null, | |
| "status": "pending" | |
| }, | |
| "subset": "full", | |
| "task_id": "T30", | |
| "unit": "score", | |
| "verdict": "withheld" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.10050251256281406, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.06030150753768848 | |
| }, | |
| "delta": -0.02010050251256279, | |
| "family_size": 98, | |
| "metric": "levenshtein,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 199, | |
| "p_holm": 1.0, | |
| "p_raw": 0.7215278472152785, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "hit_free_answer_fields", | |
| "task_id": "T30", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.14500000000000002, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.0050000000000000044 | |
| }, | |
| "delta": -0.07499999999999998, | |
| "family_size": 98, | |
| "metric": "levenshtein,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": null, | |
| "p_raw": 0.05269473052694731, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": null, | |
| "artifact_sha256": null, | |
| "status": "pending" | |
| }, | |
| "subset": "full", | |
| "task_id": "T31", | |
| "unit": "score", | |
| "verdict": "withheld" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.15822784810126583, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.006329113924050639 | |
| }, | |
| "delta": -0.08227848101265822, | |
| "family_size": 98, | |
| "metric": "levenshtein,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 158, | |
| "p_holm": 1.0, | |
| "p_raw": 0.05419458054194581, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "hit_free_answer_fields", | |
| "task_id": "T31", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.35736752328049853, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.20044404510648645 | |
| }, | |
| "delta": -0.28009458020496336, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.009799020097990201, | |
| "p_raw": 9.999000099990002e-05, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T32", | |
| "unit": "score", | |
| "verdict": "worse" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.53, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.39 | |
| }, | |
| "delta": -0.46, | |
| "family_size": 98, | |
| "metric": "exact_match,strict-match", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.009799020097990201, | |
| "p_raw": 9.999000099990002e-05, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T33", | |
| "unit": "score", | |
| "verdict": "worse" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.07330648059459305, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.016069938723859534 | |
| }, | |
| "delta": -0.028489426245139504, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.21237876212378762, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T35", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.08000000000000002, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.10000000000000003 | |
| }, | |
| "delta": 0.010000000000000009, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.9133086691330867, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T37", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -20.827199563427776, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -11.701103807248678 | |
| }, | |
| "delta": -16.089329668334905, | |
| "family_size": 98, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.009799020097990201, | |
| "p_raw": 9.999000099990002e-05, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T38", | |
| "unit": "percent", | |
| "verdict": "worse" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.0678, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.012799999999999992 | |
| }, | |
| "delta": -0.03894999999999999, | |
| "family_size": 98, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.38066193380661933, | |
| "p_raw": 0.0046995300469953, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T39", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.155, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.034999999999999976 | |
| }, | |
| "delta": -0.09499999999999997, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.32366763323667636, | |
| "p_raw": 0.0038996100389961006, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T40", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.2652249609511775, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.14013914178668174 | |
| }, | |
| "delta": -0.2023302388296171, | |
| "family_size": 98, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.009799020097990201, | |
| "p_raw": 9.999000099990002e-05, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T41", | |
| "unit": "score", | |
| "verdict": "worse" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.115, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.035 | |
| }, | |
| "delta": -0.075, | |
| "family_size": 98, | |
| "metric": "exact_match,remove_whitespace", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.024169921875, | |
| "p_raw": 0.000274658203125, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T42", | |
| "unit": "score", | |
| "verdict": "worse" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.36, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.23 | |
| }, | |
| "delta": -0.295, | |
| "family_size": 98, | |
| "metric": "exact_match,remove_whitespace", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.009799020097990201, | |
| "p_raw": 9.999000099990002e-05, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T43", | |
| "unit": "score", | |
| "verdict": "worse" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.25, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.07 | |
| }, | |
| "delta": -0.16000000000000003, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.152984701529847, | |
| "p_raw": 0.0017998200179982, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T45", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.05485222809750534, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.10008447243127017 | |
| }, | |
| "delta": 0.02331219694143971, | |
| "family_size": 98, | |
| "metric": "exact_match,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.5937406259374063, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T47", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.17000000000000004, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.0050000000000000044 | |
| }, | |
| "delta": -0.08499999999999996, | |
| "family_size": 98, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 1.0, | |
| "p_raw": 0.0744925507449255, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T48", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.19666666666666655, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": -0.053333333333333344 | |
| }, | |
| "delta": -0.125, | |
| "family_size": 98, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "model_ref": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n": 200, | |
| "p_holm": 0.08612956935215323, | |
| "p_raw": 0.0009899950500247498, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "review": { | |
| "actor": "independent reviewer", | |
| "artifact_sha256": "bba5721493d2fc6e50ebe483ee24f70b145c9834b1ab0ff6326435ba6e0cb469", | |
| "status": "accepted" | |
| }, | |
| "subset": "full", | |
| "task_id": "T50", | |
| "unit": "score", | |
| "verdict": "inconclusive" | |
| } | |
| ], | |
| "models": [ | |
| { | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "repo": "LiquidAI/LFM2.5-1.2B-Base", | |
| "revision": "7453bca97ca1e67754c4035a4b4c584e1c9dd725", | |
| "weights_sha256": "7678ab9546a0c51c1fca161876b1efc4f0906277f170b5822045f40fdaf9eeff" | |
| }, | |
| { | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "repo": "SlayerLab/Slayer1B-1.06", | |
| "revision": "ef81be58742544ad32e1afab38db011fec209609", | |
| "weights_sha256": "e10df6d50101d8424ff7338c97373ab9e151e359a6fcb23108023ac4a6400214" | |
| } | |
| ], | |
| "observations": [ | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.7611940298507465, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.8407960199004979 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 67000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "30aa8b9159f8a599cd98c94cac594a99adba3ea9038b7c56df558ef3e24ccb2a", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "9d6d5c8b444b57a4043af8030c77daca572418a54ace83cd6c16ebe8945faf67", | |
| "state": "ok", | |
| "task_id": "T01", | |
| "unit": "score", | |
| "value": 0.8009950248756219, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.8059701492537314, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.8756218905472644 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.015, | |
| "hit_rate_answer_fields": 0.015, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 197, | |
| "n_hit_free_answer_fields": 197, | |
| "n_population": 67000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "fe310fa1e741d8e0e79cc574bdbf32123a32eb6575dc51f3276211d31075360a", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "9d6d5c8b444b57a4043af8030c77daca572418a54ace83cd6c16ebe8945faf67", | |
| "state": "ok", | |
| "task_id": "T01", | |
| "unit": "score", | |
| "value": 0.8407960199004975, | |
| "value_hit_free": 0.838308457711443, | |
| "value_hit_free_answer_fields": 0.838308457711443 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.42, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.56 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc_norm,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1172, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.2495, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "a4e47cddfc2c871dc7498e25a7d382299aa78b23a783d34a64dbd43795662cae", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "1caa76f5cdf6f4d8a0a1eb8970d2537ccdb8eabe8d4a2eea6ca7aeb03835cbf9", | |
| "state": "ok", | |
| "task_id": "T02", | |
| "unit": "score", | |
| "value": 0.49, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.19, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.31 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.02, | |
| "hit_rate_answer_fields": 0.02, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 196, | |
| "n_hit_free_answer_fields": 196, | |
| "n_population": 1172, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.2495, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "2b55bec70662eb301a59f68f4e0f49349c282c03290077c22f3f828ebbf8d3e6", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "1caa76f5cdf6f4d8a0a1eb8970d2537ccdb8eabe8d4a2eea6ca7aeb03835cbf9", | |
| "state": "ok", | |
| "task_id": "T02", | |
| "unit": "score", | |
| "value": 0.25, | |
| "value_hit_free": 0.25, | |
| "value_hit_free_answer_fields": 0.25 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 1.6937118906996582, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 1.7778426149146371 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": false, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "byte_perplexity,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 62, | |
| "n_scored": 62, | |
| "n_selected": 62, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "abdc9030b69e396fd1ccc227cdb1fd4ffe97bd3dfe3b28e6d31d24b73c19366e", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "e9225fcf908fbf2f54c7e46c0d421378ac4a0e9875e88634acfa376443aa53e6", | |
| "state": "ok", | |
| "task_id": "T03", | |
| "unit": "byte_ppl", | |
| "value": 1.7347771288213312, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 1.8152554368468583, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 1.878480340094036 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": false, | |
| "hit_rate": 0.33870967741935487, | |
| "hit_rate_answer_fields": 0.33870967741935487, | |
| "metric": "byte_perplexity,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 41, | |
| "n_hit_free_answer_fields": 41, | |
| "n_population": 62, | |
| "n_scored": 62, | |
| "n_selected": 62, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "99601259b1eb621878611a70aa332e6e2e93b43f508c358b8aa3e7e4e4d7b7d4", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "e9225fcf908fbf2f54c7e46c0d421378ac4a0e9875e88634acfa376443aa53e6", | |
| "state": "ok", | |
| "task_id": "T03", | |
| "unit": "byte_ppl", | |
| "value": 1.846283835597753, | |
| "value_hit_free": 1.84293133324836, | |
| "value_hit_free_answer_fields": 1.84293133324836 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 81.5, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 91.0 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "accuracy", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 3272, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 50.0, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "af27e95954e7260ca44f2186eb6fce69bac0c667a4f8ca90a012df0712fbbaa4", | |
| "runner_sha256": "51c21059fa8ac25d76dc611567a7390d72fc8541fd0020ef69995d8b9842903f", | |
| "source_id_digest": "c9aba49d565f69d046d6062b5947cc9f9e0dfc23747684f9564646393e2496fb", | |
| "state": "ok", | |
| "task_id": "T04", | |
| "unit": "percent", | |
| "value": 86.5, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 97.5, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 100.0 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.08, | |
| "hit_rate_answer_fields": 0.08, | |
| "metric": "accuracy", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 184, | |
| "n_hit_free_answer_fields": 184, | |
| "n_population": 3272, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 50.0, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "e4bdf566cbdbf3270a806147e90b3dac49faf82a7f3096b7991b1d799508e726", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "c9aba49d565f69d046d6062b5947cc9f9e0dfc23747684f9564646393e2496fb", | |
| "state": "ok", | |
| "task_id": "T04", | |
| "unit": "percent", | |
| "value": 99.0, | |
| "value_hit_free": 98.91304347826086, | |
| "value_hit_free_answer_fields": 98.91304347826086 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.555, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.695 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc_norm,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 10042, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "655e916e75563bd313f9a6f4ae79f2eab2d0e83e17495a999c443d811e164faf", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "927e1904dd5b604b52283e5af60a0a0e94e699c37917e47b488472560ce5b052", | |
| "state": "ok", | |
| "task_id": "T05", | |
| "unit": "score", | |
| "value": 0.625, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.295, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.435 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.0, | |
| "hit_rate_answer_fields": 0.0, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 200, | |
| "n_hit_free_answer_fields": 200, | |
| "n_population": 10042, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "094d75e783c29cc45cd6a8f7689ee34d2fccdac156e434436d7dc3528756f8c0", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "927e1904dd5b604b52283e5af60a0a0e94e699c37917e47b488472560ce5b052", | |
| "state": "ok", | |
| "task_id": "T05", | |
| "unit": "score", | |
| "value": 0.365, | |
| "value_hit_free": 0.365, | |
| "value_hit_free_answer_fields": 0.365 | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc_norm,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T06", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T06", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.57, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.7 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1267, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "0162ffaac22b1036be4745bd187162f88b6bad141970841220701c979ed5b7b2", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "98a6dff8613cd375df50acd0a1882b7c9cbdccf4572795f6637a221b1723c64c", | |
| "state": "ok", | |
| "task_id": "T07", | |
| "unit": "score", | |
| "value": 0.635, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.48, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.615 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "clean", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1267, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "f3b90322173e03680a88dbbdea5a0836471d30babb40193358b18ee47d9739ba", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "98a6dff8613cd375df50acd0a1882b7c9cbdccf4572795f6637a221b1723c64c", | |
| "state": "ok", | |
| "task_id": "T07", | |
| "unit": "score", | |
| "value": 0.55, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.295, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.425 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc_norm,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 500, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "0f3986c3f4ee25818862e9c9c611383a9422639aacd79979a9579f5852bf0056", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "f375631aa3eb26df7e84fec2e490bbfa06971ffdf4d986fc3d775cf9fd931f14", | |
| "state": "ok", | |
| "task_id": "T08", | |
| "unit": "score", | |
| "value": 0.36, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.255, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.385 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.12, | |
| "hit_rate_answer_fields": 0.12, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 176, | |
| "n_hit_free_answer_fields": 176, | |
| "n_population": 500, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "f72e6b5d9bf5c2d9cd30c1d49fa308e85775f259578ada2643d55568e56a4e55", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "f375631aa3eb26df7e84fec2e490bbfa06971ffdf4d986fc3d775cf9fd931f14", | |
| "state": "ok", | |
| "task_id": "T08", | |
| "unit": "score", | |
| "value": 0.32, | |
| "value_hit_free": 0.3409090909090909, | |
| "value_hit_free_answer_fields": 0.3409090909090909 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.885, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.96 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc_norm,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "7fe0c5dd8ae879f0c8b50267ca1cf98544515a9f105e7d7b8426b1f52ec7e76b", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "7368315d4c7ec5b6e85d28a4f77f010c0790d5b7dd4b020aea7fd84524f74e70", | |
| "state": "ok", | |
| "task_id": "T09", | |
| "unit": "score", | |
| "value": 0.925, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.675, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.795 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.105, | |
| "hit_rate_answer_fields": 0.01, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 179, | |
| "n_hit_free_answer_fields": 198, | |
| "n_population": 1000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "03b7a73657f80bf917844ca81a31762eae4aabcff3cd342ec8768bff5177f243", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "7368315d4c7ec5b6e85d28a4f77f010c0790d5b7dd4b020aea7fd84524f74e70", | |
| "state": "ok", | |
| "task_id": "T09", | |
| "unit": "score", | |
| "value": 0.735, | |
| "value_hit_free": 0.7262569832402235, | |
| "value_hit_free_answer_fields": 0.7323232323232324 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.605, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.735 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 3270, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.595, | |
| "majority_value": 0.615 | |
| }, | |
| "results_sha256": "07a678d9f3c6604d19bdbe19e2bf8ac147c213fbbb6b79cbcc163317566d8170", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "35cec0f1fbcb80691dca498e533d6890402e8ede950e71e0be24da18f86398c0", | |
| "state": "ok", | |
| "task_id": "T10", | |
| "unit": "score", | |
| "value": 0.67, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.525, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.655 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.185, | |
| "hit_rate_answer_fields": 0.0, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 163, | |
| "n_hit_free_answer_fields": 200, | |
| "n_population": 3270, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.915, | |
| "majority_value": 0.615 | |
| }, | |
| "results_sha256": "90a0c1a46df7289952d73ed759e85740105bc5424470fdf0225cccb96e6af66a", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "35cec0f1fbcb80691dca498e533d6890402e8ede950e71e0be24da18f86398c0", | |
| "state": "ok", | |
| "task_id": "T10", | |
| "unit": "score", | |
| "value": 0.59, | |
| "value_hit_free": 0.6012269938650306, | |
| "value_hit_free_answer_fields": 0.59 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.66, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.83 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 100, | |
| "n_scored": 100, | |
| "n_selected": 100, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "a77766212763171efd0de5927a41488aee1c2baf35e77134416a77699795ab6e", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "fa0b4f80a2cc898e96ca6c7d6f5ae0d2dbc46d59d712fc1c20cca88bebd2e114", | |
| "state": "ok", | |
| "task_id": "T11", | |
| "unit": "score", | |
| "value": 0.75, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.53, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.72 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.21, | |
| "hit_rate_answer_fields": 0.21, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 79, | |
| "n_hit_free_answer_fields": 79, | |
| "n_population": 100, | |
| "n_scored": 100, | |
| "n_selected": 100, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "bd2c9f0cd9bc0a75e507f30f08cc0abd802b0f7d6a1cef4d3ee6114551758dce", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "fa0b4f80a2cc898e96ca6c7d6f5ae0d2dbc46d59d712fc1c20cca88bebd2e114", | |
| "state": "ok", | |
| "task_id": "T11", | |
| "unit": "score", | |
| "value": 0.63, | |
| "value_hit_free": 0.6075949367088608, | |
| "value_hit_free_answer_fields": 0.6075949367088608 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.5, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.75 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 56, | |
| "n_scored": 56, | |
| "n_selected": 56, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.3333333333333333, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.5, | |
| "majority_value": 0.5 | |
| }, | |
| "results_sha256": "017849e99eb758bca2e0132f7ffc036eeadbc4ee5bf292d544ee67c344decbee", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "310b71c1979b30e593368ced475893cd0ce5bee7693b9044e7d7a97ab4ec275e", | |
| "state": "ok", | |
| "task_id": "T12", | |
| "unit": "score", | |
| "value": 0.625, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.3392857142857143, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.5892857142857143 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "clean", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 56, | |
| "n_scored": 56, | |
| "n_selected": 56, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.3333333333333333, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.9107142857142857, | |
| "majority_value": 0.5 | |
| }, | |
| "results_sha256": "db4886f81b8ff46e1b338c913c50cabf65473fc9b1d1e2f54dc2a41ee403bcb5", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "310b71c1979b30e593368ced475893cd0ce5bee7693b9044e7d7a97ab4ec275e", | |
| "state": "ok", | |
| "task_id": "T12", | |
| "unit": "score", | |
| "value": 0.4642857142857143, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.38, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.515 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 638, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.98, | |
| "majority_value": 0.545 | |
| }, | |
| "results_sha256": "5f36bf460c59a4f0b887e8afff73ee0cbb21fbe41b9d29cf5a249568e7b7a4e9", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "bfd76be0170ef9f0a49cd916e9f91c699cef728a1f85d7d56c0cf4dfba70909f", | |
| "state": "ok", | |
| "task_id": "T13", | |
| "unit": "score", | |
| "value": 0.445, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.475, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.615 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.375, | |
| "hit_rate_answer_fields": 0.375, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 125, | |
| "n_hit_free_answer_fields": 125, | |
| "n_population": 638, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": true, | |
| "majority_prediction_share": 1.0, | |
| "majority_value": 0.545 | |
| }, | |
| "results_sha256": "04e2e8015e185a193e075d3353d05cc21d18d895fd0b99e6be2dde1147d71f34", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "bfd76be0170ef9f0a49cd916e9f91c699cef728a1f85d7d56c0cf4dfba70909f", | |
| "state": "ok", | |
| "task_id": "T13", | |
| "unit": "score", | |
| "value": 0.545, | |
| "value_hit_free": 0.52, | |
| "value_hit_free_answer_fields": 0.52 | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T14", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T14", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.49, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.625 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 277, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.55, | |
| "majority_value": 0.51 | |
| }, | |
| "results_sha256": "c5224c1ed0b0976f0211e727770b6740f7a9c544115d6e39dba5aae8b952182f", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "1b9b48d87ab1ba4f38607371c083043851102d89967b77b6ba9dd90afd9006c8", | |
| "state": "ok", | |
| "task_id": "T15", | |
| "unit": "score", | |
| "value": 0.56, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.46, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.6 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.03, | |
| "hit_rate_answer_fields": 0.03, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 194, | |
| "n_hit_free_answer_fields": 194, | |
| "n_population": 277, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.85, | |
| "majority_value": 0.51 | |
| }, | |
| "results_sha256": "c12763b0ad64f58f673a86ce7b3766449b1c7e28bd10cb7da98358df8df7169c", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "1b9b48d87ab1ba4f38607371c083043851102d89967b77b6ba9dd90afd9006c8", | |
| "state": "ok", | |
| "task_id": "T15", | |
| "unit": "score", | |
| "value": 0.53, | |
| "value_hit_free": 0.5257731958762887, | |
| "value_hit_free_answer_fields": 0.5257731958762887 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.35, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.48 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1200, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.33333333333333326, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.515, | |
| "majority_value": 0.36 | |
| }, | |
| "results_sha256": "d5963aa9df00c8a449b72f6f23e8877bb2d09032c9698eb060792daa83ae3be0", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "8f6560a9a699b3ff3acdc03af4bc854ea2943e72c81900d4de63c9390a108b7b", | |
| "state": "ok", | |
| "task_id": "T16", | |
| "unit": "score", | |
| "value": 0.415, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.295, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.425 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.035, | |
| "hit_rate_answer_fields": 0.035, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 193, | |
| "n_hit_free_answer_fields": 193, | |
| "n_population": 1200, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.33333333333333326, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.88, | |
| "majority_value": 0.36 | |
| }, | |
| "results_sha256": "b160634d004080dae805a808122eff52eab302e48d2163ccec7ab5ee17b63a9b", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "8f6560a9a699b3ff3acdc03af4bc854ea2943e72c81900d4de63c9390a108b7b", | |
| "state": "ok", | |
| "task_id": "T16", | |
| "unit": "score", | |
| "value": 0.36, | |
| "value_hit_free": 0.35233160621761656, | |
| "value_hit_free_answer_fields": 0.35233160621761656 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.375, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.515 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 5153, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "7c0fe44ca3892f4fe7183ba5a49cecd4ff64ba9d4abba1adb228bf394ba70b14", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "0e7f9f160515a068d8f2a4df367c0da497884bc3f6cd9d6b6835fcab2b3e7e9a", | |
| "state": "ok", | |
| "task_id": "T17", | |
| "unit": "score", | |
| "value": 0.445, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.33, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.465 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.0, | |
| "hit_rate_answer_fields": 0.0, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 200, | |
| "n_hit_free_answer_fields": 200, | |
| "n_population": 5153, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "971e414b21515002eff209781a4f85f9fbbb238c3567a89fc382655ecf680f9c", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "0e7f9f160515a068d8f2a4df367c0da497884bc3f6cd9d6b6835fcab2b3e7e9a", | |
| "state": "ok", | |
| "task_id": "T17", | |
| "unit": "score", | |
| "value": 0.4, | |
| "value_hit_free": 0.4, | |
| "value_hit_free_answer_fields": 0.4 | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc_norm,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T18", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "clean", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T18", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc_norm,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T19", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T19", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.23, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.36 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc_norm,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 18736, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "297d36c7dfa3612fb6d9af241cc5a9a798913037ea0662339ca79953940f2107", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "7f70b2594a3b84097657ff33afb9c50ee8039883349049423d72efd7b22a7a0d", | |
| "state": "ok", | |
| "task_id": "T20", | |
| "unit": "score", | |
| "value": 0.295, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.24, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.37 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "clean", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc_norm,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 18736, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "d3651eaa70565fd77f1a7e1b5e1aa65797472522c43d8a36b3f04f205bb45c84", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "7f70b2594a3b84097657ff33afb9c50ee8039883349049423d72efd7b22a7a0d", | |
| "state": "ok", | |
| "task_id": "T20", | |
| "unit": "score", | |
| "value": 0.305, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.635, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.765 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 872, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.755, | |
| "majority_value": 0.505 | |
| }, | |
| "results_sha256": "ac5d66fcb3238d55198246328baae16800a2e504f66dc8955b36847b8c6e8fb9", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "af3c3c75c608c2dec0e3ee4a324b77b53841a492804bdea7e67e4228bb6f81ba", | |
| "state": "ok", | |
| "task_id": "T21", | |
| "unit": "score", | |
| "value": 0.7, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.435, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.575 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.01, | |
| "hit_rate_answer_fields": 0.01, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 198, | |
| "n_hit_free_answer_fields": 198, | |
| "n_population": 872, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": true, | |
| "majority_prediction_share": 1.0, | |
| "majority_value": 0.505 | |
| }, | |
| "results_sha256": "cf21c35cecba72ed1be00e746fcf527fc93423efed29823640abb66b1b8b0634", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "af3c3c75c608c2dec0e3ee4a324b77b53841a492804bdea7e67e4228bb6f81ba", | |
| "state": "ok", | |
| "task_id": "T21", | |
| "unit": "score", | |
| "value": 0.505, | |
| "value_hit_free": 0.5, | |
| "value_hit_free_answer_fields": 0.5 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.535, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.675 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 722, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.63, | |
| "majority_value": 0.4 | |
| }, | |
| "results_sha256": "b7e1d3c20c6f69f8f4dfc40d100cc3f74f746282f1f61f20b4aea983eea3b7e1", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "69155ab8e0f845beead50b746fb16e0b9ead60229de4fbd6e9013824fd46b3ec", | |
| "state": "ok", | |
| "task_id": "T22", | |
| "unit": "score", | |
| "value": 0.605, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.26, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.39 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.02, | |
| "hit_rate_answer_fields": 0.02, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 196, | |
| "n_hit_free_answer_fields": 196, | |
| "n_population": 722, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.635, | |
| "majority_value": 0.4 | |
| }, | |
| "results_sha256": "228850f2134265180e7f82b33e7eab1aa1c986f2ad50dbc450d606b6e185faa3", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "69155ab8e0f845beead50b746fb16e0b9ead60229de4fbd6e9013824fd46b3ec", | |
| "state": "ok", | |
| "task_id": "T22", | |
| "unit": "score", | |
| "value": 0.325, | |
| "value_hit_free": 0.33163265306122447, | |
| "value_hit_free_answer_fields": 0.33163265306122447 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.505, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.64 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 4372, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.125, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.225, | |
| "majority_value": 0.16 | |
| }, | |
| "results_sha256": "425db72aa75e03d32ac6232d910c8c0907a51cadef207bd7ebc5ad270a8daa7a", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "a599d193eb6aff848ee4b1a64e7fc781b3d617f35c2293319f29b2f10debe0d1", | |
| "state": "ok", | |
| "task_id": "T23", | |
| "unit": "score", | |
| "value": 0.575, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.385, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.525 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.03, | |
| "hit_rate_answer_fields": 0.03, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 194, | |
| "n_hit_free_answer_fields": 194, | |
| "n_population": 4372, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.125, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.515, | |
| "majority_value": 0.16 | |
| }, | |
| "results_sha256": "ae22d8523778e9b2ab6f060a472800f4a43ff32d81d420fd2c9783cf92351e48", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "a599d193eb6aff848ee4b1a64e7fc781b3d617f35c2293319f29b2f10debe0d1", | |
| "state": "ok", | |
| "task_id": "T23", | |
| "unit": "score", | |
| "value": 0.455, | |
| "value_hit_free": 0.4536082474226804, | |
| "value_hit_free_answer_fields": 0.4536082474226804 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.24, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.365 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 900, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.665, | |
| "majority_value": 0.3 | |
| }, | |
| "results_sha256": "e2316eba6c6191492b6aa63959ed7108528c3e2db41bd2c5724e132c7724cc3b", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "7bb3efeaa3ff8382b06c253e6542db63c16f500c4dc685fad1a188f7aa37cb36", | |
| "state": "ok", | |
| "task_id": "T24", | |
| "unit": "score", | |
| "value": 0.3, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.235, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.365 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "clean", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 900, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.395, | |
| "majority_value": 0.3 | |
| }, | |
| "results_sha256": "aaca6733999a7b6198e172abb5d25d5b36183e38006faaecfb091254c1705875", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "7bb3efeaa3ff8382b06c253e6542db63c16f500c4dc685fad1a188f7aa37cb36", | |
| "state": "ok", | |
| "task_id": "T24", | |
| "unit": "score", | |
| "value": 0.3, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.17341040462427745, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.33980582524271846 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "f1,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1029, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.81, | |
| "majority_value": 0.0 | |
| }, | |
| "results_sha256": "443f1bacd8f8fa2c6a885eded4c9acc5fde305ad69ff1a82450e930c82705b43", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "5b74b752db2489e6e12fbc31c5f9e84c95bfae73bcdaf940df20789716c23091", | |
| "state": "ok", | |
| "task_id": "T25", | |
| "unit": "score", | |
| "value": 0.2553191489361702, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.16666666666666666, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.38596491228070173 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.8, | |
| "hit_rate_answer_fields": 0.8, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 40, | |
| "n_hit_free_answer_fields": 40, | |
| "n_population": 1029, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.59, | |
| "majority_value": 0.0 | |
| }, | |
| "results_sha256": "f7274ddd1a5c1f5c7e6d110e17b41220179187f10945b740a4531ac199adc4ce", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "5b74b752db2489e6e12fbc31c5f9e84c95bfae73bcdaf940df20789716c23091", | |
| "state": "ok", | |
| "task_id": "T25", | |
| "unit": "score", | |
| "value": 0.2777777777777778, | |
| "value_hit_free": 0.4, | |
| "value_hit_free_answer_fields": 0.4 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.35, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.49 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.33333333333333326, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.905, | |
| "majority_value": 0.41 | |
| }, | |
| "results_sha256": "68b61886b30ff397507e5e3707aaa22fb4a3f6a0a93ccc0dc38ece0458829e13", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "5ca16a5f491004ebd48a5e4a5a58f8dcb7622ad004812ca8ed84288a9d205a52", | |
| "state": "ok", | |
| "task_id": "T26", | |
| "unit": "score", | |
| "value": 0.42, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.195, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.315 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.28, | |
| "hit_rate_answer_fields": 0.28, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 144, | |
| "n_hit_free_answer_fields": 144, | |
| "n_population": 1000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.33333333333333326, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.78, | |
| "majority_value": 0.41 | |
| }, | |
| "results_sha256": "1fa89ad0ddd3e617f9ab9f0c9f42b41bfbb548d4a455228b9380b0a245bb71c5", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "5ca16a5f491004ebd48a5e4a5a58f8dcb7622ad004812ca8ed84288a9d205a52", | |
| "state": "ok", | |
| "task_id": "T26", | |
| "unit": "score", | |
| "value": 0.255, | |
| "value_hit_free": 0.2777777777777778, | |
| "value_hit_free_answer_fields": 0.2777777777777778 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.0, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.0 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "f1,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1078, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": true, | |
| "majority_prediction_share": 1.0, | |
| "majority_value": 0.0 | |
| }, | |
| "results_sha256": "ccfe333023cb4e7f855edaba59842373e8b412a91b98ffcea3e4f321e20e4728", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "1eacd6a6d277d2dab76f9f5b28de389edad6e1db1cfa8d130f315b70d37dc9c7", | |
| "state": "ok", | |
| "task_id": "T27", | |
| "unit": "score", | |
| "value": 0.0, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.0, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.0 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.0, | |
| "hit_rate_answer_fields": 0.0, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 200, | |
| "n_hit_free_answer_fields": 200, | |
| "n_population": 1078, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": true, | |
| "majority_prediction_share": 1.0, | |
| "majority_value": 0.0 | |
| }, | |
| "results_sha256": "181fd12e7c0953e8acb4a4ff9a6c6bd84db097c7ff99db1abc85d98451323715", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "1eacd6a6d277d2dab76f9f5b28de389edad6e1db1cfa8d130f315b70d37dc9c7", | |
| "state": "ok", | |
| "task_id": "T27", | |
| "unit": "score", | |
| "value": 0.0, | |
| "value_hit_free": 0.0, | |
| "value_hit_free_answer_fields": 0.0 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.17257142857142857, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.2296195652173913 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "f1,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": true, | |
| "majority_prediction_share": 1.0, | |
| "majority_value": 0.1796143250688705 | |
| }, | |
| "results_sha256": "09b3af79137a6b10a72300acda72a7c4b118bebb81eb42e4c6a7b7bb68d98679", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "2771592012d0ffcdb46329d21fd29eef4c2de815b45ab1c1b8b597b2510a5999", | |
| "state": "ok", | |
| "task_id": "T28", | |
| "unit": "score", | |
| "value": 0.1796143250688705, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.17257142857142857, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.2296195652173913 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.03, | |
| "hit_rate_answer_fields": 0.03, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 194, | |
| "n_hit_free_answer_fields": 194, | |
| "n_population": 1000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": true, | |
| "majority_prediction_share": 1.0, | |
| "majority_value": 0.1796143250688705 | |
| }, | |
| "results_sha256": "cf0bdd1e0eeabbb7ac112edbcd1436632f7b97f3dd5e68959d134e2268c1b0d9", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "2771592012d0ffcdb46329d21fd29eef4c2de815b45ab1c1b8b597b2510a5999", | |
| "state": "ok", | |
| "task_id": "T28", | |
| "unit": "score", | |
| "value": 0.1796143250688705, | |
| "value_hit_free": 0.1789173789173789, | |
| "value_hit_free_answer_fields": 0.1789173789173789 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.39, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.53 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 2058, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.16666666666666663, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.645, | |
| "majority_value": 0.4 | |
| }, | |
| "results_sha256": "5cf79411fd7efef677ced342a27de65f06c1b2707e93e84400a9ae7645e2c860", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "4c8f630dd6836f78d765b618c19c61b3196bb5ea92207845a3ed90f1bdbf2e17", | |
| "state": "ok", | |
| "task_id": "T29", | |
| "unit": "score", | |
| "value": 0.46, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.295, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.425 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.065, | |
| "hit_rate_answer_fields": 0.065, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 187, | |
| "n_hit_free_answer_fields": 187, | |
| "n_population": 2058, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.16666666666666663, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.74, | |
| "majority_value": 0.4 | |
| }, | |
| "results_sha256": "db013b473df2b6c545a0f877b8c39c75dbd17543b8f1379c550f7564b765ca5b", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "4c8f630dd6836f78d765b618c19c61b3196bb5ea92207845a3ed90f1bdbf2e17", | |
| "state": "ok", | |
| "task_id": "T29", | |
| "unit": "score", | |
| "value": 0.36, | |
| "value_hit_free": 0.3689839572192513, | |
| "value_hit_free_answer_fields": 0.3689839572192513 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.465, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.605 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "levenshtein,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 5928, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "22e7a8c2c57285918269570db982a9f97516ec280876e58a69c9851cf500a0ba", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "4315be85aabdd27ef0c986e1a461bea8afc6c4cfde4d6b65c7d6149ca3f5232d", | |
| "state": "ok", | |
| "task_id": "T30", | |
| "unit": "score", | |
| "value": 0.535, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.445, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.585 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.965, | |
| "hit_rate_answer_fields": 0.005, | |
| "metric": "levenshtein,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 7, | |
| "n_hit_free_answer_fields": 199, | |
| "n_population": 5928, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "5665d6ebaf66ff2109fc8dc3710622925b60255661faaa63be6bcd4285e2e40f", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "4315be85aabdd27ef0c986e1a461bea8afc6c4cfde4d6b65c7d6149ca3f5232d", | |
| "state": "ok", | |
| "task_id": "T30", | |
| "unit": "score", | |
| "value": 0.515, | |
| "value_hit_free": 0.14285714285714285, | |
| "value_hit_free_answer_fields": 0.5175879396984925 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.235, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.36 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "levenshtein,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 5764, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "ee82cea1101b51db90614e3b92b563d5abf3a2476acbc8fd754c4f9721aaebfb", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "4632428c7c6240e6c79251aa573e01c87e7ef25690691327016d6466f9e09ade", | |
| "state": "ok", | |
| "task_id": "T31", | |
| "unit": "score", | |
| "value": 0.295, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.165, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.28 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 1.0, | |
| "hit_rate_answer_fields": 0.21, | |
| "metric": "levenshtein,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 0, | |
| "n_hit_free_answer_fields": 158, | |
| "n_population": 5764, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "cfc97f65375c0d2e7c76457d44279946bfec77cda6c8b2076ba0ef8d516ca403", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "4632428c7c6240e6c79251aa573e01c87e7ef25690691327016d6466f9e09ade", | |
| "state": "ok", | |
| "task_id": "T31", | |
| "unit": "score", | |
| "value": 0.22, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": 0.22784810126582278 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.505538317021724, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.6212591970532435 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 14042, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.295, | |
| "majority_value": 0.28 | |
| }, | |
| "results_sha256": "0772e15260978ed97bd54538472e6024412466a4007407cfdd0f8fa1ddcd57f7", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "6ddde1e4519a7233a2918746a5f446717f5d70a5d0d9031655ae3dd63da33d03", | |
| "state": "ok", | |
| "task_id": "T32", | |
| "unit": "score", | |
| "value": 0.5632786242386014, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.22895617414916666, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.33779775392437406 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.03, | |
| "hit_rate_answer_fields": 0.03, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 194, | |
| "n_hit_free_answer_fields": 194, | |
| "n_population": 14042, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.405, | |
| "majority_value": 0.28 | |
| }, | |
| "results_sha256": "bba8b008a1bb7e6c6e00fa2e24fc0660a006bbf38714c6dd08edac8297e2800d", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "6ddde1e4519a7233a2918746a5f446717f5d70a5d0d9031655ae3dd63da33d03", | |
| "state": "ok", | |
| "task_id": "T32", | |
| "unit": "score", | |
| "value": 0.2831840440336381, | |
| "value_hit_free": 0.2935724829222906, | |
| "value_hit_free_answer_fields": 0.2935724829222906 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.43, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.57 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "exact_match,strict-match", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1319, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "094eb4a8b53de271f91d10041183ed474d17f649d770ef3a857c5288b2d78ebf", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "fba1895da6a8db6929b9c1cae827cabc98f2ba813e4a4f3cff7857a12ac4a9db", | |
| "state": "ok", | |
| "task_id": "T33", | |
| "unit": "score", | |
| "value": 0.5, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.015, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.07 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "clean", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "exact_match,strict-match", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1319, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "11b6515486899a13b8f6c0b7c79df2ba618bac8f9a74e2abf0d2f3a91d47db79", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "fba1895da6a8db6929b9c1cae827cabc98f2ba813e4a4f3cff7857a12ac4a9db", | |
| "state": "ok", | |
| "task_id": "T33", | |
| "unit": "score", | |
| "value": 0.04, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.0, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.0 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "exact_match,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 5000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "df351a29a6bbc0a1f69aa790bdd2a6a9de35313fd383020a687c493bb6e3bcaa", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "d76e87064d912355ac548e65ef497055c1708fbe759c8766a75f3ac83ebe0116", | |
| "state": "ok", | |
| "task_id": "T34", | |
| "unit": "score", | |
| "value": 0.0, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.0, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.0 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.02, | |
| "hit_rate_answer_fields": 0.02, | |
| "metric": "exact_match,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 196, | |
| "n_hit_free_answer_fields": 196, | |
| "n_population": 5000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "6c4606e511a9f2af2585e8c3e24d6f3b94e5b6ac8dd7d963ae0e7d7b5a34b0d5", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "d76e87064d912355ac548e65ef497055c1708fbe759c8766a75f3ac83ebe0116", | |
| "state": "ok", | |
| "task_id": "T34", | |
| "unit": "score", | |
| "value": 0.0, | |
| "value_hit_free": 0.0, | |
| "value_hit_free_answer_fields": 0.0 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.4170048281999294, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.530905281685233 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 817, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "22add0cb7a15606f4e6aa69726d44b46f92a6d577d61b33e2d188726dd424efe", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "93301a192273d3dc8865e3dcbd057c2e0d4c21f4946734a7c908a0a08e0afeb1", | |
| "state": "ok", | |
| "task_id": "T35", | |
| "unit": "score", | |
| "value": 0.4730409384330242, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.3838642958423616, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.5049122565642408 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.07, | |
| "hit_rate_answer_fields": 0.07, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 186, | |
| "n_hit_free_answer_fields": 186, | |
| "n_population": 817, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "2456ff951abaff77856d564abcb8234740438c3d0dba976c57d4b112fcf280a7", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "93301a192273d3dc8865e3dcbd057c2e0d4c21f4946734a7c908a0a08e0afeb1", | |
| "state": "ok", | |
| "task_id": "T35", | |
| "unit": "score", | |
| "value": 0.4445515121878845, | |
| "value_hit_free": 0.4358428387900334, | |
| "value_hit_free_answer_fields": 0.4358428387900334 | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "prompt_level_strict_acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T36", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "prompt_level_strict_acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T36", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.23, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.355 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 18821, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.58, | |
| "majority_value": 0.33 | |
| }, | |
| "results_sha256": "ebcd27776a5fc02060defd76f54723846ad47a73d46ba0ea7a14442b7ba4e253", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "afaf9c14c3bf753305551a43eac9044b6519c296b0d942f47fc0bf46126589a0", | |
| "state": "ok", | |
| "task_id": "T37", | |
| "unit": "score", | |
| "value": 0.29, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.24, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.365 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.045, | |
| "hit_rate_answer_fields": 0.045, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 191, | |
| "n_hit_free_answer_fields": 191, | |
| "n_population": 18821, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.655, | |
| "majority_value": 0.33 | |
| }, | |
| "results_sha256": "114818a564af652284faf55d7ea1c16cc9d47508260de24f2e0760a4ef541568", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "afaf9c14c3bf753305551a43eac9044b6519c296b0d942f47fc0bf46126589a0", | |
| "state": "ok", | |
| "task_id": "T37", | |
| "unit": "score", | |
| "value": 0.3, | |
| "value_hit_free": 0.29842931937172773, | |
| "value_hit_free_answer_fields": 0.29842931937172773 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 16.506203016316007, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 26.492142050376877 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "f1,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 11873, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "0db861039b901f666f2ddbc7bc3b83887e88ea3b196504bf2d76844f8b224dda", | |
| "runner_sha256": "e86eef3297f7b06d6bbfe898b51089295d76b0dd1bd338c44401a092c5f62777", | |
| "source_id_digest": "e5657b168e2c261d949e63361a31d63c3033da08f0928141995e7311a34e3ed0", | |
| "state": "ok", | |
| "task_id": "T38", | |
| "unit": "percent", | |
| "value": 21.445553515509165, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 3.6476204762255002, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 7.287952243174164 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.24, | |
| "hit_rate_answer_fields": 0.0, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 152, | |
| "n_hit_free_answer_fields": 200, | |
| "n_population": 11873, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "44c2eabb849a2f1ae87b9ee2f512c86f68712a3b586cd5493284b8437fa74450", | |
| "runner_sha256": "e86eef3297f7b06d6bbfe898b51089295d76b0dd1bd338c44401a092c5f62777", | |
| "source_id_digest": "e5657b168e2c261d949e63361a31d63c3033da08f0928141995e7311a34e3ed0", | |
| "state": "ok", | |
| "task_id": "T38", | |
| "unit": "percent", | |
| "value": 5.3562238471742605, | |
| "value_hit_free": 5.432371239488894, | |
| "value_hit_free_answer_fields": 5.3562238471742605 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.03455, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.08659999999999998 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "f1,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 9536, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "34f7dc7e011e41bd854aa02889829bce550616693531dfeb55bee55dbbc97368", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "0c4c8e2a4d4aacf9628a2d34e3ccea3528592d25348621ef86b9606723ff5230", | |
| "state": "ok", | |
| "task_id": "T39", | |
| "unit": "score", | |
| "value": 0.05874999999999998, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.010049999999999998, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.0315 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.215, | |
| "hit_rate_answer_fields": 0.0, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 157, | |
| "n_hit_free_answer_fields": 200, | |
| "n_population": 9536, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "28bdce9d0db7286033b13bd49dcc9664fcf99954fcda1db417c83fa7c6d1b0d1", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "0c4c8e2a4d4aacf9628a2d34e3ccea3528592d25348621ef86b9606723ff5230", | |
| "state": "ok", | |
| "task_id": "T39", | |
| "unit": "score", | |
| "value": 0.019800000000000005, | |
| "value_hit_free": 0.02089171974522293, | |
| "value_hit_free_answer_fields": 0.019800000000000005 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.3, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.43 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1045, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "642d7a629f4d89bd0b24957ac70522b92228e57c2af4fea2c8facfcb4de10439", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "aea2e006a0d1f85819b30a9ec7b9bae4c440d5fcd0c6cf1cdd90ff17121ce846", | |
| "state": "ok", | |
| "task_id": "T40", | |
| "unit": "score", | |
| "value": 0.365, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.21, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.33 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.06, | |
| "hit_rate_answer_fields": 0.0, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 188, | |
| "n_hit_free_answer_fields": 200, | |
| "n_population": 1045, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "9ea656b54c49116629416b0770dc8e9565addf0781a20e016ef0a06a9c8354af", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "aea2e006a0d1f85819b30a9ec7b9bae4c440d5fcd0c6cf1cdd90ff17121ce846", | |
| "state": "ok", | |
| "task_id": "T40", | |
| "unit": "score", | |
| "value": 0.27, | |
| "value_hit_free": 0.2712765957446808, | |
| "value_hit_free_answer_fields": 0.27 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.5735389344451844, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.6817363146113147 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "f1,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 500, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "b049b999de93cbec5e42aafadeb7fcb367a7e2dcaaab57d748cf81b3586387aa", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "559c0825839f86a7a9ac4ed6d7ba71c7b5ca647307dd92f0bf69ade0f67246fa", | |
| "state": "ok", | |
| "task_id": "T41", | |
| "unit": "score", | |
| "value": 0.6282142451555351, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.36869991408373765, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.48298872799313974 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.145, | |
| "hit_rate_answer_fields": 0.13, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 171, | |
| "n_hit_free_answer_fields": 174, | |
| "n_population": 500, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "1f3258492c7bf935aaaf7d1b44e907fca99ec7009ff4ca06fb0fa914609d1827", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "559c0825839f86a7a9ac4ed6d7ba71c7b5ca647307dd92f0bf69ade0f67246fa", | |
| "state": "ok", | |
| "task_id": "T41", | |
| "unit": "score", | |
| "value": 0.42588400632591805, | |
| "value_hit_free": 0.4262927797336123, | |
| "value_hit_free_answer_fields": 0.4242111034546802 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.045, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.12 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "exact_match,remove_whitespace", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 3610, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "149539c68af9bb5670f8a3e08b214d6124d8bb5a201ece10dca8ee84acd31bc8", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "ec49de267492e8a3f520ab4eb858fee702f0cecd891ed6098107ac6125d09ae8", | |
| "state": "ok", | |
| "task_id": "T42", | |
| "unit": "score", | |
| "value": 0.08, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.0, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.015 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.025, | |
| "hit_rate_answer_fields": 0.025, | |
| "metric": "exact_match,remove_whitespace", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 195, | |
| "n_hit_free_answer_fields": 195, | |
| "n_population": 3610, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "6e53646f5d5a0d2b5e750de5ba0ad9549c6997d516d69070b6fa3f02377be3b9", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "ec49de267492e8a3f520ab4eb858fee702f0cecd891ed6098107ac6125d09ae8", | |
| "state": "ok", | |
| "task_id": "T42", | |
| "unit": "score", | |
| "value": 0.005, | |
| "value_hit_free": 0.005128205128205128, | |
| "value_hit_free_answer_fields": 0.005128205128205128 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.245, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.375 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "exact_match,remove_whitespace", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 17944, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "cdb3ab12ab3ac90c0f8203f6160054efbcbe1e1bb669d34559e955d4d3ed8aba", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "bedecbef62fad879bc241807f3c27fafa9b346fe5323fe8d229763ff9f1da29e", | |
| "state": "ok", | |
| "task_id": "T43", | |
| "unit": "score", | |
| "value": 0.31, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.0, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.035 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.06, | |
| "hit_rate_answer_fields": 0.06, | |
| "metric": "exact_match,remove_whitespace", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 188, | |
| "n_hit_free_answer_fields": 188, | |
| "n_population": 17944, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "47488433a70f44dfc77501c583b0a3ed3b0b36c644c5c42f8214fd06534c6ae1", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "bedecbef62fad879bc241807f3c27fafa9b346fe5323fe8d229763ff9f1da29e", | |
| "state": "ok", | |
| "task_id": "T43", | |
| "unit": "score", | |
| "value": 0.015, | |
| "value_hit_free": 0.015957446808510637, | |
| "value_hit_free_answer_fields": 0.015957446808510637 | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T44", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T44", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.34, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.475 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 4183, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.325, | |
| "majority_value": 0.305 | |
| }, | |
| "results_sha256": "a12c9ed811d417e8204271dee963b944f022b7ba86299f9c2716f15e8ab203bd", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "42a9f00f0e8d3f1379b0c2ed8123a7fd7421b1789f19e048da83efdf4e3e5974", | |
| "state": "ok", | |
| "task_id": "T45", | |
| "unit": "score", | |
| "value": 0.405, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.185, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.305 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.045, | |
| "hit_rate_answer_fields": 0.045, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 191, | |
| "n_hit_free_answer_fields": 191, | |
| "n_population": 4183, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.25, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.88, | |
| "majority_value": 0.305 | |
| }, | |
| "results_sha256": "05a7b7f84dff46ca0397722374e1b6c28b1bb15ba3b9fa09de667b9e9a3608aa", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "42a9f00f0e8d3f1379b0c2ed8123a7fd7421b1789f19e048da83efdf4e3e5974", | |
| "state": "ok", | |
| "task_id": "T45", | |
| "unit": "score", | |
| "value": 0.245, | |
| "value_hit_free": 0.24083769633507854, | |
| "value_hit_free_answer_fields": 0.24083769633507854 | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T46", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": null, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": null, | |
| "n_scored": 0, | |
| "n_selected": null, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": null, | |
| "runner_sha256": null, | |
| "source_id_digest": null, | |
| "state": "not_run", | |
| "task_id": "T46", | |
| "unit": "score", | |
| "value": null, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.22688563676123188, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.33455068711557734 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "exact_match,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 6511, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "1fff839251972c7ddd6255f840202fce4a8c9a3287765b596cfef0b03e0c2ef2", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "9a04e7f2f5f136a8ee825fa5d5361dfef11caa4f820f27532831751663aba1a4", | |
| "state": "ok", | |
| "task_id": "T47", | |
| "unit": "score", | |
| "value": 0.2810441670140641, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.24883603571977092, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.35990228989768236 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.005, | |
| "hit_rate_answer_fields": 0.005, | |
| "metric": "exact_match,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 199, | |
| "n_hit_free_answer_fields": 199, | |
| "n_population": 6511, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": null, | |
| "results_sha256": "03bd7c99f1d9ebc25c918b204edefc11e55683216e9d7e58b6c49084493acfe5", | |
| "runner_sha256": "703c2ef4ef693bdce95a12554c1dba2668bc3180c4a4c0835186481568e60367", | |
| "source_id_digest": "9a04e7f2f5f136a8ee825fa5d5361dfef11caa4f820f27532831751663aba1a4", | |
| "state": "ok", | |
| "task_id": "T47", | |
| "unit": "score", | |
| "value": 0.30435636395550386, | |
| "value_hit_free": 0.30641332250916037, | |
| "value_hit_free_answer_fields": 0.30641332250916037 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.54, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.675 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "acc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 2000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "3f40b74bc0c63160fcd3e0926850649dd213d5cfaa05a3e2ff10907032c6a46f", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "4d23ca378e5e33e57a2de8109efdd79e7307b1ad6c0ee69211e3791b22951acc", | |
| "state": "ok", | |
| "task_id": "T48", | |
| "unit": "score", | |
| "value": 0.605, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.45, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.59 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.0, | |
| "hit_rate_answer_fields": 0.0, | |
| "metric": "acc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 200, | |
| "n_hit_free_answer_fields": 200, | |
| "n_population": 2000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": 0.5, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "70eed0d17a57ba64f8820cc18d1db49510da9a3897675d5403630887d9f5db96", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "4d23ca378e5e33e57a2de8109efdd79e7307b1ad6c0ee69211e3791b22951acc", | |
| "state": "ok", | |
| "task_id": "T48", | |
| "unit": "score", | |
| "value": 0.52, | |
| "value_hit_free": 0.52, | |
| "value_hit_free_answer_fields": 0.52 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": -0.1443378222898272, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.13502895642910745 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "mcc,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 1043, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": false, | |
| "majority_prediction_share": 0.565, | |
| "majority_value": 0.0 | |
| }, | |
| "results_sha256": "8ace22c45466f17b57036a7a1e2921eefcdb9a0954fa1a6159739f97bacc7193", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "eadab46ac0944d7d2523092310eb29d4ad35d1f616f1e891e6da12a4dd392323", | |
| "state": "ok", | |
| "task_id": "T49", | |
| "unit": "score", | |
| "value": -0.005921616513673969, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.0, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.0 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.05, | |
| "hit_rate_answer_fields": 0.05, | |
| "metric": "mcc,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 190, | |
| "n_hit_free_answer_fields": 190, | |
| "n_population": 1043, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": true, | |
| "majority_prediction_share": 1.0, | |
| "majority_value": 0.0 | |
| }, | |
| "results_sha256": "1d69a9c7d726c33e19a08ac13d7b9a68e69382c5bd113e51070f88203ed44876", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "eadab46ac0944d7d2523092310eb29d4ad35d1f616f1e891e6da12a4dd392323", | |
| "state": "ok", | |
| "task_id": "T49", | |
| "unit": "score", | |
| "value": 0.0, | |
| "value_hit_free": 0.0, | |
| "value_hit_free_answer_fields": 0.0 | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.73, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.8433333333333333 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "unknown", | |
| "contamination_receipt_sha256": null, | |
| "higher_is_better": true, | |
| "hit_rate": null, | |
| "hit_rate_answer_fields": null, | |
| "metric": "f1,none", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "n_hit_free": null, | |
| "n_hit_free_answer_fields": null, | |
| "n_population": 10000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "848e591f595c9b19660c11a9040164249fbc495909947c3a65c11b7134eda80a", | |
| "runner_sha256": "8a8ccfd4c12c40c6a8d7420af941d40643bd76804c7ddf323fda09a713f5b98e", | |
| "source_id_digest": "1c58ffbef64a26dfb93ade5d03941dd28ad2e446e60bfc1dceac74f677aad687", | |
| "state": "ok", | |
| "task_id": "T50", | |
| "unit": "score", | |
| "value": 0.7883333333333333, | |
| "value_hit_free": null, | |
| "value_hit_free_answer_fields": null | |
| }, | |
| { | |
| "ci95": { | |
| "level": 0.95, | |
| "lower": 0.5983333333333334, | |
| "method": "paired_cluster_bootstrap", | |
| "replicates": 10000, | |
| "seed": 20261028, | |
| "upper": 0.73 | |
| }, | |
| "ci95_hit_free": null, | |
| "ci95_hit_free_answer_fields": null, | |
| "contamination": "flagged", | |
| "contamination_receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7", | |
| "higher_is_better": true, | |
| "hit_rate": 0.02, | |
| "hit_rate_answer_fields": 0.005, | |
| "metric": "f1,none", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "n_hit_free": 196, | |
| "n_hit_free_answer_fields": 199, | |
| "n_population": 10000, | |
| "n_scored": 200, | |
| "n_selected": 200, | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "reason": null, | |
| "reference": { | |
| "chance_value": null, | |
| "constant_prediction": null, | |
| "majority_prediction_share": null, | |
| "majority_value": null | |
| }, | |
| "results_sha256": "fa0f39fcc704ecc13207371df0623ef23819e5b77745c118e406cd111063cc90", | |
| "runner_sha256": "f5ebb45ca18f04f3586159837d071bc33e77e90101bec98146edd7139dd6f37d", | |
| "source_id_digest": "1c58ffbef64a26dfb93ade5d03941dd28ad2e446e60bfc1dceac74f677aad687", | |
| "state": "ok", | |
| "task_id": "T50", | |
| "unit": "score", | |
| "value": 0.6633333333333333, | |
| "value_hit_free": 0.66156462585034, | |
| "value_hit_free_answer_fields": 0.6616415410385259 | |
| } | |
| ], | |
| "protocol_id": "bench50-v0.3.1-panel200", | |
| "provenance": { | |
| "amendments": [ | |
| { | |
| "artifact_sha256": "8527661ff15643cbd87aac357f8373d076a29c3d5e4f88501d8d9c4b3e62727e", | |
| "base_items_sha256": "1fff839251972c7ddd6255f840202fce4a8c9a3287765b596cfef0b03e0c2ef2", | |
| "code": "answer_leading_space_strip", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "task_id": "T47" | |
| }, | |
| { | |
| "artifact_sha256": "88f3ce2ab865fffb183d51ae5441da3b2887322ca464e02b0efb04f77505694b", | |
| "base_items_sha256": "03bd7c99f1d9ebc25c918b204edefc11e55683216e9d7e58b6c49084493acfe5", | |
| "code": "answer_leading_space_strip", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "task_id": "T47" | |
| } | |
| ], | |
| "contamination_sha256": "64852fdd8877ff1e2b561cffe929339f2fb8f0af1b070cdf9f6a19d372733ab9", | |
| "contrasts_sha256": "71ae966d5dc5865059b60214c7b9fce1d94c67d178681b8ebc661173cb7c410c", | |
| "inventory_commit": "18a798843d300fa82cceb50c8ba5032d17be36d5", | |
| "kind": "measurement", | |
| "qualification_sha256": "1e3d7aa218b6f9a518824aedd2cdb5f5551d5650f84944dc88a94cdf4bdef86c", | |
| "references_sha256": "2b9342900bbc4b50dfc593ad6df482c3898bcfd67a60e68587c902a67edbcb54", | |
| "sources": [ | |
| { | |
| "results_sha256": "f2561721cdee840e3b1044ca4c02b8e37d04c85ea52a46f76baddc97f7cd27b2", | |
| "run_id": "run7-slayer-ll-en" | |
| }, | |
| { | |
| "results_sha256": "c8710f48a37a155ae90722e469dbeba1032d1ead06db85dfe2fbfa27ff83ab22", | |
| "run_id": "run7-slayer-ll-pl" | |
| }, | |
| { | |
| "results_sha256": "5e47cd22e81bbfd1c6a64ce09cd065736af5000751baca67f494d05067ce7e41", | |
| "run_id": "run8-lfm-bos-ll-en" | |
| }, | |
| { | |
| "results_sha256": "3b7e13142929272c6c4b095c3f29353e08453315bc2d3b8c6799df92500c7c71", | |
| "run_id": "run8-lfm-bos-ll-pl" | |
| }, | |
| { | |
| "results_sha256": "65dd6c459de1996f072793fbf9576a240a5cea517b1f51865b213d9e95d4c6dd", | |
| "run_id": "run9-lfm-bos-t04-en" | |
| }, | |
| { | |
| "results_sha256": "780db12275e2b113b0769d7e830ddfe8391459a5174db2da776a8aad27730959", | |
| "run_id": "genA-slayer-en" | |
| }, | |
| { | |
| "results_sha256": "61d48f6b13b8768c97f16abd2023dfe42ba37402b3797e85db4349cb01a6ef7b", | |
| "run_id": "genA-slayer-pl" | |
| }, | |
| { | |
| "results_sha256": "78da5b851bc37e83c61890f01571cd82419a9b05780717376ec1fd797b894407", | |
| "run_id": "genB-slayer-en" | |
| }, | |
| { | |
| "results_sha256": "c810accb57392072dd8f5384ec8742aeafb8c92d9e32826e169ebf238a496c46", | |
| "run_id": "genL-lfm-en" | |
| }, | |
| { | |
| "results_sha256": "cf163e7ecec0e14a582b20daf24ecfbb2ca8cb256ccde434bea76760376eea90", | |
| "run_id": "genL-lfm-pl" | |
| }, | |
| { | |
| "results_sha256": "dc5f78da8792fe36fb3884d4783f834f911624c630c783c1861e82fb27275bb0", | |
| "run_id": "sq2S-slayer-en" | |
| }, | |
| { | |
| "results_sha256": "e670112e21ee99d55c2b6f090d10749b0157b2945e6e2fa625903e44d5e6d73b", | |
| "run_id": "sq2L-lfm-en" | |
| } | |
| ], | |
| "superseded": [ | |
| { | |
| "by_results_sha256": "65dd6c459de1996f072793fbf9576a240a5cea517b1f51865b213d9e95d4c6dd", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "replaced": [ | |
| { | |
| "results_sha256": "5e47cd22e81bbfd1c6a64ce09cd065736af5000751baca67f494d05067ce7e41", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T04" | |
| }, | |
| { | |
| "by_results_sha256": "cf163e7ecec0e14a582b20daf24ecfbb2ca8cb256ccde434bea76760376eea90", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "replaced": [ | |
| { | |
| "results_sha256": "3b7e13142929272c6c4b095c3f29353e08453315bc2d3b8c6799df92500c7c71", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T30" | |
| }, | |
| { | |
| "by_results_sha256": "cf163e7ecec0e14a582b20daf24ecfbb2ca8cb256ccde434bea76760376eea90", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "replaced": [ | |
| { | |
| "results_sha256": "3b7e13142929272c6c4b095c3f29353e08453315bc2d3b8c6799df92500c7c71", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T31" | |
| }, | |
| { | |
| "by_results_sha256": "c810accb57392072dd8f5384ec8742aeafb8c92d9e32826e169ebf238a496c46", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "replaced": [ | |
| { | |
| "results_sha256": "5e47cd22e81bbfd1c6a64ce09cd065736af5000751baca67f494d05067ce7e41", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T33" | |
| }, | |
| { | |
| "by_results_sha256": "c810accb57392072dd8f5384ec8742aeafb8c92d9e32826e169ebf238a496c46", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "replaced": [ | |
| { | |
| "results_sha256": "5e47cd22e81bbfd1c6a64ce09cd065736af5000751baca67f494d05067ce7e41", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T34" | |
| }, | |
| { | |
| "by_results_sha256": "e670112e21ee99d55c2b6f090d10749b0157b2945e6e2fa625903e44d5e6d73b", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "replaced": [ | |
| { | |
| "results_sha256": "5e47cd22e81bbfd1c6a64ce09cd065736af5000751baca67f494d05067ce7e41", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T38" | |
| }, | |
| { | |
| "by_results_sha256": "c810accb57392072dd8f5384ec8742aeafb8c92d9e32826e169ebf238a496c46", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "replaced": [ | |
| { | |
| "results_sha256": "5e47cd22e81bbfd1c6a64ce09cd065736af5000751baca67f494d05067ce7e41", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T39" | |
| }, | |
| { | |
| "by_results_sha256": "c810accb57392072dd8f5384ec8742aeafb8c92d9e32826e169ebf238a496c46", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "replaced": [ | |
| { | |
| "results_sha256": "5e47cd22e81bbfd1c6a64ce09cd065736af5000751baca67f494d05067ce7e41", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T41" | |
| }, | |
| { | |
| "by_results_sha256": "c810accb57392072dd8f5384ec8742aeafb8c92d9e32826e169ebf238a496c46", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "replaced": [ | |
| { | |
| "results_sha256": "5e47cd22e81bbfd1c6a64ce09cd065736af5000751baca67f494d05067ce7e41", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T42" | |
| }, | |
| { | |
| "by_results_sha256": "c810accb57392072dd8f5384ec8742aeafb8c92d9e32826e169ebf238a496c46", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "replaced": [ | |
| { | |
| "results_sha256": "5e47cd22e81bbfd1c6a64ce09cd065736af5000751baca67f494d05067ce7e41", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T43" | |
| }, | |
| { | |
| "by_results_sha256": "c810accb57392072dd8f5384ec8742aeafb8c92d9e32826e169ebf238a496c46", | |
| "model_id": "LiquidAI/LFM2.5-1.2B-Base", | |
| "replaced": [ | |
| { | |
| "results_sha256": "5e47cd22e81bbfd1c6a64ce09cd065736af5000751baca67f494d05067ce7e41", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T47" | |
| }, | |
| { | |
| "by_results_sha256": "61d48f6b13b8768c97f16abd2023dfe42ba37402b3797e85db4349cb01a6ef7b", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "replaced": [ | |
| { | |
| "results_sha256": "c8710f48a37a155ae90722e469dbeba1032d1ead06db85dfe2fbfa27ff83ab22", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T30" | |
| }, | |
| { | |
| "by_results_sha256": "61d48f6b13b8768c97f16abd2023dfe42ba37402b3797e85db4349cb01a6ef7b", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "replaced": [ | |
| { | |
| "results_sha256": "c8710f48a37a155ae90722e469dbeba1032d1ead06db85dfe2fbfa27ff83ab22", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T31" | |
| }, | |
| { | |
| "by_results_sha256": "780db12275e2b113b0769d7e830ddfe8391459a5174db2da776a8aad27730959", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "replaced": [ | |
| { | |
| "results_sha256": "f2561721cdee840e3b1044ca4c02b8e37d04c85ea52a46f76baddc97f7cd27b2", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T33" | |
| }, | |
| { | |
| "by_results_sha256": "78da5b851bc37e83c61890f01571cd82419a9b05780717376ec1fd797b894407", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "replaced": [ | |
| { | |
| "results_sha256": "f2561721cdee840e3b1044ca4c02b8e37d04c85ea52a46f76baddc97f7cd27b2", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T34" | |
| }, | |
| { | |
| "by_results_sha256": "dc5f78da8792fe36fb3884d4783f834f911624c630c783c1861e82fb27275bb0", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "replaced": [ | |
| { | |
| "results_sha256": "f2561721cdee840e3b1044ca4c02b8e37d04c85ea52a46f76baddc97f7cd27b2", | |
| "state": "not_run" | |
| }, | |
| { | |
| "results_sha256": "78da5b851bc37e83c61890f01571cd82419a9b05780717376ec1fd797b894407", | |
| "state": "unsupported" | |
| } | |
| ], | |
| "task_id": "T38" | |
| }, | |
| { | |
| "by_results_sha256": "78da5b851bc37e83c61890f01571cd82419a9b05780717376ec1fd797b894407", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "replaced": [ | |
| { | |
| "results_sha256": "f2561721cdee840e3b1044ca4c02b8e37d04c85ea52a46f76baddc97f7cd27b2", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T39" | |
| }, | |
| { | |
| "by_results_sha256": "78da5b851bc37e83c61890f01571cd82419a9b05780717376ec1fd797b894407", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "replaced": [ | |
| { | |
| "results_sha256": "f2561721cdee840e3b1044ca4c02b8e37d04c85ea52a46f76baddc97f7cd27b2", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T41" | |
| }, | |
| { | |
| "by_results_sha256": "78da5b851bc37e83c61890f01571cd82419a9b05780717376ec1fd797b894407", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "replaced": [ | |
| { | |
| "results_sha256": "f2561721cdee840e3b1044ca4c02b8e37d04c85ea52a46f76baddc97f7cd27b2", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T42" | |
| }, | |
| { | |
| "by_results_sha256": "78da5b851bc37e83c61890f01571cd82419a9b05780717376ec1fd797b894407", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "replaced": [ | |
| { | |
| "results_sha256": "f2561721cdee840e3b1044ca4c02b8e37d04c85ea52a46f76baddc97f7cd27b2", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T43" | |
| }, | |
| { | |
| "by_results_sha256": "78da5b851bc37e83c61890f01571cd82419a9b05780717376ec1fd797b894407", | |
| "model_id": "SlayerLab/Slayer1B-1.06", | |
| "replaced": [ | |
| { | |
| "results_sha256": "f2561721cdee840e3b1044ca4c02b8e37d04c85ea52a46f76baddc97f7cd27b2", | |
| "state": "not_run" | |
| } | |
| ], | |
| "task_id": "T47" | |
| } | |
| ] | |
| }, | |
| "run_id": "public-export-a847dabc2eec0483", | |
| "schema_version": 2, | |
| "tasks": [ | |
| { | |
| "family": "blimp", | |
| "language": "EN", | |
| "split": "train", | |
| "task": "blimp", | |
| "task_id": "T01" | |
| }, | |
| { | |
| "family": "arc", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "arc_challenge", | |
| "task_id": "T02" | |
| }, | |
| { | |
| "family": "wikitext2", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "wikitext", | |
| "task_id": "T03" | |
| }, | |
| { | |
| "family": "multiblimp", | |
| "language": "PL", | |
| "split": "pol/data.tsv", | |
| "task": "multiblimp_pl", | |
| "task_id": "T04" | |
| }, | |
| { | |
| "family": "hellaswag", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "hellaswag", | |
| "task_id": "T05" | |
| }, | |
| { | |
| "family": "piqa", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "piqa", | |
| "task_id": "T06" | |
| }, | |
| { | |
| "family": "winogrande", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "winogrande", | |
| "task_id": "T07" | |
| }, | |
| { | |
| "family": "openbookqa", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "openbookqa", | |
| "task_id": "T08" | |
| }, | |
| { | |
| "family": "sciq", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "sciq", | |
| "task_id": "T09" | |
| }, | |
| { | |
| "family": "boolq", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "boolq", | |
| "task_id": "T10" | |
| }, | |
| { | |
| "family": "copa", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "copa", | |
| "task_id": "T11" | |
| }, | |
| { | |
| "family": "cb", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "cb", | |
| "task_id": "T12" | |
| }, | |
| { | |
| "family": "wic", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "wic", | |
| "task_id": "T13" | |
| }, | |
| { | |
| "family": "wsc273", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "wsc273", | |
| "task_id": "T14" | |
| }, | |
| { | |
| "family": "rte", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "sglue_rte", | |
| "task_id": "T15" | |
| }, | |
| { | |
| "family": "anli", | |
| "language": "EN", | |
| "split": "test_r3", | |
| "task": "anli_r3", | |
| "task_id": "T16" | |
| }, | |
| { | |
| "family": "lambada", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "lambada_openai", | |
| "task_id": "T17" | |
| }, | |
| { | |
| "family": "logiqa", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "logiqa", | |
| "task_id": "T18" | |
| }, | |
| { | |
| "family": "mathqa", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "mathqa", | |
| "task_id": "T19" | |
| }, | |
| { | |
| "family": "prost", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "prost", | |
| "task_id": "T20" | |
| }, | |
| { | |
| "family": "sst2", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "sst2", | |
| "task_id": "T21" | |
| }, | |
| { | |
| "family": "polemo2", | |
| "language": "PL", | |
| "split": "test", | |
| "task": "polemo2_in_multiple_choice", | |
| "task_id": "T22" | |
| }, | |
| { | |
| "family": "8tags", | |
| "language": "PL", | |
| "split": "test", | |
| "task": "polish_8tags_multiple_choice", | |
| "task_id": "T23" | |
| }, | |
| { | |
| "family": "belebele", | |
| "language": "PL", | |
| "split": "test", | |
| "task": "polish_belebele_mc", | |
| "task_id": "T24" | |
| }, | |
| { | |
| "family": "dyk", | |
| "language": "PL", | |
| "split": "test", | |
| "task": "polish_dyk_multiple_choice", | |
| "task_id": "T25" | |
| }, | |
| { | |
| "family": "ppc", | |
| "language": "PL", | |
| "split": "test", | |
| "task": "polish_ppc_multiple_choice", | |
| "task_id": "T26" | |
| }, | |
| { | |
| "family": "psc", | |
| "language": "PL", | |
| "split": "test", | |
| "task": "polish_psc_multiple_choice", | |
| "task_id": "T27" | |
| }, | |
| { | |
| "family": "cbd", | |
| "language": "PL", | |
| "split": "test", | |
| "task": "polish_cbd_multiple_choice", | |
| "task_id": "T28" | |
| }, | |
| { | |
| "family": "klej_ner", | |
| "language": "PL", | |
| "split": "test", | |
| "task": "polish_klej_ner_multiple_choice", | |
| "task_id": "T29" | |
| }, | |
| { | |
| "family": "polqa", | |
| "language": "PL", | |
| "split": "test", | |
| "task": "polish_polqa_open_book", | |
| "task_id": "T30" | |
| }, | |
| { | |
| "family": "poquad", | |
| "language": "PL", | |
| "split": "validation", | |
| "task": "polish_poquad_open_book", | |
| "task_id": "T31" | |
| }, | |
| { | |
| "family": "mmlu", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "mmlu", | |
| "task_id": "T32" | |
| }, | |
| { | |
| "family": "gsm8k", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "gsm8k", | |
| "task_id": "T33" | |
| }, | |
| { | |
| "family": "math", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "hendrycks_math", | |
| "task_id": "T34" | |
| }, | |
| { | |
| "family": "truthfulqa", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "truthfulqa_mc2", | |
| "task_id": "T35" | |
| }, | |
| { | |
| "family": "ifeval", | |
| "language": "EN", | |
| "split": "train", | |
| "task": "ifeval", | |
| "task_id": "T36" | |
| }, | |
| { | |
| "family": "llmzszl", | |
| "language": "PL", | |
| "split": "test", | |
| "task": "n05_llmzszl", | |
| "task_id": "T37" | |
| }, | |
| { | |
| "family": "squad2", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "squadv2", | |
| "task_id": "T38" | |
| }, | |
| { | |
| "family": "drop", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "drop", | |
| "task_id": "T39" | |
| }, | |
| { | |
| "family": "race", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "race", | |
| "task_id": "T40" | |
| }, | |
| { | |
| "family": "coqa", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "coqa", | |
| "task_id": "T41" | |
| }, | |
| { | |
| "family": "nq_open", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "nq_open", | |
| "task_id": "T42" | |
| }, | |
| { | |
| "family": "triviaqa", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "triviaqa", | |
| "task_id": "T43" | |
| }, | |
| { | |
| "family": "pubmedqa", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "pubmedqa", | |
| "task_id": "T44" | |
| }, | |
| { | |
| "family": "medmcqa", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "medmcqa", | |
| "task_id": "T45" | |
| }, | |
| { | |
| "family": "social_iqa", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "social_iqa", | |
| "task_id": "T46" | |
| }, | |
| { | |
| "family": "bbh", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "bbh_fewshot", | |
| "task_id": "T47" | |
| }, | |
| { | |
| "family": "paws", | |
| "language": "EN", | |
| "split": "test", | |
| "task": "paws_en", | |
| "task_id": "T48" | |
| }, | |
| { | |
| "family": "cola", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "cola", | |
| "task_id": "T49" | |
| }, | |
| { | |
| "family": "record", | |
| "language": "EN", | |
| "split": "validation", | |
| "task": "record", | |
| "task_id": "T50" | |
| } | |
| ] | |
| } | |