{ "complete": true, "completed_utc": "2026-10-06T16:58:57.206667+00:00", "model": "Maincode/matilda-jev-fp4", "summary": { "BF16": { "Decision Index": 62.43, "Raw": 71.42, "Breadth": 61.15 }, "FP4": { "Decision Index": 61.77, "Raw": 70.9, "Breadth": 60.48 } }, "areas": [ { "area": "Knowledge & Reasoning", "BF16": 47.37, "FP4": 46.45, "delta_pp": -0.92 }, { "area": "Language Understanding", "BF16": 70.38, "FP4": 69.24, "delta_pp": -1.14 }, { "area": "Retrieval & Classification", "BF16": 64.98, "FP4": 64.41, "delta_pp": -0.57 }, { "area": "Tools & Automation", "BF16": 80.89, "FP4": 80.76, "delta_pp": -0.13 }, { "area": "Arts & Human Taste", "BF16": 41.96, "FP4": 42.05, "delta_pp": 0.09 } ], "benchmarks": [ { "id": "1", "benchmark": "BFCL", "metric": "case exact accuracy", "BF16": 96.93, "FP4": 96.69, "delta_pp": -0.24 }, { "id": "2", "benchmark": "ToolRet", "metric": "nDCG@10", "BF16": 67.36, "FP4": 67.6, "delta_pp": 0.24 }, { "id": "3", "benchmark": "API-Bank", "metric": "accuracy", "BF16": 84.84, "FP4": 84.65, "delta_pp": -0.19 }, { "id": "4", "benchmark": "BANKING77", "metric": "macro-F1", "BF16": 90.09, "FP4": 89.53, "delta_pp": -0.56 }, { "id": "5", "benchmark": "CLINC150+OOS", "metric": "macro-F1", "BF16": 91.07, "FP4": 91.2, "delta_pp": 0.13 }, { "id": "6", "benchmark": "RouterBench", "metric": "selected quality (quality objective)", "BF16": 79.62, "FP4": 79.63, "delta_pp": 0.01 }, { "id": "9", "benchmark": "Home appliance simulator", "metric": "case exact accuracy", "BF16": 76.14, "FP4": 76.14, "delta_pp": 0.0 }, { "id": "10", "benchmark": "SGD/SGD-X", "metric": "macro-F1", "BF16": 51.69, "FP4": 44.19, "delta_pp": -7.5 }, { "id": "11", "benchmark": "ContractNLI", "metric": "macro-F1", "BF16": 83.0, "FP4": 83.23, "delta_pp": 0.23 }, { "id": "12", "benchmark": "ANLI", "metric": "macro-F1", "BF16": 74.9, "FP4": 73.72, "delta_pp": -1.18 }, { "id": "20", "benchmark": "BPoMP", "metric": "accuracy", "BF16": 94.86, "FP4": 94.46, "delta_pp": -0.4 }, { "id": "21", "benchmark": "Humicroedit", "metric": "accuracy", "BF16": 62.56, "FP4": 61.91, "delta_pp": -0.65 }, { "id": "22", "benchmark": "POP909-CL", "metric": "accuracy", "BF16": 50.0, "FP4": 42.2, "delta_pp": -7.8 }, { "id": "23", "benchmark": "cfcolor", "metric": "accuracy", "BF16": 65.18, "FP4": 65.4, "delta_pp": 0.22 }, { "id": "24", "benchmark": "MMLU", "metric": "accuracy", "BF16": 88.89, "FP4": 88.23, "delta_pp": -0.66 }, { "id": "25", "benchmark": "GPQA Diamond", "metric": "accuracy", "BF16": 51.02, "FP4": 48.47, "delta_pp": -2.55 }, { "id": "26", "benchmark": "ARC-Easy", "metric": "accuracy", "BF16": 98.95, "FP4": 98.95, "delta_pp": 0.0 }, { "id": "27", "benchmark": "ARC-Challenge", "metric": "accuracy", "BF16": 96.93, "FP4": 97.01, "delta_pp": 0.08 }, { "id": "28", "benchmark": "WinoGrande", "metric": "accuracy", "BF16": 86.74, "FP4": 86.03, "delta_pp": -0.71 }, { "id": "29", "benchmark": "HellaSwag", "metric": "accuracy", "BF16": 95.7, "FP4": 95.2, "delta_pp": -0.5 }, { "id": "30", "benchmark": "GSM8K", "metric": "accuracy", "BF16": 79.45, "FP4": 79.53, "delta_pp": 0.08 }, { "id": "31", "benchmark": "ChessBench", "metric": "accuracy", "BF16": 21.62, "FP4": 20.7, "delta_pp": -0.92 }, { "id": "32", "benchmark": "MuSR", "metric": "accuracy", "BF16": 67.69, "FP4": 67.42, "delta_pp": -0.27 }, { "id": "33", "benchmark": "SATA-Bench", "metric": "case exact accuracy", "BF16": 23.58, "FP4": 25.58, "delta_pp": 2.0 }, { "id": "34", "benchmark": "SimpleBench", "metric": "accuracy", "BF16": 40.0, "FP4": 30.0, "delta_pp": -10.0 }, { "id": "36", "benchmark": "BRIGHT", "metric": "nDCG@10", "BF16": 49.33, "FP4": 48.51, "delta_pp": -0.82 }, { "id": "37", "benchmark": "Amazon ESCI", "metric": "macro-F1", "BF16": 57.65, "FP4": 58.0, "delta_pp": 0.35 }, { "id": "38", "benchmark": "ACOS", "metric": "per-review F1", "BF16": 43.59, "FP4": 39.29, "delta_pp": -4.3 }, { "id": "39", "benchmark": "FinEntity", "metric": "macro-F1", "BF16": 94.07, "FP4": 94.0, "delta_pp": -0.07 }, { "id": "40", "benchmark": "iSarcasmEval", "metric": "Sarcasm F1 \u00b7 track A, English", "BF16": 64.5, "FP4": 63.64, "delta_pp": -0.86 }, { "id": "41", "benchmark": "VAST", "metric": "macro-F1", "BF16": 80.0, "FP4": 79.3, "delta_pp": -0.7 }, { "id": "42", "benchmark": "NLI4CT", "metric": "macro-F1", "BF16": 84.49, "FP4": 84.11, "delta_pp": -0.38 }, { "id": "43", "benchmark": "CRUXEval", "metric": "accuracy", "BF16": 80.88, "FP4": 78.95, "delta_pp": -1.93 }, { "id": "44", "benchmark": "CLadder", "metric": "accuracy", "BF16": 77.4, "FP4": 77.88, "delta_pp": 0.48 }, { "id": "45", "benchmark": "HLE", "metric": "accuracy", "BF16": 17.37, "FP4": 15.77, "delta_pp": -1.6 }, { "id": "48", "benchmark": "ForecastBench", "metric": "Brier (lower is better)", "BF16": 17.99, "FP4": 17.66, "delta_pp": -0.33 }, { "id": "50", "benchmark": "Habermas Machine", "metric": "accuracy", "BF16": 40.99, "FP4": 44.21, "delta_pp": 3.22 }, { "id": "56", "benchmark": "PhishNChips phishing decisions", "metric": "accuracy", "BF16": 70.6, "FP4": 69.2, "delta_pp": -1.4 }, { "id": "57", "benchmark": "MMLU-Pro", "metric": "accuracy", "BF16": 83.6, "FP4": 81.7, "delta_pp": -1.9 }, { "id": "58", "benchmark": "BBH fixed-option tasks", "metric": "accuracy", "BF16": 79.84, "FP4": 79.54, "delta_pp": -0.3 }, { "id": "59", "benchmark": "RAGTruth response-level hallucination", "metric": "F1 on hallucinated class", "BF16": 84.16, "FP4": 84.05, "delta_pp": -0.11 }, { "id": "61", "benchmark": "HoVer claim verification", "metric": "accuracy", "BF16": 86.2, "FP4": 86.32, "delta_pp": 0.12 }, { "id": "62", "benchmark": "When2Call MCQ", "metric": "accuracy", "BF16": 86.42, "FP4": 86.12, "delta_pp": -0.3 }, { "id": "64", "benchmark": "New Yorker caption matching", "metric": "accuracy", "BF16": 68.75, "FP4": 70.83, "delta_pp": 2.08 } ], "source_DI_reused": true, "requests": 150317, "format": "FP4 weights, BF16 activations/GEMM (W4A16)", "benchmark_exposed_diagnostic": true, "serving_changed": false, "paid_api_calls": 0 }