| { |
| "complete": true, |
| "completed_utc": "2026-10-06T16:58:57.206667+00:00", |
| "model": "Maincode/matilda-jev-fp4", |
| "summary": { |
| "BF16": { |
| "Decision Index": 62.43, |
| "Raw": 71.42, |
| "Breadth": 61.15 |
| }, |
| "FP4": { |
| "Decision Index": 61.77, |
| "Raw": 70.9, |
| "Breadth": 60.48 |
| } |
| }, |
| "areas": [ |
| { |
| "area": "Knowledge & Reasoning", |
| "BF16": 47.37, |
| "FP4": 46.45, |
| "delta_pp": -0.92 |
| }, |
| { |
| "area": "Language Understanding", |
| "BF16": 70.38, |
| "FP4": 69.24, |
| "delta_pp": -1.14 |
| }, |
| { |
| "area": "Retrieval & Classification", |
| "BF16": 64.98, |
| "FP4": 64.41, |
| "delta_pp": -0.57 |
| }, |
| { |
| "area": "Tools & Automation", |
| "BF16": 80.89, |
| "FP4": 80.76, |
| "delta_pp": -0.13 |
| }, |
| { |
| "area": "Arts & Human Taste", |
| "BF16": 41.96, |
| "FP4": 42.05, |
| "delta_pp": 0.09 |
| } |
| ], |
| "benchmarks": [ |
| { |
| "id": "1", |
| "benchmark": "BFCL", |
| "metric": "case exact accuracy", |
| "BF16": 96.93, |
| "FP4": 96.69, |
| "delta_pp": -0.24 |
| }, |
| { |
| "id": "2", |
| "benchmark": "ToolRet", |
| "metric": "nDCG@10", |
| "BF16": 67.36, |
| "FP4": 67.6, |
| "delta_pp": 0.24 |
| }, |
| { |
| "id": "3", |
| "benchmark": "API-Bank", |
| "metric": "accuracy", |
| "BF16": 84.84, |
| "FP4": 84.65, |
| "delta_pp": -0.19 |
| }, |
| { |
| "id": "4", |
| "benchmark": "BANKING77", |
| "metric": "macro-F1", |
| "BF16": 90.09, |
| "FP4": 89.53, |
| "delta_pp": -0.56 |
| }, |
| { |
| "id": "5", |
| "benchmark": "CLINC150+OOS", |
| "metric": "macro-F1", |
| "BF16": 91.07, |
| "FP4": 91.2, |
| "delta_pp": 0.13 |
| }, |
| { |
| "id": "6", |
| "benchmark": "RouterBench", |
| "metric": "selected quality (quality objective)", |
| "BF16": 79.62, |
| "FP4": 79.63, |
| "delta_pp": 0.01 |
| }, |
| { |
| "id": "9", |
| "benchmark": "Home appliance simulator", |
| "metric": "case exact accuracy", |
| "BF16": 76.14, |
| "FP4": 76.14, |
| "delta_pp": 0.0 |
| }, |
| { |
| "id": "10", |
| "benchmark": "SGD/SGD-X", |
| "metric": "macro-F1", |
| "BF16": 51.69, |
| "FP4": 44.19, |
| "delta_pp": -7.5 |
| }, |
| { |
| "id": "11", |
| "benchmark": "ContractNLI", |
| "metric": "macro-F1", |
| "BF16": 83.0, |
| "FP4": 83.23, |
| "delta_pp": 0.23 |
| }, |
| { |
| "id": "12", |
| "benchmark": "ANLI", |
| "metric": "macro-F1", |
| "BF16": 74.9, |
| "FP4": 73.72, |
| "delta_pp": -1.18 |
| }, |
| { |
| "id": "20", |
| "benchmark": "BPoMP", |
| "metric": "accuracy", |
| "BF16": 94.86, |
| "FP4": 94.46, |
| "delta_pp": -0.4 |
| }, |
| { |
| "id": "21", |
| "benchmark": "Humicroedit", |
| "metric": "accuracy", |
| "BF16": 62.56, |
| "FP4": 61.91, |
| "delta_pp": -0.65 |
| }, |
| { |
| "id": "22", |
| "benchmark": "POP909-CL", |
| "metric": "accuracy", |
| "BF16": 50.0, |
| "FP4": 42.2, |
| "delta_pp": -7.8 |
| }, |
| { |
| "id": "23", |
| "benchmark": "cfcolor", |
| "metric": "accuracy", |
| "BF16": 65.18, |
| "FP4": 65.4, |
| "delta_pp": 0.22 |
| }, |
| { |
| "id": "24", |
| "benchmark": "MMLU", |
| "metric": "accuracy", |
| "BF16": 88.89, |
| "FP4": 88.23, |
| "delta_pp": -0.66 |
| }, |
| { |
| "id": "25", |
| "benchmark": "GPQA Diamond", |
| "metric": "accuracy", |
| "BF16": 51.02, |
| "FP4": 48.47, |
| "delta_pp": -2.55 |
| }, |
| { |
| "id": "26", |
| "benchmark": "ARC-Easy", |
| "metric": "accuracy", |
| "BF16": 98.95, |
| "FP4": 98.95, |
| "delta_pp": 0.0 |
| }, |
| { |
| "id": "27", |
| "benchmark": "ARC-Challenge", |
| "metric": "accuracy", |
| "BF16": 96.93, |
| "FP4": 97.01, |
| "delta_pp": 0.08 |
| }, |
| { |
| "id": "28", |
| "benchmark": "WinoGrande", |
| "metric": "accuracy", |
| "BF16": 86.74, |
| "FP4": 86.03, |
| "delta_pp": -0.71 |
| }, |
| { |
| "id": "29", |
| "benchmark": "HellaSwag", |
| "metric": "accuracy", |
| "BF16": 95.7, |
| "FP4": 95.2, |
| "delta_pp": -0.5 |
| }, |
| { |
| "id": "30", |
| "benchmark": "GSM8K", |
| "metric": "accuracy", |
| "BF16": 79.45, |
| "FP4": 79.53, |
| "delta_pp": 0.08 |
| }, |
| { |
| "id": "31", |
| "benchmark": "ChessBench", |
| "metric": "accuracy", |
| "BF16": 21.62, |
| "FP4": 20.7, |
| "delta_pp": -0.92 |
| }, |
| { |
| "id": "32", |
| "benchmark": "MuSR", |
| "metric": "accuracy", |
| "BF16": 67.69, |
| "FP4": 67.42, |
| "delta_pp": -0.27 |
| }, |
| { |
| "id": "33", |
| "benchmark": "SATA-Bench", |
| "metric": "case exact accuracy", |
| "BF16": 23.58, |
| "FP4": 25.58, |
| "delta_pp": 2.0 |
| }, |
| { |
| "id": "34", |
| "benchmark": "SimpleBench", |
| "metric": "accuracy", |
| "BF16": 40.0, |
| "FP4": 30.0, |
| "delta_pp": -10.0 |
| }, |
| { |
| "id": "36", |
| "benchmark": "BRIGHT", |
| "metric": "nDCG@10", |
| "BF16": 49.33, |
| "FP4": 48.51, |
| "delta_pp": -0.82 |
| }, |
| { |
| "id": "37", |
| "benchmark": "Amazon ESCI", |
| "metric": "macro-F1", |
| "BF16": 57.65, |
| "FP4": 58.0, |
| "delta_pp": 0.35 |
| }, |
| { |
| "id": "38", |
| "benchmark": "ACOS", |
| "metric": "per-review F1", |
| "BF16": 43.59, |
| "FP4": 39.29, |
| "delta_pp": -4.3 |
| }, |
| { |
| "id": "39", |
| "benchmark": "FinEntity", |
| "metric": "macro-F1", |
| "BF16": 94.07, |
| "FP4": 94.0, |
| "delta_pp": -0.07 |
| }, |
| { |
| "id": "40", |
| "benchmark": "iSarcasmEval", |
| "metric": "Sarcasm F1 \u00b7 track A, English", |
| "BF16": 64.5, |
| "FP4": 63.64, |
| "delta_pp": -0.86 |
| }, |
| { |
| "id": "41", |
| "benchmark": "VAST", |
| "metric": "macro-F1", |
| "BF16": 80.0, |
| "FP4": 79.3, |
| "delta_pp": -0.7 |
| }, |
| { |
| "id": "42", |
| "benchmark": "NLI4CT", |
| "metric": "macro-F1", |
| "BF16": 84.49, |
| "FP4": 84.11, |
| "delta_pp": -0.38 |
| }, |
| { |
| "id": "43", |
| "benchmark": "CRUXEval", |
| "metric": "accuracy", |
| "BF16": 80.88, |
| "FP4": 78.95, |
| "delta_pp": -1.93 |
| }, |
| { |
| "id": "44", |
| "benchmark": "CLadder", |
| "metric": "accuracy", |
| "BF16": 77.4, |
| "FP4": 77.88, |
| "delta_pp": 0.48 |
| }, |
| { |
| "id": "45", |
| "benchmark": "HLE", |
| "metric": "accuracy", |
| "BF16": 17.37, |
| "FP4": 15.77, |
| "delta_pp": -1.6 |
| }, |
| { |
| "id": "48", |
| "benchmark": "ForecastBench", |
| "metric": "Brier (lower is better)", |
| "BF16": 17.99, |
| "FP4": 17.66, |
| "delta_pp": -0.33 |
| }, |
| { |
| "id": "50", |
| "benchmark": "Habermas Machine", |
| "metric": "accuracy", |
| "BF16": 40.99, |
| "FP4": 44.21, |
| "delta_pp": 3.22 |
| }, |
| { |
| "id": "56", |
| "benchmark": "PhishNChips phishing decisions", |
| "metric": "accuracy", |
| "BF16": 70.6, |
| "FP4": 69.2, |
| "delta_pp": -1.4 |
| }, |
| { |
| "id": "57", |
| "benchmark": "MMLU-Pro", |
| "metric": "accuracy", |
| "BF16": 83.6, |
| "FP4": 81.7, |
| "delta_pp": -1.9 |
| }, |
| { |
| "id": "58", |
| "benchmark": "BBH fixed-option tasks", |
| "metric": "accuracy", |
| "BF16": 79.84, |
| "FP4": 79.54, |
| "delta_pp": -0.3 |
| }, |
| { |
| "id": "59", |
| "benchmark": "RAGTruth response-level hallucination", |
| "metric": "F1 on hallucinated class", |
| "BF16": 84.16, |
| "FP4": 84.05, |
| "delta_pp": -0.11 |
| }, |
| { |
| "id": "61", |
| "benchmark": "HoVer claim verification", |
| "metric": "accuracy", |
| "BF16": 86.2, |
| "FP4": 86.32, |
| "delta_pp": 0.12 |
| }, |
| { |
| "id": "62", |
| "benchmark": "When2Call MCQ", |
| "metric": "accuracy", |
| "BF16": 86.42, |
| "FP4": 86.12, |
| "delta_pp": -0.3 |
| }, |
| { |
| "id": "64", |
| "benchmark": "New Yorker caption matching", |
| "metric": "accuracy", |
| "BF16": 68.75, |
| "FP4": 70.83, |
| "delta_pp": 2.08 |
| } |
| ], |
| "source_DI_reused": true, |
| "requests": 150317, |
| "format": "FP4 weights, BF16 activations/GEMM (W4A16)", |
| "benchmark_exposed_diagnostic": true, |
| "serving_changed": false, |
| "paid_api_calls": 0 |
| } |
|
|
