| { |
| "benchmark": "CMDB-1500", |
| "adapter_sha256": "2982cdb8e60cec22b45da33720cd7b397381d910a12d4773be199493e2b400a4", |
| "backend": "vLLM", |
| "base_precision": "AWQ INT4", |
| "effort": 2, |
| "full": { |
| "n": 1500, |
| "valid": 1500, |
| "accuracy_percent": 81.2 |
| }, |
| "text": { |
| "n": 1200, |
| "accuracy_percent": 79.33333333333333 |
| }, |
| "images": { |
| "n": 300, |
| "accuracy_percent": 88.66666666666667 |
| }, |
| "provenance": "CMDB-1500 results for the adapter SHA and configuration specified here.", |
| "website_reported_evaluations": { |
| "source_url": "https://surdai.com/zh/blog/simplex-cd-release#cd-latest-results", |
| "checked_date": "2026-10-09", |
| "provenance": "Results reported by the linked SurdAI release page using its evaluation serving configurations.", |
| "jevbench_public231": [ |
| { |
| "effort": 1, |
| "n": 231, |
| "correct": 207, |
| "accuracy_percent": 89.6103896103896 |
| }, |
| { |
| "effort": 2, |
| "n": 231, |
| "correct": 208, |
| "accuracy_percent": 90.04329004329004 |
| } |
| ], |
| "jevbench_hard111": [ |
| { |
| "effort": 1, |
| "n": 111, |
| "correct": 88, |
| "accuracy_percent": 79.27927927927928 |
| }, |
| { |
| "effort": 2, |
| "n": 111, |
| "correct": 89, |
| "accuracy_percent": 80.18018018018019 |
| } |
| ], |
| "hard_is_subset_of_public": true, |
| "decision_index": { |
| "edition": "0.2.1", |
| "effort": 1, |
| "score": 58.3 |
| }, |
| "ece_imported": false |
| } |
| } |
|
|
