Spaces:
Running
Running
Download data.json from ritik12/WaterfallBench: direct link, hf CLI and curl.
- Browser
- Download file 24.5 kB
-
https://huggingface.co/spaces/ritik12/WaterfallBench/resolve/main/data.json
- Command line
-
hf download hf://spaces/ritik12/WaterfallBench/data.json
-
curl -L -o data.json https://huggingface.co/spaces/ritik12/WaterfallBench/resolve/main/data.json
24.5 kB
{
"name": "WaterfallBench",
"version": "1.0",
"updated": "2026-11-15",
"harness": "single call, no tools",
"models": [
{
"key": "openai/gpt-6-astra",
"name": "GPT-6 Astra",
"lab": "OpenAI",
"tasks": 20,
"fully_resolved": 95.0,
"se": 4.87,
"nontrivial": 99.17,
"line_item": 99.58,
"tolerance": 100.0,
"cost": 8.3326,
"cost_total": 166.65,
"tokens": 409597,
"input_tokens": 407039,
"output_tokens": 2558,
"seconds": 64.2,
"results": {
"fw-deere-2026-b-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-2-2025-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-3-2025-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-4-2025-10": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-1-2026-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-2-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-3-2026-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2025-b-2025-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2026-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-a-2025-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-b-2025-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2026-a-2026-02": {
"ok": false,
"nt": 83.3,
"li": 91.7
},
"fw-nissanlease-2026-b-2026-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-a-2025-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-b-2025-05": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-c-2025-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-d-2025-10": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2026-a-2026-02": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2026-b-2026-05": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2026-c-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
}
}
},
{
"key": "anthropic/claude-fable-5-2",
"name": "Claude Fable 5.2",
"lab": "Anthropic",
"tasks": 20,
"fully_resolved": 95.0,
"se": 4.87,
"nontrivial": 99.09,
"line_item": 99.58,
"tolerance": 100.0,
"cost": 5.1818,
"cost_total": 103.64,
"tokens": 635508,
"input_tokens": 632454,
"output_tokens": 3053,
"seconds": 35.4,
"results": {
"fw-carmax-2025-2-2025-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-3-2025-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-4-2025-10": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-1-2026-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-2-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-3-2026-07": {
"ok": false,
"nt": 81.8,
"li": 91.7
},
"fw-deere-2025-b-2025-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2026-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2026-b-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-a-2025-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-b-2025-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2026-a-2026-02": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2026-b-2026-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-a-2025-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-b-2025-05": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-c-2025-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-d-2025-10": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2026-a-2026-02": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2026-b-2026-05": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2026-c-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
}
}
},
{
"key": "openai/gpt-6-luna",
"name": "GPT-6 Luna",
"lab": "OpenAI",
"tasks": 20,
"fully_resolved": 90.0,
"se": 6.71,
"nontrivial": 99.17,
"line_item": 98.68,
"tolerance": 98.68,
"cost": 0.0873,
"cost_total": 1.75,
"tokens": 414850,
"input_tokens": 407039,
"output_tokens": 7811,
"seconds": 86.6,
"results": {
"fw-carmax-2025-2-2025-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-3-2025-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-4-2025-10": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-1-2026-01": {
"ok": false,
"nt": 83.3,
"li": 92.6
},
"fw-carmax-2026-2-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-3-2026-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2025-b-2025-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2026-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2026-b-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-a-2025-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-b-2025-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2026-a-2026-02": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2026-b-2026-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-a-2025-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-b-2025-05": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-c-2025-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-d-2025-10": {
"ok": false,
"nt": 100.0,
"li": 81.0
},
"fw-worldomni-2026-a-2026-02": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2026-b-2026-05": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2026-c-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
}
}
},
{
"key": "fireworks/glm-5p3",
"name": "GLM 5.3",
"lab": "Z.ai",
"tasks": 20,
"fully_resolved": 75.0,
"se": 9.68,
"nontrivial": 95.65,
"line_item": 98.04,
"tolerance": 98.45,
"cost": 0.6811,
"cost_total": 13.62,
"tokens": 441386,
"input_tokens": 420329,
"output_tokens": 21057,
"seconds": 160.9,
"results": {
"fw-deere-2026-b-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-2-2025-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-3-2025-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-4-2025-10": {
"ok": false,
"nt": 83.3,
"li": 92.6
},
"fw-carmax-2026-1-2026-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-2-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-3-2026-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2025-b-2025-08": {
"ok": false,
"nt": 85.7,
"li": 93.8
},
"fw-deere-2026-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-a-2025-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-b-2025-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2026-a-2026-02": {
"ok": false,
"nt": 83.3,
"li": 91.7
},
"fw-nissanlease-2026-b-2026-07": {
"ok": false,
"nt": 75.0,
"li": 87.5
},
"fw-worldomni-2025-a-2025-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-b-2025-05": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-c-2025-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-d-2025-10": {
"ok": false,
"nt": 85.7,
"li": 95.2
},
"fw-worldomni-2026-a-2026-02": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2026-b-2026-05": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2026-c-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
}
}
},
{
"key": "fireworks/minimax-m3p1-flash",
"name": "MiniMax M3.1 Flash",
"lab": "MiniMax",
"tasks": 20,
"fully_resolved": 50.0,
"se": 11.18,
"nontrivial": 89.03,
"line_item": 89.15,
"tolerance": 90.14,
"cost": 0.0706,
"cost_total": 1.41,
"tokens": 418289,
"input_tokens": 400871,
"output_tokens": 17419,
"seconds": 105.9,
"results": {
"fw-carmax-2025-2-2025-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-3-2025-07": {
"ok": false,
"nt": 0.0,
"li": 0.0
},
"fw-carmax-2025-4-2025-10": {
"ok": false,
"nt": 83.3,
"li": 92.6
},
"fw-carmax-2026-1-2026-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-2-2026-04": {
"ok": false,
"nt": 83.3,
"li": 92.6
},
"fw-carmax-2026-3-2026-07": {
"ok": false,
"nt": 72.7,
"li": 87.5
},
"fw-deere-2025-b-2025-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2026-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2026-b-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-a-2025-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-b-2025-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2026-a-2026-02": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2026-b-2026-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-a-2025-01": {
"ok": false,
"nt": 85.7,
"li": 95.2
},
"fw-worldomni-2025-b-2025-05": {
"ok": false,
"nt": 100.0,
"li": 95.2
},
"fw-worldomni-2025-c-2025-08": {
"ok": false,
"nt": 100.0,
"li": 80.0
},
"fw-worldomni-2025-d-2025-10": {
"ok": false,
"nt": 100.0,
"li": 76.2
},
"fw-worldomni-2026-a-2026-02": {
"ok": false,
"nt": 77.8,
"li": 72.7
},
"fw-worldomni-2026-b-2026-05": {
"ok": false,
"nt": 77.8,
"li": 90.9
},
"fw-worldomni-2026-c-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
}
}
},
{
"key": "fireworks/kimi-k3p1-pro",
"name": "Kimi K3.1 Pro",
"lab": "Moonshot AI",
"tasks": 20,
"fully_resolved": 45.0,
"se": 11.12,
"nontrivial": 92.32,
"line_item": 96.1,
"tolerance": 96.1,
"cost": 0.8904,
"cost_total": 17.81,
"tokens": 418687,
"input_tokens": 412055,
"output_tokens": 6632,
"seconds": 106.6,
"results": {
"fw-deere-2026-b-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-2-2025-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-3-2025-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-4-2025-10": {
"ok": false,
"nt": 75.0,
"li": 88.9
},
"fw-carmax-2026-1-2026-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-2-2026-04": {
"ok": false,
"nt": 83.3,
"li": 92.6
},
"fw-carmax-2026-3-2026-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2025-b-2025-08": {
"ok": false,
"nt": 85.7,
"li": 93.8
},
"fw-deere-2026-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-a-2025-01": {
"ok": false,
"nt": 83.3,
"li": 91.7
},
"fw-nissanlease-2025-b-2025-07": {
"ok": false,
"nt": 91.7,
"li": 95.8
},
"fw-nissanlease-2026-a-2026-02": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2026-b-2026-07": {
"ok": false,
"nt": 75.0,
"li": 87.5
},
"fw-worldomni-2025-a-2025-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-b-2025-05": {
"ok": false,
"nt": 100.0,
"li": 95.2
},
"fw-worldomni-2025-c-2025-08": {
"ok": false,
"nt": 85.7,
"li": 95.0
},
"fw-worldomni-2025-d-2025-10": {
"ok": false,
"nt": 100.0,
"li": 95.2
},
"fw-worldomni-2026-a-2026-02": {
"ok": false,
"nt": 88.9,
"li": 95.5
},
"fw-worldomni-2026-b-2026-05": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2026-c-2026-08": {
"ok": false,
"nt": 77.8,
"li": 90.9
}
}
},
{
"key": "anthropic/claude-haiku-5-5",
"name": "Claude Haiku 5.5",
"lab": "Anthropic",
"tasks": 20,
"fully_resolved": 40.0,
"se": 10.95,
"nontrivial": 85.78,
"line_item": 93.83,
"tolerance": 95.73,
"cost": 0.343,
"cost_total": 6.86,
"tokens": 643169,
"input_tokens": 632454,
"output_tokens": 10714,
"seconds": 58.7,
"results": {
"fw-carmax-2025-2-2025-04": {
"ok": false,
"nt": 88.9,
"li": 95.8
},
"fw-carmax-2025-3-2025-07": {
"ok": false,
"nt": 83.3,
"li": 92.6
},
"fw-carmax-2025-4-2025-10": {
"ok": false,
"nt": 83.3,
"li": 92.6
},
"fw-carmax-2026-1-2026-01": {
"ok": false,
"nt": 75.0,
"li": 88.9
},
"fw-carmax-2026-2-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-3-2026-07": {
"ok": false,
"nt": 72.7,
"li": 87.5
},
"fw-deere-2025-b-2025-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2026-2026-04": {
"ok": false,
"nt": 85.7,
"li": 93.8
},
"fw-deere-2026-b-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-a-2025-01": {
"ok": false,
"nt": 75.0,
"li": 87.5
},
"fw-nissanlease-2025-b-2025-07": {
"ok": false,
"nt": 75.0,
"li": 87.5
},
"fw-nissanlease-2026-a-2026-02": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2026-b-2026-07": {
"ok": false,
"nt": 75.0,
"li": 87.5
},
"fw-worldomni-2025-a-2025-01": {
"ok": false,
"nt": 57.1,
"li": 85.7
},
"fw-worldomni-2025-b-2025-05": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-c-2025-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-d-2025-10": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2026-a-2026-02": {
"ok": false,
"nt": 88.9,
"li": 95.5
},
"fw-worldomni-2026-b-2026-05": {
"ok": false,
"nt": 55.6,
"li": 81.8
},
"fw-worldomni-2026-c-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
}
}
}
],
"tasks": [
{
"id": "fw-carmax-2025-2-2025-04",
"family": "carmax",
"family_name": "CarMax Auto Owner Trust",
"deal": "CarMax Auto Owner Trust 2025-2",
"period": "202504",
"classes": 8,
"graded": 24,
"nontrivial": 9,
"ungraded": 3,
"ungraded_why": "Servicer report departs from the agreement's overcollateralization target",
"docs": 7,
"html_mb": 4.91,
"prompt_tokens": 629698,
"distribution_date": "2025-05-15"
},
{
"id": "fw-carmax-2025-3-2025-07",
"family": "carmax",
"family_name": "CarMax Auto Owner Trust",
"deal": "CarMax Auto Owner Trust 2025-3",
"period": "202507",
"classes": 8,
"graded": 27,
"nontrivial": 12,
"ungraded": 0,
"ungraded_why": null,
"docs": 7,
"html_mb": 5.01,
"prompt_tokens": 632551,
"distribution_date": "2025-08-15"
},
{
"id": "fw-carmax-2025-4-2025-10",
"family": "carmax",
"family_name": "CarMax Auto Owner Trust",
"deal": "CarMax Auto Owner Trust 2025-4",
"period": "202510",
"classes": 8,
"graded": 27,
"nontrivial": 12,
"ungraded": 0,
"ungraded_why": null,
"docs": 7,
"html_mb": 4.99,
"prompt_tokens": 633700,
"distribution_date": "2025-11-17"
},
{
"id": "fw-carmax-2026-1-2026-01",
"family": "carmax",
"family_name": "CarMax Auto Owner Trust",
"deal": "CarMax Auto Owner Trust 2026-1",
"period": "202601",
"classes": 8,
"graded": 27,
"nontrivial": 12,
"ungraded": 0,
"ungraded_why": null,
"docs": 8,
"html_mb": 5.08,
"prompt_tokens": 647530,
"distribution_date": "2026-02-17"
},
{
"id": "fw-carmax-2026-2-2026-04",
"family": "carmax",
"family_name": "CarMax Auto Owner Trust",
"deal": "CarMax Auto Owner Trust 2026-2",
"period": "202604",
"classes": 8,
"graded": 27,
"nontrivial": 12,
"ungraded": 0,
"ungraded_why": null,
"docs": 8,
"html_mb": 5.12,
"prompt_tokens": 647909,
"distribution_date": "2026-05-15"
},
{
"id": "fw-carmax-2026-3-2026-07",
"family": "carmax",
"family_name": "CarMax Auto Owner Trust",
"deal": "CarMax Auto Owner Trust 2026-3",
"period": "202607",
"classes": 7,
"graded": 24,
"nontrivial": 11,
"ungraded": 0,
"ungraded_why": null,
"docs": 8,
"html_mb": 4.94,
"prompt_tokens": 614998,
"distribution_date": "2026-08-17"
},
{
"id": "fw-deere-2025-b-2025-08",
"family": "deere",
"family_name": "John Deere Owner Trust",
"deal": "John Deere Owner Trust 2025-B",
"period": "202508",
"classes": 5,
"graded": 16,
"nontrivial": 7,
"ungraded": 2,
"ungraded_why": "First servicing fee needs a mid-period pool balance no document provides",
"docs": 6,
"html_mb": 7.76,
"prompt_tokens": 503783,
"distribution_date": "2025-08-15"
},
{
"id": "fw-deere-2026-2026-04",
"family": "deere",
"family_name": "John Deere Owner Trust",
"deal": "John Deere Owner Trust 2026",
"period": "202604",
"classes": 5,
"graded": 16,
"nontrivial": 7,
"ungraded": 2,
"ungraded_why": "First servicing fee needs a mid-period pool balance no document provides",
"docs": 6,
"html_mb": 7.52,
"prompt_tokens": 501473,
"distribution_date": "2026-04-15"
},
{
"id": "fw-deere-2026-b-2026-08",
"family": "deere",
"family_name": "John Deere Owner Trust",
"deal": "John Deere Owner Trust 2026-B",
"period": "202608",
"classes": 5,
"graded": 16,
"nontrivial": 7,
"ungraded": 2,
"ungraded_why": "First servicing fee needs a mid-period pool balance no document provides",
"docs": 7,
"html_mb": 7.67,
"prompt_tokens": 522651,
"distribution_date": "2026-08-17"
},
{
"id": "fw-nissanlease-2025-a-2025-01",
"family": "nissanlease",
"family_name": "Nissan Auto Lease Trust",
"deal": "Nissan Auto Lease Trust 2025-A",
"period": "202501",
"classes": 7,
"graded": 24,
"nontrivial": 12,
"ungraded": 0,
"ungraded_why": null,
"docs": 10,
"html_mb": 8.23,
"prompt_tokens": 725064,
"distribution_date": "2025-02-18"
},
{
"id": "fw-nissanlease-2025-b-2025-07",
"family": "nissanlease",
"family_name": "Nissan Auto Lease Trust",
"deal": "Nissan Auto Lease Trust 2025-B",
"period": "202507",
"classes": 7,
"graded": 24,
"nontrivial": 12,
"ungraded": 0,
"ungraded_why": null,
"docs": 10,
"html_mb": 7.76,
"prompt_tokens": 712865,
"distribution_date": "2025-08-15"
},
{
"id": "fw-nissanlease-2026-a-2026-02",
"family": "nissanlease",
"family_name": "Nissan Auto Lease Trust",
"deal": "Nissan Auto Lease Trust 2026-A",
"period": "202602",
"classes": 7,
"graded": 24,
"nontrivial": 12,
"ungraded": 0,
"ungraded_why": null,
"docs": 10,
"html_mb": 8.48,
"prompt_tokens": 739161,
"distribution_date": "2026-03-16"
},
{
"id": "fw-nissanlease-2026-b-2026-07",
"family": "nissanlease",
"family_name": "Nissan Auto Lease Trust",
"deal": "Nissan Auto Lease Trust 2026-B",
"period": "202607",
"classes": 7,
"graded": 24,
"nontrivial": 12,
"ungraded": 0,
"ungraded_why": null,
"docs": 10,
"html_mb": 8.2,
"prompt_tokens": 732825,
"distribution_date": "2026-08-17"
},
{
"id": "fw-worldomni-2025-a-2025-01",
"family": "worldomni",
"family_name": "World Omni Auto Receivables Trust",
"deal": "World Omni Auto Receivables Trust 2025-A",
"period": "202501",
"classes": 7,
"graded": 21,
"nontrivial": 7,
"ungraded": 3,
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
"docs": 7,
"html_mb": 6.37,
"prompt_tokens": 614546,
"distribution_date": "2025-02-18"
},
{
"id": "fw-worldomni-2025-b-2025-05",
"family": "worldomni",
"family_name": "World Omni Auto Receivables Trust",
"deal": "World Omni Auto Receivables Trust 2025-B",
"period": "202505",
"classes": 7,
"graded": 21,
"nontrivial": 5,
"ungraded": 3,
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
"docs": 7,
"html_mb": 6.35,
"prompt_tokens": 624079,
"distribution_date": "2025-06-16"
},
{
"id": "fw-worldomni-2025-c-2025-08",
"family": "worldomni",
"family_name": "World Omni Auto Receivables Trust",
"deal": "World Omni Auto Receivables Trust 2025-C",
"period": "202508",
"classes": 7,
"graded": 20,
"nontrivial": 7,
"ungraded": 4,
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
"docs": 7,
"html_mb": 6.58,
"prompt_tokens": 629893,
"distribution_date": "2025-09-15"
},
{
"id": "fw-worldomni-2025-d-2025-10",
"family": "worldomni",
"family_name": "World Omni Auto Receivables Trust",
"deal": "World Omni Auto Receivables Trust 2025-D",
"period": "202510",
"classes": 7,
"graded": 21,
"nontrivial": 7,
"ungraded": 3,
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
"docs": 7,
"html_mb": 8.05,
"prompt_tokens": 638804,
"distribution_date": "2025-11-17"
},
{
"id": "fw-worldomni-2026-a-2026-02",
"family": "worldomni",
"family_name": "World Omni Auto Receivables Trust",
"deal": "World Omni Auto Receivables Trust 2026-A",
"period": "202602",
"classes": 7,
"graded": 22,
"nontrivial": 9,
"ungraded": 2,
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
"docs": 7,
"html_mb": 6.98,
"prompt_tokens": 627265,
"distribution_date": "2026-03-16"
},
{
"id": "fw-worldomni-2026-b-2026-05",
"family": "worldomni",
"family_name": "World Omni Auto Receivables Trust",
"deal": "World Omni Auto Receivables Trust 2026-B",
"period": "202605",
"classes": 7,
"graded": 22,
"nontrivial": 9,
"ungraded": 2,
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
"docs": 7,
"html_mb": 6.35,
"prompt_tokens": 633802,
"distribution_date": "2026-06-15"
},
{
"id": "fw-worldomni-2026-c-2026-08",
"family": "worldomni",
"family_name": "World Omni Auto Receivables Trust",
"deal": "World Omni Auto Receivables Trust 2026-C",
"period": "202608",
"classes": 7,
"graded": 22,
"nontrivial": 9,
"ungraded": 2,
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
"docs": 7,
"html_mb": 7.76,
"prompt_tokens": 636492,
"distribution_date": "2026-09-15"
}
]
}
{
"name": "WaterfallBench",
"version": "1.0",
"updated": "2026-11-15",
"harness": "single call, no tools",
"models": [
{
"key": "openai/gpt-6-astra",
"name": "GPT-6 Astra",
"lab": "OpenAI",
"tasks": 20,
"fully_resolved": 95.0,
"se": 4.87,
"nontrivial": 99.17,
"line_item": 99.58,
"tolerance": 100.0,
"cost": 8.3326,
"cost_total": 166.65,
"tokens": 409597,
"input_tokens": 407039,
"output_tokens": 2558,
"seconds": 64.2,
"results": {
"fw-deere-2026-b-2026-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-2-2025-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-3-2025-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2025-4-2025-10": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-1-2026-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-2-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-carmax-2026-3-2026-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2025-b-2025-08": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-deere-2026-2026-04": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-a-2025-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2025-b-2025-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-nissanlease-2026-a-2026-02": {
"ok": false,
"nt": 83.3,
"li": 91.7
},
"fw-nissanlease-2026-b-2026-07": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-a-2025-01": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-b-2025-05": {
"ok": true,
"nt": 100.0,
"li": 100.0
},
"fw-worldomni-2025-c-2025-08": {
"ok": true,