{
  "examples": 1196,
  "seconds": 17.66167116165161,
  "decoding": "raw greedy, no repair, no teacher fallback",
  "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692",
  "sql_metric": "Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.",
  "metrics": {
    "json_valid": {
      "correct": 1196,
      "total": 1196,
      "rate": 1.0
    },
    "schema_valid": {
      "correct": 1191,
      "total": 1196,
      "rate": 0.995819397993311
    },
    "action_correct": {
      "correct": 1192,
      "total": 1196,
      "rate": 0.9966555183946488
    },
    "tool_correct": {
      "correct": 1088,
      "total": 1100,
      "rate": 0.9890909090909091
    },
    "arguments_exact": {
      "correct": 1014,
      "total": 1100,
      "rate": 0.9218181818181819
    },
    "sql_equivalent": {
      "correct": 896,
      "total": 912,
      "rate": 0.9824561403508771
    },
    "success": {
      "correct": 1106,
      "total": 1196,
      "rate": 0.9247491638795987
    }
  },
  "groups": {
    "language:noisy_en": {
      "json_valid": {
        "correct": 299,
        "total": 299,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 298,
        "total": 299,
        "rate": 0.9966555183946488
      },
      "action_correct": {
        "correct": 299,
        "total": 299,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 274,
        "total": 275,
        "rate": 0.9963636363636363
      },
      "arguments_exact": {
        "correct": 252,
        "total": 275,
        "rate": 0.9163636363636364
      },
      "sql_equivalent": {
        "correct": 228,
        "total": 228,
        "rate": 1.0
      },
      "success": {
        "correct": 276,
        "total": 299,
        "rate": 0.9230769230769231
      }
    },
    "backend:supabase": {
      "json_valid": {
        "correct": 588,
        "total": 588,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 588,
        "total": 588,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 588,
        "total": 588,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 541,
        "total": 544,
        "rate": 0.9944852941176471
      },
      "arguments_exact": {
        "correct": 480,
        "total": 544,
        "rate": 0.8823529411764706
      },
      "sql_equivalent": {
        "correct": 429,
        "total": 436,
        "rate": 0.9839449541284404
      },
      "success": {
        "correct": 524,
        "total": 588,
        "rate": 0.891156462585034
      }
    },
    "operation:missing_schema": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 20,
        "total": 24,
        "rate": 0.8333333333333334
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 20,
        "total": 24,
        "rate": 0.8333333333333334
      }
    },
    "language:en": {
      "json_valid": {
        "correct": 299,
        "total": 299,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 298,
        "total": 299,
        "rate": 0.9966555183946488
      },
      "action_correct": {
        "correct": 297,
        "total": 299,
        "rate": 0.9933110367892977
      },
      "tool_correct": {
        "correct": 274,
        "total": 275,
        "rate": 0.9963636363636363
      },
      "arguments_exact": {
        "correct": 246,
        "total": 275,
        "rate": 0.8945454545454545
      },
      "sql_equivalent": {
        "correct": 222,
        "total": 228,
        "rate": 0.9736842105263158
      },
      "success": {
        "correct": 268,
        "total": 299,
        "rate": 0.8963210702341137
      }
    },
    "language:hinglish": {
      "json_valid": {
        "correct": 299,
        "total": 299,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 297,
        "total": 299,
        "rate": 0.9933110367892977
      },
      "action_correct": {
        "correct": 297,
        "total": 299,
        "rate": 0.9933110367892977
      },
      "tool_correct": {
        "correct": 273,
        "total": 275,
        "rate": 0.9927272727272727
      },
      "arguments_exact": {
        "correct": 262,
        "total": 275,
        "rate": 0.9527272727272728
      },
      "sql_equivalent": {
        "correct": 228,
        "total": 228,
        "rate": 1.0
      },
      "success": {
        "correct": 284,
        "total": 299,
        "rate": 0.9498327759197325
      }
    },
    "operation:sql_error": {
      "json_valid": {
        "correct": 20,
        "total": 20,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 20,
        "total": 20,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 20,
        "total": 20,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 20,
        "total": 20,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 12,
        "total": 20,
        "rate": 0.6
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 12,
        "total": 20,
        "rate": 0.6
      }
    },
    "backend:mysql": {
      "json_valid": {
        "correct": 608,
        "total": 608,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 603,
        "total": 608,
        "rate": 0.9917763157894737
      },
      "action_correct": {
        "correct": 604,
        "total": 608,
        "rate": 0.993421052631579
      },
      "tool_correct": {
        "correct": 547,
        "total": 556,
        "rate": 0.9838129496402878
      },
      "arguments_exact": {
        "correct": 534,
        "total": 556,
        "rate": 0.960431654676259
      },
      "sql_equivalent": {
        "correct": 467,
        "total": 476,
        "rate": 0.9810924369747899
      },
      "success": {
        "correct": 582,
        "total": 608,
        "rate": 0.9572368421052632
      }
    },
    "operation:list_tables": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 12,
        "total": 24,
        "rate": 0.5
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 12,
        "total": 24,
        "rate": 0.5
      }
    },
    "operation:all": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "operation:count": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 23,
        "total": 28,
        "rate": 0.8214285714285714
      },
      "sql_equivalent": {
        "correct": 23,
        "total": 28,
        "rate": 0.8214285714285714
      },
      "success": {
        "correct": 23,
        "total": 28,
        "rate": 0.8214285714285714
      }
    },
    "operation:avg": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      }
    },
    "operation:describe": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 23,
        "total": 24,
        "rate": 0.9583333333333334
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 23,
        "total": 24,
        "rate": 0.9583333333333334
      },
      "arguments_exact": {
        "correct": 19,
        "total": 24,
        "rate": 0.7916666666666666
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 19,
        "total": 24,
        "rate": 0.7916666666666666
      }
    },
    "operation:project": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "operation:gte": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      }
    },
    "operation:min": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 24,
        "total": 28,
        "rate": 0.8571428571428571
      },
      "sql_equivalent": {
        "correct": 24,
        "total": 28,
        "rate": 0.8571428571428571
      },
      "success": {
        "correct": 24,
        "total": 28,
        "rate": 0.8571428571428571
      }
    },
    "operation:null": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      }
    },
    "operation:write": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 20,
        "total": 24,
        "rate": 0.8333333333333334
      },
      "tool_correct": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "arguments_exact": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 20,
        "total": 24,
        "rate": 0.8333333333333334
      }
    },
    "operation:or": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "operation:count_eq": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 25,
        "total": 28,
        "rate": 0.8928571428571429
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 25,
        "total": 28,
        "rate": 0.8928571428571429
      }
    },
    "operation:contains": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "success": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      }
    },
    "operation:sum": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      }
    },
    "operation:sort_desc": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      }
    },
    "operation:eq": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "operation:project_eq": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "language:hi": {
      "json_valid": {
        "correct": 299,
        "total": 299,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 298,
        "total": 299,
        "rate": 0.9966555183946488
      },
      "action_correct": {
        "correct": 299,
        "total": 299,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 267,
        "total": 275,
        "rate": 0.9709090909090909
      },
      "arguments_exact": {
        "correct": 254,
        "total": 275,
        "rate": 0.9236363636363636
      },
      "sql_equivalent": {
        "correct": 218,
        "total": 228,
        "rate": 0.956140350877193
      },
      "success": {
        "correct": 278,
        "total": 299,
        "rate": 0.9297658862876255
      }
    },
    "operation:year": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      }
    },
    "operation:bottom": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "operation:distinct": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "operation:ambiguous": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "arguments_exact": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      }
    },
    "operation:max": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      }
    },
    "operation:group_count": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "operation:sort_asc": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "operation:not_null": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      }
    },
    "operation:lte": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "operation:lt": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      }
    },
    "operation:group_sum": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "operation:between": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 21,
        "total": 28,
        "rate": 0.75
      },
      "arguments_exact": {
        "correct": 21,
        "total": 28,
        "rate": 0.75
      },
      "sql_equivalent": {
        "correct": 21,
        "total": 28,
        "rate": 0.75
      },
      "success": {
        "correct": 21,
        "total": 28,
        "rate": 0.75
      }
    },
    "operation:month": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 24,
        "total": 28,
        "rate": 0.8571428571428571
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 24,
        "total": 28,
        "rate": 0.8571428571428571
      }
    },
    "operation:and": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      }
    },
    "operation:having": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 20,
        "total": 28,
        "rate": 0.7142857142857143
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 20,
        "total": 28,
        "rate": 0.7142857142857143
      }
    },
    "operation:missing_value": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "arguments_exact": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      }
    },
    "operation:final": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "arguments_exact": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      }
    },
    "operation:date_after": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "operation:gt": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 26,
        "total": 28,
        "rate": 0.9285714285714286
      }
    },
    "operation:top": {
      "json_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      },
      "success": {
        "correct": 28,
        "total": 28,
        "rate": 1.0
      }
    },
    "operation:join_filter": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "success": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      }
    },
    "operation:join": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "success": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      }
    },
    "operation:weather": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      }
    },
    "operation:search": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 20,
        "total": 24,
        "rate": 0.8333333333333334
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 20,
        "total": 24,
        "rate": 0.8333333333333334
      },
      "arguments_exact": {
        "correct": 20,
        "total": 24,
        "rate": 0.8333333333333334
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 20,
        "total": 24,
        "rate": 0.8333333333333334
      }
    },
    "operation:ticket": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      }
    },
    "operation:read_file": {
      "json_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "schema_valid": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "action_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "tool_correct": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "arguments_exact": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      },
      "sql_equivalent": {
        "correct": 0,
        "total": 0,
        "rate": 0.0
      },
      "success": {
        "correct": 24,
        "total": 24,
        "rate": 1.0
      }
    }
  }
}