added fine-tune testing on validation set with short queries only

finetune_model.ipynb

@@ -3329,6 +3329,722 @@
     "\n",
     "#    break"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Test validation set only on short queries"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n",
+      "c:\\Users\\Dean\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\bitsandbytes\\autograd\\_functions.py:315: UserWarning: MatMul8bitLt: inputs will be cast from torch.bfloat16 to float16 during quantization\n",
+      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(fg_pct_home) FROM game WHERE team_name_home = 'Miami Heat';\n",
+      "SQLite:\n",
+      "SELECT MAX(fg_pct_home) FROM game WHERE team_name_home = 'Miami Heat'\n",
+      "['0.675']\n",
+      "['0.675']\n",
+      "Statement valid? True\n",
+      "SQLite matched? True\n",
+      "Result matched? True\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(pts_home) FROM game WHERE team_abbreviation_home = 'GSW';\n",
+      "SQLite:\n",
+      "SELECT MAX(pts_home) FROM game WHERE team_name_home = 'Golden State Warriors'\n",
+      "['149.0']\n",
+      "['155.0']\n",
+      "Statement valid? True\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(largest_lead_home) FROM other_stats WHERE team_abbreviation_home = 'CHI';\n",
+      "SQLite:\n",
+      "SELECT MAX(largest_lead_home) FROM other_stats WHERE team_name_home = 'Chicago Bulls'\n",
+      "Statement valid? False\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(pts_fb_home) FROM other_stats WHERE team_abbreviation_home = 'BKN';\n",
+      "SQLite:\n",
+      "SELECT MAX(pts_fb_home) FROM game WHERE team_name_home = 'Brooklyn Nets'\n",
+      "Statement valid? False\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(pts_away) FROM game WHERE team_abbreviation_away = 'LAL';\n",
+      "SQLite:\n",
+      "SELECT MAX(pts_away) FROM game WHERE team_name_away = 'Los Angeles Lakers'\n",
+      "['153.0']\n",
+      "['153.0']\n",
+      "Statement valid? True\n",
+      "SQLite matched? False\n",
+      "Result matched? True\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(pts_fb_away) FROM other_stats WHERE team_abbreviation_away = 'HOU';\n",
+      "SQLite:\n",
+      "SELECT MAX(pts_fb_away) FROM other_stats WHERE team_name_away = 'Houston Rockets'\n",
+      "Statement valid? False\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(pts_paint_away) FROM other_stats WHERE team_abbreviation_away = 'MIL';\n",
+      "SQLite:\n",
+      "SELECT MAX(pts_off_to_away) FROM other_stats WHERE team_name_away = 'Milwaukee Bucks'\n",
+      "Statement valid? False\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MIN(pts_away) FROM game WHERE team_abbreviation_away = 'GSW';\n",
+      "SQLite:\n",
+      "SELECT MIN(pts_away) FROM game WHERE team_name_away = 'Golden State Warriors'\n",
+      "['65.0']\n",
+      "['65.0']\n",
+      "Statement valid? True\n",
+      "SQLite matched? False\n",
+      "Result matched? True\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT COUNT(*) FROM game WHERE team_abbreviation_away = 'TOR' AND pts_away = 100;\n",
+      "SQLite:\n",
+      "SELECT COUNT(*) FROM other_stats WHERE team_name_away = 'Toronto Raptors' AND pts_away = 100\n",
+      "Statement valid? False\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(largest_lead_home) FROM other_stats WHERE team_abbreviation_home = 'BOS';\n",
+      "SQLite:\n",
+      "SELECT MAX(largest_lead_home) FROM other_stats WHERE team_name_home = 'Boston Celtics'\n",
+      "Statement valid? False\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(fg3m_home) FROM game WHERE team_name_home = 'Brooklyn Nets';\n",
+      "SQLite: SELECT MAX(fg3m_home) FROM game WHERE team_name_home = 'Brooklyn Nets'\n",
+      "['22.0']\n",
+      "['22.0']\n",
+      "Statement valid? True\n",
+      "SQLite matched? True\n",
+      "Result matched? True\n",
+      "\n",
+      "\n",
+      "SELECT MAX(pts_2nd_chance_home) FROM other_stats WHERE team_abbreviation_home = 'CHI';\n",
+      "SQLite:\n",
+      "SELECT MAX(pts_2nd_chance_home) FROM game WHERE team_name_home = 'Chicago Bulls'\n",
+      "Statement valid? False\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT COUNT(*) FROM game WHERE (pts_home + pts_away) >= 250 AND season_id = '22005';\n",
+      "SQLite:\n",
+      "SELECT COUNT(*) FROM game WHERE (pts_home + pts_away) >= 250 AND season_id = '22005'\n",
+      "['6']\n",
+      "['6']\n",
+      "Statement valid? True\n",
+      "SQLite matched? True\n",
+      "Result matched? True\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n",
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(largest_lead_home) FROM other_stats WHERE team_abbreviation_home = 'MIA';\n",
+      "SQLite:\n",
+      "SELECT MAX(largest_lead_home) FROM other_stats WHERE team_name_home = 'Miami Heat'\n",
+      "Statement valid? False\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(reb_home) FROM game WHERE team_abbreviation_home = 'NYK';\n",
+      "SQLite:\n",
+      "SELECT MAX(reb_home) FROM game WHERE team_name_home = 'New York Knicks'\n",
+      "['75.0']\n",
+      "['75.0']\n",
+      "Statement valid? True\n",
+      "SQLite matched? False\n",
+      "Result matched? True\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(pts_fb_home) FROM other_stats WHERE team_abbreviation_home = 'HOU';\n",
+      "SQLite:\n",
+      "SELECT MAX(pts_fb_home) FROM game WHERE team_name_home = 'Houston Rockets'\n",
+      "Statement valid? False\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(pts_away) FROM game WHERE team_abbreviation_away = 'CLE';\n",
+      "SQLite:\n",
+      "SELECT MAX(pts_away) FROM game WHERE team_name_away = 'Cleveland Cavaliers' AND season_type = 'Regular'\n",
+      "['140.0']\n",
+      "['None']\n",
+      "Statement valid? True\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(total_turnovers_home) FROM other_stats WHERE team_abbreviation_home = 'PHI';\n",
+      "SQLite:\n",
+      "SELECT MAX(total_turnovers_home) FROM other_stats WHERE team_name_home = 'Philadelphia 76ers'\n",
+      "Statement valid? False\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(largest_lead_away) FROM other_stats WHERE team_abbreviation_away = 'NYK';\n",
+      "SQLite:\n",
+      "SELECT MAX(largest_lead_away) FROM other_stats WHERE team_name_away = 'New York Knicks'\n",
+      "Statement valid? False\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(total_turnovers_away) FROM other_stats WHERE team_abbreviation_away = 'BKN';\n",
+      "SQLite:\n",
+      "SELECT MAX(team_turnovers_away) FROM other_stats WHERE team_name_away = 'Brooklyn Nets'\n",
+      "Statement valid? False\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT COUNT(*) FROM game WHERE ABS(pts_home - pts_away) = 1;\n",
+      "SQLite:\n",
+      "SELECT COUNT(*) FROM game WHERE pts_home - pts_away = 1\n",
+      "['3050']\n",
+      "['1572']\n",
+      "Statement valid? True\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT COUNT(*) FROM game WHERE team_abbreviation_home = 'LAL' AND season_id = '22010';\n",
+      "SQLite: SELECT COUNT(*) FROM game WHERE team_name_home = 'Los Angeles Lakers' AND season_id = '22010'\n",
+      "['41']\n",
+      "['41']\n",
+      "Statement valid? True\n",
+      "SQLite matched? False\n",
+      "Result matched? True\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(pts_home) FROM game;\n",
+      "SQLite:\n",
+      "SELECT game_id, MAX(pts_home) AS highest_scoring_home_game FROM game GROUP BY game_id ORDER BY highest_scoring_home_game DESC LIMIT 1\n",
+      "['192.0']\n",
+      "['0031600001', '192.0']\n",
+      "Statement valid? True\n",
+      "SQLite matched? False\n",
+      "Result matched? True\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MIN(fg_pct_away) FROM game;\n",
+      "SQLite:\n",
+      "SELECT MIN(fg_pct_away) FROM game\n",
+      "['0.156']\n",
+      "['0.156']\n",
+      "Statement valid? True\n",
+      "SQLite matched? True\n",
+      "Result matched? True\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT COUNT(*) FROM game WHERE ABS(pts_home - pts_away) > 40;\n",
+      "SQLite:\n",
+      "SELECT COUNT(*) FROM game WHERE pts_home > 40 OR pts_away > 40\n",
+      "['307']\n",
+      "['65696']\n",
+      "Statement valid? True\n",
+      "SQLite matched? False\n",
+      "Result matched? False\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT MAX(stl_away) FROM game;\n",
+      "SQLite:\n",
+      "SELECT MAX(stl_away) FROM game\n",
+      "['27.0']\n",
+      "['27.0']\n",
+      "Statement valid? True\n",
+      "SQLite matched? True\n",
+      "Result matched? True\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT COUNT(*) FROM game WHERE pts_home >= 120 AND pts_away >= 120;\n",
+      "SQLite:\n",
+      "SELECT COUNT(*) FROM game WHERE pts_home >= 120 AND pts_away >= 120\n",
+      "['3043']\n",
+      "['3043']\n",
+      "Statement valid? True\n",
+      "SQLite matched? True\n",
+      "Result matched? True\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
+      "Setting `pad_token_id` to `eos_token_id`:32022 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SELECT COUNT(*) FROM game WHERE ast_home >= 30 OR ast_away >= 30;\n",
+      "SQLite:\n",
+      "SELECT COUNT(*) FROM game WHERE ast_home >= 30 OR ast_away >= 30\n",
+      "['11305']\n",
+      "['11305']\n",
+      "Statement valid? True\n",
+      "SQLite matched? True\n",
+      "Result matched? True\n",
+      "\n",
+      "\n",
+      "SELECT MAX(ast_home) FROM game WHERE team_name_home = 'Indiana Pacers';\n",
+      "SQLite:\n",
+      "SELECT MAX(ast_home) FROM game WHERE team_name_home = 'Indiana Pacers'\n",
+      "['44.0']\n",
+      "['44.0']\n",
+      "Statement valid? True\n",
+      "SQLite matched? True\n",
+      "Result matched? True\n",
+      "\n",
+      "\n",
+      "Percent valid: 0.5862068965517241\n",
+      "Percent SQLite matched: 0.27586206896551724\n",
+      "Percent result matched: 0.4482758620689655\n"
+     ]
+    }
+   ],
+   "source": [
+    "num_valid = 0\n",
+    "num_sql_matched = 0\n",
+    "num_result_matched = 0\n",
+    "counter = 0\n",
+    "for v in val_dataset:\n",
+    "    # Obtain sample natural language question and sql_query\n",
+    "    #v = val_dataset[random.randint(0, len(val_dataset) - 1)]\n",
+    "    full_example = tokenizer.decode(v[\"input_ids\"], skip_special_tokens=True)\n",
+    "    user_prompt = full_example[:prompt_length]\n",
+    "    question, sql_query = full_example[prompt_length:].split(\"SQLite:\\n\")\n",
+    "    #print(question)\n",
+    "    #print(sql_query)\n",
+    "\n",
+    "    if len(sql_query) <= 90:\n",
+    "        # Obtain model output\n",
+    "        input_text = \"How many points to the Los Angeles Lakers average at home?\"\n",
+    "        message = [{'role': 'user', 'content': input_prompt + question}]\n",
+    "        inputs = tokenizer.apply_chat_template(message, add_generation_prompt=True, return_tensors=\"pt\").to(model.device)\n",
+    "\n",
+    "        # Generate SQL query\n",
+    "        outputs = model.generate(\n",
+    "            inputs,\n",
+    "            max_new_tokens=256,\n",
+    "            eos_token_id=tokenizer.convert_tokens_to_ids(\"<|endofsql|>\")\n",
+    "        )\n",
+    "        model_output = tokenizer.decode(outputs[0][len(inputs[0]):], skip_special_tokens=True)\n",
+    "\n",
+    "        print(sql_query)\n",
+    "        print(model_output.split(\";\")[0])\n",
+    "        #print()\n",
+    "        #print(model_output)\n",
+    "        result = compare_result(sql_query, model_output)\n",
+    "        print(\"Statement valid? \" + str(result[0]))\n",
+    "        print(\"SQLite matched? \" + str(result[1]))\n",
+    "        print(\"Result matched? \" + str(result[2]))\n",
+    "        print()\n",
+    "        print()\n",
+    "        counter += 1\n",
+    "\n",
+    "        if result[0]:\n",
+    "            num_valid += 1\n",
+    "        if result[1]:\n",
+    "            num_sql_matched += 1\n",
+    "        if result[2]:\n",
+    "            num_result_matched += 1\n",
+    "\n",
+    "print(\"Percent valid: \" + str(num_valid / counter))\n",
+    "print(\"Percent SQLite matched: \" + str(num_sql_matched / counter))\n",
+    "print(\"Percent result matched: \" + str(num_result_matched / counter))"
+   ]
   }
  ],
  "metadata": {