Upload 2 files

lstm-model-for-option-pricing.ipynb

@@ -0,0 +1 @@
+{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Use the kagglehub client library to attach Kaggle resources like competitions, datasets, and models to your session\n# Learn more about kagglehub: https://github.com/Kaggle/kagglehub/blob/main/README.md\n\nimport kagglehub\n# kagglehub.dataset_download('<owner>/<dataset-slug>')","metadata":{"_uuid":"9e8ca9c3-5582-4c6d-9be8-6b65b21a5257","_cell_guid":"5f311bf6-5091-4cb1-96e3-e2a2ff9c5843","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:10.247450Z","iopub.execute_input":"2026-07-12T07:04:10.247905Z","iopub.status.idle":"2026-07-12T07:04:10.254152Z","shell.execute_reply.started":"2026-07-12T07:04:10.247878Z","shell.execute_reply":"2026-07-12T07:04:10.253115Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":32},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"a482570e-0424-47d9-bcb0-f4f8ea2ddfe5","_cell_guid":"3537bfbd-32e1-426f-8540-373b62d614f2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:10.255763Z","iopub.execute_input":"2026-07-12T07:04:10.255971Z","iopub.status.idle":"2026-07-12T07:04:10.266664Z","shell.execute_reply.started":"2026-07-12T07:04:10.255953Z","shell.execute_reply":"2026-07-12T07:04:10.266102Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":33},{"cell_type":"code","source":"SEED=42\ntorch.manual_seed(42)\nnp.random.seed(42)","metadata":{"_uuid":"88de1a9f-9b4e-45ac-b582-620e79964db8","_cell_guid":"a7429c2c-da92-4a8c-bfab-cd2a44d87248","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:10.267743Z","iopub.execute_input":"2026-07-12T07:04:10.268008Z","iopub.status.idle":"2026-07-12T07:04:10.279351Z","shell.execute_reply.started":"2026-07-12T07:04:10.267980Z","shell.execute_reply":"2026-07-12T07:04:10.278388Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":34},{"cell_type":"code","source":"device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f'Using device: {device}')\nprint(f'GPUs available: f{torch.cuda.device_count()}')","metadata":{"_uuid":"75599d80-f5db-4cb2-80e7-f9f01c13390d","_cell_guid":"5b64f03f-2df1-4a02-95d8-3b4c66eb65df","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:10.280418Z","iopub.execute_input":"2026-07-12T07:04:10.280843Z","iopub.status.idle":"2026-07-12T07:04:10.291314Z","shell.execute_reply.started":"2026-07-12T07:04:10.280823Z","shell.execute_reply":"2026-07-12T07:04:10.290711Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"Using device: cuda\nGPUs available: f2\n","output_type":"stream"}],"execution_count":35},{"cell_type":"markdown","source":"# Preprocessing","metadata":{"_uuid":"6e8eaefa-8b01-4e04-905e-d765b89f84d7","_cell_guid":"66988607-d519-4d71-960b-5d684c1dd68d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"\ndf=pd.read_csv('hf://datasets/major-year-project/stock-data/spy_options_2024.csv')\nprint(f'Raw shape: {df.shape}')","metadata":{"_uuid":"1a66ff07-15bc-4a26-acae-15c25163aaf9","_cell_guid":"648042c0-0c1f-4df0-9b74-16debde05bf6","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:10.293436Z","iopub.execute_input":"2026-07-12T07:04:10.294196Z","iopub.status.idle":"2026-07-12T07:04:36.131216Z","shell.execute_reply.started":"2026-07-12T07:04:10.294162Z","shell.execute_reply":"2026-07-12T07:04:36.130180Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"Raw shape: (2292798, 20)\n","output_type":"stream"}],"execution_count":36},{"cell_type":"code","source":"df.head(3)","metadata":{"_uuid":"eb8370b7-7dfe-47b0-b6b4-76b0345f3428","_cell_guid":"b9ac2382-4e60-4f11-a47d-380b5db1cad9","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:36.132417Z","iopub.execute_input":"2026-07-12T07:04:36.132797Z","iopub.status.idle":"2026-07-12T07:04:36.151065Z","shell.execute_reply.started":"2026-07-12T07:04:36.132717Z","shell.execute_reply":"2026-07-12T07:04:36.150131Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":37,"output_type":"execute_result","data":{"text/plain":"           contractID symbol  expiration  strike  type   last   mark    bid  \\\n0  SPY240102C00402000    SPY  2024-01-02   402.0  call  69.97  70.77  70.68   \n1  SPY240102P00402000    SPY  2024-01-02   402.0   put   0.02   0.01   0.00   \n2  SPY240102C00403000    SPY  2024-01-02   403.0  call  68.97  69.78  69.68   \n\n   bid_size    ask  ask_size  volume  open_interest        date  \\\n0       100  70.85       100     130              1  2024-01-02   \n1         0   0.01      4204       0            303  2024-01-02   \n2       100  69.89       100       9              0  2024-01-02   \n\n   implied_volatility    delta    gamma    theta     vega      rho  \n0             1.24983  0.99396  0.00055 -0.32269  0.00423  0.01093  \n1             1.04145 -0.00135  0.00017 -0.05700  0.00110 -0.00002  \n2             1.25456  0.99311  0.00062 -0.35671  0.00476  0.01095  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>contractID</th>\n      <th>symbol</th>\n      <th>expiration</th>\n      <th>strike</th>\n      <th>type</th>\n      <th>last</th>\n      <th>mark</th>\n      <th>bid</th>\n      <th>bid_size</th>\n      <th>ask</th>\n      <th>ask_size</th>\n      <th>volume</th>\n      <th>open_interest</th>\n      <th>date</th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>SPY240102C00402000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>402.0</td>\n      <td>call</td>\n      <td>69.97</td>\n      <td>70.77</td>\n      <td>70.68</td>\n      <td>100</td>\n      <td>70.85</td>\n      <td>100</td>\n      <td>130</td>\n      <td>1</td>\n      <td>2024-01-02</td>\n      <td>1.24983</td>\n      <td>0.99396</td>\n      <td>0.00055</td>\n      <td>-0.32269</td>\n      <td>0.00423</td>\n      <td>0.01093</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>SPY240102P00402000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>402.0</td>\n      <td>put</td>\n      <td>0.02</td>\n      <td>0.01</td>\n      <td>0.00</td>\n      <td>0</td>\n      <td>0.01</td>\n      <td>4204</td>\n      <td>0</td>\n      <td>303</td>\n      <td>2024-01-02</td>\n      <td>1.04145</td>\n      <td>-0.00135</td>\n      <td>0.00017</td>\n      <td>-0.05700</td>\n      <td>0.00110</td>\n      <td>-0.00002</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>SPY240102C00403000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>403.0</td>\n      <td>call</td>\n      <td>68.97</td>\n      <td>69.78</td>\n      <td>69.68</td>\n      <td>100</td>\n      <td>69.89</td>\n      <td>100</td>\n      <td>9</td>\n      <td>0</td>\n      <td>2024-01-02</td>\n      <td>1.25456</td>\n      <td>0.99311</td>\n      <td>0.00062</td>\n      <td>-0.35671</td>\n      <td>0.00476</td>\n      <td>0.01095</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":37},{"cell_type":"code","source":"df.info()","metadata":{"_uuid":"240cbada-1058-4dbf-a3cc-09c5f971478e","_cell_guid":"f9134a43-3377-4fc7-97f0-146b9f39cd85","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:36.152086Z","iopub.execute_input":"2026-07-12T07:04:36.152777Z","iopub.status.idle":"2026-07-12T07:04:36.167934Z","shell.execute_reply.started":"2026-07-12T07:04:36.152716Z","shell.execute_reply":"2026-07-12T07:04:36.166992Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2292798 entries, 0 to 2292797\nData columns (total 20 columns):\n #   Column              Dtype  \n---  ------              -----  \n 0   contractID          object \n 1   symbol              object \n 2   expiration          object \n 3   strike              float64\n 4   type                object \n 5   last                float64\n 6   mark                float64\n 7   bid                 float64\n 8   bid_size            int64  \n 9   ask                 float64\n 10  ask_size            int64  \n 11  volume              int64  \n 12  open_interest       int64  \n 13  date                object \n 14  implied_volatility  float64\n 15  delta               float64\n 16  gamma               float64\n 17  theta               float64\n 18  vega                float64\n 19  rho                 float64\ndtypes: float64(11), int64(4), object(5)\nmemory usage: 349.9+ MB\n","output_type":"stream"}],"execution_count":38},{"cell_type":"code","source":"df['date']=pd.to_datetime(df['date'])\ndf['expiration']=pd.to_datetime(df['expiration'])","metadata":{"_uuid":"1a8d8016-e3ae-4e16-8cff-34b9542f99f2","_cell_guid":"070ee439-20ec-40f6-b213-0fc472d9a7b6","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:36.169076Z","iopub.execute_input":"2026-07-12T07:04:36.169443Z","iopub.status.idle":"2026-07-12T07:04:36.624179Z","shell.execute_reply.started":"2026-07-12T07:04:36.169408Z","shell.execute_reply":"2026-07-12T07:04:36.623324Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":39},{"cell_type":"code","source":"df.info()","metadata":{"_uuid":"0c78dd5d-f4d1-45cd-adbc-9824804f16d8","_cell_guid":"e728b644-a115-4865-be44-6e325f85f397","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:36.625150Z","iopub.execute_input":"2026-07-12T07:04:36.625491Z","iopub.status.idle":"2026-07-12T07:04:36.633918Z","shell.execute_reply.started":"2026-07-12T07:04:36.625467Z","shell.execute_reply":"2026-07-12T07:04:36.633099Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2292798 entries, 0 to 2292797\nData columns (total 20 columns):\n #   Column              Dtype         \n---  ------              -----         \n 0   contractID          object        \n 1   symbol              object        \n 2   expiration          datetime64[ns]\n 3   strike              float64       \n 4   type                object        \n 5   last                float64       \n 6   mark                float64       \n 7   bid                 float64       \n 8   bid_size            int64         \n 9   ask                 float64       \n 10  ask_size            int64         \n 11  volume              int64         \n 12  open_interest       int64         \n 13  date                datetime64[ns]\n 14  implied_volatility  float64       \n 15  delta               float64       \n 16  gamma               float64       \n 17  theta               float64       \n 18  vega                float64       \n 19  rho                 float64       \ndtypes: datetime64[ns](2), float64(11), int64(4), object(3)\nmemory usage: 349.9+ MB\n","output_type":"stream"}],"execution_count":40},{"cell_type":"code","source":"df['tte_days']=(df['expiration']-df['date']).dt.days","metadata":{"_uuid":"a90f9b47-6a74-4a51-818b-fcf115db255b","_cell_guid":"2a3aad81-704b-451d-977f-07d5f482c5a7","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:36.636489Z","iopub.execute_input":"2026-07-12T07:04:36.636764Z","iopub.status.idle":"2026-07-12T07:04:36.696975Z","shell.execute_reply.started":"2026-07-12T07:04:36.636708Z","shell.execute_reply":"2026-07-12T07:04:36.696297Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":41},{"cell_type":"code","source":"# Log-moneyness: how far strike is from the median strike\n# (proxy since we don't have live spot price in this file)\ndf[\"log_moneyness\"] = np.log(\n    df[\"strike\"] / df[\"strike\"].median()\n).fillna(0)","metadata":{"_uuid":"29c2a893-1070-4bee-b4bd-8ffc64d9d59f","_cell_guid":"8427b724-5c2e-4ccc-a5c1-7e34500d02a1","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:36.697918Z","iopub.execute_input":"2026-07-12T07:04:36.698188Z","iopub.status.idle":"2026-07-12T07:04:36.735202Z","shell.execute_reply.started":"2026-07-12T07:04:36.698142Z","shell.execute_reply":"2026-07-12T07:04:36.734559Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":42},{"cell_type":"code","source":"df.info()","metadata":{"_uuid":"f7c87736-39c1-411e-b9b8-a39d76d72bcd","_cell_guid":"2c0b49fd-5cb9-434a-974a-ba7e10ff54c7","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:36.736245Z","iopub.execute_input":"2026-07-12T07:04:36.736874Z","iopub.status.idle":"2026-07-12T07:04:36.745556Z","shell.execute_reply.started":"2026-07-12T07:04:36.736849Z","shell.execute_reply":"2026-07-12T07:04:36.744483Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2292798 entries, 0 to 2292797\nData columns (total 22 columns):\n #   Column              Dtype         \n---  ------              -----         \n 0   contractID          object        \n 1   symbol              object        \n 2   expiration          datetime64[ns]\n 3   strike              float64       \n 4   type                object        \n 5   last                float64       \n 6   mark                float64       \n 7   bid                 float64       \n 8   bid_size            int64         \n 9   ask                 float64       \n 10  ask_size            int64         \n 11  volume              int64         \n 12  open_interest       int64         \n 13  date                datetime64[ns]\n 14  implied_volatility  float64       \n 15  delta               float64       \n 16  gamma               float64       \n 17  theta               float64       \n 18  vega                float64       \n 19  rho                 float64       \n 20  tte_days            int64         \n 21  log_moneyness       float64       \ndtypes: datetime64[ns](2), float64(12), int64(5), object(3)\nmemory usage: 384.8+ MB\n","output_type":"stream"}],"execution_count":43},{"cell_type":"code","source":"df.head()","metadata":{"_uuid":"f41b8627-0323-4458-9996-2b3e9ca4e29f","_cell_guid":"0ef10345-d8d5-41fa-bf35-564b30e7b166","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:36.746925Z","iopub.execute_input":"2026-07-12T07:04:36.747680Z","iopub.status.idle":"2026-07-12T07:04:36.774209Z","shell.execute_reply.started":"2026-07-12T07:04:36.747645Z","shell.execute_reply":"2026-07-12T07:04:36.773307Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":44,"output_type":"execute_result","data":{"text/plain":"           contractID symbol expiration  strike  type   last   mark    bid  \\\n0  SPY240102C00402000    SPY 2024-01-02   402.0  call  69.97  70.77  70.68   \n1  SPY240102P00402000    SPY 2024-01-02   402.0   put   0.02   0.01   0.00   \n2  SPY240102C00403000    SPY 2024-01-02   403.0  call  68.97  69.78  69.68   \n3  SPY240102P00403000    SPY 2024-01-02   403.0   put   0.01   0.01   0.00   \n4  SPY240102C00404000    SPY 2024-01-02   404.0  call  68.64  68.77  68.68   \n\n   bid_size    ask  ...  open_interest       date  implied_volatility  \\\n0       100  70.85  ...              1 2024-01-02             1.24983   \n1         0   0.01  ...            303 2024-01-02             1.04145   \n2       100  69.89  ...              0 2024-01-02             1.25456   \n3         0   0.01  ...              8 2024-01-02             1.02682   \n4       100  68.85  ...              1 2024-01-02             1.21492   \n\n     delta    gamma    theta     vega      rho  tte_days  log_moneyness  \n0  0.99396  0.00055 -0.32269  0.00423  0.01093         0      -0.218156  \n1 -0.00135  0.00017 -0.05700  0.00110 -0.00002         0      -0.218156  \n2  0.99311  0.00062 -0.35671  0.00476  0.01095         0      -0.215672  \n3 -0.00137  0.00018 -0.05692  0.00111 -0.00002         0      -0.215672  \n4  0.99383  0.00058 -0.32065  0.00431  0.01099         0      -0.213193  \n\n[5 rows x 22 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>contractID</th>\n      <th>symbol</th>\n      <th>expiration</th>\n      <th>strike</th>\n      <th>type</th>\n      <th>last</th>\n      <th>mark</th>\n      <th>bid</th>\n      <th>bid_size</th>\n      <th>ask</th>\n      <th>...</th>\n      <th>open_interest</th>\n      <th>date</th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>tte_days</th>\n      <th>log_moneyness</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>SPY240102C00402000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>402.0</td>\n      <td>call</td>\n      <td>69.97</td>\n      <td>70.77</td>\n      <td>70.68</td>\n      <td>100</td>\n      <td>70.85</td>\n      <td>...</td>\n      <td>1</td>\n      <td>2024-01-02</td>\n      <td>1.24983</td>\n      <td>0.99396</td>\n      <td>0.00055</td>\n      <td>-0.32269</td>\n      <td>0.00423</td>\n      <td>0.01093</td>\n      <td>0</td>\n      <td>-0.218156</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>SPY240102P00402000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>402.0</td>\n      <td>put</td>\n      <td>0.02</td>\n      <td>0.01</td>\n      <td>0.00</td>\n      <td>0</td>\n      <td>0.01</td>\n      <td>...</td>\n      <td>303</td>\n      <td>2024-01-02</td>\n      <td>1.04145</td>\n      <td>-0.00135</td>\n      <td>0.00017</td>\n      <td>-0.05700</td>\n      <td>0.00110</td>\n      <td>-0.00002</td>\n      <td>0</td>\n      <td>-0.218156</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>SPY240102C00403000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>403.0</td>\n      <td>call</td>\n      <td>68.97</td>\n      <td>69.78</td>\n      <td>69.68</td>\n      <td>100</td>\n      <td>69.89</td>\n      <td>...</td>\n      <td>0</td>\n      <td>2024-01-02</td>\n      <td>1.25456</td>\n      <td>0.99311</td>\n      <td>0.00062</td>\n      <td>-0.35671</td>\n      <td>0.00476</td>\n      <td>0.01095</td>\n      <td>0</td>\n      <td>-0.215672</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>SPY240102P00403000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>403.0</td>\n      <td>put</td>\n      <td>0.01</td>\n      <td>0.01</td>\n      <td>0.00</td>\n      <td>0</td>\n      <td>0.01</td>\n      <td>...</td>\n      <td>8</td>\n      <td>2024-01-02</td>\n      <td>1.02682</td>\n      <td>-0.00137</td>\n      <td>0.00018</td>\n      <td>-0.05692</td>\n      <td>0.00111</td>\n      <td>-0.00002</td>\n      <td>0</td>\n      <td>-0.215672</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>SPY240102C00404000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>404.0</td>\n      <td>call</td>\n      <td>68.64</td>\n      <td>68.77</td>\n      <td>68.68</td>\n      <td>100</td>\n      <td>68.85</td>\n      <td>...</td>\n      <td>1</td>\n      <td>2024-01-02</td>\n      <td>1.21492</td>\n      <td>0.99383</td>\n      <td>0.00058</td>\n      <td>-0.32065</td>\n      <td>0.00431</td>\n      <td>0.01099</td>\n      <td>0</td>\n      <td>-0.213193</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 22 columns</p>\n</div>"},"metadata":{}}],"execution_count":44},{"cell_type":"code","source":"df[\"type_enc\"] = (df[\"type\"].str.lower() == \"call\").astype(int)","metadata":{"_uuid":"a05820ba-5613-4014-ad4e-09fb91ae838b","_cell_guid":"857cea4c-032b-440d-9214-d9230d905007","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:36.775423Z","iopub.execute_input":"2026-07-12T07:04:36.775761Z","iopub.status.idle":"2026-07-12T07:04:37.320789Z","shell.execute_reply.started":"2026-07-12T07:04:36.775709Z","shell.execute_reply":"2026-07-12T07:04:37.319672Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":45},{"cell_type":"code","source":"df.info()","metadata":{"_uuid":"4a0e22fd-99bc-4b7c-bab1-c2e0e121331c","_cell_guid":"d0d6e977-a3ef-4eb5-983a-7fd0ed518eed","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:37.321922Z","iopub.execute_input":"2026-07-12T07:04:37.322231Z","iopub.status.idle":"2026-07-12T07:04:37.331039Z","shell.execute_reply.started":"2026-07-12T07:04:37.322208Z","shell.execute_reply":"2026-07-12T07:04:37.330291Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2292798 entries, 0 to 2292797\nData columns (total 23 columns):\n #   Column              Dtype         \n---  ------              -----         \n 0   contractID          object        \n 1   symbol              object        \n 2   expiration          datetime64[ns]\n 3   strike              float64       \n 4   type                object        \n 5   last                float64       \n 6   mark                float64       \n 7   bid                 float64       \n 8   bid_size            int64         \n 9   ask                 float64       \n 10  ask_size            int64         \n 11  volume              int64         \n 12  open_interest       int64         \n 13  date                datetime64[ns]\n 14  implied_volatility  float64       \n 15  delta               float64       \n 16  gamma               float64       \n 17  theta               float64       \n 18  vega                float64       \n 19  rho                 float64       \n 20  tte_days            int64         \n 21  log_moneyness       float64       \n 22  type_enc            int64         \ndtypes: datetime64[ns](2), float64(12), int64(6), object(3)\nmemory usage: 402.3+ MB\n","output_type":"stream"}],"execution_count":46},{"cell_type":"code","source":"# Remove rows where target is missing / zero / negative\ndf = df[df[\"mark\"] > 0].copy()\ndf = df.dropna(subset=[\n    \"mark\", \"implied_volatility\",\n    \"delta\", \"gamma\", \"theta\", \"vega\", \"rho\"\n])\n \nprint(f\"    After cleaning : {df.shape}\")","metadata":{"_uuid":"59e77167-4510-4df4-bd0c-9eaaed7359db","_cell_guid":"bfdc7755-afdf-4c41-93a4-7793216241f4","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:37.332054Z","iopub.execute_input":"2026-07-12T07:04:37.332383Z","iopub.status.idle":"2026-07-12T07:04:38.394014Z","shell.execute_reply.started":"2026-07-12T07:04:37.332361Z","shell.execute_reply":"2026-07-12T07:04:38.393093Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"    After cleaning : (2292798, 23)\n","output_type":"stream"}],"execution_count":47},{"cell_type":"code","source":"FEATURES = [\n    # Theory-grounded: what Black-Scholes uses\n    \"implied_volatility\",   # market's expectation of future vol\n    \"delta\",                # price sensitivity\n    \"gamma\",                # rate of delta change\n    \"theta\",                # daily time decay\n    \"vega\",                 # vol sensitivity\n    \"rho\",                  # interest rate sensitivity\n \n    # Contract structure\n    \"tte_days\",             # time to expiry (engineered)\n    \"log_moneyness\",        # engineered from strike\n    \"strike\",               # absolute strike level\n    \"type_enc\",             # call or put\n \n    # Market activity (not price itself)\n    \"volume\",               # contracts traded today\n    \"open_interest\",        # total open contracts\n]\n \nTARGET = \"mark\"","metadata":{"_uuid":"7dda0134-b71b-4547-9a2c-80b0f80b843d","_cell_guid":"38b14507-c78d-4a29-bb94-3f3afb6bc8ef","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:38.395186Z","iopub.execute_input":"2026-07-12T07:04:38.395558Z","iopub.status.idle":"2026-07-12T07:04:38.400376Z","shell.execute_reply.started":"2026-07-12T07:04:38.395524Z","shell.execute_reply":"2026-07-12T07:04:38.399595Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":48},{"cell_type":"code","source":"df=df[FEATURES+[TARGET,'date','contractID']].dropna()\nprint(f\"    Final shape : {df.shape}\")\nprint(f\"    Features ({len(FEATURES)}) : {FEATURES}\")","metadata":{"_uuid":"81c19c9e-a0ac-4c24-95ba-d9ad20cbebe8","_cell_guid":"4c86408a-cb33-45ad-bdf0-1d93e41436cc","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:38.401469Z","iopub.execute_input":"2026-07-12T07:04:38.401842Z","iopub.status.idle":"2026-07-12T07:04:38.761544Z","shell.execute_reply.started":"2026-07-12T07:04:38.401822Z","shell.execute_reply":"2026-07-12T07:04:38.760609Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"    Final shape : (2292798, 15)\n    Features (12) : ['implied_volatility', 'delta', 'gamma', 'theta', 'vega', 'rho', 'tte_days', 'log_moneyness', 'strike', 'type_enc', 'volume', 'open_interest']\n","output_type":"stream"}],"execution_count":49},{"cell_type":"code","source":"df.head()","metadata":{"_uuid":"49039d38-727f-45d7-806e-4044a4aa8fb2","_cell_guid":"7f6e93d4-6d2a-4b97-9de6-4028d7b9160b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:38.762658Z","iopub.execute_input":"2026-07-12T07:04:38.763023Z","iopub.status.idle":"2026-07-12T07:04:38.778230Z","shell.execute_reply.started":"2026-07-12T07:04:38.763002Z","shell.execute_reply":"2026-07-12T07:04:38.777559Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":50,"output_type":"execute_result","data":{"text/plain":"   implied_volatility    delta    gamma    theta     vega      rho  tte_days  \\\n0             1.24983  0.99396  0.00055 -0.32269  0.00423  0.01093         0   \n1             1.04145 -0.00135  0.00017 -0.05700  0.00110 -0.00002         0   \n2             1.25456  0.99311  0.00062 -0.35671  0.00476  0.01095         0   \n3             1.02682 -0.00137  0.00018 -0.05692  0.00111 -0.00002         0   \n4             1.21492  0.99383  0.00058 -0.32065  0.00431  0.01099         0   \n\n   log_moneyness  strike  type_enc  volume  open_interest   mark       date  \\\n0      -0.218156   402.0         1     130              1  70.77 2024-01-02   \n1      -0.218156   402.0         0       0            303   0.01 2024-01-02   \n2      -0.215672   403.0         1       9              0  69.78 2024-01-02   \n3      -0.215672   403.0         0       1              8   0.01 2024-01-02   \n4      -0.213193   404.0         1       1              1  68.77 2024-01-02   \n\n           contractID  \n0  SPY240102C00402000  \n1  SPY240102P00402000  \n2  SPY240102C00403000  \n3  SPY240102P00403000  \n4  SPY240102C00404000  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>tte_days</th>\n      <th>log_moneyness</th>\n      <th>strike</th>\n      <th>type_enc</th>\n      <th>volume</th>\n      <th>open_interest</th>\n      <th>mark</th>\n      <th>date</th>\n      <th>contractID</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.24983</td>\n      <td>0.99396</td>\n      <td>0.00055</td>\n      <td>-0.32269</td>\n      <td>0.00423</td>\n      <td>0.01093</td>\n      <td>0</td>\n      <td>-0.218156</td>\n      <td>402.0</td>\n      <td>1</td>\n      <td>130</td>\n      <td>1</td>\n      <td>70.77</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00402000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.04145</td>\n      <td>-0.00135</td>\n      <td>0.00017</td>\n      <td>-0.05700</td>\n      <td>0.00110</td>\n      <td>-0.00002</td>\n      <td>0</td>\n      <td>-0.218156</td>\n      <td>402.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>303</td>\n      <td>0.01</td>\n      <td>2024-01-02</td>\n      <td>SPY240102P00402000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1.25456</td>\n      <td>0.99311</td>\n      <td>0.00062</td>\n      <td>-0.35671</td>\n      <td>0.00476</td>\n      <td>0.01095</td>\n      <td>0</td>\n      <td>-0.215672</td>\n      <td>403.0</td>\n      <td>1</td>\n      <td>9</td>\n      <td>0</td>\n      <td>69.78</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00403000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.02682</td>\n      <td>-0.00137</td>\n      <td>0.00018</td>\n      <td>-0.05692</td>\n      <td>0.00111</td>\n      <td>-0.00002</td>\n      <td>0</td>\n      <td>-0.215672</td>\n      <td>403.0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>8</td>\n      <td>0.01</td>\n      <td>2024-01-02</td>\n      <td>SPY240102P00403000</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1.21492</td>\n      <td>0.99383</td>\n      <td>0.00058</td>\n      <td>-0.32065</td>\n      <td>0.00431</td>\n      <td>0.01099</td>\n      <td>0</td>\n      <td>-0.213193</td>\n      <td>404.0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>68.77</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00404000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":50},{"cell_type":"markdown","source":"# Splitting by 70/30 by time","metadata":{"_uuid":"01138ec5-b832-4e77-ac2d-d7d9b2867a31","_cell_guid":"93a95221-5a30-420f-86bd-86de39a77ef4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"print('\\n[3] Splitting 70/30 by time')\ndf=df.sort_values(by=['contractID', 'date']).reset_index(drop=True)","metadata":{"_uuid":"bfa0d4c6-4ff0-466b-9910-d23bc74ff98c","_cell_guid":"60a13515-ce73-45cf-95ae-c9cf56e5060f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:38.779474Z","iopub.execute_input":"2026-07-12T07:04:38.779892Z","iopub.status.idle":"2026-07-12T07:04:39.470446Z","shell.execute_reply.started":"2026-07-12T07:04:38.779867Z","shell.execute_reply":"2026-07-12T07:04:39.469796Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"\n[3] Splitting 70/30 by time\n","output_type":"stream"}],"execution_count":51},{"cell_type":"code","source":"df.head()","metadata":{"_uuid":"d8f9898a-11bc-40c5-af0a-224fe5617dd1","_cell_guid":"52d694a9-a402-4534-9777-7c751336d042","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:39.471391Z","iopub.execute_input":"2026-07-12T07:04:39.471709Z","iopub.status.idle":"2026-07-12T07:04:39.486998Z","shell.execute_reply.started":"2026-07-12T07:04:39.471687Z","shell.execute_reply":"2026-07-12T07:04:39.486186Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":52,"output_type":"execute_result","data":{"text/plain":"   implied_volatility    delta    gamma    theta     vega      rho  tte_days  \\\n0             1.24983  0.99396  0.00055 -0.32269  0.00423  0.01093         0   \n1             1.25456  0.99311  0.00062 -0.35671  0.00476  0.01095         0   \n2             1.21492  0.99383  0.00058 -0.32065  0.00431  0.01099         0   \n3             1.14861  1.00000  0.00000 -0.05913  0.00000  0.01109         0   \n4             1.13163  1.00000  0.00000 -0.05928  0.00000  0.01112         0   \n\n   log_moneyness  strike  type_enc  volume  open_interest   mark       date  \\\n0      -0.218156   402.0         1     130              1  70.77 2024-01-02   \n1      -0.215672   403.0         1       9              0  69.78 2024-01-02   \n2      -0.213193   404.0         1       1              1  68.77 2024-01-02   \n3      -0.210721   405.0         1       0              4  67.52 2024-01-02   \n4      -0.208255   406.0         1       0              3  66.52 2024-01-02   \n\n           contractID  \n0  SPY240102C00402000  \n1  SPY240102C00403000  \n2  SPY240102C00404000  \n3  SPY240102C00405000  \n4  SPY240102C00406000  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>tte_days</th>\n      <th>log_moneyness</th>\n      <th>strike</th>\n      <th>type_enc</th>\n      <th>volume</th>\n      <th>open_interest</th>\n      <th>mark</th>\n      <th>date</th>\n      <th>contractID</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.24983</td>\n      <td>0.99396</td>\n      <td>0.00055</td>\n      <td>-0.32269</td>\n      <td>0.00423</td>\n      <td>0.01093</td>\n      <td>0</td>\n      <td>-0.218156</td>\n      <td>402.0</td>\n      <td>1</td>\n      <td>130</td>\n      <td>1</td>\n      <td>70.77</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00402000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.25456</td>\n      <td>0.99311</td>\n      <td>0.00062</td>\n      <td>-0.35671</td>\n      <td>0.00476</td>\n      <td>0.01095</td>\n      <td>0</td>\n      <td>-0.215672</td>\n      <td>403.0</td>\n      <td>1</td>\n      <td>9</td>\n      <td>0</td>\n      <td>69.78</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00403000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1.21492</td>\n      <td>0.99383</td>\n      <td>0.00058</td>\n      <td>-0.32065</td>\n      <td>0.00431</td>\n      <td>0.01099</td>\n      <td>0</td>\n      <td>-0.213193</td>\n      <td>404.0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>68.77</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00404000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.14861</td>\n      <td>1.00000</td>\n      <td>0.00000</td>\n      <td>-0.05913</td>\n      <td>0.00000</td>\n      <td>0.01109</td>\n      <td>0</td>\n      <td>-0.210721</td>\n      <td>405.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>4</td>\n      <td>67.52</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00405000</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1.13163</td>\n      <td>1.00000</td>\n      <td>0.00000</td>\n      <td>-0.05928</td>\n      <td>0.00000</td>\n      <td>0.01112</td>\n      <td>0</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>3</td>\n      <td>66.52</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00406000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":52},{"cell_type":"code","source":"df.shape","metadata":{"_uuid":"57a63ffc-12b4-4704-a02c-2aa250fd5eba","_cell_guid":"8dd66831-534a-43e9-83b6-0d4bf95881a8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:39.488141Z","iopub.execute_input":"2026-07-12T07:04:39.488443Z","iopub.status.idle":"2026-07-12T07:04:39.499488Z","shell.execute_reply.started":"2026-07-12T07:04:39.488413Z","shell.execute_reply":"2026-07-12T07:04:39.498781Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":53,"output_type":"execute_result","data":{"text/plain":"(2292798, 15)"},"metadata":{}}],"execution_count":53},{"cell_type":"code","source":"len(df['contractID'].value_counts())","metadata":{"_uuid":"cc4e9b51-3f18-4417-98b5-86208be8fe44","_cell_guid":"78259ae8-82e5-4693-872c-e2d4be4d9f82","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:39.500485Z","iopub.execute_input":"2026-07-12T07:04:39.500908Z","iopub.status.idle":"2026-07-12T07:04:39.765202Z","shell.execute_reply.started":"2026-07-12T07:04:39.500886Z","shell.execute_reply":"2026-07-12T07:04:39.764368Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":54,"output_type":"execute_result","data":{"text/plain":"76278"},"metadata":{}}],"execution_count":54},{"cell_type":"code","source":"cnt=1\nfor i in (df['contractID'].value_counts()):\n    if i>=11:\n        cnt+=i\ncnt","metadata":{"_uuid":"4d405286-914c-407a-82e8-f46067d3c93f","_cell_guid":"99022f68-cf64-4ea0-bec4-93ba18609319","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:39.769117Z","iopub.execute_input":"2026-07-12T07:04:39.769996Z","iopub.status.idle":"2026-07-12T07:04:40.014791Z","shell.execute_reply.started":"2026-07-12T07:04:39.769968Z","shell.execute_reply":"2026-07-12T07:04:40.014198Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":55,"output_type":"execute_result","data":{"text/plain":"1952207"},"metadata":{}}],"execution_count":55},{"cell_type":"code","source":"obs_per_contract=df['contractID'].value_counts()","metadata":{"_uuid":"1dd8449b-8816-4d54-a905-855779abad00","_cell_guid":"ba65da0d-3da7-4237-afe8-f17d7e2e114f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:40.015607Z","iopub.execute_input":"2026-07-12T07:04:40.015885Z","iopub.status.idle":"2026-07-12T07:04:40.247543Z","shell.execute_reply.started":"2026-07-12T07:04:40.015862Z","shell.execute_reply":"2026-07-12T07:04:40.246781Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":56},{"cell_type":"code","source":"valid_contracts=obs_per_contract[obs_per_contract>=8].index","metadata":{"_uuid":"176028b9-7b19-417c-8c40-65e3504fcca6","_cell_guid":"2b8a7366-f602-4e7e-99c4-450482f98634","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:40.248610Z","iopub.execute_input":"2026-07-12T07:04:40.248904Z","iopub.status.idle":"2026-07-12T07:04:40.259720Z","shell.execute_reply.started":"2026-07-12T07:04:40.248881Z","shell.execute_reply":"2026-07-12T07:04:40.258797Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":57},{"cell_type":"code","source":"valid_contracts","metadata":{"_uuid":"1f7149a1-155c-493d-b54c-bb2b7be1b9a8","_cell_guid":"e9245f51-8b8a-4315-912c-d0137f793c67","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:40.260737Z","iopub.execute_input":"2026-07-12T07:04:40.261130Z","iopub.status.idle":"2026-07-12T07:04:40.272870Z","shell.execute_reply.started":"2026-07-12T07:04:40.261106Z","shell.execute_reply":"2026-07-12T07:04:40.272110Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":58,"output_type":"execute_result","data":{"text/plain":"Index(['SPY241231C00457000', 'SPY241231C00490000', 'SPY250117P00475000',\n       'SPY250117P00480000', 'SPY250117P00405000', 'SPY250117P00410000',\n       'SPY250117P00415000', 'SPY250117P00180000', 'SPY250117P00425000',\n       'SPY250117P00430000',\n       ...\n       'SPY241230P00626000', 'SPY241230P00627000', 'SPY241227P00627000',\n       'SPY241227P00628000', 'SPY241227P00629000', 'SPY241227P00626000',\n       'SPY241230P00616000', 'SPY241230P00617000', 'SPY241230P00618000',\n       'SPY241230P00619000'],\n      dtype='object', name='contractID', length=58004)"},"metadata":{}}],"execution_count":58},{"cell_type":"code","source":"before=2292798\ndf=df[df['contractID'].isin(valid_contracts)].copy()\nafter=len(df)","metadata":{"_uuid":"3fb5cee2-4b1d-4eb5-93a6-77be1bb5ad62","_cell_guid":"50fd8a7d-4802-4966-b786-0474da6ac2ea","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:40.273696Z","iopub.execute_input":"2026-07-12T07:04:40.274079Z","iopub.status.idle":"2026-07-12T07:04:40.651311Z","shell.execute_reply.started":"2026-07-12T07:04:40.274028Z","shell.execute_reply":"2026-07-12T07:04:40.650601Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":59},{"cell_type":"code","source":"print(f'rows kept: {after}')\nprint(f'Original number of rows: {before}')\nprint(f'Rows dropped: {before - after}')","metadata":{"_uuid":"b1edb8dc-9b9e-42a0-8481-f47ff19e40c2","_cell_guid":"a695a3fd-9a6c-4157-9334-be426e2eeeb2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:40.652137Z","iopub.execute_input":"2026-07-12T07:04:40.652429Z","iopub.status.idle":"2026-07-12T07:04:40.657074Z","shell.execute_reply.started":"2026-07-12T07:04:40.652408Z","shell.execute_reply":"2026-07-12T07:04:40.656381Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"rows kept: 2204060\nOriginal number of rows: 2292798\nRows dropped: 88738\n","output_type":"stream"}],"execution_count":60},{"cell_type":"code","source":"df.head()","metadata":{"_uuid":"25a13baa-0e45-4998-bfe3-ec89654adb76","_cell_guid":"c9e213dc-e7dc-4233-8ef3-47a17daa62c1","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:40.657941Z","iopub.execute_input":"2026-07-12T07:04:40.658138Z","iopub.status.idle":"2026-07-12T07:04:40.679950Z","shell.execute_reply.started":"2026-07-12T07:04:40.658119Z","shell.execute_reply":"2026-07-12T07:04:40.679129Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":61,"output_type":"execute_result","data":{"text/plain":"      implied_volatility    delta    gamma    theta     vega      rho  \\\n6692             0.51083  0.97450  0.00157 -0.18256  0.04411  0.09691   \n6693             0.46327  0.98414  0.00124 -0.13792  0.02757  0.08721   \n6694             0.54482  0.97222  0.00181 -0.21802  0.04131  0.07522   \n6695             0.59497  0.97196  0.00180 -0.24861  0.03859  0.06446   \n6696             0.65137  0.99633  0.00039 -0.11028  0.00472  0.03321   \n\n      tte_days  log_moneyness  strike  type_enc  volume  open_interest   mark  \\\n6692         9      -0.208255   406.0         1       0              0  67.56   \n6693         8      -0.208255   406.0         1       2              0  63.45   \n6694         7      -0.208255   406.0         1       0              1  62.08   \n6695         6      -0.208255   406.0         1       0              1  62.67   \n6696         3      -0.208255   406.0         1       0              1  68.81   \n\n           date          contractID  \n6692 2024-01-02  SPY240111C00406000  \n6693 2024-01-03  SPY240111C00406000  \n6694 2024-01-04  SPY240111C00406000  \n6695 2024-01-05  SPY240111C00406000  \n6696 2024-01-08  SPY240111C00406000  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>tte_days</th>\n      <th>log_moneyness</th>\n      <th>strike</th>\n      <th>type_enc</th>\n      <th>volume</th>\n      <th>open_interest</th>\n      <th>mark</th>\n      <th>date</th>\n      <th>contractID</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>6692</th>\n      <td>0.51083</td>\n      <td>0.97450</td>\n      <td>0.00157</td>\n      <td>-0.18256</td>\n      <td>0.04411</td>\n      <td>0.09691</td>\n      <td>9</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>67.56</td>\n      <td>2024-01-02</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6693</th>\n      <td>0.46327</td>\n      <td>0.98414</td>\n      <td>0.00124</td>\n      <td>-0.13792</td>\n      <td>0.02757</td>\n      <td>0.08721</td>\n      <td>8</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>0</td>\n      <td>63.45</td>\n      <td>2024-01-03</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6694</th>\n      <td>0.54482</td>\n      <td>0.97222</td>\n      <td>0.00181</td>\n      <td>-0.21802</td>\n      <td>0.04131</td>\n      <td>0.07522</td>\n      <td>7</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>62.08</td>\n      <td>2024-01-04</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6695</th>\n      <td>0.59497</td>\n      <td>0.97196</td>\n      <td>0.00180</td>\n      <td>-0.24861</td>\n      <td>0.03859</td>\n      <td>0.06446</td>\n      <td>6</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>62.67</td>\n      <td>2024-01-05</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6696</th>\n      <td>0.65137</td>\n      <td>0.99633</td>\n      <td>0.00039</td>\n      <td>-0.11028</td>\n      <td>0.00472</td>\n      <td>0.03321</td>\n      <td>3</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>68.81</td>\n      <td>2024-01-08</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":61},{"cell_type":"code","source":"# now we sort the values\nunique_dates=sorted(df['date'].unique())\nsplit_date=unique_dates[int(len(unique_dates)*0.80)]\nsplit_date","metadata":{"_uuid":"46e59dd5-7773-4a50-b9bc-bbcb554f09eb","_cell_guid":"2472659a-d6bc-41f7-990d-f0d1cef8546e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:40.680973Z","iopub.execute_input":"2026-07-12T07:04:40.681364Z","iopub.status.idle":"2026-07-12T07:04:40.707213Z","shell.execute_reply.started":"2026-07-12T07:04:40.681343Z","shell.execute_reply":"2026-07-12T07:04:40.706662Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":62,"output_type":"execute_result","data":{"text/plain":"Timestamp('2024-10-18 00:00:00')"},"metadata":{}}],"execution_count":62},{"cell_type":"code","source":"df.columns","metadata":{"_uuid":"0cfeab2c-0f7a-42aa-b609-a7aeac396d11","_cell_guid":"740f3ef9-c141-448e-bdc7-d60919d1741e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:40.708008Z","iopub.execute_input":"2026-07-12T07:04:40.708381Z","iopub.status.idle":"2026-07-12T07:04:40.713521Z","shell.execute_reply.started":"2026-07-12T07:04:40.708361Z","shell.execute_reply":"2026-07-12T07:04:40.712900Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":63,"output_type":"execute_result","data":{"text/plain":"Index(['implied_volatility', 'delta', 'gamma', 'theta', 'vega', 'rho',\n       'tte_days', 'log_moneyness', 'strike', 'type_enc', 'volume',\n       'open_interest', 'mark', 'date', 'contractID'],\n      dtype='object')"},"metadata":{}}],"execution_count":63},{"cell_type":"code","source":"train_df = df[df[\"date\"] < split_date].copy()\ntest_df = df[df[\"date\"] >= split_date].copy()","metadata":{"_uuid":"a7a88afb-36fc-42d4-9122-b6e5e4152159","_cell_guid":"a611f716-19ef-46f7-b247-99fee1446634","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:40.714411Z","iopub.execute_input":"2026-07-12T07:04:40.714691Z","iopub.status.idle":"2026-07-12T07:04:40.981376Z","shell.execute_reply.started":"2026-07-12T07:04:40.714670Z","shell.execute_reply":"2026-07-12T07:04:40.980776Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":64},{"cell_type":"code","source":"print(f\"    Split date : {pd.Timestamp(split_date).date()}\")\nprint(f\"    Train : {len(train_df):,} rows  \"\n      f\"({train_df['date'].min().date()} → {train_df['date'].max().date()})\")\nprint(f\"    Test  : {len(test_df):,} rows  \"\n      f\"({test_df['date'].min().date()} → {test_df['date'].max().date()})\")","metadata":{"_uuid":"9db104fa-f59d-4ca4-ab4b-0b9cf28b3529","_cell_guid":"c52cbfa1-a568-431a-b296-58510c79a717","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:40.982244Z","iopub.execute_input":"2026-07-12T07:04:40.982560Z","iopub.status.idle":"2026-07-12T07:04:41.000546Z","shell.execute_reply.started":"2026-07-12T07:04:40.982536Z","shell.execute_reply":"2026-07-12T07:04:40.999917Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"    Split date : 2024-10-18\n    Train : 1,736,542 rows  (2024-01-02 → 2024-10-17)\n    Test  : 467,518 rows  (2024-10-18 → 2024-12-31)\n","output_type":"stream"}],"execution_count":65},{"cell_type":"code","source":"df.head()","metadata":{"_uuid":"00e403b1-c809-47bd-851d-9ec6c4b9fa83","_cell_guid":"963779dd-934a-44bc-a6fb-bbfc622f55a4","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:41.001394Z","iopub.execute_input":"2026-07-12T07:04:41.001774Z","iopub.status.idle":"2026-07-12T07:04:41.016470Z","shell.execute_reply.started":"2026-07-12T07:04:41.001744Z","shell.execute_reply":"2026-07-12T07:04:41.015644Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":66,"output_type":"execute_result","data":{"text/plain":"      implied_volatility    delta    gamma    theta     vega      rho  \\\n6692             0.51083  0.97450  0.00157 -0.18256  0.04411  0.09691   \n6693             0.46327  0.98414  0.00124 -0.13792  0.02757  0.08721   \n6694             0.54482  0.97222  0.00181 -0.21802  0.04131  0.07522   \n6695             0.59497  0.97196  0.00180 -0.24861  0.03859  0.06446   \n6696             0.65137  0.99633  0.00039 -0.11028  0.00472  0.03321   \n\n      tte_days  log_moneyness  strike  type_enc  volume  open_interest   mark  \\\n6692         9      -0.208255   406.0         1       0              0  67.56   \n6693         8      -0.208255   406.0         1       2              0  63.45   \n6694         7      -0.208255   406.0         1       0              1  62.08   \n6695         6      -0.208255   406.0         1       0              1  62.67   \n6696         3      -0.208255   406.0         1       0              1  68.81   \n\n           date          contractID  \n6692 2024-01-02  SPY240111C00406000  \n6693 2024-01-03  SPY240111C00406000  \n6694 2024-01-04  SPY240111C00406000  \n6695 2024-01-05  SPY240111C00406000  \n6696 2024-01-08  SPY240111C00406000  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>tte_days</th>\n      <th>log_moneyness</th>\n      <th>strike</th>\n      <th>type_enc</th>\n      <th>volume</th>\n      <th>open_interest</th>\n      <th>mark</th>\n      <th>date</th>\n      <th>contractID</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>6692</th>\n      <td>0.51083</td>\n      <td>0.97450</td>\n      <td>0.00157</td>\n      <td>-0.18256</td>\n      <td>0.04411</td>\n      <td>0.09691</td>\n      <td>9</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>67.56</td>\n      <td>2024-01-02</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6693</th>\n      <td>0.46327</td>\n      <td>0.98414</td>\n      <td>0.00124</td>\n      <td>-0.13792</td>\n      <td>0.02757</td>\n      <td>0.08721</td>\n      <td>8</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>0</td>\n      <td>63.45</td>\n      <td>2024-01-03</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6694</th>\n      <td>0.54482</td>\n      <td>0.97222</td>\n      <td>0.00181</td>\n      <td>-0.21802</td>\n      <td>0.04131</td>\n      <td>0.07522</td>\n      <td>7</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>62.08</td>\n      <td>2024-01-04</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6695</th>\n      <td>0.59497</td>\n      <td>0.97196</td>\n      <td>0.00180</td>\n      <td>-0.24861</td>\n      <td>0.03859</td>\n      <td>0.06446</td>\n      <td>6</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>62.67</td>\n      <td>2024-01-05</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6696</th>\n      <td>0.65137</td>\n      <td>0.99633</td>\n      <td>0.00039</td>\n      <td>-0.11028</td>\n      <td>0.00472</td>\n      <td>0.03321</td>\n      <td>3</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>68.81</td>\n      <td>2024-01-08</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":66},{"cell_type":"markdown","source":"# Scaling","metadata":{"_uuid":"f54c4ca2-e05e-4fec-82c7-b1402610999b","_cell_guid":"82b7f035-8a44-474f-829c-139b1e7e0587","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"feat_scaler=StandardScaler()\ntarget_scaler=StandardScaler()","metadata":{"_uuid":"d5a00422-f841-4572-a088-92ea92c5bfcb","_cell_guid":"f2866e49-96c9-47f4-a0f9-a026b61a5bdb","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:41.017814Z","iopub.execute_input":"2026-07-12T07:04:41.018257Z","iopub.status.idle":"2026-07-12T07:04:41.026320Z","shell.execute_reply.started":"2026-07-12T07:04:41.018213Z","shell.execute_reply":"2026-07-12T07:04:41.025526Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":67},{"cell_type":"code","source":"train_df[FEATURES]=feat_scaler.fit_transform(train_df[FEATURES].values)\ntest_df[FEATURES] = feat_scaler.transform(test_df[FEATURES].values)","metadata":{"_uuid":"f5702e77-5252-46e3-8507-12323b3de05f","_cell_guid":"87b5a7d9-d532-4d34-9ae1-aa6aaae68eca","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:41.027465Z","iopub.execute_input":"2026-07-12T07:04:41.027816Z","iopub.status.idle":"2026-07-12T07:04:41.508164Z","shell.execute_reply.started":"2026-07-12T07:04:41.027782Z","shell.execute_reply":"2026-07-12T07:04:41.507565Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":68},{"cell_type":"code","source":"print(type(train_df))\nprint(type(test_df))\nprint(TARGET)","metadata":{"_uuid":"ac24ec22-dd7f-475b-bc9b-4930c7d54d73","_cell_guid":"fb077811-1f7e-4750-bf23-795b3b11c512","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:41.509178Z","iopub.execute_input":"2026-07-12T07:04:41.509623Z","iopub.status.idle":"2026-07-12T07:04:41.514260Z","shell.execute_reply.started":"2026-07-12T07:04:41.509598Z","shell.execute_reply":"2026-07-12T07:04:41.513259Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\n<class 'pandas.core.frame.DataFrame'>\nmark\n","output_type":"stream"}],"execution_count":69},{"cell_type":"code","source":"train_df.head()","metadata":{"_uuid":"edebada9-984a-478c-94e8-b9c1e2350df4","_cell_guid":"6a2eb758-7413-4095-b466-76bafcc902c6","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:41.515146Z","iopub.execute_input":"2026-07-12T07:04:41.515533Z","iopub.status.idle":"2026-07-12T07:04:41.536968Z","shell.execute_reply.started":"2026-07-12T07:04:41.515464Z","shell.execute_reply":"2026-07-12T07:04:41.536023Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":70,"output_type":"execute_result","data":{"text/plain":"      implied_volatility     delta     gamma     theta      vega       rho  \\\n6692            0.889534  1.294105 -0.256925 -0.837888 -0.731375 -0.013656   \n6693            0.723119  1.310376 -0.279555 -0.493957 -0.754353 -0.018176   \n6694            1.008467  1.290257 -0.240467 -1.111091 -0.735265 -0.023763   \n6695            1.183945  1.289818 -0.241153 -1.346773 -0.739043 -0.028777   \n6696            1.381292  1.330952 -0.337844 -0.281003 -0.786097 -0.043338   \n\n      tte_days  log_moneyness    strike  type_enc    volume  open_interest  \\\n6692 -0.769782      -0.541181 -0.687226       1.0 -0.075309      -0.236488   \n6693 -0.774344      -0.541181 -0.687226       1.0 -0.075122      -0.236488   \n6694 -0.778907      -0.541181 -0.687226       1.0 -0.075309      -0.236384   \n6695 -0.783470      -0.541181 -0.687226       1.0 -0.075309      -0.236384   \n6696 -0.797158      -0.541181 -0.687226       1.0 -0.075309      -0.236384   \n\n       mark       date          contractID  \n6692  67.56 2024-01-02  SPY240111C00406000  \n6693  63.45 2024-01-03  SPY240111C00406000  \n6694  62.08 2024-01-04  SPY240111C00406000  \n6695  62.67 2024-01-05  SPY240111C00406000  \n6696  68.81 2024-01-08  SPY240111C00406000  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>tte_days</th>\n      <th>log_moneyness</th>\n      <th>strike</th>\n      <th>type_enc</th>\n      <th>volume</th>\n      <th>open_interest</th>\n      <th>mark</th>\n      <th>date</th>\n      <th>contractID</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>6692</th>\n      <td>0.889534</td>\n      <td>1.294105</td>\n      <td>-0.256925</td>\n      <td>-0.837888</td>\n      <td>-0.731375</td>\n      <td>-0.013656</td>\n      <td>-0.769782</td>\n      <td>-0.541181</td>\n      <td>-0.687226</td>\n      <td>1.0</td>\n      <td>-0.075309</td>\n      <td>-0.236488</td>\n      <td>67.56</td>\n      <td>2024-01-02</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6693</th>\n      <td>0.723119</td>\n      <td>1.310376</td>\n      <td>-0.279555</td>\n      <td>-0.493957</td>\n      <td>-0.754353</td>\n      <td>-0.018176</td>\n      <td>-0.774344</td>\n      <td>-0.541181</td>\n      <td>-0.687226</td>\n      <td>1.0</td>\n      <td>-0.075122</td>\n      <td>-0.236488</td>\n      <td>63.45</td>\n      <td>2024-01-03</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6694</th>\n      <td>1.008467</td>\n      <td>1.290257</td>\n      <td>-0.240467</td>\n      <td>-1.111091</td>\n      <td>-0.735265</td>\n      <td>-0.023763</td>\n      <td>-0.778907</td>\n      <td>-0.541181</td>\n      <td>-0.687226</td>\n      <td>1.0</td>\n      <td>-0.075309</td>\n      <td>-0.236384</td>\n      <td>62.08</td>\n      <td>2024-01-04</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6695</th>\n      <td>1.183945</td>\n      <td>1.289818</td>\n      <td>-0.241153</td>\n      <td>-1.346773</td>\n      <td>-0.739043</td>\n      <td>-0.028777</td>\n      <td>-0.783470</td>\n      <td>-0.541181</td>\n      <td>-0.687226</td>\n      <td>1.0</td>\n      <td>-0.075309</td>\n      <td>-0.236384</td>\n      <td>62.67</td>\n      <td>2024-01-05</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6696</th>\n      <td>1.381292</td>\n      <td>1.330952</td>\n      <td>-0.337844</td>\n      <td>-0.281003</td>\n      <td>-0.786097</td>\n      <td>-0.043338</td>\n      <td>-0.797158</td>\n      <td>-0.541181</td>\n      <td>-0.687226</td>\n      <td>1.0</td>\n      <td>-0.075309</td>\n      <td>-0.236384</td>\n      <td>68.81</td>\n      <td>2024-01-08</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":70},{"cell_type":"code","source":"train_df[TARGET]=target_scaler.fit_transform(train_df[[TARGET]].values).ravel()\ntest_df[TARGET]=target_scaler.transform(test_df[[TARGET]].values).ravel()","metadata":{"_uuid":"e912d2f3-d255-45ec-a4e2-813addf4d209","_cell_guid":"3a06082d-bb0b-45ae-8ef5-64824abc55a2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:41.537916Z","iopub.execute_input":"2026-07-12T07:04:41.538237Z","iopub.status.idle":"2026-07-12T07:04:41.577237Z","shell.execute_reply.started":"2026-07-12T07:04:41.538218Z","shell.execute_reply":"2026-07-12T07:04:41.576618Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":71},{"cell_type":"code","source":"SEQ_LEN=6\nBATCH_SIZE=1024","metadata":{"_uuid":"0fdf313a-12ae-43f3-859e-201a3516825f","_cell_guid":"44c5a4ad-e47f-4e4c-9752-2cc1b772fcd7","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:41.578231Z","iopub.execute_input":"2026-07-12T07:04:41.578453Z","iopub.status.idle":"2026-07-12T07:04:41.582248Z","shell.execute_reply.started":"2026-07-12T07:04:41.578434Z","shell.execute_reply":"2026-07-12T07:04:41.581375Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":72},{"cell_type":"code","source":"class LazyContractDataset(Dataset):\n    def __init__(self,df,features,target,seq_len):\n        self.seq_len=seq_len\n        self.features=features\n        self.target=target\n\n        self.index=[]\n        self.contract_data=[]\n        skipped=0\n        n_contracts=0\n        for contract, group in df.groupby(by='contractID',sort=False):\n            group=group.sort_values(by='date')\n            X = group[features].values.astype(np.float32)\n            y = group[target].values.astype(np.float32)\n\n\n            if len(X)<seq_len+1:\n                skipped+=1\n                continue\n\n            n_contracts+=1\n\n            arr_idx=len(self.contract_data)\n            self.contract_data.append((X,y))\n\n            for i in range(len(X)-seq_len):\n                self.index.append((arr_idx,i))\n\n        print(f\"    Contracts used    : {n_contracts:,}\")\n        print(f\"    Contracts skipped : {skipped:,}  \"\n              f\"(< {seq_len + 1} observations)\")\n        print(f\"    Total sequences   : {len(self.index):,}\")\n            \n    def __len__(self):\n        return len(self.index)\n\n    def __getitem__(self,idx):\n        arr_idx, start=self.index[idx]\n        X,y=self.contract_data[arr_idx]\n        x_seq=torch.tensor(X[start:start+self.seq_len],dtype=torch.float32)\n        label=torch.tensor(y[start+self.seq_len],dtype=torch.float32)\n        return x_seq,label","metadata":{"_uuid":"f2ab323c-d66d-414c-99bd-792989b4380b","_cell_guid":"f1f83372-8e54-41f5-96ef-cac9c29e6201","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:41.583117Z","iopub.execute_input":"2026-07-12T07:04:41.583407Z","iopub.status.idle":"2026-07-12T07:04:41.595362Z","shell.execute_reply.started":"2026-07-12T07:04:41.583372Z","shell.execute_reply":"2026-07-12T07:04:41.594796Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":73},{"cell_type":"markdown","source":"# Now bulding dataset","metadata":{"_uuid":"529fede9-19b0-4b28-9c5d-93e4010c4015","_cell_guid":"ff2f892c-eb57-456d-a0ba-c444c9e7d5b4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"train_dataset=LazyContractDataset(train_df,features=FEATURES, target=TARGET, seq_len=SEQ_LEN)\ntest_dataset=LazyContractDataset(test_df,features=FEATURES, target=TARGET, seq_len=SEQ_LEN)","metadata":{"_uuid":"947ca5ba-7b64-4056-9572-daaf08bf0a40","_cell_guid":"c01d3f0a-c892-430a-9530-aafab8a1e9da","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:04:41.596542Z","iopub.execute_input":"2026-07-12T07:04:41.596874Z","iopub.status.idle":"2026-07-12T07:05:42.177497Z","shell.execute_reply.started":"2026-07-12T07:04:41.596855Z","shell.execute_reply":"2026-07-12T07:05:42.176554Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"    Contracts used    : 47,008\n    Contracts skipped : 1,080  (< 7 observations)\n    Total sequences   : 1,450,982\n    Contracts used    : 17,388\n    Contracts skipped : 1,738  (< 7 observations)\n    Total sequences   : 357,994\n","output_type":"stream"}],"execution_count":74},{"cell_type":"code","source":"BATCH_SIZE","metadata":{"_uuid":"3162acaa-7632-47e1-a39d-c4664c2330fe","_cell_guid":"75e33e4a-1cc9-4a07-aba3-023b76b5601a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:05:42.178674Z","iopub.execute_input":"2026-07-12T07:05:42.179066Z","iopub.status.idle":"2026-07-12T07:05:42.184341Z","shell.execute_reply.started":"2026-07-12T07:05:42.179043Z","shell.execute_reply":"2026-07-12T07:05:42.183523Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":75,"output_type":"execute_result","data":{"text/plain":"1024"},"metadata":{}}],"execution_count":75},{"cell_type":"code","source":"train_loader = DataLoader(\n    train_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=True,\n    drop_last=True,\n    num_workers=16, #changed from 4\n    pin_memory=True,\n    persistent_workers=True,\n    prefetch_factor=2,\n)\n\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    drop_last=False,\n    num_workers=16, #changed from 4\n    pin_memory=True,\n    persistent_workers=True,\n    prefetch_factor=2,\n)","metadata":{"_uuid":"a994ed27-2b8c-43a8-8e03-4e406214dc03","_cell_guid":"47c058d2-56ab-486b-b9c6-5e60b18be69e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:05:42.185318Z","iopub.execute_input":"2026-07-12T07:05:42.185634Z","iopub.status.idle":"2026-07-12T07:05:42.198402Z","shell.execute_reply.started":"2026-07-12T07:05:42.185605Z","shell.execute_reply":"2026-07-12T07:05:42.197761Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":76},{"cell_type":"code","source":"","metadata":{"_uuid":"86d5bf5c-1426-4705-9d70-3800b81c6cca","_cell_guid":"59e0bbab-b603-4433-a92d-af1bd223c15c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Buidling the model","metadata":{"_uuid":"8c40b95a-3cf1-42a5-985b-f7ea8e954c56","_cell_guid":"0e04a9a2-a6ac-465c-8c46-a161e05876fb","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"class OptionPricingLSTM(nn.Module):\n    def __init__(self,input_size,hidden_size=128, num_layers=2,dropout=0.3):\n        super().__init__()\n        self.lstm=nn.LSTM(input_size=input_size,\n                         hidden_size=hidden_size,\n                         num_layers=num_layers,\n                         batch_first=True,\n                         dropout=dropout if num_layers>1 else 0.0)\n\n        self.head=nn.Sequential(nn.Dropout(dropout),\n                               nn.Linear(hidden_size,64),\n                               nn.ReLU(),\n                               nn.Dropout(0.2),\n                               nn.Linear(64,1))\n\n    def forward(self,x):\n        lstm_out,_=self.lstm(x)\n        last_step = lstm_out[:, -1, :]\n        return self.head(last_step).squeeze(-1)","metadata":{"_uuid":"7b6504af-b3cc-476e-b8b6-5765b73846b6","_cell_guid":"fbaf12b8-f74b-4395-80f8-9e8283876e74","trusted":true,"execution":{"iopub.status.busy":"2026-07-12T07:05:42.199511Z","iopub.execute_input":"2026-07-12T07:05:42.199891Z","iopub.status.idle":"2026-07-12T07:05:42.212380Z","shell.execute_reply.started":"2026-07-12T07:05:42.199857Z","shell.execute_reply":"2026-07-12T07:05:42.211485Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":77},{"cell_type":"code","source":"model=OptionPricingLSTM(input_size=len(FEATURES))","metadata":{"_uuid":"45c48deb-fa5f-44f5-b350-eea63847a725","_cell_guid":"7cdae850-3b03-4aac-acef-29c68c01e67e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:05:42.213299Z","iopub.execute_input":"2026-07-12T07:05:42.213562Z","iopub.status.idle":"2026-07-12T07:05:42.244885Z","shell.execute_reply.started":"2026-07-12T07:05:42.213536Z","shell.execute_reply":"2026-07-12T07:05:42.244264Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":78},{"cell_type":"code","source":"if torch.cuda.device_count()>1:\n    print(torch.cuda.device_count())\n    model=nn.DataParallel(model)\nelse:\n    pass","metadata":{"_uuid":"3e0e241b-39b7-48f2-a36c-335f75a6e43f","_cell_guid":"9199d4ba-a785-488b-8343-49407424df43","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:05:42.245833Z","iopub.execute_input":"2026-07-12T07:05:42.246130Z","iopub.status.idle":"2026-07-12T07:05:42.268333Z","shell.execute_reply.started":"2026-07-12T07:05:42.246100Z","shell.execute_reply":"2026-07-12T07:05:42.267769Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"2\n","output_type":"stream"}],"execution_count":79},{"cell_type":"code","source":"model=model.to(device)","metadata":{"_uuid":"097b0e00-6e64-4134-8d4c-e99219a0814d","_cell_guid":"1e5863f1-d88b-4066-97fc-2e657a24c8b4","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:05:42.269089Z","iopub.execute_input":"2026-07-12T07:05:42.269380Z","iopub.status.idle":"2026-07-12T07:05:42.614119Z","shell.execute_reply.started":"2026-07-12T07:05:42.269339Z","shell.execute_reply":"2026-07-12T07:05:42.613482Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":80},{"cell_type":"code","source":"EPOCHS=30\nLR=1e-3","metadata":{"_uuid":"1dfdef5a-79c7-470a-a31d-d31dded70046","_cell_guid":"4f90cd16-386d-47a0-b524-5d8692e6cfff","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:05:42.615085Z","iopub.execute_input":"2026-07-12T07:05:42.615432Z","iopub.status.idle":"2026-07-12T07:05:42.619578Z","shell.execute_reply.started":"2026-07-12T07:05:42.615411Z","shell.execute_reply":"2026-07-12T07:05:42.618718Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":81},{"cell_type":"code","source":"criterion=nn.MSELoss()\noptimizer=torch.optim.Adam(model.parameters(),lr=LR,weight_decay=1e-5)","metadata":{"_uuid":"f128ff0d-42ca-4c60-8070-f0ee5dd49e66","_cell_guid":"9dbfe66b-b93e-42a1-aba7-1902fe44b156","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:05:42.620567Z","iopub.execute_input":"2026-07-12T07:05:42.620902Z","iopub.status.idle":"2026-07-12T07:05:45.805803Z","shell.execute_reply.started":"2026-07-12T07:05:42.620866Z","shell.execute_reply":"2026-07-12T07:05:45.805137Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":82},{"cell_type":"code","source":"scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer, mode=\"min\", factor=0.5, patience=3\n)","metadata":{"_uuid":"7e125840-4947-4c17-83b2-253dd31e6456","_cell_guid":"3abf716c-4ae5-4836-8c93-5d08a3eb30d8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:05:45.806784Z","iopub.execute_input":"2026-07-12T07:05:45.807318Z","iopub.status.idle":"2026-07-12T07:05:45.811547Z","shell.execute_reply.started":"2026-07-12T07:05:45.807288Z","shell.execute_reply":"2026-07-12T07:05:45.810689Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":83},{"cell_type":"markdown","source":"# Training","metadata":{"_uuid":"05c0da91-03b4-4a74-9412-c77ee0999b54","_cell_guid":"e24de432-8849-4e70-9c59-a2fdfeb249c9","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"train_losses = []\n\nprint(f'training on for {EPOCHS} epochs')\n\nfor epoch in range(EPOCHS):\n    model.train()\n    running_loss=0.0\n\n    for X_batch, y_batch in train_loader:\n        X_batch=X_batch.to(device,non_blocking=True)\n        y_batch=y_batch.to(device, non_blocking=True)\n\n        optimizer.zero_grad()\n        preds=model(X_batch)\n        loss=criterion(preds,y_batch)\n        loss.backward()\n\n        nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        optimizer.step()\n        running_loss+=loss.item()\n\n    avg_loss=running_loss/len(train_loader)\n    current_lr=optimizer.param_groups[0]['lr']\n    train_losses.append(avg_loss)\n    scheduler.step(avg_loss)\n    print(f'epoch:{epoch+1} loss:{avg_loss} LR:{current_lr}')","metadata":{"_uuid":"e7523f76-4bee-4199-b2b4-c40ec8e4a795","_cell_guid":"e171c83c-01d4-4c68-8b78-f1c0b1c12354","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:05:45.812814Z","iopub.execute_input":"2026-07-12T07:05:45.813168Z","iopub.status.idle":"2026-07-12T07:16:32.491136Z","shell.execute_reply.started":"2026-07-12T07:05:45.813136Z","shell.execute_reply":"2026-07-12T07:16:32.490242Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"training on for 30 epochs\nepoch:1 loss:0.05067175640698291 LR:0.001\nepoch:2 loss:0.02886417260561875 LR:0.001\nepoch:3 loss:0.02629137144155674 LR:0.001\nepoch:4 loss:0.02488526912000935 LR:0.001\nepoch:5 loss:0.024054257234684743 LR:0.001\nepoch:6 loss:0.023198551172785208 LR:0.001\nepoch:7 loss:0.022722694424301974 LR:0.001\nepoch:8 loss:0.022274812311197153 LR:0.001\nepoch:9 loss:0.021895766447661287 LR:0.001\nepoch:10 loss:0.02128858027827142 LR:0.001\nepoch:11 loss:0.02083619935819511 LR:0.001\nepoch:12 loss:0.019970218592460166 LR:0.001\nepoch:13 loss:0.019521985312208195 LR:0.001\nepoch:14 loss:0.01977012960745369 LR:0.001\nepoch:15 loss:0.018920153472177938 LR:0.001\nepoch:16 loss:0.019095796597121798 LR:0.001\nepoch:17 loss:0.018941531774698425 LR:0.001\nepoch:18 loss:0.018922938998146666 LR:0.001\nepoch:19 loss:0.01835728722813902 LR:0.001\nepoch:20 loss:0.01817961946908532 LR:0.001\nepoch:21 loss:0.01899816576420398 LR:0.001\nepoch:22 loss:0.01829559677925045 LR:0.001\nepoch:23 loss:0.018710537478827612 LR:0.001\nepoch:24 loss:0.01802235204952711 LR:0.001\nepoch:25 loss:0.01834113232062677 LR:0.001\nepoch:26 loss:0.018370916514452232 LR:0.001\nepoch:27 loss:0.01797492826083875 LR:0.001\nepoch:28 loss:0.017949430506129217 LR:0.001\nepoch:29 loss:0.01768716086244137 LR:0.001\nepoch:30 loss:0.01792415532370303 LR:0.001\n","output_type":"stream"}],"execution_count":84},{"cell_type":"markdown","source":"# Testing","metadata":{"_uuid":"b9f68887-0d46-4894-aded-12b2e56c8bc6","_cell_guid":"150b5d27-308d-48c3-be82-f68ec050730a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"model.eval()","metadata":{"_uuid":"83dde9a8-77a1-4f18-98ea-648ca677d190","_cell_guid":"93af4ad9-182d-49f1-8ff9-edf8014916aa","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:16:32.492664Z","iopub.execute_input":"2026-07-12T07:16:32.493104Z","iopub.status.idle":"2026-07-12T07:16:32.500410Z","shell.execute_reply.started":"2026-07-12T07:16:32.493066Z","shell.execute_reply":"2026-07-12T07:16:32.499588Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":85,"output_type":"execute_result","data":{"text/plain":"DataParallel(\n  (module): OptionPricingLSTM(\n    (lstm): LSTM(12, 128, num_layers=2, batch_first=True, dropout=0.3)\n    (head): Sequential(\n      (0): Dropout(p=0.3, inplace=False)\n      (1): Linear(in_features=128, out_features=64, bias=True)\n      (2): ReLU()\n      (3): Dropout(p=0.2, inplace=False)\n      (4): Linear(in_features=64, out_features=1, bias=True)\n    )\n  )\n)"},"metadata":{}}],"execution_count":85},{"cell_type":"code","source":"all_preds=[]\nall_origs=[]\nwith torch.no_grad():\n    for X_batch, y_batch in test_loader:\n        X_batch=X_batch.to(device, non_blocking=True)\n        batch_pred=model(X_batch).cpu().numpy()\n        all_preds.extend(batch_pred)\n        all_origs.extend(y_batch)","metadata":{"_uuid":"81b65aa3-21fa-4dca-aa10-32357c15a739","_cell_guid":"3aeed4c4-09b4-499d-a830-1d4127041d88","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:16:32.501468Z","iopub.execute_input":"2026-07-12T07:16:32.501917Z","iopub.status.idle":"2026-07-12T07:16:40.286668Z","shell.execute_reply.started":"2026-07-12T07:16:32.501894Z","shell.execute_reply":"2026-07-12T07:16:40.285418Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":86},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score","metadata":{"_uuid":"636477da-8711-4f10-934c-5711831687ee","_cell_guid":"0c0a333a-26d5-4c4e-971c-8d1cd6315bc0","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:16:40.288582Z","iopub.execute_input":"2026-07-12T07:16:40.288936Z","iopub.status.idle":"2026-07-12T07:16:40.293921Z","shell.execute_reply.started":"2026-07-12T07:16:40.288891Z","shell.execute_reply":"2026-07-12T07:16:40.293243Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":87},{"cell_type":"code","source":"r2_score(all_origs,all_preds)","metadata":{"_uuid":"93310640-4b27-49fb-aec2-230bb9e95f87","_cell_guid":"59880383-0666-44c2-b290-8498e03db0e2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:16:40.295012Z","iopub.execute_input":"2026-07-12T07:16:40.295366Z","iopub.status.idle":"2026-07-12T07:16:41.959629Z","shell.execute_reply.started":"2026-07-12T07:16:40.295320Z","shell.execute_reply":"2026-07-12T07:16:41.958889Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":88,"output_type":"execute_result","data":{"text/plain":"0.9879795191757003"},"metadata":{}}],"execution_count":88},{"cell_type":"code","source":"mean_squared_error(all_origs,all_preds), mean_absolute_error(all_origs,all_preds)","metadata":{"_uuid":"241f12cf-8067-4169-81c7-34b04ba7a42b","_cell_guid":"99e76e29-0a73-4235-a5d1-83b8b664f449","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:16:41.960800Z","iopub.execute_input":"2026-07-12T07:16:41.961240Z","iopub.status.idle":"2026-07-12T07:16:45.252665Z","shell.execute_reply.started":"2026-07-12T07:16:41.961214Z","shell.execute_reply":"2026-07-12T07:16:45.251649Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":89,"output_type":"execute_result","data":{"text/plain":"(0.016767606507159045, 0.07299135060156613)"},"metadata":{}}],"execution_count":89},{"cell_type":"code","source":"unscaled_origs=target_scaler.inverse_transform(np.array(all_origs).reshape(-1,1)).ravel()\nunscaled_preds=target_scaler.inverse_transform(np.array(all_preds).reshape(-1,1)).ravel()","metadata":{"_uuid":"9824c92f-33a5-46ef-a11e-1cb56ed749f6","_cell_guid":"6bc90077-ee91-423b-8d43-44cd016ea4e6","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:16:45.253977Z","iopub.execute_input":"2026-07-12T07:16:45.254591Z","iopub.status.idle":"2026-07-12T07:16:46.950372Z","shell.execute_reply.started":"2026-07-12T07:16:45.254568Z","shell.execute_reply":"2026-07-12T07:16:46.949784Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":90},{"cell_type":"code","source":"print(f'R2 score: {r2_score(unscaled_origs, unscaled_preds)}')\nprint(f'Mean Sqaured Error: {mean_squared_error(unscaled_origs,unscaled_preds)}')\nprint(f'Root Mean Squared Error: {np.sqrt(mean_squared_error(unscaled_origs,unscaled_preds))}')\nprint(f'Mean Absolute Error: {mean_absolute_error(unscaled_origs,unscaled_preds)}')","metadata":{"_uuid":"3fef2cb3-b6a4-415e-801b-38d9781d2ebe","_cell_guid":"1b19cec5-bb67-42bf-b103-d222e285e4b2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:16:46.951236Z","iopub.execute_input":"2026-07-12T07:16:46.951542Z","iopub.status.idle":"2026-07-12T07:16:46.964448Z","shell.execute_reply.started":"2026-07-12T07:16:46.951519Z","shell.execute_reply":"2026-07-12T07:16:46.963653Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"R2 score: 0.987979531288147\nMean Sqaured Error: 95.2016372680664\nRoot Mean Squared Error: 9.757132635568013\nMean Absolute Error: 5.499941825866699\n","output_type":"stream"}],"execution_count":91},{"cell_type":"code","source":"model","metadata":{"_uuid":"60d8c48c-5822-4872-b3fa-093f90772f14","_cell_guid":"ab1aae7b-6bb6-458c-8893-c7d51169c475","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:16:46.965608Z","iopub.execute_input":"2026-07-12T07:16:46.966512Z","iopub.status.idle":"2026-07-12T07:16:46.971895Z","shell.execute_reply.started":"2026-07-12T07:16:46.966479Z","shell.execute_reply":"2026-07-12T07:16:46.971172Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":92,"output_type":"execute_result","data":{"text/plain":"DataParallel(\n  (module): OptionPricingLSTM(\n    (lstm): LSTM(12, 128, num_layers=2, batch_first=True, dropout=0.3)\n    (head): Sequential(\n      (0): Dropout(p=0.3, inplace=False)\n      (1): Linear(in_features=128, out_features=64, bias=True)\n      (2): ReLU()\n      (3): Dropout(p=0.2, inplace=False)\n      (4): Linear(in_features=64, out_features=1, bias=True)\n    )\n  )\n)"},"metadata":{}}],"execution_count":92},{"cell_type":"code","source":"len(FEATURES)","metadata":{"_uuid":"db7f3ab0-441d-4879-b8c0-d6cda41f16a3","_cell_guid":"66f787a5-27b5-4e36-b45a-d1c03ae4c502","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T07:16:46.972964Z","iopub.execute_input":"2026-07-12T07:16:46.973987Z","iopub.status.idle":"2026-07-12T07:16:46.984689Z","shell.execute_reply.started":"2026-07-12T07:16:46.973965Z","shell.execute_reply":"2026-07-12T07:16:46.984070Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":93,"output_type":"execute_result","data":{"text/plain":"12"},"metadata":{}}],"execution_count":93},{"cell_type":"code","source":"","metadata":{"_uuid":"6bd3f232-0bbb-47f7-93e1-b9cbff1ab88a","_cell_guid":"f152a542-7abf-49a4-8d6c-745fe08ba5b2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"1e3aa1bb-6a02-4237-9f87-309e2672aa1b","_cell_guid":"aab36aee-1b66-4edf-b8dc-417b662341ea","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"2763c1d9-8fc4-44f8-9072-6b6ff0351501","_cell_guid":"68e29598-f086-4549-af47-38b5ff37c95d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}

regime-lstm-with-seq-len-6.ipynb

@@ -0,0 +1 @@
+{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nimport kagglehub\n# kagglehub.dataset_download('<owner>/<dataset-slug>')","metadata":{"_uuid":"508a2511-be48-49a7-83fb-159ec6db7850","_cell_guid":"c0be5ab7-86df-414f-8a76-829a461ebcd4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:45:49.246780Z","iopub.execute_input":"2026-07-12T06:45:49.247054Z","iopub.status.idle":"2026-07-12T06:45:51.248352Z","shell.execute_reply.started":"2026-07-12T06:45:49.247021Z","shell.execute_reply":"2026-07-12T06:45:51.247630Z"}},"outputs":[],"execution_count":1},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"b8a41ebd-b005-4998-b78a-1b8d7942886f","_cell_guid":"d7fe1f54-7d32-41d4-9267-1202158471ed","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:45:51.250234Z","iopub.execute_input":"2026-07-12T06:45:51.250698Z","iopub.status.idle":"2026-07-12T06:45:56.995871Z","shell.execute_reply.started":"2026-07-12T06:45:51.250673Z","shell.execute_reply":"2026-07-12T06:45:56.995284Z"}},"outputs":[],"execution_count":2},{"cell_type":"code","source":"SEED=42\ntorch.manual_seed(42)\nnp.random.seed(42)","metadata":{"_uuid":"a1977de5-7a3b-4136-a17b-24a316ffe1af","_cell_guid":"de96a6fa-0bc8-4d51-9b36-28cf717b0065","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:45:56.996820Z","iopub.execute_input":"2026-07-12T06:45:56.997326Z","iopub.status.idle":"2026-07-12T06:45:57.007389Z","shell.execute_reply.started":"2026-07-12T06:45:56.997254Z","shell.execute_reply":"2026-07-12T06:45:57.006650Z"}},"outputs":[],"execution_count":3},{"cell_type":"code","source":"device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f'Using device: {device}')\nprint(f'GPUs available: f{torch.cuda.device_count()}')","metadata":{"_uuid":"5c5c9a52-6390-490f-9163-2bb761845c3c","_cell_guid":"bbaebdb8-8563-4190-8e6e-efd8a6e5d09e","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:45:57.008242Z","iopub.execute_input":"2026-07-12T06:45:57.008482Z","iopub.status.idle":"2026-07-12T06:45:57.330635Z","shell.execute_reply.started":"2026-07-12T06:45:57.008463Z","shell.execute_reply":"2026-07-12T06:45:57.329967Z"}},"outputs":[{"name":"stdout","text":"Using device: cuda\nGPUs available: f2\n","output_type":"stream"}],"execution_count":4},{"cell_type":"markdown","source":"# Preprocessing","metadata":{"_uuid":"dcf78362-41b0-4dff-9603-b1c97a7bb1c0","_cell_guid":"8cce33da-4528-4195-bfd0-26a711ddcca1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"df = pd.read_csv(\"hf://datasets/major-year-project/dataset_with_regimes/datasets_with_regimes.csv\")\nprint(f'Raw shape: {df.shape}')","metadata":{"_uuid":"0303ec36-3ba4-48fe-be16-a12fd6c8a123","_cell_guid":"68c96174-713a-43f9-b23c-b92dd6881e73","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:45:57.331563Z","iopub.execute_input":"2026-07-12T06:45:57.331924Z","iopub.status.idle":"2026-07-12T06:46:25.967965Z","shell.execute_reply.started":"2026-07-12T06:45:57.331847Z","shell.execute_reply":"2026-07-12T06:46:25.967264Z"}},"outputs":[{"name":"stdout","text":"Raw shape: (2292798, 22)\n","output_type":"stream"}],"execution_count":5},{"cell_type":"code","source":"df.head(3)","metadata":{"_uuid":"01b6e7ec-20f8-4a58-ac6e-aaa041ca3d2e","_cell_guid":"b7d7ecc5-516a-454e-bbb0-f0871ee4cc2a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:25.969051Z","iopub.execute_input":"2026-07-12T06:46:25.969403Z","iopub.status.idle":"2026-07-12T06:46:26.013832Z","shell.execute_reply.started":"2026-07-12T06:46:25.969381Z","shell.execute_reply":"2026-07-12T06:46:26.013071Z"}},"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"           contractID symbol  expiration  strike  type   last   mark    bid  \\\n0  SPY240102C00402000    SPY  2024-01-02   402.0  call  69.97  70.77  70.68   \n1  SPY240102P00402000    SPY  2024-01-02   402.0   put   0.02   0.01   0.00   \n2  SPY240102C00403000    SPY  2024-01-02   403.0  call  68.97  69.78  69.68   \n\n   bid_size    ask  ...  open_interest        date  implied_volatility  \\\n0       100  70.85  ...              1  2024-01-02             1.24983   \n1         0   0.01  ...            303  2024-01-02             1.04145   \n2       100  69.89  ...              0  2024-01-02             1.25456   \n\n     delta    gamma    theta     vega      rho  regime  regime_name  \n0  0.99396  0.00055 -0.32269  0.00423  0.01093       0  contraction  \n1 -0.00135  0.00017 -0.05700  0.00110 -0.00002       0  contraction  \n2  0.99311  0.00062 -0.35671  0.00476  0.01095       0  contraction  \n\n[3 rows x 22 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>contractID</th>\n      <th>symbol</th>\n      <th>expiration</th>\n      <th>strike</th>\n      <th>type</th>\n      <th>last</th>\n      <th>mark</th>\n      <th>bid</th>\n      <th>bid_size</th>\n      <th>ask</th>\n      <th>...</th>\n      <th>open_interest</th>\n      <th>date</th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>regime</th>\n      <th>regime_name</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>SPY240102C00402000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>402.0</td>\n      <td>call</td>\n      <td>69.97</td>\n      <td>70.77</td>\n      <td>70.68</td>\n      <td>100</td>\n      <td>70.85</td>\n      <td>...</td>\n      <td>1</td>\n      <td>2024-01-02</td>\n      <td>1.24983</td>\n      <td>0.99396</td>\n      <td>0.00055</td>\n      <td>-0.32269</td>\n      <td>0.00423</td>\n      <td>0.01093</td>\n      <td>0</td>\n      <td>contraction</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>SPY240102P00402000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>402.0</td>\n      <td>put</td>\n      <td>0.02</td>\n      <td>0.01</td>\n      <td>0.00</td>\n      <td>0</td>\n      <td>0.01</td>\n      <td>...</td>\n      <td>303</td>\n      <td>2024-01-02</td>\n      <td>1.04145</td>\n      <td>-0.00135</td>\n      <td>0.00017</td>\n      <td>-0.05700</td>\n      <td>0.00110</td>\n      <td>-0.00002</td>\n      <td>0</td>\n      <td>contraction</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>SPY240102C00403000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>403.0</td>\n      <td>call</td>\n      <td>68.97</td>\n      <td>69.78</td>\n      <td>69.68</td>\n      <td>100</td>\n      <td>69.89</td>\n      <td>...</td>\n      <td>0</td>\n      <td>2024-01-02</td>\n      <td>1.25456</td>\n      <td>0.99311</td>\n      <td>0.00062</td>\n      <td>-0.35671</td>\n      <td>0.00476</td>\n      <td>0.01095</td>\n      <td>0</td>\n      <td>contraction</td>\n    </tr>\n  </tbody>\n</table>\n<p>3 rows × 22 columns</p>\n</div>"},"metadata":{}}],"execution_count":6},{"cell_type":"code","source":"df.info()","metadata":{"_uuid":"ea3dad55-4a2e-4f36-bd24-dbe9aed2f93c","_cell_guid":"8e4562dd-3f6f-414a-b3f9-e457a4c971fc","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:26.016743Z","iopub.execute_input":"2026-07-12T06:46:26.017034Z","iopub.status.idle":"2026-07-12T06:46:26.050707Z","shell.execute_reply.started":"2026-07-12T06:46:26.017010Z","shell.execute_reply":"2026-07-12T06:46:26.049970Z"}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2292798 entries, 0 to 2292797\nData columns (total 22 columns):\n #   Column              Dtype  \n---  ------              -----  \n 0   contractID          object \n 1   symbol              object \n 2   expiration          object \n 3   strike              float64\n 4   type                object \n 5   last                float64\n 6   mark                float64\n 7   bid                 float64\n 8   bid_size            int64  \n 9   ask                 float64\n 10  ask_size            int64  \n 11  volume              int64  \n 12  open_interest       int64  \n 13  date                object \n 14  implied_volatility  float64\n 15  delta               float64\n 16  gamma               float64\n 17  theta               float64\n 18  vega                float64\n 19  rho                 float64\n 20  regime              int64  \n 21  regime_name         object \ndtypes: float64(11), int64(5), object(6)\nmemory usage: 384.8+ MB\n","output_type":"stream"}],"execution_count":7},{"cell_type":"code","source":"df['date']=pd.to_datetime(df['date'])\ndf['expiration']=pd.to_datetime(df['expiration'])","metadata":{"_uuid":"f0b5a298-33e9-4af7-a102-97504bca840a","_cell_guid":"65f2c63b-a7c8-4201-b1c2-4629f0e60b59","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:26.051599Z","iopub.execute_input":"2026-07-12T06:46:26.051953Z","iopub.status.idle":"2026-07-12T06:46:26.483926Z","shell.execute_reply.started":"2026-07-12T06:46:26.051931Z","shell.execute_reply":"2026-07-12T06:46:26.483317Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"df.info()","metadata":{"_uuid":"e5b79b6a-a56d-44f0-a467-303c139cefd4","_cell_guid":"f0e79c84-2e87-4f80-8673-b17b8997a021","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:26.484786Z","iopub.execute_input":"2026-07-12T06:46:26.485336Z","iopub.status.idle":"2026-07-12T06:46:26.493423Z","shell.execute_reply.started":"2026-07-12T06:46:26.485300Z","shell.execute_reply":"2026-07-12T06:46:26.492793Z"}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2292798 entries, 0 to 2292797\nData columns (total 22 columns):\n #   Column              Dtype         \n---  ------              -----         \n 0   contractID          object        \n 1   symbol              object        \n 2   expiration          datetime64[ns]\n 3   strike              float64       \n 4   type                object        \n 5   last                float64       \n 6   mark                float64       \n 7   bid                 float64       \n 8   bid_size            int64         \n 9   ask                 float64       \n 10  ask_size            int64         \n 11  volume              int64         \n 12  open_interest       int64         \n 13  date                datetime64[ns]\n 14  implied_volatility  float64       \n 15  delta               float64       \n 16  gamma               float64       \n 17  theta               float64       \n 18  vega                float64       \n 19  rho                 float64       \n 20  regime              int64         \n 21  regime_name         object        \ndtypes: datetime64[ns](2), float64(11), int64(5), object(4)\nmemory usage: 384.8+ MB\n","output_type":"stream"}],"execution_count":9},{"cell_type":"code","source":"df['tte_days']=(df['expiration']-df['date']).dt.days","metadata":{"_uuid":"3432b6a6-0196-4488-81f1-0f14679f1be7","_cell_guid":"77a33d05-3cb2-4d22-bba7-9ec46fb2307b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:26.494398Z","iopub.execute_input":"2026-07-12T06:46:26.495395Z","iopub.status.idle":"2026-07-12T06:46:26.556229Z","shell.execute_reply.started":"2026-07-12T06:46:26.495329Z","shell.execute_reply":"2026-07-12T06:46:26.555338Z"}},"outputs":[],"execution_count":10},{"cell_type":"code","source":"# Log-moneyness: how far strike is from the median strike\n# (proxy since we don't have live spot price in this file)\ndf[\"log_moneyness\"] = np.log(\n    df[\"strike\"] / df[\"strike\"].median()\n).fillna(0)","metadata":{"_uuid":"16176136-3cd0-4f90-9b5c-e2c25a7d3667","_cell_guid":"c66ba583-57c3-4d2d-a214-b522522bc07c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:26.557446Z","iopub.execute_input":"2026-07-12T06:46:26.558257Z","iopub.status.idle":"2026-07-12T06:46:26.594646Z","shell.execute_reply.started":"2026-07-12T06:46:26.558223Z","shell.execute_reply":"2026-07-12T06:46:26.593994Z"}},"outputs":[],"execution_count":11},{"cell_type":"code","source":"df.info()","metadata":{"_uuid":"940be297-fda6-42ec-ba1f-1d9576833aeb","_cell_guid":"5f57b8b9-e16b-48a2-9b70-69a0dc90261d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:26.595635Z","iopub.execute_input":"2026-07-12T06:46:26.595900Z","iopub.status.idle":"2026-07-12T06:46:26.604114Z","shell.execute_reply.started":"2026-07-12T06:46:26.595871Z","shell.execute_reply":"2026-07-12T06:46:26.603435Z"}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2292798 entries, 0 to 2292797\nData columns (total 24 columns):\n #   Column              Dtype         \n---  ------              -----         \n 0   contractID          object        \n 1   symbol              object        \n 2   expiration          datetime64[ns]\n 3   strike              float64       \n 4   type                object        \n 5   last                float64       \n 6   mark                float64       \n 7   bid                 float64       \n 8   bid_size            int64         \n 9   ask                 float64       \n 10  ask_size            int64         \n 11  volume              int64         \n 12  open_interest       int64         \n 13  date                datetime64[ns]\n 14  implied_volatility  float64       \n 15  delta               float64       \n 16  gamma               float64       \n 17  theta               float64       \n 18  vega                float64       \n 19  rho                 float64       \n 20  regime              int64         \n 21  regime_name         object        \n 22  tte_days            int64         \n 23  log_moneyness       float64       \ndtypes: datetime64[ns](2), float64(12), int64(6), object(4)\nmemory usage: 419.8+ MB\n","output_type":"stream"}],"execution_count":12},{"cell_type":"code","source":"df.head()","metadata":{"_uuid":"224a6007-1e1a-4c3f-a6fa-975f31375b0a","_cell_guid":"8ff1049f-64e3-4d39-9ef2-f1d9fb19b94b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:26.605172Z","iopub.execute_input":"2026-07-12T06:46:26.605611Z","iopub.status.idle":"2026-07-12T06:46:26.631914Z","shell.execute_reply.started":"2026-07-12T06:46:26.605575Z","shell.execute_reply":"2026-07-12T06:46:26.631092Z"}},"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"           contractID symbol expiration  strike  type   last   mark    bid  \\\n0  SPY240102C00402000    SPY 2024-01-02   402.0  call  69.97  70.77  70.68   \n1  SPY240102P00402000    SPY 2024-01-02   402.0   put   0.02   0.01   0.00   \n2  SPY240102C00403000    SPY 2024-01-02   403.0  call  68.97  69.78  69.68   \n3  SPY240102P00403000    SPY 2024-01-02   403.0   put   0.01   0.01   0.00   \n4  SPY240102C00404000    SPY 2024-01-02   404.0  call  68.64  68.77  68.68   \n\n   bid_size    ask  ...  implied_volatility    delta    gamma    theta  \\\n0       100  70.85  ...             1.24983  0.99396  0.00055 -0.32269   \n1         0   0.01  ...             1.04145 -0.00135  0.00017 -0.05700   \n2       100  69.89  ...             1.25456  0.99311  0.00062 -0.35671   \n3         0   0.01  ...             1.02682 -0.00137  0.00018 -0.05692   \n4       100  68.85  ...             1.21492  0.99383  0.00058 -0.32065   \n\n      vega      rho  regime  regime_name  tte_days  log_moneyness  \n0  0.00423  0.01093       0  contraction         0      -0.218156  \n1  0.00110 -0.00002       0  contraction         0      -0.218156  \n2  0.00476  0.01095       0  contraction         0      -0.215672  \n3  0.00111 -0.00002       0  contraction         0      -0.215672  \n4  0.00431  0.01099       0  contraction         0      -0.213193  \n\n[5 rows x 24 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>contractID</th>\n      <th>symbol</th>\n      <th>expiration</th>\n      <th>strike</th>\n      <th>type</th>\n      <th>last</th>\n      <th>mark</th>\n      <th>bid</th>\n      <th>bid_size</th>\n      <th>ask</th>\n      <th>...</th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>regime</th>\n      <th>regime_name</th>\n      <th>tte_days</th>\n      <th>log_moneyness</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>SPY240102C00402000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>402.0</td>\n      <td>call</td>\n      <td>69.97</td>\n      <td>70.77</td>\n      <td>70.68</td>\n      <td>100</td>\n      <td>70.85</td>\n      <td>...</td>\n      <td>1.24983</td>\n      <td>0.99396</td>\n      <td>0.00055</td>\n      <td>-0.32269</td>\n      <td>0.00423</td>\n      <td>0.01093</td>\n      <td>0</td>\n      <td>contraction</td>\n      <td>0</td>\n      <td>-0.218156</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>SPY240102P00402000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>402.0</td>\n      <td>put</td>\n      <td>0.02</td>\n      <td>0.01</td>\n      <td>0.00</td>\n      <td>0</td>\n      <td>0.01</td>\n      <td>...</td>\n      <td>1.04145</td>\n      <td>-0.00135</td>\n      <td>0.00017</td>\n      <td>-0.05700</td>\n      <td>0.00110</td>\n      <td>-0.00002</td>\n      <td>0</td>\n      <td>contraction</td>\n      <td>0</td>\n      <td>-0.218156</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>SPY240102C00403000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>403.0</td>\n      <td>call</td>\n      <td>68.97</td>\n      <td>69.78</td>\n      <td>69.68</td>\n      <td>100</td>\n      <td>69.89</td>\n      <td>...</td>\n      <td>1.25456</td>\n      <td>0.99311</td>\n      <td>0.00062</td>\n      <td>-0.35671</td>\n      <td>0.00476</td>\n      <td>0.01095</td>\n      <td>0</td>\n      <td>contraction</td>\n      <td>0</td>\n      <td>-0.215672</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>SPY240102P00403000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>403.0</td>\n      <td>put</td>\n      <td>0.01</td>\n      <td>0.01</td>\n      <td>0.00</td>\n      <td>0</td>\n      <td>0.01</td>\n      <td>...</td>\n      <td>1.02682</td>\n      <td>-0.00137</td>\n      <td>0.00018</td>\n      <td>-0.05692</td>\n      <td>0.00111</td>\n      <td>-0.00002</td>\n      <td>0</td>\n      <td>contraction</td>\n      <td>0</td>\n      <td>-0.215672</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>SPY240102C00404000</td>\n      <td>SPY</td>\n      <td>2024-01-02</td>\n      <td>404.0</td>\n      <td>call</td>\n      <td>68.64</td>\n      <td>68.77</td>\n      <td>68.68</td>\n      <td>100</td>\n      <td>68.85</td>\n      <td>...</td>\n      <td>1.21492</td>\n      <td>0.99383</td>\n      <td>0.00058</td>\n      <td>-0.32065</td>\n      <td>0.00431</td>\n      <td>0.01099</td>\n      <td>0</td>\n      <td>contraction</td>\n      <td>0</td>\n      <td>-0.213193</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 24 columns</p>\n</div>"},"metadata":{}}],"execution_count":13},{"cell_type":"code","source":"df[\"type_enc\"] = (df[\"type\"].str.lower() == \"call\").astype(int)","metadata":{"_uuid":"2edc369e-2555-4095-b8d7-5ff115313c2f","_cell_guid":"2a344630-5f5c-416e-a803-ae63930ec557","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:26.632975Z","iopub.execute_input":"2026-07-12T06:46:26.633298Z","iopub.status.idle":"2026-07-12T06:46:27.119063Z","shell.execute_reply.started":"2026-07-12T06:46:26.633248Z","shell.execute_reply":"2026-07-12T06:46:27.118444Z"}},"outputs":[],"execution_count":14},{"cell_type":"code","source":"df.info()","metadata":{"_uuid":"244e529e-6f3a-4700-893b-044fc9152ba0","_cell_guid":"7846d472-4c8d-4ad8-97b9-764cd22f2ac2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:27.119954Z","iopub.execute_input":"2026-07-12T06:46:27.120305Z","iopub.status.idle":"2026-07-12T06:46:27.129004Z","shell.execute_reply.started":"2026-07-12T06:46:27.120242Z","shell.execute_reply":"2026-07-12T06:46:27.128319Z"}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2292798 entries, 0 to 2292797\nData columns (total 25 columns):\n #   Column              Dtype         \n---  ------              -----         \n 0   contractID          object        \n 1   symbol              object        \n 2   expiration          datetime64[ns]\n 3   strike              float64       \n 4   type                object        \n 5   last                float64       \n 6   mark                float64       \n 7   bid                 float64       \n 8   bid_size            int64         \n 9   ask                 float64       \n 10  ask_size            int64         \n 11  volume              int64         \n 12  open_interest       int64         \n 13  date                datetime64[ns]\n 14  implied_volatility  float64       \n 15  delta               float64       \n 16  gamma               float64       \n 17  theta               float64       \n 18  vega                float64       \n 19  rho                 float64       \n 20  regime              int64         \n 21  regime_name         object        \n 22  tte_days            int64         \n 23  log_moneyness       float64       \n 24  type_enc            int64         \ndtypes: datetime64[ns](2), float64(12), int64(7), object(4)\nmemory usage: 437.3+ MB\n","output_type":"stream"}],"execution_count":15},{"cell_type":"code","source":"# Remove rows where target is missing / zero / negative\ndf = df[df[\"mark\"] > 0].copy()\ndf = df.dropna(subset=[\n    \"mark\", \"implied_volatility\",\n    \"delta\", \"gamma\", \"theta\", \"vega\", \"rho\"\n])\n\nprint(f\"    After cleaning : {df.shape}\")","metadata":{"_uuid":"79f0f90b-6224-45ed-9537-00ca8d584b7c","_cell_guid":"698f8d65-c331-41c3-a0dc-e3fec1a90cc0","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:27.129949Z","iopub.execute_input":"2026-07-12T06:46:27.130150Z","iopub.status.idle":"2026-07-12T06:46:28.290411Z","shell.execute_reply.started":"2026-07-12T06:46:27.130130Z","shell.execute_reply":"2026-07-12T06:46:28.289649Z"}},"outputs":[{"name":"stdout","text":"    After cleaning : (2292798, 25)\n","output_type":"stream"}],"execution_count":16},{"cell_type":"code","source":"FEATURES = [\n    # Theory-grounded: what Black-Scholes uses\n    \"implied_volatility\",   # market's expectation of future vol\n    \"delta\",                # price sensitivity\n    \"gamma\",                # rate of delta change\n    \"theta\",                # daily time decay\n    \"vega\",                 # vol sensitivity\n    \"rho\",                  # interest rate sensitivity\n\n    # Contract structure\n    \"tte_days\",             # time to expiry (engineered)\n    \"log_moneyness\",        # engineered from strike\n    \"strike\",               # absolute strike level\n    \"type_enc\",             # call or put\n\n    # Market activity (not price itself)\n    \"volume\",               # contracts traded today\n    \"open_interest\",        # total open contracts\n\n    \"regime\"\n]\n\nTARGET = \"mark\"","metadata":{"_uuid":"f234ed42-c39e-4ab2-acc4-e6c2951b979f","_cell_guid":"c11588ca-248d-4071-94e4-52b1a43d7a33","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:28.291389Z","iopub.execute_input":"2026-07-12T06:46:28.291714Z","iopub.status.idle":"2026-07-12T06:46:28.296714Z","shell.execute_reply.started":"2026-07-12T06:46:28.291679Z","shell.execute_reply":"2026-07-12T06:46:28.295855Z"}},"outputs":[],"execution_count":17},{"cell_type":"code","source":"# keep 'regime' around so we can one-hot encode it next\ndf = df[FEATURES + [TARGET, \"date\", \"contractID\"]].dropna()\nprint(f\"    Final shape : {df.shape}\")\nprint(f\"    Features ({len(FEATURES)}) : {FEATURES}\")","metadata":{"_uuid":"15e88fda-87f6-412c-b588-06709d5cfd4d","_cell_guid":"65938b5c-d469-469e-9d3d-1737744b39f5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:28.297641Z","iopub.execute_input":"2026-07-12T06:46:28.298137Z","iopub.status.idle":"2026-07-12T06:46:28.689168Z","shell.execute_reply.started":"2026-07-12T06:46:28.298089Z","shell.execute_reply":"2026-07-12T06:46:28.688497Z"}},"outputs":[{"name":"stdout","text":"    Final shape : (2292798, 16)\n    Features (13) : ['implied_volatility', 'delta', 'gamma', 'theta', 'vega', 'rho', 'tte_days', 'log_moneyness', 'strike', 'type_enc', 'volume', 'open_interest', 'regime']\n","output_type":"stream"}],"execution_count":18},{"cell_type":"code","source":"# # One-hot encode regime (categorical HMM state, not ordinal)\n# regime_dummies = pd.get_dummies(df[\"regime\"], prefix=\"regime\").astype(np.float32)\n# df = pd.concat([df, regime_dummies], axis=1)\n\n# FEATURES = FEATURES + list(regime_dummies.columns)\n# print(f\"    Features after regime one-hot ({len(FEATURES)}) : {FEATURES}\")","metadata":{"_uuid":"0d918cb5-269b-41b2-9c49-a143d71889ca","_cell_guid":"260984fb-e5df-49d7-9631-978fa3fbe9d0","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:28.690212Z","iopub.execute_input":"2026-07-12T06:46:28.690608Z","iopub.status.idle":"2026-07-12T06:46:28.694571Z","shell.execute_reply.started":"2026-07-12T06:46:28.690584Z","shell.execute_reply":"2026-07-12T06:46:28.693753Z"}},"outputs":[],"execution_count":19},{"cell_type":"code","source":"df.head()","metadata":{"_uuid":"e4a1da77-d03a-4b85-b8b0-b7f595a76a7c","_cell_guid":"be8dc048-44bc-4c14-929c-1b66944b4671","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:28.695402Z","iopub.execute_input":"2026-07-12T06:46:28.695658Z","iopub.status.idle":"2026-07-12T06:46:28.721827Z","shell.execute_reply.started":"2026-07-12T06:46:28.695629Z","shell.execute_reply":"2026-07-12T06:46:28.720945Z"}},"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"   implied_volatility    delta    gamma    theta     vega      rho  tte_days  \\\n0             1.24983  0.99396  0.00055 -0.32269  0.00423  0.01093         0   \n1             1.04145 -0.00135  0.00017 -0.05700  0.00110 -0.00002         0   \n2             1.25456  0.99311  0.00062 -0.35671  0.00476  0.01095         0   \n3             1.02682 -0.00137  0.00018 -0.05692  0.00111 -0.00002         0   \n4             1.21492  0.99383  0.00058 -0.32065  0.00431  0.01099         0   \n\n   log_moneyness  strike  type_enc  volume  open_interest  regime   mark  \\\n0      -0.218156   402.0         1     130              1       0  70.77   \n1      -0.218156   402.0         0       0            303       0   0.01   \n2      -0.215672   403.0         1       9              0       0  69.78   \n3      -0.215672   403.0         0       1              8       0   0.01   \n4      -0.213193   404.0         1       1              1       0  68.77   \n\n        date          contractID  \n0 2024-01-02  SPY240102C00402000  \n1 2024-01-02  SPY240102P00402000  \n2 2024-01-02  SPY240102C00403000  \n3 2024-01-02  SPY240102P00403000  \n4 2024-01-02  SPY240102C00404000  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>tte_days</th>\n      <th>log_moneyness</th>\n      <th>strike</th>\n      <th>type_enc</th>\n      <th>volume</th>\n      <th>open_interest</th>\n      <th>regime</th>\n      <th>mark</th>\n      <th>date</th>\n      <th>contractID</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.24983</td>\n      <td>0.99396</td>\n      <td>0.00055</td>\n      <td>-0.32269</td>\n      <td>0.00423</td>\n      <td>0.01093</td>\n      <td>0</td>\n      <td>-0.218156</td>\n      <td>402.0</td>\n      <td>1</td>\n      <td>130</td>\n      <td>1</td>\n      <td>0</td>\n      <td>70.77</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00402000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.04145</td>\n      <td>-0.00135</td>\n      <td>0.00017</td>\n      <td>-0.05700</td>\n      <td>0.00110</td>\n      <td>-0.00002</td>\n      <td>0</td>\n      <td>-0.218156</td>\n      <td>402.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>303</td>\n      <td>0</td>\n      <td>0.01</td>\n      <td>2024-01-02</td>\n      <td>SPY240102P00402000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1.25456</td>\n      <td>0.99311</td>\n      <td>0.00062</td>\n      <td>-0.35671</td>\n      <td>0.00476</td>\n      <td>0.01095</td>\n      <td>0</td>\n      <td>-0.215672</td>\n      <td>403.0</td>\n      <td>1</td>\n      <td>9</td>\n      <td>0</td>\n      <td>0</td>\n      <td>69.78</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00403000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.02682</td>\n      <td>-0.00137</td>\n      <td>0.00018</td>\n      <td>-0.05692</td>\n      <td>0.00111</td>\n      <td>-0.00002</td>\n      <td>0</td>\n      <td>-0.215672</td>\n      <td>403.0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>8</td>\n      <td>0</td>\n      <td>0.01</td>\n      <td>2024-01-02</td>\n      <td>SPY240102P00403000</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1.21492</td>\n      <td>0.99383</td>\n      <td>0.00058</td>\n      <td>-0.32065</td>\n      <td>0.00431</td>\n      <td>0.01099</td>\n      <td>0</td>\n      <td>-0.213193</td>\n      <td>404.0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>68.77</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00404000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":20},{"cell_type":"markdown","source":"# Splitting by 70/30 by time","metadata":{"_uuid":"e6486f4f-8a9f-431d-a68b-9461d4fc9bb4","_cell_guid":"3c2e098e-67f7-4a74-a6b1-bea06b210c4b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"print('\\n[3] Splitting 70/30 by time')\ndf=df.sort_values(by=['contractID', 'date']).reset_index(drop=True)","metadata":{"_uuid":"ec96e39c-7722-4107-9366-e2c11ce7784c","_cell_guid":"7a1615d0-082b-46f8-b42d-edfe263c5a35","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:28.722734Z","iopub.execute_input":"2026-07-12T06:46:28.723016Z","iopub.status.idle":"2026-07-12T06:46:29.449372Z","shell.execute_reply.started":"2026-07-12T06:46:28.722997Z","shell.execute_reply":"2026-07-12T06:46:29.448667Z"}},"outputs":[{"name":"stdout","text":"\n[3] Splitting 70/30 by time\n","output_type":"stream"}],"execution_count":21},{"cell_type":"code","source":"df.head()","metadata":{"_uuid":"9324d4e4-63a6-4efc-95d6-8c8de810a6a2","_cell_guid":"ac3624f9-98e9-432a-8343-46d89ff7e05b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:29.453372Z","iopub.execute_input":"2026-07-12T06:46:29.453700Z","iopub.status.idle":"2026-07-12T06:46:29.468869Z","shell.execute_reply.started":"2026-07-12T06:46:29.453678Z","shell.execute_reply":"2026-07-12T06:46:29.468110Z"}},"outputs":[{"execution_count":22,"output_type":"execute_result","data":{"text/plain":"   implied_volatility    delta    gamma    theta     vega      rho  tte_days  \\\n0             1.24983  0.99396  0.00055 -0.32269  0.00423  0.01093         0   \n1             1.25456  0.99311  0.00062 -0.35671  0.00476  0.01095         0   \n2             1.21492  0.99383  0.00058 -0.32065  0.00431  0.01099         0   \n3             1.14861  1.00000  0.00000 -0.05913  0.00000  0.01109         0   \n4             1.13163  1.00000  0.00000 -0.05928  0.00000  0.01112         0   \n\n   log_moneyness  strike  type_enc  volume  open_interest  regime   mark  \\\n0      -0.218156   402.0         1     130              1       0  70.77   \n1      -0.215672   403.0         1       9              0       0  69.78   \n2      -0.213193   404.0         1       1              1       0  68.77   \n3      -0.210721   405.0         1       0              4       0  67.52   \n4      -0.208255   406.0         1       0              3       0  66.52   \n\n        date          contractID  \n0 2024-01-02  SPY240102C00402000  \n1 2024-01-02  SPY240102C00403000  \n2 2024-01-02  SPY240102C00404000  \n3 2024-01-02  SPY240102C00405000  \n4 2024-01-02  SPY240102C00406000  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>tte_days</th>\n      <th>log_moneyness</th>\n      <th>strike</th>\n      <th>type_enc</th>\n      <th>volume</th>\n      <th>open_interest</th>\n      <th>regime</th>\n      <th>mark</th>\n      <th>date</th>\n      <th>contractID</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.24983</td>\n      <td>0.99396</td>\n      <td>0.00055</td>\n      <td>-0.32269</td>\n      <td>0.00423</td>\n      <td>0.01093</td>\n      <td>0</td>\n      <td>-0.218156</td>\n      <td>402.0</td>\n      <td>1</td>\n      <td>130</td>\n      <td>1</td>\n      <td>0</td>\n      <td>70.77</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00402000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.25456</td>\n      <td>0.99311</td>\n      <td>0.00062</td>\n      <td>-0.35671</td>\n      <td>0.00476</td>\n      <td>0.01095</td>\n      <td>0</td>\n      <td>-0.215672</td>\n      <td>403.0</td>\n      <td>1</td>\n      <td>9</td>\n      <td>0</td>\n      <td>0</td>\n      <td>69.78</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00403000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1.21492</td>\n      <td>0.99383</td>\n      <td>0.00058</td>\n      <td>-0.32065</td>\n      <td>0.00431</td>\n      <td>0.01099</td>\n      <td>0</td>\n      <td>-0.213193</td>\n      <td>404.0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>68.77</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00404000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.14861</td>\n      <td>1.00000</td>\n      <td>0.00000</td>\n      <td>-0.05913</td>\n      <td>0.00000</td>\n      <td>0.01109</td>\n      <td>0</td>\n      <td>-0.210721</td>\n      <td>405.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>4</td>\n      <td>0</td>\n      <td>67.52</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00405000</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1.13163</td>\n      <td>1.00000</td>\n      <td>0.00000</td>\n      <td>-0.05928</td>\n      <td>0.00000</td>\n      <td>0.01112</td>\n      <td>0</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>3</td>\n      <td>0</td>\n      <td>66.52</td>\n      <td>2024-01-02</td>\n      <td>SPY240102C00406000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":22},{"cell_type":"code","source":"df.shape","metadata":{"_uuid":"1bf9e1c1-4d03-4fa6-b73a-ef0c6d29eaae","_cell_guid":"671f8184-1d1f-4292-ba55-1e83bcbd777a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:29.469857Z","iopub.execute_input":"2026-07-12T06:46:29.470146Z","iopub.status.idle":"2026-07-12T06:46:29.482733Z","shell.execute_reply.started":"2026-07-12T06:46:29.470111Z","shell.execute_reply":"2026-07-12T06:46:29.481881Z"}},"outputs":[{"execution_count":23,"output_type":"execute_result","data":{"text/plain":"(2292798, 16)"},"metadata":{}}],"execution_count":23},{"cell_type":"code","source":"len(df['contractID'].value_counts())","metadata":{"_uuid":"7bb8bde6-4b90-47f2-809c-a81c17dfbc07","_cell_guid":"ceb78c15-78d5-42e2-81f9-8bc98d301d77","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:29.483767Z","iopub.execute_input":"2026-07-12T06:46:29.484146Z","iopub.status.idle":"2026-07-12T06:46:29.751783Z","shell.execute_reply.started":"2026-07-12T06:46:29.484111Z","shell.execute_reply":"2026-07-12T06:46:29.750830Z"}},"outputs":[{"execution_count":24,"output_type":"execute_result","data":{"text/plain":"76278"},"metadata":{}}],"execution_count":24},{"cell_type":"code","source":"cnt=1\nfor i in (df['contractID'].value_counts()):\n    if i>=11:\n        cnt+=i\ncnt","metadata":{"_uuid":"b6fb530c-66f7-4b02-9272-f93ed57c6c4c","_cell_guid":"40f6aefa-3a18-416c-aa66-8e110e2cceab","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:29.753004Z","iopub.execute_input":"2026-07-12T06:46:29.753789Z","iopub.status.idle":"2026-07-12T06:46:29.998921Z","shell.execute_reply.started":"2026-07-12T06:46:29.753762Z","shell.execute_reply":"2026-07-12T06:46:29.998317Z"}},"outputs":[{"execution_count":25,"output_type":"execute_result","data":{"text/plain":"1952207"},"metadata":{}}],"execution_count":25},{"cell_type":"code","source":"obs_per_contract=df['contractID'].value_counts()","metadata":{"_uuid":"550f27cf-50e6-4795-bb60-b6160bd9432f","_cell_guid":"af25b5ae-4b94-4372-80ac-6fc3499aed7d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:29.999893Z","iopub.execute_input":"2026-07-12T06:46:30.000217Z","iopub.status.idle":"2026-07-12T06:46:30.229957Z","shell.execute_reply.started":"2026-07-12T06:46:30.000193Z","shell.execute_reply":"2026-07-12T06:46:30.229129Z"}},"outputs":[],"execution_count":26},{"cell_type":"code","source":"valid_contracts=obs_per_contract[obs_per_contract>=8].index","metadata":{"_uuid":"00ba8890-16c0-4940-9859-bbb16a76df88","_cell_guid":"56fcaa1c-3f45-4af2-9a39-864d1a500fb6","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:30.231089Z","iopub.execute_input":"2026-07-12T06:46:30.232205Z","iopub.status.idle":"2026-07-12T06:46:30.245519Z","shell.execute_reply.started":"2026-07-12T06:46:30.232177Z","shell.execute_reply":"2026-07-12T06:46:30.244705Z"}},"outputs":[],"execution_count":27},{"cell_type":"code","source":"valid_contracts","metadata":{"_uuid":"ca67ba3f-94d7-490d-b570-23f92b078593","_cell_guid":"f84c0663-6cc8-4655-ae64-faab37d39fb4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:30.246571Z","iopub.execute_input":"2026-07-12T06:46:30.247338Z","iopub.status.idle":"2026-07-12T06:46:30.258631Z","shell.execute_reply.started":"2026-07-12T06:46:30.247310Z","shell.execute_reply":"2026-07-12T06:46:30.257954Z"}},"outputs":[{"execution_count":28,"output_type":"execute_result","data":{"text/plain":"Index(['SPY241231C00457000', 'SPY241231C00490000', 'SPY250117P00475000',\n       'SPY250117P00480000', 'SPY250117P00405000', 'SPY250117P00410000',\n       'SPY250117P00415000', 'SPY250117P00180000', 'SPY250117P00425000',\n       'SPY250117P00430000',\n       ...\n       'SPY241230P00626000', 'SPY241230P00627000', 'SPY241227P00627000',\n       'SPY241227P00628000', 'SPY241227P00629000', 'SPY241227P00626000',\n       'SPY241230P00616000', 'SPY241230P00617000', 'SPY241230P00618000',\n       'SPY241230P00619000'],\n      dtype='object', name='contractID', length=58004)"},"metadata":{}}],"execution_count":28},{"cell_type":"code","source":"before=2292798\ndf=df[df['contractID'].isin(valid_contracts)].copy()\nafter=len(df)","metadata":{"_uuid":"9a994601-aeda-4d3d-91a7-7f6fd128376f","_cell_guid":"e6371c7b-c9ad-4b70-a7ae-9aa0e3a4abcd","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:30.259558Z","iopub.execute_input":"2026-07-12T06:46:30.259829Z","iopub.status.idle":"2026-07-12T06:46:30.645951Z","shell.execute_reply.started":"2026-07-12T06:46:30.259797Z","shell.execute_reply":"2026-07-12T06:46:30.645316Z"}},"outputs":[],"execution_count":29},{"cell_type":"code","source":"print(f'rows kept: {after}')\nprint(f'Original number of rows: {before}')\nprint(f'Rows dropped: {before - after}')","metadata":{"_uuid":"be580222-b09c-4a9e-abab-8ce1abb81bc8","_cell_guid":"c927e2f7-13b2-4f0f-a7da-19d28d1a7352","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:30.646855Z","iopub.execute_input":"2026-07-12T06:46:30.647214Z","iopub.status.idle":"2026-07-12T06:46:30.652021Z","shell.execute_reply.started":"2026-07-12T06:46:30.647191Z","shell.execute_reply":"2026-07-12T06:46:30.651169Z"}},"outputs":[{"name":"stdout","text":"rows kept: 2204060\nOriginal number of rows: 2292798\nRows dropped: 88738\n","output_type":"stream"}],"execution_count":30},{"cell_type":"code","source":"df.head()","metadata":{"_uuid":"9880376a-5c87-46b8-a7ac-ea03270ad560","_cell_guid":"3b01d184-b4b1-49a7-a32b-f36c754a6603","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:30.653018Z","iopub.execute_input":"2026-07-12T06:46:30.653407Z","iopub.status.idle":"2026-07-12T06:46:30.676566Z","shell.execute_reply.started":"2026-07-12T06:46:30.653378Z","shell.execute_reply":"2026-07-12T06:46:30.675802Z"}},"outputs":[{"execution_count":31,"output_type":"execute_result","data":{"text/plain":"      implied_volatility    delta    gamma    theta     vega      rho  \\\n6692             0.51083  0.97450  0.00157 -0.18256  0.04411  0.09691   \n6693             0.46327  0.98414  0.00124 -0.13792  0.02757  0.08721   \n6694             0.54482  0.97222  0.00181 -0.21802  0.04131  0.07522   \n6695             0.59497  0.97196  0.00180 -0.24861  0.03859  0.06446   \n6696             0.65137  0.99633  0.00039 -0.11028  0.00472  0.03321   \n\n      tte_days  log_moneyness  strike  type_enc  volume  open_interest  \\\n6692         9      -0.208255   406.0         1       0              0   \n6693         8      -0.208255   406.0         1       2              0   \n6694         7      -0.208255   406.0         1       0              1   \n6695         6      -0.208255   406.0         1       0              1   \n6696         3      -0.208255   406.0         1       0              1   \n\n      regime   mark       date          contractID  \n6692       0  67.56 2024-01-02  SPY240111C00406000  \n6693       0  63.45 2024-01-03  SPY240111C00406000  \n6694       0  62.08 2024-01-04  SPY240111C00406000  \n6695       0  62.67 2024-01-05  SPY240111C00406000  \n6696       0  68.81 2024-01-08  SPY240111C00406000  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>tte_days</th>\n      <th>log_moneyness</th>\n      <th>strike</th>\n      <th>type_enc</th>\n      <th>volume</th>\n      <th>open_interest</th>\n      <th>regime</th>\n      <th>mark</th>\n      <th>date</th>\n      <th>contractID</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>6692</th>\n      <td>0.51083</td>\n      <td>0.97450</td>\n      <td>0.00157</td>\n      <td>-0.18256</td>\n      <td>0.04411</td>\n      <td>0.09691</td>\n      <td>9</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>67.56</td>\n      <td>2024-01-02</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6693</th>\n      <td>0.46327</td>\n      <td>0.98414</td>\n      <td>0.00124</td>\n      <td>-0.13792</td>\n      <td>0.02757</td>\n      <td>0.08721</td>\n      <td>8</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>63.45</td>\n      <td>2024-01-03</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6694</th>\n      <td>0.54482</td>\n      <td>0.97222</td>\n      <td>0.00181</td>\n      <td>-0.21802</td>\n      <td>0.04131</td>\n      <td>0.07522</td>\n      <td>7</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>62.08</td>\n      <td>2024-01-04</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6695</th>\n      <td>0.59497</td>\n      <td>0.97196</td>\n      <td>0.00180</td>\n      <td>-0.24861</td>\n      <td>0.03859</td>\n      <td>0.06446</td>\n      <td>6</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>62.67</td>\n      <td>2024-01-05</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6696</th>\n      <td>0.65137</td>\n      <td>0.99633</td>\n      <td>0.00039</td>\n      <td>-0.11028</td>\n      <td>0.00472</td>\n      <td>0.03321</td>\n      <td>3</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>68.81</td>\n      <td>2024-01-08</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":31},{"cell_type":"code","source":"# now we sort the values\nunique_dates=sorted(df['date'].unique())\nsplit_date=unique_dates[int(len(unique_dates)*0.80)]\nsplit_date","metadata":{"_uuid":"b309516a-1e86-4a84-aeec-f80f00835065","_cell_guid":"371155f7-9387-4cc0-a2e9-fcaa53df6860","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:30.677410Z","iopub.execute_input":"2026-07-12T06:46:30.678369Z","iopub.status.idle":"2026-07-12T06:46:30.706469Z","shell.execute_reply.started":"2026-07-12T06:46:30.678337Z","shell.execute_reply":"2026-07-12T06:46:30.705852Z"}},"outputs":[{"execution_count":32,"output_type":"execute_result","data":{"text/plain":"Timestamp('2024-10-18 00:00:00')"},"metadata":{}}],"execution_count":32},{"cell_type":"code","source":"df.columns","metadata":{"_uuid":"6c20c9f7-aad3-46b6-999d-b657f708cab1","_cell_guid":"5cae3083-1997-4bb1-8ef5-13a3e2988d14","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:30.707351Z","iopub.execute_input":"2026-07-12T06:46:30.707615Z","iopub.status.idle":"2026-07-12T06:46:30.712770Z","shell.execute_reply.started":"2026-07-12T06:46:30.707571Z","shell.execute_reply":"2026-07-12T06:46:30.711950Z"}},"outputs":[{"execution_count":33,"output_type":"execute_result","data":{"text/plain":"Index(['implied_volatility', 'delta', 'gamma', 'theta', 'vega', 'rho',\n       'tte_days', 'log_moneyness', 'strike', 'type_enc', 'volume',\n       'open_interest', 'regime', 'mark', 'date', 'contractID'],\n      dtype='object')"},"metadata":{}}],"execution_count":33},{"cell_type":"code","source":"train_df = df[df[\"date\"] < split_date].copy()\ntest_df = df[df[\"date\"] >= split_date].copy()","metadata":{"_uuid":"90127e5a-669a-4421-ad05-927d96946bff","_cell_guid":"a81b769f-db8e-49ae-8b02-60fd77cbda22","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:30.713741Z","iopub.execute_input":"2026-07-12T06:46:30.714098Z","iopub.status.idle":"2026-07-12T06:46:30.974692Z","shell.execute_reply.started":"2026-07-12T06:46:30.714066Z","shell.execute_reply":"2026-07-12T06:46:30.974061Z"}},"outputs":[],"execution_count":34},{"cell_type":"code","source":"train_df['regime'].value_counts()","metadata":{"_uuid":"9f7d8350-5d48-494e-8de5-881912db9761","_cell_guid":"cc545445-e9c6-42d9-bf11-257f59a88414","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T06:46:30.975560Z","iopub.execute_input":"2026-07-12T06:46:30.975838Z","iopub.status.idle":"2026-07-12T06:46:30.991925Z","shell.execute_reply.started":"2026-07-12T06:46:30.975807Z","shell.execute_reply":"2026-07-12T06:46:30.991191Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":35,"output_type":"execute_result","data":{"text/plain":"regime\n0    635852\n2    580626\n1    520064\nName: count, dtype: int64"},"metadata":{}}],"execution_count":35},{"cell_type":"code","source":"print(f\"    Split date : {pd.Timestamp(split_date).date()}\")\nprint(f\"    Train : {len(train_df):,} rows  \"\n      f\"({train_df['date'].min().date()} → {train_df['date'].max().date()})\")\nprint(f\"    Test  : {len(test_df):,} rows  \"\n      f\"({test_df['date'].min().date()} → {test_df['date'].max().date()})\")","metadata":{"_uuid":"aa91b30b-aa3a-42c4-8ffa-84ddf4307cb7","_cell_guid":"b00dfbf3-bec2-4ed9-9e05-5f84a3bb1eb5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:30.993496Z","iopub.execute_input":"2026-07-12T06:46:30.993697Z","iopub.status.idle":"2026-07-12T06:46:31.022662Z","shell.execute_reply.started":"2026-07-12T06:46:30.993679Z","shell.execute_reply":"2026-07-12T06:46:31.022017Z"}},"outputs":[{"name":"stdout","text":"    Split date : 2024-10-18\n    Train : 1,736,542 rows  (2024-01-02 → 2024-10-17)\n    Test  : 467,518 rows  (2024-10-18 → 2024-12-31)\n","output_type":"stream"}],"execution_count":36},{"cell_type":"code","source":"df.head()","metadata":{"_uuid":"c861a0fa-53b3-41c3-b6ff-9cf5434794b7","_cell_guid":"92115b26-8d31-470b-ac72-189f0cffc90a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:31.023543Z","iopub.execute_input":"2026-07-12T06:46:31.023831Z","iopub.status.idle":"2026-07-12T06:46:31.038211Z","shell.execute_reply.started":"2026-07-12T06:46:31.023808Z","shell.execute_reply":"2026-07-12T06:46:31.037235Z"}},"outputs":[{"execution_count":37,"output_type":"execute_result","data":{"text/plain":"      implied_volatility    delta    gamma    theta     vega      rho  \\\n6692             0.51083  0.97450  0.00157 -0.18256  0.04411  0.09691   \n6693             0.46327  0.98414  0.00124 -0.13792  0.02757  0.08721   \n6694             0.54482  0.97222  0.00181 -0.21802  0.04131  0.07522   \n6695             0.59497  0.97196  0.00180 -0.24861  0.03859  0.06446   \n6696             0.65137  0.99633  0.00039 -0.11028  0.00472  0.03321   \n\n      tte_days  log_moneyness  strike  type_enc  volume  open_interest  \\\n6692         9      -0.208255   406.0         1       0              0   \n6693         8      -0.208255   406.0         1       2              0   \n6694         7      -0.208255   406.0         1       0              1   \n6695         6      -0.208255   406.0         1       0              1   \n6696         3      -0.208255   406.0         1       0              1   \n\n      regime   mark       date          contractID  \n6692       0  67.56 2024-01-02  SPY240111C00406000  \n6693       0  63.45 2024-01-03  SPY240111C00406000  \n6694       0  62.08 2024-01-04  SPY240111C00406000  \n6695       0  62.67 2024-01-05  SPY240111C00406000  \n6696       0  68.81 2024-01-08  SPY240111C00406000  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>tte_days</th>\n      <th>log_moneyness</th>\n      <th>strike</th>\n      <th>type_enc</th>\n      <th>volume</th>\n      <th>open_interest</th>\n      <th>regime</th>\n      <th>mark</th>\n      <th>date</th>\n      <th>contractID</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>6692</th>\n      <td>0.51083</td>\n      <td>0.97450</td>\n      <td>0.00157</td>\n      <td>-0.18256</td>\n      <td>0.04411</td>\n      <td>0.09691</td>\n      <td>9</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>67.56</td>\n      <td>2024-01-02</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6693</th>\n      <td>0.46327</td>\n      <td>0.98414</td>\n      <td>0.00124</td>\n      <td>-0.13792</td>\n      <td>0.02757</td>\n      <td>0.08721</td>\n      <td>8</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>63.45</td>\n      <td>2024-01-03</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6694</th>\n      <td>0.54482</td>\n      <td>0.97222</td>\n      <td>0.00181</td>\n      <td>-0.21802</td>\n      <td>0.04131</td>\n      <td>0.07522</td>\n      <td>7</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>62.08</td>\n      <td>2024-01-04</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6695</th>\n      <td>0.59497</td>\n      <td>0.97196</td>\n      <td>0.00180</td>\n      <td>-0.24861</td>\n      <td>0.03859</td>\n      <td>0.06446</td>\n      <td>6</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>62.67</td>\n      <td>2024-01-05</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6696</th>\n      <td>0.65137</td>\n      <td>0.99633</td>\n      <td>0.00039</td>\n      <td>-0.11028</td>\n      <td>0.00472</td>\n      <td>0.03321</td>\n      <td>3</td>\n      <td>-0.208255</td>\n      <td>406.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>68.81</td>\n      <td>2024-01-08</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":37},{"cell_type":"markdown","source":"# Scaling","metadata":{"_uuid":"2598474b-9012-4bfb-be23-8517159d4278","_cell_guid":"a0a0e620-73b3-4ee6-8236-b282933f8031","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"feat_scaler=StandardScaler()\ntarget_scaler=StandardScaler()","metadata":{"_uuid":"10553f53-f66a-48cb-8c7c-5e35149d0c86","_cell_guid":"8a0829fe-7c64-46e1-b824-68fb8397c3ee","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:31.039295Z","iopub.execute_input":"2026-07-12T06:46:31.039574Z","iopub.status.idle":"2026-07-12T06:46:31.049045Z","shell.execute_reply.started":"2026-07-12T06:46:31.039552Z","shell.execute_reply":"2026-07-12T06:46:31.048435Z"}},"outputs":[],"execution_count":38},{"cell_type":"code","source":"train_df[FEATURES]=feat_scaler.fit_transform(train_df[FEATURES].values)\ntest_df[FEATURES] = feat_scaler.transform(test_df[FEATURES].values)","metadata":{"_uuid":"af7b5ddf-b32c-4d3f-a466-6bc0b5a26f28","_cell_guid":"de2021ad-8b85-4f51-8876-b6fb74101828","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T06:46:31.050217Z","iopub.execute_input":"2026-07-12T06:46:31.050541Z","iopub.status.idle":"2026-07-12T06:46:31.585562Z","shell.execute_reply.started":"2026-07-12T06:46:31.050518Z","shell.execute_reply":"2026-07-12T06:46:31.584833Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":39},{"cell_type":"code","source":"print(type(train_df))\nprint(type(test_df))\nprint(TARGET)","metadata":{"_uuid":"16025031-b31f-4e14-a6cf-5d01d90f3b8f","_cell_guid":"a9e12df0-739b-4a97-85d4-b30e1692bf5f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:31.586508Z","iopub.execute_input":"2026-07-12T06:46:31.586726Z","iopub.status.idle":"2026-07-12T06:46:31.591850Z","shell.execute_reply.started":"2026-07-12T06:46:31.586706Z","shell.execute_reply":"2026-07-12T06:46:31.590886Z"}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\n<class 'pandas.core.frame.DataFrame'>\nmark\n","output_type":"stream"}],"execution_count":40},{"cell_type":"code","source":"train_df.head()","metadata":{"_uuid":"40457671-2d4c-4776-b147-208a55236a13","_cell_guid":"1417f22b-45de-4610-8d8f-5c59029576a8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:31.592913Z","iopub.execute_input":"2026-07-12T06:46:31.593232Z","iopub.status.idle":"2026-07-12T06:46:31.614894Z","shell.execute_reply.started":"2026-07-12T06:46:31.593202Z","shell.execute_reply":"2026-07-12T06:46:31.614063Z"}},"outputs":[{"execution_count":41,"output_type":"execute_result","data":{"text/plain":"      implied_volatility     delta     gamma     theta      vega       rho  \\\n6692            0.889534  1.294105 -0.256925 -0.837888 -0.731375 -0.013656   \n6693            0.723119  1.310376 -0.279555 -0.493957 -0.754353 -0.018176   \n6694            1.008467  1.290257 -0.240467 -1.111091 -0.735265 -0.023763   \n6695            1.183945  1.289818 -0.241153 -1.346773 -0.739043 -0.028777   \n6696            1.381292  1.330952 -0.337844 -0.281003 -0.786097 -0.043338   \n\n      tte_days  log_moneyness    strike  type_enc    volume  open_interest  \\\n6692 -0.769782      -0.541181 -0.687226       1.0 -0.075309      -0.236488   \n6693 -0.774344      -0.541181 -0.687226       1.0 -0.075122      -0.236488   \n6694 -0.778907      -0.541181 -0.687226       1.0 -0.075309      -0.236384   \n6695 -0.783470      -0.541181 -0.687226       1.0 -0.075309      -0.236384   \n6696 -0.797158      -0.541181 -0.687226       1.0 -0.075309      -0.236384   \n\n        regime   mark       date          contractID  \n6692 -1.157626  67.56 2024-01-02  SPY240111C00406000  \n6693 -1.157626  63.45 2024-01-03  SPY240111C00406000  \n6694 -1.157626  62.08 2024-01-04  SPY240111C00406000  \n6695 -1.157626  62.67 2024-01-05  SPY240111C00406000  \n6696 -1.157626  68.81 2024-01-08  SPY240111C00406000  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>implied_volatility</th>\n      <th>delta</th>\n      <th>gamma</th>\n      <th>theta</th>\n      <th>vega</th>\n      <th>rho</th>\n      <th>tte_days</th>\n      <th>log_moneyness</th>\n      <th>strike</th>\n      <th>type_enc</th>\n      <th>volume</th>\n      <th>open_interest</th>\n      <th>regime</th>\n      <th>mark</th>\n      <th>date</th>\n      <th>contractID</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>6692</th>\n      <td>0.889534</td>\n      <td>1.294105</td>\n      <td>-0.256925</td>\n      <td>-0.837888</td>\n      <td>-0.731375</td>\n      <td>-0.013656</td>\n      <td>-0.769782</td>\n      <td>-0.541181</td>\n      <td>-0.687226</td>\n      <td>1.0</td>\n      <td>-0.075309</td>\n      <td>-0.236488</td>\n      <td>-1.157626</td>\n      <td>67.56</td>\n      <td>2024-01-02</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6693</th>\n      <td>0.723119</td>\n      <td>1.310376</td>\n      <td>-0.279555</td>\n      <td>-0.493957</td>\n      <td>-0.754353</td>\n      <td>-0.018176</td>\n      <td>-0.774344</td>\n      <td>-0.541181</td>\n      <td>-0.687226</td>\n      <td>1.0</td>\n      <td>-0.075122</td>\n      <td>-0.236488</td>\n      <td>-1.157626</td>\n      <td>63.45</td>\n      <td>2024-01-03</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6694</th>\n      <td>1.008467</td>\n      <td>1.290257</td>\n      <td>-0.240467</td>\n      <td>-1.111091</td>\n      <td>-0.735265</td>\n      <td>-0.023763</td>\n      <td>-0.778907</td>\n      <td>-0.541181</td>\n      <td>-0.687226</td>\n      <td>1.0</td>\n      <td>-0.075309</td>\n      <td>-0.236384</td>\n      <td>-1.157626</td>\n      <td>62.08</td>\n      <td>2024-01-04</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6695</th>\n      <td>1.183945</td>\n      <td>1.289818</td>\n      <td>-0.241153</td>\n      <td>-1.346773</td>\n      <td>-0.739043</td>\n      <td>-0.028777</td>\n      <td>-0.783470</td>\n      <td>-0.541181</td>\n      <td>-0.687226</td>\n      <td>1.0</td>\n      <td>-0.075309</td>\n      <td>-0.236384</td>\n      <td>-1.157626</td>\n      <td>62.67</td>\n      <td>2024-01-05</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n    <tr>\n      <th>6696</th>\n      <td>1.381292</td>\n      <td>1.330952</td>\n      <td>-0.337844</td>\n      <td>-0.281003</td>\n      <td>-0.786097</td>\n      <td>-0.043338</td>\n      <td>-0.797158</td>\n      <td>-0.541181</td>\n      <td>-0.687226</td>\n      <td>1.0</td>\n      <td>-0.075309</td>\n      <td>-0.236384</td>\n      <td>-1.157626</td>\n      <td>68.81</td>\n      <td>2024-01-08</td>\n      <td>SPY240111C00406000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":41},{"cell_type":"code","source":"train_df[TARGET]=target_scaler.fit_transform(train_df[[TARGET]].values)\ntest_df[TARGET]=target_scaler.transform(test_df[[TARGET]].values)","metadata":{"_uuid":"d57fb12f-223b-4982-8a23-81f4de708933","_cell_guid":"8384228a-2ac6-4baa-bdf5-7ae2b632b9d9","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:31.615892Z","iopub.execute_input":"2026-07-12T06:46:31.616702Z","iopub.status.idle":"2026-07-12T06:46:31.658751Z","shell.execute_reply.started":"2026-07-12T06:46:31.616678Z","shell.execute_reply":"2026-07-12T06:46:31.657856Z"}},"outputs":[],"execution_count":42},{"cell_type":"code","source":"SEQ_LEN=6\nBATCH_SIZE=1024","metadata":{"_uuid":"8ca97a81-f382-4232-a460-ecd7d27f8cae","_cell_guid":"095840af-25dc-421d-a63a-9dc99f50f85f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:31.659717Z","iopub.execute_input":"2026-07-12T06:46:31.660024Z","iopub.status.idle":"2026-07-12T06:46:31.663874Z","shell.execute_reply.started":"2026-07-12T06:46:31.660002Z","shell.execute_reply":"2026-07-12T06:46:31.663088Z"}},"outputs":[],"execution_count":43},{"cell_type":"code","source":"class LazyContractDataset(Dataset):\n    def __init__(self,df,features,target,seq_len):\n        self.seq_len=seq_len\n        self.features=features\n        self.target=target\n\n        self.index=[]\n        self.contract_data=[]\n        skipped=0\n        n_contracts=0\n        for contract, group in df.groupby(by='contractID',sort=False):\n            group=group.sort_values(by='date')\n            X = group[features].values.astype(np.float32)\n            y = group[target].values.astype(np.float32)\n\n            if len(X)<seq_len+1:\n                skipped+=1\n                continue\n\n            n_contracts+=1\n\n            arr_idx=len(self.contract_data)\n            self.contract_data.append((X,y))\n\n            for i in range(len(X)-seq_len):\n                self.index.append((arr_idx,i))\n\n        print(f\"    Contracts used    : {n_contracts:,}\")\n        print(f\"    Contracts skipped : {skipped:,}  \"\n              f\"(< {seq_len + 1} observations)\")\n        print(f\"    Total sequences   : {len(self.index):,}\")\n\n    def __len__(self):\n        return len(self.index)\n\n    def __getitem__(self,idx):\n        arr_idx, start=self.index[idx]\n        X,y=self.contract_data[arr_idx]\n        x_seq=torch.tensor(X[start:start+self.seq_len],dtype=torch.float32)\n        label=torch.tensor(y[start+self.seq_len],dtype=torch.float32)\n        return x_seq,label","metadata":{"_uuid":"d5749080-5db4-4b37-8b67-801f562bce3a","_cell_guid":"5e2ac04e-918f-4975-9c0b-3aa2a2b4327a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:31.664764Z","iopub.execute_input":"2026-07-12T06:46:31.665061Z","iopub.status.idle":"2026-07-12T06:46:31.677925Z","shell.execute_reply.started":"2026-07-12T06:46:31.665032Z","shell.execute_reply":"2026-07-12T06:46:31.677193Z"}},"outputs":[],"execution_count":44},{"cell_type":"markdown","source":"# Now bulding dataset","metadata":{"_uuid":"cc9eeec8-3fe8-4c6f-8247-717d0cebe64c","_cell_guid":"aedbc669-f406-47ad-b4b4-044353d3c8d6","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"train_dataset=LazyContractDataset(train_df,features=FEATURES, target=TARGET, seq_len=SEQ_LEN)\ntest_dataset=LazyContractDataset(test_df,features=FEATURES, target=TARGET, seq_len=SEQ_LEN)","metadata":{"_uuid":"7085ee9c-afec-43b9-9d3e-8b245e6fdcb5","_cell_guid":"f0759ee4-cd91-4b7c-a1c5-5e4ee7e0171b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:46:31.679567Z","iopub.execute_input":"2026-07-12T06:46:31.679779Z","iopub.status.idle":"2026-07-12T06:47:28.874453Z","shell.execute_reply.started":"2026-07-12T06:46:31.679759Z","shell.execute_reply":"2026-07-12T06:47:28.873559Z"}},"outputs":[{"name":"stdout","text":"    Contracts used    : 47,008\n    Contracts skipped : 1,080  (< 7 observations)\n    Total sequences   : 1,450,982\n    Contracts used    : 17,388\n    Contracts skipped : 1,738  (< 7 observations)\n    Total sequences   : 357,994\n","output_type":"stream"}],"execution_count":45},{"cell_type":"code","source":"BATCH_SIZE","metadata":{"_uuid":"64d08c69-053f-4ecc-8849-29f5ac342ddf","_cell_guid":"f5243c01-2aa6-4f68-9c65-588c0985e46c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:47:28.875568Z","iopub.execute_input":"2026-07-12T06:47:28.875887Z","iopub.status.idle":"2026-07-12T06:47:28.881504Z","shell.execute_reply.started":"2026-07-12T06:47:28.875856Z","shell.execute_reply":"2026-07-12T06:47:28.880727Z"}},"outputs":[{"execution_count":46,"output_type":"execute_result","data":{"text/plain":"1024"},"metadata":{}}],"execution_count":46},{"cell_type":"code","source":"train_loader = DataLoader(\n    train_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=True,\n    drop_last=True,\n    num_workers=16, #changed from 4\n    pin_memory=True,\n    persistent_workers=True,\n    prefetch_factor=2,\n)\n\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    drop_last=False,\n    num_workers=16, #changed from 4\n    pin_memory=True,\n    persistent_workers=True,\n    prefetch_factor=2,\n)","metadata":{"_uuid":"8d2d875e-99cc-4955-94fd-42f52c5c9b2d","_cell_guid":"7122f6e5-5441-4be5-aa7b-69a3f8498c19","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:47:28.882734Z","iopub.execute_input":"2026-07-12T06:47:28.883250Z","iopub.status.idle":"2026-07-12T06:47:28.897038Z","shell.execute_reply.started":"2026-07-12T06:47:28.883226Z","shell.execute_reply":"2026-07-12T06:47:28.896169Z"}},"outputs":[],"execution_count":47},{"cell_type":"markdown","source":"# Buidling the model","metadata":{"_uuid":"c3724558-5241-41fb-baf0-635186dd7655","_cell_guid":"f435cbcc-f603-4ced-b22a-f7b0165ba520","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"class OptionPricingLSTM(nn.Module):\n    def __init__(self,input_size,hidden_size=128, num_layers=2,dropout=0.3):\n        super().__init__()\n        self.lstm=nn.LSTM(input_size=input_size,\n                         hidden_size=hidden_size,\n                         num_layers=num_layers,\n                         batch_first=True,\n                         dropout=dropout if num_layers>1 else 0.0)\n\n        self.head=nn.Sequential(nn.Dropout(dropout),\n                               nn.Linear(hidden_size,64),\n                               nn.ReLU(),\n                               nn.Dropout(0.2),\n                               nn.Linear(64,1))\n\n    def forward(self,x):\n        lstm_out,_=self.lstm(x)\n        last_step = lstm_out[:, -1, :]\n        return self.head(last_step).squeeze(-1)","metadata":{"_uuid":"9634770d-2096-4d73-93f6-8ce29f097a99","_cell_guid":"44d4c481-1e8f-40b1-96e7-a8db55138a9b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:47:28.897903Z","iopub.execute_input":"2026-07-12T06:47:28.898317Z","iopub.status.idle":"2026-07-12T06:47:28.911142Z","shell.execute_reply.started":"2026-07-12T06:47:28.898245Z","shell.execute_reply":"2026-07-12T06:47:28.910131Z"}},"outputs":[],"execution_count":48},{"cell_type":"code","source":"model=OptionPricingLSTM(input_size=len(FEATURES))","metadata":{"_uuid":"7bbc9023-d4ab-48c5-8b74-19ecd37c34fa","_cell_guid":"55e0e427-ec04-4377-a9b9-5d777730be05","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:47:28.912308Z","iopub.execute_input":"2026-07-12T06:47:28.912871Z","iopub.status.idle":"2026-07-12T06:47:28.953829Z","shell.execute_reply.started":"2026-07-12T06:47:28.912845Z","shell.execute_reply":"2026-07-12T06:47:28.953280Z"}},"outputs":[],"execution_count":49},{"cell_type":"code","source":"if torch.cuda.device_count()>1:\n    print(torch.cuda.device_count())\n    model=nn.DataParallel(model)\nelse:\n    pass","metadata":{"_uuid":"fdd2425c-714d-4f39-ab2a-e80140c74333","_cell_guid":"bd39b797-a354-44c5-b10b-c35f65bf8644","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:47:28.954714Z","iopub.execute_input":"2026-07-12T06:47:28.955032Z","iopub.status.idle":"2026-07-12T06:47:28.977613Z","shell.execute_reply.started":"2026-07-12T06:47:28.955010Z","shell.execute_reply":"2026-07-12T06:47:28.976848Z"}},"outputs":[{"name":"stdout","text":"2\n","output_type":"stream"}],"execution_count":50},{"cell_type":"code","source":"model=model.to(device)","metadata":{"_uuid":"1d55f81b-9428-4f0f-84ac-04d60c7eff30","_cell_guid":"a54bc8f7-c19a-4d1a-8da8-32ee10922d92","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:47:28.978467Z","iopub.execute_input":"2026-07-12T06:47:28.979240Z","iopub.status.idle":"2026-07-12T06:47:29.377848Z","shell.execute_reply.started":"2026-07-12T06:47:28.979202Z","shell.execute_reply":"2026-07-12T06:47:29.377219Z"}},"outputs":[],"execution_count":51},{"cell_type":"code","source":"EPOCHS=30\nLR=1e-3","metadata":{"_uuid":"ad59d77c-65dc-49a5-9dd0-4cd7e07f394e","_cell_guid":"f4497f42-70d4-4787-a02e-ee2326903d01","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:47:29.378899Z","iopub.execute_input":"2026-07-12T06:47:29.379181Z","iopub.status.idle":"2026-07-12T06:47:29.383371Z","shell.execute_reply.started":"2026-07-12T06:47:29.379145Z","shell.execute_reply":"2026-07-12T06:47:29.382436Z"}},"outputs":[],"execution_count":52},{"cell_type":"code","source":"criterion=nn.MSELoss()\noptimizer=torch.optim.Adam(model.parameters(),lr=LR,weight_decay=1e-5)","metadata":{"_uuid":"c9ece666-9090-4fbc-bde6-fba6a5681a75","_cell_guid":"5a95c5e6-0d86-4606-8cfd-436fbcecb629","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:47:29.384358Z","iopub.execute_input":"2026-07-12T06:47:29.384645Z","iopub.status.idle":"2026-07-12T06:47:32.652224Z","shell.execute_reply.started":"2026-07-12T06:47:29.384616Z","shell.execute_reply":"2026-07-12T06:47:32.651602Z"}},"outputs":[],"execution_count":53},{"cell_type":"code","source":"scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer, mode=\"min\", factor=0.5, patience=3\n)","metadata":{"_uuid":"19033f86-1870-467e-ad6f-791030089141","_cell_guid":"57dc4b97-f29f-45f8-8cc5-9bab26b713eb","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:47:32.653132Z","iopub.execute_input":"2026-07-12T06:47:32.653652Z","iopub.status.idle":"2026-07-12T06:47:32.658612Z","shell.execute_reply.started":"2026-07-12T06:47:32.653628Z","shell.execute_reply":"2026-07-12T06:47:32.657931Z"}},"outputs":[],"execution_count":54},{"cell_type":"markdown","source":"# Training","metadata":{"_uuid":"013042ae-3811-4743-8ee9-a089415579ed","_cell_guid":"566ade2b-7481-4beb-bb34-ace95bfe2d95","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"train_losses = []\n\nprint(f'training on for {EPOCHS} epochs')\n\nfor epoch in range(EPOCHS):\n    model.train()\n    running_loss=0.0\n\n    for X_batch, y_batch in train_loader:\n        X_batch=X_batch.to(device,non_blocking=True)\n        y_batch=y_batch.to(device, non_blocking=True)\n\n        optimizer.zero_grad()\n        preds=model(X_batch)\n        loss=criterion(preds,y_batch)\n        loss.backward()\n\n        nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        optimizer.step()\n        running_loss+=loss.item()\n\n    avg_loss=running_loss/len(train_loader)\n    current_lr=optimizer.param_groups[0]['lr']\n    train_losses.append(avg_loss)\n    scheduler.step(avg_loss)\n    print(f'epoch:{epoch+1} loss:{avg_loss} LR:{current_lr}')","metadata":{"_uuid":"780402dc-e06d-470a-bdbd-b449a74a2b99","_cell_guid":"310d1ddb-bb6a-4b46-8eea-5fbb72aee4b5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:47:32.659679Z","iopub.execute_input":"2026-07-12T06:47:32.660031Z","iopub.status.idle":"2026-07-12T06:57:36.116191Z","shell.execute_reply.started":"2026-07-12T06:47:32.659995Z","shell.execute_reply":"2026-07-12T06:57:36.115334Z"}},"outputs":[{"name":"stdout","text":"training on for 30 epochs\nepoch:1 loss:0.04440996174353285 LR:0.001\nepoch:2 loss:0.02332202705954554 LR:0.001\nepoch:3 loss:0.0214343013361591 LR:0.001\nepoch:4 loss:0.02051396602646337 LR:0.001\nepoch:5 loss:0.019884127085649614 LR:0.001\nepoch:6 loss:0.01951029486125493 LR:0.001\nepoch:7 loss:0.019289785203534457 LR:0.001\nepoch:8 loss:0.0189505646664302 LR:0.001\nepoch:9 loss:0.01882317099828105 LR:0.001\nepoch:10 loss:0.018588173028876175 LR:0.001\nepoch:11 loss:0.018521703628423272 LR:0.001\nepoch:12 loss:0.018343751295732787 LR:0.001\nepoch:13 loss:0.018301234039935773 LR:0.001\nepoch:14 loss:0.01814426618659774 LR:0.001\nepoch:15 loss:0.018086510448648178 LR:0.001\nepoch:16 loss:0.01794854063201317 LR:0.001\nepoch:17 loss:0.017752943832774537 LR:0.001\nepoch:18 loss:0.017638968553073116 LR:0.001\nepoch:19 loss:0.017649108176330093 LR:0.001\nepoch:20 loss:0.017572558743453 LR:0.001\nepoch:21 loss:0.01756011074840349 LR:0.001\nepoch:22 loss:0.01752838750151486 LR:0.001\nepoch:23 loss:0.017295774861007383 LR:0.001\nepoch:24 loss:0.017212420169890223 LR:0.001\nepoch:25 loss:0.01713363904937428 LR:0.001\nepoch:26 loss:0.01704766796073103 LR:0.001\nepoch:27 loss:0.017183530769221442 LR:0.001\nepoch:28 loss:0.01701346678872419 LR:0.001\nepoch:29 loss:0.017060715615707267 LR:0.001\nepoch:30 loss:0.01691137353092495 LR:0.001\n","output_type":"stream"}],"execution_count":55},{"cell_type":"markdown","source":"# Testing","metadata":{"_uuid":"c8cbe5b3-e075-4a7c-8fba-4e17401026b5","_cell_guid":"19a9dffa-c8eb-425e-ade9-470028b3e5fb","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"model.eval()","metadata":{"_uuid":"b0875cd1-391f-42c0-bf43-c83b6ab3c8cf","_cell_guid":"fe4e4c97-feaf-4daa-85cf-fd6b9ac118bb","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:57:36.117724Z","iopub.execute_input":"2026-07-12T06:57:36.118016Z","iopub.status.idle":"2026-07-12T06:57:36.125182Z","shell.execute_reply.started":"2026-07-12T06:57:36.117978Z","shell.execute_reply":"2026-07-12T06:57:36.124394Z"}},"outputs":[{"execution_count":56,"output_type":"execute_result","data":{"text/plain":"DataParallel(\n  (module): OptionPricingLSTM(\n    (lstm): LSTM(13, 128, num_layers=2, batch_first=True, dropout=0.3)\n    (head): Sequential(\n      (0): Dropout(p=0.3, inplace=False)\n      (1): Linear(in_features=128, out_features=64, bias=True)\n      (2): ReLU()\n      (3): Dropout(p=0.2, inplace=False)\n      (4): Linear(in_features=64, out_features=1, bias=True)\n    )\n  )\n)"},"metadata":{}}],"execution_count":56},{"cell_type":"code","source":"all_preds=[]\nall_origs=[]\nwith torch.no_grad():\n    for X_batch, y_batch in test_loader:\n        X_batch=X_batch.to(device, non_blocking=True)\n        batch_pred=model(X_batch).cpu().numpy()\n        all_preds.extend(batch_pred)\n        all_origs.extend(y_batch)","metadata":{"_uuid":"91bd4d70-bf97-4473-9e61-babaaa9a48c7","_cell_guid":"9a1c1dbe-311d-4dda-985e-7682835942ec","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:57:36.126206Z","iopub.execute_input":"2026-07-12T06:57:36.126560Z","iopub.status.idle":"2026-07-12T06:57:41.773343Z","shell.execute_reply.started":"2026-07-12T06:57:36.126539Z","shell.execute_reply":"2026-07-12T06:57:41.772547Z"}},"outputs":[],"execution_count":57},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score","metadata":{"_uuid":"ef4b51da-ff0f-4aa5-af01-87ee87d8744e","_cell_guid":"a21719fd-1aa6-4ad6-bb8e-3036a9714c98","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:57:41.775557Z","iopub.execute_input":"2026-07-12T06:57:41.776137Z","iopub.status.idle":"2026-07-12T06:57:41.780570Z","shell.execute_reply.started":"2026-07-12T06:57:41.776102Z","shell.execute_reply":"2026-07-12T06:57:41.779914Z"}},"outputs":[],"execution_count":58},{"cell_type":"code","source":"r2_score(all_origs,all_preds)","metadata":{"_uuid":"86dde5e4-273d-44a3-af3c-039d32649b0d","_cell_guid":"66fb5ae6-e7e2-4932-8333-93c52e00563c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:57:41.781539Z","iopub.execute_input":"2026-07-12T06:57:41.781896Z","iopub.status.idle":"2026-07-12T06:57:42.923962Z","shell.execute_reply.started":"2026-07-12T06:57:41.781874Z","shell.execute_reply":"2026-07-12T06:57:42.923234Z"}},"outputs":[{"execution_count":59,"output_type":"execute_result","data":{"text/plain":"0.9894678010796922"},"metadata":{}}],"execution_count":59},{"cell_type":"code","source":"mean_squared_error(all_origs,all_preds), mean_absolute_error(all_origs,all_preds)","metadata":{"_uuid":"60dde317-3c75-4c27-91dc-e8b576b5f04d","_cell_guid":"a6044280-e237-4e43-819b-e00a75689793","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:57:42.924926Z","iopub.execute_input":"2026-07-12T06:57:42.925364Z","iopub.status.idle":"2026-07-12T06:57:45.128057Z","shell.execute_reply.started":"2026-07-12T06:57:42.925338Z","shell.execute_reply":"2026-07-12T06:57:45.127279Z"}},"outputs":[{"execution_count":60,"output_type":"execute_result","data":{"text/plain":"(0.014691572636083413, 0.06215482079725753)"},"metadata":{}}],"execution_count":60},{"cell_type":"code","source":"aoi=target_scaler.inverse_transform(np.array(all_origs).reshape(-1,1))\naop=target_scaler.inverse_transform(np.array(all_preds).reshape(-1,1))","metadata":{"_uuid":"8eca89d0-253d-4664-b05d-23b21e576485","_cell_guid":"2307b196-118e-4f74-8a1f-a240fd1a2131","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:57:45.129104Z","iopub.execute_input":"2026-07-12T06:57:45.129420Z","iopub.status.idle":"2026-07-12T06:57:46.637348Z","shell.execute_reply.started":"2026-07-12T06:57:45.129389Z","shell.execute_reply":"2026-07-12T06:57:46.636519Z"}},"outputs":[],"execution_count":61},{"cell_type":"code","source":"mean_squared_error(aoi,aop), mean_absolute_error(aoi,aop)","metadata":{"_uuid":"aa9105e2-4f54-4737-a499-cae6b3ef4dfb","_cell_guid":"eac07414-7ea1-4a4d-9a91-f0598c99fc2b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-07-12T06:57:46.638368Z","iopub.execute_input":"2026-07-12T06:57:46.638748Z","iopub.status.idle":"2026-07-12T06:57:46.646933Z","shell.execute_reply.started":"2026-07-12T06:57:46.638717Z","shell.execute_reply":"2026-07-12T06:57:46.646185Z"}},"outputs":[{"execution_count":62,"output_type":"execute_result","data":{"text/plain":"(83.41451263427734, 4.683403015136719)"},"metadata":{}}],"execution_count":62},{"cell_type":"code","source":"#RMSE\nnp.sqrt(mean_squared_error(aoi,aop))","metadata":{"_uuid":"a43b711d-872c-4844-b9c7-d4f32a08693e","_cell_guid":"de1187e0-8ffc-4dc0-9df2-986b99c3f691","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-07-12T06:57:46.647821Z","iopub.execute_input":"2026-07-12T06:57:46.648126Z","iopub.status.idle":"2026-07-12T06:57:46.660230Z","shell.execute_reply.started":"2026-07-12T06:57:46.648105Z","shell.execute_reply":"2026-07-12T06:57:46.659616Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":63,"output_type":"execute_result","data":{"text/plain":"np.float64(9.133154582852375)"},"metadata":{}}],"execution_count":63},{"cell_type":"code","source":"","metadata":{"_uuid":"d9e88ebc-ea40-4b5d-9be0-340f079aa1ac","_cell_guid":"c2a7e361-7bd2-430d-b395-a2e2a47e153f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}