Upload lstm-model-for-option-pricing (1).ipynb
Browse files
lstm-model-for-option-pricing (1).ipynb
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Use the kagglehub client library to attach Kaggle resources like competitions, datasets, and models to your session\n# Learn more about kagglehub: https://github.com/Kaggle/kagglehub/blob/main/README.md\n\nimport kagglehub\n# kagglehub.dataset_download('<owner>/<dataset-slug>')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:10:43.637248Z","iopub.execute_input":"2026-07-01T09:10:43.637501Z","iopub.status.idle":"2026-07-01T09:10:46.562671Z","shell.execute_reply.started":"2026-07-01T09:10:43.637470Z","shell.execute_reply":"2026-07-01T09:10:46.561758Z"}},"outputs":[],"execution_count":1},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:10:46.564459Z","iopub.execute_input":"2026-07-01T09:10:46.565007Z","iopub.status.idle":"2026-07-01T09:10:55.303085Z","shell.execute_reply.started":"2026-07-01T09:10:46.564983Z","shell.execute_reply":"2026-07-01T09:10:55.302497Z"}},"outputs":[],"execution_count":2},{"cell_type":"code","source":"SEED=42\ntorch.manual_seed(42)\nnp.random.seed(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:10:55.303943Z","iopub.execute_input":"2026-07-01T09:10:55.304396Z","iopub.status.idle":"2026-07-01T09:10:55.317474Z","shell.execute_reply.started":"2026-07-01T09:10:55.304373Z","shell.execute_reply":"2026-07-01T09:10:55.316547Z"}},"outputs":[],"execution_count":3},{"cell_type":"code","source":"device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f'Using device: {device}')\nprint(f'GPUs available: f{torch.cuda.device_count()}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:10:55.318403Z","iopub.execute_input":"2026-07-01T09:10:55.318579Z","iopub.status.idle":"2026-07-01T09:10:55.627740Z","shell.execute_reply.started":"2026-07-01T09:10:55.318563Z","shell.execute_reply":"2026-07-01T09:10:55.626945Z"}},"outputs":[{"name":"stdout","text":"Using device: cuda\nGPUs available: f2\n","output_type":"stream"}],"execution_count":4},{"cell_type":"markdown","source":"# Preprocessing\n","metadata":{}},{"cell_type":"code","source":"\ndf=pd.read_csv('hf://datasets/major-year-project/stock-data/spy_options_2024.csv')\nprint(f'Raw shape: {df.shape}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:10:55.628740Z","iopub.execute_input":"2026-07-01T09:10:55.629041Z","iopub.status.idle":"2026-07-01T09:11:21.849511Z","shell.execute_reply.started":"2026-07-01T09:10:55.629010Z","shell.execute_reply":"2026-07-01T09:11:21.848615Z"}},"outputs":[{"name":"stdout","text":"Raw shape: (2292798, 20)\n","output_type":"stream"}],"execution_count":5},{"cell_type":"code","source":"df.head(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:21.850444Z","iopub.execute_input":"2026-07-01T09:11:21.850641Z","iopub.status.idle":"2026-07-01T09:11:21.900267Z","shell.execute_reply.started":"2026-07-01T09:11:21.850622Z","shell.execute_reply":"2026-07-01T09:11:21.899588Z"}},"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":" contractID symbol expiration strike type last mark bid \\\n0 SPY240102C00402000 SPY 2024-01-02 402.0 call 69.97 70.77 70.68 \n1 SPY240102P00402000 SPY 2024-01-02 402.0 put 0.02 0.01 0.00 \n2 SPY240102C00403000 SPY 2024-01-02 403.0 call 68.97 69.78 69.68 \n\n bid_size ask ask_size volume open_interest date \\\n0 100 70.85 100 130 1 2024-01-02 \n1 0 0.01 4204 0 303 2024-01-02 \n2 100 69.89 100 9 0 2024-01-02 \n\n implied_volatility delta gamma theta vega rho \n0 1.24983 0.99396 0.00055 -0.32269 0.00423 0.01093 \n1 1.04145 -0.00135 0.00017 -0.05700 0.00110 -0.00002 \n2 1.25456 0.99311 0.00062 -0.35671 0.00476 0.01095 ","text/html":"<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>contractID</th>\n <th>symbol</th>\n <th>expiration</th>\n <th>strike</th>\n <th>type</th>\n <th>last</th>\n <th>mark</th>\n <th>bid</th>\n <th>bid_size</th>\n <th>ask</th>\n <th>ask_size</th>\n <th>volume</th>\n <th>open_interest</th>\n <th>date</th>\n <th>implied_volatility</th>\n <th>delta</th>\n <th>gamma</th>\n <th>theta</th>\n <th>vega</th>\n <th>rho</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>SPY240102C00402000</td>\n <td>SPY</td>\n <td>2024-01-02</td>\n <td>402.0</td>\n <td>call</td>\n <td>69.97</td>\n <td>70.77</td>\n <td>70.68</td>\n <td>100</td>\n <td>70.85</td>\n <td>100</td>\n <td>130</td>\n <td>1</td>\n <td>2024-01-02</td>\n <td>1.24983</td>\n <td>0.99396</td>\n <td>0.00055</td>\n <td>-0.32269</td>\n <td>0.00423</td>\n <td>0.01093</td>\n </tr>\n <tr>\n <th>1</th>\n <td>SPY240102P00402000</td>\n <td>SPY</td>\n <td>2024-01-02</td>\n <td>402.0</td>\n <td>put</td>\n <td>0.02</td>\n <td>0.01</td>\n <td>0.00</td>\n <td>0</td>\n <td>0.01</td>\n <td>4204</td>\n <td>0</td>\n <td>303</td>\n <td>2024-01-02</td>\n <td>1.04145</td>\n <td>-0.00135</td>\n <td>0.00017</td>\n <td>-0.05700</td>\n <td>0.00110</td>\n <td>-0.00002</td>\n </tr>\n <tr>\n <th>2</th>\n <td>SPY240102C00403000</td>\n <td>SPY</td>\n <td>2024-01-02</td>\n <td>403.0</td>\n <td>call</td>\n <td>68.97</td>\n <td>69.78</td>\n <td>69.68</td>\n <td>100</td>\n <td>69.89</td>\n <td>100</td>\n <td>9</td>\n <td>0</td>\n <td>2024-01-02</td>\n <td>1.25456</td>\n <td>0.99311</td>\n <td>0.00062</td>\n <td>-0.35671</td>\n <td>0.00476</td>\n <td>0.01095</td>\n </tr>\n </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":6},{"cell_type":"code","source":"df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:21.902275Z","iopub.execute_input":"2026-07-01T09:11:21.902694Z","iopub.status.idle":"2026-07-01T09:11:21.943738Z","shell.execute_reply.started":"2026-07-01T09:11:21.902673Z","shell.execute_reply":"2026-07-01T09:11:21.942939Z"}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2292798 entries, 0 to 2292797\nData columns (total 20 columns):\n # Column Dtype \n--- ------ ----- \n 0 contractID object \n 1 symbol object \n 2 expiration object \n 3 strike float64\n 4 type object \n 5 last float64\n 6 mark float64\n 7 bid float64\n 8 bid_size int64 \n 9 ask float64\n 10 ask_size int64 \n 11 volume int64 \n 12 open_interest int64 \n 13 date object \n 14 implied_volatility float64\n 15 delta float64\n 16 gamma float64\n 17 theta float64\n 18 vega float64\n 19 rho float64\ndtypes: float64(11), int64(4), object(5)\nmemory usage: 349.9+ MB\n","output_type":"stream"}],"execution_count":7},{"cell_type":"code","source":"df['date']=pd.to_datetime(df['date'])\ndf['expiration']=pd.to_datetime(df['expiration'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:21.944678Z","iopub.execute_input":"2026-07-01T09:11:21.945037Z","iopub.status.idle":"2026-07-01T09:11:22.353875Z","shell.execute_reply.started":"2026-07-01T09:11:21.945015Z","shell.execute_reply":"2026-07-01T09:11:22.353284Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:22.354746Z","iopub.execute_input":"2026-07-01T09:11:22.354947Z","iopub.status.idle":"2026-07-01T09:11:22.362880Z","shell.execute_reply.started":"2026-07-01T09:11:22.354929Z","shell.execute_reply":"2026-07-01T09:11:22.362069Z"}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2292798 entries, 0 to 2292797\nData columns (total 20 columns):\n # Column Dtype \n--- ------ ----- \n 0 contractID object \n 1 symbol object \n 2 expiration datetime64[ns]\n 3 strike float64 \n 4 type object \n 5 last float64 \n 6 mark float64 \n 7 bid float64 \n 8 bid_size int64 \n 9 ask float64 \n 10 ask_size int64 \n 11 volume int64 \n 12 open_interest int64 \n 13 date datetime64[ns]\n 14 implied_volatility float64 \n 15 delta float64 \n 16 gamma float64 \n 17 theta float64 \n 18 vega float64 \n 19 rho float64 \ndtypes: datetime64[ns](2), float64(11), int64(4), object(3)\nmemory usage: 349.9+ MB\n","output_type":"stream"}],"execution_count":9},{"cell_type":"code","source":"df['tte_days']=(df['expiration']-df['date']).dt.days","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:22.363697Z","iopub.execute_input":"2026-07-01T09:11:22.363941Z","iopub.status.idle":"2026-07-01T09:11:22.423372Z","shell.execute_reply.started":"2026-07-01T09:11:22.363923Z","shell.execute_reply":"2026-07-01T09:11:22.422808Z"}},"outputs":[],"execution_count":10},{"cell_type":"code","source":"# Log-moneyness: how far strike is from the median strike\n# (proxy since we don't have live spot price in this file)\ndf[\"log_moneyness\"] = np.log(\n df[\"strike\"] / df[\"strike\"].median()\n).fillna(0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:22.424258Z","iopub.execute_input":"2026-07-01T09:11:22.424513Z","iopub.status.idle":"2026-07-01T09:11:22.457955Z","shell.execute_reply.started":"2026-07-01T09:11:22.424487Z","shell.execute_reply":"2026-07-01T09:11:22.457397Z"}},"outputs":[],"execution_count":11},{"cell_type":"code","source":"df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:22.458765Z","iopub.execute_input":"2026-07-01T09:11:22.459055Z","iopub.status.idle":"2026-07-01T09:11:22.467002Z","shell.execute_reply.started":"2026-07-01T09:11:22.459035Z","shell.execute_reply":"2026-07-01T09:11:22.466252Z"}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2292798 entries, 0 to 2292797\nData columns (total 22 columns):\n # Column Dtype \n--- ------ ----- \n 0 contractID object \n 1 symbol object \n 2 expiration datetime64[ns]\n 3 strike float64 \n 4 type object \n 5 last float64 \n 6 mark float64 \n 7 bid float64 \n 8 bid_size int64 \n 9 ask float64 \n 10 ask_size int64 \n 11 volume int64 \n 12 open_interest int64 \n 13 date datetime64[ns]\n 14 implied_volatility float64 \n 15 delta float64 \n 16 gamma float64 \n 17 theta float64 \n 18 vega float64 \n 19 rho float64 \n 20 tte_days int64 \n 21 log_moneyness float64 \ndtypes: datetime64[ns](2), float64(12), int64(5), object(3)\nmemory usage: 384.8+ MB\n","output_type":"stream"}],"execution_count":12},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:22.467927Z","iopub.execute_input":"2026-07-01T09:11:22.468340Z","iopub.status.idle":"2026-07-01T09:11:22.501625Z","shell.execute_reply.started":"2026-07-01T09:11:22.468313Z","shell.execute_reply":"2026-07-01T09:11:22.500967Z"}},"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":" contractID symbol expiration strike type last mark bid \\\n0 SPY240102C00402000 SPY 2024-01-02 402.0 call 69.97 70.77 70.68 \n1 SPY240102P00402000 SPY 2024-01-02 402.0 put 0.02 0.01 0.00 \n2 SPY240102C00403000 SPY 2024-01-02 403.0 call 68.97 69.78 69.68 \n3 SPY240102P00403000 SPY 2024-01-02 403.0 put 0.01 0.01 0.00 \n4 SPY240102C00404000 SPY 2024-01-02 404.0 call 68.64 68.77 68.68 \n\n bid_size ask ... open_interest date implied_volatility \\\n0 100 70.85 ... 1 2024-01-02 1.24983 \n1 0 0.01 ... 303 2024-01-02 1.04145 \n2 100 69.89 ... 0 2024-01-02 1.25456 \n3 0 0.01 ... 8 2024-01-02 1.02682 \n4 100 68.85 ... 1 2024-01-02 1.21492 \n\n delta gamma theta vega rho tte_days log_moneyness \n0 0.99396 0.00055 -0.32269 0.00423 0.01093 0 -0.218156 \n1 -0.00135 0.00017 -0.05700 0.00110 -0.00002 0 -0.218156 \n2 0.99311 0.00062 -0.35671 0.00476 0.01095 0 -0.215672 \n3 -0.00137 0.00018 -0.05692 0.00111 -0.00002 0 -0.215672 \n4 0.99383 0.00058 -0.32065 0.00431 0.01099 0 -0.213193 \n\n[5 rows x 22 columns]","text/html":"<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>contractID</th>\n <th>symbol</th>\n <th>expiration</th>\n <th>strike</th>\n <th>type</th>\n <th>last</th>\n <th>mark</th>\n <th>bid</th>\n <th>bid_size</th>\n <th>ask</th>\n <th>...</th>\n <th>open_interest</th>\n <th>date</th>\n <th>implied_volatility</th>\n <th>delta</th>\n <th>gamma</th>\n <th>theta</th>\n <th>vega</th>\n <th>rho</th>\n <th>tte_days</th>\n <th>log_moneyness</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>SPY240102C00402000</td>\n <td>SPY</td>\n <td>2024-01-02</td>\n <td>402.0</td>\n <td>call</td>\n <td>69.97</td>\n <td>70.77</td>\n <td>70.68</td>\n <td>100</td>\n <td>70.85</td>\n <td>...</td>\n <td>1</td>\n <td>2024-01-02</td>\n <td>1.24983</td>\n <td>0.99396</td>\n <td>0.00055</td>\n <td>-0.32269</td>\n <td>0.00423</td>\n <td>0.01093</td>\n <td>0</td>\n <td>-0.218156</td>\n </tr>\n <tr>\n <th>1</th>\n <td>SPY240102P00402000</td>\n <td>SPY</td>\n <td>2024-01-02</td>\n <td>402.0</td>\n <td>put</td>\n <td>0.02</td>\n <td>0.01</td>\n <td>0.00</td>\n <td>0</td>\n <td>0.01</td>\n <td>...</td>\n <td>303</td>\n <td>2024-01-02</td>\n <td>1.04145</td>\n <td>-0.00135</td>\n <td>0.00017</td>\n <td>-0.05700</td>\n <td>0.00110</td>\n <td>-0.00002</td>\n <td>0</td>\n <td>-0.218156</td>\n </tr>\n <tr>\n <th>2</th>\n <td>SPY240102C00403000</td>\n <td>SPY</td>\n <td>2024-01-02</td>\n <td>403.0</td>\n <td>call</td>\n <td>68.97</td>\n <td>69.78</td>\n <td>69.68</td>\n <td>100</td>\n <td>69.89</td>\n <td>...</td>\n <td>0</td>\n <td>2024-01-02</td>\n <td>1.25456</td>\n <td>0.99311</td>\n <td>0.00062</td>\n <td>-0.35671</td>\n <td>0.00476</td>\n <td>0.01095</td>\n <td>0</td>\n <td>-0.215672</td>\n </tr>\n <tr>\n <th>3</th>\n <td>SPY240102P00403000</td>\n <td>SPY</td>\n <td>2024-01-02</td>\n <td>403.0</td>\n <td>put</td>\n <td>0.01</td>\n <td>0.01</td>\n <td>0.00</td>\n <td>0</td>\n <td>0.01</td>\n <td>...</td>\n <td>8</td>\n <td>2024-01-02</td>\n <td>1.02682</td>\n <td>-0.00137</td>\n <td>0.00018</td>\n <td>-0.05692</td>\n <td>0.00111</td>\n <td>-0.00002</td>\n <td>0</td>\n <td>-0.215672</td>\n </tr>\n <tr>\n <th>4</th>\n <td>SPY240102C00404000</td>\n <td>SPY</td>\n <td>2024-01-02</td>\n <td>404.0</td>\n <td>call</td>\n <td>68.64</td>\n <td>68.77</td>\n <td>68.68</td>\n <td>100</td>\n <td>68.85</td>\n <td>...</td>\n <td>1</td>\n <td>2024-01-02</td>\n <td>1.21492</td>\n <td>0.99383</td>\n <td>0.00058</td>\n <td>-0.32065</td>\n <td>0.00431</td>\n <td>0.01099</td>\n <td>0</td>\n <td>-0.213193</td>\n </tr>\n </tbody>\n</table>\n<p>5 rows × 22 columns</p>\n</div>"},"metadata":{}}],"execution_count":13},{"cell_type":"code","source":"df[\"type_enc\"] = (df[\"type\"].str.lower() == \"call\").astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:22.502487Z","iopub.execute_input":"2026-07-01T09:11:22.502794Z","iopub.status.idle":"2026-07-01T09:11:22.966926Z","shell.execute_reply.started":"2026-07-01T09:11:22.502772Z","shell.execute_reply":"2026-07-01T09:11:22.966375Z"}},"outputs":[],"execution_count":14},{"cell_type":"code","source":"df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:22.967667Z","iopub.execute_input":"2026-07-01T09:11:22.967850Z","iopub.status.idle":"2026-07-01T09:11:22.976882Z","shell.execute_reply.started":"2026-07-01T09:11:22.967833Z","shell.execute_reply":"2026-07-01T09:11:22.976307Z"}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2292798 entries, 0 to 2292797\nData columns (total 23 columns):\n # Column Dtype \n--- ------ ----- \n 0 contractID object \n 1 symbol object \n 2 expiration datetime64[ns]\n 3 strike float64 \n 4 type object \n 5 last float64 \n 6 mark float64 \n 7 bid float64 \n 8 bid_size int64 \n 9 ask float64 \n 10 ask_size int64 \n 11 volume int64 \n 12 open_interest int64 \n 13 date datetime64[ns]\n 14 implied_volatility float64 \n 15 delta float64 \n 16 gamma float64 \n 17 theta float64 \n 18 vega float64 \n 19 rho float64 \n 20 tte_days int64 \n 21 log_moneyness float64 \n 22 type_enc int64 \ndtypes: datetime64[ns](2), float64(12), int64(6), object(3)\nmemory usage: 402.3+ MB\n","output_type":"stream"}],"execution_count":15},{"cell_type":"code","source":"# Remove rows where target is missing / zero / negative\ndf = df[df[\"mark\"] > 0].copy()\ndf = df.dropna(subset=[\n \"mark\", \"implied_volatility\",\n \"delta\", \"gamma\", \"theta\", \"vega\", \"rho\"\n])\n \nprint(f\" After cleaning : {df.shape}\")\n ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:22.977962Z","iopub.execute_input":"2026-07-01T09:11:22.978145Z","iopub.status.idle":"2026-07-01T09:11:23.964446Z","shell.execute_reply.started":"2026-07-01T09:11:22.978128Z","shell.execute_reply":"2026-07-01T09:11:23.963706Z"}},"outputs":[{"name":"stdout","text":" After cleaning : (2292798, 23)\n","output_type":"stream"}],"execution_count":16},{"cell_type":"code","source":"FEATURES = [\n # Theory-grounded: what Black-Scholes uses\n \"implied_volatility\", # market's expectation of future vol\n \"delta\", # price sensitivity\n \"gamma\", # rate of delta change\n \"theta\", # daily time decay\n \"vega\", # vol sensitivity\n \"rho\", # interest rate sensitivity\n \n # Contract structure\n \"tte_days\", # time to expiry (engineered)\n \"log_moneyness\", # engineered from strike\n \"strike\", # absolute strike level\n \"type_enc\", # call or put\n \n # Market activity (not price itself)\n \"volume\", # contracts traded today\n \"open_interest\", # total open contracts\n]\n \nTARGET = \"mark\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:23.965349Z","iopub.execute_input":"2026-07-01T09:11:23.965652Z","iopub.status.idle":"2026-07-01T09:11:23.970036Z","shell.execute_reply.started":"2026-07-01T09:11:23.965628Z","shell.execute_reply":"2026-07-01T09:11:23.969337Z"}},"outputs":[],"execution_count":17},{"cell_type":"code","source":"df=df[FEATURES+[TARGET,'date','contractID']].dropna()\nprint(f\" Final shape : {df.shape}\")\nprint(f\" Features ({len(FEATURES)}) : {FEATURES}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:23.971133Z","iopub.execute_input":"2026-07-01T09:11:23.971485Z","iopub.status.idle":"2026-07-01T09:11:24.369772Z","shell.execute_reply.started":"2026-07-01T09:11:23.971452Z","shell.execute_reply":"2026-07-01T09:11:24.368378Z"}},"outputs":[{"name":"stdout","text":" Final shape : (2292798, 15)\n Features (12) : ['implied_volatility', 'delta', 'gamma', 'theta', 'vega', 'rho', 'tte_days', 'log_moneyness', 'strike', 'type_enc', 'volume', 'open_interest']\n","output_type":"stream"}],"execution_count":18},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:24.370852Z","iopub.execute_input":"2026-07-01T09:11:24.371257Z","iopub.status.idle":"2026-07-01T09:11:24.386026Z","shell.execute_reply.started":"2026-07-01T09:11:24.371197Z","shell.execute_reply":"2026-07-01T09:11:24.385128Z"}},"outputs":[{"execution_count":19,"output_type":"execute_result","data":{"text/plain":" implied_volatility delta gamma theta vega rho tte_days \\\n0 1.24983 0.99396 0.00055 -0.32269 0.00423 0.01093 0 \n1 1.04145 -0.00135 0.00017 -0.05700 0.00110 -0.00002 0 \n2 1.25456 0.99311 0.00062 -0.35671 0.00476 0.01095 0 \n3 1.02682 -0.00137 0.00018 -0.05692 0.00111 -0.00002 0 \n4 1.21492 0.99383 0.00058 -0.32065 0.00431 0.01099 0 \n\n log_moneyness strike type_enc volume open_interest mark date \\\n0 -0.218156 402.0 1 130 1 70.77 2024-01-02 \n1 -0.218156 402.0 0 0 303 0.01 2024-01-02 \n2 -0.215672 403.0 1 9 0 69.78 2024-01-02 \n3 -0.215672 403.0 0 1 8 0.01 2024-01-02 \n4 -0.213193 404.0 1 1 1 68.77 2024-01-02 \n\n contractID \n0 SPY240102C00402000 \n1 SPY240102P00402000 \n2 SPY240102C00403000 \n3 SPY240102P00403000 \n4 SPY240102C00404000 ","text/html":"<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>implied_volatility</th>\n <th>delta</th>\n <th>gamma</th>\n <th>theta</th>\n <th>vega</th>\n <th>rho</th>\n <th>tte_days</th>\n <th>log_moneyness</th>\n <th>strike</th>\n <th>type_enc</th>\n <th>volume</th>\n <th>open_interest</th>\n <th>mark</th>\n <th>date</th>\n <th>contractID</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>1.24983</td>\n <td>0.99396</td>\n <td>0.00055</td>\n <td>-0.32269</td>\n <td>0.00423</td>\n <td>0.01093</td>\n <td>0</td>\n <td>-0.218156</td>\n <td>402.0</td>\n <td>1</td>\n <td>130</td>\n <td>1</td>\n <td>70.77</td>\n <td>2024-01-02</td>\n <td>SPY240102C00402000</td>\n </tr>\n <tr>\n <th>1</th>\n <td>1.04145</td>\n <td>-0.00135</td>\n <td>0.00017</td>\n <td>-0.05700</td>\n <td>0.00110</td>\n <td>-0.00002</td>\n <td>0</td>\n <td>-0.218156</td>\n <td>402.0</td>\n <td>0</td>\n <td>0</td>\n <td>303</td>\n <td>0.01</td>\n <td>2024-01-02</td>\n <td>SPY240102P00402000</td>\n </tr>\n <tr>\n <th>2</th>\n <td>1.25456</td>\n <td>0.99311</td>\n <td>0.00062</td>\n <td>-0.35671</td>\n <td>0.00476</td>\n <td>0.01095</td>\n <td>0</td>\n <td>-0.215672</td>\n <td>403.0</td>\n <td>1</td>\n <td>9</td>\n <td>0</td>\n <td>69.78</td>\n <td>2024-01-02</td>\n <td>SPY240102C00403000</td>\n </tr>\n <tr>\n <th>3</th>\n <td>1.02682</td>\n <td>-0.00137</td>\n <td>0.00018</td>\n <td>-0.05692</td>\n <td>0.00111</td>\n <td>-0.00002</td>\n <td>0</td>\n <td>-0.215672</td>\n <td>403.0</td>\n <td>0</td>\n <td>1</td>\n <td>8</td>\n <td>0.01</td>\n <td>2024-01-02</td>\n <td>SPY240102P00403000</td>\n </tr>\n <tr>\n <th>4</th>\n <td>1.21492</td>\n <td>0.99383</td>\n <td>0.00058</td>\n <td>-0.32065</td>\n <td>0.00431</td>\n <td>0.01099</td>\n <td>0</td>\n <td>-0.213193</td>\n <td>404.0</td>\n <td>1</td>\n <td>1</td>\n <td>1</td>\n <td>68.77</td>\n <td>2024-01-02</td>\n <td>SPY240102C00404000</td>\n </tr>\n </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":19},{"cell_type":"markdown","source":"# Splitting by 70/30 by time","metadata":{}},{"cell_type":"code","source":"print('\\n[3] Splitting 70/30 by time')\ndf=df.sort_values(by=['contractID', 'date']).reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:24.387189Z","iopub.execute_input":"2026-07-01T09:11:24.387469Z","iopub.status.idle":"2026-07-01T09:11:25.133702Z","shell.execute_reply.started":"2026-07-01T09:11:24.387447Z","shell.execute_reply":"2026-07-01T09:11:25.133107Z"}},"outputs":[{"name":"stdout","text":"\n[3] Splitting 70/30 by time\n","output_type":"stream"}],"execution_count":20},{"cell_type":"code","source":"df.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:25.134608Z","iopub.execute_input":"2026-07-01T09:11:25.134850Z","iopub.status.idle":"2026-07-01T09:11:25.149254Z","shell.execute_reply.started":"2026-07-01T09:11:25.134828Z","shell.execute_reply":"2026-07-01T09:11:25.148425Z"}},"outputs":[{"execution_count":21,"output_type":"execute_result","data":{"text/plain":" implied_volatility delta gamma theta vega rho tte_days \\\n0 1.24983 0.99396 0.00055 -0.32269 0.00423 0.01093 0 \n1 1.25456 0.99311 0.00062 -0.35671 0.00476 0.01095 0 \n2 1.21492 0.99383 0.00058 -0.32065 0.00431 0.01099 0 \n3 1.14861 1.00000 0.00000 -0.05913 0.00000 0.01109 0 \n4 1.13163 1.00000 0.00000 -0.05928 0.00000 0.01112 0 \n\n log_moneyness strike type_enc volume open_interest mark date \\\n0 -0.218156 402.0 1 130 1 70.77 2024-01-02 \n1 -0.215672 403.0 1 9 0 69.78 2024-01-02 \n2 -0.213193 404.0 1 1 1 68.77 2024-01-02 \n3 -0.210721 405.0 1 0 4 67.52 2024-01-02 \n4 -0.208255 406.0 1 0 3 66.52 2024-01-02 \n\n contractID \n0 SPY240102C00402000 \n1 SPY240102C00403000 \n2 SPY240102C00404000 \n3 SPY240102C00405000 \n4 SPY240102C00406000 ","text/html":"<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>implied_volatility</th>\n <th>delta</th>\n <th>gamma</th>\n <th>theta</th>\n <th>vega</th>\n <th>rho</th>\n <th>tte_days</th>\n <th>log_moneyness</th>\n <th>strike</th>\n <th>type_enc</th>\n <th>volume</th>\n <th>open_interest</th>\n <th>mark</th>\n <th>date</th>\n <th>contractID</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>1.24983</td>\n <td>0.99396</td>\n <td>0.00055</td>\n <td>-0.32269</td>\n <td>0.00423</td>\n <td>0.01093</td>\n <td>0</td>\n <td>-0.218156</td>\n <td>402.0</td>\n <td>1</td>\n <td>130</td>\n <td>1</td>\n <td>70.77</td>\n <td>2024-01-02</td>\n <td>SPY240102C00402000</td>\n </tr>\n <tr>\n <th>1</th>\n <td>1.25456</td>\n <td>0.99311</td>\n <td>0.00062</td>\n <td>-0.35671</td>\n <td>0.00476</td>\n <td>0.01095</td>\n <td>0</td>\n <td>-0.215672</td>\n <td>403.0</td>\n <td>1</td>\n <td>9</td>\n <td>0</td>\n <td>69.78</td>\n <td>2024-01-02</td>\n <td>SPY240102C00403000</td>\n </tr>\n <tr>\n <th>2</th>\n <td>1.21492</td>\n <td>0.99383</td>\n <td>0.00058</td>\n <td>-0.32065</td>\n <td>0.00431</td>\n <td>0.01099</td>\n <td>0</td>\n <td>-0.213193</td>\n <td>404.0</td>\n <td>1</td>\n <td>1</td>\n <td>1</td>\n <td>68.77</td>\n <td>2024-01-02</td>\n <td>SPY240102C00404000</td>\n </tr>\n <tr>\n <th>3</th>\n <td>1.14861</td>\n <td>1.00000</td>\n <td>0.00000</td>\n <td>-0.05913</td>\n <td>0.00000</td>\n <td>0.01109</td>\n <td>0</td>\n <td>-0.210721</td>\n <td>405.0</td>\n <td>1</td>\n <td>0</td>\n <td>4</td>\n <td>67.52</td>\n <td>2024-01-02</td>\n <td>SPY240102C00405000</td>\n </tr>\n <tr>\n <th>4</th>\n <td>1.13163</td>\n <td>1.00000</td>\n <td>0.00000</td>\n <td>-0.05928</td>\n <td>0.00000</td>\n <td>0.01112</td>\n <td>0</td>\n <td>-0.208255</td>\n <td>406.0</td>\n <td>1</td>\n <td>0</td>\n <td>3</td>\n <td>66.52</td>\n <td>2024-01-02</td>\n <td>SPY240102C00406000</td>\n </tr>\n </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":21},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:25.154710Z","iopub.execute_input":"2026-07-01T09:11:25.155162Z","iopub.status.idle":"2026-07-01T09:11:25.181892Z","shell.execute_reply.started":"2026-07-01T09:11:25.155140Z","shell.execute_reply":"2026-07-01T09:11:25.181186Z"}},"outputs":[{"execution_count":22,"output_type":"execute_result","data":{"text/plain":"(2292798, 15)"},"metadata":{}}],"execution_count":22},{"cell_type":"code","source":"len(df['contractID'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:25.183041Z","iopub.execute_input":"2026-07-01T09:11:25.183414Z","iopub.status.idle":"2026-07-01T09:11:25.440329Z","shell.execute_reply.started":"2026-07-01T09:11:25.183389Z","shell.execute_reply":"2026-07-01T09:11:25.439372Z"}},"outputs":[{"execution_count":23,"output_type":"execute_result","data":{"text/plain":"76278"},"metadata":{}}],"execution_count":23},{"cell_type":"code","source":"cnt=1\nfor i in (df['contractID'].value_counts()):\n if i>=11:\n cnt+=i\ncnt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:25.441190Z","iopub.execute_input":"2026-07-01T09:11:25.441457Z","iopub.status.idle":"2026-07-01T09:11:25.681611Z","shell.execute_reply.started":"2026-07-01T09:11:25.441437Z","shell.execute_reply":"2026-07-01T09:11:25.680947Z"}},"outputs":[{"execution_count":24,"output_type":"execute_result","data":{"text/plain":"1952207"},"metadata":{}}],"execution_count":24},{"cell_type":"code","source":"obs_per_contract=df['contractID'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:25.682472Z","iopub.execute_input":"2026-07-01T09:11:25.682783Z","iopub.status.idle":"2026-07-01T09:11:25.901019Z","shell.execute_reply.started":"2026-07-01T09:11:25.682752Z","shell.execute_reply":"2026-07-01T09:11:25.900396Z"}},"outputs":[],"execution_count":25},{"cell_type":"code","source":"valid_contracts=obs_per_contract[obs_per_contract>=11].index","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:25.901936Z","iopub.execute_input":"2026-07-01T09:11:25.902172Z","iopub.status.idle":"2026-07-01T09:11:25.917859Z","shell.execute_reply.started":"2026-07-01T09:11:25.902133Z","shell.execute_reply":"2026-07-01T09:11:25.917126Z"}},"outputs":[],"execution_count":26},{"cell_type":"code","source":"valid_contracts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:25.919162Z","iopub.execute_input":"2026-07-01T09:11:25.919609Z","iopub.status.idle":"2026-07-01T09:11:25.933568Z","shell.execute_reply.started":"2026-07-01T09:11:25.919569Z","shell.execute_reply":"2026-07-01T09:11:25.932727Z"}},"outputs":[{"execution_count":27,"output_type":"execute_result","data":{"text/plain":"Index(['SPY241231C00457000', 'SPY241231C00490000', 'SPY250117P00475000',\n 'SPY250117P00480000', 'SPY250117P00405000', 'SPY250117P00410000',\n 'SPY250117P00415000', 'SPY250117P00180000', 'SPY250117P00425000',\n 'SPY250117P00430000',\n ...\n 'SPY240826P00526000', 'SPY240826P00525000', 'SPY240826P00540000',\n 'SPY240826P00539000', 'SPY240826P00538000', 'SPY240826P00537000',\n 'SPY240826P00536000', 'SPY240826P00535000', 'SPY240826P00534000',\n 'SPY240826P00532000'],\n dtype='object', name='contractID', length=30812)"},"metadata":{}}],"execution_count":27},{"cell_type":"code","source":"before=2292798\ndf=df[df['contractID'].isin(valid_contracts)].copy()\nafter=len(df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:25.934616Z","iopub.execute_input":"2026-07-01T09:11:25.934969Z","iopub.status.idle":"2026-07-01T09:11:26.306920Z","shell.execute_reply.started":"2026-07-01T09:11:25.934949Z","shell.execute_reply":"2026-07-01T09:11:26.305988Z"}},"outputs":[],"execution_count":28},{"cell_type":"code","source":"print(f'rows kept: {after}')\nprint(f'Original number of rows: {before}')\nprint(f'Rows dropped: {before - after}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:26.307969Z","iopub.execute_input":"2026-07-01T09:11:26.308256Z","iopub.status.idle":"2026-07-01T09:11:26.312839Z","shell.execute_reply.started":"2026-07-01T09:11:26.308208Z","shell.execute_reply":"2026-07-01T09:11:26.312133Z"}},"outputs":[{"name":"stdout","text":"rows kept: 1952206\nOriginal number of rows: 2292798\nRows dropped: 340592\n","output_type":"stream"}],"execution_count":29},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:26.314684Z","iopub.execute_input":"2026-07-01T09:11:26.315049Z","iopub.status.idle":"2026-07-01T09:11:26.336102Z","shell.execute_reply.started":"2026-07-01T09:11:26.315027Z","shell.execute_reply":"2026-07-01T09:11:26.335390Z"}},"outputs":[{"execution_count":30,"output_type":"execute_result","data":{"text/plain":" implied_volatility delta gamma theta vega rho \\\n14808 1.76598 0.99683 0.00005 -0.07685 0.00980 0.08284 \n14809 1.50714 0.99931 0.00002 -0.03724 0.00235 0.07856 \n14810 1.88412 0.99647 0.00006 -0.08891 0.01002 0.07304 \n14811 2.03914 0.99530 0.00007 -0.11723 0.01255 0.06794 \n14812 0.83023 1.00000 0.00000 -0.02624 0.00000 0.05416 \n\n tte_days log_moneyness strike type_enc volume open_interest \\\n14808 17 -1.021651 180.0 1 0 107 \n14809 16 -1.021651 180.0 1 0 107 \n14810 15 -1.021651 180.0 1 0 107 \n14811 14 -1.021651 180.0 1 0 107 \n14812 11 -1.021651 180.0 1 52 107 \n\n mark date contractID \n14808 293.29 2024-01-02 SPY240119C00180000 \n14809 289.24 2024-01-03 SPY240119C00180000 \n14810 287.89 2024-01-04 SPY240119C00180000 \n14811 288.60 2024-01-05 SPY240119C00180000 \n14812 294.75 2024-01-08 SPY240119C00180000 ","text/html":"<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>implied_volatility</th>\n <th>delta</th>\n <th>gamma</th>\n <th>theta</th>\n <th>vega</th>\n <th>rho</th>\n <th>tte_days</th>\n <th>log_moneyness</th>\n <th>strike</th>\n <th>type_enc</th>\n <th>volume</th>\n <th>open_interest</th>\n <th>mark</th>\n <th>date</th>\n <th>contractID</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>14808</th>\n <td>1.76598</td>\n <td>0.99683</td>\n <td>0.00005</td>\n <td>-0.07685</td>\n <td>0.00980</td>\n <td>0.08284</td>\n <td>17</td>\n <td>-1.021651</td>\n <td>180.0</td>\n <td>1</td>\n <td>0</td>\n <td>107</td>\n <td>293.29</td>\n <td>2024-01-02</td>\n <td>SPY240119C00180000</td>\n </tr>\n <tr>\n <th>14809</th>\n <td>1.50714</td>\n <td>0.99931</td>\n <td>0.00002</td>\n <td>-0.03724</td>\n <td>0.00235</td>\n <td>0.07856</td>\n <td>16</td>\n <td>-1.021651</td>\n <td>180.0</td>\n <td>1</td>\n <td>0</td>\n <td>107</td>\n <td>289.24</td>\n <td>2024-01-03</td>\n <td>SPY240119C00180000</td>\n </tr>\n <tr>\n <th>14810</th>\n <td>1.88412</td>\n <td>0.99647</td>\n <td>0.00006</td>\n <td>-0.08891</td>\n <td>0.01002</td>\n <td>0.07304</td>\n <td>15</td>\n <td>-1.021651</td>\n <td>180.0</td>\n <td>1</td>\n <td>0</td>\n <td>107</td>\n <td>287.89</td>\n <td>2024-01-04</td>\n <td>SPY240119C00180000</td>\n </tr>\n <tr>\n <th>14811</th>\n <td>2.03914</td>\n <td>0.99530</td>\n <td>0.00007</td>\n <td>-0.11723</td>\n <td>0.01255</td>\n <td>0.06794</td>\n <td>14</td>\n <td>-1.021651</td>\n <td>180.0</td>\n <td>1</td>\n <td>0</td>\n <td>107</td>\n <td>288.60</td>\n <td>2024-01-05</td>\n <td>SPY240119C00180000</td>\n </tr>\n <tr>\n <th>14812</th>\n <td>0.83023</td>\n <td>1.00000</td>\n <td>0.00000</td>\n <td>-0.02624</td>\n <td>0.00000</td>\n <td>0.05416</td>\n <td>11</td>\n <td>-1.021651</td>\n <td>180.0</td>\n <td>1</td>\n <td>52</td>\n <td>107</td>\n <td>294.75</td>\n <td>2024-01-08</td>\n <td>SPY240119C00180000</td>\n </tr>\n </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":30},{"cell_type":"code","source":"# now we sort the values\nunique_dates=sorted(df['date'].unique())\nsplit_date=unique_dates[int(len(unique_dates)*0.70)]\nsplit_date","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:26.336982Z","iopub.execute_input":"2026-07-01T09:11:26.337180Z","iopub.status.idle":"2026-07-01T09:11:26.365932Z","shell.execute_reply.started":"2026-07-01T09:11:26.337162Z","shell.execute_reply":"2026-07-01T09:11:26.365425Z"}},"outputs":[{"execution_count":31,"output_type":"execute_result","data":{"text/plain":"Timestamp('2024-09-13 00:00:00')"},"metadata":{}}],"execution_count":31},{"cell_type":"code","source":"df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:26.366621Z","iopub.execute_input":"2026-07-01T09:11:26.367319Z","iopub.status.idle":"2026-07-01T09:11:26.371988Z","shell.execute_reply.started":"2026-07-01T09:11:26.367297Z","shell.execute_reply":"2026-07-01T09:11:26.371399Z"}},"outputs":[{"execution_count":32,"output_type":"execute_result","data":{"text/plain":"Index(['implied_volatility', 'delta', 'gamma', 'theta', 'vega', 'rho',\n 'tte_days', 'log_moneyness', 'strike', 'type_enc', 'volume',\n 'open_interest', 'mark', 'date', 'contractID'],\n dtype='object')"},"metadata":{}}],"execution_count":32},{"cell_type":"code","source":"train_df = df[df[\"date\"] < split_date].copy()\ntest_df = df[df[\"date\"] >= split_date].copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:26.372787Z","iopub.execute_input":"2026-07-01T09:11:26.373028Z","iopub.status.idle":"2026-07-01T09:11:26.636611Z","shell.execute_reply.started":"2026-07-01T09:11:26.372999Z","shell.execute_reply":"2026-07-01T09:11:26.635939Z"}},"outputs":[],"execution_count":33},{"cell_type":"code","source":"print(f\" Split date : {pd.Timestamp(split_date).date()}\")\nprint(f\" Train : {len(train_df):,} rows \"\n f\"({train_df['date'].min().date()} → {train_df['date'].max().date()})\")\nprint(f\" Test : {len(test_df):,} rows \"\n f\"({test_df['date'].min().date()} → {test_df['date'].max().date()})\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:26.637564Z","iopub.execute_input":"2026-07-01T09:11:26.637844Z","iopub.status.idle":"2026-07-01T09:11:26.655157Z","shell.execute_reply.started":"2026-07-01T09:11:26.637818Z","shell.execute_reply":"2026-07-01T09:11:26.654389Z"}},"outputs":[{"name":"stdout","text":" Split date : 2024-09-13\n Train : 1,327,226 rows (2024-01-02 → 2024-09-12)\n Test : 624,980 rows (2024-09-13 → 2024-12-31)\n","output_type":"stream"}],"execution_count":34},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:26.656095Z","iopub.execute_input":"2026-07-01T09:11:26.656344Z","iopub.status.idle":"2026-07-01T09:11:26.670578Z","shell.execute_reply.started":"2026-07-01T09:11:26.656325Z","shell.execute_reply":"2026-07-01T09:11:26.669536Z"}},"outputs":[{"execution_count":35,"output_type":"execute_result","data":{"text/plain":" implied_volatility delta gamma theta vega rho \\\n14808 1.76598 0.99683 0.00005 -0.07685 0.00980 0.08284 \n14809 1.50714 0.99931 0.00002 -0.03724 0.00235 0.07856 \n14810 1.88412 0.99647 0.00006 -0.08891 0.01002 0.07304 \n14811 2.03914 0.99530 0.00007 -0.11723 0.01255 0.06794 \n14812 0.83023 1.00000 0.00000 -0.02624 0.00000 0.05416 \n\n tte_days log_moneyness strike type_enc volume open_interest \\\n14808 17 -1.021651 180.0 1 0 107 \n14809 16 -1.021651 180.0 1 0 107 \n14810 15 -1.021651 180.0 1 0 107 \n14811 14 -1.021651 180.0 1 0 107 \n14812 11 -1.021651 180.0 1 52 107 \n\n mark date contractID \n14808 293.29 2024-01-02 SPY240119C00180000 \n14809 289.24 2024-01-03 SPY240119C00180000 \n14810 287.89 2024-01-04 SPY240119C00180000 \n14811 288.60 2024-01-05 SPY240119C00180000 \n14812 294.75 2024-01-08 SPY240119C00180000 ","text/html":"<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>implied_volatility</th>\n <th>delta</th>\n <th>gamma</th>\n <th>theta</th>\n <th>vega</th>\n <th>rho</th>\n <th>tte_days</th>\n <th>log_moneyness</th>\n <th>strike</th>\n <th>type_enc</th>\n <th>volume</th>\n <th>open_interest</th>\n <th>mark</th>\n <th>date</th>\n <th>contractID</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>14808</th>\n <td>1.76598</td>\n <td>0.99683</td>\n <td>0.00005</td>\n <td>-0.07685</td>\n <td>0.00980</td>\n <td>0.08284</td>\n <td>17</td>\n <td>-1.021651</td>\n <td>180.0</td>\n <td>1</td>\n <td>0</td>\n <td>107</td>\n <td>293.29</td>\n <td>2024-01-02</td>\n <td>SPY240119C00180000</td>\n </tr>\n <tr>\n <th>14809</th>\n <td>1.50714</td>\n <td>0.99931</td>\n <td>0.00002</td>\n <td>-0.03724</td>\n <td>0.00235</td>\n <td>0.07856</td>\n <td>16</td>\n <td>-1.021651</td>\n <td>180.0</td>\n <td>1</td>\n <td>0</td>\n <td>107</td>\n <td>289.24</td>\n <td>2024-01-03</td>\n <td>SPY240119C00180000</td>\n </tr>\n <tr>\n <th>14810</th>\n <td>1.88412</td>\n <td>0.99647</td>\n <td>0.00006</td>\n <td>-0.08891</td>\n <td>0.01002</td>\n <td>0.07304</td>\n <td>15</td>\n <td>-1.021651</td>\n <td>180.0</td>\n <td>1</td>\n <td>0</td>\n <td>107</td>\n <td>287.89</td>\n <td>2024-01-04</td>\n <td>SPY240119C00180000</td>\n </tr>\n <tr>\n <th>14811</th>\n <td>2.03914</td>\n <td>0.99530</td>\n <td>0.00007</td>\n <td>-0.11723</td>\n <td>0.01255</td>\n <td>0.06794</td>\n <td>14</td>\n <td>-1.021651</td>\n <td>180.0</td>\n <td>1</td>\n <td>0</td>\n <td>107</td>\n <td>288.60</td>\n <td>2024-01-05</td>\n <td>SPY240119C00180000</td>\n </tr>\n <tr>\n <th>14812</th>\n <td>0.83023</td>\n <td>1.00000</td>\n <td>0.00000</td>\n <td>-0.02624</td>\n <td>0.00000</td>\n <td>0.05416</td>\n <td>11</td>\n <td>-1.021651</td>\n <td>180.0</td>\n <td>1</td>\n <td>52</td>\n <td>107</td>\n <td>294.75</td>\n <td>2024-01-08</td>\n <td>SPY240119C00180000</td>\n </tr>\n </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":35},{"cell_type":"markdown","source":"# Scaling","metadata":{}},{"cell_type":"code","source":"feat_scaler=StandardScaler()\ntarget_scaler=StandardScaler()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:26.671623Z","iopub.execute_input":"2026-07-01T09:11:26.672148Z","iopub.status.idle":"2026-07-01T09:11:26.683715Z","shell.execute_reply.started":"2026-07-01T09:11:26.672125Z","shell.execute_reply":"2026-07-01T09:11:26.683086Z"}},"outputs":[],"execution_count":36},{"cell_type":"code","source":"train_df[FEATURES]=feat_scaler.fit_transform(train_df[FEATURES].values)\ntest_df[FEATURES] = feat_scaler.transform(test_df[FEATURES].values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:26.684924Z","iopub.execute_input":"2026-07-01T09:11:26.685347Z","iopub.status.idle":"2026-07-01T09:11:27.124409Z","shell.execute_reply.started":"2026-07-01T09:11:26.685281Z","shell.execute_reply":"2026-07-01T09:11:27.123528Z"}},"outputs":[],"execution_count":37},{"cell_type":"code","source":"print(type(train_df))\nprint(type(test_df))\nprint(TARGET)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:27.125572Z","iopub.execute_input":"2026-07-01T09:11:27.125801Z","iopub.status.idle":"2026-07-01T09:11:27.130120Z","shell.execute_reply.started":"2026-07-01T09:11:27.125782Z","shell.execute_reply":"2026-07-01T09:11:27.129204Z"}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\n<class 'pandas.core.frame.DataFrame'>\nmark\n","output_type":"stream"}],"execution_count":38},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:27.131134Z","iopub.execute_input":"2026-07-01T09:11:27.131706Z","iopub.status.idle":"2026-07-01T09:11:27.154666Z","shell.execute_reply.started":"2026-07-01T09:11:27.131684Z","shell.execute_reply":"2026-07-01T09:11:27.154034Z"}},"outputs":[{"execution_count":39,"output_type":"execute_result","data":{"text/plain":" implied_volatility delta gamma theta vega rho \\\n14808 6.403615 1.351108 -0.417523 -0.109284 -0.848120 -0.026046 \n14809 5.325608 1.355326 -0.420085 0.248077 -0.858308 -0.027938 \n14810 6.895640 1.350496 -0.416668 -0.218089 -0.847819 -0.030377 \n14811 7.541262 1.348506 -0.415814 -0.473592 -0.844359 -0.032631 \n14812 2.506438 1.356499 -0.421794 0.347318 -0.861521 -0.038721 \n\n tte_days log_moneyness strike type_enc volume open_interest \\\n14808 -0.822152 -3.349362 -2.508474 1.0 -0.072299 -0.235896 \n14809 -0.826610 -3.349362 -2.508474 1.0 -0.072299 -0.235896 \n14810 -0.831067 -3.349362 -2.508474 1.0 -0.072299 -0.235896 \n14811 -0.835524 -3.349362 -2.508474 1.0 -0.072299 -0.235896 \n14812 -0.848895 -3.349362 -2.508474 1.0 -0.065971 -0.235896 \n\n mark date contractID \n14808 293.29 2024-01-02 SPY240119C00180000 \n14809 289.24 2024-01-03 SPY240119C00180000 \n14810 287.89 2024-01-04 SPY240119C00180000 \n14811 288.60 2024-01-05 SPY240119C00180000 \n14812 294.75 2024-01-08 SPY240119C00180000 ","text/html":"<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>implied_volatility</th>\n <th>delta</th>\n <th>gamma</th>\n <th>theta</th>\n <th>vega</th>\n <th>rho</th>\n <th>tte_days</th>\n <th>log_moneyness</th>\n <th>strike</th>\n <th>type_enc</th>\n <th>volume</th>\n <th>open_interest</th>\n <th>mark</th>\n <th>date</th>\n <th>contractID</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>14808</th>\n <td>6.403615</td>\n <td>1.351108</td>\n <td>-0.417523</td>\n <td>-0.109284</td>\n <td>-0.848120</td>\n <td>-0.026046</td>\n <td>-0.822152</td>\n <td>-3.349362</td>\n <td>-2.508474</td>\n <td>1.0</td>\n <td>-0.072299</td>\n <td>-0.235896</td>\n <td>293.29</td>\n <td>2024-01-02</td>\n <td>SPY240119C00180000</td>\n </tr>\n <tr>\n <th>14809</th>\n <td>5.325608</td>\n <td>1.355326</td>\n <td>-0.420085</td>\n <td>0.248077</td>\n <td>-0.858308</td>\n <td>-0.027938</td>\n <td>-0.826610</td>\n <td>-3.349362</td>\n <td>-2.508474</td>\n <td>1.0</td>\n <td>-0.072299</td>\n <td>-0.235896</td>\n <td>289.24</td>\n <td>2024-01-03</td>\n <td>SPY240119C00180000</td>\n </tr>\n <tr>\n <th>14810</th>\n <td>6.895640</td>\n <td>1.350496</td>\n <td>-0.416668</td>\n <td>-0.218089</td>\n <td>-0.847819</td>\n <td>-0.030377</td>\n <td>-0.831067</td>\n <td>-3.349362</td>\n <td>-2.508474</td>\n <td>1.0</td>\n <td>-0.072299</td>\n <td>-0.235896</td>\n <td>287.89</td>\n <td>2024-01-04</td>\n <td>SPY240119C00180000</td>\n </tr>\n <tr>\n <th>14811</th>\n <td>7.541262</td>\n <td>1.348506</td>\n <td>-0.415814</td>\n <td>-0.473592</td>\n <td>-0.844359</td>\n <td>-0.032631</td>\n <td>-0.835524</td>\n <td>-3.349362</td>\n <td>-2.508474</td>\n <td>1.0</td>\n <td>-0.072299</td>\n <td>-0.235896</td>\n <td>288.60</td>\n <td>2024-01-05</td>\n <td>SPY240119C00180000</td>\n </tr>\n <tr>\n <th>14812</th>\n <td>2.506438</td>\n <td>1.356499</td>\n <td>-0.421794</td>\n <td>0.347318</td>\n <td>-0.861521</td>\n <td>-0.038721</td>\n <td>-0.848895</td>\n <td>-3.349362</td>\n <td>-2.508474</td>\n <td>1.0</td>\n <td>-0.065971</td>\n <td>-0.235896</td>\n <td>294.75</td>\n <td>2024-01-08</td>\n <td>SPY240119C00180000</td>\n </tr>\n </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":39},{"cell_type":"code","source":"train_df[TARGET]=target_scaler.fit_transform(train_df[[TARGET]].values).ravel()\ntest_df[TARGET]=target_scaler.transform(test_df[[TARGET]].values).ravel()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:27.155770Z","iopub.execute_input":"2026-07-01T09:11:27.155948Z","iopub.status.idle":"2026-07-01T09:11:27.192405Z","shell.execute_reply.started":"2026-07-01T09:11:27.155931Z","shell.execute_reply":"2026-07-01T09:11:27.191757Z"}},"outputs":[],"execution_count":40},{"cell_type":"code","source":"SEQ_LEN=10\nBATCH_SIZE=1024","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:27.193120Z","iopub.execute_input":"2026-07-01T09:11:27.193355Z","iopub.status.idle":"2026-07-01T09:11:27.196895Z","shell.execute_reply.started":"2026-07-01T09:11:27.193336Z","shell.execute_reply":"2026-07-01T09:11:27.196030Z"}},"outputs":[],"execution_count":41},{"cell_type":"code","source":"class LazyContractDataset(Dataset):\n def __init__(self,df,features,target,seq_len):\n self.seq_len=seq_len\n self.features=features\n self.target=target\n\n self.index=[]\n self.contract_data=[]\n skipped=0\n n_contracts=0\n for contract, group in df.groupby(by='contractID',sort=False):\n group=group.sort_values(by='date')\n X = group[features].values.astype(np.float32)\n y = group[target].values.astype(np.float32)\n\n\n if len(X)<seq_len+1:\n skipped+=1\n continue\n\n n_contracts+=1\n\n arr_idx=len(self.contract_data)\n self.contract_data.append((X,y))\n\n for i in range(len(X)-seq_len):\n self.index.append((arr_idx,i))\n\n print(f\" Contracts used : {n_contracts:,}\")\n print(f\" Contracts skipped : {skipped:,} \"\n f\"(< {seq_len + 1} observations)\")\n print(f\" Total sequences : {len(self.index):,}\")\n \n def __len__(self):\n return len(self.index)\n\n def __getitem__(self,idx):\n arr_idx, start=self.index[idx]\n X,y=self.contract_data[arr_idx]\n x_seq=torch.tensor(X[start:start+self.seq_len],dtype=torch.float32)\n label=torch.tensor(y[start+self.seq_len],dtype=torch.float32)\n return x_seq,label\n \n \n \n ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:27.197925Z","iopub.execute_input":"2026-07-01T09:11:27.198202Z","iopub.status.idle":"2026-07-01T09:11:27.212473Z","shell.execute_reply.started":"2026-07-01T09:11:27.198174Z","shell.execute_reply":"2026-07-01T09:11:27.211841Z"}},"outputs":[],"execution_count":42},{"cell_type":"markdown","source":"# Now bulding dataset","metadata":{}},{"cell_type":"code","source":"train_dataset=LazyContractDataset(train_df,features=FEATURES, target=TARGET, seq_len=SEQ_LEN)\ntest_dataset=LazyContractDataset(test_df,features=FEATURES, target=TARGET, seq_len=SEQ_LEN)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:27.213278Z","iopub.execute_input":"2026-07-01T09:11:27.213557Z","iopub.status.idle":"2026-07-01T09:11:58.956356Z","shell.execute_reply.started":"2026-07-01T09:11:27.213537Z","shell.execute_reply":"2026-07-01T09:11:58.955650Z"}},"outputs":[{"name":"stdout","text":" Contracts used : 22,162\n Contracts skipped : 1,004 (< 11 observations)\n Total sequences : 1,100,342\n Contracts used : 14,888\n Contracts skipped : 1,062 (< 11 observations)\n Total sequences : 470,870\n","output_type":"stream"}],"execution_count":43},{"cell_type":"code","source":"BATCH_SIZE","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:58.957258Z","iopub.execute_input":"2026-07-01T09:11:58.957673Z","iopub.status.idle":"2026-07-01T09:11:58.962963Z","shell.execute_reply.started":"2026-07-01T09:11:58.957648Z","shell.execute_reply":"2026-07-01T09:11:58.962025Z"}},"outputs":[{"execution_count":44,"output_type":"execute_result","data":{"text/plain":"1024"},"metadata":{}}],"execution_count":44},{"cell_type":"code","source":"train_loader = DataLoader(\n train_dataset,\n batch_size=BATCH_SIZE,\n shuffle=True,\n drop_last=True,\n num_workers=16, #changed from 4\n pin_memory=True,\n persistent_workers=True,\n prefetch_factor=2,\n)\n\ntest_loader = DataLoader(\n test_dataset,\n batch_size=BATCH_SIZE,\n shuffle=False,\n drop_last=False,\n num_workers=16, #changed from 4\n pin_memory=True,\n persistent_workers=True,\n prefetch_factor=2,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:58.964060Z","iopub.execute_input":"2026-07-01T09:11:58.964427Z","iopub.status.idle":"2026-07-01T09:11:58.978113Z","shell.execute_reply.started":"2026-07-01T09:11:58.964397Z","shell.execute_reply":"2026-07-01T09:11:58.977273Z"}},"outputs":[],"execution_count":45},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Buidling the model","metadata":{}},{"cell_type":"code","source":"class OptionPricingLSTM(nn.Module):\n def __init__(self,input_size,hidden_size=128, num_layers=2,dropout=0.3):\n super().__init__()\n self.lstm=nn.LSTM(input_size=input_size,\n hidden_size=hidden_size,\n num_layers=num_layers,\n batch_first=True,\n dropout=dropout if num_layers>1 else 0.0)\n\n self.head=nn.Sequential(nn.Dropout(dropout),\n nn.Linear(hidden_size,64),\n nn.ReLU(),\n nn.Dropout(0.2),\n nn.Linear(64,1))\n\n def forward(self,x):\n lstm_out,_=self.lstm(x)\n last_step = lstm_out[:, -1, :]\n return self.head(last_step).squeeze(-1)\n ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:58.979154Z","iopub.execute_input":"2026-07-01T09:11:58.979485Z","iopub.status.idle":"2026-07-01T09:11:58.997732Z","shell.execute_reply.started":"2026-07-01T09:11:58.979458Z","shell.execute_reply":"2026-07-01T09:11:58.997111Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":46},{"cell_type":"code","source":"model=OptionPricingLSTM(input_size=len(FEATURES))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:58.998505Z","iopub.execute_input":"2026-07-01T09:11:58.998682Z","iopub.status.idle":"2026-07-01T09:11:59.058498Z","shell.execute_reply.started":"2026-07-01T09:11:58.998665Z","shell.execute_reply":"2026-07-01T09:11:59.057667Z"}},"outputs":[],"execution_count":47},{"cell_type":"code","source":"if torch.cuda.device_count()>1:\n print(torch.cuda.device_count())\n model=nn.DataParallel(model)\nelse:\n pass","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:59.059557Z","iopub.execute_input":"2026-07-01T09:11:59.059789Z","iopub.status.idle":"2026-07-01T09:11:59.084809Z","shell.execute_reply.started":"2026-07-01T09:11:59.059769Z","shell.execute_reply":"2026-07-01T09:11:59.084271Z"}},"outputs":[{"name":"stdout","text":"2\n","output_type":"stream"}],"execution_count":48},{"cell_type":"code","source":"model=model.to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:59.086173Z","iopub.execute_input":"2026-07-01T09:11:59.086884Z","iopub.status.idle":"2026-07-01T09:11:59.561660Z","shell.execute_reply.started":"2026-07-01T09:11:59.086862Z","shell.execute_reply":"2026-07-01T09:11:59.561082Z"}},"outputs":[],"execution_count":49},{"cell_type":"code","source":"EPOCHS=30\nLR=1e-3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:59.562540Z","iopub.execute_input":"2026-07-01T09:11:59.562725Z","iopub.status.idle":"2026-07-01T09:11:59.566593Z","shell.execute_reply.started":"2026-07-01T09:11:59.562706Z","shell.execute_reply":"2026-07-01T09:11:59.565977Z"}},"outputs":[],"execution_count":50},{"cell_type":"code","source":"criterion=nn.MSELoss()\noptimizer=torch.optim.Adam(model.parameters(),lr=LR,weight_decay=1e-5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:11:59.567427Z","iopub.execute_input":"2026-07-01T09:11:59.567960Z","iopub.status.idle":"2026-07-01T09:12:03.393747Z","shell.execute_reply.started":"2026-07-01T09:11:59.567926Z","shell.execute_reply":"2026-07-01T09:12:03.393138Z"}},"outputs":[],"execution_count":51},{"cell_type":"code","source":"scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n optimizer, mode=\"min\", factor=0.5, patience=3\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:12:03.394581Z","iopub.execute_input":"2026-07-01T09:12:03.395022Z","iopub.status.idle":"2026-07-01T09:12:03.398708Z","shell.execute_reply.started":"2026-07-01T09:12:03.395000Z","shell.execute_reply":"2026-07-01T09:12:03.398043Z"}},"outputs":[],"execution_count":52},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"train_losses = []\n\nprint(f'training on for {EPOCHS} epochs')\n\nfor epoch in range(EPOCHS):\n model.train()\n running_loss=0.0\n\n for X_batch, y_batch in train_loader:\n X_batch=X_batch.to(device,non_blocking=True)\n y_batch=y_batch.to(device, non_blocking=True)\n\n optimizer.zero_grad()\n preds=model(X_batch)\n loss=criterion(preds,y_batch)\n loss.backward()\n\n nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n optimizer.step()\n running_loss+=loss.item()\n\n avg_loss=running_loss/len(train_loader)\n current_lr=optimizer.param_groups[0]['lr']\n train_losses.append(avg_loss)\n scheduler.step(avg_loss)\n print(f'epoch:{epoch+1} loss:{avg_loss} LR:{current_lr}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:12:03.399748Z","iopub.execute_input":"2026-07-01T09:12:03.400029Z","iopub.status.idle":"2026-07-01T09:19:36.295278Z","shell.execute_reply.started":"2026-07-01T09:12:03.400009Z","shell.execute_reply":"2026-07-01T09:19:36.294169Z"}},"outputs":[{"name":"stdout","text":"training on for 30 epochs\nepoch:1 loss:0.048502970103597616 LR:0.001\nepoch:2 loss:0.026281521457908207 LR:0.001\nepoch:3 loss:0.024316823276477818 LR:0.001\nepoch:4 loss:0.02303953826673731 LR:0.001\nepoch:5 loss:0.022242416079664252 LR:0.001\nepoch:6 loss:0.02156428411235301 LR:0.001\nepoch:7 loss:0.02114208217786566 LR:0.001\nepoch:8 loss:0.020585036788871112 LR:0.001\nepoch:9 loss:0.020257482338120927 LR:0.001\nepoch:10 loss:0.020130864922928465 LR:0.001\nepoch:11 loss:0.0198568290956249 LR:0.001\nepoch:12 loss:0.019544233834228757 LR:0.001\nepoch:13 loss:0.019107741494515645 LR:0.001\nepoch:14 loss:0.018688273625673806 LR:0.001\nepoch:15 loss:0.018318067509438183 LR:0.001\nepoch:16 loss:0.017768758669967313 LR:0.001\nepoch:17 loss:0.01752916913222889 LR:0.001\nepoch:18 loss:0.017342516358116265 LR:0.001\nepoch:19 loss:0.017312257208131267 LR:0.001\nepoch:20 loss:0.01715195822846873 LR:0.001\nepoch:21 loss:0.01668822647417502 LR:0.001\nepoch:22 loss:0.01678514221572149 LR:0.001\nepoch:23 loss:0.016817185014930295 LR:0.001\nepoch:24 loss:0.016443053408728796 LR:0.001\nepoch:25 loss:0.016472981741488868 LR:0.001\nepoch:26 loss:0.01655489213771605 LR:0.001\nepoch:27 loss:0.016573218859426912 LR:0.001\nepoch:28 loss:0.016115535612545865 LR:0.001\nepoch:29 loss:0.016157236781831504 LR:0.001\nepoch:30 loss:0.016155129060100634 LR:0.001\n","output_type":"stream"}],"execution_count":53},{"cell_type":"markdown","source":"# Testing","metadata":{}},{"cell_type":"code","source":"model.eval()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:19:36.296941Z","iopub.execute_input":"2026-07-01T09:19:36.297360Z","iopub.status.idle":"2026-07-01T09:19:36.304585Z","shell.execute_reply.started":"2026-07-01T09:19:36.297324Z","shell.execute_reply":"2026-07-01T09:19:36.303755Z"}},"outputs":[{"execution_count":54,"output_type":"execute_result","data":{"text/plain":"DataParallel(\n (module): OptionPricingLSTM(\n (lstm): LSTM(12, 128, num_layers=2, batch_first=True, dropout=0.3)\n (head): Sequential(\n (0): Dropout(p=0.3, inplace=False)\n (1): Linear(in_features=128, out_features=64, bias=True)\n (2): ReLU()\n (3): Dropout(p=0.2, inplace=False)\n (4): Linear(in_features=64, out_features=1, bias=True)\n )\n )\n)"},"metadata":{}}],"execution_count":54},{"cell_type":"code","source":"all_preds=[]\nall_origs=[]\nwith torch.no_grad():\n for X_batch, y_batch in test_loader:\n X_batch=X_batch.to(device, non_blocking=True)\n batch_pred=model(X_batch).cpu().numpy()\n all_preds.extend(batch_pred)\n all_origs.extend(y_batch)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:19:36.305553Z","iopub.execute_input":"2026-07-01T09:19:36.306327Z","iopub.status.idle":"2026-07-01T09:19:45.260537Z","shell.execute_reply.started":"2026-07-01T09:19:36.306298Z","shell.execute_reply":"2026-07-01T09:19:45.259697Z"}},"outputs":[],"execution_count":55},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:19:45.262039Z","iopub.execute_input":"2026-07-01T09:19:45.262764Z","iopub.status.idle":"2026-07-01T09:19:45.267173Z","shell.execute_reply.started":"2026-07-01T09:19:45.262732Z","shell.execute_reply":"2026-07-01T09:19:45.266555Z"}},"outputs":[],"execution_count":56},{"cell_type":"code","source":"r2_score(all_origs,all_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:19:45.268363Z","iopub.execute_input":"2026-07-01T09:19:45.268919Z","iopub.status.idle":"2026-07-01T09:19:46.743833Z","shell.execute_reply.started":"2026-07-01T09:19:45.268897Z","shell.execute_reply":"2026-07-01T09:19:46.743262Z"}},"outputs":[{"execution_count":57,"output_type":"execute_result","data":{"text/plain":"0.9878501672848014"},"metadata":{}}],"execution_count":57},{"cell_type":"code","source":"mean_squared_error(all_origs,all_preds), mean_absolute_error(all_origs,all_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:19:46.744830Z","iopub.execute_input":"2026-07-01T09:19:46.745122Z","iopub.status.idle":"2026-07-01T09:19:49.573035Z","shell.execute_reply.started":"2026-07-01T09:19:46.745092Z","shell.execute_reply":"2026-07-01T09:19:49.572148Z"}},"outputs":[{"execution_count":58,"output_type":"execute_result","data":{"text/plain":"(0.016765453428708076, 0.06724213238929869)"},"metadata":{}}],"execution_count":58},{"cell_type":"code","source":"unscaled_origs=target_scaler.inverse_transform(np.array(all_origs).reshape(-1,1)).ravel()\nunscaled_preds=target_scaler.inverse_transform(np.array(all_preds).reshape(-1,1)).ravel()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:20:26.826962Z","iopub.execute_input":"2026-07-01T09:20:26.827231Z","iopub.status.idle":"2026-07-01T09:20:28.654365Z","shell.execute_reply.started":"2026-07-01T09:20:26.827189Z","shell.execute_reply":"2026-07-01T09:20:28.653286Z"}},"outputs":[],"execution_count":60},{"cell_type":"code","source":"print(f'R2 score: {r2_score(unscaled_origs, unscaled_preds)}')\nprint(f'Mean Sqaured Error: {mean_squared_error(unscaled_origs,unscaled_preds)}')\nprint(f'Root Mean Squared Error: {np.sqrt(mean_squared_error(unscaled_origs,unscaled_preds))}')\nprint(f'Mean Absolute Error: {mean_absolute_error(unscaled_origs,unscaled_preds)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:22:12.308523Z","iopub.execute_input":"2026-07-01T09:22:12.309353Z","iopub.status.idle":"2026-07-01T09:22:12.321092Z","shell.execute_reply.started":"2026-07-01T09:22:12.309323Z","shell.execute_reply":"2026-07-01T09:22:12.320406Z"}},"outputs":[{"name":"stdout","text":"R2 score: 0.9878501892089844\nMean Sqaured Error: 99.01111602783203\nRoot Mean Squared Error: 9.950432956803038\nMean Absolute Error: 5.167444229125977\n","output_type":"stream"}],"execution_count":63},{"cell_type":"code","source":"model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:22:54.902824Z","iopub.execute_input":"2026-07-01T09:22:54.903557Z","iopub.status.idle":"2026-07-01T09:22:54.910024Z","shell.execute_reply.started":"2026-07-01T09:22:54.903528Z","shell.execute_reply":"2026-07-01T09:22:54.908987Z"}},"outputs":[{"execution_count":64,"output_type":"execute_result","data":{"text/plain":"DataParallel(\n (module): OptionPricingLSTM(\n (lstm): LSTM(12, 128, num_layers=2, batch_first=True, dropout=0.3)\n (head): Sequential(\n (0): Dropout(p=0.3, inplace=False)\n (1): Linear(in_features=128, out_features=64, bias=True)\n (2): ReLU()\n (3): Dropout(p=0.2, inplace=False)\n (4): Linear(in_features=64, out_features=1, bias=True)\n )\n )\n)"},"metadata":{}}],"execution_count":64},{"cell_type":"code","source":"len(FEATURES)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:24:37.022292Z","iopub.execute_input":"2026-07-01T09:24:37.022991Z","iopub.status.idle":"2026-07-01T09:24:37.027855Z","shell.execute_reply.started":"2026-07-01T09:24:37.022961Z","shell.execute_reply":"2026-07-01T09:24:37.027097Z"}},"outputs":[{"execution_count":65,"output_type":"execute_result","data":{"text/plain":"12"},"metadata":{}}],"execution_count":65},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:24:52.244151Z","iopub.execute_input":"2026-07-01T09:24:52.244746Z","iopub.status.idle":"2026-07-01T09:24:57.313765Z","shell.execute_reply.started":"2026-07-01T09:24:52.244718Z","shell.execute_reply":"2026-07-01T09:24:57.312525Z"}},"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)","\u001b[0;32m/tmp/ipykernel_58/1718741065.py\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 60\u001b[0m \u001b[0;31m# Assuming X_test_tensor already exists in your notebook\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[0;31m# If not, adjust this line to match your variable name:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 62\u001b[0;31m \u001b[0mX_test_cpu\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX_test_tensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[0;31m# Background dataset: random sample of 200 rows from test set\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mNameError\u001b[0m: name 'X_test_tensor' is not defined"],"ename":"NameError","evalue":"name 'X_test_tensor' is not defined","output_type":"error"}],"execution_count":66},{"cell_type":"code","source":"'''\n# ── 9. EVALUATION ─────────────────────────────────────────────\nprint(\"[8] Evaluating on test set ...\")\n\nmodel.eval()\nall_preds = []\nall_labels = []\n\nwith torch.no_grad():\n for X_batch, y_batch in test_loader:\n X_batch = X_batch.to(device, non_blocking=True)\n preds = model(X_batch).cpu().numpy()\n all_preds.append(preds)\n all_labels.append(y_batch.numpy())\n\nall_preds = np.concatenate(all_preds)\nall_labels = np.concatenate(all_labels)\n\n# Inverse-transform back to original dollar price scale\npreds_orig = target_scaler.inverse_transform(\n all_preds.reshape(-1, 1)).ravel()\nlabels_orig = target_scaler.inverse_transform(\n all_labels.reshape(-1, 1)).ravel()\n\nmse = mean_squared_error(labels_orig, preds_orig)\nrmse = np.sqrt(mse)\nmae = mean_absolute_error(labels_orig, preds_orig)\nr2 = r2_score(labels_orig, preds_orig)\n\nprint(\"\\n\" + \"=\"*42)\nprint(\" TEST SET RESULTS (original price scale)\")\nprint(\"=\"*42)\nprint(f\" MSE : {mse:.4f}\")\nprint(f\" RMSE : {rmse:.4f} ← in same units as mark price\")\nprint(f\" MAE : {mae:.4f} ← avg absolute error per contract\")\nprint(f\" R² : {r2:.4f} ← 1.0 = perfect, 0.0 = baseline mean\")\nprint(\"=\"*42)\n\n\n# ── 10. PLOTS ─────────────────────────────────────────────────\nfig, axes = plt.subplots(1, 3, figsize=(18, 5))\nfig.suptitle(\"LSTM Option Pricing — SPY 2024 (per-contract sequences)\",\n fontsize=14, fontweight=\"bold\")\n\naxes[0].plot(range(1, EPOCHS + 1), train_losses,\n color=\"#4C72B0\", linewidth=2)\naxes[0].set_title(\"Training Loss (MSE) per Epoch\")\naxes[0].set_xlabel(\"Epoch\")\naxes[0].set_ylabel(\"MSE Loss (normalised scale)\")\naxes[0].grid(True, alpha=0.3)\n\nn_plot = min(3000, len(preds_orig))\nidx = np.random.choice(len(preds_orig), n_plot, replace=False)\naxes[1].scatter(labels_orig[idx], preds_orig[idx],\n alpha=0.25, s=6, color=\"#4C72B0\")\nlims = [min(labels_orig.min(), preds_orig.min()),\n max(labels_orig.max(), preds_orig.max())]\naxes[1].plot(lims, lims, \"r--\", linewidth=1.5, label=\"Perfect fit\")\naxes[1].set_title(f\"Predicted vs Actual (R²={r2:.4f})\")\naxes[1].set_xlabel(\"Actual mark price ($)\")\naxes[1].set_ylabel(\"Predicted mark price ($)\")\naxes[1].legend(fontsize=9)\naxes[1].grid(True, alpha=0.3)\n\nresiduals = preds_orig - labels_orig\naxes[2].hist(residuals, bins=80, color=\"#4C72B0\",\n edgecolor=\"white\", linewidth=0.2)\naxes[2].axvline(0, color=\"red\", linestyle=\"--\", linewidth=1.5,\n label=\"Zero error\")\naxes[2].set_title(f\"Residuals (MAE={mae:.4f})\")\naxes[2].set_xlabel(\"Prediction error (pred − actual) ($)\")\naxes[2].set_ylabel(\"Count\")\naxes[2].legend(fontsize=9)\naxes[2].grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.savefig(\"lstm_evaluation.png\", dpi=150, bbox_inches=\"tight\")\nplt.show()\nprint(\"\\nPlot saved → lstm_evaluation.png\")\n\n\n# ── 11. SAVE MODEL & SCALERS ──────────────────────────────────\n# Unwrap DataParallel before saving so the checkpoint loads\n# cleanly on any machine regardless of GPU count\nmodel_to_save = model.module if isinstance(model, nn.DataParallel) \\\n else model\n\ntorch.save({\n \"model_state_dict\": model_to_save.state_dict(),\n \"feat_scaler\": feat_scaler,\n \"target_scaler\": target_scaler,\n \"features\": FEATURES,\n \"seq_len\": SEQ_LEN,\n \"hidden_size\": 128,\n \"num_layers\": 2,\n \"metrics\": {\n \"mse\": mse,\n \"rmse\": rmse,\n \"mae\": mae,\n \"r2\": r2,\n },\n}, \"lstm_option_pricing.pt\")\n\nprint(\"Model saved → lstm_option_pricing.pt\")\nprint(\"\\nDone!\")\n'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-01T09:19:49.585660Z","iopub.status.idle":"2026-07-01T09:19:49.585951Z","shell.execute_reply.started":"2026-07-01T09:19:49.585836Z","shell.execute_reply":"2026-07-01T09:19:49.585851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}
|