muthuk1 commited on
Commit
03a64b9
·
verified ·
1 Parent(s): 7bf0a17

Upload folder using huggingface_hub

Browse files
e88186124ec611f1/dataset/sample_submission.csv CHANGED
@@ -1,6 +1,3 @@
1
- Index,demand
2
- 0,0.090767774072159
3
- 1,0.0898848048932868
4
- 2,0.0070368435076525
5
- 3,0.0790872273785951
6
- 4,0.0546359836658569
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:512c0d62e334d2ca8e4e49f7b7bbcce7abbf681388fb81fe2a18e5f8599fb4ac
3
+ size 117
 
 
 
e88186124ec611f1/dataset/test.csv CHANGED
The diff for this file is too large to render. See raw diff
 
e88186124ec611f1/dataset/train.csv CHANGED
The diff for this file is too large to render. See raw diff
 
gridlock_solution.ipynb CHANGED
@@ -1,448 +1,3 @@
1
- {
2
- "nbformat": 4,
3
- "nbformat_minor": 0,
4
- "metadata": {
5
- "colab": {
6
- "provenance": []
7
- },
8
- "kernelspec": {
9
- "display_name": "Python 3",
10
- "name": "python3"
11
- },
12
- "language_info": {
13
- "name": "python"
14
- }
15
- },
16
- "cells": [
17
- {
18
- "cell_type": "markdown",
19
- "metadata": {},
20
- "source": [
21
- "# Gridlock Hackathon 2.0 — Traffic Demand Prediction\n",
22
- "\n",
23
- "**Approach:** LightGBM + XGBoost ensemble with temporal lag features.\n",
24
- "\n",
25
- "**Key features:**\n",
26
- "- Geohash → lat/lon\n",
27
- "- Cyclical time encoding (sin/cos of hour)\n",
28
- "- **Lag feature:** same geohash + same timestamp from previous day (day 48)\n",
29
- "- Per-geohash aggregations: mean, std, median, max demand\n",
30
- "- Per-geohash per-hour mean demand\n",
31
- "- Neighbourhood (geo3-prefix) demand aggregations\n",
32
- "- Temperature (imputed via geohash+hour mean where missing)\n",
33
- "- Road features: RoadType, NumberofLanes, LargeVehicles, Landmarks, Weather\n",
34
- "\n",
35
- "**Evaluation:** `score = max(0, 100 * R2(actual, predicted))`"
36
- ]
37
- },
38
- {
39
- "cell_type": "code",
40
- "execution_count": null,
41
- "metadata": {},
42
- "outputs": [],
43
- "source": [
44
- "# Install dependencies (Colab)\n",
45
- "!pip install pygeohash lightgbm xgboost -q"
46
- ]
47
- },
48
- {
49
- "cell_type": "code",
50
- "execution_count": null,
51
- "metadata": {},
52
- "outputs": [],
53
- "source": [
54
- "import numpy as np\n",
55
- "import pandas as pd\n",
56
- "import pygeohash as pgh\n",
57
- "from lightgbm import LGBMRegressor\n",
58
- "from xgboost import XGBRegressor\n",
59
- "from sklearn.metrics import r2_score\n",
60
- "import warnings\n",
61
- "warnings.filterwarnings('ignore')\n",
62
- "\n",
63
- "print('Libraries loaded.')"
64
- ]
65
- },
66
- {
67
- "cell_type": "markdown",
68
- "metadata": {},
69
- "source": [
70
- "## 1. Load Data\n",
71
- "\n",
72
- "Upload `train.csv` and `test.csv` to Colab or mount Google Drive."
73
- ]
74
- },
75
- {
76
- "cell_type": "code",
77
- "execution_count": null,
78
- "metadata": {},
79
- "outputs": [],
80
- "source": [
81
- "# Option A: upload files manually\n",
82
- "# from google.colab import files\n",
83
- "# uploaded = files.upload() # upload train.csv and test.csv\n",
84
- "\n",
85
- "# Option B: mount Drive\n",
86
- "# from google.colab import drive\n",
87
- "# drive.mount('/content/drive')\n",
88
- "# DATA_PATH = '/content/drive/MyDrive/gridlock/'\n",
89
- "\n",
90
- "# Option C: local paths (if running locally)\n",
91
- "DATA_PATH = 'e88186124ec611f1/dataset/'\n",
92
- "\n",
93
- "train = pd.read_csv(DATA_PATH + 'train.csv')\n",
94
- "test = pd.read_csv(DATA_PATH + 'test.csv')\n",
95
- "\n",
96
- "print(f'Train: {train.shape}, Test: {test.shape}')\n",
97
- "print('Train days:', sorted(train.day.unique()))\n",
98
- "print('Test days:', sorted(test.day.unique()))\n",
99
- "train.head()"
100
- ]
101
- },
102
- {
103
- "cell_type": "markdown",
104
- "metadata": {},
105
- "source": [
106
- "## 2. Feature Engineering"
107
- ]
108
- },
109
- {
110
- "cell_type": "code",
111
- "execution_count": null,
112
- "metadata": {},
113
- "outputs": [],
114
- "source": [
115
- "def parse_timestamp(df):\n",
116
- " df = df.copy()\n",
117
- " df['hour'] = df['timestamp'].map(lambda x: int(x.split(':')[0]))\n",
118
- " df['minute'] = df['timestamp'].map(lambda x: int(x.split(':')[1]))\n",
119
- " df['time_min'] = df['day'] * 24 * 60 + df['hour'] * 60 + df['minute']\n",
120
- " # Cyclical daily time\n",
121
- " t = (df['hour'] * 60 + df['minute']) / (24 * 60) * 2 * np.pi\n",
122
- " df['time_sin'] = np.sin(t)\n",
123
- " df['time_cos'] = np.cos(t)\n",
124
- " # Cyclical day\n",
125
- " d = df['day'] / 7 * 2 * np.pi\n",
126
- " df['day_sin'] = np.sin(d)\n",
127
- " df['day_cos'] = np.cos(d)\n",
128
- " return df\n",
129
- "\n",
130
- "\n",
131
- "def decode_geohash(df):\n",
132
- " df = df.copy()\n",
133
- " decoded = df['geohash'].map(pgh.decode)\n",
134
- " df['lat'] = decoded.map(lambda x: x[0])\n",
135
- " df['lon'] = decoded.map(lambda x: x[1])\n",
136
- " df['geo3'] = df['geohash'].str[:3]\n",
137
- " df['geo4'] = df['geohash'].str[:4]\n",
138
- " return df\n",
139
- "\n",
140
- "\n",
141
- "def encode_categoricals(df):\n",
142
- " df = df.copy()\n",
143
- " df['RoadType_enc'] = df['RoadType'].map({'Residential': 0, 'Street': 1, 'Highway': 2}).fillna(-1)\n",
144
- " df['LargeVehicles_enc'] = (df['LargeVehicles'] == 'Allowed').astype(float)\n",
145
- " df['Landmarks_enc'] = (df['Landmarks'] == 'Yes').astype(float)\n",
146
- " df['Weather_enc'] = df['Weather'].map({'Sunny': 0, 'Rainy': 1, 'Foggy': 2, 'Snowy': 3}).fillna(-1)\n",
147
- " return df\n",
148
- "\n",
149
- "\n",
150
- "train = parse_timestamp(train)\n",
151
- "train = decode_geohash(train)\n",
152
- "train = encode_categoricals(train)\n",
153
- "\n",
154
- "test = parse_timestamp(test)\n",
155
- "test = decode_geohash(test)\n",
156
- "test = encode_categoricals(test)\n",
157
- "\n",
158
- "print('Basic features done.')"
159
- ]
160
- },
161
- {
162
- "cell_type": "code",
163
- "execution_count": null,
164
- "metadata": {},
165
- "outputs": [],
166
- "source": [
167
- "# ── Temperature imputation ────────────────────────────────────────────────────\n",
168
- "# Use geohash+hour mean from training data\n",
169
- "geo_hour_temp = (train.dropna(subset=['Temperature'])\n",
170
- " .groupby(['geohash', 'hour'])['Temperature']\n",
171
- " .mean().reset_index()\n",
172
- " .rename(columns={'Temperature': 'temp_impute'}))\n",
173
- "\n",
174
- "day_temp_mean = train.groupby('day')['Temperature'].mean()\n",
175
- "\n",
176
- "def impute_temp(df):\n",
177
- " df = df.merge(geo_hour_temp, on=['geohash', 'hour'], how='left')\n",
178
- " df['Temperature'] = df['Temperature'].fillna(df['temp_impute'])\n",
179
- " df['Temperature'] = df['Temperature'].fillna(day_temp_mean.mean())\n",
180
- " df = df.drop(columns=['temp_impute'])\n",
181
- " return df\n",
182
- "\n",
183
- "train = impute_temp(train)\n",
184
- "test = impute_temp(test)\n",
185
- "print(f'Temp NaN remaining - train: {train.Temperature.isna().sum()}, test: {test.Temperature.isna().sum()}')"
186
- ]
187
- },
188
- {
189
- "cell_type": "code",
190
- "execution_count": null,
191
- "metadata": {},
192
- "outputs": [],
193
- "source": [
194
- "# ── Lag feature: same geohash + same timestamp, 1 day earlier ────────────────\n",
195
- "train48 = (train[train['day'] == 48][['geohash', 'timestamp', 'demand']]\n",
196
- " .rename(columns={'demand': 'demand_lag1d'}))\n",
197
- "\n",
198
- "train = train.merge(train48, on=['geohash', 'timestamp'], how='left')\n",
199
- "test = test.merge(train48, on=['geohash', 'timestamp'], how='left')\n",
200
- "\n",
201
- "print(f'Lag1d coverage - train: {train.demand_lag1d.notna().sum()}/{len(train)}, '\n",
202
- " f'test: {test.demand_lag1d.notna().sum()}/{len(test)}')"
203
- ]
204
- },
205
- {
206
- "cell_type": "code",
207
- "execution_count": null,
208
- "metadata": {},
209
- "outputs": [],
210
- "source": [
211
- "# ── Aggregation features (from all train data) ───────────────────────────────\n",
212
- "\n",
213
- "# geohash statistics\n",
214
- "geo_stats = (train.groupby('geohash')['demand']\n",
215
- " .agg(['mean','std','median','max']).reset_index()\n",
216
- " .rename(columns={'mean':'geo_mean','std':'geo_std',\n",
217
- " 'median':'geo_median','max':'geo_max'}))\n",
218
- "geo_stats['geo_std'] = geo_stats['geo_std'].fillna(0)\n",
219
- "\n",
220
- "# geohash + hour mean\n",
221
- "geo_hour_demand = (train.groupby(['geohash','hour'])['demand']\n",
222
- " .mean().reset_index()\n",
223
- " .rename(columns={'demand':'geo_hour_mean'}))\n",
224
- "\n",
225
- "# geohash + exact timestamp mean (captures recurring patterns)\n",
226
- "geo_ts_mean = (train.groupby(['geohash','timestamp'])['demand']\n",
227
- " .mean().reset_index()\n",
228
- " .rename(columns={'demand':'geo_ts_mean'}))\n",
229
- "\n",
230
- "# geo3 neighbourhood + hour mean\n",
231
- "geo3_hour = (train.groupby(['geo3','hour'])['demand']\n",
232
- " .mean().reset_index()\n",
233
- " .rename(columns={'demand':'geo3_hour_mean'}))\n",
234
- "\n",
235
- "# geo3 overall mean\n",
236
- "geo3_mean = (train.groupby('geo3')['demand']\n",
237
- " .mean().reset_index()\n",
238
- " .rename(columns={'demand':'geo3_mean'}))\n",
239
- "\n",
240
- "# Hour-of-day global mean (captures daily rhythm)\n",
241
- "hour_mean = (train.groupby('hour')['demand']\n",
242
- " .mean().reset_index()\n",
243
- " .rename(columns={'demand':'hour_global_mean'}))\n",
244
- "\n",
245
- "def apply_aggs(df):\n",
246
- " df = df.merge(geo_stats, on='geohash', how='left')\n",
247
- " df = df.merge(geo_hour_demand, on=['geohash','hour'], how='left')\n",
248
- " df = df.merge(geo_ts_mean, on=['geohash','timestamp'], how='left')\n",
249
- " df = df.merge(geo3_hour, on=['geo3','hour'], how='left')\n",
250
- " df = df.merge(geo3_mean, on='geo3', how='left')\n",
251
- " df = df.merge(hour_mean, on='hour', how='left')\n",
252
- " # Impute missing lag1d using geo_ts_mean\n",
253
- " df['demand_lag1d'] = df['demand_lag1d'].fillna(df['geo_ts_mean'])\n",
254
- " return df\n",
255
- "\n",
256
- "train = apply_aggs(train)\n",
257
- "test = apply_aggs(test)\n",
258
- "\n",
259
- "print('Aggregation features done.')\n",
260
- "print(f'Train features: {train.shape[1]}')"
261
- ]
262
- },
263
- {
264
- "cell_type": "markdown",
265
- "metadata": {},
266
- "source": [
267
- "## 3. Model Training"
268
- ]
269
- },
270
- {
271
- "cell_type": "code",
272
- "execution_count": null,
273
- "metadata": {},
274
- "outputs": [],
275
- "source": [
276
- "FEATURES = [\n",
277
- " 'lat', 'lon',\n",
278
- " 'hour', 'minute', 'day',\n",
279
- " 'time_sin', 'time_cos', 'day_sin', 'day_cos',\n",
280
- " 'RoadType_enc', 'NumberofLanes', 'LargeVehicles_enc', 'Landmarks_enc',\n",
281
- " 'Temperature', 'Weather_enc',\n",
282
- " 'demand_lag1d',\n",
283
- " 'geo_mean', 'geo_std', 'geo_median', 'geo_max',\n",
284
- " 'geo_hour_mean', 'geo_ts_mean', 'geo3_hour_mean', 'geo3_mean',\n",
285
- " 'hour_global_mean',\n",
286
- "]\n",
287
- "\n",
288
- "X = train[FEATURES].fillna(-1)\n",
289
- "y = train['demand']\n",
290
- "X_test = test[FEATURES].fillna(-1)\n",
291
- "\n",
292
- "# Cross-val: train on day48, validate on day49\n",
293
- "mask49 = train['day'] == 49\n",
294
- "X_tr, y_tr = X[~mask49], y[~mask49]\n",
295
- "X_va, y_va = X[mask49], y[mask49]\n",
296
- "\n",
297
- "print(f'CV split — train: {X_tr.shape}, val: {X_va.shape}')"
298
- ]
299
- },
300
- {
301
- "cell_type": "code",
302
- "execution_count": null,
303
- "metadata": {},
304
- "outputs": [],
305
- "source": [
306
- "# ── LightGBM ──────────────────────────────────────────────────────────────────\n",
307
- "lgbm_params = dict(\n",
308
- " n_estimators=5000,\n",
309
- " learning_rate=0.015,\n",
310
- " num_leaves=255,\n",
311
- " min_child_samples=15,\n",
312
- " subsample=0.8, subsample_freq=1,\n",
313
- " colsample_bytree=0.8,\n",
314
- " reg_alpha=0.05, reg_lambda=0.1,\n",
315
- " random_state=42, verbose=-1, n_jobs=-1,\n",
316
- ")\n",
317
- "\n",
318
- "lgbm_cv = LGBMRegressor(**lgbm_params)\n",
319
- "lgbm_cv.fit(X_tr, y_tr)\n",
320
- "lgbm_cv_pred = lgbm_cv.predict(X_va)\n",
321
- "lgbm_r2 = r2_score(y_va, lgbm_cv_pred)\n",
322
- "print(f'LightGBM CV R2: {lgbm_r2:.4f} score: {max(0,100*lgbm_r2):.2f}')"
323
- ]
324
- },
325
- {
326
- "cell_type": "code",
327
- "execution_count": null,
328
- "metadata": {},
329
- "outputs": [],
330
- "source": [
331
- "# ── XGBoost ───────────────────────────────────────────────────────────────────\n",
332
- "xgb_params = dict(\n",
333
- " n_estimators=5000,\n",
334
- " learning_rate=0.015,\n",
335
- " max_depth=8,\n",
336
- " subsample=0.8,\n",
337
- " colsample_bytree=0.8,\n",
338
- " reg_alpha=0.05, reg_lambda=0.1,\n",
339
- " random_state=42, verbosity=0, n_jobs=-1,\n",
340
- " tree_method='hist',\n",
341
- ")\n",
342
- "\n",
343
- "xgb_cv = XGBRegressor(**xgb_params)\n",
344
- "xgb_cv.fit(X_tr, y_tr)\n",
345
- "xgb_cv_pred = xgb_cv.predict(X_va)\n",
346
- "xgb_r2 = r2_score(y_va, xgb_cv_pred)\n",
347
- "print(f'XGBoost CV R2: {xgb_r2:.4f} score: {max(0,100*xgb_r2):.2f}')"
348
- ]
349
- },
350
- {
351
- "cell_type": "code",
352
- "execution_count": null,
353
- "metadata": {},
354
- "outputs": [],
355
- "source": [
356
- "# ── Ensemble: find best blend weight ─────────────────────────────────────────\n",
357
- "best_w, best_r2 = 0, -999\n",
358
- "for w in np.arange(0, 1.05, 0.05):\n",
359
- " blend = w * lgbm_cv_pred + (1 - w) * xgb_cv_pred\n",
360
- " r2 = r2_score(y_va, blend)\n",
361
- " if r2 > best_r2:\n",
362
- " best_r2, best_w = r2, w\n",
363
- "\n",
364
- "print(f'Best blend: {best_w:.2f} * LightGBM + {1-best_w:.2f} * XGBoost')\n",
365
- "print(f'Ensemble CV R2: {best_r2:.4f} score: {max(0,100*best_r2):.2f}')"
366
- ]
367
- },
368
- {
369
- "cell_type": "code",
370
- "execution_count": null,
371
- "metadata": {},
372
- "outputs": [],
373
- "source": [
374
- "# ── Train final models on ALL data ────────────────────────────────────────────\n",
375
- "print('Training final LightGBM...')\n",
376
- "lgbm_final = LGBMRegressor(**lgbm_params)\n",
377
- "lgbm_final.fit(X, y)\n",
378
- "\n",
379
- "print('Training final XGBoost...')\n",
380
- "xgb_final = XGBRegressor(**xgb_params)\n",
381
- "xgb_final.fit(X, y)\n",
382
- "\n",
383
- "# Blend predictions\n",
384
- "lgbm_test_pred = lgbm_final.predict(X_test)\n",
385
- "xgb_test_pred = xgb_final.predict(X_test)\n",
386
- "final_preds = np.clip(\n",
387
- " best_w * lgbm_test_pred + (1 - best_w) * xgb_test_pred,\n",
388
- " 0, None\n",
389
- ")\n",
390
- "\n",
391
- "print('Done.')"
392
- ]
393
- },
394
- {
395
- "cell_type": "markdown",
396
- "metadata": {},
397
- "source": [
398
- "## 4. Generate Submission"
399
- ]
400
- },
401
- {
402
- "cell_type": "code",
403
- "execution_count": null,
404
- "metadata": {},
405
- "outputs": [],
406
- "source": [
407
- "submission = pd.DataFrame({'Index': test['Index'], 'demand': final_preds})\n",
408
- "submission.to_csv('submission.csv', index=False)\n",
409
- "print(f'Saved submission.csv shape: {submission.shape}')\n",
410
- "submission.head(10)"
411
- ]
412
- },
413
- {
414
- "cell_type": "code",
415
- "execution_count": null,
416
- "metadata": {},
417
- "outputs": [],
418
- "source": [
419
- "# Download from Colab\n",
420
- "# from google.colab import files\n",
421
- "# files.download('submission.csv')"
422
- ]
423
- },
424
- {
425
- "cell_type": "markdown",
426
- "metadata": {},
427
- "source": [
428
- "## 5. Feature Importance"
429
- ]
430
- },
431
- {
432
- "cell_type": "code",
433
- "execution_count": null,
434
- "metadata": {},
435
- "outputs": [],
436
- "source": [
437
- "import matplotlib.pyplot as plt\n",
438
- "\n",
439
- "fi = pd.Series(lgbm_final.feature_importances_, index=FEATURES).sort_values(ascending=True)\n",
440
- "fig, ax = plt.subplots(figsize=(8, 8))\n",
441
- "fi.plot.barh(ax=ax)\n",
442
- "ax.set_title('LightGBM Feature Importances')\n",
443
- "plt.tight_layout()\n",
444
- "plt.show()"
445
- ]
446
- }
447
- ]
448
- }
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:76de0010cf7e8b8130c2bdb26ef3d56ff10af4691be83f027bbe78de8d6a87d7
3
+ size 15595
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
solve.py CHANGED
@@ -2,6 +2,7 @@ import numpy as np
2
  import pandas as pd
3
  import pygeohash as pgh
4
  from lightgbm import LGBMRegressor
 
5
  from sklearn.metrics import r2_score
6
  import warnings
7
  warnings.filterwarnings('ignore')
@@ -12,16 +13,14 @@ print("Loading data...")
12
  train = pd.read_csv(f'{DATA}/train.csv')
13
  test = pd.read_csv(f'{DATA}/test.csv')
14
 
 
15
  def parse_ts(df):
16
  df = df.copy()
17
- df['hour'] = df['timestamp'].map(lambda x: int(x.split(':')[0]))
18
  df['minute'] = df['timestamp'].map(lambda x: int(x.split(':')[1]))
19
- df['time_min'] = df['day'] * 24 * 60 + df['hour'] * 60 + df['minute']
20
- mins_in_day = 24 * 60
21
- t = (df['hour'] * 60 + df['minute']) / mins_in_day * 2 * np.pi
22
  df['time_sin'] = np.sin(t)
23
  df['time_cos'] = np.cos(t)
24
- # day of week proxy (cyclical)
25
  d = df['day'] / 7 * 2 * np.pi
26
  df['day_sin'] = np.sin(d)
27
  df['day_cos'] = np.cos(d)
@@ -29,131 +28,207 @@ def parse_ts(df):
29
 
30
  def decode_geo(df):
31
  df = df.copy()
32
- decoded = df['geohash'].map(lambda x: pgh.decode(x))
33
  df['lat'] = decoded.map(lambda x: x[0])
34
  df['lon'] = decoded.map(lambda x: x[1])
35
- # geohash prefix for neighbor grouping
36
- df['geo4'] = df['geohash'].str[:4]
37
  df['geo3'] = df['geohash'].str[:3]
38
  return df
39
 
40
  def encode_cats(df):
41
  df = df.copy()
42
- road_map = {'Residential': 0, 'Street': 1, 'Highway': 2}
43
- df['RoadType_enc'] = df['RoadType'].map(road_map).fillna(-1)
44
  df['LargeVehicles_enc'] = (df['LargeVehicles'] == 'Allowed').astype(float)
45
- df['Landmarks_enc'] = (df['Landmarks'] == 'Yes').astype(float)
46
- weather_map = {'Sunny': 0, 'Rainy': 1, 'Foggy': 2, 'Snowy': 3}
47
- df['Weather_enc'] = df['Weather'].map(weather_map).fillna(-1)
48
  return df
49
 
50
- print("Parsing features...")
51
- train = parse_ts(train)
52
- train = decode_geo(train)
53
- train = encode_cats(train)
54
- test = parse_ts(test)
55
- test = decode_geo(test)
56
- test = encode_cats(test)
57
-
58
- # ── Lag features ──────────────────────────────────────────────────────────────
59
- train48 = train[train['day'] == 48][['geohash', 'timestamp', 'demand']].rename(
60
- columns={'demand': 'demand_lag1d'})
61
-
62
- # Same geohash+timestamp, 1 day earlier
63
- train = train.merge(train48, on=['geohash', 'timestamp'], how='left')
64
- test = test.merge(train48, on=['geohash', 'timestamp'], how='left')
65
-
66
- # ── Static geohash aggregations (from ALL train) ─────────────────────────────
67
- geo_stats = (train.groupby('geohash')['demand']
68
- .agg(['mean', 'std', 'median', 'max'])
69
- .reset_index()
70
- .rename(columns={'mean':'geo_mean', 'std':'geo_std',
71
- 'median':'geo_median', 'max':'geo_max'}))
72
- train = train.merge(geo_stats, on='geohash', how='left')
73
- test = test.merge(geo_stats, on='geohash', how='left')
74
-
75
- # geohash + hour bucket mean
76
- geo_hour = (train.groupby(['geohash', 'hour'])['demand']
77
- .mean().reset_index().rename(columns={'demand': 'geo_hour_mean'}))
78
- train = train.merge(geo_hour, on=['geohash', 'hour'], how='left')
79
- test = test.merge(geo_hour, on=['geohash', 'hour'], how='left')
80
-
81
- # geohash + timestamp exact mean (over train days)
82
- geo_ts_mean = (train.groupby(['geohash', 'timestamp'])['demand']
83
- .mean().reset_index().rename(columns={'demand': 'geo_ts_mean'}))
84
- train = train.merge(geo_ts_mean, on=['geohash', 'timestamp'], how='left')
85
- test = test.merge(geo_ts_mean, on=['geohash', 'timestamp'], how='left')
86
-
87
- # geo3 (neighbourhood) hour mean
88
- geo3_hour = (train.groupby(['geo3', 'hour'])['demand']
89
- .mean().reset_index().rename(columns={'demand': 'geo3_hour_mean'}))
90
- train = train.merge(geo3_hour, on=['geo3', 'hour'], how='left')
91
- test = test.merge(geo3_hour, on=['geo3', 'hour'], how='left')
92
-
93
- # Impute missing lag1d using geo_ts_mean as proxy
94
- train['demand_lag1d'] = train['demand_lag1d'].fillna(train['geo_ts_mean'])
95
- test['demand_lag1d'] = test['demand_lag1d'].fillna(test['geo_ts_mean'])
96
-
97
- # geo std fill
98
- train['geo_std'] = train['geo_std'].fillna(0)
99
- test['geo_std'] = test['geo_std'].fillna(0)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
100
 
101
  FEATURES = [
102
- 'lat', 'lon',
103
- 'hour', 'minute', 'day',
104
  'time_sin', 'time_cos', 'day_sin', 'day_cos',
105
  'RoadType_enc', 'NumberofLanes', 'LargeVehicles_enc', 'Landmarks_enc',
106
  'Temperature', 'Weather_enc',
107
- 'demand_lag1d',
 
108
  'geo_mean', 'geo_std', 'geo_median', 'geo_max',
109
- 'geo_hour_mean', 'geo_ts_mean', 'geo3_hour_mean',
 
110
  ]
111
 
112
- X_train = train[FEATURES].fillna(-1)
113
- y_train = train['demand']
114
- X_test = test[FEATURES].fillna(-1)
115
-
116
- print(f"Train: {X_train.shape}, Test: {X_test.shape}")
117
-
118
- # ── CV: train on day48, validate on day49 ────────────────────────────────────
119
- mask49 = train['day'] == 49
120
- X_cv_tr, y_cv_tr = X_train[~mask49], y_train[~mask49]
121
- X_cv_va, y_cv_va = X_train[mask49], y_train[mask49]
122
-
123
- PARAMS = dict(
124
- n_estimators=3000,
125
- learning_rate=0.02,
126
- num_leaves=255,
127
- max_depth=-1,
128
- min_child_samples=15,
129
- subsample=0.8,
130
- subsample_freq=1,
131
- colsample_bytree=0.8,
132
- reg_alpha=0.05,
133
- reg_lambda=0.1,
134
- random_state=42,
135
- verbose=-1,
136
- n_jobs=-1,
137
  )
138
 
139
- print("\nCV training (day48->day49)...")
140
- m_cv = LGBMRegressor(**PARAMS)
141
- m_cv.fit(X_cv_tr, y_cv_tr,
142
- eval_set=[(X_cv_va, y_cv_va)])
143
- cv_pred = m_cv.predict(X_cv_va)
144
- cv_r2 = r2_score(y_cv_va, cv_pred)
145
- print(f"CV R2: {cv_r2:.4f} | competition score: {max(0, 100*cv_r2):.2f}")
146
-
147
- print("\nFull training on all data...")
148
- model = LGBMRegressor(**PARAMS)
149
- model.fit(X_train, y_train)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
150
 
151
- preds = np.clip(model.predict(X_test), 0, None)
152
  submission = pd.DataFrame({'Index': test['Index'], 'demand': preds})
153
  submission.to_csv('submission.csv', index=False)
154
- print(f"Saved submission.csv ({len(submission)} rows)")
155
  print(submission.head())
156
 
157
- fi = pd.Series(model.feature_importances_, index=FEATURES).sort_values(ascending=False)
158
  print("\nTop feature importances:")
159
  print(fi.head(15))
 
2
  import pandas as pd
3
  import pygeohash as pgh
4
  from lightgbm import LGBMRegressor
5
+ from xgboost import XGBRegressor
6
  from sklearn.metrics import r2_score
7
  import warnings
8
  warnings.filterwarnings('ignore')
 
13
  train = pd.read_csv(f'{DATA}/train.csv')
14
  test = pd.read_csv(f'{DATA}/test.csv')
15
 
16
+ # ── Parse / encode ────────────────────────────────────────────────────────────
17
  def parse_ts(df):
18
  df = df.copy()
19
+ df['hour'] = df['timestamp'].map(lambda x: int(x.split(':')[0]))
20
  df['minute'] = df['timestamp'].map(lambda x: int(x.split(':')[1]))
21
+ t = (df['hour'] * 60 + df['minute']) / (24 * 60) * 2 * np.pi
 
 
22
  df['time_sin'] = np.sin(t)
23
  df['time_cos'] = np.cos(t)
 
24
  d = df['day'] / 7 * 2 * np.pi
25
  df['day_sin'] = np.sin(d)
26
  df['day_cos'] = np.cos(d)
 
28
 
29
  def decode_geo(df):
30
  df = df.copy()
31
+ decoded = df['geohash'].map(pgh.decode)
32
  df['lat'] = decoded.map(lambda x: x[0])
33
  df['lon'] = decoded.map(lambda x: x[1])
 
 
34
  df['geo3'] = df['geohash'].str[:3]
35
  return df
36
 
37
  def encode_cats(df):
38
  df = df.copy()
39
+ df['RoadType_enc'] = df['RoadType'].map({'Residential': 0, 'Street': 1, 'Highway': 2}).fillna(-1)
 
40
  df['LargeVehicles_enc'] = (df['LargeVehicles'] == 'Allowed').astype(float)
41
+ df['Landmarks_enc'] = (df['Landmarks'] == 'Yes').astype(float)
42
+ df['Weather_enc'] = df['Weather'].map({'Sunny': 0, 'Rainy': 1, 'Foggy': 2, 'Snowy': 3}).fillna(-1)
 
43
  return df
44
 
45
+ train = parse_ts(train); train = decode_geo(train); train = encode_cats(train)
46
+ test = parse_ts(test); test = decode_geo(test); test = encode_cats(test)
47
+
48
+ # ── Neighbor cache ────────────────────────────────────────────────────────────
49
+ print("Building neighbor cache...")
50
+ all_geohashes = list(set(train['geohash']) | set(test['geohash']))
51
+ neighbor_cache = {}
52
+ for gh in all_geohashes:
53
+ t = pgh.get_adjacent(gh, 'top'); b = pgh.get_adjacent(gh, 'bottom')
54
+ l = pgh.get_adjacent(gh, 'left'); r = pgh.get_adjacent(gh, 'right')
55
+ tl = pgh.get_adjacent(t, 'left'); tr = pgh.get_adjacent(t, 'right')
56
+ bl = pgh.get_adjacent(b, 'left'); br = pgh.get_adjacent(b, 'right')
57
+ neighbor_cache[gh] = [t, b, l, r, tl, tr, bl, br]
58
+
59
+ def ts_offset(ts, delta_min):
60
+ h, m = int(ts.split(':')[0]), int(ts.split(':')[1])
61
+ total = (h * 60 + m + delta_min) % (24 * 60)
62
+ return f"{total // 60}:{total % 60}"
63
+
64
+ # ── Core feature builder (given a reference day's lookup dict) ───────────────
65
+ def build_features(df, ref_df):
66
+ """
67
+ df : dataframe to featurize
68
+ ref_df : training subset used to compute all statistics (no leakage)
69
+ """
70
+ df = df.copy()
71
+
72
+ # Lag lookup from ref_df (day 48 in CV, all train for final)
73
+ lag_lookup = dict(zip(zip(ref_df['geohash'], ref_df['timestamp']), ref_df['demand']))
74
+
75
+ # Rolling lags: T, T-15, T-30, T-45, T-60
76
+ for delta in [0, 15, 30, 45, 60]:
77
+ col = 'lag1d' if delta == 0 else f'lag1d_m{delta}'
78
+ if delta == 0:
79
+ df[col] = [lag_lookup.get((gh, ts), np.nan)
80
+ for gh, ts in zip(df['geohash'], df['timestamp'])]
81
+ else:
82
+ df[col] = [lag_lookup.get((gh, ts_offset(ts, -delta)), np.nan)
83
+ for gh, ts in zip(df['geohash'], df['timestamp'])]
84
+
85
+ # Neighbor mean at same timestamp from ref_df
86
+ df['neighbor_mean'] = [
87
+ np.nanmean([lag_lookup.get((n, ts), np.nan)
88
+ for n in neighbor_cache.get(gh, [])]) or np.nan
89
+ for gh, ts in zip(df['geohash'], df['timestamp'])
90
+ ]
91
+
92
+ # Aggregations from ref_df
93
+ geo_stats = (ref_df.groupby('geohash')['demand']
94
+ .agg(['mean','std','median','max']).reset_index()
95
+ .rename(columns={'mean':'geo_mean','std':'geo_std',
96
+ 'median':'geo_median','max':'geo_max'}))
97
+ geo_stats['geo_std'] = geo_stats['geo_std'].fillna(0)
98
+ geo_hour = (ref_df.groupby(['geohash','hour'])['demand']
99
+ .mean().reset_index().rename(columns={'demand':'geo_hour_mean'}))
100
+ geo_ts = (ref_df.groupby(['geohash','timestamp'])['demand']
101
+ .mean().reset_index().rename(columns={'demand':'geo_ts_mean'}))
102
+ geo3_h = (ref_df.groupby(['geo3','hour'])['demand']
103
+ .mean().reset_index().rename(columns={'demand':'geo3_hour_mean'}))
104
+ geo3_m = (ref_df.groupby('geo3')['demand']
105
+ .mean().reset_index().rename(columns={'demand':'geo3_mean'}))
106
+ hr_mean = (ref_df.groupby('hour')['demand']
107
+ .mean().reset_index().rename(columns={'demand':'hour_global_mean'}))
108
+ # Day-49 early hours baseline (if available in ref_df)
109
+ day49_early = ref_df[ref_df['day'] == 49] if 'day' in ref_df.columns else ref_df.iloc[0:0]
110
+ if len(day49_early):
111
+ d49_base = (day49_early.groupby('geohash')['demand']
112
+ .mean().reset_index().rename(columns={'demand':'geo_d49_mean'}))
113
+ else:
114
+ d49_base = pd.DataFrame({'geohash': [], 'geo_d49_mean': []})
115
+
116
+ df = df.merge(geo_stats, on='geohash', how='left')
117
+ df = df.merge(geo_hour, on=['geohash','hour'], how='left')
118
+ df = df.merge(geo_ts, on=['geohash','timestamp'], how='left')
119
+ df = df.merge(geo3_h, on=['geo3','hour'], how='left')
120
+ df = df.merge(geo3_m, on='geo3', how='left')
121
+ df = df.merge(hr_mean, on='hour', how='left')
122
+ df = df.merge(d49_base, on='geohash', how='left')
123
+
124
+ # Daily ratio: day49_morning / day48_morning per geohash
125
+ # Signals if today is busier/quieter than yesterday overall
126
+ day48_am = (ref_df[ref_df['day'] == 48][ref_df['hour'] < 4] if len(ref_df[ref_df['day'] == 48]) else ref_df.iloc[0:0])
127
+ day49_am = (day49_early[day49_early['hour'] < 4] if len(day49_early) else pd.DataFrame())
128
+
129
+ if len(day48_am) and len(day49_am):
130
+ d48_am_mean = day48_am.groupby('geohash')['demand'].mean().rename('d48_am')
131
+ d49_am_mean = day49_am.groupby('geohash')['demand'].mean().rename('d49_am')
132
+ ratio_df = pd.concat([d48_am_mean, d49_am_mean], axis=1).reset_index()
133
+ ratio_df['daily_ratio'] = ratio_df['d49_am'] / ratio_df['d48_am'].replace(0, np.nan)
134
+ ratio_df = ratio_df[['geohash', 'daily_ratio']]
135
+ df = df.merge(ratio_df, on='geohash', how='left')
136
+ else:
137
+ df['daily_ratio'] = np.nan
138
+
139
+ # Impute missing lags with fallback chain
140
+ fallback = df['neighbor_mean'].fillna(df['geo_ts_mean']).fillna(df['geo_hour_mean'])
141
+ for col in ['lag1d','lag1d_m15','lag1d_m30','lag1d_m45','lag1d_m60']:
142
+ df[col] = df[col].fillna(fallback)
143
+
144
+ return df
145
 
146
  FEATURES = [
147
+ 'lat', 'lon', 'hour', 'minute', 'day',
 
148
  'time_sin', 'time_cos', 'day_sin', 'day_cos',
149
  'RoadType_enc', 'NumberofLanes', 'LargeVehicles_enc', 'Landmarks_enc',
150
  'Temperature', 'Weather_enc',
151
+ 'lag1d', 'lag1d_m15', 'lag1d_m30', 'lag1d_m45', 'lag1d_m60',
152
+ 'neighbor_mean',
153
  'geo_mean', 'geo_std', 'geo_median', 'geo_max',
154
+ 'geo_hour_mean', 'geo_ts_mean', 'geo3_hour_mean', 'geo3_mean',
155
+ 'hour_global_mean', 'geo_d49_mean', 'daily_ratio',
156
  ]
157
 
158
+ LGBM_PARAMS = dict(
159
+ n_estimators=3000, learning_rate=0.02, num_leaves=255,
160
+ min_child_samples=15, subsample=0.8, subsample_freq=1,
161
+ colsample_bytree=0.8, reg_alpha=0.05, reg_lambda=0.1,
162
+ random_state=42, verbose=-1, n_jobs=-1,
163
+ )
164
+ XGB_PARAMS = dict(
165
+ n_estimators=3000, learning_rate=0.02, max_depth=8,
166
+ subsample=0.8, colsample_bytree=0.8,
167
+ reg_alpha=0.05, reg_lambda=0.1,
168
+ random_state=42, verbosity=0, n_jobs=-1, tree_method='hist',
 
 
 
 
 
 
 
 
 
 
 
 
 
 
169
  )
170
 
171
+ # ── Proper CV: ref = day48 only ───────────────────────────────────────────────
172
+ print("\nBuilding CV features (ref=day48 only)...")
173
+ train48 = train[train['day'] == 48]
174
+ train49 = train[train['day'] == 49]
175
+
176
+ tr_cv = build_features(train48, train48) # train on day48, featurized vs day48
177
+ va_cv = build_features(train49, train48) # val on day49, but stats from day48 only
178
+
179
+ X_tr = tr_cv[FEATURES].fillna(-1); y_tr = train48['demand'].values
180
+ X_va = va_cv[FEATURES].fillna(-1); y_va = train49['demand'].values
181
+
182
+ print(f"CV — train: {X_tr.shape} val: {X_va.shape}")
183
+
184
+ print("\nTraining LGBM CV...")
185
+ lgbm_cv = LGBMRegressor(**LGBM_PARAMS)
186
+ lgbm_cv.fit(X_tr, y_tr)
187
+ lp = lgbm_cv.predict(X_va)
188
+ lgbm_r2 = r2_score(y_va, lp)
189
+ print(f"LGBM CV R2: {lgbm_r2:.4f} score: {max(0, 100*lgbm_r2):.2f}")
190
+
191
+ print("\nTraining XGB CV...")
192
+ xgb_cv = XGBRegressor(**XGB_PARAMS)
193
+ xgb_cv.fit(X_tr, y_tr)
194
+ xp = xgb_cv.predict(X_va)
195
+ xgb_r2 = r2_score(y_va, xp)
196
+ print(f"XGB CV R2: {xgb_r2:.4f} score: {max(0, 100*xgb_r2):.2f}")
197
+
198
+ best_w, best_r2 = 0, -999
199
+ for w in np.arange(0, 1.05, 0.05):
200
+ r2 = r2_score(y_va, w * lp + (1 - w) * xp)
201
+ if r2 > best_r2:
202
+ best_r2, best_w = r2, w
203
+ print(f"\nBest blend {best_w:.2f}*LGBM + {1-best_w:.2f}*XGB: R2={best_r2:.4f} score={max(0, 100*best_r2):.2f}")
204
+
205
+ # ── Final model: ref = ALL train ──────────────────────────────────────────────
206
+ print("\nBuilding final features (ref=all train)...")
207
+ train_full = build_features(train, train)
208
+ test_full = build_features(test, train)
209
+
210
+ X_all = train_full[FEATURES].fillna(-1)
211
+ y_all = train['demand'].values
212
+ X_test = test_full[FEATURES].fillna(-1)
213
+
214
+ print("Training final LGBM...")
215
+ lgbm_f = LGBMRegressor(**LGBM_PARAMS)
216
+ lgbm_f.fit(X_all, y_all)
217
+
218
+ print("Training final XGB...")
219
+ xgb_f = XGBRegressor(**XGB_PARAMS)
220
+ xgb_f.fit(X_all, y_all)
221
+
222
+ preds = np.clip(
223
+ best_w * lgbm_f.predict(X_test) + (1 - best_w) * xgb_f.predict(X_test),
224
+ 0, None
225
+ )
226
 
 
227
  submission = pd.DataFrame({'Index': test['Index'], 'demand': preds})
228
  submission.to_csv('submission.csv', index=False)
229
+ print(f"\nSaved submission.csv ({len(submission)} rows)")
230
  print(submission.head())
231
 
232
+ fi = pd.Series(lgbm_f.feature_importances_, index=FEATURES).sort_values(ascending=False)
233
  print("\nTop feature importances:")
234
  print(fi.head(15))
submission.csv CHANGED
The diff for this file is too large to render. See raw diff
 
traffic-management-travel-demand-forecast (1).ipynb CHANGED
@@ -1,1085 +1,3 @@
1
- {
2
- "cells": [
3
- {
4
- "cell_type": "markdown",
5
- "metadata": {},
6
- "source": [
7
- "## Introduction\n",
8
- "This notebook is a submission to **Grab AI For Sea Challenge - Traffic Management**, to forecast travel demand based on historical Grab bookings. \n",
9
- "Challenge website: https://www.aiforsea.com/traffic-management\n",
10
- "\n",
11
- "There are **four parts** in this notebook:\n",
12
- "* **Data cleaning & preprocessing**\n",
13
- "* **Model selection: Random Forest vs. XGBoost**\n",
14
- "* **Define a function to predict demands of T+1, ..., T+5 using known data till T**\n",
15
- "* **Predict demands of T+1, ..., T+5 using test data.** \n",
16
- "\n",
17
- "The test dataset can start from any time period after the timeframe of the training dataset. My model will use features from the test dataset ending at timestamp T and predict T+1 to T+5 for all the geohashes which appeared in the training dataset. \n",
18
- "\n",
19
- "Each time interval in this challenge is 15 minutes.\n",
20
- "\n",
21
- "**For evaluators**: please uncomment the code in Part 4 and fill in the link of test dataset. The code will produce a CSV file containing the demand forecasts for T+1 to T+5 for all the geohashes from the training set. Please run all codes in this notebook to avoid any errors. "
22
- ]
23
- },
24
- {
25
- "cell_type": "code",
26
- "execution_count": 1,
27
- "metadata": {
28
- "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
29
- "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
30
- },
31
- "outputs": [
32
- {
33
- "name": "stdout",
34
- "output_type": "stream",
35
- "text": [
36
- "['training.csv']\n"
37
- ]
38
- }
39
- ],
40
- "source": [
41
- "# This Python 3 environment comes with many helpful analytics libraries installed\n",
42
- "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
43
- "# For example, here's several helpful packages to load in \n",
44
- "\n",
45
- "import numpy as np # linear algebra\n",
46
- "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
47
- "\n",
48
- "# Input data files are available in the \"../input/\" directory.\n",
49
- "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
50
- "\n",
51
- "import os\n",
52
- "print(os.listdir(\"../input\"))\n",
53
- "\n",
54
- "# Any results you write to the current directory are saved as output."
55
- ]
56
- },
57
- {
58
- "cell_type": "markdown",
59
- "metadata": {},
60
- "source": [
61
- "## Part 1 - Data Cleaning & Preprocessing"
62
- ]
63
- },
64
- {
65
- "cell_type": "markdown",
66
- "metadata": {},
67
- "source": [
68
- "Take a look at training set:"
69
- ]
70
- },
71
- {
72
- "cell_type": "code",
73
- "execution_count": 2,
74
- "metadata": {
75
- "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
76
- "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
77
- },
78
- "outputs": [
79
- {
80
- "data": {
81
- "text/html": [
82
- "<div>\n",
83
- "<style scoped>\n",
84
- " .dataframe tbody tr th:only-of-type {\n",
85
- " vertical-align: middle;\n",
86
- " }\n",
87
- "\n",
88
- " .dataframe tbody tr th {\n",
89
- " vertical-align: top;\n",
90
- " }\n",
91
- "\n",
92
- " .dataframe thead th {\n",
93
- " text-align: right;\n",
94
- " }\n",
95
- "</style>\n",
96
- "<table border=\"1\" class=\"dataframe\">\n",
97
- " <thead>\n",
98
- " <tr style=\"text-align: right;\">\n",
99
- " <th></th>\n",
100
- " <th>geohash6</th>\n",
101
- " <th>day</th>\n",
102
- " <th>timestamp</th>\n",
103
- " <th>demand</th>\n",
104
- " </tr>\n",
105
- " </thead>\n",
106
- " <tbody>\n",
107
- " <tr>\n",
108
- " <th>0</th>\n",
109
- " <td>qp03wc</td>\n",
110
- " <td>18</td>\n",
111
- " <td>20:0</td>\n",
112
- " <td>0.020072</td>\n",
113
- " </tr>\n",
114
- " <tr>\n",
115
- " <th>1</th>\n",
116
- " <td>qp03pn</td>\n",
117
- " <td>10</td>\n",
118
- " <td>14:30</td>\n",
119
- " <td>0.024721</td>\n",
120
- " </tr>\n",
121
- " <tr>\n",
122
- " <th>2</th>\n",
123
- " <td>qp09sw</td>\n",
124
- " <td>9</td>\n",
125
- " <td>6:15</td>\n",
126
- " <td>0.102821</td>\n",
127
- " </tr>\n",
128
- " <tr>\n",
129
- " <th>3</th>\n",
130
- " <td>qp0991</td>\n",
131
- " <td>32</td>\n",
132
- " <td>5:0</td>\n",
133
- " <td>0.088755</td>\n",
134
- " </tr>\n",
135
- " <tr>\n",
136
- " <th>4</th>\n",
137
- " <td>qp090q</td>\n",
138
- " <td>15</td>\n",
139
- " <td>4:0</td>\n",
140
- " <td>0.074468</td>\n",
141
- " </tr>\n",
142
- " </tbody>\n",
143
- "</table>\n",
144
- "</div>"
145
- ],
146
- "text/plain": [
147
- " geohash6 day timestamp demand\n",
148
- "0 qp03wc 18 20:0 0.020072\n",
149
- "1 qp03pn 10 14:30 0.024721\n",
150
- "2 qp09sw 9 6:15 0.102821\n",
151
- "3 qp0991 32 5:0 0.088755\n",
152
- "4 qp090q 15 4:0 0.074468"
153
- ]
154
- },
155
- "execution_count": 2,
156
- "metadata": {},
157
- "output_type": "execute_result"
158
- }
159
- ],
160
- "source": [
161
- "import matplotlib.pyplot as plt\n",
162
- "%matplotlib inline\n",
163
- "import seaborn as sns\n",
164
- "\n",
165
- "df_train = pd.read_csv('../input/training.csv')\n",
166
- "df_train.head()"
167
- ]
168
- },
169
- {
170
- "cell_type": "markdown",
171
- "metadata": {},
172
- "source": [
173
- "Size of training data:"
174
- ]
175
- },
176
- {
177
- "cell_type": "code",
178
- "execution_count": 3,
179
- "metadata": {},
180
- "outputs": [
181
- {
182
- "data": {
183
- "text/plain": [
184
- "(4206321, 4)"
185
- ]
186
- },
187
- "execution_count": 3,
188
- "metadata": {},
189
- "output_type": "execute_result"
190
- }
191
- ],
192
- "source": [
193
- "df_train.shape"
194
- ]
195
- },
196
- {
197
- "cell_type": "markdown",
198
- "metadata": {},
199
- "source": [
200
- "1329 unique locations in the data"
201
- ]
202
- },
203
- {
204
- "cell_type": "code",
205
- "execution_count": 4,
206
- "metadata": {},
207
- "outputs": [
208
- {
209
- "data": {
210
- "text/plain": [
211
- "1329"
212
- ]
213
- },
214
- "execution_count": 4,
215
- "metadata": {},
216
- "output_type": "execute_result"
217
- }
218
- ],
219
- "source": [
220
- "len(df_train.geohash6.unique())"
221
- ]
222
- },
223
- {
224
- "cell_type": "markdown",
225
- "metadata": {},
226
- "source": [
227
- "Convert timestamp into hours and mininutes:"
228
- ]
229
- },
230
- {
231
- "cell_type": "code",
232
- "execution_count": 5,
233
- "metadata": {},
234
- "outputs": [
235
- {
236
- "data": {
237
- "text/html": [
238
- "<div>\n",
239
- "<style scoped>\n",
240
- " .dataframe tbody tr th:only-of-type {\n",
241
- " vertical-align: middle;\n",
242
- " }\n",
243
- "\n",
244
- " .dataframe tbody tr th {\n",
245
- " vertical-align: top;\n",
246
- " }\n",
247
- "\n",
248
- " .dataframe thead th {\n",
249
- " text-align: right;\n",
250
- " }\n",
251
- "</style>\n",
252
- "<table border=\"1\" class=\"dataframe\">\n",
253
- " <thead>\n",
254
- " <tr style=\"text-align: right;\">\n",
255
- " <th></th>\n",
256
- " <th>geohash6</th>\n",
257
- " <th>day</th>\n",
258
- " <th>timestamp</th>\n",
259
- " <th>demand</th>\n",
260
- " <th>hours</th>\n",
261
- " <th>mins</th>\n",
262
- " </tr>\n",
263
- " </thead>\n",
264
- " <tbody>\n",
265
- " <tr>\n",
266
- " <th>0</th>\n",
267
- " <td>qp03wc</td>\n",
268
- " <td>18</td>\n",
269
- " <td>20:0</td>\n",
270
- " <td>0.020072</td>\n",
271
- " <td>20</td>\n",
272
- " <td>0</td>\n",
273
- " </tr>\n",
274
- " <tr>\n",
275
- " <th>1</th>\n",
276
- " <td>qp03pn</td>\n",
277
- " <td>10</td>\n",
278
- " <td>14:30</td>\n",
279
- " <td>0.024721</td>\n",
280
- " <td>14</td>\n",
281
- " <td>30</td>\n",
282
- " </tr>\n",
283
- " <tr>\n",
284
- " <th>2</th>\n",
285
- " <td>qp09sw</td>\n",
286
- " <td>9</td>\n",
287
- " <td>6:15</td>\n",
288
- " <td>0.102821</td>\n",
289
- " <td>6</td>\n",
290
- " <td>15</td>\n",
291
- " </tr>\n",
292
- " <tr>\n",
293
- " <th>3</th>\n",
294
- " <td>qp0991</td>\n",
295
- " <td>32</td>\n",
296
- " <td>5:0</td>\n",
297
- " <td>0.088755</td>\n",
298
- " <td>5</td>\n",
299
- " <td>0</td>\n",
300
- " </tr>\n",
301
- " <tr>\n",
302
- " <th>4</th>\n",
303
- " <td>qp090q</td>\n",
304
- " <td>15</td>\n",
305
- " <td>4:0</td>\n",
306
- " <td>0.074468</td>\n",
307
- " <td>4</td>\n",
308
- " <td>0</td>\n",
309
- " </tr>\n",
310
- " </tbody>\n",
311
- "</table>\n",
312
- "</div>"
313
- ],
314
- "text/plain": [
315
- " geohash6 day timestamp demand hours mins\n",
316
- "0 qp03wc 18 20:0 0.020072 20 0\n",
317
- "1 qp03pn 10 14:30 0.024721 14 30\n",
318
- "2 qp09sw 9 6:15 0.102821 6 15\n",
319
- "3 qp0991 32 5:0 0.088755 5 0\n",
320
- "4 qp090q 15 4:0 0.074468 4 0"
321
- ]
322
- },
323
- "execution_count": 5,
324
- "metadata": {},
325
- "output_type": "execute_result"
326
- }
327
- ],
328
- "source": [
329
- "df_train['hours'] = df_train['timestamp'].map(lambda x: int(x.split(':')[0]))\n",
330
- "df_train['mins'] = df_train['timestamp'].map(lambda x: int(x.split(':')[1]))\n",
331
- "df_train.head()"
332
- ]
333
- },
334
- {
335
- "cell_type": "markdown",
336
- "metadata": {},
337
- "source": [
338
- "Convert day, hours, mins into a single feature **\"time\"**:"
339
- ]
340
- },
341
- {
342
- "cell_type": "code",
343
- "execution_count": 6,
344
- "metadata": {},
345
- "outputs": [
346
- {
347
- "data": {
348
- "text/html": [
349
- "<div>\n",
350
- "<style scoped>\n",
351
- " .dataframe tbody tr th:only-of-type {\n",
352
- " vertical-align: middle;\n",
353
- " }\n",
354
- "\n",
355
- " .dataframe tbody tr th {\n",
356
- " vertical-align: top;\n",
357
- " }\n",
358
- "\n",
359
- " .dataframe thead th {\n",
360
- " text-align: right;\n",
361
- " }\n",
362
- "</style>\n",
363
- "<table border=\"1\" class=\"dataframe\">\n",
364
- " <thead>\n",
365
- " <tr style=\"text-align: right;\">\n",
366
- " <th></th>\n",
367
- " <th>geohash6</th>\n",
368
- " <th>day</th>\n",
369
- " <th>timestamp</th>\n",
370
- " <th>demand</th>\n",
371
- " <th>hours</th>\n",
372
- " <th>mins</th>\n",
373
- " <th>time</th>\n",
374
- " </tr>\n",
375
- " </thead>\n",
376
- " <tbody>\n",
377
- " <tr>\n",
378
- " <th>0</th>\n",
379
- " <td>qp03wc</td>\n",
380
- " <td>18</td>\n",
381
- " <td>20:0</td>\n",
382
- " <td>0.020072</td>\n",
383
- " <td>20</td>\n",
384
- " <td>0</td>\n",
385
- " <td>25680</td>\n",
386
- " </tr>\n",
387
- " <tr>\n",
388
- " <th>1</th>\n",
389
- " <td>qp03pn</td>\n",
390
- " <td>10</td>\n",
391
- " <td>14:30</td>\n",
392
- " <td>0.024721</td>\n",
393
- " <td>14</td>\n",
394
- " <td>30</td>\n",
395
- " <td>13830</td>\n",
396
- " </tr>\n",
397
- " <tr>\n",
398
- " <th>2</th>\n",
399
- " <td>qp09sw</td>\n",
400
- " <td>9</td>\n",
401
- " <td>6:15</td>\n",
402
- " <td>0.102821</td>\n",
403
- " <td>6</td>\n",
404
- " <td>15</td>\n",
405
- " <td>11895</td>\n",
406
- " </tr>\n",
407
- " <tr>\n",
408
- " <th>3</th>\n",
409
- " <td>qp0991</td>\n",
410
- " <td>32</td>\n",
411
- " <td>5:0</td>\n",
412
- " <td>0.088755</td>\n",
413
- " <td>5</td>\n",
414
- " <td>0</td>\n",
415
- " <td>44940</td>\n",
416
- " </tr>\n",
417
- " <tr>\n",
418
- " <th>4</th>\n",
419
- " <td>qp090q</td>\n",
420
- " <td>15</td>\n",
421
- " <td>4:0</td>\n",
422
- " <td>0.074468</td>\n",
423
- " <td>4</td>\n",
424
- " <td>0</td>\n",
425
- " <td>20400</td>\n",
426
- " </tr>\n",
427
- " </tbody>\n",
428
- "</table>\n",
429
- "</div>"
430
- ],
431
- "text/plain": [
432
- " geohash6 day timestamp demand hours mins time\n",
433
- "0 qp03wc 18 20:0 0.020072 20 0 25680\n",
434
- "1 qp03pn 10 14:30 0.024721 14 30 13830\n",
435
- "2 qp09sw 9 6:15 0.102821 6 15 11895\n",
436
- "3 qp0991 32 5:0 0.088755 5 0 44940\n",
437
- "4 qp090q 15 4:0 0.074468 4 0 20400"
438
- ]
439
- },
440
- "execution_count": 6,
441
- "metadata": {},
442
- "output_type": "execute_result"
443
- }
444
- ],
445
- "source": [
446
- "df_train['time'] = 24*60*(df_train['day']-1) + 60*df_train['hours'] + df_train['mins']\n",
447
- "df_train.head()"
448
- ]
449
- },
450
- {
451
- "cell_type": "markdown",
452
- "metadata": {},
453
- "source": [
454
- "Convert geohash6 into latitude and longtitude:"
455
- ]
456
- },
457
- {
458
- "cell_type": "code",
459
- "execution_count": 7,
460
- "metadata": {},
461
- "outputs": [
462
- {
463
- "data": {
464
- "text/html": [
465
- "<div>\n",
466
- "<style scoped>\n",
467
- " .dataframe tbody tr th:only-of-type {\n",
468
- " vertical-align: middle;\n",
469
- " }\n",
470
- "\n",
471
- " .dataframe tbody tr th {\n",
472
- " vertical-align: top;\n",
473
- " }\n",
474
- "\n",
475
- " .dataframe thead th {\n",
476
- " text-align: right;\n",
477
- " }\n",
478
- "</style>\n",
479
- "<table border=\"1\" class=\"dataframe\">\n",
480
- " <thead>\n",
481
- " <tr style=\"text-align: right;\">\n",
482
- " <th></th>\n",
483
- " <th>geohash6</th>\n",
484
- " <th>day</th>\n",
485
- " <th>timestamp</th>\n",
486
- " <th>demand</th>\n",
487
- " <th>hours</th>\n",
488
- " <th>mins</th>\n",
489
- " <th>time</th>\n",
490
- " <th>Latitude</th>\n",
491
- " <th>Longitude</th>\n",
492
- " </tr>\n",
493
- " </thead>\n",
494
- " <tbody>\n",
495
- " <tr>\n",
496
- " <th>0</th>\n",
497
- " <td>qp02zd</td>\n",
498
- " <td>1</td>\n",
499
- " <td>0:0</td>\n",
500
- " <td>0.022396</td>\n",
501
- " <td>0</td>\n",
502
- " <td>0</td>\n",
503
- " <td>0</td>\n",
504
- " <td>-5.479431</td>\n",
505
- " <td>90.686646</td>\n",
506
- " </tr>\n",
507
- " <tr>\n",
508
- " <th>1</th>\n",
509
- " <td>qp02zu</td>\n",
510
- " <td>1</td>\n",
511
- " <td>0:0</td>\n",
512
- " <td>0.001831</td>\n",
513
- " <td>0</td>\n",
514
- " <td>0</td>\n",
515
- " <td>0</td>\n",
516
- " <td>-5.468445</td>\n",
517
- " <td>90.697632</td>\n",
518
- " </tr>\n",
519
- " <tr>\n",
520
- " <th>2</th>\n",
521
- " <td>qp02zt</td>\n",
522
- " <td>1</td>\n",
523
- " <td>0:0</td>\n",
524
- " <td>0.001112</td>\n",
525
- " <td>0</td>\n",
526
- " <td>0</td>\n",
527
- " <td>0</td>\n",
528
- " <td>-5.462952</td>\n",
529
- " <td>90.686646</td>\n",
530
- " </tr>\n",
531
- " <tr>\n",
532
- " <th>3</th>\n",
533
- " <td>qp02zv</td>\n",
534
- " <td>1</td>\n",
535
- " <td>0:0</td>\n",
536
- " <td>0.006886</td>\n",
537
- " <td>0</td>\n",
538
- " <td>0</td>\n",
539
- " <td>0</td>\n",
540
- " <td>-5.462952</td>\n",
541
- " <td>90.697632</td>\n",
542
- " </tr>\n",
543
- " <tr>\n",
544
- " <th>4</th>\n",
545
- " <td>qp08bj</td>\n",
546
- " <td>1</td>\n",
547
- " <td>0:0</td>\n",
548
- " <td>0.066376</td>\n",
549
- " <td>0</td>\n",
550
- " <td>0</td>\n",
551
- " <td>0</td>\n",
552
- " <td>-5.462952</td>\n",
553
- " <td>90.708618</td>\n",
554
- " </tr>\n",
555
- " </tbody>\n",
556
- "</table>\n",
557
- "</div>"
558
- ],
559
- "text/plain": [
560
- " geohash6 day timestamp demand ... mins time Latitude Longitude\n",
561
- "0 qp02zd 1 0:0 0.022396 ... 0 0 -5.479431 90.686646\n",
562
- "1 qp02zu 1 0:0 0.001831 ... 0 0 -5.468445 90.697632\n",
563
- "2 qp02zt 1 0:0 0.001112 ... 0 0 -5.462952 90.686646\n",
564
- "3 qp02zv 1 0:0 0.006886 ... 0 0 -5.462952 90.697632\n",
565
- "4 qp08bj 1 0:0 0.066376 ... 0 0 -5.462952 90.708618\n",
566
- "\n",
567
- "[5 rows x 9 columns]"
568
- ]
569
- },
570
- "execution_count": 7,
571
- "metadata": {},
572
- "output_type": "execute_result"
573
- }
574
- ],
575
- "source": [
576
- "import Geohash\n",
577
- "df_train['Latitude'] = df_train.geohash6.map(lambda x: float(Geohash.decode_exactly(x)[0]))\n",
578
- "df_train['Longitude'] = df_train.geohash6.map(lambda x: float(Geohash.decode_exactly(x)[1]))\n",
579
- "df_train = df_train.sort_values(by=['time','Latitude','Longitude'], ascending=True)\n",
580
- "df_train = df_train.reset_index().drop('index',axis=1)\n",
581
- "df_train.head()"
582
- ]
583
- },
584
- {
585
- "cell_type": "markdown",
586
- "metadata": {},
587
- "source": [
588
- "Not all locations appear in all time slots"
589
- ]
590
- },
591
- {
592
- "cell_type": "code",
593
- "execution_count": 8,
594
- "metadata": {},
595
- "outputs": [
596
- {
597
- "data": {
598
- "text/html": [
599
- "<div>\n",
600
- "<style scoped>\n",
601
- " .dataframe tbody tr th:only-of-type {\n",
602
- " vertical-align: middle;\n",
603
- " }\n",
604
- "\n",
605
- " .dataframe tbody tr th {\n",
606
- " vertical-align: top;\n",
607
- " }\n",
608
- "\n",
609
- " .dataframe thead th {\n",
610
- " text-align: right;\n",
611
- " }\n",
612
- "</style>\n",
613
- "<table border=\"1\" class=\"dataframe\">\n",
614
- " <thead>\n",
615
- " <tr style=\"text-align: right;\">\n",
616
- " <th></th>\n",
617
- " <th>demand</th>\n",
618
- " </tr>\n",
619
- " <tr>\n",
620
- " <th>geohash6</th>\n",
621
- " <th></th>\n",
622
- " </tr>\n",
623
- " </thead>\n",
624
- " <tbody>\n",
625
- " <tr>\n",
626
- " <th>qp02yc</th>\n",
627
- " <td>577</td>\n",
628
- " </tr>\n",
629
- " <tr>\n",
630
- " <th>qp02yf</th>\n",
631
- " <td>89</td>\n",
632
- " </tr>\n",
633
- " <tr>\n",
634
- " <th>qp02yu</th>\n",
635
- " <td>2</td>\n",
636
- " </tr>\n",
637
- " <tr>\n",
638
- " <th>qp02yv</th>\n",
639
- " <td>7</td>\n",
640
- " </tr>\n",
641
- " <tr>\n",
642
- " <th>qp02yy</th>\n",
643
- " <td>106</td>\n",
644
- " </tr>\n",
645
- " <tr>\n",
646
- " <th>qp02yz</th>\n",
647
- " <td>879</td>\n",
648
- " </tr>\n",
649
- " <tr>\n",
650
- " <th>qp02z1</th>\n",
651
- " <td>1153</td>\n",
652
- " </tr>\n",
653
- " <tr>\n",
654
- " <th>qp02z3</th>\n",
655
- " <td>567</td>\n",
656
- " </tr>\n",
657
- " <tr>\n",
658
- " <th>qp02z4</th>\n",
659
- " <td>448</td>\n",
660
- " </tr>\n",
661
- " <tr>\n",
662
- " <th>qp02z5</th>\n",
663
- " <td>1491</td>\n",
664
- " </tr>\n",
665
- " </tbody>\n",
666
- "</table>\n",
667
- "</div>"
668
- ],
669
- "text/plain": [
670
- " demand\n",
671
- "geohash6 \n",
672
- "qp02yc 577\n",
673
- "qp02yf 89\n",
674
- "qp02yu 2\n",
675
- "qp02yv 7\n",
676
- "qp02yy 106\n",
677
- "qp02yz 879\n",
678
- "qp02z1 1153\n",
679
- "qp02z3 567\n",
680
- "qp02z4 448\n",
681
- "qp02z5 1491"
682
- ]
683
- },
684
- "execution_count": 8,
685
- "metadata": {},
686
- "output_type": "execute_result"
687
- }
688
- ],
689
- "source": [
690
- "df_train[['geohash6','demand']].groupby('geohash6').count().head(10)"
691
- ]
692
- },
693
- {
694
- "cell_type": "markdown",
695
- "metadata": {},
696
- "source": [
697
- "As the training set is a huge dataset with more than 4 million data, I will only use the last 14 days' data, out of which the last five timestamps are used for testing purpose and the rest is for training purpose."
698
- ]
699
- },
700
- {
701
- "cell_type": "code",
702
- "execution_count": 9,
703
- "metadata": {},
704
- "outputs": [],
705
- "source": [
706
- "max_day = df_train.day.max()\n",
707
- "max_time = df_train.time.max()\n",
708
- "train_start = df_train[df_train.day==61-13].index[0]\n",
709
- "test_start = df_train[df_train.time==max_time-15*4].index[0]\n",
710
- "\n",
711
- "Xtrain = df_train[['time', 'Latitude','Longitude']].iloc[train_start:test_start,:]\n",
712
- "Xtest = df_train[['time', 'Latitude','Longitude']].iloc[test_start:,:]\n",
713
- "\n",
714
- "ytrain = df_train.demand.iloc[train_start:test_start]\n",
715
- "ytest = df_train.demand.iloc[test_start:]"
716
- ]
717
- },
718
- {
719
- "cell_type": "code",
720
- "execution_count": 10,
721
- "metadata": {},
722
- "outputs": [
723
- {
724
- "data": {
725
- "text/plain": [
726
- "((990189, 3), (2640, 3), (990189,), (2640,))"
727
- ]
728
- },
729
- "execution_count": 10,
730
- "metadata": {},
731
- "output_type": "execute_result"
732
- }
733
- ],
734
- "source": [
735
- "Xtrain.shape, Xtest.shape, ytrain.shape, ytest.shape"
736
- ]
737
- },
738
- {
739
- "cell_type": "markdown",
740
- "metadata": {},
741
- "source": [
742
- "## Part 2 - Model Selection"
743
- ]
744
- },
745
- {
746
- "cell_type": "markdown",
747
- "metadata": {},
748
- "source": [
749
- "### Part 2.1 - RandomForestRegressor"
750
- ]
751
- },
752
- {
753
- "cell_type": "code",
754
- "execution_count": 11,
755
- "metadata": {},
756
- "outputs": [
757
- {
758
- "name": "stdout",
759
- "output_type": "stream",
760
- "text": [
761
- "RMSE: 0.03347819383369924\n"
762
- ]
763
- }
764
- ],
765
- "source": [
766
- "from sklearn.ensemble import RandomForestRegressor\n",
767
- "from sklearn.metrics import mean_squared_error\n",
768
- "\n",
769
- "model = RandomForestRegressor(n_estimators=30, max_depth=40)\n",
770
- "model.fit(Xtrain, ytrain)\n",
771
- "ytest_pred = model.predict(Xtest)\n",
772
- "rmse = np.sqrt(mean_squared_error(ytest, ytest_pred))\n",
773
- "print('RMSE:',rmse)"
774
- ]
775
- },
776
- {
777
- "cell_type": "markdown",
778
- "metadata": {},
779
- "source": [
780
- "### Part 2.2 - XGBRegressor"
781
- ]
782
- },
783
- {
784
- "cell_type": "code",
785
- "execution_count": 12,
786
- "metadata": {},
787
- "outputs": [
788
- {
789
- "name": "stderr",
790
- "output_type": "stream",
791
- "text": [
792
- "/opt/conda/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
793
- " if getattr(data, 'base', None) is not None and \\\n",
794
- "/opt/conda/lib/python3.6/site-packages/xgboost/core.py:588: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
795
- " data.base is not None and isinstance(data, np.ndarray) \\\n"
796
- ]
797
- },
798
- {
799
- "name": "stdout",
800
- "output_type": "stream",
801
- "text": [
802
- "RMSE: 0.032064894772248616\n"
803
- ]
804
- }
805
- ],
806
- "source": [
807
- "from xgboost import XGBRegressor\n",
808
- "\n",
809
- "model = XGBRegressor(n_estimators=500, learning_rate=0.05, max_depth=35)\n",
810
- "model.fit(Xtrain, ytrain)\n",
811
- "ytest_pred = model.predict(Xtest)\n",
812
- "rmse = np.sqrt(mean_squared_error(ytest, ytest_pred))\n",
813
- "print('RMSE:',rmse)"
814
- ]
815
- },
816
- {
817
- "cell_type": "markdown",
818
- "metadata": {},
819
- "source": [
820
- "#### From above output, XGBRegressor produces a smaller RMSE than RandomForestRegressor. Hence XGBRegressor will be used. \n",
821
- "#### All the hyperparameters above have been refined.[](http://)"
822
- ]
823
- },
824
- {
825
- "cell_type": "markdown",
826
- "metadata": {},
827
- "source": [
828
- "Define a function to convert time into day, hour, minute and timestamp:"
829
- ]
830
- },
831
- {
832
- "cell_type": "code",
833
- "execution_count": 13,
834
- "metadata": {},
835
- "outputs": [],
836
- "source": [
837
- "def convert_time(time):\n",
838
- " day = int(time/(24*60)) + 1\n",
839
- " hour = int((time-(day-1)*24*60)/60)\n",
840
- " minute = time-(day-1)*24*60-hour*60\n",
841
- " timestamp = ':'.join((str(hour),str(minute)))\n",
842
- " return (day, hour, minute, timestamp)"
843
- ]
844
- },
845
- {
846
- "cell_type": "markdown",
847
- "metadata": {},
848
- "source": [
849
- "## Part 3 - Define a function to predict demands of T+1, ..., T+5 using known data till T "
850
- ]
851
- },
852
- {
853
- "cell_type": "code",
854
- "execution_count": 14,
855
- "metadata": {},
856
- "outputs": [],
857
- "source": [
858
- "def predict5ts(link, n_estimators=500, learning_rate=0.05, max_depth=35):\n",
859
- " df = pd.read_csv(link)\n",
860
- " df['hours'] = df['timestamp'].map(lambda x: int(x.split(':')[0]))\n",
861
- " df['mins'] = df['timestamp'].map(lambda x: int(x.split(':')[1]))\n",
862
- " df['time'] = 24*60*(df['day']-1) + 60*df['hours'] + df['mins']\n",
863
- " \n",
864
- " import Geohash\n",
865
- " df['Latitude'] = df.geohash6.map(lambda x: float(Geohash.decode_exactly(x)[0]))\n",
866
- " df['Longitude'] = df.geohash6.map(lambda x: float(Geohash.decode_exactly(x)[1]))\n",
867
- "\n",
868
- " df = df.sort_values(by=['time','Latitude','Longitude'], ascending=True)\n",
869
- " df = df.reset_index().drop('index',axis=1)\n",
870
- " \n",
871
- " X = df[['time', 'Latitude','Longitude']]\n",
872
- " y = df.demand\n",
873
- " \n",
874
- " from xgboost import XGBRegressor\n",
875
- " model = XGBRegressor(n_estimators=n_estimators, learning_rate=learning_rate, max_depth=max_depth)\n",
876
- " model.fit(X, y)\n",
877
- " \n",
878
- " T = df.time.max()\n",
879
- " T1 = T+15\n",
880
- " T2 = T+15*2\n",
881
- " T3 = T+15*3\n",
882
- " T4 = T+15*4\n",
883
- " T5 = T+15*5\n",
884
- " \n",
885
- " geohashes = df_train.geohash6.unique()\n",
886
- " geohashes2 = []\n",
887
- " latitudes = []\n",
888
- " longitudes = []\n",
889
- " times = []\n",
890
- " days = []\n",
891
- " timestamps = []\n",
892
- "\n",
893
- " for t in (T1,T2,T3,T4,T5):\n",
894
- " for gh in geohashes:\n",
895
- " geohashes2.append(gh)\n",
896
- " latitudes.append(float(Geohash.decode_exactly(gh)[0]))\n",
897
- " longitudes.append(float(Geohash.decode_exactly(gh)[1]))\n",
898
- " times.append(t)\n",
899
- " days.append(convert_time(t)[0])\n",
900
- " timestamps.append(convert_time(t)[-1])\n",
901
- "\n",
902
- " df_pred = pd.DataFrame({'geohash6': geohashes2, 'day': days, 'timestamp': timestamps,\n",
903
- " 'time': times, 'Latitude': latitudes, 'Longitude': longitudes})\n",
904
- " Xtest = df_pred[['time', 'Latitude','Longitude']]\n",
905
- " ypred = model.predict(Xtest)\n",
906
- "\n",
907
- " df_pred['demand'] = ypred\n",
908
- " output = df_pred[['geohash6', 'day', 'timestamp', 'demand']]\n",
909
- " output.to_csv('output.csv', index=False)"
910
- ]
911
- },
912
- {
913
- "cell_type": "markdown",
914
- "metadata": {},
915
- "source": [
916
- "Check if the above function works by testing a small portion of data from the training set."
917
- ]
918
- },
919
- {
920
- "cell_type": "code",
921
- "execution_count": 15,
922
- "metadata": {},
923
- "outputs": [
924
- {
925
- "name": "stderr",
926
- "output_type": "stream",
927
- "text": [
928
- "/opt/conda/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
929
- " if getattr(data, 'base', None) is not None and \\\n",
930
- "/opt/conda/lib/python3.6/site-packages/xgboost/core.py:588: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
931
- " data.base is not None and isinstance(data, np.ndarray) \\\n"
932
- ]
933
- },
934
- {
935
- "name": "stdout",
936
- "output_type": "stream",
937
- "text": [
938
- "(6645, 4)\n"
939
- ]
940
- },
941
- {
942
- "data": {
943
- "text/html": [
944
- "<div>\n",
945
- "<style scoped>\n",
946
- " .dataframe tbody tr th:only-of-type {\n",
947
- " vertical-align: middle;\n",
948
- " }\n",
949
- "\n",
950
- " .dataframe tbody tr th {\n",
951
- " vertical-align: top;\n",
952
- " }\n",
953
- "\n",
954
- " .dataframe thead th {\n",
955
- " text-align: right;\n",
956
- " }\n",
957
- "</style>\n",
958
- "<table border=\"1\" class=\"dataframe\">\n",
959
- " <thead>\n",
960
- " <tr style=\"text-align: right;\">\n",
961
- " <th></th>\n",
962
- " <th>geohash6</th>\n",
963
- " <th>day</th>\n",
964
- " <th>timestamp</th>\n",
965
- " <th>demand</th>\n",
966
- " </tr>\n",
967
- " </thead>\n",
968
- " <tbody>\n",
969
- " <tr>\n",
970
- " <th>0</th>\n",
971
- " <td>qp02zd</td>\n",
972
- " <td>62</td>\n",
973
- " <td>0:0</td>\n",
974
- " <td>0.014072</td>\n",
975
- " </tr>\n",
976
- " <tr>\n",
977
- " <th>1</th>\n",
978
- " <td>qp02zu</td>\n",
979
- " <td>62</td>\n",
980
- " <td>0:0</td>\n",
981
- " <td>0.039789</td>\n",
982
- " </tr>\n",
983
- " <tr>\n",
984
- " <th>2</th>\n",
985
- " <td>qp02zt</td>\n",
986
- " <td>62</td>\n",
987
- " <td>0:0</td>\n",
988
- " <td>0.137830</td>\n",
989
- " </tr>\n",
990
- " <tr>\n",
991
- " <th>3</th>\n",
992
- " <td>qp02zv</td>\n",
993
- " <td>62</td>\n",
994
- " <td>0:0</td>\n",
995
- " <td>0.042405</td>\n",
996
- " </tr>\n",
997
- " <tr>\n",
998
- " <th>4</th>\n",
999
- " <td>qp08bj</td>\n",
1000
- " <td>62</td>\n",
1001
- " <td>0:0</td>\n",
1002
- " <td>0.054124</td>\n",
1003
- " </tr>\n",
1004
- " </tbody>\n",
1005
- "</table>\n",
1006
- "</div>"
1007
- ],
1008
- "text/plain": [
1009
- " geohash6 day timestamp demand\n",
1010
- "0 qp02zd 62 0:0 0.014072\n",
1011
- "1 qp02zu 62 0:0 0.039789\n",
1012
- "2 qp02zt 62 0:0 0.137830\n",
1013
- "3 qp02zv 62 0:0 0.042405\n",
1014
- "4 qp08bj 62 0:0 0.054124"
1015
- ]
1016
- },
1017
- "execution_count": 15,
1018
- "metadata": {},
1019
- "output_type": "execute_result"
1020
- }
1021
- ],
1022
- "source": [
1023
- "df_trial = df_train[['geohash6','day','timestamp','demand']].iloc[-20000:,:]\n",
1024
- "df_trial.to_csv('df_trial.csv', index=False)\n",
1025
- "\n",
1026
- "trial_link = 'df_trial.csv'\n",
1027
- "predict5ts(link=trial_link)\n",
1028
- "\n",
1029
- "output = pd.read_csv('output.csv')\n",
1030
- "print(output.shape)\n",
1031
- "output.head()"
1032
- ]
1033
- },
1034
- {
1035
- "cell_type": "code",
1036
- "execution_count": 16,
1037
- "metadata": {},
1038
- "outputs": [],
1039
- "source": [
1040
- "os.remove(\"df_trial.csv\")\n",
1041
- "os.remove(\"output.csv\")"
1042
- ]
1043
- },
1044
- {
1045
- "cell_type": "markdown",
1046
- "metadata": {},
1047
- "source": [
1048
- "## Part 4 - Predict demands of T+1, ..., T+5 using test data\n",
1049
- "* Please uncomment below code and enter the link of test data.\n",
1050
- "* Below code will produce an output file **output.csv** which is the demand forecast of T+1,...,T+5 for all the geo-locations, where T is the last time stamp in the test data."
1051
- ]
1052
- },
1053
- {
1054
- "cell_type": "code",
1055
- "execution_count": 17,
1056
- "metadata": {},
1057
- "outputs": [],
1058
- "source": [
1059
- "#test_link = '...'\n",
1060
- "#predict5ts(link=test_link)"
1061
- ]
1062
- }
1063
- ],
1064
- "metadata": {
1065
- "kernelspec": {
1066
- "display_name": "Python 3",
1067
- "language": "python",
1068
- "name": "python3"
1069
- },
1070
- "language_info": {
1071
- "codemirror_mode": {
1072
- "name": "ipython",
1073
- "version": 3
1074
- },
1075
- "file_extension": ".py",
1076
- "mimetype": "text/x-python",
1077
- "name": "python",
1078
- "nbconvert_exporter": "python",
1079
- "pygments_lexer": "ipython3",
1080
- "version": "3.6.4"
1081
- }
1082
- },
1083
- "nbformat": 4,
1084
- "nbformat_minor": 1
1085
- }
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c261e5da02bf183147144454ea16d21eeb96273b6b53f8772334587c5c6672c
3
+ size 31277