allstate-tabular-models / model_summary.json
t22000t's picture
v1: configs + metrics + GBM + figures
001744d verified
Raw
History Blame Contribute Delete
10 kB
{
"pipeline": "dl",
"timestamp": "2026-06-29T00:17:06.382614",
"config": {
"seed": 42,
"n_tuning_trials": 30,
"cv_folds": 5,
"quick": false,
"architectures": [
"catboost",
"xgboost",
"cann",
"cann_gbm",
"ft_transformer",
"tabm",
"localglmnet",
"drn"
],
"epochs": 300,
"patience": 30,
"batch_size": 512,
"n_ensemble": 3,
"catboost_iterations": 2000,
"mono_lambda": 0.1
},
"best_model": "cann_gbm",
"double_lift_vs_glm": 1.702,
"glm_gini": 0.20404812297155872,
"models": {
"catboost": {
"metrics_train": {
"split": "train",
"n": 150568,
"n_params": 0,
"gini": 0.345528,
"mae": 1131.0156,
"rmse": 1698.0539,
"cv_rmse": 0.564025,
"ae_ratio": 1.006675,
"mean_actual": 3010.6021,
"mean_predicted": 2990.6396,
"gamma_deviance": -41071.576052
},
"metrics_test": {
"split": "test",
"n": 37750,
"n_params": 0,
"gini": 0.346107,
"mae": 1165.4098,
"rmse": 1855.6354,
"cv_rmse": 0.610391,
"ae_ratio": 1.015143,
"mean_actual": 3040.0753,
"mean_predicted": 2994.727,
"gamma_deviance": -10545.968021
},
"training_time": 135.13505816459656,
"best_params": {},
"n_params": 946
},
"xgboost": {
"metrics_train": {
"split": "train",
"n": 150568,
"n_params": 0,
"gini": 0.357934,
"mae": 1051.5452,
"rmse": 1574.9415,
"cv_rmse": 0.523132,
"ae_ratio": 1.015477,
"mean_actual": 3010.6021,
"mean_predicted": 2964.716,
"gamma_deviance": -36639.871748
},
"metrics_test": {
"split": "test",
"n": 37750,
"n_params": 0,
"gini": 0.346848,
"mae": 1152.2617,
"rmse": 1849.6937,
"cv_rmse": 0.608437,
"ae_ratio": 1.026758,
"mean_actual": 3040.0753,
"mean_predicted": 2960.8489,
"gamma_deviance": -10490.842521
},
"training_time": 38.16731905937195,
"best_params": {
"max_depth": 6,
"eta": 0.05,
"n_estimators": 1000,
"reg_alpha": 0.1,
"reg_lambda": 1.0,
"subsample": 0.85,
"colsample_bytree": 0.85,
"early_stopping_rounds": 50
},
"n_params": 778
},
"cann": {
"metrics_train": {
"split": "train",
"n": 150568,
"n_params": 0,
"gini": 0.352146,
"mae": 1099.7003,
"rmse": 1648.2283,
"cv_rmse": 0.547475,
"ae_ratio": 1.007827,
"mean_actual": 3010.6021,
"mean_predicted": 2987.2207,
"gamma_deviance": -38722.049985
},
"metrics_test": {
"split": "test",
"n": 37750,
"n_params": 0,
"gini": 0.345682,
"mae": 1168.0435,
"rmse": 1857.1383,
"cv_rmse": 0.610886,
"ae_ratio": 1.015883,
"mean_actual": 3040.0753,
"mean_predicted": 2992.5446,
"gamma_deviance": -10608.506085
},
"training_time": 918.2992675304413,
"best_params": {
"hidden_dims": [
128,
64
],
"dropout": 0.2,
"lr": 0.001,
"weight_decay": 0.001
},
"n_params": 300519,
"n_ensemble_members": 3,
"best_epochs": [
11,
10,
16
]
},
"cann_gbm": {
"metrics_train": {
"split": "train",
"n": 150568,
"n_params": 0,
"gini": 0.349731,
"mae": 1110.4079,
"rmse": 1662.8043,
"cv_rmse": 0.552316,
"ae_ratio": 1.001403,
"mean_actual": 3010.6021,
"mean_predicted": 3006.3831,
"gamma_deviance": -39527.447422
},
"metrics_test": {
"split": "test",
"n": 37750,
"n_params": 0,
"gini": 0.347317,
"mae": 1158.4278,
"rmse": 1839.3574,
"cv_rmse": 0.605037,
"ae_ratio": 1.009918,
"mean_actual": 3040.0753,
"mean_predicted": 3010.2214,
"gamma_deviance": -10437.003248
},
"training_time": 881.5784821510315,
"best_params": {
"hidden_dims": [
128,
64
],
"dropout": 0.2,
"lr": 0.001,
"weight_decay": 0.001
},
"n_params": 300519,
"n_ensemble_members": 3,
"best_epochs": [
11,
10,
8
]
},
"ft_transformer": {
"metrics_train": {
"split": "train",
"n": 150568,
"n_params": 0,
"gini": 0.026796,
"mae": 2113.1906,
"rmse": 3373.5147,
"cv_rmse": 1.120545,
"ae_ratio": 3.022223,
"mean_actual": 3010.6021,
"mean_predicted": 996.155,
"gamma_deviance": -374262.364084
},
"metrics_test": {
"split": "test",
"n": 37750,
"n_params": 0,
"gini": 0.027911,
"mae": 2141.7642,
"rmse": 3507.7363,
"cv_rmse": 1.153832,
"ae_ratio": 3.051806,
"mean_actual": 3040.0753,
"mean_predicted": 996.156,
"gamma_deviance": -95665.216816
},
"training_time": 13875.232380867004,
"best_params": {
"d_model": 64,
"n_heads": 4,
"n_layers": 3,
"dropout": 0.1,
"ffn_factor": 4,
"lr": 0.0005,
"weight_decay": 0.0001
},
"n_params": 700611,
"n_ensemble_members": 3,
"best_epochs": [
0,
0,
206
]
},
"tabm": {
"metrics_train": {
"split": "train",
"n": 150568,
"n_params": 0,
"gini": 0.35646,
"mae": 1331.63,
"rmse": 2243.6843,
"cv_rmse": 0.745261,
"ae_ratio": 1.541768,
"mean_actual": 3010.6021,
"mean_predicted": 1952.6945,
"gamma_deviance": -64699.206702
},
"metrics_test": {
"split": "test",
"n": 37750,
"n_params": 0,
"gini": 0.342687,
"mae": 1392.6337,
"rmse": 2445.7232,
"cv_rmse": 0.804494,
"ae_ratio": 1.553486,
"mean_actual": 3040.0753,
"mean_predicted": 1956.9371,
"gamma_deviance": -18795.918859
},
"training_time": 3170.9919884204865,
"best_params": {
"n_members": 8,
"hidden_dims": [
128,
64
],
"dropout": 0.2,
"lr": 0.001,
"weight_decay": 0.0001
},
"n_params": 1735956,
"n_ensemble_members": 3,
"best_epochs": [
99,
112,
0
]
},
"localglmnet": {
"metrics_train": {
"split": "train",
"n": 150568,
"n_params": 0,
"gini": 0.348541,
"mae": 1134.7861,
"rmse": 1701.8198,
"cv_rmse": 0.565276,
"ae_ratio": 1.000483,
"mean_actual": 3010.6021,
"mean_predicted": 3009.1502,
"gamma_deviance": -40236.055751
},
"metrics_test": {
"split": "test",
"n": 37750,
"n_params": 0,
"gini": 0.342759,
"mae": 1196.5108,
"rmse": 1903.4373,
"cv_rmse": 0.626115,
"ae_ratio": 1.00803,
"mean_actual": 3040.0753,
"mean_predicted": 3015.8595,
"gamma_deviance": -10907.573099
},
"training_time": 3502.3410143852234,
"best_params": {
"hidden_dims": [
64,
32
],
"dropout": 0.3,
"coeff_reg": 1.0,
"lr": 0.0001,
"weight_decay": 0.01
},
"n_params": 193134,
"n_ensemble_members": 3,
"best_epochs": [
125,
138,
147
]
},
"drn": {
"metrics_train": {
"split": "train",
"n": 150568,
"n_params": 0,
"gini": 0.350309,
"mae": 1110.3369,
"rmse": 1659.1151,
"cv_rmse": 0.551091,
"ae_ratio": 0.99661,
"mean_actual": 3010.6021,
"mean_predicted": 3020.8442,
"gamma_deviance": -39493.391953
},
"metrics_test": {
"split": "test",
"n": 37750,
"n_params": 0,
"gini": 0.345411,
"mae": 1174.6962,
"rmse": 1861.0343,
"cv_rmse": 0.612167,
"ae_ratio": 1.004295,
"mean_actual": 3040.0753,
"mean_predicted": 3027.0735,
"gamma_deviance": -10640.272218
},
"training_time": 863.1595370769501,
"best_params": {
"hidden_dims": [
128,
64
],
"dropout": 0.2,
"kl_alpha": 0.1,
"lr": 0.001,
"weight_decay": 0.0001
},
"n_params": 300714,
"n_ensemble_members": 3,
"best_epochs": [
10,
15,
9
]
},
"stacked_ensemble": {
"metrics_train": {
"split": "train",
"n": 150568,
"n_params": 0,
"gini": 0.358506,
"mae": 1045.0175,
"rmse": 1588.6652,
"cv_rmse": 0.52769,
"ae_ratio": 1.051201,
"mean_actual": 3010.6021,
"mean_predicted": 2863.963,
"gamma_deviance": -36577.102398
},
"metrics_test": {
"split": "test",
"n": 37750,
"n_params": 0,
"gini": 0.347197,
"mae": 1144.2383,
"rmse": 1863.5389,
"cv_rmse": 0.612991,
"ae_ratio": 1.062628,
"mean_actual": 3040.0753,
"mean_predicted": 2860.9032,
"gamma_deviance": -10548.74513
},
"training_time": 0.034310340881347656,
"best_params": {},
"n_params": 9,
"base_weights": {
"glm": 0.0,
"catboost": 0.0,
"xgboost": 0.900443769806392,
"cann": 0.0,
"cann_gbm": 0.0,
"ft_transformer": 0.0,
"tabm": 0.099556230193608,
"localglmnet": 0.0,
"drn": 0.0
}
}
}
}