diff --git a/contraceptive/lct_gan/eval.csv b/contraceptive/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..cccbc2cdf06e873bdb511efd69dd8de1b02fae24 --- /dev/null +++ b/contraceptive/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.00512148112797979,,0.0012481103888678906,2.623439073562622,0.02906966581940651,0.5664457678794861,0.03891143947839737,1.5334592262661317e-06,3.23850417137146,0.027567299082875252,0.06443586200475693,0.03532860800623894,0.06025194749236107,0.005062854383140802,5.861943244934082 diff --git a/contraceptive/lct_gan/history.csv b/contraceptive/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..dd8da34cf239fad72340b9d809ba58938424ac37 --- /dev/null +++ b/contraceptive/lct_gan/history.csv @@ -0,0 +1,19 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.017435819932825326,2.606467050097919,0.001229229095874701,0.13089621299295687,0.0,0.0,0.0,0.0,0.01772701053082528,320,160,155.78390192985535,0.9736493870615959,0.48682469353079794,0.07882826873228624,0.00573181561412639,12.050118238902542,4.794865197795817e-05,0.038472728827036916,0.0,0.0,0.0,0.0,0.005818093370544375,80,40,34.2833890914917,0.8570847272872925,0.42854236364364623,0.006411193099665979 +1,0.005912707145500207,5.2315221212032315,0.00010032810266796437,0.03348311016598018,0.0,0.0,0.0,0.0,0.005997926355354366,320,160,155.64752626419067,0.9727970391511918,0.4863985195755959,0.05807469860495758,0.0031659284644774744,5.03204128017187,9.458723244115674e-06,0.01957300353096798,0.0,0.0,0.0,0.0,0.003215988780993939,80,40,34.367220640182495,0.8591805160045624,0.4295902580022812,0.01497489650537318 +2,0.0038566824743270444,3.2254029402142104,2.3398139742636308e-05,0.018911924263375113,0.0,0.0,0.0,0.0,0.0039152335221046995,320,160,155.44242548942566,0.9715151593089104,0.4857575796544552,0.05645614635777747,0.004335876302411635,1.5126577647796693,3.141526110610471e-05,0.014913534704828636,0.0,0.0,0.0,0.0,0.00462017589743482,80,40,34.45174741744995,0.8612936854362487,0.43064684271812437,0.03465941670583561 +3,0.0031945571511187154,1.8064189564559683,1.6363130924196278e-05,0.01819925236850395,0.0,0.0,0.0,0.0,0.0032394818571447105,320,160,156.11653804779053,0.9757283627986908,0.4878641813993454,0.06629898547616904,0.0023899373212771025,1.704710948085517,5.742013863263562e-06,0.009725827592774294,0.0,0.0,0.0,0.0,0.002437889728753362,80,40,34.67283797264099,0.8668209493160248,0.4334104746580124,0.02997926942189224 +4,0.0023628196747608856,1.89771427981769,6.617509099889035e-06,0.013511416382971219,0.0,0.0,0.0,0.0,0.002397591866122184,320,160,156.45443058013916,0.9778401911258697,0.48892009556293486,0.06777484054055094,0.0026370916782980204,2.4481480976558276,1.1830023779024756e-05,0.012106407032115385,0.0,0.0,0.0,0.0,0.00270649099484217,80,40,34.256956577301025,0.8564239144325256,0.4282119572162628,0.025272921007854166 +5,0.002126323909101302,1.2274288133276736,7.2796363170729576e-06,0.012182644824497402,0.0,0.0,0.0,0.0,0.002155698917817972,320,160,155.61095333099365,0.9725684583187103,0.48628422915935515,0.06696215897809452,0.0019572516139305662,1.961672229903479,4.562278869368885e-06,0.009364733044640161,0.0,0.0,0.0,0.0,0.0019919197275157785,80,40,34.33668065071106,0.8584170162677764,0.4292085081338882,0.02843369234469719 +6,0.0017357339755449176,1.420432839454491,3.373285983916587e-06,0.009894938666548114,0.0,0.0,0.0,0.0,0.0017607334299384546,320,160,155.85932803153992,0.9741208001971244,0.4870604000985622,0.07427309235037001,0.002349261014842341,1.9172306640745709,1.04527538708276e-05,0.01324224536656402,0.0,0.0,0.0,0.0,0.0024387467860833567,80,40,34.37562274932861,0.8593905687332153,0.42969528436660764,0.03128219458158128 +7,0.0015943243683750551,1.599416987769855,2.6920245849319574e-06,0.010280541581232682,0.0,0.0,0.0,0.0,0.0016154164098679757,320,160,155.61981463432312,0.9726238414645195,0.48631192073225976,0.06585824833100559,0.002163761601332226,3.604303188103313,6.903144204351009e-06,0.012041708015021867,0.0,0.0,0.0,0.0,0.002217399652727181,80,40,34.502256631851196,0.86255641579628,0.43127820789814,0.02771169388506678 +8,0.001494447229151774,0.9408696638613889,3.168799174362632e-06,0.010874996512575308,0.0,0.0,0.0,0.0,0.001513177437129798,320,160,156.08727836608887,0.9755454897880554,0.4877727448940277,0.0750845561065944,0.002528560180871864,2.08794770268305,1.0474300727891794e-05,0.014760683900385629,0.0,0.0,0.0,0.0,0.002600585496657004,80,40,34.791484117507935,0.8697871029376983,0.43489355146884917,0.027971005909785164 +9,0.0014090196304223923,1.0548290274423293,2.204400118581776e-06,0.010855462366089341,0.0,0.0,0.0,0.0,0.0014265500317165447,320,160,156.41769289970398,0.9776105806231499,0.48880529031157494,0.07426661461722688,0.0021760027257187176,2.8094946333655115,7.478494145375658e-06,0.013558965211268514,0.0,0.0,0.0,0.0,0.0022287734420842753,80,40,34.604350090026855,0.8651087522506714,0.4325543761253357,0.02753840586374281 +10,0.0013136547345652615,1.0299152055348912,1.3887058074146808e-06,0.01068494772334816,0.0,0.0,0.0,0.0,0.0013296200684862925,320,160,157.22391438484192,0.982649464905262,0.491324732452631,0.07313437857337704,0.002245530338041135,1.7400220613416892,9.522791048111739e-06,0.01612092750874581,0.0,0.0,0.0,0.0,0.002284233476166264,80,40,34.46721410751343,0.8616803526878357,0.43084017634391786,0.03776466146664461 +11,0.0012481407272957768,1.2783809218519564,1.503542835857953e-06,0.010688186372863128,0.0,0.0,0.0,0.0,0.001262658101705938,320,160,155.94623494148254,0.9746639683842659,0.48733198419213297,0.07646593883546302,0.0020634316742871306,1.4383508620280794,5.810482365631309e-06,0.01460115851368755,0.0,0.0,0.0,0.0,0.002097635416976118,80,40,34.262441873550415,0.8565610468387603,0.42828052341938017,0.032811734377173704 +12,0.0011262651406582335,1.391008563175482,9.751292843113157e-07,0.00979143314070825,0.0,0.0,0.0,0.0,0.0011396489503042063,320,160,155.54917788505554,0.9721823617815971,0.48609118089079856,0.07479820074022428,0.002025444437913393,2.593721475730203,6.336484106334028e-06,0.014655754225168493,0.0,0.0,0.0,0.0,0.0020664544256987936,80,40,34.536054372787476,0.8634013593196869,0.43170067965984343,0.030719270761437656 +13,0.001076017083954639,1.4508966523469375,1.1593949275121788e-06,0.010946684941154671,0.0,0.0,0.0,0.0,0.0010878692951905578,320,160,155.46073746681213,0.9716296091675758,0.4858148045837879,0.07483407230411103,0.0023200841031211896,2.2440369691766597,1.0239675689505124e-05,0.019701136136427523,0.0,0.0,0.0,0.0,0.002370036526554031,80,40,34.30450892448425,0.8576127231121063,0.4288063615560532,0.03492432142811595 +14,0.0009635111247286332,0.9677751383686999,1.1388382714770661e-06,0.010239775705485954,0.0,0.0,0.0,0.0,0.0009741376784571543,320,160,155.40642023086548,0.9712901264429092,0.4856450632214546,0.07218089971502195,0.0024849619219821763,1.144158828597142,1.2594672893334602e-05,0.02563589295023121,0.0,0.0,0.0,0.0,0.002548753496012068,80,40,34.3231565952301,0.8580789148807526,0.4290394574403763,0.037558259995421395 +15,0.0008957717637713358,0.6864845278677636,7.343289420358814e-07,0.011444034962005389,0.0,0.0,0.0,0.0,0.0009049436197585692,320,160,155.5952959060669,0.9724705994129181,0.48623529970645907,0.07640777615497427,0.002615584134036908,1.8248634061481426,1.3346477949705005e-05,0.022938197170151397,0.0,0.0,0.0,0.0,0.00265875455661444,80,40,34.41921806335449,0.8604804515838623,0.43024022579193116,0.035906832899490836 +16,0.0009050843258659568,0.8020780563841594,7.130549941341754e-07,0.00966488631129323,0.0,0.0,0.0,0.0,0.0009149009783328665,320,160,155.25288820266724,0.9703305512666702,0.4851652756333351,0.07319629316043574,0.0025455261694332875,0.8264324518099329,1.2538950847873486e-05,0.0264737568970304,0.0,0.0,0.0,0.0,0.0025839720274234423,80,40,34.507899045944214,0.8626974761486054,0.4313487380743027,0.038317401116364634 +17,0.0007670033644302521,0.765318092200449,6.45077319996485e-07,0.010603800446915557,0.0,0.0,0.0,0.0,0.0007746850561716201,320,160,155.98394584655762,0.9748996615409851,0.48744983077049253,0.07558807322129724,0.0028201576376886807,1.2422505138596533,1.450647435639052e-05,0.023493397649144755,0.0,0.0,0.0,0.0,0.0028617149822821376,80,40,34.21730875968933,0.8554327189922333,0.42771635949611664,0.0343154217407573 diff --git a/contraceptive/lct_gan/mlu-eval.ipynb b/contraceptive/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..c1bf515343b9e74b3b62983b42d51a9bef27947e --- /dev/null +++ b/contraceptive/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2548 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:03.197867Z", + "iopub.status.busy": "2024-02-29T23:49:03.197522Z", + "iopub.status.idle": "2024-02-29T23:49:03.232223Z", + "shell.execute_reply": "2024-02-29T23:49:03.231521Z" + }, + "papermill": { + "duration": 0.049959, + "end_time": "2024-02-29T23:49:03.234245", + "exception": false, + "start_time": "2024-02-29T23:49:03.184286", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:03.260232Z", + "iopub.status.busy": "2024-02-29T23:49:03.259886Z", + "iopub.status.idle": "2024-02-29T23:49:03.266649Z", + "shell.execute_reply": "2024-02-29T23:49:03.265783Z" + }, + "papermill": { + "duration": 0.022206, + "end_time": "2024-02-29T23:49:03.268817", + "exception": false, + "start_time": "2024-02-29T23:49:03.246611", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:03.293103Z", + "iopub.status.busy": "2024-02-29T23:49:03.292849Z", + "iopub.status.idle": "2024-02-29T23:49:03.296782Z", + "shell.execute_reply": "2024-02-29T23:49:03.295945Z" + }, + "papermill": { + "duration": 0.018543, + "end_time": "2024-02-29T23:49:03.298764", + "exception": false, + "start_time": "2024-02-29T23:49:03.280221", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:03.322343Z", + "iopub.status.busy": "2024-02-29T23:49:03.321610Z", + "iopub.status.idle": "2024-02-29T23:49:03.325664Z", + "shell.execute_reply": "2024-02-29T23:49:03.324822Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017939, + "end_time": "2024-02-29T23:49:03.327645", + "exception": false, + "start_time": "2024-02-29T23:49:03.309706", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:03.351767Z", + "iopub.status.busy": "2024-02-29T23:49:03.351467Z", + "iopub.status.idle": "2024-02-29T23:49:03.356582Z", + "shell.execute_reply": "2024-02-29T23:49:03.355868Z" + }, + "papermill": { + "duration": 0.0195, + "end_time": "2024-02-29T23:49:03.358528", + "exception": false, + "start_time": "2024-02-29T23:49:03.339028", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "63997c2d", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:03.384339Z", + "iopub.status.busy": "2024-02-29T23:49:03.383785Z", + "iopub.status.idle": "2024-02-29T23:49:03.388472Z", + "shell.execute_reply": "2024-02-29T23:49:03.387640Z" + }, + "papermill": { + "duration": 0.019859, + "end_time": "2024-02-29T23:49:03.390362", + "exception": false, + "start_time": "2024-02-29T23:49:03.370503", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"lct_gan\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/lct_gan/2\"\n", + "param_index = 2\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011136, + "end_time": "2024-02-29T23:49:03.412860", + "exception": false, + "start_time": "2024-02-29T23:49:03.401724", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:03.436607Z", + "iopub.status.busy": "2024-02-29T23:49:03.436358Z", + "iopub.status.idle": "2024-02-29T23:49:03.445110Z", + "shell.execute_reply": "2024-02-29T23:49:03.444317Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.02288, + "end_time": "2024-02-29T23:49:03.446998", + "exception": false, + "start_time": "2024-02-29T23:49:03.424118", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/lct_gan/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:03.470919Z", + "iopub.status.busy": "2024-02-29T23:49:03.470380Z", + "iopub.status.idle": "2024-02-29T23:49:05.698705Z", + "shell.execute_reply": "2024-02-29T23:49:05.697737Z" + }, + "papermill": { + "duration": 2.24253, + "end_time": "2024-02-29T23:49:05.700812", + "exception": false, + "start_time": "2024-02-29T23:49:03.458282", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:05.728994Z", + "iopub.status.busy": "2024-02-29T23:49:05.727638Z", + "iopub.status.idle": "2024-02-29T23:49:05.739557Z", + "shell.execute_reply": "2024-02-29T23:49:05.738644Z" + }, + "papermill": { + "duration": 0.028027, + "end_time": "2024-02-29T23:49:05.741723", + "exception": false, + "start_time": "2024-02-29T23:49:05.713696", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:05.766620Z", + "iopub.status.busy": "2024-02-29T23:49:05.766338Z", + "iopub.status.idle": "2024-02-29T23:49:05.773814Z", + "shell.execute_reply": "2024-02-29T23:49:05.772883Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.022384, + "end_time": "2024-02-29T23:49:05.775849", + "exception": false, + "start_time": "2024-02-29T23:49:05.753465", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:05.801609Z", + "iopub.status.busy": "2024-02-29T23:49:05.801343Z", + "iopub.status.idle": "2024-02-29T23:49:05.906460Z", + "shell.execute_reply": "2024-02-29T23:49:05.905621Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.120574, + "end_time": "2024-02-29T23:49:05.908763", + "exception": false, + "start_time": "2024-02-29T23:49:05.788189", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:05.935382Z", + "iopub.status.busy": "2024-02-29T23:49:05.935091Z", + "iopub.status.idle": "2024-02-29T23:49:10.613821Z", + "shell.execute_reply": "2024-02-29T23:49:10.612849Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.694895, + "end_time": "2024-02-29T23:49:10.616322", + "exception": false, + "start_time": "2024-02-29T23:49:05.921427", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 23:49:08.207655: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 23:49:08.207734: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 23:49:08.209452: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:10.641998Z", + "iopub.status.busy": "2024-02-29T23:49:10.641390Z", + "iopub.status.idle": "2024-02-29T23:49:10.647286Z", + "shell.execute_reply": "2024-02-29T23:49:10.646505Z" + }, + "papermill": { + "duration": 0.020957, + "end_time": "2024-02-29T23:49:10.649330", + "exception": false, + "start_time": "2024-02-29T23:49:10.628373", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:10.675582Z", + "iopub.status.busy": "2024-02-29T23:49:10.675292Z", + "iopub.status.idle": "2024-02-29T23:49:18.983927Z", + "shell.execute_reply": "2024-02-29T23:49:18.982782Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.324586, + "end_time": "2024-02-29T23:49:18.986372", + "exception": false, + "start_time": "2024-02-29T23:49:10.661786", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.775,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.075,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'loss_balancer_beta': 0.675,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:19.506951Z", + "iopub.status.busy": "2024-02-29T23:49:19.506508Z", + "iopub.status.idle": "2024-02-29T23:49:19.586540Z", + "shell.execute_reply": "2024-02-29T23:49:19.585557Z" + }, + "papermill": { + "duration": 0.099476, + "end_time": "2024-02-29T23:49:19.589068", + "exception": false, + "start_time": "2024-02-29T23:49:19.489592", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../contraceptive/_cache/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache4/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache5/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/contraceptive [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T23:49:19.624457Z", + "iopub.status.busy": "2024-02-29T23:49:19.623610Z", + "iopub.status.idle": "2024-02-29T23:49:20.097352Z", + "shell.execute_reply": "2024-02-29T23:49:20.096272Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.492882, + "end_time": "2024-02-29T23:49:20.099515", + "exception": false, + "start_time": "2024-02-29T23:49:19.606633", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:20.130753Z", + "iopub.status.busy": "2024-02-29T23:49:20.130105Z", + "iopub.status.idle": "2024-02-29T23:49:20.135537Z", + "shell.execute_reply": "2024-02-29T23:49:20.134720Z" + }, + "papermill": { + "duration": 0.023581, + "end_time": "2024-02-29T23:49:20.137600", + "exception": false, + "start_time": "2024-02-29T23:49:20.114019", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:20.164842Z", + "iopub.status.busy": "2024-02-29T23:49:20.164472Z", + "iopub.status.idle": "2024-02-29T23:49:20.172753Z", + "shell.execute_reply": "2024-02-29T23:49:20.171792Z" + }, + "papermill": { + "duration": 0.024198, + "end_time": "2024-02-29T23:49:20.174703", + "exception": false, + "start_time": "2024-02-29T23:49:20.150505", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10264072" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:20.202299Z", + "iopub.status.busy": "2024-02-29T23:49:20.201973Z", + "iopub.status.idle": "2024-02-29T23:49:20.289198Z", + "shell.execute_reply": "2024-02-29T23:49:20.288078Z" + }, + "papermill": { + "duration": 0.103806, + "end_time": "2024-02-29T23:49:20.291645", + "exception": false, + "start_time": "2024-02-29T23:49:20.187839", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 40] --\n", + "├─Adapter: 1-1 [2, 1179, 40] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 41,984\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-16 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 40] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-18 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-32 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-3 [2, 8, 256] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 8, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 8, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 8, 256] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 8, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 8, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 8, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 8, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 8, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-21 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-22 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-23 [2, 8, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 8, 256] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-24 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-25 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-15 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-43 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 8, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-45 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-46 [2, 8, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 256] --\n", + "│ │ │ └─Linear: 4-39 [2, 256] 524,544\n", + "│ │ │ └─ReLU6: 4-40 [2, 256] --\n", + "│ │ └─FeedForward: 3-20 [2, 256] --\n", + "│ │ │ └─Linear: 4-41 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-42 [2, 256] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 1] --\n", + "│ │ │ └─Linear: 4-55 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-56 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,264,072\n", + "Trainable params: 10,264,072\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 39.93\n", + "========================================================================================================================\n", + "Input size (MB): 0.47\n", + "Forward/backward pass size (MB): 341.77\n", + "Params size (MB): 41.06\n", + "Estimated Total Size (MB): 383.29\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:49:20.323361Z", + "iopub.status.busy": "2024-02-29T23:49:20.322993Z", + "iopub.status.idle": "2024-03-01T00:52:45.823848Z", + "shell.execute_reply": "2024-03-01T00:52:45.822732Z" + }, + "papermill": { + "duration": 3805.519342, + "end_time": "2024-03-01T00:52:45.826226", + "exception": false, + "start_time": "2024-02-29T23:49:20.306884", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.017435819932825326, 'avg_role_model_std_loss': 2.606467050097919, 'avg_role_model_mean_pred_loss': 0.001229229095874701, 'avg_role_model_g_mag_loss': 0.13089621299295687, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01772701053082528, 'n_size': 320, 'n_batch': 160, 'duration': 155.78390192985535, 'duration_batch': 0.9736493870615959, 'duration_size': 0.48682469353079794, 'avg_pred_std': 0.07882826873228624}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00573181561412639, 'avg_role_model_std_loss': 12.050118238902542, 'avg_role_model_mean_pred_loss': 4.794865197795817e-05, 'avg_role_model_g_mag_loss': 0.038472728827036916, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005818093370544375, 'n_size': 80, 'n_batch': 40, 'duration': 34.2833890914917, 'duration_batch': 0.8570847272872925, 'duration_size': 0.42854236364364623, 'avg_pred_std': 0.006411193099665979}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005912707145500207, 'avg_role_model_std_loss': 5.2315221212032315, 'avg_role_model_mean_pred_loss': 0.00010032810266796437, 'avg_role_model_g_mag_loss': 0.03348311016598018, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005997926355354366, 'n_size': 320, 'n_batch': 160, 'duration': 155.64752626419067, 'duration_batch': 0.9727970391511918, 'duration_size': 0.4863985195755959, 'avg_pred_std': 0.05807469860495758}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0031659284644774744, 'avg_role_model_std_loss': 5.03204128017187, 'avg_role_model_mean_pred_loss': 9.458723244115674e-06, 'avg_role_model_g_mag_loss': 0.01957300353096798, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003215988780993939, 'n_size': 80, 'n_batch': 40, 'duration': 34.367220640182495, 'duration_batch': 0.8591805160045624, 'duration_size': 0.4295902580022812, 'avg_pred_std': 0.01497489650537318}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0038566824743270444, 'avg_role_model_std_loss': 3.2254029402142104, 'avg_role_model_mean_pred_loss': 2.3398139742636308e-05, 'avg_role_model_g_mag_loss': 0.018911924263375113, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0039152335221046995, 'n_size': 320, 'n_batch': 160, 'duration': 155.44242548942566, 'duration_batch': 0.9715151593089104, 'duration_size': 0.4857575796544552, 'avg_pred_std': 0.05645614635777747}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004335876302411635, 'avg_role_model_std_loss': 1.5126577647796693, 'avg_role_model_mean_pred_loss': 3.141526110610471e-05, 'avg_role_model_g_mag_loss': 0.014913534704828636, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00462017589743482, 'n_size': 80, 'n_batch': 40, 'duration': 34.45174741744995, 'duration_batch': 0.8612936854362487, 'duration_size': 0.43064684271812437, 'avg_pred_std': 0.03465941670583561}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0031945571511187154, 'avg_role_model_std_loss': 1.8064189564559683, 'avg_role_model_mean_pred_loss': 1.6363130924196278e-05, 'avg_role_model_g_mag_loss': 0.01819925236850395, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032394818571447105, 'n_size': 320, 'n_batch': 160, 'duration': 156.11653804779053, 'duration_batch': 0.9757283627986908, 'duration_size': 0.4878641813993454, 'avg_pred_std': 0.06629898547616904}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023899373212771025, 'avg_role_model_std_loss': 1.704710948085517, 'avg_role_model_mean_pred_loss': 5.742013863263562e-06, 'avg_role_model_g_mag_loss': 0.009725827592774294, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002437889728753362, 'n_size': 80, 'n_batch': 40, 'duration': 34.67283797264099, 'duration_batch': 0.8668209493160248, 'duration_size': 0.4334104746580124, 'avg_pred_std': 0.02997926942189224}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0023628196747608856, 'avg_role_model_std_loss': 1.89771427981769, 'avg_role_model_mean_pred_loss': 6.617509099889035e-06, 'avg_role_model_g_mag_loss': 0.013511416382971219, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002397591866122184, 'n_size': 320, 'n_batch': 160, 'duration': 156.45443058013916, 'duration_batch': 0.9778401911258697, 'duration_size': 0.48892009556293486, 'avg_pred_std': 0.06777484054055094}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0026370916782980204, 'avg_role_model_std_loss': 2.4481480976558276, 'avg_role_model_mean_pred_loss': 1.1830023779024756e-05, 'avg_role_model_g_mag_loss': 0.012106407032115385, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00270649099484217, 'n_size': 80, 'n_batch': 40, 'duration': 34.256956577301025, 'duration_batch': 0.8564239144325256, 'duration_size': 0.4282119572162628, 'avg_pred_std': 0.025272921007854166}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002126323909101302, 'avg_role_model_std_loss': 1.2274288133276736, 'avg_role_model_mean_pred_loss': 7.2796363170729576e-06, 'avg_role_model_g_mag_loss': 0.012182644824497402, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002155698917817972, 'n_size': 320, 'n_batch': 160, 'duration': 155.61095333099365, 'duration_batch': 0.9725684583187103, 'duration_size': 0.48628422915935515, 'avg_pred_std': 0.06696215897809452}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019572516139305662, 'avg_role_model_std_loss': 1.961672229903479, 'avg_role_model_mean_pred_loss': 4.562278869368885e-06, 'avg_role_model_g_mag_loss': 0.009364733044640161, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019919197275157785, 'n_size': 80, 'n_batch': 40, 'duration': 34.33668065071106, 'duration_batch': 0.8584170162677764, 'duration_size': 0.4292085081338882, 'avg_pred_std': 0.02843369234469719}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017357339755449176, 'avg_role_model_std_loss': 1.420432839454491, 'avg_role_model_mean_pred_loss': 3.373285983916587e-06, 'avg_role_model_g_mag_loss': 0.009894938666548114, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017607334299384546, 'n_size': 320, 'n_batch': 160, 'duration': 155.85932803153992, 'duration_batch': 0.9741208001971244, 'duration_size': 0.4870604000985622, 'avg_pred_std': 0.07427309235037001}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002349261014842341, 'avg_role_model_std_loss': 1.9172306640745709, 'avg_role_model_mean_pred_loss': 1.04527538708276e-05, 'avg_role_model_g_mag_loss': 0.01324224536656402, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024387467860833567, 'n_size': 80, 'n_batch': 40, 'duration': 34.37562274932861, 'duration_batch': 0.8593905687332153, 'duration_size': 0.42969528436660764, 'avg_pred_std': 0.03128219458158128}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015943243683750551, 'avg_role_model_std_loss': 1.599416987769855, 'avg_role_model_mean_pred_loss': 2.6920245849319574e-06, 'avg_role_model_g_mag_loss': 0.010280541581232682, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016154164098679757, 'n_size': 320, 'n_batch': 160, 'duration': 155.61981463432312, 'duration_batch': 0.9726238414645195, 'duration_size': 0.48631192073225976, 'avg_pred_std': 0.06585824833100559}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002163761601332226, 'avg_role_model_std_loss': 3.604303188103313, 'avg_role_model_mean_pred_loss': 6.903144204351009e-06, 'avg_role_model_g_mag_loss': 0.012041708015021867, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002217399652727181, 'n_size': 80, 'n_batch': 40, 'duration': 34.502256631851196, 'duration_batch': 0.86255641579628, 'duration_size': 0.43127820789814, 'avg_pred_std': 0.02771169388506678}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001494447229151774, 'avg_role_model_std_loss': 0.9408696638613889, 'avg_role_model_mean_pred_loss': 3.168799174362632e-06, 'avg_role_model_g_mag_loss': 0.010874996512575308, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001513177437129798, 'n_size': 320, 'n_batch': 160, 'duration': 156.08727836608887, 'duration_batch': 0.9755454897880554, 'duration_size': 0.4877727448940277, 'avg_pred_std': 0.0750845561065944}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002528560180871864, 'avg_role_model_std_loss': 2.08794770268305, 'avg_role_model_mean_pred_loss': 1.0474300727891794e-05, 'avg_role_model_g_mag_loss': 0.014760683900385629, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002600585496657004, 'n_size': 80, 'n_batch': 40, 'duration': 34.791484117507935, 'duration_batch': 0.8697871029376983, 'duration_size': 0.43489355146884917, 'avg_pred_std': 0.027971005909785164}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014090196304223923, 'avg_role_model_std_loss': 1.0548290274423293, 'avg_role_model_mean_pred_loss': 2.204400118581776e-06, 'avg_role_model_g_mag_loss': 0.010855462366089341, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014265500317165447, 'n_size': 320, 'n_batch': 160, 'duration': 156.41769289970398, 'duration_batch': 0.9776105806231499, 'duration_size': 0.48880529031157494, 'avg_pred_std': 0.07426661461722688}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021760027257187176, 'avg_role_model_std_loss': 2.8094946333655115, 'avg_role_model_mean_pred_loss': 7.478494145375658e-06, 'avg_role_model_g_mag_loss': 0.013558965211268514, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022287734420842753, 'n_size': 80, 'n_batch': 40, 'duration': 34.604350090026855, 'duration_batch': 0.8651087522506714, 'duration_size': 0.4325543761253357, 'avg_pred_std': 0.02753840586374281}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013136547345652615, 'avg_role_model_std_loss': 1.0299152055348912, 'avg_role_model_mean_pred_loss': 1.3887058074146808e-06, 'avg_role_model_g_mag_loss': 0.01068494772334816, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013296200684862925, 'n_size': 320, 'n_batch': 160, 'duration': 157.22391438484192, 'duration_batch': 0.982649464905262, 'duration_size': 0.491324732452631, 'avg_pred_std': 0.07313437857337704}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002245530338041135, 'avg_role_model_std_loss': 1.7400220613416892, 'avg_role_model_mean_pred_loss': 9.522791048111739e-06, 'avg_role_model_g_mag_loss': 0.01612092750874581, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002284233476166264, 'n_size': 80, 'n_batch': 40, 'duration': 34.46721410751343, 'duration_batch': 0.8616803526878357, 'duration_size': 0.43084017634391786, 'avg_pred_std': 0.03776466146664461}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012481407272957768, 'avg_role_model_std_loss': 1.2783809218519564, 'avg_role_model_mean_pred_loss': 1.503542835857953e-06, 'avg_role_model_g_mag_loss': 0.010688186372863128, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001262658101705938, 'n_size': 320, 'n_batch': 160, 'duration': 155.94623494148254, 'duration_batch': 0.9746639683842659, 'duration_size': 0.48733198419213297, 'avg_pred_std': 0.07646593883546302}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0020634316742871306, 'avg_role_model_std_loss': 1.4383508620280794, 'avg_role_model_mean_pred_loss': 5.810482365631309e-06, 'avg_role_model_g_mag_loss': 0.01460115851368755, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002097635416976118, 'n_size': 80, 'n_batch': 40, 'duration': 34.262441873550415, 'duration_batch': 0.8565610468387603, 'duration_size': 0.42828052341938017, 'avg_pred_std': 0.032811734377173704}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011262651406582335, 'avg_role_model_std_loss': 1.391008563175482, 'avg_role_model_mean_pred_loss': 9.751292843113157e-07, 'avg_role_model_g_mag_loss': 0.00979143314070825, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011396489503042063, 'n_size': 320, 'n_batch': 160, 'duration': 155.54917788505554, 'duration_batch': 0.9721823617815971, 'duration_size': 0.48609118089079856, 'avg_pred_std': 0.07479820074022428}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002025444437913393, 'avg_role_model_std_loss': 2.593721475730203, 'avg_role_model_mean_pred_loss': 6.336484106334028e-06, 'avg_role_model_g_mag_loss': 0.014655754225168493, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0020664544256987936, 'n_size': 80, 'n_batch': 40, 'duration': 34.536054372787476, 'duration_batch': 0.8634013593196869, 'duration_size': 0.43170067965984343, 'avg_pred_std': 0.030719270761437656}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001076017083954639, 'avg_role_model_std_loss': 1.4508966523469375, 'avg_role_model_mean_pred_loss': 1.1593949275121788e-06, 'avg_role_model_g_mag_loss': 0.010946684941154671, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010878692951905578, 'n_size': 320, 'n_batch': 160, 'duration': 155.46073746681213, 'duration_batch': 0.9716296091675758, 'duration_size': 0.4858148045837879, 'avg_pred_std': 0.07483407230411103}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023200841031211896, 'avg_role_model_std_loss': 2.2440369691766597, 'avg_role_model_mean_pred_loss': 1.0239675689505124e-05, 'avg_role_model_g_mag_loss': 0.019701136136427523, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002370036526554031, 'n_size': 80, 'n_batch': 40, 'duration': 34.30450892448425, 'duration_batch': 0.8576127231121063, 'duration_size': 0.4288063615560532, 'avg_pred_std': 0.03492432142811595}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009635111247286332, 'avg_role_model_std_loss': 0.9677751383686999, 'avg_role_model_mean_pred_loss': 1.1388382714770661e-06, 'avg_role_model_g_mag_loss': 0.010239775705485954, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009741376784571543, 'n_size': 320, 'n_batch': 160, 'duration': 155.40642023086548, 'duration_batch': 0.9712901264429092, 'duration_size': 0.4856450632214546, 'avg_pred_std': 0.07218089971502195}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0024849619219821763, 'avg_role_model_std_loss': 1.144158828597142, 'avg_role_model_mean_pred_loss': 1.2594672893334602e-05, 'avg_role_model_g_mag_loss': 0.02563589295023121, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002548753496012068, 'n_size': 80, 'n_batch': 40, 'duration': 34.3231565952301, 'duration_batch': 0.8580789148807526, 'duration_size': 0.4290394574403763, 'avg_pred_std': 0.037558259995421395}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008957717637713358, 'avg_role_model_std_loss': 0.6864845278677636, 'avg_role_model_mean_pred_loss': 7.343289420358814e-07, 'avg_role_model_g_mag_loss': 0.011444034962005389, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009049436197585692, 'n_size': 320, 'n_batch': 160, 'duration': 155.5952959060669, 'duration_batch': 0.9724705994129181, 'duration_size': 0.48623529970645907, 'avg_pred_std': 0.07640777615497427}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002615584134036908, 'avg_role_model_std_loss': 1.8248634061481426, 'avg_role_model_mean_pred_loss': 1.3346477949705005e-05, 'avg_role_model_g_mag_loss': 0.022938197170151397, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00265875455661444, 'n_size': 80, 'n_batch': 40, 'duration': 34.41921806335449, 'duration_batch': 0.8604804515838623, 'duration_size': 0.43024022579193116, 'avg_pred_std': 0.035906832899490836}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009050843258659568, 'avg_role_model_std_loss': 0.8020780563841594, 'avg_role_model_mean_pred_loss': 7.130549941341754e-07, 'avg_role_model_g_mag_loss': 0.00966488631129323, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009149009783328665, 'n_size': 320, 'n_batch': 160, 'duration': 155.25288820266724, 'duration_batch': 0.9703305512666702, 'duration_size': 0.4851652756333351, 'avg_pred_std': 0.07319629316043574}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025455261694332875, 'avg_role_model_std_loss': 0.8264324518099329, 'avg_role_model_mean_pred_loss': 1.2538950847873486e-05, 'avg_role_model_g_mag_loss': 0.0264737568970304, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025839720274234423, 'n_size': 80, 'n_batch': 40, 'duration': 34.507899045944214, 'duration_batch': 0.8626974761486054, 'duration_size': 0.4313487380743027, 'avg_pred_std': 0.038317401116364634}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007670033644302521, 'avg_role_model_std_loss': 0.765318092200449, 'avg_role_model_mean_pred_loss': 6.45077319996485e-07, 'avg_role_model_g_mag_loss': 0.010603800446915557, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007746850561716201, 'n_size': 320, 'n_batch': 160, 'duration': 155.98394584655762, 'duration_batch': 0.9748996615409851, 'duration_size': 0.48744983077049253, 'avg_pred_std': 0.07558807322129724}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0028201576376886807, 'avg_role_model_std_loss': 1.2422505138596533, 'avg_role_model_mean_pred_loss': 1.450647435639052e-05, 'avg_role_model_g_mag_loss': 0.023493397649144755, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0028617149822821376, 'n_size': 80, 'n_batch': 40, 'duration': 34.21730875968933, 'duration_batch': 0.8554327189922333, 'duration_size': 0.42771635949611664, 'avg_pred_std': 0.0343154217407573}\n", + "Epoch 18\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007405669536515918, 'avg_role_model_std_loss': 0.42573822759511, 'avg_role_model_mean_pred_loss': 6.796701736718965e-07, 'avg_role_model_g_mag_loss': 0.010403439644460378, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007484395295251023, 'n_size': 320, 'n_batch': 160, 'duration': 155.93792605400085, 'duration_batch': 0.9746120378375054, 'duration_size': 0.4873060189187527, 'avg_pred_std': 0.07796419600272202}\n", + "Time out: 3633.951765060425/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▃▆▂▂▁▂▁▂▁▂▁▁▂▂▂▂▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▁▃▇▆▅▆▆▆▆▆█▇▆▇█▇█▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train █▂▁▄▅▄▇▄▇▇▆▇▇▇▆▇▆▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test █▃▂▁▂▁▂▂▂▂▃▂▂▃▅▄▅▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train █▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▃▅▂▂▁▂▁▂▁▂▁▁▂▂▂▂▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▂▅▁▂▁▂▁▂▁▂▁▁▂▂▂▂▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test █▄▁▂▂▂▂▃▂▂▂▁▂▂▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▄█▅▃▃▂▂▂▁▂▂▂▂▂▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▂▃▄▇▁▂▃▄█▆▄▂▅▂▂▃▅▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▃▂▂▄▅▂▃▂▄▅█▃▂▂▂▂▁▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▂▃▄▇▁▂▃▄█▆▄▂▅▂▂▃▅▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▃▂▂▄▅▂▃▂▄▅█▃▂▂▂▂▁▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▂▃▄▇▁▂▃▄█▆▄▂▅▂▂▃▅▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▃▂▂▄▅▂▃▂▄▅█▃▂▂▂▂▁▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00286\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00077\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.03432\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.07559\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.02349\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0106\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00282\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00077\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 1e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 1.24225\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.76532\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.85543\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 0.9749\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.42772\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.48745\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 34.21731\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 155.98395\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 160\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/contraceptive/lct_gan/2/wandb/offline-run-20240229_234921-9v6ryd7h\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_234921-9v6ryd7h/logs\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 399, 'n_batch': 200, 'role_model_metrics': {'avg_loss': 0.0012481098506886472, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.007777243861423095, 'pred_duration': 3.192120313644409, 'grad_duration': 2.678847074508667, 'total_duration': 5.870967388153076, 'pred_std': 0.06025194749236107, 'std_loss': 0.005062854383140802, 'mean_pred_loss': 1.5334601357608335e-06, 'pred_rmse': 0.03532859683036804, 'pred_mae': 0.027567295357584953, 'pred_mape': 0.06443584710359573, 'grad_rmse': 0.038911424577236176, 'grad_mae': 0.029069658368825912, 'grad_mape': 0.5664457082748413}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0012481098506886472, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.007777243861423095, 'avg_pred_duration': 3.192120313644409, 'avg_grad_duration': 2.678847074508667, 'avg_total_duration': 5.870967388153076, 'avg_pred_std': 0.06025194749236107, 'avg_std_loss': 0.005062854383140802, 'avg_mean_pred_loss': 1.5334601357608335e-06}, 'min_metrics': {'avg_loss': 0.0012481098506886472, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.007777243861423095, 'pred_duration': 3.192120313644409, 'grad_duration': 2.678847074508667, 'total_duration': 5.870967388153076, 'pred_std': 0.06025194749236107, 'std_loss': 0.005062854383140802, 'mean_pred_loss': 1.5334601357608335e-06, 'pred_rmse': 0.03532859683036804, 'pred_mae': 0.027567295357584953, 'pred_mape': 0.06443584710359573, 'grad_rmse': 0.038911424577236176, 'grad_mae': 0.029069658368825912, 'grad_mape': 0.5664457082748413}, 'model_metrics': {'lct_gan': {'avg_loss': 0.0012481098506886472, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.007777243861423095, 'pred_duration': 3.192120313644409, 'grad_duration': 2.678847074508667, 'total_duration': 5.870967388153076, 'pred_std': 0.06025194749236107, 'std_loss': 0.005062854383140802, 'mean_pred_loss': 1.5334601357608335e-06, 'pred_rmse': 0.03532859683036804, 'pred_mae': 0.027567295357584953, 'pred_mape': 0.06443584710359573, 'grad_rmse': 0.038911424577236176, 'grad_mae': 0.029069658368825912, 'grad_mape': 0.5664457082748413}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:52:45.870045Z", + "iopub.status.busy": "2024-03-01T00:52:45.869665Z", + "iopub.status.idle": "2024-03-01T00:52:45.874157Z", + "shell.execute_reply": "2024-03-01T00:52:45.873220Z" + }, + "papermill": { + "duration": 0.028574, + "end_time": "2024-03-01T00:52:45.876143", + "exception": false, + "start_time": "2024-03-01T00:52:45.847569", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:52:45.912108Z", + "iopub.status.busy": "2024-03-01T00:52:45.911829Z", + "iopub.status.idle": "2024-03-01T00:52:46.195559Z", + "shell.execute_reply": "2024-03-01T00:52:46.194561Z" + }, + "papermill": { + "duration": 0.304444, + "end_time": "2024-03-01T00:52:46.198186", + "exception": false, + "start_time": "2024-03-01T00:52:45.893742", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:52:46.237721Z", + "iopub.status.busy": "2024-03-01T00:52:46.237365Z", + "iopub.status.idle": "2024-03-01T00:52:46.525891Z", + "shell.execute_reply": "2024-03-01T00:52:46.525004Z" + }, + "papermill": { + "duration": 0.310591, + "end_time": "2024-03-01T00:52:46.528111", + "exception": false, + "start_time": "2024-03-01T00:52:46.217520", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAESCAYAAAB3gfmJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABIqklEQVR4nO3de1xU1fo/8M/MwMxwB7mDCHhBvOAljQkzPR1JNMowT16OXzWPqX3TLpJpdhK6nJ94qW+e1DIrtU4p6Cmt1CxCzRK8IajkJTUUFAcFZLgzMPP8/tiydWKAGRgYgef9eu3XDGuvvfbaMPOw9t5rryUhIgJjjHUgUmtXgDHGLI0DG2Osw+HAxhjrcDiwMcY6HA5sjLEOhwMbY6zD4cDGGOtwbKxdgXuJXq9HXl4enJycIJFIrF0dxthdiAilpaXw8/ODVNp4m4wD213y8vIQEBBg7WowxhqRm5uLrl27NpqHA9tdnJycAAi/OGdnZyvXhjF2t5KSEgQEBIjf08ZwYLtL3emns7MzBzbG7lGmXCbimweMsQ6HAxtjrMPhwMYY63D4GhuzCL1eD61Wa+1qsHbO1tYWMpmsxeVwYGMtptVqkZ2dDb1eb+2qsA7A1dUVPj4+LepLyoGNtQgR4fr165DJZAgICGiy4yRjDSEiVFRU4MaNGwAAX1/fZpfFga0ZrhVXIvtmObycFQjxbrpPTUdWW1uLiooK+Pn5wd7e3trVYe2cnZ0dAODGjRvw8vJq9mlps/69rlu3DkFBQVAqlVCpVDh69Gij+bdv347Q0FAolUqEhYVhz549Buu//vprjB49Gu7u7pBIJMjMzDRYf/nyZUgkEqPL9u3bxXzG1icmJjbnEBv1n7Qr+J9PjyDxaK7Fy25vdDodAEAul1u5JqyjqPsHWVNT0+wyzA5sSUlJiI2NRXx8PE6cOIGBAwciKipKbD7+WWpqKqZMmYJZs2YhIyMDMTExiImJQVZWlpinvLwcw4cPx4oVK4yWERAQgOvXrxssb775JhwdHTF27FiDvJs2bTLIFxMTY+4hNsnDUfgSF5ZXW7zs9oqfrWWWYpHPEpkpPDyc5s2bJ/6s0+nIz8+PEhISjOafOHEiRUdHG6SpVCqaO3duvbzZ2dkEgDIyMpqsx6BBg+gf//iHQRoA2rFjR9MH0QCNRkMASKPRNJrv6xO5FLh4F039+HCz99VRVFZW0pkzZ6iystLaVWEdREOfKVO/n0REZrXYtFot0tPTERkZKaZJpVJERkYiLS3N6DZpaWkG+QEgKiqqwfymSE9PR2ZmJmbNmlVv3bx58+Dh4YHw8HBs3LgR1MgkXNXV1SgpKTFYTOHuoAAAFJRxi42xe5FZga2goAA6nQ7e3t4G6d7e3lCr1Ua3UavVZuU3xaeffoo+ffpg2LBhBulvvfUWtm3bhuTkZEyYMAHPPfcc1qxZ02A5CQkJcHFxERdTR/ZwF09Fud8Waz6JRIKdO3dauxoW9cYbb2DQoEHWrkb7e/KgsrISW7ZsMdpaW7p0KR588EEMHjwYixcvxqJFi7Bq1aoGy1qyZAk0Go245OaadjPAw1FosRWVa6HX87SsrP3avHkzXF1dLVbewoULkZKSYrHymsuswObh4QGZTIb8/HyD9Pz8fPj4+BjdxsfHx6z8Tfnvf/+LiooKTJ8+vcm8KpUKV69eRXW18VNGhUIhjuRhzogebvZCi02nJ2gqm3/nhrH2wtSnShwdHeHu7t7KtWmaWYFNLpdjyJAhBhFZr9cjJSUFERERRreJiIioF8GTk5MbzN+UTz/9FOPGjYOnp2eTeTMzM+Hm5gaFQtGsfTVEbiOFi50tAD4d/TMiQoW21ipLY9dTjdm7dy+GDx8OV1dXuLu747HHHsOlS5cAAMOGDcPixYsN8t+8eRO2trY4ePAgAOD69euIjo6GnZ0dgoODsWXLFgQFBWH16tXN+t2dPn0af/3rX2FnZwd3d3fMmTMHZWVl4voDBw4gPDwcDg4OcHV1xYMPPogrV64AAE6ePImHH34YTk5OcHZ2xpAhQ3D8+PFG93fgwAHMnDkTGo1G7B71xhtvAACCgoLw9ttvY/r06XB2dsacOXMAAIsXL0ZISAjs7e3RvXt3LF261KBbxp9PRZ9++mnExMTgnXfega+vL9zd3TFv3rwWdeUwhdkddGNjYzFjxgwMHToU4eHhWL16NcrLyzFz5kwAwPTp0+Hv74+EhAQAwIsvvoiRI0fi3XffRXR0NBITE3H8+HFs2LBBLLOoqAg5OTnIy8sDAJw/fx6A0Nq7u2V38eJFHDx4sF4/OAD47rvvkJ+fjwceeABKpRLJyclYtmwZFi5caO4hmsTdQQ5NZQ0Ky6rR08uxVfbRHlXW6NA37ger7PvMW1Gwl5v+kS4vL0dsbCwGDBiAsrIyxMXFYfz48cjMzMTUqVOxcuVKLF++XOx+kJSUBD8/Pzz00EMAhM96QUEBDhw4AFtbW8TGxjbY7cmUukRFRSEiIgLHjh3DjRs38Mwzz2D+/PnYvHkzamtrERMTg9mzZ2Pr1q3QarU4evSoWLepU6di8ODB+PDDDyGTyZCZmQlbW9tG9zls2DCsXr0acXFx4nfO0fHOZ/mdd95BXFwc4uPjxTQnJyds3rwZfn5+OH36NGbPng0nJycsWrSowf3s378fvr6+2L9/Py5evIhJkyZh0KBBmD17drN+V6YwO7BNmjQJN2/eRFxcHNRqNQYNGoS9e/eKNwhycnIMHqsZNmwYtmzZgtdffx2vvfYaevXqhZ07d6J///5inm+//VYMjAAwefJkAEB8fLz4HwQANm7ciK5du2L06NH16mVra4t169ZhwYIFICL07NkT//d//9dqvzx3Rzn+KCjnFls7NmHCBIOfN27cCE9PT5w5cwYTJ07ESy+9hF9//VUMZFu2bMGUKVMgkUhw7tw5/PTTTzh27BiGDh0KAPjkk0/Qq1evZtVly5YtqKqqwueffw4HBwcAwNq1a/H4449jxYoVsLW1hUajwWOPPYYePXoAAPr06SNun5OTg1deeQWhoaEAYFI95HI5XFxcIJFIjF4a+utf/4qXX37ZIO31118X3wcFBWHhwoVITExsNLC5ublh7dq1kMlkCA0NRXR0NFJSUu6twAYA8+fPx/z5842uO3DgQL20p556Ck899VSD5T399NN4+umnm9zvsmXLsGzZMqPrxowZgzFjxjRZhqXUdfko5C4fBuxsZTjzVpTV9m2OCxcuIC4uDkeOHEFBQYH4EH9OTg769++P0aNH48svv8RDDz2E7OxspKWl4aOPPgIgnFXY2NjgvvvuE8vr2bMn3NzcmlX3s2fPYuDAgWJQA4AHH3wQer0e58+fx4gRI/D0008jKioKjzzyCCIjIzFx4kTxecrY2Fg888wz+M9//oPIyEg89dRTYgBsrrqAfbekpCS8//77uHTpEsrKylBbW9vktel+/foZPBrl6+uL06dPt6huTWl3d0XvFXVdPgrKuMV2N4lEAnu5jVUWc3usP/744ygqKsLHH3+MI0eO4MiRIwDuXCifOnUq/vvf/6KmpgZbtmxBWFgYwsLCLP47M9WmTZuQlpaGYcOGISkpCSEhITh8+DAA4drWb7/9hujoaOzbtw99+/bFjh07WrS/u4MsIPRJnTp1Kh599FHs2rULGRkZ+Oc//9nkjYU/nxJLJJJWHwmGA1szud/u8sGPVbVPhYWFOH/+PF5//XWMGjUKffr0wa1btwzyPPHEE6iqqsLevXuxZcsWTJ06VVzXu3dv1NbWIiMjQ0y7ePFivTJM1adPH5w8eRLl5eVi2qFDhyCVStG7d28xbfDgwViyZAlSU1PRv39/bNmyRVwXEhKCBQsW4Mcff8STTz6JTZs2NblfuVwuPu/blNTUVAQGBuKf//wnhg4dil69eok3L+41HNiaSXxelFts7ZKbmxvc3d2xYcMGXLx4Efv27UNsbKxBHgcHB8TExGDp0qU4e/YspkyZIq4LDQ1FZGQk5syZg6NHjyIjIwNz5syBnZ1ds551nDp1KpRKJWbMmIGsrCzs378fzz//PKZNmwZvb29kZ2djyZIlSEtLw5UrV/Djjz/iwoUL6NOnDyorKzF//nwcOHAAV65cwaFDh3Ds2DGDa3ANCQoKQllZGVJSUlBQUICKiooG8/bq1Qs5OTlITEzEpUuX8P7777e4VdhaOLA1051rbBzY2iOpVIrExESkp6ejf//+WLBggdHO3FOnTsXJkyfx0EMPoVu3bgbrPv/8c3h7e2PEiBEYP368eIdQqVSaXR97e3v88MMPKCoqwv3334+//e1vGDVqFNauXSuuP3fuHCZMmICQkBDMmTMH8+bNw9y5cyGTyVBYWIjp06cjJCQEEydOxNixY/Hmm282ud9hw4bh2WefxaRJk+Dp6YmVK1c2mHfcuHFYsGAB5s+fj0GDBiE1NRVLly41+1jbgoTM7fzTgZWUlMDFxQUajabJC6KH/yjE5A2H0d3TAfte/kvbVPAeVFVVhezsbAQHBzfrC92RXL16FQEBAfjpp58watQoa1en3WroM2XO95MHmmwmPhVl+/btQ1lZGcLCwnD9+nUsWrQIQUFBGDFihLWr1unxqWgz1Z2KaiproK3lsf47o5qaGrz22mvo168fxo8fD09PT7Gz7pdffglHR0ejS79+/dqsjmPHjm2wHg11neoIuMXWTC52tpBJJdDpCbcqtPB27tynYZ1RVFQUoqKM99kbN24cVCqV0XVNPRFgSZ988gkqKyuNruvSpUub1aOtcWBrJqlUAjd7OQrKqlFYxoGNGXJycoKTk/Xnw/D397d2FayCT0VbgIcIZ+zexIGtBdz5BgJj9yQObC3AQ4Qzdm/iwNYCPEQ4Y/cmDmwtUDdEOI/wwdi9hQNbC7g78DU21nwdcTKXewUHthaoG+GjgE9FWTtl6clcAGFMRolEguLiYouWaw4ObC1w564on4oydi/hwNYCHjzCR31EgLbcOgtP5mKxyVyqq6uxcOFC+Pv7w8HBASqVymB07CtXruDxxx+Hm5sbHBwc0K9fP+zZsweXL1/Gww8/DEAYGkoikZg0Oral8ZMHLVDXYqus0aFCW2vWRCIdVk0FsMzPOvt+LQ+QOzSd7zaezKXhyVzmz5+PM2fOIDExEX5+ftixYwfGjBmD06dPo1evXpg3bx60Wi0OHjwIBwcHnDlzBo6OjggICMBXX32FCRMm4Pz583B2doadnV2zfictwd/EFrCXy6CwkaK6Vo/CMi3su/Cvsz3hyVyMT+aSk5ODTZs2IScnB35+wj+phQsXYu/evdi0aROWLVuGnJwcTJgwQRwqvXv37uL2dc+genl5Wfz6ncmoGdauXUuBgYGkUCgoPDycjhw50mj+bdu2Ue/evUmhUFD//v1p9+7dBuu/+uoreuSRR6hLly4EgDIyMuqVMXLkSAJgsMydO9cgz5UrV+jRRx8lOzs78vT0pIULF1JNTY3Jx6XRaAgAaTQak7cZlpBCgYt3UUbOLZO36UgqKyvpzJkzVFlZKSTo9UTVZdZZ9Hqz6v7777/T5MmTKTg4mJycnMjBwYEAiJ/PRx99VPyM/fHHHwSATp06RUREO3fuJBsbG9LpdAZlurm50XvvvWfS/gHQjh07iIhowYIF9Je//MVgfXFxMQGgn3/+mYiInn76aVIoFPTYY4/R6tWrKS8vT8wbHx9PNjY2NGrUKEpISKCLFy+aVIdNmzaRi4uLQdquXbsIADk4OBgsNjY2NHHiRCIi+vjjj8nGxoaGDRtGcXFxdPLkSXH7/fv3EwC6deuWSXX4s3qfqdvM+X6afY0tKSkJsbGxiI+Px4kTJzBw4EBERUU12ARPTU3FlClTMGvWLGRkZCAmJgYxMTHIysoS85SXl2P48OFYsWJFo/uePXs2rl+/Li53j/ap0+kQHR0NrVaL1NRUfPbZZ9i8eTPi4uLMPUSz8A2EP5FIhNNBayw8mYtFJnMpKyuDTCZDeno6MjMzxeXs2bP497//DQB45pln8Mcff2DatGk4ffo0hg4dijVr1ljsWFvM3GgaHh5O8+bNE3/W6XTk5+dHCQkJRvNPnDiRoqOjDdJUKlW91hYRUXZ2dqMtthdffLHBeu3Zs4ekUimp1Wox7cMPPyRnZ2eqrq5u4qgEzWmxPb3xCAUu3kVJR3NM3qYjaei/672uoKCAANDBgwfFtF9++cWgFVVWVkYODg707bffUt++fWn58uVi3rNnzxIAOn78uJh24cIFAtCsFtuGDRvIzc2NysrKxPW7d++u95m+2wMPPEDPP/+80XWTJ0+mxx9/vMk6fPnll+To6GiQdv78+Xq/m6a8+uqrFBYWRkREhw4dIgBUUFBg8vZ3a/MWm1arRXp6OiIjI8U0qVSKyMhIpKWlGd0mLS3NID8gjGPVUP7GfPnll/Dw8ED//v2xZMkSg4kn0tLSEBYWJk7cXLefkpIS/Pbbb0bLq66uRklJicFirjt92bjF1p7wZC4CY5O5hISEYOrUqZg+fTq+/vprZGdn4+jRo0hISMDu3bsBAC+99BJ++OEHZGdn48SJE9i/f7+4v8DAQEgkEuzatQs3b940uLPbZsyJpNeuXSMAlJqaapD+yiuvUHh4uNFtbG1tacuWLQZp69atIy8vr3p5G2uxffTRR7R37146deoUffHFF+Tv70/jx48X18+ePZtGjx5tsE15eTkBoD179hitW3x8fL3rdjCzxbZszxkKXLyL3vruN5O36Ujaa4uNiCg5OZn69OlDCoWCBgwYQAcOHDBoRREJZwIAaMSIEfW2z8vLo7Fjx5JCoaDAwEDasmULeXl50fr1603a/5/3derUKXr44YdJqVRSly5daPbs2VRaWkpERGq1mmJiYsjX15fkcjkFBgZSXFwc6XQ6qq6upsmTJ1NAQADJ5XLy8/Oj+fPnm/w3efbZZ8nd3Z0AUHx8PBERabVaiouLo6CgILK1tSVfX18aP368eI1x/vz51KNHD1IoFOTp6UnTpk0zaKG99dZb5OPjQxKJhGbMmGFSPepYosXWbgLbn6WkpBAA8SJpcwJbVVUVaTQaccnNzTU7sG34+RIFLt5FL249YfI2HUl7DmyWVvf5+emnn6xdlXbNEoHNrP4JHh4ekMlkyM/PN0jPz883uF18Nx8fH7Pym6pu2OWLFy+iR48e8PHxwdGjR+vtp64OxigUCigUihbVg0f46Lx4Mpd7l1nX2ORyOYYMGYKUlBQxTa/XIyUlBREREUa3iYiIMMgPAMnJyQ3mN1VmZiYAwNfXV9zP6dOnDe7OJicnw9nZGX379m3RvhojXmPjpw86HZ7M5d5ldo/S2NhYzJgxA0OHDkV4eDhWr16N8vJyzJw5E4DQG9vf3x8JCQkAgBdffBEjR47Eu+++i+joaCQmJuL48ePYsGGDWGZRURFycnKQl5cHAGIvaB8fH/j4+ODSpUvYsmULHn30Ubi7u+PUqVNYsGABRowYgQEDBgAARo8ejb59+2LatGlYuXIl1Go1Xn/9dcybN6/FrbLG3Bnhg28edDY8mcs9rDnnwGvWrKFu3bqRXC6n8PBwOnz4sLhu5MiR9S4Wbtu2jUJCQkgul1O/fv3qddDdtGmT0Yv4dRcyc3JyaMSIEdSlSxdSKBTUs2dPeuWVV+qda1++fJnGjh1LdnZ25OHhQS+//HKrd9C9XlxJgYt3UY8lu0lvZgfRjoCvsTFLs8Q1Np4J/i7mzDRdp7pWh96v7wUAnIwbDRf7tvtvfC+om7U7KCjIKs8Eso6nsrISly9fbtFM8Dy6RwspbGRwUgpn9J2xL5tMJgNwp7c+Yy1V1z+1Jafs/NS2BXg4KlBaVYvCMi16eFq7Nm3LxsYG9vb24pA+Uin/r2TNQ0SoqKjAjRs34OrqKv7TbA4ObBbg7iBHdkF5p7yBIJFI4Ovri+zsbHFsMMZawtXVtcXdwTiwWUBdX7bOOkS4XC5Hr169+HSUtZitrW2LWmp1OLBZgDvPVgWpVGpwoZcxa+ILIhbgwbNVMXZP4cBmAWKLrRPeFWXsXsSBzQLEa2zcYmPsnsCBzQLcHfgaG2P3Eg5sFuDBI3wwdk/hwGYBXW7fPCiuqEGtTm/l2jDGOLBZgKu9HNLbo0EXVXCrjTFr48BmATKpRGy1cZcPxqyPA5uF3LmBwIGNMWvjwGYhd4YI5zujjFkbBzYL4SHCGbt3cGCzEB4inLF7Bwc2CxH7snGLjTGr48BmIfy8KGP3Dg5sFlJ3KsrX2BizvmYFtnXr1iEoKAhKpRIqlareRMV/tn37doSGhkKpVCIsLAx79uwxWP/1119j9OjRcHd3h0QiEecMrVNUVITnn38evXv3hp2dHbp164YXXngBGo3GIJ9EIqm3JCYmNucQzcYtNsbuHWYHtqSkJMTGxiI+Ph4nTpzAwIEDERUVZTBR8d1SU1MxZcoUzJo1CxkZGYiJiUFMTAyysrLEPOXl5Rg+fDhWrFhhtIy8vDzk5eXhnXfeQVZWFjZv3oy9e/di1qxZ9fJu2rQJ169fF5eYmBhzD7FZ6lpsRdxiY8z6zJ3zLzw8nObNmyf+rNPpyM/PjxISEozmnzhxIkVHRxukqVQqmjt3br282dnZBIAyMjKarMe2bdtILpcbzBsKgHbs2GHagRjRnHlF65RUailw8S4KXLyLKqprm10Hxphx5nw/zWqxabVapKenIzIyUkyTSqWIjIxEWlqa0W3S0tIM8gPCDNoN5TdV3dyCNjaGo5vPmzcPHh4eCA8Px8aNG0GNTJtaXV2NkpISg6W5HBU2kNsIv04+HWXMuswKbAUFBdDpdPD29jZI9/b2hlqtNrqNWq02K7+p9Xj77bcxZ84cg/S33noL27ZtQ3JyMiZMmIDnnnsOa9asabCchIQEuLi4iEtAQECz6ySRSHiIcMbuEe1uMpeSkhJER0ejb9++eOONNwzWLV26VHw/ePBglJeXY9WqVXjhhReMlrVkyRLExsYalN2S4ObuqECepopbbIxZmVktNg8PD8hkMuTn5xuk5+fnNzgPoI+Pj1n5G1NaWooxY8bAyckJO3bsaHKmaJVKhatXr6K62nigUSgUcHZ2NlhagocIZ+zeYFZgk8vlGDJkCFJSUsQ0vV6PlJQUREREGN0mIiLCID8AJCcnN5i/ISUlJRg9ejTkcjm+/fZbk6Z6y8zMhJubGxQKhVn7ai4e4YOxe4PZp6KxsbGYMWMGhg4divDwcKxevRrl5eWYOXMmAGD69Onw9/dHQkICAODFF1/EyJEj8e677yI6OhqJiYk4fvw4NmzYIJZZVFSEnJwc5OXlAQDOnz8PQGjt+fj4iEGtoqICX3zxhcGFfk9PT8hkMnz33XfIz8/HAw88AKVSieTkZCxbtgwLFy5s2W/IDHceq+JTUcasqjm3XdesWUPdunUjuVxO4eHhdPjwYXHdyJEjacaMGQb5t23bRiEhISSXy6lfv360e/dug/WbNm0iAPWW+Ph4IiLav3+/0fUAKDs7m4iIvv/+exo0aBA5OjqSg4MDDRw4kNavX086nc7k42pJdw8ioo9+vkiBi3fRS4kZzdqeMdYwc76fEqJG+kN0MiUlJXBxcRG7kpjrq/SreHn7STzUywP/maVqhRoy1nmZ8/3kZ0UtqAuP8MHYPYEDmwV53L55UMTT8DFmVRzYLOju4cH5DJ8x6+HAZkF1M1XV6AglVbVWrg1jnRcHNgtS2srgpBB60HCXD8ashwObhd05HeXrbIxZCwc2CxMHnOQWG2NWw4HNwniIcMasjwObhd1psXFgY8xaOLBZmAfPCM+Y1XFgszB3HmySMavjwGZhXW6fihbwzQPGrIYDm4WJw4Nzdw/GrIYDm4XV3Tzg50UZsx4ObBZW10H3VoUWtTq9lWvDWOfEgc3C3OzlkEgAIuBWRY21q8NYp8SBzcJkUgm62HOXD8asiQNbK3DnAScZsyoObK2gbrYq7vLBmHVwYGsF3GJjzLqaFdjWrVuHoKAgKJVKqFQqHD16tNH827dvR2hoKJRKJcLCwrBnzx6D9V9//TVGjx4Nd3d3SCQSZGZm1iujqqoK8+bNg7u7OxwdHTFhwoR6EzHn5OQgOjoa9vb28PLywiuvvILa2rYf8NGj7nlRvsbGmFWYHdiSkpIQGxuL+Ph4nDhxAgMHDkRUVBRu3LhhNH9qaiqmTJmCWbNmISMjAzExMYiJiUFWVpaYp7y8HMOHD8eKFSsa3O+CBQvw3XffYfv27fj555+Rl5eHJ598Ulyv0+kQHR0NrVaL1NRUfPbZZ9i8eTPi4uLMPcQW48eqGLMyc+f2Cw8Pp3nz5ok/63Q68vPzo4SEBKP5J06cSNHR0QZpKpWK5s6dWy9vdnY2AaCMjAyD9OLiYrK1taXt27eLaWfPniUAlJaWRkREe/bsIalUSmq1Wszz4YcfkrOzM1VXVxutW1VVFWk0GnHJzc1t0byidb44fJkCF++iWZuPtagcxtgd5swralaLTavVIj09HZGRkWKaVCpFZGQk0tLSjG6TlpZmkB8AoqKiGsxvTHp6OmpqagzKCQ0NRbdu3cRy0tLSEBYWBm9vb4P9lJSU4LfffjNabkJCAlxcXMQlICDA5Do1pu7mAZ+KMmYdZgW2goIC6HQ6g+ABAN7e3lCr1Ua3UavVZuVvqAy5XA5XV9cGy2loP3XrjFmyZAk0Go245ObmmlynxtQNXcSPVTFmHTbWroA1KRQKKBQKi5fLg00yZl1mtdg8PDwgk8nq3Y3Mz8+Hj4+P0W18fHzMyt9QGVqtFsXFxQ2W09B+6ta1pbruHmXVtaiq0bXpvhljZgY2uVyOIUOGICUlRUzT6/VISUlBRESE0W0iIiIM8gNAcnJyg/mNGTJkCGxtbQ3KOX/+PHJycsRyIiIicPr0aYO7s8nJyXB2dkbfvn1N3pclOClsIJcJv1oevogxKzD3zkRiYiIpFAravHkznTlzhubMmUOurq7i3chp06bRq6++KuY/dOgQ2djY0DvvvENnz56l+Ph4srW1pdOnT4t5CgsLKSMjg3bv3k0AKDExkTIyMuj69etinmeffZa6detG+/bto+PHj1NERARFRESI62tra6l///40evRoyszMpL1795KnpyctWbLE5GMz565LUx5Y9hMFLt5FJ3Nvtbgsxph530+zAxsR0Zo1a6hbt24kl8spPDycDh8+LK4bOXIkzZgxwyD/tm3bKCQkhORyOfXr1492795tsH7Tpk0EoN4SHx8v5qmsrKTnnnuO3NzcyN7ensaPH28Q+IiILl++TGPHjiU7Ozvy8PCgl19+mWpqakw+LksGtuj3D1Lg4l2072x+i8tijJn3/ZQQEVmrtXivKSkpgYuLCzQaDZydnVtU1oyNR/Hz7zex6m8D8NRQy3QjYawzM+f7yc+KthKeEZ4x6+HA1ko8eEZ4xqyGA1sr6cLPizJmNRzYWkndg/AFfCrKWJvjwNZKPMTZqvhUlLG2xoGtlfBgk4xZDwe2VnL386Lco4axtsWBrZXUXWPT6vQorW77UXwZ68w4sLUSpa0Mjgph8BQ+HWWsbXFga0V3rrPxDQTG2hIHtlYkdvngFhtjbYoDWyty59mqGLMKDmytyIO7fDBmFRzYWtGdx6q4xcZYW+LA1orqZqvix6oYa1sc2FpR3V3RIj4VZaxNcWBrRR5884Axq+DA1or4eVHGrIMDWyuqu8ZWVKGFTs/PizLWVjiwtSI3e1tIJAARcKuCW22MtZVmBbZ169YhKCgISqUSKpUKR48ebTT/9u3bERoaCqVSibCwMOzZs8dgPREhLi4Ovr6+sLOzQ2RkJC5cuCCuP3DgACQSidHl2LFjAIDLly8bXX/48OHmHKJF2MikcLPn01HG2prZgS0pKQmxsbGIj4/HiRMnMHDgQERFRRlMVHy31NRUTJkyBbNmzUJGRgZiYmIQExODrKwsMc/KlSvx/vvvY/369Thy5AgcHBwQFRWFqqoqAMCwYcNw/fp1g+WZZ55BcHAwhg4darC/n376ySDfkCFDzD1Ei3LnvmyMtT1z5/YLDw+nefPmiT/rdDry8/OjhIQEo/knTpxI0dHRBmkqlYrmzp1LRER6vZ58fHxo1apV4vri4mJSKBS0detWo2VqtVry9PSkt956S0zLzs4mAJSRkWHysVRVVZFGoxGX3Nxci80rWmfSR6kUuHgXfZN5zWJlMtYZmTOvqFktNq1Wi/T0dERGRoppUqkUkZGRSEtLM7pNWlqaQX4AiIqKEvNnZ2dDrVYb5HFxcYFKpWqwzG+//RaFhYWYOXNmvXXjxo2Dl5cXhg8fjm+//bbR40lISICLi4u4BARYfv5Pd56tirE2Z1ZgKygogE6ng7e3t0G6t7c31Gq10W3UanWj+etezSnz008/RVRUFLp27SqmOTo64t1338X27duxe/duDB8+HDExMY0GtyVLlkCj0YhLbm5ug3mby51nq2KszdlYuwLmunr1Kn744Qds27bNIN3DwwOxsbHiz/fffz/y8vKwatUqjBs3zmhZCoUCCoWiVetb1+WDO+ky1nbMarF5eHhAJpMhPz/fID0/Px8+Pj5Gt/Hx8Wk0f92rqWVu2rQJ7u7uDQaru6lUKly8eLHJfK2JO+ky1vbMCmxyuRxDhgxBSkqKmKbX65GSkoKIiAij20RERBjkB4Dk5GQxf3BwMHx8fAzylJSU4MiRI/XKJCJs2rQJ06dPh62tbZP1zczMhK+vr8nH1xrEoYv4QXjG2ozZp6KxsbGYMWMGhg4divDwcKxevRrl5eXihfzp06fD398fCQkJAIAXX3wRI0eOxLvvvovo6GgkJibi+PHj2LBhAwBAIpHgpZdewr/+9S/06tULwcHBWLp0Kfz8/BATE2Ow73379iE7OxvPPPNMvXp99tlnkMvlGDx4MADg66+/xsaNG/HJJ5+Ye4gWxTcPGGt7Zge2SZMm4ebNm4iLi4NarcagQYOwd+9e8eJ/Tk4OpNI7DcFhw4Zhy5YteP311/Haa6+hV69e2LlzJ/r37y/mWbRoEcrLyzFnzhwUFxdj+PDh2Lt3L5RKpcG+P/30UwwbNgyhoaFG6/b222/jypUrsLGxQWhoKJKSkvC3v/3N3EO0KL55wFjbkxDxpJd1SkpK4OLiAo1GA2dnZ4uUqamswcA3fwQAnHt7DJS2MouUy1hnY873k58VbWXOShvYyiQAgCK+zsZYm+DA1sokEsmdLh98OspYm+DA1gbqunwUcF82xtoEB7Y2cOfOKLfYGGsLHNjaAI/wwVjb4sDWBsTAxjcPGGsTHNjaAJ+KMta2OLC1AfF5Ub55wFib4MDWBjz4QXjG2hQHtjZwpx8bt9gYawsc2NqA2I+tjKfhY6wtcGBrCRMfs/V2VsLFzhZanR7fZ11v5UoxxjiwNcep7cD6h4Csr0zKbiuT4ulhQQCAtfsugscdYKx1cWBrjsKLgPoUcOJzkzeZ+WAQHOQynFOXIuWs8akKGWOWwYGtOQZPBSABsn8Gbl0xaRNXezn+JyIQALB2P7faGGtNHNiaw7Ub0H2k8D7zS5M3e2Z4dyhspMjMLUbqpcJWqhxjjANbcw2eJrxmfAnodSZt4umkwJTwbgCANfsutFbNGOv0OLA1V+hjgNIVKLkK/LHf5M3mjOgOW5kEh/8oQvqVotarH2OdGAe25rJVAgMmCu8zvjB5Mz9XO0y4T5joee0+604NyFhHxYGtJepOR8/tBipMb309O7IHpBJg//mbyLqmaaXKMdZ5NSuwrVu3DkFBQVAqlVCpVDh69Gij+bdv347Q0FAolUqEhYVhz549BuuJCHFxcfD19YWdnR0iIyNx4YLhNaigoCBIJBKDZfny5QZ5Tp06hYceeghKpRIBAQFYuXJlcw7PdL4DAN+BgE4LnEoyebMgDwc8PtAPALBuP7faGLM0swNbUlISYmNjER8fjxMnTmDgwIGIiorCjRvG+2alpqZiypQpmDVrFjIyMhATE4OYmBhkZWWJeVauXIn3338f69evx5EjR+Dg4ICoqChUVVUZlPXWW2/h+vXr4vL888+L60pKSjB69GgEBgYiPT0dq1atwhtvvCHOX9pq6lptJ/5j8pMIADDv4Z4AgO+z1LiQX9oaNWOs8yIzhYeH07x588SfdTod+fn5UUJCgtH8EydOpOjoaIM0lUpFc+fOJSIivV5PPj4+tGrVKnF9cXExKRQK2rp1q5gWGBhI7733XoP1+uCDD8jNzY2qq6vFtMWLF1Pv3r0b3Kaqqoo0Go245ObmEgDSaDQNblNPRRHRW55E8c5EV9NN346I5nx+jAIX76KXEjPM2o6xzkij0Zj8/TSrxabVapGeno7IyEgxTSqVIjIyEmlpaUa3SUtLM8gPAFFRUWL+7OxsqNVqgzwuLi5QqVT1yly+fDnc3d0xePBgrFq1CrW1tQb7GTFiBORyucF+zp8/j1u3bhmtW0JCAlxcXMQlICDAxN/EXezcgL7jhPcZ/zFr0/kP9wIAfHsyD1cKy83fN2PMKLMCW0FBAXQ6nTjrex1vb2+o1Wqj26jV6kbz1702VeYLL7yAxMRE7N+/H3PnzsWyZcuwaNGiJvdz9z7+bMmSJdBoNOKSm5vb4LE3qu509PR/AW2FyZuFdXXByBBP6PSE9T9fat6+GWP12Fi7AqaKjY0V3w8YMAByuRxz585FQkICFApFs8pUKBTN3tZA0EOAayBQfAU4+x0wcJLJm87/a0/8/PtN/Df9Kl4Y1Qu+LnYtrw9jnZxZLTYPDw/IZDLk5+cbpOfn58PHx8foNj4+Po3mr3s1p0wAUKlUqK2txeXLlxvdz937aDVSKTD4f4T3Zp6O3h/UBargLqjRETYc/KMVKsdY52NWYJPL5RgyZAhSUlLENL1ej5SUFERERBjdJiIiwiA/ACQnJ4v5g4OD4ePjY5CnpKQER44cabBMAMjMzIRUKoWXl5e4n4MHD6KmpsZgP71794abm5s5h9k8g/4OQAJc/gUoMi9Azf+rcId069EcFNSNsntqO/Dhg8D1UxauKGOdgLl3JhITE0mhUNDmzZvpzJkzNGfOHHJ1dSW1Wk1ERNOmTaNXX31VzH/o0CGysbGhd955h86ePUvx8fFka2tLp0+fFvMsX76cXF1d6ZtvvqFTp07RE088QcHBwVRZWUlERKmpqfTee+9RZmYmXbp0ib744gvy9PSk6dOni2UUFxeTt7c3TZs2jbKysigxMZHs7e3po48+MvnYzLnrYtR/nhTujv70plmb6fV6GrfmFwpcvIuWf3+WKP/MnTut/3myeXVhrIMx5/tpdmAjIlqzZg1169aN5HI5hYeH0+HDh8V1I0eOpBkzZhjk37ZtG4WEhJBcLqd+/frR7t27Ddbr9XpaunQpeXt7k0KhoFGjRtH58+fF9enp6aRSqcjFxYWUSiX16dOHli1bRlVVVQblnDx5koYPH04KhYL8/f1p+fLlZh1XiwNb1tdCMHqnN5Gu1qxNf/xNTYGLd9HguG+pdu0DQjl1y/XTTRfAWAdnzvdTQsQDg9UpKSmBi4sLNBoNnJ2dzS+gthp4NxSoLAL+vh0IGW3ypno94dH3f8HEgnX4h81ewMET8AkDLu0DBkwGnvzI/Pow1oGY8/3kZ0UtyUYBDJwsvM8wfXRdAJBKJXij73UhqAGoenQNMCpOWJn1X6C4mV1RGOuEOLBZWt3d0fPfA2U3Td+u7CZUJ18HAGyqjcLnhSGA32AgeASgrwUOf9gKlWWsY+LAZmne/QC/+4RgZOqD8UTAN89BUn4DxU69sLx2CjYczEZVjQ548EUhT/pmoNL4ExSMMUMc2FrDfXWj65r4YPzRj4ELPwIyBRz+vhkeri4oKKvGtuO5QI9RgHd/oKYcOL6xdevNWAfBga019J8A2NgBN88BV483njf/DPCjcAqK0W/D1rc/nh3ZHQDw0c9/QKujO622w+uBmqoGCmKM1eHA1hqULkDfJ4T3jd1EqKkCvpoF6KqBXqOB8DkAgKeGBsDTSYFrxZVYsC0Tt4KiAeeuQPkN4FRiGxwAY+0bB7bWUnc6mvU1oG1g5I6f4oEbZ4SuHU98AEgkAAClrQz/fLQPpBJg96nrGL0mDee7Txe2SV0D6PVtcACMtV8c2FpL4INAl+6Atgz4bWf99ReSgSPrhfcxHwKOngarYwb74+vnHkQPTwfcLK3G+MM9USF1EiZrPr+nfnmMMREHttYikQCDpgrv//xgfNkNYOf/Cu9VzwK9HjFaxKAAV+x+4SHMGdEdlRIlPtWOAgBoflpl1mi9jHU2HNha06C/AxIpkJMGFNyew4EI+GYeUH4T8OoHRL7ZaBFKWxlee7QP/vtsBPY7j0c12cKlMBMffv4lSqtqGt2Wsc6KA1trcvYDet5ujdVN0Xd0g9i1AxM+EabxM8GQwC748qXHkeX5KACg58VPEfXeQfx6oaA1as5Yu8aBrbXV3UQ4uRW4fhL4canw8+h/Ad59zSrKTi7DkMlxIEjwiOwE7Eou4X8+PYJ/7jiN8urapgtgrJPgwNbaekUB9h5AWT7w2bjbXTuigPDZzSvPoyckfR4DAKz0/RkA8OWRHEStPoi0S4WWqjVj7RoHttZmI7/zYHxVMeDgBTyxTuza0SzDhA67Q4p/wPa/B8Hf1Q5Xb1ViyseH8f92n0GtjruDsM6NA1tbuG/6nfdGunaYLeB+oNswQF+D+/OTsPelhzAlvBsA4ONfsvHM58dRwjcWWCfGga0tePYGJnwKPLUZ6BXZZHaT1D1mdXwTnFCJhCfD8MHU+6C0leLA+ZuY8EEqcotMnzGLsY6EA1tbCfsb0G+85crrNRrwDAWqS4SRPwA8GuaLbXMj4O2swIUbZXhi3SEcu1xkuX0y1k5wYGuvpFJg2PPC+8MfALVaAMCArq74Zt5w9Pd3RlG5FlM/PoKv0q9asaKMNaHyltBjwII4sLVnYU8BTr5A6XXg9HYx2cdFiW1zIzC2vw+0Oj1e3n4SK/aeg17PTyuwe4BeB+QeAw4sBz55BFjZHUiaZtGnaTiwtWc2CuCB249m/enheHu5Ddb9/T7Mf1iY2u/DA5fw7Bfphv3dqkuBCj5VZW1AcxVI/wzYNh1YGQx8GgkcSACuHgVID8jkFh1ItVmBbd26dQgKCoJSqYRKpcLRo0cbzb99+3aEhoZCqVQiLCwMe/YYPsRNRIiLi4Ovry/s7OwQGRmJCxcuiOsvX76MWbNmITg4GHZ2dujRowfi4+Oh1WoN8kgkknrL4cOHm3OI7ceQpwGFM3DzLHAx2WCVVCrBwqjeeG/SQMhlUqScyUPc2o0o2fsvYOMYYEWQ8CHbu4THeWOArlYIQLlHgd92AKlrgb2vAXteAfYnAEc+Eua7vfgTcO0EcOsyUFVivKWlrQAu/CR8ttaGA+/1A757ATjzDVClARQuQJ9xwOP/Bl46DTx/HLDvYrFDsTF3g6SkJMTGxmL9+vVQqVRYvXo1oqKicP78eXHy4rulpqZiypQpSEhIwGOPPYYtW7YgJiYGJ06cQP/+/QEAK1euxPvvv4/PPvsMwcHBWLp0KaKionDmzBkolUqcO3cOer0eH330EXr27ImsrCzMnj0b5eXleOeddwz299NPP6Ffv37iz+7u7uYeYvuidBGCW+r7wKF/AyFRd9YRAQUXMF57AA93/wE2ualwLK0A/hzrD38A/PGz8IiXmU9DWFV1qTByyultwrH2nyDcoLFztXbN7k2Vt4Cb54XgVXINKMm76zVP6EROzegDKbUF7NyEwGTXRXg++uoxoTN6HYkU8B8ijAjdc5QwfL7M7PBjMrOn31OpVLj//vuxdu1aAMJM8AEBAXj++efx6quv1ss/adIklJeXY9euXWLaAw88gEGDBmH9+vUgIvj5+eHll1/GwoULAQAajQbe3t7YvHkzJk+ebLQeq1atwocffog//hBmXb98+TKCg4ORkZGBQYMGmXQs1dXVqK6+88svKSlBQEBA86ffs5aSPGD1AEBfA0xJEr7wf+wH/jggfHDvUipxxMHavjiMAfjL2IkY5aEBvnlOeChfphAe9Qqf3bIOxK2JSBhUIOMLIajV/GmsO5kCCH0UGPh3oMdfW/XLc8/S1QiDLuT/BuRnCWP+5f9W77NglNQWcPYFnP2FZ52d/YTTxIoiYVrJiiIhQNb9XNtIS9+5K9Dzr0IwCx7R4haZOdPvmfVX12q1SE9Px5IlS8Q0qVSKyMhIpKWlGd0mLS0NsbGxBmlRUVHYuXMnACA7OxtqtRqRkXf6d7m4uEClUiEtLa3BwKbRaNClS/1f1Lhx41BVVYWQkBAsWrQI48aNa/B4EhIS8OabjY+u0S44+wEDJgGZXwBbJxmuk8mBbg8A3R8Guv8Fki798HXSKaScu4H/fFuA+Q/3xPSp++C1L1Y4lf3+FeH1iXWAY/0WuMmIgMu/CmPOFecAfoOAruFA1/sBjxDhrq45SvKAzC1A5pdA0R930t17CsNDSW2E53FvnBFOo37bATh6CzdYBv1dmGSnoyESbhzl/3ZnuXFGaJXpG+ig7RIAuAYKnxkXf8MA5uwvPP5nzt9GW3FXwLv9WlMJdB0q/J2t9A/SrMBWUFAAnU4Hb29vg3Rvb2+cO3fO6DZqtdpofrVaLa6vS2soz59dvHgRa9asMTgNdXR0xLvvvosHH3wQUqkUX331FWJiYrBz584Gg9uSJUsMgm5di61devBFIOsroLYS8BkAdP8L0ONhIOABQG4vZnMEsGH6UCz//iw+/iUba/dfxNr9QDe357DApy/G3fgQsgs/gj6IgCTmA8NTW1PUaoWgkrYWUJ+6k64+BZy4PUy6wgXoOuROoOs6RDiVqVdWtTCoZsaXwKWUO6dJckfhlHPw/wABqjtfnmHPC90GTm4V7hKX5Qv1SFsrTD498O9CoGvpkx/WVKsFrhwSpnf8/Xvhn4YxcichmHv3vf3aH/DqI1y6sCS5vbC4dLVsuS3U7trp165dw5gxY/DUU09h9uw7D5J7eHgYBKn7778feXl5WLVqVYOBTaFQQKFQtHqd24RnCPDiSUAqAxw8Gs0qk0rwz+i+6OPrjE9/zcbZ6yXIuVWJBbdU+FDih3/brkWfilxgy0Sc6zYZktFvoZefF6TSRv77VhQJHYWPbhBaEYAwoc2gvwPBDwkBJ/cYkHcCqNYIM9xf2ndne4+Q20FuKNClhxDQTiUZ3inrNkwIZn2fABSO9esgkQgtQ79BwCNvCxe5T24Bzu8F1KcB9RIgeakwlFS/8YDMVujgXF1613LXz1V/WmffRQiQdy8uAa3fKqkoEkZc/v174GKKUEfxmKWAe6+7glh/wKsv4Nrt3r2c0AbMCmweHh6QyWTIz883SM/Pz4ePj4/RbXx8fBrNX/ean58PX19fgzx/vlaWl5eHhx9+GMOGDcOGDRuarK9KpUJycnKT+ToMJ++m89zlyfu64sn7uqK0qgbpV27haHYRjl12w1O5/w8L9Fsxy+Z7hOYk4vcNv2CS9CW4BA9GeHAXPNLXB8EeDkIhhZeEmw+ZW4Ca249wOXoLE9MM/ced6yp1T13oaoEbvwkXl3OPCa9Fl4CC34Ul88s/HZOvEBwHTQXce5h+cDZy4Vpb6KNCYMj6Sqhj3gkhQPz+vVm/KwCAthQovgKcu3O9GEoXwDsM8Ol/J9h5hgpdcVqi4PYQ8L/vFa4p3n1R38FLaEn3Hiu0zOUOLdtXB9Ssmwfh4eFYs2YNAOHmQbdu3TB//vwGbx5UVFTgu+++E9OGDRuGAQMGGNw8WLhwIV5++WUAwimhl5eXwc2Da9eu4eGHH8aQIUPwxRdfQCaTNVnX2bNnIz09HSdOnDDp2My5ONmRVdXokJlbjPwTuzHyTDxc9UWoJhusrJ2MjboxIEgw0/8a5th+Dx/1AUhw+yPkHQZEzAP6P2neF7u8ELh2/HawOyoEuACV0Drr8VehFWopN88Lp6rZvwC2doDCycjifHu5K03uCJTmAeqs262/08L0isauZUltAI/egEcvYTsbBWCjbPpVIrlzmll40bBM7/5AyBghmPndZ/41yg7AnO+n2YEtKSkJM2bMwEcffYTw8HCsXr0a27Ztw7lz5+Dt7Y3p06fD398fCQkJAITuHiNHjsTy5csRHR2NxMRELFu2zKC7x4oVK7B8+XKD7h6nTp0Su3tcu3YNf/nLXxAYGIjPPvvMIKjVtfg+++wzyOVyDB48GADw9ddfY+nSpfjkk08wc+ZMi//iOo3yAuh3zoP0wl4AwBnFIOgqixEmvSxm+d15GOxGvoiA+6I61+lPrRYoOH870GUJ1xHzsyzT0VRqCwQNB3o/KrTO3AJbXmY712p3RQGhBXbz5k3ExcVBrVZj0KBB2Lt3r3jxPycnB9K7/psMGzYMW7Zsweuvv47XXnsNvXr1ws6dO8WgBgCLFi1CeXk55syZg+LiYgwfPhx79+6FUikMm52cnIyLFy/i4sWL6NrV8CLl3XH57bffxpUrV2BjY4PQ0FAkJSXhb3/7m7mHyO7m4AHp3xOFWeh/+Cf6VmcCUqBWqsB3GIm1lY/g0g1/YLsOg4+kYvL9AXhsgB8cFO3u8q35bOR3Tj/rEAndKtSnhQ6stVXCTRBTX736Ar3HCF0klPzPtbnMbrF1ZNxia8LN88DPKwDPPsDQf0Bn1wW/XLiJxKO5+OlsPmpvP4vqqLDB4wP9MCU8AGH+LpB0plYcazWteirakXFga76bpdX46sRVJB7NweXCO+PA9fV1xrhBfujl5YggDwcEuNlDbtP5rg+xluPA1kwc2FqOiHD4jyIkHsvB91lqaGsNH9GRSSXwd7VDkIcDuns4IMjdHkEeDgj2cIC/qx1sZBz0mHEc2JqJA5tlFVdosTPjGo5eLkJ2QQUuF5SjskbXYH4bqQTdutgj0N0evq528HZSwsdFAS9nJbydlPB2VsDNXt54fzrWYXFgayYObK2LiHCjtBrZBeXILijH5brXwnJcLqyo17ozxlYmgZeTEl7OCvg4K+HtLLz3dFTAw1EBd0c53B0VcHeQQ2lrwW4izOpa9a4oY80lkUjgfTsYPdDdcNQVvZ5wvaQKl28HuvySatwoqUJ+SZXwvrQKBWVa1OgI14orca24ssn9OSps4O4oFwKegxDwPBzl4ntPJwW8nIRXR4UN3+ToQDiwsXuC9Pa1N39XOzzY0/gjYdpaPW6WVSO/pOp20BPeq0uEoFdYVo3CMi0Ky6tRoyOUVdeirLoWV+66mdEQO1sZvJzvBDovJyU8ne4EPy8nJbo4yOFqb8stwXaAAxtrN+Q2UjH4NYaIUFJVKwS6ciHgCYFPCHqFZVrcLKtGQWk1bpRWo6y6FpU1OlwprDApCMptpHCxszW6ONvZwvX2ezcHW/i72sPfzQ6OnaFf3z2Ef9usw5FIJGKg6W7CQB4V2lrcKKnGzbJq3Lh92nujtBo3bwe+GyVVuFlajVsVWujpdsvx9npTudjZoqubEJS7ugnBTngvLC52tnwqbEEc2FinZy+3QZCHDYI8Gn+YnEg4vdVU1txZKmoMf75rKSzTIk9TieK78vyWV2K0bEeFDfxd7eBqbwtHhQ0cFDZwVNoI7+U2cFDI4Hg7zUFxJ93V3hZeTgruJvMnHNgYM5FEIoGT0hZOSlt0NTJ8XEPKqmtx7VYlrt6qwLXiSly9VWnwc0GZFmXVtTifX9qsekklgJeTEr6uSvi52MHXRQlfVzv43fXq4ajoVN1kOLAx1socFTbo7eOE3j5ORtdXanW4VlyJvOJKaCprUH77pkdZde3t9zqU335fevu1Lr24QotaPUF9+yZKBoqN7sNWJtyR9nOxg5uDLVzthBshzna2cLUXfna5/d7FzhYu9rZwasd3ijmwMWZldnIZeno5oqeXkcEzm6DXEwrKqpGnqcL14krx9bqmCnmaSlwvrsKN0irU6AhXbwmtRVPJpBI4K22gtJVBTwSdXjgd1xFBryfoCbfTCUQQ0okgAeDnaocgdwcEutsjyN0B3epeu9jDTt76d5W5g+5duIMu64hqdXrkl1aLAa+4sgaaCq147a/4rmuFxZVCerUJnaWby8dZiUB3+9uLgxgA+/g6Q9bI6TJ30GWMiWxkpnWTuVtVjU4IdBU1qK7VQSqRQCaV3H4VrjfKbqdJJBDXSSUS1Or1uHqrEpcLypFTVIHLhRW4Uig8ZVJaVSueNh/JvjNZt41UgnNvjwFgmVNfDmyMsXqUtjIobWXwdlY2a3tfFzvcH2Q4ixwRobiiBleKhEB3uUB4vVJUASKy6J1dDmyMsTYhkUjg5iCHm4McgwJcW3Vf3PmFMdbhcGBjjHU4HNgYYx0OBzbGWIfTrMC2bt06BAUFQalUQqVS4ejRo43m3759O0JDQ6FUKhEWFoY9e/YYrCcixMXFwdfXF3Z2doiMjMSFCxcM8hQVFWHq1KlwdnaGq6srZs2ahbKyMoM8p06dwkMPPQSlUomAgACsXLmyOYfHGGvvyEyJiYkkl8tp48aN9Ntvv9Hs2bPJ1dWV8vPzjeY/dOgQyWQyWrlyJZ05c4Zef/11srW1pdOnT4t5li9fTi4uLrRz5046efIkjRs3joKDg6myslLMM2bMGBo4cCAdPnyYfvnlF+rZsydNmTJFXK/RaMjb25umTp1KWVlZtHXrVrKzs6OPPvrI5GPTaDQEgDQajbm/FsZYKzPn+2l2YAsPD6d58+aJP+t0OvLz86OEhASj+SdOnEjR0dEGaSqViubOnUtERHq9nnx8fGjVqlXi+uLiYlIoFLR161YiIjpz5gwBoGPHjol5vv/+e5JIJHTt2jUiIvrggw/Izc2NqqurxTyLFy+m3r17N3gsVVVVpNFoxCU3N5cDG2P3KHMCm1n92LRaLdLT07FkyRIxTSqVIjIyEmlpaUa3SUtLQ2xsrEFaVFQUdu7cCQDIzs6GWq1GZGSkuN7FxQUqlQppaWmYPHky0tLS4OrqiqFDh4p5IiMjIZVKceTIEYwfPx5paWkYMWIE5HK5wX5WrFiBW7duwc2t/nAMCQkJePPNN+ull5QYH1qGMWY9dd9LMuEpULMCW0FBAXQ6nTjrex1vb2+cO3fO6DZqtdpofrVaLa6vS2ssj5eXl2HFbWzQpUsXgzzBwcH1yqhbZyywLVmyxCDoXrt2DX379kVAQIDRY2GMWV9paSlcXFwazdOpnzxQKBRQKBTiz46OjsjNzYWTk1OTw7WUlJQgICAAubm5neqBeT5uPm5rISKUlpbCz8+vybxmBTYPDw/IZDLk5+cbpOfn58PHx8foNj4+Po3mr3vNz8+Hr6+vQZ5BgwaJeW7cuGFQRm1tLYqKigzKMbafu/fRFKlUiq5du5qUt46zs7PV/+DWwMfdudwrx91US62OWd095HI5hgwZgpSUFDFNr9cjJSUFERERRreJiIgwyA8AycnJYv7g4GD4+PgY5CkpKcGRI0fEPBERESguLkZ6erqYZ9++fdDr9VCpVGKegwcPoqamxmA/vXv3NnoayhjrwMy9M5GYmEgKhYI2b95MZ86coTlz5pCrqyup1WoiIpo2bRq9+uqrYv5Dhw6RjY0NvfPOO3T27FmKj4832t3D1dWVvvnmGzp16hQ98cQTRrt7DB48mI4cOUK//vor9erVy6C7R3FxMXl7e9O0adMoKyuLEhMTyd7e3qzuHuborF1D+Lj5uNsDswMbEdGaNWuoW7duJJfLKTw8nA4fPiyuGzlyJM2YMcMg/7Zt2ygkJITkcjn169ePdu/ebbBer9fT0qVLydvbmxQKBY0aNYrOnz9vkKewsJCmTJlCjo6O5OzsTDNnzqTS0lKDPCdPnqThw4eTQqEgf39/Wr58eXMOzyRVVVUUHx9PVVVVrbaPexEfNx93e8Aj6DLGOhx+VpQx1uFwYGOMdTgc2BhjHQ4HNsZYh8OBjTHW4XBgawZzx6Nr79544w1IJBKDJTQ01NrVahUHDx7E448/Dj8/P0gkEnGwhjpkwtiB7VFTx/3000/X+wyMGTPGOpU1AQc2MyUlJSE2Nhbx8fE4ceIEBg4ciKioqHqPfHU0/fr1w/Xr18Xl119/tXaVWkV5eTkGDhyIdevWGV2/cuVKvP/++1i/fj2OHDkCBwcHREVFoaqqqo1rallNHTcAjBkzxuAzsHXr1jasoZms3I+u3TF3PLqOID4+ngYOHGjtarQ5ALRjxw7xZ1PGDuwI/nzcREQzZsygJ554wir1aQ5usZmhbjy6u8eOa2o8uo7iwoUL8PPzQ/fu3TF16lTk5ORYu0ptrqmxAzu6AwcOwMvLC71798b//u//orCw0NpVahAHNjM0Nh5d3bhwHZFKpcLmzZuxd+9efPjhh8jOzsZDDz2E0tJSa1etTZkydmBHNWbMGHz++edISUnBihUr8PPPP2Ps2LHQ6XTWrppRnXo8NmaasWPHiu8HDBgAlUqFwMBAbNu2DbNmzbJizVhbmTx5svg+LCwMAwYMQI8ePXDgwAGMGjXKijUzjltsZmjOeHQdkaurK0JCQnDx4kVrV6VN3T124N06298fALp37w4PD4979jPAgc0MzRmPriMqKyvDpUuXDAYG7QxMGTuws7h69SoKCwvv2c8An4qaKTY2FjNmzMDQoUMRHh6O1atXo7y8HDNnzrR21VrNwoUL8fjjjyMwMBB5eXmIj4+HTCbDlClTrF01iysrKzNohWRnZyMzMxNdunRBt27d8NJLL+Ff//oXevXqheDgYCxduhR+fn6IiYmxXqUtoLHj7tKlC958801MmDABPj4+uHTpEhYtWoSePXsiKirKirVuhLVvy7ZHjY1H1xFNmjSJfH19SS6Xk7+/P02aNIkuXrxo7Wq1iv379xOAekvdGIOmjB3YHjV23BUVFTR69Gjy9PQkW1tbCgwMpNmzZ4uDy96LeDw2xliHw9fYGGMdDgc2xliHw4GNMdbhcGBjjHU4HNgYYx0OBzbGWIfDgY0x1uFwYGOMdTgc2BhjHQ4HNsZYh8OBjTHW4fx/aDEsd94SHRgAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:52:46.568373Z", + "iopub.status.busy": "2024-03-01T00:52:46.568049Z", + "iopub.status.idle": "2024-03-01T00:55:38.216800Z", + "shell.execute_reply": "2024-03-01T00:55:38.215940Z" + }, + "papermill": { + "duration": 171.671616, + "end_time": "2024-03-01T00:55:38.219364", + "exception": false, + "start_time": "2024-03-01T00:52:46.547748", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:55:38.261776Z", + "iopub.status.busy": "2024-03-01T00:55:38.260873Z", + "iopub.status.idle": "2024-03-01T00:55:38.282545Z", + "shell.execute_reply": "2024-03-01T00:55:38.281648Z" + }, + "papermill": { + "duration": 0.045776, + "end_time": "2024-03-01T00:55:38.284782", + "exception": false, + "start_time": "2024-03-01T00:55:38.239006", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.005121NaN0.0012482.6234390.029070.5664460.0389110.0000023.2385040.0275670.0644360.0353290.0602520.0050635.861943
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.005121 NaN 0.001248 2.623439 0.02907 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.566446 0.038911 0.000002 3.238504 0.027567 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.064436 0.035329 0.060252 0.005063 5.861943 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:55:38.326980Z", + "iopub.status.busy": "2024-03-01T00:55:38.326311Z", + "iopub.status.idle": "2024-03-01T00:55:38.676850Z", + "shell.execute_reply": "2024-03-01T00:55:38.675651Z" + }, + "papermill": { + "duration": 0.373307, + "end_time": "2024-03-01T00:55:38.679429", + "exception": false, + "start_time": "2024-03-01T00:55:38.306122", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:55:38.727223Z", + "iopub.status.busy": "2024-03-01T00:55:38.726145Z", + "iopub.status.idle": "2024-03-01T00:58:38.690068Z", + "shell.execute_reply": "2024-03-01T00:58:38.689045Z" + }, + "papermill": { + "duration": 180.00607, + "end_time": "2024-03-01T00:58:38.708335", + "exception": false, + "start_time": "2024-03-01T00:55:38.702265", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:58:38.750863Z", + "iopub.status.busy": "2024-03-01T00:58:38.750521Z", + "iopub.status.idle": "2024-03-01T00:58:38.768614Z", + "shell.execute_reply": "2024-03-01T00:58:38.767729Z" + }, + "papermill": { + "duration": 0.042796, + "end_time": "2024-03-01T00:58:38.770673", + "exception": false, + "start_time": "2024-03-01T00:58:38.727877", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:58:38.810326Z", + "iopub.status.busy": "2024-03-01T00:58:38.810027Z", + "iopub.status.idle": "2024-03-01T00:58:38.815386Z", + "shell.execute_reply": "2024-03-01T00:58:38.814407Z" + }, + "papermill": { + "duration": 0.027611, + "end_time": "2024-03-01T00:58:38.817351", + "exception": false, + "start_time": "2024-03-01T00:58:38.789740", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.43508143286059675}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:58:38.858007Z", + "iopub.status.busy": "2024-03-01T00:58:38.857712Z", + "iopub.status.idle": "2024-03-01T00:58:39.193585Z", + "shell.execute_reply": "2024-03-01T00:58:39.192623Z" + }, + "papermill": { + "duration": 0.358474, + "end_time": "2024-03-01T00:58:39.195785", + "exception": false, + "start_time": "2024-03-01T00:58:38.837311", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABATElEQVR4nO3deXhTdb4H/vfJyb53X2hLyyIg+y4yCCjXBVwYZ/zpD1QKos4VxoXhPsqdR5xxQ72j4iyXOzpK9cqizk9nvKI4jAoOKihbAcFi6Ur3LVuz55zfHycJhC40aZKTpJ/X8+RJenJy8mnafvrdvwzP8zwIIUREErEDIIQQSkSEENFRIiKEiI4SESFEdJSICCGio0RECBEdJSJCiOgoERFCREeJiBAiOkpEpF9lZWVgGAY1NTVih0JSGCUiEnXbt2/H5s2bxQ6DJBFKRCTqKBGRcFEiIoSIjhIRCdsnn3yC+fPnQ6fTQa/XY+bMmdi+fTsAYMGCBdi1axdqa2vBMAwYhkFxcfGAr81xHH7zm98gPz8farUaCxcuxKlTp1BcXIzS0tLgeZ2dnVi/fj0mTpwIrVYLvV6PG264AeXl5SHX27t3LxiGwbvvvotnnnkGBQUFUCqVuOaaa1BZWRmNj4NEgVTsAEhyKSsrw6pVqzB+/Hhs2LABRqMRR48exe7du7Fs2TL8+te/htlsxrlz5/Dyyy8DALRa7YCvv2HDBrzwwgu46aabcN1116G8vBzXXXcdnE5nyHlVVVX429/+httuuw0lJSVoaWnBn//8Z8yfPx+nTp1Cfn5+yPnPPfccJBIJ1q9fD7PZjBdeeAHLly/HwYMHB/+hkMHjCenH1q1beQB8dXU1bzKZeJ1Ox8+ePZt3OBwh53EcF3y8ZMkSfvjw4WG/V3NzMy+VSvmlS5eGHP/Nb37DA+BXrFgRPOZ0OnmfzxdyXnV1Na9QKPgnn3wyeOyLL77gAfDjxo3jXS5X8Pgrr7zCA+BPnDgRdpwk+qhqRgZsz549sFqteOyxx6BUKkOeYxhm0Nf/7LPP4PV68cADD4Qc/+Uvf9njXIVCAYlE+PX1+Xzo6OiAVqvFmDFjcOTIkR7nr1y5EnK5PPj1vHnzAAglKyI+SkRkwM6ePQsAmDBhQkyuX1tbCwAYNWpUyPH09HSkpaWFHOM4Di+//DJGjx4NhUKBzMxMZGVl4fjx4zCbzT2uXVRUFPJ14HpdXV3R/BZIhCgRkaT07LPPYt26dbjqqqvw9ttv49NPP8WePXswfvx4cBzX43yWZXu9Dk8rJScEaqwmAzZy5EgAwMmTJ3uUWi4UaTVt+PDhAIDKykqUlJQEj3d0dPQoufz1r3/FwoUL8frrr4ccN5lMyMzMjOj9iXioREQG7Nprr4VOp8OmTZt69GJdWLLQaDS9Vo8u5ZprroFUKsWWLVtCjv/xj3/scS7Lsj1KM++99x4aGhrCfl8iPioRkQHT6/V4+eWXsXr1asycORPLli1DWloaysvLYbfb8eabbwIApk+fjnfeeQfr1q3DzJkzodVqcdNNN13y+jk5OXjooYfw4osv4uabb8b111+P8vJyfPLJJ8jMzAwpad1444148sknsXLlSlx55ZU4ceIEtm3bhhEjRsTs+ycxJHKvHUlwF3bfB3z44Yf8lVdeyatUKl6v1/OzZs3id+zYEXzeZrPxy5Yt441GIw8grK58r9fLP/7443xubi6vUqn4q6++mj99+jSfkZHB/+IXvwie53Q6+V/96ld8Xl4er1Kp+Llz5/LffPMNP3/+fH7+/PnB8wLd9++9917I+1RXV/MA+K1bt4b7kZAYYHieWutIYjOZTEhLS8PTTz+NX//612KHQ2KA2ohIQnE4HD2OBSbQLliwIL7BkLihNiISF21tbfD5fH0+L5fLkZ6ejnfeeQdlZWVYvHgxtFot9u/fjx07duDaa6/F3Llz4xgxiSdKRCQuZs6cGRyw2Jv58+dj7969mDRpEqRSKV544QVYLJZgA/bTTz8dx2hJvFEbEYmLr776qtdqV0BaWhqmT58ex4hIIqFERAgRHTVWE0JEl9RtRBzHobGxETqdLiqzvwkh0cXzPKxWK/Lz84OrJfQmqRNRY2MjCgsLxQ6DEHIJ9fX1KCgo6PP5pE5EOp0OgPBN6vV6kaMhhFzMYrGgsLAw+Lfal6RORIHqmF6vp0RESAK7VNMJNVYTQkRHiYgQIjpKRIQQ0SV1GxEh0cDzPLxeb79z4UjvWJaFVCod9PAZSkRkSHO73WhqaoLdbhc7lKSlVquRl5cXsktKuCgRkSGL4zhUV1eDZVnk5+dDLpfTwNgw8DwPt9uNtrY2VFdXY/To0f0OWuwPJSKStOo77ThQ1YFsvRLzRmVCIgkvibjdbnAch8LCQqjV6hhFmdpUKhVkMhlqa2vhdrt77Hc3UJSISFKyu734sLwRbi+Hc10OyFkJ5ozMiOhakf4XJ4JofH70EyBJqbzeDLdX2L8sS6eARtH7vmUkOVCJiCSlylYrAOD6CbkYl0ej6pMdlYhI0rE4PWi3ucEwQEmmRuxwUl5xcXFw3fBYoUREkk5dh9DVnqtXQikTqmQOtw+VrTbaQjpJUdWMJJ2STA1unJQHKSv8H+U4Hm98VQ23l8Ndc4YjU6sQOcLE43a7BzXOJ9aoRESSjkYhxegcXbBaJpEwyNUL3cZNJmd/Lx0wt5fr8+b1cQM+1zOAcyOxYMECrF27FmvXroXBYEBmZiYef/zxYImwuLgYTz31FO6++27o9Xrcd999AID9+/dj3rx5UKlUKCwsxIMPPoju7u7gdVtbW3HTTTdBpVKhpKQE27Ztiyi+cFGJiKSEHL0SdZ12tFqdAAyDvt6fvqjs87mSTA2WTh0W/PrVL8/C4+u9SliQpsJtM84v3vfGV9VwuEOnkjzyb5dFFOObb76Je+65B99++y0OHTqE++67D0VFRbj33nsBAL/73e+wceNGPPHEEwCAs2fP4vrrr8fTTz+NN954A21tbcFktnXrVgBAaWkpGhsb8cUXX0Amk+HBBx9Ea2trRPGFgxIRSSpWpwdnWqzI0ChQfEFDdbZeqI61WFxihRZ3hYWFePnll8EwDMaMGYMTJ07g5ZdfDiaiq6++Gr/61a+C569evRrLly/Hww8/DAAYPXo0fv/732P+/PnYsmUL6urq8Mknn+Dbb7/FzJkzAQCvv/46xo0bF/PvhRIRSSotFhe+PNOOHL0yJBHl6ISqWbvNBR/Hgw1zlPXF1iwc1edzF1/6vqtG9nnuxTNGVs0tGUxYIa644oqQKSlz5szBiy++GJy8O2PGjJDzy8vLcfz48ZDqFs/zwakuZ86cgVQqDdnWaezYsTAajVGLuS+UiEhSMdndAIA0tSzkuF4lhVLGwunxocPmQrY+sqkGAXLpwJtPY3XuYGk0oUMbbDYb7r//fjz44IM9zi0qKsKZM2fiFVoPlIhIUumyewAARnVoDxDDMMjQytHQ5UBHt3vQiSgZHDx4MOTrAwcOYPTo0WDZ3keZT5s2DadOncKoUb2X9saOHQuv14vDhw8Hq2YVFRUwmUxRjbs31GtGkkpXoESkkfV4blqREdeOz8GwNFW8wxJFXV0d1q1bh4qKCuzYsQN/+MMf8NBDD/V5/qOPPoqvv/4aa9euxbFjx/Djjz/i73//O9auXQsAGDNmDK6//nrcf//9OHjwIA4fPozVq1dDpYr950mJiCSV81WznmNiRmXrMD7fAL2yZ5JKRXfffTccDgdmzZqFNWvW4KGHHgp20/dm0qRJ2LdvH86cOYN58+Zh6tSp2LhxI/Lz84PnbN26Ffn5+Zg/fz5uvfVW3HfffcjOzo7590JVM5I03F4O3S6hIdagGhrJpj8ymQybN2/Gli1bejxXU1PT62tmzpyJf/zjH31eMzc3Fx999FHIsbvuumtQcQ4ElYhI0rA4hfYhhUwSnNpxIZ7nUd9pR3m9qcegQ5LYqEREkoZRJcOy2UVwefpOMv93vBEuD4d8owpZOprqkSwoEZGkIWUlyOmnN4xhGKSr5WgyO2Gyu1M6Ee3du1fsEKKKqmYkpRj944vMDo/IkZBwUImIJI3TTRbY3T6UZGqQrul9Jrne34htslMiSiaUiEjSONFgRkOXA1qFtM9EZFQJx01UIkoqVDUjScPq9AIAdMq+/38GqmaB8UYkOVAiIkmB43jYwkhENpeXuvCTCFXNSFKwe3zgeB4ShoFG3vevrUrG4trxOTCoZLRZYhKhRESSQrdLKA2p5Wy/GykyDIPx+YNfGI3El6hVM5/Ph8cffxwlJSVQqVQYOXIknnrqKVoAnfRg8ycijYL+d6YiUX+qzz//PLZs2YI333wT48ePx6FDh7By5UoYDIZe10whQ5fdP8dsIBspmuxunOtyQCljMSpbG+vQSBSImoi+/vpr3HLLLViyZAkAYcHvHTt24NtvvxUzLJKARudoka1XQDKAdp/6Tgf+eboFxZnq8BMRzwM+Ebr+WVnP5Rz78NZbb+GRRx5BY2MjFIrzo8eXLl0KnU6H//3f/41VlDEjaiK68sor8eqrr+LMmTO47LLLUF5ejv379+Oll17q9XyXywWX6/yaxBaLJV6hEpEpZWyvE117E5iZb3F4w38jnwf414vhv26w5v0KkA5su5/bbrsNDz74ID788EPcdtttAITdN3bt2tXvzPpEJmob0WOPPYY77rgDY8eOhUwmw9SpU/Hwww9j+fLlvZ6/adMmGAyG4K2wsLDX88jQplcJ/1+tTk9KtjeqVCosW7YsuPMGALz99tsoKirCggULxAtsEEQtEb377rvYtm0btm/fjvHjx+PYsWN4+OGHkZ+fjxUrVvQ4f8OGDVi3bl3wa4vFQsloiDha1wWO53FZjg66Syx8plVIwTCAx8fD4fFB3U93fw+sTCidxBsb3vpK9957L2bOnImGhgYMGzYMZWVlKC0tTdohC6Imov/4j/8IlooAYOLEiaitrcWmTZt6TUQKhSKkTkyGjsO1XbA6vRhmVF8yEUlZCTRyKWwuLywOb3iJiGEGXEUS09SpUzF58mS89dZbuPbaa/H9999j165dYocVMVETkd1uh0QSWjtkWRYcRyNiyXk8z8Pu35RQPYBeM0ContlcXlicHuQaUnMh/dWrV2Pz5s1oaGjAokWLkrp2IGob0U033YRnnnkGu3btQk1NDT744AO89NJL+OlPfypmWCTBOD0cfJzQ1tPfqOoLBdattjpTd/LrsmXLcO7cObz22mtYtWqV2OEMiqgloj/84Q94/PHH8cADD6C1tRX5+fm4//77sXHjRjHDIgkmMJhRJWcHvHHi9OFpmDDMgAxt4lezImUwGPCzn/0Mu3btwtKlS8UOZ1BETUQ6nQ6bN2/G5s2bxQyDJDi72z+qWj6wahmAIbGvGQA0NDRg+fLlSd92SuPlScKj6R09dXV1Ye/evdi7dy/++7//W+xwBo1+siThOQIN1WH0fnl8HH5ossLm8uKKEelJ263dl6lTp6KrqwvPP/88xowZI3Y4g0aJiCS8CcMMKExXQ8aG17fyz9MtAICpRcYBj8pOFn3tW5asKBGRhBfO9I4AGSuBWs7C7vbB4vCkXCJKNbRCI0lZgYX0Lc7+55yl4jSQeIrG50eJiCS8I3VdOFTTGdzpdaACY4n6ep1MJjxvt9sHF+AQF/j8Ap9nJKhqRhLe0ToTLA4PhqWpgsllIAJrW1v62NGDZVkYjUa0trYCANRqdco1ascSz/Ow2+1obW2F0WgEy0Ze/aVERBKewz+OSBVmO89Aqma5ubkAEExGJHxGozH4OUaKEhFJaG4vB49PaINQhTGgEThfIrL1k4gYhkFeXh6ys7Ph8aTudJBYkclkgyoJBVAiIgnN4RHGEEklDORhdt/nG1T4+fSCAVXnWJaNyh8UiQwlIpLQAoMZVXI27PYblZxFYbo6FmGRKKNeM5LQAvPMwq2WkeRCiYgktEDVLNyG6oAzLVZ8XdmOzm7agjqRUdWMJLSRWVosn62IuFv9+Dkz6jvtSNPIka5J3SVBkh0lIpLQIpnecSGtf8Z+YAY/SUxUNSMpLdCFn8orNaYCKhGRhHaywQyHx4dRWVqkRVC1Op+IqESUyCgRkYR2osGMZrMT6Rp5RImIqmbJgapmJKEFdu+ItNdMSyWipECJiCS0wDwzdYTjiAKjqh1uH7w+2qYqUVHVjCQsjy/yeWYBCqkEP59eAJ1SOuAdQEj8USIiCStQLWP7mmfG80BHJdD2AyBTAQUzAaUh5BSGYWiaRxKgREQSltMTWDS/l3lmPA9U7QXqDpw/1nwCmLIc0GbHL0gSFdRGRBJWoETU64DGlu/PJ6H8qYAuB/A4ge//BnC+kFPrO+34urIdVW22GEdMIkWJiCSsYUYVls8uwr9dnhP6hNsOVO4RHhf/BBhzPTDpDkCuBuwdwLlDIafXd9lxsLoT1e3dcYqchIsSEUlYcqkE2Xolci7etbX+gFD60WYDw+f6T1YDJfOFx+e+DSkV6RRCzxmNJUpclIhIcnF3Aw2Hhccl8wHJBb/CuRMBhRZw2YDW08HDgbFEl9rNg4iHEhFJWBXNVnxX04lWq/P8waZywOcFdLlAxsjQF0hYIG+K8Ljl++DhgSwZS8RFiYgkrB+aLdj/YztaLS7hAMcBjUeFxwUzgN6WBskZL9x31QhtSTg/zcPp8cFDgxoTEiUikrB69Jp1ngWcFkCmBLLG9f4idbrQg8Zzwvgi/+vlUuFXnaZ6JCZKRCRhBdarDk7vaCoX7nMnAWw/Q+Cyxgr3HWeDh6h6lthoQCNJWCHLxHocQGeV8ETe5P5fmD4SqNoHmGqE9iRWihsm5EHGMtCFsUEjiR8qEZGE5PFxcHuF9hyVnAXaKoQueW0WoMns/8XabECuEZKQuR4AkKVTwKiW03yzBEWJiCSkQGmIlTBQSCVA6ynhiezxl34xwwDpI4THgVIUSWiUiEhCclywDhHjtgGmOuGJ7D4aqS+WVizc+0tE7TYXvq5sR3m9KbqBkqgQPRE1NDTgzjvvREZGBlQqFSZOnIhDhw5d+oUkpaVr5Fg+uwhLJuUJ1TKeB/T5gMo4sAsYC4V7awvgdcNkd+NgdSdON1liFjOJnKiN1V1dXZg7dy4WLlyITz75BFlZWfjxxx+RlpYmZlgkAchYYXoHAKC6QrjPvnzgF1AahJvTDFgaoFHkAqBpHolK1ET0/PPPo7CwEFu3bg0eKykp6fN8l8sFl8sV/Npiof9uKc9tB8znhMeZo8N7raFASETmemjyCgAA3S4feJ6PeJ80EhuiVs0+/PBDzJgxA7fddhuys7MxdepUvPbaa32ev2nTJhgMhuCtsLAwjtGSeKpu78Z3NZ1oqzstDE7UZg28WhYQqJ6Z6qGRS8EwAMfzwYZwkjhETURVVVXYsmULRo8ejU8//RT//u//jgcffBBvvvlmr+dv2LABZrM5eKuvr49zxCReKltt2P9jO0x1/t6yjDBLQwBgCLQTNYIFHxwYSYMaE4+oVTOO4zBjxgw8++yzAICpU6fi5MmT+J//+R+sWLGix/kKhQIKhSLeYRIR2N1eMLwXOnsdoJGEXy0DAFU6IJUDXjdgb4dGIUW3ywebywtawzGxiFoiysvLw+WXhzZAjhs3DnV1dSJFRBKF0+OD3tkMOTzC4ERdXvgXkUgArdBIDWtTcPJrt4uqZolG1BLR3LlzUVFREXLszJkzGD58uEgRkURhd/uQ5qiFVO8vDUXauKzPE8YgWZow/7JxWHBZNjSKyHYEIbEjaonokUcewYEDB/Dss8+isrIS27dvx6uvvoo1a9aIGRZJAA63F2mOWshYSWTtQwGBkpS1EUa1HAa1DNLedgQhohL1JzJz5kx88MEH2LFjByZMmICnnnoKmzdvxvLly8UMi4jM6+PAOLqg8NkglcmAtEGUkAOJyNYmzD0jCUn02fc33ngjbrzxRrHDIAnE4fHB6DwHBoA0rRBgBzFjXmkQ1rN222HtaMAJiwZggCtHXmLiLImriEpEVVU0kZDEjlouxeICB8bl6cEEJq9GimGCpSKP6RwOVnfixDlzFKIk0RRRIho1ahQWLlyIt99+G06n89IvICQMLO+DwdkIg0p2fhb9YPgTkcbVDkBoCPdx/OCvS6ImokR05MgRTJo0CevWrUNubi7uv/9+fPvtt9GOjQxVlnNCe45cA2iyBn89rbAvmtzZHlyPqNtN7UWJJKJENGXKFLzyyitobGzEG2+8gaamJvzkJz/BhAkT8NJLL6GtrS3acZIhpK3uBzSYHGiX50febX8h/xbUjL0DWrk/EdHk14QyqF4zqVSKW2+9Fe+99x6ef/55VFZWYv369SgsLMTdd9+NpqamaMVJhhBLwxnUddpR5YvS+GelAZAqAM6HdEaYKE2JKLEMKhEdOnQIDzzwAPLy8vDSSy9h/fr1OHv2LPbs2YPGxkbccsst0YqTDBXubjC2FuFxYHGzwWKYYKkojTMBoN08Ek1E3fcvvfQStm7dioqKCixevBhvvfUWFi9eDIl/182SkhKUlZWhuLg4mrGSocBUBw/Hwy5LR6ZaH73ranMAUz2MXBeAAprmkWAiSkRbtmzBqlWrUFpairy83ucAZWdn4/XXXx9UcGQIMtXB6+NgVuajSB7FqRj+EtFlGjuKx5fQNI8EE1Ei2rNnD4qKioIloACe51FfX4+ioiLI5fJeZ9AT0i9THTw+HlZt3vmNFaPB33OmcrVDpZJGpxGcRE1EbUQjR45Ee3t7j+OdnZ39rrBISL/c3UB3Ozw+DhZF3vmNFaNBnQkwEsDjBFzW6F2XREVEiYjnex8MZrPZoFQqBxUQGcJM9eB4HhbWCC+rFPYzixZWCqjT4eU4HD5Vgc9/aInetcmghVU1W7duHQCAYRhs3LgRarU6+JzP58PBgwcxZcqUqAZIhhBzPRgGmD55MkYXFEApjXI7jjYHsLbibNVZNBgMuGp0Fs3ETxBhJaKjR48CEEpEJ06cgFwuDz4nl8sxefJkrF+/ProRkqHDVAsGDIx5I2HMUF/6/HBpc8BKTkLn7QQAdLt9MKgoESWCsBLRF198AQBYuXIlXnnlFej1UexeJUObxwF0+9sdDTHaFEGbBQYMDLww6dXu9grz2YjoIuo1u3D7H0KiwlQP8DysEj1ON7qRqbVhRJY2uu/hn7em4yxgeC+Nrk4gA05Et956K8rKyqDX63Hrrbf2e+77778/6MDIEGMW1ilvZXPwVWU7LsvRRT8RybWATAk5a4XKY0K3Kz+61ycRG3AiMhgMwU3pDAZDzAIiQ5S5AQBgUeQCDkAdiwGHDANosiBn26H2dNEM/AQy4ER0YXWMqmYkqnxewD+/rEsqrJyokcdo8VBNFmRsBdSeTprmkUAi+mk7HA7wPB/svq+trcUHH3yAyy+/HNdee21UAyRDgK0Z4HyAXA0LrwVgj+5gxgtpspCjV+IGAwv5mCisdUSiIqK+y1tuuQVvvfUWAMBkMmHWrFl48cUXccstt2DLli1RDZAMAZZG4V4/DN3+7aBjmYhkrARqbxeNIUogEa/QOG/ePADAX//6V+Tm5qK2thZvvfUWfv/730c1QDIEmM8J9/phcLiFRKRRxKpq5l8032kRpnuQhBBRIrLb7dDpdACAf/zjH7j11lshkUhwxRVXoLa2NqoBkiHAXyLi9XnBdpuYlYhkKvjkWtR12rHv2GlwtHZ1Qoh48fy//e1vqK+vx6effhpsF2ptbaVBjiQ8ToswCZWRALo8LL+iCD+bVhC7xmoAjCYLjSYHGupr4fBQg3UiiCgRbdy4EevXr0dxcTFmz56NOXPmABBKR1OnTo1qgCTFWYRue2izwEgVyNQqUJShhkQSu2U6JLpsSFkGKurCTxgR/dv5+c9/jp/85CdoamrC5MmTg8evueYa/PSnP41acGQICCQi/bD4vacmC3JWArWnE3aXD9DF761J7yIu/+bm5iI3Nzfk2KxZswYdEBlizOcTUYvFiZr2bmTrlSjJ1MTuPTVZkEklUDu7YHN6Yvc+ZMAiSkTd3d147rnn8Nlnn6G1tRUcx4U8TzvBkgG5YCAj9Pk412bH12c7MDZXF9tEpM6AjGUh5Zxw2q0AjLF7LzIgESWi1atXY9++fbjrrruQl5cXnPpBSFhsLcJARpkKUKXB7hZm36tj1XUfwMogUacB1iZ4zM0AYjTbnwxYRD/xTz75BLt27cLcuXOjHQ8ZSgLtQ4YCgGGCXfeaWHXdX4DRZgMtTfDZWmP+XuTSIkpEaWlpSE9Pj3YsZKgJNlQLs+Dt/h4sdQy77gOGFRQhz1MLWQ6NI0oEEXXfP/XUU9i4cSPsdnu04yFDiTk0EXW7YzyY8QJyfQ6UUhaso+cmECT+IvrX8+KLL+Ls2bPIyclBcXExZLLQVe6OHDkSleBICgsOZGQAnT8R+Rcqi9n0jgv5F0lDdxvA87S9kMgi+okvXbo0ymGQIScw0VWTBUjl8Pq44DwznTL2iYhXGVHX5YLHa0OhrQMKXWbM35P0LaKf+BNPPBHtOMhQY/FPdDUUAAAkDIO75wyHzeWFQhr7WfGMhEW1QwWF04bMrhZKRCKL+CduMpnwl7/8BRs2bEBnp7ArwpEjR9DQ0BDR9Z577jkwDIOHH3440pBIMgku/SFUyyQSBhlaBYZnaOI2HIRTC8nHbW6Oy/uRvkVUIjp+/DgWLVoEg8GAmpoa3HvvvUhPT8f777+Purq64FpFA/Xdd9/hz3/+MyZNmhRJOCTZ+LyA1f/HH8+pHRfTZAKdgMdCmy2KLaIS0bp161BaWooff/wxZGfXxYsX48svvwzrWjabDcuXL8drr72GtLS0SMIhyeaigYwAUNvRjQNVHTjXFb+eWIkuBwDgs7bF7T1J7yJKRN999x3uv//+HseHDRuG5ubwirlr1qzBkiVLsGjRokue63K5YLFYQm4kCV00kBEAqtq78c3ZDtS0xy8RsTqh54y3dwilNCKaiKpmCoWi1yRw5swZZGUNfB3gnTt34siRI/juu+8GdP6mTZvw29/+dsDXJwnqooGMAGBzColAG4ceswCVxginRA6PxwvYOwB/CYnEX0QloptvvhlPPvkkPB5h5jLDMKirq8Ojjz6Kn/3sZwO6Rn19PR566CFs27YtpHrXnw0bNsBsNgdv9fX1kYRPxGbuufSHzT+GSBuLbYT6oFbI0C1Lh9vHC+OJiGgiSkQvvvgibDYbsrKy4HA4MH/+fIwaNQo6nQ7PPPPMgK5x+PBhtLa2Ytq0aZBKpZBKpdi3bx9+//vfQyqVwufruXKeQqGAXq8PuZEkc9GKjAHdwUQUvy2gR2Rp8JMp4zA2Vwd005wzMUVUDjYYDNizZw+++uorlJeXw2azYdq0aQNq5wm45pprcOLEiZBjK1euxNixY/Hoo4+CZeP3n5HE0QUrMkIqBwBwHB+c8BrPqplSxgJpeUDbcaCbpnqIKeyfOsdxKCsrw/vvv4+amhowDIOSkhLk5uaC5/kBjwHR6XSYMGFCyDGNRoOMjIwex0kK6WVFRqvLC47nwUoYqGVx/gd04VQPIpqwqmY8z+Pmm2/G6tWr0dDQgIkTJ2L8+PGora1FaWkpLRNLLq2X9iGLQ2hr1CmlMV2rujfftEpQ2WqDu9tE2wuJKKwSUVlZGb788kt89tlnWLhwYchzn3/+OZYuXYq33noLd999d0TB7N27N6LXkSRx0YqMAflGFVZcWQy3l+vjhbHzfYsLoxwy5Hh9kHe3AUZaJE0MYZWIduzYgf/8z//skYQA4Oqrr8Zjjz2Gbdu2RS04kmKsTcGtpQMDGQGAlTBI18iRaxhY72k0aRRSOGRp8Hip50xMYSWi48eP4/rrr+/z+RtuuAHl5eWDDoqkKFOdcG8sSphlNzQKKezydHh8HCUiEYVVNevs7EROTt+DvnJyctDV1TXooEiKCiQiQ1HI4e9qOsFxPMbm6WFQxa/7HhCWpW2UpcNNiUhUYSUin88HqbTvl7AsC6+XhsqTXnC+80t/GEMTUXm9CVanF0UZ6rgnIrVcCrssDZ5uDrC10iJpIgkrEfE8j9LSUigUil6fd7lcUQmKpCBrk9BYLVMJs979vD4uOKo63kkIALQKKRwyI1wcAK9LGGyppIGy8RZWIlqxYsUlz4m0x4ykuGD7UGFIicPs8IDnAblUAlW8xxABUCtY8IwUNokegL96Roko7sJKRFu3bo1VHCTVBRPR8JDDXXY3ACBNLRdlf7zCNDXumVcCzY+jgfYKIRFljIx7HENd7NfkJMTnAUz+CcoXJaLObmEwY7om/tUyQCiJ6ZUysIGZ99RgLQpKRCT2THUA5wUUupD2IQDo7D5fIhIVTfUQVfxmGJKhq7NKuM8Y2aNHyuSvmqVrxEtEB6s6YDNxmOvxQdndAXAcIKH/0fFEiYjEXsdZ4T69Z9vLz6cXwOzwxGcvsz6cabWh3cJgGsdCyXkBRxegyRAtnqGI0j6JLXun8IctYYG04T2elrISZGgVwpIcItEqWIBhYJcZhQO0NlHcUSIagng+jvu9t58R7g2FgLT38WdiCyzGZpOmCweonSjuqGo2hPA8j2/OduBwbRe0SimuG5+LfKMqtm/aekq4zx7b46nyehOaLU5cnqdHYbo6tnH0Q+NfntYiMQIcKBGJgEpEQ8jxc2YcrO6El+NhdnjQbInx+jv2TsDaIiwLmzmmx9PV7d041WgJ9pyJRetvnzJJDMIBGyWieKMS0RDh8XH4+mwHAGBWSTpGZGmQZ4hTaSitWFj64wI8zwcTYbZe3CpboKG8E0bhgKNLmO6RoFXJVEQloiHihyYrnB4fDCoZ5ozIiH0S4jigyb8kTM74Hk93dLvhcPsgYxlk6+K/DtGFAiUii08ujHUChAmwJG6oRDREnG4W9qGbXGgIWY61zeqC0+OLfhtNR6WwY4dMBWT1bB+q7xQ2Usw3qsDGeXnYi2VqFcI0D7kU+D5XmPhqa6HVGuOISkRDgMvrQ4tZqAaNztEFj1c0W/H2gVrsrYjyf3+eh/nMfng5DsibBLA9/9/VdgiJqCBNvEbqAFbCCNM8JAygzRYOBpa0JXFBJaIhQCFlce9VI9BockCvPD+na3iGGqyEQbvNja5uN9KiNbq5qxrnas7A5mGgHzkG4y562unxoc5fIhqZpYnOe0aLNle4t4a3dToZHCoRDRFKGYsRWdoex4b5u++r2m3ReSPOB1R+BqWMRYN6DHb/2I1//dgWMnbJ6fEhS6dAplaODG1iNAiX15uw+2QTGn3+JUDsHcL3QuKCEtEQN8JfIqlq6x7UdY6fM+Grynb4qv4FdLejJD8bwyYLG24equnCp9+3wMcJycioluOOmYX46bSCwQUfRXWddpxusqLVrQBkSiEJ0XiiuKFElOLMdg/ePVSPA1UdvT4/IlMoJTWanHB6IisBeHwcDlR1oPrEN+g8vRcAILnsWsweU4Brx+dAwjA43WTB/3f4HJrMDgAAwzDB3qpEEIil2+0DtP4lQaidKG4S5zeBxER9lx0NXQ6AB64Y0XMip0EtQ67SC669Eh2HjmGY2gfwHMDKAaUBUKUD6jRh+x+FoddZ6d/XNCGr+UsUO04jo9AIFM0GsoWWofH5BqhkLD4+0YQGkwPf1XTh5skxHjoQgcBYIpvLKySirlphMGaeyIENEZSIUlyjSSiBDEvr5Y/f5wVq9+OKtn1o7rLC6VEAF7UjhZCwQkJSGgHW3+jtNAOnf0Cuy4P8TA0kw68ARoTuezciS4u7rijGiQYzOJ4Pa2vyeAlM8+h2eYGMQImIGqzjhRJRimuxChsa9Ni80G0HTrwLWJqQq2WhTx8JY+HlQulHwgJeJ+AwCaOMAzfOB3S3Cze/brcXDpcHdkUWjFfcCuSM7jUOg1qGn4zO7PW5RKALTHx1eQGdv+fM1kprE8UJJaIU5uN4dPnncWVe2DvldQHl24U5VTIlVONvgCprTP/b6HAc4LIAjk5hoCLnBXgeFc0eHOXkKMzPhyInv+/XJ7hAicjm8grVUVYqLHFLaxPFBSWiFNbZ7YaP46GQSaBX+n/UPA/8sEtIQnINMGVZj+VbeyWRACqjcEPgUjyO1tbAJfVgbK6uz5cmg0AbkdvLwcMDMk02YGkUGqwpEcUcJaIU1m4TqmWZWsX5NpmmcqCtQqh+Tbg1mIQsTg+O1png9nL4t8v73s33Qi4vh2FGJTiOR3Fmgg1MDJNCKglO82AljFA9szQK+7HlXC52eCmPElEK8/g4qOQssgLVMnc3UPWF8LhkPmA4P46H43gcqe2ChGFw1WWZUEgvvWKiUsbi+gl54Dg+ZP5aMmIYJmTUOXT+7jJLozgBDTGUiFLYpAIjJg4zBAcSovpfgMcpzKcqmBlyrlEtR5pahi67B/WdDozK7qf37CLJnoR6pR8m3NuahUZ6iXhL2Q4F1B2Q4hiGgZSVCD1gzceFg6MW9doTNNxfvartuPQo6w6bC21WV3yXnY2xH1us2H2yCaebLIA6XViPyOcN6SUksUGJaKio+0b4z542vNdF7AGgOENIRDUd9ksmmO9qOvH2gVocrO6MeqhiabO6cLrJKoy9YpgLqmcN4gY2BFAiSlF1HXa8vr8an//QArhsQPMJ4YnieX2+ZphRBamEgcXh6Xf5Vo+Pw1n/3LQiEdeajjadv43I6vQKB/T+4QjWJpEiGjpETUSbNm3CzJkzodPpkJ2djaVLl6KiokLMkFJGm80Ji8MDu9sHNB0TSkOGYf0u9iWXSoIjsGv86wX1pqqtG24vB71KhryLB0omMb1KaDK1Oj3+A/5ERA3WMSdqItq3bx/WrFmDAwcOYM+ePfB4PLj22mvR3T24meAEaLP6BzJqZEDjMeFg/rRLvm54hgZahbTfsY0/+Fd7HJurS7ipGoMRKBFZnF6hahpIRPYOYRAoiRlRe812794d8nVZWRmys7Nx+PBhXHXVVSJFlRoCY4jyveeEpU/l6l6XbL3YlEIjphUZ+0wwdrcXNe1CaSnZBzFeTKc8P6jR5eWglGuEib9Os1AqSi8ROcLUlVBtRGazGQCQnp7e6/MulwsWiyXkRnrycXywjSfT6t9JI29yr0u2XoyVMP2Wcn5otoLjeeQalAmzqFm0yFgJVHKhm/58O5G/wZraiWIqYRIRx3F4+OGHMXfuXEyYMKHXczZt2gSDwRC8FRbS4ua9CUztUDFuqGz1wsGciWFdg+N4tPay71lg0ftxefpBx5mI9EoZGEYo+QkH/IM+zefEC2oISJgBjWvWrMHJkyexf//+Ps/ZsGED1q1bF/zaYrFQMupFoFo2Ao1geB7QZoU1X8rm8mLnt3VwenxYPW9EyL70N03KR2WbLaV6yy7006nDIJdKzu8sEmjcN9fTTPwYSohEtHbtWnz00Uf48ssvUVDQ9/KhCoUCCkVqVQdiQSphkGdQoshUC7AAsi5evr5/GjkLhVQCq9OL4+fMmFVyvqoskTC4LCe12oYuFKiaBWmyAakc8LqB7tbzS4SQqBI1vfM8j7Vr1+KDDz7A559/jpISagyMhtE5OtwxNQtj5f7BhgNopL4QwzCYPlxIPt/VdMJs96C2oxsO9xBcTF4iAQz+UpGpXtxYUpioJaI1a9Zg+/bt+Pvf/w6dTofmZmFFPIPBAJUq8ZYTTSrtZ4QlX8OslgWMy9Ph+DkTmsxObPu2Fh4vD51Siv9nZmFCrTUdbe02Fw7VdELGSnDNOP8qBIZCoOMsYK4DCmf2fwESEVFLRFu2bIHZbMaCBQuQl5cXvL3zzjtihpXUfBwPr48TlvoAwi4NBTAMgyWT8pCmlsHl4cDxPIalqaC5uOqSYrw+HqebrKG7mhgvKBGl0Ny6RCLqv7ZUmjCZKBq6HPjw0Flc03Uc43I0YbcPXUinlOHOK4ajrtMOlZxFrl6ZUgMYexMYXd3t9sLr44QJw7o8QCIFPA5hcONAFpIjYaEugBTTZnPCaK8BC074gxnk6oJSVoIRWVrkGVQpn4QAQCVjIZdKwPOA2eGf6iFhhekxAGCqEy+4FEaJKMW0Wd3IsFdBLWeDW/qQgWMYBka1MNXDFEhEAGAsEu67auIf1BBAiSjFdJotMLgaoJZLI24fGuqMKjkAwGS/IBGl+Xt0u2qE8UQkqigRpRAfxwPtZ8DwHFRpudSWESGDSigRmR0XLIWiyxO2ova6ACvNxo82SkQppLPbDWN3FVgJA2U+VcsiZVQL0zzc3gs6UyQSIK1YeNxZLUpcqSx1B4QMQR0mM4zOc9AoWDDZtPNEpMbm6jAuT39+mkdA+gig9Qegswoo6XuBORI+SkQpxGivhVQjhUyfTdWyQZCyfVQUAu1E1iahK19Gg26jhapmKSTXVYNR2ToMHztd7FBSk1IvJHieF0Zak6ihRJQqPE6gy992Qb1lg/b12Xa8+119cNmToMzLhPt2WtI4migRpQh3awXsThc4dYYwv4wMSme3Gw0mB1qtFy0RmzXGf0IV4PP0fCGJCCWiFNFVcwLl58z4orP31S1JeNI1wliiHruZaHOEKprPS4Mbo4gSUSrwOOFqrQQASGg0dVRk+pfB7bBdVCJimPPVszaqnkULJaJU0FEJh8sNh8wIQ2ae2NGkhECJqKPb3XNydrCd6IxQMiKDRokoFbSeRrfLiw7VCGTpUmefMTGlqeWQMAzcXg5W10XJxlAIKHTCKOtO6j2LBkpEyc7jgLv9LFxeDh2akcjS0VK60cBKGKRphKkenbaL2okkEiDHP2A0sIMuGRRKRMmurQI2hwt2WTrUaTkhC92TwcnQKKBXyeDtbZJrYFeUzirA3feuuGRgaGR1sms9DZvTiw71WOTqqVoWTTdMyIXk4mkeAdosQJcDWFuAttPAMBpEOhhUIkpmLhtgqkWaRobhl8/A6BTeXUMMfSahgECpqPEoLSE7SJSIkllbBcDz0GUNx+zLR6IkUyN2RCmJ5/nelzXOnSjsnmtrE/Y9IxGjRJTMmo8L9zTTPmY+/b4ZW/adxbkuR88nZcrzpaJzh+IbWIqhRJSsrC2AtRlmlw9nJcVweobgnmNx4PFxcHk4tPSy/TYAYNg04b79R8Bpjl9gKYYSUbJqKgcAnOHy8eGpLhw/R38EsZDj7wBo7isRabOBtOHCHnL138YxstRCiSgZ+TxAy0nwPI8KRlgjJ1X3ohdbnkFIRA1djr63vxp+pXDfeAxwWeMTWIqhRJSMWk8BXhesjBptbD4UMgmyaSBjTOTqlZCxDOxuHzoungAbYBwOGAoAzgvUHYxvgCmCElGy4flgFeCc+nKAYVCQpr50VzOJiJSVIN8orMTYY22iAIYBiucKjxuPAo6uOEWXOigRJZvOKqC7HWBlOMEJ1bLiDKqWxVJBmvD51vfWcxaQViIsrs95gbNfxCewFEKJKJnwPFAvFP3tmRPQaOPBMMDILK3IgaW24gw18o3K/tvhGAYYtUi4b6ugtYrCRIkomXRVA121gIRFk1YYv5JnUEKjoJk6sZStV+L2mUWYUmjs/0RtFpDv786v2A14+2hTIj3Qb3Cy4Hmgap/wOH8aRhYNw8pMN7rdNH4ooZRcBXT8KLQTnf0MGHOD2BElBSoRJYvm44C1GWBlwPA5AACjWo5hRtrSJl5cXh/K6039Dx6VKYGxS4THjceAllNxiS3ZUSJKBi4bcPZz4XHxPNhBXfVi+OBIAz7/oRWnmyz9n5hWDBTNFh7/sAuw0BbVl0KJKNFxHPDDR8J2QbocmDOn4PV/VePjE03w+npZJ4fEzNg8PQDgWL0JPu4Ss+1LFgAZo4RetBPvAbbWmMeXzCgRJbrqvcJe66wUGHsTvqnqgpfj4XD7+t6RlMTEuDwd1HIWJrsHJxsuMaVGIgEuv1lYs8htB45to5JRP+g3OVHxPFDz1fmRupfdgGqnGqebLGAYYO4o2lI63hRSFrNHZAAAvj7bAYvzEvuaSRXA5P8X0OcLJdqjbwMNh2ntol4kRCL605/+hOLiYiiVSsyePRvffjvEJw963UDFx0D1l8LXIxeiQzsan5xsAgBMLjQi10CrMYph4jADcvRKOD0+fHy8CS7vJXotZSpg8h1A5miA8wFn/gGU76Cq2kVET0TvvPMO1q1bhyeeeAJHjhzB5MmTcd1116G1dQj+oHxeoaflu9eApuPBQXJVqgl499A5uDwc8o1KzKPSkGhYCYPFE3OhkEnQZHbiWJ3p0i+SKoAJPwNGXg1IpMJYsO9eB46/B7SdoR1jATB8n1OK42P27NmYOXMm/vjHPwIAOI5DYWEhfvnLX+Kxxx7r97UWiwUGgwFmsxl6vT4e4UaPzwN47MIaNoEV/jqrwHud8HE8JCojJGMXg08rxlvf1KKz2408gxK3TBkGlZwWyBdbi8WJz39oxc+mFUAulQSPyVgJDCoZ2L7m/jlMQNUXwdU1AQhDMgwFgC4P0GQCSgOg0ANSpfAck7zzCAf6NyrqgEa3243Dhw9jw4YNwWMSiQSLFi3CN9980+N8l8sFl+v8zpsWyyW6UQM6q4HKf/Y8HpKD+QiPnX+urqMbbVYXAB48D/Dghef9p0wYZoBKJgE4Lxo6rGg2OxE4K3BJF6vGOc0E3DBrCQw6NRgIVTGLw4M5IzMgowbqhJCjV+KOmYVg/EmC53l8dLwJFodQupGxDBRSFqyEAcMIs/hvmJgHqIzA+J/ib1+fhLbjOIzdVZB5TADaABwFAKjlLEZn+9cfZxiAlQslKUbivzHnH4tp0u3C9ttRIGoiam9vh8/nQ05OTsjxnJwc/PDDDz3O37RpE37729+G/0ZelzBRNMZ8rm64HX0soAWA8cgBCKUZH8fDxQFuVgO7LB12WQa6VIWwybMBRoJuHwOD/3WXnFpARMFcUFJxejgoZRI4PRK4vRw8Ph6eC3aB1V40DafFo0K1cgagmA61pwM6Vys07jYovWZk8k4hyfCc8N/J6wJw0dbXiYCP3vCRpJrisWHDBqxbty74tcViQWFh4aVfaCwUGgxDirj+x1E5Jtyn2d2Qezj/U0zwFAbCf0VWqxS6dSUSpHNyKH0SMIzEf47wGpZhoFawVPJJMio5i+Wzh4PneTg9HNxeDi6vDz5eKB1f/PNcMikP57dLC/0dlkslgF4hVN99LiERcT7/Hz7vT1D+JIVBtKwMtlVGHr3NGkRNRJmZmWBZFi0tLSHHW1pakJub2+N8hUIBhSKCUcVyDZBeEmmYA2bUAcYBnqv130hqYRgGKjnrb8eT9XleYGmRfknlwk2R+ttEifpvVy6XY/r06fjss8+CxziOw2effYY5c+aIGBkhJJ5Er5qtW7cOK1aswIwZMzBr1ixs3rwZ3d3dWLlypdihEULiRPREdPvtt6OtrQ0bN25Ec3MzpkyZgt27d/dowCaEpC7RxxENRlKPIyJkCBjo3yh1zRBCREeJiBAiOkpEhBDRid5YPRiB5q0BT/UghMRV4G/zUk3RSZ2IrFZhe98Bja4mhIjGarXCYDD0+XxS95pxHIfGxkbodLqQeT9iCEw3qa+vpx68C9Dn0reh8NnwPA+r1Yr8/HxIJH23BCV1iUgikaCgoEDsMELo9fqU/aUaDPpc+pbqn01/JaEAaqwmhIiOEhEhRHSUiKJEoVDgiSeeiGx1gBRGn0vf6LM5L6kbqwkhqYFKRIQQ0VEiIoSIjhIRIUR0lIgIIaKjRBSGcHakfe211zBv3jykpaUhLS0NixYtStkdbCPdqXfnzp1gGAZLly6NbYAiCvezMZlMWLNmDfLy8qBQKHDZZZfh448/jlO0IuLJgOzcuZOXy+X8G2+8wX///ff8vffeyxuNRr6lpaXX85ctW8b/6U9/4o8ePcqfPn2aLy0t5Q0GA3/u3Lk4Rx5b4X4uAdXV1fywYcP4efPm8bfcckt8go2zcD8bl8vFz5gxg1+8eDG/f/9+vrq6mt+7dy9/7NixOEcef5SIBmjWrFn8mjVrgl/7fD4+Pz+f37Rp04Be7/V6eZ1Ox7/55puxClEUkXwuXq+Xv/LKK/m//OUv/IoVK1I2EYX72WzZsoUfMWIE73a74xViwqCq2QAEdqRdtGhR8Fh/O9L2xm63w+PxID09PVZhxl2kn8uTTz6J7Oxs3HPPPfEIUxSRfDYffvgh5syZgzVr1iAnJwcTJkzAs88+C5/PF6+wRZPUk17jJdwdaXvz6KOPIj8/P+QXM9lF8rns378fr7/+Oo4dOxaHCMUTyWdTVVWFzz//HMuXL8fHH3+MyspKPPDAA/B4PHjiiSfiEbZoKBHFwXPPPYedO3di7969UCqVYocjGqvVirvuuguvvfYaMjMzxQ4n4XAch+zsbLz66qtgWRbTp09HQ0MD/uu//osSEQl/R9oL/e53v8Nzzz2Hf/7zn5g0aVIsw4y7cD+Xs2fPoqamBjfddFPwGOffd1kqlaKiogIjR46MbdBxEsnvTF5eHmQyGViWDR4bN24cmpub4Xa7IZfLYxqzmKiNaAAi3ZH2hRdewFNPPYXdu3djxowZ8Qg1rsL9XMaOHYsTJ07g2LFjwdvNN9+MhQsX4tixYym10mYkvzNz585FZWVlMDkDwJkzZ5CXl5fSSQgAdd8P1M6dO3mFQsGXlZXxp06d4u+77z7eaDTyzc3NPM/z/F133cU/9thjwfOfe+45Xi6X83/961/5pqam4M1qtYr1LcREuJ/LxVK51yzcz6auro7X6XT82rVr+YqKCv6jjz7is7Oz+aefflqsbyFuKBGF4Q9/+ANfVFTEy+VyftasWfyBAweCz82fP59fsWJF8Ovhw4fzAHrcnnjiifgHHmPhfC4XS+VExPPhfzZff/01P3v2bF6hUPAjRozgn3nmGd7r9cY56vijZUAIIaKjNiJCiOgoERFCREeJiBAiOkpEhBDRUSIihIiOEhEhRHSUiAghoqNERAgRHSUiklTKyspgNBqDX//mN7/BlClTgl+Xlpam9NKzqYoSEelVaWkpGIbBL37xix7PrVmzBgzDoLS0NOT8aCeA4uJibN68OeTY7bffjjNnzvT5mldeeQVlZWXBrxcsWICHH344qnGR6KNERPpUWFiInTt3wuFwBI85nU5s374dRUVFosSkUqmQnZ3d5/MGgyGkxESSAyUi0qdp06ahsLAQ77//fvDY+++/j6KiIkydOnVQ1+6tpLJ06dJgKWvBggWora3FI488AoZhwDAMgJ5Vs4tdWDIrLS3Fvn378MorrwSvUV1djVGjRuF3v/tdyOuOHTsGhmFQWVk5qO+LRIYSEenXqlWrsHXr1uDXb7zxBlauXBnz933//fdRUFCAJ598Ek1NTWhqagr7Gq+88grmzJmDe++9N3iNoqKiHt8TAGzduhVXXXUVRo0aFa1vgYSBEhHp15133on9+/ejtrYWtbW1+Oqrr3DnnXfG/H3T09PBsix0Oh1yc3MvuRJmbwwGA+RyOdRqdfAaLMuitLQUFRUVwT3GPB4Ptm/fjlWrVkX72yADREvFkn5lZWVhyZIlKCsrA8/zWLJkSdKvN52fn48lS5bgjTfewKxZs/B///d/cLlcuO2228QObciiEhG5pFWrVqGsrAxvvvlm1EoNEokEFy+F5fF4onLtgVi9enWwIX7r1q24/fbboVar4/b+JBQlInJJ119/PdxuNzweD6677rqoXDMrKyuk3cfn8+HkyZMh58jl8kHv6dXXNRYvXgyNRoMtW7Zg9+7dVC0TGVXNyCWxLIvTp08HH/fFbDb32K8sIyOj10Xxr776aqxbtw67du3CyJEj8dJLL8FkMoWcU1xcjC+//BJ33HEHFApFRFXC4uJiHDx4EDU1NdBqtUhPT4dEIgm2FW3YsAGjR4/udxMEEntUIiIDotfrodfr+z1n7969mDp1asjtt7/9ba/nrlq1CitWrMDdd9+N+fPnY8SIEVi4cGHIOU8++SRqamowcuRIZGVlRRT3+vXrwbIsLr/8cmRlZaGuri743D333AO32x2XXkDSP1qzmgxZ//rXv3DNNdegvr6+x46sJL4oEZEhx+Vyoa2tDStWrEBubi62bdsmdkhDHlXNyJCzY8cODB8+HCaTCS+88ILY4RBQiYgQkgCoREQIER0lIkKI6CgREUJER4mIECI6SkSEENFRIiKEiI4SESFEdJSICCGi+/8BT22vkjNqcDoAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:58:39.238077Z", + "iopub.status.busy": "2024-03-01T00:58:39.237761Z", + "iopub.status.idle": "2024-03-01T00:58:39.522256Z", + "shell.execute_reply": "2024-03-01T00:58:39.521279Z" + }, + "papermill": { + "duration": 0.308346, + "end_time": "2024-03-01T00:58:39.524486", + "exception": false, + "start_time": "2024-03-01T00:58:39.216140", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+eklEQVR4nO3deZhT9b0/8PfJyZ7JMktmJrMxM+yLbLKI1AXlagW3WrcrKgNW2yve1vKjFW6fYtWLYKtUbH3o1QroLUq1Va9Vq5eLggIuLLKoMCOzM/uWfc85vz9OEshszGSSnCTzeT1PnmTOOUm+k5n5zHf9fBme53kQQoiIJGIXgBBCKBARQkRHgYgQIjoKRIQQ0VEgIoSIjgIRIUR0FIgIIaKjQEQIEZ1U7AKMBMdxaG5uhlarBcMwYheHENILz/Ow2WwoKCiARDJwvSelA1FzczOKi4vFLgYh5AIaGxtRVFQ04PmUDkRarRaA8E3qdDqRS0MI6c1qtaK4uDj8tzqQlA5EoeaYTqejQERIErtQ1wl1VhNCREeBiBAiOgpEhBDRpXQf0VDwPA+/349AICB2UUY1mUwGlmXFLgZJUmkdiLxeL1paWuB0OsUuyqjHMAyKioqQkZEhdlFIEkrbQMRxHGpra8GyLAoKCiCXy2nSo0h4nkdHRwfOnj2L8ePHU82I9JG2gcjr9YLjOBQXF0OtVotdnFHPaDSirq4OPp8vZoGosduJz2u6kKtT4rJxOZBI6B9NqkrbQBQy2LRykjixro06vX68c7wZXj+Hsz0uyFkJFozNjul7kMShv1KSko43WuD1cwAAo1YBjYKae6ks7WtEJD2dabcBAL4/LR+TTTSrPtVRjSgJXXnllXj44YfFLkbSsrp96LR7wTBAWY5G7OKQGKBAlML27t0LhmFgNpvFLkpCNXQJ0zHydUooZUKTzOUN4Ey7HbRNX2qiphlJOWU5Glw/3QQpK/wf5Tge2w7UwuvncM+CMcjJUIhcQjJco7JG5PVzA978AW7I1/qGcO1IeTwePPLIIyguLoZCocC4cePw0ksvoa6uDosWLQIAZGZmgmEYVFRUXPD1bDYbli1bBo1GA5PJhN///vd9moL//d//jTlz5kCr1SI/Px933XUX2tvbw+dDNbE9e/Zgzpw5UKvVuPTSS1FZWTni73coNAopxudpw80yiYRBvk4JAGgxuxNSBhJbo7JG9PzHZwY8V5ajwc2zCsNfv/BJNXyB/qv7RZkq3DbnXGK2bQdq4fJGLiX5+b9MGFFZ7733Xnz22Wd47rnnMGPGDNTW1qKzsxPFxcX4+9//jh/+8IeorKyETqeDSqW64OutXr0aBw4cwDvvvIO8vDysX78eR48excyZM8PX+Hw+PPHEE5g4cSLa29uxevVqVFRU4P333494rV/96ld45plnYDQa8ZOf/AQrV67EgQMHRvT9RitPp0RDtxPtNjcAvShlINEblYEoVVRVVeH111/H7t27sXjxYgBAeXl5+HxWVhYAIDc3FwaD4YKvZ7PZ8PLLL+PVV1/F1VdfDQDYvn07CgoKIq5buXJl+HF5eTmee+45zJ07F3a7PWKJxoYNG3DFFVcAANauXYulS5fC7XZDqVRG9w0Pgc3tQ1WbDdkaBUrP66jO1QnNsTarJ27vTeJnVAaiVYvGDXiu9+TcBy4fO+C1veforVxYNpJi9XHs2DGwLBv+Yx+pmpoa+Hw+zJs3L3xMr9dj4sSJEdcdOXIEv/nNb3D8+HH09PSA44QmZkNDA6ZMmRK+bvr06eHHJpMJANDe3o6SkpKYlLc/bVYPPqnqRJ5OGRGI8rRC8Ou0exDgeLA0yzqljMpAJJcOvWssXtcOxVCaWrHmcDhw7bXX4tprr8XOnTthNBrR0NCAa6+9Fl6vN+JamUwWfhyaOR0KWvFidgplyFTLIo7rVFIoZSzcvgC67B7k6uJXKyOxNyo7q1PFRRddBI7jsG/fvn7Py+VyABhyipPy8nLIZDIcOnQofMxisaCqqir89enTp9HV1YVNmzbhsssuw6RJkyI6qsXW4/QBAAxqecRxhmGQnSEc63J4+zyPJDcKREmstLQUy5cvx8qVK/H222+jtrYWe/fuxeuvvw4AGDNmDBiGwbvvvouOjg7Y7fZBX0+r1WL58uX4xS9+gY8//hjffPMN7rvvPkgkknCNpqSkBHK5HH/4wx9QU1ODd955B0888UTcv9eh6gnViDSyPudmlxhwzdQ8FGYmviZJRoYCUZLbunUrbr31Vjz44IOYNGkS7r//fjgcDgBAYWEhHnvsMaxduxZ5eXl46KGHLvh6mzdvxoIFC3D99ddj8eLFWLhwISZPnhzuYDYajdixYwfeeOMNTJkyBZs2bcLTTz8d1+9xOM41zeR9zo3L1WJqgR46Zd8gRZIbw6fwVFSr1Qq9Xg+LxdJnFw+3243a2lqUlZXFdRQn1TkcDhQWFuKZZ57BfffdF7f3icXPw+vnwlMv/u3KseFZ1SR5DfY3er5R2Vk9mn311Vc4ffo05s2bB4vFgscffxwAcNNNN4lcsguzuoX+IYVM0m8Q4nkeZ3tc6HZ4MbVAF555TZIfBaI00nt4vbdvv/0WAPD000+jsrIScrkcF198MT799FPk5OQkqphRM6hkuGt+CTy+gUfm/nGiGR4fhwKDCkYtLfVIFRSI0khBQQGOHTs26PmSkhIcOXIkcYWKISkrQd4gw/IMwyBLLUeLxQ2z00uBKIVQIEojUqkU48YNPFlzNDCoZWixuGFx+cQuChkGCkQkZZxqscLpDaAsR4MsTd9RMwDQqYQRM7OTAlEqoUBEUsbJJguaelzIUEgHDEQGlXDcTDWilELDCiRl2Nx+AIBWOfD/T4M6VCOi2dWphAIRSQkcx8M+jEBk9/j75JYiyYuaZiQlOH0BcDwPCcNAIx/411YlY3HN1DzoVTLaUDOFUCAiKcHhEWpDajk76EaKDMNgagElRks1ojbNAoEAfv3rX6OsrAwqlQpjx47FE088MeoToCfbLh7JUB57MBBpFPS/Mx2J+lN96qmnsHXrVrz88suYOnUqDh8+jBUrVkCv1+OnP/2pmEVLeV6vN5wmJB04PUKqk6FspGh2enG2xwWljMW43IwLXk/EJ2qN6ODBg7jpppuwdOlSlJaW4tZbb8U111yDL7/8Mj5vyPOA35v42zBqeBUVFdi3bx+2bNkChmHAMAyqq6tx3333hWuOEydOxJYtW/o87+abb8aGDRtQUFAQzrp48OBBzJw5E0qlEnPmzMHbb78NhmEiZmB//fXXuO6665CRkYG8vDzcc8896OzsHLA8dXV1I/5RDNf4vAzcNb8El4698FKUxm4Xdn/bhpNN5vgXjMSEqDWiSy+9FC+88AKqqqowYcIEHD9+HPv378fmzZv7vd7j8cDjOZeT2Gq1Du8NAz7g02dGUuToXPb/AOnQaidbtmxBVVUVpk2bFl6QmpmZiaKiIrzxxhvIzs7GwYMH8cADD8BkMuH2228PP3fPnj3Q6XTYvXs3AOHzueGGG7BkyRK8+uqrqK+v79PEMpvNuOqqq/CjH/0Iv//97+FyufDII4/g9ttvx0cffdRveYxGYww+lOFRytghr7bXByc1Wl3+eBaJxJCogWjt2rWwWq2YNGkSWJZFIBDAhg0bsGzZsn6v37hxIx577LEElzKx9Ho95HI51Go18vPzw8fP/77Lysrw2Wef4fXXX48IRBqNBn/+85/DTbI//elPYBgGL774IpRKJaZMmYKmpibcf//94ef88Y9/xKxZs/Dkk0+Gj23btg3FxcXhfxD9lSeZ6VTCr7XN7QPP8zR6lgJEDUSvv/46du7ciVdffRVTp07FsWPH8PDDD6OgoADLly/vc/26deuwevXq8NdWqxXFxcV9rhsQKxNqJ4nGjjxR1/PPP49t27ahoaEBLpcLXq83YgsgQEgte36/UGVlJaZPnx6R/+f8xPkAcPz4cXz88ccRu3OEVFdXY8KEkW2HFCtfNfSA43lMyNNCe4HEZxkKKRgG8AV4uHwBqAcZ7ifJQdSf0C9+8QusXbsWd955JwDhD6m+vh4bN27sNxApFAooFCNYUc0wQ24iJZNdu3ZhzZo1eOaZZ7BgwQJotVr87ne/wxdffBFxnUYz/H3g7XY7brjhBjz11FN9zoV25kgGR+p7YHP7UWhQXzAQSVkJNHIp7B4/rC4/BaIUIOpPyOl0QiKJ7C9nWTbuO0EkO7lcHpEQ/8CBA7j00kvx4IMPho9VV1df8HUmTpyIv/zlL/B4POEAfn7ifACYPXs2/v73v6O0tBRSaf+/Dr3Lk2g8z8MZ3LhSPYRRM0Bontk9fljdPuTrKUNnshN11OyGG27Ahg0b8N5776Gurg5vvfUWNm/ejB/84AdiFkt0paWl+OKLL1BXV4fOzk6MHz8ehw8fxocffoiqqir8+te/7hNQ+nPXXXeB4zg88MADOHXqFD788MNw/ulQv8mqVavQ3d2Nf/3Xf8WhQ4dQXV2NDz/8ECtWrAgHn97lSfQ/CrePQ4ATRh4Hm1V9vlDeapubFr+mAlED0R/+8IdwYvjJkydjzZo1+PGPf5xUu0aIYc2aNWBZFlOmTIHRaMS1116LW265BXfccQfmz5+Prq6uiNrRQHQ6Hf7xj3/g2LFjmDlzJn71q19h/fr1ABDuNyooKMCBAwcQCARwzTXX4KKLLsLDDz8Mg8EQrq32Lk9DQ0P8vvl+hCYzquTskDdOvHhMJm69uAiTTQPnSSbJg5LnjzI7d+7EihUrYLFYErqB40h+HvVdDrx5tAk5GXLcs6A0PgUkcUHJ8wkA4JVXXkF5eTkKCwtx/Pjx8BwhMXaRjRYt70h/9JNNc62trVi/fj1aW1thMplw2223YcOGDWIXa1hcoY7qYYx++QIcTrfYYPf4cUl5Fs0lSnIUiNLcL3/5S/zyl78UuxgjMq1Qj+IsNWTD3B7o/061AQBmlRhoD7QkR4GIJL3hLO8IkbESqOUsnN4ArC4fBaIkl/YZGlO4Lz6tiPFzCCXSt7ppzVmyS9tAJJMJv4ROp1PkkhBASEsCCBNWh+toQw8O13WHd3odqtBcouE+jyRe2jbNWJaFwWBAe3s7AECtVlOHpUg4jkNHRwfUavWAs7cH81WDGVaXD4WZqnBwGYpQbmsr7eiR9NI2EAEIrxYPBSMiHolEgpKSkqj+Gbi8wQmNw+znoaZZ6kjrQMQwDEwmE3Jzc+Hz0X9FMcnl8j7rCofC6+fgCwj9Syr58AJRqEZkp0CU9NI6EIWwLBtV3wQRn8snzCGSShjIhzl8X6BX4daLi4bVnCPiGBWBiKSu0GRGlZwddrNOJWdRnKWOR7FIjKXtqBlJD07vuQWvJH1RICJJLdQ0G25HdUhVmw0Hz3Si20FbUCczapqRpDbWmIFl8xVRT704cdaCxm4nMjVyZGlSLzvnaEGBiCS1aJZ3nC8juGI/tIKfJCdqmpG0FhrCp0yNyY1qRCSpfd1kgcsXwDhjBjKjaFqdC0RUI0pmFIhIUjvZZEGrxY0sjTyqQERNs9RATTOS1EK7d0Q7apZBNaKUQIGIJLXQOjN1lPOIQrOqXd4A/IHRvU1VMqOmGUlavkD068xCFFIJbr24CFqldMg7gJDEo0BEklaoWcYOtM6M54GuM0DHaUCmAormAkp9xCUMw9AyjxRAgYgkLbcvlDS/n3VmPA/U7AUaPj93rPUkMHMZkJGbuEKSmKA+IpK0QjWific0tn1zLggVzAK0eYDPDXzzNsBFbo/d2O3EwTOdqOmwx7nEJFoUiEjSKjSosGx+Cf5lSl7kCa8TOLNbeFz6PWDi94HpdwJyNeDsAs4ejri8sceJL2q7UdvpSFDJyXBRICJJSy6VIFenRJ6u186wjZ8LtZ+MXGDMwuDFaqDsCuHx2S8jakVahTByRnOJkhcFIpJavA6g6YjwuOwK4Pysj/kXAYoMwGMH2k+FD4fmElHK2ORFgYgkrcpWGw7VdaPd5j53sOU4EPAD2nwge2zkEyQsYJopPG77JnyYUsYmPwpEJGmdbrVi/3edaLd6hAMcBzR/JTwumgP0lxokb6pw31Mn9CXh3DIPty8AH01qTEoUiEjS6jNq1l0NuK2ATAkYJ/f/JHWWMILGc8L8ouDz5VLhV52WeiQnCkQkaYXyVYeXd7QcF+7zpwPsIFPgjJOE+67q8CFqniU3mtBIklZEmlifC+iuEU6YZgz+xKyxQM0+wFwn9CexUlw3zQQZy0BLO3okJaoRkaTkC3Dw+oX+HJWcBToqhSH5DCOgyRn8yRm5gFwjBCFLIwDAqFXAoJbTerMkRYGIJKVQbYiVMFBIJUD7t8KJ3KkXfjLDAFnlwuNQLYokNdEDUVNTE+6++25kZ2dDpVLhoosuwuHDhy/8RJLWXOflIWK8dsDcIJzIHaCTurfMUuE+WCPqtHtw8EwnjjeaY1tQEhOi9hH19PRg4cKFWLRoEf75z3/CaDTiu+++Q2ZmppjFIkkgSyPHsvkl8HM80PGNsMhVVwCoDEN7AUOxcG9rA/xemJ1efFHbDZNeiRnFQ3wNkjCiBqKnnnoKxcXF2L59e/hYWVmZiCUiyULGCss7AAC1lcJ97pShv4BSL9zcFsDaBI0iHwAt80hWojbN3nnnHcyZMwe33XYbcnNzMWvWLLz44osDXu/xeGC1WiNuJM15nYDlrPA4Z/zwnqsvEu4tjdAEJzU6PAHwPB/DApJYEDUQ1dTUYOvWrRg/fjw+/PBD/Nu//Rt++tOf4uWXX+73+o0bN0Kv14dvxcXFCS4xSZTaTgcO1XWjo+GUMDkxwzj0ZllIqHlmboRGLgXDABzPhzvCSfIQNRBxHIfZs2fjySefxKxZs/DAAw/g/vvvx5/+9Kd+r1+3bh0sFkv41tjYmOASk0Q5027H/u86YW4IjpZlD7M2BAD6UD9RM1jw4YmRNKkx+YgaiEwmE6ZMiWz3T548GQ0NDf1er1AooNPpIm4kPTm9fjC8H1pn8HdhuM0yAFBlAVK5MJ/I2RlunlE/UfIRNRAtXLgQlZWVEceqqqowZswYkUpEkoXbF4DO3Qo5fMLkRK1p+C8ikQAZQic1bC3hxa8ODzXNko2oo2Y///nPcemll+LJJ5/E7bffji+//BIvvPACXnjhBTGLRZKA0xtApqseUp1EqA31t9J+KHQmYQ6StQVXTJiMKyfkQqOIbkcQEj+i1ojmzp2Lt956C6+99hqmTZuGJ554As8++yyWLVsmZrFIEnB5/ch01UPGSqLrHwoJ1aRszTCo5dCrZZD2tyMIEZXoi16vv/56XH/99WIXgyQRf4AD4+qBImCHVGYEMkfQVA8FIntHeAEsST70UyFJx+ULwOA+CwaANLMYYEewYl6pF/JZe52wdTXhpFUDMMClYy+wcJYkVFR11JoaWkhI4kctl2JJkQuTTTowocWr0WKYcK3IZz6LL2q7cfKsJQalJLEUVSAaN24cFi1ahL/85S9wu90XfgIhw8DyAejdzdCrZOdW0Y9EMBBpPJ0AhI7wAEezq5NJVIHo6NGjmD59OlavXo38/Hz8+Mc/xpdffhnrspHRynpW6M+RawCNceSvlyHsiyZ3d4bzETm8NJcomUQViGbOnIktW7agubkZ27ZtQ0tLC773ve9h2rRp2Lx5Mzo6OmJdTjKKdDScRpPZhU55QfTD9ucLbkHNOLuQIQ8GIprUmFRGNI4plUpxyy234I033sBTTz2FM2fOYM2aNSguLsa9996LlpaWWJWTjCLWpio0dDtRE4jRHvZKPSBVAFwAWYywUJoCUXIZUSA6fPgwHnzwQZhMJmzevBlr1qxBdXU1du/ejebmZtx0002xKicZLbwOMPY24XEoudlIMUy4VpTJmQHQbh7JJqrh+82bN2P79u2orKzEkiVL8Morr2DJkiWQBHfdLCsrw44dO1BaWhrLspLRwNwAH8fDKctCjjqGawkz8gBzIwxcD4AiWuaRZKIKRFu3bsXKlStRUVEBk6n/NUC5ubl46aWXRlQ4MgqZG+APcLAoC1Aij+FSjGCNaILGidKpZbTMI8lEFYh2796NkpKScA0ohOd5NDY2oqSkBHK5HMuXL49JIckoYm6AL8DDlmE6t7FiLARHzlSeTqhU0th0gpOYiaqPaOzYsejs7OxzvLu7m1K9kuh5HYCjE74AB6vCdG5jxVhQ5wCMBPC5AY8tdq9LYiKqQDRQqk273Q6lUjmiApFRzNwIjudhZQ3ws0phP7NYYaWAOgt+jsORbyvx0em22L02GbFhNc1Wr14NAGAYBuvXr4darQ6fCwQC+OKLLzBz5syYFpCMIpZGMAxw8YwZGF9UBKU0xv04GXmArR3VNdVo0utx+XgjrcRPEsMKRF999RUAoUZ08uRJyOXy8Dm5XI4ZM2ZgzZo1sS0hGT3M9WDAwGAaC0O2+sLXD1dGHljJ19D6uwEADm8AehUFomQwrED08ccfAwBWrFiBLVu2UKpWEjs+F+AI9jvq47QpQoYRDBjoeWHRq9PrF9azEdFFNWp2/j5khMSEuRHgedgkOpxq9iInw45yY0Zs3yO4bk3LWcHwfppdnUSGHIhuueUW7NixAzqdDrfccsug17755psjLhgZZSxCkvx2Ng8HznRiQp429oFIngHIlJCzNqh8Zjg8BbF9fRK1IQcivV4PJjj3Qq/Xx61AZJSyNAEArIp8wAWo4zHhkGEAjRFythNqXw+twE8iQw5E5zfHqGlGYirgB4Lry3qkQuZEjTxOyUM1RsjYSqh93bTMI4lE9dN2uVzgeT48fF9fX4+33noLU6ZMwTXXXBPTApJRwN4KcAFAroaVzwDgjO1kxvNpjMjTKXGdnoV8YgxyHZGYiGrs8qabbsIrr7wCADCbzZg3bx6eeeYZ3HTTTdi6dWtMC0hGAWuzcK8rhCO4HXQ8A5GMlUDt76E5REkk6gyNl112GQDgb3/7G/Lz81FfX49XXnkFzz33XEwLSEYBy1nhXlcIl1cIRKFdWWNOE0ya77YKyz1IUogqEDmdTmi1WgDA//7v/+KWW26BRCLBJZdcgvr6+pgWkIwCwRoRrzOF+23iViOSqRCQZ6Ch24l9x06Bo9zVSSHq5Plvv/02Ghsb8eGHH4b7hdrb22mSIxket1VYhMpIAK0Jyy4pwQ9nF8WvsxoAozGi2exCU2M9XD7qsE4GUQWi9evXY82aNSgtLcX8+fOxYMECAELtaNasWTEtIElzVmHYHhlGMFIFcjIUKMlWQyKJX5oOiTYXUpaBiobwk0ZU/3ZuvfVWfO9730NLSwtmzJgRPn711VfjBz/4QcwKR0aBUCDSFSbuPTVGyFkJ1L5uOD0BQJu4tyb9i7r+m5+fj/z8/Ihj8+bNG3GByChjOReI2qxu1HU6kKtToixHE7/31Bghk0qgdvfA7vbF733IkEUViBwOBzZt2oQ9e/agvb0dHMdFnKedYMmQnDeREboCnO1w4mB1Fybla+MbiNTZkLEspJwbbqcNgCF+70WGJKpA9KMf/Qj79u3DPffcA5PJFF76Qciw2NuEiYwyFaDKhNMrrL5Xx2voPoSVQaLOBGwt8FlaAcRptT8Zsqh+4v/85z/x3nvvYeHChbEuDxlNQv1D+iKAYcJD95p4Dd2fh8nIBdpaELC3x/29yIVFFYgyMzORlZUV67KQ0SbcUS2sgncGR7DUcRy6DyksKoHJVw9ZHs0jSgZRDd8/8cQTWL9+PZxOZ6zLQ0YTS2QgcnjjPJnxPHJdHpRSFqyr7yYQJPGi+tfzzDPPoLq6Gnl5eSgtLYVMFpnl7ujRozEpHElj4YmMDKANBqJgorK4Le84XzBJGhwdAM/T9kIii+onfvPNN8e4GGTUCS101RgBqRz+ABdeZ6ZVxj8Q8SoDGno88PntKLZ3QaHNift7koFF9RN/9NFHY10ObNq0CevWrcPPfvYzPPvsszF/fZJkrMGFrvoiAICEYXDvgjGwe/xQSOO/Kp6RsKh1qaBw25HT00aBSGRR/8TNZjP+/Oc/Y926dejuFnZFOHr0KJqamob9WocOHcJ//dd/Yfr06dEWh6SacOoPoVkmkTDIzlBgTLYmYdNBOLUQfLyW1oS8HxlYVIHoxIkTmDBhAp566ik8/fTTMJvNAIRc1evWrRvWa9ntdixbtgwvvvgiMjMzoykOSTUBP2AL/vEncmlHb8GUID4rbbYotqgC0erVq1FRUYHvvvsuYmfXJUuW4JNPPhnWa61atQpLly7F4sWLoykKSUW9JjICQH2XA5/XdOFsT+JGYiXaPABAwNaRsPck/YuqjyjUlOqtsLAQra1Dr+bu2rULR48exaFDh4Z0vcfjgcfjCX9ttVqH/F4kifSayAgANZ0OHGswY25pFooy47C5Yj9YrTByxju7hFoam4DROtKvqGpECoWi3yBQVVUFo3FoeYAbGxvxs5/9DDt37oyoVQ1m48aN0Ov14VtxMU3NT0m9JjICgN0tDN1nJGDELESlMSAgkcPn8wPOroS9L+krqkB044034vHHH4fPJ6xcZhgGDQ0NeOSRR/DDH/5wSK9x5MgRtLe3Y/bs2ZBKpZBKpdi3bx+ee+45SKVSBAJ9E1atW7cOFoslfGtsbIym+ERslr6pP+zBOUQZ8dhGaABqhQwOWRa8AV6YT0REE/WExltvvRVGoxEulwtXXHEFWltbsWDBAmzYsGFIr3H11Vfj5MmTEcdWrFiBSZMm4ZFHHgHL9v2FVCgUUCgU0RSZJIteGRlDHOFAlLgtoMuNGoyZORmq9uOAg9aciSmqQKTX67F7924cOHAAx48fh91ux+zZs4fV4azVajFt2rSIYxqNBtnZ2X2OkzRyXkZGSOUAAI7jwwteE9k0U8pYINMEdJwAHLTUQ0zD/qlzHIcdO3bgzTffRF1dHRiGQVlZGfLz88HzPKUEIYPrJyOjzeMHx/NgJQzUssQ1zQBELvUgohlWIOJ5HjfeeCPef/99zJgxAxdddBF4nsepU6dQUVGBN998E2+//XbUhdm7d2/UzyUpop/+IatL6GvUKqVxzVXdn8/aJTC221GSzUHucwOyoQ2ckNgaViDasWMHPvnkE+zZsweLFi2KOPfRRx/h5ptvxiuvvIJ77703poUkaaJXRsaQAoMKyy8thdfPDfDE+PmmzYNxLhny/AHIHR2AgUZixTCsUbPXXnsN//Ef/9EnCAHAVVddhbVr12Lnzp0xKxxJM7aW8NbSoYmMAMBKGGRp5MjXJ742olFI4ZJlwuenkTMxDSsQnThxAt///vcHPH/dddfh+PHjIy4USVPmBuHeUJI0aTc0Cimc8iz4AhwFIhENq2nW3d2NvLy8Ac/n5eWhp6dnxIUiaSoUiPQlEYcP1XWD43hMMumgVyVu+B4Q0tI2y7LgpUAkqmEFokAgAKl04KewLAu/nzasI/3gAudSfxgiA9HxRjNsbj9KstUJD0RquRROWSZ8Dg6wt1OSNJEMe9SsoqJiwEmF568DIySCrUXorJapwqveAcAf4MKzqhMdhAAgQyGFS2aAhwPg9wiTLZW0bXqiDSsQLV++/ILX0IgZ6Ve4f6g4osZhcfnA84BcKoEq0XOIAKgVLHhGCrtEByDYPKNAlHDDCkTbt2+PVzlIugsHojERh3ucXgBAplouymTY4kw17rusDJrvxgOdlUIgyh6b8HKMdvHPyUlIwAeYgwuUewWibocwmTFLk/hmGSDUxHRKGdhgbiLqsBYHBSISf+YGgPMDCm1E/xAAdDvO1YhERUs9REWZoEj8ddcI99lj+4xImYNNsyyNeIHoi5ou2M0cFvoCUDq6AI4DJPQ/OpEoEJH466oW7rP69r3cenERLC5fYvYyG0BVux2dVgazORZKzg+4egBNtmjlGY0o7JP4cnYLf9gSFsgc0+e0lJUgO0MhpOQQSYaCBRgGTplBOEC5iRKOAtEoxPMJ3O+9s0q41xcD0uRMahdKxmaXZgkHqJ8o4ahpNorwPI/PqrtwpL4HGUoprp2ajwKDKr5v2v6tcJ87qc+p441mtFrdmGLSoTgrMQnz+6MJpqe1SgwABwpEIqAa0Shy4qwFX9R2w8/xsLh8aLW64/uGzm7A1iakhc2Z2Od0bacD3zZbwyNnYskI9k+ZJXrhgJ0CUaJRjWiU8AU4HKwWdqqYV5aFcqMGJn2CakOZpULqj/PwPB8OhLk6cZtsoY7ybhiEA64eYblHkjYl0xHViEaJ0y02uH0B6FUyLCjPjn8Q4jigJZgSJm9qn9NdDi9c3gBkLINcrbhZEUM1ImtALsx1AoQFsCRhqEY0SpxqFfahm1Gsj0jH2mHzwO0LxL6PpuuMsGOHTAUY+/YPNXYLO7oWGFRgE5wetrecDIWwzEMuBb7JFxa+2tsoW2MCUY1oFPD4A2izCM2g8Xna8PHKVhv+8nk99lbG+L8/z8NStR9+jgNM0/vdQbW+SwhEidrVdTCshBGWeUgYICNXOBhKaUsSgmpEo4BCyuL+y8vRbHZBpzy3pmtMthqshEGn3YsehxeZsZrd3FOLs3VVsPsY6MZOxORep92+ABqCNaKxRk1s3jNWMvKFe9vQt04nI0c1olFCKWNRbszoc6wwOHxf02mPzRtxAeDMHihlLJrUE/HBdw58+l1HxNwlty8Ao1aBnAw5sjOSo0P4eKMZH3zdguZAMAWIs0v4XkhCUCAa5cqDNZKaDseIXufEWTMOnOlEoOZTwNGJsoJcFM4QNtw8XNeDD79pQ4ATgpFBLcedc4vxg9lFIyt8DDV0O3GqxYZ2r0LYUogL0HyiBKJAlOYsTh9eP9yIz2u6+j1fniPUkprNbrh90dUAfAEOn9d0ofbkZ+g+tRcAIJlwDeZPLMI1U/MgYRicarHi70fOosXiAgAwDBMerUoGobI4vAEgI5gShPqJEiZ5fhNIXDT2ONHU4wJ44JLyvgs59WoZ8pV+cJ1n0HX4GArVAYDnAFYOKPWAKgtQZwrb/yj0/a5K/6auBcbWT1DqOoXsYgNQMh/IFXqGphbooZKxeP9kC5rMLhyq68GNM+I8dSAKoblEdo9fCEQ99cJkTJPIBRslKBCluWazUAMpzOznjz/gB+r345KOfWjtscHtUwC9+pEiSFghICkNABvs9HZbgFOnke/xoSBHA8mYS4DyyH3vyo0ZuOeSUpxssoDj+aTcmjy0zMPh8QPZoRoRdVgnCgWiNNdmEzY06LN5odcJnHwdsLYgP4OFLmssDMVThNqPhAX8bsBlFmYZh25cAHB0Crcgh9cPl8cHp8IIwyW3AHnj+y2HXi3D98bn9HsuGWhDC189fkAbHDmzt1NuogShQJTGAhyPnuA6rpzzR6f8HuD4q8KaKpkSqqnXQWWcOPg2OhwHeKyAq1uYqMj5AZ5HZasPX3FyFBcUQJFXMPDzk1yoRmT3+IXmKCsVUtxSbqKEoECUxrodXgQ4HgqZBDpl8EfN88Dp94QgJNcAM+/qk761XxIJoDIIN4ReisdX9XXwSH2YlK8d8KmpINRH5PVz8PGATJMLWJuFDmsKRHFHgSiNddqFZllOhuJcn0zLcaCjUmh+TbslHISsbh++ajDD6+fwL1MG3s33fB4/h0KDEhzHozQnySYmDpNCKgkv82AljNA8szYL+7HlTRG7eGmPAlEa8wU4qOQsjKFmmdcB1HwsPC67AtCfm8fDcTyO1vdAwjC4fEIOFNILZ0xUylh8f5oJHMdHrF9LRQzDRMw6hzY4XGZtFqdAowwFojQ2vciAiwr14YmEqP0U8LmF9VRFcyOuNajlyFTL0OP0obHbhXG5g4ye9ZLqQahfukLh3t4qdNJLxEtlOxrQcECaYxgGUlYijIC1nhAOjlvc70jQmGDzqr7rwrOsu+wedNg8iU07G2fftdnwwdctONViBdRZQj6igD9ilJDEBwWi0aLhM+E/e+aYfpPYA0BpthCI6rqcFwwwh+q68ZfP6/FFbXfMiyqWDpsHp1pswtwrhjmvedYkbsFGAQpEaaqhy4mX9tfio9NtgMcOtJ4UTpReNuBzCg0qSCUMrC7foOlbfQEO1cG1aSUi5pqONW2wj8jm9gsHdMHpCLYWkUo0eogaiDZu3Ii5c+dCq9UiNzcXN998MyorK8UsUtrosLthdfng9AaAlmNCbUhfOGiyL7lUEp6BXRfMF9Sfmg4HvH4OOpUMpt4TJVOYTiV0mdrcvuCBYCCiDuu4EzUQ7du3D6tWrcLnn3+O3bt3w+fz4ZprroHDMbKV4ATosAUnMmpkQPMx4WDB7As+b0y2BhkK6aBzG08Hsz1Oytcm3VKNkQjViKxuv9A0DQUiZ5cwCZTEjaijZh988EHE1zt27EBubi6OHDmCyy+/XKRSpYfQHKIC/1kh9alc3W/K1t5mFhswu8QwYIBxev2o6xRqS6k+ibE3rfLcpEaPn4NSrhEW/rotQq0oq0zkEqavpOojslgsAICsrKx+z3s8Hlit1ogb6SvA8eE+nhxbcCcN04x+U7b2xkqYQWs5p1tt4Hge+Xpl0iQ1ixUZK4FKLgzTn+snCnZYUz9RXCVNIOI4Dg8//DAWLlyIadOm9XvNxo0bodfrw7fiYkpu3p/Q0g4V44XK3igczLtoWK/BcTza+9n3LJT0frJJN+JyJiOdUgaGEWp+woHgpE/LWfEKNQokzYTGVatW4euvv8b+/fsHvGbdunVYvXp1+Gur1UrBqB+hZlk5msHwPJBhHNZ6KbvHj11fNsDtC+BHl5VH7Et/w/QCnOmwp9Vo2fl+MKsQcqnk3M4ioc59SyOtxI+jpAhEDz30EN5991188sknKCoaOH2oQqGAQpFezYF4kEoYmPRKlJjrARaAsXf6+sFp5CwUUglsbj9OnLVgXtm5prJEwmBCXnr1DZ0v1DQL0+QCUjng9wKO9nMpQkhMiRreeZ7HQw89hLfeegsfffQRysqoMzAWxudpcecsIybJg5MNh9BJfT6GYXDxGCH4HKrrhsXpQ32XAy7vKEwmL5EA+mCtyNwoblnSmKg1olWrVuHVV1/F//zP/0Cr1aK1VciIp9froVIlXzrRlNJZJaR8HWazLGSySYsTZ81osbix88t6+Pw8tEopbp9bnFS5pmOt0+7B4bpuyFgJrp4czEKgLwa6qgFLA1A8d/AXIFERtUa0detWWCwWXHnllTCZTOHbX//6VzGLldICHA9/gBNSfQDDrg2FMAyDpdNNyFTL4PFx4HgehZkqaHo3XdKMP8DjVIstclcTw3k1ojRaW5dMRP3Xlk4LJpNFU48L7xyuxtU9JzA5TzPs/qHzaZUy3H3JGDR0O6GSs8jXKdNqAmN/QrOrHV4//AFOWDCsNQESKeBzCZMbh5JIjgwLDQGkmQ67GwZnHVhwwh/MCLMLSlkJyo0ZMOlVaR+EAEAlYyGXSsDzgMUVXOohYYXlMQBgbhCvcGmMAlGa6bB5ke2sgVrOhrf0IUPHMAwMamGphzkUiADAUCLc99QlvlCjAAWiNNNtsULvaYJaLo26f2i0M6jkAACz87xAlBkc0e2pE+YTkZiiQJRGAhwPdFaB4TmoMvOpLyNKepVQI7K4zkuFojUJW1H7PYCNVuPHGgWiNNLt8MLgqAErYaAsoGZZtAxqYZmH13/eYIpEAmSWCo+7a0UpVzpL3wkho1CX2QKD+yw0ChZMLu08Ea1J+VpMNunOLfMIySoH2k8D3TVA2cAJ5sjwUSBKIwZnPaQaKWS6XGqWjYCUHaChEOonsrUIQ/kymnQbK9Q0SyP5njqMy9VizKSLxS5KelLqhADP88JMaxIzFIjShc8N9AT7Lmi0bMQOVnfi9UON4bQnYTkThPtOSmkcSxSI0oS3vRJOtwecOltYX0ZGpNvhRZPZhXZbrxSxxonBC2qAgK/vE0lUKBCliZ66kzh+1oKPu/vPbkmGJ0sjzCXqs5tJRp7QRAv4aXJjDFEgSgc+NzztZwAAEppNHRM5wTS4XfZeNSKGOdc866DmWaxQIEoHXWfg8njhkhmgzzGJXZq0EKoRdTm8fRdnh/uJqoSaERkxCkTpoP0UHB4/ulTlMGrTZ58xMWWq5ZAwDLx+DjZPr2CjLwYUWmGWdTeNnsUCBaJU53PB21kNj59Dl2YsjFpKpRsLrIRBpkZY6tFt79VPJJEAecEJo6EddMmIUCBKdR2VsLs8cMqyoM7Mi0h0T0YmW6OATiWDv79FrqFdUbprAO/Au+KSoaGZ1amu/RTsbj+61JOQr6NmWSxdNy0fkt7LPEIyjIA2D7C1AR2ngEKaRDoSVCNKZR47YK5HpkaGMVPmYHwa764hhgGDUEioVtT8FaWQHSEKRKmsoxLgeWiNYzB/yliU5WjELlFa4nm+/7TG+RcJu+faO4R9z0jUKBClstYTwj2ttI+bD79pxdZ91Tjb4+p7UqY8Vys6ezixBUszFIhSla0NsLXC4gmgWlIKt28U7jmWAL4AB4+PQ1s/228DAApnC/ed3wFuS+IKlmYoEKWqluMAgCquAO9824MTZ+mPIB7yggMArQMFooxcIHOMsIdc45cJLFl6oUCUigI+oO1r8DyPSkbIkZOue9GLzaQXAlFTj2vg7a/GXCrcNx8DPLbEFCzNUCBKRe3fAn4PbIwaHWwBFDIJcmkiY1zk65SQsQyc3gC6ei+ADTGMAfRFAOcHGr5IbAHTBAWiVMPz4SbAWfUUgGFQlKm+8FAziYqUlaDAIGRi7JObKIRhgNKFwuPmrwBXT4JKlz4oEKWa7hrA0QmwMpzkhGZZaTY1y+KpKFP4fBv7GzkLySwTkutzfqD648QULI1QIEolPA80ClV/Z840NNt5MAww1pghcsHSW2m2GgUG5eD9cAwDjFss3HdUUq6iYaJAlEp6aoGeekDCoiVDmL9i0iuhUdBKnXjK1Slxx9wSzCw2DH5hhhEoCA7nV34A+AfoUyJ90G9wquB5oGaf8LhgNsaWFGJFjhcOL80fSipllwNd3wn9RNV7gInXiV2ilEA1olTRegKwtQKsDBizAABgUMtRaKAtbRLF4w/geKN58MmjMiUwaanwuPkY0PZtQsqW6igQpQKPHaj+SHhcehmcoKF6Mbx1tAkfnW7HqRbr4BdmlgIl84XHp98DrLRF9YVQIEp2HAecflfYLkibB0vOTLz0aS3eP9kCf6CfPDkkbiaZdACAY41mBLgLrLYvuxLIHieMop18A7C3x718qYwCUbKr3Svstc5KgUk34LOaHvg5Hi5vYOAdSUlcTDZpoZazMDt9+LrpAktqJBJgyo1CziKvEzi2k2pGg6Df5GTF80DdgXMzdSdch1q3GqdarGAYYOE42lI60RRSFvPLswEAB6u7YHVfYF8zqQKY8a+ArkCo0X71F6DpCOUu6kdSBKLnn38epaWlUCqVmD9/Pr78cpQvHvR7gcr3gdpPhK/HLkJXxnj88+sWAMCMYgPy9ZSNUQwXFeqRp1PC7Qvg/RMt8PgvMGopUwEz7gRyxgNcAKj6X+D4a9RU60X0QPTXv/4Vq1evxqOPPoqjR49ixowZuPbaa9HePgp/UAG/MNJy6EWg5UR4klyNahpeP3wWHh+HAoMSl1FtSDSshMGSi/KhkEnQYnHjWIP5wk+SKoBpPwTGXgVIpMJcsEMvASfeADqqaMdYAAw/4JLixJg/fz7mzp2LP/7xjwAAjuNQXFyMf//3f8fatWsHfa7VaoVer4fFYoFOp0tEcWMn4AN8TiGHTSjDX3cNeL8bAY6HRGWAZNIS8JmleOWzenQ7vDDplbhpZiFUckqQL7Y2qxsfnW7HD2cXQS6VhI/JWAn0KhnYgdb+ucxAzcfh7JoAhCkZ+iJAawI0OYBSDyh0gFQpnGNSdx3hUP9GRZ3Q6PV6ceTIEaxbty58TCKRYPHixfjss8/6XO/xeODxnNt502q9wDBqSHctcOb/+h6PiMF8lMfOnWvocqDD5gHAg+cBHrxwPnjJtEI9VDIJwPnR1GVDq8WN0FWhl/SwapzVTMN185ZCr1WDgdAUs7p8WDA2GzLqoE4KeTol7pxbDCYYJHiex7snWmB1CbUbGctAIWXBShgwjLCK/7qLTIDKAEz9Ad4++DUyuk7A4KiBzGcG0AHgKwCAWs5ifG4w/zjDAKxcqEkxkuCNOfdYTNPvELbfjgFRA1FnZycCgQDy8vIijufl5eH06dN9rt+4cSMee+yx4b+R3yMsFI2zgMcBr2uABFoAGJ8cgFCbCXA8PBzgZTVwyrLglGWjR1UMuzwXYCRwBBjog8+74NICIgrmvJqK28dBKZPA7ZPA6+fgC/DwnbcLbEavZThtPhVqlXMAxcVQ+7qg9bRD4+2A0m9BDu8WggzPCf+d/B4Avba+TgZ87KaPpNQSj3Xr1mH16tXhr61WK4qLiy/8REOx0GEYUcUNPo7JMeE+0+mF3McFTzHhSxgI/xXZDKUwrCuRIIuTQxmQgGEkwWuE57AMA7WCpZpPilHJWSybPwY8z8Pt4+D1c/D4AwjwQu24989z6XQTzm2XFvk7LJdKAJ1CaL4HPEIg4gLBP3w+GKCCQQoj6FkZaa+MPHabNYgaiHJycsCyLNra2iKOt7W1IT8/v8/1CoUCCkUUs4rlGiCrLNpiDplBCxiGeG1G8EbSC8MwUMnZYD+ebMDrQqlFBiWVCzdF+m8TJeq/Xblcjosvvhh79uwJH+M4Dnv27MGCBQtELBkhJJFEb5qtXr0ay5cvx5w5czBv3jw8++yzcDgcWLFihdhFI4QkiOiB6I477kBHRwfWr1+P1tZWzJw5Ex988EGfDmxCSPoSfR7RSKT0PCJCRoGh/o3S0AwhRHQUiAghoqNARAgRneid1SMR6t4a8lIPQkhChf42L9QVndKByGYTtvcd0uxqQohobDYb9Hr9gOdTetSM4zg0NzdDq9VGrPsRQ2i5SWNjI43gnYc+l4GNhs+G53nYbDYUFBRAIhm4Jyila0QSiQRFRUViFyOCTqdL21+qkaDPZWDp/tkMVhMKoc5qQojoKBARQkRHgShGFAoFHn300eiyA6Qx+lwGRp/NOSndWU0ISQ9UIyKEiI4CESFEdBSICCGio0BECBEdBaJhGM6OtC+++CIuu+wyZGZmIjMzE4sXL07bHWyj3al3165dYBgGN998c3wLKKLhfjZmsxmrVq2CyWSCQqHAhAkT8P777yeotCLiyZDs2rWLl8vl/LZt2/hvvvmGv//++3mDwcC3tbX1e/1dd93FP//88/xXX33Fnzp1iq+oqOD1ej1/9uzZBJc8vob7uYTU1tbyhYWF/GWXXcbfdNNNiSlsgg33s/F4PPycOXP4JUuW8Pv37+dra2v5vXv38seOHUtwyROPAtEQzZs3j1+1alX460AgwBcUFPAbN24c0vP9fj+v1Wr5l19+OV5FFEU0n4vf7+cvvfRS/s9//jO/fPnytA1Ew/1stm7dypeXl/NerzdRRUwa1DQbgtCOtIsXLw4fG2xH2v44nU74fD5kZWXFq5gJF+3n8vjjjyM3Nxf33XdfIoopimg+m3feeQcLFizAqlWrkJeXh2nTpuHJJ59EIBBIVLFFk9KLXhNluDvS9ueRRx5BQUFBxC9mqovmc9m/fz9eeuklHDt2LAElFE80n01NTQ0++ugjLFu2DO+//z7OnDmDBx98ED6fD48++mgiii0aCkQJsGnTJuzatQt79+6FUqkUuziisdlsuOeee/Diiy8iJydH7OIkHY7jkJubixdeeAEsy+Liiy9GU1MTfve731EgIsPfkfZ8Tz/9NDZt2oT/+7//w/Tp0+NZzIQb7udSXV2Nuro63HDDDeFjXHDfZalUisrKSowdOza+hU6QaH5nTCYTZDIZWJYNH5s8eTJaW1vh9Xohl8vjWmYxUR/REES7I+1vf/tbPPHEE/jggw8wZ86cRBQ1oYb7uUyaNAknT57EsWPHwrcbb7wRixYtwrFjx9Iq02Y0vzMLFy7EmTNnwsEZAKqqqmAymdI6CAGg4fuh2rVrF69QKPgdO3bw3377Lf/AAw/wBoOBb21t5Xme5++55x5+7dq14es3bdrEy+Vy/m9/+xvf0tISvtlsNrG+hbgY7ufSWzqPmg33s2loaOC1Wi3/0EMP8ZWVlfy7777L5+bm8v/5n/8p1reQMBSIhuEPf/gDX1JSwsvlcn7evHn8559/Hj53xRVX8MuXLw9/PWbMGB5An9ujjz6a+ILH2XA+l97SORDx/PA/m4MHD/Lz58/nFQoFX15ezm/YsIH3+/0JLnXiURoQQojoqI+IECI6CkSEENFRICKEiI4CESFEdBSICCGio0BECBEdBSJCiOgoEJGUsmPHDhgMhvDXv/nNbzBz5szw1xUVFWmd8TFdUSAi/aqoqADDMPjJT37S59yqVavAMAwqKioiro91ACgtLcWzzz4bceyOO+5AVVXVgM/ZsmULduzYEf76yiuvxMMPPxzTcpHYo0BEBlRcXIxdu3bB5XKFj7ndbrz66qsoKSkRpUwqlQq5ubkDntfr9RE1JpIaKBCRAc2ePRvFxcV48803w8fefPNNlJSUYNasWSN67f5qKjfffHO4lnXllVeivr4eP//5z8EwDBiGAdC3adbb+TWziooK7Nu3D1u2bAm/Rm1tLcaNG4enn3464nnHjh0DwzA4c+bMiL4vEh0KRGRQK1euxPbt28Nfb9u2DStWrIj7+7755psoKirC448/jpaWFrS0tAz7NbZs2YIFCxbg/vvvD79GSUlJn+8JALZv347LL78c48aNi9W3QIaBAhEZ1N133439+/ejvr4e9fX1OHDgAO6+++64v29WVhZYloVWq0V+fv4FE9D1R6/XQy6XQ61Wh1+DZVlUVFSgsrIyvLWPz+fDq6++ipUrV8b62yBDRBkayaCMRiOWLl2KHTt2gOd5LF26NOXTvBYUFGDp0qXYtm0b5s2bh3/84x/weDy47bbbxC7aqEU1InJBK1euxI4dO/Dyyy/HrNYgkUjQOwONz+eLyWsPxY9+9KNwR/z27dtxxx13QK1WJ+z9SSQKROSCvv/978Pr9cLn8+Haa6+NyWsajcaIfp9AIICvv/464hq5XD7irXQGeo0lS5ZAo9Fg69at+OCDD6hZJjJqmpELYlkWp06dCj8eiMVi6bNNUHZ2dr+5qK+66iqsXr0a7733HsaOHYvNmzfDbDZHXFNaWopPPvkEd955JxQKRVRNwtLSUnzxxReoq6tDRkYGsrKyIJFIwn1F69atw/jx4wfNPU7ij2pEZEh0Oh10Ot2g1+zduxezZs2KuD322GP9Xrty5UosX74c9957L6644gqUl5dj0aJFEdc8/vjjqKurw9ixY2E0GqMq95o1a8CyLKZMmQKj0YiGhobwufvuuw9erzcho4BkcJQqloxan376Ka6++mo0Njb22QiRJBYFIjLqeDwedHR0YPny5cjPz8fOnTvFLtKoR00zMuq89tprGDNmDMxmM37729+KXRwCqhERQpIA1YgIIaKjQEQIER0FIkKI6CgQEUJER4GIECI6CkSEENFRICKEiI4CESFEdBSICCGi+/86a2m5Ec4lLgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:58:39.566077Z", + "iopub.status.busy": "2024-03-01T00:58:39.565767Z", + "iopub.status.idle": "2024-03-01T00:58:39.722761Z", + "shell.execute_reply": "2024-03-01T00:58:39.721524Z" + }, + "papermill": { + "duration": 0.181132, + "end_time": "2024-03-01T00:58:39.725971", + "exception": false, + "start_time": "2024-03-01T00:58:39.544839", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAs+ElEQVR4nO3de1RUR4I/8G83j+aNKPIMCogrGh8oHBBjNj6Q1qwOjnmgSQSJwV2Nv43Ta5whiXhQJ0yiEjRrZNcVX8lGNxvHnUkcFIm6GtEoBoNGyIgiGh4KCi0Qm4au3x8ud9KBxm4FGy7fzzl9SFfXra7ihq/V996uqxBCCBARyZDS2h0gIuouDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhItmyt3YGeyGAwoKKiAq6urlAoFNbuDhH9jBACd+/ehZ+fH5TKzudoDLgOVFRUICAgwNrdIKJOXL9+HU888USndRhwHXB1dQVw/xfo5uZm5d50H71ej0OHDiE2NhZ2dnbW7g49or6yP7VaLQICAqS/084w4DrQ9rHUzc1N9gHn5OQENzc3Wf9B9BV9bX+ac/iIJxmISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLLFy0SIeqmmpiYUFxdLzxt+0uFkUSk8PM/CxVFlVDc0NBROTk6Pu4tWx4Aj6qWKi4sRHh7ervz9DuoWFBRg3Lhx3d+pHoYBR9RLhYaGoqCgQHpeUlkHzWdFyHhhFIb59mtXty9iwBH1Uk5OTkazMuW1WqiO/4ThI8cgbPAAK/as5+BJBiKSLQYcEckWA46IZIsBR0SyxYAjItliwBGRbDHgiEi2GHBEJFsMOCKSLQYcEckWA46IZIsBR0SyxYAjItliwBGRbDHgiEi2rB5wmzdvRmBgIBwcHBAVFYVvvvnGZN0dO3ZAoVAYPRwcHIzqLFiwoF2d6dOnd/cwiKgHsuqCl3v37oVGo0FWVhaioqKQmZkJtVqNkpISeHl5dbiNm5sbSkpKpOcKhaJdnenTp2P79u3Sc5VK1a4OEcmfVWdwGRkZSE5ORlJSEkaMGIGsrCw4OTkhOzvb5DYKhQI+Pj7Sw9vbu10dlUplVMfDw6M7h0FEPZTVZnDNzc0oKChASkqKVKZUKhETE4P8/HyT2zU0NGDw4MEwGAwYN24c3n33XTz55JNGdY4ePQovLy94eHhgypQpWLt2LQYMML2Es06ng06nk55rtVoAgF6vh16vf9gh9nhtY5PzGPuSlpYW6aec96klY7NawNXU1KC1tbXdDMzb29voVmg/N2zYMGRnZ2P06NGor6/H+vXrMWHCBFy8eBFPPPEEgPsfT+fMmYOgoCCUlpbirbfewowZM5Cfnw8bG5sO201PT0daWlq78kOHDvWJW63l5uZauwvUBa43AIAtTp06hR8vWLs33aepqcnsugohhOjGvphUUVEBf39/nDx5EtHR0VL5ihUrcOzYMZw+ffqBbej1egwfPhzz5s3DmjVrOqxz5coVDBkyBIcPH8bUqVM7rNPRDC4gIAA1NTVwc3OzcGS9h16vR25uLqZNmwY7Oztrd4ce0fny23h+61n8d3IExgzqb+3udButVgtPT0/U19c/8O/TajM4T09P2NjYoLq62qi8uroaPj4+ZrVhZ2eHsWPH4vLlyybrBAcHw9PTE5cvXzYZcCqVqsMTEXZ2dn3iD7+vjFPubG1tpZ9y3p+WjM1qJxns7e0RHh6OvLw8qcxgMCAvL89oRteZ1tZWFBUVwdfX12SdGzduoLa2ttM6RCRPVj2LqtFosHXrVuzcuROXLl3C4sWL0djYiKSkJABAQkKC0UmI1atX49ChQ7hy5QrOnTuHV155BdeuXcNrr70G4P4JiDfffBOnTp1CWVkZ8vLyEBcXh5CQEKjVaquMkYisx6rXwcXHx+PWrVtITU1FVVUVwsLCkJOTI514KC8vh1L5twy+c+cOkpOTUVVVBQ8PD4SHh+PkyZMYMWIEAMDGxgbfffcddu7cibq6Ovj5+SE2NhZr1qzhtXBEfZDVTjL0ZFqtFu7u7mYdxOzN9Ho9Dhw4gGeffVbWx2z6isJrtZi95RT2Lx4v6zvbW/L3afWvahERdRcGHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWxZdT04eryampqMbujT8JMOJ4tK4eF5Fi6Of1svLzQ0tE/cbIfkjwHXhxQXFyM8PLxd+fu/eF5QUIBx48Y9nk4RdSMGXB8SGhqKgoIC6XlJZR00nxUh44VRGObbz6gekRww4PoQJycno5mZ8lotVMd/wvCRY2S9Aiz1XTzJQESyxYAjItliwBGRbDHgiEi2GHBEJFsMOCKSLasH3ObNmxEYGAgHBwdERUXhm2++MVl3x44dUCgURg8HBwejOkIIpKamwtfXF46OjoiJicFf//rX7h4GEfVAVg24vXv3QqPRYNWqVTh37hzGjBkDtVqNmzdvmtzGzc0NlZWV0uPatWtGr7///vvYtGkTsrKycPr0aTg7O0OtVuPevXvdPRwi6mGsGnAZGRlITk5GUlISRowYgaysLDg5OSE7O9vkNgqFAj4+PtLD29tbek0IgczMTLzzzjuIi4vD6NGjsWvXLlRUVGD//v2PYURE1JNY7ZsMzc3NKCgoQEpKilSmVCoRExOD/Px8k9s1NDRg8ODBMBgMGDduHN599108+eSTAICrV6+iqqoKMTExUn13d3dERUUhPz8fc+fO7bBNnU4HnU4nPddqtQAAvV4PvV7/SOPsyVpaWqSfch5nX9FX9qclY7NawNXU1KC1tdVoBgYA3t7eRite/NywYcOQnZ2N0aNHo76+HuvXr8eECRNw8eJFPPHEE6iqqpLa+GWbba91JD09HWlpae3KDx06JOtVNa43AIAtTp06hR8vWLs39Kj6yv5samoyu26v+i5qdHQ0oqOjpecTJkzA8OHD8W//9m9Ys2bNQ7ebkpICjUYjPddqtQgICEBsbCzc3Nweqc892fny20DRWYwfPx5jBvW3dnfoEfWV/dn2CcscVgs4T09P2NjYoLq62qi8uroaPj4+ZrVhZ2eHsWPH4vLlywAgbVddXQ1fX1+jNsPCwky2o1KpoFKp2pXb2dnBzs7OrL70Rra2ttJPOY+zr+gr+9OSsVntJIO9vT3Cw8ORl5cnlRkMBuTl5RnN0jrT2tqKoqIiKcyCgoLg4+Nj1KZWq8Xp06fNbpOI5MOqH1E1Gg0SExMRERGByMhIZGZmorGxEUlJSQCAhIQE+Pv7Iz09HQCwevVqjB8/HiEhIairq8O6detw7do1vPbaawDun2FdtmwZ1q5di6FDhyIoKAgrV66En58fZs+eba1hEpGVWDXg4uPjcevWLaSmpqKqqgphYWHIycmRThKUl5dDqfzbJPPOnTtITk5GVVUVPDw8EB4ejpMnT2LEiBFSnRUrVqCxsRGLFi1CXV0dJk6ciJycnHYXBBOR/CmEEMLanehptFot3N3dUV9fL+uTDIXXajF7yynsXzyeC17KQF/Zn5b8fVr9q1pERN2FAUdEssWAIyLZYsARkWz1qm8yEPV1V2sa0ahr6fC10luN0s+2i3474qyyRZCnc7f0r6dhwBH1EldrGjF5/dEH1vuX/y56YJ0jyyf1iZBjwBH1Em0zt8z4MIR4ubR//Scdvjiaj5mTouHs2P6rhwBw+WYDlu0tNDkLlBsGHFEvE+LlgpH+7u3K9Xo9qgYC4wZ7yPq7qJZgwMlYZ8drAB6zIfljwMmUucdrAB6zIfmyOOCuXLmC4ODg7ugLdaEHHa8BeMyG5M/igAsJCcEzzzyDhQsX4vnnn+eX2Hs4U8drAB6zIfmz+ELfc+fOYfTo0dBoNPDx8cE//uM/dnqrPyIia7E44MLCwrBx40ZUVFQgOzsblZWVmDhxIkaOHImMjAzcunWrO/pJRGSxh/6qlq2tLebMmYPPPvsM7733Hi5fvozly5cjICAACQkJqKys7Mp+EhFZ7KED7uzZs1iyZAl8fX2RkZGB5cuXo7S0FLm5uaioqEBcXFxX9pOIyGIWn2TIyMjA9u3bUVJSgmeffRa7du3Cs88+K628GxQUhB07diAwMLCr+0pEZBGLA27Lli149dVXsWDBAqM7V/2cl5cXtm3b9sidIyJ6FBYHXG5uLgYNGmR0rwQAEELg+vXrGDRoEOzt7ZGYmNhlnSQiehgWH4MbMmQIampq2pXfvn0bQUFBXdIpIqKuYHHAmbpHTUNDAy/6JaIexeyPqBqNBsD9e4+mpqbCyclJeq21tRWnT5/u9O7xRESPm9kB9+233wK4P4MrKiqCvb299Jq9vT3GjBmD5cuXd30P6aHoWu9B6fAjrmpLoHTo+LuoLS0tqGipwKXbl0yuJnJV2wClw4/Qtd4D0PFXvoh6KrMD7siRIwCApKQkbNy4Udb3C5WDisZrcA76EG+Z8S26j3I+6vR15yCgojEM4fDuot4RPR4Wn0Xdvn17l3Zg8+bNWLduHaqqqjBmzBh8+OGHiIyMfOB2e/bswbx58xAXF4f9+/dL5QsWLMDOnTuN6qrVauTk5HRpv3s6P+fBaLz6/7AxPgxDTKwm0tLSgq9PfI2nJj5lcgZXerMBb+wthN/kwd3ZXaJuYVbAzZkzBzt27ICbmxvmzJnTad19+/aZ/eZ79+6FRqNBVlYWoqKikJmZCbVajZKSEnh5eZncrqysDMuXL8fTTz/d4evTp083CmKVquOlgORMZeMAwz1/BLkNw4gBplcTuWp7FcP7Dze5mojhXj0M925BZcMTSNT7mHUW1d3dHQqFQvrvzh6WyMjIQHJyMpKSkjBixAhkZWXByckJ2dnZJrdpbW3Fyy+/jLS0NJPr0qlUKvj4+EgPDw8Pi/pFRPJg1gzu57OhrvqI2tzcjIKCAqSkpEhlSqUSMTExyM/PN7nd6tWr4eXlhYULF+L48eMd1jl69Ci8vLzg4eGBKVOmYO3atRgwYIDJNnU6HXQ6nfRcq9UCuD/D0ev1lg6tR2hpaZF+mhpDW3lnYzSnHXo8GnX3T/hcvvM9DLbtV1duO2lUdLPI5CGHK3caoXT4EY26Buj1Th3W6eks+f/QakuW19TUoLW1Fd7exgeuvb29UVxc3OE2J06cwLZt21BYWGiy3enTp2POnDkICgpCaWkp3nrrLcyYMQP5+fmwsbHpcJv09HSkpaW1Kz906JDR5TC9yfUGALDFiRMncK3jQ3CS3NzcLmmHute5uxVwDvoIKws6r/fR4QefNDpwshVVrn5d2LvHp6mpyey6ZgXc2LFjpY+oD3Lu3Dmz39wSd+/exfz587F161Z4enqarDd37lzpv0eNGoXRo0djyJAhOHr0KKZOndrhNikpKdJ1fsD9GVxAQABiY2N77dniixVarC86hYkTJ+JJv47HoNfrkZubi2nTppk8BmdOO/R4+Fy/id27bJDx/CgED+x4Bnf61GlEjY8yPYO71QjNfxfh2YR/wLgA08e5e7K2T1jmMCvgZs+e/bB9McnT0xM2Njaorq42Kq+uroaPj0+7+qWlpSgrK8OsWbOkMoPBAOD+2nQlJSUYMmRIu+2Cg4Ph6emJy5cvmww4lUrV4YkIOzu7XruUd9v/4La2tg8cQ2fjtKQd6l7OKhcY7vkjxGMERnp3fNvA67bXMcprlMl9pWyph+HebTirXHrt/rSk32YF3KpVqx66M6bY29sjPDwceXl5UoAaDAbk5eVh6dKl7eqHhoaiqMj47k/vvPMO7t69i40bNyIgIKDD97lx4wZqa2tNrnxCRPJl1dsGajQaJCYmIiIiApGRkcjMzERjYyOSkpIAAAkJCfD390d6ejocHBwwcuRIo+379esHAFJ5Q0MD0tLS8Nxzz8HHxwelpaVYsWIFQkJCoFarH+vYiMj6zAq4/v3744cffoCnpyc8PDw6PR53+/Zts988Pj4et27dQmpqKqqqqhAWFoacnBzpxEN5eXm7ZZk6Y2Njg++++w47d+5EXV0d/Pz8EBsbizVr1vTJa+GI+jqzAu6DDz6Aq6ur9N/mnnAwx9KlSzv8SArcv9yjMzt27DB67ujoiIMHD3ZRz4iotzMr4H6+eOWCBQu6qy9ERF3K4vXgbGxscPPmzXbltbW1Jq8zIyKyhi5b8FKn0xktoUREZG1mn0XdtGkTgPsLXv7Hf/wHXFz+dll7a2sr/vd//xehoaFd30MioodkdsB98MEHAO7P4LKysow+jtrb2yMwMBBZWVld30MioodkdsBdvXoVADB58mTs27ePK3QQUY9n8YW+bSv7EhH1dBYH3Kuvvtrp652t5UZE9DhZHHB37twxeq7X63HhwgXU1dVhypQpXdYxIqJHZXHA/fGPf2xXZjAYsHjx4g5X8yAishaLr4PrsBGlEhqNRjrTSkTUE3RJwAH312trW96aiKgnsPgj6s9XvgXuXxdXWVmJL7/80ug7q0RE1mZxwLXd4b6NUqnEwIEDsWHDhgeeYSUiepx4HRwRyVaXHYMjIuppGHBEJFsMOCKSLQYcEclWlwXcjRs3sGjRoq5qjojokXVZwNXW1mLbtm1d1RwR0SPjR1Qiki0GHBHJFgOOiGTL7G8yzJkzp9PX6+rqHqoDmzdvxrp161BVVYUxY8bgww8/RGRk5AO327NnD+bNm4e4uDjs379fKhdCYNWqVdi6dSvq6urw1FNPYcuWLRg6dOhD9Y+Iei+zZ3Du7u6dPgYPHoyEhASL3nzv3r3QaDRYtWoVzp07hzFjxkCtVnd439WfKysrw/Lly/H000+3e+3999/Hpk2bkJWVhdOnT8PZ2RlqtRr37t2zqG9E1PuZPYPbvn17l795RkYGkpOTkZSUBADIysrCl19+iezsbPzud7/rcJvW1la8/PLLSEtLw/Hjx41mjkIIZGZm4p133kFcXBwAYNeuXfD29sb+/fsxd+7cLh8DEfVcFn/Zvqs0NzejoKAAKSkpUplSqURMTAzy8/NNbrd69Wp4eXlh4cKFOH78uNFrV69eRVVVFWJiYqQyd3d3REVFIT8/32TA6XQ66HQ66blWqwVwfzl2vV7/UOOztra1+VpaWkyOoa28szGa0w49Hg/aF31lf1rSb7MDztylkMy96UxNTQ1aW1vh7e1tVO7t7Y3i4uIOtzlx4gS2bduGwsLCDl+vqqqS2vhlm22vdSQ9PR1paWntyg8dOgQnJ6fOhtFjXW8AAFucOHEC11w6r5ubm9sl7VD3MndfyH1/NjU1mV3X7IDbsWMHBg8ejLFjx0II8VAdexR3797F/PnzsXXrVnh6enZp2ykpKUYLeWq1WgQEBCA2NhZubm5d+l6Py8UKLdYXncLEiRPxpF/HY9Dr9cjNzcW0adNgZ2f30O3Q4/GgfdFX9mfbJyxzmB1wixcvxqeffoqrV68iKSkJr7zyCvr37/9QHQQAT09P2NjYoLq62qi8uroaPj4+7eqXlpairKwMs2bNksoMBgMAwNbWFiUlJdJ21dXV8PX1NWozLCzMZF9UKhVUKlW7cjs7O5P/o/R0tra20s8HjaGzcVrSDnUvc/eF3PenJf02+yzq5s2bUVlZiRUrVuDPf/4zAgIC8OKLL+LgwYMPNaOzt7dHeHg48vLypDKDwYC8vDxER0e3qx8aGoqioiIUFhZKj1/96leYPHkyCgsLERAQgKCgIPj4+Bi1qdVqcfr06Q7bJCJ5s+gkg0qlwrx58zBv3jxcu3YNO3bswJIlS9DS0oKLFy/CxcWyD/UajQaJiYmIiIhAZGQkMjMz0djYKJ1VTUhIgL+/P9LT0+Hg4ICRI0cabd+vXz8AMCpftmwZ1q5di6FDhyIoKAgrV66En58fZs+ebVHfiKj3e+izqEqlEgqFAkIItLa2PlQb8fHxuHXrFlJTU1FVVYWwsDDk5ORIJwnKy8uhVFr2ZYsVK1agsbERixYtQl1dHSZOnIicnBw4ODg8VB97q5/09/fJhR/rTdZp/EmHs7cAn2t34OzY/iM6AFy+2dAt/SN6HCwKOJ1Oh3379iE7OxsnTpzAzJkz8a//+q+YPn26xUHUZunSpVi6dGmHrx09erTTbXfs2NGuTKFQYPXq1Vi9evVD9UcuSv8vmH63r+gBNW2x+/KZB7bnrLLaFUVED83s/2uXLFmCPXv2ICAgAK+++io+/fTTLj+bSV0n9sn7J1yGeLnA0c6mwzollfX4l/8uwobnR2GYr7vJtpxVtgjydO6WfpL5HjQr54y8PbMDLisrC4MGDUJwcDCOHTuGY8eOdVhv3759XdY5enj9ne0xN3JQp3XaLvocMtAZI/1NBxz1DObNyjkj/zmzR5mQkACFQtGdfSGiTjxoVs4ZeXsWXehLRNbzoFk5Z+TtcT04IpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLJl9YDbvHkzAgMD4eDggKioKHzzzTcm6+7btw8RERHo168fnJ2dERYWht27dxvVWbBgARQKhdFj+vTp3T0MIuqBrHpzxL1790Kj0SArKwtRUVHIzMyEWq1GSUkJvLy82tXv378/3n77bYSGhsLe3h5ffPEFkpKS4OXlBbVaLdWbPn06tm/fLj1XqTq+CS4RyZtVZ3AZGRlITk5GUlISRowYgaysLDg5OSE7O7vD+pMmTcKvf/1rDB8+HEOGDMEbb7yB0aNH48SJE0b1VCoVfHx8pIeHh8fjGA4R9TBWm8E1NzejoKAAKSkpUplSqURMTAzy8/MfuL0QAl999RVKSkrw3nvvGb129OhReHl5wcPDA1OmTMHatWsxYMAAk23pdDrodDrpuVarBQDo9Xro9XpLh9ZrtN1Hs6WlRdbj7Cv6yv60ZGxWC7iamhq0trbC29vbqNzb2xvFxcUmt6uvr4e/vz90Oh1sbGzw0UcfYdq0adLr06dPx5w5cxAUFITS0lK89dZbmDFjBvLz82Fj0/5u4ACQnp6OtLS0duWHDh2Ck5PTQ46w57veAAC2OHXqFH68YO3e0KPqK/uzqanJ7LpWPQb3MFxdXVFYWIiGhgbk5eVBo9EgODgYkyZNAgDMnTtXqjtq1CiMHj0aQ4YMwdGjRzF16tQO20xJSYFGo5Gea7VaBAQEIDY2Fm5ubt06Hms6X34bKDqL8ePHY8yg/tbuDj2ivrI/2z5hmcNqAefp6QkbGxtUV1cblVdXV8PHx8fkdkqlEiEhIQCAsLAwXLp0Cenp6VLA/VJwcDA8PT1x+fJlkwGnUqk6PBFhZ2cHOzs7M0fU+9ja2ko/5TzOvqKv7E9Lxma1kwz29vYIDw9HXl6eVGYwGJCXl4fo6Giz2zEYDEbHz37pxo0bqK2tha+v7yP1l4h6H6t+RNVoNEhMTERERAQiIyORmZmJxsZGJCUlAQASEhLg7++P9PR0APePlUVERGDIkCHQ6XQ4cOAAdu/ejS1btgAAGhoakJaWhueeew4+Pj4oLS3FihUrEBISYnQZCRH1DVYNuPj4eNy6dQupqamoqqpCWFgYcnJypBMP5eXlUCr/NslsbGzEkiVLcOPGDTg6OiI0NBQff/wx4uPjAQA2Njb47rvvsHPnTtTV1cHPzw+xsbFYs2YNr4Uj6oMUQghh7U70NFqtFu7u7qivr5f1SYbCa7WYveUU9i8ej7DBpi+jod6hr+xPS/4+rf5VLSKi7sKAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki2r3heVHq+mpiYUFxdLz0sq66CruoxLFxxhqO0nlYeGhsLJyckKPSTqWgy4PqS4uBjh4eHtyl/aafy8oKAA48aNe0y9Iuo+Vv+IunnzZgQGBsLBwQFRUVH45ptvTNbdt28fIiIi0K9fPzg7OyMsLAy7d+82qiOEQGpqKnx9feHo6IiYmBj89a9/7e5h9AqhoaEoKCiQHsdOnMSKNRtw7MRJo/LQ0FBrd5WoS1h1Brd3715oNBpkZWUhKioKmZmZUKvVKCkpgZeXV7v6/fv3x9tvv43Q0FDY29vjiy++QFJSEry8vKBWqwEA77//PjZt2oSdO3ciKCgIK1euhFqtxvfffw8HB4fHPcQexcnJyWhmptfrcafmJqIjI2BnZ2fFnhF1E2FFkZGR4vXXX5eet7a2Cj8/P5Genm52G2PHjhXvvPOOEEIIg8EgfHx8xLp166TX6+rqhEqlEp9++qnZbdbX1wsAor6+3uxteqPm5maxf/9+0dzcbO2uUBf4tqxGDP7tF+Lbshprd6VbWfL3abUZXHNzMwoKCpCSkiKVKZVKxMTEID8//4HbCyHw1VdfoaSkBO+99x4A4OrVq6iqqkJMTIxUz93dHVFRUcjPz8fcuXM7bEun00Gn00nPtVotgPszHL1e/1Dj6w3axibnMfYlLS0t0k8571NLxma1gKupqUFrayu8vb2Nyr29vY3O9P1SfX09/P39odPpYGNjg48++gjTpk0DAFRVVUlt/LLNttc6kp6ejrS0tHblhw4d6hNnE3Nzc63dBeoC1xsAwBanTp3Cjxes3Zvu09TUZHbdXncW1dXVFYWFhWhoaEBeXh40Gg2Cg4MxadKkh24zJSUFGo1Geq7VahEQEIDY2Fi4ubl1Qa97Jr1ej9zcXEybNo3H4GTgfPltoOgsxo8fjzGD+lu7O92m7ROWOawWcJ6enrCxsUF1dbVReXV1NXx8fExup1QqERISAgAICwvDpUuXkJ6ejkmTJknbVVdXw9fX16jNsLAwk22qVCqoVKp25XZ2dn3iD7+vjFPubG1tpZ9y3p+WjM1ql4nY29sjPDwceXl5UpnBYEBeXh6io6PNbsdgMEjHz4KCguDj42PUplarxenTpy1qk4jkwaofUTUaDRITExEREYHIyEhkZmaisbERSUlJAICEhAT4+/sjPT0dwP1jZRERERgyZAh0Oh0OHDiA3bt3Y8uWLQAAhUKBZcuWYe3atRg6dKh0mYifnx9mz55trWESkZVYNeDi4+Nx69YtpKamoqqqCmFhYcjJyZFOEpSXl0Op/Nsks7GxEUuWLMGNGzfg6OiI0NBQfPzxx4iPj5fqrFixAo2NjVi0aBHq6uowceJE5OTk9Plr4Ij6IoUQQli7Ez2NVquFu7s76uvrZX+S4cCBA3j22Wdlfcymryi8VovZW05h/+LxCBs8wNrd6TaW/H1a/ataRETdhQFHRLLFgCMi2WLAEZFsMeCISLZ63Ve1iOg+c1doBvruKs0MOKJeytwVmoG+u0ozA46ol2pboblNw086fHkkH/8wORoujqp2dfsiBhxRL8UVmh+MJxmISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLLFgCMi2eJ1cB1oWwPUkrv39EZ6vR5NTU3QarW8bkoG+sr+bPu7NGetXgZcB+7evQsACAgIsHJPiMiUu3fvwt3dvdM6XLK8AwaDARUVFXB1dYVCobB2d7pN2/1fr1+/Luul2fuKvrI/hRC4e/cu/Pz8jO7Z0hHO4DqgVCrxxBNPWLsbj42bm5us/yD6mr6wPx80c2vDkwxEJFsMOCKSLQZcH6ZSqbBq1SqoVKoHV6Yej/uzPZ5kICLZ4gyOiGSLAUdEssWAIyLZYsD1cJMmTcKyZcus3Q2iXokBJxNHjx6FQqFAXV2dtbtCFuhp/4D1tP48KgYcUS/X3Nxs7S70WAy4XkSn0+G3v/0tAgICoFKpEBISgm3btqGsrAyTJ08GAHh4eEChUGDBggUPbO/u3bt4+eWX4ezsDF9fX3zwwQft/gXfvXs3IiIi4OrqCh8fH7z00ku4efOm9HrbzDEvLw8RERFwcnLChAkTUFJS0tXDl50FCxbg2LFj2LhxIxQKBRQKBUpLS7Fw4UIEBQXB0dERw4YNw8aNG9ttN3v2bPz+97+Hn58fhg0bBgA4efIkwsLC4ODggIiICOzfvx8KhQKFhYXSthcuXMCMGTPg4uICb29vzJ8/HzU1NSb7U1ZW9rh+Hd1DUI/2zDPPiDfeeEMIIcSLL74oAgICxL59+0Rpaak4fPiw2LNnj2hpaRGff/65ACBKSkpEZWWlqKure2Dbr732mhg8eLA4fPiwKCoqEr/+9a+Fq6ur9H5CCLFt2zZx4MABUVpaKvLz80V0dLSYMWOG9PqRI0cEABEVFSWOHj0qLl68KJ5++mkxYcKErv5VyE5dXZ2Ijo4WycnJorKyUlRWVop79+6J1NRUcebMGXHlyhXx8ccfCycnJ7F3715pu8TEROHi4iLmz58vLly4IC5cuCDq6+tF//79xSuvvCIuXrwoDhw4IP7u7/5OABDffvutEEKIO3fuiIEDB4qUlBRx6dIlce7cOTFt2jQxefJkk/1paWmxxq+myzDgeri2gCspKREARG5ubof12oLmzp07ZrWr1WqFnZ2d+Oyzz6Syuro64eTkZBRwv3TmzBkBQNy9e9fofQ8fPizV+fLLLwUA8dNPP5nVl77s5/+AmfL666+L5557TnqemJgovL29hU6nk8q2bNkiBgwYYPQ737p1q1HArVmzRsTGxhq1ff36dekfRnP705vwI2ovUVhYCBsbGzzzzDNd0t6VK1eg1+sRGRkplbm7u0sfd9oUFBRg1qxZGDRoEFxdXaX3Ly8vN6o3evRo6b99fX0BwOijLJlv8+bNCA8Px8CBA+Hi4oJ///d/b/f7HjVqFOzt7aXnJSUlGD16NBwcHKSyn+9bADh//jyOHDkCFxcX6dF2x/vS0tJuHJH1cLmkXsLR0fGxv2djYyPUajXUajU++eQTDBw4EOXl5VCr1e0ObP98Bdm2NfQMBsNj7a8c7NmzB8uXL8eGDRsQHR0NV1dXrFu3DqdPnzaq5+zsbHHbDQ0NmDVrFt577712r7X9oyQ3DLheYtSoUTAYDDh27BhiYmLavd72r3lra6tZ7QUHB8POzg5nzpzBoEGDAAD19fX44Ycf8Pd///cAgOLiYtTW1uIPf/iDtLrx2bNnu2I49H/s7e2N9tnXX3+NCRMmYMmSJVKZObOrYcOG4eOPP4ZOp5O+bH/mzBmjOuPGjcPnn3+OwMBA2Np2/Kf/y/70dvyI2ksEBgYiMTERr776Kvbv34+rV6/i6NGj+K//+i8AwODBg6FQKPDFF1/g1q1baGho6LQ9V1dXJCYm4s0338SRI0dw8eJFLFy4EEqlUpqBDRo0CPb29vjwww9x5coV/OlPf8KaNWu6fax9SWBgIE6fPo2ysjLU1NRg6NChOHv2LA4ePIgffvgBK1eubBdUHXnppZdgMBiwaNEiXLp0CQcPHsT69esB/G1G/frrr+P27duYN28ezpw5g9LSUhw8eBBJSUlSqP2yP719Fs6A60W2bNmC559/HkuWLEFoaCiSk5PR2NgIAPD390daWhp+97vfwdvbG0uXLn1gexkZGYiOjsbMmTMRExODp556CsOHD5eO4wwcOBA7duzAZ599hhEjRuAPf/iD9EdDXWP58uWwsbHBiBEjMHDgQKjVasyZMwfx8fGIiopCbW2t0WzOFDc3N/z5z39GYWEhwsLC8PbbbyM1NRUApP3p5+eHr7/+Gq2trYiNjcWoUaOwbNky9OvXT1r6+5f9+eWxv96GyyWRpLGxEf7+/tiwYQMWLlxo7e7QI/rkk0+QlJSE+vp6qxzD7Ql4DK4P+/bbb1FcXIzIyEjU19dj9erVAIC4uDgr94wexq5duxAcHAx/f3+cP38ev/3tb/Hiiy/22XADGHCyVV5ejhEjRph8/fvvvwcArF+/HiUlJbC3t0d4eDiOHz8OT0/Px9VN6kJVVVVITU1FVVUVfH198cILL+D3v/+9tbtlVfyIKlMtLS2dfs2mszNpRHLBgCMi2eJZVCKSLQYcEckWA46IZIsBR0SyxYAjq1qwYIG0uKKdnR28vb0xbdo0ZGdnW/Q1oR07dqBfv37d11ET2hafpJ6JAUdWN336dFRWVqKsrAx/+ctfMHnyZLzxxhuYOXMmWlparN096s2suRgdUWJiooiLi2tXnpeXJwCIrVu3CiGE2LBhgxg5cqRwcnISTzzxhFi8eHG7RTd//li1apUQQohdu3aJ8PBw4eLiIry9vcW8efNEdXW19D63b98WL730kvD09BQODg4iJCREZGdnS6+Xl5eLF154Qbi7uwsPDw/xq1/9Sly9elUIIcSqVavave+RI0e65fdED4czOOqRpkyZgjFjxmDfvn0AAKVSiU2bNuHixYvYuXMnvvrqK6xYsQIAMGHCBGRmZsLNzQ2VlZWorKzE8uXLAQB6vR5r1qzB+fPnsX//fpSVlRndr2LlypX4/vvv8Ze//AWXLl3Cli1bpG9y6PV6qNVquLq64vjx4/j666/h4uKC6dOno7m5GcuXL8eLL74ozUArKysxYcKEx/uLos5ZO2GpbzM1gxNCiPj4eDF8+PAOX/vss8/EgAEDpOfbt28X7u7uD3y/Xy65PmvWLJGUlNRh3d27d4thw4YJg8Eglel0OuHo6CgOHjz4wP6T9XEGRz2WEEJay+zw4cOYOnUq/P394erqivnz56O2thZNTU2dtvGgJdcXL16MPXv2ICwsDCtWrMDJkyelbc+fP4/Lly/D1dVVWuK7f//+uHfvnmyX+JYbBhz1WJcuXUJQUBDKysowc+ZMjB49Gp9//jkKCgqwefNmAJ3fE7RtyXU3Nzd88sknOHPmDP74xz8abTdjxgxcu3YNv/nNb1BRUYGpU6dKH28bGhoQHh6OwsJCo8cPP/yAl156qZtHT12B37amHumrr75CUVERfvOb36CgoAAGgwEbNmyQFmZsW8m4TUdLbZu75PrAgQORmJiIxMREPP3003jzzTexfv16jBs3Dnv37oWXlxfc3Nw67KfclviWG87gyOp0Oh2qqqrw448/4ty5c3j33XcRFxeHmTNnIiEhASEhIdDr9dLS6bt370ZWVpZRG4GBgWhoaEBeXh5qamrQ1NRk1pLrqamp+J//+R9cvnwZFy9exBdffIHhw4cDAF5++WV4enoiLi4Ox48fl5aJ/+d//mfcuHFDet/vvvsOJSUlqKmpgV6vfzy/NDKPtQ8CUt+WmJgoXWJha2srBg4cKGJiYkR2drZobW2V6mVkZAhfX1/h6Ogo1Gq12LVrV7v7wP7TP/2TGDBggNFlIv/5n/8pAgMDhUqlEtHR0eJPf/pTu3uFDh8+XDg6Oor+/fuLuLg4ceXKFanNyspKkZCQIDw9PYVKpRLBwcEiOTlZ1NfXCyGEuHnzppg2bZpwcXHhZSI9EJdLIiLZ4kdUIpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLL1/wHbIOfvPuZxfwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T00:58:39.782060Z", + "iopub.status.busy": "2024-03-01T00:58:39.781728Z", + "iopub.status.idle": "2024-03-01T00:58:40.056571Z", + "shell.execute_reply": "2024-03-01T00:58:40.055658Z" + }, + "papermill": { + "duration": 0.299911, + "end_time": "2024-03-01T00:58:40.058656", + "exception": false, + "start_time": "2024-03-01T00:58:39.758745", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEmCAYAAAAdlDeCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABrVElEQVR4nO2dd3gUVduHf7M9PYGEFEgIvUkvAUQRDQZflWYBVLqgIuBrQKQjTVABeVEERSkqTSzgp4hCBBQIoCH0DqEnISSkJ9vmfH9sZnZmd3azu9klyebc17UXm5kzZ8+wZ5952nkOQwghoFAoFC9HVtkDoFAolAcBFXYUCqVGQIUdhUKpEVBhR6FQagRU2FEolBoBFXYUCqVGQIUdhUKpEVBhR6FQagSKyh5AVYRlWdy5cwcBAQFgGKayh0OhUCwghKCgoABRUVGQyRzT2aiwk+DOnTuIjo6u7GFQKJRyuHnzJurVq+dQWyrsJAgICABg+o8MDAys5NFQKBRL8vPzER0dzf9WHYEKOwk40zUwMJAKOwqlCuOMm4kGKCgUSo2ACjsKhVIjoMKOQqHUCKjPzksxGo3Q6/WVPQxKFUUul0OhUNSo1Coq7LyQwsJC3Lp1C7QuK8Uevr6+iIyMhEqlquyhPBCosPMyjEYjbt26BV9fX4SFhdWoJzfFMQgh0Ol0yMrKQlpaGpo0aeJwYm51hgo7N7L3/F2sP3QNi59rjcggn0oZg16vByEEYWFh8PGpnDFQqj4+Pj5QKpW4fv06dDodNBpNZQ/J43i/OH+ArD90DfsvZuHP83creyhUo6OUS03Q5oTUrLv1MJn5pQAArZ6t5JFQKBRLqLBzIxllwk5vpMKOQqlqUGHnJkr1RuQWm1I9qLBznsceewz//e9/K3sYXsO+ffvAMAxyc3MreyhVBirs3ERWgZZ/rzNQYecp6I+Y4ipU2LkJzoQFAJ2R5rdRKo5Op6vsIXgVVULYrVy5ErGxsdBoNIiLi8PRo0dttl2zZg0eeeQRhISEICQkBPHx8VbtCSGYPXs2IiMj4ePjg/j4eFy6dMmj95ApEHZVyYwlhKBYZ6iUl6tJzVqtFu+++y6io6OhVqvRuHFjfPXVV7h27Rp69eoFAAgJCQHDMBgxYkS5/RUUFODll1+Gn58fIiMj8fHHH1uZzd988w06deqEgIAARERE4KWXXsLdu+aoOqdRJiUloVOnTvD19UX37t1x4cIFh+9rwYIFqFOnDgICAvDqq69i6tSpaNeuHX9+xIgR6N+/PxYuXIioqCg0a9bMobEBwM6dO9G0aVP4+PigV69euHbtmsPjqilUep7d1q1bkZiYiNWrVyMuLg7Lly9HQkICLly4gDp16li137dvH4YMGYLu3btDo9Hggw8+wJNPPokzZ86gbt26AIAPP/wQK1aswIYNG9CgQQPMmjULCQkJOHv2rMfyiTLyqqawK9Eb0XL275Xy2WfnJcBX5fwUGzZsGJKTk7FixQq0bdsWaWlpuHfvHqKjo/HDDz/gueeew4ULFxAYGOhQLmFiYiIOHjyIn3/+GeHh4Zg9ezaOHTsmEjR6vR7z589Hs2bNcPfuXSQmJmLEiBHYuXOnqK8ZM2Zg6dKlCAsLw+uvv45Ro0bh4MGD5Y5h48aNWLhwIT777DM8/PDD2LJlC5YuXYoGDRqI2iUlJSEwMBC7d+92eGw3b97EwIED8eabb2Ls2LH4999/MWnSpHLHVNOodGG3bNkyjBkzBiNHjgQArF69Gr/++ivWrl2LqVOnWrXfuHGj6O8vv/wSP/zwA5KSkjBs2DAQQrB8+XLMnDkT/fr1AwB8/fXXCA8Px/bt2zF48GCP3Mdd6rNzCxcvXsR3332H3bt3Iz4+HgDQsGFD/nytWrUAAHXq1EFwcHC5/RUUFGDDhg3YtGkTnnjiCQDAunXrEBUVJWo3atQo/n3Dhg2xYsUKdO7cGYWFhfD39+fPLVy4ED179gQATJ06FU8//TRKS0vLfYh+8sknGD16ND/PZ8+ejT/++AOFhYWidn5+fvjyyy9FS7jKG9uqVavQqFEjLF26FADQrFkznDp1Ch988EG5/z81iUoVdjqdDikpKZg2bRp/TCaTIT4+HsnJyQ71UVxcDL1ez/8I0tLSkJGRwf9QACAoKAhxcXFITk6WFHZarRZarVlY5efnO30vmSKfXdURdj5KOc7OS6i0z3aW48ePQy6X8wKloly9ehV6vR5dunThjwUFBfEmIkdKSgree+89nDhxAvfv3wfLmr7DGzduoGXLlny7Nm3a8O8jIyMBAHfv3kVMTIzdcVy4cAHjxo0THevSpQv+/PNP0bHWrVtbrVUtb2znzp1DXFyc6Jpu3brZHU9NpFKF3b1792A0GhEeHi46Hh4ejvPnzzvUx7vvvouoqCheuGVkZPB9WPbJnbNk0aJFmDt3rrPDFyE2Y6tOgIJhGJdMycqiMpa4FRUVISEhAQkJCdi4cSPCwsJw48YNJCQkWAUJlEol/55bpcIJH3fg5+fn8tgo9qkSAQpXWbx4MbZs2YKffvqpQr64adOmIS8vj3/dvHnT6T6EZqyemrEu07p1a7Asi/3790ue57Qeo9HoUH8NGzaEUqnEP//8wx/Ly8vDxYsX+b/Pnz+P7OxsLF68GI888giaN29uFQCoKM2aNRONAYDV31I4MrYWLVpYBekOHz5c8UF7GZUq7EJDQyGXy5GZmSk6npmZiYiICLvXLlmyBIsXL8Yff/whMi2465zpU61W8/tNuLLvBCGkypqx1Y3Y2FgMHz4co0aNwvbt25GWloZ9+/bhu+++AwDUr18fDMPgl19+QVZWlpXPy5KAgAAMHz4c77zzDvbu3YszZ85g9OjRkMlkvGYWExMDlUqFTz75BFevXsXPP/+M+fPnu/W+JkyYgK+++gobNmzApUuXsGDBApw8ebLcNcyOjO3111/HpUuX8M477+DChQvYtGkT1q9f7/JY80r00Boce5hUJypV2KlUKnTs2BFJSUn8MZZlkZSUZNfn8OGHH2L+/PnYtWsXOnXqJDrXoEEDREREiPrMz8/HkSNHPObHKNAaUKwzT46qFI2tjqxatQrPP/88xo0bh+bNm2PMmDEoKioCANStWxdz587F1KlTER4ejvHjx5fb37Jly9CtWzc888wziI+Px8MPP4wWLVrw1kBYWBjWr1+Pbdu2oWXLlli8eDGWLFni1nt6+eWXMW3aNEyePBkdOnRAWloaRowYUa5F4sjYYmJi8MMPP2D79u1o27YtVq9ejffff9+lcRaU6tFj8Z94ec0Rl66v0pBKZsuWLUStVpP169eTs2fPkrFjx5Lg4GCSkZFBCCFk6NChZOrUqXz7xYsXE5VKRb7//nuSnp7OvwoKCkRtgoODyY4dO8jJkydJv379SIMGDUhJSYlDY8rLyyMASF5enkPtL2Xmk/rv/sK/Xlx9yIn/AfdSUlJCzp496/C91kQKCwtJUFAQ+fLLLyt1HPHx8eSVV16ptM+XmisXMkxzufWcXZU2Lkdw9jdKCCGV7rkeNGgQsrKyMHv2bGRkZKBdu3bYtWsXH2C4ceOGqBTNqlWroNPp8Pzzz4v6mTNnDt577z0AwJQpU1BUVISxY8ciNzcXPXr0wK5duzyYY6cV/U01u6pFamoqzp8/jy5duiAvLw/z5s0DAD416UFQXFyM1atXIyEhAXK5HJs3b8aePXtE+XRVASNLRP96E5Uu7ABg/PjxNs2Rffv2if52JDOcYRjMmzePn9SehvPXMQxACPXZPUgsU0MsOXv2LACTj/fChQu86+Tvv/9GaGio28bRqlUrXL9+XfLc559/joEDB2Lnzp1YuHAhSktL0axZM/zwww+iFKmqAC/svLCkf5UQdtWdzAKTsIsI1CA9rxR6g/dNlKpKVFQUjh8/bvd8TEwMUlJSPDqOnTt32tzgKDw8HD4+PtizZ49Hx+AOWEI1O4odMsty7OqF+JiEHdXsHhgKhQKNGzeu7GGgfv36lT0Et+DNZmy1zrOrKmTmm3x20SG+AKgZS6m+cEKOJfC63emosHMDXHmnerXKhB1NKqZUU4Qanbdpd1TYuYG7+WYzFqDRWEr1RRiYMFBhRxHCsoRfKsaZsVVpbSyF4gzCZb4sNWMpQrKLdDCwBAwD1A02aXbUZ0eprlDNjmITLscu1F8NH5WppJHOwHqdc7eqExsbi+XLl/N/MwyD7du3V9p4qiusQMCxXibsaOpJBeGEXXigGiq5+dlhYAmUcrpRdWWRnp6OkJCQyh5GtUMYlPA2zY4KuwrCpZ1EBGqgVJiFm97IQimninNlUV7VnAcBIQRGoxEKRfX5mQnNWG/T7OivsYJwml2dQI1Is6syqygIAXRFlfNy0pR3ZGMcRxGasdeuXQPDMPjxxx/Rq1cv+Pr6om3btlbVsA8cOIBHHnkEPj4+iI6OxsSJE/lqK4Djm/L89ttv6NixI9RqNQ4cOOD02CsTlmp2FFvwZmyABnIZw6+P1RqNAJT2L34Q6IuB96PKb+cJpt8BVH7ltyvDkY1xKsKMGTOwZMkSNGnSBDNmzMCQIUNw+fJlKBQKXLlyBX369MGCBQuwdu1aZGVl8Wu2161bB8DxTXmmTp2KJUuWoGHDhtXOlBZqdt6WZ0eFXQXhhF1EkBoMw0Apl0FnYGn6iZM4ujFORZg8eTKefvppAMDcuXPRqlUrXL58Gc2bN8eiRYvw8ssv81pkkyZNsGLFCvTs2ROrVq2CRqNxeFOeefPmoXfv3m4b94PEm5OKqbCrIJzPrk6gqXyUihN2blhFoTUYcT69AK3rBkEmczHYofQ1aViVgdLX4aaOboxTEWxtltO8eXOcOHECJ0+eFO1eRwgBy7JIS0tDixYtHN6Ux7KgbHVCJOy8LKOACrsKwmt2nLBTyACte1ZRjP06BfsvZmHm0y3w6iMNy79ACoZxypT0ZuxtllNYWIjXXnsNEydOtLouJibGqY1vLDfNqU5QzY4iic7AIrvINNHDy4Qdl26idYNmt/9iFgBg/aFrrgu7aoJwYxxuW0JuY5xHH33U45/foUMHnD171mYFlVOnTvEb30RHRwMA/v33X4+P60HDerHPjkZjK8Ddsjp2KrkMIb4mrYFLN3Hn+lhvSwGQwpGNcTzJu+++i0OHDmH8+PE4fvw4Ll26hB07dvBFZR/EpjxVAeG0pcKOwmP216n5H6SKF3bumyheNudsUt7GOJ6kTZs22L9/Py5evIhHHnkE7du3x+zZs/kAyYPYlKcqQKOxFEnu8qsnzD9GlcL9mp23OYptERAQIAoQFBUVYe7cuRg7dmy511qW6xcu14uNjbVavhccHGx1rHPnzvjjjz9sfsaQIUMwZMgQm5/z2GOPVYtlgum5JSgoNaBuoNzqHM2zo0iSYRGcAMxmrDtr2tUEMxaoGhvjeDssIcgu0oElBFqDtXtAqM15W9UTKuwqQJCPEu1jgtEsIoA/xgUo3Fn5xNsmnT2kNsY5d+4cnnrqKZvXlLdRNsVMic7IzyepWSWcawYvyxWlwq4CDOxQDwM71BMd80iAwrvmnE3at28vuTFOSUmJ3U11KI5TpDWY/5CYV1SzoziMJ3x2NcWMtYWPj0+V2FTHGygUCjsJaWfwYp8djca6GZUnfHYuPGGrg6Oc8mAhhKBYZxT9bYk317Ojws7N8AEKN/o7nInGyuVlBUQtsvoplBK9UfTg1JaaAmzClSXeXKmYmrFuRsmZsW7V7Bxvq1Ao4Ovri6ysLCiVSshk9HlGMZFbpAUx6ABCQAw6ZJXmIzy0Fv+ABMTaHM2zo9hFVckrKBiGQWRkJNLS0nD9+nW3jYFS/cku1KJEzwIg0BsJAgKD0NaiyClNKqY4jKqsWnFl+uxUKhWaNGlCTVkKj5EleOuzgyjSGqCSy5FeqMf/htSzWoonfEYbWO/aOIoKOzdTVVJPZDLZA1lmRakenL2Tj4v3tPBTydE0whdpubmSFoPwweptqSfUoeNmPBGgoFAqytG0bABAx9havKtFKvAl2nDHy+YwFXZuxhN5dhRKRTmSlgMAiGtQCzKull8NSyqmws7NeGJtLIVSEQghOCoQdnIZV7jUvhnrbaknVNi5GVXZ2liq2VGqCjlFOr7IbOt65hL/UtFWA00qpjiK2WdHhR2lasDNRZVcBrVCDm47Eykz1ZtLPFFh52bMPjvvmiiU6gu3hzFXkUfO++zsByi8Lc+OCjs3Y/bZGctpSaE8GPRl+XKKsrnJmbGSAQovTiqmws7NeKIsO4VSETj/Mfcg5sxYKWEmWi5Go7EUeygVNEBBqVpYmbEyO2as4JDRyx7YVNi5GRVXdYSmnlCqCJwZa9bs7KSeUM1OmsLCQuTn54tezrJy5UrExsZCo9EgLi4OR48etdn2zJkzeO655xAbGwuGYbB8+XKrNu+99x4YhhG9mjdv7vS4XMUTZdkplIrAVeBRlM1NTthJKW40QCEgLS0NTz/9NPz8/BAUFISQkBCEhIQgODgYISEhTvW1detWJCYmYs6cOTh27Bjatm2LhIQE3L17V7J9cXExGjZsiMWLFyPColqDkFatWiE9PZ1/HThwwKlxVQQlXUFBqWJwKSScP9leUrE3ByicLgTwyiuvgBCCtWvXIjw8vEIbGC9btgxjxozByJEjAQCrV6/Gr7/+irVr12Lq1KlW7Tt37ozOnTsDgOR5DoVCYVcYehI+QGHwrolCqb7orAIUNTP1xGlhd+LECaSkpKBZs2YV+mCdToeUlBRMmzaNPyaTyRAfH4/k5OQK9X3p0iVERUVBo9GgW7duWLRoEWJiYmy212q10Gq1/N+umOMcdG0sparBLeg3m7Gm4+UVAvA2Yee0Gdu5c2fcvHmzwh987949GI1GhIeHi46Hh4cjIyPD5X7j4uKwfv167Nq1C6tWrUJaWhoeeeQRFBQU2Lxm0aJFCAoK4l/R0dEufz739NTSAAWlimCZelJT18Y6rdl9+eWXeP3113H79m089NBDovr1ANCmTRu3Dc4VhPuLtmnTBnFxcahfvz6+++47jB49WvKaadOmITExkf87Pz/fZYGnpGtjKVUMs7Ar0+zsJRV7cdUTp4VdVlYWrly5wvvZAFMpcEIIGIaB0ejYyoHQ0FDI5XJkZmaKjmdmZrrV3xYcHIymTZvi8uXLNtuo1Wqo1Wq3fJ7ajWasUs64NTn5xM1cfP7XFUx7qgWia/m6rV9K1YabQ44kFRvp2lgzo0aNQvv27ZGcnIyrV68iLS1N9K+jcDu+JyUl8cdYlkVSUhK6devm7LBsUlhYiCtXriAyMtJtfdpD6cYVFFxf7qLfyoPYeSoD4zYec2u/lKoN9+BVlG2+ZG9trKhSsZcJO6c1u+vXr+Pnn392y6bFiYmJGD58ODp16oQuXbpg+fLlKCoq4rXGYcOGoW7duli0aBEAU1Dj7Nmz/Pvbt2/j+PHj8Pf358czefJkPPvss6hfvz7u3LmDOXPmQC6XY8iQIRUeryO4s56dSiET7fPpLq5lF7m9T0rVxcBVPVFYmrE1S7NzWtg9/vjjOHHihFuE3aBBg5CVlYXZs2cjIyMD7dq1w65du/igxY0bN0RbAd65cwft27fn/16yZAmWLFmCnj17Yt++fQCAW7duYciQIcjOzkZYWBh69OiBw4cPIywsrMLjdQRhiSfOtK9oX+5GVoExUaofOiszlqtnZ91WaJDUeM3u2Wefxdtvv41Tp06hdevWVgGKvn37OtXf+PHjMX78eMlznADjiI2NLXen+y1btjj1+e6GSz0BTE9GzinsUl8eEnZU1tUsDJZmrB3Nzpvr2Tkt7F5//XUAwLx586zOOROg8FaEAkpvZCuknVVEUFIoHHpLM9bO2lijF6+NdVrYsV62l6S7EQoonYGFr8r1voRaojuhIrRmwZmxnGbncFJxTa56otfroVAocPr0aU+Np9ojlzG8mVjRYgDUZ0dxBwYnkopFa2O9TLNz6tekVCoRExNT401VezAM47YCnkJhZ3BjkjKVdTULy6Rixs5WiixdLmZmxowZmD59OnJycjwxHq/AXAygYgJK7P9z58Sj0q4mYZlULOdLPNGqJ3b59NNPcfnyZURFRaF+/frw8/MTnT92jCasKhUyQOsGM1Yh9v/5qOQVHRoAs8+GUn1YdzANBy7dw2evdIBa4dw8sF4bazpeboCipgu7/v37e2AY3gVfwLOCmp1CkGPozmKg1Iytfqw/dA3Xs4tx+nY+OtZ3rm6kZdUTxt4KChvCbnvqbfxy8g6WD24Pf7XTYqNK4PSo58yZ44lxeBWeKPPkVmFHzdhqB+cScUXb0gv2jQXMAQrppGJpYbf+0DUcv5mL/Rey8HSbB7P00t24LKJTUlJw7tw5AKbKwMKVDTUdd66P5aio/49SvdGXCR5XhB33oFQ4tG+s+b1BkGbGfe6d3BKnP7+q4LSwu3v3LgYPHox9+/YhODgYAJCbm4tevXphy5YtD2xZVlVG5ab1scKp6E7Njvrsqh+csHGl7JLBIkDBuTHKKwQgfFaTstl4uxoLO6ejsRMmTEBBQQHOnDmDnJwc5OTk4PTp08jPz8fEiRM9McZqh1mzc6MZ60bNriLrdSmVA5d6VBEz1nIrxfJKPBkFmh33Nj2v+go7pzW7Xbt2Yc+ePWjRogV/rGXLlli5ciWefPJJtw6uusL57NypjdHdymo2nBByJdGXM4GtkorLLctuPs4dvZNb6vTnVxWc1uxYlrVa/A+YEo7pUjIT7qpWLCx64E7NTuaZhRkUD8IJLFcqkXD+XrMZy62gsG5rS7Pj5mJ11uycnvaPP/443nrrLdy5c4c/dvv2bbz99tt44okn3Dq46oo7a9px2BKcafeK0O/TA9h12vF9O2g0tvphrECAwsBamLEuJBVzh+8V6lCqr54rqJwWdp9++iny8/MRGxuLRo0aoVGjRmjQoAHy8/PxySefeGKM1Q7VA/TZvbPtBE7cysPr36Y43Bd12VUvCCEVClBY1rOzl1RsK89O+LkZedXTlHXaZxcdHY1jx45hz549OH/+PACgRYsWiI+Pd/vgqitmn50bU09sCM78Ur3TfVFZV72w5UdzFM6MVViaseVpdoL3wpZ3cksQGypeOVUdcCnPjmEY9O7dG71793b3eLwCpZvWxgqxtTWjKyYpjcZWL4RFNF0JUNg2Y8XtCCEQdi8s8SQUjHdqimYHAElJSUhKSsLdu3etghJr1651y8CqM8LS7O7CnQnKVNRVLwwi35krqSfS0VjLviz9gSLBKnhbXROLnRZ2c+fOxbx589CpUydERkZSLUECriJsRTU74Vyz5bNz6b+ffmXVCqGGVbE8O3FSsV3hBts+uxoj7FavXo3169dj6NChnhiPV+CJAIU7+6LFO6sXUsu2rmQVYus/NzH20YYI9be/57GjScWWqSiiaKzgeI0xY3U6Hbp37+6JsXgNZjPWfaanM2ks8/7vLApK9fjw+TaSmjcVddULg4SG9cyKAyjRG3EhowAbRnUBALz38xlkFWrx6ZD2ou/dcrkY57OztIgNFtJO6nOB6qvZOZ168uqrr2LTpk2eGIvXoFS4a22sIKnYhmZnqaUZjCzWHkzDtpRbyCnSSV5DFbvqhUEiGltSluuWeuM+f+7r5Gv49WQ60i00L52VGSudZ2ep2bESeXYAkJ5b4pLvsLJxWrMrLS3FF198gT179qBNmzZWqymWLVvmtsFVVx7k2lhLwSWM2toKatCk4uqFyGdnIWSEGhwnm4p1BlEbbh4qZPbNWMu+DTaEXZHOiPwSA4J8rVdSVWWcFnYnT55Eu3btAMBq4x0arDChdlM9O1GAwqKvUr0Rp2/nWe0joBMJOzcGNSiVhl60IN9S2Fm3L9SKVzhwZiyX/8klFZcXjRWarpZt7+SVeL+w27t3ryfG4VXwlYrdGaCw0Ow++v0CvjqQZtVO+Jnu1CwplYe9UumcrBMKo2KtWbMjhPAaGqfZ2TRj7Wh23Fu5jIGRJbiTW4IWkYEu3E3lQZeEewBPrI21FJxJ5zKl2wk+09aO7lQDr14YbCT3ChEeLhQIO6Erg/Ml80nFdqKvXJ+cJsn5jyMCNQCqZ5CCCjsP4C6fnXACC/tKzyvBtexiyWu0DpixtHhn9cKuZiex9KtYZzZjhXPAsix7eWYsYNb+uFN1Q3wAVM/0EyrsPIB5Dwr3RayEQiz5SrbNdjpHAhRU2FUrhD47qwBF2b/Co0LNTqgVms3Ysr7s+Oj4z+M0u7JT9YLLhB3V7CiAp5KKzRPRrrATfKatjbVpNLZ6IVrJYCNAIZRTwmgsNx8YxqzR2SrxxH2OcL9is7Az/ctpdumeKOLp4XQWp4XdX3/9BYPBYHXcYDDgr7/+csugqjucGWtr8b6jiPLsDGbTJPlqxTQ7asZWLwyi5WLSbYRzRRiN5YsAyGS8yWs2Y8V9cJodZ5kAZoHINa1bptm5fS8KQoDfZwAHPnZvvwKcFna9evVCTk6O1fG8vDz06tXLLYOq7jizlWJWgRaLdp5D2r0iu+04wXUzpxi37tueaFqD9US3gtqx1QqRz85K+7EWXMJorN7ArZ4wf+d8NNZCS+QCWiJhZ5TW7DLzS923iTYhwO5ZwOGVwJ73gPST7unXAqeFHSFEMpqXnZ0NP7/qV+PKEzhTlv2tLan4/K+rGPjZQbvtOI3NnlYnbOfo51OqPvby7KQokjBjFQLT1GZSsUWKCmAdoAgP1EAhY2BgCbIKtM7chjSEAEnzgENlhX+fXgZEtql4vxI4nGc3cOBAAKanwogRI6BWmxcfG41GnDx5kq6ZLYP32RnKn5hH0kxa8v1i6yKcUknFnL8uQK1AgdbaneBQgKLcUVGqEvZXUJj+Fa1wkDJjBcJOxl8jvVxMLmP4fDpLn51cxiA8UIPbuSW4nVuCiCCN6zcGAPsWAQfKVl099RHQeXTF+rODw8IuKCgIgOmmAwIC4OPjw59TqVTo2rUrxowZ4/4RVkOUTuwu5qgpoDOwIITwwu7hxqHYdcZ63wlxgIL67LwBYb7kqn1X0L9dXf5vczTW3KZIwoxVCcxYmY0Axcq9l/nz1sLOfG1UsEnYmTbfCXH9xvZ9AOz/wPQ+4X0gbqzrfTmAw8Ju3bp1AIDY2FhMnjyZmqx2cNcm2UJ0BhbXsouRkV8KpZxB14a1JIWdI3l2NKm4emH5QExYbg4ESmp2AjOWM4GlzVjzNaV6Iz+f7hfrBInH4gAFAyAq2AfA/Yqln/y9FNj3vul97/lAtzdd78tBnF4uNmfOHE+Mw6twW1Kx4L3eyOJ8ej4AoGVUEAJ9pNclOrQ2tkKjojxobAaaYE4jEs4VoRlr3kbRWrMTmrFC7dFgJPz6bsuNfmQMg8ggLtfOxfSTg/8z+ekA4InZwMMTXevHSZwOUGRmZmLo0KGIioqCQqGAXC4XvSiCSsVu3iSb2zvUTyUX+WBE7RxYLkaLd1YPCrUGpFy/71ByulBwCTU7bg4oRZqd6V+hGSvMyZTJAFmZ9mewMGMZBogKrsCSseSVwO7Zpve9ZgCPTHK+DxdxWrMbMWIEbty4gVmzZtGy7DbwyL6xBpaPxMkYxqawc8SMpapd9WDAyoO4dLcQXRvWstmGN2MFx4Q+O8tadoDAZyd4GAp9vQwYPiLLErFmxzBAnQCTsLvrbDT2yOfA79NN73u+C/Sc4tz1FcRpYXfgwAH8/ffffJknijVuWy5mEY0VTjhOe7SERmO9h0t3CwEAh69a57VymKuemI8VS5ixCkkz1nyNMJhFQMyandHCZ8cwqBNoysRwKvXkny+B38qEW49E4LFpotMsS6A1sPBRec46dNqMjY6OdmuV0pUrVyI2NhYajQZxcXE4evSozbZnzpzBc889h9jYWDAMg+XLl1e4zwqT9rdViVfh7mLu+r/SGVj+SSyX2dbsdEZB2gGtZ1dzsAhQcPNO2oy11uyEVgBLYKXZEd5nB9QJMAs7h+Z3ynrg1zJztftEk5/OYhLO++UsWs3ZhYuZBeX35yJOC7vly5dj6tSpuHbtWoU/fOvWrUhMTMScOXNw7NgxtG3bFgkJCbh7965k++LiYjRs2BCLFy9GRESEW/qsEEe+ADY8A/yaKHpMCieWLb+ZI1iWZReG/1UO+OxsByiotPMWODeScK6wBCjVm757bg4I5wuntQl9dkIrgBDCa39WPjswCCsTdjoji1yJ/FARqd8C//eW6X3XN4He86wEXU6RDhuPXAdLgNO388q/aRdxWtgNGjQI+/btQ6NGjRAQEIBatWqJXs6wbNkyjBkzBiNHjkTLli2xevVq+Pr62tx7tnPnzvjoo48wePBgUVJzRfqsEJogAAyQsg7Y+Q4/I4QTy11+O72R8JNTxjB8Lp8ltOpJzcRSweIqn3BzQGzGctfY0ezknPbHlh0za3ZqhRzBZVWK7frtTmwBdow3ve/yGpCwUHLy7Th+mx+nrdxQd+C0z86W6egsOp0OKSkpmDbNbLvLZDLEx8cjOTn5gfap1Wqh1Zq/tPz8fMc+sO0ggBiB7eOAf9YAMgXQZ5FobaG7IrJGlvBmqYyBbc1OmFRsI2WBCjvvw1JEmCqfqK32jAVglUMHWAo7YlXgk29ZNnfqBKiRW6zH3YJSNIsIsB7QyW3A9jdMV3YaDTz1geTEI4Rg6z83+b/dWd3bEqeF3fDhw93ywffu3YPRaER4eLjoeHh4OM6fP/9A+1y0aBHmzp3r0mei3UsAawB+ngAcWQXI5JA/uQAyxvSErMiXZ/m05iKt9nx2jmy4I0w9uZtfip2n0jGwYz0EaqrXngIUYVKx+LvmNDsDL+wEmh3vjzOvdRebsWa/HvfAFLpQAFNE9mJmIe7mS2h2p38EfhoLEBboMBz4zxKbT9jTt/NxPsPsp7PlZ3YHLtWzu3LlCmbOnIkhQ4bwvrDffvsNZ86ccevgHhTTpk1DXl4e/7p582b5FwnpMAx4pqw0TfKnwJ73BMUA3KeWl5Ztn2dKPZGePA6lnghYvf8q3vu/s/gh5ZZ7Bkmxyf+duIOnV/yN69n2K9w4g1TqCWCuVsztXSyVegKYhZjlXOGEHcuKBSl3JR+kKLQQdmd3AD+8ChAWBS0GAc8sNyXu2eC7f8W/tYr4uMvDaWG3f/9+tG7dGkeOHMGPP/6IwkJTePzEiRNOra4IDQ2FXC5HZqZ4L4XMzEybwQdP9alWqxEYGCh6OU2nUaYnGAAcXI5E+TYAjvvs5A4sWOWczgwDx5KKbfrszJ+VU2SarPkl1kUFnCXtXhEvkCnWTNicijN38jH1h1Nu65ORKPEECH123DaK1mYsYA5SWM4VoWYnlD+coAwrSz8RaXbnfwW+HwUQI/4Pj+KRcwOQp7U9H0r1Ruw4fhuAuU6eO5UDS5wWdlOnTsWCBQuwe/duqFQq/vjjjz+Ow4cPO9yPSqVCx44dkZSUxB9jWRZJSUno1q2bs8PyWJ9O0WUM0GcxAOA15kdMlP/osM9OSthZfu2cIJHLGJFfUIh4BQWLb5Kv4fP9V0RthJ/ECVBbG7k4yuGr2ei1ZB+e/eRAhfqpCRTpKv5g4TBrduLvj8u148xCYV6mUNHivnebmh0hYs2O99lxicVlS8Yu7AK+Gw6wBpQ2H4i3Sscit5TFb6fSbY799zMZyC81oG6wD3o0DhWN1xM4LexOnTqFAQMGWB2vU6cO7t2751RfiYmJWLNmDTZs2IBz587hjTfeQFFREUaOHAkAGDZsmCjYoNPpcPz4cRw/fhw6nQ63b9/G8ePHcfnyZYf79Dhd3wCeXGAai/J7+B7+2KHLlA5odiUCM9aR1JNSPYtZO85g0W/nkSHYIEXoPiktK/ZZUWHHPaG5RFjKA8bi6+NWUZRnxnIxLFvCzmAkIs2O0yQ5M/ZugRa4tAf4bijA6oFWA3Cv9//AlomWH4/dtjlkzoR9vmM9pwreuorTAYrg4GCkp6ejQYMGouOpqamoW7eujaukGTRoELKysjB79mxkZGSgXbt22LVrFx9guHHjBmSCx9CdO3fQvn17/u8lS5ZgyZIl6NmzJ/bt2+dQnw+E7hPwU8p1DMheg3qpS4HagUCP/9q9xBEzlvPH2U09EUwWYdXiEoF5KdbsTMcrXnWWhngrA6kNdwCz9miQMmMlinNamo+cqcsSItIambJuOGEXc/8IsOV9wKgDWjwLDFwDbY7ZtD16LQc3sosRU9tX1P/NnGIcvJwNhgFe6FSP3wNZ70GfndPCbvDgwXj33Xexbds2MAwDlmVx8OBBTJ48GcOGDXN6AOPHj8f48eMlz3ECjCM2NtahjG17fT4oWr4wB0s/zcUkxTZgzxxTWkp322NSSGhqlvdqDlDAZoBCqNlp9eb3YlPEfK3ZjLV3N+VD01kcx537yvBJxTY0Oz71RGjGCjW7csxYA0tEffMBikANusnOYEHJhwCjB5o9DTy3FpArodWLCwT8lHobb8U3ER3bVhYQe7hRKOqF+PKaZ5UyY99//300b94c0dHRKCwsRMuWLfHoo4+ie/fumDlzpifGWC1pFhGAW63HY7nBVOEZf8wADq+22d6ZAIVcxkBpI8IlEnaC98LfgvCjOAFaUTOWyjr3YDCyWPDLWZuboNvC0mdXVBaN5TQ24XwRfv9ccQlbws5oIew4QRlx/198pVwCDaOHodGTwAvrAIXJhy+0KADgx9RbooetkSX4vsyEfaFTPdP4PJC9YInTmp1KpcKaNWswa9YsnD59GoWFhWjfvj2aNGlS/sU1jLfjm+LxE89DbmAxQbEd2PUuIJObAhkWKBwyY02TiGEYyGQMvxeAVBvL92ItkRG0MU3yipqxVLNzD8dv5uLLA2n4+5Jj/m+pQgCAhGYnsTYWMH/vVmasQNixlgGK68nw+W4IwGixz9gWsb1XI1ZhXtHEzamoIA3ySvS4nl2MYzfuo2N90wqrQ1fu4U5eKQI1CiS0MmVJcGa2vdp9FcVpYccRExODmJgYd47F64ip7YshXepj6eEXUDdQgYHF3wM7J5tM2k7igIkz0ViuqVIug4EVP0UtAxR8X0JTREKzq7Cwo7qdW+A0skKJ/UUksZFnxxXw5FNPLHYXYxjTnOC+dkshI9LshMdv/wNsfh7QF+EfWVu8Vvo2vi4GYgVtOGEX7KtCt0ah+OHYLfxw7DYv7LgVE/3b14VGaapywmt2Duzb4ioOCbvExETMnz8ffn5+SExMtNt22bJlbhmYtzDh8cbYlnITiTkD0LFNAOpfXAf88l+TwOswlG/niGYnNGMB0wQpsViHbStAwUr4XUx9mtpUtDoL3dfCPXAlmZzdc9jy+yviV1BYr9cGTOaokZi1Nst8UIWEZteWuQzVlo8AXSEQ+wj+VzIJ2uvFVutjtWVzSq2U4bkOdfHDsVv45cQdzH6mJUr1RvxxxmSiv9gp2vx5XHXvytbsUlNTodfr+fe2oIU8rakTqMHIhxtg1b4rGJsxAL91UUF29HPg5wkgjAxAMAAbmp2F/BGuoAAgmWunteGzEz65/zibiYmbU7Hsxba8ALXej9Q56HfvHjhNTOtgcrZNM1bHpZ5YLxcDTNFWI8wb6li6Q4Sb8hACPMRcxTeqxWC0xUD9h4GXtiLo+wsAJIRd2bxTK2To2rA2ooI0uJNXij/P30VWgRY6I4uWkYF4qG4Qf41CkOriKRwSdnv37pV8T3GM1x9thI2Hr+PC3ULseGwCBnQ2mooZ7ngT/WRvYAfbQ5QaYAsuJ44XdhIRXLEZa/7BWJqpP5+4g6ceiuC1P7rFbNWAE06OanblRWMNfNUTC81OBsAoiMZaanZys2YnyziBb1WLEMgUg42Og+yl7wCVnyDXTrwXhVnYySGTMejfvi4+23cFPx67xe9b8WJZYIKDj8Z6ULNzaW0sxTmCfJV4rWcjAMDHey5D9+QHQMcRYECwTLkKz8iSRT4VW3BaGO+zs9DsCCEWZqz9ogBag3kpUEXNWKrYmSCEoKC0nBpvduC+J0cLSJjz7CxWUOjEPjspMxawnVTMnQ/MuwC/715AMFOEY2xjkJe+B9T+AMyrKLLyLTW7MjO2bH4O7GDKv006fxdn0/OhksvQv704J9e8lUEla3bcBtmO8OOPP7o8GG9m5MOxWHfwGm7kFGNrym0MffpjaHU6qE9twnLlSizRBwB4RHSNveVigPX6WMucKPE6Wesfj9AMpgEK9zBu4zH8djoDOyc+gpZRzq+xdrb+odRWioBgbSxrXc8OEJR54jQ7i+9fIWPQlLmJhJRFkOlzcZxtiOG6qTihNpdzEq2iEMDld3Lzq3GdALStF4QTt0yFOZ9sFY5gX5XoGm58la7ZBQUF8a/AwEAkJSXh33//5c+npKQgKSmJ30ibYo2vSoEJjzcGAHySdAklBoL83svwg/ERKBgWkws+AM79Inkt94TU8oUApIWd5Q9FaMZKVZNQSGTSuwoNUJj47bRp79V1B9Ncut7V5VK2qp6Yt1K0NGPNKySE7Tgi9TewUbUQPvpc6MPbYphuGgrgK9Lgub0oLM1YTitVK8z7SQzsYDZbhYEJDs6n6EmfnUPCbt26dfwrPDwcL774ItLS0vDjjz/ixx9/xNWrVzF48GCEhoZ6bKDewJAuMagX4oO7BVqsP3QNehZ4R/8athu7QwEjsG2EaUG1BdwTkptEnGBRWTytLf08Qvkm9SMSmrYszbNzK67+dzor7MxVT6Tr2eltBCi4OcRKBCi+7huMN67/F2FMPrL8myH3ua3Ih5/p8wRftK1dxriHslppFi9920YhLECNlpGBeLixtZzgfNaeXBvrtM9u7dq1mDx5smiPWLlcjsTERM+UPvciVAoZ3o5vCgBYvf8KinUGsJBhkv4N/KV61LSQ+ruhwKXdpgskyrwDts1YeyaQlJkq1PwqvlyMSjshlj40R885W+yVL65p2Y+BhV6w17DlXOHz6IjYRzi7uwaPJo9GoCEH59hobH9oJYgmBIC19s6ZsbnFeslkdrXATRLip8L+dx7Dj+O6S2Ye8JpdVapnZzAYJKv+nj9/HqwH7W1voX/7upDLGOSV6JGRZ3oiGiHHEv9JQIu+pgXVW14GLpvLVFmmmDhqxgopkUhlKDUIzVy2QrXoqKizwFXNzkkHPZc8zCl2PkqzElKsNQq2UpSeQ9xP1mBkEc1k4sUzbwAF6cjUxOIV3XQUyoP4B6HlAy3YV8k/iO8V6vjjwmisEF+Vgk8itoSby1VKsxs5ciRGjx6NZcuW4cCBAzhw4ACWLl2KV1999cGVUarGyGUMfMu+8HxB1M4IOfD8WqD5M4BRC2x5CW10xwFYCzu5jTw74TaKlszeYV1FWrjCYuepDDw053fHM/ctodJOhL21xvaCOc7+2M2R3zIrQCHjBVCRzsBrflJ5dsJx+pfcwWbVQvhrM4HQpvi6yafIRpCo6omlQsYw5p3G7uab/XZSml158EnFlZ1nJ2TJkiWIiIjA0qVLkZ5uKswXGRmJd955B5MmTXL7AL0RjUqOAq0B+YLlDywBIFcCz68zmbIXd2Fm/jxcZKYgR95ZdL1wuZgQe7lZOUU6q2OWmpyBJfjnWg56Navj5B3RaKwl9qwxd5qxRTojDIJtNhkG8FXLoStmUaQ18MLDlivEyBIg9ybG3Xgbocw95PnWR9Dw/0PJvhwA+TCw5np2Ut9xWIAat3NLRH47KZ9deSj5pOIqpNnJZDJMmTIFt2/fRm5uLnJzc3H79m1MmTJF5Mej2MZXZa3Z8Q5mhQp48WugcW+oocVa1YdoT86KruciacLqs4DZjHVk6RkgnaUvde1fF7PwfTl7VNBorBhX9RNXtt4sKDXwn8cA8FOZdJginVGwNtbSjDX9Ky9MBzY8i1B9Oq6x4djT5UsgIIJPBWFZc6ViKbdsmET6iS0z1h4KPqm4CvnshLi8X0MNh/OrCPd9EFk9CjUw6FukKtvDj9FibuFcdGAu8qdlNnx23CTzUzumsJdK/LDkEjN62NqjmLztBC7ftb1bO41PiHE1SdsVn1V+qV6g2THwU5vml0mzs2HGyhiE4T4a/zYYuJ+GLEUEhuhmwugXCcA8x4S5m1LfMb/xToXNWK7EUxXS7ADg+++/x4svvoiuXbuiQ4cOohelfHwkNDsrH49SgwUBs3DQ2Ao+pAQbVB+gHWMqP2/LjOXMUj+VY0/UEp21Zmevrl6m1LZ5ZVAzVoyraYuu/NjzSvS8aczA/LATmrGWcyWU5GKzaiE0+deAoBgsDPsI6ajNCx1hIQDLbRSFSKWfCNfGOoqqKgYoVqxYgZEjRyI8PBypqano0qULateujatXr+Kpp57yxBi9Dk6zyyuxI+wA6Bg1XtVPwjl1GwQwJfhatRitmas2U094YeeoZidhxjpSRFQK4e/g1Q3/4peTd1zqx1uw55ezhysO+vwSg0j7MpuxBsl6dijMwtLSWWgsuwOtbyQw/GdkMGGidjKJqidSM8OcWCzls3PGjK0iScVCPvvsM3zxxRf45JNPoFKpMGXKFOzevRsTJ05EXl6eJ8bodZjNWIHPzkbbEmjwSfhCHGWbIZApxreq9xFacAGAdVIxF12tkBlrIewcXUYmvGrPuUyM32S7Ok5NwFXNzpVN1YVmLCA0YwU+O+57LcoGvu6HWPYmMkgIzvT+FqjVgBcySkvNjpjFtrRmZ72KwiUztiomFd+4cQPdu3cHAPj4+KCgwOTHGTp0KDZv3uze0XkpZjPWhs+OO8ZNM7UfRuqmIIVtgiCmGH1SXwMyTlmlnnCamq+DZqyUZmdZfcXhyUeddiJcLXPvSoAiX2jGCjU7rcFcz04hA4pzgK/7AXfPIIcJwRDdTBT51wdgXdGYj9YaBZWKJX12ZWZsfsXM2CqZVBwREYGcnBwApmrF3F6xaWlpFa6cUVOQ0uzs/ThUchmK4IMRundxnG0EjT4P+LofIrXi9Zdc4rBGKXcoOuqIGSsUdva6pNFYMQ9quRhgEaCA2GfHCQ+VPh/4pj+QeQrwC8PUgPeRRiL5cVr69oQrLOz67MrM2HuFWt4K0FUkGluVzNjHH38cP//8MwBTgvHbb7+N3r17Y9CgQZL7yVKs4TQvoc9OUrMrO8ZNwAL4YphuKrIDWwLF2Rh0bjwaMeZ9OTlhp5LLrPx5Ukjl5VmW1XN08tEAhZgHHaDg4PLsACC37HgAihH4/YtA+gnAtzYw/P+QrjRtqWC54Q7nO+OropSTelLbTwWGMQn37CKTdsdrds7k2ZV9rs7Iekxpcjqp+IsvvuCXhb355puoXbs2Dh06hL59++K1115z+wC9EY0j0VgBQnM1H37Y2/kLPH/mDfhlnMJm1UIM1s3EVRLFR1dVCpOwK68ApFSeneUwhGWy7VVGoVasJS4GKFyo5yYKUICBf5kZm1ushz+KsUG1GPL0y4BPLWDYz0CdFpDJDgIQbrgjrnunKRNUJTqjXZ+dQi5DbT817hVqcTdfizoBGpd8diq5DEE+SijlDIwscai+o7M4pdkZDAYsWLAAGRkZ/LHBgwdjxYoVmDBhAlQqlZ2rKRy+StNktLUhjiWWvjmDOggY9jOy/ZugDpOLzaoFiGXSRcLOmWKgQizHIdTs7Gl5VNaJcdWMdTlAIfDZ+ZaZsSVF+Vin+hAdZJdBNMHAsB1AxEMABFVPLDbJ5szJQB8l37e9aCwgyLUrsNDsnBB2wb4qnJjzJP6d2VtyD2V34FSvCoUCH374IQwGF9dPUgAAPqryN8Q2HTP9aynsZDIG8K2F3zt+jgtsPYQzudikWghN4Q0ApknmiBlbaih/4b9QwNkysQpK9fjn+v1y+6pJ2NPUuVOleiOKdeLfkks+uxKxz85fLYcPSvFWxnR0ll1EPvEFhm4HItvw11iujbVMPuaEXV6JOGFZCsu6dnzqiRM+uweB0yL0iSeewP79+z0xlhqDj8rae2BPE1DbKqntE4qXdTNwmY1CFJODUVcmoh6TZTJjHdqtTMKMtTC/hGasrUjZc6sO4a+LWeV+Xk2iPLcTyxL0Wf4Xei/7S7Qe1KVorHC5GMMgQKbHV8oleMhwGvnEB6OM08HUbS+6xpxHZ/rbYFEKKlAjodnZmFJ8+km+FoQQl8zYB4HTPrunnnoKU6dOxalTp9CxY0f4+fmJzvft29dtg/NWfCSSLR312QEAJ/tUchnuIQhDdDOwRbUAjfTp2KxcgJ+NaxwyBaRWUNgzY21pHRczC8v9rJrG/otZOH07T7SDlpCCUgOuZRcDMOXDBfm6nmdm0r7Koq5Ei65Hx6OW/CyKoMEI3bu4oLTewN7KjLWoaBzEaXbFekE0Vvrz+b0oCrWiwgFVTbNzWtiNGzcOgPT+sAzDwGinzBDFhJSwIwDS7hUhq0CLLg1q8ccACTPWosRTFkLwkm4GfvZfhGjDbbx87k38pZiHG/CxOw6ppGJLhD8+T6YFeCPPfHIA1xY/LXmuQCso70WEDxRXAhR6EABq6LBI/z/UykxFEVFjhH4KjpGmCJbQsLjUEl7YsZZmrKJsnAY+iGEr4h4m0OyEQTFnorEPAqdHw7KszRcVdI4hlfRLCMGo9f9g8BfJuHavSHSOMyk4pAoBZKIW5tVejBtsGIK1t7G0ZBbqwL4fTcpksorGCoUdLc7qNgoECeXCVSrOBCgCNCaBdLdAi4nfHMZnyv+hK5sKVuGDkbop+IdtDsA6URwQuEIsAhSWZiwh5hLvtjU7s89OGOGX2uqzMqlao6khSFVrzS3WI+1eEVgCnLmTLzoXGewj0gbNwk48+26xtTFENxMFmkjUY+9gs2oBwpBbobEK/XSezG6vaQiLpLKkfFeBFCFlO3QpYcCc0o/whDwVWqiQ+cwGHCUt+HaWywoBwSbYrMmdwQlcbnWPRinnfW65xSYttPwAhVmzU8llvF+wquCwGVtSUoKkpCQ888wzAIBp06ZBqzUvEZHL5Zg/fz40Go37R+llSGl2QkGSds/kA+P8MDIGaBLuj5NlW9HZ2je2VGfEbYTh905fouehEWiEdGwqy8PLhrTvyBKrAAU1Yz2CcG9ZoWZnucsXB8NYa91BPkooYMAK5SfoLU+Bligx3Wc63m3UE4C5rL+U/5Y3Y1mCO3klAAB/tUJkRQT6KJFVoEVuiY4fgxTCyidc0KuqBScAJzS7DRs24PPPP+f//vTTT3Ho0CGkpqYiNTUV3377LVatWuWRQXobPuWsXb2aJTZjGTBoUse8XydfvNNiEnMrKHQBMZhX6wOkk1poIruNjar3EQKxtmgL4Q9Kb2QdClBQnMdZM1YqlUglY7FS8xmekv8DLVFgrD4RxxTt+bWx5mtta3YsIbiTaxJ2UcFiRSVQY05OBmwLO85npzOwfK5dVfPXAU4Iu40bN2Ls2LGiY5s2bcLevXuxd+9efPTRR/juu+/cPkBvRCpAIeSKhc8OMGl2HLaKd/LLxRQyZKvr4iXdDGSSYDSX3cRG1SIEw3bxTQ7uZ/fPtRw0nfkbVu27wp+jZqz7EAo7lhCwLMGt+8U2AxSWqUQysPhvwVIk4DB0RI7X9W9jP9sWDEzzSyiYpAQl152REKTnmvLjIoPEAS0uIssJMFu5mxqlnBeMt+6bBGdVi8QCTgi7y5cvo3Xr1vzfGo0GMoHjs0uXLjh79qzUpRQLytfsCq2SjJsKhB035yyf2EITQiGXIY1E4iXdDGSRILSUXce3qkUIhP00Ee5zp3x/EoQAyVez+XOe3B/AG8gttt7nI69YL9FS7LMzsgTv/nASPT7Ya7NvYYEGGVh8pFyNR0r3wQA53tS/hb1sWR4dY9L8fQUPVCkhJTRjb/OanVjYcYnFV8sevqH+apvjqxNo0gpv3jel01RrMzY3N1fko8vKykJsbCz/N8uyovMU29jT7BjG9NQXbk3HMBCZsZzMsVXiybTDlGkyXyF1MUQ3A/dIIB6SXcM3qsUIhLXmyMGJWKnIqyd3fvIGhNoax6Avkm20Fa+L3lbOHh+c340Biw8UX+A5+QEYIcP/gqdhN9uJb8eJRGFNQ0kzlk89AdLLfHZRQWIzltPsrtw1PSC5qKsU3LmbOaa+LOdmVcDhEdWrVw+nT5+2ef7kyZOoV6+eWwbl7dgSdjG1fFG37OmaZmHK1hU8dW/mmJ6elj47vaB2mTDd4DKph5d105FD/NFWdhUbVB/AH8V2xygVjKCpJ85zPkPadVAo8tmV349cxoABi/cVX+EFxV8wEgaf1ZqG87V6idpxngahsJMKUMgEVU3ulJmxVppdWbCC0/zCHBF2nGbnRJXiB4XDwu4///kPZs+ejdLSUqtzJSUlmDt3Lp5+WjqBkiJGJmMk1fyH6gaiYZjJXDWZsuJrYmr5AgAebhwKwLYPRS23LgRwgcTgZd0M3Cf+aC+7jPWqD+GHEqtruc+U8s/RaKz7sBWgsIWSAeYr1mGIYi+MhMHb+nH4x/8xqxxMHb/pklnYSOW7cdODJeZobKRlgMJHHOiwK+zKzNhbOV5gxk6fPh05OTlo1qwZPvroI+zYsQM7duzAhx9+iGbNmuH+/fuYPn26J8fqVUiln7SKCkLDUNPyu6sCzY4TWzvfegT7Jj+GZhEmk9Yy9YRDuFGykHOkPl7RTUMe8UUn2UWsVX0EH1g+vEw/PKkfoLNmbI8P/sSnf14CAPyUegvvbDvh0tpPb6TARp6dNAST2K/wiiIJLGEwWf86fmYfhpwxm5ocXJ6br0qo2dk2Y02anUnY1Q2WDlBwhNnz2ZUJwvSyXcaqtbALDw/HoUOH0KJFC0ydOhUDBgzAgAEDMG3aNLRs2RIHDhxAeHi4J8fqVUiZsq2iAtEorEzYZRVa5bz5qxWIDTWvRZbyxQD2SzydIQ0wVDcN+cQHcbLzWKtcAg3MvlbudyeVZuKsGXvrfgmW/GHaAvLtrSewLeUWvjqQVs5V3g8h1j47IeK5QTBL8S2eM+wEALxrGIOf2EcAmExbS+2LW4TvL/LZ2TZjc4p0fKmviCDL1BMLYWdHs+POcbdSraOxANCgQQPs2rULWVlZOHz4MA4fPoysrCzs2rULDRs2dHkQK1euRGxsLDQaDeLi4nD06FG77bdt24bmzZtDo9GgdevW2Llzp+j8iBEjwDCM6NWnTx+Xx+cJpCKyraKC0CCUM2MFPjsb+U22luNolHK7hQBOkkYYrpuKAuKDbvKz+FK5BGqYAiJ8gEJCi3NHgOLQlXsV7qO68eXfV62i65bRWCE+KnnZg4xgmmITRit+AwBM1b+KbcbH+HYyhrESSGbNThiNtZ5AXIknLlUk1F9tJaACLTU7uz47saCs1nl2QmrVqoUuXbqgS5cuqFWrVoUGsHXrViQmJmLOnDk4duwY2rZti4SEBNy9e1ey/aFDhzBkyBCMHj0aqamp6N+/P/r3728VPOnTpw/S09P5V1XbDMhS2PVvF4WwADUalml2N3KKy/WR2fLZ+akV5a5LTCVNMEI3BUVEjR7yM/hCuYwXeIC0GSuVeuJsCe2zFkvhagILfj2HA5fFQt4yz06IUm7y6U5RbMVril8BACt8xmGL8XFRO5NmJ+2zK1ezKzt0qyygYJlQDEiYsXZ9duJzlmXJqgKVPqJly5ZhzJgxGDlyJFq2bInVq1fD19cXa9eulWz/v//9D3369ME777yDFi1aYP78+ejQoQM+/fRTUTu1Wo2IiAj+FRIS8iBux2G4asUAsGlMHJYPNuVJRQRq4KOUw8AS3Mq1DiAIsRXe91PJzVvn2SGFNMM4TEMxUaOn/CRWKZcDBpNJK2WySgUtnNX2sousc9FqArfvi79Le9FYpVyGt2TbME5h2utltn44/vD9j1WfJs1OesWn0Gdnz4zlNLuoIOsKOUKtUcYAtf3KN2M5vEazcxc6nQ4pKSmIj4/nj8lkMsTHxyM5WTo/KTk5WdQeABISEqza79u3D3Xq1EGzZs3wxhtvIDs7G7bQarXIz88XvTyNRiUdLZPJGDQo88txT2lbpXVsCTRflcLh0tbnVK0xSv8OSogKj8uPo/G+8YBBJ1lM1NKPV6g1iMwxewT7mn84RQ5eU92wtw+H5Tl70djRxu8whnwPAJivfwVfGxMgl6pcIqHZcfiryzFjy+YO9/CxjMQC4mhsbX+13Q3UA9QKft8KwAt8du7m3r17MBqNVoGN8PBw0T4XQjIyMspt36dPH3z99ddISkrCBx98gP379+Opp56yWYJq0aJFCAoK4l/R0dEVvLPy8RFMDMsnb4MwP8vmkkhVoVDJZWUb7jhWccJPrcBhtiVe1U9CKVEi5NYe4PuRUMBaIAnN6lK9EQ/N+R0d5u926HOEGf3emsJiz6L//K+r/HudkRWtgRWasePk2zFSZ3K5vK8fgq+MJo1OqvK0VDSWw9fBAAWHZSQWEPdtLxILmOai0G9XraOx1YnBgwejb9++aN26Nfr3749ffvkF//zzD/bt2yfZftq0acjLy+NfN2/e9PgY7ZkZjUIdE3aS/ZY90R3ZgwIwR/4Osq0xVp8IVqYCzv+C5cqVkEP8cBCatpyvxxUso8w1AWHASRiJBcya3Vj5/2GK0rS+fJ1mGL4wPsu3kdKq7Gl2oqRiO/XsOCzXxQJAgMCMteev4xCusKDCzoLQ0FDI5XJkZmaKjmdmZiIiIkLymoiICKfaA0DDhg0RGhqKy5cvS55Xq9UIDAwUvTyNRmnbzOASizmc2aaQq3jh6FZ0wuTTv9i2uPDYKkCmxDPyI/hY+ZlI4In9c67XKvPWvdQd/Z4KLZaVGQnBaPlOTFeaNLrNfq9gR8BgURup71Nux2fnJ4zGKqTMWPHfUgEKuYxBQJnQdEjYCYIU1XoFhSdQqVTo2LEjkpLMtbdYlkVSUhK6desmeU23bt1E7QFg9+7dNtsDwK1bt5CdnY3IyEj3DNwNiFMDxF9DQwfNWCk44aWUeJpLj0P8Y7lf9zFg0DfQEzn6ypPxkfJzyGDS6ISaXUX2ifVSWecwRRZ7f0Sc24BZym8BAP8zDMD2wFdE/i8Akj47uYyxKufEIVob64BmZ7lUjIPTHB3T7KgZa5fExESsWbMGGzZswLlz5/DGG2+gqKgII0eOBAAMGzYM06ZN49u/9dZb2LVrF5YuXYrz58/jvffew7///ovx48cDAAoLC/HOO+/g8OHDuHbtGpKSktCvXz80btwYCQkJlXKPUggTRy1XQjSoiBlbNvkd99lZPIEJgGZPYbx+IgxEhoHyA/hA8QUYsG4rBFD+ioGawyvy3WhxfAEAYKWhLz42PG9aAWPh4Jfy2clkjM1qwH7lRWMF1ynljE2fHFf6vTyfHSAWiFVR2Dm94Y67GTRoELKysjB79mxkZGSgXbt22LVrFx+EuHHjhqiUVPfu3bFp0ybMnDkT06dPR5MmTbB9+3Y89JBp81+5XI6TJ09iw4YNyM3NRVRUFJ588knMnz8fanX5X9iDwsdO0meARomwADVfR8wZJYoTXo5GY32U4inAiaHf2c6YqB+PFcpP8YLiLxggx/eGSQCA8ZuOIfmK7ei2FELxRmWdicHyP7FAuQ4AsNrwDD4yDALAQCmXWX3nUkJNbke9Fj7EJM1YwbXhgRqbQjMiSIPzGQX8umx7iH12Vc+MrXRhBwDjx4/nNTNLpIIKL7zwAl544QXJ9j4+Pvj999/dOTyPINLsJMyMhqF+vLBzBmc1O8s1ukJBtJPtCrmexXLlSgxR7EVwgQYgPfDLyXSnxyXU5mpigMKSF+T7sFj5JQDgdPTLWHzpP+Aea1LfndS3aS8VpHwz1vzelgkLALOfaYn4Ftl4rFmYzTYcXDEAgObZUQSIfHYSKr8wSGFroxMASOzdVPQ355h2NBrra2HGjlr/D/6+ZN7w+v/Y7pikfwMsYfBU6W/AznfgitdNpM3VcFk3UPYXPlCsAQCsNzyJ/Q3ehlCcqRRyqwiqpY/N1jEOR+vZAdZ17IQ0DPPHK13rO2Qp0GgsRRJ7ZiwAviBAeUx8ogniW5jzDrlJLjU5pT7H18KM1RlZfHv4uujYdrYH3tG/BhYM8M8azFZ8A2cllrB1l/eTEDv1V6dN4aqOvYcSR1/ZQVPQhyH4xhCP9wzDrfbvVcoZK61Nqmt7i2SE0VjJDXcEHdrT7JyhqpuxVNhVEppyzFhnghTCucwJO0mHtsQvxjLqB0hX3P2BfRRL1W8CAEYpdmG6YhOcEXhSfroR6+wXfKhM8or1+ObwdeS4cXnbM7JkUzoPQ7DZ0AuzDSMAMCi2iM6q5DKr1TFS3509M1YYZZdaJy3U7CLdJOxCfFX8uKlmR+HhzFi5jYia2Iy135dw0vvaMWOllpdJ7WFrawnYz/IngGc+BgCMVfyKKYqtcFzgWbfTVuHadhO3pGLW9tN4/ZsUt/TXR3bUlKjNEGwzPIrphtEgZT+/S3fF+4Io5TJrQSal2ZW1iQi0NkOFNQ0l69kJJlVdiRw7V5DJGISXjaW8fVYqAyrsKgkuQGErkBAd4uNwkEE4ce0lFUtpAlKanS1hZzASoNMozNKPAACMU/yMtxXfOzRGKc3O3gYulc3+iya/5dFrORXuq7fsX3yi/AQKhsUPxh541zCWF3QA8NfFLFF7pVxmZXpKanZlxzaNiZP8XD87q2mEh6RWT7jKlD7N8FJcDFrXdWyf4gcJFXaVhE85gQSFXIaWkaaVHNzO77YQaXZ2JriU70ZSs5MwYwHzCopvjE9irn4oAOAtxU+YKP/R7vgAaf3PkcoslUF2ofs2jnpcdgwrlf+DkjFih7E73tG/Dracn51pDxFLM9a6HafZNQzzxyNNQq3OmyPz9pOK3eWzA4B+7eri/QGtHU59epBUidSTmkj92n6o7adCyyjbS9M+e6UjrmcXoXEdf5ttALGzmatjJiVIpDQ7KUeyLc1Oqzdi05EbAIB1xqcgB4uZyo1IVH4PA2T4zNjf5hil6t7Z2hC6sjl5O8/u+exCLYJ9VXZ9ZgDQU3YCq5TLoWKM+MUYh0T9G+UKOgBQyRmUWgYoJNoJv3ep75GbC/Y2yfZTyW0uOfM2asZdVkH81QocnPq43RSRusE+ktUoLBFGAfmnuYSDWCpoIWXGWjrMOQq0Bkz/6RT/95fGp6GAEVOVWzBF+R2MkONzweJ1IVKaXVXdj4K1KLlUrDNg9f6r6NcuCvklegz47BB6NA7Ft6+KzUfh/24P2SlTQVTGgN+MnfFf/ZswwjE/llIiQCEV6RVOnUlPNsWec5kY3aMBf6x749q4k1eCFpHWD1ROUEcF+zgURfYGqLCrRKRMSFcQRWM581hqLaXEE76iY1ht7AsFjJis3IZpys0wQIavjNa7zEn57KqqsLOsL/d9yi2sSLqEFUmXMKB9XQCwqjwspJvsjKnUPaPHbmNHTNRPgMGJn5pSIbP6rmxVPeFoERmICwv6iDS8Oc+2wvT/tLC7XMxdkdjqQNUzrClOI/bZ2Q5QSJX6cYfA/dQ4AMsNAwEAs5QbMUK+y6pNdTJjLQuX3s03+/B2nrK/eiSOOYevlEugYfRIMrbHm/qJ0DupUyjlMquHlaRbwkIjkzJlbVkOPZuEoVP9ELzUJcapsVVnqGbnBchEPjtnU0/c87xbbngOcrCYoNiO95RfwwA5vjX25s9Xh0UTRpbglS+P4Gy6uFK18GFiL11GdfsI1qo+hC+jxT5jW4zTvwUdpOvN2UOlsE49kcsYKOWMqBhDeT5De8TU9sX3b3R3+frqCNXsvABxnp1tp7Rk6onbMt0ZLDW8gNUGk89ugXIdhsgFpbiqgbQ7fvM+kq9mI69EXFyzRC/twxRx8yhqbX8JfowWfxsfwmv6t6GF/Si61N7BgClAYflgUsgYqwdYTfG1uQuq2XkBwkX25jw76fpnlrh3wTaDxYbBkMOIMYqdWKT8CgbIATxdHWSdzRJWxbpy9sy4lQJ8+xxk+iIcMrbEGP2kcgWdXMZApZBJBoOUcimfnXXQwsE0TEoZVLPzAkp0ZtPKnL8n5bPzpGbHwWCh4WWsM5hqB36gWAMc3+z0louVga0h2opOAwDupALfDAC0+dDW7YrR+skoRfnJ0v5qhVXUl0MqGquQM3j1EfHezBUxY2siVNh5AaUG84+R215RKhrraFJxxWEw1zAM3xjiIWMIsP0NPI2/JVvuvXAXPT/ai3/dsFLBU5TYEnbpJ4Cv+wPaPCC6K+7334gSOLb0KkCjsClcTcLOslIxgzd7Ncbcvq34Y7Zq0FGkocLOC9BK+JQcXS6mVshQv3b5hRmdh8FswwhsMjwOgGARsxLPyKy3xxy57h9czy7GS2uOeGAMzmGrzp5lGXUAaM7cMAm60lyUhnfEsjoLcU/neDDCX62A0Ya0UykYq+9PITNVQunasDZ/zF7xToo1VNh5AaV66wihZMqBxG9LJmPw+38fdWiPAWchkGGGYRTQ/hXIGYLlypXoI5OudFIV0lBsaVolFj67psxNbFQtBEpygKgO6JU5ASsOZuK9n884/FmBGqXN8vQquVwyGguIN0anmp1zUGHnBZRKaHZSws6WQNEo5Qi2sSVfRSGQAc+uwE/so1AwLD5RfoInZf945LMqiq1EYaHPrhFzGxtVC1GbKQAi2wJDf0J6qSkYcfqO/WVmQvw1CrA25LtSzlhpbQoJYUc1O+egws4LkEqNkMsYhPqrRYEKeysWhH4/Kd56ogma1PFHlwa1nB4fYWSYxr6O7cbuUDJGfKpcgcdlxxy6tlRvxMJfz+LwVc8X+ly174rkcc5n15C5g82qhQhj8nGGrQ8M3Q74BPPtavs5rh37qxU2NTulZJ5dmS9W8H3SAIVzUGHnBdjKAzs09XGces+8o5pwX8+tY7ti/zuP8X9LmcJC3u7dFLsTe+LZtlFOj69YZwRLZJikfwP/Z+wKFWPEKuVyPCY7Xu616w5ew5q/0zD4i8NOf667KNYZUZ/JwCbVQtRhcnGOjcYrummAby3oBdpyqL/9dBMhARo7PjuJenadY0MAAGq5OaBE9/JwDppn5wVobQgqzuTZNCYOGXml8FHKse+CqXZanMDRDUibwlLYSpewR3ahDgQERsjxtn4c5GDxH/lRfK78GK/qJ+Fvto3Na2/eL3b681xh2R8XbJ4L0d3GV6oFiGDu4yJbF6/opuM+AnH6dp5or4cAjRMBinKisUJh9+lL7dEp1qRRC81Yg5u2tqwpUM3OC6jlZ1+j6N4oFAM71EOfhyLwyZD2+HNST6s2tgSmJbbKP9njXpGW/2EboMBE/Xj8buwENaPHGuVSdJedtnmt+/MATVzIKMCO47dBCMGlzAKs+POyZLt6TBa+JHMRxeTgMhuFl3QzkQ1TYcpnPjmAWdvNY9eW4woQEqhR4ouhHQEAs55pKTqnUshESwAfaWze2UtoxloWLKDYhwo7L+DjQe3QtWEtbHpVumItB8MweLZtlKjkO4ej0dD8Un35jSzIKtCKTDYDFBivn4g9xvbQMHp8pVyCrrJzkte6a+2uJQnL/8JbW47jz/N3bboBonAPm5QLUJe5hytsJIboZuAexBV4hUENh5aVleGvVuDJVhG4uOApjO7RAF1izb5QpZyxuYeDMFfSQIWdU1Bh5wU0ruOPLWO7oXtj62q17kZqM57yOJ9eYGWy6aHAOP1/sdfYFj6MDl8pP0T+hf1W1woreRxNy8GTH+93a7DixM1ckWnIEY4cbFItRIwsC9fYcLykm4EshNjtqzy/p5CAsoKZ3GcLzVaVXIbawpL1NuIQVLNzDirsKACAxQNbAzDtQxsWoBZt5Tjy4Vj+vSvC7my6dEqGDkq8rn8bfxlbw4/RQrF5EFZ+/S32nr/LtxFqdi9+noyLmYVuDVZojazV7lthuI/NqgWIlWXiBhuGIbqZyET5UejLFhvn2CPEjutBKZc5FOygmp1z0AAFBQAwuEsMnmodiSAfJd54rBEUMganb+cjPFAt2um9wMKMbRruj4uZ9n/kZ+7k2zynhQpj9JOwFh/hYfkZDLsyGUPPTUOvRW8BsL3/xqDPk/HFsE4IqmB+oM7AivxjYcjFZtVCNJRl4BYJxUv6mUhHbTs9uEZsbfFWmUIzXyE379IF2N6W0JVgUU2GanYUHk5wKOUyMAyD1vWCRIIOsN7P9re3HsXjzevY7ffW/RK757VQ4VX9JBxmWyCAKcHXqsXAbVMenpSJCQBH0nLw2V7poIIz6AwsL2hqIw8bVQvRWHYHd0gtDNHNwC1iCg5IpbSF+LouaOuFiCsECwWXUi6DRinHnsRH8ft/H7W5fplqds5BhR3FKd7u3VT0t1zGYLZFNNEVSqDBKN07OMo2QyBTbKokkn7CZuItAGS7YQNrnYFFqd6IEOTjW9X7aCq7jQwSgiG6mbhJwvl2whQTjj4PRbr8uZYrXOoKhB+nyTWuE4BmEQE2+zDaWoJBkYQKO4pTBGqUIn8eAMSG+uHM3AQEV0DTAYBiaDBSNwUpbBOgNBf4uh8Cc8/bbO+OTbb1Rhb6wmxsVC1CC9lN3CXBGKKbieskQtTOX0LYje4Ri17NwvBo0zCrc84iNFXLK8rJaYW9W0bYbUcRQ4UdxWmahltrG35qhagG21MPufZDLIIPRujeRWFoW6DkProeHIWmzE3Jto4mQtvDWHQfjXe9gpay68gigRiim4E0Yq2x5UhokQ1C/bFuZBdJzXbTq3EY1CladIwTaK90td73gbEVcpVgT2JPHHi3l12tj2INFXYUp5nf/yEM6RKN7W8+LDouTJ+w3Hg5UKNAkkQysxQF8MWTWf/FSbYB/I0mP1oj5rZVu4pqdgEoxujrk+CfcwbZJAAv6WbiCqkr2VYvkYfI3a9ULmDnBrXQr714ad3haU9g/zuPYW7fh6zaS5QftIlGKUe9EE+U5fJuqLCjOE2ovxqLBrZBu+hg0fEeZZn+gRoF3opvgoRWZp9X33ZRaCSRzGyLO1o1huqm4QxbH2FMvilCytwRtamIZuePYmxQLUY72VXcJ/54WTcDl0g9m+0tTcu2gnu3DCAsHtgaSrkM3RuF4qdx3fHlsE44Nqs3QvxUqF/bz8YCfrqo39NQYUdxG7OfaYlxjzXCj+O6I1CjxOdDO+GhuoGo5afChMebON1fHvzxsm46zrExqMPkYpNqIeozGfx5qaKlDqEtxDrVh+ggu4xc4odXdNNxntjfUtAygTdaEFDg9v0AgCZ1/PGMoFhC+5gQxLcML3dJX+M6jj8IKK5B8+wobiPIV4kpfZqLjn3/endoDazL+XC5CMDLuunYrFqAZrJb2KxagEG6WbhJwnHiVh7S80oQGeTERs+6ImDTi+gsu4h84otXdNNwhsQ6PS4fgTbno5Jj9jMtUaQ1YMyjDV0qdT+sW33kFuvQ0w3BDoo0VLOjeBSNUi4SdNOeMgnDUQ83cLiPHATiZd0MXGajEMXkYLNqIeoxpuotv53KKOdqAbpiYNMg4PpB5BMfDNVNxWnSsPzryhAuwk9oJQ7AjOrRABOeaOLynh5KuQyTnmzGVzehuB+GVIdtnx4w+fn5CAoKQl5eHgIDAyt7OF4FIQQ3cooRU8sX9wp1WPL7BWz9VzraakkY7mOLagEaydJxkw3DIN0s3EEoNr4ah4fLWRd8I/MeMlYPQBdyElD5Y2DBZBwjTe1eI2RE91h0aVALP6TcQu+W4Rjcxb7ZS/EsrvxGqbCTgAq7B8fGI9cx4yfbJZ4sCUcOtqjmo4EsE9fYcAzWzUQGauOTIe1tFhbNycvHheV90Y2kooioMVz3Lv4lzSXbWvJSXAyebBmObo1qi4oSUCoXV36j1IylVCoD29dD67pBGNIlBoemPl5u+0zUwku6mbjBhiFWlmmqHoz7mLA5VfoCgxZpnz2HbiQVJUSFUbopDgu6ZuEBWNj/ITzWrA4VdF4A1ewkoJpd5ZFy/T6m/Xiy3OICdZGFrer5qFdWa26wbha2TOqHRmH+MLIEe85lon1dX4TuHAvZxd9QSpQYqZ+CZLaVzT5HPhyLmznFGNYtFh3qh0iumqBUDaqtZrdy5UrExsZCo9EgLi4OR49Kb7fHsW3bNjRv3hwajQatW7fGzp07RecJIZg9ezYiIyPh4+OD+Ph4XLp0yZO3QHETHeuH4I+3e4pSMaS0vtsIw2DdTNwmtdFIlo6NqoVY/4dp17Lv/r2JN785glPLn4fs4m/QEiVe1U+2Keie61AP5+b1wZxnW+HL4Z3xaNMwKui8kEoXdlu3bkViYiLmzJmDY8eOoW3btkhISMDdu3cl2x86dAhDhgzB6NGjkZqaiv79+6N///44fdrs9/nwww+xYsUKrF69GkeOHIGfnx8SEhJQWlr6oG6LUkF+mdADx2f3xrXFT2PRwNaICvbBm70aidrcInUwRDcT6aQWmspu4+ULE7Dxz2PYe/YOlitX4gkcgZYoMFafiANsa6vP+PC5Nri08CksfbEtfFTUTPV2Kt2MjYuLQ+fOnfHpp58CAFiWRXR0NCZMmICpU6datR80aBCKiorwyy+/8Me6du2Kdu3aYfXq1SCEICoqCpMmTcLkyZMBAHl5eQgPD8f69esxePDgcsdEzdiqCSEEGfml0OpZHL2WgynfnwQANGDSsUU1H+FMLs6y9XENkfiP7DB0RI7X9W/jT7aDqJ/nO9bDR8+3KXfBPaXqUu3MWJ1Oh5SUFMTHx/PHZDIZ4uPjkZycLHlNcnKyqD0AJCQk8O3T0tKQkZEhahMUFIS4uDibfWq1WuTn54telKoHwzCIDPJBbKgfXuwUjX9nxuOJ5nWQRiJNZdNJEFrKruM/ssPQEzne1L/FC7rXHm2Ir4Z3wh9vP4oPnqOCriZSqY6Je/fuwWg0Ijw8XHQ8PDwc589Ll/bJyMiQbJ+RkcGf547ZamPJokWLMHfuXJfugVJ5hPqr8dWIzgBMdeleel+BVcY5CEEhJugnYDfbCafnJlD/GwUAXS4GAJg2bRoSExP5v/Pz8xEdHW3nCkpVQ6WQ4fvZo1CSPwAGXT5W1Y6h2htFRKUKu9DQUMjlcmRmZoqOZ2ZmIiJCuh5aRESE3fbcv5mZmYiMjBS1adeunWSfarUaarVa8hyleuETGAKUswsYpWZSqT47lUqFjh07IikpiT/GsiySkpLQrVs3yWu6desmag8Au3fv5ts3aNAAERERojb5+fk4cuSIzT4pFEoNgFQyW7ZsIWq1mqxfv56cPXuWjB07lgQHB5OMjAxCCCFDhw4lU6dO5dsfPHiQKBQKsmTJEnLu3DkyZ84colQqyalTp/g2ixcvJsHBwWTHjh3k5MmTpF+/fqRBgwakpKTEoTHl5eURACQvL8+9N0uhUNyCK7/RSvfZDRo0CFlZWZg9ezYyMjLQrl077Nq1iw8w3LhxAzJBGdfu3btj06ZNmDlzJqZPn44mTZpg+/bteOghc/XXKVOmoKioCGPHjkVubi569OiBXbt2QaPRWH0+hUKpGVR6nl1VhObZUShVm2qXZ0ehUCgPCirsKBRKjYAKOwqFUiOo9ABFVYRzY9JlYxRK1YT7bToTcqDCToKCggIAoKsoKJQqTkFBAYKCghxqS6OxErAsizt37oAQgpiYGNy8edProrLckjhvuzdvvS/Ae+/NlfsihKCgoABRUVGi1DR7UM1OAplMhnr16vGqcmBgoFdNLiHeem/eel+A996bs/flqEbHQQMUFAqlRkCFHYVCqRFQYWcHtVqNOXPmeGVFFG+9N2+9L8B77+1B3RcNUFAolBoB1ewoFEqNgAo7CoVSI6DCjkKh1AiosKNQKDWCGi/scnJy8PLLLyMwMBDBwcEYPXo0CgsL7V7zxRdf4LHHHkNgYCAYhkFubq5b+nUnrnx+aWkp3nzzTdSuXRv+/v547rnnrPb7YBjG6rVlyxZP3gpWrlyJ2NhYaDQaxMXF4ejRo3bbb9u2Dc2bN4dGo0Hr1q2xc+dO0XlCCGbPno3IyEj4+PggPj4ely5d8uQtSOLu+xoxYoTVd9OnTx9P3oIkztzXmTNn8NxzzyE2NhYMw2D58uUV7tMm7i6XXN3o06cPadu2LTl8+DD5+++/SePGjcmQIUPsXvPxxx+TRYsWkUWLFhEA5P79+27p15248vmvv/46iY6OJklJSeTff/8lXbt2Jd27dxe1AUDWrVtH0tPT+Zej5e5dYcuWLUSlUpG1a9eSM2fOkDFjxpDg4GCSmZkp2f7gwYNELpeTDz/8kJw9e5bMnDlTsmx/UFAQ2b59Ozlx4gTp27evU2X73YEn7mv48OGkT58+ou8mJyfnQd0SIcT5+zp69CiZPHky2bx5M4mIiCAff/xxhfu0RY0WdmfPniUAyD///MMf++233wjDMOT27dvlXr93715JYVfRfiuKK5+fm5tLlEol2bZtG3/s3LlzBABJTk7mjwEgP/30k8fGbkmXLl3Im2++yf9tNBpJVFQUWbRokWT7F198kTz99NOiY3FxceS1114jhBDCsiyJiIggH330EX8+NzeXqNVqsnnzZg/cgTTuvi9CTMKuX79+Hhmvozh7X0Lq168vKewq0qeQGm3GJicnIzg4GJ06deKPxcfHQyaT4ciRI1WuX09+fkpKCvR6PeLj4/ljzZs3R0xMDJKTk0Vt33zzTYSGhqJLly5Yu3atU2V2nEGn0yElJUU0JplMhvj4eKsxcSQnJ4vaA0BCQgLfPi0tDRkZGaI2QUFBiIuLs9mnu/HEfXHs27cPderUQbNmzfDGG28gOzvb/TdgA1fu60H2WaMLAWRkZKBOnTqiYwqFArVq1UJGRkaV69eTn5+RkQGVSoXg4GDR8fDwcNE18+bNw+OPPw5fX1/88ccfGDduHAoLCzFx4kS338e9e/dgNBr5zZeEYzp//rzN+5Bqz90D96+9Np7GE/cFAH369MHAgQPRoEEDXLlyBdOnT8dTTz2F5ORkyOVy99+IBa7c14Ps0yuF3dSpU/HBBx/YbXPu3LkHNBr3URXua9asWfz79u3bo6ioCB999JFHhB3FOQYPHsy/b926Ndq0aYNGjRph3759eOKJJypxZFUDrxR2kyZNwogRI+y2adiwISIiInD37l3RcYPBgJycHERERLj8+Z7q15P3FRERAZ1Oh9zcXJF2l5mZaXfMcXFxmD9/PrRardvXNoaGhkIul1tFhO2NKSIiwm577t/MzExERkaK2rRr186No7eNJ+5LioYNGyI0NBSXL19+IMLOlft6kH16pc8uLCwMzZs3t/tSqVTo1q0bcnNzkZKSwl/7559/gmVZxMXFufz5nurXk/fVsWNHKJVKJCUl8ccuXLiAGzduoFu3bjbHdPz4cYSEhHhkEbdKpULHjh1FY2JZFklJSTbH1K1bN1F7ANi9ezffvkGDBoiIiBC1yc/Px5EjR+zepzvxxH1JcevWLWRnZ4uEuidx5b4eaJ9OhTO8kD59+pD27duTI0eOkAMHDpAmTZqIUjRu3bpFmjVrRo4cOcIfS09PJ6mpqWTNmjUEAPnrr79Iamoqyc7Odrjfqnhfr7/+OomJiSF//vkn+ffff0m3bt1It27d+PM///wzWbNmDTl16hS5dOkS+eyzz4ivry+ZPXu2x+5jy5YtRK1Wk/Xr15OzZ8+SsWPHkuDgYJKRkUEIIWTo0KFk6tSpfPuDBw8ShUJBlixZQs6dO0fmzJkjmXoSHBxMduzYQU6ePEn69etXKakn7ryvgoICMnnyZJKcnEzS0tLInj17SIcOHUiTJk1IaWlplb0vrVZLUlNTSWpqKomMjCSTJ08mqamp5NKlSw736Sg1XthlZ2eTIUOGEH9/fxIYGEhGjhxJCgoK+PNpaWkEANm7dy9/bM6cOQSA1WvdunUO91sV76ukpISMGzeOhISEEF9fXzJgwACSnp7On//tt99Iu3btiL+/P/Hz8yNt27Ylq1evJkaj0aP38sknn5CYmBiiUqlIly5dyOHDh/lzPXv2JMOHDxe1/+6770jTpk2JSqUirVq1Ir/++qvoPMuyZNasWSQ8PJyo1WryxBNPkAsXLnj0HqRw530VFxeTJ598koSFhRGlUknq169PxowZ47RAcAfO3Bc3Dy1fPXv2dLhPR6ElnigUSo3AK312FAqFYgkVdhQKpUZAhR2FQqkRUGFHoVBqBFTYUSiUGgEVdhQKpUZAhR2FQqkRUGFHoVBqBFTYUaoNUmXHK6v0OKX64ZVVTyjeS58+fbBu3TrRMVtFCPR6PZRKpeiYTqeDSqVy+nNdvY5SdaCaHaVaoVarERERIXqFhIQAMG0GtGrVKvTt2xd+fn5YuHAh3nvvPbRr1w5ffvklGjRoAI1GAwC4ceMG+vXrB39/fwQGBuLFF18UlRGydR2l+kKFHcWreO+99zBgwACcOnUKo0aNAgBcvnwZP/zwA3788UccP34cLMuiX79+yMnJwf79+7F7925cvXoVgwYNEvVleR2lekPNWEq14pdffoG/v7/o2PTp0zF9+nQAwEsvvYSRI0eKzut0Onz99dcICwsDYKoDd+rUKaSlpSE6OhoA8PXXX6NVq1b4559/0LlzZ8nrKNUbKuwo1YpevXph1apVomO1atXi3ws3GeKoX7++SGCdO3cO0dHRvKADgJYtWyI4OBjnzp3jhZ3ldZTqDRV2lGqFn58fGjdubPe8I8cc/SyK90B9dpQaR4sWLXDz5k3cvHmTP3b27Fnk5uaiZcuWlTgyiiehmh2lWqHVaq22PFQoFAgNDXW4j/j4eLRu3Rovv/wyli9fDoPBgHHjxqFnz56SZjDFO6CaHaVasWvXLkRGRopePXr0cKoPhmGwY8cOhISE4NFHH0V8fDwaNmyIrVu3emjUlKoALctOoVBqBFSzo1AoNQIq7CgUSo2ACjsKhVIjoMKOQqHUCKiwo1AoNQIq7CgUSo2ACjsKhVIjoMKOQqHUCKiwo1AoNQIq7CgUSo2ACjsKhVIjoMKOQqHUCP4fqu6ucghEgDUAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.021136, + "end_time": "2024-03-01T00:58:40.101488", + "exception": false, + "start_time": "2024-03-01T00:58:40.080352", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4181.095414, + "end_time": "2024-03-01T00:58:42.845966", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/lct_gan/2/mlu-eval.ipynb", + "output_path": "eval/contraceptive/lct_gan/2/mlu-eval.ipynb", + "parameters": { + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "param_index": 2, + "path": "eval/contraceptive/lct_gan/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "lct_gan" + }, + "start_time": "2024-02-29T23:49:01.750552", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/lct_gan/model.pt b/contraceptive/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..423280b8b0febc028306780aca15369a22d9c725 --- /dev/null +++ b/contraceptive/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:424cf49a69ecb394fa723e49fa1d4bfcd5f8bf706c6fa5d31d27dae98b0bb1b4 +size 41106197 diff --git a/contraceptive/lct_gan/params.json b/contraceptive/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..7a6c31f08214473c648c7e20fda16029370834f2 --- /dev/null +++ b/contraceptive/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 8, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.775, "gradient_penalty_mode": "ALL", "synth_data": 2, "dataset_size": 2048, "batch_size": 2, "epochs": 100, "lr_mul": 0.075, "n_warmup_steps": 100, "Optim": "amsgradw", "loss_balancer_beta": 0.675, "loss_balancer_r": 0.95, "fixed_role_model": "lct_gan", "mse_mag": true, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 8, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation": "relu6", "head_activation_final": "leakyhardsigmoid", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/realtabformer/eval.csv b/contraceptive/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..bec783d34cb69e9189c19b061a50479254748197 --- /dev/null +++ b/contraceptive/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.011033064541774556,,0.0012809291818069923,2.5217413902282715,0.13429400324821472,2.274496555328369,0.22174391150474548,1.3903555782235344e-06,4.905334711074829,0.02842247299849987,0.06602984666824341,0.03579007089138031,0.05490555614233017,0.021831143647432327,7.427076101303101 diff --git a/contraceptive/realtabformer/history.csv b/contraceptive/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..bb92f48b879d6a92e71790fd6a0ed884be0be3b0 --- /dev/null +++ b/contraceptive/realtabformer/history.csv @@ -0,0 +1,17 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.009542662490628118,2.5762599885192934,0.00027902967270147837,1.262402762111742,0.0,0.0,0.0,0.0,0.009750861223506036,320,160,177.99687433242798,1.112480464577675,0.5562402322888375,0.07511623519994828,0.003741051011274976,5.616061944419175,1.4611252447579992e-05,0.20501699775923043,0.0,0.0,0.0,0.0,0.0037890895224336417,80,40,38.4509494304657,0.9612737357616424,0.4806368678808212,0.014547351159490064 +1,0.004063407685950438,2.9358797140668913,2.1007805549480562e-05,0.29748027815949174,0.0,0.0,0.0,0.0,0.004123606946563996,320,160,176.4381308555603,1.1027383178472518,0.5513691589236259,0.061613517155365116,0.0031020445742797166,3.6615094702647637,1.0131357688802912e-05,0.16994743956020103,0.0,0.0,0.0,0.0,0.00314174113664194,80,40,39.2351438999176,0.98087859749794,0.49043929874897,0.01395380900503369 +2,0.0031661894379958256,2.4117056041794633,9.836388895213808e-06,0.2652852796629304,0.0,0.0,0.0,0.0,0.0032166213203439044,320,160,177.3788866996765,1.1086180418729783,0.5543090209364891,0.06969117154612832,0.0030325393654493382,4.0810080617485855,1.0892275676884292e-05,0.2569533795583993,0.0,0.0,0.0,0.0,0.0030810200516498297,80,40,38.336137771606445,0.9584034442901611,0.47920172214508056,0.019897227934416152 +3,0.0023055057837069624,1.853058874884212,6.09967966267791e-06,0.1840183506719768,0.0,0.0,0.0,0.0,0.002340935556662771,320,160,174.05649328231812,1.0878530830144881,0.5439265415072441,0.06543248369998764,0.0030248929907429555,2.86869880397064,1.1402929371490123e-05,0.23006774671375751,0.0,0.0,0.0,0.0,0.003070625604050292,80,40,38.138808727264404,0.9534702181816102,0.4767351090908051,0.01822617864527274 +4,0.0020300468891775838,2.0495056005954213,4.800810847036593e-06,0.16669587139040232,0.0,0.0,0.0,0.0,0.002061763812719164,320,160,174.26776337623596,1.0891735211014748,0.5445867605507374,0.06477567687070404,0.0024391597434146204,2.740249975850452,7.1029316552044055e-06,0.2063442377373576,0.0,0.0,0.0,0.0,0.0024783857611396344,80,40,38.34581685066223,0.9586454212665558,0.4793227106332779,0.02674134580302052 +5,0.001967901364824343,2.1442700514508872,4.275416443720796e-06,0.1754571495053824,0.0,0.0,0.0,0.0,0.002000586862027376,320,160,173.89809226989746,1.0868630766868592,0.5434315383434296,0.06023744232916215,0.0024223781292675994,2.2352012658586773,8.114645254103658e-06,0.2092515385011211,0.0,0.0,0.0,0.0,0.0024612557746877426,80,40,37.69665455818176,0.9424163639545441,0.47120818197727204,0.028668187485891394 +6,0.0019011627552913523,1.3824890940862353,3.5074695057979445e-06,0.18085340851102955,0.0,0.0,0.0,0.0,0.0019337917172890684,320,160,173.48067259788513,1.0842542037367822,0.5421271018683911,0.07720058038594288,0.00231237440930272,3.3106582164372442,5.485264414915609e-06,0.1899933501612395,0.0,0.0,0.0,0.0,0.002348827009609522,80,40,38.06031823158264,0.951507955789566,0.475753977894783,0.024147568842454347 +7,0.0017620563637663622,2.08702039143828,4.277907744039237e-06,0.1741062898454402,0.0,0.0,0.0,0.0,0.0017928976651145235,320,160,173.46568369865417,1.0841605231165885,0.5420802615582943,0.07193254736935159,0.0022043123801552154,2.4382340427693405,4.345398951133283e-06,0.22177592268999433,0.0,0.0,0.0,0.0,0.002243927074131591,80,40,37.98831129074097,0.9497077822685241,0.4748538911342621,0.023593891369819174 +8,0.0015997363887322535,1.9942195613491454,2.12305838595916e-06,0.18833316041855142,0.0,0.0,0.0,0.0,0.0016316618306177588,320,160,172.9914493560791,1.0811965584754943,0.5405982792377472,0.07330743282886942,0.002557959837577073,3.356362864944478,9.768955913480593e-06,0.24531859559938313,0.0,0.0,0.0,0.0,0.0026019316301244544,80,40,37.9961371421814,0.9499034285545349,0.47495171427726746,0.030474860878530307 +9,0.0015520233103259785,1.2436869248155205,3.6605559843476585e-06,0.1845664474152727,0.0,0.0,0.0,0.0,0.0015831556965707704,320,160,173.16736221313477,1.0822960138320923,0.5411480069160461,0.06796933644800447,0.002226237224749639,5.8766094757972525,5.918809148441895e-06,0.24479435225948692,0.0,0.0,0.0,0.0,0.0022684186602418776,80,40,38.149120807647705,0.9537280201911926,0.4768640100955963,0.025726988267979322 +10,0.0014469290221867936,1.8593177073192813,2.8659225091814812e-06,0.17962789068405982,0.0,0.0,0.0,0.0,0.0014769007869517737,320,160,172.20347905158997,1.0762717440724372,0.5381358720362186,0.07091635333836166,0.0024056937301793367,2.3075559470113687,7.195210898430782e-06,0.26990594328381123,0.0,0.0,0.0,0.0,0.0024520762970496436,80,40,37.45506477355957,0.9363766193389893,0.46818830966949465,0.03237821003422141 +11,0.0013373809825861116,1.173929054754868,1.6501064853204766e-06,0.18660539614356822,0.0,0.0,0.0,0.0,0.0013675392799864738,320,160,172.15098762512207,1.0759436726570129,0.5379718363285064,0.07849135186097556,0.0026173408418799227,3.798281030391638,1.0865136886828441e-05,0.24395470218732954,0.0,0.0,0.0,0.0,0.0026618223850164214,80,40,37.91745567321777,0.9479363918304443,0.47396819591522216,0.026291378935275132 +12,0.001361571840274678,1.4893103715210714,2.357963301387279e-06,0.21065570718492382,0.0,0.0,0.0,0.0,0.0013948013615504352,320,160,172.53218483924866,1.078326155245304,0.539163077622652,0.06484427234754549,0.002312301666415806,2.343618094189341,8.55624849894765e-06,0.2849353780504316,0.0,0.0,0.0,0.0,0.00236011099177631,80,40,39.01480150222778,0.9753700375556946,0.4876850187778473,0.033037679741391913 +13,0.0013374171531950196,0.9150310022058135,1.4488872623798119e-06,0.22414301362587138,0.0,0.0,0.0,0.0,0.0013720730063141672,320,160,176.8967101573944,1.1056044384837151,0.5528022192418576,0.078952330530592,0.0027515849543306103,2.345467333926322,1.3061403678160666e-05,0.3832862245384604,0.0,0.0,0.0,0.0,0.0028137680678810284,80,40,38.85416507720947,0.9713541269302368,0.4856770634651184,0.032020010952692246 +14,0.001240193446210469,1.120601199339011,1.9383188680013006e-06,0.22299806029186584,0.0,0.0,0.0,0.0,0.0012742314142997202,320,160,175.8337869644165,1.098961168527603,0.5494805842638015,0.07295391643074253,0.0023316621582125663,2.4607038528601537,7.180798617589801e-06,0.32943321072962134,0.0,0.0,0.0,0.0,0.002385099265302415,80,40,38.73373031616211,0.9683432579040527,0.48417162895202637,0.032793717614549675 +15,0.0012287105860963265,0.805710650381948,1.4418534727195665e-06,0.24881427236250603,0.0,0.0,0.0,0.0,0.0012658631573629008,320,160,175.72978258132935,1.0983111411333084,0.5491555705666542,0.07813957213829781,0.0021971696589389465,3.9529951616568395,6.586546618266021e-06,0.3238095646491274,0.0,0.0,0.0,0.0,0.002248892179704853,80,40,38.49005126953125,0.9622512817382812,0.4811256408691406,0.029112126285326667 diff --git a/contraceptive/realtabformer/mlu-eval.ipynb b/contraceptive/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..021c69a16fd957ee57bba5deeb7c47089f1e9612 --- /dev/null +++ b/contraceptive/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2537 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:33.186791Z", + "iopub.status.busy": "2024-02-29T16:56:33.186353Z", + "iopub.status.idle": "2024-02-29T16:56:33.221988Z", + "shell.execute_reply": "2024-02-29T16:56:33.221148Z" + }, + "papermill": { + "duration": 0.052133, + "end_time": "2024-02-29T16:56:33.224319", + "exception": false, + "start_time": "2024-02-29T16:56:33.172186", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:33.251382Z", + "iopub.status.busy": "2024-02-29T16:56:33.250594Z", + "iopub.status.idle": "2024-02-29T16:56:33.258778Z", + "shell.execute_reply": "2024-02-29T16:56:33.257847Z" + }, + "papermill": { + "duration": 0.024165, + "end_time": "2024-02-29T16:56:33.260814", + "exception": false, + "start_time": "2024-02-29T16:56:33.236649", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:33.285728Z", + "iopub.status.busy": "2024-02-29T16:56:33.285038Z", + "iopub.status.idle": "2024-02-29T16:56:33.289434Z", + "shell.execute_reply": "2024-02-29T16:56:33.288587Z" + }, + "papermill": { + "duration": 0.018998, + "end_time": "2024-02-29T16:56:33.291329", + "exception": false, + "start_time": "2024-02-29T16:56:33.272331", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:33.316816Z", + "iopub.status.busy": "2024-02-29T16:56:33.316085Z", + "iopub.status.idle": "2024-02-29T16:56:33.320741Z", + "shell.execute_reply": "2024-02-29T16:56:33.319782Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.019549, + "end_time": "2024-02-29T16:56:33.322752", + "exception": false, + "start_time": "2024-02-29T16:56:33.303203", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:33.348944Z", + "iopub.status.busy": "2024-02-29T16:56:33.348196Z", + "iopub.status.idle": "2024-02-29T16:56:33.354487Z", + "shell.execute_reply": "2024-02-29T16:56:33.353523Z" + }, + "papermill": { + "duration": 0.021845, + "end_time": "2024-02-29T16:56:33.356617", + "exception": false, + "start_time": "2024-02-29T16:56:33.334772", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "44e5be82", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:33.384188Z", + "iopub.status.busy": "2024-02-29T16:56:33.383787Z", + "iopub.status.idle": "2024-02-29T16:56:33.389379Z", + "shell.execute_reply": "2024-02-29T16:56:33.388440Z" + }, + "papermill": { + "duration": 0.02176, + "end_time": "2024-02-29T16:56:33.391333", + "exception": false, + "start_time": "2024-02-29T16:56:33.369573", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"realtabformer\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 0\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/realtabformer/0\"\n", + "param_index = 2\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011255, + "end_time": "2024-02-29T16:56:33.414285", + "exception": false, + "start_time": "2024-02-29T16:56:33.403030", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:33.438374Z", + "iopub.status.busy": "2024-02-29T16:56:33.437616Z", + "iopub.status.idle": "2024-02-29T16:56:33.447217Z", + "shell.execute_reply": "2024-02-29T16:56:33.446391Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023814, + "end_time": "2024-02-29T16:56:33.449147", + "exception": false, + "start_time": "2024-02-29T16:56:33.425333", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/realtabformer/0\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:33.473758Z", + "iopub.status.busy": "2024-02-29T16:56:33.473457Z", + "iopub.status.idle": "2024-02-29T16:56:35.837602Z", + "shell.execute_reply": "2024-02-29T16:56:35.836545Z" + }, + "papermill": { + "duration": 2.378793, + "end_time": "2024-02-29T16:56:35.839842", + "exception": false, + "start_time": "2024-02-29T16:56:33.461049", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:35.867290Z", + "iopub.status.busy": "2024-02-29T16:56:35.866742Z", + "iopub.status.idle": "2024-02-29T16:56:35.879320Z", + "shell.execute_reply": "2024-02-29T16:56:35.878336Z" + }, + "papermill": { + "duration": 0.028436, + "end_time": "2024-02-29T16:56:35.881306", + "exception": false, + "start_time": "2024-02-29T16:56:35.852870", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:35.907250Z", + "iopub.status.busy": "2024-02-29T16:56:35.906871Z", + "iopub.status.idle": "2024-02-29T16:56:35.914907Z", + "shell.execute_reply": "2024-02-29T16:56:35.914038Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.02376, + "end_time": "2024-02-29T16:56:35.917079", + "exception": false, + "start_time": "2024-02-29T16:56:35.893319", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:35.943566Z", + "iopub.status.busy": "2024-02-29T16:56:35.943216Z", + "iopub.status.idle": "2024-02-29T16:56:36.056856Z", + "shell.execute_reply": "2024-02-29T16:56:36.055813Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.129424, + "end_time": "2024-02-29T16:56:36.059441", + "exception": false, + "start_time": "2024-02-29T16:56:35.930017", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:36.086511Z", + "iopub.status.busy": "2024-02-29T16:56:36.086166Z", + "iopub.status.idle": "2024-02-29T16:56:41.106277Z", + "shell.execute_reply": "2024-02-29T16:56:41.105225Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 5.036427, + "end_time": "2024-02-29T16:56:41.108914", + "exception": false, + "start_time": "2024-02-29T16:56:36.072487", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 16:56:38.506157: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 16:56:38.506213: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 16:56:38.507988: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:41.136994Z", + "iopub.status.busy": "2024-02-29T16:56:41.135758Z", + "iopub.status.idle": "2024-02-29T16:56:41.142315Z", + "shell.execute_reply": "2024-02-29T16:56:41.141519Z" + }, + "papermill": { + "duration": 0.02232, + "end_time": "2024-02-29T16:56:41.144336", + "exception": false, + "start_time": "2024-02-29T16:56:41.122016", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T16:56:41.173667Z", + "iopub.status.busy": "2024-02-29T16:56:41.172745Z", + "iopub.status.idle": "2024-02-29T16:56:50.303055Z", + "shell.execute_reply": "2024-02-29T16:56:50.301961Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 9.147723, + "end_time": "2024-02-29T16:56:50.305492", + "exception": false, + "start_time": "2024-02-29T16:56:41.157769", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.775,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.075,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'loss_balancer_beta': 0.675,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:00:13.846934Z", + "iopub.status.busy": "2024-02-29T17:00:13.845897Z", + "iopub.status.idle": "2024-02-29T17:03:35.330580Z", + "shell.execute_reply": "2024-02-29T17:03:35.329506Z" + }, + "papermill": { + "duration": 201.516387, + "end_time": "2024-02-29T17:03:35.346400", + "exception": false, + "start_time": "2024-02-29T17:00:13.830013", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../contraceptive/_cache/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache4/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache5/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/contraceptive [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T17:03:35.377202Z", + "iopub.status.busy": "2024-02-29T17:03:35.376810Z", + "iopub.status.idle": "2024-02-29T17:03:35.832124Z", + "shell.execute_reply": "2024-02-29T17:03:35.831090Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.473392, + "end_time": "2024-02-29T17:03:35.834373", + "exception": false, + "start_time": "2024-02-29T17:03:35.360981", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n", + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:03:35.866760Z", + "iopub.status.busy": "2024-02-29T17:03:35.866385Z", + "iopub.status.idle": "2024-02-29T17:03:35.871028Z", + "shell.execute_reply": "2024-02-29T17:03:35.870072Z" + }, + "papermill": { + "duration": 0.022921, + "end_time": "2024-02-29T17:03:35.873155", + "exception": false, + "start_time": "2024-02-29T17:03:35.850234", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:03:35.901460Z", + "iopub.status.busy": "2024-02-29T17:03:35.900845Z", + "iopub.status.idle": "2024-02-29T17:03:35.908541Z", + "shell.execute_reply": "2024-02-29T17:03:35.907582Z" + }, + "papermill": { + "duration": 0.024209, + "end_time": "2024-02-29T17:03:35.910571", + "exception": false, + "start_time": "2024-02-29T17:03:35.886362", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10911264" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:03:35.940441Z", + "iopub.status.busy": "2024-02-29T17:03:35.939572Z", + "iopub.status.idle": "2024-02-29T17:03:36.031039Z", + "shell.execute_reply": "2024-02-29T17:03:36.030056Z" + }, + "papermill": { + "duration": 0.108948, + "end_time": "2024-02-29T17:03:36.033388", + "exception": false, + "start_time": "2024-02-29T17:03:35.924440", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 16128] --\n", + "├─Adapter: 1-1 [2, 1179, 16128] --\n", + "│ └─Embedding: 2-1 [2, 1179, 24, 672] (48,384)\n", + "│ └─TensorInductionPoint: 2-2 [24, 1] 24\n", + "│ └─Sequential: 2-3 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 689,152\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-16 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 16128] (recursive)\n", + "│ └─Embedding: 2-4 [2, 294, 24, 672] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [24, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-18 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-32 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-7 [2, 8, 256] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 8, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 8, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 8, 256] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 8, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 8, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 8, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 8, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 8, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-21 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-22 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-23 [2, 8, 256] (recursive)\n", + "│ └─Encoder: 2-8 [2, 8, 256] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-24 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-25 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-15 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-43 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 8, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-45 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-46 [2, 8, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 256] --\n", + "│ │ │ └─Linear: 4-39 [2, 256] 524,544\n", + "│ │ │ └─ReLU6: 4-40 [2, 256] --\n", + "│ │ └─FeedForward: 3-20 [2, 256] --\n", + "│ │ │ └─Linear: 4-41 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-42 [2, 256] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 1] --\n", + "│ │ │ └─Linear: 4-55 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-56 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,959,648\n", + "Trainable params: 10,911,264\n", + "Non-trainable params: 48,384\n", + "Total mult-adds (M): 42.71\n", + "========================================================================================================================\n", + "Input size (MB): 0.28\n", + "Forward/backward pass size (MB): 721.87\n", + "Params size (MB): 43.84\n", + "Estimated Total Size (MB): 765.99\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:03:36.065553Z", + "iopub.status.busy": "2024-02-29T17:03:36.065151Z", + "iopub.status.idle": "2024-02-29T18:07:20.840432Z", + "shell.execute_reply": "2024-02-29T18:07:20.839341Z" + }, + "papermill": { + "duration": 3824.794063, + "end_time": "2024-02-29T18:07:20.842671", + "exception": false, + "start_time": "2024-02-29T17:03:36.048608", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.009542662490628118, 'avg_role_model_std_loss': 2.5762599885192934, 'avg_role_model_mean_pred_loss': 0.00027902967270147837, 'avg_role_model_g_mag_loss': 1.262402762111742, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009750861223506036, 'n_size': 320, 'n_batch': 160, 'duration': 177.99687433242798, 'duration_batch': 1.112480464577675, 'duration_size': 0.5562402322888375, 'avg_pred_std': 0.07511623519994828}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003741051011274976, 'avg_role_model_std_loss': 5.616061944419175, 'avg_role_model_mean_pred_loss': 1.4611252447579992e-05, 'avg_role_model_g_mag_loss': 0.20501699775923043, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0037890895224336417, 'n_size': 80, 'n_batch': 40, 'duration': 38.4509494304657, 'duration_batch': 0.9612737357616424, 'duration_size': 0.4806368678808212, 'avg_pred_std': 0.014547351159490064}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004063407685950438, 'avg_role_model_std_loss': 2.9358797140668913, 'avg_role_model_mean_pred_loss': 2.1007805549480562e-05, 'avg_role_model_g_mag_loss': 0.29748027815949174, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004123606946563996, 'n_size': 320, 'n_batch': 160, 'duration': 176.4381308555603, 'duration_batch': 1.1027383178472518, 'duration_size': 0.5513691589236259, 'avg_pred_std': 0.061613517155365116}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0031020445742797166, 'avg_role_model_std_loss': 3.6615094702647637, 'avg_role_model_mean_pred_loss': 1.0131357688802912e-05, 'avg_role_model_g_mag_loss': 0.16994743956020103, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00314174113664194, 'n_size': 80, 'n_batch': 40, 'duration': 39.2351438999176, 'duration_batch': 0.98087859749794, 'duration_size': 0.49043929874897, 'avg_pred_std': 0.01395380900503369}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0031661894379958256, 'avg_role_model_std_loss': 2.4117056041794633, 'avg_role_model_mean_pred_loss': 9.836388895213808e-06, 'avg_role_model_g_mag_loss': 0.2652852796629304, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032166213203439044, 'n_size': 320, 'n_batch': 160, 'duration': 177.3788866996765, 'duration_batch': 1.1086180418729783, 'duration_size': 0.5543090209364891, 'avg_pred_std': 0.06969117154612832}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0030325393654493382, 'avg_role_model_std_loss': 4.0810080617485855, 'avg_role_model_mean_pred_loss': 1.0892275676884292e-05, 'avg_role_model_g_mag_loss': 0.2569533795583993, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0030810200516498297, 'n_size': 80, 'n_batch': 40, 'duration': 38.336137771606445, 'duration_batch': 0.9584034442901611, 'duration_size': 0.47920172214508056, 'avg_pred_std': 0.019897227934416152}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0023055057837069624, 'avg_role_model_std_loss': 1.853058874884212, 'avg_role_model_mean_pred_loss': 6.09967966267791e-06, 'avg_role_model_g_mag_loss': 0.1840183506719768, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002340935556662771, 'n_size': 320, 'n_batch': 160, 'duration': 174.05649328231812, 'duration_batch': 1.0878530830144881, 'duration_size': 0.5439265415072441, 'avg_pred_std': 0.06543248369998764}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0030248929907429555, 'avg_role_model_std_loss': 2.86869880397064, 'avg_role_model_mean_pred_loss': 1.1402929371490123e-05, 'avg_role_model_g_mag_loss': 0.23006774671375751, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003070625604050292, 'n_size': 80, 'n_batch': 40, 'duration': 38.138808727264404, 'duration_batch': 0.9534702181816102, 'duration_size': 0.4767351090908051, 'avg_pred_std': 0.01822617864527274}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0020300468891775838, 'avg_role_model_std_loss': 2.0495056005954213, 'avg_role_model_mean_pred_loss': 4.800810847036593e-06, 'avg_role_model_g_mag_loss': 0.16669587139040232, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002061763812719164, 'n_size': 320, 'n_batch': 160, 'duration': 174.26776337623596, 'duration_batch': 1.0891735211014748, 'duration_size': 0.5445867605507374, 'avg_pred_std': 0.06477567687070404}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0024391597434146204, 'avg_role_model_std_loss': 2.740249975850452, 'avg_role_model_mean_pred_loss': 7.1029316552044055e-06, 'avg_role_model_g_mag_loss': 0.2063442377373576, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024783857611396344, 'n_size': 80, 'n_batch': 40, 'duration': 38.34581685066223, 'duration_batch': 0.9586454212665558, 'duration_size': 0.4793227106332779, 'avg_pred_std': 0.02674134580302052}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001967901364824343, 'avg_role_model_std_loss': 2.1442700514508872, 'avg_role_model_mean_pred_loss': 4.275416443720796e-06, 'avg_role_model_g_mag_loss': 0.1754571495053824, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002000586862027376, 'n_size': 320, 'n_batch': 160, 'duration': 173.89809226989746, 'duration_batch': 1.0868630766868592, 'duration_size': 0.5434315383434296, 'avg_pred_std': 0.06023744232916215}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0024223781292675994, 'avg_role_model_std_loss': 2.2352012658586773, 'avg_role_model_mean_pred_loss': 8.114645254103658e-06, 'avg_role_model_g_mag_loss': 0.2092515385011211, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024612557746877426, 'n_size': 80, 'n_batch': 40, 'duration': 37.69665455818176, 'duration_batch': 0.9424163639545441, 'duration_size': 0.47120818197727204, 'avg_pred_std': 0.028668187485891394}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0019011627552913523, 'avg_role_model_std_loss': 1.3824890940862353, 'avg_role_model_mean_pred_loss': 3.5074695057979445e-06, 'avg_role_model_g_mag_loss': 0.18085340851102955, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019337917172890684, 'n_size': 320, 'n_batch': 160, 'duration': 173.48067259788513, 'duration_batch': 1.0842542037367822, 'duration_size': 0.5421271018683911, 'avg_pred_std': 0.07720058038594288}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00231237440930272, 'avg_role_model_std_loss': 3.3106582164372442, 'avg_role_model_mean_pred_loss': 5.485264414915609e-06, 'avg_role_model_g_mag_loss': 0.1899933501612395, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002348827009609522, 'n_size': 80, 'n_batch': 40, 'duration': 38.06031823158264, 'duration_batch': 0.951507955789566, 'duration_size': 0.475753977894783, 'avg_pred_std': 0.024147568842454347}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017620563637663622, 'avg_role_model_std_loss': 2.08702039143828, 'avg_role_model_mean_pred_loss': 4.277907744039237e-06, 'avg_role_model_g_mag_loss': 0.1741062898454402, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017928976651145235, 'n_size': 320, 'n_batch': 160, 'duration': 173.46568369865417, 'duration_batch': 1.0841605231165885, 'duration_size': 0.5420802615582943, 'avg_pred_std': 0.07193254736935159}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022043123801552154, 'avg_role_model_std_loss': 2.4382340427693405, 'avg_role_model_mean_pred_loss': 4.345398951133283e-06, 'avg_role_model_g_mag_loss': 0.22177592268999433, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002243927074131591, 'n_size': 80, 'n_batch': 40, 'duration': 37.98831129074097, 'duration_batch': 0.9497077822685241, 'duration_size': 0.4748538911342621, 'avg_pred_std': 0.023593891369819174}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015997363887322535, 'avg_role_model_std_loss': 1.9942195613491454, 'avg_role_model_mean_pred_loss': 2.12305838595916e-06, 'avg_role_model_g_mag_loss': 0.18833316041855142, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016316618306177588, 'n_size': 320, 'n_batch': 160, 'duration': 172.9914493560791, 'duration_batch': 1.0811965584754943, 'duration_size': 0.5405982792377472, 'avg_pred_std': 0.07330743282886942}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002557959837577073, 'avg_role_model_std_loss': 3.356362864944478, 'avg_role_model_mean_pred_loss': 9.768955913480593e-06, 'avg_role_model_g_mag_loss': 0.24531859559938313, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026019316301244544, 'n_size': 80, 'n_batch': 40, 'duration': 37.9961371421814, 'duration_batch': 0.9499034285545349, 'duration_size': 0.47495171427726746, 'avg_pred_std': 0.030474860878530307}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015520233103259785, 'avg_role_model_std_loss': 1.2436869248155205, 'avg_role_model_mean_pred_loss': 3.6605559843476585e-06, 'avg_role_model_g_mag_loss': 0.1845664474152727, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015831556965707704, 'n_size': 320, 'n_batch': 160, 'duration': 173.16736221313477, 'duration_batch': 1.0822960138320923, 'duration_size': 0.5411480069160461, 'avg_pred_std': 0.06796933644800447}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002226237224749639, 'avg_role_model_std_loss': 5.8766094757972525, 'avg_role_model_mean_pred_loss': 5.918809148441895e-06, 'avg_role_model_g_mag_loss': 0.24479435225948692, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022684186602418776, 'n_size': 80, 'n_batch': 40, 'duration': 38.149120807647705, 'duration_batch': 0.9537280201911926, 'duration_size': 0.4768640100955963, 'avg_pred_std': 0.025726988267979322}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014469290221867936, 'avg_role_model_std_loss': 1.8593177073192813, 'avg_role_model_mean_pred_loss': 2.8659225091814812e-06, 'avg_role_model_g_mag_loss': 0.17962789068405982, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014769007869517737, 'n_size': 320, 'n_batch': 160, 'duration': 172.20347905158997, 'duration_batch': 1.0762717440724372, 'duration_size': 0.5381358720362186, 'avg_pred_std': 0.07091635333836166}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0024056937301793367, 'avg_role_model_std_loss': 2.3075559470113687, 'avg_role_model_mean_pred_loss': 7.195210898430782e-06, 'avg_role_model_g_mag_loss': 0.26990594328381123, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024520762970496436, 'n_size': 80, 'n_batch': 40, 'duration': 37.45506477355957, 'duration_batch': 0.9363766193389893, 'duration_size': 0.46818830966949465, 'avg_pred_std': 0.03237821003422141}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013373809825861116, 'avg_role_model_std_loss': 1.173929054754868, 'avg_role_model_mean_pred_loss': 1.6501064853204766e-06, 'avg_role_model_g_mag_loss': 0.18660539614356822, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013675392799864738, 'n_size': 320, 'n_batch': 160, 'duration': 172.15098762512207, 'duration_batch': 1.0759436726570129, 'duration_size': 0.5379718363285064, 'avg_pred_std': 0.07849135186097556}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0026173408418799227, 'avg_role_model_std_loss': 3.798281030391638, 'avg_role_model_mean_pred_loss': 1.0865136886828441e-05, 'avg_role_model_g_mag_loss': 0.24395470218732954, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026618223850164214, 'n_size': 80, 'n_batch': 40, 'duration': 37.91745567321777, 'duration_batch': 0.9479363918304443, 'duration_size': 0.47396819591522216, 'avg_pred_std': 0.026291378935275132}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001361571840274678, 'avg_role_model_std_loss': 1.4893103715210714, 'avg_role_model_mean_pred_loss': 2.357963301387279e-06, 'avg_role_model_g_mag_loss': 0.21065570718492382, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013948013615504352, 'n_size': 320, 'n_batch': 160, 'duration': 172.53218483924866, 'duration_batch': 1.078326155245304, 'duration_size': 0.539163077622652, 'avg_pred_std': 0.06484427234754549}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002312301666415806, 'avg_role_model_std_loss': 2.343618094189341, 'avg_role_model_mean_pred_loss': 8.55624849894765e-06, 'avg_role_model_g_mag_loss': 0.2849353780504316, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00236011099177631, 'n_size': 80, 'n_batch': 40, 'duration': 39.01480150222778, 'duration_batch': 0.9753700375556946, 'duration_size': 0.4876850187778473, 'avg_pred_std': 0.033037679741391913}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013374171531950196, 'avg_role_model_std_loss': 0.9150310022058135, 'avg_role_model_mean_pred_loss': 1.4488872623798119e-06, 'avg_role_model_g_mag_loss': 0.22414301362587138, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013720730063141672, 'n_size': 320, 'n_batch': 160, 'duration': 176.8967101573944, 'duration_batch': 1.1056044384837151, 'duration_size': 0.5528022192418576, 'avg_pred_std': 0.078952330530592}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0027515849543306103, 'avg_role_model_std_loss': 2.345467333926322, 'avg_role_model_mean_pred_loss': 1.3061403678160666e-05, 'avg_role_model_g_mag_loss': 0.3832862245384604, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0028137680678810284, 'n_size': 80, 'n_batch': 40, 'duration': 38.85416507720947, 'duration_batch': 0.9713541269302368, 'duration_size': 0.4856770634651184, 'avg_pred_std': 0.032020010952692246}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001240193446210469, 'avg_role_model_std_loss': 1.120601199339011, 'avg_role_model_mean_pred_loss': 1.9383188680013006e-06, 'avg_role_model_g_mag_loss': 0.22299806029186584, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012742314142997202, 'n_size': 320, 'n_batch': 160, 'duration': 175.8337869644165, 'duration_batch': 1.098961168527603, 'duration_size': 0.5494805842638015, 'avg_pred_std': 0.07295391643074253}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023316621582125663, 'avg_role_model_std_loss': 2.4607038528601537, 'avg_role_model_mean_pred_loss': 7.180798617589801e-06, 'avg_role_model_g_mag_loss': 0.32943321072962134, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002385099265302415, 'n_size': 80, 'n_batch': 40, 'duration': 38.73373031616211, 'duration_batch': 0.9683432579040527, 'duration_size': 0.48417162895202637, 'avg_pred_std': 0.032793717614549675}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012287105860963265, 'avg_role_model_std_loss': 0.805710650381948, 'avg_role_model_mean_pred_loss': 1.4418534727195665e-06, 'avg_role_model_g_mag_loss': 0.24881427236250603, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012658631573629008, 'n_size': 320, 'n_batch': 160, 'duration': 175.72978258132935, 'duration_batch': 1.0983111411333084, 'duration_size': 0.5491555705666542, 'avg_pred_std': 0.07813957213829781}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021971696589389465, 'avg_role_model_std_loss': 3.9529951616568395, 'avg_role_model_mean_pred_loss': 6.586546618266021e-06, 'avg_role_model_g_mag_loss': 0.3238095646491274, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002248892179704853, 'n_size': 80, 'n_batch': 40, 'duration': 38.49005126953125, 'duration_batch': 0.9622512817382812, 'duration_size': 0.4811256408691406, 'avg_pred_std': 0.029112126285326667}\n", + "Epoch 16\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011828704808237943, 'avg_role_model_std_loss': 1.521080377363036, 'avg_role_model_mean_pred_loss': 1.4230687305334667e-06, 'avg_role_model_g_mag_loss': 0.2343865339877084, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012180708587266054, 'n_size': 320, 'n_batch': 160, 'duration': 175.0330581665039, 'duration_batch': 1.0939566135406493, 'duration_size': 0.5469783067703247, 'avg_pred_std': 0.07183955616587809}\n", + "Time out: 3633.662043094635/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▅▅▅▂▂▁▁▃▁▂▃▂▄▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▁▁▃▃▆▆▅▅▇▅█▆███▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▇▂▅▃▃▁▇▅▆▄▅█▃█▆█\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▂▁▄▃▂▂▂▃▃▃▄▃▅█▆▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train █▂▂▁▁▁▁▁▁▁▁▁▁▁▁▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▅▅▅▂▂▂▁▃▁▂▃▂▄▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▅▅▆▃▄▂▁▅▂▃▅▄▇▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▇▄▅▂▂▁▃▁▃█▁▄▁▁▁▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▇█▆▄▅▅▃▅▅▂▄▂▃▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▅█▄▄▅▂▃▃▃▄▁▃▇▇▆▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train █▆▇▃▄▃▃▃▂▂▁▁▁▇▅▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▅█▄▄▅▂▃▃▃▄▁▃▇▇▆▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train █▆▇▃▄▃▃▃▂▂▁▁▁▇▅▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▅█▄▄▅▂▃▃▃▄▁▃▇▇▆▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train █▆▇▃▄▃▃▃▂▂▁▁▁▇▅▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00225\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00127\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.02911\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.07814\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.32381\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.24881\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.0022\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00123\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 1e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 3.953\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.80571\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.96225\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.09831\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.48113\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.54916\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 38.49005\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 175.72978\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 160\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/contraceptive/realtabformer/0/wandb/offline-run-20240229_170337-srnqrxxz\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_170337-srnqrxxz/logs\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 399, 'n_batch': 200, 'role_model_metrics': {'avg_loss': 0.0012809289882362033, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.01139539224805003, 'pred_duration': 4.909855365753174, 'grad_duration': 2.5098016262054443, 'total_duration': 7.419656991958618, 'pred_std': 0.054905567318201065, 'std_loss': 0.021831080317497253, 'mean_pred_loss': 1.3903560329708853e-06, 'pred_rmse': 0.03579006716609001, 'pred_mae': 0.028422458097338676, 'pred_mape': 0.06602981686592102, 'grad_rmse': 0.2217438966035843, 'grad_mae': 0.13429392874240875, 'grad_mape': 2.274496555328369}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0012809289882362033, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.01139539224805003, 'avg_pred_duration': 4.909855365753174, 'avg_grad_duration': 2.5098016262054443, 'avg_total_duration': 7.419656991958618, 'avg_pred_std': 0.054905567318201065, 'avg_std_loss': 0.021831080317497253, 'avg_mean_pred_loss': 1.3903560329708853e-06}, 'min_metrics': {'avg_loss': 0.0012809289882362033, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.01139539224805003, 'pred_duration': 4.909855365753174, 'grad_duration': 2.5098016262054443, 'total_duration': 7.419656991958618, 'pred_std': 0.054905567318201065, 'std_loss': 0.021831080317497253, 'mean_pred_loss': 1.3903560329708853e-06, 'pred_rmse': 0.03579006716609001, 'pred_mae': 0.028422458097338676, 'pred_mape': 0.06602981686592102, 'grad_rmse': 0.2217438966035843, 'grad_mae': 0.13429392874240875, 'grad_mape': 2.274496555328369}, 'model_metrics': {'realtabformer': {'avg_loss': 0.0012809289882362033, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.01139539224805003, 'pred_duration': 4.909855365753174, 'grad_duration': 2.5098016262054443, 'total_duration': 7.419656991958618, 'pred_std': 0.054905567318201065, 'std_loss': 0.021831080317497253, 'mean_pred_loss': 1.3903560329708853e-06, 'pred_rmse': 0.03579006716609001, 'pred_mae': 0.028422458097338676, 'pred_mape': 0.06602981686592102, 'grad_rmse': 0.2217438966035843, 'grad_mae': 0.13429392874240875, 'grad_mape': 2.274496555328369}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:07:20.881823Z", + "iopub.status.busy": "2024-02-29T18:07:20.881434Z", + "iopub.status.idle": "2024-02-29T18:07:20.886104Z", + "shell.execute_reply": "2024-02-29T18:07:20.885119Z" + }, + "papermill": { + "duration": 0.027044, + "end_time": "2024-02-29T18:07:20.888232", + "exception": false, + "start_time": "2024-02-29T18:07:20.861188", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:07:20.926767Z", + "iopub.status.busy": "2024-02-29T18:07:20.925659Z", + "iopub.status.idle": "2024-02-29T18:07:21.232660Z", + "shell.execute_reply": "2024-02-29T18:07:21.231735Z" + }, + "papermill": { + "duration": 0.328792, + "end_time": "2024-02-29T18:07:21.235145", + "exception": false, + "start_time": "2024-02-29T18:07:20.906353", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:07:21.274393Z", + "iopub.status.busy": "2024-02-29T18:07:21.274061Z", + "iopub.status.idle": "2024-02-29T18:07:21.571147Z", + "shell.execute_reply": "2024-02-29T18:07:21.570114Z" + }, + "papermill": { + "duration": 0.319638, + "end_time": "2024-02-29T18:07:21.573372", + "exception": false, + "start_time": "2024-02-29T18:07:21.253734", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAETCAYAAABjk1AzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8TklEQVR4nO3deVxTV94/8E8SsrHvCVEEVBDrgi2WiLVaK1O0asXaah3Hah8rtqOdttRxaRXsLMWqMz9Ha2uXp1pnili7aB/r0FrU2iqioiiKUhcUK4ZFJWFNIDm/Py5cDAZMwhKW7/v1yivh5tybk5h8PPfcc88VMMYYCCGkixA6ugKEEGILCi1CSJdCoUUI6VIotAghXQqFFiGkS6HQIoR0KRRahJAuhUKLENKlODm6Ah3FZDKhsLAQbm5uEAgEjq4OIaQJxhjKy8uhUqkgFDbfnuoxoVVYWIjAwEBHV4MQch/Xr19H7969m33ertDatGkT1q5dC41Gg4iICGzcuBFRUVHNlt+5cydWrlyJq1evIjQ0FO+++y6efPJJ/vmvv/4amzdvRlZWFm7fvo1Tp05h2LBhZtuoqanBG2+8gdTUVOj1esTGxuL999+HQqGwqs5ubm4AuA/E3d3d9jdNCGlXOp0OgYGB/G+1WcxGqampTCKRsE8//ZSdO3eOzZ8/n3l6erKioiKL5Q8fPsxEIhFbs2YNy83NZStWrGBisZjl5OTwZbZt28befvtt9vHHHzMA7NSpU/ds56WXXmKBgYEsPT2dnThxgo0YMYKNHDnS6nprtVoGgGm1WlvfMiGkA1j7G7U5tKKiotjChQv5v41GI1OpVCw5Odli+enTp7OJEyeaLVOr1WzBggX3lM3Pz7cYWmVlZUwsFrOdO3fyy86fP88AsIyMDKvqTaFFSOdm7W/UpqOHBoMBWVlZiImJ4ZcJhULExMQgIyPD4joZGRlm5QEgNja22fKWZGVloba21mw74eHh6NOnT7Pb0ev10Ol0ZjdCSNdnU2iVlpbCaDTe04+kUCig0WgsrqPRaGwq39w2JBIJPD09rd5OcnIyPDw8+Bt1whPSPXTbo4fLly9HQkIC/3dDJx+xj9FoRG1traOrQbowsVgMkUjU6u3YFFq+vr4QiUQoKioyW15UVASlUmlxHaVSaVP55rZhMBhQVlZm1tpqaTtSqRRSqdTq1yCWMcag0WhQVlbm6KqQbsDT0xNKpbJVYyVtCi2JRILIyEikp6cjLi4OADdoMz09HYsWLbK4TnR0NNLT0/Haa6/xy/bt24fo6GirXzcyMhJisRjp6emYNm0aACAvLw8FBQU2bYfYriGw/P394ezsTANziV0YY6iqqkJxcTEAICAgwO5t2bx7mJCQgDlz5mD48OGIiorC+vXrUVlZiRdeeAEA8Pzzz6NXr15ITk4GALz66qsYM2YM/vGPf2DixIlITU3FiRMn8NFHH/HbvH37NgoKClBYWAiACySAa2EplUp4eHhg3rx5SEhIgLe3N9zd3fHKK68gOjoaI0aMsPvNW6KtrsWFmzowACP6+rTptrsao9HIB5aPT8/+LEjryeVyAEBxcTH8/f3t31W059Dkxo0bWZ8+fZhEImFRUVHs6NGj/HNjxoxhc+bMMSv/xRdfsLCwMCaRSNigQYPYd999Z/b8li1bGIB7bklJSXyZ6upq9sc//pF5eXkxZ2dnNnXqVHbz5k2r62zt4dT954tY0NI9bML6Q1Zvu7uqrq5mubm5rKqqytFVId1EVVUVy83NZdXV1fc8Z+1vVMBYz7iwhU6ng4eHB7RabYsj4s8VajFxwy/wdZXgxIrfdWANO5+amhrk5+cjJCQEMpnM0dUh3UBL3ylrf6M0y0MTCnfugyytMKDWaHJwbQghTVFoNeHtLIGTkOtsLinXO7g2hJCmKLSaEAoF8HfjhkoU6WocXBvSVQkEAuzatcvR1WhTq1atumciA0eg0LLAv34XsZhaWqQL27p16z1nkbTG4sWLkZ6e3mbbs1e3HRHfGgp3rqVVTC0t0gMYDAZIJJL7lnN1dYWrq2sH1Khl1NKyoKEzvkhHLa27McZQZahzyM3Wg9xpaWkYNWoUPD094ePjg0mTJuHy5csAgJEjR2Lp0qVm5UtKSiAWi3Ho0CEAwM2bNzFx4kTI5XKEhIQgJSUFwcHBWL9+vV2fXU5ODh5//HHI5XL4+PggPj4eFRUV/PMHDx5EVFQUXFxc4OnpiUceeQTXrl0DAJw+fRpjx46Fm5sb3N3dERkZiRMnTrT4egcPHsQLL7wArVYLgUAAgUCAVatWAQCCg4Px17/+Fc8//zzc3d0RHx8PAFi6dCnCwsLg7OyMvn37YuXKlWanbjXdPZw7dy7i4uKwbt06BAQEwMfHBwsXLmz3072opWVBY2hRS+tu1bVGPJD4vUNeO/cvsXCWWP91raysREJCAoYOHYqKigokJiZi6tSpyM7OxqxZs7BmzRqsXr2aH+G/Y8cOqFQqPProowC4QdKlpaU4ePAgxGIxEhIS+NHctqqsrERsbCyio6Nx/PhxFBcX48UXX8SiRYuwdetW1NXVIS4uDvPnz8f27dthMBhw7Ngxvm6zZs3Cgw8+iA8++AAikQjZ2dkQi8UtvubIkSOxfv16JCYm8oO1724lrVu3DomJiUhKSuKXubm5YevWrVCpVMjJycH8+fPh5uaGJUuWNPs6Bw4cQEBAAA4cOIBLly5hxowZGDZsGObPn2/XZ2UNCi0L/Bo64qlPq8tqON2rwaeffgo/Pz/k5uZi+vTpeO211/DLL7/wIZWSkoKZM2dCIBDgwoUL+PHHH3H8+HEMHz4cAPDJJ58gNDTUrrqkpKSgpqYG27Ztg4uLCwDgvffew+TJk/Huu+9CLBZDq9Vi0qRJ6NevHwBg4MCB/PoFBQX485//jPDwcACwqh4SiQQeHh4QCAQWz899/PHH8cYbb5gtW7FiBf84ODgYixcvRmpqaouh5eXlhffeew8ikQjh4eGYOHEi0tPTKbQ6WkNLi/q0zMnFIuT+JdZhr22LixcvIjExEZmZmSgtLYXJxI25KygowODBg/HEE0/g888/x6OPPor8/HxkZGTgww8/BMCdRubk5ISHHnqI317//v3h5eVlV93Pnz+PiIgIPrAA4JFHHoHJZEJeXh5Gjx6NuXPnIjY2Fr/73e8QExOD6dOn8+fnJSQk4MUXX8S///1vxMTE4Nlnn+XDzV4NYXy3HTt2YMOGDbh8+TIqKipQV1d336nJBw0aZHY6TkBAAHJyclpVt/uhPi0LGjriaffQnEAggLPEySE3W0/Unjx5Mm7fvo2PP/4YmZmZyMzMBMB1OgPcLteXX36J2tpapKSkYMiQIRgyZEibf2bW2rJlCzIyMjBy5Ejs2LEDYWFhOHr0KACuL+ncuXOYOHEi9u/fjwceeADffPNNq17v7gAFuMk6Z82ahSeffBJ79uzBqVOn8NZbb/GfV3Oa7qYKBAL+P4j2QqFlgcKNa2ndqaqFvs7o4NoQW926dQt5eXlYsWIFxo0bh4EDB+LOnTtmZaZMmYKamhqkpaUhJSUFs2bN4p8bMGAA6urqcOrUKX7ZpUuX7tmGtQYOHIjTp0+jsrKSX3b48GEIhUIMGDCAX/bggw9i+fLlOHLkCAYPHoyUlBT+ubCwMLz++uv44Ycf8PTTT2PLli33fV2JRAKj0brv75EjRxAUFIS33noLw4cPR2hoKH8goLOh0LLA01kMiYj7aIrpCGKX4+XlBR8fH3z00Ue4dOkS9u/fbzYhJMC1NOLi4rBy5UqcP38eM2fO5J8LDw9HTEwM4uPjcezYMZw6dQrx8fGQy+V2Tc0za9YsyGQyzJkzB2fPnsWBAwfwyiuvYPbs2VAoFMjPz8fy5cuRkZGBa9eu4YcffsDFixcxcOBAVFdXY9GiRTh48CCuXbuGw4cP4/jx42Z9Xs0JDg5GRUUF0tPTUVpaiqqqqmbLhoaGoqCgAKmpqbh8+TI2bNjQ6tZce6HQskAgEMC/YaxWOe0idjVCoRCpqanIysrC4MGD8frrr2Pt2rX3lJs1axZOnz6NRx99FH369DF7btu2bVAoFBg9ejSmTp3KH0mz58RxZ2dnfP/997h9+zYefvhhPPPMMxg3bhzee+89/vkLFy5g2rRpCAsLQ3x8PBYuXIgFCxZAJBLh1q1beP755xEWFobp06djwoQJePvtt+/7uiNHjsRLL72EGTNmwM/PD2vWrGm27FNPPYXXX38dixYtwrBhw3DkyBGsXLnS5vfaEWiWh2ZM++AIsq7dwQezHsKEIfZPWNaV0SwPjX777TcEBgbixx9/xLhx4xxdnS6rLWZ5oKOHzaDO+J5t//79qKiowJAhQ3Dz5k0sWbIEwcHBGD16tKOr1uPR7mEz/Os742msVs9UW1uLN998E4MGDcLUqVPh5+fHDzT9/PPP+VNamt4GDRrUYXWcMGFCs/V45513OqweHY1aWs2gUfE9W2xsLGJjLY9Je+qpp6BWqy0+d7+R6m3pk08+QXV1tcXnvL29O6weHY1CqxkN09PQ0UPSlJubG9zc3BxdDfTq1cvRVXAI2j1sBrW0COmcKLSaQR3xhHROFFrNaJgIUFdTh2oDjYonpLOg0GqGu8wJMnH9qHgaYEpIp0Gh1QyBQECTARLSCVFotaDhxGlqaRFbdccLW3QWFFot8Oc746mlRbqetr6wBcBN4ywQCFBWVtam27UFhVYLaDJAQjofCq0W0LCHJhgDDJWOudGFLdrswhZ6vR6LFy9Gr1694OLiArVajYMHD/LrXrt2DZMnT4aXlxdcXFwwaNAg7N27F1evXsXYsWMBcNP/CAQCzJ07167PozVoRHwL+PMPafeQU1sFvKNyzGu/WQhIXO5frh5d2KL5C1ssWrQIubm5SE1NhUqlwjfffIPx48cjJycHoaGhWLhwIQwGAw4dOgQXFxfk5ubC1dUVgYGB+OqrrzBt2jTk5eXB3d0dcrncrs+kNSi0WsD3aVFHfJdDF7awfGGLgoICbNmyBQUFBVCpuP+AFi9ejLS0NGzZsgXvvPMOCgoKMG3aNH766b59+/LrN5zT6O/v3+b9Zdai0GpBY58WtbQAAGJnrsXjqNe2AV3YwrKcnBwYjUaEhYWZLdfr9fDx8QEA/OlPf8LLL7+MH374ATExMZg2bRqGDh1q1+u1B+rTakFDaFXo61Chr3NwbToBgYDbRXPEjS5s0SYXtqioqIBIJEJWVhays7P52/nz5/Gvf/0LAPDiiy/iypUrmD17NnJycjB8+HBs3Lixzd5ra1FotcBV6gQXCXd5JDqC2HXQhS04li5s8eCDD8JoNKK4uBj9+/c3u929GxkYGIiXXnoJX3/9Nd544w18/PHH/DYBWH3BjPZAoXUfNCq+66ELW3AsXdgiLCwMs2bNwvPPP4+vv/4a+fn5OHbsGJKTk/Hdd98BAF577TV8//33yM/Px8mTJ3HgwAH+9YKCgiAQCLBnzx6UlJSYHQHtMKyH0Gq1DADTarU2rTfjwyMsaOketuvUb+1Us86rurqa5ebmsurqakdXxWb79u1jAwcOZFKplA0dOpQdPHiQAWDffPMNX2bv3r0MABs9evQ96xcWFrIJEyYwqVTKgoKCWEpKCvP392ebN2+26vWbvtaZM2fY2LFjmUwmY97e3mz+/PmsvLycMcaYRqNhcXFxLCAggEkkEhYUFMQSExOZ0Whker2ePffccywwMJBJJBKmUqnYokWLrP43eemll5iPjw8DwJKSkhhjjBkMBpaYmMiCg4OZWCxmAQEBbOrUqezMmTOMMcYWLVrE+vXrx6RSKfPz82OzZ89mpaWl/Db/8pe/MKVSyQQCAZszZ45V9WjQ0nfK2t8oXdjiPl5NPYXd2YV468mBmD+67/1X6EbowhaN6MIWbYMubNEBaDLAnokubNF5UZ/WfTRMu0wXuOhZ6MIWnRe1tO7Dn1paPRJd2KLzsqultWnTJgQHB0Mmk0GtVuPYsWMtlt+5cyfCw8Mhk8kwZMgQ7N271+x5xhgSExMREBAAuVyOmJgYXLx40azMr7/+iilTpsDX1xfu7u4YNWoUDhw4YE/1baLgL3BBoUU4bm5u9wwXaLgFBQV1WD169erVbD0otO6yY8cOJCQkICkpCSdPnkRERARiY2ObPS/ryJEjmDlzJubNm4dTp04hLi4OcXFxOHv2LF9mzZo12LBhAzZv3ozMzEy4uLggNjYWNTWNQTFp0iTU1dVh//79yMrKQkREBCZNmgSNRmPH27be3UMeesgxi3v01PdN2l6bfJdsOl7JGIuKimILFy7k/zYajUylUrHk5GSL5adPn84mTpxotkytVrMFCxYwxhgzmUxMqVSytWvX8s+XlZUxqVTKtm/fzhhjrKSkhAFghw4d4svodDoGgO3bt8+qets75KFSX8uClu5hQUv3MG21waZ1u7q6ujqWm5trdribkNYoLS1lubm5rK6u7p7nrP2N2tSnZTAYkJWVheXLl/PLhEIhYmJikJGRYXGdjIyMewb2xcbG8rM65ufnQ6PRICYmhn/ew8MDarUaGRkZeO655+Dj44MBAwZg27ZteOihhyCVSvHhhx/C398fkZGRFl9Xr9dDr2/sPNfpdLa8VZ6zxAluMieU19ShWFcDd1nH9Vk4mkgkgqenJ9+KdnZ2tmtwJSGMMVRVVaG4uBienp4QiUR2b8um0CotLYXRaIRCoTBbrlAocOHCBYvraDQai+Ubdusa7lsqIxAI8OOPPyIuLg5ubm4QCoXw9/dHWlpasyexJicn4+2337bl7TVL4S5DeU0FinR69Pd3/EU6O1LDqR32TstCyN08PT3NTheyR5c4esgYw8KFC+Hv74+ff/4Zcrkcn3zyCSZPnozjx4/zZ8Pfbfny5WYtPJ1Oh8DAQLteX+EuxaXiih45V7xAIEBAQAD8/f1RW1vr6OqQLkwsFreqhdXAptDy9fWFSCRCUVGR2fKioqJm01OpVLZYvuG+qKjILHyKioowbNgwANxAvz179uDOnTv8SNn3338f+/btw2effYZly5bd87pSqRRSqdSWt9csBU0GCJFI1CZfOEJay6ajhxKJBJGRkUhPT+eXmUwmpKenIzo62uI60dHRZuUBYN++fXz5kJAQKJVKszI6nQ6ZmZl8maqqKq6yQvPqCoVCfp6k9kRjtQjpRGzt/U9NTWVSqZRt3bqV5ebmsvj4eObp6ck0Gg1jjLHZs2ezZcuW8eUPHz7MnJyc2Lp169j58+dZUlISE4vFLCcnhy+zevVq5unpyXbv3s3OnDnDpkyZwkJCQviTKktKSpiPjw97+umnWXZ2NsvLy2OLFy9mYrGYZWdnW1Vve48eMsbYp79cYUFL97A//ifL5nUJIdax9jdq1ywPGzduZH369GESiYRFRUWxo0eP8s+NGTPmnjO/v/jiCxYWFsYkEgkbNGgQ++6778yeN5lMbOXKlUyhUDCpVMrGjRvH8vLyzMocP36cPfHEE8zb25u5ubmxESNGsL1791pd59aE1p7ThSxo6R427f3DNq9LCLEOzfLQhL2zPADAiau38czmDAR6y/HzksfbqYaE9GzW/kbphGkr0Kh4QjoPCi0r+NWff2ioM0FbTYf9CXEkCi0ryMQieDpzI+F78rAHQjoDCi0rNY7VomEPhDgShZaV+Au3UmgR4lAUWlbiL9xKM5gS4lAUWlZSuNNkgIR0BhRaVqLrHxLSOVBoWanxAhfU0iLEkSi0rNRw0nQxtbQIcSgKLSs1dsTXwGSiUfGEOAqFlpX8XLndw1ojw50qg4NrQ0jPRaFlJYmTED4uEgDUGU+II1Fo2YCfDJA64wlxGAotG9BYLUIcj0LLBg3nH9IRREIch0LLBg0tLdo9JMRxKLRs4E+j4glxOAotGzSMiqc+LUIch0LLBnT+ISGOR6Flg4bQKqnQw0ij4glxCAotG/i6SiAQAEYTw61Kam0R4ggUWjZwEgnh69rQr0WhRYgjUGjZSEHTLhPiUBRaNmq8wAW1tAhxBAotG/nfNUUNIaTjUWjZqHH3kFpahDgChZaN+MkAqU+LEIeg0LIRzRVPiGNRaNmIRsUT4lgUWjZquNJ0aYUedUaTg2tDSM9DoWUjHxcpREIBGANKK2iueEI6GoWWjURCAX+RCxpgSkjHo9CyA42KJ8RxKLTs0HiBC+qMJ6SjUWjZoaGlVUItLUI6HIWWHej8Q0Ich0LLDgq6/iEhDkOhZQc/Ov+QEIexK7Q2bdqE4OBgyGQyqNVqHDt2rMXyO3fuRHh4OGQyGYYMGYK9e/eaPc8YQ2JiIgICAiCXyxETE4OLFy/es53vvvsOarUacrkcXl5eiIuLs6f6rdZ4/UNqaRHS0WwOrR07diAhIQFJSUk4efIkIiIiEBsbi+LiYovljxw5gpkzZ2LevHk4deoU4uLiEBcXh7Nnz/Jl1qxZgw0bNmDz5s3IzMyEi4sLYmNjUVPTGApfffUVZs+ejRdeeAGnT5/G4cOH8fvf/96Ot9x6DR3xtyoNMNTRqHhCOhSzUVRUFFu4cCH/t9FoZCqViiUnJ1ssP336dDZx4kSzZWq1mi1YsIAxxpjJZGJKpZKtXbuWf76srIxJpVK2fft2xhhjtbW1rFevXuyTTz6xtbo8rVbLADCtVmv3NhoYjSbW/83vWNDSPey3O1Wt3h4hxPrfqE0tLYPBgKysLMTExPDLhEIhYmJikJGRYXGdjIwMs/IAEBsby5fPz8+HRqMxK+Ph4QG1Ws2XOXnyJG7cuAGhUIgHH3wQAQEBmDBhgllrrSm9Xg+dTmd2aytCoQD+/BFE2kUkpCPZFFqlpaUwGo1QKBRmyxUKBTQajcV1NBpNi+Ub7lsqc+XKFQDAqlWrsGLFCuzZswdeXl547LHHcPv2bYuvm5ycDA8PD/4WGBhoy1u9r4YTp6lfi5CO1SWOHppMXL/RW2+9hWnTpiEyMhJbtmyBQCDAzp07La6zfPlyaLVa/nb9+vU2rRON1SLEMWwKLV9fX4hEIhQVFZktLyoqglKptLiOUqlssXzDfUtlAgICAAAPPPAA/7xUKkXfvn1RUFBg8XWlUinc3d3Nbm2Jzj8kxDFsCi2JRILIyEikp6fzy0wmE9LT0xEdHW1xnejoaLPyALBv3z6+fEhICJRKpVkZnU6HzMxMvkxkZCSkUiny8vL4MrW1tbh69SqCgoJseQttpvECF9TSIqRD2drDn5qayqRSKdu6dSvLzc1l8fHxzNPTk2k0GsYYY7Nnz2bLli3jyx8+fJg5OTmxdevWsfPnz7OkpCQmFotZTk4OX2b16tXM09OT7d69m505c4ZNmTKFhYSEsOrqar7Mq6++ynr16sW+//57duHCBTZv3jzm7+/Pbt++bVW92/LoIWOM7TxxnQUt3cP+8MnRNtkeIT2dtb9RJ1tDbsaMGSgpKUFiYiI0Gg2GDRuGtLQ0viO9oKAAQmFjA27kyJFISUnBihUr8OabbyI0NBS7du3C4MGD+TJLlixBZWUl4uPjUVZWhlGjRiEtLQ0ymYwvs3btWjg5OWH27Nmorq6GWq3G/v374eXlZX9it0LDXPF0pWlCOpaAMcYcXYmOoNPp4OHhAa1W2yb9W3macsSuPwRPZzGyE59ogxoS0rNZ+xvtEkcPO6OGjviyqlrU1BodXBtCeg4KLTt5yMWQOHEfXwl1xhPSYSi07CQQCGjYAyEOQKHVCjTAlJCOR6HVCo0XbqWWFiEdhUKrFRrOP6QZTAnpOBRardDQ0iqh3UNCOgyFVisoqKVFSIej0GoFf+qIJ6TDUWi1Ag15IKTjUWi1QsNMD+U1dagy1Dm4NoT0DBRareAmdYJcLAJAJ04T0lEotFqBRsUT0vEotFrJn7/aNLW0COkIFFqt1DBWiy5wQUjHoNBqJYUb7R4S0pEotFpJQXPFE9KhKLRayZ864gnpUBRardQwKp6GPBDSMSi0WomGPBDSsSi0WqlhyEOlwYgKPY2KJ6S9UWi1kqvUCa5S7kps1NoipP1RaLUB6ownpONQaLUBBXXGE9JhKLTaAHXGE9JxKLTaQOMFLqilRUh7o9BqA/78qHhqaRHS3ii02kDD7iH1aRHS/ii02gA/Vzy1tAhpdxRabeDujnjGmINrQ0j3RqHVBhpaWjW1JuhqaFQ8Ie2JQqsNyCUiuMu4UfE0GSAh7YtCq43QsAdCOgaFVhtpDC1qaRHSnii02gh//iEdQSSkXVFotZHGC1zQ7iEh7YlCq400XOCCRsUT0r4otNoIdcQT0jHsCq1NmzYhODgYMpkMarUax44da7H8zp07ER4eDplMhiFDhmDv3r1mzzPGkJiYiICAAMjlcsTExODixYsWt6XX6zFs2DAIBAJkZ2fbU/120dCn9dudKphMNMCUkPZic2jt2LEDCQkJSEpKwsmTJxEREYHY2FgUFxdbLH/kyBHMnDkT8+bNw6lTpxAXF4e4uDicPXuWL7NmzRps2LABmzdvRmZmJlxcXBAbG4uamnt3tZYsWQKVSmVrtdtdf383OEtEKNLpsePEdUdXh5Dui9koKiqKLVy4kP/baDQylUrFkpOTLZafPn06mzhxotkytVrNFixYwBhjzGQyMaVSydauXcs/X1ZWxqRSKdu+fbvZenv37mXh4eHs3LlzDAA7deqU1fXWarUMANNqtVavY6uPD11mQUv3sCFJaaxIV91ur0NId2Ttb9SmlpbBYEBWVhZiYmL4ZUKhEDExMcjIyLC4TkZGhll5AIiNjeXL5+fnQ6PRmJXx8PCAWq0222ZRURHmz5+Pf//733B2dr5vXfV6PXQ6ndmtvc0dGYzBvdyhq6nD3/acb/fXI6Qnsim0SktLYTQaoVAozJYrFApoNBqL62g0mhbLN9y3VIYxhrlz5+Kll17C8OHDraprcnIyPDw8+FtgYKBV67WGk0iI5KlDIRQA354uxME8y7vMhBD7dYmjhxs3bkR5eTmWL19u9TrLly+HVqvlb9evd0w/05DeHpg7MgQAsHL3WVQbjB3yuoT0FDaFlq+vL0QiEYqKisyWFxUVQalUWlxHqVS2WL7hvqUy+/fvR0ZGBqRSKZycnNC/f38AwPDhwzFnzhyLryuVSuHu7m526yhvPBEGlYcM129X41/plo+CEkLsY1NoSSQSREZGIj09nV9mMpmQnp6O6Ohoi+tER0eblQeAffv28eVDQkKgVCrNyuh0OmRmZvJlNmzYgNOnTyM7OxvZ2dn8kIkdO3bg73//uy1vwTp1BqAV82K5SJ3w9pTBAICPf76C8zfbvz+NkB7D1h7+1NRUJpVK2datW1lubi6Lj49nnp6eTKPRMMYYmz17Nlu2bBlf/vDhw8zJyYmtW7eOnT9/niUlJTGxWMxycnL4MqtXr2aenp5s9+7d7MyZM2zKlCksJCSEVVdbPgKXn5/ffkcPK0oZ+ziGscMbrd52cxZsO8GClu5hU977hdUZTa3eHiHdmbW/USdbQ27GjBkoKSlBYmIiNBoNhg0bhrS0NL4jvaCgAEJhYwNu5MiRSElJwYoVK/Dmm28iNDQUu3btwuDBg/kyS5YsQWVlJeLj41FWVoZRo0YhLS0NMpmstZlsuwv/B/x2jLt59AIGTbV7U6ueGoRfLpUi+3oZUjKvYXZ0cNvVk5AeSsBYz5gfWKfTwcPDA1qttuX+LcaA/y4Fjn0IiKTA87uAoJF2v+5nR64i6dtzcJM64cc3xvCn+xBCzFn7G+0SRw87lEAAjE8GwicBRj2wfSZQkmf35v4wIgjDAj1Rrq/Dqm/PtWFFCemZKLQsEYqAaZ8AvR8GasqA/zwDlBfddzVLREIB3pk6BCKhAP89q8GPufZthxDCodBqjlgOzEwFvPsC2gIg5VlAX2HXph5QuePFUdzYrcTdZ1Gpp4tfEGIvCq2WuPgCs74EnH2Am6eBL18AjPYFzqsxoejtJUehtgb/3PdrG1eUkJ6DQut+fPoBv/8CcJIDF38AvkuwawyXs8QJf4vjjphuOZyPnN+0bV1TQnoECi1r9B4OPPMpIBACJz8Dfl5n12YeG+CPyREqmBiw/JszqDOa2riihHR/FFrWCn8SmLCGe7z/b0D2drs2s3LSQLjLnHD2hg6fZVxrwwoS0jNQaNkiaj7wyKvc428XAZcP2LwJfzcZlk0YCAD4xw95uFFW3ZY1JKTbo9Cy1bhVwOBpgKkO2DEb0Jy97ypNPfdwIIYHeaHKYETS7rPoIeN7CWkTFFq2EgqBuA+AoFGAoRz4/FlAe8PGTQjwztNDIBYJ8OP5Ynx/zvJcZISQe1Fo2cNJCjz3H8AvHCgvBD5/Bqix7WhgmMINC0b3AwAkfXsOupra9qgpId0OhZa95F7ArJ2AqwIozuV2FesMNm1i0eP9EezjjCKdHuu+t/9UIUJ6Egqt1vDswwWXxBXI/4nrnLehf0omFuHvU4cAAP599Bqyrt1ur5oS0m1QaLVWQAQw/TNAIALO7ABSpgM/rgKOfQyc3wPcyAJ0NwGT5WmXH+nvi2cie4MxYMmXZ1BTS9MzE9ISmpqmrZz6D7B7YfPPC0SAmxJwCwDcAwD3XvWPVSiX+CF+52WUVBnxnLovXhwTCgjFgEgMCJ24m0jMLROKuJkoCOlmrP2NUmi1pWtHgOuZXMuqvBDQFXKPKzQAa8PR70Kn+gBzAno9CEz7FHD1a7vtE+IAFFpNdEhoNcdYB1QWNwmzQqD8ZuNjfTkqqmtgMtZCIjBBKjRBYLLyiKJiCDDnW8DZu33fByHtyNrfqM3TLRM7iJwAdxV3Q2SzxarKa/C7fx6CtroWS8YPwB/H9OP6wkx1gKkWMNbW/13/uKIY2DELKMoB/jMNeH43IOvgQCakg1FHfCfi7ybDykkPAADW/3gRl0srucATywCpG9eScvXjws8rCAh8mAsquTdQeJI7CGCodPC7IKR9UWh1MtMe6oXRYX4w1Jmw7KszMJnus/fuP5Cbx17qARRkcNND19Z0SF0JcQQKrU5GIBDgnamD4SwR4fjVO/hPphUzQQREAH/4qnG82BfP2zzQlZCugkKrE+rt5Yyl48MBAO/+9wJ+u1N1/5UCHwZ+v6N+ssLvga/m2T3LKummTCYg6zPgdGqrLkbsaBRandTsEUEYHuSFSoMRb35j5UwQwaOA5z4HRBLg/LfArpeaHdRKepgaHbDjD8D//Qn4ZgH3uPqOo2tlFwqtTkooFODdZ4ZC4iTEoV9L8PVJK2eS6D8OmL6NG8OVsxPY8xr3PyzpuUp+BT5+HMj7jvsPTSQBLuwBPhzNnbHRxVBodWL9/FzxWkwoAOAve3JRXG5lB/uACcDTH9dPD70NSFvWpXcHSCuc38MF1q2L3FkY/5MGzPsB8AoGygqA/40FMj/sUt8PCq1Obv6jfTFI5Q5tda1tF3sd/DQw5X3u8bEPgR+TutQXs9urug0c/4Sbjy1tOTfmri2ZTMD+v3Pj+AzlQNAjQPxPQK9IQPUg93jgZG7M33+XcAdvbJxeyVFoRHwXcK5Qi6feOwyjiWHzHx7C+MEB1q984lNgz+vc48feBB5b2j6V7Exqa4DrR4HeUYDE2dG1aVRnAC79CJxOAX79HjDedYRX7AKMeBl45E+AzKN1r1NdBnwdzx2QAQD1S8ATf+POX70bY1wr64cVXHh5BQPPbuVCzQHoNJ4munJoAcDa7y9g04HL8HOT4sfXx8DDWXz/lRpkvA98v5x7/Lu/NM5z3x1d+YkL6duXAffewBN/BQZNddxJ5owBN7O5C6Gc/RKoutX4nGII8MAU4Nc04MYJbpncCxj1OhAVz10w2FbF54HUWdz7d5IBk/8FRDzX8jq/ZQFfzuV2F0USIPYd4OEXO/wzo9BqoquHVk2tERM3/IzLJZV4NrI31j4bYdsGDq0D9v+VezxhLaCOb/tKOlLlLeCHt4DTDVdJEgCo/2oHjQImvAsoB3dcfXSF3FRFp1OBkguNy138gaHTuSBRcnOpgTHgwndA+l+A0vrJIN1UXKt42B+4syKskbsb+OZloLYS8AgEZvwHUA2zbt3qO8CuhVxnPQA8EAc8taH1rT4bUGg10dVDCwCyrt3GM5szwBiw7X+iMDrMxpkd0v/aeM3G3/2Vu56jk5T7H9lJxv0v6yRrXCYSd/5pcBjjgur7t4Dq2wAEXCth9J+BrC3AL/8PqKvhDkoMnweMfbP9Tiw3VHId36e3A1cOgg9NkRQInwgM+z3Qd2zzIWQyciF3MBnQXueWefcDHl/BhYiwmS5ok5G7rN0v/+T+DhkNPLOFu0K6LRgDjr4P7Evkznf1CuHmiguw8T9IO1FoNdEdQgsAVn17DluPXEUvTzl+eH00XKQ2nPPOGPfjPrrJ+nXMQkwKyD0A5VDuixwwjGu9SFxsfh9t4tZlbkhH/iHub/9B3O5Q4MONZcoKuD6b3N3c33Iv4PGVQORcbm6y1qrTcwF1bhc3Ns5Q0fhcn2ggYia3Cyj3tG2bJz4FDq1t3J0MiADGJQL9xpn/R1J9B/jqRa6vDACiFwExb1vfOrPk+nHgyxe44BRJgfHJwPD/aff/wCi0muguoVWpr0Ps+kP47U415o4MxqqnBtm2Aca4H0Pubq4FUqevvzdw90a9bdsTCAHfsMYQC4gAAoZyJ3i3lzoDcPhf3Psw6rlAfWwZ94Nt2tncIP8Q8N+l3Hz+ANef9OQaIGik7a9vqORCIvdbrkPdUN74nFcwF1RDpwPefW3f9t305Vx/5JGNja8R/CgwLokLZs1Z7ujgnavcmRBPbQSGPtu612xQdRvY9Ufg1/9yfw+exv2H0I7/rhRaTXSX0AKAny+WYPb/HoNAAOxcEI3hwW24u2MycUe1jPp7A61Oz01oePMM17lcmM39fQ8B4NOfCzDVMO5eOdS21kZzrmVwrauGfqJ+jwMT/wl4h9x/XWMd14I58LfGw/uDn+EOTnj0anndGi0XULm7gUvpQN1dF9l1VXLDBwY/zbWu2rpFUlkK/PxP4PjHjUcc+z0OFBwFaqu4axU8l9LYR9ZWGOMC88dVADMCLn5caAaN5G5+A5vfZbUDhVYT3Sm0AODPO09jZ9ZvCPSW4y9TBuOxMD8IHNH/VK4Bbp7mboXZXJjpmhm979mHa+EoBwOKwdy9Z7B1X/zqO9yPJ2sr97ezLzB+NTDkGdtDovIWd1AiaysABoidgUff4FpqYtld5UqBvL1ci+rKQW5YwN3vZeBT3K5fr+Ft+uNtVtl14KfVQHZK40y4fccCz3zavhNAFmQCX/4PoPvNfLnME+gzgguwPiO5/5ycJHa/DIVWE90ttLRVtRj/r0O4qeVGyQ/t7YFXHg9FzEB/x4TX3SpK6oMsu/52mutbskTiBigG3RVkQwD/BxrHVzEGnPsa+O8ybvZXAHhwNtc6au0P9eZpYO8SbkwXwO3ajUvkdo1ydwPXDptPk+07gGtRPfAU13J01Odckgcc2cDV95HXW9d/Za3aGuC3Y1xLt+AI1+9V22TuNic5d3An6BEgKBro/bBN/Z0UWk10t9ACgNIKPT46dAX/zriG6vqr+AwMcMcrj/fH+EFKCIWd6Mhf1W2g6BxQdJbriynKAYovNNOHJgB8+nEhVqMFrhzgFvuEcv0qwY+0Xb0YA3K+BPat5Ka/bko5lAupgU8BfgPa7nW7OmMtoDnDXRfhWgY3l1t1k0vgCZ241lefaK4j36dfi5uk0GqiO4ZWg1sVenzySz62HbmKSgMXXmEKVywc2x+Thqog6kzhdTdjLVB6sT7IchoDrbLJKS0iCbf7Nup17khme9BXAD//Azj1b+5Q/wNPca0qr+D2eb3uxmQCSn/lWqcFGVyQ3b07+WI61wprAYVWE905tBqUVRnw6S/52HLkKspruLm0+vq5YNHY/ngqQgUnURc51bSiuDHEqm5xAyz9whxdK2KrsoLGK1RNWNP8kd16FFpN9ITQaqCtrsVnR67if3/Jh7aa6zwO8nHGHx/rh6kP9obEqYuEF+lRrP2N2vXt3bRpE4KDgyGTyaBWq3Hs2LEWy+/cuRPh4eGQyWQYMmQI9u7da/Y8YwyJiYkICAiAXC5HTEwMLl68yD9/9epVzJs3DyEhIZDL5ejXrx+SkpJgMNCUwpZ4yMX407hQ/LJ0LJaMHwBvFwmu3arC0q9yMHbdQfzn6DXo62hyQNI12XzYYceOHUhISMDmzZuhVquxfv16xMbGIi8vD/7+/veUP3LkCGbOnInk5GRMmjQJKSkpiIuLw8mTJzF4MHcu2Jo1a7BhwwZ89tlnCAkJwcqVKxEbG4vc3FzIZDJcuHABJpMJH374Ifr374+zZ89i/vz5qKysxLp161r/KXRTbjIx/vhYf8wdGYzPjxbgw0NXcKOsGit2nUXy3vPo5SWH0kMOpbu0/l6GAA8ZFPX3ns5ixx+JJKQJm3cP1Wo1Hn74Ybz33nsAAJPJhMDAQLzyyitYtmzZPeVnzJiByspK7Nmzh182YsQIDBs2DJs3bwZjDCqVCm+88QYWL14MANBqtVAoFNi6dSuee87yGepr167FBx98gCtXrlhV7560e9icmlojth8rwOafLqNId/+R71InIZQeMijdZdy9hwwB7jKoPOVQecrRy1NOwUbaTLtcrNVgMCArKwvLly/nlwmFQsTExCAjI8PiOhkZGUhISDBbFhsbi127dgEA8vPzodFoEBMTwz/v4eEBtVqNjIyMZkNLq9XC27v5cTp6vR56feMPU6fT3ff9dXcysQgvPBKCP4wIQn5pJTTaGmh0Neb39Y9vVxqgrzPh2q0qXLvV/IU15GIRVJ4yPsRU/E2GXp5yKD1kkDq1wTl+hNSzKbRKS0thNBqhUCjMlisUCly4cMHiOhqNxmJ5jUbDP9+wrLkyTV26dAkbN25scdcwOTkZb7/9dstvqIcSi4QIU7ghTNH8eWQ1tUYU6/TQ6GpwU1uNIl0NbmprcLOM+/tGWQ1KK/SorjXickklLpc0f5FYPzcp+ng7IzLICyP6emN4sDfcZTbMB0bIXTpgKG3bunHjBsaPH49nn30W8+fPb7bc8uXLzVp4Op0OgYGBHVHFbkEmFqGPjzP6+DQ/82dNrREabQ0Ky6pxo6wahWXc40Jtw9/VqKk1oaRcj5JyPbKu3cFHh65AKAAeULljRIgP1H19EBXsbdukhqRHsym0fH19IRKJUFRUZLa8qKgISqXS4jpKpbLF8g33RUVFCAgIMCszbNgws/UKCwsxduxYjBw5Eh999FGLdZVKpZBK22kgIgHABVuwrwuCfS2fqsEYw52qWhSWVePXonIcy7+No1du4eqtKpy9ocPZGzp88ks+BAJgoNId6r7eUIf4QB3iDS8X+89hI92bTaElkUgQGRmJ9PR0xMXFAeA64tPT07Fo0SKL60RHRyM9PR2vvfYav2zfvn2Ijo4GAISEhECpVCI9PZ0PKZ1Oh8zMTLz88sv8Ojdu3MDYsWMRGRmJLVu2QNgRJ6iSVhEIBPB2kcDbRYLBvTzw9EO9AQAabQ0y82/h6JXbyMy/hSsllci9qUPuTR22HL4KAAhXukEd4o0RfX0wsr8vPOTUEiMcm48e7tixA3PmzMGHH36IqKgorF+/Hl988QUuXLgAhUKB559/Hr169UJycjIAbsjDmDFjsHr1akycOBGpqal45513zIY8vPvuu1i9erXZkIczZ87wQx5u3LiBxx57DEFBQfjss88gEjV27DbXwmuKjh52XsW6GmTWt8Iy82/jUnGF2fMioQAPBnrisQF+GBPmj0Eq9851XiVpE+1y9BDghjCUlJQgMTERGo0Gw4YNQ1paGt+RXlBQYNYKGjlyJFJSUrBixQq8+eabCA0Nxa5du/jAAoAlS5agsrIS8fHxKCsrw6hRo5CWlgaZjJsmZN++fbh06RIuXbqE3r17m9Wnhwzo79b83WWYHKHC5AgVAKCkXI9j+Vwr7PClUlwuqcSJa3dw4todrPvhV/i6SjA61A9jBvjh0VA/eNOuZI9Cp/GQTu/67SoculiCg3klOHKplD8pHOBmh4no7YkxYX54bIAfhvb27LwniJMW0bmHTVBodQ+GOhNOXLuNn34twU95JbigKTd73stZjEdD/fBoqC+CfFzg4yqBj4sEHnIaBNvZUWg1QaHVPWm0Nfjp12L89GsJfr5Yys9u0ZSTkDso4OMqha8rd3DAx0UKH1cJfF25x96uEvi5SqHylFNrzQEotJqg0Or+ao0mZF8vw8G8YhzPv4OSCj1KK/TNBllzZGIhwpXuGBjgjgdU7nggwB3hSjfbrnxEbEah1QSFVs+lrzPiTmUtSiv0uFVpwK0KPW5VGBof33VfXK6Hoc50zzYEAiDYxwUP3BVkAwPcoXCX0m5nG6HQaoJCi1jDaGK4dqt+3FihDufrx481d4K5t4vELMgGqdzR18+Vdi/tQKHVBIUWaY3SCj0XYHcF2eWSShhN9/58ZGIhBtYH2CCVBwap3BGmcINMTCeOt4RCqwkKLdLWamqN+LWoHLmFXIidqw+0KsO9Eyw6CQXo7+/Kh9ggFdc6c7Nw4rjRxFBpqEOV3ohKQx0q9XWo1BtRZahDpcFY/3cdhAIB5BIR5GJR8/f1j6VOwk6/G0uh1QSFFukIRhPD1VuVOHtDi9xCLsjOFWpxp6rWYvkgH2fIxSKzkKqpvbdPrbUEAvAh5ucmRZjCDQOU3EwfAxRu6O0ld/hZBhRaTVBoEUdhjOGmtgbnCnU4e0OLc4U65BZqUVh/zcrmiIQCuEhEcJE6cTeJCM4SJ7hIuXsGoNpgRE0t1wqrrjWhptaIagP3d02tCQajdQEoF4sQpnA1DzOlG/zdOu5AA4VWExRapLO5XWnAhZs6GBmrDyUnOPMhJYJE1PpdujqjCTV1JlQb6sOstg437lQjr6gcv2rKkVdUgcvFFc2Gm4dcjAEKN4QpXaHylMNJKIBIKKy/b7w53fNYCJEQfNmhvT0s7grfjUKrCQotQiyrM5pw9VYVfi0qR56mnLsvKsfV0kpYOM5gl/9bNApDenu0WKbdTpgmhHQvTiIh+vu7or+/K54c0jinXU2tEZdLKurDrAKlFXqYTAx1JgZj/Y17bEKdicHEGOqMdy9vvMklbXfklEKLEGKRTCyqP9rZcgupo9FMeoSQLoVCixDSpVBoEUK6FAotQkiXQqFFCOlSKLQIIV0KhRYhpEvpMeO0Ggb+63Q6B9eEEGJJw2/zfifp9JjQKi/nLoAQGBjo4JoQQlpSXl4OD4/mB7T2mHMPTSYTCgsL4ebmdt+TUHU6HQIDA3H9+nU6T/Eu9Lk0jz4by2z5XBhjKC8vh0qlavEK8j2mpSUUCu+50Ov9uLu70xfQAvpcmkefjWXWfi4ttbAaUEc8IaRLodAihHQpFFoWSKVSJCUlQSqVOroqnQp9Ls2jz8ay9vhcekxHPCGke6CWFiGkS6HQIoR0KRRahJAuhUKLENKlUGgRQroUCq0mNm3ahODgYMhkMqjVahw7dszRVXK4VatWQSAQmN3Cw8MdXa0Od+jQIUyePBkqlQoCgQC7du0ye54xhsTERAQEBEAulyMmJgYXL150TGU72P0+m7lz597zHRo/frxdr0WhdZcdO3YgISEBSUlJOHnyJCIiIhAbG4vi4mJHV83hBg0ahJs3b/K3X375xdFV6nCVlZWIiIjApk2bLD6/Zs0abNiwAZs3b0ZmZiZcXFwQGxuLmpqWryTdHdzvswGA8ePHm32Htm/fbt+LMcKLiopiCxcu5P82Go1MpVKx5ORkB9bK8ZKSklhERISjq9GpAGDffPMN/7fJZGJKpZKtXbuWX1ZWVsakUinbvn27A2roOE0/G8YYmzNnDpsyZUqbbJ9aWvUMBgOysrIQExPDLxMKhYiJiUFGRoYDa9Y5XLx4ESqVCn379sWsWbNQUFDg6Cp1Kvn5+dBoNGbfHw8PD6jVavr+1Dt48CD8/f0xYMAAvPzyy7h165Zd26HQqldaWgqj0QiFQmG2XKFQQKPROKhWnYNarcbWrVuRlpaGDz74APn5+Xj00Uf5OcoI+O8IfX8sGz9+PLZt24b09HS8++67+OmnnzBhwgQYjUabt9VjpqYh9pswYQL/eOjQoVCr1QgKCsIXX3yBefPmObBmpKt47rnn+MdDhgzB0KFD0a9fPxw8eBDjxo2zaVvU0qrn6+sLkUiEoqIis+VFRUVQKpUOqlXn5OnpibCwMFy6dMnRVek0Gr4j9P2xTt++feHr62vXd4hCq55EIkFkZCTS09P5ZSaTCenp6YiOjnZgzTqfiooKXL58GQEBAY6uSqcREhICpVJp9v3R6XTIzMyk748Fv/32G27dumXXd4h2D++SkJCAOXPmYPjw4YiKisL69etRWVmJF154wdFVc6jFixdj8uTJCAoKQmFhIZKSkiASiTBz5kxHV61DVVRUmLUM8vPzkZ2dDW9vb/Tp0wevvfYa/va3vyE0NBQhISFYuXIlVCoV4uLiHFfpDtLSZ+Pt7Y23334b06ZNg1KpxOXLl7FkyRL0798fsbGxtr9YmxyD7EY2btzI+vTpwyQSCYuKimJHjx51dJUcbsaMGSwgIIBJJBLWq1cvNmPGDHbp0iVHV6vDHThwgAG45zZnzhzGGDfsYeXKlUyhUDCpVMrGjRvH8vLyHFvpDtLSZ1NVVcWeeOIJ5ufnx8RiMQsKCmLz589nGo3Grtei+bQIIV0K9WkRQroUCi1CSJdCoUUI6VIotAghXQqFFiGkS6HQIoR0KRRahJAuhUKLENKlUGgRQroUCi1CSJdCoUUI6VL+P5nZcKwAb6eEAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:07:21.615562Z", + "iopub.status.busy": "2024-02-29T18:07:21.614762Z", + "iopub.status.idle": "2024-02-29T18:10:33.180446Z", + "shell.execute_reply": "2024-02-29T18:10:33.179512Z" + }, + "papermill": { + "duration": 191.589819, + "end_time": "2024-02-29T18:10:33.183090", + "exception": false, + "start_time": "2024-02-29T18:07:21.593271", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:10:33.225575Z", + "iopub.status.busy": "2024-02-29T18:10:33.225211Z", + "iopub.status.idle": "2024-02-29T18:10:33.248762Z", + "shell.execute_reply": "2024-02-29T18:10:33.247602Z" + }, + "papermill": { + "duration": 0.047837, + "end_time": "2024-02-29T18:10:33.251149", + "exception": false, + "start_time": "2024-02-29T18:10:33.203312", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.011033NaN0.0012812.5217410.1342942.2744970.2217440.0000014.9053350.0284220.066030.035790.0549060.0218317.427076
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.011033 NaN 0.001281 2.521741 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.134294 2.274497 0.221744 0.000001 4.905335 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.028422 0.06603 0.03579 0.054906 0.021831 \n", + "\n", + " total_duration \n", + "realtabformer 7.427076 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:10:33.292111Z", + "iopub.status.busy": "2024-02-29T18:10:33.291272Z", + "iopub.status.idle": "2024-02-29T18:10:33.706150Z", + "shell.execute_reply": "2024-02-29T18:10:33.705110Z" + }, + "papermill": { + "duration": 0.436775, + "end_time": "2024-02-29T18:10:33.708311", + "exception": false, + "start_time": "2024-02-29T18:10:33.271536", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:10:33.751534Z", + "iopub.status.busy": "2024-02-29T18:10:33.750562Z", + "iopub.status.idle": "2024-02-29T18:13:57.754367Z", + "shell.execute_reply": "2024-02-29T18:13:57.753398Z" + }, + "papermill": { + "duration": 204.044892, + "end_time": "2024-02-29T18:13:57.773671", + "exception": false, + "start_time": "2024-02-29T18:10:33.728779", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:13:57.815520Z", + "iopub.status.busy": "2024-02-29T18:13:57.815088Z", + "iopub.status.idle": "2024-02-29T18:13:57.833388Z", + "shell.execute_reply": "2024-02-29T18:13:57.832575Z" + }, + "papermill": { + "duration": 0.04162, + "end_time": "2024-02-29T18:13:57.835592", + "exception": false, + "start_time": "2024-02-29T18:13:57.793972", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:13:57.875152Z", + "iopub.status.busy": "2024-02-29T18:13:57.874265Z", + "iopub.status.idle": "2024-02-29T18:13:57.880039Z", + "shell.execute_reply": "2024-02-29T18:13:57.879224Z" + }, + "papermill": { + "duration": 0.027595, + "end_time": "2024-02-29T18:13:57.881915", + "exception": false, + "start_time": "2024-02-29T18:13:57.854320", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.43385773822478485}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:13:57.924453Z", + "iopub.status.busy": "2024-02-29T18:13:57.924098Z", + "iopub.status.idle": "2024-02-29T18:13:58.294427Z", + "shell.execute_reply": "2024-02-29T18:13:58.293404Z" + }, + "papermill": { + "duration": 0.394857, + "end_time": "2024-02-29T18:13:58.296848", + "exception": false, + "start_time": "2024-02-29T18:13:57.901991", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASsAAAE8CAYAAACYd5eXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCsklEQVR4nO3deXhU5fnw8e+ZNdtksm+QkBA2FQRkExEF5ee+UCqtKyBFrEKtUt4qvSruolYpaltaNwKura1aW5W6oqWubIqi7AkhO5Bksk5mOe8fJxkYspCZTDJL7s91zTUzZ86cuXMyufM8z3kWRVVVFSGECHG6YAcghBDdIclKCBEWJFkJIcKCJCshRFiQZCWECAuSrIQQYUGSlRAiLEiyEkKEBUlWQoiwIMlK+O3uu+9GUZSAHrOgoABFUdi0aVO39v/d737H4MGD0ev1jBkzJqCxiNAiyUoE1IMPPsgbb7zRJ5/17rvv8utf/5opU6awZs0aHnzwwT75XBEchmAHICLLgw8+yBVXXMHMmTN7/bM+/PBDdDodzz77LCaTqdc/TwSXlKz6iYaGhmCHEHCVlZVER0cHLFGpqkpTU1NAjtUTjY2NwQ4hJEmyikBtbUk7duzg6quvJjExkTPPPBOAF154gXHjxhEdHU1SUhJXXnklxcXFXu//73//y+zZs8nJycFsNpOdnc1tt912wj9kRVFoaGhg7dq1KIqCoijMmzcPgKKiIm6++WaGDx9OdHQ0ycnJzJ49m8LCwg6P1djYyI033khycjLx8fHMmTOH6upqr89as2YNDQ0Nns8qKCgAwOl0ct9995Gfn4/ZbCY3N5ff/OY32O12r8/Izc3lkksu4T//+Q/jx48nOjqav/zlL2zYsAFFUfjb3/7GPffcw4ABA7BYLFxxxRXU1tZit9u59dZbSUtLIy4ujuuvv77dsbt7rqdNm8bIkSPZvHkzZ511FjExMfzmN7/p8jz3V1INjGCzZ89m6NChPPjgg6iqygMPPMCdd97JT37yExYsWEBVVRVPPvkkZ511Flu3biUhIQGAV199lcbGRm666SaSk5P58ssvefLJJzl48CCvvvpqp5/3/PPPs2DBAiZOnMjChQsByM/PB+Crr77i008/5corr2TgwIEUFhayevVqpk2bxo4dO4iJifE61uLFi0lISODuu+9m586drF69mqKiIk8ief7553nqqaf48ssveeaZZwA444wzAFiwYAFr167liiuu4Fe/+hVffPEFK1as4Pvvv+f111/3+pydO3dy1VVXceONN3LDDTcwfPhwz2srVqwgOjqaO+64gz179vDkk09iNBrR6XRUV1dz99138/nnn1NQUEBeXh7Lly/3vLe75xrg8OHDXHjhhVx55ZVce+21pKen+/ib7idUEXHuuusuFVCvuuoqz7bCwkJVr9erDzzwgNe+27dvVw0Gg9f2xsbGdsdcsWKFqiiKWlRU1O5zjhUbG6vOnTu33fs7OuZnn32mAuq6des829asWaMC6rhx49SWlhbP9kceeUQF1H/+85+ebXPnzlVjY2O9jrlt2zYVUBcsWOC1fenSpSqgfvjhh55tgwYNUgF1/fr1Xvt+9NFHKqCOHDnSK4arrrpKVRRFvfDCC732nzx5sjpo0CDPc1/O9dlnn60C6p///Od250d4k2pgBPv5z3/uefzaa6/hdrv5yU9+wqFDhzy3jIwMhg4dykcffeTZNzo62vO4oaGBQ4cOccYZZ6CqKlu3bvUrlmOP6XA4OHz4MEOGDCEhIYEtW7a023/hwoUYjUbP85tuugmDwcDbb7/d5ee0vb5kyRKv7b/61a8AeOutt7y25+Xlcf7553d4rDlz5njFMGnSJFRVZf78+V77TZo0ieLiYpxOJ+DbuQYwm81cf/31Xf5cQqqBES0vL8/zePfu3aiqytChQzvc99g/ygMHDrB8+XLefPNNr3YigNraWr9iaWpqYsWKFaxZs4aSkhLUYyao7eiYx8cZFxdHZmZmp21cbYqKitDpdAwZMsRre0ZGBgkJCRQVFXltP/YcHS8nJ8frudVqBSA7O7vddrfbTW1tLcnJyT6da4ABAwbI1cxukGQVwY4tzbjdbhRF4Z133kGv17fbNy4uDgCXy8X//d//ceTIEW6//XZGjBhBbGwsJSUlzJs3D7fb7Vcsv/jFL1izZg233norkydPxmq1oigKV155pd/H7Ep3O6see46O19F56mp7WwLu7rnuTgziKElW/UR+fj6qqpKXl8ewYcM63W/79u3s2rWLtWvXMmfOHM/29957r1uf01mS+Pvf/87cuXN57LHHPNuam5upqanpcP/du3czffp0z/P6+nrKysq46KKLuvz8QYMG4Xa72b17NyeddJJne0VFBTU1NQwaNKhbP0dPdPdcC99Im1U/MWvWLPR6Pffcc49XFQy0EsHhw4eBo6WGY/dRVZXHH3+8W58TGxvbYQLS6/XtPvfJJ5/E5XJ1eJynnnoKh8Pheb569WqcTicXXnhhl5/flsxWrVrltX3lypUAXHzxxSf6EXqsu+da+EZKVv1Efn4+999/P8uWLaOwsJCZM2disVjYv38/r7/+OgsXLmTp0qWMGDGC/Px8li5dSklJCfHx8fzjH/9o13bVmXHjxvH++++zcuVKsrKyyMvLY9KkSVxyySU8//zzWK1WTj75ZD777DPef/99kpOTOzxOS0sL5557Lj/5yU/YuXMnf/rTnzjzzDO57LLLuvz80aNHM3fuXJ566ilqamo4++yz+fLLL1m7di0zZ870Kq31lu6ea+EbSVb9yB133MGwYcP4/e9/zz333ANojcXnnXeeJwkYjUb+9a9/ccstt7BixQqioqL40Y9+xOLFixk9evQJP2PlypUsXLiQ3/72tzQ1NTF37lwmTZrE448/jl6v58UXX6S5uZkpU6bw/vvvd3ol7g9/+AMvvvgiy5cvx+FwcNVVV/HEE090qy3qmWeeYfDgwRQUFPD666+TkZHBsmXLuOuuu3w4Wz3TnXMtfKOox5dThRAiBEmblRAiLEiyEkKEBUlWQoiwIMlKCBEWJFkJIcKCJCshRFiI+H5Wbreb0tJSLBZLwBc3EEL0jKqq1NXVkZWVhU7Xddkp4pNVaWlpu1HyQojQUlxczMCBA7vcJ+KTlcViAbSTER8fH+RohBDHstlsZGdne/5OuxLxyaqt6hcfHy/JSogQ1Z0mGmlgF0KEBUlWQoiwIMlKCBEWIr7NSohAcLlcXpMBiu4zGo2dTgXtC0lWQpxAfX09Bw8ebDfrp+geRVEYOHBgu7nnfSXJSoguuFwuDh48SExMDKmpqdKx2EeqqlJVVcXBgwcZOnRoj0pYkqxEv1DT2MKGnVXEmPRMH5GGUd+95lqHw4GqqqSmpsoqNH5KTU2lsLAQh8MhyUqIrqiqyjvfllNe2wxAtEnP1KGpPh1DSlT+C9S5k6uBIuKV1TZ7EhVASXWTtD+FISlZiYi3r6oBgOEZFqYMScEabTzBO0QokpKViHj7D2vJKi8lVhJVL8jNzW23TmNvkGQlIlqL083hejsAOUkxQY5G9IRUA0VEM+oVfnZmHlV1dmLNBmzNDt7fUUGTw8U1k3p/Kflw0dLSgslkCnYYXZKSlYhoiqJgiTIyOFXrkGg26Cg63EilzU5ji9Pv47Y43Z3enC53t/d1dHNfX02bNo3FixezePFirFYrKSkp3HnnnZ4LC7m5udx3333MmTOH+Ph4Fi5cCMDGjRuZOnUq0dHRZGdnc8stt9DQ0OA5bmVlJZdeeinR0dHk5eXx4osv+hybv6RkJfoVs0GPNdpIbZODIw0txJj8+xP440d7On0tLyWWmWMHeJ4/9cleHK6Orz4OTIxm9vijk0M+97/9NLW42u132/8N8znGtWvX8rOf/Ywvv/ySTZs2sXDhQnJycrjhhhsAePTRR1m+fLlnpeq9e/dywQUXcP/99/Pcc89RVVXlSXhr1qwBYN68eZSWlvLRRx9hNBq55ZZbqKys9Dk2fwS1ZPXJJ59w6aWXkpWVhaIovPHGG16vq6rK8uXLyczMJDo6mhkzZrB79+7gBCvC0v/2HOLTvYeoaz46rq+tkb2mMbLH+mVnZ/P73/+e4cOHc8011/CLX/yC3//+957XzznnHH71q1+Rn59Pfn4+K1as4JprruHWW29l6NChnHHGGTzxxBOsW7eO5uZmdu3axTvvvMPTTz/N6aefzrhx43j22Wdpamrqk58nqCWrhoYGRo8ezfz585k1a1a71x955BGeeOIJ1q5dS15eHnfeeSfnn38+O3bsICoqKggRi3CiqirbimtocboZnn50JsqEGCMHjoCtyf9ktWj6kE5f0x3XB3LhWfmd7nt8f8n5U/L8jul4p59+uleHzMmTJ/PYY4/hcmklt/Hjx3vt//XXX/PNN994Ve1UVcXtdrN//3527dqFwWBg3LhxntdHjBhBQkJCwGLuSlCT1YUXXsiFF17Y4WuqqrJq1Sp++9vfcvnllwOwbt060tPTeeONN7jyyiv7MlQRhhpaXLQ43SgKXl0WEmJaS1Y9SFYmQ/crJb21b0/FxsZ6Pa+vr+fGG2/klltuabdvTk4Ou3bt6qvQOhSybVb79++nvLycGTNmeLZZrVYmTZrEZ5991mmystvt2O12z3ObzdbrsYrQVN3QAmiJynDMWEBrtHbVK9KrgV988YXX888//7zLwcSnnXYaO3bsYMiQjkuNI0aMwOl0snnzZiZMmADAzp07qampCWjcnQnZq4Hl5eUApKene21PT0/3vNaRFStWYLVaPTdZ2ab/qm7UklVijPcl+YQYI5YoA1HGkP36B8SBAwdYsmQJO3fu5OWXX+bJJ5/kl7/8Zaf733777Xz66acsXryYbdu2sXv3bv75z3+yePFiAIYPH84FF1zAjTfeyBdffMHmzZtZsGBBnw3wjrjf1rJly6itrfXciouLgx2SCJLq1pJTYqx3skqJM7Ng6mBmndb10k/hbs6cOTQ1NTFx4kQWLVrEL3/5S08XhY6ceuqpfPzxx+zatYupU6cyduxYli9fTlZWlmefNWvWkJWVxdlnn82sWbNYuHAhaWlpffHjhG41MCMjA4CKigoyMzM92ysqKhgzZkyn7zObzZjN5t4OT4SBGk/Jqn8OsTEajaxatYrVq1e3e62wsLDD90yYMIF3332302NmZGTw73//22vbdddd16M4uytkS1Z5eXlkZGTwwQcfeLbZbDa++OILJk+eHMTIRLioa9Y6fcZH9c9kFWmCWrKqr69nz56jnev279/Ptm3bSEpKIicnh1tvvZX777+foUOHerouZGVlMXPmzOAFLcLGVRNzqG92Em1q36D80c5K9lc1cObQFIaln3iBTRF8QU1WmzZtYvr06Z7nS5YsAWDu3LkUFBTw61//moaGBhYuXEhNTQ1nnnkm69evlz5Wolv0OgVrJ1VAu8NFbZOjR32tQtmGDRuCHULABTVZTZs2rctJ0BRF4d577+Xee+/tw6hEfxBn1pJYnd3/8YGib4VsA7sQPVFa08Q3B2vJSoji1IEJ7V6PNWtVw/pmSVbhImQb2IXoico6O9+X2Sg63Njh65Yo7f90g5SswoYkKxGR2gYutyWl48WaW5NVBzMciNAkyUpEJFtTa7eFTqYxjjFqyaqpB3Naib4lbVYiIrWVrOI7KVlFm/RYogzEmAy43Cr646dKECFHkpWISDZPNbDjkpXJoGPB1MF9GZLoIakGiojjcqs02LW2qM7arET4kWQlIk7b3Oo6RSHa6P9y5SK0yL8dEXEsUUZ+cc4QmhyuLpcu/3hXFXsr65mcn8xJmfHdO7iqgitIvd71xvZTi3Zi3bp13HbbbZSWlnoN7J85cyYWi4Xnn3++t6LsNZKsREQy6HVY9F1XHJpbh9zU+9LXyuWA/z7Ww+j8NPVXYOjeclmzZ8/mlltu4c0332T27NmAtjLNW2+91eWsCqFMqoGi32qrIna0mky4i46O5uqrr/asSgPwwgsvkJOTw7Rp04IXWA9IyUpEnB2lNg4caWRIWhxD0uI63S+mdTaGRl+Sld6olXCCQe/bVDc33HADEyZMoKSkhAEDBlBQUMC8efO6rBqHMklWIuKU1DTxfZmNhBhjl8mqbeqYJocP1UBF6XZVLNjGjh3L6NGjWbduHeeddx7fffcdb731VrDD8pskKxFx2q4Gxpm7/nq3LXDqU8kqzCxYsIBVq1ZRUlLCjBkzwnpNAmmzEhGnrcE89gTJKpLbrNpcffXVHDx4kKeffpr58+cHO5wekWQlIk7bTAqxHcwQeqxoo7aUfGfjByOB1Wrlxz/+MXFxcWE/w65UA0VEcbtVT7XuRCUra4yR+WcGbgXkUFVSUsI111wT9gupSLISEaXR4UJVtd7rMScoWUW66upqNmzYwIYNG/jTn/4U7HB6TJKViChtjeuxZn3YXqIPlLFjx1JdXc3DDz/M8OHDgx1Oj0myEhElzRLFLecOpdnRvUbzf31dSlWdnQtHZZBp7ZuVhftKZ2sDhitpYBcRR69TTthe1abe7qS2yRHR3RcihSQr0a9FGbU/gROVxLpahUl0LVDnTpKViChbDlSz/ttyDnSyUMTxogxaI3yzw93h63q99npLS0tgAuyH2s5d27n0l7RZiYhSfKSRfVUNDEjoXvtTVGvHUHsnJSuDwUBMTAxVVVUYjUZ0Ovn/7gu3201VVRUxMTEYDD1LN5KsRERpmyG0bV3AEzG3VQOdHScrRVHIzMxk//79FBUVBSbIfkan05GTk9Pjq7OSrEREaejmUJs2bSWrzqqBACaTiaFDh0pV0E8mkykgJVJJViJiuN0qDS2+Jas4swFrtPGE0x/rdDqioqJ6HKPwnyQrETGaWnuvKwrEdHPu9WHpFoalW3o5MhEI0looIsbRAcwGdLIOYMQJ6WTlcrm48847ycvLIzo6mvz8fO677z7p8yI61NR6RS+mm43rIryEdDXw4YcfZvXq1axdu5ZTTjmFTZs2cf3112O1WrnllluCHZ4IMYOSY7nl3KG0ODtvLD+e3eniH5tLaHa4mHdGrpTIQlhIJ6tPP/2Uyy+/nIsvvhiA3NxcXn75Zb788ssgRyZClV6neKYr7g6jTkeFrRkAu9Pt03tF3wrpauAZZ5zBBx98wK5duwD4+uuv2bhxIxdeeGGn77Hb7dhsNq+bEJ3R6RRMhu4NuRHBFdIlqzvuuAObzcaIESPQ6/W4XC4eeOABrrnmmk7fs2LFCu65554+jFKEik/3HsLW5GB0doJPMyiYDTpanG7sPlQfRd8L6ZLV3/72N1588UVeeukltmzZwtq1a3n00UdZu3Ztp+9ZtmwZtbW1nltxcXEfRiyCqehwI9+X1fk8g4K5bchNJ73YRWgI6ZLV//t//4877riDK6+8EoBRo0ZRVFTEihUrmDt3bofvMZvNYT99q/DPsV0XfGFurQZKySq0hXTJqrGxsV03fb1ej9stXyrhTVVVn8cFtmlLVr5cRRR9L6RLVpdeeikPPPAAOTk5nHLKKWzdupWVK1eG/ZJCIvCaHC7cqoqi+F6yskRpQ250/Xwa5FAX0snqySef5M477+Tmm2+msrKSrKwsbrzxRpYvXx7s0ESIaVsrMMak97mv1Dkj0nsjJBFgIZ2sLBYLq1atYtWqVcEORYS4tipgjI+lKhE+QrrNSojuausjdaIl40X4kt+siAgnZcYzLN2Cw+V7I/m+qno+33eENIuZGSdLlTBUSbISEUOvU9DrfB8u43SrVNiaMeilgT2USTVQ9HvSzyo8SMlKRISPfqjE7nQxPjeJlDjfOgWbDV0vGiFCg5SsRETYW1XP92V1frVZSckqPEiyEmHPu/e675WFthVuHC63TOwYwiRZibDX1nsdfO+9DmDSa38Gqiqlq1AmbVYi7B3be13vx0yfBr0OS5QBnaLgdEvJKlRJshJhr7Gt93oPOoQumDo4UOGIXiLVQBH22kpWcbJQRESTZCXCXls7kz/tVSJ8yG9XhL1xgxIZk52AswfznG3cfYgDRxqZmJfIkDRZ9DQUSbISEcHfoTZtapscVNiaqWt2BjAqEUhSDRQC6RgaDqRkJcLe29vL0ClwxpAU4qOMfh2jrWOoJKvQJSUrEdZUVWVPpTbUpidkfGDok5KVCGvNDjcudxe9191uqPoe6sogJgUyRkEHbVtSDQx9kqxEWGvrYxXdUe91px2+/QdUFx3dVroVRl8JRu9FUE2ywk3Ik2qgCGuetQKP772uqrDjTS1R6Y2QNUZLUHXl8MNb2uvHiDLqiTLqZQK+ECYlKxHWGlo66b1euhUO7wGdAcZcDfFZkDUWtqyDQ7uh6gdIO8mze15KLDdNy+/L0IWPpGQlwppnaphj26sczbD/E+3x4GlaogKwZEDOZO1x4UatPUuEDUlWIqw5XG5tYdNjq4FF/wNHE8SmwIBx3m/IngjGKGg4BEf29m2wokf8Slb79u0LdBxC+GXKkBR+cc5QJuQmaRscTVoVEGDwdNAd9xU3mCHjVO1x6TbPZqfLzT82H+SlLw74Nduo6H1+JashQ4Ywffp0XnjhBZqbmwMdkxA+0esUz9U8SreCywFxqZDcSRtU5hjt/sheaLZ5jlFc3UiFrVm6L4Qov5LVli1bOPXUU1myZAkZGRnceOONfPnll4GOTQjfuJxwcJP2OHsSKJ1c2YtNButA7Ypg1U4AFEWRjqEhzq9kNWbMGB5//HFKS0t57rnnKCsr48wzz2TkyJGsXLmSqqqqQMcpRDsut8qrm4pZ/22ZVnU7tAtaGsBsgbSTu35z25XAqh88m9o6hrZINTAk9aiB3WAwMGvWLF599VUefvhh9uzZw9KlS8nOzmbOnDmUlZUFKk4h2qlvdnKwuoldFfUYdAqUfa29kHlqh73UvaQM1e5tJWCvB452DLU7JFmFoh4lq02bNnHzzTeTmZnJypUrWbp0KXv37uW9996jtLSUyy+/PFBxCtFOnd0BQJzZgNJcA9WFWtWvrQG9K1FWiM/UqoKH9wAy5CbU+ZWsVq5cyahRozjjjDMoLS1l3bp1FBUVcf/995OXl8fUqVMpKChgy5YtPQ6wpKSEa6+9luTkZKKjoxk1ahSbNm3q8XFF+PNMZxxlgLJvtI2JuRCd0L0DJLU2wFfvB8BsbG2zckqbVSjyqwf76tWrmT9/PvPmzSMzM7PDfdLS0nj22Wd7FFx1dTVTpkxh+vTpvPPOO6SmprJ7924SExN7dFwRGepbJ8qLN+ugfLu2MXN09w+QlKd1Dq0uBLebKIOOKKPM4x6q/EpW7733Hjk5OeiO68OiqirFxcXk5ORgMpmYO3duj4J7+OGHyc7OZs2aNZ5teXl5Xb7Hbrdjt9s9z202W49iEKGrrrVkleyoAHud1tkzeWj3D2DJ0vpdOZqhvpzzTsnqpUhFIPhVDczPz+fQoUPtth85cuSEycQXb775JuPHj2f27NmkpaUxduxYnn766S7fs2LFCqxWq+eWnZ0dsHhEaGkrWSU3tvZETx0Beh/+/+p0kDhIe3xEOjqHOr+SVWdLbNfX1xMVFdWjgI61b98+Vq9ezdChQ/nPf/7DTTfdxC233MLatWs7fc+yZcuora313IqLiwMWjwgtLreKDhfx9a2J5piByd2WmKvd18j3JNT5VA1csmQJoHWgW758OTExMZ7XXC4XX3zxBWPGjAlYcG63m/Hjx/Pggw8CMHbsWL799lv+/Oc/d1rFNJvNmM3mgMUgQtfMsQNwVTWibHeBKQ6sOb4fpO09toMcqKrjqwO1JMeZmDY8LbDBih7zKVlt3aqNuVJVle3bt2MymTyvmUwmRo8ezdKlSwMWXGZmJief7N2576STTuIf//hHwD5DhDd91fdad4W0k9qPA+yO2BStrcvRjNNWxoEjeGYeFaHFp2T10UcfAXD99dfz+OOPEx8f3ytBtZkyZQo7d+702rZr1y4GDRrUq58rwoTLofVahxP3WO+MooA1Gw7tJqapDMiUrgshyq82qzVr1vR6ogK47bbb+Pzzz3nwwQfZs2cPL730Ek899RSLFi3q9c8Woa3C1sz6T/5HYWVNawfPHlzJsw4EIKqxFJBOoaGq2yWrWbNmUVBQQHx8PLNmzepy39dee63HgQFMmDCB119/nWXLlnHvvfeSl5fHqlWruOaaawJyfBG+qhtbcJT/QIPbCanDOx+03B1W7Yqxqb4U1NMkWYWobicrq9WK0vqFsFqtvRbQ8S655BIuueSSPvs8ER5sDXYSmw9gjtVDyrCeHcySAXoDxpZmop01NCmJuN0quuMXoBBB1e1kdWzHzGMfCxEMjsOF6N0tGKOSIX5Azw6m04MlC311ERZ7BU3GRFpcbqJ6sBy9CDy/2qyamppobGz0PC8qKmLVqlW8++67AQtMiC4d3g2AkjLEv6uAx4vPQqcoJDgPEWXUy2yhIciv4TaXX345s2bN4uc//zk1NTVMnDgRk8nEoUOHWLlyJTfddFOg4xTiKFXFWL0HF2BMHx6YY7Y20F80yI0yUVa5CUV+zxQ6depUAP7+97+TkZFBUVER69at44knnghogEIcT60rR2224VYMxKQPCcxBW5OV0ngInC2BOaYIKL+SVWNjIxaLBYB3332XWbNmodPpOP300ykqKjrBu4XomZbynRj1OmzRA7HERJ/4Dd1htmg3VYX68sAcUwSU3wtGvPHGGxQXF/Of//yH8847D4DKyso+6X8l+jezbT+n5SRy3llnBvaKXXwmB6sb+eirr9lXVR+444qA8CtZLV++nKVLl5Kbm8ukSZOYPFlbOPLdd99l7NixAQ1QCC+OJqivAECfPDiwx7Zk0djiovHQAWqbHIE9tugxvxrYr7jiCs4880zKysoYPfroZGfnnnsuP/rRjwIWnBDt1BzQqmoxyRAV4FJ8fBYGnUJsY5V0DA1BfiUrgIyMDDIyMry2TZw4sccBCdGl6iIOHGmkqDGbzMp6hqTFBe7Ylkx0eh1mVz3OJhuQHLhjix7zK1k1NDTw0EMP8cEHH1BZWYnb7f1fSFZsFr2mpoh6u5N9hhSSA136MZhQo5Oh5iBKXRkQuIkkRc/5lawWLFjAxx9/zHXXXUdmZqZnGI4QvcpeBw2HaHa6sZmziI/2u2LQKTU+C8oOopMrgiHHr9/2O++8w1tvvcWUKVMCHY8Qnasuwq2qVCuJOPVRJMSYTvweX8VpTRv6hsrAH1v0iF9XAxMTE0lKSgp0LEJ0raYIu9NNjTkLk0FHrCnwY/d01gwUIKq5UmvIFyHDr2R13333sXz5cq/xgUL0KlWF6iKaHS5sUVlYo4290vyQMzCXSfkpnJpm1KqdImT4VQ187LHH2Lt3L+np6eTm5mI0Gr1eD8TipkJ4aa6B5lqanCq2qEzyY4wnfIs/dEaTNtVxfZXWnyvQ3SOE3/xKVjNnzgxwGEKcQHUhAI7YTGLN0ST2RntVm7gMLVnVlUGKD+sQil7lV7K66667Ah2HEF2r1sacDso/iQV5gztdDq6nnC43n1cYSCqzMTyxDJnRKnT4PRFQTU0NzzzzDMuWLePIkSOAVv0rKSkJWHBCAFp7VU3rAPnWdf56q7uMXqfwQ0MMtU0OnLayXvkM4R+/SlbffPMNM2bMwGq1UlhYyA033EBSUhKvvfYaBw4cYN26dYGOU/RnDVXQ0qittmzp3SXeFUXBGZ0KKLib67VGdrOlVz9TdI9fJaslS5Ywb948du/e7bUC80UXXcQnn3wSsOCEADxVwPqoTJ759AD/3Na7pXeD0UyjMUFbP7Cuolc/S3SfX8nqq6++4sYbb2y3fcCAAZSXS89fEWCtVcBacxZ1zU7q7c5e/TizUU+DKQWn2601souQ4FeyMpvN2Gy2dtt37dpFampqj4MSwsPt9iSrKmMmAEm9eSUQMBt0NJhStJJVvZSsQoVfyeqyyy7j3nvvxeHQ5vxRFIUDBw5w++238+Mf/zigAYp+rq5Mm2bYYKbcpS0BlxTbF8kqFadLhTqpKYQKv5LVY489Rn19PampqTQ1NXH22WczZMgQLBYLDzzwQKBjFP1Z21XAhBwON2r/HJPjzL36kWaDjkZTMm7QGtjtMmtoKPDraqDVauW9997jf//7H19//TX19fWcdtppzJgxI9Dxif6utXFdTcylukpbyCG5l0tWM05K5/xTMlC+GggNh7SqoDmA82YJv/icrNxuNwUFBbz22msUFhaiKAp5eXlkZGSgqqpMFyMCx+WE2oMA2KKycLjqMegUrNG9M9SmjUHfWuGIS9eSVV0ZJMvyXMHmUzVQVVUuu+wyFixYQElJCaNGjeKUU06hqKiIefPmyZTGIrBsB8HtBFMsLaYkBiZGk5UQ3XfLultaZ8KVdquQ4FPJqqCggE8++YQPPviA6dOne7324YcfMnPmTNatW8ecOXMCGqTop6rbeq0PIjU+itnjs/vkYw9WN7KtuIZM1cw4kCuCIcKnktXLL7/Mb37zm3aJCuCcc87hjjvu4MUXXwxYcMd76KGHUBSFW2+9tdc+Q4SQ44bY9JXGFhe7K+rZ3xwHigLNNmhp6NMYRHs+JatvvvmGCy64oNPXL7zwQr7++useB9WRr776ir/85S+ceuqpvXJ8EWKcdmgbm5cwiJY+XG3GbND+LJpVA0S3TjIpVcGg8ylZHTlyhPT09E5fT09Pp7q6usdBHa++vp5rrrmGp59+msTExIAfX4SgmmJQ3RCdgMMUz+oNe3lu436aHa5e/2hTa7KyO91gaf2+S7IKOp+SlcvlwmDovJlLr9fjdAZ+KMSiRYu4+OKLu9U1wm63Y7PZvG4iDNUUaveJuRyub8Gtqjhcbk+ppzeZDdrEMHanCyxar3kZdhN8PjWwq6rKvHnzMJs77pRnt9sDEtSxXnnlFbZs2cJXX33Vrf1XrFjBPffcE/A4RB9ra1xPGERlXTMAafHmPuka05YQW5xu1Lg0FJBG9hDgU7KaO3fuCfcJ5JXA4uJifvnLX/Lee+95ze7QlWXLlrFkyRLPc5vNRnZ231xFEgHS0gD1ravLJORQsVdr3E6zdO870FNtyUpVoSU6DfOxjeym2D6JQbTnU7Jas2ZNb8XRoc2bN1NZWclpp53m2eZyufjkk0/4wx/+gN1uR6/3nsvRbDZ3WvITYaLmgHYfmwLmOCrrDgOQHt83v1eDXodep+BWVewYMUcnQeNhrd1KOocGTeBXiQygc889l+3bt3ttu/766xkxYgS33357u0QlIkTrfOsk5uF0uTlcrw2zSe2jkhXAjWcPxqTXadVOS7okqxAQ0snKYrEwcuRIr22xsbEkJye32y4iiCdZ5VJRZ8flVokx6YmP6ruva1sjO6A1slfskEb2IAvpZCX6ocYj0FQDig4SsjE3w6gBVowGXfDGnca1dl+QRvagCrtktWHDhmCHIHpTW6nKOgAMZlLiYMbJnfft6y2bi6opq21i9MAEsuMzvHuySyN7UPR+pxUhfFG9X7tPzAtqGGW1TeyuqOdQvR0MZunJHgIkWYnQ4XYfM3g5F1uzg5KaJtzu3lkjsCvRRq3NqtnROsxHerIHnSQrETrqyrQxgQYzWDL5vtTG374qZv13fZ8gotqSlbN1eI/0ZA86SVYidBzZp90nDgKdjsLDWmfQgYnRfR5KlLF1fGDbWERpZA86SVYidBzerd0nD6GpxUVZrTbMJjel7xu027ouHK0GZsh0MUEmyUqEhuZabUFRRYHkIeyqqENVIdViJj6qd6cx7khbNbCprWQljexBJ8lKhIZDe7T7+AFgimVHmTZbxkmZ8UEJJ9rUOvPCsVPSSCN7UIVdPysRodqqgClDqbA1U17bjE5RGJFhCUo4GfFR3DQt33tKGunJHlSSrETwtTQe7bKQMoy9pdo6fcMz4og1B+crqtcp6HXHjT2VRvagkmQlTijQS6xV2Jr55mAtjS1O0ixRnKLuIV51a43YMUmcMQSyk2Kw9OFYwG45vpFderL3qRD7NohQUtPYwtvby6mqszMsPY4ZJ6dj1PesmfP7MhvvfleBW9U6eu6raqCu4hOyqMU4YgLDW/fLTorpYfQ9t2FnJfV2J2cNS9Ua+dsa2WUGhqCQBnbRIafLzZtfl1Jha8atqpTUNGFrcvT4uNWNLaio5KfFMW14KsPiW4i3V1Bnd/GZLR1V7fve6p3ZW9XA7op6GuzHTNUtjexBIyUr0aHtJbUcrm8h1qxn+vA0spNiPJfze+KM/BSGpMWRGqdNUTzWvomWQYlUGAZgyB2Ay61i0IfGqt5RRh22pmP6WoE0sgeRJCvRjtutsvVADQCnD05maHpgr8h5pid22qH8G0x6HdmnnkV2UlJAP6enojwdQ4/tvtA67MZWqs17HKxpa/ohqQaKdkpqmqhtcmA26rz6ObncKnsq62hs8X0Fox2lNipszd7VvNJt4GyBmOSgz7LQkXYdQ6G1kV2nNbA31wYpsv5JkpVop9ymDXMZkhrn1aD+z20l/OvrMn4or/PpeC1ONx/+UMFLXxygqr51BSRHMxz4VHucPTEkSyjRptbFTo9NVnrj0XYrW2kQouq/JFmJdibkJjH/zDwm5nlXywanxgGwy8dktf9QAw6XSkKMkdS41kUfijZqCSsmGTJCc5Xttmqg3XHcatDxA7R7SVZ9SpKV6JA12khCjMlr25A0LVmV1TZTb+9+VXBXhZbchqVbtP5aNQfg4KbWg54LutD8GpqNHbRZAcRnafe2kj6OqH+TBnbRbXFmAxnWKMprm9lf1cCogdYTvsfudFF4SJulYGh6nDbH+revaY3TGaNCuq/SqAFWTsmKb78KdFuyqq8AlxP08mfUF+QsCy/rvy2n2eFi0uAkMq3HzSNlr+ek6BocZSVU7CtnVGwOmOO1Wyd/sPuqGnC6VRKjDaQ2F8HOt7XqnyUDhp7XBz+R/0ydLVUflQCmGG2YUH2FNl+86HWSrISHqqrsP9RAs8PF6YOTtY1uN1TugINfQV05w1qcNB2qRXcYXI4k9LrWhnFTLERZj7nFg85A5Q/l5NRUMsZxBOXbRm3f+EwYNRsMpo4DCXWKorVbHdqtVQUlWfUJSVbCo6reTrPDhcmgI81i1i7N7/gn1La2zSgK0dYUnLVxNLW4qMZMir5Zqwq1NGi3Yxqd3aqKpbgGk9NNZrwVdFEwcDzknqldVQtxzQ4XG3cfotnp4uJRmd7jI+OzjiYr0SckWQmPkuomALISotA1HYFtL2oJyGCCnMmQORrFFEvusEZizQYSY1oTjqNJS2zNtWC3eR7rVJUxKXDEFUV0ejakDgdj309R7C+dorC9ROtL1XKy23vhU08ju1wR7CuSrIRHRWv/qoHRTvj6ZS1RxaXCyB9DdKJnv3aDjE0x2i0+s90xdUBKbwbdi0wGHUa9gsOl0tTiar9Kc9sMDPY6MAdn3q3+JDSvGYugqLDZQXWTW/mB9gcYmwKjr/ZKVN3lcqshNSjZX9Em7f95Y8tx3RcMZohN1R7XHuzjqPonSVYC0NpnjjS0MMC2jYSWcq3qN/LHWompAyU1Tbz1TRmf7zvc4es/lNt4duN+NhdV92bYvS6mdXrjdskKICFHu68p7sOI+i9JVgLQemkPjnMwzL5dG2Iz9HyI6XxgcX2zk10VdezspDf7roo66pqdOF3uDl8PF23JqqmjZGXN1u5rivowov5LkpUAwBpj5PK47xmdFQuJuZB+Spf7D0qOQacoHGloobqhxeu1BruTosNaN4VhAZ6xoa/FeKqBHfTYT2hNVg2HtD5XoldJshIaWykc2o2i6GDo/51wYHGUUe9ZfLRtOE2bna3LaGVao0iMDdO+VK081cDjh9yA1rcstvXygbRb9bqQTlYrVqxgwoQJWCwW0tLSmDlzJjt37gx2WBHJsfcT7UH6yKN/gCfQNn3M9pJa3G6tMV1VVb4r1ZbRGhGkZbQCadygRG6ens+0Yakd7+CpCh7ou6D6qZBOVh9//DGLFi3i888/57333sPhcHDeeefR0CAr4gZS0+GDbNqyia3FtTizJ3f7fUPT44gy6qlrdrK/dan3A0caOVRnx2TQBW0ZrUCKMuoxG/SdL5jR1sheK8mqt4V0P6v169d7PS8oKCAtLY3Nmzdz1llnBSmqyFO35zMAqi1DMcQld/t9Rr2OkQPi2VRYzef7DjM4JdZzdfCUrPiATIMc8trareortTGPxqjgxhPBQrpkdbzaWq03cVIX09/a7XZsNpvXTXTBXoe97Dvt8YBxPr99Qm4SligDU/JTUBSFC0ZmclKmpd1cWOGqscXJ+zsqWP9tJ3Oumy1aPzRVlXarXhY2ycrtdnPrrbcyZcoURo4c2el+K1aswGq1em7Z2dl9GGUYKt1KY3MLdeYMrOmDfH57lFHPnMm55KZoa+hZo41cMDLTcxUtEmwvqeWH8jpPu1w7ibnaffX+PoupPwqbZLVo0SK+/fZbXnnllS73W7ZsGbW1tZ5bcbF02OuUywmlW2mwOym3nKINXvZDp1OpRIAogx5F0QpOHV4RBEhqnT/+yL6+C6wfCot/f4sXL+bf//43n3zyCQMHDuxyX7PZjNns3x9dv3N4N46meurUKI5E55IWL+fteDqdQqzJQL3dSYPdSVxHy9knDNIWkWg8Ak01EJ3Q12H2CyH9L1FVVRYvXszrr7/Ohx9+SF5e6K2AEtbKt9PQ4qQqdjgJsVHeA3WFR1zrMvZ1zZ1M5WyMOjoLg5Suek1IJ6tFixbxwgsv8NJLL2GxWCgvL6e8vJympqZghxb+7HVwZB9GnY6UIePCvqd5b2orTXU573zSYO1e2q16TUgnq9WrV1NbW8u0adPIzMz03P76178GO7TwV/EdqCqxaYOYNnYEZwwJ14lcel9byaq+s5IVHG23qi4EdydtW6JHQrrNKhKmGAlJqgrl27XHGaOCG0sYsJi7GB/o2SlTm1jQ0aR1YUj0/cqq6FpIJyvRS+rKoeEQLkXP4ag8kt3q0bnURTujBloZNdDadZueomgr9ZR/q013LMkq4EK6Gih6SWupqjomlxc3V/L8Z4XBjSfEmQ367l18SBmu3R/apZVeRUBJsupvXE6o1Hqsl0UNBSDFz/5V4jhJedqSZM212vAbEVCSrPqbw3u0MWxmC8WkAZBmkfFsXVFVlQ++r+Cf20rar858LL0RElsb2g/J7CCBJsmqv2lrWE8/hco6bdI8f3uu9xeKorC7sp59VQ2d97Vqk9paFaySZBVokqz6E3u9p9OiPfUUqhsdANJzvRssrd0XbM2OrndMHgI6vTZ7qFQFA0qSVX9SuQNUN8RnUeGIAyA+2hhRg457izVaWyOxtukEycoYrV0VBKj4tpej6l8kWfUn5d9o9xkjKattXdDUKu1V3ZEQrU3PXNt4gmQF2myrABU75KpgAMm/1P6irgLqq7QqStrJDG7WodMpJIf5HOl9pdslK4CkfG1dQXudtvJN2xQyokckWfUXbQ3rKUPBGE2qEVKlYb3bEmK0ZFXT2HKCPdG6L6SdBKXbtPMuySogpBrYH7hdnr5VpMvwGn/Et5asWlzu7g0Dyxyt3Vf+IMt0BYiUrPqDI/u0PxhTLCQNpry2mdomB1kJUViijMGOLizERxm4aVp+9+eVt2SCJUMb2lT+DeSc3rsB9gNSsuoPyr7W7tNPAZ2O70preXt7GVsO1AQ1rHCiKIpvC2AoCgw4TXtcsgXc4b0ydSiQZBXpmmu1XusAmWMAOFitXQkckCBXAntV2snaxHzH/g6E3yRZRbrSbdrl88RBEJtMvd3JkYYWFAUGJsYEO7qwUnS4gTe2lrBx96HuvUFvhKyx2uMDn0o3hh6SZBXJ3K6jVcAsrUpSfERr7E2zRPWPdf0CqMXpZv+hBoqrfWgwHzhBuzpoK5NZRHtIklUkq9oJLQ1aw3qKNsNCW7LKTooOZmRhKTlO6+pxuN7e+bJcxzPFQmZr6apISlc9IckqUqkqFH+uPc4aAzo9qqpyoC1ZSRXQZwnRRgw6BYdL7V7n0DbZE0FngJpiOLy39wKMcJKsIlX1fq3Xut4AA8YDUNPooK7ZiVGvkJUgJStf6XSKp3RVWWfv/huj4mGg9jtg74cyR7ufJFlFqgOtparMMWDSSlGJsSZuOGswl5yaFdELk/amzNaxlKW1Pq6wlDNZ+z00HobSrb0QWeSTb2wkqimG6iJt4c2BE7xeijMbPEu9C9+1lUhLa3xMVsYoyJ2qPd63QVsMVfhEklWkUVWtqgGQeapndWBZKSgwshKiMBl0RBv1vp/TrLFgHQguB+x8RxrbfSTJKtJU/QC2Uq2PT9t/cuCzfYd5dVMxBw7LOLWesEQZuensfGadNhBF8XFFIEWBERdrje3VhXDwq16JMVJJsookjibY8772OHsSmLUJ9txulR2lNg5WN9HU1Rziolt0PVm2LCYJ8s/RHu/9SEtaolskWUWSPR9oUxfHJHkNnN1TVU9ds5Moo578VGmvCpTaJod/1esBp0HGSG3W1u/e0KZAFickySpSlG/XbooCwy/SqoFobVVfFR4BYHS2FYNefuU9paoqr24q5rmN+ynyp1qtKDDsAm1WBkcTfP0yNB4JfKARRr65kaD2IOxcrz3OmQwJ2Z6Xfiivo9Jmx6hXGJudGKQAI4uiKJ6JC78+WOPfQfRGOPWnEJuilYa3Pq/9HkWnJFmFu5pi+Oav4HZqK6sc06je2OLkv7urAJiYl0y0ScYCBsqpAxMA2FfVQIWt2b+DmGJg9FUQl6bNN7btZSj+Sq4SdiIsktUf//hHcnNziYqKYtKkSXz55ZfBDin4VBVKNmtVCGcLJOTAyZeD7uiv9PuyOhrsLpJiTZyWkxC8WCNQUqyJkzLjAfjg+0qcLj/nqzLHwdjrtLGbbqd2gWTr89o/IeEl5JPVX//6V5YsWcJdd93Fli1bGD16NOeffz6Vlf10TTZV1caXbVkHu97Vhm6kDoNRs8HgvfjDaTkJnDMijUtOzZS2ql4wZUgyZqOOClsz//muwv+EZTDByB/DsPO16mFtCWx9QbuVfaOtoC1Q1BDvLThp0iQmTJjAH/7wBwDcbjfZ2dn84he/4I477jjh+202G1arldraWuLj43s73MByOcDRqLVpNFRqU+Qe3qutmgLaFzvvbNxZ46h3uKi02dl/qIGRA+LJtMrYv75QeKiBf24rxa2qjMlOYPqItJ4dsLlWm52h7BvtaiFoDfKWTIgfALHJEJ0EpjitGmmI0l4PU778fYb0HOwtLS1s3ryZZcuWebbpdDpmzJjBZ5991uF77HY7dvvRQaY2m617H1b0KVR81367Vy5XO9/mtd33bc0OJ9+V2jyvKW4niuoEVdsr1WL2zJTQohh570g65dbRtOyNxb5zj1dITpebzFGSrPpCbkosPxo7gHd3lDMmO8Gz/cv9R9hUdASzQY9OAQWtYV5pfXzRqEzPoOhvS2r5vszG7PHZEGWF4RfCoCna1d2K77TxhLZS7XY8RdE6mer02r2i1x5rL7bftzO9kfBMFhj904AdLqST1aFDh3C5XKSnp3ttT09P54cffujwPStWrOCee+7x/cPs9cHt7+Jw0dLU0G6zioJDH0NU9ADIzofEXBwxA9i18QC48Yzg1ykKibFGBiREMyIzzEqQYS4nOYb5U/K8Oos2O1zYHW7sjo6rhq5j/rs0trg40nDcEl9R8ZA7Rbs112pjPRsqoeEwNFWDo0Frq1RVrQTu8mHKmr4S1Y1ly3wQ0snKH8uWLWPJkiWe5zabjezs7C7e0WrAOK2R0+s/zDGPPds72nbM9u5uO+6YerebnJPtRzfrDWCIRjGYURSddiWvdTmoaLfKdZMHeY4UZdQTY9L7PvxDBMzxvdon5iUxcoCVFqcbt6qiovXPastRbYumAgxPt5DV1Xz4UVZtnOfxXE5wNmkN825X683ZWn3sqCSP9/Z2rwWYLrDpJaSTVUpKCnq9noqKCq/tFRUVZGRkdPges9mM2ezH4p2xydotSIzAgG4WiHQ6hZQ4WaA0lEUZ9d2eNtoaY8Qa48eSaHoD6C2+vy9MhfQlIpPJxLhx4/jggw8829xuNx988AGTJ08OYmRCiL4W0iUrgCVLljB37lzGjx/PxIkTWbVqFQ0NDVx//fXBDk0I0YdCPln99Kc/paqqiuXLl1NeXs6YMWNYv359u0Z3IURkC/l+Vj0V1v2shIhwvvx9hnSblRBCtJFkJYQIC5KshBBhIeQb2HuqrUmu28NuhBB9pu3vsjtN5xGfrOrqtEG/3erFLoQIirq6OqxWa5f7RPzVQLfbTWlpKRaLJajDUdqG/RQXF8tVyePIuelcpJ8bVVWpq6sjKysLna7rVqmIL1npdDoGDhwY7DA84uPjI/JLFwhybjoXyefmRCWqNtLALoQIC5KshBBhQZJVHzGbzdx1113+zQgR4eTcdE7OzVER38AuhIgMUrISQoQFSVZCiLAgyUoIERYkWQkhwoIkqwDyZeXop59+mqlTp5KYmEhiYiIzZsyI6JWm/V1V+5VXXkFRFGbOnNm7AQaRr+empqaGRYsWkZmZidlsZtiwYbz99tt9FG0QqSIgXnnlFdVkMqnPPfec+t1336k33HCDmpCQoFZUVHS4/9VXX63+8Y9/VLdu3ap+//336rx581Sr1aoePHiwjyPvfb6emzb79+9XBwwYoE6dOlW9/PLL+ybYPubrubHb7er48ePViy66SN24caO6f/9+dcOGDeq2bdv6OPK+J8kqQCZOnKguWrTI89zlcqlZWVnqihUruvV+p9OpWiwWde3atb0VYtD4c26cTqd6xhlnqM8884w6d+7ciE1Wvp6b1atXq4MHD1ZbWlr6KsSQIdXAAGhbOXrGjBmebSdaOfp4jY2NOBwOkpKSeivMoPD33Nx7772kpaXxs5/9rC/CDAp/zs2bb77J5MmTWbRoEenp6YwcOZIHH3wQl8vVV2EHTcQPZO4L/qwcfbzbb7+drKwsry9uJPDn3GzcuJFnn32Wbdu29UGEwePPudm3bx8ffvgh11xzDW+//TZ79uzh5ptvxuFwcNddd/VF2EEjySoEPPTQQ7zyyits2LCBqKguVubtB+rq6rjuuut4+umnSUlJCXY4IcftdpOWlsZTTz2FXq9n3LhxlJSU8Lvf/U6SlTgxf1aObvPoo4/y0EMP8f7773PqqR0sER7mfD03e/fupbCwkEsvvdSzze12A2AwGNi5cyf5+fm9G3Qf8ed7k5mZidFoRK8/utrzSSedRHl5OS0tLZhMpl6NOZikzSoA/F05+pFHHuG+++5j/fr1jB8/vi9C7XO+npsRI0awfft2tm3b5rlddtllTJ8+nW3btkXUjK/+fG+mTJnCnj17PAkcYNeuXWRmZkZ0ogKk60KgvPLKK6rZbFYLCgrUHTt2qAsXLlQTEhLU8vJyVVVV9brrrlPvuOMOz/4PPfSQajKZ1L///e9qWVmZ51ZXVxesH6HX+HpujhfJVwN9PTcHDhxQLRaLunjxYnXnzp3qv//9bzUtLU29//77g/Uj9BlJVgH05JNPqjk5OarJZFInTpyofv75557Xzj77bHXu3Lme54MGDVKBdre77rqr7wPvA76cm+NFcrJSVd/PzaeffqpOmjRJNZvN6uDBg9UHHnhAdTqdfRx135MpYoQQYUHarIQQYUGSlRAiLEiyEkKEBUlWQoiwIMlKCBEWJFkJIcKCJCshRFiQZCWECAuSrEREKSgoICEhwfP87rvvZsyYMZ7n8+bNi+gpkiOZJCvhl3nz5qEoCj//+c/bvbZo0SIURWHevHle+wc6SeTm5rJq1SqvbT/96U/ZtWtXp+95/PHHKSgo8DyfNm0at956a0DjEr1DkpXwW3Z2Nq+88gpNTU2ebc3Nzbz00kvk5OQEJabo6GjS0tI6fd1qtXqVvET4kGQl/HbaaaeRnZ3Na6+95tn22muvkZOTw9ixY3t07I5KPDNnzvSU1qZNm0ZRURG33XYbiqKgKArQvhp4vGNLePPmzePjjz/m8ccf9xxj//79DBkyhEcffdTrfdu2bUNRFPbs2dOjn0v4T5KV6JH58+ezZs0az/PnnnuO66+/vtc/97XXXmPgwIHce++9lJWVUVZW5vMxHn/8cSZPnswNN9zgOUZOTk67nwlgzZo1nHXWWQwZMiRQP4LwkSQr0SPXXnstGzdupKioiKKiIv73v/9x7bXX9vrnJiUlodfrsVgsZGRknHBG1o5YrVZMJhMxMTGeY+j1eubNm8fOnTs96/c5HA5eeukl5s+fH+gfQ/hApjUWPZKamsrFF19MQUEBqqpy8cUXh/3c6VlZWVx88cU899xzTJw4kX/961/Y7XZmz54d7ND6NSlZiR6bP38+BQUFrF27NmClD51Ox/FTrTkcjoAcuzsWLFjguXiwZs0afvrTnxITE9Nnny/ak2QleuyCCy6gpaUFh8PB+eefH5BjpqamerVDuVwuvv32W699TCZTj9fL6+wYF110EbGxsaxevZr169dLFTAESDVQ9Jher+f777/3PO5MbW1tu7UAk5OTO1wE4pxzzmHJkiW89dZb5Ofns3LlSmpqarz2yc3N5ZNPPuHKK6/EbDb7Vf3Mzc3liy++oLCwkLi4OJKSktDpdJ62q2XLljF06NAuF/4QfUNKViIg4uPjiY+P73KfDRs2MHbsWK/bPffc0+G+8+fPZ+7cucyZM4ezzz6bwYMHM336dK997r33XgoLC8nPzyc1NdWvuJcuXYper+fkk08mNTWVAwcOeF772c9+RktLS59c3RQnJnOwC9GJ//73v5x77rkUFxe3WzVZ9D1JVkIcx263U1VVxdy5c8nIyODFF18MdkgCqQYK0c7LL7/MoEGDqKmp4ZFHHgl2OKKVlKyEEGFBSlZCiLAgyUoIERYkWQkhwoIkKyFEWJBkJYQIC5KshBBhQZKVECIsSLISQoSF/w9GUmdrC56k9QAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:13:58.342901Z", + "iopub.status.busy": "2024-02-29T18:13:58.341723Z", + "iopub.status.idle": "2024-02-29T18:13:58.746709Z", + "shell.execute_reply": "2024-02-29T18:13:58.745745Z" + }, + "papermill": { + "duration": 0.430571, + "end_time": "2024-02-29T18:13:58.749053", + "exception": false, + "start_time": "2024-02-29T18:13:58.318482", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASsAAAEmCAYAAAA6OrZqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/sElEQVR4nO3deXhU5d34//eZPctksm8kIWGVfRGxiAuoFUWt1D4urVoiblVsH+VHK/RbsWoVbK0PVL3so62AjwtdXLqoqEXFglV2RWUnJIHskGSyznp+f5xkIGQhM5lklnxe1zXXTM6cc+aTycwn932fe1FUVVURQogwpwt1AEII0RuSrIQQEUGSlRAiIkiyEkJEBElWQoiIIMlKCBERJFkJISKCJCshREQwhDqA/ub1eikrK8NqtaIoSqjDEUKcQlVVGhoayM7ORqfruewU9cmqrKyM3NzcUIchhOhBaWkpOTk5Pe4T9cnKarUC2puRkJAQ4miEEKey2+3k5ub6vqc9ifpk1V71S0hIkGQlRJjqTRONNLALISKCJCshRESQZCWEiAhR32Yl+k5VVdxuNx6PJ9ShiAhkNBrR6/V9Po8kK9Ejp9NJeXk5zc3NoQ5FRChFUcjJySE+Pr5P55FkJbrl9XopKipCr9eTnZ2NyWSSjrXCL6qqUl1dzdGjRxk5cmSfSliSrES3nE4nXq+X3NxcYmNjQx1On7g9XhocbnSKgtViQCdJd8CkpaVx5MgRXC6XJCvRv840DCLcqapKfasLt0dbbkCngNViDHFUg0ewSuOR/SkUohdcHtWXqLSfvcg6KZFHSlYi6jnc2lVMi0FHnMWAIcJLioOV/NVE1HO6vQCYDPo+JaojR46gKAq7du3qUzy9Pc/mzZuZMGECRqORefPm9ek1o4EkKxHVvKqK26tV+UyG4H7cP/74YxRFoa6uLqjnbbdo0SImT55MUVERa9as6ZfXiCSSrERUU4DUeBOxehW9TsHj9VLb5OR4oyPUoZ3RoUOHuPjii8nJySExMTGgczidzuAG1YP2zsP9RZKVCIjT7e325vZ4e72vq5f7+mvWrFnce++93H///WSkp/Odq+by1VdfceWVV5KTkcxZw/K4+eZbqKmp8R2zfv16zj//fBITE0lJSeGqq67i0KFDXZ7/yJEjzJ49G4CkpCQURaGwsNCv8+zdu5fzzjsPi8XC+PHj2bhxo+/ciqJw/PhxFixYgKIovpLVxo0bmT59OmazmaysLJYsWdIhQbT/3vfddx+pqanMmTPHVwJ87733mDJlCjExMVx88cVUVVXx7rvvMmbMGBISEvjBD37QofOv1+tl+fLlFBQUEBMTw6RJk/jrX//qe779vO+++y5nn302ZrOZTZs2+f236i1pYBcBefajg90+V5Aax7wpQ3w/P//JIVyerq++5STFcN20k5Mjvri5iBZn52E99397lN8xrl27lrvvvpvNmzdTV1fHxRdfzO23385Dv3qCpuZmVjzyINdffz0ffvghAE1NTSxatIiJEyfS2NjIsmXL+O53v8uuXbs6dd/Izc3l9ddf53vf+x779u0jISGBmJgYv87z05/+lJUrVzJ27Fieeuoprr76aoqKisjNzaW8vJzRo0fzyCOPcMMNN2Cz2Th27Bhz586lsLCQl156ib1793LHHXdgsVj45S9/2eXvDVBeXg7AL3/5S5555hliY2O5/vrruf766zGbzbz66qs0Njby3e9+l6effpoHHngAgOXLl/Pyyy/z+9//npEjR/LJJ59w8803k5aWxkUXXeR7vSVLlvDkk08ybNgwkpKS/P479ZoaQhs3blSvuuoqNSsrSwXUN998s8PzXq9XffDBB9XMzEzVYrGol1xyibp//36/XqO+vl4F1Pr6+iBGPji0tLSo33zzjdrS0tLpuafe39ft7c0dRzvs+/SG/d3u++etJR32fe7jg13u56+LLrpInTJlitrQ4lQbWpzqw488ol522WWqqqrqiUaHWlHfou4/VKQC6r59XZ+/urpaBdTdu3erqqqqRUXa/jt37lRVVVU/+ugjFVBra2t7jKW786xYscK3j8vlUnNyctQnnnjCt81ms6mrV6/2/fzzn/9cHT16tOr1en3bnn32WTU+Pl71eDwdfu9Ttcf5r3/9y7dt+fLlKqAeOnTIt+2uu+5S58yZo6qqqra2tqqxsbHqp59+2uFct912m/r973+/w3nfeuutHn//nj5H/nw/Q1qyampqYtKkSSxYsIBrr7220/O//vWv+d3vfsfatWspKCjgwQcfZM6cOXzzzTdYLJYQRCzaLZw9otvndKf1AbzzwuHd7nt6f8EFMwv6ElYHU6dOpdnpQQW+/OILPvroI9/4NBWtPQu0tqFRo0Zx4MABli1bxueff05NTQ1er1b9LCkpYfz48b1+3d6eZ8aMGb7HBoOBadOmsWfPnm7Pu2fPHmbMmNGhk+XMmTNpbGzk6NGj5OXlAXD22Wd3efzEiRN9jzMyMoiNjWXYsGEdtm3ZsgWAgwcP0tzczLe//e0O53A6nUyZMqXDtmnTpnUbczCFNFldccUVXHHFFV0+p6oqK1eu5Be/+AXXXHMNAC+99BIZGRm89dZb3HjjjQMZqjiNP1fW+mvfM4mLi6O98tnU1MTVV1/NE088QbPTTbPDg9mgYI0xkZWVBcDVV1/N0KFDeeGFF8jOzsbr9TJ+/Hi/G6mDdZ5AxcXFdbndaDzZa19RlA4/t29rT6yNjY0AvP322wwZMqTDfmazuVevF2xh22ZVVFRERUUFl156qW+bzWbj3HPP5T//+U+3ycrhcOBwnLzSY7fb+z1WEZ7aeiyg1ylMnTqV119/nfz8fNyqQn2LC4NOISVe++IdP36cffv28cILL3DBBRcAnLGx2GQyAXSYOsef83z22WdceOGFALjdbrZv3869997b7euNGTOG119/HVVVfaWrzZs3Y7Vaz7jYgr/Gjh2L2WympKSkQ/tUKIXt1cCKigpAK5qeKiMjw/dcV5YvX47NZvPdZGWbwcvbNqRGr1NYuHAhJ06c4Pvf/z47t2+juOgwH234gFtvvRWPx0NSUhIpKSk8//zzHDx4kA8//JBFixb1eP6hQ4eiKAr//Oc/qa6uprGx0a/zPPvss7z55pvs3buXhQsXUltby4IFC7p9vXvuuYfS0lJ+/OMfs3fvXv72t7/x0EMPsWjRoqCP37RarSxevJj777+ftWvXcujQIXbs2MHTTz/N2rVrg/pavRW2ySpQS5cupb6+3ncrLS0NdUgiRNqH/xl0CtnZ2WzevBmPx8PcKy5n1oxpPLjkpyQmJqLT6dDpdKxbt47t27czfvx47r//fn7zm9/0eP4hQ4bw8MMPs2TJEjIyMrj33nv9Os+KFStYsWIFkyZNYtOmTfz9738nNTW1x9d755132LJlC5MmTeJHP/oRt912G7/4xS8Cfo968uijj/Lggw+yfPlyxowZw+WXX87bb79NQUHw2hX9oahqeIzoVBSFN9980zes4PDhwwwfPpydO3cyefJk334XXXQRkydPZtWqVb06r91ux2azUV9fL6vb+Km1tZWioiIKCgoi8oJGXbMTh9uL1WIg1hS2LR5Rr6fPkT/fz7AtWRUUFJCZmcmGDRt82+x2O59//nmHqyhCdMfT1mill7mrokJI/900NjZy8ODJzoVFRUXs2rWL5ORk8vLyuO+++/jVr37FyJEjfV0XsrOzZVCn6JXkOBMer4ru9L4UgL3VhdPlJd5iwGLs+/zgov+FNFlt27bNN2QB8DVEzp8/nzVr1vCzn/2MpqYm7rzzTurq6jj//PNZv359RFZJxMBTFAWDvutSlaqqeFTVV/oS4S9s2qz6i7RZBS7S26x60tDqotnpIcakJ0FmDe1XwWqzklZHEZWcbi8tLg9GvdJl43p7O5ZXSlYRI2wb2IXoC7fXS6vL0+2MDe3tWN7orlhEFUlWIir5rgR20bgO+Fa38fg/+4wIEUlWIiq1J6vultxqz2FR3mQbVSRZiah06lCbruh0CjpFQa9TJGFFCElWIipdffm3eXDJ4h6rgWlWMynx5gFZZXrWrFncd999/f460UySlYg6qqr6pobpy8rLAzl/uTgzSVYi6hQWFvKfTf/mheeexaDXoSgKhw4d4rbbbvPNJz569OhO40sLCwuZN28ejz32GNnZ2YwePRqATz/9lMmTJ2OxWJg2bRpvvfVWp6W0vvrqK6644gri4+PJyMjglltOzu9eWFjIxo0bWbVqFYqioCgKR44cGai3I2pIPyvhH1UFjys0r603dp5atAu/+93vOHDgAOPGjePRRx8FtEUdcnJy+Mtf/kJKSgqffvopd955J3GJqdz0/RuJMWlDbjZs2EBCQgIffPABoHVavPrqq5k7dy6vvvoqxcXFnapzp87v/j//8z+0tLTwwAMP+OZ3X7VqFfv372f8+PE88sgjAKSlpQXxjRkcJFkJ/3hc8O/fhua1L/j/wGA64242mw2TyURcXByZmZm+7Q8//LDvcUFBARv/vZm33vgr37/xBt/2uLg4/vCHP/gm1vv973+Poii88MILWCwWxo4dy7Fjx7jjjjt8xzzzzDNMmTKFxx9/3LftxRdfJDc3l/379zNq1ChMJhOxsbEd4hH+kWQlBo1nn32WF198kZKSElpaWnA6nYybMLFDx9AJEyb4EhXAvn37mDhxYodhItOnT+9w3i9Om9/9VO3zu4u+k2Ql/KM3aiWcUL12L7Q4Pbi9J1diBli3bh2LFy/mt7/9LTNmzMBqtfL4iifYsmULp464CWQ+8cbGRt/87qdrn99d9J0kK+EfRelVVSyUXB4vBoOxw+Kfmzdv5rzzzuOee+7xbTty+DDQ85Cb0aNH8/LLL+NwOHwLJWzdurXDPqfO724wdP2VMplMHeZqF/6Tq4Ei6nhVldy8oWzbuoUjR45QU1PDyJEj2bZtG++99x779+/nwQcfZPv2bdr+PQxm/sEPfoDX6+XOO+9kz549vPfeezz55JMAvv5Zp87vvnXrVg4dOsR7773nm98dID8/n88//9wXT/sqMqL3JFmJqOPxqtz9k/sw6PWMHTuWtLQ05syZw7XXXssNN9zAueeey/Hjx7nrR3cD0NPECwkJCfzjH/9g165dTJ48mf/3//4fy5YtA/C1Y506v/tll13GhAkTuO+++3zzuwMsXrwY/SnxlJSU9O+bEIVkPivRrUidz6q6oRWvqs0UatR3///Y7fFS1+xCp1NIjut91faVV17h1ltvpb6+3rdkvOiezGclRBdUVfWVlM7Ue92g15FqNfe4D2iL6w4bNowhQ4bwxRdf+PpQSaIaWJKsRFQ5tUrXzbBAv1VUVLBs2TIqKirIysriuuuu47HHHgvOyUWvSbISUaX9yp6ubVhLMPzsZz/jZz/7WVDOJQInyUpEFaNeR7rV3GOj+anqmp24PSoJMUZMBrneFM4kWYmooygK3Sxq04mnbZUbmd44/Mm/EnFG0XzBuL0RPpp/x1AL1nsryUp0y2jUhrc0NzeHOJLea3K4qW9x4XD3rrd4+xdAFrnpP+3zgun1fVtMVqqBolt6vZ7ExESqqqoAiI2NHZBZNfuiqcWJ06PiNelRu1iC63QupwuX20ur6kKvyvqBweb1eqmuriY2NrbboUi9JclK9Kh9SpP2hBXumhxuPF6VWJMeQw8dQts5XB4cbi8mg06Wke8nOp2OvLy8Pv+jk2QleqQoCllZWaSnp+NyhWjSPT/8eWspzU43V03KJjX+zB0+vy6rZ2fRCQrS4rioIH0AIhx8TCaTb9hRX0iyEr2i1+v73ObQ37xelRMOFVXVk2iNw2I+88fbGufCZG7GZLJE1JCiwUiSlYgaLS4PqqrNYhPbyyrdqAwrozKs/RyZCAa5GiiiRpNDm78qzmTwLQ8vokdYJyuPx8ODDz7oW5Fk+PDhPProo9InRnSpxaV1V4g1h3d1VQQmrKuBTzzxBM899xxr165l3LhxbNu2jVtvvRWbzcZPfvKTUIcnwszQlDh+cslInO7eT2zncHt4ffsxWl0eCs/LlxJZGAvrZPXpp59yzTXXcOWVVwLabIuvvfYaW7ZsCXFkIlzpdYpvWa3eMOp0VNpbAXC4vX4dKwZWWFcDzzvvPDZs2MD+/fsBbRWRTZs2ccUVV3R7jMPhwG63d7gJ0R2dTvENYG51yRzp4SysS1ZLlizBbrdz1llnodfr8Xg8PPbYY9x0003dHrN8+fIO68OJwePTQzXYW1xMyk0ky9b7ifHMBh1OtxeHH9VHMfDCumT15z//mVdeeYVXX32VHTt2sHbtWp588knWrl3b7TFLly6lvr7edystLR3AiEUoFR9vZk95A81O/0pI5rZuDr0dTyhCI6xLVj/96U9ZsmQJN954I6AtQFlcXMzy5cuZP39+l8eYzWbfkklicDm164I/zG3VQClZhbewLlk1Nzd36qav1+tlGSPRiaqqNDm0klGcn10X2pOVP1cRxcAL65LV1VdfzWOPPUZeXh7jxo1j586dPPXUUyxYsCDUoYkw0+Ly4FVVFMX/kpXVYsAWYzzjAhMitMI6WT399NM8+OCD3HPPPVRVVZGdnc1dd93lW7dNiHaNbVXAWJPe775SF5+V0R8hiSAL62RltVpZuXIlK1euDHUoIsy1VwFj/SxVicgR1m1WQvRWex+p+F7MtCAik/xlRVQYk5XAqAwrLo//jeSHqxv57PAJ0q1mLh0rVcJwJclKRA29TkGv83+4jNurUmlvxdDbJXFESEg1UAx60s8qMkjJSkSFj/ZW4XB7mJaf3KvpjE9lNrT1YJexgWFNSlYiKhyqbmRPeUNAbVZSsooMkqxExOvYe93/yoLZqH0NXB6vTOwYxiRZiYjX3nsd/O+9DmBqW7JLVaV0Fc6kzUpEvFN7r+sDmOnToNdhtRjQKQpuWZo5bEmyEhGvub33eh86hN5+wbBghSP6iVQDRcRrL1nFy0IRUU2SlYh47e1MgbRXicghf10R8c4emsTk3ETcfZjnbNOBGkpONDO9IIkR6bLoaTiSZCWiQqBDbdrVt7iotLfS0OoOYlQimKQaKATSMTQSSMlKRLx3dpejU+C8EakkWIwBnaO9Y6gkq/AlJSsR0VRV5WCVNtSmL2R8YPiTkpWIaK0uLx5vD73XvV6o3gMN5RCbCpkToIu2LakGhj9JViKitfexiumq97rbAV+9DrXFJ7eV7YRJN4Kx4yKoJlnhJuxJNVBENN9agaf3XldV+ObvWqLSGyF7spagGipg79va86ewGPVYjHqZgC+MSclKRLQmZze918t2wvGDoDPA5B9AQjZkT4EdL0HNAajeC+ljfLsXpMZx96zhAxm68JOUrERE800Nc2p7lasVij7RHg+bpSUqAGsm5M3QHh/ZpLVniYghyUpENJfHqy1semo1sHgzuFogLhWGnN3xgNzpYLRAUw2cODSwwYo+CShZHT58ONhxCBGQmSNS+fHFIzknP1nb4GrRqoAAw2aD7rSPuMEMmRO1x2W7fJvdHi+vbz/Kq5+XBDTbqOh/ASWrESNGMHv2bF5++WVaW1uDHZMQftHrFN/VPMp2gscF8WmQ0k0bVNZk7f7EIWi1+85RWttMpb1Vui+EqYCS1Y4dO5g4cSKLFi0iMzOTu+66iy1btgQ7NiH843HD0W3a49xzQenmyl5cCthytCuC1fsAUBRFOoaGuYCS1eTJk1m1ahVlZWW8+OKLlJeXc/755zN+/Hieeuopqqurgx2nEJ14vCp/2VbK+q/KtapbzX5wNoHZCuljez64/Upg9V7fpvaOoU6pBoalPjWwGwwGrr32Wv7yl7/wxBNPcPDgQRYvXkxubi4//OEPKS8vD1acQnTS2OrmaG0L+ysbMegUKP9CeyJrYpe91DtIHand24+BoxE42THU4ZJkFY76lKy2bdvGPffcQ1ZWFk899RSLFy/m0KFDfPDBB5SVlXHNNdf0OcBjx45x8803k5KSQkxMDBMmTGDbtm19Pq+IfA0OFwDxZgNKax3UHtGqfu0N6D2x2CAhS6sKHj8IyJCbcBdQp9CnnnqK1atXs2/fPubOnctLL73E3Llz0bVdeSkoKGDNmjXk5+f3Kbja2lpmzpzJ7Nmzeffdd0lLS+PAgQMkJSX16bwiOvimM7YYoPxLbWNSPsQk9u4EycPBXg61RZA9GbOxrc3KLW1W4SigZPXcc8+xYMECCgsLycrK6nKf9PR0/vjHP/YpuCeeeILc3FxWr17t21ZQUNCnc4ro0dg2UV6CWQcVu7WNWZN6f4LkAq1zaO0R8HqxGHRYjDKPe7gKKFl98MEH5OXl+UpS7VRVpbS0lLy8PEwmE/Pnz+9TcH//+9+ZM2cO1113HRs3bmTIkCHcc8893HHHHd0e43A4cDgcvp/tdnufYhDhq6GtZJXiqgRHg9bZM2Vk709gzdb6XblaobGCy8Zl91OkIhgCarMaPnw4NTU1nbafOHEiqCWfw4cP89xzzzFy5Ejee+897r77bn7yk5+wdu3abo9Zvnw5NpvNd8vNzQ1aPCK8tJesUprbeqKnnQV6P/7/6nSQNFR7fEI6Ooe7gJJVd0tsNzY2YrFY+hTQqbxeL1OnTuXxxx9nypQp3Hnnndxxxx38/ve/7/aYpUuXUl9f77uVlpYGLR4RXjxeFR0eEhrbEs0pA5N7LSlfu6+Tz0m486sauGjRIkDrQLds2TJiY2N9z3k8Hj7//HMmT54ctOCysrIYO7Zjf5kxY8bw+uuvd3uM2WzGbDYHLQYRvuZNGYKnuhlltwdM8WDL8/8k7cfYj1JS3cDWknpS4k3MGp0e3GBFn/mVrHbu1MZcqarK7t27MZlMvudMJhOTJk1i8eLFQQtu5syZ7Nu3r8O2/fv3M3To0KC9hohs+uo9WneF9DGdxwH2Rlyq1tblasVtL6fkBL6ZR0V48StZffTRRwDceuutrFq1ioSEhH4Jqt3999/Peeedx+OPP87111/Pli1beP7553n++ef79XVFhPC4tF7rcOYe691RFLDlQs0BYlvKgSzpuhCmAmqzWr16db8nKoBzzjmHN998k9dee43x48fz6KOPsnLlSm666aZ+f20R3irtraz/ZDNHquraOnj24UqeLQcAS3MZIJ1Cw1WvS1bXXnsta9asISEhgWuvvbbHfd94440+B9buqquu4qqrrgra+UR0qG124qrYS5PXDWmjux+03Bs27YqxqbEM1KmSrMJUr5OVzWZDaftA2Gy2fgtIiN6wNzlIai3BHKeH1FF9O5k1E/QGjM5WYtx1tChJeL0qutMXoBAh1etkdWov8lMfCxEKruNH0HudGC0pkDCkbyfT6cGajb62GKujkhZjEk6PF0sflqMXwRdQm1VLSwvNzc2+n4uLi1m5ciXvv/9+0AITokfHDwCgpI4I7Crg6RKy0SkKie4aLEa9zBYahgIabnPNNddw7bXX8qMf/Yi6ujqmT5+OyWSipqaGp556irvvvjvYcQpxkqpirD2IBzBmjA7OOdsa6OcO9aJMl1VuwlHAM4VecMEFAPz1r38lMzOT4uJiXnrpJX73u98FNUAhTqc2VKC22vEqBmIzRgTnpG3JSmmuAbczOOcUQRVQsmpubsZqtQLw/vvvc+2116LT6fjWt75FcXHxGY4Wom+cFfsw6nXYY3Kwxsac+YDeMFu1m6pCY0VwzimCKuAFI9566y1KS0t57733uOyyywCoqqoakP5XYnAz24uYmpfEZReeH9wrdglZHK1t5qOtX3C4ujF45xVBEVCyWrZsGYsXLyY/P59zzz2XGTO0hSPff/99pkyZEtQAhejA1QKNlQDoU4YF99zWbJqdHpprSqhvcQX33KLPAmpg/6//+i/OP/98ysvLmTTp5GRnl1xyCd/97neDFpwQndSVaFW12BSwBLkUn5CNQacQ11wtHUPDUEDJCiAzM5PMzMwO26ZPn97ngIToUW0xJSeaKW7OJauqkRHp8cE7tzULnV6H2dOIu8UOpATv3KLPAkpWTU1NrFixgg0bNlBVVYXX2/G/kKzYLPpNXTGNDjeHDamkBLv0YzChxqRA3VGUhnJAptAOJwElq9tvv52NGzdyyy23kJWV5RuGI0S/cjRAUw2tbi92czYJMQFXDLqlJmRD+VF0ckUw7AT013733Xd5++23mTlzZrDjEaJ7tcV4VZVaJQm33kJirOnMx/grXmva0DdVBf/cok8CuhqYlJREcnJysGMRomd1xTjcXurM2ZgMOuJMwR+7p7NlogCW1iqtIV+EjYCS1aOPPsqyZcs6jA8Uol+pKtQW0+ryYLdkY4sx9kvzQ15OPucOT2ViulGrdoqwEVA18Le//S2HDh0iIyOD/Px8jEZjh+d37NgRlOCE8Gmtg9Z6WtwqdksWw2ONZzwkEDqjSZvquLFa688V7O4RImABJat58+YFOQwhzqD2CACuuCzizDEk9Ud7Vbv4TC1ZNZRDqh/rEIp+FVCyeuihh4IdhxA9q9XGnA4dPobbC4Z1uxxcX7k9Xj6rNJBcbmd0Ujkyo1X4CHgioLq6Ov7whz+wdOlSTpw4AWjVv2PHjgUtOCEArb2qrm2AfNs6f/3VXUavU9jbFEt9iwu3vbxfXkMEJqCS1Zdffsmll16KzWbjyJEj3HHHHSQnJ/PGG29QUlLCSy+9FOw4xWDWVA3OZm21ZWv/LvGuKArumDRAwdvaqDWym639+pqidwIqWS1atIjCwkIOHDjQYQXmuXPn8sknnwQtOCEAXxWw0ZLFHz4t4W+7+rf0bjCaaTYmausHNlT262uJ3gsoWW3dupW77rqr0/YhQ4ZQUSE9f0WQtVUB683ZNLS6aXS4+/XlzEY9TaZU3F6v1sguwkJAycpsNmO32ztt379/P2lpaX0OSggfr9eXrKqNWQAk9+eVQMBs0NFkStVKVo1SsgoXASWr73znOzzyyCO4XNqcP4qiUFJSwgMPPMD3vve9oAYoBrmGcm2aYYOZCo+2BFxy3EAkqzTcHhUapKYQLgJKVr/97W9pbGwkLS2NlpYWLrroIkaMGIHVauWxxx4LdoxiMGu/CpiYx/Fm7Z9jSry5X1/SbNDRbErBC1oDu0NmDQ0HAV0NtNlsfPDBB2zevJkvvviCxsZGpk6dyqWXXhrs+MRg19a4riblU1utLeSQ0s8lq0vHZDBnXCbK1hxoqtGqguYgzpslAuJ3svJ6vaxZs4Y33niDI0eOoCgKBQUFZGZmoqqqTBcjgsfjhvqjANgt2bg8jRh0CraY/hlq086gb6twxGdoyaqhHFJkea5Q86saqKoq3/nOd7j99ts5duwYEyZMYNy4cRQXF1NYWChTGovgsh8FrxtMcThNyeQkxZCdGDNwy7pb22bClXarsOBXyWrNmjV88sknbNiwgdmzZ3d47sMPP2TevHm89NJL/PCHPwxqkGKQqm3vtT6UtAQL103LHZCXPVrbzK7SOrJUM2eDXBEME36VrF577TV+/vOfd0pUABdffDFLlizhlVdeCVpwp1uxYgWKonDffff122uIMHLaEJuB0uz0cKCykaLWeFAUaLWDs2lAYxCd+ZWsvvzySy6//PJun7/iiiv44osv+hxUV7Zu3cr//u//MnHixH45vwgzbge0j81LHIpzAFebMRu0r0WraoCYtkkmpSoYcn4lqxMnTpCRkdHt8xkZGdTW1vY5qNM1NjZy00038cILL5CUlBT084swVFcKqhdiEnGZEnju40O8uKmIVpen31/a1JasHG4vWNs+75KsQs6vZOXxeDAYum/m0uv1uN3BHwqxcOFCrrzyyl51jXA4HNjt9g43EYHqjmj3Sfkcb3TiVVVcHq+v1NOfzAZtYhiH2wNWrde8DLsJPb8a2FVVpbCwELO56055DocjKEGdat26dezYsYOtW7f2av/ly5fz8MMPBz0OMcDaG9cTh1LV0ApAeoJ5QLrGtCdEp9uLGp+OAtLIHgb8Slbz588/4z7BvBJYWlrKf//3f/PBBx90mN2hJ0uXLmXRokW+n+12O7m5A3MVSQSJswka21aXScyj8pDWuJ1u7d1noK/ak5WqgjMmHfOpjeymuAGJQXTmV7JavXp1f8XRpe3bt1NVVcXUqVN92zweD5988gnPPPMMDocDvb7jXI5ms7nbkp+IEHUl2n1cKpjjqWo4DkBGwsD8XQ16HXqdgldVcWDEHJMMzce1divpHBoywV8lMoguueQSdu/e3WHbrbfeyllnncUDDzzQKVGJKNE23zpJBbg9Xo43asNs0gaoZAVw10XDMOl1WrXTmiHJKgyEdbKyWq2MHz++w7a4uDhSUlI6bRdRxJes8qlscODxqsSa9CRYBu7j2t7IDmiN7JXfSCN7iIV1shKDUPMJaKkDRQeJuZhbYcIQG0aDLnTjTuPbui9II3tIRVyy+vjjj0MdguhP7aUq2xAwmEmNh0vHdt+3r79sL66lvL6FSTmJ5CZkduzJLo3sIdH/nVaE8EdtkXafVBDSMMrrWzhQ2UhNowMMZunJHgYkWYnw4fWeMng5H3uri2N1LXi9/bNGYE9ijFqbVaurbZiP9GQPOUlWInw0lGtjAg1msGaxp8zOn7eWsv7rgU8QlvZk5W4b3iM92UNOkpUIHycOa/dJQ0Gn48hxrTNoTlLMgIdiMbaND2wfiyiN7CEnyUqEj+MHtPuUEbQ4PZTXa8Ns8lMHvkG7vevCyWpgpkwXE2KSrER4aK3XFhRVFEgZwf7KBlQV0qxmEiz9O41xV9qrgS3tJStpZA85SVYiPNQc1O4ThoApjm/KtdkyxmQlhCScGFPbzAunTkkjjewhFXH9rESUaq8Cpo6k0t5KRX0rOkXhrExrSMLJTLBw96zhHaekkZ7sISXJSoSes/lkl4XUURwq09bpG50ZT5w5NB9RvU5Brztt7Kk0soeUJCtxRsFeYq3S3sqXR+tpdrpJt1oYpx4kQfVqjdixyZw3AnKTY7EO4FjAXjm9kV16sg+oMPs0iHBS1+zknd0VVDc4GJURz6VjMzDq+9bMuafczvtfV+JVtY6eh6ubaKj8hGzqMZ51DqPb9stNju1j9H338b4qGh1uLhyVpjXytzeyywwMISEN7KJLbo+Xv39RRqW9Fa+qcqyuBXuLq8/nrW12oqIyPD2eWaPTGJXgJMFRSYPDw3/sGajqwPdW786h6iYOVDbS5Dhlqm5pZA8ZKVmJLu0+Vs/xRidxZj2zR6eTmxzru5zfF+cNT2VEejxp8doUxVMc23AOTaLSMARD/hA8XhWDPjxW9bYYddhbTulrBdLIHkKSrEQnXq/KzpI6AL41LIWRGcG9IuebntjtgIovMel15E68kNzk5KC+Tl9ZfB1DT+2+0Dbsxl6mzXscqmlrBiGpBopOjtW1UN/iwmzUdejn5PGqHKxqoNnp/wpG35TZqbS3dqzmle0CtxNiU0I+y0JXOnUMhbZGdp3WwN5aH6LIBidJVqKTCrs2zGVEWnyHBvW/7TrGP74oZ29Fg1/nc7q9fLi3klc/L6G6sW0FJFcrlHyqPc6dHpYllBhT22KnpyYrvfFku5W9LARRDV6SrEQn5+Qns+D8AqYXdKyWDUuLB2C/n8mqqKYJl0clMdZIWnzbog/Fm7SEFZsCmeG5ynZ7NdDhOm016IQh2r0kqwElyUp0yRZjJDHW1GHbiHQtWZXXt9Lo6H1VcH+lltxGZVi1/lp1JXB0W9tJLwFdeH4MzcYu2qwAErK1e/uxAY5ocJMGdtFr8WYDmTYLFfWtFFU3MSHHdsZjHG4PR2q0WQpGZsRrc6x/9YbWOJ05Iaz7Kk0YYmNcdkLnVaDbk1VjJXjcoJev0UCQd1l0sP6rClpdHs4dlkyW7bR5pByNjImpw1V+jMrDFUyIywNzgnbr5gt7uLoJt1clKcZAWmsx7HtHq/5ZM2HkZQPwGwXO1N1S9ZZEMMVqw4QaK7X54kW/k2QlfFRVpaimiVaXh28NS9E2er1Q9Q0c3QoNFYxyummpqUd3HDyuZPS6toZxUxxYbKfcEkBnoGpvBXl1VUx2nUD5qlnbNyELJlwHBlPXgYQ7RdHarWoOaFVBSVYDQpKV8KludNDq8mAy6Ei3mrVL89/8Derb2mYUhRhbKu76eFqcHmoxk6pv1apCzibtdkqjs1dVsZbWYXJ7yUqwgc4COdMg/3ztqlqYa3V52HSghla3hysnZHUcH5mQfTJZiQEhyUr4HKttASA70YKu5QTsekVLQAYT5M2ArEkopjjyRzUTZzaQFNuWcFwtWmJrrQeH3fdYp6pMToUTHgsxGbmQNhqMAz9FcaB0isLuY1pfKudYb8eFT32N7HJFcKBIshI+lW39q3Ji3PDFa1qiik+D8d+DmCTffp0GGZtitVtCVqdz6oDU/gy6H5kMOox6BZdHpcXp6bxKc/sMDI4GMIdm3q3BJDyvGYuQqLQ7QPWSX7VB+wLGpcKkH3RIVL3l8aphNSg5UDEm7f95s/O07gsGM8SlaY/rjw5wVIOTJCsBaO0zJ5qcDLHvItFZoVX9xn9PKzF14VhdC29/Wc5nh493+fzeCjt/3FTE9uLa/gy738W2TW/cKVkBJOZp93WlAxjR4CXJSgBaL+1h8S5GOXZrQ2xGzoHY7gcWN7a62V/ZwL5uerPvr2ygodWN2+Pt8vlI0Z6sWrpKVrZc7b6ueAAjGrwkWQkAbLFGronfw6TsOEjKh4xxPe4/NCUWnaJwoslJbZOzw3NNDjfFx7VuCqOCPGPDQIv1VQO76LGf2Jasmmq0PleiX4V1slq+fDnnnHMOVquV9PR05s2bx759+0IdVnSyl0HNARRFByO/fcaBxRaj3rf4aPtwmnb72pbRyrJZSIqL0L5UbXzVwNOH3IDWtyyu7fKBtFv1u7BOVhs3bmThwoV89tlnfPDBB7hcLi677DKammSRyWBzHfpEe5Ax/uQX8Azap4/Zfawer1drTFdVla/LtGW0zgrRMlrBdPbQJO6ZPZxZo9K63sFXFSwZuKAGqbDuurB+/foOP69Zs4b09HS2b9/OhRdeGKKook/L8aPs2rENi9HAhHNm9PpDMTIjno379TS0uik63sTwtHhKTjRT0+DAZNCFbBmtYDrj7KiJeVC2E+olWfW3sC5Zna6+XuuglxxmM0pGuoaD/wGg1joSQ3xKr48z6nWMH6KVnj47fBxVVX1XB8dlJwRlGuSw195u1ViljXkU/SZikpXX6+W+++5j5syZjB8/vtv9HA4Hdru9w030wNGAo/xr7fGQs/0+/Jz8ZKwWAzOHp6IoCpePz2JMlrXTXFiRqtnp5l/fVLL+q27mXDdbtX5oqirtVv0sYpLVwoUL+eqrr1i3bl2P+y1fvhybzea75ebmDlCEEapsJ82tThrMmdgyhvp9uMWo54cz8slP1dbQs8UYuXx8lu8qWjTYfayevRUNvna5TpLytfvaogGLaTCKiGR177338s9//pOPPvqInJycHvddunQp9fX1vltpqXTY65bHDWU7aXK4qbCO0wYvB6DbqVSigMWgR1G0glOXVwQBktvmjz9xeOACG4TC+t+fqqr8+Mc/5s033+Tjjz+moODMiwqYzWbM5sC+dIPO8QO4WhppUC2ciMknPUHet9PpdApxJgONDjdNDjfxXS1nnzhUW0Si+QS01EFM4kCHOSiE9b/EhQsX8vLLL/Pqq69itVqpqKigoqKClpaWUIcWHSp20+R0Ux03msQ4S8eBusInvm0Z+4bWbqZyNlpOzsIgpat+E9bJ6rnnnqO+vp5Zs2aRlZXlu/3pT38KdWiRz9EAJw5j1OlIHXF2xPc070/tpake551PHqbdS7tVvwn7aqDoJ5Vfg6oSlz6UWVPOCnU0Ya29ZNXYXckKtHarok+g9gh4PaCTUmqwhXXJSvQTVYWK3drjzAmhjSUCWM09jA/07ZSlTSzodkoXhn4S1iUr0U8aKqCpBo+i57ilgBSvenIuddHJhBwbE3JsPbfpKYq2Uk/FV9p0x0n+dwMRPZOS1WDUVqqqjc3nle1V/N9/joQ2njBnNuh7d/EhdbR2X7NfK72KoJJkNdh43FCl9Vgvt4wEIDXA/lXiNMkF2pJkrfXa8BsRVJKsBpvjB7UxbGYrpaQDkG61hDio8KaqKhv2VPK3Xcc6r858Kr0Rktr6AtbIVEbBJslqsGlvWM8YR1WDNmleoD3XBwtFUThQ1cjh6qbu+1q1S2urClZLsgo2SVaDiaPR12nRkTaO2mYXgPRc7wVrW/cFe6ur5x1TRmjdFppqpCoYZJKsBpOqb0D1QkI2la54ABJijFE16Li/2GK0NRLrW86QrIwx2lVBgMqv+jmqwUWS1WBS8aV2nzme8vq2BU1t0l7VG4kx2vTM9c1nSFagzbYKUPmNXBUMIvmXOlg0VEJjtVZFSR/LsFYdOp1CSoTPkT5Qel2yAkgerq0r6GjQVr5pn0JG9Ikkq8GivWE9dSQYY0gzQpo0rPdaYqyWrOqanWfYE637QvoYKNulve+SrIJCqoGDgdfj61tFhgyvCURCW8nK6fH2bsxq1iTtvmqvLNMVJFKyGgxOHNa+MKY4SB5GRX0r9S0ushMtWC3GUEcXERIsBu6eNbz388pbs8CaqQ1tqvgS8r7VvwEOAlKyGgzKv9DuM8aBTsfXZfW8s7ucHSV1IQ0rkiiK4t8CGIoCQ6Zqj4/tAG9kr0wdDiRZRbvWeq3XOkDWZACO1mpXAockypXAfpU+VpuY79S/gQiYJKtoV7ZLu3yeNBTiUmh0uDnR5ERRICcpNtTRRZTi4028tfMYmw7U9O4AvRGyp2iPSz6Vbgx9JMkqmnk9J6uA2VqVpPSE1tibbrUMjnX9gsjp9lJU00RprR8N5jnnaFcH7eUyi2gfSbKKZtX7wNmkNaynajMstCer3OSYUEYWkVLita4exxsd3S/LdTpTHGS1la6KpXTVF5KsopWqQuln2uPsyaDTo6oqJe3JSqqAfkuMMWLQKbg8au86h7bLnQ46A9SVwvFD/RdglJNkFa1qi7Re63oDDJkGQF2zi4ZWN0a9QnailKz8pdMpvtJVVYOj9wdaEiBH+xtw6EOtei78JskqWpW0laqyJoNJK0UlxZm448JhXDUxO6oXJu1PWW1jKcvq/VwOLm+G9ndoPg5lO/shsugnn9hoVFcKtcXawps553R4Kt5s8C31LvzXXiItq/MzWRktkH+B9vjwx9piqMIvkqyijapqVQ2ArIm+1YFlWbPgyE60YDLoiDHq/X9Ps6eALQc8Ltj3rjS2+0mSVbSp3gv2Mq2PT/t/cuA/h4/zl22llByXcWp9YbUYufui4Vw7NQdF8XNFIEWBs67UGttrj8DRrf0SY7SSZBVNXC1w8F/a49xzwaxNsOf1qnxTZudobQstPc0hLnpF15dly2KTYfjF2uNDH2lJS/SKJKtocnCDNnVxbHKHgbMHqxtpaHVjMeoZnibtVcFS3+IKrHo9ZCpkjtdmbf36LW0KZHFGkqyiRcVu7aYoMHquVg1Ea6vaeuQEAJNybRj08ifvK1VV+cu2Ul7cVERxINVqRYFRl2uzMrha4IvXoPlE8AONMvLJjQb1R2Hfeu1x3gxIzPU9tbeigSq7A6NeYUpuUogCjC6KovgmLvziaF1gJ9EbYeINEJeqlYZ3/p8sO38GkqwiXV0pfPkn8Lq1lVVOaVRvdrr594FqAKYXpBBjkrGAwTIxJxGAw9VNVNpbAzuJKRYmfR/i07X5xna9BqVb5SphNyIiWT377LPk5+djsVg499xz2bJlS6hDCj1VhWPbtSqE2wmJeTD2GtCd/JPuKW+gyeEhOc7E1LzE0MUahZLjTIzJSgBgw54q3J4A56syx8OUW7Sxm163doFk5/9p/4REB2GfrP70pz+xaNEiHnroIXbs2MGkSZOYM2cOVVWDdE02VdXGl+14Cfa/rw3dSBsFE64DQ8fFH6bmJXLxWelcNTFL2qr6wcwRKZiNOirtrbz3dWXgCctggvHfg1FztOph/THY+bJ2K/9SW0FboKhh3lvw3HPP5ZxzzuGZZ54BwOv1kpuby49//GOWLFlyxuPtdjs2m436+noSEhL6O9zg8rjA1ay1aTRVaVPkHj+krZoC2ge74CK82WfT6PJQZXdQVNPE+CEJZNlk7N9AOFLTxN92leFVVSbnJjL7rPS+nbC1XpudofxL7WohaA3y1ixIGAJxKRCTDKZ4rRppsGjPRyh/vp9hPQe70+lk+/btLF261LdNp9Nx6aWX8p///KfLYxwOBw7HyUGmdru9dy9W/ClUft15e4dcrna/rcN2/7e1utx8XWb3Pad43SiqG1RtrzSr2TdTglMx8sGJDCpsk3AeisOx72CHkNweL1kTJFkNhPzUOL47ZQjvf1PB5NxE3/YtRSfYVnwCs0GPTgEFrWFeaXs8d0KWb1D0V8fq2VNu57ppuWCxwegrYOhM7epu5dfaeEJ7mXY7naJonUx1eu1e0WuPtSc779ud/kh4JitMuiFopwvrZFVTU4PH4yEjI6PD9oyMDPbu3dvlMcuXL+fhhx/2/8UcjaHt7+Ly4Gxp6rRZRcGlj8USMwRyh0NSPq7YIezfVAJefCP4dYpCUpyRIYkxnJUVYSXICJeXEsuCmQUdOou2ujw4XF4crq6rhp5T/rs0Oz2caDptiS9LAuTP1G6t9dpYz6YqaDoOLbXgatLaKlVVK4F7/JiyZqBYerFsmR/COlkFYunSpSxatMj3s91uJzc3t4cj2gw5W2vk7PAf5pTHvu1dbTtle2+3nXZOvddL3ljHyc16AxhiUAxmFEWnXclrWw4qxqtyy4yhvjNZjHpiTXr/h3+IoDm9V/v0gmTGD7HhdHvxqioqWv+s9hzVvmgqwOgMK9k9zYdvsWnjPE/ncYO7RWuY93rabu626mNXJXk6bu/0XJDpgptewjpZpaamotfrqays7LC9srKSzMzMLo8xm82YzQEs3hmXot1CxAgM6WWBSKdTSI2XBUrDmcWo7/W00bZYI7bYAJZE0xtAb/X/uAgV1peITCYTZ599Nhs2bPBt83q9bNiwgRkzZoQwMiHEQAvrkhXAokWLmD9/PtOmTWP69OmsXLmSpqYmbr311lCHJoQYQGGfrG644Qaqq6tZtmwZFRUVTJ48mfXr13dqdBdCRLew72fVVxHdz0qIKOfP9zOs26yEEKKdJCshRESQZCWEiAhh38DeV+1Ncr0ediOEGDDt38veNJ1HfbJqaNAG/faqF7sQIiQaGhqw2Ww97hP1VwO9Xi9lZWVYrdaQDkdpH/ZTWloqVyVPI+9N96L9vVFVlYaGBrKzs9Hpem6VivqSlU6nIycnJ9Rh+CQkJETlhy4Y5L3pXjS/N2cqUbWTBnYhRESQZCWEiAiSrAaI2WzmoYceCmxGiCgn70335L05Keob2IUQ0UFKVkKIiCDJSggRESRZCSEigiQrIUREkGQVRP6sHP3CCy9wwQUXkJSURFJSEpdeemlUrzQd6Kra69atQ1EU5s2b178BhpC/701dXR0LFy4kKysLs9nMqFGjeOeddwYo2hBSRVCsW7dONZlM6osvvqh+/fXX6h133KEmJiaqlZWVXe7/gx/8QH322WfVnTt3qnv27FELCwtVm82mHj16dIAj73/+vjftioqK1CFDhqgXXHCBes011wxMsAPM3/fG4XCo06ZNU+fOnatu2rRJLSoqUj/++GN1165dAxz5wJNkFSTTp09XFy5c6PvZ4/Go2dnZ6vLly3t1vNvtVq1Wq7p27dr+CjFkAnlv3G63et5556l/+MMf1Pnz50dtsvL3vXnuuefUYcOGqU6nc6BCDBtSDQyC9pWjL730Ut+2M60cfbrm5mZcLhfJycn9FWZIBPrePPLII6Snp3PbbbcNRJghEch78/e//50ZM2awcOFCMjIyGD9+PI8//jgej2egwg6ZqB/IPBACWTn6dA888ADZ2dkdPrjRIJD3ZtOmTfzxj39k165dAxBh6ATy3hw+fJgPP/yQm266iXfeeYeDBw9yzz334HK5eOihhwYi7JCRZBUGVqxYwbp16/j444+xWHpYmXcQaGho4JZbbuGFF14gNTU11OGEHa/XS3p6Os8//zx6vZ6zzz6bY8eO8Zvf/EaSlTizQFaObvfkk0+yYsUK/vWvfzFxYhdLhEc4f9+bQ4cOceTIEa6++mrfNq/XC4DBYGDfvn0MHz68f4MeIIF8brKysjAajej1J1d7HjNmDBUVFTidTkwmU7/GHErSZhUEga4c/etf/5pHH32U9evXM23atIEIdcD5+96cddZZ7N69m127dvlu3/nOd5g9eza7du2KqhlfA/nczJw5k4MHD/oSOMD+/fvJysqK6kQFSNeFYFm3bp1qNpvVNWvWqN9884165513qomJiWpFRYWqqqp6yy23qEuWLPHtv2LFCtVkMql//etf1fLyct+toaEhVL9Cv/H3vTldNF8N9Pe9KSkpUa1Wq3rvvfeq+/btU//5z3+q6enp6q9+9atQ/QoDRpJVED399NNqXl6eajKZ1OnTp6ufffaZ77mLLrpInT9/vu/noUOHqkCn20MPPTTwgQ8Af96b00VzslJV/9+bTz/9VD333HNVs9msDhs2TH3sscdUt9s9wFEPPJkiRggREaTNSggRESRZCSEigiQrIUREkGQlhIgIkqyEEBFBkpUQIiJIshJCRARJViKqrFmzhsTERN/Pv/zlL5k8ebLv58LCwqiedTSaSbISASksLERRFH70ox91em7hwoUoikJhYWGH/YOdJPLz81m5cmWHbTfccAP79+/v9phVq1axZs0a38+zZs3ivvvuC2pcon9IshIBy83NZd26dbS0tPi2tba28uqrr5KXlxeSmGJiYkhPT+/2eZvN1qHkJSKHJCsRsKlTp5Kbm8sbb7zh2/bGG2+Ql5fHlClT+nTurko88+bN85XWZs2aRXFxMffffz+KoqAoCtC5Gni6U0t4hYWFbNy4kVWrVvnOUVRUxIgRI3jyySc7HLdr1y4UReHgwYN9+r1E4CRZiT5ZsGABq1ev9v384osvcuutt/b7677xxhvk5OTwyCOPUF5eTnl5ud/nWLVqFTNmzOCOO+7wnSMvL6/T7wSwevVqLrzwQkaMGBGsX0H4SZKV6JObb76ZTZs2UVxcTHFxMZs3b+bmm2/u99dNTk5Gr9djtVrJzMw84ySHXbHZbJhMJmJjY33n0Ov1FBYWsm/fPt+SWC6Xi1dffZUFCxYE+9cQfpCZQkWfpKWlceWVV7JmzRpUVeXKK6+M+OmIs7OzufLKK3nxxReZPn06//jHP3A4HFx33XWhDm1Qk5KV6LMFCxawZs0a1q5dG7TSh06n4/TZi1wuV1DO3Ru333677+LB6tWrueGGG4iNjR2w1xedSbISfXb55ZfjdDpxuVzMmTMnKOdMS0vr0A7l8Xj46quvOuxjMpn6vARVd+eYO3cucXFxPPfcc6xfv16qgGFAqoGiz/R6PXv27PE97k59fX2n5bVSUlK6nFf94osvZtGiRbz99tsMHz6cp556irq6ug775Ofn88knn3DjjTdiNpsDqn7m5+fz+eefc+TIEeLj40lOTkan0/narpYuXcrIkSN7nEtfDAwpWYmgSEhIICEhocd9Pv74Y6ZMmdLh9vDDD3e574IFC5g/fz4//OEPueiiixg2bBizZ8/usM8jjzzCkSNHGD58OGlpaQHFvXjxYvR6PWPHjiUtLY2SkhLfc7fddhtOp3NArm6KM5NpjYXoxr///W8uueQSSktLOy1EKgaeJCshTuNwOKiurmb+/PlkZmbyyiuvhDokgVQDhejktddeY+jQodTV1fHrX/861OGINlKyEkJEBClZCSEigiQrIUREkGQlhIgIkqyEEBFBkpUQIiJIshJCRARJVkKIiCDJSggRESRZCSEiwv8PdK66ifI1M9sAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:13:58.797775Z", + "iopub.status.busy": "2024-02-29T18:13:58.796842Z", + "iopub.status.idle": "2024-02-29T18:13:59.043601Z", + "shell.execute_reply": "2024-02-29T18:13:59.042585Z" + }, + "papermill": { + "duration": 0.273988, + "end_time": "2024-02-29T18:13:59.045976", + "exception": false, + "start_time": "2024-02-29T18:13:58.771988", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAupklEQVR4nO3de1hTV7438G/CJdwRiFxFAXFEqxCFA2LteOPWTh08dlpKLyBanfEypzaH2tIqPqintNUy1B4qp54iaudUp/UyndaiNJWOF0SFougALQiihSCgEIEaQrLeP3zZNSWBBMHA5vd5njyYlbVX1spOvu7svbO2gDHGQAghPCQ0dQcIIWSoUMARQniLAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxFAUcI4S1zU3dgONJoNKivr4e9vT0EAoGpu0MIuQ9jDHfu3IGnpyeEwr630SjgdKivr4e3t7epu0EI6cP169cxbty4PutQwOlgb28P4N4L6ODgYOLeDB2VSoXjx48jKioKFhYWpu4OeUCjZX0qFAp4e3tzn9O+UMDp0PO11MHBgfcBZ2NjAwcHB15/IEaL0bY+Ddl9RAcZCCG8RQFHCOEtCjhCCG9RwBFCeIsCjhDCWxRwhBDeotNECBmhOjs7UVFRwd1v/1mJM2XVcBJfgJ21SKtuQEAAbGxsHnYXTY4CjpARqqKiAsHBwb3K39VRt7i4GDNnzhz6Tg0zFHCEjFABAQEoLi7m7lc2tEL6WRkynp6OyR5jetUdjSjgCBmhbGxstLbKhNdaIDr5M6ZMC4JkgosJezZ80EEGQghvUcARQniLAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxFAUcI4S0KOEIIb1HAEUJ4iwKOEMJbFHCEEN6igCOE8BYFHCGEtyjgCCG8ZfKAy8rKgo+PD6ysrBAWFoZz587prZubmwuBQKB1s7Ky0qqzdOnSXnViYmKGehiEkGHIpBNeHjhwAFKpFNnZ2QgLC0NmZiaio6NRWVkJV1dXncs4ODigsrKSuy8QCHrViYmJwe7du7n7IpGoVx1CCP+ZdAsuIyMDK1asQFJSEqZOnYrs7GzY2NggJydH7zICgQDu7u7czc3NrVcdkUikVcfJyWkoh0EIGaZMtgXX1dWF4uJipKSkcGVCoRAREREoLCzUu1x7ezsmTJgAjUaDmTNn4q233sIjjzyiVaegoACurq5wcnLCggULsHXrVri46J/CWalUQqlUcvcVCgUAQKVSQaVSDXSIw17P2Pg8xtGku7ub+8vndWrM2EwWcM3NzVCr1b22wNzc3LQuhXa/yZMnIycnB4GBgWhra8P27dsxe/ZsXLlyBePGjQNw7+vpkiVL4Ovri+rqarzxxht4/PHHUVhYCDMzM53tpqenIy0trVf58ePHR8Wl1vLz803dBTIIrrcDgDnOnj2Lny6bujdDp7Oz0+C6AsYYG8K+6FVfXw8vLy+cOXMG4eHhXPn69evx3XffoaioqN82VCoVpkyZgvj4eGzZskVnnatXr2LixIn45ptvsHDhQp11dG3BeXt7o7m5GQ4ODkaObORQqVTIz89HZGQkLCwsTN0d8oAu1t3CH3ZdwOcrQhA03tnU3RkyCoUCYrEYbW1t/X4+TbYFJxaLYWZmhsbGRq3yxsZGuLu7G9SGhYUFZsyYgaqqKr11/Pz8IBaLUVVVpTfgRCKRzgMRFhYWo+KDP1rGyXfm5ubcXz6vT2PGZrKDDJaWlggODoZMJuPKNBoNZDKZ1hZdX9RqNcrKyuDh4aG3zo0bN9DS0tJnHUIIP5n0KKpUKsWuXbuwZ88elJeXY9WqVejo6EBSUhIAICEhQesgxObNm3H8+HFcvXoVJSUleOGFF3Dt2jW89NJLAO4dgHj11Vdx9uxZ1NbWQiaTITY2Fv7+/oiOjjbJGAkhpmPS8+Di4uLQ1NSE1NRUyOVySCQS5OXlcQce6urqIBT+ksG3b9/GihUrIJfL4eTkhODgYJw5cwZTp04FAJiZmeHSpUvYs2cPWltb4enpiaioKGzZsoXOhSNkFDLZQYbhTKFQwNHR0aCdmCOZSqXC0aNH8cQTT/B6n81oUXqtBYt3nsWRVbN4fWV7Yz6fJv+pFiGEDBUKOEIIb1HAEUJ4iwKOEMJbFHCEEN6igCOE8BYFHCGEtyjgCCG8RQFHCOEtCjhCCG9RwBFCeIsCjhDCWxRwhBDeooAjhPAWBRwhhLco4AghvEUBRwjhLQo4QghvUcARQniLAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxl8oDLysqCj48PrKysEBYWhnPnzumtm5ubC4FAoHWzsrLSqsMYQ2pqKjw8PGBtbY2IiAj8+OOPQz0MQsgwZNKAO3DgAKRSKTZt2oSSkhIEBQUhOjoaN2/e1LuMg4MDGhoauNu1a9e0Hn/33XexY8cOZGdno6ioCLa2toiOjsbdu3eHejiEkGHGpAGXkZGBFStWICkpCVOnTkV2djZsbGyQk5OjdxmBQAB3d3fu5ubmxj3GGENmZiY2bNiA2NhYBAYGYu/evaivr8eRI0cewogIIcOJuameuKurC8XFxUhJSeHKhEIhIiIiUFhYqHe59vZ2TJgwARqNBjNnzsRbb72FRx55BABQU1MDuVyOiIgIrr6joyPCwsJQWFiIZ599VmebSqUSSqWSu69QKAAAKpUKKpXqgcY5nPWMjc9jHE26u7u5v3xep8aMzWQB19zcDLVarbUFBgBubm6oqKjQuczkyZORk5ODwMBAtLW1Yfv27Zg9ezauXLmCcePGQS6Xc238us2ex3RJT09HWlpar/Ljx4/DxsbG2KGNOPn5+abuAhkE19sBwBxnz57FT5dN3Zuh09nZaXBdkwXcQISHhyM8PJy7P3v2bEyZMgX/8z//gy1btgy43ZSUFEilUu6+QqGAt7c3oqKi4ODg8EB9Hs5UKhXy8/MRGRkJCwsLU3eHPKCLdbeAsguYNWsWgsY7m7o7Q6bnG5YhTBZwYrEYZmZmaGxs1CpvbGyEu7u7QW1YWFhgxowZqKqqAgBuucbGRnh4eGi1KZFI9LYjEokgEol0tj8aPvijZZx8Z25uzv3l8/o0ZmwmO8hgaWmJ4OBgyGQyrkyj0UAmk2ltpfVFrVajrKyMCzNfX1+4u7trtalQKFBUVGRwm4QQ/jDpV1SpVIrExESEhIQgNDQUmZmZ6OjoQFJSEgAgISEBXl5eSE9PBwBs3rwZs2bNgr+/P1pbW7Ft2zZcu3YNL730EoB7R1jXrVuHrVu3YtKkSfD19cXGjRvh6emJxYsXm2qYw0ZnZ6fW/s32n5U4U1YNJ/EF2Fn/sgUbEBAwKvY9Ev4zacDFxcWhqakJqampkMvlkEgkyMvL4w4S1NXVQSj8ZSPz9u3bWLFiBeRyOZycnBAcHIwzZ85g6tSpXJ3169ejo6MDK1euRGtrK+bMmYO8vLxeJwSPRhUVFQgODu5V/u6v7hcXF2PmzJkPp1OEDCEBY4yZuhPDjUKhgKOjI9ra2nh1kOHXW3CVDa2QflaGjKenY7LHGK6ctuBGptJrLVi88yyOrJoFyQQXU3dnyBjz+RxRR1HJg7GxsdHaMhNea4Ho5M+YMi2I1x8IMnqZ/LeohBAyVCjgCCG8RQFHCOEt2gdHyAhS09yBDmW3zseqmzq4vz0n/epiKzKHr9h2SPo33FDAETJC1DR3YP72gn7r/efnZf3WOZE8b1SEHAUcISNEz5ZbZpwE/q52vR//WYkvCwrx5Lxw2Fr3/ukhAFTdbMe6A6V6twL5hgKOkBHG39UO07wce5WrVCrIxwIzJzjx+reoxqCA47G+9tcAtM+G8B8FHE8Zur8GoH02hL+MDrirV6/Cz89vKPpCBlF/+2sA2mdD+M/ogPP398fcuXOxfPly/OEPf6AfsQ9z+vbXALTPhvCf0Sf6lpSUIDAwEFKpFO7u7vjjH//Y56X+CCHEVIwOOIlEgvfffx/19fXIyclBQ0MD5syZg2nTpiEjIwNNTU1D0U9CCDHagH+qZW5ujiVLluCzzz7DO++8g6qqKiQnJ8Pb2xsJCQloaGgYzH4SQojRBhxwFy5cwOrVq+Hh4YGMjAwkJyejuroa+fn5qK+vR2xs7GD2kxBCjGb0QYaMjAzs3r0blZWVeOKJJ7B371488cQT3My7vr6+yM3NhY+Pz2D3lRBCjGJ0wO3cuRPLli3D0qVLta5cdT9XV1d8/PHHD9w5Qgh5EEYHXH5+PsaPH691rQQAYIzh+vXrGD9+PCwtLZGYmDhonSSEkIEweh/cxIkT0dzc3Kv81q1b8PX1HZROEULIYDA64PRdo6a9vZ1O+iWEDCsGf0WVSqUA7l17NDU1VeuqS2q1GkVFRX1ePZ4QQh42gwPu+++/B3BvC66srAyWlpbcY5aWlggKCkJycvLg95AMiFJ9F0Krn1CjqITQSvdvUbu7u1HfXY/yW+V6ZxOpUbRDaPUTlOq7AHT/5IuQ4crggDtx4gQAICkpCe+//z6vrhfKR/Ud12Dr+wHeMOBXdB/mfdjn47a+QH2HBMFwG6TeEfJwGH0Udffu3YPagaysLGzbtg1yuRxBQUH44IMPEBoa2u9y+/fvR3x8PGJjY3HkyBGufOnSpdizZ49W3ejoaOTl5Q1qv4c7T9sJ6Kj5M96Pk2CintlEuru7cfrUaTw651G9W3DVN9vx8oFSeM6fMJTdJWRIGBRwS5YsQW5uLhwcHLBkyZI+6x46dMjgJz9w4ACkUimys7MRFhaGzMxMREdHo7KyEq6urnqXq62tRXJyMh577DGdj8fExGgFsUikeyogPhOZWUFz1wu+DpMx1UX/bCI15jWY4jxF72wimrtt0NxtgsiMDiCRkcego6iOjo4QCATcv/u6GSMjIwMrVqxAUlISpk6diuzsbNjY2CAnJ0fvMmq1Gs8//zzS0tL0zksnEong7u7O3ZycnIzqFyGEHwzagrt/a2iwvqJ2dXWhuLgYKSkpXJlQKERERAQKCwv1Lrd582a4urpi+fLlOHnypM46BQUFcHV1hZOTExYsWICtW7fCxcVFb5tKpRJKpZK7r1AoANzbwlGpVMYObVjo7u7m/uobQ095X2M0pB3ycPS3LkbL+jSm3yabsry5uRlqtRpubto7rt3c3FBRUaFzmVOnTuHjjz9GaWmp3nZjYmKwZMkS+Pr6orq6Gm+88QYef/xxFBYWwszMTOcy6enpSEtL61V+/PhxrdNhRpLr7QBgjlOnTuGa7l1wnPz8/EFphwytq3e6ILS6jcMn/4ZCa/319ny9R+9j8p8BoZUZTpz8FtfsLfXWG846OzsNrmtQwM2YMYP7itqfkpISg5/cGHfu3MGLL76IXbt2QSwW66337LPPcv+ePn06AgMDMXHiRBQUFGDhwoU6l0lJSeHO8wPubcF5e3sjKipqxB4tvlKvwPays5gzZw4e8dQ9BpVKhfz8fERGRurdB2dIO+Th+KryAmzVm3FIDaB94O3Y+gK+ko/wxOSQQevbw9TzDcsQBgXc4sWLB9oXvcRiMczMzNDY2KhV3tjYCHd39171q6urUVtbi0WLFnFlGo0GwL256SorKzFx4sRey/n5+UEsFqOqqkpvwIlEIp0HIiwsLEbsVN49R0XNzc37HUNf4zSmHTK0vB38+jwybsxRce/5fiN2fRrTb4MCbtOmTQPujD6WlpYIDg6GTCbjAlSj0UAmk2Ht2rW96gcEBKCsTPvqTxs2bMCdO3fw/vvvw9vbW+fz3LhxAy0tLXpnPiFkpOjvyDgdFe/NpJcNlEqlSExMREhICEJDQ5GZmYmOjg4kJSUBABISEuDl5YX09HRYWVlh2rRpWsuPGTMGALjy9vZ2pKWl4amnnoK7uzuqq6uxfv16+Pv7Izo6+qGOjRBiegYFnLOzM3744QeIxWI4OTn1uT/u1q1bBj95XFwcmpqakJqaCrlcDolEgry8PO7AQ11dXa9pmfpiZmaGS5cuYc+ePWhtbYWnpyeioqKwZcuWUXkuHCGjnUEB95e//AX29vbcvw094GCItWvX6vxKCtw73aMvubm5Wvetra1x7NixQeoZIWSkMyjg7p+8cunSpUPVF0IIGVRGzwdnZmaGmzdv9ipvaWnRe54ZIYSYwqBNeKlUKrWmUCKEEFMz+Cjqjh07ANyb8PJ///d/YWf3y3k4arUa//znPxEQEDD4PSSEkAEyOOD+8pe/ALi3BZedna31ddTS0hI+Pj7Izs4e/B4SQsgAGRxwNTU1AID58+fj0KFDNEMHIWTYM/pE356ZfQkhZLgzOuCWLVvW5+N9zeVGCCEPk9EBd/v2ba37KpUKly9fRmtrKxYsWDBoHSOEkAdldMAdPny4V5lGo8GqVat0zuZBCCGmYvR5cDobEQohlUq5I62EEDIcDErAAffma+uZDpkQQoYDo7+i3j/zLXDvvLiGhgZ89dVXWr9ZJYQQUzM64HqucN9DKBRi7NixeO+99/o9wkoIIQ8TnQdHCOGtQdsHRwghww0FHCGEtyjgCCG8RQFHCOGtQQu4GzduYOXKlYPVHCGEPLBBC7iWlhZ8/PHHg9UcIYQ8MPqKSgjhLQo4QghvUcARQnjL4F8yLFmypM/HW1tbB9SBrKwsbNu2DXK5HEFBQfjggw8QGhra73L79+9HfHw8YmNjceTIEa6cMYZNmzZh165daG1txaOPPoqdO3di0qRJA+ofIWTkMngLztHRsc/bhAkTkJCQYNSTHzhwAFKpFJs2bUJJSQmCgoIQHR2t87qr96utrUVycjIee+yxXo+9++672LFjB7Kzs1FUVARbW1tER0fj7t27RvWNEDLyGbwFt3v37kF/8oyMDKxYsQJJSUkAgOzsbHz11VfIycnB66+/rnMZtVqN559/HmlpaTh58qTWliNjDJmZmdiwYQNiY2MBAHv37oWbmxuOHDmCZ599dtDHQAgZvoz+sf1g6erqQnFxMVJSUrgyoVCIiIgIFBYW6l1u8+bNcHV1xfLly3Hy5Emtx2pqaiCXyxEREcGVOTo6IiwsDIWFhXoDTqlUQqlUcvcVCgWAe9Oxq1SqAY3P1Hrm5uvu7tY7hp7yvsZoSDvk4ehvXYyW9WlMvw0OOEOnQjL0ojPNzc1Qq9Vwc3PTKndzc0NFRYXOZU6dOoWPP/4YpaWlOh+Xy+VcG79us+cxXdLT05GWltar/Pjx47CxselrGMPW9XYAMMepU6dwza7vuvn5+YPSDhlahq4Lvq/Pzs5Og+saHHC5ubmYMGECZsyYAcbYgDr2IO7cuYMXX3wRu3btglgsHtS2U1JStCbyVCgU8Pb2RlRUFBwcHAb1uR6WK/UKbC87izlz5uART91jUKlUyM/PR2RkJCwsLAbcDnk4+lsXo2V99nzDMoTBAbdq1Sp8+umnqKmpQVJSEl544QU4OzsPqIMAIBaLYWZmhsbGRq3yxsZGuLu796pfXV2N2tpaLFq0iCvTaDQAAHNzc1RWVnLLNTY2wsPDQ6tNiUSity8ikQgikahXuYWFhd43ynBnbm7O/e1vDH2N05h2yNAydF3wfX0a02+Dj6JmZWWhoaEB69evxz/+8Q94e3vjmWeewbFjxwa0RWdpaYng4GDIZDKuTKPRQCaTITw8vFf9gIAAlJWVobS0lLv9/ve/x/z581FaWgpvb2/4+vrC3d1dq02FQoGioiKdbRJC+M2ogwwikQjx8fGIj4/HtWvXkJubi9WrV6O7uxtXrlyBnZ1xX+qlUikSExMREhKC0NBQZGZmoqOjgzuqmpCQAC8vL6Snp8PKygrTpk3TWn7MmDEAoFW+bt06bN26FZMmTYKvry82btwIT09PLF682Ki+jXQ/q9QAgMs/temt0/GzEheaAPdrt2Fr3XsLFgCqbrYPSf8IeRgGfBRVKBRCIBCAMQa1Wj2gNuLi4tDU1ITU1FTI5XJIJBLk5eVxBwnq6uogFBr3Y4v169ejo6MDK1euRGtrK+bMmYO8vDxYWVkNqI8jVfX/D6bXD5X1U9Mc+6rO99uerchkB9wJGTCj3rVKpRKHDh1CTk4OTp06hSeffBL//d//jZiYGKODqMfatWuxdu1anY8VFBT0uWxubm6vMoFAgM2bN2Pz5s0D6g9fRD1yb3/kRFc7WFuY6axT2dCG//y8DO/9YTomezjqbctWZA5fse2Q9JOQoWRwwK1evRr79++Ht7c3li1bhk8//XTQj2aSweNsa4lnQ8f3WafnnKiJY20xzUt/wJHhob/dDrTLoTeDAy47Oxvjx4+Hn58fvvvuO3z33Xc66x06dGjQOkcI+YVhux1ol8P9DB5lQkICBALBUPaFENKH/nY70C6H3ow60ZcQYjr97XagXQ690XxwhBDeooAjhPAWBRwhhLco4AghvEUBRwjhLQo4QghvUcARQniLAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxFAUcI4S0KOEIIb1HAEUJ4iwKOEMJbFHCEEN6igCOE8BYFHCGEtyjgCCG8ZfKAy8rKgo+PD6ysrBAWFoZz587prXvo0CGEhIRgzJgxsLW1hUQiwb59+7TqLF26FAKBQOsWExMz1MMghAxDJr044oEDByCVSpGdnY2wsDBkZmYiOjoalZWVcHV17VXf2dkZb775JgICAmBpaYkvv/wSSUlJcHV1RXR0NFcvJiYGu3fv5u6LRLovgksI4TeTbsFlZGRgxYoVSEpKwtSpU5GdnQ0bGxvk5OTorD9v3jz8+7//O6ZMmYKJEyfi5ZdfRmBgIE6dOqVVTyQSwd3dnbs5OTk9jOEQQoYZk23BdXV1obi4GCkpKVyZUChEREQECgsL+12eMYZvv/0WlZWVeOedd7QeKygogKurK5ycnLBgwQJs3boVLi4uettSKpVQKpXcfYVCAQBQqVRQqVTGDm3E6LmOZnd3N6/HOVqMlvVpzNhMFnDNzc1Qq9Vwc3PTKndzc0NFRYXe5dra2uDl5QWlUgkzMzN8+OGHiIyM5B6PiYnBkiVL4Ovri+rqarzxxht4/PHHUVhYCDOz3lcDB4D09HSkpaX1Kj9+/DhsbGwGOMLh73o7AJjj7Nmz+OmyqXtDHtRoWZ+dnZ0G1zXpPriBsLe3R2lpKdrb2yGTySCVSuHn54d58+YBAJ599lmu7vTp0xEYGIiJEyeioKAACxcu1NlmSkoKpFIpd1+hUMDb2xtRUVFwcHAY0vGY0sW6W0DZBcyaNQtB451N3R3ygEbL+uz5hmUIkwWcWCyGmZkZGhsbtcobGxvh7u6udzmhUAh/f38AgEQiQXl5OdLT07mA+zU/Pz+IxWJUVVXpDTiRSKTzQISFhQUsLCwMHNHIY25uzv3l8zhHi9GyPo0Zm8kOMlhaWiI4OBgymYwr02g0kMlkCA8PN7gdjUajtf/s127cuIGWlhZ4eHg8UH8JISOPSb+iSqVSJCYmIiQkBKGhocjMzERHRweSkpIAAAkJCfDy8kJ6ejqAe/vKQkJCMHHiRCiVShw9ehT79u3Dzp07AQDt7e1IS0vDU089BXd3d1RXV2P9+vXw9/fXOo2EEDI6mDTg4uLi0NTUhNTUVMjlckgkEuTl5XEHHurq6iAU/rKR2dHRgdWrV+PGjRuwtrZGQEAAPvnkE8TFxQEAzMzMcOnSJezZswetra3w9PREVFQUtmzZQufCETIKCRhjzNSdGG4UCgUcHR3R1tbG64MMpddasHjnWRxZNQuSCfpPoyEjw2hZn8Z8Pk3+Uy1CCBkqFHCEEN6igCOE8BYFHCGEtyjgCCG8RQFHCOEtCjhCCG9RwBFCeIsCjhDCWxRwhBDeooAjhPAWBRwhhLco4AghvEUBRwjhLQo4QghvUcARQniLAo4QwlsUcIQQ3qKAI4Tw1oi78DMZuM7OTlRUVHD3KxtaoZRXofyyNTQtY7jygIAA2NjYmKCHhAwuCrhRpKKiAsHBwb3Kn9ujfb+4uBgzZ858SL0iZOhQwI0iAQEBKC4u5u63/6zEVycK8bv54bCzFmnVI4QPKOBGERsbG60tM5VKhdvNNxEeGgILCwsT9oyQoUEHGQghvGXygMvKyoKPjw+srKwQFhaGc+fO6a176NAhhISEYMyYMbC1tYVEIsG+ffu06jDGkJqaCg8PD1hbWyMiIgI//vjjUA+DEDIMmTTgDhw4AKlUik2bNqGkpARBQUGIjo7GzZs3ddZ3dnbGm2++icLCQly6dAlJSUlISkrCsWPHuDrvvvsuduzYgezsbBQVFcHW1hbR0dG4e/fuwxoWIWS4YCYUGhrK1qxZw91Xq9XM09OTpaenG9zGjBkz2IYNGxhjjGk0Gubu7s62bdvGPd7a2spEIhH79NNPDW6zra2NAWBtbW0GLzMSdXV1sSNHjrCuri5Td4UMgu9rm9mE175k39c2m7orQ8qYz6fJtuC6urpQXFyMiIgIrkwoFCIiIgKFhYX9Ls8Yg0wmQ2VlJX77298CAGpqaiCXy7XadHR0RFhYmEFtEkL4xWRHUZubm6FWq+Hm5qZV7ubmpnUy6q+1tbXBy8sLSqUSZmZm+PDDDxEZGQkAkMvlXBu/brPnMV2USiWUSiV3X6FQALh3lFGlUhk3sBGkZ2x8HuNo0t3dzf3l8zo1Zmwj7jQRe3t7lJaWor29HTKZDFKpFH5+fpg3b96A20xPT0daWlqv8uPHj4+KM/rz8/NN3QUyCK63A4A5zp49i58um7o3Q6ezs9PguiYLOLFYDDMzMzQ2NmqVNzY2wt3dXe9yQqEQ/v7+AACJRILy8nKkp6dj3rx53HKNjY3w8PDQalMikehtMyUlBVKplLuvUCjg7e2NqKgoODg4DGR4I4JKpUJ+fj4iIyPpPDgeuFh3Cyi7gFmzZiFovLOpuzNker5hGcJkAWdpaYng4GDIZDIsXrwYAKDRaCCTybB27VqD29FoNNzXS19fX7i7u0Mmk3GBplAoUFRUhFWrVultQyQSQSQS9Sq3sLAYFR/80TJOvjM3N+f+8nl9GjM2k35FlUqlSExMREhICEJDQ5GZmYmOjg4kJSUBABISEuDl5YX09HQA975KhoSEYOLEiVAqlTh69Cj27duHnTt3AgAEAgHWrVuHrVu3YtKkSfD19cXGjRvh6enJhSghZPQwacDFxcWhqakJqampkMvlkEgkyMvL4w4S1NXVQSj85UBvR0cHVq9ejRs3bsDa2hoBAQH45JNPEBcXx9VZv349Ojo6sHLlSrS2tmLOnDnIy8uDlZXVQx8fIcS0BIwxZupODDcKhQKOjo5oa2vj/T64o0eP4oknnuD1V5rRovRaCxbvPIsjq2ZBMsHF1N0ZMsZ8Pk3+Uy1CCBkqFHCEEN6igCOE8BYFHCGEtyjgCCG8NeJ+qkUIucfQiwgBo/dCQhRwhIxQhl5ECBi9FxKigCNkhDL0IkI9dUcjCjhCRii6iFD/6CADIYS3KOAIIbxFAUcI4S0KOEIIb1HAEUJ4iwKOEMJbFHCEEN6i8+B06JkD1JiLW4xEKpUKnZ2dUCgUdN4UD4yW9dnzuTRkrl4KOB3u3LkDAPD29jZxTwgh+ty5cweOjo591qEpy3XQaDSor6+Hvb09BAKBqbszZHouj3j9+nVeT80+WoyW9ckYw507d+Dp6al1zRZdaAtOB6FQiHHjxpm6Gw+Ng4MDrz8Qo81oWJ/9bbn1oIMMhBDeooAjhPAWBdwoJhKJsGnTJohEov4rk2GP1mdvdJCBEMJbtAVHCOEtCjhCCG9RwBFCeIsCbpiqra2FQCBAaWnpQ2nn9OnTmD59OiwsLLB48eIHek5ChgsKuBGioKAAAoEAra2tQ9K+VCqFRCJBTU0NcnNzh+Q5SG/z5s3DunXrTN0NznDrz4OigBsCXV1dpu6C0aqrq7FgwQKMGzcOY8aMGVAbD3PcjDF0d3c/tOcbzkbi++2hYeSBzZ07l61Zs4a9/PLLzMXFhc2bN4+VlZWxmJgYZmtry1xdXdkLL7zAmpqauGW+/vpr9uijjzJHR0fm7OzMfve737Gqqiru8ZqaGgaAff/999y/778lJiYa1c6nn37KwsPDmUgkYo888ggrKCjQevz+2+7duxljjBUUFLB/+7d/Y5aWlszd3Z299tprTKVS9TnuEydOMAAsLy+PSSQSZmVlxebPn88aGxvZ0aNHWUBAALO3t2fx8fGso6ODa0utVrO33nqL+fj4MCsrKxYYGMg+++wz7vGedo8ePcpmzpzJLCws2IkTJwZzNT50iYmJvV77qqoqtmzZMu51+M1vfsMyMzN7LRcbG8u2bt3KPDw8mI+PD2OMsdOnT7OgoCAmEolYcHAwO3z4MPce6tHX+1JXf2pqah7WyzEkKOAGwdy5c5mdnR179dVXWUVFBTt79iwbO3YsS0lJYeXl5aykpIRFRkay+fPnc8t8/vnn7ODBg+zHH39k33//PVu0aBGbPn06U6vVjDHtgOvu7mYHDx5kAFhlZSVraGhgra2tRrUzbtw49vnnn7N//etf7KWXXmL29vasubmZdXd3s4aGBubg4MAyMzNZQ0MD6+zsZDdu3GA2NjZs9erVrLy8nB0+fJiJxWK2adMmveOuqKjggmjWrFns1KlTrKSkhPn7+7O5c+eyqKgoVlJSwv75z38yFxcX9vbbb3Ntbd26lQUEBLC8vDxWXV3Ndu/ezUQiERfEPe0GBgay48ePs6qqKtbS0jLUq3ZItba2svDwcLZixQrW0NDAGhoa2N27d1lqaio7f/48u3r1Kvvkk0+YjY0NO3DgALdcYmIis7OzYy+++CK7fPkyu3z5Mmtra2POzs7shRdeYFeuXGFHjx5lv/nNb7QC7vbt232+L3X1p7u72xQvzaChgBsEc+fOZTNmzODub9myhUVFRWnVuX79OhdQujQ1NTEArKysjDGmHXCM/fIBv337dp990dfO/WGiUqnYuHHj2DvvvMOVOTo6cltujDH2xhtvsMmTJzONRsOVZWVlMTs7Oy48fz3u+/v5zTffcGXp6ekMAKuurubK/vjHP7Lo6GjGGGN3795lNjY27MyZM1ptLV++nMXHx2u1e+TIkT7HP9LMnTuXvfzyy33WWbNmDXvqqae4+4mJiczNzY0plUqubOfOnczFxYX9/PPPXNmuXbu03kOGvC8N6c9IQvvgBklwcDD374sXL+LEiROws7Pjbj1XFq+urgYA/Pjjj4iPj4efnx8cHBzg4+MDAKirqzPqeQ1tJzw8nPu3ubk5QkJCUF5errfd8vJyhIeHa00X9eijj6K9vR03btzQOe77BQYGcv92c3ODjY0N/Pz8tMpu3rwJAKiqqkJnZyciIyO1XrO9e/dyr1ePkJAQvX3mi6ysLAQHB2Ps2LGws7PDRx991Gt9Tp8+HZaWltz9yspKBAYGwsrKiisLDQ3VWsaQ9yXf0HRJg8TW1pb7d3t7OxYtWoR33nmnVz0PDw8AwKJFizBhwgTs2rULnp6e0Gg0mDZtmtE7jAernYG6f9z3u39GWYFA0GuGWYFAAI1GA+De6wUAX331Fby8vLTq/fp3lfqejy/279+P5ORkvPfeewgPD4e9vT22bduGoqIirXoDeR0MeV/yDQXcEJg5cyYOHjwIHx8fmJv3folbWlpQWVmJXbt24bHHHgMAnDp1qs82e/63VqvVA2rn7Nmz+O1vfwsA6O7uRnFxMdauXav3+aZMmYKDBw+CMcZtxZ0+fRr29vaDPlfe1KlTIRKJUFdXh7lz5w5q28OdpaWl1jo9ffo0Zs+ejdWrV3NlhmxdTZ48GZ988gmUSiX3n8L58+e16vT3vtTVn5GOvqIOgTVr1uDWrVuIj4/H+fPnUV1djWPHjiEpKQlqtRpOTk5wcXHBRx99hKqqKnz77beQSqV9tjlhwgQIBAJ8+eWXaGpqQnt7u1HtZGVl4fDhw6ioqMCaNWtw+/ZtLFu2TO/zrV69GtevX8ef//xnVFRU4O9//zs2bdoEqVTa7yyqxrK3t0dycjJeeeUV7NmzB9XV1SgpKcEHH3yAPXv2DOpzDTc+Pj4oKipCbW0tmpubMWnSJFy4cAHHjh3DDz/8gI0bN/YKKl2ee+45aDQarFy5EuXl5Th27Bi2b98OANx/UP29L3X1p2cre6SigBsCnp6eOH36NNRqNaKiojB9+nSsW7cOY8aMgVAohFAoxP79+1FcXIxp06bhlVdewbZt2/ps08vLC2lpaXj99dfh5uaGtWvXGtXO22+/jbfffhtBQUE4deoUvvjiC4jF4j6f7+jRozh37hyCgoLwpz/9CcuXL8eGDRse6LXRZ8uWLdi4cSPS09MxZcoUxMTE4KuvvoKvr++QPN9wkZycDDMzM0ydOhVjx45FdHQ0lixZgri4OISFhaGlpUVra04fBwcH/OMf/0BpaSkkEgnefPNNpKamAgC3X66/96Wu/hi7T3i4oemSCOGpv/71r0hKSkJbWxusra1N3R2ToH1whPDE3r174efnBy8vL1y8eBGvvfYannnmmVEbbgAFHCG8IZfLkZqaCrlcDg8PDzz99NP4r//6L1N3y6ToKyohhLfoIAMhhLco4AghvEUBRwjhLQo4QghvUcARQniLAo6Y1NKlSyEQCLgf5Lu5uSEyMhI5OTlG/UwoNzd3wDMRP4ilS5fSNSyGMQo4YnIxMTFoaGhAbW0tvv76a8yfPx8vv/wynnzySZqWnDwYk85GR0a9num3f00mkzEAbNeuXYwxxt577z02bdo0ZmNjw8aNG8dWrVrF7ty5wxj7ZTLM+289Mw/v3buXBQcHMzs7O+bm5sbi4+NZY2Mj9zy3bt1izz33HBOLxczKyor5+/uznJwc7vG6ujr29NNPM0dHR+bk5MR+//vfc9N4b9q0qdfzjvRp1PmGtuDIsLRgwQIEBQXh0KFDAAChUIgdO3bgypUr2LNnD7799lusX78eADB79mxkZmbCwcEBDQ0NaGhoQHJyMgBApVJhy5YtuHjxIo4cOYLa2losXbqUe56NGzfiX//6F77++muUl5dj586d3CQEKpUK0dHRsLe3x8mTJ3H69GnY2dkhJiYGXV1dSE5OxjPPPMNtgTY0NGD27NkP94UifTN1wpLRTd8WHGOMxcXFsSlTpuh87LPPPmMuLi7c/d27dzNHR8d+n+/8+fMMALf1t2jRIpaUlKSz7r59+3pN265UKpm1tTU7duxYv/0npkdbcGTYYvdNtvnNN99g4cKF8PLygr29PV588UW0tLSgs7OzzzaKi4uxaNEijB8/Hvb29tyEmj3TAK1atQr79++HRCLB+vXrcebMGW7ZixcvoqqqCvb29twU387Ozrh79y5vp/jmGwo4MmyVl5fD19cXtbW1ePLJJxEYGIiDBw+iuLgYWVlZAPq+JmhHRweio6Ph4OCAv/71rzh//jwOHz6stdzjjz+Oa9eu4ZVXXkF9fT0WLlzIfb1tb29HcHAwSktLtW4//PADnnvuuSEePRkMNJsIGZa+/fZblJWV4ZVXXkFxcTE0Gg3ee+89bmLGv/3tb1r1dU21XVFRgZaWFrz99tvw9vYGAFy4cKHXc40dOxaJiYlITEzEY489hldffRXbt2/HzJkzceDAAbi6usLBwUFnP/k2xTff0BYcMTmlUgm5XI6ffvoJJSUleOuttxAbG4snn3wSCQkJ8Pf3h0qlwgcffICrV69i3759yM7O1mrDx8cH7e3tkMlkaG5uRmdnJ8aPHw9LS0tuuS+++AJbtmzRWi41NRV///vfUVVVhStXruDLL7/ElClTAADPP/88xGIxYmNjcfLkSdTU1KCgoAD/8R//wV1ZzMfHB5cuXUJlZSWam5uhUqkezotGDGPqnYBkdLv/aurm5uZs7NixLCIiguXk5HDXX2WMsYyMDObh4cGsra1ZdHQ027t3b6/rxP7pT39iLi4uWqeJ/N///R/z8fFhIpGIhYeHsy+++KLXtUKnTJnCrK2tmbOzM4uNjWVXr17l2mxoaGAJCQlMLBYzkUjE/Pz82IoVK1hbWxtjjLGbN2+yyMhIZmdnR6eJDEM0HxwhhLfoKyohhLco4AghvEUBRwjhLQo4QghvUcARQniLAo4QwlsUcIQQ3qKAI4TwFgUcIYS3KOAIIbxFAUcI4S0KOEIIb/0/MXaqiocd294AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:13:59.105673Z", + "iopub.status.busy": "2024-02-29T18:13:59.104697Z", + "iopub.status.idle": "2024-02-29T18:13:59.444612Z", + "shell.execute_reply": "2024-02-29T18:13:59.443652Z" + }, + "papermill": { + "duration": 0.374621, + "end_time": "2024-02-29T18:13:59.446943", + "exception": false, + "start_time": "2024-02-29T18:13:59.072322", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATAAAAEmCAYAAADldMx1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABtEElEQVR4nO2dd3hT5RfHvzeze0BLBxQoGwTKRkBZlqWigv6YskQUAQcVpCAUAaUoiChTUJZsBQFlKJQle5UhlFVaC6WT0t1m3t8f6b29N7lJkzZpkvb9PE8eyM0db5rke88573nPoWiapkEgEAhOiMjeAyAQCITyQgSMQCA4LUTACASC00IEjEAgOC1EwAgEgtNCBIxAIDgtRMAIBILTQgSMQCA4LRJ7D6Cy0Wq1ePLkCTw9PUFRlL2HQyAQ9KBpGnl5eQgODoZIZNrGqnYC9uTJE4SEhNh7GAQCoQwePXqEOnXqmNyn2gmYp6cnAN0fx8vLy86jIRAI+uTm5iIkJIT9rZqi2gkY4zZ6eXkRASMQHBhzQjwkiE8gEJwWImAEAsFpIQJGIBCclmoXA3MWaJqGWq2GRqOx91AIBKsiFoshkUisksZEBMwBUSqVSElJQWFhob2HQiDYBDc3NwQFBUEmk1XoPHYVsFOnTmHx4sW4cuUKUlJS8Pvvv+ONN94w69gzZ86gR48eaNmyJa5du2bTcVYmWq0WCQkJEIvFCA4OhkwmIwm3hCoDTdNQKpXIyMhAQkICGjduXGayqinsKmAFBQUICwvDO++8g8GDB5t9XHZ2NkaPHo2XXnoJaWlpNhxh2SQ9LcScff/i/R4N0LWhX4XPp1QqodVqERISAjc3NyuMkEBwLFxdXSGVSvHff/9BqVTCxcWl3Oeyq4ANGDAAAwYMsPi4iRMnYsSIERCLxdi7d6/1B2YBH++MRWxSNk7ey0Diolesdt6K3JUIBEfHWt9vp/uVbNiwAQ8fPsTcuXPN2l+hUCA3N5f3sCZPsousej4CgWA+TiVg9+/fR2RkJLZs2QKJxDzjMTo6Gt7e3uzD2usgSU8nAsF+OI2AaTQajBgxAvPmzUOTJk3MPm7mzJnIyclhH48ePbLquLREwGxKYmIiKIqq8ESNuec5c+YMWrVqBalUavaEEoHPiRMnQFEUsrOzbX4tp0mjyMvLw+XLlxEbG4spU6YA0M3Y0TQNiUSCv//+G7179zY4Ti6XQy6X23BkRMEqkxMnTqBXr1549uwZfHx8rH7+iIgItGnTBocOHYKHh4fVz0+wLk4jYF5eXrh58yZv26pVq3Ds2DH89ttvCA0Ntcu4iAVmHKVSWeE8n8omPj4eEydOLLOMiykq833TNA2NRmN2SMVcnOWzs6sLmZ+fj2vXrrFmfUJCAq5du4akpCQAOvdv9OjRAHSzFi1btuQ9atWqBRcXF7Rs2RLu7u52eQ+V0dicpmkUKtV2eVjy/nr27IkpU6bgk08+gZ+fH/r164d///0XAwYMgIeHBwICAjBq1ChkZmayxxw+fBgvvPACfHx8ULNmTbz66quIj48XPH9iYiJ69eoFAPD19QVFURg7dqxF57lz5w66du3Kfm9OnjzJnpuiKDx9+hTvvPMOKIrCxo0bAQAnT55Ep06dIJfLERQUhMjISKjVapPvm3Gj/vrrL7Rt2xaurq7o3bs30tPTcejQITRv3hxeXl4YMWIEL2FZq9UiOjoaoaGhcHV1RVhYGH777Tf2dea8hw4dQvv27SGXy3H69OkyP5svv/wStWrVgqenJ959911ERkaiTZs27Otjx47FG2+8ga+++grBwcFo2rQpAOCXX35Bhw4d4OnpicDAQIwYMQLp6em8cx88eBBNmjSBq6srevXqhcTExDLHYy3saoFdvnyZ/UICOvMdAMaMGYONGzciJSWFFTNHpTIssCKVBi2i/rL9hQS4Pb8f3GTmf002bdqEDz74AGfOnEF2djZ69+6Nd999F9999x2KioowY8YMDBkyBMeOHQOgywWMiIhA69atkZ+fj6ioKAwaNAjXrl0zmGoPCQnB7t278eabb+Lu3bvw8vKCq6urReeZPn06li1bhhYtWmDp0qUYOHAgEhISEBISgpSUFDRt2hTz58/H0KFD4e3tjeTkZLz88ssYO3YsNm/ejDt37mDChAlwcXHBF198Ifi+ASAlJQUA8MUXX2DFihVwc3PDkCFDMGTIEMjlcmzbtg35+fkYNGgQli9fjhkzZgDQTTpt2bIFa9asQePGjXHq1Cm8/fbb8Pf3R48ePdjrRUZGYsmSJWjQoAF8fX1NfiZbt27FV199hVWrVqFbt27YsWMHvv32WwOvJSYmBl5eXjhy5Ai7TaVSYcGCBWjatCnS09MRERGBsWPH4uDBgwB0dfUGDx6MyZMn47333sPly5fx6aeflv1FsRIUXRkmhAORm5sLb29v5OTkWKUeWOsv/kJuse5ubI08sOLiYiQkJCA0NJRN8CtUqp1CwHr27Inc3FxcvXoVgO6u/88//+Cvv0rH/vjxY4SEhODu3buCkzGZmZnw9/fHzZs30bJlSyQmJiI0NBSxsbFo06aN2TEwY+dZtGgRKxZqtRqhoaH48MMP8dlnnwEAfHx8sGzZMtay+/zzz7F7927ExcWxKyJWrVqFGTNmICcnByKRyOB9A6WxuqNHj+Kll14CACxatAgzZ85EfHw8GjRoAECX05iYmIjDhw9DoVCgRo0aOHr0KLp06cKe691330VhYSG2bdvGnnfv3r14/fXXzfpcnn/+eXTo0AErVqxgt73wwgusBwToLLDDhw8jKSnJpOt4+fJldOzYEXl5efDw8MCsWbOwb98+3Lp1i90nMjISX3/9tcnPSOh7zmDJb9RpYmCOSmWov6tUjNvz+1XClYSvbQnt27dn/3/9+nUcP35cMBgeHx+PJk2a4P79+4iKisKFCxeQmZkJrVYLAEhKSkLLli3Nvq655+EKg0QiQYcOHRAXF2f0vHFxcejSpQtvOVe3bt2Qn5+Px48fo27dugbvm0vr1q3Z/wcEBMDNzY0VL2bbxYsXAQAPHjxAYWEh+vTpwzuHUqlE27Zteds6dOhgdMz63L17F5MmTeJt69SpE2sFM7Rq1cpAvK5cuYIvvvgC169fx7Nnz3h/1xYtWiAuLg6dO3fmHcP9G9saImAVpRIUjKIoi9w4e8KNRebn52PgwIH4+uuvDfYLCgoCAAwcOBD16tXDunXrEBwcDK1Wi5YtW0KpVFp0XWudp7wYi8FKpVL2/xRF8Z4z2xhRyM/PBwAcOHAAtWvX5u2nP5Nui5iv/jkLCgrQr18/9OvXD1u3boW/vz+SkpLQr1+/Svu7loVz/CocGG318sAtol27dti9ezfq168vOEv29OlT3L17F+vWrcOLL74IAGUGpBkLgVtmyJLznD9/Ht27dwegcyGvXLnCpuUI0bx5c+zevRs0TbNW2JkzZ+Dp6VmhmUohWrRoAblcjqSkJF68q6I0bdoUly5dYifEAODSpUtlHnfnzh08ffoUixYtYhPAL1++zNunefPm2L9/P2/b+fPnrTBq83CaRFZHhaRRGGfy5MnIysrC8OHDcenSJcTHx+Ovv/7CuHHjoNFo4Ovri5o1a2Lt2rV48OABjh07xk7kGKNevXqgKAp//vknMjIykJ+fb9F5Vq5cid9//x137tzB5MmT8ezZM7zzzjtGrzdp0iQ8evQIH374Ie7cuYN9+/Zh7ty5iIiIsPp6VU9PT0ybNg1Tp07Fpk2bEB8fj6tXr2L58uXYtGlTuc/74Ycf4ueff8amTZtw//59fPnll7hx40aZVU7q1q0LmUyG5cuX4+HDh9i/fz8WLFjA22fixIm4f/8+pk+fjrt372Lbtm3s7G1lQASsgtAkkdUowcHBOHPmDDQaDfr27YtWrVrhk08+gY+PD0QiEUQiEXbs2IErV66gZcuWmDp1KhYvXmzynLVr18a8efMQGRmJgIAATJkyxaLzLFq0CIsWLUJYWBhOnz6N/fv3w8/PeBWR2rVr4+DBg7h48SLCwsIwceJEjB8/HrNnz67Q38YYCxYswJw5cxAdHY3mzZujf//+OHDgQIXyHEeOHImZM2di2rRpaNeuHRISEjB27Ngyq0D4+/tj48aN+PXXX9GiRQssWrQIS5Ys4e1Tt25d7N69G3v37kVYWBjWrFmDhQsXlnuslkJmIStI09mHoFDrYhi2moUkEKxNnz59EBgYiF9++cUu1yezkA5CtVJ/glNSWFiINWvWoF+/fhCLxdi+fTuOHj3Ky/dyVogLWUGqmQFLcECee+45eHh4CD62bt0KiqJw8OBBdO/eHe3bt8cff/yB3bt3Izw83N5DrzDEAqsgJIhPsDcHDx6ESqUSfC0gIACurq44evRoJY+qciACVkGIBUawN/Xq1bP3EOwGcSErCJEvAsF+EAGrIMQAIxDsBxEwAoHgtBABIxAITgsRMAKB4LQQASM4DPXr18eyZcvY5xRF2b3vJ8GxIWkUBIclJSWlzGqjhOoNETCCwxIYGGjvIdisaQbBOtjVhTx16hQGDhyI4OBgs9yFPXv2oE+fPvD394eXlxe6dOnCK1dcZaFpQFlg9QetyAetyDe9n4V5Inl5eRg5ciTc3d0RFBSE7777Dj179sQnn3xi8dvmfieYpht79uxBr1694ObmhrCwMJw7d453zOnTp/Hiiy/C1dUVISEh+Oijj1BQUMC+XlaTivI2zSDYB7veVgoKChAWFoZ33nkHgwcPLnP/U6dOoU+fPli4cCF8fHywYcMGDBw4EBcuXDAouVulUBUCC4OtflrT1aBKmPUEkJlf/TMiIgJnzpzB/v37ERAQgKioKFy9epXXAacifP7551iyZAkaN26Mzz//HMOHD8eDBw8gkUgQHx+P/v3748svv8T69euRkZGBKVOmYMqUKdiwYQOAsptUMFjSNINgP+wqYAMGDMCAAQPM3p8b4AWAhQsXYt++ffjjjz+qtoA5CXl5edi0aRO2bdvGNrLYsGEDgoOtJ77Tpk3DK6/oyhbNmzcPzz33HB48eIBmzZohOjoaI0eOZK29xo0b44cffkCPHj2wevVquLi48IoXNmjQAD/88AM6duyI/Px8Xu3++fPnG9SmJzgeTu3Ya7Va5OXloUaNGvYeim2RuuksIStSpNLgQbquBnur2t6mr20mDx8+hEqlQqdOndht3t7ebI9Ba8BtksHU1U9PT0ezZs1w/fp13LhxA1u3bmX3oWkaWq0WCQkJaN68eZlNKhgsaZpBsB9OLWBLlixBfn4+hgwZYnQfhUIBhULBPs/Nza2MoVkXirLIjTMPDWip7sdr/XPbDv0mGQB4TTHef/99fPTRRwbH1a1b16ImFfZqlEywDKcVsG3btmHevHnYt28fatWqZXS/6OhozJs3rxJH5nxwG1ZUhAYNGkAqleLSpUtsu7GcnBzcu3ePbaRhS9q1a4fbt2+jUaNGgq/fvHmzzCYVBOfCKRNZd+zYgXfffRe7du0qsyjbzJkzkZOTwz4ePXpktXFcTsyy2rkqnYrrlQGenp4YM2YMpk+fjuPHj+PWrVsYP348RCKRVQSyLGbMmIGzZ89iypQpuHbtGu7fv499+/axXYfMaVJBcC6cTsC2b9+OcePGYfv27Www1xRyuRxeXl68h7V4a825sndyVGxURWPp0qXo0qULXn31VYSHh6Nbt25o3rx5pdT3b926NU6ePIl79+7hxRdfRNu2bREVFcVOIpjTpILgXNjVhczPz8eDBw/Y5wkJCbh27Rpq1KiBunXrYubMmUhOTsbmzZsB6NzGMWPG4Pvvv0fnzp2RmpoKAHB1dYW3t4lANKHS8PT05AXRCwoKMG/ePLz33ntlHpuYmMh7zi0WWb9+fYPikT4+PgbbOnbsiL///tvoNYYPH47hw4cbvU7Pnj1JkUonwq4W2OXLl9G2bVs2BSIiIoK9awK6pSRJSUns/mvXroVarcbkyZMRFBTEPj7++GO7jJ9gSGxsLLZv3872NBw5ciQA4PXXX7fzyAhVEbtaYGXd7fQbZJ44ccK2A7ISzwqUGPLjObzRtjYm9xIOKFdllixZgrt370Imk6F9+/b4559/EBcXZzLnLz8/vxJHSKgqOO0spCNz7XE27qfn44/rT6qdgLVt2xZXrlwx2F5UVIRr165V/oAIVRoiYDZAW9KqSKXR2nkkjoOrq6vR9AYCobw43SykM6ApETBlBQSMBJIJVRlrfb+JgNkAbcmHo1Jb/iExmeaFhYVWHROB4Egw32/uyoryQFxIG6CugAUmFovh4+PDlnhxc3OzSRKoQqkBrdYtnykuLq6URFMCgaZpFBYWIj09HT4+PhCLxRU6HxEwG8C4kCp1+VxIppAft06VtVFptEjP1a0RlRa6EAEjVCo+Pj5WKVhJBMwGMC6kopwxMIqiEBQUhFq1ahltGV9RHqTn44t9unWAf33SHRIxiSYQKgepVFphy4uBCJgNYHRLqdZWaKG0WCy22getj0iiRHKeBgDg4uJCBIzglJBvrQ1g0iiA0niYI+P4IyQQhCECVk5MTQNrOK8pyxkHIxAIZUMErJyYMqw0nBdJMiuBYDuIgJUTjQkF0xILjECoFIiAlROtKReSI24VycavLEjSP8FZIQJWTswWMGKBEQg2gwhYOTEVA+OKm0rjmOYNTeYeCVUAImDlRD8GJqK4r5X+3xksMCJmBGeFCFg50eoJGDdZlRfE12gqbUyWQNmiqweBUMkQASsn+jEwrhzwY2COad0Qq4tQFajQUqL8/Hy2qSiDNbv+ODIaWt+FLJUwZ5uFJBCcFYstsISEBLzyyitwd3eHt7c3fH194evrCx8fH/j6+lp0rlOnTmHgwIEIDg4GRVHYu3dvmcecOHEC7dq1g1wuR6NGjQzq5luLm49z0PvbE3hz9VnB1/UnIbnLHXlBfGeIgRFjjOCkWGyBvf3226BpGuvXr0dAQECFyrAUFBQgLCwM77zzDgYPHlzm/ox4Tpw4EVu3bkVMTAzeffddBAUFoV+/fuUehxBSCYWHGQWo4S5cDUI/iE9Rwq8RC4xAsB0WC9j169dx5coVNG3atMIXHzBggMlONfqsWbMGoaGh+PbbbwEAzZs3x+nTp/Hdd99ZXcBquMsAAM8KldBoaYhFfKE2jIFxXEiaLCUiECoDi13Ijh074tGjR7YYS5mcO3cO4eHhvG39+vXDuXPGO2QrFArk5ubyHuZQw00nYDStEzF99EJ/fAuMk/ulcAIXkkBwViy2wH766SdMnDgRycnJaNmypUFN69atW1ttcPqkpqYiICCAty0gIAC5ubkoKiqCq6urwTHR0dGYN2+exdeSiEXwcZMiu1CFp/lK+HnIea+bnIUkFhiBUClYLGAZGRmIj4/HuHHj2G0URbGF+zQOlvc0c+ZMREREsM9zc3MREhJi1rE13WU6AStQAPDkvWZqFlJLlhIRCJWCxQL2zjvvoG3btti+fXuFg/iWEhgYiLS0NN62tLQ0eHl5CVpfACCXyyGXywVfK4ua7nLEZxTgab6hC6lfD4znQpJqFARCpWCxgP3333/Yv3+/XZqUdunSBQcPHuRtO3LkCLp06WKT69X00MXBsgoMBUzfM6R4eWCl253BhSRpFARnxeIgfu/evXH9+nWrXDw/Px/Xrl1jW84nJCTg2rVrSEpKAqBz/0aPHs3uP3HiRDx8+BCfffYZ7ty5g1WrVmHXrl2YOnWqVcajDyNgT/MVBq8ZxMC4eWBO4EIS0SJUBSy2wAYOHIipU6fi5s2baNWqlUEQ/7XXXjP7XJcvX0avXr3Y50ysasyYMdi4cSNSUlJYMQOA0NBQHDhwAFOnTsX333+POnXq4KeffrJ6CgVDDXed6/lU0AIzL4ivdNBqFARCVcBiAZs4cSIAYP78+QavWRrE79mzp8na8kJZ9j179kRsbKzZ16gIfqwFJpBG4eRBfK7FSNZFEpwVi11IrVZr9OFoM5AVhUlmFYqB6dcD48XAnCCNgriQhIpyOTELHb86ij9vPLHbGCwSMJVKBYlEgn///ddW43Eoapa4kJkFhjEws5cSOagFRiBUlFP3M5GRp8CJuxl2G4NFAiaVSlG3bt0qZ2kZw9QspEEaBef/WiewwAiEilKgUAMA1Hb8jlvsQn7++eeYNWsWsrKybDEeh6JmiQuZXagyECLDiqzC5XQUTiBgxJ0klIf8Yp2A2bNsusVB/BUrVuDBgwcIDg5GvXr14O7uznv96tWrVhucvfFxk4GiStdD1vJ0YV8zjIGV/t/ZSkoTCOUhX8kImP2+4xYL2BtvvGGDYTgmYhGFGm4yPC1Q4mm+voCZmIUkLiShGlBqgTmRgM2dO9cW43BYarjrBEw/DlaV2qoRD5JQHvIVTuhCMly5cgVxcXEAgOeeew5t27a12qAciZoeMtxPBzL1svH1Y2DcoD6xwAjVgQKFE1pg6enpGDZsGE6cOAEfHx8AQHZ2Nnr16oUdO3bA39/f2mO0K0wqhX4yq74Bxn3qbBYYgVAe8hzAhbR4FvLDDz9EXl4ebt26haysLGRlZeHff/9Fbm4uPvroI1uM0a4YS6UwtMCEXyNLiQhVlYKSIL7aVJdnG2OxBXb48GEcPXoUzZs3Z7e1aNECK1euRN++fa06OEeAtcD0kln1Y2Dc5Th8C8xxcuaYmm1C2wkES6Bpmg3i29PLKNdSIv0F3IAuyVW/xVpVoIaR9ZAmg/i8GJhjiENKThG6LTqGFcfu23sohCqAQq1lLS+nciF79+6Njz/+GE+elK5/Sk5OxtSpU/HSSy9ZdXCOgF9JMqt+RQp9q5mrZ464mPu7I/fwJKcYS/6+Z++hEKoAzAwkYF8X0mIBW7FiBXJzc1G/fn00bNgQDRs2RGhoKHJzc7F8+XJbjNGuGFvQbRAD477mgLOQahOWoGPYiARngnEfAfv2PrU4BhYSEoKrV6/i6NGjuHPnDgBdezP9bkFVhZolzTz00ygMYmC8IH7p/x3FAjPl8hIIlsK1wOw5UVWuPDCKotCnTx/06dPH2uNxOJj1kHnFaijVWsgkOqPVUBA4eWAOuBaSyBfBmvBdSCeywAAgJiYGMTExSE9PNwjcr1+/3ioDcxS8XaUQiyhotDSyCpQI9NYtJ9LXJZ4FpudCGpv9q0z0XV4CoSI4igtpcQxs3rx56Nu3L2JiYpCZmYlnz57xHlUNkYiCrxsTyC91I/UtsKcFShSrdCkTXAuMpu0b5OSOozyvEQhCMDlgAKBypjywNWvWYOPGjRg1apQtxuOQ+HnIkJmv4KVSaAU+tB0XkzC2W6hBz0iVRgup2OJ7hVXRLxtNRItQEfK4FpgdvQyLf1VKpRJdu3a12gBWrlyJ+vXrw8XFBZ07d8bFixdN7r9s2TI0bdoUrq6uCAkJwdSpU1FcXGy18QghNBMpdNN5mFkAwNBdc4RAfhVM0SOYoFCpxr/JOTZLUi7gxMBo2n4hCosF7N1338W2bduscvGdO3ciIiICc+fOxdWrVxEWFoZ+/fohPT1dcP9t27YhMjISc+fORVxcHH7++Wfs3LkTs2bNssp4jCE0E6lvZQFARp7udX3rTOkAgXxTbeBIhL/qMXffLby6/DTOxT+1yfm5QXzAfmESi13I4uJirF27FkePHkXr1q0NsvKXLl1q9rmWLl2KCRMmYNy4cQB07umBAwewfv16REZGGux/9uxZdOvWDSNGjAAA1K9fH8OHD8eFCxcsfRsWUVPAAhO6szECpi9uDmGBmUi8JVQ94lJzAQBPcmzjnXBdSEB3k3aRim1yLVNYLGA3btxAmzZtAMCguYclPrBSqcSVK1cwc+ZMdptIJEJ4eDjOnTsneEzXrl2xZcsWXLx4EZ06dcLDhw9x8OBBk/E4hUIBhaLUcsrNzTV7jAyMgHFjYEImc0aJhaZvcDnCciKy3rF6kZ6r+y5WhgsJ2G8m0mIBO378uFUunJmZCY1Gg4CAAN72gIAANkFWnxEjRiAzMxMvvPACaJqGWq3GxIkTTbqQ0dHRmDdvXoXGyriQ/FlIw/1YF9IhLTATLiShSqHR0my4w1a3LUdxIe07NWYhJ06cwMKFC7Fq1SpcvXoVe/bswYEDB7BgwQKjx8ycORM5OTns49GjRxZft4bAekihWchCpQYFCnWFg/hC564oBmm3NPc1Yp1VJZ7mK9gbrK0sMH0Bs9dNutwVWSuKn58fxGIx0tLSeNvT0tIQGBgoeMycOXMwatQovPvuuwCAVq1aoaCgAO+99x4+//xziESGeiyXyyGXyys2VoGKFMaW5mTkKSoUxL/9JBdDfzyHj15qjAndG5RjtMKQRNbqQ3peqadgq8hBtbfAZDIZ2rdvj5iYGHabVqtFTEwMunTpInhMYWGhgUiJxbrAoS1jPEJpFEKzkIAuDsa8Ji9ZdmTJ3elSYhbyFGqcf2j+7NHTfEPR1IeEwKoPabmlgXubuZB6QXx7FS2wqwsZERGBdevWYdOmTYiLi8MHH3yAgoICdlZy9OjRvCD/wIEDsXr1auzYsQMJCQk4cuQI5syZg4EDB7JCZguYGFi+Qi2Ybc8lI0/BWjvMrIwlH25esQqAcYHU53JiFtp/eRQf7og1uZ8pN5GIW9WiMiww/SC+07iQp06dQteuXSGR8A9Vq9U4e/Ysunfvbva5hg4dioyMDERFRSE1NRVt2rTB4cOH2cB+UlISz+KaPXs2KIrC7NmzkZycDH9/fwwcOBBfffWVpW/DIrxcJJCKKag0uvWQwT6ugkF8gC9grlIxcopUFn24zPS0uS7f6hPxAIADN1KwcoTx/Ugia/WBb4HZRsHyHMSFtFjAevXqhZSUFNSqVYu3PScnB7169YJGY1kJ5SlTpmDKlCmCr504cYL3XCKRYO7cuZXe2o2iKNRwlyEtV7ecKNjH1ajAZOSVupAuUp34WmKB5ZZYYObeORVmiiMpp1N94FpgttAVmqZZC8xdJkaBUuM8LqSxNU9Pnz416NJdlaihVxvfWMwtPa+YFR/GhbQkiJ9roQWmMLPmPtGvqsXNxzl4kJ4v+Fo6xwKzxQdfpNKwwuhTUujA4fPABg8eDEBnjYwdO5Y3s6fRaHDjxg2rrpF0NPRnIo3FqFJzS+9+rICVx4U084tn7rlNNSEh2uZc5CvUeGvNWXi7SnHxc8NCorwYmI2uD+hyCb1cpUjOLrJbRQqzBczb2xuAzvLw9PSEq6sr+5pMJsPzzz+PCRMmWH+EDoL+ciJjn1caZ+mGa3kssCLGhTTXAiMuZHXjWYESCrWWXfmhDy8GZoOPnZmB9JBJ2AKfDm+BbdiwAYBu/eG0adOqtLsoBONCZhYIL9hmSOV8eVxl5bHASmYhzbyjmXtuIl9VB+aGSNOGIR1dFn7Z+YoVgbHAPFwkkIl117ZXVVaLg/iVHUB3FNgGt/mMBSb8xcgpsaCAUgvMsjQKxoU0b3/zLTDjr5F1ks4F9/ukpQExJySdVaDk3fxsYoExAiaXQFKSJWCvuvgWB/HT0tIwatQoBAcHQyKRQCwW8x5VlZp6y4nM0aTyxMCYWUhzlxOZ654Skao6cL9P+pY6130EbBQDK7nJusslkFriQioLrT4Wiy2wsWPHIikpCXPmzEFQUJDda71XFqULuk1bYFyYNApz704qjRbFKq3Z5wcAhcq8WUgSA6s6cAVM/3PNyOPHxWxx42LKSXua60LSNHDyG+DWHmDsAcDdz2pjsVjATp8+jX/++YctqVNdYBd05wtXnBDCUguMW2PJ7BiYmRYYSWStOpgSMAMLzIZBfHdZqXwYvUnTNHD0C+DMMt3zOweA9mOsNpZy9YWsju6IfhpFWQImolA6Q2OmyORy4mdmW2BWCOJXv0/TueG26tO/0aXrW2A2+HTzOEF85vsn6EJqtcDhSODij7rn/RZaVbyAcsTAli1bhsjISCQmJlp1II4OY4EVqTQoVKoFY2BiEcX7P9PIozwWmLlpNebeS/RjatXwHlRl4IqFvmVdGRZYASeILxUZcSG1GuCPj0rF65WlQJfJVh+LxRbY0KFDUVhYiIYNG8LNzc2gpHRWVpbVBudIeMh1OS9KtRZP85WCVqifh265EQCIKIqtRmGuBcakUADWrwlGYmBVB6XGuAtpaIFZHzYPTC5BkVIXg+VVHdaogb0TgZu/ApQIeH0l0MbEQt0KYLGALVu2zAbDcHwoikJNdxlScooNpqoZArxcWAHTWWC6u5O5FlguR8DMzcQ3F5MuJNE2p4I3C6n34TGVWD1dJMgrVtsojUInWh4uEmQX6UIq7E1arQR2vwPE/QGIJMDgdUDLwdYfRAkWC9iYMdb1YR0atQKQlC6ZqumhE7CnBQpBganl6QIgBwAgpijISlxIhbkxMJ4LaVsLjGiW88IL4uvdSNWa0lJOecVqGyWy6m607pw8MJVGC6iKgJ2jgAdHALEMGLIZaDrA6tfnUq56YPHx8Zg9ezaGDx/OtkA7dOgQbt26ZdXB2RVFPrDxVeD4QtZEqcks6M5XCt7ZArxKxU4koizLkYFeDEzgkLImTzLyFIjYeQ2XEg3deGJlVR2UeomsXBjBEtswvYlJZPWUly4lgrIA2Po/nXhJXIERO20uXkA5BOzkyZNo1aoVLly4gD179iA/X7ci/vr161UrS//eYeDxReDk18DfswGa5iWzCrmQOgtMh0RUaoGZm+rAjYHpn/+nfx6i66JjSHpqPBlw/p+3sSc2Gf9bY9jVicTAqg6mXEjme8NMKNkiY4BxId3lujp5nijE0DsfA4n/ADIP4O3dQMPeVr+uEBYLWGRkJL788kscOXIEMpmM3d67d2+cP3/eqoOzK63eAgZ8o/v/uRXAwWmo6a7zuLMKlIKCUEvPArM8jYKTB6Z3/i8PxCElpxgLD8ax2/Tdh4RM4fIqgIALyXlOmno4FzwLzMjsskRM8Z5bk/ySG62HXAIPTS62yr5C3YKbgIs3MHofUL+b9S9qBIsF7ObNmxg0aJDB9lq1aiEzM9Mqg3IYOr8PDPwBAAVc+glvPl4EEbTIzFcICpivm4y983FjYOanUZRaYMbunNzrcr/IFAWIBZqasMeRRNYqg6lEVn0X0hZVbgpKLDAf7TMMuTURrUUJyBd7A2P+BOp0sP4FTWCxgPn4+CAlJcVge2xsLGrXrm2VQTkU7ccAg9cClBjNUv/A99IVyMkvFBQEsYhi3UwxxwIzdymRpZn4XAGTiChIRObHPYjN5byYWgvJWO4ixoW0wSedr1AjEE/R8MD/ULMgHum0D1bW/wEIam31a5WFxQI2bNgwzJgxA6mpqaAoClqtFmfOnMG0adMwevRoW4zR/rQeAvxvA7QiKQaKz2NCyjxQWsNaTBQAf0+dGykSweJE1lwTMTD2GhyNUqhKzyuiTAuYSUEkauZUmLLAWBdSZBsXUqul4aN8gl2y+ZBlP0S+PBBDlHPwWFLPuhcyE4sFbOHChWjWrBlCQkKQn5+PFi1aoHv37ujatStmz55t8QBWrlyJ+vXrw8XFBZ07d8bFixdN7p+dnY3JkycjKCgIcrkcTZo0wcGDBy2+rsW0eB0J4WuhoKV4XnUek1NnwwV8EaOoUgETU5bHwLgWmLEvnoijYFwLjKZL4x5CkCB+1UG/nA4X1oVkLTDrUpR6F7tk81FXlAGtbyj+fn4jEukgqJ2lJr5MJsO6desQHx+PP//8E1u2bMGdO3fwyy+/WFxOZ+fOnYiIiMDcuXNx9epVhIWFoV+/fmxqhj5KpRJ9+vRBYmIifvvtN9y9exfr1q2rNNdV1qw/xqmmo5CWo2XRFWyUfQN3FPH28fdgLLDyLCUqO5GVa4Hpz0ZJTMTA9M9G9Mx5UZpYC2nTWci023DZ8iqCqSzcp2uDGncQSnfdb48rqkUlHeorg3L3haxbty5efvllDBkyBI0bNy7XOZYuXYoJEyZg3LhxaNGiBdasWQM3NzesX79ecP/169cjKysLe/fuRbdu3VC/fn306NEDYWFh5X0bFlHTQ4az2pYYpYxEIeWG50Vx+EUWDS8UANBZR1wLzNLGtrlmxMAocCwwPVeC60LqN/uojgvwqyoKEzEw5mNmBGz5sQeCeYEW8+QasPFliAszcFtbD+9R80B5BZfepEvivAq1Br2/PYE3Vp6pFKvMrEz8iIgILFiwAO7u7oiIiDC579KlS826sFKpxJUrV3iNa0UiEcLDw3HunGEeEwDs378fXbp0weTJk7Fv3z74+/tjxIgRmDFjhlHrT6FQQKEodfVyc3PNGp8QbjIJXKQiXFE1xSfy+fimaC7aiR5gm+wrjFJGAlwXkmOBmeNC0jStNwspvJ8xC4ymSwO3AFCo0EAuKf2bkBBY1UH/c+fCuJDcm9nyYw+w+Z1O5b/go4vAlrcARQ4K/MIw/PFkePjUBAA2WZsRq9ScYqSU9IU4/zALLzS2Xu0vIcwSsNjYWKhUKvb/xrCkuGFmZiY0Gg3bxJYhICAAd+7cETzm4cOHOHbsGEaOHImDBw/iwYMHmDRpElQqldEk2ujoaMybN8/scZVFTXc5krOLcLowBMNUc/CLbCFaihKxU7YAmUWt4O9ZA4DOGpNZYIEp1FregljjLiQ3Bsa3srjXyVeo4etemqdnGAMjsuWscG+IxhJZubHSx1kVqISacArYNgxQFQB1u+LG86uQs/k2AuU66WCqUTBjyi4svQkfuJniGAJ2/Phxwf9XNlqtFrVq1cLatWshFovRvn17JCcnY/HixUYFbObMmTyrMTc3FyEhIeUeg5+HDMnZRShUanAHdTFMOQdbZQvRRJSMOjEjkTd0N3zdpOjWqGbpYm4zLDBuLTDAlAtZCncWEtCV+mFf0xNNa1e3INgPU2kUzFPuhE5ydpHRfq4muX8U2DkSUBcDDXoBw7Yi514eAMBdrrPu9V3IbM73+K9bqVjw+nOQiMsdqSoT2525DPz8/CAWi5GWlsbbnpaWhsDAQMFjgoKC0KRJE5672Lx5c6SmpkKpVAoeI5fL4eXlxXtUhBocqwYA4una+J8yCo+0/nDLS0TAr4NwZVIjfP5KC04emLbMGBQ3/sUgdMyjZ6V3U/1F4kxpEwAo1is1bSqIT8JjlceT7CKzq+0agz/7LLzCgmuBKdRathS62cT9AWwfphOvJv2B4TsAmTunI5GujJa+C8ltapNVoMSFBNuW1zLLAmOa2prDnj17zNpPJpOhffv2iImJwRtvvAFAZ2HFxMRgypQpgsd069YN27Ztg1arhahkxu3evXsICgriLWuyJUxtfC6P6AAMUUYhxv9buOUkQrTpZWD0fsjdQwHoBEKjpU2mOTDxL0+5hK14KXRMbFI2Tt3LQPcm/gauaaGyVATtbYEVKtWIS8lD2xAfXmyuOnP8bjrGbbiEl5rVws9jO5b7POYksurnBD7JLoKfwHdXkJu/AXveA2gN0OINXUkcie73VcBZyA0YupA5hXyhPHAzBd0a2c6NNMsC8/b2Zh9eXl6IiYnB5cuX2devXLmCmJgYtvmtuURERGDdunXYtGkT4uLi8MEHH6CgoADjxo0DAIwePZoX5P/ggw+QlZWFjz/+GPfu3cOBAwewcOFCTJ5s/UqPxqjpLiyUKaiJm312AP7NgbwUYMMAyJ6WVucoy41kLDAf99ICkcbiYHEpuokIfQHjWmD6s5CV7UEOX3cBb64+i60Xkyr3wg7Mz/8kAABi7ginCZmLqcXcWjaNgv/TTn7GT/cxytVfgN3v6sSr9TDgzZ9Z8QJKK1GwLiSb61jiQpbEwOr46hpf//Vvqk1nI82ywJimtgAwY8YMDBkyBGvWrGFdOY1Gg0mTJlnsng0dOhQZGRmIiopCamoq2rRpg8OHD7OB/aSkJNbSAnT1+P/66y9MnToVrVu3Ru3atfHxxx9jxowZFl23IjD9IYVQu/nruq5sGQSkXIfL1tfRmvoUN+iGUKlpwISRyFhgPq4yPCrJLTO2fjGr5C5nYIGZiIHpLymxtZ5df5QNAPj18iOMet4+WdqOhrUq3HCXpunf40rTKPjbk7PNELALa4FD03X/bz9OVwZaTwhLe0LqbrQSfQusxIXs91wgdl99jKcFSlxMyEJXG1lhFhc0XL9+PU6fPs2LQ4nFYkRERKBr165YvHixReebMmWKUZfxxIkTBtu6dOli16oXTIduISgAcK8JjN4PbP0fqMcXsVW2EOOU06HQvARAavRYJgvf27V0H2PZ889K4hn6IlXItcBUurhbVoESNT3kZaRR0EjMLMC9tDz0fU44/khwHJQc69qYCynWcyHLFLAz3wNHonT/f36SrgGHgOKWlpPmB/HZWcgSAavpIUO/FoHYefkRDtxMsZmAWRzEV6vVgmkOd+7cgbYalDwwZYGxuPoAo34H6r8IT6oIm2VfQ5Rw0uQhzCykt1vZLuSzEjNdqTaeRqFQazD/z9to/+VRxMSlCQR7+efsueQE3vvlCk7dyzA5zrupeZiy7SriM4yX7uFCol/WR2kijaJ0KZGZLiRNAycWlYrXi9OMihfAscBcdLaPjA3i811IH1cZXm4dBEA3G1nRiQtjWGyBjRs3DuPHj0d8fDw6ddIlx124cAGLFi1iY1dVGWMxMAD8X6vcAxixC6e/6o8XqOtw2TcKcPkFaNJP8FDGAvPlCJixwDtjgZmKqynUWmw4kwgAWHToDs8C02hpXlzi60OlN6TYpGx0b+Jv9LyDVp1BoVKD2KRsnIk0o2hdNWl8XJmYKinNplHoB/FzBASMpnXCdfYH3fPec4Du00xe25gLyXwX2RuxqxRdG9aEj5sUmflKXEh4iq4NrW+FWSxgS5YsQWBgIL799lu2rE5QUBCmT5+OTz/91OoDdDSEZiEZKH17Q+aGaeJIzFN9i364DOwYCbz1M9DidYNjmRgY34XUfVnZsr0lGIuBcVHoLTPiuqPhS08iIbOAfb732hP2/8bKrxQo1MjMV7BuqlkxFQIPa3Wx5y8h47+mFUhkBQQsMK0WOPQZcGmd7nm/aKDLpDKvzTa1NepC6r6bPm5SSMUi9G0RgF2XH+PgzRSbCJjFLqRIJMJnn32G5ORkZGdnIzs7G8nJyfjss88sXsztjJiywASzBSRyTFZ9hOyGrwFaFfDrWOD6ToPdcgViYHdT8xA2728s/ovvsrMWmCkB4wT0aZrvMnLFy1x6LjmBHotPWHwcsb+sD2/FhoEFJpxG8axQVZpmo9UAf3xYIl4U8Ooys8QLAApKzuFZhgvJfI9fbqVzIw//m2YTN7JCiazWSAx1NlykYrjLhIVa6A4rk4ighgTxL3wHtHkboLXA7+8Dlzfw9mPzwFykbAD2+uNsFKk0uPCQnwyYXaSCRkub7Mqtb4FVlIw8w/pn5kA8yFKs9acwXZFV969YIOfwSXYRoFHpcrxit+h6Ng5aA3QwP/TDWmAynYAxQqnW0tBqaXYW0qckFNKtkR+8XaXIzFfgog2SWi12IQHgt99+w65du5CUlGSQAX/16lWrDMyRqeEhQ0GWeS4Us5xIRVPAa8sBqavuzvfnJ7os5+c/AFBqgXm6SCCmKGhAs3ld+XqlSWhaN11tvoCZ/dasnpVP9Mu60DRttLEtNx4m1JUoOTMbjY5FAnf+1PVsfPMn4DnD8vCmyNML4ks54Y08hZr93jEWGONG/npF50Z2aVjTouuVhcUW2A8//IBx48YhICAAsbGx6NSpE2rWrImHDx9iwADbt1FyBGpyUikm92rI/l/I2pCVVIRQqrW6nJqXFwNdP9K9eDgS+OdbAKXBTy8XKXue4pJZxjyBZUbPCpVlBPFLXUhLLDCyqsh2WMMa1f/MuW4Z93PWT6OQQ4nGx97XiZdYDgzdarF4AaWZ+B4lmfgyTsIZ01RXLKLY1wGws5GH/rX+bKTFArZq1SqsXbsWy5cvh0wmw2effYYjR47go48+Qk5OjlUH56j4cVIpwpuXVtMQ+n4aVGWlKKDPfKBnyQqDmPnAsS+RV8S4kBL2y8cs1tZf6A3o4mCmY2DGS66YxMommLUC11UBa/wl9D9zngXG+ei4AuaGYmyQfoPgzDOcno39Lb62RkuzkziMQHFjbZklYQYfVynvc+/W0A9eLhJk5iusU5uMg8UClpSUhK5duwIAXF1dkZenW50+atQobN++3aqDc1S4C7q5XxRBC4ypSMH94lEU0DMSCC8p83NqMd5X/AyA1sXASk7EupBKQwssqwwBM6e+fmVQXeXryn9ZmPHbDWRZuoi6DFR6DWK4qZdcMWOExQsF+EUWja7i2ygWuQGj9gANeyE1p1gXE7OAAs73kHEhxSKK/d5n5uveK3ciCtDdxAeGBSO8eQBcpNad6LM4BhYYGIisrCzUq1cPdevWxfnz5xEWFoaEhIRqU/WTm0rBn64WDuIDRnK2XvgEkLoBh6ZjNA5AJCmGl0tvdvEzUx5H6M/6rFBpsN6R9zrnh2MsIZZgO95crSvKWazW4PthbQFYxxrVv2lpaOMupC9ysVm2CK1Eicih3bDIJxrR9bpCodbg+egYAMC9LwcYpOkYgwngS8UUWyyToihIRSIoNVpk5OkKGXKTsRm+fKOlTaxxiy2w3r17Y//+/QB0Sa1Tp05Fnz59MHToUMF+kVWRmhZYYGXWxe/8HooGfA8tTeFtSQxqHo2AFDph0i+JwyWrQGXSAsviFJbjLvIuC2tLXXX3IM1dsWAuBi6kVtiF9FQ/xXbZVzrxEnljmHIOThXWBQBed/dCAeveGPl68S8GZqKKscB8XA0FzFahBIstsLVr17JLhiZPnoyaNWvi7NmzeO211/D+++9bfYCOCHc5EdcCE4yBiU1YYCU8azoUi/bdx1LpKkhubMdC6j9MwkQUmxCosoL4XAtMfxbTFNafhazeCsb9e1olBqYxXmWECRUE4imG/huJGqJHSKN9sCH0e8TdkkKcWwy1RovHHNdR3yU1RWklCj0Bk4gApYYN4uu7kLbEIgtMrVbjyy+/RGpqKrtt2LBh+OGHH/Dhhx9WWk0ue8Nd0M1d9S90l2HLjZgQo7xiNfZruyJS/CkgkqIvfRarpcugURiPUZQVxH9m5dgLoXxY+4agnzrDdSFpmkYIlYZfZfNRozgJj2k/DFFGQeTfDFIxBY2WRlqegmeBmdvyD+Au5OYLGNMNi8kV9HGrPB2wSMAkEgm++eYbqNWV0zLJUeG6kFQZFpjcDAuMCbhfdu0GDN+OYsjQR3wV07LmGvSeZHhWaFrA8srZ1srqnZyrtwFmdZfcMIjPeZ55H7tkCxAiykC2awiGKKLwHx0IiViEIG9dfa7kZ0X4jyNgagssMLaYoQtfwGSsC+ngFhgAvPTSSzh50nRlhaoO14XkJgyaioGZMtW5Wfho3AfTpbNRQMvRVhWLTbKvDXpPArpZSFOJrOWFJLJaF2tPbBlNo0i7Ba8dryGIysI9bW3sb/sTnkC39lBEAcE+LgB02fhJWaVLyczp18CQZ8qFhPFZSFticQxswIABiIyMxM2bN9G+fXu4u7vzXn/ttdesNjhHhZtGIaIotAnxQXpuMZoFGi6rYmZ4TIkNk6jq5ar7OK5JW2FU/kxsln+DzqI72CpbiNHKGciFB3vMs0KVoDhIRBTUDtTAo7oH8XkxMGsksuq7kFoaSL4KbBkMUdEz3NLWwxjVTEyQ+wN4CkD3Ha3t4wYgC8nZfAvMOi6k7o2VupAOLGCTJukWfQr1f6QoChqN+TNezopcIkajWh54kl2EGh4y7Pmgq66xrED3FXN6QzKJqp4lJUpEFIWrdBN8IPkCP6jmo40oHttlX2GUciayoBPJrAKlwRcJAFxlYsHMfYJ9sNQlV6g1uPk4B+3q+gr2EtAP4vs/iwU2fwIocqEMbIfhiRNRJPbkiaVYRKF2iQX2+FkRkrIq5kIazkLywySVKWAWu5BardboozqIF8PuiV1x7NOe8JBLIBJRRltHmdMbkrsOEih1S6+p6mOYcjYyaG88J/oPO2QL4I9nJceoBNMs3IwsNDcXq6dRVHMnku9Blv23+OXcf3hrzTlM2iq8ppj7PeoiuoWXr00CFLlAvW7IHLwLufAARVH82XEKqF1So/7ao2yeN2CJC2k8jYL/3XfoGJgtWLlyJerXrw8XFxd07twZFy9eNOu4HTt2gKIotqtRZeLtJkWgt0uZ+xksJRKg1IUsscA4iax36boYqpyDFLoGmoiSsUs2H8HIBE2DbZUl5yQiusnKtT6fxeoxMCfUL2uuXOCvQzV93pP3MvDlgTgAwOFbqbzZQgamHn5PUSw2Sr+BVFsMNOwNjPwNGokunKNvuIkoCsE+OgFjGsIwWORC6i3kZpDqVb7wdnXAWciioiL8+eef7HOmYSzzmD59OoqLiy0ewM6dOxEREYG5c+fi6tWrCAsLQ79+/ZCebrpzS2JiIqZNm4YXX3zR4mtWJoJLifTILS5dBwmUfgGZWNZDOpjtPRkqSsMu+XzUo0pTWVw5VlfFLTDHiZ/Zg33XktFy7l84frdinYMYuH/NwjISises59+4Y+6kGeyjVGvRT3QRa6VLIadUeFije0nPRrfSctIUxZsdF1MUapcImD7lErAyLDCHdCE3bdqEH3/8kX2+YsUKnD17FrGxsYiNjcWWLVuwevVqiwewdOlSTJgwAePGjUOLFi2wZs0auLm5Yf369UaP0Wg0GDlyJObNm4cGDRpYfM3KxBwXMo91IUtjYPo8pmvhf8ooxGuDUIfKxC7ZfDSkkgEArpz1Ze4VtMAyci2v+1WgUOPdTZex5+pjg9eczQL7eMc1FKk0GLfhknVOyFGwsgRMH6H+BEFJf2Cl9AfIKA3+1DyPA82+ASS6vETGcBRRfMedosBaYPpYEgMzFsR3Chdy69ateO+993jbtm3bhuPHj+P48eNYvHgxdu3aZdHFlUolrly5gvDw8NIBiUQIDw/HuXPnjB43f/581KpVC+PHj7foevZAP8ApBJNG4cVZICtEKmpiqDIKCaJ6CKCysVO2AM2p/3gC5lpBCyyzHAmwP556iKNxaYjYdd3gtWofA+P8n7tsx5z0ivMPs/jrXa9uRtfrMyGhtPhN0x0fqaZARZV+3owFRlF8N1JEUXCRinlVVBjKEwMzSKPguJDuMrGBoNkSs6/04MEDtGrVin3u4uLC69nYqVMn3L5926KLZ2ZmQqPRsH0gGQICAnjZ/lxOnz6Nn3/+GevWrTPrGgqFArm5ubxHZWJWEL+IkwcG4wIGAJnwxsJaS3BDGwo/Khc7ZAvQmnrAvl5hF7IcQTCS9W8c7t+Ta4Hpx9mEmr8WqTS4kqibtMGFH4H9H4ICjV/U4Ziueg9aiPhrIdmmtnwXkvk6CbmR1omBlepAZWbhAxYIWHZ2NhSKUvciIyMD9evXZ59rtVre67YgLy8Po0aNwrp16+DnZ16DgOjoaF5n8ZCQEJuOUR9z0ijYIH7JF6Osha8u3v4Yqfwcl7VN4E0VYmH+bHSkdHXz9S0wuZmVBiqCqbiZs7mQ1sZYDEy/Qsihf4Vv2KfuZwKnv9M14ABwrc5IzFGPA13y0xWqByaiKIM0CkDYjbREwNhMfBMupFcluo+ABQJWp04d/Pvvv0Zfv3HjBurUqWPRxf38/CAWi5GWxg9WpqWlITDQsMFqfHw8EhMTMXDgQEgkEkgkEmzevBn79++HRCJBfHy8wTEzZ85ETk4O+3j06JFFY6woclPldErQn4UUKGfOw89Dhjy4YbQyEme1LeBGF2GzbBG6iW4aWGC+FbwjPitQ4rUVp03uw/0tmrI0qyM0LwZW6kLqt1DVrxvWrVFNADRCri8Djn6h29j9Mxyr+zG46RhC5XQogxiY7pmwBWaFxdycL6xQJQpbYraAvfzyy4iKihKcaSwqKsK8efPwyiuvWHRxmUyG9u3bIyYmht2m1WoRExODLl26GOzfrFkz3Lx5E9euXWMfr732Gnr16oVr164JWldyuZxtPmKPJiSsBaY2/kXRn4U05UICgF9JPbJCuOBDahauuXSEK6XEeukStC7gdy23dEZI34Nc989D3HhsutIu95AzDzJ5r1X3iqxcC6mYUyVX3wLTXz3Rq4k/Zkq2YWRxSZHQl+YCvT9n0yjY83OO07AuJPRcyBIB87WSC2nCAqvMGUjAgkz8WbNmYdeuXWjatCmmTJmCJk2aAADu3r2LFStWQK1WY9asWRYPICIiAmPGjEGHDh3QqVMnLFu2DAUFBWyT3NGjR6N27dqIjo6Gi4sLWrZsyTvex8cHAAy2OwpMOR2FkS+KWqNlXQsmBlbWj54bjBXLXPG931wM/W8e+osv4c0HM3BCNBkHtc8DsHxGSN8dNKcYIm8XJ9crmVhkUWC7LJi/jX7xSYN2aJznFLQIT1yM+pIDAIDMF+bD78WPAQithTS8lqELqfuX60K6ycQoVGpMVknhotJoWQE2WErEEbDKnIEELBCwgIAAnD17Fh988AEiIyPZ4CRFUejTpw9WrVplEIw3h6FDhyIjIwNRUVFITU1FmzZtcPjwYfZcSUlJvMkCZ6OscjrcWl36mfjG4AZKPeQSaEW63pPfYjXeEJ/FculyuKiU2KPtXqE7olZL82Y4jcENVOuP3Nn0TC6xroAx6C/v0u+ozVhgImjxtWQt6j88BS0ozFKNx9vNx4KJ+OpbTEJNPXRpFNxMfEMX0k0m0QmYmS5kAed7qu9CyjgupFA1VltiUdJQaGgoDh8+jKysLDx4oJv5atSoEWrUqFGhQUyZMgVTpkwRfO3EiRMmj924cWOFrm1ryipoyPTKc5WWTj+X5UJyF5O7y3VNQDQQI0I1Cc/VDUDj5N+xVLYGLioVtG5jyzXu747cw8aziXi1pKOMKfgLlvljdzYPUi4Vl7sUkRCMqOTrCZi+ZavRaiGBGt9JV2Gg+Dy0lBgLZR9iR3EnvMlZMsZYYCJKZ31xXVTmnCIR/+8uEhAwxmhS6QfjjMDcaGUSkUEJaq4F5lOJWfhAOftC1qhRA506dbL2WKokxtIoaJrGz6cTsPCgbulIz6b+7Gtl/ei5gXl3uZidJtdChOtt5+P0f4UYJ/kLC6U/42iOJ3bA/M+K+T18H3MfALD1QlLZx8C4BeZsWHvWlvl7lmWB0WoFVkm/R1/xFShpMR68+D1O36wH5OTxSoIXloiZq1SMAqWGJ2A0xwITCbiQXGu8QKE7j6nYLJd8IzOQAD8GVtkupPP6Zk6CTCCNolilwae7ruPLA3HQ0sBb7evgu6Ft2NctscDkEjEvc99dLsU89WisUQ8EAIQnLcNk8V6zx1uetZCmSsY4m6BZXcBKxD1PwW+Nx7PAlIUYePtT9BVfgYKW4j1VBHJDX2ZnlLnpF7ef6CZUGtXSlVZivlZFSg1m7rkJwNCFZL4fFEVhYFgw/Dxk6NtCF6IxN4hfYCQHDOC7kJUdxCcCZmP0LbCUnCIM+fEc9sQmQyyiMHdgCyx+qzWv3VRZMTDuXU6l0fIET5cHRmGRehi+17wFAJgu3YVpkp0wp9ZEedZC8mP4+i6kc0mYqQ49NE2zqybMhTG0DFxI5gVFHvJ+fgP1s8+jkJZjnGo6TmjbQioWsQvzi1S6Y58VKBGfoStG2KG+LmzDWHJ/307FvTRdAxERBd6dg/sZ/DCsDc7PfIktymmuC8lYkEJL1fguJBGwKgWTI6PUaPE0X4GBy8/gxuMc+LpJ8cv4ThjXLVQgbsRZiCuiDHK7uIKlVGt5FlhpNQoKP4uHILbZpwCAKZJ9mCPZAlv03rZ20T57IufcSPRXJcz74zZaffE39lx9bPGKBUMXEkBRNujNg+CZdgF5tCtGK2fgrFY3my4VU2xSMmOBXU3SZeU38HdnBYhxIe+l5bHn1l8Lya8arCv9ZE56DxdjWfi6sTpBIiuhfHAtsBN3M5CZr0AdX1fsn/ICujYUXk3AXUomE4sMZn24qDRaXuE7rti5ysR42PgdzFbpUlLGSw7hK8l6UDB+1y2PC8mdoapKLqT+xMvGs4kAgIhd17H4r7tmnY/5e+p3hqILM4FNA0ElX0I27Y6Rylm4TDdjX9dZYLrPkomBXflPJ2Ad6vmyosS4ovfTStu3ifTqgQlFJMxZIcLFWDFD3bmIC1ll4cbAmE7IXRvWREgNN6PHcC0suVQEdxPrGxVqLS9znytgLlIxZBIRtmj66NbO0RRGSmKwRLoGYliv+OThW6XLYAxdSKtdplLgClix0viPe9UJw1UfgPBkDQCe6+mPZwj8/S0g9QYU8poYppyDG3RD3nFcASvUE7D29XxZgWJcyAfpHAET8fPABLtllXxpzBWwPCOVKJixMjjsWkhC+WAsMC0NPHqmK1BnrLQJA/fuKZeYtsDcZGI9C6x0XxdJaWrGr5qe+EQ1GWpahDfFp/GDdDmkMEwXqKiDafhbcS4F4/7Yi0w0Fhbin/sZaDL7EDacSWC3MX9PJjUjCE+xU7YA8qy7gGcQtjVfjTt0XYNzycQiuEp1n6UuX0uL64+zAZQIWMlnrqV1SbKJT0sbdYgo/TQKw7Ga02yGCzNracqFFIsokzdbW0AEzMZw706JJRU2g70tETCxYOB0+fC2aFTLA9GDW/P25y7mlktFkElKX9uv7YpJqo+hoCV4RXwRq6XfQQ7rVpJwLrkyhOYt/bFMwKbuvAZAFyvTP19esRp1qTT8Kp+HBqJUKD3qAOMO4lxuTcFzScQUx4VU44/rT1Cs0sLHTYoGfh6s1a2haSRkFvAy8vVnIYVmtS11IfNLZlGFLDBJyWB8XKWVPmlDBMzGcGe1/iu5S5ZlgfFcSIkI7nLDu9rAsGAcjeiBpoGevCCti7T0elwLjOFvbUdMUH2KYlqKcHEsfpIugSs461stMMFomsaak3xXytkTWXl5VRYeKyQUzDnccuKxSzYfdahMPNQGImHgb0CNBrzgOxepWMTejA7fSmVrrTHNPlgLTEvz3EfA0IUUKpApNaPUORdj6yCB0jBJZWfhA0TAbI6E86VOK6l2yvToMwbPApOK4GbChQRKa+jLJSK2SzJzrExs+BGf0oZhrGoGCmg5XhT/i02yr+EBnXVoSRrF8bvpWHToDm+bswfxuVkFlsw0/ngynv18+eejgdR/8VHSxwiknuGutg6GKqOgcA9CoVKN/7IM694DOlFgLDDuedvX8wVQ+h3R0jQvgK97Ta/hspALKbIsBpZf4kIKhTOYuGFlp1AARMBsDkVRBrlFZcfASv8vE5sO4nP3l0tEvGPlEjF7p9XnvLYFRilnIpd2QyfRXWyRLYQ38o3OQg5qW9tg26Msw4a7Gi2N8RutVI7ZDpTHAlOqtYjWE3KGFngAbHwFXtps/Kst6TIFH2hKLCdjf2+uC8mFETDG2tNoYWiB6adRmHQhzUyjYCqmCAhY14Z+6NsiABNerPzy7kTAKgE5xwqq6S7jJa0KwXchxSaD+Nz95VIx784rEVGCFhjDVboJhis/RxbtgTaih9gu+wpe2mzBfYXcECGO3UlHzJ3ShhjO4EKei3+KF785hlP3MvSW5ph3fJGRWvcdqDv4CQuA4mzEiZtihPJzPCvp66mladxJFXYfAZ3ACH1Pwur4ACi9aWlpGvfT+eehadrqLqSpIL63mxRrR3fAgFZlr5u1NkTAKgGuFRRUhvsIgDerqEujKP3STOvbxHB/qtSF5J/HdGY5ANyiQzFMOQcZtDdaiP7D/KzPgNwUg/0kAndx/dwmAFDoBb6doSb+8HXn8SirCKPXX9QrT2OeghWrDQWsq+hfbJZ9DQ8UAfVfxEeSuchFaRd7jRZIyCwwOI5BKqYMWuSN7lKPjYsxn7lSrTU4j0ZL61WjMDw/s/wnLiUX0YficPR2msnS4HlGihnaG8caTRWFawWVNQMJQM8NFKFuSc7YpJ4NMaV3Y4H9hQWMoiizGizco0MwRBmFrbKvUEfzCNgwALXxCZJRusBcqEu0UDKnsy0d0qc8LqT+bGUvUSzWSJdBTqlwFm3QdcQupEefBqACReksO42WRlqu8TaEFGXoQo7oXJpuwVjdCZkFUGloyCUitmGtWkvzF3MLfCYN/D0gFVN4VqjCjycf4kc8BKBbY9mxvi/a16uBjvV9UbeGGyiKMjkLaU8cazRVFCknlaGs+BfA/8LJJGIMalcbtX1d0a6ur/D+JRoll/C/8BQMm44aI4EOwhDlXPzmGo3AZwnYJZ+PEcrP8R+tK+0tZIGZgyPp2en7majv54Y6vsaTiIUKBJYFt9Jqf9FF/CBdDhmlwV+aDvhCNg1npa6sterjKsWzQhW0NI30MlrY6fc34JfD0f1hk0uSo5sEeOJmsm6ht1qj50IKfHZNAjxxcnovnIt/isv/ZeFS4jM8SM9nH9sv6kqv+3vK0aGeLzLzdNYZEbBqCNcCM9ZglItIL41CKhahWyPjTUzYWUipngspMIFgise0P2b7foOfsAC1n97HLtl8jFTOwgO6TpkVMhj03S5HEbDzD5/i7Z8vAAASFxkvfc4dv9ZMBWMSXt8Qnca30tUQUzT2a7ogQvUBarhIUaTSsIu3vUsErCwLDOCvqvBykbAVewFDS7dxLQ9WwHQLtE0vJQJ0N9M329fBm+11vSyyCpS48t8zXP4vC5cTn+HG42xk5Cl4DUc8BWJg9sSxRlNF4bpx5lhg+pn45u6vv6+4jCC+EE/FfqBHHcCdxS+huegRdsoWYLRyJkRUqFnH6//kKVBQqrUWCaktuJyYZdZ+3AqnlgTxh4mPYaHkZ4goGrvUPRCpngAtRKCo0koUuoX5up+chjZDwKSlP8/aelajvlvYKMCD9x7KCuILUcNdhj4tAtCnpNROsUqDG49zcCkxC1f/e4Z6Nd0R5F12DLcyIQJWCcgtDOLrz0KWuT8rYHouJGXYNbksYpOyEfrVZXhjDjbLFiFM9BDbZF9id0EtAMZdLwb9H33MnTS0iDqML99oiWGdDJfMVBbGYnPcTkFAaYMVwPycOL9/f8Yi6U8AgE3qPvhCPYZteyaiKORy1hEyWetFSg27nYmL6cN1IfUtd/2PtXEtT/b/ag2tt5i7fGawi1SMTqE10Cm0YhWXbQmZhawEpJa6kHqJrGXuXyJ4LgIuZHm7JOfAA28rZ+FSSe/Jkfc+Qicqrszj9H/0xSot1FoakXtuQqulMWTNOby+8oxgI1d78PGOa7zn3Nw2syywf75F49ivAABr1K9irnosK16A7jPgZrEzn21Kjs76cpGK4GLkJsV1IevodRQSciEZ1Fotb+63vALmDDiEgK1cuRL169eHi4sLOnfujIsXLxrdd926dXjxxRfh6+sLX19fhIeHm9zfEWDcJ6mYgn9JSzRT6JfTKXN/IxaYiDI/iC8E03vyjOY5yLWF2CT7Gi+Kbpg8RmviR/+sUImLiVm4/igbj58ZJsHagyO304y+ZlrAaODYl0DMfADAMvVgLFIPh/7aA4oqrUTh6SJhreuUkuB7gJeL0X4J3IYqBhYYb6JHxKtuou9CVmH9sr+A7dy5ExEREZg7dy6uXr2KsLAw9OvXD+np6YL7nzhxAsOHD8fx48dx7tw5hISEoG/fvkhOTq7kkZsPYwUFersIzgjpY6kFxrglBnlgFFXhtIYiuOAd1XQ89OkGV0qJn6RLEC66YnR/Uz/6PVdLPyNH+FHdLLPfpbE3Q+NzyVbg1GIAQLRqOJap34LQwikRRbExME8XCSs8jAUW4Oli0GKNPVZEsVa1fk9Hbpihob8H77lKbxbS3AkYZ8TuArZ06VJMmDAB48aNQ4sWLbBmzRq4ublh/fr1gvtv3boVkyZNQps2bdCsWTP89NNPbDNcR4WxwMzJAQP4Vpc5MbA+LQLQKbQGBrXjL/exVk6WAjLsb74YhzQdIafUWC1dhldF5yw+z1cHy3ZBK4t/k3Mw0IKO45n5upQHClp8KVmPCZKDAIAo1Rj8qBlo9Bwiil9Li1mq+iRHZ4HV8jJtkTMzjyF6QXzuR8t1HwFdr1GhxrZVEbsKmFKpxJUrVxAeHs5uE4lECA8Px7lz5v1ACgsLoVKpjLZ2UygUyM3N5T0qG1bAzIh/AfzlGubMQjYJ8MSu97sYVHi15o2XEsswRfURftd0g5TS4HvpCrwpOmWwn7nZ6+Wp/GpNzj98WuY+3DSKrAIlxNBgsXQt3pbEQEtTwGsrsFnTz+Q5RBTFZrF7ukg5LmSJBeZlelLn85eb4/0eDdCyNr+jPNeqaqQvYFpaLwZm8hJOjV0FLDMzExqNxqAhbkBAAFJTU40cxWfGjBkIDg7miSCX6OhoeHt7s4+QkJAKj9tSGIuqrCoUDF6cfJ+KpB9Y884rFgEaiPGp6gNsV/eCmKLxrWwNRoqP8vaztzCZiznj5O6SnVeA76Ur8Zb4FNS0CJ+oJgPtRpV5DpGo1IX0cCkN4qfmMgJm2gJ7o21tzBzQ3MCa5sbA9C0wXQyMY4FVYQWzuwtZERYtWoQdO3bg999/h4uLsDjMnDkTOTk57OPRo0eVPEqgV7NaCPCSo3cz8zqXe1pogRnDmt9bJllTCxFmqt/FBrXO8vhKuh7jxQfY/crT1agyKI+WsyKnKka9mA/wqvg8lLQYk1UfY7+2q3nXhXAQn6EsC8zoebkCFlCWBVZ1BcyueWB+fn4Qi8VIS+PPBKWlpSEwMNDksUuWLMGiRYtw9OhRtG7d2uh+crkccnnZM3+25LWwYLwWFmz2/tyMa3NiYMZgvuSrR7bD9kuPcOpeRrnP9dct7mdEYZ56NIogxyTJfsyRboUrlFihecPkLKQ9Kc+i8oy8YvyXkoGgw+MRkHISxbQUE1VTcULbBoBhc1pjpOfp4mc+rjKDBFR/z/J9NxkhlIgo1KvpbvB6WU09qgp2tcBkMhnat2/PC8AzAfkuXboYPe6bb77BggULcPjwYXTo0KEyhlqp8CwwM2Yh9WGm3F9prStvMqBVEGa/0txqY9JB4Rv1UCxR/Q8AME36K6ZLdoI280fdd9kpfHNYuIZWZWCOpfjpljNIXf0qZP+dhFLkinGqz1jxAgy7FglBUcCdVF3ctWmgh4E7V14LLLDkuLAQH8Fcv/Jk4jsjds/Ej4iIwJgxY9ChQwd06tQJy5YtQ0FBAcaNGwcAGD16NGrXro3o6GgAwNdff42oqChs27YN9evXZ2NlHh4e8PDwMHodZ8KLZ4FZLmB/Te2OJ9lFaBLgWfbOZuIr2G2GwgrNIBRBhjnSrZgs2Y9TaVLsxmCUVYtVqdZi1Yl4fNZf10qMpmncT883SAmwF17IxybZN2greoBc2hVji2bgKs0vZaRQly1gSrUWj0py3poHeRlYYOUVsLo13XD4kxdRy1P4eJ4L6QB/T1th9xjY0KFDsWTJEkRFRaFNmza4du0aDh8+zAb2k5KSkJJSWp9q9erVUCqVeOuttxAUFMQ+lixZYq+3YHW41k5ZXbqF8JBLDMTLkuC6/owXYNoN+VnzCtt7snvWbiyU/ASRid6TQvz0TwL6fncKn/1mOlHWWpj6e9RALrbLvkJb0QM8oz0wQvm5gXgBhi3UhEh8WgiNloa3qxSBXi4G4lyR6g7NAr1Qw91IGzOeBVbuSzg8drfAAGDKlCmYMmWK4GsnTpzgPU9MTLT9gOwMN43CHDfFHIwlSwrRyN8D/ybz003KEsAtmj4ohgzfSNdhhOQ4XCklpqkmQgPTMbzXV57Bzveex3dH7wEAdl99jG+HhJk9VnPRvw8Yezv+eIatsoVoIkpGBu2Nt5UzcVeg7Rlg2WfTPMgTFEVVmjVkjbWQzoDdLTCCIdyYhrXSEtRa839s5XVrftP0wC/Bs6GixRgkPoPlRnpPcrn+KBsHbqRA7QDR/2BkYpdsPpqIkpFC18BQ5Ryj4gUYVp81RbNAnVWbW6Qyud+Ljf2w473nzT6vMarLLCQRMAdl1PP10K6uD7o0FO4baCmWCITQzJi5R1/16o0PVJ9AQUvwsvgi1pjRezKrQGmRhVhRuF2yGepSadgln49QURoeaf3xP2UUHtKmZ44tscBaBOkEjOnODpRahUzTlv+1r4NfxnfG8w0q/plTZBaSYE8WvNESeyZ1K3c1CX3MnfIHIBj8NzfDXqXR4qi2Pd5VTUMRLcNL4lj8LF3M7z2pR3aR7QWM+xt++6cLPMu2IZWMX2XzUIfKRLw2CP9TRuExXavMc5oTA2NoFqT7myZzBKxmSfzqz49exLS+TRA1sIXZ5xPilZKmGi829uN9XsQCIzg93PZZXwxsYTJ43LG+4bIsc/Xl4E3drPA/2tYYq5yBfNoFL4hvYbNsETwh3APxWaFpt6qiqDT8tmfXH+ewaRTNqf+wU7YAAVQ27mhDMFQZhVSUbQHJxCKzBUxEld4UCjkdjJgihaF+7pjSuzEv/688fP1Wa3w3NAwrRrTjWcxkFpLg9HAtnLHdQnF9bl+j+7rKxHj3BX4F1l5N/Y3sbZwLdHOMUs5EDu2GjqJ7bO9JfbILTbuYFeVcvOG6R5oGwqgH2CFbAD8qFze0oRimnI1MeJt1TrnUfAFr4O/BtkhrV9eH3f7t/6w7WeEhl2BQ2zrwdpXyLMwqrF9EwKoL+kF8Y7lWPiXt4UP9S7O7R3Sui7efr1eu68bSjTFCORtPaU+EiR5ih+xL1AS/jI0lrlhZ0DSNmLg0PH5WyLpRQg1JgrJjsUUWDW+qEJe1TTBS+TmyYX7enItUbFYeGAA0Cyw978qR7TCjfzNci+pjsAjbmnATdYkLSXB6zI0xbRjbEQA//2x0l3qQVCAWd4uuj2HKOUinfdBclIRdsvkIQGmNemv+wGLi0jF+02W88PVxdPwqBmfjMw1WM3QT3cSrNybDkyrCWU0LjFZGIs+MctlcZGKR2QLWPKg0ry7I2xUf9GwIH8HEYOvBtcAcITHYVhABqyYIBebf6cZ3E3+d2AVtS1q3ceXOT6CKrKVt1p7v3A1DlHOQTNdEQ1EKdsnmow6lW5tpzR/YhYRSdzEzX4ER6y7w1pP2Fl3FeukSyGgFjmvCME71GQpRvrQRc2chmwdZb0WEuXAFrAobYETAqgshNdzwx5QXcHpGL3ab/qwXd6byGScuJbSMyNIlTvNeew6PqWAMUUThP20t1BOlY5dsHupTKby2XQDwKEs42G8OQtZcQUk9rpdF5/Gj9DvIKRUOazrifVUEFCifJaSlabNdX64FVlkQF5JQ5WhVx9tkU1eu1ZWVXypgQhaSXGp+lQx3mRgiEQUPFwmS4Y8hyig80AYjmMrCLtkCNKYe8/Y/WYGqGUJVaIeuPY9Bon90ibWUBns1XTFZ9RGUKP+sX0pOMb7Yf6vM/ZglRJUNz4UkAkaoDuhXIDWF0F3dXSYsaky8x72kJ2IadFnucdq6qEVlY6dsPp6jEtj9n5VxbVMIheqGi2PwrXQNxBSNHeqeiFBNKnOJkzkw3YaMIZeIMLpLPauV9rYE7s2oCusXETBCKdy7NtOtuUM9X8F9hcJW7kZyy9zlOrHgLlJ/Cm8MU87GNW0D1KDydYunqfu6c1cgJqYvrO+IDyFaqms4u0HdDzPV70Jrg6/9yek9Ua9mqXXbuJYH7izoj0/7NrX6tcyBm8hqDwGtLIiAEVi4fQi7NfLD8Wk9sXVCZ3bb8E6l5bhH6aVVNAv0ZAv3AcCKEW3Z/xerdLEi/d6GTO/Ji9qm8KIK8YssGp2pOEhEFKL2/YulR+5h1YkH+Pl0AsyF+2OdJN6LKOkvAIDV6oGYpx7N69lYHha80dJgW+fQGqhX0x1HI3pg7sAWmPNqCxyJ6GFX4bD/ytLKwSGqURDsj5tMjDYhPrxtoX78Sp/Rg1tj4aBWKFZpcSSutELrkandEeTjipZz/2K3vdo6GFO2xQIo7X694I2WOBp3jHfOfLhhjHIG1km/xQviW9gkW4RdSb7YfItfkbdVbe8yO0RrtHSJZUjjU8mv+FCyFwCwVPUWftAMQlk1yhjEIgrLh7fFpK1XDV4TWlIV+ygbgG4R/ji9mV27UU0UjFhgBADAzJcNG0cIQVEUXGVingvZOMATHnIJvh/WBp5yCX4Z34l3TIFCt3wmyEhbuSK4YLxqOmI0beFCqTDi4Qz0EV3m7TPkR9NdqvbGJqPl3L9w4k465ki2sOL1lWoEftCUXWCRi6+bDC+3CkJ4c8P1kLUEFrpHlhRldCQctTeBtSECRgBg/mJtBqEg/uttauP63L54sTF/2VGRGWVnFJBhomoqDmg6QUKrLO49+cnOayhWqfC/1KUYLzkEAJitGod1mlfNPgcDk3bx05iOvBSIPZO6onsTf3i7SuHlIsGdBf1xNKI7xnWrb/E1bI2zdIeqKETACAAsK3gIGK8ZVpEAvAoSfKT6ELs1L0BCafG9dAXeEp9kX++/7BRuPTHspr3vWjLE0GCJ9EeMlMRAQ1OYrnoPWzR9yjWOhrVKXWeFulR829X1hZtMgrORvXF1Th+4SMVoVMvTIYPkRMAI1Yp2dYVnG43Rvp4vIgc0w5q32xnd5+3ndcUALbFQNBBjmmoitql7Q0zRWCL9EW+LjwAA7qTmYbJeXIqmaUzbcVnXaFf8D9uz8VdNT4veD8OAloFYNrQN+1yhMkxWdZdLKrS0qjKoJvpFgvjVnbORvfH4WRHC9AL45jCxR0OTr88d+BwGta2N1nUsOzcNEWapx6MYMrwjOYwvpRvgCgXWaV7FU06OWG6xCp9tv4jV0u8QLo6FgpbgQ9WH+Fvb0azrrBzRDolPC7D4r7vsttVvt+ftU2xB1VVHoraZXeCdHYe4jaxcuRL169eHi4sLOnfujIsXL5rc/9dff0WzZs3g4uKCVq1a4eDBg5U00qpHsI9rmbN75UUqFqF9vRqCRRm3jO+MQW1rmziawnz1KKxQvw4A+Fy6DR+J94Di2Baf7zyPkQ8/Q7g4FsW0FBNUn5olXn4eMmx6pxNeaR2Eyb0aoXUd4yV0tE7qi7UI9sLSIWHYaYXy1I6M3S2wnTt3IiIiAmvWrEHnzp2xbNky9OvXD3fv3kWtWoazQGfPnsXw4cMRHR2NV199Fdu2bcMbb7yBq1evomVLwxwdgmNxbmZvPM1XomVtb7zQ2A+/xyab2JvCEvVQFNFyTJfuQoT0N7iqFUjJ7o4gFzXejp+GzuI7KKDlGK+ajvNaw4qm774QivQ8BUJquGJa36bIKlCipt7i9JAabrjx2DC2BgCrRrbHxC1XMO+15yrytu3C4HZ17D0Em0PRlk4/WZnOnTujY8eOWLFiBQBdY9uQkBB8+OGHiIyMNNh/6NChKCgowJ9//slue/7559GmTRusWbOmzOvl5ubC29sbOTk58PKq/EW2BD5R+/7F5nP/oUWQF14NC8I3h+8K7jdefBBzpFsAAHtE/dBA/QBtRPHIpd0wVvmZQduz7k38sXFsR7MmFVJyivD+L1cwukt9vNXe8EdP07RDBuqrKpb8Ru1qgSmVSly5cgUzZ85kt4lEIoSHh+PcOeEp9HPnziEiIoK3rV+/fti7d6/g/gqFAgpFaYZ4bm6u4H4E+zBzQHM0CfBEnxYBCPBywaSejZCeV4zOC2PYmTSpmMLPmpdRDBm+kq7HYO1fgAjIoj0wSjkTt+hQLBrcCkM6hICiLF86E+Ttiv1TXjD6OhEvx8WuApaZmQmNRsM2sWUICAjAnTvCbedTU1MF92c6dOsTHR2NefPmWWfABKvjKhMbVHut5emCuPn9UaTUYMelRxjcrjYy8hR4d5MLivJlWCz9EVnwwkjlLNyjQ7D7g65ob2TNJqFqY/cYmK2ZOXMmz2LLzc1FSEiIiSMIjoCLVAwXqRgf9NTNdAZ4ueD8rJeg1fZGceZEeLn74G93IlrVHbsKmJ+fH8RiMdLS0njb09LSEBgYKHhMYGCgRfvL5XLI5YbLPwjOiUhEwa2Wg6w3JNgdu6ZRyGQytG/fHjExMew2rVaLmJgYdOnSRfCYLl268PYHgCNHjhjdn0AgVF3s7kJGRERgzJgx6NChAzp16oRly5ahoKAA48aNAwCMHj0atWvXRnR0NADg448/Ro8ePfDtt9/ilVdewY4dO3D58mWsXbvWnm+DQCDYAbsL2NChQ5GRkYGoqCikpqaiTZs2OHz4MBuoT0pKgkhUaih27doV27Ztw+zZszFr1iw0btwYe/fuJTlgBEI1xO55YJUNyQMjEBwbS36jDrGUiEAgEMoDETACgeC0EAEjEAhOi92D+JUNE/IjS4oIBMeE+W2aE56vdgKWl5cHACQbn0BwcPLy8uDtbbzUEVANZyG1Wi2ePHkCT0/hUsDMUqNHjx5ViVnKqvZ+gKr3nqra+wEq9p5omkZeXh6Cg4N5KVRCVDsLTCQSoU6dsuskeXl5VZkvE1D13g9Q9d5TVXs/QPnfU1mWFwMJ4hMIBKeFCBiBQHBaiIDpIZfLMXfu3CpTwaKqvR+g6r2nqvZ+gMp7T9UuiE8gEKoOxAIjEAhOCxEwAoHgtBABIxAITgsRMAKB4LRUOwHLysrCyJEj4eXlBR8fH4wfPx75+fkmj1m7di169uwJLy8vUBSF7Oxsq5zXWpTn2sXFxZg8eTJq1qwJDw8PvPnmmwa9BiiKMnjs2LHD6uO3dmd2mqYRFRWFoKAguLq6Ijw8HPfv37f6uE1h7fc0duxYg8+if//+tnwLBljynm7duoU333wT9evXB0VRWLZsWYXPKQhdzejfvz8dFhZGnz9/nv7nn3/oRo0a0cOHDzd5zHfffUdHR0fT0dHRNAD62bNnVjmvtSjPtSdOnEiHhITQMTEx9OXLl+nnn3+e7tq1K28fAPSGDRvolJQU9lFUVGTVse/YsYOWyWT0+vXr6Vu3btETJkygfXx86LS0NMH9z5w5Q4vFYvqbb76hb9++Tc+ePZuWSqX0zZs32X0WLVpEe3t703v37qWvX79Ov/baa3RoaKjVx16Z72nMmDF0//79eZ9FVlZWpbwfmrb8PV28eJGeNm0avX37djowMJD+7rvvKnxOIaqVgN2+fZsGQF+6dInddujQIZqiKDo5ObnM448fPy4oYBU9b0Uoz7Wzs7NpqVRK//rrr+y2uLg4GgB97tw5dhsA+vfff7fZ2Gmapjt16kRPnjyZfa7RaOjg4GA6OjpacP8hQ4bQr7zyCm9b586d6ffff5+maZrWarV0YGAgvXjxYvb17OxsWi6X09u3b7fBOzDE2u+JpnUC9vrrr9tkvOZg6XviUq9ePUEBq8g5GaqVC3nu3Dn4+PigQ4cO7Lbw8HCIRCJcuHDB4c5rq2tfuXIFKpUK4eHh7LZmzZqhbt26Bh3RJ0+eDD8/P3Tq1Anr1683q8SJuTCd2bnjMKczO3d/QNeZndk/ISEBqampvH28vb3RuXNno+e0JrZ4TwwnTpxArVq10LRpU3zwwQd4+vSp9d+AAOV5T5V1zmq1mDs1NRW1atXibZNIJKhRo4bRzt72PK+trp2amgqZTAYfHx/edv0O5/Pnz0fv3r3h5uaGv//+G5MmTUJ+fj4++ugjq4zdFp3ZmX8t6d5uTWzVbb5///4YPHgwQkNDER8fj1mzZmHAgAE4d+4cxGKx9d8Ih/K8p8o6Z5UQsMjISHz99dcm94mLi6uk0VgHR3hPc+bMYf/ftm1bFBQUYPHixVYTMIL5DBs2jP1/q1at0Lp1azRs2BAnTpzASy+9ZMeR2ZcqIWCffvopxo4da3KfBg0aIDAwEOnp6bztarUaWVlZRjt7m4MtzmvL9xQYGAilUons7GyeFWaqwzkAdO7cGQsWLIBCobDKGjdbdGZn/k1LS0NQUBBvnzZt2lR4zGVRGd3mAd1n7+fnhwcPHthcwMrznirrnFUiBubv749mzZqZfMhkMnTp0gXZ2dm4cuUKe+yxY8eg1WrRuXPncl/fFue15Xtq3749pFIpr8P53bt3kZSUZLLD+bVr1+Dr62u1Bbq26MweGhqKwMBA3j65ubm4cOFCpXRvr6xu848fP8bTp095Im0ryvOeKu2cZof7qwj9+/en27ZtS1+4cIE+ffo03bhxY17KwePHj+mmTZvSFy5cYLelpKTQsbGx9Lp162gA9KlTp+jY2Fj66dOnZp/X0d7TxIkT6bp169LHjh2jL1++THfp0oXu0qUL+/r+/fvpdevW0Tdv3qTv379Pr1q1inZzc6OjoqKsOvYdO3bQcrmc3rhxI3379m36vffeo318fOjU1FSapml61KhRdGRkJLv/mTNnaIlEQi9ZsoSOi4uj586dK5hG4ePjQ+/bt4++ceMG/frrr1d6GoU131NeXh49bdo0+ty5c3RCQgJ99OhRul27dnTjxo3p4uJih3xPCoWCjo2NpWNjY+mgoCB62rRpdGxsLH3//n2zz2kO1U7Anj59Sg8fPpz28PCgvby86HHjxtF5eXns6wkJCTQA+vjx4+y2uXPn0gAMHhs2bDD7vI72noqKiuhJkybRvr6+tJubGz1o0CA6JSWFff3QoUN0mzZtaA8PD9rd3Z0OCwuj16xZQ2s0GquPf/ny5XTdunVpmUxGd+rUiT5//jz7Wo8ePegxY8bw9t+1axfdpEkTWiaT0c899xx94MAB3utarZaeM2cOHRAQQMvlcvqll16i7969a/Vxm8Ka76mwsJDu27cv7e/vT0ulUrpevXr0hAkTLPqhWwNL3hPzndN/9OjRw+xzmgMpp0MgEJyWKhEDIxAI1RMiYAQCwWkhAkYgEJwWImAEAsFpIQJGIBCcFiJgBALBaSECRiAQnBYiYAQCwWkhAkZwCIRKJtujbDLBuagS1SgIVYP+/ftjw4YNvG3GFo6rVCpIpVLeNqVSCZlMZvF1y3scwf4QC4zgMMjlcgQGBvIevr6+AHQNRlavXo3XXnsN7u7u+Oqrr/DFF1+gTZs2+OmnnxAaGgoXFxcAQFJSEl5//XV4eHjAy8sLQ4YM4ZVtMXYcwfkgAkZwGr744gsMGjQIN2/exDvvvAMAePDgAXbv3o09e/bg2rVr0Gq1eP3115GVlYWTJ0/iyJEjePjwIYYOHco7l/5xBOeEuJAEh+HPP/+Eh4cHb9usWbMwa9YsAMCIESMwbtw43utKpRKbN2+Gv78/AF0drZs3byIhIQEhISEAgM2bN+O5557DpUuX0LFjR8HjCM4JETCCw9CrVy+sXr2at61GjRrs/7mNSxjq1avHE6G4uDiEhISw4gUALVq0gI+PD+Li4lgB0z+O4JwQASM4DO7u7mjUqJHJ183ZZu61CM4PiYERqhTNmzfHo0eP8OjRI3bb7du3kZ2djRYtWthxZARbQCwwgsOgUCgMWp9JJBL4+fmZfY7w8HC0atUKI0eOxLJly6BWqzFp0iT06NFD0AUlODfEAiM4DIcPH0ZQUBDv8cILL1h0DoqisG/fPvj6+qJ79+4IDw9HgwYNsHPnThuNmmBPSElpAoHgtBALjEAgOC1EwAgEgtNCBIxAIDgtRMAIBILTQgSMQCA4LUTACASC00IEjEAgOC1EwAgEgtNCBIxAIDgtRMAIBILTQgSMQCA4LUTACASC0/J/ZBCD2vcs4HIAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.02234, + "end_time": "2024-02-29T18:13:59.491745", + "exception": false, + "start_time": "2024-02-29T18:13:59.469405", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4650.540607, + "end_time": "2024-02-29T18:14:02.238238", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/realtabformer/0/mlu-eval.ipynb", + "output_path": "eval/contraceptive/realtabformer/0/mlu-eval.ipynb", + "parameters": { + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "param_index": 2, + "path": "eval/contraceptive/realtabformer/0", + "path_prefix": "../../../../", + "random_seed": 0, + "single_model": "realtabformer" + }, + "start_time": "2024-02-29T16:56:31.697631", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/realtabformer/model.pt b/contraceptive/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..29b5859e10197313d86267d2195ae9be1e70c4e5 --- /dev/null +++ b/contraceptive/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6a5bd58ce64f40ee98a2b7d07616ebbd395ba0d21170d594a384032a311a4b0 +size 43889419 diff --git a/contraceptive/realtabformer/params.json b/contraceptive/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..797838e441bed8e1a1c4b6dafaf45a75d40b8902 --- /dev/null +++ b/contraceptive/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 8, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.775, "gradient_penalty_mode": "ALL", "synth_data": 2, "dataset_size": 2048, "batch_size": 2, "epochs": 100, "lr_mul": 0.075, "n_warmup_steps": 100, "Optim": "amsgradw", "loss_balancer_beta": 0.675, "loss_balancer_r": 0.95, "fixed_role_model": "realtabformer", "mse_mag": true, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 8, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation": "relu6", "head_activation_final": "leakyhardsigmoid", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/tab_ddpm_concat/.ipynb_checkpoints/mlu-eval-checkpoint.ipynb b/contraceptive/tab_ddpm_concat/.ipynb_checkpoints/mlu-eval-checkpoint.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..41f02105e2874fcf23419f2b0a4432c24ce8f0a7 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/.ipynb_checkpoints/mlu-eval-checkpoint.ipynb @@ -0,0 +1,2221 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.553837Z", + "iopub.status.busy": "2024-02-29T05:52:41.553487Z", + "iopub.status.idle": "2024-02-29T05:52:41.587012Z", + "shell.execute_reply": "2024-02-29T05:52:41.586303Z" + }, + "papermill": { + "duration": 0.048134, + "end_time": "2024-02-29T05:52:41.588883", + "exception": false, + "start_time": "2024-02-29T05:52:41.540749", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.614875Z", + "iopub.status.busy": "2024-02-29T05:52:41.614416Z", + "iopub.status.idle": "2024-02-29T05:52:41.621190Z", + "shell.execute_reply": "2024-02-29T05:52:41.620324Z" + }, + "papermill": { + "duration": 0.022432, + "end_time": "2024-02-29T05:52:41.623143", + "exception": false, + "start_time": "2024-02-29T05:52:41.600711", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.646395Z", + "iopub.status.busy": "2024-02-29T05:52:41.646135Z", + "iopub.status.idle": "2024-02-29T05:52:41.650123Z", + "shell.execute_reply": "2024-02-29T05:52:41.649356Z" + }, + "papermill": { + "duration": 0.017841, + "end_time": "2024-02-29T05:52:41.651962", + "exception": false, + "start_time": "2024-02-29T05:52:41.634121", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.675528Z", + "iopub.status.busy": "2024-02-29T05:52:41.675083Z", + "iopub.status.idle": "2024-02-29T05:52:41.678787Z", + "shell.execute_reply": "2024-02-29T05:52:41.677998Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017719, + "end_time": "2024-02-29T05:52:41.680705", + "exception": false, + "start_time": "2024-02-29T05:52:41.662986", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.704275Z", + "iopub.status.busy": "2024-02-29T05:52:41.704010Z", + "iopub.status.idle": "2024-02-29T05:52:41.708949Z", + "shell.execute_reply": "2024-02-29T05:52:41.708172Z" + }, + "papermill": { + "duration": 0.019176, + "end_time": "2024-02-29T05:52:41.710931", + "exception": false, + "start_time": "2024-02-29T05:52:41.691755", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b4a51afa", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.735735Z", + "iopub.status.busy": "2024-02-29T05:52:41.735440Z", + "iopub.status.idle": "2024-02-29T05:52:41.740488Z", + "shell.execute_reply": "2024-02-29T05:52:41.739696Z" + }, + "papermill": { + "duration": 0.01964, + "end_time": "2024-02-29T05:52:41.742472", + "exception": false, + "start_time": "2024-02-29T05:52:41.722832", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tab_ddpm_concat/4\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.01106, + "end_time": "2024-02-29T05:52:41.765597", + "exception": false, + "start_time": "2024-02-29T05:52:41.754537", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.789321Z", + "iopub.status.busy": "2024-02-29T05:52:41.788485Z", + "iopub.status.idle": "2024-02-29T05:52:41.797921Z", + "shell.execute_reply": "2024-02-29T05:52:41.797137Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023362, + "end_time": "2024-02-29T05:52:41.799859", + "exception": false, + "start_time": "2024-02-29T05:52:41.776497", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tab_ddpm_concat/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.824124Z", + "iopub.status.busy": "2024-02-29T05:52:41.823296Z", + "iopub.status.idle": "2024-02-29T05:52:44.077985Z", + "shell.execute_reply": "2024-02-29T05:52:44.077036Z" + }, + "papermill": { + "duration": 2.269148, + "end_time": "2024-02-29T05:52:44.080113", + "exception": false, + "start_time": "2024-02-29T05:52:41.810965", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:44.106336Z", + "iopub.status.busy": "2024-02-29T05:52:44.105917Z", + "iopub.status.idle": "2024-02-29T05:52:44.116991Z", + "shell.execute_reply": "2024-02-29T05:52:44.116261Z" + }, + "papermill": { + "duration": 0.026284, + "end_time": "2024-02-29T05:52:44.118890", + "exception": false, + "start_time": "2024-02-29T05:52:44.092606", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:44.142507Z", + "iopub.status.busy": "2024-02-29T05:52:44.142237Z", + "iopub.status.idle": "2024-02-29T05:52:44.149263Z", + "shell.execute_reply": "2024-02-29T05:52:44.148530Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021057, + "end_time": "2024-02-29T05:52:44.151151", + "exception": false, + "start_time": "2024-02-29T05:52:44.130094", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:44.174949Z", + "iopub.status.busy": "2024-02-29T05:52:44.174650Z", + "iopub.status.idle": "2024-02-29T05:52:44.283419Z", + "shell.execute_reply": "2024-02-29T05:52:44.282604Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.123895, + "end_time": "2024-02-29T05:52:44.286263", + "exception": false, + "start_time": "2024-02-29T05:52:44.162368", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:44.314871Z", + "iopub.status.busy": "2024-02-29T05:52:44.314539Z", + "iopub.status.idle": "2024-02-29T05:52:48.872369Z", + "shell.execute_reply": "2024-02-29T05:52:48.871556Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.574074, + "end_time": "2024-02-29T05:52:48.874734", + "exception": false, + "start_time": "2024-02-29T05:52:44.300660", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 05:52:46.506222: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 05:52:46.506284: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 05:52:46.507965: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:48.899749Z", + "iopub.status.busy": "2024-02-29T05:52:48.899216Z", + "iopub.status.idle": "2024-02-29T05:52:48.904784Z", + "shell.execute_reply": "2024-02-29T05:52:48.904124Z" + }, + "papermill": { + "duration": 0.020032, + "end_time": "2024-02-29T05:52:48.906657", + "exception": false, + "start_time": "2024-02-29T05:52:48.886625", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:48.931922Z", + "iopub.status.busy": "2024-02-29T05:52:48.931646Z", + "iopub.status.idle": "2024-02-29T05:52:57.087404Z", + "shell.execute_reply": "2024-02-29T05:52:57.086436Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.171018, + "end_time": "2024-02-29T05:52:57.089766", + "exception": false, + "start_time": "2024-02-29T05:52:48.918748", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "100%|██████████| 1/1 [00:00<00:00, 2.79it/s]\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-02-29T05:52:57.118044Z", + "iopub.status.busy": "2024-02-29T05:52:57.117156Z", + "iopub.status.idle": "2024-02-29T05:52:57.123889Z", + "shell.execute_reply": "2024-02-29T05:52:57.123035Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023061, + "end_time": "2024-02-29T05:52:57.125871", + "exception": false, + "start_time": "2024-02-29T05:52:57.102810", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:57.151111Z", + "iopub.status.busy": "2024-02-29T05:52:57.150850Z", + "iopub.status.idle": "2024-02-29T05:52:57.155580Z", + "shell.execute_reply": "2024-02-29T05:52:57.154701Z" + }, + "papermill": { + "duration": 0.019488, + "end_time": "2024-02-29T05:52:57.157399", + "exception": false, + "start_time": "2024-02-29T05:52:57.137911", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:57.182402Z", + "iopub.status.busy": "2024-02-29T05:52:57.182137Z", + "iopub.status.idle": "2024-02-29T05:52:57.213320Z", + "shell.execute_reply": "2024-02-29T05:52:57.212336Z" + }, + "papermill": { + "duration": 0.046054, + "end_time": "2024-02-29T05:52:57.215294", + "exception": false, + "start_time": "2024-02-29T05:52:57.169240", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset\n", + "\n", + "test_set = load_dataset(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\", \"datasets_5\", dataset_name),\n", + " preprocessor=preprocessor,\n", + " cache_dir=os.path.join(path_prefix, dataset_name, \"_cache_test\"),\n", + " start=200,\n", + " #stop=600,\n", + " val=False,\n", + " ratio=0,\n", + " drop_first_column=True,\n", + " model=single_model,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:57.241834Z", + "iopub.status.busy": "2024-02-29T05:52:57.241535Z", + "iopub.status.idle": "2024-02-29T05:52:57.563537Z", + "shell.execute_reply": "2024-02-29T05:52:57.562643Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.337952, + "end_time": "2024-02-29T05:52:57.565789", + "exception": false, + "start_time": "2024-02-29T05:52:57.227837", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.73,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'loss_balancer_beta': 0.67,\n", + " 'loss_balancer_r': 0.943,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation': torch.nn.modules.activation.Softsign,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BEST,\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:57.592167Z", + "iopub.status.busy": "2024-02-29T05:52:57.591845Z", + "iopub.status.idle": "2024-02-29T05:52:57.659947Z", + "shell.execute_reply": "2024-02-29T05:52:57.659043Z" + }, + "papermill": { + "duration": 0.083576, + "end_time": "2024-02-29T05:52:57.661879", + "exception": false, + "start_time": "2024-02-29T05:52:57.578303", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../contraceptive/_cache/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache4/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache5/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/contraceptive [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T05:52:57.690612Z", + "iopub.status.busy": "2024-02-29T05:52:57.690320Z", + "iopub.status.idle": "2024-02-29T05:52:58.154857Z", + "shell.execute_reply": "2024-02-29T05:52:58.153972Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.481713, + "end_time": "2024-02-29T05:52:58.156846", + "exception": false, + "start_time": "2024-02-29T05:52:57.675133", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n", + "[*] Embedding False True\n", + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:58.187496Z", + "iopub.status.busy": "2024-02-29T05:52:58.187196Z", + "iopub.status.idle": "2024-02-29T05:52:58.191528Z", + "shell.execute_reply": "2024-02-29T05:52:58.190632Z" + }, + "papermill": { + "duration": 0.022791, + "end_time": "2024-02-29T05:52:58.193570", + "exception": false, + "start_time": "2024-02-29T05:52:58.170779", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:58.220322Z", + "iopub.status.busy": "2024-02-29T05:52:58.219763Z", + "iopub.status.idle": "2024-02-29T05:52:58.226654Z", + "shell.execute_reply": "2024-02-29T05:52:58.225833Z" + }, + "papermill": { + "duration": 0.022269, + "end_time": "2024-02-29T05:52:58.228558", + "exception": false, + "start_time": "2024-02-29T05:52:58.206289", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "11858440" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:58.254957Z", + "iopub.status.busy": "2024-02-29T05:52:58.254488Z", + "iopub.status.idle": "2024-02-29T05:52:58.332480Z", + "shell.execute_reply": "2024-02-29T05:52:58.331558Z" + }, + "papermill": { + "duration": 0.093949, + "end_time": "2024-02-29T05:52:58.334954", + "exception": false, + "start_time": "2024-02-29T05:52:58.241005", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 10] --\n", + "├─Adapter: 1-1 [2, 1179, 10] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 11,264\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 10] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 16, 256] 1\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 1,048,832\n", + "│ │ │ └─Softsign: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 256] --\n", + "│ │ │ └─Linear: 4-55 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-56 [2, 256] --\n", + "│ │ └─FeedForward: 3-28 [2, 256] --\n", + "│ │ │ └─Linear: 4-57 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-58 [2, 256] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 11,858,440\n", + "Trainable params: 11,858,440\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 44.00\n", + "========================================================================================================================\n", + "Input size (MB): 0.12\n", + "Forward/backward pass size (MB): 375.40\n", + "Params size (MB): 47.43\n", + "Estimated Total Size (MB): 422.95\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:58.365908Z", + "iopub.status.busy": "2024-02-29T05:52:58.365521Z", + "iopub.status.idle": "2024-02-29T06:20:40.160166Z", + "shell.execute_reply": "2024-02-29T06:20:40.159139Z" + }, + "papermill": { + "duration": 1661.828434, + "end_time": "2024-02-29T06:20:40.178185", + "exception": false, + "start_time": "2024-02-29T05:52:58.349751", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n", + "Train loss {'avg_role_model_loss': 0.018960519676329567, 'avg_role_model_std_loss': 0.633159703178444, 'avg_role_model_mean_pred_loss': 0.0011727252675751032, 'avg_role_model_g_mag_loss': 0.05587912966730073, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.019387794903013855, 'n_size': 320, 'n_batch': 80, 'duration': 97.81354141235352, 'duration_batch': 1.2226692676544189, 'duration_size': 0.3056673169136047, 'avg_pred_std': 0.1214748754282482}\n", + "Val loss {'avg_role_model_loss': 0.005117437991430052, 'avg_role_model_std_loss': 1.2569677214083186, 'avg_role_model_mean_pred_loss': 4.872746470283573e-05, 'avg_role_model_g_mag_loss': 0.0313290299847722, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005207326461095363, 'n_size': 80, 'n_batch': 20, 'duration': 19.29424023628235, 'duration_batch': 0.9647120118141175, 'duration_size': 0.24117800295352937, 'avg_pred_std': 0.0514119642553851}\n", + "Epoch 1\n", + "Train loss {'avg_role_model_loss': 0.014699359831865877, 'avg_role_model_std_loss': 1.103800665530207, 'avg_role_model_mean_pred_loss': 0.0005193239165346864, 'avg_role_model_g_mag_loss': 0.15416064693126827, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01488935198285617, 'n_size': 320, 'n_batch': 80, 'duration': 97.73809790611267, 'duration_batch': 1.2217262238264084, 'duration_size': 0.3054315559566021, 'avg_pred_std': 0.09262466638174374}\n", + "Val loss {'avg_role_model_loss': 0.008836813416564837, 'avg_role_model_std_loss': 0.5016097052809527, 'avg_role_model_mean_pred_loss': 0.00014911171683706925, 'avg_role_model_g_mag_loss': 0.1304568352177739, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008931921707699075, 'n_size': 80, 'n_batch': 20, 'duration': 19.49599575996399, 'duration_batch': 0.9747997879981994, 'duration_size': 0.24369994699954986, 'avg_pred_std': 0.0690233844332397}\n", + "Epoch 2\n", + "Train loss {'avg_role_model_loss': 0.009927867040460114, 'avg_role_model_std_loss': 0.6356354271476448, 'avg_role_model_mean_pred_loss': 0.00020892808070887558, 'avg_role_model_g_mag_loss': 0.11403882297454401, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010061423623847076, 'n_size': 320, 'n_batch': 80, 'duration': 97.83805751800537, 'duration_batch': 1.2229757189750672, 'duration_size': 0.3057439297437668, 'avg_pred_std': 0.10248138114111498}\n", + "Val loss {'avg_role_model_loss': 0.004521565337199718, 'avg_role_model_std_loss': 1.0787538406874773, 'avg_role_model_mean_pred_loss': 3.4153386028634714e-05, 'avg_role_model_g_mag_loss': 0.1442047566641122, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004582326250965707, 'n_size': 80, 'n_batch': 20, 'duration': 19.448378086090088, 'duration_batch': 0.9724189043045044, 'duration_size': 0.2431047260761261, 'avg_pred_std': 0.04124836295377463}\n", + "Epoch 3\n", + "Train loss {'avg_role_model_loss': 0.01018573724286398, 'avg_role_model_std_loss': 0.7384141313583996, 'avg_role_model_mean_pred_loss': 0.00020506313351988936, 'avg_role_model_g_mag_loss': 0.06779958754777908, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010329423837538343, 'n_size': 320, 'n_batch': 80, 'duration': 97.80484867095947, 'duration_batch': 1.2225606083869933, 'duration_size': 0.30564015209674833, 'avg_pred_std': 0.08878090149955824}\n", + "Val loss {'avg_role_model_loss': 0.004938754381146282, 'avg_role_model_std_loss': 1.1424394390228372, 'avg_role_model_mean_pred_loss': 2.856015009600199e-05, 'avg_role_model_g_mag_loss': 0.03729338594712317, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005005534488009289, 'n_size': 80, 'n_batch': 20, 'duration': 19.48591160774231, 'duration_batch': 0.9742955803871155, 'duration_size': 0.24357389509677888, 'avg_pred_std': 0.03839425216428936}\n", + "Epoch 4\n", + "Train loss {'avg_role_model_loss': 0.005628427321789786, 'avg_role_model_std_loss': 0.445644349324516, 'avg_role_model_mean_pred_loss': 4.922563089544974e-05, 'avg_role_model_g_mag_loss': 0.052951197209768, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005696333260493703, 'n_size': 320, 'n_batch': 80, 'duration': 97.72880864143372, 'duration_batch': 1.2216101080179214, 'duration_size': 0.30540252700448034, 'avg_pred_std': 0.0942096491693519}\n", + "Val loss {'avg_role_model_loss': 0.004431478650076315, 'avg_role_model_std_loss': 2.1805682998136033, 'avg_role_model_mean_pred_loss': 2.3924214234227746e-05, 'avg_role_model_g_mag_loss': 0.041412954684346914, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0044817406902438964, 'n_size': 80, 'n_batch': 20, 'duration': 19.514612197875977, 'duration_batch': 0.9757306098937988, 'duration_size': 0.2439326524734497, 'avg_pred_std': 0.03819809515262022}\n", + "Epoch 5\n", + "Train loss {'avg_role_model_loss': 0.004377504003969079, 'avg_role_model_std_loss': 0.7532617844777411, 'avg_role_model_mean_pred_loss': 4.913167144098285e-05, 'avg_role_model_g_mag_loss': 0.043540800781920554, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004429807130145491, 'n_size': 320, 'n_batch': 80, 'duration': 98.63056015968323, 'duration_batch': 1.2328820019960403, 'duration_size': 0.30822050049901006, 'avg_pred_std': 0.08962908287066966}\n", + "Val loss {'avg_role_model_loss': 0.003524563132668845, 'avg_role_model_std_loss': 3.541576160071827, 'avg_role_model_mean_pred_loss': 1.2924149661341922e-05, 'avg_role_model_g_mag_loss': 0.03394877891987562, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003568227968935389, 'n_size': 80, 'n_batch': 20, 'duration': 19.638448476791382, 'duration_batch': 0.9819224238395691, 'duration_size': 0.24548060595989227, 'avg_pred_std': 0.04204475942533463}\n", + "Epoch 6\n", + "Train loss {'avg_role_model_loss': 0.004308926320300088, 'avg_role_model_std_loss': 0.655986472072349, 'avg_role_model_mean_pred_loss': 3.8958016750684914e-05, 'avg_role_model_g_mag_loss': 0.04524608214851469, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004360771635401761, 'n_size': 320, 'n_batch': 80, 'duration': 98.1361870765686, 'duration_batch': 1.2267023384571076, 'duration_size': 0.3066755846142769, 'avg_pred_std': 0.08601754870906006}\n", + "Val loss {'avg_role_model_loss': 0.0037977110616338903, 'avg_role_model_std_loss': 2.063007281812861, 'avg_role_model_mean_pred_loss': 1.837681602836483e-05, 'avg_role_model_g_mag_loss': 0.04376108571887016, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0038385945270420054, 'n_size': 80, 'n_batch': 20, 'duration': 19.570854902267456, 'duration_batch': 0.9785427451133728, 'duration_size': 0.2446356862783432, 'avg_pred_std': 0.030497054848819972}\n", + "Epoch 7\n", + "Train loss {'avg_role_model_loss': 0.0038875900178936716, 'avg_role_model_std_loss': 0.8493629617666272, 'avg_role_model_mean_pred_loss': 2.666781584003133e-05, 'avg_role_model_g_mag_loss': 0.07926730818580836, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003934197903254244, 'n_size': 320, 'n_batch': 80, 'duration': 97.9080719947815, 'duration_batch': 1.2238508999347686, 'duration_size': 0.30596272498369215, 'avg_pred_std': 0.08284469123464078}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003410281174001284, 'avg_role_model_std_loss': 2.5351196237003024, 'avg_role_model_mean_pred_loss': 8.761383719502192e-06, 'avg_role_model_g_mag_loss': 0.06262274421751499, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034496398351620884, 'n_size': 80, 'n_batch': 20, 'duration': 19.63199734687805, 'duration_batch': 0.9815998673439026, 'duration_size': 0.24539996683597565, 'avg_pred_std': 0.029967716569080947}\n", + "Epoch 8\n", + "Train loss {'avg_role_model_loss': 0.003944453283475013, 'avg_role_model_std_loss': 0.697565211437049, 'avg_role_model_mean_pred_loss': 4.258767692717491e-05, 'avg_role_model_g_mag_loss': 0.04458279046230018, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003991481203775038, 'n_size': 320, 'n_batch': 80, 'duration': 97.73626518249512, 'duration_batch': 1.2217033147811889, 'duration_size': 0.3054258286952972, 'avg_pred_std': 0.08436954310745932}\n", + "Val loss {'avg_role_model_loss': 0.007964267671923153, 'avg_role_model_std_loss': 0.92843264617959, 'avg_role_model_mean_pred_loss': 8.528030000434227e-05, 'avg_role_model_g_mag_loss': 0.0632677624002099, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0080590668701916, 'n_size': 80, 'n_batch': 20, 'duration': 19.450220584869385, 'duration_batch': 0.9725110292434692, 'duration_size': 0.2431277573108673, 'avg_pred_std': 0.06122681526467204}\n", + "Epoch 9\n", + "Train loss {'avg_role_model_loss': 0.003340029970786418, 'avg_role_model_std_loss': 0.48781303250957364, 'avg_role_model_mean_pred_loss': 2.8307830278884367e-05, 'avg_role_model_g_mag_loss': 0.041397362318821254, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003379593089266564, 'n_size': 320, 'n_batch': 80, 'duration': 96.01197457313538, 'duration_batch': 1.2001496821641922, 'duration_size': 0.30003742054104804, 'avg_pred_std': 0.0903306363383308}\n", + "Val loss {'avg_role_model_loss': 0.003354643483180553, 'avg_role_model_std_loss': 1.8730609221261603, 'avg_role_model_mean_pred_loss': 1.555475001424611e-05, 'avg_role_model_g_mag_loss': 0.04130637706257403, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033913442108314483, 'n_size': 80, 'n_batch': 20, 'duration': 18.96759533882141, 'duration_batch': 0.9483797669410705, 'duration_size': 0.23709494173526763, 'avg_pred_std': 0.04209472953807562}\n", + "Epoch 10\n", + "Train loss {'avg_role_model_loss': 0.003940557214809815, 'avg_role_model_std_loss': 0.3181843262791631, 'avg_role_model_mean_pred_loss': 3.4807291759936246e-05, 'avg_role_model_g_mag_loss': 0.04127106771338731, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0039869014210125895, 'n_size': 320, 'n_batch': 80, 'duration': 95.93454957008362, 'duration_batch': 1.1991818696260452, 'duration_size': 0.2997954674065113, 'avg_pred_std': 0.09244233099743723}\n", + "Val loss {'avg_role_model_loss': 0.003921330184675753, 'avg_role_model_std_loss': 2.439302556675841, 'avg_role_model_mean_pred_loss': 1.5360680176002007e-05, 'avg_role_model_g_mag_loss': 0.04387447247281671, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003964195022854255, 'n_size': 80, 'n_batch': 20, 'duration': 18.936774015426636, 'duration_batch': 0.9468387007713318, 'duration_size': 0.23670967519283295, 'avg_pred_std': 0.0390258968109265}\n", + "Epoch 11\n", + "Train loss {'avg_role_model_loss': 0.0029607639175083023, 'avg_role_model_std_loss': 0.6851782390538916, 'avg_role_model_mean_pred_loss': 9.521213198121579e-06, 'avg_role_model_g_mag_loss': 0.03578661805950105, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029942192559246905, 'n_size': 320, 'n_batch': 80, 'duration': 96.0065131187439, 'duration_batch': 1.2000814139842988, 'duration_size': 0.3000203534960747, 'avg_pred_std': 0.08160512297181413}\n", + "Val loss {'avg_role_model_loss': 0.0032844552013557406, 'avg_role_model_std_loss': 1.9535773819076212, 'avg_role_model_mean_pred_loss': 1.0978048215903869e-05, 'avg_role_model_g_mag_loss': 0.03746633417904377, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033219303644727916, 'n_size': 80, 'n_batch': 20, 'duration': 19.31860041618347, 'duration_batch': 0.9659300208091736, 'duration_size': 0.2414825052022934, 'avg_pred_std': 0.03969717922154814}\n", + "Epoch 12\n", + "Train loss {'avg_role_model_loss': 0.0025715101775858782, 'avg_role_model_std_loss': 0.6255118230208495, 'avg_role_model_mean_pred_loss': 7.738168527621215e-06, 'avg_role_model_g_mag_loss': 0.0324334034929052, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002600309181434568, 'n_size': 320, 'n_batch': 80, 'duration': 95.97005558013916, 'duration_batch': 1.1996256947517394, 'duration_size': 0.29990642368793485, 'avg_pred_std': 0.08734330767765641}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002402786401216872, 'avg_role_model_std_loss': 1.9713338775481133, 'avg_role_model_mean_pred_loss': 4.400266065029967e-06, 'avg_role_model_g_mag_loss': 0.031274407636374235, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024293805909110233, 'n_size': 80, 'n_batch': 20, 'duration': 19.06187415122986, 'duration_batch': 0.9530937075614929, 'duration_size': 0.23827342689037323, 'avg_pred_std': 0.034856686973944305}\n", + "Stopped False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test ▄█▃▄▃▂▃▂▇▂▃▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▆▄▄▂▂▂▂▂▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▅█▃▃▂▃▁▁▇▃▃▃▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train █▃▅▂▃▂▂▁▁▃▃▁▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▇█▁▂▁▂▃▃▂▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▂█▆▃▂▂▂▄▂▂▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test ▄█▃▄▃▂▃▂▇▂▃▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▆▄▄▂▂▂▂▂▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test ▃█▂▂▂▁▂▁▅▂▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▄▂▂▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▃▁▂▂▅█▅▆▂▄▅▄▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▄█▄▅▂▅▄▆▄▃▁▄▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▅▇▆▆▇█▇█▆▁▁▅▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▆▆▆▆▆█▇▆▆▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▅▇▆▆▇█▇█▆▁▁▅▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▆▆▆▆▆█▇▆▆▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▅▇▆▆▇█▇█▆▁▁▅▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▆▆▆▆▆█▇▆▆▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00243\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.0026\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.03486\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.08734\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.03127\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.03243\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.0024\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00257\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 1e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 1.97133\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.62551\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.95309\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.19963\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.23827\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.29991\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 19.06187\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 95.97006\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 20\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/contraceptive/tab_ddpm_concat/4/wandb/offline-run-20240229_055259-0l1vy1nr\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_055259-0l1vy1nr/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 399, 'n_batch': 100, 'role_model_metrics': {'avg_loss': 0.005469150811475807, 'avg_g_mag_loss': 0.01941445733574633, 'avg_g_cos_loss': 0.006619339273136137, 'pred_duration': 1.4712414741516113, 'grad_duration': 4.747467756271362, 'total_duration': 6.218709230422974, 'pred_std': 0.08244121074676514, 'std_loss': 0.00028081866912543774, 'mean_pred_loss': 6.793341162847355e-05, 'pred_rmse': 0.07395371049642563, 'pred_mae': 0.05780092254281044, 'pred_mape': 0.13661588728427887, 'grad_rmse': 0.25440865755081177, 'grad_mae': 0.10522309690713882, 'grad_mape': 0.8996596336364746}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.005469150811475807, 'avg_g_mag_loss': 0.01941445733574633, 'avg_g_cos_loss': 0.006619339273136137, 'avg_pred_duration': 1.4712414741516113, 'avg_grad_duration': 4.747467756271362, 'avg_total_duration': 6.218709230422974, 'avg_pred_std': 0.08244121074676514, 'avg_std_loss': 0.00028081866912543774, 'avg_mean_pred_loss': 6.793341162847355e-05}, 'min_metrics': {'avg_loss': 0.005469150811475807, 'avg_g_mag_loss': 0.01941445733574633, 'avg_g_cos_loss': 0.006619339273136137, 'pred_duration': 1.4712414741516113, 'grad_duration': 4.747467756271362, 'total_duration': 6.218709230422974, 'pred_std': 0.08244121074676514, 'std_loss': 0.00028081866912543774, 'mean_pred_loss': 6.793341162847355e-05, 'pred_rmse': 0.07395371049642563, 'pred_mae': 0.05780092254281044, 'pred_mape': 0.13661588728427887, 'grad_rmse': 0.25440865755081177, 'grad_mae': 0.10522309690713882, 'grad_mape': 0.8996596336364746}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.005469150811475807, 'avg_g_mag_loss': 0.01941445733574633, 'avg_g_cos_loss': 0.006619339273136137, 'pred_duration': 1.4712414741516113, 'grad_duration': 4.747467756271362, 'total_duration': 6.218709230422974, 'pred_std': 0.08244121074676514, 'std_loss': 0.00028081866912543774, 'mean_pred_loss': 6.793341162847355e-05, 'pred_rmse': 0.07395371049642563, 'pred_mae': 0.05780092254281044, 'pred_mape': 0.13661588728427887, 'grad_rmse': 0.25440865755081177, 'grad_mae': 0.10522309690713882, 'grad_mape': 0.8996596336364746}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:20:40.213848Z", + "iopub.status.busy": "2024-02-29T06:20:40.213502Z", + "iopub.status.idle": "2024-02-29T06:20:40.217986Z", + "shell.execute_reply": "2024-02-29T06:20:40.217117Z" + }, + "papermill": { + "duration": 0.024761, + "end_time": "2024-02-29T06:20:40.219944", + "exception": false, + "start_time": "2024-02-29T06:20:40.195183", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:20:40.253763Z", + "iopub.status.busy": "2024-02-29T06:20:40.253512Z", + "iopub.status.idle": "2024-02-29T06:20:40.571348Z", + "shell.execute_reply": "2024-02-29T06:20:40.570260Z" + }, + "papermill": { + "duration": 0.337875, + "end_time": "2024-02-29T06:20:40.573901", + "exception": false, + "start_time": "2024-02-29T06:20:40.236026", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:20:40.609831Z", + "iopub.status.busy": "2024-02-29T06:20:40.609495Z", + "iopub.status.idle": "2024-02-29T06:20:40.883725Z", + "shell.execute_reply": "2024-02-29T06:20:40.882840Z" + }, + "papermill": { + "duration": 0.294533, + "end_time": "2024-02-29T06:20:40.885823", + "exception": false, + "start_time": "2024-02-29T06:20:40.591290", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAEVCAYAAABqhMkxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABROUlEQVR4nO2deVyU1f7HPzMDM2wyKNuAsrkgiIRekhFzySTRSxplWWZqXVNvqd0iS7mpWLcrpvbLMss2pUUDbTVxCVGvJbghKMoiGIgKg4KyKtvM9/fH4zw4MSwzDMwMnvfr9byYec73Oef7wMyHc55zzvcrICICg8Fg9CCExnaAwWAwDA0TNgaD0eNgwsZgMHocTNgYDEaPgwkbg8HocTBhYzAYPQ4mbAwGo8fBhI3BYPQ4mLAxGIweBxM2BoPR47DQ56JNmzZh3bp1UCgUCAoKwsaNGxESEtKq/c6dO7FixQoUFhZi0KBBePfdd/H3v/8dANDY2Ijly5djz549+PPPPyGVShEWFoY1a9bA3d2dr+PGjRtYvHgxfv31VwiFQkybNg0ffPAB7OzseJuzZ89i4cKFOHnyJJydnbF48WK88cYbHb4vlUqF4uJi9OrVCwKBQI/fDIPB6CqICNXV1XB3d4dQ2E6fjHQkPj6exGIxbdmyhc6fP0/z5s0jBwcHKi0t1Wp/9OhREolEtHbtWsrKyqLly5eTpaUlZWZmEhFRRUUFhYWFUUJCAuXk5FBqaiqFhIRQcHCwRj2TJk2ioKAgOnbsGP3+++80cOBAmjFjBl9eWVlJrq6uNHPmTDp37hx99913ZG1tTZ9++mmH7+3y5csEgB3sYIcJH5cvX273uywg0m0TvFwux4gRI/DRRx8B4Ho5Hh4eWLx4MZYtW9bC/qmnnkJtbS12797Nnxs5ciSGDRuGzZs3a23j5MmTCAkJwaVLl+Dp6Yns7GwMGTIEJ0+exP333w8A2LdvH/7+97/jypUrcHd3xyeffII333wTCoUCYrEYALBs2TL8/PPPyMnJ6dC9VVZWwsHBAZcvX4a9vb0uvxYGg9HFVFVVwcPDAxUVFZBKpW3a6jQUbWhoQFpaGqKjo/lzQqEQYWFhSE1N1XpNamoqoqKiNM6Fh4fj559/brWdyspKCAQCODg48HU4ODjwogYAYWFhEAqFOH78OB577DGkpqZi7NixvKip23n33Xdx8+ZN9O7du0U79fX1qK+v599XV1cDAOzt7ZmwMRgmSkceE+k0eVBWVgalUglXV1eN866urlAoFFqvUSgUOtnX1dVh6dKlmDFjBi8uCoUCLi4uGnYWFhbo06cPX09r7ajLtBEbGwupVMofHh4eWu0YDIZ5YVKzoo2NjZg+fTqICJ988kmXtxcdHY3Kykr+uHz5cpe3yWAwuh6dhqJOTk4QiUQoLS3VOF9aWgqZTKb1GplM1iF7tahdunQJBw8e1BgKymQyXLt2TcO+qakJN27c4OtprR11mTYkEgkkEklrt8tgMMwUnYRNLBYjODgYycnJiIyMBMBNHiQnJ2PRokVarwkNDUVycjJeeeUV/lxSUhJCQ0P592pRy8vLw6FDh+Do6NiijoqKCqSlpSE4OBgAcPDgQahUKsjlct7mzTffRGNjIywtLfl2Bg8erPX5GsOwqFQqNDQ0GNsNhpljaWkJkUjU6Xp0nhVNSEjAnDlz8OmnnyIkJAQbNmzAjh07kJOTA1dXV8yePRt9+/ZFbGwsACAlJQXjxo3DmjVrEBERgfj4eKxevRqnT5/G0KFD0djYiCeeeAKnT5/G7t27NZ6T9enTh58MmDx5MkpLS7F582Y0Njbi+eefx/3334/t27cD4CYcBg8ejIkTJ2Lp0qU4d+4c/vGPf+D999/H/PnzO3RvVVVVkEqlqKysZJMHOtDQ0ICCggKoVCpju8LoATg4OEAmk7WYJNDl+6nzAt2nnnoK169fx8qVK6FQKDBs2DDs27ePF6SioiKNxXOjRo3C9u3bsXz5cvz73//GoEGD8PPPP2Po0KEAgKtXr2LXrl0AgGHDhmm0dejQITz44IMAgG3btmHRokWYMGECv0D3ww8/5G2lUil+++03LFy4EMHBwXBycsLKlSs7LGoM/SAilJSUQCQSwcPDo/2FkwxGKxARbt26xT92cnNz07sunXtsPZmO/keoa1TiyIXr6O9sh4Eudq3a3Qs0NjYiPz8f7u7u7a4tYjA6Qnl5Oa5duwZfX1+NYakuPTb271UP3vzpHOZ/k4Ydp9gsqlKpBACN9YMMRmewsbEBwP3T1BcmbHoQ5s+tqdt/XgHW4eVge2sZhsIQnyUmbHow1tcZYgshLpXfQm5ptbHdYTAYf4EJmx7YSiwwdpATAGD/udJ2rBkM7QgEgja3Fpojq1atajEJaAyYsOnJxABu0e/+89q3azEY5kBcXBy/J9sQLFmyBMnJyQarT1+YsOlJmL8rhAIgq6QKl2/cMrY7DEaX0tHF13Z2di0W2BsDJmx60sdWjBCfPgBYr81c2bdvH0aPHg0HBwc4OjrikUcewcWLFwFw6y+XLl2qYX/9+nVYWlriyJEjAICSkhJERETA2toaPj4+2L59O7y9vbFhwwa9/MnMzMRDDz0Ea2trODo6Yv78+aipqeHLDx8+jJCQENja2sLBwQEPPPAALl26BAA4c+YMxo8fj169esHe3h7BwcE4depUm+0dPnwYzz//PB9NRyAQYNWqVQAAb29v/Oc//8Hs2bNhb2/PrwddunQpfH19YWNjg/79+2PFihUas5d/HYo+99xziIyMxPr16+Hm5gZHR0csXLiwUzOeHYEJWycIvzMc/e08e86mhohwq6HJKIeuM9S1tbWIiorCqVOnkJycDKFQiMceewwqlQozZ85EfHy8Rp0JCQlwd3fHmDFjAACzZ89GcXExDh8+jB9++AGfffZZiz3NuvgSHh6O3r174+TJk9i5cycOHDjAb1VsampCZGQkxo0bh7NnzyI1NRXz58/nZxBnzpyJfv364eTJk0hLS8OyZcv4rYWtMWrUKGzYsAH29vYoKSlBSUkJlixZwpevX78eQUFBSE9Px4oVKwAAvXr1QlxcHLKysvDBBx/g888/x/vvv99mO4cOHcLFixdx6NAhfPXVV4iLi0NcXJxev6eOoldocAbHxAAZ3vo1Cycv3UBZTT2c7NiG+tuNSgxZud8obWe9HQ4bccc/0tOmTdN4v2XLFjg7OyMrKwvTp0/HK6+8gj/++IMXsu3bt2PGjBkQCATIycnBgQMHNIKffvHFFxg0aJBevm/fvh11dXX4+uuvYWtrCwD46KOPMGXKFLz77ruwtLREZWUlHnnkEQwYMAAA4O/vz19fVFSE119/HX5+fgDQIT/EYjGkUikEAoHWQBEPPfQQXnvtNY1zy5cv5197e3tjyZIliI+PbzMEf+/evfHRRx9BJBLBz88PERERSE5Oxrx589r1UV9Yj60T9HWwRmBfKYiAA1ms12Zu5OXlYcaMGejfvz/s7e3h7e0NgBMJZ2dnTJw4Edu2bQMAFBQUIDU1FTNnzgQA5ObmwsLCAn/729/4+gYOHKh3wIXs7GwEBQXxogYADzzwAFQqFXJzc9GnTx8899xzCA8Px5QpU/DBBx+gpKSEt42KisILL7zA5wtRD6k7w92BXdUkJCTggQcegEwmg52dHZYvX46ioqI26wkICNDYQeDm5qZ3z7ajsB5bJwkPcEXm1UrsP6/A0yGexnbH6FhbipD1drjR2taFKVOmwMvLC59//jnc3d2hUqkwdOhQ/kH5zJkz8fLLL2Pjxo3Yvn07AgMDERgY2BWud4itW7fi5Zdfxr59+5CQkIDly5cjKSkJI0eOxKpVq/DMM88gMTERe/fuRUxMDOLj4/HYY4/p3d7dIguAF/a33noL4eHhkEqliI+Px3vvvddmPX8dEgsEgi4PmMB6bJ1E/ZztaH45quu69oGoOSAQCGAjtjDKocuK9fLycuTm5mL58uWYMGEC/P39cfPmTQ2bRx99FHV1ddi3bx+2b9/O99YAYPDgwWhqakJ6ejp/Lj8/v0UdHcXf3x9nzpxBbW0tf+7o0aMQCoUYPHgwf2748OGIjo5GSkoKhg4dyke3AQBfX1+8+uqr+O233/D4449j69at7bYrFov5bXHtkZKSAi8vL7z55pu4//77MWjQIH7ywtRgwtZJBrrYob+TLRqUKhzOvW5sdxgdpHfv3nB0dMRnn32G/Px8HDx4sEVuDltbW0RGRmLFihXIzs7GjBkz+DI/Pz+EhYVh/vz5OHHiBNLT0zF//nxYW1vrtSVo5syZsLKywpw5c3Du3DkcOnQIixcvxqxZs+Dq6oqCggJER0cjNTUVly5dwm+//Ya8vDz4+/vj9u3bWLRoEQ4fPoxLly7h6NGjOHnypMYzuNbw9vZGTU0NkpOTUVZWhlu3Wl+6NGjQIBQVFSE+Ph4XL17Ehx9+iJ9++knne+0OmLB1EoFAwBbrmiFCoRDx8fFIS0vD0KFD8eqrr2LdunUt7GbOnIkzZ85gzJgx8PTUfNTw9ddfw9XVFWPHjsVjjz2GefPmoVevXrCystLZHxsbG+zfvx83btzAiBEj8MQTT2DChAl8NjgbGxvk5ORg2rRp8PX1xfz587Fw4UIsWLAAIpEI5eXlmD17Nnx9fTF9+nRMnjwZb731Vrvtjho1Cv/85z/x1FNPwdnZGWvXrm3VdurUqXj11VexaNEiDBs2DCkpKfxsqanBwhbdhb6BJtOLbuKxj1NgJ7FA2oowSCw6HwHUXKirq0NBQQF8fHz0+kL3JK5cuQIPDw8cOHAAEyZMMLY7Zktrn6kuDTTJaElQPwe42ktQWlWPlIvlGD/Ypf2LGGbPwYMHUVNTg8DAQJSUlOCNN96At7c3xo4da2zX7nnYUNQACIUCPDyEiyD8GxuO3jM0Njbi3//+NwICAvDYY4/B2dkZhw8fhqWlJbZt2wY7OzutR0BAQLf5OHny5Fb9WL16dbf50d2wHpuBCA+Q4dtjRUjKKsU7kQSRkMUn6+mEh4cjPFz70papU6fyiYb+Sns7AgzJF198gdu3b2st69OnT7f50d0wYTMQI/s7wt7KAmU1DThddBMjvHvuh4bRPr169UKvXr2M7Qb69u1rbBeMAhuKGghLkRAT/Lnh6P5zbDjKYBgTvYRt06ZN8Pb2hpWVFeRyOU6cONGm/c6dO+Hn5wcrKysEBgZiz549GuU//vgjJk6cCEdHRwgEAmRkZGiUFxYW8tEH/nrs3LmTt9NWHh8fr88t6kV4wB1hy2IhwxkMY6KzsCUkJCAqKgoxMTE4ffo0goKCEB4e3urer5SUFMyYMQNz585Feno6IiMjERkZiXPnzvE2tbW1GD16NN59912tdXh4ePDRB9THW2+9BTs7O0yePFnDduvWrRp26sTO3cFYX2dILIS4fOM2sktYyHAGw2iQjoSEhNDChQv590qlktzd3Sk2Nlar/fTp0ykiIkLjnFwupwULFrSwLSgoIACUnp7erh/Dhg2jf/zjHxrnANBPP/3U/k20QmVlJQGgyspKvet44auT5LV0N/3fb7l612FO3L59m7Kysuj27dvGdoXRQ2jtM6XL91OnHltDQwPS0tIQFhbGnxMKhQgLC0NqaqrWa1JTUzXsAW42qTX7jpCWloaMjAzMnTu3RdnChQvh5OSEkJAQbNmypc0hYX19PaqqqjSOzhLOdiEwGEZHJ2ErKyuDUqnks76rcXV1hUKh/YusUCh0su8IX375Jfz9/TFq1CiN82+//TZ27NiBpKQkTJs2DS+99BI2btzYaj2xsbGQSqX84eHhobdPasL8XSASCpCjqEZROQsZzmidnpjMxVQwu1nR27dvY/v27Vp7aytWrMADDzyA4cOHY+nSpXjjjTe07v9TEx0djcrKSv64fLnzCZAdbMSQs5DhDDPB0MlcAC7kuEAgQEVFhUHr1QWdhM3JyQkikQilpZpBFUtLS7VG4AQAmUymk317fP/997h16xZmz57drq1cLseVK1dQX1+vtVwikcDe3l7jMARsOMpgGBedhE0sFiM4OFgjvZZKpUJycjJCQ0O1XhMaGtoiHVdSUlKr9u3x5ZdfYurUqXB2dm7XNiMjA71794ZE0r0huyfeWfaRVnQT16u1iyrD+LBkLq0nc6mvr8eSJUvQt29f2NraQi6X4/Dhw/y1ly5dwpQpU9C7d2/Y2toiICAAe/bsQWFhIcaPHw+ACw0lEAjw3HPP6fX76BS6zljEx8eTRCKhuLg4ysrKovnz55ODgwMpFAoiIpo1axYtW7aMtz969ChZWFjQ+vXrKTs7m2JiYsjS0pIyMzN5m/LyckpPT6fExEQCQPHx8ZSenk4lJSUabefl5ZFAIKC9e/e28GvXrl30+eefU2ZmJuXl5dHHH39MNjY2tHLlyg7fmyFmRdVM3fg7eS3dTduOXep0XaZMixkslYqovsY4h0qlk+/ff/89/fDDD5SXl0fp6ek0ZcoUCgwMJKVSSR999BF5enqS6q46N27cqHEuLCyMhg0bRseOHaO0tDQaN24cWVtb0/vvv9+h9nHXLH5NTQ25ubnR448/TpmZmZScnEw+Pj40Z84cIiJqbGwkqVRKS5Ysofz8fMrKyqK4uDi6dIn7fAUEBNCzzz5L2dnZdOHCBdqxYwdlZGS02X59fT1t2LCB7O3tqaSkhEpKSqi6upqIiF544QUaNWoUHTlyhPLz82ndunUkkUjowoULREQUERFBDz/8MJ09e5YuXrxIv/76K/3vf/+jpqYm+uGHHwgA5ebmUklJCVVUVHT0T0JEhpkV1VnYiJr/wGKxmEJCQujYsWN82bhx4/g/hpodO3aQr68vicViCggIoMTERI3yrVu3EoAWR0xMjIZddHQ0eXh4kFKpbOHT3r17adiwYWRnZ0e2trYUFBREmzdv1mrbGoYUto8O5pHX0t00+8vjna7LlGnxIayvIYqxN85RX9Ope7l+/ToBoMzMTLp27RpZWFjQkSNH+PLQ0FBaunQpERFlZ2cTADp58iRfnpeXRwD0ErbPPvuMevfuTTU1zfeQmJhIQqGQFAoFlZeXEwA6fPiw1rp69epFcXFxOt4x992TSqUa5y5dukQikYiuXr2qcX7ChAkUHR1NRESBgYG0atUqrXUeOnSIANDNmzd19ofIMMKm117RRYsW8WnB/srd3VU1Tz75JJ588slW63vuuec61F1dvXp1qxEJJk2ahEmTJrVbR3cRHuCKdftzkXKxDFV1jbC36r6Nz4yOkZeXh5UrV+L48eMoKyvj4/AXFRVh6NChfDKXMWPG8MlcPv30UwDdn8xl7NixfDKXhx9+GGFhYZg+fTrc3NwANCdz+eabbxAWFoYnn3ySz2alK5mZmVAqlfD19dU4X19fzydDfvnll/Hiiy/it99+Q1hYGKZNm4b77rtPr/a6ArYJvosY6NIL/Z1t8ef1WhzOvY6pQe7Gdql7sLQB/l1svLZ1gCVz0U5NTQ1EIhHS0tI0sksBXKZ3AHjhhRcQHh6OxMRE/Pbbb4iNjcV7772HxYsXG+ReO41efcUeiiGHokREa/Zmk9fS3fTStjSD1GeKmOvOg7KyMgKgMdT8/fffWzz3srW1pV27dtGQIUNozZo1vK16KHrq1Cn+XFcORbUxcuRIWrx4sdayp59+mqZMmdKuD9u2bSM7OzuNc7m5uS1+N+2xbNkyCgwMJCLuuToAKisr6/D1d9PtOw8YuqFe9nE45xrqGjuWCYjRPbBkLhzakrn4+vpi5syZmD17Nn788UcUFBTgxIkTiI2NRWJiIgDglVdewf79+1FQUIDTp0/j0KFDfHteXl4QCATYvXs3rl+/rjGz223oJak9FEP32JRKFcn/e4C8lu6m5Gzt/3XNHXPtsRERJSUlkb+/P0kkErrvvvvo8OHDLfYb79mzhwDQ2LFjW1xfXFxMkydPJolEQl5eXrR9+3ZycXGhzZs3d6j9v7Z19uxZGj9+PFlZWVGfPn1o3rx5/CylQqGgyMhIcnNzI7FYTF5eXrRy5UpSKpVUX19PTz/9NHl4eJBYLCZ3d3datGhRh/8m//znP8nR0VFjwq6hoYFWrlxJ3t7eZGlpSW5ubvTYY4/R2bNniYho0aJFNGDAAJJIJOTs7EyzZs3S6KG9/fbbJJPJSCAQtJhMbA9D9NhYMpe70DeZS1us/OUcvk69hKfu98C7T5jOw1VDwZK5NMOSuRgGQyRzYUPRLkY9HD2QXQqliv0P6UkcPHgQu3btQkFBAVJSUvD000+zZC4mAhO2LibEpw+k1pYor23AqcIbxnaHYUBYMhfThS336GK4kOEu+PH0Vew/Xwp5f0dju8QwECyZi+nChK0bCA+Q3RE2BVY84q/XrBnDvGDJXIwLG4p2A2MHOcPKUoirFbdxvrjzwSwZDEbbMGHrBqzFIozz5aKR9NSEymxynWEoDPFZYsLWTTTHaCttx9K8UG+5UW9DYjA6y61bXOTpzjyLZM/YuokJfq4QCQXILa1GYVktvJ1s27/IDLCwsICNjQ0fq0woZP8rGfpBRLh16xauXbsGBweHFvtUdYEJWzchtbHEyP59cDS/HPvPK7BgnH6RF0wNgUAANzc3FBQU8EEPGYzO4ODgoHeEbTVM2LqR8ABZjxM2gIusPGjQIDYcZXQaS0vLTvXU1DBh60YmDpFh5S/ncbqoAteq6uBi33O2IAmFwnt+SxXDdGAPRLoRmdQKQR4OAICk7J41icBgmBJM2LqZ8DuJXnra7CiDYUowYetm1Ms+Uu+EDGcwGIaHCVs3M8DZDgNd7NCoJBzKuWZsdxiMHolewrZp0yZ4e3vDysoKcrkcJ06caNN+586d8PPzg5WVFQIDA7Fnzx6N8h9//BETJ06Eo6MjBAIBMjIyWtTx4IMP8rkP1cc///lPDZuioiJERETAxsYGLi4ueP3119HU1KTPLXYpzcPRnrkLgcEwNjoLW0JCAqKiohATE4PTp08jKCgI4eHhuHZNe+8jJSUFM2bMwNy5c5Geno7IyEhERkbi3LlzvE1tbS1Gjx6Nd999t822582bh5KSEv5Yu3YtX6ZUKhEREYGGhgakpKTgq6++QlxcHFauXKnrLXY5fMjw3OssZDiD0RXoFLOXiEJCQmjhwoX8e6VSSe7u7hQbG6vVfvr06RQREaFxTi6X04IFC1rYFhQUEABKT09vUTZu3Dj617/+1apfe/bsaZH44pNPPiF7e3uqr69v5644DB0avDVUKhWFruZChied75khwxkMQ9NlyVwaGhqQlpaGsLAw/pxQKERYWBhSU1O1XpOamqphD3BxrFqzb4tt27bByckJQ4cORXR0NL+nTN1OYGAgXF1dNdqpqqrC+fPntdZXX1+PqqoqjaM7EAgEmMjvHWXDUQbD0OgkbGVlZVAqlRriAQCurq5QKLR/QRUKhU72rfHMM8/g22+/xaFDhxAdHY1vvvkGzz77bLvtqMu0ERsbC6lUyh8eHh46+dQZJt55znYguxRNSlW3tctg3AuYzc6D+fPn868DAwPh5uaGCRMm4OLFi3pnvI6OjtZIuVZVVdVt4hbi3QcONpa4easRaZdussi6DIYB0anH5uTkBJFIhNJSzcWlpaWlrW5alclkOtl3FHXY5fz8/DbbUZdpQyKRwN7eXuPoLixEQowf7AIAOJjLln0wGIZEJ2ETi8UIDg5GcnIyf06lUiE5ORmhoaFarwkNDdWwB4CkpKRW7TuKekmIm5sb305mZqbG7GxSUhLs7e0xZMiQTrXVVYz3uyNs2UzYGAyDouvMRHx8PEkkEoqLi6OsrCyaP38+OTg48LORs2bNomXLlvH2R48eJQsLC1q/fj1lZ2dTTEwMWVpaUmZmJm9TXl5O6enplJiYSAAoPj6e0tPTqaSkhIiI8vPz6e2336ZTp05RQUEB/fLLL9S/f3+NJLZNTU00dOhQmjhxImVkZNC+ffvI2dmZoqOjO3xv3TUrqqaitoH6RyeS19LdVFRe2y1tMhjmii7fT70ywW/cuJE8PT1JLBZTSEgIHTt2jC8bN25ci8zPO3bsIF9fXxKLxRQQEECJiYka5Vu3biUALQ51VuqioiIaO3Ys9enThyQSCQ0cOJBef/31FjdYWFhIkydPJmtra3JycqLXXnuNGhsbO3xf3S1sRERPbk4hr6W7Ke5oQbe1yWCYIywTvJ50RSb49vj0fxcRuzcH43yd8dU/QrqlTQbDHGGZ4M2Ih+48Z0v9sxy3Gkxv+xeDYY4wYTMyA13s0K+3NRqaVDiaX25sdxiMHgETNiMjEAgwQT07yqJ9MBgGgQmbCaBe9nEo5xrLz8lgGAAmbCbAyP6OsLYUQVFVh6wSlimewegsTNhMACtLER4Y6ASALdZlMAwBEzYTYYI/217FYBgKJmwmgnrfaMblCpTX1BvZGwbDvGHCZiLIpFYY4mYPIi6yLoPB0B8mbCYEG44yGIaBCZsJoV72cST3OhpZ8EkGQ2+YsJkQQf0c4GgrRnV9E04V3jS2OwyG2cKEzYQQCQUYN9gZAHAwh2WKZzD0hQmbiTHBj8uFwLZXMRj6w4TNxBjj6wQLoQAXr9fiUnmtsd1hMMwSJmwmhr2VJUZ49wHAem0Mhr4wYTNBHmLRPhiMTsGEzQRRL/s4/ucN1Naz4JMMhq4wYTNBBjjbwsvRBg1KFf7ILzO2OwyG2cGEzQQRCATNOUdZtA8GQ2f0ErZNmzbB29sbVlZWkMvlOHHiRJv2O3fuhJ+fH6ysrBAYGIg9e/ZolP/444+YOHEiHB0dIRAI+Jyham7cuIHFixdj8ODBsLa2hqenJ15++WVUVlZq2AkEghZHfHy8PrdodNTbqw7lXoNKxYJPMhi6oLOwJSQkICoqCjExMTh9+jSCgoIQHh6ukaj4blJSUjBjxgzMnTsX6enpiIyMRGRkJM6dO8fb1NbWYvTo0Xj33Xe11lFcXIzi4mKsX78e586dQ1xcHPbt24e5c+e2sN26dStKSkr4IzIyUtdbNAlCfPrARizCtep6nC9mwScZDJ3QNbdfSEgILVy4kH+vVCrJ3d2dYmNjtdpPnz6dIiIiNM7J5XJasGBBC9uCggICQOnp6e36sWPHDhKLxRp5QwHQTz/91LEb0YIx8oq2xfyvT5LX0t20IemCsV1hMIyOLt9PnXpsDQ0NSEtLQ1hYGH9OKBQiLCwMqampWq9JTU3VsAeA8PDwVu07ijq3oIWFhcb5hQsXwsnJCSEhIdiyZUubOQTq6+tRVVWlcZgS/LIPFu2DwdAJnYStrKwMSqUSrq6uGuddXV2hUCi0XqNQKHSy76gf//nPfzB//nyN82+//TZ27NiBpKQkTJs2DS+99BI2btzYaj2xsbGQSqX84eHhobdPXYF6AuHM5Qpcr2bBJxmMjmLRvolpUVVVhYiICAwZMgSrVq3SKFuxYgX/evjw4aitrcW6devw8ssva60rOjoaUVFRGnWbkri52FshsK8UmVcrcTj3Gp6833R8YzBMGZ16bE5OThCJRCgt1Yw8UVpaCplMpvUamUymk31bVFdXY9KkSejVqxd++uknWFpatmkvl8tx5coV1Ndr7+1IJBLY29trHKbGeLYLgcHQGZ2ETSwWIzg4GMnJyfw5lUqF5ORkhIaGar0mNDRUwx4AkpKSWrVvjaqqKkycOBFisRi7du2ClZVVu9dkZGSgd+/ekEgkOrVlSqiTKf+eV4aGJhZ8ksHoCDoPRaOiojBnzhzcf//9CAkJwYYNG1BbW4vnn38eADB79mz07dsXsbGxAIB//etfGDduHN577z1EREQgPj4ep06dwmeffcbXeePGDRQVFaG4uBgAkJubC4Dr7clkMl7Ubt26hW+//VbjQb+zszNEIhF+/fVXlJaWYuTIkbCyskJSUhJWr16NJUuWdO43ZGQC+0rhZCdBWU09ThXewKg7afoYDEYb6DPtunHjRvL09CSxWEwhISF07NgxvmzcuHE0Z84cDfsdO3aQr68vicViCggIoMTERI3yrVu3EoAWR0xMDBERHTp0SGs5ACooKCAior1799KwYcPIzs6ObG1tKSgoiDZv3kxKpbLD92Vqyz3ULNmRQV5Ld9Pbv543tisMhtHQ5fspIGpjPcQ9RlVVFaRSKb+UxFTYm1mCF7edRn8nWxxc8qCx3WEwjIIu30+2V9QMGD3ICZYiAf4sq0VBGQs+yWC0BxM2M6CXlSVCfFjwSQajozBhMxP4aB8syQuD0S5M2MyECf7c7o0TBTdQw4JPMhhtwoTNTPBxsoWPky0alYQ/8q4b2x0Gw6RhwmZGqDfFJ7PgkwxGmzBhMyPUwnYo9zoLPslgtAETNjNihHcf2EksUFZTj8yrle1fwGDcozBhMyPEFkKMGcRtqWLLPhiM1mHCZmawaB8MRvswYTMz1OvZMq9W4lpVnZG9YTBMEyZsZoZzLwmC+kkBAIdz2bIPBkMbTNjMkIf8uMW6yWwXAoOhFSZsZoh62ccfeWWob1Ia2RsGw/RgwmaGBLjbw6WXBLUNSpwouGFsdxgMk4MJmxkiFAru2hTPZkcZjL/ChM1MuXvZB4sVymBowoTNTBk9yAlikRCXym/hTxZ8ksHQgAmbmWInsYC8Pxd88pABhqMqFbGJCEaPwewSJjOaecjPBb/nlSE5+xpeGNO/w9fVNymRV1qDrOIqZJVUIau4CtklVbjVqMS2F+QY2d+xC71mMLoevXpsmzZtgre3N6ysrCCXy3HixIk27Xfu3Ak/Pz9YWVkhMDAQe/bs0Sj/8ccfMXHiRDg6OkIgECAjI6NFHXV1dVi4cCEcHR1hZ2eHadOmtUjEXFRUhIiICNjY2MDFxQWvv/46mpp6blBG9bKPk4U3UFXXqNWm8lYjUi+W44vf/0TUjgxM2nAEASv345GNf+CNH84iLqUQJwpvoLq+CUoVIf5EUXfeAoPRJejcY0tISEBUVBQ2b94MuVyODRs2IDw8HLm5uXBxcWlhn5KSghkzZiA2NhaPPPIItm/fjsjISJw+fRpDhw4FANTW1mL06NGYPn065s2bp7XdV199FYmJidi5cyekUikWLVqExx9/HEePHgUAKJVKREREQCaTISUlBSUlJZg9ezYsLS2xevVqXW/TLPBytMUAZ1tcvF6L3y+UIchDyvfCzhdzPbGrFbe1Xiu1tkSAuz2GuNljiDuX8SdqxxkczLmGRqUKliL2lIJhvuicfk8ul2PEiBH46KOPAHCZ4D08PLB48WIsW7ashf1TTz2F2tpa7N69mz83cuRIDBs2DJs3b9awLSwshI+PD9LT0zFs2DD+fGVlJZydnbF9+3Y88cQTAICcnBz4+/sjNTUVI0eOxN69e/HII4+guLgYrq7cyvzNmzdj6dKluH79OsRicbv3Zqrp99riv4lZ+Pz3AggFQGsh2vr1tr4jYlIMceeEzF1qBYFAwNsoVYQR/z2AG7UN2D5PjlEDWGJmhmmhy/dTpx5bQ0MD0tLSEB0dzZ8TCoUICwtDamqq1mtSU1MRFRWlcS48PBw///xzh9tNS0tDY2MjwsLC+HN+fn7w9PTkhS01NRWBgYG8qKnbefHFF3H+/HkMHz68Rb319fWor6/n36uzy5sTfw90w+e/F0BFgIVQgEGuvTDEzZ4TMnd7+LvZQ2pt2W49IqEAD/m54Pu0K0jKKmXCxjBrdBK2srIyKJVKDfEAAFdXV+Tk5Gi9RqFQaLVXKBQdblehUEAsFsPBwaHVelprR12mjdjYWLz11lsd9sMUGe7ZG4kvjwYRMMjVDhILkd51PTzEFd+nXcGB7FKsfGSIRo+OwTAn7ukHKdHR0aisrOSPy5cvG9slvQhwl2JoX2mnRA0AxgxygsRCiMs3biO3tNpA3jEY3Y9Owubk5ASRSNRiNrK0tBQymUzrNTKZTCf71upoaGhARUVFq/W01o66TBsSiQT29vYax72MjdgCowdyQ9Ck8yxyCMN80UnYxGIxgoODkZyczJ9TqVRITk5GaGio1mtCQ0M17AEgKSmpVXttBAcHw9LSUqOe3NxcFBUV8fWEhoYiMzMT1641L1ZNSkqCvb09hgwZ0uG27nUeHsIN3w9kM2FjmDGkI/Hx8SSRSCguLo6ysrJo/vz55ODgQAqFgoiIZs2aRcuWLePtjx49ShYWFrR+/XrKzs6mmJgYsrS0pMzMTN6mvLyc0tPTKTExkQBQfHw8paenU0lJCW/zz3/+kzw9PengwYN06tQpCg0NpdDQUL68qamJhg4dShMnTqSMjAzat28fOTs7U3R0dIfvrbKykgBQZWWlrr+WHkNp1W3yXrabvJbuJkXlbWO7w2Dw6PL91FnYiIg2btxInp6eJBaLKSQkhI4dO8aXjRs3jubMmaNhv2PHDvL19SWxWEwBAQGUmJioUb5161YC0OKIiYnhbW7fvk0vvfQS9e7dm2xsbOixxx7TED4iosLCQpo8eTJZW1uTk5MTvfbaa9TY2Njh+2LCxhG56Q/yWrqbvj1WaGxXGAweXb6fOq9j68mY4zq2rmDToXys25+LBwc7I+75EGO7w2AA0O37eU/PijK0M/HOc7aU/HLU1PfcLWmMngsTNkYLBrrYwdvRBg1KFX6/wBLGMMwPJmyMFggEAoT5c722pCw2O8owP5iwMbSiXvZxMPcampQqI3vDYOgGEzaGVoK9esPBxhIVtxpx6tJNY7vDYOgEEzZjcuJzYNdioKnB2J60wEIk5OO9HWDDUYaZwYTNWNSWAfuWAae/BnL3tG9vBNSzo0nZpSxhDMOsYMJmLM4mAKo7SymyfjGuL60wZpAzxBZcwpi8azXGdofB6DBM2IwBEZC+rfn9hf1Ao/ZIt8bEVmKBBwZw+Q/Y7CjDnGDCZgyK04Fr5wGRBLCTAY21QH5y+9cZgbAhbNkHw/xgwmYM0r/lfvpPAYZO415n7zKeP22gXs+WcbkC16rrjOwNg9ExmLB1N423gczvudfDnwWGTOVe5+4Fmupbv85IuNpbIaifFACQnN35/KUMRnfAhK27yUkE6isBqQfgMw7oF8INR+urgD8PG9s7rTzMhqMMM4MJW3eT/g33c9hMQCjkDnWvLcs0h6MPD+EiEP+RX4ZbDWxTPMP0YcLWndy8BPz5P+71sBnN5/3vCFvObkCpPfGxMfF1tYNHH2s0NKlw5EKZsd1hMNqFCVt3cuY7AAT4jAV6ezef9xoF2DgBdRVAwREjOdc6AoEAD/tzvTYWMpxhDjBh6y5UKiDjztq14bM0y4QiboYUMN3Z0SHc9qqDOdegbC0zM6N1sncD7w8Fio4Z25N7AiZs3UXh70BFESCRNovY3Qx5lPuZvRtQmt5zrBDvPpBaW+JGbQPS2KZ43fnjfaDyMnDsY2N7ck/AhK27UK9dC5wGWFq3LPceDVj3Bm6VAUUp3etbB9DYFM+Go7pRrQCunuJe5yeb5LKengYTtu7gdkXzEHPYs9ptRJaAXwT32kT3jt4dfJJtiteB3L3NrxtquN47o0vRS9g2bdoEb29vWFlZQS6X48SJE23a79y5E35+frCyskJgYCD27NGMZkFEWLlyJdzc3GBtbY2wsDDk5eXx5YcPH4ZAINB6nDx5EgBQWFiotfzYMRN4pnHuB6CpDnD2B/r+rXW7IZHcz+xfuWdyJsa4wc4Qi4QoKKvFxeu1xnbHfFBHb7G0ufN+b+u2DIOgs7AlJCQgKioKMTExOH36NIKCghAeHq6RqPhuUlJSMGPGDMydOxfp6emIjIxEZGQkzp07x9usXbsWH374ITZv3ozjx4/D1tYW4eHhqKvjtvCMGjUKJSUlGscLL7wAHx8f3H///RrtHThwQMMuODhY11s0PPykwbOAQNC6nc847hlcTSlw+Xj3+KYDdhILjGSb4nWjvqZ5ic+4pdzP3L1cIARG16Frbr+QkBBauHAh/16pVJK7uzvFxsZqtZ8+fTpFRERonJPL5bRgwQIiIlKpVCSTyWjdunV8eUVFBUkkEvruu++01tnQ0EDOzs709ttv8+cKCgoIAKWnp+t6SzxdkldUcZ4oxp7orT5E1dfat/9hPme/Z6nhfDAgX6cWktfS3fT4x0eN7Yp5cP5n7u+5IYio4RbROzLufXGGsT0zO3T5furUY2toaEBaWhrCwsL4c0KhEGFhYUhNTdV6TWpqqoY9AISHh/P2BQUFUCgUGjZSqRRyubzVOnft2oXy8nI8//zzLcqmTp0KFxcXjB49Grt2tb10or6+HlVVVRqHwVH31nwnAXbO7dvzs6O7THI4GubPTSCcLrqJ69XsIXi75NwZhvpFcJNGAx7i3rPhaJeik7CVlZVBqVTC1dVV47yrqysUCoXWaxQKRZv26p+61Pnll18iPDwc/fr148/Z2dnhvffew86dO5GYmIjRo0cjMjKyTXGLjY2FVCrlDw8Pj1Zt9ULZCJyJ514Pb2XS4K8MeAgQ2wFVV4Hi04b1xwC4Sa0R2FcKIuBgDhuOtomyCcjbz70e/Pc7PydzP000anJPwexmRa9cuYL9+/dj7ty5GuednJwQFRUFuVyOESNGYM2aNXj22Wexbt26VuuKjo5GZWUlf1y+fNmwzl7Yzy3fsHMFBj7csWssrQDfcO511s+G9cdANG+KZ9E+2qQoFbh9E7DuA3jIuXODwgEIgJIzQOVVo7rXk9FJ2JycnCASiVBaqvmfurS0FDKZTOs1MpmsTXv1z47WuXXrVjg6OmLq1Knt+iuXy5Gfn99quUQigb29vcZhUNRr14KeBkQWHb9OPRzN+sUkHzKrl338kX8dtxuURvbGhFH3ynwnNf/97ZwBjxDu9QU2HO0qdBI2sViM4OBgJCc3R3tVqVRITk5GaGio1mtCQ0M17AEgKSmJt/fx8YFMJtOwqaqqwvHjx1vUSUTYunUrZs+eDUtLy3b9zcjIgJubW4fvz6BUK4C837jXra1da42BD3NLAyqKuP/sJoa/Wy/0dbBGXaMKf+SzTfFaIeJCVAGA3981y/jhKBO2rkLnoWhUVBQ+//xzfPXVV8jOzsaLL76I2tpa/kH+7NmzER0dzdv/61//wr59+/Dee+8hJycHq1atwqlTp7Bo0SIA3AbrV155Be+88w527dqFzMxMzJ49G+7u7oiMjNRo++DBgygoKMALL7zQwq+vvvoK3333HXJycpCTk4PVq1djy5YtWLx4sa63aBjOxAOk5OKtOfvqdq3YBhh4ZzLFBBfrCgSCu4aj2p+D3vNcywIqLgEWVs0TBmoG31mIXXAEqK/uft/uAXQYH3E89dRTuH79OlauXAmFQoFhw4Zh3759/MP/oqIiCIXNejlq1Chs374dy5cvx7///W8MGjQIP//8M4YOHcrbvPHGG6itrcX8+fNRUVGB0aNHY9++fbCystJo+8svv8SoUaPg5+en1bf//Oc/uHTpEiwsLODn54eEhAQ88cQTut5i5yHSXLumD0Me5WZGs34BJqxse/2bEXh4iCviUgqRnM1tihcJTcs/o6OeDe3/ICC21SxzGgT0GQDcuAhcPNj86IFhMAREJvgQx0hUVVVBKpWisrKyc8/bLp8AvrwznHwtF7DSo676amDtAEBZD7yYArgG6O9PF9CoVOFv/0lCdV0TfngxFMFefYztkmnx2YNc0p4pHwLBc1qW738TSP0ICJoBPLa5290zR3T5fprdrKhZoI6SOyRSP1EDAEkvkx6OWoqEGD+YW9P2G9uFoElVMSdqEDQ/T/sr6uUfF/abZDQXc4cJm6FpqAXO/ci9Hj6zc3XxIcNNT9iA5mUfB5iwaaKeDe03ArBz0W7jIeeiudy+AVxpe681Q3eYsBmarF1cBIfePoDXA52ry3cSILQErucA13MN458BGTfYGZYiAS5er8Wf11mmeB5+t8HfW7cRWdxZ0wa2WLcLYMJmaNRr14bP7PwDf2sHYMB47rUJJnqxt7LEyP5sU7wGdVXN4d3Vs5+twZZ9dBlM2AxJ+UXg0h8ABEDQM4ap099MhqMs+CRH/gFA1Qg4Dmx/mc/ACYBIDJTnA2V5bdsydIIJmyHJ2M79HPAQIO1rmDr9IgCBCCjN5ITTxJhwZxdC2qWbKK9hm+L5YeXgNoahaiS9AO8xmtcxDAITNkOhUt7JQgX9165pw6YPl9UKMMlEL30drBHgbg8VcYle7mmUjc27TfzaGYaqYcPRLoEJm6H48xAXkcO6d8c/1B3l7r2jJsjdIcPvaS4dBeoquVSK/UZ07Bq1sF0+DtSy7WmGggmboeCTtUwHLCSGrdvvEUAg5NZG3bxk2LoNgPo52+95ZahrvIc3xefcteldKOrYNdJ+gOw+gFTNvT1Gp2HCZghu3Wje8GzIYagaO+fmpSPZvxq+/k4S4G4Pd6kVbjcqcfRe3RRP1PycrK1lHtpQP49jz9kMBhM2Q5D5PaBsAGSBgNt9XdOGCQ9HBQIBwobc48NRRSaXN9TCGug/Xrdr1cPR/INAY53hfbsHYcJmCNRbqP6a4d2Q+D3C/bxywiQDFDYv+7gG1b2YKV7d2xownovOogtuQUAvd6CxlqXmMxBM2DpLyRlAcZZbjxT4ZNe1Y+8GeIzkXpvgcFTu44heEguU1dQj40qFsd3pftSPIjqyzOOvCAQsZLiBYcLWWdLvhCfyi+CWZnQldyd6MTHEFkKMG8wlq7nn9o5WXOb+uUHATRzoA/+cjaXmMwRM2DpDUz2QuYN7rWuUXH3wn8L9vJQCVJueeKiHo/vOK9CoNL0MW12Geg2ah7xjmci04TOGS+JTXQKUZBjMtXsVJmydIXcPl6yjl3vzns6uxMED6BsMgIAc0xuOPjjYBdaWIvx5vRZzvzqF2vp7JBxPbishwHXBQsJS8xkQJmydQb12bdgzHV+31Fn42VHTG45KrS2xaeZwWFkKceTCdTz92bGen3v0dgVQ+Af3ur1N7+3Bln0YDCZs+lJ5Fci/k4BmmIE2vHcE9ab4wj9McqX6Q36u+G7eSPSxFSPzaiUe/+QoCspqje1W15F/AFA1AU6+gNPAztU1aCK3EFuRyT23Y+gNEzZ9OfMdAAK8RgOOA7qv3T4+d1aqK5tn4kyM4Z698cOLo+DZxwaXb9zGtE9SkF5009hudQ2dmQ39K7aOzTPfF/Z1vr57GCZs+kCkGXetuzHhxbpqfJxs8cOLoxDYV4obtQ2Y8fkxJPe00EZNDVyPDTDc/mC27MMg6CVsmzZtgre3N6ysrCCXy3HiRNuhjXfu3Ak/Pz9YWVkhMDAQe/Zo/tGICCtXroSbmxusra0RFhaGvDzN+FTe3t4QCAQax5o1azRszp49izFjxsDKygoeHh5Yu3atPrfXPpdSgJsF3CyWMTIMDYnkfhb8j5u8MFGce0kQP38kxvk6o65RhXlfn8J3J4qM7ZbhKPwdqK8CbF2Avvcbpk51z6/gdy5oJUMvdBa2hIQEREVFISYmBqdPn0ZQUBDCw8Nx7Zr2kDUpKSmYMWMG5s6di/T0dERGRiIyMhLnzp3jbdauXYsPP/wQmzdvxvHjx2Fra4vw8HDU1WluL3n77bdRUlLCH3fnDK2qqsLEiRPh5eWFtLQ0rFu3DqtWrcJnn32m6y22j7q3NvTxlqnVugOngYBLAPdsx8Rn0GwlFvhizv14IrgfVARE/5iJ95MuoEckR+Njr00ChAYa/DgNBBwHccEqLya3b8/QDulISEgILVy4kH+vVCrJ3d2dYmNjtdpPnz6dIiIiNM7J5XJasGABERGpVCqSyWS0bt06vryiooIkEgl99913/DkvLy96//33W/Xr448/pt69e1N9fT1/bunSpTR48OAO31tlZSUBoMrKyrYNK64Q/W8d0dXTHa7b4ByKJYqxJ9o23TD1NTUQVZUQqVSGqe8vqFQqWr8/h7yW7iavpbvpjZ1nqLFJ2SVtdQsqFdF7/tzfIGevYevev5yr94d5hq3XzOnw95OIdPo309DQgLS0NISFhfHnhEIhwsLCkJqaqvWa1NRUDXsACA8P5+0LCgqgUCg0bKRSKeRyeYs616xZA0dHRwwfPhzr1q1DU1PzOqnU1FSMHTsWYrFYo53c3FzcvKl9uFZfX4+qqiqNo0NI+wJjlwDuwztm3xWoh8AXD+o/ZLl1Azi7A/j+H8C6AcB7g4FPRgFHPwCqSgznK7iN8q9NHIx3IodCKAASTl3GvK9P4VaDma51K8ng4u9Z2gD9xxm2bpaar9PoJGxlZWVQKpV81nc1rq6uUCgUWq9RKBRt2qt/tlfnyy+/jPj4eBw6dAgLFizA6tWr8cYbb7Tbzt1t/JXY2FhIpVL+8PDwaPXeTQ5nP26JgbKB+wJ0BCLgWg7wx/vAlkmcmP04Dzj3AxcgEQCuZQFJK4H3hwDfPA6c3Qk03DKY28+O9MLmZ4MhsRDiUO51zPjsGMrMMaS4OvbagIcAS2vD1u0RAlj3AeoqgMvHDFv3PYKFsR3oKFFRUfzr++67D2KxGAsWLEBsbCwkEv0CO0ZHR2vUW1VVZT7iJhBwa9p+Xw9k/Qzc18oG/KZ6bs3bhf3cEoKKvwSqdB0K+IZzexwdB3IzrWfiuS/UxWTuEPcCAh7lspZ7jur086SJATJsnzcSc786iTNXKvHEJyn46h8h8HI0wvNKfeFjrxk4WjLALfb2nQSc2c49Q/Uebfg2ejg6CZuTkxNEIhFKSzWn7UtLSyGTybReI5PJ2rRX/ywtLYWbm5uGzbBhw1r1RS6Xo6mpCYWFhRg8eHCr7dzdxl+RSCR6i6JJMORRTtjyDwD1NYDEjjtfXcpFY72wD7h4iAuHo0Yk4XIo+IZzh4OnZp33P88d5ReBswncer2KIm7CJP1bzv6+p4Ggpzu1fi/Yi1vrNmfLCRSW38LjH6dgy3MjEOThoHed3cbNQqD0HLeYVp0b1NAMnswJW04iMPGdzqdyvMfQ6V+vWCxGcHAwkpObZ2tUKhWSk5MRGhqq9ZrQ0FANewBISkri7X18fCCTyTRsqqqqcPz48VbrBICMjAwIhUK4uLjw7Rw5cgSNjY0a7QwePBi9e/fW5TbNB1kgl5i5qQ448Snwv7XAZ+OB93yBXYuAnN2cqNm5An+bDTz9HbC0AHj2eyBkXktRuxvHAcD4fwMvnwGe38tdL7HnRO7IWmDj34AvHgZObdF7yckAZzv8+OIoDHGzR3ltA57+7BgO5ZpBQhj1TLRnKLeotisY8BAXCutmAVB2oWva6MnoOjMRHx9PEomE4uLiKCsri+bPn08ODg6kUCiIiGjWrFm0bNky3v7o0aNkYWFB69evp+zsbIqJiSFLS0vKzMzkbdasWUMODg70yy+/0NmzZ+nRRx8lHx8fun37NhERpaSk0Pvvv08ZGRl08eJF+vbbb8nZ2Zlmz57N11FRUUGurq40a9YsOnfuHMXHx5ONjQ19+umnHb43XWZdTIbfVnIzaH89No/lZk6vniZSGmj2seEW0dmdRN9MI1rl0NzW205ECbOIcvZws6s6UnW7gZ794hh5Ld1N/aMTKeFkkWH87Sq2RnD3fXRj17bzzTSund//r2vbMRN0+X7qLGxERBs3biRPT08Si8UUEhJCx44d48vGjRtHc+bM0bDfsWMH+fr6klgspoCAAEpMTNQoV6lUtGLFCnJ1dSWJREITJkyg3NxcvjwtLY3kcjlJpVKysrIif39/Wr16NdXV1WnUc+bMGRo9ejRJJBLq27cvrVmzRqf7Mkthu5ZD9B8XondkRNtnEJ2KI6os7vp2q0qIjn5ItClUU1Df7U+UtIqovkan6uoblfRKfDq/HOSDAxdI1UVLTzpFbTnRqt7cvZZf7Nq2TnzBtfPFw13bjpmgy/dTQNQTVkoahqqqKkilUlRWVsLe3t7Y7nScuipu2GJpZZz2S85yEw6ZO4Da69w5By9gyobmUDwdgIiwdn8uPjnMJYYWCABbsQVsxCLYSixgKxHBRmwBO8mdc2IL/rytxAK2Yq787nPOdhJ49NExVHdbnEkAfpoPOPsDC7t4xrLyKjc7DQGwJE//WG89BF2+n2YzK8poAysji7Dbfdzx8NtcbLL9b3Kzr988BgQ9A4T/t0PRhQUCAZZO8oOb1Aqr92SjrlGFmvom1NQ3AZ0If9Tf2RYT/Fwwwd8V93v1hoWoE7O6hoi91lGkfQG3Ydyaubz9XZMBrYfCemx3YbY9NlOjvho4+A5w/FMABNg6A5PfBQIe7/DsXl2jEtV1Taitb0JtQxNq65WobWjCrXrlXeeaUNtw5329ErcaOBG81dBsU1JRh6a7kstIrS3x4GBnPOTnggd9XSC1sez4fTXVA2v7Aw01wAsHgX7BOv5i9ODwu8Dh1Vwyn6e3dX17Jowu308mbHfBhM3AXD4B7FoMXM/h3vtOBiLe43oi3UR1XSOOXChDcnYpDuVew81bzbPmIqEAI7x7I8zfFQ/5uaC/s13bleUlAdueAOxkQFS24faHtkXJWeDTMdwOhzf+NPxiYDOCCZueMGHrApoagD/+DziyntvYLe4FhMUA98/tHmG4C6WKkF50EweyryE5uxR512o0yvs72WKCvwse8nPFCG8tQ9ZfXwHStgLBz3PPD7sDIuD9oUDVFeCZHdzaw3sUJmx6woStC7mWw/XertwJceUxEpi6EXD2NZpLReW3kJxTiuTsazheUI5GZfNXwd7KAg8OdsEE/ztDVisR8H/+QI0Ct6cnoMZjPOoalbjVoMTtRiVuNyg13zcqcbuhCbcbVLjdyJUBQHiADCP794FAlwW3iUuAk58Dwc8BUz7o9H3/WVqJY8dTMDRgKO4bYCY7bcCETW+YsHUxKhVw6kvgwCruOZVIDIx9A3jgX4CFuN3LuxJ+yJpTikM5LYesI8UF2IZ/o4as8Lf6T9EAHZ7N/QU/WS88N8obkcP7wsqyA7ky8pOBbx/v1BBYpSKknjmPksOfY2TFbvQTlKGarJHb93EEPbEMln3aWKxtIjBh0xMmbN1ExWUgMYrb9gUALkO43ls/AwRrbLgFlJ4HFGe43AH1NYD3A9yyk97eHapC25B1iUUCFln8gkRlCBY2vgIAsLYUwUYsgpWlCNZiEazv/qkuEze/vl5dj18yinH7Tu/NwcYSM0I8MWukF9wd2nh21lQPrB0ANFQD8w7eyVTWMapu1yPltx9gffZrjGo6AUsB13YTRLAA91oJIW4NehS9xr8CuA/rcN3dDRM2PWHC1o0QcVFF9r4B3CoHIABGvgiMf7N5z2t71JY3C1jJWS5pcXk+QK3kNO3TnxO4AQ8B3mM6vEymuOI2HL8eB8mNXNT8/WNYDH8aEguhbsPJO1TeasSOU5fxVWohrty8DYDrEYYHuOK5UT4Y4d1be7075nDBDsa+Djy0vN12CgoLkLf/U/gV/whPQfMe6su2gbAeNQ9OI57E8UO7QKkfYSQy+XLyHgPBqJeBgWHd/gy0PZiw6QkTNiNQWw789uad5DgApJ7AlPe5L5YaIm5dXMlZTsQUZ7nX1cXa67Rz5fbRyu4DLKy4EOqXj3MRh9UIREC/EcDACZzQuQ9vPYXijT+BD4dz17ye36E1ee2hVBGSs0sRl1KIlIvl/PkhbvZ47gFvTA1y1xymqhcGuw4FXjyqvU6lChlHdqHxxBb87dYfEN/pndXAFlc8p8Jz4kLY9AvUuEZRWYeN237A/SXb8IjwGN+jg9NgYNQiIHC68RZ+/wUmbHrChM2I5B8Afn0VqLyTE2HoE4CdS7OY1Vdqv65Pf07A3O7jfsruA3q5trSrr+bCN108yB3l+ZrlVlLAZ1xzj663V3NZykec+HqPAZ7bbZj7vYscRRW+SinET+lXUdfI9Tb72IrxTIgnnh3pBZnUigsKum4gl53sX2c1/KsqVyB772a4X0yABzWL/Z8Sf6j+9hwGPPgsBG30glUqwlephYjb+weewV48Y3EQvcD1JmHrDIQsAEbMNYigdwYmbHrChM3I1NcAh/4LHPsEwF8+lkJLwMX/LyI2FJD00q+tm5eAPw9xIvfn4eZAm2r6DGgWuaMbuB7fpDXccLmLuFnbgIRTl/FN6iVcreCExUIowKShMjz/gDf+dvBZCC4dBSavBULm43LGAZT/71MMuXkIYgHXG62BNS64TIL7hJcgGxyiU/t5pdV4JSEDl4oVeEp0CIusk9C76U60FQtrLiPbyJe6N93kXTBh0xMmbCbClVPAic8Ba4c7AhbIRQzuqplTlRIoTm/uzV0+wfWM/spfekpdRZNShQPZpdh6tBDHC27w55f3OYgXbn2BWjtv3KxToV9Tc8avC6KBqPCficBJc2FtJ9W77YYmFTYcuIDN/7sIITVhVq90LOmVBNsb6uRLAi645qiXAU+53u3oAxM2PWHCxgDA9d7uHrbe+LPLhqHtkVVchbiUAvycUQx35VUclrzGl9WSBOn2EyAdMx9DRzyo12RGa5wsvIGoHRm4fOM2BALC6mEVeKrxFwjzf2s26jeCEzj/Kd0SCJMJm54wYWNopbqU6z1aGC/a8o3aBnx3oghuf7yJAcoCKLwfReDkeXB3demyNqvrGvH2r1nYmXYFADex8fFEG3jnxXHRXJQNnKH3GOCRDVzqwC6ECZueMGFjMFqy71wJon/MxM1bjRBbCLF0kh+ev88awpOfA6mbgKbb3GLrMUuA0a902T8AJmx6woSNwdDOteo6vPH9WRzO5eLtPTDQEeufDIKbqhRIfI2b1Qa4zGlTPgC8RhncB12+n6a1Ao/BYJgkLr2ssPW5EXgnciisLIU4ml+O8PeP4KdCCzQ+vQOY9iVg68LlZ9g6GfhlEbdExUiwHttdsB4bg9E+f16vwasJGThzhVsiYyexQOgAR4T5SPB3xWb0Ov8tZ2jjBEyKBQKfNMjkAhuK6gkTNgajYzQqVfj40EV8lVqIG7UNGmWPOBRiJT6DS10hd2LAQ1wcvj79O9UmEzY9YcLGYOiGSkU4X1yFI3nX8b8L13H60k00qQiWaMJ80W68bPETJIJGNAklKAv+F1wmvg6hpX7rEbv8GdumTZvg7e0NKysryOVynDhxok37nTt3ws/PD1ZWVggMDMSePXs0yokIK1euhJubG6ytrREWFoa8vDy+vLCwEHPnzoWPjw+sra0xYMAAxMTEoKGhQcNGIBC0OI4d6+KEGwzGPYxQKEBgPykWjh+IHQtCkb7yYXw2KxhPjeyPX6XPYGLDu/hDGQALVT1kJ9fiz/8G4/2t3+KHtCu4Vl3XdY7pmgIrPj6exGIxbdmyhc6fP0/z5s0jBwcHKi0t1Wp/9OhREolEtHbtWsrKyqLly5drzSsqlUrp559/pjNnztDUqVM18oru3buXnnvuOdq/fz9dvHiRfvnlF3JxcaHXXnuNr6OgoIAA0IEDB6ikpIQ/Gho6nufSLNPvMRgmTGFZDX199E/6fONqKl/ZlyjGnpQrpfTNm49R4NIEmrThCK3ek0V/5F2nhqa28992aV7RkJAQWrhwIf9eqVSSu7s7xcbGarWfPn06RUREaJyTy+W0YMECIuJyispkMlq3bh1fXlFRQRKJhL777rtW/Vi7di35+Pjw79XClp6erust8TBhYzC6jvrK61T69T/4HLSlKz3ppejl5LX0VxoQnUhVt9vuhOjy/dRpKNrQ0IC0tDSEhTWHlBEKhQgLC0NqaqrWa1JTUzXsASA8PJy3LygogEKh0LCRSqWQy+Wt1gkAlZWV6NOnZbSBqVOnwsXFBaNHj8auXbvavJ/6+npUVVVpHAwGo2sQ2zvBZdaXwJzdgONAuAgqsEn8IfY4bcSMwUAvK/2jEv8VnYStrKwMSqUSrq6aYWFcXV2hUCi0XqNQKNq0V//Upc78/Hxs3LgRCxYs4M/Z2dnhvffew86dO5GYmIjRo0cjMjKyTXGLjY2FVCrlDw8P84n/zmCYLT5jgBdTgHHLAJEYQ2qO4T9X5gI3Cw3WhNklTL569SomTZqEJ598EvPmzePPOzk5ISoqin8/YsQIFBcXY926dZg6darWuqKjozWuqaqqYuLGYHQHFhJgfDQwdBqw+1VuL24HQ7d3qHpdjJ2cnCASiVBaWqpxvrS0FDKZTOs1MpmsTXv1z9LSUri5uWnYDBs2TOO64uJijB8/HqNGjcJnn33Wrr9yuRxJSUmtlkskEkgkxtvYzGDc8zj7clFTGmrat9UBnYaiYrEYwcHBSE5O5s+pVCokJycjNDRU6zWhoaEa9gCQlJTE2/v4+EAmk2nYVFVV4fjx4xp1Xr16FQ8++CCCg4OxdetWCDsQjz0jI0NDLBkMhgkiEOgfMLQ1dJ3ZiI+PJ4lEQnFxcZSVlUXz588nBwcHUigUREQ0a9YsWrZsGW9/9OhRsrCwoPXr11N2djbFxMRoXe7h4OBAv/zyC509e5YeffRRjeUeV65coYEDB9KECRPoypUrGss51MTFxdH27dspOzubsrOz6b///S8JhULasmVLh++NzYoyGKZLly73ICLauHEjeXp6klgsppCQEDp27BhfNm7cOJozZ46G/Y4dO8jX15fEYjEFBARQYmKiRrlKpaIVK1aQq6srSSQSmjBhAuXm5vLlW7duJXCxolscauLi4sjf359sbGzI3t6eQkJCaOfOnTrdFxM2BsN00eX7ybZU3QXbUsVgmC4sbBGDwbinYcLGYDB6HEzYGAxGj8PsFuh2JerHjWxrFYNheqi/lx2ZFmDCdhfV1dUAwHYfMBgmTHV1NaTStnOnslnRu1CpVCguLkavXr3azdGo3n51+fJlNoPaBbDfb9dijr9fIkJ1dTXc3d3bXaDPemx3IRQK0a9fP52usbe3N5sPhjnCfr9di7n9ftvrqalhkwcMBqPHwYSNwWD0OJiw6YlEIkFMTAyLDtJFsN9v19LTf79s8oDBYPQ4WI+NwWD0OJiwMRiMHgcTNgaD0eNgwsZgMHocTNj0YNOmTfD29oaVlRXkcjlOnDhhbJd6BKtWrYJAINA4/Pz8jO2W2XLkyBFMmTIF7u7uEAgE+PnnnzXKiQgrV66Em5sbrK2tERYWhry8POM4a2CYsOlIQkICoqKiEBMTg9OnTyMoKAjh4eG4du2asV3rEQQEBKCkpIQ//vjjD2O7ZLbU1tYiKCgImzZt0lq+du1afPjhh9i8eTOOHz8OW1tbhIeHo66urps97QK6LI5vDyUkJIQWLlzIv1cqleTu7k6xsbFG9KpnEBMTQ0FBQcZ2o0cCgH766Sf+vUqlIplMRuvWrePPVVRUkEQioe+++84IHhoW1mPTgYaGBqSlpWlkrRcKhQgLC2szaz2j4+Tl5cHd3R39+/fHzJkzUVRUZGyXeiQFBQVQKBQan2WpVAq5XN4jPstM2HSgrKwMSqVSp6z1jI4jl8sRFxeHffv24ZNPPkFBQQHGjBnDh5NiGA7157WnfpZZdA+GyTB58mT+9X333Qe5XA4vLy/s2LEDc+fONaJnDHOD9dh0wMnJCSKRqM3M9gzD4eDgAF9fX+Tn5xvblR6H+vPaUz/LTNh0QCwWIzg4WCNrvUqlQnJyskbWeoZhqKmpwcWLF+Hm5mZsV3ocPj4+kMlkGp/lqqoqHD9+vEd8ltlQVEeioqIwZ84c3H///QgJCcGGDRtQW1uL559/3tiumT1LlizBlClT4OXlheLiYsTExEAkEmHGjBnGds0sqamp0ejtFhQUICMjA3369IGnpydeeeUVvPPOOxg0aBB8fHywYsUKuLu7IzIy0nhOGwpjT8uaIxs3biRPT08Si8UUEhJCx44dM7ZLPYKnnnqK3NzcSCwWU9++fempp56i/Px8Y7tlthw6dIgAtDjmzJlDRNySjxUrVpCrqytJJBKaMGEC5ebmGtdpA8HCFjEYjB4He8bGYDB6HEzYGAxGj4MJG4PB6HEwYWMwGD0OJmwMBqPHwYSNwWD0OJiwMRiMHgcTNgaD0eNgwsZgMHocTNgYDEaPgwkbg8HocTBhYzAYPY7/Bxx8/ES4fItNAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:20:40.922128Z", + "iopub.status.busy": "2024-02-29T06:20:40.921847Z", + "iopub.status.idle": "2024-02-29T06:22:16.841126Z", + "shell.execute_reply": "2024-02-29T06:22:16.840059Z" + }, + "papermill": { + "duration": 95.940371, + "end_time": "2024-02-29T06:22:16.843721", + "exception": false, + "start_time": "2024-02-29T06:20:40.903350", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:22:16.882160Z", + "iopub.status.busy": "2024-02-29T06:22:16.881399Z", + "iopub.status.idle": "2024-02-29T06:22:16.901901Z", + "shell.execute_reply": "2024-02-29T06:22:16.901026Z" + }, + "papermill": { + "duration": 0.041381, + "end_time": "2024-02-29T06:22:16.903739", + "exception": false, + "start_time": "2024-02-29T06:22:16.862358", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.0069760.0196840.0054694.7321210.1052230.899660.2544090.0000681.4780970.0578010.1366160.0739540.0824410.0002816.210219
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.006976 0.019684 0.005469 4.732121 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.105223 0.89966 0.254409 0.000068 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 1.478097 0.057801 0.136616 0.073954 0.082441 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.000281 6.210219 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:22:16.938663Z", + "iopub.status.busy": "2024-02-29T06:22:16.938408Z", + "iopub.status.idle": "2024-02-29T06:22:17.339488Z", + "shell.execute_reply": "2024-02-29T06:22:17.338514Z" + }, + "papermill": { + "duration": 0.421131, + "end_time": "2024-02-29T06:22:17.341625", + "exception": false, + "start_time": "2024-02-29T06:22:16.920494", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:22:17.380452Z", + "iopub.status.busy": "2024-02-29T06:22:17.379432Z", + "iopub.status.idle": "2024-02-29T06:23:57.383424Z", + "shell.execute_reply": "2024-02-29T06:23:57.382532Z" + }, + "papermill": { + "duration": 100.026409, + "end_time": "2024-02-29T06:23:57.386125", + "exception": false, + "start_time": "2024-02-29T06:22:17.359716", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:23:57.431576Z", + "iopub.status.busy": "2024-02-29T06:23:57.431230Z", + "iopub.status.idle": "2024-02-29T06:23:57.449621Z", + "shell.execute_reply": "2024-02-29T06:23:57.448848Z" + }, + "papermill": { + "duration": 0.044631, + "end_time": "2024-02-29T06:23:57.451907", + "exception": false, + "start_time": "2024-02-29T06:23:57.407276", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:23:57.492447Z", + "iopub.status.busy": "2024-02-29T06:23:57.491728Z", + "iopub.status.idle": "2024-02-29T06:23:57.497984Z", + "shell.execute_reply": "2024-02-29T06:23:57.496986Z" + }, + "papermill": { + "duration": 0.028607, + "end_time": "2024-02-29T06:23:57.500218", + "exception": false, + "start_time": "2024-02-29T06:23:57.471611", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.44625178128854376}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:23:57.540660Z", + "iopub.status.busy": "2024-02-29T06:23:57.540370Z", + "iopub.status.idle": "2024-02-29T06:23:57.957574Z", + "shell.execute_reply": "2024-02-29T06:23:57.956695Z" + }, + "papermill": { + "duration": 0.439641, + "end_time": "2024-02-29T06:23:57.959552", + "exception": false, + "start_time": "2024-02-29T06:23:57.519911", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABLJElEQVR4nO2deXhTZdr/vyd7mibplm7QBcome2VxEBlAGRYZBXV8HUFtRVzhZZTX91JmrgGXUWTGBVzAGV8BHVH8OYOOIyiugMuIAiLgwlKgLdDSljZpmj05z++Pk6RNm7RJmuSk7f25rlynOefJOfdJ0m+e537u5745xhgDQRCEiEjENoAgCIKEiCAI0SEhIghCdEiICIIQHRIigiBEh4SIIAjRISEiCEJ0SIgIghAdEiKCIESHhKiHUVxcjF//+tdxvQbHcXjooYe6bPfQQw+B47i42kL0DUiIEsxXX32Fhx56CEajUWxTiB7A+vXrsXnzZrHNiDskRAnmq6++wsMPP0xCRIQFCRFBEESCICFKIA899BD+93//FwAwYMAAcBwHjuNw+vRpbNq0CZdffjmys7OhVCoxfPhwbNiwIeS5PvzwQ4wdOxYqlQrDhw/Htm3bIrbH4XDgvvvug8FggFarxdVXX40zZ84EbfvFF19gwoQJUKlUKCkpwV//+teg7TiOw9KlS7FlyxYMHToUKpUK48aNw549ezq8FxzH4dixY7jpppug1+thMBjwxz/+EYwxVFdXY968edDpdMjNzcVTTz0V8f0BwPvvv4+pU6dCq9VCp9NhwoQJeP311wPavPXWWxg3bhzUajWysrJw00034ezZswFtysvLkZqairNnz2L+/PlITU2FwWDA/fffD4/HE9CW53msW7cOo0aNgkqlgsFgwOzZs7Fv3z5/m3A+7+LiYvzwww/YvXu3/7sybdq0qN6HpIcRCeP7779nN954IwPAnnnmGfb3v/+d/f3vf2ctLS1swoQJrLy8nD3zzDPsueeeYzNnzmQA2PPPPx9wjqKiIjZkyBCWlpbGHnzwQfb000+zUaNGMYlEwj788MOI7LnpppsYALZgwQL2/PPPs2uvvZaNHj2aAWCrVq3ytzt06BBTq9WssLCQrV69mj366KMsJyfH37YtANjIkSNZVlYWe+SRR9iaNWtYUVERU6vV7PDhw/52q1atYgDY2LFj2Y033sjWr1/P5s6dywCwp59+mg0dOpTdfffdbP369Wzy5MkMANu9e3dE97dp0ybGcRwbOXIke+yxx9gLL7zAFi9ezG6++eaANgDYhAkT2DPPPMMefPBBplarWXFxMWtqavK3KysrYyqVio0YMYItWrSIbdiwgV133XUMAFu/fn3AdcvLyxkANmfOHLZ27Vr25JNPsnnz5rHnnnvO3yacz/vtt99m/fv3Z8OGDfN/VyL9jHsKJEQJ5i9/+QsDwE6dOhWw32q1dmg7a9YsNnDgwIB9RUVFDAD75z//6d9nMplYXl4eKy0tDduOgwcPMgDsnnvuCdi/YMGCDkI0f/58plKpWGVlpX/fjz/+yKRSaVAhAsD27dvn31dZWclUKhW75ppr/Pt8QnTHHXf497ndbta/f3/GcRx74okn/PubmpqYWq1mZWVlYd+f0WhkWq2WXXLJJcxmswUc43meMcaY0+lk2dnZbOTIkQFt3nvvPQaArVy50r+vrKyMAWCPPPJIwLlKS0vZuHHj/M8//fRTBoAtW7asg02+6zIW/uc9YsQINnXq1DDuuGdDQ7MkQa1W+/82mUxoaGjA1KlTcfLkSZhMpoC2+fn5uOaaa/zPdTodbrnlFnz33Xeora0N63o7duwAACxbtixg/7333hvw3OPxYOfOnZg/fz4KCwv9+y+66CLMmjUr6LknTZqEcePG+Z8XFhZi3rx52LlzZ4dhzOLFi/1/S6VSjB8/Howx3Hbbbf79aWlpGDp0KE6ePBnWvQHARx99BLPZjAcffBAqlSrgmC/kYN++fairq8M999wT0Gbu3LkYNmwYtm/f3uG8d911V8DzKVOmBNj1z3/+ExzHYdWqVR1e2zbUIZLPuy9AQpQkfPnll5gxYwY0Gg3S0tJgMBjw+9//HgA6fDEHDRrUIX5nyJAhAIDTp0+Hdb3KykpIJBKUlJQE7B86dGjA8/r6ethsNgwePLjDOdq39RGs7ZAhQ2C1WlFfXx+wv624AYBer4dKpUJWVlaH/U1NTaFvqB0VFRUAgJEjR4ZsU1lZCSD4fQwbNsx/3IfP39OW9PT0ALsqKiqQn5+PjIyMTu2L5PPuC8jENoAQvrxXXHEFhg0bhqeffhoFBQVQKBTYsWMHnnnmGfA8L7aJcUMqlYa1DwCYyFmNQ9kVKX358w4FCVGCCRaJ/O9//xsOhwPvvvtuQA/hs88+C3qOEydOgDEWcK5jx44BEGZawqGoqAg8z6OioiKgR3D06NGAdgaDAWq1GsePH+9wjvZtfQRre+zYMaSkpHToUcQLX0/vyJEjGDRoUNA2RUVFAIT7uPzyywOOHT161H880uvu3LkTjY2NIXtFkXzefSVynYZmCUaj0QBAQECj75e27S++yWTCpk2bgp7j3LlzePvtt/3Pm5ub8eqrr2Ls2LHIzc0Ny445c+YAAJ599tmA/WvXrg14LpVKMWvWLLzzzjuoqqry7//pp5+wc+fOoOf+z3/+gwMHDvifV1dX41//+hdmzpwZs15FV8ycORNarRarV6+G3W4POOZ7n8ePH4/s7Gy8+OKLcDgc/uPvv/8+fvrpJ8ydOzfi61533XVgjOHhhx/ucMx33Ug+b41G0yeCX6lHlGB8Ttw//OEP+O1vfwu5XI5f/vKXUCgUuOqqq3DnnXeipaUFL730ErKzs1FTU9PhHEOGDMFtt92Gb7/9Fjk5Odi4cSPOnz8fUriCMXbsWNx4441Yv349TCYTLr30UnzyySc4ceJEh7YPP/wwPvjgA0yZMgX33HMP3G43nnvuOYwYMQKHDh3q0H7kyJGYNWsWli1bBqVSifXr1/vPkyh0Oh2eeeYZLF68GBMmTMCCBQuQnp6O77//HlarFa+88grkcjnWrFmDW2+9FVOnTsWNN96I8+fPY926dSguLsZ9990X8XWnT5+Om2++Gc8++yyOHz+O2bNng+d5fP7555g+fTqWLl2KmTNnhv15jxs3Dhs2bMCf/vQnDBo0CNnZ2R16b70C8Sbs+i6PPvoo69evH5NIJP6p/HfffZeNHj2aqVQqVlxczNasWcM2btzYYaq/qKiIzZ07l+3cuZONHj2aKZVKNmzYMPbWW29FbIfNZmPLli1jmZmZTKPRsKuuuopVV1d3mL5njLHdu3ezcePGMYVCwQYOHMhefPFF/xR8WwCwJUuWsNdee40NHjyYKZVKVlpayj777LOAdr7X1tfXB+wvKytjGo2mg61Tp05lI0aMiPge3333XXbppZcytVrNdDodmzhxInvjjTcC2rz55pustLSUKZVKlpGRwRYuXMjOnDkTll3B3gO3283+8pe/sGHDhjGFQsEMBgObM2cO279/f4Bd4XzetbW1bO7cuUyr1TIAvXYqn2OM6poRsYPjOCxZsgTPP/+82KYQPQjyEREEITrkI+qFdBXUqFarodfrE2RN7Kmvr+8QGNkWhULRZRwPkVyQEPVC8vLyOj1eVlbWo1NLTJgwoUOwYVumTp2KXbt2Jc4gotuQEPVCPvroo06P5+fnx+3aiXA5btmyBTabLeTx9PT0uNtAxBZRndXFxcVBf9nuuecevPDCCyJYRBCEGIjaI/r2228DxvpHjhzBr371K1x//fUiWkUQRKJJqun7e++9F++99x6OHz8eVmg7z/M4d+4ctFptnwmFJ4ieBGMMZrMZ+fn5kEhCT9InjY/I6XTitddew/Lly0OKisPhCAjFP3v2LIYPH54oEwmCiJLq6mr0798/5PGkEaJ33nkHRqMR5eXlIdusXr066DKB6upq6HS6OFpHEEQ0NDc3o6CgAFqtttN2STM0mzVrFhQKBf7973+HbNO+R+S7SZPJREJEEElIc3Mz9Hp9l/+jSdEjqqysxMcff9xlAnilUgmlUpkgqwiCSBRJscRj06ZNyM7OjirtAkEQPR/RhYjneWzatAllZWWQyZKig0YQRIIR/T//448/RlVVFRYtWiS2KUQfhTEGt9vd6fo1IjhSqRQymazb4TOiC9HMmTNFz0VM9F2cTidqampgtVrFNqXHkpKSgry8PCgUiqjPIboQEYRY8DyPU6dOQSqVIj8/HwqFggJjI4AxBqfTifr6epw6dQqDBw/uNGixM0iIiOTG0QKc+AhwO4FBVwCarK5fEyZOpxM8z6OgoAApKSkxO29fQq1WQy6Xo7KyEk6ns0MNuXAR3VlNECFhDPjhbaDuZ6DxJPD9VsDjivllov0VJwRi8f7RJ0AkLxdOAKYzACcBpHLAYQZqOibrJ3o+JERE8lLzvbDtPx4YMFX4+/wR8ewh4gYJEZGcOK3ABaFsNPLGANkXARwHNJ8TekZEwiguLu5Q7y7WkBARyUnTaYDxQKpBcFArU4HUHOGYsarTlxI9DxIiIjlpOiVs0we07kvzlmduCp2vmgiO0+kU24ROISEikg/GhB4RAKQXt+7XFwhbc8fqt7HG6eZDPtwePuy2rjDaRsO0adOwdOlSLF26FHq9HllZWfjjH//oDw4uLi7Go48+iltuuQU6nQ533HEHAOCLL77AlClToFarUVBQgGXLlsFisfjPW1dXh6uuugpqtRoDBgzAli1borIvUiiOiEg+7EbA3gxIpK29IADQeodmlgbA4wak8fv6vvBZx9LbPgZkaTC/tJ//+d/2VMDlCb46oH+6GtePL/A/3/jlKdicgUtJ7vvVkKhsfOWVV3Dbbbfhm2++wb59+3DHHXegsLAQt99+OwDgySefxMqVK7Fq1SoAQEVFBWbPno0//elP2LhxI+rr6/1i5itXXl5ejnPnzuGzzz6DXC7HsmXLUFdXF5V9kUBCRCQfzeeEbWqOMG3vQ6kD5GrAZQMsdYAuftVIegIFBQV45plnwHEchg4disOHD+OZZ57xC9Hll1+O//mf//G3X7x4MRYuXIh7770XADB48GA8++yzmDp1KjZs2ICqqiq8//77+OabbzBhwgQAwMsvv4yLLroo7vdCQkQkH83eoVd7oeE4QJsLNJ4CzLVxFaIl0weFPCZptwrkjl+WhGzbfsXIoskDgjeMgl/84hcBS1ImTZqEp556yr94d/z48QHtv//+exw6dChguMUY8y91OXbsGGQyGcaNG+c/PmzYMKSlpcXM5lCQEBHJh9nbI9IGKRSpMQhCZL0QVxMUsvDdp/Fq2100Gk3A85aWFtx5551YtmxZh7aFhYU4duxYokzrAAkRkVzwHsB8Xvg7WI8nJVPYxlmIegJ79+4NeP71119j8ODBkEqlQdtffPHF+PHHHzFoUPDe3rBhw+B2u7F//37/0Ozo0aMwGo0xtTsYNGtGJBctdQDvBuQqQB2kYqtfiBoTa1cSUlVVheXLl+Po0aN444038Nxzz+F3v/tdyPYPPPAAvvrqKyxduhQHDx7E8ePH8a9//QtLly4FAAwdOhSzZ8/GnXfeib1792L//v1YvHgx1Gp13O+FhIhILnxT89q8jg4WoFWI7Ka4LIDtSdxyyy2w2WyYOHEilixZgt/97nf+afpgjB49Grt378axY8cwZcoUlJaWYuXKlQElyDdt2oT8/HxMnToV1157Le644w5kZ2fH/V5oaEYkF5YGYZsa4suvSGmdObM2tk7p90HkcjnWrl2LDRs2dDh2+vTpoK+ZMGECPvzww5DnzM3NxXvvvRew7+abb+6WneFAPSIiubB4Y1Y0nfwKp2QIW/IT9RpIiIjkgTHBRwSE7hEBgNorRLam+NtEJAQamhHJg6MZcDuE/EM+X1Aw1GnC1m5KiFnJyK5du8Q2IaZQj4hIHlrqha0mU1jeEQqVXtj2YSHqbYguRGfPnsVNN92EzMxMqNVqjBo1Cvv27RPbLEIM/P4hQ+ft/EJkjKs5ROIQdWjW1NSEyZMnY/r06Xj//fdhMBhw/PhxpKcHiR8hej8tYTiqAUCVJmztzQDPA5RzuscjqhCtWbMGBQUF/pW/ADBgQOi1OA6HAw6Hw/+8ubk5rvYRCcbiHZp15qgGAEWqMHTjPYDT3NpDInosov6UvPvuuxg/fjyuv/56ZGdno7S0FC+99FLI9qtXr4Zer/c/CgoKQrYlehged2u0dFdDM4lEWIkPADZjXM0iEoOoQnTy5Els2LABgwcPxs6dO3H33Xdj2bJleOWVV4K2X7FiBUwmk/9RXV2dYIuJuGFrFFLDypSAUtt1e5o561WIOjTjeR7jx4/H448/DgAoLS3FkSNH8OKLL6KsrKxDe6VSCaVSmWgziUTgC05MyQy+tKM9NHPWqxC1R5SXl4fhw4cH7LvoootQVUXJ0fscbYUoHHxDMwf5CXsDogrR5MmTcfTo0YB9x44dQ1FRkUgWEaLhE6JwS0r7hm9UWqhXIKoQ3Xffffj666/x+OOP48SJE3j99dfxt7/9DUuWLBHTLEIMIu4RxUmIGAPczsQ/WPCc18F49dVXkZmZGTCDDADz589PyALVeCCqj2jChAl4++23sWLFCjzyyCMYMGAA1q5di4ULF4ppFpFoGEueoZnHBXz+VGzPGQ5T/geQKcJqev3112PZsmV49913cf311wMQqm9s376905X1yYzoa81+/etf49e//rXYZhBi4mgWpu8l0tZgxa7w9YjcTmF9mqzvTGKo1WosWLAAmzZt8gvRa6+9hsLCQkybNk1c46JEdCEiCH8OInV6+FHSMoUgPm6HMDyLlRBJ5ULvJNG0rVYSBrfffjsmTJiAs2fPol+/fti8eTPKy8sDkun3JEiICPHxBTL68gyFi1LrFaLm8J3cXcFxYQ+RxKS0tBRjxozBq6++ipkzZ+KHH37A9u3bxTYrakiICPGJ1D/kQ6kTelN9dOZs8eLFWLt2Lc6ePYsZM2b06JUGtFqQEB+/EEXYq+njU/gLFizAmTNn8NJLL2HRokVim9MtSIgI8Ym6R9S3hUiv1+O6665Damoq5s+fL7Y53YKEiBAXtwNwWoS/o/ERAX1WiAAhn9fChQt7/NIn8hER4uLLO61IiXzmyy9EfW+ZR1NTE3bt2oVdu3Zh/fr1YpvTbUiICHHxCVGwYopd4QtqtPc9ISotLUVTUxPWrFmDoUOHim1OtyEhIsSlW0KUKmzdDiEiOsJYnJ5MqLplPRXyERHi0h0hkqkAife31NkSO5uIhENCRIhLd4SI4wCFRvjb5/COAhbBglOiI7F4/0iICHHpjhABrcMzR+Q9IrlcGMpZrdbork0AaH3/fO9nNJCPiBAPt7NVQKIVom70iKRSKdLS0lBXJ1QPSUlJ6bFrtcSAMQar1Yq6ujqkpaVBKu2kFl0XkBAR4uGrSyZXAXJ1dOdQeKfwndHFEuXm5gKAX4yIyElLS/O/j9FCQkSIR3eHZYC/R1RT34CP6yqhkHKYNjQbOTpVWC/nOA55eXnIzs6Gy+WK3o4+ilwu71ZPyAcJESEesRAiZSrMdhe+rzmDBsNoAMB3VU2YPTIvotNIpdKY/EMR0UFCRIhHTHpEqTh9wQo5L8NAgwYDs1Ixsp8uNvYRCYOEiBAPX3HEbghRvVOOFocbSpkNk4bnIEVBX+meCE3fE+Lh6xGFmx42CEcaPACAXJUHKbLWGS+Xh0ejxdkd64gEQkJEiAPPt66a91VtjRDGGCqaeDBwyEpV+qfwqxut+Nuek9hxuCZGxhLxRlQheuihh8BxXMBj2LBhYppEJApHs1BiWiIFFKlRnYLjOFw/vhBFednQqeX+ZR5ZqUq4PDzqzQ4022kmrCcg+oB6xIgR+Pjjj/3PZTLRTSISgS91h1IXXonpEOhT5NBnZwHmWn+PSK2QIk+vwjmjHZUNVozqr4+FxUQcEf2/XiaTdTsYiuiB+GrWq2Iww6XUCkLUJkFaUaYG54x2nL5gISHqAYjuIzp+/Djy8/MxcOBALFy4sNO69w6HA83NzQEPoofiyyGkik4kGGP4+Mfz+K6qCS6pNyq7zTKPwowUAMBZo40WtfYARBWiSy65BJs3b8YHH3yADRs24NSpU5gyZQrM5uDh+qtXr4Zer/c/enLVgj6Pr0ekjK5H1Gx34/BZE/YcawDny9TYJhVItlYJmYSDzemB0Up+omRHVCGaM2cOrr/+eowePRqzZs3Cjh07YDQa8f/+3/8L2n7FihUwmUz+R3V1dYItJmKGo3s9ovPNdgCAQauETNVxBb5MKvEv8zhnskVvJ5EQRPcRtSUtLQ1DhgzBiRMngh5XKpU9Pkk44aWbPqJakyBEuXpla+7qdsnRRvbToyRbg3x9lAtqiYQhuo+oLS0tLaioqEBeXmTrhIgeBmOtPqIoh2YXLA4AgCFV1Tr9306IhufrMK4oA+ma5K/c2tcRVYjuv/9+7N69G6dPn8ZXX32Fa665BlKpFDfeeKOYZhHxxmkBeLcwbR/l0OxCixA1nZmqaJOTyCoEShI9DlGHZmfOnMGNN96ICxcuwGAw4LLLLsPXX38Ng8EgpllEvPH5hxQaIaAxQuwuD8x2NwAgQ6MApApB1BgPuG2twgSgyeLEebMd+Wlq6FR9J7l+T0NUIdq6dauYlyfEwu8fiq43ZLK5wHFAqlIGldwrZHK10CNyWgKE6LOjdai8YMWMi3IoniiJSSpnNdFH6KZ/KEenwpLpg9Di7RUBEMTHaRWCGlOz/bsNWiUqL1hR32IHQEKUrCSVs5roI3SzRwQAcqkk0Antd1gH5q42aIVZ1nqzI+prEfGHhIhIPP4YohgmMAuRRN+QKghRQ4uTIqyTGBIiIvH4kuZHmYdo+6EafPzjebQ42g7NgveI0lMUkEk4ON08TDaKsE5WSIiIxNMNH5HLw+PYeTMOnzVB2nbVvl+IApcHSSScfwh3gRKlJS0kRERicdmFWvVAVD6iZm+vRiGTQCVv8/XtpL5ZhleImkiIkhaaNSMSi88/JFcBssgjno1eIUpLkQcWQ+xEiMYUpGFIjhY5OloelKyQEBGJpZszZr6V9GnqdiIWYr0ZAPRLo7VmyQ4NzYjE0s0YIt/QTK9uFyXt6xG5ncKD6FGQEBGJpZszZkabIDJpKe2ESKoApN4OfpBeUUV9C7451Qib0xPVdYn4QkJEJJZuxhA53cKi1g49Io4LOYUPAHuO1ePLEw1oaKHAxmSEfEREzGGMBTqS29JNH9ENEwrhcHsgkwT5DVVohKKNIWbOjFYXGi1OFHjTyBLJA/WIiJhSZ7bjta8rYXeFGAJ100cEAEqZFFJJEKHrpEeUniI4t6noYnJCQkTElM9+rkNDixMHKps6HvS4W0WiG+vMQhIiqBFojSXy+ZiI5IKEiOg2JpsL57zVMsYWCHXsf6xpBs+3W9vl8w9JZULajgg5UWfGtgNncKAqiMgBncYS+XxKJkqkn5SQEBHd5oezJrz5bTU+/bkOJQYNVHIpzHY3arwJ7v20nTGLoqhindmBygvW0BHSnQmRd5bNZHN3FEhCdEiIiG5T3WQFIOQJkkklKMoUnMGVF9oJQjf9Q76sjNpQmRY7CWrUKmWQSTjwjPnPQyQPNGtGdAuXh8f5ZmFKvH+6MNwqzEjB0VozzjTagJI2jbs5Y9YqRCG+tp30iDiOw9Vj85GikCE11OsJ0aBPhOgWtSY7PDyDViXz+2F8SyrONwvH/DNc3YwhMtsF/05YQsTzQLsp/qJMTZAXEckADc2IblHr9QPl6dX+2KG0FDnUCikyUhWwOtsMg7rRI2KM+VPDapUhhmZyjTeJPgNc1oivQYhH0gjRE088AY7jcO+994ptChEBDd4UrL6UrIAwDLp9ykAsvKQo0J/TDR+R1emBm2dC0vxQPSKJpHU2LsjwzGh1Yt/pRnxfbYz4+kR8iUqITp48GVMjvv32W/z1r3/F6NGjY3peIv7Ut3QUIgAdAw55XkhsD0TVI7K5PFArpNAoZMGDGX2EKLYICCv3Pz/egENnjBFfn4gvUQnRoEGDMH36dLz22muw2+1dv6ATWlpasHDhQrz00ktIT0/v1rmIxDN1iAGXDc4KmevHnyfaaRbqjnGSVrGIgKxUJe6aWoJbJxd33rCT6Oo0/xS+i/JXJxlRCdGBAwcwevRoLF++HLm5ubjzzjvxzTffRGXAkiVLMHfuXMyYMaPLtg6HA83NzQEPQlyKMjWYUJyBFEXgcKnF4caWvZV46fOTwj9921r3wdaJhYlM2sVr/Q7rIFP4KjkkHAeXhwXmuyZEJ6pvxNixY7Fu3TqcO3cOGzduRE1NDS677DKMHDkSTz/9NOrr68M6z9atW3HgwAGsXr06rParV6+GXq/3PwoKCqIxn0gAarkUDWYnLA6P8E8fgzVmYdHJFL5Uwvln3IwUYZ1UdMtZLZPJcO211+Ktt97CmjVrcOLECdx///0oKCjALbfcgpqampCvra6uxu9+9zts2bIFKpUqrOutWLECJpPJ/6iuru6O+UQ3OWu04dh5c9DqGFIJ5x8KNVqc3Y4h2nW0DtsOnEHVhS5mwzoJagRah2fNdhKiZKJbQrRv3z7cc889yMvLw9NPP437778fFRUV+Oijj3Du3DnMmzcv5Gv379+Puro6XHzxxZDJZJDJZNi9ezeeffZZyGQyeDwdV28rlUrodLqAByEeh88Ysf1QDY6d77jIFGhdaHohBkJUY7Kj8oIVTg/fecNOekQAoPPO4jXbaGiWTEQV0Pj0009j06ZNOHr0KK688kq8+uqruPLKKyHxjv0HDBiAzZs3o7i4OOQ5rrjiChw+fDhg36233ophw4bhgQcegFQqjcY0IoE0eYc36e2zJXrJ1ChwAsCFFifg7G5UtXAtXVdR0T4hcgTvEenU1CNKRqISog0bNmDRokUoLy9HXl5e0DbZ2dl4+eWXQ55Dq9Vi5MiRAfs0Gg0yMzM77CeSE5+fRd8+kb2XjFRvGR+rE3BHL0RuDw+LQ+ghh1xn5qOT6XsAuChPi+KslI4ZHglRiUqIPvroIxQWFvp7QD4YY6iurkZhYSEUCgXKyspiYiSRfNicHn/ysw75o734Km2YLE4AvuUdkQuRb4ZLLuUCa5kFw9cj8riEJPrtShZpVfKuxYxIOFEJUUlJCWpqapCdnR2wv7GxEQMGDAjq3wmHXbt2RfU6IvE0WYVUHFqVDPIQU+ppKXJkpiqQKXeBNbvBSaRRzZq1XXUfMgWtD5kSkMoFIXK2ALKMiK9HJJ6onNWhgsFaWlrCngEjejb++mIpoYskquRS3DKpGHMHe9ehKVOjiiHy+XNSlWH+bnYS1AgAB6qa8OnP5ymWKImIqEe0fPlyAMJaopUrVyIlpTUJucfjwd69ezF27NiYGkgkJ/6yPuH4Wro5Y8bzgFohDb3qvj0KDWBrCukn+r7aCKPVhcHZ2vDFjYgrEX0K3333HQChR3T48GEoFK2/hgqFAmPGjMH9998fWwuJpGREvh7ZWlVY/8jMZgRjDJIohWhUfz1G9deHn1kxjCl8o9VFM2dJRERC9NlnnwEQptnXrVtHcTx9GL1aHtbM08FqI87sO4oRzIIBA7qXMF/S2WLXtnQR1OifwqdYoqQhqn7ppk2bYm0H0UuRSThIHc2wc3x8KncEo8sekfC1px5R8hC2EF177bXYvHkzdDodrr322k7bbtu2rduGEckLzzN8V90ErUqOEkNqp2k50lLkqPO0wM48Uc2YMcbw+jdVUMulmD0yt8Pi2qCEG9QYZGkKIQ5hC5Fer/dPner1CfplI5ISi9ONPccaIOE4/Pflgzptm6aWQ+k2w8F4eJR6RBov73DzqPPmxA4VJtCBLoIaW6OraWiWLIQtRG2HYzQ069v4/oFTVbIu/TYazg45PPCAg5mpkRbxtYReS4pCGoEQhTc0a7ELpYXC9j0RcSMqH5HNJhTT803fV1ZW4u2338bw4cMxc+bMmBpIJB9hr/sCwNmMUMqlaOJTYHLwSIswJ1qXJYSC4esRuaxBk+inKmVY+ItC6FRyEqEkIaqAxnnz5uHVV18FABiNRkycOBFPPfUU5s2bhw0bNsTUQCL58M02hSUOtkYoZRLYZfqg6UK6ossSQsGQp3SaRJ/jOGRrVVDJaWF1shB1hsYpU6YAAP7xj38gNzcXlZWVePXVV/Hss8/G1EAi+fD3iNRhiIOtCTqVHGmZ2eE5mkNcK6JaZBKJIEZASD8RkVxENTSzWq3QaoVYjQ8//BDXXnstJBIJfvGLX6CysjKmBhLJh6+XogurR9SE/DQ18gcNArIjz1Ud0bXaotAIPqIQfqLTDRacbGhBfpoaw3IpHk5sok6e/84776C6uho7d+70+4Xq6uooyLEP0Oz3EYUnRAAAdXSFEaQSDmqFNCx/VABdBDWeb7bj+2pT1xkfiYQQVY9o5cqVWLBgAe677z5cccUVmDRpEgChd1RaWhpTA4nkY87IPJhsLmSHqNzhhzG/EDFVGuxOD1RySdcr6Nswa0Su91QRVt3oYubM59+iKfzkICoh+s1vfoPLLrsMNTU1GDNmjH//FVdcgWuuuSZmxhHJiUGr7FDHLCguK+B2ggfw128uwO5pwl1TS6BWRO4kjkS8AIQR1Ch89c0UXZ0URL30ODc3F7m5uQH7Jk6c2G2DiF6EtzckUekgcykAjxsmmysqIYqYLoIafT0is90NxljkQkfElKiEyGKx4IknnsAnn3yCuro68HxgQvNYV4Ilkod6swNVjRZka1UoyEjpvLH1grBVp0Mvk6PFIQhRrj68nFXnjDZ89ON55OlVmDkit+sXtKWroZlSBgnHwcMLNc4oa6O4RCVEixcvxu7du3HzzTcjLy+Pfk36EGearNhzrAGDc1K7FiJLg7DVGKBzyHHWaIsolshkc6HR4oQmmpxBXSRHk0g4aJRSmO1umO0kRGITlRC9//772L59OyZPnhxre4gkJ6JIZ1+PKCUTeklruefIrxWNEIWu+OpDp5b7hYgQl6iEKD09HRkZlAu4LxKROFi8FX81WdBz0QiRK/xrtcfXI/K4ALdDyGXdjtkjc6GQSqCUdau8HxEDovoEHn30UaxcuRJWa/diMDZs2IDRo0f7iyVOmjQJ77//frfOScSXFodXHLoaLrkdrWWmNQZ/pY9oekQRBzMCQvUOqfd1nWRqVMml5FpIAqLqET311FOoqKhATk4OiouLIZcHflEOHDgQ1nn69++PJ554AoMHDwZjDK+88grmzZuH7777DiNGjIjGNCLOhD008/mHFBpArkZaihsDsjRIS5GHPUvVrR4RIAQ1WhuF4VkK9eCTmag+4fnz58fk4ldddVXA88ceewwbNmzA119/TUKUhPDeGSYgjLVf1lZHNQCkKGSYX9ovous1R7Pyvi0KjVeIgveITDYX9lc2AgAuH5YT3TWImBCVEK1atSrWdsDj8eCtt96CxWLxR2q3x+FwwOFw+J83NzfH3A4iNC1ONxgTll1ouooFMp8XtqmGqK7l8vBIS5F7Z7Si7BF1MYXP8wzfV5ugkEkwfWg2DdFEJOqARqPRiH/84x+oqKjA//7v/yIjIwMHDhxATk4O+vUL/5fv8OHDmDRpEux2O1JTU/15jYKxevVqPPzww9GaTHQTjUKGhZcUwur0dP1Paz4nbLX5AbvtLg8YQ5dBjXKpBAsvKeqOua0Oa4c56GGfwDndPBxuntKCiEhUzupDhw5hyJAhWLNmDZ588kkYjUYAQq7qFStWRHSuoUOH4uDBg9i7dy/uvvtulJWV4ccffwzadsWKFTCZTP5HdXV1NOYTUSKVcMjWqVCcpem8Ie8BWrwzZtrWQMTPj9djw64K7PMOh+JOF7FEMqkEGqUgPpS/WlyiEqLly5ejvLwcx48fD6jseuWVV2LPnj0RnUuhUGDQoEEYN24cVq9ejTFjxmDdunVB2yqVSv8Mm+9BJCGWeoB3C1PmbVbd+2qgRZMgLSq6GJoBbRe/khCJSVRC9O233+LOO+/ssL9fv36ora3tlkE8zwf4gYjk4fh5M/ZXNqLe3MXn03xW2GrzhEyJXnx10MIRoi+ON+CVr07jyFlT1PaGFdRIq/CTgqh8REqlMqij+NixYzAYwndOrlixAnPmzEFhYSHMZjNef/117Nq1Czt37ozGLCLO/FRrRkVdCy4fJul89X2TNzleWmHAbp8QGa2uLqfwL1gcaLQ44Qm3umswulj4CrT6iWhoJi5R9YiuvvpqPPLII3C5hA+P4zhUVVXhgQcewHXXXRf2eerq6nDLLbdg6NChuOKKK/Dtt99i586d+NWvfhWNWUScCSuuhzHAWCX8nR7obPaV8fE5hzvDJwzhVJMNia9H5LIJSfSD4LPJ6vREfx2i20Qd0Pib3/wGBoMBNpsNU6dORW1tLSZNmoTHHnss7PO8/PLL0VyeEAmzPYwYopY64R9fKheGZm2QSyVIVcr8q/BDzVIxxvzDN19EdlTIUwBOAjAecFlasza24aI8LYblammZh8hEJUR6vR4fffQRvvzyS3z//fdoaWnBxRdfjBkzZsTaPiJJcHl42Ly9hk6XXDQcE7bpxYCko9Do1a3pQHJ0wdOBWJweuDwMHNeNYEZASKKvSBGSozmDC5FSRlP2yUDEQsTzPDZv3oxt27bh9OnT4DgOAwYMQG5uLiWY6sW0eHtDClkXi0Trfxa2hqFBD5dkp8KgVXYqZr7ekFYl77ScdVgoNK1CRCQtEQkRYwxXX301duzYgTFjxmDUqFFgjOGnn35CeXk5tm3bhnfeeSdOphJi4l/aoZSF/rExnRXWmEmkQObgoE3GFXWdRN9odQLopn/IhyIVwPmQQY2AMEN3weLA1CEGpKUoun9NImIiEqLNmzdjz549+OSTTzB9+vSAY59++inmz5+PV199FbfccktMjSTEpzkcR/WZb4Rt9nBAHl4WxmDIJBJk68LMi90VXQQ1AsCpCxY0mB0Y3T+NhEgkIhKiN954A7///e87iBAAXH755XjwwQexZcsWEqLeguUCYGsEpHIMSsuA4ZJCINRIyVgF1HmHZf0ndHpau8sDs90dUmiG5moxNLejPycqwghq1KlkaDA7KJG+iEQkRIcOHcKf//znkMfnzJlDlV57A9ZG4OftgOmMf5cSQHZqNpAxAHANAPQFgNT79WmpA354R/g7fyygDb2S3eJw4297ToLjgKXTB0EmjfNsVRixRP6gRhsFNYpFRELU2NiInJzQX7KcnBw0NTV12yhCRCwXgIOvAU6r4OvRGACPUxCnljrhUbVXEKGUTOE1LXVC/FCqARjYsbfclhSFFEq5BA4Xj0arE9nawCEcYwyMCTmlY0JYKWOptJDYRCREHo8HMlnol0ilUrjd9KvSY+E9wE//EkRImwOMvA5Q6QEA+45VQ22pRomsHqrmSmGo40v1AQCGIcCQ2V36hjiOQ1aqEmebbLjQ0lGImm1uvPKf08hKVeLGiQXdn4X1Tdl34qym9WbiE/GsWXl5OZTK4GN7WiPWwzm7XxAXuQoY9V+AUhjWMMaw96wDTncWyi4dD1WKXEiMbzMKr9NkAeq0sC+TlarwC1F7Llgc8PAMnliFgviFqEWIrpZ0HArq2tQ4I8QhIiEqKyvrsg05qnsoHjdQ9bXw98DpfhECALuLh9O7JEOnkgkLWTVZwiMKMjXCD9kFS8cfrkaL09smRrNXitTW6GpnC6DqmLHBn5fIw4PnWeyGhUTYRCREmzZtipcdhNicPywMt1Q6IHdUwCHfkCVVKYuJczkzVRCZhqA9ImFfRqyESCIRRNXeLAzPgghRikKKu6eVQCmTUECuSNACG0Kg9oiw7Teuw9IMX6Szz6nbXbJShR5Rs80FuytwsWmdN8WIr01MUHrFxxE8tTDHcVTNQ2Ri880ieja2JmGqnuOAnI5FC3wr4aMq6xMElVyK0sKOwYNON48LLYIQhVuWOizCcFgT4kJCRAB1PwnbtKKgC0NNsUjJ0Y5pQ7M7mmG2gzHBZ5MaTZnpUPiGY/bQxRZ+rm3G0VozBmRpMLp/WuyuTYQFCREBXDghbLOHBT3s8xHpYihEwZBLJRiaq4U61knsuxiaAYDJ6sLJegtUcilG94/t5YmuISHq67hsQLO34kbGwKBN5ozMQ7PN1XUtswgxWp041WDB6P5pkEo45OhUuHJUXtcvjJQwhMgnsjSFLw4kRH2dptNCVLQmyx+82B6VXBrzUjuMMbz5bTWsTg+yUpUoyEiJ6fkDCCuokVLGignNmvV1Gk8K2xC9oXjBcRyKMoXlF8frzDjfbEddsz0+F/MJkdMiRI8HwdcjanG4wVg38mQTUUFC1NcxemvDpRcHPXyhxYHPfq7rXjWNEAzPE4ZMP55rxo7DNdiytwrfVcVhraJCI4QkMBayV5SqkEHCcfC0KatNJA4Sor6Mo0WYugcAXfDqvHVmBw5WG/FTTezLexdkqGHQKuHyMBitLsgkHEqyU7t+YaRwXJfDM4mE8/vAyE+UeEiI+jK++mOarJCLVZu82RLjkTCM4zjMHpmLDI0CSrkEvxqRE7NYpQ6E4bDWqmRQyCQdgiyJ+COqs3r16tXYtm0bfv75Z6jValx66aVYs2YNhg4Nnu+YiDG+fEP60PPVRqvgvE3vTjWNTshKVaLs0uL45zsPw2F9TWk/yCQcRViLgKg9ot27d2PJkiX4+uuv8dFHH8HlcmHmzJmwWCjReULw9Yg6ESJfjyg9Vmu/QhD3f/4wghrlUlprJhai9og++OCDgOebN29GdnY29u/fj1/+8pciWdVH4D2A2VsePIR/iDHWpkfUw3M5hzE0I8QjqeKITCZhZiYjIyPocYfDEZDzKFjZayJMrI2CGMkUgDp4ZY0WhxtONw8Jx8V0eYco+GKk7MaQTRotTnx+vB4SjsNVY/ITYxcBIImc1TzP495778XkyZMxcuTIoG1Wr14NvV7vfxQUFCTYyl6EpU7YagzCrFIQfL0hvVrW/fpiYqNKE7Y2ozCNHwQOwMl6C6oarRRLlGCSRoiWLFmCI0eOYOvWrSHbrFixAiaTyf+orq5OoIW9jBafEHVcfOqjICMFd08rwdVjgw/dehS+HpHHBbisQZv4E6S5edhdfKIsI5AkQ7OlS5fivffew549e9C/f2jHqVKpDJmmlogQS72wTTV02iweyztEQSoTZs4cZsBuak2q3waZVAKNUgqLwwOz3QW1ohfcdw9B1B4RYwxLly7F22+/jU8//RQDBgwQ05y+RRg9ol6Hr1fky7UdBB0l0hcFUYVoyZIleO211/D6669Dq9WitrYWtbW1sNlsYprV+3HZWuNpNMF7RDzP8PZ3Z7D7WL0/X3WPx5fg3x56uUprRQ+Krk4kogrRhg0bYDKZMG3aNOTl5fkfb775pphm9X58vSGVPmREtcnmwukGKw6fMULW0x3VPnwO605mznzpcGkVfmIR1UdEMxMi4fcPhR6W1XtTtmamKntPVYswhmZalRwKmQT0zUwsSeGsJhKMr0fUiRDVNQtCZIhlEnuxCWNoNrqfHmP66ynCOsEkzfQ9kUAsXTuqzxkFP11Mk9iLjT+o0SQUWwyChNaaiQIJUV+D57scmrk9PGq9Scr6p6sTZVn8UWi9eYl4WuqRZJAQ9TXsRqGqq1TW6rxtR22zHR6eQaOU9vylHW2RSMJa6vHBkRr8/etKf9VZIv6QEPU1Wtos7QhSBx4A7C4PUhRS9E9P6X3DFN+6OmtjyCYNLU40mB3+MkpE/CFndV8jDP/QoGwtBmalwunpJfFDbUnJAC5UALbQQqRVyVBvdtAUfgIhIepDMMbAhTFjBghOW5WkFy5xUHszO1hD58am0kKJh4ZmfQSTzYU3v62GzejNQaQxwGh14qwxMIq9rtkOnu/FUTQpPiG6ELKJb5kHDc0SBwlRH8Dl4fHu9+dQ19SMqrNCMUW3OgvbD9fgH/vOYO/JC+B5hoYWB7Z+W42NX57qvZUsUjKFrd0UsrSQLy2uLzslEX9oaNYHOFDZhAazAwaYUJyZAii14GVqZKQoUNfswFcVF/BjTTNsLg88PINBq4xt7flkQpEKSOVCOhBbk1A4oB2+bJQmmyv+ubQJANQj6vVYHG58e1pwzF6W54FSJgVSs6GQSTB7ZC5mjciFQiaB0eqCw8XDoFVi5vBcka2OIxzXZngW3GGtUwvLPHRqORy9ZcFvktNLf/YIH99XG+HyMOTqVShSeosSeFfccxyH4fk6DDRoUNVohUwiVF/t8dkYuyIlEzCfDzlzJpVwuGdaCfWEEggJUS/G7eFx2FuhdXxROrizwSOqVXIphuRoE22eeKi7dliTCCUWGpr1YirqLbA6PdCqZCjJ0rQu7ehLydCC4XNYdxLUSCQW6hH1YgozUvCr4TmQSjhInCbA7RTWWqUEr5LSZ/ALUYOQSD9I7+d0gwWfn2hApkaBK0flJdjAvgcJUS9GrZBiZD/v2qr6Y8I2JVMQo75MSibASQCXXchU6Su+2AaOAxrMDsqZlSBoaNZXsIQXUd0nkMpae4W+4Wo70rxT+Earq3cHeCYJJES9lO+qmrC/sqk1MLEvJsvvDF+ubt/70g6dSgaZhIOHZ7TUIwGQEPVCGGPYX9mEPcfqUefNKxROetg+he99sAQXIo7jkOaNsG6kCOu4Q0LUC6kzO2C2u6GQSVCYkdIaRQyQEPnw9QxDDM0AIF0jDM8aLY6QbYjYIKoQ7dmzB1dddRXy8/PBcRzeeecdMc3pNVTUtwAAijJTIJNKhH82xgBFStDCgn0SX2FJy4WQa84yNUK+7oYW6hHFG1GFyGKxYMyYMXjhhRfENKPXUVEvRFAPzEoVdrScF7apvXjpRqQodYBMKaSNDRHYaNAqkZmq6L3r7pIIUd/hOXPmYM6cOWKa0OswWV1oMDsg4TgMNHh7P/4cRJ2Xl+5TcJwwTDVWA+baoEPWQdmpGJSdKoJxfY8e5SNyOBxobm4OeBCBVDQIw7L8NFVrzXp/jyhHJKuSFK03UNFcI64dRM8SotWrV0Ov1/sfBQUFYpuUdNidHsgkHEp8v+SMtekRkRAFoMsXts3nOm3GGIO7N6bNTSJ6lBCtWLECJpPJ/6iurhbbpKTj0kFZuHNqCUbke6OFbU3CrJlU1rrYkxDQen1mlnqhskkQ/lNxAet3VeBAlTFxdvVBepQXTqlUQqnsRZVH44RC1ub3xTcs66RqR59FlQbI1YDLJsQT+XpIbZBJOTjdPC600BR+PKFvZi/C5gwyDU3+odBwXJvhWXA/UaY3lqiehCiuiNojamlpwYkTJ/zPT506hYMHDyIjIwOFhYUiWtbzcHl4bPzyFPRqOa4p7QeNb8o5zKodfRZtnlBeqPksgHEdDmfrhJLbjRYnnG4+sLdJxAxR39V9+/ahtLQUpaWlAIDly5ejtLQUK1euFNOsHkl1oxVON+8vjgjA66j2Dc1IiIKi7ydsTWeCHk5VyqBVycAY9Yriiag9omnTplGahRhx0hfEaNC0Zhd0NAOOFiHlhZaCGYOi6y+8P3YTYDMC6rQOTbJ1KpjtLTjfbEe/NHXCTewLUD+zF8AYw0lv/FCJoU0Ans/vkWoQKlcQHZEpWkXaWBW0SY5WmCDxLyAmYg4JUS+gttkOi8MDhUyC/ukprQeazwpbbcfZIKINaV5/pCl4OEh+mhoDDRrk6ak3FC961PQ9EZzj54Xe0MCsdhU4fBHDQaaliTakFQJVX4fsERVkpKAgIyXoMSI2UI+oh8MYw4k6QYgC1kXxHhKicNEXCH4im7E1XQqRUEiIegFTBmfhojwtijLbpPjwRQvLlK3J4ongyBSAvr/w94WTIZu1ONyoM5OfKB6QEPVwOI7D4BwtZo/MC4xx8U1H6/KDVqkg2pE5SNheOBH08PHzZry05yQ+/Sl4Rkeie5AQ9VaaTgvbtCJRzegx+ITIWCmUXWpHjl4IbDzf7ICTylDHHBKiHkxDiwNfnWhAk6XdPw7Ptzpe00mIwiIlA1CnC741n4i3QaeSQ6eWg2cM54y2xNvXyyEh6sH8cK4Ze0814quKdhkGLXWA2yH4PigrY3hwXGuvqP7noE0KvTNnpy5YEmVVn4GEqIfi4Rl+rhESw12U165ufVOlsNUX0or7SMi+SNg2HA06PPNlvDxZb6EVATGGvqU9lMoLQl37FIU0cLYMaHW4phcn3K4ejS5fGJ553EDDsQ6HCzNSIJNwaLa5KKF+jCEh6qF8f8YIABiWpwsMYnRaW2fMsgYl3rCeDMcBOSOEv88f6XBYLpWgMFMYnp30VkohYgMJUQ+k0eLE6QYrOA4Y2z+t3cEKoTJFqkH4dSciI3ekIEiNp4RSQ+0YW5CG2SNzcXERvbexhISoB/JdlRD9O9CQCn1Ku8Ws9UeFbdaQBFvVS1Cntzqtz+7vcLgoU4OL8nSQS+lfJ5bQu9kD0ShlUMgkuLgwLfCAo0VI8gUA2cMTblevoZ83QVrtIcAZeoaMHNaxg4SoB/KLgZm47bIBgSvtAcGvwXjB6arJEse43kB6MaDNEYoOVP6nw2HGGA5WG7Hpy9MwWV2Jt68XQkLUQ/HXLPPB88C574S/88Yk3qDeBMcBA6cJf5870GEhLMdxOFnfApPNhb2ngleJJSKDhKiH4OEZ3jt0DmearMEb1P0orB5XpNCwLBakDxB6RrwHOPqBkHa3Db8YKCwk/uFcM85SpHW3ISHqIew6Wofj51uw/VANXO2L/XlcwKk9wt/9JwgR1UT34DhgyCyhHlzTaeDMvoDD+Wlqf+24nUdqg1dQIcKGhKgHsL+yEYfOmMBxwMwRuR1nbE7tEXIuq3RAv/HiGNkbSckABk4X/q74BGgIXJn/yyEG6NVymGwuvHPwLKzO4EUaia5JCiF64YUXUFxcDJVKhUsuuQTffPON2CYlBR6e4YvjDdhzrAEAMHlQFgZktYuiPv8DUO19vwbPpN5QrOk3DsgbLQzNftgmvN9eVHIprh6bD5VcilqTHW98U02lqaNEdCF68803sXz5cqxatQoHDhzAmDFjMGvWLNTV9e28L9WNVmz9tgrfnm4EAFxakokJxW1KRvM8ULUX+OnfwvOCCUDWYBEs7eVwHDBkNmAYKviLfnxXeNiMAICsVCVumFCADI0Cw/N0kHl7q2a7C58fr8ehM0acarCgrtkOs90FD09T/sHgmMjBEJdccgkmTJiA559/HgDA8zwKCgrw3//933jwwQc7fW1zczP0ej1MJhN0Ol0izI05PM9gd3tgcXigkkugVQkBit9VNWHX0Xoo5RLMuCgHQ7JTAbddmMFpqhRiXKyCSCFvjPDPQgtc4wfPA6d2A9V7hd4RJwEyBgAZJYA2B25lGiBTQyYTZjNPN1jw9ndng55KrZDiskFZGNlPD0AQraO1ZqQoZNAopVArpNAoZFDLpZBIenZSu3D/R0VNnu90OrF//36sWLHCv08ikWDGjBn4z386xm84HA44HK1F7pqbm8O+1lmjDZ/+JBQb9ClvWwmeOCADF+UJb1StyY73j9QEtGmr1uOL0jGmIE2w4ZstOFxR3eZkvP8FDEBBugpFGRoADFanW4iKZkIDBsDD8/7XFmemQJuRAoBhNM+QbrUgP00NxXEOOOoWFmO2RaYESi4XhIiyMMYXiQQomS5ErJ/+XFgCcqHCH0AqA4TPQKYEJHJkOXnMtTlgdzPY3QxON4PLw4P3ftYZTh1wVki25rQ4Yao2wtTukhwnrG8bZEhFnjcxm9Hmws81Zv/HfbzgN2BcayjHuKIMDPc60evMduw8UhvylsYWpGNUf0EMGy1ObD8cvOw2AIzM16G0UFjW0mx3QS6RQK2QhmwfKaIKUUNDAzweD3JyAuuy5+Tk4OefO+aEWb16NR5++OGoruVy852umLa7Wmc93DwPYyeBas62fgDrBUisDSHbSmxqwCZcl3N5IHEEiqfvo5RJOHBOBjiZf3+xTgLwDr+2ARCm53X9hF/jnJHCF59IHPp+wJjfApYGYYW+sVrID+4wCz8oLjsAO1IBDGmXnYUBcHuE74+CawYswsJZpcONIqUdLg8Pp4eHy8PD7WH+Hz+J1Q7IBCGCzQlmMfuPNbQ4A4TI1vZ77GGdfufbOtfdPI8Gc+hKttY2s4I8z8AksR1I9ahyQitWrMDy5cv9z5ubm1FQUBDWa3N0Klx3sZAgPVjnIa3Nmi2DVhj3+/C15yD8kapqfdvUo+dh8EBbm0acdyMMk9RyGaCQAhwHOc8wbJQbACe04zgopRyUcpl3BT3XxrjA84GTAEotFUpMFjRZwsOXAJP3AC6b8GAeIcK97QPCJyr3PtqSCsCfJ8HbY+KZ8ONod3uQIpcBcu/3yc2j2Or0d8D76woDvtBt1x5maBT4zbj+IW9Bp2ptq1fL/f8fQduqW7/zGqUMkhj3wEUVoqysLEilUpw/fz5g//nz55Gb2zGzoFKphFIZXQ9ArZD6Uzh0hVImRX6YpYXlGUXIyui6HSC82bRmu5cikQLKVOERi9MBSPE+2qIEkBfmOVRyadj12JSy8P8/4rHgV1TvpkKhwLhx4/DJJ5/49/E8j08++QSTJk0S0TKCIBKJ6EOz5cuXo6ysDOPHj8fEiROxdu1aWCwW3HrrrWKbRhBEghBdiG644QbU19dj5cqVqK2txdixY/HBBx90cGATBNF7ET2OqDv0hjgigujNhPs/ShFwBEGIDgkRQRCiQ0JEEIToiO6s7g4+91YkSz0Igkgcvv/NrlzRPVqIzGYzAIQdXU0QhDiYzWbo9fqQx3v0rBnP8zh37hy0Wi24Hr7o07dcpbq6utfMANI9JT/xvh/GGMxmM/Lz8yHpJDtEj+4RSSQS9O8fen1MT0Sn0/WKL3hb6J6Sn3jeT2c9IR/krCYIQnRIiAiCEB0SoiRBqVRi1apVUWcXSEbonpKfZLmfHu2sJgiid0A9IoIgRIeEiCAI0SEhIghCdEiICIIQHRKiBBJJRduXXnoJU6ZMQXp6OtLT0zFjxoykrIAbbZXerVu3guM4zJ8/P74GRkik92M0GrFkyRLk5eVBqVRiyJAh2LFjR4KsDY9I72nt2rUYOnQo1Go1CgoKcN9998Fut8fXSEYkhK1btzKFQsE2btzIfvjhB3b77beztLQ0dv78+aDtFyxYwF544QX23XffsZ9++omVl5czvV7Pzpw5k2DLQxPpPfk4deoU69evH5syZQqbN29eYowNg0jvx+FwsPHjx7Mrr7ySffHFF+zUqVNs165d7ODBgwm2PDSR3tOWLVuYUqlkW7ZsYadOnWI7d+5keXl57L777ournSRECWLixIlsyZIl/ucej4fl5+ez1atXh/V6t9vNtFote+WVV+JlYsREc09ut5tdeuml7P/+7/9YWVlZUglRpPezYcMGNnDgQOZ0OhNlYsREek9Llixhl19+ecC+5cuXs8mTJ8fVThqaJQBfRdsZM2b493VW0TYYVqsVLpcLGRlh1i6KM9He0yOPPILs7GzcdtttiTAzbKK5n3fffReTJk3CkiVLkJOTg5EjR+Lxxx+Hx+MJ2j7RRHNPl156Kfbv3+8fvp08eRI7duzAlVdeGVdbe/Si155CpBVtg/HAAw8gPz8/4EslJtHc0xdffIGXX34ZBw8eTICFkRHN/Zw8eRKffvopFi5ciB07duDEiRO455574HK5sGrVqkSY3SnR3NOCBQvQ0NCAyy67DIwxuN1u3HXXXfj9738fV1upR9QDeOKJJ7B161a8/fbbUKlUYpsTFWazGTfffDNeeuklZGVliW1OTOB5HtnZ2fjb3/6GcePG4YYbbsAf/vAHvPjii2KbFjW7du3C448/jvXr1+PAgQPYtm0btm/fjkcffTSu16UeUQKItKJtW5588kk88cQT+PjjjzF69Oh4mhkRkd5TRUUFTp8+jauuusq/j+eFUswymQxHjx5FSUlJfI3uhGg+o7y8PMjlckilrbXnL7roItTW1sLpdEKhUMTV5q6I5p7++Mc/4uabb8bixYsBAKNGjYLFYsEdd9yBP/zhD53mFOoO1CNKANFWtP3zn/+MRx99FB988AHGjx+fCFPDJtJ7GjZsGA4fPoyDBw/6H1dffTWmT5+OgwcPip5lM5rPaPLkyThx4oRfUAHg2LFjyMvLE12EgOjuyWq1dhAbn9CyeC5LjasrnPCzdetWplQq2ebNm9mPP/7I7rjjDpaWlsZqa2sZY4zdfPPN7MEHH/S3f+KJJ5hCoWD/+Mc/WE1Njf9hNpvFuoUORHpP7Um2WbNI76eqqopptVq2dOlSdvToUfbee++x7Oxs9qc//UmsW+hApPe0atUqptVq2RtvvMFOnjzJPvzwQ1ZSUsL+67/+K652khAlkOeee44VFhYyhULBJk6cyL7++mv/salTp7KysjL/86KiIgagw2PVqlWJN7wTIrmn9iSbEDEW+f189dVX7JJLLmFKpZINHDiQPfbYY8ztdifY6s6J5J5cLhd76KGHWElJCVOpVKygoIDdc889rKmpKa42UhoQgiBEh3xEBEGIDgkRQRCiQ0JEEITokBARBCE6JEQEQYgOCRFBEKJDQkQQhOiQEBEEITokRESPYvPmzUhLS/M/f+ihhzB27Fj/8/Ly8qRLP0t0DQkREZTy8nJwHIe77rqrw7ElS5aA4ziUl5cHtI+1ABQXF2Pt2rUB+2644QYcO3Ys5GvWrVuHzZs3+59PmzYN9957b0ztImIPCRERkoKCAmzduhU2m82/z2634/XXX0dhYaEoNqnVamRnZ4c8rtfrA3pMRM+AhIgIycUXX4yCggJs27bNv2/btm0oLCxEaWlpt84drKcyf/58fy9r2rRpqKysxH333QeO48BxHICOQ7P2tO2ZlZeXY/fu3Vi3bp3/HKdOncKgQYPw5JNPBrzu4MGD4DgOJ06c6NZ9EdFBQkR0yqJFi7Bp0yb/840bN+LWW2+N+3W3bduG/v3745FHHkFNTQ1qamoiPse6deswadIk3H777f5zFBYWdrgnANi0aRN++ctfYtCgQbG6BSICSIiITrnpppvwxRdfoLKyEpWVlfjyyy9x0003xf26GRkZkEql0Gq1yM3N7TKTZTD0ej0UCgVSUlL855BKpSgvL8fRo0f9CeJdLhdef/11LFq0KNa3QYQJpYolOsVgMGDu3LnYvHkzGGOYO3duj885nZ+fj7lz52Ljxo2YOHEi/v3vf8PhcOD6668X27Q+C/WIiC5ZtGgRNm/ejFdeeSVmvQaJRNIh9ajL5YrJucNh8eLFfkf8pk2bcMMNNyAlJSVh1ycCISEiumT27NlwOp1wuVyYNWtWTM5pMBgC/D4ejwdHjhwJaKNQKLpdIyzUOa688kpoNBps2LABH3zwAQ3LRIaGZkSXSKVS/PTTT/6/Q2EymTrULMvMzAyaGP/yyy/H8uXLsX37dpSUlODpp5+G0WgMaFNcXIw9e/bgt7/9LZRKZVRDwuLiYuzduxenT59GamoqMjIyIJFI/L6iFStWYPDgwZ0WMSDiD/WIiLDQ6XTQ6XSdttm1axdKS0sDHg8//HDQtosWLUJZWRluueUWTJ06FQMHDsT06dMD2jzyyCM4ffo0SkpKYDAYorL7/vvvh1QqxfDhw2EwGFBVVeU/dtttt8HpdCZkFpDoHMpZTfRZPv/8c1xxxRWorq7uUA2VSCwkRESfw+FwoL6+HmVlZcjNzcWWLVvENqnPQ0Mzos/xxhtvoKioCEajEX/+85/FNocA9YgIgkgCqEdEEITokBARBCE6JEQEQYgOCRFBEKJDQkQQhOiQEBEEITokRARBiA4JEUEQovP/AUpMjqFA0wq9AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:23:57.999055Z", + "iopub.status.busy": "2024-02-29T06:23:57.998731Z", + "iopub.status.idle": "2024-02-29T06:23:58.353860Z", + "shell.execute_reply": "2024-02-29T06:23:58.352981Z" + }, + "papermill": { + "duration": 0.377542, + "end_time": "2024-02-29T06:23:58.355799", + "exception": false, + "start_time": "2024-02-29T06:23:57.978257", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEqCAYAAACx7V3bAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABIEElEQVR4nO2deXxTZfb/Pzd70qTpvkFLyw6yCsIgKKKMCg7i6CiOjrKNjmOdGeXrMow/QZxxQEcdmO84fL8yAjqiuHzRGVdUFFAQRBZFRQqlG0vpmqXZk/v8/rjJbdMmbZKmuWl73q/XfaW598m95zbJJ89znvOcwzHGGAiCICREJrUBBEEQJEQEQUgOCRFBEJJDQkQQhOSQEBEEITkkRARBSA4JEUEQkkNCRBCE5JAQEQQhOSREBEFIjqRCVFxcDI7jOmylpaVSmkUQRIJRSHnxAwcOwOfzic+//fZb/PjHP8aNN94ooVUEQSQaLpkWvd5777145513cOLECXAc12V7nudx9uxZGAyGiNoTBJFYGGOwWq0oKCiATBZ+ACZpj6gtbrcbL730EpYtWxZWVFwuF1wul/j8zJkzGD16dKJMJAgiRmpqajBw4MCwx5NGiN566y2YTCYsWrQobJvVq1dj1apVHfbX1NQgNTW1B60jCCIWLBYLCgsLYTAYOm2XNEOzq666CiqVCm+//XbYNu17RIGbNJvNJEQEkYRYLBYYjcYuv6NJ0SOqqqrCxx9/jG3btnXaTq1WQ61WJ8gqgiASRVLEEW3atAk5OTm45pprpDaFIAgJkFyIeJ7Hpk2bsHDhQigUSdFBIwgiwUj+zf/4449RXV2NJUuWSG1K0sAYg9frDYqxIohkRC6XQ6FQdDt8RnIhuvLKK5Ek/vKkwO1249y5c7Db7VKbQhARodPpkJ+fD5VKFfM5JBciohWe51FRUQG5XI6CggKoVCoK1CSSFsYY3G436uvrUVFRgWHDhnUatNgZJERJhNvtBs/zKCwshE6nk9qc5ID3AS4LwBigNgBypdQWEW3QarVQKpWoqqqC2+2GRqOJ6TwkRElIrL8qfQ7GAIcJ8LmF57wXSMkCOPr/JBPx+LzSO0okL16XIEIcAI4Tekceh9RWET0ACRGRvHj8DntlijAsA0iI+igkRIRkLFq0CNddd13og7wP8PmX8yi1gEIj9Ix8HuFYGC677DLce++9nV63uLgYa9eujcVkoocgISLiQiQCEBU+N8AAyBWCg1omB2TK1mNEj/Poo49iwoQJCbkWCRGRnHj9vSF5m7WFcn+cipeEqK9BQtRLcHv5sJvXx0fc1hNB22hZtGgRdu3ahXXr1onpfsvLy7F06VKUlJRAq9VixIgRWLduXcjXr1q1CtnZ2UhNTcVdd90Ft8vV2utRtAmSCwgRLxyz2Wy4/fbbodfrkZ+fj6effrrDuevq6jBv3jxotVqUlJRgy5YtHdpwHIf169djzpw50Gq1GDx4MN544w3xeGVlJTiOw2uvvYZLLrkEWq0WF110EcrKynDgwAFMnjwZer0ec+bMQX19fcT/t40bN+KCCy6AWq1Gfn4+7rnnHvFYdXU15s+fD71ej9TUVNx00004f/68eDzQW/nXv/6F4uJiGI1G3HzzzbBarWIbnufx5JNPYujQoVCr1SgqKsLjjz8uHn/ooYcwfPhw6HQ6DB48GI888gg8Hg8AYPPmzVi1ahW+/vpr8T3dvHlzxPcWLTR930t49tOTYY+VZKXguokDxOfP7S6Hxxc6Wn1guhY3Ti4Un2/cUwGHO9jnct+Ph0dl27p161BWVoYxY8bgscceAwCkp6dj4MCBeP3115GZmYm9e/fizjvvRH5+Pm666SbxtTt27IBGo8HOnTtRWVmJxYsXIzMjHY///rfCTJm8rRD5P668F2AMDzzwAHbt2oV///vfyMnJwR/+8AccOnQoaDixaNEinD17Fp9++imUSiV++9vfoq6ursM9PPLII1izZg3WrVuHf/3rX7j55ptx9OhRjBo1SmyzcuVKrF27FkVFRViyZAluueUWGAwGrFu3DjqdDjfddBNWrFiB9evXd/k/W79+PZYtW4Y1a9Zgzpw5MJvN2LNnj3B7PC+K0K5du+D1elFaWooFCxZg586d4jnKy8vx1ltv4Z133kFzczNuuukmrFmzRhSb5cuXY8OGDfjrX/+KGTNm4Ny5c/jhhx/E1xsMBmzevBkFBQU4evQo7rjjDhgMBjz44INYsGABvv32W3zwwQf4+OOPAQBGo7HL+4oZ1osxm80MADObzVKbEhccDgf7/vvvmcPh6HDsmQ+Ph93ePHQ6qO1/7ygL2/a1A9VBbdfvPNmhTSzMnDmT/e53v+u0TWlpKbvhhhvE5wsXLmQZGRnMZrO12rN+PdPr9czXfJqxlvrgE/A8Y5ZaxsxnmdXUyFQqFXvttdfEw42NjUyr1Yp2HD9+nAFgX375pdjm2LFjDAD761//Ku4DwO66666gS02dOpX9+te/ZowxVlFRwQCwf/7zn+LxV155hQFgO3bsEPetXr2ajRgxotP/QYCCggL28MMPhzz24YcfMrlczqqrW9+r7777LuheVq5cyXQ6HbNYLGKbBx54gE2dOpUxxpjFYmFqtZpt2LAhInsYY+wvf/kLmzRpkvh85cqVbPz48V2+rrPPbaTfUeoR9RJKZw0Ne0zWbhXInZcOCdu2/YqRJdNLumNWpzz77LPYuHEjqqur4XA44Ha7Ozg/x48fHxRFPm3aNLS0tKDm9FkMGjoi+IQcJziuvS6UnyiD2+3G1KlTxcMZGRkYMaL1NceOHYNCocCkSZPEfSNHjkRaWloHW6dNm9bh+ZEjR4L2jRs3Tvw7NzcXADB27NigfaF6W+2pq6vD2bNnccUVV4Q8fuzYMRQWFqKwsLXnOnr0aKSlpeHYsWO46KKLAAizf20zH+bn54vXP3bsGFwuV9hrAMCrr76Kv/3tbygvL0dLSwu8Xq9kCQbJR9RLUClkYTeFXBZxW2UEbePB1q1bcf/992Pp0qX48MMPceTIESxevBhudxSO5lDLOWSB4VniMxMola32BNYAtt/H81372LRabdztaX/9rq7xxRdf4NZbb8XcuXPxzjvv4PDhw3j44Yeje3/iCAkRERdUKlVQ2pI9e/bg4osvxt13342JEydi6NChKC8v7/C6r7/+Gg5Ha5Divi++gF6fgsKBBa3T9W3xC9GQ4kIolUrs379fPNTc3IyysjLx+ciRI+H1enHw4EFx3/Hjx2EymTqcdt++fR2et/UPxRODwYDi4mLs2LEj5PFRo0ahpqYGNTU14r7vv/8eJpMp4mIRw4YNg1arDXuNvXv3YtCgQXj44YcxefJkDBs2DFVVVUFt2r+nPQkNzYi4UFxcjP3796OyshJ6vR7Dhg3Diy++iO3bt6OkpAT/+te/cODAAZSUBA8F3W43li5div/3//4fKisrsfLRR3HPHYshkyuE2KH2+IVIr9Ng6dKleOCBB5CZmYmcnBw8/PDDQeueRowYgauvvhq/+tWvsH79eigUCtx7770hewuvv/46Jk+ejBkzZmDLli348ssv8fzzz8f3n9SGRx99FHfddRdycnIwZ84cWK1W7NmzB7/5zW8we/ZsjB07FrfeeivWrl0Lr9eLu+++GzNnzsTkyZMjOr9Go8FDDz2EBx98ECqVCtOnT0d9fT2+++47LF26FMOGDUN1dTW2bt2Kiy66CO+++y7efPPNoHMUFxejoqICR44cwcCBA2EwGHosVTP1iIi4cP/990Mul2P06NHIzs7GVVddheuvvx4LFizA1KlT0djYiLvvvrvD66644goMGzYMl156KRYsWIBrr5mDR5f/lzAsC5UCJSBOvA9/efIJXHLJJZg3bx5mz56NGTNmBPmDACENcUFBAWbOnInrr78ed955J3JycjqcdtWqVdi6dSvGjRuHF198Ea+88kqPlqpauHAh1q5di3/84x+44IIL8JOf/AQnTpwAIAyx/v3vfyM9PR2XXnopZs+ejcGDB+PVV1+N6hqPPPII/uu//gsrVqzAqFGjsGDBAtGHdO211+K+++7DPffcgwkTJmDv3r145JFHgl5/ww034Oqrr8asWbOQnZ2NV155JT43H4KkqeIRC5FWCOgtOJ1OVFRUoKSkJOZ0Cr0epxlw2wFVCqAJ855azwOMF1bixyEtCMdxePPNN8MvNyE6pbPPbaTfUeoREckF7xUeOxMYsVfk7Xl7iIRAQkQkD4wJi1oBVJ8+C71eH3rLLUZ1zWlJZs4iJaztej0+++wzqc1LOshZTSQPzCeIEQcUDCzqEMcj4m5BQV5m3ISoJ7wTYW0HMGDAgLDH+iuSC9GZM2fw0EMP4f3334fdbsfQoUOxadOmiGcHiD6Ezz/UkimgUCoxdGiYIE6PHXCYAZa8Q7OwthMhkVSImpubMX36dMyaNQvvv/8+srOzceLECaSnp0tpFiEVAZ9PqPihtnCtM2dE30BSIXriiSdQWFiITZs2ifvax5kQ/Qhe8A+J0dPhCBwXh3JU6aS3I6mz+j//+Q8mT56MG2+8ETk5OZg4cSI2bNgQtr3L5YLFYgnaiD6EOGPWhRBxMkF8GAQxIno9kgrRqVOnsH79egwbNgzbt2/Hr3/9a/z2t7/FCy+8ELL96tWrYTQaxa3tokCil8NYFEMzrrWSBw3P+gSSBjSqVCpMnjwZe/fuFff99re/xYEDB/DFF190aO9yueByucTnFosFhYWFFNDYF/B5AFuDIDL63K6HW/YmIYujxgioqAaclPT6gMb8/PwOYfSjRo1CdXV1yPZqtRqpqalBG9FH4FtnzCLy+QSCGmlo1ieQVIimT5+O48ePB+0rKyvDoEGDJLKIiJVuJ89vK0SR0MXMWdyT+RM9iqRCdN9992Hfvn3485//jJMnT+Lll1/Gc889h9LSUinNIqSA9wm5cCIWIv9Hl0WfY5tIQiLOI9lDvP3222zMmDFMrVazkSNHsueeey7i1/aLVLE8z5jHlfiN5yO2e+HChQz+OazAdvLkSbZkyRJWXFzMNBoNGz58OFu7dm2H182fP5/96U9/Yvl5uay4qJAxt4Pt2bOHjR8/nqnVajZp0iT25ptvMgDs8OHD4muPHj7Irp49i6Wk6FhOTg77xS9+werr68PaU1FR0Z23huiEPpEq9ic/+Ql+8pOfSG1G8uLzAJ91rE7R41zyX8EVNDqhu8nzU1NT8dG/XwUYg6XFjnnz5mHu3Ll4+eWXUVVV1WGIZTKZcPmVV+OXt92Mv65ZBYdMj4ceegg33XQTPvnkk5D2ZGdnx+f/QvQIkgsR0fsxGo1QqVTQ6XTIy8sT969atUr8u6SkBF988QVee+21ICFKSUnBP5/7H6jcZoDj8D8vvQqO47BhwwZoNBqMHj0aZ86cwR133CG+5u9//zsmTpyAP69cLuww5GLjxo0oLCxEWVkZhg8fHtIeInkhIUp25EqhdyLFdbtJJMnzx44dK+TJdgOQyXG8rAzjxo0LmgaeMmVK0Gu+/vprfPrpTugLAuu5WmfZysvLMXx4dOWQCOkhIUp2OC7iIVIyEUie//TTT2PatGkwGAz4y1/+EpRjGhB6ROLMV4SO6paWFsybNw9PrHwQ8PkAbZr4P8rPz4/nbRAJgoSIiAudJc8PECp5PoCgqfsRI0bgpZdegsvlEvMjHzhwIKj5hRdeiP/7v/9DcXEJFPCFDGpMZOJ3ovtQYjQiLrRNnt/Q0IBhw4bhq6++wvbt21FWVoZHHnmkg6CItBGiW265BTzP484778SxY8ewfft2PPXUUwBaS/iUlpaiqakJP190Jw4cPILykyewfft2LF68WBSf9vZEUuaHkA4SIiIuxJo8H0AbIZIjNTUVb7/9No4cOYIJEybg4YcfxooVKwBA9BsVFBRgz5498PE8rvzpzzF28jTce++9SEtLE6t4tLcnXLQ+kRxQ8vwkol+uNWO8kAwfAAy5rYGKbdiyZQsWL14Ms9kcXArIbReS7SvUgC4jQQYT7YnHWjPyERHSIjqqZaIIvfjiixg8eDAGDBiAr7/+WowR6lCPLFDDjNab9XpIiAhpCQzLuNZiirW1tVixYgVqa2uRn5+PG2+8EY8//njH11Kmxj4DCREhLSGm7h988EE8+OCDXb82IESMCUO8EMM6ondA7xwhLW0c1VHDca0pQ2jxa6+GhCgJ6cXzB9HDogtmDCIoUyMJkVTE4/NKQpREKJXCsgq73S6xJQmkOz0ioE06EPITSUXg8xr4/MYC+YiSCLlcjrS0NNTV1QEAdDqdGMTXJ2E84PSn/lV4AVkMvRqPD/B6ADgAXx/+XyUhjDHY7XbU1dUhLS0NcnmMPyYgIUo6AqvFA2LUp+F9gLsFAAdobLGdw+MAfG4hlkjRT2Kvkoy0tLRuZzkgIUoyOI5Dfn4+cnJy4PF4pDanZ2mqAE7sB/Q5QMlPYzvHmUPA6QOo0xTjCzYGSjmHKSWZyNKr42srERKlUtmtnlAAEqIkRS6Xx+UNTmq8ZoC3A1odEGskuS4FVrsZ35+pwNnsUcKuWjuuHmOMo6FET0NCREiHo1l41HajxLhKj8pGO5S8AoOzUzA4S48xA3r/cp/+BgkRIR0Ok/DYDSGqdyvR4vJCrXBg2uhc6FT0ke6N0PQ9IR2BHpEmLeZTfNsgTNvnaXzQKVpnzTw+Hk02d3esIxIICREhDTwPuKzC39q0mE7BGEN5Mw8GTnBOu4WZt5omO57bfQrvHT0XJ2OJnkZSIXr00UfBcVzQNnLkSClNIhKFyyLEEcnkgEof0yk4jsONk4swKD8HqVqlPxQAyNKr4fHxqLe6YHH28ZnHPoLkA+oLLrgAH3/8sfhcoZDcJCIRuCzCozo1shLTYTDqlDDmZAHWWrFHpFXJkW/U4KzJiaoGO8YOpBm0ZEfyb71CoaCSL/0Rp1l41MRhhkttEIQoMNQDMCgzBWdNTlQ22kiIegGS+4hOnDiBgoICDB48GLfeemunKT1dLhcsFkvQRvRSnP73ThObSDDG8PH353G4uhkeuT9hmrs1OrsoQ0imf8bk6F+LiHspkgrR1KlTsXnzZnzwwQdYv349KioqcMkll8BqtYZsv3r1ahiNRnErLCxMsMVE3Aj0iNSx9YgsTi+OnjFjd1kDOLVB2On3EQFAjkENhYyDw+2DyU5+omRHUiGaM2cObrzxRowbNw5XXXUV3nvvPZhMJrz22msh2y9fvhxms1ncampqEmwxETdc3esRnbc4AQDZBjUUGr+z29UqRAq5DLmpQrT2WbMjdjuJhCC5j6gtaWlpGD58OE6ePBnyuFqtFmtdEb2cbvqIas2CEOUZ1YKPCAjqEQHAmAFGDMlJQYFR2/7lRJIhuY+oLS0tLSgvL6dqnX0dxlp9RDEOzRptQvqQbL2mdfq/nRCNLkjFpEEZSE/pfZVy+xuSCtH999+PXbt2obKyEnv37sVPf/pTyOVy/PznP5fSLKKncduEhGgcF/PQrLFFiJrO1KsAVYr/vHbK1NhLkXRodvr0afz85z9HY2MjsrOzMWPGDOzbtw/Z2dlSmkX0NAH/kColpsyMTo8PVqeQ2TEjRQXIVYKoMR7wOlqFCUCzzY3zVicK0rRI1cSeQZDoWSQVoq1bt0p5eUIqRP9QbL0hs8MDjgP0agU0Sr+QKbVCj8htCxKiT4/XoarRjtmjcimeKIlJKmc10U/opn8oN1WD0llD0eLvFQEQxMdtF4Ia9Tni7myDGlWNdtS3OAGQECUrSeWsJvoJ3ewRAYBSLgt2QosO6+CUs9kGYZa13uqK+VpEz0NCRCQeMYYojgnMRId1OyHyp4xtaHFThHUSQ0JEJB6nSXiMMQ/Ru9+cw8ffn0eLq+3QLHSPKF2ngkLGwe3lYXZQhHWyQkJEJJ5u+Ig8Ph5l5604esYMedtV+6IQBS8Pksk4cQjXSInSkhYSIiKxeJyA1++vicFHZPH3alQKGTTKNh/fMEMzwD/FD2Eqn0hOaNaMSCwB/5BSAyiij3g2+YUoTacMLj7ZiRCNL0zD8FwDclNpeVCyQkJEJJZuzpgFVtKnaduJWJj1ZgAwII3WmiU7NDQjEks3Y4gCQzOjtl2UdKBH5HULG9GrICEiEks3Z8xMDkFk0nTthEiuAuT+Dn6IXlF5fQu+rGiCw+2L6bpEz0JCRCSWbsYQub3CotYOPSKOCzuFDwC7y+qx52QDGloosDEZIR8REXcYY8GO5LZ000e04KIiuLw+KGQhfkNVKULRxjAzZya7B002Nwr9aWSJ5IF6RERcqbM68dK+Kjg9YYZA3fQRAYBaIYdcFkLoOukRpesE5zYVXUxOSIiIuPLpD3VoaHHjUFVzx4M+b6tIdGOdWVjCBDUCrbFEAR8TkVyQEBHdxuzw4Ky/WsaEQqGO/ffnLOD5dmu7Av4huUJI2xElJ+us2HboNA5VhxA5oNNYooBPyUyJ9JMSEiKi23x3xoxXD9Tgkx/qMCQ7BRqlHFanF+f8Ce5F2s6YxVBUsc7qQlWjPXyEdGdC5J9lMzu8HQWSkBwSIqLb1DTbAQh5ghRyGQZlCs7gqsZ2gtBN/1AgK6MhXKbFToIaDWoFFDIOPGPieYjkgWbNiG7h8fE4bxGmxAemC8OtogwdjtdacbrJAQxp07ibM2atQhTmY9tJj4jjOFw7oQA6lQL6cK8nJIPeEaJb1Jqd8PEMBo1C9MMEllSctwjHxBmubsYQWZ2CfyciIeJ5oN0U/6DMlBAvIpIBGpoR3aLW7wfKN2rF2KE0nRJalRwZehXs7jbDoG70iBhjYmpYgzrM0EyZ4k+izwCPPeprENKRNEK0Zs0acByHe++9V2pTiCho8KdgDaRkBYRh0B2XDMatUwcF+3O64SOyu33w8kxImh+uRySTtc7GhRiemexufFXZhK9rTFFfn+hZYhKiU6dOxdWIAwcO4H//938xbty4uJ6X6HnqWzoKEYCOAYc8LyS2B2LqETk8PmhVcqSoFKGDGQOEKbYICCv3PzvRgG9Om6K+PtGzxCREQ4cOxaxZs/DSSy/B6XR2/YJOaGlpwa233ooNGzYgPT29W+ciEs/M4dmYMSwrbK4fMU+02yrUHeNkrWIRBVl6Ne6aOQSLpxd33rCT6Oo0cQrfQ/mrk4yYhOjQoUMYN24cli1bhry8PPzqV7/Cl19+GZMBpaWluOaaazB79uwu27pcLlgslqCNkJZBmSm4qDgDOlXwcKnF5cWW/VXY8Nkp4UvfttZ9qHViEaKQd/Fa0WEdYgpfo4SM4+DxseB814TkxPSJmDBhAtatW4ezZ89i48aNOHfuHGbMmIExY8bgmWeeQX19fUTn2bp1Kw4dOoTVq1dH1H716tUwGo3iVlhYGIv5RALQKuVosLphc/mEL30c1phFRCdT+HIZJ864mSjCOqnolrNaoVDg+uuvx+uvv44nnngCJ0+exP3334/CwkLcfvvtOHfuXNjX1tTU4He/+x22bNkCjUYT0fWWL18Os9ksbjU1Nd0xn+gmZ0wOlJ23hqyOIZdx4lCoyebudgzRzuN12HboNKobu5gN6ySoEWgdnlmcJETJRLeE6KuvvsLdd9+N/Px8PPPMM7j//vtRXl6Ojz76CGfPnsX8+fPDvvbgwYOoq6vDhRdeCIVCAYVCgV27duFvf/sbFAoFfL6Oq7fVajVSU1ODNkI6jp424d1vzqHsfMdFpkDrQtPGOAjRObMTVY12uH185w076REBQKp/Fs/ioKFZMhFTQOMzzzyDTZs24fjx45g7dy5efPFFzJ07FzL/2L+kpASbN29GcXFx2HNcccUVOHr0aNC+xYsXY+TIkXjooYcgl8tjMY1IIM3+4U16+2yJfjJTVDgJoLHFDbi7G1UtXCu1q6jogBC5QveIUrXUI0pGYhKi9evXY8mSJVi0aBHy8/NDtsnJycHzzz8f9hwGgwFjxowJ2peSkoLMzMwO+4nkJOBnMbZPZO8nQ+8v42N3A97Yhcjr42FzCT3ksOvMAnQyfQ8Ao/INKM7SdczwSEhKTEL00UcfoaioSOwBBWCMoaamBkVFRVCpVFi4cGFcjCSSD4fbJyY/65A/2k+g0obZ5gYQWN4RvRAFZriUci64llkoAj0in0dIot+uZJFBo+xazIiEE5MQDRkyBOfOnUNOTk7Q/qamJpSUlIT070TCzp07Y3odkXia7UIqDoNGAWWYKfU0nRKZehUylR4wixecTB7TrFnbVfdhU9AGUKgBuVIQIncLoMiI+npE4onJWR0uGKylpSXiGTCidyPWF9OFL5KoUcpx+7RiXDPMvw5NrY8phijgz9GrI/zd7CSoEQAOVTfjkx/OUyxREhFVj2jZsmUAhLVEK1asgE7XmoTc5/Nh//79mDBhQlwNJJITsaxPJL6Wbs6Y8TygVcnDr7pvjyoFcDSH9RN9XWOCye7BsBxD5OJG9ChRvQuHDx8GIPSIjh49CpWq9ddQpVJh/PjxuP/+++NrIZGUXFBgRI5BE9EXmTlMYIxBFqMQjR1oxNiBxsgzK0YwhW+ye2jmLImISog+/fRTAMI0+7p16yiOpx9j1Cojmnk6UmPC6a+O4wJmQ0lJ9xLmyzpb7NqWLoIaxSl8iiVKGmLql27atCnedhB9FIWMg9xlgZPje6ZyRyi67BEJH3vqESUPEQvR9ddfj82bNyM1NRXXX399p223bdvWbcOI5IXnGQ7XNMOgUWJItr7TtBxpOiXqfC1wMl9MM2aMMbz8ZTW0SjmuHpPXYXFtSCINagyxNIWQhoiFyGg0ilOnRmOCftmIpMTm9mJ3WQNkHIffXD6007ZpWiXUXitcjIdPbUS08fIuL486f07scGECHegiqLE1upqGZslCxELUdjhGQ7P+TeALrNcouvTbpHBOKOGDDxysTIu0qK8l9Fp0KnkUQhTZ0KzFKZQWitj3RPQYMfmIHA6hmF5g+r6qqgpvvvkmRo8ejSuvvDKuBhLJR8TrvgBwDhPUSjmaeR3MLh5pUeZE67KEUCgCPSKPPWQSfb1agVt/VIRUjZJEKEmIKaBx/vz5ePHFFwEAJpMJU6ZMwdNPP4358+dj/fr1cTWQSD4Cs00RiYOjCWqFDE6FMWS6kK7osoRQKJS6TpPocxyHHIMGGiUtrE4WYs7QeMkllwAA3njjDeTl5aGqqgovvvgi/va3v8XVQCL5EHtE2gjEwdGMVI0SaZk5kTmaw1wrqlpkMpkgRkBYPxGRXMQ0NLPb7TAYhFiNDz/8ENdffz1kMhl+9KMfoaqqKq4GEslHoJeSGlGPqBkFaVoUDB0K5ESfqzqqa7VFlSL4iML4iSobbDjV0IKCNC1G5lE8nNTEnDz/rbfeQk1NDbZv3y76herq6ijIsR9gEX1EkQkRAEAbW2EEuYyDViWPyB8VRBdBjectTnxdY+464yOREGLqEa1YsQK33HIL7rvvPlxxxRWYNm0aAKF3NHHixLgaSCQfc8bkw+zwICdM5Q4RxkQhYpo0ON0+aJSyrlfQt+GqC/L8p4qy6kYXM2cB/xZN4ScHMQnRz372M8yYMQPnzp3D+PHjxf1XXHEFfvrTn8bNOCI5yTaoO9QxC4nHDnjd4AH875eNcPqacdfMIdCqoncSRyNeACIIahQ++laKrk4KYl56nJeXh7y8vKB9U6ZM6bZBRB/C3xuSaVKh8KgAnxdmhycmIYqaLoIaAz0iq9MLxlj0QkfElZiEyGazYc2aNdixYwfq6urA88EJzeNdCZZIHuqtLlQ32ZBj0KAwQ9d5Y3uj8KhNh1GhRItLEKI8Y2Q5q86aHPjo+/PIN2pw5QV5Xb+gLV0NzdQKyDgOPl6ocUZZG6UlJiH65S9/iV27duG2225Dfn4+/Zr0I04327G7rAHDcvVdC5GtQXhMyUaqS4kzJkdUsURmhwdNNjdSYskZ1EVyNJmMQ4paDqvTC6uThEhqYhKi999/H++++y6mT58eb3uIJCeqSOdAj0iXCaOstdxz9NeKRYjCV3wNkKpVikJESEtMQpSeno6MDMoF3B+JShxs/oq/KVkwcrEIkSfya7Un0CPyeQCvS8hl3Y6rx+RBJZdBrehWeT8iDsT0Dvzxj3/EihUrYLd3LwZj/fr1GDdunFgscdq0aXj//fe7dU6iZ2lx+cWhq+GS19VaZjolW6z0EUuPKOpgRkCo3iH3v66TTI0apZxcC0lATD2ip59+GuXl5cjNzUVxcTGUyuAPyqFDhyI6z8CBA7FmzRoMGzYMjDG88MILmD9/Pg4fPowLLrggFtOIHibioVnAP6RKAZRapOm8KMlKQZpOGfEsVbd6RIAQ1GhvEoZnOurBJzMxvcPXXXddXC4+b968oOePP/441q9fj3379pEQJSG8f4YJiGDtl73VUQ0AOpUC100cENX1LLGsvG+LKsUvRKF7RGaHBwermgAAl4/Mje0aRFyISYhWrlwZbzvg8/nw+uuvw2aziZHa7XG5XHC5XOJzi8USdzuI8LS4vWBMWHaR0lUskPW88KjPjulaHh+PNJ3SP6MVY4+oiyl8nmf4usYMlUKGWSNyaIgmITEHNJpMJrzxxhsoLy/HAw88gIyMDBw6dAi5ubkYMCDyX76jR49i2rRpcDqd0Ov1Yl6jUKxevRqrVq2K1WSim6SoFLh1ahHsbl/XX1rrWeHRUBC02+nxgTF0GdSolMtw69RB3TG31WHtsoY8HBA4t5eHy8tTWhAJiclZ/c0332D48OF44okn8NRTT8FkMgEQclUvX748qnONGDECR44cwf79+/HrX/8aCxcuxPfffx+y7fLly2E2m8WtpqYmFvOJGJHLOOSkalCcldJ5Q94HtPhnzAytgYifnajH+p3l+Mo/HOpxuoglUshlSFEL4kP5q6UlJiFatmwZFi1ahBMnTgRVdp07dy52794d1blUKhWGDh2KSZMmYfXq1Rg/fjzWrVsXsq1arRZn2AIbkYTY6gHeK0yZt1l1H6iBFkuCtJjoYmgGtF38SkIkJTEJ0YEDB/CrX/2qw/4BAwagtra2WwbxPB/kByKShxPnrThY1YR6axfvj+WM8GjIFzIl+gnUQYtEiD4/0YAX9lbi2zPmmO2NKKiRVuEnBTH5iNRqdUhHcVlZGbKzI3dOLl++HHPmzEFRURGsVitefvll7Ny5E9u3b4/FLKKHOVZrRXldCy4fKet89X2zPzleWlHQ7oAQmeyeLqfwG20uNNnc8EVa3TUUXSx8BVr9RDQ0k5aYekTXXnstHnvsMXg8wpvHcRyqq6vx0EMP4YYbboj4PHV1dbj99tsxYsQIXHHFFThw4AC2b9+OH//4x7GYRfQwEcX1MAaYqoW/04OdzYEyPgHncGcEhCGSarJhCfSIPA4hiX4IAjbZ3b7Yr0N0m5gDGn/2s58hOzsbDocDM2fORG1tLaZNm4bHH3884vM8//zzsVyekAirM4IYopY64YsvVwpDszYo5TLo1QpxFX64WSrGmDh8C0Rkx4RSB3AygPGAx9aatbENo/INGJlnoGUeEhOTEBmNRnz00UfYs2cPvv76a7S0tODCCy/E7Nmz420fkSR4fDwc/l5Dp0suGsqEx/RiQNZRaIza1nQguamh04HY3D54fAwc141gRkBIoq/SCcnR3KGFSK2gKftkIGoh4nkemzdvxrZt21BZWQmO41BSUoK8vDxKMNWHafH3hlSKLhaJ1v8gPGaPCHl4SI4e2QZ1p2IW6A0ZNMpOy1lHhCqlVYiIpCUqIWKM4dprr8V7772H8ePHY+zYsWCM4dixY1i0aBG2bduGt956q4dMJaREXNqhVoT/sTGfEdaYyeRA5rCQTSYN6jqJvsnuBtBN/1AAlR7A+bBBjYAwQ9doc2Hm8Gyk6VTdvyYRNVEJ0ebNm7F7927s2LEDs2bNCjr2ySef4LrrrsOLL76I22+/Pa5GEtJjicRRffpL4TFnNKCMLAtjKBQyGXJSI8yL3RVdBDUCQEWjDQ1WF8YNTCMhkoiohOiVV17BH/7whw4iBACXX345fv/732PLli0kRH0FWyPgaALkSgxNy0D21CIg3EjJVA3U+YdlAy/q9LROjw9Wpzes0IzIM2BEXkd/TkxEENSYqlGgweqiRPoSEpUQffPNN3jyySfDHp8zZw5Veu0L2JuAH94FzKfFXWoAOfocIKME8JQAxkJA7v/4tNQB370l/F0wATCEX8luc3nx3O5T4DjgnllDoZD38GxVBLFEYlCjg4IapSIqIWpqakJubvgPWW5uLpqbm7ttFCEhtkbgyEuA2y74elKyAZ9bEKeWOmGr3i+IkC5TeE1LnRA/pM8GBnfsLbdFp5JDrZTB5eHRZHcjxxA8hGOMgTEhp3RciChlLJUWkpqohMjn80GhCP8SuVwOr5d+VXotvA849m9BhAy5wJgbAI0RAPBVWQ20thoMUdRDY6kShjqBVB8AkD0cGH51l74hjuOQpVfjTLMDjS0dhcji8OKFLyqRpVfj51MKuz8LG5iy78RZTevNpCfqWbNFixZBrQ49tqc1Yr2cMwcFcVFqgLE3AWphWMMYw/4zLri9WVh48WRodEohMb7DJLwuJQvQpkV8mSy9ShSi9jTaXPDxDL54hYKIQtQiRFfLOg4FU9vUOCOkISohWrhwYZdtyFHdS/F5gep9wt+DZ4kiBABODw+3f0lGqkYhLGRNyRK2GMhMEX7IGm0df7iabG5/mzjNXqn0rdHV7hZA0zFjg5iXyMeD51n8hoVExEQlRJs2beopOwipOX9UGG5pUoG8sUGHAkMWvVoRF+dypl4QmYaQPSJhX0a8hEgmE0TVaRGGZyGESKeS49eXDYFaIaOAXImgBTaEQO23wuOASR2WZgQinQNO3e6SpRd6RBaHB05P8GLTOn+KkUCbuKD2i48rdGphjuOomofExOeTRfRuHM3CVD3HAbkdixYEVsLHVNYnBBqlHBOLOgYPur08GlsEIYq0LHVEROCwJqSFhIgA6o4Jj2mDQi4MNccjJUc7LhuR09EMqxOMCT4bfSxlpsMRGI45wxdb+KHWguO1VpRkpWDcwLT4XZuICBIiAmg8KTzmjAx5OOAjSo2jEIVCKZdhRJ4B2ngnse9iaAYAZrsHp+pt0CjlGDcwvpcnuoaEqL/jcQAWf8WNjMEhm8wZkw+Lw9N1LbMoMdndqGiwYdzANMhlHHJTNZg7Nr/rF0ZLBEIUEFmawpcGEqL+TnOlEBWdkiUGL7ZHo5THvdQOYwyvHqiB3e1Dll6NwgxdXM8fRERBjZQyVkpo1qy/03RKeAzTG+opOI7DoExh+cWJOivOW5yoszh75mIBIXLbhOjxEAR6RC0uLxjrRp5sIiZIiPo7Jn9tuPTikIcbW1z49Ie67lXTCMPofGHI9P1ZC947eg5b9lfjcHUPrFVUpQghCYyF7RXpVQrIOA6+NmW1icRBQtSfcbUIU/cAkBq6Om+d1YUjNSYcOxf/8t6FGVpkG9Tw+BhMdg8UMg5DcvRdvzBaOK7L4ZlMxok+MPITJR4Sov5MoP5YSlbYxarN/myJPZEwjOM4XD0mDxkpKqiVMvz4gty4xSp1IAKHtUGjgEoh6xBkSfQ8kjqrV69ejW3btuGHH36AVqvFxRdfjCeeeAIjRoTOd0zEmUC+IWP4+WqTXXDepnenmkYnZOnVWHhxcc/nO4/AYf3TiQOgkHEUYS0BkvaIdu3ahdLSUuzbtw8fffQRPB4PrrzySthslOg8IQR6RJ0IUaBHlB6vtV9h6PEvfwRBjUo5rTWTCkl7RB988EHQ882bNyMnJwcHDx7EpZdeKpFV/QTeB1j95cHD+IcYY216RL08l3MEQzNCOpIqjshsFmZmMjIyQh53uVxBOY9Clb0mIsTeJIiRQgVoQ1fWaHF54fbykHFcXJd3SEIgRsppCtukyebGZyfqIeM4zBtfkBi7CABJ5KzmeR733nsvpk+fjjFjxoRss3r1ahiNRnErLCxMsJV9CFud8JiSLcwqhSDQGzJqFd2vLyY1mjTh0WESpvFDwAE4VW9DdZOdYokSTNIIUWlpKb799lts3bo1bJvly5fDbDaLW01NTQIt7GO0BISo4+LTAIUZOvz6siG4dkLooVuvItAj8nkAjz1kEzFBmpeH08MnyjICSTI0u+eee/DOO+9g9+7dGDgwvONUrVaHTVNLRImtXnjUZ3farCeWd0iCXCHMnLmsgNPcmlS/DQq5DClqOWwuH6xOD7SqPnDfvQRJe0SMMdxzzz1488038cknn6CkpERKc/oXEfSI+hyBXlEg13YIUimRviRIKkSlpaV46aWX8PLLL8NgMKC2tha1tbVwOBxSmtX38Tha42lSQveIeJ7hzcOnsausXsxX3esJJPh3hl+u0lrRg6KrE4mkQrR+/XqYzWZcdtllyM/PF7dXX31VSrP6PoHekMYYNqLa7PCgssGOo6dNUPR2R3WAgMO6k5mzQDpcWoWfWCT1EdHMhESI/qHww7J6f8rWTL2671S1iGBoZtAooVLIQJ/MxJIUzmoiwQR6RJ0IUZ1FEKLseCaxl5oIhmbjBhgxfqCRIqwTTNJM3xMJxNa1o/qsSfDTxTWJvdSIQY1modhiCGS01kwSSIj6Gzzf5dDM6+NR609SNjBdmyjLeh6VwZ+XiKelHkkGCVF/w2kSqrrKFa3O23bUWpzw8QwpannvX9rRFpksoqUeH3x7Dv/aVyVWnSV6HhKi/kZLm6UdIerAA4DT44NOJcfAdF3fG6YE1tXZm8I2aWhxo8HqEssoET0POav7GxH4h4bmGDA4Sw+3r4/ED7VFlwE0lgOO8EJk0ChQb3XRFH4CISHqRzDGwEUwYwYITluNrA8ucdD6MzvYw+fGptJCiYeGZv0Es8ODVw/UwGHy5yBKyYbJ7sYZU3AUe53FCZ7vw1E0uoAQNYZtEljmQUOzxEFC1A/w+Hj85+uzqGu2oPqMUEzRq83Cu0fP4Y2vTmP/qUbwPENDiwtbD9Rg456KvlvJQpcpPDrNYUsLBdLiBrJTEj0PDc36AYeqmtFgdSEbZhRn6gC1AbxCiwydCnUWF/aWN+L7cxY4PD74eIZsgzq+teeTCZUekCuFdCCOZqFwQDsC2SjNDk/P59ImAFCPqM9jc3lxoFJwzM7I90GtkAP6HKgUMlw9Jg9XXZAHlUIGk90Dl4dHtkGNK0fnSWx1D8JxbYZnoR3WqVphmUeqVglXX1nwm+T00Z89IsDXNSZ4fAx5Rg0Gqf1FCfwr7jmOw+iCVAzOTkF1kx0KmVB9tddnY+wKXSZgPR925kwu43D3ZUOoJ5RASIj6MF4fj6P+Cq2TB6WDOxM6olqjlGN4riHR5kmHtmuHNYlQYqGhWR+mvN4Gu9sHg0aBIVkprUs7+lMytFAEHNadBDUSiYV6RH2Yogwdfjw6F3IZB5nbDHjdwlorXegqKf0GUYgahET6IXo/lQ02fHayAZkpKswdm59gA/sfJER9GK1KjjED/Gur6suER12mIEb9GV0mwMkAj1PIVBkovtgGjgMarC7KmZUgaGjWX7BFFlHdL5ArWnuFgeFqO9L8U/gmu6dvB3gmCSREfZTD1c04WNXcGpjYH5Pld0YgV3fg/9KOVI0CChkHH89oqUcCICHqgzDGcLCqGbvL6lHnzysUSXrYfkXg/2ALLUQcxyHNH2HdRBHWPQ4JUR+kzuqC1emFSiFDUYauNYoYICEKEOgZhhmaAUB6ijA8a7K5wrYh4oOkQrR7927MmzcPBQUF4DgOb731lpTm9BnK61sAAIMydVDIZcKXjTFApQtZWLBfEigsaWsMu+YsM0XI193QQj2inkZSIbLZbBg/fjyeffZZKc3oc5TXCxHUg7P0wo6W88Kjvg8v3YgWdSqgUAtpY8MENmYb1MjUq/ruurskQtL/8Jw5czBnzhwpTehzmO0eNFhdkHEcBmf7ez9iDqLOy0v3KzhOGKaaagBrbcgh69AcPYbm6CUwrv/Rq3xELpcLFoslaCOCKW8QhmUFaZrWmvVijyhXIquSFIM/UNF6Tlo7iN4lRKtXr4bRaBS3wsJCqU1KOpxuHxQyDkMCv+SMtekRkRAFkVogPFrOdtqMMQZvX0ybm0T0KiFavnw5zGazuNXU1EhtUtJx8dAs/GrmEFxQ4I8WdjQLs2ZyRetiT0LA4PeZ2eqFyiYh+KK8Ef/YWY5D1abE2dUP6VVeOLVaDbW6D1Ue7SFUija/L4FhWSdVO/otmjRAqQU8DiGeKNBDaoNCzsHt5dHYQlP4PQl9MvsQDneIaWjyD4WH49oMz0L7iTL9sUT1JEQ9iqQ9opaWFpw8eVJ8XlFRgSNHjiAjIwNFRUUSWtb78Ph4bNxTAaNWiZ9OHICUwJRzhFU7+i2GfKG8kOUMgEkdDuekCiW3m2xuuL18cG+TiBuS/le/+uorTJw4ERMnTgQALFu2DBMnTsSKFSukNKtXUtNkh9vLi8URAfgd1YGhGQlRSIwDhEfz6ZCH9WoFDBoFGKNeUU8iaY/osssuozQLceJUIIgxO6U1u6DLArhahJQXBgpmDEnqQOH/4zQDDhOgTevQJCdVA6uzBectTgxI0ybcxP4A9TP7AIwxnPLHDw3JbhOAF/B76LOFyhVERxSqVpE2VYdskmsQJkjEBcRE3CEh6gPUWpywuXxQKWQYmK5rPWA5IzwaOs4GEW1I8/sjzaHDQQrStBicnYJ8I/WGeopeNX1PhObEeaE3NDirXQWOQMRwiGlpog1pRUD1vrA9osIMHQozdCGPEfGBekS9HMYYTtYJQhS0Lor3kRBFirFQ8BM5TK3pUoiEQkLUB7hkWBZG5RswKLNNio9AtLBC3ZosngiNQgUYBwp/N54K26zF5UWdlfxEPQEJUS+H4zgMyzXg6jH5wTEugeno1IKQVSqIdmQOFR4bT4Y8fOK8FRt2n8Inx0JndCS6BwlRX6W5UnhMGySpGb2GgBCZqoSyS+3INQqBjectLripDHXcISHqxTS0uLD3ZAOabe2+ODzf6nhNJyGKCF0GoE0XfGsBEW9DqkaJVK0SPGM4a3Ik3r4+DglRL+a7sxbsr2jC3vJ2GQZtdYDXJfg+KCtjZHBca6+o/oeQTYr8M2cVjbZEWdVvICHqpfh4hh/OCYnhRuW3q1vfXCU8GotoxX005IwSHhuOhxyeBTJenqq30YqAOEOf0l5KVaNQ116nkgfPlgGtDtf04oTb1atJLRCGZz4v0FDW4XBRhg4KGQeLw0MJ9eMMCVEv5evTJgDAyPzU4CBGt711xixraOIN681wHJB7gfD3+W87HFbKZSjKFIZnp/yVUoj4QELUC2myuVHZYAfHARMGprU7WC5UptBnC7/uRHTkjREEqalCKDXUjgmFabh6TB4uHET/23hCQtQLOVwtRP8OztbDqGu3mLX+uPCYNTzBVvURtOmtTuszBzscHpSZglH5qVDK6asTT+i/2QtJUSugUshwYVFa8AFXi5DkCwByRifcrj7DAH+CtNpvAHf4GTJyWMcPEqJeyI8GZ2LpjJLglfaA4NdgvOB0TcmSxri+QHoxYMgVig5UfdHhMGMMR2pM2LSnEma7J/H29UFIiHopYs2yADwPnD0s/J0/PvEG9SU4Dhh8mfD32UMdFsJyHIdT9S0wOzzYXxG6SiwRHSREvQQfz/DON2dxutkeukHd98LqcZWOhmXxIL1E6BnxPuD4B0La3Tb8aLCwkPi7sxacoUjrbkNC1EvYebwOJ8634N1vzsHTvtifzwNU7Bb+HniREFFNdA+OA4ZfJdSDa64ETn8VdLggTSvWjtv+bW3oCipExJAQ9QIOVjXhm9NmcBxw5QV5HWdsKnYLOZc1qcCAydIY2RfRZQCDZwl/l+8AGoJX5l86PBtGrRJmhwdvHTkDuzt0kUaia5JCiJ599lkUFxdDo9Fg6tSp+PLLL6U2KSnw8Qyfn2jA7rIGAMD0oVkoyWoXRX3+O6DG//8adiX1huLNgElA/jhhaPbdNuH/7UejlOPaCQXQKOWoNTvxypc1VJo6RiQXoldffRXLli3DypUrcejQIYwfPx5XXXUV6ur6d96XmiY7th6oxoHKJgDAxUMycVFxm5LRPA9U7weOvS08L7wIyBomgaV9HI4Dhl8NZI8Q/EXf/0fYHCYAQJZejQUXFSIjRYXR+alQ+HurVqcHn52oxzenTahosKHO4oTV6YGPpyn/UHBM4mCIqVOn4qKLLsLf//53AADP8ygsLMRvfvMb/P73v+/0tRaLBUajEWazGampqYkwN+7wPIPT64PN5YNGKYNBIwQoHq5uxs7j9VArZZg9KhfDc/SA1ynM4DRXCTEudkGkkD9e+LLQAteeg+eBil1AzX6hd8TJgIwSIGMIYMiFV50GKLRQKITZzMoGG948fCbkqbQqOWYMzcKYAUYAgmgdr7VCp1IgRS2HViVHikoBrVIOmax3J7WL9DsqafJ8t9uNgwcPYvny5eI+mUyG2bNn44svOsZvuFwuuFytRe4sFkvE1zpjcuCTY0KxwYDytpXgKSUZGJUv/KNqzU68/+25oDZt1XryoHSML0wTbPhyC46W17Q5GS++gAEoTNdgUEYKAAa72ytERTOhAQPg43nxtcWZOhgydAAYxvEM6XYbCtK0UJ3ggONeYTFmWxRqYMjlghBRFsaeRSYDhswSItYrPxOWgDSWiwGkCkB4DxRqQKZElpvHNQ4XnF4Gp5fB7WXw+Hjw/vc6w50KnBGSrbltbphrTDC3uyTHCevbhmbrke9PzGZyePDDOav4dp8o/BkY1xrKMWlQBkb7neh1Vie2f1sb9pYmFKZj7EBBDJtsbrx7NHTZbQAYU5CKiUXCshaL0wOlTAatSh62fbRIKkQNDQ3w+XzIzQ2uy56bm4sffuiYE2b16tVYtWpVTNfyePlOV0w7Pa2zHl6eh6mTQDV3Wz+AvREye0PYtjKHFnAI1+U8PshcweIZeCsVMg6cmwFuJu4vTpUBvEvUNgDC9HzqAOHXOHeM8MEnEodxADD+ZsDWIKzQN9UI+cFdVuEHxeME4IQewPB22VkYAK9P+PyoOAtgExbOql1eDFI74fHxcPt4eHw8vD4m/vjJ7E5AIQgRHG4wm1U81tDiDhIiR9vPsY91+plv61z38jwarOEr2drbzAryPAOTxXcg1avKCS1fvhzLli0Tn1ssFhQWFkb02txUDW64UEiQHqrzkNZmzVa2QRj3Bwi05yD8ode0/tu04+Zj2GBHm0ac/0EYJmmVCkAlBzgOSp5h5FgvAE5ox3FQyzmolQr/CnqujXHB5wMnA9QGKpSYLKRkCVsgASbvAzwOYWM+IcK97QbhHVX6t7boAYh5Evw9Jp4JP45Orw86pQJQ+j9PXh7FdrfYAR+YWhT0gW679jAjRYWfTRoY9hZSNa1tjVql+P0I2Vbb+plPUSsgi3MPXFIhysrKglwux/nz54P2nz9/Hnl5HTMLqtVqqNWx9QC0KrmYwqEr1Ao5CiIsLazMGISsjK7bAcI/m9Zs91FkckCtF7Z4nA6Azr+1RQ0gP8JzaJTyiOuxqRWRfz96YsGvpN5NlUqFSZMmYceOHeI+nuexY8cOTJs2TULLCIJIJJIPzZYtW4aFCxdi8uTJmDJlCtauXQubzYbFixdLbRpBEAlCciFasGAB6uvrsWLFCtTW1mLChAn44IMPOjiwCYLou0geR9Qd+kIcEUH0ZSL9jlIEHEEQkkNCRBCE5JAQEQQhOZI7q7tDwL0VzVIPgiASR+C72ZUrulcLkdVqBYCIo6sJgpAGq9UKo9EY9nivnjXjeR5nz56FwWAA18sXfQaWq9TU1PSZGUC6p+Snp++HMQar1YqCggLIOskO0at7RDKZDAMHhl8f0xtJTU3tEx/wttA9JT89eT+d9YQCkLOaIAjJISEiCEJySIiSBLVajZUrV8acXSAZoXtKfpLlfnq1s5ogiL4B9YgIgpAcEiKCICSHhIggCMkhISIIQnJIiBJINBVtN2zYgEsuuQTp6elIT0/H7Nmzk7ICbqxVerdu3QqO43Ddddf1rIFREu39mEwmlJaWIj8/H2q1GsOHD8d7772XIGsjI9p7Wrt2LUaMGAGtVovCwkLcd999cDqdPWskIxLC1q1bmUqlYhs3bmTfffcdu+OOO1haWho7f/58yPa33HILe/bZZ9nhw4fZsWPH2KJFi5jRaGSnT59OsOXhifaeAlRUVLABAwawSy65hM2fPz8xxkZAtPfjcrnY5MmT2dy5c9nnn3/OKioq2M6dO9mRI0cSbHl4or2nLVu2MLVazbZs2cIqKirY9u3bWX5+Prvvvvt61E4SogQxZcoUVlpaKj73+XysoKCArV69OqLXe71eZjAY2AsvvNBTJkZNLPfk9XrZxRdfzP75z3+yhQsXJpUQRXs/69evZ4MHD2ZutztRJkZNtPdUWlrKLr/88qB9y5YtY9OnT+9RO2lolgACFW1nz54t7uusom0o7HY7PB4PMjIirF3Uw8R6T4899hhycnKwdOnSRJgZMbHcz3/+8x9MmzYNpaWlyM3NxZgxY/DnP/8ZPp8vZPtEE8s9XXzxxTh48KA4fDt16hTee+89zJ07t0dt7dWLXnsL0Va0DcVDDz2EgoKCoA+VlMRyT59//jmef/55HDlyJAEWRkcs93Pq1Cl88sknuPXWW/Hee+/h5MmTuPvuu+HxeLBy5cpEmN0psdzTLbfcgoaGBsyYMQOMMXi9Xtx11134wx/+0KO2Uo+oF7BmzRps3boVb775JjQajdTmxITVasVtt92GDRs2ICsrS2pz4gLP88jJycFzzz2HSZMmYcGCBXj44YfxP//zP1KbFjM7d+7En//8Z/zjH//AoUOHsG3bNrz77rv44x//2KPXpR5RAoi2om1bnnrqKaxZswYff/wxxo0b15NmRkW091ReXo7KykrMmzdP3MfzQilmhUKB48ePY8iQIT1rdCfE8h7l5+dDqVRCLm+tPT9q1CjU1tbC7XZDpVL1qM1dEcs9PfLII7jtttvwy1/+EgAwduxY2Gw23HnnnXj44Yc7zSnUHahHlABirWj75JNP4o9//CM++OADTJ48ORGmRky09zRy5EgcPXoUR44cEbdrr70Ws2bNwpEjRyTPshnLezR9+nScPHlSFFQAKCsrQ35+vuQiBMR2T3a7vYPYBISW9eSy1B51hRMiW7duZWq1mm3evJl9//337M4772RpaWmstraWMcbYbbfdxn7/+9+L7desWcNUKhV744032Llz58TNarVKdQsdiPae2pNss2bR3k91dTUzGAzsnnvuYcePH2fvvPMOy8nJYX/605+kuoUORHtPK1euZAaDgb3yyivs1KlT7MMPP2RDhgxhN910U4/aSUKUQP77v/+bFRUVMZVKxaZMmcL27dsnHps5cyZbuHCh+HzQoEEMQIdt5cqViTe8E6K5p/YkmxAxFv397N27l02dOpWp1Wo2ePBg9vjjjzOv15tgqzsnmnvyeDzs0UcfZUOGDGEajYYVFhayu+++mzU3N/eojZQGhCAIySEfEUEQkkNCRBCE5JAQEQQhOSREBEFIDgkRQRCSQ0JEEITkkBARBCE5JEREr2Lz5s1IS0sTnz/66KOYMGGC+HzRokVJl/WR6BoSIiIkixYtAsdxuOuuuzocKy0tBcdxWLRoUVD7eAtAcXEx1q5dG7RvwYIFKCsrC/uadevWYfPmzeLzyy67DPfee29c7SLiDwkREZbCwkJs3boVDodD3Od0OvHyyy+jqKhIEpu0Wi1ycnLCHjcajUE9JqJ3QEJEhOXCCy9EYWEhtm3bJu7btm0bioqKMHHixG6dO1RP5brrrhN7WZdddhmqqqpw3333geM4cBwHoOPQrD1te2aLFi3Crl27sG7dOvEcFRUVGDp0KJ566qmg1x05cgQcx+HkyZPdui8iNkiIiE5ZsmQJNm3aJD7fuHEjFi9e3OPX3bZtGwYOHIjHHnsM586dw7lz56I+x7p16zBt2jTccccd4jmKioo63BMAbNq0CZdeeimGDh0ar1sgooCEiOiUX/ziF/j8889RVVWFqqoq7NmzB7/4xS96/LoZGRmQy+UwGAzIy8vrMoFcKIxGI1QqFXQ6nXgOuVyORYsW4fjx42JeZo/Hg5dffhlLliyJ920QEUIZGolOyc7OxjXXXIPNmzeDMYZrrrmm16d6LSgowDXXXIONGzdiypQpePvtt+FyuXDjjTdKbVq/hXpERJcsWbIEmzdvxgsvvBC3XoNMJuuQ8c/j8cTl3JHwy1/+UnTEb9q0CQsWLIBOp0vY9YlgSIiILrn66qvhdrvh8Xhw1VVXxeWc2dnZQX4fn8+Hb7/9NqiNSqXqdmmecOeYO3cuUlJSsH79enzwwQc0LJMYGpoRXSKXy3Hs2DHx73CYzeYOpYIyMzND5qO+/PLLsWzZMrz77rsYMmQInnnmGZhMpqA2xcXF2L17N26++Wao1eqYhoTFxcXYv38/KisrodfrkZGRAZlMJvqKli9fjmHDhnWaO5zoeahHREREamoqUlNTO22zc+dOTJw4MWhbtWpVyLZLlizBwoULcfvtt2PmzJkYPHgwZs2aFdTmscceQ2VlJYYMGYLs7OyY7L7//vshl8sxevRoZGdno7q6Wjy2dOlSuN3uhMwCEp1DqWKJfstnn32GK664AjU1NR2KEBKJhYSI6He4XC7U19dj4cKFyMvLw5YtW6Q2qd9DQzOi3/HKK69g0KBBMJlMePLJJ6U2hwD1iAiCSAKoR0QQhOSQEBEEITkkRARBSA4JEUEQkkNCRBCE5JAQEQQhOSREBEFIDgkRQRCSQ0JEEITk/H/RiuOi+qCvigAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:23:58.394609Z", + "iopub.status.busy": "2024-02-29T06:23:58.394307Z", + "iopub.status.idle": "2024-02-29T06:23:58.595978Z", + "shell.execute_reply": "2024-02-29T06:23:58.595110Z" + }, + "papermill": { + "duration": 0.223266, + "end_time": "2024-02-29T06:23:58.597895", + "exception": false, + "start_time": "2024-02-29T06:23:58.374629", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAm60lEQVR4nO3de1xUdf4/8BfXgXFARC4igaOSgJWQsCKubbormqVZ7q/YNEFy3Q1l12ItozbItY3toqKtya4bknbRaq31UeZlUVtvpEIYKpcFESgBRUUEdBiYz/cPf5x2nAEGBWbOzOv5ePDA8zmXeZ85w8sznznnM3ZCCAEiIpmxN3cBRES3guFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyZKjuQvobzqdDufOnYObmxvs7OzMXQ4R3UQIgatXr2Lo0KGwt+/8/MrmwuvcuXMICAgwdxlE1I3q6mrccccdnc63ufByc3MDcOOJcXd3N3M1fUer1WL37t2YOnUqnJyczF0O9QJbOaaNjY0ICAiQ/lY7Y3Ph1fFW0d3d3erDS6lUwt3d3apf6LbE1o5pd9067LAnIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEcmSzV0qYa1aWlpQXFwsTTdd0+BwYTkGeR2HylWht2xISAiUSmV/l0jUqxheVqK4uBgREREG7W8YWTYvLw9jx47t+6KI+hDDy0qEhIQgLy9Pmi6paUDyJ4VY9dg9CPbzMFiWSO4YXlZCqVTqnU3ZV16E4sA1hN4dhvBhg81YGVHfYIc9EckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZMns4bVu3Tqo1Wq4uLggKioKR48e7XL5hoYGLF68GH5+flAoFBg1ahR27NjRT9USkaUw6xj2W7duRXJyMjIzMxEVFYWMjAxMmzYNJSUl8PHxMVi+tbUVMTEx8PHxwaeffgp/f39UVlbCw8Oj/4snIrMya3itWrUKCxcuREJCAgAgMzMTX375JbKysvDCCy8YLJ+VlYVLly7h8OHDcHJyAgCo1er+LJmILITZwqu1tRV5eXlISUmR2uzt7TFlyhQcOXLE6Drbt29HdHQ0Fi9ejH/961/w9vbGnDlzsGzZMjg4OBhdR6PRQKPRSNONjY0AAK1WC61W24t7ZFna2tqk39a8n7ak4zha+/E0df/MFl719fVob2+Hr6+vXruvr6/eNz//rzNnzmDv3r2YO3cuduzYgbKyMixatAharRZpaWlG10lPT8fy5csN2nfv3m3V3xpd3QQAjsjNzcUPJ81dDfWmPXv2mLuEPtXS0mLScrL63kadTgcfHx/8/e9/h4ODAyIiIvDDDz/gzTff7DS8UlJSkJycLE03NjYiICAAU6dOhbu7e3+V3u9OVF0CCo9j/PjxCAv0NHc51Au0Wi327NmDmJgYqdvEGnW8O+qO2cLLy8sLDg4OqKur02uvq6vDkCFDjK7j5+cHJycnvbeIoaGhqK2tRWtrK5ydnQ3WUSgUUCgUBu1OTk5W/QJwdHSUflvzftoia3/tmrpvZrtUwtnZGREREcjJyZHadDodcnJyEB0dbXSdn/70pygrK4NOp5PaSktL4efnZzS4iMh6mfU6r+TkZGzYsAHvvfceioqKkJiYiObmZunTx7i4OL0O/cTERFy6dAlLlixBaWkpvvzyS7z22mtYvHixuXaBiMzErH1esbGxuHDhAlJTU1FbW4vw8HDs3LlT6sSvqqqCvf2P+RoQEIBdu3bh2WefxZgxY+Dv748lS5Zg2bJl5toFIjITs3fYJyUlISkpyei8/fv3G7RFR0cjNze3j6siIktn9tuDiIhuBcOLiGTJ7G8bichQS0uLwcXaTdc0OFxYjkFex6Fy/fHyn5CQEKu+4LozDC8iC1RcXIyIiAij8964aTovLw9jx47t+6IsDMOLyAKFhIQgLy9Pr62kpgHJnxRi1WP3INjPQ29ZW8TwIrJASqXS4GzKvvIiFAeuIfTuMIQPG2ymyiwHO+yJSJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsOZq7ALp1FfXNaNa0GZ1XfqFZ+u3o2PlhHqBwxHCvAX1SH1FfYnjJVEV9Mya/tb/b5f7waWG3y+xbOokBRrLD8JKpjjOujNhwBPmoDOdf0+CL/UcwY1I0BrgqjG6j7HwTntla0OnZG5ElY3jJXJCPCnf7DzRo12q1qPUGxg4bBCcnJzNURtS3etxhf+bMmV4vYt26dVCr1XBxcUFUVBSOHj3a6bLZ2dmws7PT+3Fxcen1mojIsvU4vIKCgjB58mS8//77uH79+m0XsHXrViQnJyMtLQ35+fkICwvDtGnTcP78+U7XcXd3R01NjfRTWVl523UQkbz0OLzy8/MxZswYJCcnY8iQIfjtb3/b5ZlSd1atWoWFCxciISEBo0ePRmZmJpRKJbKysjpdx87ODkOGDJF+fH19b/nxiUieetznFR4ejjVr1mDlypXYvn07srOzMXHiRIwaNQpPPfUU5s2bB29vb5O21drairy8PKSkpEht9vb2mDJlCo4cOdLpek1NTRg2bBh0Oh3Gjh2L1157DXfddZfRZTUaDTQajTTd2NgI4EafkFarNalOS9TW1ib9NrYfHW1d7WN32yDLYivHy9R9u+UOe0dHR8yePRsPPfQQ3nnnHaSkpGDp0qV48cUX8fjjj+P111+Hn59fl9uor69He3u7wZmTr68viouLja4THByMrKwsjBkzBleuXMFbb72FCRMm4NSpU7jjjjsMlk9PT8fy5csN2nfv3g2lUtmDPbYs1U0A4IiDBw+i0vDDRsmePXtuextkGTqOV25uLn44ae5q+k5LS4tJy91yeB0/fhxZWVnYsmULBgwYgKVLl2LBggX4/vvvsXz5csyaNeu23k52Jjo6GtHR0dL0hAkTEBoair/97W9YsWKFwfIpKSlITk6WphsbGxEQEICpU6fC3d291+vrL6fONeKtwlxMnDgRdw013A+tVos9e/YgJiam008bu9sGWZYTVZeAwuMYP348wgI9zV1On+l4d9SdHofXqlWrsHHjRpSUlODBBx/Epk2b8OCDD8Le/kb32fDhw5GdnQ21Wt3ttry8vODg4IC6ujq99rq6OgwZMsSkepycnHDvvfeirKzM6HyFQgGFwvA6JycnJ1lfQtBx1byjo2OX+9HVfpq6DbIMtnK8TN23HnfYr1+/HnPmzEFlZSU+//xzzJgxQwquDj4+Pnj33Xe73ZazszMiIiKQk5Mjtel0OuTk5OidXXWlvb0dhYWF3b5FJSLr0uMzrz179iAwMNAgsIQQqK6uRmBgIJydnREfH2/S9pKTkxEfH4/IyEiMGzcOGRkZaG5uRkJCAgAgLi4O/v7+SE9PBwD86U9/wvjx4xEUFISGhga8+eabqKysxK9//eue7goRyViPw2vkyJGoqamBj4+PXvulS5cwfPhwtLe392h7sbGxuHDhAlJTU1FbW4vw8HDs3LlT6sSvqqrSC8rLly9j4cKFqK2txaBBgxAREYHDhw9j9OjRPd0VIpKxHoeXEMJoe1NT0y1f6Z6UlISkpCSj8/bv3683vXr1aqxevfqWHoeIrIfJ4dXxiZ2dnR1SU1P1LjNob2/HN998g/Dw8F4vkIjIGJPD69tvvwVw48yrsLAQzs7O0jxnZ2eEhYVh6dKlvV8hkY3oanw2wLQx2mxpfDaTw2vfvn0AgISEBKxZs0bW10gRWRpTx2cDuh+jzVbGZ+txn9fGjRv7og4im9bd+GxA92O02dr4bCaF1+zZs5GdnQ13d3fMnj27y2W3bdvWK4UR2aLOxmcDOEbbzUwKr4EDB8LOzk76N5mfpv067F1+QEVjCexdDP+nbmtrw7m2cyi6VNRp/0hFYxPsXX6Apv06AB5XkheTwut/3yrybaNlONdciQHD38aL3dw++s7Od7qcP2A4cK45HBHgsEIkLxwGWqaGDhiG5orfYU1sOEYa6SNpa2vDoYOH8NOJP+30zKv8fBOWbC3A0MnD+rpcol5nUnjde++90tvG7uTn599WQWQahYMLdNf9Mdw9GKMHGx/DvsKxAqGeoZ32j+iuX4Hu+gUoHDiMNsmPSeH1yCOP9HEZREQ9Y1J4paWl9XUdREQ90uMhcYiILIFJZ16enp4oLS2Fl5cXBg0a1GX/16VLl3qtOCKizpgUXqtXr4abm5v0b1M774mI+opJ4fW/AwvOnz+/r2ohIjJZj/u8HBwcjH4h7MWLF+Hg4NArRRERdafH4dXZYIQajUZvmBwior5k8hX2a9euBXBjMMJ//OMfUKl+vKq7vb0d//nPfxASEtL7FRIRGWFyeHUMvSyEQGZmpt5bRGdnZ6jVamRmZvZ+hUQ2oLsb7YHub7a3tRvtTQ6viooKAMDkyZOxbds2DBo0qM+KIrI1pt5oD3R9s70t3Wjf4xuzO0ZUJaLe092N9kD3N9vb2o32PQ6vp556qsv5WVlZt1wMka3q7kZ7oPub7W3tRvseh9fly5f1prVaLU6ePImGhgb8/Oc/77XCiIi60uPw+uyzzwzadDodEhMTMXLkyF4pioioO71yY7a9vT2Sk5P5ZbBE1G96bVSJ8vJytLXZxreWEJH59fhtY8c3Z3cQQqCmpgZffvml3j2QRER9qcfh1fHN2R3s7e3h7e2NlStXdvtJJBFRb+F1XkQkSxxJlYhkieFFRLLE8CIiWWJ4EZEs9Vp4ff/99/jNb37TW5sjIupSr4XXxYsX8e677/bW5oiIusS3jUQkSwwvIpIlhhcRyZLJV9jPnj27y/kNDQ23WwsRkclMDq+BA7se0H/gwIGIi4u77YKIiExhcnht3LixL+sgIuoRi+jzWrduHdRqNVxcXBAVFYWjR034ChUAW7ZsgZ2dHR555JG+LZCILI7JZ16mDnfT0y/g2Lp1K5KTk5GZmYmoqChkZGRg2rRpKCkpgY+PT6frnT17FkuXLsV9993Xo8cjIutg8plXdnY29u3bh4aGBly+fLnTn55atWoVFi5ciISEBIwePRqZmZlQKpVdhmB7ezvmzp2L5cuXY8SIET1+TCKSP5PPvBITE/HRRx+hoqICCQkJePLJJ+Hp6XlbD97a2oq8vDykpKRIbfb29pgyZQqOHDnS6Xp/+tOf4OPjgwULFuDAgQNdPoZGo4FGo5GmGxsbAdz41iOtVntb9ZtTx5DbbW1tRvejo62rfexuG9R/TDkW3R1TazmeptZucnitW7cOq1atwrZt25CVlYWUlBQ89NBDWLBgAaZOnQo7O7seF1lfX4/29nb4+up/u6+vry+Ki4uNrnPw4EG8++67KCgoMOkx0tPTsXz5coP23bt3Q6lU9rhmS1HdBACOOHjwICqNf0cpAGDPnj23vQ3qez05Fp0dU2s5ni0tLSYt16ORVBUKBZ544gk88cQTqKysRHZ2NhYtWoS2tjacOnUKKlXfPmNXr17FvHnzsGHDBnh5eZm0TkpKit64+42NjQgICMDUqVPh7u7eV6X2uVPnGvFWYS4mTpyIu4Ya7odWq8WePXsQExNj9AtKTdkG9R9TjkV3x9RajmfHu6Pu9HgY6A729vaws7ODEALt7e23tA0vLy84ODigrq5Or72urg5DhgwxWL68vBxnz57FzJkzpTadTgcAcHR0RElJicF3RyoUCigUCoNtOTk5dfpHLQcdX/fu6OjY5X50tZ+mboP6Xk+ORWfH1FqOp6m19+hSCY1Gg48++ggxMTEYNWoUCgsL8de//hVVVVW3dNbl7OyMiIgI5OTkSG06nQ45OTmIjo42WD4kJASFhYUoKCiQfh5++GFMnjwZBQUFCAgI6HENRCRPJp95LVq0CFu2bEFAQACeeuopfPTRRya/detKcnIy4uPjERkZiXHjxiEjIwPNzc1ISEgAAMTFxcHf3x/p6elwcXHB3Xffrbe+h4cHABi0E5F1Mzm8MjMzERgYiBEjRuDrr7/G119/bXS5bdu29aiA2NhYXLhwAampqaitrUV4eDh27twpdeJXVVXB3t4irqUl6jPXtDe6Xk7+cKXTZZqvaXD8AjCk8jIGuBp2hZSdb+qz+iyRyeEVFxd3S58omiIpKQlJSUlG5+3fv7/LdbOzs3u/IKJ+Vv7/g+eFbYXdLOmIzWXHulxigOKWu7JlxeS9ZEgQ9Z2pd934gGqkjwquTg5GlympuYI/fFqIlf/vHgT7GR8oYYDCEcO9BvRZnZbENiKayMJ5DnDGr8YFdrlMx0WoI70H4G7/rkd5sQUML5nqro+ku/4RwPb6SMi6MLxkyrQ+ku77RwDb6SMh68JXrUx110diSv8IYFt9JGRdGF4y1V0fCftHyNrxAioikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEsmQR4bVu3Tqo1Wq4uLggKioKR48e7XTZbdu2ITIyEh4eHhgwYADCw8OxefPmfqyWiCyB2cNr69atSE5ORlpaGvLz8xEWFoZp06bh/PnzRpf39PTESy+9hCNHjuC7775DQkICEhISsGvXrn6unIjMyezhtWrVKixcuBAJCQkYPXo0MjMzoVQqkZWVZXT5SZMm4dFHH0VoaChGjhyJJUuWYMyYMTh48GA/V05E5uRozgdvbW1FXl4eUlJSpDZ7e3tMmTIFR44c6XZ9IQT27t2LkpISvP7660aX0Wg00Gg00nRjYyMAQKvVQqvV3uYeWK62tjbptzXvpy2xlWNq6r6ZNbzq6+vR3t4OX19fvXZfX18UFxd3ut6VK1fg7+8PjUYDBwcHvPPOO4iJiTG6bHp6OpYvX27Qvnv3biiVytvbAQtW3QQAjsjNzcUPJ81dDfUGWzmmLS0tJi1n1vC6VW5ubigoKEBTUxNycnKQnJyMESNGYNKkSQbLpqSkIDk5WZpubGxEQEAApk6dCnd3936sun+dqLoEFB7H+PHjERboae5yqBfYyjHteHfUHbOGl5eXFxwcHFBXV6fXXldXhyFDhnS6nr29PYKCggAA4eHhKCoqQnp6utHwUigUUCgUBu1OTk5wcnK6vR2wYI6OjtJva95PW2Irx9TUfTNrh72zszMiIiKQk5Mjtel0OuTk5CA6Otrk7eh0Or1+LSKyfmZ/25icnIz4+HhERkZi3LhxyMjIQHNzMxISEgAAcXFx8Pf3R3p6OoAbfViRkZEYOXIkNBoNduzYgc2bN2P9+vXm3A0i6mdmD6/Y2FhcuHABqampqK2tRXh4OHbu3Cl14ldVVcHe/scTxObmZixatAjff/89XF1dERISgvfffx+xsbHm2gUiMgOzhxcAJCUlISkpyei8/fv3602/+uqrePXVV/uhKiKyZGa/SJWI6FZYxJkXEelraWkxuNaxpKYBmtoyFJ10he6ih9QeEhJi1dcsdobhRWSBiouLERERYXTenPf0p/Py8jB27Nh+qMqyMLyILFBISAjy8vL02pquafDlviN4aHI0VK4KvWVtEcOLyAIplUqDsymtVovL9ecRPS7Sqi9SNRU77IlIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssQhcazEzSNvdjbqJmC7I2+SdWF4WYnORt68edRNwHZH3iTrwvCyEjePvNnZqJsdyxLJHcPLStw88iZH3SRrxw57IpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJZs7jovIQQAoLGx0cyV9C2tVouWlhY0NjbyOi8rYSvHtONvs+NvtTM2F15Xr14FAAQEBJi5EiLqytWrVzFw4MBO59uJ7uLNyuh0Opw7dw5ubm6ws7Mzdzl9prGxEQEBAaiuroa7u7u5y6FeYCvHVAiBq1evYujQobC377xny+bOvOzt7XHHHXeYu4x+4+7ubtUvdFtkC8e0qzOuDuywJyJZYngRkSwxvKyUQqFAWloaFApF9wuTLPCY6rO5Dnsisg488yIiWWJ4EZEsMbyISJYYXt2YP38+HnnkkV7d5qRJk/DMM890uYxarUZGRkavPi6RNbGp8DIlNEheXnnlFYSHh5u7jE5Z2mvO0uq5HTYVXkRy1Nraau4SLJLNhNf8+fPx9ddfY82aNbCzs4OdnR3Ky8uxYMECDB8+HK6urggODsaaNWuMrr98+XJ4e3vD3d0dTz/9tMkvqObmZsTFxUGlUsHPzw8rV640WOb8+fOYOXMmXF1dMXz4cHzwwQcGy9jZ2WH9+vWYPn06XF1dMWLECHz66afS/LNnz8LOzg4ff/wx7rvvPri6uuInP/kJSktLcezYMURGRkKlUmH69Om4cOGCic8akJWVhbvuugsKhQJ+fn5ISkqS5lVVVWHWrFlQqVRwd3fH448/jrq6Oml+x1nR5s2boVarMXDgQPzqV7+Sbo4Hbtxr+sYbbyAoKAgKhQKBgYH485//LM1ftmwZRo0aBaVSiREjRuDll1+GVqsFAGRnZ2P58uU4ceKEdEyzs7NN3re+dquvuY6uij//+c8YOnQogoODAQCHDx9GeHg4XFxcEBkZic8//xx2dnYoKCiQ1j158iSmT58OlUoFX19fzJs3D/X19Z3Wc/bs2f56OnqfsBENDQ0iOjpaLFy4UNTU1Iiamhpx/fp1kZqaKo4dOybOnDkj3n//faFUKsXWrVul9eLj44VKpRKxsbHi5MmT4osvvhDe3t7ixRdfNOlxExMTRWBgoPj3v/8tvvvuOzFjxgzh5uYmlixZIi0zffp0ERYWJo4cOSKOHz8uJkyYIFxdXcXq1aulZQCIwYMHiw0bNoiSkhLxxz/+UTg4OIjTp08LIYSoqKgQAERISIjYuXOnOH36tBg/fryIiIgQkyZNEgcPHhT5+fkiKChIPP300ybV/s477wgXFxeRkZEhSkpKxNGjR6Wa2tvbRXh4uJg4caI4fvy4yM3NFREREeL++++X1k9LSxMqlUrMnj1bFBYWiv/85z9iyJAhes/d888/LwYNGiSys7NFWVmZOHDggNiwYYM0f8WKFeLQoUOioqJCbN++Xfj6+orXX39dCCFES0uL+MMf/iDuuusu6Zi2tLSYtG/94XZfc/PmzRMnT54UJ0+eFFeuXBGenp7iySefFKdOnRI7duwQo0aNEgDEt99+K4QQ4vLly8Lb21ukpKSIoqIikZ+fL2JiYsTkyZM7raetrc0cT02vsJnwEkKI+++/Xy80jFm8eLH45S9/KU3Hx8cLT09P0dzcLLWtX79eqFQq0d7e3uW2rl69KpydncXHH38stV28eFG4urpKdZSUlAgA4ujRo9IyRUVFAoBBeN0cOlFRUSIxMVEI8WN4/eMf/5Dmf/TRRwKAyMnJkdrS09NFcHBwl3V3GDp0qHjppZeMztu9e7dwcHAQVVVVUtupU6f09iUtLU0olUrR2NgoLfPcc8+JqKgoIYQQjY2NQqFQ6IVVd958800REREhTaelpYmwsDCT1+9vt/qa8/X1FRqNRmpbv369GDx4sLh27ZrUtmHDBr3wWrFihZg6daretqurqwUAUVJSYnI9cmFzo0rcbN26dcjKykJVVRWuXbuG1tZWgw7gsLAwKJVKaTo6OhpNTU2orq7GsGHDOt12eXk5WltbERUVJbV5enpKbwMAoKioCI6OjoiIiJDaQkJC4OHhYbC96Ohog+n/fcsAAGPGjJH+7evrCwC455579NrOnz/fac0dzp8/j3PnzuEXv/iF0flFRUUICAjQGxdt9OjR8PDwQFFREX7yk58AuPGpqZubm7SMn5+f9PhFRUXQaDSdPgYAbN26FWvXrkV5eTmamprQ1tYm+xEVTHnN3XPPPXB2dpamS0pKMGbMGLi4uEht48aN01vnxIkT2LdvH1QqlcFjlpeXY9SoUb27I2ZmM31exmzZsgVLly7FggULsHv3bhQUFCAhIUHWHaT/O8Jmx3hlN7fpdLput+Pq6trr9dz8+N09xpEjRzB37lw8+OCD+OKLL/Dtt9/ipZdekvXxMfU1N2DAgB5vu6mpCTNnzkRBQYHez3//+1/87Gc/661dsBg2FV7Ozs5ob2+Xpg8dOoQJEyZg0aJFuPfeexEUFITy8nKD9U6cOIFr165J07m5uVCpVN2Oxjpy5Eg4OTnhm2++kdouX76M0tJSaTokJARtbW3Iy8uT2kpKStDQ0GCwvdzcXIPp0NDQLmu4VW5ublCr1cjJyTE6PzQ0FNXV1aiurpbaTp8+jYaGBowePdqkx7jzzjvh6ura6WMcPnwYw4YNw0svvYTIyEjceeedqKys1Fvm5mNqaW71NXez4OBgFBYWQqPRSG3Hjh3TW2bs2LE4deoU1Go1goKC9H46wtDSn6+esKnwUqvV+Oabb3D27FnU19fjzjvvxPHjx7Fr1y6Ulpbi5ZdfNnhBADc+ql6wYAFOnz6NHTt2IC0tDUlJSV2O8ggAKpUKCxYswHPPPYe9e/fi5MmTmD9/vt56wcHBeOCBB/Db3/4W33zzDfLy8vDrX//a6FnJJ598gqysLJSWliItLQ1Hjx7V+/Svt73yyitYuXIl1q5di//+97/Iz8/H22+/DQCYMmUK7rnnHsydOxf5+fk4evQo4uLicP/99yMyMtKk7bu4uGDZsmV4/vnnsWnTJpSXlyM3NxfvvvsugBvhVlVVhS1btqC8vBxr167FZ599prcNtVqNiooKFBQUoL6+Xu+P2xLc6mvuZnPmzIFOp8NvfvMbFBUVYdeuXXjrrbcA/HiGvXjxYly6dAlPPPEEjh07hvLycuzatQsJCQlSYN1cjyln4RbL3J1u/amkpESMHz9euLq6CgCiuLhYzJ8/XwwcOFB4eHiIxMRE8cILL+h1AMfHx4tZs2aJ1NRUMXjwYKFSqcTChQvF9evXTXrMq1eviieffFIolUrh6+sr3njjDYNO05qaGvHQQw8JhUIhAgMDxaZNm8SwYcMMOuzXrVsnYmJihEKhEGq1Wu8Tqo4O+47OWyGE2LdvnwAgLl++LLVt3LhRDBw40OTnLDMzUwQHBwsnJyfh5+cnfve730nzKisrxcMPPywGDBgg3NzcxGOPPSZqa2ul+cY601evXi2GDRsmTbe3t4tXX31VDBs2TDg5OYnAwEDx2muvSfOfe+456XmPjY0Vq1ev1qv/+vXr4pe//KXw8PAQAMTGjRtN3rf+cDuvuZsdOnRIjBkzRjg7O4uIiAjx4YcfStvsUFpaKh599FHh4eEhXF1dRUhIiHjmmWeETqczWk9FRUUfPwN9h0PiyISdnR0+++yzXr9VieTrgw8+QEJCAq5cudJrfZRyYvOfNhLJxaZNmzBixAj4+/vjxIkTWLZsGR5//HGbDC6A4XVbqqqquuycPn36NAIDA/uxop4x9pF6h6+++gr33XdfP1ZD3amtrUVqaipqa2vh5+eHxx57TO9uBFvDt423oa2trcvbK9RqNRwdLff/h7Kysk7n+fv72+z/6CQPDC8ikiWbulSCiKwHw4uIZInhRUSyxPAiIllieFGfmj9/vjTwnZOTE3x9fRETE4OsrKwe3ZqSnZ1tdKSNvtYX32FAvYPhRX3ugQceQE1NDc6ePYuvvvoKkydPxpIlSzBjxgy0tbWZuzySK3Pem0TWr7P79HJycgQAaSDClStXirvvvlsolUpxxx13iMTERHH16lUhxI/3aP7vT1pamhBCiE2bNomIiAihUqmEr6+veOKJJ0RdXZ30OJcuXRJz5swRXl5ewsXFRQQFBYmsrCxpflVVlXjsscfEwIEDxaBBg8TDDz8s3e+XlpZm8Lj79u3rk+eJeo5nXmQWP//5zxEWFoZt27YBAOzt7bF27VqcOnUK7733Hvbu3Yvnn38eADBhwgRkZGTA3d0dNTU1qKmpwdKlSwEAWq0WK1aswIkTJ/D555/j7NmzmD9/vvQ4L7/8Mk6fPo2vvvoKRUVFWL9+Pby8vKR1p02bBjc3Nxw4cACHDh2CSqXCAw88gNbWVixduhSPP/64dOZYU1ODCRMm9O8TRZ0zd3qSdevszEsIIWJjY0VoaKjReZ988okYPHiwNG3qaBjHjh0TAKSztpkzZ4qEhASjy27evFkEBwdLIy4IIYRGoxGurq5i165d3dZP5sUzLzIbIYQ0FtW///1v/OIXv4C/vz/c3Nwwb948XLx4ES0tLV1uIy8vDzNnzkRgYCDc3Nxw//33A7hx3ykAJCYmYsuWLQgPD8fzzz+Pw4cPS+ueOHECZWVlcHNzg0qlgkqlgqenJ65fv27SAIFkXgwvMpuioiIMHz4cZ8+exYwZMzBmzBj885//RF5eHtatWweg6+8sbG5uxrRp0+Du7o4PPvgAx44dkwYr7Fhv+vTpqKysxLPPPiuNyd/xlrOpqQkREREGwyaXlpZizpw5fbz3dLss965hsmp79+5FYWEhnn32WeTl5UGn02HlypXSKLMff/yx3vLGhi8uLi7GxYsX8Ze//EUakvv48eMGj+Xt7Y34+HjEx8fjvvvuw3PPPYe33noLY8eOxdatW+Hj49Ppl3pY07DJ1oZnXtTnNBoNamtr8cMPPyA/Px+vvfYaZs2ahRkzZiAuLg5BQUHQarV4++23cebMGWzevBmZmZl621Cr1WhqakJOTg7q6+vR0tKCwMBAODs7S+tt374dK1as0FsvNTUV//rXv1BWVoZTp07hiy++kMb9nzt3Lry8vDBr1iwcOHAAFRUV2L9/P37/+9/j+++/lx73u+++Q0lJCerr66UvvCULYO5ON7Ju8fHx0mUGjo6OwtvbW0yZMkVkZWXpfe/lqlWrhJ+fn3B1dRXTpk0TmzZtMhjC+umnnxaDBw/Wu1Tiww8/FGq1WigUChEdHS22b99u8F2GoaGhwtXVVXh6eopZs2aJM2fOSNusqakRcXFxwsvLSygUCjFixAixcOFCceXKFSGEEOfPnxcxMTFCpVLxUgkLwyFxiEiW+LaRiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSz9H+d6+BM/+nePAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:23:58.638235Z", + "iopub.status.busy": "2024-02-29T06:23:58.637938Z", + "iopub.status.idle": "2024-02-29T06:23:58.842257Z", + "shell.execute_reply": "2024-02-29T06:23:58.841314Z" + }, + "papermill": { + "duration": 0.227444, + "end_time": "2024-02-29T06:23:58.844413", + "exception": false, + "start_time": "2024-02-29T06:23:58.616969", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEpCAYAAAA3eS91AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3aUlEQVR4nO3deVhUZf8/8PeZHZhhlVU2F8QNlxQV09TEr5pb2e9yyafQrDSXckvNXPMpLa3HSrOyp2xxr9S+9ahf9XFfcUFBEAMxUEEUBIZtmOX+/TFyZNicgRkOM3xe1zWXzJkzcz4OM2/u+z7n3IdjjDEQQoiAREIXQAghFESEEMFREBFCBEdBRAgRHAURIURwFESEEMFREBFCBEdBRAgRHAURIURwFESEEMEJGkTLly8Hx3Emt7Zt2wpZEiFEABKhC+jQoQMOHTrE35dIzC/JYDDg7t27UKlU4DjOFuURQuqBMQa1Wo2AgACIRDW3ewQPIolEAj8/vzo99+7duwgKCrJyRYQQa8vIyEBgYGCNjwseRH/99RcCAgKgUCgQFRWFVatWITg4uNp1NRoNNBoNf7984oCMjAy4uro2SL2EEPMVFBQgKCgIKpWq1vU4IacB2bdvHwoLCxEeHo7MzEysWLECd+7cQUJCQrWFL1++HCtWrKiyPD8/n4KIkEaooKAAbm5uT/yOChpEleXl5SEkJASffvopJk+eXOXxyi2i8rSlICKkcTI3iATvmlXk7u6ONm3aICUlpdrH5XI55HJ5A1dFCLG1RnUcUWFhIVJTU+Hv7y90KYSQBiRoEM2bNw/Hjh3DrVu3cPr0abzwwgsQi8UYP368kGURQhqYoF2z27dvY/z48cjJyYG3tzf69OmDs2fPwtvbW8iyCCENTNAg2r59u5CbJ4Q0Eo1qjIgQYpk/rt7F5M2xyC/WCl1KvVAQEWLHZmy9jMPXs/GvQzeELqVeKIgIcQC5RWVCl1AvFESEOIBGc1RyHVEQEeIAGtEJEnVCQUSIA7DvGKIgIsQx2HkSURAR4gCYnScRBREhDsDOh4goiAhxBBREhBDBUdeMECI4ahERQgRn5zlEQUSII6AWESGkEbDvJKIgIsQBUIuIECI4O88hCiJCiPAoiAhxAHT2PSFEcPYdQxREhDgEO28QURAR4gjsPIcoiAhxBDRGRAgh9URBRIgDsPMGEQURIY6ApgEhhAiOWkSEEMFREBFCBEddM0KI4KhFRAgRnJ3nEAURIQ7BzpOIgogQB0BjRFayevVqcByHWbNmCV0KIXaHxoisIDY2Fl9//TU6deokdCmE2CU7zyHhg6iwsBATJkzApk2b4OHhIXQ5hNglOum1nqZPn45hw4YhOjr6ietqNBoUFBSY3Agh9t8ikgi58e3bt+PSpUuIjY01a/1Vq1ZhxYoVNq6KEPtj5w0i4VpEGRkZePvtt7FlyxYoFAqznvPuu+8iPz+fv2VkZNi4SkLsg53nkHAtoosXLyI7OxtPPfUUv0yv1+P48eNYv349NBoNxGKxyXPkcjnkcnlDl0oIsTHBgmjgwIGIj483WTZp0iS0bdsWCxYsqBJChJBa2HnfTLAgUqlU6Nixo8kyFxcXeHl5VVlOCKmdfcdQI9hrRgghgu41q+zo0aNCl0CIXbLznhm1iAhxBHSuGSFEcNQiIoQIjoKIECI4O88hCiJCHAGd9EoIIfVEQUSIA7DzBhEFESGOgHbfE0IERy0iQojg7DyHKIgIcQS014wQIjj7jiEKIkIcg50nUb3Ovi8sLITBYDBZ5urqWq+CCCGWs/McsrxFlJaWhmHDhsHFxQVubm7w8PCAh4cH3N3d6XJAhAjE3seILG4R/eMf/wBjDN999x18fX3BcZwt6iKEWMC+Y6gOQXTlyhVcvHgR4eHhtqiHENIEWdw1i4yMpMv4ENLI2HnPzPIW0bfffoupU6fizp076NixI6RSqcnjdP16QhqevZ/iYXEQ3b9/H6mpqZg0aRK/jOM4MMbAcRz0er1VCySEPFmTaxG9+uqr6Nq1K7Zt20aD1YQ0Ek0uiP7++2/8/vvvaN26tS3qIYQ0QRYPVj/77LO4cuWKLWohhNRRkzuOaMSIEZg9ezbi4+MRERFRZbB65MiRViuOEGIe+46hOgTR1KlTAQDvv/9+lcdosJoQYdh5g8jyIKp8bhkhRHj2vvveojEirVYLiUSChIQEW9VDCKkDe28RWRREUqkUwcHB1P0ipJGx8xyyfK/Ze++9h0WLFiE3N9cW9RBC6sDeW0QWjxGtX78eKSkpCAgIQEhICFxcXEwev3TpktWKI4SYy76TyOIgev75521QBiGkPppci2jZsmW2qIMQUg92nkN1n7P64sWL+Pnnn/Hzzz/j8uXLdXqNjRs3olOnTnB1dYWrqyuioqKwb9++upZESJPV5I6szs7Oxrhx43D06FG4u7sDAPLy8jBgwABs374d3t7eZr9WYGAgVq9ejbCwMDDG8MMPP2DUqFG4fPkyOnToYGlphDRZ9h1DdWgRzZw5E2q1GteuXUNubi5yc3ORkJCAgoICvPXWWxa91ogRI/Dcc88hLCwMbdq0wQcffAClUomzZ89aWhYhTZqdN4gsbxHt378fhw4dQrt27fhl7du3x4YNG/A///M/dS5Er9dj165dKCoqQlRUVLXraDQaaDQa/n5BQUGdt0eII7H3rpnFLSKDwVDlRFfAeLBjXU7/iI+Ph1KphFwux9SpU7F79260b9++2nVXrVoFNzc3/hYUFGTx9ghxRPYdQ3WcBuTtt9/G3bt3+WV37tzB7NmzMXDgQIsLCA8PR1xcHM6dO4c333wTMTExSExMrHbdd999F/n5+fyN5s4mxDHU6YDGkSNHIjQ0lG+RZGRkoGPHjvj5558tLkAmk/GTrHXr1g2xsbH47LPP8PXXX1dZVy6XQy6XW7yNmuj1emi1Wqu9HiENrblKDABwkYlRWlra4NuXSqUQi8X1fh2LgygoKAiXLl3CoUOHcP36dQBAu3btEB0dXe9iAGPXr+I4kC0wxpCVlYW8vDybbocQW1s+wAcAIOKMFz8Vgru7O/z8/Oo1bXSdLjnNcRwGDRqEQYMG1XnDgLGrNXToUAQHB0OtVmPr1q04evQoDhw4UK/XfZLyEPLx8YGzszPNu03sVpmTcYeNiOPQwlfVoNtmjKG4uBjZ2dkAAH9//zq/Vp2C6PDhwzh8+DCys7OrDFB/9913Zr9OdnY2XnnlFWRmZsLNzQ2dOnXCgQMH6h1wtdHr9XwIeXl52Ww7hDQETmLsjnEcB4VC0eDbd3JyAmD8Lvv4+NS5m2ZxEK1YsQLvv/8+unfvDn9//3q1Jv7973/X+bl1VT4m5Ozs3ODbJsQRlX+XtFptwwXRV199hc2bN+Pll1+u0wYbC+qOEWId1vguWbz7vqysDL179673hgkhpJzFQfTaa69h69attqiF2MDEiROtPnVL//79MWvWrFrXCQ0Nxbp166y6XSIMc37f9WVx16y0tBTffPMNDh06hE6dOlU5yvrTTz+1WnHEVP/+/dGlSxf6gjuQ5cuXY8+ePYiLi6vX69j7kdUWB9HVq1fRpUsXAKgyiT6NuxDSOGm12mpPzWosLO6aHTlypMbbf//7X1vUSGDsYh07dgyfffYZOI4Dx3FITU3F5MmT0aJFCzg5OSE8PByfffZZtc9fsWIFvL294erqiqlTp6KsrMys7RYVFeGVV16BUqmEv78/PvnkkyrrZGdnY8SIEXByckKLFi2wZcuWKutwHIeNGzdi6NChcHJyQsuWLfHLL7/wj9+6dQscx2Hnzp3o27cvnJycEBkZiRs3biA2Nhbdu3eHUqnE0KFDcf/+fTPfNePhJB06dIBcLoe/vz9mzJjBP5aeno5Ro0ZBqVTC1dUVY8aMwb179/jHly9fji5duuCnn35CaGgo3NzcMG7cOKjVan4dg8GAjz/+GK1bt4ZcLkdwcDA++OAD/vEFCxagTZs2cHZ2RsuWLbFkyRJ+z+3mzZuxYsUKXLlyhf+dbt68+Yn/p+vXr6NPnz5QKBR44dleOHviKDoFumPPnj0m7+WOHTvQr18/KBQKbNmyBTk5ORg/fjyaN28OZ2dnREREYNu2bSavbc7v2yaYHcvPz2cAWH5+vtnPKSkpYYmJiaykpIQxxpjBYGBFGq0gN4PBYHbdeXl5LCoqir3++ussMzOTZWZmstLSUrZ06VIWGxvLbt68yX7++Wfm7OzMduzYwT8vJiaGKZVKNnbsWJaQkMD++OMP5u3tzRYtWmTWdt98800WHBzMDh06xK5evcqGDx/OVCoVe/vtt/l1hg4dyjp37szOnDnDLly4wHr37s2cnJzYv/71L34dAMzLy4tt2rSJJScns8WLFzOxWMwSExMZY4ylpaUxAKxt27Zs//79LDExkfXq1Yt169aN9e/fn508eZJdunSJtW7dmk2dOtWs2r/88kumUCjYunXrWHJyMjt//jxfk16vZ126dGF9+vRhFy5cYGfPnmXdunVj/fr145+/bNkyplQq2ejRo1l8fDw7fvw48/PzM3nv5s+fzzw8PNjmzZtZSkoKO3HiBNu0aRP/+MqVK9mpU6dYWloa+/3335mvry/76KOPGGOMFRcXs7lz57IOHTrwv9Pi4uJa/086nY6Fh4ezQYMGsbi4OPb9r/9hHbt0YwDY7t27Td7L0NBQ9uuvv7KbN2+yu3fvstu3b7M1a9awy5cvs9TUVPb5558zsVjMzp07Z9Hvu7LK36mKzP2ONvkgKtJoWciCPwS5FWm0Fv1/+/XrV+sHgjHGpk+fzl588UX+fkxMDPP09GRFRUX8so0bNzKlUsn0en2tr6VWq5lMJmM7d+7kl+Xk5DAnJye+juTkZAaAnT9/nl8nKSmJAagSRJUDpGfPnuzNN99kjD3+8nz77bf849u2bWMA2OHDh/llq1atYuHh4bXWXS4gIIC999571T72f//3f0wsFrP09HR+2bVr10z+L8uWLWPOzs6soKCAX+edd95hPXv2ZIwxVlBQwORyuUnwPMmaNWtYt27d+PvLli1jnTt3Nvv5+/btYxKJhGVmZjLGGLuS8ZB9vXV3tUG0bt26J77esGHD2Ny5cxlj5v2+q2ONIKrTkdWk8diwYQO+++47pKeno6SkBGVlZfwYXrnOnTubHMAZFRWFwsJCZGRkICQkpMbXTk1NRVlZGXr27Mkv8/T0RHh4OH8/KSkJEokE3bp145e1bduWn72zosrzTEVFRVUZpO3UqRP/s6+vLwAgIiLCZFn5KQW1yc7Oxt27d2ucESIpKQlBQUEmU8m0b98e7u7uSEpKQmRkJADj3j+V6vGpE/7+/vz2k5KSoNFoap11YseOHfj888+RmpqKwsJC6HQ6uLq6PrH+miQnJyMoKAh+fn78so5dnqp23e7du5vc1+v1+PDDD7Fz507cuXMHZWVl0Gg0/GfDnN+3rTT5IHKSipH4/mDBtl0f27dvx7x58/DJJ58gKioKKpUKa9aswblz56xUYcOrOKBavvOj8jJz5r0qP/XAmvVU3v6TtnHmzBlMmDABK1aswODBg+Hm5obt27c32LhL5Ut9rVmzBp999hnWrVuHiIgIuLi4YNasWWaPF9qSxYPVx48fh06nq7Jcp9Ph+PHjVimqIXEcB2eZRJCbpXsZZTKZyVV2T506hd69e2PatGno2rUrWrdujdTU1CrPu3LlCkpKSvj7Z8+ehVKpfOLEcq1atYJUKjUJtocPH+LGjRv8/bZt20Kn0+HixYv8suTk5GpnNqg8BfDZs2dNZvq0JpVKhdDQUBw+fLjax9u1a4eMjAyTOa0SExORl5dX48R8lYWFhcHJyanGbZw+fRohISF477330L17d4SFheHvv/82Wafy7/RJwsPDkZGRYTKofu2KeRevOHXqFEaNGoV//OMf6Ny5M1q2bGnyuzTn920rFreIBgwYgMzMTPj4+Jgsz8/Px4ABA+hy1DYUGhqKc+fO4datW1AqlQgLC8OPP/6IAwcOoEWLFvjpp58QGxuLFi1amDyvrKwMkydPxuLFi3Hr1i0sW7YMM2bMgEhU+98hpVKJyZMn45133oGXlxd8fHzw3nvvmTwvPDwcQ4YMwZQpU7Bx40ZIJBLMmjWr2tbCrl270L17d/Tp0wdbtmzB+fPnbXq+4fLlyzF16lT4+Phg6NChUKvVOHXqFGbOnIno6GhERERgwoQJWLduHXQ6HaZNm4Z+/fpV6dLURKFQYMGCBZg/fz5kMhmefvpp3L9/H9euXcPkyZMRFhaG9PR0bN++HZGRkfjzzz+xe/duk9cIDQ1FWloa4uLiEBgYCJVKVeucW4MGDUKrVq0QExODjz/+GJdT72L9GuNeuif9YQsLC8Mvv/yC06dPw8PDA59++inu3bvHB685v29bsXgLjLFq/8M5OTlVmoLEuubNmwexWIz27dvD29sbgwcPxujRozF27Fj07NkTOTk5mDZtWpXnDRw4EGFhYXjmmWcwduxYjBw5EsuXLzdrm2vWrEHfvn0xYsQIREdHo0+fPibjQQDw/fffIyAgAP369cPo0aPxxhtvVPlDBRgPIdi+fTs6deqEH3/8Edu2bTO79VEXMTExWLduHb788kt06NABw4cPx19//QXA+KXdu3cvPDw88MwzzyA6OhotW7bEjh07LNrGkiVLMHfuXCxduhTt2rXD2LFj+TGkkSNHYvbs2ZgxYwa6dOmC06dPY8mSJSbPf/HFFzFkyBAMGDAA3t7eVXanVyYWi7Fnzx4UFhYiMjISK+a/jddmzgGAJ559v3jxYjz11FMYPHgw+vfvDz8/vypH3Zvz+7YFjjHzZt0ePXo0AGDv3r0YMmSISWrr9XpcvXoV4eHh2L9/v20qrUZBQQHc3NyQn59v9gBgaWkp0tLS0KJFC0GmTWiqOI7D7t276UrBVnb1dh4ux57FxNFDkZKSglatWjV4DbV9p8z9jprdNXNzcwNgbBGpVCqTprdMJkOvXr3w+uuvW/p/IIRYaPfu3XzX/OyZOHy0/F10iewpSAhZi9lB9P333wMw9mnnzZtH3TAHkJ6eXmvXKDExEcHBwQ1YkWWUSmWNj+3btw99+/ZtwGqsY8uWLZgyZUq1j4WEhODatWtQq9VYsGAB0tPT4ebhhV59+mHu0n82cKXWZXbXrDGirln96HQ63Lp1q8bHQ0NDIZE03iM8UlJSanysefPmVtuF35DUarXJHrGKpFJpleO+rt7O43/uFOhuw8pq1qBds3L37t3DvHnz+KliK+cY7TWzHxKJhL+Cij2y59prolKpTA6gbCosDqKJEyciPT0dS5YsqfdUsYQQAtQhiE6ePIkTJ05UOY2AEELqyuLjiIKCguz+OtuEkMbF4iBat24dFi5cWOsgJyGEWMLirtnYsWNRXFyMVq1awdnZucpJgbm5uVYrjhDSNFgcRDRfsn0KDQ3FrFmz+EnQ6Uhn0phYHEQxMTG2qIM0sMzMTHh4eAhdBiEA6jBGBBgnUFq8eDHGjx/Pn+C3b98+XLt2zarFEdvx8/Or9SzvhsAYq3ZKGdL0WBxEx44dQ0REBM6dO4fffvsNhYWFAIxz3ixbtszqBZLH1Go1JkyYABcXF/j7++Nf//pXna85xXFclcnWf/vtNwwYMADOzs7o3Lkzzpw5Y/KckydP8hPbBwUF4a233kJRURH/+E8//YTu3btDpVLBz88PL730kslsikePHgXHcdi3bx+6desGuVyOkydP1um9II7F4iBauHAh/vnPf+LgwYOQyWT88meffbbKxFd2gTGgrEiYm4WHQcyZMwenTp3C77//joMHD+LEiRO4dOmS1d6K9957D/PmzUNcXBzatGmD8ePH8y2W1NRUDBkyBC+++CKuXr2KHTt24OTJkyZXxdBqtVi5ciWuXLmCPXv24NatW5g4cWKV7SxcuBCrV69GUlKSydSwpOmyeIwoPj6+2iu9+vj44MGDB1YpqkFpi4EPA4TZ9qK7gMy8k4fVajV++OEHbN26lZ8juXweIGuZN28ehg0bBsA4d1CHDh2QkpKCtm3bYtWqVZgwYQLf+goLC8Pnn3+Ofv36YePGjVAoFHj11Vf512rZsiU+//xzREZGorCw0OQE1ffffx+DBg2yWt3E/lncInJ3d0dmZmaV5ZcvX0bz5s2tUhSp6ubNm9BqtejRowe/zM3NzaoTm1dsnfj7+wMA37W6cuUKNm/eDKVSyd8GDx4Mg8GAtLQ0AMDFixcxYsQIBAcHQ6VSoV+/fgCMZ/lXZO4MiKTpsLhFNG7cOCxYsAC7du3iJxI/deoU5s2bh1deecUWNdqW1NnYMhFq241IdRPXl08UX1hYiClTpuCtt96q8rzg4GAUFRVh8ODBGDx4MLZs2QJvb2+kp6dj8ODBVSZnpylkSGUWB9GHH36I6dOnIygoCHq9Hu3bt4der8dLL72ExYsX26JG2+I4s7tHQmrZsiWkUiliY2P5OYLy8/Nx48YNPPPMMzbf/lNPPYXExMQaz3iPj49HTk4OVq9ezU/Kf+HCBZvXRRyDxUEkk8mwadMmLFmyBAkJCSgsLETXrl0RFhZmi/rIIyqVCjExMXjnnXfg6ekJHx8fLFu2DCKRqEFmQFiwYAF69eqFGTNm4LXXXoOLiwsSExNx8OBBrF+/HsHBwZDJZPjiiy8wdepUJCQkYOXKlTaviziGOk/PHxwcjOeeew5jxoypcwitWrUKkZGRUKlU8PHxwfPPP4/k5OS6luTwPv30U0RFRWH48OGIjo7G008/jXbt2jXIBG+dOnXCsWPHcOPGDfTt2xddu3bF0qVL+cFyb29vbN68Gbt27UL79u2xevVqrF271uZ1Ecdg1gyNc+bMwcqVK+Hi4oI5c+bUuu6nn35q9saHDBmCcePGITIyEjqdDosWLUJCQgISExPNGkdo6jM0FhUVoXnz5vjkk08wefJkocshAmhSMzRevnwZWq2W/7kmlnYRKl/xY/PmzfDx8cHFixcbZNzD3ly+fBnXr19Hjx49kJ+fj/fffx8AMGrUKIErI6R+zAqiI0eOVPuzteXn5wMwXm+7OhqNBhqNhr9fUFBgs1oaq7Vr1yI5ORkymQzdunXDiRMnkJSUhKFDh9b4nPKj3wlprBrNzOgGgwGzZs3C008/jY4dO1a7zqpVq7BixYoGrqzx6Nq1q8mlncuVlJQgLi6u4QsixErMCqLyiyua47fffqtTIdOnT0dCQkKt5x69++67JmNUBQUFT7x+e1Pg5OTkkBPJk6bDrCAqv7giYDxjevfu3XBzc+OPkL148SLy8vIsCqyKZsyYgT/++APHjx9HYGBgjevJ5XLBzxgnhFifWUFUfnFFwHg8yZgxY/DVV19BLBYDMF5CaNq0aWbvuSrHGMPMmTOxe/duHD16FC1atLDo+fVRfsQwIaR+rPFdsvgCi97e3jh58mSVc5ySk5PRu3dv5OTkmP1a06ZNw9atW7F3716T13NzczPr4nh12X1vMBjw119/QSwWw9vbGzKZjC6JROxWctbjHTbhfpY1BOqLMYaysjLcv38fer0eYWFhEIlMD0202QUWdTodrl+/XiWIrl+/bnEybty4EQDQv39/k+Xff/99tdNHWINIJEKLFi2QmZmJu3cFOseMECvJfljC/ywrEebKts7OzggODq4SQpawOIgmTZqEyZMnIzU1lT8T/Ny5c1i9ejUmTZpk0WsJdVkimUyG4OBg6HQ6ujItsWuv/XaU//nw3P4Nvn2xWAyJRFLvXoXFQbR27Vr4+fnhk08+4acD8ff3xzvvvIO5c+fWq5iGxHEcpFJplauQEGJP7qgf/yG15zMFLA4ikUiE+fPnY/78+fwBhZYOUhNCSEX1OqCRAogQYg11CqJffvkFO3fuRHp6epVJr6w5hzIhpGmweJj7888/x6RJk+Dr64vLly+jR48e8PLyws2bN2s934kQQmpicRB9+eWX+Oabb/DFF19AJpNh/vz5OHjwIN566y3+pFVCCLGExUGUnp6O3r17AzCe46RWqwEAL7/8MrZt22bd6gghTYLFQeTn54fc3FwAxlkay69llpaWJthxQYQQ+2ZxED377LP4/fffARgPbpw9ezYGDRqEsWPH4oUXXrB6gYQQx2fxXrNvvvmGP5Vj+vTp8PLywunTpzFy5EhMmTLF6gUSQhyfRUGk0+nw4Ycf4tVXX+Wn6xg3bhzGjRtnk+IIIU2DRV0ziUSCjz/+mL8eOiGEWIPFY0QDBw7EsWPHbFELIaSJsniMaOjQoVi4cCHi4+PRrVu3Kpf9GTlypNWKI4Q0DRYH0bRp0wBUf/0yjuNoWg1CiMUsDiKaYpUQYm11n1KNEEKsxOwWUUlJCQ4fPozhw4cDMF7ap+LFDsViMVauXGnXkzMRYm84DnCEExrMDqIffvgBf/75Jx9E69evR4cOHfhJ7q9fv46AgADMnj3bNpUSQqrgADhADpnfNduyZQveeOMNk2Vbt27FkSNHcOTIEaxZswY7d+60eoGEkJo5yhVozA6ilJQURERE8PcVCoXJrP09evRAYmKidasjhNTKMWLIgq5ZXl6eyZjQ/fv3TR43GAwmjxNCbM9BGkTmt4gCAwORkJBQ4+NXr16t9XLRhBDr4xykTWR2ED333HNYunQpSktLqzxWUlKCFStWYNiwYVYtjhDyBI6RQ+Z3zRYtWoSdO3ciPDwcM2bMQJs2bQAYLzW9fv166HQ6LFq0yGaFEkKqcpAcMj+IfH19cfr0abz55ptYuHAhPxsjx3EYNGgQvvzyS/j6+tqsUEJIVY4yRmTRKR4tWrTA/v37kZubi5SUFABA69at4enpaZPiCCG1c5Qxojpd18zT05O/7j0hRDiO0iKic80IsWMOkkMURITYsyZ3ZDUhpPFxjBiiICLEvjlIElEQEWLHHCSHhA2i48ePY8SIEQgICADHcdizZ4+Q5RBid2iMyAqKiorQuXNnbNiwQcgyCLFbDpJDdTuOyFqGDh2KoUOHClkCIXbNQXJI2CCylEajMZlqpKCgQMBqCCHWYleD1atWrYKbmxt/CwoKErokQgRFY0QCePfdd5Gfn8/fMjIyhC6JEEE5RgzZWddMLpdDLpcLXQYhjYaDNIjsq0VECKnMMZJI0BZRYWEhP50IAKSlpSEuLg6enp4IDg4WsDJC7IOjtIgEDaILFy5gwIAB/P05c+YAAGJiYrB582aBqiLEfjhIDgkbRP379+dneiSEWM5RWkQ0RkQIERwFESF2rC5TxTLGcDQ5G/fVjec6hBREhNixunTNTqfmYOL3sVi8J976BdURBREhdqwuQ0Q37xcCALLyq16jUCgURITYsbqc4lHeJSsu01u7nDqjICKkiblfaAyiEi0FESHECuoyRlTeIiqlICKEWEN9goi6ZoQQq6i4+97cg4PLg6hEq280BxRTEBFixyq2iMzJFMYYP0bEGKDRGWxUmWUoiAixYxV7Zua0bfJLtNDqH69Z0ki6ZxREhNixirvvzelmVT6auriRDFhTEBFixywdq64cRNQiIoRYlTlds/LxoXKNZRd+kwgixhj+98rdRvOmE2I1Fg5WV+maNZIWkV3NWV1Xp1JyMHPbZXi6yDAuMgj/6BWCAHcnocsipN5MB6stHyNqLEdXN4kWUVGZDs3dnZBbVIYvj6ai78dHMG3LRZy7mdNojqMgpC5MB6ufvH7VMSKdtUuqkybRIhrcwQ8D2/rgUFI2fjh9C2du5uA/8Vn4T3wW2vm7YmLvEIzq0hwKqVjoUgmxiMWD1YXUIhKURCzCkI5+2PZGL+yf1RfjewRDIRUhKbMAC36NR69Vh7F633XcySsRulRCzGbpAY3lLSJnmfGPbmMZI2oyQVRRWz9XrBodgbPvDsSi59oi0MMJecVafHUsFX0/+i+m/HQBZ1Kp20YaP5NTPCwYIwr2dAbQeHbfN4muWU3cnWV445lWmNynJf57PRubT6fhVEoODly7hwPX7iHcV4WY3qF4oWtzOMmo20YaH0taRFq9ATlFZQCMQXQ9S91o9iQ3yRZRZWIRh0HtfbHltV74v9nPYELPYDhJxUi+p8ai3cZu24f/SUJGbrHQpdbbdyfTcOBaltBlEBt4Unsop9AYQmIRx+81pq5ZI9XGV4UPXojA2UUDsXhYOwR7OiO/RItvjt9EvzVH8PqPF3Aq5YFddtviMvLw/h+JmPLTRaFLIVZiySke5d2yZkoZP0bUWAarm3TXrDZuTlK81rclJj3dAkeTs7H59C2c+OsBDibew8HEe2jjq8QrUaEY/VRzOMvs420sn6uYOA5LTnq9X2ico9pbJX8cRI2kRWQf3yABiUUcBrbzxcB2vkjJVuOH03/j10u3ceNeIRbvScDH+69jTPcgvBIVimAvZ6HLrZW69PExI4yxOs13XJOSMj0MjMFFTh+phmTJGFF5i8hbKecPVWksLSLqmlmgtY8KK5/viLOLBmLp8PYI8XJGQakO355MQ7+1RzB5cyzOpOYIXWaNCkq0/M96g/W6lgYDQ8flB9Bh2QFodI3jg91UcBY0ifggUsn5VnxjGSNqGn++GLPqtXldFVK82qcFJvYOxbEb9/H96Vs4fuM+Dl/PxuHr2dj0SncMau9rte1Zi1rzuEVUpjdAIrbO3yGtwcAHW1Z+KUK8XKzyuuTJLNl9XzGInGTG332Ne830OqAkFyh6ABTnAMWP/i3KeXx/xOeAXGmV/0fTCKLUw8COlwHnZoCzJ+DsZby5VLrv3Ozxz04egLj2t0ck4jCgrQ8GtPVB6v1CrD2QjH0JWfjwP0noH+4NqZW+6JYqKdPjbn4JWnmbfkgqtoi0OgbIrLM9UYWQ11mxpUWezKyuGWNAWSEMD9PQmbuJriUP4HOnGK+Lr6J9jhbY+9OjcMl5HDyleU/e+LNLKIgsUpwLaIuB/HTjzSwc4OReIaS8HoVWM9Nlj8KslcoLH78YgfNpuUh7UITt59PxclSoDf9TNRu/6SziMvLw65u90S3Eg1+eV/w4iDR6PQCp1bdtoCCyvQqtlYiyqwgUZcKTU0N++gqgzaum9ZID6DVYCQByAHHGl+kkBVAM4HJNG+KMf5D5z3mlz73c1Wr/paYRRO1GAG9dNgZSxdSv6VbyEAAz/lvyEMhJMWszKpEUJ6Tu+FumQOEBN2j/bgWpyrtSmFW6SRX1/u8l3MnHR/uvY/7gtogIdENcRh4AYEdsukkQ5T46mA0Aymw0VzG1iCz0qLVi/EzmVgiRaj6j1bRWPgAet2xP1b4pDWR4wFRw9/KHTuGB/2YYYFB44sW+nU17BOWh4+QBiBrmQN6mEURSJ8CzpfFmDr3OGEAmH4TyD0blMHt0X1sEGLRw1txHOxEAZADXE568LZmyhu6hZ/V/iar5cIz75iwKNTokZZ7HhcWD+OVFlQYiHxQ9PuGx4rzF1mTOIDhjDHN3XoGHiwxLhre3SR229u9HB4b+O6Y7VAopijQ6nE7NQd9W7lBo82sfW6mmtWI5Y2slo8wFmVpn5DJX9O/aFgo3n0qfJU/+M/TUP0+gqEyPIy/1R5FGh9lfnISfkwIvPjPQ6u+PpZpGEFlKLAGU3sabubQl/AcrNvEv/HT4EnwlhXg7ygtKff6jD2CuabgZdMa/hmWFQJ4lXUYPkybye/oiPJSokFuiwrk9yegvysNDpoJLoQ7QhBnDjuP4I2sBYPflO5g1MAwikfUG8QEg9X4hOjZ3q3WdtAdF+O3yHQDAoufaQWzlGiyVX6JFQYkWQZ6VDr+opbVStv88RkONB5sYVM6lKMy6g8iyh1BwRXUrQqIwBoeL8fdaKvOAXuEJFw/fWlsrMzacwpVHLeDYQdG4cr8Ql9LzAA2Ah+UvroWXMpf/w+StkvN/MCzdfa83MJRq9SjV6uHuLLPa765RBNGGDRuwZs0aZGVloXPnzvjiiy/Qo0cPocuyjNQJcAsE3ALR3a8TVt8IwO9/P0RBURA++n+dqq7PGFCab9qqKq7UHK/4V5NvkjPj+EBJLpDzFwBgfMXfYhzQs7ypfhfAKgBiGZizF7brpciRqvAQKuQeU+HWw3C0DA2tpkXmCUjkdXob3t4eh8Ed/GqdUqXi8UulWn3DHXtUw56gA8fiUJSXjZFhMqj0+ZBqcsEV54IV54CrobXyZnnJOcabL8AfXcjAoQBK6J294Ozmg0ydEkHNAyHhu+nloVLhPZcZQzD+dj6+Pp6K/8RlwsCArsHuGNU5AB4SGQ5fycZLPZuhZzNP/HrpDtYeSEZWQSlfU+QHh574FjhJxXCRifkDGgs1Oryz6wpKdQY+YIy3R/d1j34uM/5csSV95t1n4e9mnQkGBQ+iHTt2YM6cOfjqq6/Qs2dPrFu3DoMHD0ZycjJ8fHyELq9OOI7Doufa4sWNZ7DrYgaiWnnBRS6BVm9Amc4ABgapWASpWASZRAWZ2A0yeStInUUI8nCCl/JxCJRq9ZBLRNDoDFCIDCZdxvwHmcjKuoP/PZMAT04NT64AnlDDg1PDgyuEFwrgxJUB+jJw6sxHXcYKEg8CiTX8J2QqwNkTGrknbqhlCPBvDi+fABRL3VEqdYdnM/8Kg5Ye4GAAe3RY2tHk+xjS0a/G90cqfhxEJXUNoketFfXDLGRn3kWgvBjfHoiFt1gNT6gR4anH7dsZaK3UwEWXB0PRA8i0BdW+1BjA+E1IM11eXiWTOCFbr0S23gV6hRcCAwOx90YZcpkx1HOYCg+ZCjlwxUOmQh6UMEAElALINb5GtNIH3z4fidsPi/HT2b/hqpCiW4gHOjdzh0Iqwokb9/H18VScSnl8HBrHAZfT83A5PY9fduR6Njo2d8OZm9UfryYTizCovS8fNIBxDqKjyfcBGN9vjuPg6iSFVMxBq2fYdfG2+e97BaVa640zckzgk6Z69uyJyMhIrF+/HgBgMBgQFBSEmTNnYuHChbU+t6CgAG5ubsjPz4erq/VG8K3lzZ8vYl+C5SeYSkQc2vm74k5eickAs6tCgoJSHXxUcgR5OuPi3w9reRUjBTQIdSqBuPQhvLgCeEANT84YVp5Qo7msGIHyYhiKcozLoYaEs/wDpmcc8qBELnNFLlQICQyCk5s3nD38IFU1g8HJE5yzFziXZrirdUH0Vwkohhwn5j9r7BLpdSgtuA9RyQMYinKQ9vffCHUqBXsUuleTUxDiVAon7UPoCh9ApS+AnNM+ubBKDIxDHlz4OnOZK3KZErmPQiSHuVYJl1LUrXVYmZ+rwqQFAwAiDnB1kvJ7NMUiDiM7B+D1vi3RTCXDn1czsTfuLvKKyyARi5CSbTxNRy4Robm7E24+eNwV/GJ8VzzTxhtuTqZ7Qw0Ghh4fHsaDQg1CvJxx7J0BAIDDSfdwJSMPCpkYCokYCqkYCqkICqkYTlIx5I9+Nj72eLlCKoZcIjKrW2/ud1TQICorK4OzszN++eUXPP/88/zymJgY5OXlYe/evSbrazQaaDSPm8oFBQUICgpqtEF0+2Ex5uy8giKNztj6EYsglXDgwEGrNxhbSHoDtDoGrd5g8qESCgcDVCg2trAqhZYHp4YXCoz3H4WWF6eGK1e3WQk0TIp8TgU5yuCGup0HV8Jkj1oiykrhonr0s6rCchXyH7VWlHIJCisc4FmZt0peZVpVS/Vp3QwKqRiHku6ZLO8c6IZAD2dc+DsX9woeT1Q2LjIYr/YJRaBH9acKZatL8caPF9FMaRzkD/FyQV5xGUasP4kynQEn5j8LmaT6Y9ey1aVYve86RnQOwIDwhutp2EUQ3b17F82bN8fp06cRFRXFL58/fz6OHTuGc+fOmay/fPlyrFixosrrNNYgslRBqRap2YVgAB6oNQhwd0IzpRynUx+gVGuARMQhp6gMIV7GGQGSs9T4K1uNAeE+2HY+Han3i6BSSNA3rBmCPJ0hE4sQl5GH/uE+UMrFSLhTgIhAN5xNzeEHLC+lP4SBgd/lDwBPt/ZClyB3PFCXYceFjCfWLYEOHig0BlWFVpePuAgqQz4fWuXdRy+oq23NGBiHh1CadHPKA6W8taKRuSND4/zE1sqcQW2w8WgqSrR6+LspkJn/uCXSI9QT7wwJR6CHEz7adx174u7iwxciIBYBH+1PxsYJT6FnSy9k5Zfi4wPX4eEsQ1s/FUp1BrTwcoGniwzn03LQPdQTv1y8jZbeLmjRzAV380rQLcQDD4u18FbKEdrMeIR5XnEZTqfmQPboANd+jw52Lb/8c3aBBkGezlVaMuYyGBgMjFntSHlrcsggsrcWkaPQGxhEnPEYIePPHCQizqRpzhhDQYkOesbgLBNDbzD+eyunGCVleohEgO7RQGdOoQZMWwQXbQFcUQCR1Amc0htQuMFZIUeZzgCVwjhu9LC4DCqFFHoDg7+bwqon6hLbMzeIBB2sbtasGcRiMe7dM2263rt3D35+VQc75XI55HLr9NeJ+cp30UrFHGraGcZxHNycq/5Fb9GsfuedVRy4J45L0LacTCZDt27dcPjwYX6ZwWDA4cOHTVpIhBDHJvju+zlz5iAmJgbdu3dHjx49sG7dOhQVFWHSpElCl0YIaSCCB9HYsWNx//59LF26FFlZWejSpQv2798PX9/GN40GIcQ2BD+OqD4a+3FEhDR15n5HG9/+PkJIk0NBRAgRHAURIURwgg9W10f58FZBQfUnMxJChFX+3XzSULRdB5FarQYABAUFCVwJIaQ2arUabm41z1Nl13vNDAYD7t69C5VKZZVD/8tPGcnIyKC9cA2A3u+G19DvOWMMarUaAQEBEIlqHgmy6xaRSCRCYGCg1V/X1dWVvhgNiN7vhteQ73ltLaFyNFhNCBEcBREhRHAURBXI5XIsW7aMzvBvIPR+N7zG+p7b9WA1IcQxUIuIECI4CiJCiOAoiAghgqMgIoQIrkkHUW5uLiZMmABXV1e4u7tj8uTJKCys+bI2ubm5mDlzJsLDw+Hk5ITg4GC89dZbyM/Pb8Cq7cuGDRsQGhoKhUKBnj174vz587Wuv2vXLrRt2xYKhQIRERH4z3/+00CVOg5L3vNNmzahb9++8PDwgIeHB6Kjo5/4O7IJ1oQNGTKEde7cmZ09e5adOHGCtW7dmo0fP77G9ePj49no0aPZ77//zlJSUtjhw4dZWFgYe/HFFxuwavuxfft2JpPJ2HfffceuXbvGXn/9debu7s7u3btX7fqnTp1iYrGYffzxxywxMZEtXryYSaVSFh8f38CV2y9L3/OXXnqJbdiwgV2+fJklJSWxiRMnMjc3N3b79u0GrbvJBlFiYiIDwGJjY/ll+/btYxzHsTt37pj9Ojt37mQymYxptVpblGnXevTowaZPn87f1+v1LCAggK1atara9ceMGcOGDRtmsqxnz55sypQpNq3TkVj6nlem0+mYSqViP/zwg61KrFaT7ZqdOXMG7u7u6N69O78sOjoaIpGoyvXUalM+BaZEYten7VldWVkZLl68iOjoaH6ZSCRCdHQ0zpw5U+1zzpw5Y7I+AAwePLjG9YmpurznlRUXF0Or1cLT09NWZVaryQZRVlYWfHxML70rkUjg6emJrCzzrlf/4MEDrFy5Em+88YYtSrRrDx48gF6vr3IRBF9f3xrf36ysLIvWJ6bq8p5XtmDBAgQEBFT5g2BrDhdECxcuBMdxtd6uX79e7+0UFBRg2LBhaN++PZYvX17/wgkR2OrVq7F9+3bs3r0bCoWiQbftcP2JuXPnYuLEibWu07JlS/j5+SE7O9tkuU6nQ25ubrVXma1IrVZjyJAhUKlU2L17N6TSul2z3JFZehVfAPDz87NofWKqLu95ubVr12L16tU4dOgQOnXqZMsyq9egI1KNSPlg9YULF/hlBw4ceOJgdX5+PuvVqxfr168fKyoqaohS7VaPHj3YjBkz+Pt6vZ41b9681sHq4cOHmyyLioqiwWoLWPqeM8bYRx99xFxdXdmZM2caosRqNdkgYsy4+75r167s3Llz7OTJkywsLMxk9/3t27dZeHg4O3fuHGPMGEI9e/ZkERERLCUlhWVmZvI3nU4n1H+j0dq+fTuTy+Vs8+bNLDExkb3xxhvM3d2dZWVlMcYYe/nll9nChQv59U+dOsUkEglbu3YtS0pKYsuWLaPd9xay9D1fvXo1k8lk7JdffjH5PKvV6gatu0kHUU5ODhs/fjxTKpXM1dWVTZo0yeQXkJaWxgCwI0eOMMYYO3LkCANQ7S0tLU2Y/0Qj98UXX7Dg4GAmk8lYjx492NmzZ/nH+vXrx2JiYkzW37lzJ2vTpg2TyWSsQ4cO7M8//2zgiu2fJe95SEhItZ/nZcuWNWjNNA0IIURwDrfXjBBifyiICCGCoyAihAiOgogQIjgKIkKI4CiICCGCoyAihAiOgogQIjgKIkKI4CiIiNVNnDix2ulXhgwZInRppJFyuGlASOMwZMgQfP/99ybLarrMsVarrTKVSllZGWQymcXbrevziLCoRURsQi6Xw8/Pz+Tm4eEBAOA4Dhs3bsTIkSPh4uKCDz74AMuXL0eXLl3w7bffokWLFvzEXOnp6Rg1ahSUSiVcXV0xZswYk/l2anoesS8UREQQy5cvxwsvvID4+Hi8+uqrAICUlBT8+uuv+O233xAXFweDwYBRo0YhNzcXx44dw8GDB3Hz5k2MHTvW5LUqP4/YH+qaEZv4448/oFQqTZYtWrQIixYtAgC89NJLmDRpksnjZWVl+PHHH+Ht7Q0AOHjwIOLj45GWloagoCAAwI8//ogOHTogNjYWkZGR1T6P2B8KImITAwYMwMaNG02WVbwyRMWrp5QLCQkxCZOkpCQEBQXxIQQA7du3h7u7O5KSkvggqvw8Yn8oiIhNuLi4oHXr1rU+bs4yc7dF7BuNEZFGq127dsjIyEBGRga/LDExEXl5eWjfvr2AlRFroxYRsQmNRlPlWloSiQTNmjUz+zWio6MRERGBCRMmYN26ddDpdJg2bRr69etXbdeO2C9qERGb2L9/P/z9/U1uffr0seg1OI7D3r174eHhgWeeeQbR0dFo2bIlduzYYaOqiVBozmpCiOCoRUQIERwFESFEcBREhBDBURARQgRHQUQIERwFESFEcBREhBDBURARQgRHQUQIERwFESFEcBREhBDB/X8bvPz19fqPPAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019035, + "end_time": "2024-02-29T06:23:58.882981", + "exception": false, + "start_time": "2024-02-29T06:23:58.863946", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + }, + "papermill": { + "default_parameters": {}, + "duration": 1881.475459, + "end_time": "2024-02-29T06:24:01.623005", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tab_ddpm_concat/4/mlu-eval.ipynb", + "output_path": "eval/contraceptive/tab_ddpm_concat/4/mlu-eval.ipynb", + "parameters": { + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "path": "eval/contraceptive/tab_ddpm_concat/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-02-29T05:52:40.147546", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/contraceptive/tab_ddpm_concat/eval.csv b/contraceptive/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..e809ae2ee1b94ae3efbf5cc7539ec75cc2269637 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,0.006976435168699962,0.019684452694278928,0.005469150488197168,4.732121467590332,0.10522309690713882,0.8996597528457642,0.25440865755081177,6.793341162847355e-05,1.4780972003936768,0.05780091881752014,0.13661587238311768,0.07395370304584503,0.08244121074676514,0.00028081866912543774,6.210218667984009 diff --git a/contraceptive/tab_ddpm_concat/history.csv b/contraceptive/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..94b8f52dad995242997fbe75cbf7cd2d25ee5917 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/history.csv @@ -0,0 +1,14 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.018960519676329567,0.633159703178444,0.0011727252675751032,0.05587912966730073,0.0,0.0,0.0,0.0,0.019387794903013855,320,80,97.81354141235352,1.2226692676544189,0.3056673169136047,0.1214748754282482,0.005117437991430052,1.2569677214083186,4.872746470283573e-05,0.0313290299847722,0.0,0.0,0.0,0.0,0.005207326461095363,80,20,19.29424023628235,0.9647120118141175,0.24117800295352937,0.0514119642553851 +1,0.014699359831865877,1.103800665530207,0.0005193239165346864,0.15416064693126827,0.0,0.0,0.0,0.0,0.01488935198285617,320,80,97.73809790611267,1.2217262238264084,0.3054315559566021,0.09262466638174374,0.008836813416564837,0.5016097052809527,0.00014911171683706925,0.1304568352177739,0.0,0.0,0.0,0.0,0.008931921707699075,80,20,19.49599575996399,0.9747997879981994,0.24369994699954986,0.0690233844332397 +2,0.009927867040460114,0.6356354271476448,0.00020892808070887558,0.11403882297454401,0.0,0.0,0.0,0.0,0.010061423623847076,320,80,97.83805751800537,1.2229757189750672,0.3057439297437668,0.10248138114111498,0.004521565337199718,1.0787538406874773,3.4153386028634714e-05,0.1442047566641122,0.0,0.0,0.0,0.0,0.004582326250965707,80,20,19.448378086090088,0.9724189043045044,0.2431047260761261,0.04124836295377463 +3,0.01018573724286398,0.7384141313583996,0.00020506313351988936,0.06779958754777908,0.0,0.0,0.0,0.0,0.010329423837538343,320,80,97.80484867095947,1.2225606083869933,0.30564015209674833,0.08878090149955824,0.004938754381146282,1.1424394390228372,2.856015009600199e-05,0.03729338594712317,0.0,0.0,0.0,0.0,0.005005534488009289,80,20,19.48591160774231,0.9742955803871155,0.24357389509677888,0.03839425216428936 +4,0.005628427321789786,0.445644349324516,4.922563089544974e-05,0.052951197209768,0.0,0.0,0.0,0.0,0.005696333260493703,320,80,97.72880864143372,1.2216101080179214,0.30540252700448034,0.0942096491693519,0.004431478650076315,2.1805682998136033,2.3924214234227746e-05,0.041412954684346914,0.0,0.0,0.0,0.0,0.0044817406902438964,80,20,19.514612197875977,0.9757306098937988,0.2439326524734497,0.03819809515262022 +5,0.004377504003969079,0.7532617844777411,4.913167144098285e-05,0.043540800781920554,0.0,0.0,0.0,0.0,0.004429807130145491,320,80,98.63056015968323,1.2328820019960403,0.30822050049901006,0.08962908287066966,0.003524563132668845,3.541576160071827,1.2924149661341922e-05,0.03394877891987562,0.0,0.0,0.0,0.0,0.003568227968935389,80,20,19.638448476791382,0.9819224238395691,0.24548060595989227,0.04204475942533463 +6,0.004308926320300088,0.655986472072349,3.8958016750684914e-05,0.04524608214851469,0.0,0.0,0.0,0.0,0.004360771635401761,320,80,98.1361870765686,1.2267023384571076,0.3066755846142769,0.08601754870906006,0.0037977110616338903,2.063007281812861,1.837681602836483e-05,0.04376108571887016,0.0,0.0,0.0,0.0,0.0038385945270420054,80,20,19.570854902267456,0.9785427451133728,0.2446356862783432,0.030497054848819972 +7,0.0038875900178936716,0.8493629617666272,2.666781584003133e-05,0.07926730818580836,0.0,0.0,0.0,0.0,0.003934197903254244,320,80,97.9080719947815,1.2238508999347686,0.30596272498369215,0.08284469123464078,0.003410281174001284,2.5351196237003024,8.761383719502192e-06,0.06262274421751499,0.0,0.0,0.0,0.0,0.0034496398351620884,80,20,19.63199734687805,0.9815998673439026,0.24539996683597565,0.029967716569080947 +8,0.003944453283475013,0.697565211437049,4.258767692717491e-05,0.04458279046230018,0.0,0.0,0.0,0.0,0.003991481203775038,320,80,97.73626518249512,1.2217033147811889,0.3054258286952972,0.08436954310745932,0.007964267671923153,0.92843264617959,8.528030000434227e-05,0.0632677624002099,0.0,0.0,0.0,0.0,0.0080590668701916,80,20,19.450220584869385,0.9725110292434692,0.2431277573108673,0.06122681526467204 +9,0.003340029970786418,0.48781303250957364,2.8307830278884367e-05,0.041397362318821254,0.0,0.0,0.0,0.0,0.003379593089266564,320,80,96.01197457313538,1.2001496821641922,0.30003742054104804,0.0903306363383308,0.003354643483180553,1.8730609221261603,1.555475001424611e-05,0.04130637706257403,0.0,0.0,0.0,0.0,0.0033913442108314483,80,20,18.96759533882141,0.9483797669410705,0.23709494173526763,0.04209472953807562 +10,0.003940557214809815,0.3181843262791631,3.4807291759936246e-05,0.04127106771338731,0.0,0.0,0.0,0.0,0.0039869014210125895,320,80,95.93454957008362,1.1991818696260452,0.2997954674065113,0.09244233099743723,0.003921330184675753,2.439302556675841,1.5360680176002007e-05,0.04387447247281671,0.0,0.0,0.0,0.0,0.003964195022854255,80,20,18.936774015426636,0.9468387007713318,0.23670967519283295,0.0390258968109265 +11,0.0029607639175083023,0.6851782390538916,9.521213198121579e-06,0.03578661805950105,0.0,0.0,0.0,0.0,0.0029942192559246905,320,80,96.0065131187439,1.2000814139842988,0.3000203534960747,0.08160512297181413,0.0032844552013557406,1.9535773819076212,1.0978048215903869e-05,0.03746633417904377,0.0,0.0,0.0,0.0,0.0033219303644727916,80,20,19.31860041618347,0.9659300208091736,0.2414825052022934,0.03969717922154814 +12,0.0025715101775858782,0.6255118230208495,7.738168527621215e-06,0.0324334034929052,0.0,0.0,0.0,0.0,0.002600309181434568,320,80,95.97005558013916,1.1996256947517394,0.29990642368793485,0.08734330767765641,0.002402786401216872,1.9713338775481133,4.400266065029967e-06,0.031274407636374235,0.0,0.0,0.0,0.0,0.0024293805909110233,80,20,19.06187415122986,0.9530937075614929,0.23827342689037323,0.034856686973944305 diff --git a/contraceptive/tab_ddpm_concat/mlu-eval.ipynb b/contraceptive/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..41f02105e2874fcf23419f2b0a4432c24ce8f0a7 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2221 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.553837Z", + "iopub.status.busy": "2024-02-29T05:52:41.553487Z", + "iopub.status.idle": "2024-02-29T05:52:41.587012Z", + "shell.execute_reply": "2024-02-29T05:52:41.586303Z" + }, + "papermill": { + "duration": 0.048134, + "end_time": "2024-02-29T05:52:41.588883", + "exception": false, + "start_time": "2024-02-29T05:52:41.540749", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.614875Z", + "iopub.status.busy": "2024-02-29T05:52:41.614416Z", + "iopub.status.idle": "2024-02-29T05:52:41.621190Z", + "shell.execute_reply": "2024-02-29T05:52:41.620324Z" + }, + "papermill": { + "duration": 0.022432, + "end_time": "2024-02-29T05:52:41.623143", + "exception": false, + "start_time": "2024-02-29T05:52:41.600711", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.646395Z", + "iopub.status.busy": "2024-02-29T05:52:41.646135Z", + "iopub.status.idle": "2024-02-29T05:52:41.650123Z", + "shell.execute_reply": "2024-02-29T05:52:41.649356Z" + }, + "papermill": { + "duration": 0.017841, + "end_time": "2024-02-29T05:52:41.651962", + "exception": false, + "start_time": "2024-02-29T05:52:41.634121", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.675528Z", + "iopub.status.busy": "2024-02-29T05:52:41.675083Z", + "iopub.status.idle": "2024-02-29T05:52:41.678787Z", + "shell.execute_reply": "2024-02-29T05:52:41.677998Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017719, + "end_time": "2024-02-29T05:52:41.680705", + "exception": false, + "start_time": "2024-02-29T05:52:41.662986", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.704275Z", + "iopub.status.busy": "2024-02-29T05:52:41.704010Z", + "iopub.status.idle": "2024-02-29T05:52:41.708949Z", + "shell.execute_reply": "2024-02-29T05:52:41.708172Z" + }, + "papermill": { + "duration": 0.019176, + "end_time": "2024-02-29T05:52:41.710931", + "exception": false, + "start_time": "2024-02-29T05:52:41.691755", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b4a51afa", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.735735Z", + "iopub.status.busy": "2024-02-29T05:52:41.735440Z", + "iopub.status.idle": "2024-02-29T05:52:41.740488Z", + "shell.execute_reply": "2024-02-29T05:52:41.739696Z" + }, + "papermill": { + "duration": 0.01964, + "end_time": "2024-02-29T05:52:41.742472", + "exception": false, + "start_time": "2024-02-29T05:52:41.722832", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tab_ddpm_concat/4\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.01106, + "end_time": "2024-02-29T05:52:41.765597", + "exception": false, + "start_time": "2024-02-29T05:52:41.754537", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.789321Z", + "iopub.status.busy": "2024-02-29T05:52:41.788485Z", + "iopub.status.idle": "2024-02-29T05:52:41.797921Z", + "shell.execute_reply": "2024-02-29T05:52:41.797137Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023362, + "end_time": "2024-02-29T05:52:41.799859", + "exception": false, + "start_time": "2024-02-29T05:52:41.776497", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tab_ddpm_concat/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:41.824124Z", + "iopub.status.busy": "2024-02-29T05:52:41.823296Z", + "iopub.status.idle": "2024-02-29T05:52:44.077985Z", + "shell.execute_reply": "2024-02-29T05:52:44.077036Z" + }, + "papermill": { + "duration": 2.269148, + "end_time": "2024-02-29T05:52:44.080113", + "exception": false, + "start_time": "2024-02-29T05:52:41.810965", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:44.106336Z", + "iopub.status.busy": "2024-02-29T05:52:44.105917Z", + "iopub.status.idle": "2024-02-29T05:52:44.116991Z", + "shell.execute_reply": "2024-02-29T05:52:44.116261Z" + }, + "papermill": { + "duration": 0.026284, + "end_time": "2024-02-29T05:52:44.118890", + "exception": false, + "start_time": "2024-02-29T05:52:44.092606", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:44.142507Z", + "iopub.status.busy": "2024-02-29T05:52:44.142237Z", + "iopub.status.idle": "2024-02-29T05:52:44.149263Z", + "shell.execute_reply": "2024-02-29T05:52:44.148530Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021057, + "end_time": "2024-02-29T05:52:44.151151", + "exception": false, + "start_time": "2024-02-29T05:52:44.130094", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:44.174949Z", + "iopub.status.busy": "2024-02-29T05:52:44.174650Z", + "iopub.status.idle": "2024-02-29T05:52:44.283419Z", + "shell.execute_reply": "2024-02-29T05:52:44.282604Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.123895, + "end_time": "2024-02-29T05:52:44.286263", + "exception": false, + "start_time": "2024-02-29T05:52:44.162368", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:44.314871Z", + "iopub.status.busy": "2024-02-29T05:52:44.314539Z", + "iopub.status.idle": "2024-02-29T05:52:48.872369Z", + "shell.execute_reply": "2024-02-29T05:52:48.871556Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.574074, + "end_time": "2024-02-29T05:52:48.874734", + "exception": false, + "start_time": "2024-02-29T05:52:44.300660", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 05:52:46.506222: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 05:52:46.506284: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 05:52:46.507965: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:48.899749Z", + "iopub.status.busy": "2024-02-29T05:52:48.899216Z", + "iopub.status.idle": "2024-02-29T05:52:48.904784Z", + "shell.execute_reply": "2024-02-29T05:52:48.904124Z" + }, + "papermill": { + "duration": 0.020032, + "end_time": "2024-02-29T05:52:48.906657", + "exception": false, + "start_time": "2024-02-29T05:52:48.886625", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:48.931922Z", + "iopub.status.busy": "2024-02-29T05:52:48.931646Z", + "iopub.status.idle": "2024-02-29T05:52:57.087404Z", + "shell.execute_reply": "2024-02-29T05:52:57.086436Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.171018, + "end_time": "2024-02-29T05:52:57.089766", + "exception": false, + "start_time": "2024-02-29T05:52:48.918748", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "100%|██████████| 1/1 [00:00<00:00, 2.79it/s]\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-02-29T05:52:57.118044Z", + "iopub.status.busy": "2024-02-29T05:52:57.117156Z", + "iopub.status.idle": "2024-02-29T05:52:57.123889Z", + "shell.execute_reply": "2024-02-29T05:52:57.123035Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023061, + "end_time": "2024-02-29T05:52:57.125871", + "exception": false, + "start_time": "2024-02-29T05:52:57.102810", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:57.151111Z", + "iopub.status.busy": "2024-02-29T05:52:57.150850Z", + "iopub.status.idle": "2024-02-29T05:52:57.155580Z", + "shell.execute_reply": "2024-02-29T05:52:57.154701Z" + }, + "papermill": { + "duration": 0.019488, + "end_time": "2024-02-29T05:52:57.157399", + "exception": false, + "start_time": "2024-02-29T05:52:57.137911", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:57.182402Z", + "iopub.status.busy": "2024-02-29T05:52:57.182137Z", + "iopub.status.idle": "2024-02-29T05:52:57.213320Z", + "shell.execute_reply": "2024-02-29T05:52:57.212336Z" + }, + "papermill": { + "duration": 0.046054, + "end_time": "2024-02-29T05:52:57.215294", + "exception": false, + "start_time": "2024-02-29T05:52:57.169240", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset\n", + "\n", + "test_set = load_dataset(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\", \"datasets_5\", dataset_name),\n", + " preprocessor=preprocessor,\n", + " cache_dir=os.path.join(path_prefix, dataset_name, \"_cache_test\"),\n", + " start=200,\n", + " #stop=600,\n", + " val=False,\n", + " ratio=0,\n", + " drop_first_column=True,\n", + " model=single_model,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:57.241834Z", + "iopub.status.busy": "2024-02-29T05:52:57.241535Z", + "iopub.status.idle": "2024-02-29T05:52:57.563537Z", + "shell.execute_reply": "2024-02-29T05:52:57.562643Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.337952, + "end_time": "2024-02-29T05:52:57.565789", + "exception": false, + "start_time": "2024-02-29T05:52:57.227837", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.73,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'loss_balancer_beta': 0.67,\n", + " 'loss_balancer_r': 0.943,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation': torch.nn.modules.activation.Softsign,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BEST,\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:57.592167Z", + "iopub.status.busy": "2024-02-29T05:52:57.591845Z", + "iopub.status.idle": "2024-02-29T05:52:57.659947Z", + "shell.execute_reply": "2024-02-29T05:52:57.659043Z" + }, + "papermill": { + "duration": 0.083576, + "end_time": "2024-02-29T05:52:57.661879", + "exception": false, + "start_time": "2024-02-29T05:52:57.578303", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../contraceptive/_cache/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache4/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache5/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/contraceptive [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T05:52:57.690612Z", + "iopub.status.busy": "2024-02-29T05:52:57.690320Z", + "iopub.status.idle": "2024-02-29T05:52:58.154857Z", + "shell.execute_reply": "2024-02-29T05:52:58.153972Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.481713, + "end_time": "2024-02-29T05:52:58.156846", + "exception": false, + "start_time": "2024-02-29T05:52:57.675133", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n", + "[*] Embedding False True\n", + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:58.187496Z", + "iopub.status.busy": "2024-02-29T05:52:58.187196Z", + "iopub.status.idle": "2024-02-29T05:52:58.191528Z", + "shell.execute_reply": "2024-02-29T05:52:58.190632Z" + }, + "papermill": { + "duration": 0.022791, + "end_time": "2024-02-29T05:52:58.193570", + "exception": false, + "start_time": "2024-02-29T05:52:58.170779", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:58.220322Z", + "iopub.status.busy": "2024-02-29T05:52:58.219763Z", + "iopub.status.idle": "2024-02-29T05:52:58.226654Z", + "shell.execute_reply": "2024-02-29T05:52:58.225833Z" + }, + "papermill": { + "duration": 0.022269, + "end_time": "2024-02-29T05:52:58.228558", + "exception": false, + "start_time": "2024-02-29T05:52:58.206289", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "11858440" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:58.254957Z", + "iopub.status.busy": "2024-02-29T05:52:58.254488Z", + "iopub.status.idle": "2024-02-29T05:52:58.332480Z", + "shell.execute_reply": "2024-02-29T05:52:58.331558Z" + }, + "papermill": { + "duration": 0.093949, + "end_time": "2024-02-29T05:52:58.334954", + "exception": false, + "start_time": "2024-02-29T05:52:58.241005", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 10] --\n", + "├─Adapter: 1-1 [2, 1179, 10] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 11,264\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 10] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 16, 256] 1\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 1,048,832\n", + "│ │ │ └─Softsign: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 256] --\n", + "│ │ │ └─Linear: 4-55 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-56 [2, 256] --\n", + "│ │ └─FeedForward: 3-28 [2, 256] --\n", + "│ │ │ └─Linear: 4-57 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-58 [2, 256] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 11,858,440\n", + "Trainable params: 11,858,440\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 44.00\n", + "========================================================================================================================\n", + "Input size (MB): 0.12\n", + "Forward/backward pass size (MB): 375.40\n", + "Params size (MB): 47.43\n", + "Estimated Total Size (MB): 422.95\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:52:58.365908Z", + "iopub.status.busy": "2024-02-29T05:52:58.365521Z", + "iopub.status.idle": "2024-02-29T06:20:40.160166Z", + "shell.execute_reply": "2024-02-29T06:20:40.159139Z" + }, + "papermill": { + "duration": 1661.828434, + "end_time": "2024-02-29T06:20:40.178185", + "exception": false, + "start_time": "2024-02-29T05:52:58.349751", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n", + "Train loss {'avg_role_model_loss': 0.018960519676329567, 'avg_role_model_std_loss': 0.633159703178444, 'avg_role_model_mean_pred_loss': 0.0011727252675751032, 'avg_role_model_g_mag_loss': 0.05587912966730073, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.019387794903013855, 'n_size': 320, 'n_batch': 80, 'duration': 97.81354141235352, 'duration_batch': 1.2226692676544189, 'duration_size': 0.3056673169136047, 'avg_pred_std': 0.1214748754282482}\n", + "Val loss {'avg_role_model_loss': 0.005117437991430052, 'avg_role_model_std_loss': 1.2569677214083186, 'avg_role_model_mean_pred_loss': 4.872746470283573e-05, 'avg_role_model_g_mag_loss': 0.0313290299847722, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005207326461095363, 'n_size': 80, 'n_batch': 20, 'duration': 19.29424023628235, 'duration_batch': 0.9647120118141175, 'duration_size': 0.24117800295352937, 'avg_pred_std': 0.0514119642553851}\n", + "Epoch 1\n", + "Train loss {'avg_role_model_loss': 0.014699359831865877, 'avg_role_model_std_loss': 1.103800665530207, 'avg_role_model_mean_pred_loss': 0.0005193239165346864, 'avg_role_model_g_mag_loss': 0.15416064693126827, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01488935198285617, 'n_size': 320, 'n_batch': 80, 'duration': 97.73809790611267, 'duration_batch': 1.2217262238264084, 'duration_size': 0.3054315559566021, 'avg_pred_std': 0.09262466638174374}\n", + "Val loss {'avg_role_model_loss': 0.008836813416564837, 'avg_role_model_std_loss': 0.5016097052809527, 'avg_role_model_mean_pred_loss': 0.00014911171683706925, 'avg_role_model_g_mag_loss': 0.1304568352177739, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008931921707699075, 'n_size': 80, 'n_batch': 20, 'duration': 19.49599575996399, 'duration_batch': 0.9747997879981994, 'duration_size': 0.24369994699954986, 'avg_pred_std': 0.0690233844332397}\n", + "Epoch 2\n", + "Train loss {'avg_role_model_loss': 0.009927867040460114, 'avg_role_model_std_loss': 0.6356354271476448, 'avg_role_model_mean_pred_loss': 0.00020892808070887558, 'avg_role_model_g_mag_loss': 0.11403882297454401, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010061423623847076, 'n_size': 320, 'n_batch': 80, 'duration': 97.83805751800537, 'duration_batch': 1.2229757189750672, 'duration_size': 0.3057439297437668, 'avg_pred_std': 0.10248138114111498}\n", + "Val loss {'avg_role_model_loss': 0.004521565337199718, 'avg_role_model_std_loss': 1.0787538406874773, 'avg_role_model_mean_pred_loss': 3.4153386028634714e-05, 'avg_role_model_g_mag_loss': 0.1442047566641122, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004582326250965707, 'n_size': 80, 'n_batch': 20, 'duration': 19.448378086090088, 'duration_batch': 0.9724189043045044, 'duration_size': 0.2431047260761261, 'avg_pred_std': 0.04124836295377463}\n", + "Epoch 3\n", + "Train loss {'avg_role_model_loss': 0.01018573724286398, 'avg_role_model_std_loss': 0.7384141313583996, 'avg_role_model_mean_pred_loss': 0.00020506313351988936, 'avg_role_model_g_mag_loss': 0.06779958754777908, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010329423837538343, 'n_size': 320, 'n_batch': 80, 'duration': 97.80484867095947, 'duration_batch': 1.2225606083869933, 'duration_size': 0.30564015209674833, 'avg_pred_std': 0.08878090149955824}\n", + "Val loss {'avg_role_model_loss': 0.004938754381146282, 'avg_role_model_std_loss': 1.1424394390228372, 'avg_role_model_mean_pred_loss': 2.856015009600199e-05, 'avg_role_model_g_mag_loss': 0.03729338594712317, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005005534488009289, 'n_size': 80, 'n_batch': 20, 'duration': 19.48591160774231, 'duration_batch': 0.9742955803871155, 'duration_size': 0.24357389509677888, 'avg_pred_std': 0.03839425216428936}\n", + "Epoch 4\n", + "Train loss {'avg_role_model_loss': 0.005628427321789786, 'avg_role_model_std_loss': 0.445644349324516, 'avg_role_model_mean_pred_loss': 4.922563089544974e-05, 'avg_role_model_g_mag_loss': 0.052951197209768, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005696333260493703, 'n_size': 320, 'n_batch': 80, 'duration': 97.72880864143372, 'duration_batch': 1.2216101080179214, 'duration_size': 0.30540252700448034, 'avg_pred_std': 0.0942096491693519}\n", + "Val loss {'avg_role_model_loss': 0.004431478650076315, 'avg_role_model_std_loss': 2.1805682998136033, 'avg_role_model_mean_pred_loss': 2.3924214234227746e-05, 'avg_role_model_g_mag_loss': 0.041412954684346914, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0044817406902438964, 'n_size': 80, 'n_batch': 20, 'duration': 19.514612197875977, 'duration_batch': 0.9757306098937988, 'duration_size': 0.2439326524734497, 'avg_pred_std': 0.03819809515262022}\n", + "Epoch 5\n", + "Train loss {'avg_role_model_loss': 0.004377504003969079, 'avg_role_model_std_loss': 0.7532617844777411, 'avg_role_model_mean_pred_loss': 4.913167144098285e-05, 'avg_role_model_g_mag_loss': 0.043540800781920554, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004429807130145491, 'n_size': 320, 'n_batch': 80, 'duration': 98.63056015968323, 'duration_batch': 1.2328820019960403, 'duration_size': 0.30822050049901006, 'avg_pred_std': 0.08962908287066966}\n", + "Val loss {'avg_role_model_loss': 0.003524563132668845, 'avg_role_model_std_loss': 3.541576160071827, 'avg_role_model_mean_pred_loss': 1.2924149661341922e-05, 'avg_role_model_g_mag_loss': 0.03394877891987562, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003568227968935389, 'n_size': 80, 'n_batch': 20, 'duration': 19.638448476791382, 'duration_batch': 0.9819224238395691, 'duration_size': 0.24548060595989227, 'avg_pred_std': 0.04204475942533463}\n", + "Epoch 6\n", + "Train loss {'avg_role_model_loss': 0.004308926320300088, 'avg_role_model_std_loss': 0.655986472072349, 'avg_role_model_mean_pred_loss': 3.8958016750684914e-05, 'avg_role_model_g_mag_loss': 0.04524608214851469, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004360771635401761, 'n_size': 320, 'n_batch': 80, 'duration': 98.1361870765686, 'duration_batch': 1.2267023384571076, 'duration_size': 0.3066755846142769, 'avg_pred_std': 0.08601754870906006}\n", + "Val loss {'avg_role_model_loss': 0.0037977110616338903, 'avg_role_model_std_loss': 2.063007281812861, 'avg_role_model_mean_pred_loss': 1.837681602836483e-05, 'avg_role_model_g_mag_loss': 0.04376108571887016, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0038385945270420054, 'n_size': 80, 'n_batch': 20, 'duration': 19.570854902267456, 'duration_batch': 0.9785427451133728, 'duration_size': 0.2446356862783432, 'avg_pred_std': 0.030497054848819972}\n", + "Epoch 7\n", + "Train loss {'avg_role_model_loss': 0.0038875900178936716, 'avg_role_model_std_loss': 0.8493629617666272, 'avg_role_model_mean_pred_loss': 2.666781584003133e-05, 'avg_role_model_g_mag_loss': 0.07926730818580836, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003934197903254244, 'n_size': 320, 'n_batch': 80, 'duration': 97.9080719947815, 'duration_batch': 1.2238508999347686, 'duration_size': 0.30596272498369215, 'avg_pred_std': 0.08284469123464078}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003410281174001284, 'avg_role_model_std_loss': 2.5351196237003024, 'avg_role_model_mean_pred_loss': 8.761383719502192e-06, 'avg_role_model_g_mag_loss': 0.06262274421751499, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034496398351620884, 'n_size': 80, 'n_batch': 20, 'duration': 19.63199734687805, 'duration_batch': 0.9815998673439026, 'duration_size': 0.24539996683597565, 'avg_pred_std': 0.029967716569080947}\n", + "Epoch 8\n", + "Train loss {'avg_role_model_loss': 0.003944453283475013, 'avg_role_model_std_loss': 0.697565211437049, 'avg_role_model_mean_pred_loss': 4.258767692717491e-05, 'avg_role_model_g_mag_loss': 0.04458279046230018, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003991481203775038, 'n_size': 320, 'n_batch': 80, 'duration': 97.73626518249512, 'duration_batch': 1.2217033147811889, 'duration_size': 0.3054258286952972, 'avg_pred_std': 0.08436954310745932}\n", + "Val loss {'avg_role_model_loss': 0.007964267671923153, 'avg_role_model_std_loss': 0.92843264617959, 'avg_role_model_mean_pred_loss': 8.528030000434227e-05, 'avg_role_model_g_mag_loss': 0.0632677624002099, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0080590668701916, 'n_size': 80, 'n_batch': 20, 'duration': 19.450220584869385, 'duration_batch': 0.9725110292434692, 'duration_size': 0.2431277573108673, 'avg_pred_std': 0.06122681526467204}\n", + "Epoch 9\n", + "Train loss {'avg_role_model_loss': 0.003340029970786418, 'avg_role_model_std_loss': 0.48781303250957364, 'avg_role_model_mean_pred_loss': 2.8307830278884367e-05, 'avg_role_model_g_mag_loss': 0.041397362318821254, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003379593089266564, 'n_size': 320, 'n_batch': 80, 'duration': 96.01197457313538, 'duration_batch': 1.2001496821641922, 'duration_size': 0.30003742054104804, 'avg_pred_std': 0.0903306363383308}\n", + "Val loss {'avg_role_model_loss': 0.003354643483180553, 'avg_role_model_std_loss': 1.8730609221261603, 'avg_role_model_mean_pred_loss': 1.555475001424611e-05, 'avg_role_model_g_mag_loss': 0.04130637706257403, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033913442108314483, 'n_size': 80, 'n_batch': 20, 'duration': 18.96759533882141, 'duration_batch': 0.9483797669410705, 'duration_size': 0.23709494173526763, 'avg_pred_std': 0.04209472953807562}\n", + "Epoch 10\n", + "Train loss {'avg_role_model_loss': 0.003940557214809815, 'avg_role_model_std_loss': 0.3181843262791631, 'avg_role_model_mean_pred_loss': 3.4807291759936246e-05, 'avg_role_model_g_mag_loss': 0.04127106771338731, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0039869014210125895, 'n_size': 320, 'n_batch': 80, 'duration': 95.93454957008362, 'duration_batch': 1.1991818696260452, 'duration_size': 0.2997954674065113, 'avg_pred_std': 0.09244233099743723}\n", + "Val loss {'avg_role_model_loss': 0.003921330184675753, 'avg_role_model_std_loss': 2.439302556675841, 'avg_role_model_mean_pred_loss': 1.5360680176002007e-05, 'avg_role_model_g_mag_loss': 0.04387447247281671, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003964195022854255, 'n_size': 80, 'n_batch': 20, 'duration': 18.936774015426636, 'duration_batch': 0.9468387007713318, 'duration_size': 0.23670967519283295, 'avg_pred_std': 0.0390258968109265}\n", + "Epoch 11\n", + "Train loss {'avg_role_model_loss': 0.0029607639175083023, 'avg_role_model_std_loss': 0.6851782390538916, 'avg_role_model_mean_pred_loss': 9.521213198121579e-06, 'avg_role_model_g_mag_loss': 0.03578661805950105, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029942192559246905, 'n_size': 320, 'n_batch': 80, 'duration': 96.0065131187439, 'duration_batch': 1.2000814139842988, 'duration_size': 0.3000203534960747, 'avg_pred_std': 0.08160512297181413}\n", + "Val loss {'avg_role_model_loss': 0.0032844552013557406, 'avg_role_model_std_loss': 1.9535773819076212, 'avg_role_model_mean_pred_loss': 1.0978048215903869e-05, 'avg_role_model_g_mag_loss': 0.03746633417904377, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033219303644727916, 'n_size': 80, 'n_batch': 20, 'duration': 19.31860041618347, 'duration_batch': 0.9659300208091736, 'duration_size': 0.2414825052022934, 'avg_pred_std': 0.03969717922154814}\n", + "Epoch 12\n", + "Train loss {'avg_role_model_loss': 0.0025715101775858782, 'avg_role_model_std_loss': 0.6255118230208495, 'avg_role_model_mean_pred_loss': 7.738168527621215e-06, 'avg_role_model_g_mag_loss': 0.0324334034929052, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002600309181434568, 'n_size': 320, 'n_batch': 80, 'duration': 95.97005558013916, 'duration_batch': 1.1996256947517394, 'duration_size': 0.29990642368793485, 'avg_pred_std': 0.08734330767765641}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002402786401216872, 'avg_role_model_std_loss': 1.9713338775481133, 'avg_role_model_mean_pred_loss': 4.400266065029967e-06, 'avg_role_model_g_mag_loss': 0.031274407636374235, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024293805909110233, 'n_size': 80, 'n_batch': 20, 'duration': 19.06187415122986, 'duration_batch': 0.9530937075614929, 'duration_size': 0.23827342689037323, 'avg_pred_std': 0.034856686973944305}\n", + "Stopped False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test ▄█▃▄▃▂▃▂▇▂▃▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▆▄▄▂▂▂▂▂▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▅█▃▃▂▃▁▁▇▃▃▃▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train █▃▅▂▃▂▂▁▁▃▃▁▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▇█▁▂▁▂▃▃▂▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▂█▆▃▂▂▂▄▂▂▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test ▄█▃▄▃▂▃▂▇▂▃▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▆▄▄▂▂▂▂▂▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test ▃█▂▂▂▁▂▁▅▂▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▄▂▂▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▃▁▂▂▅█▅▆▂▄▅▄▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▄█▄▅▂▅▄▆▄▃▁▄▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▅▇▆▆▇█▇█▆▁▁▅▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▆▆▆▆▆█▇▆▆▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▅▇▆▆▇█▇█▆▁▁▅▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▆▆▆▆▆█▇▆▆▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▅▇▆▆▇█▇█▆▁▁▅▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▆▆▆▆▆█▇▆▆▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00243\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.0026\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.03486\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.08734\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.03127\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.03243\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.0024\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00257\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 1e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 1.97133\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.62551\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.95309\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.19963\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.23827\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.29991\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 19.06187\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 95.97006\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 20\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/contraceptive/tab_ddpm_concat/4/wandb/offline-run-20240229_055259-0l1vy1nr\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_055259-0l1vy1nr/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 399, 'n_batch': 100, 'role_model_metrics': {'avg_loss': 0.005469150811475807, 'avg_g_mag_loss': 0.01941445733574633, 'avg_g_cos_loss': 0.006619339273136137, 'pred_duration': 1.4712414741516113, 'grad_duration': 4.747467756271362, 'total_duration': 6.218709230422974, 'pred_std': 0.08244121074676514, 'std_loss': 0.00028081866912543774, 'mean_pred_loss': 6.793341162847355e-05, 'pred_rmse': 0.07395371049642563, 'pred_mae': 0.05780092254281044, 'pred_mape': 0.13661588728427887, 'grad_rmse': 0.25440865755081177, 'grad_mae': 0.10522309690713882, 'grad_mape': 0.8996596336364746}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.005469150811475807, 'avg_g_mag_loss': 0.01941445733574633, 'avg_g_cos_loss': 0.006619339273136137, 'avg_pred_duration': 1.4712414741516113, 'avg_grad_duration': 4.747467756271362, 'avg_total_duration': 6.218709230422974, 'avg_pred_std': 0.08244121074676514, 'avg_std_loss': 0.00028081866912543774, 'avg_mean_pred_loss': 6.793341162847355e-05}, 'min_metrics': {'avg_loss': 0.005469150811475807, 'avg_g_mag_loss': 0.01941445733574633, 'avg_g_cos_loss': 0.006619339273136137, 'pred_duration': 1.4712414741516113, 'grad_duration': 4.747467756271362, 'total_duration': 6.218709230422974, 'pred_std': 0.08244121074676514, 'std_loss': 0.00028081866912543774, 'mean_pred_loss': 6.793341162847355e-05, 'pred_rmse': 0.07395371049642563, 'pred_mae': 0.05780092254281044, 'pred_mape': 0.13661588728427887, 'grad_rmse': 0.25440865755081177, 'grad_mae': 0.10522309690713882, 'grad_mape': 0.8996596336364746}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.005469150811475807, 'avg_g_mag_loss': 0.01941445733574633, 'avg_g_cos_loss': 0.006619339273136137, 'pred_duration': 1.4712414741516113, 'grad_duration': 4.747467756271362, 'total_duration': 6.218709230422974, 'pred_std': 0.08244121074676514, 'std_loss': 0.00028081866912543774, 'mean_pred_loss': 6.793341162847355e-05, 'pred_rmse': 0.07395371049642563, 'pred_mae': 0.05780092254281044, 'pred_mape': 0.13661588728427887, 'grad_rmse': 0.25440865755081177, 'grad_mae': 0.10522309690713882, 'grad_mape': 0.8996596336364746}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:20:40.213848Z", + "iopub.status.busy": "2024-02-29T06:20:40.213502Z", + "iopub.status.idle": "2024-02-29T06:20:40.217986Z", + "shell.execute_reply": "2024-02-29T06:20:40.217117Z" + }, + "papermill": { + "duration": 0.024761, + "end_time": "2024-02-29T06:20:40.219944", + "exception": false, + "start_time": "2024-02-29T06:20:40.195183", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:20:40.253763Z", + "iopub.status.busy": "2024-02-29T06:20:40.253512Z", + "iopub.status.idle": "2024-02-29T06:20:40.571348Z", + "shell.execute_reply": "2024-02-29T06:20:40.570260Z" + }, + "papermill": { + "duration": 0.337875, + "end_time": "2024-02-29T06:20:40.573901", + "exception": false, + "start_time": "2024-02-29T06:20:40.236026", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:20:40.609831Z", + "iopub.status.busy": "2024-02-29T06:20:40.609495Z", + "iopub.status.idle": "2024-02-29T06:20:40.883725Z", + "shell.execute_reply": "2024-02-29T06:20:40.882840Z" + }, + "papermill": { + "duration": 0.294533, + "end_time": "2024-02-29T06:20:40.885823", + "exception": false, + "start_time": "2024-02-29T06:20:40.591290", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAEVCAYAAABqhMkxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABROUlEQVR4nO2deVyU1f7HPzMDM2wyKNuAsrkgiIRekhFzySTRSxplWWZqXVNvqd0iS7mpWLcrpvbLMss2pUUDbTVxCVGvJbghKMoiGIgKg4KyKtvM9/fH4zw4MSwzDMwMnvfr9byYec73Oef7wMyHc55zzvcrICICg8Fg9CCExnaAwWAwDA0TNgaD0eNgwsZgMHocTNgYDEaPgwkbg8HocTBhYzAYPQ4mbAwGo8fBhI3BYPQ4mLAxGIweBxM2BoPR47DQ56JNmzZh3bp1UCgUCAoKwsaNGxESEtKq/c6dO7FixQoUFhZi0KBBePfdd/H3v/8dANDY2Ijly5djz549+PPPPyGVShEWFoY1a9bA3d2dr+PGjRtYvHgxfv31VwiFQkybNg0ffPAB7OzseJuzZ89i4cKFOHnyJJydnbF48WK88cYbHb4vlUqF4uJi9OrVCwKBQI/fDIPB6CqICNXV1XB3d4dQ2E6fjHQkPj6exGIxbdmyhc6fP0/z5s0jBwcHKi0t1Wp/9OhREolEtHbtWsrKyqLly5eTpaUlZWZmEhFRRUUFhYWFUUJCAuXk5FBqaiqFhIRQcHCwRj2TJk2ioKAgOnbsGP3+++80cOBAmjFjBl9eWVlJrq6uNHPmTDp37hx99913ZG1tTZ9++mmH7+3y5csEgB3sYIcJH5cvX273uywg0m0TvFwux4gRI/DRRx8B4Ho5Hh4eWLx4MZYtW9bC/qmnnkJtbS12797Nnxs5ciSGDRuGzZs3a23j5MmTCAkJwaVLl+Dp6Yns7GwMGTIEJ0+exP333w8A2LdvH/7+97/jypUrcHd3xyeffII333wTCoUCYrEYALBs2TL8/PPPyMnJ6dC9VVZWwsHBAZcvX4a9vb0uvxYGg9HFVFVVwcPDAxUVFZBKpW3a6jQUbWhoQFpaGqKjo/lzQqEQYWFhSE1N1XpNamoqoqKiNM6Fh4fj559/brWdyspKCAQCODg48HU4ODjwogYAYWFhEAqFOH78OB577DGkpqZi7NixvKip23n33Xdx8+ZN9O7du0U79fX1qK+v599XV1cDAOzt7ZmwMRgmSkceE+k0eVBWVgalUglXV1eN866urlAoFFqvUSgUOtnX1dVh6dKlmDFjBi8uCoUCLi4uGnYWFhbo06cPX09r7ajLtBEbGwupVMofHh4eWu0YDIZ5YVKzoo2NjZg+fTqICJ988kmXtxcdHY3Kykr+uHz5cpe3yWAwuh6dhqJOTk4QiUQoLS3VOF9aWgqZTKb1GplM1iF7tahdunQJBw8e1BgKymQyXLt2TcO+qakJN27c4OtprR11mTYkEgkkEklrt8tgMMwUnYRNLBYjODgYycnJiIyMBMBNHiQnJ2PRokVarwkNDUVycjJeeeUV/lxSUhJCQ0P592pRy8vLw6FDh+Do6NiijoqKCqSlpSE4OBgAcPDgQahUKsjlct7mzTffRGNjIywtLfl2Bg8erPX5GsOwqFQqNDQ0GNsNhpljaWkJkUjU6Xp0nhVNSEjAnDlz8OmnnyIkJAQbNmzAjh07kJOTA1dXV8yePRt9+/ZFbGwsACAlJQXjxo3DmjVrEBERgfj4eKxevRqnT5/G0KFD0djYiCeeeAKnT5/G7t27NZ6T9enTh58MmDx5MkpLS7F582Y0Njbi+eefx/3334/t27cD4CYcBg8ejIkTJ2Lp0qU4d+4c/vGPf+D999/H/PnzO3RvVVVVkEqlqKysZJMHOtDQ0ICCggKoVCpju8LoATg4OEAmk7WYJNDl+6nzAt2nnnoK169fx8qVK6FQKDBs2DDs27ePF6SioiKNxXOjRo3C9u3bsXz5cvz73//GoEGD8PPPP2Po0KEAgKtXr2LXrl0AgGHDhmm0dejQITz44IMAgG3btmHRokWYMGECv0D3ww8/5G2lUil+++03LFy4EMHBwXBycsLKlSs7LGoM/SAilJSUQCQSwcPDo/2FkwxGKxARbt26xT92cnNz07sunXtsPZmO/keoa1TiyIXr6O9sh4Eudq3a3Qs0NjYiPz8f7u7u7a4tYjA6Qnl5Oa5duwZfX1+NYakuPTb271UP3vzpHOZ/k4Ydp9gsqlKpBACN9YMMRmewsbEBwP3T1BcmbHoQ5s+tqdt/XgHW4eVge2sZhsIQnyUmbHow1tcZYgshLpXfQm5ptbHdYTAYf4EJmx7YSiwwdpATAGD/udJ2rBkM7QgEgja3Fpojq1atajEJaAyYsOnJxABu0e/+89q3azEY5kBcXBy/J9sQLFmyBMnJyQarT1+YsOlJmL8rhAIgq6QKl2/cMrY7DEaX0tHF13Z2di0W2BsDJmx60sdWjBCfPgBYr81c2bdvH0aPHg0HBwc4OjrikUcewcWLFwFw6y+XLl2qYX/9+nVYWlriyJEjAICSkhJERETA2toaPj4+2L59O7y9vbFhwwa9/MnMzMRDDz0Ea2trODo6Yv78+aipqeHLDx8+jJCQENja2sLBwQEPPPAALl26BAA4c+YMxo8fj169esHe3h7BwcE4depUm+0dPnwYzz//PB9NRyAQYNWqVQAAb29v/Oc//8Hs2bNhb2/PrwddunQpfH19YWNjg/79+2PFihUas5d/HYo+99xziIyMxPr16+Hm5gZHR0csXLiwUzOeHYEJWycIvzMc/e08e86mhohwq6HJKIeuM9S1tbWIiorCqVOnkJycDKFQiMceewwqlQozZ85EfHy8Rp0JCQlwd3fHmDFjAACzZ89GcXExDh8+jB9++AGfffZZiz3NuvgSHh6O3r174+TJk9i5cycOHDjAb1VsampCZGQkxo0bh7NnzyI1NRXz58/nZxBnzpyJfv364eTJk0hLS8OyZcv4rYWtMWrUKGzYsAH29vYoKSlBSUkJlixZwpevX78eQUFBSE9Px4oVKwAAvXr1QlxcHLKysvDBBx/g888/x/vvv99mO4cOHcLFixdx6NAhfPXVV4iLi0NcXJxev6eOoldocAbHxAAZ3vo1Cycv3UBZTT2c7NiG+tuNSgxZud8obWe9HQ4bccc/0tOmTdN4v2XLFjg7OyMrKwvTp0/HK6+8gj/++IMXsu3bt2PGjBkQCATIycnBgQMHNIKffvHFFxg0aJBevm/fvh11dXX4+uuvYWtrCwD46KOPMGXKFLz77ruwtLREZWUlHnnkEQwYMAAA4O/vz19fVFSE119/HX5+fgDQIT/EYjGkUikEAoHWQBEPPfQQXnvtNY1zy5cv5197e3tjyZIliI+PbzMEf+/evfHRRx9BJBLBz88PERERSE5Oxrx589r1UV9Yj60T9HWwRmBfKYiAA1ms12Zu5OXlYcaMGejfvz/s7e3h7e0NgBMJZ2dnTJw4Edu2bQMAFBQUIDU1FTNnzgQA5ObmwsLCAn/729/4+gYOHKh3wIXs7GwEBQXxogYADzzwAFQqFXJzc9GnTx8899xzCA8Px5QpU/DBBx+gpKSEt42KisILL7zA5wtRD6k7w92BXdUkJCTggQcegEwmg52dHZYvX46ioqI26wkICNDYQeDm5qZ3z7ajsB5bJwkPcEXm1UrsP6/A0yGexnbH6FhbipD1drjR2taFKVOmwMvLC59//jnc3d2hUqkwdOhQ/kH5zJkz8fLLL2Pjxo3Yvn07AgMDERgY2BWud4itW7fi5Zdfxr59+5CQkIDly5cjKSkJI0eOxKpVq/DMM88gMTERe/fuRUxMDOLj4/HYY4/p3d7dIguAF/a33noL4eHhkEqliI+Px3vvvddmPX8dEgsEgi4PmMB6bJ1E/ZztaH45quu69oGoOSAQCGAjtjDKocuK9fLycuTm5mL58uWYMGEC/P39cfPmTQ2bRx99FHV1ddi3bx+2b9/O99YAYPDgwWhqakJ6ejp/Lj8/v0UdHcXf3x9nzpxBbW0tf+7o0aMQCoUYPHgwf2748OGIjo5GSkoKhg4dyke3AQBfX1+8+uqr+O233/D4449j69at7bYrFov5bXHtkZKSAi8vL7z55pu4//77MWjQIH7ywtRgwtZJBrrYob+TLRqUKhzOvW5sdxgdpHfv3nB0dMRnn32G/Px8HDx4sEVuDltbW0RGRmLFihXIzs7GjBkz+DI/Pz+EhYVh/vz5OHHiBNLT0zF//nxYW1vrtSVo5syZsLKywpw5c3Du3DkcOnQIixcvxqxZs+Dq6oqCggJER0cjNTUVly5dwm+//Ya8vDz4+/vj9u3bWLRoEQ4fPoxLly7h6NGjOHnypMYzuNbw9vZGTU0NkpOTUVZWhlu3Wl+6NGjQIBQVFSE+Ph4XL17Ehx9+iJ9++knne+0OmLB1EoFAwBbrmiFCoRDx8fFIS0vD0KFD8eqrr2LdunUt7GbOnIkzZ85gzJgx8PTUfNTw9ddfw9XVFWPHjsVjjz2GefPmoVevXrCystLZHxsbG+zfvx83btzAiBEj8MQTT2DChAl8NjgbGxvk5ORg2rRp8PX1xfz587Fw4UIsWLAAIpEI5eXlmD17Nnx9fTF9+nRMnjwZb731Vrvtjho1Cv/85z/x1FNPwdnZGWvXrm3VdurUqXj11VexaNEiDBs2DCkpKfxsqanBwhbdhb6BJtOLbuKxj1NgJ7FA2oowSCw6HwHUXKirq0NBQQF8fHz0+kL3JK5cuQIPDw8cOHAAEyZMMLY7Zktrn6kuDTTJaElQPwe42ktQWlWPlIvlGD/Ypf2LGGbPwYMHUVNTg8DAQJSUlOCNN96At7c3xo4da2zX7nnYUNQACIUCPDyEiyD8GxuO3jM0Njbi3//+NwICAvDYY4/B2dkZhw8fhqWlJbZt2wY7OzutR0BAQLf5OHny5Fb9WL16dbf50d2wHpuBCA+Q4dtjRUjKKsU7kQSRkMUn6+mEh4cjPFz70papU6fyiYb+Sns7AgzJF198gdu3b2st69OnT7f50d0wYTMQI/s7wt7KAmU1DThddBMjvHvuh4bRPr169UKvXr2M7Qb69u1rbBeMAhuKGghLkRAT/Lnh6P5zbDjKYBgTvYRt06ZN8Pb2hpWVFeRyOU6cONGm/c6dO+Hn5wcrKysEBgZiz549GuU//vgjJk6cCEdHRwgEAmRkZGiUFxYW8tEH/nrs3LmTt9NWHh8fr88t6kV4wB1hy2IhwxkMY6KzsCUkJCAqKgoxMTE4ffo0goKCEB4e3urer5SUFMyYMQNz585Feno6IiMjERkZiXPnzvE2tbW1GD16NN59912tdXh4ePDRB9THW2+9BTs7O0yePFnDduvWrRp26sTO3cFYX2dILIS4fOM2sktYyHAGw2iQjoSEhNDChQv590qlktzd3Sk2Nlar/fTp0ykiIkLjnFwupwULFrSwLSgoIACUnp7erh/Dhg2jf/zjHxrnANBPP/3U/k20QmVlJQGgyspKvet44auT5LV0N/3fb7l612FO3L59m7Kysuj27dvGdoXRQ2jtM6XL91OnHltDQwPS0tIQFhbGnxMKhQgLC0NqaqrWa1JTUzXsAW42qTX7jpCWloaMjAzMnTu3RdnChQvh5OSEkJAQbNmypc0hYX19PaqqqjSOzhLOdiEwGEZHJ2ErKyuDUqnks76rcXV1hUKh/YusUCh0su8IX375Jfz9/TFq1CiN82+//TZ27NiBpKQkTJs2DS+99BI2btzYaj2xsbGQSqX84eHhobdPasL8XSASCpCjqEZROQsZzmidnpjMxVQwu1nR27dvY/v27Vp7aytWrMADDzyA4cOHY+nSpXjjjTe07v9TEx0djcrKSv64fLnzCZAdbMSQs5DhDDPB0MlcAC7kuEAgQEVFhUHr1QWdhM3JyQkikQilpZpBFUtLS7VG4AQAmUymk317fP/997h16xZmz57drq1cLseVK1dQX1+vtVwikcDe3l7jMARsOMpgGBedhE0sFiM4OFgjvZZKpUJycjJCQ0O1XhMaGtoiHVdSUlKr9u3x5ZdfYurUqXB2dm7XNiMjA71794ZE0r0huyfeWfaRVnQT16u1iyrD+LBkLq0nc6mvr8eSJUvQt29f2NraQi6X4/Dhw/y1ly5dwpQpU9C7d2/Y2toiICAAe/bsQWFhIcaPHw+ACw0lEAjw3HPP6fX76BS6zljEx8eTRCKhuLg4ysrKovnz55ODgwMpFAoiIpo1axYtW7aMtz969ChZWFjQ+vXrKTs7m2JiYsjS0pIyMzN5m/LyckpPT6fExEQCQPHx8ZSenk4lJSUabefl5ZFAIKC9e/e28GvXrl30+eefU2ZmJuXl5dHHH39MNjY2tHLlyg7fmyFmRdVM3fg7eS3dTduOXep0XaZMixkslYqovsY4h0qlk+/ff/89/fDDD5SXl0fp6ek0ZcoUCgwMJKVSSR999BF5enqS6q46N27cqHEuLCyMhg0bRseOHaO0tDQaN24cWVtb0/vvv9+h9nHXLH5NTQ25ubnR448/TpmZmZScnEw+Pj40Z84cIiJqbGwkqVRKS5Ysofz8fMrKyqK4uDi6dIn7fAUEBNCzzz5L2dnZdOHCBdqxYwdlZGS02X59fT1t2LCB7O3tqaSkhEpKSqi6upqIiF544QUaNWoUHTlyhPLz82ndunUkkUjowoULREQUERFBDz/8MJ09e5YuXrxIv/76K/3vf/+jpqYm+uGHHwgA5ebmUklJCVVUVHT0T0JEhpkV1VnYiJr/wGKxmEJCQujYsWN82bhx4/g/hpodO3aQr68vicViCggIoMTERI3yrVu3EoAWR0xMjIZddHQ0eXh4kFKpbOHT3r17adiwYWRnZ0e2trYUFBREmzdv1mrbGoYUto8O5pHX0t00+8vjna7LlGnxIayvIYqxN85RX9Ope7l+/ToBoMzMTLp27RpZWFjQkSNH+PLQ0FBaunQpERFlZ2cTADp58iRfnpeXRwD0ErbPPvuMevfuTTU1zfeQmJhIQqGQFAoFlZeXEwA6fPiw1rp69epFcXFxOt4x992TSqUa5y5dukQikYiuXr2qcX7ChAkUHR1NRESBgYG0atUqrXUeOnSIANDNmzd19ofIMMKm117RRYsW8WnB/srd3VU1Tz75JJ588slW63vuuec61F1dvXp1qxEJJk2ahEmTJrVbR3cRHuCKdftzkXKxDFV1jbC36r6Nz4yOkZeXh5UrV+L48eMoKyvj4/AXFRVh6NChfDKXMWPG8MlcPv30UwDdn8xl7NixfDKXhx9+GGFhYZg+fTrc3NwANCdz+eabbxAWFoYnn3ySz2alK5mZmVAqlfD19dU4X19fzydDfvnll/Hiiy/it99+Q1hYGKZNm4b77rtPr/a6ArYJvosY6NIL/Z1t8ef1WhzOvY6pQe7Gdql7sLQB/l1svLZ1gCVz0U5NTQ1EIhHS0tI0sksBXKZ3AHjhhRcQHh6OxMRE/Pbbb4iNjcV7772HxYsXG+ReO41efcUeiiGHokREa/Zmk9fS3fTStjSD1GeKmOvOg7KyMgKgMdT8/fffWzz3srW1pV27dtGQIUNozZo1vK16KHrq1Cn+XFcORbUxcuRIWrx4sdayp59+mqZMmdKuD9u2bSM7OzuNc7m5uS1+N+2xbNkyCgwMJCLuuToAKisr6/D1d9PtOw8YuqFe9nE45xrqGjuWCYjRPbBkLhzakrn4+vpi5syZmD17Nn788UcUFBTgxIkTiI2NRWJiIgDglVdewf79+1FQUIDTp0/j0KFDfHteXl4QCATYvXs3rl+/rjGz223oJak9FEP32JRKFcn/e4C8lu6m5Gzt/3XNHXPtsRERJSUlkb+/P0kkErrvvvvo8OHDLfYb79mzhwDQ2LFjW1xfXFxMkydPJolEQl5eXrR9+3ZycXGhzZs3d6j9v7Z19uxZGj9+PFlZWVGfPn1o3rx5/CylQqGgyMhIcnNzI7FYTF5eXrRy5UpSKpVUX19PTz/9NHl4eJBYLCZ3d3datGhRh/8m//znP8nR0VFjwq6hoYFWrlxJ3t7eZGlpSW5ubvTYY4/R2bNniYho0aJFNGDAAJJIJOTs7EyzZs3S6KG9/fbbJJPJSCAQtJhMbA9D9NhYMpe70DeZS1us/OUcvk69hKfu98C7T5jOw1VDwZK5NMOSuRgGQyRzYUPRLkY9HD2QXQqliv0P6UkcPHgQu3btQkFBAVJSUvD000+zZC4mAhO2LibEpw+k1pYor23AqcIbxnaHYUBYMhfThS336GK4kOEu+PH0Vew/Xwp5f0dju8QwECyZi+nChK0bCA+Q3RE2BVY84q/XrBnDvGDJXIwLG4p2A2MHOcPKUoirFbdxvrjzwSwZDEbbMGHrBqzFIozz5aKR9NSEymxynWEoDPFZYsLWTTTHaCttx9K8UG+5UW9DYjA6y61bXOTpzjyLZM/YuokJfq4QCQXILa1GYVktvJ1s27/IDLCwsICNjQ0fq0woZP8rGfpBRLh16xauXbsGBweHFvtUdYEJWzchtbHEyP59cDS/HPvPK7BgnH6RF0wNgUAANzc3FBQU8EEPGYzO4ODgoHeEbTVM2LqR8ABZjxM2gIusPGjQIDYcZXQaS0vLTvXU1DBh60YmDpFh5S/ncbqoAteq6uBi33O2IAmFwnt+SxXDdGAPRLoRmdQKQR4OAICk7J41icBgmBJM2LqZ8DuJXnra7CiDYUowYetm1Ms+Uu+EDGcwGIaHCVs3M8DZDgNd7NCoJBzKuWZsdxiMHolewrZp0yZ4e3vDysoKcrkcJ06caNN+586d8PPzg5WVFQIDA7Fnzx6N8h9//BETJ06Eo6MjBAIBMjIyWtTx4IMP8rkP1cc///lPDZuioiJERETAxsYGLi4ueP3119HU1KTPLXYpzcPRnrkLgcEwNjoLW0JCAqKiohATE4PTp08jKCgI4eHhuHZNe+8jJSUFM2bMwNy5c5Geno7IyEhERkbi3LlzvE1tbS1Gjx6Nd999t822582bh5KSEv5Yu3YtX6ZUKhEREYGGhgakpKTgq6++QlxcHFauXKnrLXY5fMjw3OssZDiD0RXoFLOXiEJCQmjhwoX8e6VSSe7u7hQbG6vVfvr06RQREaFxTi6X04IFC1rYFhQUEABKT09vUTZu3Dj617/+1apfe/bsaZH44pNPPiF7e3uqr69v5644DB0avDVUKhWFruZChied75khwxkMQ9NlyVwaGhqQlpaGsLAw/pxQKERYWBhSU1O1XpOamqphD3BxrFqzb4tt27bByckJQ4cORXR0NL+nTN1OYGAgXF1dNdqpqqrC+fPntdZXX1+PqqoqjaM7EAgEmMjvHWXDUQbD0OgkbGVlZVAqlRriAQCurq5QKLR/QRUKhU72rfHMM8/g22+/xaFDhxAdHY1vvvkGzz77bLvtqMu0ERsbC6lUyh8eHh46+dQZJt55znYguxRNSlW3tctg3AuYzc6D+fPn868DAwPh5uaGCRMm4OLFi3pnvI6OjtZIuVZVVdVt4hbi3QcONpa4easRaZdussi6DIYB0anH5uTkBJFIhNJSzcWlpaWlrW5alclkOtl3FHXY5fz8/DbbUZdpQyKRwN7eXuPoLixEQowf7AIAOJjLln0wGIZEJ2ETi8UIDg5GcnIyf06lUiE5ORmhoaFarwkNDdWwB4CkpKRW7TuKekmIm5sb305mZqbG7GxSUhLs7e0xZMiQTrXVVYz3uyNs2UzYGAyDouvMRHx8PEkkEoqLi6OsrCyaP38+OTg48LORs2bNomXLlvH2R48eJQsLC1q/fj1lZ2dTTEwMWVpaUmZmJm9TXl5O6enplJiYSAAoPj6e0tPTqaSkhIiI8vPz6e2336ZTp05RQUEB/fLLL9S/f3+NJLZNTU00dOhQmjhxImVkZNC+ffvI2dmZoqOjO3xv3TUrqqaitoH6RyeS19LdVFRe2y1tMhjmii7fT70ywW/cuJE8PT1JLBZTSEgIHTt2jC8bN25ci8zPO3bsIF9fXxKLxRQQEECJiYka5Vu3biUALQ51VuqioiIaO3Ys9enThyQSCQ0cOJBef/31FjdYWFhIkydPJmtra3JycqLXXnuNGhsbO3xf3S1sRERPbk4hr6W7Ke5oQbe1yWCYIywTvJ50RSb49vj0fxcRuzcH43yd8dU/QrqlTQbDHGGZ4M2Ih+48Z0v9sxy3Gkxv+xeDYY4wYTMyA13s0K+3NRqaVDiaX25sdxiMHgETNiMjEAgwQT07yqJ9MBgGgQmbCaBe9nEo5xrLz8lgGAAmbCbAyP6OsLYUQVFVh6wSlimewegsTNhMACtLER4Y6ASALdZlMAwBEzYTYYI/217FYBgKJmwmgnrfaMblCpTX1BvZGwbDvGHCZiLIpFYY4mYPIi6yLoPB0B8mbCYEG44yGIaBCZsJoV72cST3OhpZ8EkGQ2+YsJkQQf0c4GgrRnV9E04V3jS2OwyG2cKEzYQQCQUYN9gZAHAwh2WKZzD0hQmbiTHBj8uFwLZXMRj6w4TNxBjj6wQLoQAXr9fiUnmtsd1hMMwSJmwmhr2VJUZ49wHAem0Mhr4wYTNBHmLRPhiMTsGEzQRRL/s4/ucN1Naz4JMMhq4wYTNBBjjbwsvRBg1KFf7ILzO2OwyG2cGEzQQRCATNOUdZtA8GQ2f0ErZNmzbB29sbVlZWkMvlOHHiRJv2O3fuhJ+fH6ysrBAYGIg9e/ZolP/444+YOHEiHB0dIRAI+Jyham7cuIHFixdj8ODBsLa2hqenJ15++WVUVlZq2AkEghZHfHy8PrdodNTbqw7lXoNKxYJPMhi6oLOwJSQkICoqCjExMTh9+jSCgoIQHh6ukaj4blJSUjBjxgzMnTsX6enpiIyMRGRkJM6dO8fb1NbWYvTo0Xj33Xe11lFcXIzi4mKsX78e586dQ1xcHPbt24e5c+e2sN26dStKSkr4IzIyUtdbNAlCfPrARizCtep6nC9mwScZDJ3QNbdfSEgILVy4kH+vVCrJ3d2dYmNjtdpPnz6dIiIiNM7J5XJasGBBC9uCggICQOnp6e36sWPHDhKLxRp5QwHQTz/91LEb0YIx8oq2xfyvT5LX0t20IemCsV1hMIyOLt9PnXpsDQ0NSEtLQ1hYGH9OKBQiLCwMqampWq9JTU3VsAeA8PDwVu07ijq3oIWFhcb5hQsXwsnJCSEhIdiyZUubOQTq6+tRVVWlcZgS/LIPFu2DwdAJnYStrKwMSqUSrq6uGuddXV2hUCi0XqNQKHSy76gf//nPfzB//nyN82+//TZ27NiBpKQkTJs2DS+99BI2btzYaj2xsbGQSqX84eHhobdPXYF6AuHM5Qpcr2bBJxmMjmLRvolpUVVVhYiICAwZMgSrVq3SKFuxYgX/evjw4aitrcW6devw8ssva60rOjoaUVFRGnWbkri52FshsK8UmVcrcTj3Gp6833R8YzBMGZ16bE5OThCJRCgt1Yw8UVpaCplMpvUamUymk31bVFdXY9KkSejVqxd++uknWFpatmkvl8tx5coV1Ndr7+1IJBLY29trHKbGeLYLgcHQGZ2ETSwWIzg4GMnJyfw5lUqF5ORkhIaGar0mNDRUwx4AkpKSWrVvjaqqKkycOBFisRi7du2ClZVVu9dkZGSgd+/ekEgkOrVlSqiTKf+eV4aGJhZ8ksHoCDoPRaOiojBnzhzcf//9CAkJwYYNG1BbW4vnn38eADB79mz07dsXsbGxAIB//etfGDduHN577z1EREQgPj4ep06dwmeffcbXeePGDRQVFaG4uBgAkJubC4Dr7clkMl7Ubt26hW+//VbjQb+zszNEIhF+/fVXlJaWYuTIkbCyskJSUhJWr16NJUuWdO43ZGQC+0rhZCdBWU09ThXewKg7afoYDEYb6DPtunHjRvL09CSxWEwhISF07NgxvmzcuHE0Z84cDfsdO3aQr68vicViCggIoMTERI3yrVu3EoAWR0xMDBERHTp0SGs5ACooKCAior1799KwYcPIzs6ObG1tKSgoiDZv3kxKpbLD92Vqyz3ULNmRQV5Ld9Pbv543tisMhtHQ5fspIGpjPcQ9RlVVFaRSKb+UxFTYm1mCF7edRn8nWxxc8qCx3WEwjIIu30+2V9QMGD3ICZYiAf4sq0VBGQs+yWC0BxM2M6CXlSVCfFjwSQajozBhMxP4aB8syQuD0S5M2MyECf7c7o0TBTdQw4JPMhhtwoTNTPBxsoWPky0alYQ/8q4b2x0Gw6RhwmZGqDfFJ7PgkwxGmzBhMyPUwnYo9zoLPslgtAETNjNihHcf2EksUFZTj8yrle1fwGDcozBhMyPEFkKMGcRtqWLLPhiM1mHCZmawaB8MRvswYTMz1OvZMq9W4lpVnZG9YTBMEyZsZoZzLwmC+kkBAIdz2bIPBkMbTNjMkIf8uMW6yWwXAoOhFSZsZoh62ccfeWWob1Ia2RsGw/RgwmaGBLjbw6WXBLUNSpwouGFsdxgMk4MJmxkiFAru2hTPZkcZjL/ChM1MuXvZB4sVymBowoTNTBk9yAlikRCXym/hTxZ8ksHQgAmbmWInsYC8Pxd88pABhqMqFbGJCEaPwewSJjOaecjPBb/nlSE5+xpeGNO/w9fVNymRV1qDrOIqZJVUIau4CtklVbjVqMS2F+QY2d+xC71mMLoevXpsmzZtgre3N6ysrCCXy3HixIk27Xfu3Ak/Pz9YWVkhMDAQe/bs0Sj/8ccfMXHiRDg6OkIgECAjI6NFHXV1dVi4cCEcHR1hZ2eHadOmtUjEXFRUhIiICNjY2MDFxQWvv/46mpp6blBG9bKPk4U3UFXXqNWm8lYjUi+W44vf/0TUjgxM2nAEASv345GNf+CNH84iLqUQJwpvoLq+CUoVIf5EUXfeAoPRJejcY0tISEBUVBQ2b94MuVyODRs2IDw8HLm5uXBxcWlhn5KSghkzZiA2NhaPPPIItm/fjsjISJw+fRpDhw4FANTW1mL06NGYPn065s2bp7XdV199FYmJidi5cyekUikWLVqExx9/HEePHgUAKJVKREREQCaTISUlBSUlJZg9ezYsLS2xevVqXW/TLPBytMUAZ1tcvF6L3y+UIchDyvfCzhdzPbGrFbe1Xiu1tkSAuz2GuNljiDuX8SdqxxkczLmGRqUKliL2lIJhvuicfk8ul2PEiBH46KOPAHCZ4D08PLB48WIsW7ashf1TTz2F2tpa7N69mz83cuRIDBs2DJs3b9awLSwshI+PD9LT0zFs2DD+fGVlJZydnbF9+3Y88cQTAICcnBz4+/sjNTUVI0eOxN69e/HII4+guLgYrq7cyvzNmzdj6dKluH79OsRicbv3Zqrp99riv4lZ+Pz3AggFQGsh2vr1tr4jYlIMceeEzF1qBYFAwNsoVYQR/z2AG7UN2D5PjlEDWGJmhmmhy/dTpx5bQ0MD0tLSEB0dzZ8TCoUICwtDamqq1mtSU1MRFRWlcS48PBw///xzh9tNS0tDY2MjwsLC+HN+fn7w9PTkhS01NRWBgYG8qKnbefHFF3H+/HkMHz68Rb319fWor6/n36uzy5sTfw90w+e/F0BFgIVQgEGuvTDEzZ4TMnd7+LvZQ2pt2W49IqEAD/m54Pu0K0jKKmXCxjBrdBK2srIyKJVKDfEAAFdXV+Tk5Gi9RqFQaLVXKBQdblehUEAsFsPBwaHVelprR12mjdjYWLz11lsd9sMUGe7ZG4kvjwYRMMjVDhILkd51PTzEFd+nXcGB7FKsfGSIRo+OwTAn7ukHKdHR0aisrOSPy5cvG9slvQhwl2JoX2mnRA0AxgxygsRCiMs3biO3tNpA3jEY3Y9Owubk5ASRSNRiNrK0tBQymUzrNTKZTCf71upoaGhARUVFq/W01o66TBsSiQT29vYax72MjdgCowdyQ9Ck8yxyCMN80UnYxGIxgoODkZyczJ9TqVRITk5GaGio1mtCQ0M17AEgKSmpVXttBAcHw9LSUqOe3NxcFBUV8fWEhoYiMzMT1641L1ZNSkqCvb09hgwZ0uG27nUeHsIN3w9kM2FjmDGkI/Hx8SSRSCguLo6ysrJo/vz55ODgQAqFgoiIZs2aRcuWLePtjx49ShYWFrR+/XrKzs6mmJgYsrS0pMzMTN6mvLyc0tPTKTExkQBQfHw8paenU0lJCW/zz3/+kzw9PengwYN06tQpCg0NpdDQUL68qamJhg4dShMnTqSMjAzat28fOTs7U3R0dIfvrbKykgBQZWWlrr+WHkNp1W3yXrabvJbuJkXlbWO7w2Dw6PL91FnYiIg2btxInp6eJBaLKSQkhI4dO8aXjRs3jubMmaNhv2PHDvL19SWxWEwBAQGUmJioUb5161YC0OKIiYnhbW7fvk0vvfQS9e7dm2xsbOixxx7TED4iosLCQpo8eTJZW1uTk5MTvfbaa9TY2Njh+2LCxhG56Q/yWrqbvj1WaGxXGAweXb6fOq9j68mY4zq2rmDToXys25+LBwc7I+75EGO7w2AA0O37eU/PijK0M/HOc7aU/HLU1PfcLWmMngsTNkYLBrrYwdvRBg1KFX6/wBLGMMwPJmyMFggEAoT5c722pCw2O8owP5iwMbSiXvZxMPcampQqI3vDYOgGEzaGVoK9esPBxhIVtxpx6tJNY7vDYOgEEzZjcuJzYNdioKnB2J60wEIk5OO9HWDDUYaZwYTNWNSWAfuWAae/BnL3tG9vBNSzo0nZpSxhDMOsYMJmLM4mAKo7SymyfjGuL60wZpAzxBZcwpi8azXGdofB6DBM2IwBEZC+rfn9hf1Ao/ZIt8bEVmKBBwZw+Q/Y7CjDnGDCZgyK04Fr5wGRBLCTAY21QH5y+9cZgbAhbNkHw/xgwmYM0r/lfvpPAYZO415n7zKeP22gXs+WcbkC16rrjOwNg9ExmLB1N423gczvudfDnwWGTOVe5+4Fmupbv85IuNpbIaifFACQnN35/KUMRnfAhK27yUkE6isBqQfgMw7oF8INR+urgD8PG9s7rTzMhqMMM4MJW3eT/g33c9hMQCjkDnWvLcs0h6MPD+EiEP+RX4ZbDWxTPMP0YcLWndy8BPz5P+71sBnN5/3vCFvObkCpPfGxMfF1tYNHH2s0NKlw5EKZsd1hMNqFCVt3cuY7AAT4jAV6ezef9xoF2DgBdRVAwREjOdc6AoEAD/tzvTYWMpxhDjBh6y5UKiDjztq14bM0y4QiboYUMN3Z0SHc9qqDOdegbC0zM6N1sncD7w8Fio4Z25N7AiZs3UXh70BFESCRNovY3Qx5lPuZvRtQmt5zrBDvPpBaW+JGbQPS2KZ43fnjfaDyMnDsY2N7ck/AhK27UK9dC5wGWFq3LPceDVj3Bm6VAUUp3etbB9DYFM+Go7pRrQCunuJe5yeb5LKengYTtu7gdkXzEHPYs9ptRJaAXwT32kT3jt4dfJJtiteB3L3NrxtquN47o0vRS9g2bdoEb29vWFlZQS6X48SJE23a79y5E35+frCyskJgYCD27NGMZkFEWLlyJdzc3GBtbY2wsDDk5eXx5YcPH4ZAINB6nDx5EgBQWFiotfzYMRN4pnHuB6CpDnD2B/r+rXW7IZHcz+xfuWdyJsa4wc4Qi4QoKKvFxeu1xnbHfFBHb7G0ufN+b+u2DIOgs7AlJCQgKioKMTExOH36NIKCghAeHq6RqPhuUlJSMGPGDMydOxfp6emIjIxEZGQkzp07x9usXbsWH374ITZv3ozjx4/D1tYW4eHhqKvjtvCMGjUKJSUlGscLL7wAHx8f3H///RrtHThwQMMuODhY11s0PPykwbOAQNC6nc847hlcTSlw+Xj3+KYDdhILjGSb4nWjvqZ5ic+4pdzP3L1cIARG16Frbr+QkBBauHAh/16pVJK7uzvFxsZqtZ8+fTpFRERonJPL5bRgwQIiIlKpVCSTyWjdunV8eUVFBUkkEvruu++01tnQ0EDOzs709ttv8+cKCgoIAKWnp+t6SzxdkldUcZ4oxp7orT5E1dfat/9hPme/Z6nhfDAgX6cWktfS3fT4x0eN7Yp5cP5n7u+5IYio4RbROzLufXGGsT0zO3T5furUY2toaEBaWhrCwsL4c0KhEGFhYUhNTdV6TWpqqoY9AISHh/P2BQUFUCgUGjZSqRRyubzVOnft2oXy8nI8//zzLcqmTp0KFxcXjB49Grt2tb10or6+HlVVVRqHwVH31nwnAXbO7dvzs6O7THI4GubPTSCcLrqJ69XsIXi75NwZhvpFcJNGAx7i3rPhaJeik7CVlZVBqVTC1dVV47yrqysUCoXWaxQKRZv26p+61Pnll18iPDwc/fr148/Z2dnhvffew86dO5GYmIjRo0cjMjKyTXGLjY2FVCrlDw8Pj1Zt9ULZCJyJ514Pb2XS4K8MeAgQ2wFVV4Hi04b1xwC4Sa0R2FcKIuBgDhuOtomyCcjbz70e/Pc7PydzP000anJPwexmRa9cuYL9+/dj7ty5GuednJwQFRUFuVyOESNGYM2aNXj22Wexbt26VuuKjo5GZWUlf1y+fNmwzl7Yzy3fsHMFBj7csWssrQDfcO511s+G9cdANG+KZ9E+2qQoFbh9E7DuA3jIuXODwgEIgJIzQOVVo7rXk9FJ2JycnCASiVBaqvmfurS0FDKZTOs1MpmsTXv1z47WuXXrVjg6OmLq1Knt+iuXy5Gfn99quUQigb29vcZhUNRr14KeBkQWHb9OPRzN+sUkHzKrl338kX8dtxuURvbGhFH3ynwnNf/97ZwBjxDu9QU2HO0qdBI2sViM4OBgJCc3R3tVqVRITk5GaGio1mtCQ0M17AEgKSmJt/fx8YFMJtOwqaqqwvHjx1vUSUTYunUrZs+eDUtLy3b9zcjIgJubW4fvz6BUK4C837jXra1da42BD3NLAyqKuP/sJoa/Wy/0dbBGXaMKf+SzTfFaIeJCVAGA3981y/jhKBO2rkLnoWhUVBQ+//xzfPXVV8jOzsaLL76I2tpa/kH+7NmzER0dzdv/61//wr59+/Dee+8hJycHq1atwqlTp7Bo0SIA3AbrV155Be+88w527dqFzMxMzJ49G+7u7oiMjNRo++DBgygoKMALL7zQwq+vvvoK3333HXJycpCTk4PVq1djy5YtWLx4sa63aBjOxAOk5OKtOfvqdq3YBhh4ZzLFBBfrCgSCu4aj2p+D3vNcywIqLgEWVs0TBmoG31mIXXAEqK/uft/uAXQYH3E89dRTuH79OlauXAmFQoFhw4Zh3759/MP/oqIiCIXNejlq1Chs374dy5cvx7///W8MGjQIP//8M4YOHcrbvPHGG6itrcX8+fNRUVGB0aNHY9++fbCystJo+8svv8SoUaPg5+en1bf//Oc/uHTpEiwsLODn54eEhAQ88cQTut5i5yHSXLumD0Me5WZGs34BJqxse/2bEXh4iCviUgqRnM1tihcJTcs/o6OeDe3/ICC21SxzGgT0GQDcuAhcPNj86IFhMAREJvgQx0hUVVVBKpWisrKyc8/bLp8AvrwznHwtF7DSo676amDtAEBZD7yYArgG6O9PF9CoVOFv/0lCdV0TfngxFMFefYztkmnx2YNc0p4pHwLBc1qW738TSP0ICJoBPLa5290zR3T5fprdrKhZoI6SOyRSP1EDAEkvkx6OWoqEGD+YW9P2G9uFoElVMSdqEDQ/T/sr6uUfF/abZDQXc4cJm6FpqAXO/ci9Hj6zc3XxIcNNT9iA5mUfB5iwaaKeDe03ArBz0W7jIeeiudy+AVxpe681Q3eYsBmarF1cBIfePoDXA52ry3cSILQErucA13MN458BGTfYGZYiAS5er8Wf11mmeB5+t8HfW7cRWdxZ0wa2WLcLYMJmaNRr14bP7PwDf2sHYMB47rUJJnqxt7LEyP5sU7wGdVXN4d3Vs5+twZZ9dBlM2AxJ+UXg0h8ABEDQM4ap099MhqMs+CRH/gFA1Qg4Dmx/mc/ACYBIDJTnA2V5bdsydIIJmyHJ2M79HPAQIO1rmDr9IgCBCCjN5ITTxJhwZxdC2qWbKK9hm+L5YeXgNoahaiS9AO8xmtcxDAITNkOhUt7JQgX9165pw6YPl9UKMMlEL30drBHgbg8VcYle7mmUjc27TfzaGYaqYcPRLoEJm6H48xAXkcO6d8c/1B3l7r2jJsjdIcPvaS4dBeoquVSK/UZ07Bq1sF0+DtSy7WmGggmboeCTtUwHLCSGrdvvEUAg5NZG3bxk2LoNgPo52+95ZahrvIc3xefcteldKOrYNdJ+gOw+gFTNvT1Gp2HCZghu3Wje8GzIYagaO+fmpSPZvxq+/k4S4G4Pd6kVbjcqcfRe3RRP1PycrK1lHtpQP49jz9kMBhM2Q5D5PaBsAGSBgNt9XdOGCQ9HBQIBwobc48NRRSaXN9TCGug/Xrdr1cPR/INAY53hfbsHYcJmCNRbqP6a4d2Q+D3C/bxywiQDFDYv+7gG1b2YKV7d2xownovOogtuQUAvd6CxlqXmMxBM2DpLyRlAcZZbjxT4ZNe1Y+8GeIzkXpvgcFTu44heEguU1dQj40qFsd3pftSPIjqyzOOvCAQsZLiBYcLWWdLvhCfyi+CWZnQldyd6MTHEFkKMG8wlq7nn9o5WXOb+uUHATRzoA/+cjaXmMwRM2DpDUz2QuYN7rWuUXH3wn8L9vJQCVJueeKiHo/vOK9CoNL0MW12Geg2ah7xjmci04TOGS+JTXQKUZBjMtXsVJmydIXcPl6yjl3vzns6uxMED6BsMgIAc0xuOPjjYBdaWIvx5vRZzvzqF2vp7JBxPbishwHXBQsJS8xkQJmydQb12bdgzHV+31Fn42VHTG45KrS2xaeZwWFkKceTCdTz92bGen3v0dgVQ+Af3ur1N7+3Bln0YDCZs+lJ5Fci/k4BmmIE2vHcE9ab4wj9McqX6Q36u+G7eSPSxFSPzaiUe/+QoCspqje1W15F/AFA1AU6+gNPAztU1aCK3EFuRyT23Y+gNEzZ9OfMdAAK8RgOOA7qv3T4+d1aqK5tn4kyM4Z698cOLo+DZxwaXb9zGtE9SkF5009hudQ2dmQ39K7aOzTPfF/Z1vr57GCZs+kCkGXetuzHhxbpqfJxs8cOLoxDYV4obtQ2Y8fkxJPe00EZNDVyPDTDc/mC27MMg6CVsmzZtgre3N6ysrCCXy3HiRNuhjXfu3Ak/Pz9YWVkhMDAQe/Zo/tGICCtXroSbmxusra0RFhaGvDzN+FTe3t4QCAQax5o1azRszp49izFjxsDKygoeHh5Yu3atPrfXPpdSgJsF3CyWMTIMDYnkfhb8j5u8MFGce0kQP38kxvk6o65RhXlfn8J3J4qM7ZbhKPwdqK8CbF2Avvcbpk51z6/gdy5oJUMvdBa2hIQEREVFISYmBqdPn0ZQUBDCw8Nx7Zr2kDUpKSmYMWMG5s6di/T0dERGRiIyMhLnzp3jbdauXYsPP/wQmzdvxvHjx2Fra4vw8HDU1WluL3n77bdRUlLCH3fnDK2qqsLEiRPh5eWFtLQ0rFu3DqtWrcJnn32m6y22j7q3NvTxlqnVugOngYBLAPdsx8Rn0GwlFvhizv14IrgfVARE/5iJ95MuoEckR+Njr00ChAYa/DgNBBwHccEqLya3b8/QDulISEgILVy4kH+vVCrJ3d2dYmNjtdpPnz6dIiIiNM7J5XJasGABERGpVCqSyWS0bt06vryiooIkEgl99913/DkvLy96//33W/Xr448/pt69e1N9fT1/bunSpTR48OAO31tlZSUBoMrKyrYNK64Q/W8d0dXTHa7b4ByKJYqxJ9o23TD1NTUQVZUQqVSGqe8vqFQqWr8/h7yW7iavpbvpjZ1nqLFJ2SVtdQsqFdF7/tzfIGevYevev5yr94d5hq3XzOnw95OIdPo309DQgLS0NISFhfHnhEIhwsLCkJqaqvWa1NRUDXsACA8P5+0LCgqgUCg0bKRSKeRyeYs616xZA0dHRwwfPhzr1q1DU1PzOqnU1FSMHTsWYrFYo53c3FzcvKl9uFZfX4+qqiqNo0NI+wJjlwDuwztm3xWoh8AXD+o/ZLl1Azi7A/j+H8C6AcB7g4FPRgFHPwCqSgznK7iN8q9NHIx3IodCKAASTl3GvK9P4VaDma51K8ng4u9Z2gD9xxm2bpaar9PoJGxlZWVQKpV81nc1rq6uUCgUWq9RKBRt2qt/tlfnyy+/jPj4eBw6dAgLFizA6tWr8cYbb7Tbzt1t/JXY2FhIpVL+8PDwaPXeTQ5nP26JgbKB+wJ0BCLgWg7wx/vAlkmcmP04Dzj3AxcgEQCuZQFJK4H3hwDfPA6c3Qk03DKY28+O9MLmZ4MhsRDiUO51zPjsGMrMMaS4OvbagIcAS2vD1u0RAlj3AeoqgMvHDFv3PYKFsR3oKFFRUfzr++67D2KxGAsWLEBsbCwkEv0CO0ZHR2vUW1VVZT7iJhBwa9p+Xw9k/Qzc18oG/KZ6bs3bhf3cEoKKvwSqdB0K+IZzexwdB3IzrWfiuS/UxWTuEPcCAh7lspZ7jur086SJATJsnzcSc786iTNXKvHEJyn46h8h8HI0wvNKfeFjrxk4WjLALfb2nQSc2c49Q/Uebfg2ejg6CZuTkxNEIhFKSzWn7UtLSyGTybReI5PJ2rRX/ywtLYWbm5uGzbBhw1r1RS6Xo6mpCYWFhRg8eHCr7dzdxl+RSCR6i6JJMORRTtjyDwD1NYDEjjtfXcpFY72wD7h4iAuHo0Yk4XIo+IZzh4OnZp33P88d5ReBswncer2KIm7CJP1bzv6+p4Ggpzu1fi/Yi1vrNmfLCRSW38LjH6dgy3MjEOThoHed3cbNQqD0HLeYVp0b1NAMnswJW04iMPGdzqdyvMfQ6V+vWCxGcHAwkpObZ2tUKhWSk5MRGhqq9ZrQ0FANewBISkri7X18fCCTyTRsqqqqcPz48VbrBICMjAwIhUK4uLjw7Rw5cgSNjY0a7QwePBi9e/fW5TbNB1kgl5i5qQ448Snwv7XAZ+OB93yBXYuAnN2cqNm5An+bDTz9HbC0AHj2eyBkXktRuxvHAcD4fwMvnwGe38tdL7HnRO7IWmDj34AvHgZObdF7yckAZzv8+OIoDHGzR3ltA57+7BgO5ZpBQhj1TLRnKLeotisY8BAXCutmAVB2oWva6MnoOjMRHx9PEomE4uLiKCsri+bPn08ODg6kUCiIiGjWrFm0bNky3v7o0aNkYWFB69evp+zsbIqJiSFLS0vKzMzkbdasWUMODg70yy+/0NmzZ+nRRx8lHx8fun37NhERpaSk0Pvvv08ZGRl08eJF+vbbb8nZ2Zlmz57N11FRUUGurq40a9YsOnfuHMXHx5ONjQ19+umnHb43XWZdTIbfVnIzaH89No/lZk6vniZSGmj2seEW0dmdRN9MI1rl0NzW205ECbOIcvZws6s6UnW7gZ794hh5Ld1N/aMTKeFkkWH87Sq2RnD3fXRj17bzzTSund//r2vbMRN0+X7qLGxERBs3biRPT08Si8UUEhJCx44d48vGjRtHc+bM0bDfsWMH+fr6klgspoCAAEpMTNQoV6lUtGLFCnJ1dSWJREITJkyg3NxcvjwtLY3kcjlJpVKysrIif39/Wr16NdXV1WnUc+bMGRo9ejRJJBLq27cvrVmzRqf7Mkthu5ZD9B8XondkRNtnEJ2KI6os7vp2q0qIjn5ItClUU1Df7U+UtIqovkan6uoblfRKfDq/HOSDAxdI1UVLTzpFbTnRqt7cvZZf7Nq2TnzBtfPFw13bjpmgy/dTQNQTVkoahqqqKkilUlRWVsLe3t7Y7nScuipu2GJpZZz2S85yEw6ZO4Da69w5By9gyobmUDwdgIiwdn8uPjnMJYYWCABbsQVsxCLYSixgKxHBRmwBO8mdc2IL/rytxAK2Yq787nPOdhJ49NExVHdbnEkAfpoPOPsDC7t4xrLyKjc7DQGwJE//WG89BF2+n2YzK8poAysji7Dbfdzx8NtcbLL9b3Kzr988BgQ9A4T/t0PRhQUCAZZO8oOb1Aqr92SjrlGFmvom1NQ3AZ0If9Tf2RYT/Fwwwd8V93v1hoWoE7O6hoi91lGkfQG3Ydyaubz9XZMBrYfCemx3YbY9NlOjvho4+A5w/FMABNg6A5PfBQIe7/DsXl2jEtV1Taitb0JtQxNq65WobWjCrXrlXeeaUNtw5329ErcaOBG81dBsU1JRh6a7kstIrS3x4GBnPOTnggd9XSC1sez4fTXVA2v7Aw01wAsHgX7BOv5i9ODwu8Dh1Vwyn6e3dX17Jowu308mbHfBhM3AXD4B7FoMXM/h3vtOBiLe43oi3UR1XSOOXChDcnYpDuVew81bzbPmIqEAI7x7I8zfFQ/5uaC/s13bleUlAdueAOxkQFS24faHtkXJWeDTMdwOhzf+NPxiYDOCCZueMGHrApoagD/+DziyntvYLe4FhMUA98/tHmG4C6WKkF50EweyryE5uxR512o0yvs72WKCvwse8nPFCG8tQ9ZfXwHStgLBz3PPD7sDIuD9oUDVFeCZHdzaw3sUJmx6woStC7mWw/XertwJceUxEpi6EXD2NZpLReW3kJxTiuTsazheUI5GZfNXwd7KAg8OdsEE/ztDVisR8H/+QI0Ct6cnoMZjPOoalbjVoMTtRiVuNyg13zcqcbuhCbcbVLjdyJUBQHiADCP794FAlwW3iUuAk58Dwc8BUz7o9H3/WVqJY8dTMDRgKO4bYCY7bcCETW+YsHUxKhVw6kvgwCruOZVIDIx9A3jgX4CFuN3LuxJ+yJpTikM5LYesI8UF2IZ/o4as8Lf6T9EAHZ7N/QU/WS88N8obkcP7wsqyA7ky8pOBbx/v1BBYpSKknjmPksOfY2TFbvQTlKGarJHb93EEPbEMln3aWKxtIjBh0xMmbN1ExWUgMYrb9gUALkO43ls/AwRrbLgFlJ4HFGe43AH1NYD3A9yyk97eHapC25B1iUUCFln8gkRlCBY2vgIAsLYUwUYsgpWlCNZiEazv/qkuEze/vl5dj18yinH7Tu/NwcYSM0I8MWukF9wd2nh21lQPrB0ANFQD8w7eyVTWMapu1yPltx9gffZrjGo6AUsB13YTRLAA91oJIW4NehS9xr8CuA/rcN3dDRM2PWHC1o0QcVFF9r4B3CoHIABGvgiMf7N5z2t71JY3C1jJWS5pcXk+QK3kNO3TnxO4AQ8B3mM6vEymuOI2HL8eB8mNXNT8/WNYDH8aEguhbsPJO1TeasSOU5fxVWohrty8DYDrEYYHuOK5UT4Y4d1be7075nDBDsa+Djy0vN12CgoLkLf/U/gV/whPQfMe6su2gbAeNQ9OI57E8UO7QKkfYSQy+XLyHgPBqJeBgWHd/gy0PZiw6QkTNiNQWw789uad5DgApJ7AlPe5L5YaIm5dXMlZTsQUZ7nX1cXa67Rz5fbRyu4DLKy4EOqXj3MRh9UIREC/EcDACZzQuQ9vPYXijT+BD4dz17ye36E1ee2hVBGSs0sRl1KIlIvl/PkhbvZ47gFvTA1y1xymqhcGuw4FXjyqvU6lChlHdqHxxBb87dYfEN/pndXAFlc8p8Jz4kLY9AvUuEZRWYeN237A/SXb8IjwGN+jg9NgYNQiIHC68RZ+/wUmbHrChM2I5B8Afn0VqLyTE2HoE4CdS7OY1Vdqv65Pf07A3O7jfsruA3q5trSrr+bCN108yB3l+ZrlVlLAZ1xzj663V3NZykec+HqPAZ7bbZj7vYscRRW+SinET+lXUdfI9Tb72IrxTIgnnh3pBZnUigsKum4gl53sX2c1/KsqVyB772a4X0yABzWL/Z8Sf6j+9hwGPPgsBG30glUqwlephYjb+weewV48Y3EQvcD1JmHrDIQsAEbMNYigdwYmbHrChM3I1NcAh/4LHPsEwF8+lkJLwMX/LyI2FJD00q+tm5eAPw9xIvfn4eZAm2r6DGgWuaMbuB7fpDXccLmLuFnbgIRTl/FN6iVcreCExUIowKShMjz/gDf+dvBZCC4dBSavBULm43LGAZT/71MMuXkIYgHXG62BNS64TIL7hJcgGxyiU/t5pdV4JSEDl4oVeEp0CIusk9C76U60FQtrLiPbyJe6N93kXTBh0xMmbCbClVPAic8Ba4c7AhbIRQzuqplTlRIoTm/uzV0+wfWM/spfekpdRZNShQPZpdh6tBDHC27w55f3OYgXbn2BWjtv3KxToV9Tc8avC6KBqPCficBJc2FtJ9W77YYmFTYcuIDN/7sIITVhVq90LOmVBNsb6uRLAi645qiXAU+53u3oAxM2PWHCxgDA9d7uHrbe+LPLhqHtkVVchbiUAvycUQx35VUclrzGl9WSBOn2EyAdMx9DRzyo12RGa5wsvIGoHRm4fOM2BALC6mEVeKrxFwjzf2s26jeCEzj/Kd0SCJMJm54wYWNopbqU6z1aGC/a8o3aBnx3oghuf7yJAcoCKLwfReDkeXB3demyNqvrGvH2r1nYmXYFADex8fFEG3jnxXHRXJQNnKH3GOCRDVzqwC6ECZueMGFjMFqy71wJon/MxM1bjRBbCLF0kh+ev88awpOfA6mbgKbb3GLrMUuA0a902T8AJmx6woSNwdDOteo6vPH9WRzO5eLtPTDQEeufDIKbqhRIfI2b1Qa4zGlTPgC8RhncB12+n6a1Ao/BYJgkLr2ssPW5EXgnciisLIU4ml+O8PeP4KdCCzQ+vQOY9iVg68LlZ9g6GfhlEbdExUiwHttdsB4bg9E+f16vwasJGThzhVsiYyexQOgAR4T5SPB3xWb0Ov8tZ2jjBEyKBQKfNMjkAhuK6gkTNgajYzQqVfj40EV8lVqIG7UNGmWPOBRiJT6DS10hd2LAQ1wcvj79O9UmEzY9YcLGYOiGSkU4X1yFI3nX8b8L13H60k00qQiWaMJ80W68bPETJIJGNAklKAv+F1wmvg6hpX7rEbv8GdumTZvg7e0NKysryOVynDhxok37nTt3ws/PD1ZWVggMDMSePXs0yokIK1euhJubG6ytrREWFoa8vDy+vLCwEHPnzoWPjw+sra0xYMAAxMTEoKGhQcNGIBC0OI4d6+KEGwzGPYxQKEBgPykWjh+IHQtCkb7yYXw2KxhPjeyPX6XPYGLDu/hDGQALVT1kJ9fiz/8G4/2t3+KHtCu4Vl3XdY7pmgIrPj6exGIxbdmyhc6fP0/z5s0jBwcHKi0t1Wp/9OhREolEtHbtWsrKyqLly5drzSsqlUrp559/pjNnztDUqVM18oru3buXnnvuOdq/fz9dvHiRfvnlF3JxcaHXXnuNr6OgoIAA0IEDB6ikpIQ/Gho6nufSLNPvMRgmTGFZDX199E/6fONqKl/ZlyjGnpQrpfTNm49R4NIEmrThCK3ek0V/5F2nhqa28992aV7RkJAQWrhwIf9eqVSSu7s7xcbGarWfPn06RUREaJyTy+W0YMECIuJyispkMlq3bh1fXlFRQRKJhL777rtW/Vi7di35+Pjw79XClp6erust8TBhYzC6jvrK61T69T/4HLSlKz3ppejl5LX0VxoQnUhVt9vuhOjy/dRpKNrQ0IC0tDSEhTWHlBEKhQgLC0NqaqrWa1JTUzXsASA8PJy3LygogEKh0LCRSqWQy+Wt1gkAlZWV6NOnZbSBqVOnwsXFBaNHj8auXbvavJ/6+npUVVVpHAwGo2sQ2zvBZdaXwJzdgONAuAgqsEn8IfY4bcSMwUAvK/2jEv8VnYStrKwMSqUSrq6aYWFcXV2hUCi0XqNQKNq0V//Upc78/Hxs3LgRCxYs4M/Z2dnhvffew86dO5GYmIjRo0cjMjKyTXGLjY2FVCrlDw8P84n/zmCYLT5jgBdTgHHLAJEYQ2qO4T9X5gI3Cw3WhNklTL569SomTZqEJ598EvPmzePPOzk5ISoqin8/YsQIFBcXY926dZg6darWuqKjozWuqaqqYuLGYHQHFhJgfDQwdBqw+1VuL24HQ7d3qHpdjJ2cnCASiVBaWqpxvrS0FDKZTOs1MpmsTXv1z9LSUri5uWnYDBs2TOO64uJijB8/HqNGjcJnn33Wrr9yuRxJSUmtlkskEkgkxtvYzGDc8zj7clFTGmrat9UBnYaiYrEYwcHBSE5O5s+pVCokJycjNDRU6zWhoaEa9gCQlJTE2/v4+EAmk2nYVFVV4fjx4xp1Xr16FQ8++CCCg4OxdetWCDsQjz0jI0NDLBkMhgkiEOgfMLQ1dJ3ZiI+PJ4lEQnFxcZSVlUXz588nBwcHUigUREQ0a9YsWrZsGW9/9OhRsrCwoPXr11N2djbFxMRoXe7h4OBAv/zyC509e5YeffRRjeUeV65coYEDB9KECRPoypUrGss51MTFxdH27dspOzubsrOz6b///S8JhULasmVLh++NzYoyGKZLly73ICLauHEjeXp6klgsppCQEDp27BhfNm7cOJozZ46G/Y4dO8jX15fEYjEFBARQYmKiRrlKpaIVK1aQq6srSSQSmjBhAuXm5vLlW7duJXCxolscauLi4sjf359sbGzI3t6eQkJCaOfOnTrdFxM2BsN00eX7ybZU3QXbUsVgmC4sbBGDwbinYcLGYDB6HEzYGAxGj8PsFuh2JerHjWxrFYNheqi/lx2ZFmDCdhfV1dUAwHYfMBgmTHV1NaTStnOnslnRu1CpVCguLkavXr3azdGo3n51+fJlNoPaBbDfb9dijr9fIkJ1dTXc3d3bXaDPemx3IRQK0a9fP52usbe3N5sPhjnCfr9di7n9ftvrqalhkwcMBqPHwYSNwWD0OJiw6YlEIkFMTAyLDtJFsN9v19LTf79s8oDBYPQ4WI+NwWD0OJiwMRiMHgcTNgaD0eNgwsZgMHocTNj0YNOmTfD29oaVlRXkcjlOnDhhbJd6BKtWrYJAINA4/Pz8jO2W2XLkyBFMmTIF7u7uEAgE+PnnnzXKiQgrV66Em5sbrK2tERYWhry8POM4a2CYsOlIQkICoqKiEBMTg9OnTyMoKAjh4eG4du2asV3rEQQEBKCkpIQ//vjjD2O7ZLbU1tYiKCgImzZt0lq+du1afPjhh9i8eTOOHz8OW1tbhIeHo66urps97QK6LI5vDyUkJIQWLlzIv1cqleTu7k6xsbFG9KpnEBMTQ0FBQcZ2o0cCgH766Sf+vUqlIplMRuvWrePPVVRUkEQioe+++84IHhoW1mPTgYaGBqSlpWlkrRcKhQgLC2szaz2j4+Tl5cHd3R39+/fHzJkzUVRUZGyXeiQFBQVQKBQan2WpVAq5XN4jPstM2HSgrKwMSqVSp6z1jI4jl8sRFxeHffv24ZNPPkFBQQHGjBnDh5NiGA7157WnfpZZdA+GyTB58mT+9X333Qe5XA4vLy/s2LEDc+fONaJnDHOD9dh0wMnJCSKRqM3M9gzD4eDgAF9fX+Tn5xvblR6H+vPaUz/LTNh0QCwWIzg4WCNrvUqlQnJyskbWeoZhqKmpwcWLF+Hm5mZsV3ocPj4+kMlkGp/lqqoqHD9+vEd8ltlQVEeioqIwZ84c3H///QgJCcGGDRtQW1uL559/3tiumT1LlizBlClT4OXlheLiYsTExEAkEmHGjBnGds0sqamp0ejtFhQUICMjA3369IGnpydeeeUVvPPOOxg0aBB8fHywYsUKuLu7IzIy0nhOGwpjT8uaIxs3biRPT08Si8UUEhJCx44dM7ZLPYKnnnqK3NzcSCwWU9++fempp56i/Px8Y7tlthw6dIgAtDjmzJlDRNySjxUrVpCrqytJJBKaMGEC5ebmGtdpA8HCFjEYjB4He8bGYDB6HEzYGAxGj4MJG4PB6HEwYWMwGD0OJmwMBqPHwYSNwWD0OJiwMRiMHgcTNgaD0eNgwsZgMHocTNgYDEaPgwkbg8HocTBhYzAYPY7/Bxx8/ES4fItNAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:20:40.922128Z", + "iopub.status.busy": "2024-02-29T06:20:40.921847Z", + "iopub.status.idle": "2024-02-29T06:22:16.841126Z", + "shell.execute_reply": "2024-02-29T06:22:16.840059Z" + }, + "papermill": { + "duration": 95.940371, + "end_time": "2024-02-29T06:22:16.843721", + "exception": false, + "start_time": "2024-02-29T06:20:40.903350", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:22:16.882160Z", + "iopub.status.busy": "2024-02-29T06:22:16.881399Z", + "iopub.status.idle": "2024-02-29T06:22:16.901901Z", + "shell.execute_reply": "2024-02-29T06:22:16.901026Z" + }, + "papermill": { + "duration": 0.041381, + "end_time": "2024-02-29T06:22:16.903739", + "exception": false, + "start_time": "2024-02-29T06:22:16.862358", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.0069760.0196840.0054694.7321210.1052230.899660.2544090.0000681.4780970.0578010.1366160.0739540.0824410.0002816.210219
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.006976 0.019684 0.005469 4.732121 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.105223 0.89966 0.254409 0.000068 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 1.478097 0.057801 0.136616 0.073954 0.082441 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.000281 6.210219 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:22:16.938663Z", + "iopub.status.busy": "2024-02-29T06:22:16.938408Z", + "iopub.status.idle": "2024-02-29T06:22:17.339488Z", + "shell.execute_reply": "2024-02-29T06:22:17.338514Z" + }, + "papermill": { + "duration": 0.421131, + "end_time": "2024-02-29T06:22:17.341625", + "exception": false, + "start_time": "2024-02-29T06:22:16.920494", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:22:17.380452Z", + "iopub.status.busy": "2024-02-29T06:22:17.379432Z", + "iopub.status.idle": "2024-02-29T06:23:57.383424Z", + "shell.execute_reply": "2024-02-29T06:23:57.382532Z" + }, + "papermill": { + "duration": 100.026409, + "end_time": "2024-02-29T06:23:57.386125", + "exception": false, + "start_time": "2024-02-29T06:22:17.359716", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:23:57.431576Z", + "iopub.status.busy": "2024-02-29T06:23:57.431230Z", + "iopub.status.idle": "2024-02-29T06:23:57.449621Z", + "shell.execute_reply": "2024-02-29T06:23:57.448848Z" + }, + "papermill": { + "duration": 0.044631, + "end_time": "2024-02-29T06:23:57.451907", + "exception": false, + "start_time": "2024-02-29T06:23:57.407276", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:23:57.492447Z", + "iopub.status.busy": "2024-02-29T06:23:57.491728Z", + "iopub.status.idle": "2024-02-29T06:23:57.497984Z", + "shell.execute_reply": "2024-02-29T06:23:57.496986Z" + }, + "papermill": { + "duration": 0.028607, + "end_time": "2024-02-29T06:23:57.500218", + "exception": false, + "start_time": "2024-02-29T06:23:57.471611", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.44625178128854376}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:23:57.540660Z", + "iopub.status.busy": "2024-02-29T06:23:57.540370Z", + "iopub.status.idle": "2024-02-29T06:23:57.957574Z", + "shell.execute_reply": "2024-02-29T06:23:57.956695Z" + }, + "papermill": { + "duration": 0.439641, + "end_time": "2024-02-29T06:23:57.959552", + "exception": false, + "start_time": "2024-02-29T06:23:57.519911", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABLJElEQVR4nO2deXhTZdr/vyd7mibplm7QBcome2VxEBlAGRYZBXV8HUFtRVzhZZTX91JmrgGXUWTGBVzAGV8BHVH8OYOOIyiugMuIAiLgwlKgLdDSljZpmj05z++Pk6RNm7RJmuSk7f25rlynOefJOfdJ0m+e537u5745xhgDQRCEiEjENoAgCIKEiCAI0SEhIghCdEiICIIQHRIigiBEh4SIIAjRISEiCEJ0SIgIghAdEiKCIESHhKiHUVxcjF//+tdxvQbHcXjooYe6bPfQQw+B47i42kL0DUiIEsxXX32Fhx56CEajUWxTiB7A+vXrsXnzZrHNiDskRAnmq6++wsMPP0xCRIQFCRFBEESCICFKIA899BD+93//FwAwYMAAcBwHjuNw+vRpbNq0CZdffjmys7OhVCoxfPhwbNiwIeS5PvzwQ4wdOxYqlQrDhw/Htm3bIrbH4XDgvvvug8FggFarxdVXX40zZ84EbfvFF19gwoQJUKlUKCkpwV//+teg7TiOw9KlS7FlyxYMHToUKpUK48aNw549ezq8FxzH4dixY7jpppug1+thMBjwxz/+EYwxVFdXY968edDpdMjNzcVTTz0V8f0BwPvvv4+pU6dCq9VCp9NhwoQJeP311wPavPXWWxg3bhzUajWysrJw00034ezZswFtysvLkZqairNnz2L+/PlITU2FwWDA/fffD4/HE9CW53msW7cOo0aNgkqlgsFgwOzZs7Fv3z5/m3A+7+LiYvzwww/YvXu3/7sybdq0qN6HpIcRCeP7779nN954IwPAnnnmGfb3v/+d/f3vf2ctLS1swoQJrLy8nD3zzDPsueeeYzNnzmQA2PPPPx9wjqKiIjZkyBCWlpbGHnzwQfb000+zUaNGMYlEwj788MOI7LnpppsYALZgwQL2/PPPs2uvvZaNHj2aAWCrVq3ytzt06BBTq9WssLCQrV69mj366KMsJyfH37YtANjIkSNZVlYWe+SRR9iaNWtYUVERU6vV7PDhw/52q1atYgDY2LFj2Y033sjWr1/P5s6dywCwp59+mg0dOpTdfffdbP369Wzy5MkMANu9e3dE97dp0ybGcRwbOXIke+yxx9gLL7zAFi9ezG6++eaANgDYhAkT2DPPPMMefPBBplarWXFxMWtqavK3KysrYyqVio0YMYItWrSIbdiwgV133XUMAFu/fn3AdcvLyxkANmfOHLZ27Vr25JNPsnnz5rHnnnvO3yacz/vtt99m/fv3Z8OGDfN/VyL9jHsKJEQJ5i9/+QsDwE6dOhWw32q1dmg7a9YsNnDgwIB9RUVFDAD75z//6d9nMplYXl4eKy0tDduOgwcPMgDsnnvuCdi/YMGCDkI0f/58plKpWGVlpX/fjz/+yKRSaVAhAsD27dvn31dZWclUKhW75ppr/Pt8QnTHHXf497ndbta/f3/GcRx74okn/PubmpqYWq1mZWVlYd+f0WhkWq2WXXLJJcxmswUc43meMcaY0+lk2dnZbOTIkQFt3nvvPQaArVy50r+vrKyMAWCPPPJIwLlKS0vZuHHj/M8//fRTBoAtW7asg02+6zIW/uc9YsQINnXq1DDuuGdDQ7MkQa1W+/82mUxoaGjA1KlTcfLkSZhMpoC2+fn5uOaaa/zPdTodbrnlFnz33Xeora0N63o7duwAACxbtixg/7333hvw3OPxYOfOnZg/fz4KCwv9+y+66CLMmjUr6LknTZqEcePG+Z8XFhZi3rx52LlzZ4dhzOLFi/1/S6VSjB8/Howx3Hbbbf79aWlpGDp0KE6ePBnWvQHARx99BLPZjAcffBAqlSrgmC/kYN++fairq8M999wT0Gbu3LkYNmwYtm/f3uG8d911V8DzKVOmBNj1z3/+ExzHYdWqVR1e2zbUIZLPuy9AQpQkfPnll5gxYwY0Gg3S0tJgMBjw+9//HgA6fDEHDRrUIX5nyJAhAIDTp0+Hdb3KykpIJBKUlJQE7B86dGjA8/r6ethsNgwePLjDOdq39RGs7ZAhQ2C1WlFfXx+wv624AYBer4dKpUJWVlaH/U1NTaFvqB0VFRUAgJEjR4ZsU1lZCSD4fQwbNsx/3IfP39OW9PT0ALsqKiqQn5+PjIyMTu2L5PPuC8jENoAQvrxXXHEFhg0bhqeffhoFBQVQKBTYsWMHnnnmGfA8L7aJcUMqlYa1DwCYyFmNQ9kVKX358w4FCVGCCRaJ/O9//xsOhwPvvvtuQA/hs88+C3qOEydOgDEWcK5jx44BEGZawqGoqAg8z6OioiKgR3D06NGAdgaDAWq1GsePH+9wjvZtfQRre+zYMaSkpHToUcQLX0/vyJEjGDRoUNA2RUVFAIT7uPzyywOOHT161H880uvu3LkTjY2NIXtFkXzefSVynYZmCUaj0QBAQECj75e27S++yWTCpk2bgp7j3LlzePvtt/3Pm5ub8eqrr2Ls2LHIzc0Ny445c+YAAJ599tmA/WvXrg14LpVKMWvWLLzzzjuoqqry7//pp5+wc+fOoOf+z3/+gwMHDvifV1dX41//+hdmzpwZs15FV8ycORNarRarV6+G3W4POOZ7n8ePH4/s7Gy8+OKLcDgc/uPvv/8+fvrpJ8ydOzfi61533XVgjOHhhx/ucMx33Ug+b41G0yeCX6lHlGB8Ttw//OEP+O1vfwu5XI5f/vKXUCgUuOqqq3DnnXeipaUFL730ErKzs1FTU9PhHEOGDMFtt92Gb7/9Fjk5Odi4cSPOnz8fUriCMXbsWNx4441Yv349TCYTLr30UnzyySc4ceJEh7YPP/wwPvjgA0yZMgX33HMP3G43nnvuOYwYMQKHDh3q0H7kyJGYNWsWli1bBqVSifXr1/vPkyh0Oh2eeeYZLF68GBMmTMCCBQuQnp6O77//HlarFa+88grkcjnWrFmDW2+9FVOnTsWNN96I8+fPY926dSguLsZ9990X8XWnT5+Om2++Gc8++yyOHz+O2bNng+d5fP7555g+fTqWLl2KmTNnhv15jxs3Dhs2bMCf/vQnDBo0CNnZ2R16b70C8Sbs+i6PPvoo69evH5NIJP6p/HfffZeNHj2aqVQqVlxczNasWcM2btzYYaq/qKiIzZ07l+3cuZONHj2aKZVKNmzYMPbWW29FbIfNZmPLli1jmZmZTKPRsKuuuopVV1d3mL5njLHdu3ezcePGMYVCwQYOHMhefPFF/xR8WwCwJUuWsNdee40NHjyYKZVKVlpayj777LOAdr7X1tfXB+wvKytjGo2mg61Tp05lI0aMiPge3333XXbppZcytVrNdDodmzhxInvjjTcC2rz55pustLSUKZVKlpGRwRYuXMjOnDkTll3B3gO3283+8pe/sGHDhjGFQsEMBgObM2cO279/f4Bd4XzetbW1bO7cuUyr1TIAvXYqn2OM6poRsYPjOCxZsgTPP/+82KYQPQjyEREEITrkI+qFdBXUqFarodfrE2RN7Kmvr+8QGNkWhULRZRwPkVyQEPVC8vLyOj1eVlbWo1NLTJgwoUOwYVumTp2KXbt2Jc4gotuQEPVCPvroo06P5+fnx+3aiXA5btmyBTabLeTx9PT0uNtAxBZRndXFxcVBf9nuuecevPDCCyJYRBCEGIjaI/r2228DxvpHjhzBr371K1x//fUiWkUQRKJJqun7e++9F++99x6OHz8eVmg7z/M4d+4ctFptnwmFJ4ieBGMMZrMZ+fn5kEhCT9InjY/I6XTitddew/Lly0OKisPhCAjFP3v2LIYPH54oEwmCiJLq6mr0798/5PGkEaJ33nkHRqMR5eXlIdusXr066DKB6upq6HS6OFpHEEQ0NDc3o6CgAFqtttN2STM0mzVrFhQKBf7973+HbNO+R+S7SZPJREJEEElIc3Mz9Hp9l/+jSdEjqqysxMcff9xlAnilUgmlUpkgqwiCSBRJscRj06ZNyM7OjirtAkEQPR/RhYjneWzatAllZWWQyZKig0YQRIIR/T//448/RlVVFRYtWiS2KUQfhTEGt9vd6fo1IjhSqRQymazb4TOiC9HMmTNFz0VM9F2cTidqampgtVrFNqXHkpKSgry8PCgUiqjPIboQEYRY8DyPU6dOQSqVIj8/HwqFggJjI4AxBqfTifr6epw6dQqDBw/uNGixM0iIiOTG0QKc+AhwO4FBVwCarK5fEyZOpxM8z6OgoAApKSkxO29fQq1WQy6Xo7KyEk6ns0MNuXAR3VlNECFhDPjhbaDuZ6DxJPD9VsDjivllov0VJwRi8f7RJ0AkLxdOAKYzACcBpHLAYQZqOibrJ3o+JERE8lLzvbDtPx4YMFX4+/wR8ewh4gYJEZGcOK3ABaFsNPLGANkXARwHNJ8TekZEwiguLu5Q7y7WkBARyUnTaYDxQKpBcFArU4HUHOGYsarTlxI9DxIiIjlpOiVs0we07kvzlmduCp2vmgiO0+kU24ROISEikg/GhB4RAKQXt+7XFwhbc8fqt7HG6eZDPtwePuy2rjDaRsO0adOwdOlSLF26FHq9HllZWfjjH//oDw4uLi7Go48+iltuuQU6nQ533HEHAOCLL77AlClToFarUVBQgGXLlsFisfjPW1dXh6uuugpqtRoDBgzAli1borIvUiiOiEg+7EbA3gxIpK29IADQeodmlgbA4wak8fv6vvBZx9LbPgZkaTC/tJ//+d/2VMDlCb46oH+6GtePL/A/3/jlKdicgUtJ7vvVkKhsfOWVV3Dbbbfhm2++wb59+3DHHXegsLAQt99+OwDgySefxMqVK7Fq1SoAQEVFBWbPno0//elP2LhxI+rr6/1i5itXXl5ejnPnzuGzzz6DXC7HsmXLUFdXF5V9kUBCRCQfzeeEbWqOMG3vQ6kD5GrAZQMsdYAuftVIegIFBQV45plnwHEchg4disOHD+OZZ57xC9Hll1+O//mf//G3X7x4MRYuXIh7770XADB48GA8++yzmDp1KjZs2ICqqiq8//77+OabbzBhwgQAwMsvv4yLLroo7vdCQkQkH83eoVd7oeE4QJsLNJ4CzLVxFaIl0weFPCZptwrkjl+WhGzbfsXIoskDgjeMgl/84hcBS1ImTZqEp556yr94d/z48QHtv//+exw6dChguMUY8y91OXbsGGQyGcaNG+c/PmzYMKSlpcXM5lCQEBHJh9nbI9IGKRSpMQhCZL0QVxMUsvDdp/Fq2100Gk3A85aWFtx5551YtmxZh7aFhYU4duxYokzrAAkRkVzwHsB8Xvg7WI8nJVPYxlmIegJ79+4NeP71119j8ODBkEqlQdtffPHF+PHHHzFoUPDe3rBhw+B2u7F//37/0Ozo0aMwGo0xtTsYNGtGJBctdQDvBuQqQB2kYqtfiBoTa1cSUlVVheXLl+Po0aN444038Nxzz+F3v/tdyPYPPPAAvvrqKyxduhQHDx7E8ePH8a9//QtLly4FAAwdOhSzZ8/GnXfeib1792L//v1YvHgx1Gp13O+FhIhILnxT89q8jg4WoFWI7Ka4LIDtSdxyyy2w2WyYOHEilixZgt/97nf+afpgjB49Grt378axY8cwZcoUlJaWYuXKlQElyDdt2oT8/HxMnToV1157Le644w5kZ2fH/V5oaEYkF5YGYZsa4suvSGmdObM2tk7p90HkcjnWrl2LDRs2dDh2+vTpoK+ZMGECPvzww5DnzM3NxXvvvRew7+abb+6WneFAPSIiubB4Y1Y0nfwKp2QIW/IT9RpIiIjkgTHBRwSE7hEBgNorRLam+NtEJAQamhHJg6MZcDuE/EM+X1Aw1GnC1m5KiFnJyK5du8Q2IaZQj4hIHlrqha0mU1jeEQqVXtj2YSHqbYguRGfPnsVNN92EzMxMqNVqjBo1Cvv27RPbLEIM/P4hQ+ft/EJkjKs5ROIQdWjW1NSEyZMnY/r06Xj//fdhMBhw/PhxpKcHiR8hej8tYTiqAUCVJmztzQDPA5RzuscjqhCtWbMGBQUF/pW/ADBgQOi1OA6HAw6Hw/+8ubk5rvYRCcbiHZp15qgGAEWqMHTjPYDT3NpDInosov6UvPvuuxg/fjyuv/56ZGdno7S0FC+99FLI9qtXr4Zer/c/CgoKQrYlehged2u0dFdDM4lEWIkPADZjXM0iEoOoQnTy5Els2LABgwcPxs6dO3H33Xdj2bJleOWVV4K2X7FiBUwmk/9RXV2dYIuJuGFrFFLDypSAUtt1e5o561WIOjTjeR7jx4/H448/DgAoLS3FkSNH8OKLL6KsrKxDe6VSCaVSmWgziUTgC05MyQy+tKM9NHPWqxC1R5SXl4fhw4cH7LvoootQVUXJ0fscbYUoHHxDMwf5CXsDogrR5MmTcfTo0YB9x44dQ1FRkUgWEaLhE6JwS0r7hm9UWqhXIKoQ3Xffffj666/x+OOP48SJE3j99dfxt7/9DUuWLBHTLEIMIu4RxUmIGAPczsQ/WPCc18F49dVXkZmZGTCDDADz589PyALVeCCqj2jChAl4++23sWLFCjzyyCMYMGAA1q5di4ULF4ppFpFoGEueoZnHBXz+VGzPGQ5T/geQKcJqev3112PZsmV49913cf311wMQqm9s376905X1yYzoa81+/etf49e//rXYZhBi4mgWpu8l0tZgxa7w9YjcTmF9mqzvTGKo1WosWLAAmzZt8gvRa6+9hsLCQkybNk1c46JEdCEiCH8OInV6+FHSMoUgPm6HMDyLlRBJ5ULvJNG0rVYSBrfffjsmTJiAs2fPol+/fti8eTPKy8sDkun3JEiICPHxBTL68gyFi1LrFaLm8J3cXcFxYQ+RxKS0tBRjxozBq6++ipkzZ+KHH37A9u3bxTYrakiICPGJ1D/kQ6kTelN9dOZs8eLFWLt2Lc6ePYsZM2b06JUGtFqQEB+/EEXYq+njU/gLFizAmTNn8NJLL2HRokVim9MtSIgI8Ym6R9S3hUiv1+O6665Damoq5s+fL7Y53YKEiBAXtwNwWoS/o/ERAX1WiAAhn9fChQt7/NIn8hER4uLLO61IiXzmyy9EfW+ZR1NTE3bt2oVdu3Zh/fr1YpvTbUiICHHxCVGwYopd4QtqtPc9ISotLUVTUxPWrFmDoUOHim1OtyEhIsSlW0KUKmzdDiEiOsJYnJ5MqLplPRXyERHi0h0hkqkAife31NkSO5uIhENCRIhLd4SI4wCFRvjb5/COAhbBglOiI7F4/0iICHHpjhABrcMzR+Q9IrlcGMpZrdbork0AaH3/fO9nNJCPiBAPt7NVQKIVom70iKRSKdLS0lBXJ1QPSUlJ6bFrtcSAMQar1Yq6ujqkpaVBKu2kFl0XkBAR4uGrSyZXAXJ1dOdQeKfwndHFEuXm5gKAX4yIyElLS/O/j9FCQkSIR3eHZYC/R1RT34CP6yqhkHKYNjQbOTpVWC/nOA55eXnIzs6Gy+WK3o4+ilwu71ZPyAcJESEesRAiZSrMdhe+rzmDBsNoAMB3VU2YPTIvotNIpdKY/EMR0UFCRIhHTHpEqTh9wQo5L8NAgwYDs1Ixsp8uNvYRCYOEiBAPX3HEbghRvVOOFocbSpkNk4bnIEVBX+meCE3fE+Lh6xGFmx42CEcaPACAXJUHKbLWGS+Xh0ejxdkd64gEQkJEiAPPt66a91VtjRDGGCqaeDBwyEpV+qfwqxut+Nuek9hxuCZGxhLxRlQheuihh8BxXMBj2LBhYppEJApHs1BiWiIFFKlRnYLjOFw/vhBFednQqeX+ZR5ZqUq4PDzqzQ4022kmrCcg+oB6xIgR+Pjjj/3PZTLRTSISgS91h1IXXonpEOhT5NBnZwHmWn+PSK2QIk+vwjmjHZUNVozqr4+FxUQcEf2/XiaTdTsYiuiB+GrWq2Iww6XUCkLUJkFaUaYG54x2nL5gISHqAYjuIzp+/Djy8/MxcOBALFy4sNO69w6HA83NzQEPoofiyyGkik4kGGP4+Mfz+K6qCS6pNyq7zTKPwowUAMBZo40WtfYARBWiSy65BJs3b8YHH3yADRs24NSpU5gyZQrM5uDh+qtXr4Zer/c/enLVgj6Pr0ekjK5H1Gx34/BZE/YcawDny9TYJhVItlYJmYSDzemB0Up+omRHVCGaM2cOrr/+eowePRqzZs3Cjh07YDQa8f/+3/8L2n7FihUwmUz+R3V1dYItJmKGo3s9ovPNdgCAQauETNVxBb5MKvEv8zhnskVvJ5EQRPcRtSUtLQ1DhgzBiRMngh5XKpU9Pkk44aWbPqJakyBEuXpla+7qdsnRRvbToyRbg3x9lAtqiYQhuo+oLS0tLaioqEBeXmTrhIgeBmOtPqIoh2YXLA4AgCFV1Tr9306IhufrMK4oA+ma5K/c2tcRVYjuv/9+7N69G6dPn8ZXX32Fa665BlKpFDfeeKOYZhHxxmkBeLcwbR/l0OxCixA1nZmqaJOTyCoEShI9DlGHZmfOnMGNN96ICxcuwGAw4LLLLsPXX38Ng8EgpllEvPH5hxQaIaAxQuwuD8x2NwAgQ6MApApB1BgPuG2twgSgyeLEebMd+Wlq6FR9J7l+T0NUIdq6dauYlyfEwu8fiq43ZLK5wHFAqlIGldwrZHK10CNyWgKE6LOjdai8YMWMi3IoniiJSSpnNdFH6KZ/KEenwpLpg9Di7RUBEMTHaRWCGlOz/bsNWiUqL1hR32IHQEKUrCSVs5roI3SzRwQAcqkk0Antd1gH5q42aIVZ1nqzI+prEfGHhIhIPP4YohgmMAuRRN+QKghRQ4uTIqyTGBIiIvH4kuZHmYdo+6EafPzjebQ42g7NgveI0lMUkEk4ON08TDaKsE5WSIiIxNMNH5HLw+PYeTMOnzVB2nbVvl+IApcHSSScfwh3gRKlJS0kRERicdmFWvVAVD6iZm+vRiGTQCVv8/XtpL5ZhleImkiIkhaaNSMSi88/JFcBssgjno1eIUpLkQcWQ+xEiMYUpGFIjhY5OloelKyQEBGJpZszZr6V9GnqdiIWYr0ZAPRLo7VmyQ4NzYjE0s0YIt/QTK9uFyXt6xG5ncKD6FGQEBGJpZszZkabIDJpKe2ESKoApN4OfpBeUUV9C7451Qib0xPVdYn4QkJEJJZuxhA53cKi1g49Io4LOYUPAHuO1ePLEw1oaKHAxmSEfEREzGGMBTqS29JNH9ENEwrhcHsgkwT5DVVohKKNIWbOjFYXGi1OFHjTyBLJA/WIiJhSZ7bjta8rYXeFGAJ100cEAEqZFFJJEKHrpEeUniI4t6noYnJCQkTElM9+rkNDixMHKps6HvS4W0WiG+vMQhIiqBFojSXy+ZiI5IKEiOg2JpsL57zVMsYWCHXsf6xpBs+3W9vl8w9JZULajgg5UWfGtgNncKAqiMgBncYS+XxKJkqkn5SQEBHd5oezJrz5bTU+/bkOJQYNVHIpzHY3arwJ7v20nTGLoqhindmBygvW0BHSnQmRd5bNZHN3FEhCdEiIiG5T3WQFIOQJkkklKMoUnMGVF9oJQjf9Q76sjNpQmRY7CWrUKmWQSTjwjPnPQyQPNGtGdAuXh8f5ZmFKvH+6MNwqzEjB0VozzjTagJI2jbs5Y9YqRCG+tp30iDiOw9Vj85GikCE11OsJ0aBPhOgWtSY7PDyDViXz+2F8SyrONwvH/DNc3YwhMtsF/05YQsTzQLsp/qJMTZAXEckADc2IblHr9QPl6dX+2KG0FDnUCikyUhWwOtsMg7rRI2KM+VPDapUhhmZyjTeJPgNc1oivQYhH0gjRE088AY7jcO+994ptChEBDd4UrL6UrIAwDLp9ykAsvKQo0J/TDR+R1emBm2dC0vxQPSKJpHU2LsjwzGh1Yt/pRnxfbYz4+kR8iUqITp48GVMjvv32W/z1r3/F6NGjY3peIv7Ut3QUIgAdAw55XkhsD0TVI7K5PFArpNAoZMGDGX2EKLYICCv3Pz/egENnjBFfn4gvUQnRoEGDMH36dLz22muw2+1dv6ATWlpasHDhQrz00ktIT0/v1rmIxDN1iAGXDc4KmevHnyfaaRbqjnGSVrGIgKxUJe6aWoJbJxd33rCT6Oo0/xS+i/JXJxlRCdGBAwcwevRoLF++HLm5ubjzzjvxzTffRGXAkiVLMHfuXMyYMaPLtg6HA83NzQEPQlyKMjWYUJyBFEXgcKnF4caWvZV46fOTwj9921r3wdaJhYlM2sVr/Q7rIFP4KjkkHAeXhwXmuyZEJ6pvxNixY7Fu3TqcO3cOGzduRE1NDS677DKMHDkSTz/9NOrr68M6z9atW3HgwAGsXr06rParV6+GXq/3PwoKCqIxn0gAarkUDWYnLA6P8E8fgzVmYdHJFL5Uwvln3IwUYZ1UdMtZLZPJcO211+Ktt97CmjVrcOLECdx///0oKCjALbfcgpqampCvra6uxu9+9zts2bIFKpUqrOutWLECJpPJ/6iuru6O+UQ3OWu04dh5c9DqGFIJ5x8KNVqc3Y4h2nW0DtsOnEHVhS5mwzoJagRah2fNdhKiZKJbQrRv3z7cc889yMvLw9NPP437778fFRUV+Oijj3Du3DnMmzcv5Gv379+Puro6XHzxxZDJZJDJZNi9ezeeffZZyGQyeDwdV28rlUrodLqAByEeh88Ysf1QDY6d77jIFGhdaHohBkJUY7Kj8oIVTg/fecNOekQAoPPO4jXbaGiWTEQV0Pj0009j06ZNOHr0KK688kq8+uqruPLKKyHxjv0HDBiAzZs3o7i4OOQ5rrjiChw+fDhg36233ophw4bhgQcegFQqjcY0IoE0eYc36e2zJXrJ1ChwAsCFFifg7G5UtXAtXVdR0T4hcgTvEenU1CNKRqISog0bNmDRokUoLy9HXl5e0DbZ2dl4+eWXQ55Dq9Vi5MiRAfs0Gg0yMzM77CeSE5+fRd8+kb2XjFRvGR+rE3BHL0RuDw+LQ+ghh1xn5qOT6XsAuChPi+KslI4ZHglRiUqIPvroIxQWFvp7QD4YY6iurkZhYSEUCgXKyspiYiSRfNicHn/ysw75o734Km2YLE4AvuUdkQuRb4ZLLuUCa5kFw9cj8riEJPrtShZpVfKuxYxIOFEJUUlJCWpqapCdnR2wv7GxEQMGDAjq3wmHXbt2RfU6IvE0WYVUHFqVDPIQU+ppKXJkpiqQKXeBNbvBSaRRzZq1XXUfMgWtD5kSkMoFIXK2ALKMiK9HJJ6onNWhgsFaWlrCngEjejb++mIpoYskquRS3DKpGHMHe9ehKVOjiiHy+XNSlWH+bnYS1AgAB6qa8OnP5ymWKImIqEe0fPlyAMJaopUrVyIlpTUJucfjwd69ezF27NiYGkgkJ/6yPuH4Wro5Y8bzgFohDb3qvj0KDWBrCukn+r7aCKPVhcHZ2vDFjYgrEX0K3333HQChR3T48GEoFK2/hgqFAmPGjMH9998fWwuJpGREvh7ZWlVY/8jMZgRjDJIohWhUfz1G9deHn1kxjCl8o9VFM2dJRERC9NlnnwEQptnXrVtHcTx9GL1aHtbM08FqI87sO4oRzIIBA7qXMF/S2WLXtnQR1OifwqdYoqQhqn7ppk2bYm0H0UuRSThIHc2wc3x8KncEo8sekfC1px5R8hC2EF177bXYvHkzdDodrr322k7bbtu2rduGEckLzzN8V90ErUqOEkNqp2k50lLkqPO0wM48Uc2YMcbw+jdVUMulmD0yt8Pi2qCEG9QYZGkKIQ5hC5Fer/dPner1CfplI5ISi9ONPccaIOE4/Pflgzptm6aWQ+k2w8F4eJR6RBov73DzqPPmxA4VJtCBLoIaW6OraWiWLIQtRG2HYzQ069v4/oFTVbIu/TYazg45PPCAg5mpkRbxtYReS4pCGoEQhTc0a7ELpYXC9j0RcSMqH5HNJhTT803fV1ZW4u2338bw4cMxc+bMmBpIJB9hr/sCwNmMUMqlaOJTYHLwSIswJ1qXJYSC4esRuaxBk+inKmVY+ItC6FRyEqEkIaqAxnnz5uHVV18FABiNRkycOBFPPfUU5s2bhw0bNsTUQCL58M02hSUOtkYoZRLYZfqg6UK6ossSQsGQp3SaRJ/jOGRrVVDJaWF1shB1hsYpU6YAAP7xj38gNzcXlZWVePXVV/Hss8/G1EAi+fD3iNRhiIOtCTqVHGmZ2eE5mkNcK6JaZBKJIEZASD8RkVxENTSzWq3QaoVYjQ8//BDXXnstJBIJfvGLX6CysjKmBhLJh6+XogurR9SE/DQ18gcNArIjz1Ud0bXaotAIPqIQfqLTDRacbGhBfpoaw3IpHk5sok6e/84776C6uho7d+70+4Xq6uooyLEP0Oz3EYUnRAAAdXSFEaQSDmqFNCx/VABdBDWeb7bj+2pT1xkfiYQQVY9o5cqVWLBgAe677z5cccUVmDRpEgChd1RaWhpTA4nkY87IPJhsLmSHqNzhhzG/EDFVGuxOD1RySdcr6Nswa0Su91QRVt3oYubM59+iKfzkICoh+s1vfoPLLrsMNTU1GDNmjH//FVdcgWuuuSZmxhHJiUGr7FDHLCguK+B2ggfw128uwO5pwl1TS6BWRO4kjkS8AIQR1Ch89c0UXZ0URL30ODc3F7m5uQH7Jk6c2G2DiF6EtzckUekgcykAjxsmmysqIYqYLoIafT0is90NxljkQkfElKiEyGKx4IknnsAnn3yCuro68HxgQvNYV4Ilkod6swNVjRZka1UoyEjpvLH1grBVp0Mvk6PFIQhRrj68nFXnjDZ89ON55OlVmDkit+sXtKWroZlSBgnHwcMLNc4oa6O4RCVEixcvxu7du3HzzTcjLy+Pfk36EGearNhzrAGDc1K7FiJLg7DVGKBzyHHWaIsolshkc6HR4oQmmpxBXSRHk0g4aJRSmO1umO0kRGITlRC9//772L59OyZPnhxre4gkJ6JIZ1+PKCUTeklruefIrxWNEIWu+OpDp5b7hYgQl6iEKD09HRkZlAu4LxKROFi8FX81WdBz0QiRK/xrtcfXI/K4ALdDyGXdjtkjc6GQSqCUdau8HxEDovoEHn30UaxcuRJWa/diMDZs2IDRo0f7iyVOmjQJ77//frfOScSXFodXHLoaLrkdrWWmNQZ/pY9oekQRBzMCQvUOqfd1nWRqVMml5FpIAqLqET311FOoqKhATk4OiouLIZcHflEOHDgQ1nn69++PJ554AoMHDwZjDK+88grmzZuH7777DiNGjIjGNCLOhD008/mHFBpArkZaihsDsjRIS5GHPUvVrR4RIAQ1WhuF4VkK9eCTmag+4fnz58fk4ldddVXA88ceewwbNmzA119/TUKUhPDeGSYgjLVf1lZHNQCkKGSYX9ovous1R7Pyvi0KjVeIgveITDYX9lc2AgAuH5YT3TWImBCVEK1atSrWdsDj8eCtt96CxWLxR2q3x+FwwOFw+J83NzfH3A4iNC1ONxgTll1ouooFMp8XtqmGqK7l8vBIS5F7Z7Si7BF1MYXP8wzfV5ugkEkwfWg2DdFEJOqARqPRiH/84x+oqKjA//7v/yIjIwMHDhxATk4O+vUL/5fv8OHDmDRpEux2O1JTU/15jYKxevVqPPzww9GaTHQTjUKGhZcUwur0dP1Paz4nbLX5AbvtLg8YQ5dBjXKpBAsvKeqOua0Oa4c56GGfwDndPBxuntKCiEhUzupDhw5hyJAhWLNmDZ588kkYjUYAQq7qFStWRHSuoUOH4uDBg9i7dy/uvvtulJWV4ccffwzadsWKFTCZTP5HdXV1NOYTUSKVcMjWqVCcpem8Ie8BWrwzZtrWQMTPj9djw64K7PMOh+JOF7FEMqkEGqUgPpS/WlyiEqLly5ejvLwcx48fD6jseuWVV2LPnj0RnUuhUGDQoEEYN24cVq9ejTFjxmDdunVB2yqVSv8Mm+9BJCGWeoB3C1PmbVbd+2qgRZMgLSq6GJoBbRe/khCJSVRC9O233+LOO+/ssL9fv36ora3tlkE8zwf4gYjk4fh5M/ZXNqLe3MXn03xW2GrzhEyJXnx10MIRoi+ON+CVr07jyFlT1PaGFdRIq/CTgqh8REqlMqij+NixYzAYwndOrlixAnPmzEFhYSHMZjNef/117Nq1Czt37ozGLCLO/FRrRkVdCy4fJul89X2TNzleWmHAbp8QGa2uLqfwL1gcaLQ44Qm3umswulj4CrT6iWhoJi5R9YiuvvpqPPLII3C5hA+P4zhUVVXhgQcewHXXXRf2eerq6nDLLbdg6NChuOKKK/Dtt99i586d+NWvfhWNWUScCSuuhzHAWCX8nR7obPaV8fE5hzvDJwzhVJMNia9H5LIJSfSD4LPJ6vREfx2i20Qd0Pib3/wGBoMBNpsNU6dORW1tLSZNmoTHHnss7PO8/PLL0VyeEAmzPYwYopY64R9fKheGZm2QSyVIVcr8q/BDzVIxxvzDN19EdlTIUwBOAjAecFlasza24aI8LYblammZh8hEJUR6vR4fffQRvvzyS3z//fdoaWnBxRdfjBkzZsTaPiJJcHl42Ly9hk6XXDQcE7bpxYCko9Do1a3pQHJ0wdOBWJweuDwMHNeNYEZASKKvSBGSozmDC5FSRlP2yUDEQsTzPDZv3oxt27bh9OnT4DgOAwYMQG5uLiWY6sW0eHtDClkXi0Trfxa2hqFBD5dkp8KgVXYqZr7ekFYl77ScdVgoNK1CRCQtEQkRYwxXX301duzYgTFjxmDUqFFgjOGnn35CeXk5tm3bhnfeeSdOphJi4l/aoZSF/rExnRXWmEmkQObgoE3GFXWdRN9odQLopn/IhyIVwPmQQY2AMEN3weLA1CEGpKUoun9NImIiEqLNmzdjz549+OSTTzB9+vSAY59++inmz5+PV199FbfccktMjSTEpzkcR/WZb4Rt9nBAHl4WxmDIJBJk68LMi90VXQQ1AsCpCxY0mB0Y3T+NhEgkIhKiN954A7///e87iBAAXH755XjwwQexZcsWEqLeguUCYGsEpHIMSsuA4ZJCINRIyVgF1HmHZf0ndHpau8sDs90dUmiG5moxNLejPycqwghq1KlkaDA7KJG+iEQkRIcOHcKf//znkMfnzJlDlV57A9ZG4OftgOmMf5cSQHZqNpAxAHANAPQFgNT79WmpA354R/g7fyygDb2S3eJw4297ToLjgKXTB0EmjfNsVRixRP6gRhsFNYpFRELU2NiInJzQX7KcnBw0NTV12yhCRCwXgIOvAU6r4OvRGACPUxCnljrhUbVXEKGUTOE1LXVC/FCqARjYsbfclhSFFEq5BA4Xj0arE9nawCEcYwyMCTmlY0JYKWOptJDYRCREHo8HMlnol0ilUrjd9KvSY+E9wE//EkRImwOMvA5Q6QEA+45VQ22pRomsHqrmSmGo40v1AQCGIcCQ2V36hjiOQ1aqEmebbLjQ0lGImm1uvPKf08hKVeLGiQXdn4X1Tdl34qym9WbiE/GsWXl5OZTK4GN7WiPWwzm7XxAXuQoY9V+AUhjWMMaw96wDTncWyi4dD1WKXEiMbzMKr9NkAeq0sC+TlarwC1F7Llgc8PAMnliFgviFqEWIrpZ0HArq2tQ4I8QhIiEqKyvrsg05qnsoHjdQ9bXw98DpfhECALuLh9O7JEOnkgkLWTVZwiMKMjXCD9kFS8cfrkaL09smRrNXitTW6GpnC6DqmLHBn5fIw4PnWeyGhUTYRCREmzZtipcdhNicPywMt1Q6IHdUwCHfkCVVKYuJczkzVRCZhqA9ImFfRqyESCIRRNXeLAzPgghRikKKu6eVQCmTUECuSNACG0Kg9oiw7Teuw9IMX6Szz6nbXbJShR5Rs80FuytwsWmdN8WIr01MUHrFxxE8tTDHcVTNQ2Ri880ieja2JmGqnuOAnI5FC3wr4aMq6xMElVyK0sKOwYNON48LLYIQhVuWOizCcFgT4kJCRAB1PwnbtKKgC0NNsUjJ0Y5pQ7M7mmG2gzHBZ5MaTZnpUPiGY/bQxRZ+rm3G0VozBmRpMLp/WuyuTYQFCREBXDghbLOHBT3s8xHpYihEwZBLJRiaq4U61knsuxiaAYDJ6sLJegtUcilG94/t5YmuISHq67hsQLO34kbGwKBN5ozMQ7PN1XUtswgxWp041WDB6P5pkEo45OhUuHJUXtcvjJQwhMgnsjSFLw4kRH2dptNCVLQmyx+82B6VXBrzUjuMMbz5bTWsTg+yUpUoyEiJ6fkDCCuokVLGignNmvV1Gk8K2xC9oXjBcRyKMoXlF8frzDjfbEddsz0+F/MJkdMiRI8HwdcjanG4wVg38mQTUUFC1NcxemvDpRcHPXyhxYHPfq7rXjWNEAzPE4ZMP55rxo7DNdiytwrfVcVhraJCI4QkMBayV5SqkEHCcfC0KatNJA4Sor6Mo0WYugcAXfDqvHVmBw5WG/FTTezLexdkqGHQKuHyMBitLsgkHEqyU7t+YaRwXJfDM4mE8/vAyE+UeEiI+jK++mOarJCLVZu82RLjkTCM4zjMHpmLDI0CSrkEvxqRE7NYpQ6E4bDWqmRQyCQdgiyJ+COqs3r16tXYtm0bfv75Z6jValx66aVYs2YNhg4Nnu+YiDG+fEP60PPVRqvgvE3vTjWNTshKVaLs0uL45zsPw2F9TWk/yCQcRViLgKg9ot27d2PJkiX4+uuv8dFHH8HlcmHmzJmwWCjReULw9Yg6ESJfjyg9Vmu/QhD3f/4wghrlUlprJhai9og++OCDgOebN29GdnY29u/fj1/+8pciWdVH4D2A2VsePIR/iDHWpkfUw3M5hzE0I8QjqeKITCZhZiYjIyPocYfDEZDzKFjZayJMrI2CGMkUgDp4ZY0WhxtONw8Jx8V0eYco+GKk7MaQTRotTnx+vB4SjsNVY/ITYxcBIImc1TzP495778XkyZMxcuTIoG1Wr14NvV7vfxQUFCTYyl6EpU7YagzCrFIQfL0hvVrW/fpiYqNKE7Y2ozCNHwQOwMl6C6oarRRLlGCSRoiWLFmCI0eOYOvWrSHbrFixAiaTyf+orq5OoIW9jBafEHVcfOqjICMFd08rwdVjgw/dehS+HpHHBbisQZv4E6S5edhdfKIsI5AkQ7OlS5fivffew549e9C/f2jHqVKpDJmmlogQS72wTTV02iweyztEQSoTZs4cZsBuak2q3waZVAKNUgqLwwOz3QW1ohfcdw9B1B4RYwxLly7F22+/jU8//RQDBgwQ05y+RRg9ol6Hr1fky7UdBB0l0hcFUYVoyZIleO211/D6669Dq9WitrYWtbW1sNlsYprV+3HZWuNpNMF7RDzP8PZ3Z7D7WL0/X3WPx5fg3x56uUprRQ+Krk4kogrRhg0bYDKZMG3aNOTl5fkfb775pphm9X58vSGVPmREtcnmwukGKw6fMULW0x3VPnwO605mznzpcGkVfmIR1UdEMxMi4fcPhR6W1XtTtmamKntPVYswhmZalRwKmQT0zUwsSeGsJhKMr0fUiRDVNQtCZIhlEnuxCWNoNrqfHmP66ynCOsEkzfQ9kUAsXTuqzxkFP11Mk9iLjT+o0SQUWwyChNaaiQIJUV+D57scmrk9PGq9Scr6p6sTZVn8UWi9eYl4WuqRZJAQ9TXsRqGqq1TW6rxtR22zHR6eQaOU9vylHW2RSMJa6vHBkRr8/etKf9VZIv6QEPU1Wtos7QhSBx4A7C4PUhRS9E9P6X3DFN+6OmtjyCYNLU40mB3+MkpE/CFndV8jDP/QoGwtBmalwunpJfFDbUnJAC5UALbQQqRVyVBvdtAUfgIhIepDMMbAhTFjBghOW5WkFy5xUHszO1hD58am0kKJh4ZmfQSTzYU3v62GzejNQaQxwGh14qwxMIq9rtkOnu/FUTQpPiG6ELKJb5kHDc0SBwlRH8Dl4fHu9+dQ19SMqrNCMUW3OgvbD9fgH/vOYO/JC+B5hoYWB7Z+W42NX57qvZUsUjKFrd0UsrSQLy2uLzslEX9oaNYHOFDZhAazAwaYUJyZAii14GVqZKQoUNfswFcVF/BjTTNsLg88PINBq4xt7flkQpEKSOVCOhBbk1A4oB2+bJQmmyv+ubQJANQj6vVYHG58e1pwzF6W54FSJgVSs6GQSTB7ZC5mjciFQiaB0eqCw8XDoFVi5vBcka2OIxzXZngW3GGtUwvLPHRqORy9ZcFvktNLf/YIH99XG+HyMOTqVShSeosSeFfccxyH4fk6DDRoUNVohUwiVF/t8dkYuyIlEzCfDzlzJpVwuGdaCfWEEggJUS/G7eFx2FuhdXxROrizwSOqVXIphuRoE22eeKi7dliTCCUWGpr1YirqLbA6PdCqZCjJ0rQu7ehLydCC4XNYdxLUSCQW6hH1YgozUvCr4TmQSjhInCbA7RTWWqUEr5LSZ/ALUYOQSD9I7+d0gwWfn2hApkaBK0flJdjAvgcJUS9GrZBiZD/v2qr6Y8I2JVMQo75MSibASQCXXchU6Su+2AaOAxrMDsqZlSBoaNZXsIQXUd0nkMpae4W+4Wo70rxT+Earq3cHeCYJJES9lO+qmrC/sqk1MLEvJsvvDF+ubt/70g6dSgaZhIOHZ7TUIwGQEPVCGGPYX9mEPcfqUefNKxROetg+he99sAQXIo7jkOaNsG6kCOu4Q0LUC6kzO2C2u6GQSVCYkdIaRQyQEPnw9QxDDM0AIF0jDM8aLY6QbYjYIKoQ7dmzB1dddRXy8/PBcRzeeecdMc3pNVTUtwAAijJTIJNKhH82xgBFStDCgn0SX2FJy4WQa84yNUK+7oYW6hHFG1GFyGKxYMyYMXjhhRfENKPXUVEvRFAPzEoVdrScF7apvXjpRqQodYBMKaSNDRHYaNAqkZmq6L3r7pIIUd/hOXPmYM6cOWKa0OswWV1oMDsg4TgMNHh7P/4cRJ2Xl+5TcJwwTDVWA+baoEPWQdmpGJSdKoJxfY8e5SNyOBxobm4OeBCBVDQIw7L8NFVrzXp/jyhHJKuSFK03UNFcI64dRM8SotWrV0Ov1/sfBQUFYpuUdNidHsgkHEp8v+SMtekRkRAFoMsXts3nOm3GGIO7N6bNTSJ6lBCtWLECJpPJ/6iurhbbpKTj0kFZuHNqCUbke6OFbU3CrJlU1rrYkxDQen1mlnqhskkQ/lNxAet3VeBAlTFxdvVBepQXTqlUQqnsRZVH44RC1ub3xTcs66RqR59FlQbI1YDLJsQT+XpIbZBJOTjdPC600BR+PKFvZi/C5gwyDU3+odBwXJvhWXA/UaY3lqiehCiuiNojamlpwYkTJ/zPT506hYMHDyIjIwOFhYUiWtbzcHl4bPzyFPRqOa4p7QeNb8o5zKodfRZtnlBeqPksgHEdDmfrhJLbjRYnnG4+sLdJxAxR39V9+/ahtLQUpaWlAIDly5ejtLQUK1euFNOsHkl1oxVON+8vjgjA66j2Dc1IiIKi7ydsTWeCHk5VyqBVycAY9Yriiag9omnTplGahRhx0hfEaNC0Zhd0NAOOFiHlhZaCGYOi6y+8P3YTYDMC6rQOTbJ1KpjtLTjfbEe/NHXCTewLUD+zF8AYw0lv/FCJoU0Ans/vkWoQKlcQHZEpWkXaWBW0SY5WmCDxLyAmYg4JUS+gttkOi8MDhUyC/ukprQeazwpbbcfZIKINaV5/pCl4OEh+mhoDDRrk6ak3FC961PQ9EZzj54Xe0MCsdhU4fBHDQaaliTakFQJVX4fsERVkpKAgIyXoMSI2UI+oh8MYw4k6QYgC1kXxHhKicNEXCH4im7E1XQqRUEiIegFTBmfhojwtijLbpPjwRQvLlK3J4ongyBSAvr/w94WTIZu1ONyoM5OfKB6QEPVwOI7D4BwtZo/MC4xx8U1H6/KDVqkg2pE5SNheOBH08PHzZry05yQ+/Sl4Rkeie5AQ9VaaTgvbtCJRzegx+ITIWCmUXWpHjl4IbDzf7ICTylDHHBKiHkxDiwNfnWhAk6XdPw7Ptzpe00mIwiIlA1CnC741n4i3QaeSQ6eWg2cM54y2xNvXyyEh6sH8cK4Ze0814quKdhkGLXWA2yH4PigrY3hwXGuvqP7noE0KvTNnpy5YEmVVn4GEqIfi4Rl+rhESw12U165ufVOlsNUX0or7SMi+SNg2HA06PPNlvDxZb6EVATGGvqU9lMoLQl37FIU0cLYMaHW4phcn3K4ejS5fGJ553EDDsQ6HCzNSIJNwaLa5KKF+jCEh6qF8f8YIABiWpwsMYnRaW2fMsgYl3rCeDMcBOSOEv88f6XBYLpWgMFMYnp30VkohYgMJUQ+k0eLE6QYrOA4Y2z+t3cEKoTJFqkH4dSciI3ekIEiNp4RSQ+0YW5CG2SNzcXERvbexhISoB/JdlRD9O9CQCn1Ku8Ws9UeFbdaQBFvVS1Cntzqtz+7vcLgoU4OL8nSQS+lfJ5bQu9kD0ShlUMgkuLgwLfCAo0VI8gUA2cMTblevoZ83QVrtIcAZeoaMHNaxg4SoB/KLgZm47bIBgSvtAcGvwXjB6arJEse43kB6MaDNEYoOVP6nw2HGGA5WG7Hpy9MwWV2Jt68XQkLUQ/HXLPPB88C574S/88Yk3qDeBMcBA6cJf5870GEhLMdxOFnfApPNhb2ngleJJSKDhKiH4OEZ3jt0DmearMEb1P0orB5XpNCwLBakDxB6RrwHOPqBkHa3Db8YKCwk/uFcM85SpHW3ISHqIew6Wofj51uw/VANXO2L/XlcwKk9wt/9JwgR1UT34DhgyCyhHlzTaeDMvoDD+Wlqf+24nUdqg1dQIcKGhKgHsL+yEYfOmMBxwMwRuR1nbE7tEXIuq3RAv/HiGNkbSckABk4X/q74BGgIXJn/yyEG6NVymGwuvHPwLKzO4EUaia5JCiF64YUXUFxcDJVKhUsuuQTffPON2CYlBR6e4YvjDdhzrAEAMHlQFgZktYuiPv8DUO19vwbPpN5QrOk3DsgbLQzNftgmvN9eVHIprh6bD5VcilqTHW98U02lqaNEdCF68803sXz5cqxatQoHDhzAmDFjMGvWLNTV9e28L9WNVmz9tgrfnm4EAFxakokJxW1KRvM8ULUX+OnfwvOCCUDWYBEs7eVwHDBkNmAYKviLfnxXeNiMAICsVCVumFCADI0Cw/N0kHl7q2a7C58fr8ehM0acarCgrtkOs90FD09T/sHgmMjBEJdccgkmTJiA559/HgDA8zwKCgrw3//933jwwQc7fW1zczP0ej1MJhN0Ol0izI05PM9gd3tgcXigkkugVQkBit9VNWHX0Xoo5RLMuCgHQ7JTAbddmMFpqhRiXKyCSCFvjPDPQgtc4wfPA6d2A9V7hd4RJwEyBgAZJYA2B25lGiBTQyYTZjNPN1jw9ndng55KrZDiskFZGNlPD0AQraO1ZqQoZNAopVArpNAoZFDLpZBIenZSu3D/R0VNnu90OrF//36sWLHCv08ikWDGjBn4z386xm84HA44HK1F7pqbm8O+1lmjDZ/+JBQb9ClvWwmeOCADF+UJb1StyY73j9QEtGmr1uOL0jGmIE2w4ZstOFxR3eZkvP8FDEBBugpFGRoADFanW4iKZkIDBsDD8/7XFmemQJuRAoBhNM+QbrUgP00NxXEOOOoWFmO2RaYESi4XhIiyMMYXiQQomS5ErJ/+XFgCcqHCH0AqA4TPQKYEJHJkOXnMtTlgdzPY3QxON4PLw4P3ftYZTh1wVki25rQ4Yao2wtTukhwnrG8bZEhFnjcxm9Hmws81Zv/HfbzgN2BcayjHuKIMDPc60evMduw8UhvylsYWpGNUf0EMGy1ObD8cvOw2AIzM16G0UFjW0mx3QS6RQK2QhmwfKaIKUUNDAzweD3JyAuuy5+Tk4OefO+aEWb16NR5++OGoruVy852umLa7Wmc93DwPYyeBas62fgDrBUisDSHbSmxqwCZcl3N5IHEEiqfvo5RJOHBOBjiZf3+xTgLwDr+2ARCm53X9hF/jnJHCF59IHPp+wJjfApYGYYW+sVrID+4wCz8oLjsAO1IBDGmXnYUBcHuE74+CawYswsJZpcONIqUdLg8Pp4eHy8PD7WH+Hz+J1Q7IBCGCzQlmMfuPNbQ4A4TI1vZ77GGdfufbOtfdPI8Gc+hKttY2s4I8z8AksR1I9ahyQitWrMDy5cv9z5ubm1FQUBDWa3N0Klx3sZAgPVjnIa3Nmi2DVhj3+/C15yD8kapqfdvUo+dh8EBbm0acdyMMk9RyGaCQAhwHOc8wbJQbACe04zgopRyUcpl3BT3XxrjA84GTAEotFUpMFjRZwsOXAJP3AC6b8GAeIcK97QPCJyr3PtqSCsCfJ8HbY+KZ8ONod3uQIpcBcu/3yc2j2Or0d8D76woDvtBt1x5maBT4zbj+IW9Bp2ptq1fL/f8fQduqW7/zGqUMkhj3wEUVoqysLEilUpw/fz5g//nz55Gb2zGzoFKphFIZXQ9ArZD6Uzh0hVImRX6YpYXlGUXIyui6HSC82bRmu5cikQLKVOERi9MBSPE+2qIEkBfmOVRyadj12JSy8P8/4rHgV1TvpkKhwLhx4/DJJ5/49/E8j08++QSTJk0S0TKCIBKJ6EOz5cuXo6ysDOPHj8fEiROxdu1aWCwW3HrrrWKbRhBEghBdiG644QbU19dj5cqVqK2txdixY/HBBx90cGATBNF7ET2OqDv0hjgigujNhPs/ShFwBEGIDgkRQRCiQ0JEEIToiO6s7g4+91YkSz0Igkgcvv/NrlzRPVqIzGYzAIQdXU0QhDiYzWbo9fqQx3v0rBnP8zh37hy0Wi24Hr7o07dcpbq6utfMANI9JT/xvh/GGMxmM/Lz8yHpJDtEj+4RSSQS9O8fen1MT0Sn0/WKL3hb6J6Sn3jeT2c9IR/krCYIQnRIiAiCEB0SoiRBqVRi1apVUWcXSEbonpKfZLmfHu2sJgiid0A9IoIgRIeEiCAI0SEhIghCdEiICIIQHRKiBBJJRduXXnoJU6ZMQXp6OtLT0zFjxoykrIAbbZXerVu3guM4zJ8/P74GRkik92M0GrFkyRLk5eVBqVRiyJAh2LFjR4KsDY9I72nt2rUYOnQo1Go1CgoKcN9998Fut8fXSEYkhK1btzKFQsE2btzIfvjhB3b77beztLQ0dv78+aDtFyxYwF544QX23XffsZ9++omVl5czvV7Pzpw5k2DLQxPpPfk4deoU69evH5syZQqbN29eYowNg0jvx+FwsPHjx7Mrr7ySffHFF+zUqVNs165d7ODBgwm2PDSR3tOWLVuYUqlkW7ZsYadOnWI7d+5keXl57L777ournSRECWLixIlsyZIl/ucej4fl5+ez1atXh/V6t9vNtFote+WVV+JlYsREc09ut5tdeuml7P/+7/9YWVlZUglRpPezYcMGNnDgQOZ0OhNlYsREek9Llixhl19+ecC+5cuXs8mTJ8fVThqaJQBfRdsZM2b493VW0TYYVqsVLpcLGRlh1i6KM9He0yOPPILs7GzcdtttiTAzbKK5n3fffReTJk3CkiVLkJOTg5EjR+Lxxx+Hx+MJ2j7RRHNPl156Kfbv3+8fvp08eRI7duzAlVdeGVdbe/Si155CpBVtg/HAAw8gPz8/4EslJtHc0xdffIGXX34ZBw8eTICFkRHN/Zw8eRKffvopFi5ciB07duDEiRO455574HK5sGrVqkSY3SnR3NOCBQvQ0NCAyy67DIwxuN1u3HXXXfj9738fV1upR9QDeOKJJ7B161a8/fbbUKlUYpsTFWazGTfffDNeeuklZGVliW1OTOB5HtnZ2fjb3/6GcePG4YYbbsAf/vAHvPjii2KbFjW7du3C448/jvXr1+PAgQPYtm0btm/fjkcffTSu16UeUQKItKJtW5588kk88cQT+PjjjzF69Oh4mhkRkd5TRUUFTp8+jauuusq/j+eFUswymQxHjx5FSUlJfI3uhGg+o7y8PMjlckilrbXnL7roItTW1sLpdEKhUMTV5q6I5p7++Mc/4uabb8bixYsBAKNGjYLFYsEdd9yBP/zhD53mFOoO1CNKANFWtP3zn/+MRx99FB988AHGjx+fCFPDJtJ7GjZsGA4fPoyDBw/6H1dffTWmT5+OgwcPip5lM5rPaPLkyThx4oRfUAHg2LFjyMvLE12EgOjuyWq1dhAbn9CyeC5LjasrnPCzdetWplQq2ebNm9mPP/7I7rjjDpaWlsZqa2sZY4zdfPPN7MEHH/S3f+KJJ5hCoWD/+Mc/WE1Njf9hNpvFuoUORHpP7Um2WbNI76eqqopptVq2dOlSdvToUfbee++x7Oxs9qc//UmsW+hApPe0atUqptVq2RtvvMFOnjzJPvzwQ1ZSUsL+67/+K652khAlkOeee44VFhYyhULBJk6cyL7++mv/salTp7KysjL/86KiIgagw2PVqlWJN7wTIrmn9iSbEDEW+f189dVX7JJLLmFKpZINHDiQPfbYY8ztdifY6s6J5J5cLhd76KGHWElJCVOpVKygoIDdc889rKmpKa42UhoQgiBEh3xEBEGIDgkRQRCiQ0JEEITokBARBCE6JEQEQYgOCRFBEKJDQkQQhOiQEBEEITokRESPYvPmzUhLS/M/f+ihhzB27Fj/8/Ly8qRLP0t0DQkREZTy8nJwHIe77rqrw7ElS5aA4ziUl5cHtI+1ABQXF2Pt2rUB+2644QYcO3Ys5GvWrVuHzZs3+59PmzYN9957b0ztImIPCRERkoKCAmzduhU2m82/z2634/XXX0dhYaEoNqnVamRnZ4c8rtfrA3pMRM+AhIgIycUXX4yCggJs27bNv2/btm0oLCxEaWlpt84drKcyf/58fy9r2rRpqKysxH333QeO48BxHICOQ7P2tO2ZlZeXY/fu3Vi3bp3/HKdOncKgQYPw5JNPBrzu4MGD4DgOJ06c6NZ9EdFBQkR0yqJFi7Bp0yb/840bN+LWW2+N+3W3bduG/v3745FHHkFNTQ1qamoiPse6deswadIk3H777f5zFBYWdrgnANi0aRN++ctfYtCgQbG6BSICSIiITrnpppvwxRdfoLKyEpWVlfjyyy9x0003xf26GRkZkEql0Gq1yM3N7TKTZTD0ej0UCgVSUlL855BKpSgvL8fRo0f9CeJdLhdef/11LFq0KNa3QYQJpYolOsVgMGDu3LnYvHkzGGOYO3duj885nZ+fj7lz52Ljxo2YOHEi/v3vf8PhcOD6668X27Q+C/WIiC5ZtGgRNm/ejFdeeSVmvQaJRNIh9ajL5YrJucNh8eLFfkf8pk2bcMMNNyAlJSVh1ycCISEiumT27NlwOp1wuVyYNWtWTM5pMBgC/D4ejwdHjhwJaKNQKLpdIyzUOa688kpoNBps2LABH3zwAQ3LRIaGZkSXSKVS/PTTT/6/Q2EymTrULMvMzAyaGP/yyy/H8uXLsX37dpSUlODpp5+G0WgMaFNcXIw9e/bgt7/9LZRKZVRDwuLiYuzduxenT59GamoqMjIyIJFI/L6iFStWYPDgwZ0WMSDiD/WIiLDQ6XTQ6XSdttm1axdKS0sDHg8//HDQtosWLUJZWRluueUWTJ06FQMHDsT06dMD2jzyyCM4ffo0SkpKYDAYorL7/vvvh1QqxfDhw2EwGFBVVeU/dtttt8HpdCZkFpDoHMpZTfRZPv/8c1xxxRWorq7uUA2VSCwkRESfw+FwoL6+HmVlZcjNzcWWLVvENqnPQ0Mzos/xxhtvoKioCEajEX/+85/FNocA9YgIgkgCqEdEEITokBARBCE6JEQEQYgOCRFBEKJDQkQQhOiQEBEEITokRARBiA4JEUEQovP/AUpMjqFA0wq9AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:23:57.999055Z", + "iopub.status.busy": "2024-02-29T06:23:57.998731Z", + "iopub.status.idle": "2024-02-29T06:23:58.353860Z", + "shell.execute_reply": "2024-02-29T06:23:58.352981Z" + }, + "papermill": { + "duration": 0.377542, + "end_time": "2024-02-29T06:23:58.355799", + "exception": false, + "start_time": "2024-02-29T06:23:57.978257", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEqCAYAAACx7V3bAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABIEElEQVR4nO2deXxTZfb/Pzd70qTpvkFLyw6yCsIgKKKMCg7i6CiOjrKNjmOdGeXrMow/QZxxQEcdmO84fL8yAjqiuHzRGVdUFFAQRBZFRQqlG0vpmqXZk/v8/rjJbdMmbZKmuWl73q/XfaW598m95zbJJ89znvOcwzHGGAiCICREJrUBBEEQJEQEQUgOCRFBEJJDQkQQhOSQEBEEITkkRARBSA4JEUEQkkNCRBCE5JAQEQQhOSREBEFIjqRCVFxcDI7jOmylpaVSmkUQRIJRSHnxAwcOwOfzic+//fZb/PjHP8aNN94ooVUEQSQaLpkWvd5777145513cOLECXAc12V7nudx9uxZGAyGiNoTBJFYGGOwWq0oKCiATBZ+ACZpj6gtbrcbL730EpYtWxZWVFwuF1wul/j8zJkzGD16dKJMJAgiRmpqajBw4MCwx5NGiN566y2YTCYsWrQobJvVq1dj1apVHfbX1NQgNTW1B60jCCIWLBYLCgsLYTAYOm2XNEOzq666CiqVCm+//XbYNu17RIGbNJvNJEQEkYRYLBYYjcYuv6NJ0SOqqqrCxx9/jG3btnXaTq1WQ61WJ8gqgiASRVLEEW3atAk5OTm45pprpDaFIAgJkFyIeJ7Hpk2bsHDhQigUSdFBIwgiwUj+zf/4449RXV2NJUuWSG1K0sAYg9frDYqxIohkRC6XQ6FQdDt8RnIhuvLKK5Ek/vKkwO1249y5c7Db7VKbQhARodPpkJ+fD5VKFfM5JBciohWe51FRUQG5XI6CggKoVCoK1CSSFsYY3G436uvrUVFRgWHDhnUatNgZJERJhNvtBs/zKCwshE6nk9qc5ID3AS4LwBigNgBypdQWEW3QarVQKpWoqqqC2+2GRqOJ6TwkRElIrL8qfQ7GAIcJ8LmF57wXSMkCOPr/JBPx+LzSO0okL16XIEIcAI4Tekceh9RWET0ACRGRvHj8DntlijAsA0iI+igkRIRkLFq0CNddd13og7wP8PmX8yi1gEIj9Ix8HuFYGC677DLce++9nV63uLgYa9eujcVkoocgISLiQiQCEBU+N8AAyBWCg1omB2TK1mNEj/Poo49iwoQJCbkWCRGRnHj9vSF5m7WFcn+cipeEqK9BQtRLcHv5sJvXx0fc1hNB22hZtGgRdu3ahXXr1onpfsvLy7F06VKUlJRAq9VixIgRWLduXcjXr1q1CtnZ2UhNTcVdd90Ft8vV2utRtAmSCwgRLxyz2Wy4/fbbodfrkZ+fj6effrrDuevq6jBv3jxotVqUlJRgy5YtHdpwHIf169djzpw50Gq1GDx4MN544w3xeGVlJTiOw2uvvYZLLrkEWq0WF110EcrKynDgwAFMnjwZer0ec+bMQX19fcT/t40bN+KCCy6AWq1Gfn4+7rnnHvFYdXU15s+fD71ej9TUVNx00004f/68eDzQW/nXv/6F4uJiGI1G3HzzzbBarWIbnufx5JNPYujQoVCr1SgqKsLjjz8uHn/ooYcwfPhw6HQ6DB48GI888gg8Hg8AYPPmzVi1ahW+/vpr8T3dvHlzxPcWLTR930t49tOTYY+VZKXguokDxOfP7S6Hxxc6Wn1guhY3Ti4Un2/cUwGHO9jnct+Ph0dl27p161BWVoYxY8bgscceAwCkp6dj4MCBeP3115GZmYm9e/fizjvvRH5+Pm666SbxtTt27IBGo8HOnTtRWVmJxYsXIzMjHY///rfCTJm8rRD5P668F2AMDzzwAHbt2oV///vfyMnJwR/+8AccOnQoaDixaNEinD17Fp9++imUSiV++9vfoq6ursM9PPLII1izZg3WrVuHf/3rX7j55ptx9OhRjBo1SmyzcuVKrF27FkVFRViyZAluueUWGAwGrFu3DjqdDjfddBNWrFiB9evXd/k/W79+PZYtW4Y1a9Zgzpw5MJvN2LNnj3B7PC+K0K5du+D1elFaWooFCxZg586d4jnKy8vx1ltv4Z133kFzczNuuukmrFmzRhSb5cuXY8OGDfjrX/+KGTNm4Ny5c/jhhx/E1xsMBmzevBkFBQU4evQo7rjjDhgMBjz44INYsGABvv32W3zwwQf4+OOPAQBGo7HL+4oZ1osxm80MADObzVKbEhccDgf7/vvvmcPh6HDsmQ+Ph93ePHQ6qO1/7ygL2/a1A9VBbdfvPNmhTSzMnDmT/e53v+u0TWlpKbvhhhvE5wsXLmQZGRnMZrO12rN+PdPr9czXfJqxlvrgE/A8Y5ZaxsxnmdXUyFQqFXvttdfEw42NjUyr1Yp2HD9+nAFgX375pdjm2LFjDAD761//Ku4DwO66666gS02dOpX9+te/ZowxVlFRwQCwf/7zn+LxV155hQFgO3bsEPetXr2ajRgxotP/QYCCggL28MMPhzz24YcfMrlczqqrW9+r7777LuheVq5cyXQ6HbNYLGKbBx54gE2dOpUxxpjFYmFqtZpt2LAhInsYY+wvf/kLmzRpkvh85cqVbPz48V2+rrPPbaTfUeoR9RJKZw0Ne0zWbhXInZcOCdu2/YqRJdNLumNWpzz77LPYuHEjqqur4XA44Ha7Ozg/x48fHxRFPm3aNLS0tKDm9FkMGjoi+IQcJziuvS6UnyiD2+3G1KlTxcMZGRkYMaL1NceOHYNCocCkSZPEfSNHjkRaWloHW6dNm9bh+ZEjR4L2jRs3Tvw7NzcXADB27NigfaF6W+2pq6vD2bNnccUVV4Q8fuzYMRQWFqKwsLXnOnr0aKSlpeHYsWO46KKLAAizf20zH+bn54vXP3bsGFwuV9hrAMCrr76Kv/3tbygvL0dLSwu8Xq9kCQbJR9RLUClkYTeFXBZxW2UEbePB1q1bcf/992Pp0qX48MMPceTIESxevBhudxSO5lDLOWSB4VniMxMola32BNYAtt/H81372LRabdztaX/9rq7xxRdf4NZbb8XcuXPxzjvv4PDhw3j44Yeje3/iCAkRERdUKlVQ2pI9e/bg4osvxt13342JEydi6NChKC8v7/C6r7/+Gg5Ha5Divi++gF6fgsKBBa3T9W3xC9GQ4kIolUrs379fPNTc3IyysjLx+ciRI+H1enHw4EFx3/Hjx2EymTqcdt++fR2et/UPxRODwYDi4mLs2LEj5PFRo0ahpqYGNTU14r7vv/8eJpMp4mIRw4YNg1arDXuNvXv3YtCgQXj44YcxefJkDBs2DFVVVUFt2r+nPQkNzYi4UFxcjP3796OyshJ6vR7Dhg3Diy++iO3bt6OkpAT/+te/cODAAZSUBA8F3W43li5div/3//4fKisrsfLRR3HPHYshkyuE2KH2+IVIr9Ng6dKleOCBB5CZmYmcnBw8/PDDQeueRowYgauvvhq/+tWvsH79eigUCtx7770hewuvv/46Jk+ejBkzZmDLli348ssv8fzzz8f3n9SGRx99FHfddRdycnIwZ84cWK1W7NmzB7/5zW8we/ZsjB07FrfeeivWrl0Lr9eLu+++GzNnzsTkyZMjOr9Go8FDDz2EBx98ECqVCtOnT0d9fT2+++47LF26FMOGDUN1dTW2bt2Kiy66CO+++y7efPPNoHMUFxejoqICR44cwcCBA2EwGHosVTP1iIi4cP/990Mul2P06NHIzs7GVVddheuvvx4LFizA1KlT0djYiLvvvrvD66644goMGzYMl156KRYsWIBrr5mDR5f/lzAsC5UCJSBOvA9/efIJXHLJJZg3bx5mz56NGTNmBPmDACENcUFBAWbOnInrr78ed955J3JycjqcdtWqVdi6dSvGjRuHF198Ea+88kqPlqpauHAh1q5di3/84x+44IIL8JOf/AQnTpwAIAyx/v3vfyM9PR2XXnopZs+ejcGDB+PVV1+N6hqPPPII/uu//gsrVqzAqFGjsGDBAtGHdO211+K+++7DPffcgwkTJmDv3r145JFHgl5/ww034Oqrr8asWbOQnZ2NV155JT43H4KkqeIRC5FWCOgtOJ1OVFRUoKSkJOZ0Cr0epxlw2wFVCqAJ855azwOMF1bixyEtCMdxePPNN8MvNyE6pbPPbaTfUeoREckF7xUeOxMYsVfk7Xl7iIRAQkQkD4wJi1oBVJ8+C71eH3rLLUZ1zWlJZs4iJaztej0+++wzqc1LOshZTSQPzCeIEQcUDCzqEMcj4m5BQV5m3ISoJ7wTYW0HMGDAgLDH+iuSC9GZM2fw0EMP4f3334fdbsfQoUOxadOmiGcHiD6Ezz/UkimgUCoxdGiYIE6PHXCYAZa8Q7OwthMhkVSImpubMX36dMyaNQvvv/8+srOzceLECaSnp0tpFiEVAZ9PqPihtnCtM2dE30BSIXriiSdQWFiITZs2ifvax5kQ/Qhe8A+J0dPhCBwXh3JU6aS3I6mz+j//+Q8mT56MG2+8ETk5OZg4cSI2bNgQtr3L5YLFYgnaiD6EOGPWhRBxMkF8GAQxIno9kgrRqVOnsH79egwbNgzbt2/Hr3/9a/z2t7/FCy+8ELL96tWrYTQaxa3tokCil8NYFEMzrrWSBw3P+gSSBjSqVCpMnjwZe/fuFff99re/xYEDB/DFF190aO9yueByucTnFosFhYWFFNDYF/B5AFuDIDL63K6HW/YmIYujxgioqAaclPT6gMb8/PwOYfSjRo1CdXV1yPZqtRqpqalBG9FH4FtnzCLy+QSCGmlo1ieQVIimT5+O48ePB+0rKyvDoEGDJLKIiJVuJ89vK0SR0MXMWdyT+RM9iqRCdN9992Hfvn3485//jJMnT+Lll1/Gc889h9LSUinNIqSA9wm5cCIWIv9Hl0WfY5tIQiLOI9lDvP3222zMmDFMrVazkSNHsueeey7i1/aLVLE8z5jHlfiN5yO2e+HChQz+OazAdvLkSbZkyRJWXFzMNBoNGz58OFu7dm2H182fP5/96U9/Yvl5uay4qJAxt4Pt2bOHjR8/nqnVajZp0iT25ptvMgDs8OHD4muPHj7Irp49i6Wk6FhOTg77xS9+werr68PaU1FR0Z23huiEPpEq9ic/+Ql+8pOfSG1G8uLzAJ91rE7R41zyX8EVNDqhu8nzU1NT8dG/XwUYg6XFjnnz5mHu3Ll4+eWXUVVV1WGIZTKZcPmVV+OXt92Mv65ZBYdMj4ceegg33XQTPvnkk5D2ZGdnx+f/QvQIkgsR0fsxGo1QqVTQ6XTIy8sT969atUr8u6SkBF988QVee+21ICFKSUnBP5/7H6jcZoDj8D8vvQqO47BhwwZoNBqMHj0aZ86cwR133CG+5u9//zsmTpyAP69cLuww5GLjxo0oLCxEWVkZhg8fHtIeInkhIUp25EqhdyLFdbtJJMnzx44dK+TJdgOQyXG8rAzjxo0LmgaeMmVK0Gu+/vprfPrpTugLAuu5WmfZysvLMXx4dOWQCOkhIUp2OC7iIVIyEUie//TTT2PatGkwGAz4y1/+EpRjGhB6ROLMV4SO6paWFsybNw9PrHwQ8PkAbZr4P8rPz4/nbRAJgoSIiAudJc8PECp5PoCgqfsRI0bgpZdegsvlEvMjHzhwIKj5hRdeiP/7v/9DcXEJFPCFDGpMZOJ3ovtQYjQiLrRNnt/Q0IBhw4bhq6++wvbt21FWVoZHHnmkg6CItBGiW265BTzP484778SxY8ewfft2PPXUUwBaS/iUlpaiqakJP190Jw4cPILykyewfft2LF68WBSf9vZEUuaHkA4SIiIuxJo8H0AbIZIjNTUVb7/9No4cOYIJEybg4YcfxooVKwBA9BsVFBRgz5498PE8rvzpzzF28jTce++9SEtLE6t4tLcnXLQ+kRxQ8vwkol+uNWO8kAwfAAy5rYGKbdiyZQsWL14Ms9kcXArIbReS7SvUgC4jQQYT7YnHWjPyERHSIjqqZaIIvfjiixg8eDAGDBiAr7/+WowR6lCPLFDDjNab9XpIiAhpCQzLuNZiirW1tVixYgVqa2uRn5+PG2+8EY8//njH11Kmxj4DCREhLSGm7h988EE8+OCDXb82IESMCUO8EMM6ondA7xwhLW0c1VHDca0pQ2jxa6+GhCgJ6cXzB9HDogtmDCIoUyMJkVTE4/NKQpREKJXCsgq73S6xJQmkOz0ioE06EPITSUXg8xr4/MYC+YiSCLlcjrS0NNTV1QEAdDqdGMTXJ2E84PSn/lV4AVkMvRqPD/B6ADgAXx/+XyUhjDHY7XbU1dUhLS0NcnmMPyYgIUo6AqvFA2LUp+F9gLsFAAdobLGdw+MAfG4hlkjRT2Kvkoy0tLRuZzkgIUoyOI5Dfn4+cnJy4PF4pDanZ2mqAE7sB/Q5QMlPYzvHmUPA6QOo0xTjCzYGSjmHKSWZyNKr42srERKlUtmtnlAAEqIkRS6Xx+UNTmq8ZoC3A1odEGskuS4FVrsZ35+pwNnsUcKuWjuuHmOMo6FET0NCREiHo1l41HajxLhKj8pGO5S8AoOzUzA4S48xA3r/cp/+BgkRIR0Ok/DYDSGqdyvR4vJCrXBg2uhc6FT0ke6N0PQ9IR2BHpEmLeZTfNsgTNvnaXzQKVpnzTw+Hk02d3esIxIICREhDTwPuKzC39q0mE7BGEN5Mw8GTnBOu4WZt5omO57bfQrvHT0XJ2OJnkZSIXr00UfBcVzQNnLkSClNIhKFyyLEEcnkgEof0yk4jsONk4swKD8HqVqlPxQAyNKr4fHxqLe6YHH28ZnHPoLkA+oLLrgAH3/8sfhcoZDcJCIRuCzCozo1shLTYTDqlDDmZAHWWrFHpFXJkW/U4KzJiaoGO8YOpBm0ZEfyb71CoaCSL/0Rp1l41MRhhkttEIQoMNQDMCgzBWdNTlQ22kiIegGS+4hOnDiBgoICDB48GLfeemunKT1dLhcsFkvQRvRSnP73ThObSDDG8PH353G4uhkeuT9hmrs1OrsoQ0imf8bk6F+LiHspkgrR1KlTsXnzZnzwwQdYv349KioqcMkll8BqtYZsv3r1ahiNRnErLCxMsMVE3Aj0iNSx9YgsTi+OnjFjd1kDOLVB2On3EQFAjkENhYyDw+2DyU5+omRHUiGaM2cObrzxRowbNw5XXXUV3nvvPZhMJrz22msh2y9fvhxms1ncampqEmwxETdc3esRnbc4AQDZBjUUGr+z29UqRAq5DLmpQrT2WbMjdjuJhCC5j6gtaWlpGD58OE6ePBnyuFqtFmtdEb2cbvqIas2CEOUZ1YKPCAjqEQHAmAFGDMlJQYFR2/7lRJIhuY+oLS0tLSgvL6dqnX0dxlp9RDEOzRptQvqQbL2mdfq/nRCNLkjFpEEZSE/pfZVy+xuSCtH999+PXbt2obKyEnv37sVPf/pTyOVy/PznP5fSLKKncduEhGgcF/PQrLFFiJrO1KsAVYr/vHbK1NhLkXRodvr0afz85z9HY2MjsrOzMWPGDOzbtw/Z2dlSmkX0NAH/kColpsyMTo8PVqeQ2TEjRQXIVYKoMR7wOlqFCUCzzY3zVicK0rRI1cSeQZDoWSQVoq1bt0p5eUIqRP9QbL0hs8MDjgP0agU0Sr+QKbVCj8htCxKiT4/XoarRjtmjcimeKIlJKmc10U/opn8oN1WD0llD0eLvFQEQxMdtF4Ia9Tni7myDGlWNdtS3OAGQECUrSeWsJvoJ3ewRAYBSLgt2QosO6+CUs9kGYZa13uqK+VpEz0NCRCQeMYYojgnMRId1OyHyp4xtaHFThHUSQ0JEJB6nSXiMMQ/Ru9+cw8ffn0eLq+3QLHSPKF2ngkLGwe3lYXZQhHWyQkJEJJ5u+Ig8Ph5l5604esYMedtV+6IQBS8Pksk4cQjXSInSkhYSIiKxeJyA1++vicFHZPH3alQKGTTKNh/fMEMzwD/FD2Eqn0hOaNaMSCwB/5BSAyiij3g2+YUoTacMLj7ZiRCNL0zD8FwDclNpeVCyQkJEJJZuzpgFVtKnaduJWJj1ZgAwII3WmiU7NDQjEks3Y4gCQzOjtl2UdKBH5HULG9GrICEiEks3Z8xMDkFk0nTthEiuAuT+Dn6IXlF5fQu+rGiCw+2L6bpEz0JCRCSWbsYQub3CotYOPSKOCzuFDwC7y+qx52QDGloosDEZIR8REXcYY8GO5LZ000e04KIiuLw+KGQhfkNVKULRxjAzZya7B002Nwr9aWSJ5IF6RERcqbM68dK+Kjg9YYZA3fQRAYBaIYdcFkLoOukRpesE5zYVXUxOSIiIuPLpD3VoaHHjUFVzx4M+b6tIdGOdWVjCBDUCrbFEAR8TkVyQEBHdxuzw4Ky/WsaEQqGO/ffnLOD5dmu7Av4huUJI2xElJ+us2HboNA5VhxA5oNNYooBPyUyJ9JMSEiKi23x3xoxXD9Tgkx/qMCQ7BRqlHFanF+f8Ce5F2s6YxVBUsc7qQlWjPXyEdGdC5J9lMzu8HQWSkBwSIqLb1DTbAQh5ghRyGQZlCs7gqsZ2gtBN/1AgK6MhXKbFToIaDWoFFDIOPGPieYjkgWbNiG7h8fE4bxGmxAemC8OtogwdjtdacbrJAQxp07ibM2atQhTmY9tJj4jjOFw7oQA6lQL6cK8nJIPeEaJb1Jqd8PEMBo1C9MMEllSctwjHxBmubsYQWZ2CfyciIeJ5oN0U/6DMlBAvIpIBGpoR3aLW7wfKN2rF2KE0nRJalRwZehXs7jbDoG70iBhjYmpYgzrM0EyZ4k+izwCPPeprENKRNEK0Zs0acByHe++9V2pTiCho8KdgDaRkBYRh0B2XDMatUwcF+3O64SOyu33w8kxImh+uRySTtc7GhRiemexufFXZhK9rTFFfn+hZYhKiU6dOxdWIAwcO4H//938xbty4uJ6X6HnqWzoKEYCOAYc8LyS2B2LqETk8PmhVcqSoFKGDGQOEKbYICCv3PzvRgG9Om6K+PtGzxCREQ4cOxaxZs/DSSy/B6XR2/YJOaGlpwa233ooNGzYgPT29W+ciEs/M4dmYMSwrbK4fMU+02yrUHeNkrWIRBVl6Ne6aOQSLpxd33rCT6Oo0cQrfQ/mrk4yYhOjQoUMYN24cli1bhry8PPzqV7/Cl19+GZMBpaWluOaaazB79uwu27pcLlgslqCNkJZBmSm4qDgDOlXwcKnF5cWW/VXY8Nkp4UvfttZ9qHViEaKQd/Fa0WEdYgpfo4SM4+DxseB814TkxPSJmDBhAtatW4ezZ89i48aNOHfuHGbMmIExY8bgmWeeQX19fUTn2bp1Kw4dOoTVq1dH1H716tUwGo3iVlhYGIv5RALQKuVosLphc/mEL30c1phFRCdT+HIZJ864mSjCOqnolrNaoVDg+uuvx+uvv44nnngCJ0+exP3334/CwkLcfvvtOHfuXNjX1tTU4He/+x22bNkCjUYT0fWWL18Os9ksbjU1Nd0xn+gmZ0wOlJ23hqyOIZdx4lCoyebudgzRzuN12HboNKobu5gN6ySoEWgdnlmcJETJRLeE6KuvvsLdd9+N/Px8PPPMM7j//vtRXl6Ojz76CGfPnsX8+fPDvvbgwYOoq6vDhRdeCIVCAYVCgV27duFvf/sbFAoFfL6Oq7fVajVSU1ODNkI6jp424d1vzqHsfMdFpkDrQtPGOAjRObMTVY12uH185w076REBQKp/Fs/ioKFZMhFTQOMzzzyDTZs24fjx45g7dy5efPFFzJ07FzL/2L+kpASbN29GcXFx2HNcccUVOHr0aNC+xYsXY+TIkXjooYcgl8tjMY1IIM3+4U16+2yJfjJTVDgJoLHFDbi7G1UtXCu1q6jogBC5QveIUrXUI0pGYhKi9evXY8mSJVi0aBHy8/NDtsnJycHzzz8f9hwGgwFjxowJ2peSkoLMzMwO+4nkJOBnMbZPZO8nQ+8v42N3A97Yhcjr42FzCT3ksOvMAnQyfQ8Ao/INKM7SdczwSEhKTEL00UcfoaioSOwBBWCMoaamBkVFRVCpVFi4cGFcjCSSD4fbJyY/65A/2k+g0obZ5gYQWN4RvRAFZriUci64llkoAj0in0dIot+uZJFBo+xazIiEE5MQDRkyBOfOnUNOTk7Q/qamJpSUlIT070TCzp07Y3odkXia7UIqDoNGAWWYKfU0nRKZehUylR4wixecTB7TrFnbVfdhU9AGUKgBuVIQIncLoMiI+npE4onJWR0uGKylpSXiGTCidyPWF9OFL5KoUcpx+7RiXDPMvw5NrY8phijgz9GrI/zd7CSoEQAOVTfjkx/OUyxREhFVj2jZsmUAhLVEK1asgE7XmoTc5/Nh//79mDBhQlwNJJITsaxPJL6Wbs6Y8TygVcnDr7pvjyoFcDSH9RN9XWOCye7BsBxD5OJG9ChRvQuHDx8GIPSIjh49CpWq9ddQpVJh/PjxuP/+++NrIZGUXFBgRI5BE9EXmTlMYIxBFqMQjR1oxNiBxsgzK0YwhW+ye2jmLImISog+/fRTAMI0+7p16yiOpx9j1Cojmnk6UmPC6a+O4wJmQ0lJ9xLmyzpb7NqWLoIaxSl8iiVKGmLql27atCnedhB9FIWMg9xlgZPje6ZyRyi67BEJH3vqESUPEQvR9ddfj82bNyM1NRXXX399p223bdvWbcOI5IXnGQ7XNMOgUWJItr7TtBxpOiXqfC1wMl9MM2aMMbz8ZTW0SjmuHpPXYXFtSCINagyxNIWQhoiFyGg0ilOnRmOCftmIpMTm9mJ3WQNkHIffXD6007ZpWiXUXitcjIdPbUS08fIuL486f07scGECHegiqLE1upqGZslCxELUdjhGQ7P+TeALrNcouvTbpHBOKOGDDxysTIu0qK8l9Fp0KnkUQhTZ0KzFKZQWitj3RPQYMfmIHA6hmF5g+r6qqgpvvvkmRo8ejSuvvDKuBhLJR8TrvgBwDhPUSjmaeR3MLh5pUeZE67KEUCgCPSKPPWQSfb1agVt/VIRUjZJEKEmIKaBx/vz5ePHFFwEAJpMJU6ZMwdNPP4358+dj/fr1cTWQSD4Cs00RiYOjCWqFDE6FMWS6kK7osoRQKJS6TpPocxyHHIMGGiUtrE4WYs7QeMkllwAA3njjDeTl5aGqqgovvvgi/va3v8XVQCL5EHtE2gjEwdGMVI0SaZk5kTmaw1wrqlpkMpkgRkBYPxGRXMQ0NLPb7TAYhFiNDz/8ENdffz1kMhl+9KMfoaqqKq4GEslHoJeSGlGPqBkFaVoUDB0K5ESfqzqqa7VFlSL4iML4iSobbDjV0IKCNC1G5lE8nNTEnDz/rbfeQk1NDbZv3y76herq6ijIsR9gEX1EkQkRAEAbW2EEuYyDViWPyB8VRBdBjectTnxdY+464yOREGLqEa1YsQK33HIL7rvvPlxxxRWYNm0aAKF3NHHixLgaSCQfc8bkw+zwICdM5Q4RxkQhYpo0ON0+aJSyrlfQt+GqC/L8p4qy6kYXM2cB/xZN4ScHMQnRz372M8yYMQPnzp3D+PHjxf1XXHEFfvrTn8bNOCI5yTaoO9QxC4nHDnjd4AH875eNcPqacdfMIdCqoncSRyNeACIIahQ++laKrk4KYl56nJeXh7y8vKB9U6ZM6bZBRB/C3xuSaVKh8KgAnxdmhycmIYqaLoIaAz0iq9MLxlj0QkfElZiEyGazYc2aNdixYwfq6urA88EJzeNdCZZIHuqtLlQ32ZBj0KAwQ9d5Y3uj8KhNh1GhRItLEKI8Y2Q5q86aHPjo+/PIN2pw5QV5Xb+gLV0NzdQKyDgOPl6ocUZZG6UlJiH65S9/iV27duG2225Dfn4+/Zr0I04327G7rAHDcvVdC5GtQXhMyUaqS4kzJkdUsURmhwdNNjdSYskZ1EVyNJmMQ4paDqvTC6uThEhqYhKi999/H++++y6mT58eb3uIJCeqSOdAj0iXCaOstdxz9NeKRYjCV3wNkKpVikJESEtMQpSeno6MDMoF3B+JShxs/oq/KVkwcrEIkSfya7Un0CPyeQCvS8hl3Y6rx+RBJZdBrehWeT8iDsT0Dvzxj3/EihUrYLd3LwZj/fr1GDdunFgscdq0aXj//fe7dU6iZ2lx+cWhq+GS19VaZjolW6z0EUuPKOpgRkCo3iH3v66TTI0apZxcC0lATD2ip59+GuXl5cjNzUVxcTGUyuAPyqFDhyI6z8CBA7FmzRoMGzYMjDG88MILmD9/Pg4fPowLLrggFtOIHibioVnAP6RKAZRapOm8KMlKQZpOGfEsVbd6RIAQ1GhvEoZnOurBJzMxvcPXXXddXC4+b968oOePP/441q9fj3379pEQJSG8f4YJiGDtl73VUQ0AOpUC100cENX1LLGsvG+LKsUvRKF7RGaHBwermgAAl4/Mje0aRFyISYhWrlwZbzvg8/nw+uuvw2aziZHa7XG5XHC5XOJzi8USdzuI8LS4vWBMWHaR0lUskPW88KjPjulaHh+PNJ3SP6MVY4+oiyl8nmf4usYMlUKGWSNyaIgmITEHNJpMJrzxxhsoLy/HAw88gIyMDBw6dAi5ubkYMCDyX76jR49i2rRpcDqd0Ov1Yl6jUKxevRqrVq2K1WSim6SoFLh1ahHsbl/XX1rrWeHRUBC02+nxgTF0GdSolMtw69RB3TG31WHtsoY8HBA4t5eHy8tTWhAJiclZ/c0332D48OF44okn8NRTT8FkMgEQclUvX748qnONGDECR44cwf79+/HrX/8aCxcuxPfffx+y7fLly2E2m8WtpqYmFvOJGJHLOOSkalCcldJ5Q94HtPhnzAytgYifnajH+p3l+Mo/HOpxuoglUshlSFEL4kP5q6UlJiFatmwZFi1ahBMnTgRVdp07dy52794d1blUKhWGDh2KSZMmYfXq1Rg/fjzWrVsXsq1arRZn2AIbkYTY6gHeK0yZt1l1H6iBFkuCtJjoYmgGtF38SkIkJTEJ0YEDB/CrX/2qw/4BAwagtra2WwbxPB/kByKShxPnrThY1YR6axfvj+WM8GjIFzIl+gnUQYtEiD4/0YAX9lbi2zPmmO2NKKiRVuEnBTH5iNRqdUhHcVlZGbKzI3dOLl++HHPmzEFRURGsVitefvll7Ny5E9u3b4/FLKKHOVZrRXldCy4fKet89X2zPzleWlHQ7oAQmeyeLqfwG20uNNnc8EVa3TUUXSx8BVr9RDQ0k5aYekTXXnstHnvsMXg8wpvHcRyqq6vx0EMP4YYbboj4PHV1dbj99tsxYsQIXHHFFThw4AC2b9+OH//4x7GYRfQwEcX1MAaYqoW/04OdzYEyPgHncGcEhCGSarJhCfSIPA4hiX4IAjbZ3b7Yr0N0m5gDGn/2s58hOzsbDocDM2fORG1tLaZNm4bHH3884vM8//zzsVyekAirM4IYopY64YsvVwpDszYo5TLo1QpxFX64WSrGmDh8C0Rkx4RSB3AygPGAx9aatbENo/INGJlnoGUeEhOTEBmNRnz00UfYs2cPvv76a7S0tODCCy/E7Nmz420fkSR4fDwc/l5Dp0suGsqEx/RiQNZRaIza1nQguamh04HY3D54fAwc141gRkBIoq/SCcnR3KGFSK2gKftkIGoh4nkemzdvxrZt21BZWQmO41BSUoK8vDxKMNWHafH3hlSKLhaJ1v8gPGaPCHl4SI4e2QZ1p2IW6A0ZNMpOy1lHhCqlVYiIpCUqIWKM4dprr8V7772H8ePHY+zYsWCM4dixY1i0aBG2bduGt956q4dMJaREXNqhVoT/sTGfEdaYyeRA5rCQTSYN6jqJvsnuBtBN/1AAlR7A+bBBjYAwQ9doc2Hm8Gyk6VTdvyYRNVEJ0ebNm7F7927s2LEDs2bNCjr2ySef4LrrrsOLL76I22+/Pa5GEtJjicRRffpL4TFnNKCMLAtjKBQyGXJSI8yL3RVdBDUCQEWjDQ1WF8YNTCMhkoiohOiVV17BH/7whw4iBACXX345fv/732PLli0kRH0FWyPgaALkSgxNy0D21CIg3EjJVA3U+YdlAy/q9LROjw9Wpzes0IzIM2BEXkd/TkxEENSYqlGgweqiRPoSEpUQffPNN3jyySfDHp8zZw5Veu0L2JuAH94FzKfFXWoAOfocIKME8JQAxkJA7v/4tNQB370l/F0wATCEX8luc3nx3O5T4DjgnllDoZD38GxVBLFEYlCjg4IapSIqIWpqakJubvgPWW5uLpqbm7ttFCEhtkbgyEuA2y74elKyAZ9bEKeWOmGr3i+IkC5TeE1LnRA/pM8GBnfsLbdFp5JDrZTB5eHRZHcjxxA8hGOMgTEhp3RciChlLJUWkpqohMjn80GhCP8SuVwOr5d+VXotvA849m9BhAy5wJgbAI0RAPBVWQ20thoMUdRDY6kShjqBVB8AkD0cGH51l74hjuOQpVfjTLMDjS0dhcji8OKFLyqRpVfj51MKuz8LG5iy78RZTevNpCfqWbNFixZBrQ49tqc1Yr2cMwcFcVFqgLE3AWphWMMYw/4zLri9WVh48WRodEohMb7DJLwuJQvQpkV8mSy9ShSi9jTaXPDxDL54hYKIQtQiRFfLOg4FU9vUOCOkISohWrhwYZdtyFHdS/F5gep9wt+DZ4kiBABODw+3f0lGqkYhLGRNyRK2GMhMEX7IGm0df7iabG5/mzjNXqn0rdHV7hZA0zFjg5iXyMeD51n8hoVExEQlRJs2beopOwipOX9UGG5pUoG8sUGHAkMWvVoRF+dypl4QmYaQPSJhX0a8hEgmE0TVaRGGZyGESKeS49eXDYFaIaOAXImgBTaEQO23wuOASR2WZgQinQNO3e6SpRd6RBaHB05P8GLTOn+KkUCbuKD2i48rdGphjuOomofExOeTRfRuHM3CVD3HAbkdixYEVsLHVNYnBBqlHBOLOgYPur08GlsEIYq0LHVEROCwJqSFhIgA6o4Jj2mDQi4MNccjJUc7LhuR09EMqxOMCT4bfSxlpsMRGI45wxdb+KHWguO1VpRkpWDcwLT4XZuICBIiAmg8KTzmjAx5OOAjSo2jEIVCKZdhRJ4B2ngnse9iaAYAZrsHp+pt0CjlGDcwvpcnuoaEqL/jcQAWf8WNjMEhm8wZkw+Lw9N1LbMoMdndqGiwYdzANMhlHHJTNZg7Nr/rF0ZLBEIUEFmawpcGEqL+TnOlEBWdkiUGL7ZHo5THvdQOYwyvHqiB3e1Dll6NwgxdXM8fRERBjZQyVkpo1qy/03RKeAzTG+opOI7DoExh+cWJOivOW5yoszh75mIBIXLbhOjxEAR6RC0uLxjrRp5sIiZIiPo7Jn9tuPTikIcbW1z49Ie67lXTCMPofGHI9P1ZC947eg5b9lfjcHUPrFVUpQghCYyF7RXpVQrIOA6+NmW1icRBQtSfcbUIU/cAkBq6Om+d1YUjNSYcOxf/8t6FGVpkG9Tw+BhMdg8UMg5DcvRdvzBaOK7L4ZlMxok+MPITJR4Sov5MoP5YSlbYxarN/myJPZEwjOM4XD0mDxkpKqiVMvz4gty4xSp1IAKHtUGjgEoh6xBkSfQ8kjqrV69ejW3btuGHH36AVqvFxRdfjCeeeAIjRoTOd0zEmUC+IWP4+WqTXXDepnenmkYnZOnVWHhxcc/nO4/AYf3TiQOgkHEUYS0BkvaIdu3ahdLSUuzbtw8fffQRPB4PrrzySthslOg8IQR6RJ0IUaBHlB6vtV9h6PEvfwRBjUo5rTWTCkl7RB988EHQ882bNyMnJwcHDx7EpZdeKpFV/QTeB1j95cHD+IcYY216RL08l3MEQzNCOpIqjshsFmZmMjIyQh53uVxBOY9Clb0mIsTeJIiRQgVoQ1fWaHF54fbykHFcXJd3SEIgRsppCtukyebGZyfqIeM4zBtfkBi7CABJ5KzmeR733nsvpk+fjjFjxoRss3r1ahiNRnErLCxMsJV9CFud8JiSLcwqhSDQGzJqFd2vLyY1mjTh0WESpvFDwAE4VW9DdZOdYokSTNIIUWlpKb799lts3bo1bJvly5fDbDaLW01NTQIt7GO0BISo4+LTAIUZOvz6siG4dkLooVuvItAj8nkAjz1kEzFBmpeH08MnyjICSTI0u+eee/DOO+9g9+7dGDgwvONUrVaHTVNLRImtXnjUZ3farCeWd0iCXCHMnLmsgNPcmlS/DQq5DClqOWwuH6xOD7SqPnDfvQRJe0SMMdxzzz1488038cknn6CkpERKc/oXEfSI+hyBXlEg13YIUimRviRIKkSlpaV46aWX8PLLL8NgMKC2tha1tbVwOBxSmtX38Tha42lSQveIeJ7hzcOnsausXsxX3esJJPh3hl+u0lrRg6KrE4mkQrR+/XqYzWZcdtllyM/PF7dXX31VSrP6PoHekMYYNqLa7PCgssGOo6dNUPR2R3WAgMO6k5mzQDpcWoWfWCT1EdHMhESI/qHww7J6f8rWTL2671S1iGBoZtAooVLIQJ/MxJIUzmoiwQR6RJ0IUZ1FEKLseCaxl5oIhmbjBhgxfqCRIqwTTNJM3xMJxNa1o/qsSfDTxTWJvdSIQY1modhiCGS01kwSSIj6Gzzf5dDM6+NR609SNjBdmyjLeh6VwZ+XiKelHkkGCVF/w2kSqrrKFa3O23bUWpzw8QwpannvX9rRFpksoqUeH3x7Dv/aVyVWnSV6HhKi/kZLm6UdIerAA4DT44NOJcfAdF3fG6YE1tXZm8I2aWhxo8HqEssoET0POav7GxH4h4bmGDA4Sw+3r4/ED7VFlwE0lgOO8EJk0ChQb3XRFH4CISHqRzDGwEUwYwYITluNrA8ucdD6MzvYw+fGptJCiYeGZv0Es8ODVw/UwGHy5yBKyYbJ7sYZU3AUe53FCZ7vw1E0uoAQNYZtEljmQUOzxEFC1A/w+Hj85+uzqGu2oPqMUEzRq83Cu0fP4Y2vTmP/qUbwPENDiwtbD9Rg456KvlvJQpcpPDrNYUsLBdLiBrJTEj0PDc36AYeqmtFgdSEbZhRn6gC1AbxCiwydCnUWF/aWN+L7cxY4PD74eIZsgzq+teeTCZUekCuFdCCOZqFwQDsC2SjNDk/P59ImAFCPqM9jc3lxoFJwzM7I90GtkAP6HKgUMlw9Jg9XXZAHlUIGk90Dl4dHtkGNK0fnSWx1D8JxbYZnoR3WqVphmUeqVglXX1nwm+T00Z89IsDXNSZ4fAx5Rg0Gqf1FCfwr7jmOw+iCVAzOTkF1kx0KmVB9tddnY+wKXSZgPR925kwu43D3ZUOoJ5RASIj6MF4fj6P+Cq2TB6WDOxM6olqjlGN4riHR5kmHtmuHNYlQYqGhWR+mvN4Gu9sHg0aBIVkprUs7+lMytFAEHNadBDUSiYV6RH2Yogwdfjw6F3IZB5nbDHjdwlorXegqKf0GUYgahET6IXo/lQ02fHayAZkpKswdm59gA/sfJER9GK1KjjED/Gur6suER12mIEb9GV0mwMkAj1PIVBkovtgGjgMarC7KmZUgaGjWX7BFFlHdL5ArWnuFgeFqO9L8U/gmu6dvB3gmCSREfZTD1c04WNXcGpjYH5Pld0YgV3fg/9KOVI0CChkHH89oqUcCICHqgzDGcLCqGbvL6lHnzysUSXrYfkXg/2ALLUQcxyHNH2HdRBHWPQ4JUR+kzuqC1emFSiFDUYauNYoYICEKEOgZhhmaAUB6ijA8a7K5wrYh4oOkQrR7927MmzcPBQUF4DgOb731lpTm9BnK61sAAIMydVDIZcKXjTFApQtZWLBfEigsaWsMu+YsM0XI193QQj2inkZSIbLZbBg/fjyeffZZKc3oc5TXCxHUg7P0wo6W88Kjvg8v3YgWdSqgUAtpY8MENmYb1MjUq/ruurskQtL/8Jw5czBnzhwpTehzmO0eNFhdkHEcBmf7ez9iDqLOy0v3KzhOGKaaagBrbcgh69AcPYbm6CUwrv/Rq3xELpcLFoslaCOCKW8QhmUFaZrWmvVijyhXIquSFIM/UNF6Tlo7iN4lRKtXr4bRaBS3wsJCqU1KOpxuHxQyDkMCv+SMtekRkRAFkVogPFrOdtqMMQZvX0ybm0T0KiFavnw5zGazuNXU1EhtUtJx8dAs/GrmEFxQ4I8WdjQLs2ZyRetiT0LA4PeZ2eqFyiYh+KK8Ef/YWY5D1abE2dUP6VVeOLVaDbW6D1Ue7SFUija/L4FhWSdVO/otmjRAqQU8DiGeKNBDaoNCzsHt5dHYQlP4PQl9MvsQDneIaWjyD4WH49oMz0L7iTL9sUT1JEQ9iqQ9opaWFpw8eVJ8XlFRgSNHjiAjIwNFRUUSWtb78Ph4bNxTAaNWiZ9OHICUwJRzhFU7+i2GfKG8kOUMgEkdDuekCiW3m2xuuL18cG+TiBuS/le/+uorTJw4ERMnTgQALFu2DBMnTsSKFSukNKtXUtNkh9vLi8URAfgd1YGhGQlRSIwDhEfz6ZCH9WoFDBoFGKNeUU8iaY/osssuozQLceJUIIgxO6U1u6DLArhahJQXBgpmDEnqQOH/4zQDDhOgTevQJCdVA6uzBectTgxI0ybcxP4A9TP7AIwxnPLHDw3JbhOAF/B76LOFyhVERxSqVpE2VYdskmsQJkjEBcRE3CEh6gPUWpywuXxQKWQYmK5rPWA5IzwaOs4GEW1I8/sjzaHDQQrStBicnYJ8I/WGeopeNX1PhObEeaE3NDirXQWOQMRwiGlpog1pRUD1vrA9osIMHQozdCGPEfGBekS9HMYYTtYJQhS0Lor3kRBFirFQ8BM5TK3pUoiEQkLUB7hkWBZG5RswKLNNio9AtLBC3ZosngiNQgUYBwp/N54K26zF5UWdlfxEPQEJUS+H4zgMyzXg6jH5wTEugeno1IKQVSqIdmQOFR4bT4Y8fOK8FRt2n8Inx0JndCS6BwlRX6W5UnhMGySpGb2GgBCZqoSyS+3INQqBjectLripDHXcISHqxTS0uLD3ZAOabe2+ODzf6nhNJyGKCF0GoE0XfGsBEW9DqkaJVK0SPGM4a3Ik3r4+DglRL+a7sxbsr2jC3vJ2GQZtdYDXJfg+KCtjZHBca6+o/oeQTYr8M2cVjbZEWdVvICHqpfh4hh/OCYnhRuW3q1vfXCU8GotoxX005IwSHhuOhxyeBTJenqq30YqAOEOf0l5KVaNQ116nkgfPlgGtDtf04oTb1atJLRCGZz4v0FDW4XBRhg4KGQeLw0MJ9eMMCVEv5evTJgDAyPzU4CBGt711xixraOIN681wHJB7gfD3+W87HFbKZSjKFIZnp/yVUoj4QELUC2myuVHZYAfHARMGprU7WC5UptBnC7/uRHTkjREEqalCKDXUjgmFabh6TB4uHET/23hCQtQLOVwtRP8OztbDqGu3mLX+uPCYNTzBVvURtOmtTuszBzscHpSZglH5qVDK6asTT+i/2QtJUSugUshwYVFa8AFXi5DkCwByRifcrj7DAH+CtNpvAHf4GTJyWMcPEqJeyI8GZ2LpjJLglfaA4NdgvOB0TcmSxri+QHoxYMgVig5UfdHhMGMMR2pM2LSnEma7J/H29UFIiHopYs2yADwPnD0s/J0/PvEG9SU4Dhh8mfD32UMdFsJyHIdT9S0wOzzYXxG6SiwRHSREvQQfz/DON2dxutkeukHd98LqcZWOhmXxIL1E6BnxPuD4B0La3Tb8aLCwkPi7sxacoUjrbkNC1EvYebwOJ8634N1vzsHTvtifzwNU7Bb+HniREFFNdA+OA4ZfJdSDa64ETn8VdLggTSvWjtv+bW3oCipExJAQ9QIOVjXhm9NmcBxw5QV5HWdsKnYLOZc1qcCAydIY2RfRZQCDZwl/l+8AGoJX5l86PBtGrRJmhwdvHTkDuzt0kUaia5JCiJ599lkUFxdDo9Fg6tSp+PLLL6U2KSnw8Qyfn2jA7rIGAMD0oVkoyWoXRX3+O6DG//8adiX1huLNgElA/jhhaPbdNuH/7UejlOPaCQXQKOWoNTvxypc1VJo6RiQXoldffRXLli3DypUrcejQIYwfPx5XXXUV6ur6d96XmiY7th6oxoHKJgDAxUMycVFxm5LRPA9U7weOvS08L7wIyBomgaV9HI4Dhl8NZI8Q/EXf/0fYHCYAQJZejQUXFSIjRYXR+alQ+HurVqcHn52oxzenTahosKHO4oTV6YGPpyn/UHBM4mCIqVOn4qKLLsLf//53AADP8ygsLMRvfvMb/P73v+/0tRaLBUajEWazGampqYkwN+7wPIPT64PN5YNGKYNBIwQoHq5uxs7j9VArZZg9KhfDc/SA1ynM4DRXCTEudkGkkD9e+LLQAteeg+eBil1AzX6hd8TJgIwSIGMIYMiFV50GKLRQKITZzMoGG948fCbkqbQqOWYMzcKYAUYAgmgdr7VCp1IgRS2HViVHikoBrVIOmax3J7WL9DsqafJ8t9uNgwcPYvny5eI+mUyG2bNn44svOsZvuFwuuFytRe4sFkvE1zpjcuCTY0KxwYDytpXgKSUZGJUv/KNqzU68/+25oDZt1XryoHSML0wTbPhyC46W17Q5GS++gAEoTNdgUEYKAAa72ytERTOhAQPg43nxtcWZOhgydAAYxvEM6XYbCtK0UJ3ggONeYTFmWxRqYMjlghBRFsaeRSYDhswSItYrPxOWgDSWiwGkCkB4DxRqQKZElpvHNQ4XnF4Gp5fB7WXw+Hjw/vc6w50KnBGSrbltbphrTDC3uyTHCevbhmbrke9PzGZyePDDOav4dp8o/BkY1xrKMWlQBkb7neh1Vie2f1sb9pYmFKZj7EBBDJtsbrx7NHTZbQAYU5CKiUXCshaL0wOlTAatSh62fbRIKkQNDQ3w+XzIzQ2uy56bm4sffuiYE2b16tVYtWpVTNfyePlOV0w7Pa2zHl6eh6mTQDV3Wz+AvREye0PYtjKHFnAI1+U8PshcweIZeCsVMg6cmwFuJu4vTpUBvEvUNgDC9HzqAOHXOHeM8MEnEodxADD+ZsDWIKzQN9UI+cFdVuEHxeME4IQewPB22VkYAK9P+PyoOAtgExbOql1eDFI74fHxcPt4eHw8vD4m/vjJ7E5AIQgRHG4wm1U81tDiDhIiR9vPsY91+plv61z38jwarOEr2drbzAryPAOTxXcg1avKCS1fvhzLli0Tn1ssFhQWFkb02txUDW64UEiQHqrzkNZmzVa2QRj3Bwi05yD8ode0/tu04+Zj2GBHm0ac/0EYJmmVCkAlBzgOSp5h5FgvAE5ox3FQyzmolQr/CnqujXHB5wMnA9QGKpSYLKRkCVsgASbvAzwOYWM+IcK97QbhHVX6t7boAYh5Evw9Jp4JP45Orw86pQJQ+j9PXh7FdrfYAR+YWhT0gW679jAjRYWfTRoY9hZSNa1tjVql+P0I2Vbb+plPUSsgi3MPXFIhysrKglwux/nz54P2nz9/Hnl5HTMLqtVqqNWx9QC0KrmYwqEr1Ao5CiIsLazMGISsjK7bAcI/m9Zs91FkckCtF7Z4nA6Azr+1RQ0gP8JzaJTyiOuxqRWRfz96YsGvpN5NlUqFSZMmYceOHeI+nuexY8cOTJs2TULLCIJIJJIPzZYtW4aFCxdi8uTJmDJlCtauXQubzYbFixdLbRpBEAlCciFasGAB6uvrsWLFCtTW1mLChAn44IMPOjiwCYLou0geR9Qd+kIcEUH0ZSL9jlIEHEEQkkNCRBCE5JAQEQQhOZI7q7tDwL0VzVIPgiASR+C72ZUrulcLkdVqBYCIo6sJgpAGq9UKo9EY9nivnjXjeR5nz56FwWAA18sXfQaWq9TU1PSZGUC6p+Snp++HMQar1YqCggLIOskO0at7RDKZDAMHhl8f0xtJTU3tEx/wttA9JT89eT+d9YQCkLOaIAjJISEiCEJySIiSBLVajZUrV8acXSAZoXtKfpLlfnq1s5ogiL4B9YgIgpAcEiKCICSHhIggCMkhISIIQnJIiBJINBVtN2zYgEsuuQTp6elIT0/H7Nmzk7ICbqxVerdu3QqO43Ddddf1rIFREu39mEwmlJaWIj8/H2q1GsOHD8d7772XIGsjI9p7Wrt2LUaMGAGtVovCwkLcd999cDqdPWskIxLC1q1bmUqlYhs3bmTfffcdu+OOO1haWho7f/58yPa33HILe/bZZ9nhw4fZsWPH2KJFi5jRaGSnT59OsOXhifaeAlRUVLABAwawSy65hM2fPz8xxkZAtPfjcrnY5MmT2dy5c9nnn3/OKioq2M6dO9mRI0cSbHl4or2nLVu2MLVazbZs2cIqKirY9u3bWX5+Prvvvvt61E4SogQxZcoUVlpaKj73+XysoKCArV69OqLXe71eZjAY2AsvvNBTJkZNLPfk9XrZxRdfzP75z3+yhQsXJpUQRXs/69evZ4MHD2ZutztRJkZNtPdUWlrKLr/88qB9y5YtY9OnT+9RO2lolgACFW1nz54t7uusom0o7HY7PB4PMjIirF3Uw8R6T4899hhycnKwdOnSRJgZMbHcz3/+8x9MmzYNpaWlyM3NxZgxY/DnP/8ZPp8vZPtEE8s9XXzxxTh48KA4fDt16hTee+89zJ07t0dt7dWLXnsL0Va0DcVDDz2EgoKCoA+VlMRyT59//jmef/55HDlyJAEWRkcs93Pq1Cl88sknuPXWW/Hee+/h5MmTuPvuu+HxeLBy5cpEmN0psdzTLbfcgoaGBsyYMQOMMXi9Xtx11134wx/+0KO2Uo+oF7BmzRps3boVb775JjQajdTmxITVasVtt92GDRs2ICsrS2pz4gLP88jJycFzzz2HSZMmYcGCBXj44YfxP//zP1KbFjM7d+7En//8Z/zjH//AoUOHsG3bNrz77rv44x//2KPXpR5RAoi2om1bnnrqKaxZswYff/wxxo0b15NmRkW091ReXo7KykrMmzdP3MfzQilmhUKB48ePY8iQIT1rdCfE8h7l5+dDqVRCLm+tPT9q1CjU1tbC7XZDpVL1qM1dEcs9PfLII7jtttvwy1/+EgAwduxY2Gw23HnnnXj44Yc7zSnUHahHlABirWj75JNP4o9//CM++OADTJ48ORGmRky09zRy5EgcPXoUR44cEbdrr70Ws2bNwpEjRyTPshnLezR9+nScPHlSFFQAKCsrQ35+vuQiBMR2T3a7vYPYBISW9eSy1B51hRMiW7duZWq1mm3evJl9//337M4772RpaWmstraWMcbYbbfdxn7/+9+L7desWcNUKhV744032Llz58TNarVKdQsdiPae2pNss2bR3k91dTUzGAzsnnvuYcePH2fvvPMOy8nJYX/605+kuoUORHtPK1euZAaDgb3yyivs1KlT7MMPP2RDhgxhN910U4/aSUKUQP77v/+bFRUVMZVKxaZMmcL27dsnHps5cyZbuHCh+HzQoEEMQIdt5cqViTe8E6K5p/YkmxAxFv397N27l02dOpWp1Wo2ePBg9vjjjzOv15tgqzsnmnvyeDzs0UcfZUOGDGEajYYVFhayu+++mzU3N/eojZQGhCAIySEfEUEQkkNCRBCE5JAQEQQhOSREBEFIDgkRQRCSQ0JEEITkkBARBCE5JEREr2Lz5s1IS0sTnz/66KOYMGGC+HzRokVJl/WR6BoSIiIkixYtAsdxuOuuuzocKy0tBcdxWLRoUVD7eAtAcXEx1q5dG7RvwYIFKCsrC/uadevWYfPmzeLzyy67DPfee29c7SLiDwkREZbCwkJs3boVDodD3Od0OvHyyy+jqKhIEpu0Wi1ycnLCHjcajUE9JqJ3QEJEhOXCCy9EYWEhtm3bJu7btm0bioqKMHHixG6dO1RP5brrrhN7WZdddhmqqqpw3333geM4cBwHoOPQrD1te2aLFi3Crl27sG7dOvEcFRUVGDp0KJ566qmg1x05cgQcx+HkyZPdui8iNkiIiE5ZsmQJNm3aJD7fuHEjFi9e3OPX3bZtGwYOHIjHHnsM586dw7lz56I+x7p16zBt2jTccccd4jmKioo63BMAbNq0CZdeeimGDh0ar1sgooCEiOiUX/ziF/j8889RVVWFqqoq7NmzB7/4xS96/LoZGRmQy+UwGAzIy8vrMoFcKIxGI1QqFXQ6nXgOuVyORYsW4fjx42JeZo/Hg5dffhlLliyJ920QEUIZGolOyc7OxjXXXIPNmzeDMYZrrrmm16d6LSgowDXXXIONGzdiypQpePvtt+FyuXDjjTdKbVq/hXpERJcsWbIEmzdvxgsvvBC3XoNMJuuQ8c/j8cTl3JHwy1/+UnTEb9q0CQsWLIBOp0vY9YlgSIiILrn66qvhdrvh8Xhw1VVXxeWc2dnZQX4fn8+Hb7/9NqiNSqXqdmmecOeYO3cuUlJSsH79enzwwQc0LJMYGpoRXSKXy3Hs2DHx73CYzeYOpYIyMzND5qO+/PLLsWzZMrz77rsYMmQInnnmGZhMpqA2xcXF2L17N26++Wao1eqYhoTFxcXYv38/KisrodfrkZGRAZlMJvqKli9fjmHDhnWaO5zoeahHREREamoqUlNTO22zc+dOTJw4MWhbtWpVyLZLlizBwoULcfvtt2PmzJkYPHgwZs2aFdTmscceQ2VlJYYMGYLs7OyY7L7//vshl8sxevRoZGdno7q6Wjy2dOlSuN3uhMwCEp1DqWKJfstnn32GK664AjU1NR2KEBKJhYSI6He4XC7U19dj4cKFyMvLw5YtW6Q2qd9DQzOi3/HKK69g0KBBMJlMePLJJ6U2hwD1iAiCSAKoR0QQhOSQEBEEITkkRARBSA4JEUEQkkNCRBCE5JAQEQQhOSREBEFIDgkRQRCSQ0JEEITk/H/RiuOi+qCvigAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:23:58.394609Z", + "iopub.status.busy": "2024-02-29T06:23:58.394307Z", + "iopub.status.idle": "2024-02-29T06:23:58.595978Z", + "shell.execute_reply": "2024-02-29T06:23:58.595110Z" + }, + "papermill": { + "duration": 0.223266, + "end_time": "2024-02-29T06:23:58.597895", + "exception": false, + "start_time": "2024-02-29T06:23:58.374629", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAm60lEQVR4nO3de1xUdf4/8BfXgXFARC4igaOSgJWQsCKubbormqVZ7q/YNEFy3Q1l12ItozbItY3toqKtya4bknbRaq31UeZlUVtvpEIYKpcFESgBRUUEdBiYz/cPf5x2nAEGBWbOzOv5ePDA8zmXeZ85w8sznznnM3ZCCAEiIpmxN3cBRES3guFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyZKjuQvobzqdDufOnYObmxvs7OzMXQ4R3UQIgatXr2Lo0KGwt+/8/MrmwuvcuXMICAgwdxlE1I3q6mrccccdnc63ufByc3MDcOOJcXd3N3M1fUer1WL37t2YOnUqnJyczF0O9QJbOaaNjY0ICAiQ/lY7Y3Ph1fFW0d3d3erDS6lUwt3d3apf6LbE1o5pd9067LAnIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEcmSzV0qYa1aWlpQXFwsTTdd0+BwYTkGeR2HylWht2xISAiUSmV/l0jUqxheVqK4uBgREREG7W8YWTYvLw9jx47t+6KI+hDDy0qEhIQgLy9Pmi6paUDyJ4VY9dg9CPbzMFiWSO4YXlZCqVTqnU3ZV16E4sA1hN4dhvBhg81YGVHfYIc9EckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZMns4bVu3Tqo1Wq4uLggKioKR48e7XL5hoYGLF68GH5+flAoFBg1ahR27NjRT9USkaUw6xj2W7duRXJyMjIzMxEVFYWMjAxMmzYNJSUl8PHxMVi+tbUVMTEx8PHxwaeffgp/f39UVlbCw8Oj/4snIrMya3itWrUKCxcuREJCAgAgMzMTX375JbKysvDCCy8YLJ+VlYVLly7h8OHDcHJyAgCo1er+LJmILITZwqu1tRV5eXlISUmR2uzt7TFlyhQcOXLE6Drbt29HdHQ0Fi9ejH/961/w9vbGnDlzsGzZMjg4OBhdR6PRQKPRSNONjY0AAK1WC61W24t7ZFna2tqk39a8n7ak4zha+/E0df/MFl719fVob2+Hr6+vXruvr6/eNz//rzNnzmDv3r2YO3cuduzYgbKyMixatAharRZpaWlG10lPT8fy5csN2nfv3m3V3xpd3QQAjsjNzcUPJ81dDfWmPXv2mLuEPtXS0mLScrL63kadTgcfHx/8/e9/h4ODAyIiIvDDDz/gzTff7DS8UlJSkJycLE03NjYiICAAU6dOhbu7e3+V3u9OVF0CCo9j/PjxCAv0NHc51Au0Wi327NmDmJgYqdvEGnW8O+qO2cLLy8sLDg4OqKur02uvq6vDkCFDjK7j5+cHJycnvbeIoaGhqK2tRWtrK5ydnQ3WUSgUUCgUBu1OTk5W/QJwdHSUflvzftoia3/tmrpvZrtUwtnZGREREcjJyZHadDodcnJyEB0dbXSdn/70pygrK4NOp5PaSktL4efnZzS4iMh6mfU6r+TkZGzYsAHvvfceioqKkJiYiObmZunTx7i4OL0O/cTERFy6dAlLlixBaWkpvvzyS7z22mtYvHixuXaBiMzErH1esbGxuHDhAlJTU1FbW4vw8HDs3LlT6sSvqqqCvf2P+RoQEIBdu3bh2WefxZgxY+Dv748lS5Zg2bJl5toFIjITs3fYJyUlISkpyei8/fv3G7RFR0cjNze3j6siIktn9tuDiIhuBcOLiGTJ7G8bichQS0uLwcXaTdc0OFxYjkFex6Fy/fHyn5CQEKu+4LozDC8iC1RcXIyIiAij8964aTovLw9jx47t+6IsDMOLyAKFhIQgLy9Pr62kpgHJnxRi1WP3INjPQ29ZW8TwIrJASqXS4GzKvvIiFAeuIfTuMIQPG2ymyiwHO+yJSJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsOZq7ALp1FfXNaNa0GZ1XfqFZ+u3o2PlhHqBwxHCvAX1SH1FfYnjJVEV9Mya/tb/b5f7waWG3y+xbOokBRrLD8JKpjjOujNhwBPmoDOdf0+CL/UcwY1I0BrgqjG6j7HwTntla0OnZG5ElY3jJXJCPCnf7DzRo12q1qPUGxg4bBCcnJzNURtS3etxhf+bMmV4vYt26dVCr1XBxcUFUVBSOHj3a6bLZ2dmws7PT+3Fxcen1mojIsvU4vIKCgjB58mS8//77uH79+m0XsHXrViQnJyMtLQ35+fkICwvDtGnTcP78+U7XcXd3R01NjfRTWVl523UQkbz0OLzy8/MxZswYJCcnY8iQIfjtb3/b5ZlSd1atWoWFCxciISEBo0ePRmZmJpRKJbKysjpdx87ODkOGDJF+fH19b/nxiUieetznFR4ejjVr1mDlypXYvn07srOzMXHiRIwaNQpPPfUU5s2bB29vb5O21drairy8PKSkpEht9vb2mDJlCo4cOdLpek1NTRg2bBh0Oh3Gjh2L1157DXfddZfRZTUaDTQajTTd2NgI4EafkFarNalOS9TW1ib9NrYfHW1d7WN32yDLYivHy9R9u+UOe0dHR8yePRsPPfQQ3nnnHaSkpGDp0qV48cUX8fjjj+P111+Hn59fl9uor69He3u7wZmTr68viouLja4THByMrKwsjBkzBleuXMFbb72FCRMm4NSpU7jjjjsMlk9PT8fy5csN2nfv3g2lUtmDPbYs1U0A4IiDBw+i0vDDRsmePXtuextkGTqOV25uLn44ae5q+k5LS4tJy91yeB0/fhxZWVnYsmULBgwYgKVLl2LBggX4/vvvsXz5csyaNeu23k52Jjo6GtHR0dL0hAkTEBoair/97W9YsWKFwfIpKSlITk6WphsbGxEQEICpU6fC3d291+vrL6fONeKtwlxMnDgRdw013A+tVos9e/YgJiam008bu9sGWZYTVZeAwuMYP348wgI9zV1On+l4d9SdHofXqlWrsHHjRpSUlODBBx/Epk2b8OCDD8Le/kb32fDhw5GdnQ21Wt3ttry8vODg4IC6ujq99rq6OgwZMsSkepycnHDvvfeirKzM6HyFQgGFwvA6JycnJ1lfQtBx1byjo2OX+9HVfpq6DbIMtnK8TN23HnfYr1+/HnPmzEFlZSU+//xzzJgxQwquDj4+Pnj33Xe73ZazszMiIiKQk5Mjtel0OuTk5OidXXWlvb0dhYWF3b5FJSLr0uMzrz179iAwMNAgsIQQqK6uRmBgIJydnREfH2/S9pKTkxEfH4/IyEiMGzcOGRkZaG5uRkJCAgAgLi4O/v7+SE9PBwD86U9/wvjx4xEUFISGhga8+eabqKysxK9//eue7goRyViPw2vkyJGoqamBj4+PXvulS5cwfPhwtLe392h7sbGxuHDhAlJTU1FbW4vw8HDs3LlT6sSvqqrSC8rLly9j4cKFqK2txaBBgxAREYHDhw9j9OjRPd0VIpKxHoeXEMJoe1NT0y1f6Z6UlISkpCSj8/bv3683vXr1aqxevfqWHoeIrIfJ4dXxiZ2dnR1SU1P1LjNob2/HN998g/Dw8F4vkIjIGJPD69tvvwVw48yrsLAQzs7O0jxnZ2eEhYVh6dKlvV8hkY3oanw2wLQx2mxpfDaTw2vfvn0AgISEBKxZs0bW10gRWRpTx2cDuh+jzVbGZ+txn9fGjRv7og4im9bd+GxA92O02dr4bCaF1+zZs5GdnQ13d3fMnj27y2W3bdvWK4UR2aLOxmcDOEbbzUwKr4EDB8LOzk76N5mfpv067F1+QEVjCexdDP+nbmtrw7m2cyi6VNRp/0hFYxPsXX6Apv06AB5XkheTwut/3yrybaNlONdciQHD38aL3dw++s7Od7qcP2A4cK45HBHgsEIkLxwGWqaGDhiG5orfYU1sOEYa6SNpa2vDoYOH8NOJP+30zKv8fBOWbC3A0MnD+rpcol5nUnjde++90tvG7uTn599WQWQahYMLdNf9Mdw9GKMHGx/DvsKxAqGeoZ32j+iuX4Hu+gUoHDiMNsmPSeH1yCOP9HEZREQ9Y1J4paWl9XUdREQ90uMhcYiILIFJZ16enp4oLS2Fl5cXBg0a1GX/16VLl3qtOCKizpgUXqtXr4abm5v0b1M774mI+opJ4fW/AwvOnz+/r2ohIjJZj/u8HBwcjH4h7MWLF+Hg4NArRRERdafH4dXZYIQajUZvmBwior5k8hX2a9euBXBjMMJ//OMfUKl+vKq7vb0d//nPfxASEtL7FRIRGWFyeHUMvSyEQGZmpt5bRGdnZ6jVamRmZvZ+hUQ2oLsb7YHub7a3tRvtTQ6viooKAMDkyZOxbds2DBo0qM+KIrI1pt5oD3R9s70t3Wjf4xuzO0ZUJaLe092N9kD3N9vb2o32PQ6vp556qsv5WVlZt1wMka3q7kZ7oPub7W3tRvseh9fly5f1prVaLU6ePImGhgb8/Oc/77XCiIi60uPw+uyzzwzadDodEhMTMXLkyF4pioioO71yY7a9vT2Sk5P5ZbBE1G96bVSJ8vJytLXZxreWEJH59fhtY8c3Z3cQQqCmpgZffvml3j2QRER9qcfh1fHN2R3s7e3h7e2NlStXdvtJJBFRb+F1XkQkSxxJlYhkieFFRLLE8CIiWWJ4EZEs9Vp4ff/99/jNb37TW5sjIupSr4XXxYsX8e677/bW5oiIusS3jUQkSwwvIpIlhhcRyZLJV9jPnj27y/kNDQ23WwsRkclMDq+BA7se0H/gwIGIi4u77YKIiExhcnht3LixL+sgIuoRi+jzWrduHdRqNVxcXBAVFYWjR034ChUAW7ZsgZ2dHR555JG+LZCILI7JZ16mDnfT0y/g2Lp1K5KTk5GZmYmoqChkZGRg2rRpKCkpgY+PT6frnT17FkuXLsV9993Xo8cjIutg8plXdnY29u3bh4aGBly+fLnTn55atWoVFi5ciISEBIwePRqZmZlQKpVdhmB7ezvmzp2L5cuXY8SIET1+TCKSP5PPvBITE/HRRx+hoqICCQkJePLJJ+Hp6XlbD97a2oq8vDykpKRIbfb29pgyZQqOHDnS6Xp/+tOf4OPjgwULFuDAgQNdPoZGo4FGo5GmGxsbAdz41iOtVntb9ZtTx5DbbW1tRvejo62rfexuG9R/TDkW3R1TazmeptZucnitW7cOq1atwrZt25CVlYWUlBQ89NBDWLBgAaZOnQo7O7seF1lfX4/29nb4+up/u6+vry+Ki4uNrnPw4EG8++67KCgoMOkx0tPTsXz5coP23bt3Q6lU9rhmS1HdBACOOHjwICqNf0cpAGDPnj23vQ3qez05Fp0dU2s5ni0tLSYt16ORVBUKBZ544gk88cQTqKysRHZ2NhYtWoS2tjacOnUKKlXfPmNXr17FvHnzsGHDBnh5eZm0TkpKit64+42NjQgICMDUqVPh7u7eV6X2uVPnGvFWYS4mTpyIu4Ya7odWq8WePXsQExNj9AtKTdkG9R9TjkV3x9RajmfHu6Pu9HgY6A729vaws7ODEALt7e23tA0vLy84ODigrq5Or72urg5DhgwxWL68vBxnz57FzJkzpTadTgcAcHR0RElJicF3RyoUCigUCoNtOTk5dfpHLQcdX/fu6OjY5X50tZ+mboP6Xk+ORWfH1FqOp6m19+hSCY1Gg48++ggxMTEYNWoUCgsL8de//hVVVVW3dNbl7OyMiIgI5OTkSG06nQ45OTmIjo42WD4kJASFhYUoKCiQfh5++GFMnjwZBQUFCAgI6HENRCRPJp95LVq0CFu2bEFAQACeeuopfPTRRya/detKcnIy4uPjERkZiXHjxiEjIwPNzc1ISEgAAMTFxcHf3x/p6elwcXHB3Xffrbe+h4cHABi0E5F1Mzm8MjMzERgYiBEjRuDrr7/G119/bXS5bdu29aiA2NhYXLhwAampqaitrUV4eDh27twpdeJXVVXB3t4irqUl6jPXtDe6Xk7+cKXTZZqvaXD8AjCk8jIGuBp2hZSdb+qz+iyRyeEVFxd3S58omiIpKQlJSUlG5+3fv7/LdbOzs3u/IKJ+Vv7/g+eFbYXdLOmIzWXHulxigOKWu7JlxeS9ZEgQ9Z2pd934gGqkjwquTg5GlympuYI/fFqIlf/vHgT7GR8oYYDCEcO9BvRZnZbENiKayMJ5DnDGr8YFdrlMx0WoI70H4G7/rkd5sQUML5nqro+ku/4RwPb6SMi6MLxkyrQ+ku77RwDb6SMh68JXrUx110diSv8IYFt9JGRdGF4y1V0fCftHyNrxAioikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEsmQR4bVu3Tqo1Wq4uLggKioKR48e7XTZbdu2ITIyEh4eHhgwYADCw8OxefPmfqyWiCyB2cNr69atSE5ORlpaGvLz8xEWFoZp06bh/PnzRpf39PTESy+9hCNHjuC7775DQkICEhISsGvXrn6unIjMyezhtWrVKixcuBAJCQkYPXo0MjMzoVQqkZWVZXT5SZMm4dFHH0VoaChGjhyJJUuWYMyYMTh48GA/V05E5uRozgdvbW1FXl4eUlJSpDZ7e3tMmTIFR44c6XZ9IQT27t2LkpISvP7660aX0Wg00Gg00nRjYyMAQKvVQqvV3uYeWK62tjbptzXvpy2xlWNq6r6ZNbzq6+vR3t4OX19fvXZfX18UFxd3ut6VK1fg7+8PjUYDBwcHvPPOO4iJiTG6bHp6OpYvX27Qvnv3biiVytvbAQtW3QQAjsjNzcUPJ81dDfUGWzmmLS0tJi1n1vC6VW5ubigoKEBTUxNycnKQnJyMESNGYNKkSQbLpqSkIDk5WZpubGxEQEAApk6dCnd3936sun+dqLoEFB7H+PHjERboae5yqBfYyjHteHfUHbOGl5eXFxwcHFBXV6fXXldXhyFDhnS6nr29PYKCggAA4eHhKCoqQnp6utHwUigUUCgUBu1OTk5wcnK6vR2wYI6OjtJva95PW2Irx9TUfTNrh72zszMiIiKQk5Mjtel0OuTk5CA6Otrk7eh0Or1+LSKyfmZ/25icnIz4+HhERkZi3LhxyMjIQHNzMxISEgAAcXFx8Pf3R3p6OoAbfViRkZEYOXIkNBoNduzYgc2bN2P9+vXm3A0i6mdmD6/Y2FhcuHABqampqK2tRXh4OHbu3Cl14ldVVcHe/scTxObmZixatAjff/89XF1dERISgvfffx+xsbHm2gUiMgOzhxcAJCUlISkpyei8/fv3602/+uqrePXVV/uhKiKyZGa/SJWI6FZYxJkXEelraWkxuNaxpKYBmtoyFJ10he6ih9QeEhJi1dcsdobhRWSBiouLERERYXTenPf0p/Py8jB27Nh+qMqyMLyILFBISAjy8vL02pquafDlviN4aHI0VK4KvWVtEcOLyAIplUqDsymtVovL9ecRPS7Sqi9SNRU77IlIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssQhcazEzSNvdjbqJmC7I2+SdWF4WYnORt68edRNwHZH3iTrwvCyEjePvNnZqJsdyxLJHcPLStw88iZH3SRrxw57IpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJZs7jovIQQAoLGx0cyV9C2tVouWlhY0NjbyOi8rYSvHtONvs+NvtTM2F15Xr14FAAQEBJi5EiLqytWrVzFw4MBO59uJ7uLNyuh0Opw7dw5ubm6ws7Mzdzl9prGxEQEBAaiuroa7u7u5y6FeYCvHVAiBq1evYujQobC377xny+bOvOzt7XHHHXeYu4x+4+7ubtUvdFtkC8e0qzOuDuywJyJZYngRkSwxvKyUQqFAWloaFApF9wuTLPCY6rO5Dnsisg488yIiWWJ4EZEsMbyISJYYXt2YP38+HnnkkV7d5qRJk/DMM890uYxarUZGRkavPi6RNbGp8DIlNEheXnnlFYSHh5u7jE5Z2mvO0uq5HTYVXkRy1Nraau4SLJLNhNf8+fPx9ddfY82aNbCzs4OdnR3Ky8uxYMECDB8+HK6urggODsaaNWuMrr98+XJ4e3vD3d0dTz/9tMkvqObmZsTFxUGlUsHPzw8rV640WOb8+fOYOXMmXF1dMXz4cHzwwQcGy9jZ2WH9+vWYPn06XF1dMWLECHz66afS/LNnz8LOzg4ff/wx7rvvPri6uuInP/kJSktLcezYMURGRkKlUmH69Om4cOGCic8akJWVhbvuugsKhQJ+fn5ISkqS5lVVVWHWrFlQqVRwd3fH448/jrq6Oml+x1nR5s2boVarMXDgQPzqV7+Sbo4Hbtxr+sYbbyAoKAgKhQKBgYH485//LM1ftmwZRo0aBaVSiREjRuDll1+GVqsFAGRnZ2P58uU4ceKEdEyzs7NN3re+dquvuY6uij//+c8YOnQogoODAQCHDx9GeHg4XFxcEBkZic8//xx2dnYoKCiQ1j158iSmT58OlUoFX19fzJs3D/X19Z3Wc/bs2f56OnqfsBENDQ0iOjpaLFy4UNTU1Iiamhpx/fp1kZqaKo4dOybOnDkj3n//faFUKsXWrVul9eLj44VKpRKxsbHi5MmT4osvvhDe3t7ixRdfNOlxExMTRWBgoPj3v/8tvvvuOzFjxgzh5uYmlixZIi0zffp0ERYWJo4cOSKOHz8uJkyYIFxdXcXq1aulZQCIwYMHiw0bNoiSkhLxxz/+UTg4OIjTp08LIYSoqKgQAERISIjYuXOnOH36tBg/fryIiIgQkyZNEgcPHhT5+fkiKChIPP300ybV/s477wgXFxeRkZEhSkpKxNGjR6Wa2tvbRXh4uJg4caI4fvy4yM3NFREREeL++++X1k9LSxMqlUrMnj1bFBYWiv/85z9iyJAhes/d888/LwYNGiSys7NFWVmZOHDggNiwYYM0f8WKFeLQoUOioqJCbN++Xfj6+orXX39dCCFES0uL+MMf/iDuuusu6Zi2tLSYtG/94XZfc/PmzRMnT54UJ0+eFFeuXBGenp7iySefFKdOnRI7duwQo0aNEgDEt99+K4QQ4vLly8Lb21ukpKSIoqIikZ+fL2JiYsTkyZM7raetrc0cT02vsJnwEkKI+++/Xy80jFm8eLH45S9/KU3Hx8cLT09P0dzcLLWtX79eqFQq0d7e3uW2rl69KpydncXHH38stV28eFG4urpKdZSUlAgA4ujRo9IyRUVFAoBBeN0cOlFRUSIxMVEI8WN4/eMf/5Dmf/TRRwKAyMnJkdrS09NFcHBwl3V3GDp0qHjppZeMztu9e7dwcHAQVVVVUtupU6f09iUtLU0olUrR2NgoLfPcc8+JqKgoIYQQjY2NQqFQ6IVVd958800REREhTaelpYmwsDCT1+9vt/qa8/X1FRqNRmpbv369GDx4sLh27ZrUtmHDBr3wWrFihZg6daretqurqwUAUVJSYnI9cmFzo0rcbN26dcjKykJVVRWuXbuG1tZWgw7gsLAwKJVKaTo6OhpNTU2orq7GsGHDOt12eXk5WltbERUVJbV5enpKbwMAoKioCI6OjoiIiJDaQkJC4OHhYbC96Ohog+n/fcsAAGPGjJH+7evrCwC455579NrOnz/fac0dzp8/j3PnzuEXv/iF0flFRUUICAjQGxdt9OjR8PDwQFFREX7yk58AuPGpqZubm7SMn5+f9PhFRUXQaDSdPgYAbN26FWvXrkV5eTmamprQ1tYm+xEVTHnN3XPPPXB2dpamS0pKMGbMGLi4uEht48aN01vnxIkT2LdvH1QqlcFjlpeXY9SoUb27I2ZmM31exmzZsgVLly7FggULsHv3bhQUFCAhIUHWHaT/O8Jmx3hlN7fpdLput+Pq6trr9dz8+N09xpEjRzB37lw8+OCD+OKLL/Dtt9/ipZdekvXxMfU1N2DAgB5vu6mpCTNnzkRBQYHez3//+1/87Gc/661dsBg2FV7Ozs5ob2+Xpg8dOoQJEyZg0aJFuPfeexEUFITy8nKD9U6cOIFr165J07m5uVCpVN2Oxjpy5Eg4OTnhm2++kdouX76M0tJSaTokJARtbW3Iy8uT2kpKStDQ0GCwvdzcXIPp0NDQLmu4VW5ublCr1cjJyTE6PzQ0FNXV1aiurpbaTp8+jYaGBowePdqkx7jzzjvh6ura6WMcPnwYw4YNw0svvYTIyEjceeedqKys1Fvm5mNqaW71NXez4OBgFBYWQqPRSG3Hjh3TW2bs2LE4deoU1Go1goKC9H46wtDSn6+esKnwUqvV+Oabb3D27FnU19fjzjvvxPHjx7Fr1y6Ulpbi5ZdfNnhBADc+ql6wYAFOnz6NHTt2IC0tDUlJSV2O8ggAKpUKCxYswHPPPYe9e/fi5MmTmD9/vt56wcHBeOCBB/Db3/4W33zzDfLy8vDrX//a6FnJJ598gqysLJSWliItLQ1Hjx7V+/Svt73yyitYuXIl1q5di//+97/Iz8/H22+/DQCYMmUK7rnnHsydOxf5+fk4evQo4uLicP/99yMyMtKk7bu4uGDZsmV4/vnnsWnTJpSXlyM3NxfvvvsugBvhVlVVhS1btqC8vBxr167FZ599prcNtVqNiooKFBQUoL6+Xu+P2xLc6mvuZnPmzIFOp8NvfvMbFBUVYdeuXXjrrbcA/HiGvXjxYly6dAlPPPEEjh07hvLycuzatQsJCQlSYN1cjyln4RbL3J1u/amkpESMHz9euLq6CgCiuLhYzJ8/XwwcOFB4eHiIxMRE8cILL+h1AMfHx4tZs2aJ1NRUMXjwYKFSqcTChQvF9evXTXrMq1eviieffFIolUrh6+sr3njjDYNO05qaGvHQQw8JhUIhAgMDxaZNm8SwYcMMOuzXrVsnYmJihEKhEGq1Wu8Tqo4O+47OWyGE2LdvnwAgLl++LLVt3LhRDBw40OTnLDMzUwQHBwsnJyfh5+cnfve730nzKisrxcMPPywGDBgg3NzcxGOPPSZqa2ul+cY601evXi2GDRsmTbe3t4tXX31VDBs2TDg5OYnAwEDx2muvSfOfe+456XmPjY0Vq1ev1qv/+vXr4pe//KXw8PAQAMTGjRtN3rf+cDuvuZsdOnRIjBkzRjg7O4uIiAjx4YcfStvsUFpaKh599FHh4eEhXF1dRUhIiHjmmWeETqczWk9FRUUfPwN9h0PiyISdnR0+++yzXr9VieTrgw8+QEJCAq5cudJrfZRyYvOfNhLJxaZNmzBixAj4+/vjxIkTWLZsGR5//HGbDC6A4XVbqqqquuycPn36NAIDA/uxop4x9pF6h6+++gr33XdfP1ZD3amtrUVqaipqa2vh5+eHxx57TO9uBFvDt423oa2trcvbK9RqNRwdLff/h7Kysk7n+fv72+z/6CQPDC8ikiWbulSCiKwHw4uIZInhRUSyxPAiIllieFGfmj9/vjTwnZOTE3x9fRETE4OsrKwe3ZqSnZ1tdKSNvtYX32FAvYPhRX3ugQceQE1NDc6ePYuvvvoKkydPxpIlSzBjxgy0tbWZuzySK3Pem0TWr7P79HJycgQAaSDClStXirvvvlsolUpxxx13iMTERHH16lUhxI/3aP7vT1pamhBCiE2bNomIiAihUqmEr6+veOKJJ0RdXZ30OJcuXRJz5swRXl5ewsXFRQQFBYmsrCxpflVVlXjsscfEwIEDxaBBg8TDDz8s3e+XlpZm8Lj79u3rk+eJeo5nXmQWP//5zxEWFoZt27YBAOzt7bF27VqcOnUK7733Hvbu3Yvnn38eADBhwgRkZGTA3d0dNTU1qKmpwdKlSwEAWq0WK1aswIkTJ/D555/j7NmzmD9/vvQ4L7/8Mk6fPo2vvvoKRUVFWL9+Pby8vKR1p02bBjc3Nxw4cACHDh2CSqXCAw88gNbWVixduhSPP/64dOZYU1ODCRMm9O8TRZ0zd3qSdevszEsIIWJjY0VoaKjReZ988okYPHiwNG3qaBjHjh0TAKSztpkzZ4qEhASjy27evFkEBwdLIy4IIYRGoxGurq5i165d3dZP5sUzLzIbIYQ0FtW///1v/OIXv4C/vz/c3Nwwb948XLx4ES0tLV1uIy8vDzNnzkRgYCDc3Nxw//33A7hx3ykAJCYmYsuWLQgPD8fzzz+Pw4cPS+ueOHECZWVlcHNzg0qlgkqlgqenJ65fv27SAIFkXgwvMpuioiIMHz4cZ8+exYwZMzBmzBj885//RF5eHtatWweg6+8sbG5uxrRp0+Du7o4PPvgAx44dkwYr7Fhv+vTpqKysxLPPPiuNyd/xlrOpqQkREREGwyaXlpZizpw5fbz3dLss965hsmp79+5FYWEhnn32WeTl5UGn02HlypXSKLMff/yx3vLGhi8uLi7GxYsX8Ze//EUakvv48eMGj+Xt7Y34+HjEx8fjvvvuw3PPPYe33noLY8eOxdatW+Hj49Ppl3pY07DJ1oZnXtTnNBoNamtr8cMPPyA/Px+vvfYaZs2ahRkzZiAuLg5BQUHQarV4++23cebMGWzevBmZmZl621Cr1WhqakJOTg7q6+vR0tKCwMBAODs7S+tt374dK1as0FsvNTUV//rXv1BWVoZTp07hiy++kMb9nzt3Lry8vDBr1iwcOHAAFRUV2L9/P37/+9/j+++/lx73u+++Q0lJCerr66UvvCULYO5ON7Ju8fHx0mUGjo6OwtvbW0yZMkVkZWXpfe/lqlWrhJ+fn3B1dRXTpk0TmzZtMhjC+umnnxaDBw/Wu1Tiww8/FGq1WigUChEdHS22b99u8F2GoaGhwtXVVXh6eopZs2aJM2fOSNusqakRcXFxwsvLSygUCjFixAixcOFCceXKFSGEEOfPnxcxMTFCpVLxUgkLwyFxiEiW+LaRiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSz9H+d6+BM/+nePAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:23:58.638235Z", + "iopub.status.busy": "2024-02-29T06:23:58.637938Z", + "iopub.status.idle": "2024-02-29T06:23:58.842257Z", + "shell.execute_reply": "2024-02-29T06:23:58.841314Z" + }, + "papermill": { + "duration": 0.227444, + "end_time": "2024-02-29T06:23:58.844413", + "exception": false, + "start_time": "2024-02-29T06:23:58.616969", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEpCAYAAAA3eS91AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3aUlEQVR4nO3deVhUZf8/8PeZHZhhlVU2F8QNlxQV09TEr5pb2e9yyafQrDSXckvNXPMpLa3HSrOyp2xxr9S+9ahf9XFfcUFBEAMxUEEUBIZtmOX+/TFyZNicgRkOM3xe1zWXzJkzcz4OM2/u+z7n3IdjjDEQQoiAREIXQAghFESEEMFREBFCBEdBRAgRHAURIURwFESEEMFREBFCBEdBRAgRHAURIURwFESEEMEJGkTLly8Hx3Emt7Zt2wpZEiFEABKhC+jQoQMOHTrE35dIzC/JYDDg7t27UKlU4DjOFuURQuqBMQa1Wo2AgACIRDW3ewQPIolEAj8/vzo99+7duwgKCrJyRYQQa8vIyEBgYGCNjwseRH/99RcCAgKgUCgQFRWFVatWITg4uNp1NRoNNBoNf7984oCMjAy4uro2SL2EEPMVFBQgKCgIKpWq1vU4IacB2bdvHwoLCxEeHo7MzEysWLECd+7cQUJCQrWFL1++HCtWrKiyPD8/n4KIkEaooKAAbm5uT/yOChpEleXl5SEkJASffvopJk+eXOXxyi2i8rSlICKkcTI3iATvmlXk7u6ONm3aICUlpdrH5XI55HJ5A1dFCLG1RnUcUWFhIVJTU+Hv7y90KYSQBiRoEM2bNw/Hjh3DrVu3cPr0abzwwgsQi8UYP368kGURQhqYoF2z27dvY/z48cjJyYG3tzf69OmDs2fPwtvbW8iyCCENTNAg2r59u5CbJ4Q0Eo1qjIgQYpk/rt7F5M2xyC/WCl1KvVAQEWLHZmy9jMPXs/GvQzeELqVeKIgIcQC5RWVCl1AvFESEOIBGc1RyHVEQEeIAGtEJEnVCQUSIA7DvGKIgIsQx2HkSURAR4gCYnScRBREhDsDOh4goiAhxBBREhBDBUdeMECI4ahERQgRn5zlEQUSII6AWESGkEbDvJKIgIsQBUIuIECI4O88hCiJCiPAoiAhxAHT2PSFEcPYdQxREhDgEO28QURAR4gjsPIcoiAhxBDRGRAgh9URBRIgDsPMGEQURIY6ApgEhhAiOWkSEEMFREBFCBEddM0KI4KhFRAgRnJ3nEAURIQ7BzpOIgogQB0BjRFayevVqcByHWbNmCV0KIXaHxoisIDY2Fl9//TU6deokdCmE2CU7zyHhg6iwsBATJkzApk2b4OHhIXQ5hNglOum1nqZPn45hw4YhOjr6ietqNBoUFBSY3Agh9t8ikgi58e3bt+PSpUuIjY01a/1Vq1ZhxYoVNq6KEPtj5w0i4VpEGRkZePvtt7FlyxYoFAqznvPuu+8iPz+fv2VkZNi4SkLsg53nkHAtoosXLyI7OxtPPfUUv0yv1+P48eNYv349NBoNxGKxyXPkcjnkcnlDl0oIsTHBgmjgwIGIj483WTZp0iS0bdsWCxYsqBJChJBa2HnfTLAgUqlU6Nixo8kyFxcXeHl5VVlOCKmdfcdQI9hrRgghgu41q+zo0aNCl0CIXbLznhm1iAhxBHSuGSFEcNQiIoQIjoKIECI4O88hCiJCHAGd9EoIIfVEQUSIA7DzBhEFESGOgHbfE0IERy0iQojg7DyHKIgIcQS014wQIjj7jiEKIkIcg50nUb3Ovi8sLITBYDBZ5urqWq+CCCGWs/McsrxFlJaWhmHDhsHFxQVubm7w8PCAh4cH3N3d6XJAhAjE3seILG4R/eMf/wBjDN999x18fX3BcZwt6iKEWMC+Y6gOQXTlyhVcvHgR4eHhtqiHENIEWdw1i4yMpMv4ENLI2HnPzPIW0bfffoupU6fizp076NixI6RSqcnjdP16QhqevZ/iYXEQ3b9/H6mpqZg0aRK/jOM4MMbAcRz0er1VCySEPFmTaxG9+uqr6Nq1K7Zt20aD1YQ0Ek0uiP7++2/8/vvvaN26tS3qIYQ0QRYPVj/77LO4cuWKLWohhNRRkzuOaMSIEZg9ezbi4+MRERFRZbB65MiRViuOEGIe+46hOgTR1KlTAQDvv/9+lcdosJoQYdh5g8jyIKp8bhkhRHj2vvveojEirVYLiUSChIQEW9VDCKkDe28RWRREUqkUwcHB1P0ipJGx8xyyfK/Ze++9h0WLFiE3N9cW9RBC6sDeW0QWjxGtX78eKSkpCAgIQEhICFxcXEwev3TpktWKI4SYy76TyOIgev75521QBiGkPppci2jZsmW2qIMQUg92nkN1n7P64sWL+Pnnn/Hzzz/j8uXLdXqNjRs3olOnTnB1dYWrqyuioqKwb9++upZESJPV5I6szs7Oxrhx43D06FG4u7sDAPLy8jBgwABs374d3t7eZr9WYGAgVq9ejbCwMDDG8MMPP2DUqFG4fPkyOnToYGlphDRZ9h1DdWgRzZw5E2q1GteuXUNubi5yc3ORkJCAgoICvPXWWxa91ogRI/Dcc88hLCwMbdq0wQcffAClUomzZ89aWhYhTZqdN4gsbxHt378fhw4dQrt27fhl7du3x4YNG/A///M/dS5Er9dj165dKCoqQlRUVLXraDQaaDQa/n5BQUGdt0eII7H3rpnFLSKDwVDlRFfAeLBjXU7/iI+Ph1KphFwux9SpU7F79260b9++2nVXrVoFNzc3/hYUFGTx9ghxRPYdQ3WcBuTtt9/G3bt3+WV37tzB7NmzMXDgQIsLCA8PR1xcHM6dO4c333wTMTExSExMrHbdd999F/n5+fyN5s4mxDHU6YDGkSNHIjQ0lG+RZGRkoGPHjvj5558tLkAmk/GTrHXr1g2xsbH47LPP8PXXX1dZVy6XQy6XW7yNmuj1emi1Wqu9HiENrblKDABwkYlRWlra4NuXSqUQi8X1fh2LgygoKAiXLl3CoUOHcP36dQBAu3btEB0dXe9iAGPXr+I4kC0wxpCVlYW8vDybbocQW1s+wAcAIOKMFz8Vgru7O/z8/Oo1bXSdLjnNcRwGDRqEQYMG1XnDgLGrNXToUAQHB0OtVmPr1q04evQoDhw4UK/XfZLyEPLx8YGzszPNu03sVpmTcYeNiOPQwlfVoNtmjKG4uBjZ2dkAAH9//zq/Vp2C6PDhwzh8+DCys7OrDFB/9913Zr9OdnY2XnnlFWRmZsLNzQ2dOnXCgQMH6h1wtdHr9XwIeXl52Ww7hDQETmLsjnEcB4VC0eDbd3JyAmD8Lvv4+NS5m2ZxEK1YsQLvv/8+unfvDn9//3q1Jv7973/X+bl1VT4m5Ozs3ODbJsQRlX+XtFptwwXRV199hc2bN+Pll1+u0wYbC+qOEWId1vguWbz7vqysDL179673hgkhpJzFQfTaa69h69attqiF2MDEiROtPnVL//79MWvWrFrXCQ0Nxbp166y6XSIMc37f9WVx16y0tBTffPMNDh06hE6dOlU5yvrTTz+1WnHEVP/+/dGlSxf6gjuQ5cuXY8+ePYiLi6vX69j7kdUWB9HVq1fRpUsXAKgyiT6NuxDSOGm12mpPzWosLO6aHTlypMbbf//7X1vUSGDsYh07dgyfffYZOI4Dx3FITU3F5MmT0aJFCzg5OSE8PByfffZZtc9fsWIFvL294erqiqlTp6KsrMys7RYVFeGVV16BUqmEv78/PvnkkyrrZGdnY8SIEXByckKLFi2wZcuWKutwHIeNGzdi6NChcHJyQsuWLfHLL7/wj9+6dQscx2Hnzp3o27cvnJycEBkZiRs3biA2Nhbdu3eHUqnE0KFDcf/+fTPfNePhJB06dIBcLoe/vz9mzJjBP5aeno5Ro0ZBqVTC1dUVY8aMwb179/jHly9fji5duuCnn35CaGgo3NzcMG7cOKjVan4dg8GAjz/+GK1bt4ZcLkdwcDA++OAD/vEFCxagTZs2cHZ2RsuWLbFkyRJ+z+3mzZuxYsUKXLlyhf+dbt68+Yn/p+vXr6NPnz5QKBR44dleOHviKDoFumPPnj0m7+WOHTvQr18/KBQKbNmyBTk5ORg/fjyaN28OZ2dnREREYNu2bSavbc7v2yaYHcvPz2cAWH5+vtnPKSkpYYmJiaykpIQxxpjBYGBFGq0gN4PBYHbdeXl5LCoqir3++ussMzOTZWZmstLSUrZ06VIWGxvLbt68yX7++Wfm7OzMduzYwT8vJiaGKZVKNnbsWJaQkMD++OMP5u3tzRYtWmTWdt98800WHBzMDh06xK5evcqGDx/OVCoVe/vtt/l1hg4dyjp37szOnDnDLly4wHr37s2cnJzYv/71L34dAMzLy4tt2rSJJScns8WLFzOxWMwSExMZY4ylpaUxAKxt27Zs//79LDExkfXq1Yt169aN9e/fn508eZJdunSJtW7dmk2dOtWs2r/88kumUCjYunXrWHJyMjt//jxfk16vZ126dGF9+vRhFy5cYGfPnmXdunVj/fr145+/bNkyplQq2ejRo1l8fDw7fvw48/PzM3nv5s+fzzw8PNjmzZtZSkoKO3HiBNu0aRP/+MqVK9mpU6dYWloa+/3335mvry/76KOPGGOMFRcXs7lz57IOHTrwv9Pi4uJa/086nY6Fh4ezQYMGsbi4OPb9r/9hHbt0YwDY7t27Td7L0NBQ9uuvv7KbN2+yu3fvstu3b7M1a9awy5cvs9TUVPb5558zsVjMzp07Z9Hvu7LK36mKzP2ONvkgKtJoWciCPwS5FWm0Fv1/+/XrV+sHgjHGpk+fzl588UX+fkxMDPP09GRFRUX8so0bNzKlUsn0en2tr6VWq5lMJmM7d+7kl+Xk5DAnJye+juTkZAaAnT9/nl8nKSmJAagSRJUDpGfPnuzNN99kjD3+8nz77bf849u2bWMA2OHDh/llq1atYuHh4bXWXS4gIIC999571T72f//3f0wsFrP09HR+2bVr10z+L8uWLWPOzs6soKCAX+edd95hPXv2ZIwxVlBQwORyuUnwPMmaNWtYt27d+PvLli1jnTt3Nvv5+/btYxKJhGVmZjLGGLuS8ZB9vXV3tUG0bt26J77esGHD2Ny5cxlj5v2+q2ONIKrTkdWk8diwYQO+++47pKeno6SkBGVlZfwYXrnOnTubHMAZFRWFwsJCZGRkICQkpMbXTk1NRVlZGXr27Mkv8/T0RHh4OH8/KSkJEokE3bp145e1bduWn72zosrzTEVFRVUZpO3UqRP/s6+vLwAgIiLCZFn5KQW1yc7Oxt27d2ucESIpKQlBQUEmU8m0b98e7u7uSEpKQmRkJADj3j+V6vGpE/7+/vz2k5KSoNFoap11YseOHfj888+RmpqKwsJC6HQ6uLq6PrH+miQnJyMoKAh+fn78so5dnqp23e7du5vc1+v1+PDDD7Fz507cuXMHZWVl0Gg0/GfDnN+3rTT5IHKSipH4/mDBtl0f27dvx7x58/DJJ58gKioKKpUKa9aswblz56xUYcOrOKBavvOj8jJz5r0qP/XAmvVU3v6TtnHmzBlMmDABK1aswODBg+Hm5obt27c32LhL5Ut9rVmzBp999hnWrVuHiIgIuLi4YNasWWaPF9qSxYPVx48fh06nq7Jcp9Ph+PHjVimqIXEcB2eZRJCbpXsZZTKZyVV2T506hd69e2PatGno2rUrWrdujdTU1CrPu3LlCkpKSvj7Z8+ehVKpfOLEcq1atYJUKjUJtocPH+LGjRv8/bZt20Kn0+HixYv8suTk5GpnNqg8BfDZs2dNZvq0JpVKhdDQUBw+fLjax9u1a4eMjAyTOa0SExORl5dX48R8lYWFhcHJyanGbZw+fRohISF477330L17d4SFheHvv/82Wafy7/RJwsPDkZGRYTKofu2KeRevOHXqFEaNGoV//OMf6Ny5M1q2bGnyuzTn920rFreIBgwYgMzMTPj4+Jgsz8/Px4ABA+hy1DYUGhqKc+fO4datW1AqlQgLC8OPP/6IAwcOoEWLFvjpp58QGxuLFi1amDyvrKwMkydPxuLFi3Hr1i0sW7YMM2bMgEhU+98hpVKJyZMn45133oGXlxd8fHzw3nvvmTwvPDwcQ4YMwZQpU7Bx40ZIJBLMmjWr2tbCrl270L17d/Tp0wdbtmzB+fPnbXq+4fLlyzF16lT4+Phg6NChUKvVOHXqFGbOnIno6GhERERgwoQJWLduHXQ6HaZNm4Z+/fpV6dLURKFQYMGCBZg/fz5kMhmefvpp3L9/H9euXcPkyZMRFhaG9PR0bN++HZGRkfjzzz+xe/duk9cIDQ1FWloa4uLiEBgYCJVKVeucW4MGDUKrVq0QExODjz/+GJdT72L9GuNeuif9YQsLC8Mvv/yC06dPw8PDA59++inu3bvHB685v29bsXgLjLFq/8M5OTlVmoLEuubNmwexWIz27dvD29sbgwcPxujRozF27Fj07NkTOTk5mDZtWpXnDRw4EGFhYXjmmWcwduxYjBw5EsuXLzdrm2vWrEHfvn0xYsQIREdHo0+fPibjQQDw/fffIyAgAP369cPo0aPxxhtvVPlDBRgPIdi+fTs6deqEH3/8Edu2bTO79VEXMTExWLduHb788kt06NABw4cPx19//QXA+KXdu3cvPDw88MwzzyA6OhotW7bEjh07LNrGkiVLMHfuXCxduhTt2rXD2LFj+TGkkSNHYvbs2ZgxYwa6dOmC06dPY8mSJSbPf/HFFzFkyBAMGDAA3t7eVXanVyYWi7Fnzx4UFhYiMjISK+a/jddmzgGAJ559v3jxYjz11FMYPHgw+vfvDz8/vypH3Zvz+7YFjjHzZt0ePXo0AGDv3r0YMmSISWrr9XpcvXoV4eHh2L9/v20qrUZBQQHc3NyQn59v9gBgaWkp0tLS0KJFC0GmTWiqOI7D7t276UrBVnb1dh4ux57FxNFDkZKSglatWjV4DbV9p8z9jprdNXNzcwNgbBGpVCqTprdMJkOvXr3w+uuvW/p/IIRYaPfu3XzX/OyZOHy0/F10iewpSAhZi9lB9P333wMw9mnnzZtH3TAHkJ6eXmvXKDExEcHBwQ1YkWWUSmWNj+3btw99+/ZtwGqsY8uWLZgyZUq1j4WEhODatWtQq9VYsGAB0tPT4ebhhV59+mHu0n82cKXWZXbXrDGirln96HQ63Lp1q8bHQ0NDIZE03iM8UlJSanysefPmVtuF35DUarXJHrGKpFJpleO+rt7O43/uFOhuw8pq1qBds3L37t3DvHnz+KliK+cY7TWzHxKJhL+Cij2y59prolKpTA6gbCosDqKJEyciPT0dS5YsqfdUsYQQAtQhiE6ePIkTJ05UOY2AEELqyuLjiIKCguz+OtuEkMbF4iBat24dFi5cWOsgJyGEWMLirtnYsWNRXFyMVq1awdnZucpJgbm5uVYrjhDSNFgcRDRfsn0KDQ3FrFmz+EnQ6Uhn0phYHEQxMTG2qIM0sMzMTHh4eAhdBiEA6jBGBBgnUFq8eDHGjx/Pn+C3b98+XLt2zarFEdvx8/Or9SzvhsAYq3ZKGdL0WBxEx44dQ0REBM6dO4fffvsNhYWFAIxz3ixbtszqBZLH1Go1JkyYABcXF/j7++Nf//pXna85xXFclcnWf/vtNwwYMADOzs7o3Lkzzpw5Y/KckydP8hPbBwUF4a233kJRURH/+E8//YTu3btDpVLBz88PL730kslsikePHgXHcdi3bx+6desGuVyOkydP1um9II7F4iBauHAh/vnPf+LgwYOQyWT88meffbbKxFd2gTGgrEiYm4WHQcyZMwenTp3C77//joMHD+LEiRO4dOmS1d6K9957D/PmzUNcXBzatGmD8ePH8y2W1NRUDBkyBC+++CKuXr2KHTt24OTJkyZXxdBqtVi5ciWuXLmCPXv24NatW5g4cWKV7SxcuBCrV69GUlKSydSwpOmyeIwoPj6+2iu9+vj44MGDB1YpqkFpi4EPA4TZ9qK7gMy8k4fVajV++OEHbN26lZ8juXweIGuZN28ehg0bBsA4d1CHDh2QkpKCtm3bYtWqVZgwYQLf+goLC8Pnn3+Ofv36YePGjVAoFHj11Vf512rZsiU+//xzREZGorCw0OQE1ffffx+DBg2yWt3E/lncInJ3d0dmZmaV5ZcvX0bz5s2tUhSp6ubNm9BqtejRowe/zM3NzaoTm1dsnfj7+wMA37W6cuUKNm/eDKVSyd8GDx4Mg8GAtLQ0AMDFixcxYsQIBAcHQ6VSoV+/fgCMZ/lXZO4MiKTpsLhFNG7cOCxYsAC7du3iJxI/deoU5s2bh1deecUWNdqW1NnYMhFq241IdRPXl08UX1hYiClTpuCtt96q8rzg4GAUFRVh8ODBGDx4MLZs2QJvb2+kp6dj8ODBVSZnpylkSGUWB9GHH36I6dOnIygoCHq9Hu3bt4der8dLL72ExYsX26JG2+I4s7tHQmrZsiWkUiliY2P5OYLy8/Nx48YNPPPMMzbf/lNPPYXExMQaz3iPj49HTk4OVq9ezU/Kf+HCBZvXRRyDxUEkk8mwadMmLFmyBAkJCSgsLETXrl0RFhZmi/rIIyqVCjExMXjnnXfg6ekJHx8fLFu2DCKRqEFmQFiwYAF69eqFGTNm4LXXXoOLiwsSExNx8OBBrF+/HsHBwZDJZPjiiy8wdepUJCQkYOXKlTaviziGOk/PHxwcjOeeew5jxoypcwitWrUKkZGRUKlU8PHxwfPPP4/k5OS6luTwPv30U0RFRWH48OGIjo7G008/jXbt2jXIBG+dOnXCsWPHcOPGDfTt2xddu3bF0qVL+cFyb29vbN68Gbt27UL79u2xevVqrF271uZ1Ecdg1gyNc+bMwcqVK+Hi4oI5c+bUuu6nn35q9saHDBmCcePGITIyEjqdDosWLUJCQgISExPNGkdo6jM0FhUVoXnz5vjkk08wefJkocshAmhSMzRevnwZWq2W/7kmlnYRKl/xY/PmzfDx8cHFixcbZNzD3ly+fBnXr19Hjx49kJ+fj/fffx8AMGrUKIErI6R+zAqiI0eOVPuzteXn5wMwXm+7OhqNBhqNhr9fUFBgs1oaq7Vr1yI5ORkymQzdunXDiRMnkJSUhKFDh9b4nPKj3wlprBrNzOgGgwGzZs3C008/jY4dO1a7zqpVq7BixYoGrqzx6Nq1q8mlncuVlJQgLi6u4QsixErMCqLyiyua47fffqtTIdOnT0dCQkKt5x69++67JmNUBQUFT7x+e1Pg5OTkkBPJk6bDrCAqv7giYDxjevfu3XBzc+OPkL148SLy8vIsCqyKZsyYgT/++APHjx9HYGBgjevJ5XLBzxgnhFifWUFUfnFFwHg8yZgxY/DVV19BLBYDMF5CaNq0aWbvuSrHGMPMmTOxe/duHD16FC1atLDo+fVRfsQwIaR+rPFdsvgCi97e3jh58mSVc5ySk5PRu3dv5OTkmP1a06ZNw9atW7F3716T13NzczPr4nh12X1vMBjw119/QSwWw9vbGzKZjC6JROxWctbjHTbhfpY1BOqLMYaysjLcv38fer0eYWFhEIlMD0202QUWdTodrl+/XiWIrl+/bnEybty4EQDQv39/k+Xff/99tdNHWINIJEKLFi2QmZmJu3cFOseMECvJfljC/ywrEebKts7OzggODq4SQpawOIgmTZqEyZMnIzU1lT8T/Ny5c1i9ejUmTZpk0WsJdVkimUyG4OBg6HQ6ujItsWuv/XaU//nw3P4Nvn2xWAyJRFLvXoXFQbR27Vr4+fnhk08+4acD8ff3xzvvvIO5c+fWq5iGxHEcpFJplauQEGJP7qgf/yG15zMFLA4ikUiE+fPnY/78+fwBhZYOUhNCSEX1OqCRAogQYg11CqJffvkFO3fuRHp6epVJr6w5hzIhpGmweJj7888/x6RJk+Dr64vLly+jR48e8PLyws2bN2s934kQQmpicRB9+eWX+Oabb/DFF19AJpNh/vz5OHjwIN566y3+pFVCCLGExUGUnp6O3r17AzCe46RWqwEAL7/8MrZt22bd6gghTYLFQeTn54fc3FwAxlkay69llpaWJthxQYQQ+2ZxED377LP4/fffARgPbpw9ezYGDRqEsWPH4oUXXrB6gYQQx2fxXrNvvvmGP5Vj+vTp8PLywunTpzFy5EhMmTLF6gUSQhyfRUGk0+nw4Ycf4tVXX+Wn6xg3bhzGjRtnk+IIIU2DRV0ziUSCjz/+mL8eOiGEWIPFY0QDBw7EsWPHbFELIaSJsniMaOjQoVi4cCHi4+PRrVu3Kpf9GTlypNWKI4Q0DRYH0bRp0wBUf/0yjuNoWg1CiMUsDiKaYpUQYm11n1KNEEKsxOwWUUlJCQ4fPozhw4cDMF7ap+LFDsViMVauXGnXkzMRYm84DnCEExrMDqIffvgBf/75Jx9E69evR4cOHfhJ7q9fv46AgADMnj3bNpUSQqrgADhADpnfNduyZQveeOMNk2Vbt27FkSNHcOTIEaxZswY7d+60eoGEkJo5yhVozA6ilJQURERE8PcVCoXJrP09evRAYmKidasjhNTKMWLIgq5ZXl6eyZjQ/fv3TR43GAwmjxNCbM9BGkTmt4gCAwORkJBQ4+NXr16t9XLRhBDr4xykTWR2ED333HNYunQpSktLqzxWUlKCFStWYNiwYVYtjhDyBI6RQ+Z3zRYtWoSdO3ciPDwcM2bMQJs2bQAYLzW9fv166HQ6LFq0yGaFEkKqcpAcMj+IfH19cfr0abz55ptYuHAhPxsjx3EYNGgQvvzyS/j6+tqsUEJIVY4yRmTRKR4tWrTA/v37kZubi5SUFABA69at4enpaZPiCCG1c5Qxojpd18zT05O/7j0hRDiO0iKic80IsWMOkkMURITYsyZ3ZDUhpPFxjBiiICLEvjlIElEQEWLHHCSHhA2i48ePY8SIEQgICADHcdizZ4+Q5RBid2iMyAqKiorQuXNnbNiwQcgyCLFbDpJDdTuOyFqGDh2KoUOHClkCIXbNQXJI2CCylEajMZlqpKCgQMBqCCHWYleD1atWrYKbmxt/CwoKErokQgRFY0QCePfdd5Gfn8/fMjIyhC6JEEE5RgzZWddMLpdDLpcLXQYhjYaDNIjsq0VECKnMMZJI0BZRYWEhP50IAKSlpSEuLg6enp4IDg4WsDJC7IOjtIgEDaILFy5gwIAB/P05c+YAAGJiYrB582aBqiLEfjhIDgkbRP379+dneiSEWM5RWkQ0RkQIERwFESF2rC5TxTLGcDQ5G/fVjec6hBREhNixunTNTqfmYOL3sVi8J976BdURBREhdqwuQ0Q37xcCALLyq16jUCgURITYsbqc4lHeJSsu01u7nDqjICKkiblfaAyiEi0FESHECuoyRlTeIiqlICKEWEN9goi6ZoQQq6i4+97cg4PLg6hEq280BxRTEBFixyq2iMzJFMYYP0bEGKDRGWxUmWUoiAixYxV7Zua0bfJLtNDqH69Z0ki6ZxREhNixirvvzelmVT6auriRDFhTEBFixywdq64cRNQiIoRYlTlds/LxoXKNZRd+kwgixhj+98rdRvOmE2I1Fg5WV+maNZIWkV3NWV1Xp1JyMHPbZXi6yDAuMgj/6BWCAHcnocsipN5MB6stHyNqLEdXN4kWUVGZDs3dnZBbVIYvj6ai78dHMG3LRZy7mdNojqMgpC5MB6ufvH7VMSKdtUuqkybRIhrcwQ8D2/rgUFI2fjh9C2du5uA/8Vn4T3wW2vm7YmLvEIzq0hwKqVjoUgmxiMWD1YXUIhKURCzCkI5+2PZGL+yf1RfjewRDIRUhKbMAC36NR69Vh7F633XcySsRulRCzGbpAY3lLSJnmfGPbmMZI2oyQVRRWz9XrBodgbPvDsSi59oi0MMJecVafHUsFX0/+i+m/HQBZ1Kp20YaP5NTPCwYIwr2dAbQeHbfN4muWU3cnWV445lWmNynJf57PRubT6fhVEoODly7hwPX7iHcV4WY3qF4oWtzOMmo20YaH0taRFq9ATlFZQCMQXQ9S91o9iQ3yRZRZWIRh0HtfbHltV74v9nPYELPYDhJxUi+p8ai3cZu24f/SUJGbrHQpdbbdyfTcOBaltBlEBt4Unsop9AYQmIRx+81pq5ZI9XGV4UPXojA2UUDsXhYOwR7OiO/RItvjt9EvzVH8PqPF3Aq5YFddtviMvLw/h+JmPLTRaFLIVZiySke5d2yZkoZP0bUWAarm3TXrDZuTlK81rclJj3dAkeTs7H59C2c+OsBDibew8HEe2jjq8QrUaEY/VRzOMvs420sn6uYOA5LTnq9X2ico9pbJX8cRI2kRWQf3yABiUUcBrbzxcB2vkjJVuOH03/j10u3ceNeIRbvScDH+69jTPcgvBIVimAvZ6HLrZW69PExI4yxOs13XJOSMj0MjMFFTh+phmTJGFF5i8hbKecPVWksLSLqmlmgtY8KK5/viLOLBmLp8PYI8XJGQakO355MQ7+1RzB5cyzOpOYIXWaNCkq0/M96g/W6lgYDQ8flB9Bh2QFodI3jg91UcBY0ifggUsn5VnxjGSNqGn++GLPqtXldFVK82qcFJvYOxbEb9/H96Vs4fuM+Dl/PxuHr2dj0SncMau9rte1Zi1rzuEVUpjdAIrbO3yGtwcAHW1Z+KUK8XKzyuuTJLNl9XzGInGTG332Ne830OqAkFyh6ABTnAMWP/i3KeXx/xOeAXGmV/0fTCKLUw8COlwHnZoCzJ+DsZby5VLrv3Ozxz04egLj2t0ck4jCgrQ8GtPVB6v1CrD2QjH0JWfjwP0noH+4NqZW+6JYqKdPjbn4JWnmbfkgqtoi0OgbIrLM9UYWQ11mxpUWezKyuGWNAWSEMD9PQmbuJriUP4HOnGK+Lr6J9jhbY+9OjcMl5HDyleU/e+LNLKIgsUpwLaIuB/HTjzSwc4OReIaS8HoVWM9Nlj8KslcoLH78YgfNpuUh7UITt59PxclSoDf9TNRu/6SziMvLw65u90S3Eg1+eV/w4iDR6PQCp1bdtoCCyvQqtlYiyqwgUZcKTU0N++gqgzaum9ZID6DVYCQByAHHGl+kkBVAM4HJNG+KMf5D5z3mlz73c1Wr/paYRRO1GAG9dNgZSxdSv6VbyEAAz/lvyEMhJMWszKpEUJ6Tu+FumQOEBN2j/bgWpyrtSmFW6SRX1/u8l3MnHR/uvY/7gtogIdENcRh4AYEdsukkQ5T46mA0Aymw0VzG1iCz0qLVi/EzmVgiRaj6j1bRWPgAet2xP1b4pDWR4wFRw9/KHTuGB/2YYYFB44sW+nU17BOWh4+QBiBrmQN6mEURSJ8CzpfFmDr3OGEAmH4TyD0blMHt0X1sEGLRw1txHOxEAZADXE568LZmyhu6hZ/V/iar5cIz75iwKNTokZZ7HhcWD+OVFlQYiHxQ9PuGx4rzF1mTOIDhjDHN3XoGHiwxLhre3SR229u9HB4b+O6Y7VAopijQ6nE7NQd9W7lBo82sfW6mmtWI5Y2slo8wFmVpn5DJX9O/aFgo3n0qfJU/+M/TUP0+gqEyPIy/1R5FGh9lfnISfkwIvPjPQ6u+PpZpGEFlKLAGU3sabubQl/AcrNvEv/HT4EnwlhXg7ygtKff6jD2CuabgZdMa/hmWFQJ4lXUYPkybye/oiPJSokFuiwrk9yegvysNDpoJLoQ7QhBnDjuP4I2sBYPflO5g1MAwikfUG8QEg9X4hOjZ3q3WdtAdF+O3yHQDAoufaQWzlGiyVX6JFQYkWQZ6VDr+opbVStv88RkONB5sYVM6lKMy6g8iyh1BwRXUrQqIwBoeL8fdaKvOAXuEJFw/fWlsrMzacwpVHLeDYQdG4cr8Ql9LzAA2Ah+UvroWXMpf/w+StkvN/MCzdfa83MJRq9SjV6uHuLLPa765RBNGGDRuwZs0aZGVloXPnzvjiiy/Qo0cPocuyjNQJcAsE3ALR3a8TVt8IwO9/P0RBURA++n+dqq7PGFCab9qqKq7UHK/4V5NvkjPj+EBJLpDzFwBgfMXfYhzQs7ypfhfAKgBiGZizF7brpciRqvAQKuQeU+HWw3C0DA2tpkXmCUjkdXob3t4eh8Ed/GqdUqXi8UulWn3DHXtUw56gA8fiUJSXjZFhMqj0+ZBqcsEV54IV54CrobXyZnnJOcabL8AfXcjAoQBK6J294Ozmg0ydEkHNAyHhu+nloVLhPZcZQzD+dj6+Pp6K/8RlwsCArsHuGNU5AB4SGQ5fycZLPZuhZzNP/HrpDtYeSEZWQSlfU+QHh574FjhJxXCRifkDGgs1Oryz6wpKdQY+YIy3R/d1j34uM/5csSV95t1n4e9mnQkGBQ+iHTt2YM6cOfjqq6/Qs2dPrFu3DoMHD0ZycjJ8fHyELq9OOI7Doufa4sWNZ7DrYgaiWnnBRS6BVm9Amc4ABgapWASpWASZRAWZ2A0yeStInUUI8nCCl/JxCJRq9ZBLRNDoDFCIDCZdxvwHmcjKuoP/PZMAT04NT64AnlDDg1PDgyuEFwrgxJUB+jJw6sxHXcYKEg8CiTX8J2QqwNkTGrknbqhlCPBvDi+fABRL3VEqdYdnM/8Kg5Ye4GAAe3RY2tHk+xjS0a/G90cqfhxEJXUNoketFfXDLGRn3kWgvBjfHoiFt1gNT6gR4anH7dsZaK3UwEWXB0PRA8i0BdW+1BjA+E1IM11eXiWTOCFbr0S23gV6hRcCAwOx90YZcpkx1HOYCg+ZCjlwxUOmQh6UMEAElALINb5GtNIH3z4fidsPi/HT2b/hqpCiW4gHOjdzh0Iqwokb9/H18VScSnl8HBrHAZfT83A5PY9fduR6Njo2d8OZm9UfryYTizCovS8fNIBxDqKjyfcBGN9vjuPg6iSFVMxBq2fYdfG2+e97BaVa640zckzgk6Z69uyJyMhIrF+/HgBgMBgQFBSEmTNnYuHChbU+t6CgAG5ubsjPz4erq/VG8K3lzZ8vYl+C5SeYSkQc2vm74k5eickAs6tCgoJSHXxUcgR5OuPi3w9reRUjBTQIdSqBuPQhvLgCeEANT84YVp5Qo7msGIHyYhiKcozLoYaEs/wDpmcc8qBELnNFLlQICQyCk5s3nD38IFU1g8HJE5yzFziXZrirdUH0Vwkohhwn5j9r7BLpdSgtuA9RyQMYinKQ9vffCHUqBXsUuleTUxDiVAon7UPoCh9ApS+AnNM+ubBKDIxDHlz4OnOZK3KZErmPQiSHuVYJl1LUrXVYmZ+rwqQFAwAiDnB1kvJ7NMUiDiM7B+D1vi3RTCXDn1czsTfuLvKKyyARi5CSbTxNRy4Robm7E24+eNwV/GJ8VzzTxhtuTqZ7Qw0Ghh4fHsaDQg1CvJxx7J0BAIDDSfdwJSMPCpkYCokYCqkYCqkICqkYTlIx5I9+Nj72eLlCKoZcIjKrW2/ud1TQICorK4OzszN++eUXPP/88/zymJgY5OXlYe/evSbrazQaaDSPm8oFBQUICgpqtEF0+2Ex5uy8giKNztj6EYsglXDgwEGrNxhbSHoDtDoGrd5g8qESCgcDVCg2trAqhZYHp4YXCoz3H4WWF6eGK1e3WQk0TIp8TgU5yuCGup0HV8Jkj1oiykrhonr0s6rCchXyH7VWlHIJCisc4FmZt0peZVpVS/Vp3QwKqRiHku6ZLO8c6IZAD2dc+DsX9woeT1Q2LjIYr/YJRaBH9acKZatL8caPF9FMaRzkD/FyQV5xGUasP4kynQEn5j8LmaT6Y9ey1aVYve86RnQOwIDwhutp2EUQ3b17F82bN8fp06cRFRXFL58/fz6OHTuGc+fOmay/fPlyrFixosrrNNYgslRBqRap2YVgAB6oNQhwd0IzpRynUx+gVGuARMQhp6gMIV7GGQGSs9T4K1uNAeE+2HY+Han3i6BSSNA3rBmCPJ0hE4sQl5GH/uE+UMrFSLhTgIhAN5xNzeEHLC+lP4SBgd/lDwBPt/ZClyB3PFCXYceFjCfWLYEOHig0BlWFVpePuAgqQz4fWuXdRy+oq23NGBiHh1CadHPKA6W8taKRuSND4/zE1sqcQW2w8WgqSrR6+LspkJn/uCXSI9QT7wwJR6CHEz7adx174u7iwxciIBYBH+1PxsYJT6FnSy9k5Zfi4wPX4eEsQ1s/FUp1BrTwcoGniwzn03LQPdQTv1y8jZbeLmjRzAV380rQLcQDD4u18FbKEdrMeIR5XnEZTqfmQPboANd+jw52Lb/8c3aBBkGezlVaMuYyGBgMjFntSHlrcsggsrcWkaPQGxhEnPEYIePPHCQizqRpzhhDQYkOesbgLBNDbzD+eyunGCVleohEgO7RQGdOoQZMWwQXbQFcUQCR1Amc0htQuMFZIUeZzgCVwjhu9LC4DCqFFHoDg7+bwqon6hLbMzeIBB2sbtasGcRiMe7dM2263rt3D35+VQc75XI55HLr9NeJ+cp30UrFHGraGcZxHNycq/5Fb9GsfuedVRy4J45L0LacTCZDt27dcPjwYX6ZwWDA4cOHTVpIhBDHJvju+zlz5iAmJgbdu3dHjx49sG7dOhQVFWHSpElCl0YIaSCCB9HYsWNx//59LF26FFlZWejSpQv2798PX9/GN40GIcQ2BD+OqD4a+3FEhDR15n5HG9/+PkJIk0NBRAgRHAURIURwgg9W10f58FZBQfUnMxJChFX+3XzSULRdB5FarQYABAUFCVwJIaQ2arUabm41z1Nl13vNDAYD7t69C5VKZZVD/8tPGcnIyKC9cA2A3u+G19DvOWMMarUaAQEBEIlqHgmy6xaRSCRCYGCg1V/X1dWVvhgNiN7vhteQ73ltLaFyNFhNCBEcBREhRHAURBXI5XIsW7aMzvBvIPR+N7zG+p7b9WA1IcQxUIuIECI4CiJCiOAoiAghgqMgIoQIrkkHUW5uLiZMmABXV1e4u7tj8uTJKCys+bI2ubm5mDlzJsLDw+Hk5ITg4GC89dZbyM/Pb8Cq7cuGDRsQGhoKhUKBnj174vz587Wuv2vXLrRt2xYKhQIRERH4z3/+00CVOg5L3vNNmzahb9++8PDwgIeHB6Kjo5/4O7IJ1oQNGTKEde7cmZ09e5adOHGCtW7dmo0fP77G9ePj49no0aPZ77//zlJSUtjhw4dZWFgYe/HFFxuwavuxfft2JpPJ2HfffceuXbvGXn/9debu7s7u3btX7fqnTp1iYrGYffzxxywxMZEtXryYSaVSFh8f38CV2y9L3/OXXnqJbdiwgV2+fJklJSWxiRMnMjc3N3b79u0GrbvJBlFiYiIDwGJjY/ll+/btYxzHsTt37pj9Ojt37mQymYxptVpblGnXevTowaZPn87f1+v1LCAggK1atara9ceMGcOGDRtmsqxnz55sypQpNq3TkVj6nlem0+mYSqViP/zwg61KrFaT7ZqdOXMG7u7u6N69O78sOjoaIpGoyvXUalM+BaZEYten7VldWVkZLl68iOjoaH6ZSCRCdHQ0zpw5U+1zzpw5Y7I+AAwePLjG9YmpurznlRUXF0Or1cLT09NWZVaryQZRVlYWfHxML70rkUjg6emJrCzzrlf/4MEDrFy5Em+88YYtSrRrDx48gF6vr3IRBF9f3xrf36ysLIvWJ6bq8p5XtmDBAgQEBFT5g2BrDhdECxcuBMdxtd6uX79e7+0UFBRg2LBhaN++PZYvX17/wgkR2OrVq7F9+3bs3r0bCoWiQbftcP2JuXPnYuLEibWu07JlS/j5+SE7O9tkuU6nQ25ubrVXma1IrVZjyJAhUKlU2L17N6TSul2z3JFZehVfAPDz87NofWKqLu95ubVr12L16tU4dOgQOnXqZMsyq9egI1KNSPlg9YULF/hlBw4ceOJgdX5+PuvVqxfr168fKyoqaohS7VaPHj3YjBkz+Pt6vZ41b9681sHq4cOHmyyLioqiwWoLWPqeM8bYRx99xFxdXdmZM2caosRqNdkgYsy4+75r167s3Llz7OTJkywsLMxk9/3t27dZeHg4O3fuHGPMGEI9e/ZkERERLCUlhWVmZvI3nU4n1H+j0dq+fTuTy+Vs8+bNLDExkb3xxhvM3d2dZWVlMcYYe/nll9nChQv59U+dOsUkEglbu3YtS0pKYsuWLaPd9xay9D1fvXo1k8lk7JdffjH5PKvV6gatu0kHUU5ODhs/fjxTKpXM1dWVTZo0yeQXkJaWxgCwI0eOMMYYO3LkCANQ7S0tLU2Y/0Qj98UXX7Dg4GAmk8lYjx492NmzZ/nH+vXrx2JiYkzW37lzJ2vTpg2TyWSsQ4cO7M8//2zgiu2fJe95SEhItZ/nZcuWNWjNNA0IIURwDrfXjBBifyiICCGCoyAihAiOgogQIjgKIkKI4CiICCGCoyAihAiOgogQIjgKIkKI4CiIiNVNnDix2ulXhgwZInRppJFyuGlASOMwZMgQfP/99ybLarrMsVarrTKVSllZGWQymcXbrevziLCoRURsQi6Xw8/Pz+Tm4eEBAOA4Dhs3bsTIkSPh4uKCDz74AMuXL0eXLl3w7bffokWLFvzEXOnp6Rg1ahSUSiVcXV0xZswYk/l2anoesS8UREQQy5cvxwsvvID4+Hi8+uqrAICUlBT8+uuv+O233xAXFweDwYBRo0YhNzcXx44dw8GDB3Hz5k2MHTvW5LUqP4/YH+qaEZv4448/oFQqTZYtWrQIixYtAgC89NJLmDRpksnjZWVl+PHHH+Ht7Q0AOHjwIOLj45GWloagoCAAwI8//ogOHTogNjYWkZGR1T6P2B8KImITAwYMwMaNG02WVbwyRMWrp5QLCQkxCZOkpCQEBQXxIQQA7du3h7u7O5KSkvggqvw8Yn8oiIhNuLi4oHXr1rU+bs4yc7dF7BuNEZFGq127dsjIyEBGRga/LDExEXl5eWjfvr2AlRFroxYRsQmNRlPlWloSiQTNmjUz+zWio6MRERGBCRMmYN26ddDpdJg2bRr69etXbdeO2C9qERGb2L9/P/z9/U1uffr0seg1OI7D3r174eHhgWeeeQbR0dFo2bIlduzYYaOqiVBozmpCiOCoRUQIERwFESFEcBREhBDBURARQgRHQUQIERwFESFEcBREhBDBURARQgRHQUQIERwFESFEcBREhBDB/X8bvPz19fqPPAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019035, + "end_time": "2024-02-29T06:23:58.882981", + "exception": false, + "start_time": "2024-02-29T06:23:58.863946", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + }, + "papermill": { + "default_parameters": {}, + "duration": 1881.475459, + "end_time": "2024-02-29T06:24:01.623005", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tab_ddpm_concat/4/mlu-eval.ipynb", + "output_path": "eval/contraceptive/tab_ddpm_concat/4/mlu-eval.ipynb", + "parameters": { + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "path": "eval/contraceptive/tab_ddpm_concat/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-02-29T05:52:40.147546", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/contraceptive/tab_ddpm_concat/model.pt b/contraceptive/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e670cd1d9a24b5f53020b44a944e2e0f441b104 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:741402a51d8fa376ef10ca6f67a4ce472b2d64f4cea965d2167ec8a324acbc7c +size 47482955 diff --git a/contraceptive/tab_ddpm_concat/params.json b/contraceptive/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..6ccb729149b2c2a713a6bb3a529e2062584066e4 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.73, "gradient_penalty_mode": "ALL", "synth_data": 2, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.09, "n_warmup_steps": 100, "Optim": "amsgradw", "loss_balancer_beta": 0.67, "loss_balancer_r": 0.943, "fixed_role_model": "tab_ddpm_concat", "mse_mag": true, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation": "tanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 128, "ada_d_hid": 1024, "ada_n_layers": 9, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation": "softsign", "head_activation_final": "leakyhardsigmoid", "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/tvae/eval.csv b/contraceptive/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..e8f3bd7ab11130525d5417e235f037426c9326c1 --- /dev/null +++ b/contraceptive/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.0048602849527101385,,0.001199199252957093,2.5938045978546143,0.029055854305624962,0.575093150138855,0.03735598176717758,7.052161663523293e-07,3.237421989440918,0.02745048701763153,0.06274975836277008,0.0346294566988945,0.05399195849895477,0.026110012084245682,5.831226587295532 diff --git a/contraceptive/tvae/history.csv b/contraceptive/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..ac9023f50668285a566e1b01e6d2961252351f69 --- /dev/null +++ b/contraceptive/tvae/history.csv @@ -0,0 +1,20 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.02044109800390288,2.4362574391484584,0.0019195302865352177,0.16679631007427814,0.0,0.0,0.0,0.0,0.020752195270142694,320,160,152.6873915195465,0.9542961969971657,0.47714809849858286,0.07773836838096031,0.007787475768054719,2.2598845489101174,0.0001329833972132044,0.06387415776262059,0.0,0.0,0.0,0.0,0.007876941776339663,80,40,34.667025566101074,0.8666756391525269,0.43333781957626344,0.037622747202112804 +1,0.006578218158620075,3.5744917789010295,0.0001345584984532182,0.040657627023756505,0.0,0.0,0.0,0.0,0.006674684401968989,320,160,158.56897902488708,0.9910561189055442,0.4955280594527721,0.06574895506673784,0.002715696775703691,4.162856922139042,9.724292509610821e-06,0.020695178036112337,0.0,0.0,0.0,0.0,0.002750939411816944,80,40,35.01019263267517,0.8752548158168793,0.4376274079084396,0.022036342885985504 +2,0.0030868062563627063,1.476575678907632,1.725077691700862e-05,0.024703686686552827,0.0,0.0,0.0,0.0,0.0031247849726014467,320,160,156.26092982292175,0.9766308113932609,0.48831540569663046,0.06579984588715888,0.0027993964433790097,3.641182864084643,1.3598827807209668e-05,0.022234968321936322,0.0,0.0,0.0,0.0,0.002834092272314592,80,40,33.2850501537323,0.8321262538433075,0.41606312692165376,0.02141796936703031 +3,0.0022891667019848683,1.6068099677926182,6.612543098453473e-06,0.02071905894408701,0.0,0.0,0.0,0.0,0.0023184519005326363,320,160,154.19920873641968,0.9637450546026229,0.48187252730131147,0.07092049774619227,0.0026929985309834593,3.0403425516754,1.0568798141985525e-05,0.018767503660637886,0.0,0.0,0.0,0.0,0.002731285383561044,80,40,32.86460590362549,0.8216151475906373,0.4108075737953186,0.022138956440176116 +4,0.0017628774662000525,2.2449683045348796,3.043247930578793e-06,0.01629736885879538,0.0,0.0,0.0,0.0,0.0017833273730786913,320,160,154.66035175323486,0.9666271984577179,0.48331359922885897,0.07253363067648025,0.00217515139884199,2.751713131871395,6.859175753426428e-06,0.016465158166829495,0.0,0.0,0.0,0.0,0.0022024019488071644,80,40,32.97675704956055,0.8244189262390137,0.41220946311950685,0.027853763510938732 +5,0.0016198344877579984,1.198978344199396,2.7660729376433046e-06,0.013700315837559174,0.0,0.0,0.0,0.0,0.00163975777767007,320,160,151.12690496444702,0.9445431560277939,0.47227157801389696,0.07664285300816118,0.0023051692825561076,3.7659985534039606,6.092837360660574e-06,0.014564098932169145,0.0,0.0,0.0,0.0,0.002337455260965271,80,40,33.647725105285645,0.8411931276321412,0.4205965638160706,0.022250424231697253 +6,0.0015104103944622693,1.4739226749323517,2.151131067418738e-06,0.014532235827937256,0.0,0.0,0.0,0.0,0.0015276236336944748,320,160,150.64317202568054,0.9415198251605034,0.4707599125802517,0.07481845202296426,0.0017437607562897028,1.7716060946109806,3.2990996021275974e-06,0.013805617642356082,0.0,0.0,0.0,0.0,0.0017670220266154501,80,40,32.80771327018738,0.8201928317546845,0.41009641587734225,0.032647981911577514 +7,0.0014582326822619508,0.8133875374451442,2.600399002399001e-06,0.017096595877592335,0.0,0.0,0.0,0.0,0.0014742144544925395,320,160,150.58923768997192,0.9411827355623246,0.4705913677811623,0.07290942690669908,0.0022431900478295575,2.112379498438763,8.708145000263512e-06,0.02324506860168185,0.0,0.0,0.0,0.0,0.0022669869065794048,80,40,32.844870805740356,0.821121770143509,0.4105608850717545,0.030994260040461085 +8,0.0012292226523300087,0.9143240710843227,1.1601490665833457e-06,0.017422522912238492,0.0,0.0,0.0,0.0,0.0012417459553262233,320,160,149.79405570030212,0.9362128481268883,0.46810642406344416,0.08164471380441682,0.002094481812127924,2.5781812031031945,5.563664813551528e-06,0.023539607209386304,0.0,0.0,0.0,0.0,0.0021182637627134683,80,40,32.88525176048279,0.8221312940120697,0.4110656470060349,0.02998391728651768 +9,0.0011912165241611205,1.002233497608629,1.0378981288260842e-06,0.016518235197145258,0.0,0.0,0.0,0.0,0.0012034058073304265,320,160,149.81003165245056,0.936312697827816,0.468156348913908,0.07757201609729236,0.0021209866390563548,2.2054855651783485,7.790026517334384e-06,0.027394813470891677,0.0,0.0,0.0,0.0,0.0021441533169308967,80,40,32.834739208221436,0.8208684802055359,0.41043424010276797,0.03273092644376448 +10,0.0010440993143930656,0.6237209759905976,8.545894512740349e-07,0.016773289057891817,0.0,0.0,0.0,0.0,0.0010549161908912196,320,160,151.99514055252075,0.9499696284532547,0.47498481422662736,0.08429675764928106,0.0020188588828659705,1.6875887818820332,7.073809199660553e-06,0.020735643728403374,0.0,0.0,0.0,0.0,0.002045956869096699,80,40,36.630669593811035,0.9157667398452759,0.45788336992263795,0.033265805801784155 +11,0.0010166813490059211,0.5779329396736607,1.1541078821007341e-06,0.016200161819870117,0.0,0.0,0.0,0.0,0.0010268422765705055,320,160,154.49023914337158,0.9655639946460723,0.48278199732303617,0.08438906156479789,0.0022567970930140293,1.4275073309931146,9.043062631357477e-06,0.027866220747819172,0.0,0.0,0.0,0.0,0.0022840781810373302,80,40,33.030266523361206,0.8257566630840302,0.4128783315420151,0.03308896141534205 +12,0.0010085937371733245,0.6852384920215103,1.1549593074543923e-06,0.015572391230125503,0.0,0.0,0.0,0.0,0.0010184062243165926,320,160,151.7514419555664,0.94844651222229,0.474223256111145,0.06961961404886097,0.002278934889363882,2.298216237341036,8.917010871634101e-06,0.031079518698970788,0.0,0.0,0.0,0.0,0.002304124432612298,80,40,33.068289279937744,0.8267072319984436,0.4133536159992218,0.03493260265313438 +13,0.0009363631818700924,0.4839344189483297,8.540815376802124e-07,0.01630451331693621,0.0,0.0,0.0,0.0,0.0009456257481076591,320,160,150.87772369384766,0.9429857730865479,0.4714928865432739,0.07785468016882077,0.0022760112356991157,2.6171030896344747,1.0650064827233408e-05,0.029537392102065498,0.0,0.0,0.0,0.0,0.002303297990329156,80,40,33.033122539520264,0.8258280634880066,0.4129140317440033,0.03722803041819134 +14,0.000764540546202852,0.44834804190900857,6.100927646765161e-07,0.013185284686551313,0.0,0.0,0.0,0.0,0.0007720930538482662,320,160,150.77999806404114,0.9423749879002571,0.47118749395012854,0.08267560889125888,0.0020305066434957554,3.3567321322424286,6.930302250784948e-06,0.030277981684776023,0.0,0.0,0.0,0.0,0.002052497151453281,80,40,33.12909913063049,0.8282274782657624,0.4141137391328812,0.033659220521803944 +15,0.0007402757991229692,0.38544692055954943,4.2042679632425807e-07,0.013898816978326067,0.0,0.0,0.0,0.0,0.0007476143626305998,320,160,150.68796515464783,0.9417997822165489,0.47089989110827446,0.08294280422669545,0.0022146169496409128,1.9662903952322979,8.018624131983509e-06,0.03200231332739349,0.0,0.0,0.0,0.0,0.002240737696956785,80,40,33.06134486198425,0.8265336215496063,0.41326681077480315,0.03342230037087575 +16,0.0006711353589196279,0.5376870474081945,3.8460864332618064e-07,0.012289214101656398,0.0,0.0,0.0,0.0,0.0006777132193121815,320,160,150.60209393501282,0.9412630870938301,0.47063154354691505,0.07368135868346144,0.0023524648198872456,2.413720256314673,1.1856669321630431e-05,0.030548772783367893,0.0,0.0,0.0,0.0,0.002378911971572961,80,40,33.232566833496094,0.8308141708374024,0.4154070854187012,0.037087300876009976 +17,0.0006756978137346436,0.7821963908104526,5.090253047178393e-07,0.013120992203766946,0.0,0.0,0.0,0.0,0.0006825899582267425,320,160,152.36271023750305,0.9522669389843941,0.47613346949219704,0.07919310781685454,0.002482261122668206,1.5217115195275608,1.1857084663224882e-05,0.03310881347861141,0.0,0.0,0.0,0.0,0.002508918128592086,80,40,33.57771110534668,0.839442777633667,0.4197213888168335,0.03645229901885614 +18,0.0006414469639224763,0.8802176390026426,3.0969604700309403e-07,0.013044403970798157,0.0,0.0,0.0,0.0,0.000647848942770679,320,160,151.09337043762207,0.9443335652351379,0.47216678261756895,0.07918862322630957,0.00226858947034998,2.6035211705103833,9.649373751539902e-06,0.03664247310261999,0.0,0.0,0.0,0.0,0.0022923396401893113,80,40,33.24403095245361,0.8311007738113403,0.41555038690567014,0.033244564172491664 diff --git a/contraceptive/tvae/mlu-eval.ipynb b/contraceptive/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..ebce9eb65bc6eaab150c23f8408f90f766edaffc --- /dev/null +++ b/contraceptive/tvae/mlu-eval.ipynb @@ -0,0 +1,2557 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:18.991476Z", + "iopub.status.busy": "2024-03-01T11:21:18.991039Z", + "iopub.status.idle": "2024-03-01T11:21:19.024377Z", + "shell.execute_reply": "2024-03-01T11:21:19.023547Z" + }, + "papermill": { + "duration": 0.047845, + "end_time": "2024-03-01T11:21:19.026349", + "exception": false, + "start_time": "2024-03-01T11:21:18.978504", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:19.051388Z", + "iopub.status.busy": "2024-03-01T11:21:19.051042Z", + "iopub.status.idle": "2024-03-01T11:21:19.057731Z", + "shell.execute_reply": "2024-03-01T11:21:19.056910Z" + }, + "papermill": { + "duration": 0.021255, + "end_time": "2024-03-01T11:21:19.059623", + "exception": false, + "start_time": "2024-03-01T11:21:19.038368", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:19.082877Z", + "iopub.status.busy": "2024-03-01T11:21:19.082367Z", + "iopub.status.idle": "2024-03-01T11:21:19.086257Z", + "shell.execute_reply": "2024-03-01T11:21:19.085478Z" + }, + "papermill": { + "duration": 0.017692, + "end_time": "2024-03-01T11:21:19.088210", + "exception": false, + "start_time": "2024-03-01T11:21:19.070518", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:19.111370Z", + "iopub.status.busy": "2024-03-01T11:21:19.111116Z", + "iopub.status.idle": "2024-03-01T11:21:19.114867Z", + "shell.execute_reply": "2024-03-01T11:21:19.114006Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017519, + "end_time": "2024-03-01T11:21:19.116720", + "exception": false, + "start_time": "2024-03-01T11:21:19.099201", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:19.139949Z", + "iopub.status.busy": "2024-03-01T11:21:19.139688Z", + "iopub.status.idle": "2024-03-01T11:21:19.144765Z", + "shell.execute_reply": "2024-03-01T11:21:19.143940Z" + }, + "papermill": { + "duration": 0.018873, + "end_time": "2024-03-01T11:21:19.146653", + "exception": false, + "start_time": "2024-03-01T11:21:19.127780", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "847e48c9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:19.170889Z", + "iopub.status.busy": "2024-03-01T11:21:19.170636Z", + "iopub.status.idle": "2024-03-01T11:21:19.175182Z", + "shell.execute_reply": "2024-03-01T11:21:19.174397Z" + }, + "papermill": { + "duration": 0.018855, + "end_time": "2024-03-01T11:21:19.177150", + "exception": false, + "start_time": "2024-03-01T11:21:19.158295", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tvae\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tvae/4\"\n", + "param_index = 2\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011005, + "end_time": "2024-03-01T11:21:19.199198", + "exception": false, + "start_time": "2024-03-01T11:21:19.188193", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:19.222264Z", + "iopub.status.busy": "2024-03-01T11:21:19.221996Z", + "iopub.status.idle": "2024-03-01T11:21:19.230567Z", + "shell.execute_reply": "2024-03-01T11:21:19.229855Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022285, + "end_time": "2024-03-01T11:21:19.232435", + "exception": false, + "start_time": "2024-03-01T11:21:19.210150", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tvae/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:19.255641Z", + "iopub.status.busy": "2024-03-01T11:21:19.255364Z", + "iopub.status.idle": "2024-03-01T11:21:21.385622Z", + "shell.execute_reply": "2024-03-01T11:21:21.384784Z" + }, + "papermill": { + "duration": 2.144372, + "end_time": "2024-03-01T11:21:21.387811", + "exception": false, + "start_time": "2024-03-01T11:21:19.243439", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:21.413880Z", + "iopub.status.busy": "2024-03-01T11:21:21.413463Z", + "iopub.status.idle": "2024-03-01T11:21:21.424005Z", + "shell.execute_reply": "2024-03-01T11:21:21.423311Z" + }, + "papermill": { + "duration": 0.025516, + "end_time": "2024-03-01T11:21:21.425860", + "exception": false, + "start_time": "2024-03-01T11:21:21.400344", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:21.449346Z", + "iopub.status.busy": "2024-03-01T11:21:21.449100Z", + "iopub.status.idle": "2024-03-01T11:21:21.455681Z", + "shell.execute_reply": "2024-03-01T11:21:21.454985Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.020462, + "end_time": "2024-03-01T11:21:21.457574", + "exception": false, + "start_time": "2024-03-01T11:21:21.437112", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:21.480972Z", + "iopub.status.busy": "2024-03-01T11:21:21.480733Z", + "iopub.status.idle": "2024-03-01T11:21:21.582153Z", + "shell.execute_reply": "2024-03-01T11:21:21.581150Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.115754, + "end_time": "2024-03-01T11:21:21.584457", + "exception": false, + "start_time": "2024-03-01T11:21:21.468703", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:21.610975Z", + "iopub.status.busy": "2024-03-01T11:21:21.610344Z", + "iopub.status.idle": "2024-03-01T11:21:26.148754Z", + "shell.execute_reply": "2024-03-01T11:21:26.147955Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.55451, + "end_time": "2024-03-01T11:21:26.151252", + "exception": false, + "start_time": "2024-03-01T11:21:21.596742", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-01 11:21:23.809078: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-01 11:21:23.809133: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-01 11:21:23.810798: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:26.176831Z", + "iopub.status.busy": "2024-03-01T11:21:26.176040Z", + "iopub.status.idle": "2024-03-01T11:21:26.181620Z", + "shell.execute_reply": "2024-03-01T11:21:26.180894Z" + }, + "papermill": { + "duration": 0.02006, + "end_time": "2024-03-01T11:21:26.183512", + "exception": false, + "start_time": "2024-03-01T11:21:26.163452", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:26.209243Z", + "iopub.status.busy": "2024-03-01T11:21:26.208874Z", + "iopub.status.idle": "2024-03-01T11:21:34.293747Z", + "shell.execute_reply": "2024-03-01T11:21:34.292712Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.10031, + "end_time": "2024-03-01T11:21:34.296053", + "exception": false, + "start_time": "2024-03-01T11:21:26.195743", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.775,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.075,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'loss_balancer_beta': 0.675,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:34.824588Z", + "iopub.status.busy": "2024-03-01T11:21:34.824276Z", + "iopub.status.idle": "2024-03-01T11:21:34.892202Z", + "shell.execute_reply": "2024-03-01T11:21:34.891354Z" + }, + "papermill": { + "duration": 0.083366, + "end_time": "2024-03-01T11:21:34.894160", + "exception": false, + "start_time": "2024-03-01T11:21:34.810794", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../contraceptive/_cache/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache4/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache5/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/contraceptive [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-01T11:21:34.921773Z", + "iopub.status.busy": "2024-03-01T11:21:34.921469Z", + "iopub.status.idle": "2024-03-01T11:21:35.357515Z", + "shell.execute_reply": "2024-03-01T11:21:35.356611Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.452312, + "end_time": "2024-03-01T11:21:35.359599", + "exception": false, + "start_time": "2024-03-01T11:21:34.907287", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:35.388513Z", + "iopub.status.busy": "2024-03-01T11:21:35.388216Z", + "iopub.status.idle": "2024-03-01T11:21:35.392352Z", + "shell.execute_reply": "2024-03-01T11:21:35.391474Z" + }, + "papermill": { + "duration": 0.020703, + "end_time": "2024-03-01T11:21:35.394335", + "exception": false, + "start_time": "2024-03-01T11:21:35.373632", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:35.420678Z", + "iopub.status.busy": "2024-03-01T11:21:35.420380Z", + "iopub.status.idle": "2024-03-01T11:21:35.427052Z", + "shell.execute_reply": "2024-03-01T11:21:35.426244Z" + }, + "papermill": { + "duration": 0.021862, + "end_time": "2024-03-01T11:21:35.428877", + "exception": false, + "start_time": "2024-03-01T11:21:35.407015", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10270216" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:35.455142Z", + "iopub.status.busy": "2024-03-01T11:21:35.454873Z", + "iopub.status.idle": "2024-03-01T11:21:35.534419Z", + "shell.execute_reply": "2024-03-01T11:21:35.533582Z" + }, + "papermill": { + "duration": 0.095558, + "end_time": "2024-03-01T11:21:35.536886", + "exception": false, + "start_time": "2024-03-01T11:21:35.441328", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 46] --\n", + "├─Adapter: 1-1 [2, 1179, 46] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 48,128\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-16 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 46] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-18 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-32 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-3 [2, 8, 256] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 8, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 8, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 8, 256] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 8, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 8, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 8, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 8, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 8, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-21 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-22 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-23 [2, 8, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 8, 256] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-24 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-25 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-15 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-43 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 8, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-45 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-46 [2, 8, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 256] --\n", + "│ │ │ └─Linear: 4-39 [2, 256] 524,544\n", + "│ │ │ └─ReLU6: 4-40 [2, 256] --\n", + "│ │ └─FeedForward: 3-20 [2, 256] --\n", + "│ │ │ └─Linear: 4-41 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-42 [2, 256] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 1] --\n", + "│ │ │ └─Linear: 4-55 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-56 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,270,216\n", + "Trainable params: 10,270,216\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 39.96\n", + "========================================================================================================================\n", + "Input size (MB): 0.54\n", + "Forward/backward pass size (MB): 341.77\n", + "Params size (MB): 41.08\n", + "Estimated Total Size (MB): 383.39\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T11:21:35.566842Z", + "iopub.status.busy": "2024-03-01T11:21:35.566509Z", + "iopub.status.idle": "2024-03-01T12:26:33.657855Z", + "shell.execute_reply": "2024-03-01T12:26:33.656822Z" + }, + "papermill": { + "duration": 3898.108413, + "end_time": "2024-03-01T12:26:33.659957", + "exception": false, + "start_time": "2024-03-01T11:21:35.551544", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.02044109800390288, 'avg_role_model_std_loss': 2.4362574391484584, 'avg_role_model_mean_pred_loss': 0.0019195302865352177, 'avg_role_model_g_mag_loss': 0.16679631007427814, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.020752195270142694, 'n_size': 320, 'n_batch': 160, 'duration': 152.6873915195465, 'duration_batch': 0.9542961969971657, 'duration_size': 0.47714809849858286, 'avg_pred_std': 0.07773836838096031}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007787475768054719, 'avg_role_model_std_loss': 2.2598845489101174, 'avg_role_model_mean_pred_loss': 0.0001329833972132044, 'avg_role_model_g_mag_loss': 0.06387415776262059, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007876941776339663, 'n_size': 80, 'n_batch': 40, 'duration': 34.667025566101074, 'duration_batch': 0.8666756391525269, 'duration_size': 0.43333781957626344, 'avg_pred_std': 0.037622747202112804}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006578218158620075, 'avg_role_model_std_loss': 3.5744917789010295, 'avg_role_model_mean_pred_loss': 0.0001345584984532182, 'avg_role_model_g_mag_loss': 0.040657627023756505, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006674684401968989, 'n_size': 320, 'n_batch': 160, 'duration': 158.56897902488708, 'duration_batch': 0.9910561189055442, 'duration_size': 0.4955280594527721, 'avg_pred_std': 0.06574895506673784}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002715696775703691, 'avg_role_model_std_loss': 4.162856922139042, 'avg_role_model_mean_pred_loss': 9.724292509610821e-06, 'avg_role_model_g_mag_loss': 0.020695178036112337, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002750939411816944, 'n_size': 80, 'n_batch': 40, 'duration': 35.01019263267517, 'duration_batch': 0.8752548158168793, 'duration_size': 0.4376274079084396, 'avg_pred_std': 0.022036342885985504}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0030868062563627063, 'avg_role_model_std_loss': 1.476575678907632, 'avg_role_model_mean_pred_loss': 1.725077691700862e-05, 'avg_role_model_g_mag_loss': 0.024703686686552827, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0031247849726014467, 'n_size': 320, 'n_batch': 160, 'duration': 156.26092982292175, 'duration_batch': 0.9766308113932609, 'duration_size': 0.48831540569663046, 'avg_pred_std': 0.06579984588715888}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0027993964433790097, 'avg_role_model_std_loss': 3.641182864084643, 'avg_role_model_mean_pred_loss': 1.3598827807209668e-05, 'avg_role_model_g_mag_loss': 0.022234968321936322, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002834092272314592, 'n_size': 80, 'n_batch': 40, 'duration': 33.2850501537323, 'duration_batch': 0.8321262538433075, 'duration_size': 0.41606312692165376, 'avg_pred_std': 0.02141796936703031}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0022891667019848683, 'avg_role_model_std_loss': 1.6068099677926182, 'avg_role_model_mean_pred_loss': 6.612543098453473e-06, 'avg_role_model_g_mag_loss': 0.02071905894408701, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023184519005326363, 'n_size': 320, 'n_batch': 160, 'duration': 154.19920873641968, 'duration_batch': 0.9637450546026229, 'duration_size': 0.48187252730131147, 'avg_pred_std': 0.07092049774619227}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0026929985309834593, 'avg_role_model_std_loss': 3.0403425516754, 'avg_role_model_mean_pred_loss': 1.0568798141985525e-05, 'avg_role_model_g_mag_loss': 0.018767503660637886, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002731285383561044, 'n_size': 80, 'n_batch': 40, 'duration': 32.86460590362549, 'duration_batch': 0.8216151475906373, 'duration_size': 0.4108075737953186, 'avg_pred_std': 0.022138956440176116}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017628774662000525, 'avg_role_model_std_loss': 2.2449683045348796, 'avg_role_model_mean_pred_loss': 3.043247930578793e-06, 'avg_role_model_g_mag_loss': 0.01629736885879538, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017833273730786913, 'n_size': 320, 'n_batch': 160, 'duration': 154.66035175323486, 'duration_batch': 0.9666271984577179, 'duration_size': 0.48331359922885897, 'avg_pred_std': 0.07253363067648025}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00217515139884199, 'avg_role_model_std_loss': 2.751713131871395, 'avg_role_model_mean_pred_loss': 6.859175753426428e-06, 'avg_role_model_g_mag_loss': 0.016465158166829495, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022024019488071644, 'n_size': 80, 'n_batch': 40, 'duration': 32.97675704956055, 'duration_batch': 0.8244189262390137, 'duration_size': 0.41220946311950685, 'avg_pred_std': 0.027853763510938732}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016198344877579984, 'avg_role_model_std_loss': 1.198978344199396, 'avg_role_model_mean_pred_loss': 2.7660729376433046e-06, 'avg_role_model_g_mag_loss': 0.013700315837559174, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00163975777767007, 'n_size': 320, 'n_batch': 160, 'duration': 151.12690496444702, 'duration_batch': 0.9445431560277939, 'duration_size': 0.47227157801389696, 'avg_pred_std': 0.07664285300816118}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023051692825561076, 'avg_role_model_std_loss': 3.7659985534039606, 'avg_role_model_mean_pred_loss': 6.092837360660574e-06, 'avg_role_model_g_mag_loss': 0.014564098932169145, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002337455260965271, 'n_size': 80, 'n_batch': 40, 'duration': 33.647725105285645, 'duration_batch': 0.8411931276321412, 'duration_size': 0.4205965638160706, 'avg_pred_std': 0.022250424231697253}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015104103944622693, 'avg_role_model_std_loss': 1.4739226749323517, 'avg_role_model_mean_pred_loss': 2.151131067418738e-06, 'avg_role_model_g_mag_loss': 0.014532235827937256, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015276236336944748, 'n_size': 320, 'n_batch': 160, 'duration': 150.64317202568054, 'duration_batch': 0.9415198251605034, 'duration_size': 0.4707599125802517, 'avg_pred_std': 0.07481845202296426}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0017437607562897028, 'avg_role_model_std_loss': 1.7716060946109806, 'avg_role_model_mean_pred_loss': 3.2990996021275974e-06, 'avg_role_model_g_mag_loss': 0.013805617642356082, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017670220266154501, 'n_size': 80, 'n_batch': 40, 'duration': 32.80771327018738, 'duration_batch': 0.8201928317546845, 'duration_size': 0.41009641587734225, 'avg_pred_std': 0.032647981911577514}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014582326822619508, 'avg_role_model_std_loss': 0.8133875374451442, 'avg_role_model_mean_pred_loss': 2.600399002399001e-06, 'avg_role_model_g_mag_loss': 0.017096595877592335, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014742144544925395, 'n_size': 320, 'n_batch': 160, 'duration': 150.58923768997192, 'duration_batch': 0.9411827355623246, 'duration_size': 0.4705913677811623, 'avg_pred_std': 0.07290942690669908}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022431900478295575, 'avg_role_model_std_loss': 2.112379498438763, 'avg_role_model_mean_pred_loss': 8.708145000263512e-06, 'avg_role_model_g_mag_loss': 0.02324506860168185, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022669869065794048, 'n_size': 80, 'n_batch': 40, 'duration': 32.844870805740356, 'duration_batch': 0.821121770143509, 'duration_size': 0.4105608850717545, 'avg_pred_std': 0.030994260040461085}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012292226523300087, 'avg_role_model_std_loss': 0.9143240710843227, 'avg_role_model_mean_pred_loss': 1.1601490665833457e-06, 'avg_role_model_g_mag_loss': 0.017422522912238492, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012417459553262233, 'n_size': 320, 'n_batch': 160, 'duration': 149.79405570030212, 'duration_batch': 0.9362128481268883, 'duration_size': 0.46810642406344416, 'avg_pred_std': 0.08164471380441682}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002094481812127924, 'avg_role_model_std_loss': 2.5781812031031945, 'avg_role_model_mean_pred_loss': 5.563664813551528e-06, 'avg_role_model_g_mag_loss': 0.023539607209386304, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021182637627134683, 'n_size': 80, 'n_batch': 40, 'duration': 32.88525176048279, 'duration_batch': 0.8221312940120697, 'duration_size': 0.4110656470060349, 'avg_pred_std': 0.02998391728651768}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011912165241611205, 'avg_role_model_std_loss': 1.002233497608629, 'avg_role_model_mean_pred_loss': 1.0378981288260842e-06, 'avg_role_model_g_mag_loss': 0.016518235197145258, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012034058073304265, 'n_size': 320, 'n_batch': 160, 'duration': 149.81003165245056, 'duration_batch': 0.936312697827816, 'duration_size': 0.468156348913908, 'avg_pred_std': 0.07757201609729236}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021209866390563548, 'avg_role_model_std_loss': 2.2054855651783485, 'avg_role_model_mean_pred_loss': 7.790026517334384e-06, 'avg_role_model_g_mag_loss': 0.027394813470891677, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021441533169308967, 'n_size': 80, 'n_batch': 40, 'duration': 32.834739208221436, 'duration_batch': 0.8208684802055359, 'duration_size': 0.41043424010276797, 'avg_pred_std': 0.03273092644376448}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010440993143930656, 'avg_role_model_std_loss': 0.6237209759905976, 'avg_role_model_mean_pred_loss': 8.545894512740349e-07, 'avg_role_model_g_mag_loss': 0.016773289057891817, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010549161908912196, 'n_size': 320, 'n_batch': 160, 'duration': 151.99514055252075, 'duration_batch': 0.9499696284532547, 'duration_size': 0.47498481422662736, 'avg_pred_std': 0.08429675764928106}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0020188588828659705, 'avg_role_model_std_loss': 1.6875887818820332, 'avg_role_model_mean_pred_loss': 7.073809199660553e-06, 'avg_role_model_g_mag_loss': 0.020735643728403374, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002045956869096699, 'n_size': 80, 'n_batch': 40, 'duration': 36.630669593811035, 'duration_batch': 0.9157667398452759, 'duration_size': 0.45788336992263795, 'avg_pred_std': 0.033265805801784155}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010166813490059211, 'avg_role_model_std_loss': 0.5779329396736607, 'avg_role_model_mean_pred_loss': 1.1541078821007341e-06, 'avg_role_model_g_mag_loss': 0.016200161819870117, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010268422765705055, 'n_size': 320, 'n_batch': 160, 'duration': 154.49023914337158, 'duration_batch': 0.9655639946460723, 'duration_size': 0.48278199732303617, 'avg_pred_std': 0.08438906156479789}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022567970930140293, 'avg_role_model_std_loss': 1.4275073309931146, 'avg_role_model_mean_pred_loss': 9.043062631357477e-06, 'avg_role_model_g_mag_loss': 0.027866220747819172, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022840781810373302, 'n_size': 80, 'n_batch': 40, 'duration': 33.030266523361206, 'duration_batch': 0.8257566630840302, 'duration_size': 0.4128783315420151, 'avg_pred_std': 0.03308896141534205}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010085937371733245, 'avg_role_model_std_loss': 0.6852384920215103, 'avg_role_model_mean_pred_loss': 1.1549593074543923e-06, 'avg_role_model_g_mag_loss': 0.015572391230125503, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010184062243165926, 'n_size': 320, 'n_batch': 160, 'duration': 151.7514419555664, 'duration_batch': 0.94844651222229, 'duration_size': 0.474223256111145, 'avg_pred_std': 0.06961961404886097}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002278934889363882, 'avg_role_model_std_loss': 2.298216237341036, 'avg_role_model_mean_pred_loss': 8.917010871634101e-06, 'avg_role_model_g_mag_loss': 0.031079518698970788, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002304124432612298, 'n_size': 80, 'n_batch': 40, 'duration': 33.068289279937744, 'duration_batch': 0.8267072319984436, 'duration_size': 0.4133536159992218, 'avg_pred_std': 0.03493260265313438}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009363631818700924, 'avg_role_model_std_loss': 0.4839344189483297, 'avg_role_model_mean_pred_loss': 8.540815376802124e-07, 'avg_role_model_g_mag_loss': 0.01630451331693621, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009456257481076591, 'n_size': 320, 'n_batch': 160, 'duration': 150.87772369384766, 'duration_batch': 0.9429857730865479, 'duration_size': 0.4714928865432739, 'avg_pred_std': 0.07785468016882077}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022760112356991157, 'avg_role_model_std_loss': 2.6171030896344747, 'avg_role_model_mean_pred_loss': 1.0650064827233408e-05, 'avg_role_model_g_mag_loss': 0.029537392102065498, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002303297990329156, 'n_size': 80, 'n_batch': 40, 'duration': 33.033122539520264, 'duration_batch': 0.8258280634880066, 'duration_size': 0.4129140317440033, 'avg_pred_std': 0.03722803041819134}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.000764540546202852, 'avg_role_model_std_loss': 0.44834804190900857, 'avg_role_model_mean_pred_loss': 6.100927646765161e-07, 'avg_role_model_g_mag_loss': 0.013185284686551313, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007720930538482662, 'n_size': 320, 'n_batch': 160, 'duration': 150.77999806404114, 'duration_batch': 0.9423749879002571, 'duration_size': 0.47118749395012854, 'avg_pred_std': 0.08267560889125888}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0020305066434957554, 'avg_role_model_std_loss': 3.3567321322424286, 'avg_role_model_mean_pred_loss': 6.930302250784948e-06, 'avg_role_model_g_mag_loss': 0.030277981684776023, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002052497151453281, 'n_size': 80, 'n_batch': 40, 'duration': 33.12909913063049, 'duration_batch': 0.8282274782657624, 'duration_size': 0.4141137391328812, 'avg_pred_std': 0.033659220521803944}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007402757991229692, 'avg_role_model_std_loss': 0.38544692055954943, 'avg_role_model_mean_pred_loss': 4.2042679632425807e-07, 'avg_role_model_g_mag_loss': 0.013898816978326067, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007476143626305998, 'n_size': 320, 'n_batch': 160, 'duration': 150.68796515464783, 'duration_batch': 0.9417997822165489, 'duration_size': 0.47089989110827446, 'avg_pred_std': 0.08294280422669545}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022146169496409128, 'avg_role_model_std_loss': 1.9662903952322979, 'avg_role_model_mean_pred_loss': 8.018624131983509e-06, 'avg_role_model_g_mag_loss': 0.03200231332739349, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002240737696956785, 'n_size': 80, 'n_batch': 40, 'duration': 33.06134486198425, 'duration_batch': 0.8265336215496063, 'duration_size': 0.41326681077480315, 'avg_pred_std': 0.03342230037087575}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006711353589196279, 'avg_role_model_std_loss': 0.5376870474081945, 'avg_role_model_mean_pred_loss': 3.8460864332618064e-07, 'avg_role_model_g_mag_loss': 0.012289214101656398, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006777132193121815, 'n_size': 320, 'n_batch': 160, 'duration': 150.60209393501282, 'duration_batch': 0.9412630870938301, 'duration_size': 0.47063154354691505, 'avg_pred_std': 0.07368135868346144}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023524648198872456, 'avg_role_model_std_loss': 2.413720256314673, 'avg_role_model_mean_pred_loss': 1.1856669321630431e-05, 'avg_role_model_g_mag_loss': 0.030548772783367893, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002378911971572961, 'n_size': 80, 'n_batch': 40, 'duration': 33.232566833496094, 'duration_batch': 0.8308141708374024, 'duration_size': 0.4154070854187012, 'avg_pred_std': 0.037087300876009976}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006756978137346436, 'avg_role_model_std_loss': 0.7821963908104526, 'avg_role_model_mean_pred_loss': 5.090253047178393e-07, 'avg_role_model_g_mag_loss': 0.013120992203766946, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006825899582267425, 'n_size': 320, 'n_batch': 160, 'duration': 152.36271023750305, 'duration_batch': 0.9522669389843941, 'duration_size': 0.47613346949219704, 'avg_pred_std': 0.07919310781685454}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002482261122668206, 'avg_role_model_std_loss': 1.5217115195275608, 'avg_role_model_mean_pred_loss': 1.1857084663224882e-05, 'avg_role_model_g_mag_loss': 0.03310881347861141, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002508918128592086, 'n_size': 80, 'n_batch': 40, 'duration': 33.57771110534668, 'duration_batch': 0.839442777633667, 'duration_size': 0.4197213888168335, 'avg_pred_std': 0.03645229901885614}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006414469639224763, 'avg_role_model_std_loss': 0.8802176390026426, 'avg_role_model_mean_pred_loss': 3.0969604700309403e-07, 'avg_role_model_g_mag_loss': 0.013044403970798157, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000647848942770679, 'n_size': 320, 'n_batch': 160, 'duration': 151.09337043762207, 'duration_batch': 0.9443335652351379, 'duration_size': 0.47216678261756895, 'avg_pred_std': 0.07918862322630957}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00226858947034998, 'avg_role_model_std_loss': 2.6035211705103833, 'avg_role_model_mean_pred_loss': 9.649373751539902e-06, 'avg_role_model_g_mag_loss': 0.03664247310261999, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022923396401893113, 'n_size': 80, 'n_batch': 40, 'duration': 33.24403095245361, 'duration_batch': 0.8311007738113403, 'duration_size': 0.41555038690567014, 'avg_pred_std': 0.033244564172491664}\n", + "Epoch 19\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.000560577970895082, 'avg_role_model_std_loss': 0.8001867097902629, 'avg_role_model_mean_pred_loss': 2.9982478164879666e-07, 'avg_role_model_g_mag_loss': 0.011377948859171738, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005662478498848599, 'n_size': 320, 'n_batch': 160, 'duration': 151.1276044845581, 'duration_batch': 0.9445475280284882, 'duration_size': 0.4722737640142441, 'avg_pred_std': 0.07509421081031178}\n", + "Time out: 3733.328085899353/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▂▂▂▁▂▁▂▁▁▁▂▂▂▁▂▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▃▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test █▁▁▁▄▁▆▅▅▆▆▆▇█▆▆█▇▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▆▁▁▃▄▅▄▄▇▅██▂▆▇▇▄▆▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test █▂▂▂▁▁▁▂▂▃▂▃▃▃▃▄▃▄▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train █▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▂▂▂▁▂▁▂▁▁▁▂▂▂▁▂▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▃▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▃█▇▅▄▇▂▃▄▃▂▁▃▄▆▂▄▁▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▆█▃▄▅▃▃▂▂▂▂▁▂▁▁▁▁▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▄▅▂▁▁▃▁▁▁▁█▁▁▁▂▁▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▃█▆▅▅▂▂▂▁▁▃▅▃▂▂▂▂▃▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▄▅▂▁▁▃▁▁▁▁█▁▁▁▂▁▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▃█▆▅▅▂▂▂▁▁▃▅▃▂▂▂▂▃▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▄▅▂▁▁▃▁▁▁▁█▁▁▁▂▁▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▃█▆▅▅▂▂▂▁▁▃▅▃▂▂▂▂▃▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00229\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00065\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.03324\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.07919\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.03664\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.01304\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00227\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00064\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 1e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 2.60352\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.88022\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.8311\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 0.94433\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.41555\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.47217\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 33.24403\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 151.09337\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 160\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/contraceptive/tvae/4/wandb/offline-run-20240301_112137-576zsmsp\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240301_112137-576zsmsp/logs\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 399, 'n_batch': 200, 'role_model_metrics': {'avg_loss': 0.0011991993539862264, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.006522898674072502, 'pred_duration': 3.172057628631592, 'grad_duration': 2.5961968898773193, 'total_duration': 5.768254518508911, 'pred_std': 0.05399195849895477, 'std_loss': 0.026110012084245682, 'mean_pred_loss': 7.052161663523293e-07, 'pred_rmse': 0.0346294566988945, 'pred_mae': 0.02745048701763153, 'pred_mape': 0.06274976581335068, 'grad_rmse': 0.03735599294304848, 'grad_mae': 0.02905585803091526, 'grad_mape': 0.5750930905342102}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0011991993539862264, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.006522898674072502, 'avg_pred_duration': 3.172057628631592, 'avg_grad_duration': 2.5961968898773193, 'avg_total_duration': 5.768254518508911, 'avg_pred_std': 0.05399195849895477, 'avg_std_loss': 0.026110012084245682, 'avg_mean_pred_loss': 7.052161663523293e-07}, 'min_metrics': {'avg_loss': 0.0011991993539862264, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.006522898674072502, 'pred_duration': 3.172057628631592, 'grad_duration': 2.5961968898773193, 'total_duration': 5.768254518508911, 'pred_std': 0.05399195849895477, 'std_loss': 0.026110012084245682, 'mean_pred_loss': 7.052161663523293e-07, 'pred_rmse': 0.0346294566988945, 'pred_mae': 0.02745048701763153, 'pred_mape': 0.06274976581335068, 'grad_rmse': 0.03735599294304848, 'grad_mae': 0.02905585803091526, 'grad_mape': 0.5750930905342102}, 'model_metrics': {'tvae': {'avg_loss': 0.0011991993539862264, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.006522898674072502, 'pred_duration': 3.172057628631592, 'grad_duration': 2.5961968898773193, 'total_duration': 5.768254518508911, 'pred_std': 0.05399195849895477, 'std_loss': 0.026110012084245682, 'mean_pred_loss': 7.052161663523293e-07, 'pred_rmse': 0.0346294566988945, 'pred_mae': 0.02745048701763153, 'pred_mape': 0.06274976581335068, 'grad_rmse': 0.03735599294304848, 'grad_mae': 0.02905585803091526, 'grad_mape': 0.5750930905342102}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:26:33.698319Z", + "iopub.status.busy": "2024-03-01T12:26:33.698017Z", + "iopub.status.idle": "2024-03-01T12:26:33.702157Z", + "shell.execute_reply": "2024-03-01T12:26:33.701313Z" + }, + "papermill": { + "duration": 0.025376, + "end_time": "2024-03-01T12:26:33.703997", + "exception": false, + "start_time": "2024-03-01T12:26:33.678621", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:26:33.740641Z", + "iopub.status.busy": "2024-03-01T12:26:33.740328Z", + "iopub.status.idle": "2024-03-01T12:26:34.023630Z", + "shell.execute_reply": "2024-03-01T12:26:34.022747Z" + }, + "papermill": { + "duration": 0.30434, + "end_time": "2024-03-01T12:26:34.026198", + "exception": false, + "start_time": "2024-03-01T12:26:33.721858", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:26:34.065430Z", + "iopub.status.busy": "2024-03-01T12:26:34.065120Z", + "iopub.status.idle": "2024-03-01T12:26:34.337448Z", + "shell.execute_reply": "2024-03-01T12:26:34.336515Z" + }, + "papermill": { + "duration": 0.29489, + "end_time": "2024-03-01T12:26:34.339672", + "exception": false, + "start_time": "2024-03-01T12:26:34.044782", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2RklEQVR4nO3dfVxUZd4/8M+ZZxhghgFhGEQwlTBEKl1YetBa2bDIDdf71ri5k/yZWmlrsaXrbkn12l/0+MvN9a6tXmvbbubDrrX3qllGkruK4gOkprJKKCgOiMgMz/N0/f44zNGBGeaBh2Hg+369zmtmzlxz5pqR+Xidc65zXRxjjIEQQgKEyN8VIIQQb1BoEUICCoUWISSgUGgRQgIKhRYhJKBQaBFCAgqFFiEkoEj8XYGhYrPZUFdXh9DQUHAc5+/qEEJ6YIyhpaUFOp0OIpHr9tSoCa26ujrExcX5uxqEEDdqa2sxduxYl8+PmtAKDQ0FwH8hYWFhfq4NIaQno9GIuLg44bfqyqgJLfsuYVhYGIUWIcOYu8M3dCCeEBJQKLQIIQGFQosQElBGzTEt0j9WqxVms9nf1SABTCqVQiwW93s7FFqkT4wx6PV6NDc3+7sqZARQq9XQarX96itJoUX6ZA+sqKgoBAcHU8dc4hPGGNrb29HQ0AAAiImJ8XlbFFo9GDrMOH3ZCA5A+k0R/q6OX1mtViGwIiJG93dB+i8oKAgA0NDQgKioKJ93FelAfA/lNdfw8PsH8eI/Tvm7Kn5nP4YVHBzs55qQkcL+t9Sf46MUWj1EKOUAgGttJj/XZPigXUIyUAbib4lCq4dwpRQA0NRmAs35QcjwQ6HVg0YpAwCYrDa0max+rg0hpCcKrR6CZRIopPzX0tRKu4jENxzH4fPPP/d3NQbUiy++iFtvvdXf1fAttDZs2ICEhAQoFAqkp6ejrKysz/Lbtm1DUlISFAoFUlJSsGvXLuE5s9mM1atXIyUlBUqlEjqdDgsXLkRdXZ3DNpqampCXl4ewsDCo1WosXrwYra2tvlTfLU0w39pqaqfQIoHro48+glqtHrDtPfvssyguLh6w7fnK69DasmULCgoKUFhYiGPHjiE1NRVZWVlC/4ueDhw4gNzcXCxevBjl5eXIyclBTk4OTp48CQBob2/HsWPH8MILL+DYsWPYvn07Kisr8bOf/cxhO3l5efj++++xZ88e7NixA/v27cPSpUt9+MjuaUK6Q6uta1C2T8hwYjJ59p9zSEjI8Oj6wryUlpbGli9fLjy2Wq1Mp9OxoqIip+Xnz5/PsrOzHdalp6ezZcuWuXyPsrIyBoBduHCBMcbYqVOnGAB2+PBhocwXX3zBOI5jly5d8qjeBoOBAWAGg8Ft2f/+8CCLX72DbTtS69G2R6qOjg526tQp1tHRwRhjzGazsbYus18Wm83mVd2/+OILdueddzKVSsU0Gg3Lzs5m586dY4wxlpGRwVatWuVQvqGhgUkkEvbtt98yxhirq6tjDzzwAFMoFCwhIYF98sknLD4+nr399tsevT8A9tlnnwmPjx8/zu69916mUCiYRqNhS5YsYS0tLcLze/fuZT/60Y9YcHAwU6lU7I477mDnz59njDFWUVHB7rnnHhYSEsJCQ0PZ7bff7vBbcGbv3r0MgMNSWFjIGGMsPj6evfzyy+yRRx5hoaGhLD8/nzHG2KpVq9ikSZNYUFAQGz9+PHv++eeZyWQStllYWMhSU1OFx/n5+eyhhx5ib7zxBtNqtUyj0bAnn3zS4TU99fybupGnv1GvOpeaTCYcPXoUa9asEdaJRCJkZmaitLTU6WtKS0tRUFDgsC4rK6vP/X2DwQCO44SmbWlpKdRqNaZPny6UyczMhEgkwqFDhzB37txe2+jq6kJX1/WWktFo9OQjAgAilNTScqbDbMUta7/0y3ufejkLwTLP/1zb2tpQUFCAqVOnorW1FWvXrsXcuXNRUVGBvLw8vP7663j11VeFU/BbtmyBTqfD3XffDQBYuHAhGhsbUVJSAqlUioKCApd7E57UJSsrCxkZGTh8+DAaGhrw2GOPYcWKFfjoo49gsViQk5ODJUuW4NNPP4XJZEJZWZlQt7y8PNx222149913IRaLUVFRAalU2ud73nHHHVi3bh3Wrl2LyspKAHxLye7NN9/E2rVrUVhYKKwLDQ3FRx99BJ1OhxMnTmDJkiUIDQ3FqlWrXL7P3r17ERMTg7179+LcuXNYsGABbr31VixZssSn78oTXoVWY2MjrFYroqOjHdZHR0fjzJkzTl+j1+udltfr9U7Ld3Z2YvXq1cjNzRUG69Pr9YiKinKsuEQCjUbjcjtFRUV46aWXPPpcPYULoUUXCAeqefPmOTz+4x//iDFjxuDUqVOYP38+nn76afzrX/8SQmrTpk3Izc0Fx3E4c+YMvv76axw+fFj4j/LDDz/EpEmTfKrLpk2b0NnZiY8//hhKpRIA8Pvf/x5z5szBa6+9BqlUCoPBgAcffBATJkwAAEyePFl4fU1NDZ577jkkJSUBgEf1kMlkUKlU4DgOWq221/M/+clP8Mtf/tJh3fPPPy/cT0hIwLPPPovNmzf3GVrh4eH4/e9/D7FYjKSkJGRnZ6O4uHj4hNZgM5vNmD9/PhhjePfdd/u1rTVr1ji08OxDuXrC3tKiDqaOgqRinHo5y2/v7Y2zZ89i7dq1OHToEBobG2Gz2QDwATBlyhTcd999+OSTT3D33XejuroapaWl+MMf/gAAqKyshEQiwe233y5sb+LEiQgPD/ep7qdPn0ZqaqoQWABw5513wmazobKyEjNmzMCjjz6KrKws/PSnP0VmZibmz58vXJ9XUFCAxx57DH/+85+RmZmJ//zP/xTCzVc37rXYbdmyBe+88w6qqqrQ2toKi8XidpTf5ORkh8txYmJicOLEiX7VzR2vDsRHRkZCLBajvr7eYX19fb3TNAcArVbrUXl7YF24cAF79uxx+LK0Wm2vprnFYkFTU5PL95XL5cLQyt4OsWxvaV2l0HLAcRyCZRK/LN72pJ4zZw6amprwwQcf4NChQzh06BCA6wed8/Ly8Ne//hVmsxmbNm1CSkoKUlJSBvw789TGjRtRWlqKO+64A1u2bEFiYiIOHjwIgO9q8P333yM7OxvffPMNbrnlFnz22Wf9er8bAxTgD8Hk5eXhgQcewI4dO1BeXo7f/OY3bg/S99xN5ThO+A9isHgVWjKZDNOmTXM47Wmz2VBcXIyMjAynr8nIyOh1mnTPnj0O5e2BdfbsWXz99de9zlBkZGSgubkZR48eFdZ98803sNlsSE9P9+YjeMTe5eEadXkISFevXkVlZSWef/55zJo1C5MnT8a1a9ccyjz00EPo7OzE7t27sWnTJuTl5QnP3XzzzbBYLCgvLxfWnTt3rtc2PDV58mR89913aGtrE9bt378fIpEIN998s7Dutttuw5o1a3DgwAFMmTIFmzZtEp5LTEzEM888g6+++go///nPsXHjRrfvK5PJYLV61kH6wIEDiI+Px29+8xtMnz4dkyZNwoULF7z4lEPH6y4PBQUF+OCDD/CnP/0Jp0+fxhNPPIG2tjYsWrQIAH8A88YD9StXrsTu3bvx1ltv4cyZM3jxxRdx5MgRrFixAgAfWP/xH/+BI0eO4JNPPoHVaoVer4derxdSfvLkyZg9ezaWLFmCsrIy7N+/HytWrMDDDz8MnU43EN+DA41wTItCKxCFh4cjIiIC77//Ps6dO4dvvvmm18kgpVKJnJwcvPDCCzh9+jRyc3OF55KSkpCZmYmlS5eirKwM5eXlWLp0KYKCgny6di4vLw8KhQL5+fk4efIk9u7di6eeegqPPPIIoqOjUV1djTVr1qC0tBQXLlzAV199hbNnz2Ly5Mno6OjAihUrUFJSggsXLmD//v04fPiwwzEvVxISEtDa2ori4mI0Njaivb3dZdlJkyahpqYGmzdvRlVVFd55551+t+YGTZ/nFl1Yv349GzduHJPJZCwtLY0dPHhQeG7mzJnCKVS7rVu3ssTERCaTyVhycjLbuXOn8Fx1dXWvU7P2Ze/evUK5q1evstzcXBYSEsLCwsLYokWLHE4Zu+NNl4d/640sfvUONvXFLz3e/kjU1+np4W7Pnj1s8uTJTC6Xs6lTp7KSkpJe3RB27drFALAZM2b0en1dXR27//77mVwuZ/Hx8WzTpk0sKiqKvffeex69f8/36qvLg16vZzk5OSwmJobJZDIWHx/P1q5dy6xWK+vq6mIPP/wwi4uLYzKZjOl0OrZixQqP/00ef/xxFhER0avLg7OuG8899xyLiIhgISEhbMGCBeztt99mKpVKeN5Vl4cbrVy5ks2cOdNlfQaiywPH2Oi4KthoNEKlUsFgMLg9vnW1tQvTfvs1AODs/70fUvHovNqps7MT1dXVGD9+PBQKhb+r41cXL15EXFwcvv76a8yaNcvf1QlYff1NefobHVZnD4cLdbAMHAcwBjS3mzEmVO7vKpEh9s0336C1tRUpKSm4fPkyVq1ahYSEBMyYMcPfVRv1RmcTwg2xiIM66PoQNWT0MZvN+PWvf43k5GTMnTsXY8aMETqafvLJJwgJCXG6JCcnD1kd77//fpf1eOWVV4asHkONWlouhCtluNZuptAapbKyspCV5bxP2s9+9jOXZ63d9VQfSB9++CE6OjqcPqfRaIasHkONQsuFCKUMP1xpo24PpJfQ0FCEhob6uxqIjY31dxX8gnYPXQgPpg6mhAxHFFouRITQpTyEDEcUWi7YW1p0TIuQ4YVCywXqFU/I8ESh5QKFFiHDE4WWC+EUWqQfRuLEFsMFhZYLERRaJMAN9MQWAFBSUgKO49Dc3Dyg2/UGhZYL4TfMyDNKLs8kJCBQaLlg7/JgstjQTpO28hgDTG3+Wbz8j2P37t246667oFarERERgQcffBBVVVUA+PHTV69e7VD+ypUrkEql2LdvHwDg8uXLyM7ORlBQEMaPH49NmzYhISEB69at8+mrO3HiBH7yk58gKCgIERERWLp0qcMUeCUlJUhLS4NSqYRarcadd94pjGf13Xff4d5770VoaCjCwsIwbdo0HDlypM/3KykpwaJFi4T5FjiOw4svvgiAnz/h2WefRWxsLJRKJdLT01FSUiK89sKFC5gzZw7Cw8OhVCqRnJyMXbt24fz587j33nsB8MP/cByHRx991Kfvoz+oR7wLQVIx5BIRuiw2NLWZoJTTVwVzO/DKwI9f5pFf1wEypfty3WhiC9cTW6xYsQKnTp3C5s2bodPp8Nlnn2H27Nk4ceIEJk2ahOXLl8NkMmHfvn1QKpU4deoUQkJCEBcXh7/97W+YN28eKisrERYWhqCgIJ++k/6gX6ILHMchQilDnaETTW0mxGmC/V0l4gWa2ML5xBY1NTXYuHEjampqhAE0n332WezevRsbN27EK6+8gpqaGsybN08Yfvqmm24SXm+/pjEqKmrAj5d5ikKrD+E3hBYBIA3mWzz+em8v0MQWzp04cQJWqxWJiYkO67u6uoRhzn/xi1/giSeewFdffYXMzEzMmzcPU6dO9en9BgMd0+oD9dXqgeP4XTR/LDSxxYBMbNHa2gqxWIyjR4+ioqJCWE6fPo3f/e53AIDHHnsMP/zwAx555BGcOHEC06dPx/r16wfss/YXhVYfKLQCE01swXM2scVtt90Gq9WKhoYGTJw40WG5cTcyLi4Ojz/+OLZv345f/vKX+OCDD4RtAvB4wozBQKHVhxu7PZDAQRNb8JxNbJGYmIi8vDwsXLgQ27dvR3V1NcrKylBUVISdO3cCAJ5++ml8+eWXqK6uxrFjx7B3717h/eLj48FxHHbs2IErV644nAEdMn2OID+CeDOxhd07X/+bxa/ewVb/9btBrNnwRRNbjMyJLUwmE1u7di1LSEhgUqmUxcTEsLlz57Ljx48zxhhbsWIFmzBhApPL5WzMmDHskUceYY2NjcI2X375ZabVahnHcb0msXGHJrbwgjcTW9j95eAFPP/5Sfz0lmh8sLD3jLwjHU1scR1NbDEwaGKLQWa/lIfG1Bp9aGKL4YuOafWBLpoevWhii+GLWlp9EC6apgPxow5NbDF8UWj1wd7Sam43w2K1QTJKJ20ljmhiC/+iX2Ef7HMfAkBzh9mPNfGvUXKuhgyBgfhbotDqg0Qsgjp49E7aat/VaW9v93NNyEhh/1vqz2407R66oQmWoXmUTtoqFouhVquF0Q2Cg4N96lxJCGMM7e3taGhogFqthlgs9nlbFFpuaJQy/NDYNmq7Pdgv7fB1WBZCbqRWqx0uF/IFhZYb9oPxo3XSVo7jEBMTg6ioKJjNo/e4Huk/qVTarxaWHYWWG9TBlCcWiwfkD46Q/qID8W6M9pYWIcMNhZYbQkuLOpgSMixQaLkhDE9DLS1ChgUKLTc0IRRahAwnFFpuaKilRciwQqHlxo1DLtPlLIT4H4WWG/bQ6rLY0GGmSVsJ8TcKLTeCZWLIJPzXdLWVdhEJ8TcKLTfsk7YC1O2BkOGAQssD9m4P1MGUEP/zKbQ2bNiAhIQEKBQKpKeno6ysrM/y27ZtQ1JSEhQKBVJSUrBr1y6H57dv34777rsPERER4DgOFRUVvbZxzz33gOM4h+Xxxx/3pfpeiwihS3kIGS68Dq0tW7agoKAAhYWFOHbsGFJTU5GVleVyFIADBw4gNzcXixcvRnl5OXJycpCTk4OTJ08KZdra2nDXXXfhtdde6/O9lyxZgsuXLwvL66+/7m31fUIdTAkZRryatIwxlpaWxpYvXy48tlqtTKfTsaKiIqfl58+fz7Kzsx3Wpaens2XLlvUqW11dzQCw8vLyXs/NnDmTrVy50tvqCnyZ99Cu8O8nWfzqHey1L077/P6EkL55+hv1qqVlMplw9OhRZGZmCutEIhEyMzNRWlrq9DWlpaUO5QF+0gBX5fvyySefIDIyElOmTMGaNWv6HFGzq6sLRqPRYfGVhmblIWTY8GpomsbGRlitVkRHRzusj46OxpkzZ5y+Rq/XOy2v1+u9quh//dd/IT4+HjqdDsePH8fq1atRWVmJ7du3Oy1fVFSEl156yav3cIWmEiNk+AiY8bSWLl0q3E9JSUFMTAxmzZqFqqoqTJgwoVf5NWvWoKCgQHhsNBoRFxfn03tTlwdChg+vQisyMhJisRj19fUO6+vr610OoarVar0q7yn7vHPnzp1zGlpyuRxyubxf72FHXR4IGT68OqYlk8kwbdo0FBcXC+tsNhuKi4uRkZHh9DUZGRkO5QFgz549Lst7yt4tIiYmpl/b8QR1eSBk+PB697CgoAD5+fmYPn060tLSsG7dOrS1tWHRokUAgIULFyI2NhZFRUUAgJUrV2LmzJl46623kJ2djc2bN+PIkSN4//33hW02NTWhpqYGdXV1AIDKykoAfCtNq9WiqqoKmzZtwgMPPICIiAgcP34czzzzDGbMmIGpU6f2+0twx97Sau4ww2pjEItoRhpC/MaXU5Pr169n48aNYzKZjKWlpbGDBw8Kz82cOZPl5+c7lN+6dStLTExkMpmMJScns507dzo8v3HjRgag11JYWMgYY6ympobNmDGDaTQaJpfL2cSJE9lzzz3nVfeF/nR5MFusLH71Dha/egdrbOn0+vWEEPc8/Y1yjI2O8VaMRiNUKhUMBgPCwsK8fn3qS1/B0GHGnmdmYFK0/6dEJ2Sk8fQ3StceeiiCuj0QMixQaHmI+moRMjxQaHlIuP6Q+moR4lcUWh6iSVsJGR4otDxEk7YSMjxQaHmIWlqEDA8UWh6ilhYhwwOFlofoomlChgcKLQ8JXR5oRh5C/IpCy0NC51JqaRHiVxRaHrK3tDrNNrSbLH6uDSGjF4WWh5QyMWRi/uuiXvGE+A+Floc4jhPGir/WZvZzbQgZvSi0vHC920OXn2tCyOhFoeUF6vZAiP9RaHlBaGlRtwdC/IZCywvU0iLE/yi0vCAMT0NnDwnxGwotL2hCKLQI8TcKLS9oqKVFiN9RaHlBQ0MuE+J3FFpeEDqXtlPnUkL8hULLC+FKKQD+7KHVNipmXiNk2KHQ8oL97CFjgKGDWluE+AOFlhekYhHCFBIAQBNdykOIX1BoeSkiRA4AaKKLpgnxCwotL4UH88e1qKVFiH9QaHlJo6SWFiH+RKHlJY2SWlqE+BOFlpeopUWIf1FoeUlzQ18tQsjQo9Dykr2vFk3aSoh/UGh5KSLEPk48hRYh/kCh5SUaU4sQ/6LQ8lKEcCCeQosQf6DQ8pL9oukOsxUdJqufa0PI6EOh5aUQueT6pK10BpGQIUeh5SWO44TWVhPNykPIkKPQ8oHQwZRaWoQMOQotHwgdTOlgPCFDjkLLB9TBlBD/8Sm0NmzYgISEBCgUCqSnp6OsrKzP8tu2bUNSUhIUCgVSUlKwa9cuh+e3b9+O++67DxEREeA4DhUVFb220dnZieXLlyMiIgIhISGYN28e6uvrfal+vwmTtlJoETLkvA6tLVu2oKCgAIWFhTh27BhSU1ORlZWFhoYGp+UPHDiA3NxcLF68GOXl5cjJyUFOTg5OnjwplGlra8Ndd92F1157zeX7PvPMM/jHP/6Bbdu24dtvv0VdXR1+/vOfe1v9ARGupJYWIX7DvJSWlsaWL18uPLZarUyn07GioiKn5efPn8+ys7Md1qWnp7Nly5b1KltdXc0AsPLycof1zc3NTCqVsm3btgnrTp8+zQCw0tJSj+ptMBgYAGYwGDwq35ePD1Sz+NU72LKPj/R7W4QQnqe/Ua9aWiaTCUePHkVmZqawTiQSITMzE6WlpU5fU1pa6lAeALKyslyWd+bo0aMwm80O20lKSsK4ceNcbqerqwtGo9FhGSjhNP8hIX7jVWg1NjbCarUiOjraYX10dDT0er3T1+j1eq/Ku9qGTCaDWq32eDtFRUVQqVTCEhcX5/H7uSNM2kpdHggZciP27OGaNWtgMBiEpba2dsC2TTNNE+I/Em8KR0ZGQiwW9zprV19fD61W6/Q1Wq3Wq/KutmEymdDc3OzQ2uprO3K5HHK53OP38IY9tJq7J20Vi7hBeR9CSG9etbRkMhmmTZuG4uJiYZ3NZkNxcTEyMjKcviYjI8OhPADs2bPHZXlnpk2bBqlU6rCdyspK1NTUeLWdgWLvp2VjgJEmbSVkSHnV0gKAgoIC5OfnY/r06UhLS8O6devQ1taGRYsWAQAWLlyI2NhYFBUVAQBWrlyJmTNn4q233kJ2djY2b96MI0eO4P333xe22dTUhJqaGtTV1QHgAwngW1harRYqlQqLFy9GQUEBNBoNwsLC8NRTTyEjIwM//vGP+/0leMs+aaux04KrbSbhwDwhZAj4cmpy/fr1bNy4cUwmk7G0tDR28OBB4bmZM2ey/Px8h/Jbt25liYmJTCaTseTkZLZz506H5zdu3MgA9FoKCwuFMh0dHezJJ59k4eHhLDg4mM2dO5ddvnzZ4zoPZJcHxhib+fo3LH71DlZWfXVAtkfIaOfpb5RjjDE/ZuaQMRqNUKlUMBgMCAsL6/f25v7PfpTXNOO9/56G2VM8Pz5HCHHO09/oiD17ONiES3mo2wMhQ4pCy0c0Vjwh/kGh5SNNCIUWIf5AoeUjDbW0CPELCi0fUa94QvyDQstHGjoQT4hfUGj5yB5aV2lyC0KGFIWWj6ilRYh/UGj5yH7pTrvJik4zTdpKyFCh0PJRqFwCqZgf3YEOxhMydCi0fMRxHHUwJcQPKLT6gbo9EDL0KLT6gUKLkKFHodUPFFqEDD0KrZ5qDgJ/mgNsX+q2KHV7IGToeT1y6cjHAdX7AGUUwBjAuR7/XUOTthIy5Kil1VPMVEAkBdoagOaaPosKLS0KLUKGDIVWT9IgQJvC3794uM+i9i4P1NIiZOhQaDkTl8bfugmtCGppETLkKLScGfsj/tZdS4vOHhIy5Ci0nBk7nb+9fBwwd7osduM48TbbqJgfhBC/o9ByRh0PKMcANjOgP+662A2Tthpo0lZChgSFljMc59EuokwiQqiC7zXSRH21CBkSFFqu2HcR3RzXom4PhAwtCi1XhJbWkT6LUQdTQoYWhZYrutsBTgQYagHjZZfF7LPyUEuLkKFBoeWKPASISubv97GLSC0tQoYWhVZfPDiuZQ+tKy1dQ1EjQkY9Cq2+eHBc6xZdGADgn2evDEWNCBn1KLT6Yg+tunLA6rwf1r1JUZCKOVRdacO5htYhrBwhoxOFVl8iJgIKFWDpAOq/d1okTCFFxoRIAMCX3+uHsnaEjEoUWn0RiYBY98e1spKjAQBfUWgRMugotNwRRnxwfVzrp7dEg+OA7y4acNnQMUQVI2R0otByRziDWOaySFSoAtPGhQMAvvq+fihqRcioRaHlTuw0/rbpB6DtqstiWclaAHRci5DBRqHlTlA4EJnI37/kehfRHlqHqpuodzwhg4hCyxMejPgwLiIYSdpQWG0MxWcahqhihIw+FFqe8HDEB9pFJGTwUWh5QmhpHQVsVpfF7KG1799X0G6yDEXNCBl1KLQ8EXULIFUCphag8d8ui02OCUWcJghdFhu+raTLeggZDBRanhCJgdjb+fu1rrs+cByHrFtoF5GQwUSh5SkPZ+jJmsKHVvGZBpgstsGuFSGjjk+htWHDBiQkJEChUCA9PR1lZa5bHwCwbds2JCUlQaFQICUlBbt27XJ4njGGtWvXIiYmBkFBQcjMzMTZs2cdyiQkJIDjOIfl1Vdf9aX6vvFwJNPbx4UjMkSGlk4LDv7gul8XIcQ3XofWli1bUFBQgMLCQhw7dgypqanIyspCQ4Pz0/wHDhxAbm4uFi9ejPLycuTk5CAnJwcnT54Uyrz++ut455138N577+HQoUNQKpXIyspCZ6fj9F0vv/wyLl++LCxPPfWUt9X3nf0M4pUzQKfBZTGxiMNPb+GvRaRdREIGAfNSWloaW758ufDYarUynU7HioqKnJafP38+y87OdliXnp7Oli1bxhhjzGazMa1Wy9544w3h+ebmZiaXy9mnn34qrIuPj2dvv/22x/Xs7OxkBoNBWGpraxkAZjAYPN5GL2+nMFYYxti5b/os9s2Zeha/egf70W/3MKvV5vv7ETKKGAwGj36jXrW0TCYTjh49iszMTGGdSCRCZmYmSktLnb6mtLTUoTwAZGVlCeWrq6uh1+sdyqhUKqSnp/fa5quvvoqIiAjcdttteOONN2CxuO5WUFRUBJVKJSxxcXHefFTnPNxFvGNCBELkEjS0dKG8trn/70sIEXgVWo2NjbBarYiOjnZYHx0dDb3e+a6QXq/vs7z91t02f/GLX2Dz5s3Yu3cvli1bhldeeQWrVq1yWdc1a9bAYDAIS21trecf1BUPD8bLJWLcmxQFgIarIWSgSfxdAU8VFBQI96dOnQqZTIZly5ahqKgIcrm8V3m5XO50fb/E3RBajPGTurowO1mLf3xXhy+/1+NX9yeB66MsIcRzXrW0IiMjIRaLUV/vOPxKfX09tFqt09dotdo+y9tvvdkmAKSnp8NiseD8+fPefIT+iU4BxHKgo4kf9aEP99w8BjKJCOevtuPf9TQMMyEDxavQkslkmDZtGoqLi4V1NpsNxcXFyMjIcPqajIwMh/IAsGfPHqH8+PHjodVqHcoYjUYcOnTI5TYBoKKiAiKRCFFRUd58hP6RyADdrfx9N7uISrkEd0+kYZgJGXDeHuHfvHkzk8vl7KOPPmKnTp1iS5cuZWq1mun1esYYY4888gj71a9+JZTfv38/k0gk7M0332SnT59mhYWFTCqVshMnTghlXn31VaZWq9nf//53dvz4cfbQQw+x8ePHs46ODsYYYwcOHGBvv/02q6ioYFVVVewvf/kLGzNmDFu4cKHH9fb0zIRbu3/Nn0HcUeC26JayGha/egd74Hf7+veehIwCnv5GvQ4txhhbv349GzduHJPJZCwtLY0dPHhQeG7mzJksPz/fofzWrVtZYmIik8lkLDk5me3cudPheZvNxl544QUWHR3N5HI5mzVrFqusrBSeP3r0KEtPT2cqlYopFAo2efJk9sorr7DOzk6P6zxgoXVyOx9a793ttmhjSycb/6sdLH71DlZzta1/70vICOfpb5RjjDH/tvWGhtFohEqlgsFgQFhYmO8bMlwE3k4GODGw5iIgC+6z+II/lOJQdRNeePAWLL5rvO/vS8gI5+lvlK499FZYLBAaAzArcLnCbXFhjK2TdFyLkIFAoeUtjvO4vxYA3Nc9vdjhC01obO0azJoRMipQaPnCHlp9DFMjFA0PxpTYMDAGfH2KZuohpL8otHwxtkcnUzdojC1CBg6Fli9iUgGRBGit5w/Mu2EfY2v/uato6TQPdu0IGdEotHwhCwaip/D3PTiuNSkqBOMjlTBZbSihYZgJ6RcKLV95OOIDwA/DbD8gT7uIhPQPhZavvDiDCFzv+lBSeQVdFtcz+hBC+kah5Sv7iA+XvwMs7rsy3DpWjegwOVq7LDhwjoZhJsRXFFq+Ch8PBEcA1i5Af9JtcZGIw310FpGQfqPQ8pVDJ1P3/bWA67uIe07Vw2obFVdPETLgKLT6wz7ZhYfHtdJv0kAVJMXVNhPe+qoSFitNMUaItyi0+sPLg/FSsQiPdV80/T8lVVjw/kHUNrUPVu0IGZEotPpDdzsADmiuAVo8uESHMTx1lxbvzh0HlVyEoxeu4YHf/RN/r7g06FUlZKQImDHihyVFGBA1GWg4BRz+AFCPA9qbgI5r/JDM9vvtTfzjjmuA1YT7AcwWidAcrMJlSxga/xaGw3tjkJp0M2SqaCAkGlCOAUKi+PtBGkBE/78QAlBo9d/YH/Ghte8Nr17GMRvC2TWEi67xKwwngENfOS8cNhbIeBK4PR+Qh/SzwoQENgqt/pr+fwD9CYATAcEavlUUrAGCwvnFYV33rVgGtDUCbQ1A6xX8cP4H7D50HPLORozhDLgtwoyx0hZwrQ18C814Efjy13wwpi0D0pfx2yFkFKKRS4cJQ4cZz39+Ev/4rg4AkDZeg3ULboVOCeD4VmD/uuszAEmDgWmPAhkrAFWsv6pMRjpzB2CsAwy1gOESPziA8SLQ0QxEJgIxU/nBA9TxfU6n5ylPf6MUWsMIYwzbj13C2r+fRJvJijCFBK/Om4oHUmIAmxU4/b/AP/8foD/Ov0AkBaYuAO5cCYxJ9G/lSWBhDGi7wp9Eaq7pDqTuYLKHVHujZ9tSqADtVH6JSeXDLGISIPZuR45Cq4dACC27841tWLm5HN9dNAAA5t4Wi5WzJiEhUsn/sVV9A/zrbeD8P7tfwQGTHwTuegaInea4McaA9qvdf5i1QHPt9T/M5hqg0wAk5wB3rASUEQP7QTqagS4jP0S1SDyw2x5M7U3ApWP8j9pmBqxm/j8NmxmwWbofW3rctwLo+VPq0fq4sTUilvI/8rE/4k/gDPRkvjYb0KrvDqVaoPnC9X/z5lr+vqXT/XakSkA1lm/Rq8YCqjhAFsIfx9UfBxpOA1ZT79dJFPxIKDHdYXbzA0BodO9yN6DQ6iGQQgsAzFYb3t7zb7z7bRUYA0Qc8OBUHZ68dwKStN31rz3M7zae2XH9heNn8JcY3RhQlg73bygL4Y+VZazo//Gy+u+B0v8BTmzl/6BFUkAdB4QnXF/U8dfvB6n79379Ye4ALh8HLh29vlyrHto6hETz4RWXBoxN4+fWlAZ59lpLF9B4Frhyhl8aTvO31y7wIdsnDgjT8UGkGtt7CYvlj8v2FagWE/9++uP893j5O6D+JGDqMUHxot1AvOt5TAEKrV4CLbTsjtVcwzvFZx3G4cqcHIUn752I28eF8ysazgD7f8eHhM3ifEMhWj44VHHXb1Vx/LWT/3yL/2MDAFkof6byx096FyaMAeeKgdLfAz/svb6eE/OTgPRFoe4OsHj+JIXVxLdghFv7/R7rmZU/uREyhu8i0nO5cb1EzreGGv/tGFD13zv/ziIm8sEqlvIDPtoXsZRvNYqkPZ4T8ydjnH0vznS18O+vP977/UVSQJvCh5g9yEKigKvnrodSw2ngSiV/nNPV98uJu1tI4/jWnDqOv1V134bF8hMQDzSbja/X5YrrYbbgz4A8tM+XUWj1EKihZXfykgHvllRh18nLwu8g46YILL93Iu6cGAGO4/iWVcUn/A/lxoAKi+V/tK4wBlTuAvYWAfUn+HVyFXDHCiD9cb4/mivmDuD4FuDgu/yPCeB/vJN/BmQs53dXjXXAtfP8Lsq1845L2xANiihX8T/uni0AAFBG8Zdkxd7O11d3G9/CGArmDqCugr9+tbaMv7qi1VlHZQ69dz+7yVVAVBIwJonvNzjmZj50Q3VeH1fyJwqtHgI9tOyqrrTivZIqfFZ+CZbui65Tx6rw5L0T8dPJ0RCJ+nFsxGYDzvyDD68rp/l1CjVwx1P8ruON/1O2NgCHP+SX9u6hdmShwO0L+bLh8Z69Z1fr9TBrrgGYjW9tiaX8rb1FI5Y5rhfL+N9x+zW+60jbFX5pvXL9vn25sSUjVfKhZA+o2Gn8rtBAH1PyFWP891F7+HqQ6U/wgSsP6w6mJGBMdzhFTeantBsu9e8HCq0eRkpo2V1q7sAH+37A5sM16DTzF15PigrBE/dMwOwpWgTL+vE/rM0GnPoMKHmV350C+N2wO1cC4+8GjvyR74ZhPwCrGgf8+HHgtkf6bpX5A2NAZzMfZswGRE4KrJMCAGBq509ohESPiHByhUKrh5EWWnaNrV3YuL8aHx+4gJau6y2KCKUMYzXBiAsPwtjwYMRpghAXHow4TTB0agXkEg9+uDYrcHI7UFIENFX1fj52Or8LmTQnoHZDyPBEodXDSA0tO2OnGX8uvYCPS8+j3tj3SKocB0SHKhCn4QMtOkyBqFA5fxsmR3Qof6uQdgeb1QKc2AZ8+yq/Czd5Dn+WMS5tCD4ZGS0otHoY6aF1I0OHGRevtaO2qQMXr7Xj4rUO1Da1o7Z7XYfZszHqwxQShyAbEypFtFKCoKAgBMvEUMokCJbzt0q5GMEyCZRyCYJlYkjFdIE38Y6nv1Fq049AqiApVEEqJOtUvZ5jjKGpzYTa7iC7eK0DDS2daDB2oaGlE/XGLtQbO9FlscHYaYGxsxVnG5yccXNDJhFBKRNDqwrC1FgVpsapkDpWjZu1oRRopF+opUV6YYzB2GlBg7ETDS18iNlvm9pMaOuyot1kQZvJivYuC9pNVrR2WdDWZRHOaLoik4hwS0wYUseqMHWsGqlxKtwUGdK/s55kRKCWFvEZx3HdrTUpJkX33SGwJ5PFJgRaW5cFP1xpw/GLzTh+0YDjF5th7LSgorYZFbXNAC4AAJQyMabEqpAap0ZUqBxyqRhyiQgKqRiK7lvhsVQMhVQEuUSMYLkYoXIJ30eNjBrU0iJDhjGG81fbcfxiM76rNeDEpWacvGT0+BibM6FyCcZqgjFOE4RxmmCM0/BnSOM0wRgbHuTZWVIyLNCB+B4otIYni9WGc1dacbzWgJN1Bhg6zOg0W9FlsaHTbEWnmb812R9bbOjqvnU3oxHHAdowBR9i4cHQquSQS/iTBDKJCDIxB5lEJDyWikWQ3XBfIubAARBxHEQcB47rvi+yr+Nbpfb7SrkEYQopZBI6ZucLCq0eKLRGng6TFZea21HT1I6aq+2oaeroPkPKr2s3+Wcm7yCpGKogKcKCJPytQtr9mF9UQVKog6RIiAzGTZEhCFcOwvV/AYiOaZERL0gmxsSoUEyM6n3cjTGGq20m1DTxIVbb1I4rLV0wWRnMVhtMFptwa7Lablh3/XmLjcHGGBiDw62N8du337cxBquNCSHZYbaiw2yF3ujZ59AoZZgwRombIkMwIcp+G4K48CBI6ExrLxRaZETiOA6RIXJEhsivj4YxyKw2htZOCwwdZhg7zfxth9nhMb/OgqttXai+0oY6A39GtqnNhMPnrzlsTyrmEB+hxIQxSujUQd27tCJIRCJIJRykIhGkYg6S7t1aqYTjnxOLECwT8y07hQRhQVKEKiQj5vgehRYhA0Qs4qAKlkIVLPX4Ne0m/gxr1ZVWh9sfGlvRabbhXEMrzvnQT84ZhVSEMIVUCLEb74s5Dgx8a5LBPqKOYyvTvp7j+K4rcgl/FlcuEV1/3H2mV3hOyt9PiVUhVOH599IXCi1C/ChYJsGUWBWmxDp2BLbZGC4bO/HDlVZUNbSivqULFuv13Vezm/vtJiuM3S28lk7+mlT+pEYXGlr6vsxrMPzvijsxdax6QLZFoUXIMCQScYhVByFWHYS7J43p17asNobWLosQYsYOS/ctH2jGTjNsjB/ph+MADvYzpRD6wN24njF+ZN0uixVdZhu6LN33Lbbux/x9k+X6c0r5wEUNhRYhI5xYdL2z8EhApyYIIQHFp9DasGEDEhISoFAokJ6ejrKysj7Lb9u2DUlJSVAoFEhJScGuXbscnmeMYe3atYiJiUFQUBAyMzNx9uxZhzJNTU3Iy8tDWFgY1Go1Fi9ejNbWgTlASQgJHF6H1pYtW1BQUIDCwkIcO3YMqampyMrKQkNDg9PyBw4cQG5uLhYvXozy8nLk5OQgJycHJ0+eFMq8/vrreOedd/Dee+/h0KFDUCqVyMrKQmfn9SmO8vLy8P3332PPnj3YsWMH9u3bh6VLl/rwkQkhAY15KS0tjS1fvlx4bLVamU6nY0VFRU7Lz58/n2VnZzusS09PZ8uWLWOMMWaz2ZhWq2VvvPGG8HxzczOTy+Xs008/ZYwxdurUKQaAHT58WCjzxRdfMI7j2KVLlzyqt8FgYACYwWDw7IMSQoaUp79Rr1paJpMJR48eRWZmprBOJBIhMzMTpaWlTl9TWlrqUB4AsrKyhPLV1dXQ6/UOZVQqFdLT04UypaWlUKvVmD59ulAmMzMTIpEIhw4dcvq+XV1dMBqNDgshJPB5FVqNjY2wWq2IjnacKTY6Ohp6vd7pa/R6fZ/l7bfuykRFRTk8L5FIoNFoXL5vUVERVCqVsMTFxXn4KQkhw9mIPXu4Zs0aGAwGYamtrfV3lQghA8CrflqRkZEQi8Wor3ecTLK+vh5ardbpa7RabZ/l7bf19fWIiYlxKHPrrbcKZXoe6LdYLGhqanL5vnK5HHL59QlKWfdgFrSbSMjwZP9tMncDz3h7sCwtLY2tWLFCeGy1WllsbGyfB+IffPBBh3UZGRm9DsS/+eabDgfknB2IP3LkiFDmyy+/9OpAfG1tLUP35VO00ELL8F1qa2v7/C173SO+oKAA+fn5mD59OtLS0rBu3Tq0tbVh0aJFAICFCxciNjYWRUVFAICVK1di5syZeOutt5CdnY3NmzfjyJEjeP/99wHwlwk8/fTT+O1vf4tJkyZh/PjxeOGFF6DT6ZCTkwMAmDx5MmbPno0lS5bgvffeg9lsxooVK/Dwww9Dp9N5VG+dTofa2lqEhoa6HZ7XaDQiLi4OtbW1I37srdHyWUfL5wQC97MyxtDS0uL+N+1RM6WH9evXs3HjxjGZTMbS0tLYwYMHhedmzpzJ8vPzHcpv3bqVJSYmMplMxpKTk9nOnTsdnrfZbOyFF15g0dHRTC6Xs1mzZrHKykqHMlevXmW5ubksJCSEhYWFsUWLFrGWlhZfqu/WaOoeMVo+62j5nIyN/M86akYu9cZoGuV0tHzW0fI5gZH/WUfs2UNCyMhEoeWEXC5HYWGhw9nHkWq0fNbR8jmBkf9ZafeQEBJQqKVFCAkoFFqEkIBCoUUICSgUWoSQgEKhRQgJKBRaPXg7lHQgevHFF8FxnMOSlJTk72oNiH379mHOnDnQ6XTgOA6ff/65w/PMg6G9A4W7z/roo4/2+neePXu2fyo7gCi0buDtUNKBLDk5GZcvXxaWf/3rX/6u0oBoa2tDamoqNmzY4PR5T4b2DhTuPisAzJ492+Hf+dNPPx3CGg4Sv15ENMx4O5R0oCosLGSpqan+rsagA8A+++wz4bEnQ3sHqp6flTHG8vPz2UMPPeSX+gwmaml182Uo6UB29uxZ6HQ63HTTTcjLy0NNTY2/qzToPBnae6QpKSlBVFQUbr75ZjzxxBO4evWqv6vUbxRa3XwZSjpQpaen46OPPsLu3bvx7rvvorq6GnfffTdaWlr8XbVB5cnQ3iPJ7Nmz8fHHH6O4uBivvfYavv32W9x///2wWq3+rlq/0AzTo9D9998v3J86dSrS09MRHx+PrVu3YvHixX6sGRlIDz/8sHA/JSUFU6dOxYQJE1BSUoJZs2b5sWb9Qy2tbr4MJT1SqNVqJCYm4ty5c/6uyqC6cWjvG42Gf2MAuOmmmxAZGRnw/84UWt1kMhmmTZuG4uJiYZ3NZkNxcTEyMjL8WLPB19raiqqqKocx+kei8ePHQ6vVOvwbG41GHDp0aMT/GwPAxYsXcfXq1YD/d6bdwxu4G0p6pHj22WcxZ84cxMfHo66uDoWFhRCLxcjNzfV31fqttbXVoSVRXV2NiooKaDQajBs3zu3Q3oGkr8+q0Wjw0ksvYd68edBqtaiqqsKqVaswceJEZGVl+bHWA8Dfpy+Hm76Gkh4pFixYwGJiYphMJmOxsbFswYIF7Ny5c/6u1oDYu3ev08kS7EOAezK0d6Do67O2t7ez++67j40ZM4ZJpVIWHx/PlixZwvR6vb+r3W80nhYhJKDQMS1CSECh0CKEBBQKLUJIQKHQIoQEFAotQkhAodAihAQUCi1CSECh0CKEBBQKLUJIQKHQIoQEFAotQkhA+f/4tIA3q3MjlgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:26:34.379205Z", + "iopub.status.busy": "2024-03-01T12:26:34.378901Z", + "iopub.status.idle": "2024-03-01T12:29:19.095489Z", + "shell.execute_reply": "2024-03-01T12:29:19.094711Z" + }, + "papermill": { + "duration": 164.738732, + "end_time": "2024-03-01T12:29:19.097828", + "exception": false, + "start_time": "2024-03-01T12:26:34.359096", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:29:19.137220Z", + "iopub.status.busy": "2024-03-01T12:29:19.136925Z", + "iopub.status.idle": "2024-03-01T12:29:19.156834Z", + "shell.execute_reply": "2024-03-01T12:29:19.156030Z" + }, + "papermill": { + "duration": 0.041634, + "end_time": "2024-03-01T12:29:19.158653", + "exception": false, + "start_time": "2024-03-01T12:29:19.117019", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.00486NaN0.0011992.5938050.0290560.5750930.0373567.052162e-073.2374220.027450.062750.0346290.0539920.026115.831227
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.00486 NaN 0.001199 2.593805 0.029056 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 0.575093 0.037356 7.052162e-07 3.237422 0.02745 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.06275 0.034629 0.053992 0.02611 5.831227 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:29:19.195877Z", + "iopub.status.busy": "2024-03-01T12:29:19.195608Z", + "iopub.status.idle": "2024-03-01T12:29:19.527902Z", + "shell.execute_reply": "2024-03-01T12:29:19.526915Z" + }, + "papermill": { + "duration": 0.35345, + "end_time": "2024-03-01T12:29:19.530077", + "exception": false, + "start_time": "2024-03-01T12:29:19.176627", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:29:19.569513Z", + "iopub.status.busy": "2024-03-01T12:29:19.569193Z", + "iopub.status.idle": "2024-03-01T12:32:12.831650Z", + "shell.execute_reply": "2024-03-01T12:32:12.830770Z" + }, + "papermill": { + "duration": 173.3007, + "end_time": "2024-03-01T12:32:12.849894", + "exception": false, + "start_time": "2024-03-01T12:29:19.549194", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_test/tvae/all inf False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:32:12.889955Z", + "iopub.status.busy": "2024-03-01T12:32:12.889622Z", + "iopub.status.idle": "2024-03-01T12:32:12.907116Z", + "shell.execute_reply": "2024-03-01T12:32:12.906244Z" + }, + "papermill": { + "duration": 0.039707, + "end_time": "2024-03-01T12:32:12.909072", + "exception": false, + "start_time": "2024-03-01T12:32:12.869365", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:32:12.946674Z", + "iopub.status.busy": "2024-03-01T12:32:12.946366Z", + "iopub.status.idle": "2024-03-01T12:32:12.951506Z", + "shell.execute_reply": "2024-03-01T12:32:12.950685Z" + }, + "papermill": { + "duration": 0.02601, + "end_time": "2024-03-01T12:32:12.953431", + "exception": false, + "start_time": "2024-03-01T12:32:12.927421", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.42709620502359585}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:32:12.993121Z", + "iopub.status.busy": "2024-03-01T12:32:12.992826Z", + "iopub.status.idle": "2024-03-01T12:32:13.359794Z", + "shell.execute_reply": "2024-03-01T12:32:13.358871Z" + }, + "papermill": { + "duration": 0.3896, + "end_time": "2024-03-01T12:32:13.362073", + "exception": false, + "start_time": "2024-03-01T12:32:12.972473", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBFUlEQVR4nO3deXTTZb4/8Pc3+57uKy0tUDYRQRYFRES9jiMuyKh43EBFvVe4LoxzlDlHGLcBZ9TBcRx+dxwpekVwwxmvjjqOiogMoCyCW9na0n1v0zR78vz+eJK0oUnbpEm+Sfp5nZPT9JtvvnkS0g/P+nkExhgDIYSISCJ2AQghhAIRIUR0FIgIIaKjQEQIER0FIkKI6CgQEUJER4GIECI6CkSEENFRICKEiI4CESFEdBSISFTt2bMHv/nNb9DZ2Sl2UUgSoUBEomrPnj147LHHKBCRsFAgIoSIjgIRiZrf/OY3+NWvfgUAKC0thSAIEAQBOp0OCxcu7He+x+NBYWEhrrvuOv+xZ555BnPnzkVmZibUajVmzJiBt99+O+jrvfbaa5gxYwbUajUyMjJw4403oqamJjZvjsSUQGlASLQcOXIEGzZswLZt2/CHP/wBWVlZAICTJ0/i8ccfR11dHfLy8vzn79q1CwsWLMBbb73lD0ZFRUW4+uqrMXnyZDgcDmzfvh379+/H+++/j0WLFvmf+9RTT+HRRx/FDTfcgAULFqClpQUvvPACdDodDh06hLS0tLi+dzJMjJAo+v3vf88AsMrKSv+xiooKBoC98MILAefee++9TKfTMYvF4j/W9z5jjDkcDjZlyhR28cUX+49VVVUxqVTKnnrqqYBzjx49ymQyWb/jJPFR04zE3Pjx4zFt2jS88cYb/mNutxtvv/02rrrqKqjVav/xvvc7OjrQ1dWF+fPn4+DBg/7jO3bsgMfjwQ033IDW1lb/LS8vD2VlZfj888/j88ZI1MjELgAZGZYuXYpf//rXqKurQ2FhIXbu3Inm5mYsXbo04Lz3338fTz75JA4fPgy73e4/LgiC//7x48fBGENZWVnQ15LL5bF5EyRmKBCRuFi6dCnWrFmDt956Cw888ADefPNNGI1GXH755f5zvvzyS1x99dW48MIL8ec//xn5+fmQy+UoLy/H66+/7j/P4/FAEAR8+OGHkEql/V5Lp9PF5T2R6KFARKKqb82lr9LSUsyePRtvvPEGVq1ahR07dmDx4sVQKpX+c9555x2oVCp8/PHHAcfLy8sDrjV27FgwxlBaWorx48fH5o2QuKI+IhJVWq0WAIJOaFy6dCn27t2LzZs3o7W1tV+zTCqVQhAEuN1u/7Gqqir87W9/CzhvyZIlkEqleOyxx8DOGPRljKGtrS06b4bEDQ3fk6j6+uuvMXv2bFxxxRW48cYbIZfLcdVVV0Gr1aK2thbFxcXQ6XSQy+VobGwM6M/57LPPcMkll2D+/Pm46aab0NzcjBdffBF5eXk4cuRIQNDZsGED1qxZg7lz52Lx4sXQ6/WorKzEu+++i7vvvhsPPfSQGG+fRErEETuSop544glWWFjIJBJJv6H8efPmMQBsxYoVQZ/78ssvs7KyMqZUKtnEiRNZeXk5W7duHQv2VX3nnXfYBRdcwLRaLdNqtWzixIls5cqVrKKiIlZvjcQI1YgIIaKjPiJCiOgoEBFCREeBiBAiOgpEhBDRUSAihIiOAhEhRHRJvcTD4/Ggvr4eer0+5NICQoh4GGPo7u5GQUEBJJLQ9Z6kDkT19fUoKioSuxiEkEHU1NRg1KhRIR9P6kCk1+sB8DdpMBhELg0h5EwmkwlFRUX+v9VQkjoQ+ZpjBoOBAhEhCWywrhPqrCaEiI4CESFEdBSICCGiS+o+IkKigTEGl8sVkJCNDI1UKoVMJhv29BkKRGREczgcaGhogMViEbsoSUuj0SA/Px8KhSLia1AgIiOWx+NBZWUlpFIpCgoKoFAoaGJsGBhjcDgcaGlpQWVlJcrKygactDgQCkQkYTHGsPdUO+o7rZg7LhP5RvXgTwqDw+GAx+NBUVERNBpNVK89UqjVasjlclRXV8PhcEClUkV0HeqsJgmroqkbe0+14XS7BR8caYDL7YnJ60T6vzjhovH50b8ASVjf1nT673fbXKhs7RGvMCSmKBCRhOTxMKgVMihkEkzK58sDjjebRS4ViRUKRCQhSSQCrj6nAP+1YCzOKjACAGo7LP32MSOxV1JSgo0bN8b0NaizmiQ0iURAnlGFWSUZKEiLrCOUJD4KRCQhOd0eyKW8wi6XSnBBWZbIJUpuDodjWPN8Yo2aZiQhvbKnCn/98hRazXZRXt/h8oS8nTl6N9C5ziGcG4mLLroIq1atwqpVq2A0GpGVlYVHH33U33QtKSnBE088gdtuuw0GgwF33303AGD37t2YP38+1Go1ioqKcN9996Gnp3cQoLm5GVdddRXUajVKS0uxdevWiMoXLqoRkYRjd7nRbXMBAHRK/hW1Od2o77TC6WaYkDdwbptoePHzEyEfK83SYvH0Qv/vf9l1Ek538L6rUelqXD+zN3nf5q8qYXUELiV58D/GR1TGV155BXfeeSf279+Pb775BnfffTeKi4tx1113AQCeeeYZrF27FuvWrQMAnDx5EpdffjmefPJJbN68GS0tLf5gVl5eDgBYvnw56uvr8fnnn0Mul+O+++5Dc3NzROULBwUiknC6rE4AgFohhUouBQC09zjw98P10CllcQlEyaCoqAh/+MMfIAgCJkyYgKNHj+IPf/iDPxBdfPHF+OUvf+k/f8WKFbj55pvxwAMPAADKysrwxz/+EQsWLMCmTZtw+vRpfPjhh9i/fz9mzZoFAHj55ZcxadKkmL8XCkQk4Zi8gciolvuPZep4/4bZ7oLV4YZaIY1pGVYuHBfyMckZq0DuvnBsyHPPXDFyx7zS4RQrwPnnnx+wJGXOnDl49tln/Yt3Z86cGXD+t99+iyNHjgQ0txhj/qUux44dg0wmw4wZM/yPT5w4EWlpaVErcygUiEjC6QoSiJQyKfQqGbptLnRYHFArorvc40wK2dC7T2N17nBptdqA381mM+655x7cd999/c4tLi7GsWPH4lW0figQkYQTLBABQJpGgW6bC50WJwrSYhuIksG+ffsCft+7dy/KysoglQavLZ577rn44YcfMG5c8NrexIkT4XK5cODAAX/TrKKiAp2dnVEtdzA0akYSTqhAlK7hv3daHHEvUyI6ffo0Vq9ejYqKCmzbtg0vvPAC7r///pDnP/zww9izZw9WrVqFw4cP4/jx4/j73/+OVatWAQAmTJiAyy+/HPfccw/27duHAwcOYMWKFVCrYx/0KRCRhJOrV6EoQ4MMbeC8lzRfIPIGqpHutttug9VqxezZs7Fy5Urcf//9/mH6YKZOnYovvvgCx44dw/z58zF9+nSsXbsWBQUF/nPKy8tRUFCABQsWYMmSJbj77ruRk5MT8/dCTTOScOaOCz55MU3DA1OnhQIRAMjlcmzcuBGbNm3q91hVVVXQ58yaNQv//Oc/Q14zLy8P77//fsCxW2+9dVjlHAoKRCRp5BtVuOLsfKRr5YOfTJIKBSKSUNwePjFQeuYYOQCNguYQpSoKRCSh1LRb8O6hun4zkkmgnTt3il2EqKLOapJQzHa+tEMmDZ47uq7TioOnO9BkssWzWCTGKBCRhNJj960xC94PdLS2C19UtKC6jXbdSCUUiEhC6XHwQKRVBp+UZ1Dz3gQTDeGnFFEDkdvtxqOPPorS0lKo1WqMHTsWTzzxBGXhG8HMdr5Oyrfq/kwGFa8pmWwUiFKJqJ3VTz/9NDZt2oRXXnkFZ511Fr755hvcfvvtMBqNQdfDkNTna5ppQwQi32xrqhGlFlED0Z49e3DNNddg0aJFAHgyp23btmH//v1iFouIqLePKESNyBuIum0uMMZoQ8QUIWrTbO7cufj000/9q36//fZb7N69Gz//+c+Dnm+322EymQJuJLUUZ2hQnKEJGYj0ShkkggCXh6HHQXvVpwpRa0SPPPIITCYTJk6cCKlUCrfbjaeeego333xz0PPXr1+Pxx57LM6lJPF02Vl5Az4ukQjQqWQwWZ0wWZ0hAxZJLqL+K7755pvYunUrXn/9dZx11lk4fPgwHnjgARQUFGDZsmX9zl+zZg1Wr17t/91kMqGoiCa9jTSXTc6FXCrxJ0sjyU/UQPSrX/0KjzzyCG688UYAwNlnn43q6mqsX78+aCBSKpVQKpXxLiaJk4GWd/RVlBHDfeoZA9widIRL5f3TOYbw6quv4sEHH0R9fX3A38PixYuh1+vxv//7v7EqZcyIGogsFku/fbOlUik8ntjscU4S26kWM94/0oCSLA2unT5KnEK4ncCXz8b/def/EpANrYZ3/fXX47777sN7772H66+/HgDffeODDz4YcGV9IhO1s/qqq67CU089hQ8++ABVVVV499138dxzz+Haa68Vs1hEJBZv57NUMvDX0mRz4tDpDhyp7YxDqRKPWq3GTTfd5N95AwBee+01FBcX46KLLhKvYMMgao3ohRdewKOPPop7770Xzc3NKCgowD333IO1a9eKWSwiEquTByKNfODE+F0WJ3ZWtCBdI8fUUWnRLYRUzmsn8SYNL7XJXXfdhVmzZqGurg6FhYXYsmULli9fnrTTGUQNRHq9Hhs3boz5vtokOfj2+xpshw7f7OqYzCUShCE3kcQ0ffp0nHPOOXj11Vdx2WWX4fvvv8cHH3wgdrEiRmOfJGH4akSDBSKdSgZBgH8u0Ugdwl+xYgU2btyIuro6XHrppUk9gkyLXknC8PURqQdpmkklgj/4jOSlHjfddBNqa2vx0ksv4Y477hC7OMNCgYgkDH8f0RA2T/StOesawYHIaDTiF7/4BXQ6HRYvXix2cYZlZNZpSUIala6GViGFXjV4x61BLQc6rCO6RgQAdXV1uPnmm5N+fh0FIpIwFk4Y+rY1I71G1NHRgZ07d2Lnzp3485//LHZxho0CEUlKk/INGJ2pQZo68Ue4YmH69Ono6OjA008/jQkTJohdnGGjQEQSgsfDwDD48g4fo1rebyfYkSTUvmXJigIRSQgNJhve/LoG2Xolbjl/tNjFIXFGo2YkIfgmM8qGWCMCvIn0j7Wge5hpYyk18fBE4/OjQEQSwlBnVfd1qKYDB6s70N7jiOg15XLetLNYaEeQ4fB9fr7PMxLUNCMJwT+repDJjH0ZVHK0mR0wWV0RvaZUKkVaWhqam5sBABqNJmnXaomBMQaLxYLm5makpaVBKh36v92ZKBCRhGDxbiOkUQz9KxmNIfy8PJ4R0heMSPjS0tL8n2OkKBCRhGDzrzMbem+BIQqBSBAE5OfnIycnB07nyJyTNBxyuXxYNSEfCkQkIfSuMwu/RhSNPc6kUmlU/qBIZCgQkYSQZ1RBEIA0zdA7PH27vo7U2dWphAIRSQhzx2aF/RxfjcjqcMPuckMpoxpNsqJARJKWUibF0llFSNcoKAglOQpERHSMMbg9DDJp+NPaCtLUMSgRiTea0EhE12V14oXPTuClXafELgoRCdWIiOh8kxklYSzv8Gk12/FDvQkquRSzSzOiXTQSJ1QjIqLzDd0PJTPjmcw2Fw5Ud6Ci0RTtYpE4ohoREZ11iLmqg0nXyKB2tMNsUsHjYRHVqoj4KBAR0Q11945+etpg+O4dTG86Bg8DLN/XQXfWz4FBNmgkiYf+xYjoIqoR2buBb7dBsLRBpeBZGp2nvwEqv4hFEUmMUSAiogu7j4gx4NjHPBhps9B01p04nnkxv07NfsBMC1iTDQUiIrpsvQIlWRqka4eYf7qjCmg9DkikwOTFyEhPR5t2HJqUowHmASp3xbS8JPooEBHRzRidgWunj8LYbN3gJzMGVO3m9wumA7psZOv5VjpV+hn8eNsJwNoRo9KSWKBARJJLVw3QVQtIZEDx+QD47Oo755fi2gvOATJKebBqOCJyQUk4KBARUTHG4HJ7hv6E+kP8Z+5ZgFIPAJBLJTCo5Dy7Yt7Z/PGWn3hAIkmBAhERldPN8MJnJ/Di5yfgHCwgOSxASwW/XzA9+DmZZby2ZGmnTuskQoGIiMo3dO/xsMF38Gg8CnjcgD4XMOQHPFTTbsF739Zj16kuIHMsP9jyUyyKTGKAAhERVd/JjIMmrm86yn/mT+v3kIcxnGw240SzGcgq4wfbaRFtsqBAREQ15KT5PW2AuQUQJEDOpH4P+zI8dlmd6FYX8oPmJt6cIwmPAhERlT9X9WBJ81t+5D8zSgF5/xxESpnUP4xfb5UDumzeWd1RFc3ikhihQERE5d+9Y7Ck+c3eQJQ9MeQpviRp9Z1WIL2UH6RAlBRED0R1dXW45ZZbkJmZCbVajbPPPhvffPON2MUicTKk5R3mFqCnlc+kzhof8rRCbyCq7bQCaaP5wa7aqJWVxI6oq+87Ojowb948LFy4EB9++CGys7Nx/PhxpKeni1ksEkfpGgVKs7TI0ilDn+Rvlo0B5KqQpxWlayAIQGu3HWZVPnQAYGkDHD2AQhvVcpPoEjUQPf300ygqKkJ5ebn/WGlpqYglIvF29igjzh5lHPik1uP8Z/aEAU9TK6TIM6ggkQiwMgV02ixek+qqA7JD16SI+ERtmr333nuYOXMmrr/+euTk5GD69Ol46aWXQp5vt9thMpkCbiTF2br4xERBADLGDnr6DTOLcMPMIt5xbSziB7tqYlxIMlyiBqJTp05h06ZNKCsrw8cff4z/+q//wn333YdXXnkl6Pnr16+H0Wj034qKiuJcYhJtgy7vaDvJfxoKAIVm0OsFZGg0juI/qZ8o4QmMibcgR6FQYObMmdizZ4//2H333Yevv/4a//73v/udb7fbYbfb/b+bTCYUFRWhq6sLBoMhLmUm0fWnz45DEATcOmc0DKogu7weeYuvph+zABg9d8jXtTrcEGwdUB14iXdyz/8l/0niymQywWg0Dvo3KmqNKD8/H5MnTw44NmnSJJw+fTro+UqlEgaDIeBGkpfD5YHTzeBweaAKtkGi29k7/J45bsjX3XWsBf+z6ySOtgm8c9vjBnpaolNoEhOiBqJ58+ahoqIi4NixY8cwevRokUpE4sm3vEMmESCXBlne0VEFeFyAyghos4d8XYNaDsaA6nYroPeuSetuiEKJSayIGogefPBB7N27F7/97W9x4sQJvP766/jLX/6ClStXilksEif+XNWh1pm1neA/M8fxzuohKs7gfUkNnVa4tbn8YHfjsMpKYkvUQDRr1iy8++672LZtG6ZMmYInnngCGzduxM033yxmsUic+NaZBd29g7HejurMwUfL+krXyKFTyuDyMDTDOyeNakQJTfTthK688kpceeWVYheDiKDHzmtEOmWQr6GljSfHl8iAtOKwrisIAkalq/FTYzdOO43IB/jsbLcLkIr+lSdBiL7Eg4xcPQOtvPd1UhtHAdIgo2mDGJXOm2fVZikf9mcevhqfJCQKREQ0RrUcpVla5BqCLO9or+Q/MyKbaV+UwdedNZrscFE/UcKjeioRzaR8AyblB5mC4XEDndX8fnpJRNc2quWYXpyGDK0CzJoHdFQCZgpEiYoCEUk8pno+h0iuBnS5EV1CEARcNCGH/9KcB9SCclgnsIiaZqdOUQpOMnwhk+X7+ofSR4c1bB+SzhuQeloBTxg7hpC4iSgQjRs3DgsXLsRrr70Gm80W7TKREYAxhv+38yRe/PwEzHZX4IMd3v6h9OFlYvB4GJpNNvzYKeGjZR4XbbyYoCIKRAcPHsTUqVOxevVq5OXl4Z577sH+/fujXTaSwuwuD1we3/KOPl9Dlx0weef8RNg/1Pc1tu47jY++b4ZTlcUP0shZQoooEE2bNg3PP/886uvrsXnzZjQ0NOCCCy7AlClT8Nxzz6Glhdb1kIH5akEquRQyaZ+vYWcNH2pXpwPqtGG9hlohhUHNh/47Jd6JjT3UT5SIhjV8L5PJsGTJErz11lt4+umnceLECTz00EMoKirCbbfdhoYGms1KgrN4JzNqlWfMqu7yLngOcxJjKFk6BQCgQ5LGD5jpP8lENKxA9M033+Dee+9Ffn4+nnvuOTz00EM4efIkPvnkE9TX1+Oaa66JVjlJivHViLRnTmbs9CYxS4tOrqlMLZ+j1MK8WSCpaZaQIhq+f+6551BeXo6KigpcccUVePXVV3HFFVdAIuFxrbS0FFu2bEFJSUk0y0pSSLfNCQDQq/p8BV2O3kmHxugEogwtrxE1uPT8gL0bcFqDbklExBNRINq0aRPuuOMOLF++HPn5+UHPycnJwcsvvzyswpHUZbLxGpG+bzI0Ux3vH1IZeOqPKPA1zVptApjKAMFm4vOJ0inVTCKJKBB98sknKC4u9teAfBhjqKmpQXFxMRQKBZYtWxaVQpLUk6Xju3f4NkUE0JvS1TgqOvOHAKRrFRAEnnLEbsiGymbiSdIoECWUiALR2LFj0dDQgJycnIDj7e3tKC0thdvtjkrhSOqaXpyO6cVnbBvlS3IfpWYZAMilElw6KRd6lQyyzgag8yT1EyWgiAJRqDTXZrMZKlXofacICcnj5tv+AFEbMfOZUuht5rm9y0UobWzCCSsQrV69GgBfx7N27VpoNL27Krjdbuzbtw/Tpk2LagFJ6vF4GJweD5R981R3N/CZz3I1oMmMzQtrvZMae1p54rUoNf/I8IUViA4dOgSA14iOHj0KhULhf0yhUOCcc87BQw89FN0SkpTTarZj677TSNPIcfs87zKOvsP2UQ4QZrsLtR0WSJgM4yVSvqDW1jXsCZMkesIKRJ9//jkA4Pbbb8fzzz9Pu2iQiJi8Q/cqeZ8akcnbLIti/5BPs8mGD482IluvxHhNBp/U2NNKgSiBRDShsby8nIIQiViHhQeidI136J6x3kBkKIj666VpeM29y+oE03ibZ5bWqL8OidyQa0RLlizBli1bYDAYsGTJkgHP3bFjx7ALRlJXR48DQG+AgK0TcFj4Boi6vKi/nkElgyDwfdRsigyoAeqwTjBDDkRGo9G/5YvRGJ3JZmRk6vTWiNJ8NSLfantdTkyS28ukEuiUMnTbXOiWplEgSkBD/lcvLy8Pep+QcHVYeI0o3VcjMtXzn/roN8t80jQKdNtc6BCMyAGAnjaeJE1CadsTQUT/ClarFRaLxf97dXU1Nm7ciH/+859RKxhJTTanGxbvxoq9NaLY9Q/5+Na0dTEt36LI4+JNQpIQIgpE11xzDV599VUAQGdnJ2bPno1nn30W11xzDTZt2hTVApLU4mEM04vTUJar4/OI3K7eXNJxCETddjeg9c5T6qEO60QRcYbG+fPnAwDefvtt5OXlobq6Gq+++ir++Mc/RrWAJLVoFDJcNCEHV071Bp2e5t6JjOr0gZ88DBNy9Vg8vRAzR2cA2mzva1M/UaKIqGfQYrFAr+dpFf75z39iyZIlkEgkOP/881FdXR3VApIU5+sfMhTEdKZzpk6JTJ13gS0N4SeciJPn/+1vf0NNTQ0+/vhjXHbZZQCA5uZmml9EBlTbYYHD1WcnjTj0D/VDNaKEE1EgWrt2LR566CGUlJTgvPPOw5w5cwDw2tH06dOjWkCSOnrsLrz1TS3+3xcnYXd5MzT4hu5jHIgYY/ixwYT9le1wqLx9RJZ22l4oQUTUNLvuuutwwQUXoKGhAeecc47/+CWXXIJrr702aoUjqaWmg4+0ZmgVvKPa0dO7vU8Mh+4BvlB7Z0ULbE43SjOLkS2V8zVn1o7ezmsimohnj+Xl5SEvL3AW7OzZs4ddIJK6TrX0AACKM7xZG3y1IU0mII99+hi9Sgab041uuwvZ2iz++j0tFIgSQESBqKenBxs2bMCnn36K5uZmeM6o3tJOsORMNqcbJ5vNAIAJed780XHuHzKo5WjptqPb5uL9RL5AhIlxeX0SWkSBaMWKFfjiiy9w6623Ij8/37/0g5BQvqnqgMvDkKVXIseXHrY7Pv1DPnol/7qb7a7eDmsaOUsIEQWiDz/8EB988AHmzZsX7fKQFPRtTSe+rmoHAMwZk8n/4wpYcV8Yl3LoVH0CUTpNakwkEY2apaenIyMjI9plISnKt6XPtOI0jMvR8YOWNr59kFTWWzuJMd8eamZb3xpRO09TS0QVUSB64oknsHbt2oD1ZoSEUpShwY2zi3DR+D4Bx1cb0ufHbeGpzts063G4AKUekCn59kWWtri8PgktoqbZs88+i5MnTyI3NxclJSWQy+UBjx88eDAqhSPJyeNh+NePTZg6Kg15Rj4alm88Y0PDOM0f6ivHoMS10wt5E00QeA7rrjrePNPlDH4BEjMRBaLFixdHuRjAhg0bsGbNGtx///3YuHFj1K9P4qeiqRvf15twqrUHd15QCrk0SI0nzv1DAE9NW5Kl7T2gzfYGIpphLbaIAtG6deuiWoivv/4a//M//4OpU6dG9bpEHN/VdQEAphWlBQ9CLkfvH388l3aciZZ6JIyIG+ednZ3461//ijVr1qC9nY+IHDx4EHV1dWFdx2w24+abb8ZLL72E9PTYrb4m8dFldaK2wwpBACYXhFh32N3AR82Uen6LoxPNZnxd1Y5OiyNweyEiqogC0ZEjRzB+/Hg8/fTTeOaZZ9DZ2QmA56pes2ZNWNdauXIlFi1ahEsvvXTQc+12O0wmU8CNJJbqNj57usCohkElD35S3xX3cXbwdAd2H29Fk8neWyOydfLlHkQ0EQWi1atXY/ny5Th+/HjAzq5XXHEFdu3aNeTrbN++HQcPHsT69euHdP769ethNBr9t6Ki6G89Q4bndDsfSS3O1IQ+SYT+IR9d30mNCi2g0PDaGdWKRBVRIPr6669xzz339DteWFiIxsbGIV2jpqYG999/P7Zu3TrkbarXrFmDrq4u/62mpiascpPY8ngYatqtAIDRoQIRY6LWiLS+IXy7y3uA+okSQUSd1UqlMmiz6NixY8jOHtrktAMHDqC5uRnnnnuu/5jb7cauXbvwpz/9CXa7HVKpNOA5SqUSSqUykiKTODA7XFDIJPAwhlx9iP9cbF181b0gAfTR3zpoMDol/04FBKKOalrqIbKIAtHVV1+Nxx9/HG+++SYAnmLh9OnTePjhh/GLX/xiSNe45JJLcPTo0YBjt99+OyZOnIiHH364XxAiic+gkuPOC0phd7khkYRYf9jdd+ugEH1IMaTt2zQD+Mp/gJpmIot4QuN1112H7OxsWK1WLFiwAI2NjZgzZw6eeuqpIV1Dr9djypQpAce0Wi0yMzP7HSfJRSkb4D8REfuHgN5lHtQ0SywRBSKj0YhPPvkEX331Fb799luYzWace+65Qxr5IqmLMTZ4JgYR+4eAvss8vOvL/CNnJsBpi0teJNJf2IHI4/Fgy5Yt2LFjB6qqqiAIAkpLS5GXlze0L+IAdu7cGfFzibjcHobNuyuRoVVg0dR8qORBakVuF9DdxO+LFIj0KhmunV4IrVLGv69yFZ/LZO/m/UTGUaKUa6QLa9SMMYarr74aK1asQF1dHc4++2ycddZZqK6uxvLlyylN7AjW3uOA2e5CU7cNSlmIr5W5KS5bBw1EJpWgJEuLbL2y9z9Nap6JLqwa0ZYtW7Br1y58+umnWLhwYcBjn332GRYvXoxXX30Vt912W1QLSaKr2+bE0douZOuVKMuNzszmTu820hkaRehacZd3uoVxVEy3DgqbNgtoP0Ud1iIKq0a0bds2/PrXv+4XhADg4osvxiOPPIKtW7dGrXAk+twehncO1GJfZTveP9KAky3mqFy308pnJvu3kQ56kjcQpRVH5TUjVdXag/2V7Wgy2fgBqhGJLqxAdOTIEVx++eUhH//5z3+Ob7/9dtiFIrFzrKkbHZbe5Qz7K9ujct1O7zWNakXwEzyePjUicWfE/9hgwlcnWlHr3VWE1pyJL6xA1N7ejtzc3JCP5+bmoqOjY9iFIrFzrKkbAF8ZLxEENHbZ0NHjGPZ1fU2zkDWinhbAZQdkCkAX+jsUD72zq70jZ76dXx09/EbiLqw+IrfbDZks9FOkUilcLtewC0VigzEGu3eX1bMK+cp4pVwCmXT4/TVdgzXNfLUhw6i4ZWQMRXvm7GqZAlCnAdZOXitSaEM+l8RGWIGIMYbly5eHXGZht9ujUigSG4Ig4IaZReixu6BRSLFwYnSyEno8DGka3iQzqkMEos7T/Gea+AuVtWfOJQJ4P5EvEKWPFqdgI1hYgWjZsmWDnkMjZonP94cYLRKJgOtmDDD/xuPpDUQi9w8BQWZXA7yfqPU4dViLJKxvZHl5eazKQeIg2IRTq8ONhi4r8o1qqBUxWt9nbgScVt4EEjMjo1e/9WYAjZyJTNzGOomrrftO482vawI6p3ccqsXfD9ejrtMa8XUZYwOf0O7d+Te9BJCIv5hZ4w24DpcHDm+fWUAgGuz9kKiLbh2dJCy7y42Wbt6H13f5RaZWiWaTHW1me++eY2HaWdGCky1mnD8mE1MKjf1P8AWijDERXT/alDIJlpxbCI1CBpkvS4A6g6cmcdn5cg9ViDS3JCaoRjRCdPTwUS2tUhrQBMvS8U7mtmEM4XdYHHw/+WCc1t6FrgkSiARBwOhMvszDn65EKutddkLNs7ijQDRCdHjn+aRrAiccZur4CGibOfIRT99kxqBD9+2VvKmjyQRUQWpLicQ3sZE2XIw7CkQjhK9fyLf9s0+mt0bU3uOE2xN+34jbw/y1oTRNkFnVLT/xn9njw752LNW0W7C/sh017X12K6YOa9FQIBoh2v0znwODhV4p86d39dWawtFtc8LDGORSAdozR91cDqDtJL+fPSmicsfKyRYzvjrRiqq2PjOpKRCJhgLRCBGqRiQIgr+51mkJf0sd/xqzYKvu207wtB/q9ITb0rnfMg8gMBB5PCKUauSiUbMRwqhRwOFmyAjSfJoxOh1Otwc5hvA3JvCtug86o7rpO/4zZ2Jipf1AiEmN6nTeae12AdYOQJspUulGHgpEI8TV54SeSDghL/KcRCq5BIXpauTqzwhi1s7eYfu8xNtKvDdlbJ9AJJEA2hw+ymduokAURxSIyLBMzDNgYl6QOTf1h/hoWUYpoMmIf8EGofEvfHUHPqDL7Q1EuZNFKNnIRH1EI8BgM59dbg9q2i34sSFKW3g7rUD9QX6/4NyBzxWJr0Zkc7rhcvfpD/L1ZZmbRSjVyEU1ohFg76l2HKntxPTidMwu7V87cbg9ePtALQQBKMvRQSYd2v9PjDG4Paz/+TX7+IiZLhvIKovGW4g6pUwCmUSAy8PQY3fDqPG+B1+uJHOTeIUbgSgQjQBdVicsDnfIx9VyKRQyCRwuD7qsTv8kx8GYbC6Uf1WJNLUcy+aW8FGznlagZj8/oeTChOuk9hEEAdeeWwiVXAqdqs+fgTabl9nRA9jNgDKyZS8kPNQ0GwFMNj6yZVAH/39HEAT/rGjfKNiQrmt1gjH+fEEQeC3ox/cAjxvIHJewtSGfUekaZOmUkPbdlVam4OvOAKoVxREFohHA5A0uBlXoxPZpagXAPDB1tvVuNjhI31LA0g6nDfjuHb5vmUIDjL8sYWtDg/L1E9HExrihplmKc3uYP++OIdhcH7cLaDyC8fX7kF53CmldCqDW2xyRKfncGk0GryVoMr330wGpAp0WG5QuEwot1cDXFXzVulQGTPlF4q8rA9DQZcXpNguy9EqMze7TBNPlAs0/Uo0ojigQpTizzQXGAJkkyBKM7kbgh/cASxuMDhtawWB1MZ4zyOPmKTG6G/ktiJwmM6b32FFo1wBGNQ8+U5YA+rw4vLPhq2m3Ys/JNkwuMJwRiGjkLN4oEKU4X/+QXiULXILRUQUcfRtwO3my+HHzcEimhkqficnzx/Dj1k7A2g5Y2gN/OvhCUZvTBSZIIEsfBYyZDhRMA6QD7GuWYHwJ0iyOM1KY+EbOLG38c0ii95SsKBClOIlEQHGGBvq+I0Pmlt4glFEKTLoaBkGJC43m3lQeUjkfftdl97+oywHmdmDvrpOwMBUmzSwFtCH2M0tgOn/K2DNGFJU6HpwdPbyfKAHS26Y6CkQprjBNjV/0TWzvcgDf7+BBKH00MOU6QCqDCgieXTEYmQIuQYbR+bkwWZ3B+56SQO/C1yBJ3XS5fIlKdyMFojigQDTSVH3Jm1hKPTD5Gt65HAG5VILLpyRHX1Aovv3NrA433B4WOIyv7xOISMzR8H2KC0h2Zm4Gar/h9yf8vN9Ggq1mO76r60L9MBLpJxO1XOoPPj1n9hPp8/nP7oY4l2pkokCU4sq/qsRfdp3kifNPfQEwD8+WmDm237k/1JvwyQ9N/m2pB2JzuiPK6JhIBEHwd1j3a575AlFPK2/GkpiiplkK880hYgzQOlp4ojJBAMYsDHq+r6O6awizqz/7qRnHm8y4eGIOzh6V+HOGQrlyagFkUgFpZ/ZzqQy809pu5s2zBNihNpVRjSiF9Z1DpK7fyw/mTA6ZliNNPfRMjZ0WniI2ZpsyxkmeUYUsnTL4Ql9qnsUNBaIU5ptDlCntgdB6nNeGRs8Neb6xT43IM0CzizGGTqsvB3ZyjpgNCQWiuBE1EK1fvx6zZs2CXq9HTk4OFi9ejIqKCjGLlFJ8gajYdowfSC/t3TInCL1SBqlE4E26Mztv+7A5PbA7eQ6foClik0hztw37TrXhp8YguZgM3kBkokAUa6IGoi+++AIrV67E3r178cknn8DpdOKyyy5DT0/P4E8mgzJZXRCYCzkWbyAqHDhJmUQiwOCd+Ng1QPPMt9uHXiWDfIi5ixJVs8mOPSfbUNEYpIPeVyOydvBkbyRmRO2s/uijjwJ+37JlC3JycnDgwAFceOGFIpUqdXTbnMi0VEIDO6DMBTL6j5SdKU2jQIfFiU6LE0UhMryG2qwxGflGzczBJjXK1XyBr7WDp48NMtJIoiOhRs26uroAABkZwf8C7HY77PbeHUlNpiilNk1RmToF0lAFjULG14FJBq+9zC7NwIzR6cgaIDmarzM7XZvczTKgTxL9YIEI4M0zawfvJ6JAFDMJU6/2eDx44IEHMG/ePEyZMiXoOevXr4fRaPTfiopoSHUgM/JVmJ1m5nuZ5Z41pOcUpKlRlKEZcDQsW6/EpHw9CtM00SqqaHzLPCwOd/AOekMh/9lVF8dSjTwJE4hWrlyJ7777Dtu3bw95zpo1a9DV1eW/1dTUxLGESaj1GJ/AqMvhTYwoGZ+rx+VT8oe1DVGiUMulEASeA87iDJJO1+hdp2eqpU0XYyghmmarVq3C+++/j127dmHUqFEhz1MqlVAqw98EcCRyexhY80/8Hzh74pCf53J78FNjN0w2J+aMyey/e2uKkUgEaBUymO0u9Nhd/qaanzaHp491OfhKfH2uOAVNcaLWiBhjWLVqFd5991189tlnKC0tFbM4KaWxvRP7Dx7E0bouIHtCWM/9149N2HeqPWjCfafbg06LY8B5Rsmmd4+zIP1EEglg8P7n2EU18FgRtUa0cuVKvP766/j73/8OvV6Pxka+0tloNEKtVotZtKTnaDoGgXngVGUNOHfoTDKpBHqVHCarE51Wp78Pxaeuw4p3D9UhS6fArXNKolxqcfzH5FxIBSF0OhPjKL4Sv6sGGDUzvoUbIUStEW3atAldXV246KKLkJ+f77+98cYbYhYrJbibfgIAuDLC30kj3Ttbut3s6PdYWw8ftczQpk4TOUevQqZOGXpOlG+dWVftoBsKkMiIWiMabAdSEiGXA4J333khZ+j9Qz7ZeiWq2yxoMdsABC5obenmwSlTl/xziIZMn8/zeNvNfCg/AbfQTnYJM2pGoqj9JBxOB+wyPdRp+WE/3TeHqLU7dI1ooHlGyaajx4F9p9pw6HRH8BOk8t4NAbpq41ewEYQCUSpq+QkOlwdtmjEwqMOvufiCTIvZHlBr9XiYv7mWlUI1IpPNiT0n2/BdXVfok9KK+c/O6vgUaoShQJRq3E6wtpOwO91oV5eE3N11IBlaBaQSAQ6XByZr70hSu8UBl4dBIZMk/WLXvnwbT5psrtDdBekl/Gd7JfUTxUBCzCMiUdReCeZywJieifS8kv7zYoZAKhFw7fRCpGnkAc9v6LQBAHINqpSaX+Tb4cTh8sDm9ASfVW4YxfN7+3b28O19RqKCAlGqaa2ARBAwYcosTCiLfAlMUUb/5Rt5RhXOH5MZUS0rkcmkEuiUfFKjyeYMHoikMsBYzIfxO6ooEEUZNc1SiccNtB7n97PHR/3y2Xol5ozNxFkFyZsaNhRfcDUNlCa3b/OMRBUFolTSUQW47HBK1XDqhrcXl8fDsO9UG3YcrIXdFWQNVorp7ScaIBBleGf+d50G3KETx5HwUSBKJS08u+URWy7+9PkpHAw1HD0EEomAHxtMqG6z4FRLD6rbenCiuRsOV2ou/PTNqu7bOd+PNptvweR28WBEoia1GvsjmcfDV9sDaFAWAw5E1FHd14Q8A/aeasP39SZYnW60dtuxYEI2zi2O3kr+RHH2KCMm5ukH3rVWEIDMcUDDt0DrCSBjTPwKmOKoRpQquk7zdKZyFeoZ36++3xY5YZqcb4BEEFDTbkFrtx0KmQST8w3RKG3CMajkAy/z8Mny9r21Hadh/CiiQJQqWnhtyJk+Dj3ebo7h7klv1Mhx/pje5QwXlmVDJU/u7YOGLX00H0GzmQBzk9ilSRnUNEsFjAGtvH/IpOPNBbVCGpWgcd6YTJRmaSGVCMhMoWUdwRyobkeb2YELyrJ4et1gpHLeJGs5xpvCvqUfZFioRpQKTHV8QaZMgXYFHy0bbrOsrxyDKuWDEAAcqe3C9/UmtAXJOhAg05vRwNsnR4aPAlEqaOEpP5A5Dh023m+RSksw4sW3K8mgO91mlfHV+OYWfiPDRoEo2TEGNHsDUfYkZOuVOKvAgOLM5E9sH2++XWvbLYPUiOTq3hGz5u9jXKqRgfqIkp2pDrB387zKGWNQKpWhNEsrdqmSUobWVyMaJBABfFeU1uNA0w9A6QI+tE8iRjWiZOerDWWW8dEcEjFf06y9ZwiBKHMcD/62LspRFAUUiJIZY0DLj/x+zmTYXW60mu1wp1Bi+3hK99aITFbX4J+hVA5keTclaDwa45KlPgpEyayrxjtapgTSS1DfacP//rsa2/bT8oNIaBVSKGT8T6J7oDVnPvnn8J/N3wNOWwxLlvqoLp/Mmr21oazxgFSGNjPfgtvX10HCIwgCbpxVBINaPvgMa4Dv7qHNAnpagabvaIePYaAaUbJyu4Am74hN7mQAQEt36uWTjrchLfPwEQSg8Fx+v+4gLfkYBgpEyar1GOCyAyoDkFYCAGgy8eZBjp4CUdzkTuH9RZY2njSNRIQCUbJqPMJ/5k4BJBLYnG50eCfi5RpUIhYsuVkdbnz6YxPePVQ7tO2uZEogfxq/f/rfMS1bKqNAlIxsXTwJGgDknQ2gt1lmUMuDpzolQyKTCviuzoSqVgvMwbagDqZoNp9p3VnDbyRsFIiSUf0h3h+RVuzf7K+5m5pl0SCXSpCh5TOsfU3dQakM/v8QUP1VjEqW2igQJRu3kwciIGCUpiRTi7ljMzEhTy9SwVJHQZoaAFDbYR36k4rPBwQJz2dNOa3DRoEo2TQe5XNW1Gm9q8DBR3vOG5OJ8bkUiIZrVDpfp1fXGUYgUqf3jqCd+BfPmEmGjAJRMvG4gZr9/H7hTEBC/3yxUJDGO/tbuu2wOcPYOKDkAkCu4vOKGg7FqHSpib7JyaTxCGDtABSa3lm9AGraLTjW1A2rI/V324gHvUqONI0cjIVZK5KrgZL5/P6pnYC1MxbFS0kUiJKF2wVUeTtCi+fyBZde+yrb8cGRBvzQYBKpcKlndKYGepUMLneYkxQLzgWMhYDLAVT8gyY5DhEFomRx+t883YdSDxRM9x+2Otyo7bAAAMZl68QqXcq5YFw27rygNPzOf4kEmHglz4TQUQ1U74lNAVMMBaJk0NPWO1lu3CUB6T5ONJvBGJBjUMKooayM0aKQSSBEmmNIkwGM+w9+v3KXf2MDEhoFokTndgI//I13VGeOBbIn+h9ijOFIXScAYAKNlsWE28P8Nc6wFEwDCmfw+z/8vXcCKgmKAlEiYww49hFgbuYd1OMvD8gEWNdpRbPJDplESMn96MVmc7rx8u5TeOdAHboGy2MdzLhLeH5rjws4+hbP6EiCokCUqBjj81Eav+PBZ9JVfAav/2GGPSfaAACT8g20rCMGVHIpsnRKeBjDVydbw7+ARApMXsxrsm4X8N07QNVummMUREIEohdffBElJSVQqVQ477zzsH//frGLJC6nlX9pa7/hv09c1G97Y5eHQa2QQi4VcF6fTRBJdF1QlgVBACoau3G8qTv8C0hlwJRf8GYaY0Dll8DBLbQm7QyiB6I33ngDq1evxrp163Dw4EGcc845+NnPfobm5maxixZ/LgdQewDY/xdejZdIeU3It46pD7lUgiun5uOm80ZDr6JO6ljJ0aswY3Q6AODj7xtxssUc/kUkUmD8ZcCkK/mEx+4m4NBrwKGtPKeUawg5slOcwIaU6yB2zjvvPMyaNQt/+tOfAAAejwdFRUX47//+bzzyyCMDPtdkMsFoNKKrqwsGQxLtyc4Y74R2WvhKenMzT/vafoofB3jmv4mLAEMBPB4Gs8OFhk4bKpq6cdnkXNr6OY48Hob3vq1HZWsPAKAsV4dFZ+dHNqpmN/ORtKbv+AAEAEhkgCEfMBQCmkw+6qbQ8QmSUkVS7xAy1L9RUVPFOhwOHDhwAGvWrPEfk0gkuPTSS/Hvf/fP7WK322G32/2/m0xDnMDXXsn7W4Lxx2EW5Fif40M9FuKaHT0OnGo1A4xBwlwQPC7+KONnFWdoeGZFdTqajFPxXlMmPIesYDgJu9MDT5/XOmpUYVYJNcfiRSIRcNU5Bdh1vAXf1nTC6fb4gxBjDC/v5otcJYIAicDP94WOURkaLJyQ47/W1sNt8LCzIJcUo8R1HOdrm/hs+VApRAQJT7wmkfL7grT3frSFG/DOviGg33I4RA1Era2tcLvdyM3NDTiem5uLn376qd/569evx2OPPRb+C7nsfP2PiJjDAYe1J+CYR5DCKdWgR56JtLwyZE2YCujzYGu3wFxbB6B3yYZEEJChU6AkU4PxOTRUH29SiYCFE3JwdqExYLa1w+1Bty103iKjJjB/eGu3w/ufigIewwycP7vIG4hOA92NgLWd/+7o4TUm5uHf30TEotfpnlTJ89esWYPVq1f7fzeZTCgqKhr8iWlFwDk3nhHx+9z3Hw92rM/xoR4Lck2V04XiHm+zSyYH5BoIEjkEiQABAgxqGaDg/xx5RhVuOX80BIE/WymXQiOXQiJJ3ip6qjgzH7hcIsEt54+G28PgYfzGWG+lWKUIrLksObfQ/5hCJuHfE02GP6+UH2N82N9p5c115g1KvuAUNAgMsZclWr0xiuht5ClqIMrKyoJUKkVTU1PA8aamJuTl5fU7X6lUQqmMIPGXQgtklEZazKjQANCkD+1cpUyKbD31ASUDiURAdhjJ6IoyhrgVuCDwJpl0ZAxEiDpqplAoMGPGDHz66af+Yx6PB59++inmzJkjYskIIfEketNs9erVWLZsGWbOnInZs2dj48aN6Onpwe233y520QghcSJ6IFq6dClaWlqwdu1aNDY2Ytq0afjoo4/6dWATQlKX6POIhiNp5xERMkIM9W9U9JnVhBBCgYgQIjoKRIQQ0YneWT0cvu6tIS/1IITEle9vc7Cu6KQORN3dPC3DkGZXE0JE093dDaMxdPK+pB4183g8qK+vh16vjzy/cJT4lpvU1NTQCF4f9LmENhI+G8YYuru7UVBQAMkA+/AldY1IIpFg1KhRYhcjgMFgSNkv1XDQ5xJaqn82A9WEfKizmhAiOgpEhBDRUSCKEqVSiXXr1kWWHSCF0ecSGn02vZK6s5oQkhqoRkQIER0FIkKI6CgQEUJER4GIECI6CkRhCGdH2pdeegnz589Heno60tPTcemll6bsDraR7tS7fft2CIKAxYsXx7aAIgr3s+ns7MTKlSuRn58PpVKJ8ePH4x//+EecSisiRoZk+/btTKFQsM2bN7Pvv/+e3XXXXSwtLY01NTUFPf+mm25iL774Ijt06BD78ccf2fLly5nRaGS1tbVxLnlshfu5+FRWVrLCwkI2f/58ds0118SnsHEW7mdjt9vZzJkz2RVXXMF2797NKisr2c6dO9nhw4fjXPL4o0A0RLNnz2YrV670/+52u1lBQQFbv379kJ7vcrmYXq9nr7zySqyKKIpIPheXy8Xmzp3L/vrXv7Jly5albCAK97PZtGkTGzNmDHM4HPEqYsKgptkQ+HakvfTSS/3HBtqRNhiLxQKn04mMjNTZoTXSz+Xxxx9HTk4O7rzzzngUUxSRfDbvvfce5syZg5UrVyI3NxdTpkzBb3/7W7jd7qDnp5KkXvQaL+HuSBvMww8/jIKCgoAvZrKL5HPZvXs3Xn75ZRw+fDgOJRRPJJ/NqVOn8Nlnn+Hmm2/GP/7xD5w4cQL33nsvnE4n1q1bF49ii4YCURxs2LAB27dvx86dO6FSqcQujmi6u7tx66234qWXXkJWVpbYxUk4Ho8HOTk5+Mtf/gKpVIoZM2agrq4Ov//97ykQkfB3pO3rmWeewYYNG/Cvf/0LU6dOjWUx4y7cz+XkyZOoqqrCVVdd5T/m8fCtk2UyGSoqKjB27NjYFjpOIvnO5OfnQy6XQyrt3eV30qRJaGxshMPhgEKhiGmZxUR9REMQ6Y60v/vd7/DEE0/go48+wsyZM+NR1LgK93OZOHEijh49isOHD/tvV199NRYuXIjDhw+nVKbNSL4z8+bNw4kTJ/zBGQCOHTuG/Pz8lA5CAGj4fqi2b9/OlEol27JlC/vhhx/Y3XffzdLS0lhjYyNjjLFbb72VPfLII/7zN2zYwBQKBXv77bdZQ0OD/9bd3S3WW4iJcD+XM6XyqFm4n83p06eZXq9nq1atYhUVFez9999nOTk57MknnxTrLcQNBaIwvPDCC6y4uJgpFAo2e/ZstnfvXv9jCxYsYMuWLfP/Pnr0aAag323dunXxL3iMhfO5nCmVAxFj4X82e/bsYeeddx5TKpVszJgx7KmnnmIulyvOpY4/SgNCCBEd9RERQkRHgYgQIjoKRIQQ0VEgIoSIjgIRIUR0FIgIIaKjQEQIER0FIkKI6CgQkaSyZcsWpKWl+X//zW9+g2nTpvl/X758eUqnnk1VFIhIUMuXL4cgCPjP//zPfo+tXLkSgiBg+fLlAedHOwCUlJRg48aNAceWLl2KY8eOhXzO888/jy1btvh/v+iii/DAAw9EtVwk+igQkZCKioqwfft2WK1W/zGbzYbXX38dxcXFopRJrVYjJycn5ONGozGgxkSSAwUiEtK5556LoqIi7Nixw39sx44dKC4uxvTp04d17WA1lcWLF/trWRdddBGqq6vx4IMPQhAECIIAoH/T7Ex9a2bLly/HF198geeff95/jcrKSowbNw7PPPNMwPMOHz4MQRBw4sSJYb0vEhkKRGRAd9xxB8rLy/2/b968GbfffnvMX3fHjh0YNWoUHn/8cTQ0NKChoSHsazz//POYM2cO7rrrLv81iouL+70nACgvL8eFF16IcePGRestkDBQICIDuuWWW7B7925UV1ejuroaX331FW655ZaYv25GRgakUin0ej3y8vIGzYQZjNFohEKhgEaj8V9DKpVi+fLlqKio8O8x5nQ68frrr+OOO+6I9tsgQ0SpYsmAsrOzsWjRImzZsgWMMSxatCjp800XFBRg0aJF2Lx5M2bPno3/+7//g91ux/XXXy920UYsqhGRQd1xxx3YsmULXnnllajVGiQSCc5MheV0OqNy7aFYsWKFvyO+vLwcS5cuhUajidvrk0AUiMigLr/8cjgcDjidTvzsZz+LyjWzs7MD+n3cbje+++67gHMUCsWw9/QKdY0rrrgCWq0WmzZtwkcffUTNMpFR04wMSiqV4scff/TfD6Wrq6vffmWZmZlBk+JffPHFWL16NT744AOMHTsWzz33HDo7OwPOKSkpwa5du3DjjTdCqVRG1CQsKSnBvn37UFVVBZ1Oh4yMDEgkEn9f0Zo1a1BWVjbgJggk9qhGRIbEYDDAYDAMeM7OnTsxffr0gNtjjz0W9Nw77rgDy5Ytw2233YYFCxZgzJgxWLhwYcA5jz/+OKqqqjB27FhkZ2dHVO6HHnoIUqkUkydPRnZ2Nk6fPu1/7M4774TD4YjLKCAZGOWsJiPWl19+iUsuuQQ1NTX9dmQl8UWBiIw4drsdLS0tWLZsGfLy8rB161axizTiUdOMjDjbtm3D6NGj0dnZid/97ndiF4eAakSEkARANSJCiOgoEBFCREeBiBAiOgpEhBDRUSAihIiOAhEhRHQUiAghoqNARAgR3f8HbPuEQLWmGG8AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:32:13.401408Z", + "iopub.status.busy": "2024-03-01T12:32:13.401088Z", + "iopub.status.idle": "2024-03-01T12:32:13.726117Z", + "shell.execute_reply": "2024-03-01T12:32:13.725272Z" + }, + "papermill": { + "duration": 0.346834, + "end_time": "2024-03-01T12:32:13.728068", + "exception": false, + "start_time": "2024-03-01T12:32:13.381234", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/z0lEQVR4nO2deZiT5bn/v2/2ZZLMvjEzzLCD7CIIiIpaa921VXtEy2Ktp8LxUH5cVc45xVOXom3lApdDT7WAVJZaxVPbWql1gYKKyI4iA7Mz+5pMJnvy/P54kswMk8wkmSRvkrk/15UrmTdv3rknk/nO/dzPvQiMMQaCIAgRkYhtAEEQBAkRQRCiQ0JEEITokBARBCE6JEQEQYgOCRFBEKJDQkQQhOiQEBEEIToysQ0YDh6PBw0NDdDpdBAEQWxzCIK4BMYYuru7UVhYCIkkuN+T1ELU0NCA4uJisc0gCGII6urqUFRUFPT5pBYinU4HgP+Qer1eZGsIgrgUk8mE4uJi/99qMJJaiHzLMb1eT0JEEAnMUKETClYTBCE6JEQEQYgOCRFBEKKT1DEigggGYwwulwtut1tsU1IaqVQKmUw27PQZEiIi5XA4HGhsbITFYhHblBGBRqNBQUEBFApFxNcgISJSCo/Hg6qqKkilUhQWFkKhUFCya4xgjMHhcKC1tRVVVVUYP378oEmLg0FCRCQsjDF8XtmBhi4rFozLQoFBPeRrHA4HPB4PiouLodFo4mDlyEatVkMul6OmpgYOhwMqlSqi61CwmkhYzjV34/PKdtR2WPDXU41wuT0hvzbS/8xE+ETjvabfFpGwnKzr8j/utrlQ1dYjnjFETCEhIhISj4dBrZBBIZNgcgEvDzjfYhbZKiJWUIyISEgkEgG3zyiEx8NQ32XF2cZuXOy0gDFGwecUhDwiIqGRSATkG1S4ojQTN0zOE9ucmHLttddi9erVYpshCuQREQmJ0+2BXMr/T8qlElw1Pltki4hYQh4RkZC8/mk1XvtnJdrM9qhd0+HyBL1duiM32LnOEM8Nh2XLlmH//v3YvHkzBEGAIAgoKirCli1b+p13/PhxSCQS1NTUAAA2btyIadOmQavVori4GI8++ijM5v6xtIMHD2LRokVQq9UoLi7GY489hp6exAr8k0dEJBx2lxvdNhcAIE3JP6I2pxsNXVY43QwT8wfvbROMVz6+EPS5smwt7pw1yv/1bw9UwOkOPI29KEONe+b0NuTbeqgKVsfAUpKffGtCyLZt3rwZ5eXlmDp1Kp566ikAwAsvvIBdu3bhxz/+sf+8nTt3YuHChRg9ejQAvnX+4osvoqysDJWVlXj00Ufx05/+FP/zP/8DAKioqMBNN92EZ555Blu3bkVraytWrVqFVatWYdu2bSHbF2vIIyISDqPVCQBQK6RQyaUAgI4eB/50ogEHylvFNC1mGAwGKBQKaDQa5OfnIz8/H0uWLMGhQ4dQW1sLgGeN79mzB0uWLPG/bvXq1Vi8eDFKS0tx3XXX4ZlnnsGbb77pf37Dhg1YsmQJVq9ejfHjx2PBggV48cUXsWPHDthstrj/nMEgj4hIOExeITKo5f5jWWm8jslsd8HqcEOtkIZ93ZWLxwV9TnLJRtyPrh4b9NxLN+1WLCwL25ZQmDlzJiZPnoxdu3bhiSeewP79+9HS0oJ77rnHf84//vEPbNiwAd988w1MJhNcLhdsNhssFgs0Gg1OnjyJU6dOYefOnf7XMMb8pTCTJ0+Oie3hQh4RkXAYAwiRUiaFTsX/b3ZaHBFdVyGTBL3JpJKQz5WHeG40WLJkCXbt2gUA2LVrF2666SZkZWUBAKqrq3Hrrbdi+vTpePvtt3H06FG88sorAHipCwCYzWY88sgjOHHihP928uRJnD9/HmPHBhfbeEMeEZFwBBIiAEjXKNBtc6HL4kRh+tB1Z8mGQqEY0Lbk/vvvx3/913/h6NGjeOutt/Cb3/zG/9zRo0fh8Xjwwgsv+Mss+i7LAGD27Nn4+uuvMW5ccG8wESCPiEg4gglRhoZ/3RWhR5TolJaW4vDhw6iurkZbWxs8Hg9KS0uxYMECPPTQQ3C73bj99tv9548bNw5OpxMvvfQSKisr8fvf/76fUAHA448/jk8//RSrVq3CiRMncP78efzpT3/CqlWr4v3jDQoJEZFw5OlUKM7UIFPbv79Nuk+IvEKVaqxduxZSqRRTpkxBTk6OP0i9ZMkSnDx5EnfddRfU6l5PcMaMGdi4cSOef/55TJ06FTt37sSGDRv6XXP69OnYv38/ysvLsWjRIsyaNQvr169HYWFhXH+2oRAYY4H3KJMAk8kEg8EAo9FIUzxGABWtZrx7ogF5ehXun1cS8BybzYaqqiqUlZVF3JKCCI/B3vNQ/0YpRkQkDQUGFW6eVoAMrXzok4mkgoSISCjcHu6gSy/dTwegUcgiTmYkEhsSIiKhqOuw4J3j9QOyl4nUhoLVREJhtvPSDpk0cKuP+i4rjtV2otmUOFnBxPAhISISih67r8YscBzo9EUj9p9rRU07TehIJUiIiISix8GFSKsMXMKhV/NogilFt/BHKqIKkdvtxs9+9jOUlZVBrVZj7NixePrpp5HEGQXEMDHbeWaxr+r+UvQq7imZbCREqYSowernn38eW7Zsweuvv47LLrsMX375JZYvXw6DwYDHHntMTNMIkfAtzbRBhMiXbU0eUWohqhB9+umnuOOOO3DLLbcA4Cnuu3fvxhdffCGmWYSI9MaIgnhEXiHqtrmof3UKIerSbMGCBfjwww9RXl4OADh58iQOHjyI73znOwHPt9vtMJlM/W5EalGSqUFJpiaoEOmUMkgEAS4PQ0+AZmTJTKL1rI6nPaJ6RE888QRMJhMmTZoEqVQKt9uNZ599tl/jp75s2LABP//5z+NsJRFPbrwsf9DnJRIBaSoZTFYnTFZnUMEaqTgcjmHNoBcNJiK7d+9mRUVFbPfu3ezUqVNsx44dLDMzk23fvj3g+TabjRmNRv+trq6OAWBGozHOlhNiUtvewxq7rMzhcg94zmq1sq+//ppZrVYRLIucpUuXMgD9bhcuXGArVqxgpaWlTKVSsQkTJrBNmzYNeN0dd9zBnnnmGVZQUMBKS0sZY4wdOnSIzZgxgymVSnb55Zezd955hwFgx48f97/29OnT7KabbmJarZbl5uayBx54gLW2tga1p6qqKqDtg73nRqMxpL9RUYWoqKiIvfzyy/2OPf3002zixIkhvT7UH5JIDlxuD3O5PcO6RsA/Co+HMaddnJsntJ+nq6uLzZ8/nz388MOssbGRNTY2MpvNxtavX8+OHDnCKisr2RtvvME0Gg37wx/+4H/d0qVLWVpaGnvwwQfZmTNn2JkzZ5jRaGSZmZnsgQceYF999RV777332IQJE/oJUWdnJ8vJyWHr1q1jZ8+eZceOHWPf+ta32OLFi4Pa43K5Qn/PvYT6NyqqX2uxWAbMzZZKpfB4wpuAQKQGla1m/OVUI0qzNbhrVlH0Lux2Av98IXrXC4dF/w+QDb1UurRntY++oYiysjJ89tlnePPNN3Hvvff6j2u1Wrz22mv+JdlvfvMbCIKAV199FSqVClOmTEF9fT0efvhh/2tefvllzJo1C7/4xS/8x7Zu3Yri4mKUl5djwoQJAe2JFaIK0W233YZnn30WJSUluOyyy3D8+HFs3LgRK1asENMsQiQs3uCzVDL4HorJ5kRFixlSiYDpRelxsEw8XnnlFWzduhW1tbWwWq1wOByYOXNmv3OmTZvWLy507tw5TJ8+vV9Ljrlz5/Z7zcmTJ/Hxxx8jLS1twPesqKjAhAmhTyCJBqIK0UsvvYSf/exnePTRR9HS0oLCwkI88sgjWL9+vZhmESJhdXIh0sgHb4xvtDjxyblWZGjkoQmRVM49EzGQRt6yZM+ePVi7di1eeOEFzJ8/HzqdDr/61a9w+PDhfudptdqwr202m3Hbbbfh+eefH/BcQUFBxDZHiqhCpNPpsGnTJmzatElMM4gEwTcbbKgJHb7s6pBziQQhpOWR2Fzas/rQoUNYsGABHn30Uf+xioqKIa8zceJEvPHGG7Db7VAqlQCAI0eO9Dtn9uzZePvtt1FaWgqZLLAMBOqhHSuo1oxIGHwe0VBClKaSQRCQcrlEl/asHj9+PL788kvs27cP5eXl+NnPfjZAUAJx//33w+Px4Ec/+hHOnj2Lffv24de//jUA+EV75cqV6OjowL/8y7/gyJEjqKiowL59+7B8+XK/+ATqoR0rSIiIhMEXI1IPsTSTSgR//lAqlXpc2rP629/+Nu6++27cd999mDdvHtrb2/t5R8HQ6/X485//jBMnTmDmzJn4z//8T3+4wxc3KiwsxKFDh+B2u3HjjTdi2rRpWL16NdLT0/0bSMF6aMcC6llNJAy//7wGbd123D17FEZnDR73+OOXdbjYacVNU/MxuaD3d089qwOzc+dOLF++HEajsV8D/mhAPauJlKIoQw2tQgqdaugAr14tBzqtKeURRZMdO3ZgzJgxGDVqFE6ePInHH38c9957b9RFKFqQEBEJw+KJuSGf66vCN5IQBaSpqQnr169HU1MTCgoKcM899+DZZ58V26ygkBARScnkAj1GZ2mQrk783TAx+OlPf4qf/vSnYpsRMiRERELg8TAwBJ7eEQiDWj5gEiyRvJAQEQlBo8mGN4/UIUenxANXjhbbHCLO0PY9kRD4khllIXpEgLeRfnkrugO0jU3izeCkIxrvNQkRkRCEmlXdl+N1nThW04mOHof/mFzOl2sWC035iBe+99r33kcCLc2IhMCfVT1EMmNf9Co52s0OmKwu/zGpVIr09HS0tLQAADQaDbWTjRGMMVgsFrS0tCA9PR1Saei/u0shISISAot3jJBGEfpHMtgWvq9thU+MiNiSnp4+7FYhJEREQmDz15mFHi3QBxEiQRBQUFCA3NxcOJ2UZxRL5HL5sDwhHyRERELQW2cWvkcUbMaZVCqNyh8JEXtIiIiEIN+ggiAA6ZrQA56+qa+UXZ38kBARCcGCsdlhv8bnEVkdbthdbihl5P0kKyRERNKilElx3xXFyNAoSISSHBIiQnQYY3B7GGTS8NPaCtMTs5qcCA9KaCREx2h14qWPLuDVA5Vim0KIBHlEhOj4khklYZR3+Ggz2/F1gwkquRRzyzKjbRoRJ8gjIkTHt3WvCaO8w4fZ5sLRmk6cazJF2ywijpBHRIiONcRe1YHI0MigdnTAbFLB42EReVWE+JAQEaIT6vSOAfS0Q3/mbcxqLoeHAZav6pF22XeAIQY0EokH/cYI0YnII7J3Ayd3Q7C0Q+Wdcuqs/RKo2h8LE4kYQ0JEiE7YMSLGgPJ9XIy02Wi+7CGcz7qOX6fuC8BMxa7JBgkRITo5OgVKszXI0IbYf7qzGmg7D0ikwJQ7kZmRgXbtODQrRwPMA1QdiKm9RPQhISJE5/LRmbhrVhHG5qQNfTJjQPVB/rhwFpCWgxwdH6tcrbucH2+/AFg7Y2QtEQtIiIjkwlgHGC8CEhlQciUAnl390KIy3HXVDCCzjItV4ymRDSXCgYSIEBXGGFzuMGaqNxzn93mXAUodAEAulUCvkvNOjPnT+POt33BBIpICEiJCVJxuhpc+uoBXPr4A51CC5LAAref448JZgc/JGs+9JUsHBa2TCBIiQlR8W/ceDxt6gkfTacDjBnR5gL6g31N1HRa8e7IBByqNQNZYfrD1m1iYTMQAEiJCVPomMw7Z5L75NL8vmDngKQ9jqGgx40KLGcgezw92UBFtskBCRIhKyE3ze9oBcysgSIDcyQOe9nV4NFqd6FaP4gfNzXw5RyQ8JESEqPh7VQ/VNL/1LL/PLAPkA3sQKWVS/zZ+g1UOpOXwYHVndTTNJWIECREhKv7pHUM1zW/xClHOpKCn+JqkNXRZgYwyfpCEKCkQXYjq6+vxwAMPICsrC2q1GtOmTcOXX34ptllEnAipvMPcCvS08Uzq7AlBTxvlFaKLXVYgfTQ/aLwYNVuJ2CFq9X1nZycWLlyIxYsX429/+xtycnJw/vx5ZGRkiGkWEUcyNAqUZWuRnaYMfpJ/WTYGkKuCnlacoYEgAG3ddphVBUgDAEs74OgBFNqo2k1EF1GF6Pnnn0dxcTG2bdvmP1ZWViaiRUS8mVZkwLQiw+AntZ3n9zkTBz1NrZAiX6+CRCLAyhRI02ZzT8pYD+QE96QI8RF1afbuu+9izpw5uOeee5Cbm4tZs2bh1VdfDXq+3W6HyWTqdyNSHJuRJyYKApA5dsjT751TjHvnFPPAtaGYHzTWxdhIYriIKkSVlZXYsmULxo8fj3379uHHP/4xHnvsMbz++usBz9+wYQMMBoP/VlxcHGeLiWgzZHlHewW/1xcCCs2Q1+vXodFQxO8pTpTwCIyJV5CjUCgwZ84cfPrpp/5jjz32GI4cOYLPPvtswPl2ux12u93/tclkQnFxMYxGI/R6fVxsJqLLyx+dhyAIeHD+aOhVAaa8nvojr6Yfcw0wekHI17U63BBsnVAdfZUHuRf9P35PxBWTyQSDwTDk36ioHlFBQQGmTJnS79jkyZNRW1sb8HylUgm9Xt/vRiQvDpcHTjeDw+WBKtCARLezd/s9a1zI1z1Q3or/PVCB0+0CD2573EBPa3SMJmKCqEK0cOFCnDt3rt+x8vJyjB49WiSLiHjiK++QSQTIpQHKOzqrAY8LUBkAbU7I19Wr5WAMqOmwAjpvTVp3YxQsJmKFqEL0k5/8BJ9//jl+8Ytf4MKFC9i1axd++9vfYuXKlWKaRcQJf6/qYHVm7Rf4fdY4HqwOkZJMHktq7LLCrc3jB7ubhmUrEVtEFaIrrrgC77zzDnbv3o2pU6fi6aefxqZNm7BkyRIxzSLihK/OLOD0DsZ6A9VZQ++W9SVDI0eaUgaXh6EF3pw08ogSGtHHCd1666249dZbxTaDEIEeO/eI0pQBPoaWdt4cXyID0kvCuq4gCCjKUOObpm7UOg0oAHh2ttsFSEX/yBMBEL3Egxi59AxWee8LUhuKAGmA3bQhKMrgy7Mas5Rv+zMPr8YnEhISIkI0DGo5yrK1yNMHKO/oqOL3mZFl2hdn8rqzJpMdLooTJTzkpxKiMblAj8kFAVIwPG6gq4Y/ziiN6NoGtRyzStKRqVWAWfOBzirATEKUqJAQEYmHqYHnEMnVQFpeRJcQBAHXTszlX7TkAxdBPawTmIiWZpWV1IKTGD5Bm+X74kMZo8Patg9KmleQetoATxgTQ4i4EZEQjRs3DosXL8Ybb7wBm80WbZuIEQBjDL/5pAKvfHwBZrur/5Od3vhQxvA6MXg8DC0mG852SfhumcdFgxcTlIiE6NixY5g+fTrWrFmD/Px8PPLII/jiiy+ibRuRwthdHrg8vvKOPh9Dlx0weXN+IowP9f0eOw/X4v2vWuBUZfODtHOWkEQkRDNnzsTmzZvR0NCArVu3orGxEVdddRWmTp2KjRs3orWV6nqIwfF5QSq5FDJpn49hVx3faldnAOr0YX0PtUIKvZpv/XdJvImNPRQnSkSGtX0vk8lw9913449//COef/55XLhwAWvXrkVxcTF+8IMfoLGRslmJwFi8yYxa5SVZ1UZvwXOYSYzByE5TAAA6Jen8gJn+SSYiwxKiL7/8Eo8++igKCgqwceNGrF27FhUVFfjggw/Q0NCAO+64I1p2EimGzyPSXprM2OVtYpYenV5TWVqeo9TKvF0gaWmWkES0fb9x40Zs27YN586dw80334wdO3bg5ptvhkTCda2srAzbt29HaWlpNG0lUohumxMAoFP1+Qi6HL1Jh4boCFGmlntEjS4dP2DvBpzWgCOJCPGISIi2bNmCFStWYNmyZSgoKAh4Tm5uLn73u98NyzgidTHZuEek69sMzVTP40MqPW/9EQV8S7M2mwCm0kOwmXg+UQa1mkkkIhKiDz74ACUlJX4PyAdjDHV1dSgpKYFCocDSpUujYiSRemSn8ekdvqGIAHpbuhqKopM/BCBDq4Ag8JYjdn0OVDYTb5JGQpRQRCREY8eORWNjI3Jzc/sd7+joQFlZGdxud1SMI1KXWSUZmFVyydgoX5P7KC3LAEAuleCGyXnQqWSQdTUCXRUUJ0pAIhKiYG2uzWYzVKrgc6cIIigeNx/7A0Rtx8zH1FHeZZ7bWy5CbWMTjrCEaM2aNQB4Hc/69euh0fROVXC73Th8+DBmzpwZVQOJ1MPjYXB6PFD27VPd3cgzn+VqQJMVm2+s9SY19rTxxmtRWv4RwycsITp+/DgA7hGdPn0aCoXC/5xCocCMGTOwdu3a6FpIpBxtZjt2Hq5FukaO5Qu9ZRx9t+2jLBBmuwsXOy2QMBkmSKS8oNZmHHbCJBE9whKijz/+GACwfPlybN68maZoEBFh8m7dq+R9PCKTd1kWxfiQjxaTDX873YQcnRITNJk8qbGnjYQogYgooXHbtm0kQkTEdFq4EGVovFv3jPUKkb4w6t8vXcM9d6PVCabxLs8sbVH/PkTkhOwR3X333di+fTv0ej3uvvvuQc/du3fvsA0jUpfOHgeAXoGArQtwWPgAxLT8qH8/vUoGQeBz1GyKTKgBClgnGCELkcFg8I98MRiik2xGjEy6vB5Rus8j8lXbp+XGpLm9TCpBmlKGbpsL3dJ0EqIEJOTf+rZt2wI+Johw6bRwjyjD5xGZGvi9LvrLMh/pGgW6bS50CgbkAkBPO2+SJqG27YlARL8Fq9UKi8Xi/7qmpgabNm3C3//+96gZRqQmNqcbFu9gxV6PKHbxIR++mjYj0/IRRR4XXxISCUFEQnTHHXdgx44dAICuri7MnTsXL7zwAu644w5s2bIlqgYSqYWHMcwqScf4vDSeR+R29faSjoMQddvdgNabp9RDAetEIeIOjYsWLQIAvPXWW8jPz0dNTQ127NiBF198MaoGEqmFRiHDtRNzcet0r+j0tPQmMqozBn/xMJiYp8Ods0ZhzuhMQJvj/d4UJ0oUIooMWiwW6HS8rcLf//533H333ZBIJLjyyitRU1MTVQOJFMcXH9IXxjTTOStNiaw0b4EtbeEnHBE3z/+///s/1NXVYd++fbjxxhsBAC0tLZRfRAzKxU4LHK4+kzTiEB8aAHlECUdEQrR+/XqsXbsWpaWlmDdvHubPnw+Ae0ezZs2KqoFE6tBjd+GPX17Eb/ZXwO7ydmjwbd3HWIgYYzjbaMIXVR1wqLwxIksHjRdKECJamn3ve9/DVVddhcbGRsyYMcN//Prrr8ddd90VNeOI1KKuk++0ZmoVPFDt6Okd7xPDrXuAF2p/cq4VNqcbZVklyJHKec2ZtbM3eE2IRsTZY/n5+cjP758FO3fu3GEbRKQula09AICSTG/XBp83pMkC5LFvH6NTyWBzutFtdyFHm82/f08rCVECEJEQ9fT04LnnnsOHH36IlpYWeC5xb2kSLHEpNqcbFS1mAMDEfG//6DjHh/RqOVq77ei2uXicyCdEmBSX708EJyIh+uEPf4j9+/fjwQcfREFBgb/0gyCC8WV1J1wehmydErm+9rDd8YkP+dAp+cfdbHf1Bqxp5ywhiEiI/va3v+Gvf/0rFi5cGG17iBTkZF0XjlR3AADmj8ni/7j6VdyPiosdaao+QpRBSY2JRES7ZhkZGcjMzIy2LUSK4hvpM7MkHeNy0/hBSzsfHySV9XonMcY3Q81s6+sRdfA2tYSoRCRETz/9NNavX9+v3owgglGcqcH35xbj2gl9BMfnDekK4lZ4muZdmvU4XIBSB8iUfHyRpT0u358ITkSfgBdeeAH79u1DXl4epk2bhtmzZ/e7RcJzzz0HQRCwevXqiF5PJA4eD8Pfv2pCk9HmP1ZgUPePJcYpf6gvuXol7po1CjdPK+BZ3H17WBOiElGM6M4774yqEUeOHMH//u//Yvr06VG9LiEO55q78VWDCZVtPXjoqjLIpQH+38U5PgTw1rSl2dreA9ocPjmEMqxFJyIhevLJJ6NmgNlsxpIlS/Dqq6/imWeeidp1CfE4U28EAMwsTg8sQi5H7x9/PEs7LoVKPRKGiBfnXV1deO2117Bu3Tp0dPAdkWPHjqG+vj6s66xcuRK33HILbrjhhkhNIRIIo9WJi51WCAIwpTBI3WF3I981U+r4LY5caDHjSHUHuiwOWpolEBF5RKdOncINN9wAg8GA6upqPPzww8jMzMTevXtRW1vr71U0FHv27MGxY8dw5MiRkM632+2w2+3+r00mUyTmEzGkpp1nTxca1ND3nWvfl74V93HmWG0n6jut0KvkSM/0ekS2Ll7uIQ1iLxFzIvKI1qxZg2XLluH8+fP9JrvefPPNOHDgQEjXqKurw7//+79j586dIU+H3bBhAwwGg/9WXBz90TPE8Kjt4DupJVma4CeJEB/ykdY3qVGhBRQa7p2RVyQqEQnRkSNH8Mgjjww4PmrUKDQ1NYV0jaNHj6KlpQWzZ8+GTCaDTCbD/v378eKLL0Imk8HtHpjbsW7dOhiNRv+trq4uEvOJGOHxMNR1WAEAo4MJEWOiekRa3xa+3eU9QHGiRCCipZlSqQy4LCovL0dOTmjJaddffz1Onz7d79jy5csxadIkPP7445BKpQNeo1QqoVQqIzGZiANmhwsKmQQexpCnC+Ll2oy86l6QALrojw4aijQl/1z1E6LOGir1EJmIhOj222/HU089hTfffBMAb7FQW1uLxx9/HN/97ndDuoZOp8PUqVP7HdNqtcjKyhpwnEgO9Co5HrqqDHaXGxJJkPrD7r6jg+Ifk9H2XZoBvPIfoKWZyESc0Gg2m5GTkwOr1YprrrkG48aNg06nw7PPPhttG4kkQykb6M36ETE+BPSWedDSLLGIyCMyGAz44IMPcOjQIZw8eRJmsxmzZ88e9hb8J598MqzXE+LCGBu6E4OI8SGgb5mHNwbpEyKbCXDa4tIXiRhI2ELk8Xiwfft27N27F9XV1RAEAWVlZcjPzw/tg0ikJG4Pw9aDVcjUKnDL9AKo5AG8IrcL6G7mj0USIp1KhrtmjYJWKeOfV7mK5zLZu3mcyFAkil0jnbCWZowx3H777fjhD3+I+vp6TJs2DZdddhlqamqwbNkyahM7gunoccBsd6G52walLMjHytwcl9FBgyGTSlCarUWOTtn7T5OWZ6ITlke0fft2HDhwAB9++CEWL17c77mPPvoId955J3bs2IEf/OAHUTWSiC7dNidOXzQiR6fE+LzoZDZ3ecdIZ2oUwb1iozfdwlAU09FBYaPNBjoqKWAtImF5RLt378Z//Md/DBAhALjuuuvwxBNPYOfOnVEzjog+bg/D20cv4nBVB/5yqhEVreaoXLfL6gTQZ4x0wJO8QpReEpXvGSnVbT34oqoDzSZvdwDyiEQnLCE6deoUbrrppqDPf+c738HJkyeHbRQRO8qbu9Fpcfq//qKqIyrX7fJe06BWBD7B4+njEYmbEX+20YRDF9pw0TtVhGrOxCcsIero6EBeXl7Q5/Py8tDZ2Tlso4jYUd7cDYBXxksEAU1GGzp7HMO+rm9pFtQj6mkFXHZApgDSgn+G4kFvdrV358w3+dXRw29E3AkrRuR2uyGTBX+JVCqFy+UatlFEbGCMwe6dsnrZKF4Zr5RLIJMOP15jHGpp5vOG9EVx68gYDO2l2dUyBaBOB6xd3CtSaIO+logNYQkRYwzLli0LWmbRtzKeSDwEQcC9c4rRY3dBo5Bi8aTcqFzX42FI1/AlmUEdRIi6avl9uviFytpLc4kAHifyCVHGaHEMG8GEJURLly4d8hzaMUt8fH+I0UIiEfC9ywfJv/F4eoVI5PgQECC7GuBxorbzFLAWibA+kdu2bYuVHUQcCJRwanW40Wi0osCghloxSGnGcDA3AU4rXwKJ2ZHRy4B6M4B2zkRG3MU6EVd2Hq7Fm0fq+gWn9x6/iD+daEB9lzXi6zLGBj+hwzv5N6MUkMRI7MJA4xVch8sDhzdm1k+Ihvp5iKgTXR+dSFjsLjdau3kMr2/5RZZWiRaTHe1me+/MsTD55FwrKlrNuHJMFqaOMgw8wSdEmWMiun60UcokuHv2KGgUMsh8XQLUmbw1icvOyz1UQdrcEjGBPKIRQmcP39XSKqX9lmDZaTzI3D6MLfxOi4PPkw+E09pb6JogQiQIAkZn8TIPf7sSqay37ISWZ3GHhGiE0OnN88nQ9E84zErjO6Dt5sh3PH3JjAG37juq+FJHkwWoAnhLiYQvsZEGLsYdEqIRgi8u5Bv/7CPL6xF19Djh9oQfG3F7mN8bStcEyKpu/Ybf50wI+9qxpK7Dgi+qOlDX0WdaMQWsRYOEaITQ4c987i8WOqXM397V5zWFQ7fNCQ9jkEsFaC/ddXM5gPYK/jhnckR2x4qKVjMOXWhDdXufTGoSItEgIRohBPOIBEHwL9e6+tSghYq/xixQ1X37Bd72Q53BW8MmEAPKPID+QuTxiGDVyIV2zUYIBo0CDjdDZoDl0+WjM+B0e5CrD38wga/qPmBGdfMZfp87KbHafiBIUqM6gwet3S7A2glos0SybuRBQjRCuH1G8ETCifmR9yRSySUYlaFGnu4SEbN29W7b50+P+PqxordlbB8hkkgAbS7f5TM3kxDFERIiYlhMytdjUn6AnJuG43y3LLMM0GTG37Ah0PgLXy+Zn5eW1ytEeVNEsGxkQjGiEcBQmc8utwd1HRacbYzSCG+nFWg4xh8Xzo7ONaOMzyOyOd1wufvEg3yxLHOLCFaNXMgjGgF8XtmBUxe7MKskA3PLBnonDrcHbx29CEEAxuemQSYN7f8TYwxuDxt4ft1hvmOWlgNkj4/GjxB1lDIJZBIBLg9Dj90Ng8b7M/h6JZmbxTNuBEJCNAIwWp2wOAaO8PahlkuhkEngcHlgtDr9SY5DYbK5sO1QFdLVcixdUMp3zXragLov+AmlVydckNqHIAi4a/YoqORSpKn6/Bloc7jNjh7AbgaUkZW9EOFBS7MRgMnGd7b06sD/dwRB8GdF+3bBQrqu1QnG+OsFQeBe0Nl3AY8byBqXsN6Qj6IMDbLTlJD2nUorU/C6M4C8ojhCQjQCMHnFRa8K3tg+Xa0AmAemrvbeYYNDxJb6lXY4bcCZt/ncMoUGmHBjwnpDQ+KLE1FiY9ygpVmK4/Ywf98dfaBcH7cLaDqFCQ2HkVFfiXSjArjoXY7IlDy3RpPJvQRNlvdxBiBVoMtig9JlwihLDXDkHK9al8qAqd9N/LoyAI1GK2rbLcjWKTE2p88SLC0PaDlLHlEcISFKccw2FxgDZJIAJRjdTcDX7wKWdhgcNrSBwepivGeQx81bYnQ38VsAcpvNmNVjxyi7BjCoufhMvRvQ5cfhJxs+dR1WfFrRjimF+kuEiHbO4g0JUYrjiw/pVLL+JRid1cDptwC3kzeLH7cQx2VqqHRZmLJoDD9u7QKsHYClo/+9gxeK2pwuMEECWUYRMGYWUDgTkA4y1yzB8DVIszguaWHi2zmztPP3IYl+pmSFhCjFkUgElGRqoOu7M2Ru7RWhzDJg8u3QC0pcbTD3tvKQyvn2e1rOwIu6HGBuBz4/UAELU2HynDJAG2SeWQKT5m8Ze8mOojKNi7Ojh8eJEqC9bapDQpTijEpX47t9G9u7HMBXe7kIZYwGpn4PkMqgAgJ3VwyETAGXIMPogjyYrM7AsackoLfwNUBTt7Q8XqLS3URCFAdIiEYa1f/kSyylDphyBw8uR4BcKsFNU5MjFhQM33wzq8MNt4f138bX9REiIubQ9n2K06/ZmbkFuPglfzzxOwMGCbaZ7ThTb0TDMBrpJxNqudQvPj2Xxol0Bfy+uzHOVo1MSIhSnG2HqvDbAxW8cX7lfoB5eLfErLEDzv26wYQPvm72j6UeDJvTHVFHx0RCEAR/wHrA8swnRD1tfBlLxBRamqUwvhwixgCto5U3KhMEYMzigOf7AtXGELKrP/qmBeebzbhuUi6mFSV+zlAwbp1eCJlUQPqlcS6Vnget7Wa+PEuACbWpDHlEKUzfHCJ1w+f8YO6UoG050tWhd2rssvAWsTEbyhgn8g0qZKcpAxf60vIsbpAQpTC+HKIsaQ+EtvPcGxq9IOj5hj4ekWeQZRdjDF1WXw/s5NwxCwkSorghqhBt2LABV1xxBXQ6HXJzc3HnnXfi3LlzYpqUUviEqMRWzg9klPWOzAmATimDVCLwJd2lwds+2Jwe2J28h0/AFrFJREu3DYcr2/FNU4BeTHqvEJlIiGKNqEK0f/9+rFy5Ep9//jk++OADOJ1O3Hjjjejp6Rn6xcSQmKwuCMyFXItXiEYN3qRMIhGg9yY+GgdZnvmmfehUMshD7F2UqLSY7Pi0oh3nmgIE6H0ekbWTN3sjYoaower333+/39fbt29Hbm4ujh49iquvvlokq1KHbpsTWZYqaGAHlHlA5sCdsktJ1yjQaXGiy+JEcZAOr8GGNSYjvl0zc6CkRrmaF/haO3n72AA7jUR0SKhdM6PRCADIzAz8F2C322G3904kNZmi1No0RclKUyAd1dAoZLwOTDK09zK3LBOXj85A9iDN0XzB7Axtci/LgD5N9AMJEcCXZ9ZOHiciIYoZCeNXezwerF69GgsXLsTUqVMDnrNhwwYYDAb/rbiYtlQH4/ICFeamm/kss7zLQnpNYboaxZmaQXfDcnRKTC7QYVS6JlqmioavzMPicAcO0OtH8XtjfRytGnkkjBCtXLkSZ86cwZ49e4Kes27dOhiNRv+trq4ujhYmIW3lPIExLZcvMaLEhDwdbppaMKwxRImCWi6FIPAecBZngHa6Bm+dnukiDV2MIQmxNFu1ahX+8pe/4MCBAygqKgp6nlKphFIZ/hDAkYjbw8BavuG/4JxJIb/O5fbgm6ZumGxOzB+TNXB6a4ohkQjQKmQw213osbv8SzU/2lzePtbl4JX4ujxxDE1xRPWIGGNYtWoV3nnnHXz00UcoKysT05yUoqmjC18cO4bT9UYgZ2JYr/3H2WYcruwI2HDf6fagy+IYNM8o2eidcRYgTiSRAHrvP0cjeeCxQlSPaOXKldi1axf+9Kc/QafToamJVzobDAao1WoxTUt6HM3lEJgHTlX2oLlDlyKTSqBTyWGyOtFldfpjKD7qO61453g9stMUeHB+aZStFodvTcmDVBCCtzMxFPFKfGMdUDQnvsaNEET1iLZs2QKj0Yhrr70WBQUF/tsf/vAHMc1KCdzN3wAAXJnhT9LI8GZLd5gdA55r7+G7lpna1Fki5+pUyEpTBs+J8tWZGS8OOVCAiAxRPaKhJpASEeJyQPDOnRdyQ48P+cjRKVHTbkGr2Qagf0FrazcXp6y05M8hChldAe/jbTfzrfwEHKGd7CTMrhkRRToq4HA6YJfpoE4vCPvlvhyitu7gHtFgeUbJRmePA4cr23G8tjPwCVJ570AA48X4GTaCICFKRVq/gcPlQbtmDPTq8D0Xn8i0mu39vFaPh/mXa9kp5BGZbE58WtGOM/XG4Cell/D7rpr4GDXCICFKNdxOsPYK2J1udKhLg053HYxMrQJSiQCHywOTtXcnqcPigMvDoJBJkr7YtS++wZMmmyt4uCCjlN93VFGcKAYkRB4REUU6qsBcDhgyspCRXzowLyYEpBIBd80ahXSNvN/rG7tsAIA8vSql8ot8E04cLg9sTk/grHJ9Ee/v7Zvs4Zt9RkQFEqJUo+0cJIKAiVOvwMTxkZfAFGcOLN/IN6hw5ZisiLysREYmlSBNyZMaTTZnYCGSygBDCd/G76wmIYoytDRLJTxuoO08f5wzIeqXz9EpMX9sFi4rTN7WsMHwiatpsDa5fZdnRFQhIUolOqsBlx1OqRrOtOHN4vJ4GA5XtmPvsYuwuwLUYKUYvXGiQYQo05v5b6wF3MEbxxHhQ0KUSrTy7panbHl4+eNKHAu2HR0CEomAs40m1LRbUNnag5r2Hlxo6YbDlZqFn76s6r7B+QFoc/gIJreLixERNVJrsT+S8Xh4tT2ARmUJ4EBEgeq+TMzX4/PKdnzVYILV6UZbtx3XTMzB7JLoVfInCtOKDJiUrxt8aq0gAFnjgMaTQNsFIHNM/AxMccgjShWMtbydqVyFBsbn1Q8YkRMmUwr0kAgC6josaOu2QyGTYEqBPhrWJhx6lXzwMg8f2d7YW/t52saPIiREqUIr94acGePQ4w1zDHcmvUEjx5VjessZrh6fA5U8uccHDZuM0XwHzWYCzM1iW5My0NIsFWAMaOPxIVMaXy6oFdKoiMa8MVkoy9ZCKhGQlUJlHYE4WtOBdrMDV43P5u11AyGV8yVZazlfCvtKP4hhQR5RKmCq5wWZMgU6FHy3bLjLsr7k6lUpL0IAcOqiEV81mNAeoOtAP7K8HQ28MTli+JAQpQKtvOUHssah08bjFqlUghEvfFNJhpx0mz2eV+ObW/mNGDYkRMkOY0CLV4hyJiNHp8RlhXqUZCV/Y/t445ta22EZwiOSq3t3zFq+irFVIwOKESU7pnrA3s37KmeOQZlUhrJsrdhWJSWZWp9HNIQQAXwqStt5oPlroOwavrVPRAx5RMmOzxvKGs93c4iI8S3NOnpCEKKscVz8bUbqURQFSIiSGcaA1rP8ce4U2F1utJntcKdQY/t4kuH1iExW19DvoVQOZHuHEjSdjrFlqQ8JUTJjrPPulimBjFI0dNnw+89qsPsLKj+IBK1CCoWM/0l0D1Zz5qNgBr9v+Qpw2mJoWepDvnwy0+L1hrInAFIZ2s18BLcv1kGEhyAI+P4VxdCr5UNnWAN8uoc2G+hpA5rP0ISPYUAeUbLidgHN3h2bvCkAgNbu1OsnHW9CKvPwIQjAqNn8cf0xKvkYBiREyUpbOeCyAyo9kF4KAGg28eVBro6EKG7kTeXxIks7b5pGRAQJUbLSdIrf500FJBLYnG50ehPx8vQqEQ1LbqwONz4824x3jl8MbdyVTAkUzOSPaz+LqW2pDAlRMmIz8iZoAJA/DUDvskyvlgdudUqEhEwq4Ey9CdVtFpgDjaAORPFcnmndVcdvRNiQECUjDcd5PCK9xD/sr6WblmXRQC6VIFPLM6x9S90hUen9/xBQcyhGlqU2JETJhtvJhQjot0tTmqXFgrFZmJivE8mw1KEwXQ0AuNhpDf1FJVcCgoT3s6ae1mFDQpRsNJ3mOSvq9N4qcPDdnnljsjAhj4RouBRl8Dq9+q4whEid0buDduEfvGMmETIkRMmExw3UfcEfj5oDSOjXFwsK03mwv7XbDpszjMEBpVcBchXPK2o8HiPrUhP6JCcTTacAayeg0PRm9QKo67CgvLkbVkfqT9uIBzqVHOkaORgL0yuSq4HSRfxx5SeAtSsW5qUkJETJgtsFVHsDoSULeMGll8NVHfjrqUZ83WgSybjUY3SWBjqVDC53mEmKhbMBwyjA5QDOvUdJjiFCQpQs1H7G230odUDhLP9hq8ONi50WAMC4nDSxrEs5rhqXg4euKgs/+C+RAJNu5Z0QOmuAmk9jY2CKQUKUDPS09ybLjbu+X7uPCy1mMAbk6pUwaKgrY7RQyCQQIu0xpMkExn2LP6464B9sQASHhCjRcTuBr/+PB6qzxgI5k/xPMcZwqr4LADCRdstigtvD/B5nWBTOBEZdzh9//afeBFQiICREiQxjQPn7gLmFB6gn3NSvE2B9lxUtJjtkEiEl59GLjc3pxu8OVuLto/UwDtXHOhDjruf9rT0u4PQfeUdHIiAkRIkKYzwfpekMF5/Jt/EMXv/TDJ9eaAcATC7QU1lHDFDJpchOU8LDGA5VtIV/AYkUmHIn92TdLuDM20D1QcoxCkBCCNErr7yC0tJSqFQqzJs3D1988YXYJomL08o/tBe/5F9PumXAeGOXh0GtkEIuFTCvzxBEIrpcNT4bggCca+rG+ebu8C8glQFTv8uXaYwBVf8Ejm2nmrRLEF2I/vCHP2DNmjV48skncezYMcyYMQPf/va30dLSIrZp8cflAC4eBb74LXfjJVLuCfnqmPogl0pw6/QC3D9vNHQqClLHilydCpePzgAA7PuqCRWt5vAvIpECE24EJt/KEx67m4HjbwDHd/KeUq4QemSnOAILqddB7Jg3bx6uuOIKvPzyywAAj8eD4uJi/Nu//RueeOKJQV9rMplgMBhgNBqh1yfRTHbGeBDaaeGV9OYW3va1o5IfB3jnv0m3APpCeDwMZocLjV02nGvuxo1T8mj0cxzxeBjePdmAqrYeAMD4vDTcMq0gsl01u5nvpDWf4RsQACCRAfoCQD8K0GTxXTdFGk+QlCqSekJIqH+joraKdTgcOHr0KNatW+c/JpFIcMMNN+Czzwb2drHb7bDb7f6vTaYQE/g6qni8JRB+HWYBjvU5HuqxINfs7HGgss0MMAYJc0HwuPizjJ9VkqnhnRXVGWg2TMe7zVnwHLeCoQJ2pweePt/rtEGFK0ppORYvJBIBt80oxIHzrThZ1wWn2+MXIcYYfneQF7lKBAESgZ/vk46iTA0WT8z1X2vniXZ42GWQS0pQ6jqPK7XNPFs+WAsRQcIbr0mk/LEg7X0cbcIVvGn39otbDgdRhaitrQ1utxt5eXn9jufl5eGbb74ZcP6GDRvw85//PPxv5LLz+h8RYQ4HHNaefsc8ghROqQY98iyk549H9sTpgC4ftg4LzBfrAfSWbEgEAZlpCpRmaTAhl7bq441UImDxxFxMG2Xol23tcHvQbQvet8ig6d8/vK3b4f2nooBHfzmunFvsFaJaoLsJsHbwrx093GNiHv75TURY9ILuSdU8f926dVizZo3/a5PJhOLi4qFfmF4MzPj+JYrf57H/eKBjfY6HeizANVVOF0p6vMsumRyQayBI5BAkAgQI0KtlgIL/OvINKjxw5WgIAn+1Ui6FRi6FRJK8LnqqcGk/cLlEggeuHA23h8HD+I2xXqdYpejvudw9e5T/OYVMwj8nmkx/Xyk/jPFtf6eVL9eZV5R84hRQBEKMskQrGqOI3iBPUYUoOzsbUqkUzc3N/Y43NzcjPz9/wPlKpRJKZQSNvxRaILMsUjOjggaAJiO0c5UyKXJ0FANKBiQSATlhNKMrzgxxFLgg8CWZdGRsRIi6a6ZQKHD55Zfjww8/9B/zeDz48MMPMX/+fBEtIwginoi+NFuzZg2WLl2KOXPmYO7cudi0aRN6enqwfPlysU0jCCJOiC5E9913H1pbW7F+/Xo0NTVh5syZeP/99wcEsAmCSF1EzyMaDkmbR0QQI4RQ/0ZFz6wmCIIgISIIQnRIiAiCEB3Rg9XDwRfeCrnUgyCIuOL72xwqFJ3UQtTdzdsyhJRdTRCEaHR3d8NgCN68L6l3zTweDxoaGqDT6SLvLxwlfOUmdXV1tIPXB3pfgjMS3hvGGLq7u1FYWAjJIHP4ktojkkgkKCoqEtuMfuj1+pT9UA0Hel+Ck+rvzWCekA8KVhMEITokRARBiA4JUZRQKpV48sknI+sOkMLQ+xIcem96SepgNUEQqQF5RARBiA4JEUEQokNCRBCE6JAQEQQhOiREYRDORNpXX30VixYtQkZGBjIyMnDDDTek7ATbSCf17tmzB4Ig4M4774ytgSIS7nvT1dWFlStXoqCgAEqlEhMmTMB7770XJ2tFhBEhsWfPHqZQKNjWrVvZV199xR5++GGWnp7OmpubA55///33s1deeYUdP36cnT17li1btowZDAZ28eLFOFseW8J9X3xUVVWxUaNGsUWLFrE77rgjPsbGmXDfG7vdzubMmcNuvvlmdvDgQVZVVcU++eQTduLEiThbHn9IiEJk7ty5bOXKlf6v3W43KywsZBs2bAjp9S6Xi+l0Ovb666/HykRRiOR9cblcbMGCBey1115jS5cuTVkhCve92bJlCxszZgxzOBzxMjFhoKVZCPgm0t5www3+Y4NNpA2ExWKB0+lEZmbqTGiN9H156qmnkJubi4ceeigeZopCJO/Nu+++i/nz52PlypXIy8vD1KlT8Ytf/AJutzvg+alEUhe9xotwJ9IG4vHHH0dhYWG/D2ayE8n7cvDgQfzud7/DiRMn4mCheETy3lRWVuKjjz7CkiVL8N577+HChQt49NFH4XQ68eSTT8bDbNEgIYoDzz33HPbs2YNPPvkEKpVKbHNEo7u7Gw8++CBeffVVZGdni21OwuHxeJCbm4vf/va3kEqluPzyy1FfX49f/epXJERE+BNp+/LrX/8azz33HP7xj39g+vTpsTQz7oT7vlRUVKC6uhq33Xab/5jHw0cny2QynDt3DmPHjo2t0XEiks9MQUEB5HI5pNLeKb+TJ09GU1MTHA4HFApFTG0WE4oRhUCkE2l/+ctf4umnn8b777+POXPmxMPUuBLu+zJp0iScPn0aJ06c8N9uv/12LF68GCdOnEipTpuRfGYWLlyICxcu+MUZAMrLy1FQUJDSIgSAtu9DZc+ePUypVLLt27ezr7/+mv3oRz9i6enprKmpiTHG2IMPPsieeOIJ//nPPfccUygU7K233mKNjY3+W3d3t1g/QkwI9325lFTeNQv3vamtrWU6nY6tWrWKnTt3jv3lL39hubm57JlnnhHrR4gbJERh8NJLL7GSkhKmUCjY3Llz2eeff+5/7pprrmFLly71fz169GgGYMDtySefjL/hMSac9+VSUlmIGAv/vfn000/ZvHnzmFKpZGPGjGHPPvssc7lccbY6/lAbEIIgRIdiRARBiA4JEUEQokNCRBCE6JAQEQQhOiREBEGIDgkRQRCiQ0JEEITokBARScX27duRnp7u//q///u/MXPmTP/Xy5YtS+mOj6kKCRERkGXLlkEQBPzrv/7rgOdWrlwJQRCwbNmyfudHWwBKS0uxadOmfsfuu+8+lJeXB33N5s2bsX37dv/X1157LVavXh1Vu4joQ0JEBKW4uBh79uyB1Wr1H7PZbNi1axdKSkpEsUmtViM3Nzfo8waDoZ/HRCQHJEREUGbPno3i4mLs3bvXf2zv3r0oKSnBrFmzhnXtQJ7KnXfe6feyrr32WtTU1OAnP/kJBEGAIAgABi7NLqWvZ7Zs2TLs378fmzdv9l+jqqoK48aNw69//et+rztx4gQEQcCFCxeG9XMRkUFCRAzKihUrsG3bNv/XW7duxfLly2P+fffu3YuioiI89dRTaGxsRGNjY9jX2Lx5M+bPn4+HH37Yf42SkpIBPxMAbNu2DVdffTXGjRsXrR+BCAMSImJQHnjgARw8eBA1NTWoqanBoUOH8MADD8T8+2ZmZkIqlUKn0yE/P3/IBnSBMBgMUCgU0Gg0/mtIpVIsW7YM586d84/2cTqd2LVrF1asWBHtH4MIEerQSAxKTk4ObrnlFmzfvh2MMdxyyy1J3+a1sLAQt9xyC7Zu3Yq5c+fiz3/+M+x2O+655x6xTRuxkEdEDMmKFSuwfft2vP7661HzGiQSCS7tQON0OqNy7VD44Q9/6A/Eb9u2Dffddx80Gk3cvj/RHxIiYkhuuukmOBwOOJ1OfPvb347KNXNycvrFfdxuN86cOdPvHIVCMexROsGucfPNN0Or1WLLli14//33aVkmMrQ0I4ZEKpXi7Nmz/sfBMBqNA8YEZWVlBexFfd1112HNmjX461//irFjx2Ljxo3o6urqd05paSkOHDiA73//+1AqlREtCUtLS3H48GFUV1cjLS0NmZmZkEgk/ljRunXrMH78+EF7jxOxhzwiIiT0ej30ev2g53zyySeYNWtWv9vPf/7zgOeuWLECS5cuxQ9+8ANcc801GDNmDBYvXtzvnKeeegrV1dUYO3YscnJyIrJ77dq1kEqlmDJlCnJyclBbW+t/7qGHHoLD4YjLLiAxONQqlhix/POf/8T111+Purq6AYMQifhCQkSMOOx2O1pbW7F06VLk5+dj586dYps04qGlGTHi2L17N0aPHo2uri788pe/FNscAuQREQSRAJBHRBCE6JAQEQQhOiREBEGIDgkRQRCiQ0JEEITokBARBCE6JEQEQYgOCRFBEKJDQkQQhOj8f5iP9RozHzt6AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:32:13.769448Z", + "iopub.status.busy": "2024-03-01T12:32:13.769181Z", + "iopub.status.idle": "2024-03-01T12:32:13.986830Z", + "shell.execute_reply": "2024-03-01T12:32:13.985920Z" + }, + "papermill": { + "duration": 0.24063, + "end_time": "2024-03-01T12:32:13.988942", + "exception": false, + "start_time": "2024-03-01T12:32:13.748312", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAArd0lEQVR4nO3deVRUV6Iu8K+KoRgERJFRlMkWjUEUlqhtOg4IJC9G27zEmIEhtvbV2Dd2tW1COuJC7XA7MYaYa+S2N4hDJ9rpNnbfxKh0RXMdcMKgaAQDgkMEFBVKIBZF1X5/+DhJhSqsUrDg1PdbqxY5u/bZ7M2xvpw6wz4KIYQAEZEMKe3dASKi7sKAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGTL2d4d6ImMRiOuXLkCLy8vKBQKe3eHiH5ECIFbt24hODgYSmXn+2gMODOuXLmC0NBQe3eDiDpx6dIlDBw4sNM6DDgzvLy8ANz5A3p7e9u5N91Hr9djz549SEpKgouLi727Q/fJUbanVqtFaGio9DntDAPOjPavpd7e3rIPOA8PD3h7e8v6A+EoHG17WnP4iCcZiEi2GHBEJFsMOCKSLQYcEckWA46IZIsBR0SyxctEiHqplpYWlJWVSctN3+twqLQSvn7H0cddZVI3OjoaHh4eD7qLdseAI+qlysrKEBcX16H8LTN1i4uLMXr06O7vVA/DgCPqpaKjo1FcXCwtl9c0QP1JKVY//TCGBvXtUNcRMeCIeikPDw+TvTLlhetQ7f8ew0aMROzg/nbsWc/BkwxEJFsMOCKSLQYcEckWA46IZIsBR0SyxYAjItliwBGRbDHgiEi2GHBEJFsMOCKSLQYcEckWA46IZIsBR0SyxYAjItliwBGRbNk94NauXYuwsDC4ubkhISEBR48etVi3oKAACoXC5OXm5mZSJz09vUOdlJSU7h4GEfVAdp3wctu2bVCr1cjLy0NCQgJyc3ORnJyM8vJy+Pv7m13H29sb5eXl0rJCoehQJyUlBRs2bJCWVSpVhzpEJH923YNbvXo15s6di4yMDAwfPhx5eXnw8PBAfn6+xXUUCgUCAwOlV0BAQIc6KpXKpI6vr293DoOIeii77cG1traiuLgYmZmZUplSqURiYiKKioosrtfU1ITBgwfDaDRi9OjRePPNN/HQQw+Z1Nm3bx/8/f3h6+uLyZMnY+XKlejf3/IUzjqdDjqdTlrWarUAAL1eD71ef69D7PHaxybnMTqStrY26aect6ktY7NbwNXX18NgMHTYAwsICDB5FNqPDR06FPn5+YiJiUFjYyNWrVqF8ePH48yZMxg4cCCAO19PZ86cifDwcFRWVuL111/HY489hqKiIjg5OZltNycnB9nZ2R3K9+zZ4xCPWissLLR3F6gLXGoCAGccPnwY3522d2+6T0tLi9V1FUII0Y19sejKlSsICQnBoUOHMG7cOKl8yZIl+Oqrr3DkyJG7tqHX6zFs2DDMnj0bK1asMFvn/PnziIyMxL/+9S9MmTLFbB1ze3ChoaGor6+Ht7e3jSPrPfR6PQoLCzF16lS4uLjYuzt0n05evIH/u/44/jY3HiMH9bN3d7qNVquFn58fGhsb7/r5tNsenJ+fH5ycnFBXV2dSXldXh8DAQKvacHFxwahRo1BRUWGxTkREBPz8/FBRUWEx4FQqldkTES4uLg7xwXeUccqds7Oz9FPO29OWsdntJIOrqyvi4uKg0WikMqPRCI1GY7JH1xmDwYDS0lIEBQVZrHP58mVcv3690zpEJE92PYuqVquxfv16bNy4EWfPnsX8+fPR3NyMjIwMAEBqaqrJSYjly5djz549OH/+PE6cOIEXXngBFy5cwK9+9SsAd05A/P73v8fhw4dRXV0NjUaD6dOnIyoqCsnJyXYZIxHZj12vg5s1axauXbuGrKws1NbWIjY2Frt27ZJOPFy8eBFK5Q8ZfPPmTcydOxe1tbXw9fVFXFwcDh06hOHDhwMAnJyccOrUKWzcuBENDQ0IDg5GUlISVqxYwWvhiByQ3U4y9GRarRY+Pj5WHcTszfR6PXbu3InHH39c1sdsHEXJheuYse4wdswfK+sn29vy+bT7rVpERN2FAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki27PviZHqyWlhaUlZVJy03f63CotBK+fsfRx/2HB2NHR0fDw8PDHl0k6lJ234Nbu3YtwsLC4ObmhoSEBBw9etRi3YKCAigUCpOXm5ubSR0hBLKyshAUFAR3d3ckJibi22+/7e5h9AplZWWIi4uTXo9OGI+3lv4Oj04Yb1L+4xAk6s3suge3bds2qNVq5OXlISEhAbm5uUhOTkZ5eTn8/f3NruPt7Y3y8nJpWaFQmLz/1ltvYc2aNdi4cSPCw8OxdOlSJCcn45tvvukQho4mOjoaxcXF0nJ5TQPUn5Ri9dMPY2hQX5N6RLIg7GjMmDHi5ZdflpYNBoMIDg4WOTk5Zutv2LBB+Pj4WGzPaDSKwMBA8fbbb0tlDQ0NQqVSiY8//tjqfjU2NgoAorGx0ep1eqOvq+vF4Fc/E19X19u7K9QFHGV72vL5tNseXGtrK4qLi5GZmSmVKZVKJCYmoqioyOJ6TU1NGDx4MIxGI0aPHo0333wTDz30EACgqqoKtbW1SExMlOr7+PggISEBRUVFePbZZ822qdPpoNPppGWtVgsA0Ov10Ov19zXOnqytrU36KedxOgpH2Z62jM1uAVdfXw+DwYCAgACT8oCAAIvHgIYOHYr8/HzExMSgsbERq1atwvjx43HmzBkMHDgQtbW1Uhs/bbP9PXNycnKQnZ3doXzPnj2yPth+qQkAnHH48GF8d9revaH75Sjbs6Wlxeq6veos6rhx4zBu3Dhpefz48Rg2bBj+67/+CytWrLjndjMzM6FWq6VlrVaL0NBQJCUlwdvb+7763JOdvHgDKD2OsWPHYuSgfvbuDt0nR9me7d+wrGG3gPPz84OTkxPq6upMyuvq6hAYGGhVGy4uLhg1ahQqKioAQFqvrq4OQUFBJm3GxsZabEelUkGlUnUod3FxgYuLi1V96Y2cnZ2ln3Iep6NwlO1py9jsdpmIq6sr4uLioNFopDKj0QiNRmOyl9YZg8GA0tJSKczCw8MRGBho0qZWq8WRI0esbpOI5MOuX1HVajXS0tIQHx+PMWPGIDc3F83NzcjIyAAApKamIiQkBDk5OQCA5cuXY+zYsYiKikJDQwPefvttXLhwAb/61a8A3LlkZNGiRVi5ciWGDBkiXSYSHByMGTNm2GuYRGQndg24WbNm4dq1a8jKykJtbS1iY2Oxa9cu6STBxYsXoVT+sJN58+ZNzJ07F7W1tfD19UVcXBwOHTqE4cOHS3WWLFmC5uZmzJs3Dw0NDZgwYQJ27drl8NfAETkihRBC2LsTPY1Wq4WPjw8aGxtlfZKh5MJ1zFh3GDvmj0Xs4P727g7dJ0fZnrZ8Pu1+qxYRUXdhwBGRbDHgiEi2GHBEJFu96k4GIkdXVd+MZl2b2fcqrzVLP9sv+jXHU+WMcD/PbulfT8OAI+olquqbMWnVvrvW+93fSu9aZ+/iiQ4Rcgw4ol6ifc8td1Ysovz7dHz/ex0+21eEJyaOg6d7x1sPAaDiahMWbSuxuBcoNww4ol4myr8PRoT4dCjX6/WoHQCMHuwr63tRbcGTDEQkWww4IpItmwPu/Pnz3dEPIqIuZ3PARUVFYdKkSdiyZQtu377dHX0iIuoSNgfciRMnEBMTA7VajcDAQPz617/u9FF/RET2YnPAxcbG4r333sOVK1eQn5+PmpoaTJgwASNGjMDq1atx7dq17ugnEZHN7vkkg7OzM2bOnIlPPvkEf/rTn1BRUYHFixcjNDQUqampqKmp6cp+EhHZ7J4D7vjx41iwYAGCgoKwevVqLF68GJWVlSgsLMSVK1cwffr0ruwnEZHNbL7Qd/Xq1diwYQPKy8vx+OOPY9OmTXj88celmXfDw8NRUFCAsLCwru4rEZFNbA64devW4aWXXkJ6errJk6t+zN/fHx9++OF9d47uT2c3ZgO8OZvkz+aAKywsxKBBg0yelQAAQghcunQJgwYNgqurK9LS0rqsk2Q7a2/MBnhzNsmXzQEXGRmJmpoa+Pv7m5TfuHED4eHhMBgMXdY5und3uzEb4M3ZJH82B5ylZ9Q0NTXxyVU9kKUbswHenE3yZ3XAqdVqAHeePZqVlQUPDw/pPYPBgCNHjnT69HgiogfN6oD7+uuvAdzZgystLYWrq6v0nqurK0aOHInFixd3fQ+JiO6R1QG3d+9eAEBGRgbee+89WT8vlIjkweYLfTds2NCl4bZ27VqEhYXBzc0NCQkJVt/XunXrVigUCsyYMcOkPD09HQqFwuSVkpLSZf0lot7Dqj24mTNnoqCgAN7e3pg5c2andbdv3271L9+2bRvUajXy8vKQkJCA3NxcJCcno7y8vMNZ2h+rrq7G4sWL8cgjj5h9PyUlBRs2bJCWVSrzZwiJSN6s2oPz8fGBQqGQ/ruzly1Wr16NuXPnIiMjA8OHD0deXh48PDyQn59vcR2DwYDnn38e2dnZiIiIMFtHpVIhMDBQevn6+trULyKSB6v24H68N/Tj/74fra2tKC4uRmZmplSmVCqRmJiIoqIii+stX74c/v7+mDNnDvbv32+2zr59++Dv7w9fX19MnjwZK1euRP/+/S22qdPpoNPppGWtVgvgzmUUer3e1qH1CG1tbdJPS2NoL+9sjNa0Qw/G3baFo2xPW/ptt4fO1NfXw2AwICAgwKQ8ICAAZWVlZtc5cOAAPvzwQ5SUlFhsNyUlBTNnzkR4eDgqKyvx+uuv47HHHkNRURGcnJzMrpOTk4Ps7OwO5Xv27DG5HKY3udQEAM44cOAALpi/zldSWFjYJe1Q9zp/qxVKt5v4dP9fUeRuud7GLzZafK/2e0Dp5oS9+7/EBS9Xi/V6spaWFqvrWhVwo0aNkr6i3s2JEyes/uW2uHXrFl588UWsX78efn5+Fus9++yz0n8//PDDiImJQWRkJPbt24cpU6aYXSczM1O6zg+4swcXGhqKpKSkXnu2+MwVLVaVHsaECRPwULD5Mej1ehQWFmLq1KkWL/S1ph16MD4vPw5Pw3JsNwBouvd2PMOB8Ng/4/Gh8V3Wtwep/RuWNawKuJ+eqewKfn5+cHJyQl1dnUl5XV0dAgMDO9SvrKxEdXU1pk2bJpUZjUYAd+amKy8vR2RkZIf1IiIi4Ofnh4qKCosBp1KpzJ6IcHFx6bVX+LffPO/s7HzXMXQ2Tlvaoe4V6h2B5qrf4L1ZsYg0c/tdW1sbDh44iJ9P+LnFyRMqrzbhlW0lCJ0U0Wu3py39tirgli1bds+dscTV1RVxcXHQaDRSgBqNRmg0GixcuLBD/ejoaJSWmt4U/sYbb+DWrVt47733EBoaavb3XL58GdevX7c48wlRb6FycoPxdgjCvYdieH/zz0Wtcq7CsH7DLIaA8XYjjLevQeXkGLdV2vXBz2q1GmlpaYiPj8eYMWOQm5uL5uZmZGRkAABSU1MREhKCnJwcuLm5YcSIESbr9+3bFwCk8qamJmRnZ+Opp55CYGAgKisrsWTJEkRFRSE5OfmBjo2I7M+qgOvXrx/OnTsHPz8/+Pr6dno87saNG1b/8lmzZuHatWvIyspCbW0tYmNjsWvXLunEw8WLFztMy9QZJycnnDp1Chs3bkRDQwOCg4ORlJSEFStW8Fo4IgdkVcC9++678PLykv7b2hMO1li4cKHZr6TAncs9OlNQUGCy7O7ujt27d3dRz4iot7Mq4H48eWV6enp39YW6kM5wG0q371ClLYfSzfz1HW1tbbjSdgVnb5y1eFC6StsEpdt30BluA7DtQm4ie7P5GJyTk5PZCS+vX78Of39/TnjZQ1xpvgDP8PfxuhW39n6w64NO3/cMB640xyIOAZ3WI+ppumzCS51OZzKFEtlXsOfgTi8pAGy7rCB40uDu7C5Rt7A64NasWQPgzoSX//3f/40+fX740BgMBvzv//4voqOju76HdE/udkkBwMsKSP6sDrh3330XwJ09uLy8PJPbnlxdXREWFoa8vLyu7yER0T2yOuCqqqoAAJMmTcL27ds5QwcR9Xg2H4Nrn9mXiKinszngXnrppU7f72wuNyKiB8nmgLt586bJsl6vx+nTp9HQ0IDJkyd3WceIiO6XzQH36aefdigzGo2YP3++2dk8iIjsxeaHzphtRKmEWq2WzrQSEfUEXRJwwJ352tqnQyYi6gls/or645lvgTvXxdXU1ODzzz83uWeViMjebA649ifct1MqlRgwYADeeeedu55hJSJ6kHgdHBHJVpcdgyMi6mkYcEQkWww4IpItBhwRyVaXBdzly5cxb968rmqOiOi+dVnAXb9+HR9++GFXNUdEdN/4FZWIZIsBR0SyxYAjItmy+k6GmTNndvp+Q0PDPXVg7dq1ePvtt1FbW4uRI0fi/fffx5gxY+663tatWzF79mxMnz4dO3bskMqFEFi2bBnWr1+PhoYG/PznP8e6deswZMiQe+ofEfVeVu/B+fj4dPoaPHgwUlNTbfrl27Ztg1qtxrJly3DixAmMHDkSycnJuHr1aqfrVVdXY/HixXjkkUc6vPfWW29hzZo1yMvLw5EjR+Dp6Ynk5GTcvn3bpr4RUe9n9R7chg0buvyXr169GnPnzkVGRgYAIC8vD59//jny8/Px2muvmV3HYDDg+eefR3Z2Nvbv32+y5yiEQG5uLt544w1Mnz4dALBp0yYEBARgx44dePbZZ7t8DETUc9l8s31XaW1tRXFxMTIzM6UypVKJxMREFBUVWVxv+fLl8Pf3x5w5c7B//36T96qqqlBbW4vExESpzMfHBwkJCSgqKrIYcDqdDjqdTlrWarUA7kzHrtfr72l89tY+N19bW5vFMbSXdzZGa9qhB+Nu28JRtqct/bY64KydCsnah87U19fDYDAgICDApDwgIABlZWVm1zlw4AA+/PBDlJSUmH2/trZWauOnbba/Z05OTg6ys7M7lO/ZswceHh6dDaPHutQEAM44cOAALph/sL2ksLCwS9qh7mXttpD79mxpabG6rtUBV1BQgMGDB2PUqFEQQtxTx+7HrVu38OKLL2L9+vXw8/Pr0rYzMzNNJvLUarUIDQ1FUlISvL29u/R3PShnrmixqvQwJkyYgIeCzY9Br9ejsLAQU6dOtfhke2vaoQfjbtvCUbZn+zcsa1gdcPPnz8fHH3+MqqoqZGRk4IUXXkC/fv3uqYMA4OfnBycnJ9TV1ZmU19XVITAwsEP9yspKVFdXY9q0aVKZ0WgEADg7O6O8vFxar66uDkFBQSZtxsbGWuyLSqWCSqXqUO7i4mLxH0pP5+zsLP282xg6G6ct7VD3snZbyH172tJvq8+irl27FjU1NViyZAn+53/+B6GhoXjmmWewe/fue9qjc3V1RVxcHDQajVRmNBqh0Wgwbty4DvWjo6NRWlqKkpIS6fXkk09i0qRJKCkpQWhoKMLDwxEYGGjSplarxZEjR8y2SUTyZtNJBpVKhdmzZ2P27Nm4cOECCgoKsGDBArS1teHMmTPo08e2L/VqtRppaWmIj4/HmDFjkJubi+bmZumsampqKkJCQpCTkwM3NzeMGDHCZP2+ffsCgEn5okWLsHLlSgwZMgTh4eFYunQpgoODMWPGDJv61tt9rzcAAE5/12ixTvP3Ohy/BgReuAlP9457sABQcbWpW/pH9CDc81lUpVIJhUIBIQQMBsM9tTFr1ixcu3YNWVlZqK2tRWxsLHbt2iWdJLh48SKUSttutliyZAmam5sxb948NDQ0YMKECdi1axfc3NzuqY+9VeX/D6bXtpfepaYzNlccu2t7niq7nXAnumc2/avV6XTYvn078vPzceDAATzxxBP4z//8T6SkpNgcRO0WLlyIhQsXmn1v3759na5bUFDQoUyhUGD58uVYvnz5PfVHLpIeunM8MtK/D9xdnMzWKa9pxO/+Vop3/u/DGBrkY7EtT5Uzwv08u6WfRN3J6oBbsGABtm7ditDQULz00kv4+OOPu/xsJnWdfp6ueHbMoE7rtF8TFTnAEyNCLAcc9Qx3O+zAQw4dWR1weXl5GDRoECIiIvDVV1/hq6++Mltv+/btXdY5IvqBdYcdeMjhx6weZWpqKhQKRXf2hYg6cbfDDjzk0JFNF/oSkf3c7bADDzl0xPngiEi2GHBEJFsMOCKSLQYcEckWA46IZIsBR0SyxYAjItliwBGRbDHgiEi2GHBEJFsMOCKSLQYcEckWA46IZIsBR0SyxYAjItliwBGRbDHgiEi2GHBEJFsMOCKSLQYcEcmW3QNu7dq1CAsLg5ubGxISEnD06FGLdbdv3474+Hj07dsXnp6eiI2NxebNm03qpKenQ6FQmLxSUlK6exhE1APZ9eGI27Ztg1qtRl5eHhISEpCbm4vk5GSUl5fD39+/Q/1+/frhD3/4A6Kjo+Hq6orPPvsMGRkZ8Pf3R3JyslQvJSUFGzZskJZVKvMPwSUiebPrHtzq1asxd+5cZGRkYPjw4cjLy4OHhwfy8/PN1p84cSJ++ctfYtiwYYiMjMQrr7yCmJgYHDhwwKSeSqVCYGCg9PL19X0QwyGiHsZue3Ctra0oLi5GZmamVKZUKpGYmIiioqK7ri+EwJdffony8nL86U9/Mnlv37598Pf3h6+vLyZPnoyVK1eif//+FtvS6XTQ6XTSslarBQDo9Xro9Xpbh9ZrtD9Hs62tTdbjdBSOsj1tGZvdAq6+vh4GgwEBAQEm5QEBASgrK7O4XmNjI0JCQqDT6eDk5IQPPvgAU6dOld5PSUnBzJkzER4ejsrKSrz++ut47LHHUFRUBCenjk8DB4CcnBxkZ2d3KN+zZw88PDzucYQ936UmAHDG4cOH8d1pe/eG7pejbM+Wlhar69r1GNy98PLyQklJCZqamqDRaKBWqxEREYGJEycCAJ599lmp7sMPP4yYmBhERkZi3759mDJlitk2MzMzoVarpWWtVovQ0FAkJSXB29u7W8djTycv3gBKj2Ps2LEYOaifvbtD98lRtmf7Nyxr2C3g/Pz84OTkhLq6OpPyuro6BAYGWlxPqVQiKioKABAbG4uzZ88iJydHCrifioiIgJ+fHyoqKiwGnEqlMnsiwsXFBS4uLlaOqPdxdnaWfsp5nI7CUbanLWOz20kGV1dXxMXFQaPRSGVGoxEajQbjxo2zuh2j0Why/OynLl++jOvXryMoKOi++ktEvY9dv6Kq1WqkpaUhPj4eY8aMQW5uLpqbm5GRkQEASE1NRUhICHJycgDcOVYWHx+PyMhI6HQ67Ny5E5s3b8a6desAAE1NTcjOzsZTTz2FwMBAVFZWYsmSJYiKijK5jISIHINdA27WrFm4du0asrKyUFtbi9jYWOzatUs68XDx4kUolT/sZDY3N2PBggW4fPky3N3dER0djS1btmDWrFkAACcnJ5w6dQobN25EQ0MDgoODkZSUhBUrVvBaOCIHpBBCCHt3oqfRarXw8fFBY2OjrE8ylFy4jhnrDmPH/LGIHWz5MhrqHRxle9ry+bT7rVpERN2FAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJVq97qhbdu5aWFpNHMpbXNEBXW4Gzp91hvN5XKo+Ojpb14xLJcTDgHEhZWRni4uI6lD+30XS5uLgYo0ePfkC9Iuo+DDgHEh0djeLiYmm56XsdPt9bhP8zaRz6uKtM6hHJAQPOgXh4eJjsmen1etysv4pxY+Jl/RxNclw8yUBEssWAIyLZYsARkWwx4IhItuwecGvXrkVYWBjc3NyQkJCAo0ePWqy7fft2xMfHo2/fvvD09ERsbCw2b95sUkcIgaysLAQFBcHd3R2JiYn49ttvu3sYRNQD2TXgtm3bBrVajWXLluHEiRMYOXIkkpOTcfXqVbP1+/Xrhz/84Q8oKirCqVOnkJGRgYyMDOzevVuq89Zbb2HNmjXIy8vDkSNH4OnpieTkZNy+fftBDYuIegphR2PGjBEvv/yytGwwGERwcLDIycmxuo1Ro0aJN954QwghhNFoFIGBgeLtt9+W3m9oaBAqlUp8/PHHVrfZ2NgoAIjGxkar1+mNWltbxY4dO0Rra6u9u0Jd4OvqejH41c/E19X19u5Kt7Ll82m3PbjW1lYUFxcjMTFRKlMqlUhMTERRUdFd1xdCQKPRoLy8HL/4xS8AAFVVVaitrTVp08fHBwkJCVa1SUTyYrcLfevr62EwGBAQEGBSHhAQYHK/5E81NjYiJCQEOp0OTk5O+OCDDzB16lQAQG1trdTGT9tsf88cnU4HnU4nLWu1WgB3LoTV6/W2DawXaR+bnMfoSNra2qSfct6mtoyt193J4OXlhZKSEjQ1NUGj0UCtViMiIgITJ0685zZzcnKQnZ3doXzPnj0OcdN5YWGhvbtAXeBSEwA44/Dhw/jutL17031aWlqsrmu3gPPz84OTkxPq6upMyuvq6hAYGGhxPaVSiaioKABAbGwszp49i5ycHEycOFFar66uDkFBQSZtxsbGWmwzMzMTarVaWtZqtQgNDUVSUhK8vb3vZXi9gl6vR2FhIaZOncpbtWTg5MUbQOlxjB07FiMH9bN3d7pN+zcsa9gt4FxdXREXFweNRoMZM2YAAIxGIzQaDRYuXGh1O0ajUfp6GR4ejsDAQGg0GinQtFotjhw5gvnz51tsQ6VSQaVSdSh3cXFxiA++o4xT7pydnaWfct6etozNrl9R1Wo10tLSEB8fjzFjxiA3NxfNzc3IyMgAAKSmpiIkJAQ5OTkA7nyVjI+PR2RkJHQ6HXbu3InNmzdj3bp1AACFQoFFixZh5cqVGDJkCMLDw7F06VIEBwdLIUpEjsOuATdr1ixcu3YNWVlZqK2tRWxsLHbt2iWdJLh48SKUyh9O9DY3N2PBggW4fPky3N3dER0djS1btmDWrFlSnSVLlqC5uRnz5s1DQ0MDJkyYgF27dsHNze2Bj4+I7EshhBD27kRPo9Vq4ePjg8bGRtkfg9u5cycef/xxWX+lcRQlF65jxrrD2DF/LGIH97d3d7qNLZ9Pu9+qRUTUXRhwRCRbDDgiki0GHBHJFgOOiGSr192qRUR3WPucW8Bxn3XLgCPqpax9zi3guM+6ZcAR9VLWPue2va4jYsAR9VJ8zu3d8SQDEckWA46IZIsBR0SyxYAjItliwBGRbDHgiEi2GHBEJFu8Ds6M9jlAbXm4RW+k1+vR0tICrVbL66ZkwFG2Z/vn0pq5ehlwZty6dQsAEBoaaueeEJElt27dgo+PT6d1OGW5GUajEVeuXIGXlxcUCoW9u9Nt2h+PeOnSJVlPze4oHGV7CiFw69YtBAcHmzyzxRzuwZmhVCoxcOBAe3fjgfH29pb1B8LROML2vNueWzueZCAi2WLAEZFsMeAcmEqlwrJly6BSqe5emXo8bs+OeJKBiGSLe3BEJFsMOCKSLQYcEckWA46IZIsBJxMTJ07EokWL7N0NslFP2249rT/3iwFH1Mu1trbauws9l6BeLy0tTQAweYWEhIgPPvjApN6JEyeEQqEQ1dXVQggh3nnnHTFixAjh4eEhBg4cKObPny9u3bplss7+/fvFhAkThJubmxg4cKD4zW9+I5qamh7Y2OTM3HarqKgQL730kggLCxNubm7iZz/7mcjNze2w3vTp08XKlStFUFCQCAsLE0IIcfDgQTFy5EihUqlEXFyc+PTTTwUA8fXXX0vrlpaWipSUFOHp6Sn8/f3FCy+8IK5du2axP1VVVQ/qz9EtGHAy0NDQIMaNGyfmzp0rampqRE1NjVi8eLGYMGGCSb3f/e53JmXvvvuu+PLLL0VVVZXQaDRi6NChYv78+dL7FRUVwtPTU7z77rvi3Llz4uDBg2LUqFEiPT39gY1Nzsxtt9u3b4usrCxx7Ngxcf78ebFlyxbh4eEhtm3bJq2XlpYm+vTpI1588UVx+vRpcfr0adHY2Cj69esnXnjhBXHmzBmxc+dO8bOf/cwk4G7evCkGDBggMjMzxdmzZ8WJEyfE1KlTxaRJkyz2p62tzR5/mi7DgJOJRx99VLzyyivS8tdffy0UCoW4cOGCEEIIg8EgQkJCxLp16yy28cknn4j+/ftLy3PmzBHz5s0zqbN//36hVCrF999/37UDcFA/3W7mvPzyy+Kpp56SltPS0kRAQIDQ6XRS2bp160T//v1Ntsv69etNAm7FihUiKSnJpO1Lly4JAKK8vNzq/vQmPAYnU7GxsRg2bBg++ugjAMBXX32Fq1ev4umnn5bq/Otf/8KUKVMQEhICLy8vvPjii7h+/TpaWloAACdPnkRBQQH69OkjvZKTk2E0GlFVVWWXcTmCtWvXIi4uDgMGDECfPn3w5z//GRcvXjSp8/DDD8PV1VVaLi8vR0xMDNzc3KSyMWPGmKxz8uRJ7N2712R7tj/xvrKyshtHZD8MOBl7/vnnpYD76KOPkJKSgv79+wMAqqur8cQTTyAmJgZ///vfUVxcjLVr1wL44aB1U1MTfv3rX6OkpER6nTx5Et9++y0iIyPtMyiZ27p1KxYvXow5c+Zgz549KCkpQUZGRocTCZ6enja33dTUhGnTpplsz5KSEnz77bf4xS9+0VVD6FE4H5xMuLq6wmAwmJQ999xzeOONN1BcXIy//e1vyMvLk94rLi6G0WjEO++8I00a+Ne//tVk/dGjR+Obb75BVFRU9w/AQf10ux08eBDjx4/HggULpDJr9q6GDh2KLVu2QKfTSTfbHzt2zKTO6NGj8fe//x1hYWFwdjb/0Tf376g34x6cTISFheHIkSOorq5GfX09jEYjwsLCMH78eMyZMwcGgwFPPvmkVD8qKgp6vR7vv/8+zp8/j82bN5sEIAC8+uqrOHToEBYuXCj9n/4f//gHFi5c+KCHJ1s/3W5DhgzB8ePHsXv3bpw7dw5Lly7tEFTmPPfcczAajZg3bx7Onj2L3bt3Y9WqVQAgzUr98ssv48aNG5g9ezaOHTuGyspK7N69GxkZGVKomft31KvZ+yAgdY3y8nIxduxY4e7ubnJ6/4MPPhAARGpqaod1Vq9eLYKCgoS7u7tITk4WmzZtEgDEzZs3pTpHjx4VU6dOFX369BGenp4iJiZG/PGPf3xAo5K/n263srIykZ6eLnx8fETfvn3F/PnzxWuvvSZGjhwprdN+mchPHTx4UMTExAhXV1cRFxcnPvroI6nNdufOnRO//OUvRd++fYW7u7uIjo4WixYtEkaj0Wx/evtlIpwuiUim/vKXvyAjIwONjY1wd3e3d3fsgsfgiGRi06ZNiIiIQEhICE6ePIlXX30VzzzzjMOGG8CAI5KN2tpaZGVloba2FkFBQXj66afxxz/+0d7dsit+RSUi2eJZVCKSLQYcEckWA46IZIsBR0SyxYAjItliwJFdpaenQ6FQQKFQwMXFBQEBAZg6dSry8/Ntuk2ooKAAffv27b6OWpCeno4ZM2Y88N9L1mHAkd2lpKSgpqYG1dXV+OKLLzBp0iS88soreOKJJ9DW1mbv7lFvZt87xcjRWbqvUqPRCABi/fr1QojOp1ffu3dvh6m2ly1bJoQQYtOmTSIuLk706dNHBAQEiNmzZ4u6ujrp99y4cUM899xzws/PT7i5uYmoqCiRn58vvX/x4kXx9NNPCx8fH+Hr6yuefPJJ6f7MZcuWdfi9e/fu7Za/E90b7sFRjzR58mSMHDkS27dvBwAolUqsWbMGZ86cwcaNG/Hll19iyZIlAIDx48cjNzcX3t7eqKmpQU1NDRYvXgwA0Ov1WLFiBU6ePIkdO3aguroa6enp0u9ZunQpvvnmG3zxxRc4e/Ys1q1bBz8/P2nd5ORkeHl5Yf/+/Th48CD69OmDlJQUtLa2YvHixXjmmWekPdCamhqMHz/+wf6hqHP2TlhybJb24IQQYtasWWLYsGFm3/vp9OobNmwQPj4+d/19x44dEwCkvb9p06aJjIwMs3U3b94shg4dKs20IYQQOp1OuLu7i927d9+1/2R/3IOjHksIIc1ldrfp1S0pLi7GtGnTMGjQIHh5eeHRRx8FAGkK8Pnz52Pr1q2IjY3FkiVLcOjQIWndkydPoqKiAl5eXtIU3/369cPt27dlO8W33DDgqMc6e/YswsPDrZpe3Zzm5mYkJyfD29sbf/nLX3Ds2DF8+umnJus99thjuHDhAn7729/iypUrmDJlivT1tqmpCXFxcR2m+D537hyee+65bh49dQXOJkI90pdffonS0lL89re/tWp6dXNTbZeVleH69ev4j//4D4SGhgIAjh8/3uF3DRgwAGlpaUhLS8MjjzyC3//+91i1ahVGjx6Nbdu2wd/fH97e3mb7KbcpvuWGe3BkdzqdDrW1tfjuu+9w4sQJvPnmm5g+fTqeeOIJpKamWjW9elhYGJqamqDRaFBfX4+WlhYMGjQIrq6u0nr//Oc/sWLFCpP1srKy8I9//AMVFRU4c+YMPvvsMwwbNgzAnYf2+Pn5Yfr06di/fz+qqqqwb98+/Pu//zsuX74s/d5Tp06hvLwc9fX10Ov1D+aPRtax90FAcmw/fpq6s7OzGDBggEhMTBT5+fnCYDBI9ayZXv3f/u3fRP/+/U0uE/noo49EWFiYUKlUYty4ceKf//xnh2eFDhs2TLi7u4t+/fqJ6dOni/Pnz0tt1tTUiNTUVOHn5ydUKpWIiIgQc+fOFY2NjUIIIa5evSpN6Q5eJtLjcD44IpItfkUlItliwBGRbDHgiEi2GHBEJFsMOCKSLQYcEckWA46IZIsBR0SyxYAjItliwBGRbDHgiEi2GHBEJFv/DwUD+NE6e0qzAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-01T12:32:14.031713Z", + "iopub.status.busy": "2024-03-01T12:32:14.031365Z", + "iopub.status.idle": "2024-03-01T12:32:14.306971Z", + "shell.execute_reply": "2024-03-01T12:32:14.306085Z" + }, + "papermill": { + "duration": 0.299373, + "end_time": "2024-03-01T12:32:14.309047", + "exception": false, + "start_time": "2024-03-01T12:32:14.009674", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATwAAAEmCAYAAAD/SCz7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABmHUlEQVR4nO2dZ3hU1daA3ymZSU+AQAoGQhNEgSAlYsMSDYgXUC8icqXIB0oRNSICIqCoAUVEFOGionIFe1dEMQJK1xB6kWpoSSCQXqad78fknExNZtJD9vs885A5s88++zAza1bba6kkSZIQCASCRoC6rhcgEAgEtYUQeAKBoNEgBJ5AIGg0CIEnEAgaDULgCQSCRoMQeAKBoNEgBJ5AIGg0CIEnEAgaDdq6XkB9xGKxcPbsWYKCglCpVHW9HIFA4IAkSeTl5REVFYVa7bneJgSeC86ePUt0dHRdL0MgEFTAqVOnuOKKKzweLwSeC4KCggDrf2ZwcHAdr0YgEDiSm5tLdHS08l31FCHwXCCbscHBwULgCQT1GG9dTiJoIRAIGg1C4AkEgkaDEHgCgaDRIHx4lUSSJEwmE2azua6XIqgmNBoNWq1WpCJdxgiBVwkMBgPnzp2jsLCwrpciqGb8/f2JjIxEp9PV9VIENYAQeF5isVg4ceIEGo2GqKgodDqd0AguAyRJwmAwcP78eU6cOEGHDh28SmgVNAyEwPMSg8GAxWIhOjoaf3//ul6OoBrx8/PDx8eHf/75B4PBgK+vb10vqdFQbDTTb9HvhPj58OkjffD10dTIdYTAqyTi1//yRLyvdUN2oZGTWYVo1Cr02pp7D8S7KxAI6pycIiMAIX4+NeoiEgJPIBDUOdmFBgBC/Xxq9DpC4AkaFaNGjWLw4MF1vQyBA7KGFywEnqC6uOWWW3jiiSfqehkCgRO2Jm1NIgSeoMFhNBrregmCakYWeKH+QuDVeyRJotBgqvWHJEker3HUqFFs3LiRN954A5VKhUql4oorrmDp0qV241JTU1Gr1fzzzz8ALFy4kC5duhAQEEB0dDQTJkwgPz/f7pxNmzZx00034efnR3R0NJMnT6agoMCjdZ07d44BAwbg5+dHmzZtWL16NTExMSxatEgZo1KpWLp0KQMHDiQgIICXXnoJs9nMmDFjaNOmDX5+fnTs2JE33njDbm6z2UxiYiKhoaE0a9aMqVOnevV/Jqg9akvDE2kp1UCR0UznWT/X+nUPvJCAv86zt/CNN97g77//5pprruGFF14A4LXXXmP16tWMHz9eGbdq1SpuuOEGWrduDVjTNBYvXkybNm04fvw4EyZMYOrUqbz99tsAHDt2jH79+vHiiy+yYsUKzp8/z6RJk5g0aRLvv/9+hesaMWIEFy5cYMOGDfj4+JCYmEhmZqbTuDlz5jBv3jwWLVqEVqvFYrFwxRVX8Pnnn9OsWTO2bNnCuHHjiIyM5P7771fu74MPPmDFihVcddVVvPbaa3z99dfcdtttHv2fCWqP7MJSDU8IPEF1EBISgk6nw9/fn4iICACGDx/Oa6+9RlpaGq1atcJisfDJJ58wc+ZM5Txbn19MTAwvvvgijz76qCLwkpKSGD58uDKuQ4cOLF68mL59+7J06dJyk3cPHTrEr7/+yp9//knPnj0BePfdd+nQoYPT2AcffJDRo0fbHXv++eeVv9u0acPWrVv57LPPFIG3aNEipk+fzr333gvAsmXL+Pnn2v9hElRMbQUthMCrBvx8NBx4IaFOrlsVYmNjueqqq1i9ejXTpk1j48aNZGZmMmTIEGXMr7/+SlJSEocOHSI3NxeTyURxcTGFhYX4+/uze/du9uzZw6pVq5RzJElStuBdddVVbq9/+PBhtFot1157rXKsffv2NGnSxGmsLBBtWbJkCStWrCAtLY2ioiIMBgOxsbEA5OTkcO7cOeLi4pTxWq2Wnj17CrO2HtKoghZLliwhJiYGX19f4uLi2LFjh9uxX331FT179iQ0NJSAgABiY2P53//+ZzdGkiRmzZpFZGQkfn5+xMfHc+TIkRpbv0qlwl+nrfVHdSRoDh8+nNWrVwOwevVq+vXrR7NmzQA4efIkd999N127duXLL78kJSWFJUuWANYtdgD5+fk88sgj7Nq1S3ns3r2bI0eO0K5duyqvTyYgIMDu+SeffMKUKVMYM2YMv/zyC7t27WL06NHKugQNi2wlaFGzRRvqXOB9+umnJCYmMnv2bHbu3Em3bt1ISEhw6ccBaNq0Kc8++yxbt25lz549jB49mtGjR9uZKq+88gqLFy9m2bJlbN++nYCAABISEiguLq6t26qX6HQ6p3JWDz74IPv27SMlJYUvvviC4cOHK6+lpKRgsVh47bXXuO6667jyyis5e/as3fnXXnstBw4coH379k6PiiqOdOzYEZPJRGpqqnLs6NGjXLp0qcJ72bx5M9dffz0TJkyge/futG/fnmPHjimvh4SEEBkZyfbt25VjJpOJlJSUCucW1D65taThIdUxvXv3liZOnKg8N5vNUlRUlJSUlOTxHN27d5dmzpwpSZIkWSwWKSIiQnr11VeV17OzsyW9Xi99/PHHHs2Xk5MjAVJOTo7Ta0VFRdKBAwekoqIij9dXXxg7dqzUq1cv6cSJE9L58+cls9ksSZIk3XDDDVK3bt2koKAgqbCwUBm/a9cuCZAWLVokHTt2TFq5cqXUsmVLCZAuXbokSZIk7d69W/Lz85MmTpwopaamSn///bf0zTff2L2n5REfHy9de+210vbt26WdO3dKt956q+Tn5yctWrRIGQNIX3/9td15b7zxhhQcHCytXbtWOnz4sDRz5kwpODhY6tatmzJm3rx5UtOmTaWvv/5aOnjwoDR27FgpKChIGjRokNv1NOT3tyET+/zPUutnfpAOncv1aHx539HyqFMNz2AwkJKSQnx8vHJMrVYTHx/P1q1bKzxfkiSSk5M5fPgwN998MwAnTpwgPT3dbs6QkBDi4uLczllSUkJubq7d43JkypQpaDQaOnfuTPPmzUlLSwOsZu3u3bu555578PPzU8Z369aNhQsXMn/+fK655hpWrVpFUlKS3Zxdu3Zl48aN/P3339x00010796dWbNmERUV5dGaVq5cSXh4ODfffDP33HMPY8eOJSgoqMJKJY888gj33nsvQ4cOJS4ujqysLCZMmGA35qmnnuKhhx5i5MiR9OnTh6CgIO655x6P1iWoPSRJIrfYBNR8Hl6danhnzpyRAGnLli12x59++mmpd+/ebs/Lzs6WAgICJK1WK+n1eum9995TXtu8ebMESGfPnrU7Z8iQIdL999/vcr7Zs2dLgNPjctPwGgKnTp2SAOnXX3+tk+uL97f2yS0ySK2f+UFq/cwPUpHB5NE5ldXwGmSUNigoiF27dpGfn09ycjKJiYm0bduWW265pVLzTZ8+ncTEROW53PNSUPP89ttv5Ofn06VLF86dO8fUqVOJiYlRNHbB5Y+cg6fXqmusDp5MnQq8sLAwNBoNGRkZdsczMjKUXDFXqNVq2rdvD1hTKw4ePEhSUhK33HKLcl5GRgaRkZF2c8opC47o9Xr0en0V70bgyB9//EH//v3dvp6fn4/RaGTGjBkcP36coKAgrr/+elatWoWPTw2bNoJ6Q22lpEAdCzydTkePHj1ITk5WKlhYLBaSk5OZNGmSx/NYLBZKSkoAawJqREQEycnJioDLzc1l+/btdjsKBDVPz5492bVrV7ljEhISSEio/RxGQf2h1iK01IPE48TEREaOHEnPnj3p3bs3ixYtoqCgQMmqHzFiBC1btlSc5UlJSfTs2ZN27dpRUlLCmjVr+N///qfsCVWpVDzxxBO8+OKLdOjQgTZt2vDcc88RFRUlygLVMn5+foomLhC4I7uWCgdAPRB4Q4cO5fz588yaNYv09HRiY2NZu3Yt4eHhAKSlpdmV3S4oKGDChAmcPn0aPz8/OnXqxEcffcTQoUOVMVOnTqWgoIBx48aRnZ3NjTfeyNq1a0WPAoGgHlKbJq1KksQ+G0dyc3MJCQkhJyeH4OBgu9eKi4s5ceIEbdq0EQL0MkS8v7XP0g3HmL/2EPddewWv3d/No3PK+46WR53vtBAIBI2b2tTwhMATCAR1ihB4AoGg0ZBTVNrApxaCFkLgCTzCVRXib775ps7WI7h8aDR5eIKGy7lz51zWrRMIvEXeaSEEnqDeUt5OmNpCkiTMZjNarfgYN2QUDU+YtA0ESQJDQe0/vMwoysvLY/jw4QQEBBAZGcnrr79e6daNtibtyZMnUalUfPXVV9x66634+/vTrVs3p+o0FTX7+d///kfPnj0JCgoiIiKCBx980K4u4oYNG1CpVPz000/06NEDvV7Ppk2bvF67oH4hTNqGhrEQXvasHFK1MuMs6AIqHldKYmIimzdv5rvvviM8PJxZs2axc+dOt3uMveXZZ59lwYIFdOjQgWeffZZhw4Zx9OhRtFqtR81+jEYjc+fOpWPHjmRmZpKYmMioUaNYs2aN3XWmTZvGggULaNu2rTCrGzhmi0SeXBpKCDxBdZGXl8eHH37I6tWruf322wF4//33Pa5b5wlTpkxhwIABgLXBztVXX83Ro0fp1KmTR81+Hn74YWWutm3bsnjxYnr16kV+fj6BgYHKay+88AJ33HFHta1bUHfI+2ih5hv4gBB41YOPv1Xbqovresjx48cxGo307t1bORYSEkLHjh2rbTldu3ZV/pYr1WRmZtKpUyePmv2kpKQwZ84cdu/ezaVLl7BYLIB1e2Hnzp2V81w19BE0TGRzNkCnwUdT8x42IfCqA5XKK9PycsW2pJPcYEgWWnKzn8mTJzud16pVKwoKCpTKKatWrVIqMickJDg15nFs6CNouNRW8x4ZIfAaCW3btsXHx4c///yTVq1aAdZWhn///XetFNu0bfbjir1795KVlcW8efOU4qt//fVXja9LULfUVj9aGRGlbSQEBQUxcuRInn76adavX8/+/fsZM2YMarW6Wto9VsQzzzzDli1bmDRpErt27eLIkSN8++23St3DVq1aodPpePPNNzl+/Djfffcdc+fOrfF1CeqW7EKr9h7iVzu6lxB4jYiFCxfSp08f7r77buLj47nhhhu46qqraqUqSEXNfpo3b84HH3zA559/TufOnZk3bx4LFiyo8XUJ6hY5aBHqJ0xaQTUTFBRkFzQoKCjg+eefZ9y4cRWee/LkSbvntlXFYmJicKwyFhoa6nSsV69e/PLLL26vMWzYMIYNG+b2OrfccovTnIKGTW3m4IEQeI2K1NRUDh06RO/evcnJyeGFF14AYNCgQXW8MkFjRd5WVhuFA0AIvEbHggULOHz4sNJP5I8//uDgwYMVNtsRCGqC2g5aCIHXiOjevTspKSlOx4uKiipstiMQ1ATZwqQV1Dai2Y6grsipxQY+IKK0lUY4zy9PxPtau9Rmi0YQAs9r5N0EhYWFdbwSQU0gv6+iEXjtoAQtRFpK/USj0RAaGqqULfL396+VxF1BzSJJEoWFhWRmZhIaGopGo6nrJTUKGmVaypIlS3j11VdJT0+nW7duvPnmm3ab3G155513WLlyJfv27QOgR48evPzyy3bjR40axYcffmh3XkJCAmvXrq2W9crFL21rtQkuD0JDQ+tFcdPGQInJTJHRDDQigffpp5+SmJjIsmXLiIuLY9GiRSQkJHD48GFatGjhNH7Dhg0MGzaM66+/Hl9fX+bPn8+dd97J/v37admypTKuX79+Sp01AL1eX21rVqlUREZG0qJFC4xGY8UnCBoEPj4+QrOrRWTtTqWCIN/aEUV1LvAWLlzI2LFjGT16NADLli3jxx9/ZMWKFUybNs1pvO1OAYB3332XL7/8kuTkZEaMGKEc1+v1Nf5LrdFoxBdEIKgkcsAi2NcHtbp23EJ1GrQwGAykpKQQHx+vHFOr1cTHxzuVB3dHYWEhRqORpk2b2h3fsGEDLVq0oGPHjowfP56srCy3c5SUlJCbm2v3EAgENUttNu+RqVOBd+HCBcxmM+Hh4XbHw8PDSU9P92iOZ555hqioKDuh2a9fP1auXElycjLz589n48aN9O/fH7PZ7HKOpKQkQkJClIdcnkggENQctZ2DB/XApK0K8+bN45NPPmHDhg12FT8eeOAB5e8uXbrQtWtX2rVrx4YNG5Ty5rZMnz6dxMRE5Xlubq4QegJBDVPbEVqoYw0vLCwMjUZDRkaG3fGMjIwK/W8LFixg3rx5/PLLL3alxV3Rtm1bwsLCOHr0qMvX9Xo9wcHBdg+BQFCzNDqTVt7AnpycrByzWCwkJyfTp08ft+e98sorzJ07l7Vr13rU3+D06dNkZWUpfRYEAkHd0+g0PLC2DnznnXf48MMPOXjwIOPHj6egoECJ2o4YMYLp06cr4+fPn89zzz3HihUriImJIT09nfT0dKWiR35+Pk8//TTbtm3j5MmTJCcnM2jQINq3b09CQkKd3KNAIHCmLgRenfvwhg4dyvnz55k1axbp6enExsaydu1aJZCRlpaGWl0ml5cuXYrBYODf//633TyzZ89mzpw5aDQa9uzZw4cffkh2djZRUVHceeedzJ07t1pz8QQCQdWoi6CFShK7pZ3Izc0lJCSEnJwc4c8TCGqIhz/4k98OZTL/vi4M7dXKq3Mr+x2tc5NWIBA0Tsoa+NRO4QAQAk8gENQRDc6Hl5+frzRalhEmoEAg8IScIhNQz6O0J06cYMCAAQQEBBASEkKTJk1o0qQJoaGhNGnSpCbWKBAILjMkSSKnyGrS1uudFv/5z3+QJIkVK1YQHh4uasEJBAKvKTKaMZqt8dJ6bdLu3r2blJQUOnbsWBPrEQgEjQB5l4VWrcJfV3sVh7w2aXv16sWpU6dqYi0CgaCRYJuDV5tWotca3rvvvsujjz7KmTNnuOaaa5xq/1e0r1UgEAhqux+tjNcC7/z58xw7dkzZ+gXWCsCSJKFSqdyWYBIIBAKZsuY99VzgPfzww3Tv3p2PP/5YBC0EAkGlqO32jDJeC7x//vmH7777TjRuFggElSa7SN5lUbsCz+ugxW233cbu3btrYi0CgaCRUBa0qL1tZVAJDe9f//oXTz75JHv37qVLly5OQYuBAwdW2+IEAsHlSYMJWjz66KMAvPDCC06viaCFQCDwhAYTtHDcOyuoHSRJYvQHf9I0QMfC+2PrejkCQZWoi8IB4KUPz2g0otVq2bdvX02tR+CGo5n5bDh8nq92nqnrpQgEVaauorReCTwfHx9atWolzNY6wGJTplXUbBU0dLLroNoxVCJK++yzzzJjxgwuXrxYE+sRuME23dEi5J2ggVNXJq3XPry33nqLo0ePEhUVRevWrQkICLB7fefOndW2OIFrrBqeSPgWNEwsFqnhCLzBgwfXwDIaPhaLhEpFje08sZ1VaHiChkxeiQnZK1Pv01Jmz55dE+to0JgtEne/uYkgXy2fjruuRoSevUkrJJ6g4SIHLHx91Pj61F5pKKhCT4uUlBQ++ugjPvroI1JTU6u0iCVLlhATE4Ovry9xcXHs2LHD7dh33nmHm266Sam0HB8f7zRekiRmzZpFZGQkfn5+xMfHc+TIkSqtsTz+ySrg4Llcdpy4iKnG1C9hwgouD8py8Gp3lwVUQuBlZmZy22230atXLyZPnszkyZPp0aMHt99+O+fPn/d6AZ9++imJiYnMnj2bnTt30q1bNxISEsjMzHQ5fsOGDQwbNoz169ezdetWoqOjufPOOzlzpixd45VXXmHx4sUsW7aM7du3ExAQQEJCAsXFxV6vzxN8NGX/jUZzzeQpCg1PcLlQV/47qITAe+yxx8jLy2P//v1cvHiRixcvsm/fPnJzc5k8ebLXC1i4cCFjx45l9OjRdO7cmWXLluHv78+KFStcjl+1ahUTJkwgNjaWTp068e6772KxWEhOTgas2t2iRYuYOXMmgwYNomvXrqxcuZKzZ8/yzTffeL0+T9Bpy/4bDaaaT8wW8k7QkKmrwgFQCYG3du1a3n77ba666irlWOfOnVmyZAk//fSTV3MZDAZSUlKIj48vW5BaTXx8PFu3bvVojsLCQoxGI02bNgWsTYbS09Pt5gwJCSEuLs7jOb3FVvuqKYFnH7QQEk/QcFE0vFrOwYNKbi1zLBgA1qRkb7edXbhwAbPZTHh4uN3x8PBwDh065NEczzzzDFFRUYqAS09PV+ZwnFN+zZGSkhJKSkqU57m5uR7fg9NcNSXwbKSqiNIKGjINyqS97bbbePzxxzl79qxy7MyZMzz55JPcfvvt1bq4ipg3bx6ffPIJX3/9Nb6+vpWeJykpiZCQEOURHR3t3QQ2AshQUz48N9cTCBoaOYUNSOC99dZb5ObmEhMTQ7t27WjXrh1t2rQhNzeXN99806u5wsLC0Gg0ZGRk2B3PyMggIiKi3HMXLFjAvHnz+OWXX+z6aMjneTPn9OnTycnJUR7eNimylT81ZtKKoIXgMkGphVcHAs9rkzY6OpqdO3fy66+/KmbnVVddZecz8xSdTkePHj1ITk5WEprlAMSkSZPcnvfKK6/w0ksv8fPPP9OzZ0+719q0aUNERATJycnExsYCVhN1+/btjB8/3uV8er0evV7v9fplbOVPTUVpbRECT9CQaVA+PLD6k+644w7uuOOOKi8gMTGRkSNH0rNnT3r37s2iRYsoKChQmgSNGDGCli1bkpSUBMD8+fOZNWsWq1evJiYmRvHLBQYGEhgYiEql4oknnuDFF1+kQ4cOtGnThueee46oqKga2yUi2eh4tRKlrfErCAQ1R3YdmrSVEnjJyckkJyeTmZnpFKhwl07ijqFDh3L+/HlmzZpFeno6sbGxrF27Vgk6pKWloVaXWd5Lly7FYDDw73//226e2bNnM2fOHACmTp1KQUEB48aNIzs7mxtvvJG1a9dWyc/nKTUl8GyVOqHhCRoydRm08FrgPf/887zwwgv07NmTyMjIatlGNWnSJLcm7IYNG+yenzx5ssL5VCoVL7zwgsuqzDWBrfwpqSGT1lbECXknaMg0KIG3bNkyPvjgAx566KGaWE+DpDaCFnbXEwJP0ICpqwY+UIkorcFg4Prrr6+JtTRYbAty1pxJW3YNYdIKKiKv2MiuU9n1rlis0Wwhv8QENJC0lP/7v/9j9erVNbGWBkttRGklN38LBK6Y+c0+Bi/ZzJ8nL9X1UuyQK6UABPtWKoRQJby+YnFxMcuXL+fXX3+la9euTrsuFi5cWG2La4jUStBCbLUQVMCZS0WAtZJP7zZNlePFRjP7z+bSPToUtbry/vfNRy+w70wO425u65UfXzZng/RatJpKF2uqNF4LvD179ij5bY7NfGqq+GVDoqZ2WtjqdfXMShHUQ4ylP4qFBvv+M+P+l8Lvf59nar+OTLilfaXnn/rFHs5kF9GrTVOubdXE4/Oy66gfrYzXAm/9+vU1sY4Gja0Aqg0NTxJGraACTKU/vLK/TOb3v60l3P639Z9KC7yCEhNnsq0a5LHMfK8EXk4dNe+RqX2d8jLEVgDVVPEAW4RFK6gI2ZdcaDC5fL0qttiJCwXK32kXC706t67aM8rUvtfwMmL3qWy2Hc+yKwBaYxqezd8iSiuoCJPZ+hkpKKn+lqrHbQTeySzvBF5d7rIAIfCqxPYTWST9dIhrW4Uqx2osSiu5/lsgcIWxdAdUQYlrDa8qHD+fr/z9T1ZBOSOdESZtA0avtTYgsTVja07Dsw1aCIknKB9Zw3MMWshUJcB4/HyZkPvHSw0vp46DFl4LvN9//x2TyflXw2Qy8fvvv1fLohoKcmn3YmPZh6qmorT2e2lr5BKCywhjqcBzDFpUB7Y+vJwiI9mFBo/PrcsGPlAJgXfrrbdy8eJFp+M5OTnceuut1bKohoK+VODVioYnorQCLzBZyg9aVBZJkhSTVk7j88aPV5f7aKESAk+SJJfqcFZWFgEBAdWyqIZCmYZX8wLPFi8r6QsaIUaT7MOr3qBFZl4JBQYzahV0vSIU8M6Pl1OHDXzAi6DFvffeC1ht/1GjRtkVzDSbzezZs6fR7bEt8+GVfahqrlqK5PJvgcAVcuJxQTVreLL/LrqpPx1aBLLrVLZXfry6Dlp4LPBCQkIAq4YXFBSEn5+f8ppOp+O6665j7Nix1b/CeoyurkxaIe8EFSAnHrvT8Cobszh+wWrOtg0LICbMatGd9ErDayBpKe+//z4AMTExTJkypdGZr66QfXi2Qk6UeBfUNRaLpAS2qjstRdbw2jYPpHUzf8C7SG2Dy8ObPXt2TayjQWLbgFtGaHiCusZo4+QtMpoxWyQ0DoUCKqvhyRHaNmEBtG5qVXo8FXjFRrNiDdVFPwuoRNAiIyODhx56iKioKLRaLRqNxu7RmNDXpsCz8dsJDU9QHnJKikyR0dmsVVVyc5kcoW3bPIBWpRrehfwSj9JfZHNWrYJAXd3sefD6qqNGjSItLY3nnnuu2kq8N1RcCrxaMWlr/BKCBozJ4TNYUGIiUF91AWMwWThVWnaqXfNAQvx8aBqg42KBgX+yCrg6KqTc8239d1UpTVUVvP5f2LRpE3/88YdSIqoxI0dpbakNk1aUABWUh6OG58qPVxk9Je1iIWaLRIBOQ4sga5ZGq6b+XCwwkJZV6JXAqyu8Nmmjo6PF1qZSatWHZ/O30PAE5WFySNR0t73MW2Rztk3zAMWyiyk1az1JPq7rgAVUQuAtWrSIadOmedQ97HKnNk1au54WXkq8D7ecZNX2f6p7SYJ6islBw6uu7WVylZS2YYHKsdbN5MBFxakpZQ2462ZbGVRC4A0dOpQNGzbQrl07goKCaNq0qd3DW5YsWUJMTAy+vr7ExcWxY8cOt2P379/PfffdR0xMDCqVikWLFjmNmTNnDiqVyu7RqVMnr9flCXWl4Xkj7i4VGJj93X6e/XofRdX0Sy+o3zimRrnaXlYZD9qJ82URWpnWiobnhcCrQw3Pax+eKyFTWT799FMSExNZtmwZcXFxLFq0iISEBA4fPkyLFi2cxhcWFtK2bVuGDBnCk08+6Xbeq6++ml9//VV5rtXWTERI56Imf+0UD/Bc5DkmRfvpGlckvTHi6MPLr6btZUrScXNbgWf9O80DkzantMhAaEMSeCNHjqy2iy9cuJCxY8cyevRowNrz9scff2TFihVMmzbNaXyvXr3o1asXgMvXZbRaLREREdW2TrfX0ajRqFWYbUzM2thL+8L3B/jp8Zs8ipDbprPURgRZUPc4aXgugxbWz84fR87Tqqm/IrjKQ046bte8zKSVfXhnc4opNprx9XH/g1ofNLxK1cM7duwYM2fOZNiwYWRmZgLw008/sX//fo/nMBgMpKSkEB8fX7YYtZr4+Hi2bt1amWUpHDlyhKioKNq2bcvw4cNJS0srd3xJSQm5ubl2D09x9OPVnMArE1yH0vPYd8azNdr6c2z3/AouX0wOPt4CN66MnWmXeOi9HfR9dUOFc+YUGskqsGpoMTYmbdMAHUGlKS+nKij3nt0QBd7GjRvp0qUL27dv56uvviI/36rm7t6926tdGBcuXMBsNhMeHm53PDw8nPT0dG+XpRAXF8cHH3zA2rVrWbp0KSdOnOCmm24iLy/P7TlJSUmEhIQoj+joaI+v5+jHM1mkGmmj6GjFerop3Farqw3tU1D3uMrDc8WeU9kezymbs+HBerucPpVKpSQgV7Tjoixo0YAE3rRp03jxxRdZt24dOl1ZtOW2225j27Zt1bq4ytC/f3+GDBlC165dSUhIYM2aNWRnZ/PZZ5+5PWf69Onk5OQoj1OnTnl8vdqK1DqKUE/deLbmTW00GBLUPU55eG6CFo7bzcpD2UNrE6GViWnmWRGB+mDSeu3D27t3L6tXr3Y63qJFCy5cuODxPGFhYWg0GjIyMuyOZ2RkVKv/LTQ0lCuvvJKjR4+6HaPX6+3KXXmDy0it2VKuL6MyVDb10WiqXf+ioO5xysNzE7TwZreDsoe2ubOvz9MiAjlKteMGpOGFhoZy7tw5p+Opqam0bNnS43l0Oh09evQgOTlZOWaxWEhOTqZPnz7eLsst+fn5HDt2jMjIyGqb05ba2m3hmOztaU08243kQsNrHDgGLdyZtGovtlvYloVyxNPUlAZp0j7wwAM888wzpKeno1KpsFgsbN68mSlTpjBixAiv5kpMTOSdd97hww8/5ODBg4wfP56CggIlajtixAimT5+ujDcYDOzatYtdu3ZhMBg4c+YMu3btstPepkyZwsaNGzl58iRbtmzhnnvuQaPRMGzYMG9v1SNcpqbUI8FiNNkKPBG0aAx4YtKiAo03As9FhFZGSU0pJ2ghSVK9CFp4bdK+/PLLTJw4kejoaMxmM507d8ZsNvPggw8yc+ZMr+YaOnQo58+fZ9asWaSnpxMbG8vatWuVQEZaWhpqdZlAOXv2LN27d1eeL1iwgAULFtC3b182bNgAwOnTpxk2bBhZWVk0b96cG2+8kW3bttG8eXNvb9Uj9D61I/Cc9DmPfXjCpG1sOO60cLe1zNakNZktaF38eIN1Z49tWShHZB/e6UtFGM0Wuz7NMgUGs5K+VVcNfKASAk+n0/HOO+/w3HPPsW/fPvLz8+nevTsdOnSo1AImTZrEpEmTXL4mCzGZmJiYCvfxfvLJJ5VaR2WpreRjx9s2e+jUE0GLxoejD8/V1jJr0KLsudEs4cI7A8DZnCJKTBZ8NCquaOLn9HqLID16rZoSk4Uzl4rs0lZkZHNWp1Hj60JJqC0qvQWhVatWtGrVqjrX0iDRuwhO1IyGZy/gHH/F3SHSUhofslbvo1FhNEvugxY2Jq3BbMEP1xJPNmdbNwtwqQWq1SpaN/Pn74x8/rlY6FLgya0cg/186rSknEcCLzExkblz5xIQEEBiYmK5YxcuXFgtC2so1Nr2Mgf55mkpeXsNT/jwGgNyHl6In85tcU55n7lMeT+GSpUUF4JMpnWzAKvAyyoAnN1Hdd28R8YjgZeamorRaFT+dkdjLAZaVz48x2x6dwiTtvFhVASelgv5JW6LB9gmyJf3Ayr779q6SEmRUcpEXXAduMitBwEL8FDgrV+/3uXfAtDXUZTWcw1PBC0aG/J7HuqvAwrcbi0zebgHXC4L1c5F0rFMKyVS6zo1Jbse5OBBJffSCsqoNQ3PyaQVGp7ANXLQQhYuBpPF6QdSpfJcwyvrVOaBhucm+bg+7LIADzU8uQm3J3z11VeVXkxDpDI+vKOZ+XyRcprhca2Iburv0XWcgxYeangmIfAaEnnFRgL12iq5h+QfQ1vhUlhiJsS/7LNqkew1PHefjWKjmTPZ1j4W5fnwYmzKRLnqkibn4AU3BA3PdmN9cHAwycnJ/PXXX8rrKSkpJCcnK826GxOuorTl/Vp+u+sMA9/axLKNx3j6i90eX8dJw/PYhyeqpTQUDqXn0mXOLzz2sXs/uSfIEXxfnUb5QXZMPi4ymO1Sm9x9ZmX/ndywxx2RIb5o1SoMZgvpucVOrzeooIXchBvgmWee4f7772fZsmVKW0az2cyECRMIDg6umVXWY1xpeK5+LUtMZub+cICPtpWVqtp2/CIHzubSOari/zenoIWHGp6ttlliFBpefea9P04A8MOec7z1YOXnkYWXj1pFgF6DodDitL0st8iI2eaz4c5FYmvOlqd1ajVqopv6c+JCAf9kFdAy1D5fr76YtF778FasWMGUKVPsetBqNBoSExNZsWJFtS6uIeBJb9pTFwsZsmyrIuwm3tqOu7pYCyS8v/mER9dxTLj2NA/P9pdbFACt37jyB1cGef+0j0aNf2n/V8fARV6JyaOgxYkLFaekyJRXRCCnHjTwgUoIPJPJxKFDh5yOHzp0CIul8X2hKupr8euBDAYs/oM9p3MI9ffh/VG9eDqhEw9dFwPAhr/PV+q6Rg//r41Cw2sw6Kqpkb38Y6jVqAnQW+d0VUDA4oFJW94eWkdaN3VfRKBBmbS2jB49mjFjxnDs2DF69+4NwPbt25k3b56y6b8x4a4ensls4dVfDvPfjccBiI0OZcnwaxVVPzLEF4BiDxvrOJu0nml4tuOEhle/cfXjWRlkd4ePRkVAabFOVwLPk6DFMaVTmScanvv+FtlF1p0WFWp4ZiNoak4oei3wFixYQEREBK+99ppSJioyMpKnn36ap556qtoXWN8JxPnNPXWxkAff2c6OkxcBGH1DDNP7X2X3gZb/9jhyWsmdFvY+PBG0qM9Ul8CTA1patZqAUpPWVQGBitJSJEnihE0v2oqICXOfmlJm0roJfJTkw5opYDHDvcsr1yncA7wWeGq1mqlTpzJ16lSl90NjDFYAcDaVgev/xU7N/aw234bc/G7VdquvLlCv5ZV/d+WuLs61+OQPt8FsQZKkCtMQHNNSRB7e5YetteDJZ8Idsoan1agUk1beXubro6a41LVhqkDgZRUYyC02oVKVpZ2Uh22PWtv1my0SeaXXd6nhpe+DL0bDhb9BpYYbJkNEF09v1yuq9JMSHBzceIUdQOoqdKY8XvZ5j7d83iTIRtvrFBHE94/d6FLYgf2H2xNB5JiW4nkenthp0VDw9jPhDtviAWUanqn0GmV+Qts+xa4+G7L/rmWon0cVvK9o4odKZdUmL+QblON5xUbl82sn8CQJ/nof3r3dKuyComDkDzUm7KCS1VK++OILPvvsM9LS0jAYDHav7dy5s1oW1iDo/wqHS5rSdvcC7tZso4vqOI+bJ9Pp2r7MGXh1uR8SW/PFk5LwTgKvUntphUlbn7GtI1dkKL/lYXkoaSkaNf5K0ML63mttEoLlLmS259jiTYQWrMI0KsSPM9lF/JNVQPMga9sEOWDhr9OUfe6Lc+H7x2F/6UaFDnfC4GUQ0MzT26wUXmt4ixcvZvTo0YSHh5Oamkrv3r1p1qwZx48fp3///jWxxvqLWk36NWMZYpjNKUtzWqsz+Vr/PPNabsa3An+Mbf5eZTSvyvjwRNCi4VBUBX+rfZTWPmhh+xm4ZCPwXGmU3kRoZVz58Zxy8M6mwn9vtgo7tRbumAvDPq1xYQeVEHhvv/02y5cv580330Sn0zF16lTWrVvH5MmTycnJqYk11mt0GjW7pPYMMLzMJm0fVBYjrJ0GnzwIhRfdnqdSqbwKXFQ2SivSUhoOtg3dqyTw5Dw8dZlJK+fh2X4GLhbaanjOn6djHuyhdaRV0zI/noxcOCDEVwvblsG7d8ClExDSCkavtfrs1NUTsKkIr6+SlpbG9ddfD4Cfn5/S7/Whhx7i448/rt7VNQDkZNFcAngxYDrctQA0Oji8BpbdBGnb3Z9bquV5ouE5Jh57modn34i7ZgTerwcyeOi97aTnOG8pEniObV5ckYfpSq4w2mh4/rqyPDyLRXKr4bn6DHpr0kJZEYF/HDS8EPKZW5wEa58BixE63Q2P/g7Rvby4s6rjtcCLiIjg4kWr5tKqVSulF+2JEycqLL9+OWK3tUylgt5j4f9+habtIPc0vN8fNr0OLgSULCw98a1VVsOrjYrH/7fyL/44coE53+2vkfkbC9Wu4WlUStPsQoPJyaVxsRwfnslsUZrytPXCpFV2W9g09NGe/Ysf9TPoVbLVqgz0fxWGfgR+Tby4q+rBa4F322238d133wHWJOQnn3ySO+64g6FDh3LPPfdU+wLrOy7r80d2g0c2QpchIJnh1zmw6t+Qb7+rQueVhmf/vDIVj6vyJfKErIKSGp3/csdcXRqeSY7SqvFXfHhmih3e/9zismRkx8/TqUtFGM0Svj5qIoN9Pb52RIg1sT4zt9j6I7/pdRK2j+IK1QUu6FrCmHUQN67G8uwqwuso7fLly5UtZBMnTqRZs2Zs2bKFgQMH8sgjj1T7Aus7ttuB7PKm9EFw7zvQpi+seRqOJcOyG+G+d6HNTdYhpVE4zzSvyubhlY2zpgdUPr+rIlQ0vorX1YmlmjQ82d2hVauUH+QCg6lcl4bja3JZ95hmAV417A4PtkZmzXmZSKv+jepYMmrgO3MfDl8zl6ejYr24k+rHKw3PZDLx4osvkp6erhx74IEHWLx4MY899hg6Xd21X6srbDd8O30sVCq49iEYtx6ad4L8dFg5EDbMA4tZ0fAq41tz7EzlDttfbovkvIm8OmmEFf6rFVsly1Eb8waT2UbD05VFacsLWjlqeHJZKG8itABhgXquUx/ke59pqI4lg9aXL6KmMtk4Cb/AUK/mqgm8EnharZZXXnkFk8l1J/PKsGTJEmJiYvD19SUuLo4dO3a4Hbt//37uu+8+YmJiUKlULFq0qMpzVhVbH57bL3yLq2Dseuj+H5AssCEJVg4iQn0JqJxJ67EPz2FuubdATeBNJ3uBM7Ymrbtesp5gtN1pYbO1rLgcX7Hj56QyEVosZnz+eJVVupcIV2VTHNoexq5nnV8CoCLEv+4VIq99eLfffjsbN26slot/+umnJCYmMnv2bHbu3Em3bt1ISEggMzPT5fjCwkLatm3LvHnziIiIqJY5q4rHJX10/jBoCdyzHHwC4OQfLM5+jJvVuykxme0c1q5wfLUyPjyA3OKaE3gp/1yqsbkbA3YmbRUEnsl2L63N1jJvNDxPOpXZkZcOKwfBhpfRYOEzU1+2x38J4Z3L0lLquDQUVMKH179/f6ZNm8bevXvp0aMHAQH2/yEDBw70eK6FCxcyduxYpcrKsmXL+PHHH1mxYgXTpk1zGt+rVy969bKGsV29Xpk5q4pHGp4t3YZCyx7w+ShCMvayUjef5A2n6fpJfxK6RDNn0NUE+zp/MCq708JxnLyJuyYQic1Vwy5oUSWT1rlaSkUanqNPuKxTmQcm7dFk+GocFF4AnwCWB0/k5TOxzCu0fjeU0lANUeBNmDABcN1/VqVSYTZ79kYZDAZSUlKYPn26ckytVhMfH8/WrVu9XVaV5iwpKaGkpCzCKBdF8AStRo1GrcJskTx32oe1h//7leQ3/o/b87/n9gur+FD9J5NTJ3HXyYu88UAsPVo3tTulqj0t5DXaRuYE9QtbDa8qPjyjnQ+vtCq5RSrXnWHrR84rNpKZZ/0+lKvhmU2w/iXYVCoLwq+BIR9wfGMRnDlFRq51jvrSohEqYdJaLBa3D0+FHcCFCxcwm82Eh4fbHQ8PD7cLinhDZedMSkqy69sRHR3t1XVlLc8rF5aPL5+FP8F4w+PkSX70VP/NGv10OuVsYsiyrby+7m87oeao4Rk8zsOzjmtW2o+gJn14gqph69aoLh+eHLQA+72zMu1KfXS2Jq2s3YUF6twLqZzT8MGAMmHXc4w1/zSsAy1K01gy8qyJ6NkNWeBdjkyfPp2cnBzlcerUKa/Or2xpbp1Ww0+WOO4yvMwuS1tCVQW8q3uNZzX/4+3kg9z/360uiymCFxpe6Ti5AUtN+vAEVaPaTFpLmYanUavwK01/kndWhAXqlbFdrwgF7IMWijnrrg/t4Z+sKVantoE+GIZ8AHcvBB9rDp6cmpKZW4LRbFGEd11XOwYvTNqioiKSk5O5++67AauQsDUDNRoNc+fOxdfXsyTFsLAwNBoNGRkZdsczMjLcBiRqak69Xo9er3f7ekUoGp6X58nlgE5J4QwxzOGzdj/T/cwqxmh/Ik7zN+NPTeJfbxWwLvFm550WXlZLsX7I88gtEiZtfcVWw/O0ErYrFA2vNH8uQK+lyGhWdlY0D9JzId/63b06KpivU8/YaXhuI7QmgzWJftsS6/Oo7vDv96FpG7th4UFWGZCZV6z47wCCXPimaxuPVZMPP/yQ//73v8rzt956iy1btpCamkpqaiofffQRS5cu9fjCOp2OHj16kJycrByzWCwkJyfTp08fj+ep6Tk9QdHwvEzLsC0RZURLSqcp1qoRfk24RnWMn/TPckPJH/z+9wXnvbReanhhgULDq+9U29YyGx8eoERqZZO2ZagvN3UI47ZOLRQfnW3AyWWE9uIJWHFnmbC7biI8/IuTsANoUarhZeQWKxHaYF+tU6/ausBjDW/VqlVMnTrV7tjq1atp27YtAB999BFLlizhySef9PjiiYmJjBw5kp49e9K7d28WLVpEQUGBEmEdMWIELVu2JCkpCbAGJQ4cOKD8febMGXbt2kVgYCDt27f3aM6aoKoanvLcRwMd+8Gjm+CLMQSe2sbbusVs35JG5vWz7cZ6kocnSZLiwG5WasYIH179xVJNJq1tPTxA8ePJJq2vj4Z3R1qzHTYduQC4MWnlCO3+b+C7x6AkF3xDYfBS6HSX2+uHl/rwzueVcKm0IktIPTBnwQuBd/ToUbp0KatE6uvri9qmpEvv3r2ZOHGiVxcfOnQo58+fZ9asWaSnpxMbG8vatWuVoENaWprdNc6ePUv37t2V5wsWLGDBggX07duXDRs2eDRnTWBbRdYbHHsYKDX0Qq6AUT9y9LMZtD20nLisb8jZeJC2qrEcl6IAz3Za2Jq9zYSGV++pjqCFJElleXga609woIOGZ/t59SkdIwtJSZLKdlk00cAPifDXe9bB0XFw33sQWn5Qr1mADrXKurPnWKZVW6wPAQvwQuBlZ2fb+ezOn7ffCG+xWOxe95RJkyYxadIkl6/JQkwmJibGo4os5c1ZE8iCy9uNBnqHJt52FW41Wvz6P8/IvU143edtwnIP873uWWYaH+Zry00e7aW1NXvDAuyrzwrqH95uLUvLKuT57/cz7ua2xLW1Fs+0/ZHzUdtreLIPzzbIZttbBSA9t5hCg5n26nPEfDMIMvZZB96YCLfO8KijmFajJixQT2ZeCUdKBV6ou+Y9tYzHPrwrrriCffv2uX19z549XHHFFdWyqIaGbJp6bdI6lPB2LOkdFeLLoYBe9C9J4lhgDwJUJbyuW8qr2mX4mAs5n1fC+kOZdvlbttj2s1A0PBG0qLd4Ww/vlwPpJB/KVJpGgb2ro0zDcxB4NpaFbPbKn5UT5wsYrN7E97pnUWfsA/8w+M+XED/bq/aJsln7d4a1XmZ90fA8Fnh33XUXs2bNorjYuchjUVERzz//PAMGDKjWxTUUyjQ8L4MWThqe/XOVSkVsdCjnacLj2lm8Zvw3ZtQM0f7OJ6oZvP3pt4z+4E82H7vgcn5bR3SovzBp6zveBi1kLf+SbeViG1eHLPDk5OOyzmVlP6x2Gp6hgBbrn2KR7m38KIaYm2D8Zmgf7/W9tCjtZyELvOB6IvA8NmlnzJjBZ599RseOHZk0aRJXXnklAIcPH+att97CZDIxY8aMGltofaayPjzH/D1XTVtio0NZdyCDAxkF7JPuJaNJDxLzXqG96gzTTk+kUDOSU1nXQAfn+WWTVqdRK7+wImhRf/G2Hp6sEdoKPFsNTzZp5e1lMrYanvyj29p0EpY/S/sLhzFLKja1HEPfEa+AunKfbSX5uHS3RX3IwQMvBF54eDhbtmxh/PjxTJs2TfGlqVQq7rjjDt5+++0aDQzUZypr0jppeC4EZ/foUMDqAAY47NuVuy4k8ZrPUm7V7Ga+zzsc2XkKYt8BX/uWmUabPZXBfta3Oq+01Lc3Nc4EtYO39fDk7+ClgrIfMfk916hVynssp6XIOAYthmrW8zwfwAUjlzRNGV80gYHdhlZa2EFZ8rFMfTFpvdpL26ZNG9auXcvFixc5evQoAO3bt6dp06YVnHl5o69k0MIpSutix0aXK0JQqcq2lvlo1FwkmIeNTzPW8iNPaz+jQ+ZaWN7XmgRqU2BREXhatVKQQJIg32ByWaDAHfklJsUPJKg5HE3aioq1ysOzC51Ltdu2Y3Rsoq18zopzafbz48z3sbZKlNrHM/LMf9hToOMJb8pCuSDcoUpyfSgcAJXcWta0aVN69+5N7969G72wAxsfnpc6nqMp7MqkDfL1oUOLsi0+sl9GQs1y878YaniOSz7hcPE4vHcHbF+uSEfbTeS+PhpFMHtj1n721ymumf0zH2456fJ1dwETmYISE1+knLb7UgpcYxu0kKSKC8PKArLAYFby6ByTjgFu6tDc7jy9VgNnd8Hyvvge+gqTpCbJOIyiIR+zL9vq623rReMeV9RXDU/spa0GHBOIPcVRw3O3Jze21KwF+w8ywE7pSl6KfsfaBcpsgJ+eJvPdIVB0yc6HB2WOY28itVO/2APAbDcNeira4jbtq71M+Xw3j/wvxeNrNlYcayJWlJpim6Il/6DI+ZnyDyNARIgvHcOD5LPofOpj64/jxeNYgq/gfsMs/mv+F0cyC7FI1qiu3ES7srQIstfwhMC7jFAEl7d5eE4mrWufSWx0WXcnR4EHkG7wtXaB6v8KJZKWFmfWYXr7RnTnUkrPsS4s2NdqllZnLl5FhUu/330WgO0n3PfoFVhx/PGoKPnYdvil0i1cSotGhz6vN3UII5gClvosInbfy9Yfx44DsIz7nZ1SaQAy3RpRbds8oMp9T1o4anj1JGghBF41IJumXgctnHZauBN4ocrfKsoEmExusRFUKsy9xnGfYQ4nLeFo805z5Zr7Gav5AVlxlDW81FOXeOqz3dVSodjT3hqCinHcLlhR4MLWBJYjtfIcOofPyICmZ/hRN4P+mj+xqH2g33x4YBWagKaK7/lQqcDzpg+tO5oF6O32zgoN7zKisjstbAWeWuUsyGSuDA9USvyoVM6/3rLGll9sYp/UlrsNL5HbbiBqycSzPqt5qehFKMhSAhWvrD3MlztP84Ebv5w3VKThNTbyS0wsXPe3kn/mDYUGe1dDRakpZhcmrUGphVf6GbFYYPNiYn99gGj1ef6xtOD3G1fBdY+CSoVKpVJcHofSrYVv3ZaF8gKNWkVzmzJUofWgnwUIgVctVNaHZ3uer4/GrRmh1ajpekVI6TOVnX8GygReXkmp4MOf365O4mDPuRRLPvQ2/QXLbqC7dMDuPGM1NOb2tExVY+GlHw+yOPkId77+u9fnyonBMhX78Mr+lk1ak03xTwqy4OOhsO45VBYTB5rezsSgRXSLu9VuHlngyULaq8Y95SAHLjRqFQG6yqe4VCdC4FUDlY/Sql3+7YrYVqHWa6jK/HiyfMwtMmKxSOTZlG8/lJHPPzFDGGSYy2lNNOSdY/KpJ5mk+Zrm/tYPn8WDfckVITQ8e3ZW4CbIyi/hwFnXLQQKSuwFXIU+PIsLk7b0WHfLAWuRziO/gEYPd79O58e+5IenB9AkwF7bkj+/F/Ktc1SHSQtlycchfj411gvZW4TAqwYUH57XQYuyXz13AQuZu66JRK9V0yumiZJjJaerWEpz62w1hPQca+f4w1IrZjZ/C2KHo8bCFJ/P+a7JQpqTTXXIKqHh2VORT7PHi79y1+I/OHjOWegVlL5/TUod/BX58OxN2tKghdHIJM3XvFLwLOSdhWYdYOxv0PNhtx9Qx0BYdWl48vay+pKDB0LgVQvV4cOrSOB1iw5l//MJjLu5nfIB7RgRrMyRW2Qk30bDO5dTrKSlWHT+MPhtGLwMfAKIzNrOGv00OhX+5XSdczlFrNx60smf5A6zh701Ggue/gBsPmq//1mSJApK/8/lEuxembQFBsjLoMv60Uzx+RwNFug2DMZtgIhryp3H9nMYGeJr1wejKsjJx/VlHy0IgVctXNsqlKYBOm5oH+bVebZbyzzxA8qOaNmHF9PMX4l+5RQZ7QoDpOcW2+ThlUriWOsXIDuoA81VuSRmTIPkudbuU8BXO0/TJ+k3Zn27n1d/PuzRPZg9NIu1jWQrm6cN0h3LexUbLYrGLQu8ioIWti6JiKxtsOwGmmVupVDS81ZIItyzDPQVByBsg2XVZc4CRDe19rhoUcWcvupE7BeqBto2DyRlZrzXfgrbROOKNDxbZA0vplkAwb5azueVkFNktDNpz+UUK9n3dlHd5lfy202fUPTdVIZrk+GPBfDPZrjvPRI/260M23osy6O1mD1MS3GVPyiz70wOTQJ0tAz1U4411P2+lW2QLr93KlVZwyVHk9bx/8QiSWgw84T2SyamfwtI5AZfyT3nx9IioCueVoTU2bhWqsucBeh/TSTpOSXc0blFtc1ZVYSGV01Uxilrq+G52kfrjg4tAtGoVVzbuoldFRTboIXBZFF6i/o4ao9aX541jWFJs2dBFwRpW2HZjdyqTlWGBPlqnYqtrt13zu6YJEl8tC0NT3CMLMuculjI3W9u4oZ5vynHjmbm0+2FX3jrtyMezV2f8La5kozsv/P30SjlnGyDFj/sOcvVs39m2cZjyrGA4kxW617iMe03qJGgx2h+u+ljjkkt3f5/u8I2Z686UlJkfH00jL+lHe1bBFU8uJYQAq8OkZt4g3ca3uJh3dk2/XbahAXYmbT5Dk220y5aWzw65vepS4XzNv++8OjvEBkLRRd5X/cqM7Sr8MFEkK8PD3/wp915j360kx/3nlOer9mb7pTLt/tUtss157lpAL7fRcTypR8PkFdsYsEvf7s8pz7jaftMg8m1hheg1+JXKvBsfXjTv9xLkdHMvJ8OWQ/8/TOTj4wmTn2IPMmPaaon4V+LMGA1H8vTqB2xHdumGjW8+ogQeHWMrOW522XhCh+NWtnraCvw8hyKe8oCz7EMlayMWiQJmraFMb/AdRMAGKf9kc91z9NKdZ71h+3L+ANsPlpm6u49k+P0+hcpp/ky5TTPf7+/wsICgMuS/Q05DOKphmdwo+EF2gg8Wx+eXLHaBxO5302D1fcTYM5hj6UNdxte4vOS3tamTRbnaikVYRu0aFeNGl59RPjw6hidVk2R0eyVSWtLiE1BgDyHxNVTF4sA5197WcNT3G9aPVLCy4z73ZdXff5LrPoYbY+P4Zx6HD9betud64kG89TnVl/gnZ2dewFvPnqBIF+t0gDalXyohvTAOsPzoIX9/6NsvgbotcquGlsfnq+PhitU53nT502Cd1pLs/3e9D7+7+xADPiAJJFbbCqrluJFMrz8+dBp1LRs4lfB6IaN0PDqGDk6641Ja4u9hmcVeLKZLDdbdhR4dhpeKQazhXWWntxVkkSKpQPBqkL+q1vEHO0H6Ckr7fR5ymnlb8mFLiZf0/Y6Muk5xQx/dzsD39qsHLNdw52vbyQjt9hu1szcYv678ZjSYrC+Y/QwiOPepNWUCTwbDS+ueDNrdNPprj5KoToQhq7im/DJVmFXSnahoawGYiU0vNbN/OtF79iapF4IvCVLlhATE4Ovry9xcXHs2LGj3PGff/45nTp1wtfXly5durBmzRq710eNGoWqdJ+g/OjXr19N3kKl0VVR4AW78OHZ1s8D9z48W01Krr12ljCGGp7jbdNAAEZpf+FL3RxiVOfs5pAkyeXOErnFnyvO5hQ5HbMVeH9n5PPK2sN2Zu6IFTtI+ukQkz9JdTq3PuKpduqYliKbtAE6G5PWaAZjMax5mueL5xGsKmSnpT33WOZj7jjAaadMdqGxrFqKFz482eVRnRHa+kqdC7xPP/2UxMREZs+ezc6dO+nWrRsJCQlkZma6HL9lyxaGDRvGmDFjSE1NZfDgwQwePNipo1q/fv04d+6c8vj4449r43a8Rtbw3NXCqwhbgSfvpb0y3D4q5mzSWv+10/BsNA4TWl4xPcBIwzNckIK5Rn2SH3TPMlBt1cw2HM6k54u/8tuhDKf1yBU3wLmApa0gk6ONjl9ax2Rbeb4/jrhuVNRQcfTh2QYt5B+/4MJ/rHXrdiwHYJnpX9xvmMXh4ibsOe28U+ZSoUFxObgrROEK+UdXabx9GVPnAm/hwoWMHTuW0aNH07lzZ5YtW4a/vz8rVqxwOf6NN96gX79+PP3001x11VXMnTuXa6+9lrfeestunF6vJyIiQnk0adLE5Xx1jZwD5U3QwhZXUVpnDc+5GxrYCxtX1XU3WrpxV0kSW82dCVQVs1i3hHna5Tz6/iayCgz8nZFf7tpGrrDX1G2/oPN+OsTZ7CIcLcDq2N/bEHA0aeV9tAF6Lf46DQPVm5lzbgKk7wH/Zjypnck80zAim1p/zP44csEp6Tu70IhRbsKt9vyrnXB1BG3DAhjQJbIqt9QgqFOBZzAYSElJIT6+rA2cWq0mPj6erVu3ujxn69atduMBEhISnMZv2LCBFi1a0LFjR8aPH09WlmeJtLVNVU1aJWhRXJZ43MFRw9O6NmltBZC7bUyZNGG4cQaLTPdikVQ8oN3At7rn6KA67XJ8eThGbfOKTU5eQLNFatBBC09xDlpY37tQrZHuqc+xWLcEP6kIWt8Aj27iD2IBuK2jNYn3jyPnnSLc9hqe51/tftdE8NuUW7imZUjFgxs4dSrwLly4gNlsdup2Fh4eTnp6ustz0tPTKxzfr18/Vq5cSXJyMvPnz2fjxo30798fs9n1l7qkpITc3Fy7R21RFrSoapTWSG6phtfeQcNzTEuRTVr5C5NfYuL21za6vYYFNYtM/2a4cQaZUigd1af5TjeTIZoNeJNE4qiRmCwWJyFokVwHQ6Di/hkNCVc7LdqrTjPm0BhanvgCi6RiuWoIjPgOgqMU39wtnawCLzUtWynVL7+flwqNSlqMNyZtY6LOTdqa4IEHHmDgwIF06dKFwYMH88MPP/Dnn3+yYcMGl+OTkpIICQlRHtHR0bW21qpGaWUf3oV8g2ImNQ/SKxU3wH1aiiw+fj3g7ItzxVbL1fQvSeJ3cxf8VAZe9VnO6z5vE4BzMMIVjuar2SI5mbCS5F7D+yr1jEfXaQjYmbSSxNUZ3/O9biZhhcexBLRguHEGLxfdQ35pppFchqtNswBimvljskj89Y+1bH6T0uKa2YVlnwFvdlo0JupU4IWFhaHRaMjIsP/CZWRkEBHhnMMFEBER4dV4gLZt2xIWFqa0lnRk+vTp5OTkKI9Tp055eSeV54b2YQT5auleWu/OW1yVzg7Ua4kIKcunqigtxZs9q1mEMNL4DPOND2CS1Nyj2cz3umfprDpZ4bmOKRsmi+TkeC/Ph3fIRUmlhsqfJy9Z005K8uDrR3gwfT5+KgPnmvVBPX4zh3xjAUjLsiaPG20Ke8pdyIqN1mPy3lurhudi/7RAoU7/V3Q6HT169CA5OVk5ZrFYSE5Opk+fPi7P6dOnj914gHXr1rkdD3D69GmysrKIjHTtlNXr9QQHB9s9aotH+7Zj96w76RRRuWsG6DR2uVPy88iQsq5Rjr/2jonHGi/3AUuoWWoeyFDDc5yRmtFWnc7Xutn8R7OO8kzcEqODwDM7a3gWyX1qR0OzaC8WGNh6LMvlbhKA5Z99A8tvgT2fYpLUzDc+wOm7P4LAFrQu7SWbdtGa5mO2CUbc1MG+Ko8s8LILDTZtGoWG54o6/xlITEzknXfe4cMPP+TgwYOMHz+egoICRo8eDcCIESOYPn26Mv7xxx9n7dq1vPbaaxw6dIg5c+bw119/MWmStTZEfn4+Tz/9NNu2bePkyZMkJyczaNAg2rdvT0JCQp3cY0VUpSqISqWy0/ICSzuTRdgIPGcfnn2U1tPad46kSB0ZUPIy68zXolcZedHnfZb4vEEwrnPxHP1WJovFSRhYJMmtD8/d8frKba9tYNg72/h5v6PLQOI/mnU8euQRyDrKOakpDxhmIt34JL3aWIVZ62b+AJzMKkSSJMU3p9Wo6NOumd2PnFxO6lKhQQk+eRO0aEzU+f/K0KFDWbBgAbNmzSI2NpZdu3axdu1aJTCRlpbGuXNlSa/XX389q1evZvny5XTr1o0vvviCb775hmuusRY51Gg07Nmzh4EDB3LllVcyZswYevTowR9//IFeX3/qclUntgIvqLRRT6RN53d3eXiyrCkoqZzAA8gmiLHGp3jB+BAGScMAzQ5+0M2gq+qY09hNDrl0ZjcmrVsNrwGpeGpVWRXijX+X5ZQGU8ASnzd40ed99CojG7mW/iVJhHe5lakJHZVxrZtaBd4/WYV2ZfS1ahVBvj50t+lkp5i0BUZlf3N11rW7nKgXe2knTZqkaGiOuAo0DBkyhCFDhrgc7+fnx88//1ydy6v32FaUDdQ7a3iO5o1jHp5j8xjvUbHC3J+/LFfyls9iWqnP84VuDvNNw3jP3B+5geWnf9n7Rk0WyaknhsXi3ihuQPIOrUatBBDkgJTlVAo/6GbQSn0eg6RR/n96tG7Ka0O62Wn6rWxMWtuCBPIOips6NOev0v4ZssDLyC3GZJFQqSCuTbOav8kGSJ1reIKqIzfYBmsdO4BI26CFtvygRX5J+ZV1PWWP1I4BhiR+NPdGpzLznM9HvOuzgFBctyzcfOSCc5HLcoIWDcmktd3L6qdVY9z0Jub37qSV+jxpluYMMczmPfNdxDQL4J0RPZ2i9DHNyjQ8W1eAXAXlpivL/HhyJRVZMF4VEVxvGl/XN+qFhieoGvYmrec+PFm25JfYl5WqCnn4M9H4OFssyczS/o94TSpr1NOZbJjEX1Inu7HvbjrhdL4k4bajeUPT8MBMKHkMOfI0Phf/AOBHc2+mG8eSi1WDe390b0VDs6VVqcA7m11k96MgC7yuLUMI9tWSW2wixM8Htars/+e6tkK7c4fQ8C4DQio0acvfS+vYHrDqqFhljmew4QWOWSKJUl3kE92LTNB8i4ryq4nsPp3NjhMXXb7WkHZg+GhU9FAdZo1+Om0u/oFRpWOmcTQTjY8rwm7ybe3d+tqaB+rx12mwSFYtT0YOVmg1agZ3b4lOq6ZTRLDdZ+C6tk1r8M4aNkLgXQa4CloE6rWKtueYlqJy2FrmrhqxK7xpqHxQas2/DC/xlflGtCoLU30+5UOf+YThXDhUxtWeXpmGErRQYWGU5Ws+1c0lSnWRXP/WLGy1hI/Md2CrvkaGuq89p1KpaFUauDiWad2zrFWr7FoJPD/wanbNuoOOEUGEliYfq1TQu40QeO4QAu8ywJWGBzCgSyQtQ/2cigk4a3ieCzxZoHpKIb4kGsczxfgIhZKemzV7WaOfTh/1fq/mgYZRWCCMHD70mc8kyyq0Kgtfm29gfd/POaVv7zS2omwkOTXleGnJLcdadSqVSmmpGFrqs7sqIlgRfgJnhA/vMsBWCAXZBDDm3dfVWrfOIbHY2YfnhYanr8wWOBVfmPuyy9KOt3zepJP6FKt8XuZN8z28YboXi4e/u/Ve3B3fyBr9dFqosimSdMwyjeJzc1/e8gtCkpxzEytq/CT3dU3PKQacBZ4t8vYy4b8rH6HhXQbYFh6wFXjg+ktVJvDsNTx/D8zVQC81PFuOSlcw2PACq023olZJPK79itW6lwjHtc/OkbrW8CRJYs3ec5y6WGj/gsUM61+GlYNoocrmsOUK/mV4kc/NtyCbsI7pN1D2PrhDDmacL+0+V96OmP7XRBAZ4su917b0/IYaIULgXQbobWrpeWJylqWlWP+Ve2G42pfrSGClNLwyitEzwzSWyYZJ5Eu+XKc+yBr9dG5R76rw3Lq2aL/fc44Jq3Zy0yvrOZNdZNWMc8/ChwNh43xA4mPTrQwyzOWodIXdua6EtbstZzLNSgWeXDa/vB05Q3pGs3X67Y2ixFNVEALvMsC265StD88d8piLBQZyi42KhueZwKt4/puvbF7hmO8s13O34SX2WWJopsrjA90rTNN+jBb35nVda3i//13Wxe2Geb8x5eUFsOxG+GcT6AIxD17OdNNYiktbJfaOsQYPDp7LdRkYqqjDWdMA6zznZYEntsdWGSHwLgP0WvcmrStaN/OnfYtADGYLa/acUzpmBXsg8AI8EHhRNikx5XFSiuRew/O8b7LucX5U+z2f6V6gJc7tIaHuNTw5gqzFxDTtxyxTJUFhFkR0gUd+x9D533bj5YIlS9YfY+tx5wK0jnuLHSkrCmDNk7zcG+zUBkLgXQbovBR4KpWKe7pbfT2rtqcpxz3R8IK80CA9wYAPz5tG8ojhCXIkf65VH2WNfjp3qv90GlvXGp7BZKYl5/lM9wKPar+3Huw9Dsb8Cs3aOZW/ctXkyBbHRj6OyDsolPm8rGojcEYIvMsAe4HnWVBhYLcooKyZto9G5VHQwpP5Az0Quo78bOnNAEMSqZb2hKgKWa57ndnaD9FRtgtEkqy5eJ70xq1ukg9m4H98LWv007lWfZRcyZ9HDE/AXa+Cj1Wj/XDzSa/m9FTDk/G2jJfAGSHwLgNsnd+ealfRTf0VH5N8nrsvlG3kzxMN0hsNz5bTUnOGGGaxzHQ3AKO1P/OlbjatVdby/T/uPUfbGWu4ZcEGtz04aoJTmZf4Z9VkXpdeJURVyC5LO+4yvGzXpDwrv4TX1v1td15F8slYTpI1WFNNbOcQJm3VEQLvMsB2d4InWprM4O5lgixAr3X7hbq3e1nE0RPtzRM/nztMaJlnepBRhqe5KAXSpbRF5N3qsiZNpy8VsTPtUqWv4SmSJPHM8q/JXXIrD2vXAvBf0wCGGGZzWmphNzY1Ldvp/AoFXgVBC41aRaiNm0EoeFVHCLzLgKujrKkIIX4+Xvl5BnSJVAoLBJYj8Gx9Z54I1KoIPJkNlu7cVZLEdksnglRFvKV7k5e176LHAEBmbkmVr1ER+SmfMfPMeK5WneCiFMhow9MkmYZjdJGvf/S8fcvKmQOuqnB+T0xzW7NWaHhVRwi8y4AQPx9Sn7uDbdNv9+48fx9uK+2CFajXujVFbZNm/Xw8MWmrlqsnk04zHjQ8yxume7BIKh7U/sY3uudopzrDiQsFnLpYyF8nLzLl89288/vxarkmAMYi+P5xgn4YR5CqiB2WjtxVksR6S3e3p8i+UJn/u6mtB0GLigVes4CyorXCh1d1hMC7TGgSoMPPC3NW5j/XtUatgs5RwUy8tWy/p06jZmq/jvz0+E12As8jDU9XfTsWzWh43TSEh4zTOC+FcJX6FN/rZhJw8DPGfPgn/162lS9STvPSmoNVvtapi4Us/3INJ+ddBykfYJFUvGkazDDDTNJxvWXrQn4JE1fv5Mc951y+7ojtj5JOW/HXz1bDE/Ku6oi9tI2cGzuE8ccztxEWqLPbseGv1zDhFqsAtP3SeSJUKxOlrYjNli7cVZLE6z5LuFGzn3EXX6WZeRvPMZpCPMv7c8Xz3+9n3YEM7ugcTu7Wlcz1eR9/VQnnpWCeNE5kk6VLuee//ONBt8Iur9i+zqBKBeHBel4YdDVfp55h3E3tKlxfE2HSVitC4Alo6aJMkW2bv/BgX/6YeitBvloMLsyw1s38lZpt/a+JcFnQsjo4TygjjNMZb/mORO3n3Kf5g1jVUSYaH+eQ1Ir9Z3NYsekkj9/eQSmg6Y6cIiNf7TzN+5tP4k8x1+xYyH06a5HOTearedI4kfOEVrimdQfd9/SV98DKRIX4oVKpGNEnhhF9YiqcGyDMJhdPNOapOkLgCVyiddAmopuWCZD+10Tw07505fnnj/RhUfIRLuSV8Pbwa8kqMNTYuiyoWWIezA5LJxbr3qKd+hzf6p7jedMIBiy2lkv+cudpHu3bjmn9O7mdZ8bXe/lxzzk6qdJY4vMG7dTnMEsqFpqGsNQ80OMKLuXVEsx0EHgtm7ivf+eOQbFRbD9+kRKzhTE3tvH6fIE9QuAJ7OgUEcSh9DwGxka5HbP0Pz0wWyR2pl3i6qhg/HVaXr6nzPTz86meoEV5/Cl14q6Sl1ng819u16Tyss97XK/ez3Tj/5GHP8s2HitX4K3bn86DmmRma1eiVxk5JzXlccNEdkgVR1c9xXGvrCtNuiLatwjis0fd91wWeIfQkQV2rB57HW8O685Td15Z7jiNWkWvmKZKAUpbHNNS3h5+rVK9FzzbnlYej9/egV2z7uDOnlfzf8aneNE4HKOk4W7NNn7QzaCLyhqxvZBf4rLnblHeJRaq3+Bln/fQq4z8Zo7lrpKXq1XYAcz5V2e751Ghlfc1CqqHeiHwlixZQkxMDL6+vsTFxbFjx45yx3/++ed06tQJX19funTpwpo1a+xelySJWbNmERkZiZ+fH/Hx8Rw5cqQmb+GyoWmAjn91i7ILYFSGLdNuo3urUBYNjeWuLpH8PvVWtk2/nUNz+3lUpKA8xt7cllB/Hde3b4aEmnfN1mTgU5bmtFZn8qVuNqM0a+n54jqGv7vd7tzv164h49U47tZswyhpeNE4nDHGKVwiuNLrcVf2ftQNbVgz+SblefxV4ZW+hqB6qHOB9+mnn5KYmMjs2bPZuXMn3bp1IyEhgczMTJfjt2zZwrBhwxgzZgypqakMHjyYwYMHs2/fPmXMK6+8wuLFi1m2bBnbt28nICCAhIQEiouLa+u2Gj1RoX58PeEGu90cESG++PpoWPafHnQMD+L90b149d9dXZ7/0Zg4l8cfvqGNki/Y75oImpSWNt8ltWeA4WV+MvdCpzIzx2cly30WcjztNEUGM0gSpi1LSdj6H2LUGZyWwhhimM275gFIHn4NJt/egSfiOzgd/yWxLy/f04WZA66iW3QoO5+7Q3mtc1Qwvz99Kx+PvY7urZp4dB1BzaGSKqpCWMPExcXRq1cv3nrrLQAsFgvR0dE89thjTJs2zWn80KFDKSgo4IcfflCOXXfddcTGxrJs2TIkSSIqKoqnnnqKKVOmAJCTk0N4eDgffPABDzzwQIVrys3NJSQkhJycHIKDK//LL/AMg8nCW+uPckvH5ny/+yz394ymXfNArpz5EwAj+rTm879O0yYsgFX/F2eXqpHyzyXuW7rFZjaJ/2h+5Tnt/9CrTJyWwngv5DEGm3+hW8FmAH4y9+IZ41hyse/14Y4gXy2vDenGnVdHAHDgbC5PfrqLYpOZ7x+7keAqVIEWVI7KfkfrVOAZDAb8/f354osvGDx4sHJ85MiRZGdn8+233zqd06pVKxITE3niiSeUY7Nnz+abb75h9+7dHD9+nHbt2pGamkpsbKwypm/fvsTGxvLGG284zVlSUkJJSVlELTc3l+joaCHw6pjzeSWoVdAsUO92jMFkIWHR75y4YN8z4mrVSd7yeYM26rK0kRJJy0um4aw034lt97A7OofTMtSPQoOJf/eI5vj5fK5r2wydVo1E5YINgpqlsgKvTqO0Fy5cwGw2Ex5u79sIDw/n0KFDLs9JT093OT49PV15XT7mbowjSUlJPP/885W6B0HN0TzIvaCT0WnVrJ9yC2AVfkazhfc2nWDhOrjb8DIv+7zHIM0WTljCmWSczH7Jmtrx4uBr6NIyhE6RQU7+StHm8PJFpKUA06dPJzExUXkua3iChoVOq0anVTP59g5Mvt3qazOa7uHU0VR0YW35vmmTcvtCCC5/6lTghYWFodFoyMiwz1bPyMggIiLC5TkRERHljpf/zcjIIDIy0m6MrYlri16vR6+vWJsQNDx8tBqiO/Ws62UI6gl1GqXV6XT06NGD5ORk5ZjFYiE5OZk+fVwnW/bp08duPMC6deuU8W3atCEiIsJuTG5uLtu3b3c7p0AgaCRIdcwnn3wi6fV66YMPPpAOHDggjRs3TgoNDZXS09MlSZKkhx56SJo2bZoyfvPmzZJWq5UWLFggHTx4UJo9e7bk4+Mj7d27Vxkzb948KTQ0VPr222+lPXv2SIMGDZLatGkjFRUVebSmnJwcCZBycnKq92YFAkG1UNnvaJ378IYOHcr58+eZNWsW6enpxMbGsnbtWiXokJaWhtpmI/v111/P6tWrmTlzJjNmzKBDhw588803XHPNNcqYqVOnUlBQwLhx48jOzubGG29k7dq1+PqKTHeBoDFT53l49RGRhycQ1G8q+x2t850WAoFAUFsIgScQCBoNQuAJBIJGQ50HLeojslszNze3jlciEAhcIX83vQ1BCIHngry8PACx20IgqOfk5eUREhLi8XgRpXWBxWLh7NmzBAUFldvnVd6CdurUqcsmmivuqf5zud0PeH9PkiSRl5dHVFSUXdpaRQgNzwVqtZorrrjC4/HBwcGXzQdPRtxT/edyux/w7p680exkRNBCIBA0GoTAEwgEjQYh8KqAXq9n9uzZl1WlFXFP9Z/L7X6g9u5JBC0EAkGjQWh4AoGg0SAEnkAgaDQIgScQCBoNQuAJBIJGgxB4FXDx4kWGDx9OcHAwoaGhjBkzhvz8/HLPWb58ObfccgvBwcGoVCqys7OrZd7qoDLXLS4uZuLEiTRr1ozAwEDuu+8+p74iKpXK6fHJJ5/UyD0sWbKEmJgYfH19iYuLY8eOHeWO//zzz+nUqRO+vr506dKFNWvW2L0uSRKzZs0iMjISPz8/4uPjOXLkSI2s3R3VfU+jRo1yej/69etXk7dghzf3s3//fu677z5iYmJQqVQsWrSoynO6pVrrLl+G9OvXT+rWrZu0bds26Y8//pDat28vDRs2rNxzXn/9dSkpKUlKSkqSAOnSpUvVMm91UJnrPvroo1J0dLSUnJws/fXXX9J1110nXX/99XZjAOn999+Xzp07pzw8LanvDZ988omk0+mkFStWSPv375fGjh0rhYaGShkZGS7Hb968WdJoNNIrr7wiHThwQJo5c6bLlgAhISHSN998I+3evVsaOHCgVy0B6uM9jRw5UurXr5/d+3Hx4sV6eT87duyQpkyZIn388cdSRESE9Prrr1d5TncIgVcOBw4ckADpzz//VI799NNPkkqlks6cOVPh+evXr3cp8Ko6b2WpzHWzs7MlHx8f6fPPP1eOHTx4UAKkrVu3KscA6euvv66xtcv07t1bmjhxovLcbDZLUVFRUlJSksvx999/vzRgwAC7Y3FxcdIjjzwiSZIkWSwWKSIiQnr11VeV17OzsyW9Xi99/PHHNXAHzlT3PUmSVeANGjSoRtZbEd7ejy2tW7d2KfCqMqctwqQth61btxIaGkrPnmVt/uLj41Gr1Wzfvr3ezVsT101JScFoNBIfH68c69SpE61atWLr1q12YydOnEhYWBi9e/dmxYoVXpfuqQiDwUBKSordWtRqNfHx8U5rkdm6davdeICEhARl/IkTJ0hPT7cbExISQlxcnNs5q5OauCeZDRs20KJFCzp27Mj48ePJysqq/htwoDL3U5tziuIB5ZCenk6LFi3sjmm1Wpo2bUp6enq9m7cmrpueno5OpyM0NNTueHh4uN05L7zwArfddhv+/v788ssvTJgwgfz8fCZPnlxt679w4QJms1lp8GS7lkOHDrldv6vx8trlf8sbU5PUxD0B9OvXj3vvvZc2bdpw7NgxZsyYQf/+/dm6dSsajab6b6SUytxPbc7ZKAXetGnTmD9/frljDh48WEurqTr14X6ee+455e/u3btTUFDAq6++Wq0CT+A5DzzwgPJ3ly5d6Nq1K+3atWPDhg3cfvvtdbiyuqVRCrynnnqKUaNGlTumbdu2REREkJmZaXfcZDJx8eJFIiIiKn396p63Ju8nIiICg8FAdna2nZaXkZFR7lrj4uKYO3cuJSUl1bY/MiwsDI1G4xQhLm8tERER5Y6X/83IyCAyMtJuTGxsbLWsuzxq4p5c0bZtW8LCwjh69GiNCrzK3E9tztkofXjNmzenU6dO5T50Oh19+vQhOzublJQU5dzffvsNi8VCXFxcpa9f3fPW5P306NEDHx8fkpOTlWOHDx8mLS2NPn36uF3Trl27aNKkSbVuBtfpdPTo0cNuLRaLheTkZLdr6dOnj914gHXr1inj27RpQ0REhN2Y3Nxctm/fXu79VRc1cU+uOH36NFlZWXZCvSaozP3U6pxehTgaIf369ZO6d+8ubd++Xdq0aZPUoUMHuzSO06dPSx07dpS2b9+uHDt37pyUmpoqvfPOOxIg/f7771JqaqqUlZXl8bz16X4effRRqVWrVtJvv/0m/fXXX1KfPn2kPn36KK9/99130jvvvCPt3btXOnLkiPT2229L/v7+0qxZs6p9/Z988omk1+ulDz74QDpw4IA0btw4KTQ0VEpPT5ckSZIeeughadq0acr4zZs3S1qtVlqwYIF08OBBafbs2S7TUkJDQ6Vvv/1W2rNnjzRo0KBaT0upznvKy8uTpkyZIm3dulU6ceKE9Ouvv0rXXnut1KFDB6m4uLje3U9JSYmUmpoqpaamSpGRkdKUKVOk1NRU6ciRIx7P6SlC4FVAVlaWNGzYMCkwMFAKDg6WRo8eLeXl5SmvnzhxQgKk9evXK8dmz54tAU6P999/3+N569P9FBUVSRMmTJCaNGki+fv7S/fcc4907tw55fWffvpJio2NlQIDA6WAgACpW7du0rJlyySz2Vwj9/Dmm29KrVq1knQ6ndS7d29p27Ztymt9+/aVRo4caTf+s88+k6688kpJp9NJV199tfTjjz/avW6xWKTnnntOCg8Pl/R6vXT77bdLhw8frpG1u6M676mwsFC68847pebNm0s+Pj5S69atpbFjx3otHKqCN/cjf+YcH3379vV4Tk8R5aEEAkGjoVH68AQCQeNECDyBQNBoEAJPIBA0GoTAEwgEjQYh8AQCQaNBCDyBQNBoEAJPIBA0GoTAEwgEjQYh8AQNBldly2u7dLmgYdMoq6UIGi79+vXj/ffftzvmrkCB0WjEx8fH7pjBYECn03l93cqeJ6hfCA1P0KDQ6/VERETYPZo0aQJYGwktXbqUgQMHEhAQwEsvvcScOXOIjY3l3XffpU2bNvj6+gKQlpbGoEGDCAwMJDg4mPvvv9+u/JC78wQNGyHwBJcVc+bM4Z577mHv3r08/PDDABw9epQvv/ySr776il27dmGxWBg0aBAXL15k48aNrFu3juPHjzN06FC7uRzPEzR8hEkraFD88MMPBAYG2h2bMWMGM2bMAODBBx9k9OjRdq8bDAZWrlxJ8+bNAWvtuL1793LixAmio6MBWLlyJVdffTV//vknvXr1cnmeoOEjBJ6gQXHrrbeydOlSu2NNmzZV/rZtUCTTunVrO6F18OBBoqOjFWEH0LlzZ0JDQzl48KAi8BzPEzR8hMATNCgCAgJo3759ua97cszTawkuL4QPT9DouOqqqzh16hSnTp1Sjh04cIDs7Gw6d+5chysT1DRCwxM0KEpKSpzaJ2q1WsLCwjyeIz4+ni5dujB8+HAWLVqEyWRiwoQJ9O3b16VJLLh8EBqeoEGxdu1aIiMj7R433nijV3OoVCq+/fZbmjRpws0330x8fDxt27bl008/raFVC+oLosS7QCBoNAgNTyAQNBqEwBMIBI0GIfAEAkGjQQg8gUDQaBACTyAQNBqEwBMIBI0GIfAEAkGjQQg8gUDQaBACTyAQNBqEwBMIBI0GIfAEAkGjQQg8gUDQaPh/fPmL373P72wAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.020469, + "end_time": "2024-03-01T12:32:14.350650", + "exception": false, + "start_time": "2024-03-01T12:32:14.330181", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4259.495249, + "end_time": "2024-03-01T12:32:17.092386", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tvae/4/mlu-eval.ipynb", + "output_path": "eval/contraceptive/tvae/4/mlu-eval.ipynb", + "parameters": { + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "param_index": 2, + "path": "eval/contraceptive/tvae/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "tvae" + }, + "start_time": "2024-03-01T11:21:17.597137", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/tvae/model.pt b/contraceptive/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f67b412c47adf1cb62ea1ae271a8c4c8d485cd5 --- /dev/null +++ b/contraceptive/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83e5cfdc0fba243dca2214c9e502ba42f0fbd956ce1278f925bf3191c3d19436 +size 41130645 diff --git a/contraceptive/tvae/params.json b/contraceptive/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae4ee80e13aa6633b4517cdecc075ce5d963e6e --- /dev/null +++ b/contraceptive/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 8, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.775, "gradient_penalty_mode": "ALL", "synth_data": 2, "dataset_size": 2048, "batch_size": 2, "epochs": 100, "lr_mul": 0.075, "n_warmup_steps": 100, "Optim": "amsgradw", "loss_balancer_beta": 0.675, "loss_balancer_r": 0.95, "fixed_role_model": "tvae", "mse_mag": true, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 8, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation": "relu6", "head_activation_final": "leakyhardsigmoid", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/lct_gan/eval.csv b/insurance/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..83491199d80de2a6caa0a5dc9f5171e1c53d601c --- /dev/null +++ b/insurance/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.06709889649299153,0.04237791681767238,0.0004385647521223546,2.473484992980957,0.019484179094433784,0.6016532778739929,0.027372034266591072,5.971627103917854e-08,0.9092490673065186,0.016042448580265045,0.21174485981464386,0.02094193734228611,0.15322212874889374,0.00018641637871041894,3.3827340602874756 diff --git a/insurance/lct_gan/history.csv b/insurance/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..d03de27b8173c4d112dc4cb4f948e281e7eea323 --- /dev/null +++ b/insurance/lct_gan/history.csv @@ -0,0 +1,21 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.050247991016658486,10.28739813197949,0.017046372366257855,0.193543198867701,0.0,0.0,0.0,0.0,0.051501953338447495,320,40,56.587130069732666,1.4146782517433167,0.1768347814679146,0.09714822967071086,0.004852117493283003,0.1814691229723394,3.2799669435235045e-05,0.08009703867137433,0.0,0.0,0.0,0.0,0.004924363386817276,80,10,10.838585615158081,1.083858561515808,0.135482320189476,0.08860605396330357 +1,0.011156090041913558,11.664660499544821,0.0008925456208956461,0.08103617053711787,0.0,0.0,0.0,0.0,0.01129018383435323,320,40,56.461429834365845,1.4115357458591462,0.17644196823239328,0.07315202569152461,0.002358844585251063,0.4199162518021694,1.0129522642049472e-05,0.06003515869379043,0.0,0.0,0.0,0.0,0.002381171748857014,80,10,10.87971806526184,1.087971806526184,0.135996475815773,0.09902558447793126 +2,0.0034042925290123094,9.753277767247004,9.066182733613593e-06,0.038842388638295235,0.0,0.0,0.0,0.0,0.003444421075255377,320,40,56.57714056968689,1.4144285142421722,0.17680356428027152,0.06676940899342299,0.006381771888118237,0.19890570319257678,0.00012566872430852527,0.10561599396169186,0.0,0.0,0.0,0.0,0.006445180880837143,80,10,10.939128160476685,1.0939128160476685,0.13673910200595857,0.1349614226259291 +3,0.002452945696313691,1.2665704390786232,2.11658639054646e-06,0.028949995071161538,0.0,0.0,0.0,0.0,0.0024836566824887997,320,40,56.6224582195282,1.415561455488205,0.17694518193602563,0.0898168554296717,0.0007892202775110491,0.1727975025996784,5.349752479411052e-06,0.026769726537168026,0.0,0.0,0.0,0.0,0.000797343170415843,80,10,10.970746994018555,1.0970746994018554,0.13713433742523193,0.09117034515365958 +4,0.005982843652964221,8.70367759960153,4.6642777065386556e-05,0.05610372093506157,0.0,0.0,0.0,0.0,0.006053559554493404,320,40,56.52850890159607,1.4132127225399018,0.17665159031748773,0.0644026278750971,0.003287506682681851,16.047465970693157,1.088683212699948e-05,0.04266803655773401,0.0,0.0,0.0,0.0,0.0033236311399377884,80,10,10.905077695846558,1.0905077695846557,0.13631347119808196,0.06659559032414109 +5,0.004059205174417002,5.259179203768985,1.668044608822566e-05,0.026854972611181437,0.0,0.0,0.0,0.0,0.0041126025767880495,320,40,56.349008321762085,1.4087252080440522,0.17609065100550653,0.07893026950769126,0.006680386653169989,5.266184377827448,7.541818723666438e-06,0.033891827799379826,0.0,0.0,0.0,0.0,0.006806988373864442,80,10,10.916773557662964,1.0916773557662964,0.13645966947078705,0.03329654145054519 +6,0.003598407094614231,2.932740167534939,2.3322258428694864e-05,0.05079334197798744,0.0,0.0,0.0,0.0,0.003638804776710458,320,40,56.28336548805237,1.4070841372013092,0.17588551715016365,0.08040455909213051,0.001399435577332042,0.03829116202541627,5.023645737622928e-06,0.020581181766465305,0.0,0.0,0.0,0.0,0.0014129357805359177,80,10,10.841504096984863,1.0841504096984864,0.1355188012123108,0.08378756074234843 +7,0.0007492218805055017,0.170321739616611,4.540652907489067e-07,0.020798994018696247,0.0,0.0,0.0,0.0,0.0007574143121019006,320,40,56.28621745109558,1.4071554362773895,0.17589442953467369,0.09626067588105798,0.0007445405019097962,0.022931542915193857,7.847201817325067e-07,0.022695068223401903,0.0,0.0,0.0,0.0,0.0007516293582739309,80,10,10.79617428779602,1.079617428779602,0.13495217859745026,0.09322735751047731 +8,0.0005037498325691558,0.09364098146875222,5.574042118451682e-07,0.01803334672586061,0.0,0.0,0.0,0.0,0.0005089619417049107,320,40,56.07193899154663,1.4017984747886658,0.17522480934858323,0.09689875207841396,0.0005107733530167024,0.00858361014284128,1.0890024667808974e-06,0.017429086938500406,0.0,0.0,0.0,0.0,0.0005159668740816415,80,10,10.789738893508911,1.0789738893508911,0.1348717361688614,0.08924549822695553 +9,0.00035197669812987443,0.34276115468084073,1.0337585000970428e-07,0.013747934671118855,0.0,0.0,0.0,0.0,0.00035582845957833343,320,40,56.243980169296265,1.4060995042324067,0.17576243802905084,0.09651408242061735,0.00044438694512791697,0.4925519588992756,1.008665438284595e-06,0.01876302403397858,0.0,0.0,0.0,0.0,0.000449354921511258,80,10,10.789398670196533,1.0789398670196533,0.13486748337745666,0.08748150069732219 +10,0.00035949447574239456,0.1280737452699789,3.733976400273689e-08,0.01467946984921582,0.0,0.0,0.0,0.0,0.00036338540758151794,320,40,56.97472429275513,1.424368107318878,0.17804601341485976,0.09049497039522976,0.0006770771055016666,0.03294871362177219,5.858435483396818e-07,0.022112145693972705,0.0,0.0,0.0,0.0,0.0006841214883024805,80,10,11.102938890457153,1.1102938890457152,0.1387867361307144,0.09739410360343755 +11,0.0005287153543577005,0.5186876220466502,4.586899796882402e-07,0.016068345904932356,0.0,0.0,0.0,0.0,0.0005339632833056384,320,40,57.017521381378174,1.4254380345344544,0.1781797543168068,0.09512973661767318,0.001113348835860961,0.037260775306276625,1.0160834841599354e-05,0.026870487723499537,0.0,0.0,0.0,0.0,0.0011242192860663636,80,10,11.043520212173462,1.104352021217346,0.13804400265216826,0.09820128022693098 +12,0.0005380359987611882,0.11823862343915152,2.4365393777314724e-07,0.016488190542440863,0.0,0.0,0.0,0.0,0.0005437209121737397,320,40,56.564205169677734,1.4141051292419433,0.17676314115524291,0.09085401250049471,0.00033866083576867824,1.833931254097297,6.538905610309342e-08,0.014750928152352571,0.0,0.0,0.0,0.0,0.0003426387884246651,80,10,10.850196361541748,1.0850196361541748,0.13562745451927186,0.08306950689293444 +13,0.0005064211665740004,0.2563789238775499,2.7694440859517278e-08,0.017641309543978422,0.0,0.0,0.0,0.0,0.000511730196467397,320,40,56.38441038131714,1.4096102595329285,0.17620128244161606,0.08948770546121523,0.0009603310900274664,0.5317420201619558,1.386134010417095e-06,0.03154977913945913,0.0,0.0,0.0,0.0,0.0009698854177258909,80,10,10.881567239761353,1.0881567239761352,0.1360195904970169,0.09450749242678284 +14,0.000545748674630886,0.5983708105014387,2.0696913546770085e-07,0.01691446538316086,0.0,0.0,0.0,0.0,0.0005514411785952688,320,40,56.442954301834106,1.4110738575458526,0.17638423219323157,0.09217081667738966,0.0004825813819479663,1.5819854507564002,3.7346816483818657e-07,0.01985574811697006,0.0,0.0,0.0,0.0,0.00048788031563162804,80,10,10.967880487442017,1.0967880487442017,0.1370985060930252,0.08295555993681773 +15,0.0007052878027025145,0.5709439587292039,2.5470733571198996e-07,0.022239991917740554,0.0,0.0,0.0,0.0,0.0007124825457140105,320,40,56.290388345718384,1.4072597086429597,0.17590746358036996,0.09195324669708498,0.0005267480395559687,0.013233936694268778,2.673773188188733e-07,0.019443262554705142,0.0,0.0,0.0,0.0,0.0005322059372701915,80,10,10.92713212966919,1.092713212966919,0.13658915162086488,0.09297868777066469 +16,0.00039893748798931484,0.1465173953220642,1.9658869693291864e-07,0.013279191794572398,0.0,0.0,0.0,0.0,0.0004032720233226428,320,40,56.342530965805054,1.4085632741451264,0.1760704092681408,0.09154584540519864,0.00040967040695250034,0.07373528057341901,3.718508183148117e-07,0.014390437654219568,0.0,0.0,0.0,0.0,0.0004137940326472744,80,10,10.883796691894531,1.0883796691894532,0.13604745864868165,0.09073051768355071 +17,0.0006527752275360399,0.7538731722454941,9.391601971928554e-07,0.01640899422345683,0.0,0.0,0.0,0.0,0.000658580005983822,320,40,56.13877892494202,1.4034694731235504,0.1754336841404438,0.09342938290210441,0.0069938480504788455,4.3801720477524215,1.4892586449377632e-05,0.022057242598384617,0.0,0.0,0.0,0.0,0.007098242765641772,80,10,10.762710332870483,1.0762710332870484,0.13453387916088105,0.04032667824067175 +18,0.002608265062008286,1.1152279273919476,3.564367044772698e-06,0.03654546027537435,0.0,0.0,0.0,0.0,0.0026544345240836265,320,40,56.032405614852905,1.4008101403713227,0.17510126754641533,0.08181555005721748,0.001217631989857182,0.20784810947207005,8.713004294169657e-07,0.02198890279978514,0.0,0.0,0.0,0.0,0.0012277602480025962,80,10,10.823968410491943,1.0823968410491944,0.1352996051311493,0.07128694653511047 +19,0.0011638483822025592,0.29395967978312,1.489425534201283e-06,0.028835050016641616,0.0,0.0,0.0,0.0,0.0011756261381378863,320,40,56.04453897476196,1.401113474369049,0.17513918429613112,0.09410313908010721,0.0007999519177246839,0.12229554721852764,4.281340208578399e-07,0.02408305713906884,0.0,0.0,0.0,0.0,0.0008083746215561405,80,10,10.778013944625854,1.0778013944625855,0.1347251743078232,0.07425977652892471 diff --git a/insurance/lct_gan/mlu-eval.ipynb b/insurance/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..cb961692f5dbaa5bbbbed52a98576c2b481750ed --- /dev/null +++ b/insurance/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2582 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:14.698809Z", + "iopub.status.busy": "2024-02-29T05:45:14.697909Z", + "iopub.status.idle": "2024-02-29T05:45:14.733427Z", + "shell.execute_reply": "2024-02-29T05:45:14.732634Z" + }, + "papermill": { + "duration": 0.051669, + "end_time": "2024-02-29T05:45:14.736026", + "exception": false, + "start_time": "2024-02-29T05:45:14.684357", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:14.764031Z", + "iopub.status.busy": "2024-02-29T05:45:14.763432Z", + "iopub.status.idle": "2024-02-29T05:45:14.771855Z", + "shell.execute_reply": "2024-02-29T05:45:14.770919Z" + }, + "papermill": { + "duration": 0.024814, + "end_time": "2024-02-29T05:45:14.773896", + "exception": false, + "start_time": "2024-02-29T05:45:14.749082", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:14.799475Z", + "iopub.status.busy": "2024-02-29T05:45:14.799112Z", + "iopub.status.idle": "2024-02-29T05:45:14.803604Z", + "shell.execute_reply": "2024-02-29T05:45:14.802764Z" + }, + "papermill": { + "duration": 0.019794, + "end_time": "2024-02-29T05:45:14.805706", + "exception": false, + "start_time": "2024-02-29T05:45:14.785912", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:14.831327Z", + "iopub.status.busy": "2024-02-29T05:45:14.830979Z", + "iopub.status.idle": "2024-02-29T05:45:14.835342Z", + "shell.execute_reply": "2024-02-29T05:45:14.834452Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.0203, + "end_time": "2024-02-29T05:45:14.837797", + "exception": false, + "start_time": "2024-02-29T05:45:14.817497", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:14.866675Z", + "iopub.status.busy": "2024-02-29T05:45:14.866294Z", + "iopub.status.idle": "2024-02-29T05:45:14.872199Z", + "shell.execute_reply": "2024-02-29T05:45:14.871286Z" + }, + "papermill": { + "duration": 0.023055, + "end_time": "2024-02-29T05:45:14.874382", + "exception": false, + "start_time": "2024-02-29T05:45:14.851327", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "d5f32143", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:14.900205Z", + "iopub.status.busy": "2024-02-29T05:45:14.899549Z", + "iopub.status.idle": "2024-02-29T05:45:14.904839Z", + "shell.execute_reply": "2024-02-29T05:45:14.903790Z" + }, + "papermill": { + "duration": 0.020135, + "end_time": "2024-02-29T05:45:14.906890", + "exception": false, + "start_time": "2024-02-29T05:45:14.886755", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"lct_gan\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/lct_gan/42\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.012337, + "end_time": "2024-02-29T05:45:14.931176", + "exception": false, + "start_time": "2024-02-29T05:45:14.918839", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:14.957486Z", + "iopub.status.busy": "2024-02-29T05:45:14.956749Z", + "iopub.status.idle": "2024-02-29T05:45:14.966758Z", + "shell.execute_reply": "2024-02-29T05:45:14.965930Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.025557, + "end_time": "2024-02-29T05:45:14.968840", + "exception": false, + "start_time": "2024-02-29T05:45:14.943283", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/lct_gan/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:14.995739Z", + "iopub.status.busy": "2024-02-29T05:45:14.995340Z", + "iopub.status.idle": "2024-02-29T05:45:17.414699Z", + "shell.execute_reply": "2024-02-29T05:45:17.413760Z" + }, + "papermill": { + "duration": 2.435524, + "end_time": "2024-02-29T05:45:17.417121", + "exception": false, + "start_time": "2024-02-29T05:45:14.981597", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:17.444593Z", + "iopub.status.busy": "2024-02-29T05:45:17.443747Z", + "iopub.status.idle": "2024-02-29T05:45:17.456002Z", + "shell.execute_reply": "2024-02-29T05:45:17.455245Z" + }, + "papermill": { + "duration": 0.027936, + "end_time": "2024-02-29T05:45:17.458023", + "exception": false, + "start_time": "2024-02-29T05:45:17.430087", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:17.483605Z", + "iopub.status.busy": "2024-02-29T05:45:17.483284Z", + "iopub.status.idle": "2024-02-29T05:45:17.491235Z", + "shell.execute_reply": "2024-02-29T05:45:17.490344Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.023303, + "end_time": "2024-02-29T05:45:17.493225", + "exception": false, + "start_time": "2024-02-29T05:45:17.469922", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:17.521950Z", + "iopub.status.busy": "2024-02-29T05:45:17.521301Z", + "iopub.status.idle": "2024-02-29T05:45:17.630534Z", + "shell.execute_reply": "2024-02-29T05:45:17.629661Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.126937, + "end_time": "2024-02-29T05:45:17.633243", + "exception": false, + "start_time": "2024-02-29T05:45:17.506306", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:17.660673Z", + "iopub.status.busy": "2024-02-29T05:45:17.660281Z", + "iopub.status.idle": "2024-02-29T05:45:22.657180Z", + "shell.execute_reply": "2024-02-29T05:45:22.656333Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 5.012868, + "end_time": "2024-02-29T05:45:22.659511", + "exception": false, + "start_time": "2024-02-29T05:45:17.646643", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 05:45:20.092232: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 05:45:20.092294: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 05:45:20.094019: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:22.688021Z", + "iopub.status.busy": "2024-02-29T05:45:22.686662Z", + "iopub.status.idle": "2024-02-29T05:45:22.694184Z", + "shell.execute_reply": "2024-02-29T05:45:22.693466Z" + }, + "papermill": { + "duration": 0.023576, + "end_time": "2024-02-29T05:45:22.696113", + "exception": false, + "start_time": "2024-02-29T05:45:22.672537", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:22.721931Z", + "iopub.status.busy": "2024-02-29T05:45:22.721229Z", + "iopub.status.idle": "2024-02-29T05:45:31.775858Z", + "shell.execute_reply": "2024-02-29T05:45:31.774815Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 9.070351, + "end_time": "2024-02-29T05:45:31.778472", + "exception": false, + "start_time": "2024-02-29T05:45:22.708121", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BEST,\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:32.415306Z", + "iopub.status.busy": "2024-02-29T05:45:32.414952Z", + "iopub.status.idle": "2024-02-29T05:45:32.490368Z", + "shell.execute_reply": "2024-02-29T05:45:32.489508Z" + }, + "papermill": { + "duration": 0.093552, + "end_time": "2024-02-29T05:45:32.492795", + "exception": false, + "start_time": "2024-02-29T05:45:32.399243", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../insurance/_cache/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache4/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache5/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/insurance [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T05:45:32.523567Z", + "iopub.status.busy": "2024-02-29T05:45:32.523180Z", + "iopub.status.idle": "2024-02-29T05:45:32.979397Z", + "shell.execute_reply": "2024-02-29T05:45:32.978487Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.47377, + "end_time": "2024-02-29T05:45:32.981579", + "exception": false, + "start_time": "2024-02-29T05:45:32.507809", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:33.012530Z", + "iopub.status.busy": "2024-02-29T05:45:33.011757Z", + "iopub.status.idle": "2024-02-29T05:45:33.015989Z", + "shell.execute_reply": "2024-02-29T05:45:33.015144Z" + }, + "papermill": { + "duration": 0.022156, + "end_time": "2024-02-29T05:45:33.017941", + "exception": false, + "start_time": "2024-02-29T05:45:32.995785", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:33.046642Z", + "iopub.status.busy": "2024-02-29T05:45:33.046262Z", + "iopub.status.idle": "2024-02-29T05:45:33.053562Z", + "shell.execute_reply": "2024-02-29T05:45:33.052747Z" + }, + "papermill": { + "duration": 0.023921, + "end_time": "2024-02-29T05:45:33.055510", + "exception": false, + "start_time": "2024-02-29T05:45:33.031589", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9631369" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:33.083381Z", + "iopub.status.busy": "2024-02-29T05:45:33.083055Z", + "iopub.status.idle": "2024-02-29T05:45:33.177383Z", + "shell.execute_reply": "2024-02-29T05:45:33.176465Z" + }, + "papermill": { + "duration": 0.111148, + "end_time": "2024-02-29T05:45:33.179715", + "exception": false, + "start_time": "2024-02-29T05:45:33.068567", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 29] --\n", + "├─Adapter: 1-1 [2, 1071, 29] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 30,720\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 29] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,631,369\n", + "Trainable params: 9,631,369\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.15\n", + "========================================================================================================================\n", + "Input size (MB): 0.31\n", + "Forward/backward pass size (MB): 307.49\n", + "Params size (MB): 38.53\n", + "Estimated Total Size (MB): 346.32\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:45:33.212371Z", + "iopub.status.busy": "2024-02-29T05:45:33.212017Z", + "iopub.status.idle": "2024-02-29T06:09:52.653965Z", + "shell.execute_reply": "2024-02-29T06:09:52.653037Z" + }, + "papermill": { + "duration": 1459.479844, + "end_time": "2024-02-29T06:09:52.674951", + "exception": false, + "start_time": "2024-02-29T05:45:33.195107", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.050247991016658486, 'avg_role_model_std_loss': 10.28739813197949, 'avg_role_model_mean_pred_loss': 0.017046372366257855, 'avg_role_model_g_mag_loss': 0.193543198867701, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.051501953338447495, 'n_size': 320, 'n_batch': 40, 'duration': 56.587130069732666, 'duration_batch': 1.4146782517433167, 'duration_size': 0.1768347814679146, 'avg_pred_std': 0.09714822967071086}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004852117493283003, 'avg_role_model_std_loss': 0.1814691229723394, 'avg_role_model_mean_pred_loss': 3.2799669435235045e-05, 'avg_role_model_g_mag_loss': 0.08009703867137433, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004924363386817276, 'n_size': 80, 'n_batch': 10, 'duration': 10.838585615158081, 'duration_batch': 1.083858561515808, 'duration_size': 0.135482320189476, 'avg_pred_std': 0.08860605396330357}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.011156090041913558, 'avg_role_model_std_loss': 11.664660499544821, 'avg_role_model_mean_pred_loss': 0.0008925456208956461, 'avg_role_model_g_mag_loss': 0.08103617053711787, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01129018383435323, 'n_size': 320, 'n_batch': 40, 'duration': 56.461429834365845, 'duration_batch': 1.4115357458591462, 'duration_size': 0.17644196823239328, 'avg_pred_std': 0.07315202569152461}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002358844585251063, 'avg_role_model_std_loss': 0.4199162518021694, 'avg_role_model_mean_pred_loss': 1.0129522642049472e-05, 'avg_role_model_g_mag_loss': 0.06003515869379043, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002381171748857014, 'n_size': 80, 'n_batch': 10, 'duration': 10.87971806526184, 'duration_batch': 1.087971806526184, 'duration_size': 0.135996475815773, 'avg_pred_std': 0.09902558447793126}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0034042925290123094, 'avg_role_model_std_loss': 9.753277767247004, 'avg_role_model_mean_pred_loss': 9.066182733613593e-06, 'avg_role_model_g_mag_loss': 0.038842388638295235, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003444421075255377, 'n_size': 320, 'n_batch': 40, 'duration': 56.57714056968689, 'duration_batch': 1.4144285142421722, 'duration_size': 0.17680356428027152, 'avg_pred_std': 0.06676940899342299}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006381771888118237, 'avg_role_model_std_loss': 0.19890570319257678, 'avg_role_model_mean_pred_loss': 0.00012566872430852527, 'avg_role_model_g_mag_loss': 0.10561599396169186, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006445180880837143, 'n_size': 80, 'n_batch': 10, 'duration': 10.939128160476685, 'duration_batch': 1.0939128160476685, 'duration_size': 0.13673910200595857, 'avg_pred_std': 0.1349614226259291}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002452945696313691, 'avg_role_model_std_loss': 1.2665704390786232, 'avg_role_model_mean_pred_loss': 2.11658639054646e-06, 'avg_role_model_g_mag_loss': 0.028949995071161538, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024836566824887997, 'n_size': 320, 'n_batch': 40, 'duration': 56.6224582195282, 'duration_batch': 1.415561455488205, 'duration_size': 0.17694518193602563, 'avg_pred_std': 0.0898168554296717}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0007892202775110491, 'avg_role_model_std_loss': 0.1727975025996784, 'avg_role_model_mean_pred_loss': 5.349752479411052e-06, 'avg_role_model_g_mag_loss': 0.026769726537168026, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000797343170415843, 'n_size': 80, 'n_batch': 10, 'duration': 10.970746994018555, 'duration_batch': 1.0970746994018554, 'duration_size': 0.13713433742523193, 'avg_pred_std': 0.09117034515365958}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005982843652964221, 'avg_role_model_std_loss': 8.70367759960153, 'avg_role_model_mean_pred_loss': 4.6642777065386556e-05, 'avg_role_model_g_mag_loss': 0.05610372093506157, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006053559554493404, 'n_size': 320, 'n_batch': 40, 'duration': 56.52850890159607, 'duration_batch': 1.4132127225399018, 'duration_size': 0.17665159031748773, 'avg_pred_std': 0.0644026278750971}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003287506682681851, 'avg_role_model_std_loss': 16.047465970693157, 'avg_role_model_mean_pred_loss': 1.088683212699948e-05, 'avg_role_model_g_mag_loss': 0.04266803655773401, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033236311399377884, 'n_size': 80, 'n_batch': 10, 'duration': 10.905077695846558, 'duration_batch': 1.0905077695846557, 'duration_size': 0.13631347119808196, 'avg_pred_std': 0.06659559032414109}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004059205174417002, 'avg_role_model_std_loss': 5.259179203768985, 'avg_role_model_mean_pred_loss': 1.668044608822566e-05, 'avg_role_model_g_mag_loss': 0.026854972611181437, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0041126025767880495, 'n_size': 320, 'n_batch': 40, 'duration': 56.349008321762085, 'duration_batch': 1.4087252080440522, 'duration_size': 0.17609065100550653, 'avg_pred_std': 0.07893026950769126}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006680386653169989, 'avg_role_model_std_loss': 5.266184377827448, 'avg_role_model_mean_pred_loss': 7.541818723666438e-06, 'avg_role_model_g_mag_loss': 0.033891827799379826, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006806988373864442, 'n_size': 80, 'n_batch': 10, 'duration': 10.916773557662964, 'duration_batch': 1.0916773557662964, 'duration_size': 0.13645966947078705, 'avg_pred_std': 0.03329654145054519}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003598407094614231, 'avg_role_model_std_loss': 2.932740167534939, 'avg_role_model_mean_pred_loss': 2.3322258428694864e-05, 'avg_role_model_g_mag_loss': 0.05079334197798744, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003638804776710458, 'n_size': 320, 'n_batch': 40, 'duration': 56.28336548805237, 'duration_batch': 1.4070841372013092, 'duration_size': 0.17588551715016365, 'avg_pred_std': 0.08040455909213051}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001399435577332042, 'avg_role_model_std_loss': 0.03829116202541627, 'avg_role_model_mean_pred_loss': 5.023645737622928e-06, 'avg_role_model_g_mag_loss': 0.020581181766465305, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014129357805359177, 'n_size': 80, 'n_batch': 10, 'duration': 10.841504096984863, 'duration_batch': 1.0841504096984864, 'duration_size': 0.1355188012123108, 'avg_pred_std': 0.08378756074234843}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007492218805055017, 'avg_role_model_std_loss': 0.170321739616611, 'avg_role_model_mean_pred_loss': 4.540652907489067e-07, 'avg_role_model_g_mag_loss': 0.020798994018696247, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007574143121019006, 'n_size': 320, 'n_batch': 40, 'duration': 56.28621745109558, 'duration_batch': 1.4071554362773895, 'duration_size': 0.17589442953467369, 'avg_pred_std': 0.09626067588105798}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0007445405019097962, 'avg_role_model_std_loss': 0.022931542915193857, 'avg_role_model_mean_pred_loss': 7.847201817325067e-07, 'avg_role_model_g_mag_loss': 0.022695068223401903, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007516293582739309, 'n_size': 80, 'n_batch': 10, 'duration': 10.79617428779602, 'duration_batch': 1.079617428779602, 'duration_size': 0.13495217859745026, 'avg_pred_std': 0.09322735751047731}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005037498325691558, 'avg_role_model_std_loss': 0.09364098146875222, 'avg_role_model_mean_pred_loss': 5.574042118451682e-07, 'avg_role_model_g_mag_loss': 0.01803334672586061, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005089619417049107, 'n_size': 320, 'n_batch': 40, 'duration': 56.07193899154663, 'duration_batch': 1.4017984747886658, 'duration_size': 0.17522480934858323, 'avg_pred_std': 0.09689875207841396}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0005107733530167024, 'avg_role_model_std_loss': 0.00858361014284128, 'avg_role_model_mean_pred_loss': 1.0890024667808974e-06, 'avg_role_model_g_mag_loss': 0.017429086938500406, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005159668740816415, 'n_size': 80, 'n_batch': 10, 'duration': 10.789738893508911, 'duration_batch': 1.0789738893508911, 'duration_size': 0.1348717361688614, 'avg_pred_std': 0.08924549822695553}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00035197669812987443, 'avg_role_model_std_loss': 0.34276115468084073, 'avg_role_model_mean_pred_loss': 1.0337585000970428e-07, 'avg_role_model_g_mag_loss': 0.013747934671118855, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00035582845957833343, 'n_size': 320, 'n_batch': 40, 'duration': 56.243980169296265, 'duration_batch': 1.4060995042324067, 'duration_size': 0.17576243802905084, 'avg_pred_std': 0.09651408242061735}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00044438694512791697, 'avg_role_model_std_loss': 0.4925519588992756, 'avg_role_model_mean_pred_loss': 1.008665438284595e-06, 'avg_role_model_g_mag_loss': 0.01876302403397858, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000449354921511258, 'n_size': 80, 'n_batch': 10, 'duration': 10.789398670196533, 'duration_batch': 1.0789398670196533, 'duration_size': 0.13486748337745666, 'avg_pred_std': 0.08748150069732219}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00035949447574239456, 'avg_role_model_std_loss': 0.1280737452699789, 'avg_role_model_mean_pred_loss': 3.733976400273689e-08, 'avg_role_model_g_mag_loss': 0.01467946984921582, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00036338540758151794, 'n_size': 320, 'n_batch': 40, 'duration': 56.97472429275513, 'duration_batch': 1.424368107318878, 'duration_size': 0.17804601341485976, 'avg_pred_std': 0.09049497039522976}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0006770771055016666, 'avg_role_model_std_loss': 0.03294871362177219, 'avg_role_model_mean_pred_loss': 5.858435483396818e-07, 'avg_role_model_g_mag_loss': 0.022112145693972705, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006841214883024805, 'n_size': 80, 'n_batch': 10, 'duration': 11.102938890457153, 'duration_batch': 1.1102938890457152, 'duration_size': 0.1387867361307144, 'avg_pred_std': 0.09739410360343755}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005287153543577005, 'avg_role_model_std_loss': 0.5186876220466502, 'avg_role_model_mean_pred_loss': 4.586899796882402e-07, 'avg_role_model_g_mag_loss': 0.016068345904932356, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005339632833056384, 'n_size': 320, 'n_batch': 40, 'duration': 57.017521381378174, 'duration_batch': 1.4254380345344544, 'duration_size': 0.1781797543168068, 'avg_pred_std': 0.09512973661767318}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001113348835860961, 'avg_role_model_std_loss': 0.037260775306276625, 'avg_role_model_mean_pred_loss': 1.0160834841599354e-05, 'avg_role_model_g_mag_loss': 0.026870487723499537, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011242192860663636, 'n_size': 80, 'n_batch': 10, 'duration': 11.043520212173462, 'duration_batch': 1.104352021217346, 'duration_size': 0.13804400265216826, 'avg_pred_std': 0.09820128022693098}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005380359987611882, 'avg_role_model_std_loss': 0.11823862343915152, 'avg_role_model_mean_pred_loss': 2.4365393777314724e-07, 'avg_role_model_g_mag_loss': 0.016488190542440863, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005437209121737397, 'n_size': 320, 'n_batch': 40, 'duration': 56.564205169677734, 'duration_batch': 1.4141051292419433, 'duration_size': 0.17676314115524291, 'avg_pred_std': 0.09085401250049471}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00033866083576867824, 'avg_role_model_std_loss': 1.833931254097297, 'avg_role_model_mean_pred_loss': 6.538905610309342e-08, 'avg_role_model_g_mag_loss': 0.014750928152352571, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0003426387884246651, 'n_size': 80, 'n_batch': 10, 'duration': 10.850196361541748, 'duration_batch': 1.0850196361541748, 'duration_size': 0.13562745451927186, 'avg_pred_std': 0.08306950689293444}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005064211665740004, 'avg_role_model_std_loss': 0.2563789238775499, 'avg_role_model_mean_pred_loss': 2.7694440859517278e-08, 'avg_role_model_g_mag_loss': 0.017641309543978422, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000511730196467397, 'n_size': 320, 'n_batch': 40, 'duration': 56.38441038131714, 'duration_batch': 1.4096102595329285, 'duration_size': 0.17620128244161606, 'avg_pred_std': 0.08948770546121523}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0009603310900274664, 'avg_role_model_std_loss': 0.5317420201619558, 'avg_role_model_mean_pred_loss': 1.386134010417095e-06, 'avg_role_model_g_mag_loss': 0.03154977913945913, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009698854177258909, 'n_size': 80, 'n_batch': 10, 'duration': 10.881567239761353, 'duration_batch': 1.0881567239761352, 'duration_size': 0.1360195904970169, 'avg_pred_std': 0.09450749242678284}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.000545748674630886, 'avg_role_model_std_loss': 0.5983708105014387, 'avg_role_model_mean_pred_loss': 2.0696913546770085e-07, 'avg_role_model_g_mag_loss': 0.01691446538316086, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005514411785952688, 'n_size': 320, 'n_batch': 40, 'duration': 56.442954301834106, 'duration_batch': 1.4110738575458526, 'duration_size': 0.17638423219323157, 'avg_pred_std': 0.09217081667738966}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0004825813819479663, 'avg_role_model_std_loss': 1.5819854507564002, 'avg_role_model_mean_pred_loss': 3.7346816483818657e-07, 'avg_role_model_g_mag_loss': 0.01985574811697006, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00048788031563162804, 'n_size': 80, 'n_batch': 10, 'duration': 10.967880487442017, 'duration_batch': 1.0967880487442017, 'duration_size': 0.1370985060930252, 'avg_pred_std': 0.08295555993681773}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007052878027025145, 'avg_role_model_std_loss': 0.5709439587292039, 'avg_role_model_mean_pred_loss': 2.5470733571198996e-07, 'avg_role_model_g_mag_loss': 0.022239991917740554, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007124825457140105, 'n_size': 320, 'n_batch': 40, 'duration': 56.290388345718384, 'duration_batch': 1.4072597086429597, 'duration_size': 0.17590746358036996, 'avg_pred_std': 0.09195324669708498}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0005267480395559687, 'avg_role_model_std_loss': 0.013233936694268778, 'avg_role_model_mean_pred_loss': 2.673773188188733e-07, 'avg_role_model_g_mag_loss': 0.019443262554705142, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005322059372701915, 'n_size': 80, 'n_batch': 10, 'duration': 10.92713212966919, 'duration_batch': 1.092713212966919, 'duration_size': 0.13658915162086488, 'avg_pred_std': 0.09297868777066469}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00039893748798931484, 'avg_role_model_std_loss': 0.1465173953220642, 'avg_role_model_mean_pred_loss': 1.9658869693291864e-07, 'avg_role_model_g_mag_loss': 0.013279191794572398, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0004032720233226428, 'n_size': 320, 'n_batch': 40, 'duration': 56.342530965805054, 'duration_batch': 1.4085632741451264, 'duration_size': 0.1760704092681408, 'avg_pred_std': 0.09154584540519864}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00040967040695250034, 'avg_role_model_std_loss': 0.07373528057341901, 'avg_role_model_mean_pred_loss': 3.718508183148117e-07, 'avg_role_model_g_mag_loss': 0.014390437654219568, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0004137940326472744, 'n_size': 80, 'n_batch': 10, 'duration': 10.883796691894531, 'duration_batch': 1.0883796691894532, 'duration_size': 0.13604745864868165, 'avg_pred_std': 0.09073051768355071}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006527752275360399, 'avg_role_model_std_loss': 0.7538731722454941, 'avg_role_model_mean_pred_loss': 9.391601971928554e-07, 'avg_role_model_g_mag_loss': 0.01640899422345683, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000658580005983822, 'n_size': 320, 'n_batch': 40, 'duration': 56.13877892494202, 'duration_batch': 1.4034694731235504, 'duration_size': 0.1754336841404438, 'avg_pred_std': 0.09342938290210441}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0069938480504788455, 'avg_role_model_std_loss': 4.3801720477524215, 'avg_role_model_mean_pred_loss': 1.4892586449377632e-05, 'avg_role_model_g_mag_loss': 0.022057242598384617, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007098242765641772, 'n_size': 80, 'n_batch': 10, 'duration': 10.762710332870483, 'duration_batch': 1.0762710332870484, 'duration_size': 0.13453387916088105, 'avg_pred_std': 0.04032667824067175}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002608265062008286, 'avg_role_model_std_loss': 1.1152279273919476, 'avg_role_model_mean_pred_loss': 3.564367044772698e-06, 'avg_role_model_g_mag_loss': 0.03654546027537435, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026544345240836265, 'n_size': 320, 'n_batch': 40, 'duration': 56.032405614852905, 'duration_batch': 1.4008101403713227, 'duration_size': 0.17510126754641533, 'avg_pred_std': 0.08181555005721748}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001217631989857182, 'avg_role_model_std_loss': 0.20784810947207005, 'avg_role_model_mean_pred_loss': 8.713004294169657e-07, 'avg_role_model_g_mag_loss': 0.02198890279978514, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012277602480025962, 'n_size': 80, 'n_batch': 10, 'duration': 10.823968410491943, 'duration_batch': 1.0823968410491944, 'duration_size': 0.1352996051311493, 'avg_pred_std': 0.07128694653511047}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011638483822025592, 'avg_role_model_std_loss': 0.29395967978312, 'avg_role_model_mean_pred_loss': 1.489425534201283e-06, 'avg_role_model_g_mag_loss': 0.028835050016641616, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011756261381378863, 'n_size': 320, 'n_batch': 40, 'duration': 56.04453897476196, 'duration_batch': 1.401113474369049, 'duration_size': 0.17513918429613112, 'avg_pred_std': 0.09410313908010721}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0007999519177246839, 'avg_role_model_std_loss': 0.12229554721852764, 'avg_role_model_mean_pred_loss': 4.281340208578399e-07, 'avg_role_model_g_mag_loss': 0.02408305713906884, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008083746215561405, 'n_size': 80, 'n_batch': 10, 'duration': 10.778013944625854, 'duration_batch': 1.0778013944625855, 'duration_size': 0.1347251743078232, 'avg_pred_std': 0.07425977652892471}\n", + "Stopped False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test ▆▃▇▁▄█▂▁▁▁▁▂▁▂▁▁▁█▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▂▁▁▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▅▆█▅▃▁▄▅▅▅▅▅▄▅▄▅▅▁▄▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train █▃▂▆▁▄▄███▇█▇▆▇▇▇▇▅▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▆▅█▂▃▂▁▂▁▁▂▂▁▂▁▁▁▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train █▄▂▂▃▂▂▁▁▁▁▁▁▁▁▁▁▁▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test ▆▃▇▁▄█▂▁▁▁▁▂▁▂▁▁▁█▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▃▁▁▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test ▃▂█▁▂▁▁▁▁▁▁▂▁▁▁▁▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▁▁▁▁█▃▁▁▁▁▁▁▂▁▂▁▁▃▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▇█▇▂▆▄▃▁▁▁▁▁▁▁▁▁▁▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▃▃▅▅▄▄▃▂▂▂█▇▃▃▅▄▃▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▅▄▅▅▅▃▃▃▁▃██▅▄▄▃▃▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▃▃▅▅▄▄▃▂▂▂█▇▃▃▅▄▃▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▅▄▅▅▅▃▃▃▁▃██▅▄▄▃▃▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▃▃▅▅▄▄▃▂▂▂█▇▃▃▅▄▃▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▅▄▅▅▅▃▃▃▁▃██▅▄▄▃▃▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00081\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00118\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.07426\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.0941\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.02408\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.02884\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.0008\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00116\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 0.1223\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.29396\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 1.0778\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.40111\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.13473\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.17514\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 10.77801\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 56.04454\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 10\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/insurance/lct_gan/42/wandb/offline-run-20240229_054534-u0jglx9a\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_054534-u0jglx9a/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 399, 'n_batch': 50, 'role_model_metrics': {'avg_loss': 0.00043856672097095534, 'avg_g_mag_loss': 0.046217054526472144, 'avg_g_cos_loss': 0.07708879704015296, 'pred_duration': 0.9253897666931152, 'grad_duration': 2.459545850753784, 'total_duration': 3.3849356174468994, 'pred_std': 0.15322226285934448, 'std_loss': 0.00018639823247212917, 'mean_pred_loss': 5.971594418952009e-08, 'pred_rmse': 0.02094198390841484, 'pred_mae': 0.016042577102780342, 'pred_mape': 0.21174538135528564, 'grad_rmse': 0.027372082695364952, 'grad_mae': 0.019484277814626694, 'grad_mape': 0.6016528010368347}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.00043856672097095534, 'avg_g_mag_loss': 0.046217054526472144, 'avg_g_cos_loss': 0.07708879704015296, 'avg_pred_duration': 0.9253897666931152, 'avg_grad_duration': 2.459545850753784, 'avg_total_duration': 3.3849356174468994, 'avg_pred_std': 0.15322226285934448, 'avg_std_loss': 0.00018639823247212917, 'avg_mean_pred_loss': 5.971594418952009e-08}, 'min_metrics': {'avg_loss': 0.00043856672097095534, 'avg_g_mag_loss': 0.046217054526472144, 'avg_g_cos_loss': 0.07708879704015296, 'pred_duration': 0.9253897666931152, 'grad_duration': 2.459545850753784, 'total_duration': 3.3849356174468994, 'pred_std': 0.15322226285934448, 'std_loss': 0.00018639823247212917, 'mean_pred_loss': 5.971594418952009e-08, 'pred_rmse': 0.02094198390841484, 'pred_mae': 0.016042577102780342, 'pred_mape': 0.21174538135528564, 'grad_rmse': 0.027372082695364952, 'grad_mae': 0.019484277814626694, 'grad_mape': 0.6016528010368347}, 'model_metrics': {'lct_gan': {'avg_loss': 0.00043856672097095534, 'avg_g_mag_loss': 0.046217054526472144, 'avg_g_cos_loss': 0.07708879704015296, 'pred_duration': 0.9253897666931152, 'grad_duration': 2.459545850753784, 'total_duration': 3.3849356174468994, 'pred_std': 0.15322226285934448, 'std_loss': 0.00018639823247212917, 'mean_pred_loss': 5.971594418952009e-08, 'pred_rmse': 0.02094198390841484, 'pred_mae': 0.016042577102780342, 'pred_mape': 0.21174538135528564, 'grad_rmse': 0.027372082695364952, 'grad_mae': 0.019484277814626694, 'grad_mape': 0.6016528010368347}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:09:52.715501Z", + "iopub.status.busy": "2024-02-29T06:09:52.714572Z", + "iopub.status.idle": "2024-02-29T06:09:52.719077Z", + "shell.execute_reply": "2024-02-29T06:09:52.718205Z" + }, + "papermill": { + "duration": 0.027041, + "end_time": "2024-02-29T06:09:52.721168", + "exception": false, + "start_time": "2024-02-29T06:09:52.694127", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:09:52.758971Z", + "iopub.status.busy": "2024-02-29T06:09:52.758628Z", + "iopub.status.idle": "2024-02-29T06:09:53.045894Z", + "shell.execute_reply": "2024-02-29T06:09:53.045080Z" + }, + "papermill": { + "duration": 0.308814, + "end_time": "2024-02-29T06:09:53.048218", + "exception": false, + "start_time": "2024-02-29T06:09:52.739404", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:09:53.088564Z", + "iopub.status.busy": "2024-02-29T06:09:53.088217Z", + "iopub.status.idle": "2024-02-29T06:09:53.379190Z", + "shell.execute_reply": "2024-02-29T06:09:53.378254Z" + }, + "papermill": { + "duration": 0.313741, + "end_time": "2024-02-29T06:09:53.381291", + "exception": false, + "start_time": "2024-02-29T06:09:53.067550", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6G0lEQVR4nO3de1yUZf7/8dfMMMNJAQXlYAgeIAwVS4Ww0lopNNNwLc2vq+bX0vzqri5ZZqXY7v5i7bDZwc20LXM30uzg7pprGelWiucTppKaiqacNIfzaeb+/XHDyOigDKcZmc/z8bgfwD3Xfc91I/P2uq/7uq9boyiKghBCOAGtoysghBC1JJCEEE5DAkkI4TQkkIQQTkMCSQjhNCSQhBBOQwJJCOE03BxdgeZgNps5d+4c7du3R6PROLo6QogrKIpCUVERISEhaLX1t4PaRCCdO3eO0NBQR1dDCHEdZ86c4aabbqr39TYRSO3btwfUg/Xx8XFwbYQQVyosLCQ0NNTyWa1Pmwik2tM0Hx8fCSQhnNj1ulSkU1sI4TQkkIQQTkMCSQjhNNpEH5JoGpPJRFVVlaOrIW5ger0enU7X5P1IILkwRVHIycnh0qVLjq6KaAP8/PwICgpq0lhACSQXVhtGnTt3xsvLSwaVikZRFIXS0lLy8vIACA4ObvS+XCqQqk1m9p25hLG0il9FdUardd0PoMlksoSRv7+/o6sjbnCenp4A5OXl0blz50afvrlUp7ZJUXh4WQaPrdpNUUW1o6vjULV9Rl5eXg6uiWgrav+WmtIf6VKB5O6mw0OvHnJhmXTiwvUHqgnRUM3xt+RSgQTg52kA4FKpBJIQzqZRgbR06VLCw8Px8PAgLi6OnTt3XrP82rVriYqKwsPDgz59+rBhwwar1x999FE0Go3VMmzYsMZU7bp8PfUAGKWFJITTsTuQ1qxZQ3JyMikpKezdu5eYmBgSExMtPexX2rZtG+PHj2fq1Kns27ePpKQkkpKSOHTokFW5YcOGcf78ecvy0UcfNe6IrkMCSTSVRqNh3bp1jq5Gs1q0aBH9+vVzdDXsD6S//OUvPP7440yZMoVbbrmFZcuW4eXlxXvvvWez/Ouvv86wYcN46qmn6NWrF3/84x+57bbbeOutt6zKubu7ExQUZFk6dOjQuCO6Dp+aQLpUVtki+xeiNaxcuRI/P79m29/cuXNJT09vtv01ll2BVFlZyZ49e0hISLi8A62WhIQEMjIybG6TkZFhVR4gMTHxqvJbtmyhc+fO3HzzzcyYMYMLFy7UW4+KigoKCwutloby85IWknAdlZUN+4+3Xbt2TjH8w65AKigowGQyERgYaLU+MDCQnJwcm9vk5ORct/ywYcNYtWoV6enpLF68mP/+978MHz4ck8lkc5+pqan4+vpaFnsmZ5NTtvopikJpZXWrL/Y+PHnjxo3ceeed+Pn54e/vzwMPPMCJEycAGDRoEPPmzbMqn5+fj16v59tvvwXg/PnzjBgxAk9PT7p160ZaWhrh4eEsWbKkUb+3zMxMfvWrX+Hp6Ym/vz/Tpk2juLjY8vqWLVuIjY3F29sbPz8/7rjjDk6fPg3AgQMHuOeee2jfvj0+Pj7079+f3bt3X/P9tmzZwpQpUzAajZY+10WLFgEQHh7OH//4RyZNmoSPjw/Tpk0DYN68eURGRuLl5UX37t1ZsGCB1eX5K0/ZHn30UZKSknjllVcIDg7G39+fmTNntvgtRk4xMPKRRx6xfN+nTx/69u1Ljx492LJlC0OHDr2q/Pz580lOTrb8XDv5U0PUBpJc9r9aWZWJWxZ+2erve/gPiXgZGv6nWFJSQnJyMn379qW4uJiFCxcyevRo9u/fz4QJE3jppZf485//bLkMvWbNGkJCQrjrrrsAmDRpEgUFBWzZsgW9Xk9ycnK9faANqUtiYiLx8fHs2rWLvLw8HnvsMWbNmsXKlSuprq4mKSmJxx9/nI8++ojKykp27txpqduECRO49dZbefvtt9HpdOzfvx+9Xn/N9xw0aBBLlixh4cKFZGVlAWoLp9Yrr7zCwoULSUlJsaxr3749K1euJCQkhMzMTB5//HHat2/P008/Xe/7bN68meDgYDZv3szx48cZN24c/fr14/HHH2/U76oh7AqkgIAAdDodubm5Vutzc3MJCgqyuU1QUJBd5QG6d+9OQEAAx48ftxlI7u7uuLu721N1i9pAksv+N64xY8ZY/fzee+/RqVMnDh8+zNixY5kzZw7ff/+9JYDS0tIYP348Go2Go0eP8vXXX7Nr1y4GDBgAwLvvvktERESj6pKWlkZ5eTmrVq3C29sbgLfeeouRI0eyePFi9Ho9RqORBx54gB49egDQq1cvy/bZ2dk89dRTREVFATSoHgaDAV9fXzQajc3P0a9+9SuefPJJq3XPP/+85fvw8HDmzp3L6tWrrxlIHTp04K233kKn0xEVFcWIESNIT093nkAyGAz079+f9PR0kpKSAHWC/fT0dGbNmmVzm/j4eNLT05kzZ45l3aZNm4iPj6/3fc6ePcuFCxeadE9MfaQPqX6eeh2H/5DokPe1x7Fjx1i4cCE7duygoKAAs9kMqB/u3r17c9999/Hhhx9y1113cfLkSTIyMnjnnXcAyMrKws3Njdtuu82yv549ezb6IsqRI0eIiYmxhBHAHXfcgdlsJisri8GDB/Poo4+SmJjIvffeS0JCAmPHjrX8bScnJ/PYY4/x97//nYSEBB5++GFLcDVWbdDWtWbNGt544w1OnDhBcXEx1dXV151dNTo62uoWkODgYDIzM5tUt+ux+ypbcnIyK1as4IMPPuDIkSPMmDGDkpISpkyZAqjN4fnz51vKz549m40bN/Lqq69y9OhRFi1axO7duy0BVlxczFNPPcX27ds5deoU6enpPPjgg/Ts2ZPExOb/cPhIH1K9NBoNXga3Vl/sHeE7cuRILl68yIoVK9ixYwc7duwALnfgTpgwgU8++YSqqirS0tLo06cPffr0afbfV0O9//77ZGRkMGjQINasWUNkZCTbt28H1L6bH374gREjRvDNN99wyy238Pnnnzfp/eqGI6gXliZMmMD999/P+vXr2bdvH88999x1O7yvPHXUaDSW8G8pdgfSuHHjLOeo/fr1Y//+/WzcuNHScZ2dnc358+ct5QcNGkRaWhrLly8nJiaGTz75hHXr1tG7d28AdDodBw8eZNSoUURGRjJ16lT69+/Pd9991+jTsmuRTu0b24ULF8jKyuL5559n6NCh9OrVi19++cWqzIMPPkh5eTkbN24kLS2NCRMmWF67+eabqa6uZt++fZZ1x48fv2ofDdWrVy8OHDhASUmJZd3WrVvRarXcfPPNlnW33nor8+fPZ9u2bfTu3Zu0tDTLa5GRkfz+97/nq6++4te//jXvv//+dd/XYDDUe9HnStu2bSMsLIznnnuOAQMGEBERYelUdzaN6tSeNWtWvadoW7ZsuWrdww8/zMMPP2yzvKenJ19+2XodqZZAkj6kG1KHDh3w9/dn+fLlBAcHk52dzTPPPGNVxtvbm6SkJBYsWMCRI0cYP3685bWoqCgSEhKYNm0ab7/9Nnq9nieffBJPT89G3Ys1YcIEUlJSmDx5MosWLSI/P5/f/va3TJw4kcDAQE6ePMny5csZNWoUISEhZGVlcezYMSZNmkRZWRlPPfUUDz30EN26dePs2bPs2rXrqj4yW8LDwykuLiY9PZ2YmBi8vLzqvVE6IiKC7OxsVq9ezcCBA/niiy+a3AprKS54L5saSEUV1ZjM9l1uFo6n1WpZvXo1e/bsoXfv3vz+97/n5ZdfvqrchAkTOHDgAHfddRddu3a1em3VqlUEBgYyePBgRo8ebbni5OHhYXd9vLy8+PLLL7l48SIDBw7koYceYujQoZaBv15eXhw9epQxY8YQGRnJtGnTmDlzJtOnT0en03HhwgUmTZpEZGQkY8eOZfjw4bzwwgvXfd9BgwbxxBNPMG7cODp16sRLL71Ub9lRo0bx+9//nlmzZtGvXz+2bdvGggUL7D7W1qBR7B0E4oQKCwvx9fXFaDRet6OuymQm4rn/ALBvwb108Da0RhWdTnl5OSdPnqRbt26N+iC2JWfPniU0NJSvv/7a5lVd0TDX+ptq6GfUKcYhtSa9Tou3QUdJpQljWZXLBpIr++abbyguLqZPnz6cP3+ep59+mvDwcAYPHuzoqrk8lztlgzpjkaRj2yVVVVXx7LPPEh0dzejRo+nUqZNlkOSHH35Iu3btbC7R0dGtVsfhw4fXW48XX3yx1erR2lyuhQTg62XgnLFcrrS5qMTExHqHlIwaNYq4uDibr11vBHVzevfddykrK7P5WseOHVutHq3NNQPJUz1sCSRxpfbt21/3+fOtoUuXLo6ugkO49CmbBJIQzsW1A6lU5kQSwpm4ZCD5ealX1qSFJIRzcclAklM2IZyTSwaS3GArhHNyyUCSOZFEU7TFSf6dhUsGkp+0kMQNrrkn+Qf1xniNRsOlS5eadb/2cMlAkmlshXBOLh1I0kK6gqJAZUnrLzLJf7NN8l9RUcHcuXPp0qUL3t7exMXFWU0JdPr0aUaOHEmHDh3w9vYmOjqaDRs2cOrUKe655x5AneJFo9Hw6KOPNur30RQuOlJbDaSSShNVJjN6nUvm8tWqSuHFkNZ/32fPgcH7+uVqyCT/9U/yP2vWLA4fPszq1asJCQnh888/Z9iwYWRmZhIREcHMmTOprKzk22+/xdvbm8OHD9OuXTtCQ0P59NNPGTNmDFlZWfj4+ODp6dmo30lTuGQg1V5lA7WVFNCu+WemFC1HJvm3Pcl/dnY277//PtnZ2YSEqP+xzJ07l40bN/L+++/z4osvkp2dzZgxYyxT+nbv3t2yfe09cp07d272/qmGcslA0mk1tPdwo6i8WgKpLr2X2lpxxPvaQSb5ty0zMxOTyURkZKTV+oqKCstDIH/3u98xY8YMvvrqKxISEhgzZgx9+/Zt1Pu1BJc9V5F+JBs0GvXUqbUXmeS/WSb5Ly4uRqfTsWfPHvbv329Zjhw5wuuvvw7AY489xk8//cTEiRPJzMxkwIABvPnmm812rE0lgSRjkW4oMsm/ytYk/7feeismk4m8vDx69uxptdQ9tQsNDeWJJ57gs88+48knn2TFihWWfQINfnhAS3DZQJLns92Y6k7yf/z4cb755hurpxhDwyf537lzJ/v27WPatGlNmuTfw8ODyZMnc+jQITZv3nzVJP/z588nIyOD06dP89VXX3Hs2DF69epFWVkZs2bNYsuWLZw+fZqtW7eya9cuqz6m+tSd5L+goIDS0lIiIyOZMGECkyZN4rPPPuPkyZPs3LmT1NRUvvjiCwDmzJnDl19+ycmTJ9m7dy+bN2+2vF9YWBgajYb169eTn59vdaWw1ShtgNFoVADFaDQ2eJsZ/9ithM1br6zcerLlKubEysrKlMOHDytlZWWOrordNm3apPTq1Utxd3dX+vbtq2zZskUBlM8//9xSZsOGDQqgDB48+Krtz507pwwfPlxxd3dXwsLClLS0NKVz587KsmXLGvT+V77XwYMHlXvuuUfx8PBQOnbsqDz++ONKUVGRoiiKkpOToyQlJSnBwcGKwWBQwsLClIULFyomk0mpqKhQHnnkESU0NFQxGAxKSEiIMmvWrAb/mzzxxBOKv7+/AigpKSmKoihKZWWlsnDhQiU8PFzR6/VKcHCwMnr0aOXgwYOKoijKrFmzlB49eiju7u5Kp06dlIkTJyoFBQWWff7hD39QgoKCFI1Go0yePLlB9ah1rb+phn5GXW6S/1rzPzvIRzvPkHxvJL8b2rgrLDcymeT/Mpnkv3nIJP9N4CP3s7ksmeTfebluH5KnzInkqmSSf+flsi0kuezvumSSf+fl8oEkN9iKumSSf8dy2VO2y89mc+15tdvANQ3hJJrjb8llA8nVxyHVnn6UlpY6uCairaj9W2rKqa3Ln7K5aiDpdDr8/Pwsd7l7eXk1amCgEIqiUFpaSl5eHn5+fuh0ukbvq1GBtHTpUl5++WVycnKIiYnhzTffJDY2tt7ya9euZcGCBZw6dYqIiAgWL17M/fffb7PsE088wTvvvMNrr73GnDlzGlO9Bqm97F9eZaa8yoSHvvG/xBtV7e0EjZ16Q4i6/Pz8rG5RaQy7A2nNmjUkJyezbNky4uLiWLJkCYmJiWRlZdG5c+erym/bto3x48eTmprKAw88QFpaGklJSezdu5fevXtblf3888/Zvn27ZeqEltTe3Q2tBsyK2rHtioGk0WgIDg6mc+fOVFW5ZktRNA+9Xt+klpGFXWPDFUWJjY1VZs6cafnZZDIpISEhSmpqqs3yY8eOVUaMGGG1Li4uTpk+fbrVurNnzypdunRRDh06pISFhSmvvfZag+vUmFtHFEVRYl74Ugmbt175MafQru2EEPZp6GfUrk7tyspK9uzZQ0JCgmWdVqslISGBjIwMm9tkZGRYlQd1HEjd8mazmYkTJ/LUU081aPBZRUUFhYWFVktjuHo/khDOxq5AKigowGQyERgYaLU+MDCQnJwcm9vk5ORct/zixYtxc3Pjd7/7XYPqkZqaiq+vr2UJDQ215zAs5HFIQjgXh1/237NnD6+//jorV65s8FWe+fPnYzQaLcuZM2ca9d7SQhLCudgVSAEBAeh0OnJzc63W5+bm1tu7HhQUdM3y3333HXl5eXTt2hU3Nzfc3Nw4ffo0Tz75JOHh4Tb36e7ujo+Pj9XSGBJIQjgXuwLJYDDQv39/0tPTLevMZjPp6enEx8fb3CY+Pt6qPMCmTZss5SdOnMjBgwetptwMCQnhqaee4ssvv7T3eOwigSSEc7H7sn9ycjKTJ09mwIABxMbGsmTJEkpKSpgyZQqgPmKmS5cupKamAjB79myGDBnCq6++yogRI1i9ejW7d+9m+fLlAPj7+1smIK+l1+sJCgqymgK0JUggCeFc7A6kcePGkZ+fz8KFC8nJyaFfv35s3LjR0nGdnZ2NVnu54TVo0CDS0tJ4/vnnefbZZ4mIiGDdunVXjUFyBFe/fUQIZ+OyM0YCrNmVzbxPM/lVVGfee3RgC9ZQCNfW0M+ow6+yOZKcsgnhXFw6kC5PY+vaU5AI4SxcOpAuT2Nb7eCaCCHAxQPJ1+vyrJFtoCtNiBueawdSzSlbpclMeZXZwbURQrh0IHkbdOi06u0qrj6VrRDOwKUDSaPR4CdX2oRwGi4dSFDn0r/c8S+Ew7l8IPlIC0kIp+HygXT5cUgSSEI4mssHkp+XPDBSCGfh8oEkt48I4TwkkCSQhHAaEkgyr7YQTkMCSVpIQjgNCSQJJCGchgSSp1xlE8JZSCB5yTgkIZyFywfS5TmRZAoSIRzN5QOp9pTNZFYoqTQ5uDZCuDaXDyQPvRaDTv01SMe2EI7l8oGk0Whkbm0hnITLBxLI89mEcBYSSMilfyGchQQScvuIEM5CAgkZrS2Es5BAQgJJCGchgYQEkhDOQgIJmcZWCGchgYRcZRPCWTQqkJYuXUp4eDgeHh7ExcWxc+fOa5Zfu3YtUVFReHh40KdPHzZs2GD1+qJFi4iKisLb25sOHTqQkJDAjh07GlO1RpFxSEI4B7sDac2aNSQnJ5OSksLevXuJiYkhMTGRvLw8m+W3bdvG+PHjmTp1Kvv27SMpKYmkpCQOHTpkKRMZGclbb71FZmYm33//PeHh4dx3333k5+c3/sjsIH1IQjgJxU6xsbHKzJkzLT+bTCYlJCRESU1NtVl+7NixyogRI6zWxcXFKdOnT6/3PYxGowIoX3/9dYPqVFveaDQ2qPyVfswpVMLmrVf6LvqyUdsLIa6toZ9Ru1pIlZWV7Nmzh4SEBMs6rVZLQkICGRkZNrfJyMiwKg+QmJhYb/nKykqWL1+Or68vMTExNstUVFRQWFhotTSFpQ+pvAqzWaYgEcJR7AqkgoICTCYTgYGBVusDAwPJycmxuU1OTk6Dyq9fv5527drh4eHBa6+9xqZNmwgICLC5z9TUVHx9fS1LaGioPYdxldqbaxUFiiqqm7QvIUTjOc1VtnvuuYf9+/ezbds2hg0bxtixY+vtl5o/fz5Go9GynDlzpknv7aHX4aFXfxVypU0Ix7ErkAICAtDpdOTm5lqtz83NJSgoyOY2QUFBDSrv7e1Nz549uf322/nb3/6Gm5sbf/vb32zu093dHR8fH6ulqeR+NiEcz65AMhgM9O/fn/T0dMs6s9lMeno68fHxNreJj4+3Kg+wadOmesvX3W9FRYU91WsSudImhOO52btBcnIykydPZsCAAcTGxrJkyRJKSkqYMmUKAJMmTaJLly6kpqYCMHv2bIYMGcKrr77KiBEjWL16Nbt372b58uUAlJSU8P/+3/9j1KhRBAcHU1BQwNKlS/n55595+OGHm/FQr63u3NpCCMewO5DGjRtHfn4+CxcuJCcnh379+rFx40ZLx3V2djZa7eWG16BBg0hLS+P555/n2WefJSIignXr1tG7d28AdDodR48e5YMPPqCgoAB/f38GDhzId999R3R0dDMd5vX5SAtJCIfTKMqN/6iNwsJCfH19MRqNje5PevLjA3y69yxPD7uZ/7u7ZzPXUAjX1tDPqNNcZXM06UMSwvEkkGrU3s8ml/2FcBwJpBrSQhLC8SSQasg4JCEcTwKphq9MQSKEw0kg1ZBTNiEcTwKphgSSEI4ngVSjNpCKyqsxyRQkQjiEBFKN2kACufQvhKNIINXQ67R4G3SAnLYJ4SgSSHVIP5IQjiWBVIePPJ9NCIeSQKpDHockhGNJINUhp2xCOJYEUh3yBFshHEsCqY7L97NVOrgmQrgmCaQ6/LxkGlshHEkCqQ6ZxlYIx5JAqkM6tYVwLAmkOmROJCEcSwKpDj+5yiaEQ0kg1SGnbEI4lgRSHbWBVFJpospkdnBthHA9Ekh1+NSZgkRaSUK0PgmkOnRaDe091If5SiAJ0fokkK4g/UhCOI4E0hUsgSSX/oVodRJIV5AWkhCOI4F0BZkTSQjHkUC6grSQhHCcRgXS0qVLCQ8Px8PDg7i4OHbu3HnN8mvXriUqKgoPDw/69OnDhg0bLK9VVVUxb948+vTpg7e3NyEhIUyaNIlz5841pmpN5iO3jwjhMHYH0po1a0hOTiYlJYW9e/cSExNDYmIieXl5Nstv27aN8ePHM3XqVPbt20dSUhJJSUkcOnQIgNLSUvbu3cuCBQvYu3cvn332GVlZWYwaNappR9ZI0kISwoEUO8XGxiozZ860/GwymZSQkBAlNTXVZvmxY8cqI0aMsFoXFxenTJ8+vd732LlzpwIop0+fblCdjEajAihGo7FB5a/lw+2nlbB565WpK3c1eV9CCFVDP6N2tZAqKyvZs2cPCQkJlnVarZaEhAQyMjJsbpORkWFVHiAxMbHe8gBGoxGNRoOfn5/N1ysqKigsLLRamotMYyuE49gVSAUFBZhMJgIDA63WBwYGkpOTY3ObnJwcu8qXl5czb948xo8fj4+Pj80yqamp+Pr6WpbQ0FB7DuOaLFOQlMk0tkK0Nqe6ylZVVcXYsWNRFIW333673nLz58/HaDRaljNnzjRbHaQPSQjHcbOncEBAADqdjtzcXKv1ubm5BAUF2dwmKCioQeVrw+j06dN888039baOANzd3XF3d7en6g0m45CEcBy7WkgGg4H+/fuTnp5uWWc2m0lPTyc+Pt7mNvHx8VblATZt2mRVvjaMjh07xtdff42/v7891WpWtZf9y6vMlFeZHFYPIVyRXS0kgOTkZCZPnsyAAQOIjY1lyZIllJSUMGXKFAAmTZpEly5dSE1NBWD27NkMGTKEV199lREjRrB69Wp2797N8uXLATWMHnroIfbu3cv69esxmUyW/qWOHTtiMBia61gbpL27GxoNKIrase2h17Xq+wvhyuwOpHHjxpGfn8/ChQvJycmhX79+bNy40dJxnZ2djVZ7ueE1aNAg0tLSeP7553n22WeJiIhg3bp19O7dG4Cff/6Zf/3rXwD069fP6r02b97M3Xff3chDaxytVoOPhx5jWRXGsio6+3i06vsL4co0iqIojq5EUxUWFuLr64vRaLxm31NDDXl5M6cvlPLJE/EMCO/YDDUUwrU19DPqVFfZnIVcaRPCMSSQbJDHIQnhGBJINsgTbIVwDAkkG/wkkIRwCAkkG6QPSQjHkECyQQJJCMeQQLJBAkkIx5BAskHuZxPCMSSQbJCrbEI4hgSSDTIOSQjHkECyoe6skW3gzhohbhgSSDb4eakzDFSazJRXmR1cGyFchwSSDd4GHTqtBpB+JCFakwSSDRqNRubWFsIBJJDqYRmLJB3bQrQaCaR6yOBIIVqfBFI9JJCEaH0SSPWQQBKi9Ukg1UNuHxGi9Ukg1UNaSEK0PgmkesjtI0K0PgmkesgNtkK0Pgmkesg0tkK0PgmketS9wVYI0TokkOrh61V764gEkhCtRQKpHnWvsskUJEK0Dgmkevh5qlOQmMwKJZUmB9dGCNcggVQPD70Wg0799UjHthCtQwKpHhqNxnLpv6CowsG1EcI1SCBdQ+8uPgBsycp3cE2EcA2NCqSlS5cSHh6Oh4cHcXFx7Ny585rl165dS1RUFB4eHvTp04cNGzZYvf7ZZ59x33334e/vj0ajYf/+/Y2pVrMb2TcEgH8d+Fk6toVoBXYH0po1a0hOTiYlJYW9e/cSExNDYmIieXl5Nstv27aN8ePHM3XqVPbt20dSUhJJSUkcOnTIUqakpIQ777yTxYsXN/5IWsB90YEY3LScyC/hyPkiR1dHiLZPsVNsbKwyc+ZMy88mk0kJCQlRUlNTbZYfO3asMmLECKt1cXFxyvTp068qe/LkSQVQ9u3bZ1edjEajAihGo9Gu7Rpi+qrdSti89UrqhiPNvm8hXEVDP6N2tZAqKyvZs2cPCQkJlnVarZaEhAQyMjJsbpORkWFVHiAxMbHe8g1RUVFBYWGh1dJSRsaop23/PnBOTtuEaGF2BVJBQQEmk4nAwECr9YGBgeTk5NjcJicnx67yDZGamoqvr69lCQ0NbfS+rudXUZ3xNuj4+VIZe7Mvtdj7CCFu0Kts8+fPx2g0WpYzZ8602Ht5GnTce4saqP8+cK7F3kcIYWcgBQQEoNPpyM3NtVqfm5tLUFCQzW2CgoLsKt8Q7u7u+Pj4WC0taVQ/9bTti8zzmMxy2iZES7ErkAwGA/379yc9Pd2yzmw2k56eTnx8vM1t4uPjrcoDbNq0qd7yzujOnp3w9dSTX1TBjp8uOLo6QrRZdp+yJScns2LFCj744AOOHDnCjBkzKCkpYcqUKQBMmjSJ+fPnW8rPnj2bjRs38uqrr3L06FEWLVrE7t27mTVrlqXMxYsX2b9/P4cPHwYgKyuL/fv3N6mfqTkZ3LQM76226P59UE7bhGgxjbmE9+abbypdu3ZVDAaDEhsbq2zfvt3y2pAhQ5TJkydblf/444+VyMhIxWAwKNHR0coXX3xh9fr777+vAFctKSkpDapPS172r7X1WL4SNm+90nfRl0pFlanF3keItqihn1GNotz417ILCwvx9fXFaDS2WH+Syaxwe2o6+UUVvPfoAH4VFXj9jYQQQMM/ozfkVTZH0Gk1jOgTDMC/D5x3cG2EaJskkOxQO0jyqx9yKJM5koRodhJIdritqx9d/DwpqTSxOcv2vXtCiMaTQLKDRqOxtJL+tV+utgnR3CSQ7DQyRu1H+iYrj6JymUlS3AD+9Vv46yAoKXB0Ta5LAslOtwT70KOTN5XVZr76Iff6G7Smiz9B2jg48m9H10Q4i/wfYe8qyPsBdixzdG2uSwLJTnVP26wGSZYXws97HVQroLIEVk+AHzfCv2erPwux5/3L3+9cARXFjqtLA0ggNUJtIH1/rICLJZVgqoJVo2DFPY5pnSgK/HsO5Kkj3Sm9oP7xCddWVQb7P1S/13tD+SXY93eHVul6JJAaoUendkSH+FBtVvjPofPw/RI4t099cctiNSCaycmCEuZ9cpCtx69x/r/rXcj8GDQ66K/ewsO2N5z+f0PRwn74HMqN4NcV7vuDui5jqfofqJOSQGqkUTWtpP27t8J/a6be1eggNxOOfdUs71FUXsXUlbtYs/sME97dwYsbjlBZbbYudGYnbKy5d/DeF+D+V6Bjd7WVtEtaSS5td83pWv9Hod9vwLsTGM/Aoc8cWq1rkUBqpBF9g3Gjmkm5i8FcBTffD/H/p7747StNbiUpisL8zzL5qaAEb4MOgOXf/sTov27lRH5Ny6c4Hz6erL7/LQ9C/CzQucHgp9XXt74BFTIXuEvKyYSzO0HrpoaR3gPinlBf2/p6s7bim5MEUiPd1MGLP/h/TR/tKSrcfOCB12oCwV39Qzj1XZP2/4/tp1l/8DxuWg2rpsbxzsT++Hnp+eFcIQ+88T2rt/+E8skUKDoHAZHw4FLQaNSN+zwMHXtA2UXpS3JVta2jqAegfc19lwOnqn1JeT/A8fT6t3Ug1wukX07DxZNN30/uD4wrTQNgqec0aB+kLrdNVF//7tVG7/rg2Uv8cf0RAJ4ZHkX/sA4kRgfx5ZzB3NHTn7IqE7+sX4jm1Hcoei8Y9w9wb395Bzo3GFLTStomrSSXU1EEB9eo3w/438vrPTuop28AW5e0dq0axLUCqSgHVj0I7w+HvKON34+pCtbNQKdUs8nUnzfybyX7Qqn62h2z1WbyT1vg7B67d20sreL/PtxLpclMYnQgU+/sZnkt0MeDv/9vHO8MOM8MN/Vq3vPmGWwr9L96R70fAv+eUPYL7FzemKMUN6rMT6CyWP337zbY+rX4/1P/Pk99Bz/b//fZ0lwrkAD0nlB0Xg2lxo4b+n4JnD8AHn7886a5gObymCS/rtB3nPr9d6/YtVtFUXhy7QHO/lJG145evPRQDJra07Aa2l9+IvHYCwB8oh/JhyX9mfDuDv78n6PWHd46NxgyT/1+25vqOCnR9ikK7H5P/b7/lMun8bV8b1JP6UHtS3IyrhVI7YPg0S+gS3+1f+WDUXDqe/v2kfvD5atqw1/irtt6A1c8AODO3wMayNqglm+gFd/9xNdHcjHotPx1wm34euqtC1SWwJrfQEUhdI3n/uQVPDIwFEWBZf89wUPLtnGyoM6AyN5jwD9CWkmu5NxeyDmo9mX2+x/bZQb9Vv16+F9w4UTr1a0BXCuQALw6wqR/qk3ZyiL4xxj48cuGbVtzqma5qtZ3LMOig9HrNBzNKeLH3Jq+moAIiE5Sv29gX9LuUxdZvDELgIUjb6F3F1/rAoqijsDOOwztAuHhlXh5evLnMX15uya8Dp41MuKN73j6kwP87fuTbP3pF4puT1a3l1aSa6htHUUnqX/rtgRGQ8R9gAIZb7VWzRrE9QIJ1A7g/1mrhkp1Oaz+H/W8+3rqnKrxwGug0eDrpWdIZCfgylZSTRD88Pl1/xe6UFzBrLR9mMwKD/YLYUJc16sL7VwBmWvVsU4Pr1RbezWG9wlm45y7uL17R0orTXy8+yx/XH+YCe/uIObT9pykC5Rf4ptVf2DNrmz2n7lEaWX19Y9X3FjKLkHmp+r3dTuzbbljtvp134dQ7DxT6bhmIIE6LmPsKugzFszV8Oljly+V2lL3VO3+l60CwebTbYP7QkQiKGb4/i/17tZkVpizZj85heX06OTNi6P7XNVvxJmd8OWz6vf3/gHCBl21n2BfTz587HZWTBrA74ZGkBgdSLi/F4pGy2uVowG47ec0/vTpDpKWbiU65UsGv7SZv33fDFcchXM4uAaqy6DzLRAad+2yYXeoXRemCqc6nXfdQALQ6WH0OzBgKqDA+jm2O/quPFWr7RSskdArEA+9llMXSpm9ej9bsvKoNplh8Fy1wIHVcMn2wyyXbj7Od8cK8NBrefs3/fF2d7MuUJx3xeDHmfUfjlbDvbcEknxvJO9MHMCWp+7hhxcSmTo9GWO77vhpSkjp/C0B7dxRFMi+WMof1x9m+bfO1Y8gGqFuZ/aA/726M/tKGs3lVpIT3XTr2oEEoNXCiFcvn2JtWgjpf7AeyWrjVK0ub3c3Jg8KB+BfB87x6Pu7uD01nUX7vCkKjldbYNvevOqttx4v4LWvfwTgT0l9iAxsb13AeBZWjrA9+LGBvAxuxIT545v4HAAPVf6T3XMHsuf5BOYkRADw4oajpO3Itmu/wslkb4f8o6D3gr5jG7ZN1APqANryS+oUJU5AAgnUD3lCCiQsUn/+7lXYMBfM5mueqtX1zLAoPp0Rz6T4MDp6GygormTltlNMP30PANW7V3Lq9OXTo9zCcmav3oeiwLgBoTzU/ybrHRYch/eGQcGP4NMFxq+2Hvxor+jR0ClKvdly+zL827kzJyGSGXf3AOC5dZn8c//Pjd+/cKza1lHvMeDhe+2ytbS6y1fcnOSmW3kM0pV2vwfrkwFFHU+Uf1RtHd18PzyS1qAWSpXJzPfHCli3/2e++iGHjzTP0U97grerR7Ih8Ake7BfCV4dz2XnyIlFB7Vk38w489LrLOzh/EP7xayjJVwe3TVwHfqFNOy6AQ5/CJ/+r/sHOPgiefiiKwoJ/HuIf27Nx02p4Z2J/hvaSRzzdUEouwF+iwFQJj2+GLrc1fNuqcljSG0ryUUa/wz9KbyfjpwtMuaMbA8PruUrXCA39jEog2ZL5CXw+XT3VAvVUbeaOeltH11JSUc2hbz4ibscsihRP7qh4nULaAdDO3Y1///ZOugV4X97gdIY662OFEYL6wG8+h3admn5MoLb43h4E+Ufg7vlw9zM1qxWSP97Puv3ncHfTsnJKLPE9bIz+Fs5p6xuwaQEE94Pp/7V/+29fgW/+yBl9N+4q+hOg/qd77y2BzBsWRc/O7ZpcRXkuW1P0eQjGfQhuHurP1zhVux5vdzfiEidA52jaa8pYFb2P/mEd8DLoeOXhGOswOrYJ/j5aDaOu8eogzuYKI1D7y+6uGb2d8Vf1MjGg1Wp4+eEY7r0lkIpqM499sIv9Zy413/uKlmM2X54V8nqX+uuxu9OvKcWD0KqTDHXLJDE6EJ1Ww6bDuSQu+Zb5n2WSV1jejJWun7SQriX/RzBmQ4+hdncmXyXzE/h0qnqD45xD4H7F/zqHPoXPpqmtsp73qkMSDF5Ne09bzGZYdoc6wHLIM3DPfMtL5VUmpn6wi63HL+Drqefj6fHcHNSEfivR8k5shr8ngaE9PHn06r+razCZFd765jivp//Is7q/85jbfygJjsd7+kaO5xXz0sajfHVYnTfeU6/j8cHdmTa4O+2uvBLcANJCag6dIqFnQtPDCNRO5Y491Ns49lwx3mn3+/DJVDWMon+t9lW1RBiB2kqqvcdt+1/V+tTw0OtYPnEA/UL9MJZV8Zu/7eD0BZmb26nV/i3FjLMrjHILy/nNuzt47esfMStwvtf/omjd8D6fAWf30LNzO5ZPGsDaJ+K5tasfZVUm3kg/xt0vb+bvGaeoMpmv/yaNIIHUWrS6mnvcUIcAVNU0gb9/TR3/hKI2uce8C26Glq1Lr1HQOVq9J+7fs+HEN5ZxKN7ubqycMpCooPbkF1Uw4d0dnDeWtWx9ROMU5cDRL9Tva6cuboDNWXkMf/07Mn66gJdBx1/GxrBgwn1oej+kFth2eSzewPCOfDZjEG9PuI1uAd4UFFey4J8/cN9r3/KfzPM09wmWnLK1pupKeONWKDyrTjVrPHt5Xpo7k2HowuZpjTXEkX+rN+rW0rqpnaJhgyDsDvI73srDKw9z6kIpPTp58/H0ePzbubdO3ZqiugIuHFevjpYUQGgsBMWoLcO25tuX4Zs/qaOyp15/2uTKajOvfJXF8m9/AtRHer31P7fSvVNNyyr3B/WiBxrofrc69i0gomaJpMorkNW7zrDk62NcKKkE4Naufjx3fy8GXOeKnFxlc1Y73oH/PA1avTr6GtTbQWpHzbamo1+owXR6K1y6cmCkhspO0ay7GEZ6WQSFnQfyzhOJ+Hjobe6q1VUUQ0GW2s+Xf1Qdr5V/FH45pd6uU5d3J7VfLuJe6HGP2o9nD0VRn3l3Zoc6APH8AXWfgdGXF/+Ilm/Z1mU2wesx6hzZo9+BmEeuWTz7Qim/Xb2PAzUXKybHhzH//l7Ww01A/U+qvifnGNqBf0+qOvZkZ6E/a095cqQ6iOFD7mDOsL7XfP8WDaSlS5fy8ssvk5OTQ0xMDG+++SaxsbH1ll+7di0LFizg1KlTREREsHjxYu6//37L64qikJKSwooVK7h06RJ33HEHb7/9NhEREQ2qzw0VSFVlsKSPOsYIDYxccnkWP0e6dAZOb4PT36tfLxy/qkg+fhRqO1Ci70CFuz/VngFovDuh8+mMh28QXh0D8Q3ogl9ACG7uno2rh9kEpRfV34+txfgz5Geprcz6ePiiBNyMyeCD7mwGmjrPqFM0Oqq7DKSqewKV3YZSHXALSs1lbq1Gvf1Gq1Sjz8vE7ecd6M7uRHNmB5qSa9+Aqmj1KP4RVHe6heqAXlQF3EJFxygqvQIprTJTWFZFUXk1heXWX4vKqyguq6SkvILS8nL0encC/doT7OdBiK+n+tXPkxBfTzwNdcLjxy8hbaw6JOXJo+o8X6gXJvKLKsgrqiC/qIL84grOXyrj7xmnKaqoxtdTz0sP9SUxup6rxqYqOLsbLhxTQ77gmLr8cgoUk81NKgYl435fyjV/Py0WSGvWrGHSpEksW7aMuLg4lixZwtq1a8nKyqJz585Xld+2bRuDBw8mNTWVBx54gLS0NBYvXszevXvp3VudS2jx4sWkpqbywQcf0K1bNxYsWEBmZiaHDx/Gw8PjunW6oQIJYP9HsOVFtWUUPdrRtbGtKKcmoLZSfvw7PH7JsmvzcvRUo8ekcaNaY8CsdcOs1aNoDSg6A+gMaHR6NG4G3DBjqLiAoeIihopf0NCwDtNitw7kGMI54xbKKc1N/GjqwuHqIE6Wt6OowoSigJ5qBmizuEe7n7u1+4nUWo9GP690ZIsphl3mm+mmzWGgNosYzQk8NZVW5SoUNzKV7uxTIsmkJx0oJIJsbtZkc7PmDD4a2/1slxRvivFEhxk3qnHDjA5Tna8mdBrrj2CloqMMd0rxoFRxp7Tm+yqtB+i90Bja0dN0jMDyk6T7PcQ7no9RUBNARRX1z+LQP6wDb4y/lS5+jfjPoroSfjlpHVK139//stqpfg0tFkhxcXEMHDiQt95S51Exm82Ehoby29/+lmeeeeaq8uPGjaOkpIT169db1t1+++3069ePZcuWoSgKISEhPPnkk8ydq96MajQaCQwMZOXKlTzyyLWbovYcrGi8MmMBOdlZlFzMofxSDtVF+VCch66sAEPFBbyqfsHH9AsdFCMGje3/Se1xUWnHBcWXC/hQoPhSoPhwQfEhjw6cMAdzXOnCJewfknCTJp+7tfsZoj3AHdof8NJU2Cz3i9KO3eZI9pgj2W2OJFPpTgX1nZIpdKGAKK0aTr1qvnbXnMdN0zJXo2oNrXiZE0oXq3UGNy2d27vTqb07ndqpX6OCfXhkYCh6XTP3pSmKeoqs1V2zWEM/o3YNKKisrGTPnj3Mn3957IpWqyUhIYGMjAyb22RkZJCcnGy1LjExkXXr1gFw8uRJcnJySEhIsLzu6+tLXFwcGRkZNgOpoqKCiorLf0iFhTLxWEvz9A2gW5+A65YzmcwUXMynyPgLpeVllJaWUl5eTll5GZXlZZRXlFNVUU5lRTlVlRVUV5ZTYYJCnXo6eEnnR5HWF0Wjs/Tva9Cg0aj9/VqNhkBPPZGeenw89Ph4utV81ePj4VbzVV3fzt0NraZm29p9ABrNZPWrqVxtBR7bBD/vQfHvgTn0dkw3xeLRIYJ4NMSaFcxmBZNy+avJrKDTatBpNOi0Gty0WnS6uj9r0Go16pXUC8fVjnatTp1dQutWs+jqfK+v+VmnnjJVlUJlKVSVqF8rSygtLcR4yUhRkZGSYiPlxUWc94rgf8LuvRw+NUt7d7erp7BpKRqNOkdXM7ErkAoKCjCZTAQGWt/rFBgYyNGjtifNz8nJsVk+JyfH8nrtuvrKXCk1NZUXXnjBnqqLVqLTaQnoFEhApxvgfjitJ/Qcqi6oYaWrWZqF3gOCejdiw6uvWHnVLMFNrZOTuyGvhc6fPx+j0WhZzpyxPdeQEOLGYlcgBQQEoNPpyM3NtVqfm5tLUJDtXvugoKBrlq/9as8+3d3d8fHxsVqEEDc+uwLJYDDQv39/0tMvP/XSbDaTnp5OfHy8zW3i4+OtygNs2rTJUr5bt24EBQVZlSksLGTHjh317lMI0UYpdlq9erXi7u6urFy5Ujl8+LAybdo0xc/PT8nJyVEURVEmTpyoPPPMM5byW7duVdzc3JRXXnlFOXLkiJKSkqLo9XolMzPTUubPf/6z4ufnp/zzn/9UDh48qDz44INKt27dlLKysgbVyWg0KoBiNBrtPRwhRCto6GfU7kBSFEV58803la5duyoGg0GJjY1Vtm/fbnltyJAhyuTJk63Kf/zxx0pkZKRiMBiU6Oho5YsvvrB63Ww2KwsWLFACAwMVd3d3ZejQoUpWVlaD6yOBJIRza+hnVG4dEUK0OJl+RAhxw7F/piUnVNvIkwGSQjin2s/m9U7I2kQgFRWpj7AODW2GifCFEC2mqKgIX9/6n4rSJvqQzGYz586do3379tcdMl9YWEhoaChnzpxpk/1Nbfn42vKxQds+PkVRKCoqIiQkBO015qZqEy0krVbLTTfddP2CdbT1AZVt+fja8rFB2z2+a7WMakmnthDCaUggCSGchssFkru7OykpKbi73wDzQzdCWz6+tnxs0PaPryHaRKe2EKJtcLkWkhDCeUkgCSGchgSSEMJpSCAJIZyGBJIQwmm4XCAtXbqU8PBwPDw8iIuLY+fOnY6uUpMtWrQIjUZjtURFRTm6Wo327bffMnLkSEJCQtBoNJYn1NRSFIWFCxcSHByMp6cnCQkJHDt2zDGVbYTrHd+jjz561b/nsGHDHFPZVuZSgbRmzRqSk5NJSUlh7969xMTEkJiYSF7etZ9KeiOIjo7m/PnzluX77793dJUaraSkhJiYGJYuXWrz9Zdeeok33niDZcuWsWPHDry9vUlMTKS8vLyVa9o41zs+gGHDhln9e3700UetWEMHatFp4pxMbGysMnPmTMvPJpNJCQkJUVJTUx1Yq6ZLSUlRYmJiHF2NFgEon3/+ueVns9msBAUFKS+//LJl3aVLlxR3d3flo48+ckANm+bK41MURZk8ebLy4IMPOqQ+juYyLaTah1zWfSDl9R5yeSM5duwYISEhdO/enQkTJpCdne3oKrWI6z1YtK3YsmULnTt35uabb2bGjBlcuHDB0VVqFS4TSNd6yGV9D6S8UcTFxbFy5Uo2btzI22+/zcmTJ7nrrrss80S1JY15sOiNZtiwYaxatYr09HQWL17Mf//7X4YPH47J1PRHlDu7NjH9iKsbPny45fu+ffsSFxdHWFgYH3/8MVOnTnVgzURj1H18fJ8+fejbty89evRgy5YtDB061IE1a3ku00JqzEMub1R+fn5ERkZy/PhxR1el2TXmwaI3uu7duxMQENAm/z2v5DKB1JiHXN6oiouLOXHiBMHBbe9J8K74YNGzZ89y4cKFNvnveSWXOmVLTk5m8uTJDBgwgNjYWJYsWUJJSQlTpkxxdNWaZO7cuYwcOZKwsDDOnTtHSkoKOp2O8ePHO7pqjVJcXGzVGjh58iT79++nY8eOdO3alTlz5vCnP/2JiIgIunXrxoIFCwgJCSEpKclxlbbDtY6vY8eOvPDCC4wZM4agoCBOnDjB008/Tc+ePUlMTHRgrVuJoy/ztbZrPeTyRjVu3DglODhYMRgMSpcuXZRx48Ypx48fd3S1Gm3z5s0KcNVS+wDSpj5Y1NGudXylpaXKfffdp3Tq1EnR6/VKWFiY8vjjj1ueDN3WyXxIQgin4TJ9SEII5yeBJIRwGhJIQginIYEkhHAaEkhCCKchgSSEcBoSSEIIpyGBJIRwGhJIQginIYEkhHAaEkhCCKfx/wFj+P1ociLDAAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:09:53.422255Z", + "iopub.status.busy": "2024-02-29T06:09:53.421911Z", + "iopub.status.idle": "2024-02-29T06:10:47.256576Z", + "shell.execute_reply": "2024-02-29T06:10:47.255630Z" + }, + "papermill": { + "duration": 53.858343, + "end_time": "2024-02-29T06:10:47.259471", + "exception": false, + "start_time": "2024-02-29T06:09:53.401128", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:10:47.305828Z", + "iopub.status.busy": "2024-02-29T06:10:47.304957Z", + "iopub.status.idle": "2024-02-29T06:10:47.327593Z", + "shell.execute_reply": "2024-02-29T06:10:47.326574Z" + }, + "papermill": { + "duration": 0.04762, + "end_time": "2024-02-29T06:10:47.329916", + "exception": false, + "start_time": "2024-02-29T06:10:47.282296", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.0670990.0423780.0004392.4734850.0194840.6016530.0273725.971627e-080.9092490.0160420.2117450.0209420.1532220.0001863.382734
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.067099 0.042378 0.000439 2.473485 0.019484 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.601653 0.027372 5.971627e-08 0.909249 0.016042 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.211745 0.020942 0.153222 0.000186 3.382734 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:10:47.373226Z", + "iopub.status.busy": "2024-02-29T06:10:47.372872Z", + "iopub.status.idle": "2024-02-29T06:10:47.844265Z", + "shell.execute_reply": "2024-02-29T06:10:47.843227Z" + }, + "papermill": { + "duration": 0.495999, + "end_time": "2024-02-29T06:10:47.846556", + "exception": false, + "start_time": "2024-02-29T06:10:47.350557", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:10:47.888885Z", + "iopub.status.busy": "2024-02-29T06:10:47.888468Z", + "iopub.status.idle": "2024-02-29T06:11:43.967934Z", + "shell.execute_reply": "2024-02-29T06:11:43.966885Z" + }, + "papermill": { + "duration": 56.103839, + "end_time": "2024-02-29T06:11:43.970515", + "exception": false, + "start_time": "2024-02-29T06:10:47.866676", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:11:44.015166Z", + "iopub.status.busy": "2024-02-29T06:11:44.014787Z", + "iopub.status.idle": "2024-02-29T06:11:44.032907Z", + "shell.execute_reply": "2024-02-29T06:11:44.031966Z" + }, + "papermill": { + "duration": 0.044599, + "end_time": "2024-02-29T06:11:44.035203", + "exception": false, + "start_time": "2024-02-29T06:11:43.990604", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:11:44.076709Z", + "iopub.status.busy": "2024-02-29T06:11:44.075831Z", + "iopub.status.idle": "2024-02-29T06:11:44.081660Z", + "shell.execute_reply": "2024-02-29T06:11:44.080728Z" + }, + "papermill": { + "duration": 0.028983, + "end_time": "2024-02-29T06:11:44.083773", + "exception": false, + "start_time": "2024-02-29T06:11:44.054790", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.05525053527823563}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:11:44.126923Z", + "iopub.status.busy": "2024-02-29T06:11:44.125963Z", + "iopub.status.idle": "2024-02-29T06:11:44.531718Z", + "shell.execute_reply": "2024-02-29T06:11:44.530518Z" + }, + "papermill": { + "duration": 0.429955, + "end_time": "2024-02-29T06:11:44.533962", + "exception": false, + "start_time": "2024-02-29T06:11:44.104007", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAE8CAYAAACGvqckAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAh0lEQVR4nO3deXxU5b3A/8+ZfSbJTPYNEhIIArKjgEg1INa1VmrrbdVWcO+r2GopfSm3t1q1LdWfC7b2Um9/SvRXt+q1rb0WLVVxq6CILAIiwRACBAJZJpl9O78/JhkJWZhJJjmT5Pt+veZF5uSZc74ZMt88z3OeRVFVVUUIIVKETusAhBDiRJKUhBApRZKSECKlSFISQqQUSUpCiJQiSUkIkVIkKQkhUookJSFESpGkJIRIKZKURK+qqqpQFIX9+/drHYoYISQpiaR79tlnWb16tdZhiCFKkpJIOklKoj8kKQkhUookJZGwdevWUVlZSUZGBna7ndmzZ/Pss88CsGDBAl599VVqa2tRFAVFUSgrK4v73JFIhF/84hcUFxdjs9lYuHAhu3btoqysjKVLl8bKNTU1sWLFCqZOnUp6ejp2u52LL76Ybdu2dTrfhg0bUBSFP//5z/zqV79i9OjRWCwWFi1aRHV1dTLeDpFkBq0DEENLVVUV119/PZMnT2blypVkZmbyySef8Nprr3H11Vfzs5/9DKfTycGDB3nkkUcASE9Pj/v8K1eu5IEHHuCyyy7jwgsvZNu2bVx44YX4fL5O5b744gv++te/cuWVV1JeXs7Ro0d5/PHHqaysZNeuXRQXF3cq/5vf/AadTseKFStwOp088MADXHPNNWzatKn/b4pILlWIXqxdu1YF1JqaGrWlpUXNyMhQ586dq3q93k7lIpFI7OtLL71UHTNmTMLXOnLkiGowGNTFixd3Ov6LX/xCBdQlS5bEjvl8PjUcDncqV1NTo5rNZvXee++NHXvrrbdUQJ00aZLq9/tjxx999FEVUHfs2JFwnGJgSfNNxG39+vW0tbVx5513YrFYOn1PUZR+n/+NN94gFArxgx/8oNPxH/7wh13Kms1mdLror284HKaxsZH09HQmTJjAli1bupS/7rrrMJlMsefnnHMOEK1xidQiSUnEbd++fQBMmTJlQM5fW1sLQEVFRafj2dnZZGVldToWiUR45JFHGD9+PGazmdzcXPLy8ti+fTtOp7PLuUtLSzs97zhfc3NzMn8EkQSSlMSQ9Otf/5rly5dz7rnn8qc//YnXX3+d9evXM3nyZCKRSJfyer2+2/Ooshp0ypGObhG3cePGAfDpp592qc2cqK9NuTFjxgBQXV1NeXl57HhjY2OXGs1LL73EwoULeeKJJzodb2lpITc3t0/XF6lBakoibhdccAEZGRmsWrWqy92wE2scaWlp3TahTmXRokUYDAbWrFnT6fhjjz3Wpaxer+9Sy3nxxRc5dOhQwtcVqUVqSiJudrudRx55hBtvvJHZs2dz9dVXk5WVxbZt2/B4PDz11FMAnHHGGbzwwgssX76c2bNnk56ezmWXXXbK8xcUFHDbbbfx0EMP8fWvf52LLrqIbdu2sW7dOnJzczvVwL72ta9x7733ct1113H22WezY8cOnnnmGcaOHTtgP78YJBrf/RMp7sQhAR1eeeUV9eyzz1atVqtqt9vVOXPmqM8991zs+y6XS7366qvVzMxMFUhoeEAoFFJ//vOfq4WFharValXPO+88dffu3WpOTo76/e9/P1bO5/OpP/nJT9SioiLVarWq8+fPVz/44AO1srJSraysjJXrGBLw4osvdrpOTU2NCqhr165N9C0RA0xRVenpE6mtpaWFrKwsfvnLX/Kzn/1M63DEAJM+JZFSvF5vl2Mdk3sXLFgwuMEITUifkhgUx44dIxwO9/h9k8lEdnY2L7zwAlVVVVxyySWkp6fz3nvv8dxzz3HBBRcwf/78QYxYaEWSkhgUs2fPjg2O7E5lZSUbNmxg2rRpGAwGHnjgAVpbW2Od37/85S8HMVqhJelTEoPi/fff77Zp1iErK4szzjhjECMSqUqSkhAipUhHtxAipQzpPqVIJMLhw4fJyMhIyix1IURyqapKW1sbxcXFsVUdTmVIJ6XDhw9TUlKidRhCiFOoq6tj9OjRcZUd0kkpIyMDiP7Adrtd42iEECdrbW2lpKQk9lmNx5BOSh1NNrvdLklJiBSWSPeKdHQLIVKKJCUhREqRpCSESClDuk8pHqqqEgqFep13Jbqn1+sxGAwy3EIMqmGdlAKBAPX19Xg8Hq1DGbJsNhtFRUWddgIRYiAN26QUiUSoqalBr9dTXFyMyWSSv/gJUFWVQCDAsWPHqKmpYfz48XEPfhOiP4ZtUgoEAkQiEUpKSrDZbFqHMyRZrVaMRiO1tbUEAoEue72JziIRlbf3HsPpCbJwYj4Oq1HrkIakYf+nT/6694+8f/HbVd/K1gMt1Bx388buo1qHM2TJb5wQSbK7vjX29YEmD22+oIbRDF3DtvkmxGDyBcMcaomuF3VmWRajs2xYjN1vgCl6JzWlEa6srCy2Brbou2NtflQV7FYj54zPozw3DaNePl59Ie+aEEnQ4ok21QrsZo0jGfqk+TYMBAIBGUeksamjHYwvSCcQjhAKR6g57qbZE2R2WZYMRUnQiKwpBUKRHh+hcCTussE4yvbFggULuPXWW7n11ltxOBzk5uby85//PLZNdVlZGffddx/XXnstdrudm2++GYD33nuPc845B6vVSklJCT/60Y9wu92x8zY0NHDZZZdhtVopLy/nmWee6VN8onsWox67JToM4NUd9bxffRx3QGYSJGpE1pR+/1Z1j98rz01j8cxRsef/884+guHulzEfnWXlyjO/XGTuyfdr8J70S/jjr57WpxifeuopbrjhBj788EM2b97MzTffTGlpKTfddBMADz74IHfddRd33303APv27eOiiy7il7/8JU8++STHjh2LJba1a9cCsHTpUg4fPsxbb72F0WjkRz/6EQ0NDX2KT/TMoNfhsBpp8QRpdgdIN4/Ij1mfybuVokpKSnjkkUdQFIUJEyawY8cOHnnkkVhSOu+88/jJT34SK3/jjTdyzTXXcPvttwMwfvx4fvvb31JZWcmaNWs4cOAA69at48MPP2T27NkAPPHEE0yaNGnQf7bhxhcM8+r2erLSjCyckI+iKGTZTLR4grR4gpRkax3h0DIik9KyhRU9fk93UvP/5nPH9Vj25K6C6+eX9yesTs4666xOfRHz5s3joYceik0sPvPMMzuV37ZtG9u3b+/UJFNVNTbd5vPPP8dgMHTaxmjixIlkZmYmLeaRqtkT4ECTh2aPgfMmRv/PMizRj1abX8YqJUrzPqVDhw7x3e9+l5ycHKxWK1OnTmXz5s0Dek2TQdfjw3DSbdzeyp58y7e7MgMlLS2t03OXy8Utt9zC1q1bY49t27axd+9exo3rObGK/uu48zYqfBB2vQLOQ2S09y21+UJahjYkaVpTam5uZv78+SxcuJB169aRl5fH3r17ycrK0jKslLBp06ZOzzdu3Mj48ePR67sfkDdr1ix27dpFRUX3tcCJEycSCoX4+OOPY823PXv20NLSktS4R6I2XwhrsJnxLf8ENQ0aq7GXfTf2PZEYTZPS/fffT0lJSawjFqC8PHlNoKHswIEDLF++nFtuuYUtW7bwu9/9joceeqjH8nfccQdnnXUWt956KzfeeCNpaWns2rWL9evX89hjjzFhwgQuuugibrnlFtasWYPBYOD222/HarUO4k81PLn8QQradmE2tDe3Q36yXHuBPJlq0geaNt9eeeUVzjzzTK688kry8/OZOXMmf/zjH3ss7/f7aW1t7fQYrq699lq8Xi9z5sxh2bJl3HbbbbFb/92ZNm0ab7/9Np9//jnnnHMOM2fO5K677qK4uDhWZu3atRQXF1NZWckVV1zBzTffTH5+/mD8OMOayxciy1eL0aCDzFIAsry1XD6jmMumF5/i1aILVUNms1k1m83qypUr1S1btqiPP/64arFY1Kqqqm7L33333SrQ5eF0OruU9Xq96q5du1Sv1zvQP0bSVVZWqrfddpvWYaiqOrTfx8Hy4jvb1H//vz9RG1+9V1VbDqrqm79W1bf/H1UNh7UOTXNOp7PHz2hPNK0pRSIRZs2axa9//WtmzpzJzTffzE033cQf/vCHbsuvXLkSp9MZe9TV1Q1yxEJ0ZfRElykxOIogowj0RggHwdOocWRDk6Z9SkVFRZx++umdjk2aNIn//d//7ba82WzGbJa5RSK1XF5hIGzIRskZBTodZBRCSx0HD+zjiE3P2Lx0stNkGlC8NE1K8+fPZ8+ePZ2Off7554wZM0ajiFLDhg0btA5BJMLVgF6nQEZB9Hl6NCntP7Cfj/TZWE16SUoJ0LT59uMf/5iNGzfy61//murqap599ln+53/+h2XLlmkZlhCJcbWvMpnenpRs0SHcGZE2gC5Tj0TvNE1Ks2fP5i9/+QvPPfccU6ZM4b777mP16tVcc801WoYlRNzqjhxn94H66AJvaXnRg+1JKS3sBJBJuQnSfJrJ1772Nb72ta9pHYYQfdLacpxWT5CIMZ1RhvYmmi0n+k/EhaKG8QZkAGUiNJ9mIsRQFnRF77DpbCfMQjClg96IUQFzqA23X2pKiZCkJEQ/hN3NACjWE5KSooDFgdGgYA678EhNKSGSlIToh4inCQB92knrk1gcGPU6zCEXHulTSojmfUpCDGWqrwUAY/pJk8jNdixGPYvKLejGjer6QtEjSUpC9IPibQHA1KWmZEevKBSaA5AhOwsnQppvQvRVJIIu5AHAnNG1pgSAf/hOGh8oIyspqSqEAoP/ULtf47s7Tz/9NDk5Ofj9/k7HFy9ezPe+971kvyOiPwIuzihxMGdsLlmOzM7fs0STUsOxBjbvb8LpkSVM4jWymm/hILzb85pEA+acn4AhvmkGV155JT/60Y945ZVXuPLKK4HoLiSvvvoq//znPwcySpGogAsAvTkdTl58r72mdLShgXfVY2TajDhsxsGOcEgaWTWlIcBqtXL11Vd3WvjuT3/6E6WlpSxYsEC7wERX/ug0EswZXb9nSgfAqFPRR/z4gn3bbmskGlk1Jb0xWmvR4roJuOmmm5g9ezaHDh1i1KhRVFVVsXTpUtnUMMU0Nh6n4UgrukgxE07+pt4ARgsGnQtTxIs3KMMC4jWykpKixN2M0tLMmTOZPn06Tz/9NBdccAE7d+7k1Vdf1ToscRK3q4UWT5Cw39Q1KQGY0jHoGzGGPfgkKcVtZCWlIeTGG29k9erVHDp0iPPPP5+SkpJTv0gMqpAnemdNb7V3X8CUhkGnYAx7pfmWAOlTSlFXX301Bw8e5I9//CPXX3+91uGIboS90VUA9Laek5JRr8MUdkvzLQGSlFKUw+Hgm9/8Junp6SxevFjrcEQ3Ir5oR7ext5qSvqOmJEkpXtJ8S2GHDh3immuukSWAU5GqorbffTPYHN2XMaWTYTFyTp4V3UTZNSZekpRSUHNzMxs2bGDDhg3893//t9bhiO6EfISDAQCM1m6GBACY0jHpdeSbQ5Auf1jiJUkpBc2cOZPm5mbuv/9+Jkzo9r6O0JrfFZ0goDNjtfQwt83UvrV6+yBLER9JSilo//79WocgTiXoYcooBxFrNmTZui9jSkdF5ejxRg7ub2J6SSZGvXTjnoq8Q0L0RTA6EVdnTkOn62FQqykNBYXaI8d57/OjeGQFyrgM+6SkJjAZVnQl718PAtGkhNHacxmDBRQFo16HIeLHF5KkFI9hm5SMxujUDo/Ho3EkQ1vH+9fxfooon7eNz460sqU+0HMhnQ4MFvQ6BUPYJ8MC4jRs+5T0ej2ZmZk0NDQAYLPZZO5YAlRVxePx0NDQQGZmJvqTZ8GPcH6Pi2ZPkONuhVm9FewY1R3x4g/JqO54aJqUfvGLX3DPPfd0OjZhwgQ+++yzpJy/sLAQIJaYROIyMzNj76P4UtAbvaOmM/XQyd3BaMWg12GM+PDLVJO4aF5Tmjx5Mv/6179izw2G5IWkKApFRUXk5+cTDMoiW4kyGo1SQ+pBOOAGQG9J672g0RprvvmlTykumiclg8EQ919iv9/faUXG1tb4lhrV6/Xy4RJJFfa1JyVzeu8FjTYMOgVDxCfNtzhp3tG9d+9eiouLGTt2LNdccw0HDhzoseyqVatwOByxh8ycF1qJ+DtWnTx1TSk/w0xleTpTRvUwHUV0omlSmjt3LlVVVbz22musWbOGmpoazjnnHNra2rotv3LlSpxOZ+xRV1c3yBELAUQihAM+AIynbL7ZsJkMFFgjOKxyBzMemjbfLr744tjX06ZNY+7cuYwZM4Y///nP3HDDDV3Km81mmZwqtBfyEolEm2KmOPqUAAh6Bzio4UPzPqUTZWZmctppp1FdXa11KEL0LOChPDeNMYU5hMtzey9rtBEIRzjWcJzW+lYmFfWwzImI0bxP6UQul4t9+/ZRVFSkdShC9Kxjikn7Im69MtrwBsJUHzrGhzVNgxDc0KdpUlqxYgVvv/02+/fv59///jff+MY30Ov1XHXVVVqGJUTvgnFMMelgtGLQt999C4YGNq5hQtPm28GDB7nqqqtobGwkLy+Pr3zlK2zcuJG8vDwtwxKid0EP1Q1tuHwBxvqC2C29dGC3DwnQqWGCgV6mpIgYTZPS888/r+XlheibgIcmd4DDQPmphh7pjegN0aSlhLyEwhEMsnxJr+TdESJBIb+LsAohnQWL6RQfIUVBb7ahQPuobhlAeSqSlIRIULB9NHfIYMMUR61HMdrQt0/KlZUCTk2SkhAJCnqjSUlninPlCaMVg05BHwlITSkOKTVOSYihIOyLc4pJB4OF8rw0CsfYyUpL/R2atSZJSYgEfblCwCmWLelgtJJpNZGZBhhlYvipSPNNiEREIoT90XFKBsspVgjoYGjf7SToG6CghhepKQmRiJCXggwzeekmgtPK4nuN0YrLH6LxaCOmfC9FjjgGXY5gUlMSIhHtE2t1JitmU5z9QwYLjS4/Ow8cpbpB9oA7FUlKQiSivT8JY5yd3PDl6pMRvyyJGwdJSkIkIujlYLOHHQ0BDjbHuVOOwYJBtlmKm/QpCZGIoBunN8gBH5ji3VyyY5slqSnFRWpKQiQi4CEUUQnpLViMcX58jJb2dbr9+GVE9ylJUhIiEUEv4YhKUGfFEu+YI0O0T0lRIwSD/lOXH+EkKQmRiKCbUDhCUG/FYogzKemN6Nu3DgsHZMfmU5E+JSESEPa7CasQ1Fkwx9t8UxTMljROKwhQMj5zQOMbDiQpCZGAjhUCwgYrZkP8DQ2D2UZOmgcyBiqy4UOab0IkoGMyrs6cFt8KAR1kqkncpKYkRLxUlTRdkLnl2cyYfVpirzVaafIEaKo/TnFmCJtJPno9kXdGiHgFvaCq6BQFa1qC7TCDmdrjbj731WMbE5Sk1AtpvgkRr9guJhbQJbgEicGKvmNXE1norVeSlISIV8BNk9vPrmNBdh52JvZaowWDLjrVxC9TTXqVMknpN7/5DYqicPvtt2sdihDdC3px+8McdOtoaE1wEKRBJuXGKyWS0kcffcTjjz/OtGnTtA5FiJ4F3YQikcTGKHU4caqJNN96pXlScrlcXHPNNfzxj38kKytL63CE6FnQe8K8t0T7lNon5Yal+XYqmielZcuWcemll3L++eefsqzf76e1tbXTQ4hBE/AQCrfPe4t3ikmH2I4m0nw7Fc13yN2yZQsfffRRXOVXrVrFPffcM8BRCdGDoJtQRCWYyAoBHQwWstNMTLYaUEscAxPfMKFZTamuro7bbruNZ555BovFEtdrVq5cidPpjD3q6uoGOEohThCMbrud0AoBHQwWbCYDeRbITzMOTHzDhGY1pY8//piGhgZmzZoVOxYOh3nnnXd47LHH8Pv96PWd/+PNZjNms3mwQxUiKuDuV59STMgHpgSW0x1h+pSUvvjiC8aOHduvCy9atIgdO3Z0OnbdddcxceJE7rjjji4JSQjNBT2cMSaLyTMmYLEmWNvR6QjqTLS2tdFy+DjjyiQp9aRPSamiooLKykpuuOEGvvWtb8Xd/DpRRkYGU6ZM6XQsLS2NnJycLseF0JyqQtCHTlGwpdlBl8Bk3HY+jHx+1MUXpoOMKxszAEEOD33qU9qyZQvTpk1j+fLlFBYWcsstt/Dhhx8mOzYhUkfQC2r7XTNjnDvjnkRvir4u4vcmK6phqU9JacaMGTz66KMcPnyYJ598kvr6er7yla8wZcoUHn74YY4dO9anYDZs2MDq1av79FohBlTQgzcYZk9jgA9qWvp0CoO5PZmFfYTCMiygJ/26+2YwGLjiiit48cUXuf/++6murmbFihWUlJRw7bXXUl9fn6w4hdBW0IMvGOawW8cXx/u2oaTRZEUBDJGAjOruRb+S0ubNm/nBD35AUVERDz/8MCtWrGDfvn2sX7+ew4cPc/nllycrTiG01b6LSUJrc59EOWFTyoAkpR71qaP74YcfZu3atezZs4dLLrmEp59+mksuuQSdLprjysvLqaqqoqysLJmxCqGdoKd9F5M+DAfoYOzY/02WL+lNn5LSmjVruP7661m6dClFRUXdlsnPz+eJJ57oV3BCpIygh1A4QkhvIS2Btbk7OXGlAJn/1qM+JaX169dTWloaqxl1UFWVuro6SktLMZlMLFmyJClBCqG5juZbP2tKJdk28u1W7OkyCLgnfUr548aN4/jx412ONzU1UV5e3u+ghEg5wfbJuHpb4vPeOhisZNtMFNkgzSzL4fakT++uqqrdHne5XH0aSClEygt6Ymsp9bmmZGivHYVkl9zeJJSuly9fDoCiKNx1113YbF8OIguHw2zatIkZM2YkNUAhUkLAzWkFGYyZfBr6nD5u3ma04g2GcTW3EGn1UWCXP+DdSSgpffLJJ0C0prRjxw5MJlPseyaTienTp7NixYrkRihEKgh62ncxsUOfO7otHHf5qWt24jvUQoG9MLkxDhMJJaW33noLiE6cffTRR7Hb7QMSlBApJRKJTjOBPk8xib42evdNQSUYkCZcT/qU8teuXSsJSYwcIS+qGmHvMRdvfeHq++18nQG9Prq6QMjvSWKAw0vcNaUrrriCqqoq7HY7V1xxRa9lX3755X4HJkTKCEQHTh5xK2w92MpXTsvv23kUBZ3JAjgJ+2X77p7EnZQcDkds73SHQ5bzFCNI0N0+HMCCUa9g1Pd9dlbHSgHhgDtZ0Q07cSeltWvXdvu1EMNebOCkre/DAdrpzVYAwgFZvqQnfUr5Xq8Xj+fLNnFtbS2rV6/mn//8Z9ICEyJlBD0EwxGCeitWU/+SkqFjTaWgNN960qekdPnll/P0008D0NLSwpw5c3jooYe4/PLLWbNmTVIDFEJzAXdsayVrP2tKJmsaY3PTOKu0H3fxhrk+rzx5zjnnAPDSSy9RWFhIbW0tTz/9NL/97W+TGqAQmgt6CEYiBPWWficlo9lKgd1CaUbiy+mOFH1KSh6Ph4yM6KjWf/7zn1xxxRXodDrOOussamtrkxqgEJprrymFdFYs/Wy+xXY1kakmPepTUqqoqOCvf/0rdXV1vP7661xwwQUANDQ0yPglMfwEPZRkWfnGWeM5qzynf+cyWGn1Bak/3ownEEpOfMNMn5LSXXfdxYoVKygrK2Pu3LnMmzcPiNaaZs6cmdQAhdBcwIOiKFht9n53dGO0UHPczfb9R2h0BZIT3zDTp/UTvvWtb/GVr3yF+vp6pk+fHju+aNEivvGNbyQtOCFSQrB9TFEyNpA0tK8+GZaF3nrS50VdCgsLKSzsPKFwzpw5/Q5IiJQSDkEoQG2jm/01bmaUZ+Cw9WPbbaMVg05BH/TjC8qSuN3pU/PN7Xbz85//nLPPPpuKigrGjh3b6RGvNWvWMG3aNOx2O3a7nXnz5rFu3bq+hCTEwGivJTW4Q2w55CHQ362ROmpKql/W6e5Bn2pKN954I2+//Tbf+973KCoqik0/SdTo0aP5zW9+w/jx41FVlaeeeorLL7+cTz75hMmTJ/fpnEIkVcCDiooXMyhK//uUDJZoTSkSwB+QPqXu9CkprVu3jldffZX58+f36+KXXXZZp+e/+tWvWLNmDRs3bpSkJFJD+y4mAV10eoilr2spdTBY0LevbR+UqSbd6lNSysrKIjs7O6mBhMNhXnzxRdxud+xu3sn8fj9+/5fjO1pbW5MagxBdBNwEw9ENA0wGHYZ+TMYFQKdDZ7QAXkI+Wb6kO316h++77z7uuuuuTvPf+mrHjh2kp6djNpv5/ve/z1/+8hdOP/30bsuuWrUKh8MRe5SUlPT7+kL0qn1rpaDe2u/JuB0cjgzG5qYxOd906sIjUJ9qSg899BD79u2joKCAsrIyjMbOdyO2bNkS97kmTJjA1q1bcTqdvPTSSyxZsoS3336728S0cuXK2DrhEK0pSWISAyrgIhhJzry3DhlpGWRE3GBNyumGnT4lpcWLFyctAJPJREVFBQBnnHEGH330EY8++iiPP/54l7JmsxmzWfbLEoMo4I7VlKymfjbdOsSmmshKAd3pU1K6++67kx1HTCQS6dRvJISmAm7y0s0sPm0iobzkLPQf1JnxeIO4jzdTLHsHdNHnwZMtLS289NJL7Nu3j5/+9KdkZ2ezZcsWCgoKGDVqVFznWLlyJRdffDGlpaW0tbXx7LPPsmHDBl5//fW+hiVEcvldKIqCJS0DkrSBpA8Tu+pbOeo/xOIpSTnlsNKnd3n79u2cf/75OBwO9u/fz0033UR2djYvv/wyBw4ciK21dCoNDQ1ce+211NfX43A4mDZtGq+//jpf/epX+xKWEMkXcEX/NaUn7ZSG9tUnCXpRVbXP4/yGqz4lpeXLl7N06VIeeOCB2BImAJdccglXX3113Od54okn+nJ5IQZHOAghP4davFTv9zKpxEd+EjaQNJqjC7zpI9FR3cm6qzdc9Knn7qOPPuKWW27pcnzUqFEcOXKk30EJkRLaa0nHPWG2HPLQ5k/OUiMGkxWdQnRUt0w16aJPSclsNnc7cPHzzz8nLy+v30EJkRLadxzxYAFFIc2UnD6ljkm5hoifgCSlLvqUlL7+9a9z7733EgwGAVAUhQMHDnDHHXfwzW9+M6kBCqEZvwsVFTfRJlu/57116JiUG5HlS7rTp6T00EMP4XK5yMvLw+v1UllZSUVFBRkZGfzqV79KdoxCaCPgJhxR8SvRjum0ZCUloxW9Tte+ppLUlE7Wp/qow+Fg/fr1vP/++2zbtg2Xy8WsWbM4//zzkx2fENoJuAiGVQL6NMzGJMx762CwUJxpIT+iYEmTqSYnSzgpRSIRqqqqePnll9m/fz+KolBeXk5hYaHc3hTDS8BFIBQdzW1L5h0yo5WctPaZCSY1eecdJhJK/aqq8vWvf50bb7yRQ4cOMXXqVCZPnkxtbS1Lly6VpXDF8BJwE4pECOpt2JI0cBIAnQF07UlOppp0kdA7XVVVxTvvvMMbb7zBwoULO33vzTffZPHixTz99NNce+21SQ1SCE3428hOM/HNSRMJZiZxPoii4MOI3+sj2OIkt9CRvHMPAwnVlJ577jn+8z//s0tCAjjvvPO48847eeaZZ5IWnBCaCrhRULCk2cmw9GNd7m4c8Sjsqm9l94GjST3vcJBQUtq+fTsXXXRRj9+/+OKL2bZtW7+DEkJzkQgE29cLS8YuJifRm6J39MKy+mQXCSWlpqYmCgoKevx+QUEBzc3N/Q5KCM0F2kBVOeT08+YXbhpak9v3o+tISn5JSidLKCmFw2EMhp67ofR6PaGQ7PophgF/GwBH/Ua2HWxN2hSTDgZTdP5bOCgd3SdLqKNbVVWWLl3a40Jrsg6SGDbak5KLjoGTSbz7BujbVwqIBGSd7pMl9E4vWbLklGXkzpsYFnytqKi41GjySNoUk3aG9pUCVOlT6iKhpLR27dqBikOI1OJvIxRW8eqj6yilJ3OcEmCyRs+rBj0y6PgkyX2nhRgu/K0EwhEC+jTSzHr0uuQmDbM1ndJsGyV2ExEV9JKTYiQpCdEdfxuBUEdSSv7HRG+yMSrTCjYVkpzwhrokzTAUYphpT0p+fVrSm27Al2OfgtLRfTKpKQlxskgEAi7y7WaunTOVkCH5gycxWvEGwwS8beh9AdItslpAB6kpCXGy9oGTis6AxWYfmJqSwUpNo4dd9a3UHW1M/vmHMElKQpysfYwSpnQYqLtiOh06Y3S4QcDrGphrDFGSlIQ4WXtS2t0Mb352FKc3OCCX0bWP6g753ANy/qFK06S0atUqZs+eTUZGBvn5+SxevJg9e/ZoGZIQ4ItuilHr1rOtzkkoPDBL1uraB1AGfVJTOpGmSentt99m2bJlbNy4kfXr1xMMBrngggtwu+Uvh9CQz0k4ouIi2sGdbhmY+0F6c0dNSe7AnUjTu2+vvfZap+dVVVXk5+fz8ccfc+6552oUlRjxfC0EwhH8hgxMBh1mw8BsFmm0REd1h/zyR/hEKTUkwOl0ApCdnd3t9/1+f6dJv93tPSdEv3lb8AXD+A0Z2K3JXdztRAZztCYWCUhSOlHKdHRHIhFuv/125s+fz5QpU7ots2rVKhwOR+xRUlIyyFGKYU9VwefEH4rgM9ixD1DTDSDDbqc028aEbNm2+0Qpk5SWLVvGp59+yvPPP99jmZUrV+J0OmOPurq6QYxQjAgBF0RC+ELRrZUcA1hTSkuzMyrTyqgBGJs5lKVE8+3WW2/l//7v/3jnnXcYPXp0j+XMZnOPazkJkRTelug/Ohuqoh/Q5hvt45QISvPtRJomJVVV+eEPf8hf/vIXNmzYQHl5uZbhCAG+FgAmlpVQPmXcwF7LaMMdCOFvcZIeimAypEzDRVOaJqVly5bx7LPP8re//Y2MjAyOHDkCRHfgtVqtWoYmRipf9GaLYs3CkswNKLtjsvFZfSv+SBsVHj/5dvmdB437lNasWYPT6WTBggUUFRXFHi+88IKWYYmRrL35hjVz4K9ltGHQ61DUCD6vrEDZQfPmmxApxdeCNxjmg/1+zMHjzK/IHbhr6Y3oDEYIhAn4XED3Q2FGGmnECnEibwveQJh9bQZqGwd+pLXSvq5SwCud3R0kKQnRIRwEfxveYBifwUGWbQDvvLXrmJQb8LYN+LWGCklKQnTwNAHgjhgI6S1kpQ38wmt6SwYAIUlKMZKUhOjgjSalFuwAZNkGPikZrNGkFJSkFJMSgyeFSAme6AqQjZHoRNmstIFvvmVnZmHJtqG3D/ilhgxJSkJ08DQSDEdo1Q1eTSkrK4usTCtYZLv7DtJ8E6KDp4lAOELYkkWGxYBRPwgfD1O0+YasFBAjNSUhILo6gKeRNJOBq+bPwG8enDFDqsmGJxDC39yEI6ImfdPLoUiSkhAQXR0gHARFh2LLxqIbpOVETGl8eshJmDam+IPYrbLVkjTfhABwH4/+a82EwUpIgGLKwNg+1cTrkSYcSFISIsp9DBWV9+rhlW2HcfsHqeNZb0BpX8LE65aVVEGSkhBR7mMEQhHqQ3ZqjrkxD+IyIjpzdAiC3+0ctGumMklKQgC4GnAHwniM2WSnmzAMxp23drr2DQQCHhlACZKUhIBIBNzH8fhDuI3Z5KUP7uqmBkt0XJSM6o6SpCSEtxkiIdpCCn6DnbyMwU1Kemv7Vkte6VMCGRIgBLgbUFE5Fs4ARaHQYRnUy+dm55CWbcOYMTA78Q41kpSEcDXgD0ZoVjLR6xTyB7mmlJ2TB5lWMPpPXXgEkOabEO5jhCMq1uwiCu2WwZleciJz+2xcvzTfQGpKYqRTVWg9TJrZwAUzp6M6et7ia8CYM3AHQgS8zaQHgxiNA786QSqTmpIY2fxt0cmwig4yClEUDeaemdL4tN7FZ/WttDmbB//6KUaSkhjZ2uoJqypBSw7oNaqhKAq69mEBnjZJSpKUxMjWehinJ8A/amHdjnrNwtBbHQB4JSlpm5TeeecdLrvsMoqLi1EUhb/+9a9ahiNGorZ6nN4gLlP+wG8+2YuOpOR3t2gWQ6rQNCm53W6mT5/O73//ey3DECOVqrYnpRAuUy4l2drtUGtMywQgKElJ27tvF198MRdffHHc5f1+P37/l2M5WlvlFqroB08jfp8Xd0jBa8pmdJZNs1BMadGaUqhjh94RbEj1Ka1atQqHwxF7lJSUaB2SGMqcdbR4grhMeRQ6bJo232wZ0ZUuwx75QzukktLKlStxOp2xR11dndYhiaGs5QDNngCt5iLG5adrGoo9O4/SbBsVGcFos3IEG1KDJ81mM2bz4E4BEMOUqhJqqsXpDdKaV8TY3DRNw0m355CeZYsmpIALzBmaxqOlIVVTEiJpfC0ofhdleXbKyseTPQi74fZKb/hyuol3ZA8LkKQkRqaWA+h1CgWjylg0ZbQ2I7lP4tHbafIEcDYd0zoUTWnafHO5XFRXV8ee19TUsHXrVrKzsyktLdUwMjHstRyI/puZOr9nu1v0BI60kZlVj2Os1tFoR9OktHnzZhYuXBh7vnz5cgCWLFlCVVWVRlGJYU9VaTn0OR6XF7ttFNp2cX/JbM8hAATajmsdiqY0TUoLFixAHeF3GoQGXEdpON5Ig0dFdaZzTqHWAUXZHLm0AUFXk9ahaGpI3X0Tw5MvGKbe6SMciZCXbsFhG9iJsYGGvTR7AjgtZcwuyhrQayUiLTMfANXdGL0LlwL9XFqQpCQ0EwxHeL/6ODsOOglFvqwxj8mxsWBC/oDdETteu5uICpHssYO+Hndv7Fn5qIqOUNBP0NOCMS11EuZgkrtvQhOqqvK/Hx/kkwMthCIqWTYjBXYLigK1jR5e3nKQUHgA1qwOeGg+UgNAUdmklLjr1sFiNhI0RxNR23HtVizQmtSUhCYUReGc0/JYv/MIlRPyKcuxoSgKTk+Q9buPMrssa0D2Xms9uJNWbxCPKYc5ZRqsMtkLRVEwZOSDr5G2psNkjzld65A0IUlJaGZUppUlZ5d1qq04bEa+OWvUgNVg6vd+AoCp6HQyLKm37OyY0jIsSh1p+pG7B5w038Sg2ry/ieOuL1d66C75nHjsWJufdTvqk9OUC7gxuerQKTDqtFn9P98AGD26lNx0M1Z/o9ahaEaSkhg0NcfdvLv3OC98VIfLHzpl+XBE5W9bD/HZkTbeq07C2J2GzyjPtjFr8iTGjE6tpltMWl70X3cjhE/9Hg1H0nwTg8IXDPPG7qOgqszM8pN+5ENwNYDPCSE/hP2gdq4N6Y02LtNZeNtpYk91KRX56X1f80hVoT7adDMWTwNd6nRwn0g122kOGvB52shtOYwpJ3VGnA8WSUpiULz3+TGMTZ8z17uNuYYwNMeRFAIeCoAzIi4ajnxEzb82UfDVKzFmjkr4+s31NZia6kmzWqBwSuI/wCBRdDq2Om1YW4+iP1JLgSQlIZKv7shxAtv+zGneA5xeZEdvsEJWOdhHgS0bDBYwmEE5cZG19iU8PE2UZH1By7aPofUgh974A2XTK2HsQtDFvyhb9dZ3CNU7yRs/ngqjdsvexsOSMxpaa3Adq6WAc7QOZ9BJUhqm6p1eNu9vJsNiYFxeOqOzrJqMyQm0HuPwG2vI8rVS4LDhmLQARs8BoyWOV+dD9lhMo88kJ/9cdrzzV5SWanL2/puM1nqYvDiudYeOHT1EqH4nCpB92tn9/IkGXkb+GFw17+JtHJmLGEpSGur8bXDwI2jcB96WaO0howhTxmlUH00DRccnB1ooybbx1UkFAz6Fo5O2IzS8/SQRXyuqNYtR518PmUV9OlVZcSGfTVnMZ/u2k+P+gAznQfj4KZj6LcjoffLa/s2vASqW4olkF6Z+cyi7sIw2FPxtTajeZhTryBrZLXffhipVhYObYeMf4MAmjtYf5MDx1mincfN+Mmtf51v8i1nZAQw6hbomD898WMuBRs/gxOdpgu0vMCoNikrKGX3ezZj7mJA6LJiQx7SZsxl30a1gy4km5E/+P2j4rMfX1H/xKaEju1EUhbJZF/br+oMlP8uOx5JPIKziqq8+9QuGGUlKQ1E4CJ/+L+xdD5EQXwQzeZ2z+D/bYg6NvxrKz0VvtFBibKPSvY6lFW6KMy34gxH+tvUQXxxzDWx8ATds/zMEPCj2QsrOu5HSwrx+n9Zi1DOrNAtdeg7Muhayx0Zvm+/8C+x/r8va1iF3Mwc/eAkAa/lZZBYMjY0mjHod+txxADTW9ZxwhytJSkNN0AvbnoPje0Fn4HPHPP7GeTTaxjHztDKKR5VC2XyYe0vsQ5vxxTq+mV9PRX46oYjKjkPOgVsyJhSAHS/ibGogbLbD1P+IdmIn+zI6E+sN59KSMyN6oObdaKJuiw47oLmWmn/9DyGfi5A1lwnzLkl6DAMpa/QEAHwNX0AkrHE0g0v6lIaSgDuakFzHwGBmb8FFvFprAAXmjs1m3ricL8ua0mDqlfDFm1D3EYYv3uSSsnPY6pjE9JKsgen0jkRg9yu0NRxgR4Ofg7azuVyxMhD3ut7f18inh9uotUzgyjG5OA68GU3Ux/dGl/xQVXINAY5aMik593tYLKl9x+1k48ZWoB4pIkPxQ1MN5FZoHdKgkZrSUBFww9ZnownJlMb+0m/wjwPRTusZpZnMG5vT9TU6HYxbBOXR28r6/e9yRmQHhvaBg6qq0uIJJCc+VYW9rxNq2MPnx3zszr2QzNxCLMaB+RWbW55NTrqJNl+IZw9ks2f0t4jkTiCEDjUSAb2BrIq5zL7iNspGJz6uSWsZVjP20unRPx5Hd2gdzqCSmtJQ4HdFa0ju42BOxzXxSv7+SRsRVWVSkZ0Fp+X1XPNRFCj7CuhNUP0G1H4A4SDquEW8t6+RbXUtLJ45qv+7w9a+j3roE6qPudnhWIA+s4SFE/MHbBiCxajnW2eM5m9bD3PE6eMfNWDQTUWNTOLSSQ7GFReATk/qrJbUBwWT4eBHRI7tRRfwgEm7HXwHk9SUUp2vNVpDch+PjsmZcQ3p2YUsmJBPRX46F5xeEN8Hv2QOnHZB9OuDm4nseY3jbT6CYZW/bT3M4RZv32M89DHUvMuBJg9bTLNpSy/nkqlFmA0Du+OszWTgyjNGM78ilzSznlBEJYyefU5dQgMrU1UkrYA9bhtb9h/DtW+j1uEMGkUdwotkt7a24nA4cDqd2O12rcNJPldD9C6Wvw3MGagzrkaxZce+rapq4jWR+u2w5x/RzRjzTucV73RqWwKYDDq+OWs0hY54BjWeoO4j1Or1HGz2sik8kbrM2VwytYgJhYO7mWIkotLmC6HTQbrZkFKLt/XHv97eQFr1/5GTaee0y1YMudpSXz6jUlNKVQ27o2Nw/G1gy6F69GKe2+HCG/jyTkyfPnhF02DS10HRYTi2i6/zJmXpQQKhCC9/cpBD8daYIuHokITqfxGOqOwwTKbOcSbnjM8d9IQEoNMpOGxGMizGYZOQACZPn4PHmE1TSyvN29dpHc6gSImk9Pvf/56ysjIsFgtz587lww8/1Dok7fjbYPffYedfIRQgmDGKt2wX8vc9Ho62+thcm4SdLgpOh2lXgtGCwXWUywLrmKruIRAI8uLmOrYfbOn99a6GaMI8uBkAw7gFzF90BeefXsiZZdm9v1YkpCjThm7Cxago7N+1Cd/+TVqHNOA0b7698MILXHvttfzhD39g7ty5rF69mhdffJE9e/aQn5/f62uHVfPN2wyHP4FDWyAcJBhRqbFNY4P/NNzBaJHZZdmcPS4HXbKW3fC2wO5XwHmIUCTCXqeO7eo4zpozL7rekKLgC4Yx6XXoiEDzfkKHttJ2cCfH23yYLDbGnH0l5E1ITjyiW75gmH+te5mcY5uwmfSUn/FV7BMWRLf6TnF9+YxqnpTmzp3L7NmzeeyxxwCIRCKUlJTwwx/+kDvvvLPX1w7JpBQOQcgbvaPmOY7qaiB0vAaDtwGFaLLZ7U7jA90snMZoUs5OM7FwQj6lOQPQn6CqUL8Vat6BgAenN4jdakAx2sDiYEe9i0ZnG+mRVpRIhGA4ggo02sZSl302362cnJLLyg43TS4/H/zrJbIbP8Fs0DGjYjS6wqmomWOIWLPRWx0puSVTXz6jmqbaQCDAxx9/zMqVK2PHdDod559/Ph988EGX8n6/H7//y6VUW1tb47tQUw1U/6vr8Vg+/jIv72too8nt//JIRxlVBVRmjcnC0P6fX3PcRUObP/r6L18AgILKjJJMTPpo2dpGD0db3OjUYKfLhyMqKjBrTDbmvHIYdQYNjQ6cB1rITTcxoySL04vt6AdqUTJFgeKZUDAVGnbiOLoTnAejI8eDXnA6MftDBIGg3kpTehnunGkUFJfyzdEOSUiDJDvdTOXF32bTxlGUuD9CF3DDgY0Ea/7Nx7XN0d8PvQlVb46OT0MBRaHIYaU0J7oHsDcUYftBZ4/XKLBbKGv/wxcIqeh0xMa0dWvqf4Al+ZUBTZPS8ePHCYfDFBQUdDpeUFDAZ591nfOzatUq7rnnnsQvFPJHb6nHIeJpI+TuZUChT9/+nw4Rn4uIz99z2aAZIu1lg17CoQBhQEUhpLfgNWbiNWbRZi6gYtY88rKj/TEzbEGmjHKQnWYavE5bvQGKpkcfoUC0OelzMvn0CG1B8JsyUc0OMqxGrEb9sOpMHirSzQYWVS5ADc+Pjlxv3Iv7aB0qLYQjKkT8EDzp99FoBYsPACUYJtLW0uP5VZ0FzGnRJ+EIKMR+17t/wQBsgcUQGzy5cuVKli9fHnve2tpKSUkckywzS2D6d06q3rZ/HTsW/TfHE8AWiLQfPqFM+5c6u7X9uUKuN0B6qKNmpMSKqkr0mT7NElt2NS8QJiOogskKenPsQ20y6LAY9RhP2E5oUJcX6Y7BBBkFkFGADnBoG404iaI3Rm9WFJxO5iSVGfMCBH0eAn4vkZA/OqIdFTUSIc1kAIse1Aj6iEppRc9/RNPMerBGf/d0ERXdCb/33TKlJfcHa6dpUsrNzUWv13P06NFOx48ePUphYdc1csxmM2ZzH8bomtIguzyuotmZ8Z/WkRH/BzY9DdLjP7UQcVEUBZvFDBYz0Pu6S0ZgVG5859UyMWg6JMBkMnHGGWfwxhtvxI5FIhHeeOMN5s2bp2FkQgitaN58W758OUuWLOHMM89kzpw5rF69GrfbzXXXXad1aEIIDWielL797W9z7Ngx7rrrLo4cOcKMGTN47bXXunR+CyFGBs3HKfXHkBynJMQIInPfhBBDniQlIURKkaQkhEgpmnd090dHd1jc002EEIOq47OZSNf1kE5KbW1tAPGN6hZCaKatrQ2HI76hxkP67lskEuHw4cNkZGSM2LlYHVNt6urq5A5kH8j71z+nev9UVaWtrY3i4mJ0vc2jO8GQrinpdDpGjx6tdRgpwW63y4eqH+T965/e3r94a0gdpKNbCJFSJCkJIVKKJKUhzmw2c/fdd/dt9QQh718/DcT7N6Q7uoUQw4/UlIQQKUWSkhAipUhSEkKkFElKQoiUIklpCGpqauKaa67BbreTmZnJDTfcgMvl6vU1CxYsQFGUTo/vf//7gxSxthLdgfnFF19k4sSJWCwWpk6dyj/+8Y9BijQ1JfL+VVVVdfk9s1gsiV1QFUPORRddpE6fPl3duHGj+u6776oVFRXqVVdd1etrKisr1Ztuukmtr6+PPZxO5yBFrJ3nn39eNZlM6pNPPqnu3LlTvemmm9TMzEz16NGj3ZZ///33Vb1erz7wwAPqrl271P/6r/9SjUajumPHjkGOPDUk+v6tXbtWtdvtnX7Pjhw5ktA1JSkNMbt27VIB9aOPPoodW7dunaooinro0KEeX1dZWanedtttgxBhapkzZ466bNmy2PNwOKwWFxerq1at6rb8f/zHf6iXXnppp2Nz585Vb7nllgGNM1Ul+v6tXbtWdTgc/bqmNN+GmA8++IDMzEzOPPPM2LHzzz8fnU7Hpk2ben3tM888Q25uLlOmTGHlypV4PJ6BDldTHTswn3/++bFjve3ADNH398TyABdeeGGP5Yezvrx/AC6XizFjxlBSUsLll1/Ozp07E7rukJ6QOxIdOXKE/Pz8TscMBgPZ2dkcOXKkx9ddffXVjBkzhuLiYrZv384dd9zBnj17ePnllwc6ZM0kugMzRN/f7sr39t4OV315/yZMmMCTTz7JtGnTcDqdPPjgg5x99tns3Lkz7snzkpRSxJ133sn999/fa5ndu3f3+fw333xz7OupU6dSVFTEokWL2LdvH+PGjevzeYU40bx58zrt2Xj22WczadIkHn/8ce677764ziFJKUX85Cc/YenSpb2WGTt2LIWFhTQ0NHQ6HgqFaGpq6nZX4Z7MnTsXgOrq6mGblBLdgRmgsLAwofLDWV/ev5MZjUZmzpxJdXV13NeVPqUUkZeXx8SJE3t9mEwm5s2bR0tLCx9//HHstW+++SaRSCSWaOKxdetWAIqKipL9o6SMvuzAPG/evE7lAdavXz8id2xOxg7W4XCYHTt2JPZ71q9ucqGJiy66SJ05c6a6adMm9b333lPHjx/faUjAwYMH1QkTJqibNm1SVVVVq6ur1XvvvVfdvHmzWlNTo/7tb39Tx44dq5577rla/QiD5vnnn1fNZrNaVVWl7tq1S7355pvVzMzM2G3q733ve+qdd94ZK//++++rBoNBffDBB9Xdu3erd99994gfEpDI+3fPPfeor7/+urpv3z71448/Vr/zne+oFotF3blzZ9zXlKQ0BDU2NqpXXXWVmp6ertrtdvW6665T29raYt+vqalRAfWtt95SVVVVDxw4oJ577rlqdna2ajab1YqKCvWnP/3piBinpKqq+rvf/U4tLS1VTSaTOmfOHHXjxo2x71VWVqpLlizpVP7Pf/6zetppp6kmk0mdPHmy+uqrrw5yxKklkffv9ttvj5UtKChQL7nkEnXLli0JXU+WLhFCpBTpUxJCpBRJSkKIlCJJSQiRUiQpCSFSiiQlIURKkaQkhEgpkpSEEClFkpIQIqVIUhJDSlVVFZmZmbHnv/jFL5gxY0bs+dKlS1m8ePGgxyWSR5KS6NbSpUt7XMd72bJlKIrSaVWDgUgGZWVlrF69utOxb3/723z++ec9vubRRx+lqqoq9nzBggXcfvvtSY1LDCxJSqJHJSUlPP/883i93tgxn8/Hs88+S2lpqSYxWa3WLovcncjhcHSqSYmhR5KS6NGsWbMoKSnptDrlyy+/TGlpKTNnzuzXuburwSxevDhW+1qwYAG1tbX8+Mc/ju2KAV2bbyc7sca2dOlS3n77bR599NHYOWpqaqioqODBBx/s9LqtW7eiKEpC6/6IgSFJSfTq+uuvZ+3atbHnTz75JNddd92AX/fll19m9OjR3HvvvdTX11NfX5/wOR599FHmzZvHTTfdFDtHaWlpl58JYO3atZx77rlUVFQk60cQfSRJSfTqu9/9Lu+99x61tbXU1tby/vvv893vfnfAr5udnY1erycjI4PCwsI+rfzocDgwmUzYbLbYOfR6PUuXLmXPnj2x/cuCwSDPPvss119/fbJ/DNEHshyu6FVeXh6XXnopVVVVqKrKpZdeSm5urtZh9UtxcTGXXnopTz75JHPmzOHvf/87fr+fK6+8UuvQBFJTEnG4/vrrqaqq4qmnnkpabUKn03HyUl7BYDAp547HjTfeGOvEX7t2Ld/+9rex2WyDdn3RM0lK4pQuuugiAoEAwWCQCy+8MCnnzMvL69RPFA6H+fTTTzuVMZlMhMPhfl2np3NccsklpKWlsWbNGl577TVpuqUQab6JU9Lr9bHtnfR6fY/lnE5nbEOCDjk5OZSUlHQpe95557F8+XJeffVVxo0bx8MPP0xLS0unMmVlZbzzzjt85zvfwWw296nZWFZWxqZNm9i/fz/p6elkZ2ej0+lifUsrV65k/PjxI3JjgFQlNSURF7vdjt1u77XMhg0bmDlzZqfHPffc023Z66+/niVLlnDttddSWVnJ2LFjWbhwYacy9957L/v372fcuHHk5eX1Ke4VK1ag1+s5/fTTycvL48CBA7Hv3XDDDQQCgUG5myjiJ2t0ixHr3XffZdGiRdTV1XXZBVZoR5KSGHH8fj/Hjh1jyZIlFBYW8swzz2gdkjiBNN/EiPPcc88xZswYWlpaeOCBB7QOR5xEakpCiJQiNSUhREqRpCSESCmSlIQQKUWSkhAipUhSEkKkFElKQoiUIklJCJFSJCkJIVLK/w+AJZTVUodavAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:11:44.577151Z", + "iopub.status.busy": "2024-02-29T06:11:44.576764Z", + "iopub.status.idle": "2024-02-29T06:11:44.943723Z", + "shell.execute_reply": "2024-02-29T06:11:44.942748Z" + }, + "papermill": { + "duration": 0.390949, + "end_time": "2024-02-29T06:11:44.945979", + "exception": false, + "start_time": "2024-02-29T06:11:44.555030", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAEmCAYAAAAk84bZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+00lEQVR4nO3deXxU5b348c+ZfSbJTBLICgkJBFB2EEFwAZWKS63UulRtBXFpb21vLZdWuf1Vq74s2qtWurzoCui9WquttbYuFDdUVGQRREAQyAYEQhIyyexnZs7vj0lGAkmYJJOcSfJ9v17zgkzOnPPNkPnyPM95nu+jaJqmIYQQKcKgdwBCCHEiSUpCiJQiSUkIkVIkKQkhUookJSFESpGkJIRIKZKUhBApRZKSECKlmPQOoCei0SiHDx8mIyMDRVH0DkcIcRJN02hubqawsBCDIbE2UL9OSocPH6aoqEjvMIQQp1FdXc3w4cMTOrZfJ6WMjAwg9gM7nU6doxFCnKypqYmioqL4ZzUR/ToptXbZnE6nJCUhUlhXhldkoFsIkVIkKQkhUookJSFESunXY0qJ0DSNcDhMJBLRO5RBzWw2YzQa9Q5D9AMDOimFQiFqamrw+Xx6hzLoKYrC8OHDSU9P1zsUkeIGbFKKRqOUl5djNBopLCzEYrHIBEudaJrGsWPHOHjwIKNHj5YWk+jUgE1KoVCIaDRKUVERDodD73AGvZycHCoqKlBVdcAmpWhUY/3nx3D7VC48IxeX3ax3SP3SgB/oTnRqu+hdg6GVuqumiW1VjZTXeXlj91G9w+m35BMrRJLsrmmK/72qwUdzQNUxmv5rwHbfhOhLATXCoUY/ANNLshie5cBmHpjd1N4mLaUUNHfuXO666y69wxBdcKw5iKaB027m/NE5lA5Nw2yUj1d3yLvWj7399tsoikJjY6PeoQx6jb5YVy3PadU5kv5Pum9CJMHE4S5G56UTikQJR6KU13k57lM5uyRrUAzyJ9OgbCmFwtEOH+FINOFj1QSO7algMMjdd99NUVERVquVsrIy/vSnP1FRUcGFF14IQFZW7Bd/0aJFpz1fc3MzN910E2lpaRQUFPCLX/zilO7i//7v/zJ9+nQyMjLIz8/nxhtvpLa2Nv791hbaG2+8wfTp03E4HMyePZs9e/b0+Oftz2xmI05bbBrAyztq2LCvDm9IVhJ01aBsKf3mrX0dfq90aBoLpg6Lf/37d/ajRtrf2Xx4lp1rp39RZG7VhnL8J/0S/uBLY3oU680338wHH3zAL3/5SyZPnkx5eTl1dXUUFRXxt7/9ja997Wvs2bMHp9OJ3W4/7fmWLFnChg0beOmll8jLy+Pee+9l69atTJkyJX6Mqqo8+OCDjB07ltraWpYsWcKiRYt45ZVX2pzrxz/+MY899hg5OTl8+9vfZvHixWzYsKFHP+9AYDIacNnNNPpUjntDpFsH5ces2+TdSmF79+7lueeeY926dcybNw+AkSNHxr+fnZ0NQG5uLpmZmac9X3NzM08++STPPPMMF198MQCrV6+msLCwzXGLFy+O/33kyJH88pe/5Oyzz8bj8bRZJvLQQw8xZ84cAO655x6uuOIKAoEANputez9wPxVQI7z8SQ1ZaWYuHJuLoihkOSw0+lQafSpF2XpH2L/onpQOHTrE3XffzauvvorP56OsrIzVq1czffr0XrvmnReWdfg9w0nd/zsuGNXhsScPFSw+t7QnYZ1i27ZtGI3G+Ae/pw4cOICqqsyYMSP+nMvlYuzYsW2O27JlCz/96U/Zvn07x48fJxqNdUOrqqoYN25c/LhJkybF/15QUABAbW0txcXFSYm3vzjuC1HV4OO4z8RFZ8R+KTJssY9Wc1DmKnWVrknp+PHjnHvuuVx44YW8+uqr5OTk8Pnnn5OVldWr17WYEh9K661jE5FIdyzZvF4v8+fPZ/78+Tz99NPk5ORQVVXF/PnzCYVCbY41m79YRtE6mNuawAaT1jtvwyIHYdenMOwsMmyxf7vmQFjP0PolXZPSI488QlFREatXr44/V1qa3NZGfzZx4kSi0Sjr16+Pd99OZLFYABIuyzJy5EjMZjObNm2Kt2bcbjd79+7lggsuAOCzzz6jvr6ehx9+OL4pw+bNm5Px4wxYzYEwdvU4oxv/DVoa1O/DWfKN+PdE1+h69+2ll15i+vTpXHvtteTm5jJ16lT+8Ic/6BlSSikpKWHhwoUsXryYF198kfLyct5++22ee+45AEaMGIGiKPzrX//i2LFjeDyeTs+XkZHBwoUL+eEPf8hbb73Fzp07ufXWWzEYDPGWTnFxMRaLhV/96lccOHCAl156iQcffLDXf9b+zBNUyWvehbW17x8OkuX5HECWmnSDrknpwIEDrFy5ktGjR7N27Vr+4z/+g//8z//kySefbPf4YDBIU1NTm8dAt3LlSq655hq+853vcMYZZ3D77bfj9XoBGDZsGPfffz/33HMPeXl5fPe73z3t+R5//HFmzZrFl7/8ZebNm8e5557LmWeeGR+czsnJYc2aNTz//POMGzeOhx9+mEcffbRXf8b+zhMIkxWoxGwyQGasBZrlr+SqKYVcObnwNK8Wp9B0ZDabtVmzZrV57nvf+552zjnntHv8fffdpwGnPNxu9ynH+v1+bdeuXZrf7++V2AcKj8ejuVwu7Y9//GOvXmcg/3s8/8527f0//pdW//IDmtZ4UNPe/Jmmrf8fTYtE9A5Nd263u8PPaEd0bSkVFBS0uZsDcOaZZ1JVVdXu8cuWLcPtdscf1dXVfRHmgPLxxx/z5z//mf3797N161ZuuukmAK666iqdI+u/zL5YmRKTqwAyCsBohogKvnqdI+ufdB3oPvfcc0+ZBbx3715GjBjR7vFWqxWrVdYWdeTkW/Yn27VrFwCPPvooe/bswWKxcNZZZ/Huu+8ydOjQvgpzwLmqzETElI0yZBgYDJCRD43VHKzazxGHkZE56WSnWfQOs9/QNSn94Ac/YPbs2fzsZz/juuuu46OPPuL3v/89v//97/UMq98qLCxk27ZtnX6/uLiYLVu29F1Qg4GnFqNBgYy82NfpsaRUUVXBJmM2dotRklIX6JqUzj77bP7+97+zbNkyHnjgAUpLS3niiSfiXQrRNSaTibKyjieGil7iaakymd6SlByxKdwZ0WYwcsrSI9E53Wd0f/nLX+bLX/6y3mEI0S3VR+rwVNXgtJkZlpYTe7IlKaVF3GBGFuV20aCsEiBEsjQ11sXWuEWsYGrpojmGxP6IelC0CP6QTKDsCklKQvSA6ondYTM4TlgaZUkHoxmzAtZwM96gtJS6QpKSED0Q8R4HQLGfkJQUBWwuzCYFa8SDT1pKXSJJSYgeiPoaADCmnVSfxObCbDRgDXvwyZhSl+g+0C1Ef6YFGgEwp59U2cLqxGY2cnGpDcOoYae+UHRIWkopKNV2M0m1eFKJ4m8EwHJKS8mJUVHIt4bIzRhcRe96SpLSAHVy7SPRC6JRDGEfANaMU1tKAAQH/qLxZBtcSUnTIBzq+4fWfo3v9ixatIj169ezYsUKFEVBURT279/PrbfeSmlpKXa7nbFjx7JixYpTXrdgwQIeeughCgsL49Uk33//faZMmYLNZmP69Om8+OKLKIrSZub3p59+ymWXXUZ6ejp5eXl885vfpK6ursN4KioqevxPMSCEPJxV5GLGyKFkuTLbfs8WS0q1x2rZXNGA2yclTBI1uMaUIiq8+1jfX/f8//piDstprFixgr179zJhwgQeeOABILZbyfDhw3n++ecZMmQI77//PnfccQcFBQVcd9118de+8cYbOJ1O1q1bB0BTUxNXXnkll19+Oc888wyVlZWndMMaGxu56KKLuO222/jFL36B3+/n7rvv5rrrruPNN99sN56cnJwkvCkDQChWv8poTQfjSbvhtrSUjtbW8q52jEyHGZfDfPIZRDsGV1LqB1wuFxaLBYfDQX5+fvz5+++/P/730tJSPvjgA5577rk2SSktLY0//vGP8YqUv/3tb1EUhT/84Q/YbDbGjRvHoUOHuP322+Ov+fWvf83UqVP52c9+Fn9u1apVFBUVsXfvXsaMGdNuPAIINsf+tGac+j1LbIMFs0HDGA0SUAdfmeDuGlxJyWiOtVr0uG4P/eY3v2HVqlVUVVXh9/sJhUJttkWCWPnc1oQEsGfPHiZNmtRmd5ETNw0A2L59O2+99VabXUpa7d+/nzFjerZF1EBWX19H7ZEmDNFCxp78TaMJzDZMBg+WqB+/KtMCEjW4kpKiJNyNSiXPPvssS5cu5bHHHmPWrFlkZGTwP//zP2zcuLHNcWlpaV0+t8fj4corr+SRRx455XutO5SI9nk9jTT6VCJBy6lJCcCSjslYjzniIyBJKWGDKyn1ExaLpc1mABs2bGD27Nl85zvfiT+3f//+055n7Nix/N///R/BYDBeh2rTpk1tjpk2bRp/+9vfKCkpwWRq/9fh5HhETNgXu7NmtDvbP8CShsmgYI74pfvWBYPr7ls/UVJSwsaNG6moqKCuro7Ro0ezefNm1q5dy969e/nJT35ySnJpz4033kg0GuWOO+5g9+7drF27Nl5vu3WjgDvvvJOGhgZuuOEGNm3axP79+1m7di233HJLPBGdHM9g3EapPRG/GwCjo+OkZDYasES80n3rAklKKWjp0qUYjUbGjRtHTk4O8+fP5+qrr+b6669n5syZ1NfXt2k1dcTpdPLPf/6Tbdu2MWXKFH784x9z7733AsTHmQoLC9mwYQORSIRLLrmEiRMnctddd5GZmYnBYGg3no7KFQ820UBsoNvcWUvJ2NpSkqSUKEXTujCJJsU0NTXhcrlwu904nW1/MQKBAOXl5ZSWlg66baQ78/TTT3PLLbfgdrv7dLPLAffvoWl89vx9HG/2kT7nu0wsKzn1mKoPCe19g8aMMRjGf4Wh6YOvlHNnn9GOyJjSAPfUU08xcuRIhg0bxvbt2+NzkPTYfXdACQeIqLFZ82Z7O1MCACzpWIwGcq1hGIQJqbskKQ1wR44c4d577+XIkSMUFBRw7bXX8tBDD+kdVv8X9MQWCBis2Dtq+Vla7oaGOt8kVLQlSWmA+9GPfsSPfvQjvcMYeFQfE4a5iNqzIcvR/jGWdDQ0jtbVc7CigclFmZiNMox7OvIOCdEdamwhrsGahqF1u+6TWdJQUKg8Usd7e4/ikwqUCRnwSakfj+MPKAPu3yEUS0qYOxmbM9lAUTAbDZiiQQJhSUqJGLBJyWyOLe3w+Xw6RyLgi1IqxpMXrvZTAX8znx1pYmtNJyViDAYw2TAaFEyRgEwLSJCuY0o//elP2yw0hdgs5M8++6zH5zYajWRmZlJbWwuAw+GITxgUfSsajXLs2DEcDkeHs8b7m6DPw3GfSp1XYVpnB7bO6o76CYZl0mkidP8NGT9+PK+//nr862T+0rauam9NTEI/BoOB4uLiAfMfg+qP3VEzWDoY5G5ltmMyGjBHAwRlqUlCdE9KJpOp10piKIpCQUEBubm5qKoU2dKTxWKJzxAfCCIhLwBG22kWQZvt8e5bUMaUEqJ7Uvr8888pLCzEZrMxa9Ysli9fTnFxcbvHBoNBgsFg/OumpsRKjRqNxgEzliFSQyTQkpSsp5Z8acPswGRQMEUD0n1LkK7/dc2cOZM1a9bw2muvsXLlSsrLyzn//PNpbm5u9/jly5fjcrnij6Kioj6OWIiYaLC16uTpW0q5GVbmlKYzYZirDyLr/3RNSpdddhnXXnstkyZNYv78+bzyyis0Njby3HPPtXv8smXLcLvd8Ud1dXUfRywEEI0SCQUAMJ+2++bAYTGRZ4/isks53ETo3n07UWZmJmPGjGHfvn3tft9qtcbrAgmhm7A/Xr7FksCYEgCqv5eDGjhSauTR4/Gwf/9+qXgoUlvIR+nQNGaOGca00qGdH2t2EIpEOVRbx+4a2W4pEbompaVLl7J+/XoqKip4//33+epXv4rRaOSGG27QMywhOte6xKSliFunzA78oQj7Dh3jo/KGPgiu/9O1+3bw4EFuuOEG6uvrycnJ4bzzzuPDDz+ULXxEalMTWGLSymzHZGy5+6aGezeuAULXpPTss8/qeXkhukf1sa+2GU8gxMiAitPWyQB2y5QAgxZBlV2LE5JSY0pC9AshHw3eEBVNoJ1u6pHRjNEUS1pK2E84InOVTkeSkhBdFA56iGgQNtiwWU7zEVIUjFYHCrTM6pakdDqSlIToIrVlNnfY5MCSQNE2xezA2LIoVyoFnJ4kJSG6SPXHkpLBkmDlCbMdk0HBGA1JSykBKTV5Uoj+IBJIcIlJK5ON0pw08kc4yUrrfzs09zVJSkJ00RcVAk5TtqSV2U6m3UJmGmCWheGnI903IboiGiUSjM1TMtlOUyGglalltxM10EtBDSzSUhKiK8J+8jKs5KRbUCeVJPYasx1PMEz90XosuX4KXLLnXmekpSREV7QsrDVY7FgtCY4PmWzUe4LsrDrKvlrZA+50JCkJ0RUt40mYExzkhi+qT0aDUhI3AZKUhOgK1c/B4z521IY4eDzBnXJMNkyyzVLCZExJiK5Qvbj9KlUBsCS6uWTrNkvSUkqItJSE6IqQj3BUI2y0YTMn+PEx21rqdAcJyozu05KkJERXqH4iUQ3VYMeW6JwjU2xMSdGiqGrw9McPcpKUhOgK1Us4EkU12rGZEkxKRjPGlv0MIyHZsfl0ZExJiC6IBL1ENFANNqyJdt8UBastjTF5IYpGZ/ZqfAOBJCUhuqC1QkDEZMdqSryjYbI6GJLmg4zeimzgkO6bEF3QuhjXYE3r2hbkstQkYdJSEiJRmkaaQWVmaTZTzh7Ttdea7TT4QjTU1FGYGcZhkY9eR+SdESJRqh80DYOiYE/rYj/MZKWyzsveQA2OEaokpU5I902IRMV3MbGBoYslSEx2jK27mkiht05JUhIiUSEvDd4gu46p7Dzs7tprzTZMhthSk6AsNelUyiSlhx9+GEVRuOuuu/QORYj2qX68wQgHvQZqm7o4CdIki3ITlRJJadOmTfzud79j0qRJeociRMdUL+FotGtzlFqduNREum+d0j0peTwebrrpJv7whz+QlZWldzhCdEz1n7DuratjSi2LciPSfTsd3ZPSnXfeyRVXXMG8efNOe2wwGKSpqanNQ4g+E/IRjrSse0t0iUmr+I4m0n07Hd237d66dSubNm1K6Pjly5dz//3393JUQnRA9RKOaqhdqRDQymQjO83CeLsJrcjVO/ENELq1lKqrq/n+97/P008/jc1mS+g1y5Ytw+12xx/V1dW9HKUQJ1Bj2253qUJAK5MNh8VEjg1y08y9E98AoVtLacuWLdTW1jJt2rT4c5FIhHfeeYdf//rXBINBjMa2//BWqxWr1drXoQoRE/L2aEwpLhwASxfK6Q4y3UpKBw4cYOTIkT268MUXX8yOHTvaPHfLLbdwxhlncPfdd5+SkITQnerjrBFZjJ8yFpu9i60dgwHVYKGpuZnGw3WMKpGk1JFuJaWysjLmzJnDrbfeyjXXXJNw9+tEGRkZTJgwoc1zaWlpDBky5JTnhdCdpoEawKAoONKcYOjCYtwWAczsPerhgOUgo0pG9EKQA0O3xpS2bt3KpEmTWLJkCfn5+XzrW9/io48+SnZsQqQO1Q9ay10zc4I7457EaIm9Lhr0JyuqAalbSWnKlCmsWLGCw4cPs2rVKmpqajjvvPOYMGECjz/+OMeOHetWMG+//TZPPPFEt14rRK9SffjVCHvqQ3xQ3titU5isLcksEiAckWkBHenR3TeTycTVV1/N888/zyOPPMK+fftYunQpRUVF3HzzzdTU1CQrTiH0pfoIqBEOew0cqOvehpJmix0FMEVDMqu7Ez1KSps3b+Y73/kOBQUFPP744yxdupT9+/ezbt06Dh8+zFVXXZWsOIXQV8suJl2qzX0S5YRNKUOSlDrUrYHuxx9/nNWrV7Nnzx4uv/xynnrqKS6//HIMhliOKy0tZc2aNZSUlCQzViH0o/padjHpxnSAVubW/d+kfElnupWUVq5cyeLFi1m0aBEFBQXtHpObm8uf/vSnHgUnRMpQfYQjUcJGG2ldqM3dxomVAmT9W4e6lZTWrVtHcXFxvGXUStM0qqurKS4uxmKxsHDhwqQEKYTuWrtvPWwpFWU7yHXacabLJOCOdCvljxo1irq6ulOeb2hooLS0tMdBCZFy1JbFuEZH19e9tTLZyXZYKHBAmlXK4XakW++upmntPu/xeLo1kVKIlKf64rWUut1SMrW0jsKyS25nupSulyxZAoCiKNx77704HF9MIotEImzcuJEpU6YkNUAhUkLIy5i8DEaMH4NxSDc3bzPb8asRPMcbiTYFyHPKf+Dt6VJS+vjjj4FYS2nHjh1YLJb49ywWC5MnT2bp0qXJjVCIVKD6WnYxcUK3B7pt1HmCVB93EzjUSJ4zP7kxDhBdSkpvvfUWEFs4u2LFCpxOZ68EJURKiUZjy0yg20tMYq+N3X1T0FBD0oXrSLdS/urVqyUhicEj7EfTonx+zMNbBzzdv51vMGE0xqoLhIO+JAY4sCTcUrr66qtZs2YNTqeTq6++utNjX3jhhR4HJkTKCMUmTh7xKmw72MR5Y3K7dx5FwWCxAW4iQdm+uyMJJyWXyxXfO93lknKeYhBRvS3TAWyYjQpmY/dXZ7VWCoiEvMmKbsBJOCmtXr263b8LMeDFJ046uj8doIXRagcgEpLyJR3pVsr3+/34fF/0iSsrK3niiSf497//nbTAhEgZqg81EkU12rFbepaUTK01lVTpvnWkW0npqquu4qmnngKgsbGRGTNm8Nhjj3HVVVexcuXKpAYohO5C3vjWSvYetpQs9jRGDk3jnOIe3MUb4LpdefL8888H4K9//Sv5+flUVlby1FNP8ctf/jKpAQqhO9WHGo2iGm09Tkpmq508p43ijK6X0x0supWUfD4fGRmxWa3//ve/ufrqqzEYDJxzzjlUVlYmNUAhdNfSUgob7Nh62H2L72oiS0061K2kVFZWxosvvkh1dTVr167lkksuAaC2tlbmL4mBR/VRlGXnq+eM5pzSIT07l8lOU0Clpu44vlA4OfENMN1KSvfeey9Lly6lpKSEmTNnMmvWLCDWapo6dWpSAxRCdyEfiqJgdzh7PNCN2UZ5nZdPKo5Q7wklJ74Bplv1E6655hrOO+88ampqmDx5cvz5iy++mK9+9atJC06IlKC2zClKxgaSppbqkxEp9NaRbhd1yc/PJz+/7YLCGTNm9DggIVJKJAzhEJX1XirKvUwpzcDl6MG222Y7JoOCUQ0SUKUkbnu61X3zer385Cc/Yfbs2ZSVlTFy5Mg2j0StXLmSSZMm4XQ6cTqdzJo1i1dffbU7IQnRO1paSbXeMFsP+Qj1dGuk1paSFpQ63R3oVkvptttuY/369Xzzm9+koKAgvvykq4YPH87DDz/M6NGj0TSNJ598kquuuoqPP/6Y8ePHd+ucQiRVyIeGhh8rKErPx5RMtlhLKRoiGJIxpfZ0Kym9+uqrvPzyy5x77rk9uviVV17Z5uuHHnqIlStX8uGHH0pSEqmhZReTkCG2PMTW3VpKrUw2jC217VVZatKubiWlrKwssrOzkxpIJBLh+eefx+v1xu/mnSwYDBIMfjG/o6mpKakxCHGKkBc1EtswwGIyYOrBYlwADAYMZhvgJxyQ8iXt6dY7/OCDD3Lvvfe2Wf/WXTt27CA9PR2r1cq3v/1t/v73vzNu3Lh2j12+fDkulyv+KCoq6vH1hehUy9ZKqtHe48W4rVyuDEYOTWN8ruX0Bw9C3WopPfbYY+zfv5+8vDxKSkowm9vejdi6dWvC5xo7dizbtm3D7Xbz17/+lYULF7J+/fp2E9OyZcvidcIh1lKSxCR6VciDGk3OurdWGWkZZES9YE/K6QacbiWlBQsWJC0Ai8VCWVkZAGeddRabNm1ixYoV/O53vzvlWKvVitUq+2WJPhTyxltKdksPu26t4ktNpFJAe7qVlO67775kxxEXjUbbjBsJoauQl5x0KwvGnEE4JzmF/lWDFZ9fxVt3nELZO+AU3Z482djYyF//+lf279/PD3/4Q7Kzs9m6dSt5eXkMGzYsoXMsW7aMyy67jOLiYpqbm3nmmWd4++23Wbt2bXfDEiK5gh4URcGWlgFJ2kAygIVdNU0cDR5iwYSknHJA6da7/MknnzBv3jxcLhcVFRXcfvvtZGdn88ILL1BVVRWvtXQ6tbW13HzzzdTU1OByuZg0aRJr167lS1/6UnfCEiL5Qp7Yn5b0pJ3S1FJ9EtWPpmndnuc3UHUrKS1ZsoRFixbx85//PF7CBODyyy/nxhtvTPg8f/rTn7pzeSH6RkSFcJBDjX72Vfg5syhAbhI2kDRbYwXejNHYrO5k3dUbKLo1crdp0ya+9a1vnfL8sGHDOHLkSI+DEiIltLSS6nwRth7y0RxMTqkRk8WOQSE2q1uWmpyiW0nJarW2O3Fx79695OTk9DgoIVJCy44jPmygKKRZkjOm1Loo1xQNEpKkdIpuJaWvfOUrPPDAA6iqCoCiKFRVVXH33Xfzta99LakBCqGboAcNDS+xLluP1721al2UG5XyJe3pVlJ67LHH8Hg85OTk4Pf7mTNnDmVlZWRkZPDQQw8lO0Yh9BHyEolqBJXYwHRaspKS2Y7RYGipqSQtpZN1qz3qcrlYt24dGzZsYPv27Xg8HqZNm8a8efOSHZ8Q+gl5UCMaIWMaVnMS1r21MtkozLSRG1WwpclSk5N1OSlFo1HWrFnDCy+8QEVFBYqiUFpaSn5+vtzeFANLyEMoHJvN7UjmHTKznSFpLSsTLFryzjtAdCn1a5rGV77yFW677TYOHTrExIkTGT9+PJWVlSxatEhK4YqBJeQlHI2iGh04kjRxEgCDCQwtSU6WmpyiS+/0mjVreOedd3jjjTe48MIL23zvzTffZMGCBTz11FPcfPPNSQ1SCF0Em8lOs/C1M89AzUziehBFIYCZoD+A2uhmaL4reeceALrUUvrzn//Mf//3f5+SkAAuuugi7rnnHp5++umkBSeErkJeFBRsaU4ybD2oy92OIz6FXTVN7K46mtTzDgRdSkqffPIJl156aYffv+yyy9i+fXuPgxJCd9EoqC31wpKxi8lJjJbYHb2IVJ88RZeSUkNDA3l5eR1+Py8vj+PHj/c4KCF0F2oGTeOQO8ibB7zUNiV37MfQmpSCkpRO1qWkFIlEMJk6HoYyGo2Ew7LrpxgAgs0AHA2a2X6wKWlLTFqZLLH1bxFVBrpP1qWBbk3TWLRoUYeF1qQOkhgwWpKSh9aJk0m8+wYYWyoFRENSp/tkXXqnFy5ceNpj5M6bGBACTWhoeLRY8kjaEpMWppZKAZqMKZ2iS0lp9erVvRWHEKkl2Ew4ouE3xuoopSdznhJgscfOq6k+mXR8kuS+00IMFMEmQpEoIWMaaVYjRkNyk4bVnk5xtoMip4WoBkbJSXGSlIRoT7CZULg1KSX/Y2K0OBiWaQeHBklOeP1dklYYCjHAtCSloDEt6V034Iu5T6oMdJ9MWkpCnCwahZCHXKeVm2dMJGxK/uRJzHb8aoSQvxljIES6TaoFtJKWkhAna5k4qRhM2BzO3mkpmeyU1/vYVdNE9dH65J+/H5OkJMTJWuYoYUmH3rorZjBgMMemG4T8nt65Rj+la1Javnw5Z599NhkZGeTm5rJgwQL27NmjZ0hCxJPS7uPw5mdHcfvVXrmMoWVWdzjg7ZXz91e6JqX169dz55138uGHH7Ju3TpUVeWSSy7B65V/JKGjQGxTjEqvke3VbsKR3ilZa2iZQKkGpKV0Il0Hul977bU2X69Zs4bc3Fy2bNnCBRdcoFNUYtALuIlENTzEBrjTbb3zMTFaW1tKcgfuRCk1puR2uwHIzs7WORIxqAUaCUWiBE0ZWEwGrKbe2SzSbIvN6g4HpWdwopSZEhCNRrnrrrs499xzmTCh/Q3Wg8Fgm0W/7e09J0SP+RsJqBGCpgyc9uQWdzuRyRpriUVDkpROlDItpTvvvJNPP/2UZ599tsNjli9fjsvlij+Kior6MEIxKGgaBNwEw1ECJifOXuq6AWQ4nRRnOxibLdt2nyglktJ3v/td/vWvf/HWW28xfPjwDo9btmwZbrc7/qiuru7DKMWgEPJANEwgHNtaydWLLaW0NCfDMu0M64W5mf2Zrt03TdP43ve+x9///nfefvttSktLOz3earV2WMtJiKTwN8b+MDjQFGOvdt9omaeEKt23E+malO68806eeeYZ/vGPf5CRkcGRI0eA2GaXdrtdz9DEYBVoBOCMkiJKJ4zq3WuZHXhDYYKNbtLDUSymlOi46E7Xd2HlypW43W7mzp1LQUFB/PGXv/xFz7DEYBaI3QFW7FnYzEZsydyE8mQWB5/VNPFZ9VEafVK1tZXu3TchUkpL9w17Zu9fy+zAZDQQikQI+P3glN4BpMhAtxApI9CIX43wZkWQDfvqevdaRjMGU2zMKiSzuuMkKQlxIn8j/lCE/c0mKut7f6a10lJXKeSXwe5WkpSEaBVRIdiMX40QMLnIcvTinbcWrYtyQ/7mXr9WfyFJSYhWvgYAvFETYaONrLTeL7xmtGUAEJakFCdJSYhW/lhSasQJQJaj95OSyR5LSqokpbiUWfsmhO58sQqQ9dHYQtmstN7vvmVnZmHLdmB09vql+g1JSkK08tWjRqI0GfqupZSVlUVWph1sst19K+m+CdHK10AoEiViyyLDZsJs7IOPhyXWfUMqBcRJS0kIiFUH8NWTZjFxw7lTCFr7pqaXZnHgC4UJHm/AFdWSvullfyRJSQiIVQeIqKAYUBzZ2Ax9VE7Eksanh9xEaGZCUMVpl62WpPsmBIC3Zfa2PRP6KiEBiiUDs9GAokXx+6QLB5KUhIjxHkND470aeGn7YbzBPhp4NppQWkqY+L1SSRUkKQkR4z1GKBylJuyk/JgXax+WETFYY1MQgl53n10zlUlSEgLAU4s3FMFnziY73YKpL+68tTC0bCAQ8skESpCkJAREo+CtwxcM4zVnk5Pet9VNTbbYvCiZ1R0jSUkI/3GIhmkOKwRNTnIy+jYpGe0tWy35ZUwJZEqAEOCtRUPjWCQDFIV8l61PLz80ewhp2Q7MGb2zE29/I0lJCE8tQTXKcSUTo0Eht49bStlDciDTDmYpiQvSfRMCvMeIRDXs2QXkO219s7zkRNaW1bhB6b6BtJTEYKdp0HSYNKuJS6ZORnN1vO9gr7Fm4A2FCfmPk66qmM29X50glUlLSQxuwebYYljFABn5KIoOa88saXxa4+Gzmiaa3cf7/vopRpKSGNyaa4hoGqptCBh1aqEoCoaWaQG+ZklKuiald955hyuvvJLCwkIUReHFF1/UMxwxGDUdxu0L8UolvLqjRrcwjHYXAH5JSvomJa/Xy+TJk/nNb36jZxhiMGuuwe1X8Vhye3fjydNoTUpBb6NuMaQKXQe6L7vsMi677DI9QxCDmaa1JKUwniFDKcrWbzNIc1omAKokpf519y0YDBIMfjGXo6lJbqGKHvDVEwz48YYV/JZshmc5dAvFkhZrKYVbd+gdxPrVQPfy5ctxuVzxR1FRkd4hif7MXU2jT8VjySHf5dC1++bIiFW6jPjkP9p+lZSWLVuG2+2OP6qrq/UOSfRnjVUc94VoshYwKjdd11Cc2TkUZzsoy1Bj3cpBrF9136xWK1Zr3y4BEAOUphFuqMTtV2nKKWDk0DRdw0l3DiE9yxFLSCEPWDN0jUdP/aqlJETSBBpRgh5KcpyUlI4muw92w+2U0fTFchP/4J4WoGtLyePxsG/fvvjX5eXlbNu2jezsbIqLi3WMTAx4jVUYDQp5w0rIm6DD0pJ2+IxOAr5jGBuO4cocvL//uialzZs3c+GFF8a/XrJkCQALFy5kzZo1OkUlBoXGqtifKfTh391oJHSkmcysGlwj9Y5GP7ompblz56IN8kE9oQNNo/HQXnweP07HMPQd4v6C1TmEEBBqrtM7FF3JmJIYfDxHqa2r50CDysfuVElJ4HANBUD1NOgcib761d03MTAF1Ag17gCRaJScdBsuR+8ujA3Vfs5xXwi3rYSzC7J69VpdkZaZC4DmrY/dhdOjYkEKkKQkdKNGomzYV8eOg27C0S+68SOGOJg7NrfX7ojVVe4mqkE0e2Sf1+PujDMrF00xEFaDqL5GzGmpkzD7knTfhC40TeNvWw7ycVUj4ahGlsNMntOGokBlvY8Xth4kHOmFmtUhH8ePlANQUHKmPvWTOmCzmlGtsUTUXKdfxQK9SUtJ6EJRFM4fk8O6nUeYMzaXkiEOFEXB7VNZt/soZ5dk9crea00Hd9LkV/FZhjCjJDWmArRSFAVTRi4E6mluOEz2iHF6h6QLSUpCN8My7SycXdKmteJymPnatGG91oKp+fxjACwF48iwpV7Z2RHFJdiUatKMg3cPOOm+iT61uaKBOs8XlR7aSz4nPnesOcirO2qS05ULebF4qjEoMGzMtJ6frxcMH17M0HQr9mC93qHoRpKS6DPldV7e/byOv2yqxhMMn/b4SFTjH9sO8dmRZt7bl4S5O7WfUZrtYNr4MxkxPLW6bnFpObE/vfUQOf17NBBJ9030iYAa4Y3dR0HTmJoVJP3IR+CphYAbwkGIBEFr2xoymh1cabCx3m1hz75iynLTu1/zSNOgJtZ1MxdOAkPqDHCfSLM6Oa6aCPiaGdp4GMuQ1Jlx3lckKYk+8d7eY5gb9jLTv52ZpggcTyAphHzkAWdFPdQe2UT56xvJ+9K1mDOHdfn6x2vKsTTUkGa3Qf6Erv8AfUQxGNjmdmBvOorxSCV5kpSESL7qI3WEtj/HGH8V4wqcGE12yCoF5zBwZIPJBiYrKCcWWWsp4eFroCjrAI3bt0DTQQ698VtKJs+BkReCIfGibPu2vUO4xk3O6NGUmfUre5sI25Dh0FSO51gleZyvdzh9TpLSAFXj9rO54jgZNhOjctIZnmXXZU5OqOkYh99YSVagiTyXA9eZc2H4DDDbEnh1LmSPxDJ8OkNyL2DHOy+iNO5jyOfvk9FUA+MXJFR36NjRQ4RrdqIA2WNm9/An6n0ZuSPwlL+Lv35wFjGUpNTfBZvh4Cao3w/+xljrIaMAS8YY9h1NA8XAx1WNFGU7+NKZeb2+hKON5iPUrl9FNNCEZs9i2LzFkFnQrVOVFObz2YQFfLb/E4Z4PyDDfRC2PAkTr4GM/E5fW7H5NUDDVngG2fmp3x3Kzi+hGYVgcwOa/ziKfXDN7Ja7b/2VpsHBzfDhb6FqI0drDlJV1xQbND5eQWblWq7hdaZlhzAZFKobfDz9USVV9b6+ic/XAJ/8hWFpUFBUyvCL7sDazYTUau7YHCZNPZtRl34XHENiCfnj/4Xazzp8Tc2BTwkf2Y2iKJRMm9+j6/eV3CwnPlsuoYiGp2bf6V8wwEhS6o8iKnz6N/h8HUTDHFAzWcs5/MuxgEOjb4TSCzCabRSZm5njfZVFZV4KM20E1Sj/2HaIA8c8vRtfyAufPAchH4ozn5KLbqM4P6fHp7WZjUwrzsKQPgSm3QzZI2O3zXf+HSreO6W2ddh7nIMf/BUAe+k5ZOb1j40mzEYDxqGjAKiv7jjhDlSSlPob1Q/b/wx1n4PBxF7XLP7BRdQ7RjF1TAmFw4qh5FyY+a34hzbjwKt8LbeGstx0wlGNHYfcvVfHKhyCHc/jbqglYnXCxOtig9jJvozBwjrTBTQOmRJ7ovzdWKJujk074Hgl5a//nnDAQ9g+lLGzLk96DL0pa/hYAAK1ByAa0TmaviVjSv1JyBtLSJ5jYLLyed6lvFxpAgVmjsxm1qghXxxrSYOJ18KBN6F6E6YDb3J5yflsc53J5KKs3hn0jkZh90s011axozbIQcdsrlLs9Ma9rg376/n0cDOVtrFcO2Iorqo3Y4m67vNYyQ9NY6gpxFFbJkUXfBObLbXvuJ1s1MgytCMFZChBaCiHoWV6h9RnpKXUX4S8sO2ZWEKypFFR/FVeqYoNWk8pzmTWyCGnvsZggFEXQ2nstrKx4l3Oiu7A1DJxUNM0Gn2h5MSnafD5WsK1e9h7LMDuofPJHJqPzdw7v2IzS7MZkm6hORDmmaps9gy/hujQsYQxoEWjYDSRVTaTs6/+PiXDuz6vSW8ZdivO4smx/zyO7tA7nD4lLaX+IOiJtZC8dWBNx3PGtfzz42aimsaZBU7mjsnpuOWjKFByHhgtsO8NqPwAIiraqIt5b38926sbWTB1WM93h63cgHboY/Yd87LDNRdjZhEXnpHba9MQbGYj15w1nH9sO8wRd4BXysFkmIgWPZMrznQxqjAPDEZSp1pSN+SNh4ObiB77HEPIBxb9dvDtS9JSSnWBplgLyVsXm5Mz5SbSs/OZOzaXstx0LhmXl9gHv2gGjLkk9veDm4nueY265gBqROMf2w5zuNHf/RgPbYHyd6lq8LHVcjbN6aVcPrEAq6l3d5x1WExce9Zwzi0bSprVSDiqEcHIfrehSxMrU1U0LY89XgdbK47h2f+h3uH0GUXrx5X7m5qacLlcuN1unE6n3uEkn6c2dhcr2AzWDLQpN6I4suPf1jSt6y2Rmk9gzyuxzRhzxvGSfzKVjSEsJgNfmzacfFcikxpPUL0Jbd86Dh73szFyBtWZZ3P5xALG5vftZorRqEZzIIzBAOlWU0oVb+uJ19e/Tdq+fzEk08mYK5f2u9ZSdz6j0lJKVbW7Y3Nwgs3gGMK+4Qv48w4P/tAXd2K69cErmARnfgUUA6Zju/gKb1KSrhIKR3nh44McSrTFFI3EpiTse51IVGOHaTzVrumcP3ponyckAINBweUwk2EzD5iEBDB+8gx85mwaGps4/smreofTJ1IiKf3mN7+hpKQEm83GzJkz+eijj/QOST/BZtj9T9j5IoRDqBnDeMsxn3/u8XG0KcDmyiTsdJE3DiZdC2YbJs9Rrgy9ykRtD6GQyvObq/nkYGPnr/fUxhLmwc0AmEbN5dyLr2beuHyml2R3/lrRJQWZDgxjL0NDoWLXRgIVG/UOqdfp3n37y1/+ws0338xvf/tbZs6cyRNPPMHzzz/Pnj17yM3N7fS1A6r75j8Ohz+GQ1shoqJGNcodk3g7OAavGjvk7JJsZo8agiFZZTf8jbD7JXAfIhyN8rnbwCfaKM6ZMStWb0hRCKgRLEYDBqJwvILwoW00H9xJXXMAi83BiNnXQs7Y5MQj2hVQI7z+6gsMObYRh8VI6Vlfwjl2bmyr7xTXnc+o7klp5syZnH322fz6178GIBqNUlRUxPe+9z3uueeeTl/bL5NSJAxhf+yOmq8OzVNLuK4ck78WhViy2e1N4wPDNNzmWFLOTrNw4dhciof0wniCpkHNNih/B0I+3H4Vp92EYnaAzcWOGg/17mbSo00o0ShqJIoG1DtGUp09m2/MGZ+SZWUHmgZPkA9e/yvZ9R9jNRmYUjYcQ/5EtMwRRO3ZGO2ulNySqTufUV1TbSgUYsuWLSxbtiz+nMFgYN68eXzwwQenHB8MBgkGvyil2tTUlNiFGsph3+unPh/Px1/k5f21zTR4g18803qMpgEa00ZkYWr5xy+v81DbHIy9/osXAKCgMaUoE4sxdmxlvY+jjV4Mmtrm8pGohgZMG5GNNacUhp1Fbb0Ld1UjQ9MtTCnKYlyhE2NvFSVTFCicCnkToXYnrqM7wX0wNnNc9YPbjTUYRgVUo52G9BK8QyaRV1jM14a7JCH1kex0K3Muu56NHw6jyLsJQ8gLVR+ilr/Plsrjsd8PowXNaI3NT0MBRaHAZad4SGzDTX84yicH3R1eI89po6TlP75QWMNgID6nrV0TrwNb8hsDuialuro6IpEIeXl5bZ7Py8vjs89OXfOzfPly7r///q5fKByM3VJPQNTXTNjbyYTCgLHlHx2iAQ/RQLDjY1UrRFuOVf1EwiEigIZC2GjDb87Eb86i2ZpH2bRZ5GTHxmOmOFQmDHORnWbpu0FbowkKJsce4VCsOxlwM35clGYVgpZMNKuLDLsZu9k4oAaT+4t0q4mL58xFi5wbm7le/zneo9VoNBKJahANgnrS76PZDrYAAIoaIdrc2OH5NYMNrGmxLyJRUIj/rrf/gl7YAot+Nnly2bJlLFmyJP51U1MTRUUJLLLMLILJXz+pedvy9/hzsT+H+EI4QtGWp084puWvBqe95WuFof4Q6eHWlpESP1RTYl8Z02zxsqs5oQgZqgYWOxit8Q+1xWTAZjZiPmE7oT4tL9IekwUy8iAjDwPg0jcacRLFaI7drMgbR+aZGlNmhVADPkJBP9FwMDajHQ0tGiXNYgKbEbQoxqhGcVnH/4mmWY1gj/3uGaIahhN+79tlSUvuD9ZC16Q0dOhQjEYjR48ebfP80aNHyc8/tUaO1WrFau3GHF1LGmSXJnRodmbip3VlJP6BTU+D1Nm1XgwUiqLgsFnBZgU6r7tkBoYNTey8eiYGXacEWCwWzjrrLN544434c9FolDfeeINZs2bpGJkQQi+6d9+WLFnCwoULmT59OjNmzOCJJ57A6/Vyyy236B2aEEIHuiel66+/nmPHjnHvvfdy5MgRpkyZwmuvvXbK4LcQYnDQfZ5ST/TLeUpCDCKy9k0I0e9JUhJCpBRJSkKIlKL7QHdPtA6HJbzcRAjRp1o/m10Zuu7XSam5uRkgsVndQgjdNDc343IlNtW4X999i0ajHD58mIyMjEG7Fqt1qU11dbXcgewGef965nTvn6ZpNDc3U1hYiKGzdXQn6NctJYPBwPDhw/UOIyU4nU75UPWAvH8909n7l2gLqZUMdAshUookJSFESpGk1M9ZrVbuu+++7lVPEPL+9VBvvH/9eqBbCDHwSEtJCJFSJCkJIVKKJCUhREqRpCSESCmSlPqhhoYGbrrpJpxOJ5mZmdx66614PJ5OXzN37lwURWnz+Pa3v91HEeurqzswP//885xxxhnYbDYmTpzIK6+80keRpqauvH9r1qw55ffMZrN17YKa6HcuvfRSbfLkydqHH36ovfvuu1pZWZl2ww03dPqaOXPmaLfffrtWU1MTf7jd7j6KWD/PPvusZrFYtFWrVmk7d+7Ubr/9di0zM1M7evRou8dv2LBBMxqN2s9//nNt165d2v/7f/9PM5vN2o4dO/o48tTQ1fdv9erVmtPpbPN7duTIkS5dU5JSP7Nr1y4N0DZt2hR/7tVXX9UURdEOHTrU4evmzJmjff/73++DCFPLjBkztDvvvDP+dSQS0QoLC7Xly5e3e/x1112nXXHFFW2emzlzpvatb32rV+NMVV19/1avXq25XK4eXVO6b/3MBx98QGZmJtOnT48/N2/ePAwGAxs3buz0tU8//TRDhw5lwoQJLFu2DJ/P19vh6qp1B+Z58+bFn+tsB2aIvb8nHg8wf/78Do8fyLrz/gF4PB5GjBhBUVERV111FTt37uzSdfv1gtzB6MiRI+Tm5rZ5zmQykZ2dzZEjRzp83Y033siIESMoLCzkk08+4e6772bPnj288MILvR2ybrq6AzPE3t/2ju/svR2ouvP+jR07llWrVjFp0iTcbjePPvoos2fPZufOnQkvnpeklCLuueceHnnkkU6P2b17d7fPf8cdd8T/PnHiRAoKCrj44ovZv38/o0aN6vZ5hTjRrFmz2uzZOHv2bM4880x+97vf8eCDDyZ0DklKKeK//uu/WLRoUafHjBw5kvz8fGpra9s8Hw6HaWhoaHdX4Y7MnDkTgH379g3YpNTVHZgB8vPzu3T8QNad9+9kZrOZqVOnsm/fvoSvK2NKKSInJ4czzjij04fFYmHWrFk0NjayZcuW+GvffPNNotFoPNEkYtu2bQAUFBQk+0dJGd3ZgXnWrFltjgdYt27doNyxORk7WEciEXbs2NG137MeDZMLXVx66aXa1KlTtY0bN2rvvfeeNnr06DZTAg4ePKiNHTtW27hxo6ZpmrZv3z7tgQce0DZv3qyVl5dr//jHP7SRI0dqF1xwgV4/Qp959tlnNavVqq1Zs0bbtWuXdscdd2iZmZnx29Tf/OY3tXvuuSd+/IYNGzSTyaQ9+uij2u7du7X77rtv0E8J6Mr7d//992tr167V9u/fr23ZskX7+te/rtlsNm3nzp0JX1OSUj9UX1+v3XDDDVp6errmdDq1W265RWtubo5/v7y8XAO0t956S9M0TauqqtIuuOACLTs7W7NarVpZWZn2wx/+cFDMU9I0TfvVr36lFRcXaxaLRZsxY4b24Ycfxr83Z84cbeHChW2Of+6557QxY8ZoFotFGz9+vPbyyy/3ccSppSvv31133RU/Ni8vT7v88su1rVu3dul6UrpECJFSZExJCJFSJCkJIVKKJCUhREqRpCSESCmSlIQQKUWSkhAipUhSEkKkFElKol9Zs2YNmZmZ8a9/+tOfMmXKlPjXixYtYsGCBX0el0geSUqiXYsWLeqwZO6dd96JoihtFhD3RjIoKSnhiSeeaPPc9ddfz969ezt8zYoVK1izZk3867lz53LXXXclNS7RuyQpiQ4VFRXx7LPP4vf7488FAgGeeeYZiouLdYnJbrefUk/qRC6Xq01LSvQ/kpREh6ZNm0ZRUVGbQnAvvPACxcXFTJ06tUfnbq8Fs2DBgnjra+7cuVRWVvKDH/wgXoAeTu2+nezEFtuiRYtYv349K1asiJ+jvLycsrIyHn300Tav27ZtG4qidKnEhugdkpREpxYvXszq1avjX69atYpbbrml16/7wgsvMHz4cB544AFqamqoqanp8jlWrFjBrFmzuP322+PnKC4uPuVnAli9ejUXXHABZWVlyfoRRDdJUhKd+sY3vsF7771HZWUllZWVbNiwgW984xu9ft3s7GyMRiMZGRnk5+d3q8iay+XCYrHgcDji5zAajSxatIg9e/bEtwpSVZVnnnmGxYsXJ/vHEN0glSdFp3JycrjiiitYs2YNmqZxxRVXMHToUL3D6pHCwkKuuOIKVq1axYwZM/jnP/9JMBjk2muv1Ts0gbSURAIWL17MmjVrePLJJ5PWmjAYDJxcNUdV1aScOxG33XZbfBB/9erVXH/99Tgcjj67vuiYJCVxWpdeeimhUAhVVZk/f35SzpmTk9NmnCgSifDpp5+2OcZisRCJRHp0nY7Ocfnll5OWlsbKlSt57bXXpOuWQqT7Jk7LaDTGd1IxGo0dHud2u+O1v1sNGTKEoqKiU4696KKLWLJkCS+//DKjRo3i8ccfp7Gxsc0xJSUlvPPOO3z961/HarV2q9tYUlLCxo0bqaioID09nezsbAwGQ3xsadmyZYwePXpQ1uBOVdJSEglxOp04nc5Oj3n77beZOnVqm8f999/f7rGLFy9m4cKF3HzzzcyZM4eRI0dy4YUXtjnmgQceoKKiglGjRpGTk9OtuJcuXYrRaGTcuHHk5ORQVVUV/96tt95KKBTqk7uJInFSDlcMWu+++y4XX3wx1dXVp2y4KPQjSUkMOsFgkGPHjrFw4ULy8/N5+umn9Q5JnEC6b2LQ+fOf/8yIESNobGzk5z//ud7hiJNIS0kIkVKkpSSESCmSlIQQKUWSkhAipUhSEkKkFElKQoiUIklJCJFSJCkJIVKKJCUhREqRpCSESCn/H1Ys+gv+m40sAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:11:44.989990Z", + "iopub.status.busy": "2024-02-29T06:11:44.989575Z", + "iopub.status.idle": "2024-02-29T06:11:45.232029Z", + "shell.execute_reply": "2024-02-29T06:11:45.231018Z" + }, + "papermill": { + "duration": 0.267205, + "end_time": "2024-02-29T06:11:45.234311", + "exception": false, + "start_time": "2024-02-29T06:11:44.967106", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEmCAYAAAAdlDeCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzEklEQVR4nO3de1hTV7o/8G/CJYAQEAkELMrNiv6q0oIillYpd2qPF7RaGUVLdaYtvSG26jnVcXrmWFuwWseR40zVsdVqW5F2VNBUQRm1iHipUsSKUi0QUREiICHA+v3hyR5iuCQ1kMt+P8/jY/baa++8m528Wfu2loAxxkAIIRZOaOwACCGkP1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwgrWxAzB1HR0dqK6uhpOTEwQCgbHDIYR0whjDvXv34OXlBaGw57YbJbteVFdXw9vb29hhEEJ6cOPGDTz22GM91qFk1wsnJycAD/6YYrHYyNH0HZVKhUOHDiEmJgY2NjbGDoc8Ir7sT4VCAW9vb+572hNKdr1QH7qKxWKLT3YODg4Qi8UW/eXgC77tT11OMdEFCkIIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8ALdekKIBWhubsalS5e46cb7Spy4UIGBbqfhaC/SqBsYGAgHB4f+DtHoKNkRYgEuXbqE4OBgrfKPuqhbUlKCp556qu+DMjGU7AixAIGBgSgpKeGmy2vqkfb1BaydOQrDPV206vIRJTtCLICDg4NGa034yx2ICu9jxBNjEDR0kBEjMx10gYIQwguU7AghvECHsTxx7XYTmpRt3PT9+824duUyN93e3o5zFyrQbF0IKysrrtw34HHY2//7yt0AkTV83Qb0T9CkRw/v084qbjVx/1tbd/8159P+pGTHA9duNyEio0CjTCm/Avk/3u51WWnyOoikARpl+emTePMFMVVd7dOuLP7mQq91+LI/KdnxQF1zI4R2VUiPfhzerg9aaUqlE6oSP+fqdHS0o/xSOYYHDodQ+O+W3WDvIRCJ7AAAN+qakSG7jLrmRvjC8r8cpkzdols3KwgB7o7a8+8rsa/gJCZPCsOAh+6zU7tS24i3d5/rtnVoaSjZ8UB10y8Y4LsBm670UlEKHK//p2ZZnebkAF+guikIwfAwaIzktwlwd8QTg521ylUqFeQS4KmhA3nRn50uKNnxgNeAoWi69gbWzwqCfxetAABoa2vD8X8dx9PhT3d7jqeithFv7T4Hr4ihfRkuIX2Ckh0PiKzs0NEyGL7i4Rg5SLsVADxoCVyzvoYRriO6bQl0tDSgo+UWRFZ2fRku0YGyvQVCuypcU5RDaKf9A9bW1obqtmqU1ZV1++N1TfHg9IayvQVA158LS0LJjhAzpD41sfxUz/X+mvfXHufz6bSE2SW7jRs34uOPP4ZcLseYMWOwYcMGjBs3rsu6paWlWLFiBUpKSvDLL7/gk08+wdtvv92/ARPSB3o7NUGnJbSZVbLbvXs30tLSkJWVhdDQUKxbtw6xsbEoLy+Hu7u7Vv3m5mb4+flh5syZeOedd4wQMSF9o7dTE3RaQptZPUGxdu1aLFy4EAsWLMDIkSORlZUFBwcHbNmypcv6Y8eOxccff4zZs2dDJOr68jshhB/MJtm1traipKQEUVFRXJlQKERUVBROnjxpxMgIIebAbA5jb9++jfb2dnh4aJ5I9fDw0Oi08FEplUoolUpuWqFQAHhwWKBSqQz2Pv2pra2N+7+7bVCX97SNuqyH9I/e9gVf9qc+cZtNsusvq1evxqpVq7TKDx06ZLa9u95oBABr/Otf/8IvXd9mx5HJZAZZD+lbuu4LS9+fzc3NOtc1m2Tn5uYGKysr3Lx5U6P85s2bkEqlBnufZcuWIS0tjZtWKBTw9vZGTEwMxGKxwd6nP5VWK5Bx4QeEh4fj/3l1vQ0qlQoymQzR0dHdntDWZT2kf/S2L/iyP9VHXrowm2Rna2uL4OBgHD58GFOnTgUAdHR04PDhw0hNTTXY+4hEoi4vZtjY2JjtYzfqWw+sra173YaetlOf9ZC+peu+sPT9qU/cZpPsACAtLQ3JyckICQnBuHHjsG7dOjQ1NWHBggUAgHnz5mHw4MFYvXo1gAcXNX766SfudVVVFc6dOwdHR0cEBAR0+z6EEMtjVslu1qxZuHXrFlasWAG5XI6goCDk5eVxFy2uX78OofDfF5irq6vx5JNPctMZGRnIyMjAxIkTUVBQ0N/hE0KMyKySHQCkpqZ2e9j6cALz8fEBY6wfoiKEmDqzuc+OEEIeBSU7QggvULIjhPCC2Z2zI/q7r2oHAFysaui2TtN9JU7fAqS/3O2xG29CzBUlOx6o+L8ktTS7t8FXrPH5leJe1zdARB8bY+vtB4x+vLTRp5YHYv7fgydM/N0dYW9j1WWd8poGLP7mAjJnjMJwz+57reXT0HumTLcfMPrx6owfW8lzrgNsMXvckB7rqB8K95cM6HIAF2JaevsBox8vbZTsCDFDvf2A0Y+XNroaSwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBbNLdhs3boSPjw/s7OwQGhqKU6dO9Vj/66+/RmBgIOzs7DBq1CgcOHCgnyIlhJgSs0p2u3fvRlpaGlauXIkzZ85gzJgxiI2NRW1tbZf1T5w4gZdeegkpKSk4e/Yspk6diqlTp+LixYv9HDkhxNgEzIwGVg0NDcXYsWPxl7/8BQDQ0dEBb29vvPHGG1i6dKlW/VmzZqGpqQn79u3jysaPH4+goCBkZWXp9J4KhQLOzs5oaGiAWCw2zIaYoHO/3MHUTT8g59XxCBo6yNjhED01Nzfj0qVL3HR5TT3Svr6AtTNHYbini0bdwMBAODg49HOEfUOf76fZdN7Z2tqKkpISLFu2jCsTCoWIiorCyZMnu1zm5MmTSEtL0yiLjY1FTk5Ot++jVCqhVCq5aYVCAQBQqVRQqVSPsAWmTd3ZY1tbm0Vvp6W6ePEiQkNDtcrn/EO7blFREZ588sl+iKrv6fNZNZtkd/v2bbS3t8PDw0Oj3MPDQ+MXrTO5XN5lfblc3u37rF69GqtWrdIqP3TokMX8GnblRiMAWOOHH35AFR3lmx2lUonMzExuWtUB1LUArnaAzUMnqyorK1FTU9PPEfaN5uZmneuaTbLrL8uWLdNoDSoUCnh7eyMmJsaiD2PPX68DLpzG+PHjMWaIq7HDIY9IpVJBJpMhOjoaNjY2xg6nz6iPvHRhNsnOzc0NVlZWuHnzpkb5zZs3IZVKu1xGKpXqVR8ARCIRRCLtoedsbGws+kNjbW3N/W/J28k3lv651WfbzOZqrK2tLYKDg3H48GGurKOjA4cPH0ZYWFiXy4SFhWnUBwCZTNZtfUKI5TKblh0ApKWlITk5GSEhIRg3bhzWrVuHpqYmLFiwAAAwb948DB48GKtXrwYAvPXWW5g4cSIyMzPx/PPPY9euXTh9+jQ2b95szM0ghBiBWSW7WbNm4datW1ixYgXkcjmCgoKQl5fHXYS4fv06hMJ/N1YnTJiAnTt34r/+67+wfPlyDBs2DDk5OXjiiSeMtQmEECMxq2QHAKmpqUhNTe1yXkFBgVbZzJkzMXPmzD6OihBi6szmnB0hhDwKs2vZEcPo6o57pfwKyi7ao+OOC1duSXfbE36jZMdTly5dQnBwsFb5w3fcl5SU4KmnnuqnqAjpO5TseCowMBAlJSXcdON9Jfbnn8TzEWFwtBdp1CPEElCy4ykHBweNFptKpcLd27UIGxdi0TehEv6iCxSEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QW9k93Vq1f7Ig5CCOlTeie7gIAARERE4IsvvkBLS0tfxEQIIQand7I7c+YMRo8ejbS0NEilUvz+97/HqVOn+iI2DXV1dUhKSoJYLIaLiwtSUlLQ2NjY4zKbN2/GpEmTIBaLIRAIUF9f3+dxEkJMk97JLigoCOvXr0d1dTW2bNmCmpoahIeH44knnsDatWtx69atvogTSUlJKC0thUwmw759+3Ds2DEsWrSox2Wam5sRFxeH5cuX90lMhBAzwh5RS0sLW7t2LROJREwgEDCRSMTmzp3LqqurH3XVnJ9++okBYMXFxVxZbm4uEwgErKqqqtfl8/PzGQB29+5dvd+7oaGBAWANDQ16L2tOWltbWU5ODmttbTV2KMQA+LI/9fl+/uYxKE6fPo0tW7Zg165dGDBgANLT05GSkoJff/0Vq1atwpQpUwx2eHvy5Em4uLggJCSEK4uKioJQKERRURGmTZtmkPcBAKVSCaVSyU0rFAoAD8ZoUKlUBnsfU6PeNkveRj7hy/7UZ/v0TnZr167F1q1bUV5ejoSEBGzfvh0JCQkQCh8cEfv6+mLbtm3w8fHRd9XdksvlcHd31yiztraGq6sr5HK5wd4HAFavXo1Vq1ZplR86dIgX46fKZDJjh0AMyNL3Z3Nzs8519U52mzZtwssvv4z58+fD09Ozyzru7u747LPPel3X0qVLsWbNmh7rlJWV6RviI1m2bBnS0tK4aYVCAW9vb8TExEAsFvdrLP1JpVJBJpMhOjqaRhezAHzZn+ojL13onexkMhmGDBnCteTUGGO4ceMGhgwZAltbWyQnJ/e6rsWLF2P+/Pk91vHz84NUKkVtba1GeVtbG+rq6iCVSvXdhB6JRCKIRCKtchsbG4v+0KjxZTv5wtL3pz7bpney8/f3R01NjdZhZV1dHXx9fdHe3q7zuiQSCSQSSa/1wsLCUF9fj5KSEm4U+yNHjqCjowOhoaH6bQAhhJf0vvWEMdZleWNjI+zs7B45oK6MGDECcXFxWLhwIU6dOoXjx48jNTUVs2fPhpeXFwCgqqoKgYGBGhdF5HI5zp07hytXrgAALly4gHPnzqGurq5P4iSEmC6dW3bq81gCgQArVqzQOFnf3t6OoqIiBAUFGTxAtR07diA1NRWRkZEQCoVITEzEp59+ys1XqVQoLy/XOGGZlZWlcbHh2WefBQBs3bq118NnQszV/fv3kZaWhh9++AF5eXlYu3Yt7O3tjR2W0QlYd021h0RERAAAjh49irCwMNja2nLzbG1t4ePjg/T0dAwbNqxvIjUShUIBZ2dnNDQ0WPwFigMHDiAhIcGiz/FYuqlTp+Lbb7/VKp8yZQpycnL6P6A+ps/3U+eWXX5+PgBgwYIFWL9+vUV/8QkxR90lOgD49ttvMXXqVItMeLrSuWXHV3xo2bW3tyM/Px+5ubmIj49HREQErKysjB0W0cP9+/d1ug+0ubnZog5pDd6ymz59OrZt2waxWIzp06f3WDc7O1v3SInRZWdnY/HixaisrATw4KZxHx8fZGZm9rqviel44403dK7397//vY+jMU06XY11dnaGQCDgXvf0j5iP7OxszJgxQ+spFLlcjhkzZtAPlxn55ptvNKZjY2Px4YcfIjY2tsd6fEKHsb2w1MPY9vZ2eHl5oba2FnZ2dhp9E6qn3d3dUV1dTYe0ZkDdGAGApqYm2NjYcBecVCoVBgwYwM23pK+8Pt9P6padpwoKCrinUh7uhFU9XVtbi4KCgv4OjTwia2vrHqf5Sqe/wpNPPqnxy9GTM2fOPFJApH8cOXJE53qRkZF9HA0xJEdHR7z11lvw9fXF8uXLsX79emOHZBJ0SnZTp07t4zBIf+s8lohAINA4tOk8TWOOmIfg4GCUlJQAeHDPZEZGRrf1+EqnZLdy5cq+joP0s0uXLnGv4+LiEB8fj8uXL+Pxxx9Hbm4ucnNzteoR03X48GG4uLjoVI+v6GCep27fvs29Ligo4JIbAI37sDrXI6bL2dkZ/v7+qKio6LaOv78/r++Y0CnZubq64vLly3Bzc8PAgQN7PH9HD9mbh84nre/fv68xr/M0ndw2H1euXEFAQECXCc/f35/rEIOvdPokf/LJJ3BycuJe63qxgpiucePGcTcS91aPmI8rV66goaEB8fHx+PnnnzFs2DDk5ubyukWnplOy69wRJ/UWYhmCgoLw1Vdf6VSPmBdnZ2ccPXqUOnZ4iN732VlZWWn1GgwAd+7coZtPzUhpaalB6xFi6gzWeadSqdTo9omYtt4GGNe3HiGmTuezz+qOMgUCAf7+97/D0dGRm9fe3o5jx44hMDDQ8BGSPuHm5mbQeoSYOp2T3SeffALgQcsuKytL45BV3XlnVlaW4SMkfULXK3N8v4JHLIfOye7atWsAHvRYnJ2djYEDB/ZZUKTvXbx40aD1CDF1et9Epe6xmJg3XUdSt/QR5Ql/6J3sXn755R7nb9my5TcHQ/qPnZ0dN8BwXFwcAgICuMfFrly5gry8PK4eIZZA72R39+5djWmVSoWLFy+ivr4ezz33nMECI31r0KBB3C1E6sQGAIcOHdKqR4gl0PvWk71792r827dvH65evYpZs2Zh/PjxfREjgAePoSUlJUEsFsPFxQUpKSk93hZRV1eHN954A8OHD4e9vT2GDBmCN998Ew0NDX0WoznR9WZhuqmYWAqDdN4pFAqRlpbGXbHtC0lJSSgtLYVMJsO+fftw7NgxLFq0qNv61dXVqK6uRkZGBi5evIht27YhLy8PKSkpfRajOdH1SRh6YoZYDGYg+/fvZ25uboZanYaffvqJAWDFxcVcWW5uLhMIBKyqqkrn9Xz11VfM1taWqVQqnZdpaGhgAFhDQ4NeMZs6pVLJAPT6T6lUGjtU8hu0traynJwc1traauxQ+pQ+30+9z9mlpaU9nCxRU1OD/fv3azxDa0gnT56Ei4sLQkJCuLKoqCgIhUIUFRVh2rRpOq1H3U99Tz15KJVKKJVKblp9El+lUlnUlUldr6rn5+fTuVgzpP6sWtJntiv6bJ/eye7s2bMa00KhEBKJBJmZmb1eqf2t5HI53N3dNcqsra3h6uqqNTJWd27fvo0PPvigx0NfAFi9ejVWrVqlVX7o0CGdxuU0Fzt27AAAiEQijeSupi7/7LPPtMaoIOZDJpMZO4Q+1dzcrHNdo95nt3TpUqxZs6bHOmVlZY/8PgqFAs8//zxGjhyJP/7xjz3WXbZsmUbrVaFQwNvbGzExMRY1ulhRUREAdJnoOpcHBAQgISGh3+Iij661tRUbN27E0aNHMXHiRLz++usW+9y6+shLF0btmXHx4sW9ngD38/ODVCrV6mmlra0NdXV1kEqlPS5/7949xMXFwcnJCXv37u21uxuRSASRSKRVbmNjY1Fd5YwdO5Z7LRQK0dHR0eX02LFjLWq7Ld27776LtWvXor29HQBw4MABLF++HGlpafjoo4+MHJ3h6fXZ7PtTiI9OfYHi9OnTXNnBgwd7vUDR0NDAxo8fzyZOnMiampp+03tb6gWKwMBA7iKERCJhmzZtYlu3bmWbNm1iEomEmxcYGGjsUImOlixZ0uPFpiVLlhg7RIPT5/tpNoNkx8fH4+bNm8jKyoJKpcKCBQsQEhKCnTt3AgCqqqoQGRmJ7du3Y9y4cVAoFIiJiUFzczP27t2rMUiwRCLRue89Sx0k28HBget+3d7eXqMr9s7T9vb2ep0XIcbR2toKOzu7HgfAFggEaGlpsahDWoscJHvHjh0IDAxEZGQkEhISEB4ejs2bN3PzVSoVysvLuS/mmTNnUFRUhAsXLiAgIACenp7cvxs3bhhrM0yG+oq0i4sLWltbNea1trZyI1XRGBTm4dNPP+0x0QEP7pxQd9XGS4ZqTt64cYMtXLjQUKszGZZ6GJuYmKjTfXaJiYnGDpXoYPz48Trtz/Hjxxs7VIPS5/tpsJbdnTt38NlnnxlqdaSPvfLKKxrTUqkU8fHxWhd8Hq5HTJOug5nzedBzOkbhKfbQIY9cLtcYO7a7esQ03bt3j3s9aNAgREREoK6uDq6ursjPz8edO3e06vENJTueUl/Y0aVefHx8H0dDHlXnJwnu3LmDb775ptd6fGM2FyiIYal/4T08PCAUan4MhEIhPDw8NOoRYu50btlNnz69x/n19fWPGgvpR56engCAmzdv4vnnn0dsbCzXeefBgwexf/9+jXrEtEmlUvz666861eMrnVt2zs7OPf4bOnQo5s2b15exEgPq3PcgYwxBQUF4+umnERQUpHGeri/7KCSGM3r0aIPWs0Q6t+y2bt3al3GQfta5x+m8vDwcOHCAm+58w/XDPVMT06Tr/ZB8vm+SztnxlEQiAQD4+vpCIBBozBMIBPD19dWoR0ybrk/3WNJTQPrSOc3r2n0TDbhjHgYPHgwAqKysREJCAnx9fblzdteuXeNaeup6xLTNnTsXX3zxBezs7NDa2qrVsYOtrS1aWlowd+5cI0ZpXDo/GysUCjF06FA8+eSTPd57tXfvXoMFZwos9dnY9vZ2BAQEwM3NDbdv30ZlZSU3z9fXF4MGDcKdO3fw888/6/wcMTGe9vZ2uLq6QqFQwN3dHc888wzu3r2LgQMHorCwELW1tRCLxairq7Oo/anP91Pnlt2rr76KL7/8EteuXcOCBQvwu9/9Dq6uro8cLDEOKysrZGZmYsaMGXj++efxzjvv4Oeff8awYcMgk8mwf/9+fPPNNxb1xbBkVlZW2Lp1KxITE3Hr1i3s2bOHm6c+TbF161Z+7099nkNraWlhO3fuZFFRUczBwYHNnDmT5eXlsY6ODv0fajMTlvpsrNqePXuYj4+PxvOTvr6+bM+ePcYOjfwGe/bsYUOGDNHYn0OHDrXY/dkvXTz98ssv2LZtG7Zv3462tjaUlpbC0dHRUDnYZFjqYWxn7e3tyM/PR25uLuLj4xEREcHvFoCZ49P+7JPD2IcJhUIIBAIwxrheUYl5srKywsSJE9HU1ISJEyda7BeDL2h/dk2vW0+USiW+/PJLREdH4/HHH8eFCxfwl7/8BdevX7fIVh0hxHLo3LJ77bXXsGvXLnh7e+Pll1/Gl19+CTc3t76MjRBCDEbnZJeVlYUhQ4bAz88PR48exdGjR7usl52dbbDgCCHEUHROdvPmzdO6054QQsyFzslu27ZtfRgGIYT0LXo2lhDCC5TsCCG8YDbJrq6uDklJSRCLxXBxcUFKSgoaGxt7XOb3v/89/P39YW9vD4lEgilTpuDSpUv9FDEhxJSYTbJLSkpCaWkpZDIZ9u3bh2PHjmHRokU9LhMcHIytW7eirKwMBw8eBGMMMTExdBM0ITxkFj35lZWVIS8vD8XFxQgJCQEAbNiwAQkJCcjIyICXl1eXy3VOhj4+Pvjv//5vjBkzBpWVlfD39++X2AkhpsEskt3Jkyfh4uLCJToAiIqKglAoRFFREaZNm9brOpqamrB161b4+vrC29u723pKpRJKpZKbVigUAB6MymTJIzOpt82St5FP+LI/9dk+s0h2crkc7u7uGmXW1tZwdXWFXC7vcdm//vWvePfdd9HU1IThw4dDJpPB1ta22/qrV6/GqlWrtMoPHToEBweH37YBZkQmkxk7BGJAlr4/m5ubda5r1GS3dOlSrFmzpsc6ZWVlj/QeSUlJiI6ORk1NDTIyMvDiiy/i+PHjsLOz67L+smXLkJaWxk0rFAp4e3sjJibGYns9AR78QspkMkRHR8PGxsbY4ZBHxJf9qT7y0oVRk93ixYsxf/78Huv4+flBKpWitrZWo7ytrQ11dXW9Dg2nHv1s2LBhGD9+PAYOHIi9e/fipZde6rK+SCSCSCTSKrexsbHoD40aX7aTLyx9f+qzbUZNdhKJRKcBXcLCwlBfX4+SkhIEBwcDAI4cOYKOjg6Ehobq/H6MMTDGNM7JEUL4wSxuPRkxYgTi4uKwcOFCnDp1CsePH0dqaipmz57NXYmtqqpCYGAgTp06BQC4evUqVq9ejZKSEly/fh0nTpzAzJkzYW9vj4SEBGNuDiHECMwi2QHAjh07EBgYiMjISCQkJCA8PBybN2/m5qtUKpSXl3MnLO3s7FBYWIiEhAQEBARg1qxZcHJywokTJ7QudhBCLJ9ZXI0FAFdXV+zcubPb+T4+Phqjnnl5eWkM/EwI4TezadkRQsijoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFs0l2dXV1SEpKglgshouLC1JSUtDY2KjTsowxxMfHQyAQICcnp28DJYSYJLNJdklJSSgtLYVMJsO+fftw7NgxLFq0SKdl161bB4FA0McREkJMmVkMpVhWVoa8vDwUFxcjJCQEALBhwwYkJCQgIyODGyi7K+fOnUNmZiZOnz4NT0/P/gqZEGJizCLZnTx5Ei4uLlyiA4CoqCgIhUIUFRVh2rRpXS7X3NyMOXPmYOPGjZBKpTq9l1KphFKp5KYVCgWAB4Nwq1SqR9gK06beNkveRj7hy/7UZ/vMItnJ5XK4u7trlFlbW8PV1RVyubzb5d555x1MmDABU6ZM0fm9Vq9ejVWrVmmVHzp0CA4ODroHbaZkMpmxQyAGZOn7s7m5Wee6Rk12S5cuxZo1a3qsU1ZW9pvW/d133+HIkSM4e/asXsstW7YMaWlp3LRCoYC3tzdiYmIgFot/UyzmQKVSQSaTITo6GjY2NsYOhzwivuxP9ZGXLoya7BYvXoz58+f3WMfPzw9SqRS1tbUa5W1tbairq+v28PTIkSOoqKiAi4uLRnliYiKeeeYZFBQUdLmcSCSCSCTSKrexsbHoD40aX7aTLyx9f+qzbUZNdhKJBBKJpNd6YWFhqK+vR0lJCYKDgwE8SGYdHR0IDQ3tcpmlS5filVde0SgbNWoUPvnkE7zwwguPHjwhxKyYxTm7ESNGIC4uDgsXLkRWVhZUKhVSU1Mxe/Zs7kpsVVUVIiMjsX37dowbNw5SqbTLVt+QIUPg6+vb35tACDEys7nPbseOHQgMDERkZCQSEhIQHh6OzZs3c/NVKhXKy8v1OmFJCOEPs2jZAYCrqyt27tzZ7XwfHx8wxnpcR2/zCSGWy2xadoQQ8igo2RG0t7fj6NGjOHbsGI4ePYr29nZjh0SIwVGy47ns7Gz4+/sjOjoaa9euRXR0NPz9/ZGdnW3s0AgxKEp2PJadnY3ExEStexhra2uRmJhICY9YFEp2PNXe3o4//OEPAIDIyEgUFhbiyy+/RGFhISIjIwEAr776Kh3SmiE6LdE1SnY8VVBQgFu3biE8PBzZ2dloaWlBcXExWlpakJ2djfDwcNTW1nb7pAkxTdnZ2QgICNA4LREQEECtdFCy4y11EouKioKvr6/Gl8PX15dr3VGyMx/Z2dmYMWMGRo0apdFSHzVqFGbMmMH7hEfJjuf++Mc/oqqqSqOsqqqqy55fiOlqb2/H4sWLMXnyZOTk5CA0NBT29vYIDQ1FTk4OJk+ejPT0dF4f0lKy46lnn33WoPWIcRUWFqKyshLLly9HW1sbPv30U2zevBmffvop2trasGzZMly7dg2FhYXGDtVozOYJCmJY9+7d05gOCAiAUChER0cHrly50m09YppqamoAALt27UJ4eDjXgjtw4ADee+89pKamatTjIwGjZ6h6pFAo4OzsjIaGBovqz87f3x9Xr17ttZ6fnx8qKir6ISLyKAoKChAREQEA3I+WWufp/Px8TJo0yRgh9gl9vp90GMtT1dXVBq1HjKtzV2fx8fEaFyji4+O7rMc3lOx4ytbW1qD1iHFt2rSJe80Yw+7du7F161bs3r1bowOMzvX4hs7Z8ZRUKtWpS2tdByoixqW+8BAaGooDBw5w5YcOHeLKi4qKUFhYqDHsAJ9Qy46ndL0Fgc+3KpgTJycnAEBRUVGX89Xl6np8RMmOpxobGw1ajxjXSy+9ZNB6loiSHU/Z2dkZtB4xrh9//NGg9SwRJTueGjx4MPfa2toaY8aMQWBgIMaMGQNra+su6xHTlZOTY9B6loguUPDUtGnTcOLECQAPhqU8f/58t/WI6bt27Rr32t7eHvfv3+9yunM9vqGWHU+9+eabEAgEPdYRCAR48803+yki8ija2tq417W1tcjIyEBCQgIyMjI0+ivsXI9vKNnxlK2tLdLT03usk56eTvfZmYnOV1mdnZ2Rnp6OAwcOID09Hc7Ozl3W4xuzSXZ1dXVISkqCWCyGi4sLUlJSer1SOGnSJAgEAo1/6g4rCfDRRx9hyZIlEAo1PwZCoRBLlizBRx99ZKTIiL46PxnR+VGxh6f5/ASF2TwbGx8fj5qaGvzv//4vVCoVFixYgLFjx/Y4vOKkSZPw+OOP409/+hNX5uDgoNczrpb6bGxnra2t2LBhA44cOYLnnnsOb7zxBrXozMzBgwcRFxfXa728vDzExsb2Q0T9Q5/vp1lcoCgrK0NeXh6Ki4sREhICANiwYQN3TsLLy6vbZR0cHOgpgF7Y2trizTffREBAABISEmBjY2PskIieHm7NPWo9S2QWye7kyZNwcXHhEh3woIddoVCIoqKiHq8Y7tixA1988QWkUileeOEFvP/++3BwcOi2vlKphFKp5KbVj1SpVCqoVCoDbI1pUm+bJW+jJfv88891rhcVFdXH0fQffT6vZpHs5HI53N3dNcqsra3h6uoKuVze7XJz5szB0KFD4eXlhR9//BHvvfceysvLe+yeevXq1V320nvo0KEek6SlkMlkxg6B/AZnz57lXtvY2Ggkgc7TZ8+e1Xh21tw1NzfrXNeoyW7p0qVYs2ZNj3XKysp+8/oXLVrEvR41ahQ8PT0RGRmJiooK+Pv7d7nMsmXLNB6UVigU8Pb2RkxMjMWeswMe/ELKZDJER0fTYawZWr9+PS5dugR7e3vU1tbiX//6F7c/w8PD4e7ujvv378PT0xMJCQnGDtdgdOnMQs2oyW7x4sWYP39+j3X8/PwglUq1xjZta2tDXV2dXufj1Feirly50m2yE4lEEIlEWuU2Nja8SAJ82U5Lo/4e3L9/H7Nnz8Z7772HsWPHYsCAAZg9ezZ3U7FUKrWo/avPthg12UkkEkgkkl7rhYWFob6+HiUlJQgODgYAHDlyBB0dHXpdSj937hwAwNPT8zfFS4ip8vX15V7n5uZi//793LSVlVWX9fjGLO6zGzFiBOLi4rBw4UKcOnUKx48fR2pqKmbPns1dia2qqkJgYCBOnToFAKioqMAHH3yAkpISVFZW4rvvvsO8efPw7LPPYvTo0cbcHEIM7rnnnuNeP3zFtXM3XZ3r8Y1ZJDvgwVXVwMBAREZGIiEhAeHh4di8eTM3X6VSoby8nDthaWtri++//x4xMTEIDAzE4sWLkZiYiH/+85/G2gRC+sykSZO4oyR7e3uNeeppd3d3ixp/Ql9mcTUWAFxdXXu8gdjHx0ej+2lvb28cPXq0P0IjxOisrKyQlZWFxMRErXnqZ6A3bdqkcUjLN2bTsiOE9Gz69OnYs2eP1m1a7u7u2LNnD6ZPn26kyEyD2bTsCCG9mz59OqZMmYL8/Hzk5uYiPj4eERERvG7RqVGyI8TCWFlZYeLEiWhqasLEiRMp0f0fOowlhPACJTtCCC9QsiOE8AKds+uF+nYWfZ7BM0cqlQrNzc1QKBQW9TgRX/Flf6q/l7p0y0nJrhf37t0D8OC+PUKIabp3755G9/NdMZueio2lo6MD1dXVcHJy6nWAGnOm7t3lxo0bFt27C1/wZX8yxnDv3j14eXlpDS/wMGrZ9UIoFOKxxx4zdhj9RiwWW/SXg2/4sD97a9Gp0QUKQggvULIjhPACJTsC4EGnpStXruyy41Jifmh/aqMLFIQQXqCWHSGEFyjZEUJ4gZIdIYQXKNmZkUmTJuHtt982dhiEmCVKdhaooKAAAoEA9fX1xg6F6MHUfsxMLZ5HRcmOEAvS2tpq7BBMFiU7M6VUKvHee+/B29sbIpEIAQEB+Oyzz1BZWYmIiAgAwMCBAyEQCHodiBx48CB1UlISBgwYAE9PT3zyySdav+yff/45QkJC4OTkBKlUijlz5mgMXq5uUR4+fBghISFwcHDAhAkTUF5ebujNtzjz58/H0aNHsX79eggEAggEAlRUVCAlJQW+vr6wt7fH8OHDsX79eq3lpk6dij//+c/w8vLC8OHDAQAnTpxAUFAQ7OzsEBISgpycHAgEAm7sZAC4ePEi4uPj4ejoCA8PD8ydOxe3b9/uNp7Kysr++nP0DUbMxsSJE9lbb73FGGPsxRdfZN7e3iw7O5tVVFSw77//nu3atYu1tbWxPXv2MACsvLyc1dTUsPr6+l7X/corr7ChQ4ey77//nl24cIFNmzaNOTk5ce/HGGOfffYZO3DgAKuoqGAnT55kYWFhLD4+npufn5/PALDQ0FBWUFDASktL2TPPPMMmTJhg6D+Fxamvr2dhYWFs4cKFrKamhtXU1LCWlha2YsUKVlxczK5evcq++OIL5uDgwHbv3s0tl5yczBwdHdncuXPZxYsX2cWLF1lDQwNzdXVlv/vd71hpaSk7cOAAe/zxxxkAdvbsWcYYY3fv3mUSiYQtW7aMlZWVsTNnzrDo6GgWERHRbTxtbW3G+NMYDCU7M6JOduXl5QwAk8lkXdZTJ527d+/qtF6FQsFsbGzY119/zZXV19czBwcHjWT3sOLiYgaA3bt3T+N9v//+e67O/v37GQB2//59nWLhs84/Zt15/fXXWWJiIjednJzMPDw8mFKp5Mo2bdrEBg0apPE3/9vf/qaR7D744AMWExOjse4bN25wP5K6xmNO6DDWDJ07d44bVMUQrl69CpVKhXHjxnFlzs7O3CGRWklJCV544QUMGTIETk5O3Ptfv35do97o0aO5156engCgcbhLdLdx40YEBwdDIpHA0dERmzdv1vp7jxo1Cra2ttx0eXk5Ro8eDTs7O66s874FgPPnzyM/Px+Ojo7cv8DAQABARUVFH26R8VAXT2bo4RHf+0NTUxNiY2MRGxuLHTt2QCKR4Pr164iNjdU6Kd65Z1x1H4AdHR39Gq8l2LVrF9LT05GZmYmwsDA4OTnh448/RlFRkUa9AQMG6L3uxsZGvPDCC1izZo3WPPUPlKWhZGeGRo0ahY6ODhw9ehRRUVFa89W/8u3t7Tqtz8/PDzY2NiguLsaQIUMAAA0NDbh8+TKeffZZAMClS5dw584dfPjhh1yvzadPnzbE5pD/Y2trq7HPjh8/jgkTJuC1117jynRpdQ0fPhxffPEFlEol1xFAcXGxRp2nnnoKe/bsgY+PD6ytu04DD8dj7ugw1gz5+PggOTkZL7/8MnJycnDt2jUUFBTgq6++AgAMHToUAoEA+/btw61bt9DY2Njj+pycnJCcnIwlS5YgPz8fpaWlSElJgVAo5FpmQ4YMga2tLTZs2ICrV6/iu+++wwcffNDn28onPj4+KCoqQmVlJW7fvo1hw4bh9OnTOHjwIC5fvoz3339fK2l1Zc6cOejo6MCiRYtQVlaGgwcPIiMjA8C/W9qvv/466urq8NJLL6G4uBgVFRU4ePAgFixYwCW4h+Mx99Y5JTsztWnTJsyYMQOvvfYaAgMDsXDhQjQ1NQEABg8ejFWrVmHp0qXw8PBAampqr+tbu3YtwsLCMHnyZERFReHpp5/GiBEjuPM+EokE27Ztw9dff42RI0fiww8/5L5AxDDS09NhZWWFkSNHQiKRIDY2FtOnT8esWbMQGhqKO3fuaLTyuiMWi/HPf/4T586dQ1BQEP7zP/8TK1asAABuf3p5eeH48eNob29HTEwMRo0ahbfffhsuLi5c9+YPx/PwuUJzQ108kS41NTVh8ODByMzMREpKirHDIY9ox44dWLBgARoaGoxyztcU0Dk7AgA4e/YsLl26hHHjxqGhoQF/+tOfAABTpkwxcmTkt9i+fTv8/PwwePBgnD9/Hu+99x5efPFF3iY6gJIdL1y/fh0jR47sdv5PP/0EAMjIyEB5eTlsbW0RHByMwsJCuLm59VeYxIDkcjlWrFgBuVwOT09PzJw5E3/+85+NHZZR0WEsD7S1tfX4qE9PV+QIsRSU7AghvEBXYwkhvEDJjhDCC5TsCCG8QMmOEMILlOyIyZg/fz7XUaSNjQ08PDwQHR2NLVu26PWo0rZt2+Di4tJ3gXZD3ZEmMU2U7IhJiYuLQ01NDSorK5Gbm4uIiAi89dZbmDx5Mtra2owdHjFnxuxMj5DOkpOT2ZQpU7TKDx8+zACwv/3tb4wxxjIzM9kTTzzBHBwc2GOPPcZeffVVrQ5EO/9buXIlY4yx7du3s+DgYObo6Mg8PDzYSy+9xG7evMm9T11dHZszZw5zc3NjdnZ2LCAggG3ZsoWbf/36dTZz5kzm7OzMBg4cyP7jP/6DXbt2jTHG2MqVK7XeNz8/v0/+TuS3oZYdMXnPPfccxowZg+zsbACAUCjEp59+itLSUvzjH//AkSNH8O677wIAJkyYgHXr1kEsFqOmpgY1NTVIT08HAKhUKnzwwQc4f/48cnJyUFlZqTE+x/vvv4+ffvoJubm5KCsrw6ZNm7gnSFQqFWJjY+Hk5ITCwkIcP34cjo6OiIuLQ2trK9LT0/Hiiy9yLdOamhpMmDChf/9QpGfGzraEqHXXsmOMsVmzZrERI0Z0Oe/rr79mgwYN4qa3bt3KnJ2de32/h7uVf+GFF9iCBQu6rPv555+z4cOHs46ODq5MqVQye3t7dvDgwV7jJ8ZHLTtiFhhjXF9s33//PSIjIzF48GA4OTlh7ty5uHPnDpqbm3tcR2/dyr/66qvYtWsXgoKC8O677+LEiRPcsufPn8eVK1fg5OTEdWPu6uqKlpYWi+3G3NJQsiNmoaysDL6+vqisrMTkyZMxevRo7NmzByUlJdi4cSOAnsdMVXcrLxaLsWPHDhQXF2Pv3r0ay8XHx+OXX37BO++8g+rqakRGRnKHwI2NjQgODsa5c+c0/l2+fBlz5szp460nhkBPfxOTd+TIEVy4cAHvvPMOSkpK0NHRgczMTK6TSXUPzWpddSeua7fyEokEycnJSE5OxjPPPIMlS5YgIyMDTz31FHbv3g13d3eIxeIu47S0bswtDbXsiElRKpWQy+WoqqrCmTNn8D//8z+YMmUKJk+ejHnz5iEgIAAqlYrrHv7zzz9HVlaWxjp8fHzQ2NiIw4cP4/bt22hubtapW/kVK1bg22+/xZUrV1BaWop9+/ZhxIgRAICkpCS4ublhypQpKCws5LrCf/PNN/Hrr79y7/vjjz+ivLwct2/fhkql6p8/GtGNsU8aEqKWnJzM3bZhbW3NJBIJi4qKYlu2bGHt7e1cvbVr1zJPT09mb2/PYmNj2fbt27XGyf3DH/7ABg0apHHryc6dO5mPjw8TiUQsLCyMfffdd1pjqY4YMYLZ29szV1dXNmXKFHb16lVunTU1NWzevHnMzc2NiUQi5ufnxxYuXMgaGhoYY4zV1tay6Oho5ujoSLeemCDq4okQwgt0GEsI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF74/xg1bico2gQzAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:11:45.279815Z", + "iopub.status.busy": "2024-02-29T06:11:45.279402Z", + "iopub.status.idle": "2024-02-29T06:11:45.492124Z", + "shell.execute_reply": "2024-02-29T06:11:45.491187Z" + }, + "papermill": { + "duration": 0.238184, + "end_time": "2024-02-29T06:11:45.494165", + "exception": false, + "start_time": "2024-02-29T06:11:45.255981", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEnCAYAAAA9/18kAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABheklEQVR4nO2dd3iTVdvAf9ndAwothZZS9gYLreBApVh4UcSJOBgiioq+r4gKqAxFQUU/XhXFBbgA54uKiEIFRKaUKXsUymqhQPfIer4/0qRJmqQJTZq2nN915aJ5nvOcnIQ8d+773EsmSZKEQCAQNEDk/l6AQCAQ+Aoh4AQCQYNFCDiBQNBgEQJOIBA0WISAEwgEDRYh4AQCQYNFCDiBQNBgEQJOIBA0WISAEwgEDRalvxdQFzEajZw5c4bQ0FBkMpm/lyMQCOyQJInCwkJiY2ORy13oaVId4L333pNatmwpaTQaKTk5WdqyZYvTsR999JF07bXXShEREVJERITUv3//KuNHjhwpATaPtLQ0t9dz8uTJKteLh3iIR917nDx50uW97HcN7uuvv2bChAnMnz+flJQU5s6dS1paGgcPHqRp06ZVxq9du5bhw4fTt29fAgICeP3117n55pvZu3cvzZs3t4wbOHAgCxcutDzXaDRuryk0NBSAkydPEhYWVoN3JxAIfEFBQQFxcXGWe9UZMknyb7J9SkoKvXv35r333gNM5mFcXBxPPvkkkyZNqvZ6g8FAZGQk7733HiNGjABg1KhR5OXlsWzZsstaU0FBAeHh4eTn5wsBJxDUQdy9R/3qZNBqtWRkZJCammo5JpfLSU1NZdOmTW7NUVJSgk6no1GjRjbH165dS9OmTWnfvj2PPfYYFy5ccDpHeXk5BQUFNg+BQFD/8auAy83NxWAwEB0dbXM8Ojqa7Oxst+Z4/vnniY2NtRGSAwcO5PPPPyc9PZ3XX3+ddevWMWjQIAwGg8M5Zs2aRXh4uOURFxd3+W9KIBDUGfy+B1cTZs+ezdKlS1m7di0BAQGW4/fee6/l765du9KtWzdat27N2rVr6d+/f5V5Jk+ezIQJEyzPzfa9QCCo3/hVwEVFRaFQKMjJybE5npOTQ0xMjMtr58yZw+zZs1m9ejXdunVzOTYxMZGoqCiOHDniUMBpNBqPnBCCmiFJEnq93qlGLRAoFAqUSmWNw7T8KuDUajVJSUmkp6czdOhQwORkSE9PZ/z48U6ve+ONN3j11Vf57bff6NWrV7Wvc+rUKS5cuECzZs28tXTBZaLVajl79iwlJSX+XoqgjhMUFESzZs1Qq9WXPYffTdQJEyYwcuRIevXqRXJyMnPnzqW4uJjRo0cDMGLECJo3b86sWbMAeP3115k6dSqLFy8mISHBslcXEhJCSEgIRUVFzJgxgzvvvJOYmBiOHj3Kc889R5s2bUhLS/PJe5AkSQQEu4HRaCQzMxOFQkFsbCxqtVp8boIqSJKEVqvl/PnzZGZm0rZtW9fBvNVM5nfeffddKT4+XlKr1VJycrK0efNmy7l+/fpJI0eOtDxv2bKlw4C/adOmSZIkSSUlJdLNN98sNWnSRFKpVFLLli2lsWPHStnZ2W6vJz8/XwKk/Px8l+O+3polDXh7rfTf1Yc8er9XKqWlpdK+ffuk4uJify9FUA8oLi6W9u3bJ5WWllY55+496ncNDmD8+PFOTdK1a9faPD9+/LjLuQIDA/ntt9+8tDLXlOsNHMop4u/jF2vl9RoKl/1rLLii8Mb3RHzTakBSS1Ps3Y6sPAxGv8ZLCwQCBwgBVwPax4QSolFSVK7nYHahv5cjEAjsEAKuBijkMnrGRwCQkXXJv4sR+JQbbriB//znP/5eRoNh7dq1yGQy8vLyfPo6QsDVkKviIwHYfkIIOEHt3bgC9xACroYktTQJuG0nhKNBcGWh1Wr9vYRqEQKuhvSMj0Amg5MXSzlXUObv5dQ7JEmiRKv3y0O6zEI65eXlPP/888TFxaHRaGjTpg2ffvopx48f58YbbwQgMjISmUzGqFGjqp2vsLCQ+++/n+DgYJo1a8b//d//VTGJv/jiC3r16kVoaCgxMTHcd999nDt3znLerDmmp6fTq1cvgoKC6Nu3LwcPHnT7fc2cOZOmTZsSGhrKww8/zKRJk+jRo4fl/KhRoxg6dCivvvoqsbGxtG/f3q21AaxYsYJ27doRGBjIjTfeWG00hLeoE2Ei9ZnQABXto0M5kF1IxolLDOoqsiU8oVRnoNPU2gnrsWffy2kEqT2/BUaMGMGmTZt455136N69O5mZmeTm5hIXF8f333/PnXfeycGDBwkLCyMwMLDa+SZMmMCGDRv46aefiI6OZurUqWzfvt1GuOh0Ol555RXat2/PuXPnmDBhAqNGjWLFihU2c73wwgu89dZbNGnShHHjxvHQQw+xYcOGatfw1Vdf8eqrr/L+++9zzTXXsHTpUt566y1atWplMy49PZ2wsDBWrVrl9tpOnjzJHXfcwRNPPMEjjzzCtm3beOaZZ6pdkzcQAs4L9EqIFALuCuHQoUN88803rFq1ylLBJjEx0XLeXLaradOmREREVDtfYWEhn332GYsXL7bkSS9cuJDY2FibcQ899JDl78TERN555x169+5NUVERISEhlnOvvvoq/fr1A2DSpEkMHjyYsrIym2IUjnj33XcZM2aMJYNo6tSp/P777xQVFdmMCw4O5pNPPrFJn6pubR988AGtW7fmrbfeAqB9+/bs2bOH119/vdrPp6YIAecFklpG8uXmLLYJR4PHBKoU7HvZNyl07ry2p+zcuROFQmERIjXl2LFj6HQ6kpOTLcfCw8Mt5p+ZjIwMpk+fzq5du7h06RJGoxGArKwsOnXqZBlnXXjCnHt97tw54uPjXa7j4MGDPP744zbHkpOT+eOPP2yOde3atUpuaHVr279/PykpKTbX9OnTx+V6vIUQcF6gV0XA794z+ZTpDARcxo1zpSKTyS7LTPQX7pic3qa4uJi0tDTS0tL46quvaNKkCVlZWaSlpVXZ6FepVJa/zXm+ZoHjDYKDgy97bf5AOBm8QIvIQJqEatAZJPaczvf3cgQ+pGvXrhiNRtatW+fwvFm7cbcUVGJiIiqVir///ttyLD8/n0OHDlmeHzhwgAsXLjB79myuu+46OnToUGUTv6a0b9/eZg1AleeOcGdtHTt2ZOvWrTbHNm/eXPNFu4EQcF5AJpORVBEPt+24MFMbMgkJCYwcOZKHHnqIZcuWkZmZydq1a/nmm28AaNmyJTKZjOXLl3P+/Pkqe1j2hIaGMnLkSJ599lnWrFnD3r17GTNmDHK53KKBxcfHo1areffddzl27Bg//fQTr7zyilff15NPPsmnn37KZ599xuHDh5k5cya7d++uttqLO2sbN24chw8f5tlnn+XgwYMsXryYRYsWeXX9zhACzkv0SjAJuAyxD9fg+eCDD7jrrrt4/PHH6dChA2PHjqW4uBiA5s2bM2PGDCZNmkR0dLTLuoZm3n77bfr06cMtt9xCamoq11xzDR07drQ4Bpo0acKiRYv49ttv6dSpE7Nnz2bOnDlefU/3338/kydPZuLEiVx11VVkZmYyatSoap0T7qwtPj6e77//nmXLltG9e3fmz5/Pa6+95tX1O8PvXbXqIpfTVWt71iXueH8jjYLVZLyYKuqcOaCsrIzMzExatWpV7Y1zJVNcXEzz5s156623GDNmjN/WMWDAAGJiYvjiiy/88vquvi/u3qP1Z3e3jtMlNhy1Us7FYi2ZucUkNgmp/iKBANixYwcHDhwgOTmZ/Px8Xn75ZQBuu+22WltDSUkJ8+fPJy0tDYVCwZIlS1i9erVNvFt9RJioXkKtlNO9RTggzFRBJVlZWZZq044eWVlZgKnHSPfu3UlNTaW4uJj169cTFRXltXV07tzZ6Rq++uorZDIZK1as4PrrrycpKYmff/6Z77//3qZbXX1EaHBeJKllI/4+fomME5e4u5foyiWA2NhYdu7c6fJ8fHw8GRkZPl3HihUr0Ol0Ds9FR0cTGBjI6tWrfboGfyAEnBcxJ94LDU5gRqlU0qZNG38vg5YtW/p7CX5BmKhe5KqK2nCHzxWRV+L/IEeB4EpHCDgv0jhEQ2KUKdJ7R1aefxcjEAiEgPM2V4n6cAJBnUEIOC/TS+zDCQR1BiHgvIzZ0bDrZD46g/eSnAUCgecIAedlWjcJISxASanOwP6zBf5ejqAOkJCQwNy5cy3PZTIZy5Yt89t6riSEgPMycrlMhIsIXHL27FkGDRrk72VcEQgB5wMqG9EIASeoSkxMDBqNxq9rkCQJvV7v1zXUBkLA+QBzx3vRStANJAm0xf55eFBnwp3GMO5ibaIeP34cmUzGDz/8wI033khQUBDdu3dn06ZNNtf89ddfXHfddQQGBhIXF8dTTz1lqWAC7jel+fXXX0lKSkKj0fDXX395vPb6hshk8AHd48JRyGWczS/jdF4pzSNqvwpsvUFXAq/FVj/OF0w5A+rg6sfhXmOYmvDCCy8wZ84c2rZtywsvvMDw4cM5cuQISqWSo0ePMnDgQGbOnMmCBQs4f/4848ePZ/z48SxcuBBwvynNpEmTmDNnDomJiURGRnpl7XUZIeB8QJBaSefYMHafyifjxCUh4Oo57jaGqQkTJ05k8ODBAMyYMYPOnTtz5MgROnTowKxZs7j//vst2mLbtm1555136NevHx988AEBAQFuN6V5+eWXGTBggNfWXdcRAs5HXBUfye5T+Ww/cYkh3f2kodQHVEEmTcpfr+0G7jaGqQnOmsV06NCBXbt2sXv3br766ivLGEmSMBqNZGZm0rFjR7eb0vTq1ctra64PCAHnI5JaRrJo43GR0VAdMpnbZmJDxlWzmKKiIh599FGeeuqpKtfFx8d71PjFvmlMQ0cIOB9hLmG+/2whxeV6gjXio66vWDeGMbffMzeGuf76633++ldddRX79u1zWpVkz549lsYvcXGmMl3btm3z+brqA3XCizpv3jwSEhIICAggJSWlSgceaz7++GOuu+46IiMjiYyMJDU1tcp4SZKYOnUqzZo1IzAwkNTUVA4fPuzrt2FDs/BAYsMDMBgldp3Mq9XXFngXdxrD+JLnn3+ejRs3Mn78eHbu3Mnhw4f58ccfLf0eaqMpTX3F7wLu66+/ZsKECUybNo3t27fTvXt30tLSnLZFW7t2LcOHD2fNmjVs2rSJuLg4br75Zk6fPm0Z88Ybb/DOO+8wf/58tmzZQnBwMGlpaZSVldXW2wIgKcEULiICfus/1TWG8SXdunVj3bp1HDp0iOuuu46ePXsydepUi5OjNprS1FskP5OcnCw98cQTlucGg0GKjY2VZs2a5db1er1eCg0NlT777DNJkiTJaDRKMTEx0ptvvmkZk5eXJ2k0GmnJkiVuzZmfny8BUn5+vgfvpCoL/zomtXx+uTRywZYazdNQKC0tlfbt2yeVlpb6eyk1pqioSAoPD5c++eQTfy+lweLq++LuPepXDU6r1ZKRkWFT910ul5Oamlol0NEZJSUl6HQ6GjUyaUuZmZlkZ2fbzBkeHk5KSorTOcvLyykoKLB5eAPrgF+jUTQvq8/s2LGDJUuWcPToUbZv3879998P1G5jGIHn+FXA5ebmYjAYiI6OtjkeHR1Ndna2W3M8//zzxMbGWgSa+TpP5pw1axbh4eGWh3mjtqZ0bBZKoEpBQZmeI+ddNwAW1H0cNYbZv3+/y6YyAv9Sr117s2fPZunSpaxdu7ZGeyGTJ09mwoQJlucFBQVeEXJKhZwecRFsOnaBbccv0S46tMZzCvxDz549HTaGKS0tddlURuBf/CrgoqKiUCgU5OTk2BzPyckhJibG5bVz5sxh9uzZrF692iZI0nxdTk6OJWDS/NxZWo1Go/FZ8nOvhEg2HbtAxolL3JcS75PXEPiPwMDAOtFURuAYv5qoarWapKQk0tPTLceMRiPp6en06dPH6XVvvPEGr7zyCitXrqwSmd2qVStiYmJs5iwoKGDLli0u5/QVV1lKJ4mAXzOSB0nugisXb3xP/G6iTpgwgZEjR9KrVy+Sk5OZO3cuxcXFjB49GoARI0bQvHlzZs2aBcDrr7/O1KlTWbx4MQkJCZZ9NfOeh0wm4z//+Q8zZ86kbdu2tGrVipdeeonY2FiGDh1a6+/vqniTgDt+oYTconKiQvxbJsefmKP1S0pKCAwU+bkC15SUlAC2WR6e4ncBN2zYMM6fP8/UqVPJzs6mR48erFy50uIkyMrKQi6vVDQ/+OADtFotd911l80806ZNY/r06QA899xzFBcX88gjj5CXl8e1117LypUrayVmyZ7wQBXtokM4lFPE9hOXuLmza9O7IaNQKIiIiLDEOAYFBdVKoKygfiFJEiUlJZw7d46IiAgUCsVlzyWThL1QhYKCAsLDw8nPzycsLKzG803+YTdLtp7k0esTmfyvjl5YYf1FkiSys7PJy8vz91IEdZyIiAhiYmIc/gi6e4/6XYO7Ekhq2YglW0+KjAZMieTNmjWjadOm6HQ6fy9HUEdRqVQ10tzMCAFXC5hLmO8+nU+53oBGWfP/uPqOQqHwyhdYIHCF33NRrwQSGgfROFiNVm/kn9Oi05ZAUFsIAVcLyGQyES4iEPgBIeBqCdHxXiCofYSAqyUqe6XmiUBXgaCWEAKulujSPBy1Qk5uUTlZF0v8vRyB4IpACLhaIkCloEtzU7yOMFMFgtpBCLhaRHS8FwhqFyHgahHR8V4gqF2EgKtFrmoZAcDBnEIKykQUv0Dga4SAq0WahgYQ3ygISYIdWXn+Xo5A0OCpkYArKirySS+DhoyIhxMIag+PBVxmZiaDBw8mODiY8PBwS3/SiIgIIiMjfbHGBoXIaBAIag+Pk+0feOABJEliwYIFREdHi3peHmLueL8zKw+9wYhSIXYJBAJf4bGA27VrFxkZGbRv394X62nwtG0aSqhGSWG5ngPZhXRpHu7vJQkEDRaP1YfevXtz8uRJX6zlikAhl9GzwkzdniX24QQCX+KxBvfJJ58wbtw4Tp8+TZcuXarUS7fucCVwTFJ8JH8eOk/GiUuM6JPg7+UIBA0WjwXc+fPnOXr0qKUpDJjKAUmShEwmw2AweHWBDRFLRsNxocEJBL7EYwH30EMP0bNnT5YsWSKcDJdJj/gI5DI4nVdKdn4ZMeG13wxHIKgN/jiQw/LdZ5k+pDNhAZffHety8VjAnThxgp9++kk0u60BIRolHWLC2He2gIwTlxjcrVn1FwkE9ZCZv+zn2PliburQlFu6xdb663vsZLjpppvYtWuXL9ZyRWEOFxEBv4KGypm8Uo6dLwbgUol/UhM91uBuvfVWnn76afbs2UPXrl2rOBmGDBnitcU1ZJJaRvL5phMi4FfQYNlwJNfyd0FpPRFw48aNA+Dll1+uck44GdzH7GjYe6aAUq2BQLXoMCVoWGw8esHyd76fBJzHJqrRaHT6EMLNfZpHBBIdpkFvlNh9Ks/fyxEIvIokSfxVBzQ4jwScTqdDqVTyzz//+Go9VwwymcynBTC3Z13iue92caGo3OtzCwTVcfhcEecLK7979UKDU6lUxMfHC03NS/iyAOan6zP5Ztspftx5xutzCwTVYd5/k1dEkdULAQfwwgsvMGXKFC5eFJvjNcXSaSvrEkajdzttmQtqnskr9eq8AoE7mAVccivTj7i/BJzHTob33nuPI0eOEBsbS8uWLQkODrY5v337dq8trqHTOTaMAJWcvBIdx3KLadM0xGtzl2hNWvbZ/DKvzSkQuIPOYGTzMZMCNLhrMzYfu1h/BNzQoUN9sIwrE5VCTrcWEWzNvMj2E5d8JOCEBieoXXafyqOoXE9EkIo+rRsD9ShMZNq0ab5YxxVLUstItmZeZNuJi9zTO85r85Zq9YDQ4AS1z4YjpvCQvq0bEx6oBqCwXI/RKCGX125qp8cCzkxGRgb79+8HoHPnzvTs2dNri7qS8FUJ8+IKDe5cYbkorCmoVczhIde0iSI80JQIIElQWKYnPKh281E9/tafO3eOm266id69e/PUU0/x1FNPkZSURP/+/Tl//rzHC5g3bx4JCQkEBASQkpLC1q1bnY7du3cvd955JwkJCchkMubOnVtlzPTp05HJZDaPDh06eLyu2uKqeJOAO3q+mEvFWq/NW1oh4AxGifMiVERQS5Ro9eyoqHN4bZso1Eo5gSpTELs/9uE8FnBPPvkkhYWF7N27l4sXL3Lx4kX++ecfCgoKeOqppzya6+uvv2bChAlMmzaN7du30717d9LS0jh37pzD8SUlJSQmJjJ79mxiYmKcztu5c2fOnj1refz1118eras2iQxW07qJyVHjrQKYkiRRUmGigjBTBbXH1syL6AwSzSMCiW8UBGDR4vzRKtNjAbdy5Uref/99OnbsaDnWqVMn5s2bx6+//urRXG+//TZjx45l9OjRdOrUifnz5xMUFMSCBQscju/duzdvvvkm9957LxqNxum8SqWSmJgYyyMqKsqjddU23g74LdcbsY46yRYCTlBLmMNDrm0TZSmlFhZo2gmrFxqc0WiskmAPpiBgo9Ho9jxarZaMjAxSU1MrFyOXk5qayqZNmzxdlg2HDx8mNjaWxMRE7r//frKysmo0n6/pVRHw6619OLMH1YyIhRPUFn9VOBiuaVupVJg1uHoh4G666Sb+/e9/c+ZMZYT86dOnefrpp+nfv7/b8+Tm5mIwGIiOjrY5Hh0dTXZ2tqfLspCSksKiRYtYuXIlH3zwAZmZmVx33XUUFhY6vaa8vNyv/V3NrQR3ncxDZ3D/R8IZ1uYpCA1OUHscOWe6z3rGRViO1SsB995771FQUEBCQgKtW7emdevWtGrVioKCAt59911frNEjBg0axN133023bt1IS0tjxYoV5OXl8c033zi9ZtasWYSHh1secXHeC9dwh8SoYCKCVJTrjew9U3PhWmqnwYk9OEFtYd4aUSsrRUuYeQ/ODwLO4zCRuLg4tm/fzurVqzlw4AAAHTt2tDE13SEqKgqFQkFOTo7N8ZycHJcOBE+JiIigXbt2HDlyxOmYyZMnM2HCBMvzgoKCWhVycrmMpPhI0g+cI+PEJXpY/fpZs3RrFusOnef/hvUgQOW8vJK9iSqCfQW1hSSZJJx1tJu5VHm90ODAVAljwIABPPnkkzz55JMeCzcAtVpNUlIS6enplmNGo5H09HT69OlzOctySFFREUePHqVZM+dlwTUaDWFhYTaP2sadjvcf/nmMX//JZvOxC07HABTbmahCgxPUFhbflpWE86eJelmBvunp6aSnp3Pu3LkqjgVnHlBHTJgwgZEjR9KrVy+Sk5OZO3cuxcXFlo5dI0aMoHnz5syaNQswOSb27dtn+fv06dPs3LmTkJAQS4+IiRMncuutt9KyZUvOnDnDtGnTUCgUDB8+/HLeaq1hHfBr7lBmj3lv7dQl1xqZ2USNDtOQU1DO2fwytmddssTcCQS+okKBQ2Yl4dwRcGU6A99mnALggZR4rzWz8ljAzZgxg5dffplevXrRrFmzGi1k2LBhnD9/nqlTp5KdnU2PHj1YuXKlxfGQlZWFXF6pZJ45c8YmY2LOnDnMmTOHfv36sXbtWgBOnTrF8OHDuXDhAk2aNOHaa69l8+bNNGnS5LLXWRt0axGBUi4jp6CcU5dKiauIIbKmTGf6MTldjVfUbKK2bBRMToEpyPeO9zdyfPZgL69aIHCMzEMNrqhcz0vLTHUmH7y6pdfW4bGAmz9/PosWLeLBBx/0ygLGjx/P+PHjHZ4zCy0zCQkJFhvfGUuXLvXKumqbQLWCzs3D2XUyj+1ZlxwKuFKdSXCddlODC9aIMuiC2sP63rRWeyoDffXUNh7vwWm1Wvr27euLtVzxJMU7z0s1GiW0epMGd+pSict5zHtwQZrLTjUWCDzGWvewtuz86UX1WMA9/PDDLF682BdrueJx1fG+XF+51+muiRrkwtMqEHgba9vKkQbnykStxjC7bDz+iS8rK+Ojjz5i9erVdOvWrUpWw9tvv+21xV1pmHulHsguoKhcT4iVBmY2T8FUIaRcb0CjdCzAzCZqkOjUJahFbExUJ3twzhxovsJjAbd792569OgBUKX5TG0uvCESHRZA84hATueVsjMrj2ut0l3KrAScJMHZvDISooIdTWPR4ALVShaM6sVDi7b5duECAfYaXFUvqsEoUaI1EFyLWycev9KaNWt8sQ5BBb0SIjm9s5SME5ecCjgwmanOBZxpDy5YrSAxylQlOETsxwl8jOTERg1QyVEpZOgMEvmluloVcKIKYh3DHA+3Ys9ZDFYlQUrtBZwLT2qlBqdAozL9F5frRSc0gW+RcGyiymSyavfhJHyzCScEXB3jlm6xhAeqOJhTyNK/K6ugmGPgzLjypFqcDGqlZZ9OZ5BsBKZA4G1svKh258LczGbw9i6XEHB1jMhgNf9JbQvAW78fsnwhyu00uFMuPKmluoowEbUCjVXSs1Zf80olAoE72O/H+ytdSwi4OsgDV7ekTdMQLhZreTf9MOCZiVpcXulFtRZwwkwV+BKXGlyAf2LhPBZwf/75J3p91YhkvV7Pn3/+6ZVFXemoFHJeHGyqmLxo43GOnS+ymKjqiuYxrvJRS61MVKVCjrKik1G50OAEPsTZHhy4ocH5aPfEYwF34403Ouxqn5+fz4033uiVRQnghvZNubF9E/RGiddW7Ld4UVtVeE6zC8rQOymOWVJhogZWxMGZtTh7T6xA4E1sNTjHJmp1Gpy3A808FnDOAvUuXLhQpcu9oGa8MLgTSrmM1fvPsWqfqW5eXKMgVAoZBqNETqHjbln2gb6aiowGocEJfIlNlIinGpyPcDsg5Y477gBMm4ejRo2yafpiMBjYvXu3yFH1Mm2ahvBgn5Ys3HCclXtNZdyDNQqahQeSdbGEUxdLaB4RWOU68x5csNr032vW4Mp1QsAJfIerQhjmxjO1nXDvtoALDw8HTG8iNDSUwMDKG0utVnP11VczduxY76/wCuc//duxbMdpLpWYfvkClApaRJoEnKOcVKNRsjgk7E1UZ06G/BId+aU64htXrWAiELhLTTQ4XwUwuS3gFi5cCJhKFk2cOFGYo7VEeJCKCQPa8dKPewGT0DJrbY48qWVWQsxsogZUY6L2fOV3jBJsnHQTsQ40QoHAHdzZg6s+Ds67u3Ae78FNmzZNCLdaZnhyPO2jQwGT0GoeaRJCjjyp1v0YzB3Fq9PgzPG/u0/le23NgisQm3JJtqfcDfT1Nh4LuJycHB588EFiY2NRKpUoFAqbh8D7KBVy/ju8B0N7xHJ3rzhaRJpMSUcmaknF/lugSoG8IjzEnM1gnw0BtsG/Il9VUBNswkTszrnrRfU2Hn+jR40aRVZWFi+99FKNS5YL3KdDTBhz7zWVa88pMDWROZBdSJnOYNNhq8Qqi8GMq3zUwjKdzd+uyjAJBK5wVvAS/NdZy2MB99dff7F+/XpLySRB7dMjLoKYsACyC8pYtPE44/q1tpyzTrQ348qLau3Veuyr7cQ3CuLP50Q8o8BznBW8BNNeMpj2ge1/lMF3BS89NlHj4uKq7Ysg8C0BKgXPprUHYN4fR8gtqoyHc1Ts0qyROXIyWGtwAFkXXZdDFwic4azgJUCIWknFjolLM9Xvgb5z585l0qRJHD9+3MtLEXjC7T2b06V5GIXleuauPmQ5XlxuNlErlXNXJmpBae03AhE0DHQGI/PWHGHnyTzAPkzEVlTJ5TK/OBo8FnDDhg1j7dq1tG7dmtDQUBo1amTzENQOcrmMFwd3AmDJ1pMczikEKpPyHWlwjpwMBWW13whE0DDYfOwCb/52kNd+2Q9Ub2ZaEu5r8Tvn8R7c3LlzfbAMweVwdWJj0jpH89veHN76/RDzH0yyqgXnYA/OoQYnBJzg8jBbC5dKtEClF9WZ39FVLJyvCl56LOBGjhzpi3UILpOnB7Tjt705/HHgHEXlept+DGYsJqoDDa7QQeqMdb6xsSJIzhxyIhCY0RlM3w1L7KWlq71j3An2rRMFL48ePcqLL77I8OHDOXfuHAC//vore/fu9eriBNXTPjqUVlHBaA1G/jx0npLyyn4MZlw5GRyZC+YvbqnWQP+31zFy4VZfLF1QzzFXiDb3ADHrYM5CxywCrqQO78GtW7eOrl27smXLFn744QeKiooA2LVrF9OmTfP6AgWukclkDOgUDcDve7Mp0VUNEwlw6WSo+mXTVpRhWrn3LJm5xaw/nOv1dQvqP/oKAVdcocFJ1Whw/ki491jATZo0iZkzZ7Jq1SrUarXl+E033cTmzZu9ujiBe5gF3Io92SzffQZwP0zE0ZfNXB59/9lCyzERGiSwx1yPUKs3ojcYq92Dc+VFrTNxcHv27OH222+vcrxp06bk5opfen9wVXwkN7RvgtZg5ORFU/qWTZiIi4KXjr5sZkG470yB5ZjWSXFNwZWL3qqJUYnOYKXBVWOiuoyD83OyfUREBGfPnq1yfMeOHTRv3twrixJ4hkIuY+Go3nz1cAo3dWhKZJCKqxMrQ3Yqvai2QqpMZ+DvzKrVmc35qfvPVgo4876cQGDGuktbSbmh0g96GV5UX+GxF/Xee+/l+eef59tvv0Umk2E0GtmwYQMTJ05kxIgRvlijwA1kMhnXtInimjZRVc5ZKvraeVF/35dDYbkDE1Vv5HxhOReKtZZjOr0RNFWGCho4uUXlrNqXw5DusVUaNltrcMVaveWH1OkenB/yUT3W4F577TU6dOhAXFwcRUVFdOrUieuvv56+ffvy4osv+mKNghriLA5udUUZdHu0eiO7T+XZHNMJE/WK5L6PNzP5hz28tOyfKuese4KUaq1M1Go0OEeOLb8XvDSjVqv5+OOPeemll/jnn38oKiqiZ8+etG3b1hfrE3gBZwUvrU1Qa8r1BnZVpN9UHhMC7krkUI4pSuJ3Bz+GNhpcud4iwKrbg3MZXO7lOLjLLgAWHx9PfHy8N9ci8BGO9uDKdAaO5RY7HK/VG9llV/xSaHBXNo7kjs0enAcaXJ0zUSdMmEBxcbHlb1cPT5k3bx4JCQkEBASQkpLC1q3Og0r37t3LnXfeSUJCAjKZzGnamCdzXgk48qIeOVeEwSgREaSyaQ4NkFeqY3vWJZtjwslwZeNIaFmbqCVaQ2WYiJM5zGEixVpDrf1guqXB7dixA51OZ/nbGZ4Wv/z666+ZMGEC8+fPJyUlhblz55KWlsbBgwdp2rRplfElJSUkJiZy99138/TTT3tlzisBR3FwB7JNMW4dYkLZfMzWk7pkaxaFZXriGwVRpjNwrrBcaHBXOI7ubXsnQ6UG51gOhAVUipvCMj2NgivjaH0VZ+mWgFuzZo3Dv2vK22+/zdixYxk9ejQA8+fP55dffmHBggVMmjSpyvjevXvTu3dvAIfnL2fOK4HKXNRKDe5Axf5bh5iwKgLOnLnw0DUJLNhw3HSt2IO7onGUimwbJqKvTNVyModSISdEo6SoXE9+qc5GwJnxez04b6HVasnIyCA1NbVyMXI5qampbNq0qVbnLC8vp6CgwObhFuWF8P3DkJd1WeutLcxOhqJyPesPnwdsNbhPR/ZycI2cu3vFoVKYvnJCg7uycaSVWW9bFGsNlVqYCylV2/twbmlw5qbP7vDDDz+4NS43NxeDwUB0dLTN8ejoaA4cOOD263ljzlmzZjFjxgzPX3DlZNjzLRxZDXd+Am1Sq7/GD8SGB9C3dWM2Hr3AyAVbeX5gBw5kV2hwzcLoERdR5ZrEqBCCNUpUCtNvoBBwDRdJkli44TjtokO5tm3VOEpwpsHZhYlU/O1KCwsLVHE6r7TWBJxbGlx4eLjlERYWRnp6Otu2bbOcz8jIID093dIcur4xefJk8vPzLY+TJ0+6d+H1z0KzHlB6Cb68C9a+Dsa6JwhkMhkLRvXmnl4tMEow69cD5BZpkcmgXXSIw2taRZlaQyorNDjr/RZBw2LTsQu8vHwfD3y6xcWomu/BQeU+nH2oiK9yUd3S4MxNnwGef/557rnnHubPn29pE2gwGHj88ccJCwtz+4WjoqJQKBTk5NjG1+Tk5BATE+P2PN6YU6PRoNFcRph+ZEt46DdY+TxkLIK1r8Gpv+GOjyCoblU3DlApeP3ObnRpHs7LP+9Db5RIaBxsyVlVymU2X9iEqKCK46bfQL3wojZYTl2s2n7SHkcanPV3wtSu0nWyPVRvovq9HtyCBQuYOHGiTQ9UhULBhAkTWLBggdvzqNVqkpKSSE9PtxwzGo2kp6fTp08fT5flszmrRRUAt/4XbnsflAFwZBV82A9Ob/fN69UAmUzGiD4JfPVwCl2ahzGqb4LlnH0aTkLjCg2u4pttqEYzNRglh31aBQ0Dh2EiTpPtnVPbe3AeCzi9Xu9wP+vAgQMYPTTPJkyYwMcff8xnn33G/v37eeyxxyguLrZ4QEeMGMHkyZMt47VaLTt37mTnzp1otVpOnz7Nzp07OXLkiNtz+oye98PDqyGyFeRnwYI02LbQd7p3DUhJbMzyJ69jpJWAi40ItBljb6JWFwc3fvF2rpn9B7/vzfbuYgU+x51y4XIHEs76R8/Gi+pCDavtBtAeZzKMHj2aMWPGcPToUZKTkwHYsmULs2fP9liIDBs2jPPnzzN16lSys7Pp0aMHK1eutDgJsrKykMsrZfCZM2fo2bOn5fmcOXOYM2cO/fr1Y+3atW7N6VNiusIja2HZY3BwBSz/D5zcCoPfAnWQ71+/Brw7vCepb6+zPE8wC7iKz99QzR7cr/+YBNuHfx7j5s6Xt8UgqLs4EllO9+BczFPbnbU8FnBz5swhJiaGt956y1I2qVmzZjz77LM888wzHi9g/PjxjB8/3uE5s9Ayk5CQ4FZAoKs5fU5gBAz7Cjb+F9Jfhl2LIXs33PM5NG5d7eX+ok3TEBKjgi3pW40rYpQ8dTIY66DGKnCNO/9lDgN9rffgrDMZ3NiDc1QqX4aRILwr+DwWcHK5nOeee47nnnvOEi/miXPhikAuh2ufhuZJ8N1DkPMPfHQD3D4fOgz29+qcknmhMjfV/IU278Hp3QwTMQpva73Dnf+xavfgrHJRXelwzvbg5KUXWah6E6NcCcYhpnvIC9RolrCwMCHcXNHqenj0T4hLgfICWHofrJoGhrrZbNnRL7nFi+q2BufNFQm8wYWicqb/tNdp9Rh3cGsPrppke3Ai4E5vJ3ppGjcodtGX3XBu32Wv057Lqiby3Xff8c0335CVlYVWq7U5t3173fMe+pWwWBj1C6yaCpvfhw1z4XQG3LUAQupmbmxUSGUKjULhmQZX3V6doPaZ9MMeVu3LYdHG4xyfXdWCcM9ErXrMdg+u+mR7qGw8k1+qM71wxiL49TmUBi3HjdH8R5rAspgu1S/ITTzW4N555x1Gjx5NdHQ0O3bsIDk5mcaNG3Ps2DEGDRrktYU1KBQqGDgL7loIqmA4vh4+vB6y6laTnok3twPg/4b1sByzmKhiD67eYl+81J7L9aJa78G5U/ASKjU4bUkxLHvc5IgzaClplcYQ7UwOyRKqXYsneCzg3n//fT766CPeffdd1Go1zz33HKtWreKpp54iPz+/+gmuZLrcAY+sgaj2UHgWFg2GzR/UmVCS8Te1Ze+MNK5r28RyzFMTtY68FYEV3ih1VV09OK3BaEnnc9U4JixQRUtZNguNL5gccDI5pE4nd/ACCgiu8Trt8VjAZWVl0bdvXwACAwMpLDQlbT/44IMsWbLEu6triDRpD2P/gM53gFEPKyfBd6NNift1APuA38pAX/duEoOQcHUOXTWVYC7XRNXZxb0WlxucjjUTcWIVP6tfpJP8BMagJjDiR5NDTuabuh8ezxoTE8PFi6byOvHx8ZZeqJmZmaJ3prtoQkx7cANfB7kS9v4PPr4Jzl1ekQFfonSjmsglq+Y0wkSte9gLInvc86I6cjLYXllc0eHeoXwz6GH1dNTfPUCYrIQMY1uy7/3N5IjDPTP5cvBYwN1000389NNPgCno9+mnn2bAgAEMGzbMYb9UgRNkMrh6HIxaAaHNIPeQScjt+c7fK7OhOg1OkiRGLfrb6nmtLEvgAd7II64uFxVMfRnAgTAsOg9f3g5//R8AS+SDuVf7EhcVVSuXeLsenMde1I8++siSkvXEE0/QuHFjNm7cyJAhQ3j00Ue9vLwrgPgUeHS9yUw9vh6+H2NK2B/wCiirFgSsbZSWckmOb5INRy7YNKgRGlzdo9r9Uzf+zxztq+mrmKgOwp+ytsC3I017zqpguO1dFvweha6kqFbStTzS4PR6PTNnziQ7uzLf8N577+Wdd97hySefRK32/w1ZLwlpAg8ug2srelpsmW9yQOSf9uuyoPpk+/nrjto8F2Ei9Q9vBPqCKVTEMlaSYPN8WPQvk3CLamfae+5yZ60m3Hsk4JRKJW+88QZ6fd0MVK3XKJSQOg3uXQKacDi11RRKcmxd9df6kPAg05fRUaWQPafy+etIrs0xkclQe3yy/hi3vLuei8Xa6gfXELf24Co0uECp1JTBs/J5kyOt8+0m4da0g+m82lSJqMyqT6+vFH+P9+D69+/PunX+vekaNB3+BY+uheiuUJILXwyF9W/7rZBmUnwkAFszL1VxIs3/82iV8UK+1R4zf9nPP6cLeCf9sM9fy9UeXJC6siR+a9lpPi5/Dvb+YHKgDZxtiv/UhFquM1sFjrY9PG1cVR0e78ENGjSISZMmsWfPHpKSkggOto1dGTJkiNcWd8XSKBEeXgW/PAM7v4L0GaZ9uaEfmJL5a5HucRGoFXJyi8o5fqHEUkbpeG4xv+4xFVvo2CzMkgYk9uBqn1OXfF+Hz7GJavrRDQtQUaI10ObcKp5RzyJEKjM5zu5eBPFXV7nOvK9bG0VUPRZwjz/+OGDqXmWPTCbDYDBUOS64DFSBcNs8Ux7rimdN5Zc+6gf3fAHNutXaMgJUCrrHhfP38Uv8nXnRIuA+Wn8MowQ3tm9Ck1CNlYCrtaUJKrhUUjMT1Z3fJMe5qKYLIzQwVvkF95/8FWSwQ96Fno/+z2kqoroW+3x4bKIajUanDyHcvIxMBkkjYcxvEBEPl47DpwNgx1e1uozeCaby61uPm+IfzxWW8V3GKQDG9WttaUwD1WtwG4/k0mdWOmsOnvPRaq88LtVwD86d+FVn9eCiucj/lb3IGOWvAHygv5WJATNc5lm7E1vpLfzWNlDgAbE94ZF10PZm0JfBj4/DT0+BrqxWXr53K5OA+7tCwC3acByt3kjP+AiSWzVCrXRfwN33yRbO5pcxeuHfLscJ3OdiDTU4d3C0N9bTsIflmil01O2jQArkjYiXeF0/3FTyyAXm9D/rPThfKf5um6ilpaWkp6dzyy23AKZOVOXl5ZbzCoWCV155hYCAAO+vUmBqYjP8a1g/B9a8Bts/g7M7TYU0IxN8+tJJLSORy+DEhRIOZhfyxeYTgEl7k8lkFpMDRJhIbaG1Sr/KK6lZuIXHYSKSBBvm8gmvoJBJnNa05r6CJwiWtQMKqg3WVSudV6jxW+Pnzz77jA8//NDy/L333mPjxo3s2LGDHTt28OWXX/LBBx94eXkCG+Ry6PccPPA9BDaCs7tMDW4O/e7Tlw0LUNGrwky97+PNFJbpad0kmAEdTWXgrU1U4WOoHUq1tttBjirk2mP9Q2SNR3twZfnw9QOwejoKmcT3huv4tP1HnJBiLKla1UkpiwZXCz+Gbgu4r776ikceecTm2OLFi1mzZg1r1qzhzTff5JtvvvH6AgUOaNPfVEizeRKU5cHiu+GPV8Houz3Qqbd0Qi6DCxX7PY9e3xp5hbvfvKcCQoOrLUp0trGoZ9zoaGb9/+QpMoDsisrUB5YjKdRM1o3hGd04goJNISCWVK1q5qrNZuJuC7gjR47QtWtXy/OAgACbhjDJycns2+e9SpyCaoiIg9G/Qu+HTc//fAO+vBOKL/jk5bo0D2fMta0AiAkL4LaesZZzCiv7pbo9OKWjgCqBx9hrcCXa6n/cnH327vwk3VCWDp+kwsVjEB6HcdRKlhj6AzJLEcvKaiKu/49VDoqo+qpQh9t7cHl5eTZ7bufPn7c5bzQabc4LagGlxtSxq0Uy/PxvOLbGlP1wz2fQopfXX+6Zm9sTolFxbdvGaJSVfXHlVjdOdd9TuVwmYkm8gL1Ac0dztnYGWeNKuKjRMVX5OQ/kV/Qabt0f7vwEvTocWAmYtjAASnUVAq6adbhsRemvxs8tWrTgn3/+cXp+9+7dtGjRwiuLEnhI92EwNh0atYaCU7BgIGz92OsbYgEqBf9ObUtSy0Y2xxVWAk5rMPLE4u1sq/C42iM0OO9gFiZm3AmaVTnZg3NKXhbfqmfwgDIdIzK4YTLc/y0ENbJ5PXMrQDPVJSPUSRP1X//6F1OnTqWsrGpoQmlpKTNmzGDw4LrbMarBE93Z1JO1461g1MGKifDDI6AtrvbSmhKoUtg8/2X3WR7+fBsnL5ZUGSsEnHewN1Hd0eA82oM7sho+vJ7u8mNckkJ4o9ErcMMkkJv+r60T7c0anBlXFX2hUsDVRiaD2wJuypQpXLx4kfbt2/Pmm2/y448/8uOPP/LGG2/Qvn17Ll26xJQpU3y5VkF1BISZMh1ungkyBez5Bj7uD7lHfPqy9gIOTKELj3+1nTI7TUPpqRYhcIi9iWpfusgRKiet+GwUfaMR1s6GL++C0kvsMiZyS/mr7AnqbXONtUANDbDd6apOg6vMRa1cs1lgKrz8A+j2Hlx0dDQbN27kscceY9KkSRa7XSaTMWDAAN5///3a6R4vcI1MBn2fNAUHfzsazu83eb6GzoNOt/nkJQPUtgJOLjM1F9lzOp/pP+1l9p2VqWXe/gJfqZTaeVHd0eCqM1EjKDR55I+sNh1IGs09G/pRjppWdlqZWaDKZRAS4FnGp8VEteurChDk4MeyJnj0c9qqVStWrlzJ+fPn2bx5M5s3b+b8+fOsXLmSxMREry5MUEMSroVx6yG+L2gL4ZsR8NsLYPB+DS57Da5XQiPeGd4TmQyW/n2SpVuzLOeEieoZJVo9ReV6covKbZwBVTW4mpmoXWXHWK55wSTclAGmwg63zqUcU41He63MbF4q5XKC1fYanOde1JKKGLogzWV1MnXKZdkLjRo1Ijk5meTkZBo1alT9BQL/EBoDI38yaXQAm96Dz4ZAYbbr6zzEuo8qwHVtoriubROeGWBqQzjtp70UVcRICQ3OfQxGiU5Tf6PLtN/oNXM1E7/dbTlnvwd32U4GSaLD6e/5Tj2dFrJciGwFD6+GHvfZDLMXWgYrkzJIY/sDdzlxcOb3E6T2owYnqIcoVKY9uXs+B3UoZG00hZIc3+C1l+gRF8Ej11dq8IO6NgPg8RvaEBqgpFxvJDvfFIgqNDj3MWs1Zr7ffsrydxUB52QPzroAqcpeg9OWwLLHue7gTDQyPasMSSZHVUxX7LH/bzNrjEqFrIpZWe0enIMy+OZqwI72c2uCd/VBQd2l023QtDN88yCc2wef3QoDZkCf8dV/I6tBJpMx5V8dSe0YTanOQJumIYAp5i08UEVhmZ6CMvc0uOz8MqLDNF4vfNjQKNG550W17qiltHYyXDhq2rbI+Qcjct7Q3cOHhlvIdFJv0P5/w2xeKuUylAo5aqXckh9bfZiIuZm4tQZn+n7Yt62sKUKDu5KIamMyP7reA5IBfn/RJPDKCrwyfXKrRvRr18TmWGhFCIG5wYjSiScPYNmO01w9K50p/9vjlfXUd1wZnVU1OMejrU1XlTnQ98AvJsdTzj8Q3IRfenzAfMMQJBfiwL4eXKXX03RNsJVp6W6YiE5fdU8xUJioghqhDoY7PjJlQMhVsP/nii+7b9Lswio8bIVuaHBzfj8IwJKtJ32yloaEu3Fw1vtcAXIjrJ4OS++D8gJTMdVH/+R0ZG+H11pjr5WZX8+sjQVZORrcDhMxWjsZ6oAXVdBAkMlMOawPrYSw5nDxKHzSH3Z7v1iCRYOrqHbhypMnEvVtkVyEttmbqM40OG2FgIsinwnZz1t6k3L14zDqFwiLdbOzve3/m1lwmn+wgmw0ONc4cjKUNGQTdd68eSQkJBAQEEBKSgpbt251Of7bb7+lQ4cOBAQE0LVrV1asWGFzftSoUchkMpvHwIEDffkW6ictepmqkiTeCLoS+GGsqQ+E3ns5xeZEbHc0OHdCHa4kXBUuKLVzQBicpD3pDBJXyQ6xXDOFDmU7Tb1J71oIA2eZHFBuYv/fZv4xMmtj1uEd1cXbOcpkaLAm6tdff82ECROYNm0a27dvp3v37qSlpXHunOOS1hs3bmT48OGMGTOGHTt2MHToUIYOHVolT3bgwIGcPXvW8liyZEltvJ36R3CUqb7c9c+Znv/9CSwcBHneMRPDquzBOS+tJFoO2uJSwJkT2ys+Toc/DpJEQMZHfK1+hRjZJc6q4uGRNdDlDtthbtQTsd9Xq/SimkSItWkZGey6P7KjkuUl5Q3URH377bcZO3Yso0ePplOnTsyfP5+goCAWLFjgcPx///tfBg4cyLPPPkvHjh155ZVXuOqqq3jvvfdsxmk0GmJiYiyPyMjI2ng79RO5Am56Ae77BgIi4HSGKZTkSHqNpw6124OzdjLYJ1tfaRqcoyoeh3IKeWjR3+w+leey6IpZ4wmt0JyqfHblRfDdQzRePxWVzMDPhquZGTsPmrR3sI7q12rvG6oM9DUJq2CrWLjIINeaocpBmIjZ5K4Tgb7eQqvVkpGRQWpqquWYXC4nNTWVTZs2Obxm06ZNNuMB0tLSqoxfu3YtTZs2pX379jz22GNcuOC8Tlp5eTkFBQU2jyuSdmnw6Dpo1h1KL5rqy617o0Y9Wc0aXGHFHpx95ZGsCyWWDudXkgb3295sus/4nT8O5Ngcf/izbfxx4BxD3tvgsoyR2clg3uO00YbPH4SPb4K9PyDJlEzXjeBJ3ZOUy4M8WuPPu85Y/rbX4Mx7ZpoKjcvayRAZ5FqDc5TJYDa5G1Sgb25uLgaDoUoOa3R0NNnZjqPts7Ozqx0/cOBAPv/8c9LT03n99ddZt24dgwYNctr1a9asWYSHh1secXFxNXxn9ZjIBHjod7hqJCDBmldhyTAocVz+qDrMGpw5Ds66dtyJ3BKuf3MN187+A7iyNLhHv8igoEzPQ4u22Rw/bVWZ1y0NruLztexn/fMDfHQj5B6E0GYcGbyURYaBeFpo7XBOIU8u2WF5bu8ZzS4wVRWKCdMAdhpcdSaqg5Ll5mKZDUrA+Yp7772XIUOG0LVrV4YOHcry5cv5+++/Wbt2rcPxkydPJj8/3/I4efIKD1NQBcCQd0x9WZUBcPh3U0/WMzuqv9YOc62wQgc9A7ZkmrTqwnI9RqMkvKjY7lG6swdn1pAlfTn8Ogm+Gw26Yki4Dh79k/yoJKurHM/nSFM8ZVcC3d6LejbfJOCahQcCEKiy1uBcm6jmpjM2e3BmE1XdgEzUqKgoFAoFOTm2anpOTg4xMTEOr4mJifFoPEBiYiJRUVEcOeK4bJBGoyEsLMzmIQB6PgBjVpm0urws+DQNMj7zqJCmRYMrNWlw1jeTtcfsYonWEtLQUNiaeZF//Xe90+KfjnDlhLGm1EqDi+Yid/3zGGypaPp07dPw4DIIaeq4aq4b2Ot79l7UbIuAM3XRs9bgIqoxUc0anLUXtUGaqGq1mqSkJNLTKzezjUYj6enp9OnTx+E1ffr0sRkPsGrVKqfjAU6dOsWFCxdo1qyZdxZ+JdGsm6kna7tBYCiHn5+CH8eDrvomJ1B1D85aK7EOVj1yrsjmuoawH3fPh5vYd7aAu+bb7g+X6w1Ob2SlGx3KJEmy7IH1qOhN2qJoN2jC4d7FkDodFKYfFneq5jp6HXuNzV7gna3ILY6pEHDWmlcjN72o1j9oZhO1wYWJTJgwgY8//pjPPvuM/fv389hjj1FcXMzo0aMBGDFiBJMnT7aM//e//83KlSt56623OHDgANOnT2fbtm2MHz8egKKiIp599lk2b97M8ePHSU9P57bbbqNNmzakpaX55T3WewIjTDdO/6kgk8POL+GTAaYGJNVgvwdnrZWYK4wALN6SZXPd+iO5Xlh43SPrQgndpv/utEmMdUK8MxO1XG/EKEmMU/zEEyefoYmsgJzANqYQkA62VbV1No1d3F+nvcZmn6qVbWeiWgvsak1USxyclZOhwkS1L71UU/wu4IYNG8acOXOYOnUqPXr0YOfOnaxcudLiSMjKyuLs2bOW8X379mXx4sV89NFHdO/ene+++45ly5bRpUsXwNSAevfu3QwZMoR27doxZswYkpKSWL9+PRqNxi/vsUEgl8N1z8CD/4OgKMjZAx/eAAdWuLzMvAdXVK7HYJRsNs7NoSNQuWlt5qzVHlDWhRLWH7ZtclRfefePw5TrnWtV1mE0zgRcWdElPlK9zSTVUuQY+d5wHR+3+xAat64y9nL7HlTJJ7V6KkmS1R6cSYPTWDW0qc6LatZSjVLlD16Jj0zUOlFNZPz48RYNzB5HjoG7776bu+++2+H4wMBAfvvtN28uT2BN4g2m7IdvR8GprbB0OFw7AW58wWIWWWNdzrqoTG+zB1dkJeBy7AScuf8qwPVvrgHg+8f6ktSyfscz2gtye5Q2GpyjCf4hZOkD3KzIRCsp+bPtszzzTxfux/bH+5ttJ4mLDELrxh6coxH2XlNrDe5Sic4ipJtWeFGtBWl4YHVxcJVzma6TU6YzXd/gTFRBPSS8uSmPMWWc6flfb8OXt0NRVS1Lo1RYft0LynS2Glx5pWfVHAvn7DnAzpN5NV+7nzGbds6wjhO0927eIf8TPklFmZfJKSmKEbJXOBB7JyCzMf3/OZ3Pc9/tZvjHm217j3qwTvs9N+vn5v23qBCNpX2ktVZaXd8N61QuvVGy6RDW4ExUQT1FqYZBr8Odn5ryGzP/NGU/ZG2pMtQ64d7a7LI2URV2KoOvGgH7m+oEnFxWVYPToOVV5ae8rZ4P+lIKW9zALeWvclzd3lKuyDqG0LrL/eU6GewlnPW67D2oAPGN3A8itvYU6w1Gi3kqk0GAyrsiSQg4Qc3oeheM/QOi2kHhGVj0L9g83+ausU64t/aOFlgJOHuv3eebTpAw6Re2HHOegVIfKSzXuzxv/SkYjBLNOc836pe5X5mOUZLBDVPYf9On5BFKkFphMfesNTjrBs+u9vvMOMpFtd+Ds/7vMe+/xVgJuAGdopk8qANLH7m62tdTyGWW+bQGoyUPNVCl8HqhUyHgBDWnaQeTkOt8Oxj1sPJ5+O4hUz4ktkUvrU3UIqvgX3uvnfnGHPbRZt+uvY5hfX8HZa1hueYFS2/SUbrn4IbnKanYrwpQKSwmbW5ROUu3ZlFYprOYjWCrJUuSxNHzRdw4Zy3fbnMdzG4vZ6wFj9lEtdbgZDIZj/ZrzdWJjd14jzJLC0O9QaqsBedl8xSEgBN4C01oRRme2SBXwt4fTPmQ5w/aFL10ZqK688P9/hrf9netC8hlMmQY+bfie1quHEmkrMjSm/RPY3fAtkGL2dxbfziXST/sYdL3eyyZAlBZh8/MpO93k5lbzLPfVTawcRgHZ/+8Gg3OU6wriphbIHrbgwpCwAm8iUwGVz9mckCENjPlQ350Izfo/gJMwb55JZU3nLWAcyeawdqz2lAJlwpZqHqTp1XfI0PiS31/7tZO4zSVpeDNm/KBaoVlD87ML3vO2mjJBXbOGnNGiTWOvajOf3Ec7cF5inVFEV/loYIQcAJfEH+1KZQk4TrQFTMm+2WmKj8n+1KhTTK5tfcst8h7RTbrLaczmFf0H25Q7KJUUnPiurd4UT8GLbZhFyUONDhrDE72OSXcj4uzl29aq708+yDfy8G68UyJj1oGghBwAl8R0tSUD3nNfwB4SLmSW7aPJZrLq0pipiF6VyWjEbYtgAUDiZbOk2mM5nbty+S2udPh+FKrFnv2FZJlMts0N2stGXCc7+vgM7UXm+awHUdBvpeDdeOZShNV7MEJ6hMKJQyYwS8d51AgBdLFeIDlmin0ke+97CkvN3m8LuBIOAdQjrTsMVj+NBi0bFJdzW3amRyQ4p0K88ry3soqPS7kMplNyIi9iepIg3Mn0De/Ymshv1Rn0byjw7ywB2c0ChNVUL/JjRvArdpX2W+Mp4msgC9VrzFO8ROehZ6acFVCqK5wMLuQjBOXqhy3r3fXUpbN/9TTkO9easrxTZ3BrLAXKSAYcF4PrlRnbaLa3sJyGRhsHDmVAk6SbE1N19hKuEslpv1Ps/bWKFhNQA3Ki1t7UX3V1R6EgBPUAqEBSk5IMdyuncH3hutQyCQmqZbykeptwij2aK66XBTz1KUSjucWkzb3T+78YGOV89bCZYB8Gz+rX6CjPAtjUBMY8RNc+x+bzX1n5ZKsSwtZ54CCKX7NWawhONaAHVcTsX2eV6EJmvffYmqgvYFtZy1rjdTbCAEn8DnmkkllaHhGN47JujGUS0puVmTwk/pFOspOuD2X9U3/9qpDzF931OvrdQet3mgjSIxGiWtfX8MNc9Y6vaZcb0SBgeeVS/hY/TZhslK2GdtRNPIPaHUdYOu9rM5EDVApSIiyzSCQyWx/BGw0ODxwMtg9zy/R2ey/xUbUTMBZh4lYWgYKDU5QH7FOuAcZSwz9uUs7nVNSFAnyHP6nnsqd8j/dmsssVM7ml/JO+mFm/3rAJt+yNigq19PuxV9JnLICSZLQG4yU6R2XP7JGn5/NF6pZPKb8GYBP9YO4V/siuuDKEvzWPgNnymqJlYnasnGwjSdVLrPV4MxJ7GYc78E5cDLYqXBag5FSnYFsuzpwl4vSKkxEeFEF9ZowB9Ul9kim4NW1hu4EyHS8pZ7Pa8pP0OA61s28v2R949a20br+UGVRge1Zl0h5LZ1OU6upYJO1mUZfDaCvYh9FUgBPaJ/iFf2D6FFaNC69wcj2rDzLJU7LJVkJBJVCTsvGlVqcvQZnj/WpEq2e7zNOMW9NVS3YURRcXomOM14IEQFQWzWeESaqoF5jq8FVkkcoo3XP8rbuLoySjPuUf/CtegYtZM5rvzmq9OtoQ98XSJLET7vOcCinsvpwQZm+mgBkidGKX2HRYJTF2Rw2Nuc27Sv8YqzM2TQLpEUbj9tc6UzA2QuENk1DLOfkMpnT6+xN3k5Tf+OZb3fZHFtz4BxbMy9yILtqZ7m8Ep3X9uCsG89UdrX3vgZXJ+rBCRo2jjQ4MxJy3jHcwU6pNf9VzaObPJPl6in8R/cEa409qow3OLh5Nx694FYOZE1ZvvssT1l1mgLXpdWDKON11UfcqtgMRshLvJXb9t1OCbbCQW8wcrFYy8xf9tscd+YwNpuogRVezDZNQ/htr6lPiUzmupeDPfGNgsi6WGJ5PnrR307H5pVqHeahXg4qpTkOzkqD83LTZxAanKAWCHHD9PjT2J1byl9lpzGRCFkxC1Rv8rTyO+TYNYd2FAdXS6EjjqoKOzMHW8tO86P6JW5VbEYnKWDQGxy9/r9VhJt5jg8dOEvcMVEB2jYNtZyTuViTI+bc3b3KscSoYIdj80p0lUG+ETUzUVXyykyGUpFsL6jPyOUymrtxQ5ymCfdop/GFPhW5TOLfyh9YqHqDSCrNJUc3fW3twdlnBYBjbekW+SZ+Ur9IW/lpsqVIhmlfgpRHKXcSpHyuoJwP/6za38JpmIiu0osKdiaqXOZRwx777YPBXZvxx8QbHI49danEom3V2ES1eFElis1hLz4wUYWAE9QKXz2cwqw7ulY7TouKl/QP8bT2MUolNf0Uu/lZ8yLdZCYNx9FNX1uxv9UJOBV6pio/5z31uwTLytlo6MTg8tfYLrUDnNdm+8BJqIszOWWOpzPHwCU2qdS48kt1Ds14Z4RobAWceT/MEfvPFgIQEaSqcWlx6zg4iwYnTFRBfSUhKphBXZz3rrXnf8brGKp9mUxjNC1kuXyrnsH9itUYjVWFxHtrjnDsfJGDWbyLo+bVZgEXzUWWqGfykHIlAPP0Q3hQN5kLhFvGOssiyC10XGjAmYlqDvUwCwlr0y4qROPRHpw5RtGMWTtUOyg7vv+sSZOuqfYGlWsX9eAEDYYqnZqq4aAUzxDtq6w09EYj0/OqagFN0v8D2pIqY8cv3lF1Ai/jSIPTGyX6yPeyXDOFXvJDFEhBjNVO4E39vRiw1UjWHnTsHXZWScXZXprWIuAqP8+x17UCoHNsWBUBN7ibqR+wI3kZEqC0SdgvrQi/iXDQ+u9oxY9ITR0MUFm2XGswChNVcOVSSBDjdP/hVd196CU54Ye+r2i8YrtndfJiVaFXU/JKtKw7dN4iMOzTnkDiyA+v8KXqNZrICthvjOdW7UxWGXs5nG/J1iyHx887E3AOgnKfWLzdImitS5N3ig0DTBqlvYALUDoXHAq5zCbdy+zAcNT6z5zmFVPDGDio9KKKXFRBw+Gyy+3L+NhwC/drXyBPHgnn9hL79SDS5JUhDSqlnMM5hcz4eS+PfrGN1ftyarTU4nI9PV5excgFW/l803HA1kQNo9jSm1Qhk/jOcD23a2dwQnLfDDfjbMvsjZUHqxz7ZXdlj2BrM9LcFMaRgNOoKve7HGEt4MwmamSw89CeWC9ocGYvaolWb9FUg1TCRBVcwWyROnJzyUy2Gtuj0BXyofr/mKRcjAIDaoWcAf/3Jws3HOe3vTk8/Pm2Gr3Wgr8yLX/P/vUAUOkk6Cg7wU/qF7lZkUG5pGSS7mEm6h6lDO82Fq+uh6p1+z1z4KzBKFVxMpgF2JZMx7X4rHs4mAVco2DnzZtrmqYFlWu3Lqnu7Z6oIAScoBbxRsOkc0Ryn/YFPtb/C4BxyuUsVr9KtCKv5pNbsfFoZTcva+/nnfI/+Z96KgnyHE5JUdylnc5Sw01cjnraJLRmAlFlpXmZZV1+qY6sC7bmenVljTSqqiZqhIvu9DVN04LKXFRzIU2VQmZjcnsLIeAEtYa3GsLpUfKq/gEe0/6bIimAFPkBPi17hl6yA156BSi3S57fdPA0ryo/5S31fAJkOtYaTIHJe6TEy36NVo0dB9S6iyMT9UB2IUv/tu2Y5WoPDpyYqA6cDGa8o8GZ1msWcL7IYgAh4AT1mF+NKQzRzuSQsTlR0iWWqmcyRrECc+iv0Vi5ge0uaw6cY/OxCzZJ7y1k5wn66hZLb9K3dXcxWvcseYQ6n8gNQgOUNdLirL2o9qXLrdFU00zZ2kQ174dZOxnsNW9veFEtJmqpOQ/VN1mjQsAJ6jXHpFiGal9hnaYfSpmRl1RfMk/1X0IoIXHKCrpO/428Em0VjcwRK//JZvSiv7nXqhdrP/kuflbb9iZ9x3AHkoe3zl+Hc6sc06jkHnWEt6ZZeIBNSSNXAs7FKQCSWkZa/r69Z3PA1kSVpMr5wwKUXhFG5kyGixWFCnyx/wYi2V5Qi/iqGG8JAcwJfpbVRa14SfkFgxVb6SA7yTjd0xw2tqDHy6sAGNevNZMGdXA6z7gvMyx/yzDylOJ//Fv5A3KZxE5jIk9o/23Tvs8THvh0S5VjraKC0SgVl1UN5Yb2tutw1avC1X4awHMD26NSyAhUKxnXz2Ryd28RbjMmPFDFxWKtV/bfoLJkubnLWkINzXVnCA1OUHv4MKWqVG/kC8PNDNNO5YzUiNbys/yofokh8g2WMfPXHbUEq7oigsrepHKZxBf6VO6x603qDdo2DSUu8vIExo3tm9o8d2Q2tooKZv1zN1brZAhSK3lhcCcmDGhnySZoG21rfkdUVITxxv4b2JrXwWoFL93SySvz2iMEnKDW8GXDGPNe2w6pLbeUv8Z6QxeCZOW8o57HdOUiVJj2en7aecbh9eZaad1kR1muecHSm/Rp7WO8pH+oSm/SmhIZpKJ/x6bEXaaJek2bKJvnXZqHM/+BJNZaJcoHaxTENQpy6tz5ckyKy9ew9mqGVzgdvLH/BpVeVIBZd3ajlZMKJjVFCDhBreHLnHjrhtIXCWOkbhLv6ocCMEr5O9+oX6YZF/hv+mFmLt9XuSZJ4si5Io7nFnOfIr2i4GaupTfp/4zXeX2tMWEBbJ7Sn9AAldM9uMVjbYVP97gINk/uzxt3dWPhqN4O98EGdokhISqYhaN7kxgVzNRbOrtcx7Vto1yej7Cq4+dtDe6q+EiC1QrGXteKId1jvTKnI+qEgJs3bx4JCQkEBASQkpLC1q1bXY7/9ttv6dChAwEBAXTt2pUVK1bYnJckialTp9KsWTMCAwNJTU3l8OHDvnwLAjeIDFKR3KoRyQmNXIYheAMjct7S38ND2onkS0H0lB9huWYK18j38IlVEO/Dn23jlrd/Z/s79/Ka6lM0Mj2/GXoxRPsqB6R4j17zneE93RrXpmmIxXMZ37iqgHtlaBd6xkXaHPt4RBIx4QHc0yuOGzs0rXKNNTe2b8ofE28guVUjN1fuGOt81F4JjZDLIDmhZnOa6RQbxu7pabww2DemqRm/C7ivv/6aCRMmMG3aNLZv30737t1JS0vj3LlzDsdv3LiR4cOHM2bMGHbs2MHQoUMZOnQo//zzj2XMG2+8wTvvvMP8+fPZsmULwcHBpKWlUVbmOjJc4FtkMhlfP3I1Xz96NYtGJ9vUMfMVfxiv4hbtq/xjTKCxrJAvVLN5QrEMjEbKdAaOHtzF/9TTuFOxHoMk4zXdcB7VPU0hnpuOLSID2TP9Zra/NMDlOOsik9GhVTWiB69uaeNV7N4inKYOxrmLObbNmlF9E6q9LiKw0jnxxI1t2D09jb5tXGt9nuDK8+stZJKz3mS1REpKCr179+a9994DwGg0EhcXx5NPPsmkSZOqjB82bBjFxcUsX77ccuzqq6+mR48ezJ8/H0mSiI2N5ZlnnmHixIkA5OfnEx0dzaJFi7j33nurXVNBQQHh4eHk5+cTFhbmpXcqcETX6b9ZkscVcplHpX48QYOWGcpF3KtcC8DBsL58VdyLifqPCZOVcl4K40ndU2w2eqZRNA5WW3oyHH3tX5abNmHSLwA8cn0i0WEBjOjTEoNRQpKqhkSYx4LJFP3xiWsAk1Pk9ZUH+GREL/p3jOZymfXrfj5cZypOcHz2YAxGyS3hMvbzbayqyOk9PnvwZb++L3D3HvWrBqfVasnIyCA1NdVyTC6Xk5qayqZNmxxes2nTJpvxAGlpaZbxmZmZZGdn24wJDw8nJSXF6Zzl5eUUFBTYPAS1g7XJs3dGGg9d08onr1OOmkn6R3hON5ZySUX7go28bHiHMFkpfxvbMbh8lkfCrUWF9/O5ge3JnPUvjs8e7FBodGwWyphrW6FSyAlQKRzGe/00/hpevq0zXz9yNV89XLn3Nq5fa/6ZnlYj4QbQrXkEUJn54K7m9PgNrQG4paLcUn3Er3Fwubm5GAwGoqNt/wOjo6M5cMBx2k12drbD8dnZ2Zbz5mPOxtgza9YsZsyYcVnvQVAz3ry7O4s2HufOq5oToFIw9dZOTL21E+cKy0h+NR2AXi0j2XbiEoO6xLDrZB6fjurNjzvP8Pmm4wzqYrr5ft51hhaRgRzLLSY2PIBPRvbmz8PnWbghk3K9kbwSHa8M7cJLy2CvsRUfqP6PePl5PtEPYrZ+OE/070hqx2heXLYHrUFieHIc17SJ4u3fDzFpUAf+OHCOaT/t5V9dY3j0+tZ0ig3jeG4xbZqGVOkhCvDEja3ZmnmRf3WtXjh0axFBtxYRDs95I6h2UJcYPnowyelrOKNnfCTbXxpg42yod0h+5PTp0xIgbdy40eb4s88+KyUnJzu8RqVSSYsXL7Y5Nm/ePKlp06aSJEnShg0bJEA6c+aMzZi7775buueeexzOWVZWJuXn51seJ0+elAApPz//ct+aoBYxGo2SVm9we6wkSVLuxUvS6aN7paIyneVYdZTr3HsNge/Jz8936x71qwYXFRWFQqEgJ8e2dldOTg4xMY7rasXExLgcb/43JyeHZs2a2Yzp0aOHwzk1Gg0ajXdL3QhqD5lMZhM4Wt1YgMaRERAZ4dHr+KLahcC3+PV/TK1Wk5SURHp6uuWY0WgkPT2dPn36OLymT58+NuMBVq1aZRnfqlUrYmJibMYUFBSwZcsWp3MKBIIGSi1plE5ZunSppNFopEWLFkn79u2THnnkESkiIkLKzs6WJEmSHnzwQWnSpEmW8Rs2bJCUSqU0Z84caf/+/dK0adMklUol7dmzxzJm9uzZUkREhPTjjz9Ku3fvlm677TapVatWUmlpqVtrclf9FQgE/qFemKhgCvs4f/48U6dOJTs7mx49erBy5UqLkyArKwu5vFLR7Nu3L4sXL+bFF19kypQptG3blmXLltGlSxfLmOeee47i4mIeeeQR8vLyuPbaa1m5ciUBAd6JwhYIBPUDv8fB1UVEHJxAULepF3FwAoFA4EuEgBMIBA0WIeAEAkGDxe9OhrqIeVtSpGwJBHUT871ZnQtBCDgHFBYWAhAXF+fnlQgEAlcUFhYSHh7u9LzwojrAaDRy5swZQkNDHeYZ+pqCggLi4uI4efKk8OL6CPEZ+x5ffsaSJFFYWEhsbKxNGJk9QoNzgFwup0WLFv5eBmFhYeLm8zHiM/Y9vvqMXWluZoSTQSAQNFiEgBMIBA0WIeDqIBqNhmnTpokKJz5EfMa+py58xsLJIBAIGixCgxMIBA0WIeAEAkGDRQg4gUDQYBECTiAQNFiEgPMTFy9e5P777ycsLIyIiAjGjBlDUVGRy2vKysp44oknaNy4MSEhIdx5551V+lPIZLIqj6VLl/ryrdQZ5s2bR0JCAgEBAaSkpLB161aX47/99ls6dOhAQEAAXbt2ZcWKFTbnJUli6tSpNGvWjMDAQFJTUzl8+LAv30Kdx9uf8ahRo6p8XwcOHOi9Bfu0rrDAKQMHDpS6d+8ubd68WVq/fr3Upk0bafjw4S6vGTdunBQXFyelp6dL27Ztk66++mqpb9++NmMAaeHChdLZs2ctD3dLtddnli5dKqnVamnBggXS3r17pbFjx0oRERFSTk6Ow/EbNmyQFAqF9MYbb0j79u2TXnzxRYel78PDw6Vly5ZJu3btkoYMGeJR6fuGhi8+45EjR0oDBw60+b5evHjRa2sWAs4P7Nu3TwKkv//+23Ls119/lWQymXT69GmH1+Tl5UkqlUr69ttvLcf2798vAdKmTZssxwDpf//7n8/WXldJTk6WnnjiCctzg8EgxcbGSrNmzXI4/p577pEGDx5scywlJUV69NFHJUkytReMiYmR3nzzTcv5vLw8SaPRSEuWLPHBO6j7ePszliSTgLvtttt8sl5JkiRhovqBTZs2ERERQa9evSzHUlNTkcvlbNmyxeE1GRkZ6HQ6UlNTLcc6dOhAfHw8mzZtshn7xBNPEBUVRXJyMgsWLKi2pEx9R6vVkpGRYfPZyOVyUlNTq3w2ZjZt2mQzHiAtLc0yPjMzk+zsbJsx4eHhpKSkOJ2zIeOLz9jM2rVradq0Ke3bt+exxx7jwoULXlu3SLb3A9nZ2TRt2tTmmFKppFGjRmRnZzu9Rq1WExERYXM8Ojra5pqXX36Zm266iaCgIH7//Xcef/xxioqKeOqpp7z+PuoKubm5GAwGS6MiM9HR0Rw4cMDhNdnZ2Q7Hmz9L87+uxlxJ+OIzBhg4cCB33HEHrVq14ujRo0yZMoVBgwaxadMmFApFjdctBJwXmTRpEq+//rrLMfv37/fpGl566SXL3z179qS4uJg333yzQQs4Qf3l3nvvtfzdtWtXunXrRuvWrVm7di39+/ev8fzCRPUizzzzDPv373f5SExMJCYmhnPnztlcq9fruXjxIjExMQ7njomJQavVkpeXZ3M8JyfH6TUAKSkpnDp1ivLy8hq/v7pKVFQUCoWiikfZ1WcTExPjcrz5X0/mbMj44jN2RGJiIlFRURw5cqTmi0YIOK/SpEkTOnTo4PKhVqvp06cPeXl5ZGRkWK79448/MBqNpKSkOJw7KSkJlUpFenq65djBgwfJysqiT58+Tte0c+dOIiMjG3RSuVqtJikpyeazMRqNpKenO/1s+vTpYzMeYNWqVZbxrVq1IiYmxmZMQUEBW7Zscfl5N1R88Rk74tSpU1y4cIFmzZp5Z+E+c18IXDJw4ECpZ8+e0pYtW6S//vpLatu2rU2YyKlTp6T27dtLW7ZssRwbN26cFB8fL/3xxx/Stm3bpD59+kh9+vSxnP/pp5+kjz/+WNqzZ490+PBh6f3335eCgoKkqVOn1up78wdLly6VNBqNtGjRImnfvn3SI488IkVEREjZ2dmSJEnSgw8+KE2aNMkyfsOGDZJSqZTmzJkj7d+/X5o2bZrDMJGIiAjpxx9/lHbv3i3ddtttV3yYiDc/48LCQmnixInSpk2bpMzMTGn16tXSVVddJbVt21YqKyvzypqFgPMTFy5ckIYPHy6FhIRIYWFh0ujRo6XCwkLL+czMTAmQ1qxZYzlWWloqPf7441JkZKQUFBQk3X777dLZs2ct53/99VepR48eUkhIiBQcHCx1795dmj9/vmQwGGrzrfmNd999V4qPj5fUarWUnJwsbd682XKuX79+0siRI23Gf/PNN1K7du0ktVotde7cWfrll19szhuNRumll16SoqOjJY1GI/Xv3186ePBgbbyVOos3P+OSkhLp5ptvlpo0aSKpVCqpZcuW0tixYy0C0xuIckkCgaDBIvbgBAJBg0UIOIFA0GARAk4gEDRYhIATCAQNFiHgBAJBg0UIOIFA0GARAk4gEDRYhIATCAQNFiHgBPUGR+WtvV7iWtCgEOWSBPWKgQMHsnDhQptjzgoJ6HQ6VCqVzTGtVotarfb4dS/3OoF/ERqcoF6h0WiIiYmxeURGRgKmhjsffPABQ4YMITg4mFdffZXp06fTo0cPPvnkE1q1akVAQAAAWVlZ3HbbbYSEhBAWFsY999xjU9rH2XWC+oUQcIIGxfTp07n99tvZs2cPDz30EABHjhzh+++/54cffmDnzp0YjUZuu+02Ll68yLp161i1ahXHjh1j2LBhNnPZXyeofwgTVVCvWL58OSEhITbHpkyZwpQpUwC47777GD16tM15rVbL559/TpMmTQBTTbI9e/aQmZlJXFwcAJ9//jmdO3fm77//pnfv3g6vE9Q/hIAT1CtuvPFGPvjgA5tjjRo1svxt3cjHTMuWLW2E1P79+4mLi7MIN4BOnToRERHB/v37LQLO/jpB/UMIOEG9Ijg4mDZt2rg8784xd19LUL8Re3CCK46OHTty8uRJTp48aTm2b98+8vLy6NSpkx9XJvA2QoMT1CvKy8urtO1TKpVERUW5PUdqaipdu3bl/vvvZ+7cuej1eh5//HH69evn0MQV1F+EBieoV6xcuZJmzZrZPK699lqP5pDJZPz4449ERkZy/fXXk5qaSmJiIl9//bWPVi3wF6JkuUAgaLAIDU4gEDRYhIATCAQNFiHgBAJBg0UIOIFA0GARAk4gEDRYhIATCAQNFiHgBAJBg0UIOIFA0GARAk4gEDRYhIATCAQNFiHgBAJBg0UIOIFA0GD5f1XMUGxG48AmAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.021308, + "end_time": "2024-02-29T06:11:45.537438", + "exception": false, + "start_time": "2024-02-29T06:11:45.516130", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 1595.086316, + "end_time": "2024-02-29T06:11:48.284797", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/lct_gan/42/mlu-eval.ipynb", + "output_path": "eval/insurance/lct_gan/42/mlu-eval.ipynb", + "parameters": { + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "path": "eval/insurance/lct_gan/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "lct_gan" + }, + "start_time": "2024-02-29T05:45:13.198481", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/lct_gan/model.pt b/insurance/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab7ab7d541b5873e84845f4d07e48e9f8fbe79d5 --- /dev/null +++ b/insurance/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56f554b5f68fa3c271fd6beeb2d4991d14b4ca2a19b19c76f2f10ecfbb38ca4a +size 38583573 diff --git a/insurance/lct_gan/params.json b/insurance/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..7e124407075e1d9ed88bd41594215a5a2a164360 --- /dev/null +++ b/insurance/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "ALL", "synth_data": 2, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "fixed_role_model": "lct_gan", "mse_mag": true, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/realtabformer/eval.csv b/insurance/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..2c6103776756f53b417dbcfa5ef099f7b9fb531d --- /dev/null +++ b/insurance/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.11524768834723566,0.02341634124876461,0.0005081756423253164,0.4052395820617676,0.10590226948261261,2.4253790378570557,0.22090712189674377,8.237027770974237e-08,2.0878190994262695,0.01770336553454399,0.26909443736076355,0.02254275046288967,0.16169530153274536,3.49789515894372e-05,2.493058681488037 diff --git a/insurance/realtabformer/history.csv b/insurance/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..93f63cfe1f2e518b91cb2d080decfa2b1621c5e3 --- /dev/null +++ b/insurance/realtabformer/history.csv @@ -0,0 +1,22 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.14850416241097264,1.180862516140678,0.0921927880990836,5.234412693977356,0.0,0.0,0.0,0.0,0.14999753049341963,320,40,54.21397066116333,1.3553492665290832,0.1694186583161354,0.18854508120566607,0.01377877084451029,3.3042451123972567,0.00011845146224525039,0.21803768426179887,0.0,0.0,0.0,0.0,0.01390348743661889,80,10,9.792181253433228,0.9792181253433228,0.12240226566791534,0.05718763165641576 +1,0.010435808167676442,5.020027183283946,0.0002433187393593883,0.40798311326652764,0.0,0.0,0.0,0.0,0.01056134452810511,320,40,54.643929958343506,1.3660982489585876,0.17076228111982344,0.06234871030319482,0.0054248009575530885,7.6202680438756945,2.544027604756138e-06,0.2834674768149853,0.0,0.0,0.0,0.0,0.005489112006034702,80,10,10.194044351577759,1.0194044351577758,0.12742555439472197,0.03268122207373381 +2,0.004800849179446232,4.060577614020576,1.805691692511324e-05,0.24826885210350155,0.0,0.0,0.0,0.0,0.004860438608739059,320,40,55.247488498687744,1.3811872124671936,0.1726484015583992,0.06760974442586302,0.002636889467248693,0.5023930853232741,1.9693377169005544e-06,0.46713720858097074,0.0,0.0,0.0,0.0,0.0027079362072981892,80,10,10.467901945114136,1.0467901945114135,0.1308487743139267,0.08990103090181947 +3,0.003688591261743568,5.662890090987792,3.4259154269070546e-05,0.2902778916526586,0.0,0.0,0.0,0.0,0.003743911211495288,320,40,54.36272192001343,1.3590680480003356,0.16988350600004196,0.08270897869369946,0.004029050027020276,3.6828645429341123,5.466211582216829e-06,0.22611289098858833,0.0,0.0,0.0,0.0,0.004080505215097219,80,10,9.939324378967285,0.9939324378967285,0.12424155473709106,0.10100285978987814 +4,0.004078761015261989,3.139242228523483,3.3808381526224254e-05,0.36921092458069327,0.0,0.0,0.0,0.0,0.0041462778390268795,320,40,54.406832456588745,1.3601708114147186,0.17002135142683983,0.08321290470194072,0.0015522451751166955,0.42041910818370526,2.725768419864494e-06,0.18216939978301525,0.0,0.0,0.0,0.0,0.0015822530665900558,80,10,10.175266981124878,1.0175266981124877,0.12719083726406097,0.08142163883894682 +5,0.0020072206414624818,0.8528530288420655,8.324493640320986e-06,0.22079015588387846,0.0,0.0,0.0,0.0,0.0020452409196877854,320,40,55.4995391368866,1.3874884784221648,0.1734360598027706,0.08714587027207017,0.00045264576037880034,0.6128407033349503,2.0517467608964848e-07,0.13081572577357292,0.0,0.0,0.0,0.0,0.00047156074579106643,80,10,10.349114894866943,1.0349114894866944,0.1293639361858368,0.07619310528971254 +6,0.0009020587680424796,0.19860290411693313,4.795286915437736e-07,0.16418010434135794,0.0,0.0,0.0,0.0,0.0009270869173633401,320,40,55.20178484916687,1.3800446212291717,0.17250557765364646,0.09331575823016465,0.005614329825039022,0.15103251073694537,3.9005249729129333e-05,0.43398396372795106,0.0,0.0,0.0,0.0,0.005694820835196878,80,10,10.202214241027832,1.0202214241027832,0.1275276780128479,0.12506430204957725 +7,0.002170501317596063,0.42862962205581423,9.560629908794694e-06,0.23468926995992662,0.0,0.0,0.0,0.0,0.0022109889338025822,320,40,55.481743574142456,1.3870435893535613,0.17338044866919516,0.09381236587651073,0.001958590082358569,0.8871201357003884,2.944644425895149e-06,0.20026710480451584,0.0,0.0,0.0,0.0,0.0019938025623559954,80,10,10.465904474258423,1.0465904474258423,0.13082380592823029,0.07660999405197799 +8,0.0032185729534830896,0.39595315401531933,2.223747077226177e-05,0.26023907188791784,0.0,0.0,0.0,0.0,0.0032680820706445955,320,40,55.13088512420654,1.3782721281051635,0.17228401601314544,0.09016182171180845,0.006755840044934303,0.042443403803736145,9.365343847704821e-05,0.8566916212439537,0.0,0.0,0.0,0.0,0.0068929205401218495,80,10,10.145907640457153,1.0145907640457152,0.1268238455057144,0.1274191069416702 +9,0.003263563236214395,0.8668862596564395,2.02128691412784e-05,0.2691699764691293,0.0,0.0,0.0,0.0,0.0033155382898257812,320,40,54.284446239471436,1.3571111559867859,0.16963889449834824,0.08654828406870366,0.0004874484846368432,0.003280633598478744,2.418919596181013e-07,0.12313968688249588,0.0,0.0,0.0,0.0,0.0005053632776252925,80,10,9.947452306747437,0.9947452306747436,0.12434315383434295,0.08960323911160231 +10,0.002170001242484432,0.33971318979765786,1.1028246948152631e-06,0.2432436312083155,0.0,0.0,0.0,0.0,0.0022117360727861523,320,40,54.17906904220581,1.3544767260551454,0.16930959075689317,0.0964421829674393,0.0009476982486376073,1.3159201624483856,1.8856890096896618e-07,0.13108705095946788,0.0,0.0,0.0,0.0,0.0009683694021077827,80,10,10.008543252944946,1.0008543252944946,0.12510679066181182,0.06394510199315846 +11,0.0008773179083618743,0.4401658920556809,6.324480833997878e-07,0.12429296048358082,0.0,0.0,0.0,0.0,0.0008975059775366389,320,40,54.14188766479492,1.3535471916198731,0.16919339895248414,0.09457911597564816,0.0003643055897555314,0.9265121433396416,1.4636429152004027e-07,0.07630779892206192,0.0,0.0,0.0,0.0,0.0003756950100068934,80,10,10.02190351486206,1.002190351486206,0.12527379393577576,0.08594365753233432 +12,0.0004941477713146014,0.05355267495711606,1.4848906236212407e-07,0.09103901842609048,0.0,0.0,0.0,0.0,0.000507950432574944,320,40,54.16036105155945,1.3540090262889861,0.16925112828612326,0.09392541642300785,0.00022847578329674434,0.661697834357119,3.0550005269969204e-08,0.056163723766803744,0.0,0.0,0.0,0.0,0.00023663823449169286,80,10,10.043348550796509,1.004334855079651,0.12554185688495637,0.08091261517256498 +13,0.00029569496205112956,0.04474922703773245,6.23439235618306e-08,0.05976610332727432,0.0,0.0,0.0,0.0,0.00030461719015875135,320,40,55.389015674591064,1.3847253918647766,0.17309067398309708,0.09439536663703621,0.00025476437076576985,2.0484021956655853,4.190348910498853e-08,0.05006699915975332,0.0,0.0,0.0,0.0,0.00026224629764328713,80,10,10.323888778686523,1.0323888778686523,0.12904860973358154,0.08619057663017884 +14,0.0003340222642691515,0.05788067737812881,5.194412635303129e-08,0.06943645351566374,0.0,0.0,0.0,0.0,0.0003443146288191201,320,40,55.98161721229553,1.3995404303073884,0.17494255378842355,0.091485915472731,0.00025505621306365356,0.46580720322454,2.3517513286774872e-08,0.053071103431284426,0.0,0.0,0.0,0.0,0.00026298246484657284,80,10,10.274444580078125,1.0274444580078126,0.12843055725097657,0.08523455495014787 +15,0.00045633949812327044,0.03235854215981604,6.432831875045203e-08,0.08093514840584248,0.0,0.0,0.0,0.0,0.000468703078058752,320,40,54.44696354866028,1.361174088716507,0.17014676108956336,0.09638399491086602,0.0005625217905617319,1.3924316422228002,1.6518343430860228e-07,0.09882423281669617,0.0,0.0,0.0,0.0,0.000577750973025104,80,10,10.127650499343872,1.0127650499343872,0.1265956312417984,0.07251062926370651 +16,0.0010068999348732178,0.41138206991290643,9.277023389098193e-07,0.12433609296567738,0.0,0.0,0.0,0.0,0.0010285568052495365,320,40,54.41342616081238,1.3603356540203095,0.1700419567525387,0.08991203643381596,0.0011337338481098413,0.349766146424372,8.557302780647091e-07,0.27743667736649513,0.0,0.0,0.0,0.0,0.001173558970913291,80,10,10.310146570205688,1.031014657020569,0.12887683212757112,0.09896524278447032 +17,0.0009781627519259927,0.12210180922690057,2.4953360625475097e-07,0.14165311101824046,0.0,0.0,0.0,0.0,0.0010008670633396832,320,40,55.63052153587341,1.3907630383968352,0.1738453797996044,0.09886696673929692,0.000485295636462979,0.006752535897888379,3.532286285690134e-07,0.1760434664785862,0.0,0.0,0.0,0.0,0.0005099983332911507,80,10,10.920001983642578,1.0920001983642578,0.13650002479553222,0.09113368857651949 +18,0.0005243772658104717,0.06725221440409826,5.400551530233399e-07,0.11257906723767519,0.0,0.0,0.0,0.0,0.0005410770281741861,320,40,55.73082876205444,1.3932707190513611,0.17415883988142014,0.09228192456066608,0.0008791333420958836,0.9210377830691868,1.0548424336231932e-07,0.1540754111483693,0.0,0.0,0.0,0.0,0.0009023529979458545,80,10,10.039228677749634,1.0039228677749634,0.12549035847187043,0.06676724823191763 +19,0.0034075098868925125,0.4902075598911324,2.192158632493697e-05,0.34483206500299274,0.0,0.0,0.0,0.0,0.0034694819161813937,320,40,54.436464071273804,1.360911601781845,0.17011395022273063,0.0833871294511482,0.0008605413371697068,0.6328421436296139,3.3934636718413457e-07,0.16964451484382154,0.0,0.0,0.0,0.0,0.0008860847738105804,80,10,10.06943392753601,1.0069433927536011,0.12586792409420014,0.07390656652860343 +20,0.0011855200507852714,0.24362912205001522,2.228905412201762e-06,0.1392481838585809,0.0,0.0,0.0,0.0,0.001209167311571946,320,40,55.08751344680786,1.3771878361701966,0.17214847952127457,0.09444970786571502,0.0006605268277780852,0.30152974404791166,2.607353242090049e-07,0.1564497247338295,0.0,0.0,0.0,0.0,0.0006833117298810975,80,10,10.297621965408325,1.0297621965408326,0.12872027456760407,0.09378624190576375 diff --git a/insurance/realtabformer/mlu-eval.ipynb b/insurance/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..dbbc70d2a4f04034f3976ffd986ce51c26228650 --- /dev/null +++ b/insurance/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2607 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:37.107665Z", + "iopub.status.busy": "2024-02-29T14:45:37.107303Z", + "iopub.status.idle": "2024-02-29T14:45:37.143672Z", + "shell.execute_reply": "2024-02-29T14:45:37.142842Z" + }, + "papermill": { + "duration": 0.052589, + "end_time": "2024-02-29T14:45:37.145925", + "exception": false, + "start_time": "2024-02-29T14:45:37.093336", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:37.174706Z", + "iopub.status.busy": "2024-02-29T14:45:37.174314Z", + "iopub.status.idle": "2024-02-29T14:45:37.181833Z", + "shell.execute_reply": "2024-02-29T14:45:37.180939Z" + }, + "papermill": { + "duration": 0.024573, + "end_time": "2024-02-29T14:45:37.184089", + "exception": false, + "start_time": "2024-02-29T14:45:37.159516", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:37.210880Z", + "iopub.status.busy": "2024-02-29T14:45:37.210589Z", + "iopub.status.idle": "2024-02-29T14:45:37.214940Z", + "shell.execute_reply": "2024-02-29T14:45:37.214046Z" + }, + "papermill": { + "duration": 0.020243, + "end_time": "2024-02-29T14:45:37.217007", + "exception": false, + "start_time": "2024-02-29T14:45:37.196764", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:37.242437Z", + "iopub.status.busy": "2024-02-29T14:45:37.241857Z", + "iopub.status.idle": "2024-02-29T14:45:37.246230Z", + "shell.execute_reply": "2024-02-29T14:45:37.245289Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.019299, + "end_time": "2024-02-29T14:45:37.248203", + "exception": false, + "start_time": "2024-02-29T14:45:37.228904", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:37.276301Z", + "iopub.status.busy": "2024-02-29T14:45:37.275891Z", + "iopub.status.idle": "2024-02-29T14:45:37.282880Z", + "shell.execute_reply": "2024-02-29T14:45:37.281948Z" + }, + "papermill": { + "duration": 0.023623, + "end_time": "2024-02-29T14:45:37.284985", + "exception": false, + "start_time": "2024-02-29T14:45:37.261362", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e7d5e499", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:37.309430Z", + "iopub.status.busy": "2024-02-29T14:45:37.309143Z", + "iopub.status.idle": "2024-02-29T14:45:37.313769Z", + "shell.execute_reply": "2024-02-29T14:45:37.312964Z" + }, + "papermill": { + "duration": 0.018965, + "end_time": "2024-02-29T14:45:37.315618", + "exception": false, + "start_time": "2024-02-29T14:45:37.296653", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"realtabformer\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/realtabformer/2\"\n", + "param_index = 2\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011644, + "end_time": "2024-02-29T14:45:37.339251", + "exception": false, + "start_time": "2024-02-29T14:45:37.327607", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:37.364512Z", + "iopub.status.busy": "2024-02-29T14:45:37.364253Z", + "iopub.status.idle": "2024-02-29T14:45:37.373585Z", + "shell.execute_reply": "2024-02-29T14:45:37.372771Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024056, + "end_time": "2024-02-29T14:45:37.375465", + "exception": false, + "start_time": "2024-02-29T14:45:37.351409", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/realtabformer/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:37.402825Z", + "iopub.status.busy": "2024-02-29T14:45:37.402169Z", + "iopub.status.idle": "2024-02-29T14:45:39.648753Z", + "shell.execute_reply": "2024-02-29T14:45:39.647684Z" + }, + "papermill": { + "duration": 2.263712, + "end_time": "2024-02-29T14:45:39.651809", + "exception": false, + "start_time": "2024-02-29T14:45:37.388097", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:39.678039Z", + "iopub.status.busy": "2024-02-29T14:45:39.677589Z", + "iopub.status.idle": "2024-02-29T14:45:39.689368Z", + "shell.execute_reply": "2024-02-29T14:45:39.688605Z" + }, + "papermill": { + "duration": 0.02649, + "end_time": "2024-02-29T14:45:39.691576", + "exception": false, + "start_time": "2024-02-29T14:45:39.665086", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:39.715465Z", + "iopub.status.busy": "2024-02-29T14:45:39.715208Z", + "iopub.status.idle": "2024-02-29T14:45:39.722619Z", + "shell.execute_reply": "2024-02-29T14:45:39.721729Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021699, + "end_time": "2024-02-29T14:45:39.724571", + "exception": false, + "start_time": "2024-02-29T14:45:39.702872", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:39.750238Z", + "iopub.status.busy": "2024-02-29T14:45:39.749935Z", + "iopub.status.idle": "2024-02-29T14:45:39.855863Z", + "shell.execute_reply": "2024-02-29T14:45:39.855071Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.121939, + "end_time": "2024-02-29T14:45:39.858399", + "exception": false, + "start_time": "2024-02-29T14:45:39.736460", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:39.884482Z", + "iopub.status.busy": "2024-02-29T14:45:39.884136Z", + "iopub.status.idle": "2024-02-29T14:45:44.512489Z", + "shell.execute_reply": "2024-02-29T14:45:44.511721Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.643906, + "end_time": "2024-02-29T14:45:44.514967", + "exception": false, + "start_time": "2024-02-29T14:45:39.871061", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 14:45:42.144631: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 14:45:42.144692: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 14:45:42.146341: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:44.542280Z", + "iopub.status.busy": "2024-02-29T14:45:44.541233Z", + "iopub.status.idle": "2024-02-29T14:45:44.548651Z", + "shell.execute_reply": "2024-02-29T14:45:44.547951Z" + }, + "papermill": { + "duration": 0.022958, + "end_time": "2024-02-29T14:45:44.550597", + "exception": false, + "start_time": "2024-02-29T14:45:44.527639", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:44.574514Z", + "iopub.status.busy": "2024-02-29T14:45:44.574261Z", + "iopub.status.idle": "2024-02-29T14:45:53.307618Z", + "shell.execute_reply": "2024-02-29T14:45:53.306484Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.748598, + "end_time": "2024-02-29T14:45:53.310557", + "exception": false, + "start_time": "2024-02-29T14:45:44.561959", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.77,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.75,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': torch.nn.modules.activation.ReLU6,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.RReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:53.881907Z", + "iopub.status.busy": "2024-02-29T14:45:53.881195Z", + "iopub.status.idle": "2024-02-29T14:45:53.956863Z", + "shell.execute_reply": "2024-02-29T14:45:53.955904Z" + }, + "papermill": { + "duration": 0.092814, + "end_time": "2024-02-29T14:45:53.959350", + "exception": false, + "start_time": "2024-02-29T14:45:53.866536", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../insurance/_cache/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache4/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache5/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/insurance [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T14:45:53.992457Z", + "iopub.status.busy": "2024-02-29T14:45:53.992151Z", + "iopub.status.idle": "2024-02-29T14:45:54.445421Z", + "shell.execute_reply": "2024-02-29T14:45:54.444403Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.471474, + "end_time": "2024-02-29T14:45:54.447551", + "exception": false, + "start_time": "2024-02-29T14:45:53.976077", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n", + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:54.477041Z", + "iopub.status.busy": "2024-02-29T14:45:54.476699Z", + "iopub.status.idle": "2024-02-29T14:45:54.481566Z", + "shell.execute_reply": "2024-02-29T14:45:54.480123Z" + }, + "papermill": { + "duration": 0.02186, + "end_time": "2024-02-29T14:45:54.483648", + "exception": false, + "start_time": "2024-02-29T14:45:54.461788", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:54.510879Z", + "iopub.status.busy": "2024-02-29T14:45:54.510585Z", + "iopub.status.idle": "2024-02-29T14:45:54.517733Z", + "shell.execute_reply": "2024-02-29T14:45:54.516799Z" + }, + "papermill": { + "duration": 0.023023, + "end_time": "2024-02-29T14:45:54.519718", + "exception": false, + "start_time": "2024-02-29T14:45:54.496695", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10420884" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:54.547495Z", + "iopub.status.busy": "2024-02-29T14:45:54.547224Z", + "iopub.status.idle": "2024-02-29T14:45:54.640625Z", + "shell.execute_reply": "2024-02-29T14:45:54.639714Z" + }, + "papermill": { + "duration": 0.109819, + "end_time": "2024-02-29T14:45:54.642768", + "exception": false, + "start_time": "2024-02-29T14:45:54.532949", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 15200] --\n", + "├─Adapter: 1-1 [2, 1071, 15200] --\n", + "│ └─Embedding: 2-1 [2, 1071, 19, 800] (440,800)\n", + "│ └─TensorInductionPoint: 2-2 [19, 1] 19\n", + "│ └─Sequential: 2-3 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 820,224\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 15200] (recursive)\n", + "│ └─Embedding: 2-4 [2, 267, 19, 800] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [19, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-7 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-8 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─RReLU: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-52 [2, 128] --\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,861,684\n", + "Trainable params: 10,420,884\n", + "Non-trainable params: 440,800\n", + "Total mult-adds (M): 43.07\n", + "========================================================================================================================\n", + "Input size (MB): 0.20\n", + "Forward/backward pass size (MB): 632.87\n", + "Params size (MB): 43.45\n", + "Estimated Total Size (MB): 676.52\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T14:45:54.672966Z", + "iopub.status.busy": "2024-02-29T14:45:54.672663Z", + "iopub.status.idle": "2024-02-29T15:10:29.001114Z", + "shell.execute_reply": "2024-02-29T15:10:29.000114Z" + }, + "papermill": { + "duration": 1474.363618, + "end_time": "2024-02-29T15:10:29.020839", + "exception": false, + "start_time": "2024-02-29T14:45:54.657221", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.14850416241097264, 'avg_role_model_std_loss': 1.180862516140678, 'avg_role_model_mean_pred_loss': 0.0921927880990836, 'avg_role_model_g_mag_loss': 5.234412693977356, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.14999753049341963, 'n_size': 320, 'n_batch': 40, 'duration': 54.21397066116333, 'duration_batch': 1.3553492665290832, 'duration_size': 0.1694186583161354, 'avg_pred_std': 0.18854508120566607}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01377877084451029, 'avg_role_model_std_loss': 3.3042451123972567, 'avg_role_model_mean_pred_loss': 0.00011845146224525039, 'avg_role_model_g_mag_loss': 0.21803768426179887, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01390348743661889, 'n_size': 80, 'n_batch': 10, 'duration': 9.792181253433228, 'duration_batch': 0.9792181253433228, 'duration_size': 0.12240226566791534, 'avg_pred_std': 0.05718763165641576}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.010435808167676442, 'avg_role_model_std_loss': 5.020027183283946, 'avg_role_model_mean_pred_loss': 0.0002433187393593883, 'avg_role_model_g_mag_loss': 0.40798311326652764, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01056134452810511, 'n_size': 320, 'n_batch': 40, 'duration': 54.643929958343506, 'duration_batch': 1.3660982489585876, 'duration_size': 0.17076228111982344, 'avg_pred_std': 0.06234871030319482}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0054248009575530885, 'avg_role_model_std_loss': 7.6202680438756945, 'avg_role_model_mean_pred_loss': 2.544027604756138e-06, 'avg_role_model_g_mag_loss': 0.2834674768149853, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005489112006034702, 'n_size': 80, 'n_batch': 10, 'duration': 10.194044351577759, 'duration_batch': 1.0194044351577758, 'duration_size': 0.12742555439472197, 'avg_pred_std': 0.03268122207373381}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004800849179446232, 'avg_role_model_std_loss': 4.060577614020576, 'avg_role_model_mean_pred_loss': 1.805691692511324e-05, 'avg_role_model_g_mag_loss': 0.24826885210350155, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004860438608739059, 'n_size': 320, 'n_batch': 40, 'duration': 55.247488498687744, 'duration_batch': 1.3811872124671936, 'duration_size': 0.1726484015583992, 'avg_pred_std': 0.06760974442586302}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002636889467248693, 'avg_role_model_std_loss': 0.5023930853232741, 'avg_role_model_mean_pred_loss': 1.9693377169005544e-06, 'avg_role_model_g_mag_loss': 0.46713720858097074, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027079362072981892, 'n_size': 80, 'n_batch': 10, 'duration': 10.467901945114136, 'duration_batch': 1.0467901945114135, 'duration_size': 0.1308487743139267, 'avg_pred_std': 0.08990103090181947}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003688591261743568, 'avg_role_model_std_loss': 5.662890090987792, 'avg_role_model_mean_pred_loss': 3.4259154269070546e-05, 'avg_role_model_g_mag_loss': 0.2902778916526586, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003743911211495288, 'n_size': 320, 'n_batch': 40, 'duration': 54.36272192001343, 'duration_batch': 1.3590680480003356, 'duration_size': 0.16988350600004196, 'avg_pred_std': 0.08270897869369946}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004029050027020276, 'avg_role_model_std_loss': 3.6828645429341123, 'avg_role_model_mean_pred_loss': 5.466211582216829e-06, 'avg_role_model_g_mag_loss': 0.22611289098858833, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004080505215097219, 'n_size': 80, 'n_batch': 10, 'duration': 9.939324378967285, 'duration_batch': 0.9939324378967285, 'duration_size': 0.12424155473709106, 'avg_pred_std': 0.10100285978987814}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004078761015261989, 'avg_role_model_std_loss': 3.139242228523483, 'avg_role_model_mean_pred_loss': 3.3808381526224254e-05, 'avg_role_model_g_mag_loss': 0.36921092458069327, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0041462778390268795, 'n_size': 320, 'n_batch': 40, 'duration': 54.406832456588745, 'duration_batch': 1.3601708114147186, 'duration_size': 0.17002135142683983, 'avg_pred_std': 0.08321290470194072}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0015522451751166955, 'avg_role_model_std_loss': 0.42041910818370526, 'avg_role_model_mean_pred_loss': 2.725768419864494e-06, 'avg_role_model_g_mag_loss': 0.18216939978301525, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015822530665900558, 'n_size': 80, 'n_batch': 10, 'duration': 10.175266981124878, 'duration_batch': 1.0175266981124877, 'duration_size': 0.12719083726406097, 'avg_pred_std': 0.08142163883894682}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0020072206414624818, 'avg_role_model_std_loss': 0.8528530288420655, 'avg_role_model_mean_pred_loss': 8.324493640320986e-06, 'avg_role_model_g_mag_loss': 0.22079015588387846, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0020452409196877854, 'n_size': 320, 'n_batch': 40, 'duration': 55.4995391368866, 'duration_batch': 1.3874884784221648, 'duration_size': 0.1734360598027706, 'avg_pred_std': 0.08714587027207017}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00045264576037880034, 'avg_role_model_std_loss': 0.6128407033349503, 'avg_role_model_mean_pred_loss': 2.0517467608964848e-07, 'avg_role_model_g_mag_loss': 0.13081572577357292, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00047156074579106643, 'n_size': 80, 'n_batch': 10, 'duration': 10.349114894866943, 'duration_batch': 1.0349114894866944, 'duration_size': 0.1293639361858368, 'avg_pred_std': 0.07619310528971254}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009020587680424796, 'avg_role_model_std_loss': 0.19860290411693313, 'avg_role_model_mean_pred_loss': 4.795286915437736e-07, 'avg_role_model_g_mag_loss': 0.16418010434135794, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009270869173633401, 'n_size': 320, 'n_batch': 40, 'duration': 55.20178484916687, 'duration_batch': 1.3800446212291717, 'duration_size': 0.17250557765364646, 'avg_pred_std': 0.09331575823016465}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005614329825039022, 'avg_role_model_std_loss': 0.15103251073694537, 'avg_role_model_mean_pred_loss': 3.9005249729129333e-05, 'avg_role_model_g_mag_loss': 0.43398396372795106, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005694820835196878, 'n_size': 80, 'n_batch': 10, 'duration': 10.202214241027832, 'duration_batch': 1.0202214241027832, 'duration_size': 0.1275276780128479, 'avg_pred_std': 0.12506430204957725}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002170501317596063, 'avg_role_model_std_loss': 0.42862962205581423, 'avg_role_model_mean_pred_loss': 9.560629908794694e-06, 'avg_role_model_g_mag_loss': 0.23468926995992662, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022109889338025822, 'n_size': 320, 'n_batch': 40, 'duration': 55.481743574142456, 'duration_batch': 1.3870435893535613, 'duration_size': 0.17338044866919516, 'avg_pred_std': 0.09381236587651073}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001958590082358569, 'avg_role_model_std_loss': 0.8871201357003884, 'avg_role_model_mean_pred_loss': 2.944644425895149e-06, 'avg_role_model_g_mag_loss': 0.20026710480451584, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019938025623559954, 'n_size': 80, 'n_batch': 10, 'duration': 10.465904474258423, 'duration_batch': 1.0465904474258423, 'duration_size': 0.13082380592823029, 'avg_pred_std': 0.07660999405197799}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0032185729534830896, 'avg_role_model_std_loss': 0.39595315401531933, 'avg_role_model_mean_pred_loss': 2.223747077226177e-05, 'avg_role_model_g_mag_loss': 0.26023907188791784, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032680820706445955, 'n_size': 320, 'n_batch': 40, 'duration': 55.13088512420654, 'duration_batch': 1.3782721281051635, 'duration_size': 0.17228401601314544, 'avg_pred_std': 0.09016182171180845}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006755840044934303, 'avg_role_model_std_loss': 0.042443403803736145, 'avg_role_model_mean_pred_loss': 9.365343847704821e-05, 'avg_role_model_g_mag_loss': 0.8566916212439537, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0068929205401218495, 'n_size': 80, 'n_batch': 10, 'duration': 10.145907640457153, 'duration_batch': 1.0145907640457152, 'duration_size': 0.1268238455057144, 'avg_pred_std': 0.1274191069416702}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003263563236214395, 'avg_role_model_std_loss': 0.8668862596564395, 'avg_role_model_mean_pred_loss': 2.02128691412784e-05, 'avg_role_model_g_mag_loss': 0.2691699764691293, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033155382898257812, 'n_size': 320, 'n_batch': 40, 'duration': 54.284446239471436, 'duration_batch': 1.3571111559867859, 'duration_size': 0.16963889449834824, 'avg_pred_std': 0.08654828406870366}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0004874484846368432, 'avg_role_model_std_loss': 0.003280633598478744, 'avg_role_model_mean_pred_loss': 2.418919596181013e-07, 'avg_role_model_g_mag_loss': 0.12313968688249588, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005053632776252925, 'n_size': 80, 'n_batch': 10, 'duration': 9.947452306747437, 'duration_batch': 0.9947452306747436, 'duration_size': 0.12434315383434295, 'avg_pred_std': 0.08960323911160231}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002170001242484432, 'avg_role_model_std_loss': 0.33971318979765786, 'avg_role_model_mean_pred_loss': 1.1028246948152631e-06, 'avg_role_model_g_mag_loss': 0.2432436312083155, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022117360727861523, 'n_size': 320, 'n_batch': 40, 'duration': 54.17906904220581, 'duration_batch': 1.3544767260551454, 'duration_size': 0.16930959075689317, 'avg_pred_std': 0.0964421829674393}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0009476982486376073, 'avg_role_model_std_loss': 1.3159201624483856, 'avg_role_model_mean_pred_loss': 1.8856890096896618e-07, 'avg_role_model_g_mag_loss': 0.13108705095946788, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009683694021077827, 'n_size': 80, 'n_batch': 10, 'duration': 10.008543252944946, 'duration_batch': 1.0008543252944946, 'duration_size': 0.12510679066181182, 'avg_pred_std': 0.06394510199315846}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008773179083618743, 'avg_role_model_std_loss': 0.4401658920556809, 'avg_role_model_mean_pred_loss': 6.324480833997878e-07, 'avg_role_model_g_mag_loss': 0.12429296048358082, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008975059775366389, 'n_size': 320, 'n_batch': 40, 'duration': 54.14188766479492, 'duration_batch': 1.3535471916198731, 'duration_size': 0.16919339895248414, 'avg_pred_std': 0.09457911597564816}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0003643055897555314, 'avg_role_model_std_loss': 0.9265121433396416, 'avg_role_model_mean_pred_loss': 1.4636429152004027e-07, 'avg_role_model_g_mag_loss': 0.07630779892206192, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0003756950100068934, 'n_size': 80, 'n_batch': 10, 'duration': 10.02190351486206, 'duration_batch': 1.002190351486206, 'duration_size': 0.12527379393577576, 'avg_pred_std': 0.08594365753233432}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0004941477713146014, 'avg_role_model_std_loss': 0.05355267495711606, 'avg_role_model_mean_pred_loss': 1.4848906236212407e-07, 'avg_role_model_g_mag_loss': 0.09103901842609048, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000507950432574944, 'n_size': 320, 'n_batch': 40, 'duration': 54.16036105155945, 'duration_batch': 1.3540090262889861, 'duration_size': 0.16925112828612326, 'avg_pred_std': 0.09392541642300785}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00022847578329674434, 'avg_role_model_std_loss': 0.661697834357119, 'avg_role_model_mean_pred_loss': 3.0550005269969204e-08, 'avg_role_model_g_mag_loss': 0.056163723766803744, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00023663823449169286, 'n_size': 80, 'n_batch': 10, 'duration': 10.043348550796509, 'duration_batch': 1.004334855079651, 'duration_size': 0.12554185688495637, 'avg_pred_std': 0.08091261517256498}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00029569496205112956, 'avg_role_model_std_loss': 0.04474922703773245, 'avg_role_model_mean_pred_loss': 6.23439235618306e-08, 'avg_role_model_g_mag_loss': 0.05976610332727432, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00030461719015875135, 'n_size': 320, 'n_batch': 40, 'duration': 55.389015674591064, 'duration_batch': 1.3847253918647766, 'duration_size': 0.17309067398309708, 'avg_pred_std': 0.09439536663703621}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00025476437076576985, 'avg_role_model_std_loss': 2.0484021956655853, 'avg_role_model_mean_pred_loss': 4.190348910498853e-08, 'avg_role_model_g_mag_loss': 0.05006699915975332, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00026224629764328713, 'n_size': 80, 'n_batch': 10, 'duration': 10.323888778686523, 'duration_batch': 1.0323888778686523, 'duration_size': 0.12904860973358154, 'avg_pred_std': 0.08619057663017884}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0003340222642691515, 'avg_role_model_std_loss': 0.05788067737812881, 'avg_role_model_mean_pred_loss': 5.194412635303129e-08, 'avg_role_model_g_mag_loss': 0.06943645351566374, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0003443146288191201, 'n_size': 320, 'n_batch': 40, 'duration': 55.98161721229553, 'duration_batch': 1.3995404303073884, 'duration_size': 0.17494255378842355, 'avg_pred_std': 0.091485915472731}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00025505621306365356, 'avg_role_model_std_loss': 0.46580720322454, 'avg_role_model_mean_pred_loss': 2.3517513286774872e-08, 'avg_role_model_g_mag_loss': 0.053071103431284426, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00026298246484657284, 'n_size': 80, 'n_batch': 10, 'duration': 10.274444580078125, 'duration_batch': 1.0274444580078126, 'duration_size': 0.12843055725097657, 'avg_pred_std': 0.08523455495014787}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00045633949812327044, 'avg_role_model_std_loss': 0.03235854215981604, 'avg_role_model_mean_pred_loss': 6.432831875045203e-08, 'avg_role_model_g_mag_loss': 0.08093514840584248, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000468703078058752, 'n_size': 320, 'n_batch': 40, 'duration': 54.44696354866028, 'duration_batch': 1.361174088716507, 'duration_size': 0.17014676108956336, 'avg_pred_std': 0.09638399491086602}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0005625217905617319, 'avg_role_model_std_loss': 1.3924316422228002, 'avg_role_model_mean_pred_loss': 1.6518343430860228e-07, 'avg_role_model_g_mag_loss': 0.09882423281669617, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000577750973025104, 'n_size': 80, 'n_batch': 10, 'duration': 10.127650499343872, 'duration_batch': 1.0127650499343872, 'duration_size': 0.1265956312417984, 'avg_pred_std': 0.07251062926370651}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010068999348732178, 'avg_role_model_std_loss': 0.41138206991290643, 'avg_role_model_mean_pred_loss': 9.277023389098193e-07, 'avg_role_model_g_mag_loss': 0.12433609296567738, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010285568052495365, 'n_size': 320, 'n_batch': 40, 'duration': 54.41342616081238, 'duration_batch': 1.3603356540203095, 'duration_size': 0.1700419567525387, 'avg_pred_std': 0.08991203643381596}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0011337338481098413, 'avg_role_model_std_loss': 0.349766146424372, 'avg_role_model_mean_pred_loss': 8.557302780647091e-07, 'avg_role_model_g_mag_loss': 0.27743667736649513, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001173558970913291, 'n_size': 80, 'n_batch': 10, 'duration': 10.310146570205688, 'duration_batch': 1.031014657020569, 'duration_size': 0.12887683212757112, 'avg_pred_std': 0.09896524278447032}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009781627519259927, 'avg_role_model_std_loss': 0.12210180922690057, 'avg_role_model_mean_pred_loss': 2.4953360625475097e-07, 'avg_role_model_g_mag_loss': 0.14165311101824046, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010008670633396832, 'n_size': 320, 'n_batch': 40, 'duration': 55.63052153587341, 'duration_batch': 1.3907630383968352, 'duration_size': 0.1738453797996044, 'avg_pred_std': 0.09886696673929692}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.000485295636462979, 'avg_role_model_std_loss': 0.006752535897888379, 'avg_role_model_mean_pred_loss': 3.532286285690134e-07, 'avg_role_model_g_mag_loss': 0.1760434664785862, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005099983332911507, 'n_size': 80, 'n_batch': 10, 'duration': 10.920001983642578, 'duration_batch': 1.0920001983642578, 'duration_size': 0.13650002479553222, 'avg_pred_std': 0.09113368857651949}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005243772658104717, 'avg_role_model_std_loss': 0.06725221440409826, 'avg_role_model_mean_pred_loss': 5.400551530233399e-07, 'avg_role_model_g_mag_loss': 0.11257906723767519, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005410770281741861, 'n_size': 320, 'n_batch': 40, 'duration': 55.73082876205444, 'duration_batch': 1.3932707190513611, 'duration_size': 0.17415883988142014, 'avg_pred_std': 0.09228192456066608}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008791333420958836, 'avg_role_model_std_loss': 0.9210377830691868, 'avg_role_model_mean_pred_loss': 1.0548424336231932e-07, 'avg_role_model_g_mag_loss': 0.1540754111483693, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009023529979458545, 'n_size': 80, 'n_batch': 10, 'duration': 10.039228677749634, 'duration_batch': 1.0039228677749634, 'duration_size': 0.12549035847187043, 'avg_pred_std': 0.06676724823191763}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0034075098868925125, 'avg_role_model_std_loss': 0.4902075598911324, 'avg_role_model_mean_pred_loss': 2.192158632493697e-05, 'avg_role_model_g_mag_loss': 0.34483206500299274, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034694819161813937, 'n_size': 320, 'n_batch': 40, 'duration': 54.436464071273804, 'duration_batch': 1.360911601781845, 'duration_size': 0.17011395022273063, 'avg_pred_std': 0.0833871294511482}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008605413371697068, 'avg_role_model_std_loss': 0.6328421436296139, 'avg_role_model_mean_pred_loss': 3.3934636718413457e-07, 'avg_role_model_g_mag_loss': 0.16964451484382154, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008860847738105804, 'n_size': 80, 'n_batch': 10, 'duration': 10.06943392753601, 'duration_batch': 1.0069433927536011, 'duration_size': 0.12586792409420014, 'avg_pred_std': 0.07390656652860343}\n", + "Epoch 20\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011855200507852714, 'avg_role_model_std_loss': 0.24362912205001522, 'avg_role_model_mean_pred_loss': 2.228905412201762e-06, 'avg_role_model_g_mag_loss': 0.1392481838585809, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001209167311571946, 'n_size': 320, 'n_batch': 40, 'duration': 55.08751344680786, 'duration_batch': 1.3771878361701966, 'duration_size': 0.17214847952127457, 'avg_pred_std': 0.09444970786571502}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0006605268277780852, 'avg_role_model_std_loss': 0.30152974404791166, 'avg_role_model_mean_pred_loss': 2.607353242090049e-07, 'avg_role_model_g_mag_loss': 0.1564497247338295, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006833117298810975, 'n_size': 80, 'n_batch': 10, 'duration': 10.297621965408325, 'duration_batch': 1.0297621965408326, 'duration_size': 0.12872027456760407, 'avg_pred_std': 0.09378624190576375}\n", + "Stopped False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▄▂▃▂▁▄▂▄▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▃▁▅▆▅▄█▄█▅▃▅▅▅▅▄▆▅▄▄▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train █▁▁▂▂▂▃▃▃▂▃▃▃▃▃▃▃▃▃▂▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▂▃▅▃▂▂▄▂█▂▂▁▁▁▁▁▃▂▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▄▂▃▂▁▄▂▄▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▁▁▁▁▁▃▁▇▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▄█▁▄▁▂▁▂▁▁▂▂▂▃▁▂▁▁▂▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▂▇▆█▅▂▁▁▁▂▁▂▁▁▁▁▁▁▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▁▃▅▂▃▄▄▅▃▂▂▂▃▄▄▃▄█▃▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▁▃▅▂▂▆▅▆▅▂▁▁▁▆█▂▂▇▇▂▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▁▃▅▂▃▄▄▅▃▂▂▂▃▄▄▃▄█▃▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▁▃▅▂▂▆▅▆▅▂▁▁▁▆█▂▂▇▇▂▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▁▃▅▂▃▄▄▅▃▂▂▂▃▄▄▃▄█▃▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▁▃▅▂▂▆▅▆▅▂▁▁▁▆█▂▂▇▇▂▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00068\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00121\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.09379\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.09445\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.15645\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.13925\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00066\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00119\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 0.30153\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.24363\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 1.02976\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.37719\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.12872\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.17215\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 10.29762\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 55.08751\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 10\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/insurance/realtabformer/2/wandb/offline-run-20240229_144556-hucjnvtd\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_144556-hucjnvtd/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 399, 'n_batch': 50, 'role_model_metrics': {'avg_loss': 0.0005081958878688505, 'avg_g_mag_loss': 0.017775198748454284, 'avg_g_cos_loss': 0.10860606788991388, 'pred_duration': 2.0889976024627686, 'grad_duration': 0.40578484535217285, 'total_duration': 2.4947824478149414, 'pred_std': 0.16169536113739014, 'std_loss': 3.497965735732578e-05, 'mean_pred_loss': 8.23702563934603e-08, 'pred_rmse': 0.022543201223015785, 'pred_mae': 0.01770409755408764, 'pred_mape': 0.26909926533699036, 'grad_rmse': 0.2209186851978302, 'grad_mae': 0.10591567307710648, 'grad_mape': 2.4253857135772705}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0005081958878688505, 'avg_g_mag_loss': 0.017775198748454284, 'avg_g_cos_loss': 0.10860606788991388, 'avg_pred_duration': 2.0889976024627686, 'avg_grad_duration': 0.40578484535217285, 'avg_total_duration': 2.4947824478149414, 'avg_pred_std': 0.16169536113739014, 'avg_std_loss': 3.497965735732578e-05, 'avg_mean_pred_loss': 8.23702563934603e-08}, 'min_metrics': {'avg_loss': 0.0005081958878688505, 'avg_g_mag_loss': 0.017775198748454284, 'avg_g_cos_loss': 0.10860606788991388, 'pred_duration': 2.0889976024627686, 'grad_duration': 0.40578484535217285, 'total_duration': 2.4947824478149414, 'pred_std': 0.16169536113739014, 'std_loss': 3.497965735732578e-05, 'mean_pred_loss': 8.23702563934603e-08, 'pred_rmse': 0.022543201223015785, 'pred_mae': 0.01770409755408764, 'pred_mape': 0.26909926533699036, 'grad_rmse': 0.2209186851978302, 'grad_mae': 0.10591567307710648, 'grad_mape': 2.4253857135772705}, 'model_metrics': {'realtabformer': {'avg_loss': 0.0005081958878688505, 'avg_g_mag_loss': 0.017775198748454284, 'avg_g_cos_loss': 0.10860606788991388, 'pred_duration': 2.0889976024627686, 'grad_duration': 0.40578484535217285, 'total_duration': 2.4947824478149414, 'pred_std': 0.16169536113739014, 'std_loss': 3.497965735732578e-05, 'mean_pred_loss': 8.23702563934603e-08, 'pred_rmse': 0.022543201223015785, 'pred_mae': 0.01770409755408764, 'pred_mape': 0.26909926533699036, 'grad_rmse': 0.2209186851978302, 'grad_mae': 0.10591567307710648, 'grad_mape': 2.4253857135772705}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:10:29.061332Z", + "iopub.status.busy": "2024-02-29T15:10:29.060496Z", + "iopub.status.idle": "2024-02-29T15:10:29.064733Z", + "shell.execute_reply": "2024-02-29T15:10:29.063926Z" + }, + "papermill": { + "duration": 0.027169, + "end_time": "2024-02-29T15:10:29.066618", + "exception": false, + "start_time": "2024-02-29T15:10:29.039449", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:10:29.103059Z", + "iopub.status.busy": "2024-02-29T15:10:29.102492Z", + "iopub.status.idle": "2024-02-29T15:10:29.393521Z", + "shell.execute_reply": "2024-02-29T15:10:29.392517Z" + }, + "papermill": { + "duration": 0.312411, + "end_time": "2024-02-29T15:10:29.396441", + "exception": false, + "start_time": "2024-02-29T15:10:29.084030", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:10:29.438270Z", + "iopub.status.busy": "2024-02-29T15:10:29.437282Z", + "iopub.status.idle": "2024-02-29T15:10:29.734675Z", + "shell.execute_reply": "2024-02-29T15:10:29.733760Z" + }, + "papermill": { + "duration": 0.321378, + "end_time": "2024-02-29T15:10:29.736969", + "exception": false, + "start_time": "2024-02-29T15:10:29.415591", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA9DklEQVR4nO3de1xUdf4/8NfMMBdgBpA7GIIoqHlBRSF0lXVlQ7MLRmn8TMmvabVgJVnGbkm131/o2sUyy2q3rN1IdMv6pa6lKLYKooEXFCUvKF64CMZwZ5iZ9++PkYMjA8yM6Azyfj4e5+HMOZ/zOZ8zw7z9nM/5nM9HREQExhjrJcS2LgBjjFmCgxZjrFfhoMUY61U4aDHGehUOWoyxXoWDFmOsV+GgxRjrVRxsXYDbRa/X4/Lly1CpVBCJRLYuDmPsBkSEuro6+Pv7QyzuvD7VZ4LW5cuXERAQYOtiMMa6ceHCBdx1112dbu8zQUulUgEwfCAuLi42Lg1j7Ea1tbUICAgQfqud6TNBq+2S0MXFhYMWY3asu+YbbohnjPUqHLQYY70KBy3GWK/SZ9q02M3R6XRobW21dTFYLyaVSiGRSG46H6uC1tq1a7Fq1SqUl5cjLCwMa9asQUREhMm0x48fx/Lly5Gfn4/z58/j3XffxfPPP2+U5rXXXsPrr79utG7IkCE4efKk8L65uRkvvPACNmzYgJaWFsTGxuLDDz+Ej4+PNafAzEREKC8vR01Nja2Lwu4Abm5u8PX1vam+khYHrczMTKSkpGDdunWIjIzE6tWrERsbi+LiYnh7e3dI39jYiODgYDz66KNYsmRJp/kOHz4cO3fubC+Yg3HRlixZgq1bt2LTpk1wdXVFcnIyHn74Yezbt8/SU2AWaAtY3t7ecHJy4o65zCpEhMbGRlRWVgIA/Pz8biozi0RERFBSUpLwXqfTkb+/P6Wnp3e7b2BgIL377rsd1qelpVFYWFin+9XU1JBUKqVNmzYJ606cOEEAKDc316xyq9VqAkBqtbrLdDUNGtp3+grlna02K987mVarpaKiIqqqqrJ1UdgdoqqqioqKikir1XbYZu5v1KKGeI1Gg/z8fMTExAjrxGIxYmJikJuba33kBHDq1Cn4+/sjODgYc+bMQWlpqbAtPz8fra2tRscdOnQoBgwY0OlxW1paUFtba7SY4+ilGvyfT/Ow/PtjN3U+d4K2NiwnJycbl4TdKdr+lm6mfdSioFVVVQWdTtehHcnHxwfl5eVWFyIyMhLr16/H9u3b8dFHH6GkpASTJk1CXV0dAMMlikwmg5ubm9nHTU9Ph6urq7CY+wiPUm64LK1v0Vp9PncaviRkPaUn/pbsosvD9OnT8eijj2LUqFGIjY3Ftm3bUFNTg40bN1qdZ2pqKtRqtbBcuHDBrP1UCg5ajNkzixriPT09IZFIUFFRYbS+oqICvr6+PVYoNzc3hIaG4vTp0wAAX19faDQa1NTUGNW2ujquXC6HXC63+NhKuRQA0NCiBRFxLYMxO2NRTUsmkyE8PBxZWVnCOr1ej6ysLERFRfVYoerr63HmzBnhDkN4eDikUqnRcYuLi1FaWtqjxwUA5bWaVquO0KLV92jerO8QiUT47rvvbF2MHvXaa69h9OjRti6G5ZeHKSkp+PTTT/HFF1/gxIkTeOaZZ9DQ0ID58+cDAObNm4fU1FQhvUajweHDh3H48GFoNBpcunQJhw8fFmpRALB06VLs2bMH586dQ05ODmbOnAmJRIKEhAQAgKurKxYsWICUlBTs3r0b+fn5mD9/PqKionDPPffc7GdgxEna3vmNLxFZb7Z+/foO7cA3Y+nSpUYVB1uxuJ/W7NmzceXKFSxfvhzl5eUYPXo0tm/fLjTOl5aWGg3gdfnyZYwZM0Z4/9Zbb+Gtt95CdHQ0srOzAQAXL15EQkICqqur4eXlhd/97nfYv38/vLy8hP3effddiMVixMfHG3Uu7WlisQhKuQPqW7Sob9bCU2n5JSZjvYlGo4FMJus2nVKphFKpvA0l6sYt6o5hd8ztA0JEFPl/d1Lgsi1UeLHmNpTMfjU1NVFRURE1NTUREZFer6eGllabLHq93qKy/+c//6GJEyeSq6srubu704wZM+j06dNERBQVFUUvvfSSUfrKykpycHCgPXv2EBHR5cuX6b777iOFQkFBQUH01VdfddrP0BQAtHnzZuH90aNHacqUKaRQKMjd3Z0WLlxIdXV1wvbdu3fT+PHjycnJiVxdXWnChAl07tw5IiI6fPgw/f73vyelUkkqlYrGjh1LBw8e7PL4u3fvJgBGS1paGhEZ+ku+8cYbNHfuXFKpVJSYmEhERC+99BKFhISQo6MjDRw4kF555RXSaDRCnjf2p0xMTKSHHnqIVq1aRb6+vuTu7k5/+tOfjPa50Y1/U9cz9zfKzx6aoFQ4ALVAXTNfHl6vqVWHu5f/aJNjF70RCyeZ+X+uDQ0NSElJwahRo1BfX4/ly5dj5syZOHz4MObMmYO//e1vWLFihXCjJTMzE/7+/pg0aRIAQzNHVVUVsrOzIZVKkZKSIvTmtlRDQwNiY2MRFRWFgwcPorKyEk8++SSSk5Oxfv16aLVaxMXFYeHChfj666+h0Whw4MABoWxz5szBmDFj8NFHH0EikeDw4cOQSqVdHnPChAlYvXo1li9fjuLiYgAwqiW99dZbWL58OdLS0oR1KpUK69evh7+/PwoLC7Fw4UKoVCq89NJLnR5n9+7d8PPzw+7du3H69GnMnj0bo0ePxsKFC636rMzBQcuEtr5aDdym1WvFx8cbvf/ss8/g5eWFoqIizJo1C88//zz27t0rBKmMjAwkJCRAJBLh5MmT2LlzJw4ePIhx48YBAP7+978jJCTEqrJkZGSgubkZX375JZydnQEAH3zwAR544AGsXLkSUqkUarUa999/PwYNGgQAGDZsmLB/aWkpXnzxRQwdOhQAzCqHTCaDq6srRCKRyTvsf/jDH/DCCy8YrXvllVeE10FBQVi6dCk2bNjQZdDq168fPvjgA0gkEgwdOhQzZsxAVlYWB63bjftqmeYolaDojVibHdsSp06dwvLly5GXl4eqqiro9YY7waWlpRgxYgTuvfdefPXVV5g0aRJKSkqQm5uLjz/+GIDhzrSDgwPGjh0r5Dd48GD069fPqrKfOHECYWFhQsACgIkTJ0Kv16O4uBiTJ0/GE088gdjYWPzxj39ETEwMZs2aJdw9T0lJwZNPPol//vOfiImJwaOPPioEN2u1BePrZWZm4v3338eZM2dQX18PrVbb7Si/w4cPNxq5wc/PD4WFhTdVtu7YRedSe+N87TKkjoOWEZFIBCeZg00WS/vLPfDAA7h69So+/fRT5OXlIS8vD4Ch0RkwXHL9+9//RmtrKzIyMjBy5EiMHDmyxz8zc33++efIzc3FhAkTkJmZidDQUOzfvx+AoavB8ePHMWPGDOzatQt33303Nm/efFPHuz6AAkBubi7mzJmD++67D1u2bMGhQ4fwl7/8Rfi8OnPjZapIJBL+g7hVOGiZ0NZXq57btHql6upqFBcX45VXXsHUqVMxbNgw/Pbbb0ZpHnroITQ3N2P79u3IyMjAnDlzhG1DhgyBVqvFoUOHhHWnT5/ukIe5hg0bhiNHjqChoUFYt2/fPojFYgwZMkRYN2bMGKSmpiInJwcjRoxARkaGsC00NBRLlizBTz/9hIcffhiff/55t8eVyWTQ6XRmlTEnJweBgYH4y1/+gnHjxiEkJATnz5+34CxvHw5aJrQ/f8iD3vVG/fr1g4eHBz755BOcPn0au3btQkpKilEaZ2dnxMXF4dVXX8WJEyeEPoGA4WH8mJgYLFq0CAcOHMChQ4ewaNEiODo6WvWExJw5c6BQKJCYmIhjx45h9+7dWLx4MebOnQsfHx+UlJQgNTUVubm5OH/+PH766SecOnUKw4YNQ1NTE5KTk5GdnY3z589j3759OHjwoFGbV2eCgoJQX1+PrKwsVFVVobGxsdO0ISEhKC0txYYNG3DmzBm8//77N12bu1U4aJmg4ppWryYWi7Fhwwbk5+djxIgRWLJkCVatWtUh3Zw5c3DkyBFMmjQJAwYMMNr25ZdfwsfHB5MnT8bMmTOFO2kKhcLi8jg5OeHHH3/E1atXMX78eDzyyCOYOnUqPvjgA2H7yZMnER8fj9DQUCxatAhJSUl46qmnIJFIUF1djXnz5iE0NBSzZs3C9OnTOwyaacqECRPw9NNPY/bs2fDy8sLf/va3TtM++OCDWLJkCZKTkzF69Gjk5OTg1VdftfhcbwcREZGtC3E71NbWwtXVFWq1utvGxY/3nEH6f04ifuxdeHtW2G0qof1pbm5GSUkJBg4caNWP9U5y8eJFBAQEYOfOnZg6daqti9NrdfU3Ze5vlO8emiC0afHlYZ+1a9cu1NfXY+TIkSgrK8NLL72EoKAgTJ482dZF6/P48tAEHlOLtba24s9//jOGDx+OmTNnwsvLS+ho+tVXXwmPtNy4DB8+/LaVcfr06Z2W480337xt5bjduKZlghC0uE2rz4qNjUVsrOk+aQ8++CAiIyNNbuuup3pP+vvf/46mpiaT29zd3W9bOW43DlomtAUt7qfFTFGpVFCpVLYuBvr372/rItgEXx6awP20GLNfHLRMUF03eiljzL5w0DKhrabVoNFBp+8TPUIY6zU4aJngLG9/ALRBw7UtxuwJBy0T5A4SyCSGj4bbtRizLxy0OqHk4WnYTbgTJ7awFxy0OiF0e+CaFuulenpiCwDIzs6GSCRCTU1Nj+ZrCQ5aneBe8YzZJw5anRDuIHLQakcEaBpss1j4XP/27dvxu9/9Dm5ubvDw8MD999+PM2fOADCMfrBs2TKj9FeuXIFUKsXPP/8MACgrK8OMGTPg6OiIgQMHIiMjA0FBQVi9erVVH11hYSH+8Ic/wNHRER4eHli0aBHq6+uF7dnZ2YiIiICzszPc3NwwceJEYTyrI0eOYMqUKVCpVHBxcUF4eDh++eWXLo+XnZ2N+fPnQ61WQyQSQSQS4bXXXgMAtLS0YOnSpejfvz+cnZ0RGRkpzIwFAOfPn8cDDzyAfv36wdnZGcOHD8e2bdtw7tw5TJkyBYBh+B+RSIQnnnjCqs/jZnCP+E6o+FGejlobgTf9bXPsP18GZM7dp7uGJ7bofGKL5ORkFBUVYcOGDfD398fmzZsxbdo0FBYWIiQkBElJSdBoNPj555/h7OyMoqIiKJVKBAQE4JtvvkF8fDyKi4vh4uICR0dHqz6Tm8FBqxPO/ChPr8YTW5ie2KK0tBSff/45SktL4e9v+A9o6dKl2L59Oz7//HO8+eabKC0tRXx8vDD8dHBwsLB/2zON3t7ePd5eZi4OWp3gR3lMkDoZajy2OrYFeGIL0woLC6HT6RAaGmq0vqWlBR4eHgCAZ599Fs888wx++uknxMTEID4+HqNGjbLqeLeCVW1aa9euRVBQEBQKBSIjI3HgwIFO0x4/fhzx8fEICgqCSCQy2SaQnp6O8ePHQ6VSwdvbG3FxcUKVts3vf/974dq8bXn66aetKb5ZVDzkckcikeESzRYLT2zRIxNb1NfXQyKRID8/H4cPHxaWEydO4L333gMAPPnkkzh79izmzp2LwsJCjBs3DmvWrOmxc71ZFgetzMxMpKSkIC0tDQUFBQgLC0NsbGyn1/uNjY0IDg7GihUrTM6/BgB79uxBUlIS9u/fjx07dqC1tRX33nuv0UQAALBw4UKUlZUJS1fDx94svnvYe/HEFgamJrYYM2YMdDodKisrMXjwYKPl+t9nQEAAnn76aXz77bd44YUX8Omnnwp5AjB7woxbosv5p02IiIigpKQk4b1OpyN/f39KT0/vdl9zpxWvrKwkAMIU5URE0dHR9Nxzz1laXIG5U263+WzvWQpctoWSMwqsPmZv19UU5vZMp9ORh4cHPf7443Tq1CnKysqi8ePHd5iqfs6cORQWFkYikYjOnz9vlEdMTAyNHTuW8vLyqKCggKZMmUKOjo60evVqs8pw/bEaGhrIz8+P4uPjqbCwkHbt2kXBwcHCdPRnz56ll19+mXJycujcuXP0448/koeHB3344YfU2NhISUlJtHv3bjp37hzt3buXBg0aRC+99FK3Zdi3bx8BoJ07d9KVK1eooaFBOO+goCD65ptv6OzZs5SXl0dvvvkmbdmyhYiInnvuOdq+fTudPXuW8vPzKTIykmbNmkVERBcvXiSRSETr16+nyspKqqurM+vzaNPV35S5v1GLglZLSwtJJBKjL56IaN68efTggw92u7+5QevUqVMEgAoLC4V10dHR5OnpSR4eHjR8+HB6+eWXhS/BlObmZlKr1cJy4cIFi4LWxoOlFLhsCz3xWZ5Z6e9EvTVoERHt2LGDhg0bRnK5nEaNGkXZ2dkdgta2bdsIAE2ePLnD/pcvX6bp06eTXC6nwMBAysjIIG9vb1q3bp1Zx7/xWEePHqUpU6aQQqEgd3d3WrhwofCDLy8vp7i4OPLz8yOZTEaBgYG0fPly0ul01NLSQo899hgFBASQTCYjf39/Sk5ONvs7efrpp8nDw4MAUFpaGhERaTQaWr58OQUFBZFUKiU/Pz+aOXMmHT16lIiIkpOTadCgQSSXy8nLy4vmzp1LVVVVQp5vvPEG+fr6kkgkEgKvuW570Lp06RIBoJycHKP1L774IkVERHS7vzlBS6fT0YwZM2jixIlG6z/++GPavn07HT16lP71r39R//79aebMmZ3mk5aWRgA6LOYGrW1HL1Pgsi30yEf7zEp/J+rNQauntf2nt3PnTlsXpVfriaBld3cPk5KScOzYMezdu9do/aJFi4TXI0eOhJ+fH6ZOnYozZ86YvJOSmppqNNddbW0tAgICzC5H291Dfoynb+KJLeyXRQ3xnp6ekEgkqKioMFpfUVHRaSO7JZKTk7Flyxbs3r0bd911V5dp28boPn36tMntcrkcLi4uRosluCG+b+OJLeyXRTUtmUyG8PBwZGVlIS4uDgCg1+uRlZWF5ORkqwtBRFi8eDE2b96M7OxsDBw4sNt9Dh8+DABCX5aepuJRHvo0ntjCfll8eZiSkoLExESMGzcOERERWL16NRoaGjB//nwAhscf+vfvj/T0dACGfjFFRUXC60uXLuHw4cNQKpUYPHgwAMMlYUZGBr7//nuoVCqUl5cDAFxdXeHo6IgzZ84gIyMD9913Hzw8PHD06FEsWbIEkydPvmWd3pTXDblMRFZNh87uTDyxhY1Z05i2Zs0aGjBgAMlkMoqIiKD9+/cL26Kjo43uKJSUlJhsEI+OjhbSmNoOgD7//HMiIiotLaXJkyeTu7s7yeVyGjx4ML344otmN6oTWd7loa65lQKXbaHAZVuoSaM1+zh3krZG08bGRlsXhd0hGhsbb7ohXkRk4ePzvZS5U2630esJwX/eBgD45ZUYeCrlt7qIdken0+HXX3+Ft7e38IgHYzejuroalZWVCA0NhUQiMdpm7m/U7u4e2guxWASl3AH1LVrUN2v7ZNCSSCRwc3MTnnZwcnLiy2RmFSJCY2MjKisr4ebm1iFgWYKDVheEoNWHG+Pb7gpbOywLY9dzc3O76Z4GHLS6oFQ4ALV9u6+WSCSCn58fvL290drKD48z60ml0puqYbXhoNWFtr5aPHqp4VKxJ/7gGLtZPNxyF7ivFmP2h4NWF5xlPHopY/aGg1YXePRSxuwPB60uKHn0UsbsDgetLqi4psWY3eGg1YX2mpYNh5ZljBnhoNUFoU2LLw8ZsxsctLrAY2oxZn84aHVBybNMM2Z3OGh1QcmzTDNmdzhodYH7aTFmfzhodUF13eiljDH7wEGrC201rQaNDjp9nxgrkTG7x0GrC87y9lENGjRc22LMHnDQ6oLcQQKZxPARcbsWY/aBg1Y3lDw8DWN2hYNWN4RuD1zTYswucNDqBo9eyph94aDVDb48ZMy+cNDqhoof5WHMrlgVtNauXYugoCAoFApERkbiwIEDnaY9fvw44uPjERQUBJFIhNWrV1uVZ3NzM5KSkuDh4QGlUon4+HhUVFRYU3yLOPOjPIzZFYuDVmZmJlJSUpCWloaCggKEhYUhNja203nxGhsbERwcjBUrVnQ635k5eS5ZsgQ//PADNm3ahD179uDy5ct4+OGHLS2+xfhRHsbsDFkoIiKCkpKShPc6nY78/f0pPT29230DAwPp3XfftTjPmpoakkqltGnTJiHNiRMnCADl5uaaVW61Wk0ASK1Wm5W+zZtbiyhw2Rb63y3HLdqPMWYZc3+jFtW0NBoN8vPzERMTI6wTi8WIiYlBbm6uVUHTnDzz8/PR2tpqlGbo0KEYMGBAp8dtaWlBbW2t0WINHr2UMftiUdCqqqqCTqeDj4+P0XofHx+Ul5dbVQBz8iwvL4dMJoObm5vZx01PT4erq6uwBAQEWFU+vnvImH25Y+8epqamQq1WC8uFCxesyqd9IEAecpkxe+BgSWJPT09IJJIOd+0qKio6bWTviTx9fX2h0WhQU1NjVNvq6rhyuRxyudyqMl2Ph1xmzL5YVNOSyWQIDw9HVlaWsE6v1yMrKwtRUVFWFcCcPMPDwyGVSo3SFBcXo7S01Orjmqvt8pAf42HMPlhU0wKAlJQUJCYmYty4cYiIiMDq1avR0NCA+fPnAwDmzZuH/v37Iz09HYChob2oqEh4fenSJRw+fBhKpRKDBw82K09XV1csWLAAKSkpcHd3h4uLCxYvXoyoqCjcc889PfJBdIZrWozZGWtuTa5Zs4YGDBhAMpmMIiIiaP/+/cK26OhoSkxMFN6XlJQQgA5LdHS02XkSETU1NdGf/vQn6tevHzk5OdHMmTOprKzM7DJb2+XhVEUtBS7bQqNf/9Gi/RhjljH3Nyoioj4xJGdtbS1cXV2hVqvh4uJi9n7l6mbck54FqUSEX/93OkQi0S0sJWN9l7m/0Tv27mFPaWvTatURWrR6G5eGMcZBqxtO0vYhl7ldizHb46DVDbFYxJO2MmZHOGiZge8gMmY/OGiZgR/lYcx+cNAyA18eMmY/OGiZQcU1LcbsBgctMzjLePRSxuwFBy0z8OiljNkPDlpmaL97yMPTMGZrHLTMoOKaFmN2g4OWGXjIZcbsBwctM7T30+LLQ8ZsjYOWGbhHPGP2g4OWGbhzKWP2g4OWGZQ8yzRjdoODlhm4nxZj9oODlhlUcikAoIFrWozZHActM7TVtBo0Ouj0fWJ0asbsFgctMzjL20cvbdBwbYsxW+KgZQa5gwQyieGj4nYtxmyLg5aZeCBAxuwDBy0zCd0euKbFmE1x0DJTW9DiO4iM2ZZVQWvt2rUICgqCQqFAZGQkDhw40GX6TZs2YejQoVAoFBg5ciS2bdtmtF0kEplcVq1aJaQJCgrqsH3FihXWFN8qfHnImH2wOGhlZmYiJSUFaWlpKCgoQFhYGGJjY1FZWWkyfU5ODhISErBgwQIcOnQIcXFxiIuLw7Fjx4Q0ZWVlRstnn30GkUiE+Ph4o7zeeOMNo3SLFy+2tPhWU/GjPIzZBYuD1jvvvIOFCxdi/vz5uPvuu7Fu3To4OTnhs88+M5n+vffew7Rp0/Diiy9i2LBh+Otf/4qxY8figw8+ENL4+voaLd9//z2mTJmC4OBgo7xUKpVROmdnZ0uLbzVnfpSHMbtgUdDSaDTIz89HTExMewZiMWJiYpCbm2tyn9zcXKP0ABAbG9tp+oqKCmzduhULFizosG3FihXw8PDAmDFjsGrVKmi1nQeQlpYW1NbWGi03gx/lYcw+OFiSuKqqCjqdDj4+PkbrfXx8cPLkSZP7lJeXm0xfXl5uMv0XX3wBlUqFhx9+2Gj9s88+i7Fjx8Ld3R05OTlITU1FWVkZ3nnnHZP5pKen4/XXXzf31Lql4iGXGbMLFgWt2+Gzzz7DnDlzoFAojNanpKQIr0eNGgWZTIannnoK6enpkMvlHfJJTU012qe2thYBAQFWl4tHL2XMPlgUtDw9PSGRSFBRUWG0vqKiAr6+vib38fX1NTv9f//7XxQXFyMzM7PbskRGRkKr1eLcuXMYMmRIh+1yudxkMLMW3z1kzD5Y1KYlk8kQHh6OrKwsYZ1er0dWVhaioqJM7hMVFWWUHgB27NhhMv0//vEPhIeHIywsrNuyHD58GGKxGN7e3pacgtXaBwLky0PGbMniy8OUlBQkJiZi3LhxiIiIwOrVq9HQ0ID58+cDAObNm4f+/fsjPT0dAPDcc88hOjoab7/9NmbMmIENGzbgl19+wSeffGKUb21tLTZt2oS33367wzFzc3ORl5eHKVOmQKVSITc3F0uWLMHjjz+Ofv36WXPeFuMhlxmzDxYHrdmzZ+PKlStYvnw5ysvLMXr0aGzfvl1obC8tLYVY3F6BmzBhAjIyMvDKK6/gz3/+M0JCQvDdd99hxIgRRvlu2LABRISEhIQOx5TL5diwYQNee+01tLS0YODAgViyZIlRm9Wt1nZ5yI/xMGZbIiLqEwNE1dbWwtXVFWq1Gi4uLhbvf6j0N8z8MAd39XPE3mV/uAUlZKxvM/c3ys8emqltwlZ+9pAx2+KgZSbltSGX61u06COVU8bsEgctM7W1abXqCC1avY1Lw1jfxUHLTE7S9iGX+Q4iY7bDQctMYrGIJ21lzA5w0LIA99VizPY4aFmAH+VhzPY4aFmALw8Zsz0OWhZQcU2LMZvjoGUBZxmPXsqYrXHQsgCPXsqY7XHQsoCSRy9lzOY4aFmg/flDHr2UMVvhoGUBnmWaMdvjoGWB9n5afHnImK1w0LIA94hnzPY4aFmAO5cyZnsctCyg5FmmGbM5DloWUPLopYzZHActC6jaRi/ly0PGbIaDlgWEmpZGB52eh1xmzBY4aFnAWd4+emmDhmtbjNkCBy0LyB0kkEkMHxlfIjJmGxy0LMQDATJmW1YFrbVr1yIoKAgKhQKRkZE4cOBAl+k3bdqEoUOHQqFQYOTIkdi2bZvR9ieeeAIikchomTZtmlGaq1evYs6cOXBxcYGbmxsWLFiA+vp6a4p/U7iDKWO2ZXHQyszMREpKCtLS0lBQUICwsDDExsaisrLSZPqcnBwkJCRgwYIFOHToEOLi4hAXF4djx44ZpZs2bRrKysqE5euvvzbaPmfOHBw/fhw7duzAli1b8PPPP2PRokWWFv+mcQdTxmyMLBQREUFJSUnCe51OR/7+/pSenm4y/axZs2jGjBlG6yIjI+mpp54S3icmJtJDDz3U6TGLiooIAB08eFBY95///IdEIhFdunTJrHKr1WoCQGq12qz0nXl0XQ4FLttCW49evql8GGPGzP2NWlTT0mg0yM/PR0xMjLBOLBYjJiYGubm5JvfJzc01Sg8AsbGxHdJnZ2fD29sbQ4YMwTPPPIPq6mqjPNzc3DBu3DhhXUxMDMRiMfLy8kwet6WlBbW1tUZLT1BxTYsxm7IoaFVVVUGn08HHx8dovY+PD8rLy03uU15e3m36adOm4csvv0RWVhZWrlyJPXv2YPr06dDpdEIe3t7eRnk4ODjA3d290+Omp6fD1dVVWAICAiw51U4586M8jNmUg60LAACPPfaY8HrkyJEYNWoUBg0ahOzsbEydOtWqPFNTU5GSkiK8r62t7ZHAxUMuM2ZbFtW0PD09IZFIUFFRYbS+oqICvr6+Jvfx9fW1KD0ABAcHw9PTE6dPnxbyuLGhX6vV4urVq53mI5fL4eLiYrT0hLbLQ+5cyphtWBS0ZDIZwsPDkZWVJazT6/XIyspCVFSUyX2ioqKM0gPAjh07Ok0PABcvXkR1dTX8/PyEPGpqapCfny+k2bVrF/R6PSIjIy05hZvGo5cyZmOWtvBv2LCB5HI5rV+/noqKimjRokXk5uZG5eXlREQ0d+5cevnll4X0+/btIwcHB3rrrbfoxIkTlJaWRlKplAoLC4mIqK6ujpYuXUq5ublUUlJCO3fupLFjx1JISAg1NzcL+UybNo3GjBlDeXl5tHfvXgoJCaGEhASzy91Tdw8/23uWApdtoeSMgpvKhzFmzNzfqMVBi4hozZo1NGDAAJLJZBQREUH79+8XtkVHR1NiYqJR+o0bN1JoaCjJZDIaPnw4bd26VdjW2NhI9957L3l5eZFUKqXAwEBauHChEATbVFdXU0JCAimVSnJxcaH58+dTXV2d2WXuqaC18WApBS7bQk98lndT+TDGjJn7GxURUZ8YrqC2thaurq5Qq9U31b71n8IyPPNVAcYH9cOmpyf0YAkZ69vM/Y3ys4cWart7yG1ajNkGBy0L8bOHjNkWBy0LqXjIZcZsioOWhZRtQy63aNFHmgMZsysctCzU1qbVqiO0aPU2Lg1jfQ8HLQs5SduHXOZ2LcZuPw5aFhKLRTymFmM2xEHLCnwHkTHb4aBlBR4nnjHb4aBlBb48ZMx2OGhZQcU1LcZshoOWFZxlPHopY7bCQcsKPHopY7bDQcsK7XcPW21cEsb6Hg5aVmh//lBn45Iw1vdw0LICD7nMmO1w0LJCez8tvjxk7HbjoGUF7hHPmO1w0LICdy5lzHY4aFlBybNMM2YzHLSsoOTRSxmzGQ5aVlC1jV7Kl4eM3XYctKwg1LQ0Ouj0POQyY7cTBy0rOMvbRy9t0HBti7HbyaqgtXbtWgQFBUGhUCAyMhIHDhzoMv2mTZswdOhQKBQKjBw5Etu2bRO2tba2YtmyZRg5ciScnZ3h7++PefPm4fLly0Z5BAUFQSQSGS0rVqywpvg3Te4ggUxi+Oj4EpGx28vioJWZmYmUlBSkpaWhoKAAYWFhiI2NRWVlpcn0OTk5SEhIwIIFC3Do0CHExcUhLi4Ox44dAwA0NjaioKAAr776KgoKCvDtt9+iuLgYDz74YIe83njjDZSVlQnL4sWLLS1+j+GBABmzEbJQREQEJSUlCe91Oh35+/tTenq6yfSzZs2iGTNmGK2LjIykp556qtNjHDhwgADQ+fPnhXWBgYH07rvvWlpcgVqtJgCkVqutzuN6k1buosBlWyj//NUeyY+xvs7c36hFNS2NRoP8/HzExMQI68RiMWJiYpCbm2tyn9zcXKP0ABAbG9tpegBQq9UQiURwc3MzWr9ixQp4eHhgzJgxWLVqFbTazms5LS0tqK2tNVp6EncwZcw2HCxJXFVVBZ1OBx8fH6P1Pj4+OHnypMl9ysvLTaYvLy83mb65uRnLli1DQkICXFxchPXPPvssxo4dC3d3d+Tk5CA1NRVlZWV45513TOaTnp6O119/3ZLTswhfHjJmGxYFrVuttbUVs2bNAhHho48+MtqWkpIivB41ahRkMhmeeuoppKenQy6Xd8grNTXVaJ/a2loEBAT0WFlVXNNizCYsClqenp6QSCSoqKgwWl9RUQFfX1+T+/j6+pqVvi1gnT9/Hrt27TKqZZkSGRkJrVaLc+fOYciQIR22y+Vyk8GspzjzozyM2YRFbVoymQzh4eHIysoS1un1emRlZSEqKsrkPlFRUUbpAWDHjh1G6dsC1qlTp7Bz5054eHh0W5bDhw9DLBbD29vbklPoMTzkMmO2YfHlYUpKChITEzFu3DhERERg9erVaGhowPz58wEA8+bNQ//+/ZGeng4AeO655xAdHY23334bM2bMwIYNG/DLL7/gk08+AWAIWI888ggKCgqwZcsW6HQ6ob3L3d0dMpkMubm5yMvLw5QpU6BSqZCbm4slS5bg8ccfR79+/Xrqs7BI2+Uhdy5l7Daz5tbkmjVraMCAASSTySgiIoL2798vbIuOjqbExESj9Bs3bqTQ0FCSyWQ0fPhw2rp1q7CtpKSEAJhcdu/eTURE+fn5FBkZSa6urqRQKGjYsGH05ptvUnNzs9ll7ukuD+/v/JUCl22hl7852iP5MdbXmfsbFRFRn3h4rra2Fq6urlCr1d22l5nj830leP2HIjwQ5o81CWN6oISM9W3m/kb52UMrtffT4iGXGbudOGhZiYdcZsw2OGhZqe3uIc/Iw9jtxUHLSkq+e8iYTXDQspKK+2kxZhMctKykbBtyuUWLPnIDljG7wEHLSm1tWq06QotWb+PSMNZ3cNC6ERGw63+B0v1dJnOStg+5zHcQGbt9OGjd6MAnwM+rgIzZwJXiTpOJxSIeU4sxG+CgdaMxc4G7xgPNNcC/4oHask6Tcl8txm4/Dlo3kjkBCZmAx2BAfcEQuJpqTCblgQAZu/04aJni7AE8/i2g9AEqjwOZjwPalg7J+PKQsduPg1Zn+gUCc/4NyFTAuf8Cm58C9MZ3Cdv6al2p7xjQGGO3BgetrviNAh77ChBLgeObgR9TDXcXrwn2dAYAvPb/jiPzYCn312LsNuCg1Z3gaGDmOsPrvHXAvveETUv+GIrfD/FCi1aPZd8U4oVNR9DIj/Uwdktx0DLHyEeAe/+v4fXONODIBgCAm5MMnyWOx4uxQyAWAd8WXMJDH+zDqYo6GxaWsTsbBy1zTUgGopINr79PAk4bxr0Xi0VImjIYGQvvgbdKjlOV9Xjwg334tuCiDQvL2J2Lg5Yl/vhXYOSjgF4LZM4FLh8SNt0T7IGtz07CxMEeaGrVIWXjEbz8zVE0t+psWGA7UfJfYPebQEO1rUvC7gActCwhFgMPfQgMjAZaG4CvHgWunhU2e6nk+PJ/IvF8TAhEImDDwQuIW7sPZ6/U27DQNtRUA3yfDHxxP7BnJfD3qcCVX21dKtbLcdCylIMMmP0vwHck0HAF+OfDwKmdgN5Qo5KIRXg+JhT//J9IeDjLcLK8Dg+s2Ysfjly2ccHN0HgV+PYp4MMo4JfPTPZNM9vJrcDaSODQPw3vnTyB30qAf8QAZ7N7pLisb+KJLaxVVw78449ATanhvdIXGPUoEPZ/AJ+7AQAVtc1Y/PUhHCi5CgCIDvXCMD8XBHs6Y6CXMwZ6OsPDWQaRSHTz5blZp3Ya2urqy9vXufQHfrfE8GiTVGFePvWVwH9eMnQRAQxPFjz4AeAZAmyYA1zYD4gdgBlvA+FP9PhpsN7L3N8oB62bob4I7F0NHPs30PRb+3q/MCAsARj5KLQKd7yz41d8mH3GZBYqhQOCPZwQ7lqHMfJLCME5eOquoNZjDC76/gG/kRL1LVo0tGhR36xFXdvrFi3qmrVwdZRiUognJod6wc/V0fJz0DQAO5YDB/9ueO8ZCoycBfzyD6Du2nOXSl9g4nOGICNzMp0PEXB0I7B9meGzEEmAic8C0S+3B7zWZuD/LQYKNxreRyUDf3wDEEtM58n6FA5aN7glQauNVgOc+gk48jXw64+A/toMPWIHYPAfgdEJOK6MwoELDbhYUQVd+XE4Xj0J/5bTGCo6jyGiC3ARNXXItpUkyNEPx1Z9JH7SjUMNVF0WI9RHickhXoge4oXxQe5QSLsJBhd/Ab5dBFy9FlAjnwZiXgOkjoYAc+ifhqBce+1OqLMXMOFZYNz/AHJlez7qi8CWJYbPAAB8R0J7/xqUOQ3BueoGnKtuRIW6GToikJ4w4dJnmHzJMFlvseskfDMwDRqxE4gIegL6OUkx0MsZQR6G2qibk6ybL4DdLlfqWvBtwUX891QVBnk5Y+owH0QGu0PucPP/8XDQusEtDVrXa6gGjn8LHM4ALhe0r1e4AU4e1xruO37kOpEU5fIgnMIAXGxVIYoOYZD+fPt2SFCiCscprxhc8p0KmcoTzjIHXPitEXt+vYIjF2qgvy5bhVSMyIEeiA71wuRQLwzycm6/DNW1Gobf+fktgHSAyh+I+xAYNKXj+Wg1wJEM4L9vt18KO3kAUcnQjJkP9cEM9Nv3f+GgbYBWJMV3ro/j49b7ca5Gg1Zd539aD4hz8Jb0Y8hFrTiuD8QCzVKUw8Nk2n5OUgz0dEaQpzOCr/070NMQ1JzlFk+Sziyk0xN+/vUKNhwsRdaJSmj1xt+rs0yCyaFemDrMB1OGeMFDKbfqOLc0aK1duxarVq1CeXk5wsLCsGbNGkRERHSaftOmTXj11Vdx7tw5hISEYOXKlbjvvvuE7USEtLQ0fPrpp6ipqcHEiRPx0UcfISQkREhz9epVLF68GD/88APEYjHi4+Px3nvvQalUmjpkB7ctaF3vSrGh9nUkE6i7riFe6Qv4DAd8RwA+1xbPEEAiNd6/6hRQ9B1w/HugorB9vUgCDJwMDI8DQu4FVH6oaWrFvtPV2PNrJX7+tQrltc1GWfV3c8RwfxeEK6/g0fN/hbv6OACARjwC0Yy3AMd+nZ6GVqfH+StqNBz8CgOOfwS3ZkPNS0MSyESGGxD5+hC81LoIZ6i/sJ/MQYxAdycEejjD300BB7EYIhEgFgFikQj+9cfwyKkX4dz6G+qkXvhu2Fu4ohyGK/UtKKlqQElVAypqu74ZcFc/RwzxUWGIb/sS7KmEzIHvMd2sC1cbsfGXC9j0y0Wjv6fRAW64f5QfTlfWI+tkJa7UtX9HIhEwJsANU4f5IGaYD0J9lGa32d6yoJWZmYl58+Zh3bp1iIyMxOrVq7Fp0yYUFxfD29u7Q/qcnBxMnjwZ6enpuP/++5GRkYGVK1eioKAAI0aMAACsXLkS6enp+OKLLzBw4EC8+uqrKCwsRFFRERQKQ3vI9OnTUVZWho8//hitra2YP38+xo8fj4yMjB79QG4JvQ64kGe4G+czAlB6WZ5H1WlDACv6DigvNN4mUwLuwYZGb4/BII9BKIUfsqtcsaOkBQdKrqJVp8U8yQ6kOmRAIWpFDTnjldb/wR7pJOGmQNviKJXgVGU9TlXUobiiHmeu1ENzbUhpCXR4QJyLxQ6bMUhchgaS4wunRBT6PYoBXioEeTgj0MMJQR7O8HVRQCzu5g/2t/PXBlw8AUidgIc/AYY9IGxuaNEaLjGvNOBS5RVUVFbgavUVqGuqoW2uRy05oxoq/EYq1MMRgAgOYhGCvZwxxNcFQ3yUGOLrggHuTnCWS6CUO8BZ7gCppBcENb0OaKoBNVxBk7oSjTWVaFZfgabuCvT1VdDCAS1yj+sWd2gUHtDI3SG61k4oEokgEYmgVDjAReEAlUIKF0cHuCikhuYDXSvQUmdYNPVoaWlG7tlq7CiqwOELagCG70+pcMAfhvogdrgfgjydDdFJLIVeLMWJKxrsOatG1qkaHClrghYSYb+7+jli6lBvJE4IQrBX1xWMWxa0IiMjMX78eHzwwQeGz1WvR0BAABYvXoyXX365Q/rZs2ejoaEBW7ZsEdbdc889GD16NNatWwcigr+/P1544QUsXboUAKBWq+Hj44P169fjsccew4kTJ3D33Xfj4MGDGDduHABg+/btuO+++3Dx4kX4+/t3W26bBq2eVn0GKPresJQfBaiLMeqdPKDrF4zGZg1U1UcAAIWKcCzHMzisdoK5376jVIIQHyVCfVQY4qNCiJcjRmqOwD1oBESud93c+TTXAv+eD5zeCUAEDI4BtM1As7p9aant+jwBtMIBv5EK1aTEb6TC1WvB7CpcUE8KaCERFoilcHCQQiKVwUEqg0wqg1QmhVQqg1QihlQiMixiw+IgARyE14Z/JWLD/QfCtQt+ovb3REIjABFBpG2GWNsEsbYRYm0TJNomOOiaIL72r4OuCVJdMxx0DXBsrYGTVg0l1UFsoimhO3oS4SpUqCJXVJMLauAMR2jgLGqGEk1QognOomao0AS5qOdnSCeIoIUDmskBLZBCAwfUxv0LQ8dM7HI/c3+jFjUIaDQa5OfnIzU1VVgnFosRExOD3Nxck/vk5uYiJSXFaF1sbCy+++47AEBJSQnKy8sRExMjbHd1dUVkZCRyc3Px2GOPITc3F25ubkLAAoCYmBiIxWLk5eVh5syZHY7b0tKClpb2amttba0lp2rfPAYBk1IMi1YD/HbO0Jheffracsaw1F0GGqshaaw2NOE7OAL3/hUjxz+JzSIRmlt1uHC1EWevXYqVXDH826DRYrB3e4AK9VHhrn6OJmpNfj1zPgoXw8CLP6Yahrs+vaPztGIHQOFqWKROhoDXWAW0NkIKLbxFv8Fb9Fvn+1+PAGiuLXZMTU6G4CNSoUHsikapGzTSfpCLtXDR/QaVrgauut/goq+BUl8LsYjgiVp4isz/m28mKerhCA0c4CACHGUOcJKKIRGLrhvZhNpfk95ww0mrAXQtRv+hiECQohVSUStUMNxg8vUxrxnHHBYFraqqKuh0Ovj4+Bit9/HxwcmTJ03uU15ebjJ9eXm5sL1tXVdpbrz0dHBwgLu7u5DmRunp6Xj99dfNPLNezEEGeIUalhu11Bsa/qtPA/UVhvYvj0HCZoVUghAfFUJ8ur4reVtIHID7VgGh0wxBuC0wtS1yl2uBytFwaXIjTSPQdBVorL62XHvdUGUIappGQN8KvbYVOm0rtFoN9FoN9NpW6HVa6LWtIF0roNdCTwSCCHqh1gToYajB6Nvek6FGQWi7ELpGBIiM10AEQCuWoVXsiFaJAlqxAq0OjtCJHaGVOELn4AidRAGdgxNI6gQHpRdkLl5wcvOBUz9P9FM6w89JhmCZGXfodNpr530FaKg0nH9TjeFzk6sAuRJ6qRJNYifUkQJqnRy1egXULUBtcyt8XBS4J9jDEKwsodMagpe2BdBp2v+99lrsGdJ9Hma6Y2+9pKamGtXwamtrERAQYMMS2YBcaRgTzG+UrUtivsFTrdtP5mRYurlUFV9bpF2m6sUkDoDKx7B0QgzA+dri25PHlTgAMueeyrFTFrVGenp6QiKRoKKiwmh9RUUFfH1Nn76vr2+X6dv+7S5NZWWl0XatVourV692ely5XA4XFxejhTHW+1kUtGQyGcLDw5GVlSWs0+v1yMrKQlRUlMl9oqKijNIDwI4dO4T0AwcOhK+vr1Ga2tpa5OXlCWmioqJQU1OD/Px8Ic2uXbug1+sRGRlpySkwxno7stCGDRtILpfT+vXrqaioiBYtWkRubm5UXl5ORERz586ll19+WUi/b98+cnBwoLfeeotOnDhBaWlpJJVKqbCwUEizYsUKcnNzo++//56OHj1KDz30EA0cOJCampqENNOmTaMxY8ZQXl4e7d27l0JCQighIcHscqvVagJAarXa0lNmjN0G5v5GLQ5aRERr1qyhAQMGkEwmo4iICNq/f7+wLTo6mhITE43Sb9y4kUJDQ0kmk9Hw4cNp69atRtv1ej29+uqr5OPjQ3K5nKZOnUrFxcVGaaqrqykhIYGUSiW5uLjQ/Pnzqa6uzuwyc9BizL6Z+xvlx3gYY3bB3N9oL+gWzBhj7ThoMcZ6lTu2n9aN2q6C76ie8YzdQdp+m921WPWZoFVXZ5jWq891MGWsl6mrq4Orq2un2/tMQ7xer8fly5ehUqm6HSqjrff8hQsX7phG+zvtnO608wH4nIgIdXV18Pf3h1jcectVn6lpicVi3HWXZaMR3Ik96e+0c7rTzgfo2+fUVQ2rDTfEM8Z6FQ5ajLFehYOWCXK5HGlpaZDLrRvr2h7daed0p50PwOdkrj7TEM8YuzNwTYsx1qtw0GKM9SoctBhjvQoHLcZYr8JBizHWq3DQusHatWsRFBQEhUKByMhIHDhwwNZFstprr70GkUhktAwdOtTWxbLIzz//jAceeAD+/v4QiUTC1HNtiAjLly+Hn58fHB0dERMTg1OnTtmmsGbq7pyeeOKJDt/btGnTbFNYM6Snp2P8+PFQqVTw9vZGXFwciouLjdI0NzcjKSkJHh4eUCqViI+P7zAvhLk4aF0nMzMTKSkpSEtLQ0FBAcLCwhAbG9thUo3eZPjw4SgrKxOWvXv32rpIFmloaEBYWBjWrl1rcvvf/vY3vP/++1i3bh3y8vLg7OyM2NhYNDc3m0xvD7o7JwCYNm2a0ff29ddf38YSWmbPnj1ISkrC/v37sWPHDrS2tuLee+9FQ0ODkGbJkiX44YcfsGnTJuzZsweXL1/Gww8/bN0Bb+n4qb1MREQEJSUlCe91Oh35+/tTenq6DUtlvbS0NAoLC7N1MXoMANq8ebPwXq/Xk6+vL61atUpYV1NTQ3K5nL7++msblNByN54TEVFiYiI99NBDNilPT6isrCQAtGfPHiIyfCdSqZQ2bdokpDlx4gQBoNzcXIvz55rWNW2zZ18/03V3s2f3BqdOnYK/vz+Cg4MxZ84clJaW2rpIPaa72cl7s+zsbHh7e2PIkCF45plnUF1dbesimU2tVgMA3N3dAQD5+flobW01+p6GDh2KAQMGWPU9cdC6pqvZszubxdreRUZGYv369di+fTs++ugjlJSUYNKkScLYYr2dObOT90bTpk3Dl19+iaysLKxcuRJ79uzB9OnTodPpbF20bun1ejz//POYOHEiRowYAcDwPclkMri5uRmltfZ76jND0/RF06dPF16PGjUKkZGRCAwMxMaNG7FgwQIblox15bHHHhNejxw5EqNGjcKgQYOQnZ2NqVOtnIH7NklKSsKxY8duadsp17SusWb27N7Gzc0NoaGhOH36tK2L0iPMmZ38ThAcHAxPT0+7/96Sk5OxZcsW7N6922jsOl9fX2g0GtTU1Bilt/Z74qB1jTWzZ/c29fX1OHPmDPz8/GxdlB5hzuzkd4KLFy+iurrabr83IkJycjI2b96MXbt2YeDAgUbbw8PDIZVKjb6n4uJilJaWWvc99dQdgztBd7Nn9zYvvPACZWdnU0lJCe3bt49iYmLI09OTKisrbV00s9XV1dGhQ4fo0KFDBIDeeecdOnToEJ0/f56IzJud3N50dU51dXW0dOlSys3NpZKSEtq5cyeNHTuWQkJCqLm52dZFN+mZZ54hV1dXys7OprKyMmFpbGwU0jz99NM0YMAA2rVrF/3yyy8UFRVFUVFRVh2Pg9YNupo9u7eZPXs2+fn5kUwmo/79+9Ps2bPp9OnTti6WRXbv3k0AOixts5ibMzu5venqnBobG+nee+8lLy8vkkqlFBgYSAsXLrTr/zhNnQsA+vzzz4U0TU1N9Kc//Yn69etHTk5ONHPmTCorK7PqeDyeFmOsV+E2LcZYr8JBizHWq3DQYoz1Khy0GGO9CgctxlivwkGLMdarcNBijPUqHLQYY70KBy3GWK/CQYsx1qtw0GKM9Sr/H1K+ExjXxHNCAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:10:29.777734Z", + "iopub.status.busy": "2024-02-29T15:10:29.777395Z", + "iopub.status.idle": "2024-02-29T15:11:17.451415Z", + "shell.execute_reply": "2024-02-29T15:11:17.450613Z" + }, + "papermill": { + "duration": 47.697124, + "end_time": "2024-02-29T15:11:17.453837", + "exception": false, + "start_time": "2024-02-29T15:10:29.756713", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:11:17.493625Z", + "iopub.status.busy": "2024-02-29T15:11:17.493312Z", + "iopub.status.idle": "2024-02-29T15:11:17.513526Z", + "shell.execute_reply": "2024-02-29T15:11:17.512611Z" + }, + "papermill": { + "duration": 0.042442, + "end_time": "2024-02-29T15:11:17.515639", + "exception": false, + "start_time": "2024-02-29T15:11:17.473197", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.1152480.0234160.0005080.405240.1059022.4253790.2209078.237028e-082.0878190.0177030.2690940.0225430.1616950.0000352.493059
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.115248 0.023416 0.000508 0.40524 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.105902 2.425379 0.220907 8.237028e-08 2.087819 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.017703 0.269094 0.022543 0.161695 0.000035 \n", + "\n", + " total_duration \n", + "realtabformer 2.493059 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:11:17.553561Z", + "iopub.status.busy": "2024-02-29T15:11:17.553303Z", + "iopub.status.idle": "2024-02-29T15:11:17.983527Z", + "shell.execute_reply": "2024-02-29T15:11:17.982604Z" + }, + "papermill": { + "duration": 0.451415, + "end_time": "2024-02-29T15:11:17.985563", + "exception": false, + "start_time": "2024-02-29T15:11:17.534148", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:11:18.025744Z", + "iopub.status.busy": "2024-02-29T15:11:18.025451Z", + "iopub.status.idle": "2024-02-29T15:12:09.210182Z", + "shell.execute_reply": "2024-02-29T15:12:09.209157Z" + }, + "papermill": { + "duration": 51.208162, + "end_time": "2024-02-29T15:12:09.213170", + "exception": false, + "start_time": "2024-02-29T15:11:18.005008", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:12:09.255582Z", + "iopub.status.busy": "2024-02-29T15:12:09.255237Z", + "iopub.status.idle": "2024-02-29T15:12:09.272658Z", + "shell.execute_reply": "2024-02-29T15:12:09.271919Z" + }, + "papermill": { + "duration": 0.040881, + "end_time": "2024-02-29T15:12:09.274649", + "exception": false, + "start_time": "2024-02-29T15:12:09.233768", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:12:09.312810Z", + "iopub.status.busy": "2024-02-29T15:12:09.312518Z", + "iopub.status.idle": "2024-02-29T15:12:09.317658Z", + "shell.execute_reply": "2024-02-29T15:12:09.316743Z" + }, + "papermill": { + "duration": 0.026693, + "end_time": "2024-02-29T15:12:09.319823", + "exception": false, + "start_time": "2024-02-29T15:12:09.293130", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.0504238942397623}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:12:09.360369Z", + "iopub.status.busy": "2024-02-29T15:12:09.359650Z", + "iopub.status.idle": "2024-02-29T15:12:09.791791Z", + "shell.execute_reply": "2024-02-29T15:12:09.790850Z" + }, + "papermill": { + "duration": 0.455132, + "end_time": "2024-02-29T15:12:09.794069", + "exception": false, + "start_time": "2024-02-29T15:12:09.338937", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAE8CAYAAACGvqckAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCw0lEQVR4nO3deXhU5dn48e+ZfTJJJgshCyQkbEFEFtmKqKBSFbUVrVpRC6iIbaFqkfen9OqLolVqX7VQ21JtayKtS6tFbeu+ghs7KIqyCSEEQoDsk9nP+f0xyUDIQjKZ5EyS+3NduWBOnjnnziFz8zzPeRZF0zQNIYSIEQa9AxBCiJNJUhJCxBRJSkKImCJJSQgRUyQpCSFiiiQlIURMkaQkhIgpkpSEEDFFkpIQIqZIUhItuv/++1EUJarnLCwsRFEUNm3a1Kby//d//8fAgQMxGo2MHj06qrGI2CRJSbTLww8/zCuvvNIl13r77bf5f//v/zF58mQKCgp4+OGHu+S6Ql8mvQMQ3cvDDz/MNddcw4wZMzr9Wu+//z4Gg4G//vWvWCyWTr+eiA1SU+ohXC6X3iFEXVlZGXa7PWoJSdM03G53VM7VEXV1dXqHENMkKXVDDX09O3bs4IYbbiA5OZlzzz0XgL///e+MHTsWu91OSkoK119/PcXFxY3e/9FHH3HttdeSk5OD1WolOzubn//856f9wCqKgsvl4plnnkFRFBRFYc6cOQAUFRXx05/+lPz8fOx2O6mpqVx77bXs37+/2XPV1dVx++23k5qaSmJiIrNmzaKioqLRtQoKCnC5XOFrFRYWAhAIBHjwwQcZNGgQVquV3NxcfvGLX+D1ehtdIzc3lyuuuIK33nqLcePGYbfbefLJJ/nwww9RFIV//vOfLF26lH79+pGQkMA111xDVVUVXq+Xu+66i759+xIfH8/NN9/c5NxtvddTp05lxIgRbN68mfPPP5+4uDh+8YtftHqfeztpvnVj1157LUOGDOHhhx9G0zQeeugh/vd//5frrruOuXPncvToUZ544gnOP/98tm7dSlJSEgAvvvgidXV1/OQnPyE1NZUNGzbwxBNPcPDgQV588cUWr/e3v/2NuXPnMmHCBObNmwfAoEGDANi4cSOffvop119/Pf3792f//v2sXLmSqVOnsmPHDuLi4hqda8GCBSQlJXH//fezc+dOVq5cSVFRUThh/O1vf+Opp55iw4YN/OUvfwHgnHPOAWDu3Lk888wzXHPNNdx9992sX7+eZcuW8fXXX/Pyyy83us7OnTuZOXMmt99+O7fddhv5+fnh7y1btgy73c69997Lnj17eOKJJzCbzRgMBioqKrj//vtZt24dhYWF5OXlsWTJkvB723qvAY4fP8706dO5/vrruemmm0hPT2/nv3Qvo4lu57777tMAbebMmeFj+/fv14xGo/bQQw81Krt9+3bNZDI1Ol5XV9fknMuWLdMURdGKioqaXOdkDodDmz17dpP3N3fOzz77TAO0VatWhY8VFBRogDZ27FjN5/OFj//mN7/RAO3VV18NH5s9e7bmcDganXPbtm0aoM2dO7fR8UWLFmmA9v7774ePDRgwQAO0N998s1HZDz74QAO0ESNGNIph5syZmqIo2vTp0xuVnzRpkjZgwIDw6/bc6ylTpmiA9qc//anJ/RHNk+ZbN/bjH/84/PfVq1ejqirXXXcdx44dC39lZGQwZMgQPvjgg3BZu90e/rvL5eLYsWOcc845aJrG1q1bI4rl5HP6/X6OHz/O4MGDSUpKYsuWLU3Kz5s3D7PZHH79k5/8BJPJxOuvv97qdRq+v3DhwkbH7777bgBee+21Rsfz8vK45JJLmj3XrFmzGsUwceJENE3jlltuaVRu4sSJFBcXEwgEgPbdawCr1crNN9/c6s8lTpDmWzeWl5cX/vvu3bvRNI0hQ4Y0W/bkD9+BAwdYsmQJ//73vxv14wBUVVVFFIvb7WbZsmUUFBRQUlKCdtKCps2d89Q44+PjyczMbLEPqkFRUREGg4HBgwc3Op6RkUFSUhJFRUWNjp98j06Vk5PT6LXT6QQgOzu7yXFVVamqqiI1NbVd9xqgX79+8vSwHSQpdWMn105UVUVRFN544w2MRmOTsvHx8QAEg0G++93vUl5ezj333MOwYcNwOByUlJQwZ84cVFWNKJaf/exnFBQUcNdddzFp0iScTieKonD99ddHfM7WtHVQ58n36FTN3afWjjck2rbe67bEIJqSpNRDDBo0CE3TyMvLY+jQoS2W2759O7t27eKZZ55h1qxZ4ePvvPNOm67TUjJ46aWXmD17No899lj4mMfjobKystnyu3fv5oILLgi/rq2t5fDhw1x22WWtXn/AgAGoqsru3bs544wzwsePHDlCZWUlAwYMaNPP0RFtvdciMtKn1ENcffXVGI1Gli5d2qjpBKH/4Y8fPw6cqAWcXEbTNFasWNGm6zgcjmYTjdFobHLdJ554gmAw2Ox5nnrqKfx+f/j1ypUrCQQCTJ8+vdXrNySt5cuXNzr++OOPA3D55Zef7kfosLbeaxEZqSn1EIMGDeJXv/oVixcvZv/+/cyYMYOEhAT27dvHyy+/zLx581i0aBHDhg1j0KBBLFq0iJKSEhITE/nXv/7VpG+pJWPHjuXdd9/l8ccfJysri7y8PCZOnMgVV1zB3/72N5xOJ8OHD+ezzz7j3XffJTU1tdnz+Hw+LrroIq677jp27tzJH//4R84991y+//3vt3r9UaNGMXv2bJ566ikqKyuZMmUKGzZs4JlnnmHGjBmNal+dpa33WkRGklIPcu+99zJ06FB++9vfsnTpUiDUaXvxxReHP+xms5n//Oc/3HHHHSxbtgybzcZVV13FggULGDVq1Gmv8fjjjzNv3jx++ctf4na7mT17NhMnTmTFihUYjUaeffZZPB4PkydP5t13323xydfvf/97nn32WZYsWYLf72fmzJn87ne/a1Nf0V/+8hcGDhxIYWEhL7/8MhkZGSxevJj77ruvHXerY9pyr0VkFO3U+qcQQuhI+pSEEDFFkpIQIqZIUhJCxBRJSkKImCJJSQgRUyQpCSFiSrcep6SqKocOHSIhISHqC9wLITpO0zRqamrIysrCYGhbHahbJ6VDhw41mdEthIg9xcXF9O/fv01lu3VSSkhIAEI/cGJios7RCCFOVV1dTXZ2dviz2hbdOik1NNkSExMlKQkRw9rTvSId3UKImCJJSQgRUyQpCSFiSrfuU2oLTdMIBAItLjYmWmY0GjGZTDLcQnSpHp2UfD4fhw8flh1JOyAuLo7MzExZ+F50mR6blFRVZd++fRiNRrKysrBYLPI/fjtomobP5+Po0aPs27ePIUOGtHnwmxAd0WOTks/nQ1VVsrOzm+zOKtrGbrdjNpspKirC5/Nhs9n0DinmfVlSxY5D1Zw9IJnBfeNP/wbRRI9NSg3kf/eOkfvXdsdrvbz79RE0DY7WeslKyiXO0uM/YlEnv3FCRImiKAzPDA3i9QVUdh+p1Tmi7kmSkhBRkuKwcPGZGZw7pA8A+4+7dI6oe5Kk1Mvl5uY22UNNdEy/pNCOuKVVnib7wonT0z0plZSUcNNNN5Gamordbuess85i06ZNeoclRLtUe/wcqfYQ9HvpW/Yx2VWbqfP6qfEG9A6t29G1F66iooLJkydzwQUX8MYbb5CWlsbu3btJTk7WM6xux+fzyTgine0+UsvaXUc5T93IOMsBzgrUkGGMxx/I0zu0bkfXmtIjjzxCdnY2BQUFTJgwgby8PC6++GIGDRrUqdf1BdQWvwJBtc1l/W0oG4mpU6eyYMECFixYgNPppE+fPvzv//5vuCmQm5vLgw8+yKxZs0hMTGTevHkAfPzxx5x33nnY7Xays7O54447cLlO9GuUlZXxve99D7vdTl5eHs8++2xE8YmmKlw+jKqPvq5dAOSnJ3CuvYhUh/xn0V661pT+/e9/c8kll3DttdeyZs0a+vXrx09/+lNuu+22Zst7vV68Xm/4dXV1dUTX/cMHe1r8Xl4fBzPG9Au/fmrtXvzB5vsF+ifbuXbciUXmnv5kH25f4+ksP//u0IhifOaZZ7j11lvZsGEDmzZtYt68eeTk5ITvzaOPPsqSJUvCu8Lu3buXSy+9lF/96lc8/fTTHD16NJzYCgoKAJgzZw6HDh3igw8+wGw2c8cdd1BWVhZRfKKxijofiZ5D2M0KWBMg4AZvDdSWQUK63uF1K7ompW+//ZaVK1eycOFCfvGLX7Bx40buuOMOLBYLs2fPblJ+2bJl4S2Se7rs7Gx++9vfoigK+fn5bN++nd/+9rfhpHThhRdy9913h8vPnTuXG2+8kbvuuguAIUOG8Lvf/Y4pU6awcuVKDhw4wBtvvMGGDRsYP348AH/9618544wzuvxn64kq6nykeUuwJxohdTB4KtHKv8VXfgCrJKV20TUpqarKuHHjePjhhwEYM2YMX375JX/605+aTUqLFy9m4cKF4dcNq9q11/wLBrf4PcMpM1Hmnd9yU/LUWSu3TI5e/8F3vvOdRtNiJk2axGOPPRaeWDxu3LhG5T///HO++OKLRk0yTdPC02127dqFyWRi7Nix4e8PGzaMpKSkqMXcW3kDQVzeILm+49jMBnD2pypgZOf+zZSXb+fSAeP1DrFb0TUpZWZmMnz48EbHzjjjDP71r381W95qtWK1Wjt8XYup7V1pnVW2oxwOR6PXtbW13H777dxxxx1Nyubk5LBr166uCq3Xqazzg6bhVCsxGRwQ3xebqhBUNUx1ZfiDKmaj7g+6uw1dk9LkyZPZuXNno2O7du1iwIABOkUUO9avX9/o9bp16xgyZAhGo7HZ8meffTY7duxg8ODma4HDhg0jEAiwefPmcPNt586dVFZWRjXu3qjG48cSdOEwBEAxQFwqFoMJowJx/nJq3T6S42XeYFvpmr5//vOfs27dOh5++GH27NnDc889x1NPPcX8+fP1DCsmHDhwgIULF7Jz506ef/55nnjiCe68884Wy99zzz18+umnLFiwgG3btrF7925effVVFixYAEB+fj6XXnopt99+O+vXr2fz5s3MnTsXu93eVT9Sj5XisHJelkrfRCvEpYDBiGJLwmw2o2gqrpoKvUPsVnRNSuPHj+fll1/m+eefZ8SIETz44IMsX76cG2+8Uc+wYsKsWbNwu91MmDCB+fPnc+edd4Yf/Tdn5MiRrFmzhl27dnHeeecxZswYlixZQlZWVrhMQUEBWVlZTJkyhauvvpp58+bRt2/frvhxerQUh4WRyQH6JtjAkRY6aDCg2JMAqKuSJ5ztofsU5iuuuIIrrrhC7zBijtlsZvny5axcubLJ9/bv39/se8aPH8/bb7/d4jkzMjL473//2+jYj370ow7FKeq5K0N/xqWEDxkcqVB1FF/1cX1i6qak902IDjpU6aa68iiqpoHNGT5uig8lKH9tuV6hdUuSlITooDe+LGXzN/txeQNgSwofT3CmkhxnJsXk0S+4bkj35pto6sMPP9Q7BNFGqqpR6/ZjDdZgMSU2qillZ2bA8URwyKYV7SE1JSE6oM4fxBSoxaCpmE0msJ60U7O1fqtqb40+wXVTkpSE6IAajx9roAaLyYDB7oSTlw+2JqKh4XdXo6lSW2orab4J0QEubxBL0BUasX1yLQlQTXFs2F+FpgYZVVtFXGJKC2cRJ5OakhAd4PY1JCUFrI13LzEYDQRNoelAnloZQNlWkpSE6ACXL4AlWBeqKVma2VLJXr+RgKuqiyPrvqT5JkQHDEiNo09fEykuy4mO7ZMoltAxT11ka3/1RpKUhOiATKcdnIBmabamZLbHEwD8dfIErq2k+SZER/nq93ezNk1KRluoT8nvlj3g2kqSkhAdcLDcRW11RWiKSXM1JVuo+RbwSE2prXpXUtI0CPi6/qsde3+tWrWK1NTURmuRA8yYMUMmz8ag17YdYPuBY3j8wWaTUpLTSYrDQrJZtlpqq97VpxT0w0ePdf11z7sbTG3b1eLaa6/ljjvu4N///jfXXnstENqF5LXXXmt1BQDR9YKqhlpfAzJb7c3+G2dn9IX0BLBGtrNNb9S7akrdgN1u54YbbgjvQALw97//nZycHKZOnapfYKKJOl8AS9CFApjsTZ+8AWCOC/3pr2tXjbk36101JaM5VGvR47rtcNtttzF+/HhKSkro168fhYWFzJkzp9FGAkJ/oYGToTFKSjPDAQCwxKOhEQz4MQU8YJaVPk+ndyUlRWlzM0pPY8aMYdSoUaxatYqLL76Yr776itdee03vsMQpXOGkpDQ/cBLwqApbimpRgj4mTHRhlKR0Wr0rKXUjc+fOZfny5ZSUlDBt2rSItpISnavOF8AcrMNsMoDF0WwZq8mA12DHFvThqavBEd+ni6PsfqRPKUbdcMMNHDx4kD//+c/ccssteocjmlHnC2JSPZgNLSclRVHAEupX8smo7jaRpBSjnE4nP/jBD4iPj2fGjBl6hyOakZ0cx4g0M6nxllb7ihRzKGH56mQAZVtI8y2GlZSUcOONN0ZlA04RfRlOGyQZoNpy4ilbM4zW0PcCXklKbSFJKQZVVFTw4Ycf8uGHH/LHP/5R73BEa/zu0J+t1JQM9UnJ73V3RUTdniSlGDRmzBgqKip45JFHyM/P1zsc0YLSKg+O2mriFA1jazUlcxwaEPC4ui64bkzXPqX7778fRVEafQ0bNkzPkGLC/v37qaqqYtGiRXqHIlrx9vaDfFl0lFpvoNWaUnJSaKpJvNHfhdF1X7rXlM4880zefffd8GuTSfeQhGgTv7cOAJPJCCZbi+Xy+/eFigSwy4juttA9A5hMJjIyMtpU1uv1NpqoWl19+kesmgzt7xC5f83TNI2gN9QcM1njQgNzW9JQi/JLn1Jb6D4kYPfu3WRlZTFw4EBuvPFGDhw40GLZZcuW4XQ6w1+tDSg0m0NTO+rq6qIec2/ScP8a7qcI8QZUjMHQJpNmW/NjlMLMcaGpJj75XWwLXWtKEydOpLCwkPz8fA4fPszSpUs577zz+PLLL0lIaDqXaPHixSxcuDD8urq6usXEZDQaSUpKoqysDIC4uDiZO9YOmqZRV1dHWVkZSUlJGI1GvUOKKW5fELPqwWhQMFha7uQGKKlVKf62HLvFyCg1CAa5l63RNSlNnz49/PeRI0cyceJEBgwYwD//+U9uvfXWJuWtVmu7xuw0NAsbEpNov6SkpDY3r3sTTyCIKejBZFBOO8nWbItDQyEQVENNuGZWqBQn6N6ndLKkpCSGDh3Knj17onI+RVHIzMykb9+++P3y5KO9zGaz1JBa4K6fYmIyKmBuvflms5gJGCwoQS+avw5FklKrYiop1dbWsnfv3qivsGg0GuXDJaIqxWFhdIaVpHLbaWtKdrORgMGGSfXi89RJRek0dO3oXrRoEWvWrGH//v18+umnXHXVVRiNRmbOnKlnWEKcVlKchaHJBvom2FqdYgJgNhpQ64cM+NwygPJ0dK0pHTx4kJkzZ3L8+HHS0tI499xzWbduHWlpaXqGJUTbtGGKSQPFHAce8Htk/tvp6JqUXnjhBT0vL0TEjtd6MddUE6eqmNqSlCyhMg0DLkXLYqpPSYju4rNvj5Ow9xBDkzQyT9N8A0hJcmLzWLCq3tOW7e0kKQkRAY8vSLLqwWy0t6n5NiovE7QEsAS7ILruTfcR3UJ0R16fF4MWrB+ndPqaUnhunEw1OS1JSkJEIFi/DInJbG7bbjUmGxoaqt/TyZF1f5KUhIhAw9pIRquj9cm49T4v9bL+23J2lxzt7NC6PUlKQrSTP6hCINQMM51uMm49xWxFA6kptYEkJSHaye0PTcZVqF+2pA3M1lDy0qRP6bTk6ZsQ7WQxGhiXZSNBtYcGRbblPbbQEzo14ANZKaBVkpSEaCeb2cgZfUzgimvbkzfAYguVC6oaBLzhveBEU9J8EyISDc2wNiYXq9lE0GAmENQgIP1KrZGkJEQ7VXv8VFZX4QuqbRo4CWA1GQkoVgKqKknpNKT5JkQ77ThUTdXOgwy21DGojc03m8VAYkI8Dr8P1eeW2kArJCkJ0U4efxCz6sZsNLSrpnT2wEyo9ENQ5r+1RhK2EO3k8QcxBb1tn2LSwFS/lLM031olSUmIdvI0Wgq3bTUlAMx2NDQZq3QakpSEaCe/tw4FDZPBAKa2J6WP9tWw/ttySsurOjG67k+SkhDtFJ73ZrGCse3dskFDaKpJ0Cc1pdZIUhKinQL1m0oarW2b99bAWL/6ZMArSak18vRNiHYak2klsdaO1d7epBRaUykofUqtkqQkRDuNSDPD8TiwtW+vJJPVQRBQpabUKmm+CdFe7djF5GRma0NNSYYEtEaSkhDt4PEHqaiqwhsItjspNSxzIkMCWhczSenXv/41iqJw11136R2KEC0qrfKw9qsD7Cytad/ASSAhPgGn3Uy8MQia1kkRdn8x0ae0ceNGnnzySUaOHKl3KEK0yhMILfBmascUkwa56SmQmRh6EfCC2dYJEXZ/uteUamtrufHGG/nzn/9McnKy3uEI0SqPXw2N5jYoYG7f0zeMphPjmgLShGuJ7klp/vz5XH755UybNu20Zb1eL9XV1Y2+hOhKHn8Qc7AhKbWvpgSEdzUhIJNyW6L7tt1btmxh48aNbSq/bNkyli5d2slRCdEyj79h3puh3X1Kdb4Am/fVYPGWM2GUm9PvgdI76VZTKi4u5s477+TZZ5/FZmtb23rx4sVUVVWFv4qLizs5SiEa8/j8mFRvRDUli9GAW7OgaqH5c6J5utWUNm/eTFlZGWeffXb4WDAYZO3atfz+97/H6/ViNDZeXN1qtWK1Wrs6VCHC/A3z3oyGE7vetpHJaEAzhn5/vR43lqhH1zPolpQuuugitm/f3ujYzTffzLBhw7jnnnuaJCQhYsGwPiasSXYcjngwtL+hoZhDSckvo7pbpFtSSkhIYMSIEY2OORwOUlNTmxwXIlYMSTJCShzEJUb0fkN9P1RAmm8t0v3pmxDdSoRTTBoY6iflykoBLYuJwZMNPvzwQ71DEKJFmqZRXlmJwx/EarZH9PTMWL8lU9AnNaWWRFRT+vbbb6MdhxAxzxtQee+L/WwtrkQ1RjYa25kQj9NuxoI/ytH1HBElpcGDB3PBBRfw97//HY9HZjyL3qFhjJJRaf8Cbw1G52UwPDOR9Mhaf71CRElpy5YtjBw5koULF5KRkcHtt9/Ohg0boh2bEDHF41dPmvcW4bbbDfPdZEeTFkWUlEaPHs2KFSs4dOgQTz/9NIcPH+bcc89lxIgRPP744xw9ejTacQqhu9DWSh2YYgInxjZJUmpRh56+mUwmrr76al588UUeeeQR9uzZw6JFi8jOzmbWrFkcPnw4WnEKobvGKwREVlPafdzPpqJyvjl4VJYvaUGHktKmTZv46U9/SmZmJo8//jiLFi1i7969vPPOOxw6dIgrr7wyWnEKobvGKwREWFMy2/AHNYKBAASls7s5EQ0JePzxxykoKGDnzp1cdtllrFq1issuuwxD/QjXvLw8CgsLyc3NjWasQujqxAoBRrBEVlOyWmxoKARULdSEM8lkk1NFlJRWrlzJLbfcwpw5c8jMzGy2TN++ffnrX//aoeCEiCX9EkxoTjPxVlPEzTerxUjQYCWo+mT5khZElJTeeecdcnJywjWjBpqmUVxcTE5ODhaLhdmzZ0clSCFiQXa8FppiYjCCMbIajtVkIGCwElC9stBbCyLqUxo0aBDHjh1rcry8vJy8vLwOByVETApPMYkDJbLVkKwmIwGDhaCqEfTJE7jmRJSUtBaeGtTW1rZ5bSQhupuKqko8/iDBdi5ZcjKryUDQEFopwCeTcpvVrubbwoULAVAUhSVLlhAXd6JdHQwGWb9+PaNHj45qgELEirVfFdG3pJL8+P6kRHgOg0EhMSEBp2KWrZZa0K6ktHXrViBUU9q+fTsWy4l2tcViYdSoUSxatCi6EQoRI9T6SbSmdu6Me6pz8vvBoTJQAtEIq8dpV1L64IMPgNBibCtWrCAxMbI1ZYTobjRNCyclcweTEqb61VPl6VuzInr6VlBQEO04hIhp3oCKqf5pmdkW2WTcsIaBl/L0rVltTkpXX301hYWFJCYmcvXVV7dadvXq1R0OTIhYElohwIvRoGCKcIWABhuKXRiKykk3VJB9RpQC7EHanJScTidK/WNQp9PZaQEJEYtCKwS466eYRLhCQD2fYsEY1Aj4pKbUnDYnpZObbNJ8E71NhzehPInJakcDgpKUmhXROCW3201d3YkxFkVFRSxfvpy33347aoEJEUsS7Wbykgz0SbCCpWPNN1P9vDlVhgQ0K6KkdOWVV7Jq1SoAKisrmTBhAo899hhXXnklK1eujGqAQsSCFLuR3EQDWU57h2tKZlvo/ZpfRnQ3J+KVJ8877zwAXnrpJTIyMigqKmLVqlX87ne/i2qAQsSEhlqNooCpY0nJYq2vKQV8oAY7GlmPE1FSqqurIyEhAYC3336bq6++GoPBwHe+8x2KioqiGqAQsaC6pjo0xcRojWgTypNZ6mtKgaAmK1A2I+KNA1555RWKi4t56623uPjiiwEoKyuTAZWiR9q0u4StxZUcckU2EfdkDqsFu92O3WyUAZTNiCgpLVmyhEWLFpGbm8vEiROZNGkSEKo1jRkzps3nWblyJSNHjiQxMZHExEQmTZrEG2+8EUlIQnSqgNcFgKmjAyeBDKeN0XmZDO4bf6JZKMIiGtF9zTXXcO6553L48GFGjRoVPn7RRRdx1VVXtfk8/fv359e//jVDhgxB0zSeeeYZrrzySrZu3cqZZ54ZSWhCdIpwUurgwMmw8FQTab6dKuIdcjMyMsjIyGh0bMKECe06x/e+971Grx966CFWrlzJunXrJCmJmKJ6Qkmpw1NMGoR3NZHm26kiSkoul4tf//rXvPfee5SVlaGqaqPvR7KDbjAY5MUXX8TlcoWbg6fyer14vSf+Eaurq9t9HSHaS9M0gr76pGTv4GTc+vOt3VeDo7qc4bm1dGx8eM8TUVKaO3cua9as4Uc/+hGZmZnh6SeR2L59O5MmTcLj8RAfH8/LL7/M8OHDmy27bNkyli5dGvG1hIiEN6BirJ88a41CUlIUBbdqxhLU8HulT+lUESWlN954g9dee43Jkyd3OID8/Hy2bdtGVVUVL730ErNnz2bNmjXNJqbFixeHF5qDUE0pOzu7wzEI0RqPP7TfWzQm4zZQ6nfKlaTUVERJKTk5mZSUSNfea8xisTB48GAAxo4dy8aNG1mxYgVPPvlkk7JWqxWr1RqV6wrRVmajgSFJCiaPrcNTTBoYLPVjlWRJ3CYiGhLw4IMPsmTJkkbz36JFVdVG/UZC6M1hNTHQCTkpcWDpePMNwFhfU5KVApqKqKb02GOPsXfvXtLT08nNzcVsNjf6/pYtW9p0nsWLFzN9+nRycnKoqanhueee48MPP+Stt96KJCwhOocahIZ5alGqKRmtoZqS7GjSVERJacaMGVG5eFlZGbNmzeLw4cM4nU5GjhzJW2+9xXe/+92onF+IaKitrcLkD2IymTB1cDJug/BKAT5pvp0qoqR03333ReXisoOu6A527DtMsLiStNRUBnfgSfPJ4uMdKFYjFs0flfP1JBHPLKysrOQvf/kLixcvpry8HAg120pKSqIWnBCxwO+pAcDY0Q0DTjIqN4OR/ZLolxCdJNeTRFRT+uKLL5g2bRpOp5P9+/dz2223kZKSwurVqzlw4EB4rSUhegK/pxYrYIzWFBNoPKJb0yLecbcniqimtHDhQubMmcPu3bsb7Yh72WWXsXbt2qgFJ0QsCNbXlEz2hOidtCEpaZpMNTlFRDWljRs3NjuOqF+/fpSWlnY4KCFiieqpBcAcxaR0qMZP6cEabEaN4QEPmGW7+wYRJSWr1drsvLNdu3aRlpbW4aCEiCVq/QBHSxSTkqJAbdAIqldWCjhFRM2373//+zzwwAP4/aEnB4qicODAAe655x5+8IMfRDVAIfSkqhqKP1RTssRFLylZTUYCBitBVVafPFVESemxxx6jtraWtLQ03G43U6ZMYfDgwSQkJPDQQw9FO0YhdKNqGoOTFNITrFijWFOymgzhpCQbCDQWUfPN6XTyzjvv8Mknn/D5559TW1vL2WefzbRp06IdnxC6MhkNDHQqEBcPtugmpaBiQQN8njpkRucJ7U5KqqpSWFjI6tWr2b9/P4qikJeXR0ZGBpqmdWgZEyFiTjBwYopJB3fGPZnJaEAz28ATWilAktIJ7Wq+aZrG97//febOnUtJSQlnnXUWZ555JkVFRcyZM6ddS+EK0R3Uuapx+4MEMHR4v7dTGUwNy5fIVJOTtaumVFhYyNq1a3nvvfe44IILGn3v/fffZ8aMGaxatYpZs2ZFNUgh9LLn4GFcxZUkJ6cwLMqtAIfDgaPOiCId3Y20q6b0/PPP84tf/KJJQgK48MILuffee3n22WejFpwQevO6QkNfFFv0tw47b3gOI/slkWRRT1+4F2lXUvriiy+49NJLW/z+9OnT+fzzzzsclBCxIlAXSkomeyfsZxje0URGdJ+sXUmpvLyc9PT0Fr+fnp5ORUVFh4MSIlYEPJ2ZlOpHccveb420q08pGAytKdMSo9FIIBDocFBCxArVHUpKlrjoJ6XtZT4CxRUkpVrJi/rZu692JSVN05gzZ06L62TLMraip1G9ocm4Focz6uf2YsHvVwnU7yknQtqVlGbPnn3aMvLkTfQUqqqBNzTFxBafFPXzW+3x+AHV7wFVBUPEy5v1KO1KSgUFBZ0VhxAxR9U0BiaqBH1W7I6kqJ/fandQCwSCwdD8N4tsSwkd2LZbiJ7OpPkZ4DQD5qhOMWlgs1gIGiwE1GCos1uSEtCB5XCF6PHq+5MwWcFkifrpbZbQpNxAUAW/jOpuIElJiBa4aytx+4P4zVFcBvckNrMRv8GGP6ihybCAMElKQrRgX0kp24or2XFM65Tz281GzDYHcVaj7JR7EulTEqIF/voxSsbOGDhJaDvwqSMGQGkNqDL/rYGuNaVly5Yxfvx4EhIS6Nu3LzNmzGDnzp16hiREmN9VBXTOwMmwhpUHpE8pTNektGbNGubPn8+6det455138Pv9XHzxxbhcMphM6C/gDiUlcyeMUQozNSQl6VNqoGvz7c0332z0urCwkL59+7J582bOP/98naISIkRzVwJgi0/ptGtsOuTBWFxBhrmCzE67SvcSU31KVVWh/5lSUpr/JfB6vY2msjS3o4oQ0aCpKnhCv49xiZ2XlFyqBYtfxSdTTcJi5umbqqrcddddTJ48mREjRjRbZtmyZTidzvBXdnZ2F0cpeguvxwXB0G49nZmUTLbQgMmgV5JSg5hJSvPnz+fLL7/khRdeaLHM4sWLqaqqCn8VFxd3YYSiV/FU0z/ZTlpqCmZz9AdONrDUbwWu+aSju0FMNN8WLFjAf//7X9auXUv//v1bLGe1WltcoUCIaLIFa8lOjoPErE69jsnuIAAEfe7QFt6y8Ya+NSVN01iwYAEvv/wy77//Pnl5sqqMiBH1/UnYor9kycmstngAgkFVNqWsp2tNaf78+Tz33HO8+uqrJCQkUFpaCoT2lbPbo7tzhBDt4ao6jsEfxGROxNyJ17FZzZQbzPhVNTQsIMo7pnRHutaUVq5cSVVVFVOnTiUzMzP89Y9//EPPsIRg38FDbCuu5OtOXt3ZYTVhtMRhMRpkAGU9XWtKmtY5c4qE6Khg/RglayeOUQJIT7SRnt8fakplAGW9mHn6JkQsUesqAbAmJHf+xRp23pWkBEhSEqIpv5uAL9TpHO/s3JoSIPPfTiFJSYhTeGuOElA1fEYHCY7OXw1yfYmHrQcqOF4p25OBJCUhmnBVHAUgaE3CajJ2+vVqglY8ARWfS6ZNgSQlIZpwVx8DwODogqYbYLSFRnX7Zf4bECMjuoWIJYlaDdnJduqy+nXJ9cz1i8gFPTVdcr1YJ0lJiFMkqNUkJMdB7oAuuZ7VHh+aaiKTcgFpvgnRmKZBXXno7/auab5ZHKHtm1SvK7QpZS8nSUmIk/lc1LjqqPOrBK2dO++tgc0eSkr+QBACMlZJkpIQJ3OXs/NIDRsOByh3B7vkkg67BaM1DrPRAD5pwkmfkhAn8dUcwx/UcNucJNi65uPRPzmO/kOzwXUMfLVA3y65bqySmpIQJ6mrLANAtaVgM3f+GKUwS/2Gl7LYmyQlIU7mrgwtn2N2pnXthcNJSZpv0nwT4iS+qiMAWJ0ZXXrdzw56sB6uIK9PJV0wBTimSVISooHfjb9+qkdcctcmpdqgFQIqHplqIs03IcJcx/D4g3iN8SQlxHfppc1x9cMC6iQpSU1JiAauMrKSbFhtOfRNtHXppS2O0FSTgFuSkiQlIRq4jpHisJKSkwf2zlyZuymbIxkvkpRAmm9CnOAKLVmCo4ufvAG2+PpJuT4PBLynKd2zSVISAkDTqC0/TEWdjzpz1z//io+LJ2iw4Auq4O3dqwVIUhICwFPJsYoqvj7iYuPRLhw0WS/eZsJgS8RiMoC3dzfhpE9JCICaUup8QVzmVLISu37vtRSHhXOG50H5t1JT0jsAIWKBVn2YOl8AlyWNtHidtoa31g9DkKSkn7Vr1/K9732PrKwsFEXhlVde0TMc0Yt5K0rwBzVc1jRSHBZ9grAm1AdTq8/1Y4SuScnlcjFq1Cj+8Ic/6BmG6O00DffxEgDMSZmYjPp8LDaWBthyoIJDZUd0uX6s0LVPafr06UyfPr3N5b1eL17vicel1dW9u0NQRIm7Are7DlUxEp/StdNLTuZR4jAFVLy1VbrFEAu6VZ/SsmXLcDqd4a/s7Gy9QxI9QfUhXL4AdeZUUuO7vpO7gT0+CYBAnSSlbmPx4sVUVVWFv4qLi/UOSfQE1SX0T45jyNBhDOrbtXPeTuZwpgLg99aB36NbHHrrVkMCrFYrVqtOT0ZEz1V5ALvZSO7AfNDryRuQEO/gqMGO1+8FTyWY9WtK6qlb1ZSEiDq/O7QMLYCzv66hOO1mvKYEvAEVta73buEtSUn0blUlHHd5KfbaqPB37STcU8VbTfgsTjSgrvq4rrHoSdfmW21tLXv27Am/3rdvH9u2bSMlJYWcnBwdIxO9RlUxR6q97DbEk1teR7JeY5QARVGIc/YhMbAPtWHvuV5I16S0adMmLrjggvDrhQsXAjB79mwKCwt1ikr0JsGKA9R4/NQkZ5CdEqd3OFwwagh8swu03rtWt65JaerUqWiapmcIojfze6g9WoyqQTAxh+Q4fZtvANiTQn96KvWMQlfSpyR6r4r9VNd5cZuTSE/vi6IoekcE9tCyKaq7CtSu2Qwz1nSrIQFCRFXFPirdfqpsAxmSrH/TDeCIx0TxwRrMBBnlqYK4FL1D6nJSUxK9k6bhLdtDrTdApS2b3D4OvSMCwG41UaUl4PYFCdYe0zscXUhSEr1T3XHqqsvBYMTRN494a2w0GhKsJny2VDTAVXFY73B0IUlJ9E7HdpEcZ2H86NF896zYmUOpKArGhNAa4a7yUp2j0YckJdE7lX0NgCn9DF3HJjXHntQXAE9Vmc6R6EOSkuh96soJVJeCYoA+Q/WOpglHShYAvqoyUFWdo+l6sdGQFr2bGoS64+CpBn8dqIHQMS0IihEMRrDEg6NP6JF5Bx/dq0d28GVJFS5HNvk+I31iq6JEamoaBxUTdV4vuMtDP3cvIklJ6ENV4fgeOLQVKg+EElFb2JyQfib0Hw+WCB7jaxrH92zE7Vc5aBnIOFvsfQTSnXYOJ2bg1I6iVR9CkaQkRCerOQK73oDq0NMlDY2gwYrJkQxmBxhNYDCFaklaEIL+0LZDruPgqYKiT6FkEwyeBhkj21Vz0sr3cfhIKUGDhZz8s7Gaun47pdOxmY18Z/QIKN4INaWQOVLvkLqUJCXRtUq/hJ2vU+P2UFYH31ryKTLlYYrvw5xxA8PFVm85SGWdH6vZgM1kxGoxYLOpJLuLSK/cSj+qUb55HSr2Q/7loUTWBsVffYLLG6TSeQYX5nb9TrhtlhDqV6K6RN84dCBJqZsKqhoHK+ooq/Giqhqp8VZyU+NCi95rGlQfCvVHGEyQkBGevqCrks3Ubn+d/cddHFCy+Db5PPym0KDFFEPjZy7Vbj9Vbj+4Tz1JGgnWS5mbewy+XQNHdlBVeRz7mOuw2FtfNbL66CGO7P0CUMg+61ziLDH865+YiappuCsO4wgG2px0e4Le85P2ILuP1LB29zGq3f5Gx+1mAxcmH2VI3RYUzymbKiTnwqALISG96wI92fG9HNzwKgfL6yhJGElJyncYmpHIoDQHaQlWHKcMXrxydD/c/iDegIo3EMTjV/H6g7j9QSxGA+QMhoQM1O2r2bPrG1zfrkAdcR3D8rJJT7Q2mcdW5wuw7aN/Y1Y11D75jBgyqCt/+nY7HrDzdbEbq+phbPUhDMm9ZykfSUrdjKZp7Cmrpdrtx24xMiAlDoNBoeRYJeklb1O1rwhffydWmz3UBAj6oOZwqJmzuTCUmPqP6/ATrHapOQJfvYzdbOCII5+4YdO4Jb9vq6Ookx0WTlu3S86ldvhMggf/jNldgX/bKl4tmY4jtR9nZCaSlWQjOc6CzWzEWvktaZ4iXCYjw869HKMhBibftiLZYaXWnoWxZi9VJbtIlqQkYpWiKEwbnk6608aILGdo73mfC9X1X47ElWFPdmIdcgFkTwBj/VIc7krY+x4c3QV73g09fh96SdckJk81bH8Rgn5S++dzTt6VpDmjN/k1sU8m46++k4p1z1F+pBjT0f/yTfAi1taERml/d3g6I5KDGHe9ztD0BNyZE4hPzYra9TuLwaBg6zsYavZSdXgXySOm6R1Sl5HBk91AIKiy/WBVeO0ps9HA2TnJ4YTEtucwuI6SmdaHpHNuhtzJYDRTVu3B7QuG1ug58+rQ0ypFCT2G3/l6pw/Mc7vdfPHmX/C6KkNjbc68KqoJqYFic5Iy+WYGDz2TCdkOpqtrOMf3Kf3UEpzlX8CWVeB3Y3RmEp8/JerX7yzOfqGBna6jxaG1xHsJSUoxLqhqvLb9MO9+fYS1u0+ZNV6fkHAdC+1DP+YmcPYD4Ei1hxc3H2T11oN4/MFQMsoeD2d8L/T3w1+EElMnLbLn8vjY9HoBruOH+KY8iDbiGjDbOuVaQOjcI3+Iqf9YMp12JsYd5jrLOrKPfRz6QCekw1nXnqg9dgMD+2fhNidR4/bjPvyN3uF0GUlKMUxVNd76qpRvj7owGRTyUk9aXsPvhs+fr09ICTD6xkZr75gMCiaDQlm1l5e3luAN1C8Yln4mDL8yNMWidHunJKZab4DP3vonpspvMZvM5EyZjRLXBU//jCbIvxTGzg6N7YlPC+1QMuhCGPOjUOLuRpx2M2raGWhA2Z4teofTZaRPKUZpmsZ735Sxs7QGo0HhilFZ5KTWN338Hvj8Bag9ChYHjL6hyWJgqfFWrj67Py9tPkhplYdXtpZw1Zj+oSZf3zNCiejrf4dqTIoBhl4alT6mao+fj977LynHtmE1GRh8wQ0kZuR2+LztkpgV+uoB+gwai/vQZ1Qf3hPqn7Ml6h1Sp5OaUgzSNI01u47yZUkVigLTR2SQ17AIWcAb6jiuKQWzHUbNbHF1wrQEKz84ux9Ws4FDlR5e3VaCP1jfj5Q+/ERT7tA22P12h2tMVXV+3v3wA1JKP8FmMjD0nCtJzDmrQ+fs7fJzs+nTfzD56fFwcKPe4XQJSUox6IOdZWw9UAmEnh4NSU8IfcNbC9uehaqDYLLCqOtDTZRW9E20cXV9DelghZv/fH6IQDgxnQnDLg8lppItocTUgc7vzZs+IaPkHexmA0PHX0j84HMiPpcIsVuMDBn33dDYrENbQv2IPZwkpRiUnRyH0aDw3eHpnJnlDB2sKYWtfwuN+bHEhWpICW3b1jnDaWPGmH5YTAaMBqXxGJ2MsyD/shOJafuL7X/So2lwcBOTfZ+SEmcif/QkHGd00ZCD3iBlICRkoAX91O14S+9oOl1MJKU//OEP5ObmYrPZmDhxIhs2bNA7pC5X7TkxOntIegJzJucyop8ztITHgfWw5W+h8UY2Z6jTNjGzXefvl2TnqjH9uOTMjPBoZ19ADQ0zyBwJZ3w/1FFc/i1s/EtoTNNpmnMVLh9ffFsCO16B3e9gNSoMO3sK9rNmSEKKJkXBnXcRX5fW8uXWzyjfvU7viDqV7h3d//jHP1i4cCF/+tOfmDhxIsuXL+eSSy5h586d9O3bV+/wOpWmaRyscLPlQAUHK9zMmjSABFvokXWimVBfT/GG0GBHgD5DQs0tsz2i62UlNX7f2ztKqfUEGJebTF7aGRjjUuCrV8BdAV/+K5T4ss4OXbf+mpqmcaTKwzd791Lx7VbSanZQnm4jxWGDgVNDgzYlIUWdJTmbIynjsB38lL2frMZT5yLrrAvAEBP1iqhSNJ13g5w4cSLjx4/n97//PQCqqpKdnc3PfvYz7r333lbfW11djdPppKqqisTE2H8q4Q+qHK3xUu7yUVrlYf9xFzVuP0bNh0X1Mi3PzOA4D1QVh6aFBOtrT5a40Ae+nct0tKaqzs+qz/YTUEP//FazgZyUONIdRjIqt5Bavo24+v+yfEGVUq+V6qCZSpcHPFUYVR8AyXFm+g8YTPyI6T3miVes8vgCrHv7H1iPbAXAlpBCQu7ZOLMGkdonE4PVEXP/IUTyGdU1Kfl8PuLi4njppZeYMWNG+Pjs2bOprKzk1VdfbVTe6/Xi9XrDr6urq8nOzm7TD1xS6eb9r49w8g+bXfoudu9xNDRyU+PIdIZqA9VuH18dqj6l+VLfAaxBTkoc/ZJCAwFrvQG2H6wIlzrxltBfspNtZCfHARq1ngBbDlSihKPQsOCnb7yZDKcdu/mUtX1sTug3FjJHdcrAQ5c3wLbiSr4sqaLO13jjw7FZFs53HISyr6mrKOXzg1Xh7xkUSE5wkJadT/LgCZA6KOY+DD1VIBBk2+aP8XzzLsbAib6/8bnJmIwmMFnYfdzHcZcfRVHqm+oKWv2foDA2NxlTfb/i3qMujtV4m70WwJicpNAwEmD/8TqOVHuIt5g4s18inHXdaYcoRJKUdG2+HTt2jGAwSHp645nr6enpfPNN0xGsy5YtY+nSpRFdyx9QOVbra3Qss7YczXc09MIWB6b65o3HD3WnzLI/mccHnvqy3gB4a8LfavLR9AbBVz89RA0Sb/BiNxuJsxhx2i0k2B0YFSXUn2NLCk3HiE8PdW7Gp3fqh91hNTF5cB8mDUyltNpDSaWbYzVeajwBkpwJ0D8Xcs/F4nHh/vwbkiwqGYlxpPVJwZzQN7RMrehSJpORcROnUDNiHAe+2ULtoW9Qq0sxGgygqeD3oLlr0Nw+Wqxt1PpPNPtqa1FrW05KuPxgDJXVal2oNR5UqwlqvaHrdQLd+5TaY/HixSxcuDD8uqGm1BbpiTZ+cHb/Rp9xY+0PIOhDURQSbCawhvpzrEGVgXV+FAW0+jSjoICioCiE1uGxmEBRsAZVhrj9KEpD214JZyZFCa0iiKX+vCic3XDxhkCMllB/jY7THwwGhawke5M+pwYmm4MLJ47t4qhEaxIcDs4cex6MPS90IBiAgBsCPtJra3H6A2iahqqqJ6rv9UnEkGgL/46m1Pmw+1tOLoYEa6hqDCS7/Vi8QSxGBRKsoYG7nUDXpNSnTx+MRiNHjhxpdPzIkSNkZDR93G21WrFarRFdy24xnhgR3SCl+TV1rEBbVx0yA31iYP000csZTWBMACs4Hak42/i25BROv0RMvaT6r86ma9e9xWJh7NixvPfee+Fjqqry3nvvMWnSJB0jE0LoRffm28KFC5k9ezbjxo1jwoQJLF++HJfLxc0336x3aEIIHeielH74wx9y9OhRlixZQmlpKaNHj+bNN99s0vkthOgddB+n1BHdbZySEL1NJJ/RnjccVAjRrUlSEkLEFElKQoiYontHd0c0dIdVV7cy+loIoZuGz2Z7uq67dVKqqQlN72jrqG4hhD5qampwOts2pLNbP31TVZVDhw6RkJDQZEfU3qph6k1xcbE8kWwHuW+ROd190zSNmpoasrKyMLRxmZVuXVMyGAz0799f7zBiUmJiony4IiD3LTKt3be21pAaSEe3ECKmSFISQsQUSUo9jNVq5b777ot4NYXeSu5bZDrjvnXrjm4hRM8jNSUhREyRpCSEiCmSlIQQMUWSkhAipkhS6gHKy8u58cYbSUxMJCkpiVtvvZXa2tpW3zN16tTwFjwNXz/+8Y+7KGJ9tHcn5hdffJFhw4Zhs9k466yzeP3117so0tjSnvtWWFjY5PfKZmvn9mCa6PYuvfRSbdSoUdq6deu0jz76SBs8eLA2c+bMVt8zZcoU7bbbbtMOHz4c/qqqquqiiLveCy+8oFksFu3pp5/WvvrqK+22227TkpKStCNHjjRb/pNPPtGMRqP2m9/8RtuxY4f2y1/+UjObzdr27du7OHJ9tfe+FRQUaImJiY1+r0pLS9t1TUlK3dyOHTs0QNu4cWP42BtvvKEpiqKVlJS0+L4pU6Zod955ZxdEGBsmTJigzZ8/P/w6GAxqWVlZ2rJly5otf91112mXX355o2MTJ07Ubr/99k6NM9a0974VFBRoTqezQ9eU5ls399lnn5GUlMS4cePCx6ZNm4bBYGD9+vWtvvfZZ5+lT58+jBgxgsWLF1NXV9fZ4erC5/OxefNmpk2bFj5mMBiYNm0an332WbPv+eyzzxqVB7jkkktaLN8TRXLfAGpraxkwYADZ2dlceeWVfPXVV+26breekCugtLSUvn37NjpmMplISUmhtLS0xffdcMMNDBgwgKysLL744gvuuecedu7cyerVqzs75C7X3p2YIXRfmyvf2j3taSK5b/n5+Tz99NOMHDmSqqoqHn30Uc455xy++uqrNk+el6QUo+69914eeeSRVst8/fXXEZ9/3rx54b+fddZZZGZmctFFF7F3714GDWp+k04hTmfSpEmN9mw855xzOOOMM3jyySd58MEH23QOSUox6u6772bOnDmtlhk4cCAZGRmUlZU1Oh4IBCgvL292l+GWTJw4EYA9e/b0uKTU3p2YATIyMtpVvieK5L6dymw2M2bMGPbs2dPm60qfUoxKS0tj2LBhrX5ZLBYmTZpEZWUlmzdvDr/3/fffR1XVcKJpi23btgGQmZkZ7R9Fd5HsxDxp0qRG5QHeeeedXrVzczR2sA4Gg2zfvr19v1cd6iYXMeHSSy/VxowZo61fv177+OOPtSFDhjQaEnDw4EEtPz9fW79+vaZpmrZnzx7tgQce0DZt2qTt27dPe/XVV7WBAwdq559/vl4/Qqd74YUXNKvVqhUWFmo7duzQ5s2bpyUlJYUfV//oRz/S7r333nD5Tz75RDOZTNqjjz6qff3119p9993Xa4cEtOe+LV26VHvrrbe0vXv3aps3b9auv/56zWazaV999VWbrylJqQc4fvy4NnPmTC0+Pl5LTEzUbr75Zq2mpib8/X379mmA9sEHH2iapmkHDhzQzj//fC0lJUWzWq3a4MGDtf/5n//p0eOUNE3TnnjiCS0nJ0ezWCzahAkTtHXr1oW/N2XKFG327NmNyv/zn//Uhg4dqlksFu3MM8/UXnvttS6OODa0577ddddd4bLp6enaZZddpm3ZsqVd15OlS4QQMUX6lIQQMUWSkhAipkhSEkLEFElKQoiYIklJCBFTJCkJIWKKJCUhREyRpCSEiCmSlES3UlhYSFJSUvj1/fffz+jRo8Ov58yZw4wZM7o8LhE9kpREs+bMmdPiut3z589HUZRGqxh0RjLIzc1l+fLljY798Ic/ZNeuXS2+Z8WKFRQWFoZfT506lbvuuiuqcYnOJUlJtCg7O5sXXngBt9sdPubxeHjuuefIycnRJSa73d5kUbuTOZ3ORjUp0f1IUhItOvvss8nOzm60GuXq1avJyclhzJgxHTp3czWYGTNmhGtfU6dOpaioiJ///OfhXTGgafPtVCfX2ObMmcOaNWtYsWJF+Bz79u1j8ODBPProo43et23bNhRFade6P6JzSFISrbrlllsoKCgIv3766ae5+eabO/26q1evpn///jzwwAMcPnyYw4cPt/scK1asYNKkSdx2223hc+Tk5DT5mQAKCgo4//zzGTx4cLR+BBEhSUqiVTfddBMff/wxRUVFFBUV8cknn3DTTTd1+nVTUlIwGo0kJCSQkZER0YqPTqcTi8VCXFxc+BxGo5E5c+awc+fO8P5lfr+f5557jltuuSXaP4aIgCyHK1qVlpbG5ZdfTmFhIZqmcfnll9OnTx+9w+qQrKwsLr/8cp5++mkmTJjAf/7zH7xeL9dee63eoQmkpiTa4JZbbqGwsJBnnnkmarUJg8HAqUt5+f3+qJy7LebOnRvuxC8oKOCHP/whcXFxXXZ90TJJSuK0Lr30Unw+H36/n0suuSQq50xLS2vUTxQMBvnyyy8blbFYLASDwQ5dp6VzXHbZZTgcDlauXMmbb74pTbcYIs03cVpGozG8nZPRaGyxXFVVVXgDggapqalkZ2c3KXvhhReycOFCXnvtNQYNGsTjjz9OZWVlozK5ubmsXbuW66+/HqvVGlGzMTc3l/Xr17N//37i4+NJSUnBYDCE+5YWL17MkCFDetWGALFOakqiTRITE0lMTGy1zIcffsiYMWMafS1durTZsrfccguzZ89m1qxZTJkyhYEDB3LBBRc0KvPAAw+wf/9+Bg0aRFpaWkRxL1q0CKPRyPDhw0lLS+PAgQPh79166634fL4ueZoo2k7W6Ba91kcffcRFF11EcXFxk11ghX4kKYlex+v1cvToUWbPnk1GRgbPPvus3iGJk0jzTfQ6zz//PAMGDKCyspLf/OY3eocjTiE1JSFETJGakhAipkhSEkLEFElKQoiYIklJCBFTJCkJIWKKJCUhREyRpCSEiCmSlIQQMeX/A+H8ncWB/9SzAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:12:09.835982Z", + "iopub.status.busy": "2024-02-29T15:12:09.835655Z", + "iopub.status.idle": "2024-02-29T15:12:10.175166Z", + "shell.execute_reply": "2024-02-29T15:12:10.174252Z" + }, + "papermill": { + "duration": 0.362666, + "end_time": "2024-02-29T15:12:10.177180", + "exception": false, + "start_time": "2024-02-29T15:12:09.814514", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAEmCAYAAAAk84bZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/5ElEQVR4nO3deXxU5b348c+ZfSaZmSyELBAI+yK7CEVcQKkLXitqq7Uu4NpesbfKpVV6W9xq0Vat3OpFawvor1JarUtbF+oKlQoiiKKyRpaQhS3rJLOec35/nGQgZCGZTDKT5Pt+vfKCmTlzzjch8+V5nvM830fRdV1HCCGShCnRAQghxIkkKQkhkookJSFEUpGkJIRIKpKUhBBJRZKSECKpSFISQiQVSUpCiKRiSXQAHaFpGiUlJbjdbhRFSXQ4QoiT6LpOTU0NeXl5mExtawN166RUUlJCfn5+osMQQpxCUVER/fv3b9Ox3Topud1uwPiGPR5PgqMRQpysurqa/Pz86Ge1Lbp1Umrosnk8HklKQiSx9gyvyEC3ECKpSFISQiQVSUpCiKTSrceU2kLXdSKRCKqqJjoU0Q1ZrVbMZnOiw+hVenRSCoVClJaWUldXl+hQRDelKAr9+/cnNTU10aH0Gj02KWmaxt69ezGbzeTl5WGz2WSCpWgXXdc5cuQIBw8eZNiwYdJi6iI9NimFQiE0TSM/Px+Xy5XocEQ3lZWVxb59+wiHw21KSl8UV/FVSTWTBqYztK+0rmLRY5NSg7ZObReiOe1pXR/zBXln+yF0HY74guSlFeCy9fiPWNzJJ1aIOFEUhdG5xiTeUERj9yFfgiPqniQpCREnGSk2Ljgth7OG9QFg37HaBEfUPUlS6iX27duHoihs3bq1S86zfv16xo4di9VqZc6cOR26ZnfTL80JQFlVANnBrP0SnpSKi4u57rrryMzMxOl0MnbsWD755JNEh9XjffDBByiKQmVlZaecf8GCBUyYMIG9e/eycuXKTrlGMqkOhDlUHUANB+l7+EPyqzZTFwxTE4wkOrRuJ6GjcBUVFUyfPp2ZM2fy5ptvkpWVxe7du0lPT09kWEknFAphs9kSHUa7FBYW8oMf/KDN5Sqa05Xft67rqKqKxRLbR2L3IR/rdh3hbG0Tk20HGBupIcecSjgyKM6R9nwJbSk98sgj5Ofns2LFCqZMmcKgQYO44IILGDJkSKdeNxTRWvyKqFqbjw234dhYzJgxgzvuuIM777yTPn36cOGFF/LFF19w8cUXk5qaSnZ2Ntdffz1Hjx6Nvuett97irLPOIi0tjczMTP7jP/6DwsLCZs+/b98+Zs6cCUB6ejqKojBv3rx2nWfHjh2ceeaZOBwOxowZw9q1a6PnVhSFY8eOcdNNN6EoSrSltHbtWqZMmYLdbic3N5d77rmHSOR4S6K577uhRbdmzRomTpyI0+nkvPPO4/Dhw7z55puMGjUKj8fD9773vUaTZDVNY8mSJQwaNAin08n48eN56aWXoq83nPfNN9/k9NNPx2638+GHH8b07wVQURvCrIXoW7sLgBHZbs5y7iczpXv9Z5IMEtpS+tvf/saFF17Id77zHdauXUu/fv24/fbbufXWW5s9PhgMEgwGo4+rq6tjuu5T7+9p8bVBfVKYM7Ff9PHv1hUSVpsfF+if7uQ7k48XmVu+fi/+UOPlLHd9c3hMMT733HP853/+J+vXr6eyspLzzjuPW265hd/85jf4/X7uvvturrrqKt577z0AamtrWbBgAePGjcPn87F48WIuv/xytm7d2mRaRH5+Pn/961+58sor2blzJx6PB6fT2a7z/PjHP+aJJ55g9OjRPP7441x66aXs3buX/Px8SktLGTFiBA888ABXX301Xq+X4uJiZs+ezbx583j++efZsWMHt956Kw6Hg/vuu6/Z7xugtLQUgPvuu48nn3wSl8vFVVddxVVXXYXdbmfVqlX4fD4uv/xyfvvb33L33XcDsGTJEv74xz/y9NNPM2zYMNatW8d1111HVlYW5557bvR699xzD48++iiDBw/uUAu9oi6EJ1CC06qA3Q0RPwRrwHcY3Nkxn7c3SmhS+vrrr1m2bBkLFizgpz/9KZs2beK//uu/sNlszJ07t8nxS5Ys4f77709ApF1v2LBh/OpXvwLgF7/4BRMnTuSXv/xl9PXly5eTn5/Prl27GD58OFdeeWWj9y9fvpysrCy++uorxowZ0+g1s9lMRkYGAH379iUtLS36WlvPc8cdd0SPXbZsGW+99RZ/+MMf+MlPfkJOTg6KouD1esnJyQHg//7v/8jPz+fJJ59EURRGjhxJSUkJd999N4sXL44mvBO/bzielH7xi18wffp0AG6++WYWLVpEYWEhgwcPBuDb3/4277//PnfffTfBYJBf/vKXvPPOO0ybNg2AwYMH8+GHH/LMM880SkoPPPAA3/zmN0/9D3IKFXUhsoLFOD1myBwKgUr08q8JlR/ALkmpXRKalDRNY/LkydEP28SJE/niiy94+umnm01KixYtYsGCBdHHDVXt2mv+zKEtvmY6aa7cbee03JU8eV7dTdPjN35w+umnR//+2Wef8f777ze7/qqwsJDhw4eze/duFi9ezMaNGzl69CiaZnQdDxw40CQptaat52n4sANYLBYmT57M9u3bWzzv9u3bmTZtWqPJiNOnT8fn83Hw4EEGDBjQ5Ps+0bhx46J/z87OxuVyRRNSw3Mff/wxAHv27KGurq5JsgmFQkycOLHRc5MnT24x5rYKRlRqgyoFoWM4rCbw9qcqYmbnvs2Ul2/jooFndPgavUlCk1Jubi6jR49u9NyoUaP461//2uzxdrsdu93e4evaLG0fSuusY08lJSUl+nefz8ell17KI4880uS43NxcAC699FIGDhzIs88+S15eHpqmMWbMGEKhULuuG6/zxOrE7/tEVqs1+ndFURo9bniuIYH6fMakxddff51+/fo1Ou7k35+WrtcelXVh0HW8WiUWUwqk9sWhKaiajqXuMGFVw2pO+I3ubiOhSWn69Ons3Lmz0XO7du1i4MCBCYooOU2aNIm//vWvFBQUNHt36NixY+zcuZNnn32Ws88+G+CUg7YNd7VOLOnSnvNs2LCBc845B4BIJMLmzZu54447Wrxew382uq5HW0vr16/H7XZ36A5dc0aPHo3dbufAgQONumqdpSYQxqbWkmKKgGICVyY2kwWzAq5wOT5/iPRUR6fH0VMkNH3fddddbNiwgV/+8pfs2bOHVatW8bvf/Y758+cnMqykM3/+fMrLy7nmmmvYtGkThYWFrFmzhhtvvBFVVUlPTyczM5Pf/e537Nmzh/fee69RN7c5AwcORFEU/vGPf3DkyBF8Pl+7zvPUU0/xyiuvsGPHDubPn09FRQU33XRTi9e7/fbbKSoq4oc//CE7duzgtdde495772XBggVxX5/odrtZuHAhd911F8899xyFhYVs2bKF3/72tzz33HNxvRZARoqds/M0+nrs4MoAkxnFkYbVakXRNWprKuJ+zZ4soUnpjDPO4JVXXuFPf/oTY8aM4cEHH+SJJ57g2muvTWRYSScvL4/169ejqioXXHABY8eO5c477yQtLQ2TyYTJZGL16tVs3ryZMWPGcNddd/HrX/+61XP269eP+++/n3vuuYfs7GzuuOOOdp3n4Ycf5uGHH2b8+PF8+OGH/O1vf6NPnz6tXu+NN97g448/Zvz48fzgBz/g5ptv5mc/+1mHfjYtefDBB/n5z3/OkiVLGDVqFBdddBGvv/46gwbFf95QRoqNcekR+rodkJJlPGkyoTjTAKirOhz3a/Zkit6N58FXV1fj9XqpqqpqsptJIBBg7969DBo0CIdDms4iNm3+Pdr5FpR8CgXTYZDRrf38zd9TW7ID99jZjJlyXhdFnFxa+4y2REbfhOigkko/1ZVH0HQdHN7o85ZUY9pF2FeeqNC6JUlKQnTQm1+UsXnHPmqDEXCkRZ93ezNJd1nJsAQSF1w3JBWohOgATdPx+cPY1RpsFk+jllJ+bg4c80CKbFrRHtJSEqID6sIqlogPk65htVjAfsK4ib1+q+pgTWKC66YkKQnRATWBMPZIDTaLCZPTCydOb7B70NEJ+6vRNWkttZV034TogNqgik2tNWZs2xvfXdIsLj7eV4WuqYz3VeHyZCQoyu5FWkpCdIA/1JCUFLA3XptoMptQLcYyloBPJlC2lSQlITqgNhTBptYZLSVbM1sqOes3Eqit6uLIui/pvgnRAQMzXfTpayGj1nZ8YPsEis14LlAXW+2v3khaSkloxowZ3HnnnYkOIyrZ4kkmuV4nQ72Q7rI121KyOo3nwnVyB66tJCn1UF1VakQAofr93exNk5LZYYwphf2yB1xbSVJKMvPmzWPt2rUsXboURVFQFIXCwkJuvvnmaL3pESNGsHTp0ibvmzNnDg899BB5eXmMGDECgH//+99MmDABh8PB5MmTefXVV5tskdRa/e/m4tm3b19X/TiS3sHyWnzVFcYSk+ZaSg6j+xYJSEuprXrXmJKugxru+uuarU3LVLZg6dKl7Nq1izFjxvDAAw8ARnH//v378+KLL5KZmcm///1vbrvtNnJzc7nqqqui73333XfxeDy8/fbbgLEY8tJLL2X27NmsWrWK/fv3N+mGnar+d3PxZGVlxeGH0jO8vvUA4w4cZXx/L65mklKa14stxYbLKlsttVXvSkpqGP71WNdf9+z/BkvbdrXwer3YbDZcLle0vjXQqDb5oEGD+Oijj/jLX/7SKCmlpKTw+9//PlrA7emnn0ZRFJ599lkcDgejR4+muLi40cYMTz755CnrfzcXjwBV09HqW0BWu7PZf+P8nL6Q7QZ7bDvb9Ea9Kyl1Y0899RTLly/nwIED+P1+QqEQEyZMaHTM2LFjG+2TtnPnTsaNG9eo5MaUKVMavact9b9F8+pCEWxqLQpgcTa98waA1WX8Ga4zWuptbDH3Zr0rKZmtRqslEdftgNWrV7Nw4UIee+wxpk2bhtvt5te//jUbN25sdFws9abbUv9bNM+YOGnMUVKamQ4AgC0VHR01EsYSCYDV2bVBdkO9KykpSpu7UYlks9ka1c5ev349Z555Jrfffnv0uZY2mjzRiBEj+OMf/0gwGIwWzN+0aVOjY05V/7u5eIShNpqUlOYnTgIBTWHLfh+KGmLK1FrMkpROSe6+JaGCggI2btzIvn37OHr0KMOGDeOTTz5hzZo17Nq1i5///OdNkktzvve976FpGrfddhvbt29nzZo1PProowDR4v2nqv/dXDwNu4b0dnWhCFa1DqvFBLbmW6l2i4mgyUhEAZmr1CaSlJLQwoULMZvNjB49mqysLC688EKuuOIKrr76aqZOncqxY8catZpa4vF4+Pvf/87WrVuZMGEC//M//8PixYsBouNMp6r/3Vw8Bw4c6LxvvhupC6lYtABWU8tJSVEUsBnjSiGZ1d0mUqO7l3nhhRe48cYbqaqqim7VLVrW2u9RWVWAuk9WkR4oIn3SHMgd3+w5Pvjr09gr9zB42uVkj57eBVEnj1hqdPeuMaVe6Pnnn2fw4MH069ePzz77LDoHSRJSx+V4HZBmgmrb8btszTDbjdciQZnV3RaSlHq4srIyFi9eTFlZGbm5uXznO9/hoYceSnRYPUfYb/zZygC2qT4phYP+roio20vomNJ9990XXbrQ8DVy5MhEhtTj/OQnP2Hfvn3RbshvfvMbXK6W/1cXbVdWFaDGV42q6a23lOpfiwRquyq0bi3hLaXTTjuNd955J/q4pdvSQiSbf247yLD9Rxid68HbSkspPc0LKTZSzQlY4tQNJTwDWCwWWb4guqVwsA4Ai8UMlpZvpozo3xcq3ODstveUulTCpwTs3r2bvLw8Bg8ezLXXXtvq7eZgMEh1dXWjr1PpxjcXRRJo6fdH13XUoNEds9hdrS8faWhFhWVMqS0SmpSmTp3KypUreeutt1i2bBl79+7l7LPPpqam+UlmS5Yswev1Rr/y8/NbPLfVaiztqKur65TYRe/QUJfKbDY3ej4Y0TCrxiaTVscplvdYXcZSk5D8LrZFQrtvF198cfTv48aNY+rUqQwcOJC//OUv3HzzzU2OX7RoEQsWLIg+rq6ubjExmc1m0tLSOHz4MAAulys6i1mIttA0jSNHjuByuZqMdfpDKlYtgNmkYLK1fuOg2KdR9HU5TpuZ8ZoKJnOrx/d2CR9TOlFaWhrDhw9nz549zb5ut9uja7jaomGsqiExCdFeJpOJAQMGNPkPLRBRsagBLCbllItsrQ4XOgoRVTO6cM1UqBTHJVVS8vl8FBYWcv3118flfIqikJubS9++fQmH5c6HaD+bzRZdbnMif/0SE4tZAWvr3TeHzUrEZENRg+jhOhRJSq1KaFJauHAhl156KQMHDqSkpIR7770Xs9nMNddcE9frmM3mJmMCQnRERoqNCTl20sodp2wpOa1mIiYHFi1IKFAnDaVTSGhSOnjwINdccw3Hjh0jKyuLs846iw0bNki5VZH00lw20tJNEHa0OnESwGo2oVkcEKki5K+l7QMQvVNCk9Lq1asTeXkhOqYNS0waKFYXBCAckPVvp5LweUpCdEfHfEGqa6qJaFrbkpLNOKZhwqVoWVINdAvRXXz09THchSUMT9PJPUX3DSAjzYsjYMOuBbsguu5NkpIQMQiEVNK1AFazs00tpfGDckF3g03KCp+KdN+EiEEwFMSkq/XzlNpQdaFhbZwsNTklSUpCxECtL0NisVrbtluNxYGOjhYOdHJk3Z8kJSFi0FAbyWxPadNebp+VBdn4dTm7i490dmjdniQlIdoprGoQMbphllMtxq2nWO3oIC2lNpCkJEQ7+cPGYlyF+rIlbWC1G8lLlzGlU5K7b0K0k81sYnKeA7fmNCZFtuU9DuMOnRYJgVQKaJUkJSHayWE1M6qPBWpdbbvzBtgcxnGqpkMkGN0LTjQl3TchYtHQDWtjcrFbLagmKxFVh4iMK7VGkpIQ7VQdCFNZXUVIbdsSEwC7xUxEsRvLUiQptUq6b0K001cl1VTtPMhQWx1D2th9c9hMeNyppIRDaCG/tAZaIUlJiHYKhFWsmh+r2dSultKkwblQGQZV1r+1RhK2EO0UCKtY1GDbl5g0sNRXUpLuW6skKQnRToFGpXDb1lICwOpER5e5SqcgSUmIdgoH61DQsZhMYGl7UvrX3ho2fl1OWXlVJ0bX/UlSEqKdouvebHYwt31YVjUZS03UkLSUWiNJSYh2itRvKmm2t23dWwNzffXJSFCSUmvk7psQ7TQx147H58TubG9SMmoqqTKm1CpJSkK005gsKxxzgaN9eyVZ7CmogCYtpVZJ902I9mrHLiYnstobWkoyJaA1kpSEaIdAWKWiqopgRG13UmoocyJTAlqXNEnp4YcfRlEU7rzzzkSHIkSLyqoCrPvyADvLato3cRJwp7rxOq2kmlXQ9U6KsPtLijGlTZs28cwzzzBu3LhEhyJEqwIRo8CbpR1LTBoUZGdArsd4EAmC1dEJEXZ/CW8p+Xw+rr32Wp599lnS09MTHY4QrQqENWM2t0kBa/vuvmG2HJ/XFJEuXEsSnpTmz5/PJZdcwqxZs055bDAYpLq6utGXEF0pEFaxqg1JqX0tJSC6qwkRWZTbkoR231avXs2WLVvYtGlTm45fsmQJ999/fydHJUTLAuGGdW+mdo8p1YUibN5bgy1YzpTxfk69B0rvlLCWUlFRET/60Y944YUXcDja1rdetGgRVVVV0a+ioqJOjlKIxgKhMBYtGFNLyWY24ddtaLqxfk40L2Etpc2bN3P48GEmTZoUfU5VVdatW8eTTz5JMBjEbG5cXN1ut2O327s6VCGiwg3r3sym47vetpHFbEI3G7+/wYAfW9yj6xkSlpTOP/98tm3b1ui5G2+8kZEjR3L33Xc3SUhCJIORfSzY05ykpKSCqf0dDcVqJKWwzOpuUcKSktvtZsyYMY2eS0lJITMzs8nzQiSLYWlmyHCByxPT+03141AR6b61KOF334ToVmJcYtLAVL8oVyoFtCwpJk82+OCDDxIdghAt0nWd8spKUsIqdqszprtn5votmdSQtJRaElNL6euvv453HEIkvWBE493P9/FpUSWaObbZ2F53Kl6nFRvhOEfXc8SUlIYOHcrMmTP54x//SCAgK55F79AwR8mstL/AW4MJg3IYneshO7beX68QU1LasmUL48aNY8GCBeTk5PD973+fjz/+ON6xCZFUAmHthHVvMW673bDeTXY0aVFMSWnChAksXbqUkpISli9fTmlpKWeddRZjxozh8ccf58iRI/GOU4iEM7ZW6sASEzg+t0mSUos6dPfNYrFwxRVX8OKLL/LII4+wZ88eFi5cSH5+PjfccAOlpaXxilOIhGtcISC2ltLuY2E+2V/OjoNHpHxJCzqUlD755BNuv/12cnNzefzxx1m4cCGFhYW8/fbblJSUcNlll8UrTiESrnGFgBhbSlYHYVVHjURAlcHu5sQ0JeDxxx9nxYoV7Ny5k9mzZ/P8888ze/ZsTPUzXAcNGsTKlSspKCiIZ6xCJNTxCgFmsMXWUrLbHOgoRDTd6MJZZLHJyWJKSsuWLeOmm25i3rx55ObmNntM3759+cMf/tCh4IRIJv3cFnSvlVS7Jebum91mRjXZUbWQlC9pQUxJ6e2332bAgAHRllEDXdcpKipiwIAB2Gw25s6dG5cghUgG+am6scTEZAZzbC0cu8VExGQnogWl0FsLYhpTGjJkCEePHm3yfHl5OYMGDepwUEIkpegSExcosVVDslvMREw2VE1HDckduObElJT0Fu4a+Hy+NtdGEqK7qaiqJBBWUdtZsuREdosJ1WRUCgjJotxmtav7tmDBAgAURWHx4sW4XMf71aqqsnHjRiZMmBDXAIVIFuu+3E/f4kpGpPYnI8ZzmEwKHrcbr2KVrZZa0K6k9OmnnwJGS2nbtm3YbMf71TabjfHjx7Nw4cL4RihEktDqF9Fa2rkz7snOHNEPSg6DEolHWD1Ou5LS+++/DxjF2JYuXYrHE1tNGSG6G13Xo0nJ2sGkhKW+eqrcfWtWTHffVqxYEe84hEhqwYiGpf5umdUR22LcqIaJl3L3rVltTkpXXHEFK1euxOPxcMUVV7R67Msvv9zhwIRIJkaFgCBmk4IlxgoBDT4uqsW0v5xsUwX5o+IUYA/S5qTk9XpR6m+Der3eTgtIiGRkVAjw1y8xibFCQL2QYsOs6kRC0lJqTpuT0oldNum+id6mw5tQnsBid6IDqiSlZsU0T8nv91NXd3yOxf79+3niiSf45z//GbfAhEgmHqeVQWkm+rjtYOtY981Sv25OkykBzYopKV122WU8//zzAFRWVjJlyhQee+wxLrvsMpYtWxbXAIVIBhlOMwUeE3leZ4dbSlaH8X49LDO6mxNz5cmzzz4bgJdeeomcnBz279/P888/z//+7//GNUAhkkJDq0ZRwNKxpGSz17eUIiHQ1I5G1uPElJTq6upwu90A/POf/+SKK67AZDLxjW98g/3798c1QCGSQXVNtbHExGyPaRPKE9nqW0oRVZcKlM2IeeOAV199laKiItasWcMFF1wAwOHDh2VCpeiRPtldzKdFlZTUxrYQ90QpdhtOpxOn1SwTKJsRU1JavHgxCxcupKCggKlTpzJt2jTAaDVNnDixzedZtmwZ48aNw+Px4PF4mDZtGm+++WYsIQnRqSLBWgAsHZ04CeR4HUwYlMvQvqnHu4UiKqYZ3d/+9rc566yzKC0tZfz48dHnzz//fC6//PI2n6d///48/PDDDBs2DF3Xee6557jsssv49NNPOe2002IJTYhOEU1KHZw4GRVdaiLdt5PFvENuTk4OOTk5jZ6bMmVKu85x6aWXNnr80EMPsWzZMjZs2CBJSSQVLWAkpQ4vMWkQ3dVEum8niykp1dbW8vDDD/Puu+9y+PBhNE1r9HosO+iqqsqLL75IbW1ttDt4smAwSDB4/B+xurq63dcRor10XUcN1SclZwcX49afb93eGlKqyxld4KNj88N7npiS0i233MLatWu5/vrryc3NjS4/icW2bduYNm0agUCA1NRUXnnlFUaPHt3ssUuWLOH++++P+VpCxCIY0TDXL561xyEpKYqCX7NiU3XCQRlTOllMSenNN9/k9ddfZ/r06R0OYMSIEWzdupWqqipeeukl5s6dy9q1a5tNTIsWLYoWmgOjpZSfn9/hGIRoTSBs7PcWj8W4DZT6nXIlKTUVU1JKT08nIyPW2nuN2Ww2hg4dCsDpp5/Opk2bWLp0Kc8880yTY+12O3a7PS7XFaKtrGYTw9IULAFHh5eYNDDZ6ucqSUncJmKaEvDggw+yePHiRuvf4kXTtEbjRkIkWordwmAvDMhwga3j3TcAc31LSSoFNBVTS+mxxx6jsLCQ7OxsCgoKsFqtjV7fsmVLm86zaNEiLr74YgYMGEBNTQ2rVq3igw8+YM2aNbGEJUTn0FRoWKcWp5aS2W60lGRHk6ZiSkpz5syJy8UPHz7MDTfcQGlpKV6vl3HjxrFmzRq++c1vxuX8QsSDz1eFJaxisViwdHAxboNopYCQdN9OFlNSuvfee+NycdlBV3QHX+0tRS2qJCszk6EduNN8otTUFBS7GZsejsv5epKYVxZWVlby+9//nkWLFlFeXg4Y3bbi4uK4BSdEMggHagAwd3TDgBOML8hhXL80+rnjk+R6kphaSp9//jmzZs3C6/Wyb98+br31VjIyMnj55Zc5cOBAtNaSED1BOODDDpjjtcQEGs/o1vWYd9ztiWJqKS1YsIB58+axe/fuRjvizp49m3Xr1sUtOCGSgVrfUrI43fE7aUNS0nVZanKSmFpKmzZtanYeUb9+/SgrK+twUEIkEy3gA8Aax6RUUhOm7GANDrPO6EgArLLdfYOYkpLdbm923dmuXbvIysrqcFBCJBOtfoKjLY5JSVHAp5pBC0qlgJPE1H371re+xQMPPEA4bNw5UBSFAwcOcPfdd3PllVfGNUAhEknTdJSw0VKyueKXlOwWMxGTHVWT6pMniykpPfbYY/h8PrKysvD7/Zx77rkMHToUt9vNQw89FO8YhUgYTdcZmqaQ7bZjj2NLyW4xRZOSbCDQWEzdN6/Xy9tvv8369ev57LPP8Pl8TJo0iVmzZsU7PiESymI2MdirgCsVHPFNSqpiQwdCgTpkRedx7U5KmqaxcuVKXn75Zfbt24eiKAwaNIicnBx0Xe9QGRMhko4aOb7EpIM7457IYjahWx0QMCoFSFI6rl3dN13X+da3vsUtt9xCcXExY8eO5bTTTmP//v3MmzevXaVwhegO6mqr8YdVIpg6vN/byUyWhvIlstTkRO1qKa1cuZJ169bx7rvvMnPmzEavvffee8yZM4fnn3+eG264Ia5BCpEoew6WUltUSXp6BiPj3AtISUkhpc6MIgPdjbSrpfSnP/2Jn/70p00SEsB5553HPffcwwsvvBC34IRItGCtMfVFccR/67CzRw9gXL800mzaqQ/uRdqVlD7//HMuuuiiFl+/+OKL+eyzzzoclBDJIlJnJCWLsxP2M4zuaCIzuk/UrqRUXl5OdnZ2i69nZ2dTUVHR4aCESBaRQGcmpfpZ3LL3WyPtGlNSVaOmTEvMZjORSKTDQQmRLDS/kZRsrvgnpW2HQ0SKKkjLtDMo7mfvvtqVlHRdZ968eS3WyZYytqKn0YLGYlxbijfu5w5iIxzWiNTvKScM7UpKc+fOPeUxcudN9BSapkPQWGLiSE2L+/ntzlTCgBYOgKaBKebyZj1Ku5LSihUrOisOIZKOpusM9mioITvOlLS4n9/uTMEHRFTVWP9mk20poQPbdgvR01n0MAO9VsAa1yUmDRw2G6rJRkRTjcFuSUpAB8rhCtHj1Y8nYbGDxRb30ztsxqLciKpBWGZ1N5CkJEQL/L5K/GGVsDWOZXBP4LCaCZschFUdXaYFRElSEqIFe4vL2FpUyVdH9U45v9NqxupIwWU3y065J5AxJSFaEK6fo2TujImTGNuBzxgzEMpqQJP1bw0S2lJasmQJZ5xxBm63m759+zJnzhx27tyZyJCEiArXVgGdM3EyqqHygIwpRSU0Ka1du5b58+ezYcMG3n77bcLhMBdccAG1tTKZTCRexG8kJWsnzFGKsjQkJRlTapDQ7ttbb73V6PHKlSvp27cvmzdv5pxzzklQVEIYdH8lAI7UjE67xiclAcxFFeRYK8jttKt0L0k1plRVZfzPlJHR/C9BMBhstJSluR1VhIgHXdMgYPw+ujydl5RqNRu2sEZIlppEJc3dN03TuPPOO5k+fTpjxoxp9pglS5bg9XqjX/n5+V0cpegtgoFaUI3dejozKVkcxoRJNShJqUHSJKX58+fzxRdfsHr16haPWbRoEVVVVdGvoqKiLoxQ9CqBavqnO8nKzMBqjf/EyQa2+q3A9ZAMdDdIiu7bHXfcwT/+8Q/WrVtH//79WzzObre3WKFAiHhyqD7y013gyevU61icKUQANeQ3tvCWjTcS21LSdZ077riDV155hffee49Bg6SqjEgS9eNJOOJfsuREdkcqAKqqyaaU9RLaUpo/fz6rVq3itddew+12U1ZWBhj7yjmd8d05Qoj2qK06himsYrF6sHbidRx2K+UmK2FNM6YFxHnHlO4ooS2lZcuWUVVVxYwZM8jNzY1+/fnPf05kWEKw92AJW4sq2d7J1Z1T7BbMNhc2s0kmUNZLaEtJ1ztnTZEQHaXWz1Gyd+IcJYBsj4PsEf2hpkwmUNZLmrtvQiQTra4SALs7vfMv1rDzriQlQJKSEE2F/URCxqBzqrdzW0qArH87iSQlIU4SrDlCRNMJmVNwp3R+NciNxQE+PVDBsUrZngwkKQnRRG3FEQBUexp2i7nTr1ej2glENEK1smwKJCkJ0YS/+igAppQu6LoBZocxqzss69+AJJnRLUQy8eg15Kc7qcvr1yXXs9YXkVMDNV1yvWQnSUmIk7i1atzpLigY2CXXsztTjaUmsigXkO6bEI3pOtSVG393dk33zZZibN+kBWuNTSl7OUlKQpwoVEtNbR11YQ3V3rnr3ho4nEZSCkdUiMhcJUlKQpzIX87OQzV8XBqh3K92ySVTnDbMdhdWswlC0oWTMSUhThCqOUpY1fE7vLgdXfPx6J/uov/wfKg9CiEf0LdLrpuspKUkxAnqKg8DoDkycFg7f45SlK1+w0sp9iZJSYgT+SuN8jlWb1bXXjialKT7Jt03IU4QqjoEgN2b06XX/ehgAHtpBYP6VNIFS4CTmiQlIRqE/YTrl3q40rs2KflUO0Q0ArLURLpvQkTVHiUQVgmaU0lzp3bppa2u+mkBdZKUpKUkRIPaw+SlObA7BtDX4+jSS9tSjKUmEb8kJUlKQjSoPUpGip2MAYPA2ZmVuZtypKQTRJISSPdNiONqjZIlpHTxnTfAkVq/KDcUgEjwFEf3bJKUhADQdXzlpVTUhaizdv39r1RXKqrJRkjVINi7qwVIUhICIFDJ0Yoqth+qZdORLpw0WS/VYcHk8GCzmCDYu7twMqYkBEBNGXUhlVprJnmert97LSPFxpmjB0H519JSSuTF161bx6WXXkpeXh6KovDqq68mMhzRi+nVpdSFItTasshKTdDW8Pb6aQiSlBKntraW8ePH89RTTyUyDCEIVhQTVnVq7VlkpNgSE4TdXR+MLzHXTxIJ7b5dfPHFXHzxxYkMQQjQdfzHigGwpuViMSfm/+pNZRHMByrIsRwib0RCQkgK3WpMKRgMEgwev11aXd27BwRFnPgr8Pvr0BQzqRldu7zkRAHFhSWiEfRVJSyGZNCt7r4tWbIEr9cb/crPz090SKInqC6hNhShzppJZmrXD3I3cKamARCpk6TUbSxatIiqqqroV1FRUaJDEj1BdTH9010MGz6SIX27ds3biVK8mQCEg3UQDiQsjkTrVt03u92O3Z6gOyOi56o8gNNqpmDwCEjUnTfAnZrCEZOTYDgIgUqwJq4rmUjdqqUkRNyF/UYZWgBv/4SG4nVaCVrcBCMaWl3v3cI7oS0ln8/Hnj17oo/37t3L1q1bycjIYMCAAQmMTPQaVcUcqw1SZ/GSGraSnqDZAACpdgshmxc9dJi66mOkZiculkRKaFL65JNPmDlzZvTxggULAJg7dy4rV65MUFSiV6kq4lB1kN2mVArK60hP1BwlQFEUXN4+eCJ70Rr2nuuFEpqUZsyYga7riQxB9HJqxQFqAmFq0nPIz3AlOhxmjh8GO3aB3ntrdcuYkui9wgF8R4rQdFA9A0h3dW0NpWY504w/A5WJjCKhJCmJ3qtiH9V1QfzWNLKz+6IoSqIjAqdRNkXzV4HWNZthJptuNSVAiLiq2EulP0yVYzDD0hPfdQM4FLBQdLAGKyrjA1Xgykh0SF1OWkqid9J1gof34AtGqHTkU9AnJdERAeC0W6jS3fhDKqrvaKLDSQhJSqJ3qjtGXXU5mMyk9B1Eqj05Og1uu4WQIxMdqK0oTXQ4CSFJSfROR3eR7rJxxoQJfHNs8qyhVBQFs9uoEV5bXpbgaBJDkpLonQ5vB8CSPSqhc5Oa40zrC0Cg6nCCI0kMSUqi96krJ1JdBooJ+gxPdDRNpGTkARCqOgyaluBoul5ydKRF76apUHcMAtUQrgMtYjynq6CYwWQGWyqk9DFumXfw1r126Cu+KK6iNiWfESEzfZKroURmZhYHFQt1wSD4y43vuxeRpCQSQ9Pg2B4o+RQqDxiJqC0cXsg+DfqfAbYYbuPrOsf2bMIf1jhoG8xkR/J9BLK9Tko9OXj1I+jVJSiSlIToZDWHYNebUG3cXdLRUU12LCnpYE0BswVMFqOVpKugho1th2qPQaAK9v8bij+BobMgZ1y7Wk56+V5KD5WhmmwMGDEJu6Xrt1M6FYfVzDcmjIGiTVBTBrnjEh1Sl5KkJLpW2Rew8w1q/AEO18HXthHstwzCktqHeZMHRw97ectBKuvC2K0mHBYzdpsJh0Mj3b+f7MpP6Uc1yo43oGIfjLjESGRtUPTlemqDKpXeUZxX0PU74baZ2xhXoro4sXEkgCSlbkrVdA5W1HG4Joim6WSm2inIdBlF73UdqkuM8QiTBdw50eULCVW8Gd+2N9h3rJYDSh5fp59N2GJMWswwNb7nUu0PU+UPg//kk2Thtl/ELQVH4eu1cOgrqiqP4Zx4FTZn61Ujq4+UcKjwc0Ahf+xZuGxJ/OvvyUXTdfwVpaSokTYn3Z6g93ynPcjuQzWs232Uan+40fNOq4nz0o8wrG4LSuCkTRXSC2DIeeBOUJGeY4Uc/Pg1DpbXUeweR3HGNxie42FIVgpZbjspJ01evGxCP/xhlWBEIxhRCYQ1gmEVf1jFZjbBgKHgzkHb9jJ7du2g9uulaGOuYuSgfLI99ibr2OpCEbb+629YNR2tzwjGDBvSld99ux2LONle5MeuBTi9ugRTeu+pLyZJqZvRdZ09h31U+8M4bWYGZrgwmRSKj1aSXfxPqvbuJ9Tfi93hNLoAaghqSo1uzuaVRmLqP7nDd7DapeYQfPkKTquJQykjcI2cxU0j+rY6izo9xcYp23bpBfhGX4N68Fms/grCW5/nteKLScnsx6hcD3lpDtJdNhxWM/bKr8kK7KfWYmbkWZdgNiXB4ttWpKfY8TnzMNcUUlW8i3RJSiJZKYrCrNHZZHsdjMnzGnvPh2rRav/BIddhnOle7MNmQv4UMNeX4vBXQuG7cGQX7HnHuP0+/MKuSUyBatj2IqhhMvuP4MxBl5Hljd/iV0+fXM644kdUbFhF+aEiLEf+wQ71fNbVGLO0vzk6mzHpKuZdbzA8240/dwqpmXlxu35nMZkUHH2HQk0hVaW7SB8zK9EhdRmZPNkNRFSNbQerogXxrGYTkwakRxMSW1dhqj1CblYf0s68EQqmg9nK4eoA/pBq1Og57QrjbpWiGLfhd77R6RPz/H4/n7/1e4K1lcZcm9Muj2tCaqA4vGRMv5Ghw09jSn4KF2trOTP0b/ppxXjLP4ctz0PYj9mbS+qIc+N+/c7i7WdM7Kw9UmTUEu8lJCklOVXTeX1bKe9sP8S63SetGq9PSNQeNfahn3gdePsBcKg6wIubD/LypwcJhFUjGeWfAaMuNf5e+rmRmDqp8mdtIMQnb6yg9lgJO8pV9DHfBqujU64FGOcedzWW/qeT63Uy1VXKVbYN5B/90PhAu7Nh7HeOtx67gcH98/Bb06jxh/GX7kh0OF1GklIS0zSdNV+W8fWRWiwmhUGZJ5TXCPvhsz/VJyQ3TLi2Ue0di0nBYlI4XB3klU+LCUbqC4ZlnwajLzOWWJRt65TE5AtG+GjNX7BUfo3VYmXAuXNRXF1w989sgREXwelzjbk9qVnGDiVDzoOJ1xuJuxvxOq1oWaPQgcN7tiQ6nC4jY0pJStd13t1xmJ1lNZhNCv8xPo8BmfVdn3AAPlsNviNgS4EJ32tSDCwz1c4Vk/rz0uaDlFUFePXTYi6f2N/o8vUdZSSi7X8zWkyKCYZfFJcxpupAmH+9+w8yjm7FbjExdOb38OQUdPi87eLJM756gD5DTsdf8hHVpXuM8TmHJ9EhdTppKSUhXddZu+sIXxRXoShw8ZgcBjUUIYsEjYHjmjKwOmH8NS1WJ8xy27lyUj/sVhMllQFe21pMWK0fR8oefbwrV7IVdv+zwy2mqrow73zwPhll63FYTAw/8zI8A8Z26Jy93YiCfPr0H8qI7FQ4uCnR4XQJSUpJ6P2dh/n0QCVg3D0alu02Xgj6YOsLUHUQLHYY/12ji9KKvh4HV9S3kA5W+Pn7ZyVEoonpNBh5iZGYircYiakDg9+bP1lPTvHbOK0mhp9xHqlDz4z5XMLgtJkZNvmbxtyski3GOGIPJ0kpCeWnuzCbFL45OpvT8rzGkzVl8On/M+b82FxGC8ndtm2dc7wO5kzsh81iwmxSGs/RyRkLI2YfT0zbXmz/nR5dh4OfMD30bzJcFkZMmEbKqC6actAbZAwGdw66GqbuqzWJjqbTJUVSeuqppygoKMDhcDB16lQ+/vjjRIfU5aoDx2dnD8t2M296AWP6eY0SHgc2wpb/Z8w3cniNQVtPbrvO3y/NyeUT+3HhaTnR2c6hiGZMM8gdB6O+ZQwUl38Nm35vzGk6RXeuojbE518Xw1evwu63sZsVRk46F+fYOZKQ4klR8A86n+1lPr749CPKd29IdESdKuED3X/+859ZsGABTz/9NFOnTuWJJ57gwgsvZOfOnfTt2zfR4XUqXdc5WOFny4EKDlb4uWHaQNwO45a1x4ox1lP0sTHZEaDPMKO7ZXXGdL28tMbv++dXZfgCESYXpDMoaxRmVwZ8+Sr4K+CLvxqJL2+Scd36a+q6zqGqADsKC6n4+lOyar6iPNtBRooDBs8wJm1KQoo7W3o+hzIm4zj4bwrXv0ygrpa8sTPBlBTtirhS9ARvUTt16lTOOOMMnnzySQA0TSM/P58f/vCH3HPPPa2+t7q6Gq/XS1VVFR5P8t+VCKsaR2qClNeGKKsKsO9YLTX+MGY9hE0LMmuQlaGuAFQVGctC1PrWk81lfODbWaajNVV1YZ7/aB8Rzfjnt1tNDMhwkZ1iJqdyC5nlW3HV/5cVUjXKgnaqVSuVtQEIVGHWQgCku6z0HziU1DEX95g7XskqEIqw4Z9/xn7oUwAc7gzcBZPw5g0hs08uJntK0v2HEMtnNKFJKRQK4XK5eOmll5gzZ070+blz51JZWclrr73W6PhgMEgwGIw+rq6uJj8/v03fcHGln/e2H+LEbza/7B2cwWPo6BRkusj1Gq2Ban+IL0uqT+q+1A8A6zAgw0W/NGMioC8YYdvBiuhRx99i/CU/3UF+ugvQ8QUibDlQiRKNQsdGmL6pVnK8TpzWk2r7OLzQ73TIHd8pEw9rgxG2FlXyRXEVdaHGGx+enmfjnJSDcHg7dRVlfHawKvqaSYF0dwpZ+SNIHzoFMock3Yehp4pEVLZu/pDAjncwR46P/Z1RkI7FbAGLjd3HQhyrDaMoSn1XXUGv/xMUTi9Ix1I/rlh4pJajNcFmrwUwcUCaMY0E2HesjkPVAVJtFk7r54GxV51yikIsSSmh3bejR4+iqirZ2Y1XrmdnZ7NjR9MZrEuWLOH++++P6VrhiMZRX6jRc7m+cvTQEeOBwwWW+u5NIAx1J62yP1EgBIH6Y4MRCNZEX2ry0QyqEKpfHqKppJqCOK1mXDYzXqcNtzMFs6IY4zmONGM5Rmq2MbiZmt2pH/YUu4XpQ/swbXAmZdUBiiv9HK0JUhOIkOZ1Q/8CKDgLW6AW/2c7SLNp5HhcZPXJwOrua5SpFV3KYjEzeeq51IyZzIEdW/CV7ECrLsNsMoGuQTiA7q9B94dosbXhCx/v9vl8aL6WkxK1YTAbx+q+WrSaAJrdAr6gcb1OkPAxpfZYtGgRCxYsiD5uaCm1RbbHwZWT+jf6jJt9V4IaQlEU3A4L2I3xHLuqMbgujKKAXp9mFBRQFBQFow6PzQKKgl3VGOYPoygNfXslmpkUxagiiK3+vChMarh4QyBmmzFek8DlDyaTQl6as8mYUwOLI4Xzpp7exVGJ1rhTUjjt9LPh9LONJ9QIRPwQCZHt8+ENR9B1HU3Tjjff65OIyeOI/o5m1IVwhltOLia33WgaA+n+MLagis2sgNtuTNztBAlNSn369MFsNnPo0KFGzx86dIicnKa3u+12O3a7PaZrOW3m4zOiG2Q0X1PHDrS16pAV6JME9dNEL2e2gNkNdvCmZOJt49vSMzh1iZh6afVfnS2hQ/c2m43TTz+dd999N/qcpmm8++67TJs2LYGRCSESJeHdtwULFjB37lwmT57MlClTeOKJJ6itreXGG29MdGhCiARIeFK6+uqrOXLkCIsXL6asrIwJEybw1ltvNRn8FkL0Dgmfp9QR3W2ekhC9TSyf0Z43HVQI0a1JUhJCJBVJSkKIpJLwge6OaBgOq65uZfa1ECJhGj6b7Rm67tZJqabGWN7R1lndQojEqKmpwett25TObn33TdM0SkpKcLvdTXZE7a0alt4UFRXJHcl2kJ9bbE71c9N1nZqaGvLy8jC1scxKt24pmUwm+vfvn+gwkpLH45EPVwzk5xab1n5ubW0hNZCBbiFEUpGkJIRIKpKUehi73c69994bczWF3kp+brHpjJ9btx7oFkL0PNJSEkIkFUlKQoikIklJCJFUJCkJIZKKJKUeoLy8nGuvvRaPx0NaWho333wzPp+v1ffMmDEjugVPw9cPfvCDLoo4Mdq7E/OLL77IyJEjcTgcjB07ljfeeKOLIk0u7fm5rVy5ssnvlcPRzu3BdNHtXXTRRfr48eP1DRs26P/617/0oUOH6tdcc02r7zn33HP1W2+9VS8tLY1+VVVVdVHEXW/16tW6zWbTly9frn/55Zf6rbfeqqelpemHDh1q9vj169frZrNZ/9WvfqV/9dVX+s9+9jPdarXq27Zt6+LIE6u9P7cVK1boHo+n0e9VWVlZu64pSamb++qrr3RA37RpU/S5N998U1cURS8uLm7xfeeee67+ox/9qAsiTA5TpkzR58+fH32sqqqel5enL1mypNnjr7rqKv2SSy5p9NzUqVP173//+50aZ7Jp789txYoVutfr7dA1pfvWzX300UekpaUxefLk6HOzZs3CZDKxcePGVt/7wgsv0KdPH8aMGcOiRYuoq6vr7HATIhQKsXnzZmbNmhV9zmQyMWvWLD766KNm3/PRRx81Oh7gwgsvbPH4niiWnxuAz+dj4MCB5Ofnc9lll/Hll1+267rdekGugLKyMvr27dvoOYvFQkZGBmVlZS2+73vf+x4DBw4kLy+Pzz//nLvvvpudO3fy8ssvd3bIXa69OzGD8XNt7vjWfqY9TSw/txEjRrB8+XLGjRtHVVUVjz76KGeeeSZffvllmxfPS1JKUvfccw+PPPJIq8ds37495vPfdttt0b+PHTuW3Nxczj//fAoLCxkypPlNOoU4lWnTpjXas/HMM89k1KhRPPPMMzz44INtOockpST13//938ybN6/VYwYPHkxOTg6HDx9u9HwkEqG8vLzZXYZbMnXqVAD27NnT45JSe3diBsjJyWnX8T1RLD+3k1mtViZOnMiePXvafF0ZU0pSWVlZjBw5stUvm83GtGnTqKysZPPmzdH3vvfee2iaFk00bbF161YAcnNz4/2tJFwsOzFPmzat0fEAb7/9dq/auTkeO1irqsq2bdva93vVoWFykRQuuugifeLEifrGjRv1Dz/8UB82bFijKQEHDx7UR4wYoW/cuFHXdV3fs2eP/sADD+iffPKJvnfvXv21117TBw8erJ9zzjmJ+hY63erVq3W73a6vXLlS/+qrr/TbbrtNT0tLi96uvv766/V77rknevz69et1i8WiP/roo/r27dv1e++9t9dOCWjPz+3+++/X16xZoxcWFuqbN2/Wv/vd7+oOh0P/8ssv23xNSUo9wLFjx/RrrrlGT01N1T0ej37jjTfqNTU10df37t2rA/r777+v67quHzhwQD/nnHP0jIwM3W6360OHDtV//OMf9+h5Srqu67/97W/1AQMG6DabTZ8yZYq+YcOG6GvnnnuuPnfu3EbH/+Uvf9GHDx+u22w2/bTTTtNff/31Lo44ObTn53bnnXdGj83OztZnz56tb9mypV3Xk9IlQoikImNKQoikIklJCJFUJCkJIZKKJCUhRFKRpCSESCqSlIQQSUWSkhAiqUhSEt3KypUrSUtLiz6+7777mDBhQvTxvHnzmDNnTpfHJeJHkpJo1rx581oskTt//nwURWm0YLgzkkFBQQFPPPFEo+euvvpqdu3a1eJ7li5dysqVK6OPZ8yYwZ133hnXuETnkqQkWpSfn8/q1avx+/3R5wKBAKtWrWLAgAEJicnpdDapH3Uir9fbqCUluh9JSqJFkyZNIj8/v1Hht5dffpkBAwYwceLEDp27uRbMnDlzoq2vGTNmsH//fu66665oAXpo2n072Ykttnnz5rF27VqWLl0aPcfevXsZOnQojz76aKP3bd26FUVR2lViQ3QOSUqiVTfddBMrVqyIPl6+fDk33nhjp1/35Zdfpn///jzwwAOUlpZSWlra7nMsXbqUadOmceutt0bPMWDAgCbfE8CKFSs455xzGDp0aLy+BREjSUqiVddddx0ffvgh+/fvZ//+/axfv57rrruu06+bkZGB2WzG7XaTk5MTU3E1r9eLzWbD5XJFz2E2m5k3bx47d+6MbhUUDodZtWoVN910U7y/DREDqTwpWpWVlcUll1zCypUr0XWdSy65hD59+iQ6rA7Jy8vjkksuYfny5UyZMoW///3vBINBvvOd7yQ6NIG0lEQb3HTTTaxcuZLnnnsubq0Jk8nEyVVzwuFwXM7dFrfcckt0EH/FihVcffXVuFyuLru+aJkkJXFKF110EaFQiHA4zIUXXhiXc2ZlZTUaJ1JVlS+++KLRMTabDVVVO3Sdls4xe/ZsUlJSWLZsGW+99ZZ03ZKIdN/EKZnN5ujOKWazucXjqqqqorW+G2RmZpKfn9/k2PPOO48FCxbw+uuvM2TIEB5//HEqKysbHVNQUMC6dev47ne/i91uj6nbWFBQwMaNG9m3bx+pqalkZGRgMpmiY0uLFi1i2LBhvar2drKTlpJoE4/Hg8fjafWYDz74gIkTJzb6uv/++5s99qabbmLu3LnccMMNnHvuuQwePJiZM2c2OuaBBx5g3759DBkyhKysrJjiXrhwIWazmdGjR5OVlcWBAweir918882EQqEuuZso2k7K4Ype61//+hfnn38+RUVFTTZcFIkjSUn0OsFgkCNHjjB37lxycnJ44YUXEh2SOIF030Sv86c//YmBAwdSWVnJr371q0SHI04iLSUhRFKRlpIQIqlIUhJCJBVJSkKIpCJJSQiRVCQpCSGSiiQlIURSkaQkhEgqkpSEEElFkpIQIqn8fzTRHhC4N0ZUAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:12:10.219824Z", + "iopub.status.busy": "2024-02-29T15:12:10.219527Z", + "iopub.status.idle": "2024-02-29T15:12:10.451636Z", + "shell.execute_reply": "2024-02-29T15:12:10.450872Z" + }, + "papermill": { + "duration": 0.256131, + "end_time": "2024-02-29T15:12:10.453717", + "exception": false, + "start_time": "2024-02-29T15:12:10.197586", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEmCAYAAAAdlDeCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3tElEQVR4nO3deVwTd/4/8FfCEUAukVNFA1ILrigVq8WqeIFKdT1q69Wq6Opqi62Cttrdan3YXTzAal1b135XrWddBVlrEaWKSqkHonZRgRbLoRgEQQiXIcfn94e/zBLDkSgQknk/Hw8fZmY+mXkPk7zzmZnPfD4CxhgDIYSYOKGhAyCEkPZAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4Qwgvmhg6go1OpVHjw4AHs7OwgEAgMHQ4hpAHGGKqqqtC1a1cIhc3X3SjZteDBgwfw9PQ0dBiEkGbcu3cP3bt3b7YMJbsW2NnZAXj6x7S3tzdwNG1HLpfjzJkzCA0NhYWFhaHDIS+IL8dTKpXC09OT+542h5JdC9Snrvb29iaf7GxsbGBvb2/SXw6+4Nvx1OUSE92gIITwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1DTE0JMQG1tLbKzs7np6joZfs68i87O12BrLdIo6+vrCxsbm/YO0eAo2RFiArKzsxEYGKg1f1MjZTMyMjBgwIC2D6qDoWRHiAnw9fVFRkYGN50jqUDk0UxsecsfL3s4apXlI0p2PKXraQ9fT3mMjY2NjUZtTVhQBlFqHfz69kdAzy4GjKzjoGTHU7qe9vD1lIeYHkp2PKXraQ9fT3mI6aFkx1N02mP88h7VoEamaHTZ3dIa7n9z86a/5p1E5vBy7tQm8XU0lOwIMUJ5j2owMuZ8i+WijmW2WCZlxQheJDxKdjzRXC0AoJqAsVEfy63TA+Djaqu9vE6Gk+cvYcKIIHR6pp2dWm5JNZYdudns58KUULLjAV1rAQDVBIyNj6st+nZz0Jovl8tR7AIM6NmZF/3Z6YKSHQ+U11ZDaFWEFSG94enUeDOSOlk9Uq9lYthAf1iLLBstc6+8FjHJv6K8thpeoGRHjAslOx54UFOATl7b8XVuCwUtgPO/NF+kkxfwoCYAgXBrtfiI/mTKJxBaFSFPmgOhlfZprEKhwAPFA2SVZzV5WSJP+vRHUKZ8AkC7dmhqKNnxQNdOPVGTtxTbpgegVyPXd4CnX460n9Lw+tDXm/xy3C2pxodHbqLryJ5tGS7RgfoH7JOrzZf7KumrZpfz6cfL6JLdjh07sHnzZhQXF6N///7Yvn07Bg0a1GjZ27dvY82aNcjIyEBBQQG++OILLFu2rH0D7gBEZlZQPekGL/uX0adL47/gcrkceeZ58HPya/Iaj+pJJVRPSiEys2rLcIkOWvoBox8vbUaV7I4cOYLIyEjs3LkTgwcPxtatWzF27Fjk5OTA1dVVq3xtbS28vb3x1ltvYfny5QaImJC20dIPGP14aTOq/uy2bNmChQsXIjw8HH369MHOnTthY2OD3bt3N1r+1VdfxebNmzFjxgyIRI3ffieE8IPR1Ozq6+uRkZGB1atXc/OEQiHGjBmDS5cutdp2ZDIZZDIZNy2VSgE8/aWUy+Wttp32pFAouP+b2gf1/Ob2UZf1kPbR0rHgy/HUJ26jSXaPHj2CUqmEm5vmhVQ3NzeN3jteVHR0NNatW6c1/8yZM0bb+8e9agAwx08//YSCxu9PcJKTk1tlPaRt6XosTP141tbW6lzWaJJde1m9ejUiIyO5afWI46GhoUY7SPbtB1LEZF7G0KFD8Yeuje+DXC5HcnIyQkJCmrzGo8t6SPto6Vjw5Xiqz7x0YTTJztnZGWZmZnj48KHG/IcPH8Ld3b3VtiMSiRq9vmdhYWG0LdHVd+PMzc1b3Ifm9lOf9ZC2peuxMPXjqU/cRnODwtLSEoGBgTh79iw3T6VS4ezZswgKCjJgZIQQY2A0NTsAiIyMxNy5czFw4EAMGjQIW7duRU1NDcLDwwEAc+bMQbdu3RAdHQ3g6U2NO3fucK+Liopw8+ZN2NrawsfHx2D7QQhpf0aV7KZPn47S0lKsWbMGxcXFCAgIQFJSEnfTorCwEELh/yqrDx48wCuvvMJNx8TEICYmBsHBwTh//nx7h08IMSCjSnYAEBERgYiIiEaXPZvAxGIxGGPtEBUhpKMzmmt2hBDyIoyuZkf0VydXAgBuFVU2WaamToZrpYB7weNmO3skxFhRsuOBu/8/Sa2Kb6ljTnPsz01vcX2dRPSxIcaHPrU8EPqHp+0Qe7nawtrCrNEyOZJKRB3LROw0f7zs0XTfZtQtOzFWlOx4wKmTJWYM6tFsGfVzkr1cOjXazTfpWFq6NEGXJbRRsiPECOl2aYIuSzTEj70kxMS0dGmCLktoo2RHiBFq6dIEXZbQRu3sCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvGB0yW7Hjh0Qi8WwsrLC4MGDcfXq1WbLHz16FL6+vrCysoK/vz8SExPbKVJCSEdiVMnuyJEjiIyMxNq1a3H9+nX0798fY8eORUlJSaPlf/75Z8ycORMLFizAjRs3MHnyZEyePBm3bt1q58gJIYYmYEY0/NbgwYPx6quv4h//+AeAp4Nke3p6YunSpVi1apVW+enTp6OmpgYnT57k5r322msICAjAzp07ddqmVCqFg4MDKisrYW9v3zo70gHU1tYiOzubm86RVCDyaCa2vOWPlz0cufm+vr6wsbExQITkRdwsKMPkry8jYclrCOjZxdDhtBl9vp9G08VTfX09MjIysHr1am6eUCjEmDFjcOnSpUbfc+nSJURGRmrMGzt2LBISEtoyVKOQnZ2NwMBArfmzvtWczsjIwIABA9opKvK8GvvxkhXnIuuWNVRljhpl+foDZjTJ7tGjR1AqldyA2Gpubm4aB7mh4uLiRssXFxc3uR2ZTAaZTMZNS6VSAIBcLodcLn/e8DucXr164cqVK9x0dZ0Mp1PTMXbYq7Bt0I13r169TGq/TdWtW7cwePBgrfnP/ngBwJUrVzQGjzdm+nw2jSbZtZfo6GisW7dOa/6ZM2dM/tdwiH8vVFWUo6rif/MkEonB4iG6k8lkiI2N5ablKqD8CeBkBVg8c2U+Pz/fZI5rbW2tzmWNJtk5OzvDzMwMDx8+1Jj/8OFDuLu7N/oed3d3vcoDwOrVqzVOfaVSKTw9PREaGmpS1+yeJZfLkZycjJCQEFhYWBg6HPKC+HI81WdeujCaZGdpaYnAwECcPXsWkydPBvD0BsXZs2cRERHR6HuCgoJw9uxZLFu2jJuXnJyMoKCgJrcjEokgEmmPxmRhYWHSHxo1vuwnX5j68dRn34wm2QFAZGQk5s6di4EDB2LQoEHYunUrampqEB4eDgCYM2cOunXrhujoaADAhx9+iODgYMTGxuKNN97Ad999h2vXrmHXrl2G3A1CiAEYVbKbPn06SktLsWbNGhQXFyMgIABJSUncTYjCwkIIhf+7QDFkyBAcOnQIf/3rX/HJJ5/gpZdeQkJCAvr27WuoXSCEGIhRtbMzBFNtZ/csuVyOxMREhIWFmfRpD1/w5Xjq8/00qicoCCHkeVGyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjUCqVuHDhAi5evIgLFy5AqVQaOiRCWh0lO56Lj4+Hj48PQkJCsGXLFoSEhMDHxwfx8fGGDo2QVmVUHQGQ1hUfH49p06bhjTfewPLly/Hbb7/hpZdeQnJyMqZNm4Zjx45h6tSphg6TkFZBHQG0wFQ7AlAqlfDx8YGzszNKS0tRUFDALevZsydcXFxQVlaG3377DWZmZgaMlDwP6ghAG53G8lRqairy8/Nx7do19OvXD6mpqTh8+DBSU1PRr18/XLt2DXl5eUhNTTV0qIS0Ckp2PFVUVAQAGD9+PBISEjB48GBYW1tj8ODBSEhIwPjx4zXKEWLsKNnxVGlpKQBg6tSpGh2eAk+HqFR3fa8uR4ixo2THUy4uLgCe3qRQqVQay1QqFTe2rrocIcaOkh1PdevWDQCQlJSEyZMn4/Lly6irq8Ply5cxefJkJCUlaZQjxNhR0xOeGjZsGMRiMZydnZGZmYnhw4dzy7y8vBAYGIiysjIMGzbMgFES0noo2fGUmZkZYmNjMW3aNISFhWHChAn49ddf0bt3b+Tl5SExMRHHjh2jZifEdDAjUVZWxmbNmsXs7OyYg4MDmz9/Pquqqmr2Pf/85z9ZcHAws7OzYwDY48eP9d5uZWUlA8AqKyufM/KObeXKlczc3JwB4P6Zm5uzlStXGjo08gLq6+tZQkICq6+vN3QobUqf76fe1+x+//331s22Opo9ezZu376N5ORknDx5EhcvXsSiRYuafU9tbS3GjRuHTz75pJ2iNC7x8fGIiYmBublmBd/c3BwxMTH0yBgxLfpmUoFAwEaMGMH279/P6urqnisb6+vOnTsMAEtPT+fmnTp1igkEAlZUVNTi+1NSUqhm9wyFQsFcXV0ZACYQCDRqduppV1dXplAoDB0qeQ5Us9Om9zW769evY8+ePYiMjERERASmT5+OBQsWYNCgQa2YgjVdunQJjo6OGDhwIDdvzJgxEAqFuHLlCqZMmdJq25LJZJDJZNy0VCoF8PTxG7lc3mrbMbSUlBSUlJQAAJydnTFs2DBUVFTA0dERqampKC0tRUlJCc6ePYuRI0caOFqiL/Vn1ZQ+s43RZ//0TnYBAQHYtm0bYmNjceLECezduxdDhw5F7969MX/+fLz77rut3jaruLgYrq6uGvPMzc3h5OSE4uLiVt1WdHQ01q1bpzX/zJkzsLGxadVtGdL+/fsBPP07lpWVaZyyCoVCmJubQ6FQYNeuXairqzNUmOQFJScnGzqENlVbW6tz2ee+G2tubo6pU6fijTfewFdffYXVq1djxYoV+OSTT/D2229j48aN8PDwaHYdq1atwsaNG5stk5WV9bwhPpfVq1cjMjKSm5ZKpfD09ERoaKhJdQSwfft2AIBCoYCrqytmzpyJuro6WFtb4/Dhw1ytr6KiAmFhYYYMlTwHuVyO5ORkhISEmHxHALp67mR37do17N69G9999x06deqEFStWYMGCBbh//z7WrVuHSZMm4erVq82uIyoqCvPmzWu2jLe3N9zd3bkvn5pCoUB5eTnc3d2fdxcaJRKJIBKJtOZbWFiY1IdGXUs1MzODSCTCtm3buGWenp4wMzODUqmEjY2NSe0335ja5/ZZ+uyb3sluy5Yt2LNnD3JychAWFoZ9+/YhLCyMe77Sy8sLe/fuhVgsbnFdLi4uOp3yBgUFoaKiAhkZGQgMDAQAnDt3DiqVCoMHD9Z3F0gDSqVS62H/oqIirUfICDF2ejc9+frrrzFr1iwUFBQgISEBEyZM0HqQ3NXVFf/6179aLUg/Pz+MGzcOCxcuxNWrV5GWloaIiAjMmDEDXbt2BfD0C+rr66tRmywuLsbNmzeRm5sLAMjMzMTNmzdRXl7earEZq4aPgbFnujRsOE2PixGToe+t3ry8PKZUKrXmq1QqVlBQoO/qdFZWVsZmzpzJbG1tmb29PQsPD9doVJyXl8cAsJSUFG7e2rVrNZpUqP/t2bNH5+2aatOTJUuWNPq3efbfkiVLDB0q0VNVVRWbOHEi69mzJ5s4cWKLje+NWZs2PenVqxckEonW3dHy8nJ4eXm12WAtTk5OOHToUJPLxWKxVg3ls88+w2effdYm8Rg7R0fHVi1HOoZBgwYhPT2dmy4oKICdnR1effXVFq+hmzq9T2OfTShq1dXVsLKyeuGASPto6jg+bzlieM8muobS09PbtC2sMdC5ZqdujiEQCLBmzRqNNmdKpRJXrlxBQEBAqwdI2kZZWVmrliOGVV1d3WSiU0tPT0d1dTVsbW3bKaqORedkd+PGDQBPf+kzMzNhaWnJLbO0tET//v2xYsWK1o+QtIlbt261ajliWNOmTdO5nLqvQr7ROdmlpKQAAMLDw7Ft2zaTamDLR7o2xtSn0SYxnIZPSnTp0gV/+9vfIBKJIJPJ8Je//IWroZv6ExXN0fua3Z49eyjRmQA3Nzfu9bPXWhtONyxHOq6G7SKLi4sxf/58dO7cGfPnz9d4pJLP7Sd1qtlNnToVe/fuhb29fYuDJlO3QMahYWNue3t7LF68GLW1tbCxscGhQ4fw5MkTrXLEOCgUCo1OVxUKhQGj6Th0SnYODg4QCATca2L8GvZhV1JSgq1bt7ZYjnRc1tbWXIcN1tbWmDVrFgIDAzFv3jyNJlvW1taGCtHgdPok79mzp9HXxHj17NmzVcsRw1q1ahXWrl3LTR86dKjRdqmrVq1qz7A6FBpdjKdGjRrV5DJ1Lb6lcqTj0DWJ8TnZ6VSze+WVVzS+AM25fv36CwVE2seIESPg4OCAyspKCASCRhsPOzg4YMSIEe0fHNGbpaUlVq5cic2bNzdZZuXKlRpNxvhGp2SnHh2emBb1B//ZRKeebqyrK9Jxbdq0CQC0Ep5AIMCKFSu45XwlYPQ8ULOkUilXAzKlJjfnz5/nultveHEbeNrXnboH2JSUFKrdGZn6+nps374d586dw6hRo7B06VKTrdHp8/2ka3Y8pe7Dbvz48aisrERycjIiIyORnJyMiooKjB8/XqMcMR6Wlpb44IMPsGjRInzwwQcmm+j0pdNprJOTE3799Vc4Ozujc+fOzV6/o77ijENpaSmAp20oLSwsEBwcjJqaGgQHB8PCwgKTJ0/GqVOnuHKEGDudkt0XX3wBOzs77rWuNytIx6VuLBwfH4/58+drLFOpVEhISNAoR4ix0ynZzZ07l3vd0pgRxDioeyBOSkrC5MmTsXLlStTV1eHy5cvYvHkz97A49VRMTIXezePNzMwa7byzrKwMrq6ubdZ5J2ldw4YNg1gshrOzMzIzMzF8+HBumZeXFwIDA1FWVoZhw4YZMEpCWo/eya6pm7cymYwuhBoRMzMzxMbGYtq0aVpNTCQSCfLz83Hs2DGNZywJMWY6J7svv/wSwNM2O//3f/+n0QGgUqnExYsX4evr2/oRkjbV2I9XU42MCTFqug5sIRaLmVgsZgKBgHl6enLTYrGY9e7dm4WGhrLLly/rO16GzsrKytisWbOYnZ0dc3BwYPPnz292IJGysjIWERHBevfuzaysrJinpydbunQpq6io0Gu7pjrgjkKhYGKxmA0cOJCJxWKNQXbU8728vJhCoTB0qOQ51NfXs4SEBFZfX2/oUNpUmwy4k5eXBwAYOXIk4uPj0blz57bIvU2aPXs2JBIJkpOTIZfLER4ejkWLFjU5CM+DBw/w4MEDxMTEoE+fPigoKMDixYvx4MEDHDt2rF1j74hSU1ORn5+PgoICTJgwAfv378f9+/fRvXt3bNq0CSdPngRjDKmpqdSomJiGts+9L+7OnTsMAEtPT+fmnTp1igkEAlZUVKTzev79738zS0tLJpfLdX6PqdbsDhw4wACw8ePHM6VSqVETUCqVbPz48QwAO3DggKFDJc+Banba9L5B8WybrGft3r37eXJusy5dugRHR0cMHDiQmzdmzBgIhUJcuXIFU6ZM0Wk96kdKqI82zUbFCoWCe7woNzcXS5cupUbFxOTo/a1//PixxrRcLsetW7dQUVHRZt0BFRcXazV1MTc3h5OTk0aX08159OgR1q9fj0WLFjVbTiaTQSaTcdPqMRjkcjnkcrmekXdc6ssQn3/+ORYvXsw1GUpMTMTHH3+M7t27c+VMab/5Qn3MTP3Y6bN/eie748ePa81TqVRYsmQJevXqpde6Vq1ahY0bNzZbJisrS691NkYqleKNN95Anz59Whw0Ozo6GuvWrdOaf+bMGY3hI43dvXv3ADwdRNnc3Bz+/v7o3LkzHj9+jKysLBQUFHDlEhMTDRkq0ZNSqcSdO3fw+PFjZGZmok+fPibbhEjdYYUuWq3Xk5ycHIwYMQISiUTn95SWlrY4Lqm3tzcOHDiAqKgojVqlQqGAlZUVjh492uxpbFVVFcaOHQsbGxucPHmyxYG8G6vZeXp64tGjRybV60ldXR3X3X5jHwH1/MrKSl535W1sjh8/jpUrV6KwsJCb16NHD2zevFnnyz3GRCqVwtnZWbdeiVrrQuEPP/zAnJ2dW2t1GtQ3KK5du8bNO336dIs3KCorK9lrr73GgoODWU1NzXNt21RvUHzxxRdcUxMLCwvWv39/5uvry/r3788sLCy4ZV988YWhQyU6iouL02hC9Oy/uLg4Q4fY6tr0BkVkZOSzyRISiQQ//PCDxjO0rcnPzw/jxo3DwoULsXPnTsjlckRERGDGjBno2rUrgKddEY0ePRr79u3DoEGDIJVKERoaitraWhw4cABSqZS7/ubi4mKy1Xpd/frrrwCedgdUX1+PX375RWO5er66HOnYlEolwsPDmy0THh6OSZMm8fazr3eyu3Hjhsa0UCiEi4sLYmNjW7xT+yIOHjyIiIgIjB49GkKhEG+++Sb3VAfw9EJlTk4Odw5//fp1XLlyBQDg4+Ojsa68vDyIxeI2i9UYqC831NfXw9XVFe7u7igrK0OXLl1QXFyMkpISjXKkYzt79myLA5pLpVKcPXsWoaGh7RRVx6J3sktJSWmLOFrk5OTUZANiABCLxRrXnkaMGEGPPDWjYddNJSUlXHJ7trNO6uLJOOzdu1fncnxNdtRTMU/p2n6O2tkZB10HuuLzgFjUupandK2xUc3OOKibCqmNGTMGHh4ekEgk+PHHH5ssxyeU7Hjq1q1b3Gtzc3P07NkTdXV1sLa2RkFBARQKhVY50nE1bFxrZWWlkeCsrKzw5MkTrXJ8Q8mOp9QffuBpm8W7d++2WI4Yh2ePGR3Dp1rtmt39+/dbfBSLdBwNxxERCATo1q0b3Nzc0K1bN61lpOPTtRei9u6tqCNptWRXVlaGf/3rX621OtLGGnbbxBhDUVERHj58iKKiIq272qTj0/VRTX0f6TQldDeWp3R9plCfZw+J4ejabpTP7Usp2RFiAho+z90a5UwRJTueevDgQauWI4alHgWuqU4b1PP5PFqczndjp06d2uzyioqKF42FtCOVSgXg6eN+jDGN63QN56nLkY4tIiKCG/t33LhxqKqqQmFhIXr06AE7OzskJSVBKBQiIiLC0KEajM7JzsHBocXlc+bMeeGASPtQn86oVCpYWlpiypQpsLa2Rl1dHY4fP476+nqNcqRjs7S0RFRUFDZv3owzZ85wP1L37t3jHvyPiori9XCnOie7PXv2tGUcpJ0NGDAAycnJEAgEUCqVOHLkCLfM3Nyc689uwIABBoyS6GPTpk0AgC1btmgtW7lyJbecr6hRMU+pu7lnjMHJyQnDhw/H48eP0blzZ1y8eJF7JvbZ7vBJx7Zp0yZ8/vnn3Jgio0aNwtKlS3ldo1PTOdnp2n1TWwy4Q1qfm5sb97q0tBRxcXEtliPGwdLSEh988AF8fHwQFhYGCwsLQ4fUIeic7Pbu3YuePXvilVdeoa6TTEC3bt241+prdY1NNyxHiDHTOdktWbIEhw8fRl5eHsLDw/HOO+/AycmpLWMjbWjYsGEQi8VwdnZGaWmpRm8Ybm5ucHZ2RllZGa+bKhDTonM7ux07dkAikeCjjz7C999/D09PT7z99ts4ffo01fSMkJmZGWJjY5GRkQF/f39s27YNERER2LZtG/r27YuMjAzExMTwtgtvYnqee3SxgoIC7N27F/v27YNCocDt27dha2vb2vEZnFQqhYODg26jFxmh+Ph4REVFIT8/n5vn5eWFmJiYFttWko5LLpcjMTHR5K/Z6fP9fO67sUKhkGueoB5gmRifqVOnYtKkSUhJScGpU6cwfvx4jBw5kmp0xOTo9biYTCbD4cOHERISgt69eyMzMxP/+Mc/UFhYaJK1Or4wMzNDcHAwhg8fjuDgYEp0xCTpnOzee+89eHh4YMOGDZgwYQLu3buHo0ePIiwsDEJh2z9iW15ejtmzZ8Pe3h6Ojo5YsGABqqurm33Pn//8Z/Tq1QvW1tZwcXHBpEmTkJ2d3eaxEkI6Hp1PY3fu3IkePXrA29sbFy5cwIULFxotFx8f32rBNTR79mxIJBIkJydDLpcjPDwcixYtanbEscDAQMyePRs9evRAeXk5PvvsM4SGhiIvL49qL4TwjM7Jbs6cOQbrtTYrKwtJSUlIT0/HwIEDAQDbt29HWFgYYmJiuIGyn9Ww52SxWIzPP/8c/fv3R35+Pq87MSSEj/RqVGwoly5dgqOjI5fogKejJwmFQly5cgVTpkxpcR01NTXYs2cPvLy84Onp2WQ5mUym8fC7euBhuVxu0oOVqPfNlPeRT/hyPPXZP6N4Nra4uFjrGU1zc3M4OTmhuLi42fd+9dVX+Oijj1BTU4OXX34ZycnJzT4nGB0djXXr1mnNP3PmDGxsbJ5vB4xIcnKyoUMgrcjUj6c+PWkbNNmtWrUKGzdubLZMVlbWC21j9uzZCAkJgUQiQUxMDN5++22kpaXBysqq0fKrV69GZGQkNy2VSuHp6YnQ0FCTbGenJpfLkZycjJCQEJNul8UXfDme6jMvXRg02UVFRWHevHnNlvH29oa7uztKSko05isUCpSXl8Pd3b3Z9zs4OMDBwQEvvfQSXnvtNXTu3BnHjx/HzJkzGy0vEokgEom05ltYWJj0h0aNL/vJF6Z+PPXZN4MmOxcXF51GnA8KCkJFRQUyMjIQGBgIADh37hxUKhUGDx6s8/bUve9Sh5SE8I9RjEHh5+eHcePGYeHChbh69SrS0tIQERGBGTNmcHdii4qK4Ovri6tXrwIAfv/9d0RHRyMjIwOFhYX4+eef8dZbb8Ha2hphYWGG3B1CiAEYRbIDgIMHD8LX1xejR49GWFgYhg4dil27dnHL5XI5cnJyuAuWVlZWSE1NRVhYGHx8fDB9+nTY2dnh559/pg4pCeEho7gbCwBOTk7NNiAWi8Uava907doViYmJ7REaIcQIGE3NjhBCXgQlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8ILRJLvy8nLMnj0b9vb2cHR0xIIFC1BdXa3TexljGD9+PAQCARISEto2UEJIh2Q0yW727Nm4ffs2kpOTcfLkSVy8eBGLFi3S6b1bt26FQCBo4wgJIR2ZUYwulpWVhaSkJKSnp2PgwIEAgO3btyMsLAwxMTHc2LGNuXnzJmJjY3Ht2jV4eHi0V8iEkA7GKGp2ly5dgqOjI5foAGDMmDEQCoW4cuVKk++rra3FrFmzsGPHDri7u7dHqISQDsooanbFxcVaA1ubm5vDyckJxcXFTb5v+fLlGDJkCCZNmqTztmQyGWQyGTctlUoBPB2EWy6X6xm58VDvmynvI5/w5Xjqs38GTXarVq3Cxo0bmy2TlZX1XOs+ceIEzp07hxs3buj1vujoaKxbt05r/pkzZ2BjY/NcsRiT5ORkQ4dAWpGpH8/a2lqdywoYY6wNY2lWaWkpysrKmi3j7e2NAwcOICoqCo8fP+bmKxQKWFlZ4ejRo5gyZYrW+5YtW4Yvv/wSQuH/ztSVSiWEQiGGDRuG8+fPN7q9xmp2np6eePToEezt7fXcQ+Mhl8uRnJyMkJAQWFhYGDoc8oL4cjylUimcnZ1RWVnZ4vfToDU7FxcXuLi4tFguKCgIFRUVyMjIQGBgIADg3LlzUKlUGDx4cKPvWbVqFf70pz9pzPP398cXX3yBiRMnNrktkUgEkUikNd/CwsKkPzRqfNlPvjD146nPvhnFNTs/Pz+MGzcOCxcuxM6dOyGXyxEREYEZM2Zwd2KLioowevRo7Nu3D4MGDYK7u3ujNyV69OgBLy+v9t4FQoiBGcXdWAA4ePAgfH19MXr0aISFhWHo0KHYtWsXt1wulyMnJ0evc3hCCH8YRc0OAJycnHDo0KEml4vFYrR0+dGAlycJIQZmNDU7Qgh5EZTsCCG8QMmOEMILlOwIIbxAyY4QE6NUKnHhwgVcvHgRFy5cgFKpNHRIHQIlO0JMSHx8PHx8fBASEoItW7YgJCQEPj4+iI+PN3RoBkfJjhATER8fj2nTpsHf3x+pqak4fPgwUlNT4e/vj2nTpvE+4VGyI8QEKJVKREVFYcKECYiLi8OTJ0+Qnp6OJ0+eIC4uDhMmTMCKFSt4fUpLyY4QE5Camor8/HwMGTKk0dPYoKAg5OXlITU11dChGozRPEFBCGmaRCIBAKxevVprCIJ79+7hk08+0SjHR5TsCDEBDTu3dXV1xbp16yASiSCTybB27Vo8fPhQqxzf0GksISagvr4ewNMujwoLCzF//nx07twZ8+fPR2FhIdcVkrocH1HNjhAToO4kQy6XY/LkyZBKpSgsLMTmzZthb2/PdV9+6NAhjB8/3pChGgwlO0JMgHoMZXd3d5w6dYqbf+/ePQCAm5sbHj58qPNYy6aITmMJMQFDhw4FgCYHoFJfs1OX4yNKdoSYgPDw8FYtZ4oo2RFiAubOnduq5UwRJTtCTEBaWhr3+tl2dg2nG5bjG0p2hJgA9fCfdnZ2qKqqwuLFixEQEIDFixejqqoKdnZ2GuX4iO7GEmICnJ2dUVhYiKqqKvj6+uL+/fsAgJs3b+LkyZOoqqriyvGV0dTsysvLMXv2bNjb28PR0RELFixo8Tb6iBEjIBAINP4tXry4nSImpP0EBQVxr9WJrrHphuX4xmiS3ezZs3H79m0kJyfj5MmTuHjxIhYtWtTi+xYuXAiJRML927RpUztES0j7oruxLTOK09isrCwkJSUhPT0dAwcOBABs374dYWFhiImJ4QbKboyNjU2jg2UTYkpUKlWrljNFRpHsLl26BEdHRy7RAcCYMWMgFApx5coVTJkypcn3Hjx4EAcOHIC7uzsmTpyITz/9FDY2Nk2Wl8lkGhdxpVIpgKeP4agfuTE19fX12LFjBy5cuICcnBy8//77sLS0NHRYRA/79+/XudyYMWPaOJr2o8930iiSXXFxsVZvDebm5nBycmqyxTgAzJo1Cz179kTXrl3x3//+Fx9//DFycnKa7bE1Ojoa69at05p/5syZZpOksdq7dy9OnDjB/eInJiZi9erV+OMf/4h58+YZNjiisxs3bnCvLSwsNJJAw+kbN24gMTGx3eNrK7W1tTqXNWiyW7VqFTZu3NhsmaysrOdef8Nrev7+/vDw8MDo0aNx9+5d9OrVq9H3rF69GpGRkdy0VCqFp6cnQkNDYW9v/9yxdESrVq1CQkICXFxc4Ofnh0ePHsHZ2RlZWVlISEiAt7c3NmzYYOgwiQ62bduG7OxsWFtbo6SkBD/99BOSk5MREhKCoUOHwtXVFXV1dfDw8EBYWJihw2016jMvXRg02UVFRbVYe/D29oa7uztKSko05isUCpSXl+t1PW7w4MEAgNzc3CaTnUgkgkgk0ppvYWHBdZNjCurr6/Hll1/C0tISpaWlKC0t1VhuaWmJL7/8EtHR0XRKawTU34O6ujrMmDEDH3/8MV599VV06tQJM2bMQF1dHVfOlD7H+uyLQZOdi4sLXFxcWiwXFBSEiooKZGRkIDAwEABw7tw5qFQqLoHp4ubNmwAADw+P54rXlHz11VdQKBRNLlf3e/bVV19h2bJl7RQVeV5eXl7c61OnTuGHH37gps3MzBotxzdG0fTEz88P48aNw8KFC3H16lWkpaUhIiICM2bM4O7EFhUVwdfXF1evXgUA3L17F+vXr0dGRgby8/Nx4sQJzJkzB8OHD0e/fv0MuTsdQnZ2dquWI4Y1atQo7vWzd1wbDrLTsBzfGEWyA57eVfX19cXo0aMRFhaGoUOHYteuXdxyuVyOnJwc7oKlpaUlfvzxR4SGhsLX1xdRUVF488038f333xtqFzqUzMzMVi1HDGvEiBHcWZK1tbXGMvW0q6srRowY0d6hdRhGcTcWAJycnLjeWBsjFovBGOOmPT09ceHChfYIzSjV1NS0ajliWGZmZti5cyfefPNNrWXqjgC+/vprjVNavjGamh1pXYWFha1ajhje1KlTERcXp9VMy9XVFXFxcZg6daqBIusYBKxhdYhokUqlcHBwQGVlpUk1PTE3N9dpwGQzM7Nmb2SQjkepVCIlJQWnTp3C+PHjMXLkSJOt0enz/TSa01jSuhr+xllYWOD111+HSqWCUChEWloa1wiVfguNj5mZGYKDg1FTU4Pg4GCTTXT6omTHUxYWFtxjcXK5HOfPn2+yHCGmgK7Z8ZS3t3erliOko6Nkx1M0ZgHhG0p2PLV8+fJWLUdIR0fJjqcsLS2xcuXKZsusXLmSnoslJoNuUPCYutfmmJgYjbuuQqEQUVFR1KszMSlUs+O5TZs24cmTJ4iJieF6fq6rq6NER0wO1ewILC0t8cEHH8DHxwdhYWHU3ISYJKrZEUJ4gZIdIYQXKNkRQniBrtm1QH2XUp++7o2RXC5HbW0tpFIpXbMzAXw5nurvpS7PcFOya0FVVRWAp/3jEUI6pqqqKjg4ODRbhrp4aoFKpcKDBw9gZ2fHdYJoitSjqN27d8+kurLiK74cT8YYqqqq0LVrVwiFzV+Vo5pdC4RCIbp3727oMNqNvb29SX85+IYPx7OlGp0a3aAghPACJTtCCC9QsiMAng4Ovnbt2kYHCCfGh46nNrpBQQjhBarZEUJ4gZIdIYQXKNkRQniBkp0RyM/Ph0AgwM2bN9tlPWlpafD394eFhQUmT578QtskpKOgZGeEzp8/D4FAgIqKijZZf2RkJAICApCXl4e9e/e2yTaIthEjRmDZsmWGDoPT0eJ5UZTs2lh9fb2hQ9Db3bt3MWrUKHTv3h2Ojo7PtY723G/GGBQKRbttryMzxs9bu2GkVQUHB7P333+fffjhh6xLly5sxIgRLDMzk40bN4516tSJubq6snfeeYeVlpZy7zl16hR7/fXXmYODA3NycmJvvPEGy83N5Zbn5eUxAOzGjRvc64b/5s6dq9d6Dh8+zIKCgphIJGJ/+MMf2Pnz5zWWN/y3Z88exhhj58+fZ6+++iqztLRk7u7u7OOPP2ZyubzZ/U5JSWEAWFJSEgsICGBWVlZs5MiR7OHDhywxMZH5+voyOzs7NnPmTFZTU8OtS6lUsr///e9MLBYzKysr1q9fP3b06FFuuXq9iYmJbMCAAczCwoKlpKS05mFsd3PnztX62+fm5rL58+dzf4fevXuzrVu3ar1v0qRJ7PPPP2ceHh5MLBYzxhhLS0tj/fv3ZyKRiAUGBrLjx49znyG15j6XjcWTl5fXXn+ONkHJrpUFBwczW1tbtnLlSpadnc0uX77MXFxc2OrVq1lWVha7fv06CwkJYSNHjuTec+zYMRYXF8d+++03duPGDTZx4kTm7+/PlEolY0wz2SkUChYXF8cAsJycHCaRSFhFRYVe6+nevTs7duwYu3PnDvvTn/7E7Ozs2KNHj5hCoWASiYTZ29uzrVu3MolEwmpra9n9+/eZjY0Ne++991hWVhY7fvw4c3Z2ZmvXrm1yv7Ozs7mk9Nprr7GffvqJXb9+nfn4+LDg4GAWGhrKrl+/zi5evMi6dOnCNmzYwK3r888/Z76+viwpKYndvXuX7dmzh4lEIi4pq9fbr18/dubMGZabm8vKysra+tC2qYqKChYUFMQWLlzIJBIJk0gk7MmTJ2zNmjUsPT2d/f777+zAgQPMxsaGHTlyhHvf3Llzma2tLXv33XfZrVu32K1bt1hlZSVzcnJi77zzDrt9+zZLTExkvXv31kh2jx8/bvZz2Vg8CoXCEH+aVkPJrpUFBwezV155hZtev349Cw0N1Shz7949Llk1prS0lAFgmZmZjDHNZMfY/77sjx8/bjaWptbTMLHI5XLWvXt3tnHjRm6eg4MDV6NjjLFPPvmEvfzyy0ylUnHzduzYwWxtbblE+ux+N4zzxx9/5OZFR0czAOzu3bvcvD//+c9s7NixjDHGnjx5wmxsbNjPP/+ssa4FCxawmTNnaqw3ISGh2f03NsHBwezDDz9stsz777/P3nzzTW567ty5zM3NjclkMm7e119/zbp06cLq6uq4ed98843GZ0iXz6Uu8RgTumbXBgIDA7nXv/zyC1JSUmBra8v98/X1BfD02hgA/Pbbb5g5cya8vb1hb28PsVgMACgsLNRru7quJygoiHttbm6OgQMHIisrq8n1ZmVlISgoSKOLq9dffx3V1dW4f/9+o/vdUL9+/bjXbm5usLGxgbe3t8a8kpISAEBubi5qa2sREhKi8Tfbt28f9/dSGzhwYJMxm4odO3YgMDAQLi4usLW1xa5du7SOp7+/v8b4vjk5OejXrx+srKy4eYMGDdJ4jy6fS1NDXTy1gU6dOnGvq6urMXHiRGzcuFGrnIeHBwBg4sSJ6NmzJ7755ht07doVKpUKffv21ftic2ut53k13O+GGvaUKxAItHrOFQgEUKlUAJ7+vQDghx9+QLdu3TTKPfucZ1PbMxXfffcdVqxYgdjYWAQFBcHOzg6bN2/GlStXNMo9z99Bl8+lqaFk18YGDBiAuLg4iMVimJtr/7nLysqQk5ODb775BsOGDQMA/PTTT82uU/0rrlQqn2s9ly9fxvDhwwEACoUCGRkZiIiIaHJ7fn5+iIuLA2OMq92lpaXBzs6u1fv669OnD0QiEQoLCxEcHNyq6+7oLC0tNY5pWloahgwZgvfee4+bp0ut6+WXX8aBAwcgk8m4H4j09HSNMi19LhuLx9jRaWwbe//991FeXo6ZM2ciPT0dd+/exenTpxEeHg6lUonOnTujS5cu2LVrF3Jzc3Hu3DlERkY2u86ePXtCIBDg5MmTKC0tRXV1tV7r2bFjB44fP47s7Gy8//77ePz4MebPn9/k9t577z3cu3cPS5cuRXZ2Nv7zn/9g7dq1iIyMbLF3WH3Z2dlhxYoVWL58Ob799lvcvXsX169fx/bt2/Htt9+26rY6GrFYjCtXriA/Px+PHj3CSy+9hGvXruH06dP49ddf8emnn2olrcbMmjULKpUKixYtQlZWFk6fPo2YmBgA4H6sWvpcNhaPuvZtrCjZtbGuXbsiLS0NSqUSoaGh8Pf3x7Jly+Do6AihUAihUIjvvvsOGRkZ6Nu3L5YvX47Nmzc3u85u3bph3bp1WLVqFdzc3BAREaHXejZs2IANGzagf//++Omnn3DixAk4Ozs3u73ExERcvXoV/fv3x+LFi7FgwQL89a9/faG/TVPWr1+PTz/9FNHR0fDz88O4cePwww8/wMvLq02211GsWLECZmZm6NOnD1xcXDB27FhMnToV06dPx+DBg1FWVqZRy2uKvb09vv/+e9y8eRMBAQH4y1/+gjVr1gAAdx2vpc9lY/Hoew25o6EungjhgYMHDyI8PByVlZWwtrY2dDgGQdfsCDFB+/btg7e3N7p164ZffvkFH3/8Md5++23eJjqAkh0hJqm4uBhr1qxBcXExPDw88NZbb+Fvf/ubocMyKDqNJYTwAt2gIITwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO9JhzJs3DwKBgOsswM3NDSEhIdi9e7dejyrt3bv3uXtYfhHz5s2jMTs6MEp2pEMZN24cJBIJ8vPzcerUKYwcORIffvghJkyYQF2vkxdj0N70CGlA3cX4s86ePcsAsG+++YYxxlhsbCzr27cvs7GxYd27d2dLlixhVVVVjLH/dezZ8J+6R+V9+/axwMBAZmtry9zc3NjMmTPZw4cPue2Ul5ezWbNmMWdnZ2ZlZcV8fHzY7t27ueWFhYXsrbfeYg4ODqxz587sj3/8I9dV+dq1a7W2a+xdxZsaqtmRDm/UqFHo378/4uPjAQBCoRBffvklbt++jW+//Rbnzp3DRx99BAAYMmQItm7dCnt7e0gkEkgkEqxYsQIAIJfLsX79evzyyy9ISEhAfn4+5s2bx23n008/xZ07d3Dq1ClkZWXh66+/5jpIkMvlGDt2LOzs7JCamoq0tDTY2tpi3LhxqK+vx4oVK/D2229zNVOJRIIhQ4a07x+KNM/Q2ZYQtaZqdowxNn36dObn59fosqNHj7IuXbpw03v27GEODg4tbi89PZ0B4GqFEydOZOHh4Y2W3b9/v1bX9DKZjFlbW7PTp0+3GD8xPKrZEaPAGnQc+uOPP2L06NHo1q0b7Ozs8O6776KsrAy1tbXNriMjIwMTJ05Ejx49YGdnx3UOqu66aMmSJfjuu+8QEBCAjz76CD///DP33l9++QW5ubmws7PjujF3cnLCkydPTLYbc1NDyY4YhaysLHh5eSE/Px8TJkxAv379EBcXh4yMDOzYsQNA82Om1tTUYOzYsbC3t8fBgweRnp6O48ePa7xv/PjxKCgowPLly/HgwQOMHj2aOwWurq5GYGAgbt68qfHv119/xaxZs9p470lroF5PSId37tw5ZGZmYvny5cjIyIBKpUJsbCzXyeS///1vjfKNdSeenZ2NsrIybNiwAZ6engCAa9euaW3LxcUFc+fOxdy5czFs2DCsXLkSMTExGDBgAI4cOQJXV1fY29s3GqepdWNuaqhmRzoUmUyG4uJiFBUV4fr16/j73/+OSZMmYcKECZgzZw58fHwgl8uxfft2/P7779i/fz927typsQ6xWIzq6mqcPXsWjx49Qm1tLXr06AFLS0vufSdOnMD69es13rdmzRr85z//QW5uLm7fvo2TJ0/Cz88PADB79mw4Oztj0qRJSE1NRV5eHs6fP48PPviAG2FNLBbjv//9L3JycvDo0SPI5fL2+aMR3Rj6oiEhag1HoTc3N2cuLi5szJgxbPfu3dz4tIwxtmXLFubh4cGsra3Z2LFj2b59+7TG0V28eDHr0qWLRtOTQ4cOMbFYzEQiEQsKCmInTpzQGkvVz8+PWVtbMycnJzZp0iT2+++/c+uUSCRszpw5zNnZmYlEIubt7c0WLlzIKisrGWOMlZSUsJCQEGZra0tNTzog6s+OEMILdBpLCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBe+H+0V+hpHbxligAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T15:12:10.497739Z", + "iopub.status.busy": "2024-02-29T15:12:10.497212Z", + "iopub.status.idle": "2024-02-29T15:12:10.745650Z", + "shell.execute_reply": "2024-02-29T15:12:10.744752Z" + }, + "papermill": { + "duration": 0.273377, + "end_time": "2024-02-29T15:12:10.747757", + "exception": false, + "start_time": "2024-02-29T15:12:10.474380", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEqCAYAAABEE9ZrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTaUlEQVR4nO2dd3hT5dvHv9ndi9JFWyhQOtirUFAZliUioL+XoSJLFAEHlS0bpSggKCCIylAZDhQQEYGyp7RsaAsthTK6S0c6Ms/7R3pOT2aT9qRJy/O5rlyQkzOekyZ3vs/93INHURQFAoFAqGfwbT0AAoFAqAnEeBEIhHoJMV4EAqFeQowXgUColxDjRSAQ6iXEeBEIhHoJMV4EAqFeQowXgUColxDjRSAQ6iXEeBEIhHqJTY1XXFwcunbtCldXV/j4+GDYsGFISUmp9rjffvsN4eHhcHBwQNu2bXHw4ME6GC2BQLAnbGq8Tp48ialTp+LChQs4cuQIFAoF+vfvj9LSUqPHnDt3DqNHj8bEiRNx5coVDBs2DMOGDcPNmzfrcOQEAsHW8OwpMTs3Nxc+Pj44efIkXnjhBYP7jBw5EqWlpThw4ACzrXv37ujQoQM2bdpU7TXUajWePHkCV1dX8Hg8zsZOIBC4gaIolJSUICAgAHy+cX0lrMMxVUtRUREAwMvLy+g+58+fR2xsrNa2AQMGYO/evQb3l8lkkMlkzPPHjx8jMjKy9oMlEAhW5eHDhwgMDDT6ut0YL7VajY8++gg9e/ZEmzZtjO6XlZUFX19frW2+vr7IysoyuH9cXByWLFmit/3hw4dwc3Or3aAJBALnFBcXIygoCK6urib3sxvjNXXqVNy8eRNnzpzh9Lxz587VUmr0G+Pm5kaMF4Fgx1Tn1rEL4zVt2jQcOHAAp06dMikTAcDPzw/Z2dla27Kzs+Hn52dwf4lEAolEwtlYCQSCfWDT1UaKojBt2jT8+eefOHbsGEJCQqo9Jjo6GvHx8Vrbjhw5gujoaGsNk0Ag2CE2VV5Tp07Fzp07sW/fPri6ujJ+K3d3dzg6OgIA3nrrLTRp0gRxcXEAgA8//BC9evXC6tWrMXjwYOzevRsJCQnYvHmzze6DQCDUPTY1Xhs3bgQA9O7dW2v71q1bMW7cOABARkaG1nJpjx49sHPnTsyfPx/z5s1DaGgo9u7da9LJbykURUGpVEKlUnF2TgLBHhAIBBAKhQ0iTMiu4rzqguLiYri7u6OoqMigw14ulyMzMxNlZWU2GB2BYH2cnJzg7+8PsVhs66EYpLrvKI1dOOztBbVajfT0dAgEAgQEBEAsFjeIXygCAdDMKORyOXJzc5Geno7Q0FCTQaD2DjFeLORyOdRqNYKCguDk5GTr4RAInOPo6AiRSIQHDx5ALpfDwcHBatc6fCsLOy5mYMVrbeHv7sj5+euv2bUi9fnXiECojrr6fK86nIKTd3JxMiXXKucn31ICgcA5D/JLcSdbCgCQq9RWuQYxXgQCgXOO3K4KJFeorLMmSIwXwSj3798Hj8fD1atX6+Q8Z8+eRdu2bSESiTBs2LBaXfNZ5cSJE+DxeCgsLLTpOA6zjJeSKC+CrbH2FyM2NhYdOnRAeno6tm3bZpVrEKxPQakcCfcLmOdKNVFeBAuQy+W2HoLFpKWloW/fvggMDISHh0eNzlGX900HM3NNffzbsYlPygbbXimI8rINFEWhTK60ycOS+OHevXtj2rRp+Oijj+Dt7Y0BAwbg5s2bGDRoEFxcXODr64sxY8YgLy+POebQoUN47rnn4OHhgUaNGuHll19GWlqawfPfv38fffr0AQB4enqCx+MxWRDmnic5ORk9evSAg4MD2rRpg5MnTzLn5vF4yM/Px4QJE8Dj8RjldfLkSURFRUEikcDf3x9z5szRMhiG7ptWiP/++y86duwIR0dH9O3bFzk5Ofjnn38QEREBNzc3vP7661rByGq1GnFxcQgJCYGjoyPat2+P33//nXmdPu8///yDzp07QyKRmFUF5dNPP4WPjw9cXV3x9ttvY86cOejQoQPz+rhx4zBs2DB89tlnCAgIQFhYGADgp59+QpcuXeDq6go/Pz+8/vrryMnJ0Tr3wYMH0apVKzg6OqJPnz64f/9+teOxNrS/i18ZIqm0ks+LxHlVQ7lChciF/9rk2reXDoCT2Pw/0fbt2/Hee+/h7NmzKCwsRN++ffH2229jzZo1KC8vx+zZszFixAgcO3YMAFBaWorY2Fi0a9cOUqkUCxcuxPDhw3H16lW95fSgoCDs2bMHr732GlJSUuDm5sbkn5p7npkzZ2Lt2rWIjIzEl19+iSFDhiA9PR1BQUHIzMxEWFgYli5dipEjR8Ld3R2PHz/GSy+9hHHjxuHHH39EcnIyJk2aBAcHByxevNjgfQNAZmYmAGDx4sVYv349nJycMGLECIwYMQISiQQ7d+6EVCrF8OHDsW7dOsyePRuApvbbzz//jE2bNiE0NBSnTp3Cm2++icaNG6NXr17M9ebMmYNVq1ahefPm8PT0NPk32bFjBz777DN888036NmzJ3bv3o3Vq1frFSGIj4+Hm5sbjhw5wmxTKBRYtmwZwsLCkJOTg9jYWIwbN47p2fDw4UO8+uqrmDp1Kt555x0kJCTg448/rv6DYkUqFCqcvqv5gewW0gjn7+VbTXkR49WACA0NxRdffAFA82vfsWNHLF++nHl9y5YtCAoKwp07d9CqVSu89tprWsdv2bIFjRs3xu3bt/VyRQUCAVPh1sfHR2taZ+55pk2bxuy7ceNGHDp0CD/88ANmzZoFPz8/8Hg8uLu7M+WNvvnmGwQFBWH9+vXg8XgIDw/HkydPMHv2bCxcuJAxjOz7BqqM16effoqePXsCACZOnIi5c+ciLS0NzZs3BwD873//w/HjxzF79mzIZDIsX74cR48eZSqUNG/eHGfOnMG3336rZbyWLl2Kfv36Vf8HAbBu3TpMnDgR48ePBwAsXLgQhw8fhlQq1drP2dkZ33//vVbKzoQJE5j/N2/eHF9//TW6du0KqVQKFxcXbNy4ES1atMDq1asBAGFhYbhx4wY+//xzs8ZmDc6m5qFcoUKAuwPaBblXGi+ivGyCo0iA20sH2OzaltC5c2fm/9euXcPx48fh4uKit19aWhpatWqFu3fvYuHChbh48SLy8vKgVmt+ITMyMixKdDf3POyyRUKhEF26dEFSUpLR8yYlJSE6OlorRatnz56QSqV49OgRgoOD9e6bTbt27Zj/+/r6wsnJiTFc9Lb//vsPAJCamoqysjI9oySXy9GxY0etbV26dDE6Zl1SUlIwZcoUrW1RUVGM+qVp27atXq5hYmIiFi9ejGvXruHp06da72tkZCSSkpLQrVs3rWNsXRrqcsZTAMALrRpDItD8uCjVRHnZBB6PZ9HUzZY4Ozsz/5dKpRgyZIjBX2F/f38AwJAhQ9C0aVN89913CAgIgFqtRps2bSx2GHN1nprCvm82IpGI+T+Px9N6Tm+jDQKthP7++280adJEaz/dYpbGrlcbdM9ZWlqKAQMGYMCAAdixYwcaN26MjIwMDBgwwK4d+smZJQCACH83FJUrAFgvzqt+fCsJFtOpUyfs2bMHzZo1g1Co/2fOz89HSkoKvvvuOzz//PMAUK3zmVYG7FJBlpznwoULTFcopVKJxMRETJs2zej1IiIisGfPHlAUxaivs2fPwtXVtdqKu5YSGRkJiUSCjIwMrSlibQkLC8OlS5fw1ltvMdsuXbpU7XHJycnIz8/HihUrEBQUBABISEjQ2iciIgL79+/X2nbhwgUORl1zkrM0xivczxWJlSqMxHkRLGLq1KkoKCjA6NGjcenSJaSlpeHff//F+PHjoVKp4OnpiUaNGmHz5s1ITU3FsWPH9Loy6dK0aVPweDwcOHAAubm5kEqlFp1nw4YN+PPPP5GcnIypU6fi6dOnWn4dXaZMmYKHDx/i/fffR3JyMvbt24dFixYhNjaW8/w8V1dXzJgxA9OnT8f27duRlpaGy5cvY926ddi+fXuNz/v+++/jhx9+wPbt23H37l18+umnuH79erXVSoKDgyEWi7Fu3Trcu3cP+/fvx7Jly7T2mTx5Mu7evYuZM2ciJSUFO3futGl8XHGFAo8LywEA4X5uEFX+jUioBMEiAgICcPbsWahUKvTv3x9t27bFRx99BA8PD/D5fPD5fOzevRuJiYlo06YNpk+fjpUrV5o8Z5MmTbBkyRLMmTMHvr6+mDZtmkXnWbFiBVasWIH27dvjzJkz2L9/P7y9vU1e7+DBg/jvv//Qvn17TJ48GRMnTsT8+fNr9d4YY9myZViwYAHi4uIQERGBgQMH4u+//zarPLkx3njjDcydOxczZsxAp06dkJ6ejnHjxlVbzaFx48bYtm0bfvvtN0RGRmLFihVYtWqV1j7BwcHYs2cP9u7di/bt22PTpk1aCzR1zZ1K1eXn5gB3JxGEAo2BVlgpSJUUI2RRUVGB9PR0hISEWLVUCOHZpl+/fvDz88NPP/1kk+tb63P+84UHmL/3JnqHNca28VH46cIDLNh7EwNa++LbMeYvcpBihASCHVBWVoZNmzZhwIABEAgE2LVrF44ePaoVz9VQSKlUXmF+mn6L4krlZa0gVTJtJBBqQevWreHi4mLwsWPHDvB4PBw8eBAvvPACOnfujL/++gt79uxBTEyMrYfOOclZxQA0znoAENI+LytNG4nyIhBqwcGDB6FQKAy+5uvrC0dHRxw9erSOR1X3UBTFWmnUTPWEjPIicV4Egt3RtGlTWw/BLsgsqkBJhRJCPg8tGmsCo0UCstpIIBDsHHrK2LyxM8RCjVkRVmZmk2KEBALBbtGdMgJVysta6UHEeBEIhFqju9IIsIwXUV4EAsFeoXMaw1nGiwlSJT4vAoFgj8iVaqTlahLbw/3Z08bK1UZSBppgLZo1a4a1a9cyz3k8Hvbu3Wuz8RDqF/fypFCqKbg6CBHgXhWxz8R5KUmoBKGOyMzMrLZCKIFAw54yshPOrZ3bSIwXQQ+6kqktoSgKKpXKYDkfgn2RbMBZD7Ad9sTnZRsoCpCX2uZhQc58SUkJ3njjDTg7O8Pf3x9r1qxB79698dFHH1l8y+xpI90c448//kCfPn3g5OSE9u3b4/z581rHnDlzBs8//zwcHR0RFBSEDz74AKWlpczr1TWTqGlzC4LtSamM8Qrz006itvZqI/lZqw5FGbA8wDbXnvcEEJtXtTM2NhZnz57F/v374evri4ULF+Ly5ctaXWpqwyeffIJVq1YhNDQUn3zyCUaPHo3U1FQIhUKkpaVh4MCB+PTTT7Flyxbk5uZi2rRpmDZtGrZu3Qqg+mYSNJY0tyDYB7TyitBRXkyQKikDTTBGSUkJtm/fjp07d+LFF18EAGzduhUBAdwZ3RkzZmDw4MEAgCVLlqB169ZITU1FeHg44uLi8MYbbzAqLzQ0FF9//TV69eqFjRs3wsHBodpmEjSWNLcg2J6iMgUyiyoAAK2MThuJ8rINIieNArLVtc3g3r17UCgUiIqKYra5u7sz/f+4gN3Mgq6Bn5OTg/DwcFy7dg3Xr1/Hjh07mH0oioJarUZ6ejoiIiKqbSZBY0lzC4LtScnWqK4mHo5wc9DuESBkhUqwS3lzBTFe1cHjmT11a8joNrMAoNW84t1338UHH3ygd1xwcLBFzSSs0dyCYD10y+CwEbFKdStUFMRCYrwIOjRv3hwikQiXLl1i2oEVFRXhzp07TMMLa9KpUyfcvn0bLVu2NPj6jRs3qm0mQaifGFtpBKqUF6DJbxRzvD5IVhsbAK6urhg7dixmzpyJ48eP49atW5g4cSL4fD7nUt0Qs2fPxrlz5zBt2jRcvXoVd+/exb59+5jOQOY0kyDUTwzlNNLQPi/AOpUliPFqIHz55ZeIjo7Gyy+/jJiYGPTs2RMRERF1Uou/Xbt2OHnyJO7cuYPnn38eHTt2xMKFC5kFA3OaSRDqHxRFMcYrwl+/1ryIrbysEOtFpo0NBFdXVy2HeWlpKZYsWYJ33nmn2mPv37+v9Zzdk6VZs2bQ7dHi4eGht61r1644fPiw0WuMHj0ao0ePNnqd3r17652TYN88eloOqUwJkYCHEG99XyWPx4OAz4NKTVlFeRHj1UC4cuUKkpOTERUVhaKiIixduhQAMHToUBuPjNBQof1dLRq7aE0R2QgZ40WUF8EEq1atQkpKCsRiMTp37ozTp08jKSkJgwYNMnoM3eaeQLAUOrLe0JSRRiTgQ6ZUW6WyBDFeDYSOHTsiMTFRb3t5eTmuXr1a9wMiNHhMrTTSWLMJBzFeDRxHR0ejIQwEQm0wx3hVNeEgq411AnEcExoyXHy+ZUoV0vM0ifcRfiamjXw6yp575UWMFws6irysrMzGIyEQrAf9+WZnTVhKao4UKjUFd0cRfN0kRvcTWrH9mU2njadOncLKlSuRmJiIzMxM/Pnnnxg2bJjR/U+cOIE+ffrobc/MzOSkBpVAIICHhwdTqsXJyalOgjwJhLqAoiiUlZUhJycHHh4eEAgENT4XXYAwTKcAoS5VdewbmMO+tLQU7du3x4QJE/Dqq6+afVxKSgrc3Kqkqo+PD2djoo0gu9YUgdCQ8PDwqPWPPZ2QrVsGRxc6v9EalSVsarwGDRpkchnfGD4+PvDw8DBrX5lMBplMxjwvLi42uT+Px4O/vz98fHyMtnEnEOorIpGoVoqLpspZb9zfBbBLQTewaWNN6dChA2QyGdq0aYPFixejZ8+eRveNi4vDkiVLLL6GQCDg5I9MIDREkjPp6qnVKC8r1vSqVw57f39/bNq0CXv27MGePXsQFBSE3r174/Lly0aPmTt3LoqKipjHw4cP63DEBELD42mpHDklmtlM9caLxHkBAMLCwrQK7PXo0QNpaWlYs2YNfvrpJ4PHSCQSSCTGV0MIBIJl0FPGIC9HuEhMmxC6/ZncCsarXikvQ0RFRSE1NdXWwyAQnhnoAoRhvqb9XQA7wv4ZnzYa4urVq0xZYgKBYH2qyuCYnjICLJ9XQ3PYS6VSLdWUnp6Oq1evwsvLC8HBwZg7dy4eP36MH3/8EQCwdu1ahISEoHXr1qioqMD333+PY8eOmSzFQiAQuMWctCAapoNQQwuVSEhI0Ao6jY2NBQCMHTsW27ZtQ2ZmJjIyMpjX5XI5Pv74Yzx+/BhOTk5o164djh49ajBwlUAgcI9aTeFOdlWH7OoQCa3XeNamxqu6AnTbtm3Tej5r1izMmjXLyqMiEAjGePi0DGVyFcRCPpo1qr5ZisiKyqve+7wIBELdQU8ZQ31cmLxFUzC5jSQxm0Ag2BJ2TqM5iMhqI4FAsAdSsiurp1aTFkQj5FvP50WMF4FAMBtLVhoBVjFCeysDLZVKma7JNOxqDwQCoeFQoVDhfmUBQnNWGgHrpgdZrLzS09MxePBgODs7w93dHZ6envD09ISHhwc8PT05HyCBQLAP7mZLoaYAL2cxGrual3JnV/W83nzzTVAUhS1btsDX15cU6yMQnhGq0oJMFyBkQ/u87KKS6rVr15CYmKiVIE0gEBo+lvq7ADtbbezatSspK0MgPINYktNIY804L4uV1/fff4/Jkyfj8ePHaNOmjV4R/3bt2nE2OAKBYD+YWz2VjTWLEVpsvHJzc5GWlobx48cz23g8HiiKAo/Hg0ql4nSABALB9uRJZciTysDjAa18Xcw+jpk22oPymjBhAjp27Ihdu3YRhz2B8IxATxmbejnBSWy+2WCKESrtQHk9ePAA+/fvJ12YCYRniCQza9brIrSi8rLYYd+3b19cu3aN84EQCAT7JaUG/i7AuquNFiuvIUOGYPr06bhx4wbatm2r57B/5ZVXOBscgUCwD8zt06iLXcV5TZ48GQCwdOlSvdeIw55AaHio1BRLeVlmvKrKQNuB8tLNZSQQCA2bB/mlkCnVcBDx0dSMAoRs7Ca3UaFQQCgU4ubNm5wPhEAg2Ce06mrl6woB37LoAjpIVW7rCHuRSITg4GAyNSQQniGS6Cmjr2VTRqCqDLTNlRcAfPLJJ5g3bx4KCgo4HwyBQLA/UrJqFiYBVCkvu/B5rV+/HqmpqQgICEDTpk3h7Kw9B758+TJngyMQCLanKqfR8lp9VSVx7GC1cdiwYZwPgkAg2CdlciUeFJQBqJnyEttTbuOiRYs4HwSBQLBP7mRLQVGAt4sE3i7mFSBkY1fKiyYxMRFJSUkAgNatW6Njx46cDYpAINgHtL/L3LLPuthVkGpOTg5GjRqFEydOwMPDAwBQWFiIPn36YPfu3WjcuDHXYyQQCDYiycJWZ7pUVZWwg2KE77//PkpKSnDr1i0UFBSgoKAAN2/eRHFxMT744APOB0ggEGwH7ayvsfKyJ5/XoUOHcPToUURERDDbIiMjsWHDBvTv35/TwREIBNtBURRTtz7cwoRsGjrOyy46ZqvVar1kbEATwEpShwiEhkNuiQxPyxTg84BQCwoQsqFzGylKkyPJJTUqifPhhx/iyZMnzLbHjx9j+vTpePHFFzkdHIFAsB102edm3s5wEAlqdA56tRHg3mlvsfFav349iouL0axZM7Ro0QItWrRASEgIiouLsW7dOk4HRyAQbEdt/V1AlfICuDdeFvu8goKCcPnyZRw9ehTJyckAgIiICMTExHA6MAKBYFuSmD6NNfN3AYCQlcjNtdO+RnFePB4P/fr1Q79+/TgdDIFAsB8Y5WVBqzNd2FUouHba18h4xcfHIz4+Hjk5OXpO+i1btnAyMAKBYDuUKjXu5kgB1G7ayOPxIBLwoFBRtldeS5YswdKlS9GlSxf4+/uT7kEEQgPkfn4p5Eo1nMQCBHk61epcIgEfCpXK9sZr06ZN2LZtG8aMGcPpQAgEgv2QzCpAyLewAKEutN9LbuvVRrlcjh49enA6CAKBYF9wsdJIU1XH3sbG6+2338bOnTs5HQSBQLAvapvTyEZopfZnFk8bKyoqsHnzZhw9ehTt2rXTi7b/8ssvORscgUCwDSnZtUsLYmOtyhIWG6/r16+jQ4cOAKDXiIM47wmE+o9UpsTDgnIAXE0brVNZwmLjdfz4cU4HQCAQ7Ava3+XjKoGns7jW56N9XjZPD3qWoSgKt54UQa4kCeiEhktVcGrtp4xAVVkcBcc+L2K8LOCfm1kY/PUZrD6cYuuhEAhWI7mW1VN1sVbjWWK8LCCtMuL4QX6ZjUdCIFiP5Fr0aTQEHedFlJcNKZEpAQAVStJ0l9AwoSiKk5xGNkJ7ifM6deoUlEql3nalUolTp05ZfK4hQ4YgICAAPB4Pe/furfaYEydOoFOnTpBIJGjZsiW2bdtm0TVrQ0mFAgBQLifGi9AwySquQFG5AgI+Dy19alaAUBeRleK8LDZeffr0Mdgtu6ioCH369LHoXKWlpWjfvj02bNhg1v7p6ekYPHgw+vTpg6tXr+Kjjz7C22+/jX///dei69aU4gpaeRGHPaFhQk8ZQ7ydIRHWrAChLvRqI9fpQRaHSlAUZTCeKz8/X697dnUMGjQIgwYNMnv/TZs2ISQkBKtXrwagqSN25swZrFmzBgMGDDB4jEwmg0wmY54XFxdbNEY2JZXGS6YgyovQMOEyLYiGDlK1WYT9q6++CkATiDpu3DhIJFUNKFUqFa5fv271nMfz58/rFT0cMGAAPvroI6PHxMXFYcmSJZxcX1o5bawgxovQQEnO5HalEWAHqdpIebm7uwPQKC9XV1c4Ojoyr4nFYnTv3h2TJk3idHC6ZGVlwdfXV2ubr68viouLUV5erjUmmrlz5yI2NpZ5XlxcjKCgoBpdn1Ze5cR4EeqQ82n5cBQL0CHIw+rXYlYaOUgLorFWnJfZxmvr1q0AgGbNmmHGjBkWTxFthUQi0VKJtYE2XhUK4vMi1A2lMiXe2nIRDkIBLi/sp1UTnmsUKjXScmtfgFAXuv2ZzeO8Fi1aZDPD5efnh+zsbK1t2dnZcHNzM6i6uKaETBsJdUypTAmFikKJTGn1+MJ7uaVQqCi4SIQI9OTu+yS0Um6jxcYrOzsbY8aMQUBAAIRCIQQCgdbDmkRHRyM+Pl5r25EjRxAdHW3V6wKannOllSESMqUaaiu0LycQdGF/4WlVZC3oyPpWvi6cFllgVhs5XqW3eLVx3LhxyMjIwIIFC2pdBloqlSI1NZV5np6ejqtXr8LLywvBwcGYO3cuHj9+jB9//BEAMHnyZKxfvx6zZs3ChAkTcOzYMfz666/4+++/azwGs8daoR3bJlOq4Si2rrEmENiNWlNzpBjQ2nrX4jqnkcZaxQgtNl5nzpzB6dOnmbI4tSEhIUErNox2rI8dOxbbtm1DZmYmMjIymNdDQkLw999/Y/r06fjqq68QGBiI77//3miYBJcUV04ZaSoUKmK8CFanbpUX92ESQFV6kM2LEQYFBYGiuBlE7969TZ7LUPR87969ceXKFU6ubwklOsqLpAgR6gIVS63QubXWIoXjnEYau6kqsXbtWsyZMwf379/ndCD2jlSmY7zIiiOhDtBWXqWcCQddiisUeFxIFyDketpo4zgvmpEjR6KsrAwtWrSAk5OTXhloQ6lDDYESnWkjyW8k1AXsqZZUpkROiQy+bg6cX4dWXf7uDnB3ElWzt2WIbB3nRbN27VpOB1BfINNGgi1Q6axqp+ZIrWK8qoJTuZ0yAlWhEjavYT927FhOB1Bf0FVeJNaLUBeodKaJablS9Gzpzfl1UrK4a7ihi4jJbbSDYoRpaWmYP38+Ro8ejZycHADAP//8g1u3bnE6OHuiWDdUgvi8CHWAIeVlDZIzrbPSCLCUl62DVE+ePIm2bdvi4sWL+OOPPyCVat7Ma9euYdGiRZwOzp7QnTaS/EZCXaAbXmCNcAmKopCSbc1po50orzlz5uDTTz/FkSNHIBZXdRbp27cvLly4wOng7AkybSTYAl3llZZTyvk1nhRVoKRCCSGfhxaNuSlAyEZkpTgvi43XjRs3MHz4cL3tPj4+yMvL42RQ9oiew55MGwl1AB1eEOzlBEBT6VT3h7S20GVwWjR2gVjIfeI3s9po62mjh4cHMjMz9bZfuXIFTZo04WRQ9ghRXgRbQCsvT2cxGrtqqqPcy+VWfVlzpRFg+bw4zm202HiNGjUKs2fPRlZWFng8HtRqNc6ePYsZM2bgrbfe4nRw9gQdpOoq0SzQEp8XoS6gg1QFPKBFY001F66d9lw33NDFWrmNFhuv5cuXIzw8HEFBQZBKpYiMjMQLL7yAHj16YP78+ZwOzp6gp430rx8pBU2oC2jlJeTzmYYYXDvtue7TqIu1Wp9ZHOclFovx3XffYcGCBbh58yakUik6duyI0NBQTgdmb9DGy9tVgnt5paQJB6FOYJQXy5nOpfGSK9XMNJTL6qls7KaqBE1wcDCCg4O5HItdQ1eVoJUX8XkRakN2cQVSskrwfKi3ybJSdGK2UFDViozLaWNarhRKNQVXByEC3LmP3AdYxQhtobxiY2OxbNkyODs7a9WDN8SXX37JycDsCbWaYnxePsR4ETig23JNUc3NYzqjf2s/o/vRX3i28nqQXwaFSs1JSWj2lJHLAoRsbNr67MqVK1AoFMz/jWGtm7c1pXIl6CwNWnmVk1AJAgecTc0zabzUFO3z4sHf3QFOYgHK5Co8yC/jpCmstVcaAes1nTXLeB0/ftzg/58VaH+XSMCDu6Mm454oLwIXVPeDz/Z58Xga9XXjcRHScqXcGC8mLcg6/i6A3bfRDnIbnzVo4+XqIIKjSFM9lRgvQl3AXm0EuA+XsEaTWV2sldtolvKiG86awx9//FHjwdgrUplmyuzqIIRDpfEiidmEuoDt8wLAabhEYZkcWcUVAIBWVp022lB5ubu7Mw83NzfEx8cjISGBeT0xMRHx8fFMY9qGBl1RwkUihINI85aRIFVCXVClvDTGqypcovZR9rS/q4mHI9wcuC1AyMamNezphrMAMHv2bIwYMQKbNm1iWp2pVCpMmTIFbm7WmzfbkqppY5XyItNGAhdUt8bF9nkBQAtaeeVIQVFUrRbJ6mLKCFhvtdFin9eWLVswY8YMrR6NAoEAsbGx2LJlC6eDsxfovEZXB1GV8SKVVAkcwINp48OO8wKApo2cIODzmJLQtSHZymlBNFVBqjZOzFYqlUhOTtbbnpycDDXHEbT2gpbyEtLKy7b3WlKhwMe/XsOJlBybjoNgOZY0LKa/8PxKhSURCpgKE7V12tMxXtaKrKehDa9KTXHaQMTiCPvx48dj4sSJSEtLQ1RUFADg4sWLWLFiBcaPH8/ZwOwJWnm5OYgYn1eFjRtwnLqThz2XH+F+fil6h/nYdCwE8yksk2PMD/+Zvb+uzwvQ+L3S80prVRJaraZwp66mjfwqjaRQURALuYkHtdh4rVq1Cn5+fli9ejVTGsff3x8zZ87Exx9/zMmg7A228qIbzdp62lhUrjGomZXtqgj1g2UHknDjcRHz3HyfV5UBaOHjjKNJtevj+LiwHKVyFcQCPkK8nWt8HnOglRegyW8UcxShZbHx4vP5mDVrFmbNmoXiYo3sbKiOehpD00aFioJKTTGO1LqGDt/IKZFBrabAt9E4CJahG+JQ3V+NUV4CbeUFAKkWhEuUyZVwEld93ZPoAoQ+LpykGZmCPXYuK0vUatRubm4N3nABhh32gG1XHKWVBlWpppBfKrfZOAiWYelXV6Wz2giwYr3MLAn984UHiFz4L+b+cQMypQpXMp7ihzPpAKw/ZQR0p43c+YprVFXi999/x6+//oqMjAzI5dpfnMuXL3MyMHuCrbwkrDK55QoVnCU1LsxRuzGxOnhnF1cwOZcE+0bXYV3dtNGYzwuoKgntaiJGi6IoxlDt+i8D/97KQgHrx65PuPX9pXw+DwI+Dyo1xWmsl8XK6+uvv8b48ePh6+uLK1euICoqCo0aNcK9e/cwaNAgzgZmT5SwglT5fB5jwOxBeQEa40Wwb0oqFFCrKVi62EbXwGIrL3dHkdkloS9nPEV6XikcRHy4O4pQUCqHSMDDa50C8c+Hz+OV9gGWDaiGVBUktKHy+uabb7B582aMHj0a27Ztw6xZs9C8eXMsXLgQBQUFnA3MniiRVU0bAcBBJIBMqbZpuESpvMp4ZRHjZdc8LCjD818cR7cQL6ZKhLkYUl6AJscxt0SGtFwp2gd5GD3+98RHAIDBbQMQ278VTqTkoF+EL3ys0HXbFCIBHzKlmtNYL4uVV0ZGBnr06AEAcHR0REmJZrl1zJgx2LVrF2cDsydo5eXmoLH1TLiEDZUXu5tRdhExXvbMnssaA3IxvUBPeVVbVUKlv9oIsJz2JlYcKxQqHLimiQj4X+dANPFwxBvdmta54QKAQH4e3hH8xWl+o8XGy8/Pj1FYwcHBTK/G9PR0TgPQ7AWKorSqSgCwixQhqZbPq3aR1oS6Q/cbYvZqo47yMidB+99bWSiRKRHo6YhuIV6WDpUbKApI2IrfqY8xT7QLDqkHOTu1xcarb9++2L9/PwBNwOr06dPRr18/jBw50mA/x3pN0l8oL85jPkCulcqrqiyO7aaNbJ8XmTbWHyz9gdfNbaQxR3nRU8bXOgXaJpSmMAP4aRhw4CO4oByX1K0gdeeu14XFPq/NmzczaUBTp05Fo0aNcO7cObzyyit49913ORuYzXlyFfh1LCROjfAifwxOoAucKgNUJXanvIjxsmfY+Yt6tsvc1UaBYeVlrCT048JynEnVNIF+rVNgDUZdCygKSNwGHF4AyEsAoQPW8V7HmpK++M21GWeXsch4KZVKLF++HBMmTEBgoOYNGTVqFEaNGsXZgOwHCvBqDkH+XfwgXo0DeB688u6Akxcc6NVGG0bZk9XG+omuw143MVupUqNcoWJcFIZWGwHAz62qJHRGQRmjxGh++S8DFAVEN2+E4EZOXN+GcQozgP3vA/dOaJ4HdQeGfYM/tz2EuqTUdkGqQqEQX3zxBZRKZfU713cCOgKTTyO77btQUTy8jNPAN92BlH8Yn1e5jfIb1WoKUtZq49MyRY1U4I1HRczUgmA92D756r66U3ZcRo8Vx/AgXxMCwQSp6jj2+XwemhupqqpUqfFLwkMAwOvd6qjDV6VvC9/00BguoQMwYDkw/iDQqAWCPJ0Q4u3MaTS/xdPGF198ESdPnkSzZs04G4TdInJEStuZmJwQgK8dv0OQ9BGwaxTec4nBVYywWe/GMoWKmX7QwX85xTKLf2GHrD8DAPB3d6hxgi/BMgyFSuSWyCDg8+DlLEbCg6coqVDil0sPMWtguFGfFwC0bOyCm4+L9Zz28ck5yC6WoZGzGANMNPfgDCNqC41aMLtsnxDF+WUtNl6DBg3CnDlzcOPGDXTu3BnOztpJna+88gpng7MHSiqUuEKFYk7jDdjRIh44vx7dpUdxRJKAq08WA6j7Shr0lFHA56GJhyMyCsqQXVJR4+nB3ewSYrzqCh3bJVOq0PWzowCA1M8GobBME/3+55XHmNE/zKjPCzDutN95MQMA8L8ugRALrZi3aMC3hRcXAt0mA3xBtYfXFouN15QpUwAY7s/I4/GgUjWsIn10XqODozPQfxkQ8Qpyfp4AH1kG+l//CMB5YOAKwKnulqJpZ72LRAg/NwdkFJQhy0Csl0Klhkyphks1KUwNtWWdPaKrvHJYYS4FpXLQMZyZRRU4fy+flduob4Toqqr386qi7B8WlOHU3VwAwOtRVpwymqG2rI3FZlmtVht9NDTDBWjnNQIAgrpic8R2bFK+DDX4wPVfGF9YXcE2Xr6VXY4NOe0HrD2FNov+ZX7NjUFsl3Vhv726k0Z2ugxd5ohmz+VHzLRRN84LALP6zS6vvKvSUf98qDeaNrJCqZtqfFt1CWl9Vg3sihI0QokTVihfx7bwb4FGoYA0G9g1CvjjHaDM+ilS0gq28tLkuBkyXnTe27m0fJPnI7ar7tBVXipWuszTMs1njfZvHbqZheJy7W2mUKjU+DVBswBjFdXFituCvESjtt47B0RPrZNpoi5mTxvLy8sRHx+Pl19+GQAwd+5cyGRVklcgEGDZsmVwcKj71ANrUqyrvFCVHpQmiQAmnwaOLwfOr9eosHsngCFfAWHWS1Knf60lIj58K1M9skxE2XOZDEuoHbr+enauH13tIdLfDVKZEul5pUydeUPKS5cjt7ORJ5WhsasEMZG+3A46cWulb0ta574tY5itvLZv345vv/2Web5+/XqcO3cOV65cwZUrV/Dzzz9j48aNVhmkLdFNDQLY6UFqQOSo8YVNOFxnKoxd15w2XqbyG6stQ0LmjVZFK1RC50/BVmJPK6f3ns5ivNqxidZ+5igv2lE/oksgdyEJjNqarjFcNlZbbMy+wx07duCdd97R2rZz504cP34cx48fx8qVK/Hrr79yPkBbUzVtZCkvQyVxgrpqVFiPDwAeyxeWzF0uFw27QJ0f7fMqMWG8qmmMQkxX3aGbHsT+YaGVl6eTCMN0jJfQgMOezf28UpxJzQOPB4zqysGUkaKAhC3AN9GVvi1HYECcTXxbxjDbeKWmpqJt27bMcwcHB/BZb2hUVBRu377N7ejsANo5zjZeTB173cBQQyps92jOVRjbePm6Vk4biyqM5s3RUc3lchWW/HULF+9p+8B0hVd8UjaGbTiLexx0ZSZor+bq/oXYPi96YcXTSYwgLyd0b161gl2d8tr1n0Z19WrVGEFetYyoN6i2zgLRU2yuttiYbbwKCwu1fFy5ublagapqtVrrdUvYsGEDmjVrBgcHB3Tr1g3//We8u8q2bdvA4/G0Htb0s1WVwzEwbTSWHmRlFaaiqqKufSod9jKlWm+1ioYuQ7LpZBq2nr2PkZsvmDz/xO0JuPqwENN/ucrJeBsyUpkS7/yYgP3Xnpi1v67DXsFSxQWlmr+fp5MYAPAqKyfRUJwXjVypxm+JHDjq64HaYmO28QoMDMTNmzeNvn79+nUm39ESfvnlF8TGxmLRokW4fPky2rdvjwEDBiAnx3g/Qjc3N2RmZjKPBw8eWHxdc6GnjS4s5SUxp3cjW4V5t+JUhbEbkTqIBPB00hhWY6VxaB9ZRkGZRdehV78Ixtl0Ig2Hb2fjg11XzNpfVxwbUl5ezpq/50tt/ZnFIYmJYNM72VIUlMrh5+aAvjUt61xP1BYbs43XSy+9hIULF6KiQt+3Ul5ejiVLlmDw4MEWD+DLL7/EpEmTMH78eERGRmLTpk1wcnIy2X2bx+PBz8+Pefj6Gl9ZkclkKC4u1npYgl6cF6qmjWblNgZ1Bd49DfT8kDMVRi8e0o1Iq1YcDfu96Gmj2IgT11jXZsridhHPHgXVxNDpoltIlG286HN5VCovF4kQK//XHm8/F4I2Ae7VnntE1yAILXXU1zO1xcbsO503bx4KCgoQFhaGlStXYt++fdi3bx+++OILhIWF4enTp5g3b55FF5fL5UhMTERMTEzVgPh8xMTE4Pz580aPk0qlaNq0KYKCgjB06FDcunXL6L5xcXFwd3dnHkFBQRaN0eBqo6VVJUQOQL+lnKkwRnnxtY2XsRVHetooMtLsk3bJKFRqfPzrNWZ7A6wtyTmWl8kyEedVWuXzohnSPgDzX46sth4XnweM6mrZZ7s+qi02ZhsvX19fnDt3DhEREZgzZw6GDx+O4cOHY+7cuYiMjMSZM2dMKiBD5OXlQaVS6R3n6+uLrKwsg8eEhYVhy5Yt2LdvH37++Weo1Wr06NEDjx4Zro4wd+5cFBUVMY+HDx+aPb4KhYqJXtaO89L8YWWWFiPkSIUxyqvyA+1HGy+W8mI77+lpo7Hlc/prsSfxEVOyWHMOi4b1TKJb7aE6dJWXoTgvT2fj3YCM0SfMBwEejubtXI/VFhuLchtDQkJw6NAhFBQUIDU1FQDQsmVLeHnVXV5fdHQ0oqOjmec9evRAREQEvv32Wyxbtkxvf4lEAomkZm3BaNXF4wEuYn3jVaNihLQKCx8C7JsC5N3RqLB2I83OkdRXXpr7Y08b2V8SOlSCnaRraGUyt0TbZ/bUwinRs4g5eaHacV7GlRcdEM1WXubyRnczHfWFGcC+aUD6Sc1zG+QkckWNItm8vLwQFRWFqKioWhkub29vCAQCZGdna23Pzs6Gn595pTxEIhE6duzIGFMuYZz1YqGWbKfLQJfXppJqLVQY/YGnx2Qov5G9qqU04POSscr50F8u3c4uZXIVtp+7b8FNPXtY2jFdX3npq3dzjRc9G2ji4Yherapx1LPVVvrJequ22Ng0t1EsFqNz586Ij49ntqnVasTHx2upK1OoVCrcuHED/v7+nI/PkLMe0O4eVKumIzX0hSl1CtRVTRurlBP7F5122LMDHctYiw20w16l+80CsGi/cX8iwXLjpae8dLIfXB2EzIJQdXQK9sSnw9rgh3FdTI+jMAP4cWi99W0Zw+aJ2bGxsfjuu++wfft2JCUl4b333kNpaSnGj9fUyXrrrbcwd+5cZv+lS5fi8OHDuHfvHi5fvow333wTDx48wNtvv8352KoCVLV9EHQNezUFbsraWqjCaFWl67DP0vJ5Ve1P/7qz1VgZqxIrjCgvQvVYmlml+w4/0VlkmT0w3IJr8/Bm96YI93MzcrGGp7bY2KZXPYuRI0ciNzcXCxcuRFZWFjp06IBDhw4xTvyMjAytSP6nT59i0qRJyMrKgqenJzp37oxz584hMjKS87EZSg0CqpQXoFlx5KTgmwW+MF2HPW288qQyphkD21DRBpY9RSkzEOZhqiFqmVwJAZ/HxLgRNFjqsDcl1LeO74o+YTWM09JF17cVHA0M3dAgjBaNzY0XAEybNg3Tpk0z+NqJEye0nq9ZswZr1qypg1FVOVBddIyXWMAHn6dRXhVylVb0fa2hVdiJ5cC5dVWVKl5eC4S/BEDfYd/IWQwhnwelmkKeVAZ/d0cdn5dmf7ZKLGV1H+Ix+xn+ZpXLVYhc+C+8XSRImB9jcJ9nFb45Dnut7kGG3+Puzb24MVx6FSAcgZhFQNS7QDX5kfWNhnU3HGMoxgvQyHUHa/ZuNOYL2zMJKCvQU158Pg8+rpUrjpXTEO3VRs0TdmmcUhnL58WjfV6G7yUpSxPYmyclzW11sTTOy9jMnJMqELq+reBojW+r+3sNznABxHiZxNi0ETAjv5ELdH1hN34FNnRDs7zjALRrPOmuOFJa00aNUWIrK/reAJbyMvLNIjmOxtFKujaiqrS7Bxnep1bGi6KASz9o+7YGrgDGNQzfljHsYtporxhbbQSMlMWxBgZ8YUOTZ0It6ok7qgXMbrorjlrKy4DPq5CVxH0npwRjt/xnVFk9yLcsJ/JZgr3Kp1JTJhOoAePKy9JVS4anD4D904D0U5rnDdC3ZQxivExAqxNDPi0HsRWnjYZg+cLUZ9dhuOAspLdfByLXA+GD9VYctXxelUZLrqzalscKSP325D2zh0FRFGnYwYJtdJRqCuz1jNQcKS5nPNV20hudNlq6bFm5knhkYYP3bRmDGC8TmFZeHASqWkqlCtte0BbP31qIlsonwO7XgbYjEOz2HoCq/Ea28aIDUtnKq6b+K4rSTIMoikKuVAYf14ZV9ttS2HZcoVIz7gQAiPlSs9IX7ufKbJMbKcldXbFBLZ5htcXm2TDRNcRQIUIadqBqXZPhFInB8uW44D+G8YW9nvA/9OMnMBVV2b/2cqW+zytPWrPUH/oMvyU8QtRn8dh8Kq1G56kPfPb3bYzafJ5ZrTUEO1TC2GotXYfeFObUqGd8Wxt7aAzXM+LbMgYxXiZgmm9IDEwba5PfWEvUagoyiHE25H1g4hHAuxUcZPn4TvwlxmctB8oKtJSXnAmVqPoS5tZQedHn/eaEJh1r+cHkmt6G3fPd6XRcuFfA9EGsDkU15bZNUZ2vDE8fAD++Avwd+0ysJJrDs3nXZmJqtdGxppUlOIDdgAOBXYB3T+NpxylQUTzEKE8CG7pBnHqI2Z9WXmzjVZtpIwCE+lZNhQylFTUkTNkklYEc0ppgqKksAKK2TECMlwlKjASpAlXKq059XpXopgdB5ADRwGV4Tb4EqeoAoDQHjf4ahzWiDXCHlOXzMuywtwR6qV/dwA0WG1PChjKwqlsTDE4bidoyCXkHTGBqtVFiQ58X/SVhV7pwkQiRKg7HYPlyPO04BRSPj+GCszgqmYmoCk1hR7byoqfElkJ/WdlVKRo6pqLo2UacPW20tFem1rSRqC2zIMbLCAqVmgmDMBmkaoNpo0pXeVXi6yaBDGIktf4Yma/tR6o6AI15RVgujwP2TIJEYVkJbEPQxktuwHjllFTg+qPCWl/DHmBPhU3FYBmbNhrKHTUFE6Sqp7Z6ELVlBPJuGKGEpUxcJPrGy6nSeP2a8BCHbmbWrjSOhdC/9rpfKrqHY1ZxBSp8O2KwfDk2KYdABc2K5Nq8d9CPn1Cra9PTRpkBZRH1WTxeWX8WNx8X1eoa9gBbOZlKvmbPntnHaFXtMAMhjwIufV8ZJc9WW38TtWUEYryMQE8ZncQCg00NhnVsAj83BzwuLMfkny/jje8vIjmr9srGHJRGjBfdwzG7WAY1BcggxgrlaLxJLQO8W8GTKsR34i+xVrQeHqh++d4QppQXzQWdvpD1EbZ/0FT9ePa0kX0MO3e0OgJ5uRiV/D7w98eAopSoLTMh74wRTAWoAkCbJu44NqMX3u/bEmIhH+fS8vHSV6exYO9NppGCtaAd9nrGi5XfyFaCicoWwLun8Yv4VagoHoYJzuGIZFaNVBh9bbk1czrtAHZsl6lpo6HqHYB5naV4UONNwRH8K56F4KIEorYshBgvIziJBRjaIQAxEcabijiJhfi4fxjiY3vhpbZ+UFPATxceoPeqE9h+7r7J4MbawDjsdaYzdH5jVlGFli9GrlKDEkqwSTwWr8mXII3S+MJqosLUFHAnu0TL1xfz5Uks+athVVxlR8KbisDSnjaylFc108ZAXi5+FsXhU9FWOPNkeOLekagtCyHvkhGaN3bBV6M64rPhbavdN8jLCd+80Rm7JnVHuJ8risoVWLT/FvqvPYW1R+/g5uMiTn1ieqESldCNOLJLKvRiky7dfwqFSo2rVEu857xW4wurgQpbc+QO+q85hceF5cy29LxSbD17v+Y3ZIewne9qCohPyka/L0/q+fPYyovt5DemvNhqq6fgFsopMZYoxuBAp++I2rIQYrw4JLpFIxx4/zl8OqwNPJ1EuJdbirVH7+LldWfQY8UxfPLnDfyXXrtu2QArSFXHeAV7OQMA7mZL9XLo3t5+iflCerq7YYVyNF6TL8FddROLVNi2BtaQo0yuxP28Ur3txaySQWqKwsTtCbibI8XE7Ze09jMWKmFIeemqrYvqcAyUr8BW1SAIBSTN2FKI8eIYoYCPN7s3xYmZffDFa+3QP9IXjiIBMosqsONiBkZ8ex4/nb9fq2vQv/C6yivczxWeTiJIZUpcyXiq9VpxhRKySj8VvSp5lWqJl+WfYWMNVVhDIGb1SfRedUIrxKOwTI6Ba08zz9nqqrhc2ygZDZVgF3s0orZGyefjAaXpklVtehBBD2K8rIS7owgjugZh81tdcGVhP2wd1xUvt9N0OFqw7xa2nU2v8blVRlYb+XweerT0BgAcT9HPx3taplETEf5VDRtkEOPzGqqwhgDdAOOfm1VNjs+maq+Wsmf8unX+tSPs9UMlTKktivX1q3E9r2cYYrzqAAeRAH3CfbBudEdM7qXxayz+6za+P21+HS02xowXADxXabwupBkOV3CVCBHi7ay3nUsV9unfSVh24LbFx9kSU/mZbIOl67rUajGnFSqhqFZtsRERJ73FkHesDuHxeJg9MAxT+2gM2Kd/J2H81v/wzYlUXLyXb9byOsAyXgaCJ2njZaxuVBNPR3i7GG5qyqUK++FMzZWlLTBlvNiv6Sovg6EST+/j5auTq1VbbMi00XKIl7CO4fF4mNE/DAIeD18fS8XxlFxmiifk89A6wA0dgz3RuanmEeDhqHcOlZE4L0Cz8tmskRPuGyndHOjpBC9nickx0irsI+EevCM4gGGCc+jJv4V5iok4ou5i6S3XC0wZL7a9UukZr6r/K5UqTZT84YVoqihFOSXG58pR2K7qb9Ro0RgKhCaYhhgvG8Dj8RDbPwz9W/vhwr18XM54isQHT5FdLMO1R0W49qiIWdWL9HfDhjc6aU31TE0bAeC5UG/cz88w+FqQlyO8nKtvJ0+rsH9VXbFS9C1C+Y/xnfhL7FX1wGLFWBTCtdpz1CdM9aw0NW2kVxsDeTl44cJEoECzGnnPqT3GPx1rcIpoCLOKERK0IMbLhrRp4o42TdwBaMoqPy4sx+WMQlx+oDFmt54U4XZmMfqsOoEXw33QqVKN0T0XjaWtPNfSGz9fMGy8Aj2d4GYka8AQxlTYJ4oJOKzuavLY+lTvnp3aoztkU9V/KLUKbwqOYK5wJ5wLZJW15Bdj7b2ueFCQZfxAHYjxshxivOwEHo+HQE8nBHo64ZX2AQA0wZ//t+k88qQyxCfnID45R+sYYx/46BbeTFNcXdoFultsUAypsM3iNdWqMIoCZEoVDt7IxPOhjdHY1fR01Zaw47X01JUxVfb0Pibdn45Q0RUAQKZHR/iP+QFo1AKlyZcMH2METvo2PmOQd8yOCfF2xrk5fbF3ak8seDkSL7X1Y5rLAvrNcGncHUVoF+iht93XTYLOwZ41Ho+xFcn+fMNfVDVFYfXhFMT+eg0jN5+v8XXrAtM+L725YmUFiB4ILbuCckqMxYq3sK9DVZR8delBupBQCcshysvOEQv56BDkgQ5BHpj4XAgoisL5tHw8LixHu8oppyH6hPng6sNCrW2D2vgzU00vZzEKapBAbokKowAcvKGZOt3L1Y9iv/2kGKm5UkZp2hK2I97UtDGQl6Opt3VfE8Sa6tgOEwvH4QHlhxmsBV5zV45pyLTRcojyqmfweJpA1P/rEmSyVMvbz4cg7lXtvMzBlUGyAODhaFi1mYs5KoyitNutFVcokM+qnf/S16fxwa4rdVJCJz2vFOd1Yt/YmQ7VhUpURcnP1hguoSMw8HN8FbiGccprJ2ZbaLzItNFiyDvWQHGWCDE6KlhrG3vK6FpL4wUYjgvbLF7DxIWpKUrLKPT64jg6f3qUqZVGczdHWuuxVEefVScw+rsLSKlsQ/ZfegEW7KuqhGHKeDlIH2KHaDkTt1VVb2sy1KyvkDmJ2cYg00bLIcargXNqZh9Mj2mFS5/EaCk1dx3j1blp7X1h3yhf0VJhR/74QetLTKcn3XqiXbRRXIcBmtceFqJUpsSDfO1prCGnPK22+hwbih6C24xvi11vS2VBYrYpLO6YTSDGq6ET3MgJH8aE6q306RovQ8GwliCDGF8oR+FVlgobkjQTn1Ff6UXn6xZrtKhbdC2Ztec6Wi/6FxU6lWB1lVcgL4dRW0JVORMlv001UKveltqMxGxzqMv3oKFAHPbPKI10AlUDPBw4Oe+1ShX2ofAPvCv4y2Bc2KOn5VrHsFNjZEoVJEIBJ2MBgIX7bhpshvGoQDsDgTFeajWC03biX/HncObJUEZJkNL2Y4xKiDQYJW8oPUihUhtNzzIGSQ+yHGLun1Gm9W2p9bxJLZUXG0MqjO0Le/RU23DQqiPxQQHC5h/Cqn9TOBlHSYUCP55/gN8TH+mP0ZDyenof+PEVtLm6VCsn8W6z142m92hVUlVTKCyTo/vyeIvHSlYbLYcYr2cUbxcJZg8MZ54HuHNnvGiuGfGFNX50VCsoNLXSYb/0QBIAYP3xVJTJlVgXfxd3s2telqfERG9K7TLPavQu3g980wO4fxpKgSMWKcZilHw+Mihfk1VwdZXXljPpyK9BCAoJUrUc8o49w7Ab5tIFCrnGkAqblrsI1J6JjC9szdE7ALQd918cSsHqI3fQb82pGl/bVPsxuvsR7dsa+3SdpnNP0544G7MP21UDGLVlagbI9pUpVZTelNgcokK8EOjJ/Y9HQ4cYr2cYOkcSMNybkkt0VZjg1h4ckczEAFZcGFt9JD54aug0FmGq/ZhCoWTitnoIbqOCJwEGfQGMPYAyF+0QE1NJ2+yX5Co1ckpkRvc1xi/vdK83OaD2BDFezzDs5XxTX1CuYKswuWcoGvOK8a14Db4SrUfp0xytQE1TcVfmwjbObAJ5OXgvI1ar3tbMRhuBbu8a7NyjO21cc+QO8qUyqNUUrrHKRx+4nokzqXkWj5MYrppBjNczTEdW0Kq/FXxexrhGtcSZvn8wKmyo4BzK1nZGi7xjzD5sY1pUpsCM367hnIWGoUTHeLGj5MMrrqKMkjC+rScCfyNn0U9w/yr+Lhb/dRtx/ySZ9Kux8XGVYErvFriyoJ/W9h1vdzPvZgh6kFCJZ5jXOgWCz+OhS1NPOIoFSJgfA4oCjiVnY/aeG1a9dlKeHCuVo/CvqgtWVeZILiqLQwdRDyxSjIWacmH2XXEoGb8nPsLviY9wf8Xgas+dliuFj6sEUpZhCeTlYKVwM6IFmvLUyZK2eKd4AjIoTV9OpZlloGn+uvbErPsUC/iY9EIIZg4IN/h6z8rKtwTLIcbrGUbA5+F/nQOZ594umkDWwjKF3r5Rzbzg7iTCkdvZzDZ/dwdkVjawsJSE+5oWcLpxYUMF59CDfxPzcyfiDjRxYbv+M1ybzBC3nxTjpa81SdMf9G1ZqbaOYo5wFxO39blyFJJ9RyKjqJA5Tm3CeC35q+b1+G8s6c9p3BqhCmK8CHrkGnA6y1VqzBkUjg5BHogMcEMjZzFa+boio6AM/Y2sCD7X0tuoD4jtkKd9YWwV9q14DfapNCrM3KqtCpWaMVwA8Mfxc9gpqlJbF9XhmKl4FxmULzrorCCyY890Sz2bg0jAw/R+rXAnqwR7r1apMmK4rAcxXgQ98qT6xkuhUqNFYxdM7aMd3NrK1xWbx3SGUk2hT5gP/u/bc7j5WJO7+NnwNui18oTW/sFeTsgoKEOxAV+RURWmmIh/TVRtlSvVmPfnDSbp2pja+lHVjwl/kOr4w56WKTBk3Rm817sFpu28YvA6LhIhpDIluoV4Iau4Ap2CPfHnlccAgD+n9GSq4rKNF8F6EIc9QY/XKqeSUc28mG2GihvS9G/th5fa+sNRLMBPE7qhdYAb5g+OQNNGzjj00fNo5Vvlv/p+rOkGHrQKGy5fijvqJlorkh4oQblchd3/ZSC7uGq6uuVsOn5PfIQbj4sQyMvBTtFyLBNt04qSZ8dtAVWBsWxuPC7ClB2XDY5rcq8W2Du1Bz5/rS12TeqOkzP7YMHLkczr7FCT6sJO7LmibH2CR5kKH26AFBcXw93dHUVFRXBzc6v+gGeU9LxSBHg4ICO/DH9dz8Tbz4fAzUjlVkuJ/fUq/rj8uNr9JJAzKkzAo5BLuWGb14fYkBmBYC8nnJrVB8UVCrRbfNgstVUb0pa/pFe2pqhMgfZLDwMALs57Eb5umkDfbsuPIrtYo14NLTAkZRZj2YHb+Lh/WK2qeTRUzP2O2oXy2rBhA5o1awYHBwd069YN//33n8n9f/vtN4SHh8PBwQFt27bFwYMH62ikzw4h3s6QCAUI9XVFbL9WnBkuAIh7tS2WDW2NUzP74L3eLYzuZ0iFzXy6DF+J1qO4IBtSmRKvfnPObLWly5HpL6BZIydmTMZqak2PaWXwNTdHIZ4P9cbzod5a5bmdq1FeEf5u2DmpOzFctcTmyuuXX37BW2+9hU2bNqFbt25Yu3YtfvvtN6SkpMDHx0dv/3PnzuGFF15AXFwcXn75ZezcuROff/45Ll++jDZt2lR7PaK87A+VmoKAz8PZ1DxM3XnZ4GqnIRU2XzEBPrxCs9SWr5sEL0b44veER5Cr1Fj1f+21VloB4GxqHn659BCD2vjhvR2X4eEkwo8TotA6wN2iYoGHbmZh8s+JGNU1CCtea1ezN+UZxtzvqM2NV7du3dC1a1esX78eAKBWqxEUFIT3338fc+bM0dt/5MiRKC0txYEDB5ht3bt3R4cOHbBp0ya9/WUyGWSyKgd0cXExgoKCiPGyY4orFPj+dDpGdg3CH4mPsPrIHea1drw0rBJtQiu+9rSTvZLI5u8PnkPrgKpa/2VyJRyEApMltAHg5uMi+Ls7oJFLzfxTWUUV8HGVVHsdgj7mGi+brjbK5XIkJiZi7ty5zDY+n4+YmBicP2+428z58+cRGxurtW3AgAHYu3evwf3j4uKwZMkSzsZMsD5uDiLE9msFAHj/xVCM7hYMkYCPCdsuIfFBCwxhrUhqSlFr1FZLHzecGd8VgZ5ORs/tJDbvI9/GRHMTc7BWojuhCpsar7y8PKhUKvj6av9a+vr6Ijk52eAxWVlZBvfPyjLc4HPu3Llaxo5WXoT6Ax08u+e9HqhQqOAgEoCihiE99TYkji5Y3CQYS0h+4DNHg4/zkkgkkEjI0nRDwUGkCfrk8XhoHtraxqMh2BKbrjZ6e3tDIBAgOztba3t2djb8/PwMHuPn52fR/gQCoWFiU+MlFovRuXNnxMdXlc1Vq9WIj49HdHS0wWOio6O19geAI0eOGN2fQCA0TGw+bYyNjcXYsWPRpUsXREVFYe3atSgtLcX48eMBAG+99RaaNGmCuLg4AMCHH36IXr16YfXq1Rg8eDB2796NhIQEbN682Za3QSAQ6hibG6+RI0ciNzcXCxcuRFZWFjp06IBDhw4xTvmMjAzwWQXievTogZ07d2L+/PmYN28eQkNDsXfvXrNivAgEQsPB5nFedQ0JUiUQ7Jt6lR5EIBAIlkKMF4FAqJfY3OdV19Cz5OLiYhuPhEAgGIL+blbn0XrmjFdJiaZgHYmyJxDsm5KSEri7G0/TeuYc9mq1Gk+ePIGrq6tdtpyi05cePnxIFhSqgbxX5lOf3iuKolBSUoKAgACtSANdnjnlxefzERgYWP2ONsbNzc3uP2T2AnmvzKe+vFemFBcNcdgTCIR6CTFeBAKhXkKMl50hkUiwaNEiUgnDDMh7ZT4N8b165hz2BAKhYUCUF4FAqJcQ40UgEOolxHgRCIR6CTFeBAKhXkKMlw0oKCjAG2+8ATc3N3h4eGDixImQSvXbz7OpqKjA1KlT0ahRI7i4uOC1117TK4fN4/H0Hrt377bmrXAK182HKYrCwoUL4e/vD0dHR8TExODu3bvWvIU6g+v3aty4cXqfnYEDB1rzFmoPRahzBg4cSLVv3566cOECdfr0aaply5bU6NGjTR4zefJkKigoiIqPj6cSEhKo7t27Uz169NDaBwC1detWKjMzk3mUl5db81Y4Y/fu3ZRYLKa2bNlC3bp1i5o0aRLl4eFBZWdnG9z/7NmzlEAgoL744gvq9u3b1Pz58ymRSETduHGD2WfFihWUu7s7tXfvXuratWvUK6+8QoWEhNSb98QY1nivxo4dSw0cOFDrs1NQUFBXt1QjiPGqY27fvk0BoC5dusRs++effygej0c9fvzY4DGFhYWUSCSifvvtN2ZbUlISBYA6f/48sw0A9eeff1pt7NYkKiqKmjp1KvNcpVJRAQEBVFxcnMH9R4wYQQ0ePFhrW7du3ah3332XoiiKUqvVlJ+fH7Vy5Urm9cLCQkoikVC7du2ywh3UHVy/VxSlMV5Dhw61ynitBZk21jHnz5+Hh4cHunTpwmyLiYkBn8/HxYsXDR6TmJgIhUKBmJgYZlt4eDiCg4P1mvNOnToV3t7eiIqKwpYtW6otK2IP0M2H2fdnTvNh9v6ApvkwvX96ejqysrK09nF3d0e3bt2MnrM+YI33iubEiRPw8fFBWFgY3nvvPeTn53N/AxzyzCVm25qsrCz4+PhobRMKhfDy8jLaODcrKwtisRgeHh5a23Wb7S5duhR9+/aFk5MTDh8+jClTpkAqleKDDz7g/D64xBrNh+l/LWlQXB+wVqPmgQMH4tVXX0VISAjS0tIwb948DBo0COfPn4dAIOD+RjiAGC+OmDNnDj7//HOT+yQlJVl1DAsWLGD+37FjR5SWlmLlypV2b7wItmfUqFHM/9u2bYt27dqhRYsWOHHiBF588UUbjsw4ZNrIER9//DGSkpJMPpo3bw4/Pz/k5ORoHatUKlFQUGCy0a5cLkdhYaHW9uqa7Xbr1g2PHj2CTCar9f1ZE2s0H6b/bWgNiuuqUXPz5s3h7e2N1NTU2g/aShDjxRGNGzdGeHi4yYdYLEZ0dDQKCwuRmJjIHHvs2DGo1Wp069bN4Lk7d+4MkUik1Ww3JSUFGRkZJpvtXr16FZ6ennafjGuN5sMhISHw8/PT2qe4uBgXL16s1w2K66pR86NHj5Cfnw9/f39uBm4NbL1i8CwycOBAqmPHjtTFixepM2fOUKGhoVqhEo8ePaLCwsKoixcvMtsmT55MBQcHU8eOHaMSEhKo6OhoKjo6mnl9//791HfffUfduHGDunv3LvXNN99QTk5O1MKFC+v03mrK7t27KYlEQm3bto26ffs29c4771AeHh5UVlYWRVEUNWbMGGrOnDnM/mfPnqWEQiG1atUqKikpiVq0aJHBUAkPDw9q37591PXr16mhQ4c2mFAJLt+rkpISasaMGdT58+ep9PR06ujRo1SnTp2o0NBQqqKiwib3aA7EeNmA/Px8avTo0ZSLiwvl5uZGjR8/niopKWFeT09PpwBQx48fZ7aVl5dTU6ZMoTw9PSknJydq+PDhVGZmJvP6P//8Q3Xo0IFycXGhnJ2dqfbt21ObNm2iVCpVXd5arVi3bh0VHBxMicViKioqirpw4QLzWq9evaixY8dq7f/rr79SrVq1osRiMdW6dWvq77//1npdrVZTCxYsoHx9fSmJREK9+OKLVEpKSl3citXh8r0qKyuj+vfvTzVu3JgSiURU06ZNqUmTJjHG0F4hJXEIBEK9hPi8CARCvYQYLwKBUC8hxotAINRLiPEiEAj1EmK8CARCvYQYLwKBUC8hxotAINRLiPEiEAj1EmK8CARCvYQYL4JdYKiGer2oo06wGaSeF8FuGDhwILZu3aq1zVhFDIVCAZFIpLVNLpdDLBZbfN2aHkewLUR5EewGiUQCPz8/rYenpycATWekjRs34pVXXoGzszM+++wzLF68GB06dMD333+PkJAQODg4AAAyMjIwdOhQuLi4wM3NDSNGjNCqZ2XsOEL9ghgvQr1h8eLFGD58OG7cuIEJEyYAAFJTU7Fnzx788ccfuHr1KtRqNYYOHYqCggKcPHkSR44cwb179zBy5Eitc+keR6h/kGkjwW44cOAAXFxctLbNmzcP8+bNAwC8/vrrGD9+vNbrcrkcP/74Ixo3bgxAU2Tvxo0bSE9PR1BQEADgxx9/ROvWrXHp0iV07drV4HGE+gcxXgS7oU+fPti4caPWNi8vL+b/7I5LNE2bNtUyQElJSQgKCmIMFwBERkbCw8MDSUlJjPHSPY5Q/yDGi2A3ODs7o2XLliZfN2ebudci1G+Iz4vQoIiIiMDDhw/x8OFDZtvt27dRWFiIyMhIG46MwDVEeRHsBplMptdTUSgUwtvb2+xzxMTEoG3btnjjjTewdu1aKJVKTJkyBb169TI47STUX4jyItgNhw4dgr+/v9bjueees+gcPB4P+/btg6enJ1544QXExMSgefPm+OWXX6w0aoKtIDXsCQRCvYQoLwKBUC8hxotAINRLiPEiEAj1EmK8CARCvYQYLwKBUC8hxotAINRLiPEiEAj1EmK8CARCvYQYLwKBUC8hxotAINRLiPEiEAj1kv8HUo/RCX0rfMEAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.021118, + "end_time": "2024-02-29T15:12:10.790441", + "exception": false, + "start_time": "2024-02-29T15:12:10.769323", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 1597.887828, + "end_time": "2024-02-29T15:12:13.531575", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/realtabformer/2/mlu-eval.ipynb", + "output_path": "eval/insurance/realtabformer/2/mlu-eval.ipynb", + "parameters": { + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "param_index": 2, + "path": "eval/insurance/realtabformer/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "realtabformer" + }, + "start_time": "2024-02-29T14:45:35.643747", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/realtabformer/model.pt b/insurance/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..bcd4b89b9f4d77ae72be4f7e02672e5048e995f7 --- /dev/null +++ b/insurance/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee265eb1ca82fc7d4feb0f8c9f60cd92e13e3cd40eee0f092e0f2dbf8358cf0 +size 43503213 diff --git a/insurance/realtabformer/params.json b/insurance/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..dca8dde2042494a2f8a50fe87ab43db314626142 --- /dev/null +++ b/insurance/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.77, "head_final_mul": "identity", "gradient_penalty_mode": "ALL", "synth_data": 2, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "loss_balancer_beta": 0.75, "loss_balancer_r": 0.95, "fixed_role_model": "realtabformer", "mse_mag": true, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "relu6", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "rrelu", "head_activation_final": "softsign", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/tab_ddpm_concat/eval.csv b/insurance/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..666feff39958e68214839a36837794228037d475 --- /dev/null +++ b/insurance/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,3.508772319866949e-08,0.519334661818686,0.020264706268607824,0.5646026134490967,0.19168440997600555,0.9978110790252686,0.28464293479919434,1.485260327172e-05,0.8755850791931152,0.09591476619243622,0.7516146302223206,0.14235414564609528,0.04477740824222565,1.0563102960586548,1.440187692642212 diff --git a/insurance/tab_ddpm_concat/history.csv b/insurance/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..d00b787c34ff8198e87ee98b96e4995c49d91e17 --- /dev/null +++ b/insurance/tab_ddpm_concat/history.csv @@ -0,0 +1,16 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.03187035469454713,10.499650678998297,0.004555381951256443,0.10591729702427984,0.0,0.0,0.0,0.0,0.032666408171644436,320,40,52.7623987197876,1.31905996799469,0.16488249599933624,0.06082322220318019,0.012105482351034879,11.988290130137239,4.250274636738993e-05,0.06420383900403977,0.0,0.0,0.0,0.0,0.012274807115318254,80,10,9.881241083145142,0.9881241083145141,0.12351551353931427,0.01785293687134981 +1,0.01477175319159869,13.734025255048982,0.00038951243641720533,0.07673057662323117,0.0,0.0,0.0,0.0,0.015022071998100728,320,40,51.19189524650574,1.2797973811626435,0.15997467264533044,0.025308902200777084,0.018090815236791968,7.937299627149969,0.00019444738691163366,0.1157106451690197,0.0,0.0,0.0,0.0,0.01831991651561111,80,10,9.59651231765747,0.9596512317657471,0.11995640397071838,0.024659548606723546 +2,0.013891099636384751,15.052705611435432,0.0001513469102661702,0.07600115705281496,0.0,0.0,0.0,0.0,0.01410236083320342,320,40,51.0040807723999,1.2751020193099976,0.1593877524137497,0.02815434621879831,0.011826220527291298,16.741873883521475,2.0565837598951475e-05,0.06937152929604054,0.0,0.0,0.0,0.0,0.011988498945720493,80,10,10.122968435287476,1.0122968435287476,0.12653710544109345,0.011842810921370983 +3,0.01305226328986464,8.44533070571506,8.075568497831754e-05,0.0740173936355859,0.0,0.0,0.0,0.0,0.01324827965145232,320,40,52.304439544677734,1.3076109886169434,0.16345137357711792,0.030870095500722526,0.01087374986964278,4.948310850643611,9.443121865615466e-06,0.06728989202529193,0.0,0.0,0.0,0.0,0.011004218307789415,80,10,10.216580152511597,1.0216580152511596,0.12770725190639495,0.03216287791728974 +4,0.01341965578030795,9.745894026785027,0.00025464458190107566,0.07172424085438252,0.0,0.0,0.0,0.0,0.013624004492885432,320,40,52.158915519714355,1.303972887992859,0.16299661099910737,0.03181108951102942,0.013633261938230135,23.27936131209135,0.0001621007818215503,0.06217544944956899,0.0,0.0,0.0,0.0,0.013878957656561396,80,10,9.629613637924194,0.9629613637924195,0.12037017047405243,0.008153766090981663 +5,0.015133786734077148,15.675266945300246,0.0004782500350243524,0.07984540462493897,0.0,0.0,0.0,0.0,0.015376201126491651,320,40,51.18136239051819,1.2795340597629548,0.15994175747036934,0.03030351351480931,0.013037011679261922,60.89342765808105,0.00012217244490742019,0.06693714633584022,0.0,0.0,0.0,0.0,0.013210956694092602,80,10,9.720444917678833,0.9720444917678833,0.12150556147098542,0.002085553959477693 +6,0.014456333359703422,23.62044777232586,0.00021797772160834228,0.07543233875185251,0.0,0.0,0.0,0.0,0.014706396087422035,320,40,51.926756381988525,1.298168909549713,0.16227111369371414,0.011786041979212314,0.012131108457106165,7.805004606687544,2.6683602803245778e-05,0.07964657545089722,0.0,0.0,0.0,0.0,0.012273790544713847,80,10,9.750572919845581,0.9750572919845581,0.12188216149806977,0.02273060418665409 +7,0.012077562542981469,6.567233682959795,4.818316614121354e-05,0.07109622973948718,0.0,0.0,0.0,0.0,0.012245478003751486,320,40,51.910486936569214,1.2977621734142304,0.1622202716767788,0.03978729886002839,0.011527308775112034,13.779987310473008,8.208526573838525e-06,0.07020539008080959,0.0,0.0,0.0,0.0,0.011682888003997504,80,10,9.546999216079712,0.9546999216079712,0.1193374902009964,0.013699167082086206 +8,0.012478862586431206,5.30599477694621,0.00012620691976824582,0.07224454144015909,0.0,0.0,0.0,0.0,0.012667084962595254,320,40,51.33292007446289,1.2833230018615722,0.16041537523269653,0.0429231948684901,0.012926540290936827,12.363943309122169,7.751098961534808e-05,0.06192715894430876,0.0,0.0,0.0,0.0,0.013109061248542275,80,10,9.641941547393799,0.9641941547393799,0.12052426934242248,0.01464123004116118 +9,0.012389416692894884,6.866479976817482,7.133043326996713e-05,0.07013261341489851,0.0,0.0,0.0,0.0,0.012591721711214632,320,40,51.01964092254639,1.2754910230636596,0.15943637788295745,0.03452906697057188,0.012916690974088851,11.750201603699548,8.754061152274061e-05,0.062365250661969185,0.0,0.0,0.0,0.0,0.0131001640445902,80,10,9.570227146148682,0.9570227146148682,0.11962783932685853,0.017192536499351263 +10,0.018607557029463352,4.465180883041103,0.000800906858773942,0.09729779753834009,0.0,0.0,0.0,0.0,0.018892296141711996,320,40,51.002689361572266,1.2750672340393066,0.15938340425491332,0.06619891247246415,0.011875751259503886,17.329812236219624,1.2309143085076357e-05,0.07150064390152693,0.0,0.0,0.0,0.0,0.012017370684770868,80,10,9.58153510093689,0.958153510093689,0.11976918876171112,0.011057432601228356 +11,0.012972346472088248,19.315320107340813,0.00019001812893630897,0.06836491376161576,0.0,0.0,0.0,0.0,0.013183764470159075,320,40,50.905850887298584,1.2726462721824645,0.15908078402280806,0.014206227369140834,0.011903141811490058,18.613083130121233,3.253242537972767e-05,0.06795406974852085,0.0,0.0,0.0,0.0,0.012059342197608203,80,10,9.73936128616333,0.973936128616333,0.12174201607704163,0.009851218271069228 +12,0.013557270777528174,9.40529484031249,0.000470378745089306,0.07184866722673178,0.0,0.0,0.0,0.0,0.013794060627697035,320,40,50.82116389274597,1.2705290973186494,0.15881613716483117,0.028804797097109258,0.01135652862722054,8.427186564245494,8.580396155366543e-06,0.0712865686044097,0.0,0.0,0.0,0.0,0.011488370859296992,80,10,9.59274697303772,0.959274697303772,0.1199093371629715,0.021046570409089325 +13,0.013481573245371692,7.827111255334057,0.0002185987625528796,0.0728761725127697,0.0,0.0,0.0,0.0,0.01369951949600363,320,40,51.20417022705078,1.2801042556762696,0.1600130319595337,0.031780592259019615,0.011808508425019681,6.597967364589567,2.029107187979662e-05,0.07400116585195064,0.0,0.0,0.0,0.0,0.011944976943777875,80,10,9.59532618522644,0.959532618522644,0.1199415773153305,0.030000757053494455 +14,0.013263944795471615,6.083585756333209,0.00026106483452181806,0.07474971488118172,0.0,0.0,0.0,0.0,0.013470099665573798,320,40,51.04028844833374,1.2760072112083436,0.15950090140104295,0.04288241628091782,0.011318164970725774,7.087506137978335,1.1584629907090972e-05,0.06372334305197,0.0,0.0,0.0,0.0,0.011474482208723203,80,10,9.756741285324097,0.9756741285324096,0.1219592660665512,0.024069974571466445 diff --git a/insurance/tab_ddpm_concat/mlu-eval.ipynb b/insurance/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..f7712d6cb77b6111e1b63acaeb118514dd99ed21 --- /dev/null +++ b/insurance/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2513 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:08.508845Z", + "iopub.status.busy": "2024-02-29T17:55:08.508554Z", + "iopub.status.idle": "2024-02-29T17:55:08.541284Z", + "shell.execute_reply": "2024-02-29T17:55:08.540427Z" + }, + "papermill": { + "duration": 0.047604, + "end_time": "2024-02-29T17:55:08.543416", + "exception": false, + "start_time": "2024-02-29T17:55:08.495812", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:08.569051Z", + "iopub.status.busy": "2024-02-29T17:55:08.568678Z", + "iopub.status.idle": "2024-02-29T17:55:08.575656Z", + "shell.execute_reply": "2024-02-29T17:55:08.574681Z" + }, + "papermill": { + "duration": 0.022213, + "end_time": "2024-02-29T17:55:08.577741", + "exception": false, + "start_time": "2024-02-29T17:55:08.555528", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:08.602434Z", + "iopub.status.busy": "2024-02-29T17:55:08.602174Z", + "iopub.status.idle": "2024-02-29T17:55:08.606151Z", + "shell.execute_reply": "2024-02-29T17:55:08.605272Z" + }, + "papermill": { + "duration": 0.018676, + "end_time": "2024-02-29T17:55:08.608072", + "exception": false, + "start_time": "2024-02-29T17:55:08.589396", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:08.631438Z", + "iopub.status.busy": "2024-02-29T17:55:08.630948Z", + "iopub.status.idle": "2024-02-29T17:55:08.634667Z", + "shell.execute_reply": "2024-02-29T17:55:08.633867Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017387, + "end_time": "2024-02-29T17:55:08.636484", + "exception": false, + "start_time": "2024-02-29T17:55:08.619097", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:08.659362Z", + "iopub.status.busy": "2024-02-29T17:55:08.659112Z", + "iopub.status.idle": "2024-02-29T17:55:08.664185Z", + "shell.execute_reply": "2024-02-29T17:55:08.663465Z" + }, + "papermill": { + "duration": 0.018801, + "end_time": "2024-02-29T17:55:08.666119", + "exception": false, + "start_time": "2024-02-29T17:55:08.647318", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b08aa7c2", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:08.690852Z", + "iopub.status.busy": "2024-02-29T17:55:08.690596Z", + "iopub.status.idle": "2024-02-29T17:55:08.695150Z", + "shell.execute_reply": "2024-02-29T17:55:08.694357Z" + }, + "papermill": { + "duration": 0.019136, + "end_time": "2024-02-29T17:55:08.697010", + "exception": false, + "start_time": "2024-02-29T17:55:08.677874", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/tab_ddpm_concat/4\"\n", + "param_index = 2\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011075, + "end_time": "2024-02-29T17:55:08.719185", + "exception": false, + "start_time": "2024-02-29T17:55:08.708110", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:08.742933Z", + "iopub.status.busy": "2024-02-29T17:55:08.742271Z", + "iopub.status.idle": "2024-02-29T17:55:08.750943Z", + "shell.execute_reply": "2024-02-29T17:55:08.750120Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022697, + "end_time": "2024-02-29T17:55:08.752883", + "exception": false, + "start_time": "2024-02-29T17:55:08.730186", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/tab_ddpm_concat/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:08.776368Z", + "iopub.status.busy": "2024-02-29T17:55:08.776120Z", + "iopub.status.idle": "2024-02-29T17:55:10.945844Z", + "shell.execute_reply": "2024-02-29T17:55:10.944969Z" + }, + "papermill": { + "duration": 2.183875, + "end_time": "2024-02-29T17:55:10.947938", + "exception": false, + "start_time": "2024-02-29T17:55:08.764063", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:10.973992Z", + "iopub.status.busy": "2024-02-29T17:55:10.973563Z", + "iopub.status.idle": "2024-02-29T17:55:10.985231Z", + "shell.execute_reply": "2024-02-29T17:55:10.984330Z" + }, + "papermill": { + "duration": 0.026836, + "end_time": "2024-02-29T17:55:10.987242", + "exception": false, + "start_time": "2024-02-29T17:55:10.960406", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:11.010986Z", + "iopub.status.busy": "2024-02-29T17:55:11.010700Z", + "iopub.status.idle": "2024-02-29T17:55:11.017838Z", + "shell.execute_reply": "2024-02-29T17:55:11.017015Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021174, + "end_time": "2024-02-29T17:55:11.019749", + "exception": false, + "start_time": "2024-02-29T17:55:10.998575", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:11.043346Z", + "iopub.status.busy": "2024-02-29T17:55:11.043108Z", + "iopub.status.idle": "2024-02-29T17:55:11.143444Z", + "shell.execute_reply": "2024-02-29T17:55:11.142720Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.114573, + "end_time": "2024-02-29T17:55:11.145594", + "exception": false, + "start_time": "2024-02-29T17:55:11.031021", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:11.171390Z", + "iopub.status.busy": "2024-02-29T17:55:11.171118Z", + "iopub.status.idle": "2024-02-29T17:55:15.689698Z", + "shell.execute_reply": "2024-02-29T17:55:15.688934Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.533713, + "end_time": "2024-02-29T17:55:15.691990", + "exception": false, + "start_time": "2024-02-29T17:55:11.158277", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 17:55:13.358713: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 17:55:13.358774: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 17:55:13.360450: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:15.717148Z", + "iopub.status.busy": "2024-02-29T17:55:15.716541Z", + "iopub.status.idle": "2024-02-29T17:55:15.723594Z", + "shell.execute_reply": "2024-02-29T17:55:15.722903Z" + }, + "papermill": { + "duration": 0.021789, + "end_time": "2024-02-29T17:55:15.725575", + "exception": false, + "start_time": "2024-02-29T17:55:15.703786", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:15.750853Z", + "iopub.status.busy": "2024-02-29T17:55:15.750578Z", + "iopub.status.idle": "2024-02-29T17:55:23.941139Z", + "shell.execute_reply": "2024-02-29T17:55:23.939978Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.205955, + "end_time": "2024-02-29T17:55:23.943698", + "exception": false, + "start_time": "2024-02-29T17:55:15.737743", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.77,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.75,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': torch.nn.modules.activation.ReLU6,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.RReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:24.453808Z", + "iopub.status.busy": "2024-02-29T17:55:24.453540Z", + "iopub.status.idle": "2024-02-29T17:55:24.523974Z", + "shell.execute_reply": "2024-02-29T17:55:24.523115Z" + }, + "papermill": { + "duration": 0.086365, + "end_time": "2024-02-29T17:55:24.526177", + "exception": false, + "start_time": "2024-02-29T17:55:24.439812", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../insurance/_cache/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache4/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache5/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/insurance [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T17:55:24.555514Z", + "iopub.status.busy": "2024-02-29T17:55:24.555243Z", + "iopub.status.idle": "2024-02-29T17:55:24.998022Z", + "shell.execute_reply": "2024-02-29T17:55:24.997082Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.460231, + "end_time": "2024-02-29T17:55:25.000775", + "exception": false, + "start_time": "2024-02-29T17:55:24.540544", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:25.030228Z", + "iopub.status.busy": "2024-02-29T17:55:25.029885Z", + "iopub.status.idle": "2024-02-29T17:55:25.034055Z", + "shell.execute_reply": "2024-02-29T17:55:25.033220Z" + }, + "papermill": { + "duration": 0.02087, + "end_time": "2024-02-29T17:55:25.035923", + "exception": false, + "start_time": "2024-02-29T17:55:25.015053", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:25.062681Z", + "iopub.status.busy": "2024-02-29T17:55:25.062411Z", + "iopub.status.idle": "2024-02-29T17:55:25.069174Z", + "shell.execute_reply": "2024-02-29T17:55:25.068337Z" + }, + "papermill": { + "duration": 0.022143, + "end_time": "2024-02-29T17:55:25.071076", + "exception": false, + "start_time": "2024-02-29T17:55:25.048933", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9613953" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:25.098198Z", + "iopub.status.busy": "2024-02-29T17:55:25.097904Z", + "iopub.status.idle": "2024-02-29T17:55:25.183215Z", + "shell.execute_reply": "2024-02-29T17:55:25.182356Z" + }, + "papermill": { + "duration": 0.101187, + "end_time": "2024-02-29T17:55:25.185242", + "exception": false, + "start_time": "2024-02-29T17:55:25.084055", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 12] --\n", + "├─Adapter: 1-1 [2, 1071, 12] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 13,312\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 12] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─RReLU: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-52 [2, 128] --\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,613,953\n", + "Trainable params: 9,613,953\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.08\n", + "========================================================================================================================\n", + "Input size (MB): 0.13\n", + "Forward/backward pass size (MB): 307.47\n", + "Params size (MB): 38.46\n", + "Estimated Total Size (MB): 346.05\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T17:55:25.216339Z", + "iopub.status.busy": "2024-02-29T17:55:25.216047Z", + "iopub.status.idle": "2024-02-29T18:12:21.414874Z", + "shell.execute_reply": "2024-02-29T18:12:21.413870Z" + }, + "papermill": { + "duration": 1016.233783, + "end_time": "2024-02-29T18:12:21.433987", + "exception": false, + "start_time": "2024-02-29T17:55:25.200204", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.03187035469454713, 'avg_role_model_std_loss': 10.499650678998297, 'avg_role_model_mean_pred_loss': 0.004555381951256443, 'avg_role_model_g_mag_loss': 0.10591729702427984, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.032666408171644436, 'n_size': 320, 'n_batch': 40, 'duration': 52.7623987197876, 'duration_batch': 1.31905996799469, 'duration_size': 0.16488249599933624, 'avg_pred_std': 0.06082322220318019}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012105482351034879, 'avg_role_model_std_loss': 11.988290130137239, 'avg_role_model_mean_pred_loss': 4.250274636738993e-05, 'avg_role_model_g_mag_loss': 0.06420383900403977, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012274807115318254, 'n_size': 80, 'n_batch': 10, 'duration': 9.881241083145142, 'duration_batch': 0.9881241083145141, 'duration_size': 0.12351551353931427, 'avg_pred_std': 0.01785293687134981}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.01477175319159869, 'avg_role_model_std_loss': 13.734025255048982, 'avg_role_model_mean_pred_loss': 0.00038951243641720533, 'avg_role_model_g_mag_loss': 0.07673057662323117, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.015022071998100728, 'n_size': 320, 'n_batch': 40, 'duration': 51.19189524650574, 'duration_batch': 1.2797973811626435, 'duration_size': 0.15997467264533044, 'avg_pred_std': 0.025308902200777084}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.018090815236791968, 'avg_role_model_std_loss': 7.937299627149969, 'avg_role_model_mean_pred_loss': 0.00019444738691163366, 'avg_role_model_g_mag_loss': 0.1157106451690197, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01831991651561111, 'n_size': 80, 'n_batch': 10, 'duration': 9.59651231765747, 'duration_batch': 0.9596512317657471, 'duration_size': 0.11995640397071838, 'avg_pred_std': 0.024659548606723546}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013891099636384751, 'avg_role_model_std_loss': 15.052705611435432, 'avg_role_model_mean_pred_loss': 0.0001513469102661702, 'avg_role_model_g_mag_loss': 0.07600115705281496, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01410236083320342, 'n_size': 320, 'n_batch': 40, 'duration': 51.0040807723999, 'duration_batch': 1.2751020193099976, 'duration_size': 0.1593877524137497, 'avg_pred_std': 0.02815434621879831}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011826220527291298, 'avg_role_model_std_loss': 16.741873883521475, 'avg_role_model_mean_pred_loss': 2.0565837598951475e-05, 'avg_role_model_g_mag_loss': 0.06937152929604054, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011988498945720493, 'n_size': 80, 'n_batch': 10, 'duration': 10.122968435287476, 'duration_batch': 1.0122968435287476, 'duration_size': 0.12653710544109345, 'avg_pred_std': 0.011842810921370983}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.01305226328986464, 'avg_role_model_std_loss': 8.44533070571506, 'avg_role_model_mean_pred_loss': 8.075568497831754e-05, 'avg_role_model_g_mag_loss': 0.0740173936355859, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01324827965145232, 'n_size': 320, 'n_batch': 40, 'duration': 52.304439544677734, 'duration_batch': 1.3076109886169434, 'duration_size': 0.16345137357711792, 'avg_pred_std': 0.030870095500722526}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01087374986964278, 'avg_role_model_std_loss': 4.948310850643611, 'avg_role_model_mean_pred_loss': 9.443121865615466e-06, 'avg_role_model_g_mag_loss': 0.06728989202529193, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011004218307789415, 'n_size': 80, 'n_batch': 10, 'duration': 10.216580152511597, 'duration_batch': 1.0216580152511596, 'duration_size': 0.12770725190639495, 'avg_pred_std': 0.03216287791728974}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.01341965578030795, 'avg_role_model_std_loss': 9.745894026785027, 'avg_role_model_mean_pred_loss': 0.00025464458190107566, 'avg_role_model_g_mag_loss': 0.07172424085438252, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013624004492885432, 'n_size': 320, 'n_batch': 40, 'duration': 52.158915519714355, 'duration_batch': 1.303972887992859, 'duration_size': 0.16299661099910737, 'avg_pred_std': 0.03181108951102942}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013633261938230135, 'avg_role_model_std_loss': 23.27936131209135, 'avg_role_model_mean_pred_loss': 0.0001621007818215503, 'avg_role_model_g_mag_loss': 0.06217544944956899, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013878957656561396, 'n_size': 80, 'n_batch': 10, 'duration': 9.629613637924194, 'duration_batch': 0.9629613637924195, 'duration_size': 0.12037017047405243, 'avg_pred_std': 0.008153766090981663}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.015133786734077148, 'avg_role_model_std_loss': 15.675266945300246, 'avg_role_model_mean_pred_loss': 0.0004782500350243524, 'avg_role_model_g_mag_loss': 0.07984540462493897, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.015376201126491651, 'n_size': 320, 'n_batch': 40, 'duration': 51.18136239051819, 'duration_batch': 1.2795340597629548, 'duration_size': 0.15994175747036934, 'avg_pred_std': 0.03030351351480931}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013037011679261922, 'avg_role_model_std_loss': 60.89342765808105, 'avg_role_model_mean_pred_loss': 0.00012217244490742019, 'avg_role_model_g_mag_loss': 0.06693714633584022, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013210956694092602, 'n_size': 80, 'n_batch': 10, 'duration': 9.720444917678833, 'duration_batch': 0.9720444917678833, 'duration_size': 0.12150556147098542, 'avg_pred_std': 0.002085553959477693}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.014456333359703422, 'avg_role_model_std_loss': 23.62044777232586, 'avg_role_model_mean_pred_loss': 0.00021797772160834228, 'avg_role_model_g_mag_loss': 0.07543233875185251, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014706396087422035, 'n_size': 320, 'n_batch': 40, 'duration': 51.926756381988525, 'duration_batch': 1.298168909549713, 'duration_size': 0.16227111369371414, 'avg_pred_std': 0.011786041979212314}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012131108457106165, 'avg_role_model_std_loss': 7.805004606687544, 'avg_role_model_mean_pred_loss': 2.6683602803245778e-05, 'avg_role_model_g_mag_loss': 0.07964657545089722, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012273790544713847, 'n_size': 80, 'n_batch': 10, 'duration': 9.750572919845581, 'duration_batch': 0.9750572919845581, 'duration_size': 0.12188216149806977, 'avg_pred_std': 0.02273060418665409}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012077562542981469, 'avg_role_model_std_loss': 6.567233682959795, 'avg_role_model_mean_pred_loss': 4.818316614121354e-05, 'avg_role_model_g_mag_loss': 0.07109622973948718, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012245478003751486, 'n_size': 320, 'n_batch': 40, 'duration': 51.910486936569214, 'duration_batch': 1.2977621734142304, 'duration_size': 0.1622202716767788, 'avg_pred_std': 0.03978729886002839}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011527308775112034, 'avg_role_model_std_loss': 13.779987310473008, 'avg_role_model_mean_pred_loss': 8.208526573838525e-06, 'avg_role_model_g_mag_loss': 0.07020539008080959, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011682888003997504, 'n_size': 80, 'n_batch': 10, 'duration': 9.546999216079712, 'duration_batch': 0.9546999216079712, 'duration_size': 0.1193374902009964, 'avg_pred_std': 0.013699167082086206}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012478862586431206, 'avg_role_model_std_loss': 5.30599477694621, 'avg_role_model_mean_pred_loss': 0.00012620691976824582, 'avg_role_model_g_mag_loss': 0.07224454144015909, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012667084962595254, 'n_size': 320, 'n_batch': 40, 'duration': 51.33292007446289, 'duration_batch': 1.2833230018615722, 'duration_size': 0.16041537523269653, 'avg_pred_std': 0.0429231948684901}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012926540290936827, 'avg_role_model_std_loss': 12.363943309122169, 'avg_role_model_mean_pred_loss': 7.751098961534808e-05, 'avg_role_model_g_mag_loss': 0.06192715894430876, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013109061248542275, 'n_size': 80, 'n_batch': 10, 'duration': 9.641941547393799, 'duration_batch': 0.9641941547393799, 'duration_size': 0.12052426934242248, 'avg_pred_std': 0.01464123004116118}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012389416692894884, 'avg_role_model_std_loss': 6.866479976817482, 'avg_role_model_mean_pred_loss': 7.133043326996713e-05, 'avg_role_model_g_mag_loss': 0.07013261341489851, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012591721711214632, 'n_size': 320, 'n_batch': 40, 'duration': 51.01964092254639, 'duration_batch': 1.2754910230636596, 'duration_size': 0.15943637788295745, 'avg_pred_std': 0.03452906697057188}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012916690974088851, 'avg_role_model_std_loss': 11.750201603699548, 'avg_role_model_mean_pred_loss': 8.754061152274061e-05, 'avg_role_model_g_mag_loss': 0.062365250661969185, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0131001640445902, 'n_size': 80, 'n_batch': 10, 'duration': 9.570227146148682, 'duration_batch': 0.9570227146148682, 'duration_size': 0.11962783932685853, 'avg_pred_std': 0.017192536499351263}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.018607557029463352, 'avg_role_model_std_loss': 4.465180883041103, 'avg_role_model_mean_pred_loss': 0.000800906858773942, 'avg_role_model_g_mag_loss': 0.09729779753834009, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.018892296141711996, 'n_size': 320, 'n_batch': 40, 'duration': 51.002689361572266, 'duration_batch': 1.2750672340393066, 'duration_size': 0.15938340425491332, 'avg_pred_std': 0.06619891247246415}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011875751259503886, 'avg_role_model_std_loss': 17.329812236219624, 'avg_role_model_mean_pred_loss': 1.2309143085076357e-05, 'avg_role_model_g_mag_loss': 0.07150064390152693, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012017370684770868, 'n_size': 80, 'n_batch': 10, 'duration': 9.58153510093689, 'duration_batch': 0.958153510093689, 'duration_size': 0.11976918876171112, 'avg_pred_std': 0.011057432601228356}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012972346472088248, 'avg_role_model_std_loss': 19.315320107340813, 'avg_role_model_mean_pred_loss': 0.00019001812893630897, 'avg_role_model_g_mag_loss': 0.06836491376161576, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013183764470159075, 'n_size': 320, 'n_batch': 40, 'duration': 50.905850887298584, 'duration_batch': 1.2726462721824645, 'duration_size': 0.15908078402280806, 'avg_pred_std': 0.014206227369140834}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011903141811490058, 'avg_role_model_std_loss': 18.613083130121233, 'avg_role_model_mean_pred_loss': 3.253242537972767e-05, 'avg_role_model_g_mag_loss': 0.06795406974852085, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012059342197608203, 'n_size': 80, 'n_batch': 10, 'duration': 9.73936128616333, 'duration_batch': 0.973936128616333, 'duration_size': 0.12174201607704163, 'avg_pred_std': 0.009851218271069228}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013557270777528174, 'avg_role_model_std_loss': 9.40529484031249, 'avg_role_model_mean_pred_loss': 0.000470378745089306, 'avg_role_model_g_mag_loss': 0.07184866722673178, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013794060627697035, 'n_size': 320, 'n_batch': 40, 'duration': 50.82116389274597, 'duration_batch': 1.2705290973186494, 'duration_size': 0.15881613716483117, 'avg_pred_std': 0.028804797097109258}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01135652862722054, 'avg_role_model_std_loss': 8.427186564245494, 'avg_role_model_mean_pred_loss': 8.580396155366543e-06, 'avg_role_model_g_mag_loss': 0.0712865686044097, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011488370859296992, 'n_size': 80, 'n_batch': 10, 'duration': 9.59274697303772, 'duration_batch': 0.959274697303772, 'duration_size': 0.1199093371629715, 'avg_pred_std': 0.021046570409089325}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013481573245371692, 'avg_role_model_std_loss': 7.827111255334057, 'avg_role_model_mean_pred_loss': 0.0002185987625528796, 'avg_role_model_g_mag_loss': 0.0728761725127697, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01369951949600363, 'n_size': 320, 'n_batch': 40, 'duration': 51.20417022705078, 'duration_batch': 1.2801042556762696, 'duration_size': 0.1600130319595337, 'avg_pred_std': 0.031780592259019615}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011808508425019681, 'avg_role_model_std_loss': 6.597967364589567, 'avg_role_model_mean_pred_loss': 2.029107187979662e-05, 'avg_role_model_g_mag_loss': 0.07400116585195064, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011944976943777875, 'n_size': 80, 'n_batch': 10, 'duration': 9.59532618522644, 'duration_batch': 0.959532618522644, 'duration_size': 0.1199415773153305, 'avg_pred_std': 0.030000757053494455}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013263944795471615, 'avg_role_model_std_loss': 6.083585756333209, 'avg_role_model_mean_pred_loss': 0.00026106483452181806, 'avg_role_model_g_mag_loss': 0.07474971488118172, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013470099665573798, 'n_size': 320, 'n_batch': 40, 'duration': 51.04028844833374, 'duration_batch': 1.2760072112083436, 'duration_size': 0.15950090140104295, 'avg_pred_std': 0.04288241628091782}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011318164970725774, 'avg_role_model_std_loss': 7.087506137978335, 'avg_role_model_mean_pred_loss': 1.1584629907090972e-05, 'avg_role_model_g_mag_loss': 0.06372334305197, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011474482208723203, 'n_size': 80, 'n_batch': 10, 'duration': 9.756741285324097, 'duration_batch': 0.9756741285324096, 'duration_size': 0.1219592660665512, 'avg_pred_std': 0.024069974571466445}\n", + "Stopped False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test ▂█▂▁▄▃▂▂▃▃▂▂▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▂▂▁▁▂▂▁▁▁▃▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▅▆▃█▂▁▆▄▄▅▃▃▅▇▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▇▃▃▃▄▃▁▅▅▄█▁▃▄▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁█▂▂▁▂▃▂▁▁▂▂▂▃▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train █▃▂▂▂▃▂▂▂▁▆▁▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test ▂█▂▁▄▃▂▂▃▃▂▂▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▂▂▁▁▂▂▁▁▁▃▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test ▂█▁▁▇▅▂▁▄▄▁▂▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▂▁▁▁▂▁▁▁▁▂▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▂▁▂▁▃█▁▂▂▂▃▃▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▃▄▅▂▃▅█▂▁▂▁▆▃▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▄▂▇█▂▃▃▁▂▁▁▃▁▂▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train █▂▂▆▆▂▅▅▃▂▂▁▁▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▄▂▇█▂▃▃▁▂▁▁▃▁▂▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train █▂▂▆▆▂▅▅▃▂▂▁▁▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▄▂▇█▂▃▃▁▂▁▁▃▁▂▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train █▂▂▆▆▂▅▅▃▂▂▁▁▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.01147\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.01347\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.02407\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.04288\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.06372\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.07475\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.01132\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.01326\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 1e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.00026\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 7.08751\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 6.08359\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.97567\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.27601\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.12196\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.1595\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 9.75674\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 51.04029\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 10\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/insurance/tab_ddpm_concat/4/wandb/offline-run-20240229_175526-dhcmzcwb\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_175526-dhcmzcwb/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 399, 'n_batch': 50, 'role_model_metrics': {'avg_loss': 0.020264622956084877, 'avg_g_mag_loss': 0.5099495620207679, 'avg_g_cos_loss': 5.4264185582769756e-06, 'pred_duration': 0.8703622817993164, 'grad_duration': 0.5704290866851807, 'total_duration': 1.440791368484497, 'pred_std': 0.044777363538742065, 'std_loss': 1.0563119649887085, 'mean_pred_loss': 1.4852062122372445e-05, 'pred_rmse': 0.1423538625240326, 'pred_mae': 0.09591443091630936, 'pred_mape': 0.7516122460365295, 'grad_rmse': 0.28464236855506897, 'grad_mae': 0.19168375432491302, 'grad_mape': 0.9978110790252686}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.020264622956084877, 'avg_g_mag_loss': 0.5099495620207679, 'avg_g_cos_loss': 5.4264185582769756e-06, 'avg_pred_duration': 0.8703622817993164, 'avg_grad_duration': 0.5704290866851807, 'avg_total_duration': 1.440791368484497, 'avg_pred_std': 0.044777363538742065, 'avg_std_loss': 1.0563119649887085, 'avg_mean_pred_loss': 1.4852062122372445e-05}, 'min_metrics': {'avg_loss': 0.020264622956084877, 'avg_g_mag_loss': 0.5099495620207679, 'avg_g_cos_loss': 5.4264185582769756e-06, 'pred_duration': 0.8703622817993164, 'grad_duration': 0.5704290866851807, 'total_duration': 1.440791368484497, 'pred_std': 0.044777363538742065, 'std_loss': 1.0563119649887085, 'mean_pred_loss': 1.4852062122372445e-05, 'pred_rmse': 0.1423538625240326, 'pred_mae': 0.09591443091630936, 'pred_mape': 0.7516122460365295, 'grad_rmse': 0.28464236855506897, 'grad_mae': 0.19168375432491302, 'grad_mape': 0.9978110790252686}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.020264622956084877, 'avg_g_mag_loss': 0.5099495620207679, 'avg_g_cos_loss': 5.4264185582769756e-06, 'pred_duration': 0.8703622817993164, 'grad_duration': 0.5704290866851807, 'total_duration': 1.440791368484497, 'pred_std': 0.044777363538742065, 'std_loss': 1.0563119649887085, 'mean_pred_loss': 1.4852062122372445e-05, 'pred_rmse': 0.1423538625240326, 'pred_mae': 0.09591443091630936, 'pred_mape': 0.7516122460365295, 'grad_rmse': 0.28464236855506897, 'grad_mae': 0.19168375432491302, 'grad_mape': 0.9978110790252686}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:12:21.471053Z", + "iopub.status.busy": "2024-02-29T18:12:21.470098Z", + "iopub.status.idle": "2024-02-29T18:12:21.474784Z", + "shell.execute_reply": "2024-02-29T18:12:21.474072Z" + }, + "papermill": { + "duration": 0.025412, + "end_time": "2024-02-29T18:12:21.476720", + "exception": false, + "start_time": "2024-02-29T18:12:21.451308", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:12:21.511494Z", + "iopub.status.busy": "2024-02-29T18:12:21.510794Z", + "iopub.status.idle": "2024-02-29T18:12:21.773483Z", + "shell.execute_reply": "2024-02-29T18:12:21.772653Z" + }, + "papermill": { + "duration": 0.283017, + "end_time": "2024-02-29T18:12:21.776097", + "exception": false, + "start_time": "2024-02-29T18:12:21.493080", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:12:21.813713Z", + "iopub.status.busy": "2024-02-29T18:12:21.813419Z", + "iopub.status.idle": "2024-02-29T18:12:22.071700Z", + "shell.execute_reply": "2024-02-29T18:12:22.070773Z" + }, + "papermill": { + "duration": 0.279736, + "end_time": "2024-02-29T18:12:22.073784", + "exception": false, + "start_time": "2024-02-29T18:12:21.794048", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCy0lEQVR4nO3deXhU1d3A8e9k33eyQUgAiUEEZDERFKiSGpSCcQNpKsiLgi24pbaUWsClNa4tLrxatMX6VgRpkVpAFAKIQNgSwk5kCYQlCYSYnWwz5/3jZiZMmJCZbJOQ3+d55iG598y5J8PMb8459yw6pZRCCCE6CQd7F0AIIWwhQUsI0alI0BJCdCoStIQQnYoELSFEpyJBSwjRqUjQEkJ0Kk72LkB7MRgMnD9/Hm9vb3Q6nb2LI4RoQClFaWkp4eHhODg0Xp/qMkHr/PnzRERE2LsYQogmnDlzhh49ejR6vssELW9vb0B7QXx8fOxcGiFEQyUlJURERJg+q43pMkHL2CT08fGRoCVEB9ZU9410xAshOhUJWkKITkWClhCiU+kyfVqiZfR6PTU1NfYuhujEnJ2dcXR0bHE+ErTENSmlyMvLo6ioyN5FEdcBPz8/QkNDWzRWUoKWuCZjwAoODsbDw0MG5opmUUpRUVHBhQsXAAgLC2t2XhK0Gii+XMOR3BJ0QFzvQHsXx670er0pYAUGdu3XQrScu7s7ABcuXCA4OLjZTUXpiG8gI+dHHlm8gxf/e9jeRbE7Yx+Wh4eHnUsirhfG91JL+kclaDXQzcsVgIKyKjuXpOOQJqFoLa3xXpKg1UCglwsAheXVGAyy54cQHY0ErQYCPbWalt6gKLost/iF6GgkaDXg4uSAr7szAJekiSiaSafTsWrVKnsXo1W9+OKL3HLLLfYuhgQtS4xNxIsStEQn9sknn+Dn59dq+T3//POkpqa2Wn7NJUHLgiBTZ3y1nUsiRNurrrbufe7l5dUhhr5I0LIgqK6mJc1Dc0opKqpr7fKwdSP0devWcccdd+Dn50dgYCA/+9nPOHHiBAAjRoxgzpw5ZukvXryIs7MzW7ZsASA3N5dx48bh7u5Or169WLp0KVFRUSxcuLBZr92BAwe46667cHd3JzAwkBkzZlBWVmY6v3nzZmJjY/H09MTPz4/bb7+d06dPA7Bv3z7uvPNOvL298fHxYejQoezZs+ea19u8eTPTpk2juLgYnU6HTqfjxRdfBCAqKopXXnmFKVOm4OPjw4wZMwCYM2cO0dHReHh40Lt3b+bNm2c2NKFh8/Cxxx4jMTGRt956i7CwMAIDA5k1a1abT/eSwaUWBMmwB4su1+i5af43drn24ZcT8HCx/u1aXl5OcnIyAwcOpKysjPnz53P//feTmZlJUlISb7zxBq+99prpFvzy5csJDw9n5MiRAEyZMoWCggI2b96Ms7MzycnJptHctiovLychIYHhw4eze/duLly4wOOPP87s2bP55JNPqK2tJTExkSeeeILPP/+c6upqdu3aZSpbUlISgwcP5oMPPsDR0ZHMzEycnZ2vec0RI0awcOFC5s+fT1ZWFqDVlIzeeust5s+fz4IFC0zHvL29+eSTTwgPD+fAgQM88cQTeHt789vf/rbR62zatImwsDA2bdrE8ePHmTRpErfccgtPPPFEs14ra0jQssAYtC5J87DTevDBB81+//vf/063bt04fPgwEydO5Nlnn2Xr1q2mILV06VImT56MTqfj6NGjbNiwgd27dzNs2DAAPv74Y/r27dussixdupTKyko+/fRTPD09AXj//fcZP348r7/+Os7OzhQXF/Ozn/2MPn36ANCvXz/T83NycvjNb35DTEwMgFXlcHFxwdfXF51OR2ho6FXn77rrLn7961+bHfvDH/5g+jkqKornn3+eZcuWXTNo+fv78/777+Po6EhMTAzjxo0jNTVVglZ7M3bES03LnLuzI4dfTrDbtW1x7Ngx5s+fz86dOykoKMBgMABaALj55pu5++67+eyzzxg5ciTZ2dmkpaXx17/+FYCsrCycnJwYMmSIKb8bbrgBf3//ZpX9yJEjDBo0yBSwAG6//XYMBgNZWVmMGjWKxx57jISEBH76058SHx/PxIkTTfPzkpOTefzxx/m///s/4uPjefjhh03BrbmMwfhKy5cv59133+XEiROUlZVRW1vb5Cq//fv3N5uOExYWxoEDB1pUtqZIn5YFxprWRalpmdHpdHi4ONnlYetI6vHjx1NYWMhHH33Ezp072blzJ1Df6ZyUlMS//vUvampqWLp0KQMGDGDAgAGt/ppZa8mSJaSlpTFixAiWL19OdHQ0O3bsALS+pEOHDjFu3Dg2btzITTfdxJdfftmi610ZQAHS0tJISkri3nvvZfXq1ezdu5cXXnihyU76hs1UnU5n+oJoKxK0LJCO+M7t0qVLZGVl8Yc//IExY8bQr18/fvzxR7M09913H5WVlaxbt46lS5eSlJRkOnfjjTdSW1vL3r17TceOHz9+VR7W6tevH/v27aO8vNx0bNu2bTg4OHDjjTeajg0ePJi5c+eyfft2br75ZpYuXWo6Fx0dzXPPPce3337LAw88wJIlS5q8rouLC3q93qoybt++ncjISF544QWGDRtG3759TTcCOhoJWhZc2RFv610rYX/+/v4EBgayePFijh8/zsaNG0lOTjZL4+npSWJiIvPmzePIkSNMnjzZdC4mJob4+HhmzJjBrl272Lt3LzNmzMDd3b1Zc+eSkpJwc3Nj6tSpHDx4kE2bNvHUU0/x6KOPEhISQnZ2NnPnziUtLY3Tp0/z7bffcuzYMfr168fly5eZPXs2mzdv5vTp02zbto3du3eb9Xk1JioqirKyMlJTUykoKKCioqLRtH379iUnJ4dly5Zx4sQJ3n333RbX5tqKBC0LjEGrssZARbV131Si43BwcGDZsmWkp6dz880389xzz/Hmm29elS4pKYl9+/YxcuRIevbsaXbu008/JSQkhFGjRnH//feb7qS5ubnZXB4PDw+++eYbCgsLufXWW3nooYcYM2YM77//vun80aNHefDBB4mOjmbGjBnMmjWLmTNn4ujoyKVLl5gyZQrR0dFMnDiRe+65h5deeqnJ644YMYInn3ySSZMm0a1bN954441G006YMIHnnnuO2bNnc8stt7B9+3bmzZtn89/aHnSqi1QlSkpK8PX1pbi4uMnORaUU/eavo7LGwHe/+QmRgZ7XTH+9qqysJDs7m169ejXrw3o9OXv2LBEREWzYsIExY8bYuzid1rXeU9Z+RuXuoQU6nY4gL1fO/niZgrKqLhu0urKNGzdSVlbGgAEDyM3N5be//S1RUVGMGjXK3kXr8qR52IhAmcrTpdXU1PD73/+e/v37c//999OtWzfTQNPPPvsMLy8vi4/+/fu3WxnvueeeRsvx6quvtls52pvUtBrRTcZqdWkJCQkkJFgekzZhwgTi4uIsnmtqpHpr+vjjj7l8+bLFcwEBAe1WjvYmQasRMipeNMbb2xtvb297F4Pu3bvbuwh2Ic3DRsioeCE6JglajZCalhAdkwStRgSapvJITUuIjqRZQWvRokVERUXh5uZGXFwcu3btumb6FStWEBMTg5ubGwMGDGDt2rVm51988UViYmLw9PTE39+f+Ph401wxo8LCQpKSkvDx8cHPz4/p06ebrUfU2oKkeShEh2Rz0Fq+fDnJycksWLCAjIwMBg0aREJCQqNrDW3fvp3Jkyczffp09u7dS2JiIomJiRw8eNCUJjo6mvfff58DBw6wdetWoqKiuPvuu7l48aIpTVJSEocOHWL9+vWsXr2aLVu2mBYvawvdpHkoRMekbBQbG6tmzZpl+l2v16vw8HCVkpJiMf3EiRPVuHHjzI7FxcWpmTNnNnqN4uJiBagNGzYopZQ6fPiwAtTu3btNab7++mul0+nUuXPnrCq3Mc/i4mKr0l8qq1KRc1aryDmrVVWN3qrnXG8uX76sDh8+rC5fvmzvonQ6gPryyy/tXYwO51rvKWs/ozbVtKqrq0lPTyc+Pt50zMHBgfj4eNLS0iw+Jy0tzSw9aGNgGktfXV3N4sWL8fX1ZdCgQaY8/Pz8zNYAio+Px8HB4apmpFFVVRUlJSVmD1v4uTvj6KBNji0sl9qW6Hxae2ML0JZx1ul0FBUVtWq+trApaBUUFKDX6wkJCTE7HhISQl5ensXn5OXlWZV+9erVeHl54ebmxl/+8hfWr19PUFCQKY/g4GCz9E5OTgQEBDR63ZSUFHx9fU2PiIgIW/5UHBx0BHhKv5YQHU2HuXt45513kpmZyfbt2xk7diwTJ05s9prcAHPnzqW4uNj0OHPmjM15BMkdRHNKQXW5fR6ysUWrbWxRVVXF888/T/fu3fH09CQuLo7Nmzebnnv69GnGjx+Pv78/np6e9O/fn7Vr13Lq1CnuvPNOQFv+R6fT8dhjjzXr9WgJm0bEBwUF4ejoSH5+vtnx/Px8i+tQA4SGhlqV3tPTkxtuuIEbbriB2267jb59+/K3v/2NuXPnEhoaelUAq62tpbCwsNHrurq64urqasufd5X6xQCleQhATQW8Gm6fa//+PLhYP3FdNrZofGOL2bNnc/jwYZYtW0Z4eDhffvklY8eO5cCBA/Tt25dZs2ZRXV3Nli1b8PT05PDhw3h5eREREcG///1vHnzwQbKysvDx8cHd3b1Zr0lL2BS0XFxcGDp0KKmpqSQmJgJgMBhITU1l9uzZFp8zfPhwUlNTefbZZ03H1q9fz/Dhw695LYPBQFVVlSmPoqIi0tPTGTp0KKDNwjcYDI3OAWsNsitP5yUbW1je2CInJ4clS5aQk5NDeLj2BfT888+zbt06lixZwquvvkpOTg4PPvigafnp3r17m55vnNMYHBzc6v1l1rJ57mFycjJTp05l2LBhxMbGsnDhQsrLy5k2bRqgfUN1796dlJQUAJ555hlGjx7N22+/zbhx41i2bBl79uxh8eLFgPYt9Kc//YkJEyYQFhZGQUEBixYt4ty5czz88MOA9h84duxYnnjiCT788ENqamqYPXs2jzzyiOmFbwuy7HIDzh5ajcde17aBbGxh2YEDB9Dr9URHR5sdr6qqMm3E+vTTT/PLX/6Sb7/9lvj4eB588EEGDhzYrOu1BZv7tCZNmmTaM+2WW24hMzOTdevWmTrbc3JyyM3NNaUfMWIES5cuZfHixQwaNIh//etfrFq1iptvvhkAR0dHs1Ubx48fz6VLl/j+++/Nlvn47LPPiImJYcyYMdx7773ccccdpsDXVmR5mgZ0Oq2JZo+HbGzRKhtblJWV4ejoSHp6OpmZmabHkSNHeOeddwB4/PHHOXnyJI8++igHDhxg2LBhvPfee632t7ZYW43H6GhsHaellFIr9pxRkXNWq198vKMNS9ZxddZxWgUFBQpQW7ZsMR37/vvvzcZOlZWVKU9PT/XVV1+pm266Sb322mumtEeOHFGA2rNnj+nYsWPHFKD+8pe/WFWGK6+1ePFi5e/vr8rKykzn16xZoxwcHFReXp7F5992223qqaeesnjukUceUePHj2+yDJ999pny8vIyO5aVlXXVa9OU3/3ud2rAgAFKKaW2bdumAFVQUGD186/U7uO0upr6qTxS0+pMZGMLjaWNLaKjo0lKSmLKlCmsXLmS7Oxsdu3aRUpKCmvWrAHg2Wef5ZtvviE7O5uMjAw2bdpkul5kZCQ6nY7Vq1dz8eLFNp1K16hmhctOqDk1rQNni1TknNVq2B/Xt2HJOq7OWtNSSqn169erfv36KVdXVzVw4EC1efPmq0apr127VgFq1KhRVz3//Pnz6p577lGurq4qMjJSLV26VAUHB6sPP/zQqus3vNb+/fvVnXfeqdzc3FRAQIB64oknVGlpqVJKqby8PJWYmKjCwsKUi4uLioyMVPPnz1d6vV5VVVWpRx55REVERCgXFxcVHh6uZs+ebfX/yZNPPqkCAwMVoBYsWKCUUqq6ulrNnz9fRUVFKWdnZxUWFqbuv/9+tX//fqWUUrNnz1Z9+vRRrq6uqlu3burRRx81q1m9/PLLKjQ0VOl0OjV16lSrymHUGjUt2djiGvKKK7ktJRVHBx3H/ngPDg62f8t2ZrKxRT3Z2KJ1yMYWbcw4Il5vUBRdrjH9Lq5/srFFxyV9Wtfg4uSAr7s2iE/GanUtsrFFxyU1rSYEeblQfLmGgrIqokPsvy64aB+ysUXHJUGrCYFerpy4WC53EIWJbGxhX9I8bEL9YoBdt3nYRe7ViHbQGu8lCVpN6Mq78hibOhUVFXYuibheGN9LLWlGS/OwCaZJ06Vdr3no6OiIn5+faXUDDw+PZg2uFEIpRUVFBRcuXMDPzw9HR8dm5yVBqwmmrcTKu15NCzCtENCStc2EMPLz82t0OSlrSdBqgrF5eLGLdsTrdDrCwsIIDg6mpqbG3sURnZizs3OLalhGErSaECQd8YDWVGyNN5wQLSUd8U24cv9DuYsmhP1J0GqCsaZVWWOgvFpv59IIISRoNcHT1Ql3Z61Z1NWbiEJ0BBK0rNCVx2oJ0dFI0LJCkCy7LESHIUHLCkFS0xKiw5CgZYX6YQ9S0xLC3iRoWUH2PxSi45CgZQXpiBei45CgZQXpiBei45CgZQWpaQnRcUjQskI36YgXosOQoGUFY/Ow+HIN1bUGO5dGiK5NgpYVfN2dcazb87CrrqslREchQcsKDg46Auv2PJQmohD2JUHLSoF1TcSL0hkvhF1J0LKScSqP1LSEsC8JWlbqJqPihegQJGhZyTRWq1SClhD2JEHLSvW78kjzUAh7kqBlpUBpHgrRIUjQslL9mlpS0xLCniRoWUmWpxGiY5CgZSVj0Cosr8ZgkK3EhLCXZgWtRYsWERUVhZubG3Fxcezateua6VesWEFMTAxubm4MGDCAtWvXms7V1NQwZ84cBgwYgKenJ+Hh4UyZMoXz58+b5REVFYVOpzN7vPbaa80pfrME1I2I1xsURZdlp2Uh7MXmoLV8+XKSk5NZsGABGRkZDBo0iISEBC5cuGAx/fbt25k8eTLTp09n7969JCYmkpiYyMGDBwGoqKggIyODefPmkZGRwcqVK8nKymLChAlX5fXyyy+Tm5trejz11FO2Fr/ZXJwc8HV3BqSJKIRdKRvFxsaqWbNmmX7X6/UqPDxcpaSkWEw/ceJENW7cOLNjcXFxaubMmY1eY9euXQpQp0+fNh2LjIxUf/nLX2wtrklxcbECVHFxcbPzuOutTSpyzmq17fjFZuchhLDM2s+oTTWt6upq0tPTiY+PNx1zcHAgPj6etLQ0i89JS0szSw+QkJDQaHqA4uJidDodfn5+Zsdfe+01AgMDGTx4MG+++Sa1tbWN5lFVVUVJSYnZo6VkBVMh7M/JlsQFBQXo9XpCQkLMjoeEhHD06FGLz8nLy7OYPi8vz2L6yspK5syZw+TJk/Hx8TEdf/rppxkyZAgBAQFs376duXPnkpuby5///GeL+aSkpPDSSy/Z8uc1yRS0ZFS8EHZjU9BqazU1NUycOBGlFB988IHZueTkZNPPAwcOxMXFhZkzZ5KSkoKrq+tVec2dO9fsOSUlJURERLSofKZJ07KmlhB2Y1PQCgoKwtHRkfz8fLPj+fn5hIaGWnxOaGioVemNAev06dNs3LjRrJZlSVxcHLW1tZw6dYobb7zxqvOurq4Wg1lLmEbFl0rzUAh7salPy8XFhaFDh5Kammo6ZjAYSE1NZfjw4RafM3z4cLP0AOvXrzdLbwxYx44dY8OGDQQGBjZZlszMTBwcHAgODrblT2iR+vmHUtMSwl5sbh4mJyczdepUhg0bRmxsLAsXLqS8vJxp06YBMGXKFLp3705KSgoAzzzzDKNHj+btt99m3LhxLFu2jD179rB48WJAC1gPPfQQGRkZrF69Gr1eb+rvCggIwMXFhbS0NHbu3Mmdd96Jt7c3aWlpPPfcc/ziF7/A39+/tV6LJhmbhxelI14I+2nOrcn33ntP9ezZU7m4uKjY2Fi1Y8cO07nRo0erqVOnmqX/4osvVHR0tHJxcVH9+/dXa9asMZ3Lzs5WgMXHpk2blFJKpaenq7i4OOXr66vc3NxUv3791KuvvqoqKyutLnNrDHnYc6pQRc5ZrUakpDY7DyGEZdZ+RnVKqS4xJ6WkpARfX1+Ki4ub7C9rTM6lCka9uQk3ZweOvDwWnU7XyqUUouuy9jMqcw9tYFwIsLLGQHm13s6lEaJrkqBlA09XJ9ydHQG4JFN5hLALCVo2CvI2rqslQUsIe5CgZaNAT5nKI4Q9SdCykSwGKIR9SdCykWnZZRkVL4RdSNCykYyKF8K+JGjZqH6DCwlaQtiDBC0bBcqaWkLYlQQtG0lHvBD2JUHLRvUd8RK0hLAHCVo2Mta0Siprqa412Lk0QnQ9ErRs5OvujJODNlFa7iAK0f4kaNnIwUFn2gPxknTGC9HuJGg1g7GJeFE644VodxK0miFQOuOFsBsJWs3QzTQqXpqHQrQ3CVrNEOQt+x8KYS8StJoh0NgRLzUtIdqdBK1mkFHxQtiPBK1mMHbEX5TmoRDtToJWMwRJR7wQdiNBqxm61XXEF5ZXYzB0iR3YhOgwJGg1g3FEvN6gKLpcY+fSCNG1SNBqBmdHB/w8nAHpjBeivUnQaibjsAcZqyVE+5Kg1UymYQ/SGS9Eu5Kg1UwyKl4I+5Cg1UxBplHxErSEaE8StJrJ1DyU/Q+FaFcStJopUPY/FMIuJGg1k3GDi4uyeqkQ7UqCVjNJR7wQ9iFBq5mCPOubh0rJVB4h2osErWYK8taah5U1Bsqr9XYujRBdhwStZvJwccLd2RGASzKVR4h2I0GrBYy1LZl/KET7kaDVAqatxGSslhDtpllBa9GiRURFReHm5kZcXBy7du26ZvoVK1YQExODm5sbAwYMYO3ataZzNTU1zJkzhwEDBuDp6Ul4eDhTpkzh/PnzZnkUFhaSlJSEj48Pfn5+TJ8+nbKysuYUv9UEespYLSHam81Ba/ny5SQnJ7NgwQIyMjIYNGgQCQkJXLhwwWL67du3M3nyZKZPn87evXtJTEwkMTGRgwcPAlBRUUFGRgbz5s0jIyODlStXkpWVxYQJE8zySUpK4tChQ6xfv57Vq1ezZcsWZsyY0Yw/ufV0MzYPpaYlRPtRNoqNjVWzZs0y/a7X61V4eLhKSUmxmH7ixIlq3LhxZsfi4uLUzJkzG73Grl27FKBOnz6tlFLq8OHDClC7d+82pfn666+VTqdT586ds6rcxcXFClDFxcVWpbfGm+uOqsg5q9W8VQdaLU8huiprP6M21bSqq6tJT08nPj7edMzBwYH4+HjS0tIsPictLc0sPUBCQkKj6QGKi4vR6XT4+fmZ8vDz82PYsGGmNPHx8Tg4OLBz506LeVRVVVFSUmL2aG3GUfHSES9E+7EpaBUUFKDX6wkJCTE7HhISQl5ensXn5OXl2ZS+srKSOXPmMHnyZHx8fEx5BAcHm6VzcnIiICCg0XxSUlLw9fU1PSIiIqz6G21RPypemodCtJcOdfewpqaGiRMnopTigw8+aFFec+fOpbi42PQ4c+ZMK5WynrEjvkA64oVoN062JA4KCsLR0ZH8/Hyz4/n5+YSGhlp8TmhoqFXpjQHr9OnTbNy40VTLMubRsKO/traWwsLCRq/r6uqKq6ur1X9bc9R3xEvQEqK92FTTcnFxYejQoaSmppqOGQwGUlNTGT58uMXnDB8+3Cw9wPr1683SGwPWsWPH2LBhA4GBgVflUVRURHp6uunYxo0bMRgMxMXF2fIntCpjTaukspbqWoPdyiFEl2JrD/+yZcuUq6ur+uSTT9Thw4fVjBkzlJ+fn8rLy1NKKfXoo4+q3/3ud6b027ZtU05OTuqtt95SR44cUQsWLFDOzs7qwAHtjlt1dbWaMGGC6tGjh8rMzFS5ubmmR1VVlSmfsWPHqsGDB6udO3eqrVu3qr59+6rJkydbXe62uHuo1xtUn7lrVOSc1ep8UUWr5StEV2TtZ9TmoKWUUu+9957q2bOncnFxUbGxsWrHjh2mc6NHj1ZTp041S//FF1+o6Oho5eLiovr376/WrFljOpedna0Ai49NmzaZ0l26dElNnjxZeXl5KR8fHzVt2jRVWlpqdZnbImgppVTsn9aryDmr1f4zRa2arxBdjbWfUZ1SXWNdlZKSEnx9fSkuLjbrL2upe9/5nsO5JSyZdit33hjc9BOEEBZZ+xntUHcPOyNZDFCI9iVBq4Xqd+WRsVpCtAcJWi0kNS1hrapaPQv+c5ANh/ObTiwaJUGrhWQqj7DW1wfy+EfaaV5YdUCW6G4BCVotVL88jTQPxbXtOHkJgPySKo5dsO+ySp2ZBK0WMjYPL0rzUDRhZ3ah6ectP1y0Y0k6NwlaLRQoHfHCCvkllWQXlJt+//5YgR1L07lJ0GqhbnU1rcLyagwGBVWlUFNp51KJjsbYNPT3cAZgZ/YlqmplF6fmkKDVQgF1NS29QVFUcB7eGQSf3mfnUomOxtg0fGBID7p5u1JZYyD91I92LlXnJEGrhZwdHfCr+/asytoAFZfgzA4oav2lcETntbOupnVb70BG9g0CYIs0EZtFglYrMO7K43Dq+/qDp7fZqTSio7lQWsmJi+XodBAbFWAKWt8fk8745pCg1QqMnfFeudvrD57aaqfSiI5mV13TMCbUB18PZ26/QQtah86XyPi+ZpCg1QqCvF3pobuAZ8W5+oNS0xJ1dp7UgtZtvQMACPZ2o1+YNiF423FpItpKglYr6OblygiHQ9ovwf1B5wCFJ6Ek174FEx2C8c5hXK/6xS1HGfu1fpCgZSsJWq0g0NOlPmjFjIPQAdrPUtvq8i6V1Y9+j+0VYDo+sm83ALYevyhTemwkQasVBHm5MMLhsPZLr1EQeYf2s/RrdXnG/qwbQ7xNw2MAhkX54+rkIFN6mkGCVivoaThLsK6IKlygx60Qdbt2QmpaXZ5xfJaxP8vIzdmRuN5ac1Gm9NhGglYriCjeA8B+hxhwdoOedZt2FPwAZReu8UxxvTP1Z/UOvOrcyBuMQx+kX8sWErRaQdDFHQBsre2n9U94BGgd8iC1rS7sx/JqjuaVAub9WUYjo7WgtTP7EpU1MqXHWhK0WspgwP18GgBbam6ivLruzWdsIp6SoNVV7TqlNQ37BnuZBiBf6cYQ7/opPadlSo+1JGi1VP5BdJd/pEy5s1/1rl/BNFL6tbo64/isuN5X17IAdDrdFaPjpYloLQlaLZW9BYD9jjehx5FL5Q2C1oXDUH7JToUT9mRpfFZDo+qGPsiUHutJ0GqpuqCV5TEYgIuldetqeXWDoBu1n3O2W3qmuI4VV9RwJK8EaLymBciUnmaQoNUS+hpT8++M760A9TUtkH6tLmz3qUKUgt7dPAn2dms0XTdvV5nSYyMJWi1xPhOqy8Ddn8qAGAAKSq9YwdTUryWDTLsaa5qGRjKlxzYStFoi+zvt36iRBHq7Aw125YmqGxmfdxAuy92hrqSxQaWWjLyiX0um9DRNglZL1PVn0WvUFWvFXxG0vEMhoA+gIGdH+5dP2EVJZQ2HzhcD1tW0jFN6LpRW8UO+TOlpigSt5qqphDM7tZ97jbpi09YGG1yY+rWkidhVpJ/6EYOCqEAPQn0b788yunJKj9xFbJoEreY6uxtqK8ErBIKiTYMHC8ob3AEyTp6W8Vpdhi39WUajZLyW1SRoNdcVTUN0uvqdphvuf2isaeXug8qSdiygsJcd2dceVGqJsV9LpvQ0TYJWc10ZtKhfJ76kstZ8ayjfHuAXCcpQ35wU162yqloOnqvrz7IwSbox0SFeBMuUHqtI0GqOqjI4p63sYAxaPm7OODnoAG0PRDNRsr5WV5F++kf0BkVEgDvd/dytfp5Op+MO0y490q91LRK0miNnBxhqwa8n+EcB4OCgI9DURGwQtGQeYpfRnP4sI9OUHhmvdU0StJrDOD6rrpZl1GhnvLFf6/xeqC5HXL92moKW9f1ZRsYpPYdzZUrPtUjQag5Tf9Zos8OBxqDVsDPeLxJ8emi1M+nXum5VVNey/6zWn3WbDf1ZRt28XblJpvQ0SYKWrS7/qN0JBIgaaXbKdAexrEHzUKeTeYhdQPrpH6k1KLr7uRMR4NGsPIwLA8qUnsZJ0LLV6e2AgqBo8AkzO2VsHl6yVLW/Tvu1th4r4MWvDrE3R+54mdbPakbT0GjkDTKlpynNClqLFi0iKioKNzc34uLi2LVr1zXTr1ixgpiYGNzc3BgwYABr1641O79y5UruvvtuAgMD0el0ZGZmXpXHT37yE3Q6ndnjySefbE7xW6bBUIcr1de0LAQt4x3Ec+lQc7mtSteudpy8xP98sptPtp/i/v/dzn2LtrFq7zmqaw32Lppd7Mw2rgff/KAlU3qaZnPQWr58OcnJySxYsICMjAwGDRpEQkICFy5Y3sBh+/btTJ48menTp7N3714SExNJTEzk4MGDpjTl5eXccccdvP7669e89hNPPEFubq7p8cYbb9ha/Ja7ZtCqq2k1HPIAENAbvEJBX62Npu/kfsgvZcane6jWG+gb7IWLowP7zhTx7PJMbn99Iws3/MDFhn1717HL1Xr2nWl+f5aRTOlpms1B689//jNPPPEE06ZN46abbuLDDz/Ew8ODv//97xbTv/POO4wdO5bf/OY39OvXj1deeYUhQ4bw/vvvm9I8+uijzJ8/n/j4+Gte28PDg9DQUNPDx8fH1uK3TNkFbSVSuKo/C+o74i1+WK+jfq3c4stM/fsuSiprGRbpz3+fuoPtc+/i1z+NJtjblYulVSzccIwRr6WSvDyT/WeL7F3kNrc350eq9QZCfdzo2cz+LCPTUjUypccim4JWdXU16enpZsHFwcGB+Ph40tLSLD4nLS3tqmCUkJDQaPpr+eyzzwgKCuLmm29m7ty5VFRUNJq2qqqKkpISs0eLnfpe+zd0gLbjTgPG5uHJi+X8fWs2tfoGzaTroF+rpLKGaUt2k1tcSZ9unnw8dRhuzo4Eebny1Ji+bJ1zF+9OHsyQnn7U6BUr955jwvvbeOB/t/HVvvPUNHxNrhNXTt3R6XQtyss4pWeXTOmxyMmWxAUFBej1ekJCQsyOh4SEcPToUYvPycvLs5g+Ly/PpoL+/Oc/JzIykvDwcPbv38+cOXPIyspi5cqVFtOnpKTw0ksv2XSNJjUy1MGob7A3Q3r6kZFTxMurD7N89xlenNCf4X3qmgvGfq2zu6G2Cpyu3qGlI6uq1TPz03SO5pUS7O3KP/4nFj8PF7M0Lk4OTBgUzoRB4ew7U8Qn20+xev95MnKKyMjZS4iPK7+Ii2RyXE+LO9R0VjtbMKi0IeOUngulVaSf/tE0fktoOs3dwxkzZpCQkMCAAQNISkri008/5csvv+TEiRMW08+dO5fi4mLT48yZMy0vxDX6s0D7wK54cgSv3j8Afw9nsvJLmfzRDmYtzeB80WXtjqNnN211iHPpLS9POzIYFM+v2E/ayUt4uTqxZNqt9PC/djNoUIQff5l0C9t+dxfPxvclyMuV/JIq3l7/AyNSNvLrL/aZ5ul1ZpU1evaeKQKsW/SvKTKl59psClpBQUE4OjqSn59vdjw/P5/Q0FCLzwkNDbUpvbXi4uIAOH78uMXzrq6u+Pj4mD1apOgMFJ4EnWP9DtIWODro+HlcTzY9/xOmDI/EQQdr9ucy5u3veH/TcfQRdc/tZP1ar607yn/3ncfJQceHvxhK/3Bfq58b7O3Gs/HRbP/dXSycdAuDevhSrTfw74yz/Oy9rfx33/k2LHnbyzxTRHWtgW7ervQK8myVPGVKT+NsClouLi4MHTqU1NRU0zGDwUBqairDh1v+IA8fPtwsPcD69esbTW8t47CIsLCwaydsLcb+rO5DwK3pAOjn4cLL993M6qdGEhsVwOUaPW99+wPvnaxrKneifq2/bc1m8ZaTALz58EBTLcBWLk4OJA7uzn9m38GXvxpBfD/ttVjw1aGrJ5l3IleOz2ppf5bRlVN6utJdWGvY3DxMTk7mo48+4h//+AdHjhzhl7/8JeXl5UybNg2AKVOmMHfuXFP6Z555hnXr1vH2229z9OhRXnzxRfbs2cPs2bNNaQoLC8nMzOTwYe3OXFZWFpmZmaZ+rxMnTvDKK6+Qnp7OqVOn+Oqrr5gyZQqjRo1i4MCBLXoBrGZsGlq4a3gtN4X7sHzmbbzzyC2E+LjydWkfACqz08jOL2rlQra+Nftz+eMa7f9lztgY7h/co1XyHdzTnw9+MYQbQ7wpLK/mT2uOtEq+9lA/Pqvl/VlGMqXnGlQzvPfee6pnz57KxcVFxcbGqh07dpjOjR49Wk2dOtUs/RdffKGio6OVi4uL6t+/v1qzZo3Z+SVLlijgqseCBQuUUkrl5OSoUaNGqYCAAOXq6qpuuOEG9Zvf/EYVFxdbXebi4mIF2PQcE4NBqbf7KbXAR6njG21/fp3SyhqVsuaQKpwfrtQCH/XQC++o178+osoqa5qdZ1tKO1Gg+v5+rYqcs1rNX3VAGQyGVr9G+ulCFfW71Spyzmq19djFVs+/rVXW1KroF7TX6Fh+Savm/erawypyzmqVvDyzVfPtqKz9jOqU6hpzBUpKSvD19aW4uNj2/q1LJ+C9IeDoAnNOg0vLxuGU/WMSXtnreK3mET7UTyDUx43fj+vH+IFhrda8aKkf8kt56IPtlFTWktA/hP9NGoqjQ9uUbcF/DvKPtNNEBnrwzbOjcHN2bJPrtIXdpwp5+MM0grxc2P1CfKv+/207XkDSxzsJ9nZl5+/HdJj3Rlux9jPaae4e2pVxKZoesS0OWABe0dqQiccjzhMR4E5eSSVPf76XRxbv4Eiu/Zdkbjh49J1HBrdZwAJ4PuFGQn3cOH2pgndTj7XZddqCcahDbCv2ZxkNjfTHzVmm9DQkQcsaTQx1sFndyPigwr2sf+Z2kn8ajZuzAzuzCxn37vf89l/7yCuubJ1r2aixwaNtydvNmZfv6w/A4i0nO0TgtpZxf8PWGJ/VkJuzI7G9ZEpPQxK0mmIwQHbdncPWClohN4OrL1SX4lZwiKfH9GVD8mjGDQjDoOCLPWf5yVubeOubLEora1rnmteir4XqCqsGj7aVu/uHMrZ/KLUGxdyVB9AbOn6vRY3ewJ5T2uoWLZlveC0ypedqErSacvEIVBSAswd0H9o6eTo4QmTdkI+6oQ89/D1YlDSElb8awbBIfyprDLy/6Tij39zMP7afarvpL0rBZw+hXovg0DsPUp29HS9XR6sGj7a2Fyf0x9vVicwzRfxzx+l2vXZz7D9bzOUaPf4ezvQN9mqTa5h26TnZ/Ck9SikOnitmc9YF0k//yPELpVwoqeRytb5TLn9j0zSeLsnYNOw5HJxasdYReTv8sE4bZDriKdPhIT39WfHkcL49nM/rXx/lZEE5C746xCfbT/HbhBsZe3No6/adHPkKTm5CBwwp3cS/XTdRFtAfrwuzodsD4Nz0ZqOtJdTXjd/eE8O8VQd5Y91RfnpTCOE2bA7R3oxDHWJ7BeDQRn1+V07p2XPqR5vGyBVVVLNq7zmW7T7D0bxSi2mcHXV4uznj4+ak/evuhI+bM95uxn+1Y0Fe2sDZXkGeeLraN2xI0GpKazcNjYyTp3O2g0Gv1b7q6HQ6EvqHcldMMMt3n2Hhhh/ILijnl59lMKSnH7+/tx/Dolo+XaSkvAKnr+fjASytvRMHFA+7pOFVeAhW/RK+/QMMnQa3Tgef8BZfzxpJsT1Ztfcc6ad/ZP5/DvHRlKEd9q6ZcVBpWzUNQXsvjOzbjX9nnOX74xebDFpKKXacLGTZ7hy+PphnWtvM1cmB3t28KKuqoeRyLaWVNRgU1OgVheXVNg3uDfauD2BXPnoGeuDq1PZ3fiVoXYtBX7/tV2sHrbBB4OIFlcWQfwjCrh4k6+zowC9uiyRxcHcWbznJR1tOkpFTxEMfppHQP4Tfjo2hTzfrmyUFZVXszi5kZ3Yhu08VMiT/37zifIoC5cOfan/B7LGDcYz1h4x/wK6PoeQsfP8WbFsI/SZA3EyIiNOW2WkjDg46Uh4YwLh3v2fDkXy+OZTH2JvbadaDDWr1BvacartO+CuNig7SgtYPBcy9x3KaC6WV/Cv9LF/sPsOpS/Wrn/QL82FybAT3DeqOr4ez6bhSivJqPaWVWhArqawx/VxaWUNJZS0ll+v+rawhv7iS7IJyLpVXc6G0igulVaabEEYOOuju705UoCe96wJZVJAnvYO86O7v3mp3oGWc1rWcy4CP7tQ6zedkm9WGWsX/PQAnUmHsa3DbL5tMnl9SycINP7B89xkMSpvnODk2gmfGRNPN++oVE84VXWZX9iV2ZReyK7uQExfrdwLyoJLvXJ+jm66YL0Ofwf2OX5HQP6S+VqOvhaw1sPOv5lOOwgZB3JPQv22bjm9/m8V7G48T7O3Khl+PxsfNuekntaPMM0UkLtqGr7sze+f9tM2ah6B92Qz74wYAdr8Qb/q/1hsU3/1wgWW7zpB69ILp5oWniyMTbunO5NgIBnT3bdWaanFFDdmXyjlVUM7JgnKyC7SfswvKKauqbfR5Sx+PY0QTq1VY+xmVmta1mKbu3NH6AQu0oQ8nUrXanBVBK8THjZQHBjLt9l68/vVRUo9e4J87cvgy4xwzR/fh7v4h7M0pMgWpc0VXL+scE+rNrVEBJFV9TrcjxeDfi/sf/8PV/XWOTnDTfdoj74AWvA6s0Db1WPVL+HYeDJsGw/6nTZqOs+68gdX7c8kuKOeNdUf5Y+KAVr9GSxjHZ90a1Xb9WUZBXtqUnsO5JWw7XsDQSH9W7DnDF3vOkldSPzRmaKQ/k26NYNyAsDbrd/L1cOYWDz9uifAzO66U4mJZFacKKsguKOPkFcHs1KUKolppIjlITevaTDWh1+G2NliPPmcn/P1ucA+A35wAB9tu5qaduETK10dM21Y15Oig4+buvsT1CuDWqABujfLXhjCUXYR3b4HqMnhoCdz8gHUXLL+kNR13/01rOgI41AW3u+ZBQC+byt+UtBOXmPzRDgD+9eTwVunHay3TluxiU9ZF/jCuH4+P7N3m10v5+gh//e4k/h7OFF2uwfip9fdw5oEhPZh0awTRId5tXo7m0BsUDjqarPFJTaulaqshp2511dbuzzIKHwxO7nC5EC4ehZCbbHr68D6BrPrV7aw+kMvb32aRW1zJ4Ag/LUj1CmBIT3/L37jfva4FrPDBcFOi9Rf0DISRyTDiafOm48F/w5HV2rnbnwHn1rnjN7xPIBOH9eCLPWeZu/IAa54eiYtTK4zSuXRC66erLKnrn9OBzkH7WefQ4PerzxscnKg5FQH0adNO+CuN6tuNv353kh8rtHF7d9wQxKRbI7i7f0i7dH63RGvPppCg1Zhz6VBTAR5BENyvba7h5AIRsdo0odPbbA5aoHVcG1cKNRhU002VSycgfYn2809ftrl2B5g3HXP3aU3F7O9gcwrs+xzueQOiE2zP14Lf39uPjUcvcOxCGX/97gRPjenbsgzP74V/PggVl5qdhQPwKTo+cRtPv25jWlYeKw3vHchz8dHoleKhIT3oGdi+Y+g6Eglajbly6k5b3nKPukP7wJ/aCrFPtCgrq/pWNr6i7XR9w09bpwYZNgim/AcOfQnf/B5+PAVLJ8KN42BsCvhHtih7Pw8X5o/vz9Of7+W9jce5d2CYTXdMzZz8Dpb9XKtlhg2CW34BKG2ArTLU/Wxo5Hdl+v3E4XT6XPiW/+Er+NtJeGAxhPRv0d/ZFAcHHc/EtzBgXyckaDWmtecbNubKzS6UatsAeS5dCy7oIP7F1stXp9P6xfr+FL57A3b8r9Z8PJEKI5/XBs+24E7j+IFhrMw4y+asi/x+5QGWzbjN9jtih1bByie0Ldx6jYJJn1m1mKMlKad341Ddn3c8l+CefxAW/wTGzIfbZjWv5ipsIq+wJdUVcLZuA9q2Dlrdh4KjK5RfhII2XOFAKVi/QPt50CMQenPrX8PVG+5+BZ7cpi2WWFsJm/4IHwyHYxuana1Op+OV+27G3dmRndmFrNhz1rYM9vwdVjymBax+Ezh9zz9YdaSUr/adZ+PRfHacvMSBs8WcvFhGfkmlNvCykbmPeoNiV3Yh3xpu5eTDGyB6rJbvt3+ATydAUU6z/05hHalpWXJmp/ZG9OmhbbLalpzdoMetcHqr9ugW3TbXOb5BWzLa0RXufKFtrmEUHANT/6t10H/zgra2/mcPQszPtCajX0+bs4wI8ODXd0fzxzVH+NPaI9wZE2xxbJoZpWDLW1rgBHYG3secU1M49Wfrtq9zd3bE09URT1cnPFyc8HRxxMlRR0llLV6uTtzYpw/0XabdUV33e+31/eB2uPdNGDipbWvNXZjUtCwxNQ1Hts8br603cTXo62tZcTPAL6JtrnMlnQ4GPASzd8Pw2dqGIEdXw/uxWiCptX3d88dGRHFzdx+KL9fwyurDjaarrjWw88RFMv46wxSw3qm9n0nnJnLqxyqcHHQM6enHbb0DGNjDl97dPAn1ccPbzYkruwUv1+gpKKvm9KUKjuSWsOf0j+yom7ozvE8gTo51dxSHPgZPfq99+VSVwJczYcVUqCi0UDrRUjJOy5KPxsC5PZD4Adzy87Yv3MnvtKaFdxgkH2n9QJn5Oax6Etx84elMixvNtrn8w7D2+frR9QF9tBrJDbbdfTt4rpgJ72/FoGDJtFu588ZglFKcuFjO98cusvVYAekn83lJLeI+x+0AvFgzhe8DHmRk326M7BtEXO9AvBoZfKmUoqrWQFlVLRVVesqraymvqqW8Wk9FVS1lVbXU6BXxNwUT7N2gn05fC9v+Aptf0252eIXCfYug77V3Thcaaz+jErQaqiyG16O0u0bPHmyfWkl1BbzWEww18FQGBPZpvbxrKuH9YVB8BuJfgjuebb28baWUNqr+2z9AWd22cv3Gw4hnoMcwq4P1n9Yc5qPvs+nu586IPoFsPV5Abt2iie5U8qHzQkY77kePI7sHv0rP0VPbd7WI83th5Qwo+EH7/dbHteElLq03Kvx6JMstN9fpNC1gBfRun4AF2hLOxrW6WntrsV2LtYDl012b8GxPOh0MnKg1GW/7ldZkPPJf+Fs8LIqDrQuhtOmdx5/7aTTd/dw5V3SZFelnyS2uxMXJgYRezmwO/jOjHfejnD1wTPqC2+57sv2XtwkfDDO3aHM0AXZ/DH8dBWc71wa9HZXUtBr68bQ2LMDZvX0/5Kkvw/dvw8BH4IG/tk6el3+Ed26ByiKtmTL4F62Tb2vJOwjb34PD/4HaunmSOke4IV4ra/TYRtcwSz/9Iws3/EBMqDcj+3YjNqACt2UPQ0EWuPvDz1dAxK3t+Mc04sRGWPUrKM3V/rbRv4XBj4K+Spt1Yfq32sKxKq3vT19d92+NdqOmb0K7rnPWXqR52ECLduNpD8dT4Z8PaP0gMzaDTyssx7J+Pmx7B4Jvgie3ts2k79ZQWaJ9Uez9Z/1QEwCPQBgwEQYnQeg1Jkxf/AH+735tPqRPd/jFSu0OZkdRUQhrfg2HVrZOfm6+2iobgx5p86WCqK3SprPpayFyRKts7NIYCVoNdPigVVUG7wzUppe4+kD8Ahj6P80frFh8Ft4don1b//yLVptW0+Yu/gCZn8G+ZVB2RVMxdKBW+xrwsPmNhLPp8NlD2vzNwL7w6Jft16y31YF/aUNAKi6Bk6u2JZ2ji1abdHStP2Z27opjOgc4uRlKztXn6d9LC14DJ7XehPWSXDj2rfY4sQlq6pY0cnKH3j+BG8dqtWDv0Na5nvGyErTMdfigBdpigF89pY1cB+1bdPw7zZv7uOpX2oc/8nZ4bE3nGzOkr9WaVpn/hKNrtZsUoH2Ab7xHm4Kjc4AvpmgfqvAhkPQvbVL39cy4MOW+ZVqz2hhQQFsSfNAj2iR4dz/b8jyXAce+gR++gbz95ue9QsDBuX5lD6PwIdr/xY33aJu1tPA9JkGrgU4RtEB7A+3+WOvjqi7T3ix3PKtNh7G2HyP/kDbIEQWPp2p35jqzikLtruPef179gQLofSdM+ie4ts3mEh1Wdbm2usa+z7UaGHUfZUdXiLkXBk2GPneBo4UFFC8XaV8KP3wDx9c3mECug+5DtL6z6LshdJAWkPIPQtY6yFoL5zPM8/PpodXAbrxHmw3h1MTAXwskaDXQaYKWUfFZWPsb7Q0C2rim8Qutm1b02UTtW/Om+2Dip21azHaXu1+rQe7/QmsS9r8f7v9rsz4k15WS89prsm+ZtoOUkUeQ1qQe9Ag4udXVpr7V+qnUFbv7uPpoAS46QZtM79Xt2tcrzdMC3g/rtCak8UYKaMuI97lLC2B97wZP6zbjkKDVQKcLWqCNazryXy14Gft3bkmCu//Y+ADRU1vhk3HanapZuyDohvYrb3uqrYJLx7WbDJ2t6duWlNJqo/uWabXT8mts8hp0o1aT6psAPW+zXCOzRs1lbYD0D19rNbEr+yLRacsvJbzaZI1fglYDnTJoGVUWw4aXYM/ftN89ArV15Qc8bP6BVQo+HqP1iQ2bDj/7s33KKzoGfY3WBNy3DI6u0Y71GqkFqb4/bfWVZgFtc+PcTK0GlvV1fXN+1u4m59VK0GqgUwcto5wd8N9ntFVOQauCj/tz/Zvv0CptzpuzJzyTCV7B9iqp6GhqLgO69h/fVXxWm8s7aHKTNWIZEX896nkbzPwe7vqD1tl6YiP873BtJHnNZa3zHrT1qyRgiSs5u9tnQKpvD23+bis24aWm1VkVHIfVz2rLoQB4BkP5BfDsBk/v1da2EqITkZrW9S7oBm3NqvsWgZufFrAARs+RgCWua7IIYGem02mjxPsmwOZXtY7XIVPtXSoh2pQEreuBVzf42V/sXQoh2oU0D4UQnYoELSFEpyJBSwjRqUjQEkJ0Ks0KWosWLSIqKgo3Nzfi4uLYtWvXNdOvWLGCmJgY3NzcGDBgAGvXrjU7v3LlSu6++24CAwPR6XRkZmZelUdlZSWzZs0iMDAQLy8vHnzwQfLz85tTfCFEJ2Zz0Fq+fDnJycksWLCAjIwMBg0aREJCAhcuXLCYfvv27UyePJnp06ezd+9eEhMTSUxM5ODBg6Y05eXl3HHHHbz++uuNXve5557jv//9LytWrOC7777j/PnzPPDAA7YWXwjR2SkbxcbGqlmzZpl+1+v1Kjw8XKWkpFhMP3HiRDVu3DizY3FxcWrmzJlXpc3OzlaA2rt3r9nxoqIi5ezsrFasWGE6duTIEQWotLQ0q8pdXFysAFVcXGxVeiFE+7L2M2rTOK3q6mrS09OZO3eu6ZiDgwPx8fGkpVnetTctLY3k5GSzYwkJCaxatcrq66anp1NTU0N8fP3+cTExMfTs2ZO0tDRuu+22q55TVVVFVVX9hqDFxcWANlVACNHxGD+bqomZhTYFrYKCAvR6PSEhIWbHQ0JCOHr0qMXn5OXlWUyfl9f0VlFX5uHi4oKfn5/V+aSkpPDSSy9ddTwiooOuHy6EAKC0tBRfX99Gz1+3I+Lnzp1rVsMzGAwUFhaaOvuvpaSkhIiICM6cOXN9TK5uQ/Ja2UZer8YppSgtLSU8PPya6WwKWkFBQTg6Ol511y4/P5/QUMs7c4SGhtqUvrE8qqurKSoqMqttXSsfV1dXXF3Nl+BtWFNrio+Pj7yxrCSvlW3k9bLsWjUsI5vuHrq4uDB06FBSU1NNxwwGA6mpqQwfPtzic4YPH26WHmD9+vWNprdk6NChODs7m+WTlZVFTk6OTfkIITo/m5uHycnJTJ06lWHDhhEbG8vChQspLy9n2rRpAEyZMoXu3buTkpICwDPPPMPo0aN5++23GTduHMuWLWPPnj0sXrzYlGdhYSE5OTmcP38e0AISaDWs0NBQfH19mT59OsnJyQQEBODj48NTTz3F8OHDLXbCCyGuY825Nfnee++pnj17KhcXFxUbG6t27NhhOjd69Gg1depUs/RffPGFio6OVi4uLqp///5qzZo1ZueXLFmi0PY/MnssWLDAlOby5cvqV7/6lfL391ceHh7q/vvvV7m5uc0pfpMqKyvVggULVGVlZZvkfz2R18o28nq1XJdZuVQIcX2QuYdCiE5FgpYQolORoCWE6FQkaAkhOhUJWkKITkWCVgO2rhXWVb344ovodDqzR0xMjL2L1WFs2bKF8ePHEx4ejk6nu2qBAKUU8+fPJywsDHd3d+Lj4zl27Jh9CtvJSNC6gq1rhXV1/fv3Jzc31/TYunWrvYvUYZSXlzNo0CAWLVpk8fwbb7zBu+++y4cffsjOnTvx9PQkISGBysrKdi5pJ2TncWIdiq1rhXVlCxYsUIMGDbJ3MToFQH355Zem3w0GgwoNDVVvvvmm6VhRUZFydXVVn3/+uR1K2LlITauOca2wK9fsamqtsK7u2LFjhIeH07t3b5KSksjJybF3kTqF7Oxs8vLyzN5rvr6+xMXFyXvNChK06lxrrTBb1v7qKuLi4vjkk09Yt24dH3zwAdnZ2YwcOZLS0lJ7F63DM76f5L3WPNftelqibd1zzz2mnwcOHEhcXByRkZF88cUXTJ8+3Y4lE9c7qWnVac5aYaKen58f0dHRHD9+3N5F6fCM7yd5rzWPBK06zVkrTNQrKyvjxIkThIWF2bsoHV6vXr0IDQ01e6+VlJSwc+dOea9ZQZqHV2hqrTBR7/nnn2f8+PFERkZy/vx5FixYgKOjI5MnT7Z30TqEsrIys1pndnY2mZmZBAQE0LNnT5599ln++Mc/0rdvX3r16sW8efMIDw8nMTHRfoXuLOx9+7KjudZaYaLepEmTVFhYmHJxcVHdu3dXkyZNUsePH7d3sTqMTZs2WVwjzrjWnMFgUPPmzVMhISHK1dVVjRkzRmVlZdm30J2ErKclhOhUpE9LCNGpSNASQnQqErSEEJ2KBC0hRKciQUsI0alI0BJCdCoStIQQnYoELSFEpyJBSwjRqUjQEkJ0KhK0hBCdyv8DqPnY4j/qcjcAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:12:22.112173Z", + "iopub.status.busy": "2024-02-29T18:12:22.111872Z", + "iopub.status.idle": "2024-02-29T18:13:10.266130Z", + "shell.execute_reply": "2024-02-29T18:13:10.265079Z" + }, + "papermill": { + "duration": 48.176855, + "end_time": "2024-02-29T18:13:10.268708", + "exception": false, + "start_time": "2024-02-29T18:12:22.091853", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:13:10.306535Z", + "iopub.status.busy": "2024-02-29T18:13:10.306237Z", + "iopub.status.idle": "2024-02-29T18:13:10.326535Z", + "shell.execute_reply": "2024-02-29T18:13:10.325555Z" + }, + "papermill": { + "duration": 0.041427, + "end_time": "2024-02-29T18:13:10.328507", + "exception": false, + "start_time": "2024-02-29T18:13:10.287080", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat3.508772e-080.5193350.0202650.5646030.1916840.9978110.2846430.0000150.8755850.0959150.7516150.1423540.0447771.056311.440188
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 3.508772e-08 0.519335 0.020265 0.564603 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.191684 0.997811 0.284643 0.000015 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 0.875585 0.095915 0.751615 0.142354 0.044777 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 1.05631 1.440188 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:13:10.364482Z", + "iopub.status.busy": "2024-02-29T18:13:10.364203Z", + "iopub.status.idle": "2024-02-29T18:13:10.805989Z", + "shell.execute_reply": "2024-02-29T18:13:10.805132Z" + }, + "papermill": { + "duration": 0.46239, + "end_time": "2024-02-29T18:13:10.808109", + "exception": false, + "start_time": "2024-02-29T18:13:10.345719", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:13:10.846228Z", + "iopub.status.busy": "2024-02-29T18:13:10.845942Z", + "iopub.status.idle": "2024-02-29T18:14:00.857691Z", + "shell.execute_reply": "2024-02-29T18:14:00.856788Z" + }, + "papermill": { + "duration": 50.033773, + "end_time": "2024-02-29T18:14:00.860332", + "exception": false, + "start_time": "2024-02-29T18:13:10.826559", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:14:00.900303Z", + "iopub.status.busy": "2024-02-29T18:14:00.899498Z", + "iopub.status.idle": "2024-02-29T18:14:00.917131Z", + "shell.execute_reply": "2024-02-29T18:14:00.916436Z" + }, + "papermill": { + "duration": 0.039377, + "end_time": "2024-02-29T18:14:00.919028", + "exception": false, + "start_time": "2024-02-29T18:14:00.879651", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:14:00.954954Z", + "iopub.status.busy": "2024-02-29T18:14:00.954427Z", + "iopub.status.idle": "2024-02-29T18:14:00.959497Z", + "shell.execute_reply": "2024-02-29T18:14:00.958718Z" + }, + "papermill": { + "duration": 0.02512, + "end_time": "2024-02-29T18:14:00.961366", + "exception": false, + "start_time": "2024-02-29T18:14:00.936246", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.05494664395581067}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:14:00.997456Z", + "iopub.status.busy": "2024-02-29T18:14:00.996822Z", + "iopub.status.idle": "2024-02-29T18:14:01.377020Z", + "shell.execute_reply": "2024-02-29T18:14:01.376133Z" + }, + "papermill": { + "duration": 0.400488, + "end_time": "2024-02-29T18:14:01.379186", + "exception": false, + "start_time": "2024-02-29T18:14:00.978698", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS4AAAE8CAYAAAB+XlzTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABF5UlEQVR4nO3deXxTVdoH8N/NnrZJ2tKdrlAW2SsUBJVFeQVEB0bHcUFtRdQZYVAZ5xVmRhDQYZhRBFFxxndoZUSdccRlRhEYFHAFAdmVpUApLV2gbbpnu+f94za3WdskTZqkeb6fTz5pbk6Sk7R5es65zzmHY4wxEEJIGJEEuwKEEOItClyEkLBDgYsQEnYocBFCwg4FLkJI2KHARQgJOxS4CCFhhwIXISTsUOAihIQdCly9UHZ2Nm655ZaAvgbHcXjmmWe6LPfMM8+A47iA1oVEHgpcIejrr7/GM888g/r6+mBXhYSBV199FcXFxcGuRo+iwBWCvv76ayxfvpwCF/EIBS5CCAkDFLhCzDPPPIPf/OY3AICcnBxwHAeO43D+/HkUFRXhhhtuQFJSEpRKJYYMGYINGza4fa7t27dj1KhRUKlUGDJkCLZs2eJ1fQwGA5544gkkJiZCo9HgJz/5CS5evOiy7Jdffon8/HyoVCr0798ff/nLX1yW4zgOCxYswObNmzFo0CCoVCqMHj0ae/bscfosOI7DqVOncO+990Kn0yExMRFPP/00GGMoKyvDrFmzoNVqkZKSghdeeMHr9wcAW7duxaRJk6DRaKDVapGfn4+33nrLrsy7776L0aNHQ61WIyEhAffeey/Ky8vtyhQWFiImJgbl5eWYPXs2YmJikJiYiCeffBIWi8WuLM/zWLduHYYPHw6VSoXExERMnz4d+/fvF8t48vvOzs7G8ePHsXv3bvFvZfLkyT59DmGFkZBy+PBhdvfddzMA7MUXX2R///vf2d///nfW1NTE8vPzWWFhIXvxxRfZ+vXr2U033cQAsJdfftnuObKystjAgQNZbGwsW7x4MVuzZg0bPnw4k0gkbPv27V7V595772UA2D333MNefvlldtttt7ERI0YwAGzZsmViuSNHjjC1Ws0yMzPZqlWr2MqVK1lycrJY1hYANmzYMJaQkMBWrFjBVq9ezbKyspharWZHjx4Vyy1btowBYKNGjWJ33303e/XVV9nMmTMZALZmzRo2aNAg9stf/pK9+uqr7Nprr2UA2O7du716f0VFRYzjODZs2DD23HPPsVdeeYXNmzeP3XfffXZlALD8/Hz24osvssWLFzO1Ws2ys7NZXV2dWK6goICpVCo2dOhQNnfuXLZhwwZ2++23MwDs1VdftXvdwsJCBoDNmDGDrV27lj3//PNs1qxZbP369WIZT37f77//PktPT2eDBw8W/1a8/R2HIwpcIejPf/4zA8DOnTtnd7ylpcWp7LRp01i/fv3sjmVlZTEA7L333hOP6fV6lpqayvLy8jyux6FDhxgA9uijj9odv+eee5wC1+zZs5lKpWKlpaXisRMnTjCpVOoycAFg+/fvF4+VlpYylUrFfvrTn4rHrIHr4YcfFo+ZzWaWnp7OOI5jf/zjH8XjdXV1TK1Ws4KCAo/fX319PdNoNGzcuHGstbXV7j6e5xljjBmNRpaUlMSGDRtmV+Y///kPA8CWLl0qHisoKGAA2IoVK+yeKy8vj40ePVq8/dlnnzEAbOHChU51sr4uY57/vocOHcomTZrkwTvuPairGEbUarX4s16vx+XLlzFp0iScPXsWer3ermxaWhp++tOfire1Wi3uv/9+fP/996isrPTo9T755BMAwMKFC+2OP/7443a3LRYLtm3bhtmzZyMzM1M8ftVVV2HatGkun3v8+PEYPXq0eDszMxOzZs3Ctm3bnLpV8+bNE3+WSqUYM2YMGGN48MEHxeOxsbEYNGgQzp4969F7A4AdO3agsbERixcvhkqlsrvPmsKxf/9+VFdX49FHH7UrM3PmTAwePBgff/yx0/P+4he/sLt9/fXX29XrvffeA8dxWLZsmdNjbVNHvPl9RxoKXGHkq6++wtSpUxEdHY3Y2FgkJibit7/9LQA4/SHn5uY65U8NHDgQAHD+/HmPXq+0tBQSiQT9+/e3Oz5o0CC72zU1NWhtbcWAAQOcnsOxrJWrsgMHDkRLSwtqamrsjtsGQwDQ6XRQqVRISEhwOl5XV+f+DTkoKSkBAAwbNsxtmdLSUgCu38fgwYPF+62s41W24uLi7OpVUlKCtLQ0xMfHd1o/b37fkUYW7AoQz5SUlODGG2/E4MGDsWbNGmRkZEChUOCTTz7Biy++CJ7ng13FgJFKpR4dAwAW5JXI3dXLW5H8+/YEBa4Q5CrT/N///jcMBgM++ugjuxbI559/7vI5zpw5A8aY3XOdOnUKgHAmyhNZWVngeR4lJSV2LY6TJ0/alUtMTIRarcbp06ednsOxrJWrsqdOnUJUVJRTiyVQrC3JY8eOITc312WZrKwsAML7uOGGG+zuO3nypHi/t6+7bds21NbWum11efP7jsSZCdRVDEHR0dEAYJeAav1Pbtui0Ov1KCoqcvkcFRUVeP/998XbDQ0N2LRpE0aNGoWUlBSP6jFjxgwAwEsvvWR3fO3atXa3pVIppk2bhg8++AAXLlwQj//www/Ytm2by+f+5ptvcPDgQfF2WVkZPvzwQ9x0001+a7V05aabboJGo8GqVavQ1tZmd5/1cx4zZgySkpLw2muvwWAwiPdv3boVP/zwA2bOnOn1695+++1gjGH58uVO91lf15vfd3R0dMQlK1OLKwRZB61/97vf4a677oJcLsfEiROhUChw66234pFHHkFTUxNef/11JCUl4dKlS07PMXDgQDz44IP47rvvkJycjI0bN6KqqsptoHNl1KhRuPvuu/Hqq69Cr9djwoQJ2LlzJ86cOeNUdvny5fj0009x/fXX49FHH4XZbMb69esxdOhQHDlyxKn8sGHDMG3aNCxcuBBKpRKvvvqq+Dw9RavV4sUXX8S8efOQn5+Pe+65B3FxcTh8+DBaWlrwxhtvQC6XY/Xq1XjggQcwadIk3H333aiqqsK6deuQnZ2NJ554wuvXnTJlCu677z689NJLOH36NKZPnw6e5/HFF19gypQpWLBgAW666SaPf9+jR4/Ghg0b8OyzzyI3NxdJSUlOrcNeJ3gnNElnVq5cyfr27cskEomYGvHRRx+xESNGMJVKxbKzs9nq1avZxo0bnVInsrKy2MyZM9m2bdvYiBEjmFKpZIMHD2bvvvuu1/VobW1lCxcuZH369GHR0dHs1ltvZWVlZU7pEIwxtnv3bjZ69GimUChYv3792GuvvSamNNgCwObPn8/efPNNNmDAAKZUKlleXh77/PPP7cpZH1tTU2N3vKCggEVHRzvVddKkSWzo0KFev8ePPvqITZgwganVaqbVatnYsWPZ22+/bVfmH//4B8vLy2NKpZLFx8ezOXPmsIsXL3pUL1efgdlsZn/+85/Z4MGDmUKhYImJiWzGjBnswIEDdvXy5PddWVnJZs6cyTQaDQMQEakRHGO0ryLpWRzHYf78+Xj55ZeDXRUSpmiMixASdmiMK0J1lYSqVquh0+l6qDb+V1NT45TIakuhUHSZR0VCFwWuCJWamtrp/QUFBWG9VEp+fr5TcqitSZMmYdeuXT1XIeJXFLgi1I4dOzq9Py0tLWCv3RPDqps3b0Zra6vb++Pi4gJeBxI4NDhPCAk7NDhPCAk7vb6ryPM8KioqoNFoInJqBCGhjDGGxsZGpKWlQSLxvB3V6wNXRUUFMjIygl0NQkgnysrKkJ6e7nH5Xh+4NBoNAOGD0Wq1Qa4NIcRWQ0MDMjIyxO+pp3p94LJ2D7VaLQUuQkKUt8M4NDhPCAk7FLgIIWGHAhchJOz0+jEuTzDGYDabO53bRlyTSqWQyWSUakJ6VMQHLqPRiEuXLqGlpSXYVQlbUVFRSE1NhUKhCHZVSISI6MDF8zzOnTsHqVSKtLQ0KBQKajl4gTEGo9GImpoanDt3DgMGDPAqiZAQX0V04DIajeB5HhkZGYiKigp2dcKSWq2GXC5HaWkpjEaj0/6EkeJYuR4lNU2YelUyopUR/bXqEfTvEaBWQjdF+ufH8ww7TlThbE0zjlyM7P0Oe0pk/8UR4gd1LUbx52StMog1iRwUuAjppmaDcDY6IUaBfokxQa5NZKDARbqUnZ3ttJci6dBsNAMAohQ0ttVTKHAR0k0t7YHLwhhqGg1dlCb+QIErQhiNxq4LEZ9Yu4rlda1489tSWHhaVDjQKHC5YDTzbi9mC+9xWZOHZX0xefJkLFiwAAsWLIBOp0NCQgKefvppcT337OxsrFy5Evfffz+0Wi0efvhhAMCXX36J66+/Hmq1GhkZGVi4cCGam5vF562ursatt94KtVqNnJwcbN682af6RZJ+idGYODBBvG0w0wyMQKNOuQuvfO68xbxVTkI0Zuf1FW//dU8JTBbX/2HT49S4Y0zHIoYbvzqHVqPzH/UT/zPQp3q+8cYbePDBB7Fv3z7s378fDz/8MDIzM/HQQw8BAJ5//nksXboUy5YtAwCUlJRg+vTpePbZZ7Fx40bU1NSIwa+oqAgAUFhYiIqKCnz++eeQy+VYuHAhqqurfapfpEiPi0J6XBS+PVsLo5mHwcQjiiYRBBQFrjCWkZGBF198ERzHYdCgQTh69ChefPFFMXDdcMMN+PWvfy2WnzdvHubMmYPHH38cADBgwAC89NJLmDRpEjZs2IALFy5g69at2LdvH/Lz8wEAf/vb33DVVVf1+HsLR0qZRAhcPraiiecocLkwf0qu2/skDjOCHp7Y321Zx9lDc6/N6U61nFxzzTV2U5TGjx+PF154QZwsPmbMGLvyhw8fxpEjR+y6f4wxcerTqVOnIJPJMHr0aPH+wYMHIzY21q/17m0u6VvBWMdieG0m6ioGGgUuFxQyz4f+AlXWH6Kjo+1uNzU14ZFHHsHChQudymZmZuLUqVM9VbVe5b8nqnC5qePkh+PYJvE/ClxhbO/evXa3v/32WwwYMABSqdRl+auvvhonTpxAbq7rFuXgwYNhNptx4MABsat48uRJ1NfX+7XevY21a6iSS9FmssBIgSvg6KxiGLtw4QIWLVqEkydP4u2338b69evx2GOPuS3/1FNP4euvv8aCBQtw6NAhnD59Gh9++CEWLFgAABg0aBCmT5+ORx55BHv37sWBAwcwb948qNXqnnpLYckaqIakaXF1Vhz6RNO0n0CjFlcYu//++9Ha2oqxY8dCKpXiscceE9MeXBkxYgR2796N3/3ud7j++uvBGEP//v1x5513imWKioowb948TJo0CcnJyXj22Wfx9NNP98TbCUuMMZjMwlnlqzNjoVHJg1yjyECBK4zJ5XKsXbsWGzZscLrv/PnzLh+Tn5+P7du3u33OlJQU/Oc//7E7dt9993Wrnr2ZmWfg23PnenoMM5LRJ01IN9gmEHPg0NBmEqcAkcChwEVIN1jPICpkEuwvrcXfvjiHb89eCXKter+gBq49e/bg1ltvRVpaGjiOwwcffGB3P2MMS5cuRWpqKtRqNaZOnYrTp08Hp7IhZteuXbRiQwhQyqSYODAB1/TrA4VU+Dq5m0lB/Ceogau5uRkjR47EK6+84vL+P/3pT3jppZfw2muvYe/evYiOjsa0adPQ1tbWwzUlxDW1QorRWfEYnRUHmRi4KB0i0II6OD9jxgzMmDHD5X2MMaxduxa///3vMWvWLADApk2bkJycjA8++AB33XVXT1aVkC7JpULmPAWuwAvZMa5z586hsrISU6dOFY/pdDqMGzcO33zzjdvHGQwGNDQ02F0ICZQ2kwWX9K2obTZCTl3FHhOygauyshIAkJycbHc8OTlZvM+VVatWQafTiZeMjAy3ZQnpror6VryzrwzbjlfaBC5qcQVayAYuXy1ZsgR6vV68lJWVBbtKpBezLhoo5TjI2mfg87SQYMCFbAJqSkoKAKCqqgqpqani8aqqKowaNcrt45RKJZRKmnJBeobZGrgkHGKUMgzvq0OMKmS/Vr1GyLa4cnJykJKSgp07d4rHGhoasHfvXowfPz6INSOkg7XFJZNyiItWYOqQZFzTr0+Qa9X7BfVfQ1NTE86c6Vht9Ny5czh06BDi4+ORmZmJxx9/HM8++ywGDBiAnJwcPP3000hLS8Ps2bODV2lCbFhsWlyk5wQ1cO3fvx9TpkwRby9atAgAUFBQgOLiYvzv//4vmpub8fDDD6O+vh7XXXcdPv3004jd5p2EHmtXUSbhwBiDwczDzDPEKKm7GEhB/XQnT54sbu7gCsdxWLFiBVasWNEzFWIMsJh65rUcSeXOS6a6sWnTJjzxxBOoqKiwG8+bPXs2NBoN/v73vweqlsSBtcUl4Tg0Gy14fc9ZSDgOj00dEOSa9W70b8GWxQR88UJwXvv6XwMyz3ZYuOOOO7Bw4UJ89NFHuOOOOwAIu/N8/PHHna78QPwvLVaFa/r1QaJG2XFWkTFYeEbdxwAK2cF54p5arcY999wj7swDAG+++SYyMzMxefLk4FUsAqXHRWF8/z7ITYqxC1RmnnK5AolaXLakcqHlE6zX9sJDDz2E/Px8lJeXo2/fviguLkZhYaHd5hmkZ8lsAhdtChtYFLhscZzH3bVgy8vLw8iRI7Fp0ybcdNNNOH78OD7++ONgVyviNLSZYDTziFHKoJJLIZVwsPBMHLQngUGBK4zNmzcPa9euRXl5OaZOnUrTm4Lgm5IrOFHRgOsHJGBMdrwYuCw0XzGgaIwrjN1zzz24ePEiXn/9dcydOzfY1YlIjnlc1u4itbgCiwJXGNPpdLj99tsRExNDSblB0pHHJXyVBqZoMDRNS+vPBxh1FcNceXk55syZQ/Mzg8TSfvawPW5hyqCkINYmclDgClN1dXXYtWsXdu3ahVdffTXY1YlYZot9i4v0DApcYSovLw91dXVYvXo1Bg0aFOzqRCzr1mTWMS7hjCIPmURCCagBRIErTLnbN5H0LNu5igCw5eBFXKxrxcwRqRiYrAlm1Xo1ClyEdMOQVC0y4szQqYUEYln7uvNmSocIKApcQKcTvUnXIvnzy8uMs7stbR/rosz5wIroEUW5XPgv2dLSEuSahDfr52f9PCNZRx4XzVUMpIhucUmlUsTGxqK6uhoAEBUVRXP9vMAYQ0tLC6qrqxEbGwupVBrsKvW4+hYjOE5Ytlkq4SDhOgbpSeBEdOACOta2twYv4r3Y2Fjxc4w0m/degNHMY+61OdBFycUWFwWuwIr4wMVxHFJTU5GUlASTKUiLCIYxuVwekS0tK+sgvLR9UF5Mi4jgcb+eEPGBy0oqlUb0F5B4j+eZmMdlbWklapQYmKxBn2iayRBIFLgI8ZHtRGrr2NawvjoM66sLVpUiRkSfVSSkO2zHsWSUJd+jKHAR4iNryoOE4yCxCVysfc15EjjUVSTER9ZULWu2PAAcKK3DnlM1uCpVg+nDUt08knQXBS5CfCSXccjLjBXHtwDA2vCyUP5pQFHgIsRHUQoZJjusv2Vd3obSIQKLxrgI8SPrslwWmvITUBS4CPGRycKjsc2ENpNFPNaxLlewahUZKHAR4qPyulb83xfn8N7Bi+Ixaft4F09nFQMqpAOXxWLB008/jZycHKjVavTv3x8rV66M6GVUSOhwXEQQoCk/PSWkB+dXr16NDRs24I033sDQoUOxf/9+PPDAA9DpdFi4cGGwq0ciXMfWZB3//6OVMuQkRCMuOjw2Fg5XIR24vv76a8yaNQszZ84EAGRnZ+Ptt9/Gvn37glwzQjoSUG1bXMlaFWbn9Q1WlSJGSHcVJ0yYgJ07d+LUqVMAgMOHD+PLL7/EjBkz3D7GYDCgoaHB7kJIIFhbXBKa7tPjQrrFtXjxYjQ0NGDw4MGQSqWwWCx47rnnMGfOHLePWbVqFZYvX96DtSS93afHLqFS34af52cgStHxlXE1xkV6Rki3uP75z39i8+bNeOutt3Dw4EG88cYbeP755/HGG2+4fcySJUug1+vFS1lZWQ/WmPQ2TQYzfrjUiLoWE85UN9ndx/P2W5MBwJUmA17+7DT+uqekR+sZaUK6xfWb3/wGixcvxl133QUAGD58OEpLS7Fq1SoUFBS4fIxSqaRdnYnf6Fs7FpdsbDPb3dcnRolhfXVI1anEYxKOg8nCwHF0VjGQQjpwtbS0QOKwQ7BUKgVPWcmkh9jmYzUZ7ANXTkI0chKi7Y5Zx7sojyuwQjpw3XrrrXjuueeQmZmJoUOH4vvvv8eaNWswd+7cYFeNRIiM+ChMG5qCbccr0ewQuFyR2eRxMcZo85UACenAtX79ejz99NN49NFHUV1djbS0NDzyyCNYunRpsKtGIohGJUOCRgmtyn77NYPZAsYAuVQijnNZrxkDeAZIKW4FBMd6eRp6Q0MDdDod9Ho9tFptsKtDepHtxytxvKIB1+YmYGxOPADAaObxyudnAADzp+RCIQvp819B5+v3M6RbXIQE2xena3D+SgtGZ8ZhSJr9F8vi4qyibWoE37vbBEFFgYuQTuhbTbjcaIDRxXIPrvK4OA5Ij1PbBTPifxS4COmEqT1gff5jNfadu4LCCTli98/aopLaBS4Od4zJ6PmKRhjqgBPSCaO5o6XVbLDAYO5Ye8u6GayMRuB7HAUuQjphG7gAwGTpGLey0JSfoKGuIiGdMDgELrPNWJd1jEvikKv15rel0LeacMfodCRpVSD+R4GLkE44DsqbbDLis/tEITZKjhiV/dfIaOZhNPN2O10T/6LARUgnohUyKKQ8eMbQbLDAZNMCm5Cb4PIx1jEv2hQ2cChwEdKJggnZAIB/fleGZkOruHhgZ6xdR8rjChwKXIR4QBclh8FssVum2WThIeU4p4UErekR1FUMHApchHhg2tAUp2N/3XMWRjOPwgnZdmvM004/gUeBixA3WoxmbDlYDrmUw535mU73i1N+pK5bXLTTT+BQ4CLEDZOFoabRALmLBFPGmNs8rvgYBcw8D6VM2iP1jEQUuAhxw3YzjAOldTh6sR5D0nQYmxNvN37lOC9xyqCkHq1nJKLMeULcsN1+zGCyoK7FJC4maJvqIJPQ16in0SdOiBvWzAepRAJ5+8Rqa0KqNXBxHEAzfnqeT4Hr7Nmz/q4HISHHtsVlHceyTqy2XdLGcXnmL09fxut7zuLghboerG1k8Slw5ebmYsqUKXjzzTfR1tbm7zoREhJsx7jkUuGrYl3mRirhMCA5Bv0SY5weZ7RY0GQwo81kcbqP+IdPgevgwYMYMWIEFi1ahJSUFDzyyCPYt2+fv+tGSFBx4BCtlEItlzoFrhilDLeMSMPNw1OdHmdNUqXNqALHp8A1atQorFu3DhUVFdi4cSMuXbqE6667DsOGDcOaNWtQU1Pj73oS0uMy+0Th4Yn98bPR6WJKhCfZ8NYEVMrjCpxuDc7LZDLcdtttePfdd7F69WqcOXMGTz75JDIyMnD//ffj0qVL/qonIUGllEuhVcsRpRBys1j79mOuWE8yUuZ84HQrcO3fvx+PPvooUlNTsWbNGjz55JMoKSnBjh07UFFRgVmzZvmrnoQEVd9YNR68LgezRvUFAFyobcHa/57G2/suOJW1trhormLg+JSAumbNGhQVFeHkyZO4+eabsWnTJtx8883irtM5OTkoLi5Gdna2P+tKSI86VdWIQxfqkdknCtf062N3X2dBiZa1CTyfAteGDRswd+5cFBYWIjXVeXASAJKSkvC3v/2tW5UjJJga20wor2+FVu38Nels2eYohbCBrEZFE1MCxadPdseOHcjMzBRbWFaMMZSVlSEzMxMKhQIFBQV+qSQhwWDN2ZJKJGgzWbDlYDksPI97r8nqdKOMq1K1uCqVNh8OJJ/GuPr374/Lly87Ha+trUVOTk63K0VIKLCeFRSSTIGqhjZcbjLCzDMxOVVK032CwqdP3d3ZlKamJqhUtDkA6R1sd6qW2wQos4W53AyW9ByvuoqLFi0CIGx6uXTpUkRFRYn3WSwW7N27F6NGjfJrBcvLy/HUU09h69ataGlpQW5uLoqKijBmzBi/vg4hjsw2gUsi4SCVcLDwDCaetwtqji5cacFnP1YhQaPELSPSerTOkcKrwPX9998DEFpcR48ehULRseqjQqHAyJEj8eSTT/qtcnV1dbj22msxZcoUbN26FYmJiTh9+jTi4uL89hqEuGOx2AcnuVQCC2+B2cKgVcmRnRCFhBil0+PMPI+6FhMUjutx8Txw6RBgaAD6jgaUmkC/hV7Lq8D1+eefAwAeeOABrFu3DlptYAcgV69ejYyMDBQVFYnHaAyN9BSJBFDIJGLWvFzKoc0k7K04KEWDQSmuA4/bFVDP7QYufCv8XP0jkP8gIJUHrP69mU9jXEVFRQEPWgDw0UcfYcyYMbjjjjuQlJSEvLw8vP76650+xmAwoKGhwe5CiC9uGJyM+VNyMTorHkDHeJbjXouOxMBlW661Hijba3O7Dqj43q/1jSQet7huu+02FBcXQ6vV4rbbbuu07JYtW7pdMUBYPmfDhg1YtGgRfvvb3+K7777DwoULO021WLVqFZYvX+6X1yfEVrRS5tlcRbHFZXOw4nuAMSA+B0gYCJzaJhxLzxcW9SJe8Thw6XQ6cd0hnU4XsArZ4nkeY8aMwR/+8AcAQF5eHo4dO4bXXnvNbeBasmSJeBIBABoaGpCRkdEj9SW92x1jOv6Oth+vxOnqJkzo3wd5mfZjrmLgsl0e4vIp4Tp1FBDfDzizE2ipBZovAzGJga56r+Nx4LIdZ7L9OZBSU1MxZMgQu2NXXXUV3nvvPbePUSqVUCqdB0wJ8dbuUzWobTZgbE4f9I1V291nsjAYza67jOLqENa7W2qFi0QqtLhkCiAuC7hSAlw5Q4HLBz6NcbW2tqKlpUW8XVpairVr12L79u1+qxgAXHvttTh58qTdsVOnTiErK8uvr0OIK5X6Vpy/3IJWo/OCgB2rozp/heQyCXRqecdUoSslwrUuA5C1/1ON7ydc15f6vd6RwKcpP7NmzcJtt92GX/ziF6ivr8fYsWOhUChw+fJlrFmzBr/85S/9UrknnngCEyZMwB/+8Af8/Oc/x759+/DXv/4Vf/3rX/3y/IR0xjHJdO/ZKzh7uRkj02M7zePSquSYe53N2e+Gi8J1nM0/XF16+30VwtgXjXN5xecVUK+//noAwL/+9S+kpKSgtLQUmzZtwksvveS3yuXn5+P999/H22+/jWHDhmHlypVYu3Yt5syZ47fXIMQdx+DU0GZGpb4NjW0mMai52nPRSUOFcK3t23EsOhGQygCzQehGEq/41OJqaWmBRiPksGzfvh233XYbJBIJrrnmGpSW+rfpe8stt+CWW27x63MS4gmzUwJqxzpbnbW47BgagbYGoUWlsVlJRSIVbteXCS2y6D7un4M48XmzjA8++ABlZWXYtm0bbrrpJgBAdXV1j+R3EdITeGbfVbSuO2+08DbdSOevEM8zbN5bik3fnIehtkw4GJ0gDMrbsgaypuoA1L538ylwLV26FE8++SSys7Mxbtw4jB8/HoDQ+srLy/NrBQkJFrNDq8p2i7IkjRJ9Y9VQKZy/QhwH1DQacKXJCF7voptoFd1+NrGZ9mjwlk9dxZ/97Ge47rrrcOnSJYwcOVI8fuONN+KnP/2p3ypHSCgQA1d7i8ts4THDxe4+VhzHQcpxMDMGZm1NxSQ7F6TA5TOfl2hMSUlBSkqK3bGxY8d2u0KEhIpfTOpvt4STwrpFmSfZ81JOaLE1t69bF+0iVys6QWieGVsAYzOgiPZLvSOBT4GrubkZf/zjH7Fz505UV1eDd9hAjna6Jr2F7S7VCpkESrnEozW4pBwHKW8Ea9MDCpkQpJwKyQFVrDBvsbmGApcXfApc8+bNw+7du3HfffchNTXVaQtyQnoj64oQjDFs2FUCmYTDnGsyEaVw/hpJJRzUplowGYTla+Rq5ycEhIDWWie0zOKyA1r/3sSnwLV161Z8/PHHuPbaa/1dH0JCgsFswafHKiGVcLh5WCokNq0sM8/QZhKy6d2lQ0glHBSmOvBS5rqbaBWdAFw+3dGlJB7xKXDFxcUhPj7e33UhJGSYLAxna5rBcc5J7WabZR/kbtacj1HKoIEeHOC6m2ilbv8etdZ1r8IRxqd0iJUrV2Lp0qV28xUJ6U2sq58KG2UIkUvfYsJ7By7iXweFKTxyKWfXErN1x5gMTM+RQaOSd97iUrevLEGByys+tbheeOEFlJSUIDk5GdnZ2ZDL7VdxPHjwoF8qR0iwWFcvtd3Fx8IYLtR2/LO2JqS61XJFuO60xdUeuAwNgMUsTAMiXfLpU5o9e7afq0FIaOlY/aGjReW4h6Kss8BlNgKGJuFna3ByRREtnF20mIA2PU398ZBPgWvZsmX+rgchIcXVXETH8SxFJxOsvz1+GtHleqQlxqOPuzOKgDCApo4Tpv201lHg8pDPu1nW19fj//7v/7BkyRLU1gqz2w8ePIjy8nK/VY6QYHGcYA3YrwTRJ0bhcocfqxZ9DZoMZrTKPFgtmMa5vOZTi+vIkSOYOnUqdDodzp8/j4ceegjx8fHYsmULLly4gE2bNvm7noT0KFctLmn7jtaMAbddnY4Ypfuvj9osbNJiUlDgCgSfWlyLFi1CYWEhTp8+bbdz9c0334w9e/b4rXKEBEt2QjQeu3EA7szvWGee4zhxQN7cxU4/SpMeAGCkwBUQPrW4vvvuO/zlL39xOt63b19UVlZ2u1KEhAKJhIME9uNYSpkEEo7rcosypUkPAwCjPLbrF1K3l2mr96WaEcmnFpdSqXS5X+GpU6eQmEgL/5Pea+61ORiZrsNHhyqw/7z7lUutLa42uQctLlV7mbYGoR9KuuRT4PrJT36CFStWwGQyARCa0BcuXMBTTz2F22+/3a8VJCQYSq8045Ojl/D9Bfvum0TCodloQWObWRwHc2Jqg8zSCgAwehK4lFrh7CJvBoxN3a16RPApcL3wwgtoampCYmIiWltbMWnSJOTm5kKj0eC5557zdx0J6XG1zUacrGxERX2b033WeYoqudT1g1vrIJVwYPJocHIPtsqTSIWJ2IDQ6iJd8mmMS6fTYceOHfjqq69w+PBhNDU14eqrr8bUqVP9XT9CgoJnzmcVAeBERQPOVAutoiiFu8BVi4y4KGTkZAD9O8mat6XUCkGrTQ/oXKyWSux4Hbh4nkdxcTG2bNmC8+fPg+M45OTkICUlBYwxWuKG9Apmm7mKtlpNHXssatX2U91E1l17OsuYd6TSAfqLQuAiXfKqq8gYw09+8hPMmzcP5eXlGD58OIYOHYrS0lIUFhbSss2k13C3i0+qTiUej4tSOD0OQEdag9qLFVSsA/QG6ip6wqsWV3FxMfbs2YOdO3diypQpdvd99tlnmD17NjZt2oT777/fr5UkpKc5bpRhlapTYdrQFGjVMihkbv7vt9airsWIfecM0JguY0KuB91FVfvuWNTi8ohXLa63334bv/3tb52CFgDccMMNWLx4MTZv3uy3yhESLBbmuqvIcRyGpGmRHhfl/sGtdTBZeJxvVqKmyeDZC4opERS4POFV4Dpy5AimT5/u9v4ZM2bg8OHD3a4UIcFmcTFX0SPGFsDUBg4c2mQ69ykTjlSxwnWbnnK5POBVV7G2thbJyS62WWqXnJyMujqatkDC3+RBibhuQIL3gat9fIupNOA5meeBS9neVbSYAFMroOikRUe8a3FZLBbIZO5jnVQqhdls7nalCAk2mVQClVza9WKBjsTAFQsAngcuqaxjlx/qLnbJqxYXYwyFhYVQKl0n1RkMHvbnffTHP/4RS5YswWOPPYa1a9cG9LUI8Yl4RjEOMHSMlXlEpRP2VzQ0AHC/4SzxMnAVFBR0WSZQZxStE7tHjBgRkOcnxNaB0lrUNpswrK8WqbpOFgJ0ZBu46u031uiSSgs0VFCLywNeBa6ioqJA1aNTTU1NmDNnDl5//XU8++yzQakDiSxna5pxsa4VWX2ikOrBdENRe+CSRMVBKuHg1RAZnVn0mM8roPak+fPnY+bMmR5NKTIYDGhoaLC7EOIt69iUxNuZIO2BKyExBQtvHID7xmd7/lglBS5PhfyWIu+88w4OHjyI7777zqPyq1atwvLlywNcK9LbWRNQHfO4OmVqE84IAuC8yZq3oux5j4V0i6usrAyPPfYYNm/ebLfSameWLFkCvV4vXsrKygJcS9IbuZtk3Snr+JYiGpC5mQ7UGTF7ngJXV0K6xXXgwAFUV1fj6quvFo9ZLBbs2bMHL7/8MgwGA6RS+xn6SqXS7VlPQjwlTrLuZCcfJzYD8yYLj0+OXoKFZ5g1qq9nAdCay2VqFbY38yX4RYiQDlw33ngjjh49anfsgQcewODBg/HUU085BS1C/MXdJOtO2QQuDsIAPwCYLDykEg/+VuUqIViZjUJ3UebhkjgRKKQDl0ajwbBhw+yORUdHo0+fPk7HCfEncZK1N4PzNoHLdkcgj5NQAWGcq6mmfXNYClzuhHTgIiRYCidkw8zziFJ48RWxbXG17whkNPPe5XIpbQIXcSvsAteuXbuCXQUSAdQKKQAvhyJsk08hnJE0AjDxne8IZMc6QE9nFjsV0mcVCQkbZqMwXQcQA5d1fMy77HmbHX+IW2HX4iIk0EwWHl+croGE43D9gETPBuitrS25WhhkB8QJ2qYu9mC0o6QWlyeoxUWIA7OF4XCZHt9fqPd8yo5DNxEQUik4riMnzCO0EqpHqMVFiANz+5iUcGbQw8jlInDdnZ8JjoN3G8iILa4mgOcBCbUtXKFPhRAH3c3hspJ4E/islBphn0XGA8ZG7x4bQShwEeLAp3mKrT5sSeYKx9lsDkvdRXcocBHiwKcWV8sV4domafRQWT3+fbhC3EDWymzhwXeWlKqkOYtdocBFiAOvW1ymVmGTDMBuL8WqhjacqW5CXYvRrvjhi3oUf30eP1xyE5holYguUeAixIG4w4+n681bW1tKjd3EaHn7BG3HdIgLtc3Qt5rQZrMrth1aJaJLdFaREAdpsSrMvS7H8we0tI9vRfWxOyxrPyNom4DKGEOlXtibgWdCd3JIqtZ+c1klpUR0hQIXIQ5kUgl0ai86I9YWl2Pgam9xmW2m/DQazGgzWSDhOHx3vhatRguSNEqkxdqsa09dxS5RV5GQ7nITuBRi5nxHi6uh1QQA0KhkSNII68bVNtuPgdmtPU+bw7pEgYsQBxX1rdhzqsb94Lkjsatov1yztftnNHe0uBpahX1HtWo5dGp5+zGT/fNZ0yGsm8MSJxS4CHFQ3WjAgdI6lNQ0dV2Yt3Qknzq0uKxzFe0CV5sQpLQqGbTWwNXmELik8o6drKm76BKNcRHiwNI+JuVROkRrvZDlLpV3tJTaDUiKQe4NuXbPo5JLkahRok+MAjFKa4vLxe7vKp2QYtHWAGhSfH4vvRUFLkIcWM8CSj2ZJ2g7vuUwvUfmIp1iVEYsRmXEAgAq9W0AAL1jVxFoP7N4iVpcblBXkRAHFm8SUFsuC9cO3URPaFRCu6HZaHbOpBdzueq9ft5IQC0uQhyYvZny01QtXMckOd3VarRgz+ka8DzDjOGpYO1nCK0Tr9VyKaYPS0G0q+WhVbHCNSWhukSBixAHXrW4mmuE6+hEp7sYGE5UCIFn2tAUNBvNeOPr89BFKXDvuExIJByuStW6fl5aULBT1FUkxIHHk6wt5o5UCBeBS2EzxmW08GhoM8NkYTCa+a6Xu6EFBTtFLS5CHIzv3wejMmOhlnexWUbLFeGMolzldEYREAKfVMLBwjOYLDwabVIhrKoa2lDTaECSRokkrc1u7dYWl7FFyOeSyrv9vnoTanER4iBaKUNCjBLRyi7+rze3j29FJzqdUQQgblEGCLlctsmnVkcv6rHjRBVK2jePFcnVHcHKQAsKOqLARYivrAPz0c4D81YqufAVazPzYoa8VtURuKzBsdngkMvFcTZTf+r9U99ehAIXIQ6OlevxTckVXG4ydF6wuT0VIsZ5fMsqSiF0N1uNZjFDXmPTVYxWCvc3G10kodKCgm7RGBchDk5cakB5XSv6xCiQEKN0XYgxoKlK+NnFwLyVSm4NXLyYaKpTu2pxuVibi1aJcIsCFyEOrJnz8s4WEjQ0ChvAchIgJtltsWlDUyDhOChkElyobYFMKkFslE3gas/hanHV4lLHCtfWuZBERIGLEAdmT+YqNl4SrqMTOj3jp7I5MzlzRKrT/WJX0WABY8w+TcK68QYFLichPca1atUq5OfnQ6PRICkpCbNnz8bJkyeDXS3Sy1lXc+i0xWUNXBrnYOSNqPYWF88YWh2XcqbA5VZIB67du3dj/vz5+Pbbb7Fjxw6YTCbcdNNNaG5u7vrBhPhI3CxD2kmLq6E9cGk7D1zVjW3YfrwSn/9YLU75sSWVcJg5IhU/G51ul7AKoGPaj6mN1uVyENJdxU8//dTudnFxMZKSknDgwAFMnDjR5WMMBgMMho6zQQ0NNLBJvGNu39xC7m51CMY8bnHxPHC8fdrPiUsNuKZfH4zOst97cWCyc/IqAGHjDWWMsKt1a52Q20UAhHiLy5FeL0x/iI+Pd1tm1apV0Ol04iUjI6Onqkd6AcaYuNSyXOamxdVaB5gNgETW6RlFAIiL7hj/Mpp5KGVefuWou+hS2AQunufx+OOP49prr8WwYcPclluyZAn0er14KSsr68Fakt5gzjWZuDM/AyqZmyk/DRXCdUwSIOl8WpBSJkWMTQZ+ik7lVKa6oQ1HL+pRUe+iO0iBy6WQ7iramj9/Po4dO4Yvv/yy03JKpRJKpZvcG0K6wHEckjTOwcWO/qJwrevr0XMOSdNi37laJGiU6BOtcLr/ZFUj9p+vQ15mrP1uPwAFLjfCInAtWLAA//nPf7Bnzx6kp6cHuzok0tVfEK5jszwqPr5fHyRplEiNVbtcFSJKzOVykYRqDVzWVSgIgBAPXIwx/OpXv8L777+PXbt2ISfHi006CfFBi9GMoxf1UCukGJEe61zA2NyxXLPOs3+iEgmHAe4G4GGby+UqCZVaXK6EdOCaP38+3nrrLXz44YfQaDSorKwEAOh0OqjVdIaF+F9jmxlfl1yBRiVzHbjq28dMYxL9dpYv2pMWl6lVuNCZRQAhPji/YcMG6PV6TJ48GampqeLlH//4R7CrRnopk6WLrHl9e+DSZfrtNcX5iq6m/ciUHYsKWid1k9BucblK2CMkkMR5iu7SFupLhetY/6XZWFeQMJh4mCy8c8Z+VIKwQkTLZb++bjgL6RYXIT3NOk/RZfJpWwPQVCOsleXhwLwnlDKJ2MJrcbVKRHT7DkLNV/z2muEupFtchPQ0o7mT6T61Z4VrTWrHTtN+wHEcZgxPhVImQZTSRV5YVIJw3UJdRSsKXITYEFeGcDXBurZEuO7T3++vm5sU4/5Oa3a+dUchQl1FQmyZxHmKDi0u3gLUnhN+jvd/4OqUdbNZQ5Mw4ZpQi4sQW4NTtEjRqZ3nFNZfEHbbUUQDmhS/v25NowGV+jbERsmREe/QDbXuImRoFLqLHuaP9WbU4iLERrRShr6xauclm2va14Hrk+tyR5/uKqlpwn9/qMLJSjc7+lh3yrYuFx3hKHAR0hWeB2p+FH5OGhyQl7AmobrM5QI6lodurAzI64cb6ioSYuN0VSOaDGZk9YlGvHVCdP15IWtdEQXEZgfkdaNslnB2ybruFwUuANTiIsTO0XI9dp2sQVWDzSB49Q/CdcIgwN3igt1kXfrG5aYZAKBpb3E1XwYsbspEEApchNiwrjevsA7Om4023cSrAva61uz5FqPF9YwRpVZo8TG+YwftCEaBixAbBmvgsuZx1fwgBC91HBDrv/mJjqxL21h4hjYT71yA44CY9rOZ1mWjIxgFLkJsWFtcYjpExSHhOnVkQM4mWkklHNSKTna1BjrSMKwrsEYwGpwnxIbRYtNVbKwSggQnAVKGB/y1pw1NgUzCQatys09jbAZQio4VWCMYBS5C2ll4ZtPikgLn9wp3JA4UdtsJsJyE6M4LaPsKQbS1HmjTAypdwOsUqqirSEi7tvYNWTkOUFkaOs4mZlwTxFrZkCk7zi5GeKuLWlyEtFPKJLgzPwMGMw/u4tfCGby47C43ffUXfasJF660QC7jMDhF67qQLl3YjLauFEge2iP1CkXU4iKknUwqQVqsGjnRRqDie+FgZs+1tmoaDfjvD1X4/kK9+0Jx7fsu1JYIG9NGKApchDg6t0dYDSIuW7j0EJ1aGJSvbzG5LxSbBUjlwkoRETxvkQIXIe0q9W04duI4GkoPCwf6TwloCoQja+BqM1nE8TYnUllHML18umcqFoJojIuEHkMj0FQtLJxnbAbMbcKSMowXLoDQTZLKhYs8SlizSpMiLLrnY7A5X12Plv3vI0plgDZndECWr+mMQiZBtFKKZoMF+lYTVHI3u2QnDBCCVs2PQPZ1PRpcQwUFLhIaTK1CsmfND0L+lK+UMUDiVUBaXsda7R5SnP8czFQPaVwi0P8G3+vQDfHRSjQbWlDTaECy1s2O2gmDAOl2Yd5i4yVAm9azlQwBFLgiAc8LC9A11wBmg3BaXR0nTCEJ0KRhj5mNQOlXQPkBoVUFCC2IqD5C60kZI7SoJHIhh4njhGsA4M2AxSiM97RcBhrKhZ8vfidc4nOA9Hwgvl/XrZKLB6CuOQIDOBhzpwVt/8JEjRJltS2oaTK4LyRXAQkDgaoTQOVRClykl2m+IpwdqzomtGgcyZRA4mCg7+iO/KCeVHsWOPmpkEwJCJus9h0tfCkVXSRjumIxA3XngUuHgCtnhKWWa88Jz5s+FkgaIowR2WIMKNsHlHwGg5lHmW4M0pMHdved+SyxfQHDmsZOAhcgTEGqOgFUHhG6i758XmGMAldv1KYHzn8p/De2njKXyoUxG5lKaHU1Vwvrl186LFwSBgD9JgPRCYGvn7EFKNkJVB4Tbqu0QO7/CHXozniNVAYk5AqX1jrg4gEhiDXVAD9+DJzdBaSPEVYxlcqFcbSL+4H6C2BguBA1FOXaUeISM8GQ1ScKPxud7rwCq6PYLOH32VgptC77Te6R+oUKCly9iamto9vFt5+V6pML9L1ayP+x7RbyvLArc8VBYVniy6eFVkrKCCDnemGNc39jDKg6Dpz5r9AC5Dig7xggZyIgU/j3tdRxwICpQPa1wthZ+X6hG3l2t3CxJZGhOX0iSkyxkEol4tm9YIhWysSdrTvFcUDWtcCx94Cy74TfW1R84CsYIihw9QY8D1z6Hjj3RUeXMC4LyJkE6Pq6foxEIpSJyxK6lOd2ATWnhNZX9XEgY5xwkXXxn99TLbXAqW1CVw4Qum8DZ7ivn7/I1UDWeCBjLFB9Qnh/zTVCt1KlE8a/0segqkkGcBWIj1ZA4rjDT6hKGCCkRtSdB05uBUbeBUjcnInsZTgWBvvcv/LKK/jzn/+MyspKjBw5EuvXr8fYsWM9emxDQwN0Oh30ej20WjfTKMIVbxFaMBe+BVradzmOThDOiHkyIO1IfxEo+QzQlwu3FVFA1nVA6giha+ULY7NQP2srUCITWkEZ40LqS8YYQ02TAS0GC7K7muwcYG0mCw6W1qG60YBZo9LAdfZ7bKkF9m8UTmykjhD+GQT7hIsXfP1+hnyL6x//+AcWLVqE1157DePGjcPatWsxbdo0nDx5EklJScGuXnC01AoD7pVHhW3hAeFMU/ZEIG2U7wFBlw7k3QdcPiWMB7XUAqe3A+f3CMu6JA4GNGldfzF4XjjDV3lUaL1ZlxqO7wcM+J+Q7NJwHIckjQoIQA/ZFwdK62DmGcrrW5Ee18mu2VHxwJBZQpfx0hGhOzxoeq9fOSLkW1zjxo1Dfn4+Xn75ZQAAz/PIyMjAr371KyxevLjLx4dti4vnhcRLUyvQVi8EkeZqYX+/1vqOcopooRuUlue/bh0gtI4qDgEX99m/nlwtbNwQnSCMg8lUAJgQnAx6odupLxNOAFhpUoRxLF9agT1A32KCVi3rvGXTw/57ogpHy/WIi5Jj1qi+iIvuYgyw+kfgh38LKSISqdCNjO8v7A6kjvXv34Yf+fr9DOnAZTQaERUVhX/961+YPXu2eLygoAD19fX48MMPnR5jMBhgMHR8aRoaGpCRkdH1B1P6tdDtcmT38TAXx7tzzOa47TFmEfKb3OEkQo5S8lAhdcDXbpwneF5IW6g6JlybuzhNbyVTCImSKcOFJY+DHBR4nuHNvaXgeQaeATxj4BmDhRe6ZoNSNLjxqiRhHa4Q0Gq0YPPeUjS2Ca3VaKUUCqkE1/TvI64cUdXQhu0nqiDhgDnjsoSzp6e3C//cHEmk7TMNFO15cFz774Sz/9348/ek0AAj7+y0SK/sKl6+fBkWiwXJyfY5RsnJyfjxxx9dPmbVqlVYvny59y9maBIykUONTCm0bKLihaRMXYbQpeup/6ASSUeKAW8RTr83VwuflalFOJPJSYQvhlIDqOOFFpYmNaTGWjgOuNLk+p8BxwESjoM0hFpcaoUUd4zJwGc/VuH85RY0GyxohsVuPXqjmcflRgMk1nrHJAJ5c4TfUc2PQgBrqRVa7bxFuJja3LxiAKg6+efbTSEduHyxZMkSLFq0SLxtbXF1qe9oF3lE7T+7OmZ33NNjXTyn9RgnEbpgMlVIffkhkQpnAQN9JjAAOI7Dz0andwQpCSf+HKOUuZ8XGEQ6tRw/zUtHm8mChlYTjBbeLlUjUaPE7VenOz9Qk2I/z9JsEC4Wo3BhDABzc+1HksCFl5AOXAkJCZBKpaiqsp+7VlVVhZQU1xNglUollEofWiPRfbye20bCS0Z8J4PcIUwll7oMrCq5FJl9PHhPMmXIjnH5KoT+nTtTKBQYPXo0du7cKR7jeR47d+7E+PHjg1gzQkgwhXSLCwAWLVqEgoICjBkzBmPHjsXatWvR3NyMBx54INhVI4QEScgHrjvvvBM1NTVYunQpKisrMWrUKHz66adOA/aEkMgR0ukQ/hC2eVyERABfv58hPcZFCCGuUOAihIQdClyEkLAT8oPz3WUdwmtoaAhyTQghjqzfS2+H2nt94GpsbAQAz7LnCSFB0djYCJ3O8xUtev1ZRZ7nUVFRAY1GE1Kz/3uSddpTWVkZnVn1AX1+3dPZ58cYQ2NjI9LS0iDxYnpbr29xSSQSpKe7mM8VgbRaLX3xuoE+v+5x9/l509KyosF5QkjYocBFCAk7FLgigFKpxLJly3xbNYPQ59dNgfj8ev3gPCGk96EWFyEk7FDgIoSEHQpchJCwQ4GLEBJ2KHD1UrW1tZgzZw60Wi1iY2Px4IMPoqmpqdPHTJ48GRzH2V1+8Ytf9FCNg+uVV15BdnY2VCoVxo0bh3379nVa/t1338XgwYOhUqkwfPhwfPLJJz1U09DkzedXXFzs9HemUqm8e0FGeqXp06ezkSNHsm+//ZZ98cUXLDc3l919992dPmbSpEnsoYceYpcuXRIver2+h2ocPO+88w5TKBRs48aN7Pjx4+yhhx5isbGxrKqqymX5r776ikmlUvanP/2JnThxgv3+979ncrmcHT16tIdrHhq8/fyKioqYVqu1+zurrKz06jUpcPVCJ06cYADYd999Jx7bunUr4ziOlZeXu33cpEmT2GOPPdYDNQwtY8eOZfPnzxdvWywWlpaWxlatWuWy/M9//nM2c+ZMu2Pjxo1jjzzySEDrGaq8/fyKioqYTqfr1mtSV7EX+uabbxAbG4sxY8aIx6ZOnQqJRIK9e/d2+tjNmzcjISEBw4YNw5IlS9DS0hLo6gaV0WjEgQMHMHXqVPGYRCLB1KlT8c0337h8zDfffGNXHgCmTZvmtnxv5svnBwBNTU3IyspCRkYGZs2ahePHXewi34leP8k6ElVWViIpKcnumEwmQ3x8PCorK90+7p577kFWVhbS0tJw5MgRPPXUUzh58iS2bNkS6CoHjS+7pVdWVros39ln21v58vkNGjQIGzduxIgRI6DX6/H8889jwoQJOH78uMcLIlDgCiOLFy/G6tWrOy3zww8/+Pz8Dz/8sPjz8OHDkZqaihtvvBElJSXo37+/z89LiK3x48fb7Ys6YcIEXHXVVfjLX/6ClStXevQcFLjCyK9//WsUFhZ2WqZfv35ISUlBdXW13XGz2Yza2lq3O4C7Mm7cOADAmTNnem3g8mW39JSUFK/K92a+fH6O5HI58vLycObMGY9fl8a4wkhiYiIGDx7c6UWhUGD8+PGor6/HgQMHxMd+9tln4HleDEaeOHToEAAgNTXV328lZPiyW/r48ePtygPAjh07InJ3dX/sNm+xWHD06FHv/s66NbRPQtb06dNZXl4e27t3L/vyyy/ZgAED7NIhLl68yAYNGsT27t3LGGPszJkzbMWKFWz//v3s3Llz7MMPP2T9+vVjEydODNZb6DHvvPMOUyqVrLi4mJ04cYI9/PDDLDY2VjxFf99997HFixeL5b/66ismk8nY888/z3744Qe2bNmyiE+H8ObzW758Odu2bRsrKSlhBw4cYHfddRdTqVTs+PHjHr8mBa5e6sqVK+zuu+9mMTExTKvVsgceeIA1NjaK9587d44BYJ9//jljjLELFy6wiRMnsvj4eKZUKllubi77zW9+ExF5XIwxtn79epaZmckUCgUbO3Ys+/bbb8X7Jk2axAoKCuzK//Of/2QDBw5kCoWCDR06lH388cc9XOPQ4s3n9/jjj4tlk5OT2c0338wOHjzo1evRsjaEkLBDY1yEkLBDgYsQEnYocBFCwg4FLkJI2KHARQgJOxS4CCFhhwIXISTsUOAihIQdClykVykuLkZsbKx4+5lnnsGoUaPE24WFhZg9e3aP14v4FwUu4pPCwkK3a9LPnz8fHMfZrWQRiICRnZ2NtWvX2h278847cerUKbePWbduHYqLi8XbkydPxuOPP+7XepHAo8BFfJaRkYF33nkHra2t4rG2tja89dZbyMzMDEqd1Gq10yKKtnQ6nV2LjIQnClzEZ1dffTUyMjLsVkjdsmULMjMzkZeX163ndtUSmj17ttiKmzx5MkpLS/HEE0+IO8UAzl1FR7Ytv8LCQuzevRvr1q0Tn+PcuXPIzc3F888/b/e4Q4cOgeM4r9aMIoFDgYt0y9y5c1FUVCTe3rhxIx544IGAv+6WLVuQnp6OFStW4NKlS7h06ZLXz7Fu3TqMHz8eDz30kPgcmZmZTu8JAIqKijBx4kTk5ub66y2QbqDARbrl3nvvxZdffonS0lKUlpbiq6++wr333hvw142Pj4dUKoVGo0FKSopPq4/qdDooFApERUWJzyGVSlFYWIiTJ0+KewOaTCa89dZbmDt3rr/fBvERLd1MuiUxMREzZ85EcXExGGOYOXMmEhISgl2tbklLS8PMmTOxceNGjB07Fv/+979hMBhwxx13BLtqpB21uEi3zZ07F8XFxXjjjTf81iqRSCRwXCrOZDL55bk9MW/ePPHEQ1FREe68805ERUX12OuTzlHgIt02ffp0GI1GmEwmTJs2zS/PmZiYaDduZbFYcOzYMbsyCoUCFoulW6/j7jluvvlmREdHY8OGDfj000+pmxhiqKtIuk0qlYrbokmlUrfl9Hq9uAGHVZ8+fZCRkeFU9oYbbsCiRYvw8ccfo3///lizZg3q6+vtymRnZ2PPnj246667oFQqfeqiZmdnY+/evTh//jxiYmIQHx8PiUQijnUtWbIEAwYMiMiNMEIZtbiIX2i1Wmi12k7L7Nq1C3l5eXaX5cuXuyw7d+5cFBQU4P7778ekSZPQr18/TJkyxa7MihUrcP78efTv3x+JiYk+1fvJJ5+EVCrFkCFDkJiYiAsXLoj3PfjggzAajT1ylpR4h9acJ8SNL774AjfeeCPKysqcdmomwUWBixAHBoMBNTU1KCgoQEpKCjZv3hzsKhEH1FUkxMHbb7+NrKws1NfX409/+lOwq0NcoBYXISTsUIuLEBJ2KHARQsIOBS5CSNihwEUICTsUuAghYYcCFyEk7FDgIoSEHQpchJCw8/8wAo2F7PjHuwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:14:01.419066Z", + "iopub.status.busy": "2024-02-29T18:14:01.418287Z", + "iopub.status.idle": "2024-02-29T18:14:01.777458Z", + "shell.execute_reply": "2024-02-29T18:14:01.776499Z" + }, + "papermill": { + "duration": 0.381361, + "end_time": "2024-02-29T18:14:01.779578", + "exception": false, + "start_time": "2024-02-29T18:14:01.398217", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS4AAAEmCAYAAADcE30uAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCt0lEQVR4nO2deXxU5fX/P3f2STIzIRtJSEICCfsqmyyyCN+yqfDFClqsbIpWrCJFkVpAXIorgtYfbW0h0iK4FGtbv4oWBRSRVZBNlkgWyApJJvts9/n9cTM3s2dmMpOZSc779ZrXZO7c5WSS+5nznOc853CMMQaCIIgIQhJqAwiCIHyFhIsgiIiDhIsgiIiDhIsgiIiDhIsgiIiDhIsgiIiDhIsgiIiDhIsgiIhDFmoDgg3P8yguLoZGowHHcaE2hyAIGxhjqK2tRWpqKiQS7/2oDi9cxcXFSE9PD7UZBEF4oKioCGlpaV7v3+GFS6PRABA+GK1WG2JrCIKwpaamBunp6eJ96i0dXrisw0OtVkvCRRBhiq9hHArOEwQRcZBwEQQRcZBwEQQRcXT4GFdHxWKxwGQyhdoMgmgVuVwOqVQa0HOScEUgdXV1uHr1KqgGJBEJcByHtLQ0xMTEBOycJFwRhsViwdWrVxEVFYXExERKqiXCGsYYKioqcPXqVeTk5ATM8yLhijBMJhMYY0hMTIRarQ61OUQzjUYzDGYeGpUcUgl9mdiSmJiI/Px8mEymgAkXBecjFPK0wgfGGGqaBOFqNFpCbU7YEYz/VRIugmgjFr4l1iiX0hdKe0DCRRBtxNI8SSKTcFDKAzt7RriGhIsIaxYuXIjZs2cH9JwTJ07E8uXLPe6TmZmJTZs2eXU+nheeJTR8bzdIuIh2wxvBiET4Zo+LATBZ+NAa084888wzGDJkSLtfl4SLINqIVbhMFh6V9UbKr2sHSLg6CEYz7/ZhdvACPO3r6DG4289XFi5ciP3792Pz5s3gOA4cxyEvLw9LlixBVlYW1Go1evfujc2bN7s8fv369UhMTIRWq8VDDz0Eo9Ho1XXr6+tx3333ISYmBikpKXjttdec9ikvL8ftt98OtVqNrKws7Nixw2kfjuOwZcsWTJ8+HWq1Gj169MCHH34IAFDKJLhRchXJOjU+3v0hbhk/Hmq1GiNGjMDFixdx9OhRDB8+HDExMZg+fToqKiq8/ty2bt2K/v37Q6lUIiUlBY888oj4XmFhIWbNmoWYmBhotVrMnTsXZWVl4vtWb+hvf/sbMjMzodPpcPfdd6O2tlbch+d5vPzyy8jOzoZSqURGRgZeeOEF8f1Vq1ahV69eiIqKQo8ePbBmzRpxxUZubi7Wr1+PU6dOiX/T3Nxcr3+3tkB5XB2Et7667Pa9rIRozB7aTXz95wN5MFlcewVpXdS4a3hL4cWtB6+4nOJ//H96+WTf5s2bcfHiRQwYMADPPvssAKBLly5IS0vDBx98gPj4eHz77bdYunQpUlJSMHfuXPHYvXv3QqVSYd++fcjPz8eiRYsQHx9vd4O544knnsD+/fvx8ccfIykpCb/97W9x4sQJu+HNwoULUVxcjK+++gpyuRyPPvooysvLnc61Zs0avPjii9i8eTP+9re/4e6778bp06fRt29fRCmFW+nVDc9j0+uvIysrE4sXL8YvfvELaDQabN68GVFRUZg7dy7Wrl2LLVu2tGr7li1bsGLFCrz44ouYPn069Ho9Dh48CEAQHKto7d+/H2azGcuWLcO8efOwb98+8Rx5eXn45z//if/85z+oqqrC3Llz8eKLL4qf3erVq/H222/j9ddfx7hx41BSUoIff/xRPF6j0SA3Nxepqak4ffo0HnjgAWg0Gjz55JOYN28ezpw5g88++wz//e9/AQA6na7V3ysgsA6OXq9nAJherw+1KQGhsbGRnTt3jjU2Ntpt3/j5BbePj05ctdv3zb0X3e77/tFCu3237Lvscj9/mDBhAnvsscc87rNs2TJ25513iq8XLFjA4uLiWH19fYtNW7awmJgYZrFYPJ6rtraWKRQK9v7774vbbty4wdRqtWjHhQsXGAB25MgRcZ/z588zAOz1118XtwFgDz30kN35R40axX71q18xxhi7cuUKA8A2vrmFGU2CXTt37mQA2N69e8VjNmzYwHr37u3Rbiupqans6aefdvne559/zqRSKSssbPl7nT171u53WbduHYuKimI1NTXiPk888QQbNWoUY4yxmpoaplQq2dtvv+2VPYwx9sorr7Bhw4aJr9etW8cGDx7s8Rh3/7OM+X9/ksfVQVg2Kdvte46J3EvH93S7r+PE2OKxWW0xq1XeeustbN26FYWFhWhsbITRaHQK9g4ePBhRUVHi69GjR6Ourg5FRUXo3r2723Pn5eXBaDRi1KhR4ra4uDj07t1bfH3+/HnIZDIMGzZM3NanTx/ExsY6nW/06NFOr0+ePGk3fO7bf4AY8+ratSsAYODAgeIxXbt2denNOVJeXo7i4mJMnjzZ5fvnz59Henq6XVnyfv36ITY2FufPn8eIESMACLOjttVFU1JSxOufP38eBoPB7TUA4L333sMbb7yBvLw81NXVwWw2h0VBTopxdRAUMonbh0wq8XpfuZf7BoJdu3Zh5cqVWLJkCT7//HOcPHkSixYt8jp+FS7UNpmgbxTiPnK5HNZBuDVjXC6Xi/tyHAeebz1GGKjlXLbXdrx+a9c4dOgQ5s+fjxkzZuA///kPvv/+ezz99NNh8fch4SLaDYVCAYulJV528OBBjBkzBg8//DCGDh2K7Oxs5OXlOR136tQpNDY2iq+/++47xMTEtNoEpWfPnpDL5Th8+LC4raqqChcvXhRf9+nTB2azGcePHxe3XbhwAdXV1U7n++6775xe9+3bF7xDuJAFYFZRo9EgMzMTe/fudfl+3759UVRUhKKiInHbuXPnUF1djX79+nl1jZycHKjVarfX+Pbbb9G9e3c8/fTTGD58OHJyclBQUGC3j+PftL2goSLRbmRmZuLw4cPIz89HTEwMcnJysH37duzZswdZWVn429/+hqNHjyIry354ajQasWTJEvzud79Dfn4+1q1bh0ceeaTVdlYxMTFYsmQJnnjiCcTHxyMpKQlPP/203XG9e/fGtGnT8OCDD2LLli2QyWRYvny5S2/kgw8+wPDhwzFu3Djs2LEDR44cwV//+lcwtAiVSi6BzIc2W5545pln8NBDDyEpKQnTp09HbW0tDh48iF//+teYMmUKBg4ciPnz52PTpk0wm814+OGHMWHCBAwfPtyr86tUKqxatQpPPvkkFAoFxo4di4qKCpw9exZLlixBTk4OCgsLsWvXLowYMQKffPIJPvroI7tzZGZm4sqVKzh58iTS0tKg0WigVCoD8vt7gjwuot1YuXIlpFIp+vXrh8TEREydOhVz5szBvHnzMGrUKNy4cQMPP/yw03GTJ09GTk4Oxo8fj3nz5uGOO+7AM88849U1X3nlFdxyyy24/fbbMWXKFIwbN84ungUA27ZtQ2pqKiZMmIA5c+Zg6dKlSEpKcjrX+vXrsWvXLgwaNAjbt2/Hzp070bdvX9g6WNFKOeQBGkovWLAAmzZtwv/7f/8P/fv3x2233YZLly4BEIZ8H3/8Mbp06YLx48djypQp6NGjB9577z2frrFmzRr85je/wdq1a9G3b1/MmzdPjIHdcccdePzxx/HII49gyJAh+Pbbb7FmzRq74++8805MmzYNkyZNQmJiInbu3BmQ3701OBYIvzaMqampgU6ng16vD4ugYltpamrClStXkJWVBZVKFWpzOg0cx+Gjjz5yWn7EM4aKWgMAIFGjpGU/LvD0P+vv/UkeF0G0AcevfQvP21WLIIIDCRcRsRQWFiImJsbto7CwMOg2WAcsHIAGgxnX64yoN5i9OtaT7V9//XUQrY58QhqcP3DgAF555RUcP34cJSUlTq44Ywzr1q3D22+/jerqaowdOxZbtmxBTk5O6IwmwobU1FScPHnS4/uBwl1ERcJxiFHa30be+luebO/WrZvb94gQC1d9fT0GDx6MxYsXY86cOU7vv/zyy3jjjTfwzjvvICsrC2vWrMHUqVNx7tw5iu8QkMlkyM52n3jbHkgkHKKbhavBKHha3oaNQ217JBNS4Zo+fTqmT5/u8j3GGDZt2oTf/e53mDVrFgBg+/bt6Nq1K/75z3/i7rvvbk9TCaJVrGF5inAFn7CNcV25cgWlpaWYMmWKuE2n02HUqFE4dOiQ2+MMBgNqamrsHgQRLHieiRU4rJnyHXuePjwIW+EqLS0F0LLey0rXrl3F91yxYcMG6HQ68dFadjVBtAWThUdVgxH6JlOLx0XKFXTCVrj8ZfXq1dDr9eLDdkkEQQQacV0iOKcF6kTwCNslP8nJyQCAsrIypKSkiNvLyso8lopVKpXtsuSAIICWYSEHYYZRLZdCQn0Vg07YelxZWVlITk62WwBaU1ODw4cPO5UXIYjQ0ZzHxQEyqQRatdwpPYIIPCEVrrq6Opw8eVLMZ7Eu1iwsLATHcVi+fDmef/55/Otf/8Lp06dx3333ITU1NeBdX4j2IdyaZQTCHopmhYaQfjUcO3YMkyZNEl+vWLECgLC4NDc3F08++STq6+uxdOlSVFdXY9y4cfjss88oh6sTYzQaoVAoQm2GiDhU5ISgPGOCmElpuBhcfKqXGoF0+NLNPM+YyRCaB897bfeCBQsYhHtafFy+fJktXryYZWZmMpVKxXr16sU2bdrkdNysWbPY888/z1JSUlhmZiZjjLGDBw+ywYMHM6VSyYYNG8Y++ugjBoB9//334rGnT59m06ZNY9HR0SwpKYnde++9rKKiwq09V65c8fnvUddkYqX6RlbdYGRmC89K9Y2sVO9corgzQ6WbCWcsJuBr58417cItvwFk3nk/bW2WodVq8cUXXwAQYp233347ZsyYgXfffRcFBQVOQ77q6mrceuutuP/++/H666+jsbERq1atwty5c/Hll1+6tCcxMdHnj0Au5RCtkEEm5WDrYzHGxLwuIvCQcBHtgk6ng0KhQFRUlDhjDAg1rqxkZWXh0KFDeP/99+2EKzo6Gn/5y1/EIeIf//hHcByHt99+GyqVCv369cO1a9fwwAMPiMf84Q9/wNChQ/H73/9e3LZ161akp6fj4sWL6NWrl0t7fEUhk0LRfBfxNvlbDADJVvAg4Yp0pHLB8wnVtduIN80yBg4caBfXunDhAgYNGmQX6xw5cqTdMadOncJXX32FmJgYp2vm5eWhVy/f2qt5g73HBVKuIELCFelwnNfDtXDD2izjtddew+jRo6HRaPDKK6/Y1YgHBI/LV+rq6nD77bfjpZdecnrPNi+wrVh4HjwDpBznkL9FyhVMSLiIdsNTswwrrpplONK7d2/8/e9/h8FgEJONjx49arfPTTfdhH/84x/IzMyETOb63zwQjR7qDBY0mSyIUcoQrZSBQ3Okn/IkgkrYJqASHQ/bZhnXr19HTk4Ojh07hj179uDixYtYs2aNkwC54he/+AV4nsfSpUtx/vx57NmzB6+++iqAlpZgy5YtQ2VlJe655x4cPXoUeXl52LNnDxYtWiSKlaM93rQNc8KmkKBw/ebNvp+J8AESLqLd8LdZhiNarRb//ve/cfLkSQwZMgRPP/001q5dCwBi3Cs1NRUHDx6ExWLBz372MwwcOBDLly9HbGys2OXH0R5/KqaKAtUsWEq5FCq5lNYtBhlqlhFhULMM1+zYsQOLFi2CXq8PWDNVb6hqMMJo5qFVyaBWUOTFFcFolkGfNBGRbN++HT169EC3bt1w6tQpMUerPUULsM2cJxerPSHhIiKS0tJSrF27FqWlpUhJScFdd92FF154IQSW2A9YGGPifCKJWfAg4SIikieffBJPPvlkqM2wK2sDAFUNJpgsPHRqOVRyacjs6uiQcBFEG1DLpbAwJi6qFmcVO3TkOPSQcEUoHXxOJWKIcqi91dIwg/4+VoLxv0rpEBGGVCoMP4xGY4gtIVwhRrVIt0Ss/6vW/91AQB5XhCGTyRAVFYWKigrI5XIxJ4kIDWYLD3DCkh+O42A0mmAy82iCGRIqvgKe51FRUYGoqCi3Kxj8gT7ZCIPjOKSkpODKlSsoKCgItTmdntomExgDYpQySCQcmkwWGM08lHIJlDIKzgOARCJBRkZGQGdZSbgiEIVCgZycHBouhgHbv80HzxjuGpGOaIUMh3+6gfMVNRicHos+GV1CbV5YoFAoAj4yIOGKUCQSCWXOhxieZ6i3CDdktFoNlVyKeF0M0s0SxGlj6O8TREi4CMJPzHxLBF7SPAwa0E2HAd10oTKp00CRXYLwE4uNcMmoOUa7QsJFEH5ibi6DI3EoIsgYsxM1IvDQUJEg/MRavksmbRGt4wVVOHCxAn1TNJg2IHCVVgl7SLgIwk/kMg5DM2LF+BYAWB0vix81CQnvIeEiCD+JUsgwsXeS3TZZ87S/hZZkBRWKcRFEALGmK1n8KQNNeA0JF0H4icnCo7bJhCZTS8MNa5UIGioGFxIugvCTa1WN+MvXV/CPE1fFbdLmeBdPs4pBJayFy2KxYM2aNcjKyoJarUbPnj3x3HPPUUkXIiywJqDa5nCJHhf9jwaVsA7Ov/TSS9iyZQveeecd9O/fH8eOHcOiRYug0+nw6KOPhto8opNjzdWS2qzDi1bKkJUQjS7RkdmkN1IIa+H69ttvMWvWLMycOROA0Adv586dOHLkSIgtI4iWBFRbj6urVoXZQ7uFyqROQ1gPFceMGYO9e/fi4sWLAIBTp07hm2++wfTp090eYzAYUFNTY/cgiGBg9bgktNyn3Qlrj+upp55CTU0N+vTpA6lUCovFghdeeAHz5893e8yGDRuwfv36drSS6Oh8dqYEpfomzB2Rjiib3omuYlxE+xDWHtf777+PHTt24N1338WJEyfwzjvv4NVXX8U777zj9pjVq1dDr9eLj6Kiona0mOho1BnMOF9Si6oGEy6X19m9x4sxrhbhulFnwB++vIQ/H8hrVzs7G2HtcT3xxBN46qmncPfddwMABg4ciIKCAmzYsAELFixweYxSqYRSqWxPM4kOjL7RJP5c22S2ey8+RokB3XRI0bXU3ZJwHEwWBo6jWcVgEtbC1dDQ4FQ5USqVgqesZKKdsM3HqjPYC1dWQjSyEqLttlnjXZTHFVzCWrhuv/12vPDCC8jIyED//v3x/fffY+PGjVi8eHGoTSM6CelxUZjaPxl7zpai3kG4XCGzyeNijFE36yAR1sL15ptvYs2aNXj44YdRXl6O1NRUPPjgg1i7dm2oTSM6ERqVDAkaJbQqud12g9kCxgC5VCLGuazPjAE8A6SkW0GBYx08Db2mpgY6nQ56vR5arTbU5hAdiM/PluJscQ3GZidgZFYcAMBo5vHWV5cBAMsmZUMhC+v5r5Dj7/0Z1h4XQYSary9VIP9GA4ZldEG/VPsby+JiVtE2NYLv2D5BSCHhIggP6BtNuF5rgNFFuQdXeVwcB6R1UduJGRF4SLgIwgOmZsH66sdyHLlyAwvHZInDP6tHJbUTLg53DU9vf0M7GTQAJwgPGM0tnla9wQKDuaX2ltnS7HFRBL7dIeEiCA/YChcAmCwtcSsLLfkJGTRUJAgPGByEy2wT67LGuCQOuVp//64A+kYT7hqWhiQtdbMOBiRcBOEBx6C8ySYjPjM+CrFRcsSo7G8jo5mH0czbdbomAgsJF0F4IFohg0LKg2cM9QYLTDYe2JjsBJfHWGNe1BQ2eJBwEYQHFozJBAC8f7QI9YZGsXigJ6xDR8rjCh4kXAThBbooOQxmi12ZZpOFh5TjnAoJWtMjaKgYPEi4CMILpvZPdtr25wM/wWjmsXBMpl2Neer0E3xIuAjCDQ1GM3afuAa5lMO8ERlO74tLfqSuPS7q9BM8SLgIwg0mC0NFrQFyFwmmjDG3eVxxMQqYeR5KmbRd7OyMkHARhBtsm2EcL6jC6avV6Jeqw8isOLv4leO6xEm9k9rVzs4IZc4ThBts248ZTBZUNZjEYoK2qQ4yCd1G7Q194gThBmvmg1Qigbx5YbU1IdUqXBwH0Iqf9scv4frpp58CbQdBhB22Hpc1jmVdWG1b0saxPPM3l67j7QM/4URhVTta27nwS7iys7MxadIk/P3vf0dTU1OgbSKIsMA2xiWXCreKtcyNVMIhp2sMeiTGOB1ntFhQZzCjyWRxeo8IDH4J14kTJzBo0CCsWLECycnJePDBB3HkyJFA20YQIYUDh2ilFGq51Em4YpQy3DYoFTMGpjgdZ01SpWZUwcMv4RoyZAg2b96M4uJibN26FSUlJRg3bhwGDBiAjRs3oqKiItB2EkS7kxEfhaXje+Lnw9LElAhvsuGtCaiUxxU82hScl8lkmDNnDj744AO89NJLuHz5MlauXIn09HTcd999KCkpCZSdBBFSlHIptGo5ohRCbhZrbj/mCuskI2XOB482CdexY8fw8MMPIyUlBRs3bsTKlSuRl5eHL774AsXFxZg1a1ag7CSIkNItVo0l47Iwa0g3AEBhZQM2/fcSdh4pdNrX6nHRWsXg4VcC6saNG7Ft2zZcuHABM2bMwPbt2zFjxgyx63RWVhZyc3ORmZkZSFsJol25WFaLk4XVyIiPws094u3e8yRKVNYm+PglXFu2bMHixYuxcOFCpKQ4BycBICkpCX/961/bZBxBhJLaJhOuVTdCq3a+TTyVbY5SCA1kNSpamBIs/Ppkv/jiC2RkZIgelhXGGIqKipCRkQGFQoEFCxYExEiCCAXWnC2pRIImkwW7T1yDhedx783dPTbK6JuiRd8Uaj4cTPyKcfXs2RPXr1932l5ZWYmsrKw2G0UQ4YB1VlBIMgXKappwvc4IM8/E5FQpLfcJCX596u5mU+rq6qBSUXMAomNg26labiNQZgtz2QyWaD98GiquWLECgND0cu3atYiKihLfs1gsOHz4MIYMGRJQA69du4ZVq1bh008/RUNDA7Kzs7Ft2zYMHz48oNchCEfMNsIlkXCQSjhYeAYTz9uJmiOFNxrw5Y9lSNAocdug1Ha1ubPgk3B9//33AASP6/Tp01AoWqo+KhQKDB48GCtXrgyYcVVVVRg7diwmTZqETz/9FImJibh06RK6dOkSsGsQhDssFntxkkslsPAWmC0MWpUcmQlRSIhROh1n5nlUNZigcKzHxfNAyUnAUAN0GwYoNcH+FTosPgnXV199BQBYtGgRNm/eDK02uAHIl156Cenp6di2bZu4jWJoRHshkQAKmUTMmpdLOTSZhN6KvZM16J3sWnjcVkC9sh8o/E74ufxHYMQSQCoPmv0dGb9iXNu2bQu6aAHAv/71LwwfPhx33XUXkpKSMHToULz99tsejzEYDKipqbF7EIQ/3NqnK5ZNysaw7nEAWuJZjr0WHRGFy3a/xmqg6LDN6yqg+PuA2tuZ8NrjmjNnDnJzc6HVajFnzhyP++7evbvNhgFC+ZwtW7ZgxYoV+O1vf4ujR4/i0Ucf9ZhqsWHDBqxfvz4g1ycIW6KVMu/WKooel83G4u8BxoC4LCChF3Bxj7AtbYRQ1IvwCa+FS6fTiXWHdDpd0Ayyhed5DB8+HL///e8BAEOHDsWZM2fwxz/+0a1wrV69WpxEAICamhqkp6e3i71Ex+au4S3/R5+fLcWl8jqM6RmPoRn2MVdRuGzLQ1y/KDynDAHiegCX9wINlUD9dSAmMdimdzi8Fi7bOJPtz8EkJSUF/fr1s9vWt29f/OMf/3B7jFKphFLpHDAlCF/Zf7EClfUGjMyKR7dYtd17JguD0ex6yChWh7C+3VApPCRSweOSKYAu3YEbecCNyyRcfuBXjKuxsRENDQ3i64KCAmzatAmff/55wAwDgLFjx+LChQt22y5evIju3bsH9DoE4YpSfSPyrzeg0ehcELClOqrzLSSXSaBTy1uWCt3IE5516YCs+Us1rofwXF0QcLs7A34t+Zk1axbmzJmDhx56CNXV1Rg5ciQUCgWuX7+OjRs34le/+lVAjHv88ccxZswY/P73v8fcuXNx5MgR/PnPf8af//zngJyfIDzhmGR6+Kcb+Ol6PQanxXrM49Kq5Fg8zmb2u+aq8NzF5gtXl9b8XrEQ+6I4l0/4XQH1lltuAQB8+OGHSE5ORkFBAbZv34433ngjYMaNGDECH330EXbu3IkBAwbgueeew6ZNmzB//vyAXYMg3OEoTjVNZpTqm1DbZBJFzVXPRSdqioVnbbeWbdGJgFQGmA3CMJLwCb88roaGBmg0Qg7L559/jjlz5kAikeDmm29GQUFgXd/bbrsNt912W0DPSRDeYHZKQG2ps+XJ47LDUAs01QgelcamkopEKryuLhI8suh49+cgnPC7WcY///lPFBUVYc+ePfjZz34GACgvL2+X/C6CaA94Zj9UtNadN1p4m2Gk8y3E8ww7Dhdg+6F8GCqLhI3RCUJQ3harkNWVB8H6jo1fwrV27VqsXLkSmZmZGDVqFEaPHg1A8L6GDh0aUAMJIlSYHbwq2xZlSRolusWqoVI430IcB1TUGnCjzghe72KYaCW6eTaxnno0+IpfQ8Wf//znGDduHEpKSjB48GBx++TJk/G///u/ATOOIMIBUbiaPS6zhcd0F919rHAcBynHwcwYmNWbiunqvCMJl9/4XaIxOTkZycnJdttGjhzZZoMIIlx4aEJPuxJOCmuLMm+y56Wc4LHVN9eti3aRqxWdILhnxgbAWA8oogNid2fAL+Gqr6/Hiy++iL1796K8vBy8QwM56nRNdBRsu1QrZBIo5RKvanBJOQ5S3gjWpAcUMkGknHaSA6pYYd1ifQUJlw/4JVz3338/9u/fj1/+8pdISUlxakFOEB0Ra0UIxhi27MuDTMJh/s0ZiFI430ZSCQe1qRJMBqF8jVztfEJAELTGKsEz65IZVPs7En4J16effopPPvkEY8eODbQ9BBEWGMwWfHamFFIJhxkDUiCx8bLMPEOTScimd5cOIZVwUJiqwEuZ62GilegE4PqlliEl4RV+CVeXLl0QFxcXaFsIImwwWRh+qqgHxzkntZttyj7I3dScj1HKoIEeHOB6mGhF3XwfNVa1zeBOhl/pEM899xzWrl1rt16RIDoS1uqnQqMMQbn0DSb84/hVfHhCWMIjl3J2npgtdw1Px7QsGTQquWePS91cWYKEyyf88rhee+015OXloWvXrsjMzIRcbl/F8cSJEwExjiBChbV6qW0XHwtjKKxs+bK2JqS6peGG8OzR42oWLkMNYDELy4CIVvHrU5o9e3aAzSCI8KKl+kOLR+XYQ1HmSbjMRsBQJ/xsFSdXKKKF2UWLCWjS09IfL/FLuNatWxdoOwgirHC1FtExnqXwsMD6u7OXEH1Nj9TEOMS7m1EEhACauouw7KexioTLS/zuZlldXY2//OUvWL16NSorhdXtJ06cwLVr1wJmHEGECscF1oB9JYj4GIXLDj9WGvQVqDOY0Sjzolowxbl8xi+P64cffsCUKVOg0+mQn5+PBx54AHFxcdi9ezcKCwuxffv2QNtJEO2KK49L2tzRmjFgzk1piFG6v33UZqFJi0lBwhUM/PK4VqxYgYULF+LSpUt2natnzJiBAwcOBMw4gggVmQnReGxyDuaNaKkzz3GcGJA3t9LpR2nSAwCMJFxBwS+P6+jRo/jTn/7ktL1bt24oLS1ts1EEEQ5IJBwksI9jKWUSSDiu1RZlSpMeBgBGeWzrF1I379NU7Y+ZnRK/PC6lUumyX+HFixeRmEiF/4mOy+KxWRicpsO/ThbjWL77yqVWj6tJ7oXHpWrep6lGGIcSreKXcN1xxx149tlnYTKZAAgudGFhIVatWoU777wzoAYSRCgouFGP/ztdgu8L7YdvEgmHeqMFtU1mMQ7mhKkJMksjAMDojXAptcLsIm8GjHVtNb1T4Jdwvfbaa6irq0NiYiIaGxsxYcIEZGdnQ6PR4IUXXgi0jQTR7lTWG3GhtBbF1U1O71nXKarkUtcHN1ZBKuHA5NHg5F60ypNIhYXYgOB1Ea3iV4xLp9Phiy++wMGDB3Hq1CnU1dXhpptuwpQpUwJtH0GEBJ45zyoCwLniGlwuF7yiKIU74apEepcopGelAz09ZM3botQKotWkB3QuqqUSdvgsXDzPIzc3F7t370Z+fj44jkNWVhaSk5PBGKMSN0SHwGyzVtGWRlNLj0Wt2n6pm4i1a4+njHlHVDpAf1UQLqJVfBoqMsZwxx134P7778e1a9cwcOBA9O/fHwUFBVi4cCGVbSY6DO66+KToVOL2LlEKp+MAtKQ1qH2ooGIN0BtoqOgNPnlcubm5OHDgAPbu3YtJkybZvffll19i9uzZ2L59O+67776AGkkQ7Y1jowwrKToVpvZPhlYtg0Lm5nu/sRJVDUYcuWKAxnQdY7K9GC6qmrtjkcflFT55XDt37sRvf/tbJ9ECgFtvvRVPPfUUduzYETDjCCJUWJjroSLHceiXqkValyj3BzdWwWThkV+vREWdwbsLiikRJFze4JNw/fDDD5g2bZrb96dPn45Tp0612SiCCDUWF2sVvcLYAJiawIFDk0znPmXCEVWs8Nykp1wuL/BpqFhZWYmuXV20WWqma9euqKqiZQtE5DOxdyLG5ST4LlzN8S2m0oDnZN4Ll7J5qGgxAaZGQOHBoyN887gsFgtkMvdaJ5VKYTab22yUO1588UVwHIfly5cH7RoEAQi1tlRyaevFAh0RhSsWALwXLqmspcsPDRdbxSePizGGhQsXQql0nVRnMHg5nvcD6/rIQYMGBe0aBNFmxBnFLoChJVbmFSqd0F/RUAPAfcNZwkfhWrBgQav7BGNGsa6uDvPnz8fbb7+N559/PuDnJwhHjhdUorLehAHdtEjReSgE6IitcFXbN9ZoFZUWqCkmj8sLfBKubdu2BcsOjyxbtgwzZ87ElClTSLiIduGninpcrWpE9/gopHix3FCkWbgkUV0glXDwKURGM4teE/aV+Xft2oUTJ07g6NGjXu1vMBjshqyuqlgQRGtYY1MSX1eCNAtXQmIyHu2R7NuxShIub/G7dHN7UFRUhMceeww7duywK1joiQ0bNkCn04mP9PT01g8iCAesCaiOeVweMTUJM4IAOF+y5q1Q9rzXhLVwHT9+HOXl5bjpppsgk8kgk8mwf/9+vPHGG5DJZLBYLE7HrF69Gnq9XnwUFRWFwHIi0nG3yNoj1viWIhqQuVkO5Akxe56EqzXCeqg4efJknD592m7bokWL0KdPH6xatQpSqfPqfKVS6XbWkyC8RVxk7aGTjxM2gXmThcf/nS6BhWeYNaSbdwJozeUyNQrtzfwRv05CWAuXRqPBgAED7LZFR0cjPj7eaTtBBBJ3i6w9YiNcHIQAPwCYLDykEjclcGyRqwSxMhuF4aLMy5I4nZCwHioSRKgQF1n7Epy3ES5rRyDAhyRUgGYWvSSsPS5X7Nu3L9QmEJ2AhWMyYeZ5RCl8uEVsPa7mjkBGM+9bLpdSB9RVkHC1QsQJF0G0B2qFFIAXwztbbJNPIcxIGgGYeM8dgeywBuhpZtEjNFQkiEBgNgrLdQBRuKzxMd+y5206/hBuIY+LIBwwWXh8fakCEo7DLTmJ3gXord6WXC0E2QFxgbaplR6MdijJ4/IG8rgIwgGzheFUkR7fF1Z7v2THYZgICKkUHNeSE+YVVAnVK8jjIggHzM0xKWFm0EvlciFc94zIAMfBtwYyosdVB/A8ICHfwhX0qRCEA23N4bIi8UX4rCg1Qp9FxgPGWt+O7USQcBGEA36tU2z0oyWZKzjOpjksDRfdQcJFEA745XE13BCeo1uy3U8WVePfp4rFBrJWzBYevKekVCWtWWwNEi6CcMBnj8vUKDTJAOx6KZbVNOFyeR2qGox2u5+6qkfut/k4X+JGmKhKRKuQcBGEA2KHH2/rzVu9LaXGbmG0vHmBtmM6RGFlPfSNJjSZnKubAKAqEV5As4oE4UBqrAqLx2V5f0BDc3wrKt5us6x5RtA2AZUxhlK9UOiSZ8Jwsl+K1r65rJJSIlqDhIsgHJBJJdCpfRiMWD0uR+Fq9rjMNkt+ag1mNJkskHAcjuZXotFoQZJGidRYm7r2NFRsFRoqEkRbcSNcCjFzvsXjqmk0AQA0KhmSNELduMp6+xiYXYUIag7rEhIugnCguLoRBy5WuA+eOyIOFe3LNVuHf0Zzi8dV0yj0HdWq5dCp5c3bTPbns6ZDWJvDEk6QcBGEA+W1BhwvqEJeRV3rO/OWluRTB4/LulbRTriaBJHSqmTQWoWryUG4pPKWTtY0XHQJxbgIwgFLc0zKq3SIxmohy10qb/GUmslJikH2rdl251HJpUjUKBEfo0CM0upxuej+rtIJKRZNNYDGx25BnQASLoJwwDoLKPVmnaBtfMtheY/MRTrFkPRYDEmPBQCU6psAAHrHoSLQPLNYQh6XG2ioSBAOWHxJQG24Ljw7DBO9QaMS/IZ6o9k5k17M5ar2+bydAfK4CMIBsy9LfurKheeYJKe3Go0WHLhUAZ5nmD4wBax5htC68Fotl2LagGREuyoPrYoVnikJ1SUkXAThgE8eV32F8Byd6PQWA8O5YkF4pvZPRr3RjHe+zYcuSoF7R2VAIuHQN0Xr+rxUUNAjNFQkCAe8XmRtMbekQrgQLoVNjMto4VHTZIbJwmA0862Xu6GCgh4hj4sgHBjdMx5DMmKhlrfSLKPhhjCjKFc5zSgCgvBJJRwsPIPJwqPWJhXCSllNEypqDUjSKJGkVbUcbPW4jA1CPpdU3ubfqyNBHhdBOBCtlCEhRoloZSvf6/XN8a3oRKcZRQBiizJAyOWyTT61cvqqHl+cK0Nec/NYEbm6RawMVFDQERIugvAXa2A+2jkwb0UlF26xJjMvZshrVS3CZRXHeoNDLhfH2Sz9qQ6MvR0IEi6CcODMNT0O5d3A9TqD5x3rm1MhYpzjW1aiFMJws9FoFjPkNTZDxWil8H690UUSKhUUdAvFuAjCgXMlNbhW1Yj4GAUSYpSud2IMqCsTfnYRmLeikluFixcTTXVqVx6Xi9pcVCXCLSRcBOGANXNe7qmQoKFWaADLSYCYrm53m9o/GRKOg0ImQWFlA2RSCWKjbISrOYerwZXHpY4Vnq1rIQmRsB4qbtiwASNGjIBGo0FSUhJmz56NCxcuhNosooNj9matYm2J8Byd4HHGTyWXilUiZg5KwS9v7g6NXYyreahosIgJqiLWxhskXE6EtXDt378fy5Ytw3fffYcvvvgCJpMJP/vZz1BfX9/6wQThJ9ZqDh49LqtwaVLadK2oZo+LZwyNjqWcSbjcEtZDxc8++8zudW5uLpKSknD8+HGMHz8+RFYRHR2xWYbUg8dV0yxcWs/CVV7bhJOF1ZBLJZjYO9Ep8VQq4TBzUArUcqldwiqAlmU/piahLpdcDUIgrIXLEb1eyCKOi4tzu4/BYIDB0DIbVFNDgU3CN8zNzS3k7qpDMOa1x8XzwNnmZT/nSmpwc494DOtu33uxV1fn5FUAQuMNZYzQ1bqxioTLhrAeKtrC8zyWL1+OsWPHYsCAAW7327BhA3Q6nfhIT09vRyuJSIcxJpZalsvceFyNVYDZAEhkHmcUAaBLdEs8y2jmoZT5eMvRcNElESNcy5Ytw5kzZ7Br1y6P+61evRp6vV58FBUVtZOFREdh/s0ZmDciHSqZmyU/NcXCc0wSIPG8LEgpkyLGJgM/Wady2qe8pgmnr+pRXO2iTDMJl0siYqj4yCOP4D//+Q8OHDiAtLQ0j/sqlUoolW5ybwiiFTiOQ5LGWVzs0F8VnnXdvDpnv1QtjlypRIJGifhohdP7F8pqcSy/CkMzYu27/QAkXG4Ia+FijOHXv/41PvroI+zbtw9ZWT70uiOIYFFdKDzHdvdq99E94pGkUSIlVu2yKkSUmMvlIgnVKlzWKhQEgDAXrmXLluHdd9/Fxx9/DI1Gg9LSUgCATqeDWk2BSiLwNBjNOH1VD7VCikFpsc47GOtbyjXrPHv/ViQSDjnuAvCwzeVylYRKHpcrwjrGtWXLFuj1ekycOBEpKSni47333gu1aUQHpbbJjG/zbuDIFTceTnVzzDQmMWCzfNHeeFymRmpVZkNYe1xOmcQEEWRMllay5vXNwqXLCNg1xfWKrpb9yJRCUcGmGmFRdyzNkgNh7nERRHsjrlN0l7ZQXSA8B1BArBUkDCZeFE77HRKEZ2tjDoKEiyBssa5TdJl82lQD1FUItbK8DMx7g1ImET28BldVIqKbOwjV3wjYNSOdsB4qEkR7YzR7WO5T+ZPwrElp6TQdADiOw/SBKVDKJIhSusgLI4/LCRIugrBBrAzhaoF1ZZ7wHN8z4NfNTopx/6Y1O9/aUYigoSJB2GIS1yk6eFy8Bai8IvwcF3jh8oi12ayhTlhwTZDHRRC29EnWIlmndl5TWF0odNtRRAOa5IBft6LWgFJ9E2Kj5EiPcxiGWrsIGWqF4aKX+WMdGfK4CMKGaKUM3WLVziWbK5oLWMZnu+zo01byKurw3/NluFDqpqOPtVO2tVx0J4eEiyBag+eBih+Fn5P6BOUS1iRUl7lcQEt56NrSoFw/0qChIkHYcKmsFnUGM7rHRyPOuiC6Ol/IWldEAbGZQblulE0JZ5dY636RcAEgj4sg7Dh9TY99FypQVmMTBC8/Lzwn9AbcFRdsI9bSNy6bZgCAptnjqr8OWNzs04kg4SIIG6z15q0NLmA22gwT+wbtutbs+Qaji6YZgNBjUREFML6lg3YnhoSLIGwwWIXLmsdVcV4QL3UXIDZw6xMdsZa2sfAMTSYXy344Dohpns20lo3uxJBwEYQNVo9LTIcoPik8pwwOymyiFamEg1rhoas10JKGYa3A2omh4DxB2GC02AwVa8sEkeAkQPLAoF97av9kyCQctCo3fRpj04ECtFRg7cSQcBFEMxae2XhcUiD/sPBGYi+h206QyUqI9ryDtpsgoo3VQJMeUOmCblO4QkNFgmimqbkhK8cBKktNy2xi+s0htMoGmbJldrGTe13kcRFEM0qZBPNGpMNg5sFd/VaYweuS2WrT10ChbzSh8EYD5DIOfZK1rnfSpQnNaKsKgK7928WucIQ8LoJoRiaVIDVWjaxoI1D8vbAxo/28rYpaA/57vgzfF1a736lLc8OYyjyhMW0nhYSLIBy5ckCoBtElU3i0Ezq1EJSvbjC53ym2OyCVC5UiOvG6RRIugmimVN+EM+fOoqbglLCh56SgpkA4YhWuJpNFjLc5IZW1iOn1S+1jWBhCMS4i/DDUAnXlQuE8Yz1gbhJKyjBeeADCMEkqFx7yKKFmlSZZKLrnp9jkl1ej4dhHiFIZoM0aFpTyNZ5QyCSIVkpRb7BA32iCSu6mS3ZCjiBaFT8CmePaVVzDBRIuIjwwNQrJnhXnhfwpf1HGAIl9gdShLbXavUSR/xWYqRrSLolAz1v9t6ENxEUrUW9oQEWtAV21bjpqJ/QGpJ8L6xZrSwBtavsaGQaQcHUGeF4oQFdfAZgNwrS6uouwhCRIi4a9xmwECg4C144LXhUgeBBR8YL3pIwRPCqJXMhh4jjhGQB4M2AxCvGehutAzTXh56tHhUdcFpA2Aojr0bpXcvU41BU/wAAOxuypAeuZ6CuJGiWKKhtQUWdwv5NcBST0AsrOAaWnSbiIDkb9DWF2rOyM62aiMiWQ2AfoNqwlP6g9qfwJuPCZkEwJCE1Wuw0TbkpFK8mYrrCYgap8oOQkcOOyUGq58opw3rSRQFI/IUZkC2NA0REg70sYzDyKdMOR1rVXW38zv0lsLmBYUetBuABhCVLZOaD0B2G46M/nFcGQcHVEmvRA/jfCt7F1ylwqF2I2MpXgddWXC/XLS04Jj4QcoMdEIDoh+PYZG4C8vUDpGeG1Sgtk/49gQ1viNVIZkJAtPBqrgKvHBRGrqwB+/AT4aR+QNlyoYiqVC3G0q8eA6kIwMBRG9cc17RCxxEwo6B4fhZ8PS3OuwOpIbHfh71lbKniXPSa2i33hAglXR8LU1DLs4ptnpeKzgW43Cfk/tsNCnhe6MhefEMoSX78keCnJg4CsW4Qa54GGMaDsLHD5v4IHyHFAt+FA1nhApgjstdRdgJwpQOZYIXZ27ZgwjPxpv/CwRSJDfdp45JliIZVKxNm9UBCtlImdrT3CcUD3scCZfwBFR4W/W1Rc8A0ME0i4OgI8D5R8D1z5umVI2KU7kDUB0HVzfYxEIuzTpbswpLyyD6i4KHhf5WeB9FHCQ9bKN7+3NFQCF/cIQzlAGL71mu7evkAhVwPdRwPpI4Hyc8LvV18hDCtVOiH+lTYcZXUygCtGXLQCEscOP+FKQo6QGlGVD1z4FBh8NyBxMxPZweCYy6pl4cVbb72FV155BaWlpRg8eDDefPNNjBw50qtja2pqoNPpoNfrodW6WUYRqfAWwYMp/A5oaO5yHJ0gzIh5E5B2RH8VyPsS0F8TXiuigO7jgJRBwtDKH4z1gn1WL1AiE7yg9FFhdZMxxlBRZ0CDwYLM1hY7B5kmkwUnCqpQXmvArCGp4Dz9HRsqgWNbhYmNlEHCl0GoJ1x8wN/7M+w9rvfeew8rVqzAH//4R4waNQqbNm3C1KlTceHCBSQlJYXavNDQUCkE3EtPC23hAWGmKXM8kDrEf0HQpQFDfwlcvyjEgxoqgUufA/kHhLIuiX0ATWrrNwbPCzN8pacF781aajiuB5DzP2E5pOE4DkkaFRCEEbI/HC+ogplnuFbdiLQuHrpmR8UB/WYJQ8aSH4ThcO9pHb5yRNh7XKNGjcKIESPwhz/8AQDA8zzS09Px61//Gk899VSrx0esx8XzQuKlqRFoqhZEpL5c6O/XWN2ynyJaGAalDg3csA4QvKPik8DVI/bXk6uFxg3RCUIcTKYCwARxMuiFYae+SJgAsKJJFuJY/niB7YC+wQStWubZs2ln/nuuDKev6dElSo5ZQ7qhS3QrMcDyH4Hz/xZSRCRSYRgZ11PoDqSODez/RgDx9/4Ma+EyGo2IiorChx9+iNmzZ4vbFyxYgOrqanz88cdOxxgMBhgMLTdNTU0N0tPTW/9gCr4Vhl2O2H08zMX2tmyz2W67jVmE/CZ3cBIhR6lrfyF1wN9hnDfwvJC2UHZGeDa3Mk1vRaYQEiWTBwolj0MsCjzP8PfDBeB5Bp4BPGPgGYOFF4ZmvZM1mNw3SajDFQY0Gi3YcbgAtU2CtxqtlEIhleDmnvFi5YiymiZ8fq4MEg6YP6q7MHt66XPhy80RibR5pYGiOQ+Oa/6bcPZ/m0D+nRQaYPA8j7t0yKHi9evXYbFY0LWrfY5R165d8eOPP7o8ZsOGDVi/fr3vFzPUCZnI4YZMKXg2UXFCUqYuXRjStdc3qETSkmLAW4Tp9/py4bMyNQgzmZxEuDGUGkAdJ3hYmpSwirVwHHCjzvWXAccBEo6DNIw8LrVCiruGp+PLH8uQf70B9QYL6mGxq0dvNPO4XmuAxGp3TCIwdL7wN6r4URCwhkrBa+ctwsPU5OaKQUDl4cu3jYS1cPnD6tWrsWLFCvG11eNqlW7DXOQRNf/sapvddm+3tXJO6zZOIgzBZKqwuvkhkQqzgMGeCQwCHMfh58PSWkRKwok/xyhl7tcFhhCdWo7/HZqGJpMFNY0mGC28XapGokaJO29Kcz5Qk2y/ztJsEB4Wo/BgDABz8xxAJMGTl7AWroSEBEilUpSV2a9dKysrQ3Ky6wWwSqUSSqUf3kh0vM9r24jIIj3OQ5A7jFHJpS6FVSWXIiPei99JpgzbGJe/hNHXuTMKhQLDhg3D3r17xW08z2Pv3r0YPXp0CC0jCCKUhLXHBQArVqzAggULMHz4cIwcORKbNm1CfX09Fi1aFGrTCIIIEWEvXPPmzUNFRQXWrl2L0tJSDBkyBJ999plTwJ4giM5DWKdDBIKIzeMiiE6Av/dnWMe4CIIgXEHCRRBExEHCRRBExBH2wfm2Yg3h1dTUhNgSgiAcsd6XvobaO7xw1dbWAoB32fMEQYSE2tpa6HTeV7To8LOKPM+juLgYGo0mrFb/tyfWZU9FRUU0s+oH9Pm1DU+fH2MMtbW1SE1NhcSH5W0d3uOSSCRIS3OxnqsTotVq6cZrA/T5tQ13n58vnpYVCs4TBBFxkHARBBFxkHB1ApRKJdatW+df1QyCPr82EozPr8MH5wmC6HiQx0UQRMRBwkUQRMRBwkUQRMRBwkUQRMRBwtVBqaysxPz586HVahEbG4slS5agrq7O4zETJ04Ex3F2j4ceeqidLA4tb731FjIzM6FSqTBq1CgcOXLE4/4ffPAB+vTpA5VKhYEDB+L//u//2snS8MSXzy83N9fp/0ylUvl2QUZ0SKZNm8YGDx7MvvvuO/b111+z7Oxsds8993g8ZsKECeyBBx5gJSUl4kOv17eTxaFj165dTKFQsK1bt7KzZ8+yBx54gMXGxrKysjKX+x88eJBJpVL28ssvs3PnzrHf/e53TC6Xs9OnT7ez5eGBr5/ftm3bmFartfs/Ky0t9emaJFwdkHPnzjEA7OjRo+K2Tz/9lHEcx65du+b2uAkTJrDHHnusHSwML0aOHMmWLVsmvrZYLCw1NZVt2LDB5f5z585lM2fOtNs2atQo9uCDDwbVznDF189v27ZtTKfTtemaNFTsgBw6dAixsbEYPny4uG3KlCmQSCQ4fPiwx2N37NiBhIQEDBgwAKtXr0ZDQ0OwzQ0pRqMRx48fx5QpU8RtEokEU6ZMwaFDh1wec+jQIbv9AWDq1Klu9+/I+PP5AUBdXR26d++O9PR0zJo1C2fPuugi74EOv8i6M1JaWoqkpCS7bTKZDHFxcSgtLXV73C9+8Qt0794dqamp+OGHH7Bq1SpcuHABu3fvDrbJIcOfbumlpaUu9/f02XZU/Pn8evfuja1bt2LQoEHQ6/V49dVXMWbMGJw9e9brgggkXBHEU089hZdeesnjPufPn/f7/EuXLhV/HjhwIFJSUjB58mTk5eWhZ8+efp+XIGwZPXq0XV/UMWPGoG/fvvjTn/6E5557zqtzkHBFEL/5zW+wcOFCj/v06NEDycnJKC8vt9tuNptRWVnptgO4K0aNGgUAuHz5cocVLn+6pScnJ/u0f0fGn8/PEblcjqFDh+Ly5cteX5diXBFEYmIi+vTp4/GhUCgwevRoVFdX4/jx4+KxX375JXieF8XIG06ePAkASElJCfSvEjb40y199OjRdvsDwBdffNEpu6sHotu8xWLB6dOnffs/a1Nonwhbpk2bxoYOHcoOHz7MvvnmG5aTk2OXDnH16lXWu3dvdvjwYcYYY5cvX2bPPvssO3bsGLty5Qr7+OOPWY8ePdj48eND9Su0G7t27WJKpZLl5uayc+fOsaVLl7LY2Fhxiv6Xv/wle+qpp8T9Dx48yGQyGXv11VfZ+fPn2bp16zp9OoQvn9/69evZnj17WF5eHjt+/Di7++67mUqlYmfPnvX6miRcHZQbN26we+65h8XExDCtVssWLVrEamtrxfevXLnCALCvvvqKMcZYYWEhGz9+PIuLi2NKpZJlZ2ezJ554olPkcTHG2JtvvskyMjKYQqFgI0eOZN9995343oQJE9iCBQvs9n///fdZr169mEKhYP3792effPJJO1scXvjy+S1fvlzct2vXrmzGjBnsxIkTPl2PytoQBBFxUIyLIIiIg4SLIIiIg4SLIIiIg4SLIIiIg4SLIIiIg4SLIIiIg4SLIIiIg4SL6FDk5uYiNjZWfP3MM89gyJAh4uuFCxdi9uzZ7W4XEVhIuAi/WLhwodvSzsuWLQPHcXYLwoMhGJmZmdi0aZPdtnnz5uHixYtuj9m8eTNyc3PF1xMnTsTy5csDahcRfEi4CL9JT0/Hrl270NjYKG5ramrCu+++i4yMjJDYpFarnWqR2aLT6ew8MiIyIeEi/Oamm25Cenq6XaHB3bt3IyMjA0OHDm3TuV15QrNnzxa9uIkTJ6KgoACPP/642HABcB4qOmLr+S1cuBD79+/H5s2bxXNcuXIF2dnZePXVV+2OO3nyJDiO86n0ChE8SLiINrF48WJs27ZNfL1161YsWrQo6NfdvXs30tLS8Oyzz6KkpAQlJSU+n2Pz5s0YPXo0HnjgAfEcGRkZTr8TAGzbtg3jx49HdnZ2oH4Fog2QcBFt4t5778U333yDgoICFBQU4ODBg7j33nuDft24uDhIpVJoNBokJyf7VcRPp9NBoVAgKipKPIdUKsXChQtx4cIFscWWyWTCu+++i8WLFwf61yD8hCqgEm0iMTERM2fORG5uLhhjmDlzJhISEkJtVptITU3FzJkzsXXrVowcORL//ve/YTAYcNddd4XaNKIZ8riINrN48WLk5ubinXfeCZhXIpFI4FhxyWQyBeTc3nD//feLEw/btm3DvHnzEBUV1W7XJzxDwkW0mWnTpsFoNMJkMmHq1KkBOWdiYqJd3MpiseDMmTN2+ygUClgsljZdx905ZsyYgejoaGzZsgWfffYZDRPDDBoqEm1GKpWK3YWkUqnb/fR6vVjH3kp8fDzS09Od9r311luxYsUKfPLJJ+jZsyc2btyI6upqu30yMzNx4MAB3H333VAqlX4NUTMzM3H48GHk5+cjJiYGcXFxkEgkYqxr9erVyMnJ6ZT15MMZ8riIgKDVaqHVaj3us2/fPgwdOtTusX79epf7Ll68GAsWLMB9992HCRMmoEePHpg0aZLdPs8++yzy8/PRs2dPJCYm+mX3ypUrIZVK0a9fPyQmJqKwsFB8b8mSJTAaje0yS0r4BpVuJgg3fP3115g8eTKKioqcGp4SoYWEiyAcMBgMqKiowIIFC5CcnIwdO3aE2iTCARoqEoQDO3fuRPfu3VFdXY2XX3451OYQLiCPiyCIiIM8LoIgIg4SLoIgIg4SLoIgIg4SLoIgIg4SLoIgIg4SLoIgIg4SLoIgIg4SLoIgIg4SLoIgIo7/D0DAuOsVsHreAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:14:01.819754Z", + "iopub.status.busy": "2024-02-29T18:14:01.819467Z", + "iopub.status.idle": "2024-02-29T18:14:02.040000Z", + "shell.execute_reply": "2024-02-29T18:14:02.039139Z" + }, + "papermill": { + "duration": 0.242771, + "end_time": "2024-02-29T18:14:02.041926", + "exception": false, + "start_time": "2024-02-29T18:14:01.799155", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEmCAYAAAAdlDeCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyU0lEQVR4nO3deVwTd94H8A9nCIaAKJeWQ0QBWxUFRXx00XKIVNdrPfGAulptbWvxtlXrY7s8WrzXystu1dV61Xo8rutBKl5VBMSjihFXRHGFiIoQDgkBfs8fPswaw5FoIMd836+XL5mZ32S+wyQf5sj8xowxxkAIISbOXN8FEEJIS6CwI4TwAoUdIYQXKOwIIbxAYUcI4QUKO0IIL1DYEUJ4gcKOEMILlvouwNDV1tYiPz8fdnZ2MDMz03c5hJBXMMZQWlqKdu3awdy88X03Crsm5Ofnw93dXd9lEEIa8fDhQ7zzzjuNtqGwa4KdnR2Al79MsVis52qaj1KpRHJyMiIjI2FlZaXvcshb4sv2lMvlcHd35z6njaGwa0LdoatYLDb5sLO1tYVYLDbpDwdf8G17anKKiS5QEEJ4gcKOEMILFHaEEF6gsCOE8AKFHSGEFyjsCCG8QF894amKigrcvn2bGy57ocDFGzlo3fYyREIBN97Pzw+2trb6KJEQnaKw46nbt28jMDBQbfyq14YzMzPRs2fPlimKkGZEYcdTfn5+yMzM5IazC4oRv/8G1ozuCl83B5V2hJgCCjuesrW1VdljM3/wDILzL+D/XncEeLbRY2WENA8KO0JMgKbnYAH+noelsCPEBGh6Dhbg73lYCjtCTICm52Dr2vIRhR0hJoDOwTaNvlRMCOEFCjtCCC9Q2BFCeIHO2RFipHKflqNcUV3vtJwn5dz/lpYNf8xbCSzRoW2rZqnP0FDYEWKEcp+WY2DimSbbzfnlRpNtTs8dwIvAo7AjxAjV7dGtGxsAH2eR+vQXChw9k4ohA0LQ6rUvFde5W1iG2fuuNbh3aGoo7AgxYj7OIrzX3l5tvFKphMwJ6OnZmhcP3NEEhR0hRkhRUwlzm0fIlWfD3EZ9z666uhr51fmQFkkbPGeXKy+Duc0jKGoqAagHpqmhsCPECOWXP0CrDhuxOL3xdt+f+L7R6a06APnlAQiEiw6rM0xGF3abNm3Cd999B5lMhu7du2Pjxo3o3bt3vW2zsrKwdOlSZGZm4sGDB1i7di1mz57dsgUT0gzatfJEee6nWD82AB3rOWdXXV2NC79dwH/1+68G9+xyCsvw+b5raDfQs7nLNQhGFXb79u1DfHw8kpKSEBwcjHXr1mHQoEHIzs6Gs7OzWvuKigp4e3tj9OjR+OKLL/RQMSHNQ2Bhg9rK9ugg9kWXNvWfs8u1zIW/o3+D5+xqK0tQW/kEAgub5i7XIBjVl4rXrFmDadOmIS4uDl26dEFSUhJsbW2xdevWetv36tUL3333HcaNGweBoP4rUoQQfjCasKuqqkJmZibCw8O5cebm5ggPD0dqaqoeKyOEGAOjOYx9+vQpampq4OKieiLVxcVFpdPCt6VQKKBQKLhhuVwO4OVhgVKp1NlyDE11dTX3vymvp6loanvVjWtsW5rCNtembqMJu5aSkJCA5cuXq41PTk426d5dH5YBgCUuXbqERzf1XQ1pSt32+u233/BA/foERyKRvPVrGLKKigqN2xpN2LVt2xYWFhZ4/PixyvjHjx/D1dVVZ8tZtGgR4uPjuWG5XA53d3dERkZCLBbrbDmG5npeEXDjMvr06YPuHo76Loc0IStfjsQbl9CvXz+82079falUKiGRSBAREdHgBYqmXsMY1B15acJows7a2hqBgYE4deoUhg8fDgCora3FqVOnMGvWLJ0tRyAQ1Hsxw8rKyqS/iV739QRLS0uTXk9Toen2aux9awrbXJu6jSbsACA+Ph5TpkxBUFAQevfujXXr1qG8vBxxcXEAgMmTJ6N9+/ZISEgA8PKixq1bt7ifHz16hGvXrkEkEsHHx0dv60EIaXlGFXZjx47FkydPsHTpUshkMgQEBODEiRPcRYu8vDyYm//nAnN+fj569OjBDScmJiIxMRGhoaE4c+ZMS5dPCNEjowo7AJg1a1aDh62vB5iXlxcYYy1QFSHE0BnN9+wIIeRtGN2eHXkzjfVqC1DPtsT0UdjxgKa92gLUsy0xXRR2PNBUr7YA9WxLTB+FHY801KstQD3bGpsXyhoAwM1HJfVOL3+hwOUngOuD543+8eITCjtCjFDO/wfVwoONnXawxM67GU2+VisBP2KAH2tJiImJfPflLZIdnUUQWlmoTc8uKMGcX25g9Z+6wtet4S7X+XTBicKOB5p6XgFAzywwNo6trDGut0eD0+t6NOno1KrBUxd8Q2HHA5o+rwCgZxYQ00VhxwNNPa8AoGcWENNHYccDTT2vAKBnFhDTR7eLEUJ4gcKOEMILFHaEEF6gsCOE8AKFHSGEFyjsCCG8QGFHCOEFCjtCCC9Q2BFCeIHCjhDCC0YXdps2bYKXlxdsbGwQHByM9PTG727fv38//Pz8YGNjg65du+LYsWMtVKnheLWjx4b+XXnwHJefAFcePG+wDd86eySmxajujd23bx/i4+ORlJSE4OBgrFu3DoMGDUJ2djacnZ3V2l+8eBHjx49HQkIChgwZgt27d2P48OG4cuUK3nvvPT2sgX5o1tEjQJ09ElNmxozowarBwcHo1asX/vrXvwIAamtr4e7ujk8//RQLFy5Uaz927FiUl5fj6NGj3Lg+ffogICAASUlJGi1TLpfD3t4eJSUlEIvFulmRFlZUXoXkLFmDHT0C1NmjsauoqMDt27e54eyCYsTvv4E1o7vC181Bpa2fnx9sbW1buMLmoc3n02j+RFdVVSEzMxOLFi3ixpmbmyM8PBypqan1zpOamor4+HiVcYMGDcLhw4cbXI5CoYBCoeCG5XI5gJe9giiVyrdYA/2xszbDqB5ujbaprKwEAHi2FsDXufEPgrH+HkzZzZs3ERwcrDZ+wt/V26alpaFHjx4tUFXz0+a9aDRh9/TpU9TU1MDFRbXTSBcXF5W/aK+SyWT1tpfJZA0uJyEhAcuXL1cbn5ycbDJ/DevzsAwALHHp0iU8uqnvaoi2FAoFVq9ezQ0ra4GiSsDRBrB67cz8/fv3UVBQ0MIVNo+KigqN2xpN2LWURYsWqewNyuVyuLu7IzIy0mgPY+tTUVGB7Oxsbri0oAQK2S04hHSB2yuHsb6+viYd8qZKqVRCIpEgIiLCpJ8WV3fkpQmjCbu2bdvCwsICjx8/Vhn/+PFjuLq61juPq6urVu0BQCAQQCBQf/SclZWVSb1pcnJy6j3smfTaYU9mZiZ69uzZQlURXTO19+3rtFk3owk7a2trBAYG4tSpUxg+fDiAlxcoTp06hVmzZtU7T0hICE6dOoXZs2dz4yQSCUJCQlqgYsPm5+eHzMxMbrjshQL/PJ2KDwaGQPTKc0b9/Pz0UR4hOmc0YQcA8fHxmDJlCoKCgtC7d2+sW7cO5eXliIuLAwBMnjwZ7du3R0JCAgDg888/R2hoKFavXo0PPvgAe/fuxeXLl7FlyxZ9roZBsLW1VdljUyqVeP60ECG9g0x6T4Dwl1GF3dixY/HkyRMsXboUMpkMAQEBOHHiBHcRIi8vD+bm/zkb27dvX+zevRtfffUVFi9ejE6dOuHw4cO8+o4dIeQlo/qenT6YwvfsNKFUKnHs2DFER0fTnp0J4Mv21ObzaXS3ixFCyJugsCOE8AKFHSGEFyjsCCG8QGFHCOEFCjtCCC9Q2BFCeIHCjhDCCxR2hBBeoLAjhPAChR0hhBco7AghvEBhRwjhBQo7QggvUNgRQnhB67C7d+9ec9RBCCHNSuuw8/HxwcCBA/HTTz9xzxolhBBDp3XYXblyBd26dUN8fDxcXV3x0UcfIT09vTlqI4QQndE67AICArB+/Xrk5+dj69atKCgoQL9+/fDee+9hzZo1ePLkSXPUSQghb+WNL1BYWlpi5MiR2L9/P1auXIm7d+9i7ty5cHd3x+TJk03mieOEENPwxmF3+fJlfPzxx3Bzc8OaNWswd+5c5OTkQCKRID8/H8OGDdNlnSgqKkJMTAzEYjEcHBwwdepUlJWVNTrPli1bMGDAAIjFYpiZmaG4uFinNRFCjIfWYbdmzRp07doVffv2RX5+Pnbs2IEHDx7gm2++QYcOHdC/f39s374dV65c0WmhMTExyMrKgkQiwdGjR3Hu3DlMnz690XkqKioQFRWFxYsX67QWQojx0fq5sZs3b8aHH36I2NhYuLm51dvG2dkZP/7441sXV0cqleLEiRPIyMhAUFAQAGDjxo2Ijo5GYmIi2rVrV+98s2fPBgCcOXNGZ7UQQoyT1mEnkUjg4eGh8jBqAGCM4eHDh/Dw8IC1tTWmTJmisyJTU1Ph4ODABR0AhIeHw9zcHGlpaRgxYoTOlqVQKKBQKLhhuVwO4OVzOJVKpc6WY2jq1s2U15FP+LI9tVk/rcOuY8eOKCgogLOzs8r4oqIidOjQATU1Ndq+ZJNkMpna8iwtLeHo6AiZTKbTZSUkJGD58uVq45OTk2Fra6vTZRkiiUSi7xKIDpn69qyoqNC4rdZhxxird3xZWRlsbGy0eq2FCxdi5cqVjbaRSqVavebbWrRoEeLj47lhuVwOd3d3REZGNvnEcWOmVCohkUgQERFh0k+Q5wu+bM+6Iy9NaBx2dQFgZmaGpUuXquzl1NTUIC0tDQEBAZpXCWDOnDmIjY1ttI23tzdcXV1RWFioMr66uhpFRUVwdXXVaplNEQgEEAgEauOtrKxM+k1Thy/ryRemvj21WTeNw+7q1asAXu7Z3bhxA9bW1tw0a2trdO/eHXPnztWiTMDJyQlOTk5NtgsJCUFxcTEyMzMRGBgIAEhJSUFtbS2Cg4O1WiYhhJ80DrvTp08DAOLi4rB+/foWPaTz9/dHVFQUpk2bhqSkJCiVSsyaNQvjxo3jrsQ+evQIYWFh2LFjB3r37g3g5bk+mUyGu3fvAgBu3LgBOzs7eHh4wNHRscXqJ4QYAGYknj17xsaPH89EIhETi8UsLi6OlZaWctNzc3MZAHb69Glu3LJlyxgAtX/btm3TeLklJSUMACspKdHh2hieqqoqdvjwYVZVVaXvUshbKi0tZUOHDmWenp5s6NChKp8TU6PN59OMsQauOLxi5MiR2L59O8RiMUaOHNlo24MHD751ABsSuVwOe3t7lJSUmPwFimPHjiE6Otqkz/GYut69eyMjI0NtfK9evUyyww5tPp8aHcba29vDzMyM+5kQYngaCjoAyMjIQO/evU0y8DSl0Z4dn9GeHTEGZWVlsLOza7JdaWkpRCJRC1TUMrT5fFK37ISYgFGjRum0nSnS6DC2R48e3GFsU3TdAQAhpGmv3inRtm1bfPPNNxAIBFAoFPjqq6/w9OlTtXZ8o1HYDR8+vJnLIIS8jVfPRhUUFIAxxp2WmDp1Kndqgs9nrTQKu2XLljV3HYQQHamuroaFhYXKMKFzdoSYhFcvTgiFQsTGxiInJwexsbEQCoX1tuMbjfbsHB0dcefOHbRt2xatW7du9PxdUVGRzoojhGhm/vz5WLJkCTe8e/du7N69u952fKVR2K1du5b7i7B27VqNL1YQQlrG62HXWDu+0ijsXu2Is6leSgghLc/a2hrz5s3Dd99912CbefPmqXTgwTdan7OzsLBQ624JAJ49e6ZyUpQQ0rJWrVqFefPmqR15mZubY968eVi1apWeKjMMWoddQ5euFQoFr/9qEGIIVq1ahcrKSiQmJnLPaHnx4gXvgw7QoounDRs2AHjZeeff/vY3lVtOampqcO7cOfj5+em+QkKIVqytrfHZZ5/Bx8eHbv97hcZht3btWgAv9+ySkpJUDlmtra3h5eWFpKQk3VdICCE6oHHY5ebmAgAGDhyIgwcPonXr1s1WFCGE6JrWD9yp67GYEEKMidZh9+GHHzY6fevWrW9cDCGENBetw+758+cqw0qlEjdv3kRxcTHef/99nRVGCCG6pHXYHTp0SG1cbW0tZs6ciY4dO+qkKEII0TWddARgbm6O+Ph47ootIYQYGp31epKTk9OsXckUFRUhJiYGYrEYDg4OmDp1KsrKyhpt/+mnn8LX1xdCoRAeHh747LPPUFJS0mw1EkIMl9aHsfHx8SrDjDEUFBTgn//8p8o9tLoWExODgoICSCQSKJVKxMXFYfr06fX27AAA+fn5yM/PR2JiIrp06YIHDx5gxowZyM/Pxy+//NJsdRJCDJPWYXf16lWVYXNzczg5OWH16tVNXql9U1KpFCdOnEBGRgaCgoIAABs3buRuh6l7UPar3nvvPRw4cIAb7tixI7799ltMnDgR1dXVsLTUetUJIUbMKL5nl5qaCgcHBy7oACA8PBzm5uZIS0vDiBEjNHqduicQNRZ0CoUCCoWCG5bL5QBeXnVWKpVvuAaGr27dTHkd+YQv21Ob9TOK3RuZTAZnZ2eVcZaWlnB0dIRMJtPoNZ4+fYoVK1Zg+vTpjbZLSEjA8uXL1cYnJyfD1tZW86KNFJ8fyGKKTH17VlRUaNxWr2G3cOFCrFy5stE2Uqn0rZcjl8vxwQcfoEuXLvj6668bbbto0SKV85JyuRzu7u6IjIw0+efGSiQSRERE0I3jRq6qqgqbNm3C2bNnERoaik8++cRkeySqO/LShF7Dbs6cOU12Burt7Q1XV1e1PvSqq6tRVFQEV1fXRucvLS1FVFQU7OzscOjQoSY/yAKBAAKBQG28lZUVL0KAL+tpqubPn481a9agpqYGAHDs2DEsXrwY8fHxJtnNkzbvVb2GnZOTE5ycnJpsFxISguLiYmRmZiIwMBAAkJKSgtraWgQHBzc4n1wux6BBgyAQCHDkyBHY2NjorHZCDM38+fPr7am4pqaGG2+KgacpnX3P7t///neT58PelL+/P6KiojBt2jSkp6fjwoULmDVrFsaNG8ddiX306BH8/PyQnp4O4GXQRUZGory8HD/++CPkcjlkMhlkMhn3V48QU1FVVYXExMRG2yQmJqKqqqqFKjI8Ogu7Z8+e4ccff9TVy6nZtWsX/Pz8EBYWhujoaPTr1w9btmzhpiuVSmRnZ3MnLK9cuYK0tDTcuHEDPj4+cHNz4/49fPiw2eokRB82bNjQ5AOwGWNcJ7x8ZBRXY4GXj3Ns6AvEAODl5aWysQcMGMDrp58Tftm/f7/G7ebOndvM1Rgmekg2ISYgKytLp+1MkdHs2RFCGlZZWcn9HBUVBR8fH9y5cwedO3fG3bt3ceLECbV2fKNx2I0cObLR6cXFxW9bCyHkDb16yqYu2ICXX4ZvqB3faBx29vb2TU6fPHnyWxdECNGera1to70AvdqOrzQOu23btjVnHYSQt9CzZ0+cO3dOo3Z8RRcoCDEBnTt31mk7U6Txnp2m3TfRA3cIaXlPnjzRaTtTpHHYbd++HZ6enujRowevT3ISYojs7Ox02s4UaRx2M2fOxJ49e5Cbm4u4uDhMnDgRjo6OzVkbIURDkyZNwk8//QSRSARHR0fk5eVx0zw9PfHs2TOUlZVh0qRJeqxSvzQ+Z7dp0yYUFBRg/vz5+Mc//gF3d3eMGTMGJ0+epD09QvQsLCwMYrEYZWVlqKysxOzZszF9+nTMnj0bL168QFlZGcRiMcLCwvRdqt6YsTdMqgcPHmD79u3YsWMHqqurkZWVBZFIpOv69E4ul8Pe3p7r5dhUKZVKHDt2DNHR0dTFk5E6ePAgRo0aBTMzM5UdkLrhAwcONPl9WWOjzefzja/Gmpubc79E6kWEEP0bOXIkDhw4AHd3d5XxHh4eJhl02tIq7BQKBfbs2YOIiAh07twZN27cwF//+lfk5eWZ5F4dIcZm5MiRuHfvHiQSCeLj4yGRSJCTk8P7oAO0uEDx8ccfY+/evXB3d8eHH36IPXv2oG3bts1ZGyHkDVhYWCA0NBTl5eUIDQ2FhYWFvksyCBqHXVJSEjw8PODt7Y2zZ8/i7Nmz9bY7ePCgzoojhBBd0TjsJk+eDDMzs+ashRBCmo1WXyomhBBjRffGEkJ4gcKOEMILFHaEEF4wmrArKipCTEwMxGIxHBwcMHXq1CY7K/zoo4/QsWNHCIVCODk5YdiwYbh9+3YLVUwIMSRGE3YxMTHIysqCRCLB0aNHce7cuSafUxsYGIht27ZBKpVy9/BGRkbSHR+E8JBRPHBHKpXixIkTyMjIQFBQEABg48aNiI6ORmJiIveg7Ne9GoZeXl745ptv0L17d9y/fx8dO3ZskdoJIYbBKMIuNTUVDg4OXNABQHh4OMzNzZGWloYRI0Y0+Rrl5eXYtm0bOnTooHbv4KsUCgUUCgU3LJfLAby8UV6pVL7FWhi2unUz5XXkE75sT23WzyjCTiaTwdnZWWWcpaUlHB0dIZPJGp33+++/x/z581FeXg5fX19IJBJYW1s32D4hIQHLly9XG5+cnMyLh5VIJBJ9l0B0yNS3Z0VFhcZt9Rp2CxcuxMqVKxttI5VK32oZMTExiIiIQEFBARITEzFmzBhcuHABNjY29bZftGgR4uPjuWG5XA53d3dERkaafBdPEokEERER1MWTCeDL9qw78tKEXsNuzpw5iI2NbbSNt7c3XF1dUVhYqDK+uroaRUVFcHV1bXR+e3t72Nvbo1OnTujTpw9at26NQ4cOYfz48fW2FwgEEAgEauOtrKxM+k1Thy/ryRemvj21WTe9hp2TkxOcnJyabBcSEoLi4mJkZmYiMDAQAJCSkoLa2loEBwdrvDzGGBhjKufkCCH8YBRfPfH390dUVBSmTZuG9PR0XLhwAbNmzcK4ceO4K7GPHj2Cn58f0tPTAQD37t1DQkICMjMzkZeXh4sXL2L06NEQCoWIjo7W5+oQQvTAKMIOAHbt2gU/Pz+EhYUhOjoa/fr1w5YtW7jpSqUS2dnZ3AlLGxsbnD9/HtHR0fDx8cHYsWNhZ2eHixcvql3sIISYPqO4GgsAjo6O2L17d4PTvby8VPrdb9euHY4dO9YSpRFCjIDR7NkRQsjboLAjhPAChR0hhBco7AghvEBhRwjhBQo7QggvUNgRQniBwo4QwgsUdoQQXqCwI4TwAoUdIYQXKOwIIbxAYUcI4QUKO0IIL1DYEUJ4gcKOEMILFHaEEF6gsCOE8AKFHSGEFyjsCCG8YDRhV1RUhJiYGIjFYjg4OGDq1KkoKyvTaF7GGAYPHgwzMzMcPny4eQslhBgkowm7mJgYZGVlQSKR4OjRozh37hymT5+u0bzr1q2DmZlZM1dICDFkRvEoRalUihMnTiAjIwNBQUEAgI0bNyI6OhqJiYncg7Lrc+3aNaxevRqXL1+Gm5tbS5VMCDEwRhF2qampcHBw4IIOAMLDw2Fubo60tDSMGDGi3vkqKiowYcIEbNq0Ca6urhotS6FQQKFQcMNyuRzAy4dwK5XKt1gLw1a3bqa8jnzCl+2pzfoZRdjJZDI4OzurjLO0tISjoyNkMlmD833xxRfo27cvhg0bpvGyEhISsHz5crXxycnJsLW11bxoIyWRSPRdAtEhU9+eFRUVGrfVa9gtXLgQK1eubLSNVCp9o9c+cuQIUlJScPXqVa3mW7RoEeLj47lhuVwOd3d3REZGQiwWv1EtxkCpVEIikSAiIgJWVlb6Loe8Jb5sz7ojL03oNezmzJmD2NjYRtt4e3vD1dUVhYWFKuOrq6tRVFTU4OFpSkoKcnJy4ODgoDJ+1KhR6N+/P86cOVPvfAKBAAKBQG28lZWVSb9p6vBlPfnC1LenNuum17BzcnKCk5NTk+1CQkJQXFyMzMxMBAYGAngZZrW1tQgODq53noULF+LPf/6zyriuXbti7dq1GDp06NsXTwgxKkZxzs7f3x9RUVGYNm0akpKSoFQqMWvWLIwbN467Evvo0SOEhYVhx44d6N27N1xdXevd6/Pw8ECHDh1aehUIIXpmNN+z27VrF/z8/BAWFobo6Gj069cPW7Zs4aYrlUpkZ2drdcKSEMIfRrFnBwCOjo7YvXt3g9O9vLzAGGv0NZqaTggxXUazZ0cIIW+Dwo4QwgsUdoQQXqCwI8TE1NTU4OzZszh37hzOnj2LmpoafZdkECjsCDEhBw8ehI+PDyIiIrBmzRpERETAx8cHBw8e1HdpekdhR4iJOHjwIP70pz+ha9euOH/+PPbs2YPz58+ja9eu+NOf/sT7wKOwI8QE1NTUYM6cORgyZAgOHz6M4OBgCIVCBAcH4/DhwxgyZAjmzp3L60NaCjtCTMD58+dx//59LF68GIwxlXN2jDEsWrQIubm5OH/+vL5L1RsKO0JMQEFBAQAgJycH3t7eKufsvL29ce/ePZV2fGQ0d1AQQhpW1wv3xIkT1R5B8PDhQ0ycOFGlHR9R2BFiAvr27QszMzMwxuDk5IT//u//hkAggEKhwNKlS1FYWAgzMzP07dtX36XqDR3GEmICzpw5w937HRQUBIVCgczMTCgUCu5xBoyxBvtx5APasyPEBOzcuRMAEBYWhpMnT+LYsWPcNAsLC7z//vtISUnBzp07ERkZqa8y9YrCjhATUPcM5VOnTqlNq6mpQUpKiko7PqLDWEJMgKbn4uicHSHEqPn7++u0nSmisCPEBKxdu1an7UwRhR0hJuD333/nfn79e3avDr/ajm8o7AgxAVVVVQBePjy+pKQEM2bMQEBAAGbMmIGSkhJYWlqqtOMjuhpLiAlo3bo15HI5qqur4ezsjMrKSgDAtWvXsH37dlRXV3Pt+Mpo9uyKiooQExMDsVgMBwcHTJ06tcnL6AMGDICZmZnKvxkzZrRQxYS0nD59+nA/1wVdfcOvtuMbowm7mJgYZGVlQSKR4OjRozh37hymT5/e5HzTpk1DQUEB92/VqlUtUC0hLSsuLk6n7UyRURzGSqVSnDhxAhkZGdytLxs3bkR0dDQSExO5B2XXx9bWtt6HZRNiSmpra3XazhQZRdilpqbCwcGBCzoACA8Ph7m5OdLS0jBixIgG5921axd++uknuLq6YujQoViyZAlsbW0bbK9QKKBQKLhhuVwO4OVDuJVKpQ7WxjDVrZspr6Mpq7tdTJN24eHhzVxNy9Hm/WoUYSeTyeDs7KwyztLSEo6OjpDJZA3ON2HCBHh6eqJdu3b4/fffsWDBAmRnZzfaPXVCQgKWL1+uNj45ObnRkDQVEolE3yWQN3D16lXuZysrK5UQeHX46tWrKvfNGruKigqN2+o17BYuXIiVK1c22kYqlb7x6796Tq9r165wc3NDWFgYcnJy0LFjx3rnWbRoEeLj47lhuVwOd3d3REZGQiwWv3Ethk6pVEIikSAiIgJWVlb6Lodoaf369bh9+zaEQiEKCwvx22+/cduzX79+cHZ2xosXL+Dm5obo6Gh9l6szdUdemtBr2M2ZMwexsbGNtvH29oarqysKCwtVxldXV6OoqEir83HBwcEAgLt37zYYdgKBAAKBQG28lZUVL0KAL+tpauo+By9evMC4ceOwYMEC9OrVC61atcK4cePw4sULrp0pbV9t1kWvYefk5AQnJ6cm24WEhKC4uBiZmZkIDAwEAKSkpKC2tpYLME1cu3YNAL97ayWmqUOHDtzPx48fxz//+U9u2MLCot52fGMUXz3x9/dHVFQUpk2bhvT0dFy4cAGzZs3CuHHjuCuxjx49gp+fH9LT0wG87It/xYoVyMzMxP3793HkyBFMnjwZf/jDH9CtWzd9rg4hOvf+++9zP79+xfXVJ4q92o5vjCLsgJdXVf38/BAWFobo6Gj069cPW7Zs4aYrlUpkZ2dzJyytra3x66+/IjIyEn5+fpgzZw5GjRqFf/zjH/paBUKazYABA7ijJKFQqDKtbtjZ2RkDBgxo6dIMhlFcjQUAR0dH7N69u8HpXl5eXLfUAODu7o6zZ8+2RGmE6J2FhQWSkpIwatQotWl1HQFs3rxZ5ZCWb4xmz44Q0riRI0fiwIEDal/TcnZ2xoEDBzBy5Eg9VWYYjGbPjhDStJEjR2LYsGE4ffo0jh8/jsGDB2PgwIG83qOrQ2FHiImxsLBAaGgoysvLERoaSkH3/+gwlhDCCxR2hBBeoLAjhPACnbNrQt3XWbS5B88YKZVKVFRUQC6Xm9TtRHzFl+1Z97l89WtnDaGwa0JpaSmAl9/bI4QYptLSUtjb2zfaxoxpEok8Vltbi/z8fNjZ2ak9tcmU1PXu8vDhQ5Pu3YUv+LI9GWMoLS1Fu3btYG7e+Fk52rNrgrm5Od555x19l9FixGKxSX84+IYP27OpPbo6dIGCEMILFHaEEF6gsCMAXnZaumzZsno7LiXGh7anOrpAQQjhBdqzI4TwAoUdIYQXKOwIIbxAYadjsbGxGD58uE5fc8CAAZg9e3ajbby8vLBu3TqdLpcQU0Jh1whNQoYYl6+//hoBAQH6LqNehvZ+M7R63haFHSEmpKqqSt8lGCwKuwbExsbi7NmzWL9+PczMzGBmZoacnBxMnToVHTp0gFAohK+vL9avX1/v/MuXL4eTkxPEYjFmzJih8ZuwvLwckydPhkgkgpubG1avXq3WprCwEEOHDoVQKESHDh2wa9cutTZmZmbYvHkzBg8eDKFQCG9vb/zyyy/c9Pv378PMzAw///wz+vfvD6FQiF69euHOnTvIyMhAUFAQRCIRBg8ejCdPnmj4WwO2bt2Kd999FwKBAG5ubpg1axY3LS8vD8OGDYNIJIJYLMaYMWPw+PFjbnrdXtfOnTvh5eUFe3t7jBs3juuMAXh5r/KqVavg4+MDgUAADw8PfPvtt9z0BQsWoHPnzrC1tYW3tzeWLFkCpVIJANi+fTuWL1+O69evc9t0+/btGq9bc3rT91vdaZNvv/0W7dq1g6+vLwDg4sWLCAgIgI2NDYKCgnD48GGYmZlxz04GgJs3b2Lw4MEQiURwcXHBpEmT8PTp0wbruX//fkv9OpoHI/UqLi5mISEhbNq0aaygoIAVFBSwyspKtnTpUpaRkcHu3bvHfvrpJ2Zra8v27dvHzTdlyhQmEonY2LFj2c2bN9nRo0eZk5MTW7x4sUbLnTlzJvPw8GC//vor+/3339mQIUOYnZ0d+/zzz7k2gwcPZt27d2epqans8uXLrG/fvkwoFLK1a9dybQCwNm3asB9++IFlZ2ezr776illYWLBbt24xxhjLzc1lAJifnx87ceIEu3XrFuvTpw8LDAxkAwYMYL/99hu7cuUK8/HxYTNmzNCo9u+//57Z2NiwdevWsezsbJaens7VVFNTwwICAli/fv3Y5cuX2aVLl1hgYCALDQ3l5l+2bBkTiURs5MiR7MaNG+zcuXPM1dVV5Xc3f/581rp1a7Z9+3Z29+5ddv78efbDDz9w01esWMEuXLjAcnNz2ZEjR5iLiwtbuXIlY4yxiooKNmfOHPbuu+9y27SiokKjdWtub/t+mzRpErt58ya7efMmKykpYY6OjmzixIksKyuLHTt2jHXu3JkBYFevXmWMMfb8+XPm5OTEFi1axKRSKbty5QqLiIhgAwcObLCe6upqffxqdIbCrhGhoaEqIVOfTz75hI0aNYobnjJlCnN0dGTl5eXcuM2bNzORSMRqamoafa3S0lJmbW3Nfv75Z27cs2fPmFAo5OrIzs5mAFh6ejrXRiqVMgBqYfd6SAUHB7OZM2cyxv4Tdn/729+46Xv27GEA2KlTp7hxCQkJzNfXt9G667Rr1459+eWX9U5LTk5mFhYWLC8vjxuXlZWlsi7Lli1jtra2TC6Xc23mzZvHgoODGWOMyeVyJhAIVMKtKd999x0LDAzkhpctW8a6d++u8fwt6U3fby4uLkyhUHDjNm/ezNq0acNevHjBjfvhhx9Uwm7FihUsMjJS5bUfPnzIALDs7GyN6zEm1OuJljZt2oStW7ciLy8PL168QFVVldoJ7+7du8PW1pYbDgkJQVlZGR4+fAhPT88GXzsnJwdVVVUIDg7mxjk6OnKHJgAglUphaWmJwMBAbpyfnx8cHBzUXi8kJERt+NXDGADo1q0b97OLiwsAoGvXrirjCgsLG6y5TmFhIfLz8xEWFlbvdKlUCnd3d5V+Abt06QIHBwdIpVL06tULwMurynZ2dlwbNzc3bvlSqRQKhaLBZQDAvn37sGHDBuTk5KCsrAzV1dVG3euHJu+3rl27wtramhvOzs5Gt27dYGNjw43r3bu3yjzXr1/H6dOnIRKJ1JaZk5ODzp0763ZFDACds9PC3r17MXfuXEydOhXJycm4du0a4uLijPqk8Ku92Nb11/f6uNra2iZf5/Wn0OuinteX39QyUlNTERMTg+joaBw9ehRXr17Fl19+abTbR9P3W6tWrbR+7bKyMgwdOhTXrl1T+fevf/0Lf/jDH3S1CgaFwq4R1tbWqKmp4YYvXLiAvn374uOPP0aPHj3g4+ODnJwctfmuX7+OFy9ecMOXLl2CSCRqsrfjjh07wsrKCmlpady458+f486dO9ywn58fqqurkZmZyY3Lzs5GcXGx2utdunRJbdjf37/RGt6UnZ0dvLy8cOrUqXqn+/v74+HDh3j48CE37tatWyguLkaXLl00WkanTp0gFAobXMbFixfh6emJL7/8EkFBQejUqRMePHig0ub1bWpI3vT99jpfX1/cuHEDCoWCG5eRkaHSpmfPnsjKyoKXlxd8fHxU/tWFpyH/rt4EhV0jvLy8kJaWhvv37+Pp06fo1KkTLl++jJMnT+LOnTtYsmSJ2psIeHn5f+rUqbh16xaOHTuGZcuWYdasWU32pCoSiTB16lTMmzcPKSkpuHnzJmJjY1Xm8/X1RVRUFD766COkpaUhMzMTf/7zn+vd69m/fz+2bt2KO3fuYNmyZUhPT1e5OqprX3/9NVavXo0NGzbgX//6F65cuYKNGzcCAMLDw9G1a1fExMTgypUrSE9Px+TJkxEaGoqgoCCNXt/GxgYLFizA/PnzsWPHDuTk5ODSpUv48ccfAbwMw7y8POzduxc5OTnYsGEDDh06pPIaXl5eyM3NxbVr1/D06VOVQNC3N32/vW7ChAmora3F9OnTIZVKcfLkSSQmJgL4z977J598gqKiIowfPx4ZGRnIycnByZMnERcXxwXc6/Vosodv0PR90tCQZWdnsz59+jChUMgAsNu3b7PY2Fhmb2/PHBwc2MyZM9nChQtVTnhPmTKFDRs2jC1dupS1adOGiUQiNm3aNFZZWanRMktLS9nEiROZra0tc3FxYatWrVI7UVxQUMA++OADJhAImIeHB9uxYwfz9PRUu0CxadMmFhERwQQCAfPy8lK5ild3gaLuhDVjjJ0+fZoBYM+fP+fGbdu2jdnb22v8O0tKSmK+vr7MysqKubm5sU8//ZSb9uDBA/bHP/6RtWrVitnZ2bHRo0czmUzGTa/v4sHatWuZp6cnN1xTU8O++eYb5unpyaysrJiHhwf7y1/+wk2fN28e93sfO3YsW7t2rUr9lZWVbNSoUczBwYEBYNu2bdN43Zrb27zfXnfhwgXWrVs3Zm1tzQIDA9nu3bu516xz584dNmLECObg4MCEQiHz8/Njs2fPZrW1tfXWk5ub28y/geZFXTyZKDMzMxw6dEjnt64R47Rr1y7ExcWhpKREZ+dXjQ1djSXEBO3YsQPe3t5o3749rl+/jgULFmDMmDG8DTqAwq5F5eXlNXoy/tatW/Dw8GjBirRT39cU6hw/fhz9+/dvwWpIY2QyGZYuXQqZTAY3NzeMHj1a5U4TPqLD2BZUXV3d6C03Xl5esLQ03L8/d+/ebXBa+/bteb3XQAwfhR0hhBfoqyeEEF6gsCOE8AKFHSGEFyjsCCG8QGFHDEZsbCzXUaSVlRVcXFwQERGBrVu3anWr0vbt2+vtBaa5NcfzR4juUNgRgxIVFYWCggLcv38fx48fx8CBA/H5559jyJAhqK6u1nd5xJjp8141Ql7V0H2ep06dYgC4TjtXr17N3nvvPWZra8veeecdNnPmTFZaWsoY+8/9va/+W7ZsGWOMsR07drDAwEAmEomYi4sLGz9+PHv8+DG3nKKiIjZhwgTWtm1bZmNjw3x8fNjWrVu56Xl5eWz06NHM3t6etW7dmv3xj3/k7hddtmyZ2nJPnz7dLL8n8mZoz44YvPfffx/du3fHwYMHAQDm5ubYsGEDsrKy8Pe//x0pKSmYP38+AKBv375Yt24dxGIxCgoKUFBQgLlz5wIAlEolVqxYgevXr+Pw4cO4f/8+YmNjueUsWbIEt27dwvHjxyGVSrF582a0bduWm3fQoEGws7PD+fPnceHCBYhEIkRFRaGqqgpz587FmDFjuD3TgoIC9O3bt2V/UaRx+k5bQuo0tGfHGGNjx45l/v7+9U7bv38/a9OmDTesaU8tGRkZDAC3Vzh06FAWFxdXb9udO3cyX19frkcQxhhTKBRMKBSykydPNlk/0T/asyNGgTHG9cX266+/IiwsDO3bt4ednR0mTZqEZ8+eoaKiotHXyMzMxNChQ+Hh4QE7OzuEhoYCeHnPMgDMnDkTe/fuRUBAAObPn4+LFy9y816/fh13796FnZ0dRCIRRCIRHB0dUVlZqVGHmkT/KOyIUZBKpejQoQPu37+PIUOGoFu3bjhw4AAyMzOxadMmAI0/M7W8vByDBg2CWCzGrl27kJGRwXXsWTff4MGD8eDBA3zxxRfc8zTqDoHLysoQGBio1o35nTt3MGHChGZee6ILhnvXOSH/LyUlBTdu3MAXX3yBzMxM1NbWYvXq1VwPzj///LNK+/q6E799+zaePXuG//mf/+G6x798+bLaspycnDBlyhRMmTIF/fv3x7x585CYmIiePXti3759cHZ2bvABPqbWjbmpoT07YlAUCgVkMhkePXqEK1eu4C9/+QuGDRuGIUOGYPLkyfDx8YFSqcTGjRtx79497Ny5E0lJSSqv4eXlhbKyMpw6dQpPnz5FRUUFPDw8YG1tzc135MgRrFixQmW+pUuX4n//939x9+5dZGVl4ejRo9wzO2JiYtC2bVsMGzYM58+fR25uLs6cOYPPPvsM//73v7nl/v7778jOzsbTp0+5h3MTA6Hvk4aE1JkyZQr3tQ1LS0vm5OTEwsPD2datW1WeubtmzRrm5ubGhEIhGzRoENuxY4dad/IzZsxgbdq0Ufnqye7du5mXlxcTCAQsJCSEHTlyRO1Zqv7+/kwoFDJHR0c2bNgwdu/ePe41CwoK2OTJk1nbtm2ZQCBg3t7ebNq0aaykpIQxxlhhYSGLiIhgIpGIvnpigKiLJ0IIL9BhLCGEFyjsCCG8QGFHCOEFCjtCCC9Q2BFCeIHCjhDCCxR2hBBeoLAjhPAChR0hhBco7AghvEBhRwjhBQo7Qggv/B+0RJHVQg3QlwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T18:14:02.082424Z", + "iopub.status.busy": "2024-02-29T18:14:02.082143Z", + "iopub.status.idle": "2024-02-29T18:14:02.363265Z", + "shell.execute_reply": "2024-02-29T18:14:02.362378Z" + }, + "papermill": { + "duration": 0.30403, + "end_time": "2024-02-29T18:14:02.365346", + "exception": false, + "start_time": "2024-02-29T18:14:02.061316", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAErCAYAAAC4gcr6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUPklEQVR4nO3deVxU9foH8M8Mw7DIJoozoCCoKGAoKjJiJpVTqFjS9XdF8yoiSYt4NTRzhSwLQ+3illztupWI2i0r81KESyWEiGIqaC4QuAyIyKoyMHN+f0wcOc4AMzjDMPi8X695Kec855zvDPDwPee78RiGYUAIIaRZfGMXgBBCOjpKlIQQ0gpKlIQQ0gpKlIQQ0gpKlIQQ0gpKlIQQ0gpKlIQQ0gpKlIQQ0gpKlIQQ0gpKlIQQ0gqjJ8rNmzfD3d0dlpaWkEgkOHnyZIvxBw4cgJeXFywtLeHr64vDhw9z9jMMg9jYWDg7O8PKygpSqRSXL1/mxHz44YcYOXIkrK2t4eDg0OL17ty5g169eoHH46GioqItb5EQYuKMmij37duHmJgYxMXF4fTp0xg8eDCCg4NRWlqqMT4jIwNTp05FZGQkzpw5g9DQUISGhuL8+fNsTEJCAjZs2ICkpCRkZWWhS5cuCA4OxoMHD9gYuVyOv//973jzzTdbLWNkZCQGDRr0+G+WEGK6GCMKCAhg5syZw36tUCgYFxcXJj4+XmP85MmTmZCQEM42iUTCvP766wzDMIxSqWTEYjGzZs0adn9FRQVjYWHB7N27V+18O3bsYOzt7Zst36effsoEBQUx6enpDADm7t27Orw7QkhnITBWgpbL5cjJycGSJUvYbXw+H1KpFJmZmRqPyczMRExMDGdbcHAwDh48CAAoKCiATCaDVCpl99vb20MikSAzMxNTpkzRunx5eXl4//33kZWVhWvXrml1TF1dHerq6tivlUolysvL0a1bN/B4PK2vTQgxPIZhUF1dDRcXF/D5Ld9cGy1RlpWVQaFQQCQScbaLRCJcvHhR4zEymUxjvEwmY/c3bmsuRht1dXWYOnUq1qxZAzc3N60TZXx8PFauXKn1dQghxldcXIxevXq1GGO0RNmRLVmyBN7e3vjHP/6h83FNa7yVlZVwc3NDcXEx7Ozs9F1MQshjqKqqgqurK2xtbVuNNVqi7N69O8zMzFBSUsLZXlJSArFYrPEYsVjcYnzjvyUlJXB2dubE+Pn5aV22I0eO4Ny5c/jyyy8BqKrojWVetmxZs7VGCwsLWFhYqG23s7OjRElIB6XNYzGjtXoLhUIMGzYM6enp7DalUon09HQEBgZqPCYwMJATDwBpaWlsvIeHB8RiMSemqqoKWVlZzZ5Tk//+9784e/YscnNzkZubi88++wwA8Msvv2DOnDlan4cQ0jkY9dY7JiYG4eHh8Pf3R0BAABITE1FbW4uIiAgAwIwZM9CzZ0/Ex8cDAObNm4egoCCsW7cOISEhSElJwalTp7B161YAqr8M8+fPx6pVq+Dp6QkPDw+sWLECLi4uCA0NZa9bVFSE8vJyFBUVQaFQIDc3FwDQr18/2NjYoG/fvpxylpWVAQC8vb1b7XdJCOmEjN3svnHjRsbNzY0RCoVMQEAA89tvv7H7goKCmPDwcE78/v37mf79+zNCoZAZOHAg8/3333P2K5VKZsWKFYxIJGIsLCyYMWPGMJcuXeLEhIeHMwDUXkePHtVYxqNHj7ape1BlZSUDgKmsrNTpOEKI4eny+8ljGFpczFCqqqpgb2+PyspKnZ5RMgyDhoYGKBQKA5aOkM7NzMwMAoGg2WeQuvx+Uqt3ByOXy3Hr1i3cu3fP2EUhxORZW1vD2dkZQqHwsc5DibIDUSqVKCgogJmZGVxcXCAUCqmjOiFtwDAM5HI5bt++jYKCAnh6erbaqbwllCg7ELlcDqVSCVdXV1hbWxv0WuW1clTdr4ebozX4fErGpPOxsrKCubk5/vzzT8jlclhaWrb5XEafPYioe5y/fNq6fvceqh7Uo6y2rvVgQkyUvn6XKFE+4RRKassjpDWUKAkhpBWUKInRzJw5kzMQQB+effZZzJ8/v8UYd3d3JCYm6vW6xDi0+X7rAyXKJ5y+mnHa6weWtJ/33ntPpzkSOjNKlIQQo6qvrzd2EVpFibKDYxgG9+QNen89qFfgQb0C9+WKZmO0HbQ1c+ZMHD9+HOvXrwePxwOPx8PVq1cRGRkJDw8PWFlZYcCAAVi/fr3G41euXAknJyfY2dnhjTfegFwu1+q6tbW1mDFjBmxsbODs7Ix169apxZSWluKll16ClZUVPDw8sGfPHrUYHo+HLVu2YNy4cbCyskKfPn3YmaMAoLCwEDweD/v378czzzwDKysrDB8+HH/88Qeys7Ph7+8PGxsbjBs3Drdv39aq7ACwfft2DBw4EBYWFnB2dkZ0dDS7r6ioCBMnToSNjQ3s7OwwefJkzsxZjbW9zz//HO7u7rC3t8eUKVNQXV3NxiiVSiQkJKBfv36wsLCAm5sbPvzwQ3b/u+++i/79+8Pa2hp9+vTBihUr2KS1c+dOrFy5EmfPnmW/pzt37mz1PV28eBGjRo2CpaUlfHx88NNPP4HH47GTazd+lvv27UNQUBAsLS2xZ88e3LlzB1OnTkXPnj1hbW0NX19f7N27l3Nubb7fhkL9KDu4+/UK+MT+YJRr570fDGth6z8i69evxx9//IGnnnoK77//PgCga9eu6NWrFw4cOIBu3bohIyMDUVFRcHZ2xuTJk9lj09PTYWlpiWPHjqGwsBARERHo1q0b5xe6Oe+88w6OHz+Ob775Bj169MDSpUtx+vRpzu3izJkzcfPmTRw9ehTm5ub45z//qXFNphUrVmD16tVYv349Pv/8c0yZMgXnzp2Dt7c3GxMXF4fExES4ublh1qxZePXVV2Fra4v169fD2toakydPRmxsLLZs2dJq2bds2YKYmBisXr0a48aNQ2VlJU6cOAFAleAak+Tx48fR0NCAOXPmICwsDMeOHWPPcfXqVRw8eBCHDh3C3bt3MXnyZKxevZr97JYsWYJt27bhX//6F0aNGoVbt25xJsW2tbXFzp074eLignPnzmH27NmwtbXFokWLEBYWhvPnzyM1NRU//fQTANVqAS1RKBQIDQ2Fm5sbsrKyUF1djQULFmiMXbx4MdatW4chQ4bA0tISDx48wLBhw/Duu+/Czs4O33//PaZPn46+ffsiICAAgHbfb4Mx6KjzJ5yuk2Lcv3+fycvLY+7fv89uq62rZ3q/e8gor9q6eq3fa1BQEDNv3rwWY+bMmcNMmjSJ/To8PJxxdHRkamtr2W1btmxhbGxsGIVC0eK5qqurGaFQyOzfv5/ddufOHcbKyootx6VLlxgAzMmTJ9mY/Px8BgDzr3/9i90GgHnjjTc455dIJMybb77JMAzDFBQUMACYzz77jN2/d+9eBgCTnp7ObouPj2cGDBjQYrkbubi4MMuWLdO478cff2TMzMyYoqIidtuFCxc47yUuLo6xtrZmqqqq2Jh33nmHkUgkDMMwTFVVFWNhYcFs27ZNq/IwDMOsWbOGGTZsGPt1XFwcM3jwYK2P/9///scIBALm1q1b7La0tDQGAPP1118zDPPws0xMTGz1fCEhIcyCBQsYhtHu+62Jpt+pRrr8flKNsoOzMjdD3vvBej/v+RuVAAAnGwuI7DWPWLAyN3usa2zevBnbt29HUVER7t+/D7lcrvbXf/DgwZxRSIGBgaipqUFxcTF69+7d7LmvXr0KuVwOiUTCbnN0dMSAAQPYr/Pz8yEQCDBs2DB2m5eXl8ap8h6drzQwMJCdfq9R09U4G5cb8fX15WxrbgXRpkpLS3Hz5k2MGTNG4/78/Hy4urrC1dWV3ebj4wMHBwfk5+dj+PDhAFSt901n53Z2dmavn5+fj7q6umavAahWQd2wYQOuXr2KmpoaNDQ0PNYE05cuXYKrqytn4u3G2uCj/P39OV8rFAp89NFH2L9/P27cuAG5XI66ujr2Z0Ob77chUaLs4Hg8nla3v7qy/CsJWgnNDHL+lJQULFy4EOvWrUNgYCBsbW2xZs0aZGVl6f1a7cXc3Jz9f+MY/Ee3KZXKVs9jZWWl9/I8ev3WrpGZmYlp06Zh5cqVCA4Ohr29PVJSUtrtuV+XLl04X69Zswbr169HYmIifH190aVLF8yfP1/r59WGRo05RC+EQiFnWrgTJ05g5MiReOuttzBkyBD069cPV69eVTvu7NmzuH//Pvv1b7/9BhsbG05tSpO+ffvC3Nyck3jv3r2LP/74g/3ay8sLDQ0NyMnJYbddunQJFRUVauf77bff1L5u+nxSn2xtbeHu7q42W38jb29vFBcXo7i4mN2Wl5eHiooK+Pj4aHUNT09PWFlZNXuNjIwM9O7dG8uWLYO/vz88PT3x559/cmIe/Z62ZsCAASguLuY0OmVnZ2t17IkTJzBx4kT84x//wODBg9GnTx/O91Kb77chUY3yCaevEYzu7u7IyspCYWEhbGxs4Onpid27d+OHH36Ah4cHPv/8c2RnZ8PDw4NznFwuR2RkJJYvX47CwkLExcUhOjq61TG6NjY2iIyMxDvvvINu3bqhR48eWLZsGee4AQMGYOzYsXj99dexZcsWCAQCzJ8/X2Nt68CBA/D398eoUaOwZ88enDx5Ev/5z3/08+Fo8N577+GNN95Ajx49MG7cOFRXV+PEiROYO3cupFIpfH19MW3aNCQmJqKhoQFvvfUWgoKC1G5Zm2NpaYl3330XixYtglAoxNNPP43bt2/jwoULiIyMhKenJ4qKipCSkoLhw4fj+++/x9dff805h7u7OwoKCpCbm4tevXrB1tZW45pQjV544QX07dsX4eHhSEhIQHV1NZYvXw6g9XVpPD098eWXXyIjIwNdu3bFJ598gpKSEvYPgzbfb0OiGuUTRqFUcrr9lNXUoV7R+u1iaxYuXAgzMzP4+PjAyckJwcHB+Nvf/oawsDBIJBLcuXMHb731ltpxY8aMgaenJ0aPHo2wsDC8/PLLeO+997S65po1a/DMM8/gpZdeglQqxahRozjPIwFgx44dcHFxQVBQEP72t78hKioKPXr0UDvXypUrkZKSgkGDBmH37t3Yu3ev1rW3tggPD0diYiI+/fRTDBw4EBMmTMDly5cBqJLKN998g65du2L06NGQSqXo06cP9u3bp9M1VqxYgQULFiA2Nhbe3t4ICwtjn2G+/PLLePvttxEdHQ0/Pz9kZGRgxYoVnOMnTZqEsWPH4rnnnoOTk5Nad51HmZmZ4eDBg6ipqcHw4cPx2muvYdmyZQDQ6sw9y5cvx9ChQxEcHIxnn30WYrFYbdSWNt9vQ6EZzg1I1xnOHzx4gIKCAnh4eDzWlFDNqVcokX+rChYCM9Q1PLylcra3gpNt8zWFzo7H4+Hrr7/W+3BKorqlHjVqFK5cuaK2FlV7aOl3imY4JxpVP2gAAE6SJESfvv76a/bRy5UrVzBv3jw8/fTTRkmS+kS33qRDKioqgo2NTbOvoqIiYxexRS2V/ZdffjF28dpkz549zb6ngQMHAgCqq6sxZ84ceHl5YebMmRg+fDi++eYbI5f88Rm9Rrl582asWbMGMpkMgwcPxsaNG5vtewWoHrqvWLEChYWF8PT0xMcff4zx48ez+xmGQVxcHLZt24aKigo8/fTT2LJlCzw9PdmYDz/8EN9//z1yc3MhFArVWkHPnj2L1atX49dff0VZWRnc3d3xxhtvYN68eXp//0QzFxcXtX6Mj+7XF0M8fWqp7D179tT79drDyy+/zOnH2FRjV6UZM2ZgxowZ7VmsdmHURLlv3z7ExMQgKSkJEokEiYmJCA4OxqVLlzQ+cM/IyMDUqVMRHx+PCRMmIDk5GaGhoTh9+jSeeuopAEBCQgI2bNiAXbt2set6BwcHIy8vj31GIZfL8fe//x2BgYEaWzZzcnLQo0cPfPHFF3B1dWWH35mZmXHG4xLDEQgE6Nevn7GL0WamXPbm2Nracjq4P1FaHbtjQAEBAcycOXPYrxUKBePi4sLEx8drjJ88eTITEhLC2SaRSJjXX3+dYRjVmt5isZhZs2YNu7+iooKxsLBg9u7dq3a+HTt2MPb29lqV9a233mKee+45rWIb6WMIoz7dqaljzhbfVXuVVhnmeoQYm76GMBrtGaVcLkdOTg6kUim7jc/nQyqVIjMzU+MxmZmZnHgACA4OZuMLCgogk8k4Mfb29pBIJM2eU1uVlZVwdHR8rHMQQkyT0W69y8rKoFAo2DGzjUQiEWeGk6ZkMpnGeJlMxu5v3NZcTFtkZGRg3759+P7771uMq6urQ13dw8W6qqqq2nzN9kWrMBLSEmr1bsX58+cxceJExMXF4cUXX2wxNj4+Hvb29uyrtWF4hBDTYLRE2b17d5iZmXHGhQJASUkJZ/aRpsRicYvxjf/qcs6W5OXlYcyYMYiKimKHYrVkyZIlqKysZF9Nx+oSQkyX0RKlUCjEsGHDOIP2lUol0tPT1aa8ahQYGKg2yD8tLY2N9/DwgFgs5sRUVVUhKyur2XM258KFC3juuecQHh6u1SSyAGBhYQE7OzvOyzQYd3DWo4t9NZ0Rm5COwKjdg2JiYhAeHg5/f38EBAQgMTERtbW1iIiIAKDqk9WzZ0/Ex8cDAObNm4egoCCsW7cOISEhSElJwalTp7B161YAql+w+fPnY9WqVfD09GS7B7m4uHCGpxUVFaG8vBxFRUVQKBRsn7d+/frBxsYG58+fx/PPP4/g4GDExMSwzzfNzMzg5ORk8M9FyTAoq6mDjYWAnQ7tSXLr1i107drV2MUghGXURBkWFobbt28jNjYWMpkMfn5+SE1NZRtjioqKOLODjBw5EsnJyVi+fDmWLl0KT09PHDx4kO1DCQCLFi1CbW0toqKiUFFRgVGjRiE1NZUzzjM2Nha7du1ivx4yZAgA4OjRo3j22Wfx5Zdf4vbt2/jiiy/wxRdfsHG9e/dGYWGhoT4O1t1aOe48UNXyBvVy0Pl4hZJBg0IJCxNNsm15TKJvDMNAoVBAIDD6mAzSERig6xL5S1v7UV6+eYft48golQxTV6PTK6/wFvP7tRvMveoKzvY75eXM79duqL1K79zRfC6lUuv3WlVVxbz66quMtbU1IxaLmU8++USr5SEYhmF69+6ttjTDo0sH/Pe//2WeffZZxsrKihk0aBCTkZHBOccvv/zCjBo1irG0tGR69erFzJ07l6mpqWH37969mxk2bBhjY2PDiEQiZurUqUxJSQm7/+jRowwA5vDhw8zQoUMZc3Nz5ujRo1q/f9Ix0VIQT4r6e8BHug3Xa266Wce/XlpbehMQdmk9DqrHKCdOnMC3334LkUiE2NhYvS78tGzZMqxduxaenp5YtmwZpk6diitXrkAgEODq1asYO3YsVq1ahe3bt+P27duIjo5GdHQ0duzYAUC1JOoHH3yAAQMGoLS0FDExMZg5cyYOHz7Muc7ixYuxdu1a9OnTh27/CYsSJXls1dXV2LVrF5KTk9k1WhrngdSXhQsXIiQkBIBq7siBAwfiypUr8PLyQnx8PKZNm4b58+cDUE0Cu2HDBgQFBWHLli2wtLTErFmz2HP16dMHGzZswPDhw1FTUwMbGxt23/vvv48XXnhBb+UmnQMlyo7O3FpVs2vFg3oFbtfI4WQjxOXSGgBADzsLiGwfPpstr5XjRsV9tWPF9hZwstEw/6W5tfo2Da5du4b6+nrOZCb29vZ6Xfip6cJezs7OAFSLdHl5eeHs2bP4/fffOWt2MwwDpVKJgoICeHt7IycnB++99x7Onj2Lu3fvsmvLFBUVcSbo1XYGcfJkoUTZAXHGyfB4Wt3+FtypQr1CgOpKBkxjgjO3BIRNEmC9ORhzDaNwHo3rgDQt7NWY7GpqavD666/jn//8p9pxbm5uqK2tRXBwMIKDg7Fnzx44OTmhqKgIwcHBaotXPbroFSEAJcpOo3E5hwZ9LYKjgz59+sDc3BzZ2dlwc3MDoBob/8cff2D06NEGv/7QoUORl5fX7Iw9586dw507d7B69Wp2tNSpU6cMXi7SedAQRoLHHetta2uL8PBwvPPOOzh69Ci7gBWfz291USl9ePfdd5GRkYHo6Gjk5ubi8uXL+Oabb9gp8dzc3CAUCrFx40Zcu3YN3377LT744AODl4t0HpQoiV588sknCAwMxIQJEyCVSvH000/D29vbIGv/PGrQoEE4fvw4/vjjDzzzzDMYMmQIYmNj2cYkJycn7Ny5EwcOHICPjw9Wr16NtWvXGrxcpPOgxcUMqK2LiwkdRKhVqP6Gadvh/PfrFWrbRHaWENlxG3Ou372nFudsbwknW/0mtNraWvTs2RPr1q1DZGSkXs9NiLZocTHSoZw5cwYXL15EQEAAKisr8f777wMAJk6caOSSEfL4KFESvVm7di0uXbrETnjyyy+/ID8/H+PGjWv2mJqamnYsISFtQ4nyCWLIZpUhQ4YgJydHbfv9+/dbXGiLEFNAiZIYlJWVVadcaIs8WajVuwNq//Y1WgqCdE76+l2iRNmBNI4+UcjrWonUTn2DUstI6vhAOqd791S9PJqO7GoLuvXuQMzMzODg4IDCm6WwsnMETyDEgwcPtDqWaZCrbbtTJUd364d/C+V1co1x9XIetLwMISaBYRjcu3cPpaWlcHBwgJnZ483NSomygxGLxfj69HW43yuBuRkPwgflWh1Xeld9sgsAEN63Yv9/T96A8tp6tZg6KwEqLR/vLy4hHZGDg4NeJoKmRNnB8Hg85NwGNmSWoqslHy8NcsGzA5wwtHfLcyO+9tUxjdvTFzzL/v/HCzJ8fFR9KeCo0X0Q5u32OMUmpMMxNzd/7JpkI0qUHdSDBga3ahTYmlGMrRnFKFwd0mL8jWqFxu1NRyPUQ6Axro4xa5ehhoSYKmrM6YDaYR4JQogOKFF2cs+uOYqiO6qWP0rAhLQNJcpOrvDOPcR9e97YxSDEpBk9UW7evBnu7u6wtLSERCLByZMnW4w/cOAAvLy8YGlpCV9fX7XFoRiGQWxsLJydnWFlZQWpVIrLly9zYj788EOMHDkS1tbWcHBw0HidoqIihISEwNraGj169MA777yDhoaGx3qvxlKvoH6ShDwOoybKffv2ISYmBnFxcTh9+jQGDx6M4OBglJaWaozPyMjA1KlTERkZiTNnziA0NBShoaE4f/5hjSkhIQEbNmxAUlISsrKy0KVLFwQHB3P6I8rlcvz973/Hm2++qfE6CoUCISEhkMvlyMjIwK5du7Bz507Exsbq9wNoFt0jE9KRGDVRfvLJJ5g9ezYiIiLg4+ODpKQkWFtbY/v27Rrj169fj7Fjx+Kdd96Bt7c3PvjgAwwdOhSbNm0CoKpNJiYmYvny5Zg4cSIGDRqE3bt34+bNmzh48CB7npUrV+Ltt9+Gr6+vxuv8+OOPyMvLwxdffAE/Pz+MGzcOH3zwATZv3qy2xgohpPMzWqKUy+XIycmBVCp9WBg+H1KpFJmZmRqPyczM5MQDQHBwMBtfUFAAmUzGibG3t4dEImn2nM1dx9fXFyKRiHOdqqoqXLhwodnj6urqUFVVxXkRQkyf0RJlWVkZFAoFJxkBgEgkgkwm03iMTCZrMb7xX13Oqct1ml5Dk/j4eNjb27OvxoWsCCGmzeiNOZ3JkiVLUFlZyb6Ki4uNXSRCiB4YLVF2794dZmZmKCkp4WwvKSlpdmymWCxuMb7xX13Oqct1ml5DEwsLC9jZ2XFepiC3uALltfTslZDmPFairKmpafMzucblAtLT09ltSqUS6enpCAwM1HhMYGAgJx4A0tLS2HgPDw+IxWJOTFVVFbKyspo9Z3PXOXfuHKf1PS0tDXZ2dvDx8dH6PB1Ncx3OD5+TQfLRT+1bGEJMiM5jvQsKChAdHY1jx45xutwwDAMejweFQvOYY01iYmIQHh4Of39/BAQEIDExEbW1tYiIiAAAzJgxAz179kR8fDwAYN68eQgKCsK6desQEhKClJQUnDp1Clu3bgWgmlBi/vz5WLVqFTw9PeHh4YEVK1bAxcUFoaGh7HWLiopQXl6OoqIiKBQKdqmCfv36wcbGBi+++CJ8fHwwffp0JCQkQCaTYfny5ZgzZw4sLCx0/chMAvW1JKR5OifKf/zjH2AYBtu3b4dIJHqsBe7DwsJw+/ZtxMbGQiaTwc/PD6mpqWzDSVFREfj8h5XekSNHIjk5GcuXL8fSpUvh6emJgwcP4qmnnmJjFi1ahNraWkRFRaGiogKjRo1CamoqZ9KH2NhY7Nq1i/16yJAhAICjR4/i2WefhZmZGQ4dOoQ333wTgYGB6NKlC8LDw9mVBQkhTxad1/W2sbFBTk4OBgwYYKgydRq6ruvdaPbuU0jL4z4jbW32IPfF3ze7b1S/7vjiNQm+PnMdb+8722xca9cgpDPR5fdT52eUw4cPp9ZcQsgTRedb788++wxvvPEGbty4gaeeekptLYpBgwbprXCEENIR6Jwob9++jatXr7INLoCqEaUtjTmEEGIKdE6Us2bNwpAhQ7B3797HbswhhBBToHOi/PPPP/Htt9/SovadXPz/8nG7qg7rJg+mP4bkiadzY87zzz+Ps2ebbzklj0/faakxz/F0OPO/j1/DV2du4FJJtZ5LQ4jp0blG+dJLL+Htt9/GuXPn4Ovrq9aY8/LLL+utcMQ4mvYYa6CO6ITonijfeOMNANDY+ZoaczoHhfJhcuTTbTchuidKpVJpiHKQDkTRpEbJp/mlCNHtGWV9fT0EAgFn6QXS+TStUeryXJOQzkqnRGlubg43Nze6ve7kuLfeRiwIIR2EzjdWy5Ytw9KlS1FeXm6I8hAYf/1tTo3S2IUhpAPQ+Rnlpk2bcOXKFbi4uKB3797o0qULZ//p06f1VjhiHNxEacSCENJB6Jwom87rSDqnpolSt7mlCOmcdE6UcXFxhigHaQet1Q5vVNzH3Vo5utkI2W06zsJHSKekc6JslJOTg/z8fADAwIED2clviel6evURAEDybAm7TUl5khDdE2VpaSmmTJmCY8eOwcHBAQBQUVGB5557DikpKXByctJ3GUk7y7v5cO0jBpQpCdG51Xvu3Lmorq7GhQsXUF5ejvLycpw/fx5VVVX45z//aYgyknbW0KQaSeMLCGlDjTI1NRU//fQTvL292W0+Pj7YvHkzXnzxRb0WjhhH08YcJT2jJET3GqVSqVSbCANQdUan4Y2dA7V6E8LVpmnW5s2bh5s3b7Lbbty4gbfffhtjxozRuQCbN2+Gu7s7LC0tIZFIcPLkyRbjDxw4AC8vL1haWsLX1xeHDx/m7GcYBrGxsXB2doaVlRWkUikuX77MiSkvL8e0adNgZ2cHBwcHREZGoqamhhPzww8/YMSIEbC1tYWTkxMmTZqEwsJCnd9fW2gaNtierc+cREnPKAnRPVFu2rQJVVVVcHd3R9++fdG3b194eHigqqoKGzdu1Olc+/btQ0xMDOLi4nD69GkMHjwYwcHBKC0t1RifkZGBqVOnIjIyEmfOnEFoaChCQ0M5Y88TEhKwYcMGJCUlISsrC126dEFwcDBnDfJp06bhwoULSEtLw6FDh/Dzzz8jKiqK3V9QUICJEyfi+eefR25uLn744QeUlZXhb3/7m46flmni3nobsSCEdBA6L1cLqGo3P/30Ey5evAgA8Pb2hlQq1fniEokEw4cPx6ZNmwCobutdXV0xd+5cLF68WC0+LCwMtbW1OHToELttxIgR8PPzQ1JSEhiGgYuLCxYsWICFCxcCACorKyESibBz505MmTIF+fn58PHxQXZ2Nvz9/QGonruOHz8e169fh4uLC7788ktMnToVdXV17Lri3333HSZOnIi6ujqNjx40aetytW98noPUCzLOtmsfjQe/hYHXLS1X+4xnd3weKcE3uTcwLyW31eu/+WxfbDl2FQDw1VsjMdStq3YFJ8SEGHS5WkA1/veFF17A3LlzMXfu3DYlSblcjpycHM6xfD4fUqkUmZmZGo/JzMxUu1ZwcDAbX1BQAJlMxomxt7eHRCJhYzIzM+Hg4MAmSQCQSqXg8/nIysoCAAwbNgx8Ph87duyAQqFAZWUlPv/8c0il0haTZF1dHaqqqjgvU8R9RklVSkLa1OE8PT0d6enpKC0tVWvA2b59u1bnKCsrg0KhgEgk4mwXiURsTfVRMplMY7xMJmP3N25rKaZHjx6c/QKBAI6OjmyMh4cHfvzxR0yePBmvv/46FAoFAgMD1Z6HPio+Ph4rV65sMUYbmkbQtGe6ajqrOeVJQtpQo1y5ciVefPFFpKeno6ysDHfv3uW8OgOZTIbZs2cjPDwc2dnZOH78OIRCIf7v//6vxRrWkiVLUFlZyb6Ki4vbsdT6o2jyx4+eURLShhplUlISdu7cienTpz/Whbt37w4zMzOUlJRwtpeUlEAsFms8RiwWtxjf+G9JSQmcnZ05MX5+fmzMo41FDQ0NKC8vZ4/fvHkz7O3tkZCQwMZ88cUXcHV1RVZWFkaMGKGxfBYWFrCwsGjtrbeJKkG3z1Q+TWc4f5x+lOdvVGLrz9fwTvAAuDpa66NohBiFzjVKuVyOkSNHPvaFhUIhhg0bhvT0dHabUqlEeno6AgMDNR4TGBjIiQeAtLQ0Nt7DwwNisZgTU1VVhaysLDYmMDAQFRUVyMnJYWOOHDkCpVIJiUQ1xvnevXtsI04jMzMztoydnb46nE/Y+Cu+PXsTb3yR03owIR2YzonytddeQ3Jysl4uHhMTg23btmHXrl3Iz8/Hm2++idraWkRERAAAZsyYgSVLlrDx8+bNQ2pqKtatW4eLFy/ivffew6lTpxAdHQ1A1cg0f/58rFq1Ct9++y3OnTuHGTNmwMXFhZ0eztvbG2PHjsXs2bNx8uRJnDhxAtHR0ZgyZQpcXFwAACEhIcjOzsb777+Py5cv4/Tp04iIiEDv3r1NcvKPG3fvA9B+Et6miVIfD0ev3a59/JMQYkQ633o/ePAAW7duxU8//YRBgwaptQJ/8sknWp8rLCwMt2/fRmxsLGQyGfz8/JCamso2xhQVFXFqdiNHjkRycjKWL1+OpUuXwtPTEwcPHsRTTz3FxixatAi1tbWIiopCRUUFRo0ahdTUVFhaWrIxe/bsQXR0NMaMGQM+n49JkyZhw4YN7P7nn38eycnJSEhIQEJCAqytrREYGIjU1FRYWVnp+pHpxePkq2tlqkSlbQt2g577UVKndWLqdO5H+dxzzzV/Mh4PR44ceexCdRZt7Uf51p4cHD7H7Ud5+cNxMDdr/gagpX6UAFC4OkTrfpShfi44mKsaebV7VgBG92/bjFCNZbIQ8HFp1bg2nYMQQ9Hl91PnGuXRo0fbXDDSdu3ZTaeBM4SREEKrNhM1NHsQIVyUKE1Ec8/5Mq6W4Vblfa3OoW1jTgONzCGEo81LQRDjy7x6B69uy9L7eZVNEuWPF0pwX65EyCDnFo5oGaVaYuqoRmkiNFXssgruGORaTWuUKdnFmJN8GhX35Aa5FiGmQOdE+fPPP6OhoUFte0NDA37++We9FIoYl6bnkvfkirafkKqUxMTpnCife+45lJeXq22vrKxssesQ0T9dHx9qOwCy6aQYjfhaPt8kpDPSOVEyDKOxUeDOnTvo0qWLXgpF1P339PXHPoe2eVWhIQO3MBUmIZ2e1o05jbN783g8zJw5kzP5g0KhwO+//66XMeBE81IQy74+j2mS3u1yfYWG4TjatpgT0hlpnSjt7e0BqGqUtra2nKF8QqEQI0aMwOzZs/VfQtIsQz36a9CQKB+nRklDGImp0zpR7tixAwDg7u6OhQsX0m12J6bQwwxJn/x4SQ8lIaRj0LkfZVxcnCHKQdpCx9YcbSuFCg15Utc64YYjV3Q8gpCOS+fGnJKSEkyfPh0uLi4QCAQwMzPjvIjp01SjfJwBOjS4h5g6nWuUM2fORFFREVasWAFnZ2d6yN8JaWrMoeeM5Emmc6L89ddf8csvv7BLKxADMMDfHl3GbGtKlI+TJynFElOn8623q6srTZRggmbtzNa4uqMmmvpREvIk0zlRJiYmYvHixSgsLDRAcYihHL10W+tnhQoNI3ModZInmc633mFhYbh37x769u0La2trtaUgNA1vJIbRnv0oqZJJnmQ6J8rExEQDFIN0JJomxXicxhx6VENMnc6JMjw83BDlIG2ga/4pra7TKk5jqzflOvIEa9N8lFevXsXy5csxdepUlJaWAgD+97//4cKFC3ot3JPKUB2uPjiUp1WcxltvfReGEBOic6I8fvw4fH19kZWVha+++go1NTUAgLNnz7Zp1M7mzZvh7u4OS0tLSCQSnDx5ssX4AwcOwMvLC5aWlvD19cXhw4c5+xmGQWxsLJydnWFlZQWpVIrLly9zYsrLyzFt2jTY2dnBwcEBkZGR7Ptoep61a9eif//+sLCwQM+ePfHhhx/q/P5MkVJjjZJSJXly6ZwoFy9ejFWrViEtLQ1CoZDd/vzzz+O3337T6Vz79u1DTEwM4uLicPr0aQwePBjBwcFsLfVRGRkZmDp1KiIjI3HmzBmEhoYiNDQU58+fZ2MSEhKwYcMGJCUlISsrC126dEFwcDAePHjAxkybNg0XLlxAWloaDh06hJ9//hlRUVGca82bNw+fffYZ1q5di4sXL+Lbb79FQECATu/P0AzVCVxTjfJxUIolpk7ndb1tbGxw7tw5eHh4wNbWFmfPnkWfPn1QWFgILy8vTkJqjUQiwfDhw7Fp0yYAgFKphKurK+bOnYvFixerxYeFhaG2thaHDh1it40YMQJ+fn5ISkoCwzBwcXHBggULsHDhQgCqCYVFIhF27tyJKVOmID8/Hz4+PsjOzoa/vz8AIDU1FePHj8f169fh4uKC/Px8DBo0COfPn8eAAQN0+Xg42rqud3TyaRz6/Zba9sLVIZyv1/xwEZuPXm1z+Zoj4PPUkuUvi56Dq6O11ud4dJ3xR8tOiLHp8vupc43SwcEBt26p/xKfOXMGPXv21Po8crkcOTk5kEqlDwvD50MqlSIzM1PjMZmZmZx4AAgODmbjCwoKIJPJODH29vaQSCRsTGZmJhwcHNgkCQBSqRR8Ph9ZWaqFur777jv06dMHhw4dgoeHB9zd3fHaa6+12vWprq4OVVVVnJcp0neNkhBTp3OinDJlCt59913IZDLweDwolUqcOHECCxcuxIwZM7Q+T1lZGRQKBUQiEWe7SCSCTCbTeIxMJmsxvvHf1mJ69OjB2S8QCODo6MjGXLt2DX/++ScOHDiA3bt3Y+fOncjJycH//d//tfie4uPjYW9vz75cXV1bjDcl9IiSPMl0TpQfffQRvLy84OrqipqaGvj4+GD06NEYOXIkli9fbogytjulUom6ujrs3r0bzzzzDJ599ln85z//wdGjR3HpUvPzLC5ZsgSVlZXsq7i4uB1LbVg0KQZ5kuncj1IoFGLbtm1YsWIFzp8/j5qaGgwZMgSenp46nad79+4wMzNDSUkJZ3tJSQnEYrHGY8RicYvxjf+WlJTA2dmZE9M4iYdYLFZrLGpoaEB5eTl7vLOzMwQCAfr378/GeHt7AwCKioqafW5pYWHBWSLD0NqzlqfLteoaHmPFRkI6oDav6+3m5obx48dj8uTJOidJQJVwhw0bhvT0dHabUqlEeno6AgMDNR4TGBjIiQeAtLQ0Nt7DwwNisZgTU1VVhaysLDYmMDAQFRUVyMnJYWOOHDkCpVIJiUQCAHj66afR0NCAq1cfNpT88ccfAIDevdtn3ZqORpecnJBKs5uTzkWrGmVMTAw++OADdOnSBTExMS3GfvLJJ1pfPCYmBuHh4fD390dAQAASExNRW1uLiIgIAMCMGTPQs2dPxMfHA1B12QkKCsK6desQEhKClJQUnDp1Clu3bgWgWgBr/vz5WLVqFTw9PeHh4YEVK1bAxcUFoaGhAFQ1w7Fjx2L27NlISkpCfX09oqOjMWXKFLi4uABQNe4MHToUs2bNQmJiIpRKJebMmYMXXniBU8s0FFOf4/Ob3JvGLgIheqVVojxz5gzq6+vZ/zdH11/wsLAw3L59G7GxsZDJZPDz80NqairbGFNUVAQ+/2Gld+TIkUhOTsby5cuxdOlSeHp64uDBg3jqqafYmEWLFqG2thZRUVGoqKjAqFGjkJqaCktLSzZmz549iI6OxpgxY8Dn8zFp0iRs2LCB3c/n8/Hdd99h7ty5GD16NLp06YJx48Zh3bp1Or0/Q2vPp4a6zWf5+GvuENKR6NyPkmivrf0o5+49g+/OqtfKHu2L+HHqRWw5pv9+lJqkLwhCXycbrWJ93/sB1Q8aONuoHyXpaAzaj5I8mXT5c6pxhnRCTJhWt95/+9vftD7hV1991ebCEBVtH2C0772A9hejDuuks9GqRtm0E7WdnR3S09Nx6tQpdn9OTg7S09Nhb29vsIIS49IlKWuaVIMQU6ZVjXLHjh3s/999911MnjwZSUlJ7PK0CoUCb731lk7P4Yhp0SX1UY2SdDY6P6Pcvn07Fi5cyFnD28zMDDExMdi+fbteC0cIIR2BzomyoaEBFy9eVNt+8eJFKKlbSLswRkcF6htBnmQ6D2GMiIhAZGQkrl69ys7PmJWVhdWrV7MdxYnh1NY1YNz6XzDKsztsLXX+9rUZjfUmTzKdf9PWrl0LsViMdevWsdOtOTs745133sGCBQv0XsAnUUv99r89exNF5feQnFWE14P6tFuZtK1R0jhv0hnpnCj5fD4WLVqERYsWsfMtUiNO+zHWLbC21zXERMKEGNtj3btRgmx/nNpme84epOXFjlwsaT2IEBPTpkT55ZdfYv/+/SgqKoJcLufsO336tF4KRjQz1nQZ2tYoBXwa7EU6H51/qjds2ICIiAiIRCKcOXMGAQEB6NatG65du4Zx48YZooykiaY1yo7YvGJuZtozHxGiic6J8tNPP8XWrVuxceNGCIVCLFq0CGlpafjnP/+JyspKQ5SRNMEzWp1SO2b8jl0+QtpC50RZVFSEkSNHAgCsrKxQXV0NAJg+fTr27t2r39IRdUbKQ9reepub0a036Xx0/qkWi8XsaoRubm7sWt4FBQVG6Qj9pDHaM0otb/QFVKMknZDOifL555/Ht99+C0DV+fztt9/GCy+8gLCwMLzyyit6LyDhMtbs59r+DTSjxhzSCenc6r1161Z2qOKcOXPQrVs3ZGRk4OWXX8brr7+u9wI+iVpKhcarUWqHGnNIZ6RTomxoaMBHH32EWbNmoVevXgBU63xPmTLFIIUj6jit3u34qEPbawnoGSXphHT6qRYIBEhISEBDQ0PrweSJRM8oSWek85//MWPG4Pjx44YoC9GCsRZo1LbuSomSdEY6J8px48Zh8eLFWLhwIfbu3Ytvv/2W82qLzZs3w93dHZaWlpBIJDh58mSL8QcOHICXlxcsLS3h6+uLw4cPc/YzDIPY2Fg4OzvDysoKUqkUly9f5sSUl5dj2rRpsLOzg4ODAyIjI1FTU6PxeleuXIGtrS0cHBza9P70qWk/yvbsZKD1yBx6Rkk6IZ0bc9566y0Amtfv5vF4UCh0mz1m3759iImJQVJSEiQSCRITExEcHIxLly6hR48eavEZGRmYOnUq4uPjMWHCBCQnJyM0NBSnT59ml61NSEjAhg0bsGvXLnZt7+DgYOTl5bHL1k6bNg23bt1CWloa6uvrERERgaioKCQnJ3OuV19fj6lTp+KZZ55BRkaGTu/NEIy35Ld2mZI6nJPOSOcapVKpbPala5IEVAl39uzZiIiIgI+PD5KSkmBtbd3sbOnr16/H2LFj8c4778Db2xsffPABhg4dik2bNgFQ1SYTExOxfPlyTJw4EYMGDcLu3btx8+ZNHDx4EACQn5+P1NRUfPbZZ5BIJBg1ahQ2btyIlJQU3LzJXSZ2+fLl8PLywuTJk3V+b23VUhegpvvad11v7eJorDfpjIz6Uy2Xy5GTkwOpVMpu4/P5kEqlyMzM1HhMZmYmJx4AgoOD2fiCggLIZDJOjL29PSQSCRuTmZkJBwcH+Pv7szFSqRR8Ph9ZWVnstiNHjuDAgQPYvHmzVu+nrq4OVVVVnJe+dfTuQedu0DBW0vlofet9//59pKenY8KECQCAJUuWoK6ujt1vZmaGDz74gL211UZZWRkUCgVEIhFnu0gk0rjcBADIZDKN8TKZjN3fuK2lmEdv6wUCARwdHdmYO3fuYObMmfjiiy+0nk4uPj4eK1eu1Cq2rYzWmKNFpvyjpBo5f941fGEIaWda1yh37dqFf//73+zXmzZtQkZGBs6cOYMzZ87giy++wJYtWwxSSGOYPXs2Xn31VYwePVrrY5YsWYLKykr2VVxcrPdydeRJMXKLK4xdBEIMQutEuWfPHkRFRXG2JScn4+jRozh69CjWrFmD/fv363Tx7t27w8zMDCUl3MleS0pKIBaLNR4jFotbjG/8t7WY0tJSzv6GhgaUl5ezMUeOHMHatWshEAggEAgQGRmJyspKCASCZp+fWlhYwM7OjvPSN+PVKLWoUtJQf9JJaZ0or1y5Al9fX/ZrS0tL8Js8uA8ICEBeXp5OFxcKhRg2bBjS09PZbUqlEunp6QgMDNR4TGBgICceANLS0th4Dw8PiMViTkxVVRWysrLYmMDAQFRUVCAnJ4eNOXLkCJRKJSQSCQDVc8zc3Fz29f7778PW1ha5ublGHdPekZ9RKmlSFNJJaf2MsqKigvNM8vbt25z9SqWSs19bMTExCA8Ph7+/PwICApCYmIja2lp2RccZM2agZ8+eiI+PBwDMmzcPQUFBWLduHUJCQpCSkoJTp05h69atAFStwvPnz8eqVavg6enJdg9ycXFBaGgoAMDb2xtjx47F7NmzkZSUhPr6ekRHR2PKlClwcXFhY5o6deoU+Hw+2wXJkLRNhh2tH6WS8iTppLROlL169cL58+cxYMAAjft///13dvy3LsLCwnD79m3ExsZCJpPBz88PqampbGNMUVERp+Y6cuRIJCcnY/ny5Vi6dCk8PT1x8OBBTgJbtGgRamtrERUVhYqKCowaNQqpqamchqY9e/YgOjoaY8aMAZ/Px6RJk7Bhwwady28IzeUbhmGMODKn9SyooBol6aR4jJazHcybNw8//fQTcnJy1Fq279+/D39/f0ilUqxfv94gBTVFVVVVsLe3R2VlpU7PK9/el4uvz9xQ214QPx4/XCjBG1+oHhlEPO2OHScK9VXcFiW/JsHIft1bjNmdWYjYby5o3Fe4OsQQxSKkzXT5/dS6Rrl06VLs378fAwYMQHR0NPr37w8AuHTpEjZt2oSGhgYsXbr08UpOADR/680wHXust5LuvUknpXWiFIlEyMjIwJtvvonFixezraA8Hg8vvPACPv30U7W+i0S/GBivMUcblCdJZ6XTWG8PDw+kpqaivLwcV65cAQD069cPjo6OBikc4VI9o+y4k2JQqzfprNq0rrejoyMCAgL0XRbSCmPWKLVpzKE8STormsGgI2ohGxprzgmqUZInGSVKE8IwxhvCqE0KPF1E47xJ50SJ0oQwYDi1zZsV99vv2q3UFgvLavHDhZIWYwgxVZQoTUhJJXfk04957ZeYWqtR/lFS3S7lIMQYKFGakH/99EeH7R5kLqAfJdJ50U+3CWlQMi3Ofm5QrVQphbRMLenE6Ke7A2quwYZhmA7bPcicEiXpxOin24Qw6LgznNPqi6Qzo0RpQlQ1SiN1D6Jbb/IEo59uE9KRJ8WgGiXpzChRmhCGMeIQxlaqlHzjLThOiMFRojQhSmNmSkKeYJQoO6DmKmeqSTE67hBGQjorSpQdUHN3uUZdCqKVTEnzYZDOjBKlCTHunTdlQmICHlQBV48Cd//U62nbNB8lMawWb72NVKWkGiPpcJRKoOwP4Ho2cP0kcP0UUJoPgAFeeB94ep7eLkWJ0oQojboKIyFGdv8ucD2nSWLMAeoq1eMc3AC+flNbh7j13rx5M9zd3WFpaQmJRIKTJ0+2GH/gwAF4eXnB0tISvr6+OHz4MGc/wzCIjY2Fs7MzrKysIJVKcfnyZU5MeXk5pk2bBjs7Ozg4OCAyMhI1NTXs/mPHjmHixIlwdnZGly5d4Ofnhz179ujvTbeBcbsHGenC5MmkVAAlF4BTO4CDc4BNw4GP3YE9k4Djq4GrR1RJUmAF9H4aeHo+ELYHWPAHMP8cEDhHr8Uxeo1y3759iImJQVJSEiQSCRITExEcHIxLly6hR48eavEZGRmYOnUq4uPjMWHCBCQnJyM0NBSnT59m1/ZOSEjAhg0bsGvXLnh4eGDFihUIDg5GXl4eu9TutGnTcOvWLaSlpaG+vh4RERGIiopCcnIye51Bgwbh3XffhUgkwqFDhzBjxgzY29tjwoQJ7fcBNWHMIYyt0WapCEKaVXsHuHEKKD6pqjHeOA3INUzd59gH6DX84Us0EDAzN3jxtF7X21AkEgmGDx+OTZs2AQCUSiVcXV0xd+5cLF68WC0+LCwMtbW1OHToELttxIgR8PPzQ1JSEhiGgYuLCxYsWICFCxcCACorKyESibBz505MmTIF+fn58PHxQXZ2Nvz9/QEAqampGD9+PK5fvw4XFxeNZQ0JCYFIJML27du1em9tXdd74YGz+DLnutr2Zzy7Y760PyZtydD6XPqy6dUhmDBI8+cCABdlVRib+Euz+2ldb8JSNAClF1QJsThb9W/5VfU4oQ3Qc+hfSTEA6OUPdGl5bXldGGRdb0OQy+XIycnBkiVL2G18Ph9SqRSZmZkaj8nMzERMTAxnW3BwMA4ePAgAKCgogEwmg1QqZffb29tDIpEgMzMTU6ZMQWZmJhwcHNgkCQBSqRR8Ph9ZWVl45ZVXNF67srIS3t7ezb6furo61NU9nFy3qqqq+TffRh21exAhzaq5/dczxb8S483TQP099bhunoDrXwmxVwDQwxvgm7V/eTUwaqIsKyuDQqFQWw9cJBLh4sWLGo+RyWQa42UyGbu/cVtLMY/e1gsEAjg6OrIxj9q/fz+ys7Px73//u9n3Ex8fj5UrVza7X1vN5UJjLt5FeZJoRVEPyM791eCSrbqVrtDQVcfCDug57K/EOFz1f+uOu+y10Z9RmoKjR48iIiIC27Ztw8CBA5uNW7JkCae2W1VVBVdXV72VQ6k0Xs3OyE9oSEdVdYvbPefmGaDhwSNBPMDJS1VTbEyM3QcYb0nRNjBqouzevTvMzMxQUsJd+6WkpARisVjjMWKxuMX4xn9LSkrg7OzMifHz82NjSktLOedoaGhAeXm52nWPHz+Ol156Cf/6178wY8aMFt+PhYUFLCwsWox5HB25wUSpNHYJiME11Klqi40NLtezgcpi9ThLh4eNLa5/1RYt7du9uPpk1EQpFAoxbNgwpKenIzQ0FICqMSc9PR3R0dEajwkMDER6ejrmz5/PbktLS0NgYCAAwMPDA2KxGOnp6WxirKqqQlZWFt588032HBUVFcjJycGwYcMAAEeOHIFSqYREImHPe+zYMUyYMAEff/wxoqKi9PzudWfMSl1r165XUKbsdCqvcxtcbuUCCjk3hscHevg0SYwBgGNfk6otasPot94xMTEIDw+Hv78/AgICkJiYiNraWkRERAAAZsyYgZ49eyI+Ph4AMG/ePAQFBWHdunUICQlBSkoKTp06ha1btwJQjVyZP38+Vq1aBU9PT7Z7kIuLC5uMvb29MXbsWMyePRtJSUmor69HdHQ0pkyZwrZ4Hz16FBMmTMC8efMwadIk9tmlUCiEo6NxnqUYM1GuOHgep4vuYnmID4QaFhKjRGni6u8Dt842qS2eAqpvqsdZd+N2z+k5FLCwbf/ytjOjJ8qwsDDcvn0bsbGxkMlk8PPzQ2pqKtsYU1RUBH6Tv04jR45EcnIyli9fjqVLl8LT0xMHDx5k+1ACwKJFi1BbW4uoqChUVFRg1KhRSE1NZftQAsCePXsQHR2NMWPGgM/nY9KkSdiwYQO7f9euXbh37x7i4+PZJA0AQUFBOHbsmAE/keYZ89a7uq4BuzP/xJ0aOdZP8YPgkRnNTxfdNVLJiM4YRtXAcr1Jv0XZOUBZz43jmQHip7jdcxz7dNzOvAZk9H6UnZku/bROFZZj78liLB3vhY9TL2L/KfV+lMN6d8XS8d7t0o+SzwOUzfxkfDzJF2HD3Tjb3Bd/z/5/dH8n/PzHbc5+6kdpRPJa4GbuwwaX4pNAbal6XJce3O45Ln6AsEt7l7bdmEw/SvLQ/yWp+o3+lF8C9+7G/+Hk8XjN3usX3tHQB64Jb7GtWqIk7YRhgPJrqoTY2HdRdh5gFNw4vgAQD3rYCt1ruGqM9BNYW9QGJcoOpvJ+Pc4WV2jc156V/5Z+Xe7Lub90dFNiRHXVquF+ja3Q17OBe3fU42ydHza29BoOOA8GzK3av7wmihKlCWnuVtgQWqpYHPr9JlZM8IEZXxXU0J4Fe5Iplaqhfk2755TmAcwjDWlmQsDZ72H3nF7DAfteRilyZ0GJ0oQwAK6UapgowABU815qToBlNXIcuViKF3xUDW6yyocdjMf7ijtwb08T86ASuJHzsHvO9WzgQYV6nL0rt3uO2BcQGK4/75OIEqUJOVtc0extub619qTq3I1KNlHm3Xo4pn2AyA618gYDlqyTUiqBsksPh/1dPwXcvgi1P1YCS8BlCLeLjp2zxlMS/aFESdqkp8PDrlaV9x92K3GytUBNWb2mQ0hT98pVtcXGxHgjB6jTMIlKV3du9xyxb7tMK0a4KFESjTQ9o+zWRYg7taqRGU2fSzYNfWVIT3ySdsnApTMxSoVqiYKm3XPuXFaPM7dWDfdr7J7Tyx+wUZ+TlbQ/SpREI76GTDltRG9sSFf9giuaJMrSatXUcpOG9oKVsGNMi2VUtWXc7jk3TgPyGvU4x76PTCvmA5jRr2RHRN8VopGmZ5QOVuZ4ebALvj17Ew2Kh4ny9l+JsoedqgHhieotpGgASs5zu+eUX1OPE9qqhvs17bfYgacVI1yUKDuAhg44TlpTjdLGUgDBX12CfsyTYdYoDwAPG3NEtk9AS2tNKbd7zs0zmieh7T6A2z3HyavDTEJLdEeJsgOoqeuArcQaqpS2FgLcrlHVHn+7Vo4H9aqO5ycLygEA3Ww6WaJskAMl55p0zzkJVBSpx1nY/3X73GRaMauu7V9eYjCUKDuAd7783dhFUKPp1tvGUoBrt2vZr4vK73GeVTbOKmSyd95VN7ndc27lap6Etoc3t99iN89ON60Y4aJE2QFcyv8dQjhCjo7T7YOn4dbb1tIcNyrus1/nFlXAw+nhuHQvsQlNt9VQp5pWrGlirFKfiARWXbndc3oOAyy1XyiOdA6UKDuAL8w/goh3F3mMO04rPXFG2Q+nlZ64iW4w1krefA2XtbEQYPqI3vj8N9UaKCnZRRggfpg0endTJc0O15jDMH9NQtuke47sd82T0IoGNkmMw4FufWmiCEKJ0ujqqmHNq4MFrwFDeFcwhH+F3VXCOHAS5zmmD+ogbJdiaa5RCrAsxBuujlb46PBFFN65h1wNI4WMvmRF/f0m04r9NUN3jYZF46y7PzKt2BDAwqbdi0s6PkqUxmZhC/+6LXDjlWII7zKG8i9jCP8KfHh/QsSrwDizbIwzywYA1DNmyGN6P0yejCeuM04wRK1T4zNKCwEszc3wypBe+OjwRdy9Jzd+7ZFhgLuFj0wrdg5QPtJAxhcAoqe43XO6ulNtkWiFEmWHwEMRI0IRI8I3ylEAAEvUwZdXwCbOofzL6MGrwGDeNQzmX0MEfgAA3GbscEbpiTNKT5xWeuJ3xgP3YdnSxbQrkYb8Yf1XZ/Ku1uYtTuxrUPJa9WnFajXMfWkjemRaMT9AaN3uxSWdAyXKDuoBLJDNeCFb4QUoAIBBT5RxEqcPrxBOvCq8aJaDF81yAAANDB8XGTf2dv0M0w+FjBi61jo13Xo3bhOY8SHx6IbMaw/nPezepGuQ3mqZ7CS02Q/7LpZc0DAJrblqfkXOtGKuVFskekOJ0mTwcANOuKF0wnfKkQAAC8gxkFeIIfyHt+wuvHI8xSvEU/xCTMdPAIByxoatcZ5h+uGssi9q0fKkra2lmGf6d+ckyiXjvB7r3QH4axLaHHBW/rtfrh5n1/ORacUGAeaPX4smpDmUKE1YHYQ4zfTHaUV//OevSpYYd9ga5xD+FfjyCuDIq8EYszMYY3YGAKBkeLjE9FLdsjOqmuc1xhkMHvYFbK0y1qc7t9Gjdzcdb2uVSuDOFW6DS2ke1Hphmlmo1m5pOq2YfU/drkXIY+oQiXLz5s1Ys2YNZDIZBg8ejI0bNyIgIKDZ+AMHDmDFihUoLCyEp6cnPv74Y4wfP57dzzAM4uLisG3bNlRUVODpp5/Gli1b4OnpycaUl5dj7ty5+O6779hVGNevXw8bm4cJ4Pfff8ecOXOQnZ0NJycnzJ07F4sWLTLMh6AnMnTD/5Td8D+lan1yIerhzfuTc8vei1cGb14xvPnFeBVHAAAVTBfkKvuxjUS3mIEoQfND7sT23Bpcvx6ttBbfr3hYW2Qnoa1Uj3Nw43bPEfsCgvZp6SekOUZPlPv27UNMTAySkpIgkUiQmJiI4OBgXLp0CT16qE8xlZGRgalTpyI+Ph4TJkxAcnIyQkNDcfr0aXbJ2oSEBGzYsAG7du1i1/UODg5GXl4eu2TttGnTcOvWLaSlpaG+vh4RERGIiopCcnIyANUKbS+++CKkUimSkpJw7tw5zJo1Cw4ODoiKitLrZ7A8xBt378mx+ehVvZ4XAOQwx1mmH84q+mHHX7VOJ9zFUL6qK9IQ/mUM4l2DA68Wz5qdxbNmZwEAynoerghdVLfsjKqVHUolOwKlaWI04/PgYP0wmfEYJfrzilWJmXcZQ/iXgY9vqBdOYKWaKKLptGK2Yr1/BoQ8LqMvVyuRSDB8+HBs2rQJAKBUKuHq6oq5c+di8eLFavFhYWGora3FoUOH2G0jRoyAn58fkpKSwDAMXFxcsGDBAixcuBAAUFlZCZFIhJ07d2LKlCnIz8+Hj48PsrOz4e/vDwBITU3F+PHjcf36dbi4uGDLli1YtmwZZDIZhEJVEli8eDEOHjyIixcvavXedFkOE+Au+dqeBGiAF6/o4S077wrc+SXqgRZ2fyU2VW1v8Pa7qIQN7FGDs7O6suOhHxRmw1JZq358Vw9u9xzRQJqElhiNySxXK5fLkZOTgyVLlrDb+Hw+pFIpMjMzNR6TmZmJmJgYzrbg4GAcPHgQAFBQUACZTAapVMrut7e3h0QiQWZmJqZMmYLMzEw4ODiwSRIApFIp+Hw+srKy8MorryAzMxOjR49mk2TjdT7++GPcvXsXXbuqT3pQV1eHuro69uuqKg0zVrfg88gATP/PSZ2O0YcGCHCe6YPzij74XPEiAGCkWAmr0jNs4hwquAbLuirg2jHVC8BZS6CUcUAPXgWQ/PB8lgBqGQucVfZla6M13YegkrEHiqB6oQqA5u8xIY9r2ojemD6it97OZ9REWVZWBoVCAZFIxNkuEomarbXJZDKN8TKZjN3fuK2lmEdv6wUCARwdHTkxHh4eaudo3KcpUcbHx2PlypXNv+FWPOPphM9m+OO13acQ1N8JJVUPcFGmWkzM0pyPB/XtNx3b+68+i0VfOsJ5xCwobS2hdLUBKi4/HBd9/SRw54oqSQKqiSH+6p5zRumJSV9VQNmkcQilANA+C6MRUlZd13qQDoz+jLIzWbJkCae2W1VVBVdXV53OIfURIX1BEHp1tYK8QYni8vsQCnjw6G6D3OIK1CuUsLEQ4Prd++hqbY7c4goMENuiq7UQLg5WyL9VhdziClia85F/qxp9uneBl7MdFEolXB2tkXezCg8alOABENtZwlzAR32DEpX363GyoBx5t6rw6bShcHW0xldvPc0tnNhX9Roeqfr6XjmYssvgdffkTEI7BEBq72ocvVgKK6GZWgs5IYbm5qjfwQVGTZTdu3eHmZkZSkq4z8NKSkogFmt+qC8Wi1uMb/y3pKQEzs7OnBg/Pz82prS0lHOOhoYGlJeXc86j6TpNr/EoCwsLWFg8/pyMfZ1UicVCYAYfl4fP8Ib1fliLfaqnPQBA0qcb51gnWyeM7u/U7LkHutg3u2/SMB3XfrZ2BM9NonFXf5Et+otMaDYhQlpg1En0hEIhhg0bhvT0dHabUqlEeno6AgMDNR4TGBjIiQeAtLQ0Nt7DwwNisZgTU1VVhaysLDYmMDAQFRUVyMnJYWOOHDkCpVIJiUTCxvz888+or6/nXGfAgAEab7sJIZ0YY2QpKSmMhYUFs3PnTiYvL4+JiopiHBwcGJlMxjAMw0yfPp1ZvHgxG3/ixAlGIBAwa9euZfLz85m4uDjG3NycOXfuHBuzevVqxsHBgfnmm2+Y33//nZk4cSLj4eHB3L9/n40ZO3YsM2TIECYrK4v59ddfGU9PT2bq1Kns/oqKCkYkEjHTp09nzp8/z6SkpDDW1tbMv//9b63fW2VlJQOAqaysfJyPiBBiALr8fho9UTIMw2zcuJFxc3NjhEIhExAQwPz222/svqCgICY8PJwTv3//fqZ///6MUChkBg4cyHz//fec/UqlklmxYgUjEokYCwsLZsyYMcylS5c4MXfu3GGmTp3K2NjYMHZ2dkxERARTXV3NiTl79iwzatQoxsLCgunZsyezevVqnd4XJUpCOi5dfj+N3o+yM9O1HyUhpP3o8vtJC30QQkgrKFESQkgrKFESQkgrqMO5ATU+/tV1KCMhxPAafy+1aaahRGlA1dWqIXu6js4hhLSf6upq2Ns3PxAD6ACzB3VmSqUSN2/ehK2trcalFUxd4xDN4uLiJ7pVnz4HFVP7HBiGQXV1NVxcXMDnt/wUkmqUBsTn89Grl47DAk2QnZ2dSfxiGBp9Diqm9Dm0VpNsRI05hBDSCkqUhBDSCkqUpM0sLCwQFxenlxmTTBl9Diqd+XOgxhxCCGkF1SgJIaQVlCgJIaQVlCgJIaQVlCgJIaQVlChJizZv3gx3d3dYWlpCIpHg5MmWl9M9cOAAvLy8YGlpCV9fXxw+fLidSmpYunwO27ZtwzPPPIOuXbuia9eukEqlrX5upkLXn4dGKSkp4PF4CA0NNWwBDcVw8wcTU5eSksIIhUJm+/btzIULF5jZs2czDg4OTElJicb4EydOMGZmZkxCQgKTl5fHLF++XG2ZDlOk6+fw6quvMps3b2bOnDnD5OfnMzNnzmTs7e2Z69evt3PJ9UvXz6FRQUEB07NnT+aZZ55hJk6c2D6F1TNKlKRZAQEBzJw5c9ivFQoF4+LiwsTHx2uMnzx5MhMSEsLZJpFImNdff92g5TQ0XT+HRzU0NDC2trbMrl27DFXEdtGWz6GhoYEZOXIk89lnnzHh4eEmmyjp1ptoJJfLkZOTA6lUym7j8/mQSqXIzMzUeExmZiYnHgCCg4ObjTcFbfkcHnXv3j3U19fD0dGx9eAOqq2fw/vvv48ePXogMjKyPYppMDQpBtGorKwMCoUCIpGIs10kEuHixYsaj5HJZBrjZTKZwcppaG35HB717rvvwsXFRe2PiClpy+fw66+/4j//+Q9yc3PboYSGRYmSEANavXo1UlJScOzYMVhaWhq7OO2muroa06dPx7Zt29C9e3djF+exUaIkGnXv3h1mZmYoKSnhbC8pKYFYLNZ4jFgs1ineFLTlc2i0du1arF69Gj/99BMGDRpkyGIanK6fw9WrV1FYWIiXXnqJ3aZUKgEAAoEAly5dQt++fQ1baD2iZ5REI6FQiGHDhiE9PZ3dplQqkZ6ejsDAQI3HBAYGcuIBIC0trdl4U9CWzwEAEhIS8MEHHyA1NRX+/v7tUVSD0vVz8PLywrlz55Cbm8u+Xn75ZTz33HPIzc01vVn/jd2aRDqulJQUxsLCgtm5cyeTl5fHREVFMQ4ODoxMJmMYhmGmT5/OLF68mI0/ceIEIxAImLVr1zL5+flMXFxcp+kepMvnsHr1akYoFDJffvklc+vWLfZVXV1trLegF7p+Do8y5VZvSpSkRRs3bmTc3NwYoVDIBAQEML/99hu7LygoiAkPD+fE79+/n+nfvz8jFAqZgQMHMt9//307l9gwdPkcevfuzQBQe8XFxbV/wfVM15+Hpkw5UdI0a4QQ0gp6RkkIIa2gREkIIa2gREkIIa2gREkIIa2gREkIIa2gREkIIa2gREkIIa2gREkIIa2gREmeKDNnzgSPx1N7jR071thFIx0YzR5Enjhjx47Fjh07ONssLCw0xtbX18Pc3JyzTS6XQygU6nzdth5HjI9qlOSJY2FhAbFYzHl17doVAMDj8bBlyxa8/PLL6NKlCz788EO899578PPzw2effQYPDw92XsmioiJMnDgRNjY2sLOzw+TJkznTkDV3HDE9lCgJecR7772HV155BefOncOsWbMAAFeuXMF///tffPXVV8jNzYVSqcTEiRNRXl6O48ePIy0tDdeuXUNYWBjnXI8eR0wT3XqTJ86hQ4dgY2PD2bZ06VIsXboUAPDqq68iIiKCs18ul2P37t1wcnICoJpn89y5cygoKGDnVty9ezcGDhyI7OxsDB8+XONxxDRRoiRPnOeeew5btmzhbGu68JemiXZ79+7NSXb5+flwdXXlTEDr4+MDBwcH5Ofns4ny0eOIaaJESZ44Xbp0Qb9+/Vrcr802ba9FTB89oySkDby9vVFcXIzi4mJ2W15eHioqKuDj42PEkhFDoBoleeLU1dWpLaErEAh0Wi1QKpXC19cX06ZNQ2JiIhoaGvDWW28hKCioU6yRQ7ioRkmeOKmpqXB2dua8Ro0apdM5eDwevvnmG3Tt2hWjR4+GVCpFnz59sG/fPgOVmhgTLQVBCCGtoBolIYS0ghIlIYS0ghIlIYS0ghIlIYS0ghIlIYS0ghIlIYS0ghIlIYS0ghIlIYS0ghIlIYS0ghIlIYS0ghIlIYS0ghIlIYS04v8BKiC4P93OezAAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019608, + "end_time": "2024-02-29T18:14:02.404720", + "exception": false, + "start_time": "2024-02-29T18:14:02.385112", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 1138.024799, + "end_time": "2024-02-29T18:14:05.144850", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/tab_ddpm_concat/4/mlu-eval.ipynb", + "output_path": "eval/insurance/tab_ddpm_concat/4/mlu-eval.ipynb", + "parameters": { + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "param_index": 2, + "path": "eval/insurance/tab_ddpm_concat/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-02-29T17:55:07.120051", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/tab_ddpm_concat/model.pt b/insurance/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..456cd288d2c39552e325a7c28e1b5117a5cce956 --- /dev/null +++ b/insurance/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dd7b4d6e01bcd20c011318b4556018c310099328ffe5000b29ff8b1919e8893 +size 38511671 diff --git a/insurance/tab_ddpm_concat/params.json b/insurance/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..53f5e0d3386946cd36772e4c7ce316050438704d --- /dev/null +++ b/insurance/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.77, "head_final_mul": "identity", "gradient_penalty_mode": "ALL", "synth_data": 2, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "loss_balancer_beta": 0.75, "loss_balancer_r": 0.95, "fixed_role_model": "tab_ddpm_concat", "mse_mag": true, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "relu6", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "rrelu", "head_activation_final": "softsign", "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/tvae/eval.csv b/insurance/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..7e0f2a970983ed530808044c67747bfc9d48ddd1 --- /dev/null +++ b/insurance/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.11739331457698554,0.06104676764607803,0.00041556813743054156,2.451298952102661,0.022227462381124496,0.6887103915214539,0.03157595172524452,8.493182690472167e-08,0.8849966526031494,0.015491104684770107,0.17248360812664032,0.020385488867759705,0.15985152125358582,1.6569136278121732e-05,3.3362956047058105 diff --git a/insurance/tvae/history.csv b/insurance/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..10744200046c66ccb7ae26039d50447e598a09a1 --- /dev/null +++ b/insurance/tvae/history.csv @@ -0,0 +1,21 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.031827369896927846,28.259707012480977,0.004191242665176187,0.2307685674633831,0.0,0.0,0.0,0.0,0.0321838908130303,320,40,54.558457136154175,1.3639614284038544,0.1704951785504818,0.05414983599475818,0.010481808317126706,6.6735696544030105,2.9961921549848113e-05,0.0279149753972888,0.0,0.0,0.0,0.0,0.010719987776246854,80,10,10.396106243133545,1.0396106243133545,0.1299513280391693,0.02609058879315853 +1,0.011675631045363843,18.501674464043028,0.0009566789830486755,0.09624193962663412,0.0,0.0,0.0,0.0,0.011828435189090668,320,40,55.64720034599304,1.391180008649826,0.17389750108122826,0.08122157981561032,0.004508110485039652,22.331014108657836,2.6730329889979034e-06,0.04836642872542143,0.0,0.0,0.0,0.0,0.004554628022015094,80,10,10.792717218399048,1.0792717218399048,0.1349089652299881,0.03636421860428527 +2,0.0035327200530446135,9.784873699297304,7.98118183580887e-06,0.0508232937194407,0.0,0.0,0.0,0.0,0.0035692405959707684,320,40,55.71424317359924,1.3928560793399811,0.17410700991749764,0.07611006779770832,0.0014172189374221488,0.09624732142401626,6.146688699448788e-06,0.020977134071290492,0.0,0.0,0.0,0.0,0.0014305356336990372,80,10,10.833696365356445,1.0833696365356444,0.13542120456695556,0.08805033396929503 +3,0.0020708162381197328,2.6992595067491036,7.111737180576228e-06,0.04061916306382045,0.0,0.0,0.0,0.0,0.002090448179660598,320,40,55.8498101234436,1.39624525308609,0.17453065663576126,0.08638362209312618,0.00137065484886989,0.005912328146041546,4.321905053217279e-06,0.02791164191439748,0.0,0.0,0.0,0.0,0.0013822964770952239,80,10,10.717517137527466,1.0717517137527466,0.13396896421909332,0.09996146205812692 +4,0.0014275090521550736,0.9193552540385213,1.838910397943677e-06,0.02831868330249563,0.0,0.0,0.0,0.0,0.0014419240753341,320,40,55.6291663646698,1.390729159116745,0.17384114488959312,0.08760609640739858,0.0012192766880616546,0.21555799300662254,1.798611496894864e-06,0.03309073373675346,0.0,0.0,0.0,0.0,0.001231387781444937,80,10,10.670984268188477,1.0670984268188477,0.13338730335235596,0.08692563734948636 +5,0.0017993727393331937,0.8518799075905094,2.7848555921972105e-06,0.03595072412863374,0.0,0.0,0.0,0.0,0.0018162189619033598,320,40,55.764097690582275,1.394102442264557,0.17426280528306962,0.08750550604891032,0.0010832886851858347,0.02544177576819493,1.465394489699734e-06,0.026945540122687815,0.0,0.0,0.0,0.0,0.001093548518838361,80,10,10.761934995651245,1.0761934995651246,0.13452418744564057,0.0844166411086917 +6,0.0011671951153402916,0.5874403900238576,1.4499453566324537e-06,0.027735682530328633,0.0,0.0,0.0,0.0,0.0011789663760282565,320,40,53.97340989112854,1.3493352472782134,0.16866690590977668,0.09012857411289588,0.005621012981282547,1.7633854886284097,6.435126891801701e-06,0.026362080965191124,0.0,0.0,0.0,0.0,0.005691835869220086,80,10,10.383732795715332,1.0383732795715332,0.12979665994644166,0.046620757598429916 +7,0.0026447449334227715,1.1427540660426474,2.4505569594213126e-06,0.03827556688338518,0.0,0.0,0.0,0.0,0.0026816512978257378,320,40,54.003010749816895,1.3500752687454223,0.16875940859317778,0.07941703633405268,0.0011788314557634294,0.005349384507599098,2.9599219267595344e-06,0.03608495369553566,0.0,0.0,0.0,0.0,0.0011917953903321176,80,10,10.279337406158447,1.0279337406158446,0.12849171757698058,0.10001012030988932 +8,0.0016346701009751995,0.718084112559017,3.656406406381263e-06,0.039087726082652804,0.0,0.0,0.0,0.0,0.0016500235098646954,320,40,53.33782744407654,1.3334456861019135,0.1666807107627392,0.08958328987937421,0.0010581719019683079,0.003487239831883926,4.90262124515084e-06,0.0341863420791924,0.0,0.0,0.0,0.0,0.0010685857152566314,80,10,9.576281309127808,0.9576281309127808,0.1197035163640976,0.0981753658503294 +9,0.0013097766401187982,0.6380827577727615,1.4933229921021417e-06,0.03195373273920268,0.0,0.0,0.0,0.0,0.0013226776201918256,320,40,52.700077295303345,1.3175019323825836,0.16468774154782295,0.08942489377222955,0.0010866674332646654,0.00882318691146793,4.24079198637628e-06,0.02658762000501156,0.0,0.0,0.0,0.0,0.0010967145411996172,80,10,9.853407621383667,0.9853407621383667,0.12316759526729584,0.09188855718821287 +10,0.0006990553600189741,1.061739949541392,1.0682193031123776e-06,0.0205522009258857,0.0,0.0,0.0,0.0,0.00070628455869155,320,40,53.82054615020752,1.345513653755188,0.1681892067193985,0.08604398613679223,0.0009244225264410488,0.06421275703937682,1.4013101122145599e-06,0.04732898417860269,0.0,0.0,0.0,0.0,0.0009358287643408403,80,10,10.352828025817871,1.035282802581787,0.12941035032272338,0.08558480869978666 +11,0.0015980420044797937,0.5406122412953664,1.8725383334045098e-06,0.032535753858974205,0.0,0.0,0.0,0.0,0.0016200240766920614,320,40,52.21665906906128,1.305416476726532,0.1631770595908165,0.09230114514939487,0.002002889267168939,0.0061411346425302325,1.0156804978578294e-05,0.04757400024682283,0.0,0.0,0.0,0.0,0.0020213359617628156,80,10,9.674538373947144,0.9674538373947144,0.1209317296743393,0.10838058441877366 +12,0.0017209277200890937,1.178802874633203,1.6148575717713204e-06,0.036564521375112236,0.0,0.0,0.0,0.0,0.0017370989189657848,320,40,52.85961055755615,1.321490263938904,0.165186282992363,0.08622921356000006,0.0022714424470905215,0.13002902529697166,3.263659819907616e-06,0.057125764340162276,0.0,0.0,0.0,0.0,0.0022938565700314937,80,10,9.738993883132935,0.9738993883132935,0.12173742353916168,0.10313290562480688 +13,0.0012762842405209085,0.2588888414171834,1.5229265101335266e-06,0.04046122131403536,0.0,0.0,0.0,0.0,0.0012891990583739244,320,40,52.36062669754028,1.3090156674385072,0.1636269584298134,0.08853817647323012,0.001091636417550035,0.09408267263497691,1.7059007003439318e-06,0.029268642514944078,0.0,0.0,0.0,0.0,0.0011036358628189191,80,10,9.722130298614502,0.9722130298614502,0.12152662873268127,0.09569714600220322 +14,0.0019031960193387932,0.7488255576945889,3.0515770202299565e-06,0.038308367284480484,0.0,0.0,0.0,0.0,0.0019228730783652282,320,40,52.47055411338806,1.3117638528347015,0.1639704816043377,0.09056595326401293,0.0037872558576054873,0.9208965854093549,6.925271605950911e-06,0.0315930200740695,0.0,0.0,0.0,0.0,0.00382659753668122,80,10,10.571656942367554,1.0571656942367553,0.13214571177959442,0.063721539452672 +15,0.0013684824078154633,0.6904069223831811,8.799437353642324e-07,0.026554493221919984,0.0,0.0,0.0,0.0,0.0013836498705131818,320,40,53.85569667816162,1.3463924169540404,0.16829905211925505,0.09258326740236952,0.0008738325996091589,0.6486218484133133,1.909309010628135e-06,0.02152107618749142,0.0,0.0,0.0,0.0,0.00088074420200428,80,10,9.91654920578003,0.991654920578003,0.12395686507225037,0.07936803023330867 +16,0.005297215489918017,5.37764993282708,1.1892299855552019e-05,0.05588673622114584,0.0,0.0,0.0,0.0,0.005385978057893226,320,40,52.9380304813385,1.3234507620334626,0.16543134525418282,0.07217288188985549,0.003967790375463664,2.064500455884263,3.926296935219398e-06,0.025678274407982826,0.0,0.0,0.0,0.0,0.00401492468372453,80,10,9.976269960403442,0.9976269960403442,0.12470337450504303,0.048923579324036834 +17,0.0015846264206629713,1.2480987950442795,5.060022944709808e-06,0.03413833014201373,0.0,0.0,0.0,0.0,0.0016031169310736005,320,40,52.37343645095825,1.3093359112739562,0.16366698890924453,0.08430429067229853,0.0009885555627988651,1.2472513985600018,2.0395318431324676e-06,0.022923901677131653,0.0,0.0,0.0,0.0,0.000998050649650395,80,10,10.16307783126831,1.016307783126831,0.12703847289085388,0.08072445834986866 +18,0.0009306106423537131,0.6546383185190479,1.0302065916178993e-06,0.027319350809557365,0.0,0.0,0.0,0.0,0.0009397507692483487,320,40,53.383638858795166,1.3345909714698792,0.1668238714337349,0.08970151729881763,0.0007668180856853724,0.5179583482356975,9.070707868374938e-07,0.01832481948658824,0.0,0.0,0.0,0.0,0.0007744630362140015,80,10,9.982792615890503,0.9982792615890503,0.12478490769863129,0.08095776736736297 +19,0.0012491234574554255,0.5037455438789038,2.073498104799426e-06,0.02705336583312601,0.0,0.0,0.0,0.0,0.0012609064433490857,320,40,52.2404465675354,1.306011164188385,0.16325139552354812,0.08776604899903759,0.001464261399814859,0.010546232295291702,5.130721780943759e-06,0.039135200902819633,0.0,0.0,0.0,0.0,0.0014782442012801766,80,10,10.231428146362305,1.0231428146362305,0.1278928518295288,0.09019482526928187 diff --git a/insurance/tvae/mlu-eval.ipynb b/insurance/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..a70ca89f375e75019b9449cd4bea2cc8437a4a36 --- /dev/null +++ b/insurance/tvae/mlu-eval.ipynb @@ -0,0 +1,2582 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:09.291178Z", + "iopub.status.busy": "2024-02-29T02:48:09.290859Z", + "iopub.status.idle": "2024-02-29T02:48:09.325349Z", + "shell.execute_reply": "2024-02-29T02:48:09.324644Z" + }, + "papermill": { + "duration": 0.049609, + "end_time": "2024-02-29T02:48:09.327586", + "exception": false, + "start_time": "2024-02-29T02:48:09.277977", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:09.354201Z", + "iopub.status.busy": "2024-02-29T02:48:09.353322Z", + "iopub.status.idle": "2024-02-29T02:48:09.361172Z", + "shell.execute_reply": "2024-02-29T02:48:09.360270Z" + }, + "papermill": { + "duration": 0.023292, + "end_time": "2024-02-29T02:48:09.363197", + "exception": false, + "start_time": "2024-02-29T02:48:09.339905", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:09.387045Z", + "iopub.status.busy": "2024-02-29T02:48:09.386776Z", + "iopub.status.idle": "2024-02-29T02:48:09.391031Z", + "shell.execute_reply": "2024-02-29T02:48:09.390098Z" + }, + "papermill": { + "duration": 0.018572, + "end_time": "2024-02-29T02:48:09.393062", + "exception": false, + "start_time": "2024-02-29T02:48:09.374490", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:09.416877Z", + "iopub.status.busy": "2024-02-29T02:48:09.416575Z", + "iopub.status.idle": "2024-02-29T02:48:09.420802Z", + "shell.execute_reply": "2024-02-29T02:48:09.419881Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018507, + "end_time": "2024-02-29T02:48:09.422715", + "exception": false, + "start_time": "2024-02-29T02:48:09.404208", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:09.446886Z", + "iopub.status.busy": "2024-02-29T02:48:09.446566Z", + "iopub.status.idle": "2024-02-29T02:48:09.451973Z", + "shell.execute_reply": "2024-02-29T02:48:09.451076Z" + }, + "papermill": { + "duration": 0.020112, + "end_time": "2024-02-29T02:48:09.453972", + "exception": false, + "start_time": "2024-02-29T02:48:09.433860", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "1044138b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:09.481278Z", + "iopub.status.busy": "2024-02-29T02:48:09.480434Z", + "iopub.status.idle": "2024-02-29T02:48:09.485464Z", + "shell.execute_reply": "2024-02-29T02:48:09.484660Z" + }, + "papermill": { + "duration": 0.021092, + "end_time": "2024-02-29T02:48:09.487458", + "exception": false, + "start_time": "2024-02-29T02:48:09.466366", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"tvae\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/tvae/4\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011055, + "end_time": "2024-02-29T02:48:09.509920", + "exception": false, + "start_time": "2024-02-29T02:48:09.498865", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:09.534384Z", + "iopub.status.busy": "2024-02-29T02:48:09.533654Z", + "iopub.status.idle": "2024-02-29T02:48:09.542924Z", + "shell.execute_reply": "2024-02-29T02:48:09.542091Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023648, + "end_time": "2024-02-29T02:48:09.544914", + "exception": false, + "start_time": "2024-02-29T02:48:09.521266", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/tvae/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:09.569012Z", + "iopub.status.busy": "2024-02-29T02:48:09.568719Z", + "iopub.status.idle": "2024-02-29T02:48:11.784312Z", + "shell.execute_reply": "2024-02-29T02:48:11.783359Z" + }, + "papermill": { + "duration": 2.230021, + "end_time": "2024-02-29T02:48:11.786379", + "exception": false, + "start_time": "2024-02-29T02:48:09.556358", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:11.813553Z", + "iopub.status.busy": "2024-02-29T02:48:11.813089Z", + "iopub.status.idle": "2024-02-29T02:48:11.825078Z", + "shell.execute_reply": "2024-02-29T02:48:11.824290Z" + }, + "papermill": { + "duration": 0.028244, + "end_time": "2024-02-29T02:48:11.827063", + "exception": false, + "start_time": "2024-02-29T02:48:11.798819", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:11.851770Z", + "iopub.status.busy": "2024-02-29T02:48:11.851434Z", + "iopub.status.idle": "2024-02-29T02:48:11.858808Z", + "shell.execute_reply": "2024-02-29T02:48:11.858079Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.022259, + "end_time": "2024-02-29T02:48:11.860860", + "exception": false, + "start_time": "2024-02-29T02:48:11.838601", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:11.885414Z", + "iopub.status.busy": "2024-02-29T02:48:11.885122Z", + "iopub.status.idle": "2024-02-29T02:48:11.990199Z", + "shell.execute_reply": "2024-02-29T02:48:11.989131Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.120368, + "end_time": "2024-02-29T02:48:11.992677", + "exception": false, + "start_time": "2024-02-29T02:48:11.872309", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:12.019398Z", + "iopub.status.busy": "2024-02-29T02:48:12.019089Z", + "iopub.status.idle": "2024-02-29T02:48:16.746783Z", + "shell.execute_reply": "2024-02-29T02:48:16.746003Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.743956, + "end_time": "2024-02-29T02:48:16.749295", + "exception": false, + "start_time": "2024-02-29T02:48:12.005339", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 02:48:14.309174: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 02:48:14.309236: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 02:48:14.310961: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:16.775383Z", + "iopub.status.busy": "2024-02-29T02:48:16.774769Z", + "iopub.status.idle": "2024-02-29T02:48:16.782049Z", + "shell.execute_reply": "2024-02-29T02:48:16.781197Z" + }, + "papermill": { + "duration": 0.02223, + "end_time": "2024-02-29T02:48:16.784092", + "exception": false, + "start_time": "2024-02-29T02:48:16.761862", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:16.811497Z", + "iopub.status.busy": "2024-02-29T02:48:16.811085Z", + "iopub.status.idle": "2024-02-29T02:48:25.988742Z", + "shell.execute_reply": "2024-02-29T02:48:25.987725Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 9.195019, + "end_time": "2024-02-29T02:48:25.991498", + "exception": false, + "start_time": "2024-02-29T02:48:16.796479", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BEST,\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:26.513908Z", + "iopub.status.busy": "2024-02-29T02:48:26.513602Z", + "iopub.status.idle": "2024-02-29T02:48:26.583915Z", + "shell.execute_reply": "2024-02-29T02:48:26.582945Z" + }, + "papermill": { + "duration": 0.086186, + "end_time": "2024-02-29T02:48:26.586171", + "exception": false, + "start_time": "2024-02-29T02:48:26.499985", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../insurance/_cache/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache4/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/insurance [80, 20]\n", + "Caching in ../../../../insurance/_cache5/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/insurance [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T02:48:26.616414Z", + "iopub.status.busy": "2024-02-29T02:48:26.616082Z", + "iopub.status.idle": "2024-02-29T02:48:27.058820Z", + "shell.execute_reply": "2024-02-29T02:48:27.057866Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.459857, + "end_time": "2024-02-29T02:48:27.060892", + "exception": false, + "start_time": "2024-02-29T02:48:26.601035", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:27.092139Z", + "iopub.status.busy": "2024-02-29T02:48:27.091806Z", + "iopub.status.idle": "2024-02-29T02:48:27.095944Z", + "shell.execute_reply": "2024-02-29T02:48:27.095153Z" + }, + "papermill": { + "duration": 0.022738, + "end_time": "2024-02-29T02:48:27.097962", + "exception": false, + "start_time": "2024-02-29T02:48:27.075224", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:27.125523Z", + "iopub.status.busy": "2024-02-29T02:48:27.125233Z", + "iopub.status.idle": "2024-02-29T02:48:27.132241Z", + "shell.execute_reply": "2024-02-29T02:48:27.131360Z" + }, + "papermill": { + "duration": 0.023266, + "end_time": "2024-02-29T02:48:27.134176", + "exception": false, + "start_time": "2024-02-29T02:48:27.110910", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9638537" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:27.161638Z", + "iopub.status.busy": "2024-02-29T02:48:27.161329Z", + "iopub.status.idle": "2024-02-29T02:48:27.251186Z", + "shell.execute_reply": "2024-02-29T02:48:27.250270Z" + }, + "papermill": { + "duration": 0.106066, + "end_time": "2024-02-29T02:48:27.253292", + "exception": false, + "start_time": "2024-02-29T02:48:27.147226", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 36] --\n", + "├─Adapter: 1-1 [2, 1071, 36] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 37,888\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 36] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,638,537\n", + "Trainable params: 9,638,537\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.18\n", + "========================================================================================================================\n", + "Input size (MB): 0.39\n", + "Forward/backward pass size (MB): 307.49\n", + "Params size (MB): 38.55\n", + "Estimated Total Size (MB): 346.43\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:48:27.286078Z", + "iopub.status.busy": "2024-02-29T02:48:27.285669Z", + "iopub.status.idle": "2024-02-29T03:11:36.175645Z", + "shell.execute_reply": "2024-02-29T03:11:36.174662Z" + }, + "papermill": { + "duration": 1388.927787, + "end_time": "2024-02-29T03:11:36.195877", + "exception": false, + "start_time": "2024-02-29T02:48:27.268090", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.031827369896927846, 'avg_role_model_std_loss': 28.259707012480977, 'avg_role_model_mean_pred_loss': 0.004191242665176187, 'avg_role_model_g_mag_loss': 0.2307685674633831, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0321838908130303, 'n_size': 320, 'n_batch': 40, 'duration': 54.558457136154175, 'duration_batch': 1.3639614284038544, 'duration_size': 0.1704951785504818, 'avg_pred_std': 0.05414983599475818}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010481808317126706, 'avg_role_model_std_loss': 6.6735696544030105, 'avg_role_model_mean_pred_loss': 2.9961921549848113e-05, 'avg_role_model_g_mag_loss': 0.0279149753972888, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010719987776246854, 'n_size': 80, 'n_batch': 10, 'duration': 10.396106243133545, 'duration_batch': 1.0396106243133545, 'duration_size': 0.1299513280391693, 'avg_pred_std': 0.02609058879315853}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.011675631045363843, 'avg_role_model_std_loss': 18.501674464043028, 'avg_role_model_mean_pred_loss': 0.0009566789830486755, 'avg_role_model_g_mag_loss': 0.09624193962663412, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011828435189090668, 'n_size': 320, 'n_batch': 40, 'duration': 55.64720034599304, 'duration_batch': 1.391180008649826, 'duration_size': 0.17389750108122826, 'avg_pred_std': 0.08122157981561032}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004508110485039652, 'avg_role_model_std_loss': 22.331014108657836, 'avg_role_model_mean_pred_loss': 2.6730329889979034e-06, 'avg_role_model_g_mag_loss': 0.04836642872542143, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004554628022015094, 'n_size': 80, 'n_batch': 10, 'duration': 10.792717218399048, 'duration_batch': 1.0792717218399048, 'duration_size': 0.1349089652299881, 'avg_pred_std': 0.03636421860428527}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0035327200530446135, 'avg_role_model_std_loss': 9.784873699297304, 'avg_role_model_mean_pred_loss': 7.98118183580887e-06, 'avg_role_model_g_mag_loss': 0.0508232937194407, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0035692405959707684, 'n_size': 320, 'n_batch': 40, 'duration': 55.71424317359924, 'duration_batch': 1.3928560793399811, 'duration_size': 0.17410700991749764, 'avg_pred_std': 0.07611006779770832}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0014172189374221488, 'avg_role_model_std_loss': 0.09624732142401626, 'avg_role_model_mean_pred_loss': 6.146688699448788e-06, 'avg_role_model_g_mag_loss': 0.020977134071290492, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014305356336990372, 'n_size': 80, 'n_batch': 10, 'duration': 10.833696365356445, 'duration_batch': 1.0833696365356444, 'duration_size': 0.13542120456695556, 'avg_pred_std': 0.08805033396929503}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0020708162381197328, 'avg_role_model_std_loss': 2.6992595067491036, 'avg_role_model_mean_pred_loss': 7.111737180576228e-06, 'avg_role_model_g_mag_loss': 0.04061916306382045, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002090448179660598, 'n_size': 320, 'n_batch': 40, 'duration': 55.8498101234436, 'duration_batch': 1.39624525308609, 'duration_size': 0.17453065663576126, 'avg_pred_std': 0.08638362209312618}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00137065484886989, 'avg_role_model_std_loss': 0.005912328146041546, 'avg_role_model_mean_pred_loss': 4.321905053217279e-06, 'avg_role_model_g_mag_loss': 0.02791164191439748, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013822964770952239, 'n_size': 80, 'n_batch': 10, 'duration': 10.717517137527466, 'duration_batch': 1.0717517137527466, 'duration_size': 0.13396896421909332, 'avg_pred_std': 0.09996146205812692}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014275090521550736, 'avg_role_model_std_loss': 0.9193552540385213, 'avg_role_model_mean_pred_loss': 1.838910397943677e-06, 'avg_role_model_g_mag_loss': 0.02831868330249563, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014419240753341, 'n_size': 320, 'n_batch': 40, 'duration': 55.6291663646698, 'duration_batch': 1.390729159116745, 'duration_size': 0.17384114488959312, 'avg_pred_std': 0.08760609640739858}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0012192766880616546, 'avg_role_model_std_loss': 0.21555799300662254, 'avg_role_model_mean_pred_loss': 1.798611496894864e-06, 'avg_role_model_g_mag_loss': 0.03309073373675346, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001231387781444937, 'n_size': 80, 'n_batch': 10, 'duration': 10.670984268188477, 'duration_batch': 1.0670984268188477, 'duration_size': 0.13338730335235596, 'avg_pred_std': 0.08692563734948636}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017993727393331937, 'avg_role_model_std_loss': 0.8518799075905094, 'avg_role_model_mean_pred_loss': 2.7848555921972105e-06, 'avg_role_model_g_mag_loss': 0.03595072412863374, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018162189619033598, 'n_size': 320, 'n_batch': 40, 'duration': 55.764097690582275, 'duration_batch': 1.394102442264557, 'duration_size': 0.17426280528306962, 'avg_pred_std': 0.08750550604891032}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0010832886851858347, 'avg_role_model_std_loss': 0.02544177576819493, 'avg_role_model_mean_pred_loss': 1.465394489699734e-06, 'avg_role_model_g_mag_loss': 0.026945540122687815, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001093548518838361, 'n_size': 80, 'n_batch': 10, 'duration': 10.761934995651245, 'duration_batch': 1.0761934995651246, 'duration_size': 0.13452418744564057, 'avg_pred_std': 0.0844166411086917}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011671951153402916, 'avg_role_model_std_loss': 0.5874403900238576, 'avg_role_model_mean_pred_loss': 1.4499453566324537e-06, 'avg_role_model_g_mag_loss': 0.027735682530328633, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011789663760282565, 'n_size': 320, 'n_batch': 40, 'duration': 53.97340989112854, 'duration_batch': 1.3493352472782134, 'duration_size': 0.16866690590977668, 'avg_pred_std': 0.09012857411289588}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005621012981282547, 'avg_role_model_std_loss': 1.7633854886284097, 'avg_role_model_mean_pred_loss': 6.435126891801701e-06, 'avg_role_model_g_mag_loss': 0.026362080965191124, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005691835869220086, 'n_size': 80, 'n_batch': 10, 'duration': 10.383732795715332, 'duration_batch': 1.0383732795715332, 'duration_size': 0.12979665994644166, 'avg_pred_std': 0.046620757598429916}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0026447449334227715, 'avg_role_model_std_loss': 1.1427540660426474, 'avg_role_model_mean_pred_loss': 2.4505569594213126e-06, 'avg_role_model_g_mag_loss': 0.03827556688338518, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026816512978257378, 'n_size': 320, 'n_batch': 40, 'duration': 54.003010749816895, 'duration_batch': 1.3500752687454223, 'duration_size': 0.16875940859317778, 'avg_pred_std': 0.07941703633405268}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0011788314557634294, 'avg_role_model_std_loss': 0.005349384507599098, 'avg_role_model_mean_pred_loss': 2.9599219267595344e-06, 'avg_role_model_g_mag_loss': 0.03608495369553566, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011917953903321176, 'n_size': 80, 'n_batch': 10, 'duration': 10.279337406158447, 'duration_batch': 1.0279337406158446, 'duration_size': 0.12849171757698058, 'avg_pred_std': 0.10001012030988932}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016346701009751995, 'avg_role_model_std_loss': 0.718084112559017, 'avg_role_model_mean_pred_loss': 3.656406406381263e-06, 'avg_role_model_g_mag_loss': 0.039087726082652804, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016500235098646954, 'n_size': 320, 'n_batch': 40, 'duration': 53.33782744407654, 'duration_batch': 1.3334456861019135, 'duration_size': 0.1666807107627392, 'avg_pred_std': 0.08958328987937421}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0010581719019683079, 'avg_role_model_std_loss': 0.003487239831883926, 'avg_role_model_mean_pred_loss': 4.90262124515084e-06, 'avg_role_model_g_mag_loss': 0.0341863420791924, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010685857152566314, 'n_size': 80, 'n_batch': 10, 'duration': 9.576281309127808, 'duration_batch': 0.9576281309127808, 'duration_size': 0.1197035163640976, 'avg_pred_std': 0.0981753658503294}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013097766401187982, 'avg_role_model_std_loss': 0.6380827577727615, 'avg_role_model_mean_pred_loss': 1.4933229921021417e-06, 'avg_role_model_g_mag_loss': 0.03195373273920268, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013226776201918256, 'n_size': 320, 'n_batch': 40, 'duration': 52.700077295303345, 'duration_batch': 1.3175019323825836, 'duration_size': 0.16468774154782295, 'avg_pred_std': 0.08942489377222955}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0010866674332646654, 'avg_role_model_std_loss': 0.00882318691146793, 'avg_role_model_mean_pred_loss': 4.24079198637628e-06, 'avg_role_model_g_mag_loss': 0.02658762000501156, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010967145411996172, 'n_size': 80, 'n_batch': 10, 'duration': 9.853407621383667, 'duration_batch': 0.9853407621383667, 'duration_size': 0.12316759526729584, 'avg_pred_std': 0.09188855718821287}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006990553600189741, 'avg_role_model_std_loss': 1.061739949541392, 'avg_role_model_mean_pred_loss': 1.0682193031123776e-06, 'avg_role_model_g_mag_loss': 0.0205522009258857, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00070628455869155, 'n_size': 320, 'n_batch': 40, 'duration': 53.82054615020752, 'duration_batch': 1.345513653755188, 'duration_size': 0.1681892067193985, 'avg_pred_std': 0.08604398613679223}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0009244225264410488, 'avg_role_model_std_loss': 0.06421275703937682, 'avg_role_model_mean_pred_loss': 1.4013101122145599e-06, 'avg_role_model_g_mag_loss': 0.04732898417860269, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009358287643408403, 'n_size': 80, 'n_batch': 10, 'duration': 10.352828025817871, 'duration_batch': 1.035282802581787, 'duration_size': 0.12941035032272338, 'avg_pred_std': 0.08558480869978666}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015980420044797937, 'avg_role_model_std_loss': 0.5406122412953664, 'avg_role_model_mean_pred_loss': 1.8725383334045098e-06, 'avg_role_model_g_mag_loss': 0.032535753858974205, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016200240766920614, 'n_size': 320, 'n_batch': 40, 'duration': 52.21665906906128, 'duration_batch': 1.305416476726532, 'duration_size': 0.1631770595908165, 'avg_pred_std': 0.09230114514939487}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002002889267168939, 'avg_role_model_std_loss': 0.0061411346425302325, 'avg_role_model_mean_pred_loss': 1.0156804978578294e-05, 'avg_role_model_g_mag_loss': 0.04757400024682283, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0020213359617628156, 'n_size': 80, 'n_batch': 10, 'duration': 9.674538373947144, 'duration_batch': 0.9674538373947144, 'duration_size': 0.1209317296743393, 'avg_pred_std': 0.10838058441877366}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017209277200890937, 'avg_role_model_std_loss': 1.178802874633203, 'avg_role_model_mean_pred_loss': 1.6148575717713204e-06, 'avg_role_model_g_mag_loss': 0.036564521375112236, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017370989189657848, 'n_size': 320, 'n_batch': 40, 'duration': 52.85961055755615, 'duration_batch': 1.321490263938904, 'duration_size': 0.165186282992363, 'avg_pred_std': 0.08622921356000006}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022714424470905215, 'avg_role_model_std_loss': 0.13002902529697166, 'avg_role_model_mean_pred_loss': 3.263659819907616e-06, 'avg_role_model_g_mag_loss': 0.057125764340162276, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022938565700314937, 'n_size': 80, 'n_batch': 10, 'duration': 9.738993883132935, 'duration_batch': 0.9738993883132935, 'duration_size': 0.12173742353916168, 'avg_pred_std': 0.10313290562480688}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012762842405209085, 'avg_role_model_std_loss': 0.2588888414171834, 'avg_role_model_mean_pred_loss': 1.5229265101335266e-06, 'avg_role_model_g_mag_loss': 0.04046122131403536, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012891990583739244, 'n_size': 320, 'n_batch': 40, 'duration': 52.36062669754028, 'duration_batch': 1.3090156674385072, 'duration_size': 0.1636269584298134, 'avg_pred_std': 0.08853817647323012}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001091636417550035, 'avg_role_model_std_loss': 0.09408267263497691, 'avg_role_model_mean_pred_loss': 1.7059007003439318e-06, 'avg_role_model_g_mag_loss': 0.029268642514944078, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011036358628189191, 'n_size': 80, 'n_batch': 10, 'duration': 9.722130298614502, 'duration_batch': 0.9722130298614502, 'duration_size': 0.12152662873268127, 'avg_pred_std': 0.09569714600220322}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0019031960193387932, 'avg_role_model_std_loss': 0.7488255576945889, 'avg_role_model_mean_pred_loss': 3.0515770202299565e-06, 'avg_role_model_g_mag_loss': 0.038308367284480484, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019228730783652282, 'n_size': 320, 'n_batch': 40, 'duration': 52.47055411338806, 'duration_batch': 1.3117638528347015, 'duration_size': 0.1639704816043377, 'avg_pred_std': 0.09056595326401293}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0037872558576054873, 'avg_role_model_std_loss': 0.9208965854093549, 'avg_role_model_mean_pred_loss': 6.925271605950911e-06, 'avg_role_model_g_mag_loss': 0.0315930200740695, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00382659753668122, 'n_size': 80, 'n_batch': 10, 'duration': 10.571656942367554, 'duration_batch': 1.0571656942367553, 'duration_size': 0.13214571177959442, 'avg_pred_std': 0.063721539452672}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013684824078154633, 'avg_role_model_std_loss': 0.6904069223831811, 'avg_role_model_mean_pred_loss': 8.799437353642324e-07, 'avg_role_model_g_mag_loss': 0.026554493221919984, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013836498705131818, 'n_size': 320, 'n_batch': 40, 'duration': 53.85569667816162, 'duration_batch': 1.3463924169540404, 'duration_size': 0.16829905211925505, 'avg_pred_std': 0.09258326740236952}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008738325996091589, 'avg_role_model_std_loss': 0.6486218484133133, 'avg_role_model_mean_pred_loss': 1.909309010628135e-06, 'avg_role_model_g_mag_loss': 0.02152107618749142, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00088074420200428, 'n_size': 80, 'n_batch': 10, 'duration': 9.91654920578003, 'duration_batch': 0.991654920578003, 'duration_size': 0.12395686507225037, 'avg_pred_std': 0.07936803023330867}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005297215489918017, 'avg_role_model_std_loss': 5.37764993282708, 'avg_role_model_mean_pred_loss': 1.1892299855552019e-05, 'avg_role_model_g_mag_loss': 0.05588673622114584, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005385978057893226, 'n_size': 320, 'n_batch': 40, 'duration': 52.9380304813385, 'duration_batch': 1.3234507620334626, 'duration_size': 0.16543134525418282, 'avg_pred_std': 0.07217288188985549}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003967790375463664, 'avg_role_model_std_loss': 2.064500455884263, 'avg_role_model_mean_pred_loss': 3.926296935219398e-06, 'avg_role_model_g_mag_loss': 0.025678274407982826, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00401492468372453, 'n_size': 80, 'n_batch': 10, 'duration': 9.976269960403442, 'duration_batch': 0.9976269960403442, 'duration_size': 0.12470337450504303, 'avg_pred_std': 0.048923579324036834}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015846264206629713, 'avg_role_model_std_loss': 1.2480987950442795, 'avg_role_model_mean_pred_loss': 5.060022944709808e-06, 'avg_role_model_g_mag_loss': 0.03413833014201373, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016031169310736005, 'n_size': 320, 'n_batch': 40, 'duration': 52.37343645095825, 'duration_batch': 1.3093359112739562, 'duration_size': 0.16366698890924453, 'avg_pred_std': 0.08430429067229853}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0009885555627988651, 'avg_role_model_std_loss': 1.2472513985600018, 'avg_role_model_mean_pred_loss': 2.0395318431324676e-06, 'avg_role_model_g_mag_loss': 0.022923901677131653, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000998050649650395, 'n_size': 80, 'n_batch': 10, 'duration': 10.16307783126831, 'duration_batch': 1.016307783126831, 'duration_size': 0.12703847289085388, 'avg_pred_std': 0.08072445834986866}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009306106423537131, 'avg_role_model_std_loss': 0.6546383185190479, 'avg_role_model_mean_pred_loss': 1.0302065916178993e-06, 'avg_role_model_g_mag_loss': 0.027319350809557365, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009397507692483487, 'n_size': 320, 'n_batch': 40, 'duration': 53.383638858795166, 'duration_batch': 1.3345909714698792, 'duration_size': 0.1668238714337349, 'avg_pred_std': 0.08970151729881763}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0007668180856853724, 'avg_role_model_std_loss': 0.5179583482356975, 'avg_role_model_mean_pred_loss': 9.070707868374938e-07, 'avg_role_model_g_mag_loss': 0.01832481948658824, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007744630362140015, 'n_size': 80, 'n_batch': 10, 'duration': 9.982792615890503, 'duration_batch': 0.9982792615890503, 'duration_size': 0.12478490769863129, 'avg_pred_std': 0.08095776736736297}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012491234574554255, 'avg_role_model_std_loss': 0.5037455438789038, 'avg_role_model_mean_pred_loss': 2.073498104799426e-06, 'avg_role_model_g_mag_loss': 0.02705336583312601, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012609064433490857, 'n_size': 320, 'n_batch': 40, 'duration': 52.2404465675354, 'duration_batch': 1.306011164188385, 'duration_size': 0.16325139552354812, 'avg_pred_std': 0.08776604899903759}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001464261399814859, 'avg_role_model_std_loss': 0.010546232295291702, 'avg_role_model_mean_pred_loss': 5.130721780943759e-06, 'avg_role_model_g_mag_loss': 0.039135200902819633, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014782442012801766, 'n_size': 80, 'n_batch': 10, 'duration': 10.231428146362305, 'duration_batch': 1.0231428146362305, 'duration_size': 0.1278928518295288, 'avg_pred_std': 0.09019482526928187}\n", + "Stopped False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▄▁▁▁▁▄▁▁▁▁▂▂▁▃▁▃▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▂▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▁▂▆▇▆▆▃▇▇▇▆██▇▄▆▃▆▆▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▁▆▅▇▇▇█▆▇▇▇█▇▇██▄▆▇▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▃▆▁▃▄▃▂▄▄▂▆▆█▃▃▂▂▂▁▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train █▄▂▂▁▂▁▂▂▁▁▁▂▂▂▁▂▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▄▁▁▁▁▄▁▁▁▁▂▂▁▃▁▃▁▁▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▂▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▁▂▂▁▁▂▁▂▂▁▃▂▁▂▁▂▁▁▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▃▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▃█▁▁▁▁▂▁▁▁▁▁▁▁▁▁▂▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train █▆▃▂▁▁▁▁▁▁▁▁▁▁▁▁▂▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▆██▇▇█▅▅▁▃▅▂▂▂▇▃▃▄▃▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▆█████▄▄▃▂▄▁▂▁▁▄▂▁▃▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▆██▇▇█▅▅▁▃▅▂▂▂▇▃▃▄▃▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▆█████▄▄▃▂▄▁▂▁▁▄▂▁▃▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▆██▇▇█▅▅▁▃▅▂▂▂▇▃▃▄▃▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▆█████▄▄▃▂▄▁▂▁▁▄▂▁▃▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00148\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00126\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.09019\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.08777\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.03914\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.02705\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00146\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00125\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 1e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 0.01055\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.50375\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 1.02314\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.30601\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.12789\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.16325\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 10.23143\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 52.24045\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 10\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/insurance/tvae/4/wandb/offline-run-20240229_024828-huiz05ix\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_024828-huiz05ix/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 399, 'n_batch': 50, 'role_model_metrics': {'avg_loss': 0.00041556872023657004, 'avg_g_mag_loss': 0.04719940199397672, 'avg_g_cos_loss': 0.07775440687000901, 'pred_duration': 0.913799524307251, 'grad_duration': 2.4363996982574463, 'total_duration': 3.3501992225646973, 'pred_std': 0.15985159575939178, 'std_loss': 1.6569742001593113e-05, 'mean_pred_loss': 8.493182690472167e-08, 'pred_rmse': 0.0203855037689209, 'pred_mae': 0.01549115777015686, 'pred_mape': 0.1724841296672821, 'grad_rmse': 0.03157908841967583, 'grad_mae': 0.02223225124180317, 'grad_mape': 0.6888606548309326}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.00041556872023657004, 'avg_g_mag_loss': 0.04719940199397672, 'avg_g_cos_loss': 0.07775440687000901, 'avg_pred_duration': 0.913799524307251, 'avg_grad_duration': 2.4363996982574463, 'avg_total_duration': 3.3501992225646973, 'avg_pred_std': 0.15985159575939178, 'avg_std_loss': 1.6569742001593113e-05, 'avg_mean_pred_loss': 8.493182690472167e-08}, 'min_metrics': {'avg_loss': 0.00041556872023657004, 'avg_g_mag_loss': 0.04719940199397672, 'avg_g_cos_loss': 0.07775440687000901, 'pred_duration': 0.913799524307251, 'grad_duration': 2.4363996982574463, 'total_duration': 3.3501992225646973, 'pred_std': 0.15985159575939178, 'std_loss': 1.6569742001593113e-05, 'mean_pred_loss': 8.493182690472167e-08, 'pred_rmse': 0.0203855037689209, 'pred_mae': 0.01549115777015686, 'pred_mape': 0.1724841296672821, 'grad_rmse': 0.03157908841967583, 'grad_mae': 0.02223225124180317, 'grad_mape': 0.6888606548309326}, 'model_metrics': {'tvae': {'avg_loss': 0.00041556872023657004, 'avg_g_mag_loss': 0.04719940199397672, 'avg_g_cos_loss': 0.07775440687000901, 'pred_duration': 0.913799524307251, 'grad_duration': 2.4363996982574463, 'total_duration': 3.3501992225646973, 'pred_std': 0.15985159575939178, 'std_loss': 1.6569742001593113e-05, 'mean_pred_loss': 8.493182690472167e-08, 'pred_rmse': 0.0203855037689209, 'pred_mae': 0.01549115777015686, 'pred_mape': 0.1724841296672821, 'grad_rmse': 0.03157908841967583, 'grad_mae': 0.02223225124180317, 'grad_mape': 0.6888606548309326}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:11:36.236551Z", + "iopub.status.busy": "2024-02-29T03:11:36.236167Z", + "iopub.status.idle": "2024-02-29T03:11:36.240591Z", + "shell.execute_reply": "2024-02-29T03:11:36.239799Z" + }, + "papermill": { + "duration": 0.026275, + "end_time": "2024-02-29T03:11:36.242541", + "exception": false, + "start_time": "2024-02-29T03:11:36.216266", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:11:36.278455Z", + "iopub.status.busy": "2024-02-29T03:11:36.278211Z", + "iopub.status.idle": "2024-02-29T03:11:36.545294Z", + "shell.execute_reply": "2024-02-29T03:11:36.544407Z" + }, + "papermill": { + "duration": 0.287812, + "end_time": "2024-02-29T03:11:36.547751", + "exception": false, + "start_time": "2024-02-29T03:11:36.259939", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:11:36.587264Z", + "iopub.status.busy": "2024-02-29T03:11:36.586461Z", + "iopub.status.idle": "2024-02-29T03:11:36.860440Z", + "shell.execute_reply": "2024-02-29T03:11:36.859647Z" + }, + "papermill": { + "duration": 0.296207, + "end_time": "2024-02-29T03:11:36.862433", + "exception": false, + "start_time": "2024-02-29T03:11:36.566226", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFWklEQVR4nO3de1xUdf748dfMwHCVu1xF8BqmSF6CsIuVbFh2odws1tL8mVqrZbGV625K27bRzbLMzbXdamszzb7WtmaWoWYF3kDzbl5QSAVFBATkNvP5/XFkdHSAGUAH9P18PM6D4ZzPOedzBuY95/M5n4tOKaUQQogOQu/sDAghhCMkaAkhOhQJWkKIDkWClhCiQ5GgJYToUCRoCSE6FAlaQogOxcXZGbhYzGYzhw8fplOnTuh0OmdnRwhxDqUUJ0+eJDw8HL2+8fupyyZoHT58mMjISGdnQwjRjIKCArp06dLo9ssmaHXq1AnQ3hAfHx8n50YIca7y8nIiIyMtn9XGXDZBq6FI6OPjI0FLiHasueobqYgXQnQoErSEEB2KBC0hRIdy2dRpidYxmUzU1dU5OxuiA3N1dcVgMLT6OBK0RJOUUhQWFlJaWursrIhLgJ+fH6Ghoa1qKylBSzSpIWAFBwfj6ekpDXNFiyilqKqq4ujRowCEhYW1+FgStM5RdqqOnUfK0et0xHcLcHZ2nMpkMlkCVmBgoLOzIzo4Dw8PAI4ePUpwcHCLi4pSEX+OTfknuH/+WtK/3O7srDhdQx2Wp6enk3MiLhUN/0utqR+VoHWOQC83AEoqa5yck/ZDioSirbTF/5IErXMEeBsBKKmsReb8EKL9kaB1jkAvLWjVmRQna+qdnBshxLkkaJ3D3dWAp1GrICypqHVybkRHpdPp+OKLL5ydjTb13HPPcdVVVzk7GxK0bAk4fbd1vFKClui4PvjgA/z8/NrseE899RSZmZltdryWkqBlQ0MRsUSClrgM1Nba93/u7e3dLpq+SNCyIcAStOQJ4tmUUlTV1jtlcfShyPLly7nuuuvw8/MjMDCQ22+/nX379gEwZMgQpk2bZpX+2LFjuLq6smbNGgCOHDnCiBEj8PDwoFu3bixYsIDo6Ghmz57dovdu69at3HzzzXh4eBAYGMjEiROpqKiwbF+9ejXx8fF4eXnh5+fHtddey8GDBwH4+eefuemmm+jUqRM+Pj4MGjSIjRs3Nnm+1atXM27cOMrKytDpdOh0Op577jkAoqOj+etf/8qYMWPw8fFh4sSJAEybNo3evXvj6elJ9+7dmTFjhlXThHOLhw899BApKSm89tprhIWFERgYyOTJky94dy9pXGpDgKXZg/S1O9upOhNXzvzGKefe8Xwynkb7/10rKytJS0ujf//+VFRUMHPmTO6++242b97M6NGjeeWVV3jppZcsj+AXLVpEeHg4119/PQBjxoyhuLiY1atX4+rqSlpamqU1t6MqKytJTk4mMTGRDRs2cPToUR5++GGmTJnCBx98QH19PSkpKUyYMIFPPvmE2tpa1q9fb8nb6NGjGTBgAO+88w4Gg4HNmzfj6ura5DmHDBnC7NmzmTlzJrt37wa0O6UGr732GjNnziQ9Pd2yrlOnTnzwwQeEh4ezdetWJkyYQKdOnXjmmWcaPc+qVasICwtj1apV7N27l/vuu4+rrrqKCRMmtOi9socELRsCveVOq6MbOXKk1e/vvfcenTt3ZseOHYwaNYonnniCH3/80RKkFixYQGpqKjqdjl27dvHdd9+xYcMGBg8eDMA///lPevXq1aK8LFiwgOrqaj788EO8vLwAePvtt7njjjt4+eWXcXV1paysjNtvv50ePXoA0KdPH8v++fn5PP3008TExADYlQ+j0Yivry86nY7Q0NDztt9888384Q9/sFr37LPPWl5HR0fz1FNPsXDhwiaDlr+/P2+//TYGg4GYmBhGjBhBZmamBK2LTSribfNwNbDj+WSnndsRe/bsYebMmaxbt47i4mLMZjOgBYB+/fpxyy238PHHH3P99deTl5dHdnY2//jHPwDYvXs3Li4uDBw40HK8nj174u/v36K879y5k7i4OEvAArj22msxm83s3r2bG264gYceeojk5GR+85vfkJSUxKhRoyz989LS0nj44Yf56KOPSEpK4t5777UEt5ZqCMZnW7RoEW+99Rb79u2joqKC+vr6Zkf57du3r1V3nLCwMLZu3dqqvDVH6rRsCJCKeJt0Oh2eRhenLI62pL7jjjsoKSnh3XffZd26daxbtw44U+k8evRoPvvsM+rq6liwYAGxsbHExsa2+Xtmr/fff5/s7GyGDBnCokWL6N27N2vXrgW0uqTt27czYsQIVq5cyZVXXsnnn3/eqvOdHUABsrOzGT16NLfddhtLly5l06ZN/PnPf262kv7cYqpOp7N8QVwoLQpac+fOJTo6Gnd3dxISEli/fn2T6RcvXkxMTAzu7u7ExsaybNkyq+3PPfccMTExeHl54e/vT1JSkuWfrEFJSQmjR4/Gx8cHPz8/xo8fb1WR2Zbk6WHHdvz4cXbv3s2zzz7LsGHD6NOnDydOnLBKc9ddd1FdXc3y5ctZsGABo0ePtmy74oorqK+vZ9OmTZZ1e/fuPe8Y9urTpw8///wzlZWVlnU//fQTer2eK664wrJuwIABTJ8+naysLPr168eCBQss23r37s2TTz7Jt99+yz333MP777/f7HmNRiMmk8muPGZlZREVFcWf//xnBg8eTK9evSwPAtobh4PWokWLSEtLIz09ndzcXOLi4khOTm60kjIrK4vU1FTGjx/Ppk2bSElJISUlhW3btlnS9O7dm7fffputW7fy448/Eh0dzS233MKxY8csaUaPHs327dtZsWIFS5cuZc2aNZanHm3NUjyUxqUdkr+/P4GBgcyfP5+9e/eycuVK0tLSrNJ4eXmRkpLCjBkz2LlzJ6mpqZZtMTExJCUlMXHiRNavX8+mTZuYOHEiHh4eLeo7N3r0aNzd3Rk7dizbtm1j1apVPPbYYzz44IOEhISQl5fH9OnTyc7O5uDBg3z77bfs2bOHPn36cOrUKaZMmcLq1as5ePAgP/30Exs2bLCq82pMdHQ0FRUVZGZmUlxcTFVVVaNpe/XqRX5+PgsXLmTfvn289dZbrb6bu2CUg+Lj49XkyZMtv5tMJhUeHq4yMjJsph81apQaMWKE1bqEhAQ1adKkRs9RVlamAPXdd98ppZTasWOHAtSGDRssab7++mul0+nUoUOHbB6jurpalZWVWZaCggIFqLKysmav8WBxpYqatlTFPPt1s2kvZadOnVI7duxQp06dcnZWHLZixQrVp08f5ebmpvr3769Wr16tAPX5559b0ixbtkwB6oYbbjhv/8OHD6tbb71Vubm5qaioKLVgwQIVHBys5s2bZ9f5zz3Xli1b1E033aTc3d1VQECAmjBhgjp58qRSSqnCwkKVkpKiwsLClNFoVFFRUWrmzJnKZDKpmpoadf/996vIyEhlNBpVeHi4mjJlit1/k0ceeUQFBgYqQKWnpyullIqKilJvvPHGeWmffvppFRgYqLy9vdV9992n3njjDeXr62vZnp6eruLi4iy/jx07Vt11111Wx5g6daoaOnRoo/lp6n+q4XPf3GfUoaBVU1OjDAaD1R9DKaXGjBmj7rzzTpv7REZGnvcGzZw5U/Xv37/Rc7z66qvK19dXHTt2TCml1L/+9S/l5+dnla6urk4ZDAa1ZMkSm8dJT09XwHmLPUHrZHWdipq2VEVNW6qqauqbTX+p6shBq601fOk1fJGKlmmLoOVQ8bC4uBiTyURISIjV+pCQEAoLC23uU1hYaFf6pUuX4u3tjbu7O2+88QYrVqwgKCjIcozg4GCr9C4uLgQEBDR63unTp1NWVmZZCgoK7L5OL6MBo4v21hyXZg+XpZUrV/Lll1+Sl5dHVlYW999/P9HR0dxwww3Oztplr908PbzpppvYvHkzWVlZDB8+nFGjRrW4MR+Am5ubZWJWRydo1el0Uhl/maurq+NPf/oTffv25e6776Zz586WhqYff/wx3t7eNpe+fftetDzeeuutjebjxRdfvGj5uNgcaqcVFBSEwWCgqKjIan1RUZHNBmwAoaGhdqX38vKiZ8+e9OzZk2uuuYZevXrxr3/9i+nTpxMaGnpeAKuvr6ekpKTR87ZWgJeRI2XV0lbrMpWcnExysu02aXfeeScJCQk2tzXXUr0t/fOf/+TUqVM2twUEXLpDhTsUtIxGI4MGDSIzM5OUlBQAzGYzmZmZTJkyxeY+iYmJZGZm8sQTT1jWrVixgsTExCbPZTabqampsRyjtLSUnJwcBg0aBGi372azudF/ntaytNWSJ4jiHJ06daJTp07OzgYRERHOzoJTONwiPi0tjbFjxzJ48GDi4+OZPXs2lZWVjBs3DtD6bEVERJCRkQHA1KlTGTp0KLNmzWLEiBEsXLiQjRs3Mn/+fEDrl/W3v/2NO++8k7CwMIqLi5k7dy6HDh3i3nvvBbR2LsOHD2fChAnMmzePuro6pkyZwv333094eHhbvRdWpHgoRPvkcNC67777OHbsGDNnzqSwsJCrrrqK5cuXWyrb8/Pz0evPVJUNGTKEBQsW8Oyzz/KnP/2JXr168cUXX9CvXz8ADAYDu3bt4t///jfFxcUEBgZy9dVX88MPP1jVD3z88cdMmTKFYcOGodfrGTlyJG+99VZrr79R/tKVR4h2SafU5TEQenl5Ob6+vpSVldlVKf/2yj289u0vjBrchVd+G3cRctj+VFdXk5eXR7du3XB3d3d2dsQloKn/KXs/o+3m6WF7c2Z4GrnTEqI9kaDVCBnpQYj2SYJWIwK9pSJetNylOLFFeyFBqxHS5EF0dG09sQVowzjrdDpKS0vb9LiOkKDViIYmDydr6qmpt294DyHEhSdBqxE+7q4Y9NowJCdkrHiNUlBb6ZxFJrZos4ktampqeOqpp4iIiMDLy4uEhARWr15t2ffgwYPccccd+Pv74+XlRd++fVm2bBkHDhzgpptuArThf3Q6HQ899FCL3o/WkOGWG6HX6/D3NFJcUcPxyhpCfeWRP3VV8OKFaczbrD8dBqNX8+lOk4ktGp/YYsqUKezYsYOFCxcSHh7O559/zvDhw9m6dSu9evVi8uTJ1NbWsmbNGry8vNixYwfe3t5ERkbyf//3f4wcOZLdu3fj4+ODh4dHi96T1pCg1YRALy1oSWV8xyMTW9ie2CI/P5/333+f/Px8S2+Sp556iuXLl/P+++/z4osvkp+fz8iRIy3DT3fv3t2yf0OfxuDg4DavL7OXBK0myFjx53D11O54nHVuB8jEFrZt3boVk8lE7969rdbX1NRYJmJ9/PHHefTRR/n2229JSkpi5MiR9O/fv0XnuxCkTqsJAd4y7LIVnU4rojljkYkt2mRii4qKCgwGAzk5OWzevNmy7Ny5kzfffBOAhx9+mP379/Pggw+ydetWBg8ezJw5c9rsWltLglYTpNN0xyQTW2hsTWwxYMAATCYTR48etQwF1bCcXYyMjIzkkUceYcmSJfzhD3/g3XfftRwTsHvCjAtBglYTpFV8xyQTW2hsTWzRu3dvRo8ezZgxY1iyZAl5eXmsX7+ejIwMvvrqKwCeeOIJvvnmG/Ly8sjNzWXVqlWW80VFRaHT6Vi6dCnHjh27YDNiNenCjATd/tg7/vTZPszKU1HTlqqJH25oPvElqCOPES8TW2hsTWxRW1urZs6cqaKjo5Wrq6sKCwtTd999t9qyZYtSSqkpU6aoHj16KDc3N9W5c2f14IMPquLiYssxn3/+eRUaGqp0Op0aO3asXflo0BZjxMsoD034assRJi/I5epofxY/MuQC57D9kVEezvj111+JjIzku+++Y9iwYc7OTofVFqM8yNPDJkjx8PK1cuVKKioqiI2N5ciRIzzzzDMysUU7IXVaTZBO05cvmdii/ZI7rSY03GmVVtVRbzLjYpAYf7mQiS3aLwlaTfD3NKLTad3eTlTV0bmTm7OzJNoBmdjCueTWoQkGvQ4/D+2b83IuIl4mz2rERdAW/0sStJpxpjL+8ptpuqGoU1VV5eSciEtFw/9Sa4rRUjxsRqCXG/uOVV6Wd1oGgwE/Pz/L6Aaenp4talwphFKKqqoqjh49ip+fHwaDocXHkqDVjMu903RD146WDssixNn8/PxaPSt8i4LW3LlzefXVVyksLCQuLo45c+YQHx/faPrFixczY8YMDhw4QK9evXj55Ze57bbbAO3R8rPPPsuyZcvYv38/vr6+JCUl8dJLL1lNxBodHW0ZFK1BRkYGf/zjH1tyCXa73DtN63Q6wsLCCA4Opq5OBkMULefq6tqqO6wGDgetRYsWkZaWxrx580hISGD27NkkJyeze/dugoODz0uflZVFamoqGRkZ3H777SxYsICUlBRyc3Pp168fVVVV5ObmMmPGDOLi4jhx4gRTp07lzjvvPG90xueff54JEyZYfr8YT3Ck07TGYDC0yT+cEK3mUMchpVR8fLyaPHmy5XeTyaTCw8NVRkaGzfSjRo1SI0aMsFqXkJCgJk2a1Og51q9frwB18OBBy7qoqCj1xhtvOJpdi5b0PVRKqfd+3K+ipi1Vv/9PTovPLYRonr2fUYeeHtbW1pKTk0NSUpJlnV6vJykpiezsbJv7ZGdnW6UHreFeY+kBy7jW546M+NJLLxEYGMiAAQN49dVXqa+vb/QYNTU1lJeXWy0tcTk/PRSiPXKoeFhcXIzJZCIkJMRqfUhICLt27bK5T2Fhoc30hYWFNtNXV1czbdo0UlNTrTpNPv744wwcOJCAgACysrKYPn06R44c4fXXX7d5nIyMDP7yl784cnk2BcpM00K0K+3q6WFdXR2jRo1CKcU777xjte3s8ZD69++P0Whk0qRJZGRk4OZ2fkv16dOnW+1TXl5OZGSkw3ny95LGpUK0Jw4FraCgIAwGA0VFRVbri4qKGn2MGRoaalf6hoB18OBBVq5c2ezwMQkJCdTX13PgwAGr0R8buLm52Qxmjmq40zpRVYfZrNDrpZ2SEM7kUJ2W0Whk0KBBZGZmWtaZzWYyMzNJTEy0uU9iYqJVeoAVK1ZYpW8IWHv27OG7776zDLDflM2bN6PX620+sWxLDXdaJrOivFoe+QvhbA4XD9PS0hg7diyDBw8mPj6e2bNnU1lZybhx4wBtvriIiAgyMjIAmDp1KkOHDmXWrFmMGDGChQsXsnHjRubPnw9oAeu3v/0tubm5LF26FJPJZKnvCggIwGg0kp2dzbp16ywTVmZnZ/Pkk0/ywAMPtHiGFHu5uRjo5ObCyZp6jlfW4udpvKDnE0I0oyWPJufMmaO6du2qjEajio+PV2vXrrVsGzp06HlDsH766aeqd+/eymg0qr59+6qvvvrKsi0vL08BNpdVq1YppZTKyclRCQkJytfXV7m7u6s+ffqoF198UVVXV9ud55Y2eVBKqRteWamipi1V6/OOO7yvEMI+MtzyOVoy3HKDu//+E5vyS5n3wCCG92tdFwQhhG32fkZllAc7SKt4IdoPCVp2ONNpWhqYCuFsErTsEHC62YNMcCGE80nQsoMUD4VoPyRo2eFyH1NLiPZEgpYdLvcxtYRoTyRo2UGKh0K0HxK07HB28fAyadYmRLslQcsODZ2ma01mKmoaH8NLCHHhSdCyg4fRgIerNtSwFBGFcC4JWnY6M4KpBC0hnEmClp0CTz9BLJEniEI4lQQtO0lbLSHaBwladpLioRDtgwQtOwVKp2kh2gUJWnaSTtNCtA8StOwkreKFaB8kaNlJKuKFaB8kaNlJOk0L0T5I0LKTFA+FaB8kaNmpoXh4qs7EqVqTk3MjxOVLgpadvN1cMBq0t+u4NHsQwmlaFLTmzp1LdHQ07u7uJCQksH79+ibTL168mJiYGNzd3YmNjWXZsmWWbXV1dUybNo3Y2Fi8vLwIDw9nzJgxHD582OoYJSUljB49Gh8fH/z8/Bg/fjwVFRUtyX6L6HQ6qYwXoh1wOGgtWrSItLQ00tPTyc3NJS4ujuTkZI4ePWozfVZWFqmpqYwfP55NmzaRkpJCSkoK27ZtA6Cqqorc3FxmzJhBbm4uS5YsYffu3dx5551Wxxk9ejTbt29nxYoVLF26lDVr1jBx4sQWXHLLSat4IdoBR2eBjY+PV5MnT7b8bjKZVHh4uMrIyLCZftSoUWrEiBFW6xISEtSkSZMaPcf69esVoA4ePKiUUmrHjh0KUBs2bLCk+frrr5VOp1OHDh2yK9+tmWG6weh316qoaUvVZxsLWnwMIYRt9n5GHbrTqq2tJScnh6SkJMs6vV5PUlIS2dnZNvfJzs62Sg+QnJzcaHqAsrIydDodfn5+lmP4+fkxePBgS5qkpCT0ej3r1q2zeYyamhrKy8utltaS4qEQzudQ0CouLsZkMhESEmK1PiQkhMLCQpv7FBYWOpS+urqaadOmkZqaapkau7CwkODgYKt0Li4uBAQENHqcjIwMfH19LUtkZKRd19gUKR4K4Xzt6ulhXV0do0aNQinFO++806pjTZ8+nbKyMstSUFDQ6vxJp2khnM/FkcRBQUEYDAaKioqs1hcVFREaGmpzn9DQULvSNwSsgwcPsnLlSstdVsMxzq3or6+vp6SkpNHzurm54ebmZve12aOhVbwUD4VwHofutIxGI4MGDSIzM9Oyzmw2k5mZSWJios19EhMTrdIDrFixwip9Q8Das2cP3333HYGBgecdo7S0lJycHMu6lStXYjabSUhIcOQSWiVQiodCOJ+jNfwLFy5Ubm5u6oMPPlA7duxQEydOVH5+fqqwsFAppdSDDz6o/vjHP1rS//TTT8rFxUW99tpraufOnSo9PV25urqqrVu3KqWUqq2tVXfeeafq0qWL2rx5szpy5IhlqampsRxn+PDhasCAAWrdunXqxx9/VL169VKpqal257stnh6u239cRU1bqm54ZWWLjyGEsM3ez6jDQUsppebMmaO6du2qjEajio+PV2vXrrVsGzp0qBo7dqxV+k8//VT17t1bGY1G1bdvX/XVV19ZtuXl5SnA5rJq1SpLuuPHj6vU1FTl7e2tfHx81Lhx49TJkyftznNbBK09RSdV1LSlqt/M5S0+hhDCNns/ozqlLo/ZR8vLy/H19aWsrMyqvswRJyprGfDXFQDsfmE4bi6GtsyiEJc1ez+j7erpYXvn6+GKQa8D4ERlnZNzI8TlSYKWA/R6Hf6eroB0mhbCWSRoOUhaxQvhXBK0HCRBSwjnkqDloMDTs/JI0BLCOSRoOUjutIRwLglaDpJO00I4lwQtBwU29D+UWXmEcAoJWg6S4qEQziVBy0FniofSTksIZ5Cg5SB5eiiEc0nQclDDnVbpqTpM5sui26YQ7YoELQc1dONRCk5Uyd2WEBebBC0HuRj0+J0OXFJEFOLik6DVApbKeGn2IMRFJ0GrBQKl2YMQTiNBqwUCZFYeIZxGglYLBJxu9iBdeYS4+CRotYAUD4VwHglaLSCdpoVwHglaLWCp05Knh0JcdBK0WkA6TQvhPBK0WkCKh0I4T4uC1ty5c4mOjsbd3Z2EhATWr1/fZPrFixcTExODu7s7sbGxLFu2zGr7kiVLuOWWWwgMDESn07F58+bzjnHjjTei0+mslkceeaQl2W+1hjG1TlTVYpb+h0JcVA4HrUWLFpGWlkZ6ejq5ubnExcWRnJzM0aNHbabPysoiNTWV8ePHs2nTJlJSUkhJSWHbtm2WNJWVlVx33XW8/PLLTZ57woQJHDlyxLK88sorjma/TTTcaZnMivJqmf9QiIvK0amr4+Pj1eTJky2/m0wmFR4erjIyMmymHzVqlBoxYoTVuoSEBDVp0qTz0ubl5SlAbdq06bxtQ4cOVVOnTrU7n9XV1aqsrMyyFBQU2DXltr36zlyuoqYtVXuPnmyT4wlxuSsrK7PrM+rQnVZtbS05OTkkJSVZ1un1epKSksjOzra5T3Z2tlV6gOTk5EbTN+Xjjz8mKCiIfv36MX36dKqqqhpNm5GRga+vr2WJjIx0+HxNkcp4IZzDxZHExcXFmEwmQkJCrNaHhISwa9cum/sUFhbaTF9YWOhQRn/3u98RFRVFeHg4W7ZsYdq0aezevZslS5bYTD99+nTS0tIsv5eXl7dp4ArwMpJfUiWdpoW4yBwKWs40ceJEy+vY2FjCwsIYNmwY+/bto0ePHueld3Nzw83N7YLlR1rFC+EcDhUPg4KCMBgMFBUVWa0vKioiNDTU5j6hoaEOpbdXQkICAHv37m3VcVpKOk0L4RwOBS2j0cigQYPIzMy0rDObzWRmZpKYmGhzn8TERKv0ACtWrGg0vb0amkWEhYW16jgtFeAtbbWEcAaHi4dpaWmMHTuWwYMHEx8fz+zZs6msrGTcuHEAjBkzhoiICDIyMgCYOnUqQ4cOZdasWYwYMYKFCxeyceNG5s+fbzlmSUkJ+fn5HD58GIDdu3cD2l1aaGgo+/btY8GCBdx2220EBgayZcsWnnzySW644Qb69+/f6jehJaR4KISTtOTR5Jw5c1TXrl2V0WhU8fHxau3atZZtQ4cOVWPHjrVK/+mnn6revXsro9Go+vbtq7766iur7e+//74CzlvS09OVUkrl5+erG264QQUEBCg3NzfVs2dP9fTTTzvUfMHex6n2WryxQEVNW6oe+Ofa5hMLIZpl72dUp5S6LJp0l5eX4+vrS1lZGT4+Pq0+3qpdRxn3wQauDPNh2dTr2yCHQlze7P2MSt/DFpJ2WkI4hwStFjo7aF0mN6tCtAsStFqoodN0rclMRU29k3MjxOVDglYLeRpdcHfV3j4pIgpx8UjQaoVAmeBCiItOglYryLDLQlx8ErRaQZ4gCnHxSdBqhUAZdlmIi06CVitIp2khLj4JWq0gnaaFuPgkaLWCdJoW4uKToNUKAaebPEjQEuLikaDVCvL0UIiLT4JWKwSdrtM6drIGk8x/KMRFIUGrFbr4e+JpNFBTb2b/sQpnZ0eIy4IErVYw6HX0DdfG/dnya5mTcyPE5UGCVivFRvgBsPWQBC0hLgYJWq0U20W705KgJcTFIUGrlRrutLYfLqPeZHZuZoS4DEjQsqXmJJTstytp9yAvvIwGquvM7DtWeYEzJoSQoHWu7V/Ay9Hw5eN2JdfrdfSN8AVgy6+lFyxbQgiNBK1zhfQDcz0UrIPaKrt26X86aEm9lhAXXouC1ty5c4mOjsbd3Z2EhATWr1/fZPrFixcTExODu7s7sbGxLFu2zGr7kiVLuOWWWwgMDESn01lmjz5bdXU1kydPJjAwEG9vb0aOHElRUVFLst+0wB7gGwmmWjiYZdcusV0kaAlxsTgctBYtWkRaWhrp6enk5uYSFxdHcnIyR48etZk+KyuL1NRUxo8fz6ZNm0hJSSElJYVt27ZZ0lRWVnLdddfx8ssvN3reJ598kv/9738sXryY77//nsOHD3PPPfc4mv3m6XTQ/Ubt9f5Vdu3Sv4sfADsOl1MnlfFCXFiOzgIbHx+vJk+ebPndZDKp8PBwlZGRYTP9qFGj1IgRI6zWJSQkqEmTJp2XNi8vTwFq06ZNVutLS0uVq6urWrx4sWXdzp07FaCys7PtyrdDM0xv/UypdB+l5l5j17FNJrPqN3O5ipq2VG0/1DYzWAtxubH3M+rQnVZtbS05OTkkJSVZ1un1epKSksjOzra5T3Z2tlV6gOTk5EbT25KTk0NdXZ3VcWJiYujatWujx6mpqaG8vNxqsVu3GwEdHN0BJwubTa7X6+h3ul5rmxQRhbigHApaxcXFmEwmQkJCrNaHhIRQWGj7w11YWOhQ+saOYTQa8fPzs/s4GRkZ+Pr6WpbIyEi7z4dXIITFaa/3r7Zrl/6n67W2HCq1/zxCCIddsk8Pp0+fTllZmWUpKChw7AA9btJ+7rOvXqvhTmur9EEU4oJyKGgFBQVhMBjOe2pXVFREaGiozX1CQ0MdSt/YMWprayktLbX7OG5ubvj4+FgtDul+OmjtXwV2THvfcKe188hJauulMl6IC8WhoGU0Ghk0aBCZmZmWdWazmczMTBITE23uk5iYaJUeYMWKFY2mt2XQoEG4urpaHWf37t3k5+c7dByHdL0GXDygokir22oueYAnPu4u1JrM/FJ08sLkSQiBi6M7pKWlMXbsWAYPHkx8fDyzZ8+msrKScePGATBmzBgiIiLIyMgAYOrUqQwdOpRZs2YxYsQIFi5cyMaNG5k/f77lmCUlJeTn53P48GFAC0ig3WGFhobi6+vL+PHjSUtLIyAgAB8fHx577DESExO55pprWv0m2OTiBtHXwt7vtCJiSN8mk+t0OmK7+PLT3uNsPVRmKS4KIdpYSx5NzpkzR3Xt2lUZjUYVHx+v1q5da9k2dOhQNXbsWKv0n376qerdu7cyGo2qb9++6quvvrLa/v777yvgvCU9Pd2S5tSpU+r3v/+98vf3V56enuruu+9WR44csTvPDjV5aPDTHK3pw0f32JU8Y9lOFTVtqfrj/22x/xxCCKWU/Z9RnVJ2VNhcAsrLy/H19aWsrMz++q2i7fDOEK2YOO0AuLo3mXzZ1iP8/uNcYiN8+d9j17U+00JcRuz9jF6yTw/bRPCV4B0C9ae0vojNiD1dJNxVWE5NvelC506Iy5IErabodNZPEZvRxd8DP09X6kyK3YVSGS/EhSBBqzkOtNfS6XSWuy3pPC3EhSFBqzkNnaeP/AyVx5tNHiuNTIW4oCRoNadTKAT3BRTkrW42uaU7jwQtIS4ICVr2cKCIGHt6mJpfik5SXSeV8UK0NQla9rBUxq9utktPuK87gV5G6s2KXVIZL0Sbk6Blj6ghYDBCWQEc39tkUp1Od1bn6dKLkDkhLi8StOxh9NT6IoJdRcT+MvyyEBeMBC17OdBeKzZCKuOFuFAkaNmroTI+7wcw1TWZtGGiiz1HK6QyXog2JkHLXqFx4BEAtSfh141NJ/VxJ8jbDZNZseOIA8M8CyGaJUHLXnq93bP06HS6M/VaUkQUok1J0HKEA+21+km9lhAXhAQtRzRUxh/KgVOlTSY9M+t00+mEEI6RoOUIv0gI7AnKBAd+aDJpQ2X83qMVVNXWX4zcCXFZkKDlqB43az+bKSKG+LgT3MkNs9JmnhZCtA0JWo5yoL2WdJ4Wou1J0HJU9HWgM0DJfjhxoMmksRF+gMw6LURbkqDlKHcf6HK19rqZIuKZWaclaAnRViRotURDvVYzRcSGZg/7jlVQUdMBK+NrKmDtO1BV4uycCGEhQaslGtpr7f8ezI130+ncyY0wX3dUR62MX/U3WP5HWD7d2TkRwqJFQWvu3LlER0fj7u5OQkIC69evbzL94sWLiYmJwd3dndjYWJYtW2a1XSnFzJkzCQsLw8PDg6SkJPbs2WOVJjo6Gp1OZ7W89NJLLcl+64UPBDdfqC6FI5ubTHqm83TpBc9WmzLVwZZPtdc7/gs1MjZYR6KUYumWw+w9WuHsrLQ5h4PWokWLSEtLIz09ndzcXOLi4khOTubo0aM202dlZZGamsr48ePZtGkTKSkppKSksG3bNkuaV155hbfeeot58+axbt06vLy8SE5Oprq62upYzz//PEeOHLEsjz32mKPZbxsGF+h2vfa6mXqtDjvRxb6VUFWsva4/pQUu0WF8s72QKQs2Mfa99dSbzM7OTptyOGi9/vrrTJgwgXHjxnHllVcyb948PD09ee+992ymf/PNNxk+fDhPP/00ffr04a9//SsDBw7k7bffBrRvhNmzZ/Pss89y11130b9/fz788EMOHz7MF198YXWsTp06ERoaalm8vLwcv+K2YmeXntiOOrbWzwu1nx7+2s/NnzgvL8Jh/846CMCh0lMs317o5Ny0LYeCVm1tLTk5OSQlJZ05gF5PUlIS2dnZNvfJzs62Sg+QnJxsSZ+Xl0dhYaFVGl9fXxISEs475ksvvURgYCADBgzg1Vdfpb6+8crtmpoaysvLrZY21dBeq2CdVmHdiIY7rf3HKjlZ3fSQNu1GdRnsPl2Ev2suoIODPzbbxEO0D3uKTpK9/8zMUf/6Mc+JuWl7DgWt4uJiTCYTISEhVutDQkIoLLQdzQsLC5tM3/CzuWM+/vjjLFy4kFWrVjFp0iRefPFFnnnmmUbzmpGRga+vr2WJjIy0/0LtEdAd/LqCuQ4OZjWaLNDbjQg/DwC2HeoglfE7voT6agi6Aq64DboP1db/vMi5+RJ2+WitdpcVHx2A0aBnU34pufknnJyrttNhnh6mpaVx44030r9/fx555BFmzZrFnDlzqKmpsZl++vTplJWVWZaCgoK2zZADs0833G11mEamW04Hp7j7tOuM+532+8+fNDuxh3Cuipp6luQeAuDxYb24Iy4cgPcuobsth4JWUFAQBoOBoqIiq/VFRUWEhoba3Cc0NLTJ9A0/HTkmQEJCAvX19Rw4cMDmdjc3N3x8fKyWNtfQXmvPiiY/zLEdqZFpacGZzuCx92o/+9wORm84kQf5a52XN9GszzcdoqKmnu5BXgzpEcj467oB8PW2Qg6VnnJy7tqGQ0HLaDQyaNAgMjMzLevMZjOZmZkkJiba3CcxMdEqPcCKFSss6bt160ZoaKhVmvLyctatW9foMQE2b96MXq8nODjYkUtoW92HgosHHN9zpg7IhtiONDvP1sXaz6jrtOIvgNELrrxLe/3zAufkSzRLKcVH2QcAeOCaKPR6HVeG+5DYPRCTWfFh1gGn5q+tOFw8TEtL49133+Xf//43O3fu5NFHH6WyspJx48YBMGbMGKZPP9MYcerUqSxfvpxZs2axa9cunnvuOTZu3MiUKVMAbZTPJ554ghdeeIEvv/ySrVu3MmbMGMLDw0lJSQG0yvzZs2fz888/s3//fj7++GOefPJJHnjgAfz9/dvgbWghD3+45lHtdebzjTY0bQhaB45XUXaqHVfGK2VdNDxbXKr2c/sXUHdpfGNfatbnlfBLUQUergZGDupiWd9wt7VgfT6VHbFnxjkcDlr33Xcfr732GjNnzuSqq65i8+bNLF++3FKRnp+fz5EjRyzphwwZwoIFC5g/fz5xcXF89tlnfPHFF/Tr18+S5plnnuGxxx5j4sSJXH311VRUVLB8+XLc3d0Brai3cOFChg4dSt++ffnb3/7Gk08+yfz581t7/a137VRw94Nju840EziHv5eRyACtMn57ey4iHvlZuw4X9zN3Vg2irgXfrlBTDru+ck7+RJM+PF0BnzIgHF8PVziUC5XF3BwTTLcgL05W1/NZzq9OzmXr6ZS6PGpWy8vL8fX1paysrO3rt356E1bMBN9ImLIRXN3PS/L7j3NYtrWQP94awyNDe7Tt+dvK8umw9u/Q926494Pzt6/8G6x5BXomwQP/d9Gzd8ky1WsNllvhaHk1Q15aSb1Zsezx67mych18/FvoHAOP/MiH6w8x87/biQ70ZOUfbkSv17VR5tuOvZ/RDvP0sF2Lnwg+EdoM1Bv/ZTNJwzA17baRqaketn6mve5/v+00cafX71sJ5UdspxGO+XkhvNAZNvyzVYf5ZH0B9WbF4Ch/rgz1ghXp2oZjuyD334wc2AUfdxcOHK9i5S7bvVc6CglabcHVA278o/Z6zWta48xztPvZefavgsqj4BkIPYfZThPYAyITQJlh66cXN3+XooqjsOwZ7f1c8Zz2ewvUmcwsWK8VDR9MjNK+fI5uP5Ng9Ut4qSpS47UHKx29sakErbYS9zsI6g2nSiDr7fM29wvXglZ+SRX/bo9PcRoq4PuNBINr4+kaKuQ3S5utVvt2BtSc/hKrPamNqtECK3YUUVReQ5C3keF9AmDVC9qGG/8EAT2g8hj89CZjh0Rj0OvI3n+c7Yfb6ZenHSRotRWDC9w8Q3udPfe8b01fT1cmXK89xUn/cjsZX+/EbG4nH/qak7Bzqfa6saJhg753g8ENju1sdoQL0YS8H2DLQkAHyRnautwPoWh7k7vZ8lG2dpd1/9Vdcdv8EZTmg3coDHkMfvMXLVH2XMJ1JdzaT2v7+N6PB9rgIpxDglZb6nMHRAyCukpY8+p5m/90Wx+euqU3AP/4fj9pn26mtr4d9MDf+T9tJIfAnhAxsOm0Hn4QM0J7LZ2oW6a+Fr76g/Z68P+DxN/DlSlaMfGbPzl0B9vQz1Cvg98NCNQelAAMfQaMnhBzO0Reo/19V/3N0vzhfz8f5ujJ6iaO3H5J0GpLOh0kPae93vg+lOSds1nHlJt78epv++Oi1/HF5sM89P56yp3dkbqhqUb/+7VraM5Vp7v1bF2sfQCFY7LnQPFu8OoMw2Zq637zFzAYYf9q+OUbuw/V0M8wqU8I4Tvf14qC/t1g4BgtgU4HyaeLnZsXMMD4KwO7+lFrMvOf03doHY0ErbbW7QboMUzrSN1IHcW9gyP510NX42U0kLXvOKPmZVNY5qRvvbJDkLdGe93/Xvv26X4TeIdo9Xd7V1y4vF2KThyA70/fhd/yN+3OFcA/+kxD5W+f1QZhbMbZ/Qz/3wAfrekNwM3PWtdLdhmsFetR8O0Mxl/XHYD/rMunuq7xkXfbKwlaF0LS6cfNWxfDkS02kwzt3ZlFkxLp3MmNXYUnuefvP/FLkRNGB926GFDQNVH74NjD4AL9R2mvN2vdemrqTfx38yHyj1ddkGxeEpSCr6dpRbXo68+8hw2ufwo8g7RuYRtsN50529n9DBMOfaBV5of2h773nJ94WDroXWH/Koa7byPCz4OSylr+u/lQ21zbRSRB60IIi9OewoHWvacR/SJ8WfLoELp39uJwWTW/fSeLdWeNg3RRNAyp3P++ptOdq+Ep4i/fcPJEEePe38DUhZtJeuN73srcQ019x/sGv+B2L4NflmvBY8Ss84vi7j5w85+116szmpxQ5Ox+hpOuMqJraOeVlA56Gx/rgG5ae0LA8N1MHkrUuvn868c8Olr7cglaF8pNfwa9i1Z8OvBjo8kiAzz5v0eGMCjKn/Lqeh7813q+2nKRGm4WbtXa8xiM0DfFsX1D+mrf6uY6Ppo/i6x9WmVwbb2Z11f8wq1v/kDWvuILku0OqbZSu8sC7ale5ytspxswBoKv1OYf+P6VRg93dj/Du8v+A6Ya7e6tRyNt7ABueArcfeHoDka7Z+FpNPBLUQU/7u1YfycJWhdKYA8YOFZ7/d1zTT4R8vcy8vHDCST3DaHWZGbKJ7kXpwFgQwV87+FnhlV2wPFevwXg2soVBHkb+e/k65iTOoDOndzYf6yS3727jrRFmymusD3mWbullNZkJW8NrH8XvnoKPrhdW5oY8LFJ37+s9Zjw7Qo3PN14OoPLmYrzDe9C8R6byRr6GU7oU4tx2+mnuMPSm36Q4hlgObfnDxmMHhAEdLzGphK0LqSh08DVE37d0GwnY3dXA38fPYgxiVEoBX9duoP0/27j+IX6wJtNZ7rtxDXTNsuGnwtKufenCOqUgTj9fr68L4jYLr7cERfOd2lDGZMYhU4HSzYdYtis7/lkfX77aZfWQCntQcTeTG1+x/9NhfeGwyvd4LVe8O87YNlTWvA48IO2fDACVr5gV0W5RdEOre0ewG2vaE0RmtLjZuiVDOZ6rQHqOY6WV/PNNm1U34drF2hNJWJuh8irm89L/ERtyKGKQia7L0eng9W7j7H3aMeZbUmC1oXUKQSu+b32uomhaxoY9Dr+cmdfpg2PAeDf2Qe5JiOT33+cw/e/HMPUlh/6/auholC7w+r5G4d2/f6XY6S+u5b9VR7kGAcDEH7gc8t2Xw9Xnr+rH5///lquDPOh7FQd05ds5bfzsthV2E6GnC7cCu/eDG9cCf+5R5vfMecDyM+GUycAndZ0oPetcO0TkPKO1utBmbU2eO8Nh5L9zZ9HKa1NlrkerhgBV9xqtbm4ooZvthdSUHLOA4xbXtCqF375WvtbnaWhn2FqeBE+B74Gnf5Mw+bmuLhpd2SAX+7fuaeX9pTxvZ8O2Ld/OyCjPFxo1WXwZpz2QbhrLgx4wK7dvtleyNxVe9lyVl/FcF937h0cyb2Du9DFv5lv63OcrK5jc0EpuQdLOVRaxdTy14jI/xIGj4fbX7f7OJ9v+pWnF2+h3qy4rmcQ78YfxmPJQ9ApDJ7cDnqDVfp6k5l/Zx/k9W93U1lrwqDX8fB13Zia1AtPY+tGNmiR+hot6Pz4hhZI9C7aeP+dr9BGROgco70O7Kn1KUWrp2sYB63zwaXwvye17jdGb7jtVe2hRGPFss0L4ItHtTvuyevBL5I6k5nVu4+xeGMBK3cdpf70l1FshC/D+4UyvF8oPTp7a/0S1/8DQvrBpDWgN1BnMnPdyyspKq9mQ5c36Vy8Hq56AFLm2v8eKAX/HAaHcijqnUrCljtwd9WT/cdh+HsZW/X2toa9n1EJWhdD1hyt7Y1PF3gsx+bQNY3ZcbicTzcW8PmmQ5YPjk4H1/UM4v6ru5J0ZTBuLtaBQinFgeNV5Bw8QW7+CXIPnmB30UlLtZon1Wx0exRPXQ3/7P0Phtx4G1eGN/+ezF+zjxeX7QLgzrhwXrs3DiN1MOsKLSg/sKTRztZHyk7x/P928PXpYk2EnwcPDYkmtosvfcN96OTeRH/HtlKwAf47WWvYCVR2v43vuj1NodmX0lN1lFbVUXaqltKqhtd1lFbVUll75g45OtCTEVH1jD/6EgHFG7WVfe+B29840+aqQVUJvD0Yqo5D0l/4pdd4Fm8s4PNNh63q+aICPSkoqeLsG+neId6kXOHBpM2/xVBbBne8CYMeYtnWI/z+41xGeO5grvkF7SHKY7ng5+DELQez4P1bUTo9j/q8zfIiP55OvoLJN/V07DhtSILWOZwatOqqYc5AKD+ktYC++mFtoD2D0b4W6EB1nYlvtheyaEMBWfvONIvw93TlnoFduK5nEDuOlLMp/wS5+aWUVJ7fUj0ywIOBXf25qWYVKXl/Ic8cwk21rwM6+kX4MGpwJHfGhePnaf1tazYrXly2k3+errAdf103/nxbnzNjMn31B21oldh7YWTTQ6ys3FXEzP9u59cT1qOfdgvyom+4D/0ifImN0ALZuflosdpKrR5q7TuAwuTZmU86T2XmL92xt8Td8Gdq+LToMfN7ly950uUzDJip8QpHf898XHtcf2an/02FnA8o9e7BePfXyfm10rIpyNvI3QMiuHdwJL1DOlFcUcOKHUUs31ZI1r5i6kzaicYbljHD9T9UuvizL/VHXswsYN3+YrIDXyC0chdcMxmGv9iy92XhaNi1lCMhQ0k8OAm9Dm6OCSY1vis3XhGMobVjbimlTbG3+2utp0gz/+sStM7h1KAFkPsRfDnl/PUGN62ewWDUApmL8cw6ozd4dwav4LN+hlBo8uG/e+v4aGsVv5603XfR6KKnf4QvA6P8GdjVn4FRfgR3On2H99HdsG8lef0e59WaFFbsKLJ8SIwuem65MoRRgyO5tmcQJrPiqcU/8+XPhwH4020xTLzhnEEMf82Bf96sjZf/1C9ae6MmnKo18Z+1B9lwoITth8sbnXChi78H/cJ9ie3iyzXdAxjY1R+dnUHeYv/38L/HLXM2bu88gvGF91BYpxX94rsF0MXfAz8PI36ervh5uuLr4YqfpxE/j4bXrnRyd6Wytp61+47z095ifthbzP5jlcTp9vKm61yi9UWYlI5lfqkUD3qSvro84jO1xqOjamawXvXBRa/jpphg7h3UhZtignE12K5SLjtVR+bOIr7eVkj2L0f4Uv8U3fWFzK2/k1fr7+d2w1redn0LjJ1g6s/gFejYe9KgeA/MTQBlYlb4a8zZH27ZFObrzqjBkdx3dSThp6fAs1ttFWz7DNbP1+oOATVuObqoxud8AAla53F60DLVw3/uPtNlpo3UufpwHF+KTD6YPQNx9w3Gr3M4nUMicPEOAq8grZW1V2dtrKyqYni9j1ah/PgmCOhuaRm9aEMBuwrPPEUK93Wncyc3fv61DBe9jld+2597BnY5PxNKwdx4KP5F6/PWNRE8ArRH7B4BWmW/h9959V0NSipr2XaojG2Hy9h+qJxth8s4aKNlfVSgJ/cM6MI9AyOIDGimTq+6TCuS534IQIV7GNNrx/O/qisBGNjVjz/d1ofB0QH2vdE2HCo9xU97itnwSz7X732VO5U2ldxmcw/cqKWPvoDPTDcwP+Ap7h0UScqACDp3cnPoHJU19exctZDBaydTo1y5pfZlFnu/TnDdr1pbwKGNz/1pl69OPx0Ni2NvylIWbfyVz3J+5USVVhWh18GNVwRz/9WR3BwTjEsjgRaAkjyqsubjuuVjXGu1utgajHxRP4RhE14kKKpvk1mRoHUOpwetBmYzmGq1xoD1pxdTrTY5quX3Gq0jck251gG24qi2VDb8PL3O3IKO1q6eUFelDeY3/lurTUoptp+uQ/ti0yHKq7VJEDyNBv4+eiA3XtHEzEc/vqG1R2uUTmvY2BDIPANAZ9Aqw89dTHWYTPXU1NRQW1dLXV0dJbUuFJl9OYo/x5QvHgER9O7Zk6uujMHTP1zrC+nmrZ1q99ew9Ek4qTXS/cL1Nv58ciSVeBAd6Mm04TEM7xfq+F1bE8xmxaGfPib4+2m41WszjlcZOpF332qu7NWjdedSSmt+ceAHaj1DMVYVal9EU38+c80tVVkMb16ldQG6/Q246gFqMPDN9iI+WZdvNVN1iI8bowZHMmpwJF38Pfj1xCm2HzpBxc7v6JH3CXGn1qJHCycF5s58aPoNi01DKaUTH/6/eG7o3bnJrEjQOke7CVptRSmt1XTFWYGs6rj2s7JYu6OqPH76Z7HWuVmdVZRMmQdXpTZ6+Oo6E9/uKCJrbzEPXBNFv9MzCjWqthJ+mKWN5VRVop2vqkSroK+5OM0clNEbnUcAlOUDcNgQwdSq8WxQMQR4GZk6rBe/S+jaaLGsTZQWwOeTtIrulL+fGRGjtY5sgX/cAKeDAre+AgmT2ubYP8w6091Mp9ceGPlHQUA3SozhrDnmxeI8V7ZVBVCGNzodhLrVkly3kgcNK+ihP9OD43tzf771upOKyJu4MsKffhG+XBnmY9dTSQla57jkgpajzCY4VaoFMbMJgvvY/RCg1Ux1WvA6dcI6oCmzNhqB3kUrOuobXrtoLcP1Zy015VpwPllIxfFDFB46QFXJYTrVHSdYV4qX7szTODN65teP4I36kehc3Xn4uu5MGtr94jyhhDNfKC3oZdCk/06BTR9pjUOnbNTqPdtC3Skt0P7yrdaZuwmVOm/yTEF00x2xvOfVei/2d0nBPPj/0e2Kq/Bya1lTFgla57jsg9YlSCnFz7+W8X85v/Ld5n241RwjmFIOE8hhgvntoC6k/eYKQn3tb2LSrp06oQ1r028kdBnU9sdv6L50Ik97cHHigDYmXMPrikKr5ObOMejjJ2qd7VtbTEWC1nkkaF3aaupNrNp1lC9/PoyrQc+jN/YgJlT+zm2qtgpKD2oBzCMAIuPb9G79gk4hNnfuXKKjo3F3dychIYH169c3mX7x4sXExMTg7u5ObGwsy5ZZTyGvlGLmzJmEhYXh4eFBUlISe/ZYdxQtKSlh9OjR+Pj44Ofnx/jx46moqGhJ9sUlyM3FwPB+Yfx99CDevH+ABKwLweipVStccSt0Tbh41QvncDhoLVq0iLS0NNLT08nNzSUuLo7k5GSOHrU9/VFWVhapqamMHz+eTZs2kZKSQkpKCtu2bbOkeeWVV3jrrbeYN28e69atw8vLi+TkZKqrz4zmOXr0aLZv386KFStYunQpa9asYeLEiS24ZCFEh6YcFB8fryZPnmz53WQyqfDwcJWRkWEz/ahRo9SIESOs1iUkJKhJkyYppZQym80qNDRUvfrqq5btpaWlys3NTX3yySdKKaV27NihALVhwwZLmq+//lrpdDp16NAhm+etrq5WZWVllqWgoEABqqyszNFLFkJcBGVlZXZ9Rh2606qtrSUnJ4ekpCTLOr1eT1JSEtnZ2Tb3yc7OtkoPkJycbEmfl5dHYWGhVRpfX18SEhIsabKzs/Hz82Pw4MGWNElJSej1etatW2fzvBkZGfj6+lqWyEgH+2YJIdolh4JWcXExJpOJkJAQq/UhISEUFhba3KewsLDJ9A0/m0sTHGzdsNHFxYWAgIBGzzt9+nTKysosS0FBgZ1XKYRoz5wwNsjF4ebmhptbG7VjEUK0Gw7daQUFBWEwGCgqKrJaX1RURGhoqM19QkNDm0zf8LO5NOdW9NfX11NSUtLoeYUQlyaHgpbRaGTQoEFkZmZa1pnNZjIzM0lMtN2DOzEx0So9wIoVKyzpu3XrRmhoqFWa8vJy1q1bZ0mTmJhIaWkpOTk5ljQrV67EbDaTkJDgyCUIITo4h4uHaWlpjB07lsGDBxMfH8/s2bOprKxk3LhxAIwZM4aIiAgyMjIAmDp1KkOHDmXWrFmMGDGChQsXsnHjRubPnw9osy4/8cQTvPDCC/Tq1Ytu3boxY8YMwsPDSUlJAaBPnz4MHz6cCRMmMG/ePOrq6pgyZQr3338/4eHhNvN5LnW6DW15eTsZ7lcIYaXhs6maa+/ekkeTc+bMUV27dlVGo1HFx8ertWvXWrYNHTpUjR071ir9p59+qnr37q2MRqPq27ev+uqrr6y2m81mNWPGDBUSEqLc3NzUsGHD1O7du63SHD9+XKWmpipvb2/l4+Ojxo0bp06ePGl3nhuaPMgiiyzteykoKGjys3zZdOMxm80cPnyYTp06NTtMSHl5OZGRkRQUFFxyXX4u5WsDub6OTCnFyZMnCQ8PR29rwtnTLtmnh+fS6/V06WJjALsm+Pj4XHL/GA0u5WsDub6OytfXt9k0MoWYEKJDkaAlhOhQJGjZ4ObmRnp6+iXZOPVSvjaQ67scXDYV8UKIS4PcaQkhOhQJWkKIDkWClhCiQ5GgJYToUCRoCSE6FAla53B00o6O4rnnnkOn01ktMTExzs5Wi61Zs4Y77riD8PBwdDodX3zxhdV2ZcdkKe1Vc9f20EMPnfe3HD58uHMy6wQStM7i6KQdHU3fvn05cuSIZfnxxx+dnaUWq6ysJC4ujrlz59rcbs9kKe1Vc9cGMHz4cKu/5SeffHIRc+hkdg+TcBlwdNKOjiQ9PV3FxcU5OxsXBKA+//xzy+/2TJbSUZx7bUopNXbsWHXXXXc5JT/tgdxpndaSSTs6mj179hAeHk737t0ZPXo0+fn5zs7SBWHPZCkd3erVqwkODuaKK67g0Ucf5fjx487O0kUjQeu0lkza0ZEkJCTwwQcfsHz5ct555x3y8vK4/vrrOXnypLOz1ubsmSylIxs+fDgffvghmZmZvPzyy3z//ffceuutmEwmZ2ftorhshqa53N16662W1/379ychIYGoqCg+/fRTxo8f78ScCUfdf//9ltexsbH079+fHj16sHr1aoYNG+bEnF0ccqd1Wksm7ejI/Pz86N27N3v37nV2VtqcPZOlXEq6d+9OUFDQJfm3tEWC1mktmbSjI6uoqGDfvn2EhYU5Oyttzp7JUi4lv/76K8ePH78k/5a2SPHwLM1N2tGRPfXUU9xxxx1ERUVx+PBh0tPTMRgMpKamOjtrLVJRUWF1Z5GXl8fmzZsJCAiga9euzU6W0p41dW0BAQH85S9/YeTIkYSGhrJv3z6eeeYZevbsSXJyshNzfRE5+/Fle9PUpB0d2X333afCwsKU0WhUERER6r777lN79+51drZabNWqVTYnRWiYVMWeyVLaq6auraqqSt1yyy2qc+fOytXVVUVFRakJEyaowsJCZ2f7opHxtIQQHYrUaQkhOhQJWkKIDkWClhCiQ5GgJYToUCRoCSE6FAlaQogORYKWEKJDkaAlhOhQJGgJIToUCVpCiA5FgpYQokP5/9ZII5aUi/fFAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:11:36.902361Z", + "iopub.status.busy": "2024-02-29T03:11:36.901507Z", + "iopub.status.idle": "2024-02-29T03:12:26.941267Z", + "shell.execute_reply": "2024-02-29T03:12:26.940255Z" + }, + "papermill": { + "duration": 50.062519, + "end_time": "2024-02-29T03:12:26.944004", + "exception": false, + "start_time": "2024-02-29T03:11:36.881485", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:12:26.984358Z", + "iopub.status.busy": "2024-02-29T03:12:26.983955Z", + "iopub.status.idle": "2024-02-29T03:12:27.004770Z", + "shell.execute_reply": "2024-02-29T03:12:27.003908Z" + }, + "papermill": { + "duration": 0.043108, + "end_time": "2024-02-29T03:12:27.006622", + "exception": false, + "start_time": "2024-02-29T03:12:26.963514", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.1173930.0610470.0004162.4512990.0222270.688710.0315768.493183e-080.8849970.0154910.1724840.0203850.1598520.0000173.336296
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.117393 0.061047 0.000416 2.451299 0.022227 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 0.68871 0.031576 8.493183e-08 0.884997 0.015491 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.172484 0.020385 0.159852 0.000017 3.336296 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:12:27.044422Z", + "iopub.status.busy": "2024-02-29T03:12:27.044175Z", + "iopub.status.idle": "2024-02-29T03:12:27.496442Z", + "shell.execute_reply": "2024-02-29T03:12:27.495458Z" + }, + "papermill": { + "duration": 0.473675, + "end_time": "2024-02-29T03:12:27.498485", + "exception": false, + "start_time": "2024-02-29T03:12:27.024810", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:12:27.540205Z", + "iopub.status.busy": "2024-02-29T03:12:27.539887Z", + "iopub.status.idle": "2024-02-29T03:13:18.938208Z", + "shell.execute_reply": "2024-02-29T03:13:18.937189Z" + }, + "papermill": { + "duration": 51.422403, + "end_time": "2024-02-29T03:13:18.940772", + "exception": false, + "start_time": "2024-02-29T03:12:27.518369", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:13:18.981845Z", + "iopub.status.busy": "2024-02-29T03:13:18.981152Z", + "iopub.status.idle": "2024-02-29T03:13:18.998667Z", + "shell.execute_reply": "2024-02-29T03:13:18.997940Z" + }, + "papermill": { + "duration": 0.040037, + "end_time": "2024-02-29T03:13:19.000515", + "exception": false, + "start_time": "2024-02-29T03:13:18.960478", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:13:19.038115Z", + "iopub.status.busy": "2024-02-29T03:13:19.037872Z", + "iopub.status.idle": "2024-02-29T03:13:19.043209Z", + "shell.execute_reply": "2024-02-29T03:13:19.042379Z" + }, + "papermill": { + "duration": 0.026182, + "end_time": "2024-02-29T03:13:19.045002", + "exception": false, + "start_time": "2024-02-29T03:13:19.018820", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.055968931363778826}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:13:19.084386Z", + "iopub.status.busy": "2024-02-29T03:13:19.084111Z", + "iopub.status.idle": "2024-02-29T03:13:19.416406Z", + "shell.execute_reply": "2024-02-29T03:13:19.415452Z" + }, + "papermill": { + "duration": 0.354747, + "end_time": "2024-02-29T03:13:19.418549", + "exception": false, + "start_time": "2024-02-29T03:13:19.063802", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAE8CAYAAACGvqckAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/pklEQVR4nO3deXxU5b348c+Zc2ZPMtlJAglr2ERWgaKyKbWKVbm2iksruOG9xauW2pemr3tda9FeF6xt6XIl1F9VWi3aXi2iVRaXAopFEJDNAIEEEsieyezn98dMBgIJJJNJzoR836/XvMicPHPOd4bkm+d5zrMouq7rCCFEgjAZHYAQQpxMkpIQIqFIUhJCJBRJSkKIhCJJSQiRUCQpCSESiiQlIURCkaQkhEgokpSEEAlFkpIQIqFIUhJx9cknn/DII49QU1NjdCiih5KkJOLqk08+4dFHH5WkJGImSUkIkVAkKYm4eeSRR/jxj38MwMCBA1EUBUVRSEpKYubMmaeVD4VC9O3bl+9+97vRY08//TQXXnghGRkZ2O12JkyYwOuvv97q9f74xz8yYcIE7HY76enp3HDDDZSWlnbNmxPdRpGlS0S8bN26lSeffJJXX32V5557jszMTAD27dvHY489xuHDh8nJyYmWX79+PdOnT+e1116LJqb8/HyuvvpqRo4cic/nY8WKFWzatIm33nqLK6+8MvraJ554gv/+7//m+uuvZ/r06VRWVvLCCy+QlJTEv/71L1JTU7v1vYs40oWIo//5n//RAb2kpCR6bNeuXTqgv/DCCy3K/uAHP9CTkpJ0t9sdPXby17qu6z6fTx81apR+ySWXRI/t379fV1VVf+KJJ1qU3bZtm65p2mnHRc8izTfR5YYOHcrYsWP505/+FD0WDAZ5/fXXueqqq7Db7dHjJ39dXV1NbW0tU6dO5fPPP48eX7lyJaFQiOuvv55jx45FHzk5ORQWFrJmzZrueWOiS2hGByB6h7lz5/KTn/yEw4cP07dvX9auXUtFRQVz585tUe6tt97ipz/9KVu2bMHr9UaPK4oS/XrPnj3ouk5hYWGr1zKbzV3zJkS3kKQkusXcuXMpKiritdde47777uPPf/4zLpeLyy+/PFrmww8/5Oqrr2batGn8+te/Jjc3F7PZTHFxMa+88kq0XCgUQlEUVq1ahaqqp10rKSmpW96T6BqSlERcnVyjOdnAgQOZNGkSf/rTn7j77rtZuXIlc+bMwWq1Rsv85S9/wWazsXr16hbHi4uLW5xr8ODB6LrOwIEDGTp0aNe8EWEY6VMSceV0OgFaHTw5d+5cNmzYwLJlyzh27NhpTTdVVVEUhWAwGD22f/9+3nzzzRblrr32WlRV5dFHH0U/5eaxruscP348Pm9GGEKGBIi4+vTTT5k0aRKzZ8/mhhtuwGw2c9VVV+F0Ojl06BAFBQUkJSVhNps5cuRIi/6fDz74gEsvvZSpU6dy0003UVFRwa9+9StycnLYunVriwT05JNPUlRUxIUXXsicOXNITk6mpKSEN954gwULFnD//fcb8fZFPBh450+cox5//HG9b9++uslkOm14wEUXXaQD+h133NHqa1988UW9sLBQt1qt+vDhw/Xi4mL94Ycf1lv7Uf3LX/6iX3zxxbrT6dSdTqc+fPhwfeHChfquXbu66q2JbiA1JSFEQpE+JSFEQpGkJIRIKJKUhBAJRZKSECKhSFISQiQUSUpCiITSo6eZhEIhysrKSE5ObnN6gxDCOLquU19fT15eHiZT++pAPToplZWVkZ+fb3QYQoizKC0tpV+/fu0q26OTUnJyMhB+wykpKQZHI4Q4VV1dHfn5+dHf1fbo0UmpucmWkpIiSUmIBNaR7hXp6BZCJBRJSkKIhCJJSQiRUHp0n1J76LpOIBBosXCYaB9VVdE0TYZbiG51Ticln89HeXk5brfb6FB6LIfDQW5uLhaLxehQRC9xzialUChESUkJqqqSl5eHxWKRv/gdoOs6Pp+PyspKSkpKKCwsbPfgNyE6w/CkdPjwYR544AFWrVqF2+1myJAhFBcXc8EFF3TqvD6fj1AoRH5+Pg6HI07R9i52ux2z2cyBAwfw+XzYbDajQ0pshzbTdHgrmwJDseUOZ9LAdPlDGANDk1J1dTUXXXQRM2fOZNWqVWRlZbFnzx7S0tLidg3569458vm1U1057HmXr8tqUb272VSfhMthZniOjJ/rKEOT0lNPPUV+fn6LLXQGDhxoYERCxKj8Cxp9Aeo8AVQg072PbYfSJSnFwNA/g3/729+44IILuO6668jOzmbcuHH8/ve/b7O81+ulrq6uxUOIhFD1NbVNfmpt/XBYVFKbSimr8eALhIyOrMcxNCl9/fXXLF26lMLCQlavXs1//Md/cM899/CHP/yh1fKLFy/G5XJFHzIZt/MGDBjAkiVLjA6jZ/PUgqeWBl+IwyljyEq2cn5KI9eOy0M1SZ9SRxmalEKhEOPHj+dnP/sZ48aNY8GCBdx555385je/abV8UVERtbW10UdpaWk3RyxEK2oPA1A4aDBXTZtERoqDfkkK+bYmSUoxMDQp5ebmMnLkyBbHRowYwcGDB1stb7Vao5NvZRLuCT6fz+gQerfGCgCU5FyyXU6sqXnh4w0VBgbVcxmalC666CJ27drV4tju3bvp379/l17XFwi1+QgEQ+0u629H2VjMmDGDu+++m7vvvhuXy0VmZib//d//Hd0hdsCAATz++OPccsstpKSksGDBAgA++ugjpk6dit1uJz8/n3vuuYfGxsboeSsqKrjqqquw2+0MHDiQl19+Oab4xCnckW3CHRnRfwOhEAcOlfKvg9XGxdVDGXr37Yc//CEXXnghP/vZz7j++uvZtGkTv/vd7/jd737Xpdf91Zq9bX5vYKaTOeP6Rp//bv0+/MHW9+vsl2bnugtO9Gst+7iEJl/L6Sw//ObQmGL8wx/+wO23386mTZv47LPPWLBgAQUFBdx5550APP300zz00EM8/PDDAOzbt4/LL7+cn/70pyxbtozKyspoYmu+uzl//nzKyspYs2YNZrOZe+65h4oK+WveaY3HqWr0su2wTp6lnkJHJoGgzrbd+yipGcSYfqmYpBnXboYmpYkTJ/LGG29QVFTEY489xsCBA1myZAk333yzkWElhPz8fJ577jkURWHYsGFs27aN5557LpqULrnkEn70ox9Fy99xxx3cfPPN3HfffQAUFhbyi1/8gunTp7N06VIOHjzIqlWr2LRpExMnTgTgxRdfZMSIEd3+3s4poSA0VVPbFODLag1ThofCtAysmglnoJZASKfeE8DlMBsdaY9h+Ijub3/723z729/u1msunDmkze+d+gdtwbTBbZY9dbDubRfFb4zVN77xjRajgadMmcIzzzwTnVh86oj3L774gq1bt7Zokum6Hp1us3v3bjRNY8KECdHvDx8+nNTU1LjF3Cs1VYMeojGk4lOdpDks4AiP5HZRD7pOnccvSakDDE9KRrBo7e9K66qyneV0Ols8b2ho4K677uKee+45rWxBQQG7d+/urtB6l6Zwn1E9SaAouOxmsIb/b2yqjhbyUufxGxlhj9Mrk1JPsHHjxhbPN2zYQGFhIaqqtlp+/Pjx7NixgyFDWq8FDh8+nEAgwObNm6PNt127dlFTUxPXuHsdTx06OrV6eH5lsk0DVQOLE6vWgDXYQL0nYHCQPYtMbEpQBw8eZNGiRezatYtXX32VF154gXvvvbfN8g888ACffPIJd999N1u2bGHPnj389a9/5e677wZg2LBhXH755dx1111s3LiRzZs3c8cdd2C327vrLZ2bvLUEgjpuU7h2lGSN/J23pWDVTFgD9dQ1SU2pIyQpJahbbrmFpqYmJk2axMKFC7n33nujt/5bM3r0aNatW8fu3buZOnUq48aN46GHHiIvLy9apri4mLy8PKZPn861117LggULyM7O7o63c+7y1OENhPCpSTitKpoa+ZWyubBqKhapKXWYNN8SlNlsZsmSJSxduvS07+3fv7/V10ycOJF33323zXPm5OTw1ltvtTj2/e9/v1Nx9nqeWvzBEF4tiRTbSZ3Z1hRcDjMz02zYRkji7whJSkJ0hreONIeFm8edj9fe58RxmwuLaiLH6gGHrNrZEZKUhIhVKAi+8Ih5zZGKZjnp18kamQLlrTcgsJ5NklICWrt2rdEhiPbw1oGug0kD8ymrm1rCHd9Hj1ex/+vjnNfXdaITXJyRdHQLEStPeD2vPXUKq3cc5ViD98T3IkmprOIYn+w9RnWjTJpuL0lKQsQq0jQrbzKzo6yu5QRsS1L4HyWEqvtw+2SLr/aSpCRErHyN6Og06lYAnCf3KakaaFYsmglzsIlGnwwLaC9JSkLEytdAMKTTpIQHoDqsp4y2tyRhVk1Ygm7cXqkptZckJSFi5WvEH9Txq45wjUg95dfJ4sSsmjAH3VJT6gBJSkLEyteIPxjCp9pxWlqZk2hxYlYVzKEm3JKU2k2SkhCx8jXgD4bwqw4crd3utyRhiTTfGqX51m4ycEKIWEWabz7N0bKTu5nFSZJNY1qWA/W83O6Pr4eSpCRELEJB8DeRk2LjjimjCKitrLZgcaKZTGRZA+CUqSbt1buab7oOAV/3P/TW1/huzUsvvURGRgZer7fF8Tlz5sjk2UQSmV6CYkKzOrGZW+9TCpdt6L64zgG9q6YU9MOHz3T/daf+CLT2/aW87rrruOeee/jb3/7GddddB4R3IXn77bfPuAKA6GbNScniPH1d5GaRqSdHjldzsKSK8/JScMpUk7PqXTWlHsBut3PTTTdFdyAB+OMf/0hBQQEzZswwLjDRUiQpfVUV5J0vj1DV2jQSc7hJd7iiio/3VFIri721S+9K26o5XGsx4rodcOeddzJx4kQOHz5M3759Wb58OfPnz2+xkYAwWKRJVubW2Flex/iC1NPLRGpKFlMIkx6gyS934NqjdyUlRWl3M8pI48aNY8yYMbz00ktcdtllbN++nbffftvosMTJIlNMGiJTTGytjVNSzWDSMKsKWsh72p6AonW9Kyn1IHfccQdLlizh8OHDzJo1i/z8/LO/SHQfXyPBkI7PFG6i2Vvr6FYUMNvQTArmkAeP1JTaxdA+pUceeQRFUVo8hg8fbmRICeOmm27i0KFD/P73v+e2224zOhxxKr+bQFDHr9pbn2LSzGxHU01oQQ8ef2zbuPc2hnd0n3feeZSXl0cfH330kdEhJQSXy8V3vvMdkpKSmDNnjtHhiFP5m/CHQvhNttaHAzQzO9BMClrII31K7WR4803TNHJycowOIyEdPnyYm2++GavVanQo4lSRmlLAZGu96dbMbI9MypWk1F6GJ6U9e/aQl5eHzWZjypQpLF68mIKCglbLer3eFoMK6+rquivMblVdXc3atWtZu3Ytv/71r40OR7TG30QgpBNQbdgtZ2hwmB2kOsxcmp2MpTCz++LrwQxNSpMnT2b58uUMGzaM8vJyHn30UaZOncqXX35JcnLyaeUXL17Mo48+akCk3WvcuHFUV1fz1FNPMWzYMKPDEafSdfA3kZVk5fbJI/GbT/9ZjTLbsWoq2XZddjVpJ0OT0hVXXBH9evTo0UyePJn+/fvz5z//mdtvv/208kVFRSxatCj6vK6u7py8K9XWvm4iQQT9EAovRaJanW1upQ6AFpkT53d3Q2DnBsObbydLTU1l6NCh7N27t9XvW61W6V8RxmtOMKp29oGxZjshXedQxXHK7ceZNDBdBsGeheF3307W0NDAvn37yM2N3zIPegcmw4rTyefXCn8TAPvrdN75spzDNU1tl41MNdlTWsEn+47jDciwgLMxNCndf//9rFu3jv379/PJJ5/wb//2b6iqyo033tjpc5vN4b9gbrdUmzuj+fNr/jwF0ZrSMY/KzvJ63N4zrCppdmBSFKyEb9DIqO6zM7T5dujQIW688UaOHz9OVlYWF198MRs2bCArK6vT51ZVldTUVCoqKgBwOBxSbe4AXddxu91UVFSQmpp65n6T3iaSlJoId1yfeZxSZMQ34Qm7Tf4gaV0bXY9naFJasWJFl56/efxTc2ISHZeamirjyE4Vab65m+e9nWXwJIDZFMQU8kvzrR0SqqM73hRFITc3l+zsbPx+WTaio8xms9SQWuN3o+s6TXq4SWtvbTJuM9UMJhXNZEILeWX+Wzuc00mpmaqq8ssl4icycNJnsgFtTMZtpiigWSNTTbxSU2qHhLr7JkSP4HeHR3ObbFg0E6rpLH2Vmh21OSlJTemsekVNSYi48jcRCIXCU0zOVEtqZraR67Ixe3AaSXkpXR9fDydJSYiO8jeRbDVz88Th+JJbn6fZgmbDYdFwOAGbDK04G2m+CdFRkSEBqtV55k7uZlq474mA98zlBCBJSYiO0XXwe8Jfm1vZ6601mg1vIMi+skp2lp+bK1vEkyQlIToi4AE9REW9h9W7athb0Y493cw2mvxBtu0/ymcHqrs+xh5OkpIQHREZOFnrM7HjaFPrWyudSrOhmUyoIZ/cfWsHSUpCdESkP8mjhEdzt+vum2aLLokr45TOTpKSEB0RqSl5IvPezrjqZDPNFhmn5MMXCBEMycoLZyJJSYiOaJ6Mq7djMm4zsy06ohvAG5Am3JlIUhKiI06ZjNu+5psdRVGwKuH5l17ZaumMJCkJ0RGRybhuvbn51p6kFE5gVnyg63ikpnRGMqJbiI5o3sUkMhnXprWn+RYezzQk00H22GzSnbKBwJlIUhKiI/xNmFUT3/nGULzZgzCdbTIugEkDk4rLDq4UE7QnkfVi0nwToiOiU0wcOCzt/JuuKCemmjSPBhdtkqQkREdEOrrbPcWkmWajwRtg56EKys600YCQpCREh/jd1DT5eH9vPVsP1bT/dWYbxxu8bNpd1r6pKb2YJCUh2isUgoAXty/IjmOBjtV4NBuaGp5qIkvinpkkJSHaK9AEuk4gqBMwWbG2Z4xSs+iobplqcjaSlIRor0h/khcLKKb2DZxspp0Y1S01pTOTpCREe50yGbddU0yaRaea+KSmdBaSlIRor1Mn43aopmRHU2WbpfZImKT05JNPoigK9913n9GhCNG65poSzTWlDvz6aFZUk4Iq2yydVUKM6P7000/57W9/y+jRo40ORYi2RWpKzSsEdKimZLZj0UxM6Gtj1Oi8rojunGF4TamhoYGbb76Z3//+96SlyS7rIoFFakqXjh7IgmmDyEiytv+1mhVVUci2K+SnO7oowHOD4Ulp4cKFXHnllcyaNeusZb1eL3V1dS0eQnSbSE1JtTpwWrWzb0J5suiOJjLN5GwMTUorVqzg888/Z/Hixe0qv3jxYlwuV/SRn5/fxREKcZJYp5hANClV1daxtbSaBm8gjoGdWwxLSqWlpdx77728/PLL2Gy2dr2mqKiI2tra6KO0tLSLoxTiJH43Hn+Q9SUNfLLvWMdeG0lKpccbWLOjjKqGdmw40EsZ1tG9efNmKioqGD9+fPRYMBhk/fr1/PKXv8Tr9aKqLTsSrVYrVmsH2vFCxJO/CU8gyK6qIDalgQsHZ7b/taoZTGp0AKUsids2w5LSpZdeyrZt21ocu/XWWxk+fDgPPPDAaQlJCMP53eEpJqqtYwMnIbJ8iTUy1cSLR5bEbZNhSSk5OZlRo0a1OOZ0OsnIyDjtuBCGCwYg4CMQ0vGbY0hK0HL/N6kptcnwu29C9AiBcCe3PwRBk7VjY5SaaTZUtbn5JjWltiTE4Mlma9euNToEIVrXPBlXsYCidKKmJJNyz0ZqSkK0R2TgpC8yxaRdm1CeSrOimRRUXSblnklC1ZSESFjNU0wiSckay+L/ZjtpDgvTspyYB8jshbZIUhKiPSI1pXGD8xgxbBCaKbaaks2s0jdJheT2jc3rjWJqvn399dfxjkOIxBapKZks4V1MLFosSUmmmrRHTElpyJAhzJw5kz/+8Y94PPIBi17AF64pxTTFpJlmIxjSKTtWzZeHa+MT1zkopqT0+eefM3r0aBYtWkROTg533XUXmzZtindsQiSOyHbd/zzoZs1XFfiDMXRUazYCoRBfHqjgg68q0HU9/nGeA2JKSmPHjuX555+nrKyMZcuWUV5ezsUXX8yoUaN49tlnqaysjHecQhgrsl333poQW0prUJUOrBDQzBwePKmFPARDOv6gJKXWdGpIgKZpXHvttbz22ms89dRT7N27l/vvv5/8/HxuueUWysvL4xWnEMbyuwmEdAImG1azqX3bdZ9Ks2EygRbyA8io7jZ0Kil99tln/OAHPyA3N5dnn32W+++/n3379vHee+9RVlbGNddcE684hTCWvwl/MITfZMMWy3AAAM2KgoJNCa8QIGOVWhfTkIBnn32W4uJidu3axezZs3nppZeYPXs2psht0oEDB7J8+XIGDBgQz1iFMIauR5tvAdVGsiXWpBTuJDcrQRQ9IEmpDTElpaVLl3Lbbbcxf/58cnNzWy2TnZ3Niy++2KnghEgIQT+EAgSCIQKarWMbBpxMs4KiRPqVZKfctsSUlN577z0KCgqiNaNmuq5TWlpKQUEBFouFefPmxSVIIQwVGTjp1xWCijm2ybgQXr5EtYSnmoR8eGX5klbFlPIHDx7MsWOnr7xXVVXFwIEDOx2UEAklOhnXDorSse26T2W2k5dq54phLvqld2LM0zkspppSW+MrGhoa2r20rRA9RqSmNCAnk7vGDkIhhjtvzTQrLrsZV6oGNnOcAjy3dCgpLVq0CABFUXjooYdwOE5sFRMMBtm4cSNjx46Na4BCGO6UKSadIlNNzqpDn/C//vUvIFxT2rZtGxaLJfo9i8XCmDFjuP/+++MboRBG68wuJqfSbHj8QY4eOY7J7JY94FrRoaS0Zs0aILyW9vPPP09KSkqXBCVEQok037ZV+DmmVjC+fxoue4xNL81GbZOfjXvKsFEjSakVMXV0FxcXS0ISvUekpvR1ZIpJTPPempltaM1L4sqQgFa1u6Z07bXXsnz5clJSUrj22mvPWHblypWdDkyIhBGZjNuoh7srHLEOnoTwOt2RIQGNMniyVe1OSi6XCyUyCdHlcnVZQEIkHH8T/pBOwGRFUYh9mglElsQ1SU3pDNqdlIqLi1v9Wohznt8dHs0d2e8tpsm4zTR7dO83mWbSupj6lJqamnC73dHnBw4cYMmSJbz77rtxC0yIhOF34w/q+E22zjXdILp5gBby4QuECIVk+ZJTxZSUrrnmGl566SUAampqmDRpEs888wzXXHMNS5cujWuAQhgqFIpMxg3hN9lj21rpZM19SroXkJUCWhPzypNTp04F4PXXXycnJ4cDBw7w0ksv8Ytf/CKuAQphqEAT6OEF2QJqHGpKZjsmRWFEppmrxuSiqZ1oCp6jYkpKbreb5ORkAN59912uvfZaTCYT3/jGNzhw4EC7z7N06VJGjx5NSkoKKSkpTJkyhVWrVsUSkhBdI7I2d3Z6GgtmFDJzWHbnzqeFt2jq4zAxJNOJWZWtF08V88YBb775JqWlpaxevZrLLrsMgIqKig6NX+rXrx9PPvkkmzdv5rPPPuOSSy7hmmuuYfv27bGEJUT8+RsBMFmdOCwaTmucppkABL2dO9c5Kqak9NBDD3H//fczYMAAJk+ezJQpU4BwrWncuHHtPs9VV13F7NmzKSwsZOjQoTzxxBMkJSWxYcOGVst7vV7q6upaPIToUr5wUsIcp5HXJhVUMw3eAHsOV1Lj9sXnvOeQmJLSd7/7XQ4ePMhnn33GO++8Ez1+6aWX8txzz8UUSDAYZMWKFTQ2NkaT3KkWL16My+WKPvLz82O6lhDtFmm+7a4OsearCo41xKF2o9k4VO1mzZelHKxyn718LxNzXTQnJ4ecnJwWxyZNmtTh82zbto0pU6bg8XhISkrijTfeYOTIka2WLSoqiq5UAFBXVyeJSXStSPPtYIPCttIaBmclQVInzxnd1UTGKrUmpqTU2NjIk08+yfvvv09FRQWhUMsPtiM76A4bNowtW7ZQW1vL66+/zrx581i3bl2riclqtWK1WmMJWYjYRGpK7sgUE5slDh3TWmT+m+6V1SdbEVNSuuOOO1i3bh3f//73yc3NjU4/iYXFYmHIkCEATJgwgU8//ZTnn3+e3/72tzGfU4i48TWio9Ogh/8Ydno9JWgx/022WTpdTJ/wqlWrePvtt7noooviHQ+hUAivV+5KiAThbyQQ1PEp4bWUYl6f+2SaLTyqO+jFIzWl08SUlNLS0khPT+/0xYuKirjiiisoKCigvr6eV155hbVr17J69epOn1uIuPC5w/u9qeFNKNXOzHtrpkX6lPxeqSm1IqYG8uOPP85DDz3UYv5bLCoqKrjlllsYNmwYl156KZ9++imrV6/mm9/8ZqfOK0Rc6Hq4phTS8asOHPGoJQFoVlS1ufkmNaVTxVRTeuaZZ9i3bx99+vRhwIABmM0tV+H7/PPP23Ue2RdOJLSgH4KBcE3JbI9PfxKA2Y7TojI+z4Y+OCM+5zyHxPQpz5kzJ85hCJGAfA0ApCc7WTBlOP5QnGo1mhWrplKQokKGMz7nPIfElJQefvjheMchROKJrM2tWJzYLSp24tV8kx1NziTmQRc1NTX87//+L0VFRVRVVQHhZtvhw4fjFpwQhoqMUcIS59qMZkNH53hNLbuO1BPozJrf56CYktLWrVsZOnQoTz31FE8//TQ1NTVAeG3uoqKieMYnhHEio7n31uis+aqCivo41WwiNaWdpZX8fVs5TbIsbgsxJaVFixYxf/589uzZ02JH3NmzZ7N+/fq4BSeEoSKTcQ81wpbSGho8gfic12xDQcGKH3RdxiqdIqak9Omnn3LXXXeddrxv374cOXKk00EJkRAizbfGUHg0d1Jnly1pFqkpmU06qu6XsUqniCkpWa3WVpcN2b17N1lZWZ0OSoiE4KtH13Xq9XASccQrKZk0MKmoJpOMVWpFTEnp6quv5rHHHsPv9wOgKAoHDx7kgQce4Dvf+U5cAxTCMN4G/CEdr8mBohC/wZOKcmJSbsgjk3JPEVNSeuaZZ2hoaCArK4umpiamT5/OkCFDSE5O5oknnoh3jEIYw9eAPxjCpzqwd3ZrpVM1z38L+fBI862FmOqjLpeL9957j48//pgvvviChoYGxo8fz6xZs+IdnxDG0HXwNeIPhPCpTlLi1XRrpllPrBQgNaUWOvxJh0Ihli9fzsqVK9m/fz+KojBw4EBycnLQdb1Ty5gIkTD8TRAK4otMMUmyxqnp1sxsJzPJysV5TlKyZVT3yTqUlHRd5+qrr+bvf/87Y8aM4fzzz0fXdXbu3Mn8+fNZuXIlb775ZheFKkQ3ikwxyUpP598nDyUQ700jNSspNjMp6WZItp29fC/SoaS0fPly1q9fz/vvv8/MmTNbfO+DDz5gzpw5vPTSS9xyyy1xDVKIbuetB0CxJnV+A8rWNE818TfF/9w9XIc6ul999VV+8pOfnJaQAC655BIefPBBXn755bgFJ4RhIjUlrMldc37Nhj8Y4mhVLaWyeUALHUpKW7du5fLLL2/z+1dccQVffPFFp4MSwnDecFLaeTy8i8nxeOxicjLNRqMvwKa9ZazdXRnfc/dwHUpKVVVV9OnTp83v9+nTh+rq6k4HJYThIjWlknqFLaU1eOI9wFGzhlefDHrxyty3FjqUlILBIJrWdjeUqqoEAnGaHySEkbz16OjURaaYJNviPCTAbEczKai6jOg+VYfvvs2fP7/NbY5kwX9xzvA14A/qeJTwaG5nvFadbBbZ0UQLefEFQoRCenwHZ/ZgHfqk582bd9YycudNnBO8DfgC4dHcTosWnw0DTmZ2oJkUzMHw3TdvIITd0gV3+XqgDiWl4uLiropDiMQRGc3tCwTxqw4y4t10AzDbURQFKz7QQ3j8QUlKEXHY7lOIc4yvAfQQ3qCOT3WQ1EVJCUAzIfPfTiFJSYhTecLL8jQpTlBM8VtH6WQmFTQr+WkOZhUmk2Izn/01vUQXfNpC9HDecFIa2C+H/zh/MHqcZ5hEmR1kJXvJytCgKxJfD2VoTWnx4sVMnDiR5ORksrOzmTNnDrt27TIyJCGiNSXF5sJmVruur8csU01aY2hSWrduHQsXLmTDhg289957+P1+LrvsMhobG40MS/R2kZoS1pSuvY7ZgTcQpKzyePw2JTgHGFpnfOedd1o8X758OdnZ2WzevJlp06YZFJXo9Ty16LrO+oNeQoEKLh6SiVntgr/fZjuV9V4+3nmIdAbxzZGyWgAkWJ9SbW0tAOnp6a1+3+v1thig2do64UJ0mqcWbzDE100q9YdqmTG0i9adN9sxqyY0n0e2WTpJwtx9C4VC3HfffVx00UWMGjWq1TKLFy/G5XJFH/n5+d0cpegVvHV4/SG8ajIpNq3rFi40O8LrdAc9eHySlJolTFJauHAhX375JStWrGizTFFREbW1tdFHaWlpN0YoeoWAD/wevIEgPi0Jl6MLb9Wb7WgmE+aQ1JROlhDNt7vvvpu33nqL9evX069fvzbLWa3WNufdCREXkU5ud8hM0GTBZe/KpOTAHNnRRJLSCYbWlHRd5+677+aNN97ggw8+YODAgUaGIwR4wv2aDTgAujgp2dFUE+agB48/SCjeS+72UIbWlBYuXMgrr7zCX//6V5KTk6O767pcLux2u5Ghid4qUlOq08M/f12alDR7ZJslD7ouk3KbGZqUli5dCsCMGTNaHC8uLmb+/PndH5AQzTUlJVxTSunimpJJURicrpE+Miv+KxH0UIYmJb3Lxu8LEaOmGgC+OX4YU3MGY9G6sIcjMik3L8VGXpYlPDtXJM7dNyESQlN4OWfFno7donZt7SUyKRcAv4zobiZJSYiTeWrC/9pTu+d6ZgdN/iClR49R4/Z1zzUTnCQlIZr5m8Dv4Uidh7/tbmT30fquv6bZzuHqJj7Ytp+9FQ1df70eQJKSEM0iTbfqgIV9x/3UNfm7/poWJ2ZVwRx0y1ilCElKQjSLdHJX60kApDstXX9NS1J4rFLIg1ummgCSlIQ4oakaXdepDoaHA3RPUnJEa0pun2xPBpKUhDihqRpvIESjmoxqUrpniVqLE7NqwhxsotErNSWQpCTECZ4amvxBPFoKaQ5z9+zDZknCopowh5pokuYbIElJiBPcx8NJyZxKWnc03SA8KVczYQm6cfuCMqCYBFklQAjD+dzgcxMK6QRtaWQlddNqFBYnmklhSJqJ7JF9COmg9vLZJpKUhABwHwOgX24u//6N4QS6a8a+xYlJUeiXrEG2DWT+mzTfhADAfTz8rzMTRVG6Zk3u1qgWUCN1A79smAGSlIQIa4wkJUfr68N3GUUBSxJN/iAHj1TKVBMkKQkR5j5GRb2Hv+3xsamkqnuvbXZwqNrNmq0l7KuUqSaSlIQAcB+n0RugIpiEN9DNt+ZPGqsko7olKQkBAS946mjwBnFrqWQnd/P+axZndKySDKCUpCQEuI8T1HVqAhaCqo3c1O5OSkloMtUkSpKSEA0VNHoDuM2pJFk1kq3dPFLGmoxFNWEJNtIozTdJSkLQUEG9J0CDOZPcVFvXbT7ZFmsyZs2ENdBIo1dqSpKUhGg4Sr3Hj9uSQa7LgF10rClYtXBNqckXxB8MdX8MCURGdIveTdeh4SgOi4YtLZe87u5PArCGVyUYnGoic0QGvX1MtyQl0bs1VUPQT0FmCgVTx4LJgMaDZkVRzeSlKuSlK9Bdo8kTVO9+90I0HA3/m5RlTEKC8Khua0r4a283rAue4CQpid6t4ShuX4CgI8vYOKzJeAJBDh05SkVd795uydCktH79eq666iry8vJQFIU333zTyHBELxSqO8L2sjr+sttPVaOB886syVTWefl4x36+LKs1Lo4EYGhSamxsZMyYMfzqV78yMgzRW+k6jcdKCYR0mmzZpHblFt1nY03GopmwBBqp9/TuYQGGdnRfccUVXHHFFUaGIHozdxV19fWEFJW0Pvnds/xtW5qTUrCRml4+VqlH3X3zer14vd7o87q6OgOjET1e3WFq3D4aLVn0z0g2NhZrChbNhDXYQEMvryn1qI7uxYsX43K5oo/8/HyjQxI9mLfqEPWeAPWWbAZmOY0NxubCopqwBupx+4IEevEAyh6VlIqKiqitrY0+SktLjQ5J9GBV5V+jA5b0ft2zndKZ2FxoqoIVH2rIS10vri31qOab1WrFau2mBd3FuS3gpe54OQBZfQcZHAzhAZQWJzatFlugnhq3r3s2w0xAPSopCRE3dYfJT7Vhc6bizM81OpowWyp906qZNdBCdooB010ShKFJqaGhgb1790afl5SUsGXLFtLT0ykoKDAwMnHOqz6Aw6LRv2A4dNd2SmdjTyXDaSUjKQDdvXxKAjH0nX/22WfMnDkz+nzRokUAzJs3j+XLlxsUlegVag6E/03tb2wcJ7Olhv9tqjEyCsMZmpRmzJghO4KKbhfwuinZt4d0h5k0V0Hi3O2xpxIM6VRVHqUyuY6ReSlGR2SIhPn/EKK7lB3YzbF6L7tqzSh2l9HhnGBLxR8M8eW+A7y/82iv/YMtSUn0OhUHdwOQnDu4+1eZPBNHOlbNhD1YTzDop76XjuyWpCR6FX8whPtIOCn16T/c4GhOYUlCMduwagp2fy21br/RERlCkpLoVUoOHMTsrcFiNpNVkGBJSVHAmYndrGL3V3PcyFULDCRJSfQqh/dtA8CVOxjFnIBjgRyZOCwadn8Nxxu8Zy9/Duq9gyGE4XyBEIdrmjje4CXVYWZIdtdOiq11+/Ed3YUdyBl8fpdeK2bOTOwWFUdDNccbemdNSZKS6HZuX4CNJVXsKKvDFwhPPL1gQFo0KYVCOl8dqWd4TnJclxPxNNaSpxxHs5tJyh0Wt/PGlSMDhyXcfDvQ6EXX9cTqjO8GkpREt9pZXsfaXZV4/OFNF1PsZvJcNvqnn5ilv7eygdXbj/DFoRouG9mHjDiNuO7j+Zo+eSkEkvPAnhqXc8ZdpE9pVJqf88b0MToaQ0hSEt3mwz2VfLa/GkuggWGmI1yQ4SNLc6N4vbDXB7vDt8BT631MrKin6YiNT/encP7IUfQdMhpsnRxMePRLALTcUZ19K13HmoLJ6iRbcYOpDhSD13kygCQl0W1SA5WMqHiX82zH6ZdmR6lvvVmSbYbUHJU9R6uoq6+gdONezCXvkz14LAyaAbaODXjUdZ1d+/YxpLYcTTND1ojOv5muoiiQnAvH90H9EXD1NTqibidJSXS9YABK1jHqyCYGZgRIsjkhpS+k5oM9HcwO0CxgOvHjaNFDDPc08PnO3VQf2sXXFUdB/5zsY7uh8DLIHdPuy5dWNbFr81oammoYN/4bqBZHV7zL+EnOwVexh8Mle6kKDmJC/zSjI+pWkpREl9px8ChDK/+BVncQBUjqPxYGXAz2s/+iqS64IHsYa3d9g6379lJX8wlTbR7sX/0d6srDyekse7Xpus6mPaXkNu4hM8WKWjAxPm+sKyXn4Q0E+bpsL3tCFzC+ILVXdXZLUurhKuqa2L1rB57ynajeGqwWK+m5Axh43kSszlRDY/vyQCVH1hcTUKo5Lz8TdeQ1kDW0Q+dQFIUZw7JAgUzHSOzB7bD/Qyj7F/gbYcQ1oLb9Y/z1sUaU0k1oBMnpVwiuHrCEcnIOTouGM1CNr6mR2iY/qY7es+CbJKUe7NChUrave40UTxknrzBdfWwXdTs/oP+YGfQZPQtMarfHVnq8gdJPVpDqO4YrKx11/PchOSemcymKwsxh2ZFnF4Ezk9D2NzFV7gb/n2DUd6CVgZD+YIiPt3/N4PovyXXZsQ2ZEe6zSXTWJEzOTJIsdbi85ZTVDOhVSUlGdPdUlbvJ2/cnckJHSU92MHDUFAqnXk/mmMsJJucRCvixlW2Ef/0RfI3dGlqN28cX694g1X2A9GQHBdNiT0itcbsG8zf9Yo55gJqD8K//B57TN3D8eE8l2Yc/wK7q5PUfChmD4xZDl0sbQIrdTIqnjINV3fv/ZzSpKfUw3kAQS8U2lN3vYNJ1zht5PtrIb0fH3WQCA8fOpGzvFlyV66CuLJyYRs/tlrE5Hn+Qj9e/R0b1FyRZNQZNvR4lNb5Npi9KaykJZFGpz+Aq70fkcAw+/38w4tuQNgCA0uONVG1dRa7nEAPyUtFGzO4ZtaRmaf1x2TfgOnaYPcfdvWoQpdSUepAmX5D33n+X/Z/8BV0PQd44tHE3npZsNE2lYPgEmDAfbC68dZUcWvP7Ll/RMBTSWb9xI2mH12LVTAyefAXmLhgTNHlgOiNyk2kwZ7BSv4QdtRa8jTWw5VXY+mcoWU/fktc5L7Sbvql20sddA86MuMfRpVILSLabSQrWEmysorK+98yDk6TUQ3j8Qdas+wcZhz6gqsGLt894GPqtM/cXOdLxjb6ZzcdUSsuPcnDNi+Ct77IYG6qO4Pr672iKzqBRk3EMmdYl1zGZFL51Xg5TBmfgMyfzD/u3WFWdxxeHaqgu3Qn7P8ZUX0Zhbhr5F14HOQk6z+1MzHZM6QNx2c1ke0p61YoB0nzrAXyBEGvWrSHt4D8wq1A4fjq2Ed9qV3PE4nRhm3ATdf9czuHyctR1y+g74w6wxHnzRV8jKXvfZGyujXrrAFLHXtOlzSVFUfjGoAwK0h18tPcYJaaLKfOfj+qqJC3TBPZUTNkjOz8K3EjZI+h/dC+DUmox5/bg99FBkpQSXCAYYs1HH+I6sBrNBIVjp5Fyfsf6R8YP7c+GwPfxbvoDpaWlKOuLyZt+O5jtcYnR42nCtv01aKrBmpyJdfxNZ7xNH095qXauvyCfeo+fqkYfyTYznCv7pWUOxWa1QNOxcN9gSp7REXULab4lsEAwxLpPPiH567fRTDBk9BRcY74dUw1k8oiBmMbdhE+1c/DAfsrXvQjehk7HWFJRx/o3/5eqIwfCSW709WDAiOlkm5n+Gc5zawNHsx2yI1NiSjdyrJesryRJKYEd2/Mpjj1/RSXEoJEXkDZuTsxNIkVRuHj0UJSxN+FVHRw4sJ/Kj5Z1qvN7V1k1u97/A8mNBzjmDsL53wVHesznE63In4yu6+zY+il/Wfc5R2o9RkfU5SQpJaJQEPZ9QE75BwzOdDD4/ClkTvzuWadUnI2iKFw8ZjiMvRmTPYU06mHzcqgq6Vh4IZ0NXx1g/wcv4nIfIC3ZzuCZ88DVr1PxiVYkZaNkD8eiKgyq+pB/7CgnGDq3dzlJiKT0q1/9igEDBmCz2Zg8eTKbNm0yOiRDhEI6O3fvpv6fy+HgRgCyz5tB5gXXxm1UtqIoTBszlPOv+k80Vx74m9C/WEHNlr+Bv+msrz9a28Q7a9fh3fAiKd4j9ElLZuis21Eze9DAxJ5myCwKslNJC1biLF3LmnN8+yVFN/jd/elPf+KWW27hN7/5DZMnT2bJkiW89tpr7Nq1i+zs7DO+tq6uDpfLRW1tLSkpPffuRCAQpGT/Xsq+/AjL8a9wWEycV5CNNuJKyO7Cxe2DAdj7Hke+2kDJsUaSnU5SBo4nc9BYHBn5LWtmPjfeir18uHY1Tu9RVJNCQX4+OVNu7nljgHqiip1Uffpndh+pp8I5FOvwy5h+Xj42c/dPIeqIWH5HDU9KkydPZuLEifzyl78EIBQKkZ+fz3/+53/y4IMPnvG1PTIpBf3gd1Nx7BjHjx6m/tghvBV70Xx1AGgmhYxBYxkw8UpMjtRuCemfmz+nacdq7L7j0WMmVSNoSUE3aVxU4ABfA+g6u47WYzKp5I2egbNwGqjmbolRAIc3c3Tz/1FS2YDfZKM2pZCJ48bTJzcfrMnoiinhRn3H8jtq6JAAn8/H5s2bKSoqih4zmUzMmjWLf/7zn6eV93q9eL0n7kDU1dW170JVJbD3H9GneyrqqWr0QYt8rKProKAzoX86WqSSsK+ygcr6SOditPyJ140vSMWimkDX2X+8kSN1nhPlTjq/Dkzo58SihI81VDZQExmlqwFms0ZK/vnkj56OPaN7+2amTBhP7fBR7Nv9BfX7/4VWUwJBP0pTFQoQ9KSjRrb/GTLpItS+48Da+1ZENFzfCfRxZGD54i0OlpZirtlG2teH4ED4h3VftZ+jDQFUVUNXVBTFhK4oKCgoCowvSMOshpNWybFGKs5wN29Mv1RskV+Cg1Vu7BaVrFOXJR49t0vGgRmalI4dO0YwGKRPn5ZrEffp04evvvrqtPKLFy/m0Ucf7fiFAl5oPBZ9qjfUE2pjhKwO0MSJpktTA3rTGW7FejVQw2V1byO6t+27IwG/BYtFA5OK05WGzeHE7soho18hmfmFKFp81qKOhctpYfy4iTBuIj5/gIba4wSbajArOkpSMjjSwOIgsRsLvUDaANKm/YDUqn1UHfgSS/AYuKtAD6H7PZj8PvTIHpYn/wnVARoDJ35WGxsInWHqitLoBy38v603NKJbVFBOWYlBD8X1rTXrUYMni4qKWLRoUfR5XV0d+fntmOyZmg9jbojeTs9o9OH066fdXW+u+ppS7GBSAIVMjx+nL4gS2VVDidwbaC6rOi3RBJblC5DkD6KggKKghAvS/EqLwwY2J6gW+igKibosvMWskZ7ZBxI2wl7OZELJLCQjszD8XNch4CHfXU+214vfHyCoByEYRI+0AHQdTCnWyM81pLv9WL3BNi+hplijt8FSm/xoqgksp/xJivesgAhDk1JmZiaqqnL06NEWx48ePUpOzulLXVitVqzWGGoTFiekD4w+zejA6qIuF7R3ReikyEOIbqUoYLZjc9lp7/aaqWmQ2s6yHVsRvfMMHRJgsViYMGEC77//fvRYKBTi/fffZ8qUKQZGJoQwiuHNt0WLFjFv3jwuuOACJk2axJIlS2hsbOTWW281OjQhhAEMT0pz586lsrKShx56iCNHjjB27Fjeeeed0zq/hRC9g+HjlDqjR45TEqIXieV3NCGmmQghRDNJSkKIhCJJSQiRUAzv6O6M5u6wdk83EUJ0q+bfzY50XffopFRfH14Ev12juoUQhqmvr8flat8wzB599y0UClFWVkZycnLCzY7uLs1TbUpLS+UOZAzk8+ucs31+uq5TX19PXl4epnYuUtija0omk4l+/WS1Q4CUlBT5peoE+fw650yfX3trSM2ko1sIkVAkKQkhEookpR7OarXy8MMPx7Z6gpDPr5O64vPr0R3dQohzj9SUhBAJRZKSECKhSFISQiQUSUpCiIQiSakHqqqq4uabbyYlJYXU1FRuv/12GhoazviaGTNmoChKi8e///u/d1PExuroDsyvvfYaw4cPx2azcf755/P3v/+9myJNTB35/JYvX37az5nN1t6VwyN00eNcfvnl+pgxY/QNGzboH374oT5kyBD9xhtvPONrpk+frt955516eXl59FFbW9tNERtnxYoVusVi0ZctW6Zv375dv/POO/XU1FT96NGjrZb/+OOPdVVV9Z///Of6jh079P/6r//SzWazvm3btm6OPDF09PMrLi7WU1JSWvycHTlypEPXlKTUw+zYsUMH9E8//TR6bNWqVbqiKPrhw4fbfN306dP1e++9txsiTCyTJk3SFy5cGH0eDAb1vLw8ffHixa2Wv/766/Urr7yyxbHJkyfrd911V5fGmag6+vkVFxfrLperU9eU5lsP889//pPU1FQuuOCC6LFZs2ZhMpnYuHHjGV/78ssvk5mZyahRoygqKsLtdnd1uIZq3oF51qxZ0WNn2oEZwp/vyeUBvvWtb7VZ/lwWy+cH0NDQQP/+/cnPz+eaa65h+/btHbpuj56Q2xsdOXKE7OzsFsc0TSM9PZ0jR460+bqbbrqJ/v37k5eXx9atW3nggQfYtWsXK1eu7OqQDdPRHZgh/Pm2Vv5Mn+25KpbPb9iwYSxbtozRo0dTW1vL008/zYUXXsj27dvbPXleklKCePDBB3nqqafOWGbnzp0xn3/BggXRr88//3xyc3O59NJL2bdvH4MHD475vEKcbMqUKS32bLzwwgsZMWIEv/3tb3n88cfbdQ5JSgniRz/6EfPnzz9jmUGDBpGTk0NFRUWL44FAgKqqqlZ3FW7L5MmTAdi7d+85m5Q6ugMzQE5OTofKn8ti+fxOZTabGTduHHv37m33daVPKUFkZWUxfPjwMz4sFgtTpkyhpqaGzZs3R1/7wQcfEAqFoommPbZs2QJAbm5uvN9KwohlB+YpU6a0KA/w3nvv9codm+Oxg3UwGGTbtm0d+znrVDe5MMTll1+ujxs3Tt+4caP+0Ucf6YWFhS2GBBw6dEgfNmyYvnHjRl3XdX3v3r36Y489pn/22Wd6SUmJ/te//lUfNGiQPm3aNKPeQrdZsWKFbrVa9eXLl+s7duzQFyxYoKempkZvU3//+9/XH3zwwWj5jz/+WNc0TX/66af1nTt36g8//HCvHxLQkc/v0Ucf1VevXq3v27dP37x5s37DDTfoNptN3759e7uvKUmpBzp+/Lh+44036klJSXpKSop+66236vX19dHvl5SU6IC+Zs0aXdd1/eDBg/q0adP09PR03Wq16kOGDNF//OMf94pxSrqu6y+88IJeUFCgWywWfdKkSfqGDRui35s+fbo+b968FuX//Oc/60OHDtUtFot+3nnn6W+//XY3R5xYOvL53XfffdGyffr00WfPnq1//vnnHbqeLF0ihEgo0qckhEgokpSEEAlFkpIQIqFIUhJCJBRJSkKIhCJJSQiRUCQpCSESiiQlIURCkaQkepTly5eTmpoaff7II48wduzY6PP58+czZ86cbo9LxI8kJdGq+fPnt7mO98KFC1EUpcWqBl2RDAYMGMCSJUtaHJs7dy67d+9u8zXPP/88y5cvjz6fMWMG9913X1zjEl1LkpJoU35+PitWrKCpqSl6zOPx8Morr1BQUGBITHa7/bRF7k7mcrla1KREzyNJSbRp/Pjx5Ofnt1idcuXKlRQUFDBu3LhOnbu1GsycOXOita8ZM2Zw4MABfvjDH0Z3xYDTm2+nOrnGNn/+fNatW8fzzz8fPUdJSQlDhgzh6aefbvG6LVu2oChKh9b9EV1DkpI4o9tuu43i4uLo82XLlnHrrbd2+XVXrlxJv379eOyxxygvL6e8vLzD53j++eeZMmUKd955Z/QcBQUFp70ngOLiYqZNm8aQIUPi9RZEjCQpiTP63ve+x0cffcSBAwc4cOAAH3/8Md/73ve6/Lrp6emoqkpycjI5OTkxrfzocrmwWCw4HI7oOVRVZf78+ezatSu6f5nf7+eVV17htttui/fbEDGQ5XDFGWVlZXHllVeyfPlydF3nyiuvJDMz0+iwOiUvL48rr7ySZcuWMWnSJP7v//4Pr9fLddddZ3RoAqkpiXa47bbbWL58OX/4wx/iVpswmUycupSX3++Py7nb44477oh24hcXFzN37lwcDke3XV+0TZKSOKvLL78cn8+H3+/nW9/6VlzOmZWV1aKfKBgM8uWXX7YoY7FYCAaDnbpOW+eYPXs2TqeTpUuX8s4770jTLYFI802claqq0e2dVFVts1xtbW10Q4JmGRkZ5Ofnn1b2kksuYdGiRbz99tsMHjyYZ599lpqamhZlBgwYwPr167nhhhuwWq0xNRsHDBjAxo0b2b9/P0lJSaSnp2MymaJ9S0VFRRQWFvbKjQESldSURLukpKSQkpJyxjJr165l3LhxLR6PPvpoq2Vvu+025s2bxy233ML06dMZNGgQM2fObFHmscceY//+/QwePJisrKyY4r7//vtRVZWRI0eSlZXFwYMHo9+7/fbb8fl83XI3UbSfrNEteq0PP/yQSy+9lNLS0tN2gRXGkaQkeh2v10tlZSXz5s0jJyeHl19+2eiQxEmk+SZ6nVdffZX+/ftTU1PDz3/+c6PDEaeQmpIQIqFITUkIkVAkKQkhEookJSFEQpGkJIRIKJKUhBAJRZKSECKhSFISQiQUSUpCiITy/wFYDYW9A5c30QAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:13:19.460237Z", + "iopub.status.busy": "2024-02-29T03:13:19.459915Z", + "iopub.status.idle": "2024-02-29T03:13:19.809971Z", + "shell.execute_reply": "2024-02-29T03:13:19.809006Z" + }, + "papermill": { + "duration": 0.373697, + "end_time": "2024-02-29T03:13:19.811952", + "exception": false, + "start_time": "2024-02-29T03:13:19.438255", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASUAAAEmCAYAAAAk84bZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+QElEQVR4nO3deXxU5b348c+Zc2bNMtlJIgkJO8guQnFBXOqCtVrvVatUwb0Vb0u93Cr39merraX2qpW2Xu1tC9FbKFdba9tblbogKAoqCoIgSwghEEI2skxmnzm/P2YyEEjCZDLJTJLv+/WaF87MmTPfGZNvnuc53+d5FF3XdYQQIkkYEh2AEEKcTJKSECKpSFISQiQVSUpCiKQiSUkIkVQkKQkhkookJSFEUpGkJIRIKlqiA+iNYDBIdXU1aWlpKIqS6HCEEKfQdZ3W1lYKCwsxGKJrAw3opFRdXU1RUVGiwxBCnEFVVRXDhw+P6tgBnZTS0tKA0AdOT09PcDRCiFO1tLRQVFQU+V2NxoBOSu1dtvT0dElKQiSxngyvyEC3ECKpSFISQiQVSUpCiKQyoMeUoqHrOn6/n0AgkOhQBjVVVdE0TUozRK8N6qTk9Xo5evQoTqcz0aEMCTabjYKCAkwmU6JDEQNYwpPSkSNHePDBB3nttddwOp2MHj2aVatWMXPmzF6dNxgMUlFRgaqqFBYWYjKZ5K94H9F1Ha/XS11dHRUVFYwZMybqQjkhTpXQpHT8+HHOP/98Lr74Yl577TVyc3PZt28fmZmZvT631+slGAxSVFSEzWaLQ7SiO1arFaPRSGVlJV6vF4vFkuiQ+t/hrbiOfMaH/rFYCsYzqzRL/hDGIKFJ6fHHH6eoqIhVq1ZFHistLY3re8hf7P4zpL/rlqOw7x8cqG5G9ezlw9ZU7DYj4/Olfq6nEvpT9Ne//pWZM2dyww03kJeXx/Tp0/nNb36TyJCEiM3R7bR5/bS4/ai6jxxnOTsONyc6qgEpoUnpwIEDPPvss4wZM4Z169bxrW99i29/+9s8//zznR7v8XhoaWnpcBMiKTQeoNnlo9kyHJtJJcNVRXWTG68/mOjIBpyEdt+CwSAzZ87kJz/5CQDTp09n586dPPfccyxcuPC045cvX84jjzzS32EK0T13M7ibcXiDHEmfypRAI9lqG5OmF6IaZEyppxLaUiooKGDixIkdHpswYQKHDh3q9Phly5bR3NwcuVVVVfVHmP1u3rx5LFmyJNFhiGg1HwFgzMhRXDN3FtnpNoanKhRZXJKUYpDQltL555/Pnj17Ojy2d+9eRowY0enxZrMZs9ncH6EJEb22WgCUtALy7CmQURhKVI5aSMlJcHADT0JbSt/97nfZvHkzP/nJT9i/fz9r1qzhv//7v1m8eHGfvq/XH+zy5g8Eoz7WF8WxPbVo0SI2bNjAihUrUBQFRVEYPnw4zz77bIfjPv30UwwGA5WVlQA89dRTTJ48mZSUFIqKirjvvvtwOBwdXvPee+9x4YUXYrVaKSoq4tvf/jZtbW09jlGcwtkQ+teWHfnXHwxSebiKTw8dT1xcA1RCW0rnnnsuf/7zn1m2bBmPPvoopaWlPP300yxYsKBP3/eZ9fu7fK40J4Xrpp8Vuf/fG8vxBTrf2Xx4ppUbZp5YZG7lpgpc3o7TWb775bE9im3FihXs3buXSZMm8eijjwLw5JNPsmbNGr71rW9Fjlu9ejXnn39+pFVpMBj4xS9+QWlpKQcOHOC+++7je9/7Hv/1X/8FQHl5OVdeeSU//vGPWblyJXV1ddx///3cf//9HUoyRAzaGmhs87DjiE6hqZUxthz8AZ0de8upaBrJ1OEZGKQbF7WEF5Z85StfYceOHbjdbnbv3s3dd9+d6JASym63YzKZsNls5Ofnk5+fz4IFC9i0aVNkrC0YDLJ27doOyXvJkiVcfPHFlJSUcMkll/DjH/+YF198MfL88uXLWbBgAUuWLGHMmDGcd955/OIXv+CFF17A7Xb3++ccNIIBcB2n2eVn53GNmhY32LIxawZS/M34gzqtbn+ioxxQEj7NJBEWXzy6y+dO/YN2z9xRXR57arHuHefHt/Cz3bRp05gwYQJr1qzhoYceYsOGDdTW1nLDDTdEjnnzzTdZvnw5X3zxBS0tLfj9ftxuN06nE5vNxvbt2/nss89YvXp15DW6rkem40yYMKFPYh/0XMdBD9IWVPGqKWTaTGALVXLbaQVdp8Xtw24zJjrSASPhLaVEMGmGLm+aaoj6WGMUx8bLggULWLNmDQBr1qzhyiuvJDs7NIZx8OBBvvKVrzBlyhT+9Kc/sXXrVp555hkgNN0GwOFwcO+997Jt27bIbfv27ezbt49Ro7pOvOIMXKExo1ZSQVGwW41gDlVxW1QdLeihxe1LZIQDzpBsKSU7k8l02lIrt9xyC9///vfZunUrf/zjH3nuueciz23dupVgMMiTTz4ZmepxctcNYMaMGezatYvRo7tuJYoYuFvQ0WnWQ/Mr0ywaqBqYUjBrDswBh3TfemhItpSSXUlJCVu2bOHgwYPU19cTDAYpKSnhvPPO48477yQQCPDVr341cvzo0aPx+Xz88pe/5MCBA/zP//xPh6QF8OCDD/L+++9z//33s23bNvbt28df/vIX7r///v7+eIOLpxl/QMdpSAEg1Rz+O29Jx6wZMPtbaXFJS6knJCkloaVLl6KqKhMnTiQ3NzcywL1gwQK2b9/O1772NaxWa+T4qVOn8tRTT/H4448zadIkVq9ezfLlyzucc8qUKWzYsIG9e/dy4YUXMn36dB5++GEKCwv79bMNOu4WPP4gXjWVFLN6ovtvsWPWVEzSUuoxRdf1zq93DwAtLS3Y7Xaam5tP283E7XZTUVFBaWnp0FxGIwGG5He+9XmO1xzkr/7ZmAsm8vVZxaHH97+F9+BmGjOnYZnwZTJsQ3Phu+5+R7siY0pC9IanhUybiQXTJ+OxDjvxuMWOSTWQb3bDEE1IsZKkJESsggHwhiriNVsGmumkX6fwFTg8rQkIbGCTMSUhYuVpAV0HgwbGU1Y3NYUGvo81NLLlQAMOj4wrRUuSkhCxcofW89rXorBu1zHqHZ4Tz4WTUnVtPe/vr+d4mzcREQ5IkpSEiFW4a3bUZWRXdUvHCdim1NA/ShBV9+L0yhZf0ZKkJESsvG3o6LTpoeV0Uk4eU1I10Myhyv+AizavdN+iJUlJiFh5HQSCOi4lVDNmM6sdnzelYlQNmAJOnB5pKUVLkpIQsfK24Qvo+FRbp3MhMaVgVA0YA05pKfWAJCUhYuVtwxcI4lWtpJjU0583pWBUFYxBF05JSlGTpJSEkm2N7mSLJ2l4HfgCQXyqDZu5k5I/UyqmcPetTbpvUZPiyUHK6/ViMkklcZ8Kd9+8mq3jIHc7UwqpFo25uTbUswv6P74BSlpKSaazNbrLy8u58847KS0txWq1Mm7cOFasWHHa66677joee+wxCgsLGTduHADvv/8+06ZNw2KxMHPmTF555RUURWHbtm2R1+7cuZOrrrqK1NRUhg0bxq233kp9fX2X8Rw8eLC/vo7kFQyAz0V+uoW7Lp3EpRPyTj/GlIJmMJBr9pOVIn8gojW0Wkq6DoEELCOhGk9fprILna3RnZmZyfDhw3nppZfIzs7m/fff55577qGgoIAbb7wx8tq33nqL9PR03njjDSA0GfKaa65h/vz5rFmzhsrKytO6YU1NTVxyySXcdddd/PznP8flcvHggw9y44038vbbb3caT25ubhy+lAEuPL0ExYBmTkHr7P9vuIASr+P050SXhlZSCvjg3Sf7/30v/FfQovtLeeoa3e1O3oSztLSUDz74gBdffLFDUkpJSeG3v/1tpNv23HPPoSgKv/nNb7BYLEycOJEjR450WAf9V7/6FdOnT49sCAqwcuVKioqK2Lt3L2PHju00niGvPSmZUrr+gxOeelLTcJxDFY2cXZhOSmdjT6ID+YYGiGeeeYaVK1dy6NAhXC4XXq+XadOmdThm8uTJHcaR9uzZw5QpUzosIzJr1qwOr9m+fTvr168nNTX1tPcsLy9n7Nie7cYyZIST0heNAQ7urGFWadbpXTRjqH7pSG0jmwx1DM+0SlKKwtD6hlRjqNWSiPfthbVr17J06VKefPJJ5syZQ1paGv/5n//Jli1bOhyXkpLS43M7HA6uueYaHn/88dOeKyiQwdkuhbtk1U6N3UdbmFGccfox4ZaSyRDEoPtx+eQKXDSGVlJSlKi7UYl06hrdmzZt4rzzzuO+++6LPFZeXn7G84wbN47f//73eDyeyM7CH330UYdjZsyYwZ/+9CdKSkrQtM5/HDpbM3zIC08xcYSnmFg6q1NSjWDQMKoKWtBz2p6AonNy9S0JnbpG95gxY/j4449Zt24de/fu5f/9v/93WnLpzC233EIwGOSee+5h9+7drFu3jieeeAIAJTwOsnjxYhobG7n55pv56KOPKC8vZ926ddx+++2RRNTZmuFDnreNQFDHawh10azGTpKSooDRgmZQMAbduKWlFJWEJqUf/vCHkcvM7bfx48cnMqSkcOoa3VdccQXXX389N910E7Nnz6ahoaFDq6kr6enp/O1vf2Pbtm1MmzaN//iP/+Dhhx8GiIwzFRYWsmnTJgKBAJdffjmTJ09myZIlZGRkRHZG6WrN8CHN58Qf0PGp1s6nmLQzWtFUA1rAjdsnyTwaCe++nX322bz55puR+111IYaSsWPH8sEHH3R4bNWqVadtr33y5gBlZWWdnuu8885j+/btkfurV6/GaDRSXFwceWzMmDG8/PLLPYpnyPO58AWD+AwWLJ21ktoZbWgGBS3oljGlKCU8A2iaJpea+9ALL7zAyJEjOeuss9i+fXukBunk3VBEDMItJb/B0nnXrZ3RGp6UK0kpWglPSvv27aOwsBCLxcKcOXNYvnx5h7/iJ/N4PHg8J1b3a2lp6a8wB6yamhoefvhhampqKCgo4IYbbuCxxx5LdFgDn8+FP6jjVy1YTd2MghhtZNiMXJqXhmlMTv/FN4AlNCnNnj2bsrIyxo0bx9GjR3nkkUe48MIL2blzJ2lpaacdv3z58g5FhOLMvve97/G9730v0WEMLroOPhe5qWbunD0Rn/H0n9UIoxWzppJn1WVXkygldKD7qquu4oYbbmDKlClcccUVvPrqqzQ1NZ225XS7ZcuW0dzcHLlVVVX1c8RCEJoZEAwtRaKaU7ofU9LC3WSfsx8CGxwS3n07WUZGBmPHjmX//v2dPm82myP1NkIkTHuCUbUzF8YarQR1ncO1DRy1NjCrNCtSjiE6l1R1Sg6Hg/Ly8rhWEg/gDYAHnCHzXftcABxs0Xl951GONLm6PjY81WRfVS3vlzfg8UtZwJkkNCktXbqUDRs2cPDgQd5//32+9rWvoaoqN998c6/PbTSG/oI5ndJs7i/t33X7dz9ohVtK9W6V3UdbcXa3p5vRhkFRMBO6QCNV3WeW0O7b4cOHufnmm2loaCA3N5cLLriAzZs3x2VpDFVVycjIoLa2FgCbzSbN5j6i6zpOp5Pa2loyMjJQ1W7GWAaDcFJyERq47r5OKVzxTWjfN5cvQGbfRjfgJTQprV27tk/P317/1J6YRN/KyMgYGjVn4e6bs33e2xmKJwGMhgCGoE+6b1FIqoHueFMUhYKCAvLy8vD5ErC42xBiNBoHfwupnc+Jruu49FA31drZZNx2qhEMKprBgBb0yPy3KAzqpNROVdWh8wsj+l64cNJrCM0f7LaiW1FAM4enmnikpRSFpLr6JsSA4HOGqrkNFkyaAdVwhrFKzYranpSkpXRGQ6KlJERc+Vz4g8HQFJPuWkntjBYK7Bbmj8oktTC97+Mb4CQpCdFTPhdpZiMLzh2PN63zeZodaBZsJg1bCmAZ5OUScSDdNyF6KlwSoJpTuh/kbqeF10j3e7o/TgCSlIToGV0Hnzv038Yol3/RLHj8Acqr69h9VFa2OBNJSkL0hN8NepDaVjfr9jSxvzaKPd2MFly+ADsOHuPjyuN9H+MAJ0lJiJ4IF042ew3sOuaisc175tdoFjSDATXolatvUZCkJERPhMeT3Eqomjuqq2+aJbIkrtQpnZkkJSF6ItxScofnvXW76mQ7zRKuU/Li9QcJBIfIagoxkqQkRE+0T8bVo5iM285oiVR0A3j80oXrjiQlIXrilMm40XXfrCiKglkJzb/0yFZL3ZKkJERPhCfjOvX27ls0SSmUwMx4QddxS0upW1LRLURPtO9iEp6Ma9Gi6b6F6plG59jIm5ZHVopsINAdSUpC9ITPhVE18E9fGosnbySGM03GBTBoYFCxW8GeboBoEtkQJt03IXoiMsXEhs0U5d90RTkx1aS9Glx0SZKSED0RHuiOeopJO82Cw+Nn9+FaqrvbaEBIUhKiR3xOmlxe3trfymeHm6J/ndFCg8PDh3uro5uaMoRJUhIiWsEg+D04vQF21ft71uLRLGhqaKqJLInbPUlKQkTL7wJdxx/Q8RvMmKOpUWoXqeqWqSZnIklJiGiFx5M8mEAxRFc42U47UdUtLaXuSVISIlqnTMaNaopJu8hUE6+0lM5AkpIQ0Tp1Mm6PWkpWNFW2WYpG0iSln/70pyiKwpIlSxIdihCda28p0d5S6sGvj2ZGNSioss3SGSVFRfdHH33Er3/9a6ZMmZLoUIToWril1L5CQI9aSkYrJs3AOWdZmDSlsC+iGzQS3lJyOBwsWLCA3/zmN2Rmyi7rIomFW0qXTinlnrkjyU41R/9azYyqKORZFYqybH0U4OCQ8KS0ePFirr76ai677LIzHuvxeGhpaelwE6LfhFtKqtlGilk78yaUJ4vsaCLTTM4koUlp7dq1fPLJJyxfvjyq45cvX47dbo/cioqK+jhCIU4S6xQTiCSlxuYWPqs6jsPjj2Ngg0vCklJVVRXf+c53WL16NRaLJarXLFu2jObm5sitqqqqj6MU4iQ+J25fgI0VDt4vr+/Za8NJqarBwfpd1TQ6othwYIhK2ED31q1bqa2tZcaMGZHHAoEAGzdu5Fe/+hUejwdV7TiQaDabMZt70I8XIp58Ltz+AHsaA1gUB+eNyon+taoRDGqkgFKWxO1awpLSpZdeyo4dOzo8dvvttzN+/HgefPDB0xKSEAnnc4ammKiWnhVOQnj5EnN4qokHtyyJ26WEJaW0tDQmTZrU4bGUlBSys7NPe1yIhAv4we/FH9TxGWNIStBx/zdpKXUp4VffhBgQ/KFBbl8QAgZzz2qU2mkWVLW9+yYtpa4kRfFku3feeSfRIQjRufbJuIoJFKUXLSWZlHsm0lISIhrhwklveIpJVJtQnkozoxkUVF0m5XYnqVpKQiSt9ikm4aRkjmXxf6OVTJuJubkpGEtk9kJXJCkJEY1wS2n6qEImjBuJZoitpWQxqpyVqkJadLV5Q1FM3bcDBw7EOw4hklu4pWQwhXYxMWmxJCWZahKNmJLS6NGjufjii/n973+P2y1fsBgCvKGWUkxTTNppFgJBner64+w80hyfuAahmJLSJ598wpQpU3jggQfIz8/n3nvv5cMPP4x3bEIkj/B23R8ccrL+i1p8gRgGqjUL/mCQnZW1vP1FLbquxz/OQSCmpDRt2jRWrFhBdXU1K1eu5OjRo1xwwQVMmjSJp556irq6unjHKURihbfr3t8UZFtVE6rSgxUC2hlDxZNa0E0gqOMLSFLqTK9KAjRN4/rrr+ell17i8ccfZ//+/SxdupSioiJuu+02jh49Gq84hUgsnxN/UMdvsGA2GqLbrvtUmgWDAbSgD0CqurvQq6T08ccfc99991FQUMBTTz3F0qVLKS8v54033qC6upprr702XnEKkVg+F75AEJ/BgiWWcgAAzYyCgkUJrRAgtUqdi6kk4KmnnmLVqlXs2bOH+fPn88ILLzB//nwM4cukpaWllJWVUVJSEs9YhUgMXY903/yqhTRTrEkpNEhuVAIoul+SUhdiSkrPPvssd9xxB4sWLaKgoKDTY/Ly8vjd737Xq+CESAoBHwT9+ANB/JqlZxsGnEwzg6KEx5Vkp9yuxJSU3njjDYqLiyMto3a6rlNVVUVxcTEmk4mFCxfGJUghEipcOOnTFQKKMbbJuBBavkQ1haaaBL14ZPmSTsWU8keNGkV9/ekr7zU2NlJaWtrroIRIKpHJuFZQlJ5t130qo5XCDCtXjbMzPKsXNU+DWEwtpa7qKxwOR9RL2woxYIRbSiX5Odw7bSQKMVx5a6eZsVuN2DM0sBjjFODg0qOk9MADDwCgKAoPP/wwNtuJrWICgQBbtmxh2rRpcQ1QiIQ7ZYpJr8hUkzPq0Tf86aefAqGW0o4dOzCZTJHnTCYTU6dOZenSpfGNUIhE680uJqfSLLh9AY7VNGAwOmUPuE70KCmtX78eCK2lvWLFCtLT0/skKCGSSrj7tqPWR71ay4wRmditMXa9NAvNLh9b9lVjoUmSUidiGuhetWqVJCQxdIRbSgfCU0ximvfWzmhBa18SV0oCOhV1S+n666+nrKyM9PR0rr/++m6Pffnll3sdmBBJIzwZt00PDVfYYi2ehNA63eGSgDYpnuxU1EnJbrejhCch2u32PgtIiKTjc+EL6vgNZhSF2KeZQHhJXIO0lLoRdVJatWpVp/8txKDnc4aqucP7vcU0GbedZo3s/SbTTDoX05iSy+XC6XRG7ldWVvL000/zj3/8I26BCZE0fE58AR2fwdK7rhtENg/Qgl68/iDBoCxfcqqYktK1117LCy+8AEBTUxOzZs3iySef5Nprr+XZZ5+Na4BCJFQwGJ6MG8RnsMa2tdLJ2seUdA8gKwV0JuaVJy+88EIA/vjHP5Kfn09lZSUvvPACv/jFL6I+z7PPPsuUKVNIT08nPT2dOXPm8Nprr8USkhB9w+8CPbQgm1+NQ0vJaMWgKEzIMXLN1AI0tRddwUEqpvJUp9NJWloaAP/4xz+4/vrrMRgMfOlLX6KysjLq8wwfPpyf/vSnjBkzBl3Xef7557n22mv59NNPOfvss2MJTYj4Cq/NnZeVyT2zx9DrFWy10BZNw2wGhuWkQCy7ogxyMW8c8Morr1BVVcW6deu4/PLLAaitre1R/dI111zD/PnzGTNmDGPHjuWxxx4jNTWVzZs3xxKWEPHnawPAYE7BZtJIMcdpmglAwNO7cw1SMSWlhx9+mKVLl1JSUsLs2bOZM2cOEGo1TZ8+PaZAAoEAa9eupa2tLXK+U3k8HlpaWjrchOhT3lBSwhinymuDCqoRh8fPviN1NDm98TnvIBJTUvrnf/5nDh06xMcff8zrr78eefzSSy/l5z//eY/OtWPHDlJTUzGbzXzzm9/kz3/+MxMnTuz02OXLl2O32yO3oqKiWMIXInrh7tve40HWf1FLvSMOrRvNwuHjTtbvrOJQo/PMxw8xMXdo8/PzmT59eoeF3mbNmsX48eN7dJ5x48axbds2tmzZwre+9S0WLlzIrl27Oj122bJlNDc3R25VVVWxhi9EdMLdt0MOhW1VTTg9cSh4jOxqIrVKnYmpg9zW1sZPf/pT3nrrLWprawkGO36xPdlB12QyMXr0aADOOeccPvroI1asWMGvf/3r0441m82YzeZYQhYiNuGWkjM8xcRiisPAtBae/6Z7ZPXJTsSUlO666y42bNjArbfeSkFBQWT6STwEg0E8HhkAFEnC24aOjkMP/THs9XpK0GH+m2yzdLqYvuHXXnuNv//975x//vm9evNly5Zx1VVXUVxcTGtrK2vWrOGdd95h3bp1vTqvEHHja8Mf0PEqobWUYl6f+2SaJVTVHfDglpbSaWJKSpmZmWRlZfX6zWtrayObVtrtdqZMmcK6dev48pe/3OtzCxEXXmdovzc1tAml2pt5b+208JiSzyMtpU7ElJR+9KMf8fDDD/P88893WBK3p2QLJpHUdD3UUgrq+FQbtni0kgA0M6oa6r65ZKD7NDElpSeffJLy8nKGDRtGSUkJRmPHVfg++eSTuAQnREIFfBDwh1pKRmt8xpMAjFZSTCozCi3oo7Ljc85BJKZv+brrrotzGEIkIa8DgKy0FO6ZMx5fME6tGs2MWVMpTlchOyU+5xxEYkpKP/jBD+IdhxDJJ7w2t2JKwWpSsRKv7pvsaNKdmIsumpqa+O1vf8uyZctobGwEQt22I0eOxC04IRIqXKOEKc6tGc2Cjk5DUzN7alrx92bN70EopqT02WefMXbsWB5//HGeeOIJmpqagNDa3MuWLYtnfEIkTriae3+TzvovaqltjVPLJtxS2l1Vx6s7juKSZXE7iCkpPfDAAyxatIh9+/Z12BF3/vz5bNy4MW7BCZFQ4cm4h9tgW1UTDrc/Puc1WlBQMOMDXZdapVPElJQ++ugj7r333tMeP+uss6ipqel1UEIkhXD3rS0YquZO7e2yJe3CLSWjQUfVfVKrdIqYkpLZbO502ZC9e/eSm5vb66CESAreVnRdp1UPJRFbvJKSQQODimowhKeaSEvpZDElpa9+9as8+uij+Hw+ABRF4dChQzz44IP80z/9U1wDFCJhPA58QR2PwYaiEL/iSUU5MSk36JZJuaeIKSk9+eSTOBwOcnNzcblcXHTRRYwePZq0tDQee+yxeMcoRGJ4HfgCQbyqDWtvt1Y6Vfv8t6AXt3TfOoipPWq323njjTfYtGkT27dvx+FwMGPGDC677LJ4xydEYug6eNvw+YN41RTS49V1a6eZT6wUIC2lDnr8TQeDQcrKynj55Zc5ePAgiqJQWlpKfn4+uq7HdRkTIRLG54JgAG94ikmqOU5dt3ZGKzmpZi4oTCE9T6q6T9ajpKTrOl/96ld59dVXmTp1KpMnT0bXdXbv3s2iRYt4+eWXeeWVV/ooVCH6UXiKSW5WFt+cPRZ/vDeN1MykW4ykZxkhzXLm44eQHiWlsrIyNm7cyFtvvcXFF1/c4bm3336b6667jhdeeIHbbrstrkEK0e88rQAo5tTeb0DZmfapJj5X/M89wPVooPsPf/gD//7v/35aQgK45JJLeOihh1i9enXcghMiYcItJcxpfXN+zYIvEORYYzNVsnlABz1KSp999hlXXnlll89fddVVbN++vddBCZFwnlBS2t0Q2sWkIR67mJxMs9Dm9fPh/mre2VsX33MPcD1KSo2NjQwbNqzL54cNG8bx48d7HZQQCRduKVW0hnYxcce7wFEzh1afDHjwyNy3DnqUlAKBAJrW9TCUqqr4/XGaHyREInla0dFpCU8xSbPEuSTAaEUzKKi6VHSfqsdX3xYtWtTlNkeyC4kYNLwOfAEdtxKq5k6J16qT7cI7mmhBD15/kGBQj29x5gDWo2964cKFZzxGrryJQcHjwOsPVXOnmLT4bBhwMqMNzaBgDISuvnn8QaymPrjKNwD1KCmtWrWqr+IQInmEq7m9/gA+1UZ2vLtuAEYriqJgxgt6ELcvIEkpLA7bfQoxyHgdoAfxBHS8qo3UPkpKAJoBmf92CklKQpzKHVqWx6WkgGKI3zpKJzOooJkpyrRx2Zg00i3GM79miOiDb1uIAc4TSkqlw/P51uRR6HGeYRJhtJGb5iE3W4O+SHwDVEJbSsuXL+fcc88lLS2NvLw8rrvuOvbs2ZPIkISItJQUix2LUe27sR6jTDXpTEKT0oYNG1i8eDGbN2/mjTfewOfzcfnll9PW1pbIsMRQF24pYU7v2/cx2vD4A1TXNcRvU4JBIKFtxtdff73D/bKyMvLy8ti6dStz585NUFRiyHM3o+s6Gw95CPpruWB0Dka1D/5+G63UtXrYtPswWYzkyxNltQBIsjGl5uZmALKysjp93uPxdCjQ7GydcCF6zd2MJxDkgEul9XAz88b20brzRitG1YDmdcs2SydJmqtvwWCQJUuWcP755zNp0qROj1m+fDl2uz1yKyoq6ucoxZDgacHjC+JR00i3aH23cKHRFlqnO+DG7ZWk1C5pktLixYvZuXMna9eu7fKYZcuW0dzcHLlVVVX1Y4RiSPB7wefG4w/g1VKx2/rwUr3RimYwYAxKS+lkSdF9u//++/m///s/Nm7cyPDhw7s8zmw2dznvToi4CA9yO4NGAgYTdmtfJiUbxvCOJpKUTkhoS0nXde6//37+/Oc/8/bbb1NaWprIcIQAd2hc04ENoI+TkhVNNWAMuHH7AgTjveTuAJXQltLixYtZs2YNf/nLX0hLS4vsrmu327FarYkMTQxV4ZZSix76+evTpKRZw9ssudF1mZTbLqFJ6dlnnwVg3rx5HR5ftWoVixYt6v+AhGhvKSmhllJ6H7eUDIrCqCyNrIm58V+JYIBKaFLS+6x+X4gYuZoA+PKMcVyYPwqT1ocjHOFJuYXpFgpzTaHZuSJ5rr4JkRRcoeWcFWsWVpPat62X8KRcAHxS0d1OkpIQJ3M3hf61ZvTP+xltuHwBqo7V0+T09s97JjlJSkK087nA56amxc1f97ax91hr37+n0cqR4y7e3nGQ/bWOvn+/AUCSkhDtwl23434T5Q0+Wly+vn9PUwpGVcEYcEqtUpgkJSHahQe5j+upAGSlmPr+PU2poVqloBunTDUBJCkJcYLrOLquczwQKgfon6Rki7SUnF7ZngwkKQlxgus4Hn+QNjUN1aD0zxK1phSMqgFjwEWbR1pKIElJiBPcTbh8AdxaOpk2Y//sw2ZKxaQaMAZduKT7BkhSEuIEZ0MoKRkzyOyPrhuEJuVqBkwBJ05vQAqKSZJVAoRIOK8TvE6CQZ2AJZPc1H5ajcKUgmZQGJ1pIG/iMII6qEN8tokkJSEAnPUADC8o4JtfGo+/v2bsm1IwKArD0zTIs4DMf5PumxAAOBtC/6bkoChK36zJ3RnVBGq4beCTDTNAkpIQIW3hpGTrfH34PqMoYErF5QtwqKZOppogSUmIEGc9ta1u/rrPy4cVjf373kYbh487Wf9ZBeV1MtVEkpIQAM4G2jx+agOpePz9fGn+pFolqeqWpCQE+D3gbsHhCeDUMshL6+f910wpkVolKaCUpCQEOBsI6DpNfhMB1UJBRn8npVQ0mWoSIUlJCEctbR4/TmMGqWaNNHM/V8qY0zCpBkyBNtqk+yZJSQgctbS6/TiMORRkWPpu88mumNMwagbM/jbaPNJSkqQkhOMYrW4fTlM2BfYE7KJjTseshVpKLm8AXyDY/zEkEanoFkObroPjGDaThiWzgML+Hk8CMIdWJRiVYSBnQjZDvaZbkpIY2lzHIeCjOCed4gungSEBnQfNjKIaKcxQKMxSoL+qyZPU0P70QjiOhf5NzU1MQoJQVbc5PfTfnn5YFzzJJTQpbdy4kWuuuYbCwkIUReGVV15JZDhiKHIcw+n1E7DlJjYOcxpuf4DDNceobRna2y0lNCm1tbUxdepUnnnmmUSGIYawYEsNn1e38Ke9PhrbEjjvzJxGXYuHTbsOsrO6OXFxJIGEjildddVVXHXVVYkMQQxluk5bfRX+oI7LkkdGX27RfSbmNEyaAZO7jVb30C4LkDElMXQ5G2lpbSWoqGQOK+qf5W+70p6UAm04hnit0oC6+ubxePB4PJH7LS0tCYxGDHgtR2hyemkz5TIiOy2xsZjTMWkGzAEHDmkpDRzLly/HbrdHbkVFRYkOSQxgnsbDtLr9tJryKM1NSWwwFjsm1YDZ34rTG8A/hAsoB1RSWrZsGc3NzZFbVVVVokMSA1jj0QPogClreP9sp9Qdix1NVTDjRQ16aBnCraUB1X0zm82Yzf20oLsY3PweWhqOApB71sgEB0OogNKUgkVrxuJvpcnp7Z/NMJNQQpOSw+Fg//79kfsVFRVs27aNrKwsiouLExiZGPRajlCUYcGSkkFKUUGiowmxZHBW5nEuKzWRl56A6S5JIqFJ6eOPP+biiy+O3H/ggQcAWLhwIWVlZQmKSgwJxyuxmTRGFI+H/tpO6UysGWSnmMlO9UN/L5+SRBL6yefNmyeb74nEaKoM/ZsxIrFxnMySEfrX1ZTIKBJuQA10CxEPfo+TfeX7aGjzELQn0TCBNYNAUKeu7hi7qoduuYskJTHkVFfupb7Vw55mI4rVnuhwTrBk4AsE2VleyVu7jw3ZXoQkJTHk1B7aC0Bawaj+X2WyO7YszJoBa6CVQMBH6xCt7JakJIYUXyCIsyaUlIaNGJ/gaE5hSkUxWjBrClZfM81OX6IjSghJSmJIqag8hNHThMloJLc4yZKSokBKDlajitV3nIZErlqQQJKUxJBypHwHAPaCUSjGJKwFsuVgM2lYfU00ODxnPn4QGrrFECLhvP4gR5pcNDg8ZNiMjM7r20mxzU4f3mN7sAL5oyb36XvFLCUHq0nF5jhOg2NotpQkKYl+5/T62VLRyK7qFrz+0MTTmSWZkaQUDOp8UdPK+Py0uC4n4m5rplBpQLMaSS0YF7fzxpUtG5sp1H2rbPOg63pyDcb3A0lKol/tPtrCO3vqcPtCmy6mW40U2i2MyDoxS39/nYN1n9ew/XATl08cRnacKq6HuQ8wrDAdf1ohWDPics64C48pTcr0cfbUYYmOJiEkKYl+8+6+Oj4+eByT38E4Qw0zs73kak4Ujwf2e2Fv6BJ4RquXc2tbcdVY+OhgOpMnTuKs0VPAkt67AI7tBEArmNTbj9J3zOkYzCnkKU4wtICS4HWeEkCSkug3Gf46JtT+g7MtDQzPtKK0dt4tyTNCRr7KvmONtLTWUrVlP8aKt8gbNQ1GzgNLzwoedV1nT3k5o5uPomlGyJ3Q+w/TVxQF0gqgoRxaa8B+VqIj6neSlETfC/ihYgOTaj6kNNtPqiUF0s+CjCKwZoHRBpoJDCd+HE16kPFuB5/s3svxw3s4UHsM9E/Iq98LYy6HgqlRv31Vo4s9W9/B4Wpi+owvoZpsffEp4yctH2/tPo5U7KcxMJJzRmQmOqJ+JUlJ9Kldh44xtu5NtJZDKEDqiGlQcgFYz/yLptphZt443tnzJT4r309L0/tcaHFj/eJVaDkaSk5n2KtN13U+3FdFQds+ctLNqMXnxueD9aW0Qjz+AAeq97MvOJMZxRlDarBbktIAV9viYu+eXbiP7kb1NGE2mckqKKH07HMxp2QkNLadlXXUbFyFXznO2UU5qBOvhdyxPTqHoijMG5cLCuTYJmINfA4H34XqT8HXBhOuBbXrH+MD9W0oVR+iESB/+BiwD4AllNPySTFppPiP43W10ezykWEbOgu+SVIawA4fruLzDS+R7q7m5BWmj9fvoWX324yYOo9hUy4Dg9rvsVU1OKh6fy0Z3nrsuVmoM26FtPyYzqUoChePywvfOx9Scgh+/gqGur3g+1+Y9E/QSSGkLxBk0+cHGNW6kwK7FcvoeaExm2RnTsWQkkOqqQW75yjVTSVDKilJRfdAVbeXwvL/JT94jKw0G6WT5jDmwhvJmXolgbRCgn4fluot8OnvwdvWr6E1Ob1s3/BnMpyVZKXZKJ4be0LqjNM+ir/qF1DvBpoOwaf/A+7TN3DctK+OvCNvY1V1CkeMhexRcYuhz2WWkG41ku6u5lBj//7/SzRpKQ0wHn8AU+0OlL2vY9B1zp44GW3iVyJ1NzlA6bSLqd6/DXvdBmipDiWmKTf1S22O2xdg08Y3yD6+nVSzxsgLb0TJiG+XaXtVMxX+XOr0eVzjeY986uGT/4EJX4HMEgCqGtpo/Ow1CtyHKSnMQJswf2C0ktpljsBu3Yy9/gj7GpxDqohSWkoDiMsb4I23/sHB9/+ErgehcDra9JtPSzaaplI8/hw4ZxFY7Hha6ji8/jd9vqJhMKizccsWMo+8g1kzMGr2VRj7oCZodmkWEwrScBizeVm/hF3NJjxtTbDtD/DZi1CxkbMq/sjZwb2clWEla/q1kJId9zj6VEYxaVYjqYFmAm2N1LUOnXlwkpQGCLcvwPoNb5J9+G0aHR48w2bA2Cu6Hy+yZeGdsoCt9SpVR49xaP3vwNPaZzE6GmuwH3gVTdEZOWk2ttFz++R9DAaFK87OZ86obLzGNN60XsFrxwvZfriJ41W74eAmDK3VjCnIpOi8GyA/See5dcdoxZBVit1qJM9dMaRWDJDu2wDg9QdZv2E9mYfexKjCmBkXYZlwRVTdEVOKHcs5t9DyQRlHjh5F3bCSs+bdBaY4b77obSN9/ytMK7DQai4hY9q1fdpdUhSFL43MpjjLxnv766kwXEC1bzKqvY7MHANYMzDkTex9FXgi5U1gxLH9jExvxlgwgD9HD0lSSnL+QJD1772LvXIdmgHGTJtL+uSejY/MGDuCzf5b8Xz4PFVVVSgbV1F40Z1gtMYlRrfbheXzl8DVhDktB/OMW7q9TB9PhRlWbpxZRKvbR2OblzSLEQbLfmk5Y7GYTeCqD40NphcmOqJ+Id23JOYPBNnw/vukHfg7mgFGT5mDfepXYmqBzJ5QimH6LXhVK4cqD3J0w+/A4+h1jBW1LWx85bc01lSGktyUGyEBFdNpFiMjslMG1waORivkhafEVG2hfoisryRJKYnV7/sI276/oBJk5MSZZE6/LuYukaIoXDBlLMq0W/CoNiorD1L33speDX7vqT7OnreeJ62tknpnACb/M9iyYj6f6ETRbHRdZ9dnH/GnDZ9Q0+xOdER9TpJSMgoGoPxt8o++zagcG6MmzyHn3H8+45SKM1EUhQumjodpCzBY08mkFbaWQWNFz8IL6mz+opKDb/8Ou7OSzDQroy5eCPbhvYpPdCI1DyVvPCZVYWTju7y56yiB4ODe5SQpktIzzzxDSUkJFouF2bNn8+GHHyY6pIQIBnV2791L6wdlcGgLAHlnzyNn5vVxq8pWFIW5U8cy+Zp/QbMXgs+Fvn0tTdv+Cj7XGV9/rNnF6+9swLP5d6R7ahiWmcbYy+5EzRlAhYkDzejLKM7LIDNQR0rVO6wf5NsvKXqCP93//u//ctttt/Hcc88xe/Zsnn76aV566SX27NlDXl5et69taWnBbrfT3NxMevrAvTrh9weoOLif6p3vYWr4ApvJwNnFeWgTroa8PlzcPuCH/W9Q88VmKurbSEtJIb10Bjkjp2HLLurYMvM68dTu59131pHiOYZqUCguKiJ/zoKBVwM0ENXupvGjF9lb00ptyljM4y/norOLsBj7fwpRT8TyO5rwpDR79mzOPfdcfvWrXwEQDAYpKiriX/7lX3jooYe6fe2ATEoBH/ic1NbX03DsCK31h/HU7kfzhnZE1QwK2SOnUXLu1RhsGf0S0gdbP8G1ax1Wb0PkMYOqETCloxs0zi+2gdcBus6eY60YDCqFU+aRMmYuqMZ+iVEAR7ZybOvfqKhz4DNYaE4fw7nTZzCsoAjMaeiKIemqvmP5HU1oSYDX62Xr1q0sW7Ys8pjBYOCyyy7jgw8+OO14j8eDx3PiCkRLS5RbGzdWwP43I3f31bbS2OaFDvlYR9dBQeecEVlo4UZCeZ2Dutbw4GLk+BOvm1GcgUk1gK5zsKGNmhb3ieNOOr8OnDM8BZMSesxR56ApXKWrAUajRnrRZIqmXIQ1u3/HZuacM4Pm8ZMo37ud1oOfojVVQMCH4mpEAQLuLNTw9j+jZ52PetZ0MA+9FRET7qxzGGbLxrT9/zhUVYWxaQeZBw5DZeiHtfy4j2MOP6qqoSsqimJAVxQUFBQFZhRnYlRDSauivo3abq7mTR2egSX8S3Co0YnVpJJ76rLEU27qkzqwhCal+vp6AoEAw4Z1XIt42LBhfPHFF6cdv3z5ch555JGev5HfA231kbu6o5VgFxWyOoCLE10XlwPd1c2lWI8GauhY3dOG7un66ojfZ8Jk0sCgkmLPxGJLwWrPJ3v4GHKKxqBo8VmLOhb2FBMzpp8L08/F6/PjaG4g4GrCqOgoqWlgywSTjeTuLAwBmSVkzr2PjMZyGit3YgrUg7MR9CC6z43B50UP72F58p9QHaDNf+Jntc1BsJupK0qbD7TQ/23d0YZuUkE5ZSUGPRjXj9ZuQBVPLlu2jAceeCByv6WlhaKiKCZ7ZhTB1K9HLqdnt3lJ8emnXV1vb/oa0q1gUACFHLePFG8AJbyrhhK+NtB+rJpiiiSwXK+fVF8ABQUUBSV0IO2vNNksYEkB1cQwRSFZl4U3GTWycoZB0kY4xBkMKDljyM4ZE7qv6+B3U+RsJc/jwefzE9ADEAigh3sAug6GdHP45xqynD7MnkCXb6GmmyOXwTJcPjTVAKZT/iTFe1ZAWEKTUk5ODqqqcuzYsQ6PHzt2jPz805e6MJvNmM0xtCZMKZBVGrmb3YPVRe12iHZF6NTwTYh+pShgtGKxW4l2e82MTMiI8tierYjeewktCTCZTJxzzjm89dZbkceCwSBvvfUWc+bMSWBkQohESXj37YEHHmDhwoXMnDmTWbNm8fTTT9PW1sbtt9+e6NCEEAmQ8KR00003UVdXx8MPP0xNTQ3Tpk3j9ddfP23wWwgxNCS8Tqk3BmSdkhBDSCy/o0kxzUQIIdpJUhJCJBVJSkKIpJLwge7eaB8Oi3q6iRCiX7X/bvZk6HpAJ6XW1tAi+FFVdQshEqa1tRW7PboyzAF99S0YDFJdXU1aWlrSzY7uL+1TbaqqquQKZAzk++udM31/uq7T2tpKYWEhhigXKRzQLSWDwcDw4bLaIUB6err8UvWCfH+90933F20LqZ0MdAshkookJSFEUpGkNMCZzWZ+8IMfxLZ6gpDvr5f64vsb0APdQojBR1pKQoikIklJCJFUJCkJIZKKJCUhRFKRpDQANTY2smDBAtLT08nIyODOO+/E4XB0+5p58+ahKEqH2ze/+c1+ijixeroD80svvcT48eOxWCxMnjyZV199tZ8iTU49+f7KyspO+zmzWKJdOTxMFwPOlVdeqU+dOlXfvHmz/u677+qjR4/Wb7755m5fc9FFF+l33323fvTo0citubm5nyJOnLVr1+omk0lfuXKl/vnnn+t33323npGRoR87dqzT4zdt2qSrqqr/7Gc/03ft2qV///vf141Go75jx45+jjw59PT7W7VqlZ6ent7h56ympqZH7ylJaYDZtWuXDugfffRR5LHXXntNVxRFP3LkSJevu+iii/TvfOc7/RBhcpk1a5a+ePHiyP1AIKAXFhbqy5cv7/T4G2+8Ub/66qs7PDZ79mz93nvv7dM4k1VPv79Vq1bpdru9V+8p3bcB5oMPPiAjI4OZM2dGHrvsssswGAxs2bKl29euXr2anJwcJk2axLJly3A6nX0dbkK178B82WWXRR7rbgdmCH2/Jx8PcMUVV3R5/GAWy/cH4HA4GDFiBEVFRVx77bV8/vnnPXrfAT0hdyiqqakhLy+vw2OappGVlUVNTU2Xr7vlllsYMWIEhYWFfPbZZzz44IPs2bOHl19+ua9DTpie7sAMoe+3s+O7+24Hq1i+v3HjxrFy5UqmTJlCc3MzTzzxBOeddx6ff/551JPnJSkliYceeojHH3+822N2794d8/nvueeeyH9PnjyZgoICLr30UsrLyxk1alTM5xXiZHPmzOmwZ+N5553HhAkT+PWvf82PfvSjqM4hSSlJ/Ou//iuLFi3q9piRI0eSn59PbW1th8f9fj+NjY2d7ircldmzZwOwf//+QZuUeroDM0B+fn6Pjh/MYvn+TmU0Gpk+fTr79++P+n1lTClJ5ObmMn78+G5vJpOJOXPm0NTUxNatWyOvffvttwkGg5FEE41t27YBUFBQEO+PkjRi2YF5zpw5HY4HeOONN4bkjs3x2ME6EAiwY8eOnv2c9WqYXCTElVdeqU+fPl3fsmWL/t577+ljxozpUBJw+PBhfdy4cfqWLVt0Xdf1/fv3648++qj+8ccf6xUVFfpf/vIXfeTIkfrcuXMT9RH6zdq1a3Wz2ayXlZXpu3bt0u+55x49IyMjcpn61ltv1R966KHI8Zs2bdI1TdOfeOIJfffu3foPfvCDIV8S0JPv75FHHtHXrVunl5eX61u3btW//vWv6xaLRf/888+jfk9JSgNQQ0ODfvPNN+upqal6enq6fvvtt+utra2R5ysqKnRAX79+va7run7o0CF97ty5elZWlm42m/XRo0fr//Zv/zYk6pR0Xdd/+ctf6sXFxbrJZNJnzZqlb968OfLcRRddpC9cuLDD8S+++KI+duxY3WQy6Weffbb+97//vZ8jTi49+f6WLFkSOXbYsGH6/Pnz9U8++aRH7ydLlwghkoqMKQkhkookJSFEUpGkJIRIKpKUhBBJRZKSECKpSFISQiQVSUpCiKQiSUkMKGVlZWRkZETu//CHP2TatGmR+4sWLeK6667r97hE/EhSEp1atGhRl0vmLl68GEVROkwg7otkUFJSwtNPP93hsZtuuom9e/d2+ZoVK1ZQVlYWuT9v3jyWLFkS17hE35KkJLpUVFTE2rVrcblckcfcbjdr1qyhuLg4ITFZrdbT1pM6md1u79CSEgOPJCXRpRkzZlBUVNRhIbiXX36Z4uJipk+f3qtzd9aCue666yKtr3nz5lFZWcl3v/vdyAL0cHr37VQnt9gWLVrEhg0bWLFiReQcFRUVjB49mieeeKLD67Zt24aiKD1aYkP0DUlKolt33HEHq1atitxfuXIlt99+e5+/78svv8zw4cN59NFHOXr0KEePHu3xOVasWMGcOXO4++67I+coLi4+7TMBrFq1irlz5zJ69Oh4fQQRI0lKolvf+MY3eO+996isrKSyspJNmzbxjW98o8/fNysrC1VVSUtLIz8/P6ZF1ux2OyaTCZvNFjmHqqosWrSIPXv2RLYK8vl8rFmzhjvuuCPeH0PEQFaeFN3Kzc3l6quvpqysDF3Xufrqq8nJyUl0WL1SWFjI1VdfzcqVK5k1axZ/+9vf8Hg83HDDDYkOTSAtJRGFO+64g7KyMp5//vm4tSYMBgOnrprj8/nicu5o3HXXXZFB/FWrVnHTTTdhs9n67f1F1yQpiTO68sor8Xq9+Hw+rrjiiricMzc3t8M4USAQYOfOnR2OMZlMBAKBXr1PV+eYP38+KSkpPPvss7z++uvSdUsi0n0TZ6SqamQnFVVVuzyuubk5svZ3u+zsbIqKik479pJLLuGBBx7g73//O6NGjeKpp56iqampwzElJSVs3LiRr3/965jN5pi6jSUlJWzZsoWDBw+SmppKVlYWBoMhMra0bNkyxowZMyTX4E5W0lISUUlPTyc9Pb3bY9555x2mT5/e4fbII490euwdd9zBwoULue2227jooosYOXIkF198cYdjHn30UQ4ePMioUaPIzc2NKe6lS5eiqioTJ04kNzeXQ4cORZ6788478Xq9/XI1UURPlsMVQ9a7777LpZdeSlVV1WkbLorEkaQkhhyPx0NdXR0LFy4kPz+f1atXJzokcRLpvokh5w9/+AMjRoygqamJn/3sZ4kOR5xCWkpCiKQiLSUhRFKRpCSESCqSlIQQSUWSkhAiqUhSEkIkFUlKQoikIklJCJFUJCkJIZKKJCUhRFL5//YWzg+2a+tbAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:13:19.854237Z", + "iopub.status.busy": "2024-02-29T03:13:19.853945Z", + "iopub.status.idle": "2024-02-29T03:13:20.079830Z", + "shell.execute_reply": "2024-02-29T03:13:20.078932Z" + }, + "papermill": { + "duration": 0.24941, + "end_time": "2024-02-29T03:13:20.081910", + "exception": false, + "start_time": "2024-02-29T03:13:19.832500", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEmCAYAAAAdlDeCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAy4UlEQVR4nO3de3xM194/8M9MLpPrJCI3lxAE0UcIUWmUohHSHI57XeJaD62WcwhS+rQcj/bxIBKl1NFzcByUpxWO5gQJcYnLiQhRIqKICMmISGQkYTKTWb8//GafjNxmmGRm7/19v155mb1n7dnflZ35WnvtvdeSMMYYCCFE4KTmDoAQQpoDJTtCiChQsiOEiAIlO0KIKFCyI4SIAiU7QogoULIjhIgCJTtCiChYmzsAS6fValFQUABnZ2dIJBJzh0MIqYExhmfPnqF169aQShtuu1Gya0RBQQF8fHzMHQYhpAH5+flo27Ztg2Uo2TXC2dkZwMtfplwuN3M0TUetViMpKQlDhw6FjY2NucMhb0gsx1OpVMLHx4f7njaEkl0jdKeucrlc8MnOwcEBcrlc0F8OsRDb8TSki4kuUBBCRIGSHSFEFCjZEUJEgZIdIUQUKNkRQkSBkh0hRBTo1hNCBKCyshI3b97klsufq3D+2h20cL8EJ3uZXll/f384ODg0d4hmR8mOEAG4efMmgoKCaq1fW0fZjIwM9O7du+mDsjCU7AgRAH9/f2RkZHDLOYVPEfXTNcSOD0DXVq61yooRJTtCBMDBwUGvtSbNewJZ6nN0694Tge1bmjEyy0HJTqQM7eMRa/8OER5KdiKRW1yBCpWGW75xLRMTPhhUq9yrfTz7j5zCWwGB3LKjzBod3B2bJkhCmhAlOxG4+egJIr4/oLdOq1GhzWdfNrpt9NlfIf1Xjt66xLlj4e9Fp0aEXyjZicBvJXfg2GGTCT8vkJKdBXi1tV7TnccV3L/W1vV/zcXUUqdkJwKtHdujInc+vp0QiE6eTnWW0Wg0OHf2HN7t/269X447ReX44/5MtB7cvinDJQbILa7A4JhTjZZb9PO1RsucXDxIFAmPkp0IyKzsoH3RBh3kXfFWS5c6y6jVauRa56KbW7d6xz/TviiD9sVjyKzsmjJcYgBdi27DhED41fEfWMVzFRJOXcDwQSFwfOWmYp3bReVYsD+z3tah0FCyI4TH/Dyd0L1N7f/A1Go1FB5A7/YtRDF4pyEo2RHCQ6rqF5DaPUSuMgdSu9otO41GgwJNAbJLsuvtlshVlkNq9xCq6hcA6m7xCwklO0J4qKAiD44dNuGLiw2X23J0S4PvO3YACioCEQQvE0ZnmXiX7DZv3ox169ZBoVCgZ8+e2LRpE/r27Vtn2aysLCxfvhwZGRnIy8tDXFwcFixY0LwBE9IEGrvoRBecauNVstu/fz+ioqKwdetWBAcHY8OGDRg2bBhycnLg6elZq3xlZSU6duyI8ePHY+HChWaImJCm0dhFJ7rgVBuvxrOLjY3F7NmzMXPmTLz11lvYunUrHBwcsH379jrLv/3221i3bh0mTpwImazuK1KEEHHgTbKrqqpCRkYGhgwZwq2TSqUYMmQILly4YMbICCF8wJvT2OLiYlRXV8PLS78j1cvLS++B9jelUqmgUqm4ZaVSCeDlaYFarTbZfpqTRqPh/q2vDrr1DdXRkM8hzaOxYyGW42lM3LxJds1l9erVWLlyZa31SUlJvB39I78cAKxx9uxZ5NX9AAUnOTnZJJ9Dmpahx0Lox7OystLgsrxJdu7u7rCyssKjR4/01j969Aje3t4m28+yZcsQFRXFLSuVSvj4+GDo0KGQy+Um209zyipQIubav9C/f3/8R+u666BWq5GcnIywsLB6O7QN+RzSPBo7FmI5nrozL0PwJtnZ2toiKCgIJ06cwKhRowAAWq0WJ06cwLx580y2H5lMVufFDBsbG97eia679cDa2rrROjRUT2M+hzQtQ4+F0I+nMXHzJtkBQFRUFKZPn44+ffqgb9++2LBhAyoqKjBz5kwAwLRp09CmTRusXr0awMuLGjdu3OBeP3z4EJmZmXBycoKfn5/Z6kEIaX68SnYTJkzA48ePsXz5cigUCgQGBuLo0aPcRYv79+9DKv33BeaCggL06tWLW46JiUFMTAwGDhyIU6dONXf4hBAz4lWyA4B58+bVe9r6agLz9fUFY6wZoiKEWDre3GdHCCFvgpIdIUQUKNkRQkSBkh0hRBQo2RFCRIF3V2MJIcBzdTUA4PrDsjrfr3iuwqXHgHdeaYNzUIgJJTtCeOjO/09US+Mbmj3MGn+/nd7oZznKxJEGxFFLkWusFQBQS4Bvhv7Hy+fBO3k6wd7Gqtb7OYVlWPTzNawfF4CureqfX4LmjSWCYlgrAKCWAH+4OdpiYt929b6vG76pk4djnbOPiRH91YpAY60AgFoCRPgo2YlAY60AgFoCRPjo1hNCiChQsiOEiAIlO0KIKFCyI4SIAiU7QogoULIjhIgCJTtCiChQsiOEiAIlO0KIKFCyI4SIAu+S3ebNm+Hr6ws7OzsEBwfj4sWLDZb/6aef4O/vDzs7OwQEBCAxMbGZIiWEWBJeJbv9+/cjKioKK1aswOXLl9GzZ08MGzYMRUVFdZY/f/48Jk2ahFmzZuHKlSsYNWoURo0ahevXrzdz5IQQc5MwHk2sGhwcjLfffhvfffcdAECr1cLHxwfz58/H0qVLa5WfMGECKioqkJCQwK175513EBgYiK1btxq0T6VSCRcXF5SVlUEul5umIhYoM+8JRn3/Lxya+w4C27c0dzjESJWVlbh58ya3nFP4FFE/XUPs+AB0beWqV9bf3x8ODg7NHGHTMOb7yZtRT6qqqpCRkYFly5Zx66RSKYYMGYILFy7Uuc2FCxcQFRWlt27YsGE4dOhQvftRqVRQqVTcslKpBACo1Wqo1eo3qIFl0416otFoBF1Pobp+/TqCg4NrrZ/8t9pl09LS0KtXr2aIqukZ87fKm2RXXFyM6upqeHl56a338vLS+x+tJoVCUWd5hUJR735Wr16NlStX1lqflJQkmP8N65JfDgDW+Ne//oWHdJbPOyqVCuvXr+eW1Vqg5AXgZgfYvNJZde/ePRQWFjZzhE2jsrLS4LK8SXbNZdmyZXqtQaVSCR8fHwwdOlTQp7FX75cA1y7hnXfeQc92buYOh7whtVqN5ORkhIWFwcbGxtzhNBndmZcheJPs3N3dYWVlhUePHumtf/ToEby9vevcxtvb26jyACCTySCT1Z6DwcbGRtB/NNbW1ty/Qq6n2Aj979aYuvHmaqytrS2CgoJw4sQJbp1Wq8WJEycQEhJS5zYhISF65QEgOTm53vJiUllZicuXL3M/2devQqW4jezrV/XWG3OaQIhFYzyyb98+JpPJ2M6dO9mNGzfYnDlzmKurK1MoFIwxxqZOncqWLl3KlT937hyztrZmMTExLDs7m61YsYLZ2Niwa9euGbzPsrIyBoCVlZWZvD7mlJGRwQA0+pORkWHuUMlrqKqqYocOHWJVVVXmDqVJGfP95M1pLPDyVpLHjx9j+fLlUCgUCAwMxNGjR7mLEPfv34dU+u/Gar9+/bB37158+eWX+OKLL9C5c2ccOnQI3bt3N1cVLIa/vz8yMjK45fLnKvzz5AX8bnAInGpMpejv72+O8AgxOV7dZ2cOYrnPTq1WIzExEREREYLu4xELsRxPY76fvOmzI4SQN0HJjhAiCpTsCCGiQMmOECIKlOwIIaJAyY4QIgqU7AghokDJjhAiCpTsCCGiQMmOECIKlOwIIaJAyY4QIgqU7AghokDJjhAiCpTsCCGiQMmOECIKlOwIIaJAyY4QIgqU7AghomB0srt7925TxEEIIU3K6GTn5+eHwYMHY/fu3Xjx4kVTxEQIISZndLK7fPkyevTogaioKHh7e+Pjjz/GxYsXmyI2PSUlJYiMjIRcLoerqytmzZqF8vLyBrfZtm0bBg0aBLlcDolEgqdPnzZ5nIQQy2R0sgsMDMS3336LgoICbN++HYWFhejfvz+6d++O2NhYPH78uCniRGRkJLKyspCcnIyEhAScOXMGc+bMaXCbyspKhIeH44svvmiSmAghPPKmM3K/ePGCxcbGMplMxiQSCZPJZGzq1KmsoKDgTT+ac+PGDQaApaenc+uOHDnCJBIJe/jwYaPbnzx5kgFgpaWlRu/bmBnH+UwsM8iLhViOpzHfT+vXTZKXLl3C9u3bsW/fPjg6OmLx4sWYNWsWHjx4gJUrV2LkyJEmO729cOECXF1d0adPH27dkCFDIJVKkZaWhtGjR5tkPwCgUqmgUqm4ZaVSCeDlpMNqtdpk+7E0uroJuY5iIpbjaUz9jE52sbGx2LFjB3JychAREYFdu3YhIiICUunLM+IOHTpg586d8PX1Nfaj66VQKODp6am3ztraGm5ublAoFCbbDwCsXr0aK1eurLU+KSkJDg4OJt2XJUpOTjZ3CMSEhH48KysrDS5rdLL7/vvv8dFHH2HGjBlo1apVnWU8PT3x17/+tdHPWrp0KdasWdNgmezsbGNDfCPLli1DVFQUt6xUKuHj44OhQ4dCLpc3ayzNpaqqCps3b8bp06cxcOBAfPbZZ7C1tTV3WOQNqNVqJCcnIywsDDY2NuYOp8nozrwMYXSyS05ORrt27biWnA5jDPn5+WjXrh1sbW0xffr0Rj9r0aJFmDFjRoNlOnbsCG9vbxQVFemt12g0KCkpgbe3t7FVaJBMJoNMJqu13sbGRpB/NNHR0YiLi4NGowEAJCYm4r/+67+wcOFCrF271szRkTcl1L9bHWPqZnSy69SpEwoLC2udVpaUlKBDhw6orq42+LM8PDzg4eHRaLmQkBA8ffoUGRkZCAoKAgCkpKRAq9UiODjYuAoQTnR0NNatWwcvLy+sXLkSMpkMKpUKK1aswLp16wCAEh4RDKNvPWGM1bm+vLwcdnZ2bxxQXbp164bw8HDMnj0bFy9exLlz5zBv3jxMnDgRrVu3BgA8fPgQ/v7+ehdFFAoFMjMzcfv2bQDAtWvXkJmZiZKSkiaJk0+qqqoQFxcHLy8v5OXloVOnTrh27Ro6deqEvLw8eHl5IS4uDlVVVeYOlRCTMLhlp+vHkkgkWL58uV5nfXV1NdLS0hAYGGjyAHX27NmDefPmITQ0FFKpFGPHjsXGjRu599VqNXJycvQ6LLdu3ap3seG9994DAOzYsaPR02eh27JlCzQaDcaMGQN/f3/cu3cPwMsLUL6+vhg1ahT+/Oc/Y8uWLViwYIFZYyXGqaqqwqZNm5CSkoLbt29j/vz51AcLQMLqa6q9YvDgwQCA06dPIyQkRO+XZ2trC19fXyxevBidO3dumkjNRKlUwsXFBWVlZYK6QDF//nx89913kEgkGD58OKKjo/HgwQO0bdsWa9euRUJCAhhjmDdvHjZt2mTucImBoqOjERsbq9edZGVlhaioKEF2SRjz/TS4ZXfy5EkAwMyZM/Htt98K6osvRh06dAAA9OjRA4cOHUJ1dTWePHmC4OBgHDp0CL169cKvv/7KlSOWT9cH+6rq6mrqg8Vr9Nnt2LGDEp0ABAQEAADu37+PFy9eYOPGjdi2bRs2btyIFy9eID8/X68csWxVVVWIiYlpsExMTIyo+2ANatmNGTMGO3fuhFwux5gxYxosGx8fb5LASNMqLi4GAJSWlsLR0ZFbn5iYiMWLF9cqRyzbxo0b6714qMMYw8aNG/WOr5gY1LJzcXGBRCLhXjf0Q/ihvhvCX7ccMa+9e/fqLQcHB2PlypW1bs16tZyYGHyBQqyEeoHi+fPn3BX18PBw+Pn54datW+jSpQtu376No0ePAnj5OI69vb05QyUGkEqlXMvu2bNnkMlkSExMREREBFQqFZydnQG8vJtCq9WaM1STMub7ScOyi9SWLVu4168+DVNzuWY5Yrlqtllevd+15rKY2zYG9dn16tWLO41tzOXLl98oINI8zp49C+Dl6U5iYiK3PikpCQDQt29fXLx4EWfPnsWiRYvMEiN5PW3btsWKFStgZ2eHv/zlL3UObCFGBiW7UaNGNXEYpLk5OTkBANLS0up8X/ckiq4csWydOnXCnTt3AACPHj3Cp59+Wm85saI+u0YItc8uISEBI0aMaLTcL7/8guHDhzdDRORNlJSUoGXLlo2We/LkCdzc3JohouZBfXakUTdu3NBb7t27N95991307t27wXLEMrm5ucHLy6vBMl5eXoJKdMYyqGXn5uaGW7duwd3dHS1atGiw/05oD9kLtWXXr18/XLhwodFyISEhOH/+fDNEREzB29sbjx49qrXey8vL5APdWgKTPy4WFxfHXbqOi4sz+GIFsVyFhYUmLUcsg0KhQElJCQYMGID8/Hz4+PggNTVV1C06HYOSXc2BOMU+WohQeHl5cSOdNFaO8IubmxsyMzO5++yEPHinMYzus7Oysqo1ajDwsuPTysrKJEGRpvdqZ3ZoaCimTJmC0NDQBssRwldGj1RcXxefSqWiMbN45NW+1RMnThhUjhC+MjjZ6QbKlEgk+Mtf/qJ3/1V1dTXOnDkDf39/00dImoShndVC7NQm4mRwsouLiwPwsmW3detWvVNW3eCdW7duNX2EpEm0atXKoD47GgiACIXByS43NxfAyxGL4+Pj0aJFiyYLijS97t27c7eeuLu7Y+DAgSgtLUWLFi1w+vRpbmin7t27mzNMQkzG6D473YjFhN/c3d2518XFxThw4ECj5QjhM6OT3UcffdTg+9u3b3/tYEjzMfTKOV1hJ0Jh9K0npaWlej9FRUVISUlBfHw8nj592gQhkqYwaNAgk5YjxNIZ3bI7ePBgrXVarRZz585t0hEVSkpKMH/+fPzyyy/cVIrffvttvaNylJSUYMWKFUhKSsL9+/fh4eGBUaNGYdWqVTSiMoABAwZAKpVCq9XCzs4OL1684N7TLUulUgwYMMCMURJiOiYZCEAqlSIqKoq7YtsUIiMjkZWVheTkZCQkJODMmTOYM2dOveULCgpQUFCAmJgYXL9+HTt37sTRo0cxa9asJouRT86fP8+NWFsz0dVc1mq19FwsEQyjW3b1uXPnDjQajak+Tk92djaOHj2K9PR09OnTBwCwadMmREREICYmBq1bt661Tffu3fU63Tt16oRvvvkGU6ZMgUajgbW1yarOSw8fPuReSyQSvZvFdS2+V8sRwmdGf+OjoqL0lhljKCwsxD//+U+9Z2hN6cKFC3B1deUSHQAMGTIEUqkUaWlpGD16tEGfoxsZoaFEp1KpoFKpuGWlUgkAUKvVUKvVr1kDy/PgwQMAgLOzM/Lz8/HnP/8Zp0+fxsCBA/Hxxx/Dx8cHz549w4MHDwRVb7HQHTOhHztj6md0srty5YreslQqhYeHB9avX9/oldrXpVAo4OnpqbfO2toabm5uBt/hX1xcjFWrVjV46gsAq1evrnMY66SkJG6CGiHQDcUuk8nQuXNnPH78mFsfExPDPfqXmJhIT8bwWHJysrlDaFKVlZUGlzXrfXZLly7FmjVrGiyTnZ39xvtRKpX43e9+h7feegt/+tOfGiy7bNkyvdarUqmEj48Phg4dKqjx7HS3CBUXF9eacOfJkyfcaayrqysiIiKaPT7y+qqrq3Hq1CkkJycjLCwMgwYNEuwtRLozL0OYteNq0aJFjQ4Z1bFjR3h7e9caaUWj0aCkpATe3t4Nbv/s2TOEh4fD2dkZBw8ebHS4G5lMBplMVmu9jY2NoIbK6d+/Pw4fPgzg5eN+NS9SyGQyPH/+nCsnpHoLXXx8PKKiopCXlwcAiI2NRfv27REbG9voBPd8ZMzfplmTnYeHBzw8PBotFxISgqdPnyIjIwNBQUEAgJSUFGi12lqTANekVCoxbNgwyGQyHD58uNYUc2IWEBDAvX7//fcxbNgw/Pbbb+jcuTOOHTvGnebWLEcsW3x8PMaOHVtrfV5eHsaOHYsDBw4IMuEZjPFEeHg469WrF0tLS2Nnz55lnTt3ZpMmTeLef/DgAevatStLS0tjjDFWVlbGgoODWUBAALt9+zYrLCzkfjQajcH7LSsrYwBYWVmZyetkTl9++SUDUOePRCLhXn/55ZfmDpUYQKPRMLlcXu8xBcDkcrlRf/t8YMz3kzcT7uzZswf+/v4IDQ1FREQE+vfvj23btnHvq9Vq5OTkcB2Wly9fRlpaGq5duwY/Pz+0atWK+8nPzzdXNSxOXbftMMbqXE8s14kTJxrtv1IqlfWOWygKpsqw+fn5bPbs2ab6OIsh1Jbd8ePHG2wF6H6OHz9u7lCJASZMmGDQ8ZwwYYK5QzUps7Tsnjx5gr/+9a+m+jjSxBrq63ydcsS8DH3SRcxPxPDmNJaY1pIlS0xajphXzbsV3N3dMW7cOLz//vsYN26c3jBddc0fIxbifmZKxNLT001ajphXVVUV97q4uBg///xzo+XEhlp2IlVdXc29fvW+wprLNcsRy/XqjeFvWk6IDG7ZNXZ/Do1lx181nwWua5lYPm9vb4MGbWjsJnwhMzjZNTYGnIuLC6ZNm/bGAZHmUVFRobfcpUsXtGjRAqWlpbh161a95Yhl6tmzp0HJrmfPns0QjWUyONnt2LGjKeMgzczR0VFvuWaCa6gcsUyGDlkm5qHNxHsCL3KGzg5Hs8jxg6GDVAhpMAtjGZzmDR2+iSbc4QdDW2zUsuOHqVOnYvfu3bCzs0NVVRU3ag3w8qKEbrCHqVOnmjFK8zI42e3cuRPt27dHr1699Ea1JfwkkUhMWo6YV2hoKORyOZRKJTw9PTFgwABuHuDU1FQUFRVBLpcjNDTU3KGajcHJbu7cufjxxx+Rm5uLmTNnYsqUKXBzc2vK2EgTatOmDfdaJpPpXYGtuVyzHLFcVlZW2LFjB8aOHYvHjx/rTUmg+w9rx44dgh3XzhAG99lt3rwZhYWFiI6Oxi+//AIfHx98+OGHOHbsGLX0eKhr167c61eHtq65XLMcsWxjxozBgQMH4OPjo7e+Xbt2NLwTAAl7zUyVl5eHnTt3YteuXdBoNMjKyqp3WkM+UyqVcHFx4eavEIqqqio4OjrC1tYWKpVK7+ZhKysryGQyVFVVoaKighuinfBDdXU1Tp48iSNHjuCDDz7A4MGDBduiM+b7+drXoaVSKTcrFd1lzz+2trZYuHAh1q1bV28fz5IlSyjR8ZCVlRUGDhyIiooKDBw4ULCJzlhGJTuVSoX4+Hhs374dZ8+exfDhw/Hdd98hPDxc1I+h8NXatWsBAHFxcXp9PNbW1liyZAn3PiFCYHCy+/TTT7Fv3z74+Pjgo48+wo8//qg3mgLhp7Vr1+Lrr7/Gpk2bkJKSgvfffx/z58+nFh0RHIP77KRSKdq1a4devXo1eDtCfHy8yYKzBELts3uVWq1GYmIiIiIiaIIdARDL8WySPrtp06bRPVeEEN4y6qZiQgjhK7qqQAgRBUp2hBBR4E2yKykpQWRkJORyOVxdXTFr1iyUl5c3uM3HH3+MTp06wd7eHh4eHhg5ciRu3rzZTBETQiwJb5JdZGQksrKykJycjISEBJw5cwZz5sxpcJugoCDs2LED2dnZ3GNtQ4cOpZugCREhXozkl52djaNHjyI9PR19+vQBAGzatAkRERGIiYmpd0LnmsnQ19cXX3/9NXr27Il79+6hU6dOzRI7IcQy8CLZXbhwAa6urlyiA4AhQ4ZAKpUiLS0No0ePbvQzKioqsGPHDnTo0KHWg9I1qVQqvRFAdLOsq9XqWg/MC4mubkKuo5iI5XgaUz9eJDuFQgFPT0+9ddbW1nBzc4NCoWhw2y1btiA6OhoVFRXo2rUrkpOTG3w6YPXq1Vi5cmWt9UlJSXBwcHi9CvBIcnKyuUMgJiT041lZWWlwWbMmu6VLl2LNmjUNlsnOzn6jfURGRiIsLAyFhYWIiYnBhx9+iHPnzsHOzq7O8suWLUNUVBS3rFQq4ePjg6FDhwr+CYrk5GSEhYUJ+o57sRDL8dSdeRnCrMlu0aJFmDFjRoNlOnbsCG9v71ozmWs0GpSUlDQ6NZyLiwtcXFzQuXNnvPPOO2jRogUOHjyISZMm1VleJpPVmkcVAGxsbAT9R6MjlnqKhdCPpzF1M2uy8/DwgIeHR6PlQkJC8PTpU2RkZCAoKAgAkJKSAq1Wi+DgYIP3xxgDY4zmRSVEhHhx60m3bt0QHh6O2bNn4+LFizh37hzmzZuHiRMncldiHz58CH9/f1y8eBEAcPfuXaxevRoZGRm4f/8+zp8/j/Hjx8Pe3h4RERHmrA4hxAx4kewAYM+ePfD390doaCgiIiLQv39/bNu2jXtfrVYjJyeH67C0s7NDamoqIiIi4OfnhwkTJsDZ2Rnnz5+vdbGDECJ8vLgaCwBubm7Yu3dvve/7+vrqzYXRunVrJCYmNkdohBAe4E3LjhBC3gQlO0KIKFCyI4SIAiU7QogoULIjhIgCJTtCiChQsiOEiAIlO0KIKFCyI4SIAiU7QogoULIjhIgCJTtCiChQsiOEiAIlO0KIKFCyI4SIAiU7QogoULIjhIgCJTtCiChQsiOEiAIlO0KIKPAm2ZWUlCAyMhJyuRyurq6YNWsWysvLDdqWMYYPPvgAEokEhw4datpACSEWiTfJLjIyEllZWUhOTkZCQgLOnDmDOXPmGLTthg0bIJFImjhCQogl48VUitnZ2Th69CjS09PRp08fAMCmTZsQERGBmJgYbqLsumRmZmL9+vW4dOkSWrVq1VwhE0IsDC9adhcuXICrqyuX6ABgyJAhkEqlSEtLq3e7yspKTJ48GZs3b4a3t3dzhEoIsVC8aNkpFAp4enrqrbO2toabmxsUCkW92y1cuBD9+vXDyJEjDd6XSqWCSqXilpVKJQBArVZDrVYbGTl/6Oom5DqKiViOpzH1M2uyW7p0KdasWdNgmezs7Nf67MOHDyMlJQVXrlwxarvVq1dj5cqVtdYnJSXBwcHhtWLhk+TkZHOHQExI6MezsrLS4LISxhhrwlga9PjxYzx58qTBMh07dsTu3buxaNEilJaWcus1Gg3s7Ozw008/YfTo0bW2W7BgATZu3Aip9N9n6tXV1ZBKpRgwYABOnTpV5/7qatn5+PiguLgYcrncyBryh1qtRnJyMsLCwmBjY2PucMgbEsvxVCqVcHd3R1lZWaPfT7O27Dw8PODh4dFouZCQEDx9+hQZGRkICgoCAKSkpECr1SI4OLjObZYuXYr//M//1FsXEBCAuLg4jBgxot59yWQyyGSyWuttbGwE/UejI5Z6ioXQj6cxdeNFn123bt0QHh6O2bNnY+vWrVCr1Zg3bx4mTpzIXYl9+PAhQkNDsWvXLvTt2xfe3t51XpRo164dOnTo0NxVIISYGS+uxgLAnj174O/vj9DQUERERKB///7Ytm0b975arUZOTo5R5/CEEPHgRcsOANzc3LB379563/f19UVj3Y9m7J4khJgZb1p2hBDyJijZEUJEgZIdIUQUKNkRQkSBkh0hAlNdXY3Tp0/jzJkzOH36NKqrq80dkkWgZEeIgMTHx8PPzw9hYWGIjY1FWFgY/Pz8EB8fb+7QzI6SHSECER8fj3HjxiEgIACpqan48ccfkZqaioCAAIwbN070CY+SHSECUF1djUWLFmH48OE4dOgQgoODYW9vj+DgYBw6dAjDhw/H4sWLRX1KS8mOEAFITU3FvXv38MUXX4AxptdnxxjDsmXLkJubi9TUVHOHajaU7AgRgMLCQgDAnTt30LFjR70+u44dO+Lu3bt65cSIN4+LkaZT8+qdo6MjBg8eDCsrK3OHRYygm3JgypQpteZbyc/Px5QpU/TKiZFZx7PjA6VSCRcXF4PGy+Kj+Ph4REVFIS8vj1vXvn17xMbGYsyYMWaMjBijqqoKdnZ2YIzB09MT//3f/w2ZTAaVSoXly5ejqKgIEokEL168gK2trbnDNRljvp90Giti8fHxGDt2LIqKivTWFxUVYezYsaK/escnp06d4ga6CAoKwvXr1/Hjjz/i+vXr3BiQjLF6B60VBUYaVFZWxgCwsrIyc4diUhqNhnl4eDAAzN7engHgfnTLnp6eTKPRmDtUYoApU6YwACwgIEDvWOp+dOunTJli7lBNypjvJ7XsROrUqVN4/PgxAOD58+d67+mWi4qKxN0S4BHdhPHXrl2r833dekMnlhciSnYilZKSYtJyxLxCQkJMWk6IKNmJVG5urknLEfMy9GZhuqmYiE5BQQH3+tWrczWXa5Yjluvw4cMmLSdElOxE6t69e9zrqqoqvfdqLtcsRyxXzRb4q/fZ1VwWc0udkp1I0WmPsGg0Gu51aWkpPvnkEwQGBuKTTz6pNd+yWNETFCLl5uaGBw8eGFSOWD5nZ2duwnk3NzdotVoAQGZmpt4sfM7OzmaJzxLwpmVXUlKCyMhIyOVyuLq6YtasWY1eRh80aBAkEonezyeffNJMEVs2Q29BEPOtCnxSc7J4XaKra7m+SeXFgDfJLjIyEllZWUhOTkZCQgLOnDmDOXPmNLrd7NmzUVhYyP2sXbu2GaK1fCUlJSYtR8xr5syZJi0nRLw4jc3OzsbRo0eRnp6OPn36AAA2bdqEiIgIxMTEoHXr1vVu6+DgAG9v7+YKlTcM7bsRcx8Pn7x6UeJNywkRL5LdhQsX4OrqyiU6ABgyZAikUinS0tIwevToerfds2cPdu/eDW9vb4wYMQJfffUVHBwc6i2vUqmgUqm4ZaVSCQBQq9VQq9UmqI1l8PT05E5RbW1t9a7A6h4g15UTUr2F6vjx4waXGzx4cBNH03yM+dvkRbJTKBTw9PTUW2dtbQ03NzcoFIp6t5s8eTLat2+P1q1b49dff8Xnn3+OnJycBh9wX716NVauXFlrfVJSUoNJkm9atmzJjXH26q0nNZN9y5YtkZiY2KyxEeMlJSVxryUSCTcowKvLSUlJePfdd5s9vqZSWVlpcFmzJrulS5dizZo1DZbJzs5+7c+v2acXEBCAVq1aITQ0FHfu3EGnTp3q3GbZsmWIiorilpVKJXx8fDB06FBBDfFUXFyM9PT0Rst9/PHHiIiIaIaIyJv44YcfcPXqVVhbW+PJkyc4f/48kpOTERYWhn79+qFly5bQaDRo27atoI6n7szLEGZNdosWLcKMGTMaLNOxY0d4e3vXGoZIo9GgpKTEqP443ZWo27dv15vsZDIZZDJZrfU2NjawsbExeF+Wrr7611VOSPUWKqn05bVGjUaDyMhIREdH4+2334ajoyMiIyO5vlepVCqo42lMXcya7Dw8PODh4dFouZCQEDx9+hQZGRnc2FwpKSnQarVGXUrPzMwEIO7RWnUGDBgAX19fPH/+HI8ePar1vpeXFxwcHDBgwAAzREeM1bZtW+718ePHkZCQwC3b29vXWU5seNFn161bN4SHh2P27NnYunUr1Go15s2bh4kTJ3JXYh8+fIjQ0FDs2rULffv2xZ07d7B3715ERESgZcuW+PXXX7Fw4UK899576NGjh5lrZH5WVlZYv349xo0bh4iICNja2nKn91VVVThy5Ah+/vlnGp6dJ7p06cK9bqgPtmY50Wny0fVM5MmTJ2zSpEnMycmJyeVyNnPmTPbs2TPu/dzcXAaAnTx5kjHG2P3799l7773H3NzcmEwmY35+fmzJkiVGD8Ip1ME7dQ4cOMB8fX31Bnrs0KEDO3DggLlDI0ZQqVTM2tqaOTg4MCsrK73jaWVlxRwcHJi1tTVTqVTmDtWkjPl+0hwUjRD6HBTAy+dfT548iSNHjuCDDz6gCXd4Kjo6GuvWrYOnpycGDBiA0tJStGjRAqmpqSgqKsKSJUsEd1O9Md9PXpzGkqZlZWWFgQMHoqKiAgMHDqREx1O6RBYXF4cDBw5w662trQWZ6IzFm8fFCCGNW7t2LSoqKhATE8M9YVRRUSH6RAdQy44QwbG1tcUf/vAH+Pn5ISIiQlC3mrwJatkRQkSBkh0hRBQo2RFCRIH67BqhuzPHmGfw+EitVqOyshJKpZL6eARALMdT97005A46SnaNePbsGQDAx8fHzJEQQurz7NkzuLi4NFiGbipuhFarRUFBAZydnQU98KFudJf8/HzB3jwtJmI5nowxPHv2DK1bt+YGQ6gPtewaIZVKRfXwtFwuF/SXQ2zEcDwba9Hp0AUKQogoULIjhIgCJTsC4OWgpStWrKhz4FLCP3Q8a6MLFIQQUaCWHSFEFCjZEUJEgZIdIUQUKNkRQkSBkp0ADRo0CAsWLDB3GMRIlnbcLC2eN0XJjhABeXVmMVJD0837Q8xh+vTpejNLAWBt2rRhW7Zs0St3+fJlJpFI2L179xhjjK1fv551796dOTg4sLZt27K5c+fqzd7GGGOpqamsf//+zM7OjrVt25bNnz+flZeXN1vdhKyu43b79m320UcfMV9fX2ZnZ8e6dOnCNmzYUGu7kSNHsq+//pq1atWK+fr6MsYYO3fuHOvZsyeTyWQsKCiIHTx4kAFgV65c4ba9du0aCw8PZ46OjszT05NNmTKFPX78uN54cnNzm+vX0SQo2QnM06dPWUhICJs9ezYrLCxkhYWFbPHixax///565RYtWqS3Li4ujqWkpLDc3Fx24sQJ1rVrVzZ37lzu/du3bzNHR0cWFxfHbt26xc6dO8d69erFZsyY0Wx1E7K6jtuLFy/Y8uXLWXp6Ort79y7bvXs3c3BwYPv37+e2mz59OnNycmJTp05l169fZ9evX2dlZWXMzc2NTZkyhWVlZbHExETWpUsXvWRXWlrKPDw82LJly1h2dja7fPkyCwsLY4MHD643Ho1GY45fjclQshOggQMHsj/+8Y/c8pUrV5hEImF5eXmMMcaqq6tZmzZt2Pfff1/vZ/z000+sZcuW3PKsWbPYnDlz9MqkpqYyqVTKnj9/btoKiNSrx60un332GRs7diy3PH36dObl5aU3H+z333/PWrZsqXdcfvjhB71kt2rVKjZ06FC9z87Pz2cAWE5OjsHx8An12YlAYGAgunXrhr179wIATp8+jaKiIowfP54rc/z4cYSGhqJNmzZwdnbG1KlT8eTJE1RWVgIArl69ip07d8LJyYn7GTZsGLRaLXJzc81SLzHYvHkzgoKC4OHhAScnJ2zbtg3379/XKxMQEABbW1tuOScnBz169ICdnR23rm/fvnrbXL16FSdPntQ7nv7+/gCAO3fuNGGNzIeSnUhERkZyyW7v3r0IDw9Hy5YtAQD37t3D8OHD0aNHDxw4cAAZGRnYvHkzgH93eJeXl+Pjjz9GZmYm93P16lX89ttv6NSpk3kqJXD79u3D4sWLMWvWLCQlJSEzMxMzZ86sdRHC0dHR6M8uLy/HiBEj9I5nZmYmfvvtN7z33numqoJFofHsBMjW1hbV1dV66yZPnowvv/wSGRkZ+Pnnn7F161buvYyMDGi1Wqxfv54bAPH//u//9Lbv3bs3bty4AT8/v6avgEi9etzOnTuHfv364dNPP+XWGdLq6tq1K3bv3g2VSsUNBJCenq5Xpnfv3jhw4AB8fX1hbV13Gqjr74jPqGUnQL6+vkhLS8O9e/dQXFwMrVYLX19f9OvXD7NmzUJ1dTV+//vfc+X9/PygVquxadMm3L17F3//+9/1kiEAfP755zh//jzmzZvHtQD+8Y9/YN68ec1dPcF69bh17twZly5dwrFjx3Dr1i189dVXtZJWXSZPngytVos5c+YgOzsbx44dQ0xMDABwo21/9tlnKCkpwaRJk5Ceno47d+7g2LFjmDlzJpfg6vo74jVzdxoS08vJyWHvvPMOs7e317tlYMuWLQwAmzZtWq1tYmNjWatWrZi9vT0bNmwY27VrFwPASktLuTIXL15kYWFhzMnJiTk6OrIePXqwb775pplqJXyvHrebN2+yGTNmMBcXF+bq6srmzp3Lli5dynr27Mlto7v15FXnzp1jPXr0YLa2tiwoKIjt3buX+0ydW7dusdGjRzNXV1dmb2/P/P392YIFC5hWq60zHr7fekJDPBEiAnv27MHMmTNRVlYGe3t7c4djFtRnR4gA7dq1Cx07dkSbNm1w9epVfP755/jwww9Fm+gASnaECJJCocDy5cuhUCjQqlUrjB8/Ht988425wzIrOo0lhIgCXY0lhIgCJTtCiChQsiOEiAIlO0KIKFCyI4SIAiU7YjFmzJgBiUQCiUQCGxsbeHl5ISwsDNu3bzfqUaWdO3fC1dW16QKtx4wZMzBq1Khm3y8xDCU7YlHCw8NRWFiIe/fu4ciRIxg8eDD++Mc/Yvjw4dBoNOYOj/CZeZ9WI+Tf6nvO88SJEwwA++GHHxhjDQ8hf/LkyVrDia9YsYIxxtiuXbtYUFAQc3JyYl5eXmzSpEns0aNH3H5KSkrY5MmTmbu7O7Ozs2N+fn5s+/bt3Pv3799n48ePZy4uLqxFixbs97//Pfe86IoVK2rt9+TJk03yeyKvh1p2xOK9//776NmzJ+Lj4wEAUqkUGzduRFZWFv72t78hJSUF0dHRAIB+/fphw4YNkMvlKCwsRGFhIRYvXgwAUKvVWLVqFa5evYpDhw7h3r17mDFjBrefr776Cjdu3MCRI0eQnZ2N77//Hu7u7ty2w4YNg7OzM1JTU3Hu3Dk4OTkhPDwcVVVVWLx4MT788EOuZVpYWIh+/fo17y+KNMzc2ZYQnfpadowxNmHCBNatW7c633t1CPkdO3YwFxeXRveXnp7OAHCtwhEjRrCZM2fWWfbvf/8769q1KzciCGOMqVQqZm9vz44dO9Zo/MT8qGVHeIExxo3F1tgQ8vXJyMjAiBEj0K5dOzg7O2PgwIEAwA1zPnfuXOzbtw+BgYGIjo7G+fPnuW2vXr2K27dvw9nZmRvG3M3NDS9evBDsMOZCQ8mO8EJ2djY6dOhg0BDydamoqMCwYcMgl8uxZ88epKen4+DBg3rbffDBB8jLy8PChQtRUFCA0NBQ7hS4vLwcQUFBtYYxv3XrFiZPntzEtSemQKOeEIuXkpKCa9euYeHChQYNIV/XcOI3b97EkydP8L//+7/w8fEBAFy6dKnWvjw8PDB9+nRMnz4dAwYMwJIlSxATE4PevXtj//798PT0hFwurzNOoQ1jLjTUsiMWRaVSQaFQ4OHDh7h8+TL+53/+ByNHjsTw4cMxbdo0g4aQ9/X1RXl5OU6cOIHi4mJUVlaiXbt2sLW15bY7fPgwVq1apbfd8uXL8Y9//AO3b99GVlYWEhIS0K1bNwAvJyxyd3fHyJEjkZqaitzcXJw6dQp/+MMf8ODBA26/v/76K3JyclBcXAy1Wt08vzRiGHN3GhKiU3MWemtra+bh4cGGDBnCtm/fzqqrq7lyhgwh/8knn7CWLVvq3Xqyd+9e5uvry2QyGQsJCWGHDx+uNZdqt27dmL29PXNzc2MjR45kd+/e5T6zsLCQTZs2jbm7uzOZTMY6duzIZs+ezcrKyhhjjBUVFXHD1oNuPbE4NJ4dIUQU6DSWECIKlOwIIaJAyY4QIgqU7AghokDJjhAiCpTsCCGiQMmOECIKlOwIIaJAyY4QIgqU7AghokDJjhAiCpTsCCGi8P8Aiq/ViSCSy8oAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:13:20.125160Z", + "iopub.status.busy": "2024-02-29T03:13:20.124409Z", + "iopub.status.idle": "2024-02-29T03:13:20.340541Z", + "shell.execute_reply": "2024-02-29T03:13:20.339602Z" + }, + "papermill": { + "duration": 0.240272, + "end_time": "2024-02-29T03:13:20.342753", + "exception": false, + "start_time": "2024-02-29T03:13:20.102481", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUEAAAEmCAYAAAD8/yLTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABzqklEQVR4nO2dd3hTZd+A78zuARTaIoWyZO9RiwOBKrgRXgREBURQhih1ACpD1BeUIQoIn4OhgiC+KCqKQgUUKSBlQ0FAsKyWUbpH1vn+CEkzTlabdJ77unKRnvOc5zwJyS+//cgEQRCQkJCQqKHIK3oBEhISEhWJJAQlJCRqNJIQlJCQqNFIQlBCQqJGIwlBCQmJGo0kBCUkJGo0khCUkJCo0UhCUEJCokajrOgFVFUMBgOXLl0iJCQEmUxW0cuRkJCwQBAEcnNzqV+/PnK5c11PEoKl5NKlS8TExFT0MiQkJJxw/vx5GjRo4HSMJARLSUhICGB8k0NDQyt4NRISEpbk5OQQExNj/p46QxKCpcRkAoeGhkpCUEKikuKOq0oKjEhISNRoJCEoISFRo5GEoISERI1G8glKVHsEQUCn06HX6yt6KRJeRKVSoVAoyjyPJAQlqjUajYbLly9TUFBQ0UuR8DIymYwGDRoQHBxcpnkkIShRbTEYDJw9exaFQkH9+vVRq9VSYns1QRAErl69yoULF2jevHmZNEJJCNZAPkw6xcUbhcwZ2K5aCwWNRoPBYCAmJobAwMCKXo6El6lbty7nzp1Dq9WWSQhKgZEayIItf7Nu33mOXcqp6KWUC67KpiSqJt76Aa/wT8eSJUuIjY3F39+fuLg49u7d63DssWPHGDhwILGxschkMhYuXGg3ZubMmchkMqtHy5YtrcYUFRUxfvx46tSpQ3BwMAMHDiQjI8PbL63So9EbKnoJEhIVToUKwXXr1pGYmMiMGTPYv38/HTp0oG/fvly5ckV0fEFBAU2aNGHOnDlERUU5nLdNmzZcvnzZ/Ni5c6fV+UmTJvHDDz+wfv16duzYwaVLlxgwYIBXX1tlxWAo2VxQXo1NYQkJd6lQIbhgwQJGjx7NyJEjad26NcuWLSMwMJDly5eLju/WrRtz585lyJAh+Pn5OZxXqVQSFRVlfkRERJjPZWdn89lnn7FgwQJ69+5Nly5dWLFiBbt27WL37t1ef42VDb3FDqsKSQhKVGJGjBhB//79fX6fChOCGo2GlJQUEhISShYjl5OQkEBycnKZ5j516hT169enSZMmDBs2jLS0NPO5lJQUtFqt1X1btmxJw4YNnd63uLiYnJwcq0dVRKcvEYKSDKy83H333bz44osVvYwaQYUJwWvXrqHX64mMjLQ6HhkZSXp6eqnnjYuLY+XKlWzevJmlS5dy9uxZ7rzzTnJzcwFIT09HrVYTHh7u0X1nz55NWFiY+VFV22jpDCV+QMkclvA1Wq22opfgkgoPjHib++67j0GDBtG+fXv69u3LTz/9RFZWFl9//XWZ5p06dSrZ2dnmx/nz57204vJFb+ETVMjLJgSLtHp2nbmGRld1AiyCIFCg0VXIQ7BwRThjxIgR7Nixgw8++MAc3GvQoAFLly61GnfgwAHkcjn//vsvYHQvtWvXjqCgIGJiYhg3bhx5eXlW1+zcuZM777yTgIAAYmJimDhxIvn5+W6t6/LlyzzwwAMEBATQuHFj1qxZQ2xsrFWAUiaTsXTpUh5++GGCgoJ455130Ov1jBo1isaNGxMQEECLFi344IMPrObW6/UkJiYSHh5OnTp1ePXVV91+v8pKheUJRkREoFAo7KKyGRkZToMenhIeHs6tt97K6dOnAYiKikKj0ZCVlWWlDbq6r5+fn1M/ZFVBZ/CeObz8z7O8t/kk0x9szdN3NC7jysqHQq2e1tN/qZB7H5/Vl0C166/cBx98wN9//03btm2ZNWsWAPPnz2fNmjWMHTvWPG716tXcfvvtNGrUCDC6kz788EMaN27MP//8w7hx43j11Vf56KOPADhz5gz9+vXj7bffZvny5Vy9epUJEyYwYcIEVqxY4XJdTz31FNeuXWP79u2oVCoSExNFg5gzZ85kzpw5LFy4EKVSicFgoEGDBqxfv546deqwa9cuxowZQ3R0NI899pj59a1cuZLly5fTqlUr5s+fz7fffkvv3r1dv7FlpMI0QbVaTZcuXUhKSjIfMxgMJCUlER8f77X75OXlcebMGaKjowHo0qULKpXK6r4nT54kLS3Nq/etrFhqgoIAWr2B/kv+ZPya/R7/8p65YtQgruYVe3WNNZ2wsDDUajWBgYHm4N6wYcP4888/zf5tg8HA2rVrGTZsmPm6F198kV69ehEbG0vv3r15++23rSyg2bNnM2zYMF588UWaN29Ojx49+PDDD/n8888pKipyuqYTJ06wdetWPvnkE+Li4ujcuTOffvophYWFdmMff/xxRo4cSZMmTWjYsCEqlYo333yTrl270rhxY4YNG8bIkSOt1rZw4UKmTp3KgAEDaNWqFcuWLSMsLKysb6VbVGjFSGJiIsOHD6dr1650796dhQsXkp+fz8iRIwHjL88tt9zC7NmzAWMw5fjx4+bnFy9e5ODBgwQHB9OsWTMAXn75ZR566CEaNWrEpUuXmDFjBgqFgqFDhwLGD9ioUaNITEykdu3ahIaG8vzzzxMfH89tt91WAe9C+aK1yA00CAJpmQUcPJ/FwfNZPH17LF0a1XZ7rhsFGl8s0acEqBQcn9W3wu5dWjp27EirVq1Ys2YNU6ZMYceOHVy5coVBgwaZx2zdupXZs2dz4sQJcnJy0Ol0FBUVUVBQQGBgIIcOHeLw4cOsXr3afI0gCObywlatWjm8/8mTJ1EqlXTu3Nl8rFmzZtSqVctubNeuXe2OLVmyhOXLl5OWlkZhYSEajYaOHTsCxoyNy5cvExcXZx6vVCrp2rVruZjEFSoEBw8ezNWrV5k+fTrp6el07NiRzZs3m4MlaWlpVtn+ly5dolOnTua/582bx7x58+jZsyfbt28H4MKFCwwdOpTr169Tt25d7rjjDnbv3k3dunXN173//vvI5XIGDhxIcXExffv2NZsM1R1LTdAgCFZCcdWufz0Sgpn5RiFYTq4bryCTydwySSsjw4YNMwvBNWvW0K9fP+rUqQPAuXPnePDBBxk7dizvvPMOtWvXZufOnYwaNQqNRkNgYCB5eXk8++yzTJw40W7uhg0bem2dQUFBVn+vXbuWl19+mfnz5xMfH09ISAhz585lz549XrtnWajwT4PJJyGGSbCZiI2NdfnLsHbtWpf39Pf3Z8mSJSxZssTtdVYXdDbmsGXKzM9HL3M1tzV1Q9zzfWZVQU2wqqBWq+1afz3++OO88cYbpKSk8M0337Bs2TLzuZSUFAwGA/PnzzcrDrbBwM6dO3P8+HGz1eQJLVq0QKfTceDAAbp06QLA6dOnuXHjhstr//zzT3r06MG4cePMx86cOWN+HhYWRnR0NHv27OGuu+4CQKfTkZKSYqV5+opqFx2WcI6tT9CydE6rF1i7N03sMlHMmiBVSBWsIsTGxrJnzx7OnTvHtWvXMBgMxMbG0qNHD0aNGoVer+fhhx82j2/WrBlarZZFixbxzz//8MUXX1gJSYDJkyeza9cuJkyYwMGDBzl16hQbN250qIRY0rJlSxISEhgzZgx79+7lwIEDjBkzhoCAAJc1vM2bN2ffvn388ssv/P3330ybNo2//vrLaswLL7zAnDlz+O677zhx4gTjxo0jKyvL/TesDEhCsIZh6xO01AQBVu9JQ+dGTbFObyCnSOf19UkYefnll1EoFLRu3Zq6deuaAyLDhg3j0KFDPProowQEBJjHd+jQgQULFvDuu+/Stm1bVq9ebfalm2jfvj07duzg77//5s4776RTp05Mnz6d+vXru7Wmzz//nMjISO666y4effRRRo8eTUhICP7+/k6ve/bZZxkwYACDBw8mLi6O69evW2mFAC+99BJPPvkkw4cPN5vMjz76qFvrKisyobyScaoZOTk5hIWFkZ2dXaV2mzt8IYuHF/8JwLfjelCg0TPs0z00jggit0jLtTwNS4d15r520U7nuZZXTNe3twLwbM8mTL3PsVO9oigqKuLs2bM0btzY5RdVwnMuXLhATEwMW7dupU+fPuV+f2f/v558PyVNsIZh5ROkRDMMVCsY0s3oHF+VfM7lPDfyS/yBl7KKWPnnWQo0kmZYnfntt9/4/vvvOXv2LLt27WLIkCHExsaa/XhVFUkI1jCsfYIC2pvmsFIh5/G4hijkMnb/k8nfGblO58m0EII/HLrEzB+O89+fUn2zaAmf88cffxAcHOzwAcYSuNdee402bdrw6KOPUrduXXPidFWmwqPDEuWLpQ/QIGD2/6kVMuqHB3BPq0g2H0vni+R/eat/W4fz3Ciwrwn9/e9r3l+wRLnQtWtXDh486HRM37596du3YnIsfYkkBGsYlg0ULKPDyptpFU/FN2LzsXQ27L/Aq/1aEOIv/isvligtRYmrLgEBAaVKnakOSOZwDUNnkyxt0gxVSuNHIb5pHZrVCyZfo2fD/osO57E0h01IITaJqogkBKsQeoPAnn+uiwogt+fQi1eMqG52lJHJZDwVbyzI//aAYyF4QxKCEtUEyRyuQizZdpoFW/5GIZcR36QO/dpG0bdNlNsVHmBfMaK9+bdKUfJ72LtlPaZvPMbRi9kUavQEqO1rXsV8glK2lURVRBKCVQSNzsDnN1NX9AaBnaevsfP0NaZtPEq32Nrc3zaKfm2jiQpzng9n10XmZi9ApaIk6/+W8ACiw/y5nF3EgfM36NE0wm4eMZ+gQZKBElUQyRyuIvxyLJ1reRrqhvixNbEnU+5rSYcGYQgC7D2bycwfjnPb7CQGfPQnn/z+D+czC0TnsQyMGATB/LfaQhOUyWR0jTU2Uth3Trw2VNQnKAVGJKogkhCsIny529g9eGi3GJrVC+a5nk3ZOOEOdk7uxRsPtKJrI2NLo/1pWbzzUyp3vreNhxbt5KPtpzl7raRzsM7OJ2jKE7Su/+wWa5zvr3OZousRa54gaYKVB7GOz999912FracyI5nDVYBTGbnsOZuJXAZDulu3PGpQK5Bn7mzCM3c2ISOniF+OpfPTkcvsPZvJkYvZHLmYzXubT9IyKoT72kZTrCvpTGJqqgrWPkGArjdbau3/9wY6vQGlzXkpOly1uHz5smjvPwlJCFYJVu8xFs/3aRVJ/fAAh+MiQ/15Kj6Wp+JjuZZXzK/HMvj56GV2nbnOifRcTqRbV4EICA6FYIuoEEL8leQW6TiRnkvbW0q6/DpuniBJwcqKN7esKC2CIKDX61EqK5fYkczhSk6BRsf/Ui4A8MRtjdy+LiLYj8fjGvLFqDhS3khg7n/a07tlPSvfn8FQYh6rbMxhhVxGhwbhABy/bL29aFah+A5iVUITFATQ5FfMw8M3KDc3l2HDhhEUFER0dDTvv/9+qbfitDSHz507h0wmY8OGDfTq1YvAwEA6dOhgt+Wsq02ZvvjiC7p27UpISAhRUVE8/vjjVnuObN++HZlMxs8//0yXLl3w8/Nj586dHq/d11QukSxhx/cHL5FbrKNRnUDubGYfpXWH8EA1g7rGMKhrDDlFWtrP/BUw+gTNFSMK+9/DsEBjtUihxrq5p1iOoGm+So+2AP7rXusor/PaJVAHuR53k8TERP7880++//57IiMjmT59Ovv37ze3pS8rr7/+OvPmzaN58+a8/vrrDB06lNOnT6NUKt3alEmr1fLWW2/RokULrly5QmJiIiNGjOCnn36yus+UKVOYN28eTZo0qZQmuSQEKzGCIPDFzYDI490bIi/jFpkAof4qujaqxb5/b9ysHbbPEzRh2hOjUGstBB0la1cBEVhlyM3NZdWqVaxZs8bcpmrFihVu9/5zh5dffpkHHngAgDfffJM2bdpw+vRpWrZsabUpExgbo3744Yf07NmTpUuX4u/vz9NPP22eq0mTJnz44Yd069aNvLw8c9MFgFmzZnHPPfd4bd3eRhKClZiD57M4dikHtVLOoK7e2+y9pBGwfcWIJWYhaKsJiiRKQxUxh1WBRo2sou7tJv/88w9arZbu3bubj4WFhdGiRQuvLad9+/bm56bdGC+nZ9CyZUu3NmVKSUlh5syZHDp0iBs3bmC4mW6VlpZG69atzdeJbbxUmZCEYCXmy93GgMiD7aKpHaT22rymdugGAXOKjKl22BJTpUiR1lYIOtAEq4IUlMk8MkmrM5YtsEyfiX+u5nK7zuByU6b8/HxzV5nVq1ebu1/37dsXjcb682G78VJlQxKClZSsAg0/HjZqLMM8CIi4g0nps6wdVopogv6emsNVQAZWFZo0aYJKpeKvv/4y7wSXnZ3N33//7fMmplmFGpebMh05coTr168zZ84cYmKMVsq+fft8ui5fIUWHKynfpFygWGegVXQonRuGe3VuuYUmaK4YEdEE/VXGY3aaYFUOjFQRQkJCGD58OK+88grbtm3j2LFjjBo1Crlc7nJjI2/galOmhg0bolarzRs7ff/997z11ls+X5cvkIRgJcRgEMy5gU/c1tDrH3rTdIIgoNHdrBiROwuMWG+8ZPIJ2mqPkgj0LgsWLCA+Pp4HH3yQhIQEbr/9dlq1auXz/VJkuN6UqW7duqxcuZL169fTunVr5syZw7x583y6Ll9R4UJwyZIlxMbG4u/vT1xcHHv37nU49tixYwwcOJDY2FhkMplVWZCJ2bNn061bN0JCQqhXrx79+/fn5MmTVmPuvvtuZDKZ1eO5557z9ksrNbvOXOfstXyC/ZT073iL1+c3aYKChSZomycI9oGRTYcvc8e7v7Hj76sA1LLxU1pqgslnrrMo6RQGqZau1ISEhLB69Wry8/O5fPkyY8aM4eTJk241Pz137pxVPqEgCPTv3x8o2b/bMtUmPDycQ+dv0C3+DoxiELp168avv/5Kbm4ueXl5HDp0iNdee818zdChQzl79ixFRUXs2rWLhx56yGreu+++G0EQCA8PL+M74VsqVAiuW7eOxMREZsyYwf79++nQoQN9+/a1Sri0pKCggCZNmjBnzhyHGfA7duxg/Pjx7N69my1btqDVarn33nutkjwBRo8ezeXLl82P9957z+uvr7SY6oQf7XQLQX7ed9uWBEYcV4yAfWBk/Jr9XLhRyLW8YgBqB1oLQUtreOgnu5m/5W++P1RBkdhqwIEDB/jqq684c+YM+/fvZ9iwYQA88sgjFbyy6kWFBkYWLFjA6NGjGTlyJADLli1j06ZNLF++nClTptiN79atG926dQMQPQ+wefNmq79XrlxJvXr1SElJsXIoBwYGVopSIlvSs4vYkpoBeFYh4gkmnc8qOiwiBP2U4oERE7WCrFvvi+l8F26Id7ORcI958+Zx8uRJ1Go1Xbp04Y8//iA1NZX77rvP4TV5eXnluMKqT4UJQY1GQ0pKClOnTjUfk8vlJCQk2JXvlIXs7GwAateubXV89erVfPnll0RFRfHQQw8xbdo0AgMd53EVFxdTXFxs/jsnJ8fh2LLw1d409AaB7rG1aREV4pN7yC18gubosJg57CBFxoRd2o6IFBQTrhLu0alTJ1JSUuyOFxYWutwUqSz4PuxSuagwIXjt2jX0ej2RkZFWxyMjIzlx4oRX7mEwGHjxxRe5/fbbadu2ZOe0xx9/nEaNGlG/fn0OHz7M5MmTOXnyJBs2bHA41+zZs3nzzTe9si5HaPUG1v5lDIgMu62hi9Glx8oneFMTVHtQMWLCVgiKRYclIeh9fL4pUg2TgtU6T3D8+PEcPXrUrmh7zJgx5uft2rUjOjqaPn36cObMGZo2bSo619SpU0lMTDT/nZOTY86P8hZJqRlk5BRTJ0hNv7a+M9XFfIKimuBNIViksReCMhmEBbg2h8WSsMubKpHEXcXIyCniRoGGpnWDK+yHzlv/rxX2CY2IiEChUJCRkWF1PCMjwyu+ugkTJvDjjz+ybds2GjRo4HRsXFwcAKdPn3Y4xs/Pj9DQUKuHtzFViDzWLcbsj/MFMnOytON+ggABauMxMU0w1F+FwiZ1R+xDqRYRruWFqSKioEDyS3qbjJwiNDoDV3OLXQ/2EabKFIWibN+VCtMETY7epKQkc+jeYDCQlJRkTsgsDYIg8Pzzz/Ptt9+yfft2Gjdu7PIak3/FVD9ZEZy9ls/O09eQyYzNEnyJ2SeIYBEYsRdWzgIj4YEqyyJkQLyzdEWawwqFgvDwcHO2QWBgYLkkGldVBJ1RqGiL5RQpnWtZprE6DRQVlf97ajAYuHr1KoGBgWXuT1ih5nBiYiLDhw+na9eudO/enYULF5Kfn2+OFj/11FPccsstzJ49GzBK/uPHj5ufX7x4kYMHDxIcHGz2kYwfP541a9awceNGQkJCSE9PB4zF5wEBAZw5c4Y1a9Zw//33U6dOHQ4fPsykSZO46667rArKy5vVN9Ni7r61LjG13S+0Lw1WFSNupcgY7LQ8W1PYEsvNnH7/+yrzf/2bhUM60i22tsNrfIXJqnCUdiVRwpUbhQBoAlVkuUjNMo0t8FOSH+j4s+BL5HI5DRuWvZigQoXg4MGDuXr1KtOnTyc9PZ2OHTuyefNmc7AkLS0NuUUlw6VLl+jUqZP573nz5jFv3jx69uzJ9u3bAVi6dClgTNS0ZMWKFYwYMQK1Ws3WrVvNAjcmJoaBAwfyxhtv+PbFOqFIq2f9zcapT8b7Ji3GkpLAiIBG77piBKBYZ101Ehagcug/N5nYAH+cusb1fA07Tl6tECEok8mIjo6mXr16aLXi3W8kjDyzYTsAL/RpzsMtnSfpm8Y+2vkWJvRybW35ArVabSUfSkuFB0ZMjRrFMAk2E6ZMd2e4Oh8TE8OOHTs8WqOv+fHwZbILtdwSHkDPW+v5/oYmn6DBYrc5peMGCgD5xdbt9J1pghoLIWj633CUZlNeKBSKMvuOqjsXc43/R1qZ0mVpnmlsgU7h8zI+X1PxoTuJksapcQ1ReKFxqivMmiAW+w6L/KIq5DJz6oxtS/2wAJWtS9CMxkJrNKXNOEqzkah8yGuY31QSghXM0YvZHDqfhUohY3A376bcOEJuGR02OO4nCCWdZGw7xzjTBC3NYZNAlIRg1aFmiUBJCFY4pjrhfm2jiQj2K5d7WvoEnXWWhpLgiG0PQafmsM5eCFa0OSzhPp4ogtVBaZSEYAWSU6Rl40Fjg4En4nybFmOJ6XOrMwjmpgeOUllMwRHbbtLhgSpkDnQGS01Qd1PTLLJpxyVRefEk2loNZKAkBCuSDSkXKNTquTUymO6Nyy9yavqQF1sIJrGKESgJjmTm2/sEHWHqUWiJ7T4lEpWX6iDYPEESghWEIAh8aW6c2qhck3hNlq9GXyKYHGmC/g40wVBngRG9vdYn+QSrDjUtoVwSghXEnrOZnL6SR6BawaOdvN841Rkmn6Cl786VOeyJT1ArIgQln2DVwRMRWB3kpSQEKwhTWswjHW8hxL98M+5NH1yTEJTLcJiaYwqMZNn5BB3vfqfRSZpgVcaT/OPqoDVKQrACuJJbxC9HjeV8T/iwZZYjzD5BU46gk/peZ5qgo4+/qDks+QSrDI4CXtUVSQhWAF//dR6dQaBTw3Da1A8r9/vLbTRBsV6CJvxMeYI2G64Hqe2rL0zapKQJVm2qgXLnEWUqm8vLyzPvOm/CFy2mqhN6g8BXe88D8ESc7+uExTD5BIud9BI0YdIEr+dZt0wyblBlPTbwpmAU8wkWSyky1ZLqIC891gTPnj3LAw88QFBQEGFhYdSqVYtatWoRHh5OrVq1fLHGasW2E1e4mFVIeKCKB9pXTOsuW5+gs3ZXJiGYU6RzOMZEkFppNa8lGr3B3LFGonJT08rmPNYEn3jiCQRBYPny5URGRlYLx2h58uUeY0Dksa4xVg0KyhO5jU/QUbUIlARGxLD9vw/yc6wJAhTpDARL7fYrPTXtK+2xEDx06BApKSm0aNHCF+up1pzPLDDv2evrxqnOKNEEjX46Zy3wPRHU6ptNWE3tuWwp1OgJ9sEWohLexaPASDUQmB7/LHfr1o3z58/7Yi3VntV70hAEuLN5BLERQRW2Dts8QaUTTdATIWhqYyZmDoOUK1hV8KSRUXWIJHv8s/zpp5/y3HPPcfHiRdq2bWvex8FERXZnrswU6/R8ve9mQMRH+wm7i+ljq3HSVdpEgIgQbFYvWHSsqW2WI3NYihBXDSRz2AVXr17lzJkz5hb4YPQNCYKATCZDr5c+6GL8fCSdzHwN0WH+9GlZDo1TnSCXW9cOOxWCautzt4QHsPqZONGxpq76jjRBKVew8mLdjLhmSUGPheDTTz9Np06d+Oqrr6TAiAeYWmYN6dbQaXJyeWD2CZo1QdcpMiYm3XMrkaH+VvOYMBica4KSOVx5sZSBNe0r7bEQ/Pfff/n+++99u/lzNeNEeg77/r2BUi5jSPfyaZzqDDufoNNkaWshKGYemzC48AlK5nDlxVIP9CRFpjoITI9Vkt69e3Po0CFfrKXaYtIC720TadaiKhKzT9CNihFboWdpHts6xc3msKQJVjkMFqqgRw0UvL+UcsdjTfChhx5i0qRJHDlyhHbt2tkFRh5++GGvLa46kFes49v9F4GKqxCxxTZP0J2KERPOosVSYKTq8ndGrvl5ddDuPMFjIfjcc88BMGvWLLtzUmDEnu8OXCRfo6dJ3SDim9ap6OUA9rXDzgMjjs1hW7PX5BN0HBiRKkYqKw98uNP8XBKCLrCtFZZwjCAIZlN4WFz5Nk51hmkdpQmMWArFnCLrpgomc1jrKFla0gSrBB61168cH+ky4ZFPUKvVolQqOXr0qNcWsGTJEmJjY/H39ycuLo69e/c6HHvs2DEGDhxIbGwsMpmMhQsXlmrOoqIixo8fT506dQgODmbgwIFkZGR47TWZkMlkzBvUgcfjGvKfzg28Pn9psf3gOtMETV1kTFgKRVuz11VgRPIJVg2qgVzzCI+EoEqlomHDhl4zedetW0diYiIzZsxg//79dOjQgb59+3LlyhXR8QUFBTRp0oQ5c+YQFRVV6jknTZrEDz/8wPr169mxYweXLl1iwIABXnlNtrS9JYz/PtqOsMDybZzqDNvon9iewyacaYIOhaAUGKnSeLbRUtUXmR5Hh19//XVee+01MjMzy3zzBQsWMHr0aEaOHEnr1q1ZtmwZgYGBLF++XHR8t27dmDt3LkOGDMHPT3x7SldzZmdn89lnn7FgwQJ69+5Nly5dWLFiBbt27WL37t1lfk1VAduyKLXS/bI5S6Fo2x6rtMnS/17PJ9fGtJaoODwqm6v6MtBzn+DixYs5ffo09evXp1GjRgQFWdfA7t+/3615NBoNKSkpTJ061XxMLpeTkJBAcnKyp8tye86UlBS0Wi0JCQnmMS1btqRhw4YkJydz2223ic5dXFxMcXFJT72cnJxSrbEyYPtL70wTVCnkqBQys5/PUigW6ayFWmmiw2eu5tFn/g4CVApS3+rn3guQ8CnVQbvzBI+FYP/+/b1y42vXrqHX64mMjLQ6HhkZyYkTJ3w2Z3p6Omq1mvDwcLsx6enpDueePXs2b775ZqnWVdnwxCcI4K9UoNXrUClkVmNt9xJ2VTEiJgR3nrrm8JxExVAdtDtP8FgIzpgxwxfrqPRMnTqVxMRE8985OTnExFR89UdpsPUJOosOA/irFeQW6+xMY1sfnytzWMwnaF2zKlFWirR61uxJo0+rejSqU7pORTVMBpa+vX5KSgqpqakAtGnThk6dOnl0fUREBAqFwi4qm5GR4TDo4Y05o6Ki0Gg0ZGVlWWmDru7r5+fn0A9Z1bD1+bjSBE1+QNsgib0QNAVGHPcTtEUSgd5l89F0Zv14nMMXslg4xLPvpAnPAiNVH48DI1euXKF3795069aNiRMnMnHiRLp06UKfPn24evWq2/Oo1Wq6dOlCUlKS+ZjBYCApKYn4+HhPl+X2nF26dEGlUlmNOXnyJGlpaaW+b1XDLjrsQhM0C0GbxOli22Rpc4qMtbAzNVIVM3klRdC7XL+5K2BecendCx6Zw9XAdvZYCD7//PPk5uZy7NgxMjMzyczM5OjRo+Tk5DBx4kSP5kpMTOSTTz5h1apVpKamMnbsWPLz881tup566imrIIdGo+HgwYMcPHgQjUbDxYsXOXjwIKdPn3Z7zrCwMEaNGkViYiLbtm0jJSWFkSNHEh8f7zAoUt1x6RNUu6sJGv+1TZY2bdRu60MESRP0NiX/J6V/Z6u+WPMMj83hzZs3s3XrVlq1amU+1rp1a5YsWcK9997r0VyDBw/m6tWrTJ8+nfT0dDp27MjmzZvNgY20tDTkFpHLS5cuWZnd8+bNY968efTs2ZPt27e7NSfA+++/j1wuZ+DAgRQXF9O3b18++ugjT9+KKovHPsGb7fdtfYJBNq3y9Q7K5kIDVFzMKuRqrvWOdSD5BL2NyeVgKMPbWlkqm8qLUpXN2TZNAGMidWlK6iZMmMCECRNEz5kEm4nY2Fi3vjTO5gTw9/dnyZIlLFmyxKO1Vhc89gne1AQDbczhBY91YNK6Qzx5WyNe/d9h9AYBQRDsosMh/saP2cWsQop1evyUJfNIMtC7mFwOZflx8UQGVgdxWapWWi+88AKXLl0yH7t48SKTJk2iT58+Xl2chG+Qy219gqULjDSrF8IPz99B3zYlASWdQbDTBBUW36preRqrc4JkEHuVgpuaoCfvak3Xxj0WgosXLyYnJ4fY2FiaNm1K06ZNady4MTk5OSxatMgXa5TwMra/3mo3AyP+DrbfVFhcrzcIdmVzESElUXXb7T1r+PfPJVdyizyqpikya4Lu36Om/x94bA7HxMSwf/9+tm7dak5AbtWqlVUFhkTlxpOKEXAcGCm5vmQ+ncHeHO7QIIwfDhktB73NN66Gf/+ckpmvofs7xiyGc3MecOuawtJogp4uzILq4D4sVZ6gTCbjnnvu4Z577vH2eiTKAbvAiJN9h8FYMQLuCcFird7OKe+nlOOnlFOsM5iDJyZquhbijCMXsz2+pjQ+wTL5D6uBV7BUQjApKYmkpCSuXLliFwxx1PxAovJgFxhxUTF/a6Rxi83mkeJbbSosri8QSYhWK+XmMXN/Ocn8QR3MfkiDJAUdUpr3prAU5nBZIsnVAY+F4JtvvsmsWbPo2rUr0dHRNS6cXh3wtHZ4cLcYbm8WQYNaAQ7mk6GUy9AZBNGEaLVSbg6ObDx4iS6NavFUfGyp1l6TKI2GZvYJemDkehqcqm6BFI+F4LJly1i5ciVPPvmkL9YjUQ7Y+QRdBEZkMhkxtQOdjlEqjEJQVBNUKKwi0ifSc+3GSNhTmibuZp+gDwMjlppjddCBPI4OazQaevTo4Yu1SJQTtj5BZ7vNuYspuFKg0dmdUyvlViZ4kYWgNNR0W8wJpXlnTJq4L90M1U0T9PjT/8wzz7BmzRpfrEWinLB1AXpjM3iTz0+sSYKlTxCs/YaSDHRMqXyC5aAJWg6vBoqg5+ZwUVERH3/8MVu3bqV9+/Z21SMLFizw2uKqDcV54CceVKgI7H2CZf8om+YQ9Qkq5FbaZ76Ftvj+1r/LfO/qSmk0LnNgxJP7eOwT9Gh4pcdjIXj48GE6duwIYLfhkhQkESFlFfz2Noz8GSKaVfRqALHaYe9pgmI+QZVCZqUJSnuNuIeD3rQOEQSLwJRPNUGLjdqrwVfeYyG4bds2X6yjeqLXQcpKyL8CXz4Ko7ZCSKTLy3yN7Y+VN4SgyScoZg7LZDIrwSsmKCXs8dQcLtYZzALNk2s9VewqVBPMuQyh0V6dsuyffgnHKJTw+NdQqzFkpcHq/0BRxe9NYu8TLPvPuWkOMQEnl1nnEjracEnCGk9ljaWG7cva4QoTgldS4aM42DLDq4uQhKCvCa4LT26AoLqQfhjWPQE6jevrfIhtlr93osOmwIh9dFguk7lMqJawx1PhZOmP9ahixKO72JrD5WQPZ1+ALwdCUTakJYPOvi1baZGEYHlQu4lRI1QFwdkdsHFc6ZLAvIRPNEFzioyYOWx9T7E0Ggl7PDWHLTVszzRBj25T/ppg4Q2jAMy5CBEtYOhaUPl7bXpJCJYXt3SGwZ+DXAlH1sPW6RW2FF/4BM2BEZGgh60mKNZhWsIeT38nrTVBDy4sQ4qMz9EWwVePw9UTEBINT/wPAmt79RYef/p///13dDr7X3KdTsfvv//ulUVVW5olwCM3G7nuWgTJFdPU1b52uOxC0JQiU+TA1LUMjNi22pIQp0yaoEfmcCUtmzPoYcMzkLYL/MKMAjDc+zs8evzp79WrF5mZmXbHs7Oz6dWrl1cWVa3pMAQSZhqf//IaHPmm3Jdgpwkqy24OO0uRsdUEJdzDU1lTWOrAiGf3sUqW9tV/qyDAz5Mh9QdQqGHIaohs45NbeSwEBUEQdYZev36doKDS7XNa47j9Rej+rPH5t8/BPzvK9fZ2PkEvaIKmqhNRc1iOJARLQVk0wWu5xbzx3RGOutGOy+PASHko8n/Mh78+AWQw4GNofKfPbuV2nuCAAQMAoxYxYsQIqz149Xo9hw8flmqK3UUmg36zIS8djm+EtcPg6Z8hql253N7TjZbcwVl0WIbM7p4SrvG0pNBSE7yUXcSXu9PILdLxgYv9hz1OkbGMDvuicO7AavjtLePzfnOgzaPev4cFbgvBsLAwwPiGhYSEEBBQ0lZJrVZz2223MXr0aO+vsLoiV8CjH0P+dfh3J3z5Hxj1K9Rq5PNbW8ojpVzmlTQH5+awpAmWBk81QbFKnOxC1635K1Wy9Kkt8P3zxue3vwi3PefDmxlxWwiuWLECMO749vLLL0umrzdQ+Rt9HSvuhyvHjGkAo371evTLFkuh543IsOU84hUj1pstgWO3ikQJHucJirz3BW5swl5pfIIXUuDrp0DQQ3sL37mP8fgbMGPGDEkAepOAcHjiGwhtANdPwZrHQFPg01taKmXeyBEE55qgTCbD1u0oRYhd47k5bP+e5ruRk1kposPXz8CaQaAtgKa94ZHF5VaY7LEQzMjI4Mknn6R+/foolUoUCoXVozQsWbKE2NhY/P39iYuLY+/evU7Hr1+/npYtW+Lv70+7du346aefrM7LZDLRx9y5c81jYmNj7c7PmTOnVOsvM6H1jeF//3C48Bd887Sx7thHWPrnvFEtAiV+RbFEaLnM3icolc65xuPAiIg5nF/sxueoDE1VvSIP867AF49CwXWI7giPfQ4K+73NfYXHDRRGjBhBWloa06ZN80p7/XXr1pGYmMiyZcuIi4tj4cKF9O3bl5MnT1KvXj278bt27WLo0KHMnj2bBx98kDVr1tC/f3/2799P27ZtAbh8+bLVNT///DOjRo1i4MCBVsdnzZpl5ccMCQkp02spE/VawuPr4PNH4O+fYVMiPPSBT34NZT7RBG+awyJfRBn2PsECjZ5w582qazwea4IiP0D5bvzYlKVsrswU5xpr6rP+hVqxMGw9+JXv99BjIbhz507++OMPczutsrJgwQJGjx7NyJEjAWP7/k2bNrF8+XKmTJliN/6DDz6gX79+vPLKKwC89dZbbNmyhcWLF7Ns2TIAoqKirK7ZuHEjvXr1okmTJlbHQ0JC7MZWKA1vg4GfwddPwv5Vxgz5XlO9fhvLiJ7XfII3hZxWb/8FEdMEpdI515SldtiEO5qgx9qcpSZYFoGo0xh9gJcPQWAEPLEBgu0VH1/j8TcgJibGaz4BjUZDSkqK1Z7FcrmchIQEkpOTRa9JTk622+O4b9++DsdnZGSwadMmRo0aZXduzpw51KlTh06dOjF37lzRShgTxcXF5OTkWD18QqsH4f55xuc75sC+FV6/haVS5i0h6Cz6KxOJDjtqoqA3CHz6xz9u5bdVZ06k5/D2plTz3+585wo19j7BAo3e5RYGHvsELZ+XVhQYDPD9BDjzG6gCYdjXUKdpKScrGx5/AxYuXMiUKVM4d+5cmW9+7do19Ho9kZHWPfYiIyNJT08XvSY9Pd2j8atWrSIkJMSc52hi4sSJrF27lm3btvHss8/y3//+l1dffdXhWmfPnk1YWJj5ERPj/fIdM91GwV1GTZdNiXDiJ+fjPcRy0yNv5AiCc7Pa2EDBPSG47q/zvL0plQcX7QRg24krDPk4mfOZvg0WVTbGfrnf42scNasV0xAtqZAGCkkz4fA6kCmMPsBbunhh0tLhsRAcPHgw27dvp2nTpoSEhFC7dm2rR2Vj+fLlDBs2DH9/664TiYmJ3H333bRv357nnnuO+fPns2jRIoqLxVv0TJ06lezsbPPj/Pnzvl14r9eh0xPG9PxvRkLaHq9NbRUd9kK1iKt5jGVz1sccBUZsNxwfufIvdv+TyUvrD5V5jVWJGwXW7dY++eMfl9qgI2HnyiQud5/g7mXw5wfG5w8vgub3lG2+MuKxT3DhwoVeu3lERAQKhYKMjAyr4xkZGQ59dVFRUW6P/+OPPzh58iTr1q1zuZa4uDh0Oh3nzp2jRYsWduf9/PysqmR8jkwGD34AeVfh1C/w1WB4+leoe6s3Jjc/Uykrxhx2lLqhc5A6czXXe/3jqgK2Jux/fzrBLeGBPNDecVdlRz8sroIjnrq3rKPDHgrEoxtg801ff5/p0GmYZ9f7AI+F4PDhw712c7VaTZcuXUhKSqJ///4AGAwGkpKSmDBhgug18fHxJCUl8eKLL5qPbdmyhfj4eLuxn332GV26dKFDhw4u13Lw4EHkcrloRLrCUChh0ApY9RBcTIEvB8CoLWVuL27lE/RSJYczs1o8MCL+xdQ58F/pa9i2dGKy5Z+reU6vKbUm6LE5LFg89+DCs7/Dt88CAnQbDXckenZjH1EqNeDMmTO88cYbDB06lCtXrgDGNJRjx455PFdiYiKffPIJq1atIjU1lbFjx5Kfn2+OFj/11FNMnVoSIX3hhRfYvHkz8+fP58SJE8ycOZN9+/bZCc2cnBzWr1/PM888Y3fP5ORkFi5cyKFDh/jnn39YvXo1kyZN4oknnqBWrVoevwafog4yNmSt3RSyz99s0V+2oIGlQPJeYMTxPGIpMq9+c5iMnCK7sVoHmqAv99GtjOhL8Xod+QS9LwQ9Gw9A+lFjjbxeA60ehvverTS7NHn8DdixYwft2rVjz549bNiwgbw846/ToUOHmDFjhscLGDx4MPPmzWP69Ol07NiRgwcPsnnzZnPwIy0tzSrvr0ePHqxZs4aPP/6YDh068M033/Ddd9+ZcwRNrF27FkEQGDp0qN09/fz8WLt2LT179qRNmza88847TJo0iY8//tjj9ZcLQRHGFv3BkZBx88NUhvbilkLQW3mCSqfmsMyubA7gw6RTdsd0Iik2UP22eXSFmObrSmY40q5dbWdQFh+fW1dm3fzxLs6Bhj1gwCfG2vlKgsfm8JQpU3j77bdJTEy0Si7u3bs3ixcvLtUiJkyY4ND83b59u92xQYMGMWjQIKdzjhkzhjFjxoie69y5M7t37/Z4nRWKKZF0xQNw7g+jWTFwOXb1aG5g+WXyVsWIM2Eql1lHpE2IWbiSOWxETOi7+iFwZA7n+VATdHltQabRjZN7Geq2gqFrvNoa3xt4/A04cuQIjz5q39qmXr16XLt2zSuLknBAdAcY/AXIVXDsW/j19VKpSL6oGCmNJhjib/8brHPQU14yh13jqKu3q8R0T+9k+X/hVIvUFsJXQ+Da3xB6i7E0NKCSuZsohRAMDw+3K0sDOHDgALfccotXFiXhhKa9oP9S4/PdHxnb9HuIL3yCSifzONIEP/79H7tjjszhGicES6H5Og6MeDc67NZovQ6+GQXn94D/zdb4YZVTPnj8DRgyZAiTJ08mPT0dmUyGwWDgzz//5OWXX+app57yxRolbGk/CO592/h8yzQ4/LVHl/tECNoIOcs/ZSJ5go5wHBgp7cpqBlq9waErwet5gq6iw4IAP70MJzeBws+4O1y9Vh7epfzw+Bvw3//+l5YtWxITE0NeXh6tW7fmrrvuokePHrzxxhu+WKOEGD2eh9vGG59/N9ZYfuQmlpap1ypGbISgpWYo1k/QEY6+yDVJE3Rkvjp7B5xVhbjOE3RnVeLrEL3097mQsgKQwcBPoVHl7jjvcWBErVbzySefMG3aNI4ePUpeXh6dOnWiefPmvlifhDPufdvYov/o/2DdkzBiE9Tv6PIy636CXkqRsZlHJZdhqnmQy2Si5rAYjpKlXdW/Vic6vPmrx9c48geCO00UPO0n6OgPIGUVbHvH+Pz+udD6YY/mrgg8FoImGjZsSMOGDb25FglPkcuN/sG8K8aI8epBxs7UtRs7vcyy/ZnXosOimqDxiynDvnbYEWJdaKBmpcg4eg+c4VwT9G502KHQPLkZfpxkfH7nS9C9amy34ZYQTExM5K233iIoKIjEROdZ3gsWLPDKwiTcROlX0qI/42hJi/6gCIeXWOUJeqlixE4IWvzt7pab/1zNcxgdLk20tCbhLBfQVYt9z6PDItee/wvWjzC2xu84DHpP83DWisMtIXjgwAG0Wq35uSOkPSMqCP8wGPYNfHYvZJ4xaoQjfjRWm4hg+b/krdph21Qby7/FusiI0Xu+461Ha5JPsDSUpyZolyd47ea2ELpCaHaPz5oB+wq3hOC2bdtEn0tUIkKjjVUln90Dl/Ybf5WHrBFtU24VHfaaJih3+LexgYLFUv2V5BS5bva5/eQV8/Ma5BJ0iDNhVRafoOf9BEvGB2quwhdjoTAT6neGx1aVa2t8b+AdNUCichDR3FhnrAyAU7/Cjy+KfnOso8O+8QlaRp3lNsnS7loMI1b8ZX5ekwIjpcGkCQao7MvRXOcJenYv0/hgCvhP6ouQnQa1mxgrmhxYH5UZtzRB24akztiwYUOpFyPhBWK6w3+Ww7phcOBLY4v+3tapS5aRWq9Fh22EoOXfMpt7lsZScpQ6I2HEJASD/BR2prEvzGEVOpap3qdewSkIqmtsje/ED12ZcesbYNlROTQ0lKSkJPbt22c+n5KSQlJSknmDdokKpuX98OD7xue/z4W/PrU6beUT9FKeoK1Gafm3nSbolTtKWGLqJRigttcEvd1AwWDQM0+1jDsUxyiWBxo1QBcZCZUZtzRB08brAJMnT+axxx5j2bJl5i029Xo948aNIzQ01DerlPCcLiMgNx22z4ZNLxs70LR6CPBVKy0XgRG55+ZwTeKnI5d5Z1MqHw3rXKrrTW20AlX2X2lvN1CI2vMObRW70AoKvrt1DoPrd/JsgkqGx9+A5cuX8/LLL1vtMaxQKEhMTGT58uVeXZxEGek5GToPBwRjHee/xs2ofLHRkl102CowIrMzjyWsGbd6PxezCnn2i5RSXW/2CYpoghqdwWE5osfsWkzEkU8AeEX7LGfD4rwzbwXi8TdAp9Nx4sQJu+MnTpzA4CDHS6KCkMnggQXQ4n7QFxtb9F9JtZJC3usiY2sO2/gISxEYsaUmtNNytSmSI0wmb6CIEATnuYJua4JHvjF2LgJma4fyneEO7+5BXEF4LARHjhzJqFGjWLBgATt37mTnzp3Mnz+fZ555xtwNWqISoVAa9zJu0N3YkfrLgSjzSroA+aqfoK15XNbACMCi3+ybsFY3nOVDOhM4JuEZqBb3cDkLjrglyP7ZDt8+B8DVNk/zf/oHTRdXeTwum5s3bx5RUVHMnz/f3FIrOjqaV155hZdeesnrC5TwAupAeHwdLO8L1/4m9H9DCSWRHIJ81k/Q1sz2xm0Wbj3FuLubofZSgnelpJRCxZQnGOTnQBN0JgRd3fPyYVj7BBi00OZRLsZNgxTxfb6rIh5/muRyOa+++ioXL14kKyuLrKwsLl68yKuvvmrlJ5SoZATWNvZ0C45CeS2VT9Tz8UPju8CIk5SZsuRnV3eTuLSVMSWaoPh3MM+ZOexs4hvnjK3xNbkQeyc8+n8YLPwp1eF/o0zfgNDQUCkiXJUIbwhPfIOgDiFOfoIFqo9Qybzjx7XT/Gx8hFbmsMWXqHGEZ8m12mrkd9bqDVzOLrQ6VloZX6g1vi8BItFhgAInEWKHTVXzrxtr0fMyoF4bY4260s+mbK7qi8FSdZH55ptv+Prrr0lLS0Ojsd4kev/+/V5ZmISPiGpHwcDPUa4ZxAOKvVw6+F9otajMtZ62mp6ttueon6BYhYMz9KXosFJZGfLxblL+vcE3z5VsF+vUJ+jkpRe6MIedpcmITqvJN9YDXz8NYTHwxDfGGnWbK7wpA9/+8Th5xTpmD2hXrmlUHmuCH374ISNHjiQyMpIDBw7QvXt36tSpwz///MN9993nizVKeBkh9i5e0o4FoP7fX8DO98s8p8pC87ONFIO1JmgpIMVSOpxRnSpHUv69AcC6v86bjzkTKs60riIHKTLBfkY9x1nCtN20Bh188zRc3Af+4UY3Smh9t9ZYWq7lFfPpzrOs/es81/M1ri/wIh4LwY8++oiPP/6YRYsWoVarefXVV9myZQsTJ04kO7ts++FKlA9yGfxoiGeW9knjgaQ34eBXZZpTYRH5UCpkdoql3EGKjL/Ks4+go1ZbVRnLNmHONEFn7cTMPkEbzdq0mZXz0jnrXtGxya/D35tB6W+sRa/bwslo75B6Ocf83HUTWO/isRBMS0ujRw9ju+yAgAByc3MBePLJJ/nqq7J9kSTKB5NPbrn+Pq60u7kt6fcT4NTWUs9pGQgR6x3oKP7iqTnsaCOmqoyVj83JOGf5ziV5gtYerlB/Y0cXZ4LF8v6TlP+j7qmvQSY31qA3tE+G9mjLTTexFoKly5UsLR4LwaioKDIzMwFjd2nT/r1nz56tFk7SmoCllnb1tteg3WNGE+jrp+Bi6SoWLIWgWMTZWhMsOe7voRA8eD6LYZ/u5vCFLI/XWN4YDAJjv0xhwa8nnY9zUxN0ds5cNufnQBN0IlhMHobHFUm8oLzZAOWB+dDyAZfr8FaydOrlXPNzV1uEehuPhWDv3r35/vvvAWPi9KRJk7jnnnsYPHiw6H7E7rBkyRJiY2Px9/cnLi6OvXv3Oh2/fv16WrZsib+/P+3ateOnn36yOj9ixAhkMpnVo1+/flZjMjMzGTZsGKGhoYSHhzNq1Cjy8vJKtf6qhqVAUiuV8MgSaNILtPmw+jG4fsbjOcX8gJYoHCRLeyoEn//qAH+evs7Apbs8uq4iSP7nOj8fTefD3047HWeZ9uNMj3DWTswcGLHVBAOMmqDzPEGBe+V/8ZbSWPZ6scNE6Pq0k/GO11harDRBFw0fvI3HQvDjjz/m9deNpTPjx49n+fLltGrVilmzZrF06VKPF7Bu3ToSExOZMWMG+/fvp0OHDvTt25crV66Ijt+1axdDhw5l1KhRHDhwgP79+9O/f3+OHj1qNa5fv35cvnzZ/LA11YcNG8axY8fYsmULP/74I7///jtjxozxeP1VEbuNlpRq46buUe2h4NrNtIirHs3pKunauotMyXNbc3hYnHv71pRmH47ypsjNEjh3hYo7PkHbwEjoTU3QWZ5g0JV9fKhajEIm8JWuF5c6vuh8vV6ODhfr9Jy+UqKAOEvn8QUeCUGdTsfbb79Nenq6+diQIUP48MMPef7551Gr1R4vYMGCBYwePZqRI0fSunVrli1bRmBgoMNmDB988AH9+vXjlVdeoVWrVrz11lt07tyZxYsXW43z8/MjKirK/KhVq5b5XGpqKps3b+bTTz8lLi6OO+64g0WLFrF27VouXbrk8WuoasisusjcfO4XYmzRH94Ibpw1JsgWu68Z2/oBZTZtEhyVzdkGRu5oVjV70onhroBw16R0qgk6SJYO8XehCV49ScvfRuMv07JV34k3dE+7Tpfy8u/P6St5VlH/Sq0JKpVK3nvvPXQ670hqjUZDSkoKCQkJJQuSy0lISCA5WbwsJzk52Wo8QN++fe3Gb9++nXr16tGiRQvGjh3L9evXreYIDw+na9eu5mMJCQnI5XL27Nkjet/i4mJycnKsHlUVh11kQiKNzTED68Dlg0YfoV7r1py2fkDbL7ajfoK25nB16rLlrqxwN+DtSAbqDQIanXESWyEYGmDyCYp8Z3MuGWvJNdnsNzRjgnYiely7J7ytg1v6A6EK+AT79OnDjh2ON8TxhGvXrqHX64mMjLQ6HhkZaaVtWpKenu5yfL9+/fj8889JSkri3XffZceOHdx3333o9XrzHPXq1bOaQ6lUUrt2bYf3nT17tlVz2ZiYGI9fb2VBJpPRpG4QEcFqwgNt9oOIaAaPrwdVIJxJgu+fd0ulcVUKp3DQT9DeJ1h9pKC7gUJ3d9JzNM7S7LaNDoeYo8M22lVhFnz5H8g+T2FIY0ZpXqYIP7fWYRUY8YI9bOkPhPKPDntcMXLfffcxZcoUjhw5QpcuXQgKsi57evjhit9seciQIebn7dq1o3379jRt2pTt27fTp0+fUs05depUq+1Gc3JyqrQg/PH5O9AZBPyUIr/8DbrAoFXw1RA49BWEREHCTKfz2Wb4OzWHLY7b+gS9tO9TlcJdQeLIHLZsv2X7foaKmcO6Ylj3BFw5BsGRHOuzkhtrL3qwXovnbl/lGJMQDPZTklesK3dN0GMhOG7cOEB8f2GZTGbWttwhIiIChUJBRkaG1fGMjAyioqJEr4mKivJoPECTJk2IiIjg9OnT9OnTh6ioKLvAi06nIzMz0+E8fn5++Pm590tZFXDUcsnMrffCwx/CxvHGipKQaIh71q25xeSYVdmck+hwREj1eY/dNofdDYw4EoI3fWj+Krnd1qYmc9hcNmcwwLfPwrk/QG30Axfl1Qc8EIKWz8soBQVB4PhNIdipYTh/nLpW+fMEDQaDw4cnAhBArVbTpUsXkpKSrOZPSkoiPj5e9Jr4+Hir8QBbtmxxOB7gwoULXL9+nejoaPMcWVlZpKSU5MT99ttvGAwG4uKqfqdcr9HpiZJNmn6eDMe+dftSu4oRi0+a5Sk/m7ZYbepXn4Yc7goIdzvHOBKWVjvN2bzvJYERvXFBv0w1/j/KVTDkS4hu7/mWm17MkUnPKSKrQItCLqNjTDhQBSpGvE1iYiKffPIJq1atIjU1lbFjx5Kfn29u0PrUU08xdepU8/gXXniBzZs3M3/+fE6cOMHMmTPZt28fEyZMACAvL49XXnmF3bt3c+7cOZKSknjkkUdo1qwZffv2BaBVq1b069eP0aNHs3fvXv78808mTJjAkCFDqF+/vv0iazJ3vgzdngEE2DAGzu106zLb74mjztJ+NtFhlYt8Q0s+/t3zfMbKiLuaoCNhWWhRLWL741OSIqODPz+APcsAyLl/MSmK9kApdpuzel42gWgyhZvWDaJWoDG7xNXueN7GbXO4sLCQpKQkHnzQ2FF26tSpFBcXm88rFAreeust/P39PVrA4MGDuXr1KtOnTyc9PZ2OHTuyefNmc/AjLS0NucUXo0ePHqxZs4Y33niD1157jebNm/Pdd9/Rtm1b8zoOHz7MqlWryMrKon79+tx777289dZbVubs6tWrmTBhAn369EEulzNw4EA+/PBDj9ZeI5DJ4L73jJs2nfgRvnocnv4ZItt4NI2jPUZsO1vLPXAK/venE4y5q6lH6yhfrAMIjjqjuB1AceETFDOHTZrgvbrtsPUj48F732H8ocb8cSqZn1+403Mx5sWyOVNkuFV0qLkDjqvd8byN20Jw1apVbNq0ySwEFy9eTJs2bQgICACMe4zUr1+fSZMmebyICRMmmDU5W7Zv3253bNCgQQwaNEh0fEBAAL/88ovLe9auXZs1a9Z4tM4ai1wBAz+Fz/vD+d3GqOKoXyHccWDI3hwWzxP087BipCphW2PrKP3H3UaxjqLDlonStrcIDVByp/wwcxT/ZzwQPwF6TCBjjzHD4+KNQo+7i1uXzdlz/FIOJzNy6N/xFpctsUz+wFbRoWY/daU1h1evXm1XUbFmzRq2bdvGtm3bmDt3Ll9//bXXFyhRSVAFwNCvoG5LyDXml1GQKTpU7HPvqGLE1idYWq7kFLldoVFeWAqI0tYEW83nKEVGU+ITtH3vw24cY6lqISqZnuKWA+Cet4CSlmRFOr3HmqCr5d7/4R9MWneIP05dczlXqoUQrChN0O1P4OnTp2nXrp35b39/fysztXv37hw/fty7q5OoXJha9IfUh2sn4auhoC10fR2Oa4e9sV/IhRsFdP9vErfP+a3Mc/kKg+A4xaXM0WGzOayw+oFpKMtAvW4wwbIidurbcKXP++YIlWmuQo3e4zwXd6PDJ9KdFxQUavScu5YPQKvokBJNsLImS2dlZVn5AK9evUpsbKz5b4PBYHVeopoS1sAoCP3CjKbx/54Bg+tfbqsMGZl3NUGTxlHezThdYSkgfjpymXYzf2HbCfuaePd9guLHLaPDpre2DtmsUs1Bln+Vk8TynHYSebqS99rUkqxIZyhjdLj0TsGTGbkYBIgIVlMvxL+kAWxlTZFp0KCBXZMCSw4fPkyDBg28siiJSk5kaxi6BhRqY7Dkp5dd2kiOAiOiydplYO4v9ntiVxSWwuXFdQfJ1+gZufIvu3HuaoKOhKUpOhygNgrBQIr4TD2XxvIMCG/IlIDp5BFolYRs0gSLtfqyRYedXGubMG+LpSkMJSV/lVYTvP/++5k+fTpFRUV25woLC3nzzTd54AHx/mMS1ZDYO2DAJ4AM9i2H3+dZnbb1TTnaY8RbPkETS7ZVnrQZb+cJOgyMWPoE9TqWqD6go/wfMoVgeGIDRf7GElHLJGTTXIWaUghBNy9wVQduKwSDLLYCKM/epG5/Al977TUyMzNp0aIFc+fOZePGjWzcuJH33nuPFi1acOPGDV577TVfrlWistGmP9z3rvH5trcZpNjucKjD6LAXhGBlrbQrt4oRszksR7npBXopDlEoqBmleQUimhN8M+BgGXXVeykwUhZZVSIEQ4ASTVBvECjWld82Cm6nyERGRrJr1y7Gjh3LlClTzJJaJpNxzz338NFHH9k1NpCoAcQ9C7mXYef7zFZ+yjUhjCPYV91YaoKWX2ZLc9h2r+KqxpmreTSoFeCxie+sRZbVOBcpMvemf4L80lfoBDnjtRM5IDQHsAg4lGiCupsOxiKtwWOtyxvJ0oIgcMIiR9BynWDUBj1tuFtaPKodbty4MZs3byYzM5PTp43dcps1a0bt2rV9sjiJKkKfGZCbjvLQVyxRfchYYSbYCEJLn6DGwsNvGR32JFHaEluz6+cjl7mvXXSp5iotvx5LZ8wXKXRpVIvJ/VrSpG6Q+40R3G6gIH68SKvnCcUW4i+tBOB13Sh+M3Q2nw9yogkWar2fImPCWY7ghRuF5BbrUCvkNK0bDBg/I/4qOUVaA/nFOmoHed6ftDSUyhapXbs23bt3p3v37pIAlDBKoYcXsUPfnkBZMR/o/0s9zXmrIZYCTutACJZGExQTNGNXl//e11/tTQOM22g+9n/J9Jnvfrs5t81hB9Kn+fVtzFKuBEC4eyrr9L2szgeJpJ6Y8wRdBEZ0egPPrPqLJdtKtghw1lnaUst39r9pSpJuVi/Yqhel2Fp9TYXXDktUExQqxmpf5JChCeHkMuHiZOqSVXLaQivQWPh7LDVE05jPn+7u9m0dCZCLWe7lL3oLW60nu9C9ZrRQxlZa/+7iyUtvIZcJ/B0zCFnPyXZDgkRST0qiwwaceS9/OZbB1tQrzP2lZLMoZ620LLdEdRYYOX7JOihiItCstZZfmowkBCW8RgH+PK15hfNEEaFLZ4X6PYIpAKy7yDjaH8SkLcY1cd+6cGRKbj8pvkeNrxD7vrvfHaaU0eErqfDVEFSCll/1XTjSYZqV5DE9NZnDpnZagiCYNcFCF5pgoUgVjnVTVetzlluiOtMEbYMiJkyaYHn2FJSEoIRXuU4YExTTyFWE01Z+jqWqhaDTWGmCxQ7K20zmsKv8MkscCZDzmeWtCdofc7ZPsCXud5Gx+CP7grF0sSibE6rWPK99ngA/ax+a6T0PtBEslvMUlcInaEmRVm8OsoCNEHSiCqberCZpbdM6zZwrKGmCElWZS/IoPqo/h3zBjzsVR2HjOBSyki9HkYP0B5Mm6Mk+IwaDuNBctuOMldldEbgf8PBwXOENYxOLnIsQcSszg6dRjNpupzmzJmgjWCxNVleaoBiW4zcduUyfBSX+T63F3I7M/NwirflHqrWNOVySKyhpghJVnDT/FozTvohWUMCR9QTsmGU+5yjfrUQTdB9ngqawXAvx7Vdd1tQXW/QGAbRFxnZmV1ON3b6f+B9XdYGAfWt9kyZmEiymYIPl+1+kLUXZnM34f68XmJ9baoKOdkU9kW5MjYkO8yc80Fp7LakakTRBiSqIqTdghwbhAOwwdGCydjQAfnuX8LTiZ6fXK8yaYNnN4cqAuxsouWsOY9DBhmcgbRf4hRpruMMbUqQ1al+O9muxDYxYbm9ZqrI5J+MtI/96Bzk9tpUilpSsVdIEJaogP71wB6PvbMy7A0u6DW0w3GXMIwSmq77gIfkuh9crSqMJOrF4Ne465byAmNwua8doawSeLfwYUn8w1mwPWWNubOto43W5jSZoCozoLVS0Qq3e4x8SZ6/LUsDqHAx0FBQByxQZSROUqII0qxfC6w+0pk6wn7Uku2MS+q5GjXC+ainx8mOi15sc+R75BAXBodTs9s5W0ixMNV8iGh12Uwq6I4PGKb7nEc1Pxjs9+n/Q+E7zOcvaYUvMQlBt6tNnFII6K3PYc2HjLKXHMkiid2APH7epFLHElCIjaYISVZ7+HW8BoEndIKNU6zeHTfruqGV6PlYtoJXsX7trSmsOO4o2A6zYddbDlZcOcU3QOz7B/yh28KpqnfGPfnOg7QDzOUEQrPoJiq3JtmzO8n7Gsjm3lllyT7FjNyexTH8S0wT1BoGT6U7MYUkTlKguJLSqx6aJd/DDhDsAkCsUJGrHsdvQihBZISvV79JAdtXqGkUpKkb0gsC0jeKaZXkiFqF2u22+k3F3yw8yR/kJAGtUj8Jtz1mdt2w04MgcNvXpM5XNWQqn0pjDYlLQNKVl5FnsdZ27nk+R1oC/Sk5snSC784E2Wmt5IAlBCZ8gk8loUz/M7I+SyWQUo2aMJpEThhgiZVmsUs2xatEvF0n0dUVWgfuVGeWN+620xI+3l53hI9UHKGUGNujvYKniCbsxlq3oHQVGAi3a1hsMgp2Z6mkqkVg02STwXGmCJn9gi6hQ0R89cyRbyhOUqK7kEMQIzatcEmrTVH4Z1jyGP8aO5J5u+ANwOdu+v2VFICa03/kp1a1rxXxssbLLLFfPJVBWzO/6dkzWjkEv2N/EZAqrlXI7oWKrCQIUaPVW2ho439NDbG1iwt2kTepcRIdNQrC1SFAEJE1QooaQTh2e0kwhSwiCC3+xSLUIBXqH3aedkZnvfEuHSpxBY8Z2iRFk87lqDhGyHI4YYhmrfREtStGUG0dBESjxrfop5WatsKBYZ2emipXGmRALnIhprmYhaHFSrDwy1UlQBCx8glJgRKK6c1powDOal0Dpzz2K/bylXI7l19jd4IjOUUZuOeNJRNsWS6EURCEr1O/SUH6Vfw31GKmZTD7GbW3FhE+R1rEQNAk+mUxmkTCttzNTnUWIcy2EkUkrdG4OO/cJOmqcYMLcQKGmBUaWLFlCbGws/v7+xMXFsXfvXqfj169fT8uWLfH396ddu3b89NNP5nNarZbJkyfTrl07goKCqF+/Pk899RSXLl2ymiM2NhaZTGb1mDNnjk9en4Q4+4SWMPBT9IKMx5XbGFxQsg+0uzKlKmh6rjBpUSp0LFUtpJ38HNeFEIZrJ3ONsJJxIkLFUY4gWPtYLTUsW+HkzBy21MhMl4mbw8Z/dU58gjfyNaTnGN0XLaPEzWFzA4WapAmuW7eOxMREZsyYwf79++nQoQN9+/blyhXxLiC7du1i6NChjBo1igMHDtC/f3/69+9v3gSqoKCA/fv3M23aNPbv38+GDRs4efIkDz/8sN1cs2bN4vLly+bH888/79PXWtP5dlwP/u/JLtYHWz3EDN0IAAblfQn7VgDua1aVpWLEk7QeW4ypjgbeVX3MXYojFAh+PK15hXOCdWNYZ+awWBdmSxdhoEVjVVvh5MwczhdpvyX2jpsEtHV02NonaPIHxtQOIMRfJXq/oArQBD3qLO0LFixYwOjRoxk5ciQAy5YtY9OmTSxfvpwpU6bYjf/ggw/o168fr7zyCgBvvfUWW7ZsYfHixSxbtoywsDC2bNlidc3ixYvp3r07aWlpNGzY0Hw8JCSEqKgoH746CUs6NawFQPfGtdl7NpOHOtQH4Ev9PdST3WCi8jvYlAjBkTdTTlwLOHdL03xNWTYG0BsEJivXMUCxE50gZ5z2BQ4JzUTH2WK5v4jdmiwEc7BF/bBtAMpZjXWelSboWBXUu5EnaGqk2ipK3BQG+4435UGFaoIajYaUlBQSEhLMx+RyOQkJCSQnJ4tek5ycbDUeoG/fvg7HA2RnZyOTyQgPD7c6PmfOHOrUqUOnTp2YO3cuOp3jN764uJicnByrh0Tp+PjJLsz9T3tmDygpr1ugG8TvQf1AMMA3T9NR9rdbc7ldd1uJGSr8xHPKHwCYohvNdkNH0XFi8r7IiTlsqZxatqiy9aM6FYJFlj7Bm/+KjCsJjDj2CboKikBJioxWL5RbF6AKFYLXrl1Dr9fbbdAUGRlJenq66DXp6ekejS8qKmLy5MkMHTqU0NCSN3/ixImsXbuWbdu28eyzz/Lf//6XV1991eFaZ8+eTVhYmPkRExPj7suUsCE8UM2grjFWqRsg48uIF6H5vaAr5P8U79FUdtHlXK5K067lOY8ee4vSmsMPyHfzhuJzAN7TPsY3+p4Ox4qZ/gVOosNyEU2wQONZdNiyzb3p/mLv+e5/MknPLnKqCZrTY+o70wRLXkd5aYMV7hP0JVqtlsceewxBEFi6dKnVucTERO6++27at2/Pc889x/z581m0aBHFxeJfmqlTp5KdnW1+nD9/XnScRBlQqGDQSrilC7VkeaxSv0s9bjitJHFVlZHy7w0vL9JIdoGW5TvPciW39HmKt8mPs0D1EXKZwOe6e/hI/4jHc5RsvG7v2bJ830xmZl6xfXTYU3NY7B2f+NUBbpudZN1Ky+K5Vm/g9JU8wL6HoCUqhdy870x5+QUrVAhGRESgUCjIyMiwOp6RkeHQVxcVFeXWeJMA/Pfff9myZYuVFihGXFwcOp2Oc+fOiZ738/MjNDTU6iHhXdrUDwN1EDz+NeeEaBrIrrFK/S5hMsddosvbGk759wbfHrjApK8PMuvH44xY/hdQCp9g+lE+Vs3HT6bjZ303ZuqGl2YWFz7BkudBFo0JPNEELc1hZ9FhE5bBEEthe+ZqHhq9gRA/JQ1qBTieAIuGD+UUIa5QIahWq+nSpQtJSUnmYwaDgaSkJOLj40WviY+PtxoPsGXLFqvxJgF46tQptm7dSp06dVyu5eDBg8jlcurVq1fKVyNRWn58/g4mJdzKsz2bGA8ERTBaeI2rQhit5Gl8pJyPGvHyOFebFJlOH7mQzQdbT1GsK5t2MXDpLiatO8RvJ4zZCyZnv0dknYfV/yFUVsgeQ0te1I7HUMqvorM8QUuRako9ydPYR4dvFGgczp8vmifoGI2lJmghEE2mcMvoEJeuA7F9kn1JhUeHExMTGT58OF27dqV79+4sXLiQ/Px8c7T4qaee4pZbbmH27NkAvPDCC/Ts2ZP58+fzwAMPsHbtWvbt28fHH38MGAXgf/7zH/bv38+PP/6IXq83+wtr166NWq0mOTmZPXv20KtXL0JCQkhOTmbSpEk88cQT1KpVq2LeiBpM21vCaHtLmNWxS9RjhGYy69SzuE12jPmqpUzUTkDwUFiYvm8PLd4JGEvzxveyj7yWBUEQcLf3Qxh5xr1Bci9z0tCA0ZpEiin9/rrmFBkXeYKBFo1VbTXBCzcca9q5onmCjsXgjfwSgWopbN0JipgIKud2WhUuBAcPHszVq1eZPn066enpdOzYkc2bN5uDH2lpacgttirr0aMHa9as4Y033uC1116jefPmfPfdd7Rt2xaAixcv8v333wPQsWNHq3tt27aNu+++Gz8/P9auXcvMmTMpLi6mcePGTJo0icTExPJ50RJucUyI5TntJFao3+MhxW6uCLV4S/cEljqOpxkyqaXR3FyQVaB1KzDih4bP1PPg2t8QUp8RVyeTQ7Db9xF7rYVOK0YsAyOm/Dt7c9gRBoNgkywtOFyHCUsfqaV/0Fk3aVtqnCYIMGHCBCZMmCB6bvv27XbHBg0axKBBg0THx8bGujSROnfuzO7duz1ep0T5Yco722loxzTGMYdFjFL+TLpQi0/0D5rHudofIzrM3+yQh7IlNTtC7oYaqEDPItUiusr/Bv8weHIDlxf8U+Z7OxWCIoERY7K0e6kn+RqdVQqSwUnZnImMnJLAot5QOiEYZJHYXR5UCiEoIWGLad8MgP9pexDCdV5XreF11RquCOFsNBj7FLrSBDvG1CLBYjc0X5B85hrfHnCWziMwS7mSexUpFAsq/IauRVunBVB2Ieg0T9DiuUmw7Pj7qlUPQmfkFlkLISe50mau5JYIQZOwvZJbxLU8DXIZtIgUL5ezpEQTrAGBEQkJd5Ah4xP9A3ymuw+Auar/4w75EcB1dHj5n9adpb2vB8JzX+53ev55xbcMUyZhEGRM1I6HRj2cRmSdkZFTxGc7z5JTZAwUFTgrm7P4dpsCI0VaA9tPXrUbK4bpHiYuZRl9h87e86sW5rBJEzT5A2MjgkSFtS0l0eEakCIjIeE+Mt7WDeMH/W2oZXqWqd6njeys68tsZ/GFFHTCYMU2XlJ9A8AM3XB+MXQHSre3B8D7W/7mrR+P882+C0BJYCTQVQMFP8+NPltN8JlV+zh+KcepJngtzz4w4qpzjC2BfpImKCEhioCcl7Rj2aVvTbCsiJXq9wguuODRHGWRga58zbb0lu/nHeVnACzWPcIX+nvN54q1pSsJO3g+CyhJa3GaIlNKIZjy7w1m/XCcMzd9qTG1A2h3SxjX8zUM+TiZwxey3JqnRBM0NVJ1TwiWbAwlaYISNRjT3iSAleTSoOJZbSKphobUlWXz4OEJ1Mb9iK+8DKqgu1FVgE6yUyxRfYhSZmC97i7m6R6zOl8ac1irN3DqpmAyCT/n0eGS50FumKEmFmz5m+V/nuX174ydmWJqBbJ6dBxdGtUip0jHz0fFS1Rt0dkIQbEtNsUIstkTxUROkZZZPxxn79lMj3+QnCEJQYlKSbsGYQ7P5RLIcM1kLggRhBemsVw9lwDcLF8rgyrobkChiewSn6nnEiDTsE3fgam6Z+xuXBpzWGcQzIK4QKPnYlZhyU5zLsxhMV+c383yNFsBaWpcYLpXqL+KUH8Vnz/dnfgmrgsPTOgNAkVaPf9cywfcN4fNPQVtNMHfUq+w/M+zTN1w2KtRfkkISlR6xD7uV6jFcM1kCpWhdJSfMWpd+NaHtHjbaZdj6nKDVap3qS3L45ChCeO1L6ATScIoKqU5bGL1njRun/Mb5zONwQpXmmDdED/qBFknZTeOMO72Ft+0Dgse62B3fWSoHwCN6gQCRg1txchu9Ly1rltr1BkMnMrIQ28QCA9UERXq79Z1gQ5SZH4+ehmA+9pG211TFiQhKFFlOSPcwrD8RIoEFb0VB3lHuRxX8WKxrTHd5fuDl5yeD6aAVer3iJFf5awhkqc1r1CA+Be/tIERR7jyCfopFfz20t1mwQfQoUE4v7x4F+8P7siAzg2Yel9Lq+tnD2jH2jG38WLCreZj/ioFnw7vylejb2Pk7bFO16TXCyWmcFSo29qbmCZYoNGx429jVLtfW+/2AJWEoESVZr9wKxO0E9ELMgYrt5OoXO90fFmsqItZjsvLVOhYpnqf1vJ/uSqE8pR2CtdxbNKXNkXGEeLt9a3/DgtUERpQ0tFZJoMWUSHmLs+xEdb7AAeolNzWpI7d3CqFnPimdfBTOvczag2CubbaWfssW8y9Dy2iw9tPXqVIayCmdgBtPJjLHSQhKFHpceUC32rowuu6UQBMVH7HE4otDsfK8HyfXVfIMDBPtYw7FMfIE/wZqXmV80Kk02u8rgm68Ama+MeiemZrqvUWFraboauVzn8xXFXr6A2CR5UiJoIs6pxNmIIx97WN9nrVjyQEJSo97lQZrNX3ZqFuAACzlCvpK/9LdNz6lAu0mPYzp6/kem19U5Vf8YhiF1pBwVjtixwVmri8prQpMo5wVTtswrIhgm3D2Ya1A63+VilciAcXv05avcHjyDDYa4JFWj2/pRrb53nbFAZJCEpUYjZNvIOPhnWmc8Nwt8Yv1A1kja4XcpnAh6rFdJWdEB0nCDDvF/fa97tilGITY5SbAHhVO4Y/DO3dus7b5rCYwHKlMPnb9CAMUCusgheuhKArDT23SEdOkQ6lXEazeu43ijBrgjd9gjtPXSNfoycq1J+ODcLdnsddJCEoUWlpUz+M+9t5Yv7ImKZ7mi36zvjJtHymnkczmXgy9eZj6W7l/Y39MoXHP9ktmpf2sHwX01SrAZitHcq3hjvdXGfpzeEmdYNcD7qJq5xItYiQM0WCwQ0h6GauXrN6wS79h5aYNEFTV+vNx4ymcL+2UW41q/AUSQhKVCmeim/k9LweBc9rnyfF0JwwWQGr1O8SxXXRsYcuZHEi3XGitVZv4Oej6ew6c51/rxdYneshP8o8lXHLhhW6vvyfRWcbV2w6fLnUKTJdGrrf79KVvDC1sbfE0i8oJiRLgyf+QCiJDmt0Boq0erYc950pDJIQlKgCWGo07phVRfgxSvMyZwzR3CK7zkr1e4SSbzduwEe76LfwD6tGoJZo9SWCynK/jtayc/yf6n3UMj0/6uOYpXsST7Kwx6/ZX+oNoLo0cl8IutKgxbSzRhEWmqCrwIibRRue+AOhJE8QICn1CtmFWiKC1XSLre3RPO4iCUGJSo/ld9ndL14WIQzXTuGKEE5L+Xk+Uc/HD3Fhd/5Ggehxra7kZne+tw2ABrIrrFS/R4iskN2GVrykHetxt2uANXvTPL5GrZQ7raSxpTSaYKPaJZpgWX2CJjzVBNUKOcqbi//ffqM7457WUU433CoLkhCUqPRYfvQ9qRm9INRluGYyOUIAcfITxp3dsDdDLTU+SzQ2x2uRwyrVu9STZZFqiGFMGVrje1KHDEZN9P3HOhLqr3I9+CaufIJKEaFyi8UmSK59gu6tw1MhKJPJzMERU4L0fT4yhUESghJVAJWFxuJp2Xyq0IhntYloBAUPKPYyXfm53SwanfislsLRn2I+U8+jqfwyF4QIRmgmk4P7QYqyMrJHLA+0jzbX+7qDKyEoplnVCiwRsq58gq7MZTCW60UE+7kcZ4upnllvEAgLUBHf1P2aZU+RhKBEpWfMnU2IrRPIiwnN8VCBAiDZ0IaXtGMBGKH8lbGKH6zOv/nDMTIt/IKCILBsxxlzgq4CPYtVH9JZfposIYjhmslk4Bv/FECflvY7Hpqion4i+YCOcBVUVyrsB1hunq4SOW/Js3c1tfr7jQda2Y3xVAs0EWjR+iuhVaTrnMUyIAlBiUpPrSA121/pxYsJt5a6hdIPhh68pX0CgMmqtQyQ/24+dyI9l9e/PWL++69zN5jz8wne+vE4IPCO8jMSFAcoElTGgItwS5lejyv8nOwhbJvbB459fy41QZHzlnsCu/LB1Q5SWwnsxhH2mrGnQRETlp1tfGkKg7THiEQN4jP9/dST3eBZ5SbeVX3CdW0YOwzG7ik/HzXmDSrkMq5a7JMxSfk/hii3oxdkTNROIEVo4fN1ikVtTQJNzER1JOxcaYJiQs5fpeDQjHtRKWRu5WfOeKgNp6/m8cwdjakdZO8fdbeRqi2mfUaC1AruaB5RqjncRdIEJaoUZe2lOUc3lG/1t6OS6flItZB2spLNjn4/ZXTCf3Uzcvu4IokXlBsAmKZ7ml8N3dy6R4vIEI6+2dfqWIgHnZ3F/H4meSWTyZh6X0vG3NXE4py4sHKlyTk6HxagMgshVzSsE8iOV3rxZHysV4WgaWOoXi3rie6f4k0kIShRpXBVtO/6ejmvap/lD31bgmTFrFC/RyOZ0feXU2jcWGjn6WvcI9/HW8rlAHyge5Q1+j5u36NFVAjBfkp+e6mn+Zi/WsGIHrFuXS+WumIp6J7t2ZTX7rfwvzmQda70uLJ02RYjPLBECD5zR2Mm9GrmUbmcJfFNI/BTyhnu5ntWFiqFEFyyZAmxsbH4+/sTFxfH3r17nY5fv349LVu2xN/fn3bt2vHTTz9ZnRcEgenTpxMdHU1AQAAJCQmcOnXKakxmZibDhg0jNDSU8PBwRo0aRV5eHhKVG290VdeiZKz2RY4ZGhEhM6a91CHbrBl1kZ1kkWoRCpnAWt3dvK/7j0fzR4UZ62+b1C0RAEq5jFf6thANHtgipgk6M00dKXy1RDSzvm1Kuts8HtfQ5Vo8IdRfSY+mdegQE87U+1vxct8Wpe74MuqOxhyf1c9nCdKWVLgQXLduHYmJicyYMYP9+/fToUMH+vbty5UrV0TH79q1i6FDhzJq1CgOHDhA//796d+/P0ePHjWPee+99/jwww9ZtmwZe/bsISgoiL59+1JUVNKCfdiwYRw7dowtW7bw448/8vvvvzNmzBifv16JsuFKBk6xaQxqwjZKmUcgIzSTSTPUJVaewXL1XGTafIovH+cz9Tz8ZVq26jvdbNHl3hd55kOt6RgTzri7m9qdU8iNuW/P3NnEZZpLgIgpKiboBnQ2Bmie793c6vgHQzrS89a6vNjnVrtrFj/emVf7teDjJ7vwcIf6TtfhKTKZjNXPxPHduB5eSWz2VXK0LTLBmzuWlIK4uDi6devG4sWLATAYDMTExPD8888zZcoUu/GDBw8mPz+fH3/80Xzstttuo2PHjixbtgxBEKhfvz4vvfQSL7/8MgDZ2dlERkaycuVKhgwZQmpqKq1bt+avv/6ia9euAGzevJn777+fCxcuUL++6w9HTk4OYWFhZGdnExrq3SaPEo75am8aUzcccXj+jQda8famVPPfjeoE8suLd1GsNdBh1q924xvLLvONeiZ1ZLmkGmKoLcslUpbFAUMzHte8RqGDztBinJvzgN2x2CnGDjPdYmux/rkeAMzYeJRVyf9ajfv86e5M3XCEyFA/VozozoClf/Jg+/p8kGS0YBLvuZWJfayFnVZv4MTlXNrUD/VJY4GqjCffzwrVBDUaDSkpKSQkJJiPyeVyEhISSE5OFr0mOTnZajxA3759zePPnj1Lenq61ZiwsDDi4uLMY5KTkwkPDzcLQICEhATkcjl79uwRvW9xcTE5OTlWD4nyZ2DnBjzYvmSPiXtbWzcvvc1iI6A+Levxy4t34a9SEBYoXmlxVohmlOYVCgQ/WsnPEynL4owhmqc1L3skAC3XZMkrfY3R5KHdS0zPqfe34pk7GvPOo22Z/mBrRt3RmDubR7D9lbtZ/1wPwgJVJL10N5PuKdHkxGqGVQpjGZ0kAMtGhabIXLt2Db1eT2Sk9Qc5MjKSEyfEe8Glp6eLjk9PTzefNx1zNqZePeuEVKVSSe3atc1jbJk9ezZvvvmmm69MwleolXIWP96Z9wcbuJ6nISrMH71BYP2+83RpVIvmkSEseKwDkaH+3N7MOrXiyMx7aTfTXhs8KDRjvHYiH6sWkEkICyJn88uT9xGgVrB6TxoPtItGIZfx/aFLDOh8C18k/8ui304z5b6W3MjXUDtIzVPxsaLrHd+rGU/ENbISwv4qBW882NpurFhy8o5X7ub0lTy71yLhPaQ8QTeZOnUqiYmJ5r9zcnKIiYmpwBXVbFQKuTkAoZDLGGKhaQ3o3ED0mhB/FWdn30++Ro+fUm4WYAJwLa83qRcHUKdOBIvq1jVrV8/1LPHvmZ6/dG8LXujTHKWbVQyOtFB3aFQniEZ1yq88ryZSoUIwIiIChUJBRkaG1fGMjAyiosSzxKOiopyON/2bkZFBdHS01ZiOHTuax9gGXnQ6HZmZmQ7v6+fnh5+f5zWQEpULmUxG8M2cvXoWXZSjwwKIDrPXzhzhrgCUqPxU6P+kWq2mS5cuJCUlmY8ZDAaSkpKIj48XvSY+Pt5qPMCWLVvM4xs3bkxUVJTVmJycHPbs2WMeEx8fT1ZWFikpKeYxv/32GwaDgbi4OK+9PgkJiSqAUMGsXbtW8PPzE1auXCkcP35cGDNmjBAeHi6kp6cLgiAITz75pDBlyhTz+D///FNQKpXCvHnzhNTUVGHGjBmCSqUSjhw5Yh4zZ84cITw8XNi4caNw+PBh4ZFHHhEaN24sFBYWmsf069dP6NSpk7Bnzx5h586dQvPmzYWhQ4e6ve7s7GwBELKzs73wLkhISHgTT76fFS4EBUEQFi1aJDRs2FBQq9VC9+7dhd27d5vP9ezZUxg+fLjV+K+//lq49dZbBbVaLbRp00bYtGmT1XmDwSBMmzZNiIyMFPz8/IQ+ffoIJ0+etBpz/fp1YejQoUJwcLAQGhoqjBw5UsjNzXV7zZIQlJCovHjy/azwPMGqipQnKCFReakyeYISEhISFY0kBCUkJGo0khCUkJCo0UjJ0qXE5EqVyuckJCofpu+lOyEPSQiWktzcXACpakRCohKTm5tLWJjzbUql6HApMRgMXLp0iZCQkFL3TCsLprK98+fPS9FpLyG9p96not5TQRDIzc2lfv36yOXOvX6SJlhK5HI5DRqI16iWJ6GhodIX1stI76n3qYj31JUGaEIKjEhISNRoJCEoISFRo5GEYBXFz8+PGTNmSJ1tvIj0nnqfqvCeSoERCQmJGo2kCUpISNRoJCEoISFRo5GEoISERI1GEoISEhI1GkkIVlIyMzMZNmwYoaGhhIeHM2rUKPLy8pxeU1RUxPjx46lTpw7BwcEMHDjQbj8WmUxm91i7dq0vX0qFsWTJEmJjY/H39ycuLo69e/c6Hb9+/XpatmyJv78/7dq146effrI6LwgC06dPJzo6moCAABISEjh16pQvX0Klw9vv6YgRI+w+j/369fPlS7DHZ61dJcpEv379hA4dOgi7d+8W/vjjD6FZs2Yu2/8/99xzQkxMjJCUlCTs27dPuO2224QePXpYjQGEFStWCJcvXzY/LLcdqC6sXbtWUKvVwvLly4Vjx44Jo0ePFsLDw4WMjAzR8X/++aegUCiE9957Tzh+/LjwxhtviG7bEBYWJnz33XfCoUOHhIcffthu24bqjC/e0+HDhwv9+vWz+jxmZmaW10sSBKGStNeXsOb48eMCIPz111/mYz///LMgk8mEixcvil6TlZUlqFQqYf369eZjqampAiAkJyebjwHCt99+67O1Vxa6d+8ujB8/3vy3Xq8X6tevL8yePVt0/GOPPSY88MADVsfi4uKEZ599VhAE45YNUVFRwty5c83ns7KyBD8/P+Grr77ywSuofHj7PRUEoxB85JFHfLJed5HM4UpIcnIy4eHhdO3a1XwsISEBuVzOnj17RK9JSUlBq9WSkJBgPtayZUsaNmxIcnKy1djx48cTERFB9+7dWb58uVvthqoSGo2GlJQUq/dCLpeTkJBg916YSE5OthoP0LdvX/P4s2fPkp6ebjUmLCyMuLg4h3NWJ3zxnprYvn079erVo0WLFowdO5br1697/wU4QWqgUAlJT0+nXr16VseUSiW1a9cmPT3d4TVqtZrw8HCr45GRkVbXzJo1i969exMYGMivv/7KuHHjyMvLY+LEiV5/HRXFtWvX0Ov1REZGWh2PjIzkxIkTotekp6eLjje9d6Z/nY2pzvjiPQXo168fAwYMoHHjxpw5c4bXXnuN++67j+TkZBQKhfdfiAiSECxHpkyZwrvvvut0TGpqqk/XMG3aNPPzTp06kZ+fz9y5c6uVEJSoOgwZMsT8vF27drRv356mTZuyfft2+vTpUy5rkMzhcuSll14iNTXV6aNJkyZERUVx5coVq2t1Oh2ZmZlERUWJzh0VFYVGoyErK8vqeEZGhsNrAOLi4rhw4QLFxcVlfn2VhYiICBQKhV1k3Nl7ERUV5XS86V9P5qxO+OI9FaNJkyZERERw+vTpsi/aTSQhWI7UrVuXli1bOn2o1Wri4+PJysoiJSXFfO1vv/2GwWAgLi5OdO4uXbqgUqlISkoyHzt58iRpaWnEx8c7XNPBgwepVatWpS5w9xS1Wk2XLl2s3guDwUBSUpLD9yI+Pt5qPMCWLVvM4xs3bkxUVJTVmJycHPbs2eP0/a0u+OI9FePChQtcv36d6Oho7yzcHSo0LCPhkH79+gmdOnUS9uzZI+zcuVNo3ry5VYrMhQsXhBYtWgh79uwxH3vuueeEhg0bCr/99puwb98+IT4+XoiPjzef//7774VPPvlEOHLkiHDq1Cnho48+EgIDA4Xp06eX62srD9auXSv4+fkJK1euFI4fPy6MGTNGCA8PF9LT0wVBEIQnn3xSmDJlinn8n3/+KSiVSmHevHlCamqqMGPGDNEUmfDwcGHjxo3C4cOHhUceeaTGpch48z3Nzc0VXn75ZSE5OVk4e/assHXrVqFz585C8+bNhaKionJ7XZIQrKRcv35dGDp0qBAcHCyEhoYKI0eOFHJzc83nz549KwDCtm3bzMcKCwuFcePGCbVq1RICAwOFRx99VLh8+bL5/M8//yx07NhRCA4OFoKCgoQOHToIy5YtE/R6fXm+tHJj0aJFQsOGDQW1Wi10795d2L17t/lcz549heHDh1uN//rrr4Vbb71VUKvVQps2bYRNmzZZnTcYDMK0adOEyMhIwc/PT+jTp49w8uTJ8ngplQZvvqcFBQXCvffeK9StW1dQqVRCo0aNhNGjR5uFankhtdKSkJCo0Ug+QQkJiRqNJAQlJCRqNJIQlJCQqNFIQlBCQqJGIwlBCQmJGo0kBCUkJGo0khCUkJCo0UhCUEJCokYjCUGJaoNYq/YKadcuUaWQWmlJVCv69evHihUrrI45ag6h1WpRqVRWxzQaDWq12uP7lvY6iYpH0gQlqhV+fn5ERUVZPWrVqgUYN5launQpDz/8MEFBQbzzzjvMnDmTjh078umnn9K4cWP8/f0BSEtL45FHHiE4OJjQ0FAee+wxq7ZQjq6TqHpIQlCiRjFz5kweffRRjhw5wtNPPw3A6dOn+d///seGDRs4ePAgBoOBRx55hMzMTHbs2MGWLVv4559/GDx4sNVcttdJVE0kc1iiWvHjjz8SHBxsdey1117jtddeA+Dxxx9n5MiRVuc1Gg2ff/45devWBYw9744cOcLZs2eJiYkB4PPPP6dNmzb89ddfdOvWTfQ6iaqJJAQlqhW9evVi6dKlVsdq165tfm65eZWJRo0aWQmy1NRUYmJizAIQoHXr1oSHh5OammoWgrbXSVRNJCEoUa0ICgqiWbNmTs+7c8zde0lUfSSfoISEDa1ateL8+fOcP3/efOz48eNkZWXRunXrClyZhC+QNEGJakVxcbHdFphKpZKIiAi350hISKBdu3YMGzaMhQsXotPpGDduHD179hQ1pyWqNpImKFGt2Lx5M9HR0VaPO+64w6M5ZDIZGzdupFatWtx1110kJCTQpEkT1q1b56NVS1QkUnt9CQmJGo2kCUpISNRoJCEoISFRo5GEoISERI1GEoISEhI1GkkISkhI1GgkISghIVGjkYSghIREjUYSghISEjUaSQhKSEjUaCQhKCEhUaORhKCEhESNRhKCEhISNZr/B5I/uCNW2bDiAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.020937, + "end_time": "2024-02-29T03:13:20.385192", + "exception": false, + "start_time": "2024-02-29T03:13:20.364255", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 1515.30784, + "end_time": "2024-02-29T03:13:23.127943", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/tvae/4/mlu-eval.ipynb", + "output_path": "eval/insurance/tvae/4/mlu-eval.ipynb", + "parameters": { + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "path": "eval/insurance/tvae/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "tvae" + }, + "start_time": "2024-02-29T02:48:07.820103", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/tvae/model.pt b/insurance/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..6438251db3359dbff959842718145d314209c3b5 --- /dev/null +++ b/insurance/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5386af09b15af674d1675be1dfc37256d8947e39c05db9ef3c13096694fc763b +size 38612117 diff --git a/insurance/tvae/params.json b/insurance/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..dfb7a037492a7a46012bf064e5053b32b8604335 --- /dev/null +++ b/insurance/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "ALL", "synth_data": 2, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "fixed_role_model": "tvae", "mse_mag": true, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/lct_gan/eval.csv b/treatment/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..b8994bb5783ef65a3b27c7e7b818559b918eaac7 --- /dev/null +++ b/treatment/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.0,5.660650597639875e-07,0.002430406895076347,4.452483415603638,0.00676386896520853,0.0865572914481163,0.009492019191384315,4.4753242036676966e-06,2.381486177444458,0.03796955943107605,0.07213125377893448,0.04929915815591812,0.06725870072841644,0.018894590437412262,6.833969593048096 diff --git a/treatment/lct_gan/history.csv b/treatment/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..06f2c00d56074d970abec7ce6aa12cf30cb5a073 --- /dev/null +++ b/treatment/lct_gan/history.csv @@ -0,0 +1,21 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.26128039993345736,135.00998422801496,0.08049385482445359,0.4579842137172818,0.0,0.0,0.0,0.0,0.2694031505845487,320,80,146.7050392627716,1.833812990784645,0.45845324769616125,0.005272725281702151,0.25311135954406155,100.63954811269392,0.08570385829276618,0.4422016212542076,0.0,0.0,0.0,0.0,0.26109687432544887,80,20,23.775160551071167,1.1887580275535583,0.2971895068883896,0.00016720850671845256 +1,0.16395704752285384,46.25117130066425,0.043989320625512696,0.25818611555732784,0.0,0.0,0.0,0.0,0.1720848368597217,320,80,144.67404317855835,1.8084255397319793,0.4521063849329948,0.05303716027590326,0.0057226947756134905,3.1514409528212126,5.1956086371518494e-05,0.05613231887109578,0.0,0.0,0.0,0.0,0.005799632198613835,80,20,23.154529809951782,1.157726490497589,0.28943162262439726,0.04428817721782252 +2,0.009967373472500185,1.1435001151776631,0.00025591168522280937,0.0428276356702554,0.0,0.0,0.0,0.0,0.010189983236341505,320,80,144.73762130737305,1.809220266342163,0.45230506658554076,0.18196576089248992,0.005311523424461484,1.4322910904029869,4.043803222786302e-05,0.03152598706074059,0.0,0.0,0.0,0.0,0.005420786762260832,80,20,23.13025999069214,1.1565129995346068,0.2891282498836517,0.04619570770300925 +3,0.006134010907044285,0.35356903110854654,6.0956277070700615e-05,0.02464446089579724,0.0,0.0,0.0,0.0,0.0062516480313206555,320,80,144.51434969902039,1.8064293712377548,0.4516073428094387,0.1885365787660703,0.013837382025667467,2.5610742956889228,0.0005938085471470344,0.022871465422213078,0.0,0.0,0.0,0.0,0.014309528892044909,80,20,23.141812324523926,1.1570906162261962,0.28927265405654906,0.031223367247730494 +4,0.007267655872419709,0.3062819363175539,0.00011790961587259673,0.02917690484318882,0.0,0.0,0.0,0.0,0.0074150352023934826,320,80,144.8067650794983,1.8100845634937286,0.45252114087343215,0.18684698601136915,0.004624169710587012,1.4239937734550865,2.9078645276914284e-05,0.03463592496700585,0.0,0.0,0.0,0.0,0.004706312119378708,80,20,23.05193328857422,1.152596664428711,0.2881491661071777,0.04468875783495605 +5,0.00623816143561271,0.29020242977792704,7.033139000531819e-05,0.021522978675784545,0.0,0.0,0.0,0.0,0.0063716771262988916,320,80,150.50194144248962,1.8812742680311203,0.4703185670077801,0.17831245453562589,0.005090736831334652,1.614344134640487,4.5580479082296675e-05,0.037349450215697286,0.0,0.0,0.0,0.0,0.005194863072028965,80,20,26.129301071166992,1.3064650535583495,0.3266162633895874,0.04783064681105316 +6,0.0066317346205323705,0.155090051446291,0.00012775564626566483,0.024297820206265898,0.0,0.0,0.0,0.0,0.006774960086113424,320,80,146.69751381874084,1.8337189227342605,0.45842973068356513,0.18502795521635562,0.007387617863423656,1.8196432548054873,0.00016067005997262088,0.02709107401315123,0.0,0.0,0.0,0.0,0.007611714216182008,80,20,23.294971227645874,1.1647485613822937,0.2911871403455734,0.04717457559891045 +7,0.0050507940538409455,0.24961636728171185,6.77448583133677e-05,0.021591220212576444,0.0,0.0,0.0,0.0,0.005148960298083693,320,80,148.39960646629333,1.8549950808286666,0.46374877020716665,0.18919667988666333,0.00471943209413439,1.4062298681291623,3.5436748082773303e-05,0.03587816776707768,0.0,0.0,0.0,0.0,0.004805206473974977,80,20,25.67212200164795,1.2836061000823975,0.3209015250205994,0.05039857877418399 +8,0.004636340067554557,0.3507599200662298,7.799542100694057e-05,0.021762262954143807,0.0,0.0,0.0,0.0,0.004718717332980305,320,80,150.16857385635376,1.877107173204422,0.4692767933011055,0.1829542159830453,0.010440742748323828,1.5362174712634442,0.00041889002012851506,0.04380658604204655,0.0,0.0,0.0,0.0,0.010641950925491982,80,20,23.65154242515564,1.182577121257782,0.2956442803144455,0.04646398308686912 +9,0.00379477626629523,0.11390200727429942,7.642670419733718e-05,0.026889230590313673,0.0,0.0,0.0,0.0,0.003848425932574173,320,80,146.48870396614075,1.8311087995767594,0.45777719989418986,0.18120208581676706,0.004786214925115928,1.108299489593469,4.4953928944124756e-05,0.048245068080723284,0.0,0.0,0.0,0.0,0.00485001786146313,80,20,24.08996295928955,1.2044981479644776,0.3011245369911194,0.05166552765294909 +10,0.0033935373040549165,0.254286147947502,1.513824191111358e-05,0.031514893082203344,0.0,0.0,0.0,0.0,0.00343727340514306,320,80,149.14973759651184,1.864371719956398,0.4660929299890995,0.17849353865312878,0.0055357136952807195,1.2160269206097467,7.856696559896681e-05,0.061062258388847115,0.0,0.0,0.0,0.0,0.005603326101845596,80,20,24.012556314468384,1.2006278157234191,0.3001569539308548,0.04710417920723557 +11,0.002693801052055278,0.1464270501550981,2.100214074168402e-05,0.0323501096398104,0.0,0.0,0.0,0.0,0.0027267849328836747,320,80,146.62752771377563,1.8328440964221955,0.45821102410554887,0.18363276715390384,0.0056254648501635526,1.0173772743873997,9.926339396217898e-05,0.06872351877391339,0.0,0.0,0.0,0.0,0.005695302168896887,80,20,23.80198311805725,1.1900991559028626,0.29752478897571566,0.05559386946260929 +12,0.0014208264025228345,0.04045106382802146,5.251845141255966e-06,0.027116044610738754,0.0,0.0,0.0,0.0,0.0014380259427070996,320,80,146.92432856559753,1.8365541070699691,0.4591385267674923,0.2041609299601987,0.005773982740356587,0.8331387747311965,0.00010853523310434543,0.07785461563616991,0.0,0.0,0.0,0.0,0.005842015838425141,80,20,23.71011471748352,1.185505735874176,0.296376433968544,0.05547772371210158 +13,0.0006808990618768273,0.04152264156692178,6.086644313010679e-07,0.021928094135728316,0.0,0.0,0.0,0.0,0.0006899666342157218,320,80,147.01682090759277,1.8377102613449097,0.4594275653362274,0.18354589576229047,0.008106111267989036,0.9430381521855054,0.0002473866394328006,0.0817655582446605,0.0,0.0,0.0,0.0,0.008207306088297627,80,20,23.672499895095825,1.1836249947547912,0.2959062486886978,0.04822360556572676 +14,0.0006262306521421124,0.013846133756529965,5.347792941735277e-07,0.02022404745221138,0.0,0.0,0.0,0.0,0.0006347098269316121,320,80,147.68385481834412,1.8460481852293014,0.46151204630732534,0.19029335069935768,0.007200535701122135,0.9546419340079865,0.0001910924554870519,0.0773586924187839,0.0,0.0,0.0,0.0,0.007289312210195931,80,20,24.65805673599243,1.2329028367996215,0.3082257091999054,0.050718773249536754 +15,0.00033651087611161754,0.012355822190818566,1.6889446918767067e-07,0.014677857210335788,0.0,0.0,0.0,0.0,0.00034159754657139276,320,80,147.733962059021,1.8466745257377624,0.4616686314344406,0.19043351346626877,0.009353837548405863,1.1220808846118415,0.00032246472827637265,0.08625785815529526,0.0,0.0,0.0,0.0,0.009475567466870416,80,20,23.59881329536438,1.179940664768219,0.2949851661920547,0.04545478667132556 +16,0.0004585306428644742,0.04090626894583309,1.9009434258025014e-06,0.017395624532946387,0.0,0.0,0.0,0.0,0.00046507135782718477,320,80,148.33422374725342,1.8541777968406676,0.4635444492101669,0.19497270800638944,0.00842655439155351,0.9143219219447929,0.00028370599442979484,0.07937973253428936,0.0,0.0,0.0,0.0,0.008536131002983893,80,20,23.87963342666626,1.193981671333313,0.2984954178333282,0.054555920278653504 +17,0.00037294581165951967,0.05376572193176317,4.8404896809648184e-08,0.01513790819735732,0.0,0.0,0.0,0.0,0.0003784365873286788,320,80,146.86418867111206,1.8358023583889007,0.4589505895972252,0.19413201494608073,0.007073825206316542,0.9812158623795313,0.00019214735687924644,0.07729512881487607,0.0,0.0,0.0,0.0,0.007161703807651065,80,20,23.716655492782593,1.1858327746391297,0.29645819365978243,0.051480500027537346 +18,0.00014858139757052414,0.007059495568182683,1.036167499412745e-08,0.009602279996033757,0.0,0.0,0.0,0.0,0.0001513956265142724,320,80,146.31447434425354,1.8289309293031693,0.4572327323257923,0.19686991329072043,0.005858969881956,0.7899941703799414,0.0001155219032677457,0.07324809860438108,0.0,0.0,0.0,0.0,0.005929632572951959,80,20,23.843982934951782,1.1921991467475892,0.2980497866868973,0.05531781129539013 +19,0.00010132013832446774,0.01020061541920308,1.3937130828327704e-09,0.007747239743184764,0.0,0.0,0.0,0.0,0.00010356856200104403,320,80,147.32227063179016,1.841528382897377,0.46038209572434424,0.20688477805815636,0.006702580576529726,0.8139775842824587,0.00016960971130888236,0.07491350602358579,0.0,0.0,0.0,0.0,0.006786745876888744,80,20,23.885791540145874,1.1942895770072937,0.2985723942518234,0.053444467252120376 diff --git a/treatment/lct_gan/mlu-eval.ipynb b/treatment/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..62421649162e8bc773f453455d7e5ba5159b6f90 --- /dev/null +++ b/treatment/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2639 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:23.282587Z", + "iopub.status.busy": "2024-02-29T07:53:23.281975Z", + "iopub.status.idle": "2024-02-29T07:53:23.318278Z", + "shell.execute_reply": "2024-02-29T07:53:23.317596Z" + }, + "papermill": { + "duration": 0.051208, + "end_time": "2024-02-29T07:53:23.320176", + "exception": false, + "start_time": "2024-02-29T07:53:23.268968", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:23.346066Z", + "iopub.status.busy": "2024-02-29T07:53:23.345625Z", + "iopub.status.idle": "2024-02-29T07:53:23.353463Z", + "shell.execute_reply": "2024-02-29T07:53:23.352692Z" + }, + "papermill": { + "duration": 0.022914, + "end_time": "2024-02-29T07:53:23.355361", + "exception": false, + "start_time": "2024-02-29T07:53:23.332447", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:23.378743Z", + "iopub.status.busy": "2024-02-29T07:53:23.378177Z", + "iopub.status.idle": "2024-02-29T07:53:23.382108Z", + "shell.execute_reply": "2024-02-29T07:53:23.381324Z" + }, + "papermill": { + "duration": 0.017671, + "end_time": "2024-02-29T07:53:23.383967", + "exception": false, + "start_time": "2024-02-29T07:53:23.366296", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:23.406873Z", + "iopub.status.busy": "2024-02-29T07:53:23.406589Z", + "iopub.status.idle": "2024-02-29T07:53:23.410444Z", + "shell.execute_reply": "2024-02-29T07:53:23.409621Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017574, + "end_time": "2024-02-29T07:53:23.412448", + "exception": false, + "start_time": "2024-02-29T07:53:23.394874", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:23.437239Z", + "iopub.status.busy": "2024-02-29T07:53:23.436989Z", + "iopub.status.idle": "2024-02-29T07:53:23.442321Z", + "shell.execute_reply": "2024-02-29T07:53:23.441432Z" + }, + "papermill": { + "duration": 0.019722, + "end_time": "2024-02-29T07:53:23.444415", + "exception": false, + "start_time": "2024-02-29T07:53:23.424693", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a9065b6c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:23.469295Z", + "iopub.status.busy": "2024-02-29T07:53:23.469023Z", + "iopub.status.idle": "2024-02-29T07:53:23.473529Z", + "shell.execute_reply": "2024-02-29T07:53:23.472704Z" + }, + "papermill": { + "duration": 0.019365, + "end_time": "2024-02-29T07:53:23.475415", + "exception": false, + "start_time": "2024-02-29T07:53:23.456050", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"lct_gan\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/lct_gan/2\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011009, + "end_time": "2024-02-29T07:53:23.497610", + "exception": false, + "start_time": "2024-02-29T07:53:23.486601", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:23.520904Z", + "iopub.status.busy": "2024-02-29T07:53:23.520614Z", + "iopub.status.idle": "2024-02-29T07:53:23.529888Z", + "shell.execute_reply": "2024-02-29T07:53:23.528998Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023319, + "end_time": "2024-02-29T07:53:23.532103", + "exception": false, + "start_time": "2024-02-29T07:53:23.508784", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/lct_gan/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:23.555977Z", + "iopub.status.busy": "2024-02-29T07:53:23.555716Z", + "iopub.status.idle": "2024-02-29T07:53:25.691926Z", + "shell.execute_reply": "2024-02-29T07:53:25.691034Z" + }, + "papermill": { + "duration": 2.150499, + "end_time": "2024-02-29T07:53:25.694094", + "exception": false, + "start_time": "2024-02-29T07:53:23.543595", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:25.720419Z", + "iopub.status.busy": "2024-02-29T07:53:25.719577Z", + "iopub.status.idle": "2024-02-29T07:53:25.743440Z", + "shell.execute_reply": "2024-02-29T07:53:25.742673Z" + }, + "papermill": { + "duration": 0.039435, + "end_time": "2024-02-29T07:53:25.745749", + "exception": false, + "start_time": "2024-02-29T07:53:25.706314", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:25.771026Z", + "iopub.status.busy": "2024-02-29T07:53:25.770739Z", + "iopub.status.idle": "2024-02-29T07:53:25.780139Z", + "shell.execute_reply": "2024-02-29T07:53:25.779218Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.024181, + "end_time": "2024-02-29T07:53:25.782359", + "exception": false, + "start_time": "2024-02-29T07:53:25.758178", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:25.806356Z", + "iopub.status.busy": "2024-02-29T07:53:25.806110Z", + "iopub.status.idle": "2024-02-29T07:53:26.285996Z", + "shell.execute_reply": "2024-02-29T07:53:26.284895Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.494651, + "end_time": "2024-02-29T07:53:26.288584", + "exception": false, + "start_time": "2024-02-29T07:53:25.793933", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:26.316426Z", + "iopub.status.busy": "2024-02-29T07:53:26.316111Z", + "iopub.status.idle": "2024-02-29T07:53:39.237187Z", + "shell.execute_reply": "2024-02-29T07:53:39.236385Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 12.93783, + "end_time": "2024-02-29T07:53:39.239473", + "exception": false, + "start_time": "2024-02-29T07:53:26.301643", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 07:53:30.893014: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 07:53:30.893112: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 07:53:31.025611: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:39.265886Z", + "iopub.status.busy": "2024-02-29T07:53:39.265299Z", + "iopub.status.idle": "2024-02-29T07:53:39.293905Z", + "shell.execute_reply": "2024-02-29T07:53:39.292922Z" + }, + "papermill": { + "duration": 0.043806, + "end_time": "2024-02-29T07:53:39.295786", + "exception": false, + "start_time": "2024-02-29T07:53:39.251980", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:53:39.319505Z", + "iopub.status.busy": "2024-02-29T07:53:39.319233Z", + "iopub.status.idle": "2024-02-29T07:54:01.966487Z", + "shell.execute_reply": "2024-02-29T07:54:01.965396Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.661755, + "end_time": "2024-02-29T07:54:01.968756", + "exception": false, + "start_time": "2024-02-29T07:53:39.307001", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BEST,\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:54:02.759392Z", + "iopub.status.busy": "2024-02-29T07:54:02.759056Z", + "iopub.status.idle": "2024-02-29T07:54:02.830471Z", + "shell.execute_reply": "2024-02-29T07:54:02.829697Z" + }, + "papermill": { + "duration": 0.087216, + "end_time": "2024-02-29T07:54:02.832488", + "exception": false, + "start_time": "2024-02-29T07:54:02.745272", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../treatment/_cache/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache4/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache5/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/treatment [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T07:54:02.860835Z", + "iopub.status.busy": "2024-02-29T07:54:02.860328Z", + "iopub.status.idle": "2024-02-29T07:54:03.411029Z", + "shell.execute_reply": "2024-02-29T07:54:03.410099Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.56694, + "end_time": "2024-02-29T07:54:03.413085", + "exception": false, + "start_time": "2024-02-29T07:54:02.846145", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:54:03.442417Z", + "iopub.status.busy": "2024-02-29T07:54:03.442111Z", + "iopub.status.idle": "2024-02-29T07:54:03.446300Z", + "shell.execute_reply": "2024-02-29T07:54:03.445444Z" + }, + "papermill": { + "duration": 0.021189, + "end_time": "2024-02-29T07:54:03.448342", + "exception": false, + "start_time": "2024-02-29T07:54:03.427153", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:54:03.479042Z", + "iopub.status.busy": "2024-02-29T07:54:03.478772Z", + "iopub.status.idle": "2024-02-29T07:54:03.485938Z", + "shell.execute_reply": "2024-02-29T07:54:03.485073Z" + }, + "papermill": { + "duration": 0.024182, + "end_time": "2024-02-29T07:54:03.488090", + "exception": false, + "start_time": "2024-02-29T07:54:03.463908", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18680833" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:54:03.517373Z", + "iopub.status.busy": "2024-02-29T07:54:03.517126Z", + "iopub.status.idle": "2024-02-29T07:54:03.618381Z", + "shell.execute_reply": "2024-02-29T07:54:03.617580Z" + }, + "papermill": { + "duration": 0.118435, + "end_time": "2024-02-29T07:54:03.620360", + "exception": false, + "start_time": "2024-02-29T07:54:03.501925", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 75] --\n", + "├─Adapter: 1-1 [2, 2648, 75] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 77,824\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 75] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,680,833\n", + "Trainable params: 18,680,833\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 73.97\n", + "========================================================================================================================\n", + "Input size (MB): 1.99\n", + "Forward/backward pass size (MB): 1079.48\n", + "Params size (MB): 74.72\n", + "Estimated Total Size (MB): 1156.19\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T07:54:03.652182Z", + "iopub.status.busy": "2024-02-29T07:54:03.651871Z", + "iopub.status.idle": "2024-02-29T08:56:32.970546Z", + "shell.execute_reply": "2024-02-29T08:56:32.969707Z" + }, + "papermill": { + "duration": 3749.337625, + "end_time": "2024-02-29T08:56:32.972941", + "exception": false, + "start_time": "2024-02-29T07:54:03.635316", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.26128039993345736, 'avg_role_model_std_loss': 135.00998422801496, 'avg_role_model_mean_pred_loss': 0.08049385482445359, 'avg_role_model_g_mag_loss': 0.4579842137172818, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.2694031505845487, 'n_size': 320, 'n_batch': 80, 'duration': 146.7050392627716, 'duration_batch': 1.833812990784645, 'duration_size': 0.45845324769616125, 'avg_pred_std': 0.005272725281702151}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.25311135954406155, 'avg_role_model_std_loss': 100.63954811269392, 'avg_role_model_mean_pred_loss': 0.08570385829276618, 'avg_role_model_g_mag_loss': 0.4422016212542076, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.26109687432544887, 'n_size': 80, 'n_batch': 20, 'duration': 23.775160551071167, 'duration_batch': 1.1887580275535583, 'duration_size': 0.2971895068883896, 'avg_pred_std': 0.00016720850671845256}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.16395704752285384, 'avg_role_model_std_loss': 46.25117130066425, 'avg_role_model_mean_pred_loss': 0.043989320625512696, 'avg_role_model_g_mag_loss': 0.25818611555732784, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.1720848368597217, 'n_size': 320, 'n_batch': 80, 'duration': 144.67404317855835, 'duration_batch': 1.8084255397319793, 'duration_size': 0.4521063849329948, 'avg_pred_std': 0.05303716027590326}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0057226947756134905, 'avg_role_model_std_loss': 3.1514409528212126, 'avg_role_model_mean_pred_loss': 5.1956086371518494e-05, 'avg_role_model_g_mag_loss': 0.05613231887109578, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005799632198613835, 'n_size': 80, 'n_batch': 20, 'duration': 23.154529809951782, 'duration_batch': 1.157726490497589, 'duration_size': 0.28943162262439726, 'avg_pred_std': 0.04428817721782252}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.009967373472500185, 'avg_role_model_std_loss': 1.1435001151776631, 'avg_role_model_mean_pred_loss': 0.00025591168522280937, 'avg_role_model_g_mag_loss': 0.0428276356702554, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010189983236341505, 'n_size': 320, 'n_batch': 80, 'duration': 144.73762130737305, 'duration_batch': 1.809220266342163, 'duration_size': 0.45230506658554076, 'avg_pred_std': 0.18196576089248992}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005311523424461484, 'avg_role_model_std_loss': 1.4322910904029869, 'avg_role_model_mean_pred_loss': 4.043803222786302e-05, 'avg_role_model_g_mag_loss': 0.03152598706074059, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005420786762260832, 'n_size': 80, 'n_batch': 20, 'duration': 23.13025999069214, 'duration_batch': 1.1565129995346068, 'duration_size': 0.2891282498836517, 'avg_pred_std': 0.04619570770300925}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006134010907044285, 'avg_role_model_std_loss': 0.35356903110854654, 'avg_role_model_mean_pred_loss': 6.0956277070700615e-05, 'avg_role_model_g_mag_loss': 0.02464446089579724, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0062516480313206555, 'n_size': 320, 'n_batch': 80, 'duration': 144.51434969902039, 'duration_batch': 1.8064293712377548, 'duration_size': 0.4516073428094387, 'avg_pred_std': 0.1885365787660703}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013837382025667467, 'avg_role_model_std_loss': 2.5610742956889228, 'avg_role_model_mean_pred_loss': 0.0005938085471470344, 'avg_role_model_g_mag_loss': 0.022871465422213078, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014309528892044909, 'n_size': 80, 'n_batch': 20, 'duration': 23.141812324523926, 'duration_batch': 1.1570906162261962, 'duration_size': 0.28927265405654906, 'avg_pred_std': 0.031223367247730494}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007267655872419709, 'avg_role_model_std_loss': 0.3062819363175539, 'avg_role_model_mean_pred_loss': 0.00011790961587259673, 'avg_role_model_g_mag_loss': 0.02917690484318882, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0074150352023934826, 'n_size': 320, 'n_batch': 80, 'duration': 144.8067650794983, 'duration_batch': 1.8100845634937286, 'duration_size': 0.45252114087343215, 'avg_pred_std': 0.18684698601136915}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004624169710587012, 'avg_role_model_std_loss': 1.4239937734550865, 'avg_role_model_mean_pred_loss': 2.9078645276914284e-05, 'avg_role_model_g_mag_loss': 0.03463592496700585, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004706312119378708, 'n_size': 80, 'n_batch': 20, 'duration': 23.05193328857422, 'duration_batch': 1.152596664428711, 'duration_size': 0.2881491661071777, 'avg_pred_std': 0.04468875783495605}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00623816143561271, 'avg_role_model_std_loss': 0.29020242977792704, 'avg_role_model_mean_pred_loss': 7.033139000531819e-05, 'avg_role_model_g_mag_loss': 0.021522978675784545, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0063716771262988916, 'n_size': 320, 'n_batch': 80, 'duration': 150.50194144248962, 'duration_batch': 1.8812742680311203, 'duration_size': 0.4703185670077801, 'avg_pred_std': 0.17831245453562589}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005090736831334652, 'avg_role_model_std_loss': 1.614344134640487, 'avg_role_model_mean_pred_loss': 4.5580479082296675e-05, 'avg_role_model_g_mag_loss': 0.037349450215697286, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005194863072028965, 'n_size': 80, 'n_batch': 20, 'duration': 26.129301071166992, 'duration_batch': 1.3064650535583495, 'duration_size': 0.3266162633895874, 'avg_pred_std': 0.04783064681105316}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0066317346205323705, 'avg_role_model_std_loss': 0.155090051446291, 'avg_role_model_mean_pred_loss': 0.00012775564626566483, 'avg_role_model_g_mag_loss': 0.024297820206265898, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006774960086113424, 'n_size': 320, 'n_batch': 80, 'duration': 146.69751381874084, 'duration_batch': 1.8337189227342605, 'duration_size': 0.45842973068356513, 'avg_pred_std': 0.18502795521635562}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007387617863423656, 'avg_role_model_std_loss': 1.8196432548054873, 'avg_role_model_mean_pred_loss': 0.00016067005997262088, 'avg_role_model_g_mag_loss': 0.02709107401315123, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007611714216182008, 'n_size': 80, 'n_batch': 20, 'duration': 23.294971227645874, 'duration_batch': 1.1647485613822937, 'duration_size': 0.2911871403455734, 'avg_pred_std': 0.04717457559891045}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0050507940538409455, 'avg_role_model_std_loss': 0.24961636728171185, 'avg_role_model_mean_pred_loss': 6.77448583133677e-05, 'avg_role_model_g_mag_loss': 0.021591220212576444, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005148960298083693, 'n_size': 320, 'n_batch': 80, 'duration': 148.39960646629333, 'duration_batch': 1.8549950808286666, 'duration_size': 0.46374877020716665, 'avg_pred_std': 0.18919667988666333}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00471943209413439, 'avg_role_model_std_loss': 1.4062298681291623, 'avg_role_model_mean_pred_loss': 3.5436748082773303e-05, 'avg_role_model_g_mag_loss': 0.03587816776707768, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004805206473974977, 'n_size': 80, 'n_batch': 20, 'duration': 25.67212200164795, 'duration_batch': 1.2836061000823975, 'duration_size': 0.3209015250205994, 'avg_pred_std': 0.05039857877418399}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004636340067554557, 'avg_role_model_std_loss': 0.3507599200662298, 'avg_role_model_mean_pred_loss': 7.799542100694057e-05, 'avg_role_model_g_mag_loss': 0.021762262954143807, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004718717332980305, 'n_size': 320, 'n_batch': 80, 'duration': 150.16857385635376, 'duration_batch': 1.877107173204422, 'duration_size': 0.4692767933011055, 'avg_pred_std': 0.1829542159830453}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010440742748323828, 'avg_role_model_std_loss': 1.5362174712634442, 'avg_role_model_mean_pred_loss': 0.00041889002012851506, 'avg_role_model_g_mag_loss': 0.04380658604204655, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010641950925491982, 'n_size': 80, 'n_batch': 20, 'duration': 23.65154242515564, 'duration_batch': 1.182577121257782, 'duration_size': 0.2956442803144455, 'avg_pred_std': 0.04646398308686912}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00379477626629523, 'avg_role_model_std_loss': 0.11390200727429942, 'avg_role_model_mean_pred_loss': 7.642670419733718e-05, 'avg_role_model_g_mag_loss': 0.026889230590313673, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003848425932574173, 'n_size': 320, 'n_batch': 80, 'duration': 146.48870396614075, 'duration_batch': 1.8311087995767594, 'duration_size': 0.45777719989418986, 'avg_pred_std': 0.18120208581676706}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004786214925115928, 'avg_role_model_std_loss': 1.108299489593469, 'avg_role_model_mean_pred_loss': 4.4953928944124756e-05, 'avg_role_model_g_mag_loss': 0.048245068080723284, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00485001786146313, 'n_size': 80, 'n_batch': 20, 'duration': 24.08996295928955, 'duration_batch': 1.2044981479644776, 'duration_size': 0.3011245369911194, 'avg_pred_std': 0.05166552765294909}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0033935373040549165, 'avg_role_model_std_loss': 0.254286147947502, 'avg_role_model_mean_pred_loss': 1.513824191111358e-05, 'avg_role_model_g_mag_loss': 0.031514893082203344, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00343727340514306, 'n_size': 320, 'n_batch': 80, 'duration': 149.14973759651184, 'duration_batch': 1.864371719956398, 'duration_size': 0.4660929299890995, 'avg_pred_std': 0.17849353865312878}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0055357136952807195, 'avg_role_model_std_loss': 1.2160269206097467, 'avg_role_model_mean_pred_loss': 7.856696559896681e-05, 'avg_role_model_g_mag_loss': 0.061062258388847115, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005603326101845596, 'n_size': 80, 'n_batch': 20, 'duration': 24.012556314468384, 'duration_batch': 1.2006278157234191, 'duration_size': 0.3001569539308548, 'avg_pred_std': 0.04710417920723557}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002693801052055278, 'avg_role_model_std_loss': 0.1464270501550981, 'avg_role_model_mean_pred_loss': 2.100214074168402e-05, 'avg_role_model_g_mag_loss': 0.0323501096398104, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027267849328836747, 'n_size': 320, 'n_batch': 80, 'duration': 146.62752771377563, 'duration_batch': 1.8328440964221955, 'duration_size': 0.45821102410554887, 'avg_pred_std': 0.18363276715390384}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0056254648501635526, 'avg_role_model_std_loss': 1.0173772743873997, 'avg_role_model_mean_pred_loss': 9.926339396217898e-05, 'avg_role_model_g_mag_loss': 0.06872351877391339, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005695302168896887, 'n_size': 80, 'n_batch': 20, 'duration': 23.80198311805725, 'duration_batch': 1.1900991559028626, 'duration_size': 0.29752478897571566, 'avg_pred_std': 0.05559386946260929}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014208264025228345, 'avg_role_model_std_loss': 0.04045106382802146, 'avg_role_model_mean_pred_loss': 5.251845141255966e-06, 'avg_role_model_g_mag_loss': 0.027116044610738754, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014380259427070996, 'n_size': 320, 'n_batch': 80, 'duration': 146.92432856559753, 'duration_batch': 1.8365541070699691, 'duration_size': 0.4591385267674923, 'avg_pred_std': 0.2041609299601987}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005773982740356587, 'avg_role_model_std_loss': 0.8331387747311965, 'avg_role_model_mean_pred_loss': 0.00010853523310434543, 'avg_role_model_g_mag_loss': 0.07785461563616991, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005842015838425141, 'n_size': 80, 'n_batch': 20, 'duration': 23.71011471748352, 'duration_batch': 1.185505735874176, 'duration_size': 0.296376433968544, 'avg_pred_std': 0.05547772371210158}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006808990618768273, 'avg_role_model_std_loss': 0.04152264156692178, 'avg_role_model_mean_pred_loss': 6.086644313010679e-07, 'avg_role_model_g_mag_loss': 0.021928094135728316, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006899666342157218, 'n_size': 320, 'n_batch': 80, 'duration': 147.01682090759277, 'duration_batch': 1.8377102613449097, 'duration_size': 0.4594275653362274, 'avg_pred_std': 0.18354589576229047}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008106111267989036, 'avg_role_model_std_loss': 0.9430381521855054, 'avg_role_model_mean_pred_loss': 0.0002473866394328006, 'avg_role_model_g_mag_loss': 0.0817655582446605, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008207306088297627, 'n_size': 80, 'n_batch': 20, 'duration': 23.672499895095825, 'duration_batch': 1.1836249947547912, 'duration_size': 0.2959062486886978, 'avg_pred_std': 0.04822360556572676}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006262306521421124, 'avg_role_model_std_loss': 0.013846133756529965, 'avg_role_model_mean_pred_loss': 5.347792941735277e-07, 'avg_role_model_g_mag_loss': 0.02022404745221138, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006347098269316121, 'n_size': 320, 'n_batch': 80, 'duration': 147.68385481834412, 'duration_batch': 1.8460481852293014, 'duration_size': 0.46151204630732534, 'avg_pred_std': 0.19029335069935768}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007200535701122135, 'avg_role_model_std_loss': 0.9546419340079865, 'avg_role_model_mean_pred_loss': 0.0001910924554870519, 'avg_role_model_g_mag_loss': 0.0773586924187839, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007289312210195931, 'n_size': 80, 'n_batch': 20, 'duration': 24.65805673599243, 'duration_batch': 1.2329028367996215, 'duration_size': 0.3082257091999054, 'avg_pred_std': 0.050718773249536754}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00033651087611161754, 'avg_role_model_std_loss': 0.012355822190818566, 'avg_role_model_mean_pred_loss': 1.6889446918767067e-07, 'avg_role_model_g_mag_loss': 0.014677857210335788, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00034159754657139276, 'n_size': 320, 'n_batch': 80, 'duration': 147.733962059021, 'duration_batch': 1.8466745257377624, 'duration_size': 0.4616686314344406, 'avg_pred_std': 0.19043351346626877}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009353837548405863, 'avg_role_model_std_loss': 1.1220808846118415, 'avg_role_model_mean_pred_loss': 0.00032246472827637265, 'avg_role_model_g_mag_loss': 0.08625785815529526, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009475567466870416, 'n_size': 80, 'n_batch': 20, 'duration': 23.59881329536438, 'duration_batch': 1.179940664768219, 'duration_size': 0.2949851661920547, 'avg_pred_std': 0.04545478667132556}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0004585306428644742, 'avg_role_model_std_loss': 0.04090626894583309, 'avg_role_model_mean_pred_loss': 1.9009434258025014e-06, 'avg_role_model_g_mag_loss': 0.017395624532946387, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00046507135782718477, 'n_size': 320, 'n_batch': 80, 'duration': 148.33422374725342, 'duration_batch': 1.8541777968406676, 'duration_size': 0.4635444492101669, 'avg_pred_std': 0.19497270800638944}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00842655439155351, 'avg_role_model_std_loss': 0.9143219219447929, 'avg_role_model_mean_pred_loss': 0.00028370599442979484, 'avg_role_model_g_mag_loss': 0.07937973253428936, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008536131002983893, 'n_size': 80, 'n_batch': 20, 'duration': 23.87963342666626, 'duration_batch': 1.193981671333313, 'duration_size': 0.2984954178333282, 'avg_pred_std': 0.054555920278653504}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00037294581165951967, 'avg_role_model_std_loss': 0.05376572193176317, 'avg_role_model_mean_pred_loss': 4.8404896809648184e-08, 'avg_role_model_g_mag_loss': 0.01513790819735732, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0003784365873286788, 'n_size': 320, 'n_batch': 80, 'duration': 146.86418867111206, 'duration_batch': 1.8358023583889007, 'duration_size': 0.4589505895972252, 'avg_pred_std': 0.19413201494608073}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007073825206316542, 'avg_role_model_std_loss': 0.9812158623795313, 'avg_role_model_mean_pred_loss': 0.00019214735687924644, 'avg_role_model_g_mag_loss': 0.07729512881487607, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007161703807651065, 'n_size': 80, 'n_batch': 20, 'duration': 23.716655492782593, 'duration_batch': 1.1858327746391297, 'duration_size': 0.29645819365978243, 'avg_pred_std': 0.051480500027537346}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00014858139757052414, 'avg_role_model_std_loss': 0.007059495568182683, 'avg_role_model_mean_pred_loss': 1.036167499412745e-08, 'avg_role_model_g_mag_loss': 0.009602279996033757, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0001513956265142724, 'n_size': 320, 'n_batch': 80, 'duration': 146.31447434425354, 'duration_batch': 1.8289309293031693, 'duration_size': 0.4572327323257923, 'avg_pred_std': 0.19686991329072043}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005858969881956, 'avg_role_model_std_loss': 0.7899941703799414, 'avg_role_model_mean_pred_loss': 0.0001155219032677457, 'avg_role_model_g_mag_loss': 0.07324809860438108, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005929632572951959, 'n_size': 80, 'n_batch': 20, 'duration': 23.843982934951782, 'duration_batch': 1.1921991467475892, 'duration_size': 0.2980497866868973, 'avg_pred_std': 0.05531781129539013}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00010132013832446774, 'avg_role_model_std_loss': 0.01020061541920308, 'avg_role_model_mean_pred_loss': 1.3937130828327704e-09, 'avg_role_model_g_mag_loss': 0.007747239743184764, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00010356856200104403, 'n_size': 320, 'n_batch': 80, 'duration': 147.32227063179016, 'duration_batch': 1.841528382897377, 'duration_size': 0.46038209572434424, 'avg_pred_std': 0.20688477805815636}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006702580576529726, 'avg_role_model_std_loss': 0.8139775842824587, 'avg_role_model_mean_pred_loss': 0.00016960971130888236, 'avg_role_model_g_mag_loss': 0.07491350602358579, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006786745876888744, 'n_size': 80, 'n_batch': 20, 'duration': 23.885791540145874, 'duration_batch': 1.1942895770072937, 'duration_size': 0.2985723942518234, 'avg_pred_std': 0.053444467252120376}\n", + "Epoch 20\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 8.32226011681314e-05, 'avg_role_model_std_loss': 0.0031902712892142304, 'avg_role_model_mean_pred_loss': 1.1718505702428256e-09, 'avg_role_model_g_mag_loss': 0.007379432961170096, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 8.520250472372481e-05, 'n_size': 320, 'n_batch': 80, 'duration': 147.76828527450562, 'duration_batch': 1.8471035659313202, 'duration_size': 0.46177589148283005, 'avg_pred_std': 0.19231144490186125}\n", + "Time out: 3629.001581430435/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▅▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▁▇▇▅▇▇▇▇▇█▇██▇▇▇█▇██\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▁▃▇▇▇▇▇▇▇▇▇▇█▇▇▇████\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test █▂▁▁▁▁▁▁▁▁▂▂▂▂▂▂▂▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train █▅▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▅▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▅▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train █▃▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▃▁▁▁▁█▂▇▂▃▃▃▂▂▅▂▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▄▁▁▁▁█▄▆█▃▆▃▄▄▅▅▅▄▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▃▁▁▁▁█▂▇▂▃▃▃▂▂▅▂▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▄▁▁▁▁█▄▆█▃▆▃▄▄▅▅▅▄▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▃▁▁▁▁█▂▇▂▃▃▃▂▂▅▂▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▄▁▁▁▁█▄▆█▃▆▃▄▄▅▅▅▄▃▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00679\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.0001\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.05344\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.20688\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.07491\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.00775\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.0067\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.0001\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 0.00017\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 0.81398\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.0102\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 1.19429\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.84153\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.29857\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.46038\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 23.88579\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 147.32227\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 20\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/treatment/lct_gan/2/wandb/offline-run-20240229_075405-ym713xql\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_075405-ym713xql/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 399, 'n_batch': 100, 'role_model_metrics': {'avg_loss': 0.002430406889240993, 'avg_g_mag_loss': 9.392148260125861e-08, 'avg_g_cos_loss': 0.0, 'pred_duration': 2.3932507038116455, 'grad_duration': 4.443832159042358, 'total_duration': 6.837082862854004, 'pred_std': 0.06725870072841644, 'std_loss': 0.018894590437412262, 'mean_pred_loss': 4.4753246584150475e-06, 'pred_rmse': 0.04929915815591812, 'pred_mae': 0.03796955943107605, 'pred_mape': 0.07213124632835388, 'grad_rmse': 0.009492019191384315, 'grad_mae': 0.006763868033885956, 'grad_mape': 0.0865572914481163}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.002430406889240993, 'avg_g_mag_loss': 9.392148260125861e-08, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 2.3932507038116455, 'avg_grad_duration': 4.443832159042358, 'avg_total_duration': 6.837082862854004, 'avg_pred_std': 0.06725870072841644, 'avg_std_loss': 0.018894590437412262, 'avg_mean_pred_loss': 4.4753246584150475e-06}, 'min_metrics': {'avg_loss': 0.002430406889240993, 'avg_g_mag_loss': 9.392148260125861e-08, 'avg_g_cos_loss': 0.0, 'pred_duration': 2.3932507038116455, 'grad_duration': 4.443832159042358, 'total_duration': 6.837082862854004, 'pred_std': 0.06725870072841644, 'std_loss': 0.018894590437412262, 'mean_pred_loss': 4.4753246584150475e-06, 'pred_rmse': 0.04929915815591812, 'pred_mae': 0.03796955943107605, 'pred_mape': 0.07213124632835388, 'grad_rmse': 0.009492019191384315, 'grad_mae': 0.006763868033885956, 'grad_mape': 0.0865572914481163}, 'model_metrics': {'lct_gan': {'avg_loss': 0.002430406889240993, 'avg_g_mag_loss': 9.392148260125861e-08, 'avg_g_cos_loss': 0.0, 'pred_duration': 2.3932507038116455, 'grad_duration': 4.443832159042358, 'total_duration': 6.837082862854004, 'pred_std': 0.06725870072841644, 'std_loss': 0.018894590437412262, 'mean_pred_loss': 4.4753246584150475e-06, 'pred_rmse': 0.04929915815591812, 'pred_mae': 0.03796955943107605, 'pred_mape': 0.07213124632835388, 'grad_rmse': 0.009492019191384315, 'grad_mae': 0.006763868033885956, 'grad_mape': 0.0865572914481163}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T08:56:33.012627Z", + "iopub.status.busy": "2024-02-29T08:56:33.012293Z", + "iopub.status.idle": "2024-02-29T08:56:33.016730Z", + "shell.execute_reply": "2024-02-29T08:56:33.015857Z" + }, + "papermill": { + "duration": 0.026526, + "end_time": "2024-02-29T08:56:33.018632", + "exception": false, + "start_time": "2024-02-29T08:56:32.992106", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T08:56:33.057616Z", + "iopub.status.busy": "2024-02-29T08:56:33.057325Z", + "iopub.status.idle": "2024-02-29T08:56:33.515114Z", + "shell.execute_reply": "2024-02-29T08:56:33.514296Z" + }, + "papermill": { + "duration": 0.480293, + "end_time": "2024-02-29T08:56:33.517587", + "exception": false, + "start_time": "2024-02-29T08:56:33.037294", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T08:56:33.557328Z", + "iopub.status.busy": "2024-02-29T08:56:33.556929Z", + "iopub.status.idle": "2024-02-29T08:56:33.854813Z", + "shell.execute_reply": "2024-02-29T08:56:33.853813Z" + }, + "papermill": { + "duration": 0.320135, + "end_time": "2024-02-29T08:56:33.856875", + "exception": false, + "start_time": "2024-02-29T08:56:33.536740", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA010lEQVR4nO3de3xU9Z3/8deZmcxM7glJyEUj4RZBhGBBsqjoWrIkVC24tAUe/OTyq/AoD2nrRtTGSkJrtwFlLbWy0NKfgrvlol21XevGSyS6SgQFqVSQAgYDQq6S2ySZycx8f39MZnIhl5lkciHzeT4e5zEzZ86c+c6QefM93+/5fo+mlFIIIcQwoBvqAgghhJsEkhBi2JBAEkIMGxJIQohhQwJJCDFsSCAJIYYNCSQhxLBhGOoC+IPT6eTixYuEh4ejadpQF0cI0YlSivr6epKSktDpuq8HjYhAunjxIsnJyUNdDCFEL86fP8+1117b7fMjIpDCw8MB14eNiIgY4tIIITqrq6sjOTnZ81vtzogIJPdhWkREhASSEMNYb00q0qgthBg2JJCEEMOGBJIQYtgYEW1Ion8cDgctLS1DXQxxFQsKCkKv1/d7PxJIAUwpRVlZGTU1NUNdFDECREVFkZCQ0K9zASWQApg7jEaPHk1ISIicVCr6RClFY2MjFRUVACQmJvZ5XwEVSHaHk5OX6rlwuZH5U/v+pY0EDofDE0YxMTFDXRxxlQsODgagoqKC0aNH9/nwLaAatRtbHNzz7Pus/cNR6psDu83E3WYUEhIyxCURI4X7b6k/7ZEBFUgR5iBiw4wAnKtqHOLSDA9ymCb8xR9/SwEVSABjY0MB+KKqYYhLIoToLOACKSXGFUhSQxJi+Am4QBob5wqkEqkhiT7SNI1XX311qIvhVxs3bmT69OlDXYwADKTWGlJJtdSQxNVr165dREVF+W1/69evp7Cw0G/766vACqSWZu744D7+x/gTyiqrkGtkipHOZrN5tV1YWNiwOP0jsALJYCK47GMm60pxNtdzuTGwu/47U0rRaLMP+uLrfwwFBQXcdtttREVFERMTw913383Zs2cBuOWWW3j00Uc7bF9ZWUlQUBDvvfceAJcuXeKuu+4iODiYsWPHsmfPHlJSUti6dWufvrfjx4/zzW9+k+DgYGJiYlizZg0NDW1NAkVFRcyaNYvQ0FCioqK49dZb+fLLLwH461//yp133kl4eDgRERHMmDGDjz/+uMf3KyoqYtWqVdTW1qJpGpqmsXHjRgBSUlJ44oknWL58OREREaxZswaARx99lNTUVEJCQhg3bhwbNmzo0D3f+ZBt5cqVLFy4kC1btpCYmEhMTAwPPPDAgA8xCqgTI9E0NFMEWGsJ15ooqWpgVOiooS7VsNHU4uCG3DcG/X1P/DyTEKP3f4oWi4Xs7GymTZtGQ0MDubm53HvvvRw7doxly5bx5JNPsmnTJk839P79+0lKSmLOnDkALF++nKqqKoqKiggKCiI7O9tzlrGvLBYLmZmZzJ49m48++oiKigruv/9+1q1bx65du7Db7SxcuJDVq1ezd+9ebDYbhw8f9pRt2bJl3HTTTWzfvh29Xs+xY8cICgrq8T1vueUWtm7dSm5uLqdOnQJcNRy3LVu2kJubS15enmddeHg4u3btIikpiePHj7N69WrCw8N55JFHun2fAwcOkJiYyIEDBzhz5gyLFy9m+vTprF69uk/flTcCK5AAzK5ACqOJkqpGZoyRQLraLFq0qMPj5557jri4OE6cOMH3vvc9HnzwQd5//31PAO3Zs4elS5eiaRqff/45b7/9Nh999BEzZ84E4Pe//z0TJ07sU1n27NlDc3MzL7zwAqGhrvbJZ599lnvuuYfNmzcTFBREbW0td999N+PHjwdg8uTJnteXlpby8MMPM2nSJACvymE0GomMjETTNBISEq54/pvf/CYPPfRQh3WPP/64535KSgrr169n3759PQZSdHQ0zz77LHq9nkmTJnHXXXdRWFgogeRXJtcUmuFao/S0dRIcpOfEzzOH5H19cfr0aXJzczl06BBVVVU4nU7A9eO+8cYbmTdvHn/4wx+YM2cOJSUlFBcX89vf/haAU6dOYTAY+MY3vuHZ34QJE4iOju5T2U+ePElaWponjABuvfVWnE4np06d4vbbb2flypVkZmbyT//0T2RkZPC9733PM94rOzub+++/n//4j/8gIyOD7373u57g6it30La3f/9+nnnmGc6ePUtDQwN2u73X2VWnTJnSYQhIYmIix48f71fZehNYbUjgCaQwmuRcpE40TSPEaBj0xdczfO+55x6+/vprdu7cyaFDhzh06BDQ1oC7bNky/vjHP9LS0sKePXuYOnUqU6dO9fv35a3nn3+e4uJibrnlFvbv309qaioffvgh4Gq7+eyzz7jrrrt45513uOGGG3jllVf69X7twxGguLiYZcuW8a1vfYvXXnuNTz75hJ/+9Ke9Nnh3PnTUNM0T/gOlT4G0bds2UlJSMJvNpKenc/jw4W633blzJ3PmzCE6Opro6GgyMjKu2H7lypWexjn3kpWV1Zei9c5TQ2riiyrLwLyHGDDV1dWcOnWKxx9/nLlz5zJ58mQuX77cYZsFCxbQ3NxMQUEBe/bsYdmyZZ7nrr/+eux2O5988oln3ZkzZ67Yh7cmT57MX//6VyyWtr+lDz74AJ1Ox/XXX+9Zd9NNN5GTk8PBgwe58cYb2bNnj+e51NRU/uVf/oU333yTf/7nf+b555/v9X2NRiMOh8OrMh48eJAxY8bw05/+lJkzZzJx4kRPo/pw43Mg7d+/n+zsbPLy8jh69ChpaWlkZmZ22yhYVFTE0qVLOXDgAMXFxSQnJzNv3jy++uqrDttlZWVx6dIlz7J3796+faLemFzV1HAaOVdlka7/q0x0dDQxMTH87ne/48yZM7zzzjtkZ2d32CY0NJSFCxeyYcMGTp48ydKlSz3PTZo0iYyMDNasWcPhw4f55JNPWLNmDcHBwX0ai7Vs2TLMZjMrVqzgb3/7GwcOHOCHP/wh9913H/Hx8ZSUlJCTk0NxcTFffvklb775JqdPn2by5Mk0NTWxbt06ioqK+PLLL/nggw/46KOPOrQxdSclJYWGhgYKCwupqqqisbH72v7EiRMpLS1l3759nD17lmeeeabftbABo3w0a9Ys9cADD3geOxwOlZSUpPLz8716vd1uV+Hh4Wr37t2edStWrFALFizwtSgetbW1ClC1tbW9b/ynHyqVF6Ge/un31ZhHX1OXapr6/L5Xs6amJnXixAnV1HT1ff633npLTZ48WZlMJjVt2jRVVFSkAPXKK694tnn99dcVoG6//fYrXn/x4kU1f/58ZTKZ1JgxY9SePXvU6NGj1Y4dO7x6/87v9emnn6o777xTmc1mNWrUKLV69WpVX1+vlFKqrKxMLVy4UCUmJiqj0ajGjBmjcnNzlcPhUFarVS1ZskQlJycro9GokpKS1Lp167z+N/nBD36gYmJiFKDy8vKUUkqNGTNG/epXv7pi24cffljFxMSosLAwtXjxYvWrX/1KRUZGep7Py8tTaWlpnsdd/SZ//OMfqzvuuKPb8vT0N+Xtb9SnQLJarUqv13f4x1BKqeXLl6tvf/vbXu2jrq5Omc1m9d///d+edStWrFCRkZEqLi5Opaamqh/84Aeqqqqq2300Nzer2tpaz3L+/HnvA6ngMaXyItTeXyxXYx59TR080/37jGRXcyD5m/vv5+233x7qolzV/BFIPh2yVVVV4XA4iI+P77A+Pj6esrIyr/bx6KOPkpSUREZGhmddVlYWL7zwAoWFhWzevJl3332X+fPnd3uMnJ+fT2RkpGfx6aq15kgAEkyuBr0SaUcKOO+88w5//vOfKSkp4eDBgyxZsoSUlBRuv/32oS5awBvUXrZNmzaxb98+XnnlFcxms2f9kiVL+Pa3v83UqVNZuHAhr732Gh999BFFRUVd7icnJ4fa2lrPcv78ee8L0dqoHRtkBeBctQRSoGlpaeGxxx5jypQp3HvvvcTFxXlOkvzDH/5AWFhYl8uUKVMGrYzz58/vthy//OUvB60cg82n85BiY2PR6/WUl5d3WF9eXt7lCVrtbdmyhU2bNvH2228zbdq0HrcdN24csbGxnDlzhrlz517xvMlkwmQy+VL0di92BVKU3hVIX1RKIAWazMxMMjO7Pt/q29/+Nunp6V0+19sZ1P70+9//nqampi6fGzVq5J7M61MgGY1GZsyYQWFhIQsXLgTA6XRSWFjIunXrun3dk08+yb/+67/yxhtvdHnSVmcXLlygurq6X5OFd6tdLxtIDUl0FB4e3uv15wfDNddcM9RFGBI+H7JlZ2ezc+dOdu/ezcmTJ1m7di0Wi4VVq1YBrnFCOTk5nu03b97Mhg0beO6550hJSaGsrIyysjLP4MOGhgYefvhhPvzwQ86dO0dhYSELFixgwoQJ3f4v1i+tNaRg5Qqk0upGHE7p+hdiOPB56MjixYuprKwkNzeXsrIypk+fTkFBgaehu7S0FJ2uLee2b9+OzWbjO9/5Tof95OXlsXHjRvR6PZ9++im7d++mpqaGpKQk5s2bxxNPPNH3w7KetNaQguwNGA06bHYnF2uaSB4lk90LMdT6NJZt3bp13R6idW6IPnfuXI/7Cg4O5o03BnGEeWsNSbPWM2ZUCKcrGviiyiKBJMQwEHhj2cytAwqt9YyNcYVQSaUMshViOAi8QGqtIaGcpI5yffxzMp2tEMNC4AVSUAhorikVJkS6GrNlkK3wxUic5H+4CLxA0jRPLSklzHUm+DkJJHGV8fck/+Bq/9U0jZqaGr/u1xeBF0jg6Wm7LtQOwIXLjdjsAzvPixCidwEaSK4aUrTeSqhRj1NB6dfSjoRSYLMM/iKT/Pttkn+r1cr69eu55pprCA0NJT09vUPP95dffsk999xDdHQ0oaGhTJkyhddff51z585x5513Aq4pXjRNY+XKlX36Pvoj8KawBU9Pm2atIyU2hs8u1lFSZWHC6LBeXjjCtTTCL5MG/30fuwjG0N63ayWT/Hc/yf+6des4ceIE+/btIykpiVdeeYWsrCyOHz/OxIkTeeCBB7DZbLz33nuEhoZy4sQJwsLCSE5O5r/+679YtGgRp06dIiIiguDg4D59J/0RmIHk7mmz1pMSex2fXayTdqSriEzy3/Uk/6WlpTz//POUlpaSlOT6j2X9+vUUFBTw/PPP88tf/pLS0lIWLVrkmdJ33Lhxnte7x8iNHj3a7+1T3gr4QBoX6/ojkp42XD2Qj10cmvf1gUzy37Xjx4/jcDhITU3tsN5qtXouAvmjH/2ItWvX8uabb5KRkcGiRYt6Hew+mAK0Dant5MiU1ktrSw0JVw+kMXTwF5nk3y+T/Dc0NKDX6zly5AjHjh3zLCdPnuTXv/41APfffz9ffPEF9913H8ePH2fmzJn85je/8dtn7a8ADSR3DamOsXGuQJKJ2q4OMsm/S1eT/N900004HA4qKiqYMGFCh6X9oV1ycjI/+MEPePnll3nooYfYuXOnZ5+A1xcPGAgBGkjuGlKd55CtrK6ZRpt9CAslvCGT/Lt0Ncl/amoqy5YtY/ny5bz88suUlJRw+PBh8vPz+ctf/gLAgw8+yBtvvEFJSQlHjx7lwIEDnvcbM2YMmqbx2muvUVlZ2aGncNAM1Py6g8mnSf6VUqp4u1J5EUq9uEIppVTaz95QYx59TX32lZevHwGu5jm1ZZJ/l64m+bfZbCo3N1elpKSooKAglZiYqO6991716aefKqWUWrdunRo/frwymUwqLi5O3XfffR3mr//5z3+uEhISlKZpasWKFV6Vw80fc2prSl391wGqq6sjMjKS2traXq/GCcCxPfDqWhg/F+57mXv//QM+Ka3h35d9g29NHYBJ4Yah5uZmSkpKGDt2bIfphAPRhQsXSE5O5u233+5yhlLhnZ7+prz9jQZ8LxvA2JhQPimtkXakAPHOO+/Q0NDA1KlTuXTpEo888ohM8j9MBGgbUqdAipWG7UAik/wPXwFaQ2pr1AZIkUAKKDLJ//AV4IHUsYYk5yIJmeR/aMkhm9PpqSFVW2zUNrUMYcEG3wjo0xDDhD/+lgIzkNzT2KLA1kCYycDocNcFBQKlluQ+/GhslFkOhH+4/5b6c2gbmIdsBjPoDOC0u2pJ5ghSYkOpqLdSUmUhLTlqqEs44PR6PVFRUZ5R7iEhIX06MVAIpRSNjY1UVFQQFRWFXq/v874CM5Dcs0Y2Xfa0I42LDeVwydcB1bDtHk7Q16k3hGgvKiqq1ytY9yYwAwlcDdvtAikQe9o0TSMxMZHRo0fT0hJYbWfCv4KCgvpVM3IL7EACsNYC7XraAvDS2nq93i9/TEL0V2A2akP3J0dWWqTnSYghIoHUGkjXjQpB06DeaqfaYhvCggkRuAI3kNxd/82us7XNQXqSIl1zCAdSO5IQw0ngBlKnGhLAuLi2wzYhxODrUyBt27aNlJQUzGYz6enpHD58uNttd+7cyZw5c4iOjiY6OpqMjIwrtldKkZubS2JiIsHBwWRkZHD69Om+FM17XQSSezrbkgBs2BZiOPA5kPbv3092djZ5eXkcPXqUtLQ0MjMzuz2XpaioiKVLl3LgwAGKi4tJTk5m3rx5fPXVV55tnnzySZ555hl27NjBoUOHCA0NJTMzk+bm5r5/st506mWDjg3bQogh4NOUcEqpWbNmqQceeMDz2OFwqKSkJJWfn+/V6+12uwoPD1e7d+9WSinldDpVQkKCeuqppzzb1NTUKJPJpPbu3evVPn2eMVIppT78rWvWyP33eVa9c7JcjXn0NZX5q3e9348Qolfe/kZ9qiHZbDaOHDlCRkaGZ51OpyMjI4Pi4mKv9tHY2EhLS4tnCoWSkhLKyso67DMyMpL09PRu92m1Wqmrq+uw+KyLQ7b28yI5ndL1L8Rg8ymQqqqqcDgcxMfHd1gfHx9PWVmZV/t49NFHSUpK8gSQ+3W+7DM/P5/IyEjPkpyc7MvHcOnUywZwbXQwBp2G1e6krG4ADxeFEF0a1F62TZs2sW/fPl555ZV+zeOck5NDbW2tZzl//rzvO+mihmTQ67hulOuihdL1L8Tg8ymQYmNj0ev1lJeXd1hfXl7e66C6LVu2sGnTJt58880OV8p0v86XfZpMJiIiIjosPusikECmsxViKPkUSEajkRkzZlBYWOhZ53Q6KSwsZPbs2d2+7sknn+SJJ56goKDAcz11t7Fjx5KQkNBhn3V1dRw6dKjHffZbp1kj3QJxkK0Qw4XPg2uzs7NZsWIFM2fOZNasWWzduhWLxcKqVasAWL58Oddccw35+fkAbN68mdzcXPbs2UNKSoqnXcg9YbmmaTz44IP84he/YOLEiYwdO5YNGzaQlJTEwoUL/fdJO3MHkq0enA7QuQaXynS2QgwdnwNp8eLFVFZWkpubS1lZGdOnT6egoMDTKF1aWopO11bx2r59Ozabje985zsd9pOXl8fGjRsBeOSRR7BYLKxZs4aamhpuu+02CgoKBvZ6YaZ28ybbGsAcCcghmxBDKTAvFAmgFDwRB84W+JfPIPJaAC7WNHHLpncw6DQ+fyILgz5wR9cI4S/e/kYD99emaV12/SdEmDEZdNidiguXu74MjRBiYARuIEGXPW06ndZ22CZj2oQYVBJIcGVPW4yMaRNiKAR4ILkastsPsAUYGxe409kKMZQCPJC6OTkyRnrahBgKEkhwZSDFSSAJMRQCO5DM3Zyt3VpD+qqmieYWx2CXSoiAFdiB5K4hNXecviQ2zEi4yYBScP5rudS0EINFAgmuqCFpmuYZ0/aFHLYJMWgCPJDch2xXTvAmY9qEGHwSSNBlIMmofyEGX4AHUteHbADjJJCEGHQSSNBlIEkNSYjBF9iB1MXgWjf3yZEV9VYsVvtglkqIgBXYgdRDDSkyJIhRoUZAhpAIMVgCPJBaa0gtFteskZ3ER7gmiKustw5mqYQIWAEeSO1mjeyipy02zFVDqmqwDVaJhAhogR1IBhPoTa77XRy2xYa5nqtukBqSEIMhsAMJemxHaqshSSAJMRgkkLoZYAsQ01pDkkM2IQaHBFI3A2yh7ZBNakhCDA4JpB6Gj0ijthCDSwKpxzYkadQWYjBJIPVYQ2oNJIsNp/Oqv3ydEMOeBFIPNST3mdoOp6K2qWUwSyVEQJJA6iGQjAYdkcFBgDRsCzEYJJB6GGALECMN20IMGgkkTw2p60CSrn8hBk+fAmnbtm2kpKRgNptJT0/n8OHD3W772WefsWjRIlJSUtA0ja1bt16xzcaNG9E0rcMyadKkvhTNd6buT4wEOVtbiMHkcyDt37+f7Oxs8vLyOHr0KGlpaWRmZlJRUdHl9o2NjYwbN45NmzaRkJDQ7X6nTJnCpUuXPMv777/va9H6ptdAcnf9yyGbEAPN50B6+umnWb16NatWreKGG25gx44dhISE8Nxzz3W5/c0338xTTz3FkiVLMJlM3e7XYDCQkJDgWWJjY30tWt/IIZsQw4ZPgWSz2Thy5AgZGRltO9DpyMjIoLi4uF8FOX36NElJSYwbN45ly5ZRWlra7bZWq5W6uroOS5/10MsG0qgtxGDyKZCqqqpwOBzEx8d3WB8fH09ZWVmfC5Gens6uXbsoKChg+/btlJSUMGfOHOrruw6J/Px8IiMjPUtycnKf37unwbUgNSQhBtOw6GWbP38+3/3ud5k2bRqZmZm8/vrr1NTU8OKLL3a5fU5ODrW1tZ7l/PnzfX9zz6yRjeC48uRHd6N2tUUCSYiBZvBl49jYWPR6PeXl5R3Wl5eX99hg7auoqChSU1M5c+ZMl8+bTKYe26N8Ygxru2+th5BRHZ721JDq5ZBNiIHmUw3JaDQyY8YMCgsLPeucTieFhYXMnj3bb4VqaGjg7NmzJCYm+m2f3TIYweCaO7unOZGaWhw02uTqI0IMJJ8P2bKzs9m5cye7d+/m5MmTrF27FovFwqpVqwBYvnw5OTk5nu1tNhvHjh3j2LFj2Gw2vvrqK44dO9ah9rN+/Xreffddzp07x8GDB7n33nvR6/UsXbrUDx/RCz0MsA016jEHub4mqSUJMbB8OmQDWLx4MZWVleTm5lJWVsb06dMpKCjwNHSXlpai07Xl3MWLF7nppps8j7ds2cKWLVu44447KCoqAuDChQssXbqU6upq4uLiuO222/jwww+Ji4vr58fzkikcLBVd1pA0TSM2zMSFy01UNli5LiZkcMokRADyOZAA1q1bx7p167p8zh0ybikpKSjV89Qd+/bt60sx/KfXrn9XIMm8SEIMrGHRyzbkehlgGyfnIgkxKCSQoMc2JICYUJk5UojBIIEEvR6yxYbLAFshBoMEEvQ6wNZdQ6qyyCGbEANJAgl6H2Ab7j45UmpIQgwkCSTo/ZAt1D18RGpIQgwkCSTofYBtuAywFWIwSCBBWxtSc22XT7vHs9U0ttDicA5WqYQIOBJI0OshW1RwEHqdBsDXctgmxICRQIJeA0mn0zzXaKuUhm0hBowEEvR6YiRAjDRsCzHgJJCg1xoSQJx0/Qsx4CSQoC2Q7M1g77oG1FZDkkASYqBIIEHbIRuAraHLTdrm1pZDNiEGigQSgN4AQa3zHHXT9R8jk/0LMeAkkNx6O1tbpiARYsBJILn1dgVbadQWYsBJILn1NsDWPSeSNGoLMWAkkNy8nBOpusGG09nzlLxCiL6RQHIz93xypPtMbbtTUdd85QUlhRD9J4HkZup5Xm2TQU+E2XVNBOlpE2JgSCC5eXG2tpyLJMTAkkBy8ymQpIYkxECQQHLzZoBtWFvDthDC/ySQ3KSGJMSQk0Byk0ASYshJILmZI123XhyySaO2EANDAsnNXUPqptsfpIYkxEDrUyBt27aNlJQUzGYz6enpHD58uNttP/vsMxYtWkRKSgqaprF169Z+73NAeHXIJo3aQgwknwNp//79ZGdnk5eXx9GjR0lLSyMzM5OKioout29sbGTcuHFs2rSJhIQEv+xzQPQyuBakhiTEQPM5kJ5++mlWr17NqlWruOGGG9ixYwchISE899xzXW5/880389RTT7FkyRJMJpNf9jkg3DUkhxXsXQeOuw2p0eag0WYfrJIJETB8CiSbzcaRI0fIyMho24FOR0ZGBsXFxX0qQF/2abVaqaur67D0mzuQoNtaUpjJgMng+srksE0I//MpkKqqqnA4HMTHx3dYHx8fT1lZWZ8K0Jd95ufnExkZ6VmSk5P79N4d6PQQFOq6301Pm6ZpctgmxAC6KnvZcnJyqK2t9Sznz5/3z47NPQ+wBZk5UoiBZPBl49jYWPR6PeXl5R3Wl5eXd9tgPRD7NJlM3bZH9YspHOovScO2EEPEpxqS0WhkxowZFBYWetY5nU4KCwuZPXt2nwowEPvsMy+6/tvGs0kgCeFvPtWQALKzs1mxYgUzZ85k1qxZbN26FYvFwqpVqwBYvnw511xzDfn5+YCr0frEiROe+1999RXHjh0jLCyMCRMmeLXPQeNT178csgnhbz4H0uLFi6msrCQ3N5eysjKmT59OQUGBp1G6tLQUna6t4nXx4kVuuukmz+MtW7awZcsW7rjjDoqKirza56DpZV5tkMshCTGQfA4kgHXr1rFu3boun3OHjFtKSgpK9T4HdU/7HDReTEHS1qgtgSSEv12VvWwDxuz9IZuchySE/0kgtScDbIUYUhJI7fnQy3a5sQW7wzkYpRIiYEggtedFL1t0iBGd5rr/tUUO24TwJwmk9rzoZdPrNEa1XsW2Ug7bhPArCaT2vOhlA5kXSYiBIoHUnhdtSCAN20IMFAmk9rwYXAtyOSQhBooEUnvta0g9nMwpNSQhBoYEUnvuQHK2dDtrJMjVR4QYKBJI7Rl7nzUSpIYkxECRQGpPp2sLJS/Gs1VbJJCE8CcJpM68OBfJU0Oql0M2IfxJAqkzXwbYWqxezWQghPCOBFJnXgywHRXqOmRrcSjqmuRySEL4iwRSZ16cHGkO0hNudk0lJcNHhPAfCaTOvBhgC+3nRZJAEsJfJJA689SQanvcTC6HJIT/SSB15mUNKSa0rWFbCOEfEkideTvANry1hlQvgSSEv0ggdebtANvWGlKVTNImhN9IIHXmdQ3JfXKk1JCE8BcJpM68DKQ4uRySEH4ngdSZt43anrO15ZBNCH+RQOrME0i9dfvLIZsQ/iaB1JmXh2zuOZEsNgdNNsdAl0qIgCCB1Fn7wbU9DJwNNxkwGlxfn7QjCeEfEkideWaNtENLU7ebaZpGbKh7XiRpRxLCH/oUSNu2bSMlJQWz2Ux6ejqHDx/ucfuXXnqJSZMmYTabmTp1Kq+//nqH51euXImmaR2WrKysvhSt/4JCgdYrQUrXvxCDyudA2r9/P9nZ2eTl5XH06FHS0tLIzMykoqKiy+0PHjzI0qVL+f73v88nn3zCwoULWbhwIX/72986bJeVlcWlS5c8y969e/v2ifpLp/Nh+IjMHCmEP/kcSE8//TSrV69m1apV3HDDDezYsYOQkBCee+65Lrf/9a9/TVZWFg8//DCTJ0/miSee4Bvf+AbPPvtsh+1MJhMJCQmeJTo6um+fyB+8HmDrnltbDtmE8AefAslms3HkyBEyMjLadqDTkZGRQXFxcZevKS4u7rA9QGZm5hXbFxUVMXr0aK6//nrWrl1LdXV1t+WwWq3U1dV1WPzKx7O1K+WQTQi/8CmQqqqqcDgcxMfHd1gfHx9PWVlZl68pKyvrdfusrCxeeOEFCgsL2bx5M++++y7z58/H4ei6Oz0/P5/IyEjPkpyc7MvH6J23Xf/SqC2EXxmGugAAS5Ys8dyfOnUq06ZNY/z48RQVFTF37twrts/JySE7O9vzuK6uzr+h5MW82gBx0qgthF/5VEOKjY1Fr9dTXl7eYX15eTkJCQldviYhIcGn7QHGjRtHbGwsZ86c6fJ5k8lEREREh8WvvJhXG2ROJCH8zadAMhqNzJgxg8LCQs86p9NJYWEhs2fP7vI1s2fP7rA9wFtvvdXt9gAXLlygurqaxMREX4rnP77OiSSN2kL4hc+9bNnZ2ezcuZPdu3dz8uRJ1q5di8ViYdWqVQAsX76cnJwcz/Y//vGPKSgo4N/+7d/4/PPP2bhxIx9//DHr1q0DoKGhgYcffpgPP/yQc+fOUVhYyIIFC5gwYQKZmZl++pg+8nT7e1dDutxow+5wDnSphBjxfG5DWrx4MZWVleTm5lJWVsb06dMpKCjwNFyXlpai07Xl3C233MKePXt4/PHHeeyxx5g4cSKvvvoqN954IwB6vZ5PP/2U3bt3U1NTQ1JSEvPmzeOJJ57AZDL56WP6yMtAGhVqRNNcI0y+brQxOtw8CIUTYuTS1Ai40mFdXR2RkZHU1tb6pz2p+N/hjRy4cRF8p+vzq9xmPPEW1RYb//PjOUxO9HNblhAjhLe/URnL1hUve9mg/cmR0rAtRH9JIHXFy142aN+wLYEkRH9JIHXFy142aNf1Lz1tQvSbBFJXTJGuWx8O2eSS2kL0nwRSV7wcXAttM0dKDUmI/pNA6kr7Q7ZeOiHjpFFbCL+RQOqKO5CUE1oae9xUakhC+I8EUleMoaC1fjW9DR+RGpIQfiOB1BVN87rr3z0nUnWDjRFwjqkQQ0oCqTs+TmNrczipa7YPdKmEGNEkkLrj5Xg2c5CecJNrSKActgnRPxJI3fH0tPV+trY0bAvhHxJI3fHhbG1p2BbCPySQuuPDANu2GpIEkhD9IYHUHV8G2HqGj8ghmxD9IYHUHZ/akNxd/1JDEqI/JJC642W3P0BcmExBIoQ/SCB1x4dAig2TKUiE8AcJpO704ZBNakhC9I8EUnd86vb38XJIzbVw9gDYpUYlRHvD4sq1w5JP3f6uGlKD1U5ziwNzkL7rDZWCz16GghxoKIeEqbBwByTc6K9SC3FVkxpSd3zo9o8wGzDqXV9lt4dtX5fAH74Df/y/rjACKDsOv/tHeO8pcMg4OCEkkLrjQ6O2pmndDx9xtMD/Pg3//g9w5m3QG+EfH4MHj8P13wJnC7zzC/h/GVDxub8/xeCzW+HEn+H4H8HW81xSQnQmh2zdaT+4VinXlCQ9iA0zcam2mQf3H+PO60czJzWWWwynMRU8BJUnXRulzIG7fwWxE12Pl+yBT1+E/3kYLn4Cv50Dd/4Ubvkh6Lo57BuuLp+Dj5+HT/4TGqtc60yRMH0pzFgFoycNafGGrabLUHUGqk9D9RnXf2DBUWCOar2NBHN02zpzJOhH7s925H6y/nIfsqHA1tDucdfuSUvks4u1lFRZqK46zoRDezEZDgDQFBTF17dtJGnOCrR2V/VF0yBtMYy9Hf77R3D6TXg7Dz5/DRZubwuugaCU68dgDAVDH68Q7HTA39+Aj/8fnCkEWueDCk8CfRDUfAmHdriWMbfCzP8Lk+/p+/sNJUs1VH7u+s+l+gtXKLgDon1wtH+sD3K91mF3fRdVp6Hq767wcYeQpdL3shjDXe8RMgrC4l1LeELX94Ourqspy5Vru6MU/DwGlAOyT0JEUq8vqbXYKCnaxfhPfkm4/TIA++z/yCb7UmoIJy7cxJyJsdyRGseMMdE0tzi53GijusHG1w1W4r/4I7ee+TfMTgs2zcju4OU8b8+ittlBqMlAVEgQUSFGokOCiAo2EhUaRHTr48jg1vUhRpxKYbM7sTmc2OxOHJYqTF//HXPNaUJr/k543RkiG85gbqnBoRlojErFmZCGKfkmzNfNgPgpEBTc/QetL4OjL8CR3VB3oW39+G/CzO9DapZrxs0v3nHVmk79j+t7BAiJgZv+D8xYCaPG9eMfaIBYqqDiZGv4fA6Vp1yP3bU+XwSFuv4ja6x2HZp3JzwJYidAzETX995UA801V97aGnwvgzkSwhIgbDSExrVbYl23YaPb7hvDrjwScDpcRwnNda23tVfeH/9NuHZGj8Xw9jcqgdSTzSmuWsT4uRAc7fqfXW9svQ0Cvanjur+/AV+4akUq9nq+ui2fNxvG8b+nK/nwi69panH0+paJVLM56Hfcrj8OwCHnJB5pWcOXKqHdVgojdszYMNGCSbNhxrWEYmWs7hKp2gVStfOk6r5itFbj08d2oONi0BgqwibRED0FlZiGOXkaifUniPn8PwkteQPN6WqEV8GjYPr/Qbt5VfcBU/sVfPIfrgCrv9i2fvw3XbWm1CzXoYq13vWjs9aBtaHrxy1NrrnOUa5bz9LuMartsdPRut7R7r6z3XOt61qaXTWWxuruv5io6yBusqvmqlTrD7LGdesOjebars9dMwS3hU7sxHa3E8AU5uU/TEvH97JUuTpI3Et9Wbv75eDw8bw4g7ktmKz1rvey9d6Gyrx/hVvW9biJBJI//PYOuHTMt9cYzHD7w3DLj8Bg9Ky22h18fO4y752u5L2/V3HyUh3hJgOjwoyMCjUSE+q6HRVqYlSIgRnVfybtxBYMdgtOvRmHOQrsVjR7M3pHMxq+/bNV6EZzISiFMlMKZeZxVAaP4+vgMTgbqoj4+m/EN55iouMsN+pKiNN671n8yJnKf9ozKHDOwooRo16HyaDDaHDdBhv1hJuDiAgOIsJsINwcRKQZplkOkVb+MklVB33+DIMmagyMngxx17sCKO5612IM9e71DntrDaLG9aMOiYWIa0A3iH1ISrnev6GiNagqXLU8S6XrvqX1vqXSdb/F0vP+DGZXbcsU0Xp4GtF2/4YFMGFujy8f0EDatm0bTz31FGVlZaSlpfGb3/yGWbNmdbv9Sy+9xIYNGzh37hwTJ05k8+bNfOtb3/I8r5QiLy+PnTt3UlNTw6233sr27duZONG7NpQBC6Sa864aj90KDlvrbYvrfx73OofNdYKjw+r6B7rlhxAzvtddO50Kna7nhnIufwl/egDO/W8PG2muar7B3HYbPQZG3wBxk9p+WL20gQFYrHYu1TRSdelLWi4cxVB+nIiaz0i0nGKUsxoLZl7jdv7gmMtxe3JvV4jqUbJWzlL9Ab6nLyK2NQCdSqMBMxaCaVDBWFrvN2rBNOlCsWrBtOiD0ev1GPR6DAY9Br0Bo8F1P8igJ0ivJyjItc5oMBBiNhJqNhJqMmIMMrgOJXV60PTt7utAZ4BRYyE21fvgGUlsltaQqmprMzVHtoVQu/9c+2LAAmn//v0sX76cHTt2kJ6eztatW3nppZc4deoUo0ePvmL7gwcPcvvtt5Ofn8/dd9/Nnj172Lx5M0ePHuXGG10nBG7evJn8/Hx2797N2LFj2bBhA8ePH+fEiROYzb03yg1YIA0HTidUnHC1QRiCXY2Unluz63Cxlx5Av7BUuX6orW1LSilaHAqbw4m1xYHV7mqvstqdNLc4aLQ5qG9uoa7ZTl1TC/XNduqaWzrcr2+2Y2lqIsh6mcsOE3V2AwM5LXmoUU9cuInYsNYl3Nh2P8xEhNlAqMlAmNlAmMm1hBj1aIPx/Y5wAxZI6enp3HzzzTz77LMAOJ1OkpOT+eEPf8hPfvKTK7ZfvHgxFouF1157zbPuH/7hH5g+fTo7duxAKUVSUhIPPfQQ69evB6C2tpb4+Hh27drFkiVL/PZhxfDnDjqr3eFpmLe2tDXQN7U4sFjtWKyttzY7FqudBvdjzzoHdc0tVDfYqGywYrM7+1QeTYNQoyucQk361lsDep2Gpmlordu4btseg+ZZb9BrGHQ6gvQ6gvSa57HRoMOg0zDodRj1rlsNcCpwKoXTqTz3lVI4VPvHru9K1/omOk1Dp4GG65bWx7rWMul0Ggadht59q9d1fKxzlUnfet/zuVr36foyOn5O9/czNjaUa6NDevwevf2N+tTtb7PZOHLkCDk5OZ51Op2OjIwMiouLu3xNcXEx2dnZHdZlZmby6quvAlBSUkJZWRkZGRme5yMjI0lPT6e4uLjLQLJarVitbQ12dXW9t3mIq4OmaRgNGkaD/9pblFLUW+1U1VuparBR1WClqsFKZb371ka1xUpDsyvQ6luDzdnaNt5gtdNglTPpu/P4XZO5f45/ekx9CqSqqiocDgfx8fEd1sfHx/P5512fZVxWVtbl9mVlZZ7n3eu626az/Px8fvazn/lSdBHANE0jwhxEhDmIcXHevUYpRXOLk3prCxarg4ZmuyeYLFY7DqdCtW6nwNWxR2vNhdZOvtbHDqeixeHE7lTYHU5sDtet3b3e4bptcbgOVtw1G53OVXadBnpNa73fWvNpbX9UrbUm1VpzgrYalFO1ldHhVDic4HC63tfhVJ1uXeVwP3Z9JnXFZ8HzGM91CEeF9q99qb2r8sTInJycDrWuuro6kpOTh7BEYqTRNI1go55gox567w8QfuJTvTg2Nha9Xk95eXmH9eXl5SQkJHT5moSEhB63d9/6sk+TyURERESHRQhx9fMpkIxGIzNmzKCwsNCzzul0UlhYyOzZs7t8zezZsztsD/DWW295th87diwJCQkdtqmrq+PQoUPd7lMIMUIpH+3bt0+ZTCa1a9cudeLECbVmzRoVFRWlysrKlFJK3XfffeonP/mJZ/sPPvhAGQwGtWXLFnXy5EmVl5engoKC1PHjxz3bbNq0SUVFRak//elP6tNPP1ULFixQY8eOVU1NTV6Vqba2VgGqtrbW148jhBgE3v5GfQ4kpZT6zW9+o6677jplNBrVrFmz1Icffuh57o477lArVqzosP2LL76oUlNTldFoVFOmTFF/+ctfOjzvdDrVhg0bVHx8vDKZTGru3Lnq1KlTXpdHAkmI4c3b36gMHRFCDDhvf6MyQZsQYtiQQBJCDBtX5XlInbmPOuWMbSGGJ/dvs7cWohERSPX1rjlb5ORIIYa3+vp6IiMju31+RDRqO51OLl68SHh4eK8js91ndZ8/f35ENoCP5M83kj8bjOzPp5Sivr6epKQkdD3MCzUiakg6nY5rr73Wp9eM9DO8R/LnG8mfDUbu5+upZuQmjdpCiGFDAkkIMWwEXCCZTCby8vIwma7CS/F4YSR/vpH82WDkfz5vjIhGbSHEyBBwNSQhxPAlgSSEGDYkkIQQw4YEkhBi2JBAEkIMGwEXSNu2bSMlJQWz2Ux6ejqHDx8e6iL128aNG13Xymq3TJo0aaiL1Wfvvfce99xzD0lJSWia5rlklptSitzcXBITEwkODiYjI4PTp08PTWH7oLfPt3Llyiv+PbOysoamsIMsoAJp//79ZGdnk5eXx9GjR0lLSyMzM5OKioqhLlq/TZkyhUuXLnmW999/f6iL1GcWi4W0tDS2bdvW5fNPPvkkzzzzDDt27ODQoUOEhoaSmZlJc3PzIJe0b3r7fABZWVkd/j337t07iCUcQgM6b+UwM2vWLPXAAw94HjscDpWUlKTy8/OHsFT9l5eXp9LS0oa6GAMCUK+88ornsdPpVAkJCeqpp57yrKupqVEmk0nt3bt3CErYP50/n1JKrVixQi1YsGBIyjPUAqaG5L7qbvsr5PZ21d2ryenTp0lKSmLcuHEsW7aM0tLSoS7SgOjtSscjRVFREaNHj+b6669n7dq1VFdXD3WRBkXABFJPV93t7gq5V4v09HR27dpFQUEB27dvp6SkhDlz5njmiRpJ+nKl46tNVlYWL7zwAoWFhWzevJl3332X+fPn43A4hrpoA25ETD8S6ObPn++5P23aNNLT0xkzZgwvvvgi3//+94ewZKIvlixZ4rk/depUpk2bxvjx4ykqKmLu3LlDWLKBFzA1pL5cdfdqFRUVRWpqKmfOnBnqovhdX650fLUbN24csbGxI/Lfs7OACaS+XHX3atXQ0MDZs2dJTEwc6qL4XSBe6fjChQtUV1ePyH/PzgLqkC07O5sVK1Ywc+ZMZs2axdatW7FYLKxatWqoi9Yv69ev55577mHMmDFcvHiRvLw89Ho9S5cuHeqi9UlDQ0OH2kBJSQnHjh1j1KhRXHfddTz44IP84he/YOLEiYwdO5YNGzaQlJTEwoULh67QPujp840aNYqf/exnLFq0iISEBM6ePcsjjzzChAkTyMzMHMJSD5Kh7uYbbD1ddfdqtXjxYpWYmKiMRqO65ppr1OLFi9WZM2eGulh9duDAAQVcsbiviNzfKx0PtZ4+X2Njo5o3b56Ki4tTQUFBasyYMWr16tWeS9WPdDIfkhBi2AiYNiQhxPAngSSEGDYkkIQQw4YEkhBi2JBAEkIMGxJIQohhQwJJCDFsSCAJIYYNCSQhxLAhgSSEGDYkkIQQw8b/B6ZwzqvSaRAhAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T08:56:33.897210Z", + "iopub.status.busy": "2024-02-29T08:56:33.896912Z", + "iopub.status.idle": "2024-02-29T08:58:31.058373Z", + "shell.execute_reply": "2024-02-29T08:58:31.057525Z" + }, + "papermill": { + "duration": 117.184695, + "end_time": "2024-02-29T08:58:31.061134", + "exception": false, + "start_time": "2024-02-29T08:56:33.876439", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T08:58:31.109014Z", + "iopub.status.busy": "2024-02-29T08:58:31.108022Z", + "iopub.status.idle": "2024-02-29T08:58:31.132126Z", + "shell.execute_reply": "2024-02-29T08:58:31.131158Z" + }, + "papermill": { + "duration": 0.051029, + "end_time": "2024-02-29T08:58:31.134247", + "exception": false, + "start_time": "2024-02-29T08:58:31.083218", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.05.660651e-070.002434.4524830.0067640.0865570.0094920.0000042.3814860.037970.0721310.0492990.0672590.0188956.83397
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.0 5.660651e-07 0.00243 4.452483 0.006764 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.086557 0.009492 0.000004 2.381486 0.03797 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.072131 0.049299 0.067259 0.018895 6.83397 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T08:58:31.178297Z", + "iopub.status.busy": "2024-02-29T08:58:31.177525Z", + "iopub.status.idle": "2024-02-29T08:58:31.768325Z", + "shell.execute_reply": "2024-02-29T08:58:31.767544Z" + }, + "papermill": { + "duration": 0.615409, + "end_time": "2024-02-29T08:58:31.770452", + "exception": false, + "start_time": "2024-02-29T08:58:31.155043", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T08:58:31.813459Z", + "iopub.status.busy": "2024-02-29T08:58:31.812611Z", + "iopub.status.idle": "2024-02-29T09:00:36.493262Z", + "shell.execute_reply": "2024-02-29T09:00:36.492410Z" + }, + "papermill": { + "duration": 124.70489, + "end_time": "2024-02-29T09:00:36.495805", + "exception": false, + "start_time": "2024-02-29T08:58:31.790915", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T09:00:36.541488Z", + "iopub.status.busy": "2024-02-29T09:00:36.540633Z", + "iopub.status.idle": "2024-02-29T09:00:36.558330Z", + "shell.execute_reply": "2024-02-29T09:00:36.557386Z" + }, + "papermill": { + "duration": 0.043043, + "end_time": "2024-02-29T09:00:36.560412", + "exception": false, + "start_time": "2024-02-29T09:00:36.517369", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T09:00:36.599714Z", + "iopub.status.busy": "2024-02-29T09:00:36.599114Z", + "iopub.status.idle": "2024-02-29T09:00:36.604296Z", + "shell.execute_reply": "2024-02-29T09:00:36.603451Z" + }, + "papermill": { + "duration": 0.026875, + "end_time": "2024-02-29T09:00:36.606315", + "exception": false, + "start_time": "2024-02-29T09:00:36.579440", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.566654310339973}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T09:00:36.648618Z", + "iopub.status.busy": "2024-02-29T09:00:36.647838Z", + "iopub.status.idle": "2024-02-29T09:00:37.048270Z", + "shell.execute_reply": "2024-02-29T09:00:37.047149Z" + }, + "papermill": { + "duration": 0.42428, + "end_time": "2024-02-29T09:00:37.050489", + "exception": false, + "start_time": "2024-02-29T09:00:36.626209", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDcUlEQVR4nO3deXRU9f0//uedfZLMkn2BLBDCvq8iIqAoAqLUlvopVEGKS8W6UM7R1FO0uEA9VaGtxdYqwa8oVn9orQuKVqCoIIIgiAZCFrInZJnMklnv+/fHnZlkSCbJTCZzZzKvxzlDMnfu3HlNmLzy3t8cY4yBEEJEJBE7AEIIoURECBEdJSJCiOgoERFCREeJiBAiOkpEhBDRUSIihIiOEhEhRHSUiAghoqNERHpUVFQEjuNQXl4udihkEKNERELu9ddfx7Zt28QOg0QRSkQk5CgRkUBRIiKEiI4SEQnYRx99hHnz5kGj0UCr1WLGjBl4/fXXAQDz58/HBx98gIqKCnAcB47jkJeX1+dr8zyPxx9/HFlZWYiLi8OCBQtw9uxZ5OXlYc2aNd7zmpubsXHjRkyYMAEJCQnQarVYvHgxTp065XO9AwcOgOM4/Otf/8JTTz2FoUOHQqVS4dprr0VJSUkofhwkBGRiB0CiS1FREdauXYtx48ahsLAQer0e3377Lfbt24eVK1fi0UcfhcFgQFVVFZ5//nkAQEJCQp+vX1hYiGeeeQbLli3DokWLcOrUKSxatAhWq9XnvNLSUrz77rtYsWIFhg0bhvr6evz973/HvHnzcPbsWWRlZfmcv3XrVkgkEmzcuBEGgwHPPPMMVq1ahaNHj/b/h0L6jxHSg507dzIArKysjLW2tjKNRsNmzZrF2tvbfc7jed77/dKlS1lubm7Ar1VXV8dkMhlbvny5z/HHH3+cAWCrV6/2HrNarczlcvmcV1ZWxpRKJdu8ebP32Oeff84AsDFjxjCbzeY9vn37dgaAnT59OuA4SehR1Yz02f79+2E0GvHII49ApVL5PMZxXL+v/9lnn8HpdOLee+/1Of6b3/ymy7lKpRISifDxdblcaGpqQkJCAkaNGoUTJ050Of+OO+6AQqHw3p87dy4AoWRFxEeJiPTZhQsXAADjx48fkOtXVFQAAEaMGOFzPCkpCYmJiT7HeJ7H888/j4KCAiiVSqSkpCA1NRXfffcdDAZDl2vn5OT43Pdcr6WlJZRvgQSJEhGJSk8//TQ2bNiAq6++Gq+99ho+/vhj7N+/H+PGjQPP813Ol0ql3V6H0UrJEYEaq0mf5efnAwDOnDnTpdTSWbDVtNzcXABASUkJhg0b5j3e1NTUpeTy9ttvY8GCBXj55Zd9jre2tiIlJSWo1yfioRIR6bPrr78eGo0GW7Zs6dKL1blkER8f3231qDfXXnstZDIZduzY4XP8r3/9a5dzpVJpl9LMW2+9herq6oBfl4iPSkSkz7RaLZ5//nmsW7cOM2bMwMqVK5GYmIhTp07BYrFg165dAIBp06bhzTffxIYNGzBjxgwkJCRg2bJlvV4/PT0dDzzwAJ599lncdNNNuOGGG3Dq1Cl89NFHSElJ8Slp3Xjjjdi8eTPuuOMOXHnllTh9+jR2796N4cOHD9j7JwNI5F47EuE6d997vPfee+zKK69karWaabVaNnPmTPbGG294HzeZTGzlypVMr9czAAF15TudTvb73/+eZWRkMLVaza655hr2ww8/sOTkZHbPPfd4z7Narey3v/0ty8zMZGq1ms2ZM4d99dVXbN68eWzevHne8zzd92+99ZbP65SVlTEAbOfOnYH+SMgA4Bij1joS2VpbW5GYmIgnn3wSjz76qNjhkAFAbUQkorS3t3c55plAO3/+/PAGQ8KG2ohIWDQ2NsLlcvl9XKFQICkpCW+++SaKioqwZMkSJCQk4PDhw3jjjTdw/fXXY86cOWGMmIQTJSISFjNmzPAOWOzOvHnzcODAAUycOBEymQzPPPMM2travA3YTz75ZBijJeFGbUQkLL744otuq10eiYmJmDZtWhgjIpGEEhEhRHTUWE0IEV1UtxHxPI+amhpoNJqQzP4mhIQWYwxGoxFZWVne1RL8nSia3NxcBqDL7d577+3T8ysrK7t9Pt3oRrfIulVWVvb4uyxqiejYsWM+XbpnzpzBddddhxUrVvTp+RqNBgBQWVkJrVY7IDESQoLX1taG7Oxs7++qP6ImotTUVJ/7W7duRX5+PubNm9en53uqY1qtlhIRIRGst6aTiGkjstvteO2117Bhwwa/QdtsNthsNu/9tra2cIVHCBlAEdNr9u6776K1tdVnp4bLbdmyBTqdznvLzs4OX4CEkAETMeOIFi1aBIVCgf/85z9+z+muRJSdnQ2DwUBVM0IiUFtbG3Q6Xa+/oxFRNauoqMCnn36KvXv39nieUqmEUqkM6NqMMTidzh7nOZHuSaVSyGQyGhpBBlxEJKKdO3ciLS0NS5cuDel17XY7amtrYbFYQnrdWBIXF4fMzEyfHTAICTXRExHP89i5cydWr14NmSx04fA8j7KyMkilUmRlZUGhUNBf9gAwxmC329HY2IiysjIUFBT0PCCNkH4QPRF9+umnuHjxItauXRvS69rtdvA8j+zsbMTFxYX02rFCrVZDLpejoqICdru9y15mhISK6Ino+uuvH9AtXeiveP8M9p+f1eHCd1UGTM9NhERCJWaxDO5PGSHdOHSuEVUtFvA8wxtfX8QXJZdQXG8UO6yYRomIxBSj1YETF1vw1jdVsDhcGJMpdCmfqmwVN7AYR4koxuXl5XnXhI4FP9QawRgwJFGNBKUME4bowHFArcEKQ7tD7PBiFiUiElPKLpkAAGMyhJJQvFKGLL0aAFDaaBItrlhHiWgQsNvtYocQFawOF2oNwg61uSkdPan5qfEAgLJLZlHiIjGaiOxO3u/N6eL7fK6jD+cGY/78+bjvvvtw3333QafTISUlBb///e+9vYt5eXl44okncPvtt0Or1eKuu+4CABw+fBhz586FWq1GdnY27r//fpjNHb9cDQ0NWLZsGdRqNYYNG4bdu3cHFV+0qmqxgDEgKV4BrUruPZ6bLCSimtZ2uPiImPEUc0TvvhfDC5+X+H1sWEo8lk8Z4r3/j0MX4HB1/+EcmqjGiukdE29f+aIM7XbfqSQPXTcyqBh37dqFX/3qV/j666/xzTff4K677kJOTg7uvPNOAMCf/vQnbNq0CY899hgA4MKFC7jhhhvw5JNP4pVXXkFjY6M3me3cuRMAsGbNGtTU1ODzzz+HXC7H/fffj4aGhqDii0bVrUJpKDtJ7XM8OV4BlVwKm9OFZrMdqZrAphGR/ovJRBQNsrOz8fzzz4PjOIwaNQqnT5/G888/701E11xzDX772996z1+3bh1WrVqFBx98EABQUFCAP//5z5g3bx527NiBixcv4qOPPsLXX3+NGTNmAABefvlljBkzJuzvTSytFqEKm6H1TUQcx+GWqUOgj5NDKZOKEVrMi8lEtH7BCL+PXT6m7a6r8/2ee/mMkbVzhvUnLB9XXHGFz5SU2bNn49lnn/VO3p0+fbrP+adOncJ3333nU91ijHmnupw7dw4ymcxny57Ro0dDr9eHLOZId9OkLLRZnVDKurZIpGtp1LiYYjIRKbr5IIb73P6Kj4/3uW8ymXD33Xfj/vvv73JuTk4Ozp07F67QIhbHcdCp5b2fSMIuJhNRNDh69KjP/SNHjqCgoABSafdVh6lTp+Ls2bMYMaL70t7o0aPhdDpx/Phxb9WsuLgYra2tIY07WvE8w6HzjWg02rBsUhZUcqqihVNM9ppFg4sXL2LDhg0oLi7GG2+8gb/85S944IEH/J7/8MMP48svv8R9992HkydP4vz58/j3v/+N++67DwAwatQo3HDDDbj77rtx9OhRHD9+HOvWrYNarfZ7zcHkSGkT3jtVg4qm7rvoJRIOJQ0mVLW045LJ1u05ZOBQIopQt99+O9rb2zFz5kysX78eDzzwgLebvjsTJ07EwYMHce7cOcydOxdTpkzBpk2bkJWV5T1n586dyMrKwrx583DLLbfgrrvuQlpaWjjejuguNltwocEEi93/Anme3rIGIyWicKOqWYSSy+XYtm0bduzY0eWx8vLybp8zY8YMfPLJJ36vmZGRgffff9/n2G233davOKMBY8xbyklO8L/AW5pGhdJGMxopEYUdlYjIoGeyOWFz8JBwHJLi/CciT5JqNtNI9XCjREQGvSaTkFgS4+WQSf1/5JPiOxJRhOwpETOoahaBDhw4IHYIg0qT2V0ti+95xLReLYeE42B38jDZnNCoqKs/XKhERAa9RqNQIuqpfQgAZFIJdGoZ4hTSHhu1SehRiYgMeowxyKUcUnpJRACwclZuWAemEgElIjLoLZ6QCcYY+tLsQ0lIHKL/1Kurq/HLX/4SycnJUKvVmDBhAr755huxwyKDDMdxtDh+BBO1RNTS0oI5c+ZgwYIF+Oijj5Camorz588jMTFRzLBIDGuzOvDFie/AWdtww9VXAjJaEiQcRE1Ef/zjH5Gdne1dLwcAhg0L3Qx2QorrjPi6vBkjUhMwOz+51/NVFQeh/2EfAMCl/B7SabcBKt1AhxnzRK2avffee5g+fTpWrFiBtLQ0TJkyBS+99JLf8202G9ra2nxuhPSkyWzDJaMNFruz95PrzkBRcwwyCQenRAmbqQU49/HAB0nETUSlpaXYsWMHCgoK8PHHH+PXv/417r//fuzatavb87ds2QKdTue9ZWdnd3seIR4Gi7Azhz6ulzFBTjtQ+rnwnPQrcCb9ZrS7ADRdANpqBjhKImoi4nkeU6dOxdNPP40pU6bgrrvuwp133okXX3yx2/MLCwthMBi8t8rKyjBHTKJNizsR6dS9dN3XngJsJkCth2PITFjlerQmuJdUqfl2gKMkoiaizMxMjB071ufYmDFjcPHixW7PVyqV0Gq1PreAMCb85Qv3LYDpAq+++iqSk5Nhs/lOvFy+fHlMTFANJcYYWtuFwYw9logYA2pOCN9nz4I2XlgapT7Ovd74pfMAH9xGCKRvRG2snjNnDoqLi32OnTt3Drm5uQPzgi4H8L9nB+baPZn7W0DW+2A6AFixYgXuv/9+vPfee1ixYgUAYfeNDz74oMeZ9aQrq4OHzSEkkB5XZmwpAyzNwv9R+njo6tsBAHVcCiBXAY52oK0a0FNTwEARtUT00EMP4ciRI3j66adRUlKC119/Hf/4xz+wfv16McMSlVqtxsqVK316El977TXk5ORg/vz54gUWhTylIY1KBnkPk11Re0r4mj4BkCmgj5NDJZdCLpMDScOFx5ovDHC0sU3UEtGMGTPwzjvvoLCwEJs3b8awYcOwbds2rFq1amBeUCoXSifhJg1s8uSdd96JGTNmoLq6GkOGDEFRURHWrFnjs5g+6Z3TxZCSoECCqoePudMONLm3l8qcCAAYolfj1/PdmybU5AL1ZwFD9QBHG9tEn+Jx44034sYbbwzPi3Fcn6tIYpoyZQomTZqEV199Fddffz2+//57fPDBB2KHFXWyk+Jw2+y8nk9qKgFcTkCdCCSkA4BvwtcNFb4aawDeBUhoLeuBIHoiIt1bt24dtm3bhurqaixcuJCGKgyUhrPC17TRXfeHAoC4ZHc7kRUw1QParK7nkH4Tfa4Z6d7KlStRVVWFl156CWvXrhU7nMHJaQOay4Tv03x7b7+8cAm7vizH97VtgNa986+xNswBxg5KRBFKp9Phpz/9KRISErB8+XKxw4lKrxwuw2tHKmBod3R/QlMJwDuFUk98qs9DVoew/bTB4uh4zHxpgCOOXVQ1i2DV1dVYtWoVlEqaeBkoq8PlTUBqf3uUeRqpUwq6VMu07tUZ26wOINW904mpYUBiJZSIIlJLSwsOHDiAAwcO4G9/+5vY4UQlTxKKV0q7X2OIdwnTNwAgueumlJ5lYtvanUC8OxGZG4TBj9R7GXKUiCLQlClT0NLSgj/+8Y8YNWqU2OFEpVbPHDN/UzsMVUIbkVzd0QbUiVYt/Gq0WR1AXJbQW+a0A1YDoNYPVNgxixJRBPK3bxnpu1aLMJhR529qh6dalpwPSLqWmDwlIpPNCRckkMYlAaZGwNxIiWgAUGM1GZRa3VWzRH/7mHkTUddqGQDEK6SQSjgwBpiszo4Ga0tTqEMliIESEe1P1T/R+vPrcfkPS7Nw4yRAYvcL8XEch1SNEjxjcPC8MODR81wScoM2EcnlwgfQYrFArVaLHE30slgsADp+ntFCFyeH1enqPhF5SkP6HGGwoh+/mJnTcceUJHxtbwlhlMRj0CYiqVQKvV6PhgahyzUuLo7magWAMQaLxYKGhgbo9XpIpdE1tWHRuAz/D/ZSLeuWp0TUTiWigTBoExEAZGQIH0ZPMiKB0+v13p/joOCwAq3uBfWS8/v+vDh3ichmEnrPomDOYjQZ1ImI4zhkZmYiLS0NDoef0bXEL7lcHnUlIQDgeQaOQ/cl4OZSgPFAfEpHcvHjYpMFnxc3QB8nx82Th3TMOWtvATTpAxR9bBrUichDKpVG5S8UCc43FS04Vt6MKdl6XDkixffBzt32vZBIgGazHbynwV6dBDhqKBENAOq+J4NOq8UOu5PvuqEiz3cscNaH9iGte1VHo9Up9B5624mowTrUKBGRQcczhqhLj1lblVC1kqsA7dBer5OgkEHCcXDxDGa7C1C510i30TZWoUaJiAw6Bn/TOzzVsqTuR1NfTiLhEK8UqvRGq6Njo0WrIWSxEgElIjKo2J08TDZhM8UuJaIeJrn6o3EvM2uyOikRDSBKRGRQ8SyYr1ZIoeq8/IelWVhPiJN0LIjfBwlKdzuRzQkoOyWiKB1xHqkoEZFBpaNa5qc0pM/ucTT15ZITFEjTKqGQSjraiFwOwGkNRbjETdRE9Pjjj4PjOJ/b6NGjxQyJRDmlTIphKfEYmhjn+0Awo6kBXDE8Gatm5WL8EJ2wG4vCfV2qnoWU6OOIxo0bh08//dR7XyYTPSQSxXKS45CTfFkScliBVvfuwQEmoi5UOsBuAaxtgGYQjTgXmei/9TKZbHBNISCRp6VMGE0dl9zraGp/GGPCSG2lFkAtlYhCTPQ2ovPnzyMrKwvDhw/HqlWr/O57DwA2mw1tbW0+N0I6szpcXQ8GMJr6cu12F4q+KMPfDlwAzzPqORsgoiaiWbNmoaioCPv27cOOHTtQVlaGuXPnwmg0dnv+li1boNPpvDfa64t0ZnO6sOPABew4cAF2p7DnPRgT5pcBQSUipUwCQ7sTdicPs71TF76NElEoiZqIFi9ejBUrVmDixIlYtGgRPvzwQ7S2tuJf//pXt+cXFhbCYDB4b5WVlWGOmEQyT4+ZhEPHgvmmeqFNRyoHdIH/4eo8qNFkcwJKjfCAzRSSmIlA9DaizvR6PUaOHImSkpJuH1cqlbS1DvGr26kdnm77xLygt4vWqGQwWp3CoEa1JxF1X2onwRG9jagzk8mECxcuIDMzU+xQSBRqMbsXzO88tcNTLQtgEOPlfAY1KhKEg3azMImWhISoiWjjxo04ePAgysvL8eWXX+InP/kJpFIpfvGLX4gZFolSHQvmu0tEjnagrVr4vj+JqPM0D0WCsK8Z4wGHuV/xkg6iVs2qqqrwi1/8Ak1NTUhNTcVVV12FI0eOIDU1tfcnE3IZzxZCes/OHS3lQmN1fEq/tgDyzDczWp3CZFlFvNBGZDN1tBmRfhE1Ee3Zs0fMlyeDjGdTRW+JyNM+lNT9Th19pVfLkaZVQueZNqJIEJKQnRqsQyWiGqsJCRbPM4xM16C13S5sqti52z4p8G77zoanJmB4akLHAaUGMNbRukQhRImIDAoSCYcFo9M6DhjrhAblILvte0Rd+CFHiYgMTp7SUGIeIA3Nx9yz2SSnpC78UKNERAaFNqsDCqmkYw2i5jLhaz/bhzzePl6FOkM7fjptKDK9XfhUIgoVSkRkUPj8xwaUNppx3dh0jE9Xd3Tb+9lSOlAunofDxYQufCoRhVxEDWgkJFjN3sGMcsBQCfDuxe49O2/0k+9KjZSIQo0SEYl6DhcPg3swY1K8AmitEB7Q5wqDD0PAZyyRJxE5bcJqjaTfKBGRqNdisYMxQCWXIk4hFQYyAkJDdYh4RlebbU5AqhB64wAqFYUIJSIS9VrMntKQHJzTCpgahAcSc0P2Ghplp2keHOc754z0GyUiEvWazDYAQFK8Emip6JjWEcLpF/HuRGR0b1UERbzwlRJRSFCvGYl6noZqn/ahEFbLAKGNyHeaByWiUKJERKJeQZoGarkUWXoV8GO5cFAfumoZAGhUcqya1emaNJYopCgRkag3KkODURkaYWcNS7PQhqPPGdgXpRJRSFEiIoOHp1qmyQhoE8VAMMbAGCChRBRSlIhIVGs22+Fw8UiKV0De6l7DfIBKQ//9sR5na9pwVUEqJqupahZKlIhIVDtV2YqTla2YlpuIqw3uRKQbmETEcVzHNA8dlYhCibrvSVRrNApd92kqZ0f7kG7ogLyWdyyRzenbRuSelU+CR4mIRC3GGBpN7kTENwoH41MGrH3Iu3Z150TEeGFtbNIvlIhI1Gpzb3wolXDQ2eqEgwNULQOABO/oaoewNZFcLTxA1bN+CyoRlZaWhjoOQgJW12YFAKRqlJC2eRqqB27334ROVTPGWKfqGTVY91dQiWjEiBFYsGABXnvtNVit1pAEsnXrVnAchwcffDAk1yODnycRZcVzgNldNRug9iGgIxE5XAw2J0/zzUIoqER04sQJTJw4ERs2bEBGRgbuvvtufP3110EHcezYMfz973/HxIkTg74GiT31BiERDZE0Cw3G6sQB3d5HJpUgOykO+WkJcPKMBjWGUFCJaPLkydi+fTtqamrwyiuvoLa2FldddRXGjx+P5557Do2NjX2+lslkwqpVq/DSSy8hMTE0i1iR2DCnIAVzC1KQybk/bwNYLfP42bShuGlSllA6oqpZyPSrsVomk+GWW27BW2+9hT/+8Y8oKSnBxo0bkZ2djdtvvx21tbW9XmP9+vVYunQpFi5c2Ou5NpsNbW1tPjcSu4bo1Ziel4R4i/tzNoDVsm5R1Sxk+pWIvvnmG9x7773IzMzEc889h40bN+LChQvYv38/ampqcPPNN/f4/D179uDEiRPYsmVLn15vy5Yt0Ol03lt29sD/BSQRzuUAjJ5EFJ7PA2MMLqqahVRQI6ufe+457Ny5E8XFxViyZAleffVVLFmyBBKJkNeGDRuGoqIi5OXl+b1GZWUlHnjgAezfvx8qVd/GfRQWFmLDhg3e+21tbZSMYtTZmjZIJRzypJeg5F2AMiFk61P35GRlKw6fb8SoDC2uy6CqWagElYh27NiBtWvXYs2aNcjMzOz2nLS0NLz88st+r3H8+HE0NDRg6tSp3mMulwuHDh3CX//6V9hsNkilUp/nKJVKKJXKYEImg8yR0iYY2h34v7RKZAJCtSxE61P3RC51T/OwOahqFkJBJaL9+/cjJyfHWwLyYIyhsrISOTk5UCgUWL16td9rXHvttTh9+rTPsTvuuAOjR4/Gww8/3CUJEeLRZnXA0O4AxwEprnrh4AAOZOxM497Nw2R1Aook4aCjXdg1REKf2WAFlYjy8/NRW1uLtLQ0n+PNzc0YNmwYXC5Xr9fQaDQYP368z7H4+HgkJyd3OU5IZ9UtwpSKtAQF5KYa4WAYeswAIF4pJBujzSmMrOYkwjQPu1nYvogEJajGauZnkp/JZOpzew8hwapyJ6LhKqPQWC1TAvGpYXltz3wzm4OH3cUARZzwAFXP+iWgEpGnoZjjOGzatAlxcXHex1wuF44ePYrJkycHHcyBAweCfi6JHdUtFgBAtrRJOKDLDkv7EAAoZVIoZBLYnTxMNieSFPGAzUSJqJ8CSkTffvstAKFEdPr0aSgUCu9jCoUCkyZNwsaNG0MbISGdGNodaLEI7UOp/MBP6+iORiVDk8kOk9WJJEUCgHrAQYmoPwJKRJ9//jkAoVF5+/bt0GqpTkzCq9YgVMuytCoowtw+5JGdGAd9nAJyGUdjiUIkqMbqnTt3hjoOQvpkdIYWQ/RqWA0NwFkLIJEBCRlhjWHB6E6dNE2UiEKhz4nolltuQVFREbRaLW655ZYez927d2+/AyPEH41KDk2zu1qmzQSkIq54LKdBjaHQ5/9BnU4Hzt0gqNPpBiwgQvxhjHk/gzBUCV/DPb+sUywOF4OCqmYh0edE1Lk6RlUzIoaD5xrRYrFjRl4ShnoTUfin+FQ2W/Dut9VISlBg1UhPIrKEPY7BJKhxRO3t7bBYOn7wFRUV2LZtGz755JOQBUZIZzzPUFxnRPklC1ztRqC9Reiy1w4JeywquRRO3r2bB+34GhJBJaKbb74Zr776KgCgtbUVM2fOxLPPPoubb74ZO3bsCGmAhABAdWs7LHYXVHIphkrc44fiUwdsofyeeFZqtNhdcErdr++0AS5n2GMZLIJeoXHu3LkAgLfffhsZGRmoqKjAq6++ij//+c8hDZAQADjfYAQAjEhLgNRYLRwUoVoGACq5BDKJ0FZldsk75phRqShoQSUii8UCjUZYkvOTTz7BLbfcAolEgiuuuAIVFRUhDZAQnmc4Xy/8khekJQDejRTFaajmOM471cNod9JYohAIevH8d999F5WVlfj4449x/fXXAwAaGhpokCMJudJLZljsLsQppMjWSgFTg/CASIkI8N3Rw5uIHNRgHaygEtGmTZuwceNG5OXlYdasWZg9ezYAoXQ0ZcqUkAZIyJlqAwBgbJYWUlONe6F8vaiz3TWezRapwTokghoJ9rOf/QxXXXUVamtrMWnSJO/xa6+9Fj/5yU9CFhwhADAyXQOb04UJQ3RA/Q/CQRFLQwCQoVPD7mLQqeWAlWbg91fQQ1IzMjKQkeE7tH7mzJn9DoiQy43N0mJslrv00ypu+5DH5Gw9JmfrhTtmaiPqr6ASkdlsxtatW/HZZ5+hoaEBPM/7PE47wZIBwbsAo3uiq0g9Zt2iJWP7LahEtG7dOhw8eBC33XYbMjMzO4bdExJClc0WNBhtGJuphVohBYx1wlgduRqISxY7PDAm7Piqol6zfgsqEX300Uf44IMPMGfOnFDHQ4jX8YoWlF0yo93uwlUFKb7zy0T+42exO/Hy/8rgYgz3T1ULvT6UiIIWVK9ZYmIikpKSQh0LIV6GdgfKm4Rf7PFDPO1DF4WvEVAtU8mk4JnQgWfm3KOrqdcsaEEloieeeAKbNm3ymW9GSCh9X20AY0BOkrAIGXgeMLgTUWKuuMEBkEg470L6Jt69xZXLATjtIkYVvYKqmj377LO4cOEC0tPTkZeXB7lc7vP4iRMnQhIciU0unuH7GmE78QlD3UvOmOqEX3KZEohP6+HZ4aNRyWC0OmFySIQ1kVxOYclYmaL3JxMfQSWi5cuXh+TFd+zYgR07dqC8vBwAMG7cOGzatAmLFy8OyfVJdCq7ZIbJ5kScQor8VHePlKdaps8BJP3aKT1kEpRyAFYY7S6h56y9VWgnCsOOs4NNUInoscceC8mLDx06FFu3bkVBQQEYY9i1axduvvlmfPvttxg3blxIXoNEH89I6nFZOkjdk0s7EpH41TKPhM6jq+VxHYmIBCzoPy2tra345z//icLCQjQ3NwMQqmTV1dV9vsayZcuwZMkSFBQUYOTIkXjqqaeQkJCAI0eOBBsWiXI8z6CQSSCVcBjnGcTI874logjR7XwzarAOSlAlou+++w4LFy6ETqdDeXk57rzzTiQlJWHv3r24ePGid62iQLhcLrz11lswm83euWuXs9lssNls3vttbW3BhE8imETCYcmETFgdwtpDAABjrdAQLFcBCZHRPgQAqQlK5KclIEOnAto9gxqpAycYQZWINmzYgDVr1uD8+fM+O7suWbIEhw4dCuhap0+fRkJCApRKJe655x688847GDt2bLfnbtmyBTqdznvLzha/G5cMDG8SAny77SNo8GxOchxumpSFqTmJtBRIPwWViI4dO4a77767y/EhQ4agrq4uoGuNGjUKJ0+exNGjR/HrX/8aq1evxtmzZ7s9t7CwEAaDwXurrKwMJnwSoZrNdjSbu+n+jsD2oS68W09T1SwYQVXNlEplt9Wic+fOITU1sD3IFQoFRowYAQCYNm0ajh07hu3bt+Pvf/97t6+rVCqDCZlEga8uNOFcvRFXj0zFtFx3zxPv6lgILQLGD12OMQarg4dCFg8pQCWiIAVVIrrpppuwefNmOBwOAMKKdRcvXsTDDz+Mn/70p/0KiOd5n3YgEhusDhcuNAqliewkdccDhiqhfUgRJ6xRHWF2flGOFw9eQIvTPZaOElFQgkpEzz77LEwmE1JTU9He3o558+ZhxIgR0Gg0eOqpp/p8ncLCQhw6dAjl5eU4ffo0CgsLceDAAaxatSqYsEgUK2kwwcUzpCQokKbptCB+S5nwNXFYRLUPeXjasoye0dUOszDvgwQkqKqZTqfD/v378cUXX+DUqVMwmUyYOnUqFi5cGNB1GhoacPvtt6O2thY6nQ4TJ07Exx9/jOuuuy6YsEgUK64TFscflXHZqovN7iVlkoaFOaK+SVDJUN8GGF3u0dQuJ+ByjwAnfRZwIuJ5HkVFRdi7dy/Ky8vBcRyGDRuGjIwM3504++Dll18O9OXJIGSyOVHZInR7j0rXdDxgNwPGeuH7xMhMRBr3WCKjUwJI5UI10m6mRBSggKpmjDHcdNNNWLduHaqrqzFhwgSMGzcOFRUVWLNmDS0TS4Jyrt4IxoBMnQq6uE7zFlvKha8JaYAyQZTYeuPdzYPWru6XgEpERUVFOHToED777DMsWLDA57H//ve/WL58OV599VXcfvvtIQ2SDG6ljUID76gMje8Dze72oQitlgEdo6vNntHV7S3UYB2EgEpEb7zxBn73u991SUIAcM011+CRRx7B7t27QxYciQ03T87CjRMzfRMRY74N1RGq+2keNLo6UAElou+++w433HCD38cXL16MU6dO9TsoElvkUgkK0jWIU3QqoJvqAZtJWF4jAhZC80cXJ8eItAQMT42n+Wb9EFDVrLm5Genp6X4fT09PR0tLS7+DIgSXzgtfk4YLyShCaVVyLJuUJdwpp2kewQqoRORyuSCT+f9QSKVSOJ3OfgdFYkOTyYZXvyrH12XN3TzoTkTJBeENqj9ovlnQAvpTwxjDmjVr/E6zoBHRJBDn6k1oMtlRa2j3fcBqELrtOQ5IzhcnuAB4pnlIpWooAGFQIwlIQIlo9erVvZ5DPWakLxhjOFcvDGIcmX5Zb1lTifBVO6SjlBHB3jtVg9JGMxbnchgNUIkoCAElop07dw5UHCTGXDIJM+1lEk5o6PV50F0tS4mOalm8u5G9zTO62u6e5hGBU1IiVWQs/ktiznl3aSg3JR5KWae1hxztHct+REn7kGdQozcR8S7AaRUxouhDiYiEnW+17LIR043Fwi9yQioQL/5urn3hHUvkQMfUDqqeBYQSEQm7RqMNLRYHZBIOw1Iuq5Y1/CB8Tet+lc5IpPFM87B1nuZBiSgQkTtAgwxaUgmHsVlacIBvtcxmAlorhO9TR4sSWzC8JSKrE9DFA5YmSkQBokREwi45QYlF4zK6PnCpWGjk1WYCcdGzpXm8OxFZHS44pGrIAUpEAaJERCJHvXut8iiqlgGAUibBqAwN1AopGE9rVweDEhEJqx/r2pAUp0CqRum7dpW5SVgWluOiqloGCEslL5mQKdypcC/s5qCJr4GgxmoSNnYnj89+aMDuoxdR33bZKPw692TppHxApe365GhB0zyCQiUiEjYXGk2wO3no1HKkaztNE+JdQN1p4fvMSeIE10+eaR7gVFADgM0odkhRhRIRCRvPnvZjMrW+1bKmEmENH0V8VMwt685XpU04WtqMGSkOXAVQG1GAqGpGwqLRaENVSzskHIfxQy6retV8K3zNmABIpF2fHAU8XfgG3r0Vkt0iLKRP+kTURLRlyxbMmDEDGo0GaWlpWL58OYqLi8UMiQyQU5WtAIARaQnQqDqtS21qFJaE5Tgga7IosYWCJxG1OqSAxF3RsFP1rK9ETUQHDx7E+vXrceTIEezfvx8OhwPXX389zGZq6BtMrA4XfqwTdgaelK3zfbDqmPA1ZSSgTgxzZKHjSa5Gm6tjoX8bVc/6StQ2on379vncLyoqQlpaGo4fP46rr75apKhIqLVaHFDJpdDFSTFE32kXV7sZqP9e+D57pjjBhYhWLfwqtdtdcMTHQ45WarAOQEQ1VhsMQmNmUlL3o2ptNpvP4mttbW1hiYv0T4ZOhTVX5sFkc/o2UlceBXinMJJaO0S8AENAKZNCrZCi3e6ChVNDB1AiCkDENFbzPI8HH3wQc+bMwfjx47s9Z8uWLdDpdN5bdnbkLqpOfMmkEujjFB0HbCag+rjwfe5Vg2LtHp1aqJ6ZmGd0NSWivoqYRLR+/XqcOXMGe/bs8XtOYWEhDAaD91ZZWRnGCEmgjFYHTla2gue72Qu+8ojQq6TNjNou+8sVpCVgSo4e6gR3OxiViPosIqpm9913H95//30cOnQIQ4cO9XueUqn0u142iTwHihtR0mBCk8mGa8d02v2lvQWodnfZ580dFKUhAJie525SaGgCakGJKACiJiLGGH7zm9/gnXfewYEDBzBsWORupEcCU9JgREmDCRKOw8Sh+sse/ExoG0rMFbYLGmyU7jW4qdesz0RNROvXr8frr7+Of//739BoNKirqwMA6HQ6qNXqXp5NIpXV4cLnPzYCAKbnJSJV06kU23RBWJOakwAjrhs0pSGgY5qH2aFECiCMrqa1q/tE1DaiHTt2wGAwYP78+cjMzPTe3nzzTTHDIv30RcklmGxOJMbJMWtYpx5QRztQ/KHw/ZBpwnKwg4ih3YEXD17Am6eawABhDh3Nwu8T0atmZHCparHguyphGMa1Y9Ihk7r/1jEGnNsnVFfikoBhg2+cmEYlB8cBdl4Cu1wNpatdaCeKgi2RxBYxvWYk+jHG8HmxUCUbP0SH7KS4jgcvfgU0/ChUycYsA2QKP1eJXlIJ553q0c653zs1WPcJJSISMhzHYdG4dAxPjcfcgpSOB6qOA6UHhe8LrgO0WeIEGAaesURmqIQDNhp02xeUiEhIpWlUuHnyEKjkUmGc0PlPgfOfCA/mzAKGTBU3wAHmSURGRiWiQETEOCIS3RhjaLE4kBSvEBpoLc1A8wWg6puOX8S8q4TbIKd1JyKDJxFZDSJGEz0oEZHg2M3CgmbGOlRWVaKkuhH5eglyNQAY33GeUgOMWjxoRk/3Rh8nJKIWlxrgQImojygRkcCYLwHl/wMazwGMh93Fo66yFWqeQcnHA0wFSOVCO1DaGCB9AiCNnY9ZukaFydl6ZEoYUA1KRH0UO58Q0j8up5CAKo8KXfEAoMnAyVYNihNV0Ov0mDU1H1AlCLudxuggvsR4BRaMTgPs8UIispuFn10MJeNg0E+H9M7SDJz5/4TSEACkFADDrkaFLR5fnKgGlwAsmpYDiVYlbpyRRB4nJB+XU+g5i6INI8VAiYj0rOkCcPbfgNMGKOKAkYuB1JFwuHj895SwPfTkbD3SKAl52ZwutJgd0Ek0ULtahOoZJaIeUSIi3WNMqIaVHhC+1w0Bxt3iXQb1WHkzWi0OJChlmJ2fLG6sEebw+Uv4rsqApUyGkXJQO1EfUCIiXfE8ULIfqD4h3M+cBBRc79POoZJLoZBJMG9UKpSy6Nx5Y6B4FoDzduHToMZeUSIivlxO4If3gMZiocE5/1pg6PQujc9TcxIxJkMLlZzGxF4u0d2F3+zZWohKRL2iREQ6uBzA6beBlnJhf7Exy4QueD/UCioJdSfRXSJqcqrAwMBRIuoV/TkjApcTOLNXSEJSOTDx512SEGMMH39fh4om2u6pJ1q1HBKOg5mLh93JU4moDygREaEx+sf/AM2lQjvQxJ8DiXldTiuuN+JsTRve/64WVocr/HFGCamEgz5ODqtMh3aHS5jmQru+9ogSEelYokMiBcb/DNDndDnF6eJx+Lwwjmh6bqIwqZX4lRSvgFOigtklFRK9tVXskCIatRHFuuZSoOyQ8H3BdUBS9+uGf1dtgNHqhEYlw9Tc6N2RNVzGZGqRpVdDW5UO2C8Jg0LjU3p/YoyiRBTLHFbgxw+Fv9hZU4RbN+xOHsfKmgEAs4YlQy6lgnRvRqS5t502pQINl4D2ZnEDinD0iYplJZ8K7RfqRCD/Gr+nfXuxBRa7C/o4OcZmacMY4CDgGVHd3iJuHBFO1ER06NAhLFu2DFlZWeA4Du+++66Y4cSW1kqg7rQwPmj0Ur9Lt1odLhy/KPwSzc5PhlQSm5NZg9FksqHMrIDdxQtVM+KXqInIbDZj0qRJeOGFF8QMI/YwJpSGAGHUtN7/1t1KmQQLx6SjID0Bo9I1YQpwcNj3fR0+LXfAZHVQiagXorYRLV68GIsXLxYzhNhUfwYw1gmloLy5PZ7KcRxGpmswkpJQwFISlChu0cFscSHJZgSc9kG5aUAoRFVjtc1mg81m895va6M5PAHjeaD8sPB9zpXeSazdYYyBi9F1hUIhJUGBs1IVjC73r1l7C6BJ7/lJMSqqGqu3bNkCnU7nvWVn+69SED8afwTaWwG5Wtjk0A+b04XXjlTgeEUzXDztPxeMNI2wNEoT70721HPmV1QlosLCQhgMBu+tsrJS7JCiC2NA5RHh+yHTeqwmnKk24JLJjtNVBlCZKDipGiU4DmhhGjhcfMfCcqSLqKqaKZVKKJXK3k8k3WspA4z1wjSOHkpDThePExWtAIDpeUmQUE9ZUFRyKRLjFLDIk2Cy1SLR3CB2SBErqkpEpJ8uuktDmVOE1Rb9+KHWCJNNGEU9JpPGDfVHulYFiyIJJpuTSkQ9ELVEZDKZUFJS4r1fVlaGkydPIikpCTk5Xec7kX5oqwFaKoQtn7Nn+D2N5xm+qRDaMqbkJNK4oX6alK3DyMRxyPjhK6Gx2uUQVjcgPkRNRN988w0WLFjgvb9hwwYAwOrVq1FUVCRSVIOUpzSUPhZQ6fyeVtJoQqvFAZVciglD/J9H+iZTpwZ0aqA8AbBbhFKRNlPssCKOqIlo/vz5YIx6ZAacpRm4dE74PvsKv6cxxnCsXCgNTc7WQyGjmnvIxKcC9grA3ECJqBv0SYsFF48IPWYpBUBCqt/TOI7DdWPSMSZTg8nZ+vDFN8jVt1lxtk2JFosdMDWKHU5EokQ02NmMwkhqAMie1evpaVoVbhifScvAhtCFBhNONCnQZLIBpjqxw4lIlIgGu6pjAO8CdEN7nFNGVeSBMzQxDkZlOtqsTrC2WmF0O/FBiWgwc1iBmm+F73Nm93jq+9/V4tOz9TBaHWEILLZk6lVwKPSw8DJhipKZqmeXo0Q0mFUfFyZaxqcAyfl+T2swWlHSYMKZGgOcLioZhZpcKkGGXg2TIhWGdgfQVi12SBGHEtFg5bQL1TJAKA31MHn1WJmwRMXIdA0S42l2+EDIToyDSZEmJCJjrdjhRBxKRINV7UnA0Q6o9UDaWL+ntZjtON9gBADMyKP92QfK8NR4mBSpaG13wNVKcyQvR4loMHI5hX3rASDnCkDi/7/5WHkzGBN+UVI1NI9voKRplOC1QyHhONgMjYCVlrDpjBLRYFR7CrCZAKUGSJ/g97Rmsx0/1AqloZnDqDQ0kDiOw09nFWDq+LGIU8iA1gqxQ4oolIgGG6cNqHAvfJY7W5hp78fXZc3gGcPw1HhhKgIZULo4OSRJecKd5jJRY4k0UbUMCOmDyqPCnKa4JCBzco+nzh+VinillGbYh1NiHlj5l3BeKoWcsR47EWIJJaLBpL21o21o+Hxh59YeqORSzC3wP+WDhN4Fux7NVSbolSYUGGsBbZbYIUUEqpoNFowB5z8RGqr12UDKSL+ntpjtNJJaJBq1CvXyoWg222Gt/V7scCIGJaLBov57oOmCUAoaudhvkd9sc2LPsUq8dbwKFrszzEGSVI0SXNoY8AyoPX9S+ANCKBENCpZm4Nw+4fvcK4H45G5PY4zh4+/rYHW4YHfyUMloYmu4cRyHEaMnwcXJ0XSpAY5m6j0DKBFFP6cN+P4dYeU/fbawRZAfX5c1o6LJArmUww3jM2gtapEUZCbCnDgKDhdD5elDYocTESgRRTOXEzizFzA1CGtQj7nJ7+DFszVt+PJCEwBg/qg0pCTQ4EWxSCQcciYIG1teKj8Da1uTyBGJjxJRtHLagTNvAy3lwhrIE34OqLrvhj9TbcD+s/UAgGm5iRhPS8CKbtTw4XDp88C7eDR//6nY4YiOElE0MjcB3/4/YVCcVA6M/6nf5UdtThe+vHAJPGMYl6XF3IKUMAdLuiORcCi4YinGZmmRZSsDDFVihyQqSkTRhHcBlV8Dx3d2VMcmrwSShvmcxhgD796dVSmTYvnkIbhieDKuG5tOW0hHkPQhw6DNmyr0nP3wHzBHu9ghiSYiEtELL7yAvLw8qFQqzJo1C19//bXYIUUWlwOoOQkc+ydQ8plwPzEXmL7WZ0CczenCd1Wt2H30Io5fbPEeT9OqMDs/mZJQJBqxEFDp0N7WhKP//gcqG1p6f84gJPrI6jfffBMbNmzAiy++iFmzZmHbtm1YtGgRiouLkZaWJnZ44nE5AcNFoPGcsF+956+lIg4YNg/ImAjGcWhrd6CiyYzSRjMqmy1wuktC7XYXptK+ZJFPrgIm/AxlH/4NzFiJkn0voGbCjZg4bkJMrRvOMZGH2M6aNQszZszAX//6VwAAz/PIzs7Gb37zGzzyyCM9PretrQ06nQ4GgwFabRTOl3I5AadV6IK3tgo3SwtgrBG2huadYGDgecCh0MCSOhlJI2YCMqHH661vKlHV4lucT05QYPwQHcZmaqGSx84HOdo5Wqtx4b+70NIibOfUrkqHOmsskjLzkJKajuTEpB6Xc4lUff0dFbVEZLfbcfz4cRQWFnqPSSQSLFy4EF999VWX8202m7Dmr1tbWx/XdGkuA0q66ZnwycEMZ2sNMFsd8MnMTPhHynGYlpvoOYAf64xoa7eDMYBzH/P8KwEwIy/Re4lzdW1osTh8zpMwJzheGNk8NScREne1qbTRhCazHQDgkKrRpMpFoyoPBtcQcFUS/GakAp70oo9ToKbVinStEsNTEzA8NR7J8QqqgkUhuX4IRt/4IGpPfYKG4qOAtR4orUdzKcDi5EjO1AFSBXhOiq8rDGCczPv/zHHef5AUL0dBmsZ73aPufeo6dHw29Go5RmV0nHusvAW8n3KJRiXD2HFTgPwF3T7eX6ImokuXLsHlciE9Pd3neHp6On788ccu52/ZsgV/+MMfAn8hp61P+47zZgOc1u6nPXASDrB2/EXi29vgau9+oXkGAPaOcTrM0Q7ebvd9rY4rAzIVoNIA6kQ0OXn8qBCWFbXKdN6pGlIJB7VcCrPdCa1K2LJ4zohkzB+VCrk0+v5Skm4o4pA5YzkyJl6LS6Wn0FxdgvbmaiTI7cIfTacNPM+DuRdV86QMnz+nTAHEd/yx5o3+xygxlxwwWTvuG5vA+6kfMYdM2JpqgIjeRhSIwsJC77bUgFAiys72v0WOlz4bmPR/l82/cn/f6Vh6gR2JTv6yEoX7r46EE7YOdh/LsNiR5OQBjhPO8Px18lxXq/I+N6PdjiTPovSea0ukgEwNSJXgNErv8dEFDgzvFINMykElk0Iu5bqUdOIUUfXfR/qIU2qQOuYqpI65SjjA84DDArjs4JwO5I+2gPFOgDHwjHkL9owBKrkEiJd7r5WTa4VvquqglHNAfMcfzKF5Vr9T35QyDkjqfupQKIj6SU5JSYFUKkV9fb3P8fr6emRkZHQ5X6lUQqkMYkSwIr5LF3d3UvV9v2RSAE1SuoS+n6tRyXs/icQWiQRQCh8iKYC0AD57QxJ7P8cjS8RFOkUt0ysUCkybNg2fffaZ9xjP8/jss88we3bP+3ARQgYP0cv2GzZswOrVqzF9+nTMnDkT27Ztg9lsxh133CF2aISQMBE9Ed16661obGzEpk2bUFdXh8mTJ2Pfvn1dGrAJIYOX6OOI+iPqxxERMsj19XeU+n0JIaKjREQIER0lIkKI6ERvrO4PT/NWn6d6EELCyvO72VtTdFQnIqNRGHLep9HVhBDRGI1G6HT+VwaN6l4znudRU1MDjUYzYBM9PdNIKisrB33PXCy9VyC23q9Y75UxBqPRiKysLEh6WD0gqktEEokEQ4cODctrabXaQf9h9Yil9wrE1vsV4732VBLyoMZqQojoKBERQkRHiagXSqUSjz32WHCz/qNMLL1XILbeb6S/16hurCaEDA5UIiKEiI4SESFEdJSICCGio0RECBEdJSIEttPsSy+9hLlz5yIxMRGJiYlYuHBhVO1MG+yuunv27AHHcVi+fPnABhhigb7f1tZWrF+/HpmZmVAqlRg5ciQ+/PDDMEXbP4G+123btmHUqFFQq9XIzs7GQw89BKvV2uNzBgyLcXv27GEKhYK98sor7Pvvv2d33nkn0+v1rL6+vtvzV65cyV544QX27bffsh9++IGtWbOG6XQ6VlVVFebIAxfoe/UoKytjQ4YMYXPnzmU333xzeIINgUDfr81mY9OnT2dLlixhhw8fZmVlZezAgQPs5MmTYY48cIG+1927dzOlUsl2797NysrK2Mcff8wyMzPZQw89FObIBTGfiGbOnMnWr1/vve9yuVhWVhbbsmVLn57vdDqZRqNhu3btGqgQQyaY9+p0OtmVV17J/vnPf7LVq1dHVSIK9P3u2LGDDR8+nNnt9nCFGDKBvtf169eza665xufYhg0b2Jw5cwY0Tn9iumrm2Wl24cKF3mM97TTbHYvFAofDgaQkEfdi6YNg3+vmzZuRlpaGX/3qV+EIM2SCeb/vvfceZs+ejfXr1yM9PR3jx4/H008/DZfLFa6wgxLMe73yyitx/Phxb/WttLQUH374IZYsWRKWmC8X1ZNe+yvQnWa78/DDDyMrK8vnQxCJgnmvhw8fxssvv4yTJ0+GIcLQCub9lpaW4r///S9WrVqFDz/8ECUlJbj33nvhcDjw2GOPhSPsoATzXleuXIlLly7hqquuAmMMTqcT99xzD373u9+FI+QuYrpE1F9bt27Fnj178M4770ClUokdTkgZjUbcdttteOmll5CSkiJ2OGHB8zzS0tLwj3/8A9OmTcOtt96KRx99FC+++KLYoYXcgQMH8PTTT+Nvf/sbTpw4gb179+KDDz7AE088IUo8MV0iCnSn2c7+9Kc/YevWrfj0008xceLEgQwzJAJ9rxcuXEB5eTmWLVvmPcbzPABAJpOhuLgY+fn5Axt0PwTzf5uZmQm5XA6pVOo9NmbMGNTV1cFut0OhUAxozMEK5r3+/ve/x2233YZ169YBACZMmACz2Yy77roLjz76aI9rBw2EmC4RBbvT7DPPPIMnnngC+/btw/Tp08MRar8F+l5Hjx6N06dP4+TJk97bTTfdhAULFuDkyZMRvypmMP+3c+bMQUlJiTfhAsC5c+eQmZkZsUkICO69WiyWLsnGk4CZGNNPRWkijyB79uxhSqWSFRUVsbNnz7K77rqL6fV6VldXxxhj7LbbbmOPPPKI9/ytW7cyhULB3n77bVZbW+u9GY1Gsd5CnwX6Xi8Xbb1mgb7fixcvMo1Gw+677z5WXFzM3n//fZaWlsaefPJJsd5CnwX6Xh977DGm0WjYG2+8wUpLS9knn3zC8vPz2c9//nNR4o/5RMQYY3/5y19YTk4OUygUbObMmezIkSPex+bNm8dWr17tvZ+bm8sAdLk99thj4Q88CIG818tFWyJiLPD3++WXX7JZs2YxpVLJhg8fzp566inmdDrDHHVwAnmvDoeDPf744yw/P5+pVCqWnZ3N7r33XtbS0hL+wBljtAwIIUR0Md1GRAiJDJSICCGio0RECBEdJSJCiOgoERFCREeJiBAiOkpEhBDRUSIihIiOEhGJKkVFRdDr9d77jz/+OCZPnuy9v2bNmqhbzpZQIiJ+rFmzBhzH4Z577uny2Pr168FxHNasWeNzfqgTQF5eHrZt2+Zz7NZbb8W5c+f8Pmf79u0oKiry3p8/fz4efPDBkMZFQo8SEfErOzsbe/bsQXt7u/eY1WrF66+/jpycHFFiUqvVSEtL8/u4TqfzKTGR6ECJiPg1depUZGdnY+/evd5je/fuRU5ODqZMmdKva3dXUlm+fLm3lDV//nxUVFTgoYceAsdx4DgOQNeq2eU6l8zWrFmDgwcPYvv27d5rlJWVYcSIEfjTn/7k87yTJ0+C4ziUlJT0632R4FAiIj1au3Ytdu7c6b3/yiuv4I477hjw1927dy+GDh2KzZs3o7a2FrW1tQFfY/v27Zg9ezbuvPNO7zVycnK6vCcA2LlzJ66++mqMGDEiVG+BBIASEenRL3/5Sxw+fBgVFRWoqKjAF198gV/+8pcD/rpJSUmQSqXQaDTIyMjodcXM7uh0OigUCsTFxXmvIZVKsWbNGhQXF3sXjnc4HHj99dexdu3aUL8N0kcxvVQs6V1qaiqWLl2KoqIiMMawdOnSqF/DOisrC0uXLsUrr7yCmTNn4j//+Q9sNhtWrFghdmgxi0pEpFdr165FUVERdu3aFbJSg0Qi6bIkqcPhCMm1+2LdunXehvidO3fi1ltvRVxcXNhen/iiRER6dcMNN8But8PhcGDRokUhuWZqaqpPu4/L5cKZM2d8zlEoFP3eU8zfNZYsWYL4+Hjs2LED+/bto2qZyKhqRnollUrxww8/eL/3x2AwdNkDLTk5uduF9q+55hps2LABH3zwAfLz8/Hcc8+htbXV55y8vDwcOnQI//d//welUhlUlTAvLw9Hjx5FeXk5EhISkJSUBIlE4m0rKiwsREFBQY+bJZCBRyUi0idarRZarbbHcw4cOIApU6b43P7whz90e+7atWuxevVq3H777Zg3bx6GDx+OBQsW+JyzefNmlJeXIz8/H6mpqUHFvXHjRkilUowdOxapqam4ePGi97Ff/epXsNvtYekFJD2jNatJzPrf//6Ha6+9FpWVlV12SSXhRYmIxBybzYbGxkasXr0aGRkZ2L17t9ghxTyqmpGY88YbbyA3Nxetra145plnxA6HgEpEhJAIQCUiQojoKBERQkRHiYgQIjpKRIQQ0VEiIoSIjhIRIUR0lIgIIaKjREQIEd3/D+HUuOsdXdOFAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T09:00:37.092110Z", + "iopub.status.busy": "2024-02-29T09:00:37.091810Z", + "iopub.status.idle": "2024-02-29T09:00:37.412169Z", + "shell.execute_reply": "2024-02-29T09:00:37.411242Z" + }, + "papermill": { + "duration": 0.343463, + "end_time": "2024-02-29T09:00:37.414217", + "exception": false, + "start_time": "2024-02-29T09:00:37.070754", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEnCAYAAAANc04FAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABB0ElEQVR4nO2deZxT9bn/Pyd7ZibLLJmVzMqw7yAIiIKiCKhQW7VqlRGXttLeq/y4VeqruF2F9qqVthavC4xeUaotWrUWRBRUFEQQFJEZmB2Yfclkmazn/P44SWbCTGYmmSQny/N+vQ5Jzvmek+cwySff7/N9vs/DcBzHgSAIQkBEQhtAEARBQkQQhOCQEBEEITgkRARBCA4JEUEQgkNCRBCE4JAQEQQhOCREBEEIDgkRQRCCIxHagJHAsizOnz8PlUoFhmGENocgiAvgOA5GoxG5ubkQiQbp93ACUlBQwAHot917773DOr+hoWHA82mjjbbo2hoaGgb9LgvaIzp8+DBcLpf39YkTJ3DllVfihhtuGNb5KpUKANDQ0AC1Wh0WGwmCCJ7u7m7o9Xrvd9UfggqRTqfzeb1p0yaUlJTgsssuG9b5nuGYWq0mISKIKGYo10nU+Ijsdjtee+01rF271q/RNpsNNpvN+7q7uztS5hEEEUaiZtbsnXfeQVdXF8rKyvy22bhxIzQajXfT6/WRM5AgiLDBcFx05CNasmQJZDIZ3nvvPb9tBuoR6fV6GAwGGpoRRBTS3d0NjUYz5Hc0KoZmdXV1+Oijj7Bz585B28nlcsjl8oCuzXEcnE6nj1OciDxSqRRisVhoM4goJSqEaNu2bcjMzMTy5ctDel273Y7GxkZYLJaQXpcIHIZhMGrUKKSkpAhtChGFCC5ELMti27ZtWLVqFSSS0JnDsixqamogFouRm5sLmUxGQY8CwXEcWltbcfbsWZSWllLPiOiH4EL00Ucfob6+HqtXrw7pde12O1iWhV6vR1JSUkivTQSOTqdDbW0tHA4HCRHRD8GF6KqrrkI4/eWDhpUTESNae6NWhwvfnjVgVkEqRKLotDERoG8pkXB8WtmKs50WsCyHN76qx4EzbahoNgptVkJDQkQkFEarA0frO/HW12dhcbgwPoefUj7e0CWsYQkOCVEUsnDhQtx3331CmxGX/NBoBMcBealKpMglmJynAcMAjQYrDD0Ooc1LWEiIYph9+/aBYRh0dXUJbUrMUNNmAgCMz+Z7QslyCXK1SgBAdatJMLsSHRIiImGwOlxoNFgBAAUZvTOpJbpkAEBNm1kQu4gEFSK7k/W7OV3ssNs6htF2pNhsNjzwwAPQ6/WQy+UYPXo0Xn75ZdTW1mLRokUAgNTUVDAMM+g6PQ9GoxG33norkpOTkZOTgz/+8Y/9hoL/93//h1mzZkGlUiE7Oxu33HILWlpavMc9PbG9e/di1qxZSEpKwrx581BRUTHi+w0nZzst4DggLVkGtULq3V+QzgvR+a4euNioWPGUcAg+fS8Ez31yxu+xooxkrJye5339wqdVcLgG/nCOSlXihlm9C2+3HqhBj913Kcn9V44Zka233347vvzyS/zpT3/C1KlTUVNTg7a2Nuj1evzjH//Aj3/8Y1RUVECtVkOpVA55vbVr1+LAgQN49913kZWVhQ0bNuDo0aOYNm2at43D4cDjjz+OsWPHoqWlBWvXrkVZWRk++OADn2s99NBDePrpp6HT6fCLX/wCq1evxoEDB0Z0v+HkXBffG9Kn+f4/pSfLoJCKYXO60GG2Q6cKbBkRMXISUohihcrKSrz55pvYs2cPFi9eDAAoLi72Hk9LSwMAZGZmQqvVDnk9o9GIV155Ba+//jquuOIKAPzymtzcXJ92fYNLi4uL8ac//QkXXXQRTCaTzxKNJ554wps76sEHH8Ty5cthtVqhUCiCu+Ew02WxAwCy1b5CxDAMrp+RB22SFHIJBVsKQUIK0ZpFo/0euzCm7Z5LS/y2vTBGb/X8opGY1Y9jx45BLBYPO1HcUFRXV8PhcGD27NnefRqNBmPHjvVpd+TIETzyyCM4fvw4Ojs7wbL8ELO+vh4TJkzwtpsyZYr3eU5ODgCgpaUF+fn5IbE31Fw3NRfdVifkkv4eiSx1dIpnopCQQiQb4IMY6bbDYThDrVBjNpuxZMkSLFmyBNu3b4dOp0N9fT2WLFkCu93u01Yq7fWzeCKnPaIVjTAMA41SOnRDIuIkpLM6Vpg8eTJYlsX+/fsHPC6TyQBg2ClOiouLIZVKcfjwYe8+g8GAyspK7+tTp06hvb0dmzZtwoIFCzBu3DgfR3W8wrIc9lW04K2vG2B1UMqYSENCFMUUFhZi1apVWL16Nd555x3U1NRg3759ePPNNwEABQUFYBgG77//PlpbW2EyDR4Ho1KpsGrVKvzXf/0XPvnkE3z//fe48847IRKJvD2a/Px8yGQy/PnPf0Z1dTXeffddPP7442G/13BzsLod7x4/j7r2gafoRSIGZ1pMONvZgzaTbcA2RPggIYpytmzZgp/85Ce49957MW7cONx9990wm/kvU15eHh599FE8+OCDyMrKwq9+9ashr/fMM89g7ty5uOaaa7B48WLMnz8f48eP9zqYdTodysvL8dZbb2HChAnYtGkTnnrqqbDeYySo77CgqsUEi91/b8czW9ZiJCGKNFGTKjYYBktDabVaUVNTg6KioqidxYkGzGYz8vLy8PTTT+POO+8M2/sI+ffgOA5b9lfB5mBx68X5yFQN/P5fVrXjYHU7JuSqsWRidkRtjFdiKlUsETm++eYbnDp1CrNnz4bBYMBjjz0GAFixYoXAloUPk80Jm4OFiGGQliTz2y49hT/WYbb7bUOEBxKiOOLC6fULOXnyJADgqaeeQkVFBWQyGWbOnInPPvsMGRkZkTIz4rSbeGFJTZZCIvbvjUhL7hUijuOiNodSPEJCFEfk5ubi2LFjgx7Pz8/HkSNHImdUFNBu5n0+6cmDR0xrlVKIGAZ2JwuTzQmVgqb6IwUJURwhkUgwerT/YM1EpdXI94g8Qy9/SMQiaJQS2JwsLHYXCVEEISEi4h6O4yAVM8gYQogA4JY5BSEPTCWGhoSIiHuWTs4Bx3EYzvwwiZAwCP6/fu7cOfzsZz9Deno6lEolJk+ejK+//lpos4g4g2EYSo4fxQjaI+rs7MT8+fOxaNEi/Pvf/4ZOp8Pp06eRmpoqpFlEAtNtdeDA0W/BWLtx9aXzAAmlBIkEggrR73//e+j1emzbts27r6gotCvYicSmosmIr2o7MFqXgrkl6UO2V9Tth/aHXQAAl/x7iGfeBig04TYz4RF0aPbuu+9i1qxZuOGGG5CZmYnp06fjxRdf9NveZrOhu7vbZyOIwWg329BmtMFidw7duOkEZOcPQyJi4BTJYTN1ApW7w28kIawQVVdXY8uWLSgtLcXu3bvxy1/+Ev/xH/+BV155ZcD2GzduhEaj8W56vX7AdrFOtFXxiDZ7AsFg4StzaJOGmIp32oHqT/hzsi7GiawV6HEBaK8Cus+H2UpCUCFiWRYzZszAk08+ienTp+Oee+7B3Xffjeeff37A9uvXr4fBYPBuDQ0NEbY4drgwd1Ci0ukWIo1yiKn7xuOAzQQotXDkzYZVqkVXijsm6/w3YbaSEFSIcnJy+i1JGD9+POrr6wdsL5fLoVarfbaA4Dj+ly/SWwDrisvKyrB//35s3rwZDMOAYRhUVVXhzjvvRFFREZRKJcaOHYvNmzf3O2/lypV44oknkJub6826+MUXX2DatGlQKBSYNWsW3nnnHTAM4xOBfeLECSxduhQpKSnIysrCbbfdhra2Nr/21NbWBvb/LhAcx6GrhxfkQXtEHAecP8o/18+BOplPSNec5M433nYaiOKEb/GAoM7q+fPn96v8UFlZiYKCgvC8ocsBfPZ0eK49GAv+HyAZOpgOADZv3ozKykpMmjTJuyA1NTUVo0aNwltvvYX09HR88cUXuOeee5CTk4Mbb7zRe+7evXuhVquxZ88eAPzK52uvvRbLli3D66+/jrq6un5DrK6uLlx++eW466678Mc//hE9PT144IEHcOONN+Ljjz8e0B6dTheC/5TwY3WwsDl4ARk0M2NnDWDp4P9GWZOgae4BADQxGYBUATh6gO5zgDY+XQHRgKBCdP/992PevHl48sknceONN+Krr77CCy+8gBdeeEFIswRFo9FAJpMhKSkJ2dm9qSgeffRR7/OioiJ8+eWXePPNN32EKDk5GS+99JI3c+Pzzz8PhmHw4osvQqFQYMKECTh37hzuvvtu7zl/+ctfMH36dDz55JPefVu3boVer0dlZSXGjBkzoD2xgKc3pFJIIB1ksSsaj/OPWZMBiQzaJCcUUjGkEimQVgw0nwQ6qkiIwoigQnTRRRfh7bffxvr16/HYY4+hqKgIzz77LG699dbwvKFYyvdOIo145GuWnnvuOWzduhX19fXo6emB3W73KQEE8KllPSIEABUVFZgyZYpP/p++ifMB4Pjx4/jkk098qnN4qKqqwpgxIyuHJCROF4eMFBlSFIN8zJ12oN1dXiqHLwaQp1XilwvdRRPOF/BCZDgXZmsTG8GXeFxzzTW45pprIvNmDDPsIVI0sWPHDqxbtw5PP/005s6dC5VKhf/5n//BoUOHfNolJycHfG2TyYRrr70Wv//97/sd81TmiFX0aUm4bW7h4I3azwAuJ6BMBVKyAMA3/YdmFP9oPA+wLkBE5YbCgeBCRPRHJpP5JMQ/cOAA5s2bh3vvvde7r6qqasjrjB07Fq+99hpsNhvkcj5CuG/ifACYMWMG/vGPf6CwsBASycAfhwvtiSta+BxNyBzXvz4UACSlu/1EVsDUDKhz+7chRozga82I/hQWFuLQoUOora1FW1sbSktL8fXXX2P37t2orKzE7373u36CMhC33HILWJbFPffcgx9++AG7d+/25p/2/OqvWbMGHR0duPnmm3H48GFUVVVh9+7duOOOO7zic6E90VwyKCCcNqCjhn+e6Tt7+0VVG175ohbfN3YDanflX2NjhA1MHEiIopB169ZBLBZjwoQJ0Ol0WLJkCa6//nrcdNNNmDNnDtrb2316R/5Qq9V47733cOzYMUybNg0PPfQQNmzYAABev1Fubi4OHDgAl8uFq666CpMnT8Z9990HrVYLkUg0oD3+wiuija2f1+C1g3Uw9DgGbtB+BmCdfK8n2Xcm0Orgy08bLI7eY+a2MFucuFDy/ARj+/btuOOOO2AwGCJawDHSfw+rw4Ut+/jh65pFowdO73Hyn7wjOv9ioGSRz6Gvazvw2ek2jM9R4WpdB3DyXd5fNOO2sNseT1DyfAIA8Oqrr6K4uBh5eXk4fvy4N0ZIiCqykcTTC0qWiwcWIdbFL98AgPT+WS092Rm7e5xAcia/09zCBz9SLuuQQ0IU5zQ1NWHDhg1oampCTk4ObrjhBjzxxBNCmxV2ujxrzPwt7TCc5X1EUmWvD6gPaiX/1ei2OoCkXH62zGkHrAZAqQ2X2QkLCVGc85vf/Aa/+c1vhDYj4nRZ+GBGjb+lHZ7YofQSQNS/x+TpEZlsTrgggjgpDTC1AuZWEqIwQM5qIi7pcg/NUv3VMfMK0cDFBpJlYohFDDgOMFmdvQ5rS3uoTSWQAD2iGPbFxxWR/jsMmv7D0sFvjAhIHTgRH8Mw0KnkYDkODpblAx495xIhJ26FSCrlP4AWiyXuHbOxgCctiVgcmchkTZIUVqdrYCHy9Ia0+Xywoh9unp3f+8KUxj/2dIbQSsJD3AqRWCyGVqtFS0sLACApKYkqdwoEy7JobW1FUlKS3+jtUDNo7fohhmUD4ukR9VCPKBzErRAB8K4W94gRIRwikQj5+fnC/xg4rECXO6Feesnwz0ty94hsJn72LAbXLEYzcS1EDMMgJycHmZmZcDj8RNcSEUEmk3kjtcMNy3JgGAwseh3VAMcCyRm94uKH+nYLPqlogTZJihXT8nrXnPV0AqqsMFmfmMS1EHkQi8UR800QwvN1XScO13Zgul6LeaMzfA/2nbYfApEI6DDbwXoc7co0wHGehCgM0PQ9EXd0WeywO9n+BRVZlk9wBgzLP6R2Z3U0Wp38rJ/XT0QO61BDQkTEHZ4Yon4zZt1n+aGVVAGoRw15nRSZBCKGgYvlYLa7AIV7rZSNyliFGhIiIu4w+Fve4RmWpQ0cTX0hIhGDZDk/pDdaHb2FFq2GkNlK8JAQEXGF3cnCZOOLKfbrEQ2yyNUfKneaWZPVSUIURkiIiLjCkzBfKRNDIe0zQWHp4PMJMSI+If4wSZG7/UQ2JyDvI0QUsR9SSIiIuKJ3WOanN6TVDxpNfSHpKTJkquWQiUW9PiKXA3BaQ2Eu4UZQIXrkkUe8Rfs827hx44Q0iYhx5BIxijKSMSo1yfdAMNHUAC4uTsetcwowKU/DV2ORua9Lw7OQIngc0cSJE/HRRx95X0dqCQARn+SnJyE//QIRcliBLnd62wCFqB8KDWC3ANZuQBVbdd6iGcG/9RKJJOYK9xExRmcNH02dlD5kNLU/OI7jI7XlagCN1CMKMYL7iE6fPo3c3FwUFxfj1ltvHTQxu81mQ3d3t89GEH2xOgYoexRANPWF9NhdKD9Qg7/uqwLLcjRzFiYEFaI5c+agvLwcu3btwpYtW1BTU4MFCxbAaDQO2H7jxo3QaDTeTa+nEsBELzYnnzB/y74q2J3ukkccx68vA4ISIrlEBEOPE3YnC7O9zxS+jYQolAgqREuXLsUNN9yAKVOmYMmSJfjggw/Q1dWFN998c8D269evh8Fg8G4NDQ0RtpiIZjwzZiIGvQnzTc28T0csBTSB/3D1DWo02ZyAXMUfsJlCYjPBI7iPqC9arRZjxozBmTNnBjwul8u9FUsJ4kIGXNrhmbZPLQy6XLRKIYHR6uSDGpUeIRq4104Eh+A+or6YTCZUVVXFfM11Qhg6ze6E+X2XdniGZQEEMV6IT1CjLIXfaTfzi2iJkCCoEK1btw779+9HbW0tvvjiC/zoRz+CWCzGzTffLKRZRIzSmzDf3SNy9ADd5/jnIxGivss8ZCl8XTOOBRzmEdlL9CLo0Ozs2bO4+eab0d7eDp1Oh0suuQQHDx6ETqcb+mSCuABPCSGtp3JHZy3vrE7OGFEJIM96M6PVyS+WlSXzPiKbqddnRIwIQYVox44dQr49EWd4iip6e0Qe/1DawJU6hotWKUWmWg6NZ9mILIUXITs5rENFVDmrCSJYWJbDmCwVunrsfFHFvtP2aYFP2/elWJeCYl1K7w65CjA2UV6iEEJCRMQFIhGDReMye3cYm3iHcpDT9oNCU/ghh4SIiE88vaHUQkAcmo+5p0gkI6cp/FBDQkTEBd1WB2RiUW8Ooo4a/nGE/iEPfz9yFk2GHvx45ijkeKfwqUcUKkiIiLjgk1MtqG4148oJWZiUpeydtvdTUjpQXCwLh4vjp/CpRxRyoiqgkSCCpcMbzCgFDA0A605276m8MUJ8MzWSEIUaEiIi5nG4WBjcwYxpyTKgq44/oC3ggw9DgE8skUeInDY+WyMxYkiIiJin02IHxwEKqRhJMjEfyAjwjuoQ4YmuNtucgFjGz8YB1CsKESRERMzTafb0hqRgnFbA1MIfSC0I2Xuo5H2WeTCM75ozYsSQEBExT7vZBgBIS5YDnXW9yzpCuPwi2S1ERnepIsiS+UcSopBAs2ZEzONxVPv4h0I4LAN4H5HvMg8SolBCQkTEPKWZKiilYuRqFcCpWn6nNnTDMgBQKaS4dU6fa1IsUUghISJinrHZKozNVvGVNSwdvA9Hmx/eN6UeUUghISLiB8+wTJUdUBHFQOA4DhwHiEiIQgoJERHTdJjtcLhYpCXLIO1y5zAPU2/o41PNOHm+G5eU6jBNSUOzUEJCRMQ0xxu6cKyhCzMLUnGpwS1EmvAIEcMwvcs8NNQjCiU0fU/ENK1Gfuo+U+Hs9Q9pRoXlvbyxRDanr4/IvSqfCB4SIiJm4TgOrSa3ELGt/M7kjLD5h7y5q/sKEcfyubGJEUFCRMQs3e7Ch2IRA42tid8ZpmEZAKR4o6sdfGkiqZI/QMOzEROUEFVXV4faDoIImKZuKwBAp5JD3O1xVIev+m9Kn6EZx3F9hmfksB4pQQnR6NGjsWjRIrz22muwWq0hMWTTpk1gGAb33XdfSK5HxD8eIcpNZgCze2gWJv8Q0CtEDhcHm5Ol9WYhJCghOnr0KKZMmYK1a9ciOzsbP//5z/HVV18FbcThw4fxv//7v5gyZUrQ1yASj2YDL0R5og7eYaxMDWt5H4lYBH1aEkoyU+BkOQpqDCFBCdG0adOwefNmnD9/Hlu3bkVjYyMuueQSTJo0Cc888wxaW1uHfS2TyYRbb70VL774IlJTQ5PEikgM5pdmYEFpBnIY9+ctjMMyDz+ZOQrXTc3le0c0NAsZI3JWSyQSXH/99Xjrrbfw+9//HmfOnMG6deug1+tx++23o7GxcchrrFmzBsuXL8fixYuHbGuz2dDd3e2zEYlLnlaJWYVpSLa4P2dhHJYNCA3NQsaIhOjrr7/Gvffei5ycHDzzzDNYt24dqqqqsGfPHpw/fx4rVqwY9PwdO3bg6NGj2Lhx47Deb+PGjdBoNN5Nrw//LyAR5bgcgNEjRJH5PHAcBxcNzUJKUJHVzzzzDLZt24aKigosW7YMr776KpYtWwaRiNe1oqIilJeXo7Cw0O81Ghoa8J//+Z/Ys2cPFIrhxX2sX78ea9eu9b7u7u4mMUpQTp7vhljEoFDcBjnrAuQpIctPPRjHGrrw+elWjM1W48psGpqFiqCEaMuWLVi9ejXKysqQk5MzYJvMzEy8/PLLfq9x5MgRtLS0YMaMGd59LpcLn376Kf7yl7/AZrNBLBb7nCOXyyGXy4MxmYgzDla3w9DjwE8zG5AD8MOyEOWnHgyp2L3Mw+agoVkICUqI9uzZg/z8fG8PyAPHcWhoaEB+fj5kMhlWrVrl9xpXXHEFvvvuO599d9xxB8aNG4cHHnignwgRhIduqwOGHgcYBshwNfM7wxjI2BeVu5qHyeoEZGn8TkcPXzVERJ/ZYAlKiEpKStDY2IjMzEyf/R0dHSgqKoLL5RryGiqVCpMmTfLZl5ycjPT09H77CaIv5zr5JRWZKTJITef5nRGYMQOAZDkvNkabk4+sZkT8Mg+7mS9fRARFUM5qzs8iP5PJNGx/D0EEy1m3EBUrjLyzWiIHknUReW/PejObg4XdxQGyJP4ADc9GREA9Io+jmGEYbNiwAUlJSd5jLpcLhw4dwrRp04I2Zt++fUGfSyQO5zotAAC9uJ3fodFHxD8EAHKJGDKJCHYnC5PNiTRZMmAzkRCNkICE6JtvvgHA94i+++47yGQy7zGZTIapU6di3bp1obWQIPpg6HGg08L7h3Rs+Jd1DIRKIUG7yQ6T1Yk0WQqAZsBBQjQSAhKiTz75BADvVN68eTPUahoTE5Gl0cAPy3LVCsgi7B/yoE9NgjZJBqmEoViiEBGUs3rbtm2htoMghsW4bDXytEpYDS3ASQsgkgAp2RG1YdG4PpM07SREoWDYQnT99dejvLwcarUa119//aBtd+7cOWLDCMIfKoUUqg73sEydA4gFzHgspaDGUDDsv6BGowHjdghqNJqwGUQQ/uA4zvsZhOEs/xjp9WV9bHG4OMhoaBYShi1EfYdjNDQjhGB/ZSs6LXZcVJiGUV4hivwSn4YOC9755hzSUmS4dYxHiCwRtyOeCCqOqKenBxZL7398XV0dnn32WXz44YchM4wg+sKyHCqajKhts8DVYwR6Ovkpe3VexG1RSMVwsu5qHlTxNSQEJUQrVqzAq6++CgDo6urC7Nmz8fTTT2PFihXYsmVLSA0kCAA419UDi90FhVSMUSJ3/FCyLmyJ8gfDk6nRYnfBKXa/v9MGuJwRtyVeCDpD44IFCwAAf//735GdnY26ujq8+uqr+NOf/hRSAwkCAE63GAEAozNTIDae43cKMCwDAIVUBImI91WZXdLeNWbUKwqaoITIYrFApeJTcn744Ye4/vrrIRKJcPHFF6Ouri6kBhIEy3I43cx/yUszUwBvIUVhHNUMw3iXehjtToolCgFBJ89/55130NDQgN27d+Oqq64CALS0tFCQIxFyqtvMsNhdSJKJoVeLAVMLf0AgIQJ8K3p4hchBDutgCUqINmzYgHXr1qGwsBBz5szB3LlzAfC9o+nTp4fUQII4cc4AAJiQq4bYdN6dKF8r6Gp3lafYIjmsQ0JQkWA/+clPcMkll6CxsRFTp0717r/iiivwox/9KGTGEQQAjMlSweZ0YXKeBmj+gd8pYG8IALI1SthdHDRKKWClFfgjJeiQ1OzsbGRn+4bWz549e8QGEcSFTMhVY0Kuu/fTJax/yMM0vRbT9Fr+hZl8RCMlKCEym83YtGkT9u7di5aWFrAs63OcKsESYYF1AUb3QleBZswGhFLGjpighOiuu+7C/v37cdtttyEnJ6c37J4gQkhDhwUtRhsm5KihlIkBYxMfqyNVAknpQpsHjuMrvipo1mzEBCVE//73v/Gvf/0L8+fPD7U9BOHlSF0natrM6LG7cElphu/6MoF//Cx2J17+rAYujsN/zFDysz4kREET1KxZamoq0tLSQm0LQXgx9DhQ285/sSflefxD9fxjFAzLFBIxWI6fwDMz7uhqmjULmqCE6PHHH8eGDRt81psRRCj5/pwBHAfkp/FJyMCygMEtRKkFwhoHQCRivIn0Tay7xJXLATjtAloVuwQ1NHv66adRVVWFrKwsFBYWQiqV+hw/evTosK6zZcsWbNmyBbW1tQCAiRMnYsOGDVi6dGkwZhFxgovl8P15vpz45FHulDOmJv5LLpEDyZmDnB05VAoJjFYnTA4RnxPJ5eRTxkpkQ59M+BCUEK1cuTIkbz5q1Chs2rQJpaWl4DgOr7zyClasWIFvvvkGEydODMl7ELFHTZsZJpsTSTIxSnTuGSnPsEybD4hGVCk9ZKTIpQCsMNpd/MxZTxfvJ4pAxdl4Iyghevjhh0Py5tdee63P6yeeeAJbtmzBwYMHSYgSGE8k9cRcDcTuxaW9QiT8sMxDSt/oamlSrxARARP0T0tXVxdeeuklrF+/Hh0dHQD4Idm5c+eCup7L5cKOHTtgNpu9S0aIxINlOcgkIohFDCZ6ghhZ1rdHFCUMuN6MHNZBEVSP6Ntvv8XixYuh0WhQW1uLu+++G2lpadi5cyfq6+u9uYqGw3fffYe5c+fCarUiJSUFb7/9NiZMmDBgW5vNBpvN5n3d3d0djPlEFCMSMVg2OQdWB597CABgbOQdwVIFkBId/iEA0KXIUZKZgmyNAujxBDXSBE4wBNUjWrt2LcrKynD69Gmfyq7Lli3Dp59+GtC1xo4di2PHjuHQoUP45S9/iVWrVuHkyZMDtt24cSM0Go130+uFn8YlwoNXhADfafsoCp7NT0/CdVNzMSM/lVKBjJCghOjw4cP4+c9/3m9/Xl4empqaArqWTCbD6NGjMXPmTGzcuBFTp07F5s2bB2y7fv16GAwG79bQ0BCM+USU0mG2o8M8wPR3FPqH+uEtPU1Ds2AIamgml8sHHBZVVlZCpxtZDXKWZX2GXxe+r1wuH9H1iejly6p2VDYbcekYHWYWuGeeWFdvIrQoiB+6EI7jYHWwkEmSIQaoRxQkQQnRddddh8ceewxvvvkmAD5jXX19PR544AH8+Mc/HvZ11q9fj6VLlyI/Px9GoxGvv/469u3bh927dwdjFhHDWB0uVLXyvQl9mrL3gOEs7x+SJfE5qqOMbQdqYehx4LYJUmQAJERBEtTQ7Omnn4bJZIJOp0NPTw8uu+wyjB49GiqVCk888cSwr9PS0oLbb78dY8eOxRVXXIHDhw9j9+7duPLKK4Mxi4hhzrSY4GI5ZKTIkKnqkxC/s4Z/TC2KKv+QB48vy+iJrnaY+XUfREAE1SPSaDTYs2cPDhw4gOPHj8NkMmHGjBlYvHhxQNd5+eWXg3l7Ig6paOKT44/NviDrYoc7pUxaUYQtGh4pCgmauwGjyx1N7XICLncEODFsAhYilmVRXl6OnTt3ora2FgzDoKioCNnZ2b6VOAlimJhsTjR08tPeY7NUvQfsZsDYzD9PjU4hUrljiYxOESCW8sNIu5mEKEACGppxHIfrrrsOd911F86dO4fJkydj4sSJqKurQ1lZGaWJJYKistkIjgNyNApokvqsW+ys5R9TMgF5iiC2DYW3mgflrh4RAfWIysvL8emnn2Lv3r1YtGiRz7GPP/4YK1euxKuvvorbb789pEYS8U11K+/gHZut8j3Q4fYPRemwDOiNrjZ7oqt7OslhHQQB9YjeeOMN/Pa3v+0nQgBw+eWX48EHH8T27dtDZhyRGKyYlotrpuT4ChHH+Tqqo5SBl3lQdHWgBCRE3377La6++mq/x5cuXYrjx4+P2CgisZCKRSjNUiFJ1qeDbmoGbCY+vUYUJELzhyZJitGZKSjWJdN6sxEQ0NCso6MDWVlZfo9nZWWhs7NzxEYRBNpO849pxbwYRSlqhRTXTs3lX9TSMo9gCahH5HK5IJH4/1CIxWI4nc4RG0UkBu0mG179shZf1XQMcNAtROmlkTVqJNB6s6AJ6KeG4ziUlZX5XWbhb2kGQQxEZbMJ7SY7Gg09vgesBn7anmGA9BJhjAsAzzIPsVgJGcAHNRIBEZAQrVq1asg2NGNGDAeO41DZzAcxjsm6YLas/Qz/qM7r7WVEMe8eP4/qVjOWFjAYB1CPKAgCEqJt27aFyw4iwWgz8SvtJSKGd/T6HHQPyzJiY1iW7Hayd3uiq+3uZR4U3DtsoiP5L5FwnHb3hgoykiGX9Mk95OjpTfsRI/4hT1CjV4hYF+C0CmhR7EFCREQc32HZBRHTrRX8FzlFByQLX811OHhjiRzoXdpBw7OAICEiIk6r0YZOiwMSEYOijAuGZS0/8I+ZA6cLjkZUnmUetr7LPEiIAiF6AzSIuEUsYjAhVw0G8B2W2UxAVx3/XDdOENuCwdsjsjoBTTJgaSchChASIiLipKfIsWRidv8DbRW8k1edAyTFTknzZLcQWR0uOMRKSAESogAhISKih2Z30YQYGpYBgFwiwthsFZQyMTiWclcHAwkREVFONXUjLUkGnUrum7vK3M6nhWWYmBqWAXyq5GWTc/gXde7Ebg5a+BoI5KwmIobdyWLvDy3Yfqgezd0XROE3uRdLp5UACnX/k2MFWuYRFNQjIiJGVasJdicLjVKKLHWfZUKsC2j6jn+eM1UY40aIZ5kHGAWUAGAzCm1STEFCREQMT0378Tlq32FZ+xk+h48sOSbWlg3El9XtOFTdgYsyHLgEIB9RgAg6NNu4cSMuuugiqFQqZGZmYuXKlaioqBDSJCJMtBptONvZAxHDYFLeBUOv89/wj9mTAZG4/8kxgGcK38C6SyHZLXwifWJYCCpE+/fvx5o1a3Dw4EHs2bMHDocDV111FcxmGl/HG8cbugAAozNToFL0yUttauVTwjIMkDtNENtCgUeIuhxiQOQeaNhpeDZcBB2a7dq1y+d1eXk5MjMzceTIEVx66aUCWUWEGqvDhVNNfGXgqXqN78Gzh/nHjDGAMjXCloUOj7gabS4+0X9PFx+gGcP3FEmiatbMYOB9CGlpsRPMRgxNl8UBhVSMDJUcedo+VVztZqD5e/65frYwxoUItZL/Te+xu+CQuGfOyGE9bKLGWc2yLO677z7Mnz8fkyZNGrCNzWbzSb7W3d0dKfOIEZCtUaBsXiFMNqevk7rhEMA6+UhqdZ5wBoYAuUQMpUyMHrsLFkYJDUBCFABR0yNas2YNTpw4gR07dvhts3HjRmg0Gu+m10dvUnXCF4lYBG2SrHeHzQScO8I/L7gkLnL3aJT88MzEeaKrSYiGS1QI0a9+9Su8//77+OSTTzBq1Ci/7davXw+DweDdGhoaImglEShGqwPHGrrAsgPUgm84yM8qqXNidsr+QkozUzA9XwtlitsPRj2iYSPo0IzjOPz617/G22+/jX379qGoaPD6VXK53G++bCL62FfRijMtJrSbbLhifJ/qLz2dwDn3lH3hgrjoDQHArEK3b7OlHWgECVEACCpEa9asweuvv45//vOfUKlUaGpqAgBoNBoolcohziaimTMtRpxpMUHEMJgySnvBwb28byi1gC8XFG/I3Tm4bRTUOFwEHZpt2bIFBoMBCxcuRE5Ojnf729/+JqRZxAixOlz45FQrAGBWYSp0qj692PYqPic1IwJGXxk3vSGA7+H32F1oc3iyNJr4tCbEkAg+NCPijwNn2mCyOZGaJMWcoj6hGI4eoOID/nneTD4dbBxh6HFg24FayEQs7hUBDOviV+HHQCUSoYkKZzURP5zttODbs3w82BXjsyARuz9iHAdU7uKHK0lpQFH8BayqFFIwDGBnRbCL3a4F8hMNCxIiImRwHIdPKvgh2aQ8DfRpSb0H678EWk7xQ7Lx1wISmZ+rxC5iEeNd6tHDuO+dhGhYkBARIYNhGCyZmIViXTIWlGb0Hjh7BKjezz8vvRJQ5wpjYATwxBKZoeB32CjodjiQEBEhJVOlwIppeVBIxXyc0OmPgNMf8gfz5wB5M4Q1MMx4hMjIUY8oEKJmiQcRu3Ach06LA2nJMj7JmaUD6KgCzn7d+0UsvITf4hy1W4gMHiGyGgS0JnYgISKCw27mE5oZm9BwtgFnzrWiRCtCgQoAx/a2k6uAsUvjJnp6KLRJvBB1upQAAxKiYUJCRASGuQ2o/QxorQQ4FnYXi6aGLihZDnI2GeAUgFjK+4EyxwNZkwFx4nzMslQKTNNrkSPigHMgIRomifMJIUaGy8kLUMOh3iA9VTaOdalQkaqAVqPFnBklgCKFr3YaR4GKgZCaLMOicZmAPZkXIruZ/79LIDEOBvrfIYbG0gGc+AffGwKAjFKg6FLU2ZJx4Og5MCnAkpn5EKkVwtoZTUiTePFxOfmZsxgqGCkEJETE4LRXASf/CThtgCwJGLMU0I2Bw8Xi4+N8eehpei0ySYS82JwudJod0IhUULo6+eEZCdGgkBARA8Nx/DCseh//XJMHTLyeT4MK4HBtB7osDqTIJZhbki6srVHG56fb8O1ZA5ZzEoyRgvxEw4CEiOgPywJn9gDnjvKvc6YCpVf5+DkUUjFkEhEuG6uDXBKblTfChScBnHcKn4Iah4SEiPDF5QR+eBdoreAdziVXAKNm9XM+z8hPxfhsNRRSiom9kFT3FH6Hp7QQ9YiGhISI6MXlAL77O9BZy9cXG38tPwXvB6WMekIDkeruEbU7FeDAgSEhGhL6OSN4XE7gxE5ehMRSYMqN/USI4zjs/r4Jde1Ud24w1EopRAwDM5MMu5OlHtEwICEieGf0qfeAjmreDzTlRiC1sF+zimYjTp7vxvvfNsLqcEXezhhBLGKgTZLCKtGgx+Hil7lQ1ddBISEielN0iMTApJ8A2vx+TZwuFp+f5uOIZhWk8otaCb+kJcvgFClgdol5obd2CW1SVEM+okSnoxqo+ZR/XnolkDZwAYNvzxlgtDqhUkgwo4Cqlw7F+Bw1crVKqM9mAfY2Pig0OWPoExMUEqJExmEFTn3A/2LnTue3AbA7WRyu6QAAzClKh1RMHemhGJ3Jx1vBpANa2oCeDmENinLoE5XInPmI918oU4GSy/02+6a+Exa7C9okKSbkqiNoYBzgiaju6RTWjihHUCH69NNPce211yI3NxcMw+Cdd94R0pzEoqsBaPqOjw8at9xv6larw4Uj9fyXaG5JOsSixFzMGgztJhtqzDLYXSw/NCP8IqgQmc1mTJ06Fc8995yQZiQeHMf3hgA+alrrv3S3XCLC4vFZKM1KwdgsVYQMjA92fd+Ej2odMFkd1CMaAkF9REuXLsXSpUuFNCExaT4BGJv4XlDhgkGbMgyDMVkqjCERCpiMFDkqOjUwW1xIsxkBpz0uiwaEgphyVttsNthsNu/r7m5awxMwLAvUfs4/z5/nXcQ6EBzHgUnQvEKhICNFhpNiBYwu99espxNQZQ1+UoISU87qjRs3QqPReDe93v+QgvBD6ymgpwuQKvkih36wOV147WAdjtR1wMVSIcxgyFTxqVHaWbfY08yZX2JKiNavXw+DweDdGhoahDYptuA4oOEg/zxv5qDDhBPnDGgz2fHdWQOoTxQcOpUcDAN0cio4XGxvYjmiHzE1NJPL5ZDL5UM3JAamswYwNvPLOAbpDTldLI7WdQEAZhWmQUQzZUGhkIqRmiSDRZoGk60RqeYWoU2KWmKqR0SMkHp3byhnOp9t0Q8/NBphsvFR1ONzKG5oJGSpFbDI0mCyOalHNAiC9ohMJhPOnDnjfV1TU4Njx44hLS0N+fn91zsRI6D7PNBZx5d81l/ktxnLcvi6jvdlTM9PpbihETJVr8GY1InI/uFL3lntcvDZDQgfBBWir7/+GosWLfK+Xrt2LQBg1apVKC8vF8iqOMXTG8qaACg0fpudaTWhy+KAQirG5Dz/7YjhkaNRAholUJsC2C18r0idI7RZUYegQrRw4UJwHM3IhB1LB9BWyT/XX+y3GcdxOFzL94am6bWQSWjkHjKSdYC9DjC3kBANAH3SEoH6g/yMWUYpkKLz24xhGFw5Pgvjc1SYptdGzr44p7nbipPdcnRa7ICpVWhzohISonjHZuQjqQFAP2fI5plqBa6elENpYENIVYsJR9tlaDfZAFOT0OZEJSRE8c7ZwwDrAjSjBl1TRkPk8DEqNQlGeRa6rU5w3Y18dDvhAwlRPOOwAue/4Z/nzx206fvfNuKjk80wWh0RMCyxyNEq4JBpYWEl/BIlMw3PLoSEKJ45d4RfaJmcAaSX+G3WYrTiTIsJJ84b4HRRzyjUSMUiZGuVMMl0MPQ4gO5zQpsUdZAQxStOOz8sA/je0CCLVw/X8CkqxmSpkJpMq8PDgT41CSZZJi9ExkahzYk6SIjilcZjgKMHUGqBzAl+m3Wa7TjdYgQAXFRI9dnDRbEuGSaZDl09Dri6aI3khZAQxSMuJ1+3HgDyLwZE/v/Mh2s7wHH8F0WnonV84SJTJQerHgURw8BmaAWslMKmLyRE8UjjccBmAuQqIGuy32YdZjt+aOR7Q7OLqDcUThiGwY/nlGLGpAlIkkmArjqhTYoqSIjiDacNqHMnPiuYy6+098NXNR1gOQ7FumR+KQIRVjRJUojSCvkXHTWC2hJtxFQaEGIYNBzi1zQlpQE50wZtunCsDslyMa2wjySpheBqv4CzrRpSjht0EiGRICGKJ3q6en1DxQv5yq2DoJCKsaDU/5IPIvRU2bXoOGuCVm5CqbERUOcKbVJUQEOzeIHjgNMf8o5qrR7IGOO3aafZTpHUAqFSKtAsHYUOsx3Wxu+FNidqICGKF5q/B9qr+F7QmKV+u/xmmxM7DjfgrSNnYbE7I2wkoVPJwWSOB8sBjaeP8T8gBAlRXGDpACp38c8L5gHJ6QM24zgOu79vgtXhgt3JQiGhha2RhmEYjB43FS5Giva2Fjg6aPYMICGKfZw24Pu3+cx/Wj1fIsgPX9V0oK7dAqmYwdWTsikXtUCU5qTCnDoWDheHhu8+FdqcqICEKJZxOYETOwFTC5+Devx1foMXT57vxhdV7QCAhWMzkZFCwYtCIRIxyJ/MF7Zsqz0Ba3e7wBYJDwlRrOK0Ayf+DnTW8jmQJ98IKAaehj9xzoA9J5sBADMLUjGJUsAKztjiYri0hWBdLDq+/0hocwSHhCgWMbcD3/wfHxQnlgKTfuw3/ajN6cIXVW1gOQ4Tc9VYUJoRYWOJgRCJGJRevBwTctXItdUAhrNCmyQoJESxBOsCGr4CjmzrHY5NuwVIK/JpxnEcWHd1VrlEjJXT8nBxcTqunJBFJaSjiKy8IqgLZ/AzZz+8B87RI7RJghEVQvTcc8+hsLAQCoUCc+bMwVdffSW0SdGFywGcPwYcfgk4s5d/nVoAzFrtExBnc7rw7dkubD9UjyP1nd79mWoF5pakkwhFI6MXAwoNerrbceifL6ChpXPoc+IQwSOr//a3v2Ht2rV4/vnnMWfOHDz77LNYsmQJKioqkJmZKbR5wuFyAoZ6oLWSr1fv+bWUJQFFlwHZU8AxDLp7HKhrN6O61YyGDguc7p5Qj92FGVSXLPqRKoDJP0HNB38FZ2zAmV3P4fzkazBl4uSEyhvOcAKH2M6ZMwcXXXQR/vKXvwAAWJaFXq/Hr3/9azz44IODntvd3Q2NRgODwQC1OgbXS7mcgNPKT8Fbu/jN0gkYz/OloVknOHBgWcAhU8Gim4a00bMBCT/j9dbXDTjb6dudT0+RYVKeBhNy1FBIE+eDHOs4us6h6uNX0NnJl3PqUWRBmTsBaTmFyNBlIT01bdB0LtHKcL+jgvaI7HY7jhw5gvXr13v3iUQiLF68GF9++WW/9jabjc/566a7e5g5XTpqgDMDzEz4aDCHk40GmK0O+Cgzx/8jZhjMLEj17MCpJiO6e+zgOIBx7/P8KwJwUWGq9xKVTd3otDh82ok4JxiWj2yekZ8KkXvYVN1qQrvZDgBwiJVoVxSgVVEIgysPzFkRfj1GBo+8aJNkON9lRZZajmJdCop1yUhPltEQLAaRavMw7pr70Hj8Q7RUHAKszUB1MzqqAS5JivQcDSCWgWXE+KrOAI6ReP/ODOP9B2nJUpRmqrzXPeSuU9dL72dDq5RibHZv28O1nWD99EtUCgkmTJwOlCwa8PhIEVSI2tra4HK5kJWV5bM/KysLp06d6td+48aNePTRRwN/I6dtWHXHWbMBTuvAyx4YEQNYe3+R2J5uuHoGTjTPAYC9N06Hc/SAtdt936v3yoBEAShUgDIV7U4Wp2R8WlGrRONdqiEWMVBKxTDbnVAr+JLF80enY+FYHaTi2PulJAZAloSci1Yie8oVaKs+jo5zZ9DTcQ4pUjv/o+m0gWVZcO6kah7J8Pk55WRAcu+PNWv0H6PEuaSAydr72tgO1s/4iHNI+NJUYUJwH1EgrF+/3luWGuB7RHq9/xI5XrR6YOpPL1h/5X7eZ19WqR2pTvaCHoX7V0fE8KWD3fuyLXakOVmAYfgWnl8nz3XVCu+52T12pHmS0nuuLRIDEiUgloNRyb37x5U6UNzHBomYgUIihlTM9OvpJMli6s9HDBNGroJu/CXQjb+E38GygMMCuOxgnA6UjLOAY50Ax4HlOG/HnuMAhVQEJEu918ovsMJXqnqRSxkgufcHc1Sh1e/SN7mEAdIGXjoUCgT9JGdkZEAsFqO5udlnf3NzM7Kzs/u1l8vlkMuDiAiWJfeb4h4InXb4l0wLwCWlSRl+W5VCOnQjIrEQiQA5/yESA8gM4LOXlzp0Gw+5AibpFLRPL5PJMHPmTOzdu9e7j2VZ7N27F3PnDl6HiyCI+EHwvv3atWuxatUqzJo1C7Nnz8azzz4Ls9mMO+64Q2jTCIKIEIIL0U033YTW1lZs2LABTU1NmDZtGnbt2tXPgU0QRPwieBzRSIj5OCKCiHOG+x2leV+CIASHhIggCMEhISIIQnAEd1aPBI97a9hLPQiCiCie7+ZQruiYFiKjkQ85H1Z0NUEQgmE0GqHR+M8MGtOzZizL4vz581CpVGFb6OlZRtLQ0BD3M3OJdK9AYt2vUPfKcRyMRiNyc3MhGiR7QEz3iEQiEUaNGhWR91Kr1XH/YfWQSPcKJNb9CnGvg/WEPJCzmiAIwSEhIghCcEiIhkAul+Phhx8ObtV/jJFI9wok1v1G+73GtLOaIIj4gHpEBEEIDgkRQRCCQ0JEEITgkBARBCE4JEQIrNLsiy++iAULFiA1NRWpqalYvHhxTFWmDbaq7o4dO8AwDFauXBleA0NMoPfb1dWFNWvWICcnB3K5HGPGjMEHH3wQIWtHRqD3+uyzz2Ls2LFQKpXQ6/W4//77YbVaBz0nbHAJzo4dOziZTMZt3bqV+/7777m7776b02q1XHNz84Dtb7nlFu65557jvvnmG+6HH37gysrKOI1Gw509ezbClgdOoPfqoaamhsvLy+MWLFjArVixIjLGhoBA79dms3GzZs3ili1bxn3++edcTU0Nt2/fPu7YsWMRtjxwAr3X7du3c3K5nNu+fTtXU1PD7d69m8vJyeHuv//+CFvOk/BCNHv2bG7NmjXe1y6Xi8vNzeU2btw4rPOdTienUqm4V155JVwmhoxg7tXpdHLz5s3jXnrpJW7VqlUxJUSB3u+WLVu44uJizm63R8rEkBHova5Zs4a7/PLLffatXbuWmz9/fljt9EdCD808lWYXL17s3TdYpdmBsFgscDgcSEsTsBbLMAj2Xh977DFkZmbizjvvjISZISOY+3333Xcxd+5crFmzBllZWZg0aRKefPJJuFyuSJkdFMHc67x583DkyBHv8K26uhoffPABli1bFhGbLySmF72OlEArzQ7EAw88gNzcXJ8PQTQSzL1+/vnnePnll3Hs2LEIWBhagrnf6upqfPzxx7j11lvxwQcf4MyZM7j33nvhcDjw8MMPR8LsoAjmXm+55Ra0tbXhkksuAcdxcDqd+MUvfoHf/va3kTC5HwndIxopmzZtwo4dO/D2229DoVAIbU5IMRqNuO222/Diiy8iIyNDaHMiAsuyyMzMxAsvvICZM2fipptuwkMPPYTnn39eaNNCzr59+/Dkk0/ir3/9K44ePYqdO3fiX//6Fx5//HFB7EnoHlGglWb78tRTT2HTpk346KOPMGXKlHCaGRICvdeqqirU1tbi2muv9e5jWRYAIJFIUFFRgZKSkvAaPQKC+dvm5ORAKpVCLBZ7940fPx5NTU2w2+2QyWRhtTlYgrnX3/3ud7jttttw1113AQAmT54Ms9mMe+65Bw899NCguYPCQUL3iIKtNPuHP/wBjz/+OHbt2oVZs2ZFwtQRE+i9jhs3Dt999x2OHTvm3a677josWrQIx44di/qsmMH8befPn48zZ854BRcAKisrkZOTE7UiBAR3rxaLpZ/YeASYE2L5qSAu8ihix44dnFwu58rLy7mTJ09y99xzD6fVarmmpiaO4zjutttu4x588EFv+02bNnEymYz7+9//zjU2Nno3o9Eo1C0Mm0Dv9UJibdYs0Putr6/nVCoV96tf/YqrqKjg3n//fS4zM5P77//+b6FuYdgEeq8PP/wwp1KpuDfeeIOrrq7mPvzwQ66kpIS78cYbBbE/4YWI4zjuz3/+M5efn8/JZDJu9uzZ3MGDB73HLrvsMm7VqlXe1wUFBRyAftvDDz8cecODIJB7vZBYEyKOC/x+v/jiC27OnDmcXC7niouLuSeeeIJzOp0Rtjo4ArlXh8PBPfLII1xJSQmnUCg4vV7P3XvvvVxnZ2fkDec4jtKAEAQhOAntIyIIIjogISIIQnBIiAiCEBwSIoIgBIeEiCAIwSEhIghCcEiICIIQHBIiIqYoLy+HVqv1vn7kkUcwbdo07+uysrKYyyJJkBARfigrKwPDMPjFL37R79iaNWvAMAzKysp82odaAAoLC/Hss8/67LvppptQWVnp95zNmzejvLzc+3rhwoW47777QmoXEXpIiAi/6PV67NixAz09Pd59VqsVr7/+OvLz8wWxSalUIjMz0+9xjUbj02MiYgMSIsIvM2bMgF6vx86dO737du7cifz8fEyfPn1E1x6op7Jy5UpvL2vhwoWoq6vD/fffD4ZhwDAMgP5Dswvp2zMrKyvD/v37sXnzZu81ampqMHr0aDz11FM+5x07dgwMw+DMmTMjui8iOEiIiEFZvXo1tm3b5n29detW3HHHHWF/3507d2LUqFF47LHH0NjYiMbGxoCvsXnzZsydOxd333239xr5+fn97gkAtm3bhksvvRSjR48O1S0QAUBCRAzKz372M3z++eeoq6tDXV0dDhw4gJ/97Gdhf9+0tDSIxWKoVCpkZ2cPmahuIDQaDWQyGZKSkrzXEIvFKCsrQ0VFhTdfs8PhwOuvv47Vq1eH+jaIYZLQGRqJodHpdFi+fDnKy8vBcRyWL18e86ljc3NzsXz5cmzduhWzZ8/Ge++9B5vNhhtuuEFo0xIW6hERQ7J69WqUl5fjlVdeCVmvQSQS9csE6HA4QnLt4XDXXXd5HfHbtm3DTTfdhKSkpIi9P+ELCRExJFdffTXsdjscDgeWLFkSkmvqdDofv4/L5cKJEyd82shkshGX8vF3jWXLliE5ORlbtmzBrl27aFgmMDQ0I4ZELBbjhx9+8D73h8Fg6Fd6KD09fcD81pdffjnWrl2Lf/3rXygpKcEzzzyDrq4unzaFhYX49NNP8dOf/hRyuTyoIWFhYSEOHTqE2tpapKSkIC0tDSKRyOsrWr9+PUpLSwfNUU6EH+oREcNCrVZDrVYP2mbfvn2YPn26z/boo48O2Hb16tVYtWoVbr/9dlx22WUoLi7GokWLfNo89thjqK2tRUlJCXQ6XVB2r1u3DmKxGBMmTIBOp0N9fb332J133gm73R6RWUBicChVLJGwfPbZZ7jiiivQ0NDQrzghEVlIiIiEw2azobW1FatWrUJ2dja2b98utEkJDw3NiITjjTfeQEFBAbq6uvCHP/xBaHMIUI+IIIgogHpEBEEIDgkRQRCCQ0JEEITgkBARBCE4JEQEQQgOCRFBEIJDQkQQhOCQEBEEITgkRARBCM7/B9JCSYXa5rfHAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T09:00:37.456974Z", + "iopub.status.busy": "2024-02-29T09:00:37.456628Z", + "iopub.status.idle": "2024-02-29T09:00:37.682617Z", + "shell.execute_reply": "2024-02-29T09:00:37.681695Z" + }, + "papermill": { + "duration": 0.249909, + "end_time": "2024-02-29T09:00:37.684738", + "exception": false, + "start_time": "2024-02-29T09:00:37.434829", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAk/0lEQVR4nO3deVQUV94+8Ie1kdWFVURBSVziQsQjYlYnItHRgWGMTjSCaMyMym9MkMQw8w4e9USTUVGT8ZWJAxpNRh2VmMQdcYmJGhWjA0YwoIiJgOLCqk3TfX9/5KUmbbM0pqG6up/POX20b1V1f4tqHqpuVd+yEUIIEBEpjK3cBRARPQqGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRI9nIX0NF0Oh1u3LgBNzc32NjYyF0OET1ECIHq6mp0794dtrbN719ZXXjduHEDAQEBcpdBRK24fv06evTo0ex0qwsvNzc3AD/9YNzd3WWupv1oNBocPHgQY8aMgYODg9zlkAlYyzatqqpCQECA9LvaHLMIr7Vr12L58uUoKyvDkCFD8MEHH2D48OFNzvv888/j2LFjBu3jxo3Dnj17Wn2vxkNFd3d3iw8vZ2dnuLu7W/QH3ZpY2zZtrVtH9g77bdu2ITExEQsXLsS5c+cwZMgQREZG4ubNm03On5mZidLSUumRl5cHOzs7vPTSSx1cORHJSfbwSk1NxaxZsxAfH48BAwYgLS0Nzs7OyMjIaHL+rl27wtfXV3pkZWXB2dmZ4UVkZWQ9bKyvr0dOTg6Sk5OlNltbW4wePRonT5406jXS09Px+9//Hi4uLk1OV6vVUKvV0vOqqioAP+2CazSaX1C9eWtcN0teR2tjLdvU2PWTNbwqKiqg1Wrh4+Oj1+7j44P8/PxWlz99+jTy8vKQnp7e7DzLli3DokWLDNoPHjwIZ2fnthetMFlZWXKXQCZm6du0rq7OqPnMosP+UaWnp2PQoEHNdu4DQHJyMhITE6XnjWcyxowZY/Ed9llZWYiIiLCKzl1rYC3btPHoqDWyhpenpyfs7OxQXl6u115eXg5fX98Wl62trcXWrVuxePHiFudTqVRQqVQG7Q4ODhb9AWhkLetpTSx9mxq7brKGl6OjI0JDQ5GdnY3o6GgAP10Bn52djYSEhBaX3b59O9RqNV555ZUOqNT81dXV6R1q19xX40RuEbp4noVrJ/3w7tevn1UcMpNlk/2wMTExEXFxcRg2bBiGDx+O1atXo7a2FvHx8QCA2NhY+Pv7Y9myZXrLpaenIzo6Gt26dZOjbLOTn5+P0NBQg/a/NTFvTk4Ohg4d2v5FEbUj2cNr8uTJuHXrFlJSUlBWVoaQkBDs379f6sQvKSkx+H5TQUEBvvrqKxw8eFCOks1Sv379kJOTIz0vKL2HxO25SH1pEPr6dTaYl0jpZA8vAEhISGj2MPHo0aMGbX379gVveqTP2dlZb2/K9tptqI7fR/+BQxDSi3unZHlkv0iViOhRMLyISJEYXkSkSAwvIlIkhhcRKZJZnG0kIn0PX3QMNH/hsbVedMzwIjJDzV10DBheeGytFx0zvIjM0MMXHQPNX3hsrRcdM7yIzNDDFx0DvPD4YeywJyJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJF4h2ziczE1Ypa1Kobmp1edKtW+tfevulfXReVPYI8XdqlPnPD8CIyA1crajFqxVGj5p2/I7fF6UeSnreKAGN4EZmBxj2u1ZNDEOzt2vQ899XYffQkxj8fDpdOKoPphTdr8Pq28y3uvVkShheRGQn2dsVAf48mp2k0GpR5AUN7dYGDg0MHV2Z+2GFPRIrE8CIiRWJ4EZEisc9LwVo6tW7MaXXAuk6tk2VheCmUsafWWzutDljPqXWyLAwvhWrt1Hprp9UB6zu1TpaF4aVwzZ1a52l1snTssCciRZI9vNauXYvAwEA4OTkhLCwMp0+fbnH+e/fuYe7cufDz84NKpcLjjz+OvXv3dlC1RGQuZD1s3LZtGxITE5GWloawsDCsXr0akZGRKCgogLe3t8H89fX1iIiIgLe3N3bs2AF/f39cu3YNnTt37vjiiUhWsoZXamoqZs2ahfj4eABAWloa9uzZg4yMDLz99tsG82dkZODOnTs4ceKE1I8TGBjYkSUTkZmQLbzq6+uRk5OD5ORkqc3W1hajR4/GyZMnm1zm888/R3h4OObOnYvPPvsMXl5emDJlChYsWAA7O7sml1Gr1VCr1dLzqqoqAD91aGs0GhOuUcdqaGiQ/m1qPRrbWlrH1l6DOo4x26K1bWop29PY2mULr4qKCmi1Wvj4+Oi1+/j4ID8/v8llrly5gsOHD2Pq1KnYu3cvCgsLMWfOHGg0GixcuLDJZZYtW4ZFixYZtB88eBDOzs6/fEVkcr0GAOzx1Vdf4VrTgxAAALKysn7xa1D7a8u2aG6bWsr2rKurM2o+RV0qodPp4O3tjQ8//BB2dnYIDQ3Fjz/+iOXLlzcbXsnJyUhMTJSeV1VVISAgAGPGjIG7u3tHlW5yF29UYUXuKTz99NN4orvhemg0GmRlZSEiIqLZSyVaew3qOMZsi9a2qaVsz8ajo9bIFl6enp6ws7NDeXm5Xnt5eTl8fX2bXMbPzw8ODg56h4j9+/dHWVkZ6uvr4ejoaLCMSqWCSmV4kaaDg4Oir39q/MqPvb19i+vR0noa+xrU/tqyLZrbppayPY2tXbZLJRwdHREaGors7GypTafTITs7G+Hh4U0u89RTT6GwsBA6nU5qu3z5Mvz8/JoMLiKyXLJe55WYmIj169fjo48+wqVLlzB79mzU1tZKZx9jY2P1OvRnz56NO3fuYN68ebh8+TL27NmDpUuXYu7cuXKtAhHJRNY+r8mTJ+PWrVtISUlBWVkZQkJCsH//fqkTv6SkBLa2/83XgIAAHDhwAG+88QYGDx4Mf39/zJs3DwsWLJBrFYhIJrJ32CckJCAhIaHJaUePHjVoCw8Px6lTp9q5KiIyd7J/PYiI6FEwvIhIkWQ/bCQiQK19AFunH3G1qgC2Tk1fYdrQ0IAbDTdw6c6lJkfHvVpVA1unH6HWPgDQ9B2ILAnDS6Fa+7C39kEHrO/Dbs5u1F6DS9AH+HPLg6oAAP53//82O80lCLhRG4JQ+DQ7j6VgeCmUsR/2lj7ogHV92M1Zd5deqL36/7Bmcgj6NHPT2YaGBnz91dd46umnmvyDVHSzBvO2nUf3Ub3au1yzwPBSqNY+7K190AHr+7CbM5WdE3QP/BHk3hcDujV/09mr9lfRv2v/Jq9C1z2ohO7BLajsnNq7XLPA8FKo1j7srX3QAev7sJNl4dlGIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJHaHF5XrlxpjzqIiNqkzeEVHByMUaNG4eOPP8aDBw/aoyYiola1ObzOnTuHwYMHIzExEb6+vvjDH/6A06eN+DYpEZEJtTm8QkJCsGbNGty4cQMZGRkoLS3F008/jYEDByI1NRW3bt1qjzqJiPQ8coe9vb09YmJisH37drz33nsoLCxEUlISAgICEBsbi9LSUlPWSUSk55HD6+zZs5gzZw78/PyQmpqKpKQkFBUVISsrCzdu3EBUVJQp6yQi0tPmUSVSU1OxYcMGFBQUYNy4cdi0aRPGjRsn3eUnKCgIGzduRGBgoKlrJSKStDm81q1bhxkzZmD69Onw8/Nrch5vb2+kp6f/4uKIiJrT5vDKyspCz5499e6nCABCCFy/fh09e/aEo6Mj4uLiTFYkEdHD2tzn1adPH1RUVBi037lzB0FBQSYpioioNW0OLyFEk+01NTVwcuKInETUMYw+bExMTAQA2NjYICUlBc7OztI0rVaLb775BiEhISYvkIioKUaH17fffgvgpz2v3NxcODo6StMcHR0xZMgQJCUlmb5CIqImGB1eR44cAQDEx8djzZo1cHd3b7eiiIha0+azjRs2bGiPOoiI2sSo8IqJicHGjRvh7u6OmJiYFufNzMw0SWFERC0xKrw8PDxgY2Mj/Z+ISG5GhdfPDxV52EhE5oDDQBORIhm15/Xkk09Kh42tOXfu3C8qiIjIGEaFV3R0dDuXQUTUNkaF18KFC9u7DiKiNmnzdV5kHu5rtACAvB8rm5xee1+Ns7cA32t34dJJ1eQ8hTdr2q0+ovZmVHh17doVly9fhqenJ7p06dJi/9edO3dMVhw1r+j/guftzNwW5rLH5sIzrb6Wi4p/w0h5jPrUrlq1Cm5ubtL/je28p/Yz5glfAEAfb1d0crAzmF5QWon5O3KxcuIg9PVr/to8F5U9gjxd2q1OovZiVHj9fGDB6dOnt1ct1AZdXRzx++E9m53e0NAAAOjj5YKB/ryw2Ny11g0AtN4VYG3dAG0+XrCzs0NpaSm8vb312m/fvg1vb29otVqTFUdkLYzrBgCM6Qqwlm6ANq9lc4MRqtVqvWFyiMh4rXUDAMZ1BVhTN4DR4fX+++8D+Gkwwn/+859wdXWVpmm1Wnz55Zfo16+f6SsksgKtdQMA7Ap4mNHhtWrVKgA/7XmlpaXBzu6/fx0cHR0RGBiItLQ001dIRNQEo8Pr6tWrAIBRo0YhMzMTXbp0abeiiIha0+Y+r8YRVYmI5NTm8JoxY0aL0zMyMh65GCIiY7V5SJy7d+/qPW7evInDhw8jMzMT9+7de6Qi1q5di8DAQDg5OSEsLAynT59udt6NGzfCxsZG78FbrhFZnzbveX366acGbTqdDrNnz0afPn3aXMC2bduQmJiItLQ0hIWFYfXq1YiMjERBQYHBtWSN3N3dUVBQID3nFf9E1sckgxHa2toiMTFROiPZFqmpqZg1axbi4+MxYMAApKWlwdnZucXDTxsbG/j6+koPHx+fX1I+ESmQyS7FLSoqkq5DMVZ9fT1ycnKQnJwstdna2mL06NE4efJks8vV1NSgV69e0Ol0GDp0KJYuXYonnniiyXnVajXUarX0vKqqCgCg0Wig0WjaVK+SNG6LhoYGi15Pa2It29TYdWtzeDXeObuREAKlpaXYs2eP3ncgjVFRUQGtVmuw5+Tj44P8/Pwml+nbty8yMjIwePBgVFZWYsWKFRg5ciQuXryIHj16GMy/bNkyLFq0yKD94MGDenf9tjTXawDAHqdOncKPeXJXQ6ZgLdu0rq7OqPnaHF6Nd85uZGtrCy8vL6xcubLVM5GmEB4ejvDwcOn5yJEj0b9/f/zjH//AkiVLDOZPTk7WC9yqqioEBARgzJgxFn3j3Asld4DcsxgxYgSG9OwqdzlkAtayTRuPjloj63Venp6esLOzQ3l5uV57eXk5fH19jXoNBwcHPPnkkygsLGxyukqlgkpl+A18BwcHODg4tL1ohbC3t5f+teT1tCbWsk2NXTdZ7x7k6OiI0NBQZGdnS206nQ7Z2dl6e1ct0Wq1yM3NhZ+fX3uVSURmSPaxMxITExEXF4dhw4Zh+PDhWL16NWpraxEfHw8AiI2Nhb+/P5YtWwYAWLx4MUaMGIHg4GDcu3cPy5cvx7Vr1/Dqq6/KuRpE1MFkD6/Jkyfj1q1bSElJQVlZGUJCQrB//36pE7+kpAS2tv/dQbx79y5mzZqFsrIydOnSBaGhoThx4gQGDBgg1yoQkQxkDy8ASEhIQEJCQpPTjh49qvd81apVj3Q9GRFZFpP1ef3www947bXXTPVyREQtMll43b59G+np6aZ6OSKiFsl6tpGI6FExvIhIkRheRKRIRp9tjImJaXH6o47lRUT0KIwOLw+Plu9W4uHhgdjY2F9cEBGRMYwOrw0bNrRnHUREbcI+LyJSJKP3vIwd7oY34CCijmB0eG3cuBG9evXCk08+CSFEe9ZERNQqo8Nr9uzZ2LJlC65evYr4+Hi88sor6NrVcgdEIyLzZnSf19q1a1FaWoq33noLX3zxBQICAjBp0iQcOHCAe2JE1OHa1GGvUqnw8ssvIysrC9999x2eeOIJzJkzB4GBgaipqWmvGomIDDzy2UZbW1vY2NhACAGtVmvKmoiIWtWm8FKr1diyZQsiIiLw+OOPIzc3F3//+99RUlICV1fX9qqRiMiA0R32c+bMwdatWxEQEIAZM2Zgy5Yt8PT0bM/aiIiaZXR4paWloWfPnujduzeOHTuGY8eONTlfZmamyYojImqO0eEVGxsLGxub9qyFiMhobbpIlYjIXPC7jUSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkswivtWvXIjAwEE5OTggLC8Pp06eNWm7r1q2wsbFBdHR0+xZIRGZH9vDatm0bEhMTsXDhQpw7dw5DhgxBZGQkbt682eJyxcXFSEpKwjPPPNNBlRKROZE9vFJTUzFr1izEx8djwIABSEtLg7OzMzIyMppdRqvVYurUqVi0aBF69+7dgdUSkbmQNbzq6+uRk5OD0aNHS222trYYPXo0Tp482exyixcvhre3N2bOnNkRZRKRGbKX880rKiqg1Wrh4+Oj1+7j44P8/Pwml/nqq6+Qnp6O8+fPG/UearUaarVael5VVQUA0Gg00Gg0j1a4AjQ0NEj/WvJ6WhNr2abGrpus4dVW1dXVmDZtGtavXw9PT0+jllm2bBkWLVpk0H7w4EE4OzubukSzcb0GAOxx6tQp/JgndzVkCtayTevq6oyaT9bw8vT0hJ2dHcrLy/Xay8vL4evrazB/UVERiouLMWHCBKlNp9MBAOzt7VFQUIA+ffroLZOcnIzExETpeVVVFQICAjBmzBi4u7ubcnXMyoWSO0DuWYwYMQJDenaVuxwyAWvZpo1HR62RNbwcHR0RGhqK7Oxs6XIHnU6H7OxsJCQkGMzfr18/5Obm6rX9z//8D6qrq7FmzRoEBAQYLKNSqaBSqQzaHRwc4ODgYJoVMUP29vbSv5a8ntbEWrapsesm+2FjYmIi4uLiMGzYMAwfPhyrV69GbW0t4uPjAQCxsbHw9/fHsmXL4OTkhIEDB+ot37lzZwAwaLc2dXV1ev2EBaX3oC4rxKW8TtDd7qw3b79+/Sz6kJmsg+zhNXnyZNy6dQspKSkoKytDSEgI9u/fL3Xil5SUwNZW9is6zF5+fj5CQ0MN2qd8ZDhvTk4Ohg4d2gFVEbUfGyGEkLuIjlRVVQUPDw9UVlZaVJ/Xw3teNffV2HPkJH49KhyunfQPm7nnpUznr91G9LpT2DV7BEJ6dZO7nHZj7O+o7HteZBrOzs56e1MajQZ3K24ifPgwi+4fIevF4zEiUiSGlwXSarU4duwYvvzySxw7dgxarVbukohMjuFlYTIzMxEcHIyIiAikpqYiIiICwcHByMzMlLs0IpNieFmQzMxMTJw4EYMGDcLx48exZcsWHD9+HIMGDcLEiRMZYGRRGF4WQqvVYv78+Rg/fjx27dqFsLAwdOrUCWFhYdi1axfGjx+PpKQkHkKSxWB4WYjjx4+juLgYf/7znw2ui7O1tUVycjKuXr2K48ePy1QhkWkxvCxEaWkpgOa/adDY3jgfkdIxvCyEn58fACAvr+nhBhrbG+cjUjqGl4V45plnEBgYiKVLl0ojbTTS6XRYtmwZgoKCOGw2WQyGl4Wws7PDypUrsXv3bkRHR+PUqVO4f/8+Tp06hejoaOzevRsrVqyAnZ2d3KUSmQS/HmRBYmJisGPHDsyfPx/PPvus1B4UFIQdO3YgJiZGxuqITIvhZWFiYmIQFRWFI0eOYN++fRg7dixGjRrFPS6yOAwvC2RnZ4fnnnsOtbW1eO655xhcZJHY50VEisQ9LyIz9PD4bEDzo+Na6/hsDC8L9PNRJVxcXNjnpUDNjYwLGI6Oa60j43IkVQuTmZmJ+fPno7i4WGoLDAzEypUrebZRQZra82pudFxL2/PiSKpWqHFUifHjx2Pz5s344Ycf0KNHD/ztb3/DxIkTebmEgjw8Mi7A0XEfxg57C8FRJcjaMLwsBEeVIGvD8LIQHFWCrA3Dy0JwVAnLxvsSGGJ4WQiOKmG5eF+CpjG8LARHlbBMvC9BC4SVqaysFABEZWWl3KW0i507d4rAwEABQHoEBQWJnTt3yl0atVFDQ4MIDAwUEyZMEFqtVtTX14tdu3aJ+vp6odVqxYQJE0RQUJBoaGiQu1STMvZ3lHteFiYmJgaFhYXIyspCYmIisrKy8P333/P6LgXiGeSW8SJVC8RRJSwDzyC3jHteRGaKZ5BbxvAiMlM8g9wyHjYSmanGM8gTJ05EVFQUIiIi8P333+PatWvIysrCnj17sGPHDqvtFmB4EZmxmJgYJCUlYdWqVdi9e7fUbm9vj6SkJKs+EcPwIjJjmZmZWLFiBX79619Le16PPfYYsrKysGLFCowYMcJqA4zjeVkojUaDvXv3Yty4cRw+RaG0Wi2Cg4MxaNAg7Nq1C1qtVtqmdnZ2iI6ORl5eHr7//nuLOnQ09neUHfZEZorXebWM4UVkpnidV8sYXkRmitd5tYzhRWSmeJ1XyxheRGaKI4W0jJdKEJmxmJgY7NixA/Pnz8ezzz4rtQcFBVn9DVUYXkRmLiYmBlFRUThy5Aj27duHsWPH8l6cYHgRKQJHCjHEPi8iUiSGFxEpEsOLiBTJ6vq8Gr/KWVVVJXMl7Uuj0aCurg5VVVX8bqOFsJZt2vi72drXrq0uvKqrqwEAAQEBMldCRC2prq6Gh4dHs9OtblQJnU6HGzduwM3NDTY2NnKX026qqqoQEBCA69evW/ToGdbEWrapEALV1dXo3r27wRfSf87q9rxsbW3Ro0cPucvoMO7u7hb9QbdG1rBNW9rjasQOeyJSJIYXESkSw8tCqVQqLFy4ECqVSu5SyES4TfVZXYc9EVkG7nkRkSIxvIhIkRheRKRIDC+ZPf/883j99dflLoNIcRheCnH06FHY2Njg3r17cpdCbWBuf5zMrZ5fguFFZObq6+vlLsEsMbzMiFqtxoIFCxAQEACVSoXg4GCkp6ejuLgYo0aNAgB06dIFNjY2mD59equvV11djalTp8LFxQV+fn5YtWqVwV/ezZs3Y9iwYXBzc4Ovry+mTJmCmzdvStMb9/iys7MxbNgwODs7Y+TIkSgoKDD16luc6dOn49ixY1izZg1sbGxgY2ODoqIizJw5E0FBQejUqRP69u2LNWvWGCwXHR2Nd955B927d0ffvn0BACdOnEBISAicnJwwbNgw7Nq1CzY2Njh//ry0bF5eHsaOHQtXV1f4+Phg2rRpqKioaLae4uLijvpxmJ4gWT333HNi3rx5QgghJk2aJAICAkRmZqYoKioShw4dElu3bhUNDQ1i586dAoAoKCgQpaWl4t69e62+9quvvip69eolDh06JHJzc8Vvf/tb4ebmJr2fEEKkp6eLvXv3iqKiInHy5EkRHh4uxo4dK00/cuSIACDCwsLE0aNHxcWLF8UzzzwjRo4caeofhcW5d++eCA8PF7NmzRKlpaWitLRUPHjwQKSkpIgzZ86IK1euiI8//lg4OzuLbdu2ScvFxcUJV1dXMW3aNJGXlyfy8vJEZWWl6Nq1q3jllVfExYsXxd69e8Xjjz8uAIhvv/1WCCHE3bt3hZeXl0hOThaXLl0S586dExEREWLUqFHN1tPQ0CDHj8YkGF4yawyvgoICAUBkZWU1OV9jiNy9e9eo162qqhIODg5i+/btUtu9e/eEs7OzXng97MyZMwKAqK6u1nvfQ4cOSfPs2bNHABD37983qhZr9vM/Ts2ZO3eu+N3vfic9j4uLEz4+PkKtVktt69atE926ddP7ma9fv14vvJYsWSLGjBmj99rXr1+X/ugZW49S8LDRTJw/f166yYIpXLlyBRqNBsOHD5faPDw8pEOQRjk5OZgwYQJ69uwJNzc36f1LSkr05hs8eLD0/8Y7NP/88JKMt3btWoSGhsLLywuurq748MMPDX7egwYNgqOjo/S8oKAAgwcPhpOTk9T2820LABcuXMCRI0fg6uoqPfr16wcAKCoqasc1kofVDYljrjp16tTh71lbW4vIyEhERkbik08+gZeXF0pKShAZGWnQSfzzkTsbx0F7+C7O1LqtW7ciKSkJK1euRHh4ONzc3LB8+XJ88803evO5uLi0+bVramowYcIEvPfeewbTGv/gWBKGl5kYNGgQdDodjh07htGjRxtMb/wrrNVqjXq93r17w8HBAWfOnEHPnj0BAJWVlbh8+bJ089L8/Hzcvn0b7777rjSy7NmzZ02xOvR/HB0d9bbZ119/jZEjR2LOnDlSmzF7RX379sXHH38MtVotfTH7zJkzevMMHToUO3fuRGBgIOztm/7VfrgeJeNho5kIDAxEXFwcZsyYgV27duHq1as4evQo/v3vfwMAevXqBRsbG+zevRu3bt1CTU1Ni6/n5uaGuLg4vPnmmzhy5AguXryImTNnwtbWVtpz6tmzJxwdHfHBBx/gypUr+Pzzz7FkyZJ2X1drEhgYiG+++QbFxcWoqKjAY489hrNnz+LAgQO4fPky/vrXvxqEUFOmTJkCnU6H1157DZcuXcKBAwewYsUKAP/dE547dy7u3LmDl19+GWfOnEFRUREOHDiA+Ph4KbAerkfJe88MLzOybt06TJw4EXPmzEG/fv0wa9Ys1NbWAgD8/f2xaNEivP322/Dx8UFCQkKrr5eamorw8HCMHz8eo0ePxlNPPYX+/ftL/SZeXl7YuHEjtm/fjgEDBuDdd9+VfiHINJKSkmBnZ4cBAwbAy8sLkZGRiImJweTJkxEWFobbt2/r7YU1x93dHV988QXOnz+PkJAQ/OUvf0FKSgoASNuze/fu+Prrr6HVajFmzBgMGjQIr7/+Ojp37iwNp/xwPQ/3tSkJh8SxIrW1tfD398fKlSsxc+ZMucuhX+iTTz5BfHw8KisrZekzlRv7vCzYt99+i/z8fAwfPhyVlZVYvHgxACAqKkrmyuhRbNq0Cb1794a/vz8uXLiABQsWYNKkSVYZXADDS7FKSkowYMCAZqd/9913AIAVK1agoKAAjo6OCA0NxfHjx+Hp6dlRZZIJlZWVISUlBWVlZfDz88NLL72Ed955R+6yZMPDRoVqaGho8asdLZ1xIrIEDC8iUiSebSQiRWJ4EZEiMbyISJEYXkSkSAwvalfTp0+XBr5zcHCAj48PIiIikJGR0aavpmzcuBGdO3duv0Kb0TgwIJkfhhe1uxdffBGlpaUoLi7Gvn37MGrUKMybNw/jx49HQ0OD3OWRUsk5mBhZvri4OBEVFWXQnp2dLQCI9evXCyGEWLlypRg4cKBwdnYWPXr0ELNnzzYYEPHnj4ULFwohhNi0aZMIDQ0Vrq6uwsfHR7z88suivLxcep87d+6IKVOmCE9PT+Hk5CSCg4NFRkaGNL2kpES89NJLwsPDQ3Tp0kX85je/EVevXhVCCLFw4UKD9z1y5Ei7/Jyo7bjnRbL41a9+hSFDhiAzMxMAYGtri/fffx8XL17ERx99hMOHD+Ott94CAIwcORKrV6+Gu7s7SktLUVpaiqSkJACARqPBkiVLcOHCBezatQvFxcV64/v/9a9/xXfffYd9+/bh0qVLWLdunfQNA41Gg8jISLi5ueH48eP4+uuv4erqihdffBH19fVISkrCpEmTpD3H0tJSjBw5smN/UNQ8udOTLFtze15CCDF58mTRv3//Jqdt375ddOvWTXq+YcMG4eHh0er7PTyM9YQJE0R8fHyT827evFn07dtX6HQ6qU2tVotOnTqJAwcOtFo/yYt7XiQbIYQ0FtWhQ4fwwgsvwN/fH25ubpg2bRpu376Nurq6Fl+jtWGsZ8+eja1btyIkJARvvfUWTpw4IS174cIFFBYWws3NTRo2uWvXrnjw4IFFDptsaRheJJtLly4hKCgIxcXFGD9+PAYPHoydO3ciJycHa9euBdDyPQsbh7F2d3fHJ598gjNnzuDTTz/VW27s2LG4du0a3njjDdy4cQMvvPCCdMhZU1OD0NBQnD9/Xu9x+fJlTJkypZ3Xnn4pfnOXZHH48GHk5ubijTfeQE5ODnQ6HVauXCkNmtc4gmyjpoYvNnYYay8vL8TFxSEuLg7PPPMM3nzzTaxYsQJDhw7Ftm3b4O3tDXd39ybrtKRhky0N97yo3anVapSVleHHH3/EuXPnsHTpUkRFRWH8+PGIjY1FcHAwNBqNNBz15s2bkZaWpvcagYGBqKmpQXZ2NioqKlBXV2fUMNYpKSn47LPPUFhYiIsXL2L37t3o378/AGDq1Knw9PREVFQUjh8/Lg29/ac//Qk//PCD9L7/+c9/UFBQgIqKCmg0mo75oVHr5O50I8sWFxcnXWZgb28vvLy8xOjRo0VGRobQarXSfKmpqcLPz0906tRJREZGik2bNhncp/KPf/yj6Natm96lEv/6179EYGCgUKlUIjw8XHz++ecG9zLs37+/6NSpk+jatauIiooSV65ckV6ztLRUxMbGCk9PT6FSqUTv3r3FrFmzRGVlpRBCiJs3b4qIiAjh6urKSyXMDIfEISJF4mEjESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkX6/59ACpSgAIGyAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T09:00:37.728346Z", + "iopub.status.busy": "2024-02-29T09:00:37.728049Z", + "iopub.status.idle": "2024-02-29T09:00:38.048192Z", + "shell.execute_reply": "2024-02-29T09:00:38.047285Z" + }, + "papermill": { + "duration": 0.344492, + "end_time": "2024-02-29T09:00:38.050381", + "exception": false, + "start_time": "2024-02-29T09:00:37.705889", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEnCAYAAAA9/18kAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABR8klEQVR4nO3deVwU9R/H8dfsLrsccqgoh4GIF2oohkqaRyYKZqlleWReWVamHWSmWWJpaaZm3ml5VJ5laplRuoknHiHmgZoHBh6gSNzCsrvz+4OfWyQoi8BwfJ+PxzyC2ZnvvmdjP875/UqyLMsIgiBUQSqlAwiCIJQVUeAEQaiyRIETBKHKEgVOEIQqSxQ4QRCqLFHgBEGoskSBEwShyhIFThCEKksUOEEQqiyN0gEqIrPZzJUrV3B0dESSJKXjCILwH7Isk5GRgaenJyrVHfbT5ApgwYIFcv369WWdTie3a9dOPnjwYJHLbty4UQ4MDJSdnZ1le3t7uVWrVvJXX31VYJlhw4bJQIEpJCSk2HkSEhJuW19MYhJTxZsSEhLu+F1WfA9u/fr1hIWFsWTJEoKCgpg7dy4hISGcOXOGunXr3rZ8rVq1mDRpEn5+fmi1WrZu3cqIESOoW7cuISEhluVCQ0NZsWKF5XedTlfsTI6OjgAkJCTg5OR0D1snCEJZSE9Px8vLy/JdLYoky8o+bB8UFETbtm1ZsGABkH946OXlxdixY5kwYUKx2njggQfo1asXU6dOBWD48OGkpqayefPmEmVKT0/H2dmZtLQ0UeAEoQIq7ndU0YsMBoOB6OhogoODLfNUKhXBwcFERUXddX1ZltHr9Zw5c4bOnTsXeC0yMpK6devStGlTXn75ZW7cuFFkO7m5uaSnpxeYBEGo/BQ9RE1OTsZkMuHm5lZgvpubG6dPny5yvbS0NOrVq0dubi5qtZpFixbRvXt3y+uhoaE8+eSTNGjQgPPnz/POO+/Qs2dPoqKiUKvVt7U3ffp03n///dLbMEEQKgTFz8GVhKOjI0ePHiUzMxO9Xk9YWBi+vr48/PDDAAwcONCyrL+/Py1btqRhw4ZERkbSrVu329qbOHEiYWFhlt9vHd8LglC5KVrgXF1dUavVJCUlFZiflJSEu7t7keupVCoaNWoEQEBAAKdOnWL69OmWAvdfvr6+uLq6cu7cuUILnE6ns+oiRGVgMpnIy8tTOoZQQanVajQaTZW/DUrRAqfVagkMDESv19O3b18g/yKDXq9nzJgxxW7HbDaTm5tb5OuXLl3ixo0beHh43GvkSiEzM5NLly6h8PUjoYKzt7fHw8MDrVardJQyo/ghalhYGMOGDaNNmza0a9eOuXPnkpWVxYgRIwAYOnQo9erVY/r06UD++bI2bdrQsGFDcnNz2bZtG19//TWLFy8G8r/c77//Pv369cPd3Z3z588zfvx4GjVqVOA2kqrKZDJx6dIl7O3tqVOnTpX/F1qwnizLGAwGrl+/TlxcHI0bN77zzbLlxGA0o9WUbg7FC9yAAQO4fv06kydPJjExkYCAACIiIiwXHuLj4wt8+FlZWYwePZpLly5hZ2eHn58f33zzDQMGDADyd72PHTvGqlWrSE1NxdPTkx49ejB16tQqdxhamLy8PGRZpk6dOtjZ2SkdR6ig7OzssLGx4a+//sJgMGBra6toHrPJzIHpPUlxakb7Z97DrU7tUmlX8fvgKqLKfB9cTk4OcXFxNGjQQPE/WqFiq0h/Kyf3bKaFfhgGNJjHHMHWtf4dl68U98EJgiAgyzjuyz8Fddj1ibsWN2uIAicIgqJyj2/CO+c0WbIOh27jS7VtUeCECuHhhx/m9ddfVzpGlREZGYkkSaSmpiod5c5MRgy/5j9i+Z22D638Gpdq86LACZVKpfniCsVzbB2OmRf4W65BduDoUr/qLwqcIFRQBoNB6Qhly5iL6bf8c2+LjL3p1bZpqb+FKHBVnCzLZBuMikwlvUCfm5vL22+/jZeXFzqdjkaNGvHll19y8eJFunbtCkDNmjWRJInhw4fftb2MjAwGDx6Mg4MDHh4efPrpp7cdEn/99de0adMGR0dH3N3deeaZZ7h27Zrl9Vt7jnq9njZt2mBvb0+HDh04c+ZMsbdr2rRp1K1bF0dHR55//nkmTJhAQECA5fXhw4fTt29fPvzwQzw9PWnatGmxsgFs27aNJk2aYGdnR9euXbl48WKxcynm9+WoMy6RKNck9r4BeNe2L/W3UPw+OKFs3cwz0XzyL4q8d+wHIdhrrf8TGzp0KFFRUcybN49WrVoRFxdHcnIyXl5ebNy4kX79+nHmzBmcnJyKda9fWFgY+/bt44cffsDNzY3Jkydz5MiRAsUlLy+PqVOn0rRpU65du0ZYWBjDhw9n27ZtBdqaNGkSs2fPpk6dOrz00ks899xz7Nu3764ZVq9ezYcffsiiRYt46KGHWLduHbNnz6ZBgwYFltPr9Tg5ObF9+/ZiZ0tISODJJ5/klVdeYdSoUfz++++8+eabd82kqNwM5N2zkIB5xid5PNC3TN5GFDihQvnzzz/ZsGED27dvt3Sj5ev7zx9/rVq1AKhbty4uLi53bS8jI4NVq1axZs0ay3PIK1aswNPTs8Byzz33nOVnX19f5s2bR9u2bcnMzKRGjRqW1z788EO6dOkCwIQJE+jVqxc5OTl3vY9s/vz5jBw50vKEzuTJk/n111/JzMwssJyDgwNffPFFgcen7pZt8eLFNGzYkNmzZwPQtGlTjh8/zscff3zXz0cxBxYjZScTZ3Zji9SVCS3L5jFKUeCqODsbNbEfKPOImp3N7V1T3c3Ro0dRq9WWInKvLly4QF5eHu3atbPMc3Z2thz+3RIdHc2UKVP4448/+PvvvzGbzUD+kzTNmze3LNeyZUvLz7eebb527Rre3t53zHHmzBlGjx5dYF67du347bffCszz9/e/7dnQu2U7deoUQUFBBdZp3779HfMoKjsF9s8H4FPj0zzifx9OtjZl8laiwN2jtOw8dDYqbEvwZS4PkiSV6DBRKUo8XpaVlUVISAghISGsXr2aOnXqEB8fT0hIyG0n+m1s/vki3rrid6vglAYHB4cSZ6s09s6B3HTO4MOP5gdZ/kC9MnsrcZHhHnxz4C86zvyNbw78pXSUKsPf3x+z2cyuXbsKff3W3o3JZCpWe76+vtjY2HD48GHLvLS0NP7880/L76dPn+bGjRvMmDGDTp064efnd9tJ/HvVtGnTAhmA234vTHGyNWvWjEOHDhWYd+DAgXsPXRbSr8ChZQDMMDxN7Rp2dGrkWmZvJwrcPbBRS2TkGFkUeZ6sXKPScaoEHx8fhg0bxnPPPcfmzZuJi4sjMjKSDRs2AFC/fn0kSWLr1q1cv379tnNY/+Xo6MiwYcN466232LlzJydPnmTkyJGoVCrLHpi3tzdarZb58+dz4cIFfvjhB8v4HqVl7NixfPnll6xatYqzZ88ybdo0jh07dtf7voqT7aWXXuLs2bO89dZbnDlzhjVr1rBy5cpSzV9qdn0MxhzO2d7PTnMAfQM80ajLrgyJAncPnnzgPnxq25OSZWDFvjil41QZixcv5qmnnmL06NH4+fnxwgsvkJWVBUC9evV4//33mTBhAm5ubsXqN3DOnDm0b9+exx57jODgYB566CGaNWtmuTBQp04dVq5cybfffkvz5s2ZMWMGs2bNKtVtGjx4MBMnTmTcuHE88MADxMXFMXz48LtenChONm9vbzZu3MjmzZtp1aoVS5Ys4aOPPirV/KXixnk48jUAkzP7ARJPPnBfmb6l6E2kENb0JrLl6GVeW3cUR1sNe8c/grN92ZwsLa6K1ENERZWVlUW9evWYPXs2I0eOVCxH9+7dcXd35+uvv1bk/cv9b+W7kXDiOy67duShS6Np5uHEz691KlFTojeRcvJ4S0+aujmSkWNk2Z4LSscRChETE8PatWs5f/48R44cYfDgwQD06dOn3DJkZ2czZ84cTp48yenTpwkPD2fHjh0MGzas3DIoKvE4nPgOgDmm/L4b+5XhxYVbRIG7RyqVxBvdmwCwfF8cNzKL7jpdKH3x8fHUqFGjyCk+Ph6AWbNm0apVK4KDg8nKymLPnj24upbeye0WLVoUmWH16tVIksS2bdvo3LkzgYGB/Pjjj2zcuLHAkJlVQbbByFdRF0lMyyn4gj7/vGFmo95svFoblQS9AzwLaaF0VZ77Byqq7BRCmjrjX8+Z45fTWBx5nncfa3739YRS4enpydGjR+/4ure3N9HR0WWaY9u2bUUO8uPm5oadnR07duwo0wwVwfs/xLL+9wTWHkr45/Az/gCc/QUkNesdhwIGOjepQ13Hsj8sFgXuXhz+Ena8j9RlPG/26M/wFYf5+sBfPN/JF3dncf6rPGg0GssIa0qqX7/0OmmszL6NTgDg1NX/D54uy7Ajf8xhufWzLI/Nv1+0XxlfXLhFHKLeC7UN5KbBntl0qa+jrU9Nco1mFuw8q3QyQVCE+b+XLM/tgPj9oNYR02AUl1Nv4qjT0L25W6HrlzZR4O5Fq2egdmO4mYIUtYg3e+Q//rPuUAIJKdkKhxMEhZnNoM/fe6PdC6w7k39zdq+WHuX25E+FKHALFy7Ex8cHW1tbgoKCbrsr+9++//572rRpg4uLCw4ODgQEBNx2mV2WZSZPnoyHhwd2dnYEBwdz9mwZ7FWpNfDIpPyfoxbwoJtMp8auGM0yn+nFXpxQfdXQaSB2U/7VU60jN4NeY9vxRIAyv/ft3xQvcOvXrycsLIzw8HCOHDlCq1atCAkJKfJRmVq1ajFp0iSioqI4duwYI0aMYMSIEfzyyz9dAs2cOZN58+axZMkSDh48iIODAyEhIeTk5BTa5j1p1gc8WoEhE/bMsezFfX/kEueu3fkue0GoSm4a/nl8ro69Cn77EIB9boN448cEMnONeNWyo039muWWSfECN2fOHF544QVGjBhB8+bNWbJkCfb29ixfvrzQ5R9++GGeeOIJmjVrRsOGDXnttddo2bIle/fuBfL33ubOncu7775Lnz59aNmyJV999RVXrlxh8+bNpb8BKhV0m5z/8+EvCHDKJLiZG2YZ5u74887rCkIVcjXtpuXnx+WdkHKeFJwYdTaIiJP5e29PPeCFSlV+g5ErWuAMBgPR0dEF7gVSqVQEBwcTFRV11/VlWUav13PmzBk6d+4MQFxcHImJiQXadHZ2JigoqMg2c3NzSU9PLzBZpWE3qN8RTLkQOYOw/98Xt/XYVWKvWNmWUCI+Pj7MnTvX8rskSWXzD5pQpCup+UdIOgw8m7sOgAV5fXBxqcXYRxoxtU8LXuxSNh1bFkXRApecnIzJZLKMYn+Lm5sbiYmJRa6XlpZGjRo10Gq19OrVi/nz59O9e3cAy3rWtDl9+nScnZ0tk5eXl3UbIkn/7MUdXU1zbRKP/b8Dv893n7euLaFUXL16lZ49eyodo1q5kpq/BzdEvZ268g2uyLVZberG+71b8GaPpgxp71Pu3YopfohaEo6Ojhw9epTDhw/z4YcfEhYWRmRkZInbmzhxImlpaZYpISHB+ka8g6BJKMhm+G0aQ9v7ALDvXHKJxyYQSs7d3R2dTqdoBlmWMRqrTy8zV9JuUoNsRmu2APCpsR/tm9ajW7O6imVStMC5urqiVqtJSkoqMD8pKQl3d/ci11OpVDRq1IiAgADefPNNnnrqKaZPzx+d59Z61rSp0+lwcnIqMJXII+8BEsRuJkBzEZ1GRXKmgfPXFbzYIMtgyFJmsrKwF2dwmOL69yHqxYsXkSSJ77//nq5du2Jvb0+rVq1uO2Wxd+9eOnXqhJ2dHV5eXrz66quWXkyg+APT/PzzzwQGBqLT6SznhquDK6k3eUGzjVpSJufMnvxIZyY/1rzUhwK0hqJPMmi1WgIDA9Hr9fTt2xfI7x1Vr9cXqxucW8xmM7m5+c+ANmjQAHd3d/R6vWVQkfT0dA4ePMjLL79c2ptQkPv94P8UHP8WbeQ0AuuPZ//5G0RdSKFRXceyfe+i5GXDR2X/zF+h3rkCWoe7L/d/xRkc5l5MmjSJWbNm0bhxYyZNmsSgQYM4d+4cGo2G8+fPExoayrRp01i+fDnXr19nzJgxjBkzhhUrVgDFH5hmwoQJzJo1C19fX2rWLL8rhkrLuJHISHX+ZzHb+DTDOjXCt06Nu6xVthR/VCssLIxhw4bRpk0b2rVrx9y5c8nKyrIMzjF06FDq1atn2UObPn06bdq0oWHDhuTm5rJt2za+/vprFi9eDOT/y/36668zbdo0GjduTIMGDXjvvffw9PS0FNEy1fUdOLkJzut50r8/+887cvDCDYY8KB7luZPiDg5zL8aNG0evXr0AeP/992nRogXnzp3Dz8+P6dOnM3jwYMveYuPGjZk3bx5dunRh8eLF2NraFntgmg8++MByTrg66Zb8NTWkHI6ZG/Bnra588kjpjlJfEooXuAEDBnD9+nUmT55MYmIiAQEBREREWC4SxMfHo1L9cySdlZXF6NGjuXTpEnZ2dvj5+fHNN98wYMAAyzLjx48nKyuLUaNGkZqaSseOHYmIiCifPq9q+cIDQ+H35YQkfs443uTAhRRkWVZmV93GPn9PSgk2xR/nsriDw9yLogaM8fPz448//uDYsWOsXr3asowsy5jNZuLi4mjWrFmxB6Zp06ZNqWWuLOTUeHobfgYJXPt8yLZWndFplB+nRPECB1gOBQrz34sH06ZNY9q0aXdsT5IkPvjgAz744IPSimidzuPh6Focrx+hh81Rfs1szYXkLBoqsbsuSVYdJlZldxowJjMzkxdffJFXX331tvW8vb2tGvzlvwPHVAep26ZRUzISZW7OAy1DKkRxg0p6FbXCc/KAoFEATNJ9h4SZAxduKByqYivO4DBl6YEHHiA2NpZGjRrdNmm12nIZmKayMl07g9Of3wKw1/sVdDYVYr8JEAWu7Dz0OuicqW+M43FVFAcvpCidqEIrzuAwZentt99m//79jBkzhqNHj3L27Fm2bNliObIoj4FpKqtLGyehxsxvchuGPP2U0nEKEAWurNjXgofGAhCm+Y7D55PE/XB3cbfBYcpSy5Yt2bVrF3/++SedOnWidevWTJ482XKRozwGpqmMUs4dpH7SdsyyRHqHCRWuH0Qx6EwhrBl05o5yM5HnBSBlXWdS3nM8/8Y0GriW7fmZqjToTEUZHKaqKo2/lVMzg2mWfZiduq50fnsT6nJ6zlQMOlMR6GogdX4LgFc133P47CWFA1VsFWFwGKH4/tjzI82yD5Mnq/Hs+0G5FTdriAJX1gKHk6bzwE1KxfbIl0qnqfAKGxzm1KlTdxxYRih/OQYjmp355yCP1u1D02Yt77KGMirO5Y6qSqMj6YE3cI4az8PXVyPfnIhkV33ubrdG69atCx0c5ubNm3ccWEYofxHfr6Cv+Qw5aGk+sOJebBEFrhx4dRnB2f2f0Vi6TJr+U5wfU+j+vErKzs6uQgwsI+S7kJRGs1OfgQRXmg7Dt3b59dBrLXGIWg7sbLVscsl/9Mw+Zilklv39U+LakXA3Jf0bidy4iKZSAtmSAw36TCrlVKVLFLhyomr2OEfNDbEx3YTdZXd7gVqdfwf5f++uF4T/ys7OHxjp30943M3hc4kEJ+afS85uOwbJvmKfbhGHqOUkqGFtZu4ewBrtR8i/L0dq/wrULP0H8DUaDfb29ly/fh0bG5sCz/EKAuTvuWVnZ3Pt2jVcXFws/yjejdksE71pLi+prpOhqYVr8GtlnPTeiQJXTgLr1+QQ/uw1taAjJyFyBjyxuNTfR5IkPDw8iIuL46+//ir19oWqw8XF5Y79Lv7XzzEXeDJzLUhA53GV4hlnUeDKib1WQ8v7nPkkYQAd1ZPh2Dp46DWo61fq76XVamncuLE4TBWKZGNjU+w9N4CcPBN//fwpvaRU0nSeOHd4oQzTlR5R4MpRkG9tFsc34phjJ1pm7IHfpsLA1XdfsQRUKlWlf5JBqDjW7T7OM3nfgwR2Pd4FjVbpSMUiTtCUowd9awMw0/A0SCo4vRUu3X7flyBUJH9nGTDsmYuLlEWaYyO0rQcqHanYRIErR4H1a6JWSexNcyXL7/+9LujfVzaUINzFl78c4Fk5vyvyGj2ngKpi9PVWHKLAlaMaOg3+9ZwBmJjSC1llA3G74EKkssEEoQhxyVnUjZmPvZRLhmsA6maPKR3JKqLAlbNxPZpir1Xzw182fCv9v99+/QdWj0AlCOVhxU+RDFTpAXB89IP8HqIrEVHgylnHxq5sfuUhfF0dmJn1OFmyDi5HI5/6UeloglBA7JV0Wp1bjFYykVWvE/h2UTqS1USBU0ATN0e2jHmIwBZNWG7KH309act75OSK2zqEimP9tl95QpU/rqvDo5XzXHGFKHALFy7Ex8cHW1tbgoKCOHToUJHLLlu2jE6dOlGzZk1q1qxJcHDwbcsPHz4cSZIKTKGhoWW9GVZxtLVhybOB1Hj4DVJlB9xzL7Jo3kdc+jtb6WiCwPFLaXT4azEqSSbT91GoF6h0pBJRvMCtX7+esLAwwsPDOXLkCK1atSIkJKTIAT0iIyMZNGgQO3fuJCoqCi8vL3r06MHly5cLLBcaGsrVq1ct09q1a8tjc6wiSRIjggNIfeAVAPpnfs1LK6IwGM0KJxOqu81bNxOi/h0zqvwrp5WU4gVuzpw5vPDCC4wYMYLmzZuzZMkS7O3tWb58eaHLr169mtGjRxMQEICfnx9ffPEFZrMZvV5fYDmdToe7u7tlqsgjjPv0fAOTgxv3SckE3tjC0t3nlY4kVGPRf/3NI5c/B8i/nalO6Y1NW94ULXAGg4Ho6GiCg4Mt81QqFcHBwURFRRWrjezsbPLy8qhVq1aB+ZGRkdStW5emTZvy8ssvc+NG0cP25ebmkp6eXmAqV1p71A+PB2CMZhNf/HaCuOSs8s0gCP/364/reEh9EqNkg2Poe0rHuSf3VOAyMzPvqTAkJydjMpkso9jf4ubmRmJiYrHaePvtt/H09CxQJENDQ/nqq6/Q6/V8/PHH7Nq1i549e2IymQptY/r06Tg7O1smLy8vq7ajVLQeilzThzpSOoPlbaw7FF/+GYRq7+D5ZHpeWwrAzZZDwcVb4UT3xuoCFxcXR69evXBwcMDZ2dlyst/FxaXcDwNnzJjBunXr2LRpU4HnLgcOHEjv3r3x9/enb9++bN26lcOHDxMZGVloOxMnTiQtLc0yJSQklNMW/ItGi9Q1v/PAFzVbOXxKHKYK5UuWZfb+uIIA1QVyVXY4dp+odKR7ZvXD9s8++yyyLLN8+XLc3NzuaVBeV1dX1Go1SUlJBeYnJSXdtRuXWbNmMWPGDHbs2EHLlnce8MLX1xdXV1fOnTtHt27dbntdp9Oh0+ms34DSdv9TmPZ8itP1WLr/vY6ElEfwqmWvdCqhmth/9hq9U1aACgxtXkJXo47Ske6Z1QXujz/+IDo6mqZN7/3Eo1arJTAwEL1eT9++fQEsFwxujShemJkzZ/Lhhx/yyy+/0KZNm7u+z6VLl7hx4wYeHh73nLlMqVSogyfD2oEMV//Cj3+cxKtrW6VTCdXE6V+W8ZDqMtlqJxwfeUPpOKXC6kPUtm3bluohXFhYGMuWLWPVqlWcOnWKl19+maysLEaMyB/DYOjQoUyc+M+u8scff8x7773H8uXL8fHxITExkcTERDIzM4H884JvvfUWBw4c4OLFi+j1evr06UOjRo0ICQkptdxlpkkoiU4tsZMM1I7+TOk0QjURE5dISPIKALLajgVbZ4UTlQ6r9+C++OILXnrpJS5fvsz9999/W3/udztc/K8BAwZw/fp1Jk+eTGJiIgEBAURERFguPMTHxxfodnvx4sUYDAaeeuqpAu2Eh4czZcoU1Go1x44dY9WqVaSmpuLp6UmPHj2YOnVqxTgMvRtJwtj1PdjyNJ0ztpGdeBZ798ZKpxKqMFmWid44h+elZNI0rtTpNlbpSKVGkq0cWufAgQM888wzXLx48Z9GJAlZlpEkqcgrlZVJeno6zs7OpKWl4eTkVO7vL8syh6Z2JcgcwxXvx/F87ptyzyBUHxFHztFmS1dcpXTSgz/BqeMopSPdVXG/o1bvwT333HO0bt2atWvX3vNFBqFwkiQR03gsQWeewz1+KySdBLcWSscSqqBco4mEn2cTKqXzt60XNduPUDpSqbK6wP3111/88MMPYiDeMtYssDNbY4N4TH0QWf8B0jPrlY4kVEHrdh1lgGETSGAf8h6oiz+EYGVg9UWGRx55hD/++KMssgj/EtSgFoukgRhlFdKfERB/UOlIQhWTkmXAtGcuTtJN/nZqiq7V00pHKnVW78E9/vjjvPHGGxw/fhx/f//bLjL07t271MJVZ7Y2alq3bst30Z0ZqIlE1r+PNPynStfhoFBxrfh5P6/IP4METo9+AFVwDF2rLzLcaSBhcZGhdF1Lz2HAJ98RoXodnZQHz26ERsF3X1EQ7uL89UwOzhvKM2o96XUCcRqtr1T/eBb3O2p1yTabzUVOVaG4VSR1nWx5vFNbvjblFzV5xwdgFl0pCffuyy16nlZFAuD02LRKVdysYVWBy8vLQ6PRcOLEibLKI/zHC519WaN9ikzZFinxDzi1RelIQiW3/3wyQX8twUYykeXdFep3UDpSmbGqwNnY2ODt7S321MqRo60NQ7sF8oXpUQDM+mlgMiqcSqiszGaZNVt+oo96PwAOPStnV+TFZfUh6qRJk3jnnXdISUkpizxCIZ4Jqk+E41OkyDVQpZyDP9awav9F2k/Xc+5aptLxhErk+5jLPPF3/iNZuU37gEcrhROVLasvMrRu3Zpz586Rl5dH/fr1cXBwKPD6kSNHSjWgEirKRYZ/++GPK/yxYRrv2azG5OhJ8+sfkYuWBq4O7Bz3sNLxhEri5Y8WstjwDmZJjeqVQ+BaOe9nLbMnGW71+iGUr8f8PVi56ymu3PgZz4wrPKvewZemR0XPv0KxrTv4F8NzvgIVpDftj0slLW7WsLrAhYeHl0UO4S5UKolxj7bksxX9+NhmGa9otrDO1JUs7EjOzMW1RiXoSEBQVPIf2xioOk2ubINt93eUjlMuSnxnX3R0NN988w3ffPMNMTExpZlJKEKHRq4k+fbjvNmDWlIGz6u3AbDvXLLCyYQKz2wmNGkZAIlNn8W2duXuiry4rC5w165d45FHHqFt27a8+uqrvPrqqwQGBtKtWzeuX79eFhmFfxnfswWfmvIfqXles42apLP7T1HghDuTYzfTyHSeDNmOvA5VozPL4rC6wI0dO5aMjAxOnjxJSkoKKSkpnDhxgvT0dF599dWyyCj8S3NPJ3T+T3DC7IOjdJPRmh/Yc/Y6Vl4rEqoTkzH/9iLgC+Oj3FfvPoUDlR+rC1xERASLFi2iWbNmlnnNmzdn4cKF/Pzzz6UaTijc6z38+NQ8EICh6u2oM67wZ5K4XUQoaOHOc8z65QwcXY367/PckB35wf4JbG3USkcrNyV6VOu/D9hD/k3AZvEYUbnwqmVP36eGcrFGADopj1c137P7T3F6QPhHtsHIJ7+cYdnOWEw7ZwCwyNiHOrVdFU5WvkrUXdJrr73GlStXLPMuX77MG2+8UeiIVULZeDygHj79ZwLwtHoXf8bGkJpt4PH5e5nyw0mF0wlKu5FpAOBZ9XbUmVfI0LnxjSkY79rVa5Q2qwvcggULSE9Px8fHh4YNG9KwYUMaNGhAeno68+fPL4uMQlG8g8isH4xGMtP1ylIW7zrP8ctprNx/kb+zDEqnExSUnJlLDbJ5RZP/7PJ8Uz9y0VK/mg1DafV9cF5eXhw5coQdO3Zw+vRpAJo1a1ZgZHmh/DiEhsPnO3hUdYCFu/VAAwAi/7zGE62rz8lkoaAbmQae12yjlpTJebMHX2a2BxB7cMUhSRLdu3dn7NixjB079p6L28KFC/Hx8cHW1pagoCAOHTpU5LLLli2jU6dO1KxZk5o1axIcHHzb8rIsM3nyZDw8PLCzsyM4OJizZ8/eU8aKSvJoSYxz/uf/lmaDZf6O2GtKRRIqgMSrCZb7JGcbn8ZE/oWF+rUd7rRalWP1HhyAXq9Hr9dz7dq12y4sLF++3Kq21q9fT1hYGEuWLCEoKIi5c+cSEhLCmTNnqFu37m3LR0ZGMmjQIDp06ICtrS0ff/wxPXr04OTJk9SrVw/IHxh63rx5rFq1igYNGvDee+8REhJCbGwstra2JdnkCi2l3Tjyft3Jw+o/aGc8xSG5Gbv+vI7BaEarqXq9tAqF+/n4VdYeTuCtHk2pHbOQGlIOx80+/GxuZ1mmuh2iWv2w/fvvv88HH3xAmzZt8PDwuG1UrU2bNlkVICgoiLZt27JgwQIg/yqtl5cXY8eOZcKECXdd32QyUbNmTRYsWMDQoUORZRlPT0/efPNNxo0bB0BaWhpubm6sXLmSgQMH3rXNiviw/Z2kZBn4efpABmv0HFf5MUKaRnKWgfWjHiTIt7bS8YRykJNnwu+9CABaOWWyIfcVdFIeY1Tv8vaYV1i25wI1dBrGh/opnLR0lNnD9kuWLGHlypUMGTLkngICGAwGoqOjC4xcr1KpCA4OJioqqlhtZGdnk5eXR61atQCIi4sjMTGxwGGzs7MzQUFBREVFFVrgcnNzyc3Ntfyenp5e0k1SRC0HLfOMT9BPvRt/82n6uZzk86zGpIgLDdVGYlqO5edB2WvQafKIUbVg/rtvIqlUfNDnfgXTKcfq4xeDwUCHDqXTA2hycjImk8kyiv0tbm5uJCYmFquNt99+G09PT0tBu7WeNW1Onz4dZ2dny+Tl5WXtpijuodb+rDSFADA4axUSZjJzRceY1cXV/xc4X+kKT6t3AXC44atIVXAgGWtYvfXPP/88a9asKYssVpsxYwbr1q1j06ZN93RubeLEiaSlpVmmhISEUkxZPj5+qiX1e7+DrHPEO+8Cj6sOkCUKXLWRmH4TgI9ctqCWZPap29LpkUcVTqU8qw9Rc3JyWLp0KTt27KBly5a3PdUwZ86cYrfl6uqKWq0mKSmpwPykpCTc3d3vuO6sWbOYMWOGJcctt9ZLSkrCw8OjQJsBAQGFtqXT6dDpKnd3QzZqFT3btYCbr8HOaYRpvmVefF94qIHS0YRycDUthxZSHA/e3ANIPDTqM3Cr+OePy5rVe3DHjh0jICAAlUrFiRMniImJsUxHjx61qi2tVktgYCB6vd4yz2w2o9frad++fZHrzZw5k6lTpxIREUGbNm0KvNagQQPc3d0LtJmens7Bgwfv2GaV8eDL5Nm64qNKwunUOm4axPgZ1UFSWg7jNevzf/F/GtxaKBuoopAVtm7dOlmn08krV66UY2Nj5VGjRskuLi5yYmKiLMuyPGTIEHnChAmW5WfMmCFrtVr5u+++k69evWqZMjIyCizj4uIib9myRT527Jjcp08fuUGDBvLNmzeLlSktLU0G5LS0tNLd2HJijloky+FO8tXJ9eVtRy4oHUcoB58s+UKWw51k05RasnzjvNJxylxxv6Mlug+uNA0YMIDr168zefJkEhMTCQgIICIiwnKRID4+vsBg04sXL8ZgMPDUU08VaCc8PJwpU6YAMH78eLKyshg1ahSpqal07NiRiIiIKnkPXGGkNs+R9ttnuBuukr1nEbT+ROlIQlmSZXpdXwrAZd/+eNXyVThQxWH1fXDVQWW7D64wVyK/wDPyTf6Wa6B+4xhOLuJ+uKrKdOon1Ouf4aas5a9n9+HXuInSkcpcmY1sL1QOHp2G85fKi5pSJpd/+ljpOEIpO389k/bT9Xy9/wJ/b50MwBp64uUt9t7+TRS4KkpSazjRdCwADc6tgkzxbGpVMmHjMa6m5fD71mW4Zp0jXbanbs+3cdApftapQrG6wO3evRuj8fb7q4xGI7t37y6VUELpaNb1GY6afbGVc0j7dYbScYRSlJqdhw1GwjTfAfBb7YE8FtRc4VQVj9UFrmvXroWOap+WlkbXrl1LJZRQOnzrOvKt83MA2P6xirnf7eD4pTSFUwmlISk9hwHqndRXXeO67Iz2oVduey5cKEGBk2W50A/yxo0bt41yLyivffd+7DO1QCcZqXf0Mx5fsFcMUFMFGHKyGKvJ79hivrEvTbzc7rJG9VTsA/Ynn3wSyO8Lbvjw4QXu/DeZTBw7dqzUnlEVSs9jLT1Ze24cDx0bwZPqPXxueoyLN7Jp4Cr+MarMhqt/wU1KJcFch7Wmbkx0qV7dIBVXsQucs7MzkL8H5+joiJ2dneU1rVbLgw8+yAsvvFD6CYV75tK4Pb/EtCFE/Ttvar5l77lQUeAqs5upvKT5EYBPjf3IQ4OdtvqMlGWNYhe4FStWAODj48O4cePE4Wgl0tTdkReN/emuiqan+jDvRkfCg8OUjiWUkLxvHi5SFn+a67HZ3FHpOBWa1efgwsPDRXGrZOrXduCsfB+b/v9lCElcSlxylsKphBLJSEI+sAiA2cb+mMWdXndk9aeTlJTEkCFD8PT0RKPRoFarC0xCxaNW5V8U+tT4FAZZTSf1CaJ2fK9wKqFE9sxCZbzJUXNDfjHndzQhuqUvmtV3BQ4fPpz4+Hjee++9QrssFyqmoAa1OBgHO+wf5dGbP+J/ei45hkHYasWNoZXG33/B7/mnimYaB+Dn7sSF5CyWPPuAwsEqLqv/uvfu3cuePXuK7FtNqJiWDm3DwQs36Oz5ADc/+xV/zrHv12946LHhSkcTiityBpjz2G++n/3m+/l1UGsauDpgoxZ7cEWx+pPx8vIS91FVQs52NvRo4Y5tTQ9ivQcDcF/MbGST6PW3Urh2Go6tA+DjvP74ujrQuG4NUdzuwupPZ+7cuUyYMIGLFy+WQRyhPDTs8w6psgP1TfFc3LlC6ThCcfw2FWQzB3Qd+ENuRP+2XuL0UDFYXeAGDBhAZGQkDRs2xNHRkVq1ahWYhIrPpXYd9rk9C4DTwVlgFKNvVWTG+MNweisyKt5N74taJfFk63pKx6oUrD4HN3fu3DKIIZS3+3q+QdLK9bjlJWI4tBxth5eUjiQUYsPvCTT4KYy2wEZTR87J9xHsV4e6TtWj89Z7ZXWBGzZM3CBaFbT0cWe2dgDj8j5H3vUJtBmCbGNPeo4RZzubuzcglLlsg5EfN63la5tjGGQ1c439AHi6TeUb1lIpJTpDef78ed59910GDRrEtWv5/Yz9/PPPnDx5slTDCWVHkiRMrZ4l3lwHXW4ysZs+psHEbbR6/1cOxd3eW4xQ/q6m3uRNdf6FBVXbkTwb2onhHXzo5ldX4WSVh9UFbteuXfj7+3Pw4EG+//57MjMzAfjjjz8IDw8v9YBC2ekZ4M0c49MA3Be7DGfy/1+uPviXkrGE/8s78QMBqvPcxBbNw2/xUpeGTOndAo24clpsVn9SEyZMYNq0aWzfvh2tVmuZ/8gjj3DgwIFSDSeULf96zvxg7sBpsxdOUjYvarYC4OtaQ+Fk1ZvBaObTX09R61D+YEG/OD4BNcReW0lYXeCOHz/OE088cdv8unXrkpycbHWAhQsX4uPjg62tLUFBQRw6dKjIZU+ePEm/fv3w8fFBkqRCL3hMmTIFSZIKTH5+flbnqg4kSWLHm1054fcqACPUEdThb8QOgrJmbz/DpciV1M2JI1V24Pd6zyodqdKy+k/ZxcWFq1ev3jY/JiaGevWsu3S9fv16wsLCCA8P58iRI7Rq1YqQkBDLeb3/ys7OxtfXlxkzZlhGsC9MixYtuHr1qmXau3evVbmqE986NXhq0AtEmxtjJxkYq9lMrtGsdKxqTX88gTds8rsiX2zsTc1adRROVHlZXeAGDhzI22+/TWJiIpIkYTab2bdvH+PGjWPo0KFWtTVnzhxeeOEFRowYQfPmzVmyZAn29vYsX7680OXbtm3LJ598wsCBAwt0uPlfGo0Gd3d3y+Tq6mpVrmpHkvjKPv//3SD1b9hlJigcqHqKif+bHp/u4qG0rdwnJZMku7DK1AN3Z3FLSElZXeA++ugj/Pz88PLyIjMzk+bNm9O5c2c6dOjAu+++W+x2DAYD0dHRBAcH/xNGpSI4OJioqChrYxVw9uxZPD098fX1ZfDgwcTHx99x+dzcXNLT0wtM1U2vx/uz2+SPjWTioYSlSseplkau+p1LScmM+X9X5POMT5KDDg9R4ErM6gKn1WpZtmwZ58+fZ+vWrXzzzTecPn2ar7/+2qrukpKTkzGZTJYR7G9xc3MjMTHR2lgWQUFBrFy5koiICBYvXkxcXBydOnUiIyOjyHWmT5+Os7OzZfLyqn73GfVo4c5Z/zAA/P/+FZLELT/lLTPHmH8eVErnotmN9aaHAXB3srvzikKRStxXjre3N97e3qWZpVT07NnT8nPLli0JCgqifv36bNiwgZEjRxa6zsSJEwkLC7P8np6eXi2LnOwRwE/H29FLfQhZPxXpmXVKR6pW6thk86KcfyV7sTQA4/+/nl61RIErqWIVuLCwMKZOnYqDg0OBQlCYOXPmFOuNXV1dUavVJCUlFZiflJR0xwsI1nJxcaFJkyacO3euyGV0Ot0dz+lVFx0auvKGuT+hqsOo//yZ04d38NpeLZN6NaNzE3Giuyxl5hoZZtqEkzqbU2Zv2jz+POs3nsC7lj2OtuLJkpIqVoGLiYkhLy/P8nNRrOndQKvVEhgYiF6vp2/fvgCYzWb0ej1jxowpdjt3k5mZyfnz5xkyZEiptVlVNfd04rWBj7Fxw1b6qyP5+4f3OJM3iaHLD3FxRi+l41VJOXmm/M837jy7dREAfKF9llltvHG00xJYX3RgcS+KVeB27txZ6M/3KiwsjGHDhtGmTRvatWvH3LlzycrKYsSIEQAMHTqUevXqMX36dCD/wkRsbKzl58uXL3P06FFq1KhBo0aNABg3bhyPP/449evX58qVK4SHh6NWqxk0aFCp5a7KHvX34KfUSeTu2Et7dSydTMfZY26pdKwqa8bPpzkUl8I0zffYSnn8bm7CY08NR5IkQu/3UDpepadof9UDBgzg+vXrTJ48mcTERAICAoiIiLBceIiPj0el+uc6yJUrV2jdurXl91mzZjFr1iy6dOlCZGQkAJcuXWLQoEHcuHGDOnXq0LFjRw4cOECdOuIQq7h6dWpH4pWhuJ9azlua9ewx+Bc54LdQcicup7Fy/0W8pSQGqCMB+FI3hE8bituaSoskF6N73luDPhfH999X/sFM0tPTcXZ2Ji0tDScnJ6XjKCMrmdzZ/ujM2bxseI33J75DXUdxu0Jp8pnwEwBzbRbQV72fSFMrpCEb6SLOd95Vcb+jxbpN5N+3UDg5OaHX6/n9998tr0dHR6PX6y2DQwtVgIMr2o7550LHaTZwLjFV2TxVlJ8UT29V/n2f37k8J4pbKSvWIeqtQZ8B3n77bfr378+SJUss972ZTCZGjx5dffd2qiipw1jSdi+moeoqS1bOpsOHnygdqcowm/MPnMZp1qOSZLaaHiR8lDhPXNqsvtF3+fLljBs3rsBNvWq1mrCwsCIfsRIqKVsnltIXgNc1G7mYeEPZPFXI6NVHCJTOEKyOwSirmG18mtoO2ruvKFjF6gJnNBo5ffr0bfNPnz6N2Swe0q5qTtZ7mqtyLepJN/jhy2ncyMxVOlKVEHHyKuNt1gPwrakLcbIHKpW4iFParC5wI0aMYOTIkcyZM4e9e/eyd+9eZs+ezfPPP2+5vUOoOqY80YbPjPkXmQYbvqX37AiFE1V+OXkmuqiOEaQ6Ta5sw2fGJ2nvW1vpWFWS1beJzJo1C3d3d2bPnm3pNsnDw4O33nqLN998s9QDCsrycXVg2pTpXJi6FV9VIv0MPwB9lI5VqSXcyOQtTf7e21em7oR2CGTyY80VTlU1Wb0Hp1KpGD9+PJcvXyY1NZXU1FQuX77M+PHjrXrYXqg8NDZaS9fmL2h+gixxLu5ebFmziPtVF8mQ7Vhk7M2U3i3E4WkZuae+W52cnMSV02riJ3MQJ831cZRuYtozh+dWHubjiNvPxQp3YTLydPpXAHxhfJQlo3ooHKhqK9GTDN999x0bNmwgPj4eg6HgoMFHjhwplWBCxSKj4hPjAFZqZ8KhpcRmN+W307V5O1R0B2+Vo6upzxVS5Bo07D2eIHHurUxZvQc3b948RowYgZubGzExMbRr147atWtz4cKFAl0VCVWLSoJIcysOmv1Qmw28qsl/YsUgujcvvrwcTDtnALDQ2Icm9T0VDlT1WV3gFi1axNKlS5k/fz5arZbx48ezfft2Xn31VdLS0soio1AB7J/QDZCYmTcAgP7qXTSQrjJ27RGupecoG66C+zMpg26zIzm2ZTbqzCtckWvxjak7PrUdlI5W5Vld4OLj4+nQoQMAdnZ2lp5yhwwZwtq1a0s3nVBhuDvbcnFGLzQ+7dGbWqORzIRpvuWXk0l8tO2U0vEqtPd/PEni9WTqHV8MwGfGfuSixdZGXJQra1YXOHd3d1JS8kc+9/b2toyFGhcXRzGe2xcquafbeDHL2B+Ax9UHaCFdZPPRK+JQ9Q5SsvJ4Xr2N2lIG580efGfqzKvdGisdq1qwusA98sgj/PDDD0D+Tb9vvPEG3bt3Z8CAAYWOlypULX0DPDkl12eLKX8vftz/7+f65WTJx9GoymRZxpR5nec12wCYY3yajk3cCeveROFk1YPVV1GXLl1qeSTrlVdeoXbt2uzfv5/evXvz4osvlnpAoWLRqFXEfhBCz/BEeqkO0FX9B22Npxm7Fh5vJU6a/1d8SjZP3/wWR81NTph92GZux+J2FW8sk6rKqj04o9HItGnTCox6NXDgQObNm8fYsWPRasXDwtWBvVbDX7I7G/4/6tN4m3WAzF83shTNVREdO3mSoertAMxnELVr2NGtWV2FU1UfVhU4jUbDzJkzMRqNZZVHqCQGtvXiM+OT5Mg2tFX9ycOqo2yPTbr7itWIySxjs+8TdFIeCY6tmRL2Ktvf6IyN+p7urxesYPUn3a1bN3bt2lUWWYRKZErvFswb9SirTPl34o/XbEAfe1XhVBXLF5t+JThnBwA3HpyIh4s9NUWXSOXK6nNwPXv2ZMKECRw/fpzAwEAcHArey9O7d+9SCydUXLY2aoJ8a/OisTeD1L/RXPUXzVL0wENKR6sQMnLyqHd0Dhq1mR2m1rQLfETpSNVSscZk+Ld/DwJzW2OShMlkuudQShNjMhTfpE3HqfX7XN60+Y4EyQOvd4+DWozjuWD1d4w5OxKzLPGoYToR019WOlKVUqpjMvyb2WwucipJcVu4cCE+Pj7Y2toSFBTEoUOHilz25MmT9OvXDx8fHyRJYu7cuffcpnBvPuhzP7adxpAsO+ElX8V85BulI1UIQXELAfjB3J7TsrhqqhRFz3auX7+esLAwwsPDOXLkCK1atSIkJIRr164Vunx2dja+vr7MmDEDd3f3UmlTuDdqlcTgzi34XO4LwLWt7zP/l2PKhlLaxX20NR4hT1ZbupkSlFHsQ9SbN2+i1+t57LHHAJg4cSK5uf90X61Wq5k6dSq2tsUfWi4oKIi2bduyYMECIH/v0MvLi7FjxzJhwoQ7ruvj48Prr7/O66+/fs9t5ubmFtiW9PR0vLy8xCGqFbbFXKTl5m7cJyXzYd4zTPpwsdKRFGEymTn/8UM0McTyjbEb7xpH8nDTOqwc0U7paFVKqR+irlq1is8//9zy+4IFC9i/fz8xMTHExMTwzTffsHhx8f+oDQYD0dHRBAcH/xNGpSI4OJioqKhit1MabU6fPr3A0IheXl4lev/q7NHWPuy/73kARmt+4O+UZIUTKSPx9y00McRyU9Yyz/gk40ObsnRIG6VjVVvFLnCrV69m1KhRBeatWbOGnTt3snPnTj755BM2bNhQ7DdOTk7GZDJZRrG/xc3NrcCNxNYoaZsTJ04kLS3NMiUkJJTo/au7p58bxzmzJzWlTFK2z1Y6Tvkzm6mxbzoAq0whXKMmz3f0RasR970ppdif/Llz5/D397f8bmtrW+CKart27YiNjS3ddOVEp9NZeicWvRSXnKS24WCD0QB4nVkBmdcVTlTOTmzEOf0M6bI9i42P88XQNqK4KazYn35qamqB81TXr1/Hx8fH8rvZbC7w+t24urqiVqtJSip493tSUlKRFxCUaFOwjlPrJ/nD7IvWfJOMHTOUjlN+THkYdkwF4HPjY3z4TCeCm7vdZSWhrBW7wN13332cOHGiyNePHTvGfffdV+w31mq1BAYGotfrLfPMZjN6vZ727dsXu52yblOwTvcW7izVDAZAG7OSJz5ay6LIcwqnKgdHvkKb/hfXZSdWmELp0Vz8g1oRFLvAPfroo0yePJmcnNt7b7158ybvv/8+vXr1surNw8LCWLZsGatWreLUqVO8/PLLZGVlWcZXHTp0KBMnTrQsbzAYOHr0KEePHsVgMHD58mWOHj3KuXPnit2mULZsbdR0Ce3PflNzdJKRQdlrmRlxRulYZcuQTY4+/9zbAuMTTHpCHJpWGHIxJSYmyu7u7rK3t7c8c+ZMefPmzfLmzZvljz/+WPby8pI9PDzkxMTE4jZnMX/+fNnb21vWarVyu3bt5AMHDlhe69Klizxs2DDL73FxcTJw29SlS5dit1kcaWlpMiCnpaVZvT1CvulLV8lyuJNsnOwsPzLhc6XjlKnEbTNkOdxJTpjsKzd+e5O87+x1pSNVecX9jlr1qFZcXBwvv/wy27dvt/TeK0kS3bt3Z9GiRfj6+pZ+BVaAeFTr3n34UyxtD4yhhzqabaZ2hL7/a9Uc+/NmKoY5LdHmpfGm4SU2yZ2J/SBUdEdexor7HbXqYfsGDRoQERFBSkqK5bCwUaNG1KpV697SClWOaw0ds4z9CVYd4VH1ITLjDlOjYdW62fXghRskbHyHp/LS+NNcj03mjux4s4sobhVIiU4U1KpVi3bt2tGuXTtR3IRCedey50/Zi03m/N5FLmx4m4SUbIVTla4pa3fSM3MTALON/Xk1uCm+dWoonEr4N3EmVCgTofe7M6GnH1/bPoNBVtMy9wiHftuidKxSI8sy/W9uwEHK5ai5Ib+Y29DAVQwDWNGIAieUCUmSeKlLQza98wzH3fMHI2p7YR5UgZHXMnLyWLJlJ4PV+Z1ZzjQOACT86zkrG0y4jShwQpmSJImE+18hW9bhnR3LyZ1rOXmlcg8QPnLV79SJ/hStZGKvqQX7zffTv8194vC0AhIFTihz9zdtzApTCACayA95fN5uhROVXE6eiZSLx3hCtQeAs/5hfDMyiI/7tVQ4mVAYUeCEMteoriPX/F8kTbanqeoSfVT7yMqtnAMXXUvPZZzmW9SSTEaDUIb0e5KOjV2RpCp4C0wVIAqcUC7efaoDS4z543W8ofmO32IvKZyoZDIvHCBUfRgTKhx7vo9GjJBVoYn/O0K5sFGrWGEK4ZrsgrfqOoe++1TpSFbLzDWS8uN7AOyxfQTq+imcSLgbUeCEctPSx4N5xvwrqq9qNpGdWbkuNuyK+I6OqhMYZDXfOw9ROo5QDKLACeVm2dA25PoPJt5chzpSGgumv6V0pOKTZRofz+/Ec42pGxeMtRUOJBSHKHBCuXG2t+GTgW0tA7G8qPmRjlO+5/z1TIWT3d3N4z/QxPgn2bKOBcYncLYTQyNWBqLACeUuzj2U02YvnKVsBhk30W32LqUj3VFyejYJ3+V327XcFEqN2h6EP95C4VRCcYgCJ5S778d0Zvb/9+JGqH+hDqlY0alNuYv56XOaqC6TKjuw1PgYkW91pYmbo9KxhGIQBU4od2qVBE0f5Yi5EfZSLmM0m8iooPfFxSX9jd+p/EGclxgf54MBDymcSLCGKHCCIpYNa8snxgEADFL/RtLFitnr7xX9IrxU10mSXXjg6bfp27qe0pEEK4gCJyim7cN92G3yRyuZOLb6bf66kaV0pAKMNzO4/9wyAM75vUyPgKrRoWt1IgqcoJgxjzS27MU9odrHC7O+VjjRP3KNJpZ+Mh5n89/8Za6LXZAY06MyEgVOUIxWo+Lzt59nm6kdKklmnGYD8/VnlY6FLMvM33qIZ02bAVhIf+73rqNsKKFEKkSBW7hwIT4+Ptja2hIUFMShQ4fuuPy3336Ln58ftra2+Pv7s23btgKvDx8+HEmSCkyhoaFluQlCCXm62HG9zThMskQPdTS/7fhJ6UhsPnoZh98X4iRlc8rsxZhXJ4pRsiopxf+vrV+/nrCwMMLDwzly5AitWrUiJCSEa9euFbr8/v37GTRoECNHjiQmJoa+ffvSt2/f28ZsDQ0N5erVq5Zp7dq15bE5QgkEd+nMRlNnAN7SrOfM1XRF83y0PpLh6l8A8HjiQ7xdRT9vlZXiBW7OnDm88MILjBgxgubNm7NkyRLs7e1Zvnx5oct/9tlnhIaG8tZbb9GsWTOmTp3KAw88wIIFCwosp9PpcHd3t0w1a9Ysj80RSqCeix1zjf3IlTV0UMcybf4iRfO8qtmEnWQg2twYl4DeimYR7o2iBc5gMBAdHU1wcLBlnkqlIjg4mKioqELXiYqKKrA8QEhIyG3LR0ZGUrduXZo2bcrLL7/MjRs3isyRm5tLenp6gUkoX8tff4LVpvz/r+M06zGbzIrk+HrbTgaqdwJwoMEYEP28VWqKFrjk5GRMJhNubm4F5ru5uZGYmFjoOomJiXddPjQ0lK+++gq9Xs/HH3/Mrl276NmzJyaTqdA2p0+fjrOzs2Xy8vK6xy0TrOXn7sQa7dNkyTpaqS6g3/RFuWf4Ys8FakR9go1kYpepJaOHDyv3DELpUvwQtSwMHDiQ3r174+/vT9++fdm6dSuHDx8mMjKy0OUnTpxIWlqaZUpISCjfwAIA29/rR7TnMwD4Hv+UtMyb5fr+3277hT6q/QB49psueumtAhQtcK6urqjVapKSkgrMT0pKwt3dvdB13N3drVoewNfXF1dXV8tg1f+l0+lwcnIqMAnlT5IkOg2dQho1aChd4fuVs8vtvfNMZsZpNqCSZLaagmgc0LHc3lsoO4oWOK1WS2BgIHq93jLPbDaj1+tp3759oeu0b9++wPIA27dvL3J5gEuXLnHjxg08PDxKJ7hQZiQ7F350GghA9+sryMwq+6cbDEYzX6xZR3f1EYyyytKdk1AFyApbt26drNPp5JUrV8qxsbHyqFGjZBcXFzkxMVGWZVkeMmSIPGHCBMvy+/btkzUajTxr1iz51KlTcnh4uGxjYyMfP35clmVZzsjIkMeNGydHRUXJcXFx8o4dO+QHHnhAbty4sZyTk1OsTGlpaTIgp6Wllf4GC3eVkJQsX5nsI8vhTvL5H2aW+fst/O1POeq9IFkOd5LXTuot/52VW+bvKdyb4n5HFT8HN2DAAGbNmsXkyZMJCAjg6NGjREREWC4kxMfHc/XqVcvyHTp0YM2aNSxdupRWrVrx3XffsXnzZu6//34A1Go1x44do3fv3jRp0oSRI0cSGBjInj170Ol0imyjYJ376tZmt0f+o1F1ji5Azim7q9pGk5kD27/lQdUpcmUbzvi9gou9tszeTyhfkixX4I64FJKeno6zszNpaWnifJxCjly8Rq0VnfCREjnZdAwtBn1YJu8z+5dT9Ng3EH/VRTZoetP/3YrzPKxQtOJ+RxXfgxOEwjzgU5dDPi8D4HX6S9JTku6yRsmc37UGf9VFMmVbaodOKJP3EJQjCpxQYbUKHcFJc32cpJtsnj+u1NvPyzPwpuZbAL4wPUrHADEMYFUjCpxQYTVxd+ITY38A+pt/JvbM6VJt/+yvS2moukqKXAP3kDfRadSl2r6gPFHghApLkiTeemUMh8xNsZXyOPr1RJLSc0qn8bwcXA7l32e30NiHPkHNSqddoUIRBU6o0FrUc0HdPRyA/upIdhXxjLK1MvYtwVNK4Ypci54j3sNOK/beqiJR4IQKL7BTL34zBaCRzNju/fie2zt2/hKGnbMA+MzYjzaNxA3gVZUocEKl8Kv7KAB6q6M4enh3iduRZRn98snUljI4b/bgiRFvlVZEoQISBU6oFKaPfoYfTPmP4/3943slbif61Dme1+T3AH2s8Ss82MjtLmsIlZkocEKlIEkSAUNnYpRVdFUdJT5Gf/eV/uNaeg4xaybjKN3kpNmHJwa/UgZJhYpEFDih0vBu3JIo50cBSNw0kWvp1nWnFBf3J0PV2wH4tuZzoBJ//lWd+D8sVCouPd8lR7ahneoMq78pvFv7omT98hE6KY+DZj+e7i86s6wORIETKhX/Zs34UfcYAL1vfIFsLryX5v/aHRVF56z8gWQcen5Ai3ouZRVRqEBEgRMqncDBH5Ap29HQdIF9P3xZrHWM+mloJDN6U2vubx9SxgmFikIUOKHS8a3vzRnf/EPM+2LmkJNz56cbdvy2nUeMewGw7zmlrOMJFYgocEKl1Lr/JFJwwke6yp+/fF7kcrIso945FYAtpg482L5LeUUUKgBR4IRKSWXnxBHv5wBwi5lLalpaocsd2/8zXdV/kCerqd9vmhhIppoRBU6otDo9M55rkitupHByy5zbF5BltJH5e2+/1+pFQEBgOScUlCYKnFBp6WwduNTqNQBaXPiSnMy/C7x+8cAmmuXFkiPb4PZYyZ9+ECovUeCESs2/10tclOrhQgY7vpxsmW8ymcj+eQoAu2s+iW/DJgolFJQkCpxQqdnYaLkUEAbAwykbSLl2GYAj276kueov0mU7Wg4MVzKioKAKUeAWLlyIj48Ptra2BAUFcejQoTsu/+233+Ln54etrS3+/v5s27atwOuyLDN58mQ8PDyws7MjODiYs2fPluUmCAp66PHnOGZuQA0ph3MbP8BoyMXzSH5nlnvqDMLdvZ7CCQWlKF7g1q9fT1hYGOHh4Rw5coRWrVoREhLCtWvXCl1+//79DBo0iJEjRxITE0Pfvn3p27cvJ06csCwzc+ZM5s2bx5IlSzh48CAODg6EhITc9X4poXKSVCrMXfPPsbVK/I5dy96knpzIDZzpMkzsvVVrZT9E6521a9dOfuWVVyy/m0wm2dPTU54+fXqhy/fv31/u1atXgXlBQUHyiy++KMuyLJvNZtnd3V3+5JNPLK+npqbKOp1OXrt2bbEyiYGfKx+zySQfnfqQLIc7Waa930xVOpZQRirFwM8Gg4Ho6GiCg4Mt81QqFcHBwUQV0TV1VFRUgeUBQkJCLMvHxcWRmJhYYBlnZ2eCgoKKbDM3N5f09PQCk1C5SCoVGR3fsfx+SXYl8MkwBRMJFYGiBS45ORmTyWQZxf4WNzc3EhMTC10nMTHxjsvf+q81bU6fPh1nZ2fL5OXlVaLtEZTV8eFHiXF8GIDczu9ia2evbCBBcYqfg6sIJk6cSFpammVKSEhQOpJQQi3HrifrhX007DZC6ShCBaBogXN1dUWtVpOUVHDU8qSkJNzd3Qtdx93d/Y7L3/qvNW3qdDqcnJwKTELlpNba4lDvfqVjCBWEogVOq9USGBiIXv9P99Nmsxm9Xk/79u0LXad9+/YFlgfYvn27ZfkGDRrg7u5eYJn09HQOHjxYZJuCIFRR5XTRo0jr1q2TdTqdvHLlSjk2NlYeNWqU7OLiIicmJsqyLMtDhgyRJ0yYYFl+3759skajkWfNmiWfOnVKDg8Pl21sbOTjx49blpkxY4bs4uIib9myRT527Jjcp08fuUGDBvLNmzeLlUlcRRWEiq2431GN0gV2wIABXL9+ncmTJ5OYmEhAQAARERGWiwTx8fGo/tV3focOHVizZg3vvvsu77zzDo0bN2bz5s3cf/8/hyXjx48nKyuLUaNGkZqaSseOHYmIiMDW1rbct08QBOVIsizLSoeoaNLT03F2diYtLU2cjxOECqi431FxFVUQhCpLFDhBEKosUeAEQaiyFL/IUBHdOi0pHtkShIrp1nfzbpcQRIErREZGBoB4ZEsQKriMjAycnZ2LfF1cRS2E2WzmypUrODo6VshBStLT0/Hy8iIhIUFc5f0P8dkUrSp9NrIsk5GRgaenZ4HbyP5L7MEVQqVScd999ykd467EY2VFE59N0arKZ3OnPbdbxEUGQRCqLFHgBEGoskSBq4R0Oh3h4eHodDqlo1Q44rMpWnX8bMRFBkEQqiyxBycIQpUlCpwgCFWWKHCCIFRZosAJglBliQJXSaSkpDB48GCcnJxwcXFh5MiRZGZm3nGdpUuX8vDDD+Pk5IQkSaSmppZP2DK2cOFCfHx8sLW1JSgoiEOHDt1x+W+//RY/Pz9sbW3x9/dn27Zt5ZS0/Fnz2Zw8eZJ+/frh4+ODJEnMnTu3/IKWE1HgKonBgwdz8uRJtm/fztatW9m9ezejRo264zrZ2dmEhobyzjvv3HG5ymT9+vWEhYURHh7OkSNHaNWqFSEhIVy7dq3Q5ffv38+gQYMYOXIkMTEx9O3bl759+3LixIlyTl72rP1ssrOz8fX1ZcaMGUUOyFTplW3P6UJpiI2NlQH58OHDlnk///yzLEmSfPny5buuv3PnThmQ//777zJMWT7atWsnv/LKK5bfTSaT7OnpKU+fPr3Q5fv37y/36tWrwLygoCD5xRdfLNOcSrD2s/m3+vXry59++mkZplOG2IOrBKKionBxcaFNmzaWecHBwahUKg4ePKhgsvJlMBiIjo4mODjYMk+lUhEcHExUVFSh60RFRRVYHiAkJKTI5Surknw21YEocJVAYmIidevWLTBPo9FQq1YtEhMTFUpV/pKTkzGZTJYBiW5xc3Mr8nNITEy0avnKqiSfTXUgCpyCJkyYgCRJd5xOnz6tdExBqLREd0kKevPNNxk+fPgdl/H19cXd3f22E8VGo5GUlJSqe3K4EK6urqjVapKSkgrMT0pKKvJzcHd3t2r5yqokn011IPbgFFSnTh38/PzuOGm1Wtq3b09qairR0dGWdX/77TfMZjNBQUEKbkH50mq1BAYGotfrLfPMZjN6vZ727dsXuk779u0LLA+wffv2IpevrEry2VQLSl/lEIonNDRUbt26tXzw4EF57969cuPGjeVBgwZZXr906ZLctGlT+eDBg5Z5V69elWNiYuRly5bJgLx79245JiZGvnHjhhKbUCrWrVsn63Q6eeXKlXJsbKw8atQo2cXFRU5MTJRlWZaHDBkiT5gwwbL8vn37ZI1GI8+aNUs+deqUHB4eLtvY2MjHjx9XahPKjLWfTW5urhwTEyPHxMTIHh4e8rhx4+SYmBj57NmzSm1CqRMFrpK4ceOGPGjQILlGjRqyk5OTPGLECDkjI8PyelxcnAzIO3futMwLDw+XgdumFStWlP8GlKL58+fL3t7eslarldu1aycfOHDA8lqXLl3kYcOGFVh+w4YNcpMmTWStViu3aNFC/umnn8o5cfmx5rO59Tfz36lLly7lH7yMiO6SBEGossQ5OEEQqixR4ARBqLJEgRMEocoSBU4QhCpLFDhBEKosUeAEQaiyRIETBKHKEgVOEIQqSxQ4QRCqLFHghEpj+PDhhXYpFRoaqnQ0oYIS3SUJlUpoaCgrVqwoME+n0xW6bF5eHjY2NgXmGQwGtFqt1e9b0vUEZYk9OKFS0el0uLu7F5hq1qwJgCRJLF68mN69e+Pg4MCHH37IlClTCAgI4IsvvqBBgwbY2toCEB8fT58+fahRowZOTk7079+/QF9qRa0nVC6iwAlVypQpU3jiiSc4fvw4zz33HADnzp1j48aNfP/99xw9ehSz2UyfPn1ISUlh165dbN++nQsXLjBgwIACbf13PaHyEYeoQqWydetWatSoUWDeO++8Yxka8ZlnnmHEiBEFXjcYDHz11VfUqVMHyO/w8vjx48TFxeHl5QXAV199RYsWLTh8+DBt27YtdD2h8hEFTqhUunbtyuLFiwvMq1WrluXnf488dkv9+vULFKlTp07h5eVlKW4AzZs3x8XFhVOnTlkK3H/XEyofUeCESsXBwYFGjRrd8fXizCvuewmVmzgHJ1Q7zZo1IyEhgYSEBMu82NhYUlNTad68uYLJhNIm9uCESiU3N/e2cT41Gg2urq7FbiM4OBh/f38GDx7M3LlzMRqNjB49mi5duhR6iCtUXmIPTqhUIiIi8PDwKDB17NjRqjYkSWLLli3UrFmTzp07ExwcjK+vL+vXry+j1IJSxJgMgiBUWWIPThCEKksUOEEQqixR4ARBqLJEgRMEocoSBU4QhCpLFDhBEKosUeAEQaiyRIETBKHKEgVOEIQqSxQ4QRCqLFHgBEGosv4H9w1eP13H7SIAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.02102, + "end_time": "2024-02-29T09:00:38.093396", + "exception": false, + "start_time": "2024-02-29T09:00:38.072376", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4038.996982, + "end_time": "2024-02-29T09:00:40.838045", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/lct_gan/2/mlu-eval.ipynb", + "output_path": "eval/treatment/lct_gan/2/mlu-eval.ipynb", + "parameters": { + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "path": "eval/treatment/lct_gan/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "lct_gan" + }, + "start_time": "2024-02-29T07:53:21.841063", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/lct_gan/model.pt b/treatment/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..faedd1a53f7168c0b5ca7ec35f34f04613924a0b --- /dev/null +++ b/treatment/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7bc101bcf2b97eddd569e940acf46edd6a65e63dfeeb409cff945d6aee1ae75 +size 74778241 diff --git a/treatment/lct_gan/params.json b/treatment/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..53b4f76fe0811292b46cc38b2d018aa7c5362e5c --- /dev/null +++ b/treatment/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "ALL", "synth_data": 2, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "fixed_role_model": "lct_gan", "mse_mag": true, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/realtabformer/eval.csv b/treatment/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..369043b9d9350e0bb3bcdf1ddcc432157fa3d47f --- /dev/null +++ b/treatment/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,2.4650275154636803e-06,,0.001919547600785183,2.350700855255127,0.08960049599409103,1.7269368171691895,0.13757629692554474,4.768799499288434e-06,10.927171230316162,0.033860355615615845,0.06284471601247787,0.04381264001131058,0.06655652076005936,0.021347884088754654,13.277872085571289 diff --git a/treatment/realtabformer/history.csv b/treatment/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..4de486d1ea08af447f6cea2dfc9a9e4978a78a55 --- /dev/null +++ b/treatment/realtabformer/history.csv @@ -0,0 +1,13 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.17274063066684206,32.54312604089673,0.0552633830926005,0.3830513606852037,0.0,0.0,0.0,0.0,0.1777599465671756,320,160,252.70333337783813,1.5793958336114884,0.7896979168057442,0.12334369912808008,0.14260793848316097,34.241665986541186,0.05053621208407284,0.28937910752138124,0.0,0.0,0.0,0.0,0.14700217486024486,80,40,43.77654767036438,1.0944136917591094,0.5472068458795547,0.10342117013753124 +1,0.12808593967503334,24.135995886683254,0.03799298999619229,0.24013567787806095,0.0,0.0,0.0,0.0,0.13213618035493369,320,160,249.8136842250824,1.561335526406765,0.7806677632033825,0.17246930892296225,0.05324988574730014,17.651811327400104,0.008742467289794364,0.027332919216132723,0.0,0.0,0.0,0.0,0.059270787181776544,80,40,43.29320168495178,1.0823300421237945,0.5411650210618972,0.021186699938334642 +2,0.01842976342000071,10.15742298751733,0.0012341465806072962,0.11120925650629943,0.0,0.0,0.0,0.0,0.019104365019365587,320,160,249.94196939468384,1.562137308716774,0.781068654358387,0.11273365501998227,0.006959301792082784,24.924650703443284,0.00016421204741474661,0.061348673028260234,0.0,0.0,0.0,0.0,0.007060477700136047,80,40,43.324461698532104,1.0831115424633027,0.5415557712316513,0.003516542712429782 +3,0.0094169184052588,8.37598086759846,0.00029596344863640016,0.07017327518287857,0.0,0.0,0.0,0.0,0.00955239509055179,320,160,249.7390410900116,1.5608690068125726,0.7804345034062863,0.15416082545476115,0.005165062801461317,20.42051088246374,3.41182621838923e-05,0.04188905684277415,0.0,0.0,0.0,0.0,0.005266186463268241,80,40,43.202502489089966,1.080062562227249,0.5400312811136245,0.01200705139306706 +4,0.007788351746415656,11.70798542958924,0.00015231196534966018,0.058090974773494966,0.0,0.0,0.0,0.0,0.0079054738426243,320,160,249.75754499435425,1.560984656214714,0.780492328107357,0.13232829257080053,0.005066547100977914,13.522705953694617,7.158047726778527e-05,0.1007533791125752,0.0,0.0,0.0,0.0,0.005207566117314854,80,40,43.26472735404968,1.081618183851242,0.540809091925621,0.013474537204729131 +5,0.006294602435991692,6.33328887433686,9.684177390981152e-05,0.08273278586857487,0.0,0.0,0.0,0.0,0.006374534645516405,320,160,249.90035939216614,1.5618772462010384,0.7809386231005192,0.117095634009911,0.004170434930529154,14.81665239444008,2.34258444930191e-05,0.06965655735693872,0.0,0.0,0.0,0.0,0.004237807755816903,80,40,43.293201208114624,1.0823300302028656,0.5411650151014328,0.022184051648196146 +6,0.006196894027448252,4.8341636554777425,0.00011549125597037635,0.094308190207812,0.0,0.0,0.0,0.0,0.006269788878671534,320,160,249.95278120040894,1.5622048825025558,0.7811024412512779,0.1319531003245288,0.004221608684019884,10.302404212270737,2.561588371109702e-05,0.08412196736608166,0.0,0.0,0.0,0.0,0.004336855133806239,80,40,43.18171191215515,1.0795427978038787,0.5397713989019394,0.025219803489926564 +7,0.005747240910363871,3.7540236900401482,9.932379949015515e-05,0.09486368083598791,0.0,0.0,0.0,0.0,0.0058169683367850665,320,160,236.3643090724945,1.4772769317030907,0.7386384658515454,0.14723998532332416,0.004258396451325553,10.139248311189034,2.5635939312307342e-05,0.07636677546834107,0.0,0.0,0.0,0.0,0.004334129349547311,80,40,39.33404016494751,0.9833510041236877,0.49167550206184385,0.024249439790219186 +8,0.005594146692064328,4.709009060162941,0.00011444825762162003,0.08336758576042484,0.0,0.0,0.0,0.0,0.0056661242446011785,320,160,233.7539517879486,1.4609621986746788,0.7304810993373394,0.14480910277636666,0.005135520236672164,11.65470664921339,5.308824860196515e-05,0.05176811983110383,0.0,0.0,0.0,0.0,0.005268141837132134,80,40,38.870795249938965,0.9717698812484741,0.48588494062423704,0.028078802437062223 +9,0.005501753174223722,5.608701468630493,0.00011689694678972598,0.08811835766100558,0.0,0.0,0.0,0.0,0.005575995156732461,320,160,234.21500182151794,1.4638437613844872,0.7319218806922436,0.1474024361260035,0.005505009344597056,13.499444098733353,6.128027262813918e-05,0.1038002680579666,0.0,0.0,0.0,0.0,0.005596104772848776,80,40,42.07728934288025,1.0519322335720063,0.5259661167860031,0.035794182426798216 +10,0.00555761277170177,4.359144222357516,7.373428340302565e-05,0.11662060340167954,0.0,0.0,0.0,0.0,0.005633294901269892,320,160,245.06033635139465,1.5316271021962167,0.7658135510981083,0.14656462968980577,0.004731032052768569,12.1588466797154,3.7162138724934624e-05,0.1094296614639461,0.0,0.0,0.0,0.0,0.004811913078810903,80,40,41.566365480422974,1.0391591370105744,0.5195795685052872,0.03354522421614092 +11,0.005510396755062175,3.9457729407642987,7.684137375911747e-05,0.11754734711139463,0.0,0.0,0.0,0.0,0.005578392313691438,320,160,246.70163893699646,1.5418852433562278,0.7709426216781139,0.14803811081962975,0.003412693228585795,14.689670172936781,2.834388970897467e-05,0.11955833142856136,0.0,0.0,0.0,0.0,0.003510729462868767,80,40,42.07568120956421,1.0518920302391053,0.5259460151195526,0.025872247267534475 diff --git a/treatment/realtabformer/mlu-eval.ipynb b/treatment/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..595c8955b58b7e824667449d24df109c2171d655 --- /dev/null +++ b/treatment/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2552 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:49.139841Z", + "iopub.status.busy": "2024-02-29T20:08:49.138893Z", + "iopub.status.idle": "2024-02-29T20:08:49.174588Z", + "shell.execute_reply": "2024-02-29T20:08:49.173862Z" + }, + "papermill": { + "duration": 0.051134, + "end_time": "2024-02-29T20:08:49.176973", + "exception": false, + "start_time": "2024-02-29T20:08:49.125839", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:49.203703Z", + "iopub.status.busy": "2024-02-29T20:08:49.203268Z", + "iopub.status.idle": "2024-02-29T20:08:49.210298Z", + "shell.execute_reply": "2024-02-29T20:08:49.209488Z" + }, + "papermill": { + "duration": 0.022412, + "end_time": "2024-02-29T20:08:49.212276", + "exception": false, + "start_time": "2024-02-29T20:08:49.189864", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:49.236245Z", + "iopub.status.busy": "2024-02-29T20:08:49.235932Z", + "iopub.status.idle": "2024-02-29T20:08:49.240303Z", + "shell.execute_reply": "2024-02-29T20:08:49.239394Z" + }, + "papermill": { + "duration": 0.018591, + "end_time": "2024-02-29T20:08:49.242247", + "exception": false, + "start_time": "2024-02-29T20:08:49.223656", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:49.265906Z", + "iopub.status.busy": "2024-02-29T20:08:49.265567Z", + "iopub.status.idle": "2024-02-29T20:08:49.269732Z", + "shell.execute_reply": "2024-02-29T20:08:49.268888Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018402, + "end_time": "2024-02-29T20:08:49.271843", + "exception": false, + "start_time": "2024-02-29T20:08:49.253441", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:49.295774Z", + "iopub.status.busy": "2024-02-29T20:08:49.295002Z", + "iopub.status.idle": "2024-02-29T20:08:49.301178Z", + "shell.execute_reply": "2024-02-29T20:08:49.300322Z" + }, + "papermill": { + "duration": 0.020419, + "end_time": "2024-02-29T20:08:49.303125", + "exception": false, + "start_time": "2024-02-29T20:08:49.282706", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "c3423841", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:49.328323Z", + "iopub.status.busy": "2024-02-29T20:08:49.327987Z", + "iopub.status.idle": "2024-02-29T20:08:49.332797Z", + "shell.execute_reply": "2024-02-29T20:08:49.331962Z" + }, + "papermill": { + "duration": 0.019791, + "end_time": "2024-02-29T20:08:49.334733", + "exception": false, + "start_time": "2024-02-29T20:08:49.314942", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"realtabformer\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/realtabformer/4\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011044, + "end_time": "2024-02-29T20:08:49.356814", + "exception": false, + "start_time": "2024-02-29T20:08:49.345770", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:49.381445Z", + "iopub.status.busy": "2024-02-29T20:08:49.380601Z", + "iopub.status.idle": "2024-02-29T20:08:49.390278Z", + "shell.execute_reply": "2024-02-29T20:08:49.389381Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024239, + "end_time": "2024-02-29T20:08:49.392199", + "exception": false, + "start_time": "2024-02-29T20:08:49.367960", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/realtabformer/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:49.416208Z", + "iopub.status.busy": "2024-02-29T20:08:49.415911Z", + "iopub.status.idle": "2024-02-29T20:08:51.678401Z", + "shell.execute_reply": "2024-02-29T20:08:51.677410Z" + }, + "papermill": { + "duration": 2.277042, + "end_time": "2024-02-29T20:08:51.680494", + "exception": false, + "start_time": "2024-02-29T20:08:49.403452", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:51.707391Z", + "iopub.status.busy": "2024-02-29T20:08:51.706887Z", + "iopub.status.idle": "2024-02-29T20:08:51.721738Z", + "shell.execute_reply": "2024-02-29T20:08:51.720819Z" + }, + "papermill": { + "duration": 0.030846, + "end_time": "2024-02-29T20:08:51.723934", + "exception": false, + "start_time": "2024-02-29T20:08:51.693088", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:51.749386Z", + "iopub.status.busy": "2024-02-29T20:08:51.749032Z", + "iopub.status.idle": "2024-02-29T20:08:51.756987Z", + "shell.execute_reply": "2024-02-29T20:08:51.756131Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.023274, + "end_time": "2024-02-29T20:08:51.759049", + "exception": false, + "start_time": "2024-02-29T20:08:51.735775", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:51.784323Z", + "iopub.status.busy": "2024-02-29T20:08:51.783471Z", + "iopub.status.idle": "2024-02-29T20:08:51.890058Z", + "shell.execute_reply": "2024-02-29T20:08:51.888938Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.122074, + "end_time": "2024-02-29T20:08:51.892709", + "exception": false, + "start_time": "2024-02-29T20:08:51.770635", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:51.919143Z", + "iopub.status.busy": "2024-02-29T20:08:51.918734Z", + "iopub.status.idle": "2024-02-29T20:08:56.705424Z", + "shell.execute_reply": "2024-02-29T20:08:56.704630Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.802678, + "end_time": "2024-02-29T20:08:56.707906", + "exception": false, + "start_time": "2024-02-29T20:08:51.905228", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 20:08:54.238575: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 20:08:54.238650: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 20:08:54.240414: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:56.833457Z", + "iopub.status.busy": "2024-02-29T20:08:56.832714Z", + "iopub.status.idle": "2024-02-29T20:08:56.840565Z", + "shell.execute_reply": "2024-02-29T20:08:56.839743Z" + }, + "papermill": { + "duration": 0.117026, + "end_time": "2024-02-29T20:08:56.842758", + "exception": false, + "start_time": "2024-02-29T20:08:56.725732", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:08:56.925825Z", + "iopub.status.busy": "2024-02-29T20:08:56.924474Z", + "iopub.status.idle": "2024-02-29T20:09:19.960107Z", + "shell.execute_reply": "2024-02-29T20:09:19.958977Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 23.077531, + "end_time": "2024-02-29T20:09:19.962644", + "exception": false, + "start_time": "2024-02-29T20:08:56.885113", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:09:20.481240Z", + "iopub.status.busy": "2024-02-29T20:09:20.480368Z", + "iopub.status.idle": "2024-02-29T20:09:20.552212Z", + "shell.execute_reply": "2024-02-29T20:09:20.551234Z" + }, + "papermill": { + "duration": 0.088554, + "end_time": "2024-02-29T20:09:20.554333", + "exception": false, + "start_time": "2024-02-29T20:09:20.465779", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../treatment/_cache/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache4/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache5/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/treatment [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T20:09:20.587532Z", + "iopub.status.busy": "2024-02-29T20:09:20.587048Z", + "iopub.status.idle": "2024-02-29T20:09:21.156374Z", + "shell.execute_reply": "2024-02-29T20:09:21.155420Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.58995, + "end_time": "2024-02-29T20:09:21.158382", + "exception": false, + "start_time": "2024-02-29T20:09:20.568432", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:09:21.187441Z", + "iopub.status.busy": "2024-02-29T20:09:21.187129Z", + "iopub.status.idle": "2024-02-29T20:09:21.191339Z", + "shell.execute_reply": "2024-02-29T20:09:21.190551Z" + }, + "papermill": { + "duration": 0.020662, + "end_time": "2024-02-29T20:09:21.193236", + "exception": false, + "start_time": "2024-02-29T20:09:21.172574", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:09:21.221118Z", + "iopub.status.busy": "2024-02-29T20:09:21.220393Z", + "iopub.status.idle": "2024-02-29T20:09:21.227239Z", + "shell.execute_reply": "2024-02-29T20:09:21.226434Z" + }, + "papermill": { + "duration": 0.022925, + "end_time": "2024-02-29T20:09:21.229118", + "exception": false, + "start_time": "2024-02-29T20:09:21.206193", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "19390534" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:09:21.256952Z", + "iopub.status.busy": "2024-02-29T20:09:21.256217Z", + "iopub.status.idle": "2024-02-29T20:09:21.391331Z", + "shell.execute_reply": "2024-02-29T20:09:21.390448Z" + }, + "papermill": { + "duration": 0.151691, + "end_time": "2024-02-29T20:09:21.393933", + "exception": false, + "start_time": "2024-02-29T20:09:21.242242", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 52992] --\n", + "├─Adapter: 1-1 [2, 2648, 52992] --\n", + "│ └─Embedding: 2-1 [2, 2648, 69, 768] (215,808)\n", + "│ └─TensorInductionPoint: 2-2 [69, 1] 69\n", + "│ └─Sequential: 2-3 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 787,456\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 52992] (recursive)\n", + "│ └─Embedding: 2-4 [2, 661, 69, 768] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [69, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-7 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-8 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 19,606,342\n", + "Trainable params: 19,390,534\n", + "Non-trainable params: 215,808\n", + "Total mult-adds (M): 77.67\n", + "========================================================================================================================\n", + "Input size (MB): 1.83\n", + "Forward/backward pass size (MB): 3885.09\n", + "Params size (MB): 78.43\n", + "Estimated Total Size (MB): 3965.34\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T20:09:21.427769Z", + "iopub.status.busy": "2024-02-29T20:09:21.426827Z", + "iopub.status.idle": "2024-02-29T21:15:26.634565Z", + "shell.execute_reply": "2024-02-29T21:15:26.633637Z" + }, + "papermill": { + "duration": 3965.227463, + "end_time": "2024-02-29T21:15:26.637083", + "exception": false, + "start_time": "2024-02-29T20:09:21.409620", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.17274063066684206, 'avg_role_model_std_loss': 32.54312604089673, 'avg_role_model_mean_pred_loss': 0.0552633830926005, 'avg_role_model_g_mag_loss': 0.3830513606852037, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.1777599465671756, 'n_size': 320, 'n_batch': 160, 'duration': 252.70333337783813, 'duration_batch': 1.5793958336114884, 'duration_size': 0.7896979168057442, 'avg_pred_std': 0.12334369912808008}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.14260793848316097, 'avg_role_model_std_loss': 34.241665986541186, 'avg_role_model_mean_pred_loss': 0.05053621208407284, 'avg_role_model_g_mag_loss': 0.28937910752138124, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.14700217486024486, 'n_size': 80, 'n_batch': 40, 'duration': 43.77654767036438, 'duration_batch': 1.0944136917591094, 'duration_size': 0.5472068458795547, 'avg_pred_std': 0.10342117013753124}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.12808593967503334, 'avg_role_model_std_loss': 24.135995886683254, 'avg_role_model_mean_pred_loss': 0.03799298999619229, 'avg_role_model_g_mag_loss': 0.24013567787806095, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.13213618035493369, 'n_size': 320, 'n_batch': 160, 'duration': 249.8136842250824, 'duration_batch': 1.561335526406765, 'duration_size': 0.7806677632033825, 'avg_pred_std': 0.17246930892296225}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.05324988574730014, 'avg_role_model_std_loss': 17.651811327400104, 'avg_role_model_mean_pred_loss': 0.008742467289794364, 'avg_role_model_g_mag_loss': 0.027332919216132723, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.059270787181776544, 'n_size': 80, 'n_batch': 40, 'duration': 43.29320168495178, 'duration_batch': 1.0823300421237945, 'duration_size': 0.5411650210618972, 'avg_pred_std': 0.021186699938334642}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.01842976342000071, 'avg_role_model_std_loss': 10.15742298751733, 'avg_role_model_mean_pred_loss': 0.0012341465806072962, 'avg_role_model_g_mag_loss': 0.11120925650629943, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.019104365019365587, 'n_size': 320, 'n_batch': 160, 'duration': 249.94196939468384, 'duration_batch': 1.562137308716774, 'duration_size': 0.781068654358387, 'avg_pred_std': 0.11273365501998227}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006959301792082784, 'avg_role_model_std_loss': 24.924650703443284, 'avg_role_model_mean_pred_loss': 0.00016421204741474661, 'avg_role_model_g_mag_loss': 0.061348673028260234, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007060477700136047, 'n_size': 80, 'n_batch': 40, 'duration': 43.324461698532104, 'duration_batch': 1.0831115424633027, 'duration_size': 0.5415557712316513, 'avg_pred_std': 0.003516542712429782}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0094169184052588, 'avg_role_model_std_loss': 8.37598086759846, 'avg_role_model_mean_pred_loss': 0.00029596344863640016, 'avg_role_model_g_mag_loss': 0.07017327518287857, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00955239509055179, 'n_size': 320, 'n_batch': 160, 'duration': 249.7390410900116, 'duration_batch': 1.5608690068125726, 'duration_size': 0.7804345034062863, 'avg_pred_std': 0.15416082545476115}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005165062801461317, 'avg_role_model_std_loss': 20.42051088246374, 'avg_role_model_mean_pred_loss': 3.41182621838923e-05, 'avg_role_model_g_mag_loss': 0.04188905684277415, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005266186463268241, 'n_size': 80, 'n_batch': 40, 'duration': 43.202502489089966, 'duration_batch': 1.080062562227249, 'duration_size': 0.5400312811136245, 'avg_pred_std': 0.01200705139306706}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007788351746415656, 'avg_role_model_std_loss': 11.70798542958924, 'avg_role_model_mean_pred_loss': 0.00015231196534966018, 'avg_role_model_g_mag_loss': 0.058090974773494966, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0079054738426243, 'n_size': 320, 'n_batch': 160, 'duration': 249.75754499435425, 'duration_batch': 1.560984656214714, 'duration_size': 0.780492328107357, 'avg_pred_std': 0.13232829257080053}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005066547100977914, 'avg_role_model_std_loss': 13.522705953694617, 'avg_role_model_mean_pred_loss': 7.158047726778527e-05, 'avg_role_model_g_mag_loss': 0.1007533791125752, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005207566117314854, 'n_size': 80, 'n_batch': 40, 'duration': 43.26472735404968, 'duration_batch': 1.081618183851242, 'duration_size': 0.540809091925621, 'avg_pred_std': 0.013474537204729131}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006294602435991692, 'avg_role_model_std_loss': 6.33328887433686, 'avg_role_model_mean_pred_loss': 9.684177390981152e-05, 'avg_role_model_g_mag_loss': 0.08273278586857487, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006374534645516405, 'n_size': 320, 'n_batch': 160, 'duration': 249.90035939216614, 'duration_batch': 1.5618772462010384, 'duration_size': 0.7809386231005192, 'avg_pred_std': 0.117095634009911}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004170434930529154, 'avg_role_model_std_loss': 14.81665239444008, 'avg_role_model_mean_pred_loss': 2.34258444930191e-05, 'avg_role_model_g_mag_loss': 0.06965655735693872, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004237807755816903, 'n_size': 80, 'n_batch': 40, 'duration': 43.293201208114624, 'duration_batch': 1.0823300302028656, 'duration_size': 0.5411650151014328, 'avg_pred_std': 0.022184051648196146}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006196894027448252, 'avg_role_model_std_loss': 4.8341636554777425, 'avg_role_model_mean_pred_loss': 0.00011549125597037635, 'avg_role_model_g_mag_loss': 0.094308190207812, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006269788878671534, 'n_size': 320, 'n_batch': 160, 'duration': 249.95278120040894, 'duration_batch': 1.5622048825025558, 'duration_size': 0.7811024412512779, 'avg_pred_std': 0.1319531003245288}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004221608684019884, 'avg_role_model_std_loss': 10.302404212270737, 'avg_role_model_mean_pred_loss': 2.561588371109702e-05, 'avg_role_model_g_mag_loss': 0.08412196736608166, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004336855133806239, 'n_size': 80, 'n_batch': 40, 'duration': 43.18171191215515, 'duration_batch': 1.0795427978038787, 'duration_size': 0.5397713989019394, 'avg_pred_std': 0.025219803489926564}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005747240910363871, 'avg_role_model_std_loss': 3.7540236900401482, 'avg_role_model_mean_pred_loss': 9.932379949015515e-05, 'avg_role_model_g_mag_loss': 0.09486368083598791, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0058169683367850665, 'n_size': 320, 'n_batch': 160, 'duration': 236.3643090724945, 'duration_batch': 1.4772769317030907, 'duration_size': 0.7386384658515454, 'avg_pred_std': 0.14723998532332416}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004258396451325553, 'avg_role_model_std_loss': 10.139248311189034, 'avg_role_model_mean_pred_loss': 2.5635939312307342e-05, 'avg_role_model_g_mag_loss': 0.07636677546834107, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004334129349547311, 'n_size': 80, 'n_batch': 40, 'duration': 39.33404016494751, 'duration_batch': 0.9833510041236877, 'duration_size': 0.49167550206184385, 'avg_pred_std': 0.024249439790219186}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005594146692064328, 'avg_role_model_std_loss': 4.709009060162941, 'avg_role_model_mean_pred_loss': 0.00011444825762162003, 'avg_role_model_g_mag_loss': 0.08336758576042484, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0056661242446011785, 'n_size': 320, 'n_batch': 160, 'duration': 233.7539517879486, 'duration_batch': 1.4609621986746788, 'duration_size': 0.7304810993373394, 'avg_pred_std': 0.14480910277636666}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005135520236672164, 'avg_role_model_std_loss': 11.65470664921339, 'avg_role_model_mean_pred_loss': 5.308824860196515e-05, 'avg_role_model_g_mag_loss': 0.05176811983110383, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005268141837132134, 'n_size': 80, 'n_batch': 40, 'duration': 38.870795249938965, 'duration_batch': 0.9717698812484741, 'duration_size': 0.48588494062423704, 'avg_pred_std': 0.028078802437062223}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005501753174223722, 'avg_role_model_std_loss': 5.608701468630493, 'avg_role_model_mean_pred_loss': 0.00011689694678972598, 'avg_role_model_g_mag_loss': 0.08811835766100558, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005575995156732461, 'n_size': 320, 'n_batch': 160, 'duration': 234.21500182151794, 'duration_batch': 1.4638437613844872, 'duration_size': 0.7319218806922436, 'avg_pred_std': 0.1474024361260035}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005505009344597056, 'avg_role_model_std_loss': 13.499444098733353, 'avg_role_model_mean_pred_loss': 6.128027262813918e-05, 'avg_role_model_g_mag_loss': 0.1038002680579666, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005596104772848776, 'n_size': 80, 'n_batch': 40, 'duration': 42.07728934288025, 'duration_batch': 1.0519322335720063, 'duration_size': 0.5259661167860031, 'avg_pred_std': 0.035794182426798216}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00555761277170177, 'avg_role_model_std_loss': 4.359144222357516, 'avg_role_model_mean_pred_loss': 7.373428340302565e-05, 'avg_role_model_g_mag_loss': 0.11662060340167954, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005633294901269892, 'n_size': 320, 'n_batch': 160, 'duration': 245.06033635139465, 'duration_batch': 1.5316271021962167, 'duration_size': 0.7658135510981083, 'avg_pred_std': 0.14656462968980577}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004731032052768569, 'avg_role_model_std_loss': 12.1588466797154, 'avg_role_model_mean_pred_loss': 3.7162138724934624e-05, 'avg_role_model_g_mag_loss': 0.1094296614639461, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004811913078810903, 'n_size': 80, 'n_batch': 40, 'duration': 41.566365480422974, 'duration_batch': 1.0391591370105744, 'duration_size': 0.5195795685052872, 'avg_pred_std': 0.03354522421614092}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005510396755062175, 'avg_role_model_std_loss': 3.9457729407642987, 'avg_role_model_mean_pred_loss': 7.684137375911747e-05, 'avg_role_model_g_mag_loss': 0.11754734711139463, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005578392313691438, 'n_size': 320, 'n_batch': 160, 'duration': 246.70163893699646, 'duration_batch': 1.5418852433562278, 'duration_size': 0.7709426216781139, 'avg_pred_std': 0.14803811081962975}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003412693228585795, 'avg_role_model_std_loss': 14.689670172936781, 'avg_role_model_mean_pred_loss': 2.834388970897467e-05, 'avg_role_model_g_mag_loss': 0.11955833142856136, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003510729462868767, 'n_size': 80, 'n_batch': 40, 'duration': 42.07568120956421, 'duration_batch': 1.0518920302391053, 'duration_size': 0.5259460151195526, 'avg_pred_std': 0.025872247267534475}\n", + "Epoch 12\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0053725790354860695, 'avg_role_model_std_loss': 5.5834115966432645, 'avg_role_model_mean_pred_loss': 9.798973852376027e-05, 'avg_role_model_g_mag_loss': 0.11122669992910232, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005439527116698173, 'n_size': 320, 'n_batch': 160, 'duration': 247.18013310432434, 'duration_batch': 1.544875831902027, 'duration_size': 0.7724379159510135, 'avg_pred_std': 0.1352306238319187}\n", + "Time out: 3750.1190843582153/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▄▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▆▂▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test █▂▁▂▂▂▃▂▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▂█▁▆▃▂▃▅▅▅▅▅\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test █▁▂▁▃▂▃▂▂▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train █▅▂▁▁▂▂▂▂▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▄▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▆▂▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▂▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▆▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test █▃▅▄▂▂▁▁▁▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train █▆▃▂▃▂▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test █▇▇▇▇▇▇▂▁▆▅▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train █▇▇▇▇▇▇▂▁▁▅▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test █▇▇▇▇▇▇▂▁▆▅▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train █▇▇▇▇▇▇▂▁▁▅▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test █▇▇▇▇▇▇▂▁▆▅▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train █▇▇▇▇▇▇▂▁▁▅▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00351\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00558\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.02587\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.14804\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.11956\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.11755\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00341\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00551\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 3e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 8e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 14.68967\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 3.94577\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 1.05189\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.54189\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.52595\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.77094\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 42.07568\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 246.70164\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 160\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/treatment/realtabformer/4/wandb/offline-run-20240229_200923-25ex241w\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_200923-25ex241w/logs\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 399, 'n_batch': 200, 'role_model_metrics': {'avg_loss': 0.0019195475970936375, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 6.60189767782448e-05, 'pred_duration': 10.961475133895874, 'grad_duration': 2.385593891143799, 'total_duration': 13.347069025039673, 'pred_std': 0.06655652076005936, 'std_loss': 0.021347884088754654, 'mean_pred_loss': 4.768799044541083e-06, 'pred_rmse': 0.04381264001131058, 'pred_mae': 0.033860355615615845, 'pred_mape': 0.06284471601247787, 'grad_rmse': 0.13757629692554474, 'grad_mae': 0.08960049599409103, 'grad_mape': 1.726936936378479}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0019195475970936375, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 6.60189767782448e-05, 'avg_pred_duration': 10.961475133895874, 'avg_grad_duration': 2.385593891143799, 'avg_total_duration': 13.347069025039673, 'avg_pred_std': 0.06655652076005936, 'avg_std_loss': 0.021347884088754654, 'avg_mean_pred_loss': 4.768799044541083e-06}, 'min_metrics': {'avg_loss': 0.0019195475970936375, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 6.60189767782448e-05, 'pred_duration': 10.961475133895874, 'grad_duration': 2.385593891143799, 'total_duration': 13.347069025039673, 'pred_std': 0.06655652076005936, 'std_loss': 0.021347884088754654, 'mean_pred_loss': 4.768799044541083e-06, 'pred_rmse': 0.04381264001131058, 'pred_mae': 0.033860355615615845, 'pred_mape': 0.06284471601247787, 'grad_rmse': 0.13757629692554474, 'grad_mae': 0.08960049599409103, 'grad_mape': 1.726936936378479}, 'model_metrics': {'realtabformer': {'avg_loss': 0.0019195475970936375, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 6.60189767782448e-05, 'pred_duration': 10.961475133895874, 'grad_duration': 2.385593891143799, 'total_duration': 13.347069025039673, 'pred_std': 0.06655652076005936, 'std_loss': 0.021347884088754654, 'mean_pred_loss': 4.768799044541083e-06, 'pred_rmse': 0.04381264001131058, 'pred_mae': 0.033860355615615845, 'pred_mape': 0.06284471601247787, 'grad_rmse': 0.13757629692554474, 'grad_mae': 0.08960049599409103, 'grad_mape': 1.726936936378479}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:15:26.677986Z", + "iopub.status.busy": "2024-02-29T21:15:26.677566Z", + "iopub.status.idle": "2024-02-29T21:15:26.682516Z", + "shell.execute_reply": "2024-02-29T21:15:26.681628Z" + }, + "papermill": { + "duration": 0.027589, + "end_time": "2024-02-29T21:15:26.684597", + "exception": false, + "start_time": "2024-02-29T21:15:26.657008", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:15:26.724411Z", + "iopub.status.busy": "2024-02-29T21:15:26.723488Z", + "iopub.status.idle": "2024-02-29T21:15:26.866957Z", + "shell.execute_reply": "2024-02-29T21:15:26.866073Z" + }, + "papermill": { + "duration": 0.166229, + "end_time": "2024-02-29T21:15:26.869500", + "exception": false, + "start_time": "2024-02-29T21:15:26.703271", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:15:26.910518Z", + "iopub.status.busy": "2024-02-29T21:15:26.909527Z", + "iopub.status.idle": "2024-02-29T21:15:27.226653Z", + "shell.execute_reply": "2024-02-29T21:15:27.225762Z" + }, + "papermill": { + "duration": 0.340021, + "end_time": "2024-02-29T21:15:27.228905", + "exception": false, + "start_time": "2024-02-29T21:15:26.888884", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABB2UlEQVR4nO3de1wTZ9o//s8kkIRTgpzBInjGekCLwqJVq2ZFa2uxbLUurdRatX2g3UoPlt0q3e7vW9xqrVvrU7cHddtqtT5bbVetFhHtqigWxBOWqkVAISBYEs6B5P79ETIYCZAJgYBc79drXklm7pm5EsnlzJ37wDHGGAghpJcQ2TsAQggRgpIWIaRXoaRFCOlVKGkRQnoVSlqEkF6FkhYhpFehpEUI6VUc7B1Ad9Hr9SguLoabmxs4jrN3OISQuzDGUFVVhYCAAIhEbV9P9ZmkVVxcjMDAQHuHQQjpQFFREe677742t/eZpOXm5gbA8IHI5XI7R0MIuZtGo0FgYCD/XW1Ln0laxltCuVxOSYuQHqyj6huqiCeE9CqUtAghvQolLUJIr9Jn6rRI5+h0OjQ2Nto7DNKLOTo6QiwWd/o4lLRIuxhjUKlUqKystHco5B7g7u4OPz+/TrWVpKRF2mVMWD4+PnB2dqaGucQqjDHU1tairKwMAODv72/1sShp3YUxhuzC39DQqMfEIV72DseudDodn7A8PT3tHQ7p5ZycnAAAZWVl8PHxsfpW0aqK+E2bNiE4OBgymQwRERHIzMxss+ylS5cQExOD4OBgcByHDRs2tCpj3Hb3Eh8fz5d56KGHWm1//vnnrQm/XbuzbiDmowykfP+zzY/d2xjrsJydne0cCblXGP+WOlM/Kjhp7dq1C4mJiUhOTkZ2djZCQ0MRFRXFX/bdrba2FoMGDcKaNWvg5+dntsyZM2dQUlLCL6mpqQCAJ554wqTc0qVLTcq9++67QsPv0IwQH4hFHC7cVOPXW9U2P35vRLeExFZs8bckOGmtX78eS5cuxeLFi3H//fdj8+bNcHZ2xpYtW8yWnzBhAtauXYsnn3wSUqnUbBlvb2/4+fnxy759+zB48GBMnTrVpJyzs7NJua5o2e7pKsXkoYbbwu/OFdv8+ISQzhGUtLRaLbKysqBUKlsOIBJBqVQiIyPDJgFptVp8+eWXePbZZ1tl5e3bt8PLywujRo1CUlISamtr2zxOQ0MDNBqNyWKpuaEBAAxJiyYrIqRnEZS0ysvLodPp4Ovra7Le19cXKpXKJgHt3bsXlZWVeOaZZ0zW//GPf8SXX36J9PR0JCUl4YsvvsBTTz3V5nFSUlKgUCj4RcgIDzNH+kHqIMKvt2pwqdjyZEeIEcdx2Lt3r73DsKm33noLY8eOtXcYPa9F/GeffYbZs2cjICDAZP2yZcsQFRWF0aNHIzY2Fp9//jn27NmDa9eumT1OUlIS1Go1vxQVFVkcg6vUAcoRhsRMt4ikt9q2bRvc3d1tdrxXX30VaWlpNjuetQQlLS8vL4jFYpSWlpqsLy0tbbOSXYiCggIcPnwYzz33XIdlIyIiAABXr141u10qlfIjOlgzssOjzbeI/zlXDL2ebhHJvUur1VpUztXVtUc0fRGUtCQSCcLCwkyyrV6vR1paGiIjIzsdzNatW+Hj44M5c+Z0WDYnJwdA5xqpteeh4d5wkzmgRF2PM9dvd8k5ehvGGGq1TXZZhNYtHjx4EA8++CDc3d3h6emJRx55hL8qnzhxIlauXGlS/tatW3B0dMSPP/4IACgpKcGcOXPg5OSEgQMHYseOHQgODjbbZMcSFy5cwPTp0+Hk5ARPT08sW7YM1dUtv04fPXoU4eHhcHFxgbu7OyZNmoSCggIAwLlz5zBt2jS4ublBLpcjLCwMP/30U7vnO3r0KBYvXgy1Ws03EXrrrbcAGJoY/e1vf8OiRYsgl8uxbNkyAMDKlSsxbNgwODs7Y9CgQVi1apVJ04S7bw+feeYZREdHY926dfD394enpyfi4+O7vLuX4MaliYmJiIuLw/jx4xEeHo4NGzagpqYGixcvBgAsWrQI/fv3R0pKCgBDFs/NzeWf37x5Ezk5OXB1dcWQIUP44+r1emzduhVxcXFwcDAN69q1a9ixYwcefvhheHp64vz581ixYgWmTJmCMWPGWP3m2yNzFGPWSD/szrqB784VI2KQ/f+Hsbe6Rh3uX33ILufOfTsKzhLL/1xramqQmJiIMWPGoLq6GqtXr8a8efOQk5OD2NhYvPvuu1izZg3/Y8+uXbsQEBCAyZMnAzD8HZeXl+Po0aNwdHREYmJim816LIklKioKkZGROHPmDMrKyvDcc88hISEB27ZtQ1NTE6Kjo7F06VJ89dVX0Gq1yMzM5GOLjY3FuHHj8NFHH0EsFiMnJweOjo7tnnPixInYsGEDVq9ejby8PACGKyWjdevWYfXq1UhOTubXubm5Ydu2bQgICMCFCxewdOlSuLm54fXXX2/zPOnp6fD390d6ejquXr2KBQsWYOzYsVi6dKlVn5UlBCetBQsW4NatW1i9ejVUKhXGjh2LgwcP8pXzhYWFJuM7FxcXY9y4cfzrdevWYd26dZg6dSqOHj3Krz98+DAKCwvx7LPPtjqnRCLB4cOH+QQZGBiImJgYvPnmm0LDF+Sxsf2xO+sGDlwowVtzR8JR3OOqAEkbYmJiTF5v2bIF3t7eyM3Nxfz58/Hyyy/j+PHjfJLasWMHFi5cCI7j8PPPP+Pw4cM4c+YMxo8fDwD49NNPMXToUKti2bFjB+rr6/H555/DxcUFAPDhhx/i0Ucfxd///nc4OjpCrVbjkUceweDBgwEAI0aM4PcvLCzEa6+9hpCQEACwKA6JRAKFQgGO48xW3UyfPh2vvPKKybo7v0/BwcF49dVXsXPnznaTVr9+/fDhhx9CLBYjJCQEc+bMQVpaWs9KWgCQkJCAhIQEs9vuTESA4c1bcmk/c+bMNssFBgbi2LFjguPsrMjBnvBylaK8ugHHr5RjWohPt8fQkzg5ipH7dpTdzi3ElStXsHr1apw+fRrl5eXQ6/UADAlg1KhRmDlzJrZv347JkycjPz8fGRkZ+Oc//wkAyMvLg4ODAx544AH+eEOGDEG/fv2siv3y5csIDQ3lExYATJo0CXq9Hnl5eZgyZQqeeeYZREVF4fe//z2USiXmz5/PV30kJibiueeewxdffAGlUoknnniCT27WMibjO+3atQsffPABrl27hurqajQ1NXVYFzxy5EiT7jj+/v64cOFCp2LrCF06tEMs4vDIGMMfzrc5N+0cjf1xHAdniYNdFqEtqR999FHcvn0bn3zyCU6fPo3Tp08DaKl0jo2Nxf/93/+hsbERO3bswOjRozF69Gibf2aW2rp1KzIyMjBx4kTs2rULw4YNw6lTpwAY6pIuXbqEOXPm4MiRI7j//vuxZ8+eTp3vzgQKABkZGYiNjcXDDz+Mffv24ezZs/jLX/7SYSX93bepHMfx/0F0FUpaHZg71vAr4g+5pajT6uwcDbFERUUF8vLy8Oabb2LGjBkYMWIEfvvtN5Myjz32GOrr63Hw4EHs2LEDsbGx/Lbhw4ejqakJZ8+e5dddvXq11TEsNWLECJw7dw41NTX8uhMnTkAkEmH48OH8unHjxiEpKQknT57EqFGjsGPHDn7bsGHDsGLFCvzwww94/PHHsXXr1g7PK5FIoNNZ9jd78uRJBAUF4S9/+QvGjx+PoUOH8j8E9DSUtDowLtAdgR5OqNXqcPhyacc7ELvr168fPD098fHHH+Pq1as4cuQIEhMTTcq4uLggOjoaq1atwuXLl7Fw4UJ+W0hICJRKJZYtW4bMzEycPXsWy5Ytg5OTk1V952JjYyGTyRAXF4eLFy8iPT0dL774Ip5++mn4+voiPz8fSUlJyMjIQEFBAX744QdcuXIFI0aMQF1dHRISEnD06FEUFBTgxIkTOHPmjEmdV1uCg4NRXV2NtLQ0lJeXt9uDZOjQoSgsLMTOnTtx7do1fPDBB52+musqlLQ6wHGcSbce0vOJRCLs3LkTWVlZGDVqFFasWIG1a9e2KhcbG4tz585h8uTJGDBggMm2zz//HL6+vpgyZQrmzZvH/5Imk8kEx+Ps7IxDhw7h9u3bmDBhAv7whz9gxowZ+PDDD/ntP//8M2JiYjBs2DAsW7YM8fHxWL58OcRiMSoqKrBo0SIMGzYM8+fPx+zZs/HXv/61w/NOnDgRzz//PBYsWABvb+92BxiYO3cuVqxYgYSEBIwdOxYnT57EqlWrBL/X7sCxPtK5TqPRQKFQQK1WC25o+ktpFWa+/yMcxRx++svvoXBu/+fme0V9fT3y8/MxcOBAq76s95IbN24gMDAQhw8fxowZM+wdTq/V3t+Upd9RutKywDBfN4T4uaFRx3DwUom9wyHd4MiRI/juu++Qn5+PkydP4sknn0RwcDCmTJli79D6PEpaFjJ26/k2h24R+4LGxkb8+c9/xsiRIzFv3jx4e3vzDU23b98OV1dXs8vIkSO7LcbZs2e3Gcc777zTbXF0Nxpu2UJzQwOw9lAeMn6tQJmmHj7yvn27dK+LiopCVJT5Nmlz587l+77eraOW6rb06aefoq6uzuw2Dw+Pbouju1HSslCghzMeGOCO7MJK/Od8CZY8ONDeIRE7cXNzg5ubm73DQP/+/e0dgl3Q7aEAj401/JHQr4iE2A8lLQEeHu0PEQecK6pEQUVNxzsQQmyOkpYA3m5STGqeVuw7qpAnxC4oaQlkbGj6LY0fT4hdUNISKGqUHyQOIlwtq8blkip7h0NIn0NJSyC5zBHThxuGqKEKedKWe3Fii56CkpYVHhtL48eTns/WE1sAhvHyOI5DZWWlTY8rBCUtK0wL8YGr1AE3K+uQXWjdcCWEEOtQ0rKCzFGMmSMNw0v3qW49jAHaGvssNLGFzSa2aGhowKuvvor+/fvDxcUFERERJiMOFxQU4NFHH0W/fv3g4uKCkSNH4sCBA7h+/TqmTZsGwDD8D8dxreYn7Q7UIt5Kj43tj2+yb+LAhRKsfvT+vjF+fGMt8E5Ax+W6wp+LAYlLx+Wa0cQWbU9skZCQgNzcXOzcuRMBAQHYs2cPZs2ahQsXLmDo0KGIj4+HVqvFjz/+CBcXF+Tm5sLV1RWBgYH497//jZiYGOTl5UEul8PJycmqz6QzrPqmbdq0CcHBwZDJZIiIiEBmZmabZS9duoSYmBgEBweD4ziz/1O99dZb/P8GxsU4iL9RfX094uPj4enpCVdXV8TExLSaf7E7TRrsCU8XCSpqtDhxtdxucRDzYmJi8Pjjj2PIkCEYO3YstmzZggsXLvATWxQXF+P48eN8eXMTW3zyySeIiIjAAw880G4/v47cObHFqFGjMH36dHz44Yf44osvUFpaCo1GYzKxxYgRIxAXF8eP8VVYWAilUomQkBAMHToUTzzxBEJDQ9s9590TW/j5+cHV1RWFhYXYunUrdu/ejcmTJ2Pw4MF49dVX8eCDD/KjoRYWFmLSpEkYPXo0Bg0ahEceeQRTpkyBWCzm+zT6+PjAz88PCoXCqs+kMwRfae3atQuJiYnYvHkzIiIisGHDBkRFRSEvLw8+Pq0nfqitrcWgQYPwxBNPYMWKFW0ed+TIkTh8+HBLYHdNI7ZixQrs378fu3fvhkKhQEJCAh5//HGcOHFC6FuwCQexCHPG+OPzjAJ8d64YDw3vA5NeODobrnjsdW4BaGIL8y5cuACdTodhw4aZrG9oaOAnYn3ppZfwwgsv4IcffoBSqURMTEyXTdVnDcFXWuvXr8fSpUuxePFi3H///di8eTOcnZ2xZcsWs+UnTJiAtWvX4sknn4RUKm3zuA4ODvz/CH5+fvDy8uK3qdVqfPbZZ1i/fj2mT5+OsLAwbN26FSdPnuQH/7cHY0PTQxdVqG/sA+PHc5zhFs0eC01sYZOJLaqrqyEWi5GVlYWcnBx+uXz5Mv7xj38AAJ577jn8+uuvePrpp3HhwgWMHz8eGzdutNl77SxBSUur1SIrKwtKpbLlACIRlEolMjIyOhXIlStXEBAQgEGDBiE2NhaFhYX8tqysLDQ2NpqcNyQkBAMGDGjzvA0NDdBoNCaLrT0woB/6uzuhRqvDkZ+tq+8gtkcTWxiYm9hi3Lhx0Ol0KCsrw5AhQ0yWO+dHDAwMxPPPP49vvvkGr7zyCj755BP+mAAsnjCjKwhKWuXl5dDpdPzErEa+vr5QqVRWBxEREYFt27bh4MGD+Oijj5Cfn4/JkyejqsrQ4lylUkEikbRqc9LeeVNSUqBQKPglMDDQ6vjaIhJx/OCA1Bex56CJLQzMTWwxbNgwxMbGYtGiRfjmm2+Qn5+PzMxMpKSkYP/+/QCAl19+GYcOHUJ+fj6ys7ORnp7Ony8oKAgcx2Hfvn24deuWyS+g3YYJcPPmTQaAnTx50mT9a6+9xsLDwzvcPygoiL3//vsdlvvtt9+YXC5nn376KWOMse3btzOJRNKq3IQJE9jrr79u9hj19fVMrVbzS1FREQPA1Gp1h+cXIrdYzYJW7mND/3KAqeu0Nj22vdXV1bHc3FxWV1dn71AES01NZSNGjGBSqZSNGTOGHT16lAFge/bs4cscOHCAAWBTpkxptX9xcTGbPXs2k0qlLCgoiO3YsYP5+PiwzZs3W3T+u891/vx5Nm3aNCaTyZiHhwdbunQpq6qqYowxplKpWHR0NPP392cSiYQFBQWx1atXM51OxxoaGtiTTz7JAgMDmUQiYQEBASwhIcHif5Pnn3+eeXp6MgAsOTmZMcaYVqtlq1evZsHBwczR0ZH5+/uzefPmsfPnzzPGGEtISGCDBw9mUqmUeXt7s6effpqVl5fzx3z77beZn58f4ziOxcXFWRSHUXt/U2q12qLvqKCk1dDQwMRisck/BmOMLVq0iM2dO7fD/S1NWowxNn78ePbGG28wxhhLS0tjANhvv/1mUmbAgAFs/fr1Fh3P0g+ENTUyVvYzYzd+sui4er2eKd87yoJW7mNfnym0aJ/eojcnLVsz/qd3+PBhe4fSq9kiaQm6PZRIJAgLC0NaWhq/Tq/XIy0tDZGRkba59IOhsvDatWv8rydhYWFwdHQ0OW9eXh4KCwttel4AwOXvgE3hwIHXLCrOcRzfrYf6It47aGKLnkvwr4eJiYn45JNP8K9//QuXL1/GCy+8gJqaGixevBiAoVFeUlISX16r1fK/UGi1Wty8eRM5OTm4evUqX+bVV1/FsWPHcP36dZw8eRLz5s2DWCzm6xkUCgWWLFmCxMREpKenIysrC4sXL0ZkZCR+97vfdfYzMOXX/AtS2WVAb1llo7Fe68TVctyqarBtPMQuaGKLnktwO60FCxbg1q1bWL16NVQqFcaOHYuDBw/ylfOFhYUQiVpyYXFxMcaNG8e/XrduHdatW4epU6fyXQdu3LiBhQsXoqKiAt7e3njwwQdx6tQpeHt78/u9//77EIlEiImJQUNDA6KiovC///u/1r7vtnkMAhycDK2/b+cDXkM63CXI0wVjA92RU1SJ/eeL8cwkGj++t6OJLXoumqzVnI+nAcXZwBPbgJHzLDr+luP5eHtfLh4Y4I5v/mdS5wPuAWiyVmJrNFlrV/FtvsQvvWTxLo+MMYwfn11YiaLbtV0UmH30kf/XSDewxd8SJS1zjPVaqosW7+IjlyFysKEbxL1SIW+81amtvbeSMLEf499SZ26jaZQHc6y40gIM3XpOXK3AdznFiJ/WcV1YTycWi+Hu7s6PbuDs7GxV40pCGGOora1FWVkZ3N3dIRaLrT4WJS1zjElLXQjUVQJO7hbtNmukP1btvYS80ir8rNIgxK+DurNewNi1w9phWQi5k7u7u0l3IWtQ0jLHqR8gvw/Q3ADKcoGgiRbtpnB2xNTh3kjNLcV3OcUImdX7kxbHcfD394ePjw8aGxvtHQ7pxRwdHTt1hWVESastfqMMSUt10eKkBRjGj0/NLcV/zhfjtajh98ztlFgstskfHCGdRRXxbeHrtSyvjAeAGSG+cJGIUXS7DmeLKm0fFyF9HCWttviOMjwKTFpOEjFmjjTcs9PID4TYHiWtthiTloDuPEbGwQH3nS9Bk05v68gI6dMoabXFc7Bpdx4BHhzqhX7OjiivbkDGrxVdFCAhfRMlrbaIxIBP80BrAm8RHcUiPDzaMEIF3SISYluUtNpjZWU80HKLeLCvjB9PSDehpNUeY3cegS3jAWBCsAf8FTJUNTThaN4tGwdGSN9FSas9xistAX0QjUzGjz9305ZREdKnUdJqz53deerVgnc33iKmXS5DQxPdIhJiC5S02mPszgNYdYs4MkAOV6kDGpr0KLpt3ezEhBBTlLQ64mdsZCo8aXEchyBPw8zI18trOihNCLEEJa2O8PVaF6zaPdjTMBX69QpKWoTYAiWtjvhaf6UFAMFehiutggoaSI8QW6Ck1RG+O0+u4O48gGHSC4CutAixFauS1qZNmxAcHAyZTIaIiAhkZma2WfbSpUuIiYlBcHAwOI7Dhg0bWpVJSUnBhAkT4ObmBh8fH0RHRyMvL8+kzEMPPQSO40yW559/3prwhfEcDDjIrOrOA7TcHtKVFiG2IThp7dq1C4mJiUhOTkZ2djZCQ0MRFRXV5siWtbW1GDRoENasWdPmiIXHjh1DfHw8Tp06hdTUVDQ2NmLmzJmoqTG9Olm6dClKSkr45d133xUavnCd6M4DAMHNFfE3fquFtok6TxPSWYIHAVy/fj2WLl3KT866efNm7N+/H1u2bMEbb7zRqvyECRMwYcIEADC7HQAOHjxo8nrbtm3w8fFBVlaWyYy+zs7OFg/V2tDQgIaGlolTNRqNRfuZ5TsKKD5rSFojowXt6u0mhZOjGHWNOtysrMNALxfr4yCECLvS0mq1yMrKglKpbDmASASlUomMjAybBaVWGxpy3j3h5Pbt2+Hl5YVRo0YhKSmp3VliUlJSoFAo+CUwMND6gDpRGW/S7IHqtQjpNEFJq7y8HDqdjp9N2sjX1xcqlcomAen1erz88suYNGkSRo0axa//4x//iC+//BLp6elISkrCF198gaeeeqrN4yQlJUGtVvNLUVGR9UEZ22pZ0Z0HuKNei9pqEdJpPW6M+Pj4eFy8eBHHjx83Wb9s2TL++ejRo+Hv748ZM2bg2rVrGDx4cKvjSKVSSKVS2wR1d3cemULQ7kFexistqownpLMEXWl5eXlBLBajtLTUZH1paWmnpwUCgISEBOzbtw/p6em477772i0bEREBALh69Wqnz9uhTnbnoQamhNiOoKQlkUgQFhaGtLQ0fp1er0daWhoiIyOtDoIxhoSEBOzZswdHjhzBwIEDO9wnJycHAODv72/1eQWxcgJXgJo9EGJLgm8PExMTERcXh/HjxyM8PBwbNmxATU0N/2viokWL0L9/f6SkpAAwVN7n5ubyz2/evImcnBy4urpiyBDDLMzx8fHYsWMHvv32W7i5ufH1YwqFAk5OTrh27Rp27NiBhx9+GJ6enjh//jxWrFiBKVOmYMyYMTb5IDrkNwq4csiq7jzGVvFFt2vRpNPDQUxtegmxGrPCxo0b2YABA5hEImHh4eHs1KlT/LapU6eyuLg4/nV+fj4D0GqZOnUqX8bcdgBs69atjDHGCgsL2ZQpU5iHhweTSqVsyJAh7LXXXmNqtdrimNVqNQMgaB8TF/7NWLKcsY+nC95Vp9OzYX85wIJW7mMF5TXWnZ+Qe5yl31GOMcbski27mUajgUKhgFqthlxuxczPt34BNk0AHJ2BpBuGRqcCzHz/GH4prcbnz4ZjyjBv4ecn5B5n6XeU7lMsdWd3nt+uC949iK/Xosp4QjqDkpal7uzOY029lic1eyDEFihpCdGJlvH8aA/UwJSQTqGkJQSftKzpOE1ttQixBUpaQvhZn7SM/Q+LbtdBp+8Tv30Q0iUoaQlhbGBaKXx2ngB3J0jEImh1epSoaZILQqxFSUsIk+48uYJ2FYs4BHo4AaCW8YR0BiUtofjuPFSvRYg9UNISih+mRnizhyDqg0hIp1HSEqozHae9aA5EQjqLkpZQvqMNj1bMzkNXWoR0HiUtoTwGWd2dJ/iOYZf11OyBEKtQ0hJK7GB1d57+7k5wEHFoaNKjtKq+C4Ij5N5HScsaVtZrOYhFuK+fodnD9XK6RSTEGpS0rGGs17KqZTyN9kBIZ1DSskYnuvMY5z2k0R4IsQ4lLWt0ojuPsQ8iXWkRYh1KWtboRHeellbxdKVFiDUoaVnLyu48d15p9ZGRrgmxKauS1qZNmxAcHAyZTIaIiAhkZma2WfbSpUuIiYlBcHAwOI7Dhg0brDpmfX094uPj4enpCVdXV8TExLSaf7FbWVmvdV8/Z4g4oFarw63qhi4IjJB7m+CktWvXLiQmJiI5ORnZ2dkIDQ1FVFQUysrKzJavra3FoEGDsGbNmjYndLXkmCtWrMB//vMf7N69G8eOHUNxcTEef/xxoeHbjvFKSyUsaUkcROhPzR4IsZ7QaX7Cw8NZfHw8/1qn07GAgACWkpLS4b5BQUHs/fffF3zMyspK5ujoyHbv3s2XuXz5MgPAMjIyzJ6rvr6eqdVqfikqKurcFGJ3K8szTCn2//kxptMJ2vWpT0+xoJX72K4zhbaJhZB7gKVTiAm60tJqtcjKyoJSqeTXiUQiKJVKZGRkWJU0LTlmVlYWGhsbTcqEhIRgwIABbZ43JSUFCoWCXwIDA62Kr00m3XnyBe1KvyASYj1BSau8vBw6nQ6+vr4m6319fflZoYWy5JgqlQoSiQTu7u4WnzcpKQlqtZpfioqKrIqvTXd25xFYr0W/IBJivXv210OpVAq5XG6y2JyV9VrB1CqeEKsJSlpeXl4Qi8WtfrUrLS1ts5LdFsf08/ODVqtFZWWlzc5rE3x3HmF9EI3jahWU11KzB0IEEpS0JBIJwsLCkJaWxq/T6/VIS0tDZGSkVQFYcsywsDA4OjqalMnLy0NhYaHV57UJvq2WsNEe7uvnDI4DqhqacLtG2wWBEXLvchC6Q2JiIuLi4jB+/HiEh4djw4YNqKmpweLFiwEAixYtQv/+/ZGSkgLAUNGem5vLP7958yZycnLg6uqKIUOGWHRMhUKBJUuWIDExER4eHpDL5XjxxRcRGRmJ3/3udzb5IKxyd3cemcKi3WSOYgQonHCzsg7XK2rh6SrtwiAJucdY89Pkxo0b2YABA5hEImHh4eHs1KlT/LapU6eyuLg4/nV+fj4D0GqZOnWqxcdkjLG6ujr2P//zP6xfv37M2dmZzZs3j5WUlFgcs6U/pwr23ghD04frJwXttvDjDBa0ch/7v5+KbBsPIb2Upd9RjrG+Uami0WigUCigVqttWym/fT5w5RDw8DogfKnFuyV9cwFfZRbipelDkDhzuO3iIaSXsvQ7es/+ethtrOyD2DL0MjV7IEQISlqdxU8pJrTjNDV7IMQalLQ6y7c5aZXlAnq9xbvx04nRlRYhglDS6iyPwVZ15wnyMFxpqesaUVlLzR4IsRQlrc6ysjuPk0QMP7kMAF1tESIEJS1bsLI7D3WcJkQ4Slq2YG13HmPHaRpXixCLUdKyBSu78wR5tcw4TQixDCUtW7Bydp6WIWooaRFiKUpatuDsAcj7G54LmJ2npU6Lbg8JsRQlLVvxFT7RhbGB6e0aLdR1jV0RFSH3HEpatmJFdx5XqQO8mkd4KKSrLUIsQknLVvgpxYT+gkiV8YQIQUnLVvjbQ6HdeagPIiFCUNKyFb47T42g7jw02gMhwlDSshWxA+AdYnhuRWU8XWkRYhlKWrZkRb2Wsa1WPrWKJ8QilLRsyVf42FoDmm8Py6sbUN3Q1BVREXJPoaRlS3xlvOXdeRROjvBwkQCgW0RCLGFV0tq0aROCg4Mhk8kQERGBzMzMdsvv3r0bISEhkMlkGD16NA4cOGCyneM4s8vatWv5MsHBwa22r1mzxprwu46V3XmoZTwhlhOctHbt2oXExEQkJycjOzsboaGhiIqKQllZmdnyJ0+exMKFC7FkyRKcPXsW0dHRiI6OxsWLLbdQJSUlJsuWLVvAcRxiYmJMjvX222+blHvxxReFht+1rOzOQ30QCbGc4KS1fv16LF26FIsXL8b999+PzZs3w9nZGVu2bDFb/h//+AdmzZqF1157DSNGjMDf/vY3PPDAA/jwww/5Mn5+fibLt99+i2nTpmHQoEEmx3JzczMp5+LiIjT8rmdVd56WGacJIe0TlLS0Wi2ysrKgVCpbDiASQalUIiMjw+w+GRkZJuUBICoqqs3ypaWl2L9/P5YsWdJq25o1a+Dp6Ylx48Zh7dq1aGpqu+K6oaEBGo3GZOkWVnTnGehFV1qEWErQDNPl5eXQ6XTw9fU1We/r64uff/7Z7D4qlcpseZVKZbb8v/71L7i5ueHxxx83Wf/SSy/hgQcegIeHB06ePImkpCSUlJRg/fr1Zo+TkpKCv/71r5a+NduxotlDS1stutIipCOCklZ32LJlC2JjYyGTyUzWJyYm8s/HjBkDiUSC5cuXIyUlBVJp62nlk5KSTPbRaDQIDAzsusCN7u7OI+r4YtbYKl6lqUedVgcnibgrIySkVxN0e+jl5QWxWIzS0lKT9aWlpfDz8zO7j5+fn8Xl//vf/yIvLw/PPfdch7FERESgqakJ169fN7tdKpVCLpebLN3Ciu487s4SKJwcAQAFt+kWkZD2CEpaEokEYWFhSEtL49fp9XqkpaUhMjLS7D6RkZEm5QEgNTXVbPnPPvsMYWFhCA0N7TCWnJwciEQi+Pj4CHkLXc/K7jx8H0SqjCekXYJvDxMTExEXF4fx48cjPDwcGzZsQE1NDRYvXgwAWLRoEfr374+UlBQAwJ/+9CdMnToV7733HubMmYOdO3fip59+wscff2xyXI1Gg927d+O9995rdc6MjAycPn0a06ZNg5ubGzIyMrBixQo89dRT6NevnzXvu2v5jQJKcgz1Wvc/ZtEuQZ4uOHdDTQ1MCemA4KS1YMEC3Lp1C6tXr4ZKpcLYsWNx8OBBvrK9sLAQojvqcSZOnIgdO3bgzTffxJ///GcMHToUe/fuxahRo0yOu3PnTjDGsHDhwlbnlEql2LlzJ9566y00NDRg4MCBWLFihUmdVY9iRXceGu2BEMtwjDFm7yC6g0ajgUKhgFqt7vr6rfz/Av96BHAPAl4+b9Eu/866gVd2n8PEwZ7YsfR3XRsfIT2Qpd9R6nvYFfjuPAVAvWXtw4K9qCsPIZagpNUV7uzOU2ZZdx5jW61idR3qG3VdFRkhvR4lra5ivNpSWTbig6eLBG5SBzAG3PiNrrYIaQslra7iK6xlPMdx/IzTNCAgIW2jpNVVrOiDSEMvE9IxSlpdxW+04VHA7Dw0nRghHaOk1VU8BgNiqaDuPNRxmpCOUdLqKmIHwGeE4bmF9Vo0GCAhHaOk1ZUEDghovD28+VsdtE2WT/hKSF9CSasr+QnrzuPtJoWToxh6avZASJsoaXUlgVdaHMfRJBeEdICSVle6sztPTblFuxiHXs4vp3otQsyhpNWVnD0AvzGG578csmgXaqtFSPsoaXW14Q8bHvMOtF+uGQ1RQ0j7KGl1teGzDY/XjgCN9R0WpystQtpHSaur+YcCbgFAYy2Q/2OHxY1D1Nz4rQ6NOmr2QMjdKGl1NY5rudrK299hcV83GaQOIjTpGYor67o4OEJ6H0pa3SHEWK91sMN+iCJRS7MHqtcipDVKWt0heDIgcQWqVUDJ2Q6LU70WIW2zKmlt2rQJwcHBkMlkiIiIQGZmZrvld+/ejZCQEMhkMowePRoHDpj+kvbMM8+A4ziTZdasWSZlbt++jdjYWMjlcri7u2PJkiWorq62Jvzu5yAFhswwPM/7vsPiNJ0YIW0TnLR27dqFxMREJCcnIzs7G6GhoYiKikJZWZnZ8idPnsTChQuxZMkSnD17FtHR0YiOjsbFi6atxGfNmoWSkhJ++eqrr0y2x8bG4tKlS0hNTcW+ffvw448/YtmyZULDtx++6YMFScuLOk4T0iYmUHh4OIuPj+df63Q6FhAQwFJSUsyWnz9/PpszZ47JuoiICLZ8+XL+dVxcHHvsscfaPGdubi4DwM6cOcOv+/777xnHcezmzZsWxa1WqxkAplarLSpvczUVjL3lzliynLHb19stevzKLRa0ch+bti69e2IjpAew9Dsq6EpLq9UiKysLSqWSXycSiaBUKpGRkWF2n4yMDJPyABAVFdWq/NGjR+Hj44Phw4fjhRdeQEVFhckx3N3dMX78eH6dUqmESCTC6dOnzZ63oaEBGo3GZLErZw9gQPOs2h1cbRkr4otu10Kn7xMzvBFiMUFJq7y8HDqdjp+Y1cjX1xcqlcrsPiqVqsPys2bNwueff460tDT8/e9/x7FjxzB79mzodDr+GD4+PibHcHBwgIeHR5vnTUlJgUKh4JfAwEAhb7VrWNg63l/hBIlYhEYdNXsg5G494tfDJ598EnPnzsXo0aMRHR2Nffv24cyZMzh69KjVx0xKSoJareaXoqIi2wVsLWN7rYITQF1lm8XEIg6BHk6GotTsgRATgpKWl5cXxGIxSktLTdaXlpbCz8/P7D5+fn6CygPAoEGD4OXlhatXr/LHuLuiv6mpCbdv327zOFKpFHK53GSxO8/BgNdwQN8EXD3cblEaxZQQ8wQlLYlEgrCwMKSlpfHr9Ho90tLSEBkZaXafyMhIk/IAkJqa2mZ5ALhx4wYqKirg7+/PH6OyshJZWVl8mSNHjkCv1yMiIkLIW7A/vnV8R/Va1FaLELOE1vDv3LmTSaVStm3bNpabm8uWLVvG3N3dmUqlYowx9vTTT7M33niDL3/ixAnm4ODA1q1bxy5fvsySk5OZo6Mju3DhAmOMsaqqKvbqq6+yjIwMlp+fzw4fPsweeOABNnToUFZfX88fZ9asWWzcuHHs9OnT7Pjx42zo0KFs4cKFFsdt918PjQpOGX5BfCeQsSZtm8X+dTKfBa3cx57715k2yxByL7H0Oyo4aTHG2MaNG9mAAQOYRCJh4eHh7NSpU/y2qVOnsri4OJPyX3/9NRs2bBiTSCRs5MiRbP/+/fy22tpaNnPmTObt7c0cHR1ZUFAQW7p0KZ8EjSoqKtjChQuZq6srk8vlbPHixayqqsrimHtM0tI1Mfb3QYbEdS29zWJH88pY0Mp97Pfrj3ZfbITYkaXfUY4x1id+U9doNFAoFFCr1fav3/o2Hjj7JRDxPDD772aLFFbUYsradEgdRLj89iyIRFw3B0lI97L0O9ojfj3sc+5s+tDG/xkB7jI4iDg0NOmh0nQ8DhchfQUlLXsY9BDgIAMqC4GyXLNFHMQiBHrQjNOE3I2Slj1IXAyJC2i3oSnNzENIa5S07MWCpg/UVouQ1ihp2cuw5qR1MwvQlJgtwl9p0RA1hPAoadmLmy/Qv7kD+C8HzRahKy1CWqOkZU8d3CLeWafVR1qmENIhSlr2ZGz68OtRQNv6auq+fs4QcUBdow5lVQ3dGxshPRQlLXvyGQG4BwG6BuBaeqvNEgcR+vczjPZwvZxuEQkBKGnZF8d1OAxzMN9xmirjCQEoadmfcXqxXw4Cel2rzVQZT4gpSlr2NiASkCmA2nLgxplWm6mBKSGmKGnZm9gRGDrT8NxM63i60iLEFCWtnqCdpg/BXtTsgZA7UdLqCYYoAZEDUP4LUH7VZNN9/ZzBcUB1QxMqarR2CpCQnoOSVk8gUwDBDxqe/2J6tSVzFCNAYZzkgm4RCaGk1VMMn2N4/Ll1vZaxMj6f+iASQkmrxxg+y/BYdAqoqTDZRJNcENKCklZP4T4A8B0NMD1w5QeTTQO9jIMB0pUWIVYlrU2bNiE4OBgymQwRERHIzMxst/zu3bsREhICmUyG0aNH48CBllugxsZGrFy5EqNHj4aLiwsCAgKwaNEiFBcXmxwjODgYHMeZLGvWrLEm/J6L/xXR9BaRrrQIaSE4ae3atQuJiYlITk5GdnY2QkNDERUV1WoyVaOTJ09i4cKFWLJkCc6ePYvo6GhER0fj4sWLAIDa2lpkZ2dj1apVyM7OxjfffIO8vDzMnTu31bHefvttlJSU8MuLL74oNPyezZi0rqYBjS3jwhvbauWX11CzB0KETvMTHh7O4uPj+dc6nY4FBASwlJQUs+Xnz5/P5syZY7IuIiKCLV++vM1zZGZmMgCsoKCAXxcUFMTef/99oeHyeswUYu3R6xlbN9wwvdgvqfzq2oYmFrRyHwtauY/drm6wY4CEdB1Lv6OCrrS0Wi2ysrKgVCr5dSKRCEqlEhkZGWb3ycjIMCkPAFFRUW2WBwC1Wg2O4+Du7m6yfs2aNfD09MS4ceOwdu1aNDU1tXmMhoYGaDQak6XH47g7bhH386udJGL4yWUAqGU8IYKSVnl5OXQ6HXx9fU3W+/r6QqVSmd1HpVIJKl9fX4+VK1di4cKFJnOfvfTSS9i5cyfS09OxfPlyvPPOO3j99dfbjDUlJQUKhYJfAgMDLX2b9nXnqA933ApSH0RCDBzsHcCdGhsbMX/+fDDG8NFHH5lsS0xM5J+PGTMGEokEy5cvR0pKCqRSaatjJSUlmeyj0Wh6R+IKngw4ugBVJUBJDhAwzrDa0wWn82/TlRbp8wRdaXl5eUEsFqO0tNRkfWlpKfz8/Mzu4+fnZ1F5Y8IqKChAampqh7NAR0REoKmpCdevXze7XSqVQi6Xmyy9gqMMGDLd8PyOvohBxmYPNBgg6eMEJS2JRIKwsDCkpaXx6/R6PdLS0hAZGWl2n8jISJPyAJCammpS3piwrly5gsOHD8PT07PDWHJyciASieDj4yPkLfQOd85A3axltAe6PSR9m+Dbw8TERMTFxWH8+PEIDw/Hhg0bUFNTg8WLFwMAFi1ahP79+yMlJQUA8Kc//QlTp07Fe++9hzlz5mDnzp346aef8PHHHwMwJKw//OEPyM7Oxr59+6DT6fj6Lg8PD0gkEmRkZOD06dOYNm0a3NzckJGRgRUrVuCpp55Cv379bPVZ9BxDowBOBKguGGahdh9wxwimdKVF+jhrfprcuHEjGzBgAJNIJCw8PJydOnWK3zZ16lQWFxdnUv7rr79mw4YNYxKJhI0cOZLt37+f35afn88AmF3S09MZY4xlZWWxiIgIplAomEwmYyNGjGDvvPMOq6+vtzjmXtHk4U6fzTI0fTj1T8YYY9X1jXyzh8oarZ2DI8T2LP2Ocoz1jdaKGo0GCoUCarW6d9RvnfgASF0FDJoGLNoLAJjw/w7jVlUDXpw+BC9OHwqJA/XCIvcOS7+j9FffUxnrta4fB+rVAIBHxwQAADYeuYpZ//gR/71yy17REWI3lLR6Kq8hgOdQQN9o6NYDYNUjI/DeE6HwcpXg11s1ePqzTPzP9iwUV9bZOVhCug8lrZ7srmGYOY5DTNh9SHvlITwzMRgiDjhwQYUZ7x3D/x69Cm2T3o7BEtI9KGn1ZCHNAwNeOQToGvnVCidHvDV3JPa9OBkTgvuhrlGHdw/mYdaGH/HjL3TLSO5tlLR6svsmAM6ehjqtwtZ9Ne8PkOPr5ZFYPz8UXq5S/Fpeg0VbMvHCl1m4SbeM5B5FSasnE4mBYc0jmrYxAzXHcXj8gftw5NWpWDwpGGIRh+8vqqB87xg2pV9FQ1PrCWAJ6c0oafV0xnqtn/ebdKC+m1zmiORHR2Lfiw/yt4xrD+Vh1ob/4hjdMpJ7CCWtnm7QNEAsBSoLgFs/d1h8hL/hlvH9BYZbxvzyGsRtycTzX9AtI7k3UNLq6aSuwKCHDM/NzEBtDsdxmDfOcMv47KSBEIs4HLykwoz3jtItI+n1KGn1BvwtomVJy0guc8TqR+/H/pceRPhAD9Q36vlbxqN55ofHJqSno248vYGmBFgfYnj+yi+Am2/75c1gjOHbnGL8vwOXcauqAQCgHOGD3w3yhK9cBl+5DH5yGXzkUsgcxbaMnhCLWPodpaTVW3w8DSjOBh79AAiLs/owVfWN2HD4CradvA6d3vw/vbuzI3zdZPBVyODrJoWfQgYfectzX7kMXq5SiEWc1XEQcjdLv6M9auRS0o7hDxuSVt73nUpabjJHrHrkfswfH4hvsm+gWF2PUo1hUanr0dCkR2VtIyprG5FXWtXmcUQc4O0mbb46a75Kc5PC200KH7kU3q6GqzZPFwkcxFQLQWyHrrR6C9VFYPMkwEEGRL0DKO4D5P0BeQDg1M8wKUYnMcagqWtCaZUhgRmTWammASpNPco09VBp6nGrqgFtXKS1wnGAp4sE3m4yQ0IzJjY3KXzuWucipf9D+zK6PbxLr09ajAEfjAV+u956m6OzIYEp+gPy+wyJzPhc0d+wTWa796zTM5RXN/BXZ6VVDShVG5JZWVU9blU3oEzTgPJqy5MbALhIxM1JTAYniRgiDhBxHDjO8IuoiAM4cBCJDK85GLaLmrdzxvLG9SIA4OAg4iBxEEHqIIKkeZE6iA2PYlGb2yR3bDNudxSLIBZx/HkN56HbZFugpHWXXp+0AOBmFpDzFaC5CahvGB5rKyzbV+LWksCMj25+gMgBANd8pWbmEWi5imurDMcZRlqVuAAyBSCVAzJ36CRuuF3PDImsqgFlVQ24dcdSVlWPsipDgqtr7N3NMEQcIBZxfHIVc1xzwmVw5hoh5+ogF9XBDbVw42rhhjq4crWQQAdHNMEROjigCQ7QGZ5zzeuYcZ3h0cHk0fBc3FymkZOgDjLUi5xQyzmjHjLUipxQDyfUck6o42SogxPquJbXtXBCLZxQBxn0MJ98jf9BcNwdfwpo/s/EUIDf07juzn0AYE3MGAz2dm33M6Q6rXtR/zDDcqfGOkBT3JzIbgKaG82PN1se6ysBbZWhcaoFDVRtRQzA29EZ3lK5IZnJFIYrPpkCkMsBn5bX9WJXVOqdUdEkw60mKep1YjCmB/R6MMYMz6EH0zOA6cAYwPQ6w4zbTA80l2HN5Tm0PNczPRp1DI06w2OTyaMejfqWdY06PZp0DFqdHk164z6GdXoAjmiCG+qaE09LApIbXzdvc0VzgmKGxAQGoIcPwlHDpKiFDNVMBg1ccJu54TbkqGBy3GZuqIDh8Tf+uRxVcALaSHZ3qtPa7j8lSlq9naMT4DnYsLSlobo5sRkTWvPz6jL+Cw+wNh5h+rzNsnpAW2Po3F2vBrTVhvKNtYal2vw8l0YyAH7NS48jbl46gYGDXuIGncQNOkfjoyv0IgmYyAF6kSP0nAMY/9iyTi9yMDxyLet0JusdoYMYYtYAh6ZaODbVQNxUC4emGjjwz2ubnzeva6zln3PMkE1duAa4oAHenNri96UXOUIr7YdGqQe0Eg9opR7QSvtBK+mHhubnDRIPBDpT0iJCSF0B72GGpbvomoAGjSGBGR/rNS1JzWRd5V3r1IBe13LbieZH4+tW6+4uJ7prnQ3rnDhR8+2vvI1Hhdn1nMQVYpGos7nP9hgDmuoN/+E0VBn+s2moNvyb1JQDteXNjxUtr2srgJoKoLEGIn0jZHVlkNV10Fh5aCrQL9xWMQv34YcfsqCgICaVSll4eDg7ffp0u+W//vprNnz4cCaVStmoUaNMJrZgjDG9Xs9WrVrF/Pz8mEwmYzNmzGC//PKLSZmKigr2xz/+kbm5uTGFQsGeffZZVlVVZXHMvW5iC0J6Om0tY78VMnYzm7FfUhnL+YqxExsZS01mbG88YzsWMvbp7xn7xzjGKn7t8HCWfkcFJ62dO3cyiUTCtmzZwi5dusSWLl3K3N3dWWlpqdnyJ06cYGKxmL377rssNzeXvfnmm8zR0ZFduHCBL7NmzRqmUCjY3r172blz59jcuXPZwIEDWV1dHV9m1qxZLDQ0lJ06dYr997//ZUOGDGELFy60OG5KWoT0bF2WtMLDw1l8fDz/WqfTsYCAAJaSkmK2/Pz589mcOXNM1kVERLDly5czxgxXWX5+fmzt2rX89srKSiaVStlXX33FGGMsNzeXAWBnzpzhy3z//feM4zh28+ZNi+KmpEVIz2bpd1RQU2WtVousrCwolUp+nUgkglKpREZG65E1ASAjI8OkPABERUXx5fPz86FSqUzKKBQKRERE8GUyMjLg7u6O8ePH82WUSiVEIhFOnz5t9rwNDQ3QaDQmCyGk9xOUtMrLy6HT6eDra9ph19fXl58V+m4qlard8sbHjsr4+PiYbHdwcICHh0eb501JSYFCoeCXwMBAC98lIaQnu2c7hSUlJUGtVvNLUVGRvUMihNiAoKTl5eUFsViM0tJSk/WlpaXw8zPfwsbPz6/d8sbHjsqUlZn+pNrU1ITbt2+3eV6pVAq5XG6yEEJ6P0FJSyKRICwsDGlpafw6vV6PtLQ0REZGmt0nMjLSpDwApKam8uUHDhwIPz8/kzIajQanT5/my0RGRqKyshJZWVl8mSNHjkCv1yMiIkLIWyCE9HZCa/h37tzJpFIp27ZtG8vNzWXLli1j7u7uTKVSMcYYe/rpp9kbb7zBlz9x4gRzcHBg69atY5cvX2bJyclmmzy4u7uzb7/9lp0/f5499thjZps8jBs3jp0+fZodP36cDR06lJo8EHIPsfQ7KrhF/IIFC3Dr1i2sXr0aKpUKY8eOxcGDB/mK9MLCQohELRdwEydOxI4dO/Dmm2/iz3/+M4YOHYq9e/di1KhRfJnXX38dNTU1WLZsGSorK/Hggw/i4MGDkMlkfJnt27cjISEBM2bMgEgkQkxMDD744AMhyRkA6FdEQnoo43eTdTCGQ58Z5eHGjRv0CyIhvUBRURHuu+++Nrf3maSl1+tRXFwMNzc3cB30RdNoNAgMDERRUVGvrMCn+O2L4rcOYwxVVVUICAgwuVu7W5/pMC0SidrN3ub09l8dKX77oviFUygUHZa5Z9tpEULuTZS0CCG9CiUtM6RSKZKTkyGVSu0dilUofvui+LtWn6mIJ4TcG+hKixDSq1DSIoT0KpS0CCG9CiUtQkivQkmLENKr9NmktWnTJgQHB0MmkyEiIgKZmZntlt+9ezdCQkIgk8kwevRoHDhwoJsiNZWSkoIJEybAzc0NPj4+iI6ORl5eXrv7bNu2rXna+Jblzs7o3emtt95qFUtISEi7+/SUzx4AgoODW8XPcRzi4+PNlrf3Z//jjz/i0UcfRUBAADiOw969e022M8awevVq+Pv7w8nJCUqlEleuXOnwuEK/P7bUJ5PWrl27kJiYiOTkZGRnZyM0NBRRUVGtBho0OnnyJBYuXIglS5bg7NmziI6ORnR0NC5evNjNkQPHjh1DfHw8Tp06hdTUVDQ2NmLmzJmoqalpdz+5XI6SkhJ+KSgo6KaIWxs5cqRJLMePH2+zbE/67AHgzJkzJrGnpqYCAJ544ok297HnZ19TU4PQ0FBs2rTJ7PZ3330XH3zwATZv3ozTp0/DxcUFUVFRqK+vb/OYQr8/NtelA+T0ULaeUcieysrKGAB27NixNsts3bqVKRSK7guqHcnJySw0NNTi8j35s2eMsT/96U9s8ODBTK/Xm93ekz57AGzPnj38a0tmwjJH6PfH1vrclVZXzChkT2q1YQpzDw+PdstVV1cjKCgIgYGBeOyxx3Dp0qXuCM+sK1euICAgAIMGDUJsbCwKCwvbLNuTP3utVosvv/wSzz77bLsjh/Skz/5OlsyEdTdrvj+21ueSVlfMKGQver0eL7/8MiZNmmQyqOLdhg8fji1btuDbb7/Fl19+Cb1ej4kTJ+LGjRvdGK1BREQEtm3bhoMHD+Kjjz5Cfn4+Jk+ejKqqKrPle+pnDwB79+5FZWUlnnnmmTbL9KTP/m6WzIR1N2u+P7bWZ4amuRfFx8fj4sWL7dYJAYYx9u8cw3/ixIkYMWIE/vnPf+Jvf/tbV4dpYvbs2fzzMWPGICIiAkFBQfj666+xZMmSbo2lsz777DPMnj0bAQEBbZbpSZ/9vaLPXWl1xYxC9pCQkIB9+/YhPT1d8Dhhjo6OGDduHK5evdpF0VnO3d0dw4YNazOWnvjZA0BBQQEOHz6M5557TtB+Pemzt2QmrLtZ8/2xtT6XtLpiRqHuxBhDQkIC9uzZgyNHjmDgwIGCj6HT6XDhwgX4+/t3QYTCVFdX49q1a23G0pM++ztt3boVPj4+mDNnjqD9etJnb8lMWHez5vtjc91S3d/DdMWMQt3lhRdeYAqFgh09epSVlJTwS21tLV/m7vj/+te/skOHDrFr166xrKws9uSTTzKZTMYuXbrU7fG/8sor7OjRoyw/P5+dOHGCKZVK5uXlxcrKyszG3pM+eyOdTscGDBjAVq5c2WpbT/vsq6qq2NmzZ9nZs2cZALZ+/Xp29uxZVlBQwBizbCas6dOns40bN/KvO/r+dLU+mbQYY2zjxo1swIABTCKRsPDwcHbq1Cl+29SpU1lcXJxJ+a+//poNGzaMSSQSNnLkSLZ///5ujtgAgNll69atfJm743/55Zf59+rr68sefvhhlp2d3f3BM8YWLFjA/P39mUQiYf3792cLFixgV69e5bf35M/e6NChQwwAy8vLa7Wtp3326enpZv9ejDHq9Xq2atUq5uvry6RSKZsxY0ar9xUUFMSSk5NN1rX3/elqNJ4WIaRX6XN1WoSQ3o2SFiGkV6GkRQjpVShpEUJ6FUpahJBehZIWIaRXoaRFCOlVKGkRQnoVSlqEkF6FkhYhpFehpEUI6VX+f4YbaRypcKEqAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:15:27.269912Z", + "iopub.status.busy": "2024-02-29T21:15:27.269495Z", + "iopub.status.idle": "2024-02-29T21:19:00.436388Z", + "shell.execute_reply": "2024-02-29T21:19:00.435547Z" + }, + "papermill": { + "duration": 213.190922, + "end_time": "2024-02-29T21:19:00.439253", + "exception": false, + "start_time": "2024-02-29T21:15:27.248331", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:19:00.479539Z", + "iopub.status.busy": "2024-02-29T21:19:00.479175Z", + "iopub.status.idle": "2024-02-29T21:19:00.502681Z", + "shell.execute_reply": "2024-02-29T21:19:00.501649Z" + }, + "papermill": { + "duration": 0.04618, + "end_time": "2024-02-29T21:19:00.504874", + "exception": false, + "start_time": "2024-02-29T21:19:00.458694", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.000002NaN0.001922.3507010.08961.7269370.1375760.00000510.9271710.033860.0628450.0438130.0665570.02134813.277872
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.000002 NaN 0.00192 2.350701 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.0896 1.726937 0.137576 0.000005 10.927171 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.03386 0.062845 0.043813 0.066557 0.021348 \n", + "\n", + " total_duration \n", + "realtabformer 13.277872 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:19:00.543786Z", + "iopub.status.busy": "2024-02-29T21:19:00.543412Z", + "iopub.status.idle": "2024-02-29T21:19:00.966135Z", + "shell.execute_reply": "2024-02-29T21:19:00.965057Z" + }, + "papermill": { + "duration": 0.444926, + "end_time": "2024-02-29T21:19:00.968610", + "exception": false, + "start_time": "2024-02-29T21:19:00.523684", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:19:01.015757Z", + "iopub.status.busy": "2024-02-29T21:19:01.014556Z", + "iopub.status.idle": "2024-02-29T21:22:44.889065Z", + "shell.execute_reply": "2024-02-29T21:22:44.887999Z" + }, + "papermill": { + "duration": 223.922532, + "end_time": "2024-02-29T21:22:44.913260", + "exception": false, + "start_time": "2024-02-29T21:19:00.990728", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:22:44.958148Z", + "iopub.status.busy": "2024-02-29T21:22:44.957274Z", + "iopub.status.idle": "2024-02-29T21:22:44.976165Z", + "shell.execute_reply": "2024-02-29T21:22:44.975327Z" + }, + "papermill": { + "duration": 0.043988, + "end_time": "2024-02-29T21:22:44.978378", + "exception": false, + "start_time": "2024-02-29T21:22:44.934390", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:22:45.020711Z", + "iopub.status.busy": "2024-02-29T21:22:45.020333Z", + "iopub.status.idle": "2024-02-29T21:22:45.026287Z", + "shell.execute_reply": "2024-02-29T21:22:45.025257Z" + }, + "papermill": { + "duration": 0.029407, + "end_time": "2024-02-29T21:22:45.028548", + "exception": false, + "start_time": "2024-02-29T21:22:44.999141", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.553993238691698}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:22:45.071171Z", + "iopub.status.busy": "2024-02-29T21:22:45.070722Z", + "iopub.status.idle": "2024-02-29T21:22:45.568209Z", + "shell.execute_reply": "2024-02-29T21:22:45.566928Z" + }, + "papermill": { + "duration": 0.522366, + "end_time": "2024-02-29T21:22:45.570807", + "exception": false, + "start_time": "2024-02-29T21:22:45.048441", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASsAAAE8CAYAAACYd5eXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGA0lEQVR4nO3dd3xT1/k/8M/VlmVZ3gu8MGaFPcMKkNAMyKAkZAc7hNEGSojLt4H0FwhkOEkTSkIT2iwbWkLatJCkzSQDyGTPkNgMD/A23trj/P64lmxjyZbkaw37eb9eelm6uro6V7Yen3Puc87hGGMMhBAS4ET+LgAhhLiDghUhJChQsCKEBAUKVoSQoEDBihASFChYEUKCAgUrQkhQoGBFCAkKFKwIIUGBghXx2pNPPgmO4wQ9Zl5eHjiOw+HDh93a/09/+hMGDBgAsViM0aNHC1oWElgoWBFBPfvss3j//fd98l6ff/45/vCHP2Dq1KnIzc3Fs88+65P3Jf4h8XcBSO/y7LPP4o477sC8efN6/L2++uoriEQivPXWW5DJZD3+fsS/qGbVR2i1Wn8XQXBVVVVQKpWCBSrGGPR6vSDH6g6dTufvIgQkCla9kL0v6cyZM7j33nsRERGBadOmAQD+8Y9/YNy4cVAqlYiMjMTdd9+Nixcvtnv9N998gwULFiA5ORlyuRxJSUl49NFHu/wicxwHrVaLbdu2geM4cByHrKwsAEBxcTEefvhhDB48GEqlElFRUViwYAGKioqcHkun02HZsmWIiopCWFgYFi5ciLq6unbvlZubC61W63ivvLw8AIDFYsFTTz2F9PR0yOVypKam4vHHH4fRaGz3Hqmpqbj55pvx2WefYfz48VAqlfjb3/6GvXv3guM4/Otf/8KGDRvQr18/qNVq3HHHHWhoaIDRaMSqVasQGxuL0NBQPPjggx2O7e5nPXPmTAwfPhxHjhzBNddcg5CQEDz++OOdfs59FTUDe7EFCxYgIyMDzz77LBhjeOaZZ/DEE0/gzjvvxOLFi1FdXY0tW7bgmmuuwbFjxxAeHg4AeO+996DT6fDb3/4WUVFROHjwILZs2YJLly7hvffec/l+f//737F48WJMnDgRS5cuBQCkp6cDAA4dOoTvv/8ed999N/r374+ioiJs3boVM2fOxJkzZxASEtLuWCtWrEB4eDiefPJJ5OfnY+vWrSguLnYEkr///e94/fXXcfDgQbz55psAgClTpgAAFi9ejG3btuGOO+7A73//exw4cAA5OTn4+eefsXv37nbvk5+fj3vuuQfLli3DkiVLMHjwYMdzOTk5UCqVWLNmDc6dO4ctW7ZAKpVCJBKhrq4OTz75JH788Ufk5eUhLS0N69atc7zW3c8aAC5fvoybbroJd999N+6//37ExcV5+JvuIxjpddavX88AsHvuucexraioiInFYvbMM8+02/fUqVNMIpG0267T6TocMycnh3Ecx4qLizu8T1sqlYplZmZ2eL2zY/7www8MANu+fbtjW25uLgPAxo0bx0wmk2P7Cy+8wACwDz74wLEtMzOTqVSqdsc8fvw4A8AWL17cbvvq1asZAPbVV185tqWkpDAA7NNPP22379dff80AsOHDh7crwz333MM4jmM33XRTu/0nT57MUlJSHI89+axnzJjBALC//vWvHT4f0h41A3ux3/zmN477u3btgs1mw5133omamhrHLT4+HhkZGfj6668d+yqVSsd9rVaLmpoaTJkyBYwxHDt2zKuytD2m2WzG5cuXMXDgQISHh+Po0aMd9l+6dCmkUqnj8W9/+1tIJBJ8/PHHnb6P/fns7Ox223//+98DAD766KN229PS0nDDDTc4PdbChQvblWHSpElgjGHRokXt9ps0aRIuXrwIi8UCwLPPGgDkcjkefPDBTs+LUDOwV0tLS3PcP3v2LBhjyMjIcLpv2y9lSUkJ1q1bhw8//LBdPxEANDQ0eFUWvV6PnJwc5ObmorS0FKzNBLXOjnllOUNDQ5GQkOCyj8uuuLgYIpEIAwcObLc9Pj4e4eHhKC4ubre97Wd0peTk5HaPNRoNACApKanDdpvNhoaGBkRFRXn0WQNAv3796GqmGyhY9WJtazM2mw0cx+GTTz6BWCzusG9oaCgAwGq14le/+hVqa2vx2GOPYciQIVCpVCgtLUVWVhZsNptXZfnd736H3NxcrFq1CpMnT4ZGowHHcbj77ru9PmZn3E1WbfsZXcnZ59TZdnsAdvezdqcMpBUFqz4iPT0djDGkpaVh0KBBLvc7deoUCgoKsG3bNixcuNCxfc+ePW69j6sg8e9//xuZmZl46aWXHNsMBgPq6+ud7n/27FnMmjXL8bi5uRnl5eWYM2dOp++fkpICm82Gs2fPYujQoY7tlZWVqK+vR0pKilvn0R3uftbEM9Rn1UfMnz8fYrEYGzZsaNcEA/gaweXLlwG01hra7sMYw8svv+zW+6hUKqcBSCwWd3jfLVu2wGq1Oj3O66+/DrPZ7Hi8detWWCwW3HTTTZ2+vz2Ybd68ud32TZs2AQDmzp3b1Sl0m7ufNfEM1az6iPT0dDz99NNYu3YtioqKMG/ePKjVahQWFmL37t1YunQpVq9ejSFDhiA9PR2rV69GaWkpwsLC8J///KdD35Ur48aNwxdffIFNmzYhMTERaWlpmDRpEm6++Wb8/e9/h0ajwbBhw/DDDz/giy++QFRUlNPjmEwmXHfddbjzzjuRn5+P1157DdOmTcOtt97a6fuPGjUKmZmZeP3111FfX48ZM2bg4MGD2LZtG+bNm9euttZT3P2siWcoWPUha9aswaBBg/DnP/8ZGzZsAMB3Fl9//fWOICCVSvHf//4XK1euRE5ODhQKBX79619jxYoVGDVqVJfvsWnTJixduhT/7//9P+j1emRmZmLSpEl4+eWXIRaLsWPHDhgMBkydOhVffPGFyytxf/nLX7Bjxw6sW7cOZrMZ99xzD1555RW3+qLefPNNDBgwAHl5edi9ezfi4+Oxdu1arF+/3oNPq3vc+ayJZzh2ZT2VEEICEPVZEUKCAgUrQkhQoGBFCAkKFKwIIUGBghUhJChQsCKEBIVen2dls9lQVlYGtVot+OIGhJDuYYyhqakJiYmJEIk6rzv1+mBVVlbWYZQ8ISSwXLx4Ef379+90n14frNRqNQD+wwgLC/NzaQghbTU2NiIpKcnxPe1Mrw9W9qZfWFgYBStCApQ7XTTUwU4ICQoUrAghQYGCFSEkKPT6Pit3MMZgsVhcTgRHXBOLxZBIJJQWQnpcnw9WJpMJ5eXltApuN4SEhCAhIYEWPSA9qk8HK5vNhsLCQojFYiQmJkImk1ENwQOMMZhMJlRXV6OwsBAZGRldJvYR4q0+HaxMJhNsNhuSkpI6rAhM3KNUKiGVSlFcXAyTyQSFQuHvInmtqsmAbwpqkBEXipH9w/1dHHIF+jcIUG2gm3rL52exMtTrzfjy5yrUak3+Lg65Qu/4KyNEAInhSiRq+JphfkWTn0tDrkTBipA2+kXwC46W1uv9XBJyJQpWpEupqakd1uHrbRr0ZpgsNiRo+GBV2WjosOYf8a8+3cFOiN3nP1WgtF6PG4fHQ8RxMFlsaDZaoFZI/V000oJqVn2EyUQdxp2p15nBGBCulEGj5P+H12nNXbyK+BIFKydMFpvLm8Vqc3tfs5v7emPmzJlYsWIFVqxYAY1Gg+joaDzxxBOOpktqaiqeeuopLFy4EGFhYVi6dCkA4Ntvv8X06dOhVCqRlJSElStXQqvVOo5bVVWFW265BUqlEmlpadixY4dX5QsmFitfiwIAjVKKaLUcYUopjBYa0RBIqBnoxKtfn3P5XFq0CvPG9HM8fn3/eZitzvs2+kcosWB868R/b39XCL2p4xfg0V8N8qqc27Ztw0MPPYSDBw/i8OHDWLp0KZKTk7FkyRIAwIsvvoh169Y5ViI+f/48brzxRjz99NN4++23UV1d7Qh4ubm5AICsrCyUlZXh66+/hlQqxcqVK1FVVeVV+YJFk4EPVDKJCAqpCHNHJFBycACiYBXEkpKS8Oc//xkcx2Hw4ME4deoU/vznPzuC1bXXXovf//73jv0XL16M++67D6tWrQIAZGRk4JVXXsGMGTOwdetWlJSU4JNPPsHBgwcxYcIEAMBbb72FoUOH+vzcfKnRwDf3whQ0xjGQUbByYvmsgS6fE13xt7z0mnSX+175d79oalp3itXB1Vdf3e7LNXnyZLz00kuOAdnjx49vt/+JEydw8uTJdk07xphj2FFBQQEkEgnGjRvneH7IkCEIDw8XtNyBpkHfEqyU1JkeyChYOSGTuN+V11P7CkGlUrV73NzcjGXLlmHlypUd9k1OTkZBQYGvihZQmluagWoF/3Vo0Jvx6elyWGwM901K8WfRSBsUrILYgQMH2j3+8ccfkZGRAbFY7HT/sWPH4syZMxg40HnNcciQIbBYLDhy5IijGZifn4/6+npByx1o4jQKjOinQf8IfnyoTCxCWb0BAN/5LhHTdahA4Nffwv79+3HLLbcgMTERHMfh/fffb/c8Ywzr1q1DQkIClEolZs+ejbNnz/qnsAGopKQE2dnZyM/Px86dO7FlyxY88sgjLvd/7LHH8P3332PFihU4fvw4zp49iw8++AArVqwAAAwePBg33ngjli1bhgMHDuDIkSNYvHgxlEqlr07JL9JjQjF7WBwGxfGLFiikIkha2vtaJxdEiH/4NVhptVqMGjUKr776qtPnX3jhBbzyyiv461//igMHDkClUuGGG26AwWDwcUkD08KFC6HX6zFx4kQsX74cjzzyiCNFwZmRI0di3759KCgowPTp0zFmzBisW7cOiYmJjn1yc3ORmJiIGTNmYP78+Vi6dCliY2N9cToBg+M4hMj5RofOZPFzaYgDCxAA2O7dux2PbTYbi4+PZ3/6058c2+rr65lcLmc7d+50+7gNDQ0MAGtoaOjwnF6vZ2fOnGF6vb5bZfeHGTNmsEceecTfxWCMBffnyBhj9VoT05sszGazOba9c6CYbfo8n52tbPJjyXq/zr6fVwrYxnhhYSEqKiowe/ZsxzaNRoNJkybhhx9+cPk6o9GIxsbGdjdCXGGMYfsPRdi69zwa9a21qBAZ3+9HNavAEbDBqqKiAgAQFxfXbntcXJzjOWdycnKg0WgcN1qNmXTGaLHBYuOTekPkrRcmVDK+Gag1Up9VoOh1VwPXrl2L7Oxsx2P7iq+9zd69e/1dhF7BPqJAJhFB2uaqX5hSijClFFIxJYkGioANVvHx8QCAyspKJCQkOLZXVlZi9OjRLl8nl8shl8t7unikl9C2NPPszT67iWmRmJgW6Y8iERcCthmYlpaG+Ph4fPnll45tjY2NOHDgACZPnuzHkpHeRNdSs7I3+0jg8utvqLm5GefOtQ4aLiwsxPHjxxEZGYnk5GSsWrUKTz/9NDIyMpCWloYnnngCiYmJmDdvnv8KTXoVezNQKXOeSEsCh1+D1eHDhzFr1izHY3tfU2ZmJvLy8vCHP/wBWq0WS5cuRX19PaZNm4ZPP/00qFdQIYHFYOaDlULaPlhpjRZ8eKIMJosNCyen0ADnAODXYDVz5sxOp47lOA4bN27Exo0bfVgq0pfEhikwsr8GieHts/QlYg4VDXzysdnKIJNQsPI3aqiTPi0tWoW0aFWH7TKxCGIRB6uNwWCx+nwQOumIfgOEOMFxHJQtTUMDjQ8MCBSsSJ/WbLTAYLY67Y5QtHS6680UrAIBNQNJn7b76CXUNJtw+9j+SI4KafecoqXpZzB7N08+ERbVrNpiDLCY/HPzYI267du3IyoqCkajsd32efPm4YEHHhD6U+nV7IFIIe34VVBSzSqgUM2qLasZ+OYl/7z39N8DEplbuy5YsAArV67Ehx9+iAULFgDgV6X56KOP8Pnnn/dkKXsde+qCXNoxz0qtkEKjlEJMaQsBgWpWQUipVOLee+91rEgDAP/4xz+QnJyMmTNn+q9gQcZsbR3E7KxmNWNQDBZNS8OI/hpfF404QTWrtsRSvobjr/f2wJIlSzBhwgSUlpaiX79+yMvLQ1ZWFiUvesBeqxJxHGQ0dXHAo2DVFse53RTztzFjxmDUqFHYvn07rr/+evz000/46KOP/F2soNK2v4qCfOCjYBXEFi9ejM2bN6O0tBSzZ8/ulVPh9CT7istyFwmfFQ0GfPVLFVRyMW4b3c/pPsR3qO4bxO69915cunQJb7zxBhYtWuTv4gQdpVSMkf01joUirsTAUNloQHWT0enzxLcoWAUxjUaD22+/HaGhoTQThReiQuW4bmgcpgyMdvq8PYPdaKE8q0BAwSrIlZaW4r777qMJB3uAfSYGk8UGi5UClr9Rn1WQqqurw969e7F371689tpr/i5OULJfDZSJRRCJOnawyyUiiDgONsagN1uhpiuGfkXBKkiNGTMGdXV1eP755zF48GB/FycofX++BicuNmDSgEhMSe/YFOQ4DkqZCFqjlQ9WCs/SS4iwKFgFqaKiIn8XIegZW1IX5BLXs4QqpGJojVbHvsR/KFiRPstktQcr1827MIUUFivzZOgm6SEUrIBOZyslXQvWz89+la+zifXmjaH8qkDRp3sMpVK+D0Kn0/m5JMHN/vnZP89gYbJ0XbMigaNP16zEYjHCw8NRVVUFAAgJCaFhFx5gjEGn06Gqqgrh4eEQi4NrhRh3alYkcPTpYAW0LqZqD1jEc+Hh4Y7PMZjYa1adDWLOr2jCkeI6JEUqMT0jxldFI070+WDFcRwSEhIQGxsLs9ns7+IEHalUGnQ1KrvB8aEwmG2drhlosthQ2WiASh6c59ib9PlgZScWi4P2S0e8c+2QuC73sc9zRakL/keNdUI6YR9yQ1Mb+x8FK9InWW0MBrMVNlvnaRf2YGWgYOV3FKxIn1TTbMTWvefx9neFne5nbwYazLagzSfrLShYkT6pdahN518Be83KxhhNFeNn1MFO+iSTtWXGhS6ClVQsQqhcArGIg9lqcwQv4nsBXbOyWq144oknkJaWBqVSifT0dDz11FNUHSfd5klC6JJrBmDRtDSadcHPArpm9fzzz2Pr1q3Ytm0brrrqKhw+fBgPPvggNBoNVq5c6e/ikSDWmhBKNaVgEdDB6vvvv8dtt92GuXPnAgBSU1Oxc+dOHDx40M8lI8HOSOMCg05A/6amTJmCL7/8EgUFBQCAEydO4Ntvv8VNN93k8jVGoxGNjY3tboRcyeRBM/DAhct450AJzpTR35I/BXTNas2aNWhsbMSQIUMgFothtVrxzDPP4L777nP5mpycHGzYsMGHpSTBKEYtx+B4NWLDup67vtloQWWjAQ16Go7lTwFds/rXv/6FHTt24J133sHRo0exbds2vPjii9i2bZvL16xduxYNDQ2O28WLF31YYhIshiaEYc6IBAyJD+tyX0diqIUSQ/0poGtW//d//4c1a9bg7rvvBgCMGDECxcXFyMnJQWZmptPXyOVyWumFCMqRGGqiYOVPAV2z0ul0EInaF1EsFsNmo+Q80j0mi63LoTZ2VLMKDAFds7rlllvwzDPPIDk5GVdddRWOHTuGTZs20erDpNv+eagENc0m3D62P5KjQjrdt3V8IP2T9KeADlZbtmzBE088gYcffhhVVVVITEzEsmXLsG7dOn8XjQQ5T5JCHTMvUDPQrwI6WKnVamzevBmbN2/2d1FIL2Nf2cadYKWUiiGXiiCXBnSvSa8X0MGKkJ7AGPMozypSJcPDMwf2dLFIF+hfBelzzG3WAaQM9uBBvynS59ibgCKOg0REqxkFC2oGkj7HaG6dHsbdpdc++6kCNc1GzB4ah7gwRU8Wj7hAwYr0ORKRCIPi1BB7UKuq1ZpQ1WhEs9GCrpeZID2BghXpczQhUswdmeDRa1qnN6b0BX+hPitC3KCQ0MIR/kbBivQ5Vhtze6iNnUJGWez+Rs1A0uccv1iP/QXVGJYYhhuucm/Ze6pZ+R/VrEifY08I9SRtwd5nRYud+g8FK9LneDLUxi5EJoFcKqK8LD+iZiDpc1oXi3A/WA2OV2NwvLqnikTcQDUr0ud4Mi6QBA76bZE+x77AqVxCy3AFE2oGkj7Hm5qV2WrDh8fLYLBYcdf4JEg8aEISYVCwIn1OvEYJmUQEtcL9P3+JiMOlOj1sjEFvtkJNwcrnKFiRPmfGoBiPX8NxHBRSEXQmKwxmG9Q0ltnn6N8DIW5qnYudcq38gYIV6VMYY2DMs6E2dvbEUCOtcuMX1AwkfYrJasNrX5+HTCLCsmsGeNRR3rpwBI0P9AevalYXLlwQuhyE+IR9VRurjXk0nxVA6wf6m1fBauDAgZg1axb+8Y9/wGAwCF0mQnpM27QFd2cJtVNKxY6ARXzPq2B19OhRjBw5EtnZ2YiPj8eyZctw8OBBoctGiOC8GWpjNz0jGr+dmY4JqZFCF4u4watgNXr0aLz88ssoKyvD22+/jfLyckybNg3Dhw/Hpk2bUF1dLXQ5CRFEd4baeFoTI8Lq1tVAiUSC+fPn47333sPzzz+Pc+fOYfXq1UhKSsLChQtRXl4uVDkJEYS9z4qW4Ao+3fqNHT58GA8//DASEhKwadMmrF69GufPn8eePXtQVlaG2267TahyEiKI7tSsapqN+M+RS/jfyTKhi0Xc4FXqwqZNm5Cbm4v8/HzMmTMH27dvx5w5cyAS8X8AaWlpyMvLQ2pqqpBlJaTbQuRipEaHINaLFHQbYyip1UElp052f/AqWG3duhWLFi1CVlYWEhKcrxISGxuLt956q1uFI0Ro6TGhSI8J9eq1yjZ5Vowx6sPyMa+agXv27MFjjz3WIVAxxlBSUgIAkMlkyMzM7HYBS0tLcf/99yMqKgpKpRIjRozA4cOHu31cQjxlT1uwMeaYbZT4jlc1q/T0dJSXlyM2Nrbd9traWqSlpcFqFSZprq6uDlOnTsWsWbPwySefICYmBmfPnkVERIQgxyd9T3dqRFIxP62xxcZgMNkgF4uAxjLA0ABo+gOKMIFLS9ryKli5GlvV3NwMhUK44ejPP/88kpKSkJub69iWlpYm2PFJ3/PRqXIUX9Zh5uAYXJWo8fj1IXIJGvVm6PTN0Jz7Arh8jn9CJAYG3QAkjBK4xMTOo2CVnZ0NgM83WbduHUJCQhzPWa1WHDhwAKNHjxascB9++CFuuOEGLFiwAPv27UO/fv3w8MMPY8mSJS5fYzQaYTQaHY8bGxsFKw8JfkazDSaLDSIva1cqmRhNOj1Ep/8D2Kr5IKWMALQ1QP4ngCIciEgRttAEgIfB6tixYwD4mtWpU6cgk8kcz8lkMowaNQqrV68WrHAXLlzA1q1bkZ2djccffxyHDh3CypUrO+0Py8nJwYYNGwQrA+ldvFnZpi2VXIKBzUcgNl4CwsOA0fcCoXHALx8BFaeAs58D4x8CRJTHJTSOeTFfxoMPPoiXX34ZYWE920aXyWQYP348vv/+e8e2lStX4tChQ/jhhx+cvsZZzSopKQkNDQ09Xl4S+LZ9X4RarQl3jOuPpMiQrl9wBVZfAu74OwBjwIgFQPRA/gmzHvhxK2AxAsNuBeKuErjkvVNjYyM0Go1b30+vwn9ubq5PvvgJCQkYNmxYu21Dhw51XHF0Ri6XIywsrN2NEDt7Uqhc6sWfPmPgzn3BB6r4Ea2BCgCkSqD/BP5+2TEBSkqu5HYzcP78+cjLy0NYWBjmz5/f6b67du3qdsEAYOrUqcjPz2+3raCgACkp1CdAvGNvBsrFXiR2Vp0BmioBiQxIn9Xx+YRRQPF3QP1FQFcLhNCAZyG5Haw0Go3jkq9G4/lVFG88+uijmDJlCp599lnceeedOHjwIF5//XW8/vrrPnl/0rvYbMz74TZWC3BhH7RGCw6bh0Kb34g5I1Tt91GEARFpQO0FoPoXIGWKQCUngAfBqm36QNv7PWnChAnYvXs31q5di40bNyItLQ2bN2/Gfffd55P3J72LxcaQEhUCk8XmebAqPw4YGmCVheK4OQOqer3z/WIG8cGqpoCClcC8yrPS6/VgjDlSF4qLi7F7924MGzYM119/vaAFvPnmm3HzzTcLekzSN8kkIswf29/zF1otQMmPAABx2lTYzkmhNVqdJ5hGZQDcZ0BjOWBsAuS05LxQvOpgv+2227B9+3YAQH19PSZOnIiXXnoJt912G7Zu3SpoAQnxu8pTjsCjSBoDgB9yYzA7GXIjDwVCW0Z21Lu+EEQ85/VModOnTwcA/Pvf/0Z8fDyKi4uxfft2vPLKK4IWkBC/stkctSokTYJYIoVSxnfONxstzl8Tnsz/pGAlKK+ClU6ng1rNV28///xzzJ8/HyKRCFdffTWKi4sFLSAhQims0eLVr8/hg+Ol7r+o6gygr+dTE1qG0qhagpXO5CpYpfI/KVgJyusFI95//31cvHgRn332maOfqqqqivKaSMAyWqwwWWwwW93Mg2YMKGlJPk6ayKcsAAhtWXa+yeAiWGn6AxzHpy8YaLiXULwKVuvWrcPq1auRmpqKSZMmYfLkyQD4WtaYMWMELSAhQvE4baGmgB/zJ5EDiWMdm9VyviloczX4Q6po7bdqoqm9heLV1cA77rgD06ZNQ3l5OUaNah1lft111+HXv/61YIUjREgmT+ZfZ4xP8ASAfuP4ANTi2iGxmD0srvPXqxP4BNLGMiBmsLdFJm14vSJzfHw84uPj222bOHFitwtESE/xqGZVe4EPNmJJ6zCaFiJ3FkdVJwA4TjUrAXkVrLRaLZ577jl8+eWXqKqqgs3W/hIurdhMApFjZZuu1gxkDChuGTyfOAaQeT7gmQ9W4IMVY3wfFukWr4LV4sWLsW/fPjzwwANISEiguahJUDC6W7OqLwEaLvFzVSVN6vC03mTFJ6fLoTVZcf+kZOd//6oYvlZmMfEd7aooIU6hT/MqWH3yySf46KOPMHXqVKHLQ0iP0SilSNAoEKaUdr6j/QpgwiinGegyiQgltTowBuhMVqjkTr5GIhEQGs8HvaYyClYC8CpYRUREIDKSRpST4DI5PQqT07sIGo1lQG0hwImc1qoAQCzioJJJ0Gy0oMlgcR6sAEDdEqyaKwGM6F7hiXepC0899RTWrVsHnU4ndHkI8S97X1XcVYAy3OVuYUp7rpXZ9bFUMfxPbY1AhevbvKpZvfTSSzh//jzi4uKQmpoKqbR9tfro0aOCFI4Qn2quBmrO8p3hyZM73TVMIUUZDGjQuxGsmqsELGTf5VWwmjdvnsDFIKTn5X1XCIuN4fax/RGhknXc4eIB/md0Rpd9TOEh/OvrdG4EK5MWMOm8u6pIHLwKVuvXrxe6HIT0uCaDBRYbc54nZWzixwECQNLVXR4rQsW3Jup0Jtc7SWR8U1JfD2irAFmqx2UmrbxegqO+vh5vvvkm1q5di9raWgB886+01INBooT4iNXGYLHxw2OcZrBfOgzYrPy4Pk2/Lo8XESKDXCrqOhue+q0E41XN6uTJk5g9ezY0Gg2KioqwZMkSREZGYteuXSgpKXHMdUVIoDBaWlcJl12ZFGoxti7ykNx1rQoAYtVy/HZGetc5hqoYvh+M+q26zauaVXZ2NrKysnD27Nl2KzDPmTMH+/fvF6xwhAjFaG5NCO3QDCw/wQeskCggaqCTV3fEcZx7ydD2Ac1aClbd5VWwOnToEJYtW9Zhe79+/VBRUdHtQhEiNKOrQcw2K3DpEH8/aaJXw2I6XXpTZQ9WNfywG+I1r4KVXC53uix7QUEBYmJiul0oQoRmbwYqpFcswVVzlp9zShYCxA336JgnL9Uj77tC/HDhsuudlOF8gqnVzHfiE695FaxuvfVWbNy4EWYzf9mW4ziUlJTgsccew+233y5oAQkRgojjEK9RIDr0ipSF0iP8z4TR/Fg+D9gYn7pQ3WR0vZNIDCgj+Pu6ToIa6ZJXweqll15Cc3MzYmJioNfrMWPGDAwcOBBqtRrPPPOM0GUkpNuSIkNwz8Rk3Dg8oXVjczU/aJkT8bMreMge+DoNVkDrYqe6Wo/fg7Ty6mqgRqPBnj178N133+HEiRNobm7G2LFjMXv2bKHLR0jPKWsZaRGdwS9Q6qHoUDkAPn/LYLZ2bGLaOYIV1ay6w+NgZbPZkJeXh127dqGoqAgcxyEtLQ3x8fHO11EjJBCZDUDFKf5+v7Gd7+uCQiqGRilFg55vCiZFushQD2nJhqdg1S0eNQMZY7j11luxePFilJaWYsSIEbjqqqtQXFyMrKwsmtKYBKx9BdV485sLOHmpnt9QeZrv9FZFA+EpXh83Ws3Xrqo6awpSsBKERzWrvLw87N+/H19++SVmzZrV7rmvvvoK8+bNw/bt27Fw4UJBC0lIdzUb+OlcrDbGpxDYk0ATx3ZrFs+YUDnOVzV33m+lbGkGGpv4yfgkTsYlki55VLPauXMnHn/88Q6BCgCuvfZarFmzBjt27BCscIQIxZ66IJeI+TmrtDX81b94z9IVrpQYrkC0Wo7wkE4m9JOF8OsOAoCeOtm95VGwOnnyJG688UaXz9900004ceJEtwtFiNAcSaFSEZ+xDgAxQ/hltrohJUqFB65OwdUDupjUj5qC3eZRsKqtrUVcnOsliOLi4lBXV9ftQrny3HPPgeM4rFq1qsfeg/RORnNLzYqztM6ukDCqk1cIjIJVt3kUrKxWKyQS191cYrEYFouLVWq76dChQ/jb3/6GkSNH9sjxSe9maKlZqRrP8R3ryghAkyTY8S1WW+ezhlKw6jaPOtgZY8jKyoJc7rzqbDR2kRznpebmZtx3331444038PTTT3e6r9FobFcOZ8OCSN/CGHMMZFZUn+Y3JowUbHmsc1XN+ORUOeI1CiwY7yIAUrDqNo9qVpmZmYiNjYVGo3F6i42N7ZErgcuXL8fcuXPdSjrNyclpV6akJOH+e5LgZLUxxIbJES/RQtZcxgcpD8cBdiY8RAqLjaGy0QCbzcVgZXtiqL6OBjR7yaOaVW5ubk+Vw6V3330XR48exaFDh9zaf+3atcjOznY8bmxspIDVx0nEItwzMRk4fx4o4YDIdK8y1l2JUskgk4hgsthQ02xEbJii404KTcuAZgtgbOQfE494vXy8L1y8eBGPPPII9uzZ027erM7I5XKXzVTSh9lsrRnrCcL2e3IchwSNAsWXdShvMDgPVvYBzbrL/BhBClYe83paY184cuQIqqqqMHbsWEgkEkgkEuzbtw+vvPIKJBIJrFZr1wchfsMYQ5PB7Lpp5Eu15/mFG2Qhbk+w54kEDZ9HVd6gd72ToylIuVbeCOia1XXXXYdTp0612/bggw9iyJAheOyxxyAWuxg4SvzOZmP436lynK9qRlSoDLeP7e96MdAeVlSjxYWv9yDJ1oSMcRP4Wo7AEsP52lRZvcH1To6pYihYeSOgg5Varcbw4e07QlUqFaKiojpsJ4HlxKV6nK9qBgBcbjZhf0E1bhqR0MWreoZe2whFYyEsSgkQ3zO5VfEaBTgOaNCboTW6WKWZporploBuBpLgZLMxHC2pBwAM78f3zeRXNnWeh9SDRFU/gQODOTQBCO2ZmWzlEjFGJ4VjekY0RK5SIpTUDOyOgK5ZObN3715/F4F0oaLRgEa9GXKpCDMHx8BksUGjlLr+EvckxiCt4rsSjNE9WxufOTi28x3suVaGBv6qoIczk/Z19GkRwYUqJJiSHgUrY5CKRZg70j/NPwD8oGVdDWycBNboIf4rBwDIVPyMCxYTYKjnp6chbqNmIBFcmEKKSQOiMCU9AL6MFSdhsTLUhqRCrlD26FsxxtCgN+OXikZYrLaOO3Bca1OQMtk9RsGK+ITZakNRjRY1zT0zJMspqxmoOgOLjaFKNRhKWc9fPd55sASfnKpAtavzpE52r1GwIoKq1ZpwrqoJzcb2A9r35ldj97FS5Ff4cDmq6nzAYgKnDIc4IgUqWc/2etiTQ4FOUhiok91rFKyIoM5WNuG/J8rxTUF1u+3xLVndFQ2d5CEJreIkAGDomKnImjbA9RzpAkoM7yI51DGgmYKVpyhYEUFVNPLBKF7TfshJnEbueL7TFYyFoq8D6ooFH7TclTg1f95VjV00A6lm5TEKVkRQNc0mAOgwPi5aJYdUzMFksaFO54N8K/s4wIhUflVkH4kN44Nyg94Mg9nJcDB7FrtJB5g7GZpDOqBgRQRjttrQqOcDUWRI+0URRCIOsS21jsrGHm4Kthm03Bw5DG9+cwH/PFTSs+/ZQiEVQ63g+8acLiIhkQPyUP4+NQU9QsGKCKZOx9eqFFKx0ytv0Wo+gF1uqX31XEEKAUMjIFVAqx6AppaVbXzFXqt0uTwXdbJ7hZJCiWDqtC21KpXzlV6iVHwTqcfTF0pbVlqOHwG9hc+al7taLbkHjEkKx9B4NRLCXeR1hUTxy9ZTzcojFKyIYOw1q4gQ5+vipUSF4FfD4hz9Oj1CX89PBwMACWNgaOL7jZQ+DFZdXnWkTnavULAighkaH4aIEBlCFc7/rMJDZAh3EcgEU36CnzY4IgVQRUF/mV9tyZfBqktKSgz1BgUrIhhNiBSazhb77Gk2a+uagIljAQD6litySplvu2cv1vKzhmbEhiJCdUWAbluzYkywhSt6OwpWxKcuNxtRWq9HpEqG/hECJ2nWFLTMBqoCojMAwJE+oPBxzepgYS1KanUIkYk7Biuaj90rdDWQCEJnsuBIcR2KarSd7vdzeRO+/LmqZ4bdXDrM/0wY5ZgNVCERIyJEijCFb2t8UaEtVz61Tq582udjB6gp6AGqWRFBVDcZsb+gGpEqGVKjVS7367H0hYZSoOESHwj6jXVsnjIwGlMG+n72B/uVz1ptJ5nsusstnexpvitYEKOaFRGEPSu9Q5PnCo70Ba1R2GE3Fw/wP2OHAXK1cMf1kqNm5SooO2pWdT4qUfCjYEUEUdfS3Lkyc/1KESH8jKFGsw1ak0CrE+nr+P4qAEiaKMwxuymyJWg3GSzOh92E0LxWnqJgRQRR2xKsIlwkhNpJxCKEt1wxvCxUcuilw/xVtcg0ILR1amGTxYY39l/AjgPFzifD60Fth9047beiLHaPUbAigugqIbQtexOpRoh+K2MTUHacv580qd1TOpMFzUYL6nVmSMS+/1O3n2ets/O8cj520iXqYCfdZrRYHWPvIrvoswL4fquzaBamZlXyI2CzAJp+/AwLbdibmSE+mCHUmanp0Zg60EXTmOZj9xgFK9Jt9S2d6yEysVv5TEMT1OgfoUR0aDeH3RgaW2tVqdM7JFfqWmYr7ekZQl1xuoy8nX0+9qYKPn2BglWXKFiRbotSyXDvpGTo3ewwF2zYjaNW1b9DrQpoU7OSB9BQm7ZC7MGKOtndQcGKdJtELEJcZ7WInqCrBcqO8fdTpzkdsmKvWfmrGcgYw7GL9bjcbMK0gdEdp82hTnaPUAc78YsL1c3YX1Dteq7yrpz/CmA2ICqdvwroRGuflX/+J3Mch+Ml9Thd2uB8Whyaj90jFKxItx24cBnHSuqgM7l/VSu/oglHiutwqc6LYFVbCNSc5cfXpV/ncjellB9qo1H6b3B1p8NuaKoYj1AzkHQLYwwHC2thsTGkRavgbldUVKgcQJPnVwStFuDsHv5+v7GAKsrlrtMyojEtw78d11EqOS5Ua50Pu7E3A+3zsUt7dhHWYBfQNaucnBxMmDABarUasbGxmDdvHvLz8/1dLNJGo8ECi41BLOI8Gizsda5V8Xd8h7RMxfdVBTh7KofT85TIaD52DwR0sNq3bx+WL1+OH3/8EXv27IHZbMb1118Prbbzkf3Ed+zDbCJCpBCJ3J+XKbpljGCd1gSbzc0xgk2V/BVAABh0Q1DURKLtiaFak/OxkNTJ7raAbgZ++umn7R7n5eUhNjYWR44cwTXXXOOnUpG2alsy1z1NRQhTSiAVczBbGer15q6TSa1m4OcP+U71mMH8rRPVTUbsOnoJMWo55o/t71HZhBShkoHjAL3JCp3JCpX8iq8czcfutoCuWV2poaEBABAZGelyH6PRiMbGxnY30nMcA5jdyFxvi+O4ln6rTqZRaevs54C2hm/+ZVzf5e5NBjN0JqtjplB/kYpFjg5++5CkdqiT3W0BXbNqy2azYdWqVZg6dSqGD3e9wm5OTg42bNjgw5L1bY4BzF4keUaqZKhoMKBW28Wip2XHgfKTfC7VsFtb+3k6YR/+E3plTcYP5o3uB5VcApnESd3Anr6grfFtoYKQ/3+Tblq+fDlOnz6Nb7/9ttP91q5di+zsbMfjxsZGJCUl9XTx+iz7UBtPa1YAcPWAKExJj+o8oNQWAgWf8fdTpznNVHemuSUh1NczhDrT6Rxf9mE2ulr+Sqc4aL6SPhcUn8yKFSvwv//9D/v370f//p33P8jlcsjlPbjUE2nngckpqNOZHFf3PNFl/lNjGfDTLr6fKm4YkDLV7WPbg2iYH3Os3CIPA6QKwGzgr3Kq4/xdooAV0H1WjDGsWLECu3fvxldffYW0NJr+NdAopGIkaJSQCj0FS/1F4MROflaC8CRg8FyPVoGpc3T8+z9YGcxWfPVLJd4/VtrxiiDHAaqWObi0Vb4vXBAJ6JrV8uXL8c477+CDDz6AWq1GRUUFAECj0UCpDPzL1qRrh4pqUd5gwLSB0a1NyfKTfNPPZgHCk4ERCzxqHjHG0KBvmWa5p9cpdINExOHUpUbYGEOz0QL1lU3T0Fj+imBzJYARfiljMAjomtXWrVvR0NCAmTNnIiEhwXH75z//6e+iEQA/lTVgb34VLtXpvD5GYY0W56ua+TGCxibgp93ALx/xgSo6Axh5J5886QGzlSFBo0B4iBRhLhZc9aW2s6PWOht2o4rhfzZX+7BUwcf/v8lOCLqgABHc+Wo+0GiUUu/WAGQM/VQMzWWXYP7pMCC7xC9UyomAlCkuZ1Poikwi8mtulTORKhlqtSbUNJuQEnXF6j+hbZqBtOipSwEdrEhgs+dY2Ves6YAx/ipX4yVAX88v6GnS8jezDjDpMLRZD1F1MziZGOgfzs9NlXF9r+tojgqV4VyVi3nnVTF8gDLp+M/GjdSMvoiCFfGK1cYcV9w6LBLRXA1UnASqzgDG5k6PE6qQwiQOQbUsCUNG/AryqJRu1yzMVpvwHf7dFO1IgHXSDBRL+WE3ust8vxUFK6coWBGv1OtMsDEGmUTUmidVf5EfaFxb2LqjSAKEJfL5RPIw/osoDeEz0aUhkMlUOP99CRr0ZlRwMUgRoAn0wfEy1GqNuH5YfKcLrvqS/eLB5ZYxgtyV5xka0xKsqvg5ukgHFKyIV2rbDLPhGkuBwv1AXTH/JCfiv3AJo4CItC6v5CVoFGjQm1Fap+/Yn+Mhxhiqm4wwmK1+myHUmYgQGcQiDlIxB73Z2nFCQHUCUPUL0FTunwIGAQpWxCuXtSbIzQ0YWvstcLSS3ygSA/EjgOSrW1ccdkNyVAjOVzfD7O7sC51oMvKLioo4zqus+p4iFnFYes0A1wtqqBP4nxSsXKJgRTxnNUNatA+jK35ETIQCUIQA8SOB1KmAQuPx4QbFqTE4Ti3I2n5VjQYAQGSozC9rBXam05V/1PF8X52hkU/hkKt9V7AgQcGKeKauGMj/BGO5WliSNWCR6cCg6/g+Fy8J2Rl+sWWa5P7hQZY0LJHzg5q1NUBjORBDwepKFKyIexjjJ74r3AcwBk4eBumgG4HogYK+TZ3W1PnA3y6UtgSrfhGBF6zqdSbsza+GyWLDnROcDK4PS+SDVVMZEDPI9wUMcBSsSNdsVuCX/wGVZ/jH8SOAjF/xtQGBWG0M/zp8ERUNBmROSfWqv0lnsjhWkekXgDUrmUSEwhp+llujxQq55IpmoTqBH2rUSP1WzgRWo54EHquFHwJTeYbvQB98I85FXoP3T9XgdGmDYG8jFnGOq3c/XvBu0U/GgAmpkRgUp+44I2cACJFJoG4Z/lPV6CQ5NKwf/7OpDLDZfFiy4EDBirhmswE/f8AveyWSAMNvBxLHoLTBiMIaLao9XZmmC5PTo8Bx/DJdZ8paZ3htO+zKZmMu52xXySWYOjAac0cmCFouIdkXg61qMnR8UhXDj4O0mFoGNZO2KFgR185/BVQX8DWqEXc4khWrm/ggFRMq7LxhsWoFxqXwKQ+f/VSB7T8U4Y39F1De0PrFLq7V4Y1vLuBIcS3M1uCrfTiClbOalUgEaJL5+/UlPixVcKBgRZwrOwZcOsTfH3KzY9Vje9IlAMSqhZ/kcGp6NMamRIDjgMvNJjQbLY73A4Ciy1roTFbsL6jBtu+LcPJSPSobDdh19BKKLwf+qkdxYfxnVtnopGYFABEp/E8KVh0EXsOe+F9TJXD2C/7+gBn8LJ0tGvRmGMxWiEU9k3QpEnGYMSgGY5PDcbnZBLlU1O59rsmIQUyoHD9euIwmgwVf/tw6YV2t1oTMKakBNy6wrVg1X7Oq05mhN1mhvDLLPrylZtVQwjfDRYF7Lr5GwYq0ZzECZ97n55OKGggkT273dFk9XyOIVct7NOlSrZB2nKQOfEf88H4aDIlX41RpA05eakCz0YLEcAVmDY4N6EAFAEqZGHFhCsgkIujNToKVKpa/ymox8tnsmn7+KWgAomBFWjEGFHzKT+siVwNDOk4lXNHI5zEl+Dk1QCIWYUxyBMYkuz+sJ1DcMzGp40BmO5GIb3JX/QJcPkfBqo3A/jdEfKv8OJ+iwImAq+YBMucT6skkIiRoFD4tWm/iMlDZRWXwPy+f7fnCBBGqWRFec1VrP1XaNfwkeE5cOyQOMwfFguZw7T69yQqpmOvYnI4cwNdom6v5SQuV4f4oXsChmhXh+0d+2t3ST5XOz5rQCZGIg1hEU+92xwfHS/HXfedRdNnJ/PWykNZ/FjVUu7KjYNXXdeinutnlTJ1Gi3+XYu9N7OsZ2offdBAzhP9ZedpHJQp8FKz6uvITrf1Uw25z2U8FADsPlCDvu8J2eU/EOwNaZjAtrGl2vjBK7FD+d9JUQUvLt6Bg1Zc1lAJn9/D3067hFxN1tavOjDqdGQ16i2N8G/Fev3AlZBIRtEYrKp1ls8tUrdMbV5z0beECFAWrvkpfD5z+d+v6fF30UxVUNQEA+kcoO59EjrhFIhYhJYqvxeZXNjnfKX4k/7P8BD9esI+jYNUXmbTAqff4pZ9CY4Ght3a5okx+Bf+FGhxPk8IJZWhCGADgl/JGWJ0Nzo4aCIREAmYDn1bSx1Gw6mtMWuD4O3w/iDyUH6DcxYrHlY0GVDcZIRZxSI+hZaKEkhalgkouhs5kdT6uUSQCkibx9y8eAKxm3xYwwFCw6ku0l4Gjf28NVKPvc2vO9GMl9QCAQXGhHYeHEK+JRBwmpUXhphHxSHW1qk/ccEARxq+/WPKDbwsYYChY9QWMAZU/AUe3Afo6Pslw1L18E6MLzUYLClr6VEYnBd/QlkA3KikcQ+LDIHKVtyaWAANn8/dLfuSvDvZRFKx6O20NcPo/wJkP+eRPTX9g7EJAFeXWy0PlEswf2w/jUyMQT0NsepTeZIXJ4mSOruhB/EUQmxX46X2+Kd8HBUWwevXVV5GamgqFQoFJkybh4MGD/i5SYGOMXx35zIfAoTf5LGhOBKRNB0bfy18W90D/iBBMz/B+9RrStQvVzdj+QxE+P1PRcSZUjuMHlSs0fM34xE7AINyU0sEi4IPVP//5T2RnZ2P9+vU4evQoRo0ahRtuuAFVVVVdv7gvsZj4ZdvPfQEc+Btw7B98048x/r/yhIeA1Gn8rJ9dYIzhSHEdSuv1Pig4AfjB4UaLDWcrm/Hx6XIYzFeMFpAqgVF38/9omquBw7l8SkMfmqudY07TZwPHpEmTMGHCBPzlL38BANhsNiQlJeF3v/sd1qxZ0+XrGxsbodFo0NDQgLCwsJ4ubs9gjM+Hshj4oGTW8v9ZDQ38f9qmCkB3md/PTiwBYocBiWOBsK7nJDdZbPwS7vV6nC5tQHWTETKJCPdNSkZ4SOCsbNybFVQ24dPTFbDaGBRSMYb3C0NqlArRofLWCxv6On4cZ1PLHO3yUH5ojqY/EBLND5mSyLtMRQkUnnw/AzoV2WQy4ciRI1i7dq1jm0gkwuzZs/HDD86vjBiNRhiNrRnBjY2NTvfroPh7vibijCMIMCfbXG0XcF+rme+v6IpcDUSm4bg2CqcNkbDWysAuG8BQCMb4ozHGcPfEZIS2rP7y/bkaHLtY36GvRCYRYdrAaGiUHSfAIz1jUJwaaoUEe85U4nKzCYeL6nC4qA5D4tW4aQT/D0crUuM9/XTE6E4jrv44pLZ64PwlxzGiVDL0j1IDIgksEOFkuQ5jU9r2T7YEMWfBTOgAJ1MDo+4S7HABHaxqampgtVoRFxfXbntcXBx++eUXp6/JycnBhg0bPH8zY3Pgj8HiOEAsA6QhfP+F/RYaxy8/LudzoC7/XInq2gYAzrOebW0CotnGHIFKJhEhPkyB1OgQXJWooUx1P0jQKHH/pBScr27GuapmlDcYENVmYQ4rY6gzMNTJrsLZmCEI119EuOESQk1VkFuaoLbY+H9sNitgs8GirQd0fqplKYTNug/oYOWNtWvXIjs72/G4sbERSUmux7w59BvH9+20++/S5r4n24U+hlgKSBR8oHLjv9+Y5AgMiuMzzUUiDlzL4Thw4DggpE0QGp8SgZH9NAiRizsuukn8QiTikBGnRkbL77BtT02IVIy7JiQ5asmMpbSrjKskNkBuBawWcFYz+g9sAjQKOGrpXdbmhTwRYcNLQAer6OhoiMViVFa2X0OtsrIS8fHxTl8jl8shl3ux6ooqyu3L+YEuUiVzezEHlVwClfCL1BABtZ1ZVCIWIdHNKaXFABLDnH9PglFAXw2UyWQYN24cvvzyS8c2m82GL7/8EpMnT+7klYSQ3iaga1YAkJ2djczMTIwfPx4TJ07E5s2bodVq8eCDD/q7aIQQHwr4YHXXXXehuroa69atQ0VFBUaPHo1PP/20Q6c7IaR3C/g8q+7qFXlWhPRSnnw/A7rPihBC7ChYEUKCAgUrQkhQCPgO9u6yd8m5PeyGEOIz9u+lO13nvT5YNTXxE8e5lcVOCPGLpqYmaDSdz1rb668G2mw2lJWVQa1Wt8sEFpJ9SM/Fixf7xBXHvna+QN87Z1+dL2MMTU1NSExMhEjUea9Ur69ZiUQi9O/f3yfvFRYW1if+kO362vkCfe+cfXG+XdWo7KiDnRASFChYEUKCAgUrAcjlcqxfv9672R6CUF87X6DvnXMgnm+v72AnhPQOVLMihAQFClaEkKBAwYoQEhQoWBFCggIFKzd4siL0G2+8genTpyMiIgIRERGYPXt2UK4g7e0q2O+++y44jsO8efN6toA9wNNzrq+vx/Lly5GQkAC5XI5Bgwbh448/9lFpu8/T8928eTMGDx4MpVKJpKQkPProozAYDD4qLQBGOvXuu+8ymUzG3n77bfbTTz+xJUuWsPDwcFZZWel0/3vvvZe9+uqr7NixY+znn39mWVlZTKPRsEuXLvm45N7z9JztCgsLWb9+/dj06dPZbbfd5pvCCsTTczYajWz8+PFszpw57Ntvv2WFhYVs79697Pjx4z4uuXc8Pd8dO3YwuVzOduzYwQoLC9lnn33GEhIS2KOPPuqzMlOw6sLEiRPZ8uXLHY+tVitLTExkOTk5br3eYrEwtVrNtm3b1lNFFJw352yxWNiUKVPYm2++yTIzM4MuWHl6zlu3bmUDBgxgJpPJV0UUlKfnu3z5cnbttde225adnc2mTp3ao+Vsi5qBnbCvCD179mzHtq5WhL6STqeD2WxGZGRkTxVTUN6e88aNGxEbG4uHHnrIF8UUlDfn/OGHH2Ly5MlYvnw54uLiMHz4cDz77LOwWt1YOdvPvDnfKVOm4MiRI46m4oULF/Dxxx9jzpw5Pikz0AcGMneHNytCX+mxxx5DYmJiuz+MQObNOX/77bd46623cPz4cR+UUHjenPOFCxfw1Vdf4b777sPHH3+Mc+fO4eGHH4bZbMb69et9UWyveXO+9957L2pqajBt2jQwxmCxWPCb3/wGjz/+uC+KDIA62HvUc889h3fffRe7d++GQqHwd3F6RFNTEx544AG88cYbiI6O9ndxfMZmsyE2Nhavv/46xo0bh7vuugt//OMf8de//tXfResRe/fuxbPPPovXXnsNR48exa5du/DRRx/hqaee8lkZqGbVCW9WhLZ78cUX8dxzz+GLL77AyJEje7KYgvL0nM+fP4+ioiLccsstjm02mw0AIJFIkJ+fj/T09J4tdDd583tOSEiAVCqFWCx2bBs6dCgqKipgMpkgk7m3IrY/eHO+TzzxBB544AEsXrwYADBixAhotVosXboUf/zjH7uci0oIVLPqhLcrQr/wwgt46qmn8Omnn2L8+PG+KKpgPD3nIUOG4NSpUzh+/Ljjduutt2LWrFk4fvx4UMzQ6s3veerUqTh37pwjMANAQUEBEhISAjpQAd6dr06n6xCQ7IGa+Wp4sc+68oPUu+++y+RyOcvLy2NnzpxhS5cuZeHh4ayiooIxxtgDDzzA1qxZ49j/ueeeYzKZjP373/9m5eXljltTU5O/TsFjnp7zlYLxaqCn51xSUsLUajVbsWIFy8/PZ//73/9YbGwse/rpp/11Ch7x9HzXr1/P1Go127lzJ7tw4QL7/PPPWXp6Orvzzjt9VmYKVm7YsmULS05OZjKZjE2cOJH9+OOPjudmzJjBMjMzHY9TUlIYgA639evX+77g3eDJOV8pGIMVY56f8/fff88mTZrE5HI5GzBgAHvmmWeYxWLxcam958n5ms1m9uSTT7L09HSmUChYUlISe/jhh1ldXZ3PyktTxBBCggL1WRFCggIFK0JIUKBgRQgJChSsCCFBgYIVISQoULAihAQFClaEkKBAwYoQEhQoWJFeJS8vD+Hh4Y7HTz75JEaPHu14nJWVFZRTLhMKVsRLWVlZ4DgOv/nNbzo8t3z5cnAch6ysrHb7Cx0kUlNTsXnz5nbb7rrrLhQUFLh8zcsvv4y8vDzH45kzZ2LVqlWClov0DApWxGtJSUl49913odfrHdsMBgPeeecdJCcn+6VMSqUSsbGxLp/XaDTtal4keFCwIl4bO3YskpKSsGvXLse2Xbt2ITk5GWPGjOnWsZ3VeObNm+eorc2cORPFxcV49NFHwXEcOI4D0LEZeKW2NbysrCzs27cPL7/8suMYhYWFGDhwIF588cV2rzt+/Dg4jsO5c+e6dV7EexSsSLcsWrQIubm5jsdvv/02HnzwwR5/3127dqF///7YuHEjysvLUV5e7vExXn75ZUyePBlLlixxHCM5ObnDOQFAbm4urrnmGgwcOFCoUyAeomBFuuX+++/Ht99+i+LiYhQXF+O7777D/fff3+PvGxkZCbFYDLVajfj4+C5nbnVGo9FAJpMhJCTEcQyxWIysrCzk5+c7Fkcwm8145513sGjRIqFPg3iApjUm3RITE4O5c+ciLy8PjDHMnTs36OdiT0xMxNy5c/H2229j4sSJ+O9//wuj0YgFCxb4u2h9GtWsSLctWrQIeXl52LZtm2C1D5FI1GG6XLPZLMix3bF48WLHxYPc3FzcddddCAkJ8dn7k44oWJFuu/HGG2EymWA2m3HDDTcIcsyYmJh2/VBWqxWnT59ut49MJuv2On2ujjFnzhyoVCps3boVn376KTUBAwA1A0m3icVi/Pzzz477rjQ0NHRYWzAqKsrpohLXXnstsrOz8dFHHyE9PR2bNm1CfX19u31SU1Oxf/9+3H333ZDL5V41P1NTU3HgwAEUFRUhNDQUkZGREIlEjr6rtWvXIiMjo9MFQohvUM2KCCIsLAxhYWGd7rN3716MGTOm3W3Dhg1O9120aBEyMzOxcOFCzJgxAwMGDMCsWbPa7bNx40YUFRUhPT0dMTExXpV79erVEIvFGDZsGGJiYlBSUuJ47qGHHoLJZPLJ1U3SNZqDnRAXvvnmG1x33XW4ePFih9WLie9RsCLkCkajEdXV1cjMzER8fDx27Njh7yIRUDOQkA527tyJlJQU1NfX44UXXvB3cUgLqlkRQoIC1awIIUGBghUhJChQsCKEBAUKVoSQoEDBihASFChYEUKCAgUrQkhQoGBFCAkK/x/yuffYOdYKkwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:22:45.616087Z", + "iopub.status.busy": "2024-02-29T21:22:45.615696Z", + "iopub.status.idle": "2024-02-29T21:22:45.984779Z", + "shell.execute_reply": "2024-02-29T21:22:45.983754Z" + }, + "papermill": { + "duration": 0.394626, + "end_time": "2024-02-29T21:22:45.987129", + "exception": false, + "start_time": "2024-02-29T21:22:45.592503", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASsAAAEmCAYAAAA6OrZqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCfUlEQVR4nO3deXxTZb4/8M/J3qRpum80paXssoswLCog44KiiKPMuFFwHWHmQn/cAfQKgijoqBdGHVQUi1cW7yio1wVEEBhQWS2CLIXSBei+pknTbOf8/jhN2tIkbZKTrf2+X6++miYnJ89pm2+e85zv830YjuM4EEJIiBMFuwGEENIVFKwIIWGBghUhJCxQsCKEhAUKVoSQsEDBihASFihYEULCAgUrQkhYkAS7Af7GsixKS0uhVqvBMEywm0MIaYPjODQ2NiI1NRUikfu+U7cPVqWlpdBqtcFuBiHEjcuXLyMtLc3tNt0+WKnVagD8LyMqKirIrSGEtKXT6aDVah3vU3e6fbCyn/pFRUVRsCIkRHVliIYG2AkhYYGCFSEkLFCwIoSEhW4/ZkV8x3EcrFYrbDZbsJtCwpBUKoVYLPZ5PxSsiFtmsxllZWVoamoKdlNImGIYBmlpaYiMjPRpPxSsiEssy6KwsBBisRipqamQyWSUWEs8wnEcqqqqcOXKFfTr18+nHhYFK+KS2WwGy7LQarVQKpXBbo7fWWws9M1WyKUiKGX01hBKQkICioqKYLFYfApWNMBOOtXZNIjuguMAK8uhsdkKq40NdnO6DaF64z3jv5CQLpBJRJCJ+TdWs4WCVaihYEVIG1IJ/5YwU88q5FCwIgSAlWXBchykYv4tYbWxuHaVuqKiIjAMg7y8PJ9eq6v7OXToEIYOHQqpVIoZM2b49JrdAQUrQgDojFZUNZpgZfkAxQFgO1lSc9++fWAYBvX19X5pU05ODkaMGIHCwkLk5ub65TXCCQUr0iOYzWa3j9tagpSEYSAW8eNW9sAVLAUFBZgyZQrS0tIQHR3t1T46O24h2ZOH/YWCFfGK2cq6/Lr2Spq7bS1d3NZTkyZNwvz587FgwQLEx8fjtttuw+nTp3HHHXcgMjISSUlJeOSRR1BdXQ2O48ByHPZ+/x0mT7oJWWlJGJTRC/feczcKCgqc7r+oqAiTJ08GAMTExIBhGGRnZwMAdu7ciYkTJyI6OhpxcXG46667nO7n3LlzGD9+PBQKBYYMGYL9+/c79s0wDGpqajB37lwwDOPoWe3fvx9jxoyBXC5HSkoKlixZ0i5AODtuew9w165dGDlyJCIiIjBlyhRUVlbi22+/xaBBgxAVFYUHH3ywXfIvy7JYvXo1MjMzERERgeHDh+PTTz91PG7f77fffovrr78ecrkcBw8e9Phv1VWUTEK88vYPF10+lhmvwoyRvRw/v3egABab815KWkwE7h/dWhxx46FCGM0dp/Us/H1/j9u4adMm/PnPf8ahQ4dQX1+PKVOm4PHHH8d///d/w2g0YvHixXjggQfw3e7vAQBGgwELFy7EsGHDYDAYsGzZMtx7773Iy8vrkL6h1Wrx2Wef4b777sP58+cRFRWFiIgIAIDBYEBOTg6GDRsGvV7vcj//+Z//ibVr12Lw4MF44403MH36dBQWFkKr1aKsrAwDBgzAypUrMWvWLGg0Gly9ehXTpk1DdnY2PvroI5w7dw5PPPEEFAoFXnjhBafHDQBlZWUAgBdeeAFvvfUWlEolHnjgATzwwAOQy+XYsmUL9Ho97r33Xrz55ptYvHgxAGD16tX4+OOP8c4776Bfv344cOAAHn74YSQkJODmm292vN6SJUvw2muvoU+fPoiJifH479RVFKxIt9WvXz+8+uqrAIBVq1Zh5MiRePnllx2Pb9y4EVqtFufOn0dCWibuuXcm4iLl7R5PSEjAmTNnMGTIkHb7FovFiI2NBQAkJia2O02777772m3raj/z5893bLt+/Xrs3LkTH3zwAf72t78hOTkZDMNAo9EgOTkZAPDPf/4TWq0Wb731FhiGwcCBA1FaWorFixdj2bJljkDY9riB1mC1atUqTJgwAQDw2GOPYenSpSgoKECfPn0AAH/4wx/www8/YPHixTCZTHj55Zfx/fffY9y4cQCAPn364ODBg3j33XfbBauVK1fi97//fdf+KD6gYEW8Mm9yX5ePia7JAXzypiyX216bLzh3QqYvzWrn+uuvd9w+efIkfvjhB6fz0y5cLEBCWiaKLl3E/NWrcPjwYVRXV4Nl+dPPkpKSDsHKnQsXLmDZsmWd7sceBABAIpFg9OjROHv2rMv9nj17FuPGjWuXZDlhwgTo9XpcuXIF6enpHY67rWHDhjluJyUlQalUOgKV/b4jR44AAC5evIimpqYOQchsNmPkyJHt7hs9erTLNguJghXxikzS9eFOf23bGZVK5bit1+sxffp0vPLKKx22U8fEAwD+dP9MZGZk4J1334U6NhEsy2LimFEeD1JPnz4dvXv3xoYNG5CamgqWZTFkyJCADXa3Pe62pFKp4zbDMO1+tt9nD6x6vR4A8PXXX6NXr17ttpPL5e1+dvV6QqNgRXqEUaNG4bPPPkNGRgYkkvb/9s0WG8orqnAhPx8fvP8+JkyYiCq9CYd/OuR2nzKZDADalc6pqanB+fPnsWHDBtx4440A4HLQ+eeff8ZNN90EALBarTh+/Djmz5/v8vUGDRqEzz77DBzHOXpXhw4dglqt7nSxBU8NHjwYcrkcJSUl7U75gimoVwMPHDiA6dOnIzU1FQzD4PPPP2/3OMdxWLZsGVJSUhAREYGpU6fiwoULwWksCWvz5s1DbW0t/vSnP+Ho0aMoKCjArl27MGfOHEhFQHpqIuLi4vDee++hoOAiDu7fh+XPLna7z969e4NhGHz11VeoqqqCXq9HTEyMYz8XL17E3r17kZOT4/T5b7/9Nnbs2IFz585h3rx5qKurw9y5c12+3jPPPIPLly/jL3/5C86dO4cvvvgCy5cvR05OjuDzN9VqNRYtWoSFCxdi06ZNKCgowIkTJ/Dmm29i06ZNgr5WVwU1WBkMBgwfPhxvv/2208dfffVV/OMf/8A777yDw4cPQ6VS4bbbbkNzc3OAW0rCXWpqKg4dOgSbzYZbb70VQ4cOxYIFCxAdHQ2RSASRSIRt27bh+PHjGDp0KJY9+zcse3G123326tULK1aswJIlS5CUlIT58+e328+QIUOwcOFC/P3vf3f6/DVr1mDNmjUYPnw4Dh48iC+//BLx8fFuX++bb77BkSNHMHz4cDz99NN47LHH8F//9V8+/W5cefHFF/H8889j9erVGDRoEG6//XZ8/fXXyMwUblzRI1yIAMDt2LHD8TPLslxycjL397//3XFffX09J5fLua1bt3Z5vw0NDRwArqGhQcjm9ghGo5E7c+YMZzQag90Uv7JYbZzNxnIsyzruq9Y3c+UNRs5otgaxZd2Du/8jT96fIZsUWlhYiPLyckydOtVxn0ajwdixY/HTTz+5fJ7JZIJOp2v3RYgrHMehxmBGld7kyGIHAHHLmFBnU25I4IRssCovLwfAX05tKykpyfGYM6tXr4ZGo3F80WrMxJ22sUjUJudCZA9WVHwhZIRssPLW0qVL0dDQ4Pi6fPlysJtEQpi958SgNUABgFjEQMQwHfLASPCEbOqCPWu3oqICKSkpjvsrKiowYsQIl8+Ty+Ud8kAIccUerETXZLKq5BKo5CH79uiRQrZnlZmZieTkZOzZs8dxn06nw+HDh9tl/hLiC/swlYi6UCEvqB8der0eFy+2TogtLCxEXl4eYmNjkZ6ejgULFmDVqlXo168fMjMz8fzzzyM1NZUKkRHBOHpWFKtCXlCD1bFjxxxlNgA4kudmz56N3Nxc/O1vf4PBYMCTTz6J+vp6TJw4ETt37oRCoQhWk0k346pnZWM51BvN4DggTkVLkIUChuO697VZnU4HjUaDhoYGREVFBbs5YaW5uRmFhYXIzMzsth8QJosNJisLqViECFnrMlEsx6Gq0QQASFDL6TTRB+7+jzx5f9IIIunR5FIx5NKOa9m1DU0cx3UsD0ECLmQH2AnxxaRJk7BgwQKvn88wTGuulQDnHr62h1CwIj2cjeXAslyHlWwAwGrhS7pQFntooGBFup3s7Gzs378f69atA8MwYBgGBQUFeOyxxxz1xAcMGIB169ahvomfamO2scjOzsaMGTPw0ksvITU1Fb8bNRQA8OOPP2LEiBFQKBQYPXo0Pv/88w5Labmq7+6qPUVFRUH4zYQ3GrMinuE4wGYJzmuLpV0aO1q3bh3y8/MxZMgQrFy5EgC/qENaWhr+9a9/IS4uDj/++COefPJJqKLjMf3e+xynfHv27EFUVBR2794NndGCRp0Of7h3BqZNm4YtW7aguLi4w+mcu/rue/fuddqehIQEYX83PQAFK+IZmwX49+vBee0b/x8gkXW6mUajgUwmg1KpdMyEAIAVK1Y4bmdmZuKnn37C59s/xfR773PEQJVKhffffx8ymQyNzRa8+867YBgGGzZsgEKhwODBg3H16lU88cQTjn299dZbLuu75+fno3///k7bQzxDwYr0GG+//TY2btyIkpISGI1GmM1mXDeUr0tu71kNHTrUUQFUrZDianEBhg0b1u6S+5gxY9rt111994KCAvTv7/nKPKQjClbEM2Ip38MJ1mt7adu2bVi0aBFef/11jBs3Dmq1Gq+++ip+/PkwgNZUBW/qibur7952XivxDQUr4hmG6dKpWLDJZLJ2tdEPHTqE8ePH45lnnnHcV1BwCQA/1cZVhvqAAQPw8ccfw2QyOSbIHz16tN027uq7u2oP8RxdDSTdUkZGBg4fPoyioiJUV1ejX79+OHbsGHbt2oX8/Hw8//zzOHaMDzquApXFyuL2u++DjWXx5JNP4uzZs9i1axdee+21ds9zV9/dHqCubQ9LhbI8RsGKdEuLFi2CWCzG4MGDkZCQgNtuuw0zZ87ErFmzMHbsWNTU1ODpp/8MEQCFi+W/OAARkWr8zyefIS8vDyNGjMBzzz2HZcuWAYBjHKuz+u7O2lNSUhKIX0O3QnMDiUs9YW6gO1YbixqDGQyAxKjW49+8eTPmzJmDhoYGx5LxxDWaG0iIn9mvEH6ydTNGDO6PtLQ0nDx50pFDRYEqsChYkR6LbZn0x7gYYLffVVlRjkdWv4jy8nKkpKTg/vvvx0svvRTIphJQsCI9mN5khdFig0omRqSiY1oEP5kZmL/g/2HZc0shFdMQbzDRb5/0WPbhWneF9eyPdfOh3bBAPSvSY9nDj7uSxmIRA3Ct25LgoWBFOtVdexX2OlXuelYxytBPgA11Qv3/0GkgcUkq5cdxmpqagtwS/2g9DQxyQ7o5s5mvCyYWd6zI6gnqWRGXxGIxoqOjUVlZCQBQKpXdauEEk8kMluNgErHgXCSGEt+wLIuqqioolUqXU5G6ioIVccte0sQesLqTxmYLOA6ol0v4sSknLDYWZisLsYiBwkmtdtI5kUiE9PR0nz/oKFgRtxiGQUpKChITE2GxBKnonp/8fKkaJiuL/tpYRMicvxXyyxtxrKAa2lglbumfFOAWdg8ymcwx7cgXFKxIl4jFYp/HHELNpMFpnW4TEWGBkRXDYBX1yClHoYRO1Alxw37qZ7RQeZdgo2BFeiQby6HZYnNMuXHFHqyaKVgFHQUr0iNV601Yv68AGw8Vut1OIeXfIs0Wttvmm4ULClakRzJZ+OJ38k5SFuw9K5bjYLJSwbxgCulgZbPZ8PzzzzvWesvKysKLL75In3DEZ+aWCp6yToKVVCxCpFwCTYQUFhsFq2AK6auBr7zyCtavX49Nmzbhuuuuw7FjxzBnzhxoNBr89a9/DXbzSBiz95I6C1YA8MRNffzdHNIFIR2sfvzxR9xzzz248847AfB1rLdu3YojR44EuWUk3JntwaqbpWN0ZyF9Gjh+/Hjs2bMH+fn5APj12Q4ePIg77rgjyC0j4c7es+pszIqEjpDuWS1ZsgQ6nQ4DBw6EWCyGzWbDSy+9hIceesjlc0wmE0wmk+NnnU4XiKaSMGP24DTw8KUaFFQZMEIbjcGpVMc/WEL6Y+V///d/sXnzZmzZsgUnTpzApk2b8Nprr2HTpk0un7N69WpoNBrHl1arDWCLSbhIUMsxIFmNxCh5p9vqTVZU6JrRYOxe043CTUivbqPVarFkyRLMmzfPcd+qVavw8ccf49y5c06f46xnpdVqaXUb4rVDF6txpLAWI9KjMXlAYrCb0610m9VtmpqaOkyAFIvFbheIlMvljpVzCRGCIzHUTFnswRTSwWr69Ol46aWXkJ6ejuuuuw6//PIL3njjDcydOzfYTSNhzmxlIRExELmradzCMeXGSsEqmEI6WL355pt4/vnn8cwzz6CyshKpqal46qmnHCviEuKtT46WoFpvxn2j0pAep3S7bev8QEoKDaaQDlZqtRpr167F2rVrg90U0s14khTqqLxAp4FBFdJXAwnxF7Ot68EqQiqGXCqCXEpvl2AK6Z4VIf7AcZxHeVaxKhmemdTX380inaCPCtLjWGwc7Ak7lMEePugvRXoc+ymgiGEg6cLVQBIa6DSQ9DgmS2t5mK6uuLLrt3JU602YOigJSVFUiz0YKFiRHkciEqF/ktrl8lvO1BrMqNSZoDdZQWvcBAcFK9LjaJRS3DksxaPntJY3pvSFYKExK0K6QCGhhSOCjYIV6XFsLNfpqjbXUsgoiz3Y6DSQ9Dh5l+txIL8Kg1OjcNt1yV16DvWsgo96VqTHsSeEepK2YB+zosVOg4eCFelxPJlqY6eUSSCXiigvK4joNJD0OK2LRXQ9WA1IVmNAstpfTSJdQD0r0uN4Mi+QhA76a5Eex77AqVxCy3CFEzoNJD2ONz0ri43Fl3mlaLbaMGu0FhIPTiGJMChYkR4nWRMBmUQEtaLr//4SEYMrdUawHAejxQY1BauAo2BFepyb+yd4/ByGYaCQitBktqHZwkJNc5kDjj4eCOmi1lrslGsVDBSsSI/CcRy8XSrTnhhqolVugoJOA0mPYrax+OcPBZBJRHjqpj4eDZS3LhxB8wODwaue1aVLl4RuByEBYV/VxsZyHtWzAmj9wGDzKlj17dsXkydPxscff4zm5mah20SI37RNW+hqlVC7CKnYEbBI4HkVrE6cOIFhw4YhJycHycnJeOqpp3DkyBGh20aI4LyZamN3Y794/HlSFm7IiBW6WaQLvApWI0aMwLp161BaWoqNGzeirKwMEydOxJAhQ/DGG2+gqqpK6HYSIghfptp42hMjwvLpaqBEIsHMmTPxr3/9C6+88gouXryIRYsWQavV4tFHH0VZWZlQ7SREEPYxK1qCK/z49Bc7duwYnnnmGaSkpOCNN97AokWLUFBQgN27d6O0tBT33HOPUO0kRBC+9Kyq9SZ8dvwKvvq1VOhmkS7wKnXhjTfewIcffojz589j2rRp+OijjzBt2jSIRPw/QGZmJnJzc5GRkSFkWwnxmVIuRka8EolepKCzHIeS2iao5DTIHgxe9azWr1+PBx98EMXFxfj8889x1113OQKVXWJiIj744AOfG3j16lU8/PDDiIuLQ0REBIYOHYpjx475vF/SM2UlROLekWkYlxXn8XMj2uRZeZtYSrznVc9q9+7dSE9P7xCgOI7D5cuXkZ6eDplMhtmzZ/vUuLq6OkyYMAGTJ0/Gt99+i4SEBFy4cAExMTE+7ZcQb9jTFliOg9nGUomZAPMqWGVlZaGsrAyJiYnt7q+trUVmZiZsNmGS5l555RVotVp8+OGHjvsyMzMF2TfpmTiO8/qqnlTMlzW2shyazSzkYhGgKwWaGwBNGqCIEri1pC2vTgNddYH1ej0UCuGmo3/55ZcYPXo07r//fiQmJmLkyJHYsGGDYPsnPc/Xp8rw9g8X8Vtpg1fPV8r5z/cmox449Slw4iPgzBfA4XeAspNCNpVcw6OeVU5ODgA+32TZsmVQKpWOx2w2Gw4fPowRI0YI1rhLly5h/fr1yMnJwbPPPoujR4/ir3/9q9tTTJPJBJPJ5PhZp9MJ1h4S/kwWFmYrC5GXvSuVTIzGJiNEpz8D2CpAJAYiYgBDNXD+W0ARDcT0FrbRBICHweqXX34BwPesTp06BZlM5nhMJpNh+PDhWLRokWCNY1kWo0ePxssvvwwAGDlyJE6fPo133nnHZbBavXo1VqxYIVgbSPfizco2bankEvTVH4fYdAWIjgJGPAhEJgHnvgbKTwEXvgNGPwaIKI9LaB4Fqx9++AEAMGfOHKxbtw5RUf49R09JScHgwYPb3Tdo0CB89tlnLp+zdOlSRw8Q4HtWWq3Wb20k4cWX6TYAcFdvK5i6qwAnBwbdDahbFkntewtQnc/3sKrOAknXCdVk0sKrAfa2A97+NGHCBJw/f77dffn5+ejd23U3Wy6XQy6X+7tpJEzZg5Vc6kWw4jgwF78HOA5IHgrE9219TBoBpN0AFB0ESn+hYOUHXQ5WM2fORG5uLqKiojBz5ky3227fvt3nhgHAwoULMX78eLz88st44IEHcOTIEbz33nt47733BNk/6Xnsp4FysRdpB5VngMYKQCIDsiZ3fDxlOFB8CKi/DDTVAkqa8CykLgcrjUbjuOSr0Wj81qC2brjhBuzYsQNLly7FypUrkZmZibVr1+Khhx4KyOuT7oVlOe+n29iswKX9MJisOGYZBMN5HaYNVbXfRhEFxGQCtZeAqnNA7/ECtZwAHgSrtqd+gToNBIC77roLd911V8Bej3RfVpZD7zglzFbW82BVlgc0N8Ami0SepR9U9Ubn2yX054NVdT4FK4F5NWZlNBrBcZwjdaG4uBg7duzA4MGDceuttwraQEKEIpOIMHNUmudPtFmBkp8BAOLMCWAvSmEw2ZwnmMb1A5hdgK4MMDUCclpyXiheXRK555578NFHHwEA6uvrMWbMGLz++uu45557sH79ekEbSEjQVZxyBB6FdiQAfspNs8VJLXZ5JBDZMrOjviSAjez+vK4UeuONNwIAPv30UyQnJ6O4uBgfffQR/vGPfwjaQEKCimUdvSpox0IskSJCxg/O601W58+JTue/U7ASlFfBqqmpCWo137397rvvMHPmTIhEIvzud79DcXGxoA0kRCiF1Qa8/cNFfJF3tetPqjwDGOv51ISU4QD4LHYAaDK7ClYZ/HcKVoLyesGIzz//HJcvX8auXbsc41SVlZV+TxQlxFsmqw1mKwuLrYvlXTgOKPmJv60dw6csAIhsWXa+sdlFsNKkAQzDpy8003QvoXgVrJYtW4ZFixYhIyMDY8eOxbhx4wDwvayRI0cK2kBChOJx2oI9I10iB1JHOe5Wy/lTQdZVTSuponXcqpFKewvFq6uBf/jDHzBx4kSUlZVh+PDhjvtvueUW3HvvvYI1jhAhmT2pv85xfIInAPS6ng9ALaYMTMTUwUnun69O4RNIdaVAwgBvm0za8HpF5uTkZCQnJ7e7b8yYMT43iBB/8ahnVXuJDzZiCT+Npg1RVxZHVacAyKOelYC8ClYGgwFr1qzBnj17UFlZCZZtfwmXVmwmocixsk1nk5g5Dij+kb+dOhKQKd1v74w6hf/eWMbvj5bx8plXwerxxx/H/v378cgjjyAlJYXWUyNhwdTVnlV9CdBwha9VpR3b4WGj2YZvT5fBYLbh4bHpzv//VQl8r8xq5gfaVZ7XfCfteRWsvv32W3z99deYMGGC0O0hxG80EVKkaBSIipC639B+BTBluNMMdJlEhJLaJnAc0GS2QSV38jYSiYDIZD7oNZZSsBKAV8EqJiYGsbE0o5yEl3FZcZ2vaqMrBWoLAUbktFcFAGIRA5VMAr3JisZmq/NgBfC1rhquAPoKAEN9azzxLnXhxRdfxLJly9DU1CR0ewgJLvtYVdJ1QES0y82iIuy5VhbX+1Il8N8N1QI1rmfzqmf1+uuvo6CgAElJScjIyIBU2r5bfeLECUEaR0hA6auA6gv8YHj6OLebRimkKEUzGoxdCFb6SgEb2XN5FaxmzJghcDMI8b/cQ4WwshzuG5WGGJWs4waXD/Pf4/t1OsYUreSfX9fUhWBlNgDmJu+uKhIHr4LV8uXLhW4HIX7X2GyFleWc50mZGvl5gACg/V2n+4pR8WcTdU1m1xtJZPyppLEeMFQCsgyP20xaeb0ER319Pd5//30sXboUtbW1APjTv6tXPZgkSkiA2FgOVpafHuM0g/3KMYC18fP6NL063V+MUga5VNR5NjyNWwnGq57Vr7/+iqlTp0Kj0aCoqAhPPPEEYmNjsX37dpSUlDhqXRESKkzW1lXCO6xsYzXxizwAQHrnvSoASFTL8eebszrPMVQl8ONgNG7lM696Vjk5OcjOzsaFCxfarcA8bdo0HDhwQLDGESIUk6U1IbTDaWDZST5gKeOAuL5Ont0RwzBdS4a2T2g2ULDylVfB6ujRo3jqqac63N+rVy+Ul5f73ChChGZyNYmZtQFXjvK3tWO8mhbDuaq+AAAqe7Cq5qfdEK95FazkcrnTZdnz8/ORkJDgc6MIEZr9NFAhvWYJruoLfM0pmRJIGuLRPn+9Uo/cQ4X46VKN640iovkEU5uFH8QnXvMqWN19991YuXIlLBb+si3DMCgpKcHixYtx3333CdpAQoQgYhgkaxSIj7wmZeHqcf57ygh+Lp8HWI5PXahqNLl5YTEQEcPfbnIT1EinvApWr7/+OvR6PRISEmA0GnHzzTejb9++UKvVeOmll4RuIyE+08Yq8acx6bh9SErrnfoqftIyI+KrK3jIHvjcBiugdbHTplqPX4O08upqoEajwe7du3Ho0CGcPHkSer0eo0aNwtSpU4VuHyH+U9oy0yK+H79AqYfiI+UA+PytZout4ymmnSNYUc/KFx4HK5ZlkZubi+3bt6OoqAgMwyAzMxPJycnO11EjJBRZmoHyU/ztXqPcb+uCQiqGJkKKBiN/KqiNdZGhrmzJhqdg5ROPTgM5jsPdd9+Nxx9/HFevXsXQoUNx3XXXobi4GNnZ2VTSmISs/flVeP/fl/DrlXr+jorT/KC3Kh6I7u31fuPVfO+q0t2pIAUrQXjUs8rNzcWBAwewZ88eTJ48ud1je/fuxYwZM/DRRx/h0UcfFbSRhPhK38yXc7GxHJ9CYE8CTR3lUxXPhEg5Cir17setIlpOA02NfDE+iZN5iaRTHvWstm7dimeffbZDoAKAKVOmYMmSJdi8ebNgjSNEKPbUBblEzNesMlTzV/+SPUtXuFZqtALxajmilW4K+smU/LqDAGCkQXZveRSsfv31V9x+++0uH7/jjjtw8uRJnxvlypo1a8AwDBYsWOC31yDdkyMpVCriM9YBIGEgv8yWD3rHqfDI73rjd306KepHp4I+8yhY1dbWIinJ9RJESUlJqKur87lRzhw9ehTvvvsuhg0b5pf9k+7NZGnpWTHW1uoKKcPdPENgFKx85lGwstlskEhcD3OJxWJYrS5WqfWBXq/HQw89hA0bNiAmJkbw/ZPur7mlZ6XSXeQH1iNiAI1WsP1bbaz7qqEUrHzm0QA7x3HIzs6GXO6862wydZIc56V58+bhzjvvxNSpU7Fq1Sq325pMpnbtcDYtiPQsHMc5JjIrqk7zd6YME2x5rIuVenx7qgzJGgXuH+0iAFKw8plHwWr27NmdbiP0lcBt27bhxIkTOHr0aJe2X716NVasWCFoG0h4s7EcEqPkgKEGMn0pIBZ5PA/QnWilFFaWQ4WuGayr4n72xFBjHa0j6CWPgtWHH37or3Y4dfnyZfzHf/wHdu/e3a4UjTtLly5FTk6O42edTgetVrjuPgk/ErEIfxqTDhQUACUMEJvlVca6K3EqGWQSEcxWFtV6ExKjnPyvKjQtE5qtgEnH/0w84vXy8YFw/PhxVFZWYtSo1gxjm82GAwcO4K233oLJZIJY3H6Kg1wud3maSnowlm3NWE8R9iINwzBI0ShQXNOEsoZm58HKPqG5qYafI0jBymMhHaxuueUWnDp1qt19c+bMwcCBA7F48eIOgYqEFo7joDdZoZJJnJ8aBVJtAb9wg0zZ5QJ7nkjRRLQEKyOGa6Odb6SM5YOVsRZApuBt6O5COlip1WoMGdJ+bEGlUiEuLq7D/SS0sCyHr06VoaBSj7hIGe4bleZ6MVA/K6o24NIPu6FlG9Hv+hv4Xo7AUqP53lRpfbPrjRylYigx1BteLxhBiDsnr9SjoFIPAKjRm3EgvypobTEadFDoCmG1cUCyf3KrkjUKMAzQYLTAYHKRvkOlYnwS0j0rZ/bt2xfsJpBOsCyHEyX1AIAhvTQ4fbUB5ysaMbFfPNQKN9NS/ERU+RsYcLBEpgCR/qlkK5eIMUIbjUi5BCJXV/rscwRpyo1Xwi5YkdBXrmuGzmiBXCrCpAEJMFtZaCKkrt/E/sRxkFby456meP8OHUwakOh+A3uuVXMDf1XQw8qkPR39tojgIhUSjM+Kg43jIBWLcOewlM6f5C+6UqCpGiwjgS1+YPDaAQAyFV9xwWoGmuv58jSky2jMigguSiHF2D5xGJ8VAm/G8l9htXGoVWZArojw60txHIcGowXnynWw2tiOGzBM66kgZbJ7jIIVCQiLjUVRtQHVev9MyXLKZgEqz8DKcqhUDUCEzP+pLluPlODbU+WocnWcNMjuNQpWRFC1BjMuVjZCf80VsX3nq7Djl6s4Xx7A5aiqzgNWM5iIaIhjekMl8++ohz05FHCTwkCD7F6jYEUEdaGiEf93sgz/viZVIbklq7u8wU0ektDKfwUADBo5AdkT+7iukS6g1Gj+VLOsweh8A8eEZgpWnqJgRQRVruODUbKm/ZSTJI3c8bjbFYyFYqwD6or5cSIBJy13JknNH3elrpPTQOpZeYyCFRFUtd4MAB3mx8Wr5JCKGZitLOqa3NR9Eop9HmBMBr8qcoAkRvFBucFoQXNLwb927Fns5ibA4qL3RZyiYEUEY7Gx0Bn5QBSrbL8ogkjEILGl11Gh8/OpYJtJy/rYwXj/35fwydES/75mC4VUDLWCHxtzuoiERA7II/nbdCroEQpWRDB1TXyvSiEVO73yFq/mA1hNS+/Lfw0pBJp1gFQBg7oPGltWtgkUe6/S5fJcNMjuFUoKJYKpM7T0qlTOp9TEqfhTJL+nL1xtWWk5eSiMVj5rXu5qtWQ/GKmNxqBkNVKiXeR1KeP4ZeupZ+URClZEMPaeVYzS+bp4veOU+P3gJMe4jl8Y6/lyMACQMhLNjfy4UUQAg1WnVx1pkN0rFKyIYAYlRyFGKUOkwvm/VbRShmgXgUwwZSf5ssExvQFVHIw1/GpLgQxWnYqgxFBvULAigtEopdC4W+zT31hb65qAqXx1WWPLFbkIWWCHZy/X8lVD+yVGIkZ1TYBu27OieuxdRsGKBFSN3oSr9UbEqmRIixE4SbM6v6UaqAqI7wcAjvQBRYB7VkcKa1FS2wSlTNwxWFE9dq/Q1UAiiCazFceL61BUbXC73dmyRuw5W+mfaTdXjvHfU4Y7qoEqJGLEKKWICnAdrbjIliufBidXPu312AE6FfQA9ayIIKoaTTiQX4VYlQwZ8SqX2/ktfaHhKtBwhQ8EvVoXGBnfNx7j+wa++oP9ymetwU0mO9Vj9wj1rIgg7FnpHU55ruFIXzCYhJ12c/kw/z1xMCBXC7dfLzl6Vq6CsqNnVRegFoU/ClZEEHUtpzvXZq5fK0bJVww1WVgYzE6mo3jDWMePVwGAdoww+/RRbEvQbmy2Op92o6S6Vp6iYEUEUdsSrGJcJITaScQiRLdcMawRKjn0yjH+qlpsJhDZWlrYbGWx4cAlbD5c7LwYnh+1nXbjdNyKstg9RsGKCKKzhNC27KdI1UKMW5kagdI8/rZ2bLuHmsxW6E1W1DdZIBEH/l/dfpy1zo7z2nrspFM0wE58ZrLaHHPvYjsZswL4casL0AvTsyr5GWCtgKYXX2GhDftppjIAFUKdmZAVjwl9XZwaUz12j1GwIj6rbxlcV8rEXcpnGpSiRlpMBOIjfZx206xr7VVl3NghubKppVqpvyuEuuJ0GXk7ez32xnI+fYGCVacoWBGfxalkeHBsOoxdHDAXbNqNo1eV1qFXBbTpWclDaKpNW0p7sKJB9q6gYEV8JhGLkOSuF+EPTbVA6S/87YyJTqes2HtWwToN5DgOv1yuR43ejIl94zuWzaFBdo/QADsJiktVehzIr3Jdq7wzBXsBjgXisvirgE60jlkF5zOZYRjkldTj9NUG52VxqB67RyhYEZ8dvlSDX0rq0GTu+lWt8+WNOF5chyt1XgSr2kKg+gI/vy7rFpebRUj5qTaaiOBNrnY77YZKxXiETgOJTziOw5HCWlhZDpnxKnR1KCouUg6g0fMrgjYrcGE3f7vXKEAV53LTif3iMbFfcAeu41RyXKoyOJ92Yz8NtNdjl/p3EdZwF9I9q9WrV+OGG26AWq1GYmIiZsyYgfPnzwe7WaQNXbMVVpaDWMR4NFnY61yr4kP8gLRMxY9VhTh7KofT45TIqB67B0I6WO3fvx/z5s3Dzz//jN27d8NiseDWW2+FweB+Zj8JHPs0mxilFCJR1+syxbfMEawzmMGyXZwj2FjBXwEEgP63hUVPJN6eGGowO58LSYPsXRbSp4E7d+5s93Nubi4SExNx/Phx3HTTTUFqFWmrtiVz3dNUhKgICaRiBhYbh3qjpfNkUpsFOPslP6ieMID/cqOq0YTtJ64gQS3HzFFpHrVNSDEqGRgGMJptaDLboJJf85ajeuxdFtI9q2s1NDQAAGJjY11uYzKZoNPp2n0R/3FMYO5C5npbDMO0jFu5KaPS1oXvAEM1f/rX79ZON29stqDJbHNUCg0WqVjkGOC3T0lqhwbZuyyke1ZtsSyLBQsWYMKECRgyxPUKu6tXr8aKFSsC2LKezTGB2Yskz1iVDOUNzag1dLLoaWkeUPYrn0s1+O7WcR437NN/Iq/tyQTBjBG9oJJLIJM46RvY0xcM1YFtVBgK/l+yi+bNm4fTp0/j4MGDbrdbunQpcnJyHD/rdDpotVp/N6/Hsk+18bRnBQC/6xOH8Vlx7gNKbSGQv4u/nTHRaaa6M/qWhNBAVwh1xm2NL/s0m6Za/kqnOGzekgEXFr+Z+fPn46uvvsKBAweQluZ+/EEul0Mu9+NST6SdR8b1Rl2T2XF1zxOd5j/pSoHftvPjVEmDgd4TurxvexCNCmKOVZfIowCpArA081c51UnBblHICukxK47jMH/+fOzYsQN79+5FZiaVfw01CqkYKZoISIUuwVJ/GTi5la9KEK0FBtzp0SowdY6B/+AHq2aLDXvPVeDzX652vCLIMICqpQaXoTLwjQsjId2zmjdvHrZs2YIvvvgCarUa5eXlAACNRoOIiNC/bE06d7SoFmUNzZjYN771VLLsV/7Uj7UC0enA0Ps9Oj3iOA4NxpYyy/5ep7ALJCIGp67owHIc9CYr1NeemkYm8lcE9RUAhgaljeEgpHtW69evR0NDAyZNmoSUlBTH1yeffBLsphEAv5U2YN/5Slypa/J6H4XVBhRU6vk5gqZG4LcdwLmv+UAV3w8Y9gCfPOkBi41DikaBaKUUUS4WXA2kttVRa51Nu1El8N/1VQFsVfgJ/l/SDUEXFCCCK6jiA40mQurdGoAch14qDvrSK7D8dgyQXeEXKmVEQO/xLqspdEYmEQU1t8qZWJUMtQYzqvVm9I67ZvWfyDangbToqUshHaxIaLPnWNlXrOmA4/irXLorgLGeX9DTbOC/LE2AuQmD9EaIqvRgZGIgLZqvTdXv1m430BwXKcPFShd151UJfIAyN/G/my6kZvREFKyIV2ws57ji1mGRCH0VUP4rUHkGMOnd7idSIYVZrESVTIuBQ38PeVxvn3sWFhsr/IC/j+IdCbBOTgPFUn7aTVMNP25FwcopClbEK/VNZrAcB5lE1JonVX+Zn2hcW9i6oUgCRKXy+UTyKP6NKFXymehSJWQyFQp+LEGD0YJyJgG9BTgF+iKvFLUGE24dnOx2wdVAsl88qGmZI8hce5yRCS3BqpKv0UU6oGBFvFLbZpoNo7sKFB4A6or5BxkR/4ZLGQ7EZHZ6JS9Fo0CD0YKrdcaO4zke4jgOVY0mNFtsQasQ6kyMUgaxiIFUzMBosXUsCKhOASrPAY1lwWlgGKBgRbxSYzBDbmnAoNqDwIkK/k6RGEgeCqT/rnXF4S5Ij1OioEoPS1erL7jRaOIXFRUxjFdZ9f4iFjF48qY+rhfUUKfw3ylYuUTBinjOZoG0aD9GlP+MhBgFoFACycOAjAmAQuPx7vonqTEgSS3I2n6VumYAQGykLChrBbrjduUfdTI/Vtes41M45OrANSxMULAinqkrBs5/i1FMLazpGnCxWUD/W/gxFy8JORh+uaVMclp0mCUNS+T8pGZDNaArAxIoWF2LghXpGo7jC98V7gc4Dow8CtL+twPxfQV9mTqD2f3E305cbQlWvWJCL1jVN5mx73wVzFYWD9zgZHJ9VCofrBpLgYT+gW9giKNgRTrH2oBzXwEVZ/ifk4cC/X7P9wYEYmM5/O+xyyhvaMbs8RlejTc1ma2OVWR6hWDPSiYRobCar3Jrstogl1xzWqhO4aca6WjcypnQOqknocdm5afAVJzhB9AH3I6LsTfh81PVOH21QbCXEYsYx9W7ny95t+gnxwE3ZMSif5K6Y0XOEKCUSaBumf5TqXOSHBrVi//eWAqwbABbFh4oWBHXWBY4+wW/7JVIAgy5D0gdiasNJhRWG1Dl6co0nRiXFQeG4ZfpOlPaWuG17bQrluVc1mxXySWY0Dcedw5LEbRdQrIvBlvZ2NzxQVUCPw/Sam6Z1EzaomBFXCvYC1Tl8z2qoX9wJCtWNfJBKiFS2LphiWoFru/Npzzs+q0cH/1UhA0HLqGsofWNXVzbhA3/voTjxbWw2MKv9+EIVs56ViIRoEnnb9eXBLBV4YGCFXGu9BfgylH+9sC7HKse25MuASBRLXyRwwlZ8RjVOwYMA9TozdCbrI7XA4CiGgOazDYcyK/Gph+L8OuVelTomrH9xBUU14T+qkdJUfzvrELnpGcFADG9+e8UrDoIvRN7EnyNFcCF7/nbfW7mq3S2aDBa0GyxQSzyT9KlSMTg5v4JGJUejRq9GXKpqN3r3NQvAQmRcvx8qQaNzVbsOdtasK7WYMbs8RkhNy+wrUQ137Oqa7LAaLYh4tos++iWnlVDCX8aLgrdYwk0ClakPasJOPM5X08qri+QPq7dw6X1fI8gUS33a9KlWiHtWKQO/ED8kF4aDExW49TVBvx6pQF6kxWp0QpMHpAY0oEKACJkYiRFKSCTiGC0OAlWqkT+KqvVxGeza3oFp6EhiIIVacVxQP5OvqyLXA0M7FhKuFzH5zGlBDk1QCIWYWR6DEamd31aT6j40xhtx4nMdiIRf8pdeQ6ouUjBqo3Q/hgigVWWx6coMCLguhmAzHlBPZlEhBSNIqBN605cBiq7uH7895oL/m9MGKGeFeHpK1vHqTJv4ovgOTFlYBIm9U8E1XD1ndFsg1TMdDydju3D92j1VXzRwojoYDQv5FDPivDjI7/taBmnyuKrJrghEjEQi6j0ri++yLuKd/YXoKjGSf16mbL1w6Kaeld2FKx6ug7jVHe5rNRpsgZ3KfbuxL6eoX36TQcJA/nvFacD1KLQR8Gqpys72TpONfgel+NUALD1cAlyDxW2y3si3unTUsG0sFrvfGGUxEH836SxnJaWb0HBqidruApc2M3fzryJX0zU1aZNFtQ1WdBgtDrmtxHv9YqOgEwigsFkQ4WzbHaZqrW8cfmvgW1ciKJg1VMZ64HTn7auz9fJOFV+ZSMAIC0mwn0ROdIlErEIveP4Xuz5ikbnGyUP47+XneTnC/ZwFKx6IrMBOPUvfumnyERg0N2drihzvpx/Qw1IpqJwQhmUEgUAOFemg83Z5Oy4voAyFrA082klPRwFq57GbADytvDjIPJIfoJyJyseV+iaUdVogljEICuBlokSSmacCiq5GE1mm/N5jSIRoB3L3758GLBZAtvAEEPBqicx1AAn/qc1UI14qEs1038pqQcA9E+K7Dg9hHhNJGIwNjMOdwxNRoarVX2ShgCKKH79xZKfAtvAEEPBqifgOKDiN+DEJsBYxycZDn+QP8XohN5kRX7LmMoIbfhNbQl1w7XRGJgcBZGrvDWxBOg7lb9d8jN/dbCHomDV3RmqgdOfAWe+5JM/NWnAqEcBVVyXnh4pl2DmqF4YnRGDZJpi41dGsw1mq5MaXfH9+YsgrA347XP+VL4HCotg9fbbbyMjIwMKhQJjx47FkSNHgt2k0MZx/OrIZ74Ejr7PZ0EzIiDzRmDEg/xlcQ+kxShxYz/vV68hnbtUpcdHPxXhuzPlHSuhMgw/qVyh4XvGJ7cCzcKVlA4XIR+sPvnkE+Tk5GD58uU4ceIEhg8fjttuuw2VlZWdP7knsZr5Zdsvfg8cfhf45WP+1I/j+E/lGx4DMibyVT87wXEcjhfX4Wq9MQANJwA/OdxkZXGhQo9vTpeh2XLNbAFpBDD8j/wHjb4KOPYhn9LQg2q1M5zT9NnQMXbsWNxwww146623AAAsy0Kr1eIvf/kLlixZ0unzdTodNBoNGhoaEBUV5e/m+gfH8flQ1mY+KFkM/CdrcwP/SdtYDjTV8NvZiSVA4mAgdRQQ1XlNcrOV5Zdwrzfi9NUGVDWaIJOI8NDYdEQrQ2dl4+4sv6IRO0+Xw8ZyUEjFGNIrChlxKsRHylsvbBjr+HmcjS012uWR/NQcTRqgjOenTEnknaaihApP3p8hnYpsNptx/PhxLF261HGfSCTC1KlT8dNPzq+MmEwmmEytGcE6nc7pdh0U/8j3RJxxBAHOyX2u7hdwW5uFH6/ojFwNxGYizxCH082xsNXKwNU0g0MhOI7fG8dx+OOYdES2rP7y48Vq/HK5vsNYiUwiwsS+8dBEdCyAR/yjf5IaaoUEu89UoEZvxrGiOhwrqsPAZDXuGMp/4BhEavzLeCMSmk4jqT4PUrYeKLji2EecSoa0ODUgksAKEX4ta8Ko3m3HJ1uCmLNgJnSAk6mB4bME211IB6vq6mrYbDYkJSW1uz8pKQnnzp1z+pzVq1djxYoVnr+YSR/6c7AYBhDLAKmSH7+wf0Um8cuPy/kcqJqzFaiqbQDgPOuZbRMQLSznCFQyiQjJUQpkxCtxXaqGMtWDIEUTgYfH9kZBlR4XK/Uoa2hGXJuFOWwch7pmDnWy63AhYSCijZcR3XwFkeZKyK2NUFtZ/oONtQEsC6uhHmgKUi9LIWzWfUgHK28sXboUOTk5jp91Oh20Wtdz3hx6Xc+P7bT7dGlz25P7hd6HWApIFHyg6sKn38j0GPRP4jPNRSIGTMvuGDBgGEDZJgiN7h2DYb00UMrFHRfdJEEhEjHol6RGv5a/YduRGqVUjFk3aB29ZI7r3a4zrpKwgNwG2KxgbBak9W0ENAo4eumd9uaFPBBhw0tIB6v4+HiIxWJUVLRfQ62iogLJyclOnyOXyyGXe7Hqiiquy5fzQ12sStblxRxUcglUwi9SQwTUtrKoRCxCahdLSosBpEY5f5+Eo5C+GiiTyXD99ddjz549jvtYlsWePXswbtw4N88khHQ3Id2zAoCcnBzMnj0bo0ePxpgxY7B27VoYDAbMmTMn2E0jhARQyAerWbNmoaqqCsuWLUN5eTlGjBiBnTt3dhh0J4R0byGfZ+WrbpFnRUg35cn7M6THrAghxI6CFSEkLFCwIoSEhZAfYPeVfUiuy9NuCCEBY39fdmXovNsHq8ZGvnBcl7LYCSFB0djYCI3GfdXabn81kGVZlJaWQq1Wt8sEFpJ9Ss/ly5d7xBXHnna8QM875kAdL8dxaGxsRGpqKkQi96NS3b5nJRKJkJaWFpDXioqK6hH/yHY97XiBnnfMgTjeznpUdjTATggJCxSsCCFhgYKVAORyOZYvX+5dtYcw1NOOF+h5xxyKx9vtB9gJId0D9awIIWGBghUhJCxQsCKEhAUKVoSQsEDBqgs8WRF6w4YNuPHGGxETE4OYmBhMnTo1LFeQ9nYV7G3btoFhGMyYMcO/DfQDT4+5vr4e8+bNQ0pKCuRyOfr3749vvvkmQK31nafHu3btWgwYMAARERHQarVYuHAhmpubA9RaABxxa9u2bZxMJuM2btzI/fbbb9wTTzzBRUdHcxUVFU63f/DBB7m3336b++WXX7izZ89y2dnZnEaj4a5cuRLglnvP02O2Kyws5Hr16sXdeOON3D333BOYxgrE02M2mUzc6NGjuWnTpnEHDx7kCgsLuX379nF5eXkBbrl3PD3ezZs3c3K5nNu8eTNXWFjI7dq1i0tJSeEWLlwYsDZTsOrEmDFjuHnz5jl+ttlsXGpqKrd69eouPd9qtXJqtZrbtGmTv5ooOG+O2Wq1cuPHj+fef/99bvbs2WEXrDw95vXr13N9+vThzGZzoJooKE+Pd968edyUKVPa3ZeTk8NNmDDBr+1si04D3bCvCD116lTHfZ2tCH2tpqYmWCwWxMbG+quZgvL2mFeuXInExEQ89thjgWimoLw55i+//BLjxo3DvHnzkJSUhCFDhuDll1+GzdaFlbODzJvjHT9+PI4fP+44Vbx06RK++eYbTJs2LSBtBnrARGZfeLMi9LUWL16M1NTUdv8YocybYz548CA++OAD5OXlBaCFwvPmmC9duoS9e/fioYcewjfffIOLFy/imWeegcViwfLlywPRbK95c7wPPvggqqurMXHiRHAcB6vViqeffhrPPvtsIJoMgAbY/WrNmjXYtm0bduzYAYVCEezm+EVjYyMeeeQRbNiwAfHx8cFuTsCwLIvExES89957uP766zFr1iw899xzeOedd4LdNL/Yt28fXn75Zfzzn//EiRMnsH37dnz99dd48cUXA9YG6lm54c2K0HavvfYa1qxZg++//x7Dhg3zZzMF5ekxFxQUoKioCNOnT3fcx7IsAEAikeD8+fPIysryb6N95M3fOSUlBVKpFGKx2HHfoEGDUF5eDrPZDJmsaytiB4M3x/v888/jkUceweOPPw4AGDp0KAwGA5588kk899xzndaiEgL1rNzwdkXoV199FS+++CJ27tyJ0aNHB6KpgvH0mAcOHIhTp04hLy/P8XX33Xdj8uTJyMvLC4sKrd78nSdMmICLFy86AjMA5OfnIyUlJaQDFeDd8TY1NXUISPZAzQVqenHAhvLD1LZt2zi5XM7l5uZyZ86c4Z588kkuOjqaKy8v5ziO4x555BFuyZIlju3XrFnDyWQy7tNPP+XKysocX42NjcE6BI95eszXCsergZ4ec0lJCadWq7n58+dz58+f57766isuMTGRW7VqVbAOwSOeHu/y5cs5tVrNbd26lbt06RL33XffcVlZWdwDDzwQsDZTsOqCN998k0tPT+dkMhk3ZswY7ueff3Y8dvPNN3OzZ892/Ny7d28OQIev5cuXB77hPvDkmK8VjsGK4zw/5h9//JEbO3YsJ5fLuT59+nAvvfQSZ7VaA9xq73lyvBaLhXvhhRe4rKwsTqFQcFqtlnvmmWe4urq6gLWXSsQQQsICjVkRQsICBStCSFigYEUICQsUrAghYYGCFSEkLFCwIoSEBQpWhJCwQMGKdCu5ubmIjo52/PzCCy9gxIgRjp+zs7PDsoopoWBFvJSdnQ2GYfD00093eGzevHlgGAbZ2dntthc6SGRkZGDt2rXt7ps1axby8/NdPmfdunXIzc11/Dxp0iQsWLBA0HYR/6BgRbym1Wqxbds2GI1Gx33Nzc3YsmUL0tPTg9KmiIgIJCYmunxco9G063mR8EHBinht1KhR0Gq12L59u+O+7du3Iz09HSNHjvRp3856PDNmzHD01iZNmoTi4mIsXLgQDMOAYRgAHU8Dr9W2h5ednY39+/dj3bp1jn0UFhaib9++eO2119o9Ly8vDwzD4OLFiz4dF/EeBSvik7lz5+LDDz90/Lxx40bMmTPH76+7fft2pKWlYeXKlSgrK0NZWZnH+1i3bh3GjRuHJ554wrGP9PT0DscEAB9++CFuuukm9O3bV6hDIB6iYEV88vDDD+PgwYMoLi5GcXExDh06hIcfftjvrxsbGwuxWAy1Wo3k5OROiyE6o9FoIJPJoFQqHfsQi8XIzs7G+fPnHfXGLRYLtmzZgrlz5wp9GMQDVCmU+CQhIQF33nkncnNzwXEc7rzzzrAvb5yamoo777wTGzduxJgxY/B///d/MJlMuP/++4PdtB6NelbEZ3PnzkVubi42bdokWO9DJBJ1qEBpsVgE2XdXPP74446LBx9++CFmzZoFpVIZsNcnHVGwIj67/fbbYTabYbFYcNtttwmyz4SEhHbjUDabDadPn263jUwm83npK1f7mDZtGlQqFdavX4+dO3fSKWAIoNNA4jOxWIyzZ886brvS0NDQYbmuuLg4p3Xap0yZgpycHHz99dfIysrCG2+8gfr6+nbbZGRk4MCBA/jjH/8IuVzu1elnRkYGDh8+jKKiIkRGRiI2NhYikcgxdrV06VL069fPbc19EhjUsyKCiIqKQlRUlNtt9u3bh5EjR7b7WrFihdNt586di9mzZ+PRRx/FzTffjD59+mDy5Mnttlm5ciWKioqQlZWFhIQEr9q9aNEiiMViDB48GAkJCSgpKXE89thjj8FsNgfk6ibpHJU1JsSFf//737jllltw+fLlDguCksCjYEXINUwmE6qqqjB79mwkJydj8+bNwW4SAZ0GEtLB1q1b0bt3b9TX1+PVV18NdnNIC+pZEULCAvWsCCFhgYIVISQsULAihIQFClaEkLBAwYoQEhYoWBFCwgIFK0JIWKBgRQgJCxSsCCFh4f8DNXUETTRaPqIAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:22:46.037341Z", + "iopub.status.busy": "2024-02-29T21:22:46.036918Z", + "iopub.status.idle": "2024-02-29T21:22:46.214864Z", + "shell.execute_reply": "2024-02-29T21:22:46.213747Z" + }, + "papermill": { + "duration": 0.20533, + "end_time": "2024-02-29T21:22:46.217203", + "exception": false, + "start_time": "2024-02-29T21:22:46.011873", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA0pElEQVR4nO3de1hTV7o/8C8JEEAuotwRDUgVUAHFEfAy3rhpdXToTFFbRbT2jMoca0q1OC0cqqe09VKmPVROPUXUTqutVeuoRRGl4wW8QLWogIIgXgABhQhoCMn6/eGPPaYJEBAM7Lyf58mDe+21V9bKTl7Xvq1lwBhjIIQQHhLougKEENJTKMARQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4iwIcIYS3KMARQnjLUNcV6I2USiXu3bsHCwsLGBgY6Lo6hJDfYIzh0aNHcHJygkDQdj+NApwG9+7dg4uLi66rQQjpwO3btzFo0KA211OA08DCwgLA0w/P0tJSx7XpOXK5HMeOHUNISAiMjIx0XR3ynPRpf0qlUri4uHC/1bZQgNOg9bDU0tKS9wHOzMwMlpaWvP9B6AN93J8dnUKiiwyEEN6iAEcI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtuk2EkD6qqakJhYWF3HLDYxnO5pfA2uYizE1FKnk9PDxgZmb2oquocxTgCOmjCgsL4efnp5b+iYa8ubm5GDNmTM9XqpehAEdIH+Xh4YHc3FxuuaiiDpLv87Hlz6Mw3LG/Wl59RAGOkD7KzMxMpVcmuFUL0anH8BzpA98hA3VYs96DLjIQQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLcowBFCeEvnAS45ORlisRgmJibw9/fH+fPn281fV1eHlStXwtHRESKRCMOGDcORI0e49f/1X/8FAwMDlZe+PqZCiL7T6aNae/bsgUQiQUpKCvz9/ZGUlITQ0FAUFRXBzs5OLX9zczOCg4NhZ2eHvXv3wtnZGbdu3UL//v1V8o0YMQLHjx/nlg0N6Yk0QvSRTn/5W7ZswbJlyxAVFQUASElJweHDh5Gamop3331XLX9qaioePHiAs2fPcrMGicVitXyGhoZwcHDo0boTQno/nQW45uZm5ObmIjY2lksTCAQICgpCdna2xm0OHjyIwMBArFy5Ej/++CNsbW2xYMECrF27FkKhkMt348YNODk5wcTEBIGBgUhMTMTgwYPbrItMJoNMJuOWpVIpgKfTsMnl8udtaq/V2jY+t1GftLS0cH/5vk+1bZ/OAlxNTQ0UCgXs7e1V0u3t7VUG8XvWzZs3ceLECbz22ms4cuQIiouLsWLFCsjlcsTHxwMA/P39kZaWhuHDh6OiogIJCQmYNGkSrly50uYksYmJiUhISFBLP3bsmF4MEpiRkaHrKpBucLsBAAyRk5ODu1d0XZue1dTUpFW+PnVySqlUws7ODl9++SWEQiH8/Pxw9+5dbNy4kQtwM2bM4PJ7e3vD398fQ4YMwXfffYelS5dqLDc2NhYSiYRbbp01OyQkhPcTP2dkZCA4OFhvJgrms8vlD4D8iwgICIDP4AG6rk6Paj3K6ojOApyNjQ2EQiGqqqpU0quqqto8f+bo6AgjIyOVw1FPT09UVlaiubkZxsbGatv0798fw4YNQ3FxcZt1EYlEEIlEaulGRkZ68cPXl3byXevFNENDQ97vT23bp7MAZ2xsDD8/P2RmZmLu3LkAnvbQMjMzER0drXGbCRMm4JtvvoFSqYRA8PQOl+vXr8PR0VFjcAOAhoYGlJSUYOHChT3Sjr6ExvAneofp0O7du5lIJGJpaWns2rVr7M0332T9+/dnlZWVjDHGFi5cyN59910uf3l5ObOwsGDR0dGsqKiIHTp0iNnZ2bENGzZwed5++22WlZXFSktL2ZkzZ1hQUBCzsbFh9+/f17pe9fX1DACrr6/vvsb2Arm5uQyAVq/c3FxdV5d00i9lNWzI2kPsl7IaXVelx2n7G9XpObiIiAhUV1cjLi4OlZWV8PX1RXp6Onfhoby8nOupAYCLiwuOHj2K1atXw9vbG87Ozli1ahXWrl3L5blz5w7mz5+P2tpa2NraYuLEicjJyYGtre0Lb19vQ2P4E31jwBhjuq5EbyOVSmFlZYX6+npeX2S4dKsWc7fm4MDyABrDnwf0aX9q+xvV+aNahBDSUyjAEUJ4iwIcIYS3KMARQniLAhwhhLcowBFCeKtPPYtKiL4rrWlEo6xF47qS6kbub3tjIPYTGcLVpl+P1K+3oQBHSB9RWtOIqZuyOsz39t78DvOcjJmiF0GOAhwhfURrzy0pwhfudubq6x/LcCgrG7OmBKKfqfrgEQBQfL8Bb+251GYvkG8owBHSx7jbmWOks5VaulwuR6UtMGaINe9HE9EWXWQghPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwFgU4QghvUYAjhPAWBThCCG9RgCOE8JbOA1xycjLEYjFMTEzg7++P8+fPt5u/rq4OK1euhKOjI0QiEYYNG4YjR448V5mEEH7SaYDbs2cPJBIJ4uPjkZeXBx8fH4SGhuL+/fsa8zc3NyM4OBhlZWXYu3cvioqKsG3bNjg7O3e5TEIIf+k0wG3ZsgXLli1DVFQUvLy8kJKSAjMzM6SmpmrMn5qaigcPHuDAgQOYMGECxGIxJk+eDB8fny6XSQjhL509bN/c3Izc3FzExsZyaQKBAEFBQcjOzta4zcGDBxEYGIiVK1fixx9/hK2tLRYsWIC1a9dCKBR2qUwAkMlkkMlk3LJUKgXw9OFluVz+vE3ttVpaWri/fG4nX3S0v1rT2tuXfNnn2tZdZwGupqYGCoWCm+S5lb29PQoLCzVuc/PmTZw4cQKvvfYajhw5guLiYqxYsQJyuRzx8fFdKhMAEhMTkZCQoJZ+7NgxmJmZdaF1fcPtBgAwRE5ODu5e0XVtSEda99fp06dxS320JE5GRsZzl9HbNTU1aZWvTw2XpFQqYWdnhy+//BJCoRB+fn64e/cuNm7ciPj4+C6XGxsbC4lEwi1LpVK4uLggJCSE1xM/Xy5/AORfREBAAHwGD9B1dUgHrt6TYlN+DiZOnIgRTurfS7lcjoyMDAQHB7c5XFJHZfQVrUdZHdFZgLOxsYFQKERVVZVKelVVFRwcHDRu4+joCCMjIwiFQi7N09MTlZWVaG5u7lKZACASiSASqQ8QaGRkxOtxtVqHtTY0NOR1O/lC2/3V3veWL/tc27rr7CKDsbEx/Pz8kJmZyaUplUpkZmYiMDBQ4zYTJkxAcXExlEoll3b9+nU4OjrC2Ni4S2USQvhLp1dRJRIJtm3bhh07dqCgoADLly9HY2MjoqKiAACLFi1SuWCwfPlyPHjwAKtWrcL169dx+PBhfPjhh1i5cqXWZRJC9IdOz8FFRESguroacXFxqKyshK+vL9LT07mLBOXl5RAI/h2DXVxccPToUaxevRre3t5wdnbGqlWrsHbtWq3LJIToD51fZIiOjkZ0dLTGdVlZWWppgYGByMnJ6XKZhBD9ofNHtQghpKdQgCOE8JbOD1EJIdqRKZ5AYHIXpdIiCEzU79JtaWnBvZZ7KHhQ0ObM9qXSBghM7kKmeAJAfepBvqEAR0gfca/xFvq5fo51HQyO80X6F+2u7+cK3Gv0hR/4f+GNAhwhfYRTvyFoLP0r/h7hi6EaZrZvaWnBmdNnMGHihDZ7cCX3G7BqzyU4TR3S09XtFSjA8dTjlsc4c+saHjcr2sxTVvP0cOVU+WXcbmr7wURTYyEmDPGCqaFpT1SVaEkkNIHyiTNcLYfDa6Dmme1LDUvhOcCzzTv9lU/qoXxSDZHQpKer2ytQgOOpM7euYfXpxR3m6+cKfHkTwM32832KNAQN9euWuhHyolCA4ylro0FoLP0rYoKHwWWA5hFRHsuacepiPiaNHQVTkbHGPLcfNGFTxnVYTx3Uk9UlpEdQgOOp1sOZ34tHY6Sz5qtlcrkcolu1mDnCv81Dmit36/HJk0a9OaQh/EL3wRFCeIsCHCGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4iwIcIYS3ekWAS05OhlgshomJCfz9/XH+fNsDXqWlpcHAwEDlZWKi+hjR4sWL1fKEhYX1dDMIIb2Mzp9F3bNnDyQSCVJSUuDv74+kpCSEhoaiqKgIdnZ2GrextLREUVERt2xgYKCWJywsDNu3b+eWNU3sTAjht0734G7e7GBcnU7asmULli1bhqioKHh5eSElJQVmZmZITU1tcxsDAwM4ODhwL01TAopEIpU81tbW3VpvQkjv1+kenLu7OyZPnoylS5fiT3/6k9rhYWc0NzcjNzdXZXJngUCAoKAgZGdnt7ldQ0MDhgwZAqVSiTFjxuDDDz/EiBEjVPJkZWXBzs4O1tbWmDZtGjZs2ICBAwdqLE8mk0Emk3HLUqkUwNPRNuRyeZfbp0stLS3c37ba0JreXhu1KYe8GB3tC33an9rWvdMBLi8vD9u3b4dEIkF0dDQiIiKwdOlSjBs3rtOVrKmpgUKhUOuB2dvbo7CwUOM2w4cPR2pqKry9vVFfX49NmzZh/PjxuHr1KgYNejpmWVhYGMLDw+Hq6oqSkhKsW7cOM2bMQHZ2NoRCoVqZiYmJSEhIUEs/duwYzMw0j6XW291uAABDnD59GrfaHqwXAJCRkdEt5ZCepe2+0If92dTUpFU+A8YY68obtLS04ODBg0hLS0N6ejqGDRuGJUuWYOHChbC1tdWqjHv37sHZ2Rlnz55FYGAgl75mzRr8/PPPOHfuXIdlyOVyeHp6Yv78+Vi/fr3GPDdv3sTQoUNx/PhxTJ8+XW29ph6ci4sLampqYGlpqVVbepur96SYuzUHB5YHYIST5jbI5XJkZGQgODi4zfHgtCmHvBgd7Qt92p9SqRQ2Njaor69v9zfa5YsMhoaGCA8Px8svv4wvvvgCsbGxiImJwbp16/Dqq6/i448/hqOjY7tl2NjYQCgUoqqqSiW9qqoKDg4OWtXDyMgIo0ePRnFxcZt53NzcYGNjg+LiYo0BTiQSabwIYWRk1OYXpbdrnXTE0NCwwza0187OlEN6lrb7Qh/2p7Z17/JtIhcvXsSKFSvg6OiILVu2ICYmBiUlJcjIyMC9e/cwZ86cDsswNjaGn58fMjMzuTSlUonMzEyVHl17FAoF8vPz2w2md+7cQW1tbYcBlxDCL53uwW3ZsgXbt29HUVERZs6ciZ07d2LmzJkQCJ7GSldXV6SlpUEsFmtVnkQiQWRkJMaOHYtx48YhKSkJjY2NiIqKAgAsWrQIzs7OSExMBAB88MEHCAgIgLu7O+rq6rBx40bcunULb7zxBoCnFyASEhLwyiuvwMHBASUlJVizZg3c3d0RGhra2eYSQvqwTge4rVu3YsmSJVi8eHGbPSI7Ozt89dVXWpUXERGB6upqxMXFobKyEr6+vkhPT+cuPJSXl3PBEwAePnyIZcuWobKyEtbW1vDz88PZs2fh5eUFABAKhfj111+xY8cO1NXVwcnJCSEhIVi/fj3dC0eInul0gMvIyMDgwYNVgg4AMMZw+/ZtDB48GMbGxoiMjNS6zOjoaERHR2tcl5WVpbL86aef4tNPP22zLFNTUxw9elTr9yaE8Fenz8ENHToUNTU1aukPHjyAq6trt1SKEEK6Q6cDXFt3lTQ0NDzXTb+EENLdtD5ElUgkAJ4+JhUXF6dyA6xCocC5c+fg6+vb7RUkhJCu0jrA/fLLLwCe9uDy8/NhbPzvmdCNjY3h4+ODmJiY7q8hIYR0kdYB7uTJkwCAqKgo/P3vf++zd/gTQvRHp6+iPjsEESGE9GZaBbjw8HCkpaXB0tIS4eHh7ebdt29ft1SMEEKel1YBzsrKihtU0srKqkcrRLrHY7kCAHDlbn2beRofy3CxGnC49RD9TDXfBF18v6FH6kfIi6BVgHv2sJQOUfuGkv8fmN7dl99BTkPsKr7QYXn9RDof/JmQTqNvLU+FjHg6GstQO3OYGqmPgQcARRX1eHtvPjb/aRSGO7bdM+8nMoSrTb8eqSchPUmrADd69GiN8x5okpeX91wVIt1jQD9jzBs3uN08raO7DrXth5HOdOqB8I9WAW7u3Lk9XA1CSEc6Oq9K51TVaRXg4uPje7oehJAOaHdelc6pPks/WkkID3R0XpXOqarTKsANGDAA169fh42NDaytrds9H/fgwYNuqxwh5N86Oq9K51TVaRXgPv30U1hYWHD/1vaCAyGE6JJWAe7ZwSsXL17cU3UhhJBu1enx4IRCIe7fv6+WXltbq3HOUUII0ZVuG/BSJpOpDKFECCG6pvVV1M8++wzA0wEv/+///g/m5v+eFluhUOBf//oXPDw8ulSJ5ORkbNy4EZWVlfDx8cHnn3+OcePGacyblpbGzbjVSiQS4cmTJ9wyYwzx8fHYtm0b6urqMGHCBGzduhUvvfRSl+pHCOmbtA5wrRO9MMaQkpKicjhqbGwMsViMlJSUTldgz549kEgkSElJgb+/P5KSkhAaGoqioiLY2dlp3MbS0hJFRUXc8m8venzyySf47LPPsGPHDri6uuL9999HaGgorl27RsOqE6JPWCdNmTKFPXjwoLObtWncuHFs5cqV3LJCoWBOTk4sMTFRY/7t27czKyurNstTKpXMwcGBbdy4kUurq6tjIpGIffvtt1rVqb6+ngFg9fX12jWij/qlrIYNWXuI/VJWo+uqkG6gT/tT299op8/BnTx5EtbW1t0SXJubm5Gbm4ugoCAuTSAQICgoCNnZ2W1u19DQgCFDhsDFxQVz5szB1atXuXWlpaWorKxUKdPKygr+/v7tlkkI4Z9OP8mwZMmSdtenpqZqXVZNTQ0UCgU3yXMre3t7FBYWatxm+PDhSE1Nhbe3N+rr67Fp0yaMHz8eV69exaBBg1BZWcmV8dsyW9f9lkwmg0wm45alUikAQC6XQy6Xa92evqb1xtCWlhZet1Nf6NP+1LZ9nQ5wDx8+VHujK1euoK6uDtOmTetscZ0WGBiIwMBAbnn8+PHw9PTE//7v/2L9+vVdKjMxMREJCQlq6ceOHVOZPYxvbjcAgCFycnJw94qua0Oelz7tz6amJq3ydTrA7d+/Xy1NqVRi+fLlGDp0aKfKsrGxgVAoRFVVlUp6VVUVHBwctCrDyMgIo0ePRnFxMQBw21VVVcHR0VGlzLamNYyNjeWmRQSe9uBcXFwQEhLC68l1Lpc/APIvIiAgAD6DB+i6OuQ56dP+bD3K6ki3PGwvEAggkUgwZcoUrFmzRuvtjI2N4efnh8zMTG5IJqVSiczMTERHR2tVhkKhQH5+PmbOnAkAcHV1hYODAzIzM7mAJpVKce7cOSxfvlxjGSKRCCKR+vAyRkZGMDIy0ro9fY2hoSH3l8/t1Bf6tD+1bV+3jSZSUlLCnQPoDIlEgsjISIwdOxbjxo1DUlISGhsbuXvdFi1aBGdnZyQmJgIAPvjgAwQEBMDd3R11dXXYuHEjbt26hTfeeAPA01tG3nrrLWzYsAEvvfQSd5uIk5OT3o9r19TUpHJus6iiDrLKYhRcMYWytr9KXg8PD14fnhP90OkA9+yhHPD0vriKigocPnxY5ZlVbUVERKC6uhpxcXGorKyEr68v0tPTuYsE5eXlEAj+fbH34cOHWLZsGSorK2FtbQ0/Pz+cPXsWXl5eXJ41a9agsbERb775Jurq6jBx4kSkp6fr/T1whYWF8PPzU0tfsEM9b25uLsaMGfMCakVIzzFgrI1nr9owdepUlWWBQABbW1tMmzYNS5Ys4brJfZlUKoWVlRXq6+t5dQ7utz24hscyHD6ZjZenBsL8NyPAUg+u77l0qxZzt+bgwPIA+A4ZqOvq9Chtf6OdjkatM9yTvsfMzEylVyaXy/Gw5j4Cx43l/Tkbop86faMvIYT0FRTgCCG8RQGOEMJbFOAIIbzVbQHuzp07ePPNN7urOEIIeW7dFuBqa2vx1VdfdVdxhBDy3OgQlRDCWxTgCCG8RQGOEMJbWj/JEB4e3u76urq6560LIYR0K60DnJWVVYfrFy1a9NwVIoSQ7qJ1gNu+fXtP1oMQQrodnYMjhPCW1j24jiabadWZSWcIIaQnaR3g0tLSMGTIEIwePRqdHEKOEEJ0QusAt3z5cnz77bcoLS1FVFQUXn/9dQwYwO+JLQghfZvW5+CSk5NRUVGBNWvW4J///CdcXFzw6quv4ujRo9SjI4T0Sp26yCASiTB//nxkZGTg2rVrGDFiBFasWAGxWIyGhoaeqiMhhHRJl6+iCgQCGBgYgDEGhULxXJVITk6GWCyGiYkJ/P39cf78ea222717NwwMDNRmy1q8eDEMDAxUXmFhYc9VR0JI39OpACeTyfDtt98iODgYw4YNQ35+Pv7nf/4H5eXlMDc371IF9uzZA4lEgvj4eOTl5cHHxwehoaG4f/9+u9uVlZUhJiYGkyZN0rg+LCwMFRUV3Ovbb7/tUv0IIX2X1gFuxYoVcHR0xEcffYRZs2bh9u3b+P777zFz5kyVaf06a8uWLVi2bBmioqLg5eWFlJQUmJmZtXu7iUKhwGuvvYaEhAS4ublpzCMSieDg4MC9rK2tu1xHQkjfpPVV1JSUFAwePBhubm74+eef8fPPP2vMt2/fPq3fvLm5Gbm5uYiNjeXSBAIBgoKCkJ2d3eZ2H3zwAezs7LB06VKcOnVKY56srCzY2dnB2toa06ZNw4YNGzBwIL+nUuuMhoYGLFiwAL/++iu++uorfPPNN13uhRPSW2kd4BYtWgQDA4NuffOamhooFApukudW9vb2KvN3Puv06dP46quvcOnSpTbLDQsLQ3h4OFxdXVFSUoJ169ZhxowZyM7OhlAoVMsvk8kgk8m4ZalUCuDptHpyubwLLevdxo8fj4sXL3LLt27dgoWFBcaOHYuzZ8/qsGbkebS0tHB/+fi9fZa27evUjb669ujRIyxcuBDbtm2DjY1Nm/nmzZvH/XvUqFHw9vbG0KFDkZWVhenTp6vlT0xMREJCglr6sWPHeDf5cUxMDIqLizWuu3jxIry8vLBp06YXXCvSHW43AIAhcnJycPeKrmvTs5qamrTKp9Np6G1sbCAUClFVVaWSXlVVBQcHB7X8JSUlKCsrw+zZs7k0pVIJADA0NERRURGGDh2qtp2bmxtsbGxQXFysMcDFxsZCIpFwy1KpFC4uLggJCeHVzPYNDQ1tBrdWxcXF+P3vf0+Hq33Q5fIHQP5FBAQEwGcwv2/Cbz3K6ohOA5yxsTH8/PyQmZnJ3eqhVCqRmZmJ6OhotfweHh7Iz89XSXvvvffw6NEj/P3vf4eLi4vG97lz5w5qa2vh6Oiocb1IJIJIJFJLNzIy4tWM76+88orW+U6cONHDtSHdzdDQkPvLp++tJtq2T6cBDgAkEgkiIyMxduxYjBs3DklJSWhsbERUVBSAp+f+nJ2dkZiYCBMTE4wcOVJl+/79+wMAl97Q0ICEhAS88sorcHBwQElJCdasWQN3d3eEhoa+0Lb1NidPnuzWfIT0djoPcBEREaiurkZcXBwqKyvh6+uL9PR07sJDeXl5p25DEQqF+PXXX7Fjxw7U1dXByckJISEhWL9+vcZemj5zcHDAq6++iu+++w6VlZW6rg4h3c6A0YOkaqRSKaysrFBfX8+rc3DPXgW/e/cubG1tceTIEcycORPV1dVwdnbm1tPXou+5dKsWc7fm4MDyAPgO4fctUdr+RmnASz3l7OyMxYsXo6SkBIsXL1YJboTwhc4PUYnufPPNN/jmm290XQ1Cegz14PSIhYVFt+YjpLejAKdHCgoKujUfIb0dBTg9Ympq2q35COntKMDpkd/97nfdmo+Q3o4CnB65efMm9+/jx4/D2toaQqEQ1tbWOH78uMZ8hPRldBVVT02fPh1VVVXcfXB8f7SH6Cfqwemp386hQXNqED6iAKenLCwsMGHCBOTl5WHChAl0awjhJTpE1SOnTp1SmcPiwoULuHDhgsZ8hPAB9eD0yMSJE7s1HyG9HQU4PdPRQ/T0kD3hEwpweogxpnYYeurUKQpuhHcowOmpiRMnorm5GQcOHEBzczMdlhJeogBHCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhrV4R4JKTkyEWi2FiYgJ/f3+cP39eq+12794NAwMDbk7VVowxxMXFwdHREaampggKCsKNGzd6oOaEkN5M5wFuz549kEgkiI+PR15eHnx8fBAaGor79++3u11ZWRliYmJUHj1q9cknn+Czzz5DSkoKzp07h379+iE0NBRPnjzpqWYQQnohnQe4LVu2YNmyZYiKioKXlxdSUlJgZmaG1NTUNrdRKBR47bXXkJCQADc3N5V1jDEkJSXhvffew5w5c+Dt7Y2dO3fi3r17OHDgQA+3hhDSm+j0Yfvm5mbk5uYiNjaWSxMIBAgKCkJ2dnab233wwQews7PD0qVL1e7ILy0tRWVlJYKCgrg0Kysr+Pv7Izs7G/PmzVMrTyaTQSaTcctSqRQAIJfLIZfLu9y+3q61bXxuoz5paWnh/vJ9n2rbPp0GuJqaGigUCm4W+1b29vYoLCzUuM3p06fx1Vdf4dKlSxrXt87QrqnMtmZvT0xMREJCglr6sWPHYGZm1lEz+ryMjAxdV4F0g9sNAGCInJwc3L2i69r0rKamJq3y9anhkh49eoSFCxdi27ZtsLGx6bZyY2NjIZFIuGWpVAoXFxeEhITwamb735LL5cjIyEBwcDCN6MsDl8sfAPkXERAQAJ/BA3RdnR7VepTVEZ0GOBsbGwiFQlRVVamkV1VVwcHBQS1/SUkJysrKMHv2bC5NqVQCAAwNDVFUVMRtV1VVBUdHR5UyfX19NdZDJBJBJBKppRsZGenFD19f2sl3hoaG3F++709t26fTiwzGxsbw8/NDZmYml6ZUKpGZmYnAwEC1/B4eHsjPz8elS5e41x/+8AdMnToVly5dgouLC1xdXeHg4KBSplQqxblz5zSWSQjhL50fokokEkRGRmLs2LEYN24ckpKS0NjYiKioKADAokWL4OzsjMTERJiYmGDkyJEq2/fv3x8AVNLfeustbNiwAS+99BJcXV3x/vvvw8nJSe1+OUIIv+k8wEVERKC6uhpxcXGorKyEr68v0tPTuYsE5eXlEAg619Fcs2YNGhsb8eabb6Kurg4TJ05Eeno6TExMeqIJhJBeyoDRKIdqpFIprKysUF9fz/uLDDRtIH9culWLuVtzcGB5AHyHDNR1dXqUtr9Rnd/oSwghPYUCHCGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4iwIcIYS3dD6iLyGka5qamlSm1yyqqIOsshgFV0yhrO2vktfDw0MvpsD8LQpwhPRRhYWF8PPzU0tfsEM9b25uLsaMGfMCatW7UIAjpI/y8PBAbm4ut9zwWIbDJ7Px8tRAmJuK1PLqo15xDi45ORlisRgmJibw9/fH+fPn28y7b98+jB07Fv3790e/fv3g6+uLXbt2qeRZvHgxDAwMVF5hYWE93QxCXigzMzOMGTOGewWOG4vxo4YicNxYlfQxY8bo5eEp0At6cHv27IFEIkFKSgr8/f2RlJSE0NBQFBUVwc7OTi3/gAED8Le//Q0eHh4wNjbGoUOHEBUVBTs7O4SGhnL5wsLCsH37dm5Z08TOhBB+03kPbsuWLVi2bBmioqLg5eWFlJQUmJmZITU1VWP+KVOm4I9//CM8PT0xdOhQrFq1Ct7e3jh9+rRKPpFIBAcHB+5lbW39IppDCOlFdNqDa25uRm5uLmJjY7k0gUCAoKAgZGdnd7g9YwwnTpxAUVERPv74Y5V1WVlZsLOzg7W1NaZNm4YNGzZg4EDNU6nJZDLIZDJuWSqVAng6rZ5cLu9K0/qE1rbxuY36RJ/2p7Zt1GmAq6mpgUKh4CZ5bmVvb69y+fu36uvr4ezsDJlMBqFQiC+++ALBwcHc+rCwMISHh8PV1RUlJSVYt24dZsyYgezsbAiFQrXyEhMTkZCQoJZ+7NgxvTh3kZGRoesqkOekUChw7do1PHz4EPn5+fDy8tL4XeeLpqYmrfLp/BxcV1hYWODSpUtoaGhAZmYmJBIJ3NzcMGXKFADAvHnzuLyjRo2Ct7c3hg4diqysLEyfPl2tvNjYWEgkEm5ZKpXCxcUFISEhvJ/4OSMjA8HBwTTxcx+2f/9+rF27FmVlZVyaWCzGxx9/jD/+8Y+6q1gPaj3K6ohOA5yNjQ2EQiGqqqpU0quqquDg4NDmdgKBAO7u7gAAX19fFBQUIDExkQtwv+Xm5gYbGxsUFxdrDHAikUjjRQgjIyO9+OHrSzv5aN++fZg3bx5mzZqFXbt24c6dOxg0aBA++eQTzJs3D3v37kV4eLiuq9nttP2+6vQig7GxMfz8/JCZmcmlKZVKZGZmIjAwUOtylEqlyjm037pz5w5qa2vh6Oj4XPUlpDdRKBR4++23MWvWLBw4cAD+/v4wNTWFv78/Dhw4gFmzZiEmJgYKhULXVdUZnV9FlUgk2LZtG3bs2IGCggIsX74cjY2NiIqKAgAsWrRI5SJEYmIiMjIycPPmTRQUFGDz5s3YtWsXXn/9dQBAQ0MD3nnnHeTk5KCsrAyZmZmYM2cO3N3dVW4jIaSvO3XqFMrKyrBu3ToIBKo/ZYFAgNjYWJSWluLUqVM6qqHu6fwcXEREBKqrqxEXF4fKykr4+voiPT2du/BQXl6usvMaGxuxYsUK3LlzB6ampvDw8MDXX3+NiIgIAIBQKMSvv/6KHTt2oK6uDk5OTggJCcH69evpXjjCKxUVFQCAkSNHalzfmt6aTx/pPMABQHR0NKKjozWuy8rKUlnesGEDNmzY0GZZpqamOHr0aHdWj5BeqfWUy5UrVxAQEKC2/sqVKyr59JHOD1EJIV0zadIkiMVifPjhh1AqlSrrlEolEhMT4erqikmTJumohrrXK3pwhJDOEwqF2Lx5M/70pz9hzpw5CA4Oxo0bN3Dr1i1kZGTg8OHD2Lt3L6/vh+sIBThC+rDw8HDExMTg008/xaFDh7h0Q0NDxMTE8PIWkc6gAEdIH7Zv3z5s2rQJL7/8MteDe+mll5CRkYFNmzYhICBAr4OcAWOM6boSvY1UKoWVlRXq6+t5/yTDkSNHMHPmTLrRtw9SKBRwd3fHqFGjcODAASgUCm5/CoVCzJ07F1euXMGNGzd4d5iq7W+ULjIQ0kfRfXAdowBHSB9F98F1jAIcIX3Us/fBaUL3wVGAI6TPovvgOkYBjpA+qvU+uEOHDmHu3LnIycnB48ePkZOTg7lz5+LQoUPYtGkT7y4wdAbdJkJIHxYeHo69e/fi7bffxu9//3su3dXVlbdDJXUGBThC+rjw8HDMmTMHJ0+exE8//YQZM2Zg6tSpet1za0UBjhAeEAqFmDx5MhobGzF58mQKbv8fnYMjhPAWBThCCG9RgCOE8Badg9Og9fFcbWfu6avkcjmampoglUrpWVQe0Kf92frb7OhRegpwGjx69AgA4OLiouOaEELa8+jRI1hZWbW5nkYT0UCpVOLevXuwsLCAgYGBrqvTY1rnf719+zavR03RF/q0PxljePToEZycnNQGGngW9eA0EAgEGDRokK6r8cJYWlry/gehT/Rlf7bXc2tFFxkIIbxFAY4QwlsU4PSYSCRCfHw8zRfLE7Q/1dFFBkIIb1EPjhDCWxTgCCG8RQGOEMJbFOB6qbKyMhgYGODSpUsvpJwzZ85g1KhRMDIywty5c5/rPQnpLSjA9RFZWVkwMDBAXV1dj5QvkUjg6+uL0tJSpKWl9ch7EM2mTJmCt956S9fV4PS2+jwPCnA9oLm5WddV6LSSkhJMmzYNgwYNQv/+/btUxotsN2MMLS0tL+z9eru++J17IRh5bpMnT2YrV65kq1atYgMHDmRTpkxh+fn5LCwsjPXr14/Z2dmx119/nVVXV3Pb/PTTT2zChAnMysqKDRgwgL388susuLiYW19aWsoAsF9++YX797OvyMjITpXz7bffssDAQCYSidiIESNYVlaWyvpnX9u3b2eMMZaVlcV+97vfMWNjY+bg4MDWrl3L5HJ5u+0+efIkA8DS09OZr68vMzExYVOnTmVVVVXsyJEjzMPDg1lYWLD58+ezxsZGriyFQsE+/PBDJhaLmYmJCfP29mbff/89t7613CNHjrAxY8YwIyMjdvLkye7cjToRGRmp9vkXFxezJUuWcJ/FsGHDWFJSktp2c+bMYRs2bGCOjo5MLBYzxhg7c+YM8/HxYSKRiPn5+bH9+/dz36NW7X03NdWntLT0RX0c3Y4CXDeYPHkyMzc3Z++88w4rLCxkOTk5zNbWlsXGxrKCggKWl5fHgoOD2dSpU7lt9u7dy3744Qd248YN9ssvv7DZs2ezUaNGMYVCwRhTDXAtLS3shx9+YABYUVERq6ioYHV1dZ0qZ9CgQWzv3r3s2rVr7I033mAWFhaspqaGtbS0sIqKCmZpacmSkpJYRUUFa2pqYnfu3GFmZmZsxYoVrKCggO3fv5/Z2Niw+Pj4NttdWFjIBaKAgAB2+vRplpeXx9zd3dnkyZNZSEgIy8vLY//617/YwIED2UcffcSVtWHDBubh4cHS09NZSUkJ2759OxOJRFwgbi3X29ubHTt2jBUXF7Pa2tqe3rU9rq6ujgUGBrJly5axiooKVlFRwZ48ecLi4uLYhQsX2M2bN9nXX3/NzMzM2J49e7jtIiMjmbm5OVu4cCG7cuUKu3LlCquvr2cDBgxgr7/+Ort69So7cuQIGzZsmEqAe/jwYbvfTU31aWlp0cVH0y0owHWDyZMns9GjR3PL69evZyEhISp5bt++zQUoTaqrqxkAlp+fzxhTDXCM/fsH/vDhw3br0lY5zwYTuVzOBg0axD7++GMuzcrKiuu5McbYunXr2PDhw5lSqeTSkpOTmbm5ORc8f9vuZ+t5/PhxLi0xMZEBYCUlJVzaf/zHf7DQ0FDGGGNPnjxhZmZm7OzZsyplLV26lM2fP1+l3AMHDrTb/r5o8uTJbNWqVe3mWblyJXvllVe45cjISGZvb89kMhmXtnXrVjZw4ED2+PFjLm3btm0q3yNtvpva1KevoHNw3cTPz4/79+XLl3Hy5EmYm5tzLw8PDwBPz3UBwI0bNzB//ny4ubnB0tISYrEYAFBeXt6p99W2nMDAQO7fhoaGGDt2LAoKCtost6CgAIGBgSrDRU2YMAENDQ24c+eOxnY/y9vbm/u3vb09zMzM4ObmppJ2//59AEBxcTGampoQHBys8pnt3LmT+7xajR07ts0680lycjL8/Pxga2sLc3NzfPnll2r7dNSoUTA2NuaWi4qK4O3tDRMTEy5t3LhxKtto893kExouqZv069eP+3dDQwNmz56Njz/+WC2fo6MjAGD27NkYMmQItm3bBicnJyiVSowcObLTJ4u7q5yuerbdz3p2RFkDAwO1EWYNDAy42dgbGhoAAIcPH4azs7NKvt8+V9nW+/HJ7t27ERMTg82bNyMwMBAWFhbYuHEjzp07p5KvK5+FNt9NPqEA1wPGjBmDH374AWKxGIaG6h9xbW0tioqKsG3bNkyaNAkAcPr06XbLbP2fWqFQdKmcnJwcbmLglpYW5ObmIjo6us338/T0xA8//ADGGNeLO3PmDCwsLLp9rDwvLy+IRCKUl5dj8uTJ3Vp2X2BsbKyyX8+cOYPx48djxYoVXJo2vavhw4fj66+/hkwm4/5juHDhgkqejr6bmurTl9Ehag9YuXIlHjx4gPnz5+PChQsoKSnB0aNHERUVBYVCAWtrawwcOBBffvkliouLceLECUgkknbLHDJkCAwMDHDo0CFUV1ejoaGhU+UkJydj//79KCwsxMqVK/Hw4UMsWbKkzfdbsWIFbt++jb/+9a8oLCzEjz/+iPj4eEgkknZHUO0KCwsLxMTEYPXq1dixYwdKSkqQl5eHzz//HDt27OjW9+qNxGIxzp07h7KyMtTU1OCll17CxYsXcfToUVy/fh3vv/++WqDSZMGCBVAqlXjzzTdRUFCAo0ePYtOmTQDA/SfV0XdTU31ae9p9EQW4HuDk5IQzZ85AoVAgJCQEo0aNwltvvYX+/ftDIBBAIBBg9+7dyM3NxciRI7F69Wps3Lix3TKdnZ2RkJCAd999F/b29oiOju5UOR999BE++ugj+Pj44PTp0zh48CBsbGzafb8jR47g/Pnz8PHxwV/+8hcsXboU77333nN9Nm1Zv3493n//fSQmJsLT0xNhYWE4fPgwXF1de+T9epOYmBgIhUJ4eXnB1tYWoaGhCA8PR0REBPz9/VFbW6vSm2uLpaUl/vnPf+LSpUvw9fXF3/72N8TFxQEAd16uo++mpvp09rxwb0LDJRHCY//4xz8QFRWF+vp6mJqa6ro6LxydgyOER3bu3Ak3Nzc4Ozvj8uXLWLt2LV599VW9DG4ABThCeKWyshJxcXGorKyEo6Mj/vznP+O///u/dV0tnaFDVEIIb9FFBkIIb1GAI4TwFgU4QghvUYAjhPAWBThCCG9RgCM6tXjxYhgYGHAP5Nvb2yM4OBipqamdekQoLS2tyyMRP4/FixfTHBa9GAU4onNhYWGoqKhAWVkZfvrpJ0ydOhWrVq3CrFmzaFhy8nx0Ohod0XutQ2//VmZmJgPAtm3bxhhjbPPmzWzkyJHMzMyMDRo0iC1fvpw9evSIMfbvwTCffbWOPLxz507m5+fHzM3Nmb29PZs/fz6rqqri3ufBgwdswYIFzMbGhpmYmDB3d3eWmprKrS8vL2d//vOfmZWVFbO2tmZ/+MMfuCG84+Pj1d6XD8Oo8wn14EivNG3aNPj4+GDfvn0AAIFAgM8++wxXr17Fjh07cOLECaxZswYAMH78eCQlJcHS0hIVFRWoqKhATEwMAEAul2P9+vW4fPkyDhw4gLKyMixevJh7n/fffx/Xrl3DTz/9hIKCAmzdupUbhEAulyM0NBQWFhY4deoUzpw5A3Nzc4SFhaG5uRkxMTF49dVXuR5oRUUFxo8f/2I/KNI+XUdYot/a6sExxlhERATz9PTUuO77779nAwcO5Ja3b9/OrKysOny/CxcuMABc72/27NksKipKY95du3apDdsuk8mYqakpO3r0aIf1J7pHPTjSa7FnBts8fvw4pk+fDmdnZ1hYWGDhwoWora1FU1NTu2Xk5uZi9uzZGDx4MCwsLLgBNVuHAFq+fDl2794NX19frFmzBmfPnuW2vXz5MoqLi2FhYcEN7z1gwAA8efKEl8N78xEFONJrFRQUwNXVFWVlZZg1axa8vb3xww8/IDc3F8nJyQDanw+0sbERoaGhsLS0xD/+8Q9cuHAB+/fvV9luxowZuHXrFlavXo179+5h+vTp3OFtQ0MD/Pz8cOnSJZXX9evXsWDBgh5uPekONJoI6ZVOnDiB/Px8rF69Grm5uVAqldi8eTM3KON3332nkl/TMNuFhYWora3FRx99BBcXFwDAxYsX1d7L1tYWkZGRiIyMxKRJk/DOO+9g06ZNGDNmDPbs2QM7OztYWlpqrCefhvfmI+rBEZ2TyWSorKzE3bt3kZeXhw8//BBz5szBrFmzsGjRIri7u0Mul+Pzzz/HzZs3sWvXLqSkpKiUIRaL0dDQgMzMTNTU1KCpqQmDBw+GsbExt93Bgwexfv16le3i4uLw448/ori4GFevXsWhQ4fg6ekJAHjttddgY2ODOXPm4NSpUygtLUVWVhb+8z//k5tZTCwW49dff0VRURFqamogl8tfzIdGtKPrk4BEvz07k7qhoSGztbVlQUFBLDU1lZt/lTHGtmzZwhwdHZmpqSkLDQ1lO3fuVJsn9i9/+QsbOHCgym0i33zzDROLxUwkErHAwEB28OBBtXlCPT09mampKRswYACbM2cOu3nzJldmRUUFW7RoEbOxsWEikYi5ubmxZcuWsfr6esYYY/fv32fBwcHM3NycbhPphWg8OEIIb9EhKiGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLcowBFCeOv/AV9SdtV3ht1LAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T21:22:46.265736Z", + "iopub.status.busy": "2024-02-29T21:22:46.265326Z", + "iopub.status.idle": "2024-02-29T21:22:46.574900Z", + "shell.execute_reply": "2024-02-29T21:22:46.573961Z" + }, + "papermill": { + "duration": 0.336808, + "end_time": "2024-02-29T21:22:46.577518", + "exception": false, + "start_time": "2024-02-29T21:22:46.240710", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABVmklEQVR4nO2dd3xT5frAvxlN2lJKWS2rUJC9ESiCgyFYuIii3p9c5DJFRcBBHQwVKA7wgogy5KIyvIKggmxBKEuGjFJ2QQqFsjqgdEPSJuf3R0iatEmatE2TwPu9n3NtznxyOOfJ8z7vM2SSJEkIBAKBlyF3twACgUBQEoTyEggEXolQXgKBwCsRyksgEHglQnkJBAKvRCgvgUDglQjlJRAIvBKhvAQCgVeidLcA5Y1er+f69etUrFgRmUzmbnEEAkEhJEkiKyuLWrVqIZfbtq8eOOV1/fp1QkND3S2GQCAohitXrlCnTh2b2x845VWxYkXAcGMCAwPdLI1AIChMZmYmoaGhpnfVFg+c8jIOFQMDA4XyEgg8mOLcOsJhLxAIvBKhvAQCgVcilJdAIPBKHjiflyNIkkR+fj46nc7doggEZYpCoUCpVN4XYUJCeRVCq9Vy48YNcnNz3S2KQOAS/P39qVmzJiqVyt2ilAq3Kq89e/Ywc+ZMYmJiuHHjBr/99hv9+/d36Nh9+/bRtWtXWrZsybFjx8pEHr1eT0JCAgqFglq1aqFSqe6LXyiBAAwjCq1WS2pqKgkJCTRq1MhuEKin41bllZOTQ5s2bRgxYgTPP/+8w8elp6czZMgQnnzySZKTk8tMHq1Wi16vJzQ0FH9//zI7r0DgKfj5+eHj48Ply5fRarX4+vqW2bn/99dl9sff5Kt/tUOldL1SdKvy6tOnD3369HH6uFGjRvHSSy+hUChYu3Ztmcvlzb9GAkFxuOr5/v7Pi1y6lcvIa+m0r1fFJdcwx+ve0iVLlnDx4kWmTJni0P4ajYbMzEyLRSAQlD35ekMvnztafblcz6uU1/nz55kwYQI//vgjSqVjRuP06dOpVKmSaRF5jQKBazD2IbubVz6z9F6jvHQ6HS+99BJRUVE0btzY4eMmTpxIRkaGably5YoLpby/uHTpEjKZrNQTIo6eZ9++fbRq1QofHx+HJ24EluzatQuZTEZ6enq5X1t/T3vdzS8f5eU1oRJZWVkcOXKE2NhYxo4dCxhmByVJQqlU8scff9CjR48ix6nVatRqdXmLe1+ya9cuunfvzu3btwkKCirz80dGRtK2bVt+//13AgICyvz8AtdiUl555TNs9BrlFRgYyMmTJy3WLViwgB07dvDrr79Sv359N0nmmWi1Wq+L47lw4QKjRo2yWwalOMrze0uShE6nc9iF4Sje+G8HBcNGTTlZXm4dNmZnZ3Ps2DHTcCIhIYFjx46RmJgIGIZ8Q4YMAQwzJC1btrRYgoOD8fX1pWXLllSoUMElMkqSRK423y2LM83Mu3XrxtixY3n77bepVq0aERERnDp1ij59+hAQEEBISAiDBw/m5s2bpmO2bNnCY489RlBQEFWrVuXpp5/mwoULVs9/6dIlunfvDkDlypWRyWQMGzbMqfOcPXuWLl26mP7Ndu/ebTq3TCbj1q1bjBgxAplMxtKlSwHYvXs34eHhqNVqatasyYQJE8jPz7f7vY1Dp61bt9KuXTv8/Pzo0aMHKSkp/P777zRr1ozAwEBeeukli2BkvV7P9OnTqV+/Pn5+frRp04Zff/3VtN143t9//5327dujVqvZu3dvsf82n3zyCcHBwVSsWJGRI0cyYcIE2rZta9o+bNgw+vfvz6effkqtWrVo0qQJAP/73//o0KEDFStWpEaNGrz00kukpKRYnHvz5s00btwYPz8/unfvzqVLl4qVx1XoTT6vB8DyOnLkiOmFAMOwAWDo0KEsXbqUGzdumBSZu7iTp6P55K1uufaZaRH4qxz/J1q2bBmvv/46+/btIz09nR49ejBy5Ei+/PJL7ty5w/jx43nxxRfZsWMHYIizi4yMpHXr1mRnZzN58mSee+45jh07VmQ6PTQ0lNWrV/PCCy9w7tw5AgMD8fPzc+o87733HnPmzKF58+bMnj2bfv36kZCQQGhoKDdu3KBJkyZMmzaNAQMGUKlSJa5du8Y//vEPhg0bxg8//MDZs2d55ZVX8PX1ZerUqVa/N8CNGzcAmDp1KvPmzcPf358XX3yRF198EbVazYoVK8jOzua5555j7ty5jB8/HjBM7vz4448sXLiQRo0asWfPHv79739TvXp1unbtarrehAkTmDVrFg0aNKBy5cp2/02WL1/Op59+yoIFC3j00UdZuXIlX3zxRZGRQnR0NIGBgWzbts20Li8vj48//pgmTZqQkpJCZGQkw4YNY/PmzYChJt3zzz/PmDFjePXVVzly5AjvvPNO8Q+Ki5BMw8bysbxkkjM/7/cBmZmZVKpUiYyMjCL1vO7evUtCQgL169c3Be/lavO9Qnl169aNzMxMjh49Chh+7f/880+2bi2Q/erVq4SGhnLu3Dmrkx43b96kevXqnDx5kpYtW3Lp0iXq169PbGwsbdu2ddjnZes8M2bMMCmK/Px86tevzxtvvMH7778PQFBQEHPmzDFZdB988AGrV68mLi7OlOmwYMECxo8fT0ZGBnK5vMj3hgLf3Pbt23nyyScBmDFjBhMnTuTChQs0aNAAMMQLXrp0iS1btqDRaKhSpQrbt2+nc+fOpnONHDmS3NxcVqxYYTrv2rVrefbZZx36d3nkkUfo0KED8+bNM6177LHHTKMOMFheW7ZsITEx0e5w8ciRI3Ts2JGsrCwCAgKYNGkS69at4/Tp06Z9JkyYwOeff27338jac14WtJv2B7dz83izR0Min2pS4vPYe0fN8Rqfl7vw81FwZlqE267tDO3btzf9ffz4cXbu3GnV8X3hwgUaN27M+fPnmTx5MgcPHuTmzZvo9QZzPzExkZYtWzp8XUfPY64UlEolHTp0IC4uzuZ54+Li6Ny5s0WK1qOPPkp2djZXr16lbt26Rb63Oa1btzb9HRISgr+/v0lxGdcdOnQIgPj4eHJzc+nVq5fFObRaLe3atbNY16FDB5syF+bcuXOMHj3aYl14eLjJ+jXSqlWrIoorJiaGqVOncvz4cW7fvm1xX5s3b05cXBydOnWyOMb8Hpc3epPP6wEYNnoDMpnMqaGbOzH3+2VnZ9OvXz8+//zzIvvVrFkTgH79+lGvXj2+/fZbatWqhV6vp2XLlmi1WqeuW1bnKSm2/J0+Pj6mv2UymcVn4zqjQsjOzgZg06ZN1K5d22K/wrPVrvCvFj5nTk4OERERREREsHz5cqpXr05iYiIRERHldl+dRV/Ow0bveCsFTvPwww+zevVqwsLCrM6G3bp1i3PnzvHtt9/y+OOPAxTrfDZaBualgpw5z19//cUTTzwBGIaNMTExprAXazRr1ozVq1cjSZLJ+tq3bx8VK1Ys1YykNZo3b45arSYxMdHCv1VamjRpwuHDh00TTwCHDx8u9rizZ89y69YtZsyYYQqsPnLkiMU+zZo1Y/369Rbr/vrrrzKQumRI5eyw95ogVYFzjBkzhrS0NAYOHMjhw4e5cOECW7duZfjw4eh0OipXrkzVqlVZtGgR8fHx7NixwzRhYot69eohk8nYuHEjqampZGdnO3We+fPn89tvv3H27FnGjBnD7du3GTFihM3rjR49mitXrvDGG29w9uxZ1q1bx5QpU4iMjCzz/LyKFSvy7rvvMm7cOJYtW8aFCxc4evQoc+fOZdmyZSU+7xtvvMH333/PsmXLOH/+PJ988gknTpwotlpJ3bp1UalUzJ07l4sXL7J+/Xo+/vhji31GjRrF+fPnee+99zh37hwrVqwwzdK6A6mcg1SF8rpPqVWrFvv27UOn0/HUU0/RqlUr3n77bYKCgpDL5cjlclauXElMTAwtW7Zk3LhxzJw50+45a9euTVRUFBMmTCAkJISxY8c6dZ4ZM2YwY8YM2rRpw969e1m/fj3VqlWze73Nmzdz6NAh2rRpw6hRo3j55Zf58MMPS3VvbPHxxx/z0UcfMX36dJo1a0bv3r3ZtGlTqWIIBw0axMSJE3n33Xd5+OGHSUhIYNiwYcU6yqtXr87SpUv55ZdfaN68OTNmzGDWrFkW+9StW5fVq1ezdu1a2rRpw8KFC/nss89KLGtpMfm8ysnyErONZrhqFkYgMKdXr17UqFGD//3vf265vque8yYf/o4mX0+3JtVZOjy8xOcRs40CgQeQm5vLwoULiYiIQKFQ8NNPP7F9+3aLeK77BZGYLRB4ES1atCAgIMDqsnz5cmQyGZs3b+aJJ56gffv2bNiwgdWrV9OzZ093i17miNxGgcCL2Lx5M3l5eVa3hYSE4Ofnx/bt28tZKvdgVF4izksg8ALq1avnbhE8BqPzXCOGjQKBwFuQJEn4vAQCgfdhHrNwt5yGjUJ5CQSCUqM3015i2CgQCLwGvbC8BAKBN2Jueen0Enk61yswobwEhIWFMWfOHNNnmUzmkn6YgvuXwnk65eG0F6ESgiLcuHGj2AqhAoE5EpbaS5Ovp6KLrymUl6AINWrUcLcILmtuIXANejdYXmLYWBySBNoc9yxO5MxnZWUxaNAgKlSoQM2aNfnyyy/p1q0bb7/9ttNf2XzYaGyOsWbNGrp3746/vz9t2rThwIEDFsfs3buXxx9/HD8/P0JDQ3nzzTfJyckxbS+umURJm1sIPAN9oWe1PFKE3PqztmfPHmbOnElMTAw3btzgt99+s9tsdM2aNXzzzTccO3YMjUZDixYtmDp1KhERLizTnJcLn9Vy3fntMek6qByr2hkZGcm+fftYv349ISEhTJ48maNHj1p0qSkNH3zwAbNmzaJRo0Z88MEHDBw4kPj4eJRKJRcuXKB379588sknLF68mNTUVMaOHcvYsWNZsmQJUHwzCSPONLcQeA5SIV113/u8cnJyaNOmDSNGjOD5558vdv89e/bQq1cvPvvsM4KCgliyZAn9+vXj4MGDReqMP0hkZWWxbNkyVqxYYWo4sWTJEmrVKjul++6779K3b18AoqKiaNGiBfHx8TRt2pTp06czaNAgk5XXqFEjvv76a7p27co333yDr6+vRdHBBg0a8PXXX9OxY0eys7Mt6uxPmzatSB15gedT2PIqj/xGtyqvPn360KdPH4f3N58RA/jss89Yt24dGzZscJ3y8vE3WEDuwMffod0uXrxIXl4e4eEFNZQqVapk6v9XFpg3szDWwE9JSaFp06YcP36cEydOsHz5ctM+kiSh1+tJSEigWbNmxTaTMOJMcwuB51BEed3vlldp0ev1ZGVlUaVKFZv7aDQaNBqN6XNmZqZzF5HJHB663c8UbmYBWDSveO2113jzzTeLHFe3bl2nmkm4qnmwwLUUcdiXQylor1Zes2bNIjs7mxdffNHmPtOnTycqKqocpSp/GjRogI+PD4cPHza1A8vIyODvv/82NbxwJQ8//DBnzpyhYcOGVrefPHmy2GYSAu+mcKhEeTjsvXa2ccWKFURFRfHzzz8THBxsc7+JEyeSkZFhWq5cuVKOUpYPFStWZOjQobz33nvs3LmT06dP8/LLLyOXy4tt9FAWjB8/nv379zN27FiOHTvG+fPnWbdunakzkCPNJATeTeGJcU05WF5eqbxWrlzJyJEj+fnnn4utSKlWqwkMDLRY7kdmz55N586defrpp+nZsyePPvoozZo1K5da/K1bt2b37t38/fffPP7447Rr147JkyebJgwcaSYh8G4euFCJkvDTTz8xYsQIVq5caZr9EhisL3OHeU5ODlFRUbz66qvFHnvp0iWLz+Y9WcLCwijcoyUoKKjIuo4dO/LHH3/YvMbAgQMZOHCgzet069atyDkF3oM7glTdqryys7OJj483fU5ISODYsWNUqVKFunXrMnHiRK5du8YPP/wAGIaKQ4cO5auvvqJTp04kJSUB4OfnR6VKldzyHTyF2NhYzp49S3h4OBkZGUybNg2AZ5991s2SCR4E9PoHzOd15MgR2rVrZwpziIyMNA05wJBjl5iYaNp/0aJF5OfnM2bMGGrWrGla3nrrLbfI72nMmjWLNm3a0LNnT3Jycvjzzz+Ji4uz2SDCPL5KICgN7vB5udXyKm6oULj7765du1wrkBfTrl07YmJiiqy/c+cOx44dK3+BBA8UwuclKHP8/PxshjAIBGVFYRNEJGa7CeE4FtzPuOL5LpoeJJRXuWKMIs/NzXWzJAKB6zA+3+ZZE6WlsELUiGFj+aJQKAgKCjKVavH39y+XIE+BoDyQJInc3FxSUlIICgpCoVCU2blFepAHYCzEZ15rSiC4nwgKCirzgpPCYe8ByGQyatasSXBwsM027gKBt+Lj41OmFpcR/YNWz8uTUSgULvlHFgjuR9xRz0s47AUCQZkjQiUEAoFXUNTnJZSXQCDwAoomZotho0Ag8AKEz0sgEHglRYNUxbBRIBB4AcZho6+PQaWUR5CqUF4CgaDUGOt5+fkYwovydBK6wo6wMkYoL4FAUGqMaspfVRA66uoZR6G8BAJBqTE67P1UBYHdrnbaC+UlEAhKjdFfr5TLUCnu+b1cbHmVKj0oOzvb1HjUyP3anUcgENjGaHnJZDLUSjland7zho0JCQn07duXChUqUKlSJSpXrkzlypUJCgqicuXKrpBRIBB4OEbfvAxQ33PauzpQ1Wnl9e9//5vbt2+zePFioqOj2bFjBzt27GDnzp3s2LHDqXPt2bOHfv36UatWLWQyGWvXri32mF27dvHwww+jVqtp2LBhkTr3AoGg/DFaXnJ5QbiEq6upOj1sPH78ODExMTRp0qTUF8/JyaFNmzaMGDGC559/vtj9jVbfqFGjWL58OdHR0YwcOZKaNWsSERFRankEAkHJMAapymUyfMvJ8nJaeXXs2JErV66UifLq06cPffr0cXj/hQsXUr9+fb744gsAmjVrxt69e/nyyy+F8hII3IjRYW/0eYHrA1WdVl7fffcdo0aN4tq1a7Rs2bJIHezWrVuXmXCFOXDgAD179rRYFxERwdtvv23zGI1Gg0ajMX3OzMx0lXgCwQOL0ecll2GyvFydIuS08kpNTeXChQsMHz7ctE4mkyFJEjKZDJ3OdQInJSUREhJisS4kJITMzEzu3LmDn59fkWOmT59OVFSUy2QSCARmPi+ZzMzn5WHDxhEjRtCuXTt++uknQkJCPL5BxcSJE4mMjDR9zszMJDQ01I0SCQT3HwU+L/BVGn1eHmZ5Xb58mfXr17ulkWmNGjVITk62WJecnExgYKBVqwtArVajVqvLQzyB4IGlIFRChtqYnO1poRI9evTg+PHjrpClWDp37kx0dLTFum3bttG5c2e3yCMQCAwUBKl6sOXVr18/xo0bx8mTJ2nVqlURh/0zzzzj8Lmys7OJj483fU5ISODYsWNUqVKFunXrMnHiRK5du8YPP/wAwKhRo5g3bx7vv/8+I0aMYMeOHfz8889s2rTJ2a8hEAjKkAKHvcwUpOpxPq9Ro0YBMG3atCLbnHXYHzlyhO7du5s+G31TQ4cOZenSpdy4cYPExETT9vr167Np0ybGjRvHV199RZ06dfjuu+9EmIRA4GYkK0GqHmd5Fc5lLA3dunUrUoHRHGvR8926dSM2NrbMZBAIBKVHMre8lB6YHpSXl4dSqeTUqVOukkcgEHgh5onZ5VVN1Snl5ePjQ926dV0ayyUQCLwP60GqHmR5AXzwwQdMmjSJtLQ0V8gjEAi8EJPlBfh6anrQvHnziI+Pp1atWtSrV48KFSpYbD969GiZCScQCLwD88RstaemB/Xv398FYggEAm/GPDHbt5yCVJ1WXlOmTHGFHAKBwIux8HkpjXFeHmZ5GYmJiSEuLg6AFi1a0K5duzITSiAQeBd6b6jnlZKSwr/+9S927dpFUFAQAOnp6XTv3p2VK1dSvXr1spZRIBB4OOZBqupyClJ1erbxjTfeICsri9OnT5OWlkZaWhqnTp0iMzOTN9980xUyCgQCD8ciMdsYpOppw8YtW7awfft2mjVrZlrXvHlz5s+fz1NPPVWmwgkEAu/AIjHbWM/L0+K89Hp9kWRsMASwlmXqkEAg8B7ME7MLfF4eNmzs0aMHb731FtevXzetu3btGuPGjePJJ58sU+EEAoF3YFGM0Ki8PK1j9rx588jMzCQsLIyHHnqIhx56iPr165OZmcncuXNdIaNAIPBwLBOzDWpFm69Hr7ddeKG0OO3zCg0N5ejRo2zfvp2zZ88Chi4+hRtjCASCBwfLxGyFab1Wp8dXrrB1WKkoUZyXTCajV69e9OrVq6zlEQgEXohlkGrBgO5uns5CmZUlJVJe0dHRREdHk5KSUsRJv3jx4jIRTCAQeA/ms41KhRylXEa+XnJpoKrTyisqKopp06bRoUMHatas6fHdgwQCgesxT8wGUCvl5Gt1Lp1xdFp5LVy4kKVLlzJ48GBXyCMQCLwQvVliNhhmHHO0OpfWsXd6tlGr1dKlS5cyE2D+/PmEhYXh6+tLp06dOHTokN3958yZQ5MmTfDz8yM0NJRx48Zx9+7dMpNHIBA4j2Tm8wLKJdbLaeU1cuRIVqxYUSYXX7VqFZGRkUyZMoWjR4/Spk0bIiIiSElJsbr/ihUrmDBhAlOmTCEuLo7vv/+eVatWMWnSpDKRRyAQlAx94WFjOeQ3Oj1svHv3LosWLWL79u20bt26SLT97NmzHT7X7NmzeeWVVxg+fDhgGJJu2rSJxYsXM2HChCL779+/n0cffZSXXnoJgLCwMAYOHMjBgwed/RoCgaAMMQ9SBczyGz3IYX/ixAnatm0LUKQRhzPOe61WS0xMDBMnTjStk8vl9OzZkwMHDlg9pkuXLvz4448cOnSI8PBwLl68yObNm+363zQaDRqNxvQ5MzPTYRkFAoFjFPV5GfMbPcjy2rlzZ5lc+ObNm+h0OkJCQizWh4SEmIJfC/PSSy9x8+ZNHnvsMSRJIj8/n1GjRtkdNk6fPp2oqKgykVkgEFjHPFQCzLpme5LD3p3s2rWLzz77jAULFnD06FHWrFnDpk2b+Pjjj20eM3HiRDIyMkzLlStXylFigeDBwDwxG8qn8WyJK6mWlmrVqqFQKEhOTrZYn5ycTI0aNawe89FHHzF48GBGjhwJQKtWrcjJyeHVV1/lgw8+QC4vqovVajVqtbrsv4BAIDBhy+flymGj2ywvlUpF+/btiY6ONq3T6/VER0fTuXNnq8fk5uYWUVAKheEm2eu8LRAIXIvx7StsebkyzsttlhdAZGQkQ4cOpUOHDoSHhzNnzhxycnJMs49Dhgyhdu3aTJ8+HYB+/foxe/Zs2rVrR6dOnYiPj+ejjz6iX79+JiUmEAjKH2P1CPMgVfCwYeOePXvo0qULSqXlofn5+ezfv58nnnjC4XMNGDCA1NRUJk+eTFJSEm3btmXLli0mJ35iYqKFpfXhhx8ik8n48MMPuXbtGtWrV6dfv358+umnzn4NgUBQhuhtBql6kOXVvXt3bty4QXBwsMX6jIwMunfvjk7nnKYdO3YsY8eOtbpt165dFp+VSiVTpkwR7dcEAg+jSJCq0vUOe6d9XpIkWY3nunXrVpHu2QKB4MFAKhQqYeqa7Qk+r+effx4wjGmHDRtmMYOn0+k4ceJEmeY8CgQCA2k5WqpUULlbDLvYClL1CJ9XpUqVAIOGrVixIn5+fqZtKpWKRx55hFdeeaXsJRQIHmBmbT3HvJ3xzPxna/6vQ6i7xbGJvlCoRHkEqTqsvJYsWQIY8gnfffddMUQUCMqBeTvjAZiy/rRHK6/CoRIemZgtnOUCgaAwhYNUjZaXR9XzSk5OZvDgwdSqVQulUolCobBYBAJB2ePpMdjGavAeHec1bNgwEhMT+eijj0QZaIGgnJDwbO1VOFTCI6tK7N27lz///NNUFkfw4CFJltHUAkHBbKPhv6Z6Xi4MUnV62BgaGiryCB9g9HqJ/gv28+/vD4rnoBzx9FtdxOdlym30oCDVOXPmMGHCBC5duuQCcQSezqVbORy/ks6++Ftoda5t5y4owMN1l5VhowemBw0YMIDc3Fweeugh/P39i5SBTktLKzPhBJ6Hefd2uRg2lh8err2M4hUJUnWh5eW08pozZ44LxBB4DwVvkacPZQTlR+HE7AKflwcpr6FDh7pCDoGXYK6wPH0GTFB+2OoepMnX28yHLi0lKkZ44cIFPvzwQwYOHGhqU/b7779z+vTpMhVO4HmYqytheZUfnv5DUdRhbywSist8o04rr927d9OqVSsOHjzImjVryM7OBuD48eMi+v4BQCgs9+Dp990YpGqMlTBG2IPrnPZOK68JEybwySefsG3bNlSqgkz3Hj168Ndff5WpcALPQy8Jn5c78PRbXTgx20chM8V8uSpQ1WnldfLkSZ577rki64ODg7l582aZCCXwXITPyz14ekxd4e5BMpnM5fmNTiuvoKAgbty4UWR9bGwstWvXLhOhBJ6LJGYbBVaxtLzA9TW9nFZe//rXvxg/fjxJSUnIZDL0ej379u3j3XffZciQIa6QUeBBCIUlsEbhYoTg+kBVp5XXZ599RtOmTQkNDSU7O5vmzZvzxBNP0KVLFz788EOnBZg/fz5hYWH4+vrSqVMnDh06ZHf/9PR0xowZQ82aNVGr1TRu3JjNmzc7fV1BybAcNgrKC0+/14VDJcCsjr2LAlWdjvNSqVR8++23fPTRR5w6dYrs7GzatWtHo0aNnL74qlWriIyMZOHChXTq1Ik5c+YQERHBuXPnijT4ANBqtfTq1Yvg4GB+/fVXateuzeXLlwkKCnL62oKSYTls9PRX6v7B02914SBVKLC8NC6yvErct7Fu3brUrVu3VBefPXs2r7zyiqlP48KFC9m0aROLFy9mwoQJRfZfvHgxaWlp7N+/35SWFBYWVioZBM6hF5aXwAqFG3BAQRMOV/m8HFJekZGRfPzxx1SoUIHIyEi7+86ePduhC2u1WmJiYpg4caJpnVwup2fPnhw4cMDqMevXr6dz586MGTOGdevWUb16dV566SXGjx9vsxCiRqNBo9GYPmdmZjokn8A6kgiVEFjB2rDR1xOGjbGxseTl5Zn+toUzKQA3b95Ep9OZGswaCQkJ4ezZs1aPuXjxIjt27GDQoEFs3ryZ+Ph4Ro8eTV5ens0A2enTpxMVFeWwXAL7SDY/CB5kCldSBXPLy43Dxp07d1r9u7zR6/UEBwezaNEiFAoF7du359q1a8ycOdOm8po4caKFtZiZmUloqOc2MvB0RJyXwBqStVAJpWtrepXY51VaqlWrhkKhIDk52WJ9cnIyNWrUsHpMzZo18fHxsRgiNmvWjKSkJLRarUXEvxG1Wm3RY1JQWsSwUVCUwkGq4PpQCYeUl7HhrCOsWbPGof1UKhXt27cnOjqa/v37AwbLKjo6mrFjx1o95tFHH2XFihXo9XrkcoNW//vvv6lZs6ZVxSUoe0SohMAahROzwUOCVCtVqmRaAgMDiY6O5siRI6btMTExREdHmxrTOkpkZCTffvsty5YtIy4ujtdff52cnBzT7OOQIUMsHPqvv/46aWlpvPXWW/z9999s2rSJzz77jDFjxjh1XUHJ0QuNJbCCtSBVY00vV+U2OmR5GRvOAowfP54XX3yRhQsXmoZvOp2O0aNHExgY6NTFBwwYQGpqKpMnTyYpKYm2bduyZcsWkxM/MTHRZGGBoX7+1q1bGTduHK1bt6Z27dq89dZbjB8/3qnrCkqO5Wyj0GQCA8bZRvMpO1+zml6uwGmf1+LFi9m7d6+F30mhUBAZGUmXLl2YOXOmU+cbO3aszWHirl27iqzr3LmzqF7hRkScl+vY/XcqURtOM/OfrWlfr4q7xXEK+z4vD8ltzM/PtxrKcPbsWfR60ZDhfkckZruO7WeSuZiaw65zqe4WxWlMPi8zjeIRDntzhg8fzssvv8yFCxcIDw8H4ODBg8yYMcPkqxLcv4hQCdeRd6/iqN4LfxUkqz4vDwhSNWfWrFnUqFGDL774wlQap2bNmrz33nu88847ZS6gwLPQiXGjy8jTGW6oN3aUs5qY7Wm5jXK5nPfff5/333/flGrjrKNe4L1YVFJ1oxz3I/n33C7eOBFiNTHb0ywvc4TSevCwGDZ63zvm0eSbLC/vu7GStdxGT0jMLsyvv/7Kzz//TGJiIlqt1mLb0aNHy0QwgWdiaXlZf8lc1erqfqfA5+VmQUqAtVAJk8/LU4oRfv311wwfPpyQkBBiY2MJDw+natWqXLx4kT59+rhCRoEHUdyLpddLvPjfAwxfYr+opKAo+fdurjc67O1VUvWYGvYLFixg0aJFzJ07F5VKxfvvv8+2bdt48803ycjIcIWMAg+iuO5BiWm5HL50m53nUl3aLfl+xLtnG62lB7k2wt5p5ZWYmEiXLl0A8PPzIysrC4DBgwfz008/la10Ao9Dr7fvsDdfJ0aOzmH0dXmnz8vwX7nc3PLygNxGc2rUqEFaWhpgqKZqjHZPSEjwylkSgXNYREqIf+8yxeiw90LdVaRvIxTkNt71lGFjjx49WL9+PWAIWB03bhy9evViwIABVvs5Cu4vRNNZ15F3L1RC74Xay7rP615uo6fMNi5atMiUBjRmzBiqVq3K/v37eeaZZ3jttdfKXECBZ+GN/hhvocDy8r57bLUMtI9rLS+nlFd+fj6fffYZI0aMoE6dOoChj+O//vUvlwgncD2SJDF+9QnqVwvg9W4PObC/9b8FpcebQyVM6UFm64wds3V6iTydHh+F0wM9uzh1NqVSyX/+8x/y8/PLVAiB+zhy+TY/H7nK51us9w0ojCNxXub8dCiR3nP2cD39TollfFDw7lAJa+lBBerFFU57p1Xhk08+ye7du8tcEIF7yNU691CZz4Q58o5NXHOSs0lZfLo5zlnRHjjyvTpUwvBfi9ZnygL14opYL6d9Xn369GHChAmcPHmS9u3bU6FCBYvtzzzzTJkJJyh/cjT5VFDbfixK+l65yml7P5HnxelB1iwvmUyGWilHk693ieXltPIaPXo0YL0/o0wmQ6cTD6m3suNsMiOXHWFKvxYM7RJmdR9nErPNFZ1IFyqegsRsNwtSAkyJ2YXGcr4+invKywMsL1Fw8P7l5NVM9BKcumY7U6K4OC9bKupBV113tDr8VNYbIxu53xKzwTy/0QN8XoL7C/NHzdhfT2fnp7+kJXEOXLzFkn0PZiDz3vM3aTZ5C/8pZlLEm9ODrAWpgmvzGx1WXnfu3GHjxo2mz8Zmrsblvffe4+7duyUSYv78+YSFheHr60unTp04dMixpN6VK1cik8lMrdMEpUObX3yQpFTCINWsu/lEbTjDvvhbJZbPW4nacBqABbsu2N3Pu2cbjX9Zai9XBqo6rLyWLVvGf//7X9PnefPmsX//fmJjY4mNjeXHH3/km2++cVqAVatWERkZyZQpUzh69Cht2rQhIiKClJQUu8ddunSJd999l8cff9zpawqso733y6+z8+5YDmmcf8kup+U4fcyDwv2WHgTmgapuVF7Lly/n1VdftVi3YsUKdu7cyc6dO5k5cyY///yz0wLMnj2bV155heHDh9O8eXMWLlyIv78/ixcvtnmMTqdj0KBBREVF0aBBA6evKSjA/D1xxPKy9Hk5fz1vfDHLC2N6kDf6vIwPkm2flxuHjfHx8bRq1cr02dfX16KnYnh4OGfOnHHq4lqtlpiYGHr27FkgkFxOz549OXDggM3jpk2bRnBwMC+//HKx19BoNGRmZlosAusYlZe9l6e0ZaC9MW/PUbadSWbH2eQSHavTS6YfA+8cNlp32LuymqrDs43p6eloNBrT59RUy/ZMer3eYrsj3Lx5E51OZ2oyayQkJMRqezWAvXv38v3333Ps2DGHrjF9+nSioqKckutBwsJhbxo22vN5Wf/bUawpxmxNPpl38qgV5Of8CT2ErLt5vPKDoYv82Y97m15aR8k3m8X3TuVl+G/hiBhT12x3Ouzr1KnDqVOnbG4/ceKEKd/RVWRlZTF48GC+/fZbqlWr5tAxEydOJCMjw7RcuXLFpTJ6M44NG0v3YukliV3nUnh+wT7iU7IBGL7kEF1m7CA+JatU53Ynd8wyFUpiZeSbORq9MRrJZHnJrTvs3Wp5/eMf/2Dy5Mn07dsXX19fi2137twhKiqKvn37OnXxatWqoVAoSE62NLWTk5OpUaNGkf0vXLjApUuX6Nevn2mdMe5MqVRy7tw5HnrIMrlYrVajVqudkutBxTRstBsqUfB3Sfo26vQSw5YcBmDsiqNsefsJDl+6DcCBC7doGFzR6XN6AuYvbUlGxubKy97991RMxQhtWF5u9XlNmjSJtLQ0mjRpwsyZM1m3bh3r1q3jP//5D02aNOH27dtMmjTJqYurVCrat29PdHS0aZ1eryc6OprOnTsX2b9p06acPHmSY8eOmZZnnnmG7t27c+zYMUJDQ526vsASZ31e1vYrLpDe/MU8m5Rl8Yv89Y54R0X1KJIz73LmeoEvtSQO9zwzc8sbY+EKGnB4oOUVEhLC/v37ef3115kwYYLpBstkMnr16sWCBQuK+K4cITIykqFDh9KhQwfCw8OZM2cOOTk5pu7bQ4YMoXbt2kyfPh1fX19atmxpcXxQUBBAkfUCxzBXNsYgVXtDQ/MhZd+v97Lq1Ufo1KCqw9crPCQds7yg21RqloZ8nR5lGZdOcTWdPou2+FySobWF5eWFkxruCFJ1Kj2ofv36bNmyhbS0NOLjDb+SDRs2pEqVKiUWYMCAAaSmpjJ58mSSkpJo27YtW7ZsMSnCxMREi1lNgeswxXk5GCoBMHr5UWI+6uXwNQp3g44+axnPp5Ok0jUTLWduZBQt9VMS5ZWnM3fYl0okt2AUuXAOq0dYXuZUqVKF8PDwMhNi7NixjB071uq2Xbt22T126dKlZSbHg06Bw972PoVfzGxNvlOF5orz53jbiMl8uGikJJZTvtkx3jbbKElSsT4vjTuDVAX3P4447Av7YzT5ehp98Dvv/XLcdLw9iovz8rYhU56VdISS6J58nfeGSpiLWzTOywOCVAX3P4457K2v/yXmKmuOXi32xS3uxczR5rPzbIoX9Xws+n1K5LC38HmVSqByx/zf1FaQqrC8BC7FEZ+XPass/U5esdco7r1+/9cTDF96mAmrTxR7Lk/A2u3IL9Gw0XtnGy2+bmGHvSeESgjufzROhkoUxhG/V3GW165zhsyNtceuF3suT6VkDnvvnW20tLwst6ld6LAXyktgwuSwdzA9qDAqZfGPk7e9mMVh7dvk2yvLYYP71edVEKQqlJfAhThkednZ5iOXFTubeN8pLytfp7SzjV6muywyLWw57N2a2yi4PykcEQ2OpwcVxkchZ8XBRLvX8zaroiSUJL3HPM7L29KDzJ+JwhkWrqwqIZSXoAglTcxWKeXEJt62e+77zvKyOtvovJVhkZjtdcrLtuXlEfW8BA8OjtawL4yPQoaymGwIb3sxi8P6sNH581gEqXpZqIRkJq/t9CBheQnKAWci7Atjrru+ij5fZPv9ZnlZI78klpcX1/Myl7doetA95SUsr/JDkiTe++U4O8/ar6V/P1KSIFXDcZaK7xsrDSec0V0ZucXHjbkba1+nJJaTNydm2wuVMEXYC8ur/Nh8MgnZsf8xaekWXl56mEs3H5zGEc6kB5mjl6RirQ5nykD/cSbJ4X3dhbX7URLLy5sTsy0d9tZDJfJ0UpkrZW9K4C9XulW+SV+fb8EHDl9szA9fdaFqxxcZFtGJCur7+7bZddgXM6QsLrrcmZm09349waMNq3lceejMu3lIeqjk72N1e4lK4nhzYjbWy+FAgeUFhhnHsnx3hOVlgwoyLdR7FICO8r+ZrFjKqJi+nJnRnWNr5yDl3L/9B0vqsNdLktVEZYtzO/nraygPnc3U9adJyihZX9CyRKeXaD31D9pM+8OmE/pBDVItPNMIBelBUPaxXve3CVEa6rSH4Zsh4xrS6d/IPLKKSmkn6CidgGMnyD82jazajxPQ/kV8mvcD30B3S1xmlDS3Ua8vPkxg44kbTsvTc/ZuAGKvpLNuzKNOH18WnLyaQd2q/mw8UZC2lJajLbMg1fshPcia8pLLZagUcrQ6fZnHegnlVRyVaiPrMpZKXcaiSblAzObFVE7YQDPZZSpf2wXXdqFd/xY3gh+n7hODkTWOAFUFd0vtMNbKNtvvmG37XDoHho2l4fiVdJed2x57/k5lyOJDBFdUk5JV0CFLYaPmdWEF78gdsUzMLpGYbsNW5yAjaqVQXm5HHfwQXYZ9yvX0D/l8wzYqXdxAL92fPCS/Qb2UHfDrDvDxhyZ9oOUL0LAnKL2v+UdJh42SJJVoyOTpbD1tmDgwV1wACrnMRpCqaywvSZKKOMQ9AeOPnU3l5aMgS5Nf5oGqQnmVgFpBfowf/AyS1I/LN3OYt/0P5Kd/41nlQWrnJcOp1YZFXQma9jUosgZdQWHdwetOrOki+055+9vyva0YlQNYGw7ZW1+i3EYHIuz1Eig8T3fZ9XlBgdNel3kDapWde8UjHPbz588nLCwMX19fOnXqxKFDh2zu++233/L4449TuXJlKleuTM+ePe3u70pkMhlh1QMY+c9n+SlwOI/enc0PLRfDI2OgYk3QZMDxFbD8BZjVGDa8DQl7QO/ZhfZKannp9K4dNroLa7NoRlxRz8u28vLMe2vP50XmdV7Sb+Q31WRaruwE6WXXN9XtymvVqlVERkYyZcoUjh49Sps2bYiIiCAlxXpw6K5duxg4cCA7d+7kwIEDhIaG8tRTT3Ht2rVylrwAXx8Fk59uAcj4ONaPC+0nwbgzMPx36DgS/KvBnTSIWQLL+sHsZvD7eEg86PZcEGvPW0mrSkgu9nm5C1tDNQlblmvpho22DvdU5WWUynSbcm7C4e9hSV+Y3ZzRmu9pJ4837HDlYJld1+3Ka/bs2bzyyisMHz6c5s2bs3DhQvz9/Vm8eLHV/ZcvX87o0aNp27YtTZs25bvvvjP1enQnPZsF071JdfJ0ElPXn0aSyaBeF+j7BbxzDgb/Bu0Gg28lyE6Ggwth8VPwVWv44yO4fsyjPLW2XkD7oRKuHza6o8qoreGQLVlKZHmZV5Ww6fNy+rTlgl6SCCSH/uyC/z1vGGVsioTLewGJOJ/mTM4bys6+u6HVP8vsum71eWm1WmJiYpg4caJpnVwup2fPnhw4cMChc+Tm5pKXl1eq9mtlgUwmY0q/FuyL38Of52+y9XQSvVvWNGxUKOGhHoal72y4sANOr4GzmyDjCuz/2rBUeQhaPm/wkQU3c+v30UkScivlcuynB7ne8krKvEvNSuUXtJpwM4fF+xKsbpOwPpNY2npeYN0573GWlzYHzv1OSMzPHFZHoyYfjBlhNdsanuMWz/HJr9fYF3+L9krH+3s6gluV182bN9HpdEWa1YaEhHD27FmHzjF+/Hhq1apFz549rW7XaDRoNAWzRJmZRVtVlRVh1SrwWtcGzN0Rz8cb4+jaOBg/lcJyJ6UKmvQ2LHl34PwfBuf+31sh7QLsmWlYgpsbFFmL56HqQy6T2RY6vYSPouj64tODXPuClfds5ozf42xukyTr96NkysvSYtXpJZSFvPMeobvy7kL8NsMze24L5N8hAEAGl+R1Ces6xPDcmj2zvkrDbK0IlTBjxowZrFy5kl27duHr62t1n+nTpxMVFVVuMo3u1pA1R69xLf0O83fG825EE9s7+/hB82cNiybL8DCcWg3x2yHlDOw4Azs+sfgVIyi0XL6HvRkve8e4ftjo0tNbcDE1mz/OJNuWxUYEV2lnG8H6fXab5aXLg4u7DM/m2U2gMTMAqjTgUIXufBjfmBZtH+HLrm2LHK52UfsztyqvatWqoVAoSE62fECSk5OpUaOG3WNnzZrFjBkz2L59O61bt7a538SJE4mMjDR9zszMJDTUdQrAT6Xgo6ebM+rHGBbtucgL7etQv5oDQavqitD6/wzLnduGh+TUari4G24cMyzbPkJXO5wN+s7kNXmGf3ZtX+q4H1tH27KgivN5udpf/9fFW2w9ncTIx+u7PObp2z8T7CtLycawsZQNOMD6fS7XuRC9Di7vMzyDZ9YZnkkjgXWg5XOGH9SabVmy/Ch/S0n8X03rYRC+Lqpj71blpVKpaN++PdHR0fTv3x/A5Hy31UEb4D//+Q+ffvopW7dupUOHDnavoVarUavLN1A0okUITzSuzp6/U4nacJolwzo696L5VYZ2/zYs2akQtw5O/QaX96G4doj+HEJ3/WuSTnSkZpeXDJabf9n6/Gw57EvaWaiseP9eS7QbGXeZ3K+5y66TmqVh9dGrdveRTP9nSVkMG63dS5dPVuj1cPWwwR97+jfDxJKRCtUNln/LF6BOuEXhtjM3DJZYMxvKS20qSHgfWV4AkZGRDB06lA4dOhAeHs6cOXPIyclh+PDhAAwZMoTatWszffp0AD7//HMmT57MihUrCAsLIynJMJ4OCAggICDAbd/DHJlMxtR+zYmYs4dd51LZdiaZp1rYtyRtElDdEG7RcSRkXmfP2m8JiF/Pw/J4aqYdgo2HYPO70KC7wdfQtK9hRtNBbL0OJZnxKkmIQElZvC+BCmoFwYG+DH6kXpmff23stWI7gNu6F2U1bCysrFxyeyUJbhw3WFinfzNMIBnxDYLmzxgUVr3HDBNPhcjW5HP5Vi4AzWpWtHoJXxe1P3O78howYACpqalMnjyZpKQk2rZty5YtW0xO/MTERORmWv6bb75Bq9Xyz39aTrlOmTKFqVOnlqfodmlQPYCRjzfgm10XmLbxDE80rm6qKlliAmtx4aEhRJ3pQB1ZCv0Uf/FOrVMoU04ZnKjx20ChgkZPGRRZ494lzrO0NfQpbthYnszdEQ/Ak02Dy6xszsmrGaTlarmdqwWgolpJlibf6r7Svf8VpiSTFnm6og77wre6TC3blLMFmSBpZkUjVQHQ9Ol7WSHdDBNMdjiXZLC6QgLVVA2wPsJRu6jxrNuVF8DYsWNtDhN37dpl8fnSpUuuF6iMeKNHQ9bGXuPq7Tss2HWByF6NS31O4wNwVQrmm/xnaNplMs/WyYVTa+DUr3Dzbzi70bD4+BsUmDHP0qfopIatwayt2NniSuK4g1ytdeVSEvrN22v4b5tagO18PTDONhZdX6IGHFZCJQqfutS3N+3ivedkDaScLliv9C14Thr1MkwkOciZ6wbl1dzGkBFcV03VI5TX/Yq/SsmHfZszZsVRFu6+wAsP16Ze1dJVnChseu88m8KzbdtBt/HQ9X1IPn1vCLAGbl+6579YA+pAszzLbsXmWdq2vGwfU57DRnOKa/pREq7eNgyF5HZyg2wPuZ2/nnXLq6hCc5qMq4bh4Kk1cP1owXq5j+EHreULhrAdtfUhX3GcuZEF2PZ3gevq2Avl5WL+0aoGjzasyr74W0zbcIbvh3Us1fmMTs/WdSpx4moGu/9ORaeXUMhlBjOhRkvD8uRkw8Nq/KXNug7HfzIsfpUNTv4Wz4O+qdXr2FJE9uO8SvXVSky2jWFdaTB+TVtlb0z7WVlX2tZnYH3m1uH7m51imCE8tRoSzYK9ZXKo39WgsJo9bXgOSonRWd/cTsK1r1JYXl6JTCYj6pkW9J7zJ9FnU4iOS+bJZiHFH2gDo+XV+aGqJNzM4XZuHseupNO+XqEHUSaD2u0NS6+PDTllp1bDmbWQkwoxSyFmKR18qzNF+TAbdY9wVGqEdC9jzJbTuaSFCl3J03P3cmlG31Kdo4iVc++/9maJJamoXwpK2vqs6Gxj4YGj3WH5ndvcObGWuG1LaJt/Ejlm56vbxeADbd7fMAFURuj0ksnnZc/yMs023m8O+weBhsEVefmx+vx3z0WiNpzh0YbVSuy8N5YerqBS8kTj6mw6cYOdZ1OKKi9z5HKo19mw9J5hyDk7tRrOrEd1N5Xhyq0MV27lmlSVjbpH2KDrjE7/hNVTFRek6i6i45JpHFKR0Cr+Th+r10v8338PWK2vrrAzIrU921iSBhxFFVVxDnv9nUzyz25GFfcbxEfjp8/jYePG2u0NlnWL56BSbaflcYSEmznczdPj6yMnzI47xNdFQapuT8x+UHjjyUaEBKpJTMtl0Z6LJT6P8QHw9ZHTo0kwADvPOdGeTaE0+LyemQvvnieu+3es1j1GluRHbdktXlNuYqP6Q0J/fBSiP4bkMxaH2xs2ujN95eVlR3j8PztLdOy19DvEXL7Nnr9TTeuM9lbxw0Yr6UElasBR1OdV5FoSSNpcw5Dw5yHkff4QqnWvwd9bQJ9Hqn9D/pM3gMc1X8IrO6DLWJcpLoC4e0PGpjUCDW4LGxiDVMu68aywvMqJALWSD/o2582fYpm/M57n2tUukZVgfAB8fRR0bVIdmQxOX88kOfMuIYHWU6RsolRxu0533snzR42WbvJj9FMc4El5LH6Zl+HPWYalejNTnmVxidneiL0Xz/6w0fr6sojzMp/J9CGfx+UnOP7VAipzhACZoRGJGrior4G+5Qs07DaEpbGwYGfRXpmuorjgVCPGUYawvLyYfq1r0rlBVTT5ej7eeKb4A6xgsryUCqoFqGldJwig5M1x770gGlRs1YczNu8t2msWcqXHXGjyD8OsVGoc7PwU5rXny/Q3eVWxgdqkFjmVx1U9cBCr0ez3/mtPsUnY8nmVQZxXfh5c3MkM5SIOq19nsWoWT8v2EiC7y1WpGnfC36Cv5jN6aL8goeVbENy0RL620hDngLMeDDXsoeyDVIXyKkdkMhlRz7ZAKZfxx5lkdjkz3LuH8QEwJruWaOhYDLn4kt7gWRj4E7wXD88ugIeeBJmChroLTPL5iX2+b7FaNYVhii1Ux5D35gm6qySK439/Xba5za7yspHbWLJKqhIy9HSQnSVKuYQ6Sx7Gb+UL/Eu5iyBZDilSEEvyI3heM5XHNF+R2mkSp6UwQGbyy5X3j4dJedmIrDdidNiL2UYvp3FIRYZ1CeO7vQlMXX+areOqmiKQHcGkvO4d06NpMF9u/5u952+iydc5dS7AZpSqyW/jFwTtBhmWnJt8NfcLwnN20UkeR3v5edrLzzNZ+T8O6puRktKXdTzEbdzXBm7m1nNM6GM9/KMwmXfzGLHkMEcu37a5j90gVRvrHVGgOZp8Tl3LoEO9yiiSjvHa3SV0U/9JLVmaYYc7oFUF8UtuezboO3NI3xS9ma1hzAAwyGgQsjzj7G5la0jO1CCTQZMaxQ0bDXKLOK/7gLd6NmLd8etcupXLd38mMKZ7Q4ePNcZ5GR+IFrUCqRag5ma2hsMJt3msUbUykdHqC1ihGhtVvfky/TGCuU1fxV88rfiL9vLzdFacgatn6KuWs0/fkg36zvyh60Am5dsGbuHuC1y5nctXA9qitDdVCPSbu9eUl1cEe3XZTbtYD7EvVnlJEh8v/pU61zbTvGIMFe9c5SUAGWRKfvyh78gjz7zC1twmfPx7vNVTpJkpL+OkQnmGqsTdC06tV8WfgGK6YBf4vMSw0eup6OvDB/8wVEqdu+M819LvOHys8QEwPhByuYzuTQyxOztK6veyQnFNIFKozBJdH17QRvGY5ium5w3kqm8jlDI9XRUnmOXzXw6rX2eRzxc8I9+PP+XX7XrTiRtsOll8c1ubigszn5c95YWtIFXr966+7Aa3f/8EFjzCjOTXGKtcR8U7V0Hpx1ZZF17VjqOj5hvezRtFVp1uJNzWWj0PwO2cgm3Gf5PynDA5cyMDKN7fBWY+rzKuKiGUl5t4tm0twutX4W6enk8cdN5r8/Vk3MkDCh4IMAwdgRL50Arjd08pOlNV4qpUnf/q+jG97iK6a77gi7x/cl5fG7Usn6cUMXytmkeMehTzfL4mQn4INbZfyrIiPiW7TM5jNz3IkdnG9ET6ZPzMBtUkdqrfofLBmZB6Fo2k5A9de6b7vwfvxfOe9DZ/6DuiwZAIrZcku8o1zUx5GWcqXeXzytXm8+4vxxn/6wnT0PTgRcPwtlkxQ0Yo+KHV5uvLdGgrho1uQiaTMe3ZFvT9ei+/n0riz/OpPN7IfvTz1A2nuZmtpaJaSYPqBeV/Hm1UDaVcxsWbOVy6mUOYI8UPjXIUcnr5qRTcydOVqAGHNl9PglSTubrnmat7jiayKzyt+It+8gOEyZN5+t4wM0vy4w99ezboOrNP34o8FzyGG45f552n7FSxdRB7bc+wMdtYQXsLDv7XEAh85SDvyAAZ5EuGIXXX50fRcZUPmVSgqaoiE9UBRZz8er0hCBSgTZ1KHL+aYbHd3OdlnKk0n210pEGtI/vk6fSMWX6UnecMs8uDO9ejkp8PO+79UPZpVXypJ/OAbK1Oj6+8lNVV7iEsLzfStEYgQzobalFNWX/abv2o//11mRUHE5HJ4KuBbankV5BYHejrQ8cwQzHC0g4dTZZXCRKzLYvNyTgn1eWL/Bfppp1NP80n/De/L9ekqlSU3eEFxV6WqmZyWP0605Xf0kV+yjKlpZRcsmO1gME35gj2fV4FQbtBZDFQEc0Kn0/44uoA+P39e22+ZBzQNWdS3suEaxYwNG8Cd1v+y+QLNJ7faD0ZleXdfB3X77kTGocUnc1Ly8kz/Z137x/F/AenuBnP+TvjaTttGzGX0+x8P4mJa06aFBfA9fQ7/HDgEpIEjzeqRsPg4hO6fc1GCWXp9xLKy82M69WYagFqLqbm8P3eBKv7/HXxFlHrDSVM3otoQo+mRXMjjUNHZ0MmCkeIm7oblyC30XbumoyTUgOm5w/iMc1XvKCZwtL8p0iVKhEky2GgcicrVJ9xUD2GqcqldJCdRVYGimxtrO1enjN+d6zBi71ho0yTyUPXN7DE53MOq0cz3ed7uijOIEeCOh0NqViRcQzM+5AVuidJuzcLe+lWTpFz5d2LsFfde9Ev3cxBL0EFlcJqnSxzn1fevR8N8x+c4vxfa2OvkXEnjw/Xnra57xd//M2vMVeRy8DnXjOQ+NRsVh42FCwc/miY3WsYUSrkppCTsgxUFcrLzQT6+jDx3tT+3B3nuZFh6by/kpbL6OVHyddLPNOmFq93td5JqPs95XXwYho5TlRZuHbb8nrGbke2hof20oO0DkRJSsiJkZowNX8YnTTzGaj9gBX53bktBVBdlsEw5R/8qp7GPvWbTFIup5XsIraDEuzz9qpjVtfbi+sqTKEGPviioa/8Lxb6fEmDZe14/PRHdFccx0em47S+HjPy/sWbwctg5HZ45HUIrFnknBdSCpRXlibPovCgMdTl4r0hY72qFayGa2w5nWT625haZG55FQ56NeeOVseFVINPMO5GJqsOF+1i/b8Dl5i30zDT+dlzrRj8SBgAi/deIutuPmFV/enWONjmNQrj64JAVaG8PIDnH65Nh3qVydXq+GRTQautHE0+r/xwhLQcLS1rB/L5C61t+igeql6B0Cp+aHV69sXfdPjasVfSTX8r5TKqVDD8yt/IsD47aO8HPTYx3fZGa+dCzgF9Cyblv0JHzQKGad9nte5xsiQ/asnSeFW5iQ3qD9mtGse7ylU0kSXirCLrOXu3xedb2Ro+WnvK4ePlMhkq8ugpj+ErH8PEw3zV1/RWHEau05DuX58v817gSc1M+mqns1D3DEkK+1VDjIoD4HZOnoWiMVpeF+/tU79aBYIr2u/BoL035HTU8jqXnGXx7zjrj3OmiSCALaduMPmepT+uZ2P+FV6XWkGG1LOb2YY2gkO7hNm1Sgvj64I69kJ5eQDGyHu5zDDNvy/+Jnq9xLu/HOdsUhbVAtQsGtyhaA/IQucoiLYvmrpjC3OFo1LKaVXbMLQ5cSXD6v6umtHKR8kufVveyXudDppveE07jo26R7gjqagnT2Gsch1b1RP4Q/U+byrWUF9WfCgEGGYdzet9TV532s7eBcj1+TwhP87ojC84on6d71Rf8KxiPxVkGq7oq7Mg/xkS/rmVNZ3X8JXuBS5IBQnQxQ3ZzJVXtiafO9oCa8Q4i2ycTawWoGJgeF3TsM0axnZz+Q76vIzVTzs3qErD4ADScrR8HX0egEMJaby58hiSBAPD6/Lmk4YYRPMy2xVUCv7Zvo7d71gYV8R6idlGD6FFrUr8+5F6/HDgMlPWn6ZPyxr8fioJlULOfwc/7FCN9u5Ng1l24DK7zqU4NJOk00umFA8wKC9jruTxq+lWjymPUCJDnmVHtuo74s9dnpQfpZ/iAF3lx2ksv0ak/Fci+ZVT+jA26h5ho74zVyXbM7W7zqXwdOtapOVo7cZ/ydHTUXaOfor99Lt9mEqqTLgLyCBJqmwqF3RcegiQ0b1qc6T0W0XOE3P5NqsOJzKgY12rw2xz5QWQklXQFNloeeVqC+L5fH0UzB3YjlE/HsUaRsvNfMLHXnNeY4xW69BKdHmoGkMXH2LZ/ku0rB3IlHWGiaNezUP4+NkWpmeoZqWCpP//6xBKRV/7lXgLo3ZBEw6PUF7z589n5syZJCUl0aZNG+bOnUt4eLjN/X/55Rc++ugjLl26RKNGjfj888/5xz/+UY4Su4Z3ejVh04kbxKdkm5pLfNK/Je3rOdbW7JEGVfH1kXMj4y71J24mQK2koq+SQF8fAv0M/63k50Ognw+BvsoiM4oqhZw295TX2aQsMnLzkMuxeFBd3n6rELn4skHfhQ36LgSSQy95DP0UB3hMfpKW8ku0lF9iAis5qm/IBl1nNukeIQXL2mZjV8TSo2kwL337l5UrSLSVXaCf4gB9FX9RQ3bbuJpbUkX2qR7jx+wOHJaamAo1mo6UbN+P8atP0iikotU4KHOfF0BKVsEQXXUvK8BojRnzAhV2Sl0fSkhDLpNZKMWNJ67zUHAAVSuoqFJBRbUAtcn6Ma8737VxdXo2C2Z7XArjVh0HoH29yswd2M4iQyG0ij8ymeE7D+0SZlMWWzSrEUigr0/pm9CY4XbltWrVKiIjI1m4cCGdOnVizpw5REREcO7cOYKDizoE9+/fz8CBA5k+fTpPP/00K1asoH///hw9epSWLVu64RuUHZX8fRjfu6mpN+GwLmG82NHxBrm+Pgp6NA1m80mDMzdbk0+2Jt+m/6owah85Ncx+YdtM+wOAuvdSQM6YWWnuIJMKrNY/wWr9E1Qmk96Kw/STH+AReRwPy+N5WB7PR8ofOSQ1ZYOuM7/rwk0zfM0nbzU7k0Rz2WVTDFqovGCYnSn5s0XXkQ36zuzXtyAsMJALWUVnBw1nkdh/oajlZWR//E0qqIq+YncKWR+fbymY+TSGThgd9sZhpNLOsHF7XArb4yxnmc19p0b8VQqqVFCRdO95aFGrEgBT+rXgevpd0nO1NK8VyKz/a1NEyVQLUPP5C61RK+WONVEuxPxBDxe/k5PIpPL+KS1Ep06d6NixI/PmzQMMTWdDQ0N54403mDBhQpH9BwwYQE5ODhs3bjSte+SRR2jbti0LFy4s9nqZmZlUqlSJjIwMAgPdl0BsC71eYvL6U+glmPZMi2Lz8wqTnqvlYEIa9ar6o1YqyL6bT+bdPDLv5JFxJ+/e3wXr7ubpuZuv41a2lufa1WbEY/UJm7DJRd/ONVTnNn0VB3la8Rcd5H+b1udLcvbrW7BB35mtuo5Ul6XTT3GAfvIDPCQvGD7mSGq23wua3aNvjZYCS1OllNuMv2tQvQIXUw1KplP9KhxMsIyZqhag4ma2c9kEfVvXZNOJAtn+O7g9ES1qcPlWDt1n7bIYtgeolfirFHQIq4yPQo6PQk7cjUzqVfUn624+aTlabmVrScvRFpkJrhag5uCkJ+1WzXAXjr6jblVeWq0Wf39/fv31V1PHbIChQ4eSnp7OunXrihxTt25dIiMjefvtt03rpkyZwtq1azl+/HiR/TUaDRpNgU8hMzOT0NBQj1VensDmkzcYvfwo1SuqiX6nK9FxyayOuUaO1uBcPpuUZdq3QfUK9G9bm0BfJbl5OlIyNSzdfwmA59vVZo1ZrFWj4ABu5+ahzdeRebfsm2YA1CaVvoq/6Kc4QCv5JdP6fEmOUlbwAmskH3bo27JB15kd+nbcxfqMnrXo9sI81TyE9yKa0OvLPQ7J+GKHOtQK8qN7k2Ci45JJzdby06FEWtWuxJrRXTh2JZ303DyqBqh4uG7BEDgp4y43szW88sMR+rWpxaR7+bHFIUkSWZp80rK13MrRcCtbS9MagdSt6nwxzPLAK5TX9evXqV27Nvv376dz586m9e+//z67d+/m4MGDRY5RqVQsW7aMgQMHmtYtWLCAqKgokpOTi+w/depUoqKiiqwXyqt47Dn9r6ffoUagr9Xpcr1e4sDFW3QMq4JWp2fXuRR6NA3G32wIJUkSt+5ZBnk6PUH+PqTnGmKealf2Y+vpJL6OPk9oZX9TyZo2oUEcNwvtKI4w2Q2elhsUWRP5VfIkBX/qW7FB15lt+vZkY3h561X1N+URVvRVkmWmWPeO787nW85x4MItIlqEEBLoy+FLafj6KGhfrzKV/Hzo37Y2fioFWXfzWHnoCm3rBpF9N5/T1zMI8ldRQa0g4WYu2nw9rz7RgCoVijZyTcm8i79aWWyFBuO9K24yxptxVHm53eflaiZOnEhkZKTps9HyEhSPvRfE3uynXC7j0YaG0jwqpZynW9eyeu5qAWqqmUWP1zHzsw/qVI9Bneo5JKfxZZYkiXy9xK1sLTKZwVJRKv4PtUrJqeQLpGrVdGjWgHY6iU+UctRKuWlYbvwNt/ad5w5s55AcFX19eOWJBqbPxsBhRwh2ooT3/ay4nMGtyqtatWooFIoiFlNycjI1alhP+KxRo4ZT+6vVatRq+0F+Au/G+DLLZDJ8FDLTpINFTf9qrR06h8B7cGuQqkqlon379kRHR5vW6fV6oqOjLYaR5nTu3Nlif4Bt27bZ3F8gENyfuH3YGBkZydChQ+nQoQPh4eHMmTOHnJwchg8fDsCQIUOoXbs206dPB+Ctt96ia9eufPHFF/Tt25eVK1dy5MgRFi1a5M6vIRAIyhm3K68BAwaQmprK5MmTSUpKom3btmzZsoWQEEN+WGJiInKzAL0uXbqwYsUKPvzwQyZNmkSjRo1Yu3at18d4CQQC53B7nFd54+lxXgLBg46j76hIzBYIBF6JUF4CgcArEcpLIBB4JW532Jc3RhdfZqZ7k4wFAoF1jO9mce74B055ZWUZ8vJElL1A4NlkZWVRqVIlm9sfuNlGvV7P9evXqVixotdGVRtTnK5cuSJmTB1E3DPncdc9kySJrKwsatWqZREmVZgHzvKSy+XUqeNcCVtPJTAwULyITiLumfO4457Zs7iMCIe9QCDwSoTyEggEXolQXl6IWq1mypQpolqGE4h75jyefs8eOIe9QCC4PxCWl0Ag8EqE8hIIBF6JUF4CgcArEcpLIBB4JUJ5eQFpaWkMGjSIwMBAgoKCePnll8nOzrZ7zKJFi+jWrRuBgYHIZDLS09PLR1g3Mn/+fMLCwvD19aVTp04cOnTI7v6//PILTZs2xdfXl1atWrF58+ZyktRzcOaenT59mhdeeIGwsDBkMhlz5swpP0GtIJSXFzBo0CBOnz7Ntm3b2LhxI3v27OHVV1+1e0xubi69e/dm0qRJ5SSlezF2Xp8yZQpHjx6lTZs2REREkJKSYnV/Y+f1l19+mdjYWPr370///v05depUOUvuPpy9Z7m5uTRo0IAZM2bYbHhTrkgCj+bMmTMSIB0+fNi07vfff5dkMpl07dq1Yo/fuXOnBEi3b992oZTuJzw8XBozZozps06nk2rVqiVNnz7d6v4vvvii1LdvX4t1nTp1kl577TWXyulJOHvPzKlXr5705ZdfulC64hGWl4dz4MABgoKC6NChg2ldz549kcvlVpvyPohotVpiYmLo2bOnaZ1cLqdnz54cOHDA6jEHDhyw2B8gIiLC5v73GyW5Z56GUF4eTlJSEsHBls1LlUolVapUISkpyU1SeRY3b95Ep9OZmrYYCQkJsXmPkpKSnNr/fqMk98zTEMrLTUyYMAGZTGZ3OXv2rLvFFAg8lgeuJI6n8M477zBs2DC7+zRo0IAaNWoUcaDm5+eTlpbmGU5TD6A8Oq/fb5TknnkawvJyE9WrV6dp06Z2F5VKRefOnUlPTycmJsZ07I4dO9Dr9XTq1MmN38BzEJ3Xnack98zjcOt0gcAhevfuLbVr1046ePCgtHfvXqlRo0bSwIEDTduvXr0qNWnSRDp48KBp3Y0bN6TY2Fjp22+/lQBpz549UmxsrHTr1i13fAWXs3LlSkmtVktLly6Vzpw5I7366qtSUFCQlJSUJEmSJA0ePFiaMGGCaf99+/ZJSqVSmjVrlhQXFydNmTJF8vHxkU6ePOmur1DuOHvPNBqNFBsbK8XGxko1a9aU3n33XSk2NlY6f/68W+QXyssLuHXrljRw4EApICBACgwMlIYPHy5lZWWZtickJEiAtHPnTtO6KVOmSECRZcmSJeX/BcqJuXPnSnXr1pVUKpUUHh4u/fXXX6ZtXbt2lYYOHWqx/88//yw1btxYUqlUUosWLaRNmzaVs8Tux5l7ZnzOCi9du3Ytf8ElSRIlcQQCgVcifF4CgcArEcpLIBB4JUJ5CQQCr0QoL4FA4JUI5SUQCLwSobwEAoFXIpSXQCDwSoTyEggEXolQXgKPYNiwYVYra/Tu3dvdogk8FFFVQuAx9O7dmyVLlliss9WtOS8vDx8fH4t1Wq0WlUrl9HVLepzAvQjLS+AxqNVqatSoYbFUrlwZAJlMxjfffMMzzzxDhQoV+PTTT5k6dSpt27blu+++o379+vj6+gKQmJjIs88+S0BAAIGBgbz44osWpV9sHSfwLoTyEngNU6dO5bnnnuPkyZOMGDECgPj4eFavXs2aNWs4duwYer2eZ599lrS0NHbv3s22bdu4ePEiAwYMsDhX4eME3ocYNgo8ho0bNxIQEGCxbtKkSaYOSC+99BLDhw+32K7Vavnhhx+oXr06YKjJdfLkSRISEggNDQXghx9+oEWLFhw+fJiOHTtaPU7gfQjlJfAYunfvzjfffGOxrkqVKqa/zZuQGKlXr56FAoqLiyM0NNSkuACaN29OUFAQcXFxJuVV+DiB9yGUl8BjqFChAg0bNrS73ZF1jl5L4N0In5fgvqJZs2ZcuXKFK1eumNadOXOG9PR0mjdv7kbJBGWNsLwEHoNGoynSdkupVFKtWjWHz9GzZ09atWrFoEGDmDNnDvn5+YwePZquXbtaHXYKvBdheQk8hi1btlCzZk2L5bHHHnPqHDKZjHXr1lG5cmWeeOIJevbsSYMGDVi1apWLpBa4C1EGWiAQeCXC8hIIBF6JUF4CgcArEcpLIBB4JUJ5CQQCr0QoL4FA4JUI5SUQCLwSobwEAoFXIpSXQCDwSoTyEggEXolQXgKBwCsRyksgEHglQnkJBAKv5P8BD7vzBHOaM1cAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.022376, + "end_time": "2024-02-29T21:22:46.622308", + "exception": false, + "start_time": "2024-02-29T21:22:46.599932", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4441.699721, + "end_time": "2024-02-29T21:22:49.368713", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/realtabformer/4/mlu-eval.ipynb", + "output_path": "eval/treatment/realtabformer/4/mlu-eval.ipynb", + "parameters": { + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "path": "eval/treatment/realtabformer/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "realtabformer" + }, + "start_time": "2024-02-29T20:08:47.668992", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/realtabformer/model.pt b/treatment/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..7cc44fd98226a95fa3af7f0a4ad27a55b0fd274d --- /dev/null +++ b/treatment/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2282dc82cbf2866b884ef9a8d23f69514e620c68a3aeb53ae9042021a38d04d1 +size 78481207 diff --git a/treatment/realtabformer/params.json b/treatment/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..c9c7cc261fc8a3ddc58b27ee45dede26e7d1eef7 --- /dev/null +++ b/treatment/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "ALL", "synth_data": 2, "dataset_size": 2048, "batch_size": 2, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "fixed_role_model": "realtabformer", "mse_mag": true, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/tab_ddpm_concat/eval.csv b/treatment/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..51d6f20286d6723f3e989cda11670f093a98a471 --- /dev/null +++ b/treatment/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,0.0,0.00014312503468425018,0.0016991006075357902,4.4409027099609375,0.02741244249045849,0.4385460913181305,0.03964341804385185,4.390428784972755e-06,2.2897465229034424,0.029980136081576347,0.05906405299901962,0.041220150887966156,0.06956713646650314,0.012020673602819443,6.73064923286438 diff --git a/treatment/tab_ddpm_concat/history.csv b/treatment/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..637db8270df01cfad3eac5a0a7f5b8a55c0cbda3 --- /dev/null +++ b/treatment/tab_ddpm_concat/history.csv @@ -0,0 +1,21 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.3250283277593553,104.9569591542182,0.14964723405428232,0.5211862199008465,0.0,0.0,0.0,0.0,0.3362019395455718,320,80,146.38697052001953,1.8298371315002442,0.45745928287506105,0.02428697562621096,0.18027530759572982,31.412489253808427,0.045921629713848236,0.3813539147377014,0.0,0.0,0.0,0.0,0.18535085618495942,80,20,23.593388080596924,1.179669404029846,0.2949173510074615,0.09171277673522127 +1,0.0653022439641063,3.1109383892979623,0.00567231331072886,0.17495179391116836,0.0,0.0,0.0,0.0,0.06682286564828246,320,80,146.9430272579193,1.8367878407239915,0.45919696018099787,0.22626246082063517,0.0061504256300395355,3.498529246501312,3.916562487162878e-05,0.032489415258169174,0.0,0.0,0.0,0.0,0.006242727432982065,80,20,23.766483545303345,1.1883241772651671,0.2970810443162918,0.03535210953559727 +2,0.007219767475180561,1.1950939540983199,7.097379191712827e-05,0.03277589186327532,0.0,0.0,0.0,0.0,0.007339380298071773,320,80,146.65993809700012,1.8332492262125015,0.4583123065531254,0.17631570195662788,0.007198175770463422,5.956227582395968,0.00011806738390589544,0.022729567158967255,0.0,0.0,0.0,0.0,0.0073918548005167395,80,20,23.585861682891846,1.1792930841445923,0.2948232710361481,0.05584214173723012 +3,0.00775521779214614,0.5126722554007301,0.00021560695235042204,0.0332988631009357,0.0,0.0,0.0,0.0,0.007902447441665572,320,80,146.36689710617065,1.8295862138271333,0.4573965534567833,0.18655094002606348,0.003850602741295006,6.929230917982568,9.300656232844599e-06,0.026165280397981404,0.0,0.0,0.0,0.0,0.003912569362728391,80,20,24.091975927352905,1.2045987963676452,0.3011496990919113,0.035354171460494396 +4,0.005576713894515706,0.5243572926787815,7.799909240078506e-05,0.026431062967458275,0.0,0.0,0.0,0.0,0.0056679701934626795,320,80,147.8123037815094,1.8476537972688676,0.4619134493172169,0.17163944796193392,0.004904342864756472,6.613062943945624,4.787976637761204e-05,0.02067490922054276,0.0,0.0,0.0,0.0,0.0050057682892656885,80,20,24.22531223297119,1.2112656116485596,0.3028164029121399,0.05586188876768574 +5,0.005746945142709592,0.3063628292962107,8.83000647067575e-05,0.020610145634418587,0.0,0.0,0.0,0.0,0.005859691742443829,320,80,147.5134036540985,1.8439175456762313,0.46097938641905783,0.18795285349478946,0.006080282232142053,5.922124939222977,5.200857143190518e-05,0.021447001822525635,0.0,0.0,0.0,0.0,0.006272930890554562,80,20,24.122392892837524,1.2061196446418763,0.30152991116046907,0.03159960038028657 +6,0.005959707218971743,0.11086495585541982,8.815349536440697e-05,0.025197071458387656,0.0,0.0,0.0,0.0,0.006072522566682892,320,80,145.82239770889282,1.8227799713611603,0.4556949928402901,0.19075714307837188,0.004875349982467014,6.456333731907944,3.336970914707971e-05,0.026122358883731066,0.0,0.0,0.0,0.0,0.004957216732145752,80,20,23.773096084594727,1.1886548042297362,0.29716370105743406,0.04347092452226207 +7,0.005319609342222975,0.3930861216696401,6.20743445605716e-05,0.024280186876421795,0.0,0.0,0.0,0.0,0.0054104587794427065,320,80,145.5336833000183,1.8191710412502289,0.4547927603125572,0.18721119775436817,0.004346008354877995,5.974730668690517,1.5303319651938097e-05,0.02063523129618261,0.0,0.0,0.0,0.0,0.004420949433551868,80,20,23.65921902656555,1.1829609513282775,0.2957402378320694,0.03134176780004054 +8,0.005300425899258698,0.1536639502475623,9.251545014040408e-05,0.021025176101829857,0.0,0.0,0.0,0.0,0.005399322827724973,320,80,145.47741270065308,1.8184676587581634,0.45461691468954085,0.18479676478891632,0.003750909139489522,5.9032170891565325,7.884279494407842e-06,0.019338129594689234,0.0,0.0,0.0,0.0,0.003807382771628909,80,20,23.828125,1.19140625,0.2978515625,0.033805052132811396 +9,0.004899124511211994,0.24653815754437575,5.619041067117326e-05,0.0216044840090035,0.0,0.0,0.0,0.0,0.004984687315391057,320,80,145.63873076438904,1.820484134554863,0.45512103363871576,0.19259733236394821,0.0059519269168959,3.8484517917624546,7.430727260597792e-05,0.019101872900500895,0.0,0.0,0.0,0.0,0.0060627352468145546,80,20,23.49578022956848,1.174789011478424,0.293697252869606,0.03844869087915868 +10,0.00507157752654166,0.12483040868133913,8.54523543355759e-05,0.021162668641045455,0.0,0.0,0.0,0.0,0.005164033845721861,320,80,145.65456414222717,1.8206820517778397,0.4551705129444599,0.19211700824089348,0.00464706384591409,6.332309863651451,2.2397835272180976e-05,0.02283783564926125,0.0,0.0,0.0,0.0,0.004722811375540914,80,20,23.64812207221985,1.1824061036109925,0.29560152590274813,0.04854622206185013 +11,0.004837600045084401,0.3505868666707556,5.091351759973674e-05,0.022111102837880026,0.0,0.0,0.0,0.0,0.00492111863768514,320,80,146.0148582458496,1.82518572807312,0.45629643201828,0.17585730996215715,0.004068510180877638,5.871831947068415,1.4263169245479212e-05,0.01976338555687107,0.0,0.0,0.0,0.0,0.004133973495845567,80,20,23.57652258872986,1.178826129436493,0.29470653235912325,0.03373235072940588 +12,0.005061780314099451,0.12903543715087834,8.679314863810369e-05,0.020122559062292565,0.0,0.0,0.0,0.0,0.005155922670746804,320,80,146.0060043334961,1.825075054168701,0.45626876354217527,0.19230934586375953,0.005099401987536112,4.219143859660525,4.85376546165095e-05,0.016910381577326918,0.0,0.0,0.0,0.0,0.005196944052295293,80,20,23.720768213272095,1.1860384106636048,0.2965096026659012,0.038260015891864896 +13,0.004984028046055755,0.15418263155418116,6.304440869404939e-05,0.020150290539095294,0.0,0.0,0.0,0.0,0.005073844128310157,320,80,146.04027271270752,1.825503408908844,0.456375852227211,0.1910679372958839,0.00501167065667687,5.14628453936366,3.0035914827664102e-05,0.01827869672415545,0.0,0.0,0.0,0.0,0.005116376957448665,80,20,23.532910346984863,1.1766455173492432,0.2941613793373108,0.03470718542812392 +14,0.004734398943764972,0.6279221150001397,4.761278923632872e-05,0.017200588568812236,0.0,0.0,0.0,0.0,0.004827282899350393,320,80,145.9920961856842,1.8249012023210525,0.4562253005802631,0.17317850722465664,0.007601118210004643,3.2803518016677558,0.00017301160700378612,0.012077987159136683,0.0,0.0,0.0,0.0,0.00790572552505182,80,20,23.845383882522583,1.1922691941261292,0.2980672985315323,0.03440641919150948 +15,0.0045964237675889304,0.569245081065904,5.5843747120097085e-05,0.014247680191328983,0.0,0.0,0.0,0.0,0.004697687765110458,320,80,146.36437106132507,1.8295546382665635,0.4573886595666409,0.18270513406023384,0.007547395752771991,2.9282909621120212,0.00017729704157093407,0.012870692153228447,0.0,0.0,0.0,0.0,0.007793798694547149,80,20,23.560658931732178,1.1780329465866088,0.2945082366466522,0.0382234871853143 +16,0.0046578587127442,0.5405012748799436,8.542011439159336e-05,0.016841917944839226,0.0,0.0,0.0,0.0,0.00475033148932198,320,80,146.28253817558289,1.828531727194786,0.4571329317986965,0.18576831596437843,0.008764273906126618,3.272361376248591,0.0002117462956683802,0.011796211288310588,0.0,0.0,0.0,0.0,0.009110148320905865,80,20,23.770928859710693,1.1885464429855346,0.29713661074638364,0.0332530245417729 +17,0.004992562824554625,0.2701526861071898,6.218077271888996e-05,0.01973752847407013,0.0,0.0,0.0,0.0,0.005098153510880365,320,80,146.03065609931946,1.8253832012414932,0.4563458003103733,0.19569849580875598,0.005505499814171344,5.217980894941865,5.959937391786241e-05,0.02082536361485836,0.0,0.0,0.0,0.0,0.005607066205629963,80,20,23.852116107940674,1.1926058053970336,0.2981514513492584,0.038161135837435724 +18,0.004348099110757175,0.20937616627059583,3.1978708130590886e-05,0.015712628203618805,0.0,0.0,0.0,0.0,0.004428885674678895,320,80,146.06442213058472,1.825805276632309,0.45645131915807724,0.18260755122173578,0.00345534470397979,3.8281147816255725,4.558243014707742e-06,0.013488705200143159,0.0,0.0,0.0,0.0,0.0035211247907682265,80,20,23.61065983772278,1.180532991886139,0.29513324797153473,0.044103194237686695 +19,0.0045102617631528116,0.175469460278498,5.030859646430498e-05,0.00875355452990334,0.0,0.0,0.0,0.0,0.004684712074777053,320,80,145.9878635406494,1.8248482942581177,0.4562120735645294,0.1969025250407867,0.0036117414470936637,3.6396715161072066,1.5647155543276753e-05,0.01432776392903179,0.0,0.0,0.0,0.0,0.003690452836053737,80,20,23.642552614212036,1.182127630710602,0.2955319076776505,0.04286287054419517 diff --git a/treatment/tab_ddpm_concat/mlu-eval.ipynb b/treatment/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..1c90fe82592057cbbb92c8c68d503ff8ae64a380 --- /dev/null +++ b/treatment/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2642 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:17.478621Z", + "iopub.status.busy": "2024-02-29T02:42:17.478107Z", + "iopub.status.idle": "2024-02-29T02:42:17.512878Z", + "shell.execute_reply": "2024-02-29T02:42:17.511936Z" + }, + "papermill": { + "duration": 0.050564, + "end_time": "2024-02-29T02:42:17.515134", + "exception": false, + "start_time": "2024-02-29T02:42:17.464570", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:17.542515Z", + "iopub.status.busy": "2024-02-29T02:42:17.542112Z", + "iopub.status.idle": "2024-02-29T02:42:17.548690Z", + "shell.execute_reply": "2024-02-29T02:42:17.547897Z" + }, + "papermill": { + "duration": 0.023202, + "end_time": "2024-02-29T02:42:17.550948", + "exception": false, + "start_time": "2024-02-29T02:42:17.527746", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:17.575203Z", + "iopub.status.busy": "2024-02-29T02:42:17.574915Z", + "iopub.status.idle": "2024-02-29T02:42:17.579037Z", + "shell.execute_reply": "2024-02-29T02:42:17.578270Z" + }, + "papermill": { + "duration": 0.018643, + "end_time": "2024-02-29T02:42:17.580908", + "exception": false, + "start_time": "2024-02-29T02:42:17.562265", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:17.604662Z", + "iopub.status.busy": "2024-02-29T02:42:17.604377Z", + "iopub.status.idle": "2024-02-29T02:42:17.608272Z", + "shell.execute_reply": "2024-02-29T02:42:17.607477Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018144, + "end_time": "2024-02-29T02:42:17.610341", + "exception": false, + "start_time": "2024-02-29T02:42:17.592197", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:17.634504Z", + "iopub.status.busy": "2024-02-29T02:42:17.634218Z", + "iopub.status.idle": "2024-02-29T02:42:17.639909Z", + "shell.execute_reply": "2024-02-29T02:42:17.638907Z" + }, + "papermill": { + "duration": 0.020068, + "end_time": "2024-02-29T02:42:17.641836", + "exception": false, + "start_time": "2024-02-29T02:42:17.621768", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "606b214a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:17.667483Z", + "iopub.status.busy": "2024-02-29T02:42:17.667190Z", + "iopub.status.idle": "2024-02-29T02:42:17.672315Z", + "shell.execute_reply": "2024-02-29T02:42:17.671430Z" + }, + "papermill": { + "duration": 0.02038, + "end_time": "2024-02-29T02:42:17.674292", + "exception": false, + "start_time": "2024-02-29T02:42:17.653912", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 1\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/tab_ddpm_concat/1\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011138, + "end_time": "2024-02-29T02:42:17.696740", + "exception": false, + "start_time": "2024-02-29T02:42:17.685602", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:17.720838Z", + "iopub.status.busy": "2024-02-29T02:42:17.720126Z", + "iopub.status.idle": "2024-02-29T02:42:17.729372Z", + "shell.execute_reply": "2024-02-29T02:42:17.728535Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023481, + "end_time": "2024-02-29T02:42:17.731358", + "exception": false, + "start_time": "2024-02-29T02:42:17.707877", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/tab_ddpm_concat/1\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:17.755301Z", + "iopub.status.busy": "2024-02-29T02:42:17.755049Z", + "iopub.status.idle": "2024-02-29T02:42:19.985283Z", + "shell.execute_reply": "2024-02-29T02:42:19.984295Z" + }, + "papermill": { + "duration": 2.244597, + "end_time": "2024-02-29T02:42:19.987342", + "exception": false, + "start_time": "2024-02-29T02:42:17.742745", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:20.015166Z", + "iopub.status.busy": "2024-02-29T02:42:20.014756Z", + "iopub.status.idle": "2024-02-29T02:42:20.029240Z", + "shell.execute_reply": "2024-02-29T02:42:20.028513Z" + }, + "papermill": { + "duration": 0.031259, + "end_time": "2024-02-29T02:42:20.031236", + "exception": false, + "start_time": "2024-02-29T02:42:19.999977", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:20.055313Z", + "iopub.status.busy": "2024-02-29T02:42:20.055050Z", + "iopub.status.idle": "2024-02-29T02:42:20.062600Z", + "shell.execute_reply": "2024-02-29T02:42:20.061676Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021937, + "end_time": "2024-02-29T02:42:20.064623", + "exception": false, + "start_time": "2024-02-29T02:42:20.042686", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:20.089072Z", + "iopub.status.busy": "2024-02-29T02:42:20.088512Z", + "iopub.status.idle": "2024-02-29T02:42:20.198308Z", + "shell.execute_reply": "2024-02-29T02:42:20.196876Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.126734, + "end_time": "2024-02-29T02:42:20.202680", + "exception": false, + "start_time": "2024-02-29T02:42:20.075946", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:20.233519Z", + "iopub.status.busy": "2024-02-29T02:42:20.233188Z", + "iopub.status.idle": "2024-02-29T02:42:25.001513Z", + "shell.execute_reply": "2024-02-29T02:42:25.000740Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.787387, + "end_time": "2024-02-29T02:42:25.003798", + "exception": false, + "start_time": "2024-02-29T02:42:20.216411", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 02:42:22.560931: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 02:42:22.560984: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 02:42:22.562830: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:25.028881Z", + "iopub.status.busy": "2024-02-29T02:42:25.028281Z", + "iopub.status.idle": "2024-02-29T02:42:25.035005Z", + "shell.execute_reply": "2024-02-29T02:42:25.034095Z" + }, + "papermill": { + "duration": 0.021447, + "end_time": "2024-02-29T02:42:25.037167", + "exception": false, + "start_time": "2024-02-29T02:42:25.015720", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:25.063439Z", + "iopub.status.busy": "2024-02-29T02:42:25.063086Z", + "iopub.status.idle": "2024-02-29T02:42:47.304936Z", + "shell.execute_reply": "2024-02-29T02:42:47.303907Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.258062, + "end_time": "2024-02-29T02:42:47.307725", + "exception": false, + "start_time": "2024-02-29T02:42:25.049663", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BEST,\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:47.823092Z", + "iopub.status.busy": "2024-02-29T02:42:47.822825Z", + "iopub.status.idle": "2024-02-29T02:42:47.894259Z", + "shell.execute_reply": "2024-02-29T02:42:47.893329Z" + }, + "papermill": { + "duration": 0.087887, + "end_time": "2024-02-29T02:42:47.896449", + "exception": false, + "start_time": "2024-02-29T02:42:47.808562", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../treatment/_cache/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache4/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache5/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/treatment [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T02:42:47.925952Z", + "iopub.status.busy": "2024-02-29T02:42:47.925602Z", + "iopub.status.idle": "2024-02-29T02:42:48.469988Z", + "shell.execute_reply": "2024-02-29T02:42:48.469052Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.561404, + "end_time": "2024-02-29T02:42:48.472041", + "exception": false, + "start_time": "2024-02-29T02:42:47.910637", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:48.501145Z", + "iopub.status.busy": "2024-02-29T02:42:48.500424Z", + "iopub.status.idle": "2024-02-29T02:42:48.504847Z", + "shell.execute_reply": "2024-02-29T02:42:48.503908Z" + }, + "papermill": { + "duration": 0.021334, + "end_time": "2024-02-29T02:42:48.507004", + "exception": false, + "start_time": "2024-02-29T02:42:48.485670", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:48.537147Z", + "iopub.status.busy": "2024-02-29T02:42:48.536653Z", + "iopub.status.idle": "2024-02-29T02:42:48.544847Z", + "shell.execute_reply": "2024-02-29T02:42:48.543986Z" + }, + "papermill": { + "duration": 0.025663, + "end_time": "2024-02-29T02:42:48.546676", + "exception": false, + "start_time": "2024-02-29T02:42:48.521013", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18616321" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:48.575431Z", + "iopub.status.busy": "2024-02-29T02:42:48.575139Z", + "iopub.status.idle": "2024-02-29T02:42:48.675200Z", + "shell.execute_reply": "2024-02-29T02:42:48.674181Z" + }, + "papermill": { + "duration": 0.116914, + "end_time": "2024-02-29T02:42:48.677379", + "exception": false, + "start_time": "2024-02-29T02:42:48.560465", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 12] --\n", + "├─Adapter: 1-1 [2, 2648, 12] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 13,312\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 12] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,616,321\n", + "Trainable params: 18,616,321\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 73.71\n", + "========================================================================================================================\n", + "Input size (MB): 0.32\n", + "Forward/backward pass size (MB): 1079.48\n", + "Params size (MB): 74.47\n", + "Estimated Total Size (MB): 1154.27\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T02:42:48.711024Z", + "iopub.status.busy": "2024-02-29T02:42:48.710213Z", + "iopub.status.idle": "2024-02-29T03:44:51.349094Z", + "shell.execute_reply": "2024-02-29T03:44:51.348208Z" + }, + "papermill": { + "duration": 3722.657979, + "end_time": "2024-02-29T03:44:51.351551", + "exception": false, + "start_time": "2024-02-29T02:42:48.693572", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.3250283277593553, 'avg_role_model_std_loss': 104.9569591542182, 'avg_role_model_mean_pred_loss': 0.14964723405428232, 'avg_role_model_g_mag_loss': 0.5211862199008465, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.3362019395455718, 'n_size': 320, 'n_batch': 80, 'duration': 146.38697052001953, 'duration_batch': 1.8298371315002442, 'duration_size': 0.45745928287506105, 'avg_pred_std': 0.02428697562621096}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.18027530759572982, 'avg_role_model_std_loss': 31.412489253808427, 'avg_role_model_mean_pred_loss': 0.045921629713848236, 'avg_role_model_g_mag_loss': 0.3813539147377014, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.18535085618495942, 'n_size': 80, 'n_batch': 20, 'duration': 23.593388080596924, 'duration_batch': 1.179669404029846, 'duration_size': 0.2949173510074615, 'avg_pred_std': 0.09171277673522127}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0653022439641063, 'avg_role_model_std_loss': 3.1109383892979623, 'avg_role_model_mean_pred_loss': 0.00567231331072886, 'avg_role_model_g_mag_loss': 0.17495179391116836, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.06682286564828246, 'n_size': 320, 'n_batch': 80, 'duration': 146.9430272579193, 'duration_batch': 1.8367878407239915, 'duration_size': 0.45919696018099787, 'avg_pred_std': 0.22626246082063517}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0061504256300395355, 'avg_role_model_std_loss': 3.498529246501312, 'avg_role_model_mean_pred_loss': 3.916562487162878e-05, 'avg_role_model_g_mag_loss': 0.032489415258169174, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006242727432982065, 'n_size': 80, 'n_batch': 20, 'duration': 23.766483545303345, 'duration_batch': 1.1883241772651671, 'duration_size': 0.2970810443162918, 'avg_pred_std': 0.03535210953559727}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007219767475180561, 'avg_role_model_std_loss': 1.1950939540983199, 'avg_role_model_mean_pred_loss': 7.097379191712827e-05, 'avg_role_model_g_mag_loss': 0.03277589186327532, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007339380298071773, 'n_size': 320, 'n_batch': 80, 'duration': 146.65993809700012, 'duration_batch': 1.8332492262125015, 'duration_size': 0.4583123065531254, 'avg_pred_std': 0.17631570195662788}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007198175770463422, 'avg_role_model_std_loss': 5.956227582395968, 'avg_role_model_mean_pred_loss': 0.00011806738390589544, 'avg_role_model_g_mag_loss': 0.022729567158967255, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0073918548005167395, 'n_size': 80, 'n_batch': 20, 'duration': 23.585861682891846, 'duration_batch': 1.1792930841445923, 'duration_size': 0.2948232710361481, 'avg_pred_std': 0.05584214173723012}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00775521779214614, 'avg_role_model_std_loss': 0.5126722554007301, 'avg_role_model_mean_pred_loss': 0.00021560695235042204, 'avg_role_model_g_mag_loss': 0.0332988631009357, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007902447441665572, 'n_size': 320, 'n_batch': 80, 'duration': 146.36689710617065, 'duration_batch': 1.8295862138271333, 'duration_size': 0.4573965534567833, 'avg_pred_std': 0.18655094002606348}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003850602741295006, 'avg_role_model_std_loss': 6.929230917982568, 'avg_role_model_mean_pred_loss': 9.300656232844599e-06, 'avg_role_model_g_mag_loss': 0.026165280397981404, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003912569362728391, 'n_size': 80, 'n_batch': 20, 'duration': 24.091975927352905, 'duration_batch': 1.2045987963676452, 'duration_size': 0.3011496990919113, 'avg_pred_std': 0.035354171460494396}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005576713894515706, 'avg_role_model_std_loss': 0.5243572926787815, 'avg_role_model_mean_pred_loss': 7.799909240078506e-05, 'avg_role_model_g_mag_loss': 0.026431062967458275, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0056679701934626795, 'n_size': 320, 'n_batch': 80, 'duration': 147.8123037815094, 'duration_batch': 1.8476537972688676, 'duration_size': 0.4619134493172169, 'avg_pred_std': 0.17163944796193392}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004904342864756472, 'avg_role_model_std_loss': 6.613062943945624, 'avg_role_model_mean_pred_loss': 4.787976637761204e-05, 'avg_role_model_g_mag_loss': 0.02067490922054276, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0050057682892656885, 'n_size': 80, 'n_batch': 20, 'duration': 24.22531223297119, 'duration_batch': 1.2112656116485596, 'duration_size': 0.3028164029121399, 'avg_pred_std': 0.05586188876768574}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005746945142709592, 'avg_role_model_std_loss': 0.3063628292962107, 'avg_role_model_mean_pred_loss': 8.83000647067575e-05, 'avg_role_model_g_mag_loss': 0.020610145634418587, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005859691742443829, 'n_size': 320, 'n_batch': 80, 'duration': 147.5134036540985, 'duration_batch': 1.8439175456762313, 'duration_size': 0.46097938641905783, 'avg_pred_std': 0.18795285349478946}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006080282232142053, 'avg_role_model_std_loss': 5.922124939222977, 'avg_role_model_mean_pred_loss': 5.200857143190518e-05, 'avg_role_model_g_mag_loss': 0.021447001822525635, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006272930890554562, 'n_size': 80, 'n_batch': 20, 'duration': 24.122392892837524, 'duration_batch': 1.2061196446418763, 'duration_size': 0.30152991116046907, 'avg_pred_std': 0.03159960038028657}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005959707218971743, 'avg_role_model_std_loss': 0.11086495585541982, 'avg_role_model_mean_pred_loss': 8.815349536440697e-05, 'avg_role_model_g_mag_loss': 0.025197071458387656, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006072522566682892, 'n_size': 320, 'n_batch': 80, 'duration': 145.82239770889282, 'duration_batch': 1.8227799713611603, 'duration_size': 0.4556949928402901, 'avg_pred_std': 0.19075714307837188}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004875349982467014, 'avg_role_model_std_loss': 6.456333731907944, 'avg_role_model_mean_pred_loss': 3.336970914707971e-05, 'avg_role_model_g_mag_loss': 0.026122358883731066, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004957216732145752, 'n_size': 80, 'n_batch': 20, 'duration': 23.773096084594727, 'duration_batch': 1.1886548042297362, 'duration_size': 0.29716370105743406, 'avg_pred_std': 0.04347092452226207}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005319609342222975, 'avg_role_model_std_loss': 0.3930861216696401, 'avg_role_model_mean_pred_loss': 6.20743445605716e-05, 'avg_role_model_g_mag_loss': 0.024280186876421795, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0054104587794427065, 'n_size': 320, 'n_batch': 80, 'duration': 145.5336833000183, 'duration_batch': 1.8191710412502289, 'duration_size': 0.4547927603125572, 'avg_pred_std': 0.18721119775436817}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004346008354877995, 'avg_role_model_std_loss': 5.974730668690517, 'avg_role_model_mean_pred_loss': 1.5303319651938097e-05, 'avg_role_model_g_mag_loss': 0.02063523129618261, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004420949433551868, 'n_size': 80, 'n_batch': 20, 'duration': 23.65921902656555, 'duration_batch': 1.1829609513282775, 'duration_size': 0.2957402378320694, 'avg_pred_std': 0.03134176780004054}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005300425899258698, 'avg_role_model_std_loss': 0.1536639502475623, 'avg_role_model_mean_pred_loss': 9.251545014040408e-05, 'avg_role_model_g_mag_loss': 0.021025176101829857, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005399322827724973, 'n_size': 320, 'n_batch': 80, 'duration': 145.47741270065308, 'duration_batch': 1.8184676587581634, 'duration_size': 0.45461691468954085, 'avg_pred_std': 0.18479676478891632}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003750909139489522, 'avg_role_model_std_loss': 5.9032170891565325, 'avg_role_model_mean_pred_loss': 7.884279494407842e-06, 'avg_role_model_g_mag_loss': 0.019338129594689234, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003807382771628909, 'n_size': 80, 'n_batch': 20, 'duration': 23.828125, 'duration_batch': 1.19140625, 'duration_size': 0.2978515625, 'avg_pred_std': 0.033805052132811396}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004899124511211994, 'avg_role_model_std_loss': 0.24653815754437575, 'avg_role_model_mean_pred_loss': 5.619041067117326e-05, 'avg_role_model_g_mag_loss': 0.0216044840090035, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004984687315391057, 'n_size': 320, 'n_batch': 80, 'duration': 145.63873076438904, 'duration_batch': 1.820484134554863, 'duration_size': 0.45512103363871576, 'avg_pred_std': 0.19259733236394821}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0059519269168959, 'avg_role_model_std_loss': 3.8484517917624546, 'avg_role_model_mean_pred_loss': 7.430727260597792e-05, 'avg_role_model_g_mag_loss': 0.019101872900500895, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0060627352468145546, 'n_size': 80, 'n_batch': 20, 'duration': 23.49578022956848, 'duration_batch': 1.174789011478424, 'duration_size': 0.293697252869606, 'avg_pred_std': 0.03844869087915868}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00507157752654166, 'avg_role_model_std_loss': 0.12483040868133913, 'avg_role_model_mean_pred_loss': 8.54523543355759e-05, 'avg_role_model_g_mag_loss': 0.021162668641045455, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005164033845721861, 'n_size': 320, 'n_batch': 80, 'duration': 145.65456414222717, 'duration_batch': 1.8206820517778397, 'duration_size': 0.4551705129444599, 'avg_pred_std': 0.19211700824089348}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00464706384591409, 'avg_role_model_std_loss': 6.332309863651451, 'avg_role_model_mean_pred_loss': 2.2397835272180976e-05, 'avg_role_model_g_mag_loss': 0.02283783564926125, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004722811375540914, 'n_size': 80, 'n_batch': 20, 'duration': 23.64812207221985, 'duration_batch': 1.1824061036109925, 'duration_size': 0.29560152590274813, 'avg_pred_std': 0.04854622206185013}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004837600045084401, 'avg_role_model_std_loss': 0.3505868666707556, 'avg_role_model_mean_pred_loss': 5.091351759973674e-05, 'avg_role_model_g_mag_loss': 0.022111102837880026, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00492111863768514, 'n_size': 320, 'n_batch': 80, 'duration': 146.0148582458496, 'duration_batch': 1.82518572807312, 'duration_size': 0.45629643201828, 'avg_pred_std': 0.17585730996215715}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004068510180877638, 'avg_role_model_std_loss': 5.871831947068415, 'avg_role_model_mean_pred_loss': 1.4263169245479212e-05, 'avg_role_model_g_mag_loss': 0.01976338555687107, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004133973495845567, 'n_size': 80, 'n_batch': 20, 'duration': 23.57652258872986, 'duration_batch': 1.178826129436493, 'duration_size': 0.29470653235912325, 'avg_pred_std': 0.03373235072940588}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005061780314099451, 'avg_role_model_std_loss': 0.12903543715087834, 'avg_role_model_mean_pred_loss': 8.679314863810369e-05, 'avg_role_model_g_mag_loss': 0.020122559062292565, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005155922670746804, 'n_size': 320, 'n_batch': 80, 'duration': 146.0060043334961, 'duration_batch': 1.825075054168701, 'duration_size': 0.45626876354217527, 'avg_pred_std': 0.19230934586375953}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005099401987536112, 'avg_role_model_std_loss': 4.219143859660525, 'avg_role_model_mean_pred_loss': 4.85376546165095e-05, 'avg_role_model_g_mag_loss': 0.016910381577326918, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005196944052295293, 'n_size': 80, 'n_batch': 20, 'duration': 23.720768213272095, 'duration_batch': 1.1860384106636048, 'duration_size': 0.2965096026659012, 'avg_pred_std': 0.038260015891864896}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004984028046055755, 'avg_role_model_std_loss': 0.15418263155418116, 'avg_role_model_mean_pred_loss': 6.304440869404939e-05, 'avg_role_model_g_mag_loss': 0.020150290539095294, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005073844128310157, 'n_size': 320, 'n_batch': 80, 'duration': 146.04027271270752, 'duration_batch': 1.825503408908844, 'duration_size': 0.456375852227211, 'avg_pred_std': 0.1910679372958839}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00501167065667687, 'avg_role_model_std_loss': 5.14628453936366, 'avg_role_model_mean_pred_loss': 3.0035914827664102e-05, 'avg_role_model_g_mag_loss': 0.01827869672415545, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005116376957448665, 'n_size': 80, 'n_batch': 20, 'duration': 23.532910346984863, 'duration_batch': 1.1766455173492432, 'duration_size': 0.2941613793373108, 'avg_pred_std': 0.03470718542812392}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004734398943764972, 'avg_role_model_std_loss': 0.6279221150001397, 'avg_role_model_mean_pred_loss': 4.761278923632872e-05, 'avg_role_model_g_mag_loss': 0.017200588568812236, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004827282899350393, 'n_size': 320, 'n_batch': 80, 'duration': 145.9920961856842, 'duration_batch': 1.8249012023210525, 'duration_size': 0.4562253005802631, 'avg_pred_std': 0.17317850722465664}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007601118210004643, 'avg_role_model_std_loss': 3.2803518016677558, 'avg_role_model_mean_pred_loss': 0.00017301160700378612, 'avg_role_model_g_mag_loss': 0.012077987159136683, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00790572552505182, 'n_size': 80, 'n_batch': 20, 'duration': 23.845383882522583, 'duration_batch': 1.1922691941261292, 'duration_size': 0.2980672985315323, 'avg_pred_std': 0.03440641919150948}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0045964237675889304, 'avg_role_model_std_loss': 0.569245081065904, 'avg_role_model_mean_pred_loss': 5.5843747120097085e-05, 'avg_role_model_g_mag_loss': 0.014247680191328983, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004697687765110458, 'n_size': 320, 'n_batch': 80, 'duration': 146.36437106132507, 'duration_batch': 1.8295546382665635, 'duration_size': 0.4573886595666409, 'avg_pred_std': 0.18270513406023384}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007547395752771991, 'avg_role_model_std_loss': 2.9282909621120212, 'avg_role_model_mean_pred_loss': 0.00017729704157093407, 'avg_role_model_g_mag_loss': 0.012870692153228447, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007793798694547149, 'n_size': 80, 'n_batch': 20, 'duration': 23.560658931732178, 'duration_batch': 1.1780329465866088, 'duration_size': 0.2945082366466522, 'avg_pred_std': 0.0382234871853143}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0046578587127442, 'avg_role_model_std_loss': 0.5405012748799436, 'avg_role_model_mean_pred_loss': 8.542011439159336e-05, 'avg_role_model_g_mag_loss': 0.016841917944839226, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00475033148932198, 'n_size': 320, 'n_batch': 80, 'duration': 146.28253817558289, 'duration_batch': 1.828531727194786, 'duration_size': 0.4571329317986965, 'avg_pred_std': 0.18576831596437843}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008764273906126618, 'avg_role_model_std_loss': 3.272361376248591, 'avg_role_model_mean_pred_loss': 0.0002117462956683802, 'avg_role_model_g_mag_loss': 0.011796211288310588, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009110148320905865, 'n_size': 80, 'n_batch': 20, 'duration': 23.770928859710693, 'duration_batch': 1.1885464429855346, 'duration_size': 0.29713661074638364, 'avg_pred_std': 0.0332530245417729}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004992562824554625, 'avg_role_model_std_loss': 0.2701526861071898, 'avg_role_model_mean_pred_loss': 6.218077271888996e-05, 'avg_role_model_g_mag_loss': 0.01973752847407013, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005098153510880365, 'n_size': 320, 'n_batch': 80, 'duration': 146.03065609931946, 'duration_batch': 1.8253832012414932, 'duration_size': 0.4563458003103733, 'avg_pred_std': 0.19569849580875598}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005505499814171344, 'avg_role_model_std_loss': 5.217980894941865, 'avg_role_model_mean_pred_loss': 5.959937391786241e-05, 'avg_role_model_g_mag_loss': 0.02082536361485836, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005607066205629963, 'n_size': 80, 'n_batch': 20, 'duration': 23.852116107940674, 'duration_batch': 1.1926058053970336, 'duration_size': 0.2981514513492584, 'avg_pred_std': 0.038161135837435724}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004348099110757175, 'avg_role_model_std_loss': 0.20937616627059583, 'avg_role_model_mean_pred_loss': 3.1978708130590886e-05, 'avg_role_model_g_mag_loss': 0.015712628203618805, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004428885674678895, 'n_size': 320, 'n_batch': 80, 'duration': 146.06442213058472, 'duration_batch': 1.825805276632309, 'duration_size': 0.45645131915807724, 'avg_pred_std': 0.18260755122173578}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00345534470397979, 'avg_role_model_std_loss': 3.8281147816255725, 'avg_role_model_mean_pred_loss': 4.558243014707742e-06, 'avg_role_model_g_mag_loss': 0.013488705200143159, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0035211247907682265, 'n_size': 80, 'n_batch': 20, 'duration': 23.61065983772278, 'duration_batch': 1.180532991886139, 'duration_size': 0.29513324797153473, 'avg_pred_std': 0.044103194237686695}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0045102617631528116, 'avg_role_model_std_loss': 0.175469460278498, 'avg_role_model_mean_pred_loss': 5.030859646430498e-05, 'avg_role_model_g_mag_loss': 0.00875355452990334, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004684712074777053, 'n_size': 320, 'n_batch': 80, 'duration': 145.9878635406494, 'duration_batch': 1.8248482942581177, 'duration_size': 0.4562120735645294, 'avg_pred_std': 0.1969025250407867}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0036117414470936637, 'avg_role_model_std_loss': 3.6396715161072066, 'avg_role_model_mean_pred_loss': 1.5647155543276753e-05, 'avg_role_model_g_mag_loss': 0.01432776392903179, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003690452836053737, 'n_size': 80, 'n_batch': 20, 'duration': 23.642552614212036, 'duration_batch': 1.182127630710602, 'duration_size': 0.2955319076776505, 'avg_pred_std': 0.04286287054419517}\n", + "Epoch 20\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004609042530137231, 'avg_role_model_std_loss': 0.1913827067049482, 'avg_role_model_mean_pred_loss': 7.204572566021255e-05, 'avg_role_model_g_mag_loss': 0.01329805545246927, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004719520112121245, 'n_size': 320, 'n_batch': 80, 'duration': 146.35307240486145, 'duration_batch': 1.8294134050607682, 'duration_size': 0.45735335126519205, 'avg_pred_std': 0.1828528445912525}\n", + "Time out: 3607.3026695251465/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test █▁▄▁▄▁▂▁▁▂▃▁▂▁▁▂▁▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▁█▆▇▆▇▇▇▇▇▇▆▇▇▆▆▇▇▆▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train █▃▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test █▁▂▂▂▂▂▂▂▁▂▂▁▂▁▁▁▂▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▂▄▂▇█▇▄▃▄▁▂▂▃▁▄▂▄▄▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▄▅▅▄█▇▂▁▁▁▂▃▃▃▃▄▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▂▄▂▇█▇▄▃▄▁▂▂▃▁▄▂▄▄▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▄▅▅▄█▇▂▁▁▁▂▃▃▃▃▄▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▂▄▂▇█▇▄▃▄▁▂▂▃▁▄▂▄▄▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▄▅▅▄█▇▂▁▁▁▂▃▃▃▃▄▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00369\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00468\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.04286\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.1969\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.01433\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.00875\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00361\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00451\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 2e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 5e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 3.63967\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.17547\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 1.18213\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.82485\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.29553\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.45621\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 23.64255\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 145.98786\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 20\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/treatment/tab_ddpm_concat/1/wandb/offline-run-20240229_024250-b8vt892f\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_024250-b8vt892f/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 399, 'n_batch': 100, 'role_model_metrics': {'avg_loss': 0.0016991006310960338, 'avg_g_mag_loss': 0.00028921539953522793, 'avg_g_cos_loss': 0.0, 'pred_duration': 2.2586166858673096, 'grad_duration': 4.4722723960876465, 'total_duration': 6.730889081954956, 'pred_std': 0.06956713646650314, 'std_loss': 0.012020673602819443, 'mean_pred_loss': 4.390428784972755e-06, 'pred_rmse': 0.041220150887966156, 'pred_mae': 0.029980136081576347, 'pred_mape': 0.059064049273729324, 'grad_rmse': 0.039643414318561554, 'grad_mae': 0.02741244062781334, 'grad_mape': 0.4385460913181305}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0016991006310960338, 'avg_g_mag_loss': 0.00028921539953522793, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 2.2586166858673096, 'avg_grad_duration': 4.4722723960876465, 'avg_total_duration': 6.730889081954956, 'avg_pred_std': 0.06956713646650314, 'avg_std_loss': 0.012020673602819443, 'avg_mean_pred_loss': 4.390428784972755e-06}, 'min_metrics': {'avg_loss': 0.0016991006310960338, 'avg_g_mag_loss': 0.00028921539953522793, 'avg_g_cos_loss': 0.0, 'pred_duration': 2.2586166858673096, 'grad_duration': 4.4722723960876465, 'total_duration': 6.730889081954956, 'pred_std': 0.06956713646650314, 'std_loss': 0.012020673602819443, 'mean_pred_loss': 4.390428784972755e-06, 'pred_rmse': 0.041220150887966156, 'pred_mae': 0.029980136081576347, 'pred_mape': 0.059064049273729324, 'grad_rmse': 0.039643414318561554, 'grad_mae': 0.02741244062781334, 'grad_mape': 0.4385460913181305}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.0016991006310960338, 'avg_g_mag_loss': 0.00028921539953522793, 'avg_g_cos_loss': 0.0, 'pred_duration': 2.2586166858673096, 'grad_duration': 4.4722723960876465, 'total_duration': 6.730889081954956, 'pred_std': 0.06956713646650314, 'std_loss': 0.012020673602819443, 'mean_pred_loss': 4.390428784972755e-06, 'pred_rmse': 0.041220150887966156, 'pred_mae': 0.029980136081576347, 'pred_mape': 0.059064049273729324, 'grad_rmse': 0.039643414318561554, 'grad_mae': 0.02741244062781334, 'grad_mape': 0.4385460913181305}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:44:51.400689Z", + "iopub.status.busy": "2024-02-29T03:44:51.400339Z", + "iopub.status.idle": "2024-02-29T03:44:51.406303Z", + "shell.execute_reply": "2024-02-29T03:44:51.404294Z" + }, + "papermill": { + "duration": 0.035656, + "end_time": "2024-02-29T03:44:51.409321", + "exception": false, + "start_time": "2024-02-29T03:44:51.373665", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:44:51.459870Z", + "iopub.status.busy": "2024-02-29T03:44:51.459533Z", + "iopub.status.idle": "2024-02-29T03:44:51.927277Z", + "shell.execute_reply": "2024-02-29T03:44:51.926205Z" + }, + "papermill": { + "duration": 0.497623, + "end_time": "2024-02-29T03:44:51.929870", + "exception": false, + "start_time": "2024-02-29T03:44:51.432247", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:44:51.972252Z", + "iopub.status.busy": "2024-02-29T03:44:51.971768Z", + "iopub.status.idle": "2024-02-29T03:44:52.298553Z", + "shell.execute_reply": "2024-02-29T03:44:52.297628Z" + }, + "papermill": { + "duration": 0.351797, + "end_time": "2024-02-29T03:44:52.301120", + "exception": false, + "start_time": "2024-02-29T03:44:51.949323", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAEWCAYAAADPS+pKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3UklEQVR4nO3de3xU1b3//9eeSWZyv0DIZWgkXA1yiyUmoqJVIwnHUrG0DRx+BfJo4dSCPTZFLQoJiqfhpoeqHDjVh4JWLvZxqud81RO1kdRTjaAgQgtSoNGAMIFEM7lPkpn1+2MyOxlIQiaEzJD5PB+P/chkz9571sbM27XXXnstTSmlEEIIP2DwdQGEEMJNAkkI4TckkIQQfkMCSQjhNySQhBB+QwJJCOE3JJCEEH5DAkkI4TckkIQQfkMCSQjhN/oUSJs3byYlJYWQkBAyMzPZt29ft9v+8Y9/JD09nZiYGMLDw0lLS+Pll1/22GbRokVomuax5OTk9KVoQoirWJC3O+zevZv8/Hy2bt1KZmYmmzZtIjs7m2PHjhEfH3/R9kOGDOHRRx8lNTUVk8nEG2+8QV5eHvHx8WRnZ+vb5eTk8OKLL+q/m83mXpfJ6XRy5swZIiMj0TTN21MSQlxhSinq6uqwWCwYDD3Ug5SXMjIy1NKlS/XfHQ6HslgsqqioqNfHuP7669XKlSv13xcuXKjuueceb4uiO3XqlAJkkUUWP19OnTrV43fZqxpSS0sL+/fvZ8WKFfo6g8FAVlYWZWVll9xfKcV7773HsWPHWLduncd7paWlxMfHExsbyx133METTzzB0KFDuzyO3W7Hbrd7HBfg1KlTREVFeXNKQogBUFtbS3JyMpGRkT1u51UgVVVV4XA4SEhI8FifkJDA559/3u1+NpuN4cOHY7fbMRqN/Md//Ad33XWX/n5OTg7f//73GTlyJCdPnuSRRx5h5syZlJWVYTQaLzpeUVERjz322EXro6KiJJCE8GOXalLxug2pLyIjIzl48CD19fWUlJSQn5/PqFGj+M53vgPA3Llz9W0nTZrE5MmTGT16NKWlpdx5550XHW/FihXk5+frv7vTVwhxdfMqkOLi4jAajVRWVnqsr6ysJDExsdv9DAYDY8aMASAtLY2jR49SVFSkB9KFRo0aRVxcHCdOnOgykMxms1eN3kKIq4NXt/1NJhNTp06lpKREX+d0OikpKWHatGm9Po7T6fRoA7rQ6dOnqa6uJikpyZviCSGucl5fsuXn57Nw4ULS09PJyMhg06ZNNDQ0kJeXB8CCBQsYPnw4RUVFgKu9Jz09ndGjR2O323nrrbd4+eWX2bJlCwD19fU89thjzJkzh8TERE6ePMlDDz3EmDFjPLoFiCvH4XDQ2trq62KIq1hwcHCX7b3e8jqQcnNzOX/+PAUFBVitVtLS0iguLtYbuisqKjz6GTQ0NPDzn/+c06dPExoaSmpqKr///e/Jzc0FwGg0cujQIbZv305NTQ0Wi4UZM2awZs0auSy7wpRSWK1WampqfF0UMQjExMSQmJh4WX0BNaWu/kH+a2triY6OxmazyV02L5w9e5aamhri4+MJCwuTTqWiT5RSNDY2cu7cOWJiYrpsauntd3RA7rL5izaHk09P1VDb1Mrt18ZjMATuF9DhcOhh1F1/LyF6KzQ0FIBz584RHx/f58u3gHq41qngh1vL+Mn2T6hrbvN1cXzK3WYUFhbm45KIwcL9t3Q57ZEBFUimIAOhwa7krm2WRly4dEc1IXqrP/6WAiqQAKJCXVeptiYJJCH8TcAFUnRoMCCBJPpO0zRef/11XxejX61evZq0tDRfF0MCSYir0bZt24iJiem34y1fvtyjw7OvBNRdNugIpFoJJBEAWlpaMJlMl9wuIiKCiIiIAShRzwKuhhQVIjWkq11xcTG33HILMTExDB06lO9+97ucPHkSgJtuuomHH37YY/vz588THBzM+++/D7j6X919992EhoYycuRIduzYQUpKCps2bepTeQ4fPswdd9xBaGgoQ4cOZcmSJdTX1+vvl5aWkpGRQXh4ODExMdx88818+eWXAHz22WfcfvvtREZGEhUVxdSpU/nkk096/LzS0lLy8vKw2Wz6CKurV68GICUlhTVr1rBgwQKioqJYsmQJAA8//DDjxo0jLCyMUaNGsWrVKo+7YRdesi1atIjZs2ezceNGkpKSGDp0KEuXLr3iPfoDroYUJZds3VJK0dTqGPDPDQ02enWHpqGhgfz8fCZPnkx9fT0FBQXce++9HDx4kPnz57N+/XrWrl2rH3P37t1YLBamT58OuB5vqqqqorS0lODgYPLz8zl37lyfyt7Q0EB2djbTpk3j448/5ty5c/z0pz9l2bJlbNu2jba2NmbPns3ixYvZuXMnLS0t7Nu3Ty/b/Pnzuf7669myZQtGo5GDBw8SHBzc42fedNNNbNq0iYKCAo4dOwbgUbvZuHEjBQUFFBYW6usiIyPZtm0bFouFw4cPs3jxYiIjI3nooYe6/Zw9e/aQlJTEnj17OHHiBLm5uaSlpbF48eI+/Vv1RsAFkrQhda+p1cF1BW8P+OceeTybMFPv/xTnzJnj8fsLL7zAsGHDOHLkCD/60Y944IEH+Mtf/qIH0I4dO5g3bx6apvH555/zpz/9iY8//pj09HQAnn/+ecaOHdunsu/YsYPm5mZeeuklwsPDAXj22WeZNWsW69atIzg4GJvNxne/+11Gjx4NwPjx4/X9KyoqePDBB0lNTQXoVTlMJhPR0dFomtblKBt33HEHv/rVrzzWrVy5Un+dkpLC8uXL2bVrV4+BFBsby7PPPovRaCQ1NZW7776bkpKSKxpIgXfJ5m5DCvCOkVez48ePM2/ePEaNGkVUVBQpKSmA68s9bNgwZsyYwSuvvAJAeXk5ZWVlzJ8/H4Bjx44RFBTEt7/9bf14Y8aMITY2tk9lOXr0KFOmTNHDCODmm2/G6XRy7NgxhgwZwqJFi8jOzmbWrFn89re/5ezZs/q2+fn5/PSnPyUrK4u1a9fql56Xwx20ne3evZubb76ZxMREIiIiWLlyJRUVFT0eZ8KECR49rpOSkvpck+wtqSEJXWiwkSOPD/wIC+7Oqr01a9YsRowYwXPPPYfFYsHpdDJx4kRaWloA12XQL37xC5555hl27NjBpEmTmDRp0pUoeq+8+OKL/OIXv6C4uJjdu3ezcuVK3n33XW688UZWr17NP//zP/Pmm2/yv//7vxQWFrJr1y7uvffePn9e53AE9EB+7LHHyM7OJjo6ml27dvHkk0/2eJwLLx01TcPpdPa5XL0RcDUkCaTuaZpGmClowBdv2o+qq6s5duwYK1eu5M4772T8+PF88803Htvcc889NDc3U1xczI4dO/TaEcC1115LW1sbn376qb7uxIkTFx2jt8aPH89nn31GQ0ODvu6DDz7AYDBw7bXX6uuuv/56VqxYwYcffsjEiRPZsWOH/t64ceP45S9/yTvvvMP3v/99j9l3umMymXA4etfe9+GHHzJixAgeffRR0tPTGTt2rN6o7m8CNpDktv/VKTY2lqFDh/K73/2OEydO8N5773kMZwyuGsLs2bNZtWoVR48eZd68efp7qampZGVlsWTJEvbt28enn37KkiVLCA0N7dOjD/PnzyckJISFCxfy17/+lT179nD//ffz4x//mISEBMrLy1mxYgVlZWV8+eWXvPPOOxw/fpzx48fT1NTEsmXLKC0t5csvv+SDDz7g448/9mhj6k5KSoo+JHRVVRWNjY3dbjt27FgqKirYtWsXJ0+e5Omnn+a1117z+lwHQsAFkvvREQmkq5PBYGDXrl3s37+fiRMn8stf/pINGzZctN38+fP57LPPmD59Otdcc43Hey+99BIJCQnceuut3Hvvvfodp5CQEK/LExYWxttvv83XX3/NDTfcwA9+8APuvPNOnn32Wf39zz//nDlz5jBu3DiWLFnC0qVL+Zd/+ReMRiPV1dUsWLCAcePG8aMf/YiZM2d2OYHFhW666SZ+9rOfkZuby7Bhw1i/fn23237ve9/jl7/8JcuWLSMtLY0PP/yQVatWeX2uAyHgxkM6a2tiWtF7BBk0jv/bzIB9uLS5uZny8nJGjhzZpy/iYHL69GmSk5P505/+1OUY7qJ3evqbkvGQuuG+ZGtzKhpbHISbA+6fIOC999571NfXM2nSJM6ePctDDz1ESkoKt956q6+LFvAC7pItNNhIsNFVK5KG7cDU2trKI488woQJE7j33nsZNmyY3knylVde0R+juHCZMGHCgJVx5syZ3ZbjN7/5zYCVY6AFXPVA0zSiQoKpbmihtrkVC6G+LpIYYNnZ2d1OIPG9732PzMzMLt+7VA/q/vT888/T1NTU5XtDhgwZsHIMtIALJHBdtlU3tGBrlBqS8BQZGXnJ6Z4HwvDhw31dBJ8IuEs2kOfZhPBXfQqkzZs3k5KSQkhICJmZmezbt6/bbf/4xz+Snp5OTEwM4eHhpKWl8fLLL3tso5SioKCApKQkQkNDycrK4vjx430pWq9I50gh/JPXgbR7927y8/MpLCzkwIEDTJkyhezs7G6fcRkyZAiPPvooZWVlHDp0iLy8PPLy8nj77Y6HONevX8/TTz/N1q1b2bt3L+Hh4WRnZ9Pc3Nz3M+uBPM8mhJ9SXsrIyFBLly7Vf3c4HMpisaiioqJeH+P6669XK1euVEop5XQ6VWJiotqwYYP+fk1NjTKbzWrnzp29Op7NZlOAstlsvdr+0dcOqREPv6GefOdYr8s82DQ1NakjR46opqYmXxdFDBI9/U319jvqVQ2ppaWF/fv3k5WVpa8zGAxkZWVRVlbWm/CjpKSEY8eO6X0+ysvLsVqtHseMjo4mMzOz22Pa7XZqa2s9Fm/I4yNC+CevAqmqqgqHw6FPm+2WkJCA1Wrtdj+bzUZERAQmk4m7776bZ555hrvuugtA38+bYxYVFREdHa0vycnJ3pyGtCGJyzIYB/n3FwNyly0yMpKDBw/y8ccf82//9m/k5+dTWlra5+OtWLECm82mL6dOnfJqf/cwtlJDEler/h7kH1xD42qaRk1NTb8e1xte9UOKi4vDaDRSWVnpsb6ysrLLkevcDAYDY8aMASAtLY2jR49SVFTEd77zHX2/yspKjznBKysru52WxWw2YzabvSm6B6khCeGfvKohmUwmpk6d6jFditPppKSkhGnTpvX6OE6nE7vdDsDIkSNJTEz0OGZtbS179+716pjekEC6uskg/90P8m+321m+fDnDhw8nPDyczMxMj6uRL7/8klmzZhEbG0t4eDgTJkzgrbfe4osvvuD2228HXEO8aJrGokWL+vTvcTm87qmdn5/PwoULSU9PJyMjg02bNtHQ0EBeXh7gGkB9+PDhFBUVAa72nvT0dEaPHo3dbuett97i5ZdfZsuWLYDrevyBBx7giSeeYOzYsYwcOZJVq1ZhsViYPXt2/51pJ9IxshtKQWv34+pcMcFhIIP898sg/8uWLePIkSPs2rULi8XCa6+9Rk5ODocPH2bs2LEsXbqUlpYW3n//fcLDwzly5AgREREkJyfzX//1X8yZM4djx44RFRVFaOjAP1bldSDl5uZy/vx5CgoKsFqtpKWlUVxcrDdKV1RUYDB0VLwaGhr4+c9/zunTpwkNDSU1NZXf//735Obm6ts89NBDNDQ0sGTJEmpqarjlllsoLi6+YsNi6HfZmiWQPLQ2wm8sA/+5j5wBU/ilt2sng/x3Pch/RUUFL774IhUVFVgsrv+Oy5cvp7i4mBdffJHf/OY3VFRUMGfOHH1I31GjRun7u5+Ri4+P7/f2qd7q07Nsy5YtY9myZV2+d2Fj9RNPPMETTzzR4/E0TePxxx/n8ccf70txvOauITW3OrG3OTAHeTems/Ct48ePU1BQwN69e6mqqtLHea6oqGDixIn6IP/Tp0/XB/n/z//8T2DgB/m/9dZb9UH+77rrLrKysvjRj36kt5e6B/l/+eWXycrK4oc//KEeXN46fPgwDoeDcePGeay32+0MHToUgF/84hfcd999vPPOO2RlZTFnzhwmT57cp8+7EgLy4dpIcxCa5rpCsTW1Eh8pgQS4Lp0eOeObz/WCDPLftfr6eoxGI/v37/eYLQQ6Lul++tOfkp2dzZtvvsk777xDUVERTz75JPfff3+/nOtlu0KdNgeUtz21lVJq8uq31YiH31DHK2uvYMn819XaU7uqqkoB6v3339fX/d///Z8C1GuvvaaUUqq+vl6Fh4er//mf/1HXXXedWrt2rb7t0aNHFaA++eQTfd3x48cVoP793/+9V2Xo/Fm/+93vVGxsrKqvr9fff/PNN5XBYFBWq7XL/W+88UZ1//33d/ne3Llz1axZsy5ZhldeeUVFRER4rDt27NhF/zaX8utf/1pNmjRJKaXUBx98oABVVVXV6/07G/Ce2oOJe2xtW5M8z3Y1kUH+Xboa5H/cuHHMnz+fBQsW8Mc//pHy8nL27dtHUVERb775JgAPPPAAb7/9NuXl5Rw4cIA9e/bonzdixAg0TeONN97g/PnzHncKB0yfotDP9KWGdPfT76sRD7+h3jtaeQVL5r+u1hqSUkq9++67avz48cpsNqvJkyer0tJSj1qLUkq99dZbClC33nrrRfufOXNGzZw5U5nNZjVixAi1Y8cOFR8fr7Zu3dqrz7/wsw4dOqRuv/12FRISooYMGaIWL16s6urqlFJKWa1WNXv2bJWUlKRMJpMaMWKEKigoUA6HQ9ntdjV37lyVnJysTCaTslgsatmyZb3+b/Kzn/1MDR06VAGqsLBQKaVUS0uLKigoUCkpKSo4OFglJSWpe++9Vx06dEgppdSyZcvU6NGjldlsVsOGDVM//vGPPWpEjz/+uEpMTFSapqmFCxf2qhxu/VFDCrhB/t3mP/8RH5yoZlNuGrOvD7zBsGSQ/w4yyH//kEH+L4N0jgxcMsi//wrcNiR5ni1gySD//ktqSBJIAUcG+fdfARtI8viI6IoM8u9bAXvJJjUkIfxPwAZSlDzPBrhG8RSiP/TH31LABlJHDSkwO0a620MaG33wdL8YlNx/S5fT1hawbUiBPq620WgkJiZGH3YjLCysTz2VhVBK0djYyLlz54iJibnoOTpvBHwgBXIbknvoir6OBSREZzExMT2OHNsbARtIUSGuU6+3t9HmcBJkDLyrV03TSEpKIj4+ntbWwA1mcfmCg4Mvq2bkFriBFNpxnVvX3EZsuMmHpfEto9HYL39MQlyuwKsWtAs2Ggg3ub6EgXzZJoQ/CdhAAmlHEsLfBHQgSV8kIfyLBBJSQxLCX/QpkDZv3kxKSgohISFkZmayb9++brd97rnnmD59OrGxscTGxpKVlXXR9osWLdLnl3IvOTk5fSmaV+SSTQj/4nUg7d69m/z8fAoLCzlw4ABTpkwhOzu7274spaWlzJs3jz179lBWVkZycjIzZszgq6++8tguJyeHs2fP6svOnTv7dkZekEASwr94HUhPPfUUixcvJi8vj+uuu46tW7cSFhbGCy+80OX2r7zyCj//+c9JS0sjNTWV559/Xp/ttjOz2UxiYqK+9HVaGm90jIkUmI+PCOFvvAqklpYW9u/fT1ZWVscBDAaysrIoKyvr1TEaGxtpbW29aEyX0tJS4uPjufbaa7nvvvuorq7u9hh2u53a2lqPpS+khiSEf/EqkKqqqnA4HPostW4JCQlYrdZeHePhhx/GYrF4hFpOTg4vvfQSJSUlrFu3jj//+c/MnDkTh8PR5TGKioqIjo7Wl+TkZG9OQxfdPvNIoD7PJoS/GdCe2mvXrmXXrl2UlpZ6DAI+d+5c/fWkSZOYPHkyo0ePprS0tMtB11esWOEx9U1tbW2fQik6TGpIQvgTr2pIcXFxGI1GKisrPdZXVlZe8qG6jRs3snbtWt55551LTt07atQo4uLiOHHiRJfvm81moqKiPJa+0NuQpB+SEH7Bq0AymUxMnTrVo0Ha3UA9bdq0bvdbv349a9asobi4mPT09Et+zunTp6murtbnP79SpA1JCP/i9V22/Px8nnvuObZv387Ro0e57777aGhoIC8vD4AFCxawYsUKfft169axatUqXnjhBVJSUrBarVitVn1WzPr6eh588EE++ugjvvjiC0pKSrjnnnsYM2ZMtwOx9xcJJCH8i9dtSLm5uZw/f56CggKsVitpaWkUFxfrDd0VFRUYDB05t2XLFlpaWvjBD37gcZzCwkJWr16N0Wjk0KFDbN++nZqaGiwWCzNmzGDNmjWYzebLPL2edR6kzelUGAwyQJkQvhSwM9cCNLc6SF1VDMDh1TOIDBm4aW6ECCS9/Y4G9LNsIcFGTEGufwK5bBPC9wI6kEDakYTwJxJIofL4iBD+IuADyT22ttSQhPC9gA+kQJ8OSQh/IoEkbUhC+A0JJBnGVgi/EfCBJMPYCuE/Aj6Q5JJNCP8R8IEkNSQh/EfAB5LcZRPCfwR8ILnHRJIakhC+F/CB1NGGJD21hfA1CaSwjku2QTDwgRBXtYAPJPejIy0OJ/Y2p49LI0RgC/hAijAHYWwfmE3akYTwrYAPJE3T5AFbIfxEwAcSSOdIIfyFBBIdnSOlL5IQviWBhNSQhPAXEkjI4yNC+Is+BdLmzZtJSUkhJCSEzMxM9u3b1+22zz33HNOnTyc2NpbY2FiysrIu2l4pRUFBAUlJSYSGhpKVlcXx48f7UrQ+kRqSEP7B60DavXs3+fn5FBYWcuDAAaZMmUJ2djbnzp3rcvvS0lLmzZvHnj17KCsrIzk5mRkzZvDVV1/p26xfv56nn36arVu3snfvXsLDw8nOzqa5ubnvZ+YFfUpt6a0thG8pL2VkZKilS5fqvzscDmWxWFRRUVGv9m9ra1ORkZFq+/btSimlnE6nSkxMVBs2bNC3qampUWazWe3cubNXx7TZbApQNpvNizPp8B97TqgRD7+h8ncf7NP+Qoie9fY76lUNqaWlhf3795OVlaWvMxgMZGVlUVZW1qtjNDY20traypAhQwAoLy/HarV6HDM6OprMzMxuj2m326mtrfVYLodcsgnhH7wKpKqqKhwOhz5ttltCQgJWq7VXx3j44YexWCx6ALn38+aYRUVFREdH60tycrI3p3ERGYJECP8woHfZ1q5dy65du3jttdcICQnp83FWrFiBzWbTl1OnTl1WuaJCXT21ZVxtIXwryJuN4+LiMBqNVFZWeqyvrKwkMTGxx303btzI2rVr+dOf/sTkyZP19e79KisrSUpK8jhmWlpal8cym82YzWZvit4juWQTwj94VUMymUxMnTqVkpISfZ3T6aSkpIRp06Z1u9/69etZs2YNxcXFpKene7w3cuRIEhMTPY5ZW1vL3r17ezxmf5JAEsI/eFVDAsjPz2fhwoWkp6eTkZHBpk2baGhoIC8vD4AFCxYwfPhwioqKAFi3bh0FBQXs2LGDlJQUvV0oIiKCiIgINE3jgQce4IknnmDs2LGMHDmSVatWYbFYmD17dv+daQ/cgdTY4qDV4STYKP1FhfAFrwMpNzeX8+fPU1BQgNVqJS0tjeLiYr1RuqKiAoOh4wu9ZcsWWlpa+MEPfuBxnMLCQlavXg3AQw89RENDA0uWLKGmpoZbbrmF4uLiy2pn8kZkez8kcDVsD43ov8tBIUTvaUpd/cMk1tbWEh0djc1mIyoqqk/HmFT4NnX2Nt771W2MGhbRzyUUIrD19jsaeNcmTgfUX9yrXJ5nE8L3AiuQ7HWwZhhsHAstjR5vScO2EL4XWIFkigCjyfW63rPTZUdfJHmeTQhfCaxA0jSIbO8RXufZl0pqSEL4XmAFEkBke+fLC2pI8viIEL4XeIEU4a4hdR1IUkMSwncCL5Ai2x9xuSCQOsZEkkASwlcCL5DcNaT6C9qQwqSGJISvBV4gdVNDkks2IXwvcAPpghqSdIwUwvcCL5Ai3DWksx6r9TYkGRNJCJ8JvEBy15CavoE2u75av2RrlEASwlcCL5BCYzv11u64bHMHUp29Dafzqn/eWIirUuAFkqZ16ovUEUjuR0eUgjp5fEQInwi8QIJOt/477rSZg4yEBLv+OaQdSQjfCMxAklv/QvglCaROJJCE8K3ADCT3rf9uHrCVQBLCNwIzkLoZgkSeZxPCtwIzkKSGJIRf6lMgbd68mZSUFEJCQsjMzGTfvn3dbvu3v/2NOXPmkJKSgqZpbNq06aJtVq9ejaZpHktqampfitY7ehuSPD4ihD/xOpB2795Nfn4+hYWFHDhwgClTppCdnc25cxcPnA/Q2NjIqFGjWLt2bY+z206YMIGzZ8/qy1/+8hdvi9Z77kBqOA+Ojj5HUkMSwre8DqSnnnqKxYsXk5eXx3XXXcfWrVsJCwvjhRde6HL7G264gQ0bNjB37twep78OCgoiMTFRX+Li4rwtWu+FxYFmBBQ0dASpu4Yk42oL4RteBVJLSwv79+8nKyur4wAGA1lZWZSVlV1WQY4fP47FYmHUqFHMnz+fioqKbre12+3U1tZ6LF4xGCAi3vW6061/qSEJ4VteBVJVVRUOh0OfpdYtISFBnyK7LzIzM9m2bRvFxcVs2bKF8vJypk+fTl1dXZfbFxUVER0drS/Jycnef2gXA7VJIAnhW35xl23mzJn88Ic/ZPLkyWRnZ/PWW29RU1PDq6++2uX2K1aswGaz6cupU6e8/1D3YP9d1JDktr8QvhHkzcZxcXEYjUYqKz3vTlVWVvbYYO2tmJgYxo0bx4kTJ7p832w299ge1SuRFw/2r8/NJoEkhE94VUMymUxMnTqVkpISfZ3T6aSkpIRp06b1W6Hq6+s5efIkSUlJ/XbMi3TRF6nzJZtSMgSJEAPNqxoSQH5+PgsXLiQ9PZ2MjAw2bdpEQ0MDeXl5ACxYsIDhw4dTVFQEuBrCjxw5or/+6quvOHjwIBEREYwZMwaA5cuXM2vWLEaMGMGZM2coLCzEaDQyb968/jrPi3XRW9sdSG1ORWOLg3Cz1/88QojL4PU3Ljc3l/Pnz1NQUIDVaiUtLY3i4mK9obuiogKDoaPidebMGa6//nr9940bN7Jx40Zuu+02SktLATh9+jTz5s2jurqaYcOGccstt/DRRx8xbNiwyzy9HnRRQwoNNhJs1Gh1KGxNrRJIQgwwTQ2Ca5Pa2lqio6Ox2WxERUX1bqevDsBzt0OkBX51VF89dc27VDe0UPzAdFITe3ksIUSPevsd9Yu7bD7RefYRp1NfLWNrC+E7gRtI4fGABsoBjVX6anmeTQjfCdxAMgZBePvjKV31RZLHR4QYcIEbSNCpYbvzYP9SQxLCVwI7kLroHBnd3jlSAkmIgRfggdR950jprS3EwAvsQIq4eLB/CSQhfCewA6mL2Ufc42rLJZsQAy+wA0mGIBHCrwR2IHUxtrYEkhC+I4EErkbt9idoOoaxlUASYqAFdiC5L9kcLdD0DSA1JCF8KbADKcgMobGu1+0N2+4aUnOrE3ubw1clEyIgBXYgwUXDkESag9A01yqpJQkxsCSQLhiozWDQOk2pLc+zCTGQJJDcg/3XXzy2ttSQhBhYEkgR3Q9lK721hRhYEkh6X6Sz+iq50yaEb0ggddFbW29Dkr5IQgwoCaQunmeTYWyF8A0JpM5ja7f31pZLNiF8o0+BtHnzZlJSUggJCSEzM5N9+/Z1u+3f/vY35syZQ0pKCpqmsWnTpss+Zr9y90NqbQR7HSCjRgrhK14H0u7du8nPz6ewsJADBw4wZcoUsrOzOXfuXJfbNzY2MmrUKNauXdvtdNveHrNfmcLA3D4tS3s7kjzPJoRveB1ITz31FIsXLyYvL4/rrruOrVu3EhYWxgsvvNDl9jfccAMbNmxg7ty5mM3mfjlmv9Nv/bvutMklmxC+4VUgtbS0sH//frKysjoOYDCQlZVFWVlZnwrQl2Pa7XZqa2s9lstywTAkHYEkPbWFGEheBVJVVRUOh0OfNtstISEBq9XazV79f8yioiKio6P1JTk5uU+frdNv/bs+TzpGCuEbV+VdthUrVmCz2fTl1KlTl3fAC279R4W4Hh2RQBJiYAV5s3FcXBxGo5HKykqP9ZWVld02WF+JY5rN5m7bo/ok0nN+NncNqc7ehsOpMBq0/vssIUS3vKohmUwmpk6dSklJib7O6XRSUlLCtGnT+lSAK3FMr10w+4j7LhtILUmIgeRVDQkgPz+fhQsXkp6eTkZGBps2baKhoYG8vDwAFixYwPDhwykqKgJcjdZHjhzRX3/11VccPHiQiIgIxowZ06tjXnEXTBgZbDQQbjLS0OLA1tRKbLhpYMohRIDzOpByc3M5f/48BQUFWK1W0tLSKC4u1hulKyoqMBg6Kl5nzpzh+uuv13/fuHEjGzdu5LbbbqO0tLRXx7ziuplSu6HFIX2RhBhAmlLtz0tcxWpra4mOjsZmsxEVFeX9AZprYW37nbpHzoIpjJxN7/O5tY6Xf5LB9LHD+rfAQgSY3n5Hr8q7bP3OHAnBYa7X9Z7tSNI5UoiBI4EEoGmdemt79kWSQBJi4EgguV3UF0nG1RZioEkguV0wUJvUkIQYeBJIbu7B/uWSTQifkUByi7ywhiSPjwgx0CSQ3LrprS39kIQYOBJIbpFyl00IX5NAcrtgSm0JJCEGngSSm/u2f9M30GaXQBLCBySQ3EJjwdj+EG19ZUcbUlMrg+DpGiGuChJIbprWqWG7Uq8hORXU26VzpBADQQKps8iOwf5Dgo2Yglz/PHLZJsTAkEDqrJve2l83tPiqREIEFAmkzi54ni01MRKAvf/42lclEiKgSCB1Ful56//O1HgASj6v7G4PIUQ/kkDqLMJzfrY7Ul2XcB9/8Y20IwkxACSQOrughnTN0DDGxkfgcCre//t5HxZMiMAggdTZBYO0Adwx3nXZtufzc74okRABRQKpM3cNqaEKHK6+R3e2X7btOXYOh1M6SApxJfUpkDZv3kxKSgohISFkZmayb9++Hrf/wx/+QGpqKiEhIUyaNIm33nrL4/1FixahaZrHkpOT05eiXZ6wONCMgIIGV43o29fEEB0azDeNrRw89c3Al0mIAOJ1IO3evZv8/HwKCws5cOAAU6ZMITs7m3Pnur6k+fDDD5k3bx4/+clP+PTTT5k9ezazZ8/mr3/9q8d2OTk5nD17Vl927tzZtzO6HAbDRZdtQUYDt41zzTpSclQu24S4krwOpKeeeorFixeTl5fHddddx9atWwkLC+OFF17ocvvf/va35OTk8OCDDzJ+/HjWrFnDt7/9bZ599lmP7cxmM4mJifoSGxvbtzO6XBcM1AZwZ3s70nvSjiTEFeVVILW0tLB//36ysrI6DmAwkJWVRVlZWZf7lJWVeWwPkJ2dfdH2paWlxMfHc+2113LfffdRXV3dbTnsdju1tbUeS7/Rb/2f1VfdNm4YBg0+t9Zx+pvG/vssIYQHrwKpqqoKh8Nx0YyyCQkJWK3WLvexWq2X3D4nJ4eXXnqJkpIS1q1bx5///GdmzpyJw+Ho8phFRUVER0frS3Jysjen0TP9ebaOGlJMmIn0EUMAudsmxJXkF3fZ5s6dy/e+9z0mTZrE7NmzeeONN/j444/1qbYvtGLFCmw2m76cOnWq/wrjHuy/3jNg75DLNiGuOK8CKS4uDqPRSGWl56MUlZWVJCYmdrlPYmKiV9sDjBo1iri4OE6cONHl+2azmaioKI+l30RcXEMCuKP9MZIPTlbT2CLDkQhxJXgVSCaTialTp1JSUqKvczqdlJSUMG3atC73mTZtmsf2AO+++2632wOcPn2a6upqkpKSvCle/7igt7bb2PgIvhUbSkubkw9PdN++JYToO68v2fLz83nuuefYvn07R48e5b777qOhoYG8vDwAFixYwIoVK/Tt//Vf/5Xi4mKefPJJPv/8c1avXs0nn3zCsmXLAKivr+fBBx/ko48+4osvvqCkpIR77rmHMWPGkJ2d3U+n6YVuakiapnV62FYu24S4EoK83SE3N5fz589TUFCA1WolLS2N4uJiveG6oqICg6Ej52666SZ27NjBypUreeSRRxg7diyvv/46EydOBMBoNHLo0CG2b99OTU0NFouFGTNmsGbNGsxmcz+dphf0GlIlOB1gMOpv3TE+ge1lX/Le55UoNRFN0wa+fEIMYpoaBANG19bWEh0djc1mu/z2JEcbrIkDFCw/DhHx+lvNrQ6+veZdGlscvHH/LUwcHn15nyVEgOjtd9Qv7rL5FWMQhLt6Znd+yBYgJNjIzWPiALnbJsSVIIHUlS56a7u525EkkITofxJIXblgWu3Obm8PpM9O13C+zj6QpRJi0JNA6krkxeMiuSVEhTBpeDRKQekxqSUJ0Z8kkLoS0XVfJLc75LJNiCtCAqkrkd1fskHH0//v//08LW3OgSqVEIOeBFJXOvdF6sJESzRxEWYaWhzsK5cpkoToLxJIXblg9pELGQwad6S6ugbIZZsQ/UcCqSv6bX8rdNNv1D1FUsnnlQyCvqVC+AUJpK64n2dztEBT1+No3zI2DpPRwJfVjfyjqmEACyfE4CWB1JUgM4S6BmTrrmE7whxE5ijXNu/JWNtC9AsJpO50MwxJZzLVthD9SwKpO90MQ9KZTLUtRP+SQOpOL2pI1wwNY4xMtS1Ev5FA6k4X02p3xX3ZJoP/C3H5JJC64x7s/xKB5H6MRKbaFuLySSB1p4chSDqbOiKWqJAgmWpbiH4ggdSdHoYg6SzIaOC2a9vvtsntfyEui9djageMzjUkpaCH8bPvTI3n/312hvc+P8dDOamXPLRSCodTEWSU/x8Meo42cLaBZmhftI6f3W3f9A00fQ2NX0Njdcdr/ec3rp/BoZA0xbVY0iBmRI9/p1cDCaTuuGtIrY1QU+H6o2qugWYbNLX/bK6Bphpm1n9DUPAxtGon//VUKieCRnNMG81ZZwz2Ngf2Vif2NqfrdZtTHyFgWKSZ5NhQvhUbRvIQ189vxYaSHBtGUkwI5iBjd6UT/qa5FqqOQ9UxqPo7nP+76+fX/wDV9QzMoHmGFBo4vBz072SnKcZCY9sDKs0VUElpEJtyVYWUDPLfk6JksNf2efdKFcNh50j+pkZy2DmSvzpTsDIEuPQfSLDWxqgIJ+OiHYwIayVS1RHiqCW01UaYo5ZQRy1hjlrC3YuzjnBnLeHOBpq1EOoMUdRqkdRpkdRqkdi0KGxEUkOE/vMbFUlbUChaUCiayYwxOBRjcAjBphBCTEGEBBsIDTYSajISEmwkyNB1ubv6ezdoGqYgA8FG12Kmhejms0Q2nyG8+SvCG04T2nCakIbTGFsbaDPH0GaOpdW9mGJoaf9pN8ViN0VjD46lNTgKQ7AJo6YRZNAwGC74qWkEGTWMmoamaTjba6P6ohTOVjvYazE028Beh6GlFkNLHaDQ3DUZXPtr7WGhaei1HINyENpwmtDak4TaThJSc4Lgxv7tHKtCYiBsCCp0CCo0Ftp/qtAhqJAhEBqLs+kbtLMHMVg/I6jqczTnxX3h2kzR1A+ZQFNsKo6oZLSYZAwxyQQNScYcNYxQUxDBl1FTV0qhlOuB85709jvap0DavHkzGzZswGq1MmXKFJ555hkyMjK63f4Pf/gDq1at4osvvmDs2LGsW7eOf/qnf/I4qcLCQp577jlqamq4+eab2bJlC2PHju1Vea5YIL26EI687nptioCQGAiJhtCYC15HU0s4JytrGFL3d4bY/kZEfTmaunisJEdYHI74yTiGjqOxsRF7/de0NdZAsw1DSx3mtjrCnPWEa74dHteugrETjJ0g7Jiwq2BaCKYZE83KRLP7tf67CTvt65WJcK2ZZO0c39KqSNbOkaDV9FvZ2pSh/fNcZXCV1fP3Zky0YSCSJiK1JqJoIFJrIpJGQrWWfitLZ5UqhpNOCyfUcE4qCyeUhZNOC/WEogEaTgwoNFT7z4vXNWHCRgROL5t3TbRyrXaKSYZyJmrlTDSUk6pVYNK6q51BozJzRg3lLHFUanGcNyZQHTSMWmMsyglKOcDpdNXwlBOn04lBOVDKCU4nSjnQlOKmW7OYk3NXj+W7YoG0e/duFixYwNatW8nMzGTTpk384Q9/4NixY8THx1+0/Ycffsitt95KUVER3/3ud9mxYwfr1q3jwIED+txs69ato6ioiO3btzNy5EhWrVrF4cOHOXLkCCEhIZcs0xULJKVc1+vmSDAGe7dvSwNY/wpnP4OzB10/zx3tofretdagcOzGCJqDo2kJjsYeHE2LKZrW4BhaTDG0mqJpNUfTZorFYY6mzRSF2dlMSFsNphYb5tYazK02TC01mFpqCLbXENxSQ5D9a4LsNRjamtAcLRgcdjSubGW5UQuj0pCA1RDPWS2er4jntIqnVoURo9UTQy0x1BNLHVGqlhjqiFJ1RKtaIlUdEaoeQz+WsYFQGrVwGgzhNGmhKAzQHg6aUh2v2z9TUwoNJwo4pw3jS2045XyLk1j4hxqOTYXqNbE2p8KpXD8H4hok2KgRbDQQZGj/adQI0RyM004xnn+Q7DjNUMc54p3nSVTnidNs/fbZH43J58b/r7DHba5YIGVmZnLDDTfw7LPPAq6ptJOTk7n//vv59a9/fdH2ubm5NDQ08MYbb+jrbrzxRtLS0ti6dStKKSwWC7/61a9Yvnw5ADabjYSEBLZt28bcuXMvWaYrFkj9rbUJKo/A2U/h63IwhbtqWSHRYI7qeN15nXGAmvmUAkcrtDVDm93VltFmb/+9ueN1azO0NXXzs9l1jm3NEBQCsSNcDa0x17jaMkJjL689w+lwXUK7P0v/PLurDG32js9va3Y1EJsjISTqgn/jKNdrw8C00bkva5xK4VSg8PzdqRTKCQ6l9It5TQMNzdXM1L5SwzWDstb+vqH9stVo0LyetFS1NtHyzWlaqytwfFOBs+YUmu00xtrTGJqrXZeuBmP7ZaoRzdD+s32963cDmsEI316AadLsHj+vt99Rr/7aW1pa2L9/v8dU2QaDgaysLMrKyrrcp6ysjPz8fI912dnZvP766wCUl5djtVrJysrS34+OjiYzM5OysrIuA8lut2O3d1zS1Nb2vZ1nQAWHwremuhZ/o2kQZHIt/spgdIVaqK8L4h1XWxQYetF2OFC04FDM8WMxx/euWWSgeHWhWlVVhcPh0KfNdktISMBq7bq/jtVq7XF7909vjllUVER0dLS+JCcne3MaQgg/dVV2hFmxYgU2m01fTp065esiCSH6gVeBFBcXh9FopLLS8xZnZWUliYmJXe6TmJjY4/bun94c02w2ExUV5bEIIa5+XgWSyWRi6tSplJR0dMZyOp2UlJQwbdq0LveZNm2ax/YA7777rr79yJEjSUxM9NimtraWvXv3dntMIcQgpby0a9cuZTab1bZt29SRI0fUkiVLVExMjLJarUoppX784x+rX//61/r2H3zwgQoKClIbN25UR48eVYWFhSo4OFgdPnxY32bt2rUqJiZG/fd//7c6dOiQuueee9TIkSNVU1NTr8pks9kUoGw2m7enI4QYAL39jnodSEop9cwzz6hrrrlGmUwmlZGRoT766CP9vdtuu00tXLjQY/tXX31VjRs3TplMJjVhwgT15ptverzvdDrVqlWrVEJCgjKbzerOO+9Ux44d63V5JJCE8G+9/Y7KoyNCiCvuivRD8lfuTL1q+iMJEWDc381L1X8GRSDV1dUBSH8kIfxcXV0d0dHR3b4/KC7ZnE4nZ86cITIy8pJd6Gtra0lOTubUqVOD8vJuMJ/fYD43GNznp5Sirq4Oi8WCwdD9zf1BUUMyGAx861vf8mqfwd5/aTCf32A+Nxi859dTzcjtquypLYQYnCSQhBB+I+ACyWw2U1hYiNls9nVRrojBfH6D+dxg8J9fbwyKRm0hxOAQcDUkIYT/kkASQvgNCSQhhN+QQBJC+I2AC6TNmzeTkpJCSEgImZmZ7Nu3z9dFumyrV6/W5xBzL6mpl55B11+9//77zJo1C4vFgqZp+vjrbkopCgoKSEpKIjQ0lKysLI4fP+6bwvbBpc5v0aJFF/33zMnJ8U1hB1hABdLu3bvJz8+nsLCQAwcOMGXKFLKzszl37pyvi3bZJkyYwNmzZ/XlL3/5i6+L1GcNDQ1MmTKFzZs3d/n++vXrefrpp9m6dSt79+4lPDyc7OxsmpubB7ikfXOp8wPIycnx+O+5c+fOASyhD13RQVD8TEZGhlq6dKn+u8PhUBaLRRUVFfmwVJevsLBQTZkyxdfFuCIA9dprr+m/O51OlZiYqDZs2KCvq6mpUWazWe3cudMHJbw8F56fUkotXLhQ3XPPPT4pj68FTA3JPYVT5+mWLjWF09Xk+PHjWCwWRo0axfz586moqPB1ka6IS02bNViUlpYSHx/Ptddey3333Ud1dbWvizQgAiaQ+jKF09UiMzOTbdu2UVxczJYtWygvL2f69On6sCyDSV+mzbra5OTk8NJLL1FSUsK6dev485//zMyZM3E4vJv1+Go0KJ72D3QzZ87UX0+ePJnMzExGjBjBq6++yk9+8hMflkz0RefJUSdNmsTkyZMZPXo0paWl3HnnnT4s2ZUXMDWkvkzhdLWKiYlh3LhxnDhxwtdF6Xd9mTbrajdq1Cji4uIG5X/PCwVMIPVlCqerVX19PSdPniQpKcnXRel3gTht1unTp6murh6U/z0vFFCXbPn5+SxcuJD09HQyMjLYtGkTDQ0N5OXl+bpol2X58uXMmjWLESNGcObMGQoLCzEajcybN8/XReuT+vp6j9pAeXk5Bw8eZMiQIVxzzTU88MADPPHEE4wdO5aRI0eyatUqLBYLs2fP9l2hvdDT+Q0ZMoTHHnuMOXPmkJiYyMmTJ3nooYcYM2YM2dnZPiz1APH1bb6B1tMUTler3NxclZSUpEwmkxo+fLjKzc1VJ06c8HWx+mzPnj0KuGhxT691udNm+VpP59fY2KhmzJihhg0bpoKDg9WIESPU4sWL9XkPBzsZfkQI4TcCpg1JCOH/JJCEEH5DAkkI4TckkIQQfkMCSQjhNySQhBB+QwJJCOE3JJCEEH5DAkkI4TckkIQQfkMCSQjhNySQhBB+4/8HOHVh0JRAbsgAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:44:52.347609Z", + "iopub.status.busy": "2024-02-29T03:44:52.347258Z", + "iopub.status.idle": "2024-02-29T03:46:47.193644Z", + "shell.execute_reply": "2024-02-29T03:46:47.192628Z" + }, + "papermill": { + "duration": 114.870137, + "end_time": "2024-02-29T03:46:47.196437", + "exception": false, + "start_time": "2024-02-29T03:44:52.326300", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:46:47.238851Z", + "iopub.status.busy": "2024-02-29T03:46:47.238042Z", + "iopub.status.idle": "2024-02-29T03:46:47.258116Z", + "shell.execute_reply": "2024-02-29T03:46:47.257224Z" + }, + "papermill": { + "duration": 0.043451, + "end_time": "2024-02-29T03:46:47.260225", + "exception": false, + "start_time": "2024-02-29T03:46:47.216774", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.00.0001430.0016994.4409030.0274120.4385460.0396430.0000042.2897470.029980.0590640.041220.0695670.0120216.730649
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.0 0.000143 0.001699 4.440903 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.027412 0.438546 0.039643 0.000004 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 2.289747 0.02998 0.059064 0.04122 0.069567 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.012021 6.730649 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:46:47.301125Z", + "iopub.status.busy": "2024-02-29T03:46:47.300408Z", + "iopub.status.idle": "2024-02-29T03:46:47.883587Z", + "shell.execute_reply": "2024-02-29T03:46:47.882815Z" + }, + "papermill": { + "duration": 0.605575, + "end_time": "2024-02-29T03:46:47.885910", + "exception": false, + "start_time": "2024-02-29T03:46:47.280335", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:46:47.928276Z", + "iopub.status.busy": "2024-02-29T03:46:47.927556Z", + "iopub.status.idle": "2024-02-29T03:48:50.390744Z", + "shell.execute_reply": "2024-02-29T03:48:50.389505Z" + }, + "papermill": { + "duration": 122.48825, + "end_time": "2024-02-29T03:48:50.394545", + "exception": false, + "start_time": "2024-02-29T03:46:47.906295", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:48:50.446988Z", + "iopub.status.busy": "2024-02-29T03:48:50.446610Z", + "iopub.status.idle": "2024-02-29T03:48:50.464112Z", + "shell.execute_reply": "2024-02-29T03:48:50.463389Z" + }, + "papermill": { + "duration": 0.042961, + "end_time": "2024-02-29T03:48:50.466138", + "exception": false, + "start_time": "2024-02-29T03:48:50.423177", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:48:50.506604Z", + "iopub.status.busy": "2024-02-29T03:48:50.506277Z", + "iopub.status.idle": "2024-02-29T03:48:50.511596Z", + "shell.execute_reply": "2024-02-29T03:48:50.510763Z" + }, + "papermill": { + "duration": 0.028239, + "end_time": "2024-02-29T03:48:50.513837", + "exception": false, + "start_time": "2024-02-29T03:48:50.485598", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.5738095187005543}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:48:50.556434Z", + "iopub.status.busy": "2024-02-29T03:48:50.555590Z", + "iopub.status.idle": "2024-02-29T03:48:50.957428Z", + "shell.execute_reply": "2024-02-29T03:48:50.956520Z" + }, + "papermill": { + "duration": 0.425833, + "end_time": "2024-02-29T03:48:50.959661", + "exception": false, + "start_time": "2024-02-29T03:48:50.533828", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFfElEQVR4nO3deXhTZdo/8O/JvjRJ932DFiiLQGUTEVlkQEAE9XUcRWxF1BlhGOX1dynjpYgbOuMCbjjjDCAjyrzOgDIiIqMCbqiALAKytqV0L23TpNlznt8fJw0NXZOmOU1zf64rV5Nznpzcp8vd5zznWTjGGAMhhIhIInYAhBBCiYgQIjpKRIQQ0VEiIoSIjhIRIUR0lIgIIaKjREQIER0lIkKI6CgREUJER4kozGRnZ+OGG27o0c/gOA5PPvlkp+WefPJJcBzXo7GQyECJKMS+/fZbPPnkk2hoaBA7FBIG3nzzTWzYsEHsMHocJaIQ+/bbb7Fy5UpKRKRLKBERQkiIUCIKoSeffBL/7//9PwBAv379wHEcOI5DcXEx1q9fj6lTpyIxMRFKpRJDhgzB2rVr2z3WZ599hpEjR0KlUmHIkCHYsmWL3/HY7XY89NBDSEhIgE6nw4033ogLFy60Wfbrr7/GmDFjoFKpkJOTg7/85S9tluM4DkuWLMGmTZswaNAgqFQqjBo1Cnv37m31veA4DqdOncKdd94Jg8GAhIQEPP7442CMobS0FHPnzoVer0dycjJeeuklv88PAHbs2IFJkyZBp9NBr9djzJgxeO+993zKfPDBBxg1ahTUajXi4+Nx5513oqyszKdMYWEhoqKiUFZWhnnz5iEqKgoJCQl4+OGH4Xa7fcryPI81a9bgiiuugEqlQkJCAq6//nrs37/fW6YrP+/s7GwcO3YMe/bs8f6uTJ48OaDvQ6/HSMgcPnyY3X777QwAe+WVV9g//vEP9o9//IOZzWY2ZswYVlhYyF555RX22muvsenTpzMA7PXXX/c5RlZWFhs4cCCLjo5mjz76KHv55ZfZFVdcwSQSCfvss8/8iufOO+9kANgdd9zBXn/9dXbzzTez4cOHMwBsxYoV3nJHjhxharWaZWZmslWrVrGnn36aJSUlecu2BIANGzaMxcfHs6eeeoq98MILLCsri6nVanb06FFvuRUrVjAAbOTIkez2229nb775Jps9ezYDwF5++WU2aNAg9rvf/Y69+eabbMKECQwA27Nnj1/nt379esZxHBs2bBh79tln2RtvvMEWLVrEFixY4FMGABszZgx75ZVX2KOPPsrUajXLzs5m9fX13nIFBQVMpVKxoUOHsoULF7K1a9eyW265hQFgb775ps/nFhYWMgBs5syZbPXq1ezFF19kc+fOZa+99pq3TFd+3lu3bmXp6eksLy/P+7vi7884XFAiCrE///nPDAArKiry2W6xWFqVnTFjBuvfv7/PtqysLAaA/fvf//ZuMxqNLCUlheXn53c5jkOHDjEA7IEHHvDZfscdd7RKRPPmzWMqlYqVlJR4tx0/fpxJpdI2ExEAtn//fu+2kpISplKp2E033eTd1pyI7rvvPu82l8vF0tPTGcdx7Pnnn/dur6+vZ2q1mhUUFHT5/BoaGphOp2Pjxo1jVqvVZx/P84wxxhwOB0tMTGTDhg3zKfPxxx8zAOyJJ57wbisoKGAA2FNPPeVzrPz8fDZq1Cjv6y+++IIBYEuXLm0VU/PnMtb1n/fQoUPZpEmTunDG4Y0uzXoJtVrtfW40GlFbW4tJkybh3LlzMBqNPmVTU1Nx0003eV/r9Xrcdddd+Omnn1BZWdmlz/vkk08AAEuXLvXZ/uCDD/q8drvd2LlzJ+bNm4fMzEzv9sGDB2PGjBltHnv8+PEYNWqU93VmZibmzp2LnTt3trqMWbRokfe5VCrF6NGjwRjDPffc490eHR2NQYMG4dy5c106NwDYtWsXTCYTHn30UahUKp99zV0O9u/fj+rqajzwwAM+ZWbPno28vDxs37691XF/+9vf+ryeOHGiT1z//ve/wXEcVqxY0eq9Lbs6+PPzjgSUiHqJb775BtOmTYNWq0V0dDQSEhLwxz/+EQBa/WLm5ua26r8zcOBAAEBxcXGXPq+kpAQSiQQ5OTk+2wcNGuTzuqamBlarFQMGDGh1jMvLNmur7MCBA2GxWFBTU+OzvWVyAwCDwQCVSoX4+PhW2+vr69s/ocucPXsWADBs2LB2y5SUlABo+zzy8vK8+5s1t/e0FBMT4xPX2bNnkZqaitjY2A7j8+fnHQlkYgdAhF/e6667Dnl5eXj55ZeRkZEBhUKBTz75BK+88gp4nhc7xB4jlUq7tA0AmMizGrcXl78i+efdHkpEIdZWT+T//Oc/sNvt2LZtm08N4csvv2zzGGfOnAFjzOdYp06dAiDcaemKrKws8DyPs2fP+tQITp486VMuISEBarUap0+fbnWMy8s2a6vsqVOnoNFoWtUoekpzTe/nn39Gbm5um2WysrIACOcxdepUn30nT5707vf3c3fu3Im6urp2a0X+/Lwjpec6XZqFmFarBQCfDo3N/2lb/sc3Go1Yv359m8coLy/H1q1bva8bGxuxceNGjBw5EsnJyV2KY+bMmQCAV1991Wf76tWrfV5LpVLMmDEDH374Ic6fP+/dfuLECezcubPNY3/33Xc4ePCg93VpaSk++ugjTJ8+PWi1is5Mnz4dOp0Oq1atgs1m89nX/H0ePXo0EhMT8dZbb8Fut3v379ixAydOnMDs2bP9/txbbrkFjDGsXLmy1b7mz/Xn563VaiOi8yvViEKsuRH3sccew29+8xvI5XJce+21UCgUmDNnDu6//36YzWa8/fbbSExMREVFRatjDBw4EPfccw9+/PFHJCUlYd26daiqqmo3cbVl5MiRuP322/Hmm2/CaDTi6quvxueff44zZ860Krty5Up8+umnmDhxIh544AG4XC689tprGDp0KI4cOdKq/LBhwzBjxgwsXboUSqUSb775pvc4oaLX6/HKK69g0aJFGDNmDO644w7ExMTg8OHDsFgseOeddyCXy/HCCy/g7rvvxqRJk3D77bejqqoKa9asQXZ2Nh566CG/P3fKlClYsGABXn31VZw+fRrXX389eJ7HV199hSlTpmDJkiWYPn16l3/eo0aNwtq1a/HMM88gNzcXiYmJrWpvfYJ4N+wi19NPP83S0tKYRCLx3srftm0bGz58OFOpVCw7O5u98MILbN26da1u9WdlZbHZs2eznTt3suHDhzOlUsny8vLYBx984HccVquVLV26lMXFxTGtVsvmzJnDSktLW92+Z4yxPXv2sFGjRjGFQsH69+/P3nrrLe8t+JYAsMWLF7N3332XDRgwgCmVSpafn8++/PJLn3LN762pqfHZXlBQwLRabatYJ02axIYOHer3OW7bto1dffXVTK1WM71ez8aOHcvef/99nzL//Oc/WX5+PlMqlSw2NpbNnz+fXbhwoUtxtfU9cLlc7M9//jPLy8tjCoWCJSQksJkzZ7IDBw74xNWVn3dlZSWbPXs20+l0DECfvZXPMUbrmpHg4TgOixcvxuuvvy52KCSMUBsRIUR01EbUB3XWqVGtVsNgMIQomuCrqalp1TGyJYVC0Wk/HtK7UCLqg1JSUjrcX1BQENZTS4wZM6ZVZ8OWJk2ahN27d4cuINJtlIj6oF27dnW4PzU1tcc+OxRNjps2bYLVam13f0xMTI/HQIKLGqsJIaKjxmpCiOjC+tKM53mUl5dDp9NFTFd4QsIJYwwmkwmpqamQSNqv94R1IiovL0dGRobYYRBCOlFaWor09PR294d1ItLpdACEk9Tr9SJHQwi5XGNjIzIyMrx/q+0J60TUfDmm1+spERHSi3XWdEKN1YQQ0VEiIoSIjhIRIUR0Yd1G1BWMMbhcrg7HJpG2SaVSyGQy6hpBelyfTkQOhwMVFRWwWCxihxK2NBoNUlJSoFAoxA6F9GGiJ6KysjI88sgj2LFjBywWC3Jzc7F+/XqMHj26W8fleR5FRUWQSqVITU2FQqGg/+x+YIzB4XCgpqYGRUVFGDBgQIcd0gjpDlETUX19PSZMmIApU6Zgx44dSEhIwOnTp4MyaNHhcIDneWRkZECj0QQh2sijVqshl8tRUlICh8PRan2wPo0x4Pw+oPYkoE8H+k8CpHKxo+qzRE1EL7zwAjIyMnzmWu7Xr19QP4P+i3dPxH7/zn0JnP9eeN5YATgtwJAbxY2pDxP1t2zbtm0YPXo0br31ViQmJiI/Px9vv/12u+XtdjsaGxt9HoQEXWM5UPqD8DxtFMBxQNUxwHhB3Lj6MFET0blz57B27VoMGDAAO3fuxO9+9zssXboU77zzTpvlV61aBYPB4H3QODPSI4q/Fi7NkoYCA6cDSZ7VYssOdvw+EjBRExHP87jyyivx3HPPIT8/H/fddx/uvfdevPXWW22WX758OYxGo/dRWloa4oj7nuzs7FZrmUU0cw1w8axQC8q+RtiWdqXwteYk4HKIF1sfJmoiSklJwZAhQ3y2DR482Gchv5aUSqV3XBmNLyM9otKzTltcLqDxzHutSwHU0QDvAhran6KWBE7URDRhwoRWyxafOnUqoKV+I5nDQf+lg4LngerjwvPk4Ze2cxwQKyxhjYtnQx9XBBA1ET300EPYt28fnnvuOZw5cwbvvfce/vrXv2Lx4sU9+rkOF9/uw+Xmu1zW2YWygZg8eTKWLFmCJUuWwGAwID4+Ho8//rh3Pujs7Gw8/fTTuOuuu6DX63HfffcBAL7++mtMnDgRarUaGRkZWLp0KZqamrzHra6uxpw5c6BWq9GvXz9s2rQpoPj6rIYSwG4G5CogLsd3X0y28NVIzQE9QdTb92PGjMHWrVuxfPlyPPXUU+jXrx9Wr16N+fPn9+jnvvFl62WVm/WL12Jefpr39V/3noXT3fa03ukxatw6+lKD+bpvimB1+A4leehXAwOK8Z133sE999yDH374Afv378d9992HzMxM3HvvvQCAF198EU888QRWrFgBADh79iyuv/56PPPMM1i3bh1qamq8yay5e0RhYSHKy8vx5ZdfQi6XY+nSpaiurg4ovj6p+oTwNWEwIJH67tN7FhywXAScNiFZkaARvWf1DTfcgBtuuEHsMHqdjIwMvPLKK+A4DoMGDcLRo0fxyiuveBPR1KlT8b//+7/e8osWLcL8+fPx4IMPAgAGDBiAV199FZMmTcLatWtx/vx57NixAz/88APGjBkDAPj73/+OwYMHh/zceiXGgIuef1AJg1rvV0YJ7UTWBsBUAcQGt79bpBM9EYlh8ZTcdvdJLhsFct+1OW0XhNB00NLCCcH75bzqqqt8hqSMHz8eL730knfw7uVDYA4fPowjR474XG4xxrxDXU6dOgWZTIZRo0Z59+fl5SE6OjpoMYc1UwXgaBJ6T0dntl1GnyokosZySkRBFpGJSCHretNYT5XtLq1W6/PabDbj/vvvx9KlS1uVzczMxKlTp0IVWnjy1IZ+cSTg7M/VkEs5KGQSKGQSKGUSJOlVSItKAld1HDBXiRxs3xORiSgcfP/99z6v9+3bhwEDBkAqlbZZ/sorr8Tx48eRm9t2bS8vLw8ulwsHDhzwXpqdPHkSDQ0NQY07bHkSURFScKrK1GaRwUo3pvE8ZE21oYwsIkToQKLe7/z581i2bBlOnjyJ999/H6+99hr+8Ic/tFv+kUcewbfffoslS5bg0KFDOH36ND766CMsWbIEADBo0CBcf/31uP/++/H999/jwIEDWLRoEdRqdahOqddxuHh8+FMZGo11gKkK4Djk5o3AlLxEXDMgHmP7xWJEhgEDkqIgl3KwK+KEN1rrAbdL3OD7GKoR9VJ33XUXrFYrxo4dC6lUij/84Q/e2/RtGT58OPbs2YPHHnsMEydOBGMMOTk5uO2227xl1q9fj0WLFmHSpElISkrCM888g8cffzwUp9Mr7T5ZjaLaJuyv/xlTOQC6FAxIT2qzrNHiRJRSCul3GuGumeUioGu7LPEfJaJeSi6XY/Xq1Vi7dm2rfcXFxW2+Z8yYMfjss8/aPWZycjI+/vhjn20LFizoVpzhqqzBimPljeA4YIS2HrCgdd+hFgwazxQg2gSgoRRoqqFEFER0aUYiDmMMX52qAQAMTdEhzlkh7Ijp/E6YWWJA8cUmVFaW92SIEYcSEYk4ZQ1WVBhtkEk4TEhyCpdaMqUwpqwTZ8xyVBhtKKdEFFR0adYL7d69W+wQ+rQDJfUAgCGpemjMnk6MMVlAFyaBS0tJwZlfAGNdNdw8g/TyjmckIFQjIhHFbHehqFYYf5efGQPUFws7mseSdSI+IRkKKQeZ3YjyBmvPBBmBKBGRiCLlOFzVPw55yTrEKgE0lgk7utA+BACcOgZ6tRwy3o7y2vqeCzTC0KUZiShqhRRX9ff0B7p4FuDdgMoAqLu4YINMAY0uGjBX42JNJTAwrdO3kM5RjYhErvoi4WtMduuBgx3QxSQAENqJeL7tmRmIfygRkYhxptqE01WmS/NE1XkSkZ8DWKOiEyCVcFC6TDDZqId1MNClGYkY352rQ63JjhlDkzEkjgOaaoWaULR/M4JKVAaMzIjGmPQocBpa6ywYqEZEIoLR4kStyQ4Jx6F/gvbS3bKoJEDh5wKcKj0UUgk4e9uDY4n/KBGRiHCmxgwASItRQyWX+n3b3odSJ3y107p6wUKJiESEc55ElJOgFWZjbE5EgUxwpjTA7nLjl+IL+L8f215xhvgnshIRY8K6VKF+sK7fWdm4cSPi4uJgt9t9ts+bNy9iB6h2l93lRnmDDQDQPz5KaBuymwGpTFjX3l8qPaQSDvWmJlTWGWFzujt/D+lQZDVWu53AVy+F/nMn/i8gU3Sp6K233oqlS5di27ZtuPXWWwEIq29s3769w5H1pH0X6q3gGUO0Ri6Moi8tFnYYMoVk5C+pHDKlFgppA5QuM4xWp3C5RwIWWTWiMKBWq3HHHXd4V94AgHfffReZmZmYPHmyeIGFsepGoXaZFedplG7ZfyhQKj1UcikUbjMaLM7uBUgirEYklQu1EzE+1w/33nsvxowZg7KyMqSlpWHDhg0oLCz0mUyfdN34nDgMSdUDDEJP6gZPu053EpFSSERKlxn1FlrgsrsiKxFxXJcvkcSUn5+PESNGYOPGjZg+fTqOHTuG7du3ix1WWDOoPf8M6kuES3SFBohKDPyAKoM3EVGNqPsiKxGFkUWLFmH16tUoKyvDtGnTkJGR0fmbSOda3rbvTg1TqYdKLoHC1oQGqhF1m6htRE8++SQ4jvN55OXliRlSr3HHHXfgwoULePvtt7Fw4UKxwwlbnx2rxEeHylBh9EzZ4U1E3VyXTKmDSi6Fmm+iOYmCQPQa0dChQ/Hf//7X+1omEz2kXsFgMOCWW27B9u3bMW/ePLHDCUuMMRTVNsHicGNUVgzgtAoLKQLdax8CAIUWGoUU1+VowY2m2mp3if5XL5PJkJycLHYYvVJZWRnmz58PpVIpdihhqd7ihMXhhkzCIVmvAupOC326tPGASt+9gyt14MABDnNwgo1wot++P336NFJTU9G/f3/Mnz8f58+331PVbrejsbHR59EX1dfXY+vWrdi9ezcWL14sdjhhq6xeuBxLNqggk0oujbbvbm0IABSelXbdTsBl77gs6ZSoiWjcuHHYsGEDPv30U6xduxZFRUWYOHEiTKa2BxOuWrUKBoPB++irDbj5+fkoLCzECy+8gEGDBokdTtgqa7AAANKiPYtIdmd82eVkSkAqR1mDFR989wt+LjN2/5gRTNRLs5kzZ3qfDx8+HOPGjUNWVhb+7//+D/fcc0+r8suXL8eyZcu8rxsbG/tkMmpv3TLinzLPsI60GDVgMwortHISIDozOB+g1MHpbkBjQz3qLanBOWaEEr2NqKXo6GgMHDgQZ86caXO/Uqmk9hLSJUarE41WJyQchxSDGqg9JuzQJQu1mWBQaKGQSSB3W2iCtG4SvY2oJbPZjLNnzyIlpfP1pbqK+THglLQWrt8/u8uN9Bg1UqJVUMgkQkdGIHi1IQBQREEpk0DhtsBko06N3SFqjejhhx/GnDlzkJWVhfLycqxYsQJSqRS33357t48tlws9aS0WC9RqdbePF6ksFqGdpfn7GS4SdSrcOjpDSKSMAQ2eRBTj32yMHVJGQSmTQm6xoJ5qRN0iaiK6cOECbr/9dly8eBEJCQm45pprsG/fPiQkJHT72FKpFNHR0aiurgYAaDQaGqvlB8YYLBYLqqurER0dDak0PEeXcxwHWOoAWyMgkQKGILYpKqKgkEmg4C0w21204GI3iJqINm/e3KPHb+6f1JyMiP+io6PDrp+Xy83DxbNLU3M0D3LVp/o9ALlDiijIpRyUvBWMCYs3ese0Eb/0qsbqYOM4DikpKUhMTITTSdfw/pLL5WFZEyprsGLLwTJkx2twU376pcuyYLYPAYAyChw4GKQOxGoVcLr54B4/gvTpRNRMKpWG5R8UCUylUbhtr5RJPdPCNieiILYPAYAiCgAwLlWOcVdnB/fYEaZX3TUjJBgqG4VElGxQAZaLgKMJkMgAfZBXZfUkIrjsQg9rEjBKRKRPYYx5a0TJetWlyzJDWmDTwnZEphQSHEBjzrqJEhHpUxqtLlgcbkglHBJ1yp67LAOE+YyUUbjYZMeWfaew+yTdFAlURLQRkcjRfFkWH6WETMIBxlJhR7AbqpvJNXDzDI2NRsBAE6QFimpEpE9pTkQpBpUwtsxhES6fdMHrre+jeZgHb0WTnTo1BopqRKRPSY9Rw+50Cyt2GM8KG/UpwW8faibXQCGVQO62os5B65sFihIR6VNyEqKQk+C5m/XLBeGrIYBFFLtKofHWiKwON1xuXpj7iPiFvmOk7zI2J6IenCpGroVUwkHFC5eETVQrCgglItJnNFgcqDbZ4OaZsKS0pU64sxXs/kMtKTTgwEErEWZppHaiwNClGekzDpU24KfzDcjPjMbkmDphozYekKt67kPlwuqxBqkTsVqFkASJ3ygRkT6jxiTUShJ0ytBclgHe3tUjkuQYQcM8AkaXZqRPYIyhxtwyEXn6D/VkQzUgrBgLAC4bwNOg10BRIiJ9QqPVBbuTh1TCIU7FAWZPL+eeTkQytdAOxRjgtPTsZ/VhlIhIn1BtEu5axUUpIDWVA4wX1i5TGXr2gyUSQK5GncWBf+07hf8er+rZz+ujKBGRPqG5fShRpwpd+1AzuQaMMTQ2NqCuiYZ5BIISEekTqls2VDeWCxt78rZ9Swot5FIJZG4bLA66fR8IumtG+oRRWTFIi1EjM0YNlDQnohCtNSbXQC4VVvOoc1KHxkBQIiJ9QkasBhmxGqETo9MmDHSNSgzNhyu0kEs5yHkr7E6ehnkEgL5bpG9pLBO+6pKEVTtCQa6BVMJB4RnmYaFakd8oEZGwV1pnwakqE8x2F9BYIWwM1WUZcGmYByc0VFtpvJnfKBGRsHfkghHbj1TgREVjixpRKBOR0LvaIBNW8+DDdHVcMfWaRPT888+D4zg8+OCDYodCwkxtc49qteRSR8ZQ1og8481GpShRcHU2Ugy0srC/ekUi+vHHH/GXv/wFw4cPFzsUEmacbh71FuGSKFFiFDoyKrQ935GxpeZhHs4moYc18ZvoichsNmP+/Pl4++23ERMTI3Y4JMxcNDvAGKBVSqGxeXo161OFYRehItcKX90uWlYoQKInosWLF2P27NmYNm1ap2XtdjsaGxt9HiSyNfeojo9q2ZExhJdlgLCMtVSGi012/PPbX/AlrebhN1H7EW3evBkHDx7Ejz/+2KXyq1atwsqVK3s4KhJOaszCLfMEnRKo9SSinpoovz0c51nNw4zGRiPkUbGh/fw+QLQaUWlpKf7whz9g06ZNUKm6NnHV8uXLYTQavY/S0tIejpL0drUmoX0oQekCbEbPjIwhrhEBPpPoW+j2vd9EqxEdOHAA1dXVuPLKK73b3G439u7di9dffx12u73VevVKpRJKpTLUoZJebPrQJNSY7Eh1ewa6auKEFVhDzTPeTO6wUSIKgGiJ6LrrrsPRo0d9tt19993Iy8vDI4880ioJEdKWaI0C0RoFcK5FQ7UY5BrIpBxkvA1WpxuMMXChbDAPc6IlIp1Oh2HDhvls02q1iIuLa7WdkE6J0aO6JbkaMs+lmZtnsLt4qOT0z7SraNArCVunq0xosDqRFatGoqm5oVqkRKTQQspxUEO4i2dzuikR+aFXJaLdu3eLHQIJIycqTThbbYYyU4JEl0NYzVWbIE4wzat5yByIi1LARat5+KVXJSJC/OGdlZFdFDboUoSpW8WgEDo1Xp2pwdWjs8WJIYyJ3qGRkEDYnG40WoVezLHuWmGjWO1DACD3jC9z0AT6gaBERMJS80BXnUoGhaVS2BiqqWHbIm8eb2ah8WYBoEREwlKt2TPQVcsBTZ4aUah7VLfkuTSrbGjCpu/O4PtzF8WLJQwFlIjOnTsX7DgI8Utz+1CapFEYca/UCcsHiUUqB6RyuHkGY0MDjFYa/OqPgBJRbm4upkyZgnfffRc2my3YMRHSqYueS7NEzrPGvV7E2lAzuQYyiTB3tZWmi/VLQIno4MGDGD58OJYtW4bk5GTcf//9+OGHH4IdGyHt+p9R6Zh/VSaS0HzHTMSG6mYKjTCJvttG08X6KaBENHLkSKxZswbl5eVYt24dKioqcM0112DYsGF4+eWXUVNTE+w4CfEhk0qQqFNBYRF5aEdLcq3Qu9ozzIN0Xbcaq2UyGW6++WZ88MEHeOGFF3DmzBk8/PDDyMjIwF133YWKiopgxUlIa3YzYGsURtzrksWOxlMjEoZ5UCLyT7cS0f79+/HAAw8gJSUFL7/8Mh5++GGcPXsWu3btQnl5OebOnRusOAnxOlBSj8+OVaKqzHPTRKwR95eTqyGTcJB51jdzU+/qLguoZ/XLL7+M9evX4+TJk5g1axY2btyIWbNmQeLp1dqvXz9s2LAB2dnZwYyVEADA2WozyhqsGBTtmfpDzP5DLcm1kEk4RHHCMA+nm4c0VGurhbmAEtHatWuxcOFCFBYWIiWl7bsViYmJ+Pvf/96t4Ai5HM8z1HjumMXyzT2qe8EdM0BY34zjMH2gHhiRLXY0YSWgRLRr1y5kZmZ6a0DNGGMoLS1FZmYmFAoFCgoKghIkIc3qLQ44XDzkEkBr99wU6Q13zIBLvasdTeLGEYYCaiPKyclBbW1tq+11dXXo169ft4MipD3Vno6M6UoLJG6RR9xfruUwD+KXgBIRa2csjdls7vL804QEoqpR6ECbJmsQNog54v5ynvXNzlfV4h/fFuF4Oa0y01V+XZotW7YMAMBxHJ544gloNBrvPrfbje+//x4jR44MaoCEtFTd1tQfvYVnfTOX04V6kxmNNhGHnIQZvxLRTz/9BECoER09ehQKhcK7T6FQYMSIEXj44YeDGyEhHowxON08ACCO9ySi3tCRsZlUBsgU3iljqS9R1/mViL788ksAwiT3a9asgV5PGZ+EDsdxmD8uCzabDYrvmseY9aJEBHgm0TdCzttgo2EeXRbQXbP169cHOw5CukxlqwHAhKk3lL3sn6FcA7mE1jfzV5cT0c0334wNGzZAr9fj5ptv7rDsli1buh0YIZfzLtFjarFiR29bskehFZYVctB4M390OREZDAbvOk0Gg6HHAiKkPR/svwCOA2Zw56EHet9lGSDUiDxtRI2UiLqsy4mo5eUYXZqRULO73Cg3WsEYoFR5akS96Y5ZM7kaMimHKIkDTgUN7+iqgNqIrFYrGGPe2/clJSXYunUrhgwZgunTpwc1QEIAoLrRDsaAWJkNSpdZvDXuO6PQQiWTYtYgPTA0S+xowkZAPcHmzp2LjRs3AgAaGhowduxYvPTSS5g7dy7Wrl0b1AAJAYAKo9CRMVveIGyISuwdI+4v5+1dbRU3jjAT8AyNEydOBAD861//QnJyMkpKSrBx40a8+uqrXT7O2rVrMXz4cOj1euj1eowfPx47duwIJCTSx1UYhT/sNK55oGu6iNF0QNGciGi8mT8CSkQWiwU6nQ4A8Nlnn+Hmm2+GRCLBVVddhZKSki4fJz09Hc8//zwOHDiA/fv3Y+rUqZg7dy6OHTsWSFikj2KMeWtE8c0j7g29ZOqPy3lqRKdKq/GP74pRWkfjzroi4MnzP/zwQ5SWlmLnzp3edqHq6mq/OjnOmTMHs2bNwoABAzBw4EA8++yziIqKwr59+wIJi/RRRqsTVocbcrigc3k6Mhp6aY3Ik4hcdjNqTXaYbC6RAwoPASWiJ554Ag8//DCys7Mxbtw4jB8/HoBQO8rPzw8oELfbjc2bN6Opqcl7vMvZ7XY0Njb6PEjf5+IZ+idoMUDdCCmYZ+mgXtqFxJOI5BJAytupL1EXBXTX7H/+539wzTXXoKKiAiNGjPBuv+6663DTTTf5dayjR49i/PjxsNlsiIqK8t59a8uqVauwcuXKQEImYSw+Som5I9OA4mKgCL33sgzwjDdTQiZtEoZ5UCLqkoDnT0hOTkZ+fr7P5Ghjx45FXl6eX8cZNGgQDh06hO+//x6/+93vUFBQgOPHj7dZdvny5TAajd5HaWlpoOGTcNRYJnw1ZIgbR2fkGsglHA3z8ENANaKmpiY8//zz+Pzzz1FdXQ2e5332+7MSrEKhQG5uLgBg1KhR+PHHH7FmzRr85S9/aVVWqVRCqeyFt2xJj3G4eNhdbuiUMsDYy+aobk/zah60rFCXBZSIFi1ahD179mDBggVISUnxDv0IBp7nYbfbg3Y8Et6KLzZh+5EK5OltmOmyC5c+UYlih9UxuQYyKQe5w0oj8LsooES0Y8cObN++HRMmTOjWhy9fvhwzZ85EZmYmTCYT3nvvPezevRs7d+7s1nFJ31FWL/QfSnB75qfWpwG9fWUMhRZyqQQ6qQO8vJfMHtnLBZSIYmJiEBsb2+0Pr66u9i7EaDAYMHz4cOzcuRO/+tWvun1s0jdcaPB0ZES1sCE6U8Roukiuhl4lx5ycaGBgL7+M7CUCSkRPP/00nnjiCbzzzjs+08X6i5YbIh2xOd2oNdkBxhDn8iwtHRaJSJgylibR77qAEtFLL72Es2fPIikpCdnZ2ZDL5T77Dx48GJTgSGS74LksS1E0QeG2AhJZ71k6qCMKWs3DXwElonnz5gU5DEJaK/NclvWXXQScEPoPSQP6lQ0tT6fGI0XlOGQtxpzhqYjRKjp5U2QL6Ke6YsWKYMdBSCvNDdVpnKehOhwuywBvInJYzLhodsBsd1Ei6kTATfoNDQ3429/+huXLl6OuThj/c/DgQZSVlQUtOBLZxmTHYGS6AXGuSmFDdJjM7+O5NFNxQvsW9SXqXEA1oiNHjmDatGkwGAwoLi7Gvffei9jYWGzZsgXnz5/3zlVESHcMSNJhgNYGVHr6D/XGGRnb4h1vxkHG22GlvkSdCqhGtGzZMhQWFuL06dM+K7vOmjULe/fuDVpwhKDhvPBVnx4e7UOA0M9JpoRcykHO0zCPrggoEf3444+4//77W21PS0tDZWVlt4Mi5Fi5EWUNVvAXzwobYsLksqyZp1OjsNAiTQXSmYD+xSiVyjan4Dh16hQSEhK6HRSJbA4Xj89PVIN3O3G/tAhqCYDYHLHD8o9nNQ8Zb6MaURcEVCO68cYb8dRTT8HpdAIQVuA8f/48HnnkEdxyyy1BDZBEngqjFW6eIYWrg0riFhZS7O3jyy6n0EAu5aCXOiCX0jCPzgT0HXrppZdgNpuRkJAAq9WKSZMmITc3FzqdDs8++2ywYyQRprROuG2fI60CBw6I7df7FlLsjFyDOK0SNw2NwYyhyWJH0+sFdGlmMBiwa9cufPPNNzh8+DDMZjOuvPJKTJs2LdjxkQhUWi/0SE7jKwApgNj+4gYUCDn1rvaH34mI53ls2LABW7ZsQXFxMTiOQ79+/ZCcnHxpSWBCAmRzulHVaIPc1YRYWQPAyYCYbLHD8p+Cxpv5w69LM8YYbrzxRixatAhlZWW44oorMHToUJSUlKCwsNDvaWIJuVxZg7CaayYqoZRJgaikS3/U4USuAQPDj6cvYON3xbA46M5ZR/yqEW3YsAF79+7F559/jilTpvjs++KLLzBv3jxs3LgRd911V1CDJJGjeVhHf65c2BA/QMRoukGuBgcOtiYTLpodsDjc0CjCpB+UCPyqEb3//vv44x//2CoJAcDUqVPx6KOPYtOmTUELjkSea3Lj8ZtRSegv9cw/FD9Q3IAC5anFaThhtlHqXd0xvxLRkSNHcP3117e7f+bMmTh8+HC3gyKRSyLhkOKqgEYGQB0NaMO0X5qnsVoJB8B46kvUCb8SUV1dHZKSktrdn5SUhPr6+m4HRSJc7Snha/zA8Ltt38yTiBSe8WbURtQxvy5a3W43ZLL23yKVSuFy0TecBOb7cxdhstgxrvIX6KQI38syAJBIALkKcqnZM8yDakQd8SsRMcZQWFjY7pI+tPoG6Y6TVSa4as7iSokViInu/csGdUauhUxaBzlvpTaiTviViAoKCjotQ3fMSCBMNicumh3ItZyBPkEGJOQJtYpwptBAIZVAL3VCKgnTS8wQ8SsRrV+/vqfiIBGu5KIFEt6FLP4CZBINkNj2suNhRa5Bkl6FWwfEAelhNlYuxML8Xw7pK0ouWhBjLUaMCoDKABjSxQ6p+7zDPJrEjSMMUCIiouN5hvN1FsRbziJaLQeShoTv3bKWmlfzcNAwj85QIiKiqzLZ4LaZEO8oRZRSBiQNEzuk4JBr4eJ5fHuyFBu/K4abZ2JH1GuJmohWrVqFMWPGQKfTITExEfPmzcPJkyfFDImIwO7kkeM+h2iVDJwhDdDGix1ScCg0kHIczCYjLpodsNEt/HaJmoj27NmDxYsXY9++fdi1axecTiemT5+Opia6po4k2XEazIyrQm5iFJCaL3Y4wSNXg+M4aCB0a6He1e0TdRTep59+6vN6w4YNSExMxIEDB3DttdeKFBUJufoiwNoAiVwFJA4WO5rg8Sw9reYcAGi8WUd61XBgo9EIAIiNjW1zv91u9+k02da82SS8mGxOaEr3QwoIbUNSeWdvCR/N65sxOzjmhoUm0W9Xr2ms5nkeDz74ICZMmIBhw9purFy1ahUMBoP3kZGREeIoSbB9degkDhz8ERctDiBtlNjhBJdcA3ASKGTCah50ada+XpOIFi9ejJ9//hmbN29ut8zy5cthNBq9j9LS0hBGSILN7nLDfX4f3DyDImkQoI0TO6Tg4rgWywpZ0GSnGlF7esWl2ZIlS/Dxxx9j7969SE9vvyObUqlsd5wbCT8l5dWIM5+CWi5BVO4EscPpGcooKGQSREvtkPaFvlE9RNRExBjD73//e2zduhW7d+9Gv379xAyHhFjdiS/BMR7axBxw0X30MlsRhVSDGr8eGA2k9ZFuCT1A1ES0ePFivPfee/joo4+g0+m8q8QaDAao1WoxQyM9zGqsBSs7BACIu6IPr/6i1Alf7WZx4+jlRG0jWrt2LYxGIyZPnoyUlBTv45///KeYYZEQqDy0E2A8+Jh+iE0L03mpu0IRJXx1UCLqiOiXZiQC1Z2DufQoAA76IX24NgQAyii4eYb9v5Tgl7pi3HlVFk0J0oZec9eMRAi3Ezi1EwOToqDPGYcBObliR9SzFFGQSIAmkxF1TQ6aMrYdlIhIaJ3eBVgboNRGY+g1c6CSS8WOqGcpdeDAIYoTlklqslNforZQIiKhU34I7rJDQv+avNmALAK6YnjaiNRwgGMuNFGNqE2UiEhoVB0HO7kDv1Q24ivnQDSownw+6q6SqwGJFHIpB4XbCgvViNpEiYj0LLcLOLcHOP4RqhptOCPNwSFuGCSR0mDLcYAiCgqpBHJ3E8zUu7pNvaJnNQlPjDFcqLfiQr0VjDFEq+XIMnDQMgtgawTMlUDlUcDWCKPViR8c/XAu9ipMGpgAvaoPDW7tjDIKcpkECreFGqvbQYmIBMRsd+HTw+dhKf8FMbbz0DjqwbsaYZO6kZ8ZDQ4cGBjsTh7VTgW+co9EdXQO+idokZ8RLXb4oeWpERkkdsikdBHSFkpExG8Wmw17/7sdSTUHoWR2xGoVkKo5mO0MUUoFOIUWUOrhVOjxwTkV6tT9wCtk6J+gxawrUsBF2pgrpQ5JehXmZeqBnDBdQruHUSIi/mmsQMXezYipKoNSJkFev2xo0oYKq26oY8GUOkCmAAA47C401BQhUafE8HQDhqToIy8JAdS7ugsoEZGuqzwKnNyB/loX1CkJ0Az+FTRZI30WQmyZZqKUMvz+uj48fKOrvOPNTOLG0YtRIiKdYwwo2guUfAsA4OIHInXQrEvL5ZCOqfRgYDh8+jx+airCr0dnQKukP72W6LtBOsbzwKkdQMURNNldUOZeA1nO5L6x7lioKPXgwMFlMaKhyQGz3UWJ6DLUhE/ax7uBE9uAiiPgAeziR2N9eRZqmxxiRxZelDqA46CUMsh5K0w2p9gR9TqUiEjb3C7g2Fag+gQgkaIs9XoUyXPg4hkM6gjqAxQMEimgiIJSJoHCZYbJRn2JLkeJiLTGu4UkVHsakMiAYbfgoDURADAkVQ859YXxn0oPhUwCldtEvavbQL9RxBdjwC8fAxfPCElo+K1oispCUa2w6OUVaQaRAwxTKgMUVCNqFyUi4uvs50DVcYCTAENvAmKycabaDMaAZIMKsVqF2BGGJ6UeSpkESrcZZkpErVAiIpdUHgVKfxSeD74BiBcmLTtVJfR/GZgUJVZk4c9zaabnrFAp+vgcTAGge4hEYK4GTnqWAM+6GkgaCgBosrtQ1iBM6pWbqBMruvCnNECnlOO6OCUwIlXsaHodSkTk0m163gXE5QD9rvXu0iikmD8uC+UNVrpb1h0qT9uanZZJbwslIgKUfAOYa4RJvAbN8umsyHEcEnRKJOgiYDbFnqTSC1+dNsDl8I7HIwJqI4p0ljrg/D7h+cAZgJLagXqETAnIlCiqbcJ7e4/iTDUNgG2JElGkO/elcGkW2w9IyPPZdf6iBZ/+XOm9dU+6SR0NF8/DYboIo5V6p7dEiSiS1ZcANaeES7Gc61qNHztdbcKJikYUUyIKDnUMVDIpVC5hxkpyiaiJaO/evZgzZw5SU1PBcRw+/PBDMcOJPMVfC19TRgJRvhN2Mca8NaHseG2IA+uj1DFQyiVQuRrRYKFE1JKoiaipqQkjRozAG2+8IWYYkamhFGg4L4yDyrq61e6LTQ6YbC7IJBzSY9QiBNgHqWOgpBpRm0S9azZz5kzMnDmzy+Xtdjvsdrv3dWMj3QoNmGduISRfcemOTgvNl2MZsRoaWxYs6hio5BKoXEY0Wl3geRY5q5l0Iqx+w1atWgWDweB9ZGRkiB1SeDJVAnXnhGEcmVe1WaT4ogUAkBVHk58FjToGCqkEKrcZjHfBRINfvcIqES1fvhxGo9H7KC0tFTuk8HTBM4wjcTCgjmm12+HiUe7pTZ0dR+1DQaOIAieTQ6eUIlPjhMvNix1RrxFWHRqVSiWUSupY1y2OJmGOIQBIH91mEbPdhRitAg4Xj2gN9aYOGo4D1DEYmurC0MFaIIp+l5uFVSIiQVBxWOg3pE8B9G2PeYrVKrDgqiw4XHxkrrrRk9QxQi92a73YkfQqYXVpRrqJ54Hyn4TnaaM6La6Q0a9H0GnihK9NteB5Jm4svYioNSKz2YwzZ854XxcVFeHQoUOIjY1FZmamiJH1URdPC0tBy9VAwuA2izg97RZ0p6yHaOLRZHfhp8MnUFQ/GAuuyhI7ol5B1ES0f/9+TJkyxft62bJlAICCggJs2LBBpKj6sLIDwtfUkYC07R/9qSoTvjhRjWHpBkwZlBi62CKFNgEyKQfWVIt6sx1unkFKt/DFTUSTJ08GY1Q9DYmmWmFIB8cBqfntFiuts8DFMyipRtQzNHFQyKRQwgGJswlGq5NmvQS1EUWO5rahuNxLc+NchjGGEk//oYxY6j/UI6QycJpYqOVSaJz1qGuyd/6eCECJKBK4HEDlEeF52pXtFqsx22FxuKGQSZAaTcM6eow2Hmq5FGpXPeqaaKgHQIkoMlQfF5KROgaI6ddusfOe2lB6jJraLXqSNgFqhRQaRx3ViDwoEfV1jAHlB4XnqfkdLhXdfFmWSZdlPUub4Lk0q8NFWjUXAHVo7PtMFYCpSlijLPmKdos53ZeGdVAi6mG6ZGgUUiRyRkh11HMdoETU95V5akOJeYCi/QTDM4YJA+JRZbTRXZyepoqGShOFEWlSIJ0ugQFKRH2bw3JpXFlq+43UAKCUSXFlZusBsKQHcBygSxFmQDCVC8NtIhy1EfVl5T8JSwTpktodV0ZEoksGALiN5TDZ6M4ZJaK+yu0CyvYLzzPGddhIXd/kwM9lRphpfpzQ0afCaHXiqwNHsO1wudjRiI4SUV9V9bNwaabSt1qd43Inq0zYdbwKX/xSHaLgCHQpUMokUDoa0GBsjPi5iSgR9UU8D5T+IDxPHyPMS92BszXCGlv9aZL80FFGQWlIgEIKaK0VqGy0iR2RqCgR9UXVxwHLRWFRv5QRHRatb3KgutEOCcehfwIlolDiYrKhV8thsJfhQr1V7HBERYmor+HdQPFXwvPMq4Rk1IFTVSahaJwaGgXdRA2p6CzoVXLobeWUiMQOgARZ5RHA2iD0GUpreyrYlpoT0YBEXQ8HRlqJzoROrYDGWY/a2pqIbieiRNSXOK1A0V7heebVgKzjjonVJhtqzQ5IJRxyE2nN+5BTaKCOS4dCKoG+qRilEVwrorp4X1K0V7hTpo3vcJR9s5ZtQyp5xw3apGdwCXlIiT6L2KhaxEdFbo92SkR9RcP5S3MODZje6Z0yABiWZkC/eC1cbpqcTjTxA5BqUANcHSBxAIjMsWd0adYXOCzA8W3CSPuU4UBM1+dB1iplMNCSQeLRxApDPBgPVP4sdjSioUQU7ng3cGIbYDcJv9S5v+r0LW6eoTrC+630Kikj4WYMFb/sw76ztWJHIwpKROGMMeDkDqCuSJgMf8i8ThuoAeBERSM2fX8eu45X9XyMpHOJQ2DlpSguLcXJYz9F5FAbSkThincDJ/4DVB4V1rAferMwuLUTNqcb33r+69J0H72ETIGofmOhV8mQ3rAfPxZdFDuikKNEFI5sjcDhzUDVMSEJDb4BiMvp0lu/OVOLJrsbMRo5RqS3PYk+EUHmVUiLj4bGeRFVv+xDgyWyZm6kRBROGBOWjN6/TrhLJpUDw24BkoZ26e0nK004csEIAJialwQZLRnUe8jViB46DdFqOdLr9uHLn05E1EqwveI38Y033kB2djZUKhXGjRuHH374QeyQeheeFyY4O7Ae+OUToeNiVCIweiEQn9ulQ5yrMWPnsUoAwJjsWGTG0XSwvU7aKGTlDoaCcyHmzFbsPnwqYtb9Ez0R/fOf/8SyZcuwYsUKHDx4ECNGjMCMGTNQXR3hU1LwPGAsA858Dny/Fjj2oTD3tEwB5F4HjCoU7pJ1gc3pxqfHKuHmGQYm6XB1TlyPhk4CxHHQjLwF/TIyoHKZoTryLppKjwg14T6OYyKn3HHjxmHMmDF4/fXXAQA8zyMjIwO///3v8eijj3b43sbGRhgMBhiNRuj1+lCEG1w8D7hswsNuAmwNgKUOMFcBjWXCEkDN5Gqht3TaKEDR/ih5p5tHk92FGpMdyQYVdCqhj9CZahOKai2YmpdISwX1drZGVHy7CQpLFeK0SkATB2fsQJznYxATl4wofSzkchm4Dia76y26+jcqas9qh8OBAwcOYPny5d5tEokE06ZNw3fffdeqvN1uh91+aR2oxsbGrn1QXRFw5r9t7/PmYdbGthbbu7qtjWPWNTlQVGv22ccxHhJeSDSZsVrvHSyj1emdH8gtkcOozkRDVA6MkgzwpXJMULoxxDPr64V6C7YfqfB+sptncLguDZycNjgJV3gapHMTdcilga3hQaVHyqR7gdLvgfPfAZaLMFbvRnW1Gd7rBIkcnEwurM7CSTEgSYe4KCUADrVmO87WNnkKtkxWwvP+iVokRAmzMtQ1OXG62tRuKP3itEjUtzODwxW/FibeCwJRE1FtbS3cbjeSknxvOyclJeGXX35pVX7VqlVYuXKl/x/ksgtrv4uEt9vhsDa1u9/JyYTFD9UxsDhU+MXGw6RMgkUeK9wVAwAHALjgbDFCmzHA4nC3Op5MwiFGq6CaTziTyoDsCUD6aKDmJKznTkBpPQeHxQjmdgG8E8xxaa5rzuYCJMI/M2axgzeb2z00p4kC4EkuTXbwpvbLMqUWkKja2Rm82QLCaqzZ8uXLsWzZMu/rxsZGZGRkdP7G6AxgxG8um7e5xXPv9ra2tdje1W2XHVPtcCGjydliEyckGJkSTKZClFYFKIUfRYzTjSkDHeA8x2p+T/Nbmy+1ACBJr8KC8Vne/RKOg0YphUIqCYtqO+kCmRJIGY6MlOHIAMB4Hg6bGTa7HQ6HDXC5wPMuaNQKQCb8zLV2F7KszrZr+wA0ajmgEMYiqp1uZHaw7LVWIwPam6eqgyYCf4maiOLj4yGVSlFV5dvDt6qqCsnJya3KK5VKKJUdT/TVJoUWiG1/qeWepgWg7Vq7MlRyKVIMXVt3XiGTID4qgO8HCVucRAKlRg9lBzc9NZ5HV6gBpAUhru4S9a6ZQqHAqFGj8Pnnn3u38TyPzz//HOPHjxcxMkJIKIl+abZs2TIUFBRg9OjRGDt2LFavXo2mpibcfffdYodGCAkR0RPRbbfdhpqaGjzxxBOorKzEyJEj8emnn7ZqwCaE9F2i9yPqjrDvR0RIH9fVv1HRe1YTQgglIkKI6CgREUJEJ3pjdXc0N291eagHISSkmv82O2uKDutEZDIJY2S61LuaECIak8kEg6H9ifjC+q4Zz/MoLy+HTqfrsSENzcNISktLI+LOXKSdL0Dn3JPnzBiDyWRCamoqJJL2W4LCukYkkUiQnp4eks/S6/UR80sKRN75AnTOPaWjmlAzaqwmhIiOEhEhRHSUiDqhVCqxYsWKwEb9h6FIO1+Azrk3COvGakJI30A1IkKI6CgREUJER4mIECI6SkSEENFRIoJ/K82+/fbbmDhxImJiYhATE4Np06aF3cq0ga6su3nzZnAch3nz5vVsgD3A33NuaGjA4sWLkZKSAqVSiYEDB+KTTz4JUbTB4e85r169GoMGDYJarUZGRgYeeugh2Gy20ATLItzmzZuZQqFg69atY8eOHWP33nsvi46OZlVVVW2Wv+OOO9gbb7zBfvrpJ3bixAlWWFjIDAYDu3DhQogjD4y/59usqKiIpaWlsYkTJ7K5c+eGJtgg8fec7XY7Gz16NJs1axb7+uuvWVFREdu9ezc7dOhQiCMPnL/nvGnTJqZUKtmmTZtYUVER27lzJ0tJSWEPPfRQSOKN+EQ0duxYtnjxYu9rt9vNUlNT2apVq7r0fpfLxXQ6HXvnnXd6KsSgCuR8XS4Xu/rqq9nf/vY3VlBQEHaJyN9zXrt2Levfvz9zOByhCjHo/D3nxYsXs6lTp/psW7ZsGZswYUKPxtksoi/NmleanTZtmndbRyvNtsViscDpdCI2tovrBYko0PN96qmnkJiYiHvuuScUYQZVIOe8bds2jB8/HosXL0ZSUhKGDRuG5557Dm5368Use6NAzvnqq6/GgQMHvJdv586dwyeffIJZs2aFJOawHvTaXf6uNNuWRx55BKmpqT4/9N4qkPP9+uuv8fe//x2HDh0KQYTBF8g5nzt3Dl988QXmz5+PTz75BGfOnMEDDzwAp9OJFStWhCLsbgnknO+44w7U1tbimmuuAWMMLpcLv/3tb/HHP/4xFCFTY3V3PP/889i8eTO2bt0KlaqdZXnDmMlkwoIFC/D2228jPj5e7HBChud5JCYm4q9//StGjRqF2267DY899hjeeustsUPrMbt378Zzzz2HN998EwcPHsSWLVuwfft2PP300yH5/IiuEfm70mxLL774Ip5//nn897//xfDhw3syzKDx93zPnj2L4uJizJkzx7uN54X1zmUyGU6ePImcnJyeDbqbAvkZp6SkQC6XQyqVercNHjwYlZWVcDgcUCgUPRpzdwVyzo8//jgWLFiARYsWAQCuuOIKNDU14b777sNjjz3W4VxCwRDRNaJAV5r905/+hKeffhqffvopRo8eHYpQg8Lf883Ly8PRo0dx6NAh7+PGG2/ElClTcOjQobCYGTOQn/GECRNw5swZb9IFgFOnTiElJaXXJyEgsHO2WCytkk1zImahGI4akibxXmzz5s1MqVSyDRs2sOPHj7P77ruPRUdHs8rKSsYYYwsWLGCPPvqot/zzzz/PFAoF+9e//sUqKiq8D5PJJNYp+MXf871cON418/ecz58/z3Q6HVuyZAk7efIk+/jjj1liYiJ75plnxDoFv/l7zitWrGA6nY69//777Ny5c+yzzz5jOTk57Ne//nVI4o34RMQYY6+99hrLzMxkCoWCjR07lu3bt8+7b9KkSaygoMD7OisriwFo9VixYkXoAw+QP+d7uXBMRIz5f87ffvstGzduHFMqlax///7s2WefZS6XK8RRd48/5+x0OtmTTz7JcnJymEqlYhkZGeyBBx5g9fX1IYmVpgEhhIguotuICCG9AyUiQojoKBERQkRHiYgQIjpKRIQQ0VEiIoSIjhIRIUR0lIgIIaKjRETCyoYNGxAdHe19/eSTT2LkyJHe14WFhWE5lW2ko0RE2lRYWAiO4/Db3/621b7FixeD4zgUFhb6lA92AsjOzsbq1at9tt122204depUu+9Zs2YNNmzY4H09efJkPPjgg0GNiwQfJSLSroyMDGzevBlWq9W7zWaz4b333kNmZqYoManVaiQmJra732Aw+NSYSHigRETadeWVVyIjIwNbtmzxbtuyZQsyMzORn5/frWO3VVOZN2+et5Y1efJklJSU4KGHHgLHceA4DkDrS7PLtayZFRYWYs+ePVizZo33GEVFRcjNzcWLL77o875Dhw6B4zicOXOmW+dFAkOJiHRo4cKFWL9+vff1unXrcPfdd/f4527ZsgXp6el46qmnUFFRgYqKCr+PsWbNGowfPx733nuv9xiZmZmtzgkA1q9fj2uvvRa5ubnBOgXiB0pEpEN33nknvv76a5SUlKCkpATffPMN7rzzzh7/3NjYWEilUuh0OiQnJ3c6Y2ZbDAYDFAoFNBqN9xhSqRSFhYU4efKkd6J4p9OJ9957DwsXLgz2aZAuiuipYknnEhISMHv2bGzYsAGMMcyePTvs569OTU3F7NmzsW7dOowdOxb/+c9/YLfbceutt4odWsSiGhHp1MKFC7Fhwwa88847Qas1SCSSVlOQOp3OoBy7KxYtWuRtiF+/fj1uu+02aDSakH0+8UWJiHTq+uuvh8PhgNPpxIwZM4JyzISEBJ92H7fbjZ9//tmnjEKh6PZaYu0dY9asWdBqtVi7di0+/fRTuiwTGV2akU5JpVKcOHHC+7w9RqOx1fpncXFxbU6yP3XqVCxbtgzbt29HTk4OXn75ZTQ0NPiUyc7Oxt69e/Gb3/wGSqUyoEvC7OxsfP/99yguLkZUVBRiY2MhkUi8bUXLly/HgAEDOlwsgfQ8qhGRLtHr9dDr9R2W2b17N/Lz830eK1eubLPswoULUVBQgLvuuguTJk1C//79MWXKFJ8yTz31FIqLi5GTk4OEhISA4n744YchlUoxZMgQJCQk4Pz5895999xzDxwOR0juApKO0ZzVJGJ99dVXuO6661BaWtpqVVQSWpSISMSx2+2oqalBQUEBkpOTsWnTJrFDinh0aUYizvvvv4+srCw0NDTgT3/6k9jhEFCNiBDSC1CNiBAiOkpEhBDRUSIihIiOEhEhRHSUiAghoqNERAgRHSUiQojoKBERQkT3/wHAvJAvpmZEEAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:48:51.003507Z", + "iopub.status.busy": "2024-02-29T03:48:51.003174Z", + "iopub.status.idle": "2024-02-29T03:48:51.321739Z", + "shell.execute_reply": "2024-02-29T03:48:51.320859Z" + }, + "papermill": { + "duration": 0.342716, + "end_time": "2024-02-29T03:48:51.323725", + "exception": false, + "start_time": "2024-02-29T03:48:50.981009", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCPUlEQVR4nO3deXwU5f3A8c/svZs9cpCEBBIS7vsQEDkUD34q3sUqrUdBUWtFW6G0lvoTRKuorRb6q6WtVtAWRWu1tfVAxSIKopyKgiCQkEgSQgg5N9lrnt8fkyyEJJBsNpkk+7x57Su7M7M730nCN888pyKEEEiSJOnIoHcAkiRJMhFJkqQ7mYgkSdKdTESSJOlOJiJJknQnE5EkSbqTiUiSJN3JRCRJku5MegfQFqqqUlBQgMvlQlEUvcORJOkUQggqKytJT0/HYGi+3NOlE1FBQQEZGRl6hyFJ0hnk5+fTu3fvZvd36UTkcrkA7SLdbrfO0UiSdKqKigoyMjLC/1eb06UTUf3tmNvtlolIkjqxM1WdyMpqSZJ0JxORJEm6k4lIkiTdyUQkSZLuZCKSJEl3uieiw4cPc9NNN5GUlITdbmfEiBFs3bpV77AkSepAujbfHz9+nMmTJ3PBBRfw9ttvk5yczDfffENCQoKeYUkSCAF5m6FkL7h7Q9+pYDTrHVW3pWsievzxx8nIyGDlypXhbdnZ2TpGJEl1Dv4X8j7VnlcUQsALQ6/SN6ZuTNdbszfeeINx48Zx3XXXkZKSwpgxY3jmmWeaPd7n81FRUdHgIUlRV1EA+Z9pz3uNBUWBI19B+bf6xtWN6ZqIDh48yIoVKxgwYABr167lRz/6ET/+8Y95/vnnmzx+6dKleDye8EOOM5PaRe7H2q1Z6jAYeDGkDte2H96ub1zdmKLnckIWi4Vx48axadOm8LYf//jHbNmyhU8++aTR8T6fD5/PF35dP46lvLxcDvGQoqPqKGx5VisFnX0HOBK1EtK258Fggsk/AZNF7yi7jIqKCjwezxn/j+paIkpLS2Po0KENtg0ZMoS8vLwmj7dareFxZXJ8mdQuir7Qvib115IQgCsN7PGgBqHskG6hdWe6JqLJkyezd+/eBtv27dtHnz59dIpIimmqCsW7tec9R57YriiQ2E97fuxAx8cVA3RNRPPmzWPz5s08+uij7N+/nxdffJE///nPzJ07V8+wpFhVdgh8VWC2QVK/hvsSsrSv5fkdHlYs0DURjR8/ntdff52XXnqJ4cOH8/DDD7Ns2TJuvPFGPcOSYlXxHu1r8hAwGBvuc6drX73HIFDbsXHFAN3nI7riiiu44oor9A5DinVCwLH92vPkQY33W51aPVFNGVQWQqLs7xZNug/xkKROobIQ/NVa7+n4zKaPqS8VVRR0XFwxQvcSkSR1CnWloa/9yRz4shizUcFiMmAxGbCaDKS6bfRypqIc2Q1VR3QOtvuRiUiSIJyIckhj35HKJg8ZYg0xTVUxVZd0ZGQxQSYiKWb5gypv7Srkwr523JVHQFHoP3gUacJKIKTiD6r4giG8/hC5JdX4LEkQAGqOQygIRvnfJ1rkd1KKWev3FpNTUs3W419yoQK40hjQO7XJY8u9AZxWI8ZPHFqrmfcYuJo+Vmo9WVktxaTDZTV8VVCBosCouOPaxlP7Dp3E4zBjNBogLlnbUH20A6KMHTIRSTFHCMFH+7REMizNRVKgUNuRcOYm+SqDh9xj1RQVyZazaJKJSIo5h8tqKCyvxWRQmJwa0G61TFZtTNkZ7K8yU1heS4FMRFElE5EUc7Yd0m7Fhqa7cVTVDdlI6AOnWZu9Xq80LVmVlxYTUnWbuKLbkYlIiilVviA5JdUAjMlMgOO52o76sWRn0CO5JxajgslXTkFZTfsEGYNkIpJiilFROKdvEoN7uki0AhWHtR0tqB8CUOwJuO1mTKqPgpLj7RdojJHN91JMsVuMnNM3SXtx7ACoIbB5wN7CBRtMFhyueKgq5tjRIhjYq91ijSWyRCTFruM52teELG3OoRZyJWhN+OWlxaiynigqZCKSYsb+4kq+OVKJP6hqG0rrElErR9I745MxGhSswUoqa4NRjjI2yVszKWZ8crCUkkoflwzrydAkBapLtJJQfOtmBDXYPIzOiGd8byeKQ651Fg2yRCTFhHJvgJJKHwZFoW9y3InWMmcqWByt+zCbG4vRgOJrenCs1HoyEUkxYf/RKgB6JdixmY2tbrZvwOrSvvrkunrRIhORFBMO1iWifslx2myM9YkokpkWrR58wRBf537LK1uaXnFGah2ZiKRuzxcMUVCmzTPdt4dTqxvyVWnTeLh7t/4DbW6MBoXjldUUlZZTGwhFOeLYIxOR1O19e7wGVQjiHWY8DvOJ0pAnM7I5hYxmTNY4LEYFa7CK8ppAVOONRTIRSd1ecYW2OnCfpLpK6ZP7D0XK5sZmNmIJVVHmlYmorWTzvdTtTeyXxNB0Nwi0ntRldfU6bUlEVi0RWYNVHPf6oxFmTJOJSIoJHntdf5/jhyAU0JrsnSmRf6DNE05EskTUdrremj344IMoitLgMXjwYD1Dkrq7k5vtWzGsoxGrG5vZgCVYTZksEbWZ7iWiYcOG8f7774dfm0y6hyR1I+9+VURNIMTZ2YmkeewnJaI2LpBodWEzG7Gr1dQa2pDQJKATJCKTyUTPnj31DkPqhoQQ5JRU4/WHGNsnAQI12kKK0Lb6IQBLHA6LkYv6xaGMy2hzrLFO91azb775hvT0dPr27cuNN95IXp7sICZFx3FvAK8/hMmg0NNt0yqphYC4HmBzt+3DrS4UFBR/VXSCjXG6logmTJjAqlWrGDRoEIWFhSxZsoRzzz2XL7/8EpfL1eh4n8+Hz+cLv66okF3speYdPq7NoNjTY8NkNJwYbd/W0hCAJU77GgpA0KfNeS1FTNcS0fTp07nuuusYOXIkl1xyCW+99RZlZWW88sorTR6/dOlSPB5P+JGRIYvEUvMOl3kB6BVv1za0ZXzZqUxWMJo5XFbD3z/5mi8Pl7f9M2OY7rdmJ4uPj2fgwIHs37+/yf0LFy6kvLw8/MjPz+/gCKWu5HDdsI5eCXaoLddWaFUMEJ8ZnRNYXQRCKhVlx2VfojbqVImoqqqKAwcOkJbW9LIuVqsVt9vd4CFJTSmvCVBRE8CgKFprWX0nRlfP6N1GWeKwmAyYQ145QVob6ZqIFixYwIcffkhubi6bNm3iO9/5Dkajke9///t6hiV1A75giN4JdtLibVhMBq0jI0SvNARgcWI1GbCEvFTWyk6NbaFrZfW3337L97//fY4dO0ZycjJTpkxh8+bNJCcn6xmW1A2kuGxcNy4DIYTWUlZWl4gSWjcb42lZnVhNRsxeL8dliahNdE1Ea9as0fP0UgxQFAW8pVBbAQYjeKLYwGFxYjEZsKheqnxBQqrAKDs3RqRT1RFJUjQEQ2rDOYLq64fc6WCM4hzTFidmo4JVrUEIbfFGKTIyEUndzuGyGlasP8DrO77VNpS1Q/0QgNWJgoLH6CcxzkIgpEb382OI7kM8JCnaisq1ZnuryVg3LWx9Iopi/RCAxQnAhHQzEyZlRfezY4wsEUndTlGFloh6emzgPQb+ajCYwB3lVVnrEhFBn9bDWoqYTERStyKECJeItPFldaUhT6/IpoU9HZNVS3AAcsxZm8hEJHUrFTVBvP4QRoNCisvafrdloM1nZHVyrNrHa5v3sX5vcfTPESNkHZHUrdTflvVwWjEZFCivGwYU7YrqemYHIVVQUVEOHjnMI1KyRCR1K/WJKM1j08aW+b3a7ZOr6WFDbVY/zEOtoVo230dMloikbqV3gh1fIKSt2FF+QNvoTot+/VA9swOL0YA5VEOpX65vFimZiKRupV+yk37Jda1ZX9f1I/JEsIhiS1kc4RJRjT9EMKRqcx9JrSK/Y1L3VV6fiNpx3ipzHEaDgk3VbgmrZakoIjIRSd1GmddPcWUtIVVoS0p7S7WWrWj3HzqZxYGCQpxBmzlU1hNFRt6aSd3GzvwyduSVMSYznvMTSrWNcT3AbGu/k5q11WM9xgCJcRYtCUqtJhOR1G0crdRKJckua8fclkG4d/WoVDOj5DCPiMlbM6lbEEJwtOrkRFTXf6g9K6pBWzEWIFgLqhz0GimZiKRuoaImiC+gYjQoJNkUqKrr5dzeichk1+qhhICAt33P1Y3JRCR1C8WVWqtVktOCsbIAhKqtXWbztO+JDQYw2yn1+nl18z7e332kfc/XTclEJHUL9fVDKS5bx9UP1TM7EEJQUVFGabUc5hEJmYikbqH45IrqigJtY3s225/MEofZaMAUqsXrl833kZCtZlK3MLZPAr0S7GQm2OFQfSJK75iTmx2YjdpqHqUB2aExEjIRSd1CRqKDjESH1okxUKsNdHWmdMzJLXGYjQpmtQZfQJXDPCIgv1tS91JxWPvqStVW7egIZgdGg4KlbpiHV5aKWk0mIqnLyy/1su9IpbaKRkWhtrGjbsvgxDAPRauorpHjzVpNJiKpy/vi23Le/KKQPYUVJ5WIOjIRab2rPSZtNQ9VyGEerdVpEtFjjz2Goijce++9eocidTEl9T2q7YYTHRk7skRUN95sbJqVWZOySPPYO+7c3USnSERbtmzhT3/6EyNHjtQ7FKmLCYRUjnu1W6IUQ7nWkdES1/4dGU9WP8wjUK31sJZaTfdEVFVVxY033sgzzzxDQkKC3uFIXcyxKj9CQJzViKO2rlezO10bdtFRzHHa11BQLisUId0T0dy5c7n88suZNm3aGY/1+XxUVFQ0eEixrb5HdQ/nyR0ZO/C2DLRlrI0mjlX7eHnT1/xXrubRarr2I1qzZg3bt29ny5YtLTp+6dKlLFmypJ2jkrqSo1Vak3myywoldYmovSbKb46i1K3mUUVFRTlmZ2LHnr8b0K1ElJ+fz09+8hNWr16NzdayiasWLlxIeXl5+JGfn9/OUUqdXUmlVj+UbA1CbXndjIwdXCKCBpPoe2XzfavpViLatm0bxcXFnHXWWeFtoVCIDRs28Pvf/x6fz4fR2LBDmtVqxWq1dnSoUid28bBUjlb6SA/VDXR1JGkrsHa0uvFmZn+tTEQR0C0RXXTRRezatavBtltuuYXBgwdz3333NUpCktSUeIeFeIcFDp5UUa0HswOTUcGk1lITCCGEQOnICvMuTrdE5HK5GD58eINtcXFxJCUlNdouSWekR4/qk5ntmOpuzUKqwBdUsZnlH9OWkoNepS7rmyOVlNUE6JNoJ6WyvqJap0RkicOoKNjRWvFqAyGZiFqhUyWi9evX6x2C1IXsKarkQHEV1kwDKUG/tpprXLI+wdSv5mHyk+S0EJSrebRKp0pEktQa4VkZxTFtgytNm7pVDxatU+OkTAeTxmXpE0MXpnuHRkmKRG0gREWN1os5MVSibdSrfgjAXDe+zC8n0I+ETERSl1Q/0NVlM2HxFmkbO2pq2KaY68ebeeV4swjIW7NOKhQKEQjIcUvNOVpWhd0QIt1hoLbaCwYHWJKgtlafgFQjGBwcrfKzefNeMnt4GJXR/cdOms3mqHS1iSgRHTx4kL59+7b55FJjQgiKioooKyvTO5ROTQRCjElQsRn95LjGg2KAgqPAUf2Ccp9D0KnSR6nFVBMkJ6dMv1g6UHx8PD179mxTv6mIElH//v2ZOnUqc+bM4bvf/W6Lh2hIZ1afhFJSUnA4HLJTXDPKvH6CqsBtDGAJebXe1B059UdTqo8RCAWpxInJbMZtt+gbTzsTQuD1eiku1gb5pqVFPsYvokS0fft2Vq5cyfz587n77ruZOXMmc+bM4eyzz444EEm7HatPQklJSXqH06mlWq0EVYHRV4YhaAZrHFh1/oMYsmIKKPgwYTBbsdm6/3Aku12rpC8uLiYlJSXi27SIKqtHjx7N8uXLKSgo4LnnnqOwsJApU6YwfPhwnnrqKY4e1bF43IXV1wk5HA6dI+n8FEXBbDRgUOvWETOa9Q0IQDGgKKAIQSx1I6r/fW1LnWabWs1MJhMzZszg73//O48//jj79+9nwYIFZGRk8IMf/IDCwsK2fHzMkrdjLaSGtIdC50lEgCLUmJq3Ohq/r21KRFu3buWuu+4iLS2Np556igULFnDgwAHee+89CgoKuPrqq9scoCSdqtoXpLwmQCCgNeFjMGmV1XpTDKAoKKgIodWhSC0T0U/vqaeeYsSIEUyaNImCggJeeOEFDh06xK9+9Suys7M599xzWbVqFdu3b492vFI3Mnv2bK655ppWv88XVKkNhFCDdbcChhOVwueff/4ZF2DIyspi2bJlrT7vGdWViAwIjAZFdidqhYgS0YoVK7jhhhs4dOgQ//znP7niiiswnNK1PiUlhb/85S9RCVLq/FqSAKJBCEEwpAJgEnWJqDPclgEYtETksRnp4bRiMHTtW+wHH3yQ0aNHd8i5Imo1e++998jMzGyUfIQQ5Ofnk5mZicViYdasWVEJUpLqhVSBQKsWMohOVFENJ24PhZwYrbUiKhH169ePkpKSRttLS0vJzs5uc1BS1zJ79mw+/PBDli9fjqIoKIrCgQMHmDNnDtnZ2djtdgYNGsTy5cubfP+SJUtITk7G7XZz55134vf7mz1XIKTd75gVFW9VNT/44Y9xehJIS0vjySefbHR8cXExV155JXa7nezsbFavXt3oGEVRWLFiBdOnT8dut9O3b19effXV8P7c3FwUReGVV17h3HPPxW63M378ePbt28eWLVsYN24cTqeT6VdczdGSY9qSRi3w3HPPMWzYMKxWK2lpadx9993hfXl5eVx99dU4nU7cbjfXX389R44cCe+vL6389a9/JSsrC4/Hw/e+9z0qKyvDx6iqyhNPPEH//v2xWq1kZmbyyCOPhPffd999DBw4EIfDQd++fXnggQfCLV+rVq1iyZIlfP755+Gf6apVq1p0XREREVAURRw5cqTR9tzcXOFwOCL5yIiUl5cLQJSXl3fYOdtTTU2N2L17t6ipqWm0zxcINfsIBEMtPtbfgmNbq6ysTEycOFHcfvvtorCwUBQWFora2lqxaNEisWXLFnHw4EHxt7/9TTgcDvHyyy+H3zdr1izhdDrFzJkzxZdffin+85//iOTkZPHLX/6y2XNV1PhFUXmNqKqsED+aM0tkZvQW77//vvjiiy/EFVdcIVwul/jJT34SPn769Oli1KhR4pNPPhFbt24VkyZNEna7Xfz2t78NHwOIpKQk8cwzz4i9e/eK//3f/xVGo1Hs3r1bCCFETk6OAMTgwYPFO++8I3bv3i3OOeccMXbsWHH++eeLjz/+WGzfvl30799f3HnrD0Rt6beipLJWeH3BZq/jD3/4g7DZbGLZsmVi79694rPPPgvHFAqFxOjRo8WUKVPE1q1bxebNm8XYsWPF1KlTw+9fvHixcDqdYsaMGWLXrl1iw4YNomfPng2+dz//+c9FQkKCWLVqldi/f7/46KOPxDPPPBPe//DDD4uNGzeKnJwc8cYbb4jU1FTx+OOPCyGE8Hq94qc//akYNmxY+Gfq9XqbvJbT/d629P9oqxLRvHnzxLx584TBYBA//OEPw6/nzZsnfvzjH4sJEyaISZMmteYj2ySWEtFT7+5t9vH69m8bHPt/6/Y1e+wrW/IaHLti/f5Gx0Ri6tSpDRJAU+bOnSuuvfba8OtZs2aJxMREUV1dfSKeFSuE0+kUoVDTCfFYlU8UldeIksI8YbFYxCt/W3Vi37Fjwm63h+PYu3evAMRnn30WPmbPnj0CaJSI7rzzzgbnmTBhgvjRj34khDiRiJ599tnw/pdeekkAYt26deFtSx99VAwa0E/UHvtWFJVVi8raQLPfi/T0dHH//fc3ue/dd98VRqNR5OWd+Fl99dVXDa5l8eLFwuFwiIqKivAxP/vZz8SECROEEEJUVFQIq9XaIPGcya9//WsxduzY8OvFixeLUaNGnfF90UhEraoj2rFjR30pil27dmGxnGitsFgsjBo1igULFrS9mCZ1C08//TTPPfcceXl51NTU4Pf7G1V+jho1qkEHzokTJ1JVVUV+fj59+vRpcKzQ/nACcChnP36/nwnnTAzvT0xMZNCgQeHXe/bswWQyMXbs2PC2wYMHEx8f3yjWiRMnNnq9c+fOBttOXok4NTUVgBEjRpzY1rMnxUePgQIKotm+RMXFxRQUFHDRRRc1uX/Pnj1kZGSQkZER3jZ06FDi4+PZs2cP48ePB7TWP5fLFT4mLS0tPNxiz549+Hy+Zs8B8PLLL/O73/2OAwcOUFVVRTAYxO12N3t8e2pVIvrvf/8LaJPcL1++XLegY9HcC/o3u+/Uxpk7zuvX7LGn9j27dXL71OmtWbOGBQsW8OSTTzJx4kRcLhe//vWv+fTTTyP+TEVRSHJaUVWVQrWuQtjYcRNImM0nKsXrO/Gduk0VarhTo2ime3X9sIhoxhM+v6q26ByffPIJN954I0uWLOGSSy7B4/GwZs2aJuvZOkJEldUrV66USaiDWUyGZh8mo6HFx5pbcGxE8VkshEInWos2btzIpEmTuOuuuxgzZgz9+/fnwIEDjd73+eefU1NTE369efNmnE5ng9LAqQxqkP7ZWZjNZj79bGt4+/Hjx9m3b1/49eDBgwkGg2zbti28be/evU3ObLB58+ZGr4cMGXL6i26GUtepsblhHi6Xi6ysLNatW9fk/iFDhpCfn99g3b7du3dTVlbG0KFDWxTDgAEDsNvtzZ5j06ZN9OnTh/vvv59x48YxYMAADh061OCYU3+m7anFf05mzJjBqlWrcLvdzJgx47THvvbaa20OTOpasrKy+PTTT8nNzcXpdDJgwABeeOEF1q5dS3Z2Nn/961/ZsmVLo1ZVv9/PnDlz+N///V9yc3NZvHgxd999d6OuIcCJJXrUAE5nHHNm3cjPfv5zknr0ICUlhfvvv7/B+wYNGsSll17KD3/4Q1asWIHJZOLee+9tsrTw97//nXHjxjFlyhRWr17NZ599FnE/OK1E1PytGWitXnfeeScpKSlMnz6dyspKNm7cyD333MO0adMYMWIEN954I8uWLSMYDHLXXXcxdepUxo0b16IYbDYb9913Hz//+c+xWCxMnjyZo0eP8tVXXzFnzhwGDBhAXl4ea9asYfz48bz55pu8/vrrDT4jKyuLnJwcdu7cSe/evXG5XO22rmCL//x5PJ5wcdTj8Zz2IcWeBQsWYDQaGTp0KMnJyVxyySXMmDGDmTNnMmHCBI4dO8Zdd93V6H0XXXQRAwYM4LzzzmPmzJlcddVVPPjgg02e47g3QGm1n1Dd0I5fP7aUc889lyuvvJJp06YxZcqUBvVBoJXe09PTmTp1KjNmzOCOO+4gJSWl0WcvWbKENWvWMHLkSF544QVeeumlFpc+TqUNfG2+RAQwa9Ysli1bxh/+8AeGDRvGFVdcwTfffFP3foV//etfJCQkcN555zFt2jT69u3Lyy+/3Ko4HnjgAX7605+yaNEihgwZwsyZM8N1SFdddRXz5s3j7rvvZvTo0WzatIkHHnigwfuvvfZaLr30Ui644AKSk5N56aWXWveNaAVFiK7bEb2iogKPx0N5eXm3uFWsra0lJyeH7OxsOcfTKVQhwpPlJyvlGIQKjsSorOqqKAqvv/56RMNNGqmtQPVXURmyEDS7SHJ2/6lATvd729L/oxFVCNTU1OD1npgk/NChQyxbtox33303ko+TpDOqH9ZhRNWSUGcZcX8qgxEDCh6bMSaSULRElIiuvvpqXnjhBQDKyso4++yzefLJJ7n66qtZsWJFVAOUYlNeXh5OpzP8SIz30De9B1npKTjT+5N3+EjnGHF/qrqYnMkZDeI/+fHRRx/pHGTnE/EMjb/97W8BePXVV+nZsyc7duzgH//4B4sWLeJHP/pRiz5nxYoVrFixgtzcXACGDRvGokWLmD59eiRhSd1Ienp6g348FTV+/EGBy1CLVfWR3rv5VrXWimrtRF0i2rlpHTianmWzVy8dVxvppCJKRF6vN9yR6t1332XGjBkYDAbOOeecRk2Ap9O7d28ee+wxBgwYgBCC559/nquvvpodO3YwbNiwSEKTugmTyUT//lrfKSEER6t8CAFJSiUmEQRrJ53Fsi4R9c7IpNqcqC13ZJJLT59JRGXb/v37889//pP8/HzWrl3LxRdfDGg9RltTaXzllVdy2WWXMWDAAAYOHMgjjzyC0+ls1KdDim0hVSCE1lvZ2NlG3J+qLhEJNUQwJAi1bPxrzIsoES1atIgFCxaQlZXFhAkTwt3j3333XcaMGRNRIKFQiDVr1lBdXd2ou309n89HRUVFg4fU/QnAajJgVUIoAAajNitjZ1SXiLSeLrE1ZWxbRPTT/O53v8uUKVMoLCxk1KhR4e0XXXQR3/nOd1r1Wbt27WLixInU1tbidDp5/fXXm+2/sXTpUpYsWRJJyFIXZjYaiHdYwOcHH523NAR1nYgUtH9CThfbQrr3I/L7/eTl5VFeXs6rr77Ks88+y4cffthkMvL5fPh8vvDriooKMjIyZD+iWOEthaAPbG6wxOkdTfOqivEHApQrbqxWG257J06cURCNfkQRlYiqq6t57LHHWLduHcXFxeGBdvUOHjzY4s+yWCzhSsmxY8eyZcsWli9fzp/+9KdGx1qt1nbrYi51TmrdiHujokCobsK0zlwighPLCslbsxaLKBHddtttfPjhh9x8882kpaVFdfkbVVUblHqk2OYPqpTXBLAZVDxCaLc+hi6QiFDOOMxDOiGiRPT222/z5ptvMnny5DadfOHChUyfPp3MzEwqKyt58cUXWb9+PWvXrm3T50od7/zzz2f06NFRXx3DH6ybKJ+TWsta8IevveJpEYMRRQEjQusBLp1RRIkoISGBxMTENp+8uLg4vBCjx+Nh5MiRrF27lv/5n/9p82dLXY/f728w2R5AoK7920z9ih1dYD15xYBRUYi3GsHWBeLtDFowg2Qjf/3rX8V3v/vdBlN86iGWportzGbNmiXQWtnDj/3794tbb71VZGVlCZvNJgYOHCiWLVvW6H1XX321+NWvfiXS0tJEVlaWEEKIjRs3ilGjRgmr1SpGjj5LrPzbywIQOz56V4hArRBCiF27dolLL71UxMXFiZSUFHHTTTeJo0ePNhtPTk5Ox31DfFVClBcI4S3tuHPqqMOniq335JNPcuDAAVJTU8nKymo0U5xcWDGKhIBQ5GuKR6yFt0AAy5cvZ9++fQwfPpyHHnoI0ErNvXv35u9//ztJSUls2rSJO+64g7S0NK6//vrwe9etW4fb7ea9994DtFaW+o6uK5//K3v2H2TRL36mHawoYLRQVlbGhRdeyG233cZvf/tbampquO+++7j++uv54IMPmownOTk5it+cMwgvKyR7M7ZURIkoKtMlSC0TCsBHOkzfee5PwdSy2wqPx4PFYsHhcNCzZ8/w9pP7fGVnZ/PJJ5/wyiuvNEhEcXFxPPvss+Fbsj/+8Y8oisIzzzxDACPp2QM4XpjH3ff8uG5paYXf//73jBkzhkcffTT8Oc899xwZGRns27ePgQMHNhlPh6lLRF5fAK/qI95ubjSLptRQRIlo8eLF0Y5D6oZaMnn+iBEjGtQL7d27l5EjR2Kz2aiu0lpPzxlX12m2rtn+888/57///S9Op7PROQ8cOMDAgQPb54JaKjzMQyWknn6mRkkTcT/5srIyXn31VQ4cOMDPfvYzEhMT2b59O6mpqXJ0cTQZzVrpRI/ztkFLJ8+Pi2u+Y2Kc1YQ/qJ4YX1a3xn1VVRVXXnkljz/+eKP3pKWltSnuqKhLRAZUEMgm/BaIKBF98cUXTJs2DY/HQ25uLrfffjuJiYm89tpr5OXlhecqkqJAUVp8i6Sn002eX6+pyfNPNWjQIP72t7/h8/mwWa3YDCrbtmnLWNWv2HHWWWfxj3/8g6ysLEympn+FO3Li90bkeLNWi+jGdf78+cyePZtvvvmmQZfuyy67jA0bNkQtOKnrOHny/JKSEgYMGMDWrVtZu3Yt+/bt44EHHmDLli1n/JwbbrgBVVW544472LNnD2vffovf/N8fAVDqJsafO3cupaWlfP/732fLli0cOHCAtWvXcsstt4STz6nxnNr7v101GG+m0pGn7qoiSkRbtmzhhz/8YaPtvXr1oqioqM1BSV1PpJPnn8rtdvPvf/+bHTt2MHr0aO5ftJhF980DCP/RS09PZ+PGjYRCIS6++GJGjBjBvffeS3x8fHgVj1PjycvLa7+Lb4pirJtEX9YRtUgk/QaSk5PF9u3bhRBCOJ1OceDAASGEtlRu7969I/nIiMh+RN1TSFVFUXmNKCrzilB5gfjbM78XZrO52bXXO6WqEuE//q04erxMHK/26R1Nu4pGP6KISkRXXXUVDz30EIGA1r9FURTy8vK47777uPbaa6OYJqVYtHLV83z6yUYKDu3nX/95m/sWP8L1118ftRVSO4RBG29mRER1LGZ3FVEievLJJ6mqqiI5OZmamhqmTp1K//79cblcPPLII9GOUYoxBQWFzL3jViaePZ75Cx/kuhnX8Oc//1nvsFpHMWAyKCTYjXi6+TQg0RBRq5nH4+G9995j48aNfP7551RVVXHWWWcxbdq0aMcnxaC77p3PHffMI4EKLITAHg/mLlQaAtm7upVanYhUVWXVqlW89tpr5ObmoigK2dnZ9OzZ88SSwJIUIVUVBEMCRaiYlGB4WEeXIxNRq7Tq1kwIwVVXXcVtt93G4cOHGTFiBMOGDePQoUPMnj271dPEStKp/HWj7S0EMFA395ChC66CoRgQQHWtn5IqHyHZq/G0WlUiWrVqFRs2bGDdunVccMEFDfZ98MEHXHPNNbzwwgv84Ac/iGqQsaZD+7x0MvXTflipm40xCktK60IxoKD9LOuHeRi76eRE0fh9bVUieumll/jlL3/ZKAkBXHjhhfziF79g9erVMhFFyGKxYDAYKCgoIDk5GYvFEnO3uiYhsBtCqLU11AIYAVGrc1QRCAXAHyAQEASEjxpDiFA3W99MCIHf7+fo0aMYDIZGc0m1RqsS0RdffMETTzzR7P7p06fzu9/9LuJgYp3BYCA7O5vCwkIKCgr0Dkc/oQAEvFo9i7Va72giI1TwVeIPqdQayim1GDF30xH4DoeDzMzMcGfSSLQqEZWWlpKamtrs/tTUVI4fPx5xMJJWKsrMzCQYDOo3Vkpv+z+Aim+g50joM1LvaCKjqrDlGXJKvGxxT+esfqkMTPfoHVXUGY1GTCZTm0vurUpEoVCo2UGG9UEFg8E2BSRpHUTNZnOjCee6u08PHqPS62NC0Ze4jCFI7Q9deVklo4pNrSLor6VWGOUSUafRqkQkhGD27NnNLukjV9+Q2mLvkUqCRw9wlqEGEuLB3cWnkzHHYTKWYlZrqPHHaOm2hVqViGbNmnXGY2RFtRSJytoAx6r89Pfux51sguTB0IY6h07B4sBiNOA2BjAaYqvRobValYhWrlzZXnFIMe7QMS8GNUgf9VtMBgekNL3seJdidpDqtnHdgCTonaJ3NJ1aF/+TI3UXh455SajJJcEG2Dzg6a13SG1ndmhfA1205a8DyUQk6U5VBXmlXnp4DxBvN0Pq0BavINKpWeoSkd+rbxxdgExEku6OVNYSqq2khz8fp9UEqcP1Dik6zHEEVZVNe/N54ZNcOczjNHRNREuXLmX8+PG4XC5SUlK45ppr2Lt3r54hSTrwBVT6hQ4SbzOheHpBXA+9Q4oOiwOjolBVWc6xKj+1Adly1hxdE9GHH37I3Llz2bx5M++99x6BQICLL76Y6mp5Tx1LspIcTE86Qv8UJ6SP0Tuc6DHbURQFB1q3Fq9swm9WxMsJRcM777zT4PWqVatISUlh27ZtnHfeeTpFJXW44zlQU4bBbIOUIXpHEz1mbakku6IN4JV9iZqnayI6VXl5OQCJiYlN7vf5fA06TVZUVHRIXFL7qawN4MjfihG0uqE2rqfWqdRVVtuED0WE8AbkqIPmdJrKalVVuffee5k8eTLDhzddWbl06VI8Hk/4kZGR0cFRStH20c69bNu+hWNeP/Qaq3c40WV2gGLAYjJgDtXIW7PT6DSJaO7cuXz55ZesWbOm2WMWLlxIeXl5+JGfn9+BEUrR5guGCOVtJqQKLKmDIC5J75CiS1HAEofZaMAc8lLtkyWi5nSKW7O7776b//znP2zYsIHevZvvyGa1Wpsd5yZ1PYcKikmq2ofdbMDZf7Le4bQPqxOLyUC80YexO/SNaie6JiIhBPfccw+vv/4669evJzs7W89wpA5Wuue/KEIlLqUfSnw3vc22OEn32Ll+YDz06ibdEtqBrolo7ty5vPjii/zrX//C5XKFV4n1eDxdaw0rqdVqyksQh3cCkDSiG6/+YnVpX31V+sbRyelaR7RixQrKy8s5//zzSUtLCz9efvllPcOSOkDRzrUgVNSEbBJ7DdA7nPZjcWpf/TIRnY7ut2ZSDCo9SFX+LkDBPbQbl4YArE5CqmDr14f4ujSXm87pI6cEaUKnaTWTYkQoAPvWMjDVibvfBAb06693RO3L4sRggOrKckqr/Xj9suWsKTIRSR3rm/egpgxrXDzDplyJzdy9VrZoxOpCQcGp1ABQ7ZN9iZoiE5HUcQp2Ejq8U+tfM/jyrrtmWWvU1RHZ8aOIINWyRNQkmYikjnFkN2Lv23xdVMFHgYGU2br4fNQtZbaDwYjZqGAJ1eCVJaImyUQkta9QEA5+CLv/xZGKWvYb+7FTGY4hVipsFQUsTixGA+ZQNVWyd3WTOkXPaqlrEkLw7fEavj1egxCCeLuZPh6FOOGF2gqoKoKiXVBbQXlNgM/82RxMPIepA5Nx27rR4NYzsToxmwxYQl5ZWd0MmYikiFT5grzzeR7egq9JqM3D4T+OGqyg1hhiTGY8CgoCgS+gUhyw8FFoNMXx/eibHMeYjHi9w+9YdSUij8GHqZuu9tpWMhFJreatrWXD+2+SenQ7VuEjMc6C0a5Q5RM4rRYUSxxY3QQsbv5+0EapPRvVYqJvchyXjUhr86qgXY7VRarbxjWZbuiXrHc0nZJMRFLrVBRSuGENCUcOYzUZGJydhaPXMG3VDXsiwuoCkwUAvy9I2dEcUlxWRvb2MDTNHXtJCGTv6haQiUhquaJdsPdt+sYFsacl4xjyPzj6jG6wEOLJacZpNXHPRd14+EZLhcebVeobRycmE5F0ZkJAzgY4tAkApcdA0gdddmK5HOn0bG4Egs+/yWNHdQ7Xj8sgzir/651Mfjek01NV2Pc2FH5BtS+Itf8UTP3O7x7rjnUUqxsFhaC3nLJqP1W+oExEp5BV+FLz1BDseQMKv0AF3lPHsbKgDyXVfr0j61qsLlAUrEaBWa2hsjagd0SdjkxEUtNCQfjqdSjeAwYjh9MvJcfcj6Aq8NhjqA9QNBiMYHFiNRmwBKuorJV9iU4lE5HUmBrSklDJN2AwwfBr2V6TAsDQdDdm2Rem9WxuLCYDtlCl7F3dBPkbJTUkBHz9Hzi2X0tCI6+j2tmHnBJt0csRvTw6B9hF2TxYZImoWTIRSQ0dWAdHdoNigGHfgYQs9hdXIQT09NhIjLPoHWHXZHVjNRmwhqqokomoEZmIpBOKdkH+Fu35kCughzZp2b4jWv+XgalOvSLr+upuzdxKDTZLN5+DKQKyDVHSVBXD3rolwPtMgtRhAFT7ghwu0yb16p/i0iu6rs/qwWU1c1GSFUal6x1NpyMTkXSimV4NQlI/yD4vvMthMXLjhD4UlNXI1rK2sNXVrfnkMulNkYlIgkMboeqoNonXoMsadFZUFIVkl5VkVwzMptiebG7ta6AWgv7weDxJI+uIYp23FPI2a88HXgJWWQ/ULkxWMFnJKanmxQ272F8sB8CeTCaiWHfwv9qtWWI2JA9usCvvmJd3viwKN91LbWSPJ6iq+CuPUV4je6efTNdEtGHDBq688krS09NRFIV//vOfeoYTe44fgqP7tFuxfhc1Gj/2TXEleworyJWJKDrsCdhMRmxBbcZK6QRdE1F1dTWjRo3i6aef1jOM2JX7sfY1bTQ4G07YJYQIl4SyesR1cGDdlD0Bq9mALVhBmVcmopPpWlk9ffp0pk+frmcIsassH8rytHFQfSY12n2s2k9lbRCTQaF3gl2HALshewLWuhJRoSwRNdClWs18Ph8+ny/8uqJCNoVGrG5uIXqOONGic5L627GMRIccWxYt9gRsZgO2YDkVNUFUVcTOaiZn0KV+w5YuXYrH4wk/MjIy9A6pa6osgtKD2jCOzHOaPCT3mBeAPkly8rOosSdgMRqwhaoQapBKOfg1rEslooULF1JeXh5+5Ofn6x1S1/Rt3TCOlCFgT2i02x9UKajrTZ2VJOuHosbiRDGZcVmNZDoCBEOq3hF1Gl3q1sxqtWK1yo51beKv1uYYAug9rslDqnxBEuIs+IMq8Q7ZmzpqFAXsCQxLDzJsSBw45e9yvS6ViKQoKPxc6zfkTgN302OeEuMs3HxOH/xBNTZX3WhP9gStF3vNcb0j6VR0TURVVVXs378//DonJ4edO3eSmJhIZmamjpF1U6oKBTu0573GnvFwi6lL3bl3DY4k7Wt1iaysPomuv2lbt25lzJgxjBkzBoD58+czZswYFi1apGdY3dexb7SloM12SB7S5CGBkEpA1l20H0cPqn1BPv58D6s/y9M7mk5D1xLR+eefjxBCzxBiy+Ft2tf00WBs+ke/70glH+wpZnhvDxcMSum42GJFXDImo4KoLuF4lY+QKjDKUlHXajWT2qC6RBvSoSiQPqbZw/JLvQRVgVX2HWofjiQsJiNW/BgC1XKoRx352xYr6uuGkvqfmBvnFEIIDtX1H8pIlP2H2oXRhOJIxG424ggcp7Tad+b3xACZiGJB0A9FX2jPe53V7GFHq3x4/SEsJgPp8XJYR7uJ64HdbMQePE5ptSwRgUxEsaF4t5aM7AmQkN3sYXl1paHeCXZZb9Ge4pKxW4w4/KWyRFRHJqLuTggo2K49Tx9z2qWi62/LMuVtWfuKS667NSvlmFw1F5AdGru/ykKoPKKtUdZzRLOHBUInhnXIRNTOXD1xWIykKOUYXbLnOshE1P0drisNpQwGS/MJRhWCyQN6cKS8Vq5d1t5s8dgcTkb1MkJveQsMMhF1b37viXFl6c1XUgNYTUbOymw8AFZqB4oCrjRtBoTKAm24TYyTdUTdWcEObYkgV2qz48oknbh6AhAqL6CyVracyUTUXYWCcHir9jxjwmkrqY9X+/nycDlVcn6cjuNOp7wmwEfbvuCNzwv0jkZ3MhF1V0e+1G7NbO5Gq3Ocau+RSt7bfYQPvi7uoOAkXGlYTQas/jLKyitifm4imYi6I1WF/M+0573Ha/NSn8aBo9oaW33lJPkdx+rE6knGYoS4mkKKKmr1jkhXMhF1R8W7wXtMW9QvbdRpDz1e7ae4wodBUeibLBNRR1ISsnDbzXh8h/n2eI3e4ehKJqLuRg1B7kfa88xztGR0GvuOVGqHJtlxWGQjaoeK74PbZsZdWyATkd4BSFFW9AXUlGl9hno1PRXsyeoT0YAUVzsHJjUSn4nLbsEROE5JydGYrieSiag7CdRAzgbteeYkMJ2+Y2JxZS0lVX6MBoX+KXLN+w5ncWBP6o3FaMBdnUt+DJeKZFm8O8nZoLWUxfU47Sj7eifXDdnMp6/QltqHkjyYtPgDJDpL6OGM3R7tMhF1F2V5J+YcGnDxGVvKAIb38pDdI45gSM6SqZseA0j32EEpBYMfiM2xZ/LWrDvwe2H3G9pI+7SRkNCnxW+Ns5rwyCWD9ONI1IZ4CBWKvtQ7Gt3IRNTVqSHY8wb4KrVf6v7/c8a3hFRBcYz3W+lU0kYTEoLCrzez+UCJ3tHoQiairkwI2Ps2lOZok+EPveaMFdQAeworWP1pHu/tPtL+MUpnljKUGtVIbn4+e7/aEZNDbWQi6qrUEOz5NxTt0tawHzZDG9x6BrWBEJvq/urK6T46CZMFZ/bZuG0mepdtZUvOMb0j6nAyEXVFtRXw+Ro48pWWhIZcAUn9WvTWjftLqPaFSHCYGdW76Un0JR1knkOvHvE4Asc48vVmyryxNXOjTERdiRDaktFbn9NayYxmGH4tpA5r0dv3FlXyxbflAFw4OBWTXDKo8zDbiR82jXi7md6lm/nvjj2oauy0ZnaK38Snn36arKwsbDYbEyZM4LPPPtM7pM5FVbUJzrathK/f0jouOlNg3K3Qo3+LPuLg0SrWflUEwPisRDKT5HSwnU6vsfTpPwSLEiRh/+us/3xfzCxAqnsievnll5k/fz6LFy9m+/btjBo1iksuuYTi4hifkkJVofww7F8Hn66Ar/6pzT1tskD/i2DsbK2VrAVqAyHe+aqIkCoYmOpiUr+kdg1dipCi4Bh9LdkZGdiCVdi++BvV+V9oJeFuThE6p9wJEyYwfvx4fv/73wOgqioZGRncc889/OIXvzjteysqKvB4PJSXl+N2uzsi3OhSVQjWag9fJdSWgbcUqo5AxWFtCaB6ZrvWW7rXWLA0P0o+EFKp9gU5Wumjp8eGy6b1EdpfXElOiZcLB6fIpYI6u9oKCjetxuI9QlKcFRxJBBIHkqcmkJDUE6c7EbPZhHKaye46i5b+H9W1Z7Xf72fbtm0sXLgwvM1gMDBt2jQ++eSTRsf7fD58vhPrQFVUVLTsRKU5sP/9pveF87BoYttJ21u6rYnPLK32k1NS1WCfIlQMqpZoMhPjwi1Y5TWB8PxAIYOZcnsmZc5+lBsyUPPNTLaGGFo36+u3x728+UVh+MwhVeAPnhg4OW1IKiPqKqT7p7joLwe2dg02N2lTb4f8TyHvE/Aeo7x4PcXFVYTvEwxmFJNZW51FMTIg1UWS0woolFT5OFBSXXfgyclKe943JY5kpzYrQ2l1gG+KK5sNJTspjhR3MzM4jLhem3gvCnRNRCUlJYRCIVJTGzY7p6am8vXXXzc6funSpSxZsqT1Jwr6tLXfdaL6fPhrqpvdH1BM2uKH9gS8fhtf16pUWlPxmhO1VjEAP0CQwEkjtIUArz/U6PNMBoWEOIss+XRlRhNkTYbe4+DoXmoO7sFacxC/txwRCoIaQPhPzHWt1AbBoP0xE14falVVsx+tOJxAXXKp9qFWNn+ssMaBwdbMzujNFtClxpotXLiQ+fPnh19XVFSQkZFx5jfGZ8Co750yb/NJz8Pbm9p20vaWbjvlM+3+IBnVgZM2KVqCMVkRJhvOOBtYtR9FQiDEBQP9KHWfVf+e+rfW32oBpLpt3DyxT3i/QVFwWI1YjIYuUWyXWsBkhbSRZKSNJAMQqoq/topanw+/vxaCQVQ1iMNuAZP2M4/zBelTE2i6tA847GawaGMR7YEQmadZ9jrOYYLm5qk6TRVBa+maiHr06IHRaOTIkYY9fI8cOULPnj0bHW+1WrFaTz/RV5MscZDY/FLL7S0OiGtZvTI2s5E0T8vWnbeYDPRwRvD9kLosxWDA6nBjPU2jp6Pu0RJ2oFcU4morXVvNLBYLY8eOZd26deFtqqqybt06Jk6cqGNkkiR1JN1vzebPn8+sWbMYN24cZ599NsuWLaO6uppbbrlF79AkSeoguieimTNncvToURYtWkRRURGjR4/mnXfeaVSBLUlS96V7P6K26PL9iCSpm2vp/1Hde1ZLkiTJRCRJku5kIpIkSXe6V1a3RX31VouHekiS1KHq/2+eqSq6SyeiykptjEyLeldLkqSbyspKPJ7mJ+Lr0q1mqqpSUFCAy+VqtyEN9cNI8vPzY6JlLtauF+Q1t+c1CyGorKwkPT0dg6H5mqAuXSIyGAz07t27Q87ldrtj5pcUYu96QV5zezldSaierKyWJEl3MhFJkqQ7mYjOwGq1snjx4shG/XdBsXa9IK+5M+jSldWSJHUPskQkSZLuZCKSJEl3MhFJkqQ7mYgkSdKdTES0bqXZZ555hnPPPZeEhAQSEhKYNm1al1uZNtKVddesWYOiKFxzzTXtG2A7aO01l5WVMXfuXNLS0rBarQwcOJC33nqrg6KNjtZe87Jlyxg0aBB2u52MjAzmzZtHbW1txwQrYtyaNWuExWIRzz33nPjqq6/E7bffLuLj48WRI0eaPP6GG24QTz/9tNixY4fYs2ePmD17tvB4POLbb7/t4Mgj09rrrZeTkyN69eolzj33XHH11Vd3TLBR0tpr9vl8Yty4ceKyyy4TH3/8scjJyRHr168XO3fu7ODII9faa169erWwWq1i9erVIicnR6xdu1akpaWJefPmdUi8MZ+Izj77bDF37tzw61AoJNLT08XSpUtb9P5gMChcLpd4/vnn2yvEqIrkeoPBoJg0aZJ49tlnxaxZs7pcImrtNa9YsUL07dtX+P3+jgox6lp7zXPnzhUXXnhhg23z588XkydPbtc468X0rVn9SrPTpk0LbzvdSrNN8Xq9BAIBEhNbuF6QjiK93oceeoiUlBTmzJnTEWFGVSTX/MYbbzBx4kTmzp1Lamoqw4cP59FHHyUUaryYZWcUyTVPmjSJbdu2hW/fDh48yFtvvcVll13WITF36UGvbdXalWabct9995Gent7gh95ZRXK9H3/8MX/5y1/YuXNnB0QYfZFc88GDB/nggw+48cYbeeutt9i/fz933XUXgUCAxYsXd0TYbRLJNd9www2UlJQwZcoUhBAEg0HuvPNOfvnLX3ZEyLKyui0ee+wx1qxZw+uvv47N1syyvF1YZWUlN998M8888ww9evTQO5wOo6oqKSkp/PnPf2bs2LHMnDmT+++/nz/+8Y96h9Zu1q9fz6OPPsof/vAHtm/fzmuvvcabb77Jww8/3CHnj+kSUWtXmj3Zb37zGx577DHef/99Ro4c2Z5hRk1rr/fAgQPk5uZy5ZVXhrepqrbeuclkYu/evfTr1699g26jSH7GaWlpmM1mjEZjeNuQIUMoKirC7/djsVjaNea2iuSaH3jgAW6++WZuu+02AEaMGEF1dTV33HEH999//2nnEoqGmC4RRbrS7BNPPMHDDz/MO++8w7hx4zoi1Kho7fUOHjyYXbt2sXPnzvDjqquu4oILLmDnzp1dYmbMSH7GkydPZv/+/eGkC7Bv3z7S0tI6fRKCyK7Z6/U2Sjb1iVh0xHDUDqkS78TWrFkjrFarWLVqldi9e7e44447RHx8vCgqKhJCCHHzzTeLX/ziF+HjH3vsMWGxWMSrr74qCgsLw4/Kykq9LqFVWnu9p+qKrWatvea8vDzhcrnE3XffLfbu3Sv+85//iJSUFPGrX/1Kr0totdZe8+LFi4XL5RIvvfSSOHjwoHj33XdFv379xPXXX98h8cZ8IhJCiP/7v/8TmZmZwmKxiLPPPlts3rw5vG/q1Kli1qxZ4dd9+vQRQKPH4sWLOz7wCLXmek/VFROREK2/5k2bNokJEyYIq9Uq+vbtKx555BERDAY7OOq2ac01BwIB8eCDD4p+/foJm80mMjIyxF133SWOHz/eIbHKaUAkSdJdTNcRSZLUOchEJEmS7mQikiRJdzIRSZKkO5mIJEnSnUxEkiTpTiYiSZJ0JxOR1KWsWrWK+Pj48OsHH3yQ0aNHh1/Pnj27S84gGetkIpKaNHv2bBRF4c4772y0b+7cuSiKwuzZsxscH+0EkJWVxbJlyxpsmzlzJvv27Wv2PcuXL2fVqlXh1+effz733ntvVOOSok8mIqlZGRkZrFmzhpqamvC22tpaXnzxRTIzM3WJyW63k5KS0ux+j8fToMQkdQ0yEUnNOuuss8jIyOC1114Lb3vttdfIzMxkzJgxbfrspkoq11xzTbiUdf7553Po0CHmzZuHoigoigI0vjU71ckls9mzZ/Phhx+yfPny8Gfk5OTQv39/fvOb3zR4386dO1EUhf3797fpuqTIyEQkndatt97KypUrw6+fe+45brnllnY/72uvvUbv3r156KGHKCwspLCwsNWfsXz5ciZOnMjtt98e/ozMzMxG1wSwcuVKzjvvPPr37x+tS5BaQSYi6bRuuukmPv74Yw4dOsShQ4fYuHEjN910U7ufNzExEaPRiMvlomfPnmecqK4pHo8Hi8WCw+EIf4bRaGT27Nns3bs3PD9zIBDgxRdf5NZbb432ZUgtFNMzNEpnlpyczOWXX86qVasQQnD55Zd3+Wlj09PTufzyy3nuuec4++yz+fe//43P5+O6667TO7SYJUtE0hndeuutrFq1iueffz5qpQaDwdBo5r9AIBCVz26J2267LVwRv3LlSmbOnInD4eiw80sNyUQkndGll16K3+8nEAhwySWXROUzk5OTG9T7hEIhvvzyywbHWCyWNi/h09xnXHbZZcTFxbFixQreeecdeVumM3lrJp2R0Whkz5494efNKS8vb7TsUFJSUpNzW1944YXMnz+fN998k379+vHUU09RVlbW4JisrCw2bNjA9773PaxWa0S3hFlZWXz66afk5ubidDpJTEzEYDCE64oWLlzIgAEDTjtHudT+ZIlIahG3243b7T7tMevXr2fMmDENHkuWLGny2FtvvZVZs2bxgx/8gKlTp9K3b18uuOCCBsc89NBD5Obm0q9fP5KTkyOKe8GCBRiNRoYOHUpycjJ5eXnhfXPmzMHv93dIK6B0enKqWClmffTRR1x00UXk5+c3WoxQ6lgyEUkxx+fzcfToUWbNmkXPnj1ZvXq13iHFPHlrJsWcl156iT59+lBWVsYTTzyhdzgSskQkSVInIEtEkiTpTiYiSZJ0JxORJEm6k4lIkiTdyUQkSZLuZCKSJEl3MhFJkqQ7mYgkSdKdTESSJOnu/wEGUyoi/IlNCgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:48:51.367761Z", + "iopub.status.busy": "2024-02-29T03:48:51.367399Z", + "iopub.status.idle": "2024-02-29T03:48:51.603486Z", + "shell.execute_reply": "2024-02-29T03:48:51.602542Z" + }, + "papermill": { + "duration": 0.260439, + "end_time": "2024-02-29T03:48:51.605772", + "exception": false, + "start_time": "2024-02-29T03:48:51.345333", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEnCAYAAAA9/18kAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3tElEQVR4nO3de1xTR94/8A8ECHe8IFcRUCx4AxQrYvVRKxd1tVq6LVpbkFrcouyjTSkWW6FUV556QWrryrOueN1HaV1ru2pRGktXK3iBxaIiFATRSlBQiICGkMzvD3+cNiZAQDBw8n2/XnnhmTNnMsMhXydnzpkxYIwxEEIIDxnqugKEENJTKMARQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLeMdF2B3kipVOL27duwsrKCgYGBrqtDCHkCYwwPHjyAk5MTDA3b6aexXuCLL75grq6uTCgUsgkTJrBz5861mXfq1KkMgNpr9uzZXB6lUsnWrFnDHBwcmKmpKZsxYwYrKSnRuj43b97U+B70ohe9etfr5s2b7X6Wdd6Dy8jIgEgkQlpaGvz9/ZGamoqQkBAUFxfDzs5OLf/hw4fR3NzMbdfW1sLHxwevvvoql7ZhwwZs3boVe/bsgbu7O9asWYOQkBBcvXoVpqamHdbJysoKAHDz5k1YW1t3Qyt7J7lcjpMnTyI4OBjGxsa6rg55Svp0PqVSKVxcXLjPapu07tb0kAkTJrDly5dz2wqFgjk5ObHk5GStjt+yZQuzsrJiDQ0NjLHHvTcHBwe2ceNGLk9dXR0TCoXswIEDWpVZX1/PALD6+vpOtKTvaW5uZkeOHGHNzc26rgrpBvp0PrX9jOq0B9fc3Iy8vDzEx8dzaYaGhggMDEROTo5WZezcuRMLFiyAhYUFAKC8vBwSiQSBgYFcHhsbG/j7+yMnJwcLFixQK0Mmk0Emk3HbUqkUwOP/EeVyeZfa1he0to3PbdQn+nQ+tW2jTgNcTU0NFAoF7O3tVdLt7e1x7dq1Do8/f/48Ll++jJ07d3JpEomEK+PJMlv3PSk5ORlJSUlq6SdPnoS5uXmH9ejrsrKydF0F0o304Xw2NTVplU/n1+Cexs6dOzFmzBhMmDDhqcqJj4+HSCTitlu/3wcHB/P+GlxWVhaCgoJ4f81GH+jT+Wz9ltURnQY4W1tbCAQCVFdXq6RXV1fDwcGh3WMbGxtx8OBBfPLJJyrprcdVV1fD0dFRpUxfX1+NZQmFQgiFQrV0Y2Nj3v+hAPrTTn2hD+dT2/bp9EZfExMT+Pn5QSwWc2lKpRJisRgBAQHtHvvVV19BJpPhjTfeUEl3d3eHg4ODSplSqRTnzp3rsExCCL/o/CuqSCRCREQExo8fjwkTJiA1NRWNjY2IjIwEAISHh8PZ2RnJyckqx+3cuRPz58/HwIEDVdINDAywcuVKrFu3DsOHD+duE3FycsL8+fOfVbMIIb2AzgNcWFgY7t69i4SEBEgkEvj6+iIzM5MbJKisrFS7U7m4uBhnzpzByZMnNZYZFxeHxsZGLF26FHV1dZg8eTIyMzO1ugeOz5qamlQGbxoeynC2sAz9bS/C0kz1K7qXl5deDLAQfjNgjBadeZJUKoWNjQ3q6+t5NciQn58PPz8/rfLm5eVh3LhxPVwj0p3kcjmOHz+O2bNn8/4anLafUZ334Miz4+Xlhby8PG67uKoOoq8KkfLqGHg69lPLS0hfRwFOj5ibm6v0ygxv1EJ4+iFGjPaBr+vAdo4kpG+i6ZIIIbxFAY4QwlsU4AghvEUBjhDCWzTIQEgfRfc1dowCHCF91LVr1zTe17hBQ159va+RAhwhfRTd19gxCnCE9FF0X2PHaJCBEMJbFOAIIbxFAY4QwlsU4AghvEUBjhDCWxTgCCG8RQGOEMJbFOAIIbyl8wC3bds2uLm5wdTUFP7+/jh//ny7+evq6rB8+XI4OjpCKBTiueeew/Hjx7n9H3/8MQwMDFRe+noXNyH6TqdPMmRkZEAkEiEtLQ3+/v5ITU1FSEgIiouLYWdnp5a/ubkZQUFBsLOzw6FDh+Ds7IwbN26gX79+KvlGjRqF77//nts2MqIHNgjRRzr95KekpCAqKopbIjAtLQ3Hjh1Deno6PvjgA7X86enpuHfvHs6ePcstquHm5qaWz8jIqMOFowkh/KezANfc3Iy8vDzEx8dzaYaGhggMDEROTo7GY7799lsEBARg+fLl+OabbzBo0CC8/vrrWLVqFQQCAZfvl19+gZOTE0xNTREQEIDk5GQMGTKkzbrIZDLIZDJuWyqVAni8SpFcLn/apvZaLS0t3E8+t1Nf6NP51LZ9OgtwNTU1UCgU3Pqnrezt7VXmuPq969ev49SpU1i0aBGOHz+O0tJSLFu2DHK5HImJiQAAf39/7N69G56enqiqqkJSUhKmTJmCy5cvw8rKSmO5ycnJSEpKUks/efIkr+fQutkAAEbIzc3Fr5d1XRvytPTpfDY1NWmVr09dnFIqlbCzs8Pf/vY3CAQC+Pn54ddff8XGjRu5ADdr1iwuv7e3N/z9/eHq6oovv/wSS5Ys0VhufHw8RCIRty2VSuHi4oLg4GBerYv6pEuV94DCi5g4cSJ8hgzQdXXIU9Kn89n6LasjOgtwtra2EAgEqK6uVkmvrq5u8/qZo6MjjI2NVb6OjhgxAhKJBM3NzTAxMVE7pl+/fnjuuedQWlraZl2EQiGEQqFaurGxMa8X0G0dfDEyMuJ1O/WFPp1Pbduns9tETExM4OfnB7FYzKUplUqIxWIEBARoPOaFF15AaWkplEoll1ZSUgJHR0eNwQ0AGhoaUFZWBkdHx+5tACGk19PpfXAikQg7duzAnj17UFRUhOjoaDQ2NnKjquHh4SqDENHR0bh37x5WrFiBkpISHDt2DOvXr8fy5cu5PLGxsfjxxx9RUVGBs2fP4uWXX4ZAIMDChQufefsIIbql02twYWFhuHv3LhISEiCRSODr64vMzExu4KGyshKGhr/FYBcXF5w4cQLvvvsuvL294ezsjBUrVmDVqlVcnlu3bmHhwoWora3FoEGDMHnyZOTm5mLQoEHPvH2EEN3S+SBDTEwMYmJiNO7Lzs5WSwsICEBubm6b5R08eLC7qkYI6eN0/qgWIYT0FApwhBDeogBHCOEtCnCEEN7S+SAD6TnlNY1olLW0ub/sbiP3s70ZVyyERnC3tej2+hHS0yjA8VR5TSOmb8rWKu97hwo7zPND7DQKcqTPoQDHU609t9QwX3jYWWrO81CGo9k5mDMtABZm6o+qAUDpnQaszChotydInp32euXUI1dHAY7nPOwsMdrZRuM+uVwOySBgnGt/3j+7yAfa9sqpR/4bCnCE9BEd9cqpR66OAhwhfUxbvXLqkauj20QIIbxFAY4QwlsU4AghvEUBjhDCWxTgCCG8RQGOEMJbFOAIIbxFAY4Qwls6D3Dbtm2Dm5sbTE1N4e/vj/Pnz7ebv66uDsuXL4ejoyOEQiGee+45HD9+/KnKJITwk04DXEZGBkQiERITE5Gfnw8fHx+EhITgzp07GvM3NzcjKCgIFRUVOHToEIqLi7Fjxw44Ozt3uUxCCH/pNMClpKQgKioKkZGRGDlyJNLS0mBubo709HSN+dPT03Hv3j0cOXIEL7zwAtzc3DB16lT4+Ph0uUxCCH/p7FnU5uZm5OXlqax7amhoiMDAQOTk5Gg85ttvv0VAQACWL1+Ob775BoMGDcLrr7+OVatWQSAQdKlMAJDJZJDJZNy2VCoF8PjZPrlc/rRN1YmWlhbuZ1ttaE1vr43alEOejY7OhT6dT23rrrMAV1NTA4VCwa2B2sre3h7Xrl3TeMz169dx6tQpLFq0CMePH0dpaSmWLVsGuVyOxMTELpUJAMnJyUhKSlJLP3nyJMzNzbvQOt272QAARjhz5gxuaJ4OjpOVldUt5ZCepe250Ifz2dTUpFW+PjWbiFKphJ2dHf72t79BIBDAz88Pv/76KzZu3IjExMQulxsfHw+RSMRtS6VSuLi4IDg4GNbW1t1R9Wfuym0pNhXmYvLkyRjlpLkNcrkcWVlZCAoKanP2CW3KIc9GR+dCn85n67esjugswNna2kIgEKC6ulolvbq6Gg4ODhqPcXR0hLGxMQQCAZc2YsQISCQSNDc3d6lMABAKhRAK1efPMjY27rPTzrTO6GpkZNRhG9prZ2fKIT1L23OhD+dT27rrbJDBxMQEfn5+EIvFXJpSqYRYLEZAQIDGY1544QWUlpZCqVRyaSUlJXB0dISJiUmXyiSE8JdOR1FFIhF27NiBPXv2oKioCNHR0WhsbERkZCQAIDw8XGXAIDo6Gvfu3cOKFStQUlKCY8eOYf369Vi+fLnWZRJC9IdOr8GFhYXh7t27SEhIgEQiga+vLzIzM7lBgsrKShga/haDXVxccOLECbz77rvw9vaGs7MzVqxYgVWrVmldJiFEf+h8kCEmJgYxMTEa92VnZ6ulBQQEIDc3t8tl6guZ4hEMTX9FubQYhqaah8taWlpwu+U2iu4VtbkKU7m0AYamv0KmeARA8+I1hPRWOg9wpGfcbrwBC/fPsVqLp9T+mvnXdvdbuAO3G33hB+oFk76FAhxPOVm4orH8z/gszBfD2lgXtaWlBT+d+QkvTH6hzR5c2Z0GrMgogNN0156sLiE9ggIcTwkFplA+coa7tSdGDmx7XdRyo3KMGDCizWF35aN6KB/dhVBg2pPVJVro6LIDXXJQRwGOkD5C28sOdMnhNxTgCOkjOrrsQJcc1FGAI6SP6OiyA11yUKfzCS8JIaSnUIAjhPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwFgU4QghvUYAjhPAWBThCCG91OsBdv369J+pBCCHdrtMBzsPDA9OnT8f+/fvx6NGjnqgTIYR0i04HuPz8fHh7e0MkEsHBwQF/+tOfcP68FrMqEkLIM9bpAOfr64vPPvsMt2/fRnp6OqqqqjB58mSMHj0aKSkpuHv3bqcrsW3bNri5ucHU1BT+/v7tBszdu3fDwMBA5WVqqvrg8OLFi9XyzJw5s9P1IoT0bV0eZDAyMkJoaCi++uorfPrppygtLUVsbCxcXFwQHh6OqqoqrcrJyMiASCRCYmIi8vPz4ePjg5CQENy5c6fNY6ytrVFVVcW9bty4oZZn5syZKnkOHDjQ1aYSQvqoLge4ixcvYtmyZXB0dERKSgpiY2NRVlaGrKws3L59G/PmzdOqnJSUFERFRSEyMhIjR45EWloazM3NkZ6e3uYxBgYGcHBw4F6aVswSCoUqefr379/VphJC+qhOzweXkpKCXbt2obi4GLNnz8bevXsxe/Zsbnk/d3d37N69G25ubh2W1dzcjLy8PJW1Tw0NDREYGIicnJw2j2toaICrqyuUSiXGjRuH9evXY9SoUSp5srOzYWdnh/79++PFF1/EunXrMHDgQI3lyWQyyGQyblsqlQJ4PL+WXC7vsB29UUtLC/ezrTa0prfXRm3KIc9GR+dCn86ntnXvdIDbvn073nrrLSxevBiOjo4a89jZ2WHnzp0dllVTUwOFQqHWA7O3t8e1a9c0HuPp6Yn09HR4e3ujvr4emzZtwqRJk3DlyhUMHjwYwOOvp6GhoXB3d0dZWRlWr16NWbNmIScnBwKBQK3M5ORkJCUlqaWfPHkS5ubmHbajN7rZAABGOHPmDG5oXnOGk5WV1S3lkJ6l7bnQh/PZ1NSkVT4DxhjrTMEVFRUYMmSIyoLMAMAYw82bNzFkyBCty7p9+zacnZ1x9uxZBAQEcOlxcXH48ccfce7cuQ7LkMvlGDFiBBYuXIi1a9dqzHP9+nUMGzYM33//PWbMmKG2X1MPzsXFBTU1NbC2tta6Pb3JldtSzN+eiyPREzHKSXMb5HI5srKyEBQU1OYMsNqUQ56Njs6FPp1PqVQKW1tb1NfXt/sZ7XQPbtiwYaiqqoKdnZ1K+r179+Du7g6FQqF1Wba2thAIBKiurlZJr66uhoODg1ZlGBsbY+zYsSgtLW0zz9ChQ2Fra4vS0lKNAU4oFEIoFGosu60/lN6udU5+IyOjDtvQXjs7Uw7pWdqeC304n9rWvdODDG11+BoaGtRu1+iIiYkJ/Pz8IBaLuTSlUgmxWKzSo2uPQqFAYWFhm1+XAeDWrVuora1tNw8hhH+07sGJRCIAj0cwExISVK5NKRQKnDt3Dr6+vp2ugEgkQkREBMaPH48JEyYgNTUVjY2NiIyMBACEh4fD2dkZycnJAIBPPvkEEydOhIeHB+rq6rBx40bcuHEDb7/9NoDHgTYpKQmvvPIKHBwcUFZWhri4OHh4eCAkJKTT9SOE9F1aB7j//Oc/AB734AoLC2FiYsLtMzExgY+PD2JjYztdgbCwMNy9excJCQmQSCTw9fVFZmYmN/BQWVmpcr3v/v37iIqKgkQiQf/+/eHn54ezZ89i5MiRAACBQICff/4Ze/bsQV1dHZycnBAcHIy1a9dq/BpKCOEvrQPcDz/8AACIjIzEZ5991q0X32NiYhATE6NxX3Z2tsr2li1bsGXLljbLMjMzw4kTJ7qtboSQvqvTgwy7du3qiXoQQki30yrAhYaGYvfu3bC2tkZoaGi7eQ8fPtwtFSOEkKelVYCzsbGBgYEB92/S+z2UP75d5/Kv9W3maXwow8W7gMON+7Aw03x9svROQ4/Uj5BnQasA9/uvpfQVtW8o+/+B6YPDhR3kNMK+0gsdlmch7PTVDEJ0jv5qeSp41OMbpYfZWcLMWP3xNAAorqrHe4cKsfmPY+Dp2HbP3EJoBHdbix6pJyE9SasAN3bsWO4rakfy8/OfqkKkewywMMGCCe0/Ntf64PWwQRYY7UyXHgj/aBXg5s+f38PVIISQ7qdVgEtMTOzpehBCSLejZQMJIbylVQ9uwIABKCkpga2tLfr379/u9bh79+51W+UIIeRpaBXgtmzZAisrK+7f2g44EEK6T0f3NtJ9jeq0CnARERHcvxcvXtxTdSGEtEO7exvpvsbf63QrBQKBxgkva2trYWdn16kJLwkh2uvo3ka6r1FdpwNcWxNeymQylSmUCCHdq6N7G+m+RnVaB7itW7cCeDzh5d///ndYWv62YoVCocC///1veHl5dX8NCSGki7QOcK1zsDHGkJaWprI6lYmJCdzc3JCWltb9NSSEkC7SOsCVl5cDAKZPn47Dhw/TQsqEkF6v09fgWmf2JYSQ3q7TAe6tt95qd396enqnK7Ft2zZs3LgREokEPj4++PzzzzFhwgSNeXfv3s0tSNNKKBTi0aNH3DZjDImJidixYwfq6urwwgsvYPv27Rg+fHin60YI6bs6/ajW/fv3VV537tzBqVOncPjwYdTV1XW6AhkZGRCJREhMTER+fj58fHwQEhKCO3futHmMtbU1qqqquNeNGzdU9m/YsAFbt25FWloazp07BwsLC4SEhKgEQUII/3W6B/f111+rpSmVSkRHR2PYsGGdrkBKSgqioqK4XllaWhqOHTuG9PR0fPDBBxqPMTAwaHNhaMYYUlNT8dFHH2HevHkAgL1798Le3h5HjhzBggULOl1HQkjf1C23MxsaGkIkEmHatGmIi4vT+rjm5mbk5eUhPj5epazAwEDk5OS0eVxDQwNcXV2hVCoxbtw4rF+/HqNGjQLweDBEIpEgMDCQy29jYwN/f3/k5ORoDHAymQwymYzblkqlAAC5XA65XK51e/qa1vumWlpaeN1OfaFP51Pb9nXb8xplZWXcL1hbNTU1UCgU3Bqorezt7XHt2jWNx3h6eiI9PR3e3t6or6/Hpk2bMGnSJFy5cgWDBw+GRCLhyniyzNZ9T0pOTkZSUpJa+smTJ1UWuOabmw0AYITc3Fz8elnXtSFPS5/OZ1NTk1b5Oh3gWle4b8UYQ1VVFY4dO6byzGpPCQgIQEBAALc9adIkjBgxAv/7v/+LtWvXdqnM+Ph4lXZJpVK4uLggODi4W9d/7W0uVd4DCi9i4sSJ8BkyQNfVIU9Jn85n67esjnQ6wLWucN/K0NAQgwYNwubNmzscYX2Sra0tBAIBqqurVdKrq6vbvMb2JGNjY4wdOxalpaUAwB1XXV0NR0dHlTJ9fX01liEUCjWuem9sbAxjY2Ot6tEXGRkZcT/53E59oU/nU9v26fQ+OBMTE/j5+UEsFnPToiuVSojF4jZXun+SQqFAYWEhZs+eDQBwd3eHg4MDxGIxF9CkUinOnTuH6Ojobqs7IaT30/mcKSKRCBERERg/fjwmTJiA1NRUNDY2cqOq4eHhcHZ2RnJyMgDgk08+wcSJE+Hh4YG6ujps3LgRN27cwNtvvw3g8QjrypUrsW7dOgwfPhzu7u5Ys2YNnJycaG0JQvSMzgNcWFgY7t69i4SEBEgkEvj6+iIzM5MbJKisrISh4W+3692/fx9RUVGQSCTo378//Pz8cPbsWYwcOZLLExcXh8bGRixduhR1dXWYPHkyMjMzYWpq+szbRwjRHQPW1vxHekwqlcLGxgb19fW8HmQouFGL+dtzcSR6InxdB+q6OuQp6dP51PYzSovOEEJ4q9sC3K1bt7B06dLuKo4QQp5atwW42tpa7Ny5s7uKI4SQp0ZfUQkhvEUBjhDCWxTgCCG8pfV9cKGhoe3u78pccIQQ0pO0DnA2Nu0vQ2ZjY4Pw8PCnrhAhhHQXrQPcrl27erIehBDS7egaHCGEt7TuwWk7FVJXFp0hhJCeoHWA2717N1xdXTF27FjQ46uEkL5A6wAXHR2NAwcOoLy8HJGRkXjjjTcwYAC/Zw0lhPRtWl+D27ZtG6qqqhAXF4d//etfcHFxwWuvvYYTJ05Qj44Q0it1apBBKBRi4cKFyMrKwtWrVzFq1CgsW7YMbm5uaGho6Kk6EkJIl3R5FNXQ0BAGBgZgjEGhUHRnnQghpFt0KsDJZDIcOHAAQUFBeO6551BYWIgvvvgClZWVsLS07Kk6EkJIl2g9yLBs2TIcPHgQLi4ueOutt3DgwAHY2tr2ZN0IIeSpaN2DS0tLg7W1NYYOHYoff/wRS5cuRWhoqNqrK7Zt2wY3NzeYmprC398f58+f1+q4gwcPwsDAQG0xmcWLF8PAwEDlNXPmzC7VjRDSd2ndgwsPD4eBgUG3VyAjIwMikQhpaWnw9/dHamoqQkJCUFxcDDs7uzaPq6ioQGxsLKZMmaJx/8yZM1UeL9O07ikhhN86daNvT0hJSUFUVBS3TGBaWhqOHTuG9PR0fPDBBxqPUSgUWLRoEZKSknD69GmNM5kIhUKtF48mhPCTTpcNbG5uRl5eHuLj47k0Q0NDBAYGIicnp83jPvnkE9jZ2WHJkiU4ffq0xjzZ2dmws7ND//798eKLL2LdunUYOFDzSkMymQwymYzblkqlAAC5XA65XN6VpvUJLS0t3E8+t1Nf6NP51LZ9Og1wNTU1UCgU3Bqorezt7XHt2jWNx5w5cwY7d+5EQUFBm+XOnDkToaGhcHd3R1lZGVavXo1Zs2YhJycHAoFALX9ycjKSkpLU0k+ePAlzc/PONaoPudkAAEbIzc3Fr5d1XRvytPTpfDY1NWmVT+cLP3fGgwcP8Oabb2LHjh3tjuAuWLCA+/eYMWPg7e2NYcOGITs7GzNmzFDLHx8fD5FIxG1LpVK4uLggODiY1+uiXqq8BxRexMSJE+EzhB676+v06Xy2fsvqiE4DnK2tLQQCAaqrq1XSq6urNV4/KysrQ0VFBebOnculKZVKAICRkRGKi4sxbNgwteOGDh0KW1tblJaWagxwQqFQ4yCEsbExjI2NO92uvsLIyIj7yed26gt9Op/atk+n88GZmJjAz88PYrGYS1MqlRCLxQgICFDL7+XlhcLCQhQUFHCvl156CdOnT0dBQQFcXFw0vs+tW7dQW1sLR0fHHmsLIaT30flXVJFIhIiICIwfPx4TJkxAamoqGhsbuVHV8PBwODs7Izk5Gaamphg9erTK8f369QMALr2hoQFJSUl45ZVX4ODggLKyMsTFxcHDwwMhISHPtG2EEN3SeYALCwvD3bt3kZCQAIlEAl9fX2RmZnIDD5WVlTA01L6jKRAI8PPPP2PPnj2oq6uDk5MTgoODsXbtWroXjhA9o/MABwAxMTGIiYnRuC87O7vdY5+8P8/MzAwnTpzoppoRQvoyWpOBEMJbFOAIIbxFAY4QwlsU4AghvEUBjhDCWxTgCCG8RQGOEMJbFOAIIbxFAY4QwlsU4AghvEUBjhDCWxTgCCG8RQGOEMJbFOAIIbxFAY4QwlsU4AghvEUBjhDCWxTgCCG81SsC3LZt2+Dm5gZTU1P4+/vj/PnzWh138OBBGBgYYP78+SrpjDEkJCTA0dERZmZmCAwMxC+//NIDNSeE9GY6D3AZGRkQiURITExEfn4+fHx8EBISgjt37rR7XEVFBWJjYzFlyhS1fRs2bMDWrVuRlpaGc+fOwcLCAiEhIXj06FFPNYMQ0gvpPMClpKQgKioKkZGRGDlyJNLS0mBubo709PQ2j1EoFFi0aBGSkpIwdOhQlX2MMaSmpuKjjz7CvHnz4O3tjb179+L27ds4cuRID7eGENKb6HRVrebmZuTl5SE+Pp5LMzQ0RGBgIHJycto87pNPPoGdnR2WLFmC06dPq+wrLy+HRCJBYGAgl2ZjYwN/f3/k5ORgwYIFauXJZDLIZDJuWyqVAgDkcjnkcnmX29fbNDU1obi4mNsuqaqHTFKKywUmaK62Ucnr6ekJc3PzZ11F8hRaWlq4n3z6u9VE2/bpNMDV1NRAoVBwa6C2sre3x7Vr1zQec+bMGezcuRMFBQUa90skEq6MJ8ts3fek5ORkJCUlqaWfPHmSVx/ysrIyvPfee2rpb+5Rz7t582YMGzbsGdSKdJebDQBghNzcXPx6Wde16VlNTU1a5esV66Jq68GDB3jzzTexY8cO2Nradlu58fHxEIlE3LZUKoWLiwuCg4NhbW3dbe+ja01NTZg8eTK33fBQhhOnLyBkyvOwNFNdFJt6cH3Ppcp7QOFFTJw4ET5DBui6Oj2q9VtWR3Qa4GxtbSEQCFBdXa2SXl1dDQcHB7X8ZWVlqKiowNy5c7k0pVIJADAyMkJxcTF3XHV1NRwdHVXK9PX11VgPoVCocdV7Y2NjGBsbd7pdvZWNjQ0mTJjAbcvlcjyou4cpkybyqp36ysjIiPvJ9/Opbft0OshgYmICPz8/iMViLk2pVEIsFiMgIEAtv5eXFwoLC1FQUMC9XnrpJUyfPh0FBQVwcXGBu7s7HBwcVMqUSqU4d+6cxjIJIfyl86+oIpEIERERGD9+PCZMmIDU1FQ0NjYiMjISABAeHg5nZ2ckJyfD1NQUo0ePVjm+X79+AKCSvnLlSqxbtw7Dhw+Hu7s71qxZAycnJ7X75Qgh/KbzABcWFoa7d+8iISEBEokEvr6+yMzM5AYJKisrYWjYuY5mXFwcGhsbsXTpUtTV1WHy5MnIzMyEqalpTzSBENJLGTDGmK4r0dtIpVLY2Nigvr6eV4MMT5LL5Th+/Dhmz57N+2s2+qDgRi3mb8/FkeiJ8HUdqOvq9ChtP6M6v9GXEEJ6CgU4QghvUYAjhPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwFgU4QghvUYAjhPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwFgU4QghvUYAjhPCWzie8JIR0TVNTk8rqc8VVdZBJSlF02QzK2n4qeb28vPRyESEKcIT0UdeuXYOfn59a+usaloHMy8vDuHHjnkGtehcKcIT0UV5eXsjLy+O2Gx7KcOyHHPxheoDaMpBeXl7Punq9Qq+4Brdt2za4ubnB1NQU/v7+OH/+fJt5Dx8+jPHjx6Nfv36wsLCAr68v9u3bp5Jn8eLFMDAwUHnNnDmzp5tByDNlbm6OcePGca+ACeMxacwwBEwYr5I+btw4vfx6CvSCHlxGRgZEIhHS0tLg7++P1NRUhISEoLi4GHZ2dmr5BwwYgA8//BBeXl4wMTHB0aNHERkZCTs7O4SEhHD5Zs6ciV27dnHbmtY9JYTwm857cCkpKYiKikJkZCRGjhyJtLQ0mJubIz09XWP+adOm4eWXX8aIESMwbNgwrFixAt7e3jhz5oxKPqFQCAcHB+7Vv3//Z9EcQkgvotMeXHNzM/Ly8hAfH8+lGRoaIjAwEDk5OR0ezxjDqVOnUFxcjE8//VRlX3Z2Nuzs7NC/f3+8+OKLWLduHQYO1LzSkEwmg0wm47alUimAx6tOyeXyrjStT2htG5/bqE/06Xxq20adBriamhooFApuDdRW9vb2KsPfT6qvr4ezszNkMhkEAgH++te/IigoiNs/c+ZMhIaGwt3dHWVlZVi9ejVmzZqFnJwcCAQCtfKSk5ORlJSkln7y5Em9uHaRlZWl6yqQp6RQKHD16lXcv38fhYWFGDlypMa/db5oamrSKp/Or8F1hZWVFQoKCtDQ0ACxWAyRSIShQ4di2rRpAIAFCxZweceMGQNvb28MGzYM2dnZmDFjhlp58fHxEIlE3LZUKoWLiwuCg4N5vy5qVlYWgoKCaF3UPuzrr7/GqlWrUFFRwaW5ubnh008/xcsvv6y7ivWg1m9ZHdFpgLO1tYVAIEB1dbVKenV1NRwcHNo8ztDQEB4eHgAAX19fFBUVITk5mQtwTxo6dChsbW1RWlqqMcAJhUKNgxDGxsZ68cHXl3by0eHDh7FgwQLMmTMH+/btw61btzB48GBs2LABCxYswKFDhxAaGqrranY7bf9edTrIYGJiAj8/P4jFYi5NqVRCLBYjICBA63KUSqXKNbQn3bp1C7W1tXB0dHyq+hLSmygUCrz33nuYM2cOjhw5An9/f5iZmcHf3x9HjhzBnDlzEBsbC4VCoeuq6ozOR1FFIhF27NiBPXv2oKioCNHR0WhsbERkZCQAIDw8XGUQIjk5GVlZWbh+/TqKioqwefNm7Nu3D2+88QYAoKGhAe+//z5yc3NRUVEBsViMefPmwcPDQ+U2EkL6utOnT6OiogKrV6+GoaHqR9nQ0BDx8fEoLy/H6dOndVRD3dP5NbiwsDDcvXsXCQkJkEgk8PX1RWZmJjfwUFlZqXLyGhsbsWzZMty6dQtmZmbw8vLC/v37ERYWBgAQCAT4+eefsWfPHtTV1cHJyQnBwcFYu3Yt3QtHeKWqqgoAMHr0aI37W9Nb8+kjnQc4AIiJiUFMTIzGfdnZ2Srb69atw7p169osy8zMDCdOnOjO6hHSK7Vecrl8+TImTpyotv/y5csq+fSRzr+iEkK6ZsqUKXBzc8P69euhVCpV9imVSiQnJ8Pd3R1TpkzRUQ11r1f04AghnScQCLB582b88Y9/xLx58xAUFIRffvkFN27cQFZWFo4dO4ZDhw7x+n64jlCAI6QPCw0NRWxsLLZs2YKjR49y6UZGRoiNjeXlLSKdQQGOkD7s8OHD2LRpE/7whz9wPbjhw4cjKysLmzZtwsSJE/U6yBkwxpiuK9HbSKVS2NjYoL6+nvdPMhw/fhyzZ8+mG337IIVCAQ8PD4wZMwZHjhyBQqHgzqdAIMD8+fNx+fJl/PLLL7z7mqrtZ5QGGQjpo+g+uI5RgCOkj6L74DpGAY6QPur398FpQvfBUYAjpM+i++A6RgGOkD6q9T64o0ePYv78+cjNzcXDhw+Rm5uL+fPn4+jRo9i0aRPvBhg6g24TIaQPCw0NxaFDh/Dee+/hv/7rv7h0d3d33k6V1BkU4Ajp40JDQzFv3jz88MMP+O677zBr1ixMnz5dr3turSjAEcIDAoEAU6dORWNjI6ZOnUrB7f+ja3CEEN6iAEcI4S0KcIQQ3qJrcBq0Pp6r7co9fZVcLkdTUxOkUik9i8oD+nQ+Wz+bHT1KTwFOgwcPHgAAXFxcdFwTQkh7Hjx4ABsbmzb302wiGiiVSty+fRtWVlYwMDDQdXV6TOv6rzdv3uT1rCn6Qp/OJ2MMDx48gJOTk9pEA79HPTgNDA0NMXjwYF1X45mxtrbm/QdCn+jL+Wyv59aKBhkIIbxFAY4QwlsU4PSYUChEYmIirRfLE3Q+1dEgAyGEt6gHRwjhLQpwhBDeogBHCOEtCnDdYPHixZg/f363ljlt2jSsXLmy3Txubm5ITU3t1vclz5425/pZ6m31eRoU4J7Ap5NLHvv444/h6+ur62r0qObmZl1XoVeiAEeIDi1evBg//vgjPvvsMxgYGMDAwABlZWVYsmQJ3N3dYWZmBk9PT3z22Wdqx82fPx9/+ctf4OTkBE9PTwDA2bNn4evrC1NTU4wfPx5HjhyBgYEBCgoKuGMvX76MWbNmwdLSEvb29njzzTdRU1PTZn0qKiqe1a+j21GA+52u/rG1SkpKwqBBg2BtbY133nlH6/9VGxsbER4eDktLSzg6OmLz5s1qee7cuYO5c+fCzMwM7u7u+Mc//qGWx8DAANu3b8esWbNgZmaGoUOH4tChQ9z+iooKGBgY4Msvv8SUKVNgZmaG559/HiUlJbhw4QLGjx8PS0tLzJo1C3fv3tXytwakp6dj1KhREAqFcHR0RExMDLevsrIS8+bNg6WlJaytrfHaa6+hurqa29/au9q3bx/c3NxgY2ODBQsWcBMeAI+fDd6wYQM8PDwgFAoxZMgQ/OUvf+H2r1q1Cs899xzMzc0xdOhQrFmzBnK5HACwe/duJCUl4dKlS9w53b17t9Zt62mfffYZAgICEBUVhaqqKlRVVWHw4MEYPHgwvvrqK1y9ehUJCQlYvXo1vvzyS5VjxWIxiouLkZWVhaNHj0IqlWLu3LkYM2YM8vPzsXbtWqxatUrlmLq6Orz44osYO3YsLl68iMzMTFRXV+O1115rsz59etIJRjh1dXUsICCARUVFsaqqKlZVVcUePXrEEhIS2IULF9j169fZ/v37mbm5OcvIyOCOi4iIYJaWliwsLIxdvnyZHT16lA0aNIitXr1aq/eNjo5mQ4YMYd9//z37+eef2Zw5c5iVlRVbsWIFl2fWrFnMx8eH5eTksIsXL7JJkyYxMzMztmXLFi4PADZw4EC2Y8cOVlxczD766CMmEAjY1atXGWOMlZeXMwDMy8uLZWZmsqtXr7KJEycyPz8/Nm3aNHbmzBmWn5/PPDw82DvvvKNV3f/6178yU1NTlpqayoqLi9n58+e5OikUCubr68smT57MLl68yHJzc5mfnx+bOnUqd3xiYiKztLRkoaGhrLCwkP373/9mDg4OKr+7uLg41r9/f7Z7925WWlrKTp8+zXbs2MHtX7t2Lfvpp59YeXk5+/bbb5m9vT379NNPGWOMNTU1sffee4+NGjWKO6dNTU1ate1ZmTp1qsq51mT58uXslVde4bYjIiKYvb09k8lkXNr27dvZwIED2cOHD7m0HTt2MADsP//5D2Ps8e8qODhYpeybN28yAKy4uFjr+vQVFOCe0NU/tgEDBrDGxkYubfv27czS0pIpFIp2y3rw4AEzMTFhX375JZdWW1vLzMzMuHoUFxczAOz8+fNcnqKiIgZALcA9GZj8/f1ZdHQ0Y+y3APf3v/+d23/gwAEGgInFYi4tOTmZeXp6tlvvVk5OTuzDDz/UuO/kyZNMIBCwyspKLu3KlSsqbUlMTGTm5uZMKpVyed5//33m7+/PGGNMKpUyoVCoEtA6snHjRubn58dtJyYmMh8fH62Pf9Y0/c198cUXbNy4cczW1pZZWFgwY2Nj9vzzz3P7IyIiWGBgoMoxK1euZNOnT1dJu3TpkkqA++Mf/8iMjY2ZhYWFygsAO378eJv16atoNhEtbNu2Denp6aisrMTDhw/R3NysdtHax8cH5ubm3HZAQAAaGhpw8+ZNuLq6tll2WVkZmpub4e/vz6UNGDCAu6YCAEVFRTAyMoKfnx+X5uXlhX79+qmVFxAQoLb9++svAODt7c39297eHgAwZswYlbQ7d+60WedWd+7cwe3btzFjxgyN+4uKiuDi4qLyFWfkyJHo168fioqK8PzzzwN4PBpsZWXF5XF0dOTev6ioCDKZrM33AICMjAxs3boVZWVlaGhoQEtLS5+eTePgwYOIjY3F5s2bERAQACsrK2zcuBHnzp1TyWdhYdHpshsaGjB37lx8+umnavscHR27XOfeigJcB7T9Y+tLfj/ba+t8d0+mPblSuiZmZmbdXp8n37+j98jJycGiRYuQlJSEkJAQ2NjY4ODBgxqvY/ZWJiYmUCgU3PZPP/2ESZMmYdmyZVxaWVlZh+V4enpi//79kMlk3POoFy5cUMkzbtw4/POf/4SbmxuMjDR//J+sT19GgwxPaO+PbezYsfDw8ND4x3bp0iU8fPiQ287NzYWlpWWHF2iHDRsGY2NjlYB5//59lJSUcNteXl5oaWlBXl4el1ZcXIy6ujq18nJzc9W2R4wY0W4dusrKygpubm4Qi8Ua948YMQI3b97EzZs3ubSrV6+irq4OI0eO1Oo9hg8fDjMzszbf4+zZs3B1dcWHH36I8ePHY/jw4bhx44ZKnt7+gXVzc8O5c+dQUVGBmpoaDB8+HBcvXsSJEydQUlKCNWvWqAUqTV5//XUolUosXboURUVFOHHiBDZt2gTgt//Ili9fjnv37mHhwoW4cOECysrKcOLECURGRnK/oyfro81/dr0VBbgndPWPrbm5GUuWLMHVq1dx/PhxJCYmIiYmpt3ZRgHA0tISS5Yswfvvv49Tp07h8uXLWLx4scpxnp6emDlzJv70pz/h3LlzyMvLw9tvv62xd/PVV18hPT0dJSUlSExMxPnz51VGNbvbxx9/jM2bN2Pr1q345ZdfkJ+fj88//xwAEBgYiDFjxmDRokXIz8/H+fPnER4ejqlTp2L8+PFalW9qaopVq1YhLi4Oe/fuRVlZGXJzc7Fz504AjwNgZWUlDh48iLKyMmzduhVff/21Shlubm4oLy9HQUEBampqIJPJuveX8JRiY2MhEAgwcuRIDBo0CCEhIQgNDUVYWBj8/f1RW1ur0ptri7W1Nf71r3+hoKAAvr6++PDDD5GQkADg8e8RAJycnPDTTz9BoVAgODgYY8aMwcqVK9GvXz/ub+7J+lRWVvZc43uari8C9jbFxcVs4sSJzMzMjAFg165dY4sXL2Y2NjasX79+LDo6mn3wwQcqF60jIiLYvHnzWEJCAhs4cCCztLRkUVFR7NGjR1q954MHD9gbb7zBzM3Nmb29PduwYYPahd6qqir2hz/8gQmFQjZkyBC2d+9e5urqqjbIsG3bNhYUFMSEQiFzc3NTGe1tHWRoveDMGGM//PADA8Du37/Ppe3atYvZ2Nho/TtLS0tjnp6ezNjYmDk6OrI///nP3L4bN26wl156iVlYWDArKyv26quvMolEwu3XNACwZcsW5urqym0rFAq2bt065urqyoyNjdmQIUPY+vXruf3vv/8+93sPCwtjW7ZsUan/o0eP2CuvvML69evHALBdu3Zp3ba+bv/+/czY2LjXjRw/KzRdEo8YGBjg66+/7vbHxkjfsXfvXgwdOhTOzs64dOkSYmJiMG3aNOzfv1/XVdMJGmQghEckEgkSEhIgkUjg6OiIV199VeWmaH1DPbgeVllZ2e4F9atXr2LIkCHd8l490YOztLRsc993332HKVOmdNt7EdLdKMD1sJaWlnaf5WtvuL43KC0tbXOfs7Nzt90qQkhPoABHCOEtuk2EEMJbFOAIIbxFAY4QwlsU4AghvEUBjujU4sWLuYkojY2NYW9vj6CgIKSnp3fqGcjdu3drnF2lp/XEehyk+1CAIzo3c+ZMVFVVoaKiAt999x2mT5+OFStWYM6cOWhpadF19UhfprunxAj57TneJ4nFYgaAm+hy8+bNbPTo0czc3JwNHjyYRUdHswcPHjDGfnue9vevxMRExhhje/fuZX5+fszS0pLZ29uzhQsXsurqau597t27x15//XVma2vLTE1NmYeHB0tPT+f2V1ZWsldffZXZ2Niw/v37s5deeomVl5czxh4/R/vk+/7www898nsiXUM9ONIrvfjii/Dx8cHhw4cBAIaGhti6dSuuXLmCPXv24NSpU4iLiwMATJo0CampqbC2tubWEYiNjQUAyOVyrF27FpcuXcKRI0dQUVGBxYsXc++zZs0aXL16Fd999x2Kioqwfft22NracseGhITAysoKp0+fxk8//QRLS0vMnDkTzc3NiI2NxWuvvcb1QKuqqjBp0qRn+4si7dN1hCX6ra0eHGOMhYWFsREjRmjc99VXX7GBAwdy29rOgHLhwgUGgOv9zZ07l0VGRmrMu2/fPubp6cmUSiWXJpPJmJmZGTtx4kSH9Se6Rz040msxxriJGr///nvMmDEDzs7OsLKywptvvona2lo0NTW1W0ZeXh7mzp2LIUOGwMrKClOnTgUAbo6z6OhoHDx4EL6+voiLi8PZs2e5Yy9duoTS0lJYWVnB0tISlpaWGDBgAB49eqTVDLtE9yjAkV6rqKgI7u7uqKiowJw5c+Dt7Y1//vOfyMvLw7Zt2wC0v+BxY2MjQkJCYG1tjX/84x+4cOECNxlm63GzZs3CjRs38O6773LrS7R+vW1oaICfnx8KCgpUXiUlJXj99dd7uPWkO/Tep7yJXjt16hQKCwvx7rvvIi8vD0qlEps3b+ZmnX1yjVBN05Jfu3YNtbW1+J//+R9u6viLFy+qvdegQYMQERGBiIgITJkyBe+//z42bdqEcePGISMjA3Z2dm0uYtPbp0PXd9SDIzonk8kgkUjw66+/Ij8/H+vXr8e8efMwZ84chIeHw8PDA3K5HJ9//jmuX7+Offv2IS0tTaUMNzc3NDQ0QCwWo6amBk1NTRgyZAhMTEy447799lusXbtW5biEhAR88803KC0txZUrV3D06FFuDYtFixbB1tYW8+bNw+nTp1FeXo7s7Gz893//N27dusW9788//4zi4mLU1NRwC06TXkLXFwGJfouIiOBusTAyMmKDBg1igYGBLD09XWVN2ZSUFObo6MjMzMxYSEgI27t3r9pU6++88w4bOHCgym0i//d//8fc3NyYUChkAQEB7Ntvv1VbCHnEiBHMzMyMDRgwgM2bN49dv36dK7OqqoqFh4czW1tbJhQK2dChQ1lUVBSrr69njDF2584dFhQUxCwtLek2kV6IpksihPAWfUUlhPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwFgU4QghvUYAjhPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwFgU4Qghv/T8QsOzCvPSo1wAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T03:48:51.651436Z", + "iopub.status.busy": "2024-02-29T03:48:51.650512Z", + "iopub.status.idle": "2024-02-29T03:48:51.923066Z", + "shell.execute_reply": "2024-02-29T03:48:51.922100Z" + }, + "papermill": { + "duration": 0.297895, + "end_time": "2024-02-29T03:48:51.925242", + "exception": false, + "start_time": "2024-02-29T03:48:51.627347", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABa00lEQVR4nO2dd3iUxdbAf1uz6QkEUiCQhBKqgDQpiiVcQBTwegUR6Z/YsCFIUQgCCgIiqFy8ooDYKF7bRUAxUqRjCD2A1ISSEALpZbO78/2x2U2WbEISNtmU+T3P+2R33nnnPdtOzsw5c45CCCGQSCSSGojS2QJIJBJJRSEVnEQiqbFIBSeRSGosUsFJJJIai1RwEomkxiIVnEQiqbFIBSeRSGosUsFJJJIai9rZAlRFTCYTV65cwdPTE4VC4WxxJBLJLQghSE9PJygoCKWyeDtNKjg7XLlyheDgYGeLIZFIbkN8fDwNGzYs9rxUcHbw9PQEzG+el5eXk6WRSCS3kpaWRnBwsPW3WhxSwdnBMi318vKSCk4iqcLcbglJOhkkEkmNRSo4iURSY5EKTiKR1FjkGlw5EUJgMBgwGo3OFkUiqbaoVCrUanWFhWNJBVcO9Ho9V69eJSsry9miSCTVHjc3NwIDA9FqtQ4fu0oouKVLl7JgwQISEhJo164dH330EV26dLHbd/ny5axevZpjx44B0LFjR959912b/kIIIiMjWb58OSkpKfTo0YNly5bRrFmzO5bVZDJx/vx5VCoVQUFBaLVaGQwskZQDIQR6vZ6kpCTOnz9Ps2bNSgzaLe9NnMqaNWuEVqsVK1asEMePHxfPPPOM8PHxEYmJiXb7P/XUU2Lp0qUiJiZGxMbGilGjRglvb29x6dIla5958+YJb29v8eOPP4rDhw+LAQMGiNDQUJGdnV0qmVJTUwUgUlNTi5zLzs4WJ06cEJmZmeV7wRJJLeFKSpaIv5EpTCZTif0yMzPFiRMnSv37FKLk32hhnK7gunTpIl588UXrc6PRKIKCgsTcuXNLdb3BYBCenp7iiy++EEIIYTKZREBAgFiwYIG1T0pKinBxcRHffvttqcYsjYIry4chkdQ28gxGcTj+pjgcf1Pk6A0l9i3Pb6q0Cs6pXlS9Xk90dDQRERHWNqVSSUREBHv27CnVGFlZWeTl5VGnTh0Azp8/T0JCgs2Y3t7edO3atdgxc3NzSUtLszkkEkn5yTGYrI9zCz2ubJyq4K5fv47RaMTf39+m3d/fn4SEhFKNMXnyZIKCgqwKzXJdWcacO3cu3t7e1kPuQ5VI7oycvILoghyD8yINqnUc3Lx581izZg0//PADOp2u3ONMnTqV1NRU6xEfH+9AKas3o0aNYtCgQQ4d8/777+fVV18tsU9ISAiLFy926H0llUduIQU3sN8/bvt5VxROVXB+fn6oVCoSExNt2hMTEwkICCjx2oULFzJv3jx+++037rrrLmu75bqyjOni4mLdd1qT95+WRrFIqhczZ86kffv2zhajCDl5BdNSkxNLLztVwWm1Wjp27EhUVJS1zWQyERUVRbdu3Yq9bv78+cyePZvNmzfTqVMnm3OhoaEEBATYjJmWlsa+fftKHFMikRRPXl5eqfsKIWympSaTQDhJyTl9ijphwgSWL1/OF198QWxsLM8//zyZmZmMHj0agBEjRjB16lRr//fee4/p06ezYsUKQkJCSEhIICEhgYyMDMCcXeDVV19lzpw5/Pzzzxw9epQRI0YQFBTk8KmWBSEEWXpDpR9l+dKMGjWK7du3s2TJEhQKBQqFgrNnzzJ27FhCQ0NxdXUlPDycJUuW2L3+7bffpl69enh5efHcc8+h1+tLdd/MzExGjBiBh4cHgYGBvP/++0X6XLt2jUcffRRXV1dCQ0P5+uuvi/RRKBQsW7aMfv364erqSlhYGN999531/IULF1AoFKxbt457770XV1dXOnfuzOnTpzlw4ACdOnXCw8ODfv36kZSUVMp3DVasWEHr1q1xcXEhMDCQ8ePHW8/FxcUxcOBAPDw88PLyYvDgwTYzB4t19eWXXxISEoK3tzdPPvkk6enp1j4mk4n58+fTtGlTXFxcaNSoEe+88471/OTJk2nevDlubm6EhYUxffp0q7JZtWoVb7/9NocPH7Z+pqtWrbrtazp58iQ9e/ZEp9PRqlUrfv/9dxQKBT/++KPNe7l27Vp69eqFTqfj66+/Jjk5maFDh9KgQQPc3Nxo27Yt3377rc3YmZmZDB8+gs7NGvBQxxas/s/H5tfpJCPO6YG+Q4YMISkpiRkzZpCQkED79u3ZvHmz1UkQFxdnE/y3bNky9Ho9//rXv2zGiYyMZObMmQC88cYbZGZmMm7cOFJSUujZsyebN2++o3W6ksjOM9Jqxq8VMnZJnJjVBzdt6T7CJUuWcPr0adq0acOsWbMA8PX1pWHDhqxfv566deuye/duxo0bR2BgIIMHD7ZeGxUVhU6nY9u2bVy4cIHRo0dTt25dmx9icUyaNInt27fz008/Ub9+faZNm8bBgwdtplWjRo3iypUrbN26FY1Gw8svv8y1a9eKjDV9+nTmzZvHkiVL+PLLL3nyySc5evQoLVu2tPaJjIxk8eLFNGrUiDFjxvDUU0/h6enJkiVLcHNzY/DgwcyYMYNly5bdVvZly5YxYcIE5s2bR79+/UhNTWXXrl2AWTFZlNv27dsxGAy8+OKLDBkyhG3btlnHOHv2LD/++CMbNmzg5s2bDB48mHnz5lnfu6lTp7J8+XI++OADevbsydWrVzl58qT1ek9PT1atWkVQUBBHjx7lmWeewdPTkzfeeIMhQ4Zw7NgxNm/ezO+//w6YIwZKwmg0MmjQIBo1asS+fftIT0/n9ddft9t3ypQpvP/++3To0AGdTkdOTg4dO3Zk8uTJeHl58csvvzB8+HCaNGliDbSfNGkSO3bsYMnnXxMQ4M+id98m9tgROne8+7bvd4VQ6sCTWkRZ4+Ayc/NE48kbKv3IzM0r0+vq1auXeOWVV0rs8+KLL4rHH3/c+nzkyJGiTp06NoHNy5YtEx4eHsJoNJY4Vnp6utBqtWLdunXWtuTkZOHq6mqV49SpUwIQ+/fvt/aJjY0VgPjggw+sbYB47rnnbMbv2rWreP7554UQQpw/f14A4rPPPrOe//bbbwUgoqKirG1z584V4eHhJcptISgoSLz55pt2z/32229CpVKJuLg4a9vx48dtXktkZKRwc3MTaWlp1j6TJk0SXbt2FUIIkZaWJlxcXMTy5ctLJY8QQixYsEB07NjR+jwyMlK0a9eu1Ndv2rRJqNVqcfXqVWvbli1bBCB++OEHIUTBe7l48eLbjte/f3/x+uuvCyEKPu/lX3wlDsffFBevZ4pDf8cJnc5VjHt+fLFjVGQcnNMtuJqAq0bFiVl9nHLfO2Xp0qWsWLGCuLg4srOz0ev1RRat27Vrh5ubm/V5t27dyMjIID4+nsaNGxc79tmzZ9Hr9XTt2tXaVqdOHcLDw63PY2NjUavVdOzY0drWokULfHx8iox36xpqt27dOHTokE1bYYeTZRbQtm1bmzZ71uGtXLt2jStXrvDQQw/ZPR8bG0twcLBNSFGrVq3w8fEhNjaWzp07A2ZvcOGss4GBgdb7x8bGkpubW+w9ANauXcuHH37I2bNnycjIwGAw3JET7NSpUwQHB9s43IrbFnnr+rbRaOTdd99l3bp1XL58Gb1eT25urvW7Yfm8W7c3X6fTKPGvV4/GTZpiNDknFk4qOAegUChKPVWsSqxZs4aJEyfy/vvv061bNzw9PVmwYAH79u1ztmjlRqPRWB9b9gjf2mYqxY/N1dXV4fLcev/b3WPPnj0MGzaMt99+mz59+uDt7c2aNWvsrmNWBO7u7jbPFyxYwJIlS1i8eDFt27bF3d2dV199tch6bG6eEXdAp1FZPahGJ8X6Ot3JIKk8tFqtTXqnXbt20b17d1544QU6dOhA06ZNOXv2bJHrDh8+THZ2tvX53r178fDwuG1AdJMmTdBoNDYK8+bNm5w+fdr6vEWLFhgMBqKjo61tp06dIiUlpch4e/fuLfK88PqbI/H09CQkJMTGG1+Yli1bEh8fbxMzeeLECVJSUmjVqlWp7tGsWTNcXV2Lvcfu3btp3Lgxb775Jp06daJZs2ZcvHjRps+tn+ntCA8PJz4+3sYZcuDAgVJdu2vXLgYOHMjTTz9Nu3btCAsLs/ksLZ939F/m8XQaJVnpqVw8dxaDyeQUT2r1Mzsk5SYkJIR9+/Zx4cIFPDw8aNasGatXr+bXX38lNDSUL7/8kgMHDhAaGmpznV6vZ+zYsbz11ltcuHCByMhIxo8ff9vMDx4eHowdO5ZJkyZRt25d6tevz5tvvmlzXXh4OH379uXZZ59l2bJlqNVqXn31VbvWzfr16+nUqRM9e/bk66+/Zv/+/Xz++eeOeXPsMHPmTJ577jnq169Pv379SE9PZ9euXbz00ktERETQtm1bhg0bxuLFizEYDLzwwgv06tWryNSuOHQ6HZMnT+aNN95Aq9XSo0cPkpKSOH78OGPHjqVZs2bExcWxZs0aOnfuzC+//MIPP/xgM0ZISAjnz5/n0KFDNGzYEE9PT1xcXIq9Z+/evWnSpAkjR45k/vz5pKen89ZbbwG3r2/QrFkzvvvuO3bv3o2vry+LFi0iMTHRqtA9PDwYNXoMi+ZMx8fXF0XLUN588y2USiUCMJgEGlUlZ94p9apeLaKmbrY/deqUuOeee4Srq6sAxMmTJ63ZWHx8fMTzzz8vpkyZYrNoPXLkSDFw4EAxY8YMUbduXeHh4SGeeeYZkZOTU6p7pqeni6efflq4ubkJf39/MX/+/CLOjqtXr4r+/fsLFxcX0ahRI7F69WrRuHHjIk6GpUuXit69ewsXFxcREhIi1q5daz1vWRiPiYmxtm3dulUA4ubNm9a2lStXCm9v71K/Z5988okIDw8XGo1GBAYGipdeesl67uLFi2LAgAHC3d1deHp6iieeeEIkJCRYz9tzAHzwwQeicePG1udGo1HMmTNHNG7cWGg0GtGoUSPx7rvvWs9PmjTJ+r4PGTJEfPDBBzby5+TkiMcff1z4+PgIQKxcufK2ryk2Nlb06NFDaLVa0aJFC/G///1PAGLz5s1CCPvvpRBmB9HAgQOFh4eHqF+/vnjrrbfEiBEjxMCBA6194hOTRf9/DhaurgWfd+duPcWwsc+J9Gz7TrGKdDIohHBimHEVJS0tDW9vb1JTU4ss6Obk5HD+/HlCQ0MrLOxEUhSFQsEPP/xQYbGMtZldu3bRs2dPzpw5Q5MmTe5orITUHK6l51DHXUtDX7Pz4cL1TNJy8mjg40pdj6LWZXl+UyX9Rgsjp6gSSS3jhx9+sC5RnDlzhldeeYUePXrcsXKDgk32ukIefheNEnJsM4xUFtLJICk3cXFxeHh4FHvExcU5W8QSKUn2P//809nilYuvv/662NfUunVrANLT03nxxRdp0aIFo0aNonPnzvz0008Oub9li5ZOXaBaXNRmZVd4A35lIS04SbkJCgoqEod263lHURErKSXJ3qBBA4ffrzIYMGCATdxhYSwhKyNGjGDEiBEOv7fRJNDnW2mFLTiLsnNGXjip4CTlRq1W07RpU2eLUW6qs+zF4enpaRNYXJnk5ltvaqUStaqQBacxP84zmjCYTKgdXXehBOQUVSKROISC9TdbtaJSKtHkK7zcvMq14qSCk0gkDsGSA05nZwuhi5OmqVLBSSQSh2DPg2rB0lbZjgap4CQSiUMosOCKqhWLBVfZoSJSwUkkkjvG4kCAgrCQwrhIC05Slbm1CEzhDLASiUVxadVKVMqi+00toSJ6owljJab3lWEiknJx9epVfH19nS2GpIpgnZ7asd4A1ColaqUSg8mE3mDE1V56sQqIdZQWnKRcBAQElJi1ojIQQmAwGJwqg8SMdQdDCUlYLfFwxa7DbXgVomaBPsthckkF5wiEAH1m5R9l/I+Xnp7OsGHDcHd3JzAwkA8++KDcpQTtFSn5/vvveeCBB3Bzc6Ndu3bs2bPH5pqdO3daC8IEBwfz8ssvk5mZaT3/5Zdf0qlTJzw9PQkICOCpp56yyb67bds2FAoFmzZtomPHjri4uLBz584yyy5xPCU5GCxYQ0XsrcPps+DiLtizFDJKV/S9NMgpqiPIy4J3HbctqdRMuwJa99v3y2fChAns2rWLn3/+GX9/f2bMmFGkAMyd8Oabb7Jw4UKaNWvGm2++ydChQzlz5gxqtZqzZ8/St29f5syZw4oVK0hKSmL8+PGMHz+elStXAubSdLNnzyY8PJxr164xYcIERo0axcaNG23uM2XKFBYuXEhYWJicJlcBhBAlhohYsJzLuTXY12SEnJvmxz0nQJ0wh8kmFVwtIT09nS+++IJvvvnGWgNg5cqVDt0vOnHiRPr37w+Yywy2bt2aM2fO0KJFC+bOncuwYcOs1mKzZs348MMP6dWrF8uWLUOn0zFmzBjrWGFhYXz44Yd07tyZjIwMPDw8rOdmzZpF7969HSa35M7QG02YhEChUFitNHsUG+ybmWRWct7B0OMVh8omFZwj0LiZrSln3LeUnDt3jry8PJsCI97e3jYFYO6UwgVfAgMDAXPxlhYtWnD48GGOHDliU/NUCIHJZOL8+fO0bNmS6OhoZs6cyeHDh7l586a1dkFcXJxNGvDSZsyVVA6W7VcuamWJWYEtDgi9wawQlQqFefaTfcPc4b43QOPYHItSwTkChaJMU8Wair2CLxYllZGRwbPPPsvLL79c5LpGjRqRmZlJnz596NOnD19//TX16tUjLi6OPn36FClqcmsxFIlzsUxPb1flTa1SoFIoMApz1hGdWgkpl8wnNW7QuLPDZZMKrpYQFhaGRqPhwIEDNGrUCIDU1FROnz7NfffdV+H3v/vuuzlx4kSxGTyOHj1KcnIy8+bNsxaz+euvvypcLsmdY1FwLiU4GMD8T89FoyJLbyAnz4hOnwJ5mYASXH0qRDbpRa0leHp6MnLkSCZNmsTWrVuthU2UypKnFY5i8uTJ7N69m/Hjx3Po0CH+/vtvfvrpJ8aPHw+YrTitVstHH33EuXPn+Pnnn5k9e3aFyyW5cyxhH8XFwBXGsg6Xl5cHaZfNje5+oKwYW0squFrEokWL6NatG4888ggRERH06NGDli1bVkptibvuuovt27dz+vRp7r33Xjp06MCMGTOsTo569eqxatUq1q9fT6tWrZg3bx4LFy6scLkkd4ZJCOsaXEkeVAuWMBLXnEQQRlDrwK1Ohckni87YobYUncnMzKRBgwa8//77jB071tniSKoh2Xojf19LR6VU0CrQ67azgbTsPK4l36CpMt8pV7cZOUIti85I7pyYmBhOnjxJly5dSE1NZdasWQAMHDjQyZJJqiu51hoMqlItdbiolTRQXAdAuNZB4eIBOTkVJp9UcLWMhQsXcurUKbRaLR07duTPP/8kNjaWfv36FXtNRkZGJUooqU5kF5PFtzi0uckoFHoMQonRLYCK3uwnFVwtokOHDkRHRxdpz87OLrEAi0RSHGVZf8OYhyLdvA0rkTp4mpRSwUkqHldX1xpZgEVS8ZRmi5aVtMsgjOQqdNwweaIxGPFCc/vr7gDpRS0n0jcjqe0YTSb0xoJdDCWSmw7Z5v2mmbpABAXWX0X+lqSCKyOWaP2sLMeldJFIqiOWTfMalW2ZwCIIE6TGmx+7+aFyMW8xtDgoLL+lwjthHIWcopYRlUqFj4+PNY2Pm5tbpQTKSiRVjbRMPcKgR6NSk1OSJzQzyewpVahB6wvGPIRBT5YBrl/PJSkpCR8fH1SqUkxzy4hUcOUgICAAwCZXmURS20jJ0pORa8RTpyYvpRjry2SA9Kvm3IVudSEjHiEESSk5CAQKLx1+detYf1OORiq4cqBQKAgMDKR+HR/yoldDkwjwbuBssSSSSuW1tTEcuZTK5L4taB9ajIL6ZSKc3wZBd8Nj/zEnpgBmr9zPheQs5j3RgbaB/hUmo1Rwd4Bq8yRUMV9C+MMw9FtniyORVBpCCPZcTCcly0hYgK/9HQinNsPRL837TCOmgqur9ZS3hztXL6ZzJimLXo7L2FUE6WS4A4z3vIhRoYJTG+HUJmeLI5FUGknpuaRk5aFUQNP6HkU76LNg0yTz43tegPotbU438zdfc+ZaeoXKKRXcHRC5x8DyvIcBMG18w6HFMiSSqszJBLNiCvFztx8Dt3MRpMSBVwPoNbnIaYtSPHOtYnfJSAV3Bwzr2pgV6ie4LOqiTI3DuENmv5DUDk7lK7gWAZ5FT14/A7uWmB/3nQcuRS08i4L7+1qGjIOrqrQM9OLjkffyrmkkAGLnEkzXTjtZKomk4rFYcOH+t2TyEAI2vg5GPTSNgJaP2r2+ST0PFApIycojOVNvt48jcLqCW7p0KSEhIeh0Orp27cr+/fuL7Xv8+HEef/xxQkJCUCgUNpXWLcycOROFQmFztGjRosLk7xJah8eefJatpvaoMXDxy+cRpmLqPkokNYTTifkKLuAW6+z4D3BuG6hcoN98q9f0VnQaFcG+5oDfvxMrbprqVAW3du1aJkyYQGRkJAcPHqRdu3b06dOn2PiyrKwswsLCmDdvXolxM61bt+bq1avWo6JrZ0a0DiAnYh45QkNo+l9s+e6TCr2fROJMjCZRSMEVsuBy0mDzVPPjeydA3SYljmNdh0uqoQpu0aJFPPPMM4wePZpWrVrxySef4ObmxooVK+z279y5MwsWLODJJ58ssaq6Wq0mICDAevj5+VXUS7DS775uHAszJ41sd/w94q46rnitRFKVuJicSa7BhE6jpFGdQpXdts0zF232DYUer952nGYWBZdYcZ5Upyk4vV5PdHQ0ERERBcIolURERBSpiF5W/v77b4KCgggLC2PYsGHExcWV2D83N5e0tDSbozx0euptEtRB+CtSuPT99HKNIZFUdSzWW3N/T1TK/ClowjHYlz9zeXhhqcr/NanJFtz169cxGo34+9tGMfv7+5OQUH7rp2vXrqxatYrNmzezbNkyzp8/z7333kt6evH/JebOnYu3t7f1sFR1KjMaHcn3vWuW49p6Us8fLN84EkklcOJKGr8eTyizF9PiYGjun+9BNZnglwnmGgstB0CziBKuLsBiwdXYNbiKoF+/fjzxxBPcdddd9OnTh40bN5KSksK6deuKvWbq1KmkpqZaj/j4+HLfv9W9g/hT2xOVQpD5/SvmD18iqYKM//Ygz34Zzfu/lc3zXyRE5PA3EL8PNO7Qd26px7FYcNfSc0nNziuTDKXljhRcRkZGuad2fn5+qFQqEhMTbdoTExMduvHWx8eH5s2bc+bMmWL7uLi44OXlZXOUF4VCQdYDs8gULgSlHyEv+styjyWRVCRJabkAfLz1DEu3Fv/7uJVTVgeDJ2TdgC0zzCfunwLeDUs9jpdOQ4CXeSpbUQG/ZVZw58+fp3///ri7u+Pt7Y2vry++vr74+Pjg6+tb6nEsNQGioqKsbSaTiaioKLp161ZWsYolIyODs2fPEhgY6LAxb8dDXTqwQjMUAONvM8xfAomkipGTn48NYMGvp/h85/nbX5Nn5ML1TADC/T0hahZkJUO9lnDP82WWweJJPVtBCq7Mm+2ffvpphBCsWLECf3//O8qFNmHCBEaOHEmnTp3o0qULixcvJjMzk9GjRwMwYsQIGjRowNy5ZrNXr9dz4sQJ6+PLly9z6NAhPDw8rCm3J06cyKOPPkrjxo25cuUKkZGRqFQqhg4dWm45y4papcTjvhc5+XsULfLiMf0+E+WADyvt/hLJ7TCaBHlG89rbqO4hrNp9gdkbTqDTKBnWtXGx1525loFJgK+bhnqpRyF6lflE//dBVfaElU3re7DzzHX+rqA9qWVWcIcPHyY6Oprw8DtPATBkyBCSkpKYMWMGCQkJtG/fns2bN1sdD3FxcSiVBUbmlStX6NChg/X5woULWbhwIb169WLbtm0AXLp0iaFDh5KcnEy9evXo2bMne/fupV69encsb1kY3DWMF/54hi+YgeLgaugwHII7V6oMEklx5Bay3t7oG46LRsl/tp/j7Z9PMKBdEJ46+8rK4mBo4e+G4pcJgIB2QyGkR7nkqOg9qWVWcJ07dyY+Pt4hCg5g/PjxjB8/3u45i9KyEBIScluPz5o1axwi153i7qKm9T19+G7X7/xLtcPsZXpmK6hkhiqJ87GkGwdzTdMpfVvwxe4L5OSZSMnKK1bBWUJEnlb9DpeOgM4bes8qtxzNCu1JrQjK/Gv77LPPeO6557h8+TJt2rQpkkf9rrvucphw1Z1R3UMY8Ocweiv/wjvhCPz1OXR91tliSSTWalhalRJlfiybi1pFTp6JXEPxnv+TCenUI4XeCcvNDQ/NAI/65ZbDYsFduplNlt6Am9axBkCZR0tKSuLs2bPWdTIwew6FECgUCoxGYwlX1y7qe+no1aEV82Oe5B3NCvhjDrQaCJ4Vk55ZIiktFgXnUqhgsza/Mpa+BAV3KiGNqZpv0BoyIKgDdBxdbN/SUNfDhTruWm5k6jmXlEmbBt53NN6tlNmLOmbMGDp06MCePXs4d+4c58+ft/krseWZ+0L51vggh01hkJsGv8kdDhLnk2OnYLOl9F/h9bnCpGTpCc2I4Z+qnQgUZseC8s4LxTStZ5mmOt7RUGYL7uLFi/z888+yUHApaVrfkwdaBPDmqTH87DId5dF1cPdwCL3P2aJJajGWEBFdGSy405eTma1eCYCi0xho0NEhsjT192D/hRsV4mgoswX34IMPcvjwYYcLUpMZd18Yx0QY35p6mxt+eR0MFZcDSyK5HdYpqrrAAtPm1za1FHO+FdX+ZTRTXiZN6QMPOW4mYrXgKmDLVpktuEcffZTXXnuNo0eP0rZt2yJOhgEDBjhMuJpCl9A6tAv24b34fzHI8y/cr5+GPR+bU8pIJE4g1zpFLbBxXPKnq7l5dhRcSjxtz/4HgD9DX6G/a+mD+m+HtT5DBWy6L7OCe+655wCYNauoa1g6GeyjUCgYd28YL36TwruGp3iHj2H7fGj7L/Bp5GzxJLUQiwWnK2TBuZRkwW2egtaUwz5TCwxtBjtUFosn9WJyFnqDyTpVdgRlHslkMhV7SOVWPH3bBNCojhtfZ3fjktfdYMguSA4okVQyBWtwhRScphgnw+lf4eQGDCiZnjea8MDy79W2R4CXDg8XNUaT4EJypkPHLpOCy8vLQ61Wc+zYMYcKURtQKRVM7BMOKBh3YyhCqYaTG8y1IyWSSibHzhTVugZX2MmQlw0bzeX/Pjf045yiEWF+dsoE3gEKhcKaWcTR63BlUnAajYZGjRpJS62cPHpXID2b+nHC0ICN7o+ZGzdNkuUGJZVOQRxcISeDNUykkILb+QGkXCTH1Z8lhsdpUs/DoVNIC80qaMtWmSV98803mTZtGjduyAwZZUWhUDB7UBu0aiWTkvqR7Rpgrh25c5GzRZPUMqwWnLpoHJzVgks+a1ZwwPbQ18lCR3N7ZQIdQEEZQcfGwpVZwX388cfs2LGDoKAgwsPDufvuu20OScmE+rnzfK8mZKEjUj/c3LhribmWpERSSVidDHbi4HINpvzyfxPN5f+aPMQmozlRhN06qA6goiy4MntRBw0a5FABaiPP39+Enw5dZl1ye56pfw/N0vaaa0kO/7HYMmsSiSOx62TIt+ZyDSY48SOc/cNc/u/hBZz66gqQnwOuArBYcOeuZ2I0iYJaD3dImRVcZGSkQ25cm9FpVMwe1Ibhn+9nXNJgotxiUJ7bZq4p2eafzhZPUguwFwdnXVvLTYfN08yPe75Gnk8oZ/MLmodXkAXX0NeN5v4eNKrjTkaOAW+3sueWs0e5t+5HR0cTGxsLmOuQFs7TJrk99zarx6PtgvjfYVjj8i+eyvoafp0GzXqDS8V8iSQSC3bj4PIV3D3xyyH9CviGQM9XuXA9E73RhLtWRQMf1wqRR6VU8NtrvRw+bpnX4K5du8aDDz5I586defnll3n55Zfp2LEjDz30EElJSQ4XsCYzvX9LPF3UvH2jN2luwZB+1VxbUiKpYCyeUt0tXtRwRRxdE/MLND28EDSu1hoMzQM8ramVqgtlVnAvvfQS6enpHD9+nBs3bnDjxg2OHTtGWloaL7/8ckXIWGOp76VjYp9wctHyRla+w2HvMnONSYmkDCSk5nDgQukjG+w6GVQwR7MCFUZo+ah5NkFBFa2KWn+rSMqs4DZv3sy///1vWrZsaW1r1aoVS5cuZdOmTQ4Vrjbw9D2NadvAm805bTjkeZ+5tuQvE2S5QUmZePGbgzzxyR7+LmWVeHtxcG2SNtJZeZpchQ76FswkLGnKK2r9rSIp11atWzfYgzkI2CR/lGVGpVTwzmNtUCjg+aQnMKrdzDUmD3/jbNEk1QSTSXDscipg3s9ZGorkg8u6QZcziwHY4DvCpvzfqdqk4B588EFeeeUVrly5Ym27fPkyr732Gg899JBDhast3NXQhxH3NOYqdVmuzN/IvEWWG5SUjqtpOdY1tdIWULaEiVgcC/wxG53+JqdNDdjk8Zi1X5beQNwNs9KsFVPUjz/+mLS0NEJCQmjSpAlNmjQhNDSUtLQ0Pvroo4qQsVbwep9w6nm6sDDtQZLdwsy1JqPKX8xDUnuw1CkFSMsppYIrbMFdioa/zIksp+eNIdtUoBZO5+8N9fNwoa6Hi6NErjTKHCYSHBzMwYMH+f333zl58iQALVu2JCIiwuHC1Sa8dBqmP9KKl7+N4aW04Xyjfttcc7LD09Cwk7PFk1RhzhVScKW14HItTgalgF9eAwSXGw1k3+mWdCqUD+5UQhpQcTsYKppyxcEpFAp69+5N7969HS1PrebRuwJZdyCenWfgT68I7s36HTa8BuO2OST3vaRmcqEcCs7iZGhwdg1cPQwu3py7ezKcvmCTD+5UgtmCq47rb1BOBRcVFUVUVBTXrl0r4lhYsWKFQwSrjVg24/dZvIPXbjzObs/9aBOOwIHPoes4Z4snqaKUS8EZTPiRSmD0AnPDQ9NRuPsDF2wy+p5KNFtw1XH9DcqxBvf222/zj3/8g6ioKK5fv87NmzdtDsmdYdmMfx1vPjAOMTf+MQfSE50rmKTKcr5Qksi0MlhwUzVfo9KnQWA76DTGmvDS1oKrvh5UKIcF98knn7Bq1SqGDx9eEfJIKNiM/5/kXjxd508aZJ2ELdPhn586WzRJFcNgNBF/oyA0JC3bcNtrhBDcZTjG41pz+T9F/w9AqSqS8PJ6Ri7XM/QoFNC8tlhwer2e7t27V4Qsknwsm/FNKHkxZZi5BuWRtXD+T2eLJqliXEnJIc8orM9LM0XN0+uZpTYvJenbj4SG5vJ/t6YsP51vvTWu44artnquAZdZwf3f//0f33wjg1ArGstm/EOmJmzS9TM3ynKDkluwTE8tW0RLo+BMe5fSXHmZZOGJeLCg/J/FgrPE1Fl2MFRX6w3KMUXNycnh008/5ffff+euu+4qsqth0SKZndZRTO/fkm0nrzElZRAPeu1Bd/0U7P039HzV2aJJqggWB0Nzf09OJqTfXsGlXsLlz/kAzDU8xQLPutZTt6Yst6y/VdcQESiHBXfkyBHat2+PUqnk2LFjxMTEWI9Dhw5VgIi1F8tm/DQ8mKV/yty4/T1IiXeuYBKnERWbyOvrDpOZa15rO5+v4No19AEgO89YbGV6ADZPQWHIZr8pnF+UvVAUSrBqSXipN5gQQliziIQHOLaKVmVSZgtu69atFSGHpBievqcx30Vf4pvL3Rnj+ydNs4/A5inw5NfOFk3iBMZ+8RcAwXVceTWiuVXBtW3ozdq/zP/40nLy8LO36+DvLRD7P4RCxfS80bi42s6+CheTyTWYOG1VcI6tolWZOL48jsShFGzGV/BCyjCEQmUuN3j6N2eLJnEi1zNyAax1RJvU88BTZ7ZX7E5T87LNNRaA663HcEo0skl2CYX2pQLnkjLJ0hvRqpWE1HWviJdQKUgFVw2wbMY/LYJZp37U3LhpkvlLewsnE9L4dMdZ8uxVJ5fUGLQqFXlGE5dumr8DoX7ueOdbZHYV3M7FcPMCeAYS386ct7FwLjjzmAXPj15OAaBpPQ/UquqrJqqv5LUMy2b8t9MHkKGtb/6y5pd0s5CYlsNTy/fx7saT7DgtsyvXZDRqBfE3sjCaBK4aFf5eLsUruELl/+g7lyzcANtsvgBKpcKq5I5cMqdfqq4BvhakgqsmWDbjZ6Fjana+w2HnB+YvL2A0CV5be4gbmeYwEstfSc1Eq1Jap6eN67qhUCjw0pkVnM1uBiHMlemNudDkQWg1qCDZpZ0CzpZ1uFqr4Hbs2IHBUDRa2mAwsGPHDocIJbHPo3cF0rOpH//L68wRXSdzzcqNE0EIPtl+lt1nk619s/O/xJKawZ9/JxETV7AVUqtScv66eQdDqJ95jcxiwdkouNif4WwUqLTmGgsKRUEuOE3R4F2LgjuZn0Wk1im4Bx54wG5V+9TUVB544AGHCCWxj2Uzvlat4qXUpzAqtXD2D87u+JpFW8xl3Szes2y9VHA1hcS0HIZ/vp/H/r3b2qZRK60xcCG3KDjrFDU3HTZNMT/u8SrUbQLYyeZbCItVZ9kdUZ1j4KAcCk4IYRM7YyE5ORl39+rrbakuWDbjXxQBrGAQAJ5bZ6AzZTGwfRB9WvsD0oKrSSSk5hRpKzxFDc33clpqiVoV3Pb3zOX/fBrDvROs1xaUDCx+igrgqVMT4KVzzItwEqWOg/vnP80FiRUKBaNGjcLFpSDOxmg0cuTIEblHtZKwbMZfmPwwfXXbCSaR6R4/03/Q53wY9TcgFVxNQthp06iV1hi40HpmBedVOEwk8QTs+be5c375PwsFFbWKt+DAbL3ZM2aqE6W24Ly9vfH29kYIgaenp/W5t7c3AQEBjBs3jq+++qoiZZXkY9mMn4uW6fqRAAwxbsAz9TSu+V/aHDlFrTEIYUfFCcHlFHOIiCVOzTPfyZCRk2fetyyM0OIRaP4Pm0sLaqKWbMFV9/U3KIMFt3KlOWd7SEgIEydOlNNRJ3Nvs3r8s0MDvo+B8/UeJDTpD/jldXSh5roY0oKr2cTdyEII8HBR4+ehBUCTH+LRKfU3SNoNGjeb8n8WSrbgCtqqa5LLwpR5DS4yMtKhym3p0qWEhISg0+no2rUr+/fvL7bv8ePHefzxxwkJCUGhULB48eI7HrM6896/7iLq9V6EPv0RaNwhbg9tr28EIDtPBvrWFOxNUc8lWRwMbtZppFqlwIsM/pX8H3OnXm+AT3CRa0tScIWDfavzHlQLZVZwiYmJDB8+nKCgINRqNSqVyuYoC2vXrmXChAlERkZy8OBB2rVrR58+fbh27Zrd/llZWYSFhTFv3jwCAgIcMmZ1RqNS0qSeh7mG5f2TAej892K8ySjiRf3jZCJPfrrHJjmipOpyPSPX+lnZm6FaCs0U3kalUSmYpF6HlykF/MLhnhftjm31ot7GyVATLLgyb7YfNWoUcXFxTJ8+ncDAwDtahFy0aBHPPPMMo0ePBszZgn/55RdWrFjBlClTivTv3LkznTt3BrB7vjxj1hjueQEOfYMu6SST1GvZlPeG9VR6Th4T1x/hRqae304kMrZnqBMFlZSGTnN+ByBmem/s2XBWB4NfgYLzSz3OQFWU+Un/90GttTu2var2FixOhkBvndUrW50ps4LbuXMnf/75J+3bt7+jG+v1eqKjo5k6daq1TalUEhERwZ49eyp1zNzcXHJzc63P09LSynV/p6LSmL/Uq/rzlOoPjmQ+CtwDwH+2n7PubCgxlY6kSlB4H/HZpIwS+1otOJORuw7PQqkQbNc9QK/Qe4u9JsdQfBycxYKrzkkuC1PmKWpwcLB9r04ZuX79OkajEX9/f5t2f39/EhISKnXMuXPn2niFg4OLrltUC0J6khgyCKVC8H9pH4PJSGJaDp/tPGftIjfhV33Scwp2Ct1uhmQJ8iV6Jd43j5EmXFnhNqbEawrW4Ir+/C1Kr7oH+Foos4JbvHgxU6ZM4cKFCxUgjnOYOnUqqamp1iM+vvomlLzSdRppwo3mxjPw1wo+2HLauuYC0oKrDhTeapWTZ7TrZLAQ6ucOGdfg91kALDQM5prwLXH8gkDfohbcYx0a0KmxL493bFh2wasgZZ6iDhkyhKysLJo0aYKbm1uRlOX2tnHZw8/PD5VKRWKibTm8xMTEYh0IFTWmi4uLTeBydSYgqBHvGwbztmYV+t/e5o/M+YA3HRv7En3xprTgqgFpOQUKLj3HQF0P+2tp3q4afN008OMMyE0lo05rvrrSmya3+YxzS9iq1aOpHz2a+t2B9FWLMiu44kIzyopWq6Vjx45ERUUxaNAgAEwmE1FRUYwfP77KjFndCPR2RXQaw9GYbbQ1XGCy+mu2NJ9FWD13oi/etAZ5SqouhUv/ZeYaqONuX8GF+LmjuLgbDn8LKIjrNhvTf3MwmEpeQrJstrc3Ra1plFnBjRw50mE3nzBhAiNHjqRTp0506dKFxYsXk5mZafWAjhgxggYNGjB37lzA7EQ4ceKE9fHly5c5dOgQHh4eNG3atFRj1gZe+0dLXjoyjtWmN3lctZN72t5g/XXzmoq04Ko+hfO5ZeqLr3PapI4WfnnJ/KTjSHL97wZ23/YztlhwLnamqDWNMis4gLNnz7Jy5UrOnj3LkiVLqF+/Pps2baJRo0a0bt261OMMGTKEpKQkZsyYQUJCAu3bt2fz5s1WJ0FcXBxKZcF/mStXrtChQwfr84ULF7Jw4UJ69erFtm3bSjVmbcDXXcvA/gP45sconlZH0WDXW7i0WAVIBVcduHWKWpxTb2Duz5AUC2514aFINDctmUBK/oylBVcC27dvp1+/fvTo0YMdO3bwzjvvUL9+fQ4fPsznn3/Od999V6bxxo8fX+z00aK0LISEhJTKg1vSmLWFJzoFc63RvxGreqBIOsndXmuALtLJUA0o7GTIzDXYdTIEkEz3+OXmJ71ngVsdNGnmIjEG422mqCXsZKhplFmFT5kyhTlz5rBlyxa02oK1gQcffJC9e/c6VDjJnVG/fgCK3mbv2t3nPyGQZJsq6JKqSWELLquYpAnTNV+iMWZD8D3QzpzhWa0yh5Tc1oLLK36zfU2jzK/w6NGjPPbYY0Xa69evz/Xr1x0ilMSBtBsKjbqhMeUwXfMlejlFrfIUdjKYhCiyVauX8jD9VfsxKVTm4O78ZRxN/t/bOhmsKculBVcEHx8frl69WqQ9JiaGBg0aOEQoiQNRKqH/+5gUKh5W7adFurSyqzqFLTjjLcrKBT1vq1cBcC5sGAS0sZ4rjQUnhCiULkkquCI8+eSTTJ48mYSEBBQKBSaTiV27djFx4kRGjBhRETJK7hT/1pxrMhyAp298bLfcoKTqUHgNziQEotAq3HOq/xGiTCRB+HK21cs211nSJeUZRbFr1YXDhOQU1Q7vvvsuLVq0IDg4mIyMDFq1asV9991H9+7deeuttypCRokDONdqPAnCF3/jVXONTEmVpXCYyKmEdH49Zt5m2EiRyAvqnwGYnTcclattOiONqmBb162Wn4WcQnkCpQVnB61Wy/Llyzl79iwbNmzgq6++4uTJk3z55ZdlTpckqTw0bt7MyjNbcYXLDUqqHmmF9qIejEvhiz0XAcEs9SpcFHn8aWzDL6auuNxigRUu0FzcOpzFwaBSKqwWX02mXHFwAI0aNaJRo0aOlEVSgXi7adho6so+ZXu6Gg+Za2U+/V+o5jn3ayJpdirT91Ee4H7VYXKFmhmG0YCiiJNArSz4LPVGk10LraSCMzWRUim4CRMmMHv2bNzd3ZkwYUKJfRctWuQQwSSOxcdVAyiYZRzNL+pJ5lqZsT9Dq4HOFk1yC4WdDABu5BCpWQ3Af4yPcF4EArbJKQEbi6y4WLikDHNaMFdtuW2bakWpXmVMTAx5eXnWx8VR3Svw1GQsNTOP59bDdN8rKP9cYK6Z2eRBcKkZqXFqArkGo032F4CX1d8TpLhBnKkeSw2DrO23VqZXKRUoFWASYCjGk/rzoSsA9Gha17GCV1FKpeC2bt1q97Gk+mBRcAApHV+iztF1kHLRXDvzH3OcKJmkMIVzwQE0U1xirGoTADMNI8mlILj+VgUH5nU4vcFEnp01uGy9kZ8PmxXcv2pIOqTbUTsm4hLUKiWeLub/Zyl5KnOtTIC9y8w1NCVVglSb9TfBHM0KNAojvxk78ofpbpu+9lKOa/LX4fJu2ZJ3PSOX57+OJjU7j0BvHd2b1JyUSCVRKgvOUvS5NHz//fflFkZSsXi7aUjPNZCSnWeuldniETi5wVxDc/RG6XCoAhR2MDym3ElX5UmyhZa384rGmNqz4DRqJeiNGEwFCu73E4lM+f4I1zP0aFVK5j1+Fypl7fisS2XBFU7n7eXlRVRUFH/99Zf1fHR0NFFRUXh7e1eYoJI7xye/iEhqVv6PqO88c+3MuN1weI0TJZNYsISIeJHJNM3XAHxo+CeXqVek761OBgC1siDYNzPXwNTvj/B/q//ieoaeFgGe/DS+B72aFx2rplIqC85S9Blg8uTJDB48mE8++cQa92Y0GnnhhRfw8qr+dRRrMj6u5vWblGx9fkOwuXbm7zPht7cgvC+4mtNdm0yCK6nZNPR1c5K0tROLBfe6eh31FGmcMQXxmfFhu33tWnD5wb77z9/gua+iuZichUIBz9wbxoTezWtFcG9hyrwGt2LFCiZOnGgT1KtSqZgwYQIrVqxwqHASx2IpA5eSVWid554XzTU0s67DHwXOhs92nqPne1uZ/uMxhxQZkpSOtJw82ijOMVxlLhs43TCavGLsEK2dQF3LftTIn49zMTmLIG8d3/zfPUx7uGWtU25QDgVnMBg4efJkkfaTJ09iMslMFVUZH1c7Ck6tNWekADjwOVw+CMCes8kAfLn3Iu9ujJVKrgJISs8lKT2XjNwCz2l6Vi5zNCtQKgQ/Gruzx1R8All7YVmFY+H+2aEBm169j25NakdIiD3KHO03evRoxo4dy9mzZ+nSpQsA+/btY968ebUqLXh1xBIqknprpHzovXDXEDiyFn6ZAP8XxZlC9TiX/3me+p46nrkvrDLFrVGcS8pg49GrjO4RiruLmrScPDq/87v1fOysvrhqVYRc/I72ynNk4MY7ecPKfJ+Ilv78T3+Ft/q3ov9dgY58CdWSMiu4hQsXEhAQwPvvv29NmxQYGMikSZN4/fXXHS6gxHFYnQx2tgLRezac2gRXYtDvW8Glm0EAjLsvjE93nOPb/XFSwd0BH/9xhu9jLuPrrmVY18b8nZhuc/7va+nc5ZNHr/ilAHzlNpyknJLL/9lj2sMtmfZwS4fIXBMo8xRVqVTyxhtvcPnyZVJSUkhJSeHy5cu88cYbcrN9FcfqZMjSFz3p6Q8PTgdAtXUWdUQqvm4anu7aGIBLKdmYbpNIUVI8lsy855Iy81tsp5fZeiP8HomrMYPjpsbs9h1UuQLWUO4o0NfLy0t6TqsRVieDPQsOoPNYCLgLlT6NqZpvaVrfg0AfHUqFuWD09fx9jJLyE3cjCzDXWijMwuWr4JA5LOStvDF4uesqW7QaSbl23H733XesW7eOuLg49Hpba+DgwYMOEUzieCxOhtSsYhScUgWPfID4LIJ/qXaQ7PYEGlV3Ar1duZySTfzNbOp7yR9eWTCahE1QbXy+givsWFBjYLbGHIr1m64vMTnNaONmvxaqpGyU2YL78MMPGT16NP7+/sTExNClSxfq1q3LuXPn6NevX0XIKHEQPm6WOLhiFBxAw0786dUfgMGJi8GYRwNfVwB2n7nO6j0Xit3ILbEl+uINWkduZvWeC9asvPE3shBCkF4oY8go1a+0UMZzQ3jwb7XZseBbTLFnSdkos4L797//zaeffspHH32EVqvljTfeYMuWLbz88sukpqZWhIwSB+FjjYPTl6ikFpueJFl44pt5FvYuo2G+gnt/y2lm/HSchb+drhR5qzuvrj1ETp6JGT8dt7Zl6o3czMqzrskFkMyr6v8CMM8wlEs55vfa101TdEBJmSmzgouLi6N79+4AuLq6kp5u9gYNHz6cb7/91rHSSRyKt6sGlVKBScCgf+/i6KWi/5DyjCaO3FAxzzDU3LBtHi3cbD1+3+y7WBniVnuKCwuNu5FlrY3wluYrPBQ5RJuasd7Yy1pRq4604BxCmRVcQEAAN27cAMxZfS21UM+fPy+DQas4Oo2K959oh5dOzbHLafxz2S52n7Et9XgxORODSbBJ9QAi+B7Iy6Tf5Q9t+qTnFl9tXXJ74m9kkZNn5D7lYR5R7cMoFLyVNwaB0lrW0UeuwTmEMiu4Bx98kJ9/Nhe+GD16NK+99hq9e/dmyJAhduulSqoWgzo0IOr1+3moRX3yjILnvormzLWCoF7L47D6Xij6vw8KFcFXf6OX8rC1jxBwtlAgsMQ+xRV+ibuRhUGfbS3/t8rYl1jR2HpeqQBPXcn+v5aBMnqhNJRZwX366ae8+eabALz44ousWLGCli1bMmvWLJYtW+ZwASWOp56nC0uH3U3Hxr6k5RgYs+oAyfkhIH8nmhVX0/oe5pqbXZ8DYLZ2FW39XejY2Bx8uufcDecIX40wFbJyCxu8l25m0fHSl4QqE0kUPnxgeNzmOi9XjbWIsz1Wju7MmnH3OFzemkiZFJzBYGDOnDkkJCRY25588kk+/PBDXnrpJbRaaVZXF3QaFZ8O70ijOm7E3chi3JfR5OQZrVu0mtb3MHe8fwp4BtKIRL6/64A11c7ec8nOEr3aUNwkPufaGXpe/QKAOXlPk4FtxhYvnYYS9BsPhNe3ydAsKZ4yKTi1Ws38+fMxGAy37yyp8tT1cGHFqM546tREX7zJ5P8eKbDg6uUrOJ0X9HkXAM3uD+jlZz6/79wNuQ53G+y/P4LB1z5GI/TsNLbmf6ZuRXp4uaprTULKiqbMU9SHHnqI7du3V4QsEifQtL4H/3m6I2qlgp8OXeHE1TQAmvkXKkTT+jEIewCMubQ9MhuNypwC+9LNbCdJXT2wp9/6KP+imykaAwXl/27FS6dBJbMrO4Qy72To168fU6ZM4ejRo3Ts2BF3d3eb8wMGDHCYcJLKoXtTP+4Pr8/vsYmAOc9YcH7sG2BOZf7wQljWDeXZKEb4dOXz5DbEXk0juI5MiFkchdfgfjuRiCs5zMgv/7fe5Z+cywmye523qwZlMRZc47ry/S4LZVZwL7zwAmC//qlCocBoNN65VJJK56GWBQou1M/dpko6AH5NoccrsGMB43M/41vmEXs1nX+0DnCCtNWDW52oL6t/oIEimUvCj7mZ/Yu9zkunKWL9qZQKHr0rkDf6tqgASWsuZVZwMqllzeSB8PrWx77uxSxg3/s6HFmHb8pFXlb/wKGrIZUjXDWlsAXXVHGJ/1NtBCAybyRppuKdBF6uapuiMR8MaccjdwXZJLOUlA75jkkACPAu2ETvW1yQqcYV+s0HYKxqI9lXjgGwes8Ffj2eYP+aWoLRJNh95jrrDsQD5n2oBTVOBXM0K9EojGwxdiTK1LHEsbx0GpvK9APbNZDKrZyU2oLLzs4mKiqKRx55BICpU6eSm1uQPkelUjF79mx0Opltorry+chOLN16hqn9SkiYGN4XfdN+aM9s4oXMf3M4boB1r+WFecVPu2oyu89eZ+yqv8jOMy/PGEyCaT8ctZ4fpNzFPcpYc/k/Q9Hyf7fi5aohxK9gbbu49TjJ7Sm1gvviiy/45ZdfrAru448/pnXr1ri6mhejT548SVBQEK+99lrFSCqpcB5q6c9DLf1v20/7yHxyFkfRVXmSP/asBpoC5rAIe3UCajrPro62KjeAtX/FWx97kcmb+eX/djcYzUsdHmTyf48WGaMwXq5qPFzUHJze225pQEnpKfW79/XXXzNu3Dibtm+++YatW7eydetWFixYwLp16xwuoKQK4tOI//kMB6Dz6Q/wwpyl1lBLM/7qb8nMoi9UVX6Cej31FKkk6xrz0JjZtA66fe1gSxBvHXctHi7lStkoyafUCu7MmTO0bdvW+lyn06EsFG7dpUsXTpw44VjpJFWWi81Hc8YUhKfxJq+rzf/Ycg210wGVd4uCu5lpTgJrLv+3BYArPWaD2oVGpQjz8NLJXQqOotQKLiUlxWbNLSkpiZCQEOtzk8lkc15SswlvUJfpBnMVteGq32mjOEdunpHLKdncv2Arn/15zskSVh63Gq4JaTkoMDFHsxKVQvCTsTtNu5qXdrx0mttusyrs8JHcGaVWcA0bNuTYsWPFnj9y5AgNGzZ0iFCSqk/LQC/2mFrzo7E7SoVgjmYF+rw81uyP40JyFnN+ibVJy13beFK1lfbKs6QLV/7r9zyu2oKCTI1KCI6e/kgrGvrKYF5HUWoF9/DDDzNjxgxycnKKnMvOzubtt9+mf//a6UWrjYT6uaPTKHknbxhpwpX2ynNoDn/FqYSC5Jg/HbrsRAkrlgvXM3l4yZ/8GFP0NdYhjcnqNQAsMvyLVwb1tDlfnIL75Om7Gdsz1PHC1mJKreCmTZvGjRs3CA8PZ8GCBfz000/89NNPzJ8/n/DwcG7evMm0adPKJcTSpUsJCQlBp9PRtWtX9u/fX2L/9evX06JFC3Q6HW3btmXjxo0250eNGoVCobA5+vbtWy7ZJPZRKRWE+3uShC/vGwYD4LP7XY79fcba54eDNVfBvf2/45y4msaraw8VOTdF/S0+ikxOmBqz2vgP3LS2joKGdVyLXAO1dw2zIim1gvP392f37t20bNmSKVOm8Nhjj/HYY48xdepUWrVqxc6dO/H3v32Iwa2sXbuWCRMmEBkZycGDB2nXrh19+vTh2rVrdvvv3r2boUOHMnbsWGJiYhg0aBCDBg0qMn3u27cvV69etR4ynbrjsSRd/MoYwTFTCGp9Kq+Jr63nD8WnFCmPV1MoThl1VJxisNqcjOKtvNEYUeGmta0XXNiCq1soNfndjcpe6FlSMmUKsgkNDWXz5s0kJSWxd+9e9u7dS1JSEps3byYsrHxVzxctWsQzzzzD6NGjadWqFZ988glubm6sWLHCbv8lS5bQt29fJk2aRMuWLZk9ezZ33303H3/8sU0/FxcXAgICrIevr/zyOBqLgjOiYnqe2eHwhHoHke3SaOjrisEk2H++5iXGvJaew+6zRfPhqTAyR2P+3n5reICDojlAEQsuOH+NTatSsnfaQ8RM7822iffLxAUVQLmiCOvUqUOXLl3o0qULderUKffN9Xo90dHRREREFAikVBIREcGePXvsXrNnzx6b/gB9+vQp0n/btm3Ur1+f8PBwnn/+eZKTi0/QmJubS1pams0huT2F02bHiGZ8Y3gAgCGJi7g3zAeA0asOkJRec7zr8Tey6PJOlN1zo1S/0lIZT7bai/mGIdZ2dxdbC66ZvwdKhTmzskalxNdda7NzQeI4nBomff36dYxGY5Gprb+/v03W4MIkJCTctn/fvn1ZvXo1UVFRvPfee2zfvp1+/foVm+lk7ty5eHt7W4/g4OA7fGW1gxaBnjbP5xueJAVP3FJOM0yxqaB988nKFq3CuHf+Vrvt/tzgNfV3AOwOfZmbFCh/ndpWwQV6u/LFmC4sH9Gp4gSVADV0s/2TTz7JgAEDaNu2LYMGDWLDhg0cOHCAbdu22e0/depUUlNTrUd8fLzdfhJbvHQaggstmKfgyZYGLwLQ6tRSAjBbzWdqQYGa6fnl/w6amnK92RM25+ztJb23WT1aBcnCMRWNUxWcn58fKpWKxMREm/bExEQCAuznGQsICChTf4CwsDD8/Pw4c+aM3fMuLi54eXnZHJLS0TLA9r2q02MUNOyCMi+TtzRfARATl1L5glUAhavRF+Ze5REeUe3FKBR8qHuBPm3sJ7KUVD5OVXBarZaOHTsSFVWwpmEymYiKiqJbt6K56gG6detm0x9gy5YtxfYHuHTpEsnJyQQGBjpGcImVwutw7loVPZrVh0cWgULJI6p93Ks8AsC1tKLxk9WNL/cWLXitJc9a/m+18R+M/OejMvtHFcLpU9QJEyawfPlyvvjiC2JjY3n++efJzMxk9GizV27EiBFMnTrV2v+VV15h8+bNvP/++5w8eZKZM2fy119/MX78eAAyMjKYNGkSe/fu5cKFC0RFRTFw4ECaNm1Knz59nPIaazKFFdz94fXRaVQQ0NZabvA919W4oGd99CVniegw5m8+VaRtnGoDYcoErgkfFhmesFdiQeJEnK7ghgwZwsKFC5kxYwbt27fn0KFDbN682epIiIuL4+rVq9b+3bt355tvvuHTTz+lXbt2fPfdd/z444+0adMGMOelO3LkCAMGDKB58+aMHTuWjh078ueff+Li4uKU11iTua+5n/Xxk10KOWfunwoeAQQZrzBOtYE1B+Iw1bBsI8GKRMarfwTM5f/ScUOB1HFVCYWQtd+KkJaWhre3N6mpqXI9rpTk5BnN1lthjv0XvhtDrtDQWz+fd8cMoGczP/sDVAN6zPuDyymWSmKCzzULeUgVwy5ja4blTQMUrBrdmfua1SNsWsHumtqaCLQiKe1v1OkWnKRmUES5AbT+J4Tdj4sij5nqL/h2f9E1rOqEtRg20FsZzUOqGIRSwwzDKCx2W06eEaVSIQszVxGkgpNUHPnlBk1KDQ+qDmGK3UByRi5CCM4lZWCsZlPW7aeTAHAlh8j88n+i23jOigbWPpm55lhLvdxXWiWQCk5Ssfg1Q9njFQDeUn3BzP/uJ3TqRh58fzuvrIlxsnClp3CIyFfNdtBQcR28G6Hs9YZNvyy9ee9tRCvzGnKrQLnE4UykgpNUPPe+ToZrAxookmn193+szRuOXC3hoqpFlt5smTVRXKb9JXN8H/3eA63t/tHM/H7vPNaG2QNbs3psl0qVU2KLVHCSikfrhqq/udzg/6k20lRRvUJGrqXn0PXdKEAwW70SlTBA837Q4uEifd3zM4d46TQM7xaCn4f03DsTqeAklYJrm0c44dUTjcLIbPUqoPqsvz28ZCcAA5S76a46AWod9Jtnt+8TneQ+5qqEVHCSyqPvPLKFlm6qEwxU7gKK3/5UVfhmXxzXM3LxJIvp+VvPuG8i+IZY+7z7WFv63xXI6Tn97HuTJU5DKjhJpdGyZRs21zUXPn5L8zVeZPLr8UQmrT/M6cT021xd+aTl5FkLOFvK/501BUL3l236PdW1EUufulvWMK2CyE9EUmkoFAoee2EuccqG1FOkMkG9nonrD7M++hKjVpScpt4Z/BFrzirdWnGeEarfAMwxb2q5rlZdkApOUrmotayr/yoAw1VbaK04D8CV1KqxGX/fuWTmbowl12Dk1bWHbMr//c94D7tF29sPIqkyyLLZkkrnmEs7fjJ2Z6BqN+9oVvCY/m0UCvP/2jyjCY3Kef93h3y6F4DUbPPa4BDVNjooz5AuXJmdN1xOQ6sZ8tOSVDp5RhNz8oaRLlxprzzLk6qteLtqCJnyC83e3MT565lOkSvXUJDxec2BeOqQxhS1uVjRB4Z/cQ1ftE5UvpKyIz8tSaWTZxT55QbNmW8nq9egyCqomTFvU6xT5Jr1vxM2zyer1+CjyCTW1IgvjP8AQCMtuGqF/LQklU6e0bxP80tjb46bGuOjyLQWSga4mVn5oSNXU7P5el+c9fnditMMUW8D4M28MRgxh39IC656IT8tSaVjMJqDfAuXGxyi3kZHhTmh5P4LNyp9I/6bPxTU1VVh5J388n9rDfdby/8Bcg2umiE/LUml88x9BTV0D4rmfJtfbnCOZgUqzOtg3+yr3NRKf5wsKDQ+UvUbLZVxmHS+zDM8adNPKrjqhfy0JJXOgHZBbJ90Pw+E1wPgPcOT3BAetFTGM0r1KwDTfzpeafJsO1Wg3Opz01r+z/jgDJvyf4BTPbySsiM/LYlTaFzX3VrsOAVP3jMMBeA19Xf4c6PS5Dh+JZVRKw9Yn7+l+QpPRTY06AR3jyjSv7Us9VetkApO4jQebltQ5WydsRcpddrjocixlhusDPp/uNP6uIfyKANUezCihEcWoVGr+WhoBxYPac+PL/ZgRLfGTO/fqtJkk9w5UsFJnEbnkDp8McacL02g5HKPdzCh5FHVXnoqj3LmWnqFOhssySnBXP5vVn75v5z2oyGwHQCPtgtiUIcGtA/2YdbANni7yVTk1Qmp4CROxcOlIPtGXv02KLqOA2CWeiUPL4pi9Z4LFXbvM9cyrI+fUf1CE+VVkoQ3WT2nVNg9JZWLVHASpxLmV1DIRW8woXjgTdI1foQpExin2sD6vyouOeaBCzcBaKi4xkvqHwCYnfc0uHhX2D0llYtUcBKn4uuutT6u66EFnReeA94DYLz6RxqrzIVeHF3E5Vp6DrM3mHcuRKpXo1PksdvYip9N3ZGF6WsOcrO9xOn8+up9XEnJpkm9fGuuzeNk7Pkcjyu7+Vfih4RM0aJSKlk8pD2PtgtyyD0fW7obgAhlNL1VB8kTKqYbRgMKlAqp4WoK0oKTOJ3wAE8eaFG/oEGhQPPoIvRCxUOqGHorozGaBC9967gqXJdTsnElh5maLwA4HjLCWv5PKriag1RwkiqJS2BLvlIOBCBSsxpXHJcvzuJcGK/+kYaK6+S4BaG4b5L1vEL+KmoM8qOUVFl+rzecS8KPhorrjFf/CMBPhy7f8bjnkjJoorjMM6pfANA9uoAmDf2t5+WG+pqD/CQlVZYG9eoyM28kkB/GobjMK2sOlbtQjckk+ONkIuO+/ItZ6lVoFUYu1OkJLfrj4aLmu+e68e0z98jCMTUIqeAkVZbQeu78burIFuPdaBVGZqtXAoLfjieWa7xvD8QxZtVfDFDuoYfqODlCg8/jH0D+mlunkDp0a1LXga9A4mykgpNUWf7RKgCAuWIU2UJLd9UJBih3o1aVzwnwv8NX8CSrYCvYvRPxadC85Isk1Rqp4CRVlqb1PdjwUk++njSY/cFjAHO5wbfW7C7XeHvP3eA19XfUV6RwzhSA7v7XHCmupAoiFZykStOmgTeB3q70GjWLRG0w9RUpTFCvZ9zqv8q0Fvff6Eu0UlxgZH46pvc142T5v1qAVHCS6oHaBffHFgMwQvUbl2P38fYtNRRKYuL6GHNCTYVgg/EeDqrbV4yckiqFVHCSaoNHywhSwgagUgjmaFbw3+i4218E/HXhBoNV27lbeYYMoWN23tP0axN4+wsl1R6p4CTVCp9B80kXrnRQnmGIapu1gE1xCCF45pNfreX/rnd+ncmDH+CNvuGVIK3E2UgFJ6leeAVyvfPrgLms3/RvtpfY/a0fj/GGei2+igyu6sII6TeBf97dUMa61RKkgpNUOwJ7v0ysqRG+igzan1rMscupdvslZ+QSu/93hqq3AuD1+IegkvklahNSwUmqHS5aLW/mmcNGnlRvY8bHK+z26zLnV+ZoVgJwKnAg7s3urTQZJVUDqeAk1Q6FQsFlz7asMdwPwDuaFazc8bdNn5QsPSNUv9FKeZEU4U6ToQudIKnE2UgFJ6mW7Jr8IFc6Team8KClMo64X5cgREH9huhjsUzIL/93scMk1F71ixtKUoORCk5SLVGrlEwY1J1LHd8AYIL6O7b9dQQAo0mg+n06nops/taE027Ay84UVeJEqoSCW7p0KSEhIeh0Orp27cr+/ftL7L9+/XpatGiBTqejbdu2bNy40ea8EIIZM2YQGBiIq6srERER/P3338WMJqnOtH3kJeLdWuGpyCb958lk5Br4Y9M67tdvx4gCvyEfg1J6TGsrTldwa9euZcKECURGRnLw4EHatWtHnz59uHbtmt3+u3fvZujQoYwdO5aYmBgGDRrEoEGDOHbsmLXP/Pnz+fDDD/nkk0/Yt28f7u7u9OnTh5wcxyVNlFQRlEq8//URRqFggGoPn634D00PzATgdPAQfJt2ca58EucinEyXLl3Eiy++aH1uNBpFUFCQmDt3rt3+gwcPFv3797dp69q1q3j22WeFEEKYTCYREBAgFixYYD2fkpIiXFxcxLffflsqmVJTUwUgUlNTy/pyJE7ij0UjhYj0Ejkz6ggR6SWSZzYSeRk3nC2WpIIo7W/UqRacXq8nOjqaiIgIa5tSqSQiIoI9e/bYvWbPnj02/QH69Olj7X/+/HkSEhJs+nh7e9O1a9dix8zNzSUtLc3mkFQv7h33AckKH1wU5mLOCV3fQu3u62SpJM7GqQru+vXrGI1G/P39bdr9/f1JSEiwe01CQkKJ/S1/yzLm3Llz8fb2th7BwcHlej0S56F29yWx+9sAnHS9m5b/GOtkiSRVAaevwVUFpk6dSmpqqvWIj493tkiSctCq9yhMz+8j/LUNKJTyqy1xcl1UPz8/VCoViYm2KagTExMJCAiwe01AQECJ/S1/ExMTCQwMtOnTvn17u2O6uLjg4iJzg9UElP4tnC2CpArh1H9zWq2Wjh07EhUVZW0zmUxERUXRrVs3u9d069bNpj/Ali1brP1DQ0MJCAiw6ZOWlsa+ffuKHVMikdRQKsnpUSxr1qwRLi4uYtWqVeLEiRNi3LhxwsfHRyQkJAghhBg+fLiYMmWKtf+uXbuEWq0WCxcuFLGxsSIyMlJoNBpx9OhRa5958+YJHx8f8dNPP4kjR46IgQMHitDQUJGdnV0qmaQXVSKp2pT2N+r01ApDhgwhKSmJGTNmkJCQQPv27dm8ebPVSRAXF4ey0HpK9+7d+eabb3jrrbeYNm0azZo148cff6RNmzbWPm+88QaZmZmMGzeOlJQUevbsyebNm9HpdJX++iQSifNQCFFoA58EME9pvb29SU1NxcvLy9niSCSSWyjtb1S6miQSSY1FKjiJRFJjkQpOIpHUWJzuZKiKWJYl5ZYtiaRqYvlt3s6FIBWcHdLT0wHkli2JpIqTnp6Ot7d3seelF9UOJpOJK1eu4OnpiUKhcLY4tyUtLY3g4GDi4+NrvddXvhcF1OT3QghBeno6QUFBNmFktyItODsolUoaNmzobDHKjJeXV437IpcX+V4UUFPfi5IsNwvSySCRSGosUsFJJJIai1RwNQAXFxciIyNlRhTke1EY+V5IJ4NEIqnBSAtOIpHUWKSCk0gkNRap4CQSSY1FKjiJRFJjkQqumnLjxg2GDRuGl5cXPj4+jB07loyMjBKv+fTTT7n//vvx8vJCoVCQkpJSOcI6mKVLlxISEoJOp6Nr167s37+/xP7r16+nRYsW6HQ62rZty8aNGytJ0oqlLO/D8ePHefzxxwkJCUGhULB48eLKE9SJSAVXTRk2bBjHjx9ny5YtbNiwgR07djBu3LgSr8nKyqJv375MmzatkqR0PGvXrmXChAlERkZy8OBB2rVrR58+fbh27Zrd/rt372bo0KGMHTuWmJgYBg0axKBBgzh27FglS+5Yyvo+ZGVlERYWxrx584ot6FQjqdjM6ZKK4MSJEwIQBw4csLZt2rRJKBQKcfny5dtev3XrVgGImzdvVqCUFUOXLl3Eiy++aH1uNBpFUFCQmDt3rt3+gwcPFv3797dp69q1q3j22WcrVM6KpqzvQ2EaN24sPvjggwqUruogLbhqyJ49e/Dx8aFTp07WtoiICJRKJfv27XOiZBWLXq8nOjqaiIgIa5tSqSQiIoI9e/bYvWbPnj02/QH69OlTbP/qQHneh9qKVHDVkISEBOrXr2/TplarqVOnDgkJCU6SquK5fv06RqPRWpDIgr+/f7GvOyEhoUz9qwPleR9qK1LBVSGmTJmCQqEo8Th58qSzxZRIqg0yXVIV4vXXX2fUqFEl9gkLCyMgIKDIYrLBYODGjRs1egHZz88PlUpFYmKiTXtiYmKxrzsgIKBM/asD5XkfaivSgqtC1KtXjxYtWpR4aLVaunXrRkpKCtHR0dZr//jjD0wmE127dnXiK6hYtFotHTt2JCoqytpmMpmIioqiW7dudq/p1q2bTX+ALVu2FNu/OlCe96HW4mwvh6R89O3bV3To0EHs27dP7Ny5UzRr1kwMHTrUev7SpUsiPDxc7Nu3z9p29epVERMTI5YvXy4AsWPHDhETEyOSk5Od8RLKxZo1a4SLi4tYtWqVOHHihBg3bpzw8fERCQkJQgghhg8fLqZMmWLtv2vXLqFWq8XChQtFbGysiIyMFBqNRhw9etRZL8EhlPV9yM3NFTExMSImJkYEBgaKiRMnipiYGPH333876yVUClLBVVOSk5PF0KFDhYeHh/Dy8hKjR48W6enp1vPnz58XgNi6dau1LTIyUgBFjpUrV1b+C7gDPvroI9GoUSOh1WpFly5dxN69e63nevXqJUaOHGnTf926daJ58+ZCq9WK1q1bi19++aWSJa4YyvI+WL4Ptx69evWqfMErEZkuSSKR1FjkGpxEIqmxSAUnkUhqLFLBSSSSGotUcBKJpMYiFZxEIqmxSAUnkUhqLFLBSSSSGotUcBKJpMYiFZyk2jBq1Ci7GVb69u3rbNEkVRSZTURSrejbty8rV660aSuucnteXh4ajcamTa/Xo9Vqy3zf8l4ncS7SgpNUK1xcXAgICLA5fH19AVAoFCxbtowBAwbg7u7OO++8w8yZM2nfvj2fffYZoaGh6HQ6AOLi4hg4cCAeHh54eXkxePBgm/RDxV0nqV5IBSepUcycOZPHHnuMo0ePMmbMGADOnDnDf//7X77//nsOHTqEyWRi4MCB3Lhxg+3bt7NlyxbOnTvHkCFDbMa69TpJ9UNOUSXVig0bNuDh4WHTNm3aNGulsKeeeorRo0fbnNfr9axevZp69eoB5nxwR48e5fz58wQHBwOwevVqWrduzYEDB+jcubPd6yTVD6ngJNWKBx54gGXLltm01alTx/q4cCEeC40bN7ZRUrGxsQQHB1uVG0CrVq3w8fEhNjbWquBuvU5S/ZAKTlKtcHd3p2nTpiWeL01bae8lqd7INThJraNly5bEx8cTHx9vbTtx4gQpKSm0atXKiZJJHI204CTVitzc3CKl8dRqNX5+fqUeIyIigrZt2zJs2DAWL16MwWDghRdeoFevXnanuJLqi7TgJNWKzZs3ExgYaHP07NmzTGMoFAp++uknfH19ue+++4iIiCAsLIy1a9dWkNQSZyFTlkskkhqLtOAkEkmNRSo4iURSY5EKTiKR1FikgpNIJDUWqeAkEkmNRSo4iURSY5EKTiKR1FikgpNIJDUWqeAkEkmNRSo4iURSY5EKTiKR1FikgpNIJDWW/wf83IJEwkjGsAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.02138, + "end_time": "2024-02-29T03:48:51.968906", + "exception": false, + "start_time": "2024-02-29T03:48:51.947526", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 3998.664046, + "end_time": "2024-02-29T03:48:54.713769", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/tab_ddpm_concat/1/mlu-eval.ipynb", + "output_path": "eval/treatment/tab_ddpm_concat/1/mlu-eval.ipynb", + "parameters": { + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "path": "eval/treatment/tab_ddpm_concat/1", + "path_prefix": "../../../../", + "random_seed": 1, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-02-29T02:42:16.049723", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/tab_ddpm_concat/model.pt b/treatment/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd65d3c85d702875cf482c1d57a26159fd327f66 --- /dev/null +++ b/treatment/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a87179687479b48242d52171b430ddd09c5d849427c23f8d3695b72fb175a4e1 +size 74520513 diff --git a/treatment/tab_ddpm_concat/params.json b/treatment/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..7ceeaa1f48f00b758e1abe08081eb558c953d8a6 --- /dev/null +++ b/treatment/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "ALL", "synth_data": 2, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "fixed_role_model": "tab_ddpm_concat", "mse_mag": true, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/tvae/eval.csv b/treatment/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..0dd256378dcca5ee0a4bd67b016097ca836e91b7 --- /dev/null +++ b/treatment/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.05819453223652783,0.0695375685346334,0.0022529522749189204,4.37041163444519,0.12434521317481995,1.0819274187088013,0.37018877267837524,1.2502448953455314e-05,2.501197099685669,0.034844670444726944,0.06411760300397873,0.04746527224779129,0.06811939179897308,0.016122760251164436,6.871608734130859 diff --git a/treatment/tvae/history.csv b/treatment/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..bbbd607f0a74cc5a6d8343e328b0f0d8ff4bdaaf --- /dev/null +++ b/treatment/tvae/history.csv @@ -0,0 +1,21 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.24920250670984387,164.7863325173366,0.07509165827068501,0.5582694623619318,0.0,0.0,0.0,0.0,0.25664645032957195,320,80,146.59210634231567,1.8324013292789458,0.45810033231973646,0.03322597231261568,0.1373219751752913,0.6786565123853506,0.030190404797664883,2.0351780995726587,0.0,0.0,0.0,0.0,0.1389026703312993,80,20,23.60568141937256,1.180284070968628,0.295071017742157,0.17340120701119305 +1,0.036638771339130474,0.17430866390531036,0.0033743313750846937,1.1754540567984804,0.0,0.0,0.0,0.0,0.037131248999867236,320,80,146.09285140037537,1.8261606425046921,0.45654016062617303,0.22347859146539123,0.0072257344153058515,7.664773586231467,0.00015237315602210913,0.36747238576062957,0.0,0.0,0.0,0.0,0.00733818464716478,80,20,23.38762593269348,1.1693812966346742,0.29234532415866854,0.03263759407855105 +2,0.007566167920958833,2.0967797871943104,0.00013614335445514964,0.3681490157265216,0.0,0.0,0.0,0.0,0.007682312215547426,320,80,146.0426082611084,1.825532603263855,0.45638315081596376,0.17641356889216694,0.014423893456114455,0.9881936308058357,0.0008802180807781668,0.9822598381899297,0.0,0.0,0.0,0.0,0.014696174215350766,80,20,23.47496509552002,1.173748254776001,0.29343706369400024,0.05968952318653464 +3,0.005512200832890812,1.2029536205733167,0.00011784128748176409,0.297210247523617,0.0,0.0,0.0,0.0,0.005601800218391873,320,80,146.18077564239502,1.8272596955299378,0.45681492388248446,0.20022519901249325,0.010828381861938397,1.097018459134415,0.0006196479804458476,0.672297232132405,0.0,0.0,0.0,0.0,0.011013832199387253,80,20,23.627057552337646,1.1813528776168822,0.29533821940422056,0.06078822268173099 +4,0.005430534946663101,0.7386607706095158,0.00012488243415409995,0.20501269819797016,0.0,0.0,0.0,0.0,0.0055031314195275625,320,80,145.68419861793518,1.8210524827241898,0.45526312068104746,0.20288582288485485,0.007501897183828987,0.5299218334849585,0.00019840551146526052,0.6603707912378013,0.0,0.0,0.0,0.0,0.007664241141173989,80,20,23.400787353515625,1.1700393676757812,0.2925098419189453,0.06306147864088416 +5,0.005516164661639777,0.5138193142290902,0.00011131920733932709,0.2790927076945081,0.0,0.0,0.0,0.0,0.005602113036184164,320,80,145.87071084976196,1.8233838856220246,0.45584597140550615,0.19050200787605717,0.011137592178420164,0.3970680694670136,0.00047781592774405745,0.8084684604313225,0.0,0.0,0.0,0.0,0.011349219386465848,80,20,23.60161590576172,1.1800807952880858,0.29502019882202146,0.06168455402366817 +6,0.004157086526447529,0.47138002963091596,9.107222525662207e-05,0.2180600252642762,0.0,0.0,0.0,0.0,0.00422368691797601,320,80,145.74650716781616,1.821831339597702,0.4554578348994255,0.1932424872822594,0.015274102193507133,0.5915673881600924,0.0008221152255256214,0.8690497508272529,0.0,0.0,0.0,0.0,0.01553036894329125,80,20,23.404094457626343,1.170204722881317,0.29255118072032926,0.062114660441875455 +7,0.004875784217642831,0.6354096500271702,2.7955386938532945e-05,0.3303372459486127,0.0,0.0,0.0,0.0,0.004964619871316245,320,80,145.63225388526917,1.8204031735658646,0.45510079339146614,0.1817692676151637,0.005858663807157427,2.4625674771152717,0.00011203609905844348,0.3289946096483618,0.0,0.0,0.0,0.0,0.005963563869590871,80,20,23.52048945426941,1.1760244727134705,0.29400611817836764,0.04835958873154596 +8,0.0031127968944019814,0.9152548183759223,1.806079729388954e-05,0.23512825798243284,0.0,0.0,0.0,0.0,0.003174385150487069,320,80,145.92120909690857,1.8240151137113572,0.4560037784278393,0.19399424150469713,0.010585741262184456,0.42557602440410847,0.000421495523987403,0.7471775893121958,0.0,0.0,0.0,0.0,0.010785871403641067,80,20,24.11756443977356,1.205878221988678,0.3014695554971695,0.06357551766559481 +9,0.002172920881457685,0.7474334114474791,1.1737176422308738e-05,0.16109042560565284,0.0,0.0,0.0,0.0,0.0022150414097268367,320,80,146.69107341766357,1.8336384177207947,0.4584096044301987,0.18242204396228773,0.007691837668244261,0.4472601311212202,0.00021899929366062665,0.624529043212533,0.0,0.0,0.0,0.0,0.007850766999763437,80,20,23.481932163238525,1.1740966081619262,0.29352415204048155,0.06110418327152729 +10,0.0014581183171458179,0.35039385620373764,9.355699793850425e-07,0.11253745577996596,0.0,0.0,0.0,0.0,0.0014872381031409531,320,80,146.05332899093628,1.8256666123867036,0.4564166530966759,0.17785912672406995,0.00777593698585406,0.425841556390331,0.00018371345812004947,0.6284717622678727,0.0,0.0,0.0,0.0,0.007937150979705621,80,20,23.56994867324829,1.1784974336624146,0.29462435841560364,0.06416571224108339 +11,0.0011472264940891818,0.20379624920139322,1.6752244385284664e-06,0.0788252267288044,0.0,0.0,0.0,0.0,0.0011692211403953935,320,80,145.7802698612213,1.8222533732652664,0.4555633433163166,0.18026062222197653,0.006733461194380652,0.3946822235986474,0.0001618913361181029,0.5735587835311889,0.0,0.0,0.0,0.0,0.006877737153263297,80,20,23.571922779083252,1.1785961389541626,0.29464903473854065,0.0630436526145786 +12,0.0011293102395939058,0.3500383800602442,1.0578881475417635e-06,0.06207347880699672,0.0,0.0,0.0,0.0,0.0011483672262102118,320,80,146.05268120765686,1.8256585150957108,0.4564146287739277,0.18688904533628375,0.006275050291151274,0.3357202685957191,0.00013311508634341606,0.5655373450368643,0.0,0.0,0.0,0.0,0.0064153664527111685,80,20,23.82382082939148,1.1911910414695739,0.29779776036739347,0.06374898608773946 +13,0.0009926263401212054,0.1724021290308157,1.289044932948724e-06,0.06592325130477547,0.0,0.0,0.0,0.0,0.0010106949302098656,320,80,146.17100930213928,1.827137616276741,0.45678440406918525,0.19642419164301828,0.007367409224389121,0.36254158444035767,0.00015201675970146766,0.6065524470061063,0.0,0.0,0.0,0.0,0.007521036107209511,80,20,23.605858325958252,1.1802929162979126,0.29507322907447814,0.06694348715245724 +14,0.0009602923819784337,0.16205345961572756,1.0564527323226082e-06,0.06139939285349101,0.0,0.0,0.0,0.0,0.0009780729826161406,320,80,146.2790331840515,1.8284879148006439,0.45712197870016097,0.18868204548489304,0.006614773662295193,0.3056405092829664,0.0001378399396664065,0.5676033802330493,0.0,0.0,0.0,0.0,0.006757720473251539,80,20,23.757816553115845,1.1878908276557922,0.29697270691394806,0.06606760704889894 +15,0.0009396908400617576,0.052085603167329,5.237988241696672e-06,0.06155546120571671,0.0,0.0,0.0,0.0,0.0009575500280902816,320,80,146.49028730392456,1.831128591299057,0.45778214782476423,0.1988693069666624,0.006925884122756543,0.2924848004357045,0.00015806480612332586,0.5791323280427605,0.0,0.0,0.0,0.0,0.007072590430470882,80,20,23.813348293304443,1.190667414665222,0.2976668536663055,0.06646179044619202 +16,0.0008116556904951722,0.06441403787640639,2.4224949994013457e-06,0.04650832151528448,0.0,0.0,0.0,0.0,0.0008253834799461402,320,80,146.83057975769043,1.8353822469711303,0.45884556174278257,0.19988758593099193,0.007390370311622973,0.2927192774079003,0.0001821246734236115,0.5955257194116712,0.0,0.0,0.0,0.0,0.0075435809114424044,80,20,23.692466974258423,1.1846233487129212,0.2961558371782303,0.0673548685386777 +17,0.0007709122988444506,0.11857457724095502,7.210952682872376e-07,0.048199543548980726,0.0,0.0,0.0,0.0,0.0007847596802548651,320,80,146.50294041633606,1.8312867552042007,0.4578216888010502,0.18162176406476646,0.007169439975405112,0.2753648401154919,0.00016239425925624885,0.5857524920254946,0.0,0.0,0.0,0.0,0.007319234154419973,80,20,23.768996238708496,1.1884498119354248,0.2971124529838562,0.06929642865434289 +18,0.0007421654891004437,0.048927120538128345,6.539175803788044e-07,0.04725941644865088,0.0,0.0,0.0,0.0,0.0007555454189059674,320,80,146.3331480026245,1.8291643500328063,0.4572910875082016,0.19472089679911733,0.006925846241938416,0.299348188659269,0.00015758154320471006,0.5665402918122708,0.0,0.0,0.0,0.0,0.0070700943761039525,80,20,23.746676921844482,1.187333846092224,0.296833461523056,0.06835744129493833 +19,0.0007015017705725768,0.029490066485919186,2.4620001898843893e-06,0.04654637995117809,0.0,0.0,0.0,0.0,0.0007143976176166688,320,80,146.42631363868713,1.830328920483589,0.45758223012089727,0.18967951615341008,0.006687229181989096,0.2554799986785845,0.00014406391818007903,0.5665019916370511,0.0,0.0,0.0,0.0,0.006830559222726152,80,20,23.93708348274231,1.1968541741371155,0.29921354353427887,0.06820279331877828 diff --git a/treatment/tvae/mlu-eval.ipynb b/treatment/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..43b9124aa65d7cd545656440a59a428ae4678b73 --- /dev/null +++ b/treatment/tvae/mlu-eval.ipynb @@ -0,0 +1,2639 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:45.367004Z", + "iopub.status.busy": "2024-02-29T05:01:45.366632Z", + "iopub.status.idle": "2024-02-29T05:01:45.401517Z", + "shell.execute_reply": "2024-02-29T05:01:45.400589Z" + }, + "papermill": { + "duration": 0.050852, + "end_time": "2024-02-29T05:01:45.403867", + "exception": false, + "start_time": "2024-02-29T05:01:45.353015", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:45.430496Z", + "iopub.status.busy": "2024-02-29T05:01:45.430087Z", + "iopub.status.idle": "2024-02-29T05:01:45.437002Z", + "shell.execute_reply": "2024-02-29T05:01:45.436152Z" + }, + "papermill": { + "duration": 0.02263, + "end_time": "2024-02-29T05:01:45.439054", + "exception": false, + "start_time": "2024-02-29T05:01:45.416424", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:45.463408Z", + "iopub.status.busy": "2024-02-29T05:01:45.463100Z", + "iopub.status.idle": "2024-02-29T05:01:45.467369Z", + "shell.execute_reply": "2024-02-29T05:01:45.466534Z" + }, + "papermill": { + "duration": 0.018805, + "end_time": "2024-02-29T05:01:45.469330", + "exception": false, + "start_time": "2024-02-29T05:01:45.450525", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:45.493342Z", + "iopub.status.busy": "2024-02-29T05:01:45.493043Z", + "iopub.status.idle": "2024-02-29T05:01:45.497229Z", + "shell.execute_reply": "2024-02-29T05:01:45.496506Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018444, + "end_time": "2024-02-29T05:01:45.499122", + "exception": false, + "start_time": "2024-02-29T05:01:45.480678", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:45.523381Z", + "iopub.status.busy": "2024-02-29T05:01:45.522664Z", + "iopub.status.idle": "2024-02-29T05:01:45.528162Z", + "shell.execute_reply": "2024-02-29T05:01:45.527347Z" + }, + "papermill": { + "duration": 0.019778, + "end_time": "2024-02-29T05:01:45.530095", + "exception": false, + "start_time": "2024-02-29T05:01:45.510317", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b6647d11", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:45.556027Z", + "iopub.status.busy": "2024-02-29T05:01:45.555444Z", + "iopub.status.idle": "2024-02-29T05:01:45.560120Z", + "shell.execute_reply": "2024-02-29T05:01:45.559343Z" + }, + "papermill": { + "duration": 0.019902, + "end_time": "2024-02-29T05:01:45.562083", + "exception": false, + "start_time": "2024-02-29T05:01:45.542181", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"tvae\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/tvae/2\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011064, + "end_time": "2024-02-29T05:01:45.584902", + "exception": false, + "start_time": "2024-02-29T05:01:45.573838", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:45.609093Z", + "iopub.status.busy": "2024-02-29T05:01:45.608500Z", + "iopub.status.idle": "2024-02-29T05:01:45.618164Z", + "shell.execute_reply": "2024-02-29T05:01:45.617340Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023917, + "end_time": "2024-02-29T05:01:45.620086", + "exception": false, + "start_time": "2024-02-29T05:01:45.596169", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/tvae/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:45.644561Z", + "iopub.status.busy": "2024-02-29T05:01:45.643899Z", + "iopub.status.idle": "2024-02-29T05:01:47.970906Z", + "shell.execute_reply": "2024-02-29T05:01:47.969979Z" + }, + "papermill": { + "duration": 2.341446, + "end_time": "2024-02-29T05:01:47.972893", + "exception": false, + "start_time": "2024-02-29T05:01:45.631447", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:48.000302Z", + "iopub.status.busy": "2024-02-29T05:01:47.999220Z", + "iopub.status.idle": "2024-02-29T05:01:48.014337Z", + "shell.execute_reply": "2024-02-29T05:01:48.013331Z" + }, + "papermill": { + "duration": 0.031194, + "end_time": "2024-02-29T05:01:48.016410", + "exception": false, + "start_time": "2024-02-29T05:01:47.985216", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:48.040530Z", + "iopub.status.busy": "2024-02-29T05:01:48.040250Z", + "iopub.status.idle": "2024-02-29T05:01:48.048173Z", + "shell.execute_reply": "2024-02-29T05:01:48.047244Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.022361, + "end_time": "2024-02-29T05:01:48.050153", + "exception": false, + "start_time": "2024-02-29T05:01:48.027792", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:48.075091Z", + "iopub.status.busy": "2024-02-29T05:01:48.074481Z", + "iopub.status.idle": "2024-02-29T05:01:48.177171Z", + "shell.execute_reply": "2024-02-29T05:01:48.176431Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.117617, + "end_time": "2024-02-29T05:01:48.179436", + "exception": false, + "start_time": "2024-02-29T05:01:48.061819", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:48.206274Z", + "iopub.status.busy": "2024-02-29T05:01:48.205971Z", + "iopub.status.idle": "2024-02-29T05:01:52.918879Z", + "shell.execute_reply": "2024-02-29T05:01:52.918075Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.729223, + "end_time": "2024-02-29T05:01:52.921175", + "exception": false, + "start_time": "2024-02-29T05:01:48.191952", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 05:01:50.506157: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 05:01:50.506215: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 05:01:50.507965: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:52.946298Z", + "iopub.status.busy": "2024-02-29T05:01:52.945669Z", + "iopub.status.idle": "2024-02-29T05:01:52.952009Z", + "shell.execute_reply": "2024-02-29T05:01:52.951307Z" + }, + "papermill": { + "duration": 0.020927, + "end_time": "2024-02-29T05:01:52.953959", + "exception": false, + "start_time": "2024-02-29T05:01:52.933032", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:01:52.979820Z", + "iopub.status.busy": "2024-02-29T05:01:52.979466Z", + "iopub.status.idle": "2024-02-29T05:02:15.161641Z", + "shell.execute_reply": "2024-02-29T05:02:15.160557Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.197988, + "end_time": "2024-02-29T05:02:15.164084", + "exception": false, + "start_time": "2024-02-29T05:01:52.966096", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BEST,\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:02:15.693168Z", + "iopub.status.busy": "2024-02-29T05:02:15.692867Z", + "iopub.status.idle": "2024-02-29T05:02:15.760257Z", + "shell.execute_reply": "2024-02-29T05:02:15.759389Z" + }, + "papermill": { + "duration": 0.083572, + "end_time": "2024-02-29T05:02:15.762295", + "exception": false, + "start_time": "2024-02-29T05:02:15.678723", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../treatment/_cache/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache4/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/treatment [80, 20]\n", + "Caching in ../../../../treatment/_cache5/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/treatment [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T05:02:15.791351Z", + "iopub.status.busy": "2024-02-29T05:02:15.791015Z", + "iopub.status.idle": "2024-02-29T05:02:16.343182Z", + "shell.execute_reply": "2024-02-29T05:02:16.342248Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.569061, + "end_time": "2024-02-29T05:02:16.345323", + "exception": false, + "start_time": "2024-02-29T05:02:15.776262", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:02:16.375735Z", + "iopub.status.busy": "2024-02-29T05:02:16.375332Z", + "iopub.status.idle": "2024-02-29T05:02:16.379835Z", + "shell.execute_reply": "2024-02-29T05:02:16.378988Z" + }, + "papermill": { + "duration": 0.022218, + "end_time": "2024-02-29T05:02:16.381846", + "exception": false, + "start_time": "2024-02-29T05:02:16.359628", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:02:16.409727Z", + "iopub.status.busy": "2024-02-29T05:02:16.409445Z", + "iopub.status.idle": "2024-02-29T05:02:16.416598Z", + "shell.execute_reply": "2024-02-29T05:02:16.415762Z" + }, + "papermill": { + "duration": 0.023413, + "end_time": "2024-02-29T05:02:16.418446", + "exception": false, + "start_time": "2024-02-29T05:02:16.395033", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18701313" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:02:16.446675Z", + "iopub.status.busy": "2024-02-29T05:02:16.446402Z", + "iopub.status.idle": "2024-02-29T05:02:16.547561Z", + "shell.execute_reply": "2024-02-29T05:02:16.546549Z" + }, + "papermill": { + "duration": 0.118847, + "end_time": "2024-02-29T05:02:16.550838", + "exception": false, + "start_time": "2024-02-29T05:02:16.431991", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 95] --\n", + "├─Adapter: 1-1 [2, 2648, 95] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 98,304\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 95] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,701,313\n", + "Trainable params: 18,701,313\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 74.05\n", + "========================================================================================================================\n", + "Input size (MB): 2.51\n", + "Forward/backward pass size (MB): 1079.48\n", + "Params size (MB): 74.81\n", + "Estimated Total Size (MB): 1156.80\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T05:02:16.585268Z", + "iopub.status.busy": "2024-02-29T05:02:16.584488Z", + "iopub.status.idle": "2024-02-29T06:04:20.288410Z", + "shell.execute_reply": "2024-02-29T06:04:20.287395Z" + }, + "papermill": { + "duration": 3723.724584, + "end_time": "2024-02-29T06:04:20.291222", + "exception": false, + "start_time": "2024-02-29T05:02:16.566638", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.24920250670984387, 'avg_role_model_std_loss': 164.7863325173366, 'avg_role_model_mean_pred_loss': 0.07509165827068501, 'avg_role_model_g_mag_loss': 0.5582694623619318, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.25664645032957195, 'n_size': 320, 'n_batch': 80, 'duration': 146.59210634231567, 'duration_batch': 1.8324013292789458, 'duration_size': 0.45810033231973646, 'avg_pred_std': 0.03322597231261568}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.1373219751752913, 'avg_role_model_std_loss': 0.6786565123853506, 'avg_role_model_mean_pred_loss': 0.030190404797664883, 'avg_role_model_g_mag_loss': 2.0351780995726587, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.1389026703312993, 'n_size': 80, 'n_batch': 20, 'duration': 23.60568141937256, 'duration_batch': 1.180284070968628, 'duration_size': 0.295071017742157, 'avg_pred_std': 0.17340120701119305}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.036638771339130474, 'avg_role_model_std_loss': 0.17430866390531036, 'avg_role_model_mean_pred_loss': 0.0033743313750846937, 'avg_role_model_g_mag_loss': 1.1754540567984804, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.037131248999867236, 'n_size': 320, 'n_batch': 80, 'duration': 146.09285140037537, 'duration_batch': 1.8261606425046921, 'duration_size': 0.45654016062617303, 'avg_pred_std': 0.22347859146539123}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0072257344153058515, 'avg_role_model_std_loss': 7.664773586231467, 'avg_role_model_mean_pred_loss': 0.00015237315602210913, 'avg_role_model_g_mag_loss': 0.36747238576062957, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00733818464716478, 'n_size': 80, 'n_batch': 20, 'duration': 23.38762593269348, 'duration_batch': 1.1693812966346742, 'duration_size': 0.29234532415866854, 'avg_pred_std': 0.03263759407855105}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007566167920958833, 'avg_role_model_std_loss': 2.0967797871943104, 'avg_role_model_mean_pred_loss': 0.00013614335445514964, 'avg_role_model_g_mag_loss': 0.3681490157265216, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007682312215547426, 'n_size': 320, 'n_batch': 80, 'duration': 146.0426082611084, 'duration_batch': 1.825532603263855, 'duration_size': 0.45638315081596376, 'avg_pred_std': 0.17641356889216694}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.014423893456114455, 'avg_role_model_std_loss': 0.9881936308058357, 'avg_role_model_mean_pred_loss': 0.0008802180807781668, 'avg_role_model_g_mag_loss': 0.9822598381899297, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014696174215350766, 'n_size': 80, 'n_batch': 20, 'duration': 23.47496509552002, 'duration_batch': 1.173748254776001, 'duration_size': 0.29343706369400024, 'avg_pred_std': 0.05968952318653464}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005512200832890812, 'avg_role_model_std_loss': 1.2029536205733167, 'avg_role_model_mean_pred_loss': 0.00011784128748176409, 'avg_role_model_g_mag_loss': 0.297210247523617, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005601800218391873, 'n_size': 320, 'n_batch': 80, 'duration': 146.18077564239502, 'duration_batch': 1.8272596955299378, 'duration_size': 0.45681492388248446, 'avg_pred_std': 0.20022519901249325}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010828381861938397, 'avg_role_model_std_loss': 1.097018459134415, 'avg_role_model_mean_pred_loss': 0.0006196479804458476, 'avg_role_model_g_mag_loss': 0.672297232132405, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011013832199387253, 'n_size': 80, 'n_batch': 20, 'duration': 23.627057552337646, 'duration_batch': 1.1813528776168822, 'duration_size': 0.29533821940422056, 'avg_pred_std': 0.06078822268173099}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005430534946663101, 'avg_role_model_std_loss': 0.7386607706095158, 'avg_role_model_mean_pred_loss': 0.00012488243415409995, 'avg_role_model_g_mag_loss': 0.20501269819797016, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0055031314195275625, 'n_size': 320, 'n_batch': 80, 'duration': 145.68419861793518, 'duration_batch': 1.8210524827241898, 'duration_size': 0.45526312068104746, 'avg_pred_std': 0.20288582288485485}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007501897183828987, 'avg_role_model_std_loss': 0.5299218334849585, 'avg_role_model_mean_pred_loss': 0.00019840551146526052, 'avg_role_model_g_mag_loss': 0.6603707912378013, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007664241141173989, 'n_size': 80, 'n_batch': 20, 'duration': 23.400787353515625, 'duration_batch': 1.1700393676757812, 'duration_size': 0.2925098419189453, 'avg_pred_std': 0.06306147864088416}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005516164661639777, 'avg_role_model_std_loss': 0.5138193142290902, 'avg_role_model_mean_pred_loss': 0.00011131920733932709, 'avg_role_model_g_mag_loss': 0.2790927076945081, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005602113036184164, 'n_size': 320, 'n_batch': 80, 'duration': 145.87071084976196, 'duration_batch': 1.8233838856220246, 'duration_size': 0.45584597140550615, 'avg_pred_std': 0.19050200787605717}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011137592178420164, 'avg_role_model_std_loss': 0.3970680694670136, 'avg_role_model_mean_pred_loss': 0.00047781592774405745, 'avg_role_model_g_mag_loss': 0.8084684604313225, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011349219386465848, 'n_size': 80, 'n_batch': 20, 'duration': 23.60161590576172, 'duration_batch': 1.1800807952880858, 'duration_size': 0.29502019882202146, 'avg_pred_std': 0.06168455402366817}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004157086526447529, 'avg_role_model_std_loss': 0.47138002963091596, 'avg_role_model_mean_pred_loss': 9.107222525662207e-05, 'avg_role_model_g_mag_loss': 0.2180600252642762, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00422368691797601, 'n_size': 320, 'n_batch': 80, 'duration': 145.74650716781616, 'duration_batch': 1.821831339597702, 'duration_size': 0.4554578348994255, 'avg_pred_std': 0.1932424872822594}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.015274102193507133, 'avg_role_model_std_loss': 0.5915673881600924, 'avg_role_model_mean_pred_loss': 0.0008221152255256214, 'avg_role_model_g_mag_loss': 0.8690497508272529, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01553036894329125, 'n_size': 80, 'n_batch': 20, 'duration': 23.404094457626343, 'duration_batch': 1.170204722881317, 'duration_size': 0.29255118072032926, 'avg_pred_std': 0.062114660441875455}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004875784217642831, 'avg_role_model_std_loss': 0.6354096500271702, 'avg_role_model_mean_pred_loss': 2.7955386938532945e-05, 'avg_role_model_g_mag_loss': 0.3303372459486127, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004964619871316245, 'n_size': 320, 'n_batch': 80, 'duration': 145.63225388526917, 'duration_batch': 1.8204031735658646, 'duration_size': 0.45510079339146614, 'avg_pred_std': 0.1817692676151637}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005858663807157427, 'avg_role_model_std_loss': 2.4625674771152717, 'avg_role_model_mean_pred_loss': 0.00011203609905844348, 'avg_role_model_g_mag_loss': 0.3289946096483618, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005963563869590871, 'n_size': 80, 'n_batch': 20, 'duration': 23.52048945426941, 'duration_batch': 1.1760244727134705, 'duration_size': 0.29400611817836764, 'avg_pred_std': 0.04835958873154596}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0031127968944019814, 'avg_role_model_std_loss': 0.9152548183759223, 'avg_role_model_mean_pred_loss': 1.806079729388954e-05, 'avg_role_model_g_mag_loss': 0.23512825798243284, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003174385150487069, 'n_size': 320, 'n_batch': 80, 'duration': 145.92120909690857, 'duration_batch': 1.8240151137113572, 'duration_size': 0.4560037784278393, 'avg_pred_std': 0.19399424150469713}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010585741262184456, 'avg_role_model_std_loss': 0.42557602440410847, 'avg_role_model_mean_pred_loss': 0.000421495523987403, 'avg_role_model_g_mag_loss': 0.7471775893121958, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010785871403641067, 'n_size': 80, 'n_batch': 20, 'duration': 24.11756443977356, 'duration_batch': 1.205878221988678, 'duration_size': 0.3014695554971695, 'avg_pred_std': 0.06357551766559481}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002172920881457685, 'avg_role_model_std_loss': 0.7474334114474791, 'avg_role_model_mean_pred_loss': 1.1737176422308738e-05, 'avg_role_model_g_mag_loss': 0.16109042560565284, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022150414097268367, 'n_size': 320, 'n_batch': 80, 'duration': 146.69107341766357, 'duration_batch': 1.8336384177207947, 'duration_size': 0.4584096044301987, 'avg_pred_std': 0.18242204396228773}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007691837668244261, 'avg_role_model_std_loss': 0.4472601311212202, 'avg_role_model_mean_pred_loss': 0.00021899929366062665, 'avg_role_model_g_mag_loss': 0.624529043212533, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007850766999763437, 'n_size': 80, 'n_batch': 20, 'duration': 23.481932163238525, 'duration_batch': 1.1740966081619262, 'duration_size': 0.29352415204048155, 'avg_pred_std': 0.06110418327152729}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014581183171458179, 'avg_role_model_std_loss': 0.35039385620373764, 'avg_role_model_mean_pred_loss': 9.355699793850425e-07, 'avg_role_model_g_mag_loss': 0.11253745577996596, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014872381031409531, 'n_size': 320, 'n_batch': 80, 'duration': 146.05332899093628, 'duration_batch': 1.8256666123867036, 'duration_size': 0.4564166530966759, 'avg_pred_std': 0.17785912672406995}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00777593698585406, 'avg_role_model_std_loss': 0.425841556390331, 'avg_role_model_mean_pred_loss': 0.00018371345812004947, 'avg_role_model_g_mag_loss': 0.6284717622678727, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007937150979705621, 'n_size': 80, 'n_batch': 20, 'duration': 23.56994867324829, 'duration_batch': 1.1784974336624146, 'duration_size': 0.29462435841560364, 'avg_pred_std': 0.06416571224108339}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011472264940891818, 'avg_role_model_std_loss': 0.20379624920139322, 'avg_role_model_mean_pred_loss': 1.6752244385284664e-06, 'avg_role_model_g_mag_loss': 0.0788252267288044, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011692211403953935, 'n_size': 320, 'n_batch': 80, 'duration': 145.7802698612213, 'duration_batch': 1.8222533732652664, 'duration_size': 0.4555633433163166, 'avg_pred_std': 0.18026062222197653}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006733461194380652, 'avg_role_model_std_loss': 0.3946822235986474, 'avg_role_model_mean_pred_loss': 0.0001618913361181029, 'avg_role_model_g_mag_loss': 0.5735587835311889, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006877737153263297, 'n_size': 80, 'n_batch': 20, 'duration': 23.571922779083252, 'duration_batch': 1.1785961389541626, 'duration_size': 0.29464903473854065, 'avg_pred_std': 0.0630436526145786}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011293102395939058, 'avg_role_model_std_loss': 0.3500383800602442, 'avg_role_model_mean_pred_loss': 1.0578881475417635e-06, 'avg_role_model_g_mag_loss': 0.06207347880699672, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011483672262102118, 'n_size': 320, 'n_batch': 80, 'duration': 146.05268120765686, 'duration_batch': 1.8256585150957108, 'duration_size': 0.4564146287739277, 'avg_pred_std': 0.18688904533628375}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006275050291151274, 'avg_role_model_std_loss': 0.3357202685957191, 'avg_role_model_mean_pred_loss': 0.00013311508634341606, 'avg_role_model_g_mag_loss': 0.5655373450368643, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0064153664527111685, 'n_size': 80, 'n_batch': 20, 'duration': 23.82382082939148, 'duration_batch': 1.1911910414695739, 'duration_size': 0.29779776036739347, 'avg_pred_std': 0.06374898608773946}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009926263401212054, 'avg_role_model_std_loss': 0.1724021290308157, 'avg_role_model_mean_pred_loss': 1.289044932948724e-06, 'avg_role_model_g_mag_loss': 0.06592325130477547, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010106949302098656, 'n_size': 320, 'n_batch': 80, 'duration': 146.17100930213928, 'duration_batch': 1.827137616276741, 'duration_size': 0.45678440406918525, 'avg_pred_std': 0.19642419164301828}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007367409224389121, 'avg_role_model_std_loss': 0.36254158444035767, 'avg_role_model_mean_pred_loss': 0.00015201675970146766, 'avg_role_model_g_mag_loss': 0.6065524470061063, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007521036107209511, 'n_size': 80, 'n_batch': 20, 'duration': 23.605858325958252, 'duration_batch': 1.1802929162979126, 'duration_size': 0.29507322907447814, 'avg_pred_std': 0.06694348715245724}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009602923819784337, 'avg_role_model_std_loss': 0.16205345961572756, 'avg_role_model_mean_pred_loss': 1.0564527323226082e-06, 'avg_role_model_g_mag_loss': 0.06139939285349101, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009780729826161406, 'n_size': 320, 'n_batch': 80, 'duration': 146.2790331840515, 'duration_batch': 1.8284879148006439, 'duration_size': 0.45712197870016097, 'avg_pred_std': 0.18868204548489304}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006614773662295193, 'avg_role_model_std_loss': 0.3056405092829664, 'avg_role_model_mean_pred_loss': 0.0001378399396664065, 'avg_role_model_g_mag_loss': 0.5676033802330493, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006757720473251539, 'n_size': 80, 'n_batch': 20, 'duration': 23.757816553115845, 'duration_batch': 1.1878908276557922, 'duration_size': 0.29697270691394806, 'avg_pred_std': 0.06606760704889894}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009396908400617576, 'avg_role_model_std_loss': 0.052085603167329, 'avg_role_model_mean_pred_loss': 5.237988241696672e-06, 'avg_role_model_g_mag_loss': 0.06155546120571671, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009575500280902816, 'n_size': 320, 'n_batch': 80, 'duration': 146.49028730392456, 'duration_batch': 1.831128591299057, 'duration_size': 0.45778214782476423, 'avg_pred_std': 0.1988693069666624}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006925884122756543, 'avg_role_model_std_loss': 0.2924848004357045, 'avg_role_model_mean_pred_loss': 0.00015806480612332586, 'avg_role_model_g_mag_loss': 0.5791323280427605, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007072590430470882, 'n_size': 80, 'n_batch': 20, 'duration': 23.813348293304443, 'duration_batch': 1.190667414665222, 'duration_size': 0.2976668536663055, 'avg_pred_std': 0.06646179044619202}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008116556904951722, 'avg_role_model_std_loss': 0.06441403787640639, 'avg_role_model_mean_pred_loss': 2.4224949994013457e-06, 'avg_role_model_g_mag_loss': 0.04650832151528448, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008253834799461402, 'n_size': 320, 'n_batch': 80, 'duration': 146.83057975769043, 'duration_batch': 1.8353822469711303, 'duration_size': 0.45884556174278257, 'avg_pred_std': 0.19988758593099193}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007390370311622973, 'avg_role_model_std_loss': 0.2927192774079003, 'avg_role_model_mean_pred_loss': 0.0001821246734236115, 'avg_role_model_g_mag_loss': 0.5955257194116712, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0075435809114424044, 'n_size': 80, 'n_batch': 20, 'duration': 23.692466974258423, 'duration_batch': 1.1846233487129212, 'duration_size': 0.2961558371782303, 'avg_pred_std': 0.0673548685386777}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007709122988444506, 'avg_role_model_std_loss': 0.11857457724095502, 'avg_role_model_mean_pred_loss': 7.210952682872376e-07, 'avg_role_model_g_mag_loss': 0.048199543548980726, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007847596802548651, 'n_size': 320, 'n_batch': 80, 'duration': 146.50294041633606, 'duration_batch': 1.8312867552042007, 'duration_size': 0.4578216888010502, 'avg_pred_std': 0.18162176406476646}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007169439975405112, 'avg_role_model_std_loss': 0.2753648401154919, 'avg_role_model_mean_pred_loss': 0.00016239425925624885, 'avg_role_model_g_mag_loss': 0.5857524920254946, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007319234154419973, 'n_size': 80, 'n_batch': 20, 'duration': 23.768996238708496, 'duration_batch': 1.1884498119354248, 'duration_size': 0.2971124529838562, 'avg_pred_std': 0.06929642865434289}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007421654891004437, 'avg_role_model_std_loss': 0.048927120538128345, 'avg_role_model_mean_pred_loss': 6.539175803788044e-07, 'avg_role_model_g_mag_loss': 0.04725941644865088, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007555454189059674, 'n_size': 320, 'n_batch': 80, 'duration': 146.3331480026245, 'duration_batch': 1.8291643500328063, 'duration_size': 0.4572910875082016, 'avg_pred_std': 0.19472089679911733}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006925846241938416, 'avg_role_model_std_loss': 0.299348188659269, 'avg_role_model_mean_pred_loss': 0.00015758154320471006, 'avg_role_model_g_mag_loss': 0.5665402918122708, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0070700943761039525, 'n_size': 80, 'n_batch': 20, 'duration': 23.746676921844482, 'duration_batch': 1.187333846092224, 'duration_size': 0.296833461523056, 'avg_pred_std': 0.06835744129493833}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007015017705725768, 'avg_role_model_std_loss': 0.029490066485919186, 'avg_role_model_mean_pred_loss': 2.4620001898843893e-06, 'avg_role_model_g_mag_loss': 0.04654637995117809, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007143976176166688, 'n_size': 320, 'n_batch': 80, 'duration': 146.42631363868713, 'duration_batch': 1.830328920483589, 'duration_size': 0.45758223012089727, 'avg_pred_std': 0.18967951615341008}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006687229181989096, 'avg_role_model_std_loss': 0.2554799986785845, 'avg_role_model_mean_pred_loss': 0.00014406391818007903, 'avg_role_model_g_mag_loss': 0.5665019916370511, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006830559222726152, 'n_size': 80, 'n_batch': 20, 'duration': 23.93708348274231, 'duration_batch': 1.1968541741371155, 'duration_size': 0.29921354353427887, 'avg_pred_std': 0.06820279331877828}\n", + "Epoch 20\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006478702302104011, 'avg_role_model_std_loss': 0.02238980184622843, 'avg_role_model_mean_pred_loss': 9.630960396186244e-07, 'avg_role_model_g_mag_loss': 0.04726045283750864, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006607293610443321, 'n_size': 320, 'n_batch': 80, 'duration': 146.8006751537323, 'duration_batch': 1.8350084394216537, 'duration_size': 0.4587521098554134, 'avg_pred_std': 0.19540356531506403}\n", + "Time out: 3604.423198223114/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test █▁▂▂▃▂▂▂▃▂▃▃▃▃▃▃▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train ▁█▆▇▇▇▇▆▇▆▆▆▇▇▇▇▇▆▇▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test █▁▄▂▂▃▃▁▃▂▂▂▂▂▂▂▂▂▂▂\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▄█▃▃▂▂▂▃▂▂▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test ▁█▂▂▁▁▁▃▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▃▁▂▃▁▃▁▂█▂▃▃▅▃▅▅▄▅▄▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▇▄▃▄▁▂▂▁▃▇▃▂▃▄▅▆█▆▅▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▃▁▂▃▁▃▁▂█▂▃▃▅▃▅▅▄▅▄▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▇▄▃▄▁▂▂▁▃▇▃▂▃▄▅▆█▆▅▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▃▁▂▃▁▃▁▂█▂▃▃▅▃▅▅▄▅▄▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▇▄▃▄▁▂▂▁▃▇▃▂▃▄▅▆█▆▅▆\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00683\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00071\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.0682\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.18968\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.5665\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.04655\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00669\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.0007\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 0.00014\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 0.25548\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 0.02949\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 1.19685\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 1.83033\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.29921\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.45758\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 23.93708\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 146.42631\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 20\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/treatment/tvae/2/wandb/offline-run-20240229_050218-7ygncdgy\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_050218-7ygncdgy/logs\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 399, 'n_batch': 100, 'role_model_metrics': {'avg_loss': 0.0022529522714906497, 'avg_g_mag_loss': 0.053525562122824034, 'avg_g_cos_loss': 0.06272150501422118, 'pred_duration': 2.4670803546905518, 'grad_duration': 4.40449857711792, 'total_duration': 6.871578931808472, 'pred_std': 0.06811939179897308, 'std_loss': 0.016122760251164436, 'mean_pred_loss': 1.2502448953455314e-05, 'pred_rmse': 0.04746527597308159, 'pred_mae': 0.034844670444726944, 'pred_mape': 0.06411760300397873, 'grad_rmse': 0.37018877267837524, 'grad_mae': 0.12434521317481995, 'grad_mape': 1.0819274187088013}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0022529522714906497, 'avg_g_mag_loss': 0.053525562122824034, 'avg_g_cos_loss': 0.06272150501422118, 'avg_pred_duration': 2.4670803546905518, 'avg_grad_duration': 4.40449857711792, 'avg_total_duration': 6.871578931808472, 'avg_pred_std': 0.06811939179897308, 'avg_std_loss': 0.016122760251164436, 'avg_mean_pred_loss': 1.2502448953455314e-05}, 'min_metrics': {'avg_loss': 0.0022529522714906497, 'avg_g_mag_loss': 0.053525562122824034, 'avg_g_cos_loss': 0.06272150501422118, 'pred_duration': 2.4670803546905518, 'grad_duration': 4.40449857711792, 'total_duration': 6.871578931808472, 'pred_std': 0.06811939179897308, 'std_loss': 0.016122760251164436, 'mean_pred_loss': 1.2502448953455314e-05, 'pred_rmse': 0.04746527597308159, 'pred_mae': 0.034844670444726944, 'pred_mape': 0.06411760300397873, 'grad_rmse': 0.37018877267837524, 'grad_mae': 0.12434521317481995, 'grad_mape': 1.0819274187088013}, 'model_metrics': {'tvae': {'avg_loss': 0.0022529522714906497, 'avg_g_mag_loss': 0.053525562122824034, 'avg_g_cos_loss': 0.06272150501422118, 'pred_duration': 2.4670803546905518, 'grad_duration': 4.40449857711792, 'total_duration': 6.871578931808472, 'pred_std': 0.06811939179897308, 'std_loss': 0.016122760251164436, 'mean_pred_loss': 1.2502448953455314e-05, 'pred_rmse': 0.04746527597308159, 'pred_mae': 0.034844670444726944, 'pred_mape': 0.06411760300397873, 'grad_rmse': 0.37018877267837524, 'grad_mae': 0.12434521317481995, 'grad_mape': 1.0819274187088013}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:04:20.333276Z", + "iopub.status.busy": "2024-02-29T06:04:20.332924Z", + "iopub.status.idle": "2024-02-29T06:04:20.337610Z", + "shell.execute_reply": "2024-02-29T06:04:20.336666Z" + }, + "papermill": { + "duration": 0.0284, + "end_time": "2024-02-29T06:04:20.339576", + "exception": false, + "start_time": "2024-02-29T06:04:20.311176", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:04:20.391686Z", + "iopub.status.busy": "2024-02-29T06:04:20.391330Z", + "iopub.status.idle": "2024-02-29T06:04:20.848476Z", + "shell.execute_reply": "2024-02-29T06:04:20.847663Z" + }, + "papermill": { + "duration": 0.490131, + "end_time": "2024-02-29T06:04:20.850835", + "exception": false, + "start_time": "2024-02-29T06:04:20.360704", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:04:20.891849Z", + "iopub.status.busy": "2024-02-29T06:04:20.891524Z", + "iopub.status.idle": "2024-02-29T06:04:21.165711Z", + "shell.execute_reply": "2024-02-29T06:04:21.164849Z" + }, + "papermill": { + "duration": 0.297863, + "end_time": "2024-02-29T06:04:21.167968", + "exception": false, + "start_time": "2024-02-29T06:04:20.870105", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzVUlEQVR4nO3deXRUVb7o8W9VJVWZEyBkwpgQIMyTDGlQtJU0CSoaLrbA4gnkKVy94r0aEU23JNh2G6broxUWdOMT5D4ZtFu896k3DpFooxGUQVCQBxgIUxISJPNYdd4fJ3VSFSohlYRUkfp91jqrqk7tc2qfJPXL3r9z9j46RVEUhBDCDehdXQEhhLCSgCSEcBsSkIQQbkMCkhDCbUhAEkK4DQlIQgi3IQFJCOE2vFxdga5gsVi4ePEigYGB6HQ6V1dHCNGCoihUVFQQFRWFXt96O6hHBKSLFy8SHR3t6moIIa7j3Llz3HLLLa2+3yMCUmBgIKAebFBQkItrI4Roqby8nOjoaO272poeEZCs3bSgoCAJSEK4seulVCSpLYRwGxKQhBBuQwKSEMJt9Igckugcs9lMQ0ODq6shbmLe3t4YDIZO70cCkgdTFIXCwkKuXr3q6qqIHiAkJISIiIhOXQsoAcmDWYNRWFgYfn5+clGp6BBFUaiurqa4uBiAyMjIDu/LowKS2aJwsOAXymsa+PXgMAx6z/0Cms1mLRj16dPH1dURNzlfX18AiouLCQsL63D3zaOS2maLwm835fHoW99RWdvo6uq4lDVn5Ofn5+KaiJ7C+rfUmXykRwUko5ceH2/1kMtrJYkL179QTYj26oq/pQ4FpA0bNhAbG4uPjw8JCQns37+/1bKbN29mypQp9OrVi169epGYmHhN+YULF6LT6eyW5OTkjlTtuoJ8vAEJSEK4I6cD0q5du0hLSyMzM5ODBw8yevRokpKStIRWS7m5ucydO5c9e/aQl5dHdHQ006ZN48KFC3blkpOTuXTpkrbs2LGjY0d0HUG+TQGpxrO7bEK4I6cD0quvvsqiRYtITU1l2LBhbNq0CT8/P958802H5d9++23+5V/+hTFjxjBkyBDeeOMNLBYLOTk5duVMJhMRERHa0qtXr44d0XUE+ah5fGkhiY7S6XS8//77rq5Gl1qxYgVjxoxxdTWcC0j19fUcOHCAxMTE5h3o9SQmJpKXl9eufVRXV9PQ0EDv3r3t1ufm5hIWFsbgwYN54oknKC0tbXUfdXV1lJeX2y3t1dxCkoAkbl5bt24lJCSky/a3dOnSaxoJruBUQCopKcFsNhMeHm63Pjw8nMLCwnbt4/nnnycqKsouqCUnJ7Nt2zZycnJYtWoVX3zxBdOnT8dsNjvcR1ZWFsHBwdrizFxIgU05pAoPP8smPEN9fX27ygUEBLjF5R/depZt5cqV7Ny5k927d+Pj46OtnzNnDg888AAjR44kJSWFDz74gG+//Zbc3FyH+0lPT6esrExbzp071+46SJetdYqiUF3f2O2LszdPzs7O5o477iAkJIQ+ffpw//33c/r0aQAmT57M888/b1f+8uXLeHt78+WXXwJw6dIl7rvvPnx9fenfvz/bt28nNjaWdevWdejndvToUe655x58fX3p06cPixcvprKyUns/NzeXiRMn4u/vT0hICLfffjtnz54F4Pvvv+fuu+8mMDCQoKAgxo0bx3fffdfm5+Xm5pKamkpZWZl2EmjFihUAxMbG8vLLLzN//nyCgoJYvHgxoDYE4uPj8fPzIy4ujuXLl9udnm/ZZVu4cCEpKSmsXbuWyMhI+vTpw5NPPnnDhxg5dWFkaGgoBoOBoqIiu/VFRUVERES0ue3atWtZuXIln332GaNGjWqzbFxcHKGhoZw6dYqpU6de877JZMJkMjlTdY0ktVtX02BmWMbH3f65x/6QhJ+x/X+KVVVVpKWlMWrUKCorK8nIyGDmzJkcPnyYefPmsXr1alauXKmdht61axdRUVFMmTIFgPnz51NSUkJubi7e3t6kpaW1elKmPXVJSkpi0qRJfPvttxQXF/PYY4+xZMkStm7dSmNjIykpKSxatIgdO3ZQX1/P/v37tbrNmzePsWPHsnHjRgwGA4cPH8bb27vNz5w8eTLr1q0jIyODEydOAGoLx2rt2rVkZGSQmZmprQsMDGTr1q1ERUVx9OhRFi1aRGBgIMuWLWv1c/bs2UNkZCR79uzh1KlTzJ49mzFjxrBo0aIO/azaw6mAZDQaGTduHDk5OaSkpABoCeolS5a0ut3q1av505/+xMcff8z48eOv+znnz5+ntLS0U5egt0ZO+9/8Zs2aZff6zTffpG/fvhw7doyHH36Yp59+mr1792oBaPv27cydOxedTsdPP/3EZ599xrfffqv9Lb7xxhsMGjSoQ3XZvn07tbW1bNu2DX9/fwDWr1/PjBkzWLVqFd7e3pSVlXH//fczYMAAAIYOHaptX1BQwHPPPceQIUMA2lUPo9FIcHAwOp3OYUPgnnvu4dlnn7Vb9+KLL2rPY2NjWbp0KTt37mwzIPXq1Yv169djMBgYMmQI9913Hzk5Oe4TkADS0tJYsGAB48ePZ+LEiaxbt46qqipSU1MB9b9Pv379yMrKAmDVqlVkZGRozWJrrikgIICAgAAqKyt56aWXmDVrFhEREZw+fZply5YxcOBAkpKSuvBQVUG+TV02SWpfw9fbwLE/dP3PvD2f64yTJ0+SkZHBvn37KCkpwWKxAOqXe8SIEUybNo23336bKVOmkJ+fT15eHn/5y18AOHHiBF5eXtx2223a/gYOHNjhs7rHjx9n9OjRWjACuP3227FYLJw4cYI777yThQsXkpSUxG9+8xsSExN5+OGHtX+2aWlpPPbYY/zHf/wHiYmJ/Pa3v9UCV0c5+qe/a9cuXnvtNU6fPk1lZSWNjY3XnV11+PDhdkNAIiMjOXr0aKfqdj1O55Bmz56tNQnHjBnD4cOHyc7O1hLdBQUFXLp0SSu/ceNG6uvreeihh4iMjNSWtWvXAmAwGDhy5AgPPPAA8fHxPProo4wbN45//OMfHe6WtSVQWkit0ul0+Bm9un1x9grfGTNmcOXKFTZv3sy+ffvYt28f0JzAnTdvHn/7299oaGhg+/btjBw5kpEjR3b5z6u9tmzZQl5eHpMnT2bXrl3Ex8fzzTffAGru5scff+S+++7j888/Z9iwYezevbtTn2cbHAHy8vKYN28e9957Lx988AGHDh3i97///XUT3i27jjqdTgv+N0qHBtcuWbKk1S5ay0T0mTNn2tyXr68vH3/cfXkLa1JbzrLdnEpLSzlx4oQ2AgBg7969dmUefPBBFi9eTHZ2Ntu3b2f+/Pnae4MHD6axsZFDhw4xbtw4AE6dOsUvv/zSofoMHTqUrVu3UlVVpQWCr776Cr1ez+DBg7VyY8eOZezYsaSnpzNp0iS2b9/Or371KwDi4+OJj4/nmWeeYe7cuWzZsoWZM2e2+blGo7HVs9Atff3118TExPD73/9eW2dNqrsbjxrLBjZJbWkh3ZR69epFnz59+Otf/8qpU6f4/PPPSUtLsyvj7+9PSkoKy5cv5/jx48ydO1d7b8iQISQmJrJ48WL279/PoUOHWLx4Mb6+vh0aizVv3jx8fHxYsGABP/zwA3v27OGpp57ikUceITw8nPz8fNLT08nLy+Ps2bN88sknnDx5kqFDh1JTU8OSJUvIzc3l7NmzfPXVV3z77bd2OabWxMbGUllZSU5ODiUlJVRXV7dadtCgQRQUFLBz505Onz7Na6+91ulW2I3ieQHJR86y3cz0ej07d+7kwIEDjBgxgmeeeYY1a9ZcU27evHl8//33TJkyhVtvvdXuvW3bthEeHs6dd97JzJkztTNOtpeitJefnx8ff/wxV65cYcKECTz00ENMnTqV9evXa+//9NNPzJo1i/j4eBYvXsyTTz7JP//zP2MwGCgtLWX+/PnEx8fz8MMPM336dF566aXrfu7kyZN5/PHHmT17Nn379mX16tWtln3ggQd45plnWLJkCWPGjOHrr79m+fLlTh9rd9Apzl4E4obKy8sJDg6mrKzsuom64opaJv4pB70OTv3pXvQeOidSbW0t+fn59O/fv0NfxJ7k/PnzREdH89lnnzm8zES0T1t/U+39jnrUBG3Q3EKyKFBV36gluYXn+Pzzz6msrGTkyJFcunSJZcuWERsby5133unqqnk8j+uymbz0GA3qYUti2zM1NDTwu9/9juHDhzNz5kz69u2rXST59ttva5ektFyGDx/ebXWcPn16q/V45ZVXuq0e3c3jWkg6nY4gXy9KKuspr20gCl9XV0l0s6SkpFavcXvggQdISEhw+N71rqDuSm+88QY1NTUO32s5ML0n8biABGq3raSyXhLb4hqBgYHXvf98d+jXr5+rq+ASHtdlAwiUKUiEcEseGZBkxL8Q7slDA5LMiSSEO/LMgCQDbIVwS54ZkGSArRBuyTMDkkzSJjqhJ07y7y48MiAFSlJb3OS6epJ/UGfq0Ol0XL16tUv36wyPDEiS1BbCPXlmQPKVFpJDigL1Vd2/yCT/XTbJf11dHUuXLqVfv374+/uTkJBgN0fZ2bNnmTFjBr169cLf35/hw4fz0UcfcebMGe6++25AneJFp9OxcOHCDv08OsNjr9QGOct2jYZqeCWq+z/3dxfB6H/9ck1kkv/WJ/lfsmQJx44dY+fOnURFRbF7926Sk5M5evQogwYN4sknn6S+vp4vv/wSf39/jh07RkBAANHR0fz9739n1qxZnDhxgqCgIHx9u39YlWcGJG2SNumy3Yxkkn/Hk/wXFBSwZcsWCgoKiIpS/7EsXbqU7OxstmzZwiuvvEJBQQGzZs3SpvSNi4vTtreOkQsLC+vy/FR7eWZAsmkhKYrSoZkCeyRvP7W14orPdYJM8u/Y0aNHMZvNxMfH262vq6vTbgL5r//6rzzxxBN88sknJCYmMmvWrOvelqw7eWQOyXqWrdGiUNPQvnmJPYJOp3adunuRSf67ZJL/yspKDAYDBw4c4PDhw9py/Phx/vznPwPw2GOP8fPPP/PII49w9OhRxo8fz+uvv95lx9pZHhmQ/IwGDE0zRcqZtpuLdZL/F198kalTpzJ06NBrJuh/8MEHqa2t1Sb5nzdvnvae7ST/Vp2d5P/777+nqqpKW9faJP/p6el8/fXXjBgxgu3bt2vvWSf4/+STT/inf/ontmzZct3PdTTJ/9ixYzGbzRQXFzNw4EC7xbZrFx0dzeOPP857773Hs88+y+bNm7V9Au2+ecCN4JEBSafTNQ+wlcT2TUUm+Vc5muQ/Pj6eefPmMX/+fN577z3y8/PZv38/WVlZfPjhhwA8/fTTfPzxx+Tn53Pw4EH27NmjfV5MTAw6nY4PPviAy5cv250p7DZKD1BWVqYASllZWbu3uXP150rM8x8o350pvYE1c181NTXKsWPHlJqaGldXxWmffvqpMnToUMVkMimjRo1ScnNzFUDZvXu3Vuajjz5SAOXOO++8ZvuLFy8q06dPV0wmkxITE6Ns375dCQsLUzZt2tSuz2/5WUeOHFHuvvtuxcfHR+ndu7eyaNEipaKiQlEURSksLFRSUlKUyMhIxWg0KjExMUpGRoZiNpuVuro6Zc6cOUp0dLRiNBqVqKgoZcmSJe3+nTz++ONKnz59FEDJzMxUFEVR6uvrlYyMDCU2Nlbx9vZWIiMjlZkzZypHjhxRFEVRlixZogwYMEAxmUxK3759lUceeUQpKSnR9vmHP/xBiYiIUHQ6nbJgwYJ21cOqrb+p9n5HPW6Sf6sZr+/l6IUytiycwN1Dwm5wDd2PTPLfTCb57xoyyX8nyPARzyWT/Lsvj8whge2If0lqexqZ5N99eWwLSeZE8lwyyb/78tyAJHMiCQdkkn/X8twum8yJBEAPOKch3ERX/C15bEDy9KS2tftRXV3t4pqInsL6t9SZrm2HumwbNmxgzZo1FBYWMnr0aF5//XUmTpzosOzmzZvZtm0bP/zwAwDjxo3jlVdesSuvKAqZmZls3ryZq1evcvvtt7Nx48YOD3hsD0+fE8lgMBASEqKNcvfz85MxfaJDFEWhurqa4uJiQkJCMBgMHd6X0wFp165dpKWlsWnTJhISEli3bh1JSUmcOHGCsLBrr+fJzc1l7ty5TJ48GR8fH1atWsW0adP48ccftX7y6tWree2113jrrbfo378/y5cvJykpiWPHjt2wa2SC5N5s2nCCjk69IYStkJAQuyEqHeH0hZEJCQlMmDCB9evXA2CxWIiOjuapp57ihRdeuO72ZrOZXr16sX79eubPn4+iKERFRfHss8+ydOlSAMrKyggPD2fr1q3MmTPnuvvsyIWR+34uZfZfvyGurz+fP/vrdm3TU5nNZhoaPDcwi87z9vZus2V0Qy6MrK+v58CBA6Snp2vr9Ho9iYmJ5OXltWsf1dXVNDQ0aKcu8/PzKSwsJDExUSsTHBxMQkICeXl5DgNSXV0ddXV12uvy8nJnDgOQpLYtg8HQqWa2EF3FqaR2SUkJZrOZ8PBwu/Xh4eEUFha2ax/PP/88UVFRWgCybufMPrOysggODtaW6OhoZw4DkKS2EO6oW8+yrVy5kp07d7J79+5O5YbS09MpKyvTlnPnzjm9D2sLqb7RQq3MiSSEW3AqIIWGhmIwGCgqKrJbX1RUdN1k1tq1a1m5ciWffPKJ3Qx11u2c2afJZCIoKMhucVaA0UubF8xTz7QJ4W6cCkhGo5Fx48aRk5OjrbNYLOTk5DBp0qRWt1u9ejUvv/wy2dnZ2jzGVv379yciIsJun+Xl5ezbt6/NfXaWXq8j0CTdNiHcidOn/dPS0liwYAHjx49n4sSJrFu3jqqqKlJTUwH1jg79+vUjKysLgFWrVpGRkaHdasaaF7IOFNTpdDz99NP88Y9/ZNCgQdpp/6ioKFJSUrruSB0I8vWmvLbRo0/9C+FOnA5Is2fP5vLly2RkZFBYWMiYMWPIzs7WktIFBQXo9c0Nr40bN1JfX89DDz1kt5/MzEztXlLLli2jqqqKxYsXc/XqVe644w6ys7Nv+Dw96sWRNTLiXwg34bETtAHM/kse+/Kv8PrcscwY7YL7kQnhIdr7HfXYsWzQfKZNktpCuAfPDkgyBYkQbsWzA5JM0iaEW/HsgCQtJCHcikcHJG34iIxnE8IteHRAak5qSwtJCHfg2QFJ7jwihFvx7IAkSW0h3IpnByRJagvhViQgIUltIdyFZwekpi5bTYOZBrPFxbURQnh0QAowNY8tluEjQrieRwckL4NeC0qS2BbC9Tw6IAEEydzaQrgNjw9IgZLYFsJteHxA0q5FkhaSEC4nAclHho8I4S4kIMkNI4VwGxKQJKkthNuQgKS1kCQgCeFqHh+Qmm+pLV02IVzN4wOSJLWFcB8SkCSpLYTbkIAkU5AI4TYkIMkkbUK4DY8PSIEyja0QbsPjA5L1OqTKukbMlpv+ruJC3NQ8PiBZW0gAldJKEsKlPD4gGb30+HobAElsC+FqHh+QoDmxXSaJbSFcSgIStoltCUhCuFKHAtKGDRuIjY3Fx8eHhIQE9u/f32rZH3/8kVmzZhEbG4tOp2PdunXXlFmxYgU6nc5uGTJkSEeq1iHWxLbMqy2EazkdkHbt2kVaWhqZmZkcPHiQ0aNHk5SURHFxscPy1dXVxMXFsXLlSiIiIlrd7/Dhw7l06ZK27N2719mqdZgMsBXCPTgdkF599VUWLVpEamoqw4YNY9OmTfj5+fHmm286LD9hwgTWrFnDnDlzMJlMre7Xy8uLiIgIbQkNDW21bF1dHeXl5XZLZ8gttYVwD04FpPr6eg4cOEBiYmLzDvR6EhMTycvL61RFTp48SVRUFHFxccybN4+CgoJWy2ZlZREcHKwt0dHRnfpsuVpbCPfgVEAqKSnBbDYTHh5utz48PJzCwsIOVyIhIYGtW7eSnZ3Nxo0byc/PZ8qUKVRUVDgsn56eTllZmbacO3euw58NktQWwl14Xb/IjTd9+nTt+ahRo0hISCAmJoZ33nmHRx999JryJpOpze6fs+SW2kK4B6daSKGhoRgMBoqKiuzWFxUVtZmwdlZISAjx8fGcOnWqy/bZFmuXTeZEEsK1nApIRqORcePGkZOTo62zWCzk5OQwadKkLqtUZWUlp0+fJjIyssv22RaZgkQI9+B0ly0tLY0FCxYwfvx4Jk6cyLp166iqqiI1NRWA+fPn069fP7KysgA1EX7s2DHt+YULFzh8+DABAQEMHDgQgKVLlzJjxgxiYmK4ePEimZmZGAwG5s6d21XH2SaZpE0I9+B0QJo9ezaXL18mIyODwsJCxowZQ3Z2tpboLigoQK9vbnhdvHiRsWPHaq/Xrl3L2rVrueuuu8jNzQXg/PnzzJ07l9LSUvr27csdd9zBN998Q9++fTt5eO0jdx4Rwj3oFEW56efcKC8vJzg4mLKyMoKCgpze/lRxJYmvfkGQjxdHViTdgBoK4dna+x31rLFs9VXwX0/B2w+Dxayttia1K+sascicSEK4jFuc9u82Xj5w6P+AYoGqyxConhm0JrUtClTVN9rNkSSE6D6e1ULSGyCg6aLOikvaah9vA0Yv9Uchw0eEcB3PCkigtYoov2S3uvniSElsC+EqHhiQotTHipYBScazCeFqHhiQmlpIFfZj7wJ9rXewlS6bEK7ieQEpqOnq74qL9qvlWiQhXM7zAlKgNSDZt5BkkjYhXM8DA5LjLptM0iaE63lgQGpKape30mWTFpIQLuOBAamphVRzBRrrtNVBktQWwuU8LyD59lKv2Aa7U/+S1BbC9TwvIOl0DvNIWlJbApIQLuN5AQlszrTZtpBkTiQhXM2zA5LN8JFA6bIJ4XKeHZBsW0hyHZIQLuehAclBDsmn+SxbD5izToibkmcGpKBrB9haJ2lrtCjUNJgdbSWEuME8MyBpLaTmgOTrbcBLrwMksS2Eq3hoQLJJajd1z3Q6nZz6F8LFPDQgNbWQGqqgrvl23YEyfEQIl/LMgGT0B1Ow+ryVxLYQovt5ZkACh/MiWRPb0mUTwjU8NyC1cepfumxCuIYHB6Q2ho9Il00Il5CA5Gj4iLSQhHAJCUiOho9IC0kIl/DcgBTkqMsmSW0hXMlzA5KDyf5lgK0QruXBAcnmLJvFAkhSWwhX61BA2rBhA7Gxsfj4+JCQkMD+/ftbLfvjjz8ya9YsYmNj0el0rFu3rtP77BIB4YAOLA1QXQo0J7UrpIUkhEs4HZB27dpFWloamZmZHDx4kNGjR5OUlERxcbHD8tXV1cTFxbFy5UoiIiK6ZJ9dwuAN/n3V5015JBnLJoRrOR2QXn31VRYtWkRqairDhg1j06ZN+Pn58eabbzosP2HCBNasWcOcOXMwmUxdss+6ujrKy8vtlg5pcXGknGUTwrWcCkj19fUcOHCAxMTE5h3o9SQmJpKXl9ehCnRkn1lZWQQHB2tLdHR0hz67eV4kdfiI9SxbfaOFWpkTSYhu51RAKikpwWw2Ex4ebrc+PDycwsLCVrbq+n2mp6dTVlamLefOnevQZ7dsIfkbvWiaEkm6bUK4gJerK9ARJpOp1e6fU1rcxVav1xFg8qK8tpHymkbCAjv/EUKI9nOqhRQaGorBYKCoqMhufVFRUasJa1fss93k/mxCuBWnApLRaGTcuHHk5ORo6ywWCzk5OUyaNKlDFbgR+2y3NgbYypxIQnQ/p7tsaWlpLFiwgPHjxzNx4kTWrVtHVVUVqampAMyfP59+/fqRlZUFqEnrY8eOac8vXLjA4cOHCQgIYODAge3a5w3jaPiIrwywFcJVnA5Is2fP5vLly2RkZFBYWMiYMWPIzs7WktIFBQXo9c0Nr4sXLzJ27Fjt9dq1a1m7di133XUXubm57drnDWNtIVVdBnMDGLxtrtaWgCREd9MpPeAmZOXl5QQHB1NWVkZQUFD7N1QUeLmverX20z9ASDRL3/2evx04z/PJQ3ji1wNuXKWF8CDt/Y567lg2AJ3umkG2ckttIVzHswMSXHOPtuaktgQkIbqbBKQWie3mKUjkLJsQ3U0CUotT/zJJmxCuIwGptQG2ctpfiG4nAanF8JHmpLZ02YTobhKQWraQJKkthMtIQNKmIFFzSMGS1BbCZSQgWVtIdeVQV6m1kGoazNQ3WlxYMSE8jwQkUyAYA9TnlUUE+DSPppFumxDdSwIS2NzF9iKGpjmRQBLbQnQ3CUjgILEtI/6FcAUJSHDtxZG+MieSEK4gAQmuHT4iU5AI4RISkMBBC0m6bEK4ggQksElqqwEpUFpIQriEBCRofYCtXBwpRLeSgAT2Z9kUxSapLS0kIbqTBCRoDkjmOqj5xSapLS0kIbqTBCQALxP49VGfV1ySpLYQLiIBycomjySn/YVwDQlIVjZn2rSzbJLUFqJbSUCysklsW7tsktQWontJQLLS5kW6KEltIVxEApKVXQtJDUiVdY00mmVOJCG6iwQkK5ukdqDNnEiVddJKEqK7SECysklqexv0+Hob1JeS2Bai20hAsrIGpKpiMDc2X4skiW0huo0EJCv/UNAZQLFA1WW5FkkIF5CAZKU32CS2L8ottYVwgQ4FpA0bNhAbG4uPjw8JCQns37+/zfLvvvsuQ4YMwcfHh5EjR/LRRx/Zvb9w4UJ0Op3dkpyc3JGqdY7tmTa5pbYQ3c7pgLRr1y7S0tLIzMzk4MGDjB49mqSkJIqLix2W//rrr5k7dy6PPvoohw4dIiUlhZSUFH744Qe7csnJyVy6dElbduzY0bEj6gybyf6br9aWgCREd3E6IL366qssWrSI1NRUhg0bxqZNm/Dz8+PNN990WP7Pf/4zycnJPPfccwwdOpSXX36Z2267jfXr19uVM5lMREREaEuvXr1arUNdXR3l5eV2S5fQTv0X2iS1pcsmRHdxKiDV19dz4MABEhMTm3eg15OYmEheXp7DbfLy8uzKAyQlJV1TPjc3l7CwMAYPHswTTzxBaWlpq/XIysoiODhYW6Kjo505jNbZddlkTiQhuptTAamkpASz2Ux4eLjd+vDwcAoLCx1uU1hYeN3yycnJbNu2jZycHFatWsUXX3zB9OnTMZvNDveZnp5OWVmZtpw7d86Zw2id7fARSWoL0e28rl/kxpszZ472fOTIkYwaNYoBAwaQm5vL1KlTrylvMpkwmUxdXxEHLSRJagvRfZxqIYWGhmIwGCgqKrJbX1RUREREhMNtIiIinCoPEBcXR2hoKKdOnXKmep0X2NRCKr+oDR+RpLYQ3cepgGQ0Ghk3bhw5OTnaOovFQk5ODpMmTXK4zaRJk+zKA3z66aetlgc4f/48paWlREZGOlO9zrO2kGqvEuKtdhclqS1E93H6LFtaWhqbN2/mrbfe4vjx4zzxxBNUVVWRmpoKwPz580lPT9fK/9u//RvZ2dn8+7//Oz/99BMrVqzgu+++Y8mSJQBUVlby3HPP8c0333DmzBlycnJ48MEHGThwIElJSV10mO3kEwxevgD0tlwBJKktRHdyOoc0e/ZsLl++TEZGBoWFhYwZM4bs7GwtcV1QUIBe3xznJk+ezPbt23nxxRf53e9+x6BBg3j//fcZMWIEAAaDgSNHjvDWW29x9epVoqKimDZtGi+//PKNyRO1RadT72J75WdCzCWAdNmE6E46RVEUV1eis8rLywkODqasrIygoKDO7WzLvXD2K8rv+yuj/h6ATgen/3Qver2uayorhAdq73dUxrK11JRH8q1TrzxXFKislzySEN1BAlJLTVdre1cVYvRSfzzSbROie0hAasl2+EjTtUgllfUurJAQnkMCUktBzVPZjokOBuBvB7roSnAhRJskILVkM7f2Y1PiAHj3u/OUVNa5sFJCeAYJSC3ZDB9JiO3F6OgQ6hotbPv6jEurJYQnkIDUkrWF1FCNrq6cx+9UW0lv5Z2lSu5AIsQNJQGpJW9f8AlRn1cUMm14BLF9/CiraeCd7ySXJMSNJAHJEZs8kkGvY1FTK+mNf+TLjSOFuIEkIDlic6YNYNZtt9DH38iFqzV8ePSSCysmRM8mAcmRQPuA5ONtYOHkWAD+8sXP9IDRNkK4JQlIjtjcxdbqkUkx+HobOHapnL2nSlxUMSF6NglIjmin/psDUoifkTkT1bm7//LFz66olRA9ngQkR2yGj9h69I7+GPQ69p4q4YcLZS6omBA9mwQkR1okta1u6eXHjFHqe3/5UlpJQnQ1CUiO2LaQLPan+RffOQCAj45e4tyV6u6umRA9mgQkR/zDQKcHxQzV9gnsYVFBTBkUitmi8L/35ruogkL0TBKQHDF4qUEJoPziNW8/fpfaStr5bQFXqmRqEiG6igSk1tgMsm1p8oA+jOgXRG2Dhf/IO9vNFROi55KA1Bqbu9i2pNPp+OemXNJbeWeoqXd8h123VnkZjv0XlJ52dU2E0LjFnWvdUhstJIDpIyKI7u3LuSs1vLf/FPP0n8KFA9BvHAy4G8KGqXcxcSclJ+GnD+HER3BuP6CAwQhTM+FX/wJ6+f8kXEsCUmsCHZ/6t/Iy6Fl8RwyHP/griTn/CkpT8vvH99THgAiI+zUMuEd9DAy/4VW+hsUM57+DEx/CTx9B6Un794NvhbIC+OT3cPITSNkIwf26v55CNJGA1BoHw0c0igKnPmPeoQweMR4DBWp8I/Ad/z/g0mE48xVUFsKRneoCED6iOUDFTFanObkRGmrg51y1JfT/sqHqcvN7em/oPwUG36suQVFwYCt8/DvI/wI2ToYZf4bhKTembkJchwSk1rRytTYXDsKnGXDmH+iBWkMA/6t2Bt/1/i1/u+cedDodNNTCuX3w8x44/Tlc+h6KflCXvPVgMEHMJIi9A2KnQNRt4GXsWD0VBYqPq5/z8x41GDbWNL9vCoZBv4Eh98LARPXuvLbGp6p1eO8xuHgI3l0AJ+dB8krw6cA97qqvwPH/C3UV0O82iBwDRr+OHZvwOHKjyNYU/ai2GPxCYdlpuPIz5Lzc3CUzGGHiYq7c9hSTXztEbYOF7YsSmDwg9Np9VZWorZaf98DpPVB+wf59L1+IngAxd0Ds7dBvPHj7tF63ymJ1f6c/V/dX2SJoBt2iBqDB90LM7e0LduYGyF0Je18FxQIhMfBPm+HWhOtvW18N/++/4ci7cOozsNjcNkpngPDhcMt4uGWCemx9Bkq+ysO09zsqAak11VdgdX/1+YRFatfG0gDoYNRsuOf3EHIrABn/+QPb8s5yV3xf3vqfE9ver6KoyeWfc+HsXrVF0+LiSwwm9csbe7saUCJHwcXDzQGo6Kh9eS9ftWzc3Z1PqJ/Ng92L4WqBenHolKVw1zIweNuXMzeq3byj76otovrK5vciRqoB7fx31wZLUFtp/capwemWCRA6yPrDUX8+iqX5EevrpnV6A/j2Br/e4NXNt1oXHSYBqbMUBf4YDmabu40MTITEFeoXzkZBaTW/XrsHiwLDIoOI6+tPXN8ABvT1Z0DfAPqH+uNvaqV3rChw+URzcDr7FVQWXb9+EaPUfNSAuyH6V223qJxVWwb//Tx8v0N93W+c2lrqHad2646+Cz/83b6eIbfCyN/CyIchbEjzsZVfUAPT+W/Vs5AXD0FjbdfU0xioBia/Pi2WpnW+vcDbT/3ZePk2PTYt3r7Nzw1Nv5vGOrU1W12q/pOosj6W2DyWQs0vagvZFASmQLVrawps8dq6BAA6MNerrVBzvbpYGpuf265vrFPzgI21Dh5r1e64tYxiVvOCeq+mxaA+GrztX+u9W7zX9Fp77qX+DBw+t27bxuvQQdArts1flQSkrrDxdjXvEzkGfvMHiLur1aLp7x1hx/7W59yOCPJhQJg/caEBxPX1J6aPH94GPXqdDp0O9DqduqDgU3GGoMJvCCzah/+lfRirC2nwC6ei352UR91BWeTtNPj0waKARVGwKAooYFFAQaGXn5GwIBN9/E0Y9J249OCH9+CDp9UA5e2vDjouPdX8vm8vGD5TbTFGJ7SvVWZuULvDF75rDlRlF9RtdXp1QWfzWmezTq+2Umt+aWo9dRG9lxpgGmRsYodM+yNMfqrNIhKQusIvZ+BKPvS/67o5D0VROFNazeniSn4uqeTny1Wcvqw+lnZqeIlCMFWU4Q84F1wMeh19A0yEB5kIC/IhPMhEeKAP4UE+hAWZCAtUW1WVdY1U1jVQUduoPm96rKhtxFB5gd8W/IkhtYcBaNCbOB/2ayoGzcRvWBK3hAbj423oxPF1gMUCtVfVbnV1qYPlCtRcUR8ba5paFrU2rYxa+5avLb1XUysrFPybHv36gL/No28vNbDWlUNtuZrAr6tQX9dVqAHcdh06NeAZvJsebZ/brvO6tgVn92iyaen5qi0VS2PzYm60f20xqwHc0qjW1/ra3NC03tz8XNu2wb5sy221z7F5b/JTMOrhNn9lEpDcSFl1A6dLKpuCVRU/X67k/C81mC2KmipB0Vo7ik2rx2JRA51FAb1OvUJcr29uTTW3rECHTmugXKmqp6SyDksX/WZ1WJip34sFPZ9axlGF/SULEUE+RPf2Jbq3H7c2LRHBPoT4Ggnx8ybEzxtfb4N6BtJdWCzNQaqxVu0u+QSrd5xxp3r2EDc0IG3YsIE1a9ZQWFjI6NGjef3115k4sfVk7rvvvsvy5cs5c+YMgwYNYtWqVdx7773a+4qikJmZyebNm7l69Sq33347GzduZNCgQa3u05a7ByRXaDRbKK2qp6i8lqLyOorKaym2Pq9QHy9X1KLX6Qjw8SLQ5EWAjxcBJi/8TbavvZseDVTUNlJQWk3BlWrO/VJDQWkVVe0cNmP00hPiqwanEF8jwX7e2utgX+/mz23xmdbnft4G9J3pfgqXau931OnrkHbt2kVaWhqbNm0iISGBdevWkZSUxIkTJwgLC7um/Ndff83cuXPJysri/vvvZ/v27aSkpHDw4EFGjBgBwOrVq3nttdd466236N+/P8uXLycpKYljx47h49OFyVoP4mXQEx6kds9uFEVR+KW6gYIrTUGqaTlbWs3lyjquVjdQVlNPg1mhvtFCcUUdxRUduyW5Tgf+Ri/8jAaMXnqMBj3eBj3eXjr10WBd1/TaS4+3XmfXKtNd80RtWVoZrK1PvQ5DU8tTr1dbo4amR71O7Qo376v5ufWjbFurOuwbXK21Eq2rrZ9hbQWrn2tTj6aWsUGvPtp9Vou6WN9rWT9a1M+2Xnbb62x+Pjr7srbH1T/Un1t6dc21Zk63kBISEpgwYQLr168HwGKxEB0dzVNPPcULL7xwTfnZs2dTVVXFBx98oK371a9+xZgxY9i0aROKohAVFcWzzz7L0qVLASgrKyM8PJytW7cyZ86ca/ZZV1dHXV3zH3Z5eTnR0dHSQnJDiqJQXW/mak0DV6vrKatu4JfqBq7W1DcFrAbKqhuorFdzV1V1zfmryqbn5q7qe4ob4sX7hvLYlLg2y9yQFlJ9fT0HDhwgPT1dW6fX60lMTCQvL8/hNnl5eaSlpdmtS0pK4v333wcgPz+fwsJCEhMTtfeDg4NJSEggLy/PYUDKysripZdecqbqwkV0Oh3+Td2xfiHOD5dRFIW6RgsVNsGq0aLQYLbQ0Gih3myhwdz02myhvtH+tbqPpn2h2OzXuq75tZq3UzA35e0sFjWXZ25ab1HA3LTO0X5t99n8b97BZ15nOy2HaFsHi2J3VtWiqD8baw5SUez323TiVft4289qPmblmp+B9lpR7I7D9jNabt/bv4OjDBxwKiCVlJRgNpsJD7cfKBoeHs5PP/3kcJvCwkKH5QsLC7X3retaK9NSenq6XZCztpBEz6PT6fDxNuDjbaBvoFwI2dPdlGPZTCYTJpP8cQrR0zg1oCg0NBSDwUBRkf2VxEVFRURERDjcJiIios3y1kdn9imE6JmcCkhGo5Fx48aRk5OjrbNYLOTk5DBp0iSH20yaNMmuPMCnn36qle/fvz8RERF2ZcrLy9m3b1+r+xRC9FCKk3bu3KmYTCZl69atyrFjx5TFixcrISEhSmFhoaIoivLII48oL7zwglb+q6++Ury8vJS1a9cqx48fVzIzMxVvb2/l6NGjWpmVK1cqISEhyn/+538qR44cUR588EGlf//+Sk1NTbvqVFZWpgBKWVmZs4cjhOgG7f2OOh2QFEVRXn/9deXWW29VjEajMnHiROWbb77R3rvrrruUBQsW2JV/5513lPj4eMVoNCrDhw9XPvzwQ7v3LRaLsnz5ciU8PFwxmUzK1KlTlRMnTrS7PhKQhHBv7f2OytARIcQN197vqMySJYRwGzflaf+WrI288vJyF9dECOGI9bt5vQ5ZjwhIFRUVAHJxpBBurqKiguDg4Fbf7xE5JIvFwsWLFwkMDLzuFBfWq7rPnTvXI/NNPfn4evKxQc8+PkVRqKioICoqCn0bc4v1iBaSXq/nlltucWqboKCgHvdLt9WTj68nHxv03ONrq2VkJUltIYTbkIAkhHAbHheQTCYTmZmZPXZwbk8+vp58bNDzj689ekRSWwjRM3hcC0kI4b4kIAkh3IYEJCGE25CAJIRwGxKQhBBuw+MC0oYNG4iNjcXHx4eEhAT279/v6ip12ooVK9R7ZdksQ4YMcXW1OuzLL79kxowZREVFodPptDvUWCmKQkZGBpGRkfj6+pKYmMjJkyddU9kOuN7xLVy48JrfZ3Jysmsq2808KiBZb3KZmZnJwYMHGT16NElJSRQXF7u6ap02fPhwLl26pC179+51dZU6rKqqitGjR7NhwwaH71tvLLpp0yb27duHv78/SUlJ1NbWdnNNO+Z6xweQnJxs9/vcsWNHN9bQhW7oNHFuZuLEicqTTz6pvTabzUpUVJSSlZXlwlp1XmZmpjJ69GhXV+OGAJTdu3drry0WixIREaGsWbNGW3f16lXFZDIpO3bscEENO6fl8SmKoixYsEB58MEHXVIfV/OYFpL1Jpe2N6S83k0ubyYnT54kKiqKuLg45s2bR0FBgaurdENc78aiPUVubi5hYWEMHjyYJ554gtLSUldXqVt4TEBq6yaXrd2Q8maRkJDA1q1byc7OZuPGjeTn5zNlyhRtnqiepCM3Fr3ZJCcns23bNnJycli1ahVffPEF06dPx2w2u7pqN1yPmH7E002fPl17PmrUKBISEoiJieGdd97h0UcfdWHNREfY3j5+5MiRjBo1igEDBpCbm8vUqVNdWLMbz2NaSB25yeXNKiQkhPj4eE6dOuXqqnQ5T7yxaFxcHKGhoT3y99mSxwSkjtzk8mZVWVnJ6dOniYyMdHVVupwn3lj0/PnzlJaW9sjfZ0se1WVLS0tjwYIFjB8/nokTJ7Ju3TqqqqpITU11ddU6ZenSpcyYMYOYmBguXrxIZmYmBoOBuXPnurpqHVJZWWnXGsjPz+fw4cP07t2bW2+9laeffpo//vGPDBo0iP79+7N8+XKioqJISUlxXaWd0Nbx9e7dm5deeolZs2YRERHB6dOnWbZsGQMHDiQpKcmFte4mrj7N193ausnlzWr27NlKZGSkYjQalX79+imzZ89WTp065epqddiePXsU4JrFegPSzt5Y1NXaOr7q6mpl2rRpSt++fRVvb28lJiZGWbRokXZn6J5O5kMSQrgNj8khCSHcnwQkIYTbkIAkhHAbEpCEEG5DApIQwm1IQBJCuA0JSEIItyEBSQjhNiQgCSHchgQkIYTbkIAkhHAb/x8s9hILHIr0VgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:04:21.209201Z", + "iopub.status.busy": "2024-02-29T06:04:21.208546Z", + "iopub.status.idle": "2024-02-29T06:06:18.478358Z", + "shell.execute_reply": "2024-02-29T06:06:18.477558Z" + }, + "papermill": { + "duration": 117.293264, + "end_time": "2024-02-29T06:06:18.481058", + "exception": false, + "start_time": "2024-02-29T06:04:21.187794", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:06:18.522517Z", + "iopub.status.busy": "2024-02-29T06:06:18.522198Z", + "iopub.status.idle": "2024-02-29T06:06:18.542555Z", + "shell.execute_reply": "2024-02-29T06:06:18.541649Z" + }, + "papermill": { + "duration": 0.043342, + "end_time": "2024-02-29T06:06:18.544547", + "exception": false, + "start_time": "2024-02-29T06:06:18.501205", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.0581950.0695380.0022534.3704120.1243451.0819270.3701890.0000132.5011970.0348450.0641180.0474650.0681190.0161236.871609
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.058195 0.069538 0.002253 4.370412 0.124345 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 1.081927 0.370189 0.000013 2.501197 0.034845 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.064118 0.047465 0.068119 0.016123 6.871609 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:06:18.583646Z", + "iopub.status.busy": "2024-02-29T06:06:18.583344Z", + "iopub.status.idle": "2024-02-29T06:06:19.170819Z", + "shell.execute_reply": "2024-02-29T06:06:19.169896Z" + }, + "papermill": { + "duration": 0.609708, + "end_time": "2024-02-29T06:06:19.172988", + "exception": false, + "start_time": "2024-02-29T06:06:18.563280", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:06:19.215212Z", + "iopub.status.busy": "2024-02-29T06:06:19.214407Z", + "iopub.status.idle": "2024-02-29T06:08:22.722502Z", + "shell.execute_reply": "2024-02-29T06:08:22.721519Z" + }, + "papermill": { + "duration": 123.532066, + "end_time": "2024-02-29T06:08:22.725180", + "exception": false, + "start_time": "2024-02-29T06:06:19.193114", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:08:22.768193Z", + "iopub.status.busy": "2024-02-29T06:08:22.767856Z", + "iopub.status.idle": "2024-02-29T06:08:22.785840Z", + "shell.execute_reply": "2024-02-29T06:08:22.784940Z" + }, + "papermill": { + "duration": 0.042177, + "end_time": "2024-02-29T06:08:22.788153", + "exception": false, + "start_time": "2024-02-29T06:08:22.745976", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:08:22.828457Z", + "iopub.status.busy": "2024-02-29T06:08:22.828130Z", + "iopub.status.idle": "2024-02-29T06:08:22.833762Z", + "shell.execute_reply": "2024-02-29T06:08:22.832865Z" + }, + "papermill": { + "duration": 0.028416, + "end_time": "2024-02-29T06:08:22.835836", + "exception": false, + "start_time": "2024-02-29T06:08:22.807420", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.5513597200986436}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:08:22.877647Z", + "iopub.status.busy": "2024-02-29T06:08:22.877300Z", + "iopub.status.idle": "2024-02-29T06:08:23.262433Z", + "shell.execute_reply": "2024-02-29T06:08:23.261355Z" + }, + "papermill": { + "duration": 0.409251, + "end_time": "2024-02-29T06:08:23.264991", + "exception": false, + "start_time": "2024-02-29T06:08:22.855740", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDgklEQVR4nO2deXxTddb/Pzd72izd95ZWdmQrqwjIIuOCIowziiMqHUScRxhHeXheTmdeI6Kj4Dzq4DgOjo5S/bkwowPq44aMCrghyCKrZSu0dF+TptmT7++Pm6RNm7TNepP2vF+vvJLce3Nzbpt8cr7ne875cowxBoIgCAERCW0AQRAECRFBEIJDQkQQhOCQEBEEITgkRARBCA4JEUEQgkNCRBCE4JAQEQQhOCREBEEIDgkRQRCCQ0JEhJVvvvkGjzzyCNra2oQ2hYgjSIiIsPLNN99gw4YNJEREQJAQEQQhOCRERNh45JFH8D//8z8AgKKiInAcB47joFKpMG/evB7HO51O5Obm4uc//7ln21NPPYUrr7wSqampUCqVmDx5Mt555x2f7/f6669j8uTJUCqVSElJwW233YaqqqrIXBwRUThqA0KEi6NHj2LTpk1466238Oc//xlpaWkAgHPnzuHRRx9FdXU1srKyPMfv3bsXc+bMwdtvv+0Ro/z8fNx0000YM2YMrFYrtm3bhv379+ODDz7ADTfc4Hnt448/jj/84Q+49dZbMWfOHDQ2NuK5556DSqXC4cOHkZSUFNVrJ0KEEUQY+d///V8GgFVUVHi2lZeXMwDsueee8zr2vvvuYyqVihmNRs+2ro8ZY8xqtbKxY8ey+fPne7ZduHCBicVi9vjjj3sde+zYMSaRSHpsJ2IfGpoREWfEiBGYOHEi/vnPf3q2ORwOvPPOO1i0aBGUSqVne9fHra2t0Ol0mD17Ng4dOuTZvn37djidTtx6661oamry3LKysjB8+HB88cUX0bkwImxIhDaAGBwsXboUv/vd71BdXY3c3Fzs3r0bDQ0NWLp0qddxH3zwAf74xz/iyJEjsFgsnu0cx3kenzlzBowxDB8+3Od7SaXSyFwEETFIiIiosHTpUpSWluLtt9/GAw88gH/961/QarW47rrrPMd8+eWXuOmmm3DVVVfhb3/7G7KzsyGVSrF161a8+eabnuOcTic4jsPHH38MsVjc471UKlVUrokIHyRERFjp6rl0paioCNOmTcM///lPrFmzBtu3b8eSJUsgl8s9x/z73/+GQqHAzp07vbZv3brV61xDhw4FYwxFRUUYMWJEZC6EiCoUIyLCSmJiIgD4TGhcunQp9u3bh1deeQVNTU09hmVisRgcx8HhcHi2XbhwAe+++67XcTfffDPEYjE2bNgA1m3SlzGG5ubm8FwMETVo+p4IKwcOHMC0adOwcOFC3HbbbZBKpVi0aBESExNx6dIlFBQUQKVSQSqVoq6uziue8/nnn+Pqq6/G7Nmzcfvtt6OhoQHPP/88srKycPToUS/R2bRpE0pLS3HllVdiyZIlUKvVqKiowI4dO7Bq1SqsW7dOiMsngkXAGTtigPLYY4+x3NxcJhKJekzlz5w5kwFgK1eu9Pnal19+mQ0fPpzJ5XI2atQotnXrVrZ+/Xrm66P673//m82aNYslJiayxMRENmrUKLZ69WpWXl4eqUsjIgR5RARBCA7FiAiCEBwSIoIgBIeEiCAIwSEhIghCcEiICIIQHBIigiAEJ65LPJxOJ2pqaqBWq/2WFhAEIRyMMbS3tyMnJwcikX+/J66FqKamBvn5+UKbQRBEH1RVVSEvL8/v/rgWIrVaDYC/SI1GI7A1BEF0R6/XIz8/3/Nd9UdcC5F7OKbRaEiICCKG6St0QsFqgiAEh4SIIAjBISEiCEJw4jpG1B8YY7Db7V7Ntoj+IRaLIZFIKDWCiDgDWoisVitqa2thNBqFNiVuSUhIQHZ2NmQymdCmEAOYAStETqcTFRUVEIvFyMnJgUwmo1/2AGCMwWq1orGxERUVFRg+fHivCWkEEQoDVoisViucTify8/ORkJAgtDlxiVKphFQqxcWLF2G1WqFQKIQ2KWI4nQyHKltxrtGAdLUcM4elQS7puUIIERkE/4mrrq7GHXfc4VnrfNy4cfj+++/Ddn76FQ+NwfD3Y4zhP6fq8eWZJtS0mfFDlQ6fHK/r0ZifiByCekStra2YOXMm5s2bh48//hjp6ek4c+YMkpOThTSLGGTU6c04WasHxwGThyTjcGUbzjd2oObsD8i1VQKaXCB3EkBD+4ghqBA9+eSTyM/P91q3qqioSECLiMFItlaJZdOHoLnDglFZGjicDLUnvkL79weALA1QfwIwtwHDrhba1AGLoH73+++/jylTpuCWW25BRkYGiouL8dJLL/k93mKxQK/Xe92I0CgsLMTmzZuFNkNw0tVyjMriy4QmpDhQ0PYdrHYnnCmX8QdcOgAYGgW0cGAjqBCdP38eW7ZswfDhw7Fz507813/9F+6//368+uqrPo/fuHEjtFqt50aV90Qo6Ew2NBssPbYnN3yHyXlqjB8/CaLxtwJpwwHGgOrwxS4JbwQVIqfTiUmTJuGJJ55AcXExVq1ahXvuuQcvvPCCz+NLS0uh0+k8t6qqqihbHJtYrVahTYhLdpc34PV9lTherevcaGoFGn+EQioGLpvLx4XypvD7Gk4CDrsgtg50BBWi7OxsjBkzxmvb6NGjUVlZ6fN4uVzuqbQPpeLeanf6vdkdzn4fa+vHscEwd+5crFmzBmvWrIFWq0VaWhr+8Ic/eGZxCgsL8dhjj+Guu+6CRqPBqlWrAABfffUVZs+eDaVSifz8fNx///3o6OjwnLehoQGLFi2CUqlEUVER3njjjaDsGwhcaOrA+Ub+b5Ot7ZKWUHuU935SigB1Jv83TxoCyFWA3QrofH82idAQNFg9c+ZMlJeXe207ffo0hgwZEtH3ff6Ls373FaUlYklxruf5i3vPwebwPY2bl6zELVM6h4evfF0Bk9W7lOTBn4wIysZXX30Vd999N/bv34/vv/8eq1atQkFBAe655x4AwFNPPYWHH34Y69evBwCcO3cO1113Hf74xz/ilVdeQWNjo0fM3JMBJSUlqKmpwRdffAGpVIr7778fDQ0NQdkXzzicDHtO8/GeiQVJSFXJ+R2M8YFpAPaMsfjgcDVqdWasmFUIecpQoPYHoPkc4I4bEWFDUCF68MEHceWVV+KJJ57Arbfeiv379+PFF1/Eiy++KKRZMUF+fj7+/Oc/g+M4jBw5EseOHcOf//xnjxDNnz8f//3f/+05fuXKlVi2bBkeeOABAMDw4cPxl7/8BXPmzMGWLVtQWVmJjz/+GPv378fUqVMBAC+//DJGjx4d9WsTmm/PNaOlw4oEmRjTi1I6d+iqALMOkMggyRiJprNVMNscaDJYkZtSxAtRG3lEkUBQIZo6dSp27NiB0tJSPProoygqKsLmzZuxbNmyiL7v6nnD/O4TdUsVWXXVUL/Hdk8rWTEzfKkHV1xxhVdJyowZM/D00097inenTJnidfwPP/yAo0ePeg23GGOeUpfTp09DIpFg8uTJnv2jRo1CUlJS2GyOB843GvD9xRYAwNWjM/hYkJuGU/x9+ihALEWaSo52sx1N7Rbkpru85I5Gfogmodq7cCJ4iceNN96IG2+8MarvKZP0PzQWqWNDJTEx0eu5wWDAvffei/vvv7/HsQUFBTh9+nS0TItZrHYnPj5eB8aA8XlaDMvo0r6UMaDpDP84fRR/p5ajoqkDTQYLkJ8JKDSAWQ+01wDJhdG/gAGM4EJE+Oa7777zer5v3z4MHz4cYrHv+qdJkybh5MmTGDbMt7c3atQo2O12HDx40DM0Ky8vR1tbW1jtjmVkEhGmFaWgps2EuSMzvHca6gFLOyCW8sFpAGmu2FGTe4pfk8MLkZ6EKNwM/EKiOKWyshJr165FeXk53nrrLTz33HP4zW9+4/f4hx56CN988w3WrFmDI0eO4MyZM3jvvfewZs0aAMDIkSNx3XXX4d5778V3332HgwcPYuXKlVAqldG6JEFoNljQ2tGZ3jC5IBk3js+BuPsY3O0NpRQBYv73OTlRCgBoNdr4feps/t5QH1GbByMkRDHKXXfdBZPJhGnTpmH16tX4zW9+45mm98X48eOxZ88enD59GrNnz0ZxcTEefvhh5OTkeI7ZunUrcnJyMGfOHNx8881YtWoVMjIy/J4z3qnXm7HtQBX+72gNLHY+tiYScT1FCACaXUKUOtyzKUnJx4FMVgfMNgeQmM7v6GiKqN2DERqaxShSqRSbN2/Gli1beuy7cOGCz9dMnToVn376qd9zZmVl4YMPPvDaduedd4ZkZ6xisjrw3pFqWO1OKCRiOHtL6TLrgfZ6fvYhtXNyQiYRIUMjh0wsgsXuhMItRMYWPrFRTF+fcEF/SWJA8uWZRnRYHEhVybC4OKf33kIt5/h7dTYg854EWDa9S04bkwASOWC3AMZmQJ0ZAcsHJzQ0IwYcbUYrTtbyBdELRmf23eCs9QJ/31eiIscBKtdQtoMKYMMJeUQxyO7du4U2Ia45XNkGxvgs+ZykPoLxTmcXIfKfB8YY4/O6ElKBtirA1BI+gwkSImJg4XAy/FjXDgAoLkjq+wWGOsBm5odc6pweuy82d2DXyXokJcjw88l5gNLVtM/UGkarCRIiYkBhsTtQlJaAOp0Z+cn96FXeUsHfJw8BfLTFlYhFaDfbIXJnubuFyEgeUTghISIGFAkyCa4bm905lOqLVrcQ+R6WqRX8V8RgsfPnVLpq00ytfDY2tY8NCxSsJgYk/RIhuwXQVfOP/cSHVDIJOI4f8nVYHYAyqfO1NlN4jCVIiIiBg9nmQLPB0v/VN9oqAebkxcU95OqGSMRBJXd5RWY7XwIid9WoUZwobJAQEQOGc40GvPbtRbx3pKZ/L3DPlvkZlrlxD8/aza5SjwT38IziROGChIgYMFxq5YdK6Wp5/17gDlT3Mm0PACo5X3OmN7vaxCqS+HuzzvcLiIChYDUxYKjXmwF0a/3qD7OOz47mRJ5qe39kaOTosNqRIHMlRipcLYrNtIpMuCAhIgYEVrsTLa4q+0xNP4TI7Q1psgFp78dPLUzB1MIunRzlLiGykBCFi8E1NGOM764X7VsASxe/9tprSE1NhcXivczNkiVLBmyBajhoMljAGKCSS5Ao78fvax/T9r1CHlHYGVwekcMGfPl09N939n/3u7XoLbfcgvvvvx/vv/8+brnlFgD86hsffvhhr5X1g52Gdl64MzT9iA/1s6yjO57cJI9HpKNcojAxuDyiOECpVOL222/3Wob79ddfR0FBAebOnSucYTFOgys+1K9AtaesQ+azrKM7erMNL+09jy17XFX6biFy2CmXKEwMLo9ILOW9EyHeNwDuueceTJ06FdXV1cjNzUVZWRlKSkr6l6Q3SBmZpYZKIUFRWmLfB7u9oSTfZR3dkUtEMFj4GTOr3QmZRMK3C7F28EFvWT9KSYheGVxCxHFxsfpCcXExJkyYgNdeew3XXHMNTpw4gQ8//FBos2KaIamJGJLaDxEC+j1t70YmFkEq5mBzMBitdsgkMj5OZO1wBayzgzOa8DC4hCiOWLlyJTZv3ozq6mosWLAA+fn5fb+I6Bu7FdC7yjr6GajmOA4JMgl0Jhs6rA4kJcA1PKulgHWYoBhRjHL77bfj0qVLeOmll7BixQqhzYlp9GYbLjR1dGY+90ZbJeB01YwlpPR5uJtEOZ9DZLS4kxq7BKyJkBFUiB555BFwHOd1GzVqlJAmxQxarRY/+9nPoFKpsGTJEqHNiWkqGjuw43A1Pv+xH8tnBzltnyDjBw8d7iXF5Vr+njyisCD40Ozyyy/Hf/7zH89ziURwk2KG6upqLFu2DHJ5P0sWBinuRMaUxH7E/wKMD7np4RG5C1+thoDOQ/hG8G+9RCJBVlaW0GbEFK2trdi9ezd2796Nv/3tb0KbE/O4hSg5oQ8h8pR1cH2WdXQnTSVHXrISKlcBLOQq/t5CQhQOBBeiM2fOICcnBwqFAjNmzMDGjRtRUFDg81iLxeKVcazXD0y3uLi4GK2trXjyyScxcuRIoc2JeVqN/fSI3NP2mpw+yzq6Mz4vCePzkjo3yFxCZDVQUmMYEFSIpk+fjrKyMowcORK1tbXYsGEDZs+ejePHj0OtVvc4fuPGjdiwYYMAlkYXf+uWET1xOJknx0er7CNfy9MWtjD0N3YLkdPBJzVSLlFICBqsvv7663HLLbdg/PjxuPbaa/HRRx+hra0N//rXv3weX1paCp1O57lVVVVF2WIi1mg328AYIBVzndXxvmAMaLvIPw5BiDxN18SSTq/K2hH0+QgewYdmXUlKSsKIESNw9uxZn/vlcnnAgdt+d+sjfBLrfz+diZ+y1yilvWeedzQBViMvIJrcgN+n3WzDW/srYXcy3Dd3GL9RpuJLRaztANKDsJ5wE1N5RAaDAefOnUN2duiZqlIp76YbjcaQzzWYcf/93H/PWCNVJcc1l2d6t+nwhTs+pC0ARH0suOgDmUSEDosDFpsTNodr/Wr3zBkFrENGUI9o3bp1WLRoEYYMGYKamhqsX78eYrEYv/jFL0I+t1gsRlJSEhoa+NyShIQEqtUKAMYYjEYjGhoakJSUBLE48C9vNFDJJbg8R9v3gZ5hWWCzZW5kYhHEIg4OJ4PJ5oBULOoSsKahWagIKkSXLl3CL37xCzQ3NyM9PR2zZs3Cvn37kJ4eHjfXnRbgFiMicJKSkuI/vcLp7BSiAKft3XAcB6VUDIPFDrPVAY1Cyhe+ApRLFAYEFaJt27ZF9PwcxyE7OxsZGRmw2fqR/k94IZVKY9YTcnO2oR1SsQhZWoX/Ne4NdXyNmUQOqDKDfi+FjBcik82dXe0emrUHfU6CJ6aC1ZFCLBbH/BeKCI7PTjXAaHVg2fQCZGj8/I9b3d5QQb/afvhDKXVlV7vLPLrmEhEhEVPBaoIIBKvd6REFTW85RP1cNqgv3ELU6RFRjChcDAqPiBiY6F3V9nKpCAqpH2/IYQd0l/jHQQaq3aSr5TB2Xc3DHSOyUHZ1qJAQEXGL3pVD1GtGtb4acNp50UhIDen9phWlYFpRlzQBmStG5LQDdjMgVYZ0/sEMDc2IuKXdteChWtGLEHWdtg+3x9I1u5pyiUKChIiIW9w1Zurelg9qc5UBJfkupA4Gr2xzd8DaRnGiUKChGRG3uD0iT2uO7jjsgL6Gf6wNXYhqdSa8f6QGCXIJ7rzCFW+SuopdrZTBHwokRETcMqkgCQUpCf6XEGqvdcWHEgJqC+sPiUjUOXXvxl11byMhCgUSIiJuydAokNHb8tLu2TJtXljiQ0rXbJnZ5uxcbFFK2dXhgGJExMBF54oPhWFYBgAKCf91cTIGi91V+CqjoVk4ICEi4hKr3Ynj1TpcaOrw3arE6ewUoqTwLMUkEYsgc4mRyZNd7fKIaGgWEiRERFyiN9uw62Q9PjlR57urQkejq75MBiRmhO193YmTZrtLiDxDM5o1CwUSIiIuMbhnzPxN3bu9IU1eSPVl3VFI+XOZbd2HZiREoUDBaiIucecQ9SlE2rywvm+mWuFZghpA5/Q95RGFBAkREZe09+YRMdYlkTG8S3UvGNOtjYg7odFh7xwKEgFDQzMiLvF4RL6SGU2t/FBJJAbUOZE1RCzlSz0A8opCgISIiEsMFr7g1adH5M4fUmd3ikSk8MolIiEKFhIiIi4xeApefQiNp6wj8NU6+uJ4tQ5/33MOn56o69xIuUQhQzEiIi6ZNyoDepPdd3mHvpq/j9CwzGh1dDZHAzo9IhqaBQ0JERGX5CUnAMk+dtitfA4RwC8tHWY6p++7CJGniT55RMFCQzNiYNFey8+aydWAQhP207sb9HvyiAAqfA0DJERE3KEz2nC8WofqNlPPne21/H0EvCGgS2a1r6EZFb4GDQkREXdcajNi18l6fHe+uedOd3woiGWl+4N7aGaxOztr3ChYHTIkRETc0WHhvZHE7lP3jHXOmGlCX7bcF+6hmcPJYHO4hEhKQ7NQiRkh2rRpEziOwwMPPCC0KUSM0+GvvMOi53tHcyI+hygCSMUcMjUK5CUr4XC6PSJaVihUYmLW7MCBA/j73/+O8ePHC20KEQf4rTPTu+JDqnQ+4zkCcByH26d362/kCVabAKeDz+gmAkJwj8hgMGDZsmV46aWXkJzsaz6WILxxe0Q9hmYRjg/5RaLs7ABJw7OgEFyIVq9ejRtuuAELFizo81iLxQK9Xu91IwYf/j0iV3woQsMyv4hEnWuaUcA6KAQdmm3btg2HDh3CgQMH+nX8xo0bsWHDhghbRcQyjLEuweouQyCnA2h3lV1E2CP67FQ9zjYYMHNYGsbmavmN0gRehMgjCgrBPKKqqir85je/wRtvvAGFopcG6F0oLS2FTqfz3KqqqiJsJRFrMAb8tDgX116ehURZl99RQwO/YodEHpYVO3rD7mQ9yzyoZWxICOYRHTx4EA0NDZg0aZJnm8PhwN69e/HXv/4VFosFYrF30E8ul0Mu97N0DDEoEIk4FKQm9NzR7p62z434GvS+kxppaBYKggnR1VdfjWPHjnlt++Uvf4lRo0bhoYce6iFCBNErnvyhCPcfQudqHpauZR5U+BoSggmRWq3G2LFjvbYlJiYiNTW1x3aCcNOgN6Oh3YJ0tRyZXdc0i6YQdW+gD1B2dYgIPmtGEIFwrrEDu07W43i1rnOjzQQYW/jHUZgxk3dvoA9QdnWIxERCo5vdu3cLbQIR4/jMIXJ7QwkpnZ5JBFFIfMSIKFgdEuQREXFFh9VHDlEUh2UAkCAXI10tR0pil0b5UhqahUJMeUQE0ReG3jyiSDfKd5GhVuCOK4Z4b5RRsDoUgvKIzp8/H247CKJfdA7NXLOqjHWZuo+OEPnEPX1vt/JLCxEBEZQQDRs2DPPmzcPrr78Os9kcbpsIwicOVyIh0GVoZmwBbGZAJAFU4VtaOmAkCr7qHyCvKAiCEqJDhw5h/PjxWLt2LbKysnDvvfdi//794baNILzosNrBGCAWcVC6ptA93pA6K6pV72/tr8QLe86hpcPKb+A4msIPgaCEaOLEiXj22WdRU1ODV155BbW1tZg1axbGjh2LZ555Bo2NjeG2kyCglIrx88l5uH5sFjh39nSUA9VuTFYHTFZHt+xqmsIPlpBmzSQSCW6++Wa8/fbbePLJJ3H27FmsW7cO+fn5uOuuu1BbWxsuOwkCUrEI+SkJGJ6p7tzoaf0RXSHyWeZBU/hBE5IQff/997jvvvuQnZ2NZ555BuvWrcO5c+ewa9cu1NTUYPHixeGykyB64rABhsgtHdQbit6SGmloFjBBTd8/88wz2Lp1K8rLy7Fw4UK89tprWLhwIUQi/p9TVFSEsrIyFBYWhtNWYpBT2WyEzmRDdpICaSo53/aDOQG5CpCHf+mg3nD3rrb4KvOgYHXABCVEW7ZswYoVK1BSUoLsbN8p9RkZGXj55ZdDMo4gunKqTo+TNXrMGp7GC1HX+FCEK+67Qx5ReAlKiHbt2oWCggKPB+SGMYaqqioUFBRAJpNh+fLlYTGSIIAuOUTuPkRCtYaFn8JXClYHTVAxoqFDh6KpqanH9paWFhQVFYVsFEH4osfqHQLNmAGARiFFulru3ZzNs/Q0Dc0CJSiPyLOwXDcMBkO/uy0SRKC0d82qNusBSzufRKjKirot4/K0GJen9d5IHlHQBCREa9euBcAvqfLwww8jIaGz0tnhcOC7777DxIkTw2ogQQCAzeH0NCJLlEuA1gp+hyodkMh6eWUUkXURIsaiHreKZwISosOHDwPgPaJjx45BJuv8AMhkMkyYMAHr1q0Lr4UEgc5hmVTMQS4RCRof8ou7S6PDzqcWxIpAxgEBCdEXX3wBgG/p+uyzz0Kjie6UKTF46Vp1z3GcoPEhAGgzWvHu4WqIRBzumlHIbxRLAbGEFyJbBwlRAAQVI9q6dWu47SCIXklXy/HzyXn8Ms9RXDrIHyIRh1ajDWIRB8YYL44cx8eJHHp+Cl9JC4b2l34L0c0334yysjJoNBrcfPPNvR67ffv2kA0jiK7IJWLkp7hiMPpafukgqUKwL7u7S6PDyWBzMMgkrniQNIEPpFPAOiD6LURardZTaKjVavs4miAiiGdYFvmlg/whFXMQizg4nAxmuwMy18oeNIUfHP0Woq7DMRqaEdHmVK0edgfDkLQEaAQqdO0Kx3FQSEXosPAV+BqFlN9BU/hBEVRCo8lkgtHY+Ye+ePEiNm/ejE8//TRshhFEVw5ebMV/TtWj2WAVPFDtxlNv1rXMQ0ZCFAxBCdHixYvx2muvAQDa2towbdo0PP3001i8eDG2bNkSVgMJAgDaza6sapEFMLXyG6PUo9of7nozr8JX9xQ+1ZsFRNAdGmfPng0AeOedd5CVlYWLFy/itddew1/+8pewGkgQNofT0/dHY3W1/UhM44PVApKcIEO6Wg5R1zgVeURBEZQQGY1GqNV8c6pPP/0UN998M0QiEa644gpcvHix3+fZsmULxo8fD41GA41GgxkzZuDjjz8OxiRiAOP2hmQSEWQdrmZ7Ag/LAOCay7NwxxVDcFm6qnOjpwKfgtWBEHTz/HfffRdVVVXYuXMnrrnmGgBAQ0NDQEmOeXl52LRpEw4ePIjvv/8e8+fPx+LFi3HixIlgzCIGKAaXEKkVEnD6S/xGbZ6AFvUCBauDIighevjhh7Fu3ToUFhZi+vTpmDFjBgDeOyouLu73eRYtWoSFCxdi+PDhGDFiBB5//HGoVCrs27cvGLOIAYrebAMAqKXgc4gAQJsvnEG94Rmamfh6M6JfBJVZ/fOf/xyzZs1CbW0tJkyY4Nl+9dVX46c//WlQhjgcDrz99tvo6OjwCFt3LBYLLBaL57lerw/qvYj4wl3ekcZa+URGWUJMZC2frm/HN2ebkJecgAVjMvmN7mC10wHYLYLHseKFoFd6zcrKQlaWd/uFadOmBXyeY8eOYcaMGTCbzVCpVNixYwfGjBnj89iNGzdiw4YNQdlLxC/jcrXITVJCVX8AMIAflsVAZbvDydBqtEGtsHVuFEv4GjO7lR+ekRD1i6CEqKOjA5s2bcJnn32GhoYGOJ1Or/2BrAQ7cuRIHDlyBDqdDu+88w6WL1+OPXv2+BSj0tJSTysSgPeI8vNj1EUnwkaiXMK3/rjUwG/QFghrkAu5xD197/35hzSRFyJrB5CQIoBl8UdQQrRy5Urs2bMHd955J7KzszvXmAoCmUyGYcOGAQAmT56MAwcO4Nlnn8Xf//73HsfK5XLI5fKg34uIYxgDYixQ7XNJIYAfOppaKWAdAEEJ0ccff4wPP/wQM2fODLc9cDqdXnEgYnDjdDJ8e74ZKdBhpNUEkUQGqDKFNguAn77VAE3hB0FQQpScnIyUlNBdztLSUlx//fUoKChAe3s73nzzTezevRs7d+4M+dzEwMBgtWN/RQuyO05hlAZ8oasopOX4woZ7aGa1O+F0MohEXSrwAfKIAiCo/+hjjz2Ghx9+2KveLBgaGhpw1113YeTIkbj66qtx4MAB7Ny5Ez/5yU9COi8xcNAZ+UBwprOBDwHEyLAM6PSIGAOsDh/1ZlTm0W+C8oiefvppnDt3DpmZmSgsLIRUKvXaf+jQoX6dh9Y9I/pCZ7IBjCHVXg9ADCTFzuSEWMQhVSWDiONgd3bJGZLS0tOBEpQQLVmyJMxmEIRv9CYbFHYdVJwJECXFVo9qoLNNbFeo3ixgghKi9evXh9sOgvCJzmSD1lzNd0TU5vJ9oWMdClYHTNBRv7a2NvzjH/9AaWkpWlpaAPBDsurq6rAZR8QmjDGcbzR4rW/nb627UNGZbNBaaiCXioCkIRF5j7Ajo6FZoAQlREePHsWIESPw5JNP4qmnnkJbWxsAvld1aWlpOO0jYpAzDQa8d6QG355vBsC36XjvSA1O1oS35IYxhtYOKzTmGj4wnFwY1vOHg6/ONKHs6wqcqNF1bpQq+XubCeiW7Ev4JighWrt2LUpKSnDmzBmvlV0XLlyIvXv3hs04IvZwOhm+POO93PiJGj0qmjqw62Q9LjSFdzhy5zg5xmXKoFAoAXV2WM8dDkw2B1qNNk+HAACdQzPGALtJGMPijKCE6MCBA7j33nt7bM/NzUVdXV3IRhGxy8UWIx9AlooxtZDPJZuQp8XobA2cjOHDY7VoaDeH5b04joPKWIMkpQzi5CExkz/UFXcukblrmYdI3FljRlP4/SKo/6xcLvdZ+X769Gmkp6eHbBQRu5yq5f/vo7PVkIr5jw/HcfjJmEzkpyTAanfivcM1ntYdIdPmarSXHJvxIXcukaV7mQdN4QdEUEJ000034dFHH4XNxn/YOI5DZWUlHnroIfzsZz8Lq4FE7OB0Mlxs5r9YIzLVXvvEIg43js9GmkoGg8WO9w5X96zBCpDjVc24eO4U3wYkRgPV7r7V5u6FrzSFHxBBCdHTTz8Ng8GA9PR0mEwmzJkzB8OGDYNarcbjjz8ebhuJGKFWb4bZ5oBCKkaWpmd7C4VUjMXFuVDJJWgyWLHzRGjD9OoL5ahpaUc7UwCqjJDOFSn8Fr5KKbs6EILKI9Jqtdi1axe+/vpr/PDDDzAYDJg0aRIWLFgQbvuIGOJiMx+IHpKa0FlX1Q2NQorFxTn48GgtZgxNDen9nE1nAQCyjOEx0X/IF35bgXim8CmXqD8ELEROpxNlZWXYvn07Lly4AI7jUFRUhKysrM41wIkBycT8JGRqFEiQiXs9LkOtwPIZhV5iVdViRLZWAYm4f064wWKHQn8BHAB17shQzI4oSpkYGqUUKnm3v4l7Cp88on4RkBAxxnDTTTfho48+woQJEzBu3DgwxnDq1CmUlJRg+/btePfddyNkKiE0CTIJhnZdsaIXuopQvd6MHYerkaqS4YZx2UhKkPX5+qaGGijsesjlUsjSLgva5kiToVbg7llFPXdIySMKhICEqKysDHv37sVnn32GefPmee37/PPPsWTJErz22mu46667wmokEd9YbE7IJCI06C14c38lbpmcj3R17w3uDDWnAQDi5AJAEofN8KgCPyACCla/9dZb+N3vftdDhABg/vz5+O1vf4s33ngjbMYRscPF5g58c64JtbrAE/QKUhOwbHoBsrQKWGxOvHu42tMQ3x/Wel6IZBnDg7JXcKRdVvMg+iQgITp69Ciuu+46v/uvv/56/PDDDyEbRcQe5xs78N35FpypNwT1erVCip8W53qm9z85Xgen03d9GrMawekqAQDJ+b4XUogldhy+hLKvK9DaYe3cSMHqgAhIiFpaWpCZ6b9NZ2ZmJlpbW0M2iog9mgx8+940VfDDJIVUjBvH50AmEaGqxYj9F1p8Hsc1n8XEPA3GjRyG9IzYK+voTpvRhlajDcauU/gej8jMLy1E9EpAQuRwOCCR+A8ricVi2O29u9xE/MEYQ7Pr1z5N1XeguTeSE2WYN5LPCWrpsPqu2m8sBwcOqryxftMEYgm5xEcukVTZmXJASY19EvCsWUlJid+VNKjp/cDEaHXAZHWA43ghCZXR2WqoFRLkJSt7pnvYLXC2nOd/IdNid9q+K+7saoutSy4Rx/FiZDXyN7naz6sJIEAhWr58eZ/H0IzZwKPZwHtDSUqpp74sFDiOQ35Kgs99ukuncLqiCeqUTIxMSEXs+0N9rOZhNVKcqB8EJERbt26NlB1EDNPoig+lhhAf8ofZ5sBnpxowMkuNYRkqXPzxIOxOhiZlEUbFYLW9Lzz1Zj3WN0sEOppoCr8fBL3kNDF4aHHFh1JDjA/54nBlG07Xt6OiyYAxaVIoqn8EB2Do2Klhf69I4Y4ReQ3NAJrCDwASIqJP5o1Mx5QhyZBKwu+hTC1MRp3ehAtNRtSfPYRCMKRk5SM7JzaWle4PiXIJNEopZN3/PjSF329IiIg+kYhFYQlS+zv3TRNycaJGB/HhKqSlJSJj/JURea9IMTE/CRPzk3ruoAr8fkNCRAiOWMRhfJINUBkBLgHIjP0kxn5BPYn6jaDRwI0bN2Lq1KlQq9XIyMjAkiVLUF5eLqRJRDd0Jht2nqjDwYu+kw/DRu0R/j51aOeQJt6hZYX6jaBCtGfPHqxevRr79u3Drl27YLPZcM0116Cjg/5xsUKzwYKTNXqcqm2P3JvYrUDdUf5xTnHk3idC6Ew2vLW/Etv2V3rvkJJH1F8EHZp98sknXs/LysqQkZGBgwcP4qqrrupxvMVi8Uqa9NU3mwgvOhPfDlirjODChvXHeTFKSAFSYrflhz9EHFCnM0PEcd49uWh9s34TU4kaOh2/NlRKSorP/Rs3boRWq/Xc8vNjZx30gUrEhYgxoOYQ/zhnUsx2YuwNd0KjkzFYHV2m8N0ekd0KOMK0mMAAJWaEyOl04oEHHsDMmTMxduxYn8eUlpZCp9N5blVVVVG2cvDhFqKkhAgJUct5wNAIiCVA1rjIvEeEkYg4SFw1cWZrFyGSyPnrAgBLBIe2A4CYmTVbvXo1jh8/jq+++srvMXK53G+dGxEZIuoRMQZc/IZ/nFPcuRZYnMFxHJQyMdrNdphsDmghde8AZGrA1ApYDfzQk/BJTHhEa9aswQcffIAvvvgCeXl5QptDuGCMQWeMoBC1VQK6S4BIAuRPD//5o4jS1cfbaO3WfULuaq1rCa6P02BBUI+IMYZf//rX2LFjB3bv3o2iIh+9fwnBMFodcDAGEcdBrQizEDEGXHB5v9nj47463b2ggKl7vZn7uqwkRL0hqBCtXr0ab775Jt577z2o1WrPctVarRZKpVJI0wjwpQtr5g2DwWKHONx9gZrP8h6RSAIUXBHecwuAWi6FRikF171fgMztEVGMqDcEFaItW7YAAObOneu1fevWrSgpKYm+QUQPJGJRv1bdCAinAzj3Of84fyqg0Ib3/AKwYIyfzqVuj4iEqFcEH5oRg5CL3wDGFr4EomCG0NZEFrdHREOzXomZWTMi9thf0YI2oxVjc7XISQrTULm9vnOmbNiC+FwqKBAoWN0vSIgIv1Q0GVDTZkZhWphqv2xm4OR7AHMC6SOAjAFS3AqgVmfC7vJGqBUS3Dg+p3OHxyNq5wP0cZiwGQ1IiAi/tLmm7pPCMXXvsPMiZGzm4yYjrhtQX0on48s8TNZufyt3jMhhB+yWuM2VijQxkUdExB5WuxNGKz8VrQlViGxm4Pg7fBa1WAKM/dnAqbB3kSD1M30vlnaKD8WJ/EIeEeETd0a1Qir21FIFjNMJNP4InP8CMOt5ERp3C6CJ/bXKAsWd0Gi1O2F3OCHpusiATMWLsaUdSEwTyMLYhoSI8ElANWZWI9DRCJh1/JfNfdNXd/ZrViYBYxYDmpxeTxWvyCUiiDgOTsZgsjmg7ipEcrWriT55RP4gISJ80meNmcMG1B3jG5q11/s/kVQJ5E4G8qcN6Bkyvt5MhA4LvwacVyY6JTX2CQkR4ROTKz7kU4gay4Ezu7y/WMpkPjFRoeE9AJkKSEwHNLlAnCwLFCpKmYQXoh5lHjSF3xckRIRPZg1Pw/TLUuBwdkk6ddiA058Adcf55wotkDcVyBjd+WUbxGgUEljtTji75+nK3PVm5BH5g4SI8ItULIInTm3WAcf/zQ/DOBFQMB0YMpOfFSIAAIsn5vre4Z7CN1NHUX+QEBF9Y2gEjm7jhxZSJXD5EiC5UGir4gd3LZ2FhMgfJERED9rNNnxyvA6pKhnmZ9uBY//ip58T0/jpd2WS0CbGFwoNf2818sNb8iJ7QEJE9KDNaMOlVhNErRVAw3d8VrAmBxh/K+8RET6pbDbi63NNSEmU4drLszp3SBS8+Dhs/PAsMVU4I2OUwTGdQQSEzmSD1nQJo5t38SKUUgRM+AWJUB84GEOdzozGdov3Do7rMjzTRd+wOICEiOiBubECI5s+hVLMgLTh/HBMEpklpwcSiXI+st9hsffc6RYiMwmRL0iICG/a66A6vQMiZgdLHQaMWQKIgizxGGQkyvhIh8nmgLP7HL5HiChg7QsSIqITsx449jZsVgt0ilw4xyzpXA6H6JMEmdi1yCJg7JHU6ApY08yZT0iICB6nAzixHbAY0MppcDptAbSJFBMKBI7jPE30ewzP3DNnNDTzCQkRwXPhS0BfC7tYjoqshXCI5JFdZnqAkijnPcieQkRDs94gv5vgV9Oo3AcAkIxaiOUZo2BzOCEV0+9UoGiVUphtDvToxu4ZmrXz7VEGSf1dfyEhGuw4HUD5J3wb0+zxQMYoACARCpIbxvvptSRX80F/p4OvORsAK5eEE/q0DXaqD/LtW2UJwNCrhbZm4MJxnV6RqU1QU2IREqLBjMXAx4YAoGgOIFXgyzONeOfgJVQ0dQhr20BEmczfm1qFtSMGEVSI9u7di0WLFiEnJwccx+Hdd98V0pzBR9U+wG4F1FlA9gQAQG2bGVUtRljtToGNi09qdSa8tb8S7x2p7rmThMgvggpRR0cHJkyYgOeff15IMwYnFgNQfZh/XHSVZ0WNNpMVQD9bxBI9EHEc6nRmNOgtPXeSEPlF0GD19ddfj+uvv15IEwYvVfsAp6uYNeUyAHzj9w5LL50ZiT5RK/ivlMFi79lEn4TIL3E1a2axWGCxdP7S6PWUkxEUViNQ4/KGCmd5vKGwrNwxyFFKxZCKOdgcDAaLHUkJXWr0ugoRLbboRVwFqzdu3AitVuu55efnC21SfFL7A19Vr870eEMAoKNhWchwHOdpnN9u7pbUqEzixcdhoxU9uhFXQlRaWgqdTue5VVVVCW1S/OF08FP2AN9vusuvclhXdh3EuIdnerPNe4dI3Jk/RMMzL+JqaCaXyyGXD9wlaaJCYzmf3StLBNJHe+3iOL5wU0seUUj49YgAfnhmagOMLUBSQXQNi2HiSoiIMHDpAH+fU9yjsn7ykBRMHpLSs4UFERBJCVIkJUghEfmIASlTAFQAppao2xXLCCpEBoMBZ8+e9TyvqKjAkSNHkJKSgoIC+rUIO7pqQF/DDxFyiv0eJvL1BSL6zdTCFEwtTPG9073kdEdT9AyKAwQVou+//x7z5s3zPF+7di0AYPny5SgrKxPIqgFMzSH+ntYhE47EdP6+o1FYO2IMQYVo7ty5YIyGAVHBagQafuQf507usftSqxGfHK9DQUoCruna+J0IL26PyKwH7JYBvQx3IMTVrBkRAnVH+QRGdRag7lkh3tJhRbvZDqPV4ePFRCAwxvD291V4Yc+5njNnUmWnN0pekQcSosEAY50JjLmTfCbSNXfwOUQpidQkP1Q4jkOHxQ6T1YG2DlvPAzzDM4oTuSEhGgy0nOenjCVyIGOMz0NaSYjCSrLr79hqtPbcSQHrHpAQDQaqXUHqrPF+VxltNpAQhRN3aUebqTePqCGKFsU2JEQDHVMb0HKOf+xnyt5otcPg6rGcqiIhCgfJrqTQNl8ekco1GdBeyw+bCRKiAU/NYf7Dnlzod6lj98qkyQlSyCVU7BoOkl0ekXvI60ViOp9MarfyGdYECdGAxmHjC1wBPkjdCzlJCuQk0fJB4cJdOKwz8e1AvBCJunhFNVG2LDahEo+BTN0xwGbiCy1Th/s9bEhqIoakJkbRsIGPSi5BSqIMGqUEFnu3vkQAoMkGdJcAfS2QNU4YI2MIEqKBCmOddWV5U2n5mijDcRyWX1no/wB1Dn9PHhEAGpoNXJrP8vEHiZxfJsgPdocT5u7LIxORR+NKKjU08EPoQQ4J0UCEMaDyW/5xTnGvZQTVbSa8sOec72bvRFiw2H0IvSKJX+vM6eCHaIMcEqKBSMt5vtJeJAHypvR6aHWbCYyBZssiQJvRipf2nsfWry/0rKnkOCB5CP+49ULUbYs1SIgGGowBFXv5x7mT+F/dXqhpM/OH0oxZ2FHJJTDZHDBZHZ5+4F4kF/L3bRejalcsQkI00Gg6DbTX8RnUBVf0eqjDyVCnMwHgp++J8CIRi5Cl4f+u1W2mngckuTyi9jq+O8IghoRoIOGwAWc/4x/nTeXbwfZCTZsJNgdDgkxMpR0Rwp2b5fY8vVBoAFUG78U2n4myZbEFCdFAovJbwKzjP+AFM/o8/EIzv6z0kNREcLS0TURwe5rVrX48nvSR/H3j6ShZFJuQEA0U2uuAyn3846FXA5K+PZwLrvXti9IomTFS5CQpwXFAq9HWszcRAKS5hKi1ArD58JoGCSREAwG7FTj5Hj8VnD6i81e2FxhjmDQkGSMy1RiSmhAFIwcnCqnYMzw71+BjLbPENP7mdAD1J6JsXexAQhTvMAb8+AGfvChXAyOu79cKohzH4fIcLW4Yn02rukaYy3M0mJif5LuWj+M6uyLUHh601fgkRPEMY3xwurGcX5ljzE2AjLybWOPyHC3mjcpApsbPzGTm5Xw1vqGRH6INQkiI4hXGgDO7OuvJRt3Q7wX7Kpo68M3ZJnRYfCwASEQfqRLIdnlFFXsHpVdEQhSPWI3A0X/xS0dzHDDiWv5XtR8wxvDtuWZ8V9GCI1VtkbWT8MAYQ02bCZ+dqofV7ux5QMEVvFekr+W7JgwySIjiCcb4gOb3L/NlHCIJMPqmPnsNdeVEjR71ejNkEhEm5idFzlbCCycDPj5eh6OXdDhU6WPde7kKGDKLf3x216BrmBYTQvT888+jsLAQCoUC06dPx/79+4U2KbZgjM8zOfz/gJPvAxYDkJAKTF4OZPpuhu+LJoMFe07zS9hccVkKEuXUBSZaiEUcZg7jO2Tur2hBnc7HVH3+dECTw8+CHv0XnxM2SBBciP75z39i7dq1WL9+PQ4dOoQJEybg2muvRUPDIG8szhjvpp/7AvjuBeD4v/lCVrEEKLoKmLKCz8rtJ5dajXjn4CVY7U7kpySgOD85gsYTvhiZqcawDBUcToYdh6txqXuSo0gEjL2Zb2RnagUOlgENpwZFzIhjAi+1On36dEydOhV//etfAQBOpxP5+fn49a9/jd/+9re9vlav10Or1UKn00Gj0UTD3PDidAJ2M3+zGvgPn6kVaK8H9Jf4X0Y3Ejk/zZs3pc9CVoCPSbizpU/Xt+PDo7UAgEyNAj8tzoVSRlP2QmC2OfDu4WrU6szgOGBYhgrTi1KRru7SqsXUxv/wGFw/xolpQPooQJsLKFP4/78oPv5//f2OCuqbW61WHDx4EKWlpZ5tIpEICxYswLffftvjeIvFAovF4nmu1+v790YtFcDZ//je59Fh5mNbl+393dbnOV33TkePhlhWhxOnavWew50iKfTKPLSqhkInHoIRLAVXukTIbHPg/3170XVGBsY6383JGEZnazBvJO8x5SUrIRFxGJWtwVUj0qjlh4AopGL8dFIu9pQ34kSNHmfqDRiVpfEI0fFqHfadb4GYzUZ2xw/IbDsCEWsDys8CAC5LS4RaIQXEUjQanbjYagHjRGDgf3TyUhKRo1X6ziULdxnPuFv5cqIwIKgQNTU1weFwIDMz02t7ZmYmfvzxxx7Hb9y4ERs2bAj8jeyW2F7MTiwFZIlwSDSoaLbAKE2GQZ6JDmkKwLlGzxZ4LQfNGDxLAPmi3dy5L0EmwYpZRRQTihHkEjGuuTwLxQXJOF6j8+p8YLE7Pf+7NvkEnE4fhVRjBTSWGqgsjRji/jw4bHBazLAZO7zOzeRmPh0gGjAfs39BElefzNLSUqxdu9bzXK/XIz8/v+8XJuUDE27r9ovQ5bFnu69tXbb3d1uf5+T4xxIFP+RyudlShxOTisyewzrvOXAAEmWd/y65RIRl0ws6zQAHjuOfchyHhG5DLxKh2CNdLfd4rW5GZ6uRn6yEgzHYHW4fd6hnv0IlA2ABHFYkGk0oMJgA5vqBYgxqpRSQu//3vjz3MNJHd4dAEPTTmZaWBrFYjPr6eq/t9fX1yMrK6nG8XC6HXO6/7alfZIlASlGwZkYNiViEgn7WfYlEHDL8ZeoScUuCTIIEWV9fSwmARKiUyVD5Xqou7hB01kwmk2Hy5Mn47LPPPNucTic+++wzzJjRdxsLgiAGBoL762vXrsXy5csxZcoUTJs2DZs3b0ZHRwd++ctfCm0aQRBRQnAhWrp0KRobG/Hwww+jrq4OEydOxCeffNIjgE0QxMBF8DyiUIj7PCKCGOD09zsqeGY1QRAECRFBEIJDQkQQhOAIHqwOBXd4q9+lHgRBRBX3d7OvUHRcC1F7ezsA9C+7miAIwWhvb4dWq/W7P65nzZxOJ2pqaqBWqyO2Lpe7jKSqqmpQzMwNtusF6Jojec2MMbS3tyMnJwcikf9IUFx7RCKRCHl5eVF5L41GM2g+pMDgu16ArjlS9OYJuaFgNUEQgkNCRBCE4JAQ9YFcLsf69euDq/qPQwbb9QJ0zbFAXAerCYIYGJBHRBCE4JAQEQQhOCREBEEIDgkRQRCCQ0KEwFaafemllzB79mwkJycjOTkZCxYsiLuVaYNdWXfbtm3gOA5LliyJrIERINBrbmtrw+rVq5GdnQ25XI4RI0bgo48+ipK14SHQa968eTNGjhwJpVKJ/Px8PPjggzCbfaxIGwnYIGfbtm1MJpOxV155hZ04cYLdc889LCkpidXX1/s8/vbbb2fPP/88O3z4MDt16hQrKSlhWq2WXbp0KcqWB0eg1+umoqKC5ebmstmzZ7PFixdHx9gwEeg1WywWNmXKFLZw4UL21VdfsYqKCrZ792525MiRKFsePIFe8xtvvMHkcjl74403WEVFBdu5cyfLzs5mDz74YFTsHfRCNG3aNLZ69WrPc4fDwXJyctjGjRv79Xq73c7UajV79dVXI2ViWAnmeu12O7vyyivZP/7xD7Z8+fK4E6JAr3nLli3ssssuY1arNVomhp1Ar3n16tVs/vz5XtvWrl3LZs6cGVE73QzqoZl7pdkFCxZ4tvW20qwvjEYjbDYbUlJSImVm2Aj2eh999FFkZGTg7rvvjoaZYSWYa37//fcxY8YMrF69GpmZmRg7diyeeOIJOBwOn8fHGsFc85VXXomDBw96hm/nz5/HRx99hIULF0bF5rgueg2VQFea9cVDDz2EnJwcr396rBLM9X711Vd4+eWXceTIkShYGH6Cuebz58/j888/x7Jly/DRRx/h7NmzuO+++2Cz2bB+/fpomB0SwVzz7bffjqamJsyaNQuMMdjtdvzqV7/C7373u2iYTMHqUNi0aRO2bduGHTt2QKEYeIsdtre3484778RLL72EtLQ0oc2JGk6nExkZGXjxxRcxefJkLF26FL///e/xwgsvCG1axNi9ezeeeOIJ/O1vf8OhQ4ewfft2fPjhh3jsscei8v6D2iMKdKXZrjz11FPYtGkT/vOf/2D8+PGRNDNsBHq9586dw4ULF7Bo0SLPNqeTX+9cIpGgvLwcQ4cO7fG6WCKY/3F2djakUinE4s5lu0ePHo26ujpYrVbIZLKI2hwqwVzzH/7wB9x5551YuXIlAGDcuHHo6OjAqlWr8Pvf/77XXkLhYFB7RMGuNPunP/0Jjz32GD755BNMmTIlGqaGhUCvd9SoUTh27BiOHDniud10002YN28ejhw5EhedMYP5H8+cORNnz571iC4AnD59GtnZ2TEvQkBw12w0GnuIjVuIWTTKUaMSEo9htm3bxuRyOSsrK2MnT55kq1atYklJSayuro4xxtidd97Jfvvb33qO37RpE5PJZOydd95htbW1nlt7e7tQlxAQgV5vd+Jx1izQa66srGRqtZqtWbOGlZeXsw8++IBlZGSwP/7xj0JdQsAEes3r169narWavfXWW+z8+fPs008/ZUOHDmW33nprVOwd9ELEGGPPPfccKygoYDKZjE2bNo3t27fPs2/OnDls+fLlnudDhgxhAHrc1q9fH33DgySQ6+1OPAoRY4Ff8zfffMOmT5/O5HI5u+yyy9jjjz/O7HZ7lK0OjUCu2WazsUceeYQNHTqUKRQKlp+fz+677z7W2toaFVupDQhBEIIzqGNEBEHEBiREBEEIDgkRQRCCQ0JEEITgkBARBCE4JEQEQQgOCRFBEIJDQkQQhOCQEBFxRVlZGZKSkjzPH3nkEUycONHzvKSkJC5b2Q52SIgIn5SUlIDjOPzqV7/qsW/16tXgOA4lJSVex4dbAAoLC7F582avbUuXLsXp06f9vubZZ59FWVmZ5/ncuXPxwAMPhNUuIvyQEBF+yc/Px7Zt22AymTzbzGYz3nzzTRQUFAhik1KpREZGht/9Wq3Wy2Mi4gMSIsIvkyZNQn5+PrZv3+7Ztn37dhQUFKC4uDikc/vyVJYsWeLxsubOnYuLFy/iwQcfBMdx4DgOQM+hWXe6emYlJSXYs2cPnn32Wc85KioqMGzYMDz11FNerzty5Ag4jsPZs2dDui4iOEiIiF5ZsWIFtm7d6nn+yiuv4Je//GXE33f79u3Iy8vDo48+itraWtTW1gZ8jmeffRYzZszAPffc4zlHQUFBj2sCgK1bt+Kqq67CsGHDwnUJRACQEBG9cscdd+Crr77CxYsXcfHiRXz99de44447Iv6+KSkpEIvFUKvVyMrK6rNjpi+0Wi1kMhkSEhI85xCLxSgpKUF5ebmnUbzNZsObb76JFStWhPsyiH4yqFvFEn2Tnp6OG264AWVlZWCM4YYbboj7/tU5OTm44YYb8Morr2DatGn4v//7P1gsFtxyyy1CmzZoIY+I6JMVK1agrKwMr776ati8BpFI1KMFqc1mC8u5+8PKlSs9gfitW7di6dKlSEhIiNr7E96QEBF9ct1118FqtcJms+Haa68NyznT09O94j4OhwPHjx/3OkYmk4W8lpi/cyxcuBCJiYnYsmULPvnkExqWCQwNzYg+EYvFOHXqlOexP3Q6XY/1z1JTU3022Z8/fz7Wrl2LDz/8EEOHDsUzzzyDtrY2r2MKCwuxd+9e3HbbbZDL5UENCQsLC/Hdd9/hwoULUKlUSElJgUgk8sSKSktLMXz48F4XSyAiD3lERL/QaDTQaDS9HrN7924UFxd73TZs2ODz2BUrVmD58uW46667MGfOHFx22WWYN2+e1zGPPvooLly4gKFDhyI9PT0ou9etWwexWIwxY8YgPT0dlZWVnn133303rFZrVGYBid6hntXEoOXLL7/E1Vdfjaqqqh6rohLRhYSIGHRYLBY0NjZi+fLlyMrKwhtvvCG0SYMeGpoRg4633noLQ4YMQVtbG/70pz8JbQ4B8ogIgogByCMiCEJwSIgIghAcEiKCIASHhIggCMEhISIIQnBIiAiCEBwSIoIgBIeEiCAIwfn/SoFq5zqgUsMAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:08:23.317166Z", + "iopub.status.busy": "2024-02-29T06:08:23.316078Z", + "iopub.status.idle": "2024-02-29T06:08:23.699948Z", + "shell.execute_reply": "2024-02-29T06:08:23.698932Z" + }, + "papermill": { + "duration": 0.413109, + "end_time": "2024-02-29T06:08:23.702253", + "exception": false, + "start_time": "2024-02-29T06:08:23.289144", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCHElEQVR4nO2deZxT9dX/Pzd7MpNk9o3JLOwgDDuIiIpSFVxAn6JVVHBtKzx9kB+v6rRPodrqaB/1EVuLrS1MbUGealHbqiBihYqCbIOyyDIMwzD7mkz25d7fHzfJTGaSmew3mTnv1yuvJPfe3JybmXxyvud7zvkyHMdxIAiCEBCR0AYQBEGQEBEEITgkRARBCA4JEUEQgkNCRBCE4JAQEQQhOCREBEEIDgkRQRCCIxHagEhgWRYNDQ1Qq9VgGEZocwiC6APHceju7kZBQQFEosB+T1ILUUNDA3Q6ndBmEAQxCHV1dSgsLAy4P6mFSK1WA+AvUqPRCGwNQRB9MRgM0Ol03u9qIJJaiDzDMY1GQ0JEEAnMYKETClYTBCE4JEQEQQgOCRFBEIKT1DGiYOA4Dk6nEy6XS2hThjRisRgSiYTSKIiwGNJCZLfb0djYCLPZLLQpwwKVSoX8/HzIZDKhTSGSjCErRCzLoqamBmKxGAUFBZDJZPRrHSM4joPdbkdraytqamowZsyYAZPXCKIvQ1aI7HY7WJaFTqeDSqUS2pwhj1KphFQqRW1tLex2OxQKhdAmhQTLcjh6qRPVrUZkq+WYNzoLcolYaLOGDYL/bNXX1+O+++5DZmYmlEolJk+ejMOHD0ft/PTLHD+S9bPmOA6fnG7Gv8+1oaHLiuN1euw80QRq5x4/BPWIOjs7MW/ePCxYsAAfffQRsrOzce7cOaSnpwtpFjHMaDJYcarRAIYBZhSn49ilLlxoNaHh/HGMcFwCNCOAEdMBGtrHDEGF6IUXXoBOp8OWLVu820pLSwW0iBiO5GuVWD6nGO0mG8bnaeBiOTSe/Bzdhw8BeRqg+SRg7QJG3yC0qUMWQX3pv//975g5cyaWLVuGnJwcTJs2DW+88UbA4202GwwGg8+NIKJBtlqO8Xl8mdCUDBeKug7C7mTBZozkD7h8CDC2Cmjh0EZQIbpw4QI2bdqEMWPGYNeuXfjhD3+IH/3oR/jTn/7k9/iKigpotVrvbahW3l933XVYs2aN0GYMefQWB9qNtn7b01sOYkahGmVl0yEquwvIGgNwHFAfvdgl4YugQsSyLKZPn47nnnsO06ZNw2OPPYZHH30Ur7/+ut/jy8vLodfrvbe6uro4W0wMJT4704K/HLiEE/X6no2WTqD1WyikYmDkdXxcqHAmv6/lFOByCmLrUEdQIcrPz8fEiRN9tk2YMAGXLl3ye7xcLvdW2kdScW93sgFvThcb9LGOII4NlZUrV2Lv3r3YuHEjGIYBwzAoLCzEpk2bfI47duwYRCIRamtrAQAvv/wyJk+ejJSUFOh0Ojz++OMwGo0+r/n8888xf/58KJVK6HQ6/OhHP4LJZArZxqHAxTYTLrTy156v7ZVq0Pg17/1klALqXH7mLK0YkKcCTjug9/+/SUSGoMHqefPm4cyZMz7bzp49i+Li4pi+72v/Oh9wX2lWCpZOG+F9/vt91XC4/E/jFqYrsWxmz/Bw8/4aWOy+pSRPfGdsSLZt3LgRZ8+exaRJk/DMM88AAF566SVs27YNP/zhD73Hbd26FfPmzfN+ViKRCK+++ipKS0tx4cIFPP744/jxj3+M3/72twCA6upq3HzzzfjlL3+JzZs3o7W1FatXr8bq1at9JguGAy6Ww96zfLxnalEaMlPl/A6O4wPTAJw5k/DPY/Vo1Fvx0NUlkGeMAhqPA+3VgCduREQNQT2iJ554AgcOHMBzzz2H8+fPY9u2bfj973+PVatWCWmWoGi1WshkMqhUKuTl5SEvLw/Lly/H/v37vZ4iy7LYvn07li9f7n3dmjVrsGDBApSUlOD666/HL3/5S/z1r3/17q+oqMDy5cuxZs0ajBkzBldddRVeffVVvPnmm7BarXG/TiH5srodHSY7VDIx5pRm9OzQ1wFWPSCRQZIzDm1GG6wOF9qMdt5DAoAu8ohigaAe0axZs/Duu++ivLwczzzzDEpLS/HKK6/4fMFiwaoFowPuE/VJFXnsmlEBj+2bVvLQvNikHkydOhUTJkzAtm3b8NRTT2Hv3r1oaWnBsmXLvMd88sknqKiowLfffguDwQCn0wmr1Qqz2QyVSoXjx4/j66+/xtatW72v4TjOWwozYcKEmNieaFxoNeJwbQcA4IYJOXwsyEPLaf4+ezwgliIrVY5uqxNt3TaMyHZ7yaZWfogmoXq6aCJ4icett96KW2+9Na7vKZME7wjG6thQWb58uVeItm3bhptvvhmZmZkAgIsXL+LWW2/FD3/4Qzz77LPIyMjA559/jocffhh2ux0qlQpGoxHf//738aMf/ajfuYuKimJmdyJhd7L46EQTOA4oK9RidE6v9qUcB7Sd4x9nj+fv1HLUtJnQZrQBulxAoQGsBqC7AUgvif8FDGEEFyKiPzKZrF/bknvvvRf//d//jSNHjuCdd97xmVk8cuQIWJbFSy+95C2z6D0sA4Dp06fj1KlTGD06sDc41JFJRJhdmoGGLguuG5fju9PYDNi6AbGUD04DyHLHjto8U/yaAl6IDCRE0SY5i4OGOCUlJTh48CAuXryItrY2sCyLkpISXHXVVXj44Yfhcrlw++23e48fPXo0HA4Hfv3rX+PChQv485//3C8F4sknn8QXX3yB1atXo6qqCufOncP777+P1atXx/vy4kq70YZOk937fEZROm4tK4C47xjc4w1llAJi/vc5PUUKAOg0O/h96nz+3tgcU5uHIyRECci6desgFosxceJEZGdne4PUy5cvx/Hjx3HHHXdAqVR6j58yZQpefvllvPDCC5g0aRK2bt2KiooKn3OWlZVh7969OHv2LObPn49p06Zh/fr1KCgoiOu1xZNmgxXbD9XhH183wObkPUyRiOkvQgDQ7haizDHeTWlKPg5ksbtgdbiAlGx+h6ktpnYPRxguiUuMDQYDtFot9Hp9v5wiq9WKmpoalJaWJl1LimQlkT5zi92FPx+4CJPNhRFpStw2pQBKWYC2HlYD8OVr/OzDVf8JyFK8u7YerIVMLMKNV+RBy1jcx4mA+f/P6zkRgRnoO9ob+iSJIcm/z7XCZHMhM1WGJdMKBu4t1FHN36vzfUQIAJbP6ZXTxkkAiRxw2gBzO6DOjYHlwxMamhFDji6zHaca+YLohRNyB29w1nmRvx8sUZFhgFR3kNtEBbDRhISIGHIcu9QFjuOz5AvSlAMfzLK9hChwHpg3gqHiUyZg6YjcUMILDc2IIYWL5fBtUzcAYFpR2uAvMDYBDis/5FL3D9zXtpuw+1Qz0lQyfHdGIaB0N+2zdEbRaoKEiBhS2JwulGap0KS3QpceRK/yjhr+Pr0Y8NPqViIWodvqhMiTRu8RIjN5RNGEhIgYUqhkEtw8KR8cxwW3akunR4j8D8vUCv4rYrQ5+XMq3bVplk4+G5vax0YFihERQ5KgRMhpA/T1/OMA8aFUmQQMww/5THYXoEzrea3DEh1jCRIiYuhgdbjQbrQFv/pG1yWAY3lxUfpfsEEkYpAqd3tFVidfAiJ316hRnChqkBARQ4bqViPe/LIW71c1BPcCz2xZgGGZB8/wrNvqLvVQeYZnFCeKFiRECUii9axONHsCcbmTHyplq+XBvcATqB5g2h4AUuV8zZnB6m4Tq0jj7616/y8gQoaC1UMUu90+7NagbzbwDd58Wr8Gwqrns6MZkbfaPhA5GjlMdidUnhIRhbtUwUqryEQL8ogSDH89q6urq/Hwww+jtLQUSqUS48aNw8aNG/u9bunSpXj22WdRUFCAcePGAQC++OILTJ06FQqFAjNnzsR7770HhmFQVVXlfe2JEyewaNEipKamIjc3F/fffz/a2toC2nPx4sV4fRxBY3ey6HBX2edqghAijzekyQekAx8/qyQDd83UYUK+W4Dk7nsbCVG0GF4eEccBLkf831csDXqa11/P6vT0dBQWFuLtt99GZmYmvvjiCzz22GPIz8/HXXfd5X3tnj17oNFosHv3bgB8weFtt92GxYsXY9u2baitre03xOrq6sL111+PRx55BP/7v/8Li8WCJ598EnfddRc+/fRTv/ZkZ2dH4UOJLm1GGzgOSJVLkCIP4t96kGn7ASGPKOoMLyFyOYB/vxT/953//4JuLdq3Z7WHp59+2vu4tLQUX375Jf7617/6CFFKSgr+8Ic/eIdkr7/+OhiGwRtvvAGFQoGJEyeivr4ejz76qPc1v/nNbzBt2jQ899xz3m2bN2+GTqfD2bNnMXbsWL/2JBot3XzzshxNEPGhIMs6+uLNTfJ6RHrKJYoSw0uIkpjXXnsNmzdvxqVLl2CxWGC32zF16lSfYyZPnuwTFzpz5gzKysp8WnLMnj3b5zXHjx/Hv/71L6SmpvZ7z+rqaowdG9oqJELR4o4PBRWo9pZ1yPyWdfTFYHXg/76qg4Nl8fh1o3uEyOXkc4lkQWRwEwMyvIRILOW9EyHeNwK2b9+OdevW4aWXXsLcuXOhVqvxP//zPzh48KDPcSkpKQHOEBij0YjbbrsNL7zwQr99+fn5Ydscb8blqZGqkKA0K4jPwOMNpfkv6+iLXCKC0cbPmNmdLGQSCd8uxG7ig94kRBEzvISIYZJi9YW+Pav379+Pq666Co8//rh3W3V19aDnGTduHP7yl7/AZrNBLuc9hUOHDvkcM336dPztb39DSUkJJBL//w7+emgnGsWZKSjODFKIg5y29yATiyAVM3C4OJjtTsgkMj5OZDe5A9bJI9iJCs2aJSB9e1aPGTMGhw8fxq5du3D27Fn87Gc/6yco/rj33nvBsiwee+wxnD59Grt27cKLL74IoKcEYtWqVejo6MA999yDQ4cOobq6Grt27cKDDz7oFR9/PbSTFqcdMLjLOoIMVDMMA5WMF2mTZwFNOQWsowkJUQLSt2f1TTfdhDvvvBN333035syZg/b2dh/vKBAajQb/+Mc/UFVVhalTp+KnP/0p1q9fDwDeuFFBQQH2798Pl8uFG2+8EZMnT8aaNWuQlpbmXREkUA/tRMFgdeBim6kn83kgui4BrLtmzJMhHQQpcj6HyGzzJDX2ClgTESNoz+qf//znPrNBAD+c+Pbbb4N6PfWsDp2tW7fiwQcfhF6v92nAHw2E+syP13Xh029bMDI7BUumjhj44HO7gcuHgYJpwLibg36PfxxvwPkWIxaMz8FUXRpQdwg4/wmQPQ6YdGdkFzCESZqe1VdccQU++eQT7/NAcQoiPN58802MHDkSI0aMwPHjx705QtEWISHxJDJmpAQR/wsxPuShn0fkKXy1G0M6D+Efwb/1EokkofNTkp2mpiasX78eTU1NyM/Px7Jly/Dss88KbVZU8QhRumoQIfKWdTCDlnX0JStVjsJ0JVLdBbCQu9MdbCRE0UBwITp37hwKCgqgUCgwd+5cVFRUBFwC2WazwWazeZ8bDBQoHIwf//jH+PGPfyy0GTGl0xykR+SZttcUDFrW0ZeywjSUFab1bJC5hchupKTGKCBosHrOnDmorKzEzp07sWnTJtTU1GD+/Pno7u72e3xFRQW0Wq33ptPp4mwxkWi4WM6b46NVDpKv5W0LWxL5G3uEiHVRg7QoIKgQLVq0CMuWLUNZWRluuukmfPjhh+jq6uq3bruH8vJy6PV6762uri7OFhOJRrfVAY4DpGKmpzreHxwHdNXyjyMQIu/cjljS41XZTWGfj+ARfGjWm7S0NIwdOxbnz5/3u18ul3sT84IliReyTTqE+Kz1Fn7KXqOUDtwe1tQG2M28gGgGmVnzQ7fVgbe+ugQny/FlHgDvFTmsgL0bQOIVAicTCZVHZDQaUV1dHZXSAqmUd9PNZnPE5yKCw/NZez77eJCZKseNV+RiVskgOUGe+JC2CBANsuCiH2QSEUw2F2wOFg6XO6HTM3NGAeuIEdQjWrduHW677TYUFxejoaEBGzZsgFgsxj333BPxucViMdLS0tDS0gIAUKlUwTVUJ0KG4ziYzWa0tLQgLS0NYnHoX/RwSZVLcEWBdvADvcOy0GbLPMjEIohFDFwsB4vDBalY1CtgTUOzSBFUiC5fvox77rkH7e3tyM7OxtVXX40DBw5Erd+NJy3AI0ZEbElLS0vMVAyW7RGiEKftPTAMA6VUDKPNCavdBY1Cyhe+ApRLFAUEFaLt27fH9PwMwyA/Px85OTlwOARoiDaMkEqlcfWEPJxv6YZULEKeVhF4jXtjE19jJpEDqblhv5dCxguRxeGpN/MMzfzP8hLBk1DB6lghFosF+ZIQsWfP6RaY7S4sn1OEHE2Av3GnxxsqCqrtRyCUUnd2tafwtXcuERERCRWsJohQsDtZryhoBsohCnLZoMHwCFGPR0QxomgxLDwiYmhicFfby6UiKKQBvCGXE9Bf5h+HGaj2kK2Ww9x7NQ9PjMhG2dWRQkJEJC0Gdw7RgBnVhnqAdfKiocqM6P1ml2ZgdmmvNAGZO0bEOgGnFZAOnULieENDMyJp6XYveKhWDCBEvafto+2x9M6uplyiiCAhIpIWT42ZeqDlg7rcZUBp/gupw8Eng9wTsHZQnCgSaGhGJC0ej8jbmqMvLidgaOAfayMXoka9BX+vaoBKLsH9V7rjTVJ343w7ZfBHAgkRkbRML0pDUYYq8BJC3Y3u+JAqpLawgZCIRD1T9x48K3g4SIgigYSISFpyNArkDLS8tGe2TFsYlfiQ0j1bZnWwPYstSim7OhpQjIgYuujd8aEoDMsAQCHhvy4sx8HmdBe+ymhoFg1IiIikxO5kcaJej4ttJv/tR1i2R4jSotNATyIWQeYWI4s3u9rtEdHQLCJIiIikxGB1YPepZuw82eS/q4Kp1V1fJgNScqL2vp7ESavTLUTeoRnNmkUCCRGRlBg9M2aBpu493pCmMKL6sr4opPy5rI6+QzMSokigYDWRlHhyiAYVIm1hVN83V63wLkENoGf6nvKIIoKEiEhKugfyiDiuVyJjdBdYWDixTxsRT0Kjy9kzFCRChoZmRFLi9Yj8JTNaOvmhkkgMqAtia4hYypd6AOQVRQAJEZGUGG18watfj8iTP6TO7xGJWOGTS0RCFC4kRERSYvQWvPoRGm9ZR+irdQzGiXo9fre3Gh+fbOrZSLlEEUMxIiIpWTA+BwaL0395h6Gev4/RsMxsd/U0RwN6PCIamoUNCRGRlBSmq4B0Pzucdj6HCOCXlo4yPdP3vYTI20SfPKJwoaEZMbTobuRnzeRqQKGJ+uk9Dfq9eUQAFb5GARIiIunQmx04Ua9HfZefNee7G/n7GHhDQK/Man9DMyp8DRsSIiLpuNxlxu5TzTh4ob3/Tk98KIxlpYPBMzSzOdmeGjcKVkdMwgjR888/D4ZhsGbNGqFNIRIck433RlL6Tt1zXM+MmSbyZcv94RmauVgODpdbiKQ0NIuUhAhWHzp0CL/73e9QVlYmtClEEmAKVN5hM/C9oxkRn0MUA6RiBrkaBaRifvlpALT0dBQQ3CMyGo1Yvnw53njjDaSn+5sGIQhfAtaZGdzxodRsPuM5BjAMg3vnFGHZTJ23UVpPsNoCsK7ALyYCIrgQrVq1CrfccgsWLlwotClEkuDxiPoNzWIcHwqIRNnTAZKGZ2Eh6NBs+/btOHr0KA4dOhTU8TabDTabzfvcYDDEyjQigQnsEbnjQzEalgVEJOLXNLOb+ZtcHd/3HwII5hHV1dXhv/7rv7B161YoFAP0He5FRUUFtFqt96bTRbeymkh8OI7rFazutbor6wK63WUXMfaI9pxuxu/2VuNEvb5nIwWsI0IwITpy5AhaWlowffp0SCQSSCQS7N27F6+++iokEglcrv5j7fLycuj1eu+trq5OAMsJIeE44I5pI3DTFXlIkfXyiIwt/IodEnlUVuwYCCfL9S/zoJaxESHY0OyGG27AN99847PtwQcfxPjx4/Hkk09CLO6/lrlcLodcHmDpGGJYIBIxKMpU9d/R7Zm2HxHzNej9JzW6l5umXKKwEEyI1Go1Jk2a5LMtJSUFmZmZ/bYTxKB484di3H8IPat52HqXeVDha0QkRB4RQQRLi8GKlm4bstVy5PZe0yyeQtS3gT5A2dURklBC9NlnnwltApHgVLeacOBCO8oKtT1C5LAA5g7+cRxmzOR9G+gDFKyOEMHziAgiFPzmEHm8IVVGj2cSQxQSPzEiClZHBAkRkVSY7H5yiOI4LAMAlVyMbLUcGSm9GuVLaWgWCQk1NCOIwTAO5BHFulG+mxy1AvddWey7UUbB6kgIyyO6cOFCtO0giKDoGZq50zs4rtfUfXyEyC+e6XunnV9aiAiJsIRo9OjRWLBgAf7yl7/AarVG2yaC8IvLnUgI9BqamTsAhxUQSYDU6C0tHTISBV/1D5BXFAZhCdHRo0dRVlaGtWvXIi8vD9///vfx1VdfRds2gvDBZHeC4wCxiIHSPYXu9YbUefw6ZnHira8u4fW91egw2fkNDENT+BEQlhBNnToVGzduRENDAzZv3ozGxkZcffXVmDRpEl5++WW0trZG206CgFIqxndnFGLRpDwwnuzpOAeqPVjsLljsrj7Z1TSFHy4RzZpJJBLceeedePvtt/HCCy/g/PnzWLduHXQ6HR544AE0NjZGy06CgFQsgi5DhTG5varbva0/4itEfss8aAo/bCISosOHD+Pxxx9Hfn4+Xn75Zaxbtw7V1dXYvXs3GhoasGTJkmjZSRD9cTkAY+yWDhoIxUBJjTQ0C5mwpu9ffvllbNmyBWfOnMHixYvx5ptvYvHixRCJ+D9OaWkpKisrUVJSEk1biWHOpXYz9BYH8tMUyEqV820/OBaQpwLy6C8dNBCe3tU2f2UeFKwOmbCEaNOmTXjooYewcuVK5Of7T6nPycnBH//4x4iMI4jenG4y4FSDAVePyeKFqHd8KMYV930hjyi6hCVEu3fvRlFRkdcD8sBxHOrq6lBUVASZTIYVK1ZExUiCAHrlEHn6EAnVGhYBCl8pWB02YcWIRo0ahba2tn7bOzo6UFpaGrFRBOGPfqt3CDRjBgAahRTZarlvczbv0tM0NAuVsDwi78JyfTAajUG3fSWIUOnunVVtNQC2bj6JMDUv7rZMLtRicqHWdyN5RGETkhCtXbsWAL+kyvr166FS9VQ6u1wuHDx4EFOnTo2qgQQBAA4X621EliKXAJ01/I7UbEAiG+CVcUTWS4g4Lu5xq2QmJCE6duwYAN4j+uabbyCT9fwDyGQyTJkyBevWrYuuhQSBnmGZVMxALhEJGh8KiKdLo8vJpxYkikAmASEJ0b/+9S8AfG/pjRs3QqOJ75QpMXzpXXXPMIyg8SEA6DLb8d6xeohEDB6YW8JvFEsBsYQXIoeJhCgEwooRbdmyJdp2EMSAZKvl+O6MQn6Z5zguHRQIkYhBp9kBsYgBx3G8ODIMHydyGfgpfCWtXBwsQQvRnXfeicrKSmg0Gtx5550DHrtjx46IDSOI3sglYugy3DEYQyO/dJBUIdiX3dOl0cVycLg4yCTueJBUxQfSKWAdEkELkVar9RYaarXaQY4miBjiHZbFfumgQEjFDMQiBi6Wg9Xpgsy9sgdN4YdH0ELUezhGQzMi3pxuNMDp4lCcpYJGoELX3jAMA4VUBJONr8DXKKT8DprCD4uwEhotFgvM5p4Pura2Fq+88go+/vjjqBlGEL05UtuJT043o91oFzxQ7cFbb9a7zENGQhQOYQnRkiVL8OabbwIAurq6MHv2bLz00ktYsmQJNm3aFFUDCQIAuq3urGqRDbB08hvj1KM6EJ56M5/CV88UPtWbhUTYHRrnz58PAHjnnXeQl5eH2tpavPnmm3j11VejaiBBOFyst++Pxu5u+5GSxQerBSRdJUO2Wg5R7zgVeURhEZYQmc1mqNV8c6qPP/4Yd955J0QiEa688krU1tYGfZ5NmzahrKwMGo0GGo0Gc+fOxUcffRSOScQQxuMNySQiyEzuZnsCD8sA4MYr8nDflcUYmZ3as9FbgU/B6lAIu3n+e++9h7q6OuzatQs33ngjAKClpSWkJMfCwkI8//zzOHLkCA4fPozrr78eS5YswcmTJ8MxixiiGN1CpFZIwBgu8xu1hQJaNAAUrA6LsIRo/fr1WLduHUpKSjBnzhzMnTsXAO8dTZs2Lejz3HbbbVi8eDHGjBmDsWPH4tlnn0VqaioOHDgQjlnEEMVgdQAA1FLwOUQAoNUJZ9BAeIdmFr7ejAiKsDKrv/vd7+Lqq69GY2MjpkyZ4t1+ww034I477gjLEJfLhbfffhsmk8krbH2x2Wyw2Wze5waDIaz3IpILT3lHFtfJJzLKVAmRtXy2uRtfnG9DYboKCyfm8hs9wWrWBThtgsexkoWwV3rNy8tDXp5v+4XZs2eHfJ5vvvkGc+fOhdVqRWpqKt59911MnDjR77EVFRV4+umnw7KXSF4mj9BiRJoSqc2HACP4YVkCVLa7WA6dZgfUCkfPRrGErzFz2vnhGQlRUIQlRCaTCc8//zz27NmDlpYWsCzrsz+UlWDHjRuHqqoq6PV6vPPOO1ixYgX27t3rV4zKy8u9rUgA3iPS6RLURSeiRopcwrf+uNzCb9AWCWuQG7nEM33v+/8PaQovRHYToMoQwLLkIywheuSRR7B3717cf//9yM/P71ljKgxkMhlGjx4NAJgxYwYOHTqEjRs34ne/+12/Y+VyOeRyedjvRSQxHAckWKDa75JCAD90tHRSwDoEwhKijz76CB988AHmzZsXbXvAsqxPHIgY3rAshy8vtCMDeoyzWyCSyIDUXKHNAhCgbzVAU/hhEJYQpaenIyMjcpezvLwcixYtQlFREbq7u7Ft2zZ89tln2LVrV8TnJoYGRrsTX9V0IN90GuM14AtdRREtxxc1PEMzu5MFy3IQiXpV4APkEYVAWH/RX/ziF1i/fr1PvVk4tLS04IEHHsC4ceNwww034NChQ9i1axe+853vRHReYuigN/OB4Fy2hQ8BJMiwDOjxiDgOsLv81JtRmUfQhOURvfTSS6iurkZubi5KSkoglUp99h89ejSo89C6Z8Rg6C0OgOOQ6WwGIAbSEmdyQixikJkqg4hh4GR75QxJaenpUAlLiJYuXRplMwjCPwaLAwqnHqmMBRClJVaPaqCnTWxvqN4sZMISog0bNkTbDoLwi97igNZaz3dE1I7g+0InOhSsDpmwo35dXV34wx/+gPLycnR0dADgh2T19fVRM45ITDiOw4VWo8/6doHWuosUvcUBra0BcqkISCuOyXtEHRkNzUIlLCH6+uuvMXbsWLzwwgt48cUX0dXVBYDvVV1eXh5N+4gE5FyLEe9XNeDLC+0A+DYd71c14FRDdEtuOI5Dp8kOjbWBDwynl0T1/NHg83NtqNxfg5MN+p6NUiV/77AAfZJ9Cf+EJURr167FypUrce7cOZ+VXRcvXox9+/ZFzTgi8WBZDv8+57vc+MkGA2raTNh9qhkX26I7HLl/shyTc2VQKJSAOj+q544GFocLnWaHt0MAgJ6hGccBToswhiUZYQnRoUOH8P3vf7/f9hEjRqCpqSlio4jEpbbDzAeQpWLMKuFzyaYUajEhXwOW4/DBN41o6bZG5b0YhkGquQFpShnE6cUJkz/UG08ukbV3mYdI3FNjRlP4QRHWX1Yul/utfD979iyys7MjNopIXE438n/3CflqSMX8vw/DMPjOxFzoMlSwO1m8f6zB27ojYrrcjfbSEzM+5MklsvUt86Ap/JAIS4huv/12PPPMM3A4+H82hmFw6dIlPPnkk/iP//iPqBpIJA4sy6G2nf9ijc1V++wTixjcWpaPrFQZjDYn3j9W378GK0RO1LWjtvo03wYkQQPVnr7V1r6FrzSFHxJhCdFLL70Eo9GI7OxsWCwWXHvttRg9ejTUajWeffbZaNtIJAiNBiusDhcUUjHyNP3bWyikYiyZNgKpcgnajHbsOhnZML3+4hk0dHSjm1MAqTkRnStWBCx8lVJ2dSiElUek1Wqxe/du7N+/H8ePH4fRaMT06dOxcOHCaNtHJBC17XwgujhT1VNX1QeNQool0wrwwdeNmDsqM6L3Y9vOAwBkOWMSov+QPwK2AvFO4VMuUTCELEQsy6KyshI7duzAxYsXwTAMSktLkZeX17MGODEkmapLQ65GAZVMPOBxOWoFVswt8RGrug4z8rUKSMTBOeFGmxMKw0UwANQjxkVidkxRysTQKKVIlff5TDxT+OQRBUVIQsRxHG6//XZ8+OGHmDJlCiZPngyO43D69GmsXLkSO3bswHvvvRcjUwmhUckkGNV7xYoB6C1CzQYr3j1Wj8xUGW6ZnI80lWzQ17e1NEDhNEAul0KWNTJsm2NNjlqBh68u7b9DSh5RKIQkRJWVldi3bx/27NmDBQsW+Oz79NNPsXTpUrz55pt44IEHomokkdzYHCxkEhFaDDZs++oSls3QIVs9cIM7Y8NZAIA4vQiQJGEzPKrAD4mQgtVvvfUWfvKTn/QTIQC4/vrr8dRTT2Hr1q1RM45IHGrbTfiiug2N+tAT9IoyVVg+pwh5WgVsDhbvHav3NsQPhL2ZFyJZzpiw7BUcaa/VPIhBCUmIvv76a9x8880B9y9atAjHjx+P2Cgi8bjQasLBCx0412wM6/VqhRR3TBvhnd7feaIJLOu/Po2zm8HoLwEA0nX+F1JIJN49dhmV+2vQabL3bKRgdUiEJEQdHR3IzQ3cpjM3NxednZ0RG0UkHm1Gvn1vVmr4wySFVIxbywogk4hQ12HGVxc7/B7HtJ/H1EINJo8bjeycxCvr6EuX2YFOswPm3lP4Xo/Iyi8tRAxISELkcrkgkQQOK4nFYjidA7vcRPLBcRza3b/2WamDB5oHIj1FhgXj+JygDpPdf9V+6xkwYJBaOClgmkAiIZf4ySWSKntSDiipcVBCnjVbuXJlwJU0qOn90MRsd8Fid4FheCGJlAn5aqgVEhSmK/unezhtYDsu8L+QWYk7bd8bT3a1zdErl4hheDGym/mbXB3g1QQQohCtWLFi0GNoxmzo0W7kvaE0pdRbXxYJDMNAl6Hyu09/+TTO1rRBnZGLcapMJL4/NMhqHnYzxYmCICQh2rJlS6zsIBKYVnd8KDOC+FAgrA4X9pxuwbg8NUbnpKL22yNwshzalKUYn4DV9v7w1pv1W98sBTC10RR+EIS95DQxfOhwx4cyI4wP+ePYpS6cbe5GTZsRE7OkUNR/CwbAqEmzov5escITI/IZmgE0hR8CJETEoCwYl42ZxemQSqLvocwqSUeTwYKLbWY0nz+KEnDIyNMhvyAxlpUOhhS5BBqlFLK+nw9N4QcNCRExKBKxKCpB6kDnvn3KCJxs0EN8rA5ZWSnIKbsqJu8VK6bq0jBVl9Z/B1XgB42gg/CKigrMmjULarUaOTk5WLp0Kc6cOSOkSYQAiEUMytIcuCLVjFytCkxu4icxBgX1JAoaQYVo7969WLVqFQ4cOIDdu3fD4XDgxhtvhMlErmyioLc4sOtkE47U+k8+jBqNVfx95qieIU2yQ8sKBY2gQ7OdO3f6PK+srEROTg6OHDmCa665RiCriN60G2041WBAtlqOGcUZsXkTpx1o+pp/XDAtNu8RQ/QWBz78phEMgO/N7hXbkpJHFCwJFSPS6/klWTIy/P/D22w2n6RJf32zieiit/DtgLXKGC5s2HyCFyNVBpCRuC0/AiFigCa9FSKG8e3JReubBU3CJGqwLIs1a9Zg3rx5mDRpkt9jKioqoNVqvTedLnHWQR+qxFyIOA5oOMo/LpiesJ0YB8KT0MhyHOyuXlP4Ho/IaQdcUVpMYIiSMEK0atUqnDhxAtu3bw94THl5OfR6vfdWV1cXRwuHJx4hSlPFSIg6LgDGVkAsAfImx+Y9YoxExEDiromz2nsJkUTOXxcA2LoFsCx5SIih2erVq/HPf/4T+/btQ2FhYcDj5HJ5wDo3IjbE1CPiOKD2C/5xwbSetcCSDIZhoJSJ0W11wuJwQQupZwcgUwOWTsBu5IeehF8E9Yg4jsPq1avx7rvv4tNPP0VpqZ+Wm4RgcBwHvTmGQtR1CdBfBkQSQDcn+uePI0p3H2+zvU/3Cbm7ta4tvD5OwwVBPaJVq1Zh27ZteP/996FWq72rxGq1WiiVSiFNI8BX3bs4DiKGgVoRZSHiOODi5/zj/LKkr073LChg6Vtv5rkuOwnRQAgqRJs2bQIAXHfddT7bt2zZgpUrV8bfIMKHFLkEqxeMhtHmhDjafYHaz/MekUgCFF0Z3XMLgFouhUYpBdO3X4DM4xFRjGggBBUiv02xiIRCIhYFtepGSLAuoPpT/rFuFqDQRvf8ArBwYoDOpR6PiIRoQBJm1owYRtR+AZg7+BKIorlCWxNbPB4RDc0GJCFmzYjE5KuaDnSZ7Zg0QouCtCjF7Lqbe2bKRi9MzqWCQoGC1UFBQkQEpKbNiIYuK0qyolT75bACp94HOBbIHgvkDJHiVgCNegs+O9MKtUKCW8sKenZ4PaJuPkCfhAmb8YCEiAhIl3vqPi0aU/cuJy9C5nY+bjL25iH1pWQ5vszDYu/zWXliRC4n4LQlba5UrKEYEeEXu5OF2c5PRWsiFSKHFTjxDp9FLZYAk/5j6FTYu1FJA0zfi6U94kNxooCQR0T4xZNRrZCKvbVUIcOyQOu3wIV/AVYDL0KTlwGaxF+rLFQ8CY12Jwuni4Wk9yIDslRejG3dQEqWQBYmNiREhF9CqjGzmwFTK2DV8182z81Q39OvWZkGTFwCaAoGPFWyIpeIIGIYsBwHi8MFdW8hkqvdTfTJIwoECRHhl0FrzFwOoOkbvqFZd3PgE0mVwIgZgG72kJ4h4+vNRDDZ+DXgfDLRKalxUEiICL9Y3PEhv0LUegY4t9v3i6VM5xMTFRreA5ClAinZgGYEkCTLAkWKUibhhahfmQdN4Q8GCRHhl6vHZGHOyAy42F7Z7y4HcHYn0HSCf67QAoWzgJwJPV+2YYxGIYHdyYLtWzAg89SbkUcUCBIiIiBSsQjeOLVVD5z4Gz8MY0RA0RygeB4/K0QAAJZMHeF/h2cK30odRQNBQkQMjrEV+Ho7P7SQKoErlgLpJUJblTx4aulsJESBICEi+tFtdWDniSZkpspwfb4T+Oav/PRzShY//a5ME9rE5EKh4e/tZn54S15kP0iIiH50mR243GmBqLMGaDnIZwVrCoCyu3iPiPDLpXYz9le3ISNFhpuuyOvZIVHw4uNy8MOzlEzhjExQhsd0BhESeosDWstlTGjfzYtQRikw5R4SoUFwcRya9Fa0dtt8dzBMr+GZPv6GJQEkREQ/rK01GNf2MZRiDsgaww/HJLFZcnookSLnI/smm7P/To8QWUmI/EFCRPjS3YTUs+9CxDnBZY4GJi4FRGGWeAwzUmR8pMPicIHtO4fvFSIKWPuDhIjowWoAvnkbDrsNesUIsBOX9iyHQwyKSiZ2L7IImPslNboD1jRz5hcSIoKHdQEndwA2IzoZDc5mLYQ2hWJCocAwjLeJfr/hmWfmjIZmfiEhIngu/hswNMIplqMmbzFcInlsl5keoqTIeQ+yvxDR0GwgyO8m+NU0Lh0AAEjGL8aKnPFwuFhIxfQ7FSpapRRWhwv9loXwDs26+fYow6T+LlhIiIY7rAs4s5NvY5pfBuSMBwASoTC5pSxAryW5mg/6sy6+5mwIrFwSTei/bbhTf4Rv3ypTAaNuENqaoQvD9HhFli5BTUlESIiGMzYjHxsCgNJrAakC/z7XineOXEZNm0lY24YiynT+3tIprB0JiKBCtG/fPtx2220oKCgAwzB47733hDRn+FF3AHDaAXUekD8FANDYZUVdhxl2JyuwcclJo96Ct766hPer6vvvJCEKiKBCZDKZMGXKFLz22mtCmjE8sRmB+mP849JrvCtqdFnsAIJsEUv0Q8QwaNJb0WKw9d9JQhQQQYPVixYtwqJFi4Q0YfhSdwBg3cWsGSMB8I3fTbYBOjMSg6JW8F8po83Zv4k+CVFAkmrWzGazwWbr+aUxGCgnIyzsZqDB7Q2VXO31hqKycscwRykVQypm4HBxMNqcSFP1qtHrLUS02KIPSRWsrqiogFar9d50Op3QJiUnjcf5qnp1rtcbAgA9DcsihmEYb+P8bmufpEZlGi8+Lget6NGHpBKi8vJy6PV6762urk5ok5IP1sVP2QN8v+lev8pRXdl1GOMZnhmsDt8dInFP/hANz3xIqqGZXC6HXD50l6SJC61n+OxeWQqQPcFnF8PwhZta8ogiIqBHBPDDM0sXYO4A0oria1gCk1RCRESBy4f4+4Jp/SrrZxRnYEZxRv8WFkRIpKmkSFNJIRH5iQEpMwDUAJaOuNuVyAgqREajEefPn/c+r6mpQVVVFTIyMlBURL8WUUdfDxga+CFCwbSAh4n8fYGIoJlVkoFZJRn+d3qWnDa1xc+gJEBQITp8+DAWLFjgfb527VoAwIoVK1BZWSmQVUOYhqP8Pa1DJhwp2fy9qVVYOxIMQYXouuuuA8fRMCAu2M1Ay7f84xEz+u2+3GnGzhNNKMpQ4cbejd+J6OLxiKwGwGkb0stwh0JSzZoREdD0NZ/AqM4D1P0rxDtMdnRbnTDbXX5eTIQCx3F4+3AdXt9b3X/mTKrs8UbJK/JCQjQc4LieBMYR0/0m0rWb+ByijBRqkh8pDMPAZHPCYnehy+Tof4B3eEZxIg8kRMOBjgv8lLFEDuRM9HtIJwlRVEl3f46dZnv/nRSw7gcJ0XCg3h2kzisLuMpou5GEKJp4Sju6LAN5RC1xtCixISEa6li6gI5q/nGAKXuz3Qmju8dyZioJUTRIdyeFdvnziFLdkwHdjfywmSAhGvI0HOP/2dNLAi517FmZNF0lhVxCxa7RIN3tEXmGvD6kZPPJpE47n2FNkBANaVwOvsAV4IPUA1CQpkBBGi0fFC08hcN6C98OxAeRqJdX1BBnyxITKvEYyjR9AzgsfKFl5piAhxVnpqA4MyWOhg19UuUSZKTIoFFKYHP26UsEAJp8QH8ZMDQCeZOFMTKBICEaqnBcT11Z4SxavibOMAyDFVeVBD5AXcDfk0cEgIZmQ5f283z8QSLnlwkKgNPFwtp3eWQi9mjcSaXGFn4IPcwhIRqKcBxw6Uv+ccG0AcsI6rsseH1vtf9m70RUsDn9CL0ijV/rjHXxQ7RhDgnRUKTjAl9pL5IAhTMHPLS+ywKOA82WxYAusx1v7LuALfsv9q+pZBggvZh/3Hkx7rYlGiREQw2OA2r28Y9HTOd/dQegocvKH0ozZlEnVS6BxeGCxe7y9gP3Ib2Ev++qjatdiQgJ0VCj7SzQ3cRnUBddOeChLpZDk94CgJ++J6KLRCxCnob/XOu7LP0PSHN7RN1NfHeEYQwJ0VDC5QDO7+EfF87i28EOQEOXBQ4XB5VMTKUdMcKTm+XxPH1QaIDUHN6LbT8XZ8sSCxKiocSlLwGrnv8HL5o76OEX2/llpYszU8DQ0jYxweNp1ncG8Hiyx/H3rWfjZFFiQkI0VOhuAi4d4B+PugGQDO7hXHSvb1+aRcmMsaIgTQmGATrNjv69iQAgyy1EnTWAw4/XNEwgIRoKOO3Aqff5qeDssT2/sgPAcRymF6djbK4axZmqOBg5PFFIxd7hWXWLn7XMUrL4G+sCmk/G2brEgYQo2eE44Nt/8smLcjUwdlFQK4gyDIMrCrS4pSyfVnWNMVcUaDBVl+a/lo9heroiNB4bttX4JETJDMfxwenWM/zKHBNvB2Tk3SQaVxRosWB8DnI1AWYmc6/gq/GNrfwQbRhCQpSscBxwbndPPdn4W4JesK+mzYQvzrfBZPOzACARf6RKIN/tFdXsG5ZeEQlRMmI3A1//lV86mmGAsTfxv6pBwHEcvqxux8GaDlTVdcXWTsILx3Fo6LJgz+lm2J1s/wOKruS9IkMj3zVhmEFClExwHB/QPPxHvoxDJAEm3D5or6HenGwwoNlghUwiwlRdWuxsJXxgOeCjE034+rIeRy/5WfdengoUX80/Pr972DVMSwgheu2111BSUgKFQoE5c+bgq6++EtqkxILj+DyTY38GTv0dsBkBVSYwYwWQ678Zvj/ajDbsPcsvYXPlyAykyKkLTLwQixjMG813yPyqpgNNej9T9bo5gKaAnwX9+q98TtgwQXAh+r//+z+sXbsWGzZswNGjRzFlyhTcdNNNaGkZ5o3FOY5306v/BRx8HTjxN76QVSwBSq8BZj7EZ+UGyeVOM945chl2JwtdhgrTdOkxNJ7wx7hcNUbnpMLFcnj3WD0u901yFImASXfyjewsncCRSqDl9LCIGTGcwEutzpkzB7NmzcJvfvMbAADLstDpdPjP//xPPPXUUwO+1mAwQKvVQq/XQ6PRxMPc6MKygNPK3+xG/p/P0gl0NwOGy/wvoweJnJ/mLZw5aCErwMckPNnSZ5u78cHXjQCAXI0Cd0wbAaWMpuyFwOpw4b1j9WjUW8EwwOicVMwpzUS2ulerFksX/8NjdP8Yp2QB2eMB7QhAmcH//UXJ8fcL9jsqqG9ut9tx5MgRlJeXe7eJRCIsXLgQX375Zb/jbTYbbDab97nBYAjujTpqgPOf+N/n1WHOz7Ze24PdNug53fesq19DLLuLxelGg/dwViSFQVmIztRR0IuLMZbLwFVuEbI6XPjzl7XuM3LguJ53YzkOE/I1WDCO95gK05WQiBiMz9fgmrFZ1PJDQBRSMe6YPgJ7z7TiZIMB55qNGJ+n8QrRiXo9DlzogJibj3zTceR2VUHEdQFnzgMARmalQK2QAmIpWs0sajtt4BgROPA/OoUZKSjQKv3nkkW7jGfyXXw5URQQVIja2trgcrmQm5vrsz03Nxfffvttv+MrKirw9NNPh/5GTltiL2YnlgKyFLgkGtS022CWpsMoz4VJmgEw7tGzDT7LQXMcvEsA+aPb2rNPJZPgoatLKSaUIMglYtx4RR6mFaXjRIPep/OBzcl6/3Zd8ik4mz0emeYaaGwNSLW1otjz/+BygLVZ4TCbfM7Nya18OkA84PzM/oVJUv1nlpeXY+3atd7nBoMBOp1u8Bem6YAp3+vzi9DrsXe7v229tge7bdBzMvxjiYIfcrndbKmLxfRSq/ewnnsGDIAUWc+fSy4RYfmcoh4zwIBh+KcMw0DVZ+hFIpR4ZKvlXq/Vw4R8NXTpSrg4Dk6Xx8cd5d2vSJUBsAEuO1LMFhQZLQDn/oHiOKiVUkDu+dv789yjyCDdHUJB0P/OrKwsiMViNDc3+2xvbm5GXl5ev+Plcjnk8sBtTwMiSwEySsM1M25IxCIUBVn3JRIxyAmUqUskLSqZBCrZYF9LCYAUpCrTkep/qbqkQ9BZM5lMhhkzZmDPnj3ebSzLYs+ePZg7d/A2FgRBDA0E99fXrl2LFStWYObMmZg9ezZeeeUVmEwmPPjgg0KbRhBEnBBciO6++260trZi/fr1aGpqwtSpU7Fz585+AWyCIIYugucRRULS5xERxBAn2O+o4JnVBEEQJEQEQQgOCRFBEIIjeLA6EjzhraBLPQiCiCue7+ZgoeikFqLu7m4ACC67miAIweju7oZWqw24P6lnzViWRUNDA9RqdczW5fKUkdTV1Q2Lmbnhdr0AXXMsr5njOHR3d6OgoAAiUeBIUFJ7RCKRCIWFhXF5L41GM2z+SYHhd70AXXOsGMgT8kDBaoIgBIeEiCAIwSEhGgS5XI4NGzaEV/WfhAy36wXomhOBpA5WEwQxNCCPiCAIwSEhIghCcEiICIIQHBIigiAEh4QIoa00+8Ybb2D+/PlIT09Heno6Fi5cmHQr04a7su727dvBMAyWLl0aWwNjQKjX3NXVhVWrViE/Px9yuRxjx47Fhx9+GCdro0Oo1/zKK69g3LhxUCqV0Ol0eOKJJ2C1+lmRNhZww5zt27dzMpmM27x5M3fy5Enu0Ucf5dLS0rjm5ma/x997773ca6+9xh07dow7ffo0t3LlSk6r1XKXL1+Os+XhEer1eqipqeFGjBjBzZ8/n1uyZEl8jI0SoV6zzWbjZs6cyS1evJj7/PPPuZqaGu6zzz7jqqqq4mx5+IR6zVu3buXkcjm3detWrqamhtu1axeXn5/PPfHEE3Gxd9gL0ezZs7lVq1Z5n7tcLq6goICrqKgI6vVOp5NTq9Xcn/70p1iZGFXCuV6n08ldddVV3B/+8AduxYoVSSdEoV7zpk2buJEjR3J2uz1eJkadUK951apV3PXXX++zbe3atdy8efNiaqeHYT0086w0u3DhQu+2gVaa9YfZbIbD4UBGRkaszIwa4V7vM888g5ycHDz88MPxMDOqhHPNf//73zF37lysWrUKubm5mDRpEp577jm4XC6/xyca4VzzVVddhSNHjniHbxcuXMCHH36IxYsXx8XmpC56jZRQV5r1x5NPPomCggKfP3qiEs71fv755/jjH/+IqqqqOFgYfcK55gsXLuDTTz/F8uXL8eGHH+L8+fN4/PHH4XA4sGHDhniYHRHhXPO9996LtrY2XH311eA4Dk6nEz/4wQ/wk5/8JB4mU7A6Ep5//nls374d7777LhSKobfYYXd3N+6//3688cYbyMrKEtqcuMGyLHJycvD73/8eM2bMwN13342f/vSneP3114U2LWZ89tlneO655/Db3/4WR48exY4dO/DBBx/gF7/4RVzef1h7RKGuNNubF198Ec8//zw++eQTlJWVxdLMqBHq9VZXV+PixYu47bbbvNtYll/vXCKR4MyZMxg1alS/1yUS4fyN8/PzIZVKIRb3LNs9YcIENDU1wW63QyaTxdTmSAnnmn/2s5/h/vvvxyOPPAIAmDx5MkwmEx577DH89Kc/HbCXUDQY1h5RuCvN/upXv8IvfvEL7Ny5EzNnzoyHqVEh1OsdP348vvnmG1RVVXlvt99+OxYsWICqqqqk6IwZzt943rx5OH/+vFd0AeDs2bPIz89PeBECwrtms9ncT2w8QszFoxw1LiHxBGb79u2cXC7nKisruVOnTnGPPfYYl5aWxjU1NXEcx3H3338/99RTT3mPf/755zmZTMa98847XGNjo/fW3d0t1CWERKjX25dknDUL9ZovXbrEqdVqbvXq1dyZM2e4f/7zn1xOTg73y1/+UqhLCJlQr3nDhg2cWq3m3nrrLe7ChQvcxx9/zI0aNYq766674mLvsBcijuO4X//611xRUREnk8m42bNncwcOHPDuu/baa7kVK1Z4nxcXF3MA+t02bNgQf8PDJJTr7UsyChHHhX7NX3zxBTdnzhxOLpdzI0eO5J599lnO6XTG2erICOWaHQ4H9/Of/5wbNWoUp1AoOJ1Oxz3++ONcZ2dnXGylNiAEQQjOsI4REQSRGJAQEQQhOCREBEEIDgkRQRCCQ0JEEITgkBARBCE4JEQEQQgOCRGRVFRWViItLc37/Oc//zmmTp3qfb5y5cqk7CA53CEhIvyycuVKMAyDH/zgB/32rVq1CgzDYOXKlT7HR1sASkpK8Morr/hsu/vuu3H27NmAr9m4cSMqKyu9z6+77jqsWbMmqnYR0YeEiAiITqfD9u3bYbFYvNusViu2bduGoqIiQWxSKpXIyckJuF+r1fp4TERyQEJEBGT69OnQ6XTYsWOHd9uOHTtQVFSEadOmRXRuf57K0qVLvV7Wddddh9raWjzxxBNgGAYMwwDoPzTrS2/PbOXKldi7dy82btzoPUdNTQ1Gjx6NF1980ed1VVVVYBgG58+fj+i6iPAgISIG5KGHHsKWLVu8zzdv3owHH3ww5u+7Y8cOFBYW4plnnkFjYyMaGxtDPsfGjRsxd+5cPProo95zFBUV9bsmANiyZQuuueYajB49OlqXQIQACRExIPfddx8+//xz1NbWora2Fvv378d9990X8/fNyMiAWCyGWq1GXl7eoI3q/KHVaiGTyaBSqbznEIvFWLlyJc6cOePtz+xwOLBt2zY89NBD0b4MIkiGdYdGYnCys7Nxyy23oLKyEhzH4ZZbbkn6trEFBQW45ZZbsHnzZsyePRv/+Mc/YLPZsGzZMqFNG7aQR0QMykMPPYTKykr86U9/iprXIBKJ+nX+czgcUTl3MDzyyCPeQPyWLVtw9913Q6VSxe39CV9IiIhBufnmm2G32+FwOHDTTTdF5ZzZ2dk+cR+Xy4UTJ074HCOTySJewifQORYvXoyUlBRs2rQJO3fupGGZwNDQjBgUsViM06dPex8HQq/X91t2KDMz029v6+uvvx5r167FBx98gFGjRuHll19GV1eXzzElJSXYt28fvve970Eul4c1JCwpKcHBgwdx8eJFpKamIiMjAyKRyBsrKi8vx5gxYwbsUU7EHvKIiKDQaDTQaDQDHvPZZ59h2rRpPrenn37a77EPPfQQVqxYgQceeADXXnstRo4ciQULFvgc88wzz+DixYsYNWoUsrOzw7J73bp1EIvFmDhxIrKzs3Hp0iXvvocffhh2uz0us4DEwFCrWGLY8u9//xs33HAD6urq+i1GSMQXEiJi2GGz2dDa2ooVK1YgLy8PW7duFdqkYQ8NzYhhx1tvvYXi4mJ0dXXhV7/6ldDmECCPiCCIBIA8IoIgBIeEiCAIwSEhIghCcEiICIIQHBIigiAEh4SIIAjBISEiCEJwSIgIghAcEiKCIATn/wOv6qJ/gQsDcgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:08:23.746860Z", + "iopub.status.busy": "2024-02-29T06:08:23.746474Z", + "iopub.status.idle": "2024-02-29T06:08:23.982867Z", + "shell.execute_reply": "2024-02-29T06:08:23.981919Z" + }, + "papermill": { + "duration": 0.261068, + "end_time": "2024-02-29T06:08:23.985134", + "exception": false, + "start_time": "2024-02-29T06:08:23.724066", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAwQUlEQVR4nO3dfVxTV54/8A8JEEB50hBAREFRwSqi+BOwulblSUeLpdPiQ+VBB3dRZm1TRovT4lKdMq0WmXapbN0iasfqjGOtqxakUbo+gFqsFh9AQRAfCIoIEWhDSM7vD5c7jQkSEAzcfN+vV170nnvu4Rwv+fbee849x4wxxkAIITwkMHYFCCGkt1CAI4TwFgU4QghvUYAjhPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwlrmxK9AXaTQa3L17F7a2tjAzMzN2dQghT2CM4dGjRxgyZAgEgo6v0yjA6XH37l24u7sbuxqEkE7cunULQ4cO7XA/BTg9bG1tATz+x7OzszNybXqPSqXC0aNHERoaCgsLC2NXhzwjUzqfCoUC7u7u3He1IxTg9Gi/LbWzs+N9gLOxsYGdnR3vvxCmwBTPZ2ePkKiTgRDCWxTgCCG8RQGOEMJbFOAIIbxFAY4QwlsU4AghvEXDRAjpp1paWlBaWsptN/2sxOmSCjiKf8BAa5FWXm9vb9jY2DzvKhodBThC+qnS0lL4+/vrpH+kJ29xcTEmTZrU+5XqYyjAEdJPeXt7o7i4mNsuq2mA9O8lSH9tPMa4OujkNUUU4Ajpp2xsbLSuygQ3H0B04mf4jJsAv+GDjVizvoM6GQghvEUBjhDCWxTgCCG8RQGOEMJbFOAIIbxFAY4QwlsU4AghvEUBjhDCWxTgCCG8ZfQAl5mZCQ8PD1hZWSEgIABnz559av6GhgasWrUKrq6uEIlEGD16NI4cOcLt/4//+A+YmZlpfUz1NRVCTJ1RX9Xau3cvpFIpsrKyEBAQgIyMDISFhaGsrAwSiUQnf2trK0JCQiCRSLBv3z64ubnh5s2bcHBw0Mr3wgsv4LvvvuO2zc3pjTRCTJFRv/np6emIj49HXFwcACArKwuHDx9GdnY23nnnHZ382dnZqK+vx+nTp7lVgzw8PHTymZubw8XFpVfrTgjp+4wW4FpbW1FcXIzk5GQuTSAQIDg4GIWFhXqPOXjwIIKCgrBq1Sp88803cHJywuLFi7F27VoIhUIu3/Xr1zFkyBBYWVkhKCgIaWlpGDZsWId1USqVUCqV3LZCoQDweBk2lUr1rE3ts9rbxuc2mpK2tjbuJ9/PqaHtM1qAq6urg1qthrOzs1a6s7Oz1iR+v3bjxg0cO3YMS5YswZEjR1BeXo6VK1dCpVJh/fr1AICAgADk5ORgzJgxqKmpQWpqKqZPn45Lly51uEhsWloaUlNTddKPHj1qEpME5ufnG7sKpAfcagIAcxQVFeHOJWPXpne1tLQYlK9fPZzSaDSQSCT4/PPPIRQK4e/vjzt37mDTpk1cgJszZw6X39fXFwEBARg+fDj+9re/Yfny5XrLTU5OhlQq5bbbV80ODQ3l1cLPLS0tKCsr47abflYi78Q5hE3/fzozwI4ZM8YkgjufXKyuB0p+QGBgICYMG2Ts6vSq9ruszhgtwInFYgiFQtTW1mql19bWdvj8zNXVFRYWFlq3oz4+PpDL5WhtbYWlpaXOMQ4ODhg9ejTKy8s7rItIJIJIJNJJt7Cw4NUK4RUVFQgICNBJpxlg+aG9M83c3JxXf7f6GNo+owU4S0tL+Pv7QyaTYcGCBQAeX6HJZDIkJibqPebFF1/E7t27odFoIBA8HuFy7do1uLq66g1uANDU1ISKigosXbq0V9rRn9AMsMTkMCPas2cPE4lELCcnh125coWtWLGCOTg4MLlczhhjbOnSpeydd97h8ldXVzNbW1uWmJjIysrK2KFDh5hEImEbN27k8rz99tusoKCAVVZWslOnTrHg4GAmFovZvXv3DK5XY2MjA8AaGxt7rrF90I9VdWz42kPsx6o6Y1eF9ABTOp+GfkeN+gwuKioK9+/fR0pKCuRyOfz8/JCbm8t1PFRXV3NXagDg7u6OvLw8vPXWW/D19YWbmxtWr16NtWvXcnlu376NRYsW4cGDB3BycsK0adNQVFQEJyen594+QohxGb2TITExscNb0oKCAp20oKAgFBUVdVjenj17eqpqhJB+zuivahFCSG+hAEcI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtow8TIYQYrrKuGc3KNr37Ku43cz+fNgfiAJE5PMUDeqV+fQ0FOEL6icq6ZszcXNBpvrf3lXSa53jSSyYR5CjAEdJPtF+5ZUT5wUsyUHf/z0ocKijEvJeCMMBad/IIACi/14Q3917o8CqQbyjAEdLPeEkGYpybvU66SqWC3AmYNNyR97OJGIo6GQghvEUBjhDCWxTgCCG8RQGOEMJbFOAIIbxFAY4QwlsU4AghvEXj4Hjsaa/1APRqD+E/CnA8ZehrPQC92kP4y+gBLjMzE5s2bYJcLseECRPw6aefYsqUKR3mb2howB//+Efs378f9fX1GD58ODIyMjB37txul8lHnb3WA9CrPYT/jBrg9u7dC6lUiqysLAQEBCAjIwNhYWEoKyuDRCLRyd/a2oqQkBBIJBLs27cPbm5uuHnzJhwcHLpdJt919FoPQK/2EP4zaidDeno64uPjERcXh7FjxyIrKws2NjbIzs7Wmz87Oxv19fU4cOAAXnzxRXh4eGDGjBmYMGFCt8skhPCX0a7gWltbUVxcjOTkZC5NIBAgODgYhYWFeo85ePAggoKCsGrVKnzzzTdwcnLC4sWLsXbtWgiFwm6VCQBKpRJKpZLbVigUAB5f4ahUqmdtqlG0tbVxPztqQ3v609poSDnk+ejsXJjS+TS07kYLcHV1dVCr1dwiz+2cnZ1RWlqq95gbN27g2LFjWLJkCY4cOYLy8nKsXLkSKpUK69ev71aZAJCWlobU1FSd9KNHj8LGxqYbrTO+W00AYI6TJ0/ipv5HcJz8/PweKYf0LkPPhSmcz5aWFoPyGb2ToSs0Gg0kEgk+//xzCIVC+Pv7486dO9i0aRPWr1/f7XKTk5MhlUq5bYVCAXd3d4SGhsLOzq4nqv7cXb6rwOaSIkybNg0vDNHfBpVKhfz8fISEhHT4DM6Qcsjz0dm5MKXz2X6X1RmjBTixWAyhUIja2lqt9NraWri4uOg9xtXVFRYWFhAKhVyaj48P5HI5Wltbu1UmAIhEIohEur2IFhYW/fbhe/u4NnNz807b8LR2dqUc0rsMPRemcD4NrbvROhksLS3h7+8PmUzGpWk0GshkMgQFBek95sUXX0R5eTk0Gg2Xdu3aNbi6usLS0rJbZRJC+MuovahSqRTbtm3Djh07cPXqVSQkJKC5uRlxcXEAgOjoaK0Og4SEBNTX12P16tW4du0aDh8+jA8++ACrVq0yuExCiOkw6jO4qKgo3L9/HykpKZDL5fDz80Nubi7XSVBdXQ2B4J8x2N3dHXl5eXjrrbfg6+sLNzc3rF69GmvXrjW4TEKI6TB6J0NiYiISExP17isoKNBJCwoKQlFRUbfLJISYDppNhBDCWxTgCCG8ZfRbVEKIYZTqXyCwuoNKRRkEVrqjdNva2nC37S6u1l/tcPqrSkUTBFZ3oFT/AkD/O8p8QgGOkH7ibvNNDPD8FOvOPj3fZ7mfPXX/AE/gbrMf/MH/jjcKcIT0E0MGDEdz5e/xlyg/jNQzBVZbWxtOnTyFF6e92OEVXMW9JqzeewFDZg7v7er2CRTgCOknREIraH5xg6fdGIwdrH9l+0rzSvgM8ulwpL/ml0ZofrkPkdCqt6vbJ1AnAyGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4iwb68lRn7y0C9O4i4T8KcDxl6HuLAL27SPiLAhxPdfbeIkDvLhL+owDHU529twjQu4uE/6iTgRDCW30iwGVmZsLDwwNWVlYICAjA2bMdPzjKycmBmZmZ1sfKSvvqIjY2VidPeHh4bzeDENLHGP0Wde/evZBKpcjKykJAQAAyMjIQFhaGsrIySCQSvcfY2dmhrKyM2zYzM9PJEx4eju3bt3Pb+hZ2JoTwW5ev4G7cuNGjFUhPT0d8fDzi4uIwduxYZGVlwcbGBtnZ2R0eY2ZmBhcXF+6jb0lAkUiklcfR0bFH600I6fu6fAXn5eWFGTNmYPny5fjtb3+rc3vYFa2trSguLtZa3FkgECA4OBiFhYUdHtfU1IThw4dDo9Fg0qRJ+OCDD/DCCy9o5SkoKIBEIoGjoyNmzZqFjRs3YvDgwXrLUyqVUCqV3LZCoQDw+CG8SqXqdvuMqa2tjfvZURva05/WRkPKIc9HZ+fClM6noXXvcoA7f/48tm/fDqlUisTERERFRWH58uWYMmVKlytZV1cHtVqtcwXm7OyM0tJSvceMGTMG2dnZ8PX1RWNjIzZv3oypU6fi8uXLGDp0KIDHt6eRkZHw9PRERUUF1q1bhzlz5qCwsBBCoVCnzLS0NKSmpuqkHz16FDY2Nl1uV19wqwkAzHHy5Enc1D9KhJOfn98j5ZDeZei5MIXz2dLSYlA+M8YY684vaGtrw8GDB5GTk4Pc3FyMHj0ay5Ytw9KlS+Hk5GRQGXfv3oWbmxtOnz6NoKAgLn3NmjX4/vvvcebMmU7LUKlU8PHxwaJFi7Bhwwa9eW7cuIGRI0fiu+++w+zZs3X267uCc3d3R11dHezs7AxqS19z+a4CC7YW4UBCIF4Yor8NKpUK+fn5CAkJ6XCYiCHlkOejs3NhSudToVBALBajsbHxqd/RbncymJubIzIyEr/5zW/w2WefITk5GUlJSVi3bh1ef/11fPjhh3B1dX1qGWKxGEKhELW1tVrptbW1cHFxMageFhYWmDhxIsrLyzvMM2LECIjFYpSXl+sNcCKRSG8nhIWFRYd/KH1d+8Bdc3PzTtvwtHZ2pRzSuww9F6ZwPg2te7eHifzwww9YuXIlXF1dkZ6ejqSkJFRUVCA/Px93795FREREp2VYWlrC398fMpmMS9NoNJDJZFpXdE+jVqtRUlLy1GB6+/ZtPHjwoNOASwjhly5fwaWnp2P79u0oKyvD3LlzsXPnTsydOxcCweNY6enpiZycHHh4eBhUnlQqRUxMDCZPnowpU6YgIyMDzc3NiIuLAwBER0fDzc0NaWlpAID3338fgYGB8PLyQkNDAzZt2oSbN2/id7/7HYDHHRCpqal49dVX4eLigoqKCqxZswZeXl4ICwvranMJIf1YlwPc1q1bsWzZMsTGxnZ4RSSRSPDFF18YVF5UVBTu37+PlJQUyOVy+Pn5ITc3l+t4qK6u5oInADx8+BDx8fGQy+VwdHSEv78/Tp8+jbFjxwIAhEIhfvrpJ+zYsQMNDQ0YMmQIQkNDsWHDBhoLR4iJ6XKAy8/Px7Bhw7SCDgAwxnDr1i0MGzYMlpaWiImJMbjMxMREJCYm6t1XUFCgtb1lyxZs2bKlw7Ksra2Rl5dn8O8mhPBXl5/BjRw5EnV1dTrp9fX18PT07JFKEUJIT+hygOtoVElTU9MzDfolhJCeZvAtqlQqBfD4NamUlBStAbBqtRpnzpyBn59fj1eQEEK6y+AA9+OPPwJ4fAVXUlICS0tLbp+lpSUmTJiApKSknq8hIYR0k8EB7vjx4wCAuLg4/OUvf+m3I/wJIaajy72ov56CiBBC+jKDAlxkZCRycnJgZ2eHyMjIp+bdv39/j1SMEEKelUEBzt7enptU0t6elo4jhPQPBgW4X9+W0i0qIaS/6BNrMhBCSG8w6Apu4sSJetc90Of8+fPPVCFCCOkpBgW4BQsW9HI1CCGd+VmlBgBcutOod3/zz0r8cB9wufkQA6z1TyxRfq+p1+rXFxkU4NavX9/b9SCEdKLi/4LTO/tLnpLLHLvKz3Va1gCR0RfUey5Mo5WE8EDoC49nuR4pGQhrC921RcpqGvH2vhJ8/NvxGOPa8WiHASJzeIoH9Fo9+xKDAtygQYNw7do1iMViODo6PvV5XH19fY9VjhDyT4MGWGLhlGEd7m9fMWuk0wCMc6PhXICBAW7Lli2wtbXl/tvQDgdCCDEmgwLcryevjI2N7a26EEJIj+ryODihUIh79+7ppD948EDvmqOEEGIsPTbhpVKp1JpCiRBCjM3gXtRPPvkEwOMJL//7v/8bAwf+c1lstVqN//3f/4W3t3e3KpGZmYlNmzZBLpdjwoQJ+PTTTzFlyhS9eXNycrgVt9qJRCL88ssv3DZjDOvXr8e2bdvQ0NCAF198EVu3bsWoUaO6VT9CSP9kcIBrX+iFMYasrCyt21FLS0t4eHggKyuryxXYu3cvpFIpsrKyEBAQgIyMDISFhaGsrAwSiUTvMXZ2digrK+O2n+z0+Oijj/DJJ59gx44d8PT0xHvvvYewsDBcuXKFplUnxIQYHOAqKysBADNnzsT+/fvh6OjYIxVIT09HfHw8d1WWlZWFw4cPIzs7G++8847eY8zMzODi4qJ3H2MMGRkZePfdd7nFp3fu3AlnZ2ccOHAACxcu7JF6E0L6vi4/gzt+/HiPBbfW1lYUFxcjODj4nxUSCBAcHIzCwsIOj2tqasLw4cPh7u6OiIgIXL58mdtXWVkJuVyuVaa9vT0CAgKeWiYhhH+6/CbDsmXLnro/Ozvb4LLq6uqgVqu5RZ7bOTs7o7S0VO8xY8aMQXZ2Nnx9fdHY2IjNmzdj6tSpuHz5MoYOHQq5XM6V8WSZ7fuepFQqoVQquW2FQgEAUKlUUKlUBrenL2kf9NnW1tZhG9rTn9ZGQ8ohfYMpnStD29flAPfw4UOdX3Tp0iU0NDRg1qxZXS2uy4KCghAUFMRtT506FT4+Pviv//ovbNiwoVtlpqWlITU1VSf96NGjWquH9Se3mgDAHCdPnsTNgU/Pm5+f3yPlEONqP1dFRUW4c8nYteldLS0tBuXrcoD7+uuvddI0Gg0SEhIwcuTILpUlFoshFApRW1urlV5bW9vhM7YnWVhYYOLEiSgvLwcA7rja2lq4urpqldnRsobJycncsojA4ys4d3d3hIaG9tvFdS7fVWBzSRGmTZuGF4bob4NKpUJ+fj5CQkJgYWHR7XJI33Cxuh4o+QGBgYGYMGyQsavTq9rvsjrTIy/bCwQCSKVSvPTSS1izZo3Bx1laWsLf3x8ymYybkkmj0UAmkyExMdGgMtRqNUpKSjB37lwAgKenJ1xcXCCTybiAplAocObMGSQkJOgtQyQSQSTSnV7GwsKiwy9+X6dij3uWS2ubYW6u/zRz0+vcbepwep2q+sfDb8zNzfvtv4WpaD/PpnCuDG1fj80mUlFRwT0D6AqpVIqYmBhMnjwZU6ZMQUZGBpqbm7le1ejoaLi5uSEtLQ0A8P777yMwMBBeXl5oaGjApk2bcPPmTfzud78D8LiH9c0338TGjRsxatQobpjIkCFDTGpeO8Om1gFoeh3CZ13+q/31rRzweFhGTU0NDh8+rPXOqqGioqJw//59pKSkQC6Xw8/PD7m5uVwnQXV1NQSCf3b2Pnz4EPHx8ZDL5XB0dIS/vz9Onz6NsWPHcnnWrFmD5uZmrFixAg0NDZg2bRpyc3NNagxcZ1PrADS9DuE/M9bRu1cdmDlzpta2QCCAk5MTZs2ahWXLlnV4O9SfKBQK2Nvbo7Gxsd8+gzPEhZsPsGBrEQ4kBMJv+GBjV4c8I1M6n4Z+R7scjdpXuCeEkL6OVtUihPAWBThCCG9RgCOE8BYFOEIIb/VYgLt9+zZWrFjRU8URQsgz67EA9+DBA3zxxRc9VRwhhDwzukUlhPAWBThCCG9RgCOE8JbBbzJERkY+dX9DQ8Oz1oUQQnqUwQHO3r7jl7Hb90dHRz9zhQghpKcYHOC2b9/em/UghJAeR8/gCCG8ZfAVXGeLzbTryqIzhBDSmwwOcDk5ORg+fDgmTpyILk4hRwghRmFwgEtISMBXX32FyspKxMXF4Y033sCgQfxe2IIQ0r8Z/AwuMzMTNTU1WLNmDf7nf/4H7u7ueP3115GXl0dXdISQPqlLnQwikQiLFi1Cfn4+rly5ghdeeAErV66Eh4cHmpqaequOhBDSLd3uRRUIBDAzMwNjDGq1+pkqkZmZCQ8PD1hZWSEgIABnz5416Lg9e/bAzMxMZ7Ws2NhYmJmZaX3Cw8OfqY6EkP6nSwFOqVTiq6++QkhICEaPHo2SkhL853/+J6qrqzFwYPeWPd+7dy+kUinWr1+P8+fPY8KECQgLC8O9e/eeelxVVRWSkpIwffp0vfvDw8NRU1PDfb766qtu1Y8Q0n8ZHOBWrlwJV1dX/PnPf8a8efNw69Yt/P3vf8fcuXO1lvXrqvT0dMTHxyMuLg5jx45FVlYWbGxsnjrcRK1WY8mSJUhNTcWIESP05hGJRHBxceE+jo6O3a4jIaR/MrgXNSsrC8OGDcOIESPw/fff4/vvv9ebb//+/Qb/8tbWVhQXFyM5OZlLEwgECA4ORmFhYYfHvf/++5BIJFi+fDlOnDihN09BQQEkEgkcHR0xa9YsbNy4EYMH83spNUKINoMDXHR0NMzMzHr0l9fV1UGtVnOLPLdzdnZGaWmp3mNOnjyJL774AhcuXOiw3PDwcERGRsLT0xMVFRVYt24d5syZg8LCQgiFuosgK5VKKJVKbluhUAAAVCoVVCpVN1rWP7S1tXE/+dxOU2FK59PQ9nVpoK+xPXr0CEuXLsW2bdsgFos7zLdw4ULuv8ePHw9fX1+MHDkSBQUFmD17tk7+tLQ0pKam6qQfPXoUNjY2PVP5PuhWEwCYo6ioCHcuGbs25FmZ0vlsaWkxKJ9Rl6EXi8UQCoWora3VSq+trYWLi4tO/oqKClRVVWH+/PlcmkajAQCYm5ujrKwMI0eO1DluxIgREIvFKC8v1xvgkpOTIZVKuW2FQgF3d3eEhobyemX7i9X1QMkPCAwMxIRhNGi7vzOl89l+l9UZowY4S0tL+Pv7QyaTcUM9NBoNZDIZEhMTdfJ7e3ujpKREK+3dd9/Fo0eP8Je//AXu7u56f8/t27fx4MEDuLq66t0vEokgEol00i0sLGBhYdHFVvUf5ubm3E8+t9NUmNL5NLR9Rg1wACCVShETE4PJkydjypQpyMjIQHNzM+Li4gA8fvbn5uaGtLQ0WFlZYdy4cVrHOzg4AACX3tTUhNTUVLz66qtwcXFBRUUF1qxZAy8vL4SFhT3XthFCjMvoAS4qKgr3799HSkoK5HI5/Pz8kJuby3U8VFdXd2kYilAoxE8//YQdO3agoaEBQ4YMQWhoKDZs2KD3Ko0Qwl9GD3AAkJiYqPeWFHg83ONpnuz8sLa2Rl5eXg/VjBDSn9GEl4QQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4S0KcIQQ3qIARwjhLQpwhBDeogBHCOEtCnCEEN6iAEcI4a0+MWU5eT5aWlq0FtQuq2mAUl6Oq5esoXngoJXX29ub12vCEtNAAc6ElJaWwt/fXyd98Q7dvMXFxZg0adJzqBUhvadP3KJmZmbCw8MDVlZWCAgIwNmzZw06bs+ePTAzM+PWVG3HGENKSgpcXV1hbW2N4OBgXL9+vRdq3r94e3ujuLiY+3x/8jTWbPgY3588rZVeXFwMb29vY1eXkGdm9Cu4vXv3QiqVIisrCwEBAcjIyEBYWBjKysogkUg6PK6qqgpJSUmYPn26zr6PPvoIn3zyCXbs2AFPT0+89957CAsLw5UrV2BlZdWbzenTbGxstK7KVCoVHtbdQ9CUybxfKJiYJqNfwaWnpyM+Ph5xcXEYO3YssrKyYGNjg+zs7A6PUavVWLJkCVJTUzFixAitfYwxZGRk4N1330VERAR8fX2xc+dO3L17FwcOHOjl1hBC+hKjXsG1traiuLgYycnJXJpAIEBwcDAKCws7PO7999+HRCLB8uXLceLECa19lZWVkMvlCA4O5tLs7e0REBCAwsJCLFy4UKc8pVIJpVLJbSsUCgCPr3BUKlW329fXtbeNz200JW1tbdxPvp9TQ9tn1ABXV1cHtVrNrWLfztnZWau379dOnjyJL774AhcuXNC7Xy6Xc2U8WWb7vielpaUhNTVVJ/3o0aMm0ZOYn59v7CqQHnCrCQDMUVRUhDuXjF2b3tXS0mJQPqM/g+uKR48eYenSpdi2bRvEYnGPlZucnAypVMptKxQKuLu7IzQ0FHZ2dj32e/oalUqF/Px8hISE0DM4HrhYXQ+U/IDAwEBMGDbI2NXpVe13WZ0xaoATi8UQCoWora3VSq+trYWLi4tO/oqKClRVVWH+/PlcmkajAQCYm5ujrKyMO662thaurq5aZfr5+emth0gkgkgk0km3sLAwiS++qbST78zNzbmffD+fhrbPqJ0MlpaW8Pf3h0wm49I0Gg1kMhmCgoJ08nt7e6OkpAQXLlzgPi+//DJmzpyJCxcuwN3dHZ6ennBxcdEqU6FQ4MyZM3rLJITwl9FvUaVSKWJiYjB58mRMmTIFGRkZaG5uRlxcHAAgOjoabm5uSEtLg5WVFcaNG6d1vIODAwBopb/55pvYuHEjRo0axQ0TGTJkiM54OUIIvxk9wEVFReH+/ftISUmBXC6Hn58fcnNzuU6C6upqCARdu9Bcs2YNmpubsWLFCjQ0NGDatGnIzc016TFwhJgiM8YYM3Yl+hqFQgF7e3s0NjbyvpPhyJEjmDt3Lu+f2ZiCCzcfYMHWIhxICITf8MHGrk6vMvQ7avSBvoQQ0lsowBFCeIsCHCGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4iwIcIYS3KMARQniLAhwhhLcowBFCeIsCHCGEtyjAEUJ4y+gz+hJCuqelpUVrec2ymgYo5eW4eskamgcOWnm9vb1NYgnMJ1GAI6SfKi0thb+/v0764h26eYuLizFp0qTnUKu+hQIcIf2Ut7c3iouLue2mn5U4fLwQv5kZhIHWIp28pqhPPIPLzMyEh4cHrKysEBAQgLNnz3aYd//+/Zg8eTIcHBwwYMAA+Pn5YdeuXVp5YmNjYWZmpvUJDw/v7WYQ8lzZ2Nhg0qRJ3CdoymRMHT8SQVMma6VPmjTJJG9PgT5wBbd3715IpVJkZWUhICAAGRkZCAsLQ1lZGSQSiU7+QYMG4Y9//CO8vb1haWmJQ4cOIS4uDhKJBGFhYVy+8PBwbN++ndvWt7AzIYTfjH4Fl56ejvj4eMTFxWHs2LHIysqCjY0NsrOz9eZ/6aWX8Morr8DHxwcjR47E6tWr4evri5MnT2rlE4lEcHFx4T6Ojo7PozmEkD7EqFdwra2tKC4uRnJyMpcmEAgQHByMwsLCTo9njOHYsWMoKyvDhx9+qLWvoKAAEokEjo6OmDVrFjZu3IjBg/UvpaZUKqFUKrlthUIB4PGyeiqVqjtN6xfa28bnNpoSUzqfhrbRqAGurq4OarWaW+S5nbOzs1b395MaGxvh5uYGpVIJoVCIzz77DCEhIdz+8PBwREZGwtPTExUVFVi3bh3mzJmDwsJCCIVCnfLS0tKQmpqqk3706FGTeHaRn59v7CqQZ6RWq3HlyhU8fPgQJSUlGDt2rN6/db5oaWkxKJ/Rn8F1h62tLS5cuICmpibIZDJIpVKMGDECL730EgBg4cKFXN7x48fD19cXI0eOREFBAWbPnq1TXnJyMqRSKbetUCjg7u6O0NBQ3i/8nJ+fj5CQEFr4uR/7+uuvsXbtWlRVVXFpHh4e+PDDD/HKK68Yr2K9qP0uqzNGDXBisRhCoRC1tbVa6bW1tXBxcenwOIFAAC8vLwCAn58frl69irS0NC7APWnEiBEQi8UoLy/XG+BEIpHeTggLCwuT+OKbSjv5aP/+/Vi4cCHmzZuHXbt24fbt2xg6dCg++ugjLFy4EPv27UNkZKSxq9njDP17NWong6WlJfz9/SGTybg0jUYDmUyGoKAgg8vRaDRaz9CedPv2bTx48ACurq7PVF9C+hK1Wo23334b8+bNw4EDBxAQEABra2sEBATgwIEDmDdvHpKSkqBWq41dVaMxei+qVCrFtm3bsGPHDly9ehUJCQlobm5GXFwcACA6OlqrEyItLQ35+fm4ceMGrl69io8//hi7du3CG2+8AQBoamrCH/7wBxQVFaGqqgoymQwRERHw8vLSGkZCSH934sQJVFVVYd26dRAItL/KAoEAycnJqKysxIkTJ4xUQ+Mz+jO4qKgo3L9/HykpKZDL5fDz80Nubi7X8VBdXa118pqbm7Fy5Urcvn0b1tbW8Pb2xpdffomoqCgAgFAoxE8//YQdO3agoaEBQ4YMQWhoKDZs2EBj4Qiv1NTUAADGjRund397ens+U2T0AAcAiYmJSExM1LuvoKBAa3vjxo3YuHFjh2VZW1sjLy+vJ6tHSJ/U/sjl0qVLCAwM1Nl/6dIlrXymyOi3qISQ7pk+fTo8PDzwwQcfQKPRaO3TaDRIS0uDp6cnpk+fbqQaGl+fuIIjhHSdUCjExx9/jN/+9reIiIhASEgIrl+/jps3byI/Px+HDx/Gvn37eD0erjMU4AjpxyIjI5GUlIQtW7bg0KFDXLq5uTmSkpJ4OUSkKyjAEdKP7d+/H5s3b8ZvfvMb7gpu1KhRyM/Px+bNmxEYGGjSQc6MMcaMXYm+RqFQwN7eHo2Njbx/k+HIkSOYO3cuDfTth9RqNby8vDB+/HgcOHAAarWaO59CoRALFizApUuXcP36dd7dphr6HaVOBkL6KRoH1zkKcIT0UzQOrnMU4Ajpp349Dk4fGgdHAY6QfovGwXWOAhwh/VT7OLhDhw5hwYIFKCoqws8//4yioiIsWLAAhw4dwubNm3nXwdAVNEyEkH4sMjIS+/btw9tvv41/+Zd/4dI9PT15O1VSV1CAI6Sfi4yMREREBI4fP45vv/0Wc+bMwcyZM036yq0dBThCeEAoFGLGjBlobm7GjBkzKLj9H3oGRwjhLQpwhBDeogBHCOEteganR/vruYau3NNfqVQqtLS0QKFQ0LuoPGBK57P9u9nZq/QU4PR49OgRAMDd3d3INSGEPM2jR49gb2/f4X6aTUQPjUaDu3fvwtbWFmZmZsauTq9pX//11q1bvJ41xVSY0vlkjOHRo0cYMmSIzkQDv0ZXcHoIBAIMHTrU2NV4buzs7Hj/hTAlpnI+n3bl1o46GQghvEUBjhDCWxTgTJhIJML69etpvVieoPOpizoZCCG8RVdwhBDeogBHCOEtCnCEEN6iAEcI4S0KcDzx0ksv4c033zR2NUg39LVz19fq8ywowBHCA62trcauQt/ESL8XExPDAGh93Nzc2GeffaaV7/z588zMzIxVVVUxxhj7+OOP2bhx45iNjQ0bOnQoS0hIYI8ePdI65sSJE2zatGnMysqKDR06lP3+979nTU1Nz61tfKfv3JWXl7Nly5YxDw8PZmVlxUaPHs0yMjJ0jouIiGAbN25krq6uzMPDgzHG2KlTp9iECROYSCRi/v7+7Ouvv2YA2I8//sgdW1JSwsLDw9mAAQOYRCJhb7zxBrt//36H9amsrHxe/xw9jgIcDzQ0NLCgoCAWHx/PampqWE1NDUtKSmLTpk3Tyvf2229rpW3ZsoUdO3aMVVZWMplMxsaMGcMSEhK4/eXl5WzAgAFsy5Yt7Nq1a+zUqVNs4sSJLDY29rm1je/0nbtffvmFpaSksHPnzrEbN26wL7/8ktnY2LC9e/dyx8XExLCBAweypUuXskuXLrFLly6xxsZGNmjQIPbGG2+wy5cvsyNHjrDRo0drBbiHDx8yJycnlpyczK5evcrOnz/PQkJC2MyZMzusT1tbmzH+aXoEBTiemDFjBlu9ejW3/eOPPzIzMzN28+ZNxhhjarWaubm5sa1bt3ZYxt///nc2ePBgbnv58uVsxYoVWnlOnDjBBAIB+/nnn3u2ASbsyXOnz6pVq9irr77KbcfExDBnZ2emVCq5tK1bt7LBgwdrnZtt27ZpBbgNGzaw0NBQrbJv3brFALCysjKD69Nf0DM4nvLz84OPjw92794NAPj+++9x7949vPbaa1ye7777DrNnz4abmxtsbW2xdOlSPHjwAC0tLQCAixcvIicnBwMHDuQ+YWFh0Gg0qKysNEq7TEVmZib8/f3h5OSEgQMH4vPPP0d1dbVWnvHjx8PS0pLbLisrg6+vL6ysrLi0KVOmaB1z8eJFHD9+XOucent7AwAqKip6sUXGQQGOx5YsWcIFuN27dyM8PByDBw8GAFRVVWHevHnw9fXFP/7xDxQXFyMzMxPAPx9YNzU14V//9V9x4cIF7nPx4kVcv34dI0eONE6jTMCePXuQlJSE5cuX4+jRo7hw4QLi4uJ0OhIGDBjQ5bKbmpowf/58rXN64cIFXL9+XWtdVb6g+eB4wtLSEmq1Witt8eLFePfdd1FcXIx9+/YhKyuL21dcXAyNRoOPP/6YmzDwb3/7m9bxkyZNwpUrV+Dl5dX7DTBhT567U6dOYerUqVi5ciWXZsjV1ZgxY/Dll19CqVRyL9yfO3dOK8+kSZPwj3/8Ax4eHjA31//11/e31F/RFRxPeHh44MyZM6iqqkJdXR00Gg08PDwwdepULF++HGq1Gi+//DKX38vLCyqVCp9++ilu3LiBXbt2aQVAAFi7di1Onz6NxMRE7v/y33zzDRITE59383jtyXM3atQo/PDDD8jLy8O1a9fw3nvv6QQqfRYvXgyNRoMVK1bg6tWryMvLw+bNmwGAm5l61apVqK+vx6JFi3Du3DlUVFQgLy8PcXFxXFDT97fUbxn7ISDpGWVlZSwwMJBZW1trde1/9tlnDACLjo7WOSY9PZ25uroya2trFhYWxnbu3MkAsIcPH3J5zp49y0JCQtjAgQPZgAEDmK+vL/vTn/70nFplGp48d6WlpSw2NpbZ29szBwcHlpCQwN555x02YcIE7pj2YSJPOnXqFPP19WWWlpbM39+f7d69myuz3bVr19grr7zCHBwcmLW1NfP29mZvvvkm02g0euvTn4eJ0HRJhPDYX//6V8TFxaGxsRHW1tbGrs5zR8/gCOGRnTt3YsSIEXBzc8PFixexdu1avP766yYZ3AAKcITwilwuR0pKCuRyOVxdXfHaa6/hT3/6k7GrZTR0i0oI4S3qRSWE8BYFOEIIb1GAI4TwFgU4QghvUYAjhPAWBThiVLGxsTAzM4OZmRksLCzg7OyMkJAQZGdnd+kVoZycHDg4OPReRTsQGxuLBQsWPPffSwxDAY4YXXh4OGpqalBVVYVvv/0WM2fOxOrVqzFv3jy0tbUZu3qkPzPum2LE1HX0TqVMJmMA2LZt2xhjT59e/fjx4zrTbK9fv54xxtjOnTuZv78/GzhwIHN2dmaLFi1itbW13O+pr69nixcvZmKxmFlZWTEvLy+WnZ3N7a+urmavvfYas7e3Z46Ojuzll1/m3s1cv369zu89fvx4r/w7ke6hKzjSJ82aNQsTJkzA/v37AQACgQCffPIJLl++jB07duDYsWNYs2YNAGDq1KnIyMiAnZ0dampqUFNTg6SkJACASqXChg0bcPHiRRw4cABVVVWIjY3lfs97772HK1eu4Ntvv8XVq1exdetWiMVi7tiwsDDY2trixIkTOHXqFAYOHIjw8HC0trYiKSkJr7/+OncFWlNTg6lTpz7ffyjydMaOsMS0dXQFxxhjUVFRzMfHR+++J6dX3759O7O3t+/09507d44B4K7+5s+fz+Li4vTm3bVrFxszZgw3ywZjjCmVSmZtbc3y8vI6rT8xPrqCI30WY4ybx6yz6dU7UlxcjPnz52PYsGGwtbXFjBkzAICb/jshIQF79uyBn58f1qxZg9OnT3PHXrx4EeXl5bC1teWm9x40aBB++eUXXk7vzUcU4EifdfXqVXh6eho0vbo+zc3NCAsLg52dHf7617/i3Llz+Prrr7WOmzNnDm7evIm33noLd+/exezZs7nb26amJvj7++tM733t2jUsXry4l1tPegLNJkL6pGPHjqGkpARvvfWWQdOr65tmu7S0FA8ePMCf//xnuLu7AwB++OEHnd/l5OSEmJgYxMTEYPr06fjDH/6AzZs3Y9KkSdi7dy8kEgns7Oz01pNP03vzEV3BEaNTKpWQy+W4c+cOzp8/jw8++AARERGYN28eoqOjDZpe3cPDA01NTZDJZKirq0NLSwuGDRsGS0tL7riDBw9iw4YNWselpKTgm2++QXl5OS5fvoxDhw7Bx8cHwONFe8RiMSIiInDixAlUVlaioKAA//7v/47bt29zv/enn35CWVkZ6urqoFKpns8/GjGMsR8CEtP265XUzc3NmZOTEwsODmbZ2dlMrVZz+QyZXv3f/u3f2ODBg7WGiezevZt5eHgwkUjEgoKC2MGDB3XWCfXx8WHW1tZs0KBBLCIigt24cYMrs6amhkVHRzOxWMxEIhEbMWIEi4+PZ42NjYwxxu7du8dN6Q4aJtLn0HxwhBDeoltUQghvUYAjhPAWBThCCG9RgCOE8BYFOEIIb1GAI4TwFgU4QghvUYAjhPAWBThCCG9RgCOE8BYFOEIIb1GAI4Tw1v8H/nmReNpPo+UAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T06:08:24.032923Z", + "iopub.status.busy": "2024-02-29T06:08:24.032202Z", + "iopub.status.idle": "2024-02-29T06:08:24.315961Z", + "shell.execute_reply": "2024-02-29T06:08:24.315058Z" + }, + "papermill": { + "duration": 0.310128, + "end_time": "2024-02-29T06:08:24.318011", + "exception": false, + "start_time": "2024-02-29T06:08:24.007883", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEqCAYAAABEE9ZrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABZZUlEQVR4nO2dd3hU1daH3ymZFEhCTwKEjhB6EwiogKKAKMWrIKIUEVTAFgEBC1XBT6qKIHoRCwiCilxUFJEigkhJ6J1AAiSElp5Mppzvj2Em0zMTJpmZZL/PMw/MmX3OWSfll7XXXnstmSRJEgKBQOBnyL1tgEAgEBQHIV4CgcAvEeIlEAj8EiFeAoHALxHiJRAI/BIhXgKBwC8R4iUQCPwSIV4CgcAvEeIlEAj8EiFeAoHAL/GqeC1dupRWrVoRFhZGWFgYsbGx/Prrr07PWbduHU2bNiUoKIiWLVvyyy+/lJK1AoHAl/CqeNWuXZu5c+dy4MAB9u/fz/3330///v05duyY3fG7d+9myJAhjBo1ivj4eAYMGMCAAQM4evRoKVsuEAi8jczXNmZXqVKFDz74gFGjRtl8NnjwYHJycti0aZPpWOfOnWnTpg3Lli1z6fp6vZ4rV64QGhqKTCbzmN0CgcAzSJJEVlYWNWvWRC537F8pS9Emp+h0OtatW0dOTg6xsbF2x+zZs4e4uDiLY7169WLDhg0Or6tWq1Gr1ab3ly9fplmzZh6xWSAQlBzJycnUrl3b4edeF68jR44QGxtLfn4+FStW5Mcff3QoLqmpqURERFgci4iIIDU11eH158yZw4wZM2yOJycnExYWdmfGCwQCj5OZmUl0dDShoaFOx3ldvJo0aUJCQgIZGRmsX7+e4cOHs2PHDo95R1OmTLHw1oxfGOMigUAg8E2KCut4XbxUKhWNGjUCoH379uzbt4/Fixfz6aef2oyNjIzk6tWrFseuXr1KZGSkw+sHBgYSGBjoWaMFAoHX8bk8L71ebxGjMic2NpatW7daHNuyZYvDGJlAICi7eNXzmjJlCn369KFOnTpkZWWxevVqtm/fzm+//QbAsGHDqFWrFnPmzAHglVdeoVu3bsyfP5++ffuyZs0a9u/fz/Lly735GAKBwAt4VbzS0tIYNmwYKSkphIeH06pVK3777TcefPBBAJKSkiyWSrt06cLq1at56623mDp1Ko0bN2bDhg20aNHCW48gKGEkSUKr1aLT6bxtisBDKBQKlErlHacq+VyeV0mTmZlJeHg4GRkZImDv4xQUFJCSkkJubq63TRF4mJCQEKKiolCpVDafufo76vWAvS8jSRJ5Gh0hKvFlKm30ej2JiYkoFApq1qyJSqUSScVlAEmSKCgo4Nq1ayQmJtK4cWOniajOEL+VDjh6OYO3fzpKnSohLH6yrbfNKXcUFBSg1+uJjo4mJCTE2+YIPEhwcDABAQFcvHiRgoICgoKCinUdn1tt9CXik9LZeOgKx69ketuUcktx/yoLfBtPfF/FT4YDWtQKp2+rKCQJ5v1+ytvmCAQCK4R4OeH1B+9CIZfx58k09l246W1zBAKBGUK8nNCgekUGdTBsDP2/zScpZwuzAkGxGTFiBAMGDCjRewjxKoKXH2hMoFLOvgu32H7qmrfNEfgB3bt359VXX/W2GWUeIV5FEBUezPAu9QD4v99OodcL70tQPtBoNN42wSlCvFzgxW4NCQ1UciIlk/8dvuJtc8olkiSRW6D1ysudcMGIESPYsWMHixcvRiaTIZPJqF27NkuXLrUYFx8fj1wu5+LFiwAsWLCAli1bUqFCBaKjoxk7dizZ2dkW5+zatYt7772X4OBgoqOjefnll8nJyXHJrpSUFPr27UtwcDD169dn9erV1KtXj0WLFpnGyGQyli5dSr9+/ahQoQLvvvsuOp2OUaNGUb9+fYKDg2nSpAmLFy+2uLZOpyMuLo5KlSpRtWpVJk2aVCohFpHn5QKVK6gYc18D5m85zYItp3m4ZRQBCqH7pUmeRkezd37zyr2Pz+zlcqLy4sWLOX36NC1atGDmzJkAzJ8/n9WrV/Piiy+axq1atYquXbtSt25dwJA68OGHH1K/fn3Onz/P2LFjmTRpEp988gkA586do3fv3syePZsVK1Zw7do1xo8fz/jx4/niiy+KtGvYsGFcv36d7du3ExAQQFxcHGlpaTbjpk+fzty5c1m0aBFKpRK9Xk/t2rVZt24dVatWZffu3YwZM4aoqCgGDRpker6VK1eyYsUKYmJimD9/Pj/++CP333+/S1+z4iK2B7lIjlpLtw+2cT27gNkDWvB057olaKUgPz+fxMRE6tevT1BQELkFWr8QLzDEvNq0aWPyahISEmjXrh0XLlygTp066PV66tSpw1tvvcULL7xg9xrr16/nhRde4Pr16wA899xzKBQKi1JRu3btolu3buTk5DhN9Dx58iQxMTHs27ePDh06AHD27FkaN27MwoULTfE5mUzGq6++ysKFC50+3/jx40lNTWX9+vUA1KxZk9dee42JEycCoNVqqV+/Pu3bt3dY5dj6+2uO2B7kYSoEKhnfoxHT/3ecD7ee4T/tahOsUnjbrHJDcICC4zN7ee3ed0KbNm2IiYlh9erVTJ48mR07dpCWlsYTTzxhGvPHH38wZ84cTp48SWZmJlqtlvz8fHJzcwkJCeHQoUMcPnyYVatWmc6RJMm0jSomJsbh/U+dOoVSqaRdu3amY40aNaJy5co2Y43iZs6SJUtYsWIFSUlJ5OXlUVBQQJs2bQDIyMggJSWFTp06mcYrlUo6dOhQ4lNHMfdxgyGd6lCrUjBpWWpW7r7gbXPKFTKZjBCV0isvT+ypHDp0KKtXrwZg9erV9O7dm6pVqwJw4cIFHnnkEVq1asX333/PgQMHWLJkCWDYJgWQnZ3N888/T0JCgul16NAhzpw5Q8OGDe/YPiMVKlSweL9mzRomTJjAqFGj+P3330lISGDkyJEmu7yJEC83CFQqiHvwLgCWbj9LRq5vr8YIvINKpbIp4fPUU09x9OhRDhw4wPr16xk6dKjpswMHDqDX65k/fz6dO3fmrrvu4soVy4Whdu3acfz4cRo1amTzsleZwZwmTZqg1WqJj483HTt79iy3bt0q8ln+/vtvunTpwtixY2nbti2NGjXi3Llzps/Dw8OJiopi7969pmNarZYDBw4Uee07RYiXmwxoW4u7IiqSma/l053nij5BUO6oV68ee/fu5cKFC1y/fh29Xk+9evXo0qULo0aNQqfT0a9fP9P4Ro0aodFo+Oijjzh//jxff/21TSu/N954g927dzN+/HgSEhI4c+YMP/30E+PHjy/SnqZNm9KzZ0/GjBnDv//+S3x8PGPGjCE4OLhIr7Jx48bs37+f3377jdOnT/P222+zb98+izGvvPIKc+fOZcOGDZw8eZKxY8eSnp7u+hesmAjxchOFXMaEh5oAsOLvRNIy871skcDXmDBhAgqFgmbNmlG9enWSkpIAw9Tx0KFDDBw4kODgYNP41q1bs2DBAt5//31atGjBqlWrTNWDjbRq1YodO3Zw+vRp7r33Xtq2bcs777xDzZo1XbLpq6++IiIigvvuu4+BAwcyevRoQkNDi6zo8Pzzz/PYY48xePBgOnXqxI0bNxg7dqzFmNdff51nnnmG4cOHExsbS2hoKAMHDnTJrjtBrDYWA0mSeGzpbuKT0nmmc11mDRCVXD2Ns9UowZ1z6dIloqOj+eOPP3jggQdK/f6eWG0UnlcxkMlkTOrVFIBv/03i4g3XEgUFAm/x559/snHjRhITE9m9ezdPPvkk9erV47777vO2acVGiFcxiW1YlXsbV0Orl1i45bS3zRGUY/766y8qVqzo8AWGrT5Tp06lefPmDBw4kOrVq5sSVv0Vked1B0zq1ZS/zuzip0NXeL5bQ2KiRE18QenToUMHEhISnI7p1asXvXp5J0+upBDidQe0rB1O35ZR/HwkhXm/neK/I+72tkmCckhwcLCpcXN5Qkwb75C4hwwFC7eeTGO/KFgoEJQaQrzukIbVK/JEe0PBwvdFwUKBoNQQ4uUBXunZGJUoWCgQlCpCvDxAVHgww2MNVSZEwUKBoHQQ4uUhxnZvJAoWCgSliFfFa86cOdx9992EhoZSo0YNBgwYwKlTztuMrVy50lSh0vjyhQzsyhVUjL6vAQALtpxGo9N72SKBP2KvuqmjmljlHa+K144dOxg3bhz//PMPW7ZsQaPR8NBDDxVZ2jYsLIyUlBTTy1hK19uMuqc+VSuouHgjl+/2J3vbHEEZICUlhT59+njbDJ/Eq3lemzdvtni/cuVKatSowYEDB5xuW5DJZERGRrp0D7VajVqtNr3PzCy57tcVApWMv78RM/53nMV/nOGxtqJgoeDOcPXnvCSRJAmdTodS6VtpoT4V88rIyACgSpUqTsdlZ2dTt25doqOj6d+/P8eOHXM4ds6cOYSHh5te0dHRHrXZmqfMChZ+uedCid6rXCFJUJDjnZeb6S9ZWVkMHTqUChUqEBUVxcKFC4vdDs182njhwgVkMhk//PADPXr0ICQkhNatW7Nnzx6Lc4pq1PH111/ToUMHQkNDiYyM5KmnnrKoZ799+3ZkMhm//vor7du3JzAwkF27drlte0njM1Kq1+t59dVX6dq1Ky1aOK7S0KRJE1asWEGrVq3IyMhg3rx5dOnShWPHjlG7dm2b8VOmTCEuLs70PjMzs0QFLFCp4LUH72LCukN8su0sgzpEU6WC82JxAhfQ5MJ7rpV/8ThTr4CqQtHjbhMXF8fff//Nxo0biYiI4J133uHgwYOm0sl3yptvvsm8efNo3Lgxb775JkOGDOHs2bMolUqXGnVoNBpmzZpFkyZNSEtLIy4ujhEjRvDLL79Y3Gfy5MnMmzePBg0a2C0Z7W18RrzGjRvH0aNHi1T42NhYYmNjTe+7dOlCTEwMn376KbNmzbIZHxgYSGBgoMftdcbAtrX4/K/znEzN4v1fT/L+461K9f4C75GVlcWXX37J6tWrTaVmvvjiC5frbrnChAkT6Nu3LwAzZsygefPmnD17lqZNmzJnzhyGDh1q8vIaN27Mhx9+SLdu3Vi6dClBQUE8++yzpms1aNCADz/8kLvvvpvs7GzTRm6AmTNn8uCDD3rMbk/jE+I1fvx4Nm3axM6dO+16T84ICAigbdu2nD17toSscx+FXMbsAS14fNke1u5PZtDd0bSv63t/ufyKgBCDB+Ste7vI+fPn0Wg0dOzY0XQsPDycJk2aeMycVq0K/xhGRUUBkJaWRtOmTV1q1HHgwAGmT5/OoUOHuHXrFnq9YWU8KSmJZs2amc6z14zDl/CqeEmSxEsvvcSPP/7I9u3bqV+/vtvX0Ol0HDlyhIcffrgELCw+HepV4fH2tVl/4BJvbzjKxvFdUYpej8VHJnNr6laWMS9jYyzjbBQgY6OOl19+2ea8OnXqkJOTY6owsWrVKlOl1169etk01bBuxuFreFW8xo0bx+rVq/npp58IDQ0lNTUVMPylMpbJHTZsGLVq1TKVxZ05cyadO3emUaNGpKen88EHH3Dx4kWee+45rz2HIyb3acrvx1I5npLJqr1JDO9Sz9smCUqYBg0aEBAQwL59+6hTpw5gWIg6ffp0qRT+M2/UYY8jR45w48YN5s6da4r97t+/v8TtKgm86gosXbqUjIwMunfvTlRUlOm1du1a05ikpCRSUlJM72/dusXo0aOJiYnh4YcfJjMzk927d1u4u75CtYqBTOxtqLg67/dTXMtSF3GGwN8JDQ1l+PDhTJw4kW3btnHs2DFGjRqFXC73SAu1oiiqUUedOnVQqVSmZh8bN260Gyv2B7wqXpIk2X2NGDHCNGb79u2sXLnS9H7hwoVcvHgRtVpNamoqP//8M23bti19413kqY51aFkrnKx8LXN+OeFtcwSlwIIFC4iNjeWRRx6hZ8+edO3alZiYmFLZCVJUo47q1auzcuVK1q1bR7NmzZg7dy7z5s0rcbtKAtGAoxRISE5n4Cd/I0mwdkxnOjWoWir39WfKUgOOnJwcatWqxfz58xk1apS3zfEJRAMOP6FNdCWevNsQ/3j7p6Ni32MZJz4+nm+//ZZz585x8OBBU4PZ/v37e9mysoUQr1JiUq8mVA4J4PTVbFb+fcHb5ghKmHnz5tG6dWt69uxJTk4Of/31FydOnCiyUYbAdXwiz6s8ULmCisl9mvLG90dY9MdpHm1dk8hw/54OCezTtm1bu+3u8/LyimyUIXAdIV6lyBPto1m7L5mDSenM/vk4Hz/VztsmCUqR8tooo6QQ08ZSRC6XMWtAC+Qy2HQ4hb/PXve2ST5POVtPKjd44vsqxKuUaV4znGGx9QBD8F6t1XnXIB/FmEWem5vrZUsEJYHx+3onTW/FtNELxD10F5sOp3D+Wg6f/5XIuB5iKmGNQqGgUqVKplItISEhpZLkKShZJEkiNzeXtLQ0KlWqhEJR/Hp3Qry8QFhQAG/2bcpraw/x0Z9nGNC2FrUqBXvbLJ/DWIjPvNaUoGxQqVKlOy60KMTLSwxoU4s1/yazN/EmM/93jE+f8e0d/N5AJpMRFRVFjRo10Gg03jZH4CECAgLuyOMyIsTLS8hkhuD9w4v/4rdjV9l2Ko0eTWp42yyfRKFQeOSHXVC2EAF7L3JXRCjP3mMoAzR94zHyNSJ4LxC4ihAvL/PyA42JCAvk4o1cVu1N8rY5AoHfIMTLy1QMVDL6XkO/xz3nRN6XQOAqQrx8gLZ1DCWiE5IzRFKmQOAiQrx8gOY1w1DKZVzPVpOSke9tcwQCv0CIlw8QFKCgaVQoADtPX/OyNQKBfyDEy0d4MMaQsDdr03FOX83ysjUCge8jxMtHGNujIZ0bVCGnQMeYr/aTkSuSMgUCZwjx8hECFHKWPNWOWpWCuXAjl5fWxIvgvUDgBCFePkTVioEsH9aeAIWMnaevcf56jrdNEgh8FiFePkbzmuFUrRAIQF6ByLgXCBxxR3sbs7OzTZ16jZRWR56yjFJhKP0iGnUIBI5x2/NKTEykb9++VKhQgfDwcCpXrkzlypWpVKkSlStXLgkbyx1KuUG8dHoR8xIIHOG25/X0008jSRIrVqwgIiJCFIgrARS3xUsrxEsgcIjb4nXo0CEOHDhAkyZN7vjmc+bM4YcffuDkyZMEBwfTpUsX3n///SKvvW7dOt5++20uXLhA48aNef/993n44Yfv2B5fQSk3OMTC8xIIHOP2tPHuu+8mOTnZIzffsWMH48aN459//mHLli1oNBoeeughcnIcr7Lt3r2bIUOGMGrUKOLj4xkwYAADBgzg6NGjHrHJFxAxL4GgaGSSm8lE586d44UXXuDpp5+mRYsWNgX0W7VqVWxjrl27Ro0aNdixYwf33Xef3TGDBw8mJyeHTZs2mY517tyZNm3asGzZsiLv4WorcW/S/+NdHLqUwX+Hd+CBmAhvmyMQlCqu/o66PW28du0a586dY+TIkaZjMpkMSZKQyWTodMVf3s/IyACgSpUqDsfs2bOHuLg4i2O9evViw4YNdser1WrUarXpfWZmZrHtKy2sY15pWfnM/eUkz95Tnxa1wr1pmkDgM7gtXs8++yxt27bl22+/9WjAXq/X8+qrr9K1a1datGjhcFxqaioREZbeSEREBKmpqXbHz5kzhxkzZnjExtJCqbCMeX228zw/xF/mh/jLJM55WCySCAQUQ7wuXrzIxo0bPd75d9y4cRw9epRdu3Z59LpTpkyx8NQyMzOJjo726D08jTFVwl7Ma+eZ60SEBdI00jenvAJBaeG2eN1///0cOnTIo+I1fvx4Nm3axM6dO6ldu7bTsZGRkVy9etXi2NWrVx22UQoMDCQwMNBjtpYGCqs8r4qBhXHF4Sv+BWDHxO7UrVqh9I0TCHwEt8Xr0Ucf5bXXXuPIkSO0bNnSJmDfr18/l68lSRIvvfQSP/74I9u3b6d+/fpFnhMbG8vWrVt59dVXTce2bNlCbGysy/f1dZRWMS+t3tYDu3gjV4iXoFzjtni98MILAMycOdPmM3cD9uPGjWP16tX89NNPhIaGmuJW4eHhBAcbmrAOGzaMWrVqMWfOHABeeeUVunXrxvz58+nbty9r1qxh//79LF++3N1H8VmMMS+tzihetgvCKqXYlioo37j9G6DX6x2+3F1pXLp0KRkZGXTv3p2oqCjTa+3ataYxSUlJpKSkmN536dKF1atXs3z5clq3bs369evZsGGD0yC/v1G4PcjgcWlvx756NKluGiOq5QjKO255XhqNhuDgYBISEjwiFq6kmG3fvt3m2BNPPMETTzxxx/f3VaxTJYz/xkSFcelWHmfSsr1mm0DgK7jleQUEBFCnTp07yuUSFE2AVaqEcfpo9MgAJITrJSjfuD1tfPPNN5k6dSo3b94sCXsEFHpeGquYl1IhR6R4CQQG3A7Yf/zxx5w9e5aaNWtSt25dKlSwXPE6ePCgx4wrrziKeSnMPC/heAnKO26L14ABA0rADIE51jEv4/QxQCFDhn3XS6PTm6abAkF5wG3xmjZtWknYITDDOualuf2vQl4oTuaO15FLGfxn6W5e7N6Q1x68q9TsFAi8SbHLQB84cIATJ04A0Lx5c9q2besxo8o7NjGv29PGAIV9r+tg0i0KdHqWbj/HkI51iAwPKh1DBQIv4rZ4paWl8eSTT7J9+3YqVaoEQHp6Oj169GDNmjVUr17d+QUERWIT8zIG7OX2A/bGPZAFOj3Ldpxjer/mpWOoQOBF3A6SvPTSS2RlZXHs2DFu3rzJzZs3OXr0KJmZmbz88sslYWO5w1iM0JTndVucLFIlzOaNam3h9qFv/00iLTO/FKwUCLyL2+K1efNmPvnkE2JiYkzHmjVrxpIlS/j11189alx5xRjbst4epHQwbSwwEy+1Vs+Kvy+UrIECgQ9QrO1B1puxwZDAat0GTVA8bDZm64wBe/tJqsZpYwWVAoBTqb5fcFEguFPcFq/777+fV155hStXrpiOXb58mddee40HHnjAo8aVVxRWMa/CVAm53UKERs8rJNAQwtSJHDBBOcBt8fr444/JzMykXr16NGzYkIYNG1K/fn0yMzP56KOPSsLGckeAVcxLo7dNUjWPeRk9r+AAg+elF12HBOUAt1cbo6OjOXjwIH/88QcnT54EICYmhp49e3rcuPKKTcxLZ56kakuBtXiJkhOCckCx8rxkMhkPPvggDz74oKftEWDbMds8VcIextXGoADR71FQfiiWeG3dupWtW7eSlpZmE6RfsWKFRwwrzxRuD7Lc22hZVaIQYzJroPC8BOUIt8VrxowZzJw5kw4dOhAVFSU62ZQAxpiXzmpvo6OqEgVaQ4mioNviJTwvQXnAbfFatmwZK1eu5JlnnikJewQUxryMHpX9gL15qoTh/8G3p41CuwTlAbdXGwsKCujSpUtJ2CK4jXXMS2cesLfreRljXmLaKCg/uC1ezz33HKtXry4JWwS3KdweZBClwqoS9mNeJvFSimmjoPzg9rQxPz+f5cuX88cff9CqVSubbPsFCxZ4zLjyiinD3qaqhNxuPS9TqoRKiJeg/OC2eB0+fJg2bdoAcPToUYvPRPDeM5jyvGxSJZzvbQw0xbyEeAnKPm6L17Zt20rCDoEZNnleOjt5XnYy7I3TRuF4CcoDom6wD2JdEkdnVlXCbsDeatootgcJygNCvHwQU5Kqzhiwt5ekWihQhQH72xn2YtooKAcI8fJBjNNDnV5Cr5dMm7CVCrndvY0aEbAXlEOEePkg5tNGjdn2K0dVJdRWeV7C8RKUB9wWr507d6LVam2Oa7Vadu7c6fa1Hn30UWrWrIlMJmPDhg1Ox2/fvh2ZTGbzSk1Ndeu+vo55wF5rVpwrQCHDXtDL6HkFijwvQTnCbfHq0aOH3W7ZGRkZ9OjRw61r5eTk0Lp1a5YsWeLWeadOnSIlJcX0qlGjhlvn+zqF3YP0pqA9OK4qYYx5maaNwvUSlAPcTpWQJMluPteNGzdsumcXRZ8+fejTp4+7JlCjRg1T56KiUKvVqNVq0/vMTN8vkWwe8zIG7Q3HbaeNWp3elBphDNi7utqo1elRyGUiP0/gl7gsXo899hhgSEQdMWIEgYGBps90Oh2HDx8utT2Pbdq0Qa1W06JFC6ZPn07Xrl0djp0zZw4zZswoFbs8hXnMyzgFlMtALrfNr9eYTSvd2duo1up4cMFO6lYN4etRnTxjuEBQirgsXuHh4YDB8woNDSU4ONj0mUqlonPnzowePdrzFpoRFRXFsmXL6NChA2q1ms8//5zu3buzd+9e2rVrZ/ecKVOmEBcXZ3qfmZlJdHR0idp5pyjNUiU0DgoRGuXJvHOQOyVxUtLzSbqZS1qWaJMm8E9cFq8vvvgCgHr16jFhwgS3p4ieoEmTJjRp0sT0vkuXLpw7d46FCxfy9ddf2z0nMDDQwkv0BxRm3YOMFSWM3pj1DK/AbFoZqHS9JI5x07do+CTwV9wO2E+bNs0rwuWIjh07cvbsWW+b4VECFIUxL3u1vKCwnpdRvFRKuVnXoaLVq0B7O3tfBPcFforb4nX16lWeeeYZatasiVKpRKFQWLxKm4SEBKKiokr9viWJuedV2HzD8K2yjnmZNmUr5Mhvn+dKzMvkeQnxEvgpbq82jhgxgqSkJN5+++07LgOdnZ1t4TUlJiaSkJBAlSpVqFOnDlOmTOHy5ct89dVXACxatIj69evTvHlz8vPz+fzzz/nzzz/5/fffi22DL2K+qmgUJ+uKEkbJMeZ4BSjlGIe4IkjGQL8kFXpxCcnpNKhekfBg26bCAoGv4bZ47dq1i7/++stUFudO2L9/v0VumDGwPnz4cFauXElKSgpJSUmmzwsKCnj99de5fPkyISEhtGrVij/++MPt/DJfx3yKmKcx1KcvqhyOSiFHIXN92qgxi5Xp9BKHL2fw2Ce7ebhlJJ8MbV9s2wWC0qJYfRslD001unfv7vRaK1eutHg/adIkJk2a5JF7+zLGKSIYUhrAsK8RbGumGbcGqZTm00bH+XhGzDP39RJcupUHwNVMtaNTBAKfwu2Y16JFi5g8eTIXLlwoAXMEYOl55WscTBtva49p2qiQmTwv888dYe556SUJ9W0PTyDwF9z2vAYPHkxubi4NGzYkJCTEpgy0va1DAvcwF6F847TRmCphNdY0bVQqTJ4XGFYR7degMGA9bVRrRc6EwL9wW7wWLVpUAmYIzJHLZchlhumcUbwUNvsab7dFM6ZKKGSYO2c6vUSAk8Vf8z2Tekky3Ucg8BfcFq/hw4eXhB0CK5QKOQVavckjCnCUpKq1zfOColccLaaNeoTnJfA7ilXP69y5c7z11lsMGTKEtLQ0AH799VeOHTvmUePKM8YYV76D1UajNpknqcpl5uLl/PrmeyJ1kpg2CvwPt8Vrx44dtGzZkr179/LDDz+QnZ0NwKFDh5g2bZrHDfQq57dDfoZXbq2wES/n5XACFJaeV1HpEo4C9uarv2fTskjNEHsfBb6J2+I1efJkZs+ezZYtW1CpVKbj999/P//8849HjfMqlw/CqkHw2f1w7XSp396YLmH0iAoD9pYemMnzUlh5XkWIl9Zi2mjreaVl5tNzwU46z9lazCcQCEoWt8XryJEjDBw40OZ4jRo1uH79ukeM8glkcqhQHW6cNQjYyV9K9fbWnpfN3sbb/2q0thn2UPSeRdtpo2XA/kxadnHMFghKDbfFq1KlSqSkpNgcj4+Pp1atWh4xyieo2QbGbIe6XaEgC9YMge3vl1oZhsKYV+G0ELDJlTB6XoEKOTJZ4YpjUZ6X5bSx8D4Cgb/gtng9+eSTvPHGG6SmpiKTydDr9fz9999MmDCBYcOGlYSN3qNidRj2E3QcY3i//T347hlQZ5X4rYv0vIwBe7PVRvNxRQXsLVIl9Lael0Dg67gtXu+99x5NmzYlOjqa7OxsmjVrxn333UeXLl146623SsJG76IIgIc/gH4fg0IFJzfB5z3hxrkSva11zCvAUZKqddUJ4/7GIqaN5kUMdXoJtfC8BH6G2+KlUqn47LPPOHfuHJs2beKbb77h5MmTfP31114piVNqtHsGRv4KoVFw7SQs7wFntpTY7YpabTQ2nbXxvG6LV5EBe73VaqNIlRD4GW4nqRqpU6cOderU8aQtvk/tDoY42HfDIHkvrHoCHngH7nnNbkuyO8EY8yqqqkTh3kbLaWNRqRKWG7MLM+wdnSVJEnvO3eCfxJt0bViVTg2quvYgAkEJ4ZJ4xcXFMWvWLCpUqGBRD94eCxYs8IhhPktoJAzfBL9OhAMrYesMSDkEAz4BlecqzBpTI2xSJZxk2AMu1/QqsNjbWHSG/X93JTL75xMA/Hz4Cltf7170QwgEJYhL4hUfH49GozH93xHlpoWWUgWPLoao1vDLJDi+Aa6fgSdXQZX6HrmFcS+j2sHeRuuAvbF+vavVVK09L2cBe0mSuHAjx/Q+I8+26bBAUNq4JF7btm2z+/9yT4dnoXqMYRqZdgw+6wGPfwEN77w4om2qhP0kVfOSOIBZQULn17euKuEsVUKSrKehonS0wPsUa2+jwIy6sYY4WM12kHcLvnkMdn9cdEGtIjAF7LXOk1TVZhn2UOh5Fb09yA3Py2q8K5VaBYKSxiXPy9hw1hV++OGHYhvjt4TXMqxE/hwHCavg9zcNcbBHF4MqpFiXNHpSaqskVeuZuXmGPZitNhY1bdRbJqk6i3npJclCsIR4CXwBlzyv8PBw0yssLIytW7eyf/9+0+cHDhxg69atpsa05ZKAIOi/BPp8ADIFHPkOVvSC9KSiz7WDMcZl9Lxsq0pYtT5TuBewt502Gjdm246VJOv6X64+hUBQcrjkeRkbzgK88cYbDBo0iGXLlpnyunQ6HWPHjiUsLKxkrPQXZDLoNAZqxMC64ZB6GJZ3hye+hPr3unWpokriGLFZbSz2tLEoz8tS7AQCb+N2zGvFihVMmDDBIiFVoVAQFxfHihUrPGqc31L/XkMcLLIV5N6Ar/rD3uVuxcGsA/ZKR9NGK89L4eJqo7nnpdbonZomSZark6JRrcAXcFu8tFotJ0+etDl+8uRJ9KJ3fCGV6sCzv0HLJ0DSGfLCfhoPGtfqYxnzuoxYB+yNOMywL2pvo5kY5RY4T32QEDEvge/hdob9yJEjGTVqFOfOnaNjx44A7N27l7lz5zJy5EiPG+jXqELgsc8M+WBb3oGEb+DaCRj8DYTVdHqqdV6Xo1QJ272NhuNFCYx5kmpugfNN2XqrmJcQL4Ev4LZ4zZs3j8jISObPn28qjRMVFcXEiRN5/fXXPW6g3yOTQZeXIKI5rH8WLh+AT7vB4K+hTmeHp1nHuBwnqRqEx6aqhBvFCPOKaL4hWa02Gq8vd+ANCgSlgdvTRrlczqRJk7h8+TLp6emkp6dz+fJlJk2aVLY3Zt8pDe+H0dugRnPISYOVj8B+xzFCa/Fy2IBDZxWwd7GqhLkn5ZrnZRkSEHEvgbe5oyTVsLCwO1ph3LlzJ48++ig1a9ZEJpOxYcOGIs/Zvn077dq1IzAwkEaNGtl01fZpqtSH57ZAswGg18Cm1+B/r4C2wGaodczLUVUJjdbwr23A3rkp5iVxctWFMS97p9nzvMTUUeA2Oo1HL1cs8Vq/fj2DBg2ic+fOtGvXzuLlDjk5ObRu3ZolS5a4ND4xMZG+ffvSo0cPEhISePXVV3nuuef47bffivMY3kFVAZ5YCQ9MA2SGzd1fPgJZqRbDrAP0DlMlHHheRZfEKfy86GmjZWoF2Bev7afSmP/7qSLvLShnSBL8s8ywfU7tufLibovXhx9+yMiRI4mIiCA+Pp6OHTtStWpVzp8/T58+fdy6Vp8+fZg9e7bdmvj2WLZsGfXr12f+/PnExMQwfvx4Hn/8cRYuXOjuY3gXmQzujYOh6yAw3FBeZ3l3uHTANMTa07L2xIyzNo3WMgPf9TwvdwL2djwvq2njpVu5jPhiHx/9eZYFW0q/YYnAR9Hkw4axsPkNSD0Ch7712KXdFq9PPvmE5cuX89FHH6FSqZg0aRJbtmzh5ZdfJiOjZNuE7dmzh549e1oc69WrF3v27HF4jlqtJjMz0+LlMzR+EMZsg2pNICsFvugN8d8A9gL29j0vtc46VcJwvMiYVxGpEuanS1h6amDr2T39+V7T/z/edlZ4XwLIuAxf9IFDqw27TnrNgbuf89jl3RavpKQkunTpAkBwcDBZWYZ67s888wzffus5VbVHamoqERERFsciIiLIzMwkLy/P7jlz5syx2N4UHR1doja6TdWGMHorNH0EdAXw0zj4ZSIBMktvyLrMMxhiUaY8L6uYl+RGkmqenYoSkln0yzrDHmw9u0u3LL/+/yTecHp/QRnn4h5Y3g2uHITgyvDMDxA71qNFO90Wr8jISG7evAkYqqkaezUmJiYW+QvjDaZMmUJGRobplZyc7G2TbAkMhUFfQ/ephvf/Luepky9RlUJP1l4DDnNvSGVdw96Nkjh5djwvc22yvhdYenZand7m8+/2+eDXWVA67PuvIY6bcw0iWhh2mzTo7vHbuC1e999/Pxs3bgQMCauvvfYaDz74IIMHD3Y5dlVcIiMjuXr1qsWxq1evEhYWRnBwsN1zAgMDTauid7o6WqLI5dD9DXjyW1CFEp0Vz8bAt2ghOw/Yb8BhvmJonWHvzrQxR20b8zL/Q2Rbz8vyfb6dfZG/HE0lI9ezq0sCH0erNqye/xwHei00HwijfofK9Urkdm4nqS5fvty0DWjcuHFUrVqV3bt3069fP55//nmPG2hObGwsv/xi2fx1y5YtxMbGluh9S5WmD8Pordxa8Ti18pJYr5rBG5rRKOUdLYZJWIqXqRihi0mqBY6SVG+Llrn26SXJQuzASrzsrFYWaPVsOXGVx9vXdmqHoIyQlVrY2wEZ9JwGXV/1eG8Hc9zyvLRaLbNnzyY1tXBZ/8knn+TDDz/kpZdeQqVSuXXz7OxsEhISSEhIAAxTz4SEBJKSDGVkpkyZYtEL8oUXXuD8+fNMmjSJkydP8sknn/Ddd9/x2muvuXVfn6d6E75v9xVbdW0JkmlYrPqERvFzQGc5vTNO/eSywo3brq42WqRK2FltNI95SXauZx4CsydeABPWHWLNv8UrCSTwIy7tN6yWJ+81rJ4PXVciTWmscUu8lEol//d//4dW65ka5vv376dt27a0bdsWMDT6aNu2Le+88w4AKSkpJiEDqF+/Pj///DNbtmyhdevWzJ8/n88//5xevXp5xB5fQh8YxnOa1/lIOwCAmif+C988RkW9YbVUMitjY5wygmv1vPR6y9SHHHsxL73leGcxL2clpCf/cMThZ4IyQPw3hhXFrBTDqvmYbYZV9FLA7WnjAw88wI4dO6hXr94d37x79+5Og/z2sue7d+/utAlIWUEplyMhZ752EMf1dfk45DMUiTuYrjjFedkrQCubtmfgWiVVjdXKoT3PSW8T83K82ujI8zKy6fAVHmnlfCO6wM/QaeC3qfDvcsP7po/AwGWGxadSwm3x6tOnD5MnT+bIkSO0b9+eChUs233169fPY8aVZ8yTUn/Vd+Jcv4e5a9vzVL91ge9V0zl0JYCC2oOBws5BYD5tdHxt6/iVvSRVy3Ybtp7XzE3H+epZQxyuKPF6+dt47mlUjUoh7oUVBD5KznX4bjhc3GV4330q3DfRsOhUirgtXmPHjgXs92eUyWTodM5/kAWuYZ0aoakWA6O3cfTD/9Ai/yCxByeQpjuPnC52PS9nq40aK2WzK15m5+slW8Hbefqa6f/Opo3G87PytUK8ygJXEmDt05CRDKpQeGy5YZHJC7gtlXq93uFLCJfnCLCp5yWHkCosipjDMu0jANQ49AkrAj6giiLXNM54mrPpuPU+RXtYZNjb2R5kTlGeF9jmiQn8kMO3+zJkJEMVY3K1d4QLROszn8XRxmy9TMFc7VPsaft/6BRBdFccYnn+REgzdLM2lcRxIhbW5W3MMZ5lfrq9kjjm5Dtpm2a6Z1FZswLfRaeF396EH0aDNh8aPwSj/4TqTbxqlsvTxry8PLZu3cojjxj+6k+ZMgW1Wm36XKFQMGvWLIKCgjxvZTnEUUkc49Hkmg8TENmUyF+epbY+BT7vCQOXoZAb8qqciZexjI4zrLcHOXOcipo2gmvensAHyb0J60fC+e2G9/dOgB5TQe792n0ue15ffvkln376qen9xx9/zO7du4mPjyc+Pp5vvvmGpUuXloiR5REbz8u6qgQSt0Kb8qh6NocDWkFBNqx9moevf4EMvVurjfYwFyvrGJk1RZXUAeeem8BHST1qyN86vx0CQgxdsB542yeEC9wQr1WrVjFmzBiLY6tXr2bbtm1s27aNDz74gO+++87jBpZXbEri2KkqUaDVc4sw5lR5Dzq9CECv61+yPGABCo3juklFiRFYxsyK8prULojXt/8mc/FGTpHjBD7CsQ3w3wch/SJUqgujtkDzAd62ygKXxevs2bO0bNnS9D4oKAi52S9Yx44dOX78uGetK8dYi5W91mdGEVIEqKDPXBiwDI1MxYOKgww8MAyun7F7beuVQ3uYO25FxatcCdh/+28S0zYeK3KcwMvodbB1pqHvqCbXsKF6zHaIbOFty2xwWbzS09MtYlzXrl2zSFTV6/UWnwvuDEURrc8kybbtGW2G8GnDJVyRqlAl9wJ8dj+c2mxzbVc8L/NpZ0GR4uXalPDCdeF5+TR56fDtk/DXfMP72PEw9HsIqeJVsxzhsnjVrl2bo0ePOvz88OHD1K4tNuF6CttUCaN4FYpYgSnDvvBYSoUY+qnf5XJYG1BnGn4Yd3xgsd/H2TTQqFmSRczLuafmiucFcDVT7ZNlkwTAtVOGP3ZnfgdlkKFlX693QeF2Kmip4bJ4Pfzww7zzzjvk59s2Tc3Ly2PGjBn07dvXo8aVZ2xTJawbcJh7XoUBVLlMxnXCWdf8k9tVKyXYNhvWDQO1oXCkK2kLeqt6XYZr2x/rSqoEGAL7mfme2Rcr8CAnf4bPHoCb5yCstqFZcqtB3raqSFyW1alTp/Ldd9/RpEkTxo8fz1133QXAqVOn+Pjjj9FqtUydOrXEDC1v2KZK2LY+s+d5GUVPixL6zofIVvDz63Dif3D9LAxZTYGuYpH3N/ePjNNMpVxudwrp6rQRIC0zn/DgAJfHC0oQvR52/h9sn2N4X/ceQ3OYitW9aparuCxeERER7N69mxdffJHJkyeb3H+ZTMaDDz7IJ598YlOiWVB8zD0vuQybBq+SVNh8w2Jvo/X2oPbDoUYMrH3G0K17eXfCOi4A7BdvLLy+ecxLKrTJjpPlSqqEkdTMfBpHlN7mXYED1Fnw4wtwcpPhfcfnb08T/ecPi1sT2vr167N582Zu3rzJ2bNnAWjUqBFVqvhmQM+fMY95WU8ZjZjanpnvbbz9X4tihNEdDStG3z0Dl/bRdudonlcM5lPdI1jWZi3EIualNXpeDpqAuCNeGbZhB0Epc+McrHkKrp0EhQoeWQhtn/a2VW5TrGhclSpV6NixY9EDBcXG3PMyn0JalIG2UxJH7qgkTlgUjPgZfn4dWfzXTAn4lhbyRCZpxpCH7a4Ie0mq1iugRtyaNmapb5+jQy6ToVLK0eslG89SUEKc2QLrR4E6A0KjYPA3ULuDt60qFr67lFDOUdqJY5kjIdmmSlBESRxlIPT7iHhtPVocfo9HFf/QUJbCGE0clyTLOIe5+GluK5kjD9DV1UYweF75Gh2tpv9O5QoBDGxbmzX7kvj1lXuJCnc+lRXcAZIEuxYacriQILqToelLqP+GesTGbB/FfIpm7llZBOy1tp5XkcUIZTJO1xnEUwVvck0Ko5n8IhtVbxIrNySQGvc0WgTsi5g2urraCHA1M59z17Ip0Om5mqlm2Y5zpOdq+GTbOZevIXCTghzD/sStMwAJ2g2H4f/za+ECIV4+i9Ii5mXH85IKp3P2PS/nJXH2SU3pp36Xw/r6VJFl83XAHJ5V/GrWgMMsVeJ2jpijxrfuTBuvZuZjT1dLuNx5+eXWBfjvQ3DsR5ArDfGtfh8avHA/R4iXj2IeXzIXL5l5kqrd1UbDv86KERrztlKoyhMF0/hedy9KmZ53Ar4mLnsBaPLsJqlap28YcWfaeDVTbdcrlAv18jzntxs2Vl89ChVqwPBN0OFZb1vlMYR4+ShKi4C97bdJolBU7E0bXS1GqEbF65oXmKF5Bq0k5/6CbbCiN8F5KWbji/K8XBeva9lqUZiwpJEk2LMEvh4IebegZlvDanPdMtQiECFePouFeDlKUXAasHenJI6ML3R9eEYzhQxZGKQk8Og/Q+goMxQ4LExSlVlk2RsF0p1po04vMXHdIZvjwvHyEJo8+PF5Q3MMSQ+tn4KRmyG8lrct8zhCvHwUi5iXeaqE8b+SZDdVQuFstfE2jooR7tE357XwhRDRkmDNLVap3uNpxRZTwF4hl1t4gUbnzh3PC+DcNdsN2jIH+WYCN0hPNpRpPrwWZAro/T4M+AQCymaBUCFePorCIlXC/rdJU8y+jc4KA6YpImDU75yL6EWATMfsgC/off49VGhQymUEmLleEob4mSemgcLzcp9DyemMX32QS7dy4cLfhvhWyiEIrgLDNkDnF8r0F1aIl49imSphx/PCfoa9wyRVM4wxL3uzUUkCVCFsbfYe72mGoJNktLvxP9aoZlFNumkR99JLEvlaz1RILbu/YiXHqr0X2XT4Cud/WQRf9YPc6xDZ0hDfqn+ft80rcYR4+ShFxbwsq0rYJrQ6T5UwrlI6LuerR8Zy3aOM0LxBriKUdvKzLEh/mXbywgKHeklye8roiDLsILiMTi/xxLLdvLLGtabKBfl5zFV+xn1n3ge9Flo8Ds/+DpXrlrClvoEQLx/FYnuQ2bTRPDZkyvNSKGzOczptNIpXgONvv/H0v/StWFjvU07qo6msv8Vn+mkMVmwzjfGUeJmnSnx/4BIJyekeua4/cexKBvsu3OKnhCtFD85M4aXkV3lSuR09cnhwFvznc1CFlLyhPoJPiNeSJUuoV68eQUFBdOrUiX///dfh2JUrVyKTySxeZbFjkUwmMwmRvfwq80qqltPKoj0vY5UI8/wwlVU6hrn4pSiieKxgBv8G30MAWt4P+IxZyhXotWq3xCs00MlutNuPsPf8DV5fd4gBS/52+bplBfO/N3pnccTkf2F5NxqqT5AuVWBZ9PvQ9eVy5756XbzWrl1LXFwc06ZN4+DBg7Ru3ZpevXqRlpbm8JywsDBSUlJMr4sXL5aixaWHcbroKL+qwE6GfeH2IMfX1dqZNpoLme14iVyCWFLtHT4PGIpekvGM8g8CVw1Ek3HVtYcBqoc5zuo2epRnrzluHFLWsehP4GhR5cCX8MXDkH2VZGVd+hXMZr+ibekY6GN4XbwWLFjA6NGjGTlyJM2aNWPZsmWEhISwYsUKh+fIZDIiIyNNr7JaR8woXuapEEYPRZLsb8y2WxLHCuPqoLlgWU8hzc835Xkp5KwNHswozQQypWAUl/6h4Y99aSVzbV+itXdnTjlzGuxiPnW2aZKiLTAUlfzfy6DXQMyjTAyfT5IUQXY5rU7rVfEqKCjgwIED9OzZ03RMLpfTs2dP9uzZ4/C87Oxs6tatS3R0NP379+fYMcddadRqNZmZmRYvf0HhqudlZ7XR2fagAjsxL+vgvcXGbH1hMUKlQs42fVsGFMxCV6UxqtxU1qlm8h/5ziKfx1kjD6Fdllg0SclOg6/6w77PARn0eAue+IoMvSFckpmv8Y6RXsar4nX9+nV0Op2N5xQREUFqaqrdc5o0acKKFSv46aef+Oabb9Dr9XTp0oVLly7ZHT9nzhzCw8NNr+joaI8/R0lhTAgNcFDPy36eV9ExL7vTxttCZtQ8i5I4xqoSCpnJlvNSTTKf/o1rNR8gUKZhvmoZ05RfosSxF+BMoJztbdx5+hrf7U92cnbZw7SF6/JBQ/5W0m4IDIMh30K3iSCXo75dzSNbLTwvvyA2NpZhw4bRpk0bunXrxg8//ED16tUtunmbM2XKFDIyMkyv5GT/+SUwThvt1dGScBDzkhv3Njq+rsZOwN7G87JXjFAut0jb0KsqsrfThyzSPgbASOVvfBUwlyrY925lTgTK+JG9TPthK/5l0vrDnL6a5fihygA666l6wrewojdkXoaqjeG5rdCkj2mM+va2rCwxbSx9qlWrhkKh4OpVy6Dv1atXiYyMdOkaAQEBtG3b1lSW2prAwEDCwsIsXv5CoXjZriYaSuLYbsx2rSSO4Yc+KKBQsIKsYl4WHbNNxQhlVkmqkKeRWKR9nDEFr5EtBdFFcZyNgW/RXHbB5r53OjW8mumfJaQlSSIjr+ipnTEWqUBHxe1vw4YXQKeGu3rD6K1Q/S6L8ca9rdlqbblsKedV8VKpVLRv356tW7eajun1erZu3UpsrGs74HU6HUeOHCEqKqqkzPQaxi1C9lIlzGMi9lYbnZfEMXxmHisLsvK8LMpAm/Y2yiymd5JZhv3v+rsZUDCTRH0EtWXXWa+aTj/5bucPaMbGQ1d4duU+spzEb/x1/+ObG47Sesbv7D1/w+k4nV6iMpl8FTCXsITPDAfvmwRPfgtB4TbjjdNGnV5yqwlKWcHr08a4uDg+++wzvvzyS06cOMGLL75ITk4OI0eOBGDYsGFMmTLFNH7mzJn8/vvvnD9/noMHD/L0009z8eJFnnvuOW89QolhbMJhb2+j2mxbjmXA3vCvs9VGjb2AvbXnhZ3VRjuel3nzjbNSbfoXzGabrjXBsgI+VH3MFOUqFLdbDjlbUbx4I5c/T6Yx59eTpmMJyelM3+h4McZfWL03CYBFf5xxOk51/Sj/C3yLropj6JUhhjLN978JjhqwmP0MlMcVR6/XsB88eDDXrl3jnXfeITU1lTZt2rB582ZTED8pKQm52Tfv1q1bjB49mtTUVCpXrkz79u3ZvXs3zZo189YjlBgKU6qEbcC+wJF4uZBhb5wGmntbTj0vXWHA3mZvo9Vf/EwqMEozkdel7xin3Mjzyp+JkSXxkuYlZLjX8sw6UbVMp1McWU/LzeNQyPK5oI9A0/8bGjdz3ORGkiSLP2CZ+Vpq+E9ExCN4XbwAxo8fz/jx4+1+tn37dov3CxcuZOHChaVglfdxlipRYF5jy+zzwmmj4+ua+j3a8byMp9mtpCqXW0wbDeJlm/6gR84H2ic5rq/HBwGfcp/iCBtlb/Ge7m1OUdWxYWUcCTvfFL3OUFv+78UogB26VrykGc/K8MZOr2WddlIeVxy9Pm0UOMYY67LXgMNe8w0w29voNEnVtny0dYa9RcDerJKquZAWtbfxZ31nHiuYQZK+OnXlaSzKnkgf+V6H48s6Ns5w3i1Y9QT8vRiAC01HM1IziUwq2iapWqG2qubhLFZYVhHi5cMYUyTsVZWwlyYBheLmPM/LmCphvtroJEnVYcxLsts5yHx6d1KqQ7+C2fyla0Ew+SxVLWaCci1y3C+lU6ZmjWknYHkPOLcVlMHwn/9yssUEwyZrrJJU7aC28njLY8xLiJcPYzdV4va/xh9eh56XKxn2ZsJnLV7mnptR7BRymWlaargH5BXY/pIFWAWY0wllhOYNvg8cCMB45U98FjCfMGwrqpZlTF/RE/+Dz3vCrUQIrwOjfoeWj9vmeTlBbfVHozzmegnx8mEKq0rYfpvsCRC40LcRM88rwMm00c69XPW87KV26FDw35BRvFIwlnwpgAcU8WxQvU1D2WWHdtrg566XTK+Hbe/B2qehIBvq3WsoHBjVCrCscOv2tNHHY17XstRsPHTF6YzAXXwiYC+wj9Grshuwvy0a1tNGd5JUnU0b9XZjXnKLxQFJkixSJYw4ahgC8JP+Hs4W1GK5agEN5KlsUL1DnOZFtuj9s+W8q1Qkl4npi2HH7ZhfpxfhoVmgCDCNccvz0vh2zCtbreXfxBvsOnOD3eeuczLVsDuiXtUQWtWu5JF7CPHyYeymStz2rOzV8gLzMtCOr6ux47XZBuwL/2+8llIhw/x2esl+5yBrQbXmmFSffurZLFF9SGf5CT5TLWCh5j98qBuIVAYnAw1kV1gesIBG6iugCIRHF0Gbp2zGmfcC0BThoVhPG70d89Lo9CQkp7PrzHX+PnudhOR0m94GzaLCPGqnEC8fprCel+Npo7VQGGeYTgP2esfTRuMqo73tJgqrtAxHZaAdVcEwf4wbhPN0wRTeVK5ipPI3Xgv4nubyC8RpXiQb+9VANx9NpUG1ikSG+1HxydO/sUH1NmGyPG7Iq1L12XVQq73doRaeVxG9AWxXG0tXvCRJ4mRqFn+fNYjV3sSb5BZY/ixEVwnmnkbV6NqoGrENqlK1ome7dAvx8mFcSVK1Dti71oDDXlUJx0mqRpRWAXtJwhTzUinkdluxOUOLkhna4RyT6vGucgUPKQ7wo2waYzRxJEq2272+2nOR7w9c4tjM3i5dvyT5+M8zbEi4wrrnY6lcQWU7QJLgr3nw57uEyST+1TdhWbVprHAgXGDpeTnr8AS24lUaeV6XbuXeFivDVPB6doHF51UqqIhtWNUgWA2rUadqyZakFuLlw/SMieDYlUw61q9iOmacNsYnpQO2Bf6KyvOSJMluVQnbVAn7npdthr1xk7fcIrBvD0d7E9frunFGX4tlqkU0ll/mJ9XbvKwZx3a9bYXQnAId3+1LZtDd3i1tNO/30wAs2XaWtx6x2t2hzoYNL8KJjQB8re3JTO0wWsorOb2mzizOpSkqYG+9s6EEYl63cgrYc/4Gu85eZ/fZ61y4kWvxeXCAgo71q3BPo2p0aVSVmMgwC8+8pBHi5cMMujva5pe0f5uabD+Vxo0cw1+9ilZ14YsqRmj+191ZGWi7npfCOmAPebenCsEqBZm3py6OSt/YzTC/zSGpEf3Us1mqWkQH+WlWBMxjnnYQn+j6Yb3MOOn7w9zTuBo1KwU7vF5pcSvXSjRunoc1QyHtOMgDoO883l5X3aVrad0I2JdEhn1egY59F27y9znDVPDYlUyL2KdCLqN17XDTVLBtncpFxjdLEiFefsZ9d1Vn79QHDH8Rz1znkVY1LT4vLEZo/3zzJfggZ6kSLkwb9ZJkChyHqJSAGnC8B7Go5f9rVGJIwVtMV37JUOVWJgWspbk8kYmaF8jFMs6VW6Bl26k0mtcMo0ao92JgOeaicXYrrH8W8tOhYoRhY3WdTrDuZwAn0m1AZye3zhHG1UaVUk6BVl+smJckSWTma9l15jrf/HORAxdv2YjiXREV6dKwGvc0qkanBlUIDQpwcLXSR4iXH6JUyLm3cXXubWz7F72wGKH9H36LDd3Kwgx+64KHjgL21lUlCqeNhV6co6qornTW1qDkTe0ojkn1mK5cSV/FvzSQpTBGE0eyVFhx95cjqSzYcpoKKoVXY2A5BVqD0u/+CP6YBpIeanWAwd9AWJRbdbbMvz7OSmZDYcyresVALqfnubWKp9Hpee+XE3z7b5LNanFUeBBdG1Wja6OqdG1YjRphvrs4IsSrjGFabTRbNTyeksmeczfYc+4G/ybeBAzekVG87HUOshfwt5ukejv2Yu7FOQp7aIv4hTRnte4BTulrs1S1mBh5Mv9TvcV4zcvs0rcEYMEWQ8wpp8C7dawK8rPhh9FwZJ3hQNunoe8CUBpW1sy9qaJ0zC3P67bHW7WiisvpeU7zvPR6iYs3czlyOYMjl9LZdfYGJ1IKq91WraBiaOe69G9TkwbVKjiteOtLCPEqY1jXsJ+7+SSf7jhvMSY0SMnTnesSFRaMSimnXrUKps8u38rjv7sSycyz/Utu7XlptHqTtxDsIc/LnANSEx5Vz+ZT1ULayM/xZcBc5mqH8JmuL6Wdbv/Vngt8veciK0bcTXQVwypaLa4x+8ZiuHYe5EroPRfufs5i3mz+zEU9vTsxL6PnVfX2SmdOgQ6dXkIug0u38jh8KYPDl9M5cimDI5czbKaVISoF859oTY+mNWwWa/wFIV5lDFOqxO1fhJ8PpwDQuUEV7m9ag9gG1WhWM8wkQtsmdCc0SEmuWkflkABu5WqYtem43Wtbl8TJNVvxCnbhF6Aob8IeV6nC4IK3maX8gkHKHbwZsJrm8gtM1owmH4N3cyU9j42HrjDk7jqEh3g+JpOQnM47PxmKIq7+N4k3ejels/w4SwIWU1WfBSFVYdBXUO8em3OLEiFzdGbpEQ77Nt7GGPMyz536cvcFvv7nIonXbfeMqpRymkWF0ap2OC1rhXNP42pEhXt/weNOEOJVxijcmA0pGXlcupWHXAafDetgN9ha6/aKXVhQADsn9WBDwhVW702ymFYYCVbJMc/MyFUXilegxbTRM56XETUqJmnHcESqzzvKrxmg2E0j2RWeL3iNy1TniWV7uJyex6HkdJY+7TiPyh2y1Vq0Oj0KucyiKGLVkAD0/yzjm4D3UMr0nKA+MWP+B5Xsp25YCHYR80ati9NGnV7iZKrh+xMapDQF7Wfe/qMToJARExVGy1rht8WqEo0jKrqcf+cvCPEqY5j2NkoS+y/cAqBZzTCXVolCgwJ4pnNdnu5Uh4TkdHafu0FGnoafD6cYEhAbVLNITFz0hyHuFKiUW8RJHGXY64rwJpwj42vdQ5zWR/OJahEt5BfYGPgW4zSv8E+6Ic/q16OpfPtvEkM61rmD+xhoPeN3dHqJX1+513QskAJ6nJqB/PJPyGXwo64rUzTPcdKBcIGlB+WsrwCATmd/2ng1M5/4pHQSktNJSL7FkUsZplhfoFJB7+aRbD6aSt2qIQxoW4sRXepRIbDs/2qX/ScsZyjMpo37LhiC8x3qVnF2ig0ymYy2dSrTtk5lAKY+HGP67D/tajNp/WEAzqRlA4a4i7lcOQzYe6CiwF4phkfV7/KpagEt5Rf4JuA9Zmmf4UvdQ4CMKT8c8Yh4GWOGZ28/YyQ3WKZaSMPL55FkcmYXDOG/uocpKvZm7kEVNW02//pk5mn4dMc5thy/yv6LtxyeE6iU8+GQtkiS5DeBdk8hxKuMYRQOnSSx77bndXc998TLGQq5jHPvPcyRyxlcuJ7DxRu5xDasyld7LhQOcjRtLEbMyx5XqMbjBdOZE/A5jyl2MSPgS1rKE3lT8yxq7GzVcRPz9IY8jY72slMsUy2iuiyDfGU41/ss47/rXHsW82d2lv5wJT2PCzcKY1UbEq6wIeEKYPhyNokIpW2dSrSJrkSNsCBGfrEPKJyulzfhAiFeZQ65Kc8LU1zk7nqVPXoPhVxGm2jDL5KRr/+5WGiDDFpHV+JQcrrFeZ6s5aRGRZzmRY7p6zFVuYrHFTtpJLvECwWv8cuRFP44fpX3HmtZrJU0czOjz6/hW9UcVDIdJ/TRHGj/MW0i2wK7XLqW+bTRfCqYnlvA48v2cDYtm0ohAaRbZ+qbsf6FWNqbec+pGYX9K62bBZcnylYET2Czcbpu1ZBSSTS0nDbK+GlcV7a+3o2TswoTSKvY28B8h3f9r+5hhmkmc0uqSBv5ef4X+CYrVq/mh/jLrPg7sVjNWHV6iQC0vKf8nNjjs1HJdGzSdeKxghncCKjltG6/NeaeV/LNPCZ/fxiNTs+gT/eYpqTpuRrkMmhXx/IPAsDbjzSzEC6w3Kjvze053qb8PnkZxXpjrLvxLo/YcNuEhtUrWng+d0WGMqVPUwC6N3Ftv58r/K1vyaMFszmhr0N1WSbfqt5lqOIP/m/zKfos/sttAdNnpvCtajZPKf9EQsb7micZr3mZPILQ6PQ2DV6tN8FLksSZq1nUm/wzvx5Nsfhszb5kvj9widNXs03HljzVjvi3H+KHsV2Z1b+FxfhIO394zCvrKsrhdNGImDaWMayD5R3re3bK6Ajz3yFH1SMkSeL5bg15vltDAOpN/tlj978k1eCxgul8ELCcRxT/8G7ACprLLjA9dThqrd6l6aMkSSxb/R1PJU6lg/w6mVII6+vNYOmpWqYxGr3eZktNrkZHjlrLP+dv8N4vJ7iaqTZ9Zq/R7Hf7k5HLDNPTvVMfIMJMoCLCLGteRYbb1sAy97zKWPaDWwjxKmNYpyl08GCw3hnmd3XkDDirMeYJ8ghivOYljurrMUm5lqeUf3KX/BKJic2IueuuIs8/9stSnj39NoEyLWf0tRitiaMerYBrpjEarURmnmV8qsW034q89t31KrNkaDti5/zJwdvljMC2fFDVioHIZIUpYZF2EknN96E6yqkrDwjxKmOY/zBXraCigdnWH2/YYI4raV4X5vZFkiR2n7vBG98f5tKtPDfvLmOZrh8npLp8GPARHeSnubrqIQYWvMoJZRNOzOxtuzKn08Dvb9Fi3zKQwRZde167XdE12MyLAljxd6Kb9hioWiGQGqFBdL+rOltPppmOWzdXMXZo0t5Wrxqhzj2v8ixe5djpLJuYe14d6lUutSV08/vYqVoNOK/nZX2tro2qseuN+7kwty/zn2jttj079K3pXzCL0/paRMhusUY1i0f1f5J800oMc67D1wNh7zIAFmkfY4zmNVMpans7DYpDtVDDYsXj7WtbHLcuJgmWeyDtZcW78rUuD/jEoy9ZsoR69eoRFBREp06d+Pfff52OX7duHU2bNiUoKIiWLVvyyy+/lJKlvo/5X2JP5ncV1wZzipspEawqjFfVruz6frwLUhQDC2bym64DgTItHwQsJ2zbZIOnBaSc3Eveknvhwl/ky4J5O2gKi7SPO20C8kT72ozoUo96bpY4rlrB4EE9EBNBZbP9l/baxLmzwCA8Ly+ydu1a4uLimDZtGgcPHqR169b06tWLtLQ0u+N3797NkCFDGDVqFPHx8QwYMIABAwZw9OjRUrbcNzEPoZRWvAusY16OA/bmWHc+coR5uZfoyoWicf69h4s8N4dgXtC8ynzN4wBUOvoliQt6cun3j6n07SME514hUR/BI/kz+Dq9pdNrffd8LB880Zrp/ZqzfWIPl2w3YnxWlVJO/zaFCwD2Sma7o/GOtmKVB7wuXgsWLGD06NGMHDmSZs2asWzZMkJCQlixYoXd8YsXL6Z3795MnDiRmJgYZs2aRbt27fj444/tjler1WRmZlq8yjIymYxBHWrTM6YGLWqGldp9zb2jzg0sRdNYcaJLw2oWx+0VU7SHeWJ6lFnnIFfrpUvI+Uj3GKMKXidLCqZ+TgK1d79JsKyAbbrW9C+YzVmpcDp3b+NqTOnTlNWjO5lSOwCLXgLuUimkMMftiQ6Ge4UHB9gV+lFd6wPQq3mEzWfWNI0MLbZN/o5XA/YFBQUcOHCAKVOmmI7J5XJ69uzJnj177J6zZ88e4uLiLI716tWLDRs22B0/Z84cZsyY4TGb/YH/e9z9GNGd8mbfGK5lqakcomL0vQ0sPtsSdx9/nbnOY+1qWRyf90RrPt1xjic61ObH+Ms0rxlu99r929Rk7b4k7m8awaC7a7P/4i2e6mTYv7j4yTZM33iM1tGV2HH6mtPCDVv17RlQMJPlAQtoKE9hibYf87WDCAlUgVqLSiFn31s9CQ8unNY1jwpnxd+J3N+0hs315j/Rmvm/n+LK7Yz3T59pz8UbObz3y0nuaVSNXWev0zOmBrUrh1jEuprXDOeToe0IDbL/6zehVxPurl+FLg2rOn6W17uRlqmmUY3yK15IXuTy5csSIO3evdvi+MSJE6WOHTvaPScgIEBavXq1xbElS5ZINWrUsDs+Pz9fysjIML2Sk5MlQMrIyPDMQwi8jl6vN/2r1uikjLwC6VaOWsrXaKVbOWrpWla+dDUzT7p8K1eSJElKz8ySzp85Iel0eunC9WzTeem5BXavr9PpS+1ZBJKUkZHh0u9omU+VCAwMJDDQs80uBb6Fceolk8lQKWUWW2bs7f0LD61IeKhhOli3qiGVxPo8c0qznZfAdbwa86pWrRoKhYKrV69aHL969SqRkZF2z4mMjHRrvEAgKJt4VbxUKhXt27dn69atpmN6vZ6tW7cSGxtr95zY2FiL8QBbtmxxOF4gEJRNvD5tjIuLY/jw4XTo0IGOHTuyaNEicnJyGDlyJADDhg2jVq1azJkzB4BXXnmFbt26MX/+fPr27cuaNWvYv38/y5cv9+ZjCASCUsbr4jV48GCuXbvGO++8Q2pqKm3atGHz5s1ERBiWiZOSkpCbpRF36dKF1atX89ZbbzF16lQaN27Mhg0baNGihaNbCASCMohMkkp4t6yPkZmZSXh4OBkZGYSFlV4elEAgcA1Xf0e9nqQqEAgExUGIl0Ag8Eu8HvMqbYyz5LK+TUgg8FeMv5tFRbTKnXhlZWUBEB3tuNeeQCDwPllZWYSH298yBuUwYK/X67ly5QqhoaGmzOzMzEyio6NJTk4uV0F88dziuX0RSZLIysqiZs2aFpkG1pQ7z0sul1O7dm27n4WFhfn0N7WkEM9dvvCH53bmcRkRAXuBQOCXCPESCAR+iRAvDJUnpk2bVu6qT4jnFs/tz5S7gL1AICgbCM9LIBD4JUK8BAKBXyLESyAQ+CVCvAQCgV9SbsXr5s2bDB06lLCwMCpVqsSoUaPIzs52Ov6ll16iSZMmBAcHU6dOHV5++WUyMjJK0eo7x93nBli+fDndu3cnLCwMmUxGenp66Rh7B5TXRsbuPPexY8f4z3/+Q7169ZDJZCxatKj0DPUA5Va8hg4dyrFjx9iyZQubNm1i586djBkzxuH4K1eucOXKFebNm8fRo0dZuXIlmzdvZtSoUaVo9Z3j7nMD5Obm0rt3b6ZOnVpKVt4Z5bWRsbvPnZubS4MGDZg7d65/9oAo2SZGvsnx48clQNq3b5/p2K+//irJZDLp8uXLLl/nu+++k1QqlaTRaErCTI9zp8+9bds2CZBu3bpVglbeOR07dpTGjRtneq/T6aSaNWtKc+bMsTt+0KBBUt++fS2OderUSXr++edL1E5P4+5zm1O3bl1p4cKFJWid5ymXnteePXuoVKkSHTp0MB3r2bMncrmcvXv3unwdY6VHpdI/toh66rl9GWMj4549e5qOudLI2Hw8GBoZOxrvixTnuf2dcileqamp1Khh2QFZqVRSpUoVUlNTXbrG9evXmTVrVpFTLl/CE8/t61y/fh2dTmfqgWAkIiLC4TOmpqa6Nd4XKc5z+ztlSrwmT56MTCZz+jp58uQd3yczM5O+ffvSrFkzpk+ffueG3yGl9dwCgS/hH/MdF3n99dcZMWKE0zENGjQgMjLSJoip1Wq5efNmkYHLrKwsevfuTWhoKD/++CMBAQF3avYdUxrP7S+U10bGxXluf6dMiVf16tWpXr16keNiY2NJT0/nwIEDtG/fHoA///wTvV5Pp06dHJ6XmZlJr169CAwMZOPGjQQFBXnM9juhpJ/bnzBvZDxgwACgsJHx+PHj7Z5jbGT86quvmo75WyPj4jy33+PtFQNv0bt3b6lt27bS3r17pV27dkmNGzeWhgwZYvr80qVLUpMmTaS9e/dKkiRJGRkZUqdOnaSWLVtKZ8+elVJSUkwvrVbrrcdwG3efW5IkKSUlRYqPj5c+++wzCZB27twpxcfHSzdu3PDGIxTJmjVrpMDAQGnlypXS8ePHpTFjxkiVKlWSUlNTJUmSpGeeeUaaPHmyafzff/8tKZVKad68edKJEyekadOmSQEBAdKRI0e89QjFwt3nVqvVUnx8vBQfHy9FRUVJEyZMkOLj46UzZ8546xHcotyK140bN6QhQ4ZIFStWlMLCwqSRI0dKWVlZps8TExMlQNq2bZskSYVpAvZeiYmJ3nmIYuDuc0uSJE2bNs3uc3/xxRel/wAu8tFHH0l16tSRVCqV1LFjR+mff/4xfdatWzdp+PDhFuO/++476a677pJUKpXUvHlz6eeffy5liz2DO89t/F5bv7p161b6hhcDURJHIBD4JWVqtVEgEJQfhHgJBAK/RIiXQCDwS4R4CQQCv0SIl0Ag8EuEeAkEAr9EiJdAIPBLhHgJBAK/RIiXQCDwS4R4CXyCESNG2C3l07t3b2+bJvBRylRVCYF/07t3b7744guLY45a02s0GptyRAUFBahUKrfvW9zzBN5FeF4CnyEwMJDIyEiLV+XKlQGQyWQsXbqUfv36UaFCBd59912mT59OmzZt+Pzzz6lfv76pRFFSUhL9+/enYsWKhIWFMWjQIIs6V47OE/gXQrwEfsP06dMZOHAgR44c4dlnnwXg7NmzfP/99/zwww8kJCSg1+vp378/N2/eZMeOHWzZsoXz588zePBgi2tZnyfwP8S0UeAzbNq0iYoVK1ocmzp1qqnl2lNPPcXIkSMtPi8oKOCrr74yFWPcsmULR44cITExkejoaAC++uormjdvzr59+7j77rvtnifwP4R4CXyGHj16sHTpUotjVapUMf3fvOuRkbp161oI0IkTJ4iOjjYJF0CzZs2oVKkSJ06cMImX9XkC/0OIl8BnqFChAo0aNXL6uSvHXL2XwL8RMS9BmSImJobk5GSSk5NNx44fP056ejrNmjXzomUCTyM8L4HPoFarbXoMKpVKqlWr5vI1evbsScuWLRk6dCiLFi1Cq9UyduxYunXrZnfaKfBfhOcl8Bk2b95MVFSUxeuee+5x6xoymYyffvqJypUrc99999GzZ08aNGjA2rVrS8hqgbcQNewFAoFfIjwvgUDglwjxEggEfokQL4FA4JcI8RIIBH6JEC+BQOCXCPESCAR+iRAvgUDglwjxEggEfokQL4FA4JcI8RIIBH6JEC+BQOCX/D/75yvJhJa8YQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.021284, + "end_time": "2024-02-29T06:08:24.361297", + "exception": false, + "start_time": "2024-02-29T06:08:24.340013", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4003.146627, + "end_time": "2024-02-29T06:08:27.108053", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/tvae/2/mlu-eval.ipynb", + "output_path": "eval/treatment/tvae/2/mlu-eval.ipynb", + "parameters": { + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "path": "eval/treatment/tvae/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "tvae" + }, + "start_time": "2024-02-29T05:01:43.961426", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/tvae/model.pt b/treatment/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..c243f67991b91af4337a968d955e10736869610a --- /dev/null +++ b/treatment/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb354a7fdd1d44ac008ae0f52498d5db753f2aebcebc84cee15e0a40919c4987 +size 74860097 diff --git a/treatment/tvae/params.json b/treatment/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..abf1d8b9d9a17b938050708aad05962542f67d36 --- /dev/null +++ b/treatment/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "ALL", "synth_data": 2, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "fixed_role_model": "tvae", "mse_mag": true, "mse_mag_target": 1.0, "mse_mag_multiply": true, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file