diff --git a/contraceptive/lct_gan/eval.csv b/contraceptive/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..b7f5166d5661141b49b8ad1738e0852cf72b43b9 --- /dev/null +++ b/contraceptive/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.03109242501225145,0.09055740089827473,0.0031398469558362625,12.489086151123047,0.04181426391005516,1.1387050151824951,0.15034998953342438,3.8336263969540596e-05,3.991712808609009,0.04172290489077568,0.13467051088809967,0.05603433772921562,0.09302742779254913,0.02486800216138363,16.480798959732056 diff --git a/contraceptive/lct_gan/history.csv b/contraceptive/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..6b3e39327488bb68d3a49edbb6b16cfb34af9fa1 --- /dev/null +++ b/contraceptive/lct_gan/history.csv @@ -0,0 +1,13 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.017055697603565123,0.7837836365105371,0.000854024517475194,0.0,0.0,0.0,0.0,0.0,0.017055697603565123,900,225,209.5561990737915,0.9313608847724066,0.23284022119310166,0.11550631119145287,0.02723778669618898,0.35535829481002906,0.0011922009129072346,0.0,0.0,0.0,0.0,0.0,0.02723778669618898,450,113,91.00356817245483,0.8053413112606622,0.20223015149434406,0.12002328037391458 +1,0.007602974076354359,0.6694789405275434,0.00012108089975418718,0.0,0.0,0.0,0.0,0.0,0.007602974076354359,900,225,209.41460299491882,0.9307315688663059,0.23268289221657648,0.0941959698839734,0.004413039641092635,0.45001505491487465,4.256673724010495e-05,0.0,0.0,0.0,0.0,0.0,0.004413039641092635,450,113,91.49867558479309,0.8097227927857795,0.20333039018842908,0.06237377326902563 +2,0.005956742855616742,0.6394465912903183,6.889980071151411e-05,0.0,0.0,0.0,0.0,0.0,0.005956742855616742,900,225,209.05843949317932,0.9291486199696859,0.23228715499242147,0.09135436112475064,0.004399605713939915,1.215607570140407,2.922537710473547e-05,0.0,0.0,0.0,0.0,0.0,0.004399605713939915,450,113,88.99790143966675,0.7875920481386438,0.19777311431037056,0.04754088749589844 +3,0.005293925739824772,0.6930031799355281,5.091876555952298e-05,0.0,0.0,0.0,0.0,0.0,0.005293925739824772,900,225,209.8546106815338,0.9326871585845947,0.23317178964614868,0.09368091402575374,0.005122498869895935,0.6850604875349726,0.0001063596828682662,0.0,0.0,0.0,0.0,0.0,0.005122498869895935,450,113,90.19163846969604,0.7981560926521774,0.2004258632659912,0.06673125488749515 +4,0.004406307417407839,0.5391749190130491,3.865118440087487e-05,0.0,0.0,0.0,0.0,0.0,0.004406307417407839,900,225,208.6917781829834,0.9275190141465929,0.23187975353664822,0.0995840290437142,0.005340991177492671,0.6295711460246051,7.291253590576869e-05,0.0,0.0,0.0,0.0,0.0,0.005340991177492671,450,113,91.7313449382782,0.8117818136130814,0.20384743319617377,0.06550657832418132 +5,0.00469050889802424,0.9336842445089064,3.8414434089242215e-05,0.0,0.0,0.0,0.0,0.0,0.00469050889802424,900,225,208.08384490013123,0.9248170884450276,0.2312042721112569,0.09579447591263388,0.004810444195496125,0.3487526955594519,8.624510295827805e-05,0.0,0.0,0.0,0.0,0.0,0.004810444195496125,450,113,90.39807367324829,0.7999829528606044,0.20088460816277398,0.0735004077937487 +6,0.005119060436975107,0.6433215488478984,6.873547569881813e-05,0.0,0.0,0.0,0.0,0.0,0.005119060436975107,900,225,207.70152282714844,0.9231178792317708,0.2307794698079427,0.09771538318652245,0.004957494798161659,1.346057123426031,5.000207313663115e-05,0.0,0.0,0.0,0.0,0.0,0.004957494798161659,450,113,90.1600124835968,0.7978762166689982,0.20035558329688177,0.06382843967428249 +7,0.00532732381252572,0.4333305762650606,5.976084272669491e-05,0.0,0.0,0.0,0.0,0.0,0.00532732381252572,900,225,207.5520989894867,0.9224537732866075,0.23061344332165187,0.09995691900038058,0.0050655570465864405,1.4249071690357547,4.881459755913574e-05,0.0,0.0,0.0,0.0,0.0,0.0050655570465864405,450,113,90.09133553504944,0.7972684560623844,0.2002029678556654,0.05493936902612646 +8,0.004071665801651155,0.3260331416654134,2.1085142560956504e-05,0.0,0.0,0.0,0.0,0.0,0.004071665801651155,900,225,209.66889786720276,0.931861768298679,0.23296544207466974,0.10195785622629855,0.005279655439727422,0.9929788724251226,6.48734679797379e-05,0.0,0.0,0.0,0.0,0.0,0.005279655439727422,450,113,91.38696694374084,0.8087342207410694,0.20308214876386854,0.06704739362940984 +9,0.004601976428077453,0.3923278355509547,4.1437573981469074e-05,0.0,0.0,0.0,0.0,0.0,0.004601976428077453,900,225,202.2607822418213,0.8989368099636502,0.22473420249091255,0.1009762748144567,0.003612730009287285,1.519655740890833,3.1731855463335754e-05,0.0,0.0,0.0,0.0,0.0,0.003612730009287285,450,113,86.41371130943298,0.7647231089330353,0.19203046957651773,0.05825823241436805 +10,0.003639938447223459,0.2671018096537945,2.6972246297165055e-05,0.0,0.0,0.0,0.0,0.0,0.003639938447223459,900,225,204.09564805030823,0.907091769112481,0.22677294227812025,0.10696077811221281,0.002935796806381808,1.918275244656198,1.3822624714984066e-05,0.0,0.0,0.0,0.0,0.0,0.002935796806381808,450,113,87.83020257949829,0.7772584299070645,0.19517822795444065,0.04963276007591821 +11,0.0033298865797243907,0.48618020502341713,2.140108548165483e-05,0.0,0.0,0.0,0.0,0.0,0.0033298865797243907,900,225,201.81425046920776,0.89695222430759,0.2242380560768975,0.10155585827512874,0.002946255624992773,1.365417978676351,1.4400359725914503e-05,0.0,0.0,0.0,0.0,0.0,0.002946255624992773,450,113,87.90818929672241,0.777948577847101,0.19535153177049425,0.05489195802813577 diff --git a/contraceptive/lct_gan/mlu-eval.ipynb b/contraceptive/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..88926fa9bf42cee55054c96312ecb2d59a3bacfd --- /dev/null +++ b/contraceptive/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2303 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:59.926982Z", + "iopub.status.busy": "2024-03-22T19:36:59.926612Z", + "iopub.status.idle": "2024-03-22T19:36:59.959766Z", + "shell.execute_reply": "2024-03-22T19:36:59.959082Z" + }, + "papermill": { + "duration": 0.048315, + "end_time": "2024-03-22T19:36:59.961821", + "exception": false, + "start_time": "2024-03-22T19:36:59.913506", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:59.986986Z", + "iopub.status.busy": "2024-03-22T19:36:59.986611Z", + "iopub.status.idle": "2024-03-22T19:36:59.993605Z", + "shell.execute_reply": "2024-03-22T19:36:59.992766Z" + }, + "papermill": { + "duration": 0.021906, + "end_time": "2024-03-22T19:36:59.995635", + "exception": false, + "start_time": "2024-03-22T19:36:59.973729", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:00.019438Z", + "iopub.status.busy": "2024-03-22T19:37:00.018936Z", + "iopub.status.idle": "2024-03-22T19:37:00.022856Z", + "shell.execute_reply": "2024-03-22T19:37:00.021963Z" + }, + "papermill": { + "duration": 0.018194, + "end_time": "2024-03-22T19:37:00.024950", + "exception": false, + "start_time": "2024-03-22T19:37:00.006756", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:00.048617Z", + "iopub.status.busy": "2024-03-22T19:37:00.048315Z", + "iopub.status.idle": "2024-03-22T19:37:00.052533Z", + "shell.execute_reply": "2024-03-22T19:37:00.051690Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018212, + "end_time": "2024-03-22T19:37:00.054427", + "exception": false, + "start_time": "2024-03-22T19:37:00.036215", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:00.077831Z", + "iopub.status.busy": "2024-03-22T19:37:00.077295Z", + "iopub.status.idle": "2024-03-22T19:37:00.082918Z", + "shell.execute_reply": "2024-03-22T19:37:00.082045Z" + }, + "papermill": { + "duration": 0.019446, + "end_time": "2024-03-22T19:37:00.084750", + "exception": false, + "start_time": "2024-03-22T19:37:00.065304", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f4f0a115", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:00.111051Z", + "iopub.status.busy": "2024-03-22T19:37:00.110431Z", + "iopub.status.idle": "2024-03-22T19:37:00.115765Z", + "shell.execute_reply": "2024-03-22T19:37:00.114973Z" + }, + "papermill": { + "duration": 0.020918, + "end_time": "2024-03-22T19:37:00.117637", + "exception": false, + "start_time": "2024-03-22T19:37:00.096719", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"lct_gan\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 1\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/lct_gan/1\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.010803, + "end_time": "2024-03-22T19:37:00.139438", + "exception": false, + "start_time": "2024-03-22T19:37:00.128635", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:00.162833Z", + "iopub.status.busy": "2024-03-22T19:37:00.162271Z", + "iopub.status.idle": "2024-03-22T19:37:00.171358Z", + "shell.execute_reply": "2024-03-22T19:37:00.170598Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022705, + "end_time": "2024-03-22T19:37:00.173192", + "exception": false, + "start_time": "2024-03-22T19:37:00.150487", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/lct_gan/1\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:00.196567Z", + "iopub.status.busy": "2024-03-22T19:37:00.196330Z", + "iopub.status.idle": "2024-03-22T19:37:02.274147Z", + "shell.execute_reply": "2024-03-22T19:37:02.273085Z" + }, + "papermill": { + "duration": 2.091878, + "end_time": "2024-03-22T19:37:02.276201", + "exception": false, + "start_time": "2024-03-22T19:37:00.184323", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:02.303841Z", + "iopub.status.busy": "2024-03-22T19:37:02.303357Z", + "iopub.status.idle": "2024-03-22T19:37:02.316110Z", + "shell.execute_reply": "2024-03-22T19:37:02.315206Z" + }, + "papermill": { + "duration": 0.02929, + "end_time": "2024-03-22T19:37:02.318006", + "exception": false, + "start_time": "2024-03-22T19:37:02.288716", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:02.342193Z", + "iopub.status.busy": "2024-03-22T19:37:02.341909Z", + "iopub.status.idle": "2024-03-22T19:37:02.349491Z", + "shell.execute_reply": "2024-03-22T19:37:02.348614Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021752, + "end_time": "2024-03-22T19:37:02.351453", + "exception": false, + "start_time": "2024-03-22T19:37:02.329701", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:02.375787Z", + "iopub.status.busy": "2024-03-22T19:37:02.375495Z", + "iopub.status.idle": "2024-03-22T19:37:02.474933Z", + "shell.execute_reply": "2024-03-22T19:37:02.474086Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.114926, + "end_time": "2024-03-22T19:37:02.477611", + "exception": false, + "start_time": "2024-03-22T19:37:02.362685", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:02.504711Z", + "iopub.status.busy": "2024-03-22T19:37:02.504080Z", + "iopub.status.idle": "2024-03-22T19:37:07.156105Z", + "shell.execute_reply": "2024-03-22T19:37:07.155324Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.668007, + "end_time": "2024-03-22T19:37:07.158408", + "exception": false, + "start_time": "2024-03-22T19:37:02.490401", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 19:37:04.751058: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 19:37:04.751112: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 19:37:04.752906: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:07.183123Z", + "iopub.status.busy": "2024-03-22T19:37:07.182562Z", + "iopub.status.idle": "2024-03-22T19:37:07.188539Z", + "shell.execute_reply": "2024-03-22T19:37:07.187524Z" + }, + "papermill": { + "duration": 0.020379, + "end_time": "2024-03-22T19:37:07.190540", + "exception": false, + "start_time": "2024-03-22T19:37:07.170161", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:07.215777Z", + "iopub.status.busy": "2024-03-22T19:37:07.215511Z", + "iopub.status.idle": "2024-03-22T19:37:15.361034Z", + "shell.execute_reply": "2024-03-22T19:37:15.359853Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.161045, + "end_time": "2024-03-22T19:37:15.363624", + "exception": false, + "start_time": "2024-03-22T19:37:07.202579", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T19:37:15.390983Z", + "iopub.status.busy": "2024-03-22T19:37:15.390664Z", + "iopub.status.idle": "2024-03-22T19:37:15.397343Z", + "shell.execute_reply": "2024-03-22T19:37:15.396543Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022416, + "end_time": "2024-03-22T19:37:15.399240", + "exception": false, + "start_time": "2024-03-22T19:37:15.376824", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:15.423691Z", + "iopub.status.busy": "2024-03-22T19:37:15.423432Z", + "iopub.status.idle": "2024-03-22T19:37:15.427872Z", + "shell.execute_reply": "2024-03-22T19:37:15.427059Z" + }, + "papermill": { + "duration": 0.018915, + "end_time": "2024-03-22T19:37:15.429769", + "exception": false, + "start_time": "2024-03-22T19:37:15.410854", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:15.454450Z", + "iopub.status.busy": "2024-03-22T19:37:15.454193Z", + "iopub.status.idle": "2024-03-22T19:37:15.967529Z", + "shell.execute_reply": "2024-03-22T19:37:15.966629Z" + }, + "papermill": { + "duration": 0.528118, + "end_time": "2024-03-22T19:37:15.969855", + "exception": false, + "start_time": "2024-03-22T19:37:15.441737", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_bs_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_synth_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:15.998281Z", + "iopub.status.busy": "2024-03-22T19:37:15.997971Z", + "iopub.status.idle": "2024-03-22T19:37:16.319621Z", + "shell.execute_reply": "2024-03-22T19:37:16.318764Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.338255, + "end_time": "2024-03-22T19:37:16.321610", + "exception": false, + "start_time": "2024-03-22T19:37:15.983355", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.73,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.Softsign,\n", + " 'loss_balancer_beta': 0.67,\n", + " 'loss_balancer_r': 0.943,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.65, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:16.350046Z", + "iopub.status.busy": "2024-03-22T19:37:16.349720Z", + "iopub.status.idle": "2024-03-22T19:37:16.457775Z", + "shell.execute_reply": "2024-03-22T19:37:16.456979Z" + }, + "papermill": { + "duration": 0.125285, + "end_time": "2024-03-22T19:37:16.460059", + "exception": false, + "start_time": "2024-03-22T19:37:16.334774", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/contraceptive [400, 0]\n", + "Caching in ../../../../contraceptive/_cache_aug_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/contraceptive [0, 200]\n", + "Caching in ../../../../contraceptive/_cache_bs_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/contraceptive [100, 0]\n", + "Caching in ../../../../contraceptive/_cache_bs_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/contraceptive [0, 50]\n", + "Caching in ../../../../contraceptive/_cache_synth/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/contraceptive [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T19:37:16.488607Z", + "iopub.status.busy": "2024-03-22T19:37:16.487899Z", + "iopub.status.idle": "2024-03-22T19:37:16.945681Z", + "shell.execute_reply": "2024-03-22T19:37:16.944677Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.474366, + "end_time": "2024-03-22T19:37:16.947922", + "exception": false, + "start_time": "2024-03-22T19:37:16.473556", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:16.976131Z", + "iopub.status.busy": "2024-03-22T19:37:16.975778Z", + "iopub.status.idle": "2024-03-22T19:37:16.979707Z", + "shell.execute_reply": "2024-03-22T19:37:16.979026Z" + }, + "papermill": { + "duration": 0.020246, + "end_time": "2024-03-22T19:37:16.981682", + "exception": false, + "start_time": "2024-03-22T19:37:16.961436", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:17.007348Z", + "iopub.status.busy": "2024-03-22T19:37:17.007080Z", + "iopub.status.idle": "2024-03-22T19:37:17.014104Z", + "shell.execute_reply": "2024-03-22T19:37:17.013253Z" + }, + "papermill": { + "duration": 0.021998, + "end_time": "2024-03-22T19:37:17.015955", + "exception": false, + "start_time": "2024-03-22T19:37:16.993957", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "11889160" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:17.041708Z", + "iopub.status.busy": "2024-03-22T19:37:17.041437Z", + "iopub.status.idle": "2024-03-22T19:37:17.120909Z", + "shell.execute_reply": "2024-03-22T19:37:17.120024Z" + }, + "papermill": { + "duration": 0.094727, + "end_time": "2024-03-22T19:37:17.122862", + "exception": false, + "start_time": "2024-03-22T19:37:17.028135", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 40] --\n", + "├─Adapter: 1-1 [2, 1179, 40] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 41,984\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 40] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 16, 256] 1\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 1,048,832\n", + "│ │ │ └─Softsign: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 256] --\n", + "│ │ │ └─Linear: 4-55 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-56 [2, 256] --\n", + "│ │ └─FeedForward: 3-28 [2, 256] --\n", + "│ │ │ └─Linear: 4-57 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-58 [2, 256] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 11,889,160\n", + "Trainable params: 11,889,160\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 44.13\n", + "========================================================================================================================\n", + "Input size (MB): 0.47\n", + "Forward/backward pass size (MB): 375.40\n", + "Params size (MB): 47.56\n", + "Estimated Total Size (MB): 423.43\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:37:17.152093Z", + "iopub.status.busy": "2024-03-22T19:37:17.151777Z", + "iopub.status.idle": "2024-03-22T20:44:11.433524Z", + "shell.execute_reply": "2024-03-22T20:44:11.432528Z" + }, + "papermill": { + "duration": 4014.314032, + "end_time": "2024-03-22T20:44:11.450901", + "exception": false, + "start_time": "2024-03-22T19:37:17.136869", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.017055697603565123, 'avg_role_model_std_loss': 0.7837836365105371, 'avg_role_model_mean_pred_loss': 0.000854024517475194, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.017055697603565123, 'n_size': 900, 'n_batch': 225, 'duration': 209.5561990737915, 'duration_batch': 0.9313608847724066, 'duration_size': 0.23284022119310166, 'avg_pred_std': 0.11550631119145287}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.02723778669618898, 'avg_role_model_std_loss': 0.35535829481002906, 'avg_role_model_mean_pred_loss': 0.0011922009129072346, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.02723778669618898, 'n_size': 450, 'n_batch': 113, 'duration': 91.00356817245483, 'duration_batch': 0.8053413112606622, 'duration_size': 0.20223015149434406, 'avg_pred_std': 0.12002328037391458}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007602974076354359, 'avg_role_model_std_loss': 0.6694789405275434, 'avg_role_model_mean_pred_loss': 0.00012108089975418718, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007602974076354359, 'n_size': 900, 'n_batch': 225, 'duration': 209.41460299491882, 'duration_batch': 0.9307315688663059, 'duration_size': 0.23268289221657648, 'avg_pred_std': 0.0941959698839734}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004413039641092635, 'avg_role_model_std_loss': 0.45001505491487465, 'avg_role_model_mean_pred_loss': 4.256673724010495e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004413039641092635, 'n_size': 450, 'n_batch': 113, 'duration': 91.49867558479309, 'duration_batch': 0.8097227927857795, 'duration_size': 0.20333039018842908, 'avg_pred_std': 0.06237377326902563}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005956742855616742, 'avg_role_model_std_loss': 0.6394465912903183, 'avg_role_model_mean_pred_loss': 6.889980071151411e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005956742855616742, 'n_size': 900, 'n_batch': 225, 'duration': 209.05843949317932, 'duration_batch': 0.9291486199696859, 'duration_size': 0.23228715499242147, 'avg_pred_std': 0.09135436112475064}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004399605713939915, 'avg_role_model_std_loss': 1.215607570140407, 'avg_role_model_mean_pred_loss': 2.922537710473547e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004399605713939915, 'n_size': 450, 'n_batch': 113, 'duration': 88.99790143966675, 'duration_batch': 0.7875920481386438, 'duration_size': 0.19777311431037056, 'avg_pred_std': 0.04754088749589844}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005293925739824772, 'avg_role_model_std_loss': 0.6930031799355281, 'avg_role_model_mean_pred_loss': 5.091876555952298e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005293925739824772, 'n_size': 900, 'n_batch': 225, 'duration': 209.8546106815338, 'duration_batch': 0.9326871585845947, 'duration_size': 0.23317178964614868, 'avg_pred_std': 0.09368091402575374}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005122498869895935, 'avg_role_model_std_loss': 0.6850604875349726, 'avg_role_model_mean_pred_loss': 0.0001063596828682662, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005122498869895935, 'n_size': 450, 'n_batch': 113, 'duration': 90.19163846969604, 'duration_batch': 0.7981560926521774, 'duration_size': 0.2004258632659912, 'avg_pred_std': 0.06673125488749515}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004406307417407839, 'avg_role_model_std_loss': 0.5391749190130491, 'avg_role_model_mean_pred_loss': 3.865118440087487e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004406307417407839, 'n_size': 900, 'n_batch': 225, 'duration': 208.6917781829834, 'duration_batch': 0.9275190141465929, 'duration_size': 0.23187975353664822, 'avg_pred_std': 0.0995840290437142}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005340991177492671, 'avg_role_model_std_loss': 0.6295711460246051, 'avg_role_model_mean_pred_loss': 7.291253590576869e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005340991177492671, 'n_size': 450, 'n_batch': 113, 'duration': 91.7313449382782, 'duration_batch': 0.8117818136130814, 'duration_size': 0.20384743319617377, 'avg_pred_std': 0.06550657832418132}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00469050889802424, 'avg_role_model_std_loss': 0.9336842445089064, 'avg_role_model_mean_pred_loss': 3.8414434089242215e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00469050889802424, 'n_size': 900, 'n_batch': 225, 'duration': 208.08384490013123, 'duration_batch': 0.9248170884450276, 'duration_size': 0.2312042721112569, 'avg_pred_std': 0.09579447591263388}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004810444195496125, 'avg_role_model_std_loss': 0.3487526955594519, 'avg_role_model_mean_pred_loss': 8.624510295827805e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004810444195496125, 'n_size': 450, 'n_batch': 113, 'duration': 90.39807367324829, 'duration_batch': 0.7999829528606044, 'duration_size': 0.20088460816277398, 'avg_pred_std': 0.0735004077937487}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005119060436975107, 'avg_role_model_std_loss': 0.6433215488478984, 'avg_role_model_mean_pred_loss': 6.873547569881813e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005119060436975107, 'n_size': 900, 'n_batch': 225, 'duration': 207.70152282714844, 'duration_batch': 0.9231178792317708, 'duration_size': 0.2307794698079427, 'avg_pred_std': 0.09771538318652245}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004957494798161659, 'avg_role_model_std_loss': 1.346057123426031, 'avg_role_model_mean_pred_loss': 5.000207313663115e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004957494798161659, 'n_size': 450, 'n_batch': 113, 'duration': 90.1600124835968, 'duration_batch': 0.7978762166689982, 'duration_size': 0.20035558329688177, 'avg_pred_std': 0.06382843967428249}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00532732381252572, 'avg_role_model_std_loss': 0.4333305762650606, 'avg_role_model_mean_pred_loss': 5.976084272669491e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00532732381252572, 'n_size': 900, 'n_batch': 225, 'duration': 207.5520989894867, 'duration_batch': 0.9224537732866075, 'duration_size': 0.23061344332165187, 'avg_pred_std': 0.09995691900038058}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0050655570465864405, 'avg_role_model_std_loss': 1.4249071690357547, 'avg_role_model_mean_pred_loss': 4.881459755913574e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0050655570465864405, 'n_size': 450, 'n_batch': 113, 'duration': 90.09133553504944, 'duration_batch': 0.7972684560623844, 'duration_size': 0.2002029678556654, 'avg_pred_std': 0.05493936902612646}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004071665801651155, 'avg_role_model_std_loss': 0.3260331416654134, 'avg_role_model_mean_pred_loss': 2.1085142560956504e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004071665801651155, 'n_size': 900, 'n_batch': 225, 'duration': 209.66889786720276, 'duration_batch': 0.931861768298679, 'duration_size': 0.23296544207466974, 'avg_pred_std': 0.10195785622629855}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005279655439727422, 'avg_role_model_std_loss': 0.9929788724251226, 'avg_role_model_mean_pred_loss': 6.48734679797379e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005279655439727422, 'n_size': 450, 'n_batch': 113, 'duration': 91.38696694374084, 'duration_batch': 0.8087342207410694, 'duration_size': 0.20308214876386854, 'avg_pred_std': 0.06704739362940984}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004601976428077453, 'avg_role_model_std_loss': 0.3923278355509547, 'avg_role_model_mean_pred_loss': 4.1437573981469074e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004601976428077453, 'n_size': 900, 'n_batch': 225, 'duration': 202.2607822418213, 'duration_batch': 0.8989368099636502, 'duration_size': 0.22473420249091255, 'avg_pred_std': 0.1009762748144567}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003612730009287285, 'avg_role_model_std_loss': 1.519655740890833, 'avg_role_model_mean_pred_loss': 3.1731855463335754e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003612730009287285, 'n_size': 450, 'n_batch': 113, 'duration': 86.41371130943298, 'duration_batch': 0.7647231089330353, 'duration_size': 0.19203046957651773, 'avg_pred_std': 0.05825823241436805}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003639938447223459, 'avg_role_model_std_loss': 0.2671018096537945, 'avg_role_model_mean_pred_loss': 2.6972246297165055e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003639938447223459, 'n_size': 900, 'n_batch': 225, 'duration': 204.09564805030823, 'duration_batch': 0.907091769112481, 'duration_size': 0.22677294227812025, 'avg_pred_std': 0.10696077811221281}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002935796806381808, 'avg_role_model_std_loss': 1.918275244656198, 'avg_role_model_mean_pred_loss': 1.3822624714984066e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002935796806381808, 'n_size': 450, 'n_batch': 113, 'duration': 87.83020257949829, 'duration_batch': 0.7772584299070645, 'duration_size': 0.19517822795444065, 'avg_pred_std': 0.04963276007591821}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0033298865797243907, 'avg_role_model_std_loss': 0.48618020502341713, 'avg_role_model_mean_pred_loss': 2.140108548165483e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033298865797243907, 'n_size': 900, 'n_batch': 225, 'duration': 201.81425046920776, 'duration_batch': 0.89695222430759, 'duration_size': 0.2242380560768975, 'avg_pred_std': 0.10155585827512874}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002946255624992773, 'avg_role_model_std_loss': 1.365417978676351, 'avg_role_model_mean_pred_loss': 1.4400359725914503e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002946255624992773, 'n_size': 450, 'n_batch': 113, 'duration': 87.90818929672241, 'duration_batch': 0.777948577847101, 'duration_size': 0.19535153177049425, 'avg_pred_std': 0.05489195802813577}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003180694187467452, 'avg_role_model_std_loss': 0.5309610087041662, 'avg_role_model_mean_pred_loss': 1.2343771629586152e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003180694187467452, 'n_size': 900, 'n_batch': 225, 'duration': 202.47150707244873, 'duration_batch': 0.8998733647664388, 'duration_size': 0.2249683411916097, 'avg_pred_std': 0.09903836669710775}\n", + "Time out: 3780.621705532074/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.003139846928090611, 'avg_g_mag_loss': 0.10179269966281267, 'avg_g_cos_loss': 0.03099184789007441, 'pred_duration': 3.9727766513824463, 'grad_duration': 12.556280612945557, 'total_duration': 16.529057264328003, 'pred_std': 0.09302742779254913, 'std_loss': 0.02486800216138363, 'mean_pred_loss': 3.833626033156179e-05, 'pred_rmse': 0.05603433772921562, 'pred_mae': 0.04172290489077568, 'pred_mape': 0.13467051088809967, 'grad_rmse': 0.15034998953342438, 'grad_mae': 0.04181426018476486, 'grad_mape': 1.1387050151824951}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.003139846928090611, 'avg_g_mag_loss': 0.10179269966281267, 'avg_g_cos_loss': 0.03099184789007441, 'avg_pred_duration': 3.9727766513824463, 'avg_grad_duration': 12.556280612945557, 'avg_total_duration': 16.529057264328003, 'avg_pred_std': 0.09302742779254913, 'avg_std_loss': 0.02486800216138363, 'avg_mean_pred_loss': 3.833626033156179e-05}, 'min_metrics': {'avg_loss': 0.003139846928090611, 'avg_g_mag_loss': 0.10179269966281267, 'avg_g_cos_loss': 0.03099184789007441, 'pred_duration': 3.9727766513824463, 'grad_duration': 12.556280612945557, 'total_duration': 16.529057264328003, 'pred_std': 0.09302742779254913, 'std_loss': 0.02486800216138363, 'mean_pred_loss': 3.833626033156179e-05, 'pred_rmse': 0.05603433772921562, 'pred_mae': 0.04172290489077568, 'pred_mape': 0.13467051088809967, 'grad_rmse': 0.15034998953342438, 'grad_mae': 0.04181426018476486, 'grad_mape': 1.1387050151824951}, 'model_metrics': {'lct_gan': {'avg_loss': 0.003139846928090611, 'avg_g_mag_loss': 0.10179269966281267, 'avg_g_cos_loss': 0.03099184789007441, 'pred_duration': 3.9727766513824463, 'grad_duration': 12.556280612945557, 'total_duration': 16.529057264328003, 'pred_std': 0.09302742779254913, 'std_loss': 0.02486800216138363, 'mean_pred_loss': 3.833626033156179e-05, 'pred_rmse': 0.05603433772921562, 'pred_mae': 0.04172290489077568, 'pred_mape': 0.13467051088809967, 'grad_rmse': 0.15034998953342438, 'grad_mae': 0.04181426018476486, 'grad_mape': 1.1387050151824951}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:44:11.484448Z", + "iopub.status.busy": "2024-03-22T20:44:11.484132Z", + "iopub.status.idle": "2024-03-22T20:44:11.488584Z", + "shell.execute_reply": "2024-03-22T20:44:11.487707Z" + }, + "papermill": { + "duration": 0.023713, + "end_time": "2024-03-22T20:44:11.490684", + "exception": false, + "start_time": "2024-03-22T20:44:11.466971", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:44:11.522326Z", + "iopub.status.busy": "2024-03-22T20:44:11.522055Z", + "iopub.status.idle": "2024-03-22T20:44:11.619261Z", + "shell.execute_reply": "2024-03-22T20:44:11.618274Z" + }, + "papermill": { + "duration": 0.115695, + "end_time": "2024-03-22T20:44:11.621500", + "exception": false, + "start_time": "2024-03-22T20:44:11.505805", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:44:11.655717Z", + "iopub.status.busy": "2024-03-22T20:44:11.655043Z", + "iopub.status.idle": "2024-03-22T20:44:11.931221Z", + "shell.execute_reply": "2024-03-22T20:44:11.930263Z" + }, + "papermill": { + "duration": 0.295167, + "end_time": "2024-03-22T20:44:11.933247", + "exception": false, + "start_time": "2024-03-22T20:44:11.638080", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8m0lEQVR4nO3deVhTV/4/8HcSSMIaNkkAgVArYhGxLqRYq21lSpVqcZzRUkasY9XOqK3SzlhmqrTT769ot3FqnVpn0ZlprctMV7VailpbiaKAdUGpWhYtm4AkEPbk/P64JBJIIMGEsHxez3Mf4d6Tm5Nb8u655557Lo8xxkAIIYME39EVIIQQa1BoEUIGFQotQsigQqFFCBlUKLQIIYMKhRYhZFCh0CKEDCpOjq5Af9HpdCgrK4OHhwd4PJ6jq0MI6YIxhvr6egQGBoLPN9+eGjahVVZWhuDgYEdXgxDSi+vXr2PkyJFmtw+b0PLw8ADAHRBPT08H14YQ0pVarUZwcLDhu2rOsAkt/Smhp6cnhRYhA1hv3TfUEU8IGVQotAghgwqFFiFkUBk2fVrkzmi1WrS1tTm6GmQQc3Z2hkAguOP9UGiRHjHGUFFRgbq6OkdXhQwBXl5ekMlkdzRWkkKL9EgfWP7+/nB1daWBuaRPGGNobGxEVVUVACAgIKDP+6LQ6qqpDqi8yP0sv9+hVXE0rVZrCCxfX19HV4cMci4uLgCAqqoq+Pv79/lUkTriu7qeA+ycDRxa5+iaOJy+D8vV1dXBNSFDhf5v6U76Rym0uvKQcv/WVzq2HgMInRISW7HF3xKFVlfuMu5fzU1A2+7YuhBCuqHQ6srND+DxATAuuAghAwqFVld8AeDmz/3cUOHYupBBi8fj4dNPP3V0NWzq5ZdfxoQJExxdDQotk6hfiwwBO3fuhJeXl83298ILLyArK8tm++srCi1T9P1a1NIiw0Bra6tF5dzd3QfE0BcKLVOopWUSYwyNre0OWax9EPqhQ4cwbdo0eHl5wdfXF4899hiuXbsGAJg6dSrWrTMe0nLz5k04Ozvj+PHjAIDy8nIkJCTAxcUFYWFh2LVrF+RyOTZv3tynY3f+/Hk8/PDDcHFxga+vL5YvX46GhgbD9mPHjiEmJgZubm7w8vLC/fffj5KSEgDA999/j4ceeggeHh7w9PTEpEmTcObMmR7f79ixY1iyZAlUKhV4PB54PB5efvllAIBcLserr76KlJQUeHp6Yvny5QCAdevWITw8HK6urrjrrruwfv16o6EJXU8Pn3rqKSQmJuLNN99EQEAAfH19sXLlSrvf7kWDS02hlpZJTW1a3LPhsEPeu+BP8XAVWv7nqtFokJqaivHjx6OhoQEbNmzAvHnzcPbsWSQnJ+P111/Hxo0bDZfg9+zZg8DAQDzwwAMAgJSUFFRXV+PYsWNwdnZGamqqYTS3tTQaDeLj4xEbG4vTp0+jqqoKTz/9NFatWoWdO3eivb0diYmJWLZsGT766CO0trYiJyfHULfk5GTce++9eO+99yAQCHD27Fk4Ozv3+J5Tp07F5s2bsWHDBhQWFgLgWkp6b775JjZs2ID09HTDOg8PD+zcuROBgYE4f/48li1bBg8PD/z+9783+z5Hjx5FQEAAjh49iqtXr2LhwoWYMGECli1b1qdjZQkKLVOopTXozZ8/3+j3f/7znxgxYgQKCgqwYMECrFmzBt99950hpHbt2oWkpCTweDxcvnwZX3/9NU6fPo3JkycDAP7+979j9OjRfarLrl270NzcjH//+99wc3MDALz77ruYM2cONm3aBGdnZ6hUKjz22GMYNWoUAGDs2LGG15eWluJ3v/sdIiIiAMCiegiFQkgkEvB4PMhksm7bH374YTz//PNG61566SXDz3K5HC+88AJ2797dY2h5e3vj3XffhUAgQEREBBISEpCVlUWh1e+opWWSi7MABX+Kd9h7W+PKlSvYsGEDTp06herqauh0OgBcAIwbNw6PPPIIPvzwQzzwwAMoKiqCUqnE+++/DwAoLCyEk5MTJk6caNjf3XffDW9v7z7V/dKlS4iOjjYEFgDcf//90Ol0KCwsxPTp0/HUU08hPj4eP/vZzxAXF4cFCxYY7s9LTU3F008/jf/85z+Ii4vDL3/5S0O49ZU+jDvbs2cP3nnnHVy7dg0NDQ1ob2/vdZbfyMhIo9txAgICcP78+TuqW2+oT8sUj47QopaWER6PB1ehk0MWa0dSz5kzB7W1tfjb3/6GU6dO4dSpUwBudzonJyfjv//9L9ra2rBr1y5ERUUhKirK5sfMUjt27IBSqcTUqVOxZ88ehIeH4+TJkwC4vqSLFy8iISEBR44cwT333INPPvnkjt6vc4ACgFKpRHJyMmbPno39+/cjPz8ff/zjH3vtpO96msrj8Qz/g7AXCi1T3DtODxsqASs7gInj1dTUoLCwEC+99BJmzpyJsWPH4tatW0ZlHn/8cTQ3N+PQoUPYtWsXkpOTDdvGjBmD9vZ25OfnG9ZdvXq12z4sNXbsWHz//ffQaDSGdSdOnACfz8eYMWMM6+69916kpaUhOzsb48aNw65duwzbwsPDsXbtWnz11Vf4+c9/jh07dvT6vkKhEFqt1qI6ZmdnIzQ0FH/84x8xefJkjB492nAhYKCh0DJFH1q6NqCx1rF1IVbz9vaGr68vtm/fjqtXr+LIkSNITU01KuPm5obExESsX78ely5dQlJSkmFbREQE4uLisHz5cuTk5CA/Px/Lly+Hi4tLn+6dS05OhlgsxuLFi3HhwgUcPXoUq1evxqJFiyCVSlFUVIS0tDQolUqUlJTgq6++wpUrVzB27Fg0NTVh1apVOHbsGEpKSnDixAmcPn3aqM/LHLlcjoaGBmRlZaG6uhqNjY1my44ePRqlpaXYvXs3rl27hnfeeeeOW3P2QqFlipMQcPHhfqZ+rUGHz+dj9+7dyM3Nxbhx47B27Vq88cYb3colJyfj+++/xwMPPICQkBCjbf/+978hlUoxffp0zJs3z3AlTSwWW10fV1dXHD58GLW1tZgyZQp+8YtfYObMmXj33XcN2y9fvoz58+cjPDwcy5cvx8qVK7FixQoIBALU1NQgJSUF4eHhWLBgAWbNmoVXXnml1/edOnUqnnnmGSxcuBAjRozA66+/brbs3LlzsXbtWqxatQoTJkxAdnY21q9fb/Vn7Q88Zu0AmEFKrVZDIpFApVJZ9gixv8YCVQXAok+AUQ/bv4IDUHNzM4qKihAWFtanL+tQcuPGDQQHB+Prr7/GzJkzHV2dQaunvylLv6N9amlt3boVcrkcYrEYCoUCOTk5PZbft28fIiIiIBaLERUVhYMHDxq2tbW1Yd26dYiKioKbmxsCAwORkpKCsrIyo33I5XLDIDn9snHjxr5U3zLuHfcfUmf8sHTkyBF8/vnnKCoqQnZ2Np544gnI5XJMnz7d0VUb9qwOrT179iA1NRXp6enIy8tDdHQ04uPjzQ68y87ORlJSEpYuXYr8/HwkJiYiMTERFy5cAAA0NjYiLy8P69evR15eHj7++GMUFhZi7ty53fb1pz/9CeXl5YZl9erV1lbfcjTsYVhra2vDH/7wB0RGRmLevHkYMWKEYaDphx9+CHd3d5NLZGRkv9Vx1qxZZuvx2muv9Vs9+pvVp4cKhQJTpkwxnI/rdDoEBwdj9erVePHFF7uVX7hwITQaDfbv329Yd99992HChAnYtm2byfc4ffo0YmJiUFJSYuhrkMvlWLNmDdasWWNNdQ2sPj3M3ACc+Aug+A0wy44tugGMTg9Nq6+vR2Wl6Ra4s7MzQkND+6UeP/30E5qamkxu8/HxgY+PT7/Uwxq2OD20anBpa2srcnNzkZaWZljH5/MRFxcHpVJp8jVKpbLblZv4+Pgep+3Q3y/V9Q71jRs34tVXX0VISAiefPJJrF27Fk5Opj9CS0sLWlpaDL+r1epePl0X1NIiZnh4eMDDw8PR1UBQUJCjq+AQVoVWdXU1tFotpFKp0XqpVIrLly+bfE1FRYXJ8hUVpsOgubkZ69atQ1JSklHaPvvss5g4cSJ8fHyQnZ2NtLQ0lJeX4+233za5n4yMDIuusJhFt/IQMiANqNt42trasGDBAjDG8N577xlt69xaGz9+PIRCIVasWIGMjAyIRKJu+0pLSzN6jVqtRnBwsOWVoZYWIQOSVaHl5+cHgUDQ7Xy+srLS5E2ZACCTySwqrw+skpISHDlypNd+J4VCgfb2dhQXFxuNKtYTiUQmw8xidCsPIQOSVVcPhUIhJk2aZDR7oU6nQ1ZWFmJjY02+JjY2tttsh5mZmUbl9YF15coVfP311xZNNHb27Fnw+Xz4+/tb8xEspx8V36YBWurt8x6EEKtZfXqYmpqKxYsXY/LkyYiJicHmzZuh0WiwZMkSANw8REFBQcjIyAAAPPfcc5gxYwbeeustJCQkYPfu3Thz5gy2b98OgAusX/ziF8jLy8P+/fuh1WoN/V0+Pj4QCoVQKpU4deqUYSI0pVKJtWvX4le/+lWf77zvlcgdELoDrQ1ca0vk+I5XQggA1gdbtmxhISEhTCgUspiYGHby5EnDthkzZrDFixcbld+7dy8LDw9nQqGQRUZGsgMHDhi2FRUVMQAml6NHjzLGGMvNzWUKhYJJJBImFovZ2LFj2Wuvvcaam5strrNKpWIAmEqlsvyD/uVextI9GSv61vLXDCFNTU2soKCANTU1Oboqgw4A9sknnzi6GgNOT39Tln5H+xRag1GfQuufs7jQOrfPfhUbwCi0+m4ghNaOHTuYRCKx6T6PHj3KALBbt2716fW2CC26YbonnaeoIYQMCBRaPTFcQaRhDwC4ucVaNY5Z6MEWNnuwRUtLC1544QUEBQXBzc0NCoUCx44dM7y2pKQEc+bMgbe3N9zc3BAZGYmDBw+iuLgYDz30EABu+h8ej4ennnqqT8fjTgyocVoDDrW0jLU1Aq8FOua9/1AGCN16L9eBHmxh/sEWq1atQkFBAXbv3o3AwEB88sknePTRR3H+/HmMHj0aK1euRGtrK44fPw43NzcUFBTA3d0dwcHB+N///of58+ejsLAQnp6ecHFx6dMxuRMUWj2hltagRQ+2MP1gi9LSUuzYsQOlpaUIDOT+B/TCCy/g0KFD2LFjB1577TWUlpZi/vz5humn77rrLsPr9fcz+vv72/RBsNag0OqJoaXVt//DDjnOrlyLx1HvbQV6sIVp58+fh1arRXh4uNH6lpYWw/jIZ599Fr/5zW/w1VdfIS4uDvPnz8f48eP79H72QH1aPfGgW3mM8HjcKZojFnqwhU0ebNHQ0ACBQIDc3FycPXvWsFy6dAl/+ctfAABPP/00fvzxRyxatAjnz5/H5MmTsWXLFpt91jvWp+uWg1CfhjxoarghD+mejLVZPiZsqBisQx6qq6sZAHb8+HHDum+//dZoGEJDQwNzc3Njn3/+ObvnnnvYxo0bDWUvXbrEALAzZ84Y1l25coUBYH/+858tqkPn99q+fTvz9vZmDQ0Nhu0HDhxgfD6fVVRUmHz9fffdx1avXm1y2xNPPMHmzJnTax0+/PBD5u7ubrSusLCw27HpzYsvvsiioqIYY4ydOHGCAWDV1dUWv74zGvJgby7egEDI/Uyd8YMGPdiCY+rBFuHh4UhOTkZKSgo+/vhjFBUVIScnBxkZGThw4AAAYM2aNTh8+DCKioqQl5eHo0ePGt4vNDQUPB4P+/fvx82bN42ugPabPsXlINSnlhZjjL0dybW0SnPsU7EBbLC2tBhjLDMzk40dO5aJRCI2fvx4duzYsW4DPg8ePMgAsOnTp3d7fVlZGZs1axYTiUQsNDSU7dq1i/n7+7Nt27ZZ9P5d3+vcuXPsoYceYmKxmPn4+LBly5ax+vp6xhhjFRUVLDExkQUEBDChUMhCQ0PZhg0bmFarZS0tLeyJJ55gwcHBTCgUssDAQLZq1SqL/5s888wzzNfXlwFg6enpjDHGWltb2YYNG5hcLmfOzs4sICCAzZs3j507d44xxtiqVavYqFGjmEgkYiNGjGCLFi0yaln96U9/YjKZjPF4vG53v/TGFi0terBFb/42E/jpDLDwA2DsHPtVcACimUtvowdb2Ea/z1w6LNGwh2HpyJEjaGhoQFRUFMrLy/H73/+eHmwxQFCfVm9ogOmwRA+2GLiopdUbamkNS/Hx8YiPjze5be7cuVAoFCa39TZS3Zb+/ve/9/hgi6GKQqs31NIiXdCDLRyLTg97Qy0tDJNrNaQf2OJviUKrN8O4paU/1WlsbHRwTchQof9bupPTaDo97I2+paW5Cei0AF/g2Pr0I4FAAC8vL8PsBq6urn0aXEkIYwyNjY2oqqqCl5cXBIK+f48otHrjNgLg8QGm44LLw/RTh4Yq/QwBfZ2WhZDOvLy8zD65y1IUWr3hC7jgaqjk+rWGWWjxeDwEBATA398fbW1tjq4OGcScnZ3vqIWlR6FlCXcpF1rDsF9LTyAQ2OQPjpA7RR3xljBMUTN8Q4uQgYJCyxL6K4j0tGlCHI5CyxKGYQ/Dd6wWIQMFhZYlaIApIQMGhZYlhvEAU0IGGgotSxhaWhRahDgahZYlOvdp0X14hDgUhZYl9KGlbQWabjm2LoQMcxRalnAWA2Iv7mfq1yLEoSi0LEVXEAkZECi0LEVXEAkZECi0LEUtLUIGBAotS1FLi5ABoU+htXXrVsjlcojFYigUCuTk5PRYft++fYiIiIBYLEZUVBQOHjxo2NbW1oZ169YhKioKbm5uCAwMREpKCsrKyoz2UVtbi+TkZHh6esLLywtLly7t36fbUkuLkAHB6tDas2cPUlNTkZ6ejry8PERHRyM+Pt7sJHHZ2dlISkrC0qVLkZ+fj8TERCQmJuLChQsAuOlX8/LysH79euTl5eHjjz9GYWEh5s6da7Sf5ORkXLx4EZmZmdi/fz+OHz+O5cuX9+Ej9xG1tAgZGKx6pjVjLCYmhq1cudLwu1arZYGBgSwjI8Nk+QULFrCEhASjdQqFgq1YscLse+Tk5DAArKSkhDHGWEFBAQPATp8+bSjz5ZdfMh6Px3766SeT+2hubmYqlcqwXL9+3aJHbptV9C1j6Z6M/eXevr2eENIjlUpl0XfUqpZWa2srcnNzERcXZ1jH5/MRFxcHpVJp8jVKpdKoPMA9U85ceQBQqVTg8Xjw8vIy7MPLywuTJ082lImLiwOfz8epU6dM7iMjIwMSicSwBAcHW/oxTXPXz6lF0w4T4khWhVZ1dTW0Wi2kUqnReqlUiooK0309FRUVVpVvbm7GunXrkJSUBE9PT8M+/P39jco5OTnBx8fH7H7S0tKgUqkMy/Xr1y36jGZ5dHyG1nqgVXNn+yKE9NmAmm65ra0NCxYsAGMM77333h3tSyQSQSQS2ahmAEQegLMb0KbhOuN9R9lu34QQi1nV0vLz84NAIEBlpXFndGVlpdknbMhkMovK6wOrpKQEmZmZhlaWfh9dO/rb29tRW1t7x0/2sIp7R2uPOuMJcRirQksoFGLSpEnIysoyrNPpdMjKykJsbKzJ18TGxhqVB4DMzEyj8vrAunLlCr7++mv4+vp220ddXR1yc3MN644cOQKdTgeFQmHNR7gzNOyBEMeztod/9+7dTCQSsZ07d7KCggK2fPly5uXlxSoqKhhjjC1atIi9+OKLhvInTpxgTk5O7M0332SXLl1i6enpzNnZmZ0/f54xxlhrayubO3cuGzlyJDt79iwrLy83LC0tLYb9PProo+zee+9lp06dYt999x0bPXo0S0pKsrjell6Z6NGeFO4KovKvfd8HIcQkS7+jVocWY4xt2bKFhYSEMKFQyGJiYtjJkycN22bMmMEWL15sVH7v3r0sPDycCYVCFhkZyQ4cOGDYVlRUxACYXI4ePWooV1NTw5KSkpi7uzvz9PRkS5YsYfX19RbX2SahdfD3XGh9taHv+yCEmGTpd5TH2PCY1U6tVkMikUClUhn1l1nl27eBrFeA6CRg3jbbVpCQYc7S7yjde2gN6tMixOEotKxBt/IQ4nAUWtaglhYhDkehZQ39rTxNtUB7q2PrQsgwRaFlDVcfgO/M/UyniIQ4BIWWNXg86tcixMEotKylv3Ga+rUIcQgKLWsZpqihlhYhjkChZS0POj0kxJEotKzlTsMeCHEkCi1rUUuLEIei0LIWtbQIcSgKLWvRRICEOBSFlrU8Oj3gQqd1bF0IGYYotKzl5g+ABzAt0Fjj6NoQMuxQaFlL4AS4+XE/U78WIf2OQquL5jYtcopq8e2Vm+YL0QBTQhyGQquLrwoqseB9JV4/VGi+EN3KQ4jDUGh1MUXuDQC4WKZCQ0u76UKGlhaFFiH9jUKriwCJC0Z6u0DHgPzSW6YLGVpadHpISH+j0DJhitwHAHC62ExoUUuLEIeh0DJhcscp4pniWtMFqKVFiMNQaJmgb2nll9ahTavrXoBaWoQ4DIWWCXePcIeXqzOa2rQoKFN3L2C4aboKGB6PjSRkwKDQMoHP52FyKHeKeNrUKaK+pdXeDDSr+rFmhBAKLTMmGzrjTYSWsxgQS7ifaYApIf2KQsuMKYbO+Ftgpk4BaYoaQhyCQsuMcUESCJ34qNG0oqha070ATQZIiENQaJkhchJgQrAXgF76tailRUi/otDqgf4U0eQgU5oMkBCHoNDqgb4z3uQgUw9qaRHiCBRaPZgY4g0eDyiuaURVfbPxRpqehhCH6FNobd26FXK5HGKxGAqFAjk5OT2W37dvHyIiIiAWixEVFYWDBw8abf/444/xyCOPwNfXFzweD2fPnu22jwcffBA8Hs9oeeaZZ/pSfYtJXJwxRuoBgLuKaISmpyHEIawOrT179iA1NRXp6enIy8tDdHQ04uPjUVVVZbJ8dnY2kpKSsHTpUuTn5yMxMRGJiYm4cOGCoYxGo8G0adOwadOmHt972bJlKC8vNyyvv/66tdW3WkyYmfFa1NIixDGYlWJiYtjKlSsNv2u1WhYYGMgyMjJMll+wYAFLSEgwWqdQKNiKFSu6lS0qKmIAWH5+frdtM2bMYM8995y11TVQqVQMAFOpVFa97rOzP7HQdfvZY+98a7yhqY6xdE9uadH0uV6EEI6l31GrWlqtra3Izc1FXFycYR2fz0dcXByUSqXJ1yiVSqPyABAfH2+2fE8+/PBD+Pn5Ydy4cUhLS0NjY6PZsi0tLVCr1UZLX5idFFDkCTi5cD/TjdOE9BurQqu6uhparRZSqdRovVQqRUWF6S9uRUWFVeXNefLJJ/HBBx/g6NGjSEtLw3/+8x/86le/Mls+IyMDEonEsAQHB1v1fnoBEhcEeXGTAp4trbu9gcejKWoIcQAnR1fAUsuXLzf8HBUVhYCAAMycORPXrl3DqFGjupVPS0tDamqq4Xe1Wt3n4IoJ88En+T8hp7gW00b73d7gLgNuFVNLi5B+ZFVLy8/PDwKBAJWVxi2LyspKyGQyk6+RyWRWlbeUQqEAAFy9etXkdpFIBE9PT6Olr8xOCkgtLUL6nVWhJRQKMWnSJGRlZRnW6XQ6ZGVlITY21uRrYmNjjcoDQGZmptnyltIPiwgICLij/VjC7KSAdAWRkH5n9elhamoqFi9ejMmTJyMmJgabN2+GRqPBkiVLAAApKSkICgpCRkYGAOC5557DjBkz8NZbbyEhIQG7d+/GmTNnsH37dsM+a2trUVpairKyMgBAYSH3+C6ZTAaZTIZr165h165dmD17Nnx9fXHu3DmsXbsW06dPx/jx4+/4IPTm7hHukLg4Q9XUhoIyNaI77kmkm6YJcYC+XJrcsmULCwkJYUKhkMXExLCTJ08ats2YMYMtXrzYqPzevXtZeHg4EwqFLDIykh04cMBo+44dOxiAbkt6ejpjjLHS0lI2ffp05uPjw0QiEbv77rvZ7373O6uGL/R1yIPer3fksNB1+9nfjl+7vTLvA27Iw7/n9WmfhJDbLP2O8hgbHvMFq9VqSCQSqFSqPvVvbfvmGjZ+eRnxkVK8v2gyt/Lq18AH8wHpOOA3J2xcY0KGF0u/o3TvoYVMTgpI09MQ0u8otCxkclJA/UwPjdWAts1xlSNkGKHQspDISYAJI70AdLp52sUH4Hdcy2gwfe8lIcS2KLSsMCWMO0XM0Y/X4vMBN/1kgHSKSEh/oNCygslJAWmAKSH9ikLLCiYnBaSnTRPSryi0rNB5UsBcfb8WtbQI6VcUWlbS39Jj6NeilhYh/YpCy0pTwvT9WtTSIsQRKLSs1G1SQGppEdKvKLSs1G1SQGppEdKvKLT64PZDXGtvt7Q0VYBO18OrCCG2QKHVB/rxWlxo+QPgAbp2oMnEQ10JITZFodUH+seK5ZfWoQ0CwNWX20A3ThNidxRafaCfFLCpTYuCMvXtG6epM54Qu6PQ6gM+n4fJoZ37tagznpD+QqHVR0b9WtTSIqTfUGj1UUxYp0kBqaVFSL+h0OqjzpMC1vC4AKOWFiH2R6HVR50nBfxB48atpJYWIXZHoXUH9A9xza8VciuopUWI3VFo3QH9jA8nKgXcivpKYHg83IgQh6HQugMTQ7lJAfNuibkV7U1Ai9qxlSJkiKPQugP6SQGbIUKbkzu3kvq1CLErCq07pD9FVAm4f6lfixD7otC6Q/rO+DKthFtBLS1C7IpC6w7pb54ubuHmjqeWFiH2RaF1h/STAlYyL24FzfRAiF1RaNnAFLk3qvShRU+aJsSuKLRsYLLcp1NoUUuLEHui0LKBKXIfVIHrkGfUEU+IXVFo2cBof3c0Cf0AAFp1uYNrQ8jQ1qfQ2rp1K+RyOcRiMRQKBXJycnosv2/fPkREREAsFiMqKgoHDx402v7xxx/jkUcega+vL3g8Hs6ePdttH83NzVi5ciV8fX3h7u6O+fPno7JyYLRq+HweRgaHAQCcWtVAW5ODa0TI0GV1aO3ZswepqalIT09HXl4eoqOjER8fj6oq0x3Q2dnZSEpKwtKlS5Gfn4/ExEQkJibiwoULhjIajQbTpk3Dpk2bzL7v2rVr8cUXX2Dfvn345ptvUFZWhp///OfWVt9uIu8KRjNz5n5pGBhhSsiQxKwUExPDVq5cafhdq9WywMBAlpGRYbL8ggULWEJCgtE6hULBVqxY0a1sUVERA8Dy8/ON1tfV1TFnZ2e2b98+w7pLly4xAEypVFpUb5VKxQAwlUplUXlrnS6qYSXrRzGW7sl0JZbViRBym6XfUataWq2trcjNzUVcXJxhHZ/PR1xcHJRKpcnXKJVKo/IAEB8fb7a8Kbm5uWhrazPaT0REBEJCQszup6WlBWq12mixp6iREtzsmAywqqzEru9FyHBmVWhVV1dDq9VCKpUarZdKpaioMH2pv6Kiwqry5vYhFArh5eVl8X4yMjIgkUgMS3BwsMXv1xciJwHaxP4AgJ+uF9v1vQgZzobs1cO0tDSoVCrDcv36dbu/p9CLe8DFrUr7vxchw5WTNYX9/PwgEAi6XbWrrKyETCYz+RqZTGZVeXP7aG1tRV1dnVFrq6f9iEQiiEQii9/DFrz8g4FKoKWurF/fl5DhxKqWllAoxKRJk5CVlWVYp9PpkJWVhdjYWJOviY2NNSoPAJmZmWbLmzJp0iQ4Ozsb7aewsBClpaVW7cfeZCPlAADXlmpU1Tc7tjKEDFFWtbQAIDU1FYsXL8bkyZMRExODzZs3Q6PRYMmSJQCAlJQUBAUFISMjAwDw3HPPYcaMGXjrrbeQkJCA3bt348yZM9i+fbthn7W1tSgtLUVZGddCKSwsBMC1sGQyGSQSCZYuXYrU1FT4+PjA09MTq1evRmxsLO677747Pgi24uoTBADw59Uht/gWZkUFOLhGhAxBfbk0uWXLFhYSEsKEQiGLiYlhJ0+eNGybMWMGW7x4sVH5vXv3svDwcCYUCllkZCQ7cOCA0fYdO3YwAN2W9PR0Q5mmpib229/+lnl7ezNXV1c2b948Vl5ebnGd7T3kgTHGWNn3jKV7sqoNI9krn1+03/sQMgRZ+h3lMTY8nsSgVqshkUigUqng6elpnzdpqALeHA0d4+Hnfp/h09Uz7PM+hAxBln5Hh+zVQ4dw9QPjCcDnMVSVX4empd3RNSJkyKHQsiU+Hzx3bqyWD7uF/NI6x9aHkCGIQsvW3LmBtP68OpwurnVwZQgZeii0bM2DGzfmz6vDmRIKLUJsjULL1vQtLdQhr6QObVqdgytEyNBCoWVrHS2tkc4qNLVpUVBGT5wmxJYotGyto6U12kUDANSvRYiNUWjZWkdLK9CJa2GdKb7lyNoQMuRQaNmaOxdaXjquhXWmpBbDZPwuIf2CQsvWPLjTQ2HTTQideKhuaEVRtcbBlSJk6KDQsjU3bnApT9eGaYE8AHSKSIgtUWjZmpMQcPEBAEyTaQFQZzwhtkShZQ8dnfETvVsAAKeKaqHVUb8WIbZAoWUPHcMewt0aIRTwUVrbiEX/OIUqNU0MSMidotCyh46WlmvLTby5IBouzgJkX6vB7He+xfEfbjq4coQMbhRa9tDR0kJDJeZGB+KL1dMQIfNAdUMrUv6Zg9cPXUY73d5DSJ9QaNlDR0sL9dzjze72d8enK+9HsiIEAPDXY9ewcPtJ/FTX5KgaEjJoUWjZg6GlVWVYJXYW4P/Ni8LWJyfCQ+SE3JJbmP2Xb5FZUGlmJ4QQUyi07EHf0mro/iDZhPEBOPDsAxg/UgJVUxuW/fsMXvniIlratf1cSUIGJwote9C3tOpNt6JCfF3x32em4ulpYQCAHSeK8Yv3lCipoZHzhPSGQsse9C2tNg3QUm+yiNCJj5ceuwd/T5kML1dnnP9JhYR3vsPn39ODXgnpCYWWPQjdAKEH97OZ1pZe3D1SHHz2AUyRe6OhpR3PfpSPtI/PoamVThcJMYVCy1489J3x3fu1ugr0csFHy+7D6ofvBo8HfJRzHYlbT+BKpelWGiHDGYWWvbgbD3vojZOAj+cfGYP//FoBP3cRCivrMffdE9h75jpNbUNIJxRa9uJxe4CpNaaN9sOXzz2AaXf7oalNi9//9xzW7jmLBnqGIiEAKLTsx8qWVmcjPET4969j8Lv4MRDwefj0bBnmbPkOF8tUNq4kIYMPhZa99LGlpcfn87Dyobuxe/l9CJCIUVStwby/ZuMf3xVB3dxmw4oSMrhQaNnLHbS0Opsi98HBZx9A3Fh/tLbr8Or+Akz8UyaS/34S//iuCMU0KyoZZpwcXYEhy52bwbSvLa3OvN2E+FvKZPznZAl2Zhfjx5sanLhagxNXa/Dq/gLcNcINcWOleDjCH5NDveEkoP8XkaGLx4bJpSm1Wg2JRAKVSgVPT0/7v2HVJeCv9wFiL+DFEpvuuqhag6xLlThyuQo5RbVo7zTBoKfYCQ+O8cfMsf6YET4CXq5Cm743IfZi6XeUQsteGmuB17nbdPDHSsBZbJe3UTe34dsfqpF1qRJHC6twq/F2f5eAz8OkUG/MjOBCbNQId/B4PLvUg5A7RaHVRb+HFmPA/0kBbQvw3DnAO9Tub6nVMZy9fgtfX6rCkUtVKOwyODXExxUzx/pjZoQUMWE+EDrZ8DSyvZWbH5+QPrJraG3duhVvvPEGKioqEB0djS1btiAmJsZs+X379mH9+vUoLi7G6NGjsWnTJsyePduwnTGG9PR0/O1vf0NdXR3uv/9+vPfeexg9erShjFwuR0mJ8WlWRkYGXnzxRYvq3O+hBQB/jgJUpcDSTCDY/PGxl+u1jThyuQpZl6tw8loNWjtNPOgucsL0cD/MnzgSD47xh4BvogXGGNCi5i4m1Fdw/XPd/i3nblVqrQckwUBILBAaC4RMBUaMAQZJy661XYfqhhbcrO9YOv/c6XcBn4dfTwtD0pRg6/oOWzUdx6uSu0uioQpobwbG/QKQBNnvgw0idgutPXv2ICUlBdu2bYNCocDmzZuxb98+FBYWwt/fv1v57OxsTJ8+HRkZGXjsscewa9cubNq0CXl5eRg3bhwAYNOmTcjIyMC//vUvhIWFYf369Th//jwKCgogFnOnVXK5HEuXLsWyZcsM+/bw8ICbm5tND4hN/T0OuHEaWPgBMHZO/7ynGZqWdnx7pRpHLlfiyOWbqG1owiheGWS8WoS7avBQkBYTvFvg3lp9+4tVXwm038FEha6+XIjpg0wWDQj699qPpqUd1281GgdQ11BqaEFdo3XDSMKl7vjj7AjMGOnUcaw6gkh/3BoqO4V7FRfqprh4A/O2A+GP2ODTDm52Cy2FQoEpU6bg3XffBQDodDoEBwdj9erVJls9CxcuhEajwf79+w3r7rvvPkyYMAHbtm0DYwyBgYF4/vnn8cILLwAAVCoVpFIpdu7ciSeeeAIAF1pr1qzBmjVrrKmugUNCa3cycHk/MPtNIGZZ7+XtTdsOFH8LVvA52gu+gHOThfPViyTcuDN3KTeDhbsU8Ajo9LOMu+BQeQEoVQIl2VxYt3d5kIezGxA8hWuFhcYCQZMBoavNPl6tphUXy1S4WKbmlp9UKKrRwNRfOA86uKAVbmiCG68ZbmiGJ78FgS7tkLpo4S9sg6+wDd5OrfAStMCD3wo3XhMaVbVQV9+AD7sFP6gg5FlxY7uzq/ExrLkGVJ7ntt3/HPDwekDgbJuDMQhZ+h216n97ra2tyM3NRVpammEdn89HXFwclEqlydcolUqkpqYarYuPj8enn34KACgqKkJFRQXi4uIM2yUSCRQKBZRKpSG0AGDjxo149dVXERISgieffBJr166Fk5Ppj9DS0oKWlhbD72q12pqPahuGyQAdODtpeyvw4zHg0mfA5YNAUy14AJwBQOgOnSQY1TwfXKp3wcV6V1QxL1QxL2hdpYiJvgez7otG4Ahfy97L/SFg1EO337f8LBdgpUpuaVZxdfnxGFeG7wwETuhoiU0FghWAq0+vb8N0OlTU3MKVkhsovvETyisrUHOzEtrGOkh4Gkh4GkyEBg/yNJA4aeAjaIYHvwUe/Ga4smaIWROEOjMtyHYAltyn3umst4Z5oN1lBLylwRB6BXKB5C7tCHrZ7Z9FHl3eqwXI3ACc2gac+AtQehL4xT8ByUgLKjB8WRVa1dXV0Gq1kEqlRuulUikuX75s8jUVFRUmy1dUVBi269eZKwMAzz77LCZOnAgfHx9kZ2cjLS0N5eXlePvtt02+b0ZGBl555RVrPp7t2WiAqdXamoCrWcClz4HCL7l+KT1XXyAiAbjncUA+HXwnIfwB+AMYebMBe05fx8ncG6htaMXhE034f9kn8eAYfyTFhOChMSMs78dxEnL9eMExANYAOh1w89LtECtRAvVlXIvsxmkg+x3udf73cCHm6gM01YE11aFJXYPm+hqwpjo4targomtAANoRAGB65/fs6TqArmPpiscHhO4dixu3iDw6fnbv8rsbIPI0hFBpqwcyjtfgy0u1QAvg0eyE1aPuxuKpcoicBBYcIxEwaxMQej/w2Srg+ilg2zRg3vtAeLxlx3kYGjSDSzu31saPHw+hUIgVK1YgIyMDIpGoW/m0tDSj16jVagQHB/dLXQ3u8FYeq7Q0AFe+Ago+A65kchMQ6rnLgLGPcUEVMtVsv9KoEe74w+yxeP6RcHx1sRK7TpVC+WMNjlyuwpHLVZB6irBwcjAWxoQgyMvFuvrx+YA0kltilnGd/HUlXHiVZoOVKMGruQJUFXBLBx4A146lKy34aBJ4QCuSQODqDbGHL5zcvAGxhDtddfHi/hV7cvObiToFk7AjiJxd+nyxIATAe2FA9rVq/N/+SygoV+O1g5fxwclSpM2KwKPjZJYNMblnLiCLAv67BCjLB3YtAKY+C8zcMKxPF82xKrT8/PwgEAhQWWn8JaysrIRMJjP5GplM1mN5/b+VlZUICAgwKjNhwgSzdVEoFGhvb0dxcTHGjBnTbbtIJDIZZv3K3i2tpjrgh0NAwefAtSzjPiRJMDB2LveFGBnDhYaFRE4CzIkOxJzoQPzY0fral3sDleoWvHPkKrYcvYoHw0cgKSYED0f4W9T60uoYKtXNuF7biOu3mnDjViOu1zbh+i05btT6o0L9KHyYCpP5hZjE/wFCtEEFN6iZGxr57vDwHoERI/wRKAtEWHAgwkYGQewmgfsAuDo5dZQfvlg9Df/Lu4E3DheitLYRv/kwDzFhPlifcA+iRkp634lPGPDrw7dPF7Pf4U4Xf7nD7OkiYwxXqxqQU1yLCz+pMTbAA/MnjoSbaNC0RfqkTx3xMTEx2LJlCwCuIz4kJASrVq0y2xHf2NiIL774wrBu6tSpGD9+vFFH/AsvvIDnn38eANcq8vf3N+qI7+rDDz9ESkoKqqur4e3t3Wu9HdIRX/498P50wM0fWHuROxXh8W7/2xeaGqDwABdUPx4DdJ2uevncdTuoAifadLhBS7sWmQVc6yv7Wo1hvdRThAWTg7FgcjDEzgJcv9WI67WNuGEUTI0oq2tCm7bnPzWREx8jvV0Q7OOKUSPcMS7IE5GBEtzl5zZobk3StLTj/W+uYfu3P6K5jTsf/fnEIPw+PgIyiYUDjAs+504XW1Tc1cXEbcCYR9Gm1eFimRqni2qRU1yLM8W1RoOJAcBD5IQFU4KxOFaOEF/bXeToD3Yd8rB48WK8//77iImJwebNm7F3715cvnwZUqkUKSkpCAoKQkZGBgBuyMOMGTOwceNGJCQkYPfu3Xjttde6DXnYuHGj0ZCHc+fOGYY8KJVKnDp1Cg899BA8PDygVCqxdu1azJo1C//6179sekBsqr4SeCu8hwK82yEGfZjxTazD7d+bVQDrdMVqRMTtoJKO65dxUUXVGuw+XYr/nrmBGk2rxa9z4vMQ6OWCYB8XBHu7GgJqpLcrgn1cMMJdNGRG7JfVNeGNw4X4JP8nAIDYmY8V00dhxYy74Cq0oCVUWwTdviXgl+cDAPa7/xJ/UCdC3Wp8fMTOfNwb7I3IQE8cuVyFHztuoOfxgLixUiyZKkfsKN9BcVztOrj03XffNQwunTBhAt555x0oFAoAwIMPPgi5XI6dO3cayu/btw8vvfSSYXDp66+/bnJw6fbt21FXV4dp06bhr3/9K8LDuS98Xl4efvvb3+Ly5ctoaWlBWFgYFi1ahNTUVItPAR0SWjod8I+fAT+dse1+ZVHA2Me5oBrR/dS4v7S267jWV04JTlytAY8HyDzFXCB1CaZgH1fIPMWmB7EOYWev1+H/9hfgTMktAFzL9HfxEfj5vUHgdzkWdY2tOFN8C6eLuZZU4Y1q/I7/IZY4HQYA5OpG4w/8tQgOC8cUuQ+mhPlgXKDEcGeDTsdw/MpN7DhRjG9+uD2cJULmgaemypF4bxDEzhZcIHAQuo2nC4eEFnB7VDljAOu4fMV0nX7v+NfkOsYtncuIPAfkCGpVUxtcnAW2vTVoiGCM4eD5CmR8eQk3bnFDLaKCJEh9JBzqpjacLq7F6aJb3W67AoAAiRhLfS8gpeoNCNvrwVy8wes4XezJ1aoG/Cu7GP/Lu4HGjoekeLk6IykmBIvuC0WgtRdS+gGFVhcOCy1COjS3abEzuxjvHrlqdvrsu0a4IUbug5gwH0yR+2Cktwt3aldbdPvqIgBMXQ3MTO/16qKqqQ37zlzHzuxiQ2AK+Dw8Ok6GJVPlmBTqPWBOHSm0uqDQIgNFdUML3s78AQfOlSPExxVT5D6ICfPGZLkP/Nx76O5obwEy04FT73G/j4zhBqN69T6UR6tjyLpUiR0niqH88faFlKggCZbcL0fC+ADLxpbZEYVWFxRaZMjofHVR7MUNRu3ldLGzS+Vq7DxRjE/P/oSWdq7Lws9dhGRFCJLvC4G/h32mUeoNhVYXFFpkSLlVDOx7yurTxc5qNa34KKcU/1GWoELNjfFzFvDw2PhAzIkOQJifO0Z6u8C5n4abUGh1QaFFhpyup4uuvkD4o9wy6mHuDgALtGl1OHShAjuzi5HbcZVTT8DnIcjLBaG+rgj1dYXc1w0hPq6Q+3H/2vJqJIVWFxRaZMgq+BzYvxZorL69TiACwqYDY2Zxi2egRbv6/nodPjhZgnM3VCip1RgGyJoj8xQbwizUzxWhPm6GgPMQW3cLEoVWFxRaZEjTtnE3ohd+CRQe5E4fOwuYAIyZzQWYLMqiQciMMVTVt6C4WoOS2kaU1GhQXNOI0ppGFFdrUN/LA4R93YSGQPvtQ6Nwt79Hj+UptLqg0CLDBmPAzUIuvAq/5GbRQKevuefI2y0w+TRutgmr34LhVmMbSmo0KKlpRHGNhguzjt+73ilx4NlpiAzs+R5MCq0uKLTIsNVQBfxwmAuwa0eMZ6MVegB3z+RaYaN/ZtF8Zpaob25DSU0jt9RqsDhW3uuN3BRaXVBoEQJurrUfv+FaYT8cMp42icfn5jIbM4ub1wyso4HW9c4MZsG6Lj9bEIgUWl1QaBHShU4HlOd39IN9yU2XbS/Lv+Fmqe2BXaZbJoQMIXw+EDSJWx5+CbhVwrW+fjjEnVKCZzzDCI93+99u6/hdtvOMt3edavoOUEuLEDIgWPodpVvyCSGDCoUWIWRQodAihAwqFFqEkEGFQosQMqhQaBFCBhUKLULIoDJsBpfqh6Op1epeShJCHEH/3ext6OiwCa36eu5JJ8HBvc+nTQhxnPr6ekgk5meEGDYj4nU6HcrKyuDh4dHr00fUajWCg4Nx/fr1QTl6nurvWFT/vmGMob6+HoGBgeDzzfdcDZuWFp/Px8iRI616jaen56D8o9Oj+jsW1d96PbWw9KgjnhAyqFBoEUIGFQotE0QiEdLT0yESWT8N7UBA9Xcsqr99DZuOeELI0EAtLULIoEKhRQgZVCi0CCGDCoUWIWRQodAihAwqwza0tm7dCrlcDrFYDIVCgZycnB7L79u3DxERERCLxYiKisLBgwf7qabGMjIyMGXKFHh4eMDf3x+JiYkoLCzs8TU7d+4Ej8czWsRicT/V2NjLL7/crS4RERE9vmagHHsAkMvl3erP4/GwcuVKk+UdfeyPHz+OOXPmIDAwEDweD59++qnRdsYYNmzYgICAALi4uCAuLg5Xrlzpdb/Wfn9saViG1p49e5Camor09HTk5eUhOjoa8fHxqKqqMlk+OzsbSUlJWLp0KfLz85GYmIjExERcuGDH58SZ8c0332DlypU4efIkMjMz0dbWhkceeQQajabH13l6eqK8vNywlJSU9FONu4uMjDSqy3fffWe27EA69gBw+vRpo7pnZmYCAH75y1+afY0jj71Go0F0dDS2bt1qcvvrr7+Od955B9u2bcOpU6fg5uaG+Ph4NDc3m92ntd8fm2PDUExMDFu5cqXhd61WywIDA1lGRobJ8gsWLGAJCQlG6xQKBVuxYoVd62mJqqoqBoB98803Zsvs2LGDSSSS/qtUD9LT01l0dLTF5QfysWeMseeee46NGjWK6XQ6k9sH0rEHwD755BPD7zqdjslkMvbGG28Y1tXV1TGRSMQ++ugjs/ux9vtja8OupdXa2orc3FzExcUZ1vH5fMTFxUGpVJp8jVKpNCoPAPHx8WbL9yeVSgUA8PHp+ZHjDQ0NCA0NRXBwMB5//HFcvHixP6pn0pUrVxAYGIi77roLycnJKC0tNVt2IB/71tZWfPDBB/j1r3/d48whA+nYd1ZUVISKigqj4yuRSKBQKMwe3758f2xt2IVWdXU1tFotpFKp0XqpVIqKigqTr6moqLCqfH/R6XRYs2YN7r//fowbN85suTFjxuCf//wnPvvsM3zwwQfQ6XSYOnUqbty40Y+15SgUCuzcuROHDh3Ce++9h6KiIjzwwAOG+c66GqjHHgA+/fRT1NXV4amnnjJbZiAd+670x9Ca49uX74+tDZupaYailStX4sKFCz32CQFAbGwsYmNjDb9PnToVY8eOxfvvv49XX33V3tU0MmvWLMPP48ePh0KhQGhoKPbu3YulS5f2a13u1D/+8Q/MmjULgYGBZssMpGM/VAy7lpafnx8EAgEqKyuN1ldWVkImk5l8jUwms6p8f1i1ahX279+Po0ePWj1PmLOzM+69915cvXrVTrWznJeXF8LDw83WZSAeewAoKSnB119/jaefftqq1w2kY68/htYc3758f2xt2IWWUCjEpEmTkJWVZVin0+mQlZVl9H/EzmJjY43KA0BmZqbZ8vbEGMOqVavwySef4MiRIwgLC7N6H1qtFufPn0dAQIAdamidhoYGXLt2zWxdBtKx72zHjh3w9/dHQkKCVa8bSMc+LCwMMpnM6Piq1WqcOnXK7PHty/fH5vqlu3+A2b17NxOJRGznzp2soKCALV++nHl5ebGKigrGGGOLFi1iL774oqH8iRMnmJOTE3vzzTfZpUuXWHp6OnN2dmbnz5/v97r/5je/YRKJhB07doyVl5cblsbGRkOZrvV/5ZVX2OHDh9m1a9dYbm4ue+KJJ5hYLGYXL17s9/o///zz7NixY6yoqIidOHGCxcXFMT8/P1ZVVWWy7gPp2OtptVoWEhLC1q1b123bQDv29fX1LD8/n+Xn5zMA7O2332b5+fmspKSEMcbYxo0bmZeXF/vss8/YuXPn2OOPP87CwsJYU1OTYR8PP/ww27Jli+H33r4/9jYsQ4sxxrZs2cJCQkKYUChkMTEx7OTJk4ZtM2bMYIsXLzYqv3fvXhYeHs6EQiGLjIxkBw4c6OcacwCYXHbs2GEo07X+a9asMXxWqVTKZs+ezfLy8vq/8oyxhQsXsoCAACYUCllQUBBbuHAhu3r1qmH7QD72eocPH2YAWGFhYbdtA+3YHz161OTfi76OOp2OrV+/nkmlUiYSidjMmTO7fa7Q0FCWnp5utK6n74+90XxahJBBZdj1aRFCBjcKLULIoEKhRQgZVCi0CCGDCoUWIWRQodAihAwqFFqEkEGFQosQMqhQaBFCBhUKLULIoEKhRQgZVP4/dG2AIsMePw4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:44:11.968723Z", + "iopub.status.busy": "2024-03-22T20:44:11.968034Z", + "iopub.status.idle": "2024-03-22T20:48:05.896124Z", + "shell.execute_reply": "2024-03-22T20:48:05.895337Z" + }, + "papermill": { + "duration": 233.948452, + "end_time": "2024-03-22T20:48:05.898722", + "exception": false, + "start_time": "2024-03-22T20:44:11.950270", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:05.934156Z", + "iopub.status.busy": "2024-03-22T20:48:05.933853Z", + "iopub.status.idle": "2024-03-22T20:48:05.952653Z", + "shell.execute_reply": "2024-03-22T20:48:05.951833Z" + }, + "papermill": { + "duration": 0.038451, + "end_time": "2024-03-22T20:48:05.954486", + "exception": false, + "start_time": "2024-03-22T20:48:05.916035", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.0310920.0905570.0031412.4890860.0418141.1387050.150350.0000383.9917130.0417230.1346710.0560340.0930270.02486816.480799
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.031092 0.090557 0.00314 12.489086 0.041814 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 1.138705 0.15035 0.000038 3.991713 0.041723 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.134671 0.056034 0.093027 0.024868 16.480799 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:05.987875Z", + "iopub.status.busy": "2024-03-22T20:48:05.987334Z", + "iopub.status.idle": "2024-03-22T20:48:06.323344Z", + "shell.execute_reply": "2024-03-22T20:48:06.322430Z" + }, + "papermill": { + "duration": 0.354866, + "end_time": "2024-03-22T20:48:06.325490", + "exception": false, + "start_time": "2024-03-22T20:48:05.970624", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:06.360856Z", + "iopub.status.busy": "2024-03-22T20:48:06.360522Z", + "iopub.status.idle": "2024-03-22T20:52:09.563033Z", + "shell.execute_reply": "2024-03-22T20:52:09.562228Z" + }, + "papermill": { + "duration": 243.222766, + "end_time": "2024-03-22T20:52:09.565475", + "exception": false, + "start_time": "2024-03-22T20:48:06.342709", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/lct_gan/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/lct_gan/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:52:09.602257Z", + "iopub.status.busy": "2024-03-22T20:52:09.601874Z", + "iopub.status.idle": "2024-03-22T20:52:09.628172Z", + "shell.execute_reply": "2024-03-22T20:52:09.627453Z" + }, + "papermill": { + "duration": 0.046497, + "end_time": "2024-03-22T20:52:09.630047", + "exception": false, + "start_time": "2024-03-22T20:52:09.583550", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:52:09.663146Z", + "iopub.status.busy": "2024-03-22T20:52:09.662872Z", + "iopub.status.idle": "2024-03-22T20:52:09.668072Z", + "shell.execute_reply": "2024-03-22T20:52:09.667239Z" + }, + "papermill": { + "duration": 0.023904, + "end_time": "2024-03-22T20:52:09.670018", + "exception": false, + "start_time": "2024-03-22T20:52:09.646114", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.38558200279871624}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:52:09.706391Z", + "iopub.status.busy": "2024-03-22T20:52:09.705913Z", + "iopub.status.idle": "2024-03-22T20:52:10.051187Z", + "shell.execute_reply": "2024-03-22T20:52:10.050352Z" + }, + "papermill": { + "duration": 0.366373, + "end_time": "2024-03-22T20:52:10.053189", + "exception": false, + "start_time": "2024-03-22T20:52:09.686816", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABB9klEQVR4nO3deXxTdb4//tfJnqZJ2tIdWloom+xIYQAREAZRRmGc8eqVUQqizIiDDsN9KHdmcBsH9LrgLJev40gLDwVcfow6w1VBRmBABQVk3yml+94mafacz++P06RNm7ZJmuSk6fv5eOSR9OSTk89p0nc/+4djjDEQQoiIJGJngBBCKBARQkRHgYgQIjoKRIQQ0VEgIoSIjgIRIUR0FIgIIaKjQEQIER0FIkKI6CgQkW4VFRWB4zhcv35d7KyQGEaBiITc9u3bsWnTJrGzQfoQCkQk5CgQkUBRICKEiI4CEQnYp59+ilmzZkGr1UKn0yE/Px/bt28HAMyePRu7d+9GSUkJOI4Dx3HIycnx+9w8z+PZZ59FZmYm4uLiMGfOHJw7dw45OTkoKCjwpGtoaMDatWsxduxYxMfHQ6fT4Y477sDJkye9zrd//35wHIf3338fL774IgYNGgSVSoW5c+fiypUrofh1kBCQiZ0B0rcUFRVh+fLlGD16NNatW4eEhAScOHECn332GR544AH85je/QXNzM8rKyvD6668DAOLj4/0+/7p16/Dyyy/jrrvuwu23346TJ0/i9ttvh9Vq9Up37do1fPTRR7j33nuRm5uL6upqvPnmm5g1axbOnTuHzMxMr/QbN26ERCLB2rVr0dzcjJdffhlLlizBkSNHev9LIb3HCOlGYWEhA8CKi4tZU1MT02q1bOrUqcxisXil43ne83jhwoVs8ODBAb9XVVUVk8lkbPHixV7Hn332WQaALV261HPMarUyl8vlla64uJgplUr2/PPPe459+eWXDAAbNWoUs9lsnuNvvPEGA8BOnz4dcD5J6FHVjPht7969MBqNePrpp6FSqbye4ziu1+fft28fnE4nHnvsMa/jv/zlLzulVSqVkEiEr6/L5UJ9fT3i4+MxYsQIHD9+vFP6ZcuWQaFQeH6eOXMmAKFkRcRHgYj47erVqwCAMWPGhOX8JSUlAIC8vDyv40lJSUhMTPQ6xvM8Xn/9dQwbNgxKpRLJyclISUnBqVOn0Nzc3Onc2dnZXj+7z9fY2BjKSyBBokBE+qQ//OEPWLNmDW699Va88847+Pzzz7F3716MHj0aPM93Si+VSn2eh9FKyVGBGquJ34YOHQoAOHPmTKdSS3vBVtMGDx4MALhy5Qpyc3M9x+vr6zuVXD788EPMmTMHb7/9ttfxpqYmJCcnB/X+RDxUIiJ+mz9/PrRaLTZs2NCpF6t9yUKj0fisHvVk7ty5kMlk2Lx5s9fxP//5z53SSqXSTqWZDz74AOXl5QG/LxEflYiI33Q6HV5//XWsWLEC+fn5eOCBB5CYmIiTJ0/CbDZj69atAICbb74Z7733HtasWYP8/HzEx8fjrrvu6vH8aWlpeOKJJ/Dqq6/i7rvvxoIFC3Dy5El8+umnSE5O9ipp/ehHP8Lzzz+PZcuWYfr06Th9+jTeffddDBkyJGzXT8JI5F47EuXad9+7ffLJJ2z69OlMrVYznU7HpkyZwnbs2OF53mQysQceeIAlJCQwAAF15TudTva73/2OpaenM7VazW677TZ2/vx5NmDAAPbzn//ck85qtbJf//rXLCMjg6nVajZjxgz29ddfs1mzZrFZs2Z50rm77z/44AOv9ykuLmYAWGFhYaC/EhIGHGPUWkeiW1NTExITE/H73/8ev/nNb8TODgkDaiMiUcVisXQ65p5AO3v27MhmhkQMtRGRiKitrYXL5eryeYVCgaSkJLz33nsoKirCnXfeifj4eBw6dAg7duzA/PnzMWPGjAjmmEQSBSISEfn5+Z4Bi77MmjUL+/fvx7hx4yCTyfDyyy/DYDB4GrB///vfRzC3JNKojYhExOHDh31Wu9wSExNx8803RzBHJJpQICKEiI4aqwkhouvTbUQ8z6OiogJarTYks78JIaHFGIPRaERmZqZntQRf+nQgqqioQFZWltjZIIT0oLS0FIMGDery+T4diLRaLQDhInU6nci5IYR0ZDAYkJWV5flb7UqfDkTu6phOp6NAREgU66nphBqrCSGio0BECBEdBSJCiOj6dBuRPxhjcDqd3c5zIr5JpVLIZDIaGkHCTtRA5HK58Oyzz+Kdd95BVVUVMjMzUVBQgN/+9rch+fLb7XZUVlbCbDaHILf9U1xcHDIyMrx2wCAk1EQNRC+99BI2b96MrVu3YvTo0fjuu++wbNky6PV6rF69ulfn5nkexcXFkEqlyMzMhEKhoP/sAWCMwW63o7a2FsXFxRg2bFi3A9II6Q1RA9FXX32FRYsWYeHChQCAnJwc7NixA0ePHu31ue12O3ieR1ZWFuLi4np9vv5IrVZDLpejpKQEdru9015mfUmLzYk4hZT+GUUpUf/FTZ8+Hfv27cOlS5cAACdPnsShQ4dwxx13+Exvs9lgMBi8bj2h/+K9Eyu/v92nK3GitIm2D4pSopaInn76aRgMBowcORJSqRQulwsvvvgilixZ4jP9hg0b8Nxzz0U4l6SvM9mcKG+0oKrZipsydFDJfe9xRsQj6r+7999/H++++y62b9+O48ePY+vWrXjllVc8u0F0tG7dOjQ3N3tupaWlEc4x6YuqmoV1kBI1CvCModpg7eEVJNJEDUT/9V//haeffhr3338/xo4diwcffBC/+tWvsGHDBp/plUqlZzoHTesIjZycHM+a0LGqqtkGAFBKJXjzwDV8eKyMqmhRRtRAZDabO7VBSKVSn1sGExKsGqNQAhqeroWE42B38jDZnCLnirQnahvRXXfdhRdffBHZ2dkYPXo0Tpw4gddeew3Lly8XM1t9jt1up3E+3WgyOwAAyfEKJMTJ0dBiR0OLHVqVXOScETdRS0R/+tOf8NOf/hSPPfYYRo0ahbVr12LlypV44YUXwvq+diff5c3p4v1O6/AjbTBmz56Nxx9/HI8//jj0ej2Sk5Pxu9/9zlOdyMnJwQsvvICHHnoIOp0Ojz76KADg0KFDmDlzJtRqNbKysrB69Wq0tLR4zltTU4O77roLarUaubm5ePfdd4PKX1/i4hkMViEQJcQpkKgRAnZja3Ai0UHUEpFWq8WmTZsi3kbxly+vdPlcbrIGiycO9Pz814NX4XD5bk8YlKjGvZPbFmbbcrgYFrv3VJJf/XB4UHncunUrHn74YRw9ehTfffcdHn30UWRnZ+ORRx4BALzyyitYv349nnnmGQDA1atXsWDBAvz+97/Hli1bUFtb6wlmhYWFAICCggJUVFTgyy+/hFwux+rVq1FTUxNU/vqKFrsTCpkEPM+gUUihVQlfeZOVqmbRJObnmvVVWVlZeP3118FxHEaMGIHTp0/j9ddf9wSi2267Db/+9a896VesWIElS5bgySefBAAMGzYMf/zjHzFr1ixs3rwZN27cwKeffoqjR48iPz8fAPD2229j1KhREb+2SNKp5PjFrKGwOXlwHAddayAyWqlEFE36ZSBaNSevy+ckHQbePnrr0C7Tdhyku3xGbm+y5eUHP/iB1yjgadOm4dVXX/VM3p08ebJX+pMnT+LUqVNe1S3GmGeqy6VLlyCTyby27Bk5ciQSEhJCludoxXGcZ+xQvFJoFzJSiSiq9MtApJD53zQWrrS9pdFovH42mUxYuXKlzzl62dnZntHr/V2KVonJOYkYoFGKnRXSTr8MRH3BkSNHvH7+5ptvMGzYMEilvkcFT5o0CefOnUNenu/S3siRI+F0OnHs2DFP1ezixYtoamoKab6jzZFr9agyWDF2oB5DUuKRpFFg5rAUsbNFOoiNiUQx6MaNG1izZg0uXryIHTt24E9/+hOeeOKJLtM/9dRT+Oqrr/D444/j+++/x+XLl/Hxxx/j8ccfBwCMGDECCxYswMqVK3HkyBEcO3YMK1asgFqtjtQliaKy2YprtS0w22k9qmhGgShKPfTQQ7BYLJgyZQpWrVqFJ554wtNN78u4ceNw4MABXLp0CTNnzsTEiROxfv16ZGZmetIUFhYiMzMTs2bNwj333INHH30Uqampkbgc0RhbBy7GK9sK/warA5XNFticFJyiBVXNopRcLsemTZuwefPmTs9dv37d52vy8/OxZ8+eLs+Znp6Of/7zn17HHnzwwV7lM9q1tAYiTbtAtOtYGRrNDtw7eRAGJdISMdGASkQkZjldvGdcl3v8EADEKYTHHcd8EfFQICIxq6U10EglHJTtejTVCqHBn9qNogdVzaLQ/v37xc5CTLA6hEDTcWXGOApEUYdKRCRm2Z08FDJJp4XQ3CUii4MGNUYLKhGRmJWVFIdVc/Lg4r3nCrrbiKhEFD2oRERinrTDvB2qmkUfKhGRfic5XokpuUlIiKP1iKIFBSISs45cq0dlsxVjB+kxNCXeczxJo8CMvGQRc0Y6oqoZiVlVBiuK61pgtlEVLNpRICIxy90G5O4lc2OMoclsR1WzFTxPi+hHAwpEJGa5R07HKTqvWFD01XXsOHoDFgeVlqIBBSISs9xBRt1hHBHHcVDKhGNWCkRRoX8FIsYApz3ytwD20Nq2bRsGDBgAm83mdXzx4sUxP0E1lJyuts0LOlbNAHimfNiC3OCAhFb/6jVzOYB/vxr59535a0Dm33Y/9957L1avXo1PPvkE9957LwBh943du3d3O7OeeHOXhiSc9zwzN5VcimaLg0pEUaJ/lYj6ALVajQceeMCz8wYAvPPOO8jOzsbs2bPFy1gf43AxKGQSKOUSr3lmblQiii79q0QklQulEzHeNwCPPPII8vPzUV5ejoEDB6KoqAgFBQU+/6CIb0kaBVbNyeuyV8w9/4xKRNGhfwUijvO7iiSmiRMnYvz48di2bRvmz5+Ps2fPYvfu3WJnq0+SdNyWpZW7RGR1UIkoGvSvQNSHrFixAps2bUJ5eTnmzZuHrKysnl9E/JaTrIFaIcWgxNhes7uvoDaiKPXAAw+grKwMb731FpYvXy52dvqcS9VGfHSiHCduNPp8Pi81HjPykpGVREvFRgMKRFFKr9fjJz/5CeLj47F48WKxs9Pn1JvsKK5rQUOLXeysED9Q1SyKlZeXY8mSJVAqaTPAQLl36HAPXOzI4eJhsjrBIDRsE3FRIIpCjY2N2L9/P/bv34///d//FTs7fZK7W14p913or2iyYNfxciRrlXjwB4MjmTXiAwWiKDRx4kQ0NjbipZdewogRI8TOTp/k7pZXdVEicm8PbqdxRFGBAlEU6mrfMuK/nkpECikFomhCjdUkJnkCkY/pHYB3iYgFMBeQhEfMByL6kvVOX/39uUdUd9zBw83diM0zBietSSS6mK2ayeXCtAqz2Qy1mgatBctsNgNo+332FUun58DFM3Q1KUYu5cBxwsIIdicPuTTm/ydHtZgNRFKpFAkJCaipqQEAxMXF0VytADDGYDabUVNTg4SEBEilvksW0azj7h3tcRwHhUwCm4OHzclDQyMkRBWzgQgA0tPTAcATjEjgEhISPL/HWDNhUAIY2tqLiHhiOhBxHIeMjAykpqbC4XCInZ0+Ry6X98mSkMHqwL/O10CjlOGHN6V1mW467eQRNWI6ELlJpdI++QdFgtNic6K4rgVaVb/4escEKpOSmGNrXdqjqx4zN6vDhSazndYkigIUiEjM6WkMkdu+8zUoPHwd5ysNkcgW6QYFIhJz3CUcZQ8lIiVN84gaFIhIzHGXiFQ9lIg8o6tdFIjEJnogKi8vx89+9jMMGDAAarUaY8eOxXfffSd2tkgf5lkCpIcSEU18jR6idis0NjZixowZmDNnDj799FOkpKTg8uXLSExMFDNbpI9zuoQpGz21ESloJ4+oIWogeumll5CVleW1dU5ubq6IOSKxYM7IVNw6PKXHeXI0Az96iFo1++STTzB58mTce++9SE1NxcSJE/HWW291md5ms8FgMHjdCPFFKuEg62H+GDVWRw9RA9G1a9ewefNmDBs2DJ9//jl+8YtfYPXq1di6davP9Bs2bIBer/fcaGcL0hsJcQqMz9JjWFq82Fnp9zgm4joPCoUCkydPxldffeU5tnr1anz77bf4+uuvO6W32Wxee8IbDAZkZWWhubkZOp0uInkm0W/P2Sq4eIbpQ5Ohj+tbqwbEGoPBAL1e3+PfqKglooyMDNx0001ex0aNGoUbN274TK9UKqHT6bxuhHR0tbYFF6qMcPJU5eorRA1EM2bMwMWLF72OXbp0CYMH02LmJDiMMb+77xljMNudaDLb++wCcLFC1ED0q1/9Ct988w3+8Ic/4MqVK9i+fTv++te/YtWqVWJmi/RhNicPd0zpaUCj3cXjzQPXUHj4Oq3SKDJRA1F+fj7+/ve/Y8eOHRgzZgxeeOEFbNq0CUuWLBEzW6QPc48JkvnRa6Zo97yDRleLSvR1En70ox/hRz/6kdjZIDHC5pln1vP/WPcqjXYnD7uTRxztsyga0ad4EBJKnnlmPbQPudGgxuhAgYjEFH+XAHGjaR7RQfSqGSGhlJcaj9Vzh/ndde8ORNRGJC4KRCTmSCUcpJIAq2YUiERFgYj0a7kpGiRq5NCraQS2mCgQkZhysrQJFU0WjEjXYkhKz3PIJmXTkjPRgBqrSUypaLLgQpURjWbaPqovoRIRiSmB9pq5eAa7kwfH+d/lT0KPSkQkprjnman8GNAIAMdKGvH/DlzFwUu14cwW6QEFIhJTrA53icjPXjNaQD8qUCAiMaVt5r2fAxqlNI4oGlAgIjHFFmyJiEZWi4oCEYkZThfvWc7D3zYimmsWHajXjMQMmVSC1XOHweZ0eS3x0R2aaxYdKBCRmCKVcIhT+P+1bptrRgujiYkCEenX1HIpRmVo/W5TIuFBgYjEjKpmK07caESKVonJOUl+vUatkGLBmIww54z0hBqrScxoaLHjQpURJfVmsbNCAkSBiMSMtlHVgVWzXDyDxe6CixbQFw0FIhIzAp1n5rblUDH+34GrqDfZek5MwoICEYkZnkDk5xgiN+rCF19QgejatWuhzgchvWZ1BFc1o/lm4gsqEOXl5WHOnDl45513YLVaQ50nQoISbNWM5puJL6hAdPz4cYwbNw5r1qxBeno6Vq5ciaNHj4Y6b4QExF0iCnRMkJzmm4kuqEA0YcIEvPHGG6ioqMCWLVtQWVmJW265BWPGjMFrr72G2lpa24VE3k8mDcLKWUMwJEUT0Otovpn4etVYLZPJcM899+CDDz7ASy+9hCtXrmDt2rXIysrCQw89hMrKylDlk5Aeuad3yP2cZ+ampBKR6HoViL777js89thjyMjIwGuvvYa1a9fi6tWr2Lt3LyoqKrBo0aJQ5ZOQ0LA0AsZqgLWNGUrTqTAyXYtkrVLEjPVvHGMs4FFcr732GgoLC3Hx4kXceeedWLFiBe68805IJG1xraysDDk5OXA6nSHNcHsGgwF6vR7Nzc3Q6XRhex8S/Vw8w56zVVDKJbh1WApkHUtFPA9c+hSoPCX8rMsExv4UUARWjSOB8fdvNKi5Zps3b8by5ctRUFCAjAzf83RSU1Px9ttvB3N6QgJmc7pwocoIAJg9PLVzgmtftgUhTgIYKoCzHwETHgA4LnIZJT4FFYj27t2L7OxsrxIQADDGUFpaiuzsbCgUCixdujQkmSSkJ561quUSSCQdAouxGij7Vng8+seAJhk4VgQ03QBqLwCpo+DiGVw884wpIpEV1G996NChqKur63S8oaEBubm5vc4UIYHyrFXtq+u+5JDQJpQ6UrhpkoHsHwjPXT+EK9VG/HHfZew6XhbBHJP2ggpEXTUrmUwmqFSqXmWIkGC4S0Sdloi1NAJ1l4XHOTPbjg+cDEjlQEsd1Bahd5cGNIonoKrZmjVrAAAcx2H9+vWIi4vzPOdyuXDkyBFMmDAhpBkkxB9dlogqTgiloaQhQknITa4CUkcBlacQ13AWwGiaayaigALRiRMnAAglotOnT0OhUHieUygUGD9+PNauXRvaHBLiB58lIsaAmvPC48wJnV+UNgaoPAVl01WAjaK5ZiIKKBB9+eWXAIBly5bhjTfeoC5zEjVsvqZ3GMoBq0GogiUN6fwifRYgV0FmNUNrr4FJkg7GGDjqRYu4oHrNCgsLQ50PQnplck4Sxg7Sg0O7IFJzQbhPHi4Eo44kEiBpCCSVZ5BoKYFRmQ4nzyCXUiCKNL8D0T333IOioiLodDrcc8893abdtWtXrzNGSCA67d7BGFB3UXicOqrrFw7Ig7T6LBKtpbiBqbA7+YCniJDe8zsQ6fV6T5FVr9eHLUOEhIS5QaiWSWRAYk7X6RJzwYHDILkR9mQZjW0Uid+BqH11jKpmJNp8c60eZrsTYwcmIEWrBBqLhSf0g3xXy9wUcYAmGcPTgOGDnEAAe6KR0AmqDGqxWGA2t+2UUFJSgk2bNmHPnj0hyxghgbhcY8LJ0maY7a1zGxtaA1GSHwNs9VnCfXNpeDJHehRUIFq0aBG2bdsGAGhqasKUKVPw6quvYtGiRdi8eXNIM0iIP7x6zVxOoOm68ESiP4FoEACAbyoFTzt5iCLoFRpnzhRGqX744YdIT09HSUkJtm3bhj/+8Y9BZWTjxo3gOA5PPvlkUK8n/Zt7MKJKLhG67V1OYWZ9vI8JsB0lZOFClQHfnDyLixUNYc4p8SWoQGQ2m6HVagEAe/bswT333AOJRIIf/OAHKCkpCfh83377Ld58802MGzcumOyQfs7FM8+iZiq5VAhEAJCQ5d/MeqUOvCwOHBiYqSaMOSVdCXrx/I8++gilpaX4/PPPMX/+fABATU1NwIMcTSYTlixZgrfeeguJiYnBZIf0c+7pHUDrsq+GCuEH3SD/TsBxcGjShIdGWlVUDEEFovXr12Pt2rXIycnB1KlTMW3aNABC6WjixIkBnWvVqlVYuHAh5s2b12Nam80Gg8HgdSPEawkQDm0lIl2m3+fgNcK6WpypKtTZI34Iqq/ypz/9KW655RZUVlZi/PjxnuNz587Fj3/8Y7/Ps3PnThw/fhzffvutX+k3bNiA5557LuD8ktjmNeHV2gTYzYBECsSn+X0OXpsOAOBaqsORRdKDoAdNpKenIz093evYlClT/H59aWkpnnjiCezdu9fvpUPWrVvnWQEAEJahzMrK8vs9SWxK06qwctYQOHkGNF8SDsanAdIAvt5aIWhJzfWA0w7IFD28gIRSUIGopaUFGzduxL59+1BTUwOe95617M9OsMeOHUNNTQ0mTZrkOeZyuXDw4EH8+c9/hs1mg1TqvaSDUqmEUkkLnBNvkvbTO0rd7UMDAzqHVKWDXRoHnncCpmqhoZtETFCBaMWKFThw4AAefPBBZGRkBDVbee7cuTh9+rTXsWXLlmHkyJF46qmnOgUhQvwSRPsQACSo5XAmZULjqgBaaigQRVhQgejTTz/F7t27MWPGjKDfWKvVYsyYMV7HNBoNBgwY0Ok4Id25UmNEaYMF2QlyDDW2tvEEGIhykjXIGT0CuNEItHReBpmEV1CBKDExEUlJSaHOCyFBKWu04PvSJsTbrBjKeGEgoyqIidmaFOGexhJFXFCB6IUXXsD69euxdetWr+Vie2v//v0hOxfpP9x73mvtrQFElxncFkHxqWBg4FpqhGVEaCp+xAQViF599VVcvXoVaWlpyMnJgVzuPbv5+PHjIckcIf6wtAYijc1dLQusoRoAmsx2vHukAfmlzZgyOAGwNgPqhNBlknQrqEC0ePHiEGeDkOBZ7EKvbZylWvhGB9g+BAAyqQR2XgKTNKG1VFRHgSiCggpEzzzzTKjzQUjQLA4XFE4TFJwJkCsAre/dh7ujaF2VsUWWCJ41QNpSAyTnhTqrpAtBr4nZ1NSEv/3tb1i3bh0aGoQZy8ePH0d5eXnIMkeIP6wOF+LttZBLJEKDcxCDEeVSDhwHmBVJcPGMGqwjLKgS0alTpzBv3jzo9Xpcv34djzzyCJKSkrBr1y7cuHHDs1YRIeHmdPGwO3nE26ohi+eCah8ChL365FIJzPLWQNRSG+Kcku4EVSJas2YNCgoKcPnyZa/pGXfeeScOHjwYsswR0hOphMPPZw3F7dkuSCVcUO1DbkqZEIh4nglrXvOunl9EQiKoQPTtt99i5cqVnY4PHDgQVVU0e5lEDsdxUMuAeFudsJVQkCUiAFDIJLBLNXBwMoDxgKUpdBkl3QoqECmVSp9LcFy6dAkpKSm9zhQhATHVALwTkCmBuOAH2mbq1chNiYdEM0A4YKYR1pESVCC6++678fzzz8PhcAAQ/ivduHEDTz31FH7yk5+ENIOEdKeiyYJvT59GjdEqlIZ6MQhx3k1pWDxxIHRJratKmOtDlEvSk6AC0auvvgqTyYSUlBRYLBbMmjULeXl50Gq1ePHFF0OdR0K6VGu0oar0GhrNjl61D3nRJAv3NOcsYoLqNdPr9di7dy8OHz6MkydPwmQyYdKkSX6tskhIKFkcLsTbaiBT966h2kscVc0iLeBAxPM8ioqKsGvXLly/fh0cxyE3Nxfp6elgjAW1JAghwbJZTFA5DZBL1b0ORF9dqcOJ0ibkp3CYAghVM5pzFhEBVc0YY7j77ruxYsUKlJeXY+zYsRg9ejRKSkpQUFAQ0DKxhIQC515/KC4JkKt7dS4GwO7k0SLTCUvNupzCnDMSdgGViIqKinDw4EHs27cPc+bM8XruX//6FxYvXoxt27bhoYceCmkmCekKZ2xdkVEffLe9m0Im/F+2uwCoE4U2InM9zTmLgIBKRDt27MB///d/dwpCAHDbbbfh6aefxrvvvhuyzBHSE1nrrhuSUASi1vlmDhffrp2Ies4iIaBAdOrUKSxYsKDL5++44w6cPHmy15kixC88D1mLsA+ZPNHPPcy6IW8NRHYnBaJIC6hq1tDQgLS0rrdoSUtLQ2NjY68zRYhfzPWYmBkHO9NCnhrCqpmTpy78CAsoELlcLshkXb9EKpXC6XT2OlOE+MVQDplEAllCFtDN99JfSk8bEe/dhU89Z2EX0KfHGENBQUGXW/rYbLaQZIoQvwS5Y0dX1AopMhNU0KsVQiDiOMBhBRxmYR1sEjYBBaKlS5f2mIZ6zEikmGpKUFNngi1Zh1EhOF9yvBL35We3HVDqhO77ljoKRGEWUCAqLCwMVz4ICYzDApuhBtUGGypatCEJRJ1okoVAZK4HEgeH4x1Iq6BXaCREVIYKOFwMVpkeyrj48LwH9ZxFTO9b+AgRQ3MZHE4eRmUa4hWh2RWYMYa3DxXD5uRRMD0HGgpEEUMlItI3GcphdwmByLPvfS9xHAebU1h6lrrwI4sCEel7eB4wlMPh4mFUpEETokAEdNGFbzMCTuoRDicKRKTvaakBXE7YmBwWeSLUIaqaAR0GNcrVbb1lVD0LKwpExG9Gq0PsLAiahfFDjbIUgOOgUYYuEHmmebiETRs9pSKqnoUVBSLilyazHVu/uo7L1UaxswI0lQAApt08AQXTc5Ac73uAbTAU7eebAW3tRFQiCisKRMQvl6pNcLgY9l2ogdUh4jY7jAFNNwAAygE5SNQoPKWYUPCqmgFAHAWiSKBARLpU2mDGqbImWOwuTB6ciCSNAha7C6fKRFwsrKUWcFgAqSx0S8O2M0CjQGaCqq3dyb0rCAWisKJARLp0orQJ+87X4FhJIyQSDpNzEgEAZ8qbwRgTJ1ONQrWsSZGBLy7U4Ux5aIPi9Lxk3JefjeFpWuGAu2pmaRRWbCRhQYGI+GR38rhe1wIAGJEu/FEOT9NCKZeg2eJAWaNFnIy1tg/Vy9JwurwZV2tN4X0/RTwgUwhVQktDeN+rH6NARHyqaLLAxTNoVTIkxysACD1KeSnCdIorNWEOAL7wvKd9qFGZAQCIV4Z5cgDHUTtRBFAgIj65SzxZSXFeO7PkpQqB6GqtKfLVM1OVMLBQpkADJ7TdhDoQXagy4G//vobPzrTbOp268MOO5poRn8oazQCAQYneO2NkJ8Vh7EA9hqRoIr9eWP1V4T4xFya70KsVrwrtV5gxwGh1osXWrj2IuvDDjgIR6cTh4lFtEKY0DEqM83pOJpVg3k1dLxccVvVXhPsBeTBdFwKFVikP6Vuo5EJvmc3dfQ+0q5pRiShcqGpGOmlosYNnDBqlFLoQlziCZjMCxiqhCJY0BEarEIhCXSJSyYU/CUv7sVKeLvxGoZ2KhFyUfMtINEnTqfCL2UNhsDp87tzLGENFsxUl9S2YlJ3oKUWElbtaps2AVaL2DDgMdRuRSiZci9egTVUCIJEBvBOwNrUFJhIyFIhIG5cTuLoPqD4LlVILVd5cAEM6JeM4Dl+cq0ZDix2pWiXyUrXhz1vdJeF+QB5UcilWzcmD0erwjIQOFXdQtTt5uHgGqYQDJBIgLhEw1QLmBgpEYUBVM9Lm4m6g/LjQM9VSB5z+EGi87jNp9gCh7aik3hz+fNnNQEOx8DhlJABhKsaAEM4xc1O2C2w2Z/vqGbUThRMFIiKovwpUnwMPDl9wU3HOlQmXywlc+D/Aae+UfHCSEIiu15vD341fewFgPKBNAzQDwvpWEgmHdL0KmQkquPh210WLpIWVqIFow4YNyM/Ph1arRWpqKhYvXoyLFy+KmaX+iTHg+iEAgCF5PE47BuGgZAokar2weHzZ0U4vGZQYB6mEg8HiQJM5zMuDVJ8V7lNHAwCO32jEF+eqPUMMQu0/p2TjvvxsaFXteuRo2diwEjUQHThwAKtWrcI333yDvXv3wuFwYP78+WhpaREzW/2PsRIwVAASGWr04wEASdp4cEPmCM+XfdtphUKFTILMBGGMUXF9GD8vUy3QXCb0lqUKe3UU17bgdHkzmi0RXB+pfdVMrHl2MUzUxurPPvvM6+eioiKkpqbi2LFjuPXWW0XKVT9UfU64Tx6GOocwnSNRoxDaY+IOCaWAiu+B7KleL8tN1qC0wYziWqH3LCzKj3nyBpUOADwBKCFOEZ739EWdKARDpx2wmwBlBBro+5GoaiNqbhZmUicl+e6VsNlsMBgMXjfSSzwP1LRWfdLGoKFFaA9K0iiE3qKsKcJzld93KgnkJgvLqDa02L3bU0LFYQGqzwiPB04GALh45hlDFK4xTv++XIu//fsaTpY2tR2UyoRgBFA7URhETSDieR5PPvkkZsyYgTFjxvhMs2HDBuj1es8tKysrwrmMQc2lQq+UXA0k5aKxfSACgNSbhNnn5oZOPWiJcXL855RsPHxLrtDNHWqlRwCXA4hPARKEHVgbzcJgS4VMErYJrw4XL0zzsHdY9oPmnIVN1ASiVatW4cyZM9i5c2eXadatW4fm5mbPrbS0NII5jFEN14T7pCHgIUFja8OzJxDJFEBa6z+GihNeL+U4oYdJEo4gZDUIbVMAkDvLM6nNXWIboFH4HGwZCu5BjTZHh1HU8anCfUttWN63P4uKAY2PP/44/vnPf+LgwYMYNGhQl+mUSiWUytCPHenX2gUik90JuVQCCcd7V3syJwrji+ouA/YWn/vA8zwDxyE0wYEx4PIeYYClfiAwIM/zVJ1JaDQPxxgiN1Xr6oyWjkviatyBqCZs791fiRqIGGP45S9/ib///e/Yv38/cnNzxcxO/2MzAqaa1vlbudAp5Pj5rCGwOXnvgBKfKizLaqgAqs50arT+6qqwUuKdYzM6TZINStl3QtCTSIHhd3hN8TfbhODgKbGFgc9pHoB3iYjnhTY0EhKiBqJVq1Zh+/bt+Pjjj6HValFVJawBo9froVare3g16TX3aOX4NE8ph+M433PHMsYLgajypNCA3S44mKxOtNhcuFBp7H0gqjwlTDMBgCGzhfahdubdlIaZw5N79x49cE98tXasmqkShEZrF805CzVRQ/rmzZvR3NyM2bNnIyMjw3N77733xMxW/9Hc2saWmNNz2tRRgFQudOU3e7fNjcoQutUv1RjhdAU5O91UI0wpubBbqJplTgQG5ftMqpRJoZSFb6KtOxB3KhFJJIAmpS2/JGREr5oRETWXCfd6ofdx96lKOHkeM/KSO+8VJlMKPWiVJ4Vbay8WICyeplXJYLQ6caHKiDED9f7nwdIIFP8bqDkHz0pr2T/waqCOtDiFFIlxct/jlDSpgKGytZ1oZMTzFquiorGaiMBmErrkOQ7QDwRjDNfrW2B3CoHIp4zxQhCqvQDk/RCQqwAI1bmJ2Qk4eKkO311vwE0Zup570hxWYVpJ+TFhHhkApIwQAlAX88m+L23CpWojxmTqcVOmLtgr71FCnAIFM7por6QSUVhQIOqv3KUhTTIgV6PF5oTdyUPCcUjsasSyLlNI31InDIIceLPnqTED9Tha3IhGswOny5sxPiuh08sZY6gz2dFSU4ykkv+DFlahUTwpVwhAuoxus1zWaEZ5owVDkjv32kWMu82KuvBDigJRf9WhWuYeyKhXy7oenMhxQMYE4MoXQsmoXSBSyqSYNnQAvrxQgxM3GjF2oN5TKqo32XCx2ojL1Sag9iKG1X2BSjBAnYQp8/8D0uShPWaXMYby1gX90/WqIC86BNxd+Jam1oX8aThJKFAg6q/cDc56YdyWe6BgYk/d4mmjgWtfAsZqYelWbbrnqXED9WhoseHmwUmeIPT+d6WeAKKx1WBsw78Qr5SiQpED9di7IE0e6Fd2a4w2mO0uKGQSZOjD36P66elKlDVa8MOb0pDTvgSmiAOU8ULVtqXW8/sjvUOBqD9y2tvaONyByNwaiHqaSKqIA5KHAzXnhVJRu0AkkXC4baT3wvpapVDCGqp1YnLDVxiQrYMsZThuGn0PXKyt5GWxuyCRoMveMPdmj1lJceGZTtKBzcnDZHPCZHN2flKTKgQiUw0FohChEVn9kaFcaCBW6YUbgCZzhzlm3ckQlgpB1Wlhnlo3ZgxLxqPTM7EQh5CmdEGmSwduuhtSqdSzzKvV4cL/d7wMH5+o8KxF3R5jDBerjQAQsfYhdVejq4G2gY2m6ojkpT+gQNQfdaiWAYBUIoFCJum5agYI447iU4UJqT4WTWtPp5BCdemfQgO3Mh4Y+9NO7SoGqwMGqwPlTRZ8crICjg5jkaoMVtSb7JBLOc8Gj+EW1xqIWnyViLStjerGyojkpT+gQNQfeRqq2wLR3eMz8djsocj0pyGY44CcmcLjsu+EaoovjAFX9grz2aQyYMxPPSWw9lK1Kvx44kAoZBKUNpjxj5MVXgMj1XIp8lLjMSxNG5kdQ9AWiCx2HyUid3XUVCuMsia9RoGov+FdQtUM8PSYuXEc5/+k1eRhwh+kyyH0ovlSelSYLMtxwKhF3XbPZ+jVWNwajErqzfj7iXLUGKzgeYaEOAXuGp+JeaMit7FjnEJoPjX7CkQqvdBWxniaABsiFIj6G1O18F9crmpbED4YHAeMuAPgJG0N1+2VHweu/kt4PPQ2IGV4j6ccmKDG3eMzIZdyKGu04N0jN2BsVzWKRCO1m7tEZPbVRsRxVD0LMQpE/Y27WqYb5JlC8e31Bmz7+jq+b78ioT+06UDODOHxxU+BaweEibQXdgOXPheOZ09tW+XRD1lJcVgydTCGpGggk3Cw+QoEEaBRypAYJ0diXBdbWrurZ8aqyGUqhlH3fX/jo6G6zmhDvcneqZHYL4NnCG1EFSeAkq+Em1vOLcItQIkaBRZNEKadhGvxs54kxyu7nuYBtJWIDBWRyVCMo0DUnzAGNLvbh9oCkXtVxh7HEPnCccDw24HEwUJ1zGYUqnxZU4GE3i3lK1YQ8ou7RGSuF8ZlySK4kH8MokDUn1gahRUWJTLPf3TGGBo9gxm7qIb0xL3VT+t2P/2CUts2wtpU3eug299RG1F/4m4f0qYL3ekAWuwuz2TXiG7P0wfsO1+Nvx68igtVXewWQ9WzkKFA1J/4GD/k12TXfsrhYmixuTzbF3Xi/j020yYOvUWBqD/xNFS3VSPq/Z3s2g9pWzcQMPUYiMpo99deokDUX1ibWxdCk3iViGQSDslaJVK1Ii6tEaXc+6YZfU3zAISqmUQmbARpbohgzmIPNVb3F+6F8nUZnpUVAWFBs4CWdu1H3CUio9XhO4FEKiwW13RDKG12sbIk6RmViPoL9y6t/iyUTwAA8Z5A1M18svbVMxI0CkT9AWM+AxHPM/Dh2LM+RujVwnAGi93VeUcPTyJqsA4FCkT9galaaMeQygFd24qIJQ1m/OXLK9h9iuZL+aKUSZGuV2HwgDjYuxp1rm+dKmNpEtrhSFCojag/qL8i3CfmCO0a7sMmG5ytW0UT3/5zSnb3CWRKodHaUCG0w2VOiEi+Yg2ViPqD2ovCfbL3DHjPPvLUdd87SUOE+8ZicfPRh1EginWWxtb97SXAgDyvp2qNQiBK1tJOFD1xddeWltQ6ObbxOsAHudNtP0eBKNbVXhLuE7KFxbxa2Z28ZzBjmo7GEHWltMGMvx68ig++66YxWpspVNEcVsBEy4IEgwJRLGMMqDolPE4Z4fVUjdEKxoSxMu6Be6QzlVyKFpsLDWZ711ukSyTC6gMAUH81cpmLIRSIYpmhQli0XioT9q1vp9pgBQCkUmmoW0kaBaQSDjYHD4Olm/FE7mpv3cXIZCzGUCCKZRUnhPuUkV6jqQFAr1YgLzUeg5PifLyQuEklHAbEC435tSZr1wmThwvtcKZamu4RBApEscrSBFSfFR5nTur0dF5qPO4an+lzj3riLSVeaMyvMdi6TiRXt1XPaqlUFCgKRLHqxjfCLhOJOYDev22diW/u6muNsZtABLS1w9WeD3OOYg8FolhkrAYqvxceuxe3b6fJbEdTd42vxEtq6/CGaoO1+9+Zu3pmrBaqaMRvFIhijcsJXNwt9JiljhK67Ts4WtyAwsPXcaSY2jL8kapVIjNBhZsydXB2N55IoQEGDBUed9xeiXSLAlEsYQy4uk/4jyxXC/uJdUrCcKNB2K8+nXrM/CKTSnBffjZmDkuBXNrDn0zmROG++jTtAhsACkSxpOQrYScNABh1F6DSdUpSa7LBaHVCJuGQmaCOcAb7gcRc4ffusALVZ8TOTZ9BgSgW8Lyw7XPxQeHnvLltVYQOrtQI+9QPTtZAIaOPPxAOF48rNUbYnd1M45BIgIGThcc3vqEpH36ib2Jf57ACZ3cBpd8KPw+Z3eXOqowxTyDKS4mPUAZjx3vfluIfJytRXNfSfcLMicK4LUsjUHMuMpnr4ygQ9WWGCuBYIVB3WVg7efRiYPC0LpNXGayoN9khk3AYkqKJXD5jRG6y8Ds7U97DukMyhbDBJABc2y9swEi6RYGoL2IMKD0KnHhHGLio0gMTl/S4weHlaqE0NDxdC5Vc2m1a0tnYQXpIOA43GsyoMXYzyhoABuULn4vNCJQcikwG+zAKRH2NzQic/gC4sg/gXcIgusnLhUXce3BLXjIWjEnHxOyE8OczBulUcgxPE6q0By/VdT+mSCoHhv1QeFx6lCbD9oACUV/BGFB5Cjj6lvCllsiA4fOB0T/uNI+sKxIJh1EZOto6qBem5yVDJuFQ2mDGqbIeqmjJw4T2IsaA858IwyqIT1ERiP7yl78gJycHKpUKU6dOxdGjR8XOUnRpLge+fxe4sBtw2oQto28uAAbeDH/WeT1b0QyLvYvF30lA9Go5pucJ2wbtv1iLiiZL9y/ImyeUVh1W4OSOtk0MiBfRA9F7772HNWvW4JlnnsHx48cxfvx43H777aipqRE7a+LieaDuCnDqfeD4NqCpVCgFDZkNTFoKxKf0eAqrw4V956ux52w13vv2BmxOCkahMCk7EaMytBiUqO55UTmpDBh3n7CfnMMCnNwJXNojVLGJB8dEnnA0depU5Ofn489//jMAgOd5ZGVl4Ze//CWefvrpbl9rMBig1+vR3NwMna7z4L0+gTHAZQfsLcIuEC21rQuxXxNKP4BQ6kkfC+TcIjSAdsFsd6LeZEd9ix2VTRZcq2vxjHm5ZVgy8nOSInFF/QLPMzh55hmLZbQ68MX5amTo1dCr5dAoZOA4YRkRjgMSlYDq+r+AylNwuHjYXACfMBhIzAGnS4NUnQipWgeZTA6pJHZ2M/D3b1TUpfnsdjuOHTuGdevWeY5JJBLMmzcPX3/9daf0NpsNNlvbDGiDweDfGzUUCwP+fPHEYdbp2LU6E+pbF5j33ttceDw+KwGK1i9NSX2LZ7ExrvV51u514wbpoJIJPVWljWZUthbpJcwJCXN5vcWoDC3iFDJArsZlyWAcNg+GrVoPVl0HoM4r7T2TBnr+K58ua8ZXV+u9Li85XoFbh6dg8ADqrg8liYTzfPYAcLq8GdfrzLheZ/aZftGETAwZuRBIG43qE3tQWnIJKGsE8L1XOsZJkJeRhNQELcBJUN9ix+WaFnDuKjjHQfiGCT8PSdF4JuU2tDhwubrrklZOsgZpOiFts8WBC1Vdp81KikOmXvheGa1OnKts+1urSsrH/FtvhVoRup5XUQNRXV0dXC4X0tLSvI6npaXhwoULndJv2LABzz33XOBv5LQJKxUGiJlNcLZ0s/SDVQK0zj3irS1wWbvp0rXLASZ8cMxugcsuBCJ3ZcnFyeGQxsEsT8TQrNGIy8wDdANRf70RjVfrgS7aePh2AVKjlCEhTo4kjQLJ8UrkJGuQqVe1fYlJ2IxM10Etl6LaYIPR6oDV4QLPhM+HMbTNUUvMQctN/4nL/CXoLSVQmyuhtDdC5jBCwlzgGA+ZywZYhc+VN9vAzCZ0VW3h1PEA17r5gdkO3tR1cGEqDSBprUpaHOCN3fwjV8QB0tYpQDbvtCa5EazLHAWnTy1WvG7dOqxZs8bzs8FgQFZWVs8vTMgCxt8vPPb8Ubb74+ziWFKLHSq7q8N/o7bnpTqVcIzjkGSxQ+Vg7c7Atb2EA2QaJSAVAlGSzQGVo3XoPycD5HFCd2/ryzRxCk+AGztQj6Gto6C93r71Xte6GylA+9iLKUmjQJKf2zKNSNdiRPrNAG72HGM8D6fdApfDDhlzAHACjEHrcCLXbAfvnirCGNqX3jVqOdBaMolz8Bhs7nrwZLxaBiiEP3mVk0e2qYe0rWuZK5w8soe3pR2kToJSFtpxaKIGouTkZEilUlRXe3drVldXIz09vVN6pVIJpTKIrW8UmrYtXwKQqAcS/Uyr1wL+hoB4DeDvBAuNUgYNLW4f8ziJBHKVBnKVdxVaBaDzX4JvKgA9jyYTKAEMTA192mCJ2mumUChw8803Y9++fZ5jPM9j3759mDat66kKhJDYIvq/2jVr1mDp0qWYPHkypkyZgk2bNqGlpQXLli0TO2uEkAgRPRDdd999qK2txfr161FVVYUJEybgs88+69SATQiJXaKPI+qNmBhHREgM8/dvVPSR1YQQQoGIECI6CkSEENGJ3ljdG+7mLb+nehBCIsr9t9lTU3SfDkRGozCc3a/R1YQQ0RiNRuj1XQ/57dO9ZjzPo6KiAlqtNirnU7mnoJSWlsZcrx5dW98U6WtjjMFoNCIzMxMSSdctQX26RCSRSDBo0CCxs9EjnU4Xc19oN7q2vimS19ZdSciNGqsJIaKjQEQIER0FojBSKpV45plnglsxIMrRtfVN0XptfbqxmhASG6hERAgRHQUiQojoKBARQkRHgYgQIjoKRAEKdFfaDz74ACNHjoRKpcLYsWPxf//3f17PM8awfv16ZGRkQK1WY968ebh8+XI4L6FLgVzbW2+9hZkzZyIxMRGJiYmYN29ep/QFBQXgOM7rtmDBgnBfhk+BXFtRUVGnfKtU3hsp9tXPbfbs2Z2ujeM4LFy40JNGlM+NEb/t3LmTKRQKtmXLFnb27Fn2yCOPsISEBFZdXe0z/eHDh5lUKmUvv/wyO3fuHPvtb3/L5HI5O336tCfNxo0bmV6vZx999BE7efIku/vuu1lubi6zWCyRuizGWODX9sADD7C//OUv7MSJE+z8+fOsoKCA6fV6VlZW5kmzdOlStmDBAlZZWem5NTQ0ROqSPAK9tsLCQqbT6bzyXVVV5ZWmr35u9fX1Xtd15swZJpVKWWFhoSeNGJ8bBaIATJkyha1atcrzs8vlYpmZmWzDhg0+0//Hf/wHW7hwodexqVOnspUrVzLGGON5nqWnp7P/+Z//8Tzf1NTElEol27FjRxiuoGuBXltHTqeTabVatnXrVs+xpUuXskWLFoU6qwEL9NoKCwuZXq/v8nyx9Lm9/vrrTKvVMpPJ5DkmxudGVTM/uXelnTdvnudYd7vSAsDXX3/tlR4Abr/9dk/64uJiVFVVeaXR6/WYOnVql+cMh2CurSOz2QyHw4GkJO9trffv34/U1FSMGDECv/jFL1BfX9/FGcIj2GszmUwYPHgwsrKysGjRIpw9e9bzXCx9bm+//Tbuv/9+aDTe2xhF+nOjQOSn7nalraqq8vmaqqqqbtO77wM5ZzgEc20dPfXUU8jMzPT6o1iwYAG2bduGffv24aWXXsKBAwdwxx13wOXyvWttOARzbSNGjMCWLVvw8ccf45133gHP85g+fTrKysoAxM7ndvToUZw5cwYrVqzwOi7G59anZ9+T6LBx40bs3LkT+/fv92rUvf/++z2Px44di3HjxmHo0KHYv38/5s6dK0ZW/TJt2jSvffWmT5+OUaNG4c0338QLL7wgYs5C6+2338bYsWMxZcoUr+NifG5UIvJToLvSAkB6enq36d33gZwzHIK5NrdXXnkFGzduxJ49ezBu3Lhu0w4ZMgTJycm4cuVKr/Psr95cm5tcLsfEiRM9+Y6Fz62lpQU7d+7Eww8/3OP7ROJzo0Dkp2B2pZ02bZpXegDYu3evJ31ubi7S09O90hgMBhw5ciSiO90Gu+Puyy+/jBdeeAGfffYZJk+e3OP7lJWVob6+HhkZGSHJtz9CsZuwy+XC6dOnPfnu658bIAwrsdls+NnPftbj+0Tkc4to03gft3PnTqZUKllRURE7d+4ce/TRR1lCQoKna/fBBx9kTz/9tCf94cOHmUwmY6+88go7f/48e+aZZ3x23yckJLCPP/6YnTp1ii1atEi0buBArm3jxo1MoVCwDz/80Kub12g0MsYYMxqNbO3atezrr79mxcXF7IsvvmCTJk1iw4YNY1arNaqv7bnnnmOff/45u3r1Kjt27Bi7//77mUqlYmfPnvW6/r74ubndcsst7L777ut0XKzPjQJRgP70pz+x7OxsplAo2JQpU9g333zjeW7WrFls6dKlXunff/99Nnz4cKZQKNjo0aPZ7t27vZ7neZ797ne/Y2lpaUypVLK5c+eyixcvRuJSOgnk2gYPHswAdLo988wzjDHGzGYzmz9/PktJSWFyuZwNHjyYPfLII53G40RKINf25JNPetKmpaWxO++8kx0/ftzrfH31c2OMsQsXLjAAbM+ePZ3OJdbnRsuAEEJER21EhBDRUSAihIiOAhEhRHQUiAghoqNARAgRHQUiQojoKBARQkRHgYgQIjoKRKRPKSoqQkJCgufnZ599FhMmTPD8XFBQgMWLF0c8X6R3KBARn9zrFv/85z/v9NyqVavAcRwKCgq80oc6AOTk5GDTpk1ex+677z5cunSpy9e88cYbKCoq8vw8e/ZsPPnkkyHNFwk9CkSkS1lZWdi5cycsFovnmNVqxfbt25GdnS1KntRqNVJTU7t8Xq/Xe5WYSN9AgYh0adKkScjKysKuXbs8x3bt2oXs7GxMnDixV+f2VVJZvHixp5Q1e/ZslJSU4Fe/+pVnJwmgc9Wso/Yls4KCAhw4cABvvPGG5xzFxcXIy8vDK6+84vW677//HhzHRXStJNKGAhHp1vLly1FYWOj5ecuWLVi2bFnY33fXrl0YNGgQnn/+eVRWVqKysjLgc7zxxhuYNm0aHnnkEc85srOzO10TABQWFuLWW29FXl5eqC6BBIACEenWz372Mxw6dAglJSUoKSnB4cOH/VpMq7eSkpIglUqh1WqRnp4e1MqHer0eCoUCcXFxnnNIpVIUFBTg4sWLnv2/HA4Htm/fjuXLl4f6MoifaM1q0q2UlBQsXLgQRUVFYIxh4cKFSE5OFjtbvZKZmYmFCxdiy5YtmDJlCv7xj3/AZrPh3nvvFTtr/RaViEiPli9fjqKiImzdujVkpQaJRIKOS2E5HI6QnNsfK1as8DTEFxYW4r777kNcXFzE3p94o0BEerRgwQLY7XY4HA7cfvvtITlnSkqKV7uPy+XCmTNnvNIoFIpeb2HT1TnuvPNOaDQabN68GZ999hlVy0RGVTPSI6lUivPnz3sed6W5uRnff/+917EBAwYgKyurU9rbbrsNa9aswe7duzF06FC89tpraGpq8kqTk5ODgwcP4v7774dSqQyqSpiTk4MjR47g+vXriI+PR1JSEiQSiaetaN26dRg2bFhEF70nnVGJiPhFp9NBp9N1m2b//v2YOHGi1+25557zmXb58uVYunQpHnroIcyaNQtDhgzBnDlzvNI8//zzuH79OoYOHYqUlJSg8r127VpIpVLcdNNNSElJwY0bNzzPPfzww7Db7RHpBSTdozWrSb/173//G3PnzkVpaWmn3VJJZFEgIv2OzWZDbW0tli5divT0dLz77rtiZ6nfo6oZ6Xd27NiBwYMHo6mpCS+//LLY2SGgEhEhJApQiYgQIjoKRIQQ0VEgIoSIjgIRIUR0FIgIIaKjQEQIER0FIkKI6CgQEUJE9/8DmwpE9pbng2sAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:52:10.087766Z", + "iopub.status.busy": "2024-03-22T20:52:10.087487Z", + "iopub.status.idle": "2024-03-22T20:52:10.406559Z", + "shell.execute_reply": "2024-03-22T20:52:10.405809Z" + }, + "papermill": { + "duration": 0.338846, + "end_time": "2024-03-22T20:52:10.408588", + "exception": false, + "start_time": "2024-03-22T20:52:10.069742", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAW0lEQVR4nO2deXxU5fX/P3f2ZDKZJGQnCUkI+44sBVxAqKhUobaKdSPg1krr1/KjVb5tUfRlwVat2PqirZXFryLVlqqtdUEUUFBAdhASCEkIZF9nX+/z++NmbjLJTDKZzMydTM779ZrXTO597p1zc2c+8zznOec8HGOMgSAIQkJkUhtAEARBQkQQhOSQEBEEITkkRARBSA4JEUEQkkNCRBCE5JAQEQQhOSREBEFIjkJqA/oDz/Oorq6GTqcDx3FSm0MQRBcYYzAajcjOzoZM5r/fM6CFqLq6Grm5uVKbQRBEL1RVVSEnJ8fv/gEtRDqdDoBwkYmJiRJbQxBEVwwGA3Jzc8Xvqj8GtBB5hmOJiYkkRAQRxfTmOiFnNUEQkkNCRBCE5JAQEQQhOQPaRxQIjDG4XC643W6pTRnUKJVKyOVyqc0gohRJhcjtduOpp57CG2+8gdraWmRnZ6O4uBi//vWvQxIX5HA4UFNTA4vFEgJrif7AcRxycnKQkJAgtSlEFCKpED333HPYtGkTtm3bhnHjxuGbb77B8uXLodfr8eijj/br3DzPo7y8HHK5HNnZ2VCpVBT0KBGMMTQ0NODy5csYMWIE9YyIbkgqRAcOHMDixYuxaNEiAEB+fj7eeustHDp0qN/ndjgc4Hkeubm5iI+P7/f5iP6RlpaGiooKOJ1OSYTIbHchXiWnH6MoRVJn9ezZs7F7926UlpYCAE6cOIEvv/wSN910k8/2drsdBoPB69EbPYWVE5FDagH44FQNjlW1gkq0RyeS9oieeOIJGAwGjB49GnK5HG63G88++yzuvvtun+3Xr1+PdevWRdhKYqBjsrtwpcWK2jYbxmYlQqOkoWG0IWl34e2338abb76J7du34+jRo9i2bRuef/55bNu2zWf7NWvWoK2tTXxUVVVF2GJiIFLbZgUAJGtV4BlDncEmsUVEVyTtEf3iF7/AE088gTvvvBMAMGHCBFRWVmL9+vVYtmxZt/ZqtRpqtTrSZkacuXPnYvLkyXjppZekNiUmqG2zAwDUchn+svciVAoZHpk7XPLhItGBpD0ii8XSzYcjl8vB87xEFg0s9uzZA47j0NraKrUpUU29UegBjczUQcZxcLh4mOwuia0iOiNpj+iWW27Bs88+i7y8PIwbNw7Hjh3Diy++iBUrVkhpFhFjtFqcAIDUBBWS4pVoNjvQbHZAp1FKbBnhQdIe0R//+Ef88Ic/xCOPPIIxY8Zg9erVePjhh/HMM8+E9X0dLt7vw+XmA27rDKBtf7Hb7Xj88ceRm5sLtVqNoqIivPbaa6ioqMC8efMAAMnJyeA4DsXFxb2ez2g04u6774ZWq0VWVhb+8Ic/YO7cuXjsscfENv/3f/+HadOmQafTITMzE3fddRfq6+vF/Z6e2O7duzFt2jTEx8dj9uzZKCkp6ff1hho3z2CwCUKUFK9CslYFAGhpFyciOpC0R6TT6fDSSy9F3BfyyucX/O4rSNViyZSh4t9/3VcGp9v3lG9Ochxun9ZRmG3z/nJYHd6pJD//7sh+2Xrffffhq6++wssvv4xJkyahvLwcjY2NyM3NxT//+U/84Ac/QElJCRITExEXF9fr+VatWoX9+/fj/fffR0ZGBtauXYujR49i8uTJYhun04lnnnkGo0aNQn19PVatWoXi4mL897//9TrXr371K7zwwgtIS0vDj3/8Y6xYsQL79+/v1/WGGrPDBZVCBp5n0Krk0GmEj7zJRkOzaCLmc80GMqWlpXj77bexa9cuLFiwAABQWFgo7k9JSQEApKenIykpqdfzGY1GbNu2Ddu3b8f8+fMBAFu2bEF2drZXu85D48LCQrz88suYPn06TCaTV4rGs88+i+uuuw6AEIqxaNEi2Gw2aDSa4C44DCRqlPjJdcNhd/HgOA6J7UJktFGPKJoYlEK0cl6R332yLhMpD1073G/brpMuK+YU9Mesbhw/fhxyuVz8sveXixcvwul0YsaMGeI2vV6PUaNGebU7cuQInnrqKZw4cQItLS3i5MGlS5cwduxYsd3EiRPF11lZWQCA+vp65OXlhcTeUMFxnBg7lKAW/EJG6hFFFYNSiFSKwF1j4WobCIEMtUKN2WzGwoULsXDhQrz55ptIS0vDpUuXsHDhQjgcDq+2SmWHs9czFR7tM55pOjWm5SdjiDb2w0AGEpT/EMVMmDABPM9j7969PverVILjNdASJ4WFhVAqlTh8+LC4ra2tTUyxAYBz586hqakJGzZswDXXXIPRo0d7OaoHGgcvNuG941dwscEEAEjRqnDNiDSMzabSwtEECVEUk5+fj2XLlmHFihV49913UV5ejj179uDtt98GAAwbNgwcx+E///kPGhoaYDKZejyfTqfDsmXL8Itf/AKff/45zpw5g/vvvx8ymUzs0eTl5UGlUuGPf/wjLl68iPfffz/ss5jhpKbNhosNZlgcVI8qmiEhinI2bdokhjiMHj0aDz74IMxmMwBg6NChWLduHZ544glkZGTgpz/9aa/ne/HFFzFr1ix873vfw4IFCzBnzhyMGTNGdDCnpaVh69ateOeddzB27Fhs2LABzz//fFivMZwY2wMXE9QdXgiDzYmaNivsLhKnaIFjAzgd2WAwQK/Xo62trdsqHjabDeXl5SgoKIiqWZxow2w2Y+jQoXjhhRdw//33h+19pLoff95bBqvDjXu+MwxpOsEvtHV/OVosTtw+LQc5yVQiJpz09B3tzKB0Vg9mjh07hnPnzmHGjBloa2vD008/DQBYvHixxJaFHpebF+O6PPFDABCvUqDF4uwW80VIBwlRDNF1er0r3377LQDg+eefR0lJCVQqFa666ip88cUXSE1NjZSZEcPcLjRyGQd1pxnNOJUwlU9+o+iBhCiGyM7OxvHjx3vcn5eXhyNHjkTOKAmxOQWh6VqZMZ6EKOogIYohFAoFior8B2sONhwuHiqFrFshNE+PyOqkoMZogYSIiFlyU+Kxcl4R3Lz3fEy8SvjYU48oeqDpeyLmkXfJ26GhWfRBPSJi0JGaoMaMghQkxVM9omiBhIiIWQ5ebEJNmw0TcvQYntZRNSBFq8KcotibJRzI0NCMiFlqDTaUN5phsdMQLNohISJiFo8PyDNL5oExhlaLA7VtNvD8gE0siClIiKKQrqVbpSba7AkUT+R0vKr7OmZbD1TgrUOXYHVSbykaICGKUbrWDhqMeEQmrkscEcdxUCuEbTYSoqhgcAkRY4DLEflHH/KKi4uLsXfvXmzcuBEcx4HjOJSVleH+++9HQUEB4uLiMGrUKGzcuLHbcUuWLMGzzz6L7OxsserigQMHMHnyZGg0GkybNg3vvvsuOI7zisA+ffo0brrpJiQkJCAjIwP33nsvGhsb/dpTUVHR71sRblzujsULug7NAIgpH/YQLHBA9J/BNWvmdgJfvBD5973m/wEKVUBNN27ciNLSUowfP15MSE1OTkZOTg7eeecdDBkyBAcOHMBDDz2ErKws3HHHHeKxu3fvRmJiInbt2gVAyHy+5ZZbcPPNN2P79u2orKzsNsRqbW3F9ddfjwceeAB/+MMfYLVa8fjjj+OOO+7AZ5995tOetLS0EPxTwounNyTjvPPMPGiUcrRZndQjihIGlxANAPR6PVQqFeLj45GZmSluX7dunfi6oKAAX331Fd5++20vIdJqtfjb3/4mVm7885//DI7j8Oqrr0Kj0WDs2LG4cuUKHnzwQfGYP/3pT5gyZQp++9vfits2b96M3NxclJaWYuTIkT7tiXacbgaVQga5jPO5oiv1iKKLwSVEcqXQO5HiffvJK6+8gs2bN+PSpUuwWq1wOBxeSwABQmlZjwgBQElJCSZOnOhV/6dz4XwAOHHiBD7//HOv1Tk8lJWVYeTI/i2HJBUpWhVWzivyOyvmyT+jHlF0MLiEiOMCHiJFEzt27MDq1avxwgsvYNasWdDpdPj973+PgwcPerXTarV9PrfJZMItt9yC5557rts+z8ocAxlZ12VZ2vH0iGxO6hFFA4NLiAYIKpXKqyD+/v37MXv2bDzyyCPitrKysl7PM2rUKLzxxhuw2+1Qq4XqhJ0L5wPA1KlT8c9//hP5+flQKHx/HLraEwvkp2oRp5IjJznyK6UQ3Rlcs2YDhPz8fBw8eBAVFRVobGzEiBEj8M033+Djjz9GaWkpfvOb33QTFF/cdddd4HkeDz30EM6ePYuPP/5YrD/t8ZusXLkSzc3N+NGPfoTDhw+jrKwMH3/8MZYvXy6KT1d7on3JIAAorTPi3WNXcOxSi8/9RekJmFOUitwUKhUbDZAQRSGrV6+GXC7H2LFjkZaWhoULF+K2227D0qVLMXPmTDQ1NXn1jvyRmJiIf//73zh+/DgmT56MX/3qV1i7di0AiH6j7Oxs7N+/H263GzfccAMmTJiAxx57DElJSZDJZD7tuXTpUvguPkQ0mRwobzSj2UzxVAMBKp4/yHjzzTexfPlytLW1RXQBx0jfjz0l9Th2qRXT81Nw9YjuCa5ONw+TzQUGwbFNhAcqnk8AAF5//XUUFhZi6NChOHHihBgjJMUqspHEMy2vVvru9Fe3WrHz6BWk6tS49zvDImka4QMSohintrYWa9euRW1tLbKysnD77bfj2WefldqssOOZltcoukdVAx3LgzsojigqICGKcX75y1/il7/8pdRmRJzeekQqOQlRNEHOaiImEYXIR3oH4N0jGsBu0pgh5oWIPmTRQaTvgyeiuusKHh482fc8Y3BRTSLJidmhmVIppFVYLJaYd8wOBDxlSeRy38IQapbNzoebZ/AdVw0o5Rw4TiiM4HDxUMpj/jc5qolZIZLL5UhKSkJ9fT0AID4+3mfyIxF+eJ5HQ0MD4uPj/UZvh4Ouq3d0huM4qBQy2J087C4eWnXEzCJ8ELNCBEDMFveIESEdMpkMeXl5UfVjMDknCQwd/iJCOmJaiDiOQ1ZWFtLT0+F0OqU2Z1CjUqnESO1wY7A58dnZemjVCnx3bIbfdrNpJY+oIaaFyINcLo+Yb4KQHrPdhfJGM3SaQfHxjgmoT0rEHPb20h7+Zsw82JxutFocVJMoCiAhImKO3mKIPOw+W48t+ytwtsYQCbOIHiAhImIOTw9H3UuPSE1pHlGD5EJ05coV3HPPPRgyZAji4uIwYcIEfPPNN1KbRQxgPD0iTS89IjG62k1CJDWSevNaWlowZ84czJs3Dx9++CHS0tJw/vx5JCcnS2kWMcCxuwLrEVHia/QgqRA999xzyM3NxZYtW8RtBQUFElpExAIut5Cy0ZuPSEUreUQNkg7N3n//fUybNg2333470tPTMWXKFLz66qtSmkTEAPNGp+PR+SMwbVjPPWvKwI8eJBWiixcvYtOmTRgxYgQ+/vhj/OQnP8Gjjz6Kbdu2+Wxvt9thMBi8HgThC7mMg6KX/DFyVkcPkpaKValUmDZtGg4cOCBue/TRR3H48GF89dVX3do/9dRTXgsNeuitDCVB+KLBaMepK61IjldhSh75JcNBoKViJe0RZWVlYezYsV7bxowZ47c4+5o1a9DW1iY+qqqqImEmMcD45EwtPjxVgzZLz2k9aTo1rh+dQSIUBUjqrJ4zZw5KSkq8tpWWlmLYMN81hNVqtbg+F0H4o6zBDJvTjRkFKVKbQgSIpD2in//85/j666/x29/+FhcuXMD27dvx17/+FStXrpTSLGIAwxgLePqeMQaLw4VWi4MK6EmMpEI0ffp0/Otf/8Jbb72F8ePH45lnnsFLL72Eu+++W0qziAGM3cXDoym9BTQ63Dz+svcituyvoCqNEiN5evL3vvc9fO9735PaDCJG8MQEKQKYNVN12u90U5VGKaH/PBFT2MU8s94/2p4qjQBN4UsNCRERU4h5Zr34hzxQUGN0QEJExBSBlgDxQGke0YHkPiKCCCVF6Ql4dP4IuPjAhMUjRE7KwJcUEiIi5pDLOMhlfRyakRBJCgkRMagpSNMiWauEPk4ptSmDGhIiIqY4UdWK6lYrRmXqUJiW0Gv7qZTeERWQs5qIKapbrThXa0RLL3lmRHRBPSIipujrrJmbZ3C4eHBc4FP+ROihHhERU3jyzDQBBDQCwJHKFvx5bxn2lTaE0yyiF0iIiJjC5vT0iAKcNaMC+lEBCRERU3Rk3gcY0CinOKJogISIiCnswfaIKLJaUkiIiJjB5ebFch6B+ogo1yw6oFkzImZQyGV4dP4I2F1urxIfPUG5ZtEBCRERU8hlHOJVgX+sO3LNqDCalJAQEYOaOKUcY7J0AfuUiPBAQkTEDLVtNhy71II0nRrT8gMrnB+nkuPG8VlhtozoDXJWEzFDs9mBc7VGVDZZpDaF6CMkRETM0BFV3bdhlptnsDrccFMBfckgISJihr7mmXnY/GU5/ry3DE0mezjMIgKAhIiIGUQhCjCGyANN4UtPUEJ08eLFUNtBEP3G5gxuaEb5ZtITlBAVFRVh3rx5eOONN2Cz2UJtE0EERbBDM8o3k56ghOjo0aOYOHEiVq1ahczMTDz88MM4dOhQqG0jiD7h6RH1NSZISflmkhOUEE2ePBkbN25EdXU1Nm/ejJqaGlx99dUYP348XnzxRTQ0UG0XIvL8YGoOHr6uEIVp2j4dR/lm0tMvZ7VCocBtt92Gd955B8899xwuXLiA1atXIzc3F/fddx9qampCZSdB9IonvaOvS0erqUckOf0Som+++QaPPPIIsrKy8OKLL2L16tUoKyvDrl27UF1djcWLF4fKToIIDdYWwFgHsI6YoYxEDUZn6pCqU0to2OCGY4z1OYrrxRdfxJYtW1BSUoKbb74ZDzzwAG6++WbIZB26dvnyZeTn58PlcoXU4M4YDAbo9Xq0tbUhMTExbO9DRD9unuGTM7VQK2W4dkQaFF17RTwPlH4I1JwU/k7MBib8EFD1bRhH9I1Av6NB5Zpt2rQJK1asQHFxMbKyfOfppKen47XXXgvm9ATRZ+wuN87VGgEAc0emd29w8fMOEeJkgKEaOPMuMPkugOMiZyjhk6CEaNeuXcjLy/PqAQEAYwxVVVXIy8uDSqXCsmXLQmIkQfSGWKtaKYNM1kVYjHXA5cPC63HfB7SpwJGtQOsloOEckD4Gbp7BzTMxpoiILEH914cPH47GxsZu25ubm1FQUNBvowiir4i1qn1N3Vd+KfiE0kcLD20qkPcdYV/Fl7hQZ8TLu89j59HLEbSY6ExQQuTPrWQymaDRaPplEEEEg6dH1K1ErLUFaDwvvM6/pmP70GmAXAmYGxFnFWZ3KaBROvo0NFu1ahUAgOM4rF27FvHx8eI+t9uNgwcPYvLkySE1kCACwW+PqPqY0BtKKRR6Qh6UGiB9DFBzEvHNZwCMo1wzCemTEB07dgyA0CM6deoUVCqVuE+lUmHSpElYvXp1aC0kiADw2SNiDKg/K7zOntz9oIzxQM1JqFvLADaGcs0kpE9C9PnnnwMAli9fjo0bN9KUORE12H2ldxiuADaDMARLKex+kD4XUGqgsFmgc9TDJMsEYwwczaJFnKBmzbZs2RJqOwiiX0zLT8GEHD04dBKR+nPCc+pIQYy6IpMBKYWQ1ZxGsrUSRnUmXDyDUk5CFGkCFqLbbrsNW7duRWJiIm677bYe2+7cubPfhhFEX+i2egdjQGOJ8Dp9jP8DhxRBXncGybYqXMJMOFx8n1NEiP4TsBDp9Xqxy6rX68NmEEGEBEuzMCyTKYDkfP/tkgvAgUOO0ghHqoJiGyUiYCHqPByjoRkRbXx9sQkWhwsThiYhTacGWsqFHfoc38MyD6p4QJuKkRnAyBwX0Ic10YjQEVQf1Gq1wmLpWCmhsrISL730Ej755JOQGUYQfeF8vQknqtpgcbTnNja3C1FKAAG2+lzhua0qPMYRvRKUEC1evBivv/46AKC1tRUzZszACy+8gMWLF2PTpk0hNZAgAsFr1sztAlorhB3JgQhRDgCAb60CTyt5SELQFRqvuUaIUv3HP/6BzMxMVFZW4vXXX8fLL78clCEbNmwAx3F47LHHgjqeGNx4ghE1Spkwbe92CZn1CT4SYLuSlItztQZ8feIMSqqbw2wp4YughMhisUCn0wEAPvnkE9x2222QyWT4zne+g8rKyj6f7/Dhw/jLX/6CiRMnBmMOMchx80wsaqZRygUhAoCk3MAy69WJ4BXx4MDATPVhtJTwR9DF8999911UVVXh448/xg033AAAqK+v73OQo8lkwt13341XX30VycnJwZhDDHI86R1Ae9lXQ7XwR2JOYCfgODi1GcJLI1UVlYKghGjt2rVYvXo18vPzMXPmTMyaNQuA0DuaMmVKn861cuVKLFq0CAsWLOi1rd1uh8Fg8HoQhFcJEA4dPaLE7IDPwWuFulqcqTbU5hEBENRc5Q9/+ENcffXVqKmpwaRJk8Tt8+fPx/e///2Az7Njxw4cPXoUhw8fDqj9+vXrsW7duj7bS8Q2XgmvtlbAYQFkciAhI+Bz8LpMAABnrguHiUQvBB00kZmZiczMTK9tM2bMCPj4qqoq/M///A927doVcOmQNWvWiBUAAKEMZW5ubsDvScQmGToNHr6uEC6eAW2lwsaEDEDeh4+3ThAtuaUJcDkAhaqXA4hQEpQQmc1mbNiwAbt370Z9fT143jtrOZCVYI8cOYL6+npMnTpV3OZ2u7Fv3z786U9/gt1uh1zuXdJBrVZDraYC54Q3ss7pHVUe/9DQPp1DrkmEQx4PnncBpjrB0U1EjKCE6IEHHsDevXtx7733IisrK6hs5fnz5+PUqVNe25YvX47Ro0fj8ccf7yZCBBEQQfiHACApTglXSja07mrAXE9CFGGCEqIPP/wQH3zwAebMmRP0G+t0OowfP95rm1arxZAhQ7ptJ4ieuFBvRFWzFXlJSgw3tvt4+ihE+ala5I8bBVxqAczdyyAT4SUoIUpOTkZKSkqobSGIoLjcYsXxqlYk2G0YznghkFETRGK2Nk14pliiiBOUED3zzDNYu3Yttm3b5lUutr/s2bMnZOciBg+eNe91jnYBScwObomghHQwMHDmeqGMCKXiR4yghOiFF15AWVkZMjIykJ+fD6XSO7v56NGjITGOIALB2i5EWrtnWNY3RzUAtFocePNgM6ZXtWHGsCTA1gbEJYXOSKJHghKiJUuWhNgMgggeq0OYtY231gmf6D76hwBAIZfBwctgkie194oaSYgiSFBC9OSTT4baDoIIGqvTDZXLBBVnApQqQOd79eGeULVXZTQrksGzZsjN9UBqUahNJfwQdE3M1tZW/O1vf8OaNWvQ3CxkLB89ehRXrlwJmXEEEQg2pxsJjgYoZTLB4RxEMKJSzoHjAIsqBW6ekcM6wgTVIzp58iQWLFgAvV6PiooKPPjgg0hJScHOnTtx6dIlsVYRQYQbl5uHw8UjwV4HRQIXlH8IENbqU8plsCjbhcjcEGJLiZ4Iqke0atUqFBcX4/z5817pGTfffDP27dsXMuMIojfkMg4/vm44Fua5IZdxQfmHPKgVghDxPBNqXvPu3g8iQkJQQnT48GE8/PDD3bYPHToUtbWUvUxEDo7jEKcAEuyNwlJCQfaIAEClkMEh18LJKQDGA9bW0BlK9EhQQqRWq32W4CgtLUVaWlq/jSKIPmGqB3gXoFAD8cEH2mbr41CQlgCZdoiwwUIR1pEiKCG69dZb8fTTT8PpdAIQfpUuXbqExx9/HD/4wQ9CaiBB9ER1qxWHT51CvdEm9Ib6EYS4YGwGlkwZisSU9qoSlqYQWUn0RlBC9MILL8BkMiEtLQ1WqxXXXXcdioqKoNPp8Oyzz4baRoLwS4PRjtqqi2ixOPvlH/JCmyo8U85ZxAhq1kyv12PXrl3Yv38/Tpw4AZPJhKlTpwZUZZEgQonV6UaCvR6KuP45qr2Ip6FZpOmzEPE8j61bt2Lnzp2oqKgAx3EoKChAZmYmGGNBlQQhiGCxW03QuAxQyuP6LUQHLjTiWFUrpqdxmAEIQzPKOYsIfRqaMcZw66234oEHHsCVK1cwYcIEjBs3DpWVlSguLu5TmViCCAWcp/5QfAqgjOvXuRgAh4uHWZEolJp1u4ScMyLs9KlHtHXrVuzbtw+7d+/GvHnzvPZ99tlnWLJkCV5//XXcd999ITWSIPzBGdsrMuqDn7b3oFIIv8sON4C4ZMFHZGminLMI0Kce0VtvvYX//d//7SZCAHD99dfjiSeewJtvvhky4wiiNxTtq27IQiFE7flmTjffyU9EM2eRoE9CdPLkSdx4441+99900004ceJEv40iiIDgeSjMwjpkyuQA1zDrAWW7EDlcJESRpk9Ds+bmZmRk+F+iJSMjAy0tLf02iiACwtKEKdnxcDAdlOkhHJq5eJrCjzB9EiK32w2Fwv8hcrkcLper30YRREAYrkAhk0GRlAv08LkMFLXoI+K9p/Bp5izs9OnuMcZQXFzsd0kfu90eEqMIIiCCXLHDH3EqObKTNNDHqQQh4jjAaQOcFqEONhE2+iREy5Yt67UNzZgRkcJUX4n6RhPsqYkYE4LzpSaosXR6XscGdaIwfW9uJCEKM30Soi1btoTLDoLoG04r7IZ61BnsqDbrQiJE3dCmCkJkaQKSh4XjHYh2gq7QSBCSYqiG081gU+ihjk8Iz3vQzFnE6L+HjyCkoO0ynC4eRnUGElShWRWYMYbXviyH3cWjeHY+tCREEYN6RMTAxHAFDrcgROK69/2E4zjYXULpWZrCjywkRMTAg+cBwxU43TyMqgxoQyREgJ8pfLsRcNGMcDghISIGHuZ6wO2CnSlhVSYjLkRDM6BLUKMyrmO2jIZnYYWEiAgYo80ptQkCbUL8UIsiDeA4aNWhEyIxzcMtLNoo9opoeBZWSIiIgGi1OLDtQAXO1xmlNgVorQQAzLpqMopn5yM1wXeAbTCoOuebAR1+IuoRhRUSIiIgSutMcLoZdp+rh80p4TI7jAGtlwAA6iH5SNaqxF5MKPAamgFAPAlRJCAhIvxS1WzBycutsDrcmDYsGSlaFawON05elrBYmLkBcFoBuSJ0pWE7MUSrQnaSpsPv5FkVhIQorJAQEX45VtWK3WfrcaSyBTIZh2n5yQCA01fawBiTxqgWYVjWqsrCp+cacfpKaEVxdlEqlk7Pw8gMnbDBMzSztggVG4mwQEJE+MTh4lHRaAYAjMoUvpQjM3RQK2VoszpxucUqjWHt/qEmRQZOXWlDWYMpvO+nSgAUKmFIaG0O73sNYkiICJ9Ut1rh5hl0GgVSE1QAhBmlojQhneJCfZgFwBc8L/qHWtRZAIAEdZiTAziO/EQRgISI8Imnx5ObEu+1MktRuiBEZQ2myA/PTLVCYKFChWZO8N2EWojO1Rrwty8u4qPTnZZOpyn8sEO5ZoRPLrdYAAA5yd4rY+SlxGPCUD0K07SRrxfWVCY8JxfA5BBmtRI0of0IMwYYbS6Y7Z38QTSFH3ZIiIhuON086gxCSkNOcrzXPoVchgVj/ZcLDitNF4TnIUUwVQhCoVMrQ/oWGqUwW2b3TN8DnYZm1CMKFzQ0I7rRbHaAZwxatRyJIe5xBI3dCBhrhS5YSiGMNkGIQt0j0iiFr4S1c6yUOIXfIvipiJATJZ8yIprISNTgJ3OHw2Bz+ly5lzGG6jYbKpvMmJqXLPYiwopnWKbLgk0WJwYchtpHpFEI1+IVtKlJAmQKgHcBttYOYSJCBgkR0YHbBZTtBurOQKPWQVM0H0Bht2Ycx+HTb+vQbHYgXadGUbou/LY1lgrPQ4qgUcqxcl4RjDanGAkdKjyi6nDxcPMMchkHyGRAfDJgagAszSREYYCGZkQHJR8AV44KM1PmRuDUP4CWCp9N84YIvqPKJkv47XJYgOZy4XXaaABCKsaQEOaYeVB3Eja7q/PwjPxE4YSEiBBoKgPqvgUPDp9yM/GtOxtutws491/A5ejWfFiKIEQVTZbwT+M3nAMYD+gyAO2QsL6VTMYhU69BdpIGbr7TdVGRtLAiqRCtX78e06dPh06nQ3p6OpYsWYKSkhIpTRqcMAZUfAkAMKROwilnDvbJZkAWpxeKx18+1O2QnOR4yGUcDFYnWi1hLg9Sd0Z4Th8HADh6qQWfflsnhhiEmh/NyMPS6XnQaTrNyFHZ2LAiqRDt3bsXK1euxNdff41du3bB6XTihhtugNlsltKswYexBjBUAzIF6vWTAAApugRwhfOE/ZcPd6tQqFLIkJ0kxBiVN4XxfpkagLbLwmxZurBWR3mDGaeutKHNGsH6SJ2HZlLl2cUwkjqrP/roI6+/t27divT0dBw5cgTXXnutRFYNQuq+FZ5TR6DRKaRzJGtVgj8m/kuhF1B9HMib6XVYQaoWVc0WlDcIs2dh4coR0TZoEgFAFKCkeFV43tMXccmCGLocgMMEqCPgoB9ERJWPqK1NyKROSfE9K2G322EwGLweRD/heaC+feiTMR7NZsEflKJVCbNFuTOEfTXHu/UEClKFMqrNZoe3PyVUOK1A3Wnh9dBpAAA3z8QYonDFOH1xvgF/++IiTlS1dmyUKwQxAshPFAaiRoh4nsdjjz2GOXPmYPz48T7brF+/Hnq9Xnzk5uZG2MoYpK1KmJVSxgEpBWjpLEQAkD5WyD63NHebQUuOV+JHM/Jw/9UFwjR3qKk6CLidQEIakCSswNpiEYItVQpZ2BJenW5eSPNwdCn7QTlnYSNqhGjlypU4ffo0duzY4bfNmjVr0NbWJj6qqqoiaGGM0nxReE4pBA8ZWtodz6IQKVRARvsPQ/Uxr0M5TphhkoVDhGwGwTcFAAXXiUltnh7bEK3KZ7BlKPAENdqdXaKoE9KFZ3NDWN53MBMVAY0//elP8Z///Af79u1DTk6O33ZqtRpqdehjRwY1nYTI5HBBKZdBxvHew57sKUJ8UeN5wGH2uQ48zzNwHEIjDowB5z8RAiz1Q4EhReKuRpPgNA9HDJEHTXt1RmvXkrhajxDVh+29ByuSChFjDD/72c/wr3/9C3v27EFBQYGU5gw+7EbAVN+ev1WARJUSP76uEHYX7y0oCelCWVZDNVB7upvT+kCZUCnx5glZ3ZJkg+LyN4LoyeTAyJu8UvwtdkEcxB5bGPCZ5gF494h4XvChESFBUiFauXIltm/fjvfeew86nQ61tUINGL1ej7i4uF6OJvqNJ1o5IUPs5XAc5zt3LGuSIEQ1JwQHdidxMNlcMNvdOFdj7L8Q1ZwU0kwAoHCu4B/qxIKxGbhmZGr/3qMXPImvtq5DM02S4LR2U85ZqJFU0jdt2oS2tjbMnTsXWVlZ4uPvf/+7lGYNHtrafWzJ+b23TR8DyJXCVH6bt29uTJYwrV5ab4TLHWR2uqleSCk594EwNMueAuRM99lUrZBDrQhfoq1HiLv1iGQyQJvWYS8RMiQfmhES0nZZeNYLs48fnKyBi+cxpyi1+1phCrUwg1ZzQni0z2IBQvE0nUYBo82Fc7VGjB+qD9wGawtQ/gVQ/y3ESmt53/FyUEeaeJUcyfFK33FK2nTAUNPuJxodcdtilahwVhMSYDcJU/IcB+iHgjGGiiYzHC5BiHySNUkQoYZzQNF3AaUGgDCcm5KXhH2ljfimohljsxJ7n0lz2oS0kitHhDwyAEgbJQiQn3yy41WtKK0zYny2HmOzE4O98l5JileheI4ffyX1iMICCdFgxdMb0qYCyjiY7S44XDxkHIdkfxHLidlCe3OjEAQ59Cpx1/ihehwqb0GLxYlTV9owKTep2+GMMTSaHDDXlyOl8r/QwSY4xVMKBAFKzOrR5MstFlxpsaIwtfusXcTw+KxoCj+kkBANVroMyzyBjPo4hf/gRI4DsiYDFz4VekadhEitkGPW8CH4/Fw9jl1qwYSherFX1GSyo6TOiPN1JqChBCMaP0UNGBCXghk33AF56vBezWWM4Up7Qf9MvSbIiw4Bnil8a2t7IX8KJwkFJESDFY/DWS/EbXkCBZN7mxbPGAdc/Bww1gmlW3WZ4q6JQ/VoNttx1bAUUYTe/qZKFBCtvR4Tmj9DglqOalU+4ibcAnnq0IDMrTfaYXG4oVLIkKUP/4zqh6dqcLnFiu+OzUB+5x6YKh5QJwhDW3OD+P8j+gcJ0WDE5ejwcXiEyNIuRL0lkqrigdSRQP1ZoVfUSYhkMg7Xj/YurK9TCz2s4ToXpjUfwJC8RCjSRmLsuNvgZh09L6vDDZkMfmfDPIs95qbEhyedpAt2Fw+T3QWT3dV9pzZdECJTPQlRiKCIrMGI4YrgINbohQeAVkuXHLOeyBJKhaD2lJCn1gNzRqTiodnZWIQvkaF2Q5GYCYy9FXK5XCzzanO68c+jl/HesWqxFnVnGGMoqTMCQMT8Q3H+oquBjsBGU11EbBkMkBANRroMywBALpNBpZD1PjQDhLijhHQhIdVH0bTOJKrk0JT+R3BwqxOACT/s5lcx2Jww2Jy40mrF+yeq4ewSi1RrsKHJ5IBSzokLPIab+HYhMvvqEenanerGmojYMhggIRqMiI7qDiG6dVI2Hpk7HNmBOII5Dsi/Rnh9+RthmOILxoALu4R8NrkCGP9DsQfWmXSdBt+fMhQqhQxVzRb8+0S1V2BknFKOovQEjMjQRWbFEHQIkdXho0fkGY6aGoQoa6LfkBANNni3MDQDxBkzDxzHBZ60mjpC+EK6ncIsmi+qDgnJshwHjFnc4/R8lj4OS9rFqLLJgn8du4J6gw08z5AUr8Itk7KxYEzkFnaMVwnuU4svIdLoBV8Z4ykBNkSQEA02THXCr7hS01EQPhg4Dhh1E8DJOhzXnblyFCj7THg9/HogbWSvpxyaFIdbJ2VDKedwucWKNw9egrHT0CgSTmoPnh6RxZePiONoeBZiSIgGG55hWWKOmEJxuKIZr39VgeOdKxIGgi4TyJ8jvC75ELi4V0ikPfcBUPqxsD1vZkeVxwDITYnH3TOHoTBNC4WMg92XEEQArVqB5HglkuP9LGntGZ4ZayNnVAxD0/eDDR+O6kajHU0mRzcncUAMmyP4iKqPAZUHhIeH/KuFRx9J1qqweLKQdhKu4me9kZqg9p/mAXT0iAzVkTEoxiEhGkwwBrR5/EMdQuSpythrDJEvOA4YuRBIHiYMx+xGYciXOxNI6l8pX6lEKCA8PSJLkxCXpYhgIf8YhIRoMGFtESosyhTiLzpjDC1iMKOfYUhveJb6aV/uZ1Cg1nVEWJvq+i26gx3yEQ0mPP4hXaYwnQ7A7HCLya4RXZ5nALD7bB3+uq8M52r9rBZDw7OQQUI0mPARPxRQsusgxelmMNvd4vJF3fD8H9toEYf+QkI0mBAd1R3DiKZAk10HIbr2BQRMvQrRZVr9tZ+QEA0WbG3thdBkXj0ihYxDqk6NdJ2EpTWiFM+6aUZfaR6AMDSTKYSFIC3NEbQs9iBn9WDBUyg/MUusrAgIBc36VNp1EOHpERltTt8NZHKhWFzrJaG36aeyJNE71CMaLHhWaQ2kUD4BAEgQhaiHfLLOwzMiaEiIBgOM+RQinmfgw7FmfYygjxPCGawOd/cVPcRG5LAOBSREgwFTneDHkCuBxI6KiJXNFrzy+QV8cJLypXyhVsiRqddg2JB4OPxFnevbU2WsrYIfjggK8hENBpouCM/J+YJfw7PZZIerfalowjc/mpHXcwOFWnBaG6oFP1z25IjYFWtQj2gw0FAiPKd6Z8CL68jT1H3/SCkUnlvKpbVjAENCFOtYW9rXt5cBQ4q8djUYBSFK1dFKFL3h7smXltKeHNtSAfBBrnQ7yCEhinUaSoXnpDyhmFc7DhcvBjNmJFIMkT+qmi34674yvPNND85oXbYwRHPaABOVBQkGEqJYhjGg9qTwOm2U1656ow2MCbEynsA9ojsapRxmuxvNFof/JdJlMqH6AAA0lUXOuBiChCiWMVQLRevlCmHd+k7UGWwAgHTqDfVIilYFuYyD3cnDYO0hnsgz7G0siYxhMQYJUSxTfUx4ThvtFU0NAPo4FYrSEzAsJd7HgYQHuYzDkATBmd9gsvlvmDpS8MOZGijdIwhIiGIVaytQd0Z4nT212+6i9ATcMinb5xr1hDdpCYIzv95g999IGdcxPGugXlFfISGKVS59LawykZwP6ANb1pnwjWf4Wm/sQYiADj9cw9kwWxR7kBDFIsY6oOa48NpT3L4TrRYHWntyvhJepLeHN9QZbD3/zzzDM2OdMEQjAoaEKNZwu4CSD4QZs/QxwrR9Fw6VN2PL/gocLCdfRiCk69TITtJgbHYiXD3FE6m0wJDhwuuuyysRPUJCFEswBpTtFn6RlXHCemLdmjBcahbWq8+kGbOAUMhlWDo9D9eMSINS3stXJnuK8Fx3ilaB7QMkRLFE5QFhJQ0AGHMLoEns1qTBZIfR5oJCxiE7KS7CBg4CkguE/7vTBtSdltqaAQMJUSzA88Kyz+X7hL+L5ncMEbpwoV5Yp35YqhYqBd3+vuB087hQb4TD1UMah0wGDJ0mvL70NaV8BAh9Egc6ThtwZidQdVj4u3Cu35VVGWOiEBWlJUTIwNjh74er8O8TNShvNPfcMHuKELdlbQHqv42McQMcEqKBjKEaOLIFaDwv1E4etwQYNstv81qDDU0mBxQyDoVp2sjZGSMUpAr/s9NXeqk7pFAJC0wCwMU9wgKMRI+QEA1EGAOqDgHH3hACFzV6YMrdvS5weL5O6A2NzNRBo5T32JbozoQcPWQch0vNFtQbe4iyBoCc6cJ9sRuByi8jY+AAhoRooGE3AqfeAS7sBni3EEQ3bYVQxL0Xri5KxY3jMzElLyn8dsYgiRolRmYIQ9p9pY09xxTJlcCI7wqvqw5RMmwvkBANFBgDak4Ch14VPtQyBTDyBmDc97vlkflDJuMwJiuRlg7qB7OLUqGQcahqtuDk5V6GaKkjBH8RY8DZ94WwCsInUSFEr7zyCvLz86HRaDBz5kwcOnRIapOii7YrwPE3gXMfAC67sGT0VcXA0KsQSJ3XM9VtsDr8FH8n+oQ+TonZRcKyQXtKGlDdau35gKIFQm/VaQNOvNWxiAHhheRC9Pe//x2rVq3Ck08+iaNHj2LSpElYuHAh6uvrpTZNWngeaLwAnHwbOPo60Fol9IIK5wJTlwEJab2ewuZ0Y/fZOnxypg5/P3wJdheJUSiYmpeMMVk65CTH9V5UTq4AJi4V1pNzWoETO4DST4QhNiHCMYkTjmbOnInp06fjT3/6EwCA53nk5ubiZz/7GZ544okejzUYDNDr9Whra0NiYvfgvQEBY4DbATjMwioQ5ob2QuwXhd4PIPR6MicA+VcLDlA/WBwuNJkcaDI7UNNqxcVGsxjzcvWIVEzPT4nEFQ0KeJ7BxTMxFstoc+LTs3XI0sdBH6eEVqUAxwllRDgOSFYDmorPgJqTcLp52N0AnzQMSM4Hl5gBeVwy5HGJUCiUkMtiZzWDQL+jkpbmczgcOHLkCNasWSNuk8lkWLBgAb766qtu7e12O+z2jgxog8EQ2Bs1lwsBf74QdZh123ax0YSm9gLz3mubC68n5SZB1f6hqWwyi8XGuPb9rNNxE3MSoVEIM1VVLRbUtHfpZcwFGXN7vcWYLB3iVQpAGYfzsmHYbxkGe50erK4RQKNX29umDhV/lU9dbsOBsiavy0tNUOHakWkYNoSm60OJTMaJ9x4ATl1pQ0WjBRWNFp/tF0/ORuHoRUDGONQd+wRVlaXA5RYAx73aMU6GoqwUpCfpAE6GJrMD5+vN4DxDcI6D8AkT/i5M04pJuc1mJ87X+e9p5adqkZEotG2zOnGu1n/b3JR4ZOuFz5XR5sK3NR3ftdqU6bjh2msRpwrdzKukQtTY2Ai3242MjAyv7RkZGTh37ly39uvXr8e6dev6/kYuu1CpsI8wiwkucw+lH2wyoD33iLeZ4bb1MKXrUAJMuHHMYYXbIQiRZ7Dk5pRwyuNhUSZjeO44xGcXAYlD0VTRgpayJsCPj4fvJJBatQJJ8UqkaFVITVAjP1WLbL2m40NMhI3RmYmIU8pRZ7DDaHPC5nSDZ8L9YQwdOWrJ+TCP/RHO86XQWysRZ6mB2tEChdMIGXODYzwUbjtgE+4rb7GDWUzwN2zh4hIArn3xA4sDvMm/uDCNFpC1DyWtTvDGHn7IVfGAvD0FyO7d1qQ0gvm1KDgGVLHiNWvWYNWqVeLfBoMBubm5vR+YlAtMulN4LX4pO305/WxLMTugcbi7/Bp17JcnaoRtHIcUqwMaJ+t0Bq7jEA5QaNWAXBCiFLsTGmd76D+nAJTxwnRv+2HaeJUocBOG6jG8PQra6+3bnxPbVyMFaB17KUnRqpAS4LJMozJ1GJV5FYCrxG2M5+FyWOF2OqBgTgAugDHonC4UWBzgPakijKFz710bpwTaeybxTh7DLP6DJxPiFIBK+MprXDzyTL20ba9lrnLxyBvZ0TYnLgVqRWjj0CQVotTUVMjlctTVeU9r1tXVITMzs1t7tVoNtTqIpW9U2o4lX/pAsh5IDrCtXgcEKgEJWiDQBAutWgEtFbePeTiZDEqNFkqN9xBaA6D7N8E3GgC9R5MJqAEMTQ9922CRdNZMpVLhqquuwu7du8VtPM9j9+7dmDXLf6oCQRCxheQ/tatWrcKyZcswbdo0zJgxAy+99BLMZjOWL18utWkEQUQIyYVo6dKlaGhowNq1a1FbW4vJkyfjo48+6ubAJggidpE8jqg/xEQcEUHEMIF+RyWPrCYIgiAhIghCckiICIKQHMmd1f3B494KONWDIIiI4vlu9uaKHtBCZDQK4ewBRVcTBCEZRqMRer3/kN8BPWvG8zyqq6uh0+miMp/Kk4JSVVUVc7N6dG0Dk0hfG2MMRqMR2dnZkMn8e4IGdI9IJpMhJydHajN6JTExMeY+0B7o2gYmkby2nnpCHshZTRCE5JAQEQQhOSREYUStVuPJJ58MrmJAlEPXNjCJ1msb0M5qgiBiA+oREQQhOSREBEFIDgkRQRCSQ0JEEITkkBD1kb6uSvvOO+9g9OjR0Gg0mDBhAv773/967WeMYe3atcjKykJcXBwWLFiA8+fPh/MS/NKXa3v11VdxzTXXIDk5GcnJyViwYEG39sXFxeA4zutx4403hvsyfNKXa9u6dWs3uzUa74UUB+p9mzt3brdr4zgOixYtEttIct8YETA7duxgKpWKbd68mZ05c4Y9+OCDLCkpidXV1flsv3//fiaXy9nvfvc79u2337Jf//rXTKlUslOnToltNmzYwPR6PXv33XfZiRMn2K233soKCgqY1WqN1GUxxvp+bXfddRd75ZVX2LFjx9jZs2dZcXEx0+v17PLly2KbZcuWsRtvvJHV1NSIj+bm5khdkkhfr23Lli0sMTHRy+7a2lqvNgP1vjU1NXld1+nTp5lcLmdbtmwR20hx30iI+sCMGTPYypUrxb/dbjfLzs5m69ev99n+jjvuYIsWLfLaNnPmTPbwww8zxhjjeZ5lZmay3//+9+L+1tZWplar2VtvvRWGK/BPX6+tKy6Xi+l0OrZt2zZx27Jly9jixYtDbWqf6eu1bdmyhen1er/ni6X79oc//IHpdDpmMpnEbVLcNxqaBYhnVdoFCxaI23palRYAvvrqK6/2ALBw4UKxfXl5OWpra73a6PV6zJw50+85w0Ew19YVi8UCp9OJlBTvZa337NmD9PR0jBo1Cj/5yU/Q1NTk5wzhIdhrM5lMGDZsGHJzc7F48WKcOXNG3BdL9+21117DnXfeCa3WexmjSN83EqIA6WlV2traWp/H1NbW9tje89yXc4aDYK6tK48//jiys7O9vhQ33ngjXn/9dezevRvPPfcc9u7di5tuuglut+9Va8NBMNc2atQobN68Ge+99x7eeOMN8DyP2bNn4/LlywBi574dOnQIp0+fxgMPPOC1XYr7NqCz74noYMOGDdixYwf27Nnj5dS98847xdcTJkzAxIkTMXz4cOzZswfz58+XwtSAmDVrlte6erNnz8aYMWPwl7/8Bc8884yEloWW1157DRMmTMCMGTO8tktx36hHFCB9XZUWADIzM3ts73nuyznDQTDX5uH555/Hhg0b8Mknn2DixIk9ti0sLERqaiouXLjQb5sDpT/X5kGpVGLKlCmi3bFw38xmM3bs2IH777+/1/eJxH0jIQqQYFalnTVrlld7ANi1a5fYvqCgAJmZmV5tDAYDDh48GNGVboNdcfd3v/sdnnnmGXz00UeYNm1ar+9z+fJlNDU1ISsrKyR2B0IoVhN2u904deqUaPdAv2+AEFZit9txzz339Po+EblvEXWND3B27NjB1Go127p1K/v222/ZQw89xJKSksSp3XvvvZc98cQTYvv9+/czhULBnn/+eXb27Fn25JNP+py+T0pKYu+99x47efIkW7x4sWTTwH25tg0bNjCVSsX+8Y9/eE3zGo1GxhhjRqORrV69mn311VesvLycffrpp2zq1KlsxIgRzGazRfW1rVu3jn388cesrKyMHTlyhN15551Mo9GwM2fOeF3/QLxvHq6++mq2dOnSbtulum8kRH3kj3/8I8vLy2MqlYrNmDGDff311+K+6667ji1btsyr/dtvv81GjhzJVCoVGzduHPvggw+89vM8z37zm9+wjIwMplar2fz581lJSUkkLqUbfbm2YcOGMQDdHk8++SRjjDGLxcJuuOEGlpaWxpRKJRs2bBh78MEHu8XjRIq+XNtjjz0mts3IyGA333wzO3r0qNf5Bup9Y4yxc+fOMQDsk08+6XYuqe4blQEhCEJyyEdEEITkkBARBCE5JEQEQUgOCRFBEJJDQkQQhOSQEBEEITkkRARBSA4JETGg2Lp1K5KSksS/n3rqKUyePFn8u7i4GEuWLIm4XUT/ICEifOIpF/rjH/+4276VK1eC4zgUFxd7tQ+1AOTn5+Oll17y2rZ06VKUlpb6PWbjxo3YunWr+PfcuXPx2GOPhdQuIvSQEBF+yc3NxY4dO2C1WsVtNpsN27dvR15eniQ2xcXFIT093e9+vV7v1WMiBgYkRIRfpk6ditzcXOzcuVPctnPnTuTl5WHKlCn9OrevnsqSJUvEXtbcuXNRWVmJn//852IBd6D70KwrnXtmxcXF2Lt3LzZu3Cieo7y8HEVFRXj++ee9jjt+/Dg4jotoiRKiAxIiokdWrFiBLVu2iH9v3rwZy5cvD/v77ty5Ezk5OXj66adRU1ODmpqaPp9j48aNmDVrFh588EHxHHl5ed2uCQC2bNmCa6+9FkVFRaG6BKIPkBARPXLPPffgyy+/RGVlJSorK7F///6Aatj0l5SUFMjlcuh0OmRmZgZVcEyv10OlUiE+Pl48h1wuR3FxMUpKSsRld5xOJ7Zv344VK1aE+jKIAKFSsUSPpKWlYdGiRdi6dSsYY1i0aBFSU1OlNqtfZGdnY9GiRdi8eTNmzJiBf//737Db7bj99tulNm3QQj0ioldWrFiBrVu3Ytu2bSHrNchkMnStQON0OkNy7kB44IEHREf8li1bsHTpUsTHx0fs/QlvSIiIXrnxxhvhcDjgdDqxcOHCkJwzLS3Ny+/jdrtx+vRprzYqlarfK0f4O8fNN98MrVaLTZs24aOPPqJhmcTQ0IzoFblcjrNnz4qv/dHW1objx497bRsyZAhyc3O7tb3++uuxatUqfPDBBxg+fDhefPFFtLa2erXJz8/Hvn37cOedd0KtVgc1JMzPz8fBgwdRUVGBhIQEpKSkQCaTib6iNWvWYMSIERGtNU10h3pEREAkJiYiMTGxxzZ79uzBlClTvB7r1q3z2XbFihVYtmwZ7rvvPlx33XUoLCzEvHnzvNo8/fTTqKiowPDhw5GWlhaU3atXr4ZcLsfYsWORlpaGS5cuifvuv/9+OByOiMwCEj1DpWKJQcsXX3yB+fPno6qqqtsihURkISEiBh12ux0NDQ1YtmwZMjMz8eabb0pt0qCHhmbEoOOtt97CsGHD0Nrait/97ndSm0OAekQEQUQB1CMiCEJySIgIgpAcEiKCICSHhIggCMkhISIIQnJIiAiCkBwSIoIgJIeEiCAIySEhIghCcv4/dxPYPI7V8kYAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:52:10.445970Z", + "iopub.status.busy": "2024-03-22T20:52:10.445338Z", + "iopub.status.idle": "2024-03-22T20:52:10.657820Z", + "shell.execute_reply": "2024-03-22T20:52:10.656962Z" + }, + "papermill": { + "duration": 0.233183, + "end_time": "2024-03-22T20:52:10.659660", + "exception": false, + "start_time": "2024-03-22T20:52:10.426477", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnOElEQVR4nO3dfVQU970/8PcuDwsI+AQsSND1oRFNVBR+ICbexBYhaUxtzQNHrRBi6anKrcnWxtCcQoltSBpFbErk1opGk1ZuGmNyo/WJSK5GfMJgfQIPKGIUEFRAIC4L+/394WWadRdYzMIwu+/XORycme/MfoaBtzPfmf2uSgghQESkMGq5CyAiuh8MLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRXOUuoL+ZTCZcu3YNPj4+UKlUcpdDRPcQQuD27dsYMWIE1Oquz6+cLryuXbuGkJAQucsgoh5cuXIFDzzwQJfLnS68fHx8ANz9wfj6+spcTd8xGo3Yu3cvYmNj4ebmJnc5ZAfOckybmpoQEhIi/a12xenCq/NS0dfX1+HDy8vLC76+vg79i+5MnO2Y9tStww57IlIkhhcRKRLDi4gUieFFRIoke3jl5ORAp9PBw8MDUVFROHbsWLftGxoasGzZMgQFBUGj0eDBBx/Erl27+qlaIhooZL3bmJ+fD71ej9zcXERFRSE7OxtxcXEoKytDQECARfu2tjbMnj0bAQEB+Mc//oHg4GBcvnwZQ4YM6f/iiUhWsoZXVlYWkpOTkZSUBADIzc3Fzp07kZeXh1dffdWifV5eHm7evInDhw9Lt4p1Ol1/ljxgtba2orS0VJpu/saAw6crMNTvBLw9NWZtQ0ND4eXl1d8lEtmVbOHV1taG4uJipKamSvPUajViYmJQVFRkdZ1PP/0U0dHRWLZsGT755BP4+/tjwYIFWLlyJVxcXKyuYzAYYDAYpOmmpiYAd5+ZMRqNdtwjeZ05cwZRUVEW8/9ope3Ro0cxderUvi+K7Krz99WRfm+tsXX/ZAuv+vp6dHR0QKvVms3XarVmZxDfdvHiRXz++edYuHAhdu3ahfLycixduhRGoxHp6elW18nMzERGRobF/L179zrU2YfBYMCaNWuk6dpWYEu5CxLGdUB7z25WVlaiurq6nyske9m3b5/cJfSp1tZWm9op6gl7k8mEgIAA/OUvf4GLiwvCw8Nx9epVvP32212GV2pqKvR6vTTd+daD2NhYh37C/lTVTeRvOIHn5kdgyshhcpdDdmA0GrFv3z7Mnj3boZ+w77w66ols4eXn5wcXFxfU1taaza+trUVgYKDVdYKCguDm5mZ2iThhwgTU1NSgra0N7u7uFutoNBpoNBqL+W5ubg79C+Dq6ip9d+T9dEaO/rtr677J9qiEu7s7wsPDUVBQIM0zmUwoKChAdHS01XUeeeQRlJeXw2QySfMuXLiAoKAgq8FFRI5L1ue89Ho9NmzYgPfeew/nz5/HkiVL0NLSIt19TEhIMOvQX7JkCW7evInly5fjwoUL2LlzJ9544w0sW7ZMrl0gIpnI2ucVHx+Puro6pKWloaamBmFhYdi9e7fUiV9VVWU2GFlISAj27NmDl19+GZMnT0ZwcDCWL1+OlStXyrULRCQT2TvsU1JSkJKSYnVZYWGhxbzo6GgcOXKkj6siooFO9vAiIkv3PnQMdP3gsbM+dMzwIhqASktLER4ebnXZvQ8eFxcXY9q0aX1f1ADD8CIagEJDQ1FcXGw2r6y6AfoPTyPruUkYHzTErK0zYngRDUBeXl4WZ1PqyzegOfgNJjw8BWGjhstU2cAh+5A4RET3g+FFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRRoQ4ZWTkwOdTgcPDw9ERUXh2LFjXbbdvHkzVCqV2ZeHh0c/VktEA4Hs4ZWfnw+9Xo/09HScPHkSU6ZMQVxcHK5fv97lOr6+vqiurpa+Ll++3I8VE9FAIHt4ZWVlITk5GUlJSZg4cSJyc3Ph5eWFvLy8LtdRqVQIDAyUvrRabT9WTEQDgaucL97W1obi4mKkpqZK89RqNWJiYlBUVNTles3NzRg1ahRMJhOmTZuGN954Aw899JDVtgaDAQaDQZpuamoCABiNRhiNRjvtycDT3t4ufXfk/XQmznJMbd03WcOrvr4eHR0dFmdOWq0WpaWlVtcZP3488vLyMHnyZDQ2NmL16tWYMWMGzp49iwceeMCifWZmJjIyMizm7927F15eXvbZkQHoSjMAuOLIkSO4ekbuasgenOWYtra22tRO1vC6H9HR0YiOjpamZ8yYgQkTJuC//uu/sGrVKov2qamp0Ov10nRTUxNCQkIQGxsLX1/ffqlZDqeqbgKnT2D69OmYMnKY3OWQHTjLMe28OuqJrOHl5+cHFxcX1NbWms2vra1FYGCgTdtwc3PD1KlTUV5ebnW5RqOBRqOxup6bm1vvi1YIV1dX6bsj76czcZZjauu+ydph7+7ujvDwcBQUFEjzTCYTCgoKzM6uutPR0YHTp08jKCior8okogFI9stGvV6PxMREREREIDIyEtnZ2WhpaUFSUhIAICEhAcHBwcjMzAQAvP7665g+fTrGjRuHhoYGvP3227h8+TJ+9rOfybkbRNTPZA+v+Ph41NXVIS0tDTU1NQgLC8Pu3bulTvyqqiqo1f8+Qbx16xaSk5NRU1ODoUOHIjw8HIcPH8bEiRPl2gUikoHs4QUAKSkpSElJsbqssLDQbHrt2rVYu3ZtP1RFRAOZ7A+pEhHdD4YXESkSw4uIFInhRUSKNCA67IkIuFTfghZDe5fLK+papO+dD6zea5DGFaP9BvVJfQMNw4toALhU34JZqwttavurf5zudvmBFY87RYAxvIgGgM4zruz4MIwL8Lbe5hsDPisswpzHozHI0/Itb+XXm/FSfkm3Z2+OhOFFNICMC/DGw8GDrS4zGo2o8QemjRrq0O9ttBU77IlIkXjmpWDddfDa0rkLOFcHLzkWhpdC2drB21PnLuA8HbzkWBheCtVTB29PnbuA83XwkmNheClcVx287NwlR8cOeyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiR+JyXQhk67kDtcRWXmsqg9rB8SLW9vR3X2q/h/M3zXb496FJTM9QeV2HouAPA+puBiQYqhpdCXWu5jEGj38FvjnXf7t3d73a7fNBo4FpLGMKhtWN1RH2P4aVQIwaNQsul/8S6+DCMtfL2oPb2dnx56Es88ugjXZ55VVxvxvL8EoyYNaqvyyWyO4aXQmlcPGC6E4zRvuMxcbj1twddcr2ECcMmdPn2INOdRpju1EHj4tHX5RLZXa877C9evNgXdRAR9Uqvw2vcuHGYNWsW3n//fdy5c6cvaiIi6lGvLxtPnjyJTZs2Qa/XIyUlBfHx8Vi8eDEiIyP7oj4ip9DT3WOg5zvIznb3uNfhFRYWhnXr1mHNmjX49NNPsXnzZjz66KN48MEH8eKLL2LRokXw9/fvi1qJHJatd4+B7u8gO9Pd4/vusHd1dcW8efPw1FNP4d1330VqaipWrFiB3/zmN3j++efx1ltvISgoyJ61Ejmsnu4eAz3fQXa2u8f3HV4nTpxAXl4etm3bhkGDBmHFihVYvHgxvv76a2RkZGDu3Lk4dsyG/0aIqMe7x0DPd5Cd7e5xr8MrKysLmzZtQllZGX74wx9iy5Yt+OEPfwi1+m7f/+jRo7F582bodDp710pEJOn13cb169djwYIFuHz5Mnbs2IE5c+ZIwdUpICAAGzdutHmbOTk50Ol08PDwQFRUlM1nbNu2bYNKpcKPf/zj3uwCETmAXofXvn37sHLlSov+LCEEqqqqAADu7u5ITEy0aXv5+fnQ6/VIT0/HyZMnMWXKFMTFxeH69evdrldZWYkVK1Zg5syZvd0FInIAvQ6vsWPHor6+3mL+zZs3MXr06F4XkJWVheTkZCQlJWHixInIzc2Fl5cX8vLyulyno6MDCxcuREZGBsaMGdPr1yQi5et1n5cQwur85uZmeHj0rqOwra0NxcXFSE1Nleap1WrExMSgqKioy/Vef/11BAQEYPHixTh48GC3r2EwGGAwGKTppqYmAHc7P41GY6/qHUja29ul79b2o3Ned/vY0zao/9hyLHo6po5yPG2t3ebw0uv1AACVSoW0tDR4eXlJyzo6OnD06FGEhYX1qsj6+np0dHRAqzV/JkWr1aK0tNTqOocOHcLGjRtRUlJi02tkZmYiIyPDYv7evXvN9kFprjQDgCsOHTqEy9bvrAO4e5n/XbdBfa83x6KrY+oox7O1tdWmdjaH11dffQXg7pnX6dOn4e7uLi1zd3fHlClTsGLFil6W2Tu3b9/GokWLsGHDBvj5+dm0TmpqqhS8wN0zr5CQEMTGxsLX17evSu1zZ681YfXpI3j00Ufx0AjL/TAajdi3bx9mz57d5Ruze9oG9R9bjkVPx9RRjmfn1VFPbA6vAwcOAACSkpKwbt06u/zh+/n5wcXFBbW1tWbza2trERgYaNG+oqIClZWVePrpp6V5JpMJwN2HZsvKyjB27FizdTQaDTQay0+MdnNzU/SHsXY+pOjq6trtfnS3n7Zug/peb45FV8fUUY6nrbX3usN+06ZNdjtjcXd3R3h4OAoKCqR5JpMJBQUFiI6OtmgfGhqK06dPo6SkRPr60Y9+hFmzZqGkpAQhISF2qYuIBj6bzrzmzZuHzZs3w9fXF/Pmzeu27fbt23tVgF6vR2JiIiIiIhAZGYns7Gy0tLQgKSkJAJCQkIDg4GBkZmbCw8MDDz/8sNn6Q4YMAQCL+UTk2GwKr8GDB0OlUkn/tqf4+HjU1dUhLS0NNTU1CAsLw+7du6VO/KqqKouHYImIbAqvTZs2Wf23vaSkpCAlJcXqssLCwm7X3bx5s93rIaKBj6c0RKRINp15TZ06Vbps7MnJkye/U0FERLawKbz4xmciGmhsCq/09PS+roOIqFfY50VEimTTmdewYcNw4cIF+Pn5YejQod32f928edNuxRERdcWm8Fq7di18fHykf9vaeU9E1FdsCq9vDyz4wgsv9FUtREQ263Wfl4uLi9VRTm/cuAEXFxe7FEVE1JNeh1dXgxEaDAazYXKIiPqSzUPi/OlPfwJwdzDCv/71r/D2/vdoZx0dHfjf//1fhIaG2r9CIiIrbA6vtWvXArh75pWbm2t2ieju7g6dTofc3Fz7V0hEZIXN4XXp0iUAwKxZs7B9+3YMHTq0z4oiIupJrz+Ao3NEVSIiOfU6vF588cVul3f3kWVERPbS6/C6deuW2bTRaMSZM2fQ0NCA73//+3YrjIioO70Or48//thinslkwpIlSyw+/IKIqK/Y5Y3ZarUaer1euiNJRNTX7DaqREVFhfSJvUREfa3Xl43f/gBX4O5zX9XV1di5c6fZeyCJiPpSr8Or85OzO6nVavj7+2PNmjU93okkIrIXPudFRIrEkVSJSJEYXkSkSAwvIlIkhhcRKZLdwuvrr7/Gz3/+c3ttjoioW3YLrxs3bmDjxo322hwRUbd42UhEisTwIiJFYngRkSLZ/IT9vHnzul3e0NDwXWshIrKZzeE1ePDgHpcnJCTcVxE5OTl4++23UVNTgylTpuCdd95BZGSk1bbbt2/HG2+8gfLychiNRnzve9/Dr371KyxatOi+XlupvjF2AADOXG20urzlGwNO1AGBl29hkKfGapvy6819Vh9RX7M5vDZt2tQnBeTn50Ov1yM3NxdRUVHIzs5GXFwcysrKEBAQYNF+2LBheO211xAaGgp3d3d89tlnSEpKQkBAAOLi4vqkxoGo4v+C59Xtp7tp5Yqt5cd73NYgTa/f4kokO9l/a7OyspCcnIykpCQAQG5uLnbu3Im8vDy8+uqrFu0ff/xxs+nly5fjvffew6FDh5wqvGIfCgQAjA3whqeb5SeVl1U34lf/OI01z07C+KCuz5oHaVwx2m9Qn9VJ1FdsDi9bh7vpzQdwtLW1obi4GKmpqdI8tVqNmJgYFBUV9bi+EAKff/45ysrK8NZbb1ltYzAYYDAYpOmmpiYAd8feNxqNNtc60Pi4q/DM1KAul9+5cwcAMGqoBuMDvLrdlpJ/Do7i9jd3f0dPVd3sclDPljt3uwL8LtZhkIdlV0B5XQsAoL29XdHH1NbabQ6vzZs3Y9SoUZg6dSqEEPdd2LfV19ejo6MDWq3WbL5Wq0VpaWmX6zU2NiI4OBgGgwEuLi549913MXv2bKttMzMzkZGRYTF/79698PLq/o9aya40A4Arjhw5gqtn5K6GelJUqwLggtc+OddDS1dsLf+q2xbHiw7hsqfdSut3ra2tNrWzObyWLFmCv//977h06RKSkpLw05/+FMOGDbvvAr8LHx8flJSUoLm5GQUFBdDr9RgzZozFJSUApKammo3+2tTUhJCQEMTGxsLX17cfq+5fp6puAqdPYPr06ZgyUp7jRLab3tKGSeevY4z/IKvdAABwoaYRr3x8Hn/8yQQ8GGi9K2CQxgW64cruBui8OuqJzeGVk5ODrKwsbN++HXl5eUhNTcVTTz2FxYsXIzY2FiqVqtdF+vn5wcXFBbW1tWbza2trERgY2OV6arUa48aNAwCEhYXh/PnzyMzMtBpeGo0GGo3lKbabmxvc3Nx6XbNSuLq6St8deT8dhXaIGxZGj7ap7YOBgxE2angfVyQfW39fe/WQqkajwfz587Fv3z6cO3cODz30EJYuXQqdTofm5t7fdnd3d0d4eDgKCgqkeSaTCQUFBYiOjrZ5OyaTyaxfi4gc333fbVSr1VCpVBBCoKOj474L0Ov1SExMREREBCIjI5GdnY2Wlhbp7mNCQgKCg4ORmZkJ4G4fVkREBMaOHQuDwYBdu3Zh69atWL9+/X3XQETK06vwMhgM0mXjoUOHMGfOHPz5z3/GE088AbX6/t5pFB8fj7q6OqSlpaGmpgZhYWHYvXu31IlfVVVltu2WlhYsXboUX3/9NTw9PREaGor3338f8fHx9/X6RKRMNofX0qVLsW3bNoSEhODFF1/E3//+d/j5+dmliJSUFKSkpFhdVlhYaDb9+9//Hr///e/t8rpEpFw2h1dubi5GjhyJMWPG4IsvvsAXX3xhtd327dvtVhwRUVdsDq+EhIT7uqNIRNQXevWQKhHRQMHxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEp0oAIr5ycHOh0Onh4eCAqKgrHjh3rsu2GDRswc+ZMDB06FEOHDkVMTEy37YnIMckeXvn5+dDr9UhPT8fJkycxZcoUxMXF4fr161bbFxYWYv78+Thw4ACKiooQEhKC2NhYXL16tZ8rJyI5yR5eWVlZSE5ORlJSEiZOnIjc3Fx4eXkhLy/PavsPPvgAS5cuRVhYGEJDQ/HXv/4VJpMJBQUF/Vw5EcnJVc4Xb2trQ3FxMVJTU6V5arUaMTExKCoqsmkbra2tMBqNGDZsmNXlBoMBBoNBmm5qagIAGI1GGI3G71D9wNbe3i59d+T9dCbOckxt3TdZw6u+vh4dHR3QarVm87VaLUpLS23axsqVKzFixAjExMRYXZ6ZmYmMjAyL+Xv37oWXl1fvi1aIK80A4IojR47g6hm5qyF7cJZj2traalM7WcPru3rzzTexbds2FBYWwsPDw2qb1NRU6PV6abqpqUnqJ/P19e2vUvvdqaqbwOkTmD59OqaMtH5WSsriLMe08+qoJ7KGl5+fH1xcXFBbW2s2v7a2FoGBgd2uu3r1arz55pvYv38/Jk+e3GU7jUYDjUZjMd/NzQ1ubm73V7gCuLq6St8deT+dibMcU1v3TdYOe3d3d4SHh5t1tnd2vkdHR3e53h//+EesWrUKu3fvRkRERH+USkQDjOyXjXq9HomJiYiIiEBkZCSys7PR0tKCpKQkAEBCQgKCg4ORmZkJAHjrrbeQlpaGv/3tb9DpdKipqQEAeHt7w9vbW7b9IKL+JXt4xcfHo66uDmlpaaipqUFYWBh2794tdeJXVVVBrf73CeL69evR1taGZ5991mw76enp+N3vftefpRORjGQPLwBISUlBSkqK1WWFhYVm05WVlX1fEBENeLI/pEpEdD8YXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUaUCMKkHfXWtrq9m4/2XVDTDUlOP8GU+YbgwxaxsaGurQ4/eTc2B4OYjS0lKEh4dbzF/wnmXb4uJiTJs2rR+qIuo7DC8HERoaiuLiYmm6+RsDdh4owlOzouHtqbFoS6R0DC8H4eXlZXY2ZTQacav+OqIjIxz6wxrIebHDnogUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSLKHV05ODnQ6HTw8PBAVFYVjx4512fbs2bN45plnoNPpoFKpkJ2d3X+FEtGAIut4Xvn5+dDr9cjNzUVUVBSys7MRFxeHsrIyBAQEWLRvbW3FmDFj8Nxzz+Hll1+WoWKi/nHvsN5A10N7O+2w3kJGkZGRYtmyZdJ0R0eHGDFihMjMzOxx3VGjRom1a9f2+jUbGxsFANHY2NjrdZWkra1N7NixQ7S1tcldCt2H4uJiAcCmr+LiYrnLtStb/0ZlO/Nqa2tDcXExUlNTpXlqtRoxMTEoKiqy2+sYDAYYDAZpuqmpCcDdkUaNRqPdXmeg6dw3R95HRzZ27FgcPXrUbF7zNwbsOXgccTP/n9nQ3mPHjnWo42zrvsgWXvX19ejo6IBWqzWbr9VqLU6Xv4vMzExkZGRYzN+7d69TnGrv27dP7hLIjmZMGovbDTdxu+Hf86qrq2Wrpy+0trba1M7hx7BPTU2FXq+XppuamhASEoLY2Fj4+vrKWFnfMhqN2LdvH2bPns0x7B2EsxzTzqujnsgWXn5+fnBxcUFtba3Z/NraWgQGBtrtdTQaDTQajcV8Nzc3h/4F6OQs++lMHP2Y2rpvsj0q4e7ujvDwcBQUFEjzTCYTCgoKEB0dLVdZRKQQsl426vV6JCYmIiIiApGRkcjOzkZLSwuSkpIAAAkJCQgODkZmZiaAu538586dk/599epVlJSUwNvbG+PGjZNtP4io/8kaXvHx8airq0NaWhpqamoQFhaG3bt3S534VVVVUKv/fXJ47do1TJ06VZpevXo1Vq9ejcceewyFhYX9XT4RyUj2DvuUlBSkpKRYXXZvIOl0Oggh+qEqIhroZH97EBHR/ZD9zKu/dZ652Xo7VqmMRiNaW1vR1NTk0HemnImzHNPOv82errKcLrxu374NAAgJCZG5EiLqzu3btzF48OAul6uEk3UimUwmXLt2DT4+PlCpVHKX02c6H8a9cuWKQz+M60yc5ZgKIXD79m2MGDHC7IbdvZzuzEutVuOBBx6Qu4x+4+vr69C/6M7IGY5pd2dcndhhT0SKxPAiIkVieDkojUaD9PR0q+/rJGXiMTXndB32ROQYeOZFRIrE8CIiRWJ4EZEiMbxk9vjjj+Oll16SuwwixWF4KURhYSFUKhUaGhrkLoV6YaD95zTQ6vkuGF5EA1xbW5vcJQxIDK8BxGAwYOXKlQgJCYFGo8G4ceOwceNGVFZWYtasWQCAoUOHQqVS4YUXXuhxe7dv38bChQsxaNAgBAUFYe3atRb/827duhURERHw8fFBYGAgFixYgOvXr0vLO8/4CgoKEBERAS8vL8yYMQNlZWX23n2H88ILL+CLL77AunXroFKpoFKpUFFRgcWLF2P06NHw9PTE+PHjsW7dOov1fvzjH+MPf/gDRowYgfHjxwMADh8+jLCwMHh4eCAiIgI7duyASqVCSUmJtO6ZM2fw5JNPwtvbG1qtFosWLUJ9fX2X9VRWVvbXj8P++vjzI6kHjz32mFi+fLkQQojnn39ehISEiO3bt4uKigqxf/9+sW3bNtHe3i4++ugjAUCUlZWJ6upq0dDQ0OO2f/azn4lRo0aJ/fv3i9OnT4uf/OQnwsfHR3o9IYTYuHGj2LVrl6ioqBBFRUUiOjpaPPnkk9LyAwcOCAAiKipKFBYWirNnz4qZM2eKGTNm2PtH4XAaGhpEdHS0SE5OFtXV1aK6ulrcuXNHpKWliePHj4uLFy+K999/X3h5eYn8/HxpvcTEROHt7S0WLVokzpw5I86cOSMaGxvFsGHDxE9/+lNx9uxZsWvXLvHggw8KAOKrr74SQghx69Yt4e/vL1JTU8X58+fFyZMnxezZs8WsWbO6rKe9vV2OH41dMLxk1hleZWVlAoDYt2+f1XadIXLr1i2bttvU1CTc3NzEhx9+KM1raGgQXl5eZuF1r+PHjwsA4vbt22avu3//fqnNzp07BQDxzTff2FSLM/v2f05dWbZsmXjmmWek6cTERKHVaoXBYJDmrV+/XgwfPtzsZ75hwwaz8Fq1apWIjY012/aVK1ek//RsrUcpeNk4QJSUlMDFxQWPPfaYXbZ38eJFGI1GREZGSvMGDx4sXYJ0Ki4uxtNPP42RI0fCx8dHev2qqiqzdpMnT5b+HRQUBABml5dku5ycHISHh8Pf3x/e3t74y1/+YvHznjRpEtzd3aXpsrIyTJ48GR4eHtK8bx9bADh16hQOHDgAb29v6Ss0NBQAUFFR0Yd7JA+nGxJnoPL09Oz312xpaUFcXBzi4uLwwQcfwN/fH1VVVYiLi7PoJP72yJ2d46CZTKZ+rdcRbNu2DStWrMCaNWsQHR0NHx8fvP322zh69KhZu0GDBvV6283NzXj66afx1ltvWSzr/A/HkTC8BohJkybBZDLhiy++QExMjMXyzv+FOzo6bNremDFj4ObmhuPHj2PkyJEAgMbGRly4cAH/8R//AQAoLS3FjRs38Oabb0ojy544ccIeu0P/x93d3eyYffnll5gxYwaWLl0qzbPlrGj8+PF4//33YTAYpDdmHz9+3KzNtGnT8NFHH0Gn08HV1fqf9r31KBkvGwcInU6HxMREvPjii9ixYwcuXbqEwsJC/Pd//zcAYNSoUVCpVPjss89QV1eH5ubmbrfn4+ODxMRE/PrXv8aBAwdw9uxZLF68GGq1WjpzGjlyJNzd3fHOO+/g4sWL+PTTT7Fq1ao+31dnotPpcPToUVRWVqK+vh7f+973cOLECezZswcXLlzAb3/7W4sQsmbBggUwmUz4+c9/jvPnz2PPnj1YvXo1gH+fCS9btgw3b97E/Pnzcfz4cVRUVGDPnj1ISkqSAuveepR89szwGkDWr1+PZ599FkuXLkVoaCiSk5PR0tICAAgODkZGRgZeffVVaLXaLj8u7tuysrIQHR2NOXPmICYmBo888ggmTJgg9Zv4+/tj8+bN+PDDDzFx4kS8+eab0h8E2ceKFSvg4uKCiRMnwt/fH3FxcZg3bx7i4+MRFRWFGzdumJ2FdcXX1xf/8z//g5KSEoSFheG1115DWloaAEjHc8SIEfjyyy/R0dGB2NhYTJo0CS+99BKGDBkiDad8bz339rUpCYfEcSItLS0IDg7GmjVrsHjxYrnLoe/ogw8+QFJSEhobG2XpM5Ub+7wc2FdffYXS0lJERkaisbERr7/+OgBg7ty5MldG92PLli0YM2YMgoODcerUKaxcuRLPP/+8UwYXwPBSrKqqKkycOLHL5efOnQMArF69GmVlZXB3d0d4eDgOHjwIPz+//iqT7KimpgZpaWmoqalBUFAQnnvuOfzhD3+QuyzZ8LJRodrb27t9a0d3d5yIHAHDi4gUiXcbiUiRGF5EpEgMLyJSJIYXESkSw4v61AsvvCANfOfm5gatVovZs2cjLy+vV29N2bx5M4YMGdJ3hXahc2BAGngYXtTnnnjiCVRXV6OyshL//Oc/MWvWLCxfvhxz5sxBe3u73OWRUsk5mBg5vsTERDF37lyL+QUFBQKA2LBhgxBCiDVr1oiHH35YeHl5iQceeEAsWbLEYkDEb3+lp6cLIYTYsmWLCA8PF97e3kKr1Yr58+eL2tpa6XVu3rwpFixYIPz8/ISHh4cYN26cyMvLk5ZXVVWJ5557TgwePFgMHTpU/OhHPxKXLl0SQgiRnp5u8boHDhzok58T9R7PvEgW3//+9zFlyhRs374dAKBWq/GnP/0JZ8+exXvvvYfPP/8cr7zyCgBgxowZyM7Ohq+vL6qrq1FdXY0VK1YAAIxGI1atWoVTp05hx44dqKysNBvf/7e//S3OnTuHf/7znzh//jzWr18vvcPAaDQiLi4OPj4+OHjwIL788kt4e3vjiSeeQFtbG1asWIHnn39eOnOsrq7GjBkz+vcHRV2TOz3JsXV15iWEEPHx8WLChAlWl3344Ydi+PDh0vSmTZvE4MGDe3y9e4exfvrpp0VSUpLVtlu3bhXjx48XJpNJmmcwGISnp6fYs2dPj/WTvHjmRbIRQkhjUe3fvx8/+MEPEBwcDB8fHyxatAg3btxAa2trt9voaRjrJUuWYNu2bQgLC8Mrr7yCw4cPS+ueOnUK5eXl8PHxkYZNHjZsGO7cueOQwyY7GoYXyeb8+fMYPXo0KisrMWfOHEyePBkfffQRiouLkZOTA6D7zyzsHMba19cXH3zwAY4fP46PP/7YbL0nn3wSly9fxssvv4xr167hBz/4gXTJ2dzcjPDwcJSUlJh9XbhwAQsWLOjjvafviu/cJVl8/vnnOH36NF5++WUUFxfDZDJhzZo10qB5nSPIdrI2fLGtw1j7+/sjMTERiYmJmDlzJn79619j9erVmDZtGvLz8xEQEABfX1+rdTrSsMmOhmde1OcMBgNqampw9epVnDx5Em+88Qbmzp2LOXPmICEhAePGjYPRaJSGo966dStyc3PNtqHT6dDc3IyCggLU19ejtbXVpmGs09LS8Mknn6C8vBxnz57FZ599hgkTJgAAFi5cCD8/P8ydOxcHDx6Uht7+5S9/ia+//lp63X/9618oKytDfX09jEZj//zQqGdyd7qRY0tMTJQeM3B1dRX+/v4iJiZG5OXliY6ODqldVlaWCAoKEp6eniIuLk5s2bLF4nMqf/GLX4jhw4ebPSrxt7/9Teh0OqHRaER0dLT49NNPLT7LcMKECcLT01MMGzZMzJ07V1y8eFHaZnV1tUhISBB+fn5Co9GIMWPGiOTkZNHY2CiEEOL69eti9uzZwtvbm49KDDAcEoeIFImXjUSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gU6f8DGQKm1bQ9fUUAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:52:10.696552Z", + "iopub.status.busy": "2024-03-22T20:52:10.696267Z", + "iopub.status.idle": "2024-03-22T20:52:10.902785Z", + "shell.execute_reply": "2024-03-22T20:52:10.901904Z" + }, + "papermill": { + "duration": 0.22726, + "end_time": "2024-03-22T20:52:10.904698", + "exception": false, + "start_time": "2024-03-22T20:52:10.677438", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCQUlEQVR4nO3dd3xT9f748VeSTkoXoy2FFgqFllE2ZZahYEEcCC7kKku8XkVUVAT0p+ICr6BcReUqIuhXQJShl6VQ9pZRKKusQgt0MDvpSs7vj0BK6ErapEna9/PxyOPRnnNyzvu0zbufz+d8hkpRFAUhhHAwalsHIIQQFSHJSwjhkCR5CSEckiQvIYRDkuQlhHBIkryEEA5JkpcQwiFJ8hJCOCQnWwdQ1XQ6HZcuXcLT0xOVSmXrcIQQd1EUhczMTAIDA1GrSy9f1bjkdenSJYKCgmwdhhCiHElJSTRq1KjU/TUueXl6egL6H4yXl5eNoxFC3C0jI4OgoCDDZ7U0NS553a4qenl5SfISwo6V16wjDfZCCIckyUsI4ZAkeQkhHFKNa/MS1qUoCoWFhWi1WluHIuyURqPBycmp0l2VJHkJi8nPzyc5OZmcnBxbhyLsXK1atWjQoAEuLi4VPockL2EROp2OhIQENBoNgYGBuLi4SCdgUYyiKOTn53P58mUSEhJo3rx5mR1RyyLJS1hEfn4+Op2OoKAgatWqRX6hjos3blKvtguebs62Dk/YEXd3d5ydnTl//jz5+fm4ublV6DzSYC8s6vZ/0QvXc8jMLSDhSraNIxL2qKKlLaNzWCAOIYop1Mq6LsK6JHkJIRySJC9R4/Xt25dXXnnF1mFUG5s3b0alUnHjxg2rXkeSlxAmqqoPpTCNJC9hFdLi5djy8/NtHUK5JHkJq1EUhdwCLTn5hVX+quhC8Hl5ebz55psEBQXh6upKaGgo33//PefOnaNfv34A+Pr6olKpGDVqVLnny8zMZMSIEXh4eNCgQQM+//zzYtXUn376ic6dO+Pp6UlAQABPPfUUaWlphv23S3wxMTF07tyZWrVq0aNHD+Lj402+rw8//BA/Pz88PT159tlnmTx5Mu3btzfsHzVqFEOGDOGjjz4iMDCQsLAwk2IDWLNmDS1atMDd3Z1+/fpx7tw5k+OqDOnnJawmr1DH4//dbZNrH3s/mlou5v95P/PMM+zatYsvvviCdu3akZCQwJUrVwgKCmLZsmUMGzaM+Ph4vLy8cHd3L/d8EydOZMeOHfzxxx/4+/vzzjvvcODAAaPEUVBQwAcffEBYWBhpaWlMnDiRUaNGsWbNGqNzvfXWW8yaNYv69evz/PPPM2bMGHbs2FFuDD///DMfffQRX3/9NT179mTJkiXMmjWLkJAQo+NiYmLw8vJi/fr1JseWlJTE0KFDefHFF3nuuefYt28fr732WrkxWYIkLyFuOXnyJEuXLmX9+vX0798fgKZNmxr216lTBwA/Pz98fHzKPV9mZiYLFy5k0aJF3HvvvQD88MMPBAYGGh03ZswYw9dNmzbliy++oEuXLmRlZVG7dm3Dvo8++og+ffoAMHnyZAYPHkxubm65nTy//PJLxo4dy+jRowF45513+Ouvv8jKyjI6zsPDg3nz5hkN2Skvtm+++YZmzZoxa9YsAMLCwoiLi+OTTz4p9+dTWZK8hNW4OqlZ+s9utGnoXeXXdnfWmP2e2NhYNBqNIUFU1tmzZykoKCAyMtKwzdvb21Alu23//v289957HDp0iOvXr6PT6QBITEykVatWhuPatm1r+LpBgwYApKWlERwcXGYc8fHxvPDCC0bbIiMj2bhxo9G2iIiIYmMNy4vt+PHjdO3a1eg93bt3LzMeS5HkJaxGpVLh5qypUPXNFkypBlpadnY20dHRREdH8/PPP1O/fn0SExOJjo4u1mju7Fw0zOr2uNHbycQSPDw8KhybLUiDvRC3REREoNPp2LJlS4n7b5dKTJ3up2nTpjg7O/P3338btqWnp3Py5EnD9ydOnODq1avMmDGDqKgowsPDizWIV1ZYWJhRDECx70tiSmwtW7Zk7969Rtt2766adk5JXkLc0qRJE0aOHMmYMWNYuXIlCQkJbN68maVLlwLQuHFjVCoVq1at4vLly8XajO7m6enJyJEjeeONN9i0aRNHjx5l7NixqNVqQ8kpODgYFxcXvvzyS86ePcsff/zBBx98YNH7eumll/j+++9ZuHAhp06d4sMPP+Tw4cPlzvphSmzPP/88p06d4o033iA+Pp5FixaxYMECi8ZfGkleQtzhm2++4dFHH+WFF14gPDyccePGkZ2tH1zesGFDpk2bxuTJk/H392f8+PHlnu+zzz6je/fuPPDAA/Tv35+ePXvSsmVLQyN7/fr1WbBgAb/++iutWrVixowZzJw506L3NGLECKZMmcLrr79Ox44dSUhIYNSoUeU29JsSW3BwMMuWLWPlypW0a9eOuXPn8vHHH1s0/tKolIp2iHFQGRkZeHt7k56eLqsHWVBubi4JCQmEhITg5uZGfEomeYX66lXbRj62Dc6OZGdn07BhQ2bNmsXYsWNtFseAAQMICAjgp59+ssn17/57uZOpn1HHaEkVDiPjZj5oZP6u2w4ePMiJEyeIjIwkPT2d999/H4CHH364ymLIyclh7ty5REdHo9FoWLx4MRs2bDDqz+WIJHkJi7lZoCU5PZeUbB2uTuZ3VXA0d3dluNuxY8cAmDlzJvHx8bi4uNCpUye2bdtGvXr1LBZH69atOX/+fIn7/vvf/zJ06FDWrFnDRx99RG5uLmFhYSxbtszQl81RSfISFpNfqKPqOxvYTmBgILGxsWXuDw4OZv/+/VaNY82aNRQUFJS4z9/fH3d3dzZs2GDVGGxBkpcQFeTk5ERoaKitw6Bx48a2DsEm5GmjEMIh2TR5TZ8+nS5duuDp6Ymfnx9DhgwxaaT8r7/+Snh4OG5ubkRERBQbwCqEqP5smry2bNnCiy++yO7du1m/fj0FBQXcd999hn41Jdm5cyfDhw9n7NixHDx4kCFDhjBkyBCOHDlShZELIWzNrvp5Xb58GT8/P7Zs2ULv3r1LPOaJJ54gOzubVatWGbZ169aN9u3bM3fu3HKvIf28rCM3N5fYYydxrxOAyskFVyeN9PMSpbJEPy+7avNKT08HiqYeKcmuXbuKPeKNjo5m165dVo1NCGFf7CZ56XQ6XnnlFXr27EmbNm1KPS4lJQV/f3+jbf7+/qSkpJR4fF5eHhkZGUYvISypSZMmzJ492/C9SqVi5cqVNounprCbrhIvvvgiR44cYfv27RY97/Tp05k2bZpFzylEWZKTk/H19bV1GNWeXZS8xo8fz6pVq9i0aRONGjUq89iAgABSU1ONtqWmphIQEFDi8VOmTCE9Pd3wSkpKsljcQpQkICAAV1dXm8agKAqFhYU2jcHabJq8FEVh/PjxrFixgo0bNxabU7sk3bt3JyYmxmjb+vXrS5290dXVFS8vL6OXqCKKgqogB/Kzq/5l5nMoUxbKMNWd1cZz586hUqlYvnw5/fr1o1atWrRr165YG+327duJiorC3d2doKAgJkyYYPTU3dRFOtauXUunTp1wdXW1eC3G3ti02vjiiy+yaNEifv/9dzw9PQ3tVt7e3oZZLZ955hkaNmzI9OnTAXj55Zfp06cPs2bNYvDgwSxZsoR9+/bx7bff2uw+RMlUhTeJWNjSNhefeglcPMo/7hZTFsqojLfeeouZM2fSvHlz3nrrLYYPH87p06dxcnLizJkzDBw4kA8//JD58+dz+fJlxo8fz/jx4/nhhx8A0xfpmDx5MjNnzqRp06bVvupq0+T1zTffAPoVi+/0ww8/GJaVSkxMRK0uKiD26NGDRYsW8fbbbzN16lSaN2/OypUry2zkF6Ispi6UURmvv/46gwcPBmDatGm0bt2a06dPEx4ezvTp0xkxYoShlNe8eXO++OIL+vTpwzfffIObm5vJi3S8//77DBgwwGJx2zObJi9Tupht3ry52LbHHnuMxx57zAoRCUtSnNyJG3mcCBsswIFzLZMPNXWhjMoobfGM8PBwDh06xOHDh/n5558NxyiKgk6nIyEhgZYtW5q8SEfnzp0tFrO9s5unjaIaUqlQnGuZVX2rrspaPCMrK4t//vOfTJgwodj7goODzVoI4+5FNKozSV6ixrtzoYzby4jdXiijtJEeltSxY0eOHTtW6gwVcXFxhoUwgoKCANi3b5/V47J3dtFVQlRHdjPqrFymLJRhTW+++SY7d+5k/PjxxMbGcurUKX7//XfDHPlVsUiHI5LkJQTlL5RhTW3btmXLli2cPHmSqKgoOnTowDvvvGN4YFAVi3Q4IrsamF0VZGC2dRQfmK0mr1DfpuOIA7PtZaGM6koW4BDCQuxhoQxhHkleQtxS0kIZx48fZ9CgQaW+p7yFZ4X1SPISVuFobREdOnQocaGMmzdvlrnIhrAdSV5ClMHd3d0uFtkQxcnTRmFZNev5j6ggSzwnlOQlLMLZ2RmtTkEpzC//YFHj5eTkAMYjD8wl1UZhERqNhqRsFf7aK/jWAZ2bG4pW/981NzfXxtEJe6EoCjk5OaSlpeHj44NGU/GV1SV5CYtJyHbi73PXuLepFndnDYU6ffJyuVmT1tEWpvDx8Sl1AlFTSfISFqRi2fFsVp/KobW/BykZNwGIea2vbcMSdsXZ2blSJa7bJHkJi8stVLh8U8fFTP3SZ1UxxEbUPNJgLyxGnjOKqiTJSwjhkCR5CYux/uQxQhSR5CWsQpFKpLAySV5CCIckyUtYjJS1RFWS5CWEcEiSvIQQDkmSlxDCIUnyElYhM+MIa6vU8KCsrCzDwpm3yaIWNVcNW8tF2JjZJa+EhAQGDx6Mh4cH3t7e+Pr64uvri4+PD76+vtaIUQghijG75PWPf/wDRVGYP38+/v7+VbIopxBC3M3s5HXo0CH2799PWFiYNeIRQgiTmF1t7NKlC0lJSdaIRVQj0vwlrM3skte8efN4/vnnuXjxIm3atCk2B3Xbtm0tFpwQQpTG7OR1+fJlzpw5w+jRow3bVCoViqKgUqnQarUWDVA4DiltiapkdvIaM2YMHTp0YPHixdJgLyxOURSW7kuidaA3bRp62zocYcfMTl7nz5/njz/+kIU4hVVsPJHGm8viADg3Y7CNoxH2zOwG+3vuuYdDhw5ZIxYhOJGSaesQhIMwu+T14IMP8uqrrxIXF0dERESxBvuHHnrIYsGJmkd66QtTmZ28nn/+eQDef//9YvvMbbDfunUrn376Kfv37yc5OZkVK1YwZMiQUo/fvHkz/fr1K7Y9OTm50mvAicqzRNqR3CVMZXbyunssY2VkZ2fTrl07xowZw9ChQ01+X3x8vNEYSj8/P4vFJIRwDGYlr4KCAtzd3YmNjaVNmzaVvvigQYMYNGiQ2e/z8/PDx8en0tcX9kcKXsJUZjXYOzs7ExwcbPO+XO3bt6dBgwYMGDCAHTt2lHlsXl4eGRkZRi9hv6TaKExl9tPGt956i6lTp3Lt2jVrxFOmBg0aMHfuXJYtW8ayZcsICgqib9++HDhwoNT3TJ8+HW9vb8MrKCioCiMW5pJVh4SpzG7zmjNnDqdPnyYwMJDGjRvj4eFhtL+sRFJZYWFhRgPCe/TowZkzZ/j888/56aefSnzPlClTmDhxouH7jIwMSWBWIqUmUZXMTl5lPQ20hcjISLZv317qfldXV1xdXaswIlEZkgCFqcxOXu+++6414qiw2NhYGjRoYOswhIVI7hKmqvA00Pv37+f48eMAtG7dmg4dOph9jqysLE6fPm34PiEhgdjYWOrUqUNwcDBTpkzh4sWL/PjjjwDMnj2bkJAQWrduTW5uLvPmzWPjxo389ddfFb2NauV0WhbnrmTTv5W/rUOpOCl6CROZnbzS0tJ48skn2bx5s6G7wo0bN+jXrx9Lliyhfv36Jp9r3759Rp1Ob7dNjRw5kgULFpCcnExiYqJhf35+Pq+99hoXL16kVq1atG3blg0bNpTYcbUm6v/ZFgCW/as7nRrXsXE0FSOpS5jK7OT10ksvkZmZydGjR2nZsiUAx44dY+TIkUyYMIHFixebfK6+ffuWORxkwYIFRt9PmjSJSZMmmRtyjXP0UobNk1dFh/lIwUuYyuzktW7dOjZs2GBIXACtWrXiq6++4r777rNocEIIURqz+3npdLpig7FB34HVkkOHRM0k/byEqSo0Jc7LL7/MpUuXDNsuXrzIq6++yr333mvR4ETNI9VGYSqzk9ecOXPIyMigSZMmNGvWjGbNmhESEkJGRgZffvmlNWIUNYjkLmEqs9u8goKCOHDgABs2bODEiRMAtGzZkv79+1s8OOFY7qzyVTQJSclLmKpC/bxUKhUDBgxgwIABlo5H1HDS5iVMVaHkFRMTQ0xMDGlpacUa6efPn2+RwETlKIpCgVbBxcnslgHbktwlTGT2X/a0adO47777iImJ4cqVK1y/ft3oJezDuB/30ebdP7mWnW/rUISwCrNLXnPnzmXBggU8/fTT1ohHWMiG42kArDp8iWe6N7FtMGaQgpcwldklr/z8fHr06GGNWISDs0RjuyzAIUxldvJ69tlnWbRokTViEdVIRXOQ5C5hKrOrjbm5uXz77bds2LCBtm3bFutt/9lnn1ksOCGEKI3Zyevw4cO0b98egCNHjhjtU6lUFglK1FxS8BKmMjt5bdq0yRpxCAFItbG6SsvMxd1Zg6db8XHRFeVgnYCEKRy5/CudVKund38/St9PN7PxRKrFzlnhmVSFY7BVSUaSkLht//nrrD2SgloFDX1qWey8UvISdkWqjdWLoih8vEY/XfxjnYIIC/C02LkleQkhrObPo6nsP38dN2c1rw5oYdFzm528tm7dSmFhYbHthYWFbN261SJBiZpLOqlWHwVaHZ+s0888My6qKQHebhY9v9nJq1+/fiWulp2eni4LYdRwlkg8krqqjyV7E0m4kk1dDxee693U4uc3O3kpilJif66rV68WWz1bCHNJwat6yMwtYPaGUwC80r+5RbtI3Gby08ahQ4cC+o6oo0aNMlqFWqvVcvjwYRnzKAwkCdVs3249y9XsfJrW8+DJyGCrXMPk5OXt7Q3oS16enp64u7sb9rm4uNCtWzfGjRtn+QhFpThaG5J0sXB8Kem5fLftLACTBobjrLHOc0GTk9cPP/wAQJMmTXj99deliiiswsFyrSjB5+tPklugo3NjX6JbW2/1drM7qb777rvWiENUA5bIO5K7HFt8Sia/7k8CYMr9La063tns8lxqaipPP/00gYGBODk5odFojF5CiJprxtrj6BQY1CaATo19rXots0teo0aNIjExkf/3//4fDRo0kJkkRIlk9aCaZ+fpK2yKv4yTWsWkgeFWv57ZyWv79u1s27bNMC2OEJYl2csR6XQKH6/VDwMa0TWYkHrWbxM3u9oYFBTkcE+whOOQPy3H9L/DlzhyMYPark5MuLd5lVzT7OQ1e/ZsJk+ezLlz56wQjrA0R8sFkrwcT26Bln+viwfgX32bUbe2aznvsAyzq41PPPEEOTk5NGvWjFq1ahWbBrqkoUNCmEr6eTmen3ad5+KNmwR4uTGmZ0iVXdfs5DV79mwrhCGEnpS8HMuNnHy+3KgfBjTxvha4u1RdjwOzk9fIkSOtEYeoZiQJ1QxfbTpNRm4h4QGeDOvYqEqvXaF++2fOnOHtt99m+PDhpKXpFzddu3YtR48etWhwwrH8uOt8pc8hOc9xJF3LYeFO/e988qBwNOqq7TZldvLasmULERER7Nmzh+XLl5OVlQXAoUOHpPe9vXDgvndSYnMcM/+KJ1+ro2doXfq0qF/l1zc7eU2ePJkPP/yQ9evX4+LiYth+zz33sHv3brPOtXXrVh588EECAwNRqVSsXLmy3Pds3ryZjh074urqSmhoKAsWLDDzDmoAB84A0mDvGA5fuMHvsZcAmDLIusOASmN28oqLi+ORRx4ptt3Pz48rV66Yda7s7GzatWvHV199ZdLxCQkJDB48mH79+hEbG8srr7zCs88+y59//mnWdWsSh8tjjhZvDXTnvPSPdGhIm4beNonD7AZ7Hx8fkpOTCQkxfiR68OBBGjZsaNa5Bg0axKBBg0w+fu7cuYSEhDBr1iwAWrZsyfbt2/n888+Jjo4269rVml1UGyuWhSR32b9N8WnsPnsNFyc1r91n2XnpzWF2yevJJ5/kzTffJCUlBZVKhU6nY8eOHbz++us888wz1ojRYNeuXfTv399oW3R0NLt27Sr1PXl5eWRkZBi9hBAVU6jVMX2Nfl760T2a0MjXckuZmcvs5PXxxx8THh5OUFAQWVlZtGrVit69e9OjRw/efvtta8RokJKSgr+/8fxA/v7+ZGRkcPPmzRLfM336dLy9vQ2voKAgq8ZoDy7dKPpZ2EUhzAwy9My+/bb/AqfSsvCp5cwL/UJtGovZycvFxYXvvvuOM2fOsGrVKv7v//6PEydO8NNPP9nllDhTpkwhPT3d8EpKSrJ1SFa3cOc5W4dQYZK67FdOfiGfrT8JwPh+oXi7W35eenNUeMXs4OBggoOtMzd1aQICAkhNNV4uPDU1FS8vL6Npqe/k6upqNN++sG9S8LJf87YlkJaZR1Add57u3tjW4ZiWvCZOnMgHH3yAh4cHEydOLPPYzz77zCKBlaR79+6sWbPGaNv69evp3r271a4pKsYSSai0lapE1bucmcd/t5wB4I3ocFydbF/LMil5HTx4kIKCAsPXpTH3Dy0rK4vTp08bvk9ISCA2NpY6deoQHBzMlClTuHjxIj/++CMAzz//PHPmzGHSpEmMGTOGjRs3snTpUlavXm3WdYX9ujPnKYrjtdlVV/+JOUl2vpZ2jbx5IKKBrcMBTExemzZtKvHrytq3b5/RQrW3S3UjR45kwYIFJCcnk5iYaNgfEhLC6tWrefXVV/nPf/5Do0aNmDdvnnSTKIOjVcPubLB3sNCrrTOXs1i8t2heenUVDwMqTYXbvCyhb9++ZT5dKqn3fN++fcss/QnHJgnL/nyy9gRanUL/ln50a1rX1uEYmJS8bi84a4rly5dXOBhhGY5W2iqN/h+bffyXr6n+PneNv46lolbBm1UwL705TOoqcWc/KS8vL2JiYti3b59h//79+4mJiTEsTCtEhSklfils4M5hQE90Caa5v6eNIzJmUsnr9oKzAG+++SaPP/44c+fONfTr0mq1vPDCC3h5eVknSmEWe2jkrvDqQXe8s7qUIB3V2iMpHEy8QS0XDa/2r5p56c1hdifV+fPn8/rrrxt1SNVoNEycOJH58+dbNDhR80jCsg/5hTr+vU4/DGhcVFP8vNxsHFFxZievwsJCTpw4UWz7iRMn0Ol0FglK1FyKUbVRMpmtLNpznnNXc6hX25VxvZvaOpwSmf20cfTo0YwdO5YzZ84QGRkJwJ49e5gxYwajR4+2eIDCfIqF242OXEwnqE6tKhkOItVG28vILeCLjfr+l6/0b05tV5t2SiiV2VHNnDmTgIAAZs2aRXJyMgANGjTgjTfe4LXXXrN4gMK2dp65wlPf7cGnljOx79xn63BEFZi7+QzXsvNpWt+DJ7vY70QGZicvtVrNpEmTmDRpkmF6GWmor742HNOvUXAjp8Cs91V0dggpbdlWcvpNvt+eAMDkgeE4aSq0zEWVqFR5UJKWfbKHp40VdffwIFG1Zv11krxCHZFN6jCglX/5b7ChCiWv3377jaVLl5KYmEh+fr7RvgMHDlgkMFFxlvzQV3UilAZ72zmenMGyAxcAmHJ/uN0Pije7TPjFF18wevRo/P39OXjwIJGRkdStW5ezZ8+aNaWzqFoF2oo9Ca7on+91M6uZwvamrz2BosDgtg3oEOxr63DKZXby+vrrr/n222/58ssvcXFxYdKkSaxfv54JEyaQnp5ujRiFme7+h/nqL7FEvPcnaZm5lT6X9cnTRlvYduoyW09exlmjYlJ0mK3DMYnZySsxMZEePXoA4O7uTmZmJgBPP/00ixcvtmx0okKMql6KwoqDF8kt0LH0b/ufRdbS3TxE+XQ6xTAv/T+6NaZxXQ8bR2Qas5NXQEAA165dA/Szqd5eqzEhIUHmH6+GqrrdQ/6Cqt7K2IscS87A09WJl+6xv2FApTE7ed1zzz388ccfgL7D6quvvsqAAQN44oknSlzPUVjXhes5vLUijrOXswzbLJlvqrrWaDSfl/wztLrcAi0z/4wH4IV+odTxcCnnHfbD7KeN3377rWEY0IsvvkjdunXZuXMnDz30EP/85z8tHqAo29gF+4hPzWTdkRTDNkf+zCulfC2sY8HOc1xKzyXQ243RPZvYOhyzmJW8CgsL+fjjjxkzZgyNGjUC9Os4Pvnkk1YJTpQvPlXf5ng1u6jLikW7GNiyq4RkL6u6np3PV5v0w4Beuy8MN2fbz0tvDrOqjU5OTvz73/+msLDQWvEIC1BZMOOUdK7P1p/k261nLHYNYRtfbjxNZm4hLRt4MaSDeavd2wOz27zuvfdetmzZYo1YhIVYtM3rrnNdunGTL2JO8fEa/dTAlqaU+o2wpPNXs/lp9zkApt4fjsZO5qU3h9ltXoMGDWLy5MnExcXRqVMnPDyMH6s+9NBDFgtOWJYlnhzeLNBaIJLSGS/AIdnLWj79M54CrUJU83pENa9v63AqxOzk9cILLwAlr8+oUqnQaq37xy3KZ622otSMXLJypcnA0cUm3WDV4WRUKpgyqKWtw6kws5OXTDhYs9xZVuv6cUyVXlsa7C3vznnph3ZoRKtAx51cwX7nuxB2oayapjX6YUkPe+uKOZ7G3oRruDqpee2+FrYOp1JMLnndvHmTmJgYHnjgAQCmTJlCXl6eYb9Go+GDDz7Azc3+5roWjsN4JlVJX5ZUqNUxfa2+1DWmVwiBPu42jqhyTE5eCxcuZPXq1YbkNWfOHFq3bo27u/4HcOLECQIDA3n11VetE6mwibK6XVgjtUi+sp6l+y5w5nI2vrWc+VffZrYOp9JMrjb+/PPPPPfcc0bbFi1axKZNm9i0aROffvopS5cutXiAwrJ0ZnZvsO18XsJSsvMK+Wz9SQAm3NscLzfrr0dgbSYnr9OnTxMREWH43s3NDbW66O2RkZEcO3bMstGJCimti8HPu8/T5r0/2X/+mmWuY4XsIgtwWMd3285yJSuPxnVrMaJrY1uHYxEmJ68bN24YtXFdvnyZJk2aGL7X6XRG+4XtlFbVu5SeS06+lpeXxJpxrqqlMyp5SfayhLTMXL7dehaASdHhuDhV8XO69IuwbRb8tw/kZZV/vIlMbvNq1KgRR44cISys5InKDh8+bBjvKGzLolW9Mk5mleQi+criZm84RU6+lvZBPtwfEVA1F83PgROrIHYRnN2M4Rd7/A9o/5RFLmFy8rr//vt55513GDx4cLEnijdv3mTatGkMHjzYIkGJyrHoHPaWO5VJdNLoZVGn0zL55dYklFPvb2nd+dkUBRJ36RPW0ZWQn1m0r3EvaD8cWj5oscuZnLymTp3K0qVLCQsLY/z48bRooe8jEh8fz5w5cygsLGTq1KkWC0zYv4okybSMXJYduMjjnRtRt7Zr8XOW8rWomBlr49HqFAa08icypI51LnL9PBxaAocWwfVzRdt9GutLWe2eBN8mFr+sycnL39+fnTt38q9//YvJkycb+uCoVCoGDBjA119/jb+/fS+VJMxn6X/Uz8zfy4mUTLacTGPJc92L7ddJK73F7Dl7lQ3HU9GoVbw5MNyyJ8/LgmO/60tZ57cXbXfxhNYPQ7unILg7qK3XvmbW8KCQkBDWrVvHtWvXOH1aPw9QaGgodepYKaOLCrFkW5Qlp9cBOJGir0rsPlvyE0+Zz8sy7hwG9GSXIEL9alf+pDodnNumT1jH/4CCnFs7VNC0D7QfAeEPgEutyl/LBBVat7FOnTpERkZaOhZRRSyVFKzSVUJmlbCIVYeTOXQhnVouGl7pX8lhQFfP6BPW4V8g/Y5FXOqGQrvh+mqhd9U/rKvUitnCPt1ZWnK00ousmF15eYVa/v2nfjWgf/ZuRn3P4m2L5bp5A46ugEOLIWlP0XZXb4gYpq8WNups0+XZJXmJMpU5MNsKJSNp86q8/9udSNK1m9T3dGVc7xDT36jTwplN+ob3E6uh8NY6nyo1hPbXl7LC7gdn+xi/bBezSnz11Vc0adIENzc3unbtyt69e0s9dsGCBahUKqOXDAa3nqpfPeiOr6v42tVB+s0Cvtx4CoCJA1pQy8WE8knaCVj/DnzeGn4eBkeW6RNX/ZYw4AOYeBxG/ApthtpN4gI7KHn98ssvTJw4kblz59K1a1dmz55NdHQ08fHx+Pn5lfgeLy8v4uPjDd9X9dqCQs8ahSSjHvZSCjPb15tPcyOngOZ+tXmsUxntUDnX9Ekq9me4dLBou3sdiHhM3yerQXubVgvLY/Pk9dlnnzFu3DhGjx4NwNy5c1m9ejXz589n8uTJJb5HpVIREFBFPYVruKofmC0Jq6Iu3rjJDzvOATB5UDhOmrsqVtoCOL1Bn7Di14GuQL9d7QTNo/UJq3k0ODnG2o02TV75+fns37+fKVOmGLap1Wr69+/Prl27Sn1fVlYWjRs3RqfT0bFjRz7++GNat25d4rF5eXlGYy4zMjIsdwM1QFmlWmtPiSN5zDyz/ownv1BHt6Z1uCf8jlpLSpz+aWHcr5B9uWh7QFt9J9KIx8CjXtUHXEk2TV5XrlxBq9UW69zq7+/PiRMnSnxPWFgY8+fPp23btqSnpzNz5kx69OjB0aNHSxxbOX36dKZNm2aV+IXlSfeIijlyMZ0VsReBW8OAsq9A3FKIXQypcUUHevhB28f1je8BbWwUrWXYvNporu7du9O9e1HP7B49etCyZUv++9//8sEHHxQ7fsqUKUycONHwfUZGBkFBQVUSqz2wZjKwRhVPJyWvCvlk3QmclQLebHqOtlt/gtPrQXdrsRSNC4QN0ncibXYvaBzuY18im95FvXr10Gg0pKamGm1PTU01uU3L2dmZDh06GHr8383V1RVX1wr0cxGAtHnZPUXhwO6N9E/4hi9cd+F76Y4pZxp20lcLWw+FWtVvFIxNk5eLiwudOnUiJiaGIUOGAPp5wWJiYhg/frxJ59BqtcTFxXH//fdbMVLHUlXDg6ze5iVVyNJlJMPhX1AOLabj5RN0vP1J9myg7/HebjjUL3n6qurC5uXHiRMnMnLkSDp37kxkZCSzZ88mOzvb8PTxmWeeoWHDhkyfPh2A999/n27duhEaGsqNGzf49NNPOX/+PM8++6wtb8OuWHo8YlW6s5OqFMLuUnBT33n00GI4sxEUHSogV3EmRtWVPo9NoHbL/qDW2DrSKmHz5PXEE09w+fJl3nnnHVJSUmjfvj3r1q0zNOInJiYaTTd9/fp1xo0bR0pKCr6+vnTq1ImdO3fSqlUrW91CtVb20meWv57kq7soCiTt1fd6P7IC8tINuw6pWrIovxdrtF0ZP6gjtVs7/qIa5rB58gIYP358qdXEzZs3G33/+eef8/nnn1dBVNWXPbcrGZW8bBiHzd1IgsNL9E8Lr50p2uwSwOK8nizJ78l5JYA6Hi6M69GEZ6Oa2jBY27CL5CXsw897zvPL30nMH9WFercmCiyzAmqVWSXu/LqGpa/8bDj2h76UlbCN2z9grVMt9rpH8eXVzuzKbYmCmlC/2szoFcKQDg1xc64Z1cS7SfKqhiq6As9bK44AMOuvk0wfql8p6tKNmxaNrTw1bmyjTgeJO/WdSI/9DvlFTwuv1O/Kz7k9+e/lNuRk6ccURjWvx9heIfRpUb/GD4uT5FUN5RboKvX+m/n6/kFr45JZuOt8qcdZ42lgjSltXTt7a+rkxXAj0bBZ5xPCAd+BTL/Ynv1JngC4aNQ83iGQsb2aEhbgaauI7Y4kL1GiC9dz+NfPB6r8utW6k2puBhxbqW/HStxZtN3Vi6zQB1ha2JtZx33ITtH/86nr4cI/ujXmH90aV2xOrmpOkpcDmbzsMKkZucwf1cWqVYak6zfp9cmmco+z9qKz1aLiqNNCwhZ9wjr+Pyi8XQ1XoTTrR0LDh5l9oQWrDly/lbh1NPerzbNRITzcvua2Z5lCkpcDWXJrCavjyZm0CvSy2nX2n79utXOXp9qUvK6cKpo6OeNi0fZ6LdC2Hc5Gl37M2Z/DoaM3AP3Pu3eL+oztFULv5vVqfHuWKSR5OaDKzjZ6KT3XInFYu4e9w7l5HY4s1yeti/uKtrv5QMSjZIU/zqILdVmw4zyX0i8B4OKk5pH2DRnTK0Tas8wkycsBmfNP2Zr/wPcmXGNgG8vOq6Y4Wj8vbSGcidEnrPi1oL01/ZJKA80HQLvhJNbvw/zdl/h1YRLZ+VcAfXvW09317Vn1Sli/UpRPkpeosOf/bz+7ptxDA293i53TYRbgSD1aNEdW1h0TC/i1hvZPoUQ8yr6rLny/LYE/j+003EsL/9o826spD7UPlPasSpLkVc1ZOwFEf76Vw+9FW+x8dr0AR/ZVfbI6tAiSDxVtr1UXIh6H9sMpqN+GNUdSmL/wNIcuFA3l6XOrPStK2rMsRpKXg7DX/k8ZuYUWPZ/dzSpRmA+n/tL3xzq5rmiOLLUztIjWTzkTOoD0AhVL9iayYMFmkm+1Kbo4qRnaQd+e1cJf2rMsTZKXg7DT3GVxdjGrhKLoS1aHFutLWjlXi/Y1aK9PWG0eBY+6nL+azQ9rTrF0XxI5+VoA6tV24eluTRjRLVjas6xIkpewL7bsKpGZemvq5EWQdqxoe23/W1MnPwX+rVAUhb/PXef77fv461iqIc4wf0/GRoXwUDtpz6oKkrwcxJ2fY0eer+tOLy85yPsPt8Hb3dmwrcrbvApy4eRafcI6HQOKvvSExhXC79dPndy0H2icKNDqWBN7ke+3J3D4rvasZ6NC6BUq7VlVSZKXg7DXNq/K+D32EnU9XHnnwaK52Iz711vpnhUFLuy7NUfWMsgtSkQ06nJr6uRHwN0XgPScAhb/fYaFO88Z2rNcndQM7diQMT1DaC7tWTYhyUvYVGqGcYdZq7Z5pV8smiPr6qmi7V4Ni6ZOrtfcsPnclWx+2JHAr/svGLVnPdO9CSO6BlNX2rNsSpKXg6jo59jey2t317IsnrDyc+DEKn218OxmDD8RJ3do9ZA+YYX0NkydfLs9a962s6w/XtSeFR7gydheITzUPhBXJ2nPsgeSvEpTmAff3wfN79P/V65r2yl27/xQV6dmFY3a+GYskrwUBRJ36RPW0ZWQn1m0r3FPfcJq9TC4FY0PLdDqWBOXzLxtCcRdLKpG9gurz9heTekZWlfas+yMJK/SnPoLkmP1r63/hqCu+j/61o+Au0+Zb1UUhf/EnCInX4uLRk2P0Lr0aFa5FYntos+TFWjuSggVnUgRgOvnb82RtQiunyva7tNY/7tr9yTUCTF6S3pOAYv2JrJw5zlSMu5sz2rE2F5NCPWT9ix7JcmrNKH9Ydj3RSu1JO3Rv9a+CeGD9R+GZvcYFvBMTr9JcnouHYN9WR2XzOwNRW0qczadJmH6/VzPKeB/hy7RuYkvrQO9bXVnNpOTX7xDq/qukpfO3E6qeVn6GUhjF8H57UXbXWpDqyH6xvfg7nDHIi4ACbfbs/Zd4GbB7fYsV0Z2b8yIbo2p4+Fi8n0J25DkVRpnd4h4VP/KSC5aOv3ycTi6XP+q7Y+2zWMsLejFe3sgr1DH3H90ZPqaE8VO98ovsfwee8nw/bkZg80KpzpUG3vM2Fhsm7pYm5cJCUung3Pb9Anr+B9QkHNrh0rfftX+KWj5ILh4FDv33oRrzNuewAZpz3J4krxM4dUAer4MPSYY97zOSkWzew7DmUOEugnLNFFM+yWL5ILaBHq7cV/rABbsPAdglLhqqhs5BcW26ZTSvy+Wx66eKZojKz2paHudZtB+OLR9EnyCil0jv/BWe9b2sxy5mGHY3i+sPs9GNaVHM2nPckSSvMyhUkFge654teSTjEe5cWgNwzTbuFdzkDbqc7RRn2OqsojNzu3wbvsMkdG92HgijcRrOSWe7mRqJseTM3ioXaDRh2fDsVTiLqbzSv/mhu3VsJsXAFrdnW1cxjepANy8AUdX6P9hJO0p2unqDW0e0XcibdSlxOLojZx8Q3tWaoZ+qhpXJzXDOjViTM8QQv1qW+GORFWR5GUGnU7hl31JzFh7gvSbBahUnfHrOJQevf1wPvMH52K+o0lePAM0B+DvA3BkGlNVPflGFckhpRl3LyQ27JudZOYWkleo49ilDNo09ObRTo149kf9RHbtgry5J9y/UjHbe9Ir1BVvoFejI0odR+NNSyDxLyi81RdMpda3M7Z/CsLu11ftS1BSe1Z9T3171lNdpT2rupDkZaLjyRm8tSKOA4k3AGjVwIuPHmlDh2B9L2zqjsO5xT/4ct0Gnqm1G+9TyyAzmYGsYqDrKk7rAlmujWK5thcp1EVRFDJvzcgwdXmc4UPcL6y+4ZrJ6bkkp98k5nga90c0qNL7taS+n27i53HdStynuzN5XT7BZKfFDNFsJ0B1Hc7e2lE/XJ+wIh7XV+FvuXA9hw9WHSPp2k3cnNW4OmnIK9QafkcALRt4MbZXCA+2ayDtWdWMJK9yZOcVMnvDSebvOIdWp+DhomHifWGM7N4YJ43xE6yGPu689OSDwIOgex/ObmbDkv/Qs2AXoepLTFL/wutOS9mha03Gnuu4485N3IxKH2uOpBid88Evt3MlK5+4O8bSOZpzV3OY9sfREve5FdyAvd9B7CI0lw7w/K2/yOtKbQpaDaNOj5GccWpOiwBPo6r1hes5PDRnB9ey84udU6WCfmF+PNsrhO7SnlVtSfIqhaIo/HUslff+OGoYz3Z/RADvPNCaAG+38k+g1kDovYQ+340uM9cwSLOXRzVb6ao+QZTmCKx7kb9d3VirjWSZrjd7dOEoqCkoLFpzUYWKK1n6D+eaI8lG2+1JfqEOFyd1mcccTLph+NqJQvqoDzFMs437zh2Ac/oSqE7lRExhO37TRrFJ14HF3frw1qYzrD+2jY8eacOIro0B/e/mrRVHuJadT3iAJ29Eh1GoU8gr1FGo1dGlSR2C6tSy2v0K+yDJqxQrYy/y6i/62TKD6rjz/kNt6BfuZ/Z5mtTzIIta/Krty6/avgSpUnlEvYNhmq00VqfxmNNWHmMrSbr6LNf1YuHqKEA/L/zUFXGG82TeMelfVl4Bb62I46F2gZW7SQvZHJ9G37CyfzaXM/NoqTrPo5qtPKTZQX1V0VO/S+7NmZfRjd+1PbiKcf+39cf0Uyx/vy3BkLz+dziZLScv46JRM+epjtLwXkNJ8irFoDYN+HrTGaJbB/Biv1DcXSzTXpKk+POFdihfaB+hsyqeYZptDNbsJkh9mZfVK3jZaQX7dC1Ypo1itbYbGXgUO8f4RQdJTs/l5z2JJVyh6j330/5S99UlnSGaHQzTbKOVumj17cuKF79re7JM25vjuY1LfG/urcZ2wPCs40ZOPu//T18FfbFfqCSuGkySVyncnDWsnhBVbnWo4lTsU8LZVxjOe4UjuU+9j2GabUSpD9NZfZLO6pO85/Qjf+k6sUzbm226CLToE2iyhZYusxYXCrhHfZBhmq30VR/CWaVPQnmKEzG6jizTRrFF147Ccv780jKL3+f0NSe4kpVPqF9tnu/b1CrxC8cgyasM1ktcxvJw4X+6HvxP1wM/rjNEs51hmm2EqS/woGY3D2p2k6b4sFLbk2XaKOKVYJPPPW/b2fIPsgiFtqqzDNNs5SHNLnxVWYY9sbqm/Kbtw/+03UnH9JLS7b5Zt+06c5Vf9uk7p04fGiFPD2s4SV52Jg1fvtU+yLfaB2ijSmCYZhsPa3bgp7rBc06rec5pNUd0TVimjeJ3bU+uUfbK2VdLeBpnSX5c5xHNdoZpttJCXbQydIriy0ptL37TRnFaaVShc99u7wLIK9Dx1q02wKe6BtOlSZ3KBS4cniQvu6XiiNKUI4VN+bhwBH3VsQzTbOMe9YGi3vxOi9isa8cybW826jqQj3P5p7UAV/KNqrkalb6rR67izF+6zvym7c12XQQ6Kldy3X/+uuHrizduAuDn6cqbA8MrdV5RPUjysgGNWmU0LKY8BTixXteZ9brO+JLBg5pdDNNso536LAM0BxigOcB1pTb/03ZnmTaqxN78lafQUXWKRzVbeUCzGy9V0ZCnfboW/KbtzWptNzKxbheFcVFNjea8FzWXJC8bcNaYl7zudB0vftRG86M2mlDVBYZptvHIrR7pzzit5xmn9cV681dGIFcM1cKm6qIOtBeUeizX9mK5NopzStX1/h/YJqDKriXsW9W0SJfjq6++okmTJri5udG1a1f27t1b5vG//vor4eHhuLm5ERERwZo1a6ooUstwUlvmx35aacQnhcPpkfclT+dPZqW2BzcVF31vfudf2Ok6gZ+cP2aIejvulPyEUqdTOHIxnQJtUedYd3IZqt7Kz84fsd31Zd5wXkpTdQo5iivLtFEMz3+LqLzZfFb4uNUSV987hknd1rKBl3Q+FQY2L3n98ssvTJw4kblz59K1a1dmz55NdHQ08fHx+PkV7/i4c+dOhg8fzvTp03nggQdYtGgRQ4YM4cCBA7Rp08YGd1C+wREN2H32qqHx3MvNiaw8y600rUPNNl1btunaUpucYr35ozRHyFLcWKPtynJdFHt0+jaj9JwCZqw7weK9iajQ0VUVz6OaLQzS7KW2qijZ7dK2YpkuirXaSLIpeTC0pc39RyfC/986o233tarcIHVRvagUG6+p1bVrV7p06cKcOXMA0Ol0BAUF8dJLLzF58uRixz/xxBNkZ2ezatUqw7Zu3brRvn175s6dW+71MjIy8Pb2Jj09HS+vsp/UWYqiKCgK7E+8zuEL6exNuMqfR1PLf2MlBalSGarezlDNNhqr0wzbk3T1+R+9+aWgJwoqhmm2MVS9jSD1ZcMx53V+LNP2ZrkuigtK8VKQtZ2bMZiJv8Sy/GDRE8w1E6JoFVg1vzNhO6Z+Rm1a8srPz2f//v1MmTLFsE2tVtO/f3927dpV4nt27drFxIkTjbZFR0ezcuXKEo/Py8sjL6+ov1BGRkaJx1mTSqVCpYIuTerQpUkdbuTkGyUvLzcnWjbwYk/CNYteN0nx5z/aYfxHO5QuqniG3tGb/wWW8YLrMqPjMxV3Vmu7skzbm7+VMMxp9H+ud1MGtglg+YEL/N9u83v+P9C2Aa8OaMG9s7YYtn3yaFsa1/Xg8w0nAWjZQOaTF0VsmryuXLmCVqvF39+4OuDv78+JE8WnUgZISUkp8fiUlJQSj58+fTrTpk2zTMAW8kLfUHILtAxsE4CzRk1wnVr41HIxzPjZvVld/Dxd+XbrWU6nZbEmLpnsfC1RzesxsE0ABxNv8Nv+C2ZcUcXfSjh/3+rNH63exzDNVnqp41AB23Vt+E3bm790ncnFtLUIR/VowoR7m5OTX4izRo2/l36wesdgXx7p0JCR8//mvlb+vHRvc4J83fn0z3gSrmTjpFHxSIdGDLhVBdx55goxx9N4e3BLVCoV/3myPbVc9H+Wzho1E+4NxcNVQ7emMjuEMGbTauOlS5do2LAhO3fupHv37obtkyZNYsuWLezZs6fYe1xcXFi4cCHDhw83bPv666+ZNm0aqanFq2IllbyCgoKqtNpoK1qdglqln+RPpYJTaVlodQouTmqOJ2eQfPEC/zt8kQe7t6NjYx9aB3pTqFNwUqtw0ahRqZCEIaqcQ1Qb69Wrh0ajKZZ0UlNTCQgo+ZF4QECAWce7urri6lozVza+vSbi7fzT4o5l6ZvVrw1tAxk3yBaRCVF5Nu0q4eLiQqdOnYiJiTFs0+l0xMTEGJXE7tS9e3ej4wHWr19f6vFCiOrJ5l0lJk6cyMiRI+ncuTORkZHMnj2b7OxsRo8eDcAzzzxDw4YNmT59OgAvv/wyffr0YdasWQwePJglS5awb98+vv32W1vehhCiitk8eT3xxBNcvnyZd955h5SUFNq3b8+6desMjfKJiYmo7+jU2aNHDxYtWsTbb7/N1KlTad68OStXrrTbPl5CCOuweT+vqmaLfl5CCNOZ+hm1i+FBQghhLkleQgiHJMlLCOGQbN5gX9VuN/HZYpiQEKJ8tz+b5TXH17jklZmZCUBQUJCNIxFClCUzMxNvb+9S99e4p406nY5Lly7h6alfgfn2cKGkpKQa9/Sxpt57Tb1vcIx7VxSFzMxMAgMDjbpJ3a3GlbzUajWNGhVfEMLLy8tuf5nWVlPvvabeN9j/vZdV4rpNGuyFEA5JkpcQwiHV+OTl6urKu+++WyNnnqip915T7xuq173XuAZ7IUT1UONLXkIIxyTJSwjhkCR5CSEckiQvIYRDqpHJ69q1a4wYMQIvLy98fHwYO3YsWVlZZR7/0ksvERYWhru7O8HBwUyYMIH09PQqjLpiatpq5LeZc9/fffcdUVFR+Pr64uvrS//+/cv9Odkzc3/nty1ZsgSVSsWQIUOsG6ClKDXQwIEDlXbt2im7d+9Wtm3bpoSGhirDhw8v9fi4uDhl6NChyh9//KGcPn1aiYmJUZo3b64MGzasCqM235IlSxQXFxdl/vz5ytGjR5Vx48YpPj4+SmpqaonH79ixQ9FoNMq///1v5dixY8rbb7+tODs7K3FxcVUceeWYe99PPfWU8tVXXykHDx5Ujh8/rowaNUrx9vZWLly4UMWRV565935bQkKC0rBhQyUqKkp5+OGHqybYSqpxyevYsWMKoPz999+GbWvXrlVUKpVy8eJFk8+zdOlSxcXFRSkoKLBGmBYRGRmpvPjii4bvtVqtEhgYqEyfPr3E4x9//HFl8ODBRtu6du2q/POf/7RqnJZm7n3frbCwUPH09FQWLlxorRCtpiL3XlhYqPTo0UOZN2+eMnLkSIdJXjWu2rhr1y58fHzo3LmzYVv//v1Rq9UlrhNZmttT1Do52efw0Nurkffv39+wzZTVyO88HvSrkZd2vD2qyH3fLScnh4KCAurUqWOtMK2iovf+/vvv4+fnx9ixY6siTIuxz0+eFaWkpODn52e0zcnJiTp16pS66vbdrly5wgcffMBzzz1njRAtoipWI7dHFbnvu7355psEBgYWS+T2riL3vn37dr7//ntiY2OrIELLqjYlr8mTJ6NSqcp8mfrHW5aMjAwGDx5Mq1ateO+99yofuLArM2bMYMmSJaxYsQI3Nzdbh2NVmZmZPP3003z33XfUq1fP1uGYrdqUvF577TVGjRpV5jFNmzYlICCAtLQ0o+2FhYVcu3at1FW3b8vMzGTgwIF4enqyYsUKnJ2dKxu21VTFauT2qCL3fdvMmTOZMWMGGzZsoG3bttYM0yrMvfczZ85w7tw5HnzwQcM2nU4H6Gsj8fHxNGvWzLpBV4atG92q2u0G+3379hm2/fnnn+U22KenpyvdunVT+vTpo2RnZ1dFqJUWGRmpjB8/3vC9VqtVGjZsWGaD/QMPPGC0rXv37g7ZYG/OfSuKonzyySeKl5eXsmvXrqoI0WrMufebN28qcXFxRq+HH35Yueeee5S4uDglLy+vKkM3W41LXoqi7yrRoUMHZc+ePcr27duV5s2bG3WVuHDhghIWFqbs2bNHURR94uratasSERGhnD59WklOTja8CgsLbXUb5VqyZIni6uqqLFiwQDl27Jjy3HPPKT4+PkpKSoqiKIry9NNPK5MnTzYcv2PHDsXJyUmZOXOmcvz4ceXdd9912K4S5tz3jBkzFBcXF+W3334z+t1mZmba6hYqzNx7v5sjPW2skcnr6tWryvDhw5XatWsrXl5eyujRo43+UBMSEhRA2bRpk6IoirJp0yYFKPGVkJBgm5sw0ZdffqkEBwcrLi4uSmRkpLJ7927Dvj59+igjR440On7p0qVKixYtFBcXF6V169bK6tWrqzhiyzDnvhs3blzi7/bdd9+t+sAtwNzf+Z0cKXnJlDhCCIdUbZ42CiFqFkleQgiHJMlLCOGQJHkJIRySJC8hhEOS5CWEcEiSvIQQDkmSlxDCIUnyEnZh1KhRJc4EMnDgQFuHJuxUtZlVQji+gQMH8sMPPxhtK21l54KCgmKzeuTn5+Pi4mL2dSv6PmFbUvISdsPV1ZWAgACjl6+vLwAqlYpvvvmGhx56CA8PDz766CPee+892rdvz7x58wgJCTHMv5WYmMjDDz9M7dq18fLy4vHHHzeaJqa09wnHIslLOIz33nuPRx55hLi4OMaMGQPA6dOnWbZsGcuXLyc2NhadTsfDDz/MtWvX2LJlC+vXr+fs2bM88cQTRue6+33C8Ui1UdiNVatWUbt2baNtU6dOZerUqQA89dRTjB492mh/fn4+P/74I/Xr1wdg/fr1xMXFkZCQQFBQEAA//vgjrVu35u+//6ZLly4lvk84Hklewm7069ePb775xmjbnYtg3Lloym2NGzc2SkDHjx8nKCjIkLgAWrVqhY+PD8ePHzckr7vfJxyPJC9hNzw8PAgNDS1zvynbTL2WcGzS5iWqlZYtW5KUlERSUpJh27Fjx7hx4watWrWyYWTC0qTkJexGXl5esWXWnJyczFrZpn///kRERDBixAhmz55NYWEhL7zwAn369Cmx2ikcl5S8hN1Yt24dDRo0MHr16tXLrHOoVCp+//13fH196d27N/3796dp06b88ssvVopa2IpMAy2EcEhS8hJCOCRJXkIIhyTJSwjhkCR5CSEckiQvIYRDkuQlhHBIkryEEA5JkpcQwiFJ8hJCOCRJXkIIhyTJSwjhkCR5CSEc0v8HMU5L4PcKsmEAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.01794, + "end_time": "2024-03-22T20:52:10.940967", + "exception": false, + "start_time": "2024-03-22T20:52:10.923027", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4515.294993, + "end_time": "2024-03-22T20:52:13.681599", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/lct_gan/1/mlu-eval.ipynb", + "output_path": "eval/contraceptive/lct_gan/1/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/lct_gan/1", + "path_prefix": "../../../../", + "random_seed": 1, + "single_model": "lct_gan" + }, + "start_time": "2024-03-22T19:36:58.386606", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/lct_gan/model.pt b/contraceptive/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc85da0fedcef26069a7fd72d9a4088d46d773fd --- /dev/null +++ b/contraceptive/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf30df9b8181b82f551f57bee82a280b3f49fc96eb848404aabd6765c121091e +size 47605515 diff --git a/contraceptive/lct_gan/params.json b/contraceptive/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..ba36ceaba6301710cb0e8855269baf3549fec327 --- /dev/null +++ b/contraceptive/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.73, "gradient_penalty_mode": "NONE", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "head_activation": "softsign", "loss_balancer_beta": 0.67, "loss_balancer_r": 0.943, "tf_activation": "tanh", "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.09, "n_warmup_steps": 100, "Optim": "amsgradw", "fixed_role_model": "lct_gan", "mse_mag": false, "mse_mag_target": 0.65, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation_final": "leakyhardtanh", "tf_num_inds": 128, "ada_d_hid": 1024, "ada_n_layers": 9, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation_final": "leakyhardsigmoid", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/realtabformer/eval.csv b/contraceptive/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..f75a05d332b9a4a44c671acbe040618c2ce1a456 --- /dev/null +++ b/contraceptive/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.01376279273286768,0.006844550730648704,0.002682759171099557,8.117686986923218,0.21929873526096344,5.288559913635254,0.37969863414764404,2.591257907624822e-05,8.684867143630981,0.037153489887714386,0.12090718746185303,0.0517953597009182,0.09103206545114517,0.03082490712404251,16.8025541305542 diff --git a/contraceptive/realtabformer/history.csv b/contraceptive/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..dedd3944f172549c17a8c8878de73cd5e828724b --- /dev/null +++ b/contraceptive/realtabformer/history.csv @@ -0,0 +1,11 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.015436244466497252,0.6971071543745386,0.0005885604529329952,0.0,0.0,0.0,0.0,0.0,0.015436244466497252,900,225,233.85006093978882,1.0393336041768393,0.2598334010442098,0.1107698762230575,0.014223382426425814,0.39072058395795967,0.0007193372912492969,0.0,0.0,0.0,0.0,0.0,0.014223382426425814,450,113,101.00125312805176,0.8938163993632899,0.22444722917344834,0.1077716978992113 +1,0.006033640582399029,0.4650218232954108,0.00013023597284128328,0.0,0.0,0.0,0.0,0.0,0.006033640582399029,900,225,229.77366161346436,1.0212162738376194,0.25530406845940484,0.09818139906144804,0.0075027576179450585,0.3676717987151777,0.00021706114066001486,0.0,0.0,0.0,0.0,0.0,0.0075027576179450585,450,113,105.09904932975769,0.9300800825642274,0.2335534429550171,0.09515106879932954 +2,0.004512168372780757,0.38041619546195704,8.644590021956824e-05,0.0,0.0,0.0,0.0,0.0,0.004512168372780757,900,225,229.63529801368713,1.020601324505276,0.255150331126319,0.10015762750473288,0.0042034335448018585,1.679159187696908,0.00013376625657448635,0.0,0.0,0.0,0.0,0.0,0.0042034335448018585,450,113,100.8151388168335,0.8921693700604734,0.22403364181518554,0.06495650028253584 +3,0.0035539452421491863,0.47440502540713714,2.18277604056823e-05,0.0,0.0,0.0,0.0,0.0,0.0035539452421491863,900,225,229.67535948753357,1.0207793755001493,0.2551948438750373,0.10116369463089439,0.0038588620393743946,3.2039021881799323,5.9090333590049076e-05,0.0,0.0,0.0,0.0,0.0,0.0038588620393743946,450,113,102.2182068824768,0.904585901614839,0.22715157084994847,0.04514441881030056 +4,0.00319372646672289,0.37545427183122226,1.550665479323714e-05,0.0,0.0,0.0,0.0,0.0,0.00319372646672289,900,225,230.0189757347107,1.0223065588209364,0.2555766397052341,0.10247180342260334,0.0040023055252256905,3.8167529671252023,7.529611734713794e-05,0.0,0.0,0.0,0.0,0.0,0.0040023055252256905,450,113,101.59934687614441,0.8991092643906585,0.22577632639143203,0.04470131049484872 +5,0.0029433389956506693,0.4368025242733054,1.580498963409443e-05,0.0,0.0,0.0,0.0,0.0,0.0029433389956506693,900,225,229.80718541145325,1.0213652684953478,0.25534131712383695,0.1031194214626319,0.0028074885386094035,2.145103556606243,1.1650951160528297e-05,0.0,0.0,0.0,0.0,0.0,0.0028074885386094035,450,113,101.39577078819275,0.8973077060902013,0.22532393508487278,0.05228710767763576 +6,0.0026049669001885277,0.44792478884607234,1.0308605452903701e-05,0.0,0.0,0.0,0.0,0.0,0.0026049669001885277,900,225,230.53775358200073,1.0246122381422256,0.2561530595355564,0.10098279579128656,0.002687264719667534,4.189337836805609,1.241189197620803e-05,0.0,0.0,0.0,0.0,0.0,0.002687264719667534,450,113,100.64273428916931,0.8906436662758346,0.22365052064259847,0.042642332664979375 +7,0.002563537800257715,0.34648435719401377,9.35702638039536e-06,0.0,0.0,0.0,0.0,0.0,0.002563537800257715,900,225,227.69733333587646,1.011988148159451,0.25299703703986276,0.10545726788747642,0.0028965300656919784,3.1744629383202634,1.6787577942173726e-05,0.0,0.0,0.0,0.0,0.0,0.0028965300656919784,450,113,100.22019076347351,0.8869043430395885,0.22271153502994112,0.05538224283527517 +8,0.002383246352579186,0.3083862728209352,9.936972519990587e-06,0.0,0.0,0.0,0.0,0.0,0.002383246352579186,900,225,227.7537636756897,1.012238949669732,0.253059737417433,0.10515566083292166,0.0030194523966767723,3.0041199968275354,1.1394547741565071e-05,0.0,0.0,0.0,0.0,0.0,0.0030194523966767723,450,113,100.15035581588745,0.8862863346538713,0.22255634625752768,0.04854994764514432 +9,0.0021668040852819105,0.2028817977175666,7.674208641294594e-06,0.0,0.0,0.0,0.0,0.0,0.0021668040852819105,900,225,226.46019649505615,1.0064897622002495,0.2516224405500624,0.10634001894543568,0.0023035917821754184,3.5878168070930125,7.551760705609701e-06,0.0,0.0,0.0,0.0,0.0,0.0023035917821754184,450,113,100.57623744010925,0.8900551985850377,0.22350274986690946,0.05207151871749439 diff --git a/contraceptive/realtabformer/mlu-eval.ipynb b/contraceptive/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..72d1006c1da51c1d36509801eb62ac2561f48ddd --- /dev/null +++ b/contraceptive/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2335 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:43.605195Z", + "iopub.status.busy": "2024-03-22T12:35:43.604819Z", + "iopub.status.idle": "2024-03-22T12:35:43.644853Z", + "shell.execute_reply": "2024-03-22T12:35:43.644072Z" + }, + "papermill": { + "duration": 0.055356, + "end_time": "2024-03-22T12:35:43.646922", + "exception": false, + "start_time": "2024-03-22T12:35:43.591566", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:43.674064Z", + "iopub.status.busy": "2024-03-22T12:35:43.673215Z", + "iopub.status.idle": "2024-03-22T12:35:43.680255Z", + "shell.execute_reply": "2024-03-22T12:35:43.679294Z" + }, + "papermill": { + "duration": 0.023112, + "end_time": "2024-03-22T12:35:43.682214", + "exception": false, + "start_time": "2024-03-22T12:35:43.659102", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:43.706766Z", + "iopub.status.busy": "2024-03-22T12:35:43.706033Z", + "iopub.status.idle": "2024-03-22T12:35:43.710392Z", + "shell.execute_reply": "2024-03-22T12:35:43.709503Z" + }, + "papermill": { + "duration": 0.018857, + "end_time": "2024-03-22T12:35:43.712298", + "exception": false, + "start_time": "2024-03-22T12:35:43.693441", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:43.736920Z", + "iopub.status.busy": "2024-03-22T12:35:43.736376Z", + "iopub.status.idle": "2024-03-22T12:35:43.740389Z", + "shell.execute_reply": "2024-03-22T12:35:43.739499Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018587, + "end_time": "2024-03-22T12:35:43.742294", + "exception": false, + "start_time": "2024-03-22T12:35:43.723707", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:43.765878Z", + "iopub.status.busy": "2024-03-22T12:35:43.765618Z", + "iopub.status.idle": "2024-03-22T12:35:43.771408Z", + "shell.execute_reply": "2024-03-22T12:35:43.770490Z" + }, + "papermill": { + "duration": 0.019878, + "end_time": "2024-03-22T12:35:43.773220", + "exception": false, + "start_time": "2024-03-22T12:35:43.753342", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "dab922fa", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:43.800275Z", + "iopub.status.busy": "2024-03-22T12:35:43.799948Z", + "iopub.status.idle": "2024-03-22T12:35:43.805639Z", + "shell.execute_reply": "2024-03-22T12:35:43.804690Z" + }, + "papermill": { + "duration": 0.022128, + "end_time": "2024-03-22T12:35:43.807576", + "exception": false, + "start_time": "2024-03-22T12:35:43.785448", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"realtabformer\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/realtabformer/42\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011418, + "end_time": "2024-03-22T12:35:43.830446", + "exception": false, + "start_time": "2024-03-22T12:35:43.819028", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:43.854762Z", + "iopub.status.busy": "2024-03-22T12:35:43.854452Z", + "iopub.status.idle": "2024-03-22T12:35:43.864057Z", + "shell.execute_reply": "2024-03-22T12:35:43.863192Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024153, + "end_time": "2024-03-22T12:35:43.866019", + "exception": false, + "start_time": "2024-03-22T12:35:43.841866", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/realtabformer/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:43.890548Z", + "iopub.status.busy": "2024-03-22T12:35:43.890224Z", + "iopub.status.idle": "2024-03-22T12:35:45.938938Z", + "shell.execute_reply": "2024-03-22T12:35:45.937941Z" + }, + "papermill": { + "duration": 2.063269, + "end_time": "2024-03-22T12:35:45.941053", + "exception": false, + "start_time": "2024-03-22T12:35:43.877784", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:45.967664Z", + "iopub.status.busy": "2024-03-22T12:35:45.967217Z", + "iopub.status.idle": "2024-03-22T12:35:45.979942Z", + "shell.execute_reply": "2024-03-22T12:35:45.979099Z" + }, + "papermill": { + "duration": 0.028225, + "end_time": "2024-03-22T12:35:45.981873", + "exception": false, + "start_time": "2024-03-22T12:35:45.953648", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:46.006585Z", + "iopub.status.busy": "2024-03-22T12:35:46.006026Z", + "iopub.status.idle": "2024-03-22T12:35:46.015973Z", + "shell.execute_reply": "2024-03-22T12:35:46.015048Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.024645, + "end_time": "2024-03-22T12:35:46.018227", + "exception": false, + "start_time": "2024-03-22T12:35:45.993582", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:46.045285Z", + "iopub.status.busy": "2024-03-22T12:35:46.044964Z", + "iopub.status.idle": "2024-03-22T12:35:46.557585Z", + "shell.execute_reply": "2024-03-22T12:35:46.556639Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.528712, + "end_time": "2024-03-22T12:35:46.559974", + "exception": false, + "start_time": "2024-03-22T12:35:46.031262", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:46.587265Z", + "iopub.status.busy": "2024-03-22T12:35:46.586890Z", + "iopub.status.idle": "2024-03-22T12:36:00.550837Z", + "shell.execute_reply": "2024-03-22T12:36:00.549914Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 13.980286, + "end_time": "2024-03-22T12:36:00.553228", + "exception": false, + "start_time": "2024-03-22T12:35:46.572942", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 12:35:51.446205: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 12:35:51.446341: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 12:35:51.596074: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:36:00.581350Z", + "iopub.status.busy": "2024-03-22T12:36:00.580701Z", + "iopub.status.idle": "2024-03-22T12:36:00.587040Z", + "shell.execute_reply": "2024-03-22T12:36:00.586192Z" + }, + "papermill": { + "duration": 0.022543, + "end_time": "2024-03-22T12:36:00.589119", + "exception": false, + "start_time": "2024-03-22T12:36:00.566576", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:36:00.613610Z", + "iopub.status.busy": "2024-03-22T12:36:00.612976Z", + "iopub.status.idle": "2024-03-22T12:36:09.140045Z", + "shell.execute_reply": "2024-03-22T12:36:09.138949Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.542098, + "end_time": "2024-03-22T12:36:09.142632", + "exception": false, + "start_time": "2024-03-22T12:36:00.600534", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T12:36:09.171331Z", + "iopub.status.busy": "2024-03-22T12:36:09.170391Z", + "iopub.status.idle": "2024-03-22T12:36:09.178157Z", + "shell.execute_reply": "2024-03-22T12:36:09.177162Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.024242, + "end_time": "2024-03-22T12:36:09.180261", + "exception": false, + "start_time": "2024-03-22T12:36:09.156019", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:36:09.206025Z", + "iopub.status.busy": "2024-03-22T12:36:09.205673Z", + "iopub.status.idle": "2024-03-22T12:36:09.210792Z", + "shell.execute_reply": "2024-03-22T12:36:09.209904Z" + }, + "papermill": { + "duration": 0.020501, + "end_time": "2024-03-22T12:36:09.212728", + "exception": false, + "start_time": "2024-03-22T12:36:09.192227", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:36:09.237797Z", + "iopub.status.busy": "2024-03-22T12:36:09.237481Z", + "iopub.status.idle": "2024-03-22T12:44:37.517411Z", + "shell.execute_reply": "2024-03-22T12:44:37.516303Z" + }, + "papermill": { + "duration": 508.308484, + "end_time": "2024-03-22T12:44:37.533155", + "exception": false, + "start_time": "2024-03-22T12:36:09.224671", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_bs_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/bs_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_synth_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:44:37.561052Z", + "iopub.status.busy": "2024-03-22T12:44:37.560681Z", + "iopub.status.idle": "2024-03-22T12:44:38.165582Z", + "shell.execute_reply": "2024-03-22T12:44:38.164523Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.621448, + "end_time": "2024-03-22T12:44:38.167853", + "exception": false, + "start_time": "2024-03-22T12:44:37.546405", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.73,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.Softsign,\n", + " 'loss_balancer_beta': 0.67,\n", + " 'loss_balancer_r': 0.943,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.65, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:44:38.197028Z", + "iopub.status.busy": "2024-03-22T12:44:38.196103Z", + "iopub.status.idle": "2024-03-22T12:55:26.894260Z", + "shell.execute_reply": "2024-03-22T12:55:26.893195Z" + }, + "papermill": { + "duration": 648.727448, + "end_time": "2024-03-22T12:55:26.908814", + "exception": false, + "start_time": "2024-03-22T12:44:38.181366", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_train/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/contraceptive [400, 0]\n", + "Caching in ../../../../contraceptive/_cache_aug_val/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/contraceptive [0, 200]\n", + "Caching in ../../../../contraceptive/_cache_bs_train/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/contraceptive [100, 0]\n", + "Caching in ../../../../contraceptive/_cache_bs_val/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/contraceptive [0, 50]\n", + "Caching in ../../../../contraceptive/_cache_synth/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/contraceptive [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T12:55:26.938038Z", + "iopub.status.busy": "2024-03-22T12:55:26.937496Z", + "iopub.status.idle": "2024-03-22T12:55:27.377636Z", + "shell.execute_reply": "2024-03-22T12:55:27.376636Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.4571, + "end_time": "2024-03-22T12:55:27.379821", + "exception": false, + "start_time": "2024-03-22T12:55:26.922721", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n", + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:55:27.409753Z", + "iopub.status.busy": "2024-03-22T12:55:27.409425Z", + "iopub.status.idle": "2024-03-22T12:55:27.413860Z", + "shell.execute_reply": "2024-03-22T12:55:27.412935Z" + }, + "papermill": { + "duration": 0.021547, + "end_time": "2024-03-22T12:55:27.415899", + "exception": false, + "start_time": "2024-03-22T12:55:27.394352", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:55:27.443636Z", + "iopub.status.busy": "2024-03-22T12:55:27.443384Z", + "iopub.status.idle": "2024-03-22T12:55:27.450566Z", + "shell.execute_reply": "2024-03-22T12:55:27.449690Z" + }, + "papermill": { + "duration": 0.023232, + "end_time": "2024-03-22T12:55:27.452543", + "exception": false, + "start_time": "2024-03-22T12:55:27.429311", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "12536352" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:55:27.480763Z", + "iopub.status.busy": "2024-03-22T12:55:27.480493Z", + "iopub.status.idle": "2024-03-22T12:55:27.568738Z", + "shell.execute_reply": "2024-03-22T12:55:27.567842Z" + }, + "papermill": { + "duration": 0.10513, + "end_time": "2024-03-22T12:55:27.570916", + "exception": false, + "start_time": "2024-03-22T12:55:27.465786", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 16128] --\n", + "├─Adapter: 1-1 [2, 1179, 16128] --\n", + "│ └─Embedding: 2-1 [2, 1179, 24, 672] (48,384)\n", + "│ └─TensorInductionPoint: 2-2 [24, 1] 24\n", + "│ └─Sequential: 2-3 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 689,152\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 16128] (recursive)\n", + "│ └─Embedding: 2-4 [2, 294, 24, 672] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [24, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-7 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 16, 256] 1\n", + "│ └─Encoder: 2-8 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 1,048,832\n", + "│ │ │ └─Softsign: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 256] --\n", + "│ │ │ └─Linear: 4-55 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-56 [2, 256] --\n", + "│ │ └─FeedForward: 3-28 [2, 256] --\n", + "│ │ │ └─Linear: 4-57 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-58 [2, 256] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 12,584,736\n", + "Trainable params: 12,536,352\n", + "Non-trainable params: 48,384\n", + "Total mult-adds (M): 46.91\n", + "========================================================================================================================\n", + "Input size (MB): 0.28\n", + "Forward/backward pass size (MB): 755.51\n", + "Params size (MB): 50.34\n", + "Estimated Total Size (MB): 806.13\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:55:27.602224Z", + "iopub.status.busy": "2024-03-22T12:55:27.601876Z", + "iopub.status.idle": "2024-03-22T14:00:22.928909Z", + "shell.execute_reply": "2024-03-22T14:00:22.927881Z" + }, + "papermill": { + "duration": 3895.361184, + "end_time": "2024-03-22T14:00:22.946984", + "exception": false, + "start_time": "2024-03-22T12:55:27.585800", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding True True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.015436244466497252, 'avg_role_model_std_loss': 0.6971071543745386, 'avg_role_model_mean_pred_loss': 0.0005885604529329952, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.015436244466497252, 'n_size': 900, 'n_batch': 225, 'duration': 233.85006093978882, 'duration_batch': 1.0393336041768393, 'duration_size': 0.2598334010442098, 'avg_pred_std': 0.1107698762230575}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.014223382426425814, 'avg_role_model_std_loss': 0.39072058395795967, 'avg_role_model_mean_pred_loss': 0.0007193372912492969, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014223382426425814, 'n_size': 450, 'n_batch': 113, 'duration': 101.00125312805176, 'duration_batch': 0.8938163993632899, 'duration_size': 0.22444722917344834, 'avg_pred_std': 0.1077716978992113}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006033640582399029, 'avg_role_model_std_loss': 0.4650218232954108, 'avg_role_model_mean_pred_loss': 0.00013023597284128328, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006033640582399029, 'n_size': 900, 'n_batch': 225, 'duration': 229.77366161346436, 'duration_batch': 1.0212162738376194, 'duration_size': 0.25530406845940484, 'avg_pred_std': 0.09818139906144804}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0075027576179450585, 'avg_role_model_std_loss': 0.3676717987151777, 'avg_role_model_mean_pred_loss': 0.00021706114066001486, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0075027576179450585, 'n_size': 450, 'n_batch': 113, 'duration': 105.09904932975769, 'duration_batch': 0.9300800825642274, 'duration_size': 0.2335534429550171, 'avg_pred_std': 0.09515106879932954}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004512168372780757, 'avg_role_model_std_loss': 0.38041619546195704, 'avg_role_model_mean_pred_loss': 8.644590021956824e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004512168372780757, 'n_size': 900, 'n_batch': 225, 'duration': 229.63529801368713, 'duration_batch': 1.020601324505276, 'duration_size': 0.255150331126319, 'avg_pred_std': 0.10015762750473288}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0042034335448018585, 'avg_role_model_std_loss': 1.679159187696908, 'avg_role_model_mean_pred_loss': 0.00013376625657448635, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0042034335448018585, 'n_size': 450, 'n_batch': 113, 'duration': 100.8151388168335, 'duration_batch': 0.8921693700604734, 'duration_size': 0.22403364181518554, 'avg_pred_std': 0.06495650028253584}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0035539452421491863, 'avg_role_model_std_loss': 0.47440502540713714, 'avg_role_model_mean_pred_loss': 2.18277604056823e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0035539452421491863, 'n_size': 900, 'n_batch': 225, 'duration': 229.67535948753357, 'duration_batch': 1.0207793755001493, 'duration_size': 0.2551948438750373, 'avg_pred_std': 0.10116369463089439}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0038588620393743946, 'avg_role_model_std_loss': 3.2039021881799323, 'avg_role_model_mean_pred_loss': 5.9090333590049076e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0038588620393743946, 'n_size': 450, 'n_batch': 113, 'duration': 102.2182068824768, 'duration_batch': 0.904585901614839, 'duration_size': 0.22715157084994847, 'avg_pred_std': 0.04514441881030056}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00319372646672289, 'avg_role_model_std_loss': 0.37545427183122226, 'avg_role_model_mean_pred_loss': 1.550665479323714e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00319372646672289, 'n_size': 900, 'n_batch': 225, 'duration': 230.0189757347107, 'duration_batch': 1.0223065588209364, 'duration_size': 0.2555766397052341, 'avg_pred_std': 0.10247180342260334}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0040023055252256905, 'avg_role_model_std_loss': 3.8167529671252023, 'avg_role_model_mean_pred_loss': 7.529611734713794e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0040023055252256905, 'n_size': 450, 'n_batch': 113, 'duration': 101.59934687614441, 'duration_batch': 0.8991092643906585, 'duration_size': 0.22577632639143203, 'avg_pred_std': 0.04470131049484872}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0029433389956506693, 'avg_role_model_std_loss': 0.4368025242733054, 'avg_role_model_mean_pred_loss': 1.580498963409443e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029433389956506693, 'n_size': 900, 'n_batch': 225, 'duration': 229.80718541145325, 'duration_batch': 1.0213652684953478, 'duration_size': 0.25534131712383695, 'avg_pred_std': 0.1031194214626319}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0028074885386094035, 'avg_role_model_std_loss': 2.145103556606243, 'avg_role_model_mean_pred_loss': 1.1650951160528297e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0028074885386094035, 'n_size': 450, 'n_batch': 113, 'duration': 101.39577078819275, 'duration_batch': 0.8973077060902013, 'duration_size': 0.22532393508487278, 'avg_pred_std': 0.05228710767763576}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0026049669001885277, 'avg_role_model_std_loss': 0.44792478884607234, 'avg_role_model_mean_pred_loss': 1.0308605452903701e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026049669001885277, 'n_size': 900, 'n_batch': 225, 'duration': 230.53775358200073, 'duration_batch': 1.0246122381422256, 'duration_size': 0.2561530595355564, 'avg_pred_std': 0.10098279579128656}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002687264719667534, 'avg_role_model_std_loss': 4.189337836805609, 'avg_role_model_mean_pred_loss': 1.241189197620803e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002687264719667534, 'n_size': 450, 'n_batch': 113, 'duration': 100.64273428916931, 'duration_batch': 0.8906436662758346, 'duration_size': 0.22365052064259847, 'avg_pred_std': 0.042642332664979375}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002563537800257715, 'avg_role_model_std_loss': 0.34648435719401377, 'avg_role_model_mean_pred_loss': 9.35702638039536e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002563537800257715, 'n_size': 900, 'n_batch': 225, 'duration': 227.69733333587646, 'duration_batch': 1.011988148159451, 'duration_size': 0.25299703703986276, 'avg_pred_std': 0.10545726788747642}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0028965300656919784, 'avg_role_model_std_loss': 3.1744629383202634, 'avg_role_model_mean_pred_loss': 1.6787577942173726e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0028965300656919784, 'n_size': 450, 'n_batch': 113, 'duration': 100.22019076347351, 'duration_batch': 0.8869043430395885, 'duration_size': 0.22271153502994112, 'avg_pred_std': 0.05538224283527517}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002383246352579186, 'avg_role_model_std_loss': 0.3083862728209352, 'avg_role_model_mean_pred_loss': 9.936972519990587e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002383246352579186, 'n_size': 900, 'n_batch': 225, 'duration': 227.7537636756897, 'duration_batch': 1.012238949669732, 'duration_size': 0.253059737417433, 'avg_pred_std': 0.10515566083292166}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0030194523966767723, 'avg_role_model_std_loss': 3.0041199968275354, 'avg_role_model_mean_pred_loss': 1.1394547741565071e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0030194523966767723, 'n_size': 450, 'n_batch': 113, 'duration': 100.15035581588745, 'duration_batch': 0.8862863346538713, 'duration_size': 0.22255634625752768, 'avg_pred_std': 0.04854994764514432}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0021668040852819105, 'avg_role_model_std_loss': 0.2028817977175666, 'avg_role_model_mean_pred_loss': 7.674208641294594e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021668040852819105, 'n_size': 900, 'n_batch': 225, 'duration': 226.46019649505615, 'duration_batch': 1.0064897622002495, 'duration_size': 0.2516224405500624, 'avg_pred_std': 0.10634001894543568}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023035917821754184, 'avg_role_model_std_loss': 3.5878168070930125, 'avg_role_model_mean_pred_loss': 7.551760705609701e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023035917821754184, 'n_size': 450, 'n_batch': 113, 'duration': 100.57623744010925, 'duration_batch': 0.8900551985850377, 'duration_size': 0.22350274986690946, 'avg_pred_std': 0.05207151871749439}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0022073838575144894, 'avg_role_model_std_loss': 0.18685059259163805, 'avg_role_model_mean_pred_loss': 1.00230588112276e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022073838575144894, 'n_size': 900, 'n_batch': 225, 'duration': 227.7475564479828, 'duration_batch': 1.0122113619910347, 'duration_size': 0.25305284049775867, 'avg_pred_std': 0.10737030942820841}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0024145089968806133, 'avg_role_model_std_loss': 2.368802141273148, 'avg_role_model_mean_pred_loss': 1.0890554641507788e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024145089968806133, 'n_size': 450, 'n_batch': 113, 'duration': 99.10306596755981, 'duration_batch': 0.8770182828987594, 'duration_size': 0.22022903548346626, 'avg_pred_std': 0.04590538341193501}\n", + "Time out: 3648.574878692627/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.002682760669579563, 'avg_g_mag_loss': 0.008364482207499076, 'avg_g_cos_loss': 0.010354253430961675, 'pred_duration': 8.742810010910034, 'grad_duration': 8.114531993865967, 'total_duration': 16.857342004776, 'pred_std': 0.09103205054998398, 'std_loss': 0.0308249332010746, 'mean_pred_loss': 2.5912582714227028e-05, 'pred_rmse': 0.05179537460207939, 'pred_mae': 0.037153515964746475, 'pred_mape': 0.1209072470664978, 'grad_rmse': 0.3796982765197754, 'grad_mae': 0.21929873526096344, 'grad_mape': 5.288557529449463}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.002682760669579563, 'avg_g_mag_loss': 0.008364482207499076, 'avg_g_cos_loss': 0.010354253430961675, 'avg_pred_duration': 8.742810010910034, 'avg_grad_duration': 8.114531993865967, 'avg_total_duration': 16.857342004776, 'avg_pred_std': 0.09103205054998398, 'avg_std_loss': 0.0308249332010746, 'avg_mean_pred_loss': 2.5912582714227028e-05}, 'min_metrics': {'avg_loss': 0.002682760669579563, 'avg_g_mag_loss': 0.008364482207499076, 'avg_g_cos_loss': 0.010354253430961675, 'pred_duration': 8.742810010910034, 'grad_duration': 8.114531993865967, 'total_duration': 16.857342004776, 'pred_std': 0.09103205054998398, 'std_loss': 0.0308249332010746, 'mean_pred_loss': 2.5912582714227028e-05, 'pred_rmse': 0.05179537460207939, 'pred_mae': 0.037153515964746475, 'pred_mape': 0.1209072470664978, 'grad_rmse': 0.3796982765197754, 'grad_mae': 0.21929873526096344, 'grad_mape': 5.288557529449463}, 'model_metrics': {'realtabformer': {'avg_loss': 0.002682760669579563, 'avg_g_mag_loss': 0.008364482207499076, 'avg_g_cos_loss': 0.010354253430961675, 'pred_duration': 8.742810010910034, 'grad_duration': 8.114531993865967, 'total_duration': 16.857342004776, 'pred_std': 0.09103205054998398, 'std_loss': 0.0308249332010746, 'mean_pred_loss': 2.5912582714227028e-05, 'pred_rmse': 0.05179537460207939, 'pred_mae': 0.037153515964746475, 'pred_mape': 0.1209072470664978, 'grad_rmse': 0.3796982765197754, 'grad_mae': 0.21929873526096344, 'grad_mape': 5.288557529449463}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:00:22.983471Z", + "iopub.status.busy": "2024-03-22T14:00:22.983097Z", + "iopub.status.idle": "2024-03-22T14:00:22.987681Z", + "shell.execute_reply": "2024-03-22T14:00:22.986775Z" + }, + "papermill": { + "duration": 0.02542, + "end_time": "2024-03-22T14:00:22.989686", + "exception": false, + "start_time": "2024-03-22T14:00:22.964266", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:00:23.023599Z", + "iopub.status.busy": "2024-03-22T14:00:23.023288Z", + "iopub.status.idle": "2024-03-22T14:00:23.126595Z", + "shell.execute_reply": "2024-03-22T14:00:23.125728Z" + }, + "papermill": { + "duration": 0.122795, + "end_time": "2024-03-22T14:00:23.129060", + "exception": false, + "start_time": "2024-03-22T14:00:23.006265", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:00:23.165615Z", + "iopub.status.busy": "2024-03-22T14:00:23.165292Z", + "iopub.status.idle": "2024-03-22T14:00:23.485004Z", + "shell.execute_reply": "2024-03-22T14:00:23.484004Z" + }, + "papermill": { + "duration": 0.34071, + "end_time": "2024-03-22T14:00:23.487074", + "exception": false, + "start_time": "2024-03-22T14:00:23.146364", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAEWCAYAAAAzXsGAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABESUlEQVR4nO3de1xUdf4/8NfMwMxwHy4yA8jNC2JKaiCEmVayUl8qKTfLddVcU2uxLHJTSqXdfkWZbW5qa5dNazdDLa1WzSLU2hRRuSikkhcUFAdEZLjfZt6/Pw6MDgwwg8Aw8H4+Hucx8Dmfc857jHn3mc/5nM9HREQExhizEmJLB8AYY+bgpMUYsyqctBhjVoWTFmPMqnDSYoxZFU5ajDGrwkmLMWZVOGkxxqwKJy3GmFXhpMUYsypdSlobNmxAQEAA5HI5IiIicOTIkQ7rb9++HcHBwZDL5QgJCcGePXsM9u/YsQNTp06Fu7s7RCIRsrOzjZ4nLS0N9913HxwcHODs7IxJkyahtra2K2+BMWalbMw9YOvWrYiPj8fGjRsRERGBtWvXIjo6Gnl5efD09GxT/9ChQ5g5cyaSkpLw4IMPYsuWLYiNjUVmZiZGjx4NAKiursbEiRMxY8YMLFiwwOh109LScP/99yMhIQHr1q2DjY0Njh8/DrHYtLyr0+lQVFQEJycniEQic982Y6yHEREqKyvh7e3d8eeazBQeHk5xcXH637VaLXl7e1NSUpLR+jNmzKCYmBiDsoiICFq0aFGbuvn5+QSAsrKy2uyLiIigFStWmBuuXmFhIQHgjTfe+vhWWFjY4WfZrJZWQ0MDMjIykJCQoC8Ti8WIiopCWlqa0WPS0tIQHx9vUBYdHY2vv/7a5OuWlJQgPT0ds2bNwoQJE3Du3DkEBwfj9ddfx8SJE40eU19fj/r6ev3v1DyZRWFhIZydnU2+NmOsd1RUVMDX1xdOTk4d1jMraZWWlkKr1UKpVBqUK5VKnD592ugxarXaaH21Wm3ydc+fPw8AePXVV7FmzRqMHTsWn332GaZMmYLc3FwMHz68zTFJSUn461//2qbc2dmZkxZjfVhn3TdWcfdQp9MBABYtWoR58+Zh3LhxePfddzFixAh88sknRo9JSEiARqPRb4WFhb0ZMmOsh5jV0vLw8IBEIkFxcbFBeXFxMVQqldFjVCqVWfWN8fLyAgDcdtttBuUjR45EQUGB0WNkMhlkMpnJ12CMWQezWlpSqRShoaFITU3Vl+l0OqSmpiIyMtLoMZGRkQb1ASAlJaXd+sYEBATA29sbeXl5BuW//fYb/P39zXgHjDFrZ/aQh/j4eMydOxdhYWEIDw/H2rVrUV1djXnz5gEA5syZAx8fHyQlJQEAlixZgsmTJ+Odd95BTEwMkpOTcezYMXz44Yf6c5aVlaGgoABFRUUAoE9OKpUKKpUKIpEIf/nLX5CYmIgxY8Zg7Nix+PTTT3H69Gl8+eWXt/yPwDqn1WrR2Nho6TCYFbO1tYVEIrnl85idtB5//HFcvXoVq1atglqtxtixY7F37159Z3tBQYHBGIsJEyZgy5YtWLFiBV5++WUMHz4cX3/9tX6MFgB8++23+qQHAE888QQAIDExEa+++ioA4Pnnn0ddXR1eeOEFlJWVYcyYMUhJScHQoUO79MaZaYgIarUa5eXllg6F9QMKhULfEOkqEdHAWNiioqICLi4u0Gg0fPfQDFeuXEF5eTk8PT1hb2/PA3NZlxARampqUFJSAoVCoe+nvpmpn1GzW1r9XV2jFqeuVOB6TQPuC1Z2fkA/ptVq9QnL3d3d0uEwK2dnZwdAGHfp6enZ5a+KVjHkoTedu1qFR94/hPhtxzFAGqHtaunDsre3t3AkrL9o+Vu6lf5RTlqtDPFwhEgElNc0oqy6wdLh9An8lZB1l+74W+Kk1YqdVAIfhdCMPXe12sLRMMZa46RlxNBBjgCAsyVVFo6EMdYaJy0jhnkKSevcVU5arGtEIpFZkwJYg1dffRVjx461dBictIzhlhbrDzZv3gyFQtFt51u6dGmbp1ssgYc8GMEtLTaQNDQ0QCqVdlrP0dERjo6OvRBRx7ilZcTQQQ4AgMvltaht0Fo4mr6FiFDT0NTrm7nDT/bu3YuJEydCoVDA3d0dDz74IM6dOwdAeEpj2bJlBvWvXr0KW1tb/PzzzwCEQbUxMTGws7NDYGAgtmzZgoCAAKxdu7ZL/245OTm47777YGdnB3d3dyxcuBBVVTf+p3jgwAGEh4fDwcEBCoUCd911Fy5evAgAOH78OO699144OTnB2dkZoaGhOHbsWIfXO3DgAObNmweNRgORSASRSKR/uiQgIACvvfYa5syZA2dnZyxcuBAAsGzZMgQFBcHe3h5DhgzBypUrDYYmtP56+OSTTyI2NhZr1qyBl5cX3N3dERcX1+OPe3FLywg3BykU9rYor2nE+dIqjPJ2sXRIfUZtoxa3rfq+16978m/RsJea/udaXV2N+Ph43H777aiqqsKqVavwyCOPIDs7G7NmzcLq1avx5ptv6m/Bb926Fd7e3rj77rsBCM/QlpaW4sCBA7C1tUV8fDxKSkq6FHt1dTWio6MRGRmJo0ePoqSkBE899RQWL16MzZs3o6mpCbGxsViwYAG++OILNDQ04MiRI/rYZs2ahXHjxuGf//wnJBIJsrOzYWtr2+E1J0yYgLVr12LVqlX6Z3lvbiWtWbMGq1atQmJior7MyckJmzdvhre3N3JycrBgwQI4OTnhpZdeavc6+/fvh5eXF/bv34+zZ8/i8ccfx9ixY9udNr07cNIyQiQSYdggRxy7eB3nrlZz0rJC06dPN/j9k08+waBBg3Dy5EnMmDEDzz//PH755Rd9ktqyZQtmzpwJkUiE06dP48cff8TRo0cRFhYGAPj444+NTjZpii1btqCurg6fffYZHByEVvz69evx0EMP4a233oKtrS00Gg0efPBB/bO0I0eO1B9fUFCAv/zlLwgODgYAk+KQSqVwcXGBSCQyOg3UfffdhxdffNGgbMWKFfqfAwICsHTpUiQnJ3eYtFxdXbF+/XpIJBIEBwcjJiYGqampnLQsYWhz0uLOeEN2thKc/Fu0Ra5rjjNnzmDVqlVIT09HaWmpfiLJgoICjB49GlOnTsXnn3+Ou+++G/n5+UhLS8MHH3wAQJhlxMbGBnfccYf+fMOGDYOrq2uXYj916hTGjBmjT1gAcNddd0Gn0yEvLw+TJk3Ck08+iejoaPzud79DVFQUZsyYoX8+Lz4+Hk899RT+/e9/IyoqCo899tgtTxTQkoxvtnXrVrz33ns4d+4cqqqq0NTU1OlzuqNGjTJ4HMfLyws5OTm3FFtnuE+rHdwZb5xIJIK91KbXN3NHUj/00EMoKyvDRx99hPT0dKSnpwMQOp0B4SvXl19+icbGRmzZsgUhISEICQnp9n8vU23atAlpaWmYMGECtm7diqCgIBw+fBiA0Jf066+/IiYmBvv27cNtt92GnTt33tL1bk6ggLCWw6xZs/B///d/2LVrF7KysvDKK6/o/73a0/prqkgk0v8Poqdw0mrHUE/hP+o5bmlZnWvXriEvLw8rVqzAlClTMHLkSFy/ft2gzrRp01BXV4e9e/diy5YtmDVrln7fiBEj0NTUhKysLH3Z2bNn25zDVCNHjsTx48dRXX3jCYuDBw9CLBZjxIgR+rJx48YhISEBhw4dwujRo7Flyxb9vqCgILzwwgv44Ycf8Oijj2LTpk2dXlcqlUKrNe1G0qFDh+Dv749XXnkFYWFhGD58uP5GQF/DSasdLWO1zpdWQ6sb2A9OWxtXV1e4u7vjww8/xNmzZ7Fv3742K0I5ODggNjYWK1euxKlTpzBz5kz9vuDgYERFRWHhwoU4cuQIsrKysHDhQtjZ2XXp2blZs2ZBLpdj7ty5yM3Nxf79+/Hss89i9uzZUCqVyM/PR0JCAtLS0nDx4kX88MMPOHPmDEaOHIna2losXrwYBw4cwMWLF3Hw4EEcPXrUoM+rPQEBAaiqqkJqaipKS0tRU1PTbt3hw4ejoKAAycnJOHfuHN57771bbs31FE5a7Rjsag+pjRgNTTpcvs6rWFsTsViM5ORkZGRkYPTo0XjhhRfw9ttvt6k3a9YsHD9+HHfffTf8/PwM9n322WdQKpWYNGkSHnnkEf2dNLlcbnY89vb2+P7771FWVobx48fj97//PaZMmYL169fr958+fRrTp09HUFAQFi5ciLi4OCxatAgSiQTXrl3DnDlzEBQUhBkzZuCBBx4wutJUaxMmTMDTTz+Nxx9/HIMGDcLq1avbrfvwww/jhRdewOLFizF27FgcOnQIK1euNPu99gaeBLAD96/9GafVlfjkybABObdWXV0d8vPzERgY2KUPa39y6dIl+Pr64scff8SUKVMsHY7V6uhvytTPKLe0OjC0pTO+hGd7GGj27duHb7/9Fvn5+Th06BCeeOIJBAQEYNKkSZYObcDjpNUBfgZx4GpsbMTLL7+MUaNG4ZFHHsGgQYP0A00///xz/SMtrbdRo0b1WowPPPBAu3G88cYbvRZHb+NxWsY01gGaS/rHeXjYw8ATHR2N6Gjj49EefvhhREREGN3X2Uj17vTxxx+jttZ4f6ubm1uvxdHbOGm1dikD+CQacPLCsMd/AcBJixlycnKCk5OTpcOAj4+PpUOwiC59PdywYQMCAgIgl8sRERGBI0eOdFh/+/btCA4OhlwuR0hICPbs2WOwf8eOHZg6dSrc3d0hEomQnZ3d7rmICA888EDPzVc0KAjQNQGaAgyRV0EkAq7XNOJaVX33X4sxZjazk9bWrVsRHx+PxMREZGZmYsyYMYiOjm73YdJDhw5h5syZmD9/PrKyshAbG4vY2Fjk5ubq61RXV2PixIl46623Or3+2rVre3bOcpkT4HkbAMCuOJOnXmasryEzhYeHU1xcnP53rVZL3t7elJSUZLT+jBkzKCYmxqAsIiKCFi1a1KZufn4+AaCsrCyj58rKyiIfHx+6cuUKAaCdO3eaHLdGoyEApNFoOq/87XNEic5E36+gOf9KJ/9lu+jzwxdNvlZ/UVtbSydPnqTa2lpLh8L6iY7+pkz9jJrV0mpoaEBGRgaioqL0ZWKxGFFRUUhLSzN6TFpamkF9QOjkbK9+e2pqavCHP/wBGzZsMPrUemv19fWoqKgw2Ew2eLzweumY/g4i92sx1jeYlbRKS0uh1WqhVBoOtFQqlVCr1UaPUavVZtVvzwsvvIAJEyZg2rRpJtVPSkqCi4uLfvP19TX9YoPDhdeiLAz3kAHgpMVYX2EV47S+/fZb7Nu3z6xZIxMSEqDRaPRbYWGh6Rd0HwbIFUBTLUbbCMfxWC1mjv64sEVfYVbS8vDwgEQiQXFxsUF5cXFxu1/ZVCqVWfWN2bdvH86dOweFQgEbGxvY2AgjNaZPn4577rnH6DEymQzOzs4Gm8nEYmCwMN/QkNqTAHjqZWZ9unthC0CYxlkkEqG8vLxbz2sOs5KWVCpFaGiowYocOp0OqampiIyMNHpMZGRkmxU8UlJS2q1vzPLly3HixAlkZ2frNwB49913TZqio0uavyLaX82Cwt4WRMD5Um5tMWZpZn89jI+Px0cffYRPP/0Up06dwjPPPIPq6mrMmzcPgDC3dkJCgr7+kiVLsHfvXrzzzjs4ffo0Xn31VRw7dgyLFy/W1ykrK0N2djZOnhRaNXl5ecjOztb3e6lUKowePdpgAwA/Pz8EBgZ2/d13pLmlJbp05KbOeB72ACKgobr3N17YotsWtqivr8fSpUvh4+MDBwcHRERE4MCBA/pjL168iIceegiurq5wcHDAqFGjsGfPHly4cAH33nsvAGH6H5FIhCeffLJL/x63wuwR8Y8//jiuXr2KVatWQa1WY+zYsdi7d6++s72goABi8Y1cOGHCBGzZsgUrVqzAyy+/jOHDh+Prr7/WJx5A6LNqSXoA8MQTTwAAEhMT9f/QvW5wGAARcP0CxgQ3IuMiTwgIAGisAd7w7v3rvlwESB06r9eMF7Zof2GLxYsX4+TJk0hOToa3tzd27tyJ+++/Hzk5ORg+fDji4uLQ0NCAn3/+GQ4ODjh58iQcHR3h6+uLr776CtOnT0deXh6cnZ1hZ2fXpX+TW9Glx3gWL15s0FK62c0Zu8Vjjz2Gxx57rN3zPfnkk2ZnbOrpGXXkLsCgEcDV04iwPYdPoMRZvoNoNXhhC+MLWxQUFGDTpk0oKCiAt7fwP5+lS5di79692LRpE9544w0UFBRg+vTp+umnhwwZoj++5ZlGT0/Pbu8vMxU/e9iRweOBq6cxoukUACW3tADA1l5o9VjiumbghS2My8nJgVarRVBQkEF5fX093N3dAQDPPfccnnnmGfzwww+IiorC9OnTcfvtt3fpej3BKoY8WEzzIFNVhfDIEU+9DEAkEr6m9fbGC1t0y8IWVVVVkEgkyMjIMLixderUKfzjH/8AADz11FM4f/48Zs+ejZycHISFhWHdunXd9l5vFSetjvgKdxBlJdmwsyGeetlK8MIWAmMLW4wbNw5arRYlJSUYNmyYwXbz10hfX188/fTT2LFjB1588UV89NFH+nMCMHnBjJ7ASasjHiMAmTNEjTW4V1EKgEfGWwNe2EJgbGGLoKAgzJo1C3PmzMGOHTuQn5+PI0eOICkpCbt37wYAPP/88/j++++Rn5+PzMxM7N+/X389f39/iEQi7Nq1C1evXjW4A9preui5yD7HrAemb/bpNKJEZ/pi/SryX7aLPvzpXI/E1xdZ8wPTKSkpNHLkSJLJZHT77bfTgQMH2jxkv2fPHgJAkyZNanN8UVERPfDAAySTycjf35+2bNlCnp6etHHjRpOu3/paJ06coHvvvZfkcjm5ubnRggULqLKykoiI1Go1xcbGkpeXF0mlUvL396dVq1aRVqul+vp6euKJJ8jX15ekUil5e3vT4sWLTf5v8vTTT5O7uzsBoMTERCIiamhooFWrVlFAQADZ2tqSl5cXPfLII3TixAkiIlq8eDENHTqUZDIZDRo0iGbPnk2lpaX6c/7tb38jlUpFIpGI5s6da1IcLbrjgWle2KIz+14Hfl6NXwfFIKZwFp4Y74s3p/edTsmexAtb3MALW3SP7ljYgu8edqa5X8u/RuiM52cQB4Z9+/ahqqoKISEhuHLlCl566SVe2KKP4D6tzviEAgAcqy9CgUru0xogeGGLvotbWp2xdwPchwPXzmCc+Cz214xDWXUD3Byklo6M9SBe2KLv4qRlisHjgWtnMNnuAvZXj8PZkiqEB/bfPwrWMV7YwrL466EpfIVBpuNtzgIYeMMeBsi9GtYLuuNviZOWKZqnqRnWmAcxdAOmM77lq05NTY2FI2H9Rcvf0q18jeavh6bwHAlIHSFrqMJw0SWcu6rs/Jh+QCKRQKFQ6Gc3sLe379mVkFi/RUSoqalBSUkJFAoFJBJJl8/FScsUYgngcweQ/zPGic/i4NURnR/TT7Q82tHVaVkYu5lCoTBr1mJjOGmZavB4IP9n3CE6g63Xa1HXqIXctuv/t7AWIpEIXl5e8PT0RGNjo6XDYVbM1tb2llpYLThpmaq5XyvM5iyoCTh/tRq3eZsxst7KSSSSbvmDY+xWcUe8qVoWusBlOKOKJwRkzEI4aZnKwQNwE2ZwHCc+xxMCMmYhnLTM0Twp4DjxGW5pMWYhnLTM0Zy07hCd4ZYWYxbSpaS1YcMGBAQEQC6XIyIiAkeOHOmw/vbt2xEcHAy5XI6QkBDs2bPHYP+OHTswdepUuLu7QyQS6dc1bFFWVoZnn30WI0aMgJ2dHfz8/PDcc89Bo9F0Jfyua05aY8XncKG0kqdeZswCzE5aW7duRXx8PBITE5GZmYkxY8YgOjq63XE8hw4dwsyZMzF//nxkZWUhNjYWsbGxyM3N1deprq7GxIkT8dZbbxk9R1FREYqKirBmzRrk5uZi8+bN2Lt3L+bPn29u+LdGORpkYwdnUQ0Gay/x1MuMWYJZ0w4SUXh4OMXFxel/12q15O3tTUlJSUbrz5gxg2JiYgzKIiIiaNGiRW3q5ufnEwDKysrqNI5t27aRVCqlxsZGk+Lu8sylrX3yAFGiMy19eSntO1V8a+dijOmZ+hk1q6XV0NCAjIwMREVF6cvEYjGioqKQlpZm9Ji0tDSD+oAw7Ud79U3VMruhjY3xoWb19fWoqKgw2LrFTf1aA+UZRMb6ErOSVmlpKbRarX416RZKpVK/hH1rarXarPqmxvHaa69h4cKF7dZJSkqCi4uLfvP19e3y9Qzo7yCeHXCzPTDWF1jd3cOKigrExMTgtttuw6uvvtpuvYSEBGg0Gv1WWFjYPQE0T78cJLqEK8XF3XNOxpjJzHqMx8PDAxKJBMWtPqzFxcXtPgSpUqnMqt+RyspK3H///XBycsLOnTs7nN5CJpNBJpOZfY1OOXqiwckX0spC2F09DuB33X8Nxli7zGppSaVShIaGIjU1VV+m0+mQmpqKyMhIo8dERkYa1AeAlJSUduu3p6KiAlOnToVUKsW3335r0dVhxM2treENp1BW3WCxOBgbiMx+YDo+Ph5z585FWFgYwsPDsXbtWlRXV2PevHkAgDlz5sDHxwdJSUkAgCVLlmDy5Ml45513EBMTg+TkZBw7dgwffvih/pxlZWUoKChAUVERACAvLw+A0EpTqVT6hFVTU4P//Oc/Bh3rgwYN6vUHeW38I4CTX2Gc+CxPvcxYb+vKrcl169aRn58fSaVSCg8Pp8OHD+v3TZ48uc0Cjtu2baOgoCCSSqU0atQo2r17t8H+TZs2EYA2W8vikvv37ze6HwDl5+ebFHO3DXkgIrp0jCjRmcpWedOWwxdu/XyMMV6stbUuL9ZqTFMDGl/3gS014P3RW/Hn39/fPUEyNoCZ+hm1uruHfYKNFNcVwvp2tleOWTgYxgYWTlpd1OQlzK81SHPcwpEwNrBw0uoix2F3AgCCGvJQ16i1cDSMDRyctLrIadgEAMAIUQEuXOZBpoz1Fk5aXSRy9sZV8SBIRISys+mWDoexAYOT1i0ochwNANAVdDyfGGOs+3DSugU1nuMAAC7Xsi0bCGMDCCetW2DjL3TG+9bkAgNjuBtjFsdJ6xYMChqPerKBgiqgvZZv6XAYGxA4ad0C30GuOEmBAIDy336xcDSMDQyctG6BRCxCvnwkAKA2n+8gMtYbOGndoutuYwEAcnWGZQNhbIDgpHWLtD7C9Muulb8BDTUWjoax/o+T1i1SDh4CNblCAi1QlGXpcBjr9zhp3aKhnk7I1A0Xfrl01LLBMDYAcNK6RUMGOSBLNwwA0HCBO+MZ62mctG6RvdQGhQ7C4zy4fJQHmTLWwzhpdYNGz9vRSBJIa68C5QWWDoexfo2TVjfwU7rhV/IXfuF+LcZ6FCetbjDM0xFZ3BnPWK/gpNUNhg66KWkV8jQ1jPWkLiWtDRs2ICAgAHK5HBEREThypOMP6vbt2xEcHAy5XI6QkBDs2bPHYP+OHTswdepUuLu7QyQSITs7u8056urqEBcXB3d3dzg6OmL69OltVq62lGGejsgk4Q4iqU8AjbUWjoix/svspLV161bEx8cjMTERmZmZGDNmDKKjo1FSUmK0/qFDhzBz5kzMnz8fWVlZiI2NRWxsLHJzc/V1qqurMXHiRLz11lvtXveFF17Af//7X2zfvh0//fQTioqK8Oijj5obfo9wd5CiUuaNq+QCka4JuMKLXTDWY8xdUDE8PJzi4uL0v2u1WvL29qakpCSj9WfMmEExMTEGZREREbRo0aI2dfPz8wkAZWVlGZSXl5eTra0tbd++XV926tQpAkBpaWkmxd2ti7Ua8ciGX+j7FfcSJToTHXyvR67BWH9m6mfUrJZWQ0MDMjIyEBUVpS8Ti8WIiopCWlqa0WPS0tIM6gNAdHR0u/WNycjIQGNjo8F5goOD4efn1+556uvrUVFRYbD1pGGejjdGxnO/FmM9xqykVVpaCq1WC6VSaVCuVCqhVquNHqNWq82q3945pFIpFAqFyedJSkqCi4uLfvP19TX5el1h0Bl/iRdwZayn9Nu7hwkJCdBoNPqtsLCwR683zNMRJygQWoiByiJAc6lHr8fYQGVW0vLw8IBEImlz1664uBgqlcroMSqVyqz67Z2joaEB5eXlJp9HJpPB2dnZYOtJQwc5ohZynOJBpoz1KLOSllQqRWhoKFJTU/VlOp0OqampiIyMNHpMZGSkQX0ASElJabe+MaGhobC1tTU4T15eHgoKCsw6T0/ydbOHVCJGplYY+oBCTlqM9QQbcw+Ij4/H3LlzERYWhvDwcKxduxbV1dWYN28eAGDOnDnw8fFBUlISAGDJkiWYPHky3nnnHcTExCA5ORnHjh3Dhx9+qD9nWVkZCgoKUFRUBEBISIDQwlKpVHBxccH8+fMRHx8PNzc3ODs749lnn0VkZCTuvPPOW/5H6A4SsQiBHg7IvDocc5DCLS3GekpXbk2uW7eO/Pz8SCqVUnh4OB0+fFi/b/LkyTR37lyD+tu2baOgoCCSSqU0atQo2r17t8H+TZs2EYA2W2Jior5ObW0t/fnPfyZXV1eyt7enRx55hK5cuWJyzD095IGI6Jn/HKNJyz8Whj38zYOosa7HrsVYf2PqZ1RENDDmUqmoqICLiws0Gk2P9W/9/Yc8vLfvDHId/gxHrQZ4KhUYHNYj12KsvzH1M9pv7x5awlBPRwAinLYZIRTweC3Guh0nrW40dJAjAOBww1ChgPu1GOt2nLS60ZBBDgCAg/VDhAJOWox1O05a3cheagMfhR1O6IaARGJAUwhUXLF0WIz1K5y0utlQT0dUww7ljs3jtbi1xVi34qTVzYY2f0U8L79NKLjEnfGMdSdOWt1smKfQGd+yrBg/PM1Y9+Kk1c1a7iDuqw4QCoqygKYGywXEWD/DSaub6Yc9aBQguQJoqgOKczs+iDFmMk5a3czDUQoXO1voSIzqQeOEQu6MZ6zbcNLqZiKRSN8Zf8lxlFDISYuxbsNJqwe0dMafFAcLBfw4D2PdhpNWD2jp1zpU5w9ABJRfBKqMr1bEGDMPJ60e0JK0fi0TAYOaW1v8FZGxbsFJqwe0fD08f7UKNHi8UMhJi7FuwUmrBwx2tYNUIkZ9kw7X3cYIhTz9MmPdgpNWD7CRiBHoIdxBPCMbKRQWZQLaJgtGxVj/wEmrhwz1FJJWTp0SkLkAjTVAya8Wjoox68dJq4e0dMafK60BBocKhdyvxdgt46TVQ1o648+WVAEtnfHcr8XYLeOk1UP0La2r1cDgcKGQW1qM3bIuJa0NGzYgICAAcrkcEREROHKk4xHf27dvR3BwMORyOUJCQrBnzx6D/USEVatWwcvLC3Z2doiKisKZM2cM6vz222+YNm0aPDw84OzsjIkTJ2L//v1dCb9XtEy9XFbdgOtutwuFZeeA6msWjIox62d20tq6dSvi4+ORmJiIzMxMjBkzBtHR0SgpMT7i+9ChQ5g5cybmz5+PrKwsxMbGIjY2Frm5N2Y+WL16Nd577z1s3LgR6enpcHBwQHR0NOrq6vR1HnzwQTQ1NWHfvn3IyMjAmDFj8OCDD0KtVnfhbfe8lqmXAeBspQ3gESTs4NYWY7fG3AUVw8PDKS4uTv+7Vqslb29vSkpKMlp/xowZFBMTY1AWERFBixYtIiIinU5HKpWK3n77bf3+8vJykslk9MUXXxAR0dWrVwkA/fzzz/o6FRUVBIBSUlKMXreuro40Go1+Kyws7PHFWlv748eHyX/ZLvoi/SLRzmeERVx//FuvXZ8xa2LqYq1mtbQaGhqQkZGBqKgofZlYLEZUVBTS0tKMHpOWlmZQHwCio6P19fPz86FWqw3quLi4ICIiQl/H3d0dI0aMwGeffYbq6mo0NTXhgw8+gKenJ0JDQ41eNykpCS4uLvrN19fXnLfaLYx2xvP0y4zdErOSVmlpKbRaLZRKpUG5Uqls92uaWq3usH7La0d1RCIRfvzxR2RlZcHJyQlyuRx///vfsXfvXri6uhq9bkJCAjQajX4rLCw05612ixud8TclrcuZgE7b67Ew1l9Yxd1DIkJcXBw8PT3xv//9D0eOHEFsbCweeughXLlifIkumUwGZ2dng623tbS0zl2tBjxHAlJHoKEKKDnV67Ew1l+YlbQ8PDwgkUhQXFxsUF5cXAyVSmX0GJVK1WH9lteO6uzbtw+7du1CcnIy7rrrLtxxxx14//33YWdnh08//dSct9CrWlpahddrUKcF4HOHsIM74xnrMrOSllQqRWhoKFJTU/VlOp0OqampiIyMNHpMZGSkQX0ASElJ0dcPDAyESqUyqFNRUYH09HR9nZqaGiFYsWG4YrEYOp3OnLfQqzwcpXCW24AIyC/l8VqMdQtze/iTk5NJJpPR5s2b6eTJk7Rw4UJSKBSkVquJiGj27Nm0fPlyff2DBw+SjY0NrVmzhk6dOkWJiYlka2tLOTk5+jpvvvkmKRQK+uabb+jEiRM0bdo0CgwMpNraWiIS7h66u7vTo48+StnZ2ZSXl0dLly4lW1tbys7ONiluU+9MdLdHNvxC/st20bfZl4ny9gp3ENeF9WoMjFkDUz+jZictIqJ169aRn58fSaVSCg8Pp8OHD+v3TZ48mebOnWtQf9u2bRQUFERSqZRGjRpFu3fvNtiv0+lo5cqVpFQqSSaT0ZQpUygvL8+gztGjR2nq1Knk5uZGTk5OdOedd9KePXtMjtlSSWvptmzyX7aL3k3JI6oqFZJWojNR9bVejYOxvs7Uz6iIiMiybb3eUVFRARcXF2g0ml7tlP/gp3NI+u40HhrjjXUzxwHv3SGMjJ/1FTA8qvMTMDZAmPoZtYq7h9aspTP+bEmVUODX3Pf3604LRcSYdeOk1cOG3jT1sk5HQOiTwo6cbUBlcfsHMsaM4qTVw3xvmnr5cnkt4Dse8I0AtA3AkQ8tHR5jVoeTVg+zkYgR4GEPADh7tfkrYuRi4fXox0BDtYUiY8w6cdLqBfqR8S39WsExgGsgUFcOZG+xXGCMWSFOWr3A4BlEABBLgDv/LPyctoGfRWTMDJy0eoE+aZXc9FVw3CxArgCu5wN5e4wfyBhrg5NWL9BPUdPS0gIAqQMQ9ifh50PrLRAVY9aJk1YvuHnq5bLqhhs7IhYBYlug8DBw6ZiFomPMunDS6gU3T718/ubWlpMKuH2G8POhdRaIjDHrw0mrl7S0tvQj41tExgmvp74Frl/o3aAYs0KctHrJjQkBWyUt5Shg6H0A6YDD/7RAZIxZF05avaTNM4g3axlsmvlvoPZ6L0bFmPXhpNVLDBZvbbPzPsBzFNBYDWRs7t3AGLMynLR6ScvXw8LrNahrbDWYVCS60beV/gHQ1ADGmHGctHpJm6mXWwv5PeCoBCqvAL/u6P0AGbMSnLR6iUgkar8zHgBsZMK4LUAYbDow5mZkzGyctHpRh53xABA6D7C1B4pzgPMHei8wxqwIJ61eNNSzg854ALB3A8b9Ufg5jR/tYcwYTlq9aNigVlPUGHPnMwBEwNkfeVFXxozoUtLasGEDAgICIJfLERERgSNHjnRYf/v27QgODoZcLkdISAj27DGc1YCIsGrVKnh5ecHOzg5RUVE4c+ZMm/Ps3r0bERERsLOzg6urK2JjY7sSvsXop14ubZ562Ri3IcDIB4WfubXFWBtmJ62tW7ciPj4eiYmJyMzMxJgxYxAdHY2SkhKj9Q8dOoSZM2di/vz5yMrKQmxsLGJjY5Gbm6uvs3r1arz33nvYuHEj0tPT4eDggOjoaNTV1enrfPXVV5g9ezbmzZuH48eP4+DBg/jDH/7QhbdsOS1TL9c1Nk+93J7IZ4XXEzyPPGNtmLs2WXh4OMXFxel/12q15O3tTUlJSUbrz5gxg2JiYgzKIiIiaNGiRUQkrHmoUqno7bff1u8vLy8nmUxGX3zxBRERNTY2ko+PD3388cfmhqtnqXUPW/vd3w+Q/7JdtO90cccVP5oirI+Y+lrvBMaYhZn6GTWrpdXQ0ICMjAxERd1Yr08sFiMqKgppaWlGj0lLSzOoDwDR0dH6+vn5+VCr1QZ1XFxcEBERoa+TmZmJy5cvQywWY9y4cfDy8sIDDzxg0Fprrb6+HhUVFQZbXzBc6QRAWA+xtqGDGUsnNLe2eB55xgyYlbRKS0uh1WqhVCoNypVKJdRqtdFj1Gp1h/VbXjuqc/78eQDAq6++ihUrVmDXrl1wdXXFPffcg7KyMqPXTUpKgouLi37z9fU15632mGcmD4WjzAaHz5dh4b+PtR0d3yL4QcA1QHgWkeeRZ0zPKu4e6nQ6AMArr7yC6dOnIzQ0FJs2bYJIJML27duNHpOQkACNRqPfCgsLezPkdo32ccGmeeNhL5Xgf2dK8efPM9HQpGtb8eZ55A+/z/PIM9bMrKTl4eEBiUSC4mLDzuHi4mKoVCqjx6hUqg7rt7x2VMfLywsAcNttt+n3y2QyDBkyBAUFBUavK5PJ4OzsbLD1FeMD3PCvueMhtxVj3+kSLN6SiUatkcQ1dhYgdwHKzgN53/V+oIz1QWYlLalUitDQUKSmpurLdDodUlNTERkZafSYyMhIg/oAkJKSoq8fGBgIlUplUKeiogLp6en6OqGhoZDJZMjLy9PXaWxsxIULF+Dv72/OW+gzIoe64+M54yG1EeOHk8V4PjkbTa0Tl8zxxjzyPPyBMYG5PfzJyckkk8lo8+bNdPLkSVq4cCEpFApSq9VERDR79mxavny5vv7BgwfJxsaG1qxZQ6dOnaLExESytbWlnJwcfZ0333yTFAoFffPNN3TixAmaNm0aBQYGUm1trb7OkiVLyMfHh77//ns6ffo0zZ8/nzw9PamsrMykuPvK3cPW9p0upuEv7yH/ZbvouS8yqUmrM6ygKSL6q7twJ7HwmGWCZKwXmPoZNTtpERGtW7eO/Pz8SCqVUnh4OB0+fFi/b/LkyTR37lyD+tu2baOgoCCSSqU0atQo2r17t8F+nU5HK1euJKVSSTKZjKZMmUJ5eXkGdRoaGujFF18kT09PcnJyoqioKMrNzTU55r6atIiIfvhVTUMTdpP/sl304rZs0rZOXDsWCUlr21yLxMdYbzD1MyoiGhjTCVRUVMDFxQUajaZP9W+1+C7nChZ/kQWtjjAz3Bevx4ZALBYJO9W5wMa7AJEYeC5LuKvIWD9j6mfUKu4eDgQPhHjh7zPGQCwCvjhSiFf/+yv0/z9RjQaG3Ns8j/xGywbKmIVx0upDpo31wdu/HwORCPgs7SL+3+5TNxLXhOZ55LP+DdSWWyxGxiyNk1YfMz10MN58NAQA8K9f8vHW3jwhcQ2dAnjeBjRU8TzybEDjpNUHPT7eD6/FjgYAbPzpHN798QzPI89YM05afdTsO/2x6kFhMO17qWewft8ZIOSx5nnki4Bfd1o4QsYsg5NWH/aniYFIeCAYALDmh9/wwcFLQPgCYeehdTyPPBuQOGn1cYsmD8XSqUEAgKTvTuPzpqgb88jn/2Th6BjrfZy0rMDi+4bjuSnDAQCv/FCE014PCzsO8aM9bODhpGUlXogajqcnDwUALDoTDoIIOJsClJy2cGSM9S5OWlZCJBJh2f0jMH9iIC6SCt9rw4Qd/CA1G2A4aVkRkUiEFTEjMSfSHx81/R8AQHs8GagyPj8/Y/0RJy0rIxKJ8OpDoxAUFoUs3TBIdI04u+tdS4fFWK/hpGWFxGIRXn8kBDl+swEAbqc+Q+qJC5YNirFewknLSonFIsx6Mg6ltl5wE1Xh5+3/wP48/prI+j9OWlZMYmML1/uWAADmivbg6X8fxS9nSi0cFWM9i5OWlZPcMRskd8EQsRqTdMfw1GdHcegsJy7Wf3HSsnYyR4hC5wEAljqloK5Rhz/+Kx1Je051vK4iY1aKk1Z/ELEIENtgRH0OnhuhgY6AD34+j/v/8TMOneNWF+tfOGn1B87ewOjfAwDiHX/Ax3PCoHKW4+K1Gvzho3Qk7DgBTW2jhYNkrHtw0uovWmY2PfkNorzq8EP8JMyK8AMgTN889d2f8MOvxlcBZ8yacNLqL1QhwJB7hHnkU/8G56oLeD12NLYuvBOBHg4orqjHwn9nIO7zTFytrLd0tIx1WZeS1oYNGxAQEAC5XI6IiAgcOXKkw/rbt29HcHAw5HI5QkJCsGfPHoP9RIRVq1bBy8sLdnZ2iIqKwpkzZ4yeq76+HmPHjoVIJEJ2dnZXwu+/Ip8VXnO/BNaHAW8PQ0T6s/ghIhuvhdbATtyE3TlXEPX3n/BlxiUMkIWYWD9jdtLaunUr4uPjkZiYiMzMTIwZMwbR0dEoKTE+sPHQoUOYOXMm5s+fj6ysLMTGxiI2Nha5ubn6OqtXr8Z7772HjRs3Ij09HQ4ODoiOjkZdXV2b87300kvw9vY2N+yBYdgUIObvgN8EQCIDakqBvN2wTU3E7F+fwq92C7DL8XUsavw3dn+1GU9/nIrCshpLR82YWcxe9zAiIgLjx4/H+vXC7AI6nQ6+vr549tlnsXz58jb1H3/8cVRXV2PXrl36sjvvvBNjx47Fxo0bQUTw9vbGiy++iKVLlwIANBoNlEolNm/ejCeeeEJ/3HfffYf4+Hh89dVXGDVqFLKysjB27FiT4u7r6x52u6Z64MpxoCANKEgXXmvL2lQ7Q4OhHRyBoPG/g9jvTmFNRZGo9+LUaYHqUqBKLbyqbgccB/Xe9VmfYepn1MackzY0NCAjIwMJCQn6MrFYjKioKKSlpRk9Ji0tDfHx8QZl0dHR+PrrrwEA+fn5UKvViIqK0u93cXFBREQE0tLS9EmruLgYCxYswNdffw17e/tOY62vr0d9/Y2+m4qKCpPfZ79gIwN8w4XtLghTM187q09iDRcOQVp+HsNFl4DLl4DLXwnHOSoBvzsB3zuFV1UIILE1//qNdUIiqioBKtVAVfGN15t/rr4q9MO1kDkD0z8GgqK75Z+B9T9mJa3S0lJotVoolUqDcqVSidOnjU9Gp1arjdZXq9X6/S1l7dUhIjz55JN4+umnERYWhgsXLnQaa1JSEv7617+a9L4GBJEI8BgubHfMgRSArrIEv+zfjfMZP+J2Oo3RonxIq4qBk98IGyBM7ewTCvhFAn4RgDIEqCs3noAqm5NUlRqo05gTHOAwCBDbCIt2bHkcuO8V4O6lvdvqY1bBrKRlKevWrUNlZaVBC68zCQkJBi28iooK+Pr69kR4Vkvs5IlJD89D0OSZWPF1Lv53qhBjROdwv3M+pg+6DJfSTCH5XPifsJlLIgOclELrzVEJOKmM/+wwCJDYCMui7V0GHPsE2Pf/AHUOMO19QObY/W+eWS2zkpaHhwckEgmKi4sNyouLi6FSqYweo1KpOqzf8lpcXAwvLy+DOi39Vfv27UNaWhpkMpnBecLCwjBr1ix8+umnba4rk8na1GfGqVzk+GhOKHbneCPxG0f8TTMSr1UA8yL98Zc7CHbqo0DBYWErvwjIXTpPRE5KQK4wr6VkIwUefFfo19rzF6G1V3oGeOJzwG1Ij71/ZmXITOHh4bR48WL971qtlnx8fCgpKclo/RkzZtCDDz5oUBYZGUmLFi0iIiKdTkcqlYrWrFmj36/RaEgmk9EXX3xBREQXL16knJwc/fb9998TAPryyy+psLDQpLg1Gg0BII1GY9b7HWjKqurpha1Z5L9sF/kv20V3vZlKP/9WcqNCU2PvBHLxMNHbw4kSnYmS/IjOpvbOdZnFmPoZNTtpJScnk0wmo82bN9PJkydp4cKFpFAoSK1WExHR7Nmzafny5fr6Bw8eJBsbG1qzZg2dOnWKEhMTydbWlnJycvR13nzzTVIoFPTNN9/QiRMnaNq0aRQYGEi1tbVGY8jPzycAlJWVZXLcnLTMcyCvhCYkpeqT14vbsul6dX3vBqG5TPThvULielVB9Ms/iHS63o2B9ZoeS1pEROvWrSM/Pz+SSqUUHh5Ohw8f1u+bPHkyzZ0716D+tm3bKCgoiKRSKY0aNYp2795tsF+n09HKlStJqVSSTCajKVOmUF5eXrvX56TVO6rqGinxm1wKWC4krtDXUujb7MvU0KTtvSAaaol2/llIXInORF/OJ6qv7r3rs15j6mfU7HFa1mrAjdPqRhkXr2PZVydwtqQKAGBnK0GovyvCA90QHuiGsb4KyG0lPRcAEXD0Y2DvckDXJPR5PfE5oPDruWuyXmfqZ5STFjNJfZMW7+8/h8/SLuB6jeGMEVKJGGN9Ffokdoe/KxxlPXBj+sIvwLY5QM01wN4deOxTIPDu7r8OswhOWq1w0uoeOh3h7NUqpOeX4Uh+GdLPX0NJqwewJWIRRns7Nycxd4wPcIXCXto9AZQXAsl/ANQnAJEEuD8JCF/I47n6AU5arXDS6hlEhIvXaoQEll+GIxeuobCs1qCOSASMUDohoiWJBbrC00ne9Ys21gL/XQKc2Cr8PnaW8Myl7S2ck1kcJ61WOGn1nqLy2htJLP8azl2tblNniIeD/utkeKAbBrt2/miWASLg8PvADyuEx4C87wAe/w/g4tNN74L1Nk5arXDSspyrlfU4dkFIYun5ZTitrkDrvzofhR3CA90wZrALgpROGK50goejFKLOvvad2w98OQ+ovQ44eAKP/1t4ZpJZHU5arXDS6js0NY04drFM3xrLuayBVtf2z9DNQYrhno4IUjohSOmI4UonBCmd4ObQqn/s+gUgeRZQnAuIbYH/Ww2E/al33gzrNpy0WuGk1XdV1zchq6AcR/Kv4eSVSpwpqURBWU2b1lgLD0cZgpSOzS2y5qSmEMMl5Xng151CpdAngQfeFh4NYlaBk1YrnLSsS22DFmdLqvBbcSV+K6nEmWLh50vXa9s9RukkxQv23+FxzScQgVDlGQrM+BSOHgP4QXmdTvjqXFMqzFdWUypMB1R97UZZ7XXAeyxw558BR0+LhcpJqxVOWv1DdX0TzjQnszPFlfitOZld0dyY5fYecTbes10PZ1EN1OSKV6TL0eR1B4JVThjnp8Ad/rd499KSdFohyRgkoFJh7Fp18+8tP9c0l988X1lHbOTAHXOBu54DXAb37PswgpNWK5y0+reKukacKa7CmeJK5BVXouJSHuKKV2IILqGebLCi6U/Yrr1HX9/PzR6h/q76LUjpBIm4D4z10jYC5QVA2XnDrbxASEi1101PQjeTuwhTANl7AA4ewuBch0HCzzZyIOs/wOVjQl2xLTB2JjDxhV6dXYOTViuctAag+ko0fLkQ0jPCQirpbg/j+7rRyCm3QRk54Ro5QwMHEMRwktlgrJ8Cof6uCPN3w1g/Rc+M6geEWV3LL7ZNTGXnhcGzZMLK4HauxhNQS9nNP9u7dz77LBGQ/xPw85obc6eJxMJ6mne/CHgG3/r77gQnrVY4aQ1QOh3w89vAgTeM7tZCjOvkhDJyRBmccY2cUEbOuA4n2Dp5wt3TGz7ePhgaEACllw9E9h6mde431Ah3NfUJ6Vzzaz6guQSgg4+drb3QwnELbH4dAij8hf4mew/A3q1rU2CbquCwkLzOptwoG/mQMJOs99geuywnrVY4aQ1wv/0AZGwGqkua+3yuAfXmTAl9Q4PEEWTvDltnT4hbWjL2bsJXt7J8ITlVFnV8EqnjjYTUenNS9Y3Hkoqygf+9A5z69kbZsN8Bk/4iTL3dzThptcJJi7XR1CAksJqb7qQ1/15zvRiaa1dQrymBqLYMDk3X4YpKSESmf1xI5gyd2xCI3IZA7D4UcBt6IzE5ePSNxGSKktPAL38Hcrbf6E8LuBuYtBQInNxt74OTViuctNitqGvU4nhBGXLPF+DchYsoKiqErP46XEWVcEMF3ESVqCR7XCAlLpIKF0iJcjgCED7QYhEgtRFDKhFDaiOBzEYMmY1YKNOXC5tQLtGXyWzEsJdK4KWww2CFHXxc7eCjsINDT/W5tefaOeDgWiD7C0DXPNOHT5jQ8gqKvuXkxUmrFU5arDsREc6XViPjwnVkXLyOrMLruF7TiIYmHRqadKhv0sLIIP9u5Wpvq09gPgp7/c+Dm18V9radPwbVFZpLwMH3gMxPgabmoSbKEGDSi8DIhwFx1+ZW46TVCict1tuatDo0aHU3JTLh9/pGw/IGrVa/v77p5vKWulpU1TXhcnkdLl2vweXyWlTWNXV6fQep5EZSc22b2AY5yiC+lWEeVSVA2nrg6L+ABmGCSLgPF+42hvze7JsFnLRa4aTF+pOKukZcvl4rbOXN2/VafVIrrWro9BxSiRheCjkGu9rBz80egR4OCHB3QKCHA/zc7SGzMbHFVFMGpH8ApG8U1sQEhLudE58Xpg2yMW1VLE5arXDSYgNJXaNWn8hufr10vQaXr9dCXVHX4ddXsQjwVtgh0MPhRjIb5IBAdwcMdrWDjUTc9qD6SqHVlbZeGAgLAE5ewITnhGdBpR1PP8RJqxVOWozd0KjVQa2pa05ktSi4Vo3zpdW4cK0aF0prUFXf/tdPG7EIfm72CGiVzAI87OHtYgdxUy2Q+Rlw6D2g4rJw0JRE4O74ds8JmP4ZtYoVphlj3ctWIoavmz183dq2fogIV6vqcaG0BhdKq5F/rRr5V5sT2rVq1DXqcL5USHKtyWzE8He3R4B7GIaN+AL31KZiVNGXaBo9Gy7dFLuRNl7nNmzYgICAAMjlckRERODIkSMd1t++fTuCg4Mhl8sREhKCPXv2GOwnIqxatQpeXl6ws7NDVFQUzpw5o99/4cIFzJ8/H4GBgbCzs8PQoUORmJiIhobOv7czxswjEong6SRHeKAbZoz3xbL7g7Fxdij2Pj8JJ/96Pw4tvw9bnorA64+MxlMTAxE10hNDBznAViJCfZMOvxVX4YeTxXj/f5cw49gIjCp6GSnn6zu/sKnMXZssOTmZpFIpffLJJ/Trr7/SggULSKFQUHFxsdH6Bw8eJIlEQqtXr6aTJ0/SihUrjC7W6uLiQl9//TUdP36cHn74YYPFWr/77jt68skn6fvvv6dz587RN998Q56envTiiy+aHDeve8hYz2ps0tLF0mo6kFdCmw/mU+I3uTTnX+k0afU+yrhY1unxPbZYa3h4OMXFxel/12q15O3tTUlJSUbrz5gxg2JiYgzKIiIiaNGiRUQkLNSqUqno7bff1u8vLy8nmUxGX3zxRbtxrF69mgIDA9vdX1dXRxqNRr8VFhZy0mKsDzM1aZn19bChoQEZGRmIiorSl4nFYkRFRSEtLc3oMWlpaQb1ASA6OlpfPz8/H2q12qCOi4sLIiIi2j0nAGg0Gri5ubW7PykpCS4uLvrN13cATwTHWD9iVtIqLS2FVquFUqk0KFcqlVCr1UaPUavVHdZveTXnnGfPnsW6deuwaNGidmNNSEiARqPRb4WFhR2/OcaYVbC6u4eXL1/G/fffj8ceewwLFixot55MJoNMZtqgNsaY9TCrpeXh4QGJRILi4mKD8uLiYqhUKqPHqFSqDuu3vJpyzqKiItx7772YMGECPvzwQ3NCZ4z1E2YlLalUitDQUKSmpurLdDodUlNTERkZafSYyMhIg/oAkJKSoq8fGBgIlUplUKeiogLp6ekG57x8+TLuuecehIaGYtOmTRCLuzRagzFm7czt4U9OTiaZTEabN2+mkydP0sKFC0mhUJBarSYiotmzZ9Py5cv19Q8ePEg2Nja0Zs0aOnXqFCUmJhod8qBQKOibb76hEydO0LRp0wyGPFy6dImGDRtGU6ZMoUuXLtGVK1f0m6l4yANjfVuPDXkgIlq3bh35+fmRVCql8PBwOnz4sH7f5MmTae7cuQb1t23bRkFBQSSVSmnUqFG0e/dug/06nY5WrlxJSqWSZDIZTZkyhfLy8vT7N23aRBDmp22zmYqTFmN9m6mf0QHz7KFGo4FCoUBhYSE/e8hYH1RRUQFfX1+Ul5fDxaX9h36s7u5hV1VWVgIAj9dirI+rrKzsMGkNmJaWTqdDUVERnJycOp3NsSXjW2OrjGO3DI791hERKisr4e3t3eGNtgHT0hKLxRg82LxVc52dna3uD7AFx24ZHPut6aiF1YLHDTDGrAonLcaYVeGkZYRMJkNiYqJVPgbEsVsGx957BkxHPGOsf+CWFmPMqnDSYoxZFU5ajDGrwkmLMWZVOGkxxqwKJ61WzF0era9ISkrC+PHj4eTkBE9PT8TGxiIvL8/SYZntzTffhEgkwvPPP2/pUExy+fJl/PGPf4S7uzvs7OwQEhKCY8eOWTqsTmm1WqxcudJgWb7XXnsN1jCYgJPWTbZu3Yr4+HgkJiYiMzMTY8aMQXR0NEpKSiwdWqd++uknxMXF4fDhw0hJSUFjYyOmTp2K6uq2C2r2VUePHsUHH3yA22+/3dKhmOT69eu46667YGtri++++w4nT57EO++8A1dXV0uH1qm33noL//znP7F+/XqcOnUKb731FlavXo1169ZZOrTO9fAUOVbF3OXR+rKSkhICQD/99JOlQzFJZWUlDR8+nFJSUmjy5Mm0ZMkSS4fUqWXLltHEiRMtHUaXxMTE0J/+9CeDskcffZRmzZploYhMxy2tZl1ZHq0v02g0ANDhMmt9SVxcHGJiYtosN9eXffvttwgLC8Njjz0GT09PjBs3Dh999JGlwzLJhAkTkJqait9++w0AcPz4cfzyyy944IEHLBxZ5wbMLA+d6Wh5tNOnT1soqq7R6XR4/vnncdddd2H06NGWDqdTycnJyMzMxNGjRy0dilnOnz+Pf/7zn4iPj8fLL7+Mo0eP4rnnnoNUKsXcuXMtHV6Hli9fjoqKCgQHB0MikUCr1eL111/HrFmzLB1apzhp9UNxcXHIzc3FL7/8YulQOlVYWIglS5YgJSUFcrnc0uGYRafTISwsDG+88QYAYNy4ccjNzcXGjRv7fNLatm0bPv/8c2zZsgWjRo1CdnY2nn/+eXh7e/f52LlPq1l9fT1JJBLauXOnQfmcOXPo4YcftkxQXRAXF0eDBw+m8+fPWzoUk+zcuZMAkEQi0W8ASCQSkUQioaamJkuH2C4/Pz+aP3++Qdn7779P3t7eForIdIMHD6b169cblL322ms0YsQIC0VkOu7TataV5dH6EiLC4sWLsXPnTuzbtw+BgYGWDskkU6ZMQU5ODrKzs/VbWFgYZs2ahezsbEgkEkuH2K677rqrzbCS3377Df7+/haKyHQ1NTVtZgeVSCTQ6XQWisgMls6afUlny6P1Zc888wy5uLjQgQMHDJZYq6mpsXRoZrOWu4dHjhwhGxsbev311+nMmTP0+eefk729Pf3nP/+xdGidmjt3Lvn4+NCuXbsoPz+fduzYQR4eHvTSSy9ZOrROcdJqpaPl0foytLPE2qZNmywdmtmsJWkREf33v/+l0aNHk0wmo+DgYPrwww8tHZJJKioqaMmSJeTn50dyuZyGDBlCr7zyCtXX11s6tE7xfFqMMavCfVqMMavCSYsxZlU4aTHGrAonLcaYVeGkxRizKpy0GGNWhZMWY8yqcNJijFkVTlqMMavCSYsxZlU4aTHGrMr/B4f06pvMPvuBAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:00:23.524381Z", + "iopub.status.busy": "2024-03-22T14:00:23.523671Z", + "iopub.status.idle": "2024-03-22T14:04:32.132930Z", + "shell.execute_reply": "2024-03-22T14:04:32.132048Z" + }, + "papermill": { + "duration": 248.630794, + "end_time": "2024-03-22T14:04:32.135514", + "exception": false, + "start_time": "2024-03-22T14:00:23.504720", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:04:32.172721Z", + "iopub.status.busy": "2024-03-22T14:04:32.172390Z", + "iopub.status.idle": "2024-03-22T14:04:32.193148Z", + "shell.execute_reply": "2024-03-22T14:04:32.192186Z" + }, + "papermill": { + "duration": 0.041408, + "end_time": "2024-03-22T14:04:32.195213", + "exception": false, + "start_time": "2024-03-22T14:04:32.153805", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.0137630.0068450.0026838.1176870.2192995.288560.3796990.0000268.6848670.0371530.1209070.0517950.0910320.03082516.802554
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.013763 0.006845 0.002683 8.117687 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.219299 5.28856 0.379699 0.000026 8.684867 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.037153 0.120907 0.051795 0.091032 0.030825 \n", + "\n", + " total_duration \n", + "realtabformer 16.802554 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:04:32.230875Z", + "iopub.status.busy": "2024-03-22T14:04:32.230010Z", + "iopub.status.idle": "2024-03-22T14:04:32.737553Z", + "shell.execute_reply": "2024-03-22T14:04:32.736522Z" + }, + "papermill": { + "duration": 0.527728, + "end_time": "2024-03-22T14:04:32.739757", + "exception": false, + "start_time": "2024-03-22T14:04:32.212029", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:04:32.777689Z", + "iopub.status.busy": "2024-03-22T14:04:32.777370Z", + "iopub.status.idle": "2024-03-22T14:09:14.837137Z", + "shell.execute_reply": "2024-03-22T14:09:14.836205Z" + }, + "papermill": { + "duration": 282.081664, + "end_time": "2024-03-22T14:09:14.839804", + "exception": false, + "start_time": "2024-03-22T14:04:32.758140", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/realtabformer/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/realtabformer/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:09:14.877034Z", + "iopub.status.busy": "2024-03-22T14:09:14.876694Z", + "iopub.status.idle": "2024-03-22T14:09:14.903623Z", + "shell.execute_reply": "2024-03-22T14:09:14.902858Z" + }, + "papermill": { + "duration": 0.04813, + "end_time": "2024-03-22T14:09:14.905644", + "exception": false, + "start_time": "2024-03-22T14:09:14.857514", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:09:14.939512Z", + "iopub.status.busy": "2024-03-22T14:09:14.939241Z", + "iopub.status.idle": "2024-03-22T14:09:14.944671Z", + "shell.execute_reply": "2024-03-22T14:09:14.943801Z" + }, + "papermill": { + "duration": 0.024708, + "end_time": "2024-03-22T14:09:14.946665", + "exception": false, + "start_time": "2024-03-22T14:09:14.921957", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.3827591784511294}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:09:14.983564Z", + "iopub.status.busy": "2024-03-22T14:09:14.983242Z", + "iopub.status.idle": "2024-03-22T14:09:15.352971Z", + "shell.execute_reply": "2024-03-22T14:09:15.352073Z" + }, + "papermill": { + "duration": 0.390816, + "end_time": "2024-03-22T14:09:15.355058", + "exception": false, + "start_time": "2024-03-22T14:09:14.964242", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHCElEQVR4nO3deXhTVf4/8PfNnrZZ2tKVpgul7LIvsgkqKogKOqKoM7Qi4iiM4zA8P2WeGdxGi46ijAujo7bgxox+cRtHBFTAUXZEQZC1LaWUtnRLm6RZz++Pm6RNm7RJmuS26ef1PHna3JzcnJs0n579cIwxBkIIEZBI6AwQQggFIkKI4CgQEUIER4GIECI4CkSEEMFRICKECI4CESFEcBSICCGCo0BECBEcBSLi02OPPQaO40J6zuLiYnAchwMHDviV/m9/+xsGDBgAsViM0aNHhzQvpOegQEQC8vTTT+Pjjz+OyGtt3boV/+///T9MnToVRUVFePrppyPyuiTyJEJngPQuTz/9NG699VbMnz8/7K/19ddfQyQS4c0334RMJgv76xHhUIkoShgMBqGzEHLV1dVQKpUhC0KMMZhMppCcqzuMRqPQWehxKBD1Qq62m2PHjuHOO+9EfHw8pk2bBgB45513MG7cOCiVSiQkJGDhwoUoLy/3eP63336LBQsWIDMzE3K5HDqdDn/4wx+6/JJyHAeDwYANGzaA4zhwHIeCggIAQFlZGR544AEMHjwYSqUSiYmJWLBgAUpLS72ey2g04r777kNiYiLUajUWLVqE+vp6j9cqKiqCwWBwv1ZxcTEAwGaz4cknn0Rubi7kcjmys7Pxpz/9CWaz2eM1srOzccMNN+DLL7/E+PHjoVQq8dprr2HHjh3gOA7//ve/8fjjj6N///5QqVS49dZb0djYCLPZjIceegjJycmIi4vD3Xff3eHc/r7XM2fOxIgRI3Dw4EFcccUViImJwZ/+9KdO3+e+iKpmvdiCBQuQl5eHp59+GowxPPXUU/jLX/6C2267DUuWLEFNTQ1eeuklXHHFFfjhhx+g1WoBAB988AGMRiPuv/9+JCYmYt++fXjppZdw/vx5fPDBBz5f7+2338aSJUswceJELF26FACQm5sLANi/fz++//57LFy4EBkZGSgtLcX69esxc+ZMHDt2DDExMR7nWr58ObRaLR577DGcOHEC69evR1lZmTtIvP3223j99dexb98+vPHGGwCAKVOmAACWLFmCDRs24NZbb8Uf//hH7N27F4WFhTh+/Dg++ugjj9c5ceIE7rjjDtx333249957MXjwYPdjhYWFUCqVeOSRR3D69Gm89NJLkEqlEIlEqK+vx2OPPYY9e/aguLgYOTk5WL16tfu5/r7XAFBbW4s5c+Zg4cKF+PWvf42UlJQAP+k+gJFe59FHH2UA2B133OE+VlpaysRiMXvqqac80h45coRJJBKP40ajscM5CwsLGcdxrKysrMPrtBUbG8vy8/M7PN/bOXfv3s0AsI0bN7qPFRUVMQBs3LhxzGKxuI8/++yzDAD75JNP3Mfy8/NZbGysxzkPHz7MALAlS5Z4HF+5ciUDwL7++mv3saysLAaAbdmyxSPtN998wwCwESNGeOThjjvuYBzHsTlz5niknzx5MsvKynLfD+S9njFjBgPA/vGPf3R4f0grqpr1Yr/97W/dv2/evBkOhwO33XYbLl265L6lpqYiLy8P33zzjTutUql0/24wGHDp0iVMmTIFjDH88MMPQeWl7TmtVitqa2sxcOBAaLVaHDp0qEP6pUuXQiqVuu/ff//9kEgk+O9//9vp67geX7FihcfxP/7xjwCAzz//3ON4Tk4OrrvuOq/nWrRokUceJk2aBMYYFi9e7JFu0qRJKC8vh81mAxDYew0Acrkcd999d6fX1ddR1awXy8nJcf9+6tQpMMaQl5fnNW3bL9y5c+ewevVqfPrppx7tMgDQ2NgYVF5MJhMKCwtRVFSEiooKsDYLf3o7Z/t8xsXFIS0tzWebkktZWRlEIhEGDhzocTw1NRVarRZlZWUex9u+R+1lZmZ63NdoNAAAnU7X4bjD4UBjYyMSExMDeq8BoH///tTr1wUKRL1Y21KIw+EAx3H44osvIBaLO6SNi4sDANjtdlxzzTWoq6vDww8/jCFDhiA2NhYVFRUoKCiAw+EIKi+/+93vUFRUhIceegiTJ0+GRqMBx3FYuHBh0OfsjL8DLdu+R+15e586O+4Krv6+1/7kgfAoEEWJ3NxcMMaQk5ODQYMG+Ux35MgRnDx5Ehs2bMCiRYvcx7dt2+bX6/gKAB9++CHy8/Px/PPPu4+1tLSgoaHBa/pTp07hyiuvdN9vbm5GZWUlrr/++k5fPysrCw6HA6dOncLQoUPdx6uqqtDQ0ICsrCy/rqM7/H2vif+ojShK3HLLLRCLxXj88cc9qkUA/5+8trYWQOt/+7ZpGGNYt26dX68TGxvrNbiIxeIOr/vSSy/Bbrd7Pc/rr78Oq9Xqvr9+/XrYbDbMmTOn09d3BaoXX3zR4/jatWsBAHPnzu3qErrN3/ea+I9KRFEiNzcXf/3rX7Fq1SqUlpZi/vz5UKlUKCkpwUcffYSlS5di5cqVGDJkCHJzc7Fy5UpUVFRArVbj//7v/zq0Ffkybtw4bN++HWvXrkV6ejpycnIwadIk3HDDDXj77beh0WgwbNgw7N69G9u3b0diYqLX81gsFlx99dW47bbbcOLECbz66quYNm0abrrppk5ff9SoUcjPz8frr7+OhoYGzJgxA/v27cOGDRswf/58j1JWuPj7XhP/USCKIo888ggGDRqEF154AY8//jgAvuH12muvdX/BpVIpPvvsMzz44IMoLCyEQqHAzTffjOXLl2PUqFFdvsbatWuxdOlS/PnPf4bJZEJ+fj4mTZqEdevWQSwW491330VLSwumTp2K7du3++yxevnll/Huu+9i9erVsFqtuOOOO/D3v//dr7afN954AwMGDEBxcTE++ugjpKamYtWqVXj00UcDeLe6x5/3mviPY+3LloQQEmHURkQIERwFIkKI4CgQEUIER4GIECI4CkSEEMFRICKECK5XjyNyOBy4cOECVCpVyBd5J4R0H2MMTU1NSE9Ph0jku9zTqwPRhQsXOsyUJoT0POXl5cjIyPD5eK8ORCqVCgB/kWq1WuDcEELa0+v10Ol07u+qL706ELmqY2q1mgIRIT1YV00n1FhNCBEcBSJCiOAoEBFCBNer24j8wRiDzWbzuUAX8U0sFkMikdDQCBJ2UR2ILBYLKisraWfNboiJiUFaWhot/k7CKmoDkcPhQElJCcRiMdLT0yGTyeg/ewAYY7BYLKipqUFJSQny8vI6HZBGSHcIGoiys7M7bP8CAA888ABeeeWVbp3bYrHA4XBAp9N12GWU+EepVEIqlaKsrAwWiwUKhULoLJEoJWgg2r9/v0fbzdGjR3HNNddgwYIFIXsN+i/ePdH0/pltdsgl3rcKIsISNBAlJSV53F+zZg1yc3MxY8YMgXJEolVtsxnv7zuH4ekaTM/rB32LDQmx1O7VU/SYNiKLxYJ33nkHK1as8NmWYzabYTab3ff1en2kskd6ufJ6E6x2hlPVTThS0Qi5RISlVwygdsMeoseUuz/++GM0NDSgoKDAZ5rCwkJoNBr3jSa8dl92dnaHPcKi0YUGEwBgRH8NOABGix2NJmvnTyIR02MC0Ztvvok5c+YgPT3dZ5pVq1ahsbHRfSsvL49gDklvVtPEl6TTNUokxPFVskvNFiGzRNroEVWzsrIybN++HZs3b+40nVwuh1wuj1Cueg+LxULjfDphszvQYORLP/1UciTEyFCtN6PeSIGop+gRJaKioiIkJydHZLtgALDYHD5vNrvD77RWP9IGY+bMmVi+fDmWL18OjUaDfv364S9/+Yt7e+Ps7Gw8+eSTWLRoEdRqNZYuXQoA+N///ofp06dDqVRCp9PhwQcfhMFgcJ+3uroaN954I5RKJXJycvDuu+8Glb/eps5ggYMxKKRixMrEiHc2UtcbKBD1FIKXiBwOB4qKipCfnw+JJDLZeeWb0z4fy+kXi/lj+rvvv77rDKx273tQZsQrsWB8azvVW9+VwGTxnEryh2sGBZXHDRs24J577sG+fftw4MABLF26FJmZmbj33nsBAM899xxWr17t3t30zJkzmD17Nv7617/irbfeQk1NjTuYFRUVAQAKCgpw4cIFfPPNN5BKpXjwwQdRXV0dVP56kzpnyScxlh/UGh/jDERUIuoxBA9E27dvx7lz57B48WKhs9Kj6HQ6vPDCC+A4DoMHD8aRI0fwwgsvuAPRVVddhT/+8Y/u9EuWLMFdd92Fhx56CACQl5eHv//975gxYwbWr1+Pc+fO4YsvvsC+ffswYcIEAHy73NChQyN+bZEmE4uQEa9EqoYfkKmNkQIANVb3IIIHomuvvRaR3vV62ZUDfT4matebu/SKXJ9p2/f8Lp6a051sebj88ss9upYnT56M559/3j0AdPz48R7pf/zxR/z0008e1S3GmHuqy8mTJyGRSDBu3Dj340OGDIFWqw1ZnnuqAUlxGJAU576vUUoxor8GaoUEjDHqwu8BBA9EQpBJ/G8aC1fa7oqNjfW439zcjPvuuw8PPvhgh7SZmZk4efJkpLLW4ymkYlwzLEXobJA2+mQg6g327t3rcX/Pnj3Iy8uDWOx9isLYsWNx7NgxDBzovbQ3ZMgQ2Gw2HDx40F01O3HiBBoaGkKa757I7mAQty/qkh6lR/SakY7OnTuHFStW4MSJE3j//ffx0ksv4fe//73P9A8//DC+//57LF++HIcPH8apU6fwySefYPny5QCAwYMHY/bs2bjvvvuwd+9eHDx4EEuWLIFSqYzUJQnC4WB49ZvT+OeuszBabO7jFpsDdQYLDGZbJ88mkUKBqIdatGgRTCYTJk6ciGXLluH3v/+9u5vem5EjR2Lnzp04efIkpk+fjjFjxmD16tUeA0SLioqQnp6OGTNm4JZbbsHSpUuRnJwcicsRTLPFBpuDwWixQ9FmwuvXv1Rhw/elOF5J04R6Aqqa9VBSqRQvvvgi1q9f3+Gx0tJSr8+ZMGECtm7d6vOcqamp+M9//uNx7De/+U238tnT6Z09YyqFBKI21bNYOf+nb7DQyp09AZWISFRrdla9VArP/7kxMr50ZLJQ1awnoEBEoprRWeJxlYBclFKJx+NEWFQ164F27NghdBaihmuku1Lm2dvoKhFRIOoZqEREopqrVyxG6j0QtZ+SQ4RBgYhEtYRYGXQJMUiM81ydQNmmRBTpkf2kI6qakag2PjsB47MTOhyPkUkwSqeBUiqBgwFiGu8oKApEpE8SizhcNYSmefQUVDUjUYuf9EvVrt6ASkQkalnsDqzfcQZKqRj3TMuBROz5f7fFaofJYodCKu7Qq0Yii0pEJGqZLHYwBtgcrEMQAoCtx6pQ/H0pTlU3CZA70hYFIhK1XNM3lFLvpR2Fc9mWFmtwS/qS0KFARKKWa/pGjI9ql8IZoFqsNJZIaH0rEDEG2CyRvwUwTmXjxo1ITEz02EgSAObPnx/1E1RDzTVqOkbuvSmUAlHPIXhjdUVFBR5++GF88cUXMBqNGDhwIIqKijoshRoSdivw7fOhP29Xpv8RkPi33c+CBQvw4IMP4tNPP8WCBQsA8LtvfP75553OrCcduQORr6qZlP8/bA5ytxUSOoKWiOrr6zF16lRIpVJ88cUXOHbsGJ5//nnEx8cLmS1BKZVK3Hnnne6dNwDgnXfeQWZmJmbOnClcxnoh1/QNqpr1fIKWiJ555hnodDqPL11OTugWoO9ALOVLJ5EmlgaU/N5778WECRNQUVGB/v37o7i4GAUFBbTIe4A0MVJkJsS4d3ZtT+5qrKYSkeAEDUSffvoprrvuOixYsAA7d+5E//798cADD7i3zGnPbDZ7tJ3o9QGursdxfleRhDRmzBiMGjUKGzduxLXXXouff/4Zn3/+udDZ6nXGZsZjbKbv0rVWKcNonRZqpeAtFH2eoFWzs2fPYv369cjLy8OXX36J+++/Hw8++CA2bNjgNX1hYSE0Go37ptPpvKaLBkuWLEFxcTGKioowa9asqL5WoWhipLhySDLGZXWci0Yii2MCTj2WyWQYP348vv/+e/exBx98EPv378fu3bs7pPdWItLpdGhsbIRarfZI29LSgpKSEuTk5EChUITvIsKksbER6enpsNls2LhxI26//XZB8tGb30eHg3ksD0siT6/XQ6PReP2OtiVoiSgtLQ3Dhg3zODZ06FCcO3fOa3q5XA61Wu1xi1YajQa/+tWvEBcXh/nz5wudnV7H7mD4+9en8I+dZzptjDZZ7Kg3WGCzUzuRkAQNRFOnTsWJEyc8jp08eRJZWVkC5ahnqaiowF133QW5XC50Vnodo8UGxgCz1eFulPZmw+5SFH9figbaflpQgrbS/eEPf8CUKVPw9NNP47bbbsO+ffvw+uuv4/XXXxcyW4Krr6/Hjh07sGPHDrz66qtCZ6dXatt131lvo0Iigslipy58gQkaiCZMmICPPvoIq1atwhNPPIGcnBy8+OKLuOuuu4TMluDGjBmD+vp6PPPMMxg8eLDQ2emVDD7Wqm6PH0tkpflmAhO83/KGG27ADTfcIHQ2ehRf+5YR/7l2dY2Vdx6IZBLX6GoqEQmpb801I32Ge/cOaef/a+XO3V8tNKhRUFEfiGhh9O7pre+fsYvpHS5yCc036wmiNhBJpfy0CqPRKHBOejfX++d6P3sLjZKf3tF+9472XFUzKhEJS/A2onARi8XQarWorq4GAMTExNBcrQAwxmA0GlFdXQ2tVguxuHctpTpKp8UonbbLdP3jlbA7GNK1yvBnivgUtYEIAFJTUwHAHYxI4LRarft9jEa5SXHITYoTOht9XlQHIo7jkJaWhuTkZFitNGAtUFKptNeVhFxoekfvEtWByEUsFvfaLxQJHGMML39zGjKJCL+5PAuxPlZoBPiAZbLaYWcMakXvageLJlHbWE36rharA3YHc28V1JmKBhNe33UWHx2qiFDuiDcUiEjUcQ1mlEtFEHdRPZNTr1mPQIGIRB3XGKJYWdctD64BjTSyWlgUiEjUMfo5zwxoHUdktTPYaXtqwVAgIlHH2MV+Zm3J2iwRQtUz4VAgIlHHFEDVTCziaHR1D0CBiEQdtXN6R784/xaUk4lpBr7Q+sQ4ItK3jOivwYj+Gr/TD0lTwWZn7oZrEnkUiEifNz0vSegs9HlUNSNRx0G9X70OlYhI1Fm/8wxEHIe7Ls/0a9qG3cFgttkh4rguR2KT8KBARKKKxeZw9351tntHW9/8Uo0jFY2YnJuIywckhjN7xAdBq2aPPfYYOI7zuA0ZMkTILJFeztV1LxFx7t6wrlD3vfAELxENHz4c27dvd9+XSATPEunFjFbnYEa5xO+F8Gi5WOEJ/q2XSCRRvfAWiSx/16pui0pEwhO81+zUqVNIT0/HgAEDcNddd/ncbhoAzGYz9Hq9x42QtozmwAOReycPOw1oFIqggWjSpEkoLi7Gli1bsH79epSUlGD69Oloamrymr6wsBAajcZ90+l0Ec4x6ela55n5X9inEpHwONaD9otpaGhAVlYW1q5di3vuuafD42azGWaz2X1fr9dDp9OhsbERarU6klklPdSxC3r8clGPAUlxGO3H4vkAUF5nxIcHz6NfnAy/mZwd1vz1NXq9HhqNpsvvqOBtRG1ptVoMGjQIp0+f9vq4XC6HXO7f/CHSNw1LV2NYemD/lOLkEgxNU9FSsQISvI2orebmZpw5cwZpaWlCZ4X0IfGxMswekYYpA/sJnZU+S9BAtHLlSuzcuROlpaX4/vvvcfPNN0MsFuOOO+4QMlukF6PpHb2ToFWz8+fP44477kBtbS2SkpIwbdo07NmzB0lJNAmRBOf1b8+CMWDhBB3iYzvf5bUtm90Bi90BhURM2xAJQNBAtGnTJiFfnkQZh3PnDoBfON9fjDG8uuMM7A6GJdNzoKK2oojrUW1EhHSH0coHIY4DFAGsLcRxtEqj0CgQkajRdq3qQKtXrnlpFjsFIiH0qO57QrrD5N69o5M/6xY9cG43YG4CUkcCSYMA0KBGoVEgIlHDPc/M15pC5mbgh7f5YAQAl04BQ28AUi9zByKa+CoMqpqRqNHlNkKnvuSDkDIeSBnuPLYVMDfRjq8CoxIRiRpxcimyEmOQrFZ0fLDpIlBzkm/JHnELENMPMNUB+krg/H7IxEMBUIlIKBSISNQYnKrC4FSV9wfP7eF/Jg8D4pL537OmAUc+AC4cRv+skeA4FeJjqOteCBSISPSzmoBLJ/nfdZNajyfmAnIVYG7CSGUtRmYOFiZ/hNqISPTwuXd99XHAYQfikgBVSutxjgNShvG/V/0c/gwSnygQkajxxrdn8eqO06gzWDwfqD7G/0y5rOOTkpxrpNeXwG6z0W6vAqFARKKCw8Fgstphtjo8d++wtgCNFfzvzjFDHlRpgFSJmvomFH3xHT7/qTIyGSYeKBCRqGC02sEYX9tSth1HVF8CMAcQk8h327fHcUBCDsQiDtqWcuq+FwgFIhIVfE7vqD3D/0wc4PvJ8dkQiziozBdpiodAggpEZ8+eDXU+COkW16L5HtM7GAPqnH+rCbm+n6zOgFjEIc5SA6vVGsZcEl+CCkQDBw7ElVdeiXfeeQctLS2hzhMhATM4S0SxbUdVG2sBiwEQSQBNJxstxCRAJIuBiNkhNlaHOafEm6AC0aFDhzBy5EisWLECqampuO+++7Bv375Q540Qv5nc+5m1KRE1lvM/1emAuJMhcxwHcXwGAEBhqEQP2k+izwgqEI0ePRrr1q3DhQsX8NZbb6GyshLTpk3DiBEjsHbtWtTU1IQ6n4R0Kk4hQXa/GCSr22yu0OAMRJqMLp8v1vJpVJYqmuYhgG41VkskEtxyyy344IMP8Mwzz+D06dNYuXIldDodFi1ahMpK6golkTEkVY2bx2RgbGabnrHG8/xPbdf734k16UiIlSFH0RymHJLOdCsQHThwAA888ADS0tKwdu1arFy5EmfOnMG2bdtw4cIFzJs3L1T5JCQwLY38jeMAdf8uk3NxKRicosIwrQMKUIN1pAUViNauXYvLLrsMU6ZMwYULF7Bx40aUlZXhr3/9K3JycjB9+nQUFxfj0KFDfp9zzZo14DgODz30UDBZIn1ch+kdrtJQXAog8WMvPFkMoHDuh2agButIC2rS6/r167F48WIUFBT43IMsOTkZb775pl/n279/P1577TWMHDkymOwQgn9+exYOxnDHhEx+9w79Bf4BP9qH3OJS4DA1gukvQqzNDE9GiVdBlYi2bduGhx9+uEMQYozh3LlzAACZTIb8/Pwuz9Xc3Iy77roL//znPxEf72XkKyFdsDt37zBbHa27dzRX8T9VqX6fZ1elCHtL6nDxQlkYckk6E1Qgys3NxaVLlzocr6urQ05OTkDnWrZsGebOnYtZs2Z1mdZsNkOv13vcCHGNqhZxHD+9gzF+ITQAiPM/EFmV/H56XFNVyPNIOhdU1czXOIvm5mYoFF5Wx/Nh06ZNOHToEPbv3+9X+sLCQjz++ON+n5/0De61qmVicBwHGOsAu5UfyBiT6Pd5WCy/YBoz1vLLhoj835KIdE9AgWjFihUA+H2gVq9ejZiYGPdjdrsde/fuxejRo/06V3l5OX7/+99j27ZtfgevVatWufMAAHq9Hjpd112zJLo1m52jquXOP2d3aSgJEPlf6BcpNbBzUjjsNsDUAMT6H8RI9wQUiH744QcAfInoyJEjkMlat/SVyWQYNWoUVq5c6de5Dh48iOrqaowdO9Z9zG63Y9euXXj55ZdhNpshFnv+R5LL5ZDL/egBIX2KwR2InH8vrvahAKplACCTSNAi1cDuaAaMlygQRVBAgeibb74BANx9991Yt24d1Gp10C989dVX48iRIx7H7r77bgwZMgQPP/xwhyBEiC/NLXwgUimcf87uhuoUH8/wTiYRoU6aALujCTBcApJo6dhICaqNqKioqNsvrFKpMGLECI9jsbGxSExM7HCckM5oYqT89A6Vol1DdeCByCSNh91aypeISMT4HYhuueUWFBcXQ61W45Zbbuk07ebNm7udMUL8NTxdg+HpGv5OSyO/WD4nApyNz/6Kj5EiMSUdsTVH+BIRiRi/A5FGo+F7JJy/h8OOHTvCcl7ShzQ7R0XHJnY+496LrMRYZI0bBuzZxfe8ORwBNXaT4Pn9SbWtjoWiakZIqFjtDkjFzoARxPghD3INH8DsNqClAYhJCEkeSeeCCvcmkwlGo9F9v6ysDC+++CK2bt0asowR4g+r3YGXvz6NV3ec5nfgCGJEtQeRCA6ls7eMqmcRE1QgmjdvHjZu3AgAaGhowMSJE/H8889j3rx5WL9+fUgzSEhnXD1mDgeDTCxqUyIKrH0IABqNVrz89Sl8WeKcfW+sDVU2SReCXqFx+vTpAIAPP/wQqampKCsrw8aNG/H3v/89pBkkpDOuwYxxcgk4qxEwN/EPBNhjBgBSCQernaFZpOZnD5jqQ5lV0omgApHRaIRKxe8xvnXrVtxyyy0QiUS4/PLLUVZGEwZJ5LjXqpZLWqtlMQn+Lf3RjszZzmSSaGBnDDDVhSyfpHNBL57/8ccfo7y8HF9++SWuvfZaAEB1dXW3BjkSEihX1SxOLgFck1WDKA0BgEQsgljEoUWi4dc3MlIgipSgAtHq1auxcuVKZGdnY9KkSZg8eTIAvnQ0ZsyYkGaQkM64q2YKCdAc3EDGtuQSEVqkaj4QWQyAzRyKbJIuBDWy+tZbb8W0adNQWVmJUaNGuY9fffXVuPnmm0OWOUK64jHhtTa4qR1tySQiGEVyWDklACtfKlJ7X/yPhE5QgQgAUlNTkZrq2UU6ceLEbmeIkECkqBWw2Rn6KdDauBzsGCLwgQgALDItYK/hz0mBKOyCCkQGgwFr1qzBV199herqajgcntuv0E6wJFImZCdgQjaABn5lUCjU/PrTQUrXKhEnl0BsTwT0NdRgHSFBBaIlS5Zg586d+M1vfoO0tDT31A9CBNPNhmqXKwc7xx+VpQL6X6jBOkKCCkRffPEFPv/8c0ydOjXU+SHEb3YHg93B+OpUCBqqPSid66dTiSgiggpE8fHxSEigOThEWJWNJnxw4DxS1ArcKerm1I52mDIeHMCXiBjj90cjYRNU9/2TTz6J1atXe8w3IyTS9Ca+x0wusgMG53SMbpaI9pXU4eWvT2HXef7csJn5ZUVIWAVVInr++edx5swZpKSkIDs7G1Kp1OPxQDZWJCRYTS38nLB+nB5gDr6RWq7q1jk5DrDaGcwOMd/w3aLnq2fdaAAnXQsqEM2fPz/E2SAkcHrnqOp4h7MdJy6l21Uo1zQPs80BKBP4QGSsC2yjRhKwoALRo48+Gup8EBIwV4lIY28TiLrJPY7I5gDi4oH6UmqwjoCgl59raGjAG2+8gVWrVqGujv+gDh06hIqKipBljpDONLkWzbc624dC0FDtDkR2R+uiaDQLP+yCCkQ//fQTBg0ahGeeeQbPPfccGhoaAPBrVa9atcrv86xfvx4jR46EWq2GWq3G5MmT8cUXXwSTJdLHMMbQ1GIFx+xQWkLTUA20Vs0sNkebLnwKROEWVCBasWIFCgoKcOrUKY/NEa+//nrs2rXL7/NkZGRgzZo1OHjwIA4cOICrrroK8+bNw88//xxMtkgfYncwDElVY5DKDLmIARJZa+DoBrnUSyBydeGTsAmqjWj//v147bXXOhzv378/Ll686Pd5brzxRo/7Tz31FNavX489e/Zg+PDhwWSN9BESsQizhqUAlVXALxw/vywEY32UUjH6a5WIkYsBhZY/p93Kz8SXx3U/48SroAKRXC6HXq/vcPzkyZNISkoKKiN2ux0ffPABDAaDe1mR9sxmM8zm1mUZvOWB9DGupWG7MeO+LZVCitsmtNnGXK7mtygy1VMgCqOgqmY33XQTnnjiCVitfK8Fx3E4d+4cHn74YfzqV78K6FxHjhxBXFwc5HI5fvvb3+Kjjz7CsGHDvKYtLCyERqNx32jf+77LaLE5F8vv5q4dXaGpHhERVCB6/vnn0dzcjKSkJJhMJsyYMQMDBw6ESqXCU089FdC5Bg8ejMOHD2Pv3r24//77kZ+fj2PHjnlNu2rVKjQ2Nrpv5eXlwWSfRIHvTtfi1a9P4Xx5KX9AFaalOqjnLCKCqpppNBps27YN3333HX788Uc0Nzdj7NixmDVrVsDnkslkGDhwIABg3Lhx2L9/P9atW+e1DUoul0MuD3wtYhJ9GowWKK0NkIscgFga0v3H3tlThkaTFbeN1yGJes4iIuBA5HA4UFxcjM2bN6O0tBQcxyEnJwepqalgjHV7SRCHw+HRDkSIN40mK2Ktl6BQiPjxQyGclGqzO2CxOfixRG17zkjYBBSIGGO46aab8N///hejRo3CZZddBsYYjh8/joKCAmzevBkff/yx3+dbtWoV5syZg8zMTDQ1NeG9997Djh078OWXXwZ6HaQPsdodaGqxIdFSA0WcOOTtQzKJGIDV2YXfpmpGs/DDJqBAVFxcjF27duGrr77ClVde6fHY119/jfnz52Pjxo1YtGiRX+errq7GokWLUFlZCY1Gg5EjR+LLL7/ENddcE0i2SB+jNzmndtjqIBFzIesxc3GNrjbb7IBCQ134ERBQIHr//ffxpz/9qUMQAoCrrroKjzzyCN59912/A9Gbb74ZyMsTAgBoMFkB5kAC6sEhNuQN1R7zzcSSNl34dRSIwiSgXrOffvoJs2fP9vn4nDlz8OOPP3Y7U4R0psFohdLWiBiRM1AoQ7tIn8c0D4B6ziIgoEBUV1eHlBTfxeCUlBTU19OHRcIrKU6O0VoTtDFSfn6ZKOi5217JJe0CEfWchV1AVTO73Q6JxPdTxGIxbDZbtzNFSGcyE2OQmdgCmBWAOj3k54+PlaG/Vslv2ghQz1kEBNxrVlBQ4HMsD3W7k4jRO5ebUYd+wbLROi1G67StB5RUNQu3gAJRfn5+l2n8bagmJBg2uwN1+iYkNlVDzAHQ9A//i7atmlEXflgEFIiKiorClQ9C/FJnsOCz/x3C6No6jBuc3e01qv2i1Lbpwm+OzGv2MaFt5SMkzGoNFqjMVVBIxYA6PKWh8joj/rnrLP59wDmXUSTmxxMBVD0LEwpEpEuNJit2nqzB7jO1/CA/AdUZLIizVEMpC18g4jig2WyD0dym44V6zsIqqEmvpO9osdrxwYFy9/rQ5XVG3DouAyKRMO0ktc1mxJuroYwRh619yGPdahdlAoAS6jkLEyoRkU4dLKtHU4sNMTIxZBIRKhpMOHy+QbD8mOovQuJogVKhCN320u3IxWIAbcYRAVQiCjMKRKRTIo6DSiHB1UNTMD2vHwDgQGkdrG1LCxFidzCwhnMAAHm/LL7tJgxc61Zb7Qx2h3OtagpEYUVVM9KpybmJuHwAP47Gwfgtmc02B6qbzOivVUY0L3UGC1SmCkhEHBRJOWF7Ham49f+zxebg26NiaBZ+OFEgIl1yrTEl5oD5Y/pDrZC621EiSSnlMErVCCgU4OKzw/Y6YhEHqZiD1c5aA5HHLHzqwg81CkTEpyp9C/rFySFu0zDdL064FTLjLLWIixMBkvjwrVHtlKJWwNF2CyFXF76pgW+wpkAUUhSIiFfNZhve23sOMokIS68Y4FFdAfjpPiarHTGyCP4J1ZfyP7WZIZ/o2t6C8V42ZlAm8IHIVA/EZ4X19fsaCkTEk7kZOLUVxspzyKxPQEvG1A5BqLbZjM+PVMJmZ7h7ana3lwf2B2MMF0uOQmu1QxGfDUFaaKjBOmwoEJFWdivw4/uA4RIMjQakN52HutkOsAEejbNqpRRNLTZYbA5caGyJSKO1Xq9H6ZmTEHEM46flIjz9ZV2grYXChrrvSavyvYDhEiCLxWnNZDg4MVLMZUC15/ZOUrEIuUn8SoW/VEZmk8v6878AYOBUKRDHdH9r6a5880s1/rnrLI5WNLYepAXSwoYCEeHZLMD5/QAANvBqnJQMRoV6DGJkYuDsDsDhObVjWJoaAHCyqrl1rE0YGS4cBwBIkweF/bUAflR1s9mGFmub624/C5+EjKCBqLCwEBMmTIBKpUJycjLmz5+PEydOCJmlvqvmOGBtAZTxaIzLhcXmQJV2FBSxGqBFD1T97JE8I16JOLkELVY7Si4Zwps3awvsNacAAHEZ3ncBDjVZ+1UaAWcXvgiw2wBzU0Ty0VcIGoh27tyJZcuWYc+ePdi2bRusViuuvfZaGAxh/sMmHVU5q19pI3HJwO+SkaCKgUg3gT9ecdAjuUjEYXAq34X9y8XwVs+sVcdhNJlhkmqRnBaZ3iq52LWTR5tARLPww0bQxuotW7Z43C8uLkZycjIOHjyIK664QqBc9UEWA9BQxv+ePBSJDjmm5fXj125O7geUfgs0XQSaq4G4ZPfThqSpcLCsHmdrDGix2vmlOcKgqewwGACDdgjUSmlYXqO91i2F2k1lUcbzQYi68EOqR/WaNTbyDYMJCd53ZTCbzR7L0er1kWkojXp1Z/k2D1UKoIxHPIAJsW0+g8SBQM0JoPInIK91W/FklQLjs+Ohi49x73wRcoZaGKvOAuAg7X9ZRIYKAD5m4APUcxYmPaax2uFw4KGHHsLUqVMxYsQIr2kKCwuh0WjcN53Oy6AzEjjXQMF4H/O30kbxP6uOAg7PL+b0vCRk94sN37Ig5/cjVaNAzpCRGDEgcp+3XMKX7szWdusvUc9ZWPSYQLRs2TIcPXoUmzZt8plm1apVaGxsdN/Ky8sjmMMoxRhQV8L/njAAdgfD6epm1DabwVw9Q/E5gFQJWE1AQ2knpwpxT5K5Cag6AolIhNRhVyAjPia05+9ErFyMxDhZx6ogDWoMix5RNVu+fDn+85//YNeuXcjI8L0rg1wu97mDCAmSoYZvIxJLAXV/NJqs+OzHC5BJRHhgZi6fRiQCkoYAF34Aqn8BEgZ4nMJkseNAWR2q9Gb8amz/0FWfSv/H91Bp+vPTOiIoIz4GiyZnd3yAFtIPC0FLRIwxLF++HB999BG+/vpr5OSEb2kH4kOjs1SpyQDEEtQbLQAAbYzUM6AkD+F/XjrRYUyRnTH8dL4R5XVG/Hi+ESHRWAFU/oiyWgMOSsag2SLsErVu1IUfFoIGomXLluGdd97Be++9B5VKhYsXL+LixYswmUxCZqtv0V/gfzrXf25wBqL4GJlnOk0mIIvhxxq52pSc4uQSTB3IL5q262QNyuuM3cuT1QQc/xQ2ux0/23XYVSX3XD9aSB5d+NRgHSqCBqL169ejsbERM2fORFpamvv2r3/9S8hs9S2Nro0K+R1T651jiLQx7dpGXNUzgO9Ba2dUhgZ5KXGwOxg++qEC35+5hEaTNfD8WIzAT/8GTA2osSpwWjsFCbEyJKkiXyV/b+85vL7rDPQt7a4jlg+6MNRGPE/RStA2opA3bpLAWAytja7OElG9rxIRwAeiikPO6tl1Hku1chyH2cNTAVzEqapm7D1bh71n63DDyDTkpfADH5vNNhgtNiTGeq5x5FZ3FjixBWhpBJMo8C13BexWOYanqyPWbd+WwWyDwWxHi8UOtaJNYI5JBHAKMF6KeJ6iVY9orCYCcZWGYvsBUgUAoMHI//f3Gog0OkAWywew+lIgMdfjYYlYhLmXpeFUSjMOn2tAZWML4mNbz3PiYhN2nayBQirGiP5qjNJp+S94czVw5hs+EAGAMh6nk69BxRk7ZBIRRvTXhPzS/aGQitBsBkztu/DdJSIKRKFCgagvc+8fz5eGzDY7mp1tMR2qZkBr9aziID8jv10gAviS0aAUFQalqGCxOSBpU/KxOxjkUhFarHYcKK3Hj2cvYgr3IwazUsTKxHwJK30s9GmT8PXBagDA+Kz4sI3Y7orrdVus7QY1xibxP6lEFDIUiPoyVyBy7g8m4jjMHZmGphar7y9/yjA+ENWcAPKuAyReSk5O7de1npiTgPFZ8SipNeD4iRNQnf4UZrsBPwGQpA7FuJk3QxSbgAO/VMFosaOfSo5xWeFf8sMX13vQoUQUk8h321uMfOlQFitA7qILBaK+yuEAmir5350lIqlYhEEpXazFrO7P7wVvagBqTwEpwwN6WZGIQ67kEnId36A5VYIyUzJ2yy5Hmi4PIue0kim5/VBvsOKa4SmQhGvqiB+U7hJRu0AklrauX+1cv4l0DwWivsp4iR8LI5E5G1/9xHF88Cn9jl8aJMBABGMdcPT/ALsVcWl5GD78ZvS3eDZEK6Ri/Gqc74GtkeKzRAQAMf2cC+lfosmvIdBjpniQCHONH4pLdY8OLrlkwNmaZpi6GjyY7Aw+dSV81cRfdhtw7GN+ETZNBjDiVkAihzZGBq23xnGBqZUSJMbJ3CUjD7HO4E1d+CFBJaK+quki/1Od5j60+0wtqvQtuHFUOgYmx/l+bmwi/zx9JV8q0k307zXPfA00VfHz1obNA8Q9+89vZIYWIzO03h+McfacUYN1SFCJqK9ytQ+p+EDEGGszhsiPNX9cM/IrDnaYke9V9fHWxdWG3ggo1IHmuGdxd+HXCJuPKEGBqC+y21q/QCp+o0KjxQ6LzQGOAzT+LD6WMoIfe2RqAOrOdJ7WWAec+C//e9Zkr93+vY6rRGQx8jfSLRSI+iJDDT9xVaoEFFoArSOq1Qqpfz1VYmlrqajse9+LybdvF8ruPStvNptteHtPGd78X0nHByWy1jlnVD3rNgpEfVGTs6FaleZuqHaNqPY6kNGXjAl8O4/+AnDplPc0p7d5tguFeYfWUJKJRbjUZIbeZPVcRN/FVT1rpupZd/WevwoSOq6GalXr/vHu9qHYAHqv5Cogw9lQfXobPzO/rYpDwIXDfLDrhe1CMokIUjEfqI0WL7P/Xet3G6ojmKvoRIGoL2rXUA0A9Z3NMetM5mR+gGOLvrUKxhjfMH1qK58m54pe2y4UI+N79gzehjTEpfA/m6simKPo1LP7T0no2a2tkzXblIim5CZiUEocUtWKwM4nkfFVrsPv8eOK9v4DkCgAo3N8Tf+xfLDqpWLlYjSarDB5LRG5AlEN33PYi6qdPQ29c31NcxVfYpHHeVSV+sXJMSRVHdzAQnU6MPJ2vvHWYuCDkEgC5F4J5F3bq5dTdZeIzF5KRMp4vtHeYWsNvCQoVCLqa/Qdq2UhodUBE5cCjef4njKtjm+g7uViZPyoaoO3EhHH8e1EjRV8gI9LinDuogeViPoad/tQa7XsUrMZP5yrR0VDN5foFUv4hfWTBkVFEAL4XsR+cTL39kIdxDnfR2on6hYqEfU17h6z1hJReZ0RO07UYGByHPproyOAhMq4rASMy/K+4SeA1p6zZuo56w4qEfUlFmNrW0abQNTpqoykc217zmjp46AJGoh27dqFG2+8Eenp6eA4Dh9//LGQ2Yl+rhn3MYn8jhxObbcQIgGKTeK3F7KagJYQbaXUBwkaiAwGA0aNGoVXXnlFyGz0Hfrz/E/niowu7jFEgQxm7CMMZhve3l2KN749632zB7GktXrmCvQkYIK2Ec2ZMwdz5swRMgt9S6PnGtUAYLU70NTiqppRiag9mUSES818idFsc3hfQlfdn297a7rAL6VLAtarGqvNZjPMZrP7vl6vFzA3vYzD3jrHTNO6+mG90QLG+NUIvS4A1sdJxSIoZWKYLHbofa3lrU4DKkAlom7oVY3VhYWF0Gg07ptOpxM6S71HcxU/vkeq8Fgattb53z4xVibI3mG9gUrB/79ubvGx26yrhNlU1WE7buKfXhWIVq1ahcbGRvetvLxc6Cz1Hg3O90qd4THSeWByHO6YmIlpef0EyljPp3JurtjkKxAp4/kA77DReKIg9aqqmVwuh1we+a2Ho4Jrv/p2C71LxSKkagKcX9bHuEpEPgMRx/Glotoz/Mh15/bdxH+9qkREgmS38VMvACA+W9Cs9EZqdyCydpLIGXxcPZMkIIKWiJqbm3H69Gn3/ZKSEhw+fBgJCQnIzMwUMGdRRn+eD0ay2NZdSgFYbA58d/oSEmJlGJmhoTYiHzRKGRLjZIiVd/J1cbUTNZTzAxvpvQyIoIHowIEDuPLKK933V6xYAQDIz89HcXGxQLmKQu5qWbbHF6TeaMHh8gbEyMQYpdMKkbNeYWByXOe7mgB8T6RIDJibAFM9ENPJtBDSgaCBaObMmd4HiZHQci3jmpDjebiZHwqRGEftbt0mlvLVs4ZyoKGMAlGAqI0o2hlq+YXQOBGQONDjoeomPhD1i6MR1f5gjHX+j1Pr7AioL4tMhqIIBaJod+kk/zM+q8PSHNV6fo3plEBXZeyDthytxKs7zqDkUic727p6JBvKaAJsgCgQRTPGgKqj/O9Jgz0ecjgYapwlIgpEXbM7+MZ917w8r1Tp/Nwzi5E2XgwQBaJo1lTJV8tEEiBpqMdDdUYLrHYGmUREc8z84HqP6g0W34nEktbqWe1p3+lIBxSIotmFw/zPpMH8yN82XFM7klVy6rb3g2stb9eSKT652uF87fNGvOpVI6tJAMxNrdWy9NEdHh6cqkJWYgxM3rbJIR0kOJdIaeisagYA/fKAk1/yE2DNzfwmBaRLVCKKVuV7+QmYmgxA431ysEIqpjWI/ORaNK7ZbOs8eMtV/Gx8AKilUpG/KBBFI2Mdv8sqAGRNoVG+IaCQiqFR8sHI1cjvU2Ie/7PmZJhzFT0oEEUbxoCTW/jSUMIA/tbO2Zpm/PtAOX463xD5/PVi2f1iMCApFmJxF4E92dkxUF/CV5FJl6iNKNqUfc8PqBNLgLxrvJaGymqNqKg30UDGAF01JMW/hDEJ/HK8jRVA1c9A5uXhzVgUoBJRNKk6BpR+y/+ed53XaQaMMZx1DsrLSoyNZO76ltTL+J8Xj9DgRj9QIIoWNSeB45/xf/T9xwJpI70nazZDb7JCKuaQmRDjNQ3pXKPJihZrF72NSUP5+WeGS3wVjXSKAlE0qDgI/LwZYA4gdQS/37wPZ6r50pAuIQZSMX38gfrsxwt4638lOFPT3HlCqQJIG8X/Xr4v/Bnr5egvsTdjDDjzNXByK/972khg8FyfvWSMMRyr5DccyEtWRTKnUcM1nuhcrbHrxBkT+M+iroQW1u8CBaLeymYGfv4IOLeXv59zBTD4ekDk+yMtrzNBb7JCJhEhL4UG2gUjpx/frnb2kgFWu6PzxEotkDKc//30dmor6gQFot7IWAcc2gjUnOAX4xp6I5A9tcvxQrFyMXKT4zA0TUXVsiClaRRQKSSw2Bwoq+1kJr5Lzgy+B7OxonWkO+mA/hp7m0ungIPFfCOoPA4YfSffLuSHxDg5bhqVjpmDksObxyjGcRyGpKoBAD+ca+j6CQo1kDWV//3UVv6fCOmAAlFvYbMAJ7YARz7kq2WaDGBcgcdmib7YHZ4LeolENNK6O0bpNBBxHM7Xm3C+3o+2It3lgFbHf4ZH/w+w+FGS6mN6RCB65ZVXkJ2dDYVCgUmTJmHfPuplcGMMqP4FOPAmcOEH/ljGBL4kJO+6wZkxhm3HqvCfnyphsXXRpkH8olJIMaI/Xyo6dsGP3YZFImDoTXwJ1nAJOPwelYzaETwQ/etf/8KKFSvw6KOP4tChQxg1ahSuu+46VFdXC501Ydlt/KjcQxv4RmlTAx94Ri0E8mbxbUNdaDbb8NlPlTheqcfZGgOqm1rCn+8+YurAfrhmWAquGebnaGuFGhh1J7+TiuEScLCI79a3dzGbv4/gmMCr10+aNAkTJkzAyy+/DABwOBzQ6XT43e9+h0ceeaTT5+r1emg0GjQ2NkKtVkciu6HHGGC3AFYjv/tDcw2//U9dSesfqVgC6CbxRXxJ59MyapvNqG4y41ydEaeqmmC1M4hFHK4dnuJu2yChV2ew4KfzDchKjEVCrAwqucR7FbhFDxz7mG+8BvjPMzGPr7rFJgEKLSCN6bT3szfx9zsq6Fwzi8WCgwcPYtWqVe5jIpEIs2bNwu7duzukN5vNMJtbZz7r9X4UiwH+S33mK+/dpx7HmO9jXaR1MIZD5+p8PM0BbYwMeUmtUyoOltXBwQCRwwqOeY7SVSkk/DgfeRyQPgbvnkuAoVwOdq7c49SMAakaOW4e09pOtGl/uUcVLFWjwFVDkmk52DDbX1qHYxf0Hg3YUjEHmUQEsUiEBeMzoFZIAYUaBxOuR2XNfqTUH4LMWg2Uepb+B6eooFDIAbEcFXorLjS2AODAOI7/CQ7gAIDD8HQ14pz7rV1oaEG5s81KzHEYnx0f3ou+7Da+pBcCggaiS5cuwW63IyXFs3ibkpKCX375pUP6wsJCPP7444G/kM3MlzTCiTFYjb5nWttFFsDa+l/OZjbC0SZYOTgxzBIVTNJ4NMX3R964SYAqFeA4NJaehtnHlAKz1bPdJz5GBhHHB6BBKSqkaRS0AmMEDErh2+suNrag0WSF3cFgtTNY7XYAnp+dwcJwSpSLUwkDoDJfhLblPFSWKsitesjtBjgY46vmdhscJiNsBpPvF242A1bn17jZBEcTH4g4EQc0+9giO1RY6Noce9Xs+1WrVrk3YQT4EpFO533RLw9aHd+2ArQZa9Pmy+nxRQ3kOOfxqG5gi/Nhz7QcALlUBMS2lkqyh7TwhSmxFEwaA04sc59aJhYBbRYsWzBOBwYGzvl6rtNzACTtxgPdOYl2yBVCTr9Y92BHh4PBbHPAYnPAbLfD7mCIkba26Y3M0CCnX6yzMK0DMKH1RA47JHEcACtgs0BjNEPUYnGWvFlrqdwZBKQqGeD8G4gz25Bp4oMPxwHQhLkULAvdpGlBA1G/fv0gFotRVVXlcbyqqgqpqakd0svlcsjlQWwGKIvtsLlgqHEAMgIopQYysDlJRRsg9iYiEQelTAylTAyg48YE2hiZew3srqhVgL9/Vmr4n7anEbRFTCaTYdy4cfjqq6/cxxwOB7766itMnjxZwJwRQiJJ8KrZihUrkJ+fj/Hjx2PixIl48cUXYTAYcPfddwudNUJIhAgeiG6//XbU1NRg9erVuHjxIkaPHo0tW7Z0aMAmhEQvwccRdUdUjCMiJIr5+x2NjlFThJBejQIRIURwFIgIIYITvLG6O1zNW35P9SCERJTru9lVU3SvDkRNTfyUCr9GVxNCBNPU1ASNRuPz8V7da+ZwOHDhwgWoVKoeOZ/KNQWlvLw8Knv1ovn6ovnagMhdH2MMTU1NSE9Ph6iTFQV6dYlIJBIhI6PrFQqFplaro/KP2SWary+arw2IzPV1VhJyocZqQojgKBARQgRHgSiM5HI5Hn300eBWDOgFovn6ovnagJ53fb26sZoQEh2oREQIERwFIkKI4CgQEUIER4GIECI4CkQBCnRX2g8++ABDhgyBQqHAZZddhv/+978ejzPGsHr1aqSlpUGpVGLWrFk4depUOC/Bp0Cu7Z///CemT5+O+Ph4xMfHY9asWR3SFxQUgOM4j9vs2bPDfRk+BXJ9xcXFHfKuUHguRt9bP7uZM2d2uDaO4zB37lx3moh/doz4bdOmTUwmk7G33nqL/fzzz+zee+9lWq2WVVVVeU3/3XffMbFYzJ599ll27Ngx9uc//5lJpVJ25MgRd5o1a9YwjUbDPv74Y/bjjz+ym266ieXk5DCTyRSpy2KMBX5td955J3vllVfYDz/8wI4fP84KCgqYRqNh58+fd6fJz89ns2fPZpWVle5bXV1dpC7JQ6DXV1RUxNRqtUfeL1686JGmt352tbW1Htd19OhRJhaLWVFRkTtNpD87CkQBmDhxIlu2bJn7vt1uZ+np6aywsNBr+ttuu43NnTvX49ikSZPYfffdxxhjzOFwsNTUVPa3v/3N/XhDQwOTy+Xs/fffD8MV+BbotbVns9mYSqViGzZscB/Lz89n8+bNC3VWgxLo9RUVFTGNRuPzfNH02b3wwgtMpVKx5uZm97FIf3ZUNfOTa1faWbNmuY91tistAOzevdsjPQBcd9117vQlJSW4ePGiRxqNRoNJkyb5PGc4BHNt7RmNRlitViQkJHgc37FjB5KTkzF48GDcf//9qK2tDWne/RHs9TU3NyMrKws6nQ7z5s3Dzz//7H4smj67N998EwsXLkRsrOc+ZZH87CgQ+amzXWkvXrzo9TkXL17sNL3rZyDnDIdgrq29hx9+GOnp6R5fiNmzZ2Pjxo346quv8Mwzz2Dnzp2YM2cO7Hbvu9aGSzDXN3jwYLz11lv45JNP8M4778DhcGDKlCk4f/48gOj57Pbt24ejR49iyZIlHscj/dn16tn3pGdYs2YNNm3ahB07dng06C5cuND9+2WXXYaRI0ciNzcXO3bswNVXXy1EVv02efJkj731pkyZgqFDh+K1117Dk08+KWDOQuvNN9/EZZddhokTJ3ocj/RnRyUiPwW6Ky0ApKamdpre9TOQc4ZDMNfm8txzz2HNmjXYunUrRo4c2WnaAQMGoF+/fjh9+nS38xyI7lyfi1QqxZgxY9x5j4bPzmAwYNOmTbjnnnu6fJ1wf3YUiPwUzK60kydP9kgPANu2bXOnz8nJQWpqqkcavV6PvXv3RnSn22B33H322Wfx5JNPYsuWLRg/fnyXr3P+/HnU1tYiLS0tJPn2Vyh2FLbb7Thy5Ig77739swP4oSVmsxm//vWvu3ydsH92EWsWjwKbNm1icrmcFRcXs2PHjrGlS5cyrVbr7tb9zW9+wx555BF3+u+++45JJBL23HPPsePHj7NHH33Ua/e9Vqtln3zyCfvpp5/YvHnzBOsCDuTa1qxZw2QyGfvwww89unibmpoYY4w1NTWxlStXst27d7OSkhK2fft2NnbsWJaXl8daWloiem3BXN/jjz/OvvzyS3bmzBl28OBBtnDhQqZQKNjPP//sTtNbPzuXadOmsdtvv73DcSE+OwpEAXrppZdYZmYmk8lkbOLEiWzPnj3ux2bMmMHy8/M90v/73/9mgwYNYjKZjA0fPpx9/vnnHo87HA72l7/8haWkpDC5XM6uvvpqduLEiUhcSgeBXFtWVhYD0OH26KOPMsYYMxqN7Nprr2VJSUlMKpWyrKwsdu+993YYixNJgVzfQw895E6bkpLCrr/+enbo0CGP8/XWz44xxn755RcGgG3durXDuYT47GgZEEKI4KiNiBAiOApEhBDBUSAihAiOAhEhRHAUiAghgqNARAgRHAUiQojgKBARQgRHgYj0KsXFxdBqte77jz32GEaPHu2+X1BQgPnz50c8X6R7KBARr1xrFv/2t7/t8NiyZcvAcRwKCgo80oc6AGRnZ+PFF1/0OHb77bfj5MmTPp+zbt06FBcXu+/PnDkTDz30UEjzRUKPAhHxSafTYdOmTTCZTO5jLS0teO+995CZmSlInpRKJZKTk30+rtFoPEpMpHegQER8Gjt2LHQ6HTZv3uw+tnnzZmRmZmLMmDHdOre3ksr8+fPdpayZM2eirKwMf/jDH9y7SAAdq2bttS2ZFRQUYOfOnVi3bp37HCUlJRg4cCCee+45j+cdPnwYHMdFfK0kwqNARDq1ePFiFBUVue+/9dZbuPvuu8P+ups3b0ZGRgaeeOIJVFZWorKyMuBzrFu3DpMnT8a9997rPkdmZmaHawKAoqIiXHHFFRg4cGCoLoEEgAIR6dSvf/1r/O9//0NZWRnKysrw3Xff+bWQVnclJCRALBZDpVIhNTU1qFUPNRoNZDIZYmJi3OcQi8UoKCjAiRMn3Ht/Wa1WvPfee1i8eHGoL4P4idasJp1KSkrC3LlzUVxcDMYY5s6di379+gmdrW5JT0/H3Llz8dZbb2HixIn47LPPYDabsWDBAqGz1mdRiYh0afHixSguLsaGDRtCVmoQiURovxSW1WoNybn9sWTJEndDfFFREW6//XbExMRE7PWJJwpEpEuzZ8+GxWKB1WrFddddF5JzJiUlebT72O12HD161CONTCbr9vY1vs5x/fXXIzY2FuvXr8eWLVuoWiYwqpqRLonFYhw/ftz9uy+NjY04fPiwx7HExETodLoOaa+66iqsWLECn3/+OXJzc7F27Vo0NDR4pMnOzsauXbuwcOFCyOXyoKqE2dnZ2Lt3L0pLSxEXF4eEhASIRCJ3W9GqVauQl5cX0QXvSUdUIiJ+UavVUKvVnabZsWMHxowZ43F7/PHHvaZdvHgx8vPzsWjRIsyYMQMDBgzAlVde6ZHmiSeeQGlpKXJzc5GUlBRUvleuXAmxWIxhw4YhKSkJ586dcz92zz33wGKxRKQXkHSO1qwmfda3336Lq6++GuXl5R12SiWRRYGI9Dlmsxk1NTXIz89Hamoq3n33XaGz1OdR1Yz0Oe+//z6ysrLQ0NCAZ599VujsEFCJiBDSA1CJiBAiOApEhBDBUSAihAiOAhEhRHAUiAghgqNARAgRHAUiQojgKBARQgT3/wGwzN7+n6CW1gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:09:15.392278Z", + "iopub.status.busy": "2024-03-22T14:09:15.391924Z", + "iopub.status.idle": "2024-03-22T14:09:15.784425Z", + "shell.execute_reply": "2024-03-22T14:09:15.783446Z" + }, + "papermill": { + "duration": 0.413747, + "end_time": "2024-03-22T14:09:15.786550", + "exception": false, + "start_time": "2024-03-22T14:09:15.372803", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDiElEQVR4nO2deXxTVd7/Pzd70ybpvtHQFig7ZRUEXMAVRAV0XEcFRWdU9HmQH88DPK8ZcBkFHXXkUWTUkVYfRWbcmBkXEEVgZARZREH20g3ovqdJs9x7fn/cJG3apE3SJJem3/frlVeSe09OvrdJPv2e7znn++UYYwwEQRASIpPaAIIgCBIigiAkh4SIIAjJISEiCEJySIgIgpAcEiKCICSHhIggCMkhISIIQnIUUhvQGwRBwIULF6DT6cBxnNTmEATRCcYYWlpakJmZCZnMt9/Tp4XowoULMBqNUptBEEQPlJeXIysry+f5Pi1EOp0OgHiRer1eYmsIguhMc3MzjEaj+7fqiz4tRK7hmF6vJyEiiIuYnkInFKwmCEJySIgIgpAcEiKCICSnT8eIiN7DGIPD4QDP81KbQvRB5HI5FApFr5fPkBD1Y2w2GyoqKmA2m6U2hejDaLVaZGRkQKVSBd2HpEKUk5OD0tLSLscfffRRrF+/XgKL+g+CIKC4uBhyuRyZmZlQqVS0KJQICMYYbDYbampqUFxcjLy8vG4XLXaHpEK0f/9+jyHB0aNHce211+K2226T0Kr+gc1mgyAIMBqN0Gq1UptD9FFiYmKgVCpRWloKm80GjUYTVD+SClFKSorH87Vr12Lw4MG48sorJbKo/xHsf7C+iMAYZOT1hZxQfIcumm+hzWbDe++9hwceeICGCETIcfACalusaLbYxQA9L0htEtGBiyZYvWXLFjQ2NmLhwoU+21itVlitVvfz5ubmCFhGRAM2XgADYHXwsLTwkHFAcpya/uldJFw0HtHbb7+N2bNnIzMz02ebNWvWwGAwuG+04ZXwF7tD9IBilHIAgMAAXmivpFVSUgKO43D48OFevY+//ezZswdjxoyBUqnEvHnzevWe0cBFIUSlpaX4+uuv8eCDD3bbbuXKlWhqanLfysvLI2Qh0dexO0VHKZdBIRO9IIfgu6Tfzp07wXEcGhsbw2LP0qVLMW7cOBQXF6OwsDAs79GXuCiGZgUFBUhNTcWcOXO6badWq6FWqyNkFdFXsNls3a5hYYy5vR+FU4gcAvPwiCJNUVERHn744W5TY/RET9cdShhj4HkeCkV4JENyj0gQBBQUFGDBggVhu0giMGwOweetc5C3u7Z2P9sGyowZM/DYY49hyZIlSE5OxvXXX4+jR49i9uzZiIuLQ1paGu69917U1tYCED2fHV9/hZuvvwpJiQnINWbgnttvwekzZ7z2X1JSgpkzZwIAEhISwHGcO3a5detWXHbZZYiPj0dSUhJuvPFGFBUVdenjxIkTmDZtGjQaDUaPHo1du3a5++Y4DnV1de6JGZdHtGvXLkyePBlqtRoZGRlYsWIFHA5Ht9ft8ty2bduG8ePHIyYmBldddRWqq6vx5ZdfYsSIEdDr9bj77rs9Fq4KgoA1a9YgNzcXMTExGDt2LD766CP3eVe/X375JSZOnAi1Wo3vvvsu4M/KXyT/5X/99dcoKyvDAw88ILUphJP133r/gQJAbnIs5o0f4H7+5u4i2HnvnkVWQgxum9Qex9u4pxgWW9etJE9cOzRgG9955x088sgj2LNnDxobG3HVVVfhwQcfxJ/+9CdYLBYsX74ct99+O3bs2AGHwGBubcWjjy/B9MkTUNfQhFWrV+PXd9yGIz//1GX62Wg04uOPP8att96KkydPQq/XIyYmBgDQ2tqKpUuXIj8/HyaTCatWrcL8+fNx+PBhj37+67/+C6+88gpGjhyJl19+GTfddBOKi4thNBpRUVGBYcOG4emnn8Ydd9wBg8GA8+fP44YbbsDChQvx7rvv4sSJE3jooYeg0Wjw5JNPer1uAKioqAAAPPnkk3jttdeg1Wpx++234/bbb4darcamTZtgMpkwf/58vPrqq1i+fDkAMd763nvv4c9//jPy8vKwe/du3HPPPUhJSfFYPrNixQq8+OKLGDRoEBISEgL+nPxFciG67rrrwJh0LjLRN8nLy8MLL7wAAPjDH/6A8ePH47nnnnOf37hxI4xGI06dOoXs3MGYf8utUMo56DRKZPMC/rT+zxg1yIhjx45h9OjRHn3L5XIkJiYCAFJTUxEfH+8+d+utt3q03bhxI1JSUrr089hjj7nbbtiwAVu3bsXbb7+N//7v/0Z6ejo4joPBYEB6ejoA4PXXX4fRaMRrr70GjuMwfPhwXLhwAcuXL8eqVavcItfxuoF2IfrDH/6A6dOnAwAWLVqElStXoqioCIMGDQIA/OpXv8K3336L5cuXw2q14rnnnsPXX3+NqVOnAgAGDRqE7777Dm+88YaHED399NO49tprA/psgkFyISIuPhbPHOLznKzTbPdvrhjss23nmfEHpuf2xiwPJk6c6H78008/4dtvv0VcXFyXdkVFRRg6dCjKSs5i1apV2LdvH2prayEI4pCwtLS0ixB1x+nTp732U1ZW5tGP6wcOAAqFApMmTcLx48d99nv8+HFMnTrVYznB9OnTYTKZcO7cOQwcOLDLdXckPz/f/TgtLQ1ardYtQq5jP/zwAwDgzJkzMJvNXQTGZrNh/PjxHscmTZrk0+ZQQkJEdEGl8D90GK62PREbG+t+bDKZcNNNN+H555/v0i4jIwMAcNNNNyE7OxtvvfUWMjMzIQgCRo8eDbvdHtD7+urHZrP17oL8pON1d0SpVLofcxzn8dx1zCWaJpMJAPD5559jwIABHu06Twb5er9QQ0JE9HkmTJiAjz/+GDk5OV4nPGpra3Hy5Em89dZbuPzyywGgx8Crazaq417Iuro6v/vZu3cvrrjiCgCAw+HAwYMH8dhjj/l8vxEjRuDjjz8GY8ztFe3Zswc6na5XM2veGDlyJNRqNcrKyi6a7VSSz5oRRG9ZvHgx6uvrcdddd2H//v0oKirCtm3bcP/998PhcMCu0CIxMQlvvPEmzpw5gx07dmDp0qUAAMHHFH52djY4jsNnn32GmpoamEwmJCQkICkpCW++2bWfzqxfvx6ffvopTpw4gcWLF6OhoaHbCZlHH30U5eXlePzxx3HixAn8/e9/x+rVq7F06dKQ7wfU6XRYtmwZnnjiCbzzzjsoKirCoUOH8Oqrr+Kdd94J6Xv5CwkR0efJzMzEnj17wPM8rrvuOowZMwZLliwRg8wcB5lMhj9vfBeHDh3E6NGj8cQTT2D1H8TAts3HnrMBAwbgqaeewooVK5CWlobHHnsMMpkMmzdvxsGD7f388Y9/9Pr6tWvXYu3atRg7diy+++47/OMf/0BycrLPaxgwYAC++OIL/PDDDxg7diwefvhhLFq0CL/73e96/ffxxjPPPIPf//73WLNmDUaMGIFZs2bh888/R25u6OJ4gcCxPjxl1dzcDIPBgKamJqriESBtbW0oLi5Gbm5u0Kkb+gI2B48Gsx1yjkOyrj3+0dJmh9nGQ6uSQ6dRdtMD0RPdfZf8/Y2SR0RENa4lTrJO032udCBC3/0/HFWQEBFRjSsGJO+0lMAtRJQN5KKAhIiIalyRB66LRyTek0d0cUBCREQ17qFZp9WVrqGahPteiQ6QEBFRjULGeaT+cNHRI+rD8zVRAy1oJKKaWLUCsV4yx8g4DjFKOeWwvkggISL6JRzHQR9D0/YXCzQ0I6IWRsOuPgN5RETUwgDUtFh9JsoXBOYuMdR5nRERWcgjIvocM2bMwJIlS3ps51pDxBi8VutobrOjrtWGNkfXZG3hsIfwDQkREbUIPqbuATH3jkucaPQmPSRERJ9i4cKF2LVrF9atWweO48BxHIqKirBo0SJ3/uVhw4Zh3bp17sWKMpn4unnz5uHZZ59FZmYmhg0bBhkH7N/3PaZOngiNRoNJkyZhy5YtXcoBdZcP25s9JSUlEvxl+jYUIyLaYQzgA0sUFjLkyq4pHb2wbt06nDp1CqNHj8bTTz8NQExwn5WVhQ8//BBJSUn497//jd/85jdITEnFdTfOd3tE33zzDfR6PbZv3w4AMLW04L47foVrZ83C5g8+QGlpaZchVk/5sL3Z07mUOtEzkgvR+fPnsXz5cnz55Zcwm80YMmQICgoKIpaikugAbwf+9ZI07335/wMUPZfGMRgMUKlU0Gq17nzPAPDUU0+5H+fm5uL777/Hxx995CFEsbGx+Mtf/uJOeva/r60Hx3FY99oGpCfqMXLkSJw/fx4PPfSQu6/XXnut23zYQ4cO9WoPERiSClFDQwOmT5+OmTNn4ssvv0RKSgpOnz4d1moBRHSyfv16bNy4EWVlZbBYLLDZbBiTPxZA+yrqMWPGeNQBO336FEaMGg2Vuj11xeTJkz369ScfNtF7JBWi559/HkajEQUFBe5jUiVmIiAOjy7/f9K9d5Bs3rwZy5Ytw0svvYSpU6dCp9Phj3/8I77fuxcquQxyuRgK7Zx/2TUQ7G6tkT/5sIneI6kQ/eMf/8D111+P2267Dbt27cKAAQPw6KOPerjGHbFarbBare7nzc3NkTK1f8Bxfg2PpEalUnnkkt6zZw+mTZuGRx991H2sqKgIMo5DQqzv6xk2bBg2bdoETnAAEPeB7N+/36NNT/mwvdlDBI6ks2Znz57Fhg0bkJeXh23btuGRRx7Bf/zHf/jMm7tmzRoYDAb3zWg0em1HRDc5OTnYt28fSkpKUFtbi7y8PBw4cADbtm3DqVOn8Pvf/76LoHjj3nvuARMEPPH4ozh+/Di2bduGF198EUD7uqPu8mG7xKezPQIlOQocJiFKpZJNnTrV49jjjz/OLr30Uq/t29raWFNTk/tWXl7OALCmpqZImBtVWCwWduzYMWaxWKQ2JWBOnjzJLr30UhYTE8MAsBMnTrCFCxcyg8HA4uPj2SOPPMJWrFjBxo4d637NggUL2Ny5c7v0tWfPHpafn89UKhWbOHEi27Rpk7tPF6dOnWLz589n8fHxLCYmhg0fPpwtWbKECYLg1Z7i4uIw/wUuLrr7LjU1Nfn1G5VUiAYOHMgWLVrkcez1119nmZmZfr3e34skutKXhcgfBEFglU0WVtVsYTwv+GzH8wKzO3i3qLz33ntMqVQys9kcKVP7PKEQIkljRNOnT8fJkyc9jp06dQrZ2dkSWUREC67FjOL2Dt/tXn9rIwbm5GD44BwcO3rEvUbIVeueiAySCtETTzyBadOm4bnnnsPtt9+OH374AW+++SbefPNNKc0iooCO2zu87TNzUVNdheefexo1VVXIyMjAbbfdhmeffTZCVhIuJC8n9Nlnn2HlypU4ffo0cnNzsXTpUp+zZp2hckLBE+3lhKx2Ho0WOxQyDklxXjKjOalvtcLOMxhilNAo5RG0MHoIRTkhyVdW33jjjbjxxhulNoOIMtr3mXW/bYQDB4DyFkkNbXolopLudt53xHWadEhaSIj6OdHqCbg9oh720brrm4XboCgmFN8hEqJ+ilIpbqkwm80SWxIe5DIOKi/VOzrT7hFFpyBHAtd3yPWdCgbJY0SENMjlcsTHx6O6uhoAoNVqu51d6mvIAMTIAQgC2tocPtsJdh5yQYBg59EG2qYRCIwxmM1mVFdXIz4+HnJ58MF+EqJ+jCtthUuMCCIY4uPje50ChYSoH8NxHDIyMpCamgq7XaKEaGFCEBglxI8ASqWyV56QCxIiAnK5PCRfposFxhhe3XEGKoUM916ajVi176+5IDBY7Dx4xqDXUJ0zqaBgNRF1tNkF8AKDxcb3uEjxfKMFb+4+i08PnY+QdYQ3SIiIqMNsE4PTaqUM8h6GZ2qF+BOwOWgCX0pIiIiow2wTZ79iVT1HHtQK0WOy9rK2GdE7SIiIqMMlRDGqnuNeKqdHZOcZeIHWEkkFCRERdbiGZtoAhAig4ZmUkBARUYclgKGZXMa5xYiESDpIiIioQx+jxMBELZK7Sf/REZWzygfFiaSD1hERUcfoAQaMHmDwu/3wDB0cPHMHronIQ0JE9Hsuz6MS0VJDQzMi6hBo9qvPQR4REXVs2CUWV/z1pQP92rbBCwxWBw8Zx1G6WIkgISKiCptDcM9+qRX+OfzfnqjGkfNNmDo4CZcOSgqneYQPJB2aPfnkk+CcVRZct+HDh0tpEtHHcU3dK5yJ0fyBpu+lR3KPaNSoUfj666/dz33VFycIfzDbnYsZ1Qq/E725PCcrCZFkSP6rVygUvU6qRBAuXNs7/FlV7YI8IumRfNbs9OnTyMzMxKBBg/DrX/8aZWVlPttarVY0Nzd73AiiI2Zr4ELkWj9k42lBo1RIKkRTpkxBYWEhtm7dig0bNqC4uBiXX345WlpavLZfs2YNDAaD+2Y0GiNsMXGx077PzH9nnzwi6ZG80mtHGhsbkZ2djZdffhmLFi3qct5qtcJqtbqfNzc3w2g0UqVXws2xC804UdmMQSlxGGeM9+s15fVmfHTwHJLjVLh3ak5Y7etv9JlKrx2Jj4/H0KFDcebMGa/n1Wo11Gr/9g8R/ZORmXqMzAzsn1KcWoERGTpKFSshkseIOmIymVBUVISMjAypTSH6EQmxKswanYFpQ5KlNqXfIqkQLVu2DLt27UJJSQn+/e9/Y/78+ZDL5bjrrrukNIvow9D2jr6JpEOzc+fO4a677kJdXR1SUlJw2WWXYe/evUhJoU2IRHC8+a+zYAy48xIjEmJVfr/OwQuw8QI0CjmVIZIASYVo8+bNUr49EWUIzsodgJg4318YY3h9ZxF4geHBy3Oho1hRxLmoYkQE0RvMdlGEOA7QBJBbiOMoS6PUkBARUUPHXNWBDq9c+9JsPAmRFFxU0/cE0Rss7uod3Xyt25qBsu8BawuQng+kDAVAixqlhoSIiBrc+8x85RSymoAf/08UIwCoPQ2MuBFIH+MWItr4Kg00NCOihh7LCJ3eJopQTAKQNsp57CvA2kIVXyWGPCIiaohTK5GdpEWqXtP1ZEslUHNKjGSPvgXQJgOWeqC5Aji3Hyr5CADkEUkFCRERNQxL12FYus77ybK94n3qSCAuVXycfRlw5EPgwmEMyM4Hx+mQoKWpeykgISKiH7sFqD0lPjZOaT+eNBhQ6wBrC/Jj6pA/cJg09hEUIyKiB5+166uPAwIPxKUAurT24xwHpI0UH1f9En4DCZ+QEBFRw1/+dRav7zyD+lab54nqY+J92piuL0px5khvKAbvcFC1V4kgISKiAkFgsNh5WO2CZ/UOexvQdF587Fwz5IEuA1DGoKahBQVf7sHnP1dExmDCAxIiIiow23kwJo62YjquI2ooBpgAaJPEafvOcByQmAu5jEN8WzlN30sECRERFfjc3lFXJN4nDfL94oQcyGUcdNZK2uIhEUEJ0dmzZ0NtB0H0ClfSfI/tHYwB9c7vauJg3y/WZ0Eu4xBnq4Hdbg+jlYQvghKiIUOGYObMmXjvvffQ1tYWapsIImBanR5RbMdV1eY6wNYKyBSAoZtCC9pEyFRayBgPubk6zJYS3ghKiA4dOoT8/HwsXboU6enp+O1vf4sffvgh1LYRhN9Y3PXMOnhETeXivT4TkHezZI7jIE/IAgBoWitwEdWT6DcEJUTjxo3DunXrcOHCBWzcuBEVFRW47LLLMHr0aLz88suoqakJtZ0E0S1xGgVykrVI1XcortDoFCJDVo+vl8eLbXS2KtrmIQG9ClYrFArccsst+PDDD/H888/jzJkzWLZsGYxGI+677z5UVNBUKBEZhqfrMX98FiYM7DAz1nROvI/vuf6d3JCJxFgVcjWmMFlIdEevhOjAgQN49NFHkZGRgZdffhnLli1DUVERtm/fjgsXLmDu3Ll+97V27VpwHIclS5b0xiSCEGlrEm8cB+gH9Nici0vDsDQdRsYL0IAC1pEmqL1mL7/8MgoKCnDy5EnccMMNePfdd3HDDTdAJhN1LTc3F4WFhcjJyfGrv/379+ONN95Afn5+MOYQBHiBQd5x2t7lDcWlAQo/auGptIBGL6YJaa0G4geGx1DCK0F5RBs2bMDdd9+N0tJSbNmyBTfeeKNbhFykpqbi7bff7rEvk8mEX//613jrrbeQkOBlwRlB+MFbzu0dDa7tHc0XxHs/4kNu4tIgMAa+uTL0BhLdEpQQbd++HcuXL+9SCJExhrKyMgCASqXCggULeuxr8eLFmDNnDq655ppgTCEI8M7qHVa70F69w1Ql3uvS/e5nd4UM+4rrUXmhNAxWEt0R1NBs8ODBqKioQGpqqsfx+vp65Obmguf92zi4efNmHDp0CPv37/ervdVqhdVqdT9vbm7232gianGtqpZxnLi9gzExERoAxPkvRPaYFCgBcC1VYbCS6I6gPCJf6yxMJhM0Gi/Z8bxQXl6O//zP/8T777/v92vWrFkDg8HgvhmNPc+GENGPO1e1Sg6O4wBLA8DbxYWM2iS/+2Gx4j9WZq4T04YQESMgj2jp0qUAxDpQq1atglardZ/jeR779u3DuHHj/Orr4MGDqK6uxoQJEzz62L17N1577TVYrVbI5Z65h1euXOm2ARA9IhIjwmR1rqpWO7/Obm8oBZD5/79WFmMAzykh8A7A0gjE+i9iRO8ISIh+/PFHAKJHdOTIEahU7SV9VSoVxo4di2XLlvnV19VXX40jR454HLv//vsxfPhwLF++vIsIAYBarYZa7ccMCNGvaHULkfM744oPBTAsAwCVQoE2pQG8YALMtSREESQgIfr2228BiIKxbt066PX6oN9Yp9Nh9OjRHsdiY2ORlJTU5ThBdIepTRQincb5dXYHqtN8vMI7KoUM9cpE8EIL0FoLpFDq2EgRVLC6oKAg1HYQRNAYtEpxe4dO0ylQHbgQWZQJ4O0lokdERAy/heiWW25BYWEh9Ho9brnllm7bfvLJJ0EZs3PnzqBeR/RvRmUaMCrTID5paxKT5XMyIDa1+xd2IkGrRFJaJmJrjogeEREx/BYig8Egzkg4HxPERYnJmcYjNqn7HfdeyE6KRfbEkcDe3YC5HhCEgILdRPD4/Ul1HI7R0Iy4mLDzApRyp2AEsX7IA7VBFDDeAbQ1AtrEkNhIdE9Qcm+xWGA2m93PS0tL8corr+Crr74KmWEE4Q92XsBrO87g9Z1nxAocQayo9kAmgxDjnC2j4VnECEqI5s6di3fffRcA0NjYiMmTJ+Oll17C3LlzsWHDhpAaSBDd4ZoxEwQGlVzWwSMKLD4EAE1mO17bcRrbip277811oTKT6IGgMzRefvnlAICPPvoI6enpKC0txbvvvov//d//DamBBNEdrsWMcWoFOLsZsLaIJwKcMQMApYKDnWcwyfTi7gFLQyhNJbohKCEym83Q6cQa41999RVuueUWyGQyXHrppSgtpQ2DRORw56pWK9qHZdpE/1J/dELljDNZFAbwjAGW+pDZSXRP0Mnzt2zZgvLycmzbtg3XXXcdAKC6urpXixwJIlBcQ7M4tQJwbVYNwhsCAIVcBrmMQ5vCIJavNpMQRYqghGjVqlVYtmwZcnJyMGXKFEydOhWA6B2NHz8+pAYSRHe4h2YaBWAKbiFjR9QKGdqUelGIbK2Aw9rzi4heE9TK6l/96le47LLLUFFRgbFjx7qPX3311Zg/f37IjCOInvDY8FoX3NaOjqgUMphlati5GAB20SvSZ/T4OqJ3BCVEAJCeno70dM8p0smTJ/faIIIIhDS9Bg6eIVmD9uBysGuIIAoRANhU8QBfI/ZJQhR2ghKi1tZWrF27Ft988w2qq6shCJ7lV6gSLBEpLslJxCU5ABrFzKDQ6MX800GSGR+DOLUCcj4JaK6hgHWECEqIHnzwQezatQv33nsvMjIy3Fs/CEIyehmodjFzmHP9UWk60HyCAtYRIigh+vLLL/H5559j+vTpobaHIPyGFxh4gYnDqRAEqj2IcRZyII8oIgQlRAkJCUhMpD04hLRUNFnw4YFzSNNrcLesl1s7OsFiEsABokfEmFgfjQgbQU3fP/PMM1i1apXHfjOCiDTNFnHGTC3jgVbndoxeekQ/FNfjtR2nsfuc2DccVjGtCBFWgvKIXnrpJRQVFSEtLQ05OTlQKpUe5w8dOhQS4wiiO1raxD1hyVwzwAQxSK3W9apPjgPsPINVkLcXXLTU9yoATvRMUEI0b968EJtBEIHT7FxVnSA44zhxab0eQrm2eVgdAhCTKAqRuT6wQo1EwAQlRKtXrw61HQQRMC6PyMB3EKJe4l5H5BCAuASgoYQC1hEg6PRzjY2N+Mtf/oKVK1eivl78oA4dOoTz58+HzDiC6I4WV9J8uzM+FIJAtVuIeKE9KRrtwg87QQnRzz//jKFDh+L555/Hiy++iMbGRgBiruqVK1f63c+GDRuQn58PvV4PvV6PqVOn4ssvvwzGJKKfwRhDS5sdHOMRYwtNoBpoH5rZHEKHKXwSonATlBAtXboUCxcuxOnTpz2qtN5www3YvXu33/1kZWVh7dq1OHjwIA4cOICrrroKc+fOxS+//BKMWUQ/ghcYhqfrMVRnhVrGAIWqXTh6gVrpRYhcU/hE2AgqRrR//3688cYbXY4PGDAAlZWVfvdz0003eTx/9tlnsWHDBuzduxejRo0KxjSin6CQy3DNyDSgogo4wYn7y0Kw1idGKceA+Bho1XJAEy/2ydvFnfjquN4bTnglKCFSq9Vobm7ucvzUqVNISUkJyhCe5/Hhhx+itbXVnVakM1arFVZre1oGbzYQ/QxXathe7LjviE6jxO2XdChjrtaLJYosDSREYSSoodnNN9+Mp59+Gna7OGvBcRzKysqwfPly3HrrrQH1deTIEcTFxUGtVuPhhx/Gp59+ipEjR3ptu2bNGhgMBveN6t73X8w2hzNZfi+rdvQEbfWICEEJ0UsvvQSTyYSUlBRYLBZceeWVGDJkCHQ6HZ599tmA+ho2bBgOHz6Mffv24ZFHHsGCBQtw7Ngxr21XrlyJpqYm9628vDwY84koYM+ZOry+4zTOlZeIB3RhStVBM2cRIaihmcFgwPbt27Fnzx789NNPMJlMmDBhAq655pqA+1KpVBgyZAgAYOLEidi/fz/WrVvnNQalVquhVgeei5iIPhrNNsTYG6GWCYBcGdL6Y+/tLUWTxY7bJxmRQjNnESFgIRIEAYWFhfjkk09QUlICjuOQm5uL9PR0MMZ6nRJEEASPOBBBeKPJYkesvRYajUxcPxTCTakOXoDNIYhriTrOnBFhIyAhYozh5ptvxhdffIGxY8dizJgxYIzh+PHjWLhwIT755BNs2bLF7/5WrlyJ2bNnY+DAgWhpacGmTZuwc+dObNu2LdDrIPoRdl5AS5sDSbYaaOLkIY8PqRRyAHbnFH6HoRntwg8bAQlRYWEhdu/ejW+++QYzZ870OLdjxw7MmzcP7777Lu677z6/+quursZ9992HiooKGAwG5OfnY9u2bbj22msDMYvoZzRbnFs7HPVQyLmQzZi5cK2utjp4QGOgKfwIEJAQffDBB/if//mfLiIEAFdddRVWrFiB999/328hevvttwN5e4IAADRa7AATkIgGcIgNeaDaY7+ZXNFhCr+ehChMBDRr9vPPP2PWrFk+z8+ePRs//fRTr40iiO5oNNsR42iCVuYUipjQJunz2OYB0MxZBAhIiOrr65GW5tsNTktLQ0MDfVhEeEmJU2NcvAXxWqW4v0wW9N5tr6gVnYSIZs7CTkBDM57noVD4folcLofD4ei1UQTRHQOTtBiY1AZYNYA+M+T9J8SqMCA+RizaCNDMWQQIeNZs4cKFPtfy0LQ7ETGanelm9KFPWDbOGI9xxvj2AzE0NAs3AQnRggULemzjb6CaIILBwQuob25BUks15BwAw4Dwv2nHoRlN4YeFgISooKAgXHYQhF/Ut9rwz+8OYVxdPSYOy+l1jmq/iInvMIVvisx79jNCG+UjiDBT12qDzloFjVIO6MPjDZXXm/HW7rP42wHnXkaZXFxPBNDwLEyQEBE90mSxY9epGnxfVCcu8pOQ+lYb4mzViFGFT4g4DjBZHTBbO0y80MxZWAlq0yvRf2iz8/jwQLk7P3R5vRm/mpgFmUyaOEmdyYoEazVitPKwxYc88la7iEkEUEwzZ2GCPCKiWw6WNqClzQGtSg6VQobzjRYcPtcomT2WhkoohDbEaDShKy/dCbVcDqDDOiKAPKIwQ0JEdIuM46DTKHD1iDRcnpcMADhQUg97R28hQvACA2ssAwCok7PF2E0YcOWttvMMvODMVU1CFFZoaEZ0y9TBSbh0kLiORmBiSWarQ0B1ixUD4mMiakt9qw06y3koZBw0Kblhex+lvP3/s80hiPEoLe3CDyckRESPuHJMyTlg3vgB0GuU7jhKJIlRchirawI0GnAJOWF7H7mMg1LOwc6zdiHy2IVPU/ihhoSI8ElVcxuS49SQdwhMJ8dJlyEzzlaHuDgZoEgIX45qJ2l6DYSOJYRcU/iWRjFgTUIUUkiICK+YrA5s2lcGlUKG31wxyGO4AojbfSx2HlpVBL9CDSXiffzAkG907cxtk7wUZohJFIXI0gAkZIf1/fsbJESEJ1YTcPormCvKMLAhEW1Z07uIUJ3Jis+PVMDBM9w/PafX6YH9gTGGyuKjiLfz0CTkQJIIDQWswwYJEdEObwd++gBorUVrUysyW85Bb+IBNsgjOKuPUaKlzQGbQ8CFpraIBK2bm5tRUnQKMo5h0mWDEZ75sh6g0kJhg6bviXbK9wGttYAqFmcMUyFwcqRZS4Fqz/JOSrkMg1PETIUnKiJT5LLh3AkADJwuDXJt70tL98S3J6rx1u6zOHq+qf0gJUgLGyREhIjDBpzbDwBgQ67GKcUwnNePh1YlB87uBATPrR0jM/QAgFNVpva1NmGk9cJxAIAydWjY3wsQV1WbrA602Ttcd+dd+ETIkFSI1qxZg0suuQQ6nQ6pqamYN28eTp48KaVJ/Zea44C9DYhJQFPcYNgcAqrix0ITawDamoGqXzyaZyXEIE6tQJudR3Fta3hts7eBrzkNAIjL8l4FONSoOmdpBJxT+DKAdwDWlojY0V+QVIh27dqFxYsXY+/evdi+fTvsdjuuu+46tLaG+YtNdKXKOfzKyEdtq1glI1Gnhcx4iXj8/EGP5jIZh2Hp4hT2icrwDs/sVcdhtlhhUcYjNSMys1VquauSRwchol34YUPSYPXWrVs9nhcWFiI1NRUHDx7EFVdcIZFV/RBbK9BYKj5OHYEkQY3L8pLF3M2pyUDJv4CWSsBUDcSlul82PEOHg6UNOFvTijY7L6bmCAMtpYfBALTGD4c+RhmW9+hMe0mhTltZYhJEEaIp/JByUc2aNTWJgcHERO9VGaxWq0c62ubmyARKo576s2LMQ5cGxCQgAcAlsR0+g6QhQM1JoOJnIK+9rHiqToNJOQkwJmjdlS9CTmsdzFVnAXBQDhgTkaUCgI8d+ADNnIWJiyZYLQgClixZgunTp2P06NFe26xZswYGg8F9Mxq9LDojAse1UDDBx/6tjLHifdVRQPD8YV6el4Kc5NjwpQU5tx/pBg1yh+dj9KDIfd5qhejdWe2d8i/RzFlYuGiEaPHixTh69Cg2b97ss83KlSvR1NTkvpWXl0fQwiiFMaC+WHycOAi8wHCm2oQ6kxXMNTOUkAsoYwC7BWgs6aarEM8kWVuAqiNQyGRIH3kFshK0oe2/G2LVciTFqboOBWlRY1i4KIZmjz32GD777DPs3r0bWVm+qzKo1WqfFUSIIGmtEWNEciWgH4Amix3//OkCVAoZHp0xWGwjkwEpw4ELPwLVJ4DEQR5dWGw8DpTWo6rZilsnDAjd8KnkO3GGyjBA3NYRQbIStLhvak7XE5RIPyxI6hExxvDYY4/h008/xY4dO5CbG77UDoQPmpxepSELkCvQYLYBAOK1Sk9BSR0u3tee7LKmiGcMP59rQnm9GT+da0JIaDoPVPyE0rpWHFSMh8kmbYpaNzSFHxYkFaLFixfjvffew6ZNm6DT6VBZWYnKykpYLBYpzepfNF8Q7535nxudQpSgVXm2MwwEVFpxrZErpuQkTq3A9CFi0rTdp2pQXm/unU12C3D8H3DwPH7hjdhdpfbMHy0lHlP4FLAOFZIK0YYNG9DU1IQZM2YgIyPDffvrX/8qpVn9iyZXoUKxYmqDcw1RvLZTbMQ1PAPEGbROjM0yIC8tDrzA8OmP5/Hvolo0WeyB22MzAz//DbA0osauwZn4aUiMVSFFF/kh+aZ9ZXhzdxGa2zpdR6woumiti7hN0YqkMaKQBzeJwLC1tgddnR5Rgy+PCBCF6Pwh5/Dseo9UrRzHYdaodACVOF1lwr6z9dh3th435mcgL01c+GiyOmC2OZAU65njyE39WeDkVqCtCUyhwb+4K8Db1RiVqY/YtH1HWq0OtFp5tNl46DUdhFmbBOA0YK6NuE3RykURrCYkwuUNxSYDSg0AoNEs/vf3KkQGI6CKFQWsoQRIGuxxWiGXYc6YDJxOM+FwWSMqmtqQENvez8nKFuw+VQONUo7RA/QYa4wXf+CmaqDoW1GIACAmAWdSr8X5Ih4qhQyjBxhCfun+oFHKYLICls5T+G6PiIQoVJAQ9Wfc9eNFb8jq4GFyxmK6DM2A9uHZ+YPijvxOQgSIntHQNB2GpulgcwhQdPB8eIFBrZShzc7jQEkDfjpbiWncTxjGShCrkoseVuYENGdMwY6D1QCASdkJYVux3ROu922zd1rUGJsi3pNHFDJIiPozLiFy1geTcRzm5Gegpc3u+8efNlIUopqTQN71gMKL5+Skc17rybmJmJSdgOK6Vhw/eRK6M/+AlW/FzwAU6SMwccZ8yGITceBEFcw2Hsk6NSZmhz/lhy9cf4MuHpE2SZy2t5lF71AVK4F10QUJUX9FEICWCvGx0yNSymUYmtZDLmb9ALEWvKURqDsNpI0K6G1lMg6DFbUYLHwLU7oCpZZUfK+6FBnGPMic20qmDU5GQ6sd145KgyJcW0f8IMbtEXUSIrmyPX+1M38T0TtIiPor5lpxLYxC5Qy++gnHieJTskdMDRKgEMFcDxz9GODtiMvIw6hR8zHA5hmI1ijluHWi74WtkcKnRwQA2mRnIv1a2vwaAi6aLR5EhHGtH4pLd68OLq5txdkaEyw9LR5MdYpPfbE4NPEX3gEc2yImYTNkAaN/BSjUiNeqEO8tOC4x+hgFkuJUbs/Ig1ineNMUfkggj6i/0lIp3usz3Ie+L6pDVXMbbhqbiSGpcb5fG5skvq65QvSKjJP9e8+iHUBLlbhvbeRcQH5xf/3ys+KRnxXv/aTWOXNGAeuQQB5Rf8UVH9KJQsQY67CGyI+cP64d+ecPdtmR75Xq4+3J1UbcBGj0gVp8ceGewq+R1o4ogYSoP8I72n9AOrFQodnGw+YQwHGAwZ/kY2mjxbVHlkagvqj7tuZ64OQX4uPsqV6n/fscLo/IZhZvRK8gIeqPtNaIG1eVMYAmHkD7imq9RunfTJVc2e4Vlf7bdzL5znGhnL6TedNkdeD/9pbi7e+Ku55UqNr3nNHwrNeQEPVHWpyBal2GO1DtWlHtdSGjL7IuEeM8zReA2tPe25zZ7hkXCnOF1lCikstQ22JFs8XumUTfhWt4ZqLhWW/pO98KInS4AtW69vrx7vhQbACzV2odkOUMVJ/ZLu7M78j5Q8CFw6LY9cG4kEohg1IuCrXZ5mX3vyt/d2t1BK2KTkiI+iOdAtUA0NDdHrPuGDhVXODY1tw+BGNMDEyf/kpsk3tFn40LaVXizF6rtyUNcWnivakqghZFJxf3/CkRenh7+2bNDh7RtMFJGJoWh3S9JrD+FCpxyHV4k7iuaN+fAYUGMDvX1wyYIIpVHyVWLUeTxQ6LV4/IJUQ14sxhHxp2XmzQX66/YaoSPRZ1nMdQKTlOjeHp+uAWFuozgfw7xOCtrVUUIZkCGDwTyLuuT6dTdXtEVi8eUUyCGLQXHO3CSwQFeUT9jeauw7KQEG8EJv8GaCoTZ8rijWKAuo+jVYmrqlu9eUQcJ8aJms6LAh+XEmHrogfyiPob7vhQ+7Cs1mTFj2UNON/YyxS9coWYWD9laFSIECDOIibHqdzlhboQ5/w7UpyoV5BH1N9wz5i1e0Tl9WbsPFmDIalxGBAfHQISKiZmJ2JitveCnwDaZ85MNHPWGyT1iHbv3o2bbroJmZmZ4DgOW7ZskdKc6Mdmbo9ldBCibrMyEt3TceaMUh8HjaRC1NrairFjx2L9+vVSmtF/cO241yaJFTmcdCwhRARIbIpYXshuAdpCVEqpHyLp0Gz27NmYPXu2lCb0L5rPiffOjIwu3GuIAlnM2E9otTrwyaFzsDoELLost2sSf7lCHJ61VIpCHxMviZ19HQpW9yeaPHNUA4CdF9DS5hqakUfUGZVChlqTDS1tDli9bfMA2v+erq0zRMD0qWC11WqF1Wp1P29ubpbQmj6GwLf/UAzt2Q8bzDYwJmYj9JoArJ+jlMsQo5LDYuPR7CuXtz4DOI/2oS8RMH3KI1qzZg0MBoP7ZjQapTap72CqEtf3KDUeqWHrTGJ8KClWJUntsL6ATiP+vza1+ag26/aIqrqU4yb8o08J0cqVK9HU1OS+lZeXS21S36HR+bfSZ3msdB6SGoe7Jg/EZXnJEhl28aNzFlds8SVEMQmiwAsOWk8UJH1qaKZWq6FWR770cFTgqlffKdG7Ui5DuiHA/WX9DJdH5FOIOE70iuqKxJXrzvLdhP9IKkQmkwlnzpxxPy8uLsbhw4eRmJiIgQMHSmhZlME7xK0XAJCQI6kpfRG9W4js3TTKdArROQATI2NYFCGpEB04cAAzZ850P1+6dCkAYMGCBSgsLJTIqiik+ZwoRqrY9iqlAGwOAXvO1CIxVoX8LAPFiHxgiFEhKU6FWHU3PxdXnKixXFzYSH/LgJBUiGbMmAFGq1HDj3tYluPxA2kw23C4vBFalRxjjfFSWNYnGJIa131VE0CciZTJAWsLYGkAtN1sCyG60KeC1USQuNK4JuZ6HjaJSyGS4iju1mvkyvbYUGOptLb0QUiIop3WOjERGicDkoZ4nKpuEYUoOY5WVPsDY6x7Dz7eORHQQEIUKCRE0U7tKfE+IbtLao7qZjHHdFqgWRn7IVuPVuD1nUUoru2msq1rRrKxlDbABggJUTTDGFB1VHycMszjlCAw1Dg9IhKinuEFMbjv2pfnFV2muPfMZqbCiwFCQhTNtFSIwzKZAkgZ4XGq3myDnWdQKWS0x8wPXH+jhlab70ZyRfvwrO6M73ZEF0iIopkLh8X7lGHiyt8OuLZ2pOrUNG3vB65c3q6UKT5xxeF81XkjvNKnVlYTAWBtaR+WZY7rcnpYug7ZSVpYvJXJIbqQ6EyR0tjd0AwAkvOAU9vEDbBWk1ikgOgR8oiilfJ94gZMQxZg8L45WKOUUw4iP3EljTNZHd2Lt1on7sYHgDryivyFhCgaMdeLVVYBIHsarfINARqlHIYYUYxcQX6fJOWJ9zWnwmxV9EBCFG0wBpzaKnpDiYPEWyfO1pjwtwPl+PlcY+Tt68PkJGsxKCUWcnkPwp7qnBhoKBaHyESPUIwo2ij9t7igTq4A8q716g2V1plxvsFCCxkD5Krhaf411CaK6XibzgNVvwADLw2vYVEAeUTRRNUxoORf4uO8673ud2KM4axzUV52UmwkretfpI8R7yuP0OJGPyAhihZqTgHH/yl+6QdMADLyvTczWdFssUMp5zAwUeu1DdE9TRY72uw9zDamjBD3n7XWikM0oltIiKKB8weBXz4BmACkjxbrzfugqFr0hoyJWijl9PEHyj9/uoCN3xWjqMbUfUOlBsgYKz4u/yH8hvVx6JvYl2EMKNoBnPpKfJyRDwyb43OWjDGGYxViwYG8VF0kLY0aXOuJyurMPTfOukT8LOqLKbF+D5AQ9VUcVuCXT4GyfeLz3CuAYTcAMt8faXm9Bc0WO1QKGfLSaKFdMOQmi3G1s7WtsPM+ygu5iIkH0kaJj898TbGibiAh6ouY64FD7wI1J8VkXCNuAnKm97heKFYtx+DUOIzI0NGwLEgyDBroNArYHAJK67rZie8i90pxBrPpfPtKd6IL9G3sa9SeBg4WikFQdRww7m4xLuQHSXFq3Dw2EzOGpobXxiiG4zgMT9cDAH4sa+z5BRo9kD1dfHz6K/GfCNEFEqK+gsMGnNwKHPlIHJYZsoCJCz2KJfqCFzwTeslktNK6N4w1GiDjOJxrsOBcgx+xIuOlQLxR/AyPfgzY/PCk+hkXhRCtX78eOTk50Gg0mDJlCn74gWYZ3DAGVJ8ADrwNXPhRPJZ1iegJqXsOODPGsP1YFT77uQI2XyWTiYDQaZQYPUD0io5d8KPasEwGjLhZ9GBba4HDm8gz6oTkQvTXv/4VS5cuxerVq3Ho0CGMHTsW119/Paqrq6U2TVp4h7gq99A7YlDa0igKz9g7gbxrxNhQD5isDvzz5wocr2jG2ZpWVLe0hd/ufsL0Icm4dmQarh3p52prjR4Ye7dYSaW1FjhYIE7r8z3s5u8ncEziMhpTpkzBJZdcgtdeew0AIAgCjEYjHn/8caxYsaLb1zY3N8NgMKCpqQl6vT4S5oYexgDeBtjNYvUHU41Y/qe+uP1LKlcAximii6/ofltGncmK6hYryurNOF3VAjvPIJdxuG5Umju2QYSe+lYbfj7XiOykWCTGqqBTK7wPgduagWNbxOA1IH6eSXni0C02BdDEA0ptt7OffQl/f6OS7jWz2Ww4ePAgVq5c6T4mk8lwzTXX4Pvvv+/S3mq1wmpt3/nc3OyHWwyIP+qib7xPn3ocY76P9dBWYAyHyup9vExAvFaFvJT2LRUHS+shMEAm2MExz1W6Oo1CXOejjgMyx+P9skS0lqvByso9umYMSDeoMX98e5xo8/5yjyFYukGDq4anUjrYMLO/pB7HLjR7BLCVcg4qhQxymQy3TcqCXqMENHocTLwBFTX7kdZwCCp7NVDi6f0PS9NBo1EDcjXON9txoakNAAfGceI9OIADAA6jMvWIc9Zbu9DYhnJnzErOcZiUkxDeix5zu+jphQBJhai2thY8zyMtzdO9TUtLw4kTJ7q0X7NmDZ566qnA38hhFT2NcMIY7GbfO615mQ2wt/+Xc1jNEDqIlcDJYVXoYFEmoCVhAPImTgF06QDHoankDKw+thRY7Z5xnwStCjJOFKChaTpkGDSUgTECDE0T43WVTW1ostjBCwx2nsHO8wA8P7tWG8Np2WCcThwEnbUS8W3noLNVQW1vhppvhcCYODTnHRAsZjhaLb7f2GQF7M6fsckCoUUUIk7GASYfJbJDBQtdzLFP7b5fuXKluxosIHpERqP3pF8exBvF2ArQYa1Nhx+nxw81kOOcx1njkDbnac+2HAC1UgbEtnslOcPbRGdKrgRTasHJVe6uVXIZ0CFh2W0TjWBg4Jzv5+qeA6DotB7o7ilUqlsKcpNj3YsdBYHB6hBgcwiw8jx4gUGrbI/p5WcZkJsc63SmjQAuae9I4KGI4wDYAYcNBrMVsjab0/Nm7V65UwSUOhXg/A7EWR0YaBHFh+MAGMLsBatCt2laUiFKTk6GXC5HVVWVx/Gqqiqkp6d3aa9Wq6FWB1EMUBXbpbhgqOEAZAXgpQaysDlFRwUQ+xIyGYcYlRwxKjmAroUJ4rUqdw7sntDrAH+/Vnr43/ZiQ9KImEqlwsSJE/HNN9+4jwmCgG+++QZTp06V0DKCICKJ5EOzpUuXYsGCBZg0aRImT56MV155Ba2trbj//vulNo0giAghuRDdcccdqKmpwapVq1BZWYlx48Zh69atXQLYBEFEL5KvI+oNUbGOiCCiGH9/o9GxaoogiD4NCRFBEJJDQkQQhORIHqzuDa7wlt9bPQiCiCiu32ZPoeg+LUQtLeKWCr9WVxMEIRktLS0wGAw+z/fpWTNBEHDhwgXodLqLcj+VawtKeXl5VM7qRfP1RfO1AZG7PsYYWlpakJmZCVk3GQX6tEckk8mQldVzhkKp0ev1UflldhHN1xfN1wZE5vq684RcULCaIAjJISEiCEJySIjCiFqtxurVq4PLGNAHiObri+ZrAy6+6+vTwWqCIKID8ogIgpAcEiKCICSHhIggCMkhISIIQnJIiAIk0Kq0H374IYYPHw6NRoMxY8bgiy++8DjPGMOqVauQkZGBmJgYXHPNNTh9+nQ4L8EngVzbW2+9hcsvvxwJCQlISEjANddc06X9woULwXGcx23WrFnhvgyfBHJ9hYWFXWzXaDyT0ffVz27GjBldro3jOMyZM8fdJuKfHSP8ZvPmzUylUrGNGzeyX375hT300EMsPj6eVVVVeW2/Z88eJpfL2QsvvMCOHTvGfve73zGlUsmOHDnibrN27VpmMBjYli1b2E8//cRuvvlmlpubyywWS6QuizEW+LXdfffdbP369ezHH39kx48fZwsXLmQGg4GdO3fO3WbBggVs1qxZrKKiwn2rr6+P1CV5EOj1FRQUML1e72F7ZWWlR5u++tnV1dV5XNfRo0eZXC5nBQUF7jaR/uxIiAJg8uTJbPHixe7nPM+zzMxMtmbNGq/tb7/9djZnzhyPY1OmTGG//e1vGWOMCYLA0tPT2R//+Ef3+cbGRqZWq9kHH3wQhivwTaDX1hmHw8F0Oh1755133McWLFjA5s6dG2pTgyLQ6ysoKGAGg8Fnf9H02f3pT39iOp2OmUwm97FIf3Y0NPMTV1Xaa665xn2su6q0APD99997tAeA66+/3t2+uLgYlZWVHm0MBgOmTJnis89wEMy1dcZsNsNutyMxMdHj+M6dO5Gamophw4bhkUceQV1dXUht94dgr89kMiE7OxtGoxFz587FL7/84j4XTZ/d22+/jTvvvBOxsZ51yiL52ZEQ+Ul3VWkrKyu9vqaysrLb9q77QPoMB8FcW2eWL1+OzMxMjx/ErFmz8O677+Kbb77B888/j127dmH27Nngee9Va8NFMNc3bNgwbNy4EX//+9/x3nvvQRAETJs2DefOnQMQPZ/dDz/8gKNHj+LBBx/0OB7pz65P774nLg7Wrl2LzZs3Y+fOnR4B3TvvvNP9eMyYMcjPz8fgwYOxc+dOXH311VKY6jdTp071qK03bdo0jBgxAm+88QaeeeYZCS0LLW+//TbGjBmDyZMnexyP9GdHHpGfBFqVFgDS09O7be+6D6TPcBDMtbl48cUXsXbtWnz11VfIz8/vtu2gQYOQnJyMM2fO9NrmQOjN9blQKpUYP3682/Zo+OxaW1uxefNmLFq0qMf3CfdnR0LkJ8FUpZ06dapHewDYvn27u31ubi7S09M92jQ3N2Pfvn0RrXQbbMXdF154Ac888wy2bt2KSZMm9fg+586dQ11dHTIyMkJit7+EoqIwz/M4cuSI2/a+/tkB4tISq9WKe+65p8f3CftnF7GweBSwefNmplarWWFhITt27Bj7zW9+w+Lj493Tuvfeey9bsWKFu/2ePXuYQqFgL774Ijt+/DhbvXq11+n7+Ph49ve//539/PPPbO7cuZJNAQdybWvXrmUqlYp99NFHHlO8LS0tjDHGWlpa2LJly9j333/PiouL2ddff80mTJjA8vLyWFtbW0SvLZjre+qpp9i2bdtYUVERO3jwILvzzjuZRqNhv/zyi7tNX/3sXFx22WXsjjvu6HJcis+OhChAXn31VTZw4ECmUqnY5MmT2d69e93nrrzySrZgwQKP9n/729/Y0KFDmUqlYqNGjWKff/65x3lBENjvf/97lpaWxtRqNbv66qvZyZMnI3EpXQjk2rKzsxmALrfVq1czxhgzm83suuuuYykpKUypVLLs7Gz20EMPdVmLE0kCub4lS5a426alpbEbbriBHTp0yKO/vvrZMcbYiRMnGAD21VdfdelLis+O0oAQBCE5FCMiCEJySIgIgpAcEiKCICSHhIggCMkhISIIQnJIiAiCkBwSIoIgJIeEiOhTFBYWIj4+3v38ySefxLhx49zPFy5ciHnz5kXcLqJ3kBARXnGlCn344Ye7nFu8eDE4jsPChQs92odaAHJycvDKK694HLvjjjtw6tQpn69Zt24dCgsL3c9nzJiBJUuWhNQuIvSQEBE+MRqN2Lx5MywWi/tYW1sbNm3ahIEDB0piU0xMDFJTU32eNxgMHh4T0TcgISJ8MmHCBBiNRnzyySfuY5988gkGDhyI8ePH96pvb57KvHnz3F7WjBkzUFpaiieeeMKdvB3oOjTrTEfPbOHChdi1axfWrVvn7qO4uBhDhgzBiy++6PG6w4cPg+O4iKcoIURIiIhueeCBB1BQUOB+vnHjRtx///1hf99PPvkEWVlZePrpp1FRUYGKioqA+1i3bh2mTp2Khx56yN3HwIEDu1wTABQUFOCKK67AkCFDQnUJRACQEBHdcs899+C7775DaWkpSktLsWfPHr/y1/SWxMREyOVy6HQ6pKenB5VszGAwQKVSQavVuvuQy+VYuHAhTp486S65Y7fbsWnTJjzwwAOhvgzCTyhVLNEtKSkpmDNnDgoLC8EYw5w5c5CcnCy1Wb0iMzMTc+bMwcaNGzF58mT885//hNVqxW233Sa1af0W8oiIHnnggQdQWFiId955J2Reg0wmQ+cMNHa7PSR9+8ODDz7oDsQXFBTgjjvugFarjdj7E56QEBE9MmvWLNhsNtjtdlx//fUh6TMlJcUj7sPzPI4ePerRRqVS9bpqhK8+brjhBsTGxmLDhg3YunUrDcskhoZmRI/I5XIcP37c/dgXTU1NOHz4sMexpKQkGI3GLm2vuuoqLF26FJ9//jkGDx6Ml19+GY2NjR5tcnJysHv3btx5551Qq9VBDQlzcnKwb98+lJSUIC4uDomJiZDJZO5Y0cqVK5GXlxfRPNNEV8gjIvxCr9dDr9d322bnzp0YP368x+2pp57y2vaBBx7AggULcN999+HKK6/EoEGDMHPmTI82Tz/9NEpKSjB48GCkpKQEZfeyZcsgl8sxcuRIpKSkoKyszH1u0aJFsNlsEZkFJLqHUsUS/ZZ//etfuPrqq1FeXt6lQCERWUiIiH6H1WpFTU0NFixYgPT0dLz//vtSm9TvoaEZ0e/44IMPkJ2djcbGRrzwwgtSm0OAPCKCIC4CyCMiCEJySIgIgpAcEiKCICSHhIggCMkhISIIQnJIiAiCkBwSIoIgJIeEiCAIySEhIghCcv4/489I8leaFbEAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:09:15.827277Z", + "iopub.status.busy": "2024-03-22T14:09:15.826461Z", + "iopub.status.idle": "2024-03-22T14:09:16.050286Z", + "shell.execute_reply": "2024-03-22T14:09:16.049185Z" + }, + "papermill": { + "duration": 0.245593, + "end_time": "2024-03-22T14:09:16.052340", + "exception": false, + "start_time": "2024-03-22T14:09:15.806747", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEpCAYAAADCh6TFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAo50lEQVR4nO3de1gU56E/8O8ul0UEVEQuIrqiTdAooFARE48xEUiTWPOYJvzMBcTUtio9xj00hrSBQ80J2hglzSFyYoOXJI00XpI0GtSgtNF4hehBBSyK4gUQVEAgLsvue/7wx8TN7sKCwDLL9/M8+8C8887MO8z6debd2XcUQggBIiKZUdq6AUREXcHwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZsnl4ZWZmQq1Ww8XFBRERETh69Gi79evq6rBkyRL4+flBpVLhvvvuw65du3qptUTUVzjacuM5OTnQaDTIyspCREQEMjIyEBMTg9LSUnh7e5vUb2lpQVRUFLy9vbF161b4+/vj4sWLGDx4sNXbNBgMuHr1Ktzd3aFQKLpxb4ioOwghcOvWLQwfPhxKZTvnV8KGpkyZIpYsWSJN6/V6MXz4cJGenm62/rp160RgYKBoaWnp8jYvXbokAPDFF199/HXp0qV2/y0rhLDNqBItLS1wdXXF1q1b8dRTT0nl8fHxqKurw+eff26yzOOPPw5PT0+4urri888/x7Bhw/Dcc89h+fLlcHBwMLsdrVYLrVYrTdfX12PkyJEoLy+Hu7t7t+9XX6HT6bB//37MnDkTTk5Otm4OdYP+ckxv3bqF0aNHo66uDoMGDbJYz2aXjbW1tdDr9fDx8TEq9/HxQUlJidllzp8/j3379uH555/Hrl27UFZWhsWLF0On0yE1NdXsMunp6UhLSzMpP3ToEFxdXe99R/owV1dXHDlyxNbNoG7UH45pc3MzAHTYrWPTPq/OMhgM8Pb2xvvvvw8HBweEhYXhypUreOuttyyGV3JyMjQajTTd0NCAgIAAREdHw8PDo7ea3ut0Oh327t2LqKgou/5fuj/pL8e0oaHBqno2Cy8vLy84ODigurraqLy6uhq+vr5ml/Hz84OTk5PRJeK4ceNQVVWFlpYWODs7myyjUqmgUqlMyp2cnOz6DdCmv+xnf2Lvx9TafbPZrRLOzs4ICwtDXl6eVGYwGJCXl4fIyEizyzz44IMoKyuDwWCQys6ePQs/Pz+zwUVE9sum93lpNBqsX78emzZtQnFxMRYtWoSmpiYkJCQAAOLi4pCcnCzVX7RoEW7cuIGlS5fi7Nmz2LlzJ958800sWbLEVrtARDZi0z6v2NhY1NTUICUlBVVVVQgNDUVubq7UiV9RUWF0n0dAQAB2796NZcuWITg4GP7+/li6dCmWL19uq10gIhuxeYd9YmIiEhMTzc7Lz883KYuMjMThw4d7uFVE1NfZPLyIyFRzc7PJLUON32vxbdE5DPE6DrcBP3wIFRQUZPe3/ZjD8CLqg0pKShAWFmZ23p9+NF1QUIDJkyf3fKP6GIYXUR8UFBSEgoICo7LSyjpoPi3Cmmcm4n6/wUZ1+yOGF1Ef5OrqanI2pbx4Hapvvse4CSEIHTXURi3rO2w+JA4RUVcwvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLHMPeTvz4UVmWHpMF9N9HZZF9YXjZCUuPyvrxY7KA/vuoLLIvDC878eNHZVl6TFZbXSK5Y3jZiR8/KouPySJ7xw57IpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEs9YnwyszMhFqthouLCyIiInD06FGLdTdu3AiFQmH0cnFx6cXWElFfYPPwysnJgUajQWpqKgoLCxESEoKYmBhcu3bN4jIeHh6orKyUXhcvXuzFFhNRX2Dz8FqzZg0WLlyIhIQEjB8/HllZWXB1dUV2drbFZRQKBXx9faWXj49PL7aYiPoCm95h39LSgoKCAiQnJ0tlSqUSs2bNwqFDhywu19jYiFGjRsFgMGDy5Ml488038cADD5itq9VqodVqpemGhgYAgE6ng06n66Y96XtaW1uln/a8n/1Jfzmm1u6bTcOrtrYWer3e5MzJx8fHaISEu91///3Izs5GcHAw6uvrsXr1akybNg2nT5/GiBEjTOqnp6cjLS3NpHzPnj12PbLCpUYAcMThw4dx5ZStW0Pdob8c0+bmZqvqye67jZGRkYiMjJSmp02bhnHjxuF//ud/sGLFCpP6ycnJ0Gg00nRDQwMCAgIQHR0NDw+PXmmzLZysuAEUHcfUqVMRMtLT1s2hbtBfjmnb1VFHbBpeXl5ecHBwQHV1tVF5dXU1fH19rVqHk5MTJk2ahLKyMrPzVSoVVCqVSbmTkxOcnJw632iZcHR0lH7a8372J/3lmFq7bzbtsHd2dkZYWBjy8vKkMoPBgLy8PKOzq/bo9XoUFRXBz8+vp5pJRH2QzS8bNRoN4uPjER4ejilTpiAjIwNNTU1ISEgAAMTFxcHf3x/p6ekAgD/+8Y+YOnUqxo4di7q6Orz11lu4ePEifvnLX9pyN4iol9k8vGJjY1FTU4OUlBRUVVUhNDQUubm5Uid+RUUFlMofThBv3ryJhQsXoqqqCkOGDEFYWBi+/fZbjB8/3la7QEQ2YPPwAoDExEQkJiaanZefn280vXbtWqxdu7YXWkVEfZnNb1IlIuoKhhcRyRLDi4hkieFFRLLE8CIiWeoTnzYSEVBe24QmbavF+edqmqSfbXfb/9hAlSNGew3skfb1NQwvoj6gvLYJM1fnW1X3P7YWtTt/f9LD/SLAGF5EfUDbGVdGbCjGeruZr/O9Fl/mH8KTD0di4ADT7+uWXWvEyzkn2j17sycML6I+ZKy3Gyb4DzI7T6fToWoYMHnUELv+Yra12GFPRLLE8CIiWeJlo4y19+mUNZ9MAf3r0ymyLwwvmbL206mOPpkC+s+nU2RfGF4y1dGnUx19MgX0v0+nyL4wvGTO0qdT/GSK7B077IlIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiV+t1GmtPrbULpcQXlDKZQupl/Mbm1txdXWqyi+UWxxSJzyhkYoXa5Aq78NwPzonUR9FcNLpq42XcTA0e/itaPt13sv97125w8cDVxtCkUYfLqxdUQ9j+ElU8MHjkJT+W/xTmwoxpgZEqe1tRUHDxzEgw89aPHM69y1RizNOYHhM0f1dHOpAx2dSQMdn033tzNphpdMqRxcYLjtj9Ee92P8UPND4pQ7lmOc5ziLQ+IYbtfDcLsGKgeXnm4udcDaM2mg/bPp/nQmzfAi6gM6OpMGOj6b7m9n0gwvoj6gozNpoOOz6f52Jt3pWyXOnz/fE+0gIuqUTofX2LFjMXPmTHz00Ue4fft2tzQiMzMTarUaLi4uiIiIwNGjVlz4A9iyZQsUCgWeeuqpbmkHEclHp8OrsLAQwcHB0Gg08PX1xa9//Wurw8acnJwcaDQapKamorCwECEhIYiJicG1a9faXe7ChQtISkrC9OnTu7xtIpKvTodXaGgo3nnnHVy9ehXZ2dmorKzEQw89hAkTJmDNmjWoqanp1PrWrFmDhQsXIiEhAePHj0dWVhZcXV2RnZ1tcRm9Xo/nn38eaWlpCAwM7OwuEJEd6HKHvaOjI+bOnYsnnngC7733HpKTk5GUlITXXnsNzz77LFatWgU/P79219HS0oKCggIkJydLZUqlErNmzcKhQ4csLvfHP/4R3t7eeOmll/DNN9+0uw2tVgutVitNNzQ0ALjT+anT6azZ1T6ptbVV+mluP9rK2tvHjtZBvceaY9HRMbWX42lt27scXsePH0d2dja2bNmCgQMHIikpCS+99BIuX76MtLQ0zJkzp8PLydraWuj1evj4GN+T4uPjg5KSErPLHDhwAB988AFOnDhhVTvT09ORlpZmUr5nzx64urpatY6+6FIjADjiwIEDuGj+k3UAwN69e+95HdTzOnMsLB1Tezmezc3NVtXrdHitWbMGGzZsQGlpKR5//HFs3rwZjz/+OJTKO1ego0ePxsaNG6FWqzu76g7dunULL774ItavXw8vLy+rlklOToZGo5GmGxoaEBAQgOjoaHh4eHR7G3vL6asNWF10GA899BAeGG66HzqdDnv37kVUVJTFm1Q7Wgf1HmuORUfH1F6OZ9vVUUc6HV7r1q3DggULMH/+fIuXhd7e3vjggw86XJeXlxccHBxQXV1tVF5dXQ1fX1+T+ufOncOFCxcwe/ZsqcxgMAC4cxlbWlqKMWPGGC2jUqmgUpk+MdrJyUnWD2Ntu0nR0dGx3f1obz+tXQf1vM4cC0vH1F6Op7Vt73R47d27FyNHjpTOtNoIIXDp0iWMHDkSzs7OiI+P73Bdzs7OCAsLQ15ennS7g8FgQF5eHhITE03qBwUFoaioyKjsD3/4A27duoV33nkHAQEBnd0dIpKpTofXmDFjUFlZCW9vb6PyGzduYPTo0dDr9Z1an0ajQXx8PMLDwzFlyhRkZGSgqakJCQkJAIC4uDj4+/sjPT0dLi4umDBhgtHygwcPBgCTciKyb50OLyGE2fLGxka4uHT+awmxsbGoqalBSkoKqqqqEBoaitzcXKkTv6KiwuQsj4jI6vBq6/RWKBRISUkx+qROr9fjyJEjCA0N7VIjEhMTzV4mAkB+fn67y27cuLFL2yQiebM6vL777jsAd868ioqK4OzsLM1zdnZGSEgIkpKSur+FRERmWB1e+/fvBwAkJCTgnXfekfVtBkQkf53u89qwYUNPtIOIqFOsCq+5c+di48aN8PDwwNy5c9utu3379m5pGBFRe6wKr0GDBkGhUEi/ExHZmlXhdfelIi8biagv4A1URCRLVp15TZo0Sbps7EhhYeE9NYiIyBpWhReHWSaivsaq8EpNTe3pdhARdQr7vIhIlqw68/L09MTZs2fh5eWFIUOGtNv/dePGjW5rHBGRJVaF19q1a+Hu7i79bm3nPRFRT7EqvO4eWHD+/Pk91RYiIqt1us/LwcHB7DMVr1+/DgcHh25pFBFRRzodXpYGI9RqtUbD5BAR9SSrR5X485//DODOYIR/+ctf4Ob2w7OV9Ho9/vnPfyIoKKj7W0hEZIbV4bV27VoAd868srKyjC4RnZ2doVarkZWV1f0tJCIyw+rwKi8vBwDMnDkT27dvx5AhQ3qsUUREHen0YIRtI6oSEdlSp8NrwYIF7c7Pzs7ucmOIiKzV6fC6efOm0bROp8OpU6dQV1eHRx55pNsaRkTUnk6H144dO0zKDAYDFi1ahDFjxnRLo4iIOtItX8xWKpXQaDTSJ5JERD2t20aVOHfuHFpbW7trdURE7er0ZWPbk7PbCCFQWVmJnTt3Gn0HkoioJ3U6vNqenN1GqVRi2LBhePvttzv8JJKIqLvwPi8ikiWOpEpEssTwIiJZYngRkSwxvIhIlrotvC5fvoxf/epX3bU6IqJ2dVt4Xb9+HR988EGXls3MzIRarYaLiwsiIiJw9OhRi3W3b9+O8PBwDB48GAMHDkRoaCg+/PDDrjZbtr7X6QEAp67Um30VXryJ4zVA4cWbFuuUXWu08V4QdV2nb5Xobjk5OdBoNMjKykJERAQyMjIQExOD0tJSeHt7m9T39PTE73//ewQFBcHZ2RlffvklEhIS4O3tjZiYGBvsgW2c+//B8+r2onZqOeLDsmMdrmugyuZvA6JOs/m7ds2aNVi4cCESEhIAAFlZWdi5cyeys7Px6quvmtR/+OGHjaaXLl2KTZs24cCBA/0qvKIf8AUAjPF2wwAn0weflFbW4z+2FuHtX0zE/X6DLK5noMoRo70G9lg7iXqKTcOrpaUFBQUFSE5OlsqUSiVmzZqFQ4cOdbi8EAL79u1DaWkpVq1aZbaOVquFVquVphsaGgDcGcpHp9Pd4x7YjruzAk9P8rM4//bt2wCAUUNUuN/btd11yfnvYC/avhfc2tpq8Xi0lVuab8065MDatlsdXnPnzm13fl1dnbWrktTW1kKv18PHx8eo3MfHByUlJRaXq6+vh7+/P7RaLRwcHPDee+8hKirKbN309HSkpaWZlO/Zsweuru3/o5azS40A4IjDhw/jyilbt4Y6cq4BAByxZfdBjBho/gldOgNwQwuc37YXTmZ6q6u/VwBwwIEDB3DRzXS+XDQ3N1tVz+rwGjTI8qVH2/y4uDhrV3dP3N3dceLECTQ2NiIvLw8ajQaBgYEml5QAkJycbPRl8oaGBgQEBCA6OhoeHh690l5bOFlxAyg6jqlTpyJkpKetm0Md+Nvxy8DpM9hy/t6ffRrz6Ayoh8q3K6Dt6qgjVofXhg0butwYS7y8vODg4IDq6mqj8urqavj6+lpcTqlUYuzYsQCA0NBQFBcXIz093Wx4qVQqqFQqk3InJyc4OTnd2w70YY6OjtJPe95Pe/GzYH84ODhY7MMErOvHtIc+TGvfrzbt83J2dkZYWBjy8vLw1FNPAbgzKmteXh4SExOtXo/BYDDq1yKSG8+Bzvh/U0a2W6etT2vMsIGY4N/+lVB/YHV4WTvcTWcfwKHRaBAfH4/w8HBMmTIFGRkZaGpqkj59jIuLg7+/P9LT0wHc6cMKDw/HmDFjoNVqsWvXLnz44YdYt25dp7ZLRPJmdXht3LgRo0aNwqRJkyCE+Q7FroiNjUVNTQ1SUlJQVVWF0NBQ5ObmSp34FRUVUCp/6J1samrC4sWLcfnyZQwYMABBQUH46KOPEBsb221tIqK+z+rwWrRoET755BOUl5cjISEBL7zwAjw9u6cjODEx0eJlYn5+vtH0G2+8gTfeeKNbtktE8mX114MyMzNRWVmJV155BX//+98REBCAZ599Frt37+7WMzEiImt06ruNKpUK8+bNw969e3HmzBk88MADWLx4MdRqNRob+T05Iuo9Xf5itlKphEKhgBACer2+O9tERNShToWXVqvFJ598gqioKNx3330oKirCf//3f6OiogJubjK+pZeIZMfqDvvFixdjy5YtCAgIwIIFC/DJJ5/Ay8urJ9tGRGSR1eGVlZWFkSNHIjAwEP/4xz/wj3/8w2y97du3d1vjiIgssTq84uLioFAoerItRERW69RNqkREfQUfwEFEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEstQnwiszMxNqtRouLi6IiIjA0aNHLdZdv349pk+fjiFDhmDIkCGYNWtWu/WJyD7ZPLxycnKg0WiQmpqKwsJChISEICYmBteuXTNbPz8/H/PmzcP+/ftx6NAhBAQEIDo6GleuXOnllhORLdk8vNasWYOFCxciISEB48ePR1ZWFlxdXZGdnW22/scff4zFixcjNDQUQUFB+Mtf/gKDwYC8vLxebjkR2ZKjLTfe0tKCgoICJCcnS2VKpRKzZs3CoUOHrFpHc3MzdDodPD09zc7XarXQarXSdENDAwBAp9NBp9PdQ+v7ttbWVumnPe9nf9Jfjqm1+2bT8KqtrYVer4ePj49RuY+PD0pKSqxax/LlyzF8+HDMmjXL7Pz09HSkpaWZlO/Zsweurq6db7RMXGoEAEccPnwYV07ZujXUHfrLMW1ubraqnk3D616tXLkSW7ZsQX5+PlxcXMzWSU5OhkajkaYbGhqkfjIPD4/eamqvO1lxAyg6jqlTpyJkpPmzUpKX/nJM266OOmLT8PLy8oKDgwOqq6uNyqurq+Hr69vusqtXr8bKlSvx9ddfIzg42GI9lUoFlUplUu7k5AQnJ6euNVwGHB0dpZ/2vJ/9SX85ptbum0077J2dnREWFmbU2d7W+R4ZGWlxuT/96U9YsWIFcnNzER4e3htNJaI+xuaXjRqNBvHx8QgPD8eUKVOQkZGBpqYmJCQkAADi4uLg7++P9PR0AMCqVauQkpKCv/71r1Cr1aiqqgIAuLm5wc3NzWb7QUS9y+bhFRsbi5qaGqSkpKCqqgqhoaHIzc2VOvErKiqgVP5wgrhu3Tq0tLTgF7/4hdF6UlNT8Z//+Z+92XQisiGbhxcAJCYmIjEx0ey8/Px8o+kLFy70fIOIqM+z+U2qRERdwfAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIlnqE18PonvX3NxsNIBjaWUdtFVlKD41AIbrg43qBgUF2fVAjNQ/MLzsRElJCcLCwkzKn9tkWregoACTJ0/uhVYR9RyGl50ICgpCQUGBNN34vRY79x/CEzMj4TZAZVKXSO4YXnbC1dXV6GxKp9PhZu01RE4Jt+tRN6n/Yoc9EckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyZPPwyszMhFqthouLCyIiInD06FGLdU+fPo2nn34aarUaCoUCGRkZvddQIupTbPoAjpycHGg0GmRlZSEiIgIZGRmIiYlBaWkpvL29Teo3NzcjMDAQzzzzDJYtW2aDFhP1jh8/hxOw/CzOfvscTmFDU6ZMEUuWLJGm9Xq9GD58uEhPT+9w2VGjRom1a9d2epv19fUCgKivr+/0snLS0tIiPvvsM9HS0mLrplAXFBQUCABWvQoKCmzd3G5l7b9Rm515tbS0oKCgAMnJyVKZUqnErFmzcOjQoW7bjlarhVarlaYbGhoA3Hk0mE6n67bt9DVt+2bP+2jPxowZgyNHjhiVNX6vxe5vjiFm+k+NnsU5ZswYuzrO1u6LzcKrtrYWer0ePj4+RuU+Pj4mp8v3Ij09HWlpaSble/bs6Ren2nv37rV1E6gbTZs4BrfqbuBW3Q9llZWVNmtPT2hubraqnt0/dDY5ORkajUaabmhoQEBAAKKjo+Hh4WHDlvUsnU6HvXv3Iioqig+dtRP95Zi2XR11xGbh5eXlBQcHB1RXVxuVV1dXw9fXt9u2o1KpoFKpTMqdnJzs+g3Qpr/sZ39i78fU2n2z2a0Szs7OCAsLQ15enlRmMBiQl5eHyMhIWzWLiGTCppeNGo0G8fHxCA8Px5QpU5CRkYGmpiYkJCQAAOLi4uDv74/09HQAdzr5z5w5I/1+5coVnDhxAm5ubhg7dqzN9oOIep9Nwys2NhY1NTVISUlBVVUVQkNDkZubK3XiV1RUQKn84eTw6tWrmDRpkjS9evVqrF69GjNmzEB+fn5vN5+IbMjmHfaJiYlITEw0O+/HgaRWqyGE6IVWEVFfZ/OvBxERdYXNz7x6W9uZm7Ufx8qVTqdDc3MzGhoa7PqTqf6kvxzTtn+bHV1l9bvwunXrFgAgICDAxi0hovbcunULgwYNsjhfIfpZJ5LBYMDVq1fh7u4OhUJh6+b0mLabcS9dumTXN+P2J/3lmAohcOvWLQwfPtzoA7sf63dnXkqlEiNGjLB1M3qNh4eHXb/R+6P+cEzbO+Nqww57IpIlhhcRyRLDy06pVCqkpqaa/V4nyROPqbF+12FPRPaBZ15EJEsMLyKSJYYXEckSw8tGLly4AIVCgRMnTvTKeg4ePIiJEyfCyckJTz311D1tk6gvYHj1Efn5+VAoFKirq+uR9Ws0GoSGhqK8vBwbN27skW2QqYcffhgvv/yyrZsh6WvtuRcMry5oaWmxdRM67dy5c3jkkUcwYsQIDB48uEvr6M39FkKgtbW117bXl8nx/dYrevYJbPZhxowZYsmSJWLp0qVi6NCh4uGHHxZFRUXiscceEwMHDhTe3t7ihRdeEDU1NdIyX331lXjwwQfFoEGDhKenp3jiiSdEWVmZNL+8vFwAEN999530+92v+Pj4Tq3nk08+EZGRkUKlUokHHnhA5OfnG82/+7VhwwYhhBD5+fnipz/9qXB2dha+vr5i+fLlQqfTtbvf+/fvFwBEbm6uCA0NFS4uLmLmzJmiurpa7Nq1SwQFBQl3d3cxb9480dTUJK1Lr9eLN998U6jVauHi4iKCg4PFp59+Ks1vW++uXbvE5MmThZOTk9i/f393HsZeFx8fb/K3LysrEwsWLJD+Dvfdd5/IyMgwWW7OnDnijTfeEH5+fkKtVgshhDh48KAICQkRKpVKhIWFiR07dkjvoTbtvS/Ntae8vLy3/hzdjuFlhRkzZgg3Nzfxu9/9TpSUlIjDhw+LYcOGieTkZFFcXCwKCwtFVFSUmDlzprTM1q1bxbZt28S//vUv8d1334nZs2eLiRMnCr1eL4QwDq/W1laxbds2AUCUlpaKyspKUVdX16n1jBgxQmzdulWcOXNG/PKXvxTu7u6itrZWtLa2isrKSuHh4SEyMjJEZWWlaG5uFpcvXxaurq5i8eLFori4WOzYsUN4eXmJ1NRUi/tdUlIihczUqVPFgQMHRGFhoRg7dqyYMWOGiI6OFoWFheKf//ynGDp0qFi5cqW0rjfeeEMEBQWJ3Nxcce7cObFhwwahUqmkkG1bb3BwsNizZ48oKysT169f7+lD26Pq6upEZGSkWLhwoaisrBSVlZXi9u3bIiUlRRw7dkycP39efPTRR8LV1VXk5ORIy8XHxws3Nzfx4osvilOnTolTp06J+vp64enpKV544QVx+vRpsWvXLnHfffcZhdfNmzfbfV+aa09ra6st/jTdguFlhRkzZohJkyZJ0ytWrBDR0dFGdS5duiSFjzk1NTUCgCgqKhJCGIeXED/8471582a7bbG0nruDQqfTiREjRohVq1ZJZYMGDZLOuIQQ4rXXXhP333+/MBgMUllmZqZwc3OTgvHH+313O7/++mupLD09XQAQ586dk8p+/etfi5iYGCGEELdv3xaurq7i22+/NVrXSy+9JObNm2e03s8++6zd/ZebGTNmiKVLl7ZbZ8mSJeLpp5+WpuPj44WPj4/QarVS2bp168TQoUPF999/L5WtX7/e6D1kzfvSmvbIBfu8rBQWFib9fvLkSezfvx9ubm7SKygoCMCdviUA+Ne//oV58+YhMDAQHh4eUKvVAO6My98Z1q7n7icuOTo6Ijw8HMXFxRbXW1xcjMjISKNhgR588EE0Njbi8uXLZvf7bsHBwdLvPj4+cHV1RWBgoFHZtWvXAABlZWVobm5GVFSU0d9s8+bN0t+rTXh4uMU224vMzEyEhYVh2LBhcHNzw/vvv29yPCdOnAhnZ2dpurS0FMHBwXBxcZHKpkyZYrSMNe9Le9LvhsTpqoEDB0q/NzY2Yvbs2Vi1apVJPT8/PwDA7NmzMWrUKKxfvx7Dhw+HwWDAhAkTOt352l3r6aq79/tud4/kqVAoTEb2VCgUMBgMAO78vQBg586d8Pf3N6r34+/pWdqevdiyZQuSkpLw9ttvIzIyEu7u7njrrbdw5MgRo3pd+TtY8760JwyvLpg8eTK2bdsGtVoNR0fTP+H169dRWlqK9evXY/r06QCAAwcOtLvOtv9l9Xp9l9Zz+PBh/Nu//RsAoLW1FQUFBRYfbAIA48aNw7Zt2yCEkM6+Dh48CHd3924f72z8+PFQqVSoqKjAjBkzunXdfZ2zs7PRMT148CCmTZuGxYsXS2XWnBXdf//9+Oijj6DVaqXAP3bsmFGdjt6X5tojZ7xs7IIlS5bgxo0bmDdvHo4dO4Zz585h9+7dSEhIgF6vx5AhQzB06FC8//77KCsrw759+6DRaNpd56hRo6BQKPDll1+ipqYGjY2NnVpPZmYmduzYgZKSEixZsgQ3b97EggULLG5v8eLFuHTpEn7729+ipKQEn3/+OVJTU6HRaNodvbIr3N3dkZSUhGXLlmHTpk04d+4cCgsL8e6772LTpk3duq2+Rq1W48iRI7hw4QJqa2vxk5/8BMePH8fu3btx9uxZvP766yYhZM5zzz0Hg8GAX/3qVyguLsbu3buxevVqAJD+8+nofWmuPW1nx3LE8OqC4cOH4+DBg9Dr9YiOjsbEiRPx8ssvY/DgwVAqlVAqldiyZQsKCgowYcIELFu2DG+99Va76/T390daWhpeffVV+Pj4IDExsVPrWblyJVauXImQkBAcOHAAX3zxBby8vNrd3q5du3D06FGEhITgN7/5DV566SX84Q9/uKe/jSUrVqzA66+/jvT0dIwbNw6PPfYYdu7cidGjR/fI9vqKpKQkODg4YPz48Rg2bBhiYmIwd+5cxMbGIiIiAtevXzc6C7PEw8MDf//733HixAmEhobi97//PVJSUgBA6gfr6H1prj2d7YPtSzgkDpFMffzxx0hISEB9fT0GDBhg6+b0OvZ5EcnE5s2bERgYCH9/f5w8eRLLly/Hs88+2y+DC2B4EclGVVUVUlJSUFVVBT8/PzzzzDP4r//6L1s3y2Z42UhEssQOeyKSJYYXEckSw4uIZInhRUSyxPAiIllieFGPmj9/PhQKhfTlbR8fH0RFRSE7O7tTX03ZuHFjl0eAvRfz58/nmP99FMOLetxjjz2GyspKXLhwAV999RVmzpyJpUuX4sknn+RQz9R1Nh1NjOxe25DGP5aXlycAiPXr1wshhHj77bfFhAkThKurqxgxYoRYtGiRuHXrlhDih4EK7361jfi6efNmERYWJtzc3ISPj4+YN2+eqK6ulrZz48YN8dxzzwkvLy/h4uIixo4dK7Kzs6X5FRUV4plnnhGDBg0SQ4YMET//+c+loZFTU1NNtiv3oantCc+8yCYeeeQRhISEYPv27QAApVKJP//5zzh9+jQ2bdqEffv24ZVXXgEATJs2DRkZGfDw8EBlZSUqKyuRlJQEANDpdFixYgVOnjyJzz77DBcuXMD8+fOl7bz++us4c+YMvvrqKxQXF2PdunXSF9Z1Oh1iYmLg7u6Ob775BgcPHoSbmxsee+wxtLS0ICkpCc8++6x05lhZWYlp06b17h+KLLN1epJ9s3TmJYQQsbGxYty4cWbnffrpp2Lo0KHS9IYNG8SgQYM63N6xY8cEAOmsbfbs2SIhIcFs3Q8//NBkKGytVisGDBggdu/e3WH7ybZ45kU2I+4aCPHrr7/Go48+Cn9/f7i7u+PFF1/E9evX0dzc3O46CgoKMHv2bIwcORLu7u7SYIdtQ70sWrQIW7ZsQWhoKF555RV8++230rInT55EWVkZ3N3dpWGTPT09cfv2bbscNtneMLzIZoqLizF69GhcuHABTz75JIKDg7Ft2zYUFBQgMzMTQPvPLGxqakJMTAw8PDzw8ccf49ixY9ixY4fRcj/72c9w8eJFLFu2DFevXsWjjz4qXXI2NjYiLCwMJ06cMHqdPXsWzz33XA/vPd0rjipBNrFv3z4UFRVh2bJlKCgogMFgwNtvvy0Nmve3v/3NqL654YtLSkpw/fp1rFy5EgEBAQCA48ePm2xr2LBhiI+PR3x8PKZPn47f/e53WL16NSZPnoycnBx4e3vDw8PDbDvtadhke8MzL+pxWq0WVVVVuHLlCgoLC/Hmm29izpw5ePLJJxEXF4exY8dCp9Ph3Xffxfnz5/Hhhx8iKyvLaB1qtRqNjY3Iy8tDbW0tmpubMXLkSDg7O0vLffHFF1ixYoXRcikpKfj8889RVlaG06dP48svv8S4ceMAAM8//zy8vLwwZ84cfPPNNygvL0d+fj7+/d//XXqCklqtxv/+7/+itLQUtbW10Ol0vfNHo47ZutON7NvdT2l2dHQUw4YNE7NmzRLZ2dnS8yGFEGLNmjXCz89PDBgwQMTExIjNmzebPMfyN7/5jRg6dKjRrRJ//etfhVqtFiqVSkRGRoovvvjC5FmG48aNEwMGDBCenp5izpw54vz589I6KysrRVxcnPDy8hIqlUoEBgaKhQsXivr6eiGEENeuXRNRUVHCzc2Nt0r0MRzPi4hkiZeNRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGTp/wAUEDbIgVJSKgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:09:16.091828Z", + "iopub.status.busy": "2024-03-22T14:09:16.091522Z", + "iopub.status.idle": "2024-03-22T14:09:16.340630Z", + "shell.execute_reply": "2024-03-22T14:09:16.339642Z" + }, + "papermill": { + "duration": 0.271426, + "end_time": "2024-03-22T14:09:16.342666", + "exception": false, + "start_time": "2024-03-22T14:09:16.071240", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABbRUlEQVR4nO2dd3hTZRfAfzdt091SVgeUDaXsjS3KRkREcIGIbHAAKqLMD0GGFhUEFWWICMqSJSpTQIYMlVUom5ZCWWXTSUeS+/0RGprOJM1s39/z3KfJHe89uc09Oe+5Z0iyLMsIBAKBg6GwtQACgUBgCkJ5CQQCh0QoL4FA4JAI5SUQCBwSobwEAoFDIpSXQCBwSITyEggEDolQXgKBwCFxtrUA1kaj0XD9+nW8vb2RJMnW4ggEghzIskxSUhJBQUEoFPnbVyVOeV2/fp3g4GBbiyEQCArhypUrVKxYMd/tJU55eXt7A9oL4+PjY2NpBAJBThITEwkODtbdq/lR4pRX1lTRx8dHKC+BwI4pzK0jHPYCgcAhEcpLIBA4JEJ5CQQCh6TE+bwcBVmWUalUqNVqW4siEJgVJycnnJ2dixyqJJSXHZKRkcGNGzdITU21tSgCgUXw8PAgMDAQpVJp8hhCedkZGo2G2NhYnJycCAoKQqlUimBaQbFBlmUyMjK4ffs2sbGx1KxZs8BA1IIQyisfztxI5MM1xyntqeTnwS2tdt6MjAw0Gg3BwcF4eHhY7bwCgbVwd3fHxcWFy5cvk5GRgZubm0njCOWVDxpZ5tT1RMp7u9rk/Kb+GgkEjoA5vt/iDskHT6VWr6ekq2wsiUAgyAuhvPLB01WrvFIz1Wg0osGSQGBvCOWVD56uTgDIMqSpRLiCrbh06RKSJBEZGWmVcfbv30/9+vVxcXGhR48eRTpnSWX37t1IksSDBw8seh6bKq958+bRoEEDXZ5hWFgYW7ZsyXf/JUuWIEmS3mKqs68w3JydyHrIlyymjnaDpW+MUaNG0ahRI2JjY1myZIlFziEwDzZVXhUrVmTGjBkcOXKEw4cP0759e7p3786pU6fyPcbHx4cbN27olsuXL1tENoVCwsNFa32lpgvLyxQyMjJsLYLRxMTE0L59eypWrEipUqVMGsOanzsrmNncOML/zqbKq1u3bjz77LPUrFmTWrVq8cknn+Dl5cU///yT7zGSJBEQEKBb/P39LSZflt8rJcO2lpcsy6RmqGyyGNNQvW3btowYMYKRI0dStmxZOnfuzMmTJ+nSpQteXl74+/vTt29f7ty5oztm69atPPnkk5QqVYoyZcrw3HPPERMTk+f4ly5dol27dgD4+fkhSRIDBgwwapyzZ88SHh6Om5sb9erVY8+ePbqxJUni7t27DBo0CEmSdJbXnj17aNGiBa6urgQGBjJu3Dg9hZHX586yELdt20bjxo1xd3enffv23Lp1iy1bthAaGoqPjw+vvfaaXjCyRqMhIiKCqlWr4u7uTsOGDVm7dq1ue9a4W7ZsoWnTpri6urJv375C/zfTp0+nfPnyeHt7M2TIEMaNG0ejRo102wcMGECPHj345JNPCAoKIiQkBICff/6ZZs2a4e3tTUBAAK+99hq3bt3SG3vz5s3UqlULd3d32rVrx6VLlwqVxxzYTaiEWq1mzZo1pKSkEBYWlu9+ycnJVK5cGY1GQ5MmTfj000+pW7duvvunp6eTnp6ue5+YmGiwTJ6uzpCUToqNLa+HmWrqTNpmk3OfntoZD6XhX5OlS5fy9ttvs3//fh48eED79u0ZMmQIs2fP5uHDh4wdO5aePXvy119/AZCSksKoUaNo0KABycnJTJo0iRdeeIHIyMhcj9ODg4NZt24dL730EufOncPHxwd3d3ejxhk9ejRz5syhTp06fPnll3Tr1o3Y2FiCg4O5ceMGISEhTJ06lV69euHr68u1a9d49tlnGTBgAD/99BNnz55l6NChuLm58fHHH+f5uQFu3LgBwMcff8zcuXPx8PCgZ8+e9OzZE1dXV1asWEFycjIvvPAC33zzDWPHjgUgIiKCZcuWMX/+fGrWrMnevXt5/fXXKVeuHG3atNGdb9y4ccycOZNq1arh5+dX4P9k+fLlfPLJJ3z33Xe0atWKVatWMWvWLKpWraq3386dO/Hx8WH79u26dZmZmUybNo2QkBBu3brFqFGjGDBgAJs3bwa0dfFefPFFhg8fzhtvvMHhw4f54IMPCv+imAGbK6+oqCjCwsJIS0vDy8uLX3/9lTp16uS5b0hICIsXL6ZBgwYkJCQwc+ZMwsPDOXXqVL4VFyMiIpgyZYpJsnkotdNGW1tejkTNmjX5/PPPAe2vfePGjfn000912xcvXkxwcDDnz5+nVq1avPTSS3rHL168mHLlynH69Gnq1aunt83JyYnSpUsDUL58eb1pnaHjjBgxQrfvvHnz2Lp1Kz/88ANjxowhICAASZLw9fUlICAAgO+++47g4GDmzp2LJEnUrl2b69evM3bsWCZNmqRTjNk/NzxWXtOnT6dVq1YADB48mPHjxxMTE0O1atUAePnll9m1axdjx44lPT2dTz/9lB07duh+wKtVq8a+fftYsGCBnvKaOnUqnTp1KvwfAnzzzTcMHjyYgQMHAjBp0iT+/PNPkpOT9fbz9PRk0aJFeik7gwYN0r2uVq0aX3/9Nc2bNyc5ORkvLy/mzZtH9erVmTVrFqC9R6Oiovjss88Mkq0o2Fx5hYSEEBkZSUJCAmvXrqV///7s2bMnTwUWFhamZ5WFh4cTGhrKggULmDZtWp7jjx8/nlGjRuneZ1VpNARduISNLS93FydOT+1ss3MbQ9OmTXWvjx8/zq5du/Dy8sq1X0xMDLVq1eLChQtMmjSJf//9lzt37qDRaACIi4vLpbwKwtBxsn9/nJ2dadasGWfOnMl33DNnzhAWFqaXotWqVSuSk5O5evUqlSpVyvW5s9OgQQPda39/fzw8PHSKK2vdf//9B0B0dDSpqam5lFJGRgaNGzfWW9esWbN8Zc7JuXPnGDZsmN66Fi1a6KzfLOrXr58r1/DIkSN8/PHHHD9+nPv37+td1zp16nDmzBlattTPQClo5mRObK68lEolNWrUALRfgEOHDvHVV1+xYMGCQo91cXGhcePGREdH57uPq6srrq6mRcl72onlJUmSUVM3W+Lp6al7nZycTLdu3fL8FQ4MDAS0fs/KlSvz/fffExQUhEajoV69ekY7jM01jqlk/9zZcXFx0b2WJEnvfda6LIWQZQlt2rSJChUq6O2X8zuc3/mKQs4xU1JS6Ny5M507d2b58uWUK1eOuLg4OnfubBcOfbuL89JoNHo+qoJQq9VERUXpbgRz4+EqouyLQpMmTTh16hRVqlShRo0aeounpyd3797l3LlzTJw4kQ4dOhAaGsr9+/cLHDPLMsheKsiYcbI/DFKpVBw5coTQ0NB8zxcaGsrBgwf1Hlzs378fb2/vAptDmEKdOnVwdXUlLi4u1/UqStOYkJAQDh06pLcu5/u8OHv2LHfv3mXGjBk89dRT1K5dO5ezPjQ0VGc5ZlHQAzdzYlPlNX78ePbu3culS5eIiopi/Pjx7N69mz59+gDQr18/xo8fr9t/6tSp/Pnnn1y8eJGjR4/y+uuvc/nyZYYMGWIR+bweWTupGSJUwhSGDx/OvXv36N27N4cOHSImJoZt27YxcOBA1Go1fn5+lClThoULFxIdHc1ff/2lN8XPi8qVKyNJEhs3buT27dskJycbNc63337Lr7/+ytmzZxk+fDj379/X8+vkZNiwYVy5coV33nmHs2fP8ttvvzF58mRGjRpl9vxTb29vPvzwQ95//32WLl1KTEwMR48e5ZtvvmHp0qUmj/vOO+/www8/sHTpUi5cuMD06dM5ceJEodVKKlWqhFKp5JtvvuHixYv8/vvvudwzb731FhcuXGD06NGcO3eOFStWWC0+zqbK69atW/Tr14+QkBA6dOjAoUOH2LZtm27OHxcXp3N8Aty/f5+hQ4cSGhrKs88+S2JiIgcOHMjXwV9UPB5F2QvLyzSCgoLYv38/arWap59+mvr16zNy5EhKlSqFQqFAoVCwatUqjhw5Qr169Xj//ff54osvChyzQoUKTJkyhXHjxuHv78+IESOMGmfGjBnMmDGDhg0bsm/fPn7//XfKli1b4Pk2b97Mf//9R8OGDXnrrbcYPHgwEydOLNK1yY9p06bx0UcfERERQWhoKM888wybNm3K9WTQGPr06cP48eP58MMPadKkCbGxsQwYMKDQAO9y5cqxZMkS1qxZQ506dZgxYwYzZ87U26dSpUqsW7eODRs20LBhQ+bPn6/3gMaSSLIxgTzFgMTERHx9fUlISCi0e9DMbeeYuyua/mGVmdLdcOdxUUhLSyM2NpaqVataLHtAIOjUqRMBAQH8/PPPNjl/Qd9zQ+9Rx/AC2wid5SWmjQIHJjU1lfnz59O5c2ecnJxYuXIlO3bs0IvnckTszmFvT3hlhUqIOC+BHVO3bl28vLzyXJYvX44kSWzevJnWrVvTtGlT/vjjD9atW0fHjh1tLXqREJZXAWSFJySL3EaBHbN582YyMzPz3Obv74+7uzs7duywslSWRyivAsiK80oVDnuBHVO5cmVbi2ATxLSxAB4nZgvLSyCwN4TyKoCsgoTC5yUQ2B9CeRWAh6hjLxDYLUJ5FcDjJhxi2igQ2BtCeRVA1rTxYaYatWjCIRDYFUJ5FUCWwx6E38saVKlShTlz5ujeS5LEhg0bTB5Po5HFj04xRoRKFICrswKFBBpZm5zt7eZS+EECs3Hjxo1Cq4TmhyzLnL6RiEaWqVfBF0UhScgCx0NYXgUgSdLjcAnhtLc6AQEBJtdik9F2PQfIVGlMlsFSDS4ERUcor0KwC6e9LENGim0WI/P2k5KS6NOnD56engQGBjJ79mzatm3LyJEjjf7Y2aeNWQ0y1q9fT7t27fDw8KBhw4YcPHhQ75h9+/bx1FNP4enhwdMt6jJj0lhSU1N02wtrKGFqgwuB9RHTxkJ4nJxtw1/fzFT4NMg2555wHZSGV+0cNWoU+/fv5/fff8ff359JkyZx9OhRvU41ReF///sfM2fOpGbNmvzvf/+jd+/eREdH4+zsTExMDM888wzTp0/n+0U/8N+ZWCI+GsPId99l6dIlQOENJbIwpsGFwDYI5VUIIjnbcJKSkli6dCkrVqygQ4cOAPz4448EBZlP8X744Yd07doVgClTplC3bl2io6OpXbs2ERER9OnTh5EjR6LWaMjwLM/YKTMY/MpzLFgwHzc3t0IbSmRhTIMLgW0QyqsQdB2EbDltdPHQWkC2OreBXLx4kczMTFq0aKFb5+vrq+sBaA6yN7TIKv9969YtateuzfHjxzlx4gTLly8HtA9aZFlGo9EQGxtLaGhooQ0lsjCmwYXANgjlVQie9hBlL0lGTd2KMzkbWgB6DSzefPNN3n33XVRqDdG3tA0tqpf3onrVKkY1lLBEgwuBeRHKqxA8RHK2wVSrVg0XFxcOHTqkawmWkJDA+fPnad26tcXP36RJE06fPk2NGjXIVGvI8NQ2GK4Z6IOzk4KoqChdQ4mshhaHDx+2uFwCy2DTp43z5s2jQYMG+Pj44OPjQ1hYGFu2bCnwmDVr1lC7dm3c3NyoX79+LkerufFyFWVxDMXb25v+/fszevRodu3axalTpxg8eDAKhaLQZg/mYOzYsRw4cIARI0YQGRnJ5dgYdm3bzLvvvgMY1lBC4DjYVHlVrFiRGTNmcOTIEQ4fPkz79u3p3r07p06dynP/AwcO0Lt3bwYPHsyxY8fo0aMHPXr04OTJkxaTUVeQUDjsDeLLL78kLCyM5557jo4dO9KqVStCQ0OtUo+/QYMG7Nmzh/Pnz9O+bRt6PdOG72Z9SmCg9oGBIQ0lBA6EbGf4+fnJixYtynNbz5495a5du+qta9mypfzmm28aPH5CQoIMyAkJCQbtP2vbWbny2I3yxF+jDD5HUXj48KF8+vRp+eHDh1Y5n6VJTk6WfX198/2fWor0TJV8/Mp9+fiV+3KGSm3VcwsKp6DvuaH3qN0EqarValatWkVKSkq+7cIPHjyYq+52586dcwUqZic9PZ3ExES9xRgeFyQUlpchHDt2jJUrV+p6Dmb14OzevbtV5RAZjcUfmzvso6KiCAsLIy0tDS8vL3799dd8+zDGx8fj7++vt87f35/4+Ph8x4+IiGDKlCkmyye6ZhvPzJkzOXfuHEqlkqZNm/L3339z5swZunTpku8xWa3uzUXJauhXMrG58goJCSEyMpKEhATWrl1L//792bNnj9kayY4fP16ve3JiYqJRrdN1dezF00aDaNy4MUeOHMm1/uHDh0RGRlpfIEGxxebKS6lUUqNGDQCaNm3KoUOH+Oqrr1iwYEGufQMCArh586beups3bxIQEJDv+K6uriYn9wIiMdtMuLu76/7P1kAYXsUfu/F5ZaHRaEhPT89zW1hYGDt37tRbt3379nx9ZOYgK0jV2paXLOY9RUNcP7vGHN9vm1pe48ePp0uXLlSqVImkpCRWrFjB7t272bZtGwD9+vWjQoUKREREAPDee+/Rpk0bZs2aRdeuXVm1ahWHDx9m4cKFFpMxKzE72UqWV1YEeWpqKu7u7lY5Z3FEqC47RJa12SJov9+gnzFhLDZVXrdu3aJfv37cuHEDX19fGjRowLZt23QJsXFxcSgUj43D8PBwVqxYwcSJE5kwYQI1a9Zkw4YN1KtXz2IyWtvycnJyolSpUroyLR4eHlYJ8CxupGeokFXalJ+0tDTUTnY3yShZpN6H9ERknwqkpqVz69YtSpUqhZOTk8lDSnIJm58kJibi6+tLQkICPj4+he5/9X4qT362C1dnBeem5/+0zJzIskx8fDwPHjywyvmKIxkqDbeStO6HQF83nBTiB8BmZD6ElDuADO5+4OpNqVKlCAgIyPOH2dB71OYOe3sny/JKV2lQqTU4W+EXXJIkAgMDKV++fL5t3AUFE3XtAR//HgnAL2+GUdbL9Ic2giJw7RhsHQHqdAh9Htp/hItSWSSLKwuhvAohy+cF2uRsX3frTT+cnJzM8k8uiaglF64laaf6LkpXq6QnCXJw8xSs6QVpCRDyLDwzFZzMp3KEI6AQXJ2dcHF65GQUUfYOgyZb16CS5RixEx7EwbKXtIqrUhi8vNisiguE8jII0Tnb8VALjWU7Uu7Czy9A0g0oXwd6rwQX8z85F8rLADztoZqqwChEv0YbkZ4My1+Gu9HgGwyvr9M66S2AUF4GIJKzHQ9NNstLFlFf1kGVAav7wvWj4F4aXl8PPpZrHCOUlwFkJWenCsvLYVCb3qpRYAoaDfw2DGL+0vY96LMWytWy6CmF8jIA3bRRWF4Og1o47K2HLMOf/4OoNaBwhl4/Q8WmFj+tUF4G4GEPjWcFRqERGst67J8D/3ynfd1jHtToWODu5kIoLwPQ1bEXlpfDIBz2VuLYMtjxsfZ150+hQU+rnVooLwPI8nlZKzlbUHT0HfYCi3BuC/z+rvZ1q5EQNtyqpxfKywBEQULHQ1heFibuX1gzAGQ1NHodOn5sdRGKFPKanJysa/iZhSHJzo6GKEjoeAjlZUFunYEVPUGVBrWegW5f6UrdWBOjLa/Y2Fi6du2Kp6cnvr6++Pn54efnR6lSpfDzs0wwmq2xi67ZAqPQf9ooFJnZeHAFfn4R0h5AcEt4+Uezp/0YitFnff3115FlmcWLF+Pv718iak1lJWeLrtmOg0gPsgApd2HZi5B0HcqFQu9VoPSwmThGK6/jx49z5MgRQkJCLCGPXfK4IKGwvBwFkZhtZjJStFPFO+fBp6I27cejtE1FMnra2Lx5c65cuWIJWeyWxz4vYXk5CsLnZUbUmbC6P1w7rM1T7LsefCvYWirjLa9Fixbx1ltvce3aNerVq5erBnWDBg3MJpy98DgxW1hejoJa6C7zoNHAb8Mhenu2tB/7mHUZrbxu375NTEwMAwcO1K2TJAlZlpEkCbXacOskIiKC9evXc/bsWdzd3QkPD+ezzz4rcEq6ZMkSvXODtr1ZWlqasR/FYHS5jcLn5TBohOVlHrZ/BCd+0ab99PwJKjaztUQ6jFZegwYNonHjxqxcubLIDvs9e/YwfPhwmjdvjkqlYsKECTz99NOcPn0aT0/PfI/z8fHh3LlzuveWfmjg5SpyGx0N4bA3A/u/goNzta+7fws1O9lWnhwYrbwuX77M77//bpYGolu3btV7v2TJEsqXL8+RI0do3bp1vsdJklRgo1lzI4oROh4iMbuIRK6A7ZO0r5+eDg1fta08eWC0w759+/YcP37cErKQkJAAQOnSBT/FSE5OpnLlygQHB9O9e3dOnTqV777p6ekkJibqLcaS9bQxUy2ToRK1VhwBMW0sAue3wW8jtK/D34Xwd2wrTz4YbXl169aN999/n6ioKOrXr5/LYf/888+bJIhGo2HkyJG0atWqwD6MISEhLF68mAYNGpCQkMDMmTMJDw/n1KlTVKxYMdf+ERERTJkyxSSZssjehCM1Q4XSWVmk8QSWR0wbTeTKf9oni7IaGvaGjkW7dyyJ0X0bszeBzTWYkQ777Lz99tts2bKFffv25amE8iMzM5PQ0FB69+7NtGnTcm1PT08nPT1d9z4xMZHg4GCD+zZmUWviFjJUGvaPa0+FUqKTtb3z5Z/n+PqvaAD2jG5L5TL5+1AFj7h1FhZ31kbP13waXl0BTqZ3tDYVi/VtzJnLaA5GjBjBxo0b2bt3r1GKC7Ttwhs3bkx0dHSe211dXXF1LXrPPk+lExkqjfB7OQjC8jKShKva6Pm0B1CxObyyxCaKyxiM8nllZmbi7OzMyZMnzXJyWZYZMWIEv/76K3/99RdVq1Y1egy1Wk1UVBSBgYFmkSk/hNPescheBlrosUJIvafNV0y8BmVD4LXVoLR/S9Uoy8vFxYVKlSqZPDXMyfDhw1mxYgW//fYb3t7exMfHA+Dr64u7u3Zq1q9fPypUqEBERAQAU6dO5YknnqBGjRo8ePCAL774gsuXLzNkyBCzyJQfXiLWy6EQlVQNRJf2cw58Kmij522c9mMoRj9t/N///seECRO4d+9ekU8+b948EhISaNu2LYGBgbrll19+0e0TFxfHjRs3dO/v37/P0KFDCQ0N5dlnnyUxMZEDBw5Qp06dIstTEFlOe1GQ0DEQ6UEGoM7U1uS6egjcSmm7/fga57axJUb7vObOnUt0dDRBQUFUrlw5VzDp0aNHDR7LkGcFu3fv1ns/e/ZsZs+ebfA5zIVIznYs9OK8bCiH3SLL8Ps7cOFPcHaHPmugfG1bS2UURiuvHj16WEAM+8fTVTSedSTEtLEQtk+C4ytBcoKeSyG4ha0lMhqjldfkyZMtIYfdIwoSWhdZlrmTnEE5b9OeFItpYwEc+AYOfK193X0u1OpsW3lMxOQSiEeOHOHMmTMA1K1bl8aNG5tNKHtEFCS0LjO2nGXB3ot89lJ9ejWvZPTxeg04hBX2mOOr4M+J2tedpkKj12wrTxEwWnndunWLV199ld27d1OqVCkAHjx4QLt27Vi1ahXlypUzt4x2gaeua7awvKzBgr0XAZj6x2mTlJdK1MTJzYXt2vI2AGEjoNV7tpWniBj9tPGdd94hKSmJU6dOce/ePe7du8fJkydJTEzk3XfftYSMdoFu2igsL4dALVqf6XP1MKzuBxoVNOgFnXJnozgaRlteW7duZceOHYSGhurW1alTh2+//Zann37arMLZEx6iIKHViL2TonttarkjkZidjdvnYPnLkJkKNTppy9sUkObnKBj9CTQaTa5kbNAGsFoidche0E0bRaiERbmfkkG7mbt1702t1CZmjY9IuKaNnn94Hyo00z5ZtPO0H0MxqSTOe++9x/Xr13Xrrl27xvvvv0+HDh3MKpw9IerYW4dLd1P0V5iovUQDDrRpP8tehMSrULaWNpbLAdJ+DMVo5TV37lwSExOpUqUK1atXp3r16lStWpXExES++eYbS8hoF+jq2AvLy6qYbHmV9GljRiqsfBVunwXvIG30vIOk/RiK0T6v4OBgjh49yo4dOzh79iwAoaGhdOzY0ezC2RMiMds2mOrz0q8qUcIUmToT1g6EK/+Cm682X7FUsK2lMjsmxXlJkkSnTp3o1Mm+alpbEpGYbRtMbU9QYh32sgx/vAfnt4Kzm7ZCRPnQwo9zQExSXjt37mTnzp3cunUrl5N+8eLFZhHM3hCJ2dbBXCqnxNbz2vExRC7Xpv28sgQqPWFriSyG0cprypQpTJ06lWbNmhEYGGjxzj32wuPEbLWuzZvA8pjD51Vi9NjBb2H/HO3r57+GkC42FcfSGK285s+fz5IlS+jbt68l5LFbsiwvtUYmXaXBzcWpkCME5sDkOK8So7EecWI1bJugfd3xY2j8uk3FsQZGP23MyMggPDzcErLYNVmWFwi/lzURTxsNIHoHbHhb+/qJYdBqpE3FsRZGK68hQ4awYsUKS8hi1zgpJNxctJdLPHG0f7K7You1Grt6BH55lPZT/xV4+hPTn3I4GEZPG9PS0li4cCE7duygQYMGuaLtv/zyS7MJZ294Kp1Jy8wQsV5WxNT7sEQ47O9ceJT2kwLVO0D374pF2o+hGP1JT5w4QaNGjVAoFJw8eZJjx47plsjISKPGioiIoHnz5nh7e1O+fHl69OjBuXPnCj1uzZo11K5dGzc3N+rXr8/mzZuN/RgmUdQoe1GaxXoUe4d94nX4+QV4eA+CmkDPn6CE9RM12vLatWuX2U6+Z88ehg8fTvPmzVGpVEyYMIGnn36a06dP5yovncWBAwfo3bs3ERERPPfcc6xYsYIePXpw9OjRApvVmoOiJGe/8dNhbiWls/7tcBSKkmHWmwfhsM/Fw/uw7CVIuAJlamjTfly9bC2V1TG5GKE52Lp1q977JUuWUL58eY4cOULr1q3zPOarr77imWeeYfTo0QBMmzaN7du3M3fuXObPn29ReYuSnP3n6ZsAnI1Pok6QtpHmvgt3CPB1pUZ5b/MJWcwwedpYXB32mQ9hZW+4dRq8A6Hvr+BZ1tZS2QS7miAnJCQAULp0/jlYBw8ezJWK1LlzZw4ePGhR2cA8ydnyI/fx+ZtJvP7Dv3T8cq9ZZCsu5DSYzBLn5WAu++R0FRduJuXeoFbB2kEQd1Cb9vP6OihlfKHG4oJNLa/saDQaRo4cSatWrQqc/sXHx+Pv76+3zt/fX9fzMSfp6emkp6fr3icmJposozmSs7NuzrPxeXw5BbkwOT3IQaeN6So1vRYc5PSNRJYMbEGbWo8qE8sybHwPzm3Wpv30XgX+dW0rrI2xG8tr+PDhnDx5klWrVpl13IiICHx9fXVLcLDpCaqPk7OLHuclnPd5k1NZSSbaXioHddh/uf08p64nIsvw6aYzjy3Iv6bBsWUgKeDlH6FyyYu1zInRymvv3r2oVLktD5VKxd69pk2BRowYwcaNG9m1axcVKxbc9DIgIICbN2/qrbt58yYBAQF57j9+/HgSEhJ0y5UrV0ySER63PxMFCS2HuRSNIyZm/xd7j4WPave7Ois4dzOJX49dg3/mw9+ztDt1+wpqP2tDKe0Ho5VXu3bt8uyWnZCQQLt27YwaS5ZlRowYwa+//spff/1F1apVCz0mLCyMnTt36q3bvn07YWFhee7v6uqKj4+P3mIqZvF5Od49ZVNKSpxXUlomo1ZHIsvQs1lF3u9UC4CoLYtg61jtTh0mQZN+NpTSvjDa55VfUvLdu3fzDW/Ij+HDh7NixQp+++03vL29dX4rX19f3N3dAejXrx8VKlQgIiICgPfee482bdowa9YsunbtyqpVqzh8+DALFy409qMYjacZ6thnOY+z31t/nb3Jr8eu88kL9fBxKx4les2FqQ57vQh7B9Bj0zae5ur9h1T0c+ej5+rg4qTgzL4N/C/ja+1FaPkWPDnK1mLaFQYrrxdffBHQJsoOGDAAV9fHzUDVajUnTpwwOudx3rx5ALRt21Zv/Y8//siAAQMAiIuLQ5Etajg8PJwVK1YwceJEJkyYQM2aNdmwYYPFY7wgm8/LDNPG7E/ABi05DEBZLyWTu5VsJ2xOTC5G6EDTxm2n4ll9+CqSBF/2bIS3mwtcO8IszRc4S2o204rw1lMoVULSfgzFYOXl6+sLaC0vb29vnWUEoFQqeeKJJxg6dKhRJzfEab179+5c61555RVeeeUVo85lDsxZkDCvj34rKT33yhKGue5PR5k23k5KZ/z6KADeaF2NFlVLw51oWP4KzuqHHHFuxHvJbzJwTywTni2eRQVNxWDl9eOPPwJQpUoVPvzwQ6OniMUBcxQkLOieEr+rlnHY22uclyzLjF9/gnspGdQO8GZUp1qQeEOb9pN6F4Iakxz2A5nLz7LkwCX6h1ehQin3wgcuIRjtsJ88eXKJVFyQvSBhUXxej/7mcT+JAofmwxEsr18OXWHHmVsonRTMebURrplJj9J+4qB0deizltb1qtGyamkyVBq+/PO8rUW2K4xWXjdv3qRv374EBQXh7OyMk5OT3lKc0aUHGfm0Ma/pcV63lkh5zI050oPsUY9dvpvC1I2nAfiwcy1ql3F5lPZzCrwCdGk/kiQx/tF0cf2xq5y5YXqQdXHD6KeNAwYMIC4ujo8++qhElYGGx4nZRZs2ynp/s1NyrqThFMcGHGqNzKjVx0nNUNOyamkGh1eCtQMg7gC4Pkr78aus279RcCm61g9kU9QNPtt6liUDW9hOeDvCaOW1b98+/v77bxo1amQBcewbTzM47OUcf7OjKEE/BIZiaoS9PU8b5++J4cjl+3i5OjPrlQY4bR4FZzeCkyv0XgkBuZ+cj+4cwrZT8ew+d5sDMXcIr14yk7GzY/S0MTg4uMSmtmTPbTTmGuS5q/B5GYTplpd55TAXJ68lMHu71nf18fN1qRg5G47+9CjtZzFUaZXncVXKevJaS20S9owtZ0vsPZgdo5XXnDlzGDduHJcuXbKAOPZNluUly5CWWbS7I68nYEJ35cbkqhJ2eHOnZap5/5dIVBqZZ+oG8FLmRtj7hXbjc7Mh9LkCj3+nfU08lU6cuJrApqgbVpDYvjFaefXq1Yvdu3dTvXp1vL29KV26tN5SnHHP1jHIVL+XCJUwDnMEqdqLHvti2zku3EqmrJcrM+tEI20dp93QbiI0HVDo8eW8XRnauppurAyVnZqXVsJon9ecOXMsIIZjoFBIeCidSM1QPwqXcC30mNzkTg/SjS9ML7Ngj876A9F3+GFfLADfP5mI16bhgAwt3oDWHxo8ztCnqrHsnzgu301l5X9x9A+vYhmBHQCjlVf//v0tIYfD4OnqTGqG2qjkbANdXmLaCOS8MqZcEnubMiY8zOTDNccB+LB+Ko0PjARNJtR9AZ6ZYdQ/3tPVmfc61uSjDSf5eucFXmxSQZtOVAIxqZ5XTEwMEydOpHfv3ty6dQuALVu2cOrUKbMKZ48UtSBh1n0lglQNxIRLkjOv0dYR9h//forrCWmE+z1g+LVxkJEMVdvACwtAYXxs5KvNg6la1pO7KRl8/3esBSR2DIxWXnv27KF+/fr8+++/rF+/nuTkZACOHz/O5MmTzS6gvfG4IKGJyquAbSJI1TzYUxXVTSdu8Ouxa/hL9/nR6VOk1DsQ2BBeXQ7OprgdwMVJwejOIQAs+vsit5LSzCmyw2C08ho3bhzTp09n+/btKJWPWy21b9+ef/75x6zC2SNFTc7WWV7iaWMuztxIpP/iQ3rrTJo22onP62ZiGv/bEIU3qfzhNxvX5KtQuhr0WQeuRWu60qVeAI2CS5GaoearHRfMJLFjYbTyioqK4oUXXsi1vnz58ty5c8csQtkzpiRn55kelNe0sYQ/b3x14T+5rqspU+lc00Yb6DJZlhm99gQPU1NY4TWH8qnR4OUPr68Hr3JFHl+SJMZ3qQ3AqkNXiLmdXOQxHQ2jlVepUqW4cSN3jMmxY8eoUKGCWYSyZ3TJ2UVIEQKR25gXCQ8zzTKOPVhey/6NY9/5m3yj/Jb6qpPg6qNN+yldeLVgQ2lZrQwdapdHrZH5YmvhzZqLG0Yrr1dffZWxY8cSHx+PJEloNBr279/Phx9+SL9+xb9Era7xrMnTxvw99sJhnxtzPG20tiq7eDuZTzadYrrzYp5WHMqW9lPf7Oca80xtFBJsPRXP0bj7Zh/fnjFaeX366afUrl2b4OBgkpOTqVOnDq1btyY8PJyJEydaQka7wpTGs4bePEJ35caUa2LL1CCVWsP7q4/ztryG15z/QpYU8NIiqPKkRc4XEuDNS020TWtmbC5ZaUNGKy+lUsn3339PTEwMGzduZNmyZZw9e5aff/652JfEgccdhExtwlFQYnZJ9HkVFlBqyjWxZZzXt7tiqH99De85rwdA6joL6jxv0XOOeroWrs4K/rt0j51nbln0XPaEyX0bK1WqxLPPPkvPnj2pWbOmSWPs3buXbt26ERQUhCRJbNiwocD9d+/ejSRJuZb8Gs5agiKHShQQ51XSfF4fbThJ8092cDc5//LXplleOR321lFmx6884MKun5nqvES7ou0EaDbI4ucN9HVnYCutL+2zrWdRqUtG2pBBEfajRo1i2rRpeHp6MmpUwR1MvvzyS4NPnpKSQsOGDRk0aJCuwYchnDt3Tq+FWfny5Q0+tqgUOVRClx5kWKhE9K1kfj9+ncFPVsXXvXhFUv/8z2UAVvwbR8/mpjcDzoktHPYPM9T8tOInZjl/i0KSkZsNQWozxmrnf7ttdVYdiuPCrWTWHb1Kr+aVrHZuW2GQ8jp27BiZmZm61/lhrMO5S5cudOnSxahjQKusSpUqZfRx5sCUgoR5/fDnnR6U+/p1mr0HWYar91P5smcjg8/pSCgUEu+szP97ZSy2cNgvXbeBj1M/xVVSkRHSDeWzn1vVienr7sKIdjWYvukMs7df4PmGFXBXFm83jkHKa9euXXm+thWNGjUiPT2devXq8fHHH9OqVd41kCyBKQ57PQqqKpHHdz3rPoyMe2Da+RyEE1cf5LnelCew1k7M/u/IIV46OxJv6SH3/cPwe+UHk9J+ikrfsMr8uP8S1x485McDsQxrW8PqMlgTk31etiAwMJD58+ezbt061q1bR3BwMG3btuXo0aP5HpOenk5iYqLeUhR0oRJF6JoNeVtjf5+/Q4dZu/N85F2cnyGZu5qGNR32CbeuEPRHH8pJiVx3r4XfwNUmp/0UFVdnJz54Wttpe97uGO6nZNhEDmthkOVljD9q/fr1JgtTGCEhIYSEhOjeh4eHExMTw+zZs/n555/zPCYiIoIpU6aYTYYsn5cxidl5pQLldXudftRcoe+ifzk19RmT5HNEJCn/KHhzpAdZTJelJZD4Qw+Cuck1KYDSQ38HN5/Cj7MgPRpV4Pu/YzlzI5G5u6L56Lk6NpUni5//ucz1Bw95q3V1fD3M47s1yPLy9fXVLT4+PuzcuZPDhw/rth85coSdO3fqGtNakxYtWhAdHZ3v9vHjx5OQkKBbrly5UqTzPX7aWMRQiQLuqLwCYItz/E5BT1ntNs4rM43bi14mOD2a27IviS+vxr10oBVOXDAKhcS4R2lDPx+8zJV7qTaWSOsfnr39PPN2x7DtlPkiAwyyvLIazgKMHTuWnj17Mn/+fF1cl1qtZtiwYXpPAK1FZGQkgYH5f2lcXV1xdTWfGZ8V52Wqz8tUHRSfWHwrBxQ0bTRFeVl82qhR8/CXQZS78x9JsjvbG3/La3UbWvacRtC6Zlla1SjD/ui7fLn9PLN7NbKpPIv+vsi9lAyqlfPkxSbmSyE02ue1ePFiPvzwQ72AVCcnJ0aNGsXixYuNGis5OZnIyEgiIyMBiI2NJTIykri4OEBrNWVPOZozZw6//fYb0dHRnDx5kpEjR/LXX38xfPhwYz+GyWTvIGSKY1guoJJqQRS1Zr69k27Gksa5QyXMqMxkGXnTh7hHbyJdduazUpPo2a2r+cY3A5IkMe4Zba/HDZHXOHU9wWay3E1O5/u9FwH4oFMIzk7mc7MbPZJKpeLs2bO51p89exaNkfb64cOHady4MY0bNwa08WSNGzdm0qRJANy4cUOnyAAyMjL44IMPqF+/Pm3atOH48ePs2LGDDh06GPsxTCYrMRsgNdOwqWPeoRLFdxpoLL8fv57vNlMi7C1az2vPZ0hHFqORJcZoRjDo9X5mvSHNRf2KvnRrGIQsa7sN2YrvdseQkqGmfgVfutQLMOvYRpeBHjhwIIMHDyYmJoYWLbTNL//9919mzJjBwIEDjRqrbdu2BfpylixZovd+zJgxjBljvcC/vHBzUaCQQCNrK0tkOfDz4tT1BC7dSaVD6OMg2mLsujKZE1fztwxMmjZaymF/6AfYHQHAJNUAmnUbRLVyXmYa3PyMfjqErSdv8PeFO+y7cIcna1q31+O1Bw91gcijO4egMHMKidHKa+bMmQQEBDBr1ixdaZzAwEBGjx7NBx98YFbh7BFJkvBUOpOUriq0ssSbPx/h6v2HbHr3cVLuY4e9BYUsRpjydbdInNepDcibPkAC5qhe5Er115jW0r6j2CuV8aBPy8osOXCJiC1n+KP6k2ZXIAXx1Y7zZKg0PFGtNE9ZQHEabe8qFArGjBnDtWvXePDgAQ8ePODatWuMGTOmRCRmw+OChAXlN95MTOPq/YcAXHv0NzslUXfdTU7n+bn7dL/GBmFKMUJz/zLE7oX1Q5GQWa7qwBKXV/n85QYOUcLonfY18HJ15tT1RP44kf/03NxE30pm7ZGrgLZsjyWuVZEm6z4+PjZ5wmhrPA1Izs4+FbqXR7BgSbS8vt55gRNXE/how0mDjzFLnJcJY+i4cRxWvgbqDLaqm/ORaiCfvNAAfx+3ooxqNcp4ufJWG22vx5l/niNdVbTgakP5cvs5NDJ0quNPk0p+FjmH0dNGgLVr17J69Wri4uLIyNC/MQuKdi8ueOjCJfL/IkRlS3e5l/r4GhXneK3CeJjtAcfoR63ACsOkOC9zXeN7F2HZy5CRxDFFPd5LG073xsF0bWD7eC5jGPRkVX46eJkr9x6y/J84Bj1pvmqueXHi6gM2R8UjSfDh0yGFH2AiRlteX3/9NQMHDsTf359jx47RokULypQpw8WLF01KsnZEsiyvgpKzT1zLZnklZ1Neur8lT4llj+da82hKYQlUajM47JNvwc8vQsotrrvVoF/qSMr4+vDx83XNI6QV8VA6M7KjNm3om78ukJhmnnLb+fHFNm1J6hcaVyAkoGiNRgrCaOX13XffsXDhQr755huUSiVjxoxh+/btvPvuuyQk2C6exJoUlpwtyzJRYtqYC2s5i4tseaUlwrKX4H4sDz0r0v3B+yThwcxXGjpsWaKezSpSvZwn91MzWbAnxmLnORB9h78v3MHFSeL9RwrTUhitvOLi4ggPDwfA3d2dpKQkAPr27cvKlSvNK52dUlhy9vWENO5mU1jZX5dAg0uHKbrLNJ+XCQdloUqHX/pA/Ak0HuXonTaO2/gx+MmqhNewbqiBOXF2UjDmGW3a0A/7YrlpgYwNWZb57JHV9VqLSgSX9jD7ObJjtPIKCAjg3r17gLaaalavxtjY2BLjz/EqxPKKylHe5X52n1cBxQiLO6ZUjzCp9VnOel6GXmuNGta/AbF7kZVefFp6KpEppalZ3kvX5NWRebqOP00r+5GWqWHOjvNmH//P0zc5fuUBHkonRrQ3rbqyMRitvNq3b8/vv/8OaANW33//fTp16kSvXr3y7OdYHPHQ+bzytryynjRWevTLk33aWAJ1lg5TrCirxXnJMmwZC6c3gJOSv5t8xaJoX1ycJGb3aoSbi+OHAWXv9fjLoStE30oy29hqjczMR1bXoFZVKedt+bJARj9tXLhwoS4NaPjw4ZQpU4YDBw7w/PPP8+abb5pdQHuksOTsqEfO+rYh5fjp4GXh83qEKVaUOSLsDWLvF3Doe0DiTqdvGLbFG1AxsmMt6lWwfrUUS9GsSmk61fFn++mbfLb1HN/3a2aWcX89do0Lt5LxdXdhaOtqZhmzMIyyvFQqFdOnT9drePHqq6/y9ddf884776BUKs0uoD2S5bDPy+cly7LO8mobou2MnFdIRQnUXThZyWFvdBnowz/Crk8A0HT5nOHHK5OcrqJpZT/ealPdMkLakLHPhKCQYPvpmxy+dK/I46Wr1Mzerp2GDmtb3WoPNYxSXs7Oznz++eeoVEXrFu3oeCrzj7CPu5dKwsNMlM4KnqhWJtf2kmhxZWGaw97CZaBP/w6bHjWVaT2GH9I78m/sPTyUTnzZs6HVFK41qVHem16PGp5EbCl6r8cV/8Zx7cFD/H1c6R9exQwSGobRPq8OHTqwZ88eS8hif6hVeWobXUHCPKaNWVZXaKAPHkpnvHMkbpfk3EaTyj2boZ5Xvtf60j5YNwRkDTQdwNnQEboYpUnP1aFyGU/jT+4gjOxYCzcXBUcu3+fP0zdNHiclXcXcv7TFQN/tUNOqvkGjfV5dunRh3LhxREVF0bRpUzw99f/Bzz9v2QabViM9CX7pC5VbQZvReps8C4iwz/J3NXjkJ/HzVJKUh4VWEoNUTfJ5mXAegyyv+ChY2RvU6VD7OdI7f8HI7/4hQ62hY2h5nWVSXPH3cWPIk9WYuyuaz7eepUPt8iaV9lm8L5a7KRlUKeNBz2bWvWZGK69hw4YBefdnlCQJtdo6uVMW5/w2uLhLu3iWhWaPy/089nnlZXk9ALT1lABKeyqJy1aKN8tEL5mWl/HHWMRhf/+SNgg1PVH74/TSD8zecZGz8UmU9lQS8aJjJF0XlTfaVGP5v5eJuZ3C6sNXec3IKhn3UzJY+KjQ4KinQ3Cxcl0zo8+m0WjyXYqN4gKo/zI89aH29aZRcOYP3ab8po0ajczJa9omGg2yKa/syDn+FoXEtEy2nownzcCiiLbGpDgvE2yvHNlB+lZu8m34+QVIvgn+9eDVFfx3NZUFe7VR5xEv1rfKY357wMfNhXcexWPN3nHe6NLm8/bEkJSuok6gD8/Vt36+p/2VgLQn2k+EJv20PpG1g7U+ErJNG3M8bYy9m0Jyugo3FwU1HhWpy6W8zOD0unIvlSFLD9N46nbeWnaE6ZtOmzyWNbGW7zvfaWN6Eix/WZtwXaoSvL6OJMmTUasjkWV4pWlFOtc1b7VPe6fPE5UILu3O7aR0Fu+LNfi4GwkPWXrgEgCjnzF/oUFDMFh5PXz4kI0bN+rejx8/nlGjRumW0aNHk5ZWzJpESBJ0nQ0hXbW+kZW9IT7qcUmcHL9UWfmMdYN8df6DnMorK++uKJbXOyuPsePMTd30aM1hyyU5mxNrTcVy1fOS0ab9rOoDNyLBoyz03QDeAUzbeJqr9x9S0c+dSd3so02YNXF1dtJVfpi/5yJ3k9MNOu7rnRdIV2loUaU0bWuVs6SI+WKw8lq6dCkLFizQvZ87dy4HDhzg2LFjHDt2jGXLljFv3jyjTr537166detGUFAQkiSxYcOGQo/ZvXs3TZo0wdXVlRo1auQqFW12nJzh5R+gUrjWR7LsJbwfXgO0TTFU2RLpsp401s8W1JhTeZmjntLVPIobOgKmpQcZf55cPi9ZA7++BbF7QOkFfdZAmepsOxXP6sNXkST4smcjvN0cM+m6qHRrEES9Cj4kp6v45q/82whmcfF2MqsPZxUaDLGZf9Bg5bV8+XLeeOMNvXUrVqxg165d7Nq1iy+++ILVq1cbdfKUlBQaNmzIt99+a9D+sbGxdO3alXbt2hEZGcnIkSMZMmQI27ZtM+q8RuPiDr1XQvm6kHwTn7U9KYNWUWVvwhF17QHw2N8FUNojh/J61AVIOOwNw7S+jdkvrkzVw1Ph1HpQuECvn6FCE24npTN+fRQAb7SuRouqpY0/UTFBoXjcbWj5v5eJu1twr8cvt59HrZHpULs8zarY7roZrLyio6OpX7++7r2bmxsKxePDW7RowenTxvleunTpwvTp0w3OiZw/fz5Vq1Zl1qxZhIaGMmLECF5++WVmz55t1HlNwr0UvL4OfCuhuH+RJcrP8eShzu+lUmtyOetBGyqRnYxHllrRQiXMWCnUipjiFzHNYf/4igx3+o3Acz8DErwwH6q3R5Zlxq8/wb2UDGoHeDOqk2VLtzgCT9Ysy1M1y5Kplhn5yzF2nb1FZh7lOU5eS2DjiRvaQoM2TlY3WHk9ePCA9PTH8+Hbt29TpUoV3XuNRqO33RIcPHiQjh076q3r3LkzBw8ezPeY9PR0EhMT9RaT8QmEvr+CRxnqK2KZ7zKb5FTtr1TM7RQeZqrxVDpRrezjjjI5p40ZqqJbXo5qtZkUo1oEy6uX0y5GuzyaDXT5TPsEGW1S8o4zt1A6KZjzaiNcnR0/6docjOtSG6WTgqNxDxi45BBPfLqTj38/ReSVB7oQn6wg3ucbBhEaaNsS8AYrr4oVK3LyZP61x0+cOEHFihXNIlR+xMfH4+/vr7fO39+fxMREHj7M2w8UERGBr6+vbgkOLmIgXdka0GcNqbjxlNNJym5/DzQazt/UZujXDvTRszBy+7yyLC/TcVDdZVqEvQmoZZmnFYf41HkRAFfqvQ0ttUUDLt9NYepG7Qzhw861qB1Q8now5EfdIF9+f6cVA8KrUMZTyd2UDJYcuESPb/fTftYeJm6IYs/52zgrJLuwVg1WXs8++yyTJk3K84niw4cPmTJlCl272lfnYNA+FU1ISNAtV65cKfqgFZrysfs4MmUnSl38HbaOI/Z2MgDVyupnHORSXo98ZGa1nuxMm0VsPkOvBQdzTTus9TQ98MFRvnGZi5Mks0rVlssNtC351BqZUauPk5qhpmXV0gx+0jrVDxyJ2gHaUtf/TOjAjwOb071REG4uCmLvpLDsH20D6FdbBNtF6pTBEfYTJkxg9erVhISEMGLECGrV0mrec+fOMXfuXFQqFRMmTLCYoKAthHjzpn4e1s2bN/Hx8cHd3T3PY1xdXXF1NX/Q4TnP5nyQ8BZfK7+F/xZQvZwaaE/Vcvr/VB83/Utszrb29sqCR1HXO07fpEu24EWrFCOMP8nL50fjKmXyp7op/1MNZumjMebvieHI5ft4uTozq5gmXZsLFycF7ULK0y6kPCnpKv48Hc+vx66TnJbJex1sb3WBEcrL39+fAwcO8PbbbzNu3DjdHFiSJDp16sR3332Xa0pnbsLCwti8ebPeuu3btxMWFmbR8+aFh9KZ3zWtGFrfh/pREXS9vYi9ThLVyjbR20+SJFydFTql9XjaaLq55ChVWDNyWV4Wzm28fxmWvYSbOpl/NbV5J/Md1DghI3PyWoKueujHz9elop9lSxQXJzxdnXmhcUVeaGxZt5CxGJXbWLVqVbZu3cq9e/eIjtbGg9SoUYPSpU17XJqcnKwbB7ShEJGRkZQuXZpKlSoxfvx4rl27xk8//QTAW2+9xdy5cxkzZgyDBg3ir7/+YvXq1WzatMmk8xeFrCj7k8GvUc8nDWn/bD51XkR80hPAy3r7lvFUcj1BO93WdRwqRP+kZao5GHPX3GJbnPiE3G6Fvy/c5ljcA7zdjE6lNdxhn3IHlr0IyfHcdKvO0AcfkI52yp6eqeH9XyLJVMt0ruvPS00qGC2HwP4wqW9j6dKladGiRZFPfvjwYdq1a6d7P2qUtq5S//79WbJkCTdu3CAuLk63vWrVqmzatIn333+fr776iooVK7Jo0SI6d+5cZFmMJXty9r0nxrFjzzF6Oe8maOdwCAqCyuG6ff2yKa+1R64yqFXVQu2ucetOsCEy7w7HOY+1pwoVn24+o3udlU3Q94f/AGhhQkyQQborPVmb9nM3GnwrsbTiLBKPPI5V+mLbOS7cSqaslyufvlC/RCRdlwRMUl7mom3btgVOgfKKnm/bti3Hjh2zoFSGoUvOTlcTezeVCarBVFCm8KTqEKx4FQZtAX9tj7+cTvvPt52lol/ePros8lNceZGZMxPZyqg1ss5/lL3Sxt3kDH46eEn3PvZuitFjqwqrEKHKgF9eh+vHwKMM9P2VB38/BB7/6J179CT485frU8arZCRdlwREYraJZFVTTc1QcfFOCmqc+DHwIwh+AtITtA1L718Gciuv1HS17smNKdiTyyv2TgoNp/zJrD/P5do2fdMZJv12Svc+r6DHwkhMK6DSgUYDG97Wli1y8dSm/ZStkWdidu8WlWhf27I+WYF1EcrLRDxcHydnx97RWhQVy5eB11ZBuVBIjtf6YFLuUCdHMN9/Zqgbbi/M/POcLiduc9SNAvfNNOFJa+LDfLo7yzJsmwAn14LC+VHaT1Mg73peE7uGGn1ugX0jlJeJeLk+bjwbe1urvKqW9QR3P+i7HnyDtT6Y5a/Qrpp++ER49dy17Q0hK+PfXp82Dlt+NHdFh2yYMr3N+mEASMiuyPbNhn8fFQLoMR9qdNBtyimDJD32UQqKD0J5mchjn9djy6vqoxpe+ATB6+vBvTRcP0qt3W8z7bnHsTGm+ovn7LgA2FlMag5hdp+7ne+uOUMnDOXag4es+DeOhlP+ZOV/cXD0Z9g5RbvxmRnQ4BW9/XNOG10U4mteHBH/VRPJCpVITlfpHNF60fXlaml9MC4eEPMXfW9+xsuNgwDTHex5NfwoCRyKvceEX7UVIHZuWAJ/vKvd8OT78MTbufbPeXmdncTTxeKIUF4mklWQ8MKtZDJUGpROCoJK5XiCWLEZ9PxZ65OJWkP3W98Csl4NMGMwpcJCcSDLUm0mnWWuy9fa+lyNX4cOk/PcP6flJSLpiydCeZlIlg/ldpLWD1W5jEfeN0nNjtBD65t56u4a3nL6g4eZpimvdUevas9nV/NGy6ORZUKkOH5QzsRNyoRaXeC5r/Kdf6s0+tdXaeXGEALrIP6rJuKh1C+jUrVsAYmqDXpC508BGOeyitYpphdP/HL7uZKmu3BJusZS5Wf4Sqkc0tSClxdrK9zmgyrHvFFMG4snQnmZiFeOp1c5E7JzETac/f6vAzA641s6KI6YdN40A622dJWaeykZJp3DGEx1whuKH4m02DeEAOk+5zQVGZzxISgLzkvMzDFtdBYO+2KJ+K+aiEcO5ZWzFE5e7K00nLXq1jij4VuXr2kq5Q7sLIxfj117nB9ZAG2/2E2TadvzzDU0F59uPsP2InRbLgwP0vhR+QXlM+K4KpelX8Y4EvEq1GeYc7uwvIonQnmZiGeuaaNXPns+xslJwbjMIexUN8ZNymSx8gtqSWaoL5YHNx4prX3RdywyPqBrOGoJXFAxz2UOjRQx3JO96J8xlptocyPvp+YTuPqInNNG4bAvngjlZSJZcV5ZVCts2gg4OylQ4czwzHc5oqmJr5TKT8oZVCD/2KiiYq8BrQUhoeFzlwW0cTpBquzKoIwxxMiPK0HcTSm43HimcNiXCMR/1USUzgrdTeHt5kyZHPmLeeH8yAJIw5VBGaM5r6lAgHSfn5Qz8KMItfXRxpsNW36k0BQd+0fmf87LecFpP5myE29njiRSrqG3x93kgn15OXMoxbSxeCKUVxHweBSoWq2sp0FlVrLfRAl40S9jHNfkMlRX3OBH5Re4Y7p/au5f0WyOimfY8qN66x2t/MubThsZ4rwFgNGZb7JH0zDXPklpxk4bxde8OCL+q0UgK1C1wDCJbDjn8L3EU4Z+GeO4L3vRSBHDPJevcMa0KPqseDNH5mWnPYx3WQnAtMw+bNA8med+qRkFN+7NaXm5CJ9XsUQoryKQlSJkiLMe8rYAYuQKDMoYTbrkRlun43zushAJ48MPHNG3lZ32iqPMcP4egPmqbvygzr+ZS0ohyitnDTDhsC+eCOVVBHwetYc3xFkP4JKP7+WYXJOlFaegkhW86LSPCc4rMDaM3pFVVxPpPN+6fI2zpGGtujUzVK8WuH9qIaEiOaeNLsJhXywR/9Ui8H6nWvQPq0ynOoYVuSvIAjjr9QSjM7W9BYc6b+YNp41GyeKolldN6SqLlV/gLmWwU92YcZlDKKz4c2GWl3DYlwzsQnl9++23VKlSBTc3N1q2bMl///2X775LlixBkiS9xc3NzYrSPqZVjbJM6V4PNxfDOi4XWJpFgl81TzE9sw8AE1xW8pJir8Gy5Fvv3o6VWhB3+Ek5g1JSCkc0NRme+S4qAyqTPyykukbOaaOIsC+e2Py/+ssvvzBq1CgmT57M0aNHadiwIZ07d+bWrVv5HuPj48ONGzd0y+XLl60osenkLJL36Qv1c+2zSN2VBSqtv+czl4W0UxStXv/otScsGmVvKn4k8pNyBoHSPc5rtH6/NAyrL2+s5ZXfdF3g2NhceX355ZcMHTqUgQMHUqdOHebPn4+HhweLFy/O9xhJkggICNAtlu4XaS5yWgSlPV10r7OXu5mh6s069VM4Sxq+c/mKJtL5Ip33iYidXLmXWviOVsKdNBYrZ1JDcZ1rsvaJawKGPfQA431ezsLnVSyx6X81IyODI0eO0LFjR906hUJBx44dOXjwYL7HJScnU7lyZYKDg+nevTunTp3Kd9/09HQSExP1FluhzGEBuDrnPd2UUTA2cyh/qRvhLmWwWPkFNaSrRTr3mz+blghubpxR8Z3LVzRWRHNf1sa6xWNcWezCQiVylsTJGaIiKB7YVHnduXMHtVqdy3Ly9/cnPj4+z2NCQkJYvHgxv/32G8uWLUOj0RAeHs7Vq3nf3BEREfj6+uqW4OBgs38OQ8kZKuHq/Ph9zljSrDSio5oalJJStFMsTG9Ce/5R+y9bok37WUg7p+M8lJUMyhitl/ZjKAUpL1mWc1WqrVHecKtO4Dg4nD0dFhZGv379aNSoEW3atGH9+vWUK1eOBQsW5Ln/+PHjSUhI0C1XrlgmEdoQcloAri4FX/6HuDEoYzQXNBUIku5pndsUroRS7bRc9HjnlbzotA+VrODtzPc4Jtc0aZyCymFnn5p/368Zb7WpzpCnqpp0HoF9Y1PlVbZsWZycnLh5U7+sys2bNwkICDBoDBcXFxo3bkx0dHSe211dXfHx8dFbbEXOUIns08b8JjYP8KZfxjiuy6WpqbimDSsoJI2oziTTix1aiqFOG3nDeRMAYzLfYLemscljpabnb3ll93eFVy/DuC61852eCxwbmyovpVJJ06ZN2blzp26dRqNh586dhIWFGTSGWq0mKiqKwMBAS4lpNnJZXgVMG7Nz41Ea0QPZkyaKaG1Ap5FpRLZMcXxRsZf/uawA4JPM11ivaV3oMdULCPxNzcz/s/9z8fHUWsR3FW9sPm0cNWoU33//PUuXLuXMmTO8/fbbpKSkMHDgQAD69evH+PHjdftPnTqVP//8k4sXL3L06FFef/11Ll++zJAhQ2z1EQymIMurMKLligzKGM1DWUl7p0g+c/neqDQiWYbXvv+HaRtPG3xMfkzfeJrPtp41aN92imN87rIQgAWqrnyvfs6g43Z+0DbfbflZXvdSMhi45JDuvWh5VryxeSfOXr16cfv2bSZNmkR8fDyNGjVi69atOid+XFwcimxfwvv37zN06FDi4+Px8/OjadOmHDhwgDp16tjqIxhMTkvArRCfV06OyrUYlvke37vM4iWnv7kt+zJD9ZpBx6o0Mgdi7nIg5i7vd6qVq4x1QURsOcOec7dZPyyc5DQVi/bFGnRcE+k837l8hbOkYZ36SWaoehd6TMfQ8rzctGKB++Tl89JoZD5YHal7XzfIB4V4ylissbnyAhgxYgQjRozIc9vu3bv13s+ePZvZs2dbQSrzk/tpY3afl2E32i5NY8ZmvsEs5Xzect7IHdmXRQUkMedFUlqmUcprwR5txdS1R67SLqS8QcfUyJb2s0vdkLGZbyAbYOgv6t+80H3SMjWoNbKeJTt3VzS7zt3G1VnBiqFP0LCir0FyChwXYVdbkZw+L6WBPq+crNO0JiJTa8VMdFnOC4q/jZJjc1TeYSiFkbMfYn4EcleX9nNUU4Nhme8ZlPZjDA8zH08d95y/zewd2kDe6T3q0bSynwhMLQGI/7AVyW4p1KvgY/S0MTsL1M/xvepZAD53WUhbRaTBx5rq95IkicJSJX1J5iflDIKke0RrgrR+Osyfe5oVZX/1firvrTqGLEPvFpV4pZnt4vgE1kUoLyuS3fL6dVgrvSqn2S2vPi0rMeHZ2oWMJvGp6jXWq5/ERVJro9alC2aWWB+FBHIBxXfcSGex8gtqKq5xXS6tfUKKt8Hjv9rccMWTkqEmLVPNsOVHeZCaSYOKvkzuZv9+T4H5EMrLimS3vJxyzRMlnqim7Y7zdtvqvNG6Oq+1rFTgeDIKbcyUuiEeklZxVJeumVvsbCLmP7d1RqVt56a4wAPZk/4Z47hOWYOHntq9LjNeamDw/qkZKqb8cZoTVxMo5eHCd32aGFzdQ1A8EMrLihRUzyu8ehlWDHmCk1M6U9FP21Q1U1V4KIQKZ97OfI9ITXX8JO2ULaAIaUQFIUE+00aZGS6L6OB0TJf2c0HO+4nh1O51da/HPBOie+3r7pLX7vny04HLrPwvDkmCr15trLtmgpKDUF5WJLvyyjJiDoxrz8K+TXmuQSAKhaT3FDBnaZf8eIgbAzNGE6MJpIKkdZb7kmwWmdXZnPQKScpz0jjOeRUvO+1FJSu0+ZhyrXzH6xD6OI+1W4Mg3evsAbuG8MthbZrXyA61aFOrnFHHCooHQnlZkezliLMsmKBS7jxdNyDPLj85E4wL4j4+9M0Yzw25NLUU1/hBORM3itaUY/6eGOpO3qp7L0m5ixsOdtrEW85/ADBONZS/NE0KHDPQx41p3esyqlMtynk/rt9lSgpPu5ByvNO+RuE7CoolQnlZkerlvKgd4M0T1UobFBphqOWVxXXK0j9jLAmyB80U501KI8rOjC1nSct8LMPhS/dpP2uP7v0Lir/5yGU5ABGZvVmrblPomAqFRN+wKrzboabeNSgsST0nFf3cmd2rkQhELcEI5WVFnBQSm999ipVDnzCon6KrCQ7o83Kwtiqp7EIHp2NEOC/CXO051h19XHaorSJSl/bzvepZFhiY9pOd7IG5hjrbB4RXoWllPxb2bUYpj8Ib/QqKL0J5WRmFQjK4Eez4LrWpV8H4KhhH5BBtPXhZwSvOexnrvCrXPmmZao5cvm9w4Gl2GknRfOfyFS6SmvXqJ/lU9RqFNc3ICz3LKw+f17Qe9QDo2iCQr15txLvtazC5Wx3WvR1OnSDbVQcR2Ad2kR4kyJugUu5sfOcpqozbZPSxOzVNGa8awhcuC3nb+Q/uyL78oH5Wt732R1pf1sSuoQx5qlqu41PyKbVcXbrGj8rP8ZDS2a1uyJhC0n7Gd6nNjYQ0lhy4VKC8efm8+j5RmddbVnK4rt8C6yAsr2LMGnVbPsvU9kD8yGUZ3RX7cu2z4r+4PI9d9k/upiYBj9J+/KRkIjXVDUr76d6oAu1qa/MhS3noh0M4KyRa1ShDw+BSVMun67hQXIL8EJZXMWeeuhtlpQQGO29hpssCHmR6s0fTULddkYdyuJHwkIgt+iVvstJ+Kkh3idEEMjBjNKl5pP04KyS9aqYKCVrXLMu6t8OolqOzuCRJLBvcUvdaIDAGYXk5MGveCuO/CR0K2UtiuqoPG9ThuEhq5rnMoZH0uOpsXg/r3lmh327NjXR+UM6kluIaNx6l/dwnb5/T0Nb6U1APV2ckSaJp5dL4eeZ2sGf13hQIjEUoLwfizTbVaFDRlx/6N+PSjK40r1Ka8j5uHP2oU4HHySgYnfkWe9X1H6URfa5LI8qrFM+Jqwm6106omevyNc0U50mQPeifMZZr5B8U6pJNGy7q18yo0jsCgTEI5eUAdK0fSGlPJSPa1eD3EU/qRakDlM7DoslJJs68lfk+kZpqlJaSWar8DH/ucS6PrkIZuvgymQjnRXR0Okaa7MLgjA85L+efPP1Wm+o6y6tKGQ861nGMfpoCx0T8LDoAc19rjFojF7lGVSpuDMoYwxrlFKorbrBU+Rk9Mz4iPiENdxcnfD1cuJX4uLnHGOdf6Om8R5f2c1jOv9LFpRmPCyKentpZL5tAILAE4hvmAEiSZJTi2jryKTa/+1Se2+7hQ//MccTLftRWXOEH5UzaRGyh4dQ/uZHwkORHIRKDnLYwzPl3AMarhrBT0zTf83WorV9d1UPpLJSXwOLYxTfs22+/pUqVKri5udGyZUv++++/Avdfs2YNtWvXxs3Njfr167N582YrSWq/vJHNUV47wKfAIM6rcjldGlFzxXnmunyNE2p2nL7J/ug7dFfsY5LLzwB8lvkqa9RtCzz3DwMKL90sEJgbmyuvX375hVGjRjF58mSOHj1Kw4YN6dy5M7du3cpz/wMHDtC7d28GDx7MsWPH6NGjBz169ODkyZNWlty+GN62Bo2CSzHl+cclZz55oR7tQsrRMTR33flzciWGZHxImuxCJ6ejfOr8Ax/9dpLtf6xgpou2ge9i1TPMU3cr8LyzXmlY4HaBwFJIcs4yAVamZcuWNG/enLlz5wLavo3BwcG88847jBs3Ltf+vXr1IiUlhY0bN+rWPfHEEzRq1Ij58+cXer7ExER8fX1JSEiwaQNaa/MgNYNGU7fnWt9RcYQFLl/iJMn8ra5HE8UFPKV0flOHMzJzWIHR8zXKe7FjVOHJ2AKBMRh6j9rU8srIyODIkSN07NhRt06hUNCxY0cOHjyY5zEHDx7U2x+gc+fO+e6fnp5OYmKi3lISKeWh5O8x7fh7TDuWDW5Jj0baWlo7HqURATzldBJPKZ296vp8mPlWgYqrZ7OKQnEJbIpNnzbeuXMHtVqt69GYhb+/P2fP5t3UND4+Ps/94+Pz7ogTERHBlClTzCOwgxNc2kP3t1kVPzqE+vPOymOsVrejLImMcfmFSE013sp8n0ycaVG1NM83DMLX3YW2IeXwdjOu2qlAYEmKfajE+PHjGTVqlO59YmIiwcGiw4ybixPdGgbRraHWApPlZzl9ehhlylflVFkfEfUusHtsqrzKli2Lk5MTN2/e1Ft/8+ZNAgIC8jwmICDAqP1dXV1xdXXNc5vgMZIkUaduI1uLIRAYjE19XkqlkqZNm7Jz507dOo1Gw86dOwkLC8vzmLCwML39AbZv357v/gKBoHhi82njqFGj6N+/P82aNaNFixbMmTOHlJQUBg4cCEC/fv2oUKECERERALz33nu0adOGWbNm0bVrV1atWsXhw4dZuHChLT+GQCCwMjZXXr169eL27dtMmjSJ+Ph4GjVqxNatW3VO+bi4OBSKxwZieHg4K1asYOLEiUyYMIGaNWuyYcMG6tWrZ6uPIBAIbIDN47ysTUmN8xIIHAWHiPMSCAQCUxHKSyAQOCRCeQkEAofE5g57a5Pl4iupaUICgb2TdW8W5o4vccorKUlbOVRE2QsE9k1SUhK+vr75bi9xTxs1Gg3Xr1/H29tbpMDkICt16sqVK+JJbA7Etckfc18bWZZJSkoiKChIL0wqJyXO8lIoFFSsWNHWYtg1Pj4+4gbNB3Ft8sec16YgiysL4bAXCAQOiVBeAoHAIRHKS6DD1dWVyZMniyoceSCuTf7Y6tqUOIe9QCAoHgjLSyAQOCRCeQkEAodEKC+BQOCQCOUlEAgcEqG8ShiiO3n+GHNtvv/+e5566in8/Pzw8/OjY8eOhV5LR8bY700Wq1atQpIkevToYX6hZEGJYdWqVbJSqZQXL14snzp1Sh46dKhcqlQp+ebNm3nuv3//ftnJyUn+/PPP5dOnT8sTJ06UXVxc5KioKCtLbnmMvTavvfaa/O2338rHjh2Tz5w5Iw8YMED29fWVr169amXJLY+x1yaL2NhYuUKFCvJTTz0ld+/e3exyCeVVgmjRooU8fPhw3Xu1Wi0HBQXJERERee7fs2dPuWvXrnrrWrZsKb/55psWldMWGHttcqJSqWRvb2956dKllhLRZphybVQqlRweHi4vWrRI7t+/v0WUl5g2lhCs0Z3cUTHl2uQkNTWVzMxMSpcubSkxbYKp12bq1KmUL1+ewYMHW0y2EpeYXVKxRndyR8WUa5OTsWPHEhQUlEvZOzqmXJt9+/bxww8/EBkZaVHZhPISCIrIjBkzWLVqFbt378bNzc3W4tiUpKQk+vbty/fff0/ZsmUtei6hvEoI1uhO7qiYcm2ymDlzJjNmzGDHjh00aNDAkmLaBGOvTUxMDJcuXaJbt266dRqNBgBnZ2fOnTtH9erVzSKb8HmVEER38vwx5doAfP7550ybNo2tW7fSrFkza4hqdYy9NrVr1yYqKorIyEjd8vzzz9OuXTsiIyPNW8HY7I8ABHbLqlWrZFdXV3nJkiXy6dOn5TfeeEMuVaqUHB8fL8uyLPft21ceN26cbv/9+/fLzs7O8syZM+UzZ87IkydPLtahEsZcmxkzZshKpVJeu3atfOPGDd2SlJRkq49gMYy9Njmx1NNGobxKGN98841cqVIlWalUyi1atJD/+ecf3bY2bdrI/fv319t/9erVcq1atWSlUinXrVtX3rRpk5Ulth7GXJvKlSvLQK5l8uTJ1hfcChj7vcmOpZSXKIkjEAgcEuHzEggEDolQXgKBwCERyksgEDgkQnkJBAKHRCgvgUDgkAjlJRAIHBKhvAQCgUMilJdAIHBIhPIS2AUDBgxAkqRcyzPPPGNr0QR2iqgqIbAbnnnmGX788Ue9dfl1Yc7MzMTFxUVvXUZGBkql0ujzmnqcwLYIy0tgN7i6uhIQEKC3+Pn5ASBJEvPmzeP555/H09OTTz75hI8//phGjRqxaNEiqlatqqulFRcXR/fu3fHy8sLHx4eePXvqlXTJ7ziBYyGUl8Bh+Pjjj3nhhReIiopi0KBBAERHR7Nu3TrWr19PZGQkGo2G7t27c+/ePfbs2cP27du5ePEivXr10hsr53ECx0NMGwV2w8aNG/Hy8tJbN2HCBCZMmADAa6+9xsCBA/W2Z2Rk8NNPP1GuXDlAW28sKiqK2NhYXe2on376ibp163Lo0CGaN2+e53ECx0MoL4Hd0K5dO+bNm6e3LntDi7wK/lWuXFlPAZ05c4bg4GC9ond16tShVKlSnDlzRqe8ch4ncDyE8hLYDZ6entSoUaPA7YasM/RcAsdG+LwExYrQ0FCuXLnClStXdOtOnz7NgwcPqFOnjg0lE5gbYXkJ7Ib09PRcbdWcnZ2N6kLTsWNH6tevT58+fZgzZw4qlYphw4bRpk2bYltnvqQiLC+B3bB161YCAwP1lieffNKoMSRJ4rfffsPPz4/WrVvTsWNHqlWrxi+//GIhqQW2QpSBFggEDomwvAQCgUMilJdAIHBIhPISCAQOiVBeAoHAIRHKSyAQOCRCeQkEAodEKC+BQOCQCOUlEAgcEqG8BAKBQyKUl0AgcEiE8hIIBA6JUF4CgcAh+T/KxfwyZPqHaQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019751, + "end_time": "2024-03-22T14:09:16.382119", + "exception": false, + "start_time": "2024-03-22T14:09:16.362368", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 5617.022719, + "end_time": "2024-03-22T14:09:19.125511", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/realtabformer/42/mlu-eval.ipynb", + "output_path": "eval/contraceptive/realtabformer/42/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/realtabformer/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "realtabformer" + }, + "start_time": "2024-03-22T12:35:42.102792", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/realtabformer/model.pt b/contraceptive/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1fc128a5aa7beb569a8baa83aabbd47ecfa2676 --- /dev/null +++ b/contraceptive/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fedab6ff24590104807dd04fe22f394cb4a5e57d14949b65f1a9438d41d21a1 +size 50388737 diff --git a/contraceptive/realtabformer/params.json b/contraceptive/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..117e8887b5aabab5832bcd3912f0c40c981d4217 --- /dev/null +++ b/contraceptive/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.73, "gradient_penalty_mode": "NONE", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "head_activation": "softsign", "loss_balancer_beta": 0.67, "loss_balancer_r": 0.943, "tf_activation": "tanh", "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.09, "n_warmup_steps": 100, "Optim": "amsgradw", "fixed_role_model": "realtabformer", "mse_mag": false, "mse_mag_target": 0.65, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation_final": "leakyhardtanh", "tf_num_inds": 128, "ada_d_hid": 1024, "ada_n_layers": 9, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation_final": "leakyhardsigmoid", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/tab_ddpm_concat/eval.csv b/contraceptive/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..e5d1d57e97bce77eab5b95326d1cb066ee74d83b --- /dev/null +++ b/contraceptive/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,0.009295055350909631,0.044223827542087174,0.0025627935946630756,12.520719766616821,0.03127303719520569,0.8461189270019531,0.043997641652822495,6.7261853473610245e-06,3.9416027069091797,0.037461522966623306,0.11356883496046066,0.05062403902411461,0.10244974493980408,0.00610949145630002,16.462322473526 diff --git a/contraceptive/tab_ddpm_concat/history.csv b/contraceptive/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..71ed585e4c3562c59ae8db4a75f2d660c727568f --- /dev/null +++ b/contraceptive/tab_ddpm_concat/history.csv @@ -0,0 +1,12 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.02347481836991695,0.5124517322441693,0.002339297849103639,0.0,0.0,0.0,0.0,0.0,0.02347481836991695,900,225,223.80061292648315,0.9946693907843696,0.2486673476960924,0.13361252259876993,0.008576864742596323,0.43590039322429097,0.0002531669465378804,0.0,0.0,0.0,0.0,0.0,0.008576864742596323,450,113,98.30213046073914,0.8699303580596384,0.21844917880164252,0.06112736243490888 +1,0.008378146543917763,0.5896725166847361,0.00015907265293133425,0.0,0.0,0.0,0.0,0.0,0.008378146543917763,900,225,223.44736337661743,0.9930993927849664,0.2482748481962416,0.10096969366073609,0.004039558287994522,1.381628237529861,3.3886849553694544e-05,0.0,0.0,0.0,0.0,0.0,0.004039558287994522,450,113,98.5942611694336,0.8725155855702088,0.21909835815429687,0.052547984426857625 +2,0.010694745298888949,0.8260207763179106,0.00029481776721037526,0.0,0.0,0.0,0.0,0.0,0.010694745298888949,900,225,222.78780055046082,0.9901680024464925,0.24754200061162313,0.10044084888158573,0.006571635709972017,0.6343476063295598,7.651879237107392e-05,0.0,0.0,0.0,0.0,0.0,0.006571635709972017,450,113,98.61985659599304,0.8727420937698499,0.21915523687998453,0.0684763636147455 +3,0.007882197938549022,0.5442602433114256,0.00010187150954560703,0.0,0.0,0.0,0.0,0.0,0.007882197938549022,900,225,222.28374481201172,0.9879277547200521,0.24698193868001303,0.10185433081868622,0.010345627184336384,0.40530077661136166,0.00020547013780565572,0.0,0.0,0.0,0.0,0.0,0.010345627184336384,450,113,97.98167514801025,0.8670944703363739,0.21773705588446723,0.0684052016874528 +4,0.004765987225756463,0.33747415092830846,4.3614522179710367e-05,0.0,0.0,0.0,0.0,0.0,0.004765987225756463,900,225,222.1056261062622,0.987136116027832,0.246784029006958,0.10019376738617818,0.004264881536364555,1.3016880564829196,2.9635327498389935e-05,0.0,0.0,0.0,0.0,0.0,0.004264881536364555,450,113,99.50796246528625,0.880601437745896,0.2211288054784139,0.06006413273538042 +5,0.004931693592548577,0.49496390551068475,4.218088193112541e-05,0.0,0.0,0.0,0.0,0.0,0.004931693592548577,900,225,224.20910120010376,0.9964848942226834,0.24912122355567085,0.0999689629011684,0.004869184112176299,3.0701807380044164,3.4871858414750014e-05,0.0,0.0,0.0,0.0,0.0,0.004869184112176299,450,113,98.850257396698,0.8747810389088319,0.21966723865932888,0.057760195345082116 +6,0.004765996111996679,0.4421812962271803,3.558628672912029e-05,0.0,0.0,0.0,0.0,0.0,0.004765996111996679,900,225,223.28885769844055,0.9923949231041802,0.24809873077604505,0.10080746096455388,0.004297750624181289,2.2529975814067793,3.2225562565454145e-05,0.0,0.0,0.0,0.0,0.0,0.004297750624181289,450,113,98.77103543281555,0.8740799595824386,0.21949118985070123,0.05614519555768641 +7,0.0035699382384255943,0.3529214430276707,1.8523309853427525e-05,0.0,0.0,0.0,0.0,0.0,0.0035699382384255943,900,225,222.1509132385254,0.987337392171224,0.246834348042806,0.10173722859472037,0.0035803620515636996,2.1854055460992443,2.0855983310910675e-05,0.0,0.0,0.0,0.0,0.0,0.0035803620515636996,450,113,98.14665222167969,0.8685544444396432,0.21810367160373265,0.05454383128058923 +8,0.0035860408845150636,0.2396017117707293,2.2730216006070548e-05,0.0,0.0,0.0,0.0,0.0,0.0035860408845150636,900,225,221.9519350528717,0.9864530446794298,0.24661326116985746,0.10408703482813306,0.0036512146290624513,1.5390193092272952,3.486896375240715e-05,0.0,0.0,0.0,0.0,0.0,0.0036512146290624513,450,113,98.24018931388855,0.8693822063175978,0.2183115318086412,0.0619035135116016 +9,0.0029799208378406346,0.2575948480180224,1.6468858607504945e-05,0.0,0.0,0.0,0.0,0.0,0.0029799208378406346,900,225,222.18345999717712,0.9874820444318984,0.2468705111079746,0.10268474409770634,0.00386869330644711,2.897202919005004,3.095258879357379e-05,0.0,0.0,0.0,0.0,0.0,0.00386869330644711,450,113,98.27193331718445,0.8696631267007473,0.21838207403818766,0.06031067110495065 +10,0.002677944270108128,0.2724713624029512,1.7092457439067723e-05,0.0,0.0,0.0,0.0,0.0,0.002677944270108128,900,225,222.04818081855774,0.9868808036380344,0.2467202009095086,0.10342358542606235,0.002709867898762847,3.8843037776192246,8.260416360805687e-06,0.0,0.0,0.0,0.0,0.0,0.002709867898762847,450,113,98.23442721366882,0.8693312142802551,0.21829872714148627,0.045337541868489865 diff --git a/contraceptive/tab_ddpm_concat/mlu-eval.ipynb b/contraceptive/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..6cdc1d76cb7cc893a7a53b59ee78d3e12ae78b33 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2291 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:21.629336Z", + "iopub.status.busy": "2024-03-23T05:04:21.628994Z", + "iopub.status.idle": "2024-03-23T05:04:21.663130Z", + "shell.execute_reply": "2024-03-23T05:04:21.662384Z" + }, + "papermill": { + "duration": 0.049429, + "end_time": "2024-03-23T05:04:21.665394", + "exception": false, + "start_time": "2024-03-23T05:04:21.615965", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:21.691608Z", + "iopub.status.busy": "2024-03-23T05:04:21.691206Z", + "iopub.status.idle": "2024-03-23T05:04:21.697898Z", + "shell.execute_reply": "2024-03-23T05:04:21.697089Z" + }, + "papermill": { + "duration": 0.022113, + "end_time": "2024-03-23T05:04:21.699856", + "exception": false, + "start_time": "2024-03-23T05:04:21.677743", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:21.723534Z", + "iopub.status.busy": "2024-03-23T05:04:21.723253Z", + "iopub.status.idle": "2024-03-23T05:04:21.727276Z", + "shell.execute_reply": "2024-03-23T05:04:21.726484Z" + }, + "papermill": { + "duration": 0.018099, + "end_time": "2024-03-23T05:04:21.729236", + "exception": false, + "start_time": "2024-03-23T05:04:21.711137", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:21.753343Z", + "iopub.status.busy": "2024-03-23T05:04:21.753062Z", + "iopub.status.idle": "2024-03-23T05:04:21.757108Z", + "shell.execute_reply": "2024-03-23T05:04:21.756385Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018266, + "end_time": "2024-03-23T05:04:21.758957", + "exception": false, + "start_time": "2024-03-23T05:04:21.740691", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:21.783476Z", + "iopub.status.busy": "2024-03-23T05:04:21.783146Z", + "iopub.status.idle": "2024-03-23T05:04:21.788977Z", + "shell.execute_reply": "2024-03-23T05:04:21.788132Z" + }, + "papermill": { + "duration": 0.020768, + "end_time": "2024-03-23T05:04:21.790943", + "exception": false, + "start_time": "2024-03-23T05:04:21.770175", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "813921ec", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:21.816694Z", + "iopub.status.busy": "2024-03-23T05:04:21.816369Z", + "iopub.status.idle": "2024-03-23T05:04:21.821371Z", + "shell.execute_reply": "2024-03-23T05:04:21.820530Z" + }, + "papermill": { + "duration": 0.019782, + "end_time": "2024-03-23T05:04:21.823201", + "exception": false, + "start_time": "2024-03-23T05:04:21.803419", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 3\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tab_ddpm_concat/3\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011191, + "end_time": "2024-03-23T05:04:21.845732", + "exception": false, + "start_time": "2024-03-23T05:04:21.834541", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:21.870203Z", + "iopub.status.busy": "2024-03-23T05:04:21.869483Z", + "iopub.status.idle": "2024-03-23T05:04:21.879301Z", + "shell.execute_reply": "2024-03-23T05:04:21.878432Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024218, + "end_time": "2024-03-23T05:04:21.881253", + "exception": false, + "start_time": "2024-03-23T05:04:21.857035", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tab_ddpm_concat/3\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:21.905979Z", + "iopub.status.busy": "2024-03-23T05:04:21.905257Z", + "iopub.status.idle": "2024-03-23T05:04:23.968980Z", + "shell.execute_reply": "2024-03-23T05:04:23.967963Z" + }, + "papermill": { + "duration": 2.078671, + "end_time": "2024-03-23T05:04:23.971481", + "exception": false, + "start_time": "2024-03-23T05:04:21.892810", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:24.001446Z", + "iopub.status.busy": "2024-03-23T05:04:24.000945Z", + "iopub.status.idle": "2024-03-23T05:04:24.013907Z", + "shell.execute_reply": "2024-03-23T05:04:24.013083Z" + }, + "papermill": { + "duration": 0.02948, + "end_time": "2024-03-23T05:04:24.015902", + "exception": false, + "start_time": "2024-03-23T05:04:23.986422", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:24.040475Z", + "iopub.status.busy": "2024-03-23T05:04:24.040170Z", + "iopub.status.idle": "2024-03-23T05:04:24.047825Z", + "shell.execute_reply": "2024-03-23T05:04:24.046905Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.022157, + "end_time": "2024-03-23T05:04:24.049871", + "exception": false, + "start_time": "2024-03-23T05:04:24.027714", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:24.075125Z", + "iopub.status.busy": "2024-03-23T05:04:24.074801Z", + "iopub.status.idle": "2024-03-23T05:04:24.175336Z", + "shell.execute_reply": "2024-03-23T05:04:24.174308Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.116307, + "end_time": "2024-03-23T05:04:24.178232", + "exception": false, + "start_time": "2024-03-23T05:04:24.061925", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:24.208789Z", + "iopub.status.busy": "2024-03-23T05:04:24.208119Z", + "iopub.status.idle": "2024-03-23T05:04:29.151684Z", + "shell.execute_reply": "2024-03-23T05:04:29.150603Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.962449, + "end_time": "2024-03-23T05:04:29.154200", + "exception": false, + "start_time": "2024-03-23T05:04:24.191751", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-23 05:04:26.617688: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-23 05:04:26.617749: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-23 05:04:26.619528: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:29.181423Z", + "iopub.status.busy": "2024-03-23T05:04:29.180284Z", + "iopub.status.idle": "2024-03-23T05:04:29.186445Z", + "shell.execute_reply": "2024-03-23T05:04:29.185689Z" + }, + "papermill": { + "duration": 0.021155, + "end_time": "2024-03-23T05:04:29.188348", + "exception": false, + "start_time": "2024-03-23T05:04:29.167193", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:29.215147Z", + "iopub.status.busy": "2024-03-23T05:04:29.214438Z", + "iopub.status.idle": "2024-03-23T05:04:38.508662Z", + "shell.execute_reply": "2024-03-23T05:04:38.507453Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 9.310896, + "end_time": "2024-03-23T05:04:38.511484", + "exception": false, + "start_time": "2024-03-23T05:04:29.200588", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-23T05:04:38.541045Z", + "iopub.status.busy": "2024-03-23T05:04:38.540681Z", + "iopub.status.idle": "2024-03-23T05:04:38.547632Z", + "shell.execute_reply": "2024-03-23T05:04:38.546771Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.024359, + "end_time": "2024-03-23T05:04:38.550023", + "exception": false, + "start_time": "2024-03-23T05:04:38.525664", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:38.576002Z", + "iopub.status.busy": "2024-03-23T05:04:38.575642Z", + "iopub.status.idle": "2024-03-23T05:04:38.580702Z", + "shell.execute_reply": "2024-03-23T05:04:38.579793Z" + }, + "papermill": { + "duration": 0.020761, + "end_time": "2024-03-23T05:04:38.582699", + "exception": false, + "start_time": "2024-03-23T05:04:38.561938", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:38.609782Z", + "iopub.status.busy": "2024-03-23T05:04:38.608931Z", + "iopub.status.idle": "2024-03-23T05:04:39.112505Z", + "shell.execute_reply": "2024-03-23T05:04:39.111467Z" + }, + "papermill": { + "duration": 0.519352, + "end_time": "2024-03-23T05:04:39.114672", + "exception": false, + "start_time": "2024-03-23T05:04:38.595320", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:39.144548Z", + "iopub.status.busy": "2024-03-23T05:04:39.143559Z", + "iopub.status.idle": "2024-03-23T05:04:39.480832Z", + "shell.execute_reply": "2024-03-23T05:04:39.479905Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.354487, + "end_time": "2024-03-23T05:04:39.483055", + "exception": false, + "start_time": "2024-03-23T05:04:39.128568", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.73,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.Softsign,\n", + " 'loss_balancer_beta': 0.67,\n", + " 'loss_balancer_r': 0.943,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.65, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:39.512547Z", + "iopub.status.busy": "2024-03-23T05:04:39.512183Z", + "iopub.status.idle": "2024-03-23T05:04:39.627133Z", + "shell.execute_reply": "2024-03-23T05:04:39.626206Z" + }, + "papermill": { + "duration": 0.132353, + "end_time": "2024-03-23T05:04:39.629484", + "exception": false, + "start_time": "2024-03-23T05:04:39.497131", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_train/tab_ddpm_concat/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/contraceptive [400, 0]\n", + "Caching in ../../../../contraceptive/_cache_aug_val/tab_ddpm_concat/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/contraceptive [0, 200]\n", + "Caching in ../../../../contraceptive/_cache_bs_train/tab_ddpm_concat/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/contraceptive [100, 0]\n", + "Caching in ../../../../contraceptive/_cache_bs_val/tab_ddpm_concat/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/contraceptive [0, 50]\n", + "Caching in ../../../../contraceptive/_cache_synth/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/contraceptive [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-23T05:04:39.660099Z", + "iopub.status.busy": "2024-03-23T05:04:39.659223Z", + "iopub.status.idle": "2024-03-23T05:04:40.129666Z", + "shell.execute_reply": "2024-03-23T05:04:40.128573Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.487633, + "end_time": "2024-03-23T05:04:40.131872", + "exception": false, + "start_time": "2024-03-23T05:04:39.644239", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:40.162151Z", + "iopub.status.busy": "2024-03-23T05:04:40.161734Z", + "iopub.status.idle": "2024-03-23T05:04:40.166284Z", + "shell.execute_reply": "2024-03-23T05:04:40.165331Z" + }, + "papermill": { + "duration": 0.022852, + "end_time": "2024-03-23T05:04:40.168893", + "exception": false, + "start_time": "2024-03-23T05:04:40.146041", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:40.196772Z", + "iopub.status.busy": "2024-03-23T05:04:40.196158Z", + "iopub.status.idle": "2024-03-23T05:04:40.203527Z", + "shell.execute_reply": "2024-03-23T05:04:40.202609Z" + }, + "papermill": { + "duration": 0.023357, + "end_time": "2024-03-23T05:04:40.205581", + "exception": false, + "start_time": "2024-03-23T05:04:40.182224", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "11858440" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:40.234725Z", + "iopub.status.busy": "2024-03-23T05:04:40.233809Z", + "iopub.status.idle": "2024-03-23T05:04:40.315887Z", + "shell.execute_reply": "2024-03-23T05:04:40.314947Z" + }, + "papermill": { + "duration": 0.099156, + "end_time": "2024-03-23T05:04:40.318529", + "exception": false, + "start_time": "2024-03-23T05:04:40.219373", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 10] --\n", + "├─Adapter: 1-1 [2, 1179, 10] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 11,264\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 10] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 16, 256] 1\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 1,048,832\n", + "│ │ │ └─Softsign: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 256] --\n", + "│ │ │ └─Linear: 4-55 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-56 [2, 256] --\n", + "│ │ └─FeedForward: 3-28 [2, 256] --\n", + "│ │ │ └─Linear: 4-57 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-58 [2, 256] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 11,858,440\n", + "Trainable params: 11,858,440\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 44.00\n", + "========================================================================================================================\n", + "Input size (MB): 0.12\n", + "Forward/backward pass size (MB): 375.40\n", + "Params size (MB): 47.43\n", + "Estimated Total Size (MB): 422.95\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:04:40.350454Z", + "iopub.status.busy": "2024-03-23T05:04:40.350086Z", + "iopub.status.idle": "2024-03-23T06:11:47.496586Z", + "shell.execute_reply": "2024-03-23T06:11:47.495631Z" + }, + "papermill": { + "duration": 4027.180957, + "end_time": "2024-03-23T06:11:47.514491", + "exception": false, + "start_time": "2024-03-23T05:04:40.333534", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.02347481836991695, 'avg_role_model_std_loss': 0.5124517322441693, 'avg_role_model_mean_pred_loss': 0.002339297849103639, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.02347481836991695, 'n_size': 900, 'n_batch': 225, 'duration': 223.80061292648315, 'duration_batch': 0.9946693907843696, 'duration_size': 0.2486673476960924, 'avg_pred_std': 0.13361252259876993}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008576864742596323, 'avg_role_model_std_loss': 0.43590039322429097, 'avg_role_model_mean_pred_loss': 0.0002531669465378804, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008576864742596323, 'n_size': 450, 'n_batch': 113, 'duration': 98.30213046073914, 'duration_batch': 0.8699303580596384, 'duration_size': 0.21844917880164252, 'avg_pred_std': 0.06112736243490888}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.008378146543917763, 'avg_role_model_std_loss': 0.5896725166847361, 'avg_role_model_mean_pred_loss': 0.00015907265293133425, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008378146543917763, 'n_size': 900, 'n_batch': 225, 'duration': 223.44736337661743, 'duration_batch': 0.9930993927849664, 'duration_size': 0.2482748481962416, 'avg_pred_std': 0.10096969366073609}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004039558287994522, 'avg_role_model_std_loss': 1.381628237529861, 'avg_role_model_mean_pred_loss': 3.3886849553694544e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004039558287994522, 'n_size': 450, 'n_batch': 113, 'duration': 98.5942611694336, 'duration_batch': 0.8725155855702088, 'duration_size': 0.21909835815429687, 'avg_pred_std': 0.052547984426857625}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.010694745298888949, 'avg_role_model_std_loss': 0.8260207763179106, 'avg_role_model_mean_pred_loss': 0.00029481776721037526, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010694745298888949, 'n_size': 900, 'n_batch': 225, 'duration': 222.78780055046082, 'duration_batch': 0.9901680024464925, 'duration_size': 0.24754200061162313, 'avg_pred_std': 0.10044084888158573}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006571635709972017, 'avg_role_model_std_loss': 0.6343476063295598, 'avg_role_model_mean_pred_loss': 7.651879237107392e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006571635709972017, 'n_size': 450, 'n_batch': 113, 'duration': 98.61985659599304, 'duration_batch': 0.8727420937698499, 'duration_size': 0.21915523687998453, 'avg_pred_std': 0.0684763636147455}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007882197938549022, 'avg_role_model_std_loss': 0.5442602433114256, 'avg_role_model_mean_pred_loss': 0.00010187150954560703, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007882197938549022, 'n_size': 900, 'n_batch': 225, 'duration': 222.28374481201172, 'duration_batch': 0.9879277547200521, 'duration_size': 0.24698193868001303, 'avg_pred_std': 0.10185433081868622}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010345627184336384, 'avg_role_model_std_loss': 0.40530077661136166, 'avg_role_model_mean_pred_loss': 0.00020547013780565572, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010345627184336384, 'n_size': 450, 'n_batch': 113, 'duration': 97.98167514801025, 'duration_batch': 0.8670944703363739, 'duration_size': 0.21773705588446723, 'avg_pred_std': 0.0684052016874528}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004765987225756463, 'avg_role_model_std_loss': 0.33747415092830846, 'avg_role_model_mean_pred_loss': 4.3614522179710367e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004765987225756463, 'n_size': 900, 'n_batch': 225, 'duration': 222.1056261062622, 'duration_batch': 0.987136116027832, 'duration_size': 0.246784029006958, 'avg_pred_std': 0.10019376738617818}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004264881536364555, 'avg_role_model_std_loss': 1.3016880564829196, 'avg_role_model_mean_pred_loss': 2.9635327498389935e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004264881536364555, 'n_size': 450, 'n_batch': 113, 'duration': 99.50796246528625, 'duration_batch': 0.880601437745896, 'duration_size': 0.2211288054784139, 'avg_pred_std': 0.06006413273538042}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004931693592548577, 'avg_role_model_std_loss': 0.49496390551068475, 'avg_role_model_mean_pred_loss': 4.218088193112541e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004931693592548577, 'n_size': 900, 'n_batch': 225, 'duration': 224.20910120010376, 'duration_batch': 0.9964848942226834, 'duration_size': 0.24912122355567085, 'avg_pred_std': 0.0999689629011684}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004869184112176299, 'avg_role_model_std_loss': 3.0701807380044164, 'avg_role_model_mean_pred_loss': 3.4871858414750014e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004869184112176299, 'n_size': 450, 'n_batch': 113, 'duration': 98.850257396698, 'duration_batch': 0.8747810389088319, 'duration_size': 0.21966723865932888, 'avg_pred_std': 0.057760195345082116}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004765996111996679, 'avg_role_model_std_loss': 0.4421812962271803, 'avg_role_model_mean_pred_loss': 3.558628672912029e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004765996111996679, 'n_size': 900, 'n_batch': 225, 'duration': 223.28885769844055, 'duration_batch': 0.9923949231041802, 'duration_size': 0.24809873077604505, 'avg_pred_std': 0.10080746096455388}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004297750624181289, 'avg_role_model_std_loss': 2.2529975814067793, 'avg_role_model_mean_pred_loss': 3.2225562565454145e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004297750624181289, 'n_size': 450, 'n_batch': 113, 'duration': 98.77103543281555, 'duration_batch': 0.8740799595824386, 'duration_size': 0.21949118985070123, 'avg_pred_std': 0.05614519555768641}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0035699382384255943, 'avg_role_model_std_loss': 0.3529214430276707, 'avg_role_model_mean_pred_loss': 1.8523309853427525e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0035699382384255943, 'n_size': 900, 'n_batch': 225, 'duration': 222.1509132385254, 'duration_batch': 0.987337392171224, 'duration_size': 0.246834348042806, 'avg_pred_std': 0.10173722859472037}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0035803620515636996, 'avg_role_model_std_loss': 2.1854055460992443, 'avg_role_model_mean_pred_loss': 2.0855983310910675e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0035803620515636996, 'n_size': 450, 'n_batch': 113, 'duration': 98.14665222167969, 'duration_batch': 0.8685544444396432, 'duration_size': 0.21810367160373265, 'avg_pred_std': 0.05454383128058923}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0035860408845150636, 'avg_role_model_std_loss': 0.2396017117707293, 'avg_role_model_mean_pred_loss': 2.2730216006070548e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0035860408845150636, 'n_size': 900, 'n_batch': 225, 'duration': 221.9519350528717, 'duration_batch': 0.9864530446794298, 'duration_size': 0.24661326116985746, 'avg_pred_std': 0.10408703482813306}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0036512146290624513, 'avg_role_model_std_loss': 1.5390193092272952, 'avg_role_model_mean_pred_loss': 3.486896375240715e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0036512146290624513, 'n_size': 450, 'n_batch': 113, 'duration': 98.24018931388855, 'duration_batch': 0.8693822063175978, 'duration_size': 0.2183115318086412, 'avg_pred_std': 0.0619035135116016}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0029799208378406346, 'avg_role_model_std_loss': 0.2575948480180224, 'avg_role_model_mean_pred_loss': 1.6468858607504945e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029799208378406346, 'n_size': 900, 'n_batch': 225, 'duration': 222.18345999717712, 'duration_batch': 0.9874820444318984, 'duration_size': 0.2468705111079746, 'avg_pred_std': 0.10268474409770634}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00386869330644711, 'avg_role_model_std_loss': 2.897202919005004, 'avg_role_model_mean_pred_loss': 3.095258879357379e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00386869330644711, 'n_size': 450, 'n_batch': 113, 'duration': 98.27193331718445, 'duration_batch': 0.8696631267007473, 'duration_size': 0.21838207403818766, 'avg_pred_std': 0.06031067110495065}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002677944270108128, 'avg_role_model_std_loss': 0.2724713624029512, 'avg_role_model_mean_pred_loss': 1.7092457439067723e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002677944270108128, 'n_size': 900, 'n_batch': 225, 'duration': 222.04818081855774, 'duration_batch': 0.9868808036380344, 'duration_size': 0.2467202009095086, 'avg_pred_std': 0.10342358542606235}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002709867898762847, 'avg_role_model_std_loss': 3.8843037776192246, 'avg_role_model_mean_pred_loss': 8.260416360805687e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002709867898762847, 'n_size': 450, 'n_batch': 113, 'duration': 98.23442721366882, 'duration_batch': 0.8693312142802551, 'duration_size': 0.21829872714148627, 'avg_pred_std': 0.045337541868489865}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0024496534266250417, 'avg_role_model_std_loss': 0.241057577306981, 'avg_role_model_mean_pred_loss': 9.327678098924875e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024496534266250417, 'n_size': 900, 'n_batch': 225, 'duration': 221.71574759483337, 'duration_batch': 0.9854033226437039, 'duration_size': 0.24635083066092597, 'avg_pred_std': 0.10465546546503901}\n", + "Time out: 3768.357307434082/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.002562793581552493, 'avg_g_mag_loss': 0.04073966250407536, 'avg_g_cos_loss': 0.008872987412878067, 'pred_duration': 3.948948383331299, 'grad_duration': 12.55395245552063, 'total_duration': 16.50290083885193, 'pred_std': 0.10244974493980408, 'std_loss': 0.00610949145630002, 'mean_pred_loss': 6.7261853473610245e-06, 'pred_rmse': 0.05062403902411461, 'pred_mae': 0.03746151924133301, 'pred_mape': 0.11356884986162186, 'grad_rmse': 0.04399764537811279, 'grad_mae': 0.03127303719520569, 'grad_mape': 0.8461189866065979}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.002562793581552493, 'avg_g_mag_loss': 0.04073966250407536, 'avg_g_cos_loss': 0.008872987412878067, 'avg_pred_duration': 3.948948383331299, 'avg_grad_duration': 12.55395245552063, 'avg_total_duration': 16.50290083885193, 'avg_pred_std': 0.10244974493980408, 'avg_std_loss': 0.00610949145630002, 'avg_mean_pred_loss': 6.7261853473610245e-06}, 'min_metrics': {'avg_loss': 0.002562793581552493, 'avg_g_mag_loss': 0.04073966250407536, 'avg_g_cos_loss': 0.008872987412878067, 'pred_duration': 3.948948383331299, 'grad_duration': 12.55395245552063, 'total_duration': 16.50290083885193, 'pred_std': 0.10244974493980408, 'std_loss': 0.00610949145630002, 'mean_pred_loss': 6.7261853473610245e-06, 'pred_rmse': 0.05062403902411461, 'pred_mae': 0.03746151924133301, 'pred_mape': 0.11356884986162186, 'grad_rmse': 0.04399764537811279, 'grad_mae': 0.03127303719520569, 'grad_mape': 0.8461189866065979}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.002562793581552493, 'avg_g_mag_loss': 0.04073966250407536, 'avg_g_cos_loss': 0.008872987412878067, 'pred_duration': 3.948948383331299, 'grad_duration': 12.55395245552063, 'total_duration': 16.50290083885193, 'pred_std': 0.10244974493980408, 'std_loss': 0.00610949145630002, 'mean_pred_loss': 6.7261853473610245e-06, 'pred_rmse': 0.05062403902411461, 'pred_mae': 0.03746151924133301, 'pred_mape': 0.11356884986162186, 'grad_rmse': 0.04399764537811279, 'grad_mae': 0.03127303719520569, 'grad_mape': 0.8461189866065979}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:11:47.549770Z", + "iopub.status.busy": "2024-03-23T06:11:47.549391Z", + "iopub.status.idle": "2024-03-23T06:11:47.553901Z", + "shell.execute_reply": "2024-03-23T06:11:47.553001Z" + }, + "papermill": { + "duration": 0.024518, + "end_time": "2024-03-23T06:11:47.555978", + "exception": false, + "start_time": "2024-03-23T06:11:47.531460", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:11:47.588382Z", + "iopub.status.busy": "2024-03-23T06:11:47.588078Z", + "iopub.status.idle": "2024-03-23T06:11:47.684253Z", + "shell.execute_reply": "2024-03-23T06:11:47.683423Z" + }, + "papermill": { + "duration": 0.115126, + "end_time": "2024-03-23T06:11:47.686601", + "exception": false, + "start_time": "2024-03-23T06:11:47.571475", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:11:47.721771Z", + "iopub.status.busy": "2024-03-23T06:11:47.721405Z", + "iopub.status.idle": "2024-03-23T06:11:48.001514Z", + "shell.execute_reply": "2024-03-23T06:11:48.000578Z" + }, + "papermill": { + "duration": 0.299913, + "end_time": "2024-03-23T06:11:48.003551", + "exception": false, + "start_time": "2024-03-23T06:11:47.703638", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATIAAAESCAYAAAB+alnzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABATElEQVR4nO3de1xUdf7H8dfMwHCTi4DcjIs3xLymBmGlmRSZW2GWZf7SXEtrtTXZWrOLdNnSrHbLcrvsbtruZl52s4u6liJ2EbykmBeU1BQ0BUTkfp/5/v44MooCMjAwIJ/n4zEPZ875nnO+M8O8PZfv93t0SimFEEK0Y3p7V0AIIZpLgkwI0e5JkAkh2j0JMiFEuydBJoRo9yTIhBDtngSZEKLdc7B3BVqL2Wzm5MmTuLu7o9Pp7F0dIcRFlFIUFRURFBSEXm/dPlaHCbKTJ08SHBxs72oIIS7j+PHjXHXVVVYt02GCzN3dHdA+JA8PDzvXRghxscLCQoKDgy2/VWt0mCCrOZz08PCQIBOiDWvKqR852S+EaPckyIQQ7Z4EmRCi3esw58hE85hMJqqqquxdDdGOOTo6YjAYWmTdEmSiQUopsrKyyM/Pt3dVxBXAy8uLgIAAm7fllCATDaoJMT8/P1xdXaUxsWgSpRSlpaXk5OQAEBgYaNP1S5BdpKi8ioNZRVSbFNE9fOxdHbsymUyWEPPx6difhWg+FxcXAHJycvDz87PpYaac7L/Itl/yuPf9FP60Ns3eVbG7mnNirq6udq6JuFLU/C3Z+nyrBNlFQn20DzrzTClyOwONHE4KW2mpvyUJsosEe7ui00FRRTV5JZX2ro4QohEkyC7i7GggwMMZgIy8UjvXRgjRGBJkdag5vMw4U2Lnmoj2SqfT8fnnn9u7Gjb1wgsvMGjQIHtXo04SZHUI9XYDIOOM7JGJ9mvp0qV4eXnZbH1PPvkkiYmJNlufLUnzizqE+tbskUmQiStfZWUlRqPxsuU6depEp06dWqFG1pM9sjqc3yOTQ8sLKaUoray2y8PaK8jr16/nhhtuwMvLCx8fH37zm99w5MgRAIYNG8acOXNqlT99+jSOjo589913AJw6dYoxY8bg4uJCt27dWLZsGWFhYbz11ltN+uz27t3LzTffjIuLCz4+PkybNo3i4mLL/M2bNxMZGYmbmxteXl5cf/31ZGRkAPDTTz8xcuRI3N3d8fDwYMiQIfz4448Nbm/z5s1MmTKFgoICdDodOp2OF154AYCwsDBefvllJk2ahIeHB9OmTQNgzpw5hIeH4+rqSvfu3Xn++edrNZO4+NDyoYceIi4ujjfeeIPAwEB8fHyYMWOGXbqyyR5ZHc6fI5M9sguVVZm4et7Xdtl22kuxuBob/+daUlJCfHw8AwYMoLi4mHnz5jF27Fh2797NxIkTWbhwIQsWLLA0B1ixYgVBQUHceOONAEyaNInc3Fw2b96Mo6Mj8fHxllbp1iopKSE2Npbo6Gh27NhBTk4ODz/8MDNnzmTp0qVUV1cTFxfHI488wqeffkplZSXbt2+31G3ixIlcc801vPfeexgMBnbv3o2jo2OD2xw2bBhvvfUW8+bNIz09HaDW3tQbb7zBvHnzSEhIsExzd3dn6dKlBAUFsXfvXh555BHc3d354x//WO92kpKSCAwMJCkpicOHD3PfffcxaNAgHnnkkSZ9Vk0lQVaHmiA7U1JJUXkV7s4N/9GItmfcuHG1Xn/00Ud06dKFtLQ0xo8fzxNPPMEPP/xgCa5ly5YxYcIEdDodBw8eZOPGjezYsYOhQ4cC8Pe//51evXo1qS7Lli2jvLycf/7zn7i5aXv77777LnfccQevvfYajo6OFBQU8Jvf/IYePXoA0KdPH8vymZmZPPXUU0RERAA0qh5GoxFPT090Oh0BAQGXzL/55pv5wx/+UGvac889Z3keFhbGk08+yfLlyxsMss6dO/Puu+9iMBiIiIhgzJgxJCYmSpC1Be7Ojvi4GTlTUknGmVL6dfW0d5XaBBdHA2kvxdpt29Y4dOgQ8+bNY9u2beTm5mI2mwEtFPr168ett97KJ598wo033sjRo0dJSUnhgw8+ACA9PR0HBwcGDx5sWV/Pnj3p3Llzk+p+4MABBg4caAkxgOuvvx6z2Ux6ejrDhw/noYceIjY2lltuuYWYmBjGjx9v6Y8YHx/Pww8/zL/+9S9iYmK49957LYHXVDUBfaEVK1awaNEijhw5QnFxMdXV1ZcdTblv3761uhoFBgayd+/eZtWtKeQcWT1Calr4S1syC51Oh6vRwS4Pa1uE33HHHeTl5fG3v/2Nbdu2sW3bNkA7sQ3a4dp//vMfqqqqWLZsGf3796d///42/8waa8mSJaSkpDBs2DBWrFhBeHg4W7duBbRzU/v372fMmDFs2rSJq6++mtWrVzdrexeGKkBKSgoTJ07k9ttvZ82aNaSmpvLss89aPq/6XHyIq9PpLP9ptCYJsnqE+Whf9DE54d/unDlzhvT0dJ577jlGjRpFnz59OHv2bK0yd911F+Xl5axfv55ly5YxceJEy7zevXtTXV1NamqqZdrhw4cvWUdj9enTh59++omSkvN/S1u2bEGv19O7d2/LtGuuuYa5c+eSnJxMv379WLZsmWVeeHg4s2fP5ptvvuHuu+9myZIll92u0WjEZDI1qo7JycmEhoby7LPPMnToUHr16mW52NAeSJDVI8T7fJ9L0b507twZHx8fPvzwQw4fPsymTZuIj4+vVcbNzY24uDief/55Dhw4wIQJEyzzIiIiiImJYdq0aWzfvp3U1FSmTZuGi4tLk/oKTpw4EWdnZyZPnsy+fftISkri8ccf58EHH8Tf35+jR48yd+5cUlJSyMjI4JtvvuHQoUP06dOHsrIyZs6cyebNm8nIyGDLli3s2LGj1jm0+oSFhVFcXExiYiK5ubmUltb/t9yrVy8yMzNZvnw5R44cYdGiRc3e62tNEmT1CDvXlkz2yNofvV7P8uXL2blzJ/369WP27Nm8/vrrl5SbOHEiP/30EzfeeCMhISG15v3zn//E39+f4cOHM3bsWMsVPGdnZ6vr4+rqytdff01eXh7XXnst99xzD6NGjeLdd9+1zD948CDjxo0jPDycadOmMWPGDKZPn47BYODMmTNMmjSJ8PBwxo8fz+jRo3nxxRcvu91hw4bx6KOPct9999GlSxcWLlxYb9k777yT2bNnM3PmTAYNGkRycjLPP/+81e/VXnSqgwzxUFhYiKenJwUFBY26HdzOjLOMey+ZIE9nkueOaoUatj3l5eUcPXqUbt26NekHfCU5ceIEwcHBbNy4kVGjOubfgy009Ddl7W/0QnLVsh5h5072nyosp7zKhLOVV81E+7Zp0yaKi4vp378/p06d4o9//CNhYWEMHz7c3lUTdZBDy3p4uxnp5OSAUnDirJwn62iqqqp45pln6Nu3L2PHjqVLly6WxrGffPKJpbvOxY++ffu2Wh1Hjx5dbz1effXVVqtHWyB7ZPXQ6XSE+riy/2Qhx3JL6eln/W3cRfsVGxtLbGzdbebuvPNOoqKi6px3uRb3tvT3v/+dsrKyOud5e3u3Wj3aAgmyBtQEmYxLJi7k7u6Ou7v9/2Pr2rWrvavQZsihZQNCfaTzuBDtgQRZA0K9pfO4EO2BBFkDZI9MiPZBgqwBNaNgnDhbRrWp9fuPCSEaR4KsAQEezhgd9FSbFSfzy+1dHSFEPSTIGqDX6yx9LjPy5PBSNN6VePORtkyC7DJqWvgfkxP+op2x9c1HQBtCW6fTkZ+fb9P1NpcE2WWEnBu/P1NO+AvRZkmQXUaY3FHpPKWgssQ+D7n5iM1uPlJRUcGTTz5J165dcXNzIyoqis2bN1uWzcjI4I477qBz5864ubnRt29f1q1bx7Fjxxg5ciSgDZWk0+l46KGHmvR52Jq07L+MEGlLdl5VKbwaZJ9tP3MSjG6XL3eO3Hyk/puPzJw5k7S0NJYvX05QUBCrV6/mtttuY+/evfTq1YsZM2ZQWVnJd999h5ubG2lpaXTq1Ing4GD++9//Mm7cONLT0/Hw8MDFxaVJn4mtSZBdRs1IsRl5JSilmjSwnmh9cvORum8+kpmZyZIlS8jMzCQoSPtP6cknn2T9+vUsWbKEV199lczMTMaNG2cZ+rt79+6W5Wv6cPr5+dn8/FtzSJBdRtfOLhj0OsqrzOQUVeDv0YHH5XJ01faM7LVtK8jNR+q2d+9eTCYT4eHhtaZXVFTg4+MDwO9//3see+wxvvnmG2JiYhg3bhwDBgxo0vZaS5POkS1evJiwsDCcnZ2Jiopi+/btDZZftWoVERERODs7079/f9atW2eZV1VVxZw5c+jfvz9ubm4EBQUxadIkTp6s/YPJy8tj4sSJeHh44OXlxdSpU2udY2gpjgY9Xb203ecOf3ip02mHd/Z4yM1HbHLzkeLiYgwGAzt37mT37t2Wx4EDB3j77bcBePjhh/nll1948MEH2bt3L0OHDuWdd96x2XttEcpKy5cvV0ajUX300Udq//796pFHHlFeXl4qOzu7zvJbtmxRBoNBLVy4UKWlpannnntOOTo6qr179yqllMrPz1cxMTFqxYoV6uDBgyolJUVFRkaqIUOG1FrPbbfdpgYOHKi2bt2qvv/+e9WzZ081YcKERte7oKBAAaqgoMDat6z+7+9bVeicNWrFjkyrl23PysrKVFpamiorK7N3VaySm5urAPXdd99Zpn3//fcKUKtXr1ZKKVVcXKzc3NzUl19+qa6++mq1YMECS9kDBw4oQP3444+WaYcOHVKA+stf/tKoOly4rQ8//FB17txZFRcXW+avXbtW6fV6lZWVVefy1113nXr88cfrnHf//ferO+6447J1+OSTT1SnTp1qTUtPT7/ks7mcp59+WvXv318ppf2eAZWbm9vo5S/U0N9Uc36jVgdZZGSkmjFjhuW1yWRSQUFBav78+XWWHz9+vBozZkytaVFRUWr69On1bmP79u0KUBkZGUoppdLS0hSgduzYYSnzv//9T+l0OvXrr782qt7N+ZCeXb1Hhc5Zo15ff9DqZduz9hpkJpNJ+fj4qP/7v/9Thw4dUomJieraa6+tFS5KKTVx4kQ1cOBApdPpLH9rNWJiYtTgwYPVtm3b1K5du9TIkSOVi4uLeuuttxpVhwu3VVJSogIDA9W4cePU3r171aZNm1T37t3V5MmTlVJK/fLLL+rpp59WycnJ6tixY+rrr79WPj4+6q9//asqLS1VM2bMUElJSerYsWPqhx9+UD169FB//OMfL1uHmtDZuHGjOn36tCopKbG877CwMPXf//5X/fLLL2rbtm3q1VdfVWvWrFFKKTVr1iy1fv169csvv6idO3eqqKgoNX78eKWUUidOnFA6nU4tXbpU5eTkqKKiokZ9HjXaRJBVVFQog8FQ649BKaUmTZqk7rzzzjqXCQ4OvuR/sXnz5qkBAwbUu50NGzYonU5neUP/+Mc/lJeXV60yVVVVymAwqM8++6zOdZSXl6uCggLL4/jx403+kD789ogKnbNGzfhkp9XLtmftNciU0v6G+vTpo5ycnNSAAQPU5s2bLwmydevWKUANHz78kuVPnjypRo8erZycnFRoaKhatmyZ8vPzU++//36jtn/xtvbs2aNGjhypnJ2dlbe3t3rkkUcsIZCVlaXi4uJUYGCgMhqNKjQ0VM2bN0+ZTCZVUVGh7r//fhUcHKyMRqMKCgpSM2fObPR38uijjyofHx8FqISEBKWUUpWVlWrevHkqLCxMOTo6qsDAQDV27Fi1Z88epZRSM2fOVD169FBOTk6qS5cu6sEHH6y1B/bSSy+pgIAApdPpLGHcWG0iyH799VcFqOTk5FrTn3rqKRUZGVnnMo6OjmrZsmW1pi1evFj5+fnVWb6srEwNHjxYPfDAA5Zpr7zyigoPD7+kbJcuXdRf//rXOteTkJCggEseTfmQvt53SoXOWaPueOd7q5dtz9pzkNlazX+EGzdutHdV2rWWCrI21SC2qqqK8ePHo5Tivffea9a65s6dS0FBgeVx/PjxJq+rZjifY7nSur+j2LRpE19++SVHjx4lOTmZ+++/X24+0oZZFWS+vr4YDAays7NrTc/Ozq7VVuVCAQEBjSpfE2IZGRls2LCh1u2gAgICLmmMWF1dTV5eXr3bdXJywsPDo9ajqWoaxRaWV5Nf2vAt5MWVQW4+0r5Y1Y7MaDQyZMgQEhMTiYuLA8BsNpOYmMjMmTPrXCY6OprExESeeOIJy7QNGzYQHR1teV0TYocOHSIpKcnSnuXCdeTn57Nz506GDBkCaP9jms3mem8CYUsuRgP+Hk5kF1Zw7Ewpg1yNLb5NYV9y85H2xeoGsfHx8UyePJmhQ4cSGRnJW2+9RUlJCVOmTAG0rh1du3Zl/vz5AMyaNYsRI0bw5ptvMmbMGJYvX86PP/7Ihx9+CGghds8997Br1y7WrFmDyWQiKysL0L4Mo9FInz59uO2223jkkUd4//33qaqqYubMmdx///2W1sktLdTHjezCCjLOlDAo2KtVtinaJrn5SBvUlBN277zzjgoJCVFGo1FFRkaqrVu3WuaNGDHikisZK1euVOHh4cpoNKq+ffuqtWvXWuYdPXq0zpPygEpKSrKUO3PmjJowYYLq1KmT8vDwUFOmTLHq0m9zTiQqpdSTK3er0Dlr1Nsbf27S8u1RzYnZ0tJSe1dFXCFKS0tb5GS/TikrhxVop5pzO3aAxUmHef3rdMYNvoo3xw9sgRq2PSaTiZ9//hk/P79LDveFaIozZ86Qk5NDeHg4BoOh1rzm/Ealr2UjnR8Fo+NcuTQYDHh5eVkutLi6ukqnedEkSilKS0vJycnBy8vrkhBrLgmyRjo/CkbH6m9Zc1W4qUPYCHEhLy+velsaNIcEWSOFnBvy+nRRBSUV1bg5dYyPTqfTERgYiJ+fH1VVVfaujmjHHB0dbb4nVqNj/BptwNPFES9XR/JLq8jMK6VPYNPbpbVHBoOhxf4IhWiuNtWyv62TG/YK0TZJkFkhVIa9FqJNkiCzgtwaToi2SYLMCiHnDi0z5Wa9QrQpEmRWsOyR5coemRBtiQSZFWqaYJwqKKOi2mTn2gghakiQWaFLJydcjQbMCk6crXvUASFE65Mgs4JOp7N0VcqUE/5CtBkSZFaq6ap0TNqSCdFmSJBZKdRH2pIJ0dZIkFlJWvcL0fZIkFnJskfWwUbBEKItkyCzUk2QHc8rxWTuEGNSCtHmSZBZKdDTBUeDjiqT4lSBNMEQoi2QILOSQa8jWDqPC9GmSJA1gYyCIUTbIkHWBHLlUoi2RYKsCaQtmRBtiwRZE0jrfiHaFgmyJqgZBSMzr5QOcltQIdo0CbImuKqzC3odlFaaOF1cYe/qCNHhSZA1gZODgUBPF0BGwRCiLZAga6IwXxm/X4i2QoKsiUK8z43fLyf8hbA7CbImkjsqCdF2SJA1kYyCIUTbIUHWRNK6X4i2Q4KsiWrG7s8vraKgtMrOtRGiY5MgayI3Jwe6uDsBkCE37BXCriTImkFGwRCibZAgawY5TyZE2yBB1gwyCoYQbYMEWTNIkAnRNkiQNYPl0FJO9gthVxJkzVDTuj+7sIKySpOdayNExyVB1gxerkY8nB0AbWwyIYR9NCnIFi9eTFhYGM7OzkRFRbF9+/YGy69atYqIiAicnZ3p378/69atqzX/s88+49Zbb8XHxwedTsfu3bsvWcdNN92ETqer9Xj00UebUn2bCvOV0WKFsDerg2zFihXEx8eTkJDArl27GDhwILGxseTk5NRZPjk5mQkTJjB16lRSU1OJi4sjLi6Offv2WcqUlJRwww038NprrzW47UceeYRTp05ZHgsXLrS2+jZX08JfxiUTwo6UlSIjI9WMGTMsr00mkwoKClLz58+vs/z48ePVmDFjak2LiopS06dPv6Ts0aNHFaBSU1MvmTdixAg1a9asRtezvLxcFRQUWB7Hjx9XgCooKGj0Ohrj9fUHVeicNeqZz/bYdL1CdDQFBQVN/o1atUdWWVnJzp07iYmJsUzT6/XExMSQkpJS5zIpKSm1ygPExsbWW74hn3zyCb6+vvTr14+5c+dSWlr/XtD8+fPx9PS0PIKDg63eXmNcOH6/EMI+HKwpnJubi8lkwt/fv9Z0f39/Dh48WOcyWVlZdZbPysqyqqIPPPAAoaGhBAUFsWfPHubMmUN6ejqfffZZneXnzp1LfHy85XVhYWGLhJncUUkI+7MqyOxp2rRpluf9+/cnMDCQUaNGceTIEXr06HFJeScnJ5ycnFq8XjWNYn89W0ZltRmjg1wIFqK1WfWr8/X1xWAwkJ2dXWt6dnY2AQEBdS4TEBBgVfnGioqKAuDw4cPNWk9z+bk74eyox6zg1/wyu9ZFiI7KqiAzGo0MGTKExMREyzSz2UxiYiLR0dF1LhMdHV2rPMCGDRvqLd9YNU00AgMDm7We5tLpdIR6S+dxIezJ6kPL+Ph4Jk+ezNChQ4mMjOStt96ipKSEKVOmADBp0iS6du3K/PnzAZg1axYjRozgzTffZMyYMSxfvpwff/yRDz/80LLOvLw8MjMzOXnyJADp6emAtjcXEBDAkSNHWLZsGbfffjs+Pj7s2bOH2bNnM3z4cAYMGNDsD6G5Qn1cSc8ukj6XQthLUy6TvvPOOyokJEQZjUYVGRmptm7dapk3YsQINXny5FrlV65cqcLDw5XRaFR9+/ZVa9eurTV/yZIlCrjkkZCQoJRSKjMzUw0fPlx5e3srJycn1bNnT/XUU09ZdZm2OZd2L+dPa/ar0Dlr1Itf7rf5uoXoKJrzG9UppZQdc7TVFBYW4unpSUFBAR4eHjZd97+3ZvDc5/sYFeHHPx661qbrFqKjaM5vVC6x2YDcUUkI+5Igs4GatmSZeaWYzR1iB1eINkWCzAYCPZ1x0OuorDaTVVhu7+oI0eFIkNmAg0FPsHfNncelCYYQrU2CzEZkFAwh7EeCzEZqRos9JkEmRKuTILOREMsJfzm0FKK1SZDZiGWPLFf2yIRobRJkNhJ6wbhkHaSNsRBthgSZjVzV2RWdDoorqjlTUmnv6gjRoUiQ2Yizo4FAD2dAbtgrRGuTILOhEMudx+WEvxCtSYLMhmq6KskemRCtS4LMhmSPTAj7kCCzIcsemYyCIUSrkiCzoZpuSnJoKUTrkiCzoZq2ZHkllRSWV9m5NkJ0HBJkNuTu7IiPmxGQzuNCtCYJMhuzjBYrQSZEq5Egs7FQufO4EK1OgszGLH0uZY9MiFYjQWZjoT4yUqwQrU2CzMZCL7gRiRCidUiQ2VjoubZkpwrKKa8y2bk2QnQMEmQ25u1mxN3JAYDjslcmRKuQILMxnU5n6XMp4/cL0TokyFrA+VEw5IS/EK1BgqwFhEijWCFalQRZC6i5EYmMgiFE65AgawEh3ueaYMihpRCtQoKsBYT5antkJ86WUW0y27k2Qlz5JMhagL+7M0YHPdVmxcn8cntXR4grngRZC9DrdZaGsdJVSYiWJ0HWQkLlhL8QrUaCrIXU9LnMyG25PbLyKhMFpTISrRASZC2kpffIjuWWMHxhEjcu3MSh7KIW2YYQ7YUEWQsJbcHW/TlF5Uz6aDs5RRUUllczY9kuyiqlg7rouCTIWkjNyf7MvFLMZmWz9RaVVzFlyQ4y80oJ8Xali7sTP2cX8+JX+222DSHaGwmyFtK1swsGvY7yKjM5RRU2WWdFtYnp/9rJ/pOF+HYy8s/fRvLWfYPQ6WD5juN8sfvX5m/kwFewYR6Y5NybaD+aFGSLFy8mLCwMZ2dnoqKi2L59e4PlV61aRUREBM7OzvTv359169bVmv/ZZ59x66234uPjg06nY/fu3Zeso7y8nBkzZuDj40OnTp0YN24c2dnZTal+q3A06Onq5QLY5vDSbFbEr/yJ5CNncDMaWDolkjBfN67v6cvjN/cC4JnP9vLL6eKmb6TgBPz3YdjyNuz/vNl1FqK1WB1kK1asID4+noSEBHbt2sXAgQOJjY0lJyenzvLJyclMmDCBqVOnkpqaSlxcHHFxcezbt89SpqSkhBtuuIHXXnut3u3Onj2br776ilWrVvHtt99y8uRJ7r77bmur36psdUclpRQvrUlj7Z5TOBp0fPDgUPp19bTMnzWqF1HdvCmpNDFjWWrTB3Tc9CeoPteAd99/m1VnIVqVslJkZKSaMWOG5bXJZFJBQUFq/vz5dZYfP368GjNmTK1pUVFRavr06ZeUPXr0qAJUampqren5+fnK0dFRrVq1yjLtwIEDClApKSmNqndBQYECVEFBQaPK28Kzq/eo0Dlr1ML1B5q1nsVJh1TonDUqdM4a9cXuX+ssk1VQpq556RsVOmeNem71Xus38muqUgmeSiV4aI8XfZQqzWtWvYWwRnN+o1btkVVWVrJz505iYmIs0/R6PTExMaSkpNS5TEpKSq3yALGxsfWWr8vOnTupqqqqtZ6IiAhCQkLqXU9FRQWFhYW1Hq0tzHJruKbvka388TgL16cDMO83V3PnwKA6y/l7OPPn8QMB+NfWDNbtPdX4jSgF3zwHKOh/L/j1BXMVHFjT5HoL0ZqsCrLc3FxMJhP+/v61pvv7+5OVlVXnMllZWVaVr28dRqMRLy+vRq9n/vz5eHp6Wh7BwcGN3p6thHg379ZwiQeymfvZXgAeHdGD397QrcHyN/X247GbegAw5z97Gn9u7tA3cOx7MDjBzc9Dv3OH7HJ4KdqJK/aq5dy5cykoKLA8jh8/3up1CPM9f7NepaxrgrEz4ywzlu3CZFaMG3wVc27r3ajl/nBLOENDO1NUUc3MZalUVF/mfJmpGr55Xnt+3aPQOfR8kB39FopPW1VvIezBqiDz9fXFYDBccrUwOzubgICAOpcJCAiwqnx966isrCQ/P7/R63FycsLDw6PWo7XV7JEVlVeTb0VXosM5RUz9eAflVWZG9u7CgnH90el0jVrWwaBn0YRr8HJ1ZO+vBSz438GGF0j9J+Smg4s33BCvTfPuDkGDQZkh7fNG11sIe7EqyIxGI0OGDCExMdEyzWw2k5iYSHR0dJ3LREdH1yoPsGHDhnrL12XIkCE4OjrWWk96ejqZmZlWrae1OTsaCPBwBho/CsapgjIm/WM7+aVVDAr2YvHEwTgarNtxDvJy4Y17tPNlS7Yc4+v99RzGVxRB0qva85ueBhev8/P636P9K4eXoh2w+tAyPj6ev/3tb3z88cccOHCAxx57jJKSEqZMmQLApEmTmDt3rqX8rFmzWL9+PW+++SYHDx7khRde4Mcff2TmzJmWMnl5eezevZu0tDRAC6ndu3dbzn95enoydepU4uPjSUpKYufOnUyZMoXo6Giuu+66Zn0ALa1m/P7G3LC3oLSKyR9t52RBOd27uPHRQ9fianRo0nZjrvbn4XPn1J5a9RMnztax/S1vQ8lp8O4BQ6bUntd3LKCDzBStfZkQbVlTLpO+8847KiQkRBmNRhUZGam2bt1qmTdixAg1efLkWuVXrlypwsPDldFoVH379lVr166tNX/JkiUKuOSRkJBgKVNWVqZ+97vfqc6dOytXV1c1duxYderUqUbX2R7NL5RS6qlVu1XonDXqrQ0/N1iurLJa3fPeFhU6Z42KfGWDOp5X0uxtV1SZ1J3v/qBC56xRcYt/UJXVpvMz808o9bK/1tQi7cu6V/DRaG3+lkXNrosQl9Oc36hOKSvPQrdThYWFeHp6UlBQ0KrnyxYnHeb1r9O5e3BX/jx+UJ1lqk1mHvtkFxvSsnF3dmDVo9FEBNimjsfzSrl90fcUlVczfXh35t7eR5vx+e9g9ycQEg1T/gd1nYPb8XdY+wcIugambbZJfYSoT3N+o1fsVcu24nKt+5VSPP/FPjakZWN00PP3SUNtFmIAwd6uvH7PAAA++O4Xkg7mwKk9sHuZVuDWP9UdYgB97gKdAU6mwpkjNquTELYmQdbCQr1rhvOpO8j+svEQn24/jl4Hi+6/hqjuPjavw239ApkcHQpA/IpUKtY9AyjoNw6uGlr/gp26QPcR2vP9n9m8XkLYigRZC6s52Z9bXEFxRXWtef/amsGixEMAvBzXj9v6Nb5JirWeGdOHfl09GFjxI07Hv0cZjDBq3uUX7DdO+3efBJlouyTIWpiniyOdXR2B2i38/7f3FPO+0DrOPxHTi4lRoS1aDycHA+/eN4DnHLVDyh/9xkPnsMsvGPEbMBghJw2y01q0jkI0lQRZK7h4tNiUI2eYtXw3SsEDUSHMGtWrVeoRdnw1PXUnOKs68fCx4Xx/qBGt9l28oOct2nNpUybaKAmyuigFlbYba//C8fvTThYy7Z8/UmkyE9vXn5fv6tfoVvvNUlEMm14B4Ieuv6VAdWL2it3kFDbivpsX9r3sGBe5RTsjQXaxjGT4cASsn2OzVdbskaUcOcNDS7ZTVFFNZDdv3r7/Ggz6VggxgORFUJIDnbtxy6RniAhwJ7e4kidW7MZ0uaG4e48GR1c4e1S7gilEGyNBdjGdAU79BD8th6LGj9DRkJrx+7/9+TQ5RRVEBLjzt0lDcXY02GT9l1V4ErYs0p7f8iLOzi68+8BgXI0Gko+c4d1Nhxte3ugG4bdpz+XwUrRBEmQXC4mC4OvAVAlb37PJKsN8XS3Pu3q58PFvI/F0cbTJuhsl6RWoLoPgKOhzJwA9/Trxp7h+ALyd+DMpR840vI6aq5f7V4PZ3JK1FcJqEmR1uX6W9u+PH0F58wdk7OXvTicnB7zdjHz820j8z3UkbxVZ+yD1E+35ra/Uavx69+CruGfIVZgVzFqeSm5xAzdJ6RkDTh5Q+Csc39bClRbCOhJkdQm/DXzDoaIQdi5t9uo8nB3ZGD+CTX8YQU+/Ts2vnzU2PA8orRN48LWXzH7prr709OtETlEF8St/qv/WdY7OWlMMkMNL0eZIkNVFr4dhv9eeb/0rVFc2e5UBns54uRqbvR6rHN4IRzaB3hFGJdRZxNXowOIHBuPsqOe7n0/z/ncNdEWqObxM+1wbkFGINkKCrD4DxkOnACg6BXtX2bs21jObzo/8GjUdvOsfJrt3gDsv3tkXgDe/+Zkfj+XVXbD7CHD10Yb+OfadrWssRJNJkNXHwQmue0x7nryo/Z3g3v2J1hrf2Qtu/MNli48fGsxdg4IwmRWPf5pKfmkde6EGR7j6Lu25HF6KNkSCrCFDp4DRHU4f1G7Q0V5c0PiVEX8EV+/LLqLT6XhlbH+6+bpxqqCcF7+qpztSzeHlga+g2jZ3UBeiuSTIGuLsqYUZaKOpthcp70JxltaX8tqHG71YJycH3hw/EL0OVqf+WvcQ2SHR4B4I5QXa+Tch2gAJssu57nfayfLMZDi+3d61ubyirPOhG/OCdohshcEhnZk2XLul3LOr95JXctEhpt5wbhhs5PBStBkSZJfjEQgD79Oet4e9sqRXoKoUrroWro5r0iqeiOlFL79O5BZXkvDl/ksL1BxeHlxn0z6pQjSVBFlj1DTFOLgWTv9s37o0JHs/pP5be35R41drODsaeOPegRj0Or766eSldy3vOgS8QqCqBA593cxKC9F8EmSN0aU39L4dUJDyjr1rU78N87R7UV59l9bVqhkGBnvx2AjtEPO5z/fVbvWv053fK9v7n2ZtRwhbkCBrrJpuSzbsTG5ThxO1BrANNH611uOjehIR4E5eSSXPf76v9t3Sa4Ls0AbtxL8QdiRB1lgh12mdrk2VsO19e9emNrNJ2xsDiHwEfHrYZLVODtohpoNex//2ZbFmzwWHmP79wLc3mCq0c2VC2JEEmTVq9sp22KYzuc389Clk79Oaiwx/yqar7tfVkxkjewLw/Bf7yCk6NxDjhYeXcvVS2JkEmTXCR4NPL6gogF0f27s2msoS2PQn7fnwpxrV+NVaM0b25OpAD/JLq3hu9QWHmDUjx/6SBCWXGQZIiBYkQWYNvR6uP3cFM8U2ncmbLWWx1h/UKwQip7XIJowOet64dyCOBh3fpGXz5U8ntRm+vSBgAJir4cCXLbJtIRpDgsxaA+4715n8JOyz8xW7omz44S3teRMav1rj6iAPfn+zdpOUeV/sPz/WvxxeijZAgsxaF3Ym32LnzuRJr2htuboOhb53t/jmHr2pB/27elJQVsUzq/dqh5g1rfyP/QCFpxpegRAtRIKsKSydyQ/YrzP5/tXnz9Pd+qcmN361hqNBO8Q0GvRsPJDDZ7t+hc6hcFUkoLRxyoSwAwmyprB3Z/JTP8Hqc3uF182A0OhW23TvAHeeuEU7xHzhq/1kFZTL4aWwOwmyprruMft0Ji/Khk8naDcT6TEKbnmp9bZ9zrQbuzMw2Iui8mqe/mwPqm8c6PRwYgecPdbq9RFCgqypPIK0E//QentlVeWwYqJ2AxCfXnDPR2BwaJ1tX8DBoOfNewdgdNCzOf00qw5WQdgN2sz9q1u9PkJIkDXHsMe1fw+uhdxDLbstpWDNE9pej7MXPLACXLxadpsN6OnnzpO3hgPw8po0znbXbjMnh5fCHiTImsMvQmski4LkFu5MnrxIa8GvM8C9S23WDak5pt7QncEhXhRVVPPMgTCU3gGy9rbtEULEFUmCrLksnck/1c5ftYSfv4YN5zqC37YAeoxsme1YyaDX8ca9A3Fy0PO/Xyo56XPuosP+z+xbMdHhSJA1V2h0y3YmzzkA/5kKKBgyResU3oZ079KJP94WAcA72QO0iXv/ox0KC9FKJMhswdKZ/B9QUWS79Zbmwaf3Q2URhN4At7/eKu3FrDVlWBiRYd6sqbyGShzhzCHtEFOIViJBZgsXdibfaaPO5KYqWDlJa87gFQrj/6ndjq0N0ut1LLxnACZHdxJNg7SJctJftCIJMluo1Zl8cfM7kysF656CY9+DsZN2hdLNp/n1bEFhvm48PTqCL03DAKjeI4eXovVIkNnKgPugk79tOpPv+DvsXALoYNzfwa+PTarY0h68LpSSkJEUK2ccik5gPr7D3lUSHYQEma3YqjP5L5vhf3O05zEvQO/Rtqhdq9DrdbwyPookhgJwYOMSO9dIdBRNCrLFixcTFhaGs7MzUVFRbN/ecBedVatWERERgbOzM/3792fdutpDIyulmDdvHoGBgbi4uBATE8OhQ7UbmIaFhaHT6Wo9FixY0JTqt5whF3QmP7zB+uXPHIGVk0GZYMD95y8itCPB3q50GqL1eOiS8T+O5rShkXTFFcvqIFuxYgXx8fEkJCSwa9cuBg4cSGxsLDk5OXWWT05OZsKECUydOpXU1FTi4uKIi4tj3759ljILFy5k0aJFvP/++2zbtg03NzdiY2MpLy+vta6XXnqJU6dOWR6PP/64tdVvWS5eMPQh7bm13ZbKC7QrlOX52rA8d7zdJq9QNsZNt99Hia4TfrqzLP30E0xmOVcmWpiyUmRkpJoxY4bltclkUkFBQWr+/Pl1lh8/frwaM2ZMrWlRUVFq+vTpSimlzGazCggIUK+//rplfn5+vnJyclKffvqpZVpoaKj6y1/+0uh6lpeXq4KCAsvj+PHjClAFBQWNXkeTFPyq1Is+SiV4KJW5vXHLmKqV+tfd2jJv9lGqMKtl69gKilc+qlSCh/rk2Tj1t++OtMg2qk3mFlmvsI+CgoIm/0at6nFcWVnJzp07mTt3rmWaXq8nJiaGlJSUOpdJSUkhPj6+1rTY2Fg+//xzAI4ePUpWVhYxMTGW+Z6enkRFRZGSksL9999vmb5gwQJefvllQkJCeOCBB5g9ezYODnW/hfnz5/Piiy9a8/Zso6Yz+e5/Q/LbcN+/L7/MhnnardwcXOD+ZeDu3/L1bGFug8fD/mWMNmznhq/3MzLCjx5dOtVb3mRW5JdWcra0krySKvJKKsgrqTr3+vzjbGklecUVeJRm0L36F4459qDANQRvNyc6uxnxdjPi7Wqks5sRHzfj+Wnnpnu6OKLXt889XVE/q4IsNzcXk8mEv3/tH5q/vz8HDx6sc5msrKw6y2dlZVnm10yrrwzA73//ewYPHoy3tzfJycnMnTuXU6dO8ec//7nO7c6dO7dWgBYWFhIcHNzId9pMwx7XguzAGsg9DL496y+b+m9IeVd7PvY9CBrUKlVscWE3oty60LnkNNea9zB7RWfuGBBEXmklZy8IpprX+WVVDbbWcKaC6/Rp3K3/iZH63YTqc8CozTtR6sv3Rf353tyfRHM/Cqg/MPU68HK9MPAc8XZzwtvNkc6uRrq4OzEivAterkYbfyCiJbX+GDBNdGEoDRgwAKPRyPTp05k/fz5OTpeOVe/k5FTn9FZR05n85/9pdya/o57zZZlb4asntOcjnj4/bPSVwOCAru9Y2P4hYx23MuvEIPacuPyNfD2cHfDp5ERnV0fCjblEVe9kQNl2Qot24WA+f7dzpTdi8u2NITedq8y5THBIYgJJKHScdOtDmstQdjpcw47qHuSWmckrqaSovBqzwhKi9fF0cWR2TC8mXheKo0Eu7LcHVgWZr68vBoOB7OzanaOzs7MJCAioc5mAgIAGy9f8m52dTWBgYK0ygwYNqrcuUVFRVFdXc+zYMXr37m3N22gd18/Sgmz3p3DTM5ceLuZnwor/A3MV9LkTRsyxTz1bUr9xsP1DxjjuZG2oFy6urrUO/bzdjHR2NeLTSfvXy2jG8cRWbfjwQ9/AiYuGRvK4CnrdAr1uRddtOA5OnbTb4WUkw5FNcGQTutMH6VqSRteSNG7hn1qD4m7DoftIKsNuIt85mLyyqvOHqiWVlkPYMyWV7D9ZwC+nS3jhqzT+tTWD535zNSN7+9nn8xONZlWQGY1GhgwZQmJiInFxcQCYzWYSExOZOXNmnctER0eTmJjIE088YZm2YcMGoqO1kRK6detGQEAAiYmJluAqLCxk27ZtPPbYY/XWZffu3ej1evz82ugfWch12lj2J7ZrncljEs7PqyiGTx+AktMQ0B/Gvq/1DrjSXBUJHlfhUHiCD6PzoM/1l5bJPw6Hv4BDG+CXb7WbqdTQO0BItCW86BJx6ZVco9u5+bdorwt+1driHdmk3W+z9Aykr4P0dRgBP88Q/HqMhB43Q68R4BJUa3XVJjPLdxznzxt+5sjpEqYs2cFNvbvw3Jg+9PRzt+nHI2xHp5R1/UhWrFjB5MmT+eCDD4iMjOStt95i5cqVHDx4EH9/fyZNmkTXrl2ZP38+oDW/GDFiBAsWLGDMmDEsX76cV199lV27dtGvXz8AXnvtNRYsWMDHH39Mt27deP7559mzZw9paWk4OzuTkpLCtm3bGDlyJO7u7qSkpDB79mxGjx7Nxx83rm9jYWEhnp6eFBQU4OHhYeXH1EQH1mgjujp5Qvx+cHLXGsqufBAOrgE3P3hkE3i10rk7e/jmOW2str5jtXHUTFVwfNu5va4NkJNWu3wn//PB1f0m7f4ITWU2Q9Yey94amVu1PeAaOj0EDdZCrcfNcNVQS3/WgrIq3t10iKXJx6gyKQx6HQ9eF8oTMb3k/FkLac5v1OogA3j33Xd5/fXXycrKYtCgQSxatIioqCgAbrrpJsLCwli6dKml/KpVq3juuec4duwYvXr1YuHChdx+++2W+UopEhIS+PDDD8nPz+eGG27gr3/9K+Hh2giku3bt4ne/+x0HDx6koqKCbt268eCDDxIfH9/o82B2CTKzGRZfC2cOw62vwLCZsOkV+G4hGIzw0FoIjmydutjLyVT48CbtimyvW7S9pYoLGsnq9NqeW68YLbwCBrRc+7nKEji2RdtTO7IJTl90gcrorh2GRozRHi5eHM0t4ZW1B9h4QDs9IufPWk6rB1l7ZJcgA200jK9+Dx5dYVQCrD53N/C492DQA61XD3tRCt4ZDHm/nJ/m6gM9zx0O9rgZXL3tU7eCX8+H2pEkKMs7P89ghJ4x2nm+8Nv4IbOcl9ekkZ6tDdPUo4ubnD+zMQmyRrBbkFWVw9sDoPiCCx7Dfg+3vtx6dbC39PWw429aj4Vet0LQNW3vnKDZDFk/wc/faCPcXri35uAC4bGYrr6blQURvL4p03LVc0S4dv6sl7+cP2suCbJGsFuQAXz/Z0g81zi3VyxM+BT0htatg7BOdpo2ptr+z2rvTRrdqew1mtWVkbyY5k+pyWA5fzZrVC86u8n5s6aSIGsEuwZZWT58MBxcOsPkr8C5lbcvmk4pOLVbC7V9q6HwhGWWycmLLcZhvH9mENvMfejk4swTMb34v/Z2/qyqDIpOQVEWuAeCdze7VEOCrBHsGmSg/SCUanuHVKLxzGbtdnz7/qvdv7Pk/EAJZ3VefFl1LV+Zosn3uYZn7+hn//NnZhMU55wLqXOPwnOBVXTy3PNT2kAFFjoYNBFuflbrbteKJMgawe5BJq4sZhNkbNFCLe0LKDtrmXVSebPWdB2/dr2diWPvoleADf/eTNXaXearyrU2ckUntWAqPHl+r6rw3LTibG1IqMZwcIFOXbSG2jWvh83UGnY7tc75PwmyRpAgEy3GVKU1K9n3X9SBNegqz9+AJkP5kRlwG/7BPdCZKtCbytFVV6CvLkNnqkRvKj/3qLjg3woM554bzBUYzr02mCvRq2rr6qbTQ6cAcA/Q9rDcA7TDR/dA8AgE93PTnD21Zi/Hd2ht/45v1ZZ36wIjn4FrJrX4Xe0lyBpBgky0iqpyOLyR4l0rcDz8DU6q/PLLNFG+ciOHzlS4+OPs3ZUuQd3w8g85H1TugdDJz/oLS0rBga9gY8L5Cx2+veGWlyA8tsXa+UmQNYIEmWh1lSWkf7eKMzs/h+oyqnROVOqMVOmMVOmcqNI7af/qHKnSO1Gtd9Lm6Z2o1jlRrTdq0/TOmPRGTHonqg1OVOudqcaR1BOF/JpfVmuT3X3duKm3HyMjuhDZzRsnh2ZcHa+u1O4dsXnB+TZ2YTfCrX9qkVFaJMgaQYJMXGmUUhw5XUzSwdNsOpjDjmN5VF8wGq+Lo4Hre/oyMqILN/X2o6uXS9M2VJYPP/wFtr4HpnMjkAy4D25+3qbd6yTIGkGCTFzpisqr2HI4l6SDp0lKzyGnqKLW/N7+7twU0YWRvf0YEtrZ+iYi+ZmQ+DLsXam9NjhB9O/ghtnN6xN7jgRZI0iQiY5EKUXaqUI2p58m6WAOuzLPcuGtE9ydHLgx3JeRvf0Y0bsLfu7OjV/5r7vgm+ch4wfttauPNp7e0CnNuom0BFkjSJCJjuxsSSXfHTrN5vTTfPvz6UsGluzf1ZORvbsQ1d2Hbr5uBHg4NzwkuFLw83ptmPbcn7Vp3j20CwIRY5p0QUCCrBEkyITQmMyKPSfySUo/zeb0nDpH7nVy0BPm40aYr+u5f90I83Gjm68b/h5O6GqCylQNuz6GpFehNFebFhKtXRC4aqhV9ZIgawQJMiHqllNUzrfp2t7agVOFZOaV1rpocDEXRwOhPq5089UCrpuPG909zPT55SNcd36ArvrcldR+42DUPOgc1qh6SJA1ggSZEI1TbTLza34ZR3NLOJZbwrEzpdrzMyWcOFvW4H1Kuxvzmev8GaMqE9GjMOkdOd1nMh63zsXV07fB7UqQNYIEmRDNV1lt5sTZUo6dKeFobum5oCvhaG4JJ/PLLBcUrtYdY67DMm40aDfiTh/xV3qPnNjgupvzG203d1ESQtif0UFP9y6d6F7HPUorqk0czyvlWG4px870Yf3pG0k++S1Xn91M5OC7W7ReEmRCCJtwcjDQ08/9opu0DAAeb/Fty5gyQoh2T4JMCNHuSZAJIdo9CTIhRLsnQSaEaPckyIQQ7Z4EmRCi3esw7chqOjAUFhbauSZCiLrU/Dab0tmowwRZUZF2Q4jgYNuNaCmEsL2ioiI8Pa0bqLHD9LU0m82cPHkSd3f380OQ1KOwsJDg4GCOHz9+xfTLvNLe05X2fkDek1KKoqIigoKC0Ft5/9cOs0em1+u56qqrrFrGw8PjivmDqnGlvacr7f1Ax35P1u6J1ZCT/UKIdk+CTAjR7kmQ1cHJyYmEhAScnJzsXRWbudLe05X2fkDeU3N0mJP9Qogrl+yRCSHaPQkyIUS7J0EmhGj3JMiEEO2eBJkQot3rsEG2ePFiwsLCcHZ2Jioqiu3btzdYftWqVURERODs7Ez//v1Zt25dK9X08ubPn8+1116Lu7s7fn5+xMXFkZ6e3uAyS5cuRafT1Xo4Ozu3Uo0v74UXXrikfhEREQ0u05a/I4CwsLBL3pNOp2PGjBl1lm9r39F3333HHXfcQVBQEDqdjs8//7zWfKUU8+bNIzAwEBcXF2JiYjh06NBl12vtb7EuHTLIVqxYQXx8PAkJCezatYuBAwcSGxtLTk5OneWTk5OZMGECU6dOJTU1lbi4OOLi4ti3b18r17xu3377LTNmzGDr1q1s2LCBqqoqbr31VkpKShpczsPDg1OnTlkeGRkZrVTjxunbt2+t+v3www/1lm3r3xHAjh07ar2fDRs2AHDvvffWu0xb+o5KSkoYOHAgixcvrnP+woULWbRoEe+//z7btm3Dzc2N2NhYysvL612ntb/FeqkOKDIyUs2YMcPy2mQyqaCgIDV//vw6y48fP16NGTOm1rSoqCg1ffr0Fq1nU+Xk5ChAffvtt/WWWbJkifL09Gy9SlkpISFBDRw4sNHl29t3pJRSs2bNUj169FBms7nO+W35OwLU6tWrLa/NZrMKCAhQr7/+umVafn6+cnJyUp9++mm967H2t1ifDrdHVllZyc6dO4mJibFM0+v1xMTEkJKSUucyKSkptcoDxMbG1lve3goKCgDw9vZusFxxcTGhoaEEBwdz1113sX///taoXqMdOnSIoKAgunfvzsSJE8nMzKy3bHv7jiorK/n3v//Nb3/72wZHY2nr31GNo0ePkpWVVes78PT0JCoqqt7voCm/xfp0uCDLzc3FZDLh7+9fa7q/vz9ZWVl1LpOVlWVVeXsym8088cQTXH/99fTr16/ecr179+ajjz7iiy++4N///jdms5lhw4Zx4sSJVqxt/aKioli6dCnr16/nvffe4+jRo9x4442WceUu1p6+I4DPP/+c/Px8HnrooXrLtPXv6EI1n7M130FTfov16TDD+HQUM2bMYN++fQ2eTwKIjo4mOjra8nrYsGH06dOHDz74gJdffrmlq3lZo0ePtjwfMGAAUVFRhIaGsnLlSqZOnWrHmtnGP/7xD0aPHk1QUFC9Zdr6d9SWdLg9Ml9fXwwGA9nZ2bWmZ2dnExAQUOcyAQEBVpW3l5kzZ7JmzRqSkpKsHnvN0dGRa665hsOHD7dQ7ZrHy8uL8PDweuvXXr4jgIyMDDZu3MjDDz9s1XJt+Tuq+Zyt+Q6a8lusT4cLMqPRyJAhQ0hMTLRMM5vNJCYm1vrf70LR0dG1ygNs2LCh3vKtTSnFzJkzWb16NZs2baJbt25Wr8NkMrF3714CAwNboIbNV1xczJEjR+qtX1v/ji60ZMkS/Pz8GDNmjFXLteXvqFu3bgQEBNT6DgoLC9m2bVu930FTfov1surSwBVi+fLlysnJSS1dulSlpaWpadOmKS8vL5WVlaWUUurBBx9UTz/9tKX8li1blIODg3rjjTfUgQMHVEJCgnJ0dFR79+6111uo5bHHHlOenp5q8+bN6tSpU5ZHaWmppczF7+nFF19UX3/9tTpy5IjauXOnuv/++5Wzs7Pav3+/Pd7CJf7whz+ozZs3q6NHj6otW7aomJgY5evrq3JycpRS7e87qmEymVRISIiaM2fOJfPa+ndUVFSkUlNTVWpqqgLUn//8Z5WamqoyMjKUUkotWLBAeXl5qS+++ELt2bNH3XXXXapbt26qrKzMso6bb75ZvfPOO5bXl/stNlaHDDKllHrnnXdUSEiIMhqNKjIyUm3dutUyb8SIEWry5Mm1yq9cuVKFh4cro9Go+vbtq9auXdvKNa4fUOdjyZIlljIXv6cnnnjC8v79/f3V7bffrnbt2tX6la/HfffdpwIDA5XRaFRdu3ZV9913nzp8+LBlfnv7jmp8/fXXClDp6emXzGvr31FSUlKdf2c1dTabzer5559X/v7+ysnJSY0aNeqS9xkaGqoSEhJqTWvot9hYMh6ZEKLd63DnyIQQVx4JMiFEuydBJoRo9yTIhBDtngSZEKLdkyATQrR7EmRCiHZPgkwI0e5JkAkh2j0JMiFEuydBJoRo9/4fc+uEG2GqACkAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:11:48.039593Z", + "iopub.status.busy": "2024-03-23T06:11:48.039236Z", + "iopub.status.idle": "2024-03-23T06:16:08.303401Z", + "shell.execute_reply": "2024-03-23T06:16:08.302174Z" + }, + "papermill": { + "duration": 260.284912, + "end_time": "2024-03-23T06:16:08.306020", + "exception": false, + "start_time": "2024-03-23T06:11:48.021108", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:16:08.343916Z", + "iopub.status.busy": "2024-03-23T06:16:08.343316Z", + "iopub.status.idle": "2024-03-23T06:16:08.364777Z", + "shell.execute_reply": "2024-03-23T06:16:08.363869Z" + }, + "papermill": { + "duration": 0.04325, + "end_time": "2024-03-23T06:16:08.367118", + "exception": false, + "start_time": "2024-03-23T06:16:08.323868", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.0092950.0442240.00256312.520720.0312730.8461190.0439980.0000073.9416030.0374620.1135690.0506240.102450.00610916.462322
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.009295 0.044224 0.002563 12.52072 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.031273 0.846119 0.043998 0.000007 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 3.941603 0.037462 0.113569 0.050624 0.10245 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.006109 16.462322 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:16:08.405034Z", + "iopub.status.busy": "2024-03-23T06:16:08.404241Z", + "iopub.status.idle": "2024-03-23T06:16:08.777407Z", + "shell.execute_reply": "2024-03-23T06:16:08.776483Z" + }, + "papermill": { + "duration": 0.393214, + "end_time": "2024-03-23T06:16:08.779613", + "exception": false, + "start_time": "2024-03-23T06:16:08.386399", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:16:08.815397Z", + "iopub.status.busy": "2024-03-23T06:16:08.815075Z", + "iopub.status.idle": "2024-03-23T06:20:38.512427Z", + "shell.execute_reply": "2024-03-23T06:20:38.511395Z" + }, + "papermill": { + "duration": 269.718079, + "end_time": "2024-03-23T06:20:38.515093", + "exception": false, + "start_time": "2024-03-23T06:16:08.797014", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:20:38.551763Z", + "iopub.status.busy": "2024-03-23T06:20:38.551407Z", + "iopub.status.idle": "2024-03-23T06:20:38.578575Z", + "shell.execute_reply": "2024-03-23T06:20:38.577590Z" + }, + "papermill": { + "duration": 0.048154, + "end_time": "2024-03-23T06:20:38.580791", + "exception": false, + "start_time": "2024-03-23T06:20:38.532637", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:20:38.615312Z", + "iopub.status.busy": "2024-03-23T06:20:38.614988Z", + "iopub.status.idle": "2024-03-23T06:20:38.620736Z", + "shell.execute_reply": "2024-03-23T06:20:38.619807Z" + }, + "papermill": { + "duration": 0.02595, + "end_time": "2024-03-23T06:20:38.622954", + "exception": false, + "start_time": "2024-03-23T06:20:38.597004", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.38102179209391274}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:20:38.659411Z", + "iopub.status.busy": "2024-03-23T06:20:38.659091Z", + "iopub.status.idle": "2024-03-23T06:20:39.094482Z", + "shell.execute_reply": "2024-03-23T06:20:39.093517Z" + }, + "papermill": { + "duration": 0.456354, + "end_time": "2024-03-23T06:20:39.096686", + "exception": false, + "start_time": "2024-03-23T06:20:38.640332", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABKKklEQVR4nO3deXxTVd4/8M/NnqZJum90ZSs7FCgIiiAyLDIK6uO4oFIVdUZ4GOVhXiPz/AZEZx5wRkVcYBxfAjIuzOjgMoOKDLLMuICA7Eih0IUudE+aZs89vz9ukzY0bZM0zW3T7/v1Cm3uPbk5t6Hfnv1wjDEGQggRkUTsDBBCCAUiQojoKBARQkRHgYgQIjoKRIQQ0VEgIoSIjgIRIUR0FIgIIaKjQEQIER0Foj4mOzsbP/3pT3v0PTiOwzPPPNNlumeeeQYcx/VoXkj/QIEozL755hs888wzaGxsFDsrpA/YtGkTtm3bJnY2ehwFojD75ptvsHbtWgpExC8UiAghJEwoEIXRM888g1/96lcAgJycHHAcB47jUFxcjK1bt2LmzJlISkqCUqnEiBEjsHnz5g6v9eWXX2LcuHFQqVQYMWIEdu7cGXB+bDYbnnrqKSQmJkKr1eK2227DlStXfKb9z3/+g/z8fKhUKgwaNAhvvPGGz3Qcx2HZsmV49913kZubC5VKhQkTJuDgwYPtfhYcx6GwsBD3338/9Ho9EhMT8dvf/haMMZSVlWHBggXQ6XRISUnBiy++GPD9AcDnn3+O6dOnQ6vVQqfTIT8/H++9955Xmg8++AATJkyAWq1GQkIC7r//fpSXl3ulKSgoQHR0NMrLy7Fw4UJER0cjMTERK1euhMvl8krL8zw2btyI0aNHQ6VSITExEXPnzsWRI0c8afz5vLOzs3HmzBkcOHDA839lxowZQf0cej1GwubEiRPs3nvvZQDYhg0b2F/+8hf2l7/8hZlMJpafn88KCgrYhg0b2Kuvvspmz57NALDXXnvN6xpZWVls6NChLCYmhj399NPspZdeYqNHj2YSiYR9+eWXAeXn/vvvZwDYfffdx1577TV2xx13sDFjxjAAbM2aNZ50J0+eZGq1mmVmZrJ169ax5557jiUnJ3vStgWAjRo1iiUkJLBnn32WPf/88ywrK4up1Wp26tQpT7o1a9YwAGzcuHHs3nvvZZs2bWLz589nANhLL73EcnNz2S9+8Qu2adMmdv311zMA7MCBAwHd39atWxnHcWzUqFHs97//PXv99dfZkiVL2AMPPOCVBgDLz89nGzZsYE8//TRTq9UsOzubNTQ0eNItXryYqVQqNnLkSPbwww+zzZs3szvvvJMBYJs2bfJ634KCAgaAzZs3j7388svshRdeYAsWLGCvvvqqJ40/n/dHH33E0tPT2bBhwzz/VwL9jPsKCkRh9sc//pEBYJcvX/Y6bjab26WdM2cOGzhwoNexrKwsBoD9/e9/9xwzGAwsNTWV5eXl+Z2P48ePMwDsiSee8Dp+3333tQtECxcuZCqVipWUlHiOnT17lkmlUp+BCAA7cuSI51hJSQlTqVTs9ttv9xxzB6LHHnvMc8zpdLL09HTGcRxbv36953hDQwNTq9Vs8eLFft9fY2Mj02q1bPLkycxisXid43meMcaY3W5nSUlJbNSoUV5p/vnPfzIAbPXq1Z5jixcvZgDYs88+63WtvLw8NmHCBM/zr776igFgy5cvb5cn9/sy5v/nPXLkSDZ9+nQ/7rhvo6pZL6FWqz3fGwwG1NbWYvr06bh06RIMBoNX2rS0NNx+++2e5zqdDg8++CB++OEHVFVV+fV+n332GQBg+fLlXseffPJJr+culwu7d+/GwoULkZmZ6Tk+fPhwzJkzx+e1p0yZggkTJnieZ2ZmYsGCBdi9e3e7asySJUs830ulUkycOBGMMTzyyCOe4zExMcjNzcWlS5f8ujcA2LNnD5qamvD0009DpVJ5nXMPOThy5Aiqq6vxxBNPeKWZP38+hg0bhl27drW77s9//nOv59OmTfPK19///ndwHIc1a9a0e23boQ6BfN79AQWiXuLrr7/GrFmzoNFoEBMTg8TERPzmN78BgHb/MQcPHtxu/M7QoUMBAMXFxX69X0lJCSQSCQYNGuR1PDc31+t5TU0NLBYLhgwZ0u4a16Z185V26NChMJvNqKmp8TreNrgBgF6vh0qlQkJCQrvjDQ0NHd/QNYqKigAAo0aN6jBNSUkJAN/3MWzYMM95N3d7T1uxsbFe+SoqKkJaWhri4uI6zV8gn3d/IBM7A0T4z3vzzTdj2LBheOmll5CRkQGFQoHPPvsMGzZsAM/zYmexx0ilUr+OAQATeVXjjvIVqP78eXeEAlGY+RqJ/I9//AM2mw2ffvqpVwlh3759Pq9x8eJFMMa8rlVYWAhA6GnxR1ZWFnieR1FRkVeJ4Pz5817pEhMToVarceHChXbXuDatm6+0hYWFiIqKalei6Cnukt7p06cxePBgn2mysrIACPcxc+ZMr3Pnz5/3nA/0fXfv3o36+voOS0WBfN79ZeQ6Vc3CTKPRAIDXgEb3X9q2f/ENBgO2bt3q8xoVFRX46KOPPM+NRiO2b9+OcePGISUlxa98zJs3DwDwyiuveB1/+eWXvZ5LpVLMmTMHH3/8MUpLSz3Hz507h927d/u89rfffotjx455npeVleGTTz7B7NmzQ1aq6Mrs2bOh1Wqxbt06WK1Wr3Pun/PEiRORlJSEP/3pT7DZbJ7zn3/+Oc6dO4f58+cH/L533nknGGNYu3Ztu3Pu9w3k89ZoNP1i8CuViMLM3Yj7v//7v7jnnnsgl8tx4403QqFQ4NZbb8Xjjz8Ok8mEN998E0lJSaisrGx3jaFDh+KRRx7B999/j+TkZGzZsgVXr17tMHD5Mm7cONx7773YtGkTDAYDpk6dir179+LixYvt0q5duxZffPEFpk2bhieeeAJOpxOvvvoqRo4ciZMnT7ZLP2rUKMyZMwfLly+HUqnEpk2bPNcJF51Ohw0bNmDJkiXIz8/Hfffdh9jYWJw4cQJmsxlvv/025HI5nn/+eTz00EOYPn067r33Xly9ehUbN25EdnY2nnrqqYDf96abbsIDDzyAV155BRcuXMDcuXPB8zz+/e9/46abbsKyZcswe/Zsvz/vCRMmYPPmzfjd736HwYMHIykpqV3pLSKI12HXfz333HNswIABTCKReLryP/30UzZmzBimUqlYdnY2e/7559mWLVvadfVnZWWx+fPns927d7MxY8YwpVLJhg0bxj744IOA82GxWNjy5ctZfHw802g07NZbb2VlZWXtuu8ZY+zAgQNswoQJTKFQsIEDB7I//elPni74tgCwpUuXsnfeeYcNGTKEKZVKlpeXx/bt2+eVzv3ampoar+OLFy9mGo2mXV6nT5/ORo4cGfA9fvrpp2zq1KlMrVYznU7HJk2axN5//32vNH/9619ZXl4eUyqVLC4uji1atIhduXLFr3z5+hk4nU72xz/+kQ0bNowpFAqWmJjI5s2bx44ePeqVL38+76qqKjZ//nym1WoZgIjtyucYo33NSOhwHIelS5fitddeEzsrpA+hNiJCiOiojSgCdTWoUa1WQ6/Xhyk3oVdTU9NuYGRbCoWiy3E8pHehQBSBUlNTOz2/ePHiPr20RH5+frvBhm1Nnz4d+/fvD1+GSLdRIIpAe/bs6fR8Wlpaj713OJoc3333XVgslg7Px8bG9ngeSGhRYzUhRHTUWE0IEV2frprxPI+Kigpotdp+MxSekL6EMYampiakpaVBIum43NOnA1FFRQUyMjLEzgYhpAtlZWVIT0/v8HyfDkRarRaAcJM6nU7k3BBCrmU0GpGRkeH5Xe1Inw5E7uqYTqejQERIL9ZV0wk1VhNCREeBiBAiOgpEhBDR9ek2In8wxuB0Ojudm0R8k0qlkMlkNDSC9LiIDkR2ux2VlZUwm81iZ6XPioqKQmpqKhQKhdhZIREsYgMRz/O4fPkypFIp0tLSoFAo6C97ABhjsNvtqKmpweXLlzFkyJBOB6QR0h0RG4jsdjt4nkdGRgaioqLEzk6fpFarIZfLUVJSArvd3m5/sL7GaHXgYGENNAoZbhiSALmUAmtvEfGfBP0V755I+vntPXcVF66aUFLXjIZmu9jZIW1EbImIkLYazXYU1wpthXdMSIdOJRc5R6StyPlzR0gniuuEIJQRF0VBqBeiQNTPZWdnt9vLLBKVNwgLqWXECnvOu3gGq4OGdPQWFIhIxGOMoaJRCERpMWqcLjdg076L2Pdjtcg5I24UiCKA3U4Nr51ptrtgsjkh4Tik6FWIVsrg5Bmqm2xdv5iERb8MRHYn3+HD6eL9TuvwI20wZsyYgWXLlmHZsmXQ6/VISEjAb3/7W8960NnZ2Xjuuefw4IMPQqfT4bHHHgMA/Oc//8G0adOgVquRkZGB5cuXo7m52XPd6upq3HrrrVCr1cjJycG7774bVP76GinHYdqQBIzPioFcKkGCVgkAaDDb233eRBz9stfs9X3tt1V2y0nQYGHeAM/zPx8sgsPle1nv9Fg17prYujDblq8vw2L3bnd46idDg8rj22+/jUceeQSHDx/GkSNH8NhjjyEzMxOPPvooAOCFF17A6tWrsWbNGgBAUVER5s6di9/97nfYsmULampqPMHMvRV1QUEBKioqsG/fPsjlcixfvhzV1ZFfPVErpJiY3bq9kEYhhUImgd3Jw2BxID5aKWLuCNBPA1FfkJGRgQ0bNoDjOOTm5uLUqVPYsGGDJxDNnDkT//M//+NJv2TJEixatAhPPvkkAGDIkCF45ZVXMH36dGzevBmlpaX4/PPPcfjwYeTn5wMA3nrrLQwfPjzs9yY2juOgV8tR02RDIwWiXqFfBqKlNw3u8Jzkmlkgj904qMO0184Yefj6nO5ky8t1113nNSVlypQpePHFFz2TdydOnOiV/sSJEzh58qRXdYsx5pnqUlhYCJlMhgkTJnjODxs2DDExMSHLc291pcEMpUyK2Cg5ZC2jqWOjFEIgMlP7Wm/QLwORQuZ/01hPpe0ujUbj9dxkMuHxxx/H8uXL26XNzMxEYWFhuLLW63x2qhLNNhfum5yJZJ0wTSUmShhL1Gh2iJk10qJfBqK+4NChQ17Pv/vuOwwZMgRSqdRn+vHjx+Ps2bMYPNh3aW/YsGFwOp04evSop2p2/vx5NDY2hjTfvY3DxaPZJpQi2w5kTItRIzdF6wlMRFz9stesLygtLcWKFStw/vx5vP/++3j11Vfxy1/+ssP0v/71r/HNN99g2bJlOH78OC5cuIBPPvkEy5YtAwDk5uZi7ty5ePzxx3Ho0CEcPXoUS5YsgVqtDtcticJoEUo8CpkEKnnrf/ecBA1uGZ2KUQP0YmWNtCF6ICovL8f999+P+Ph4qNVqjB49GkeOHBE7W6J78MEHYbFYMGnSJCxduhS//OUvPd30vowZMwYHDhxAYWEhpk2bhry8PKxevdpre+mtW7ciLS0N06dPxx133IHHHnsMSUlJ4bgd0RitTgCATi2nZWB6MVGrZg0NDbj++utx00034fPPP0diYiIuXLhAe5cDkMvlePnll7F58+Z254qLi32+Jj8/H19++WWH10xJScE///lPr2MPPPBAt/LZ2xlaSkR6dfv5ZTzP0Gx3QqOQQXJtLwUJK1ED0fPPP4+MjAzPOBcAyMkJXc8TIe6qmU7V/r/6n/99CRa7C4unZiNOQytQiknUqtmnn36KiRMn4q677kJSUhLy8vLw5ptvdpjeZrPBaDR6PQjpjNHacYlIoxAa/k0t1TciHlED0aVLl7B582YMGTIEu3fvxi9+8QssX74cb7/9ts/069atg16v9zwidbvp/fv394sZ8eEwKk2PaUMSkB7bfpXO6JZSUpONuvDFxjH3BCYRKBQKTJw4Ed98843n2PLly/H999/j22+/bZfeZrPBZmudqOjeztZgMLTb6dVqteLy5cvIycnp80uciimSf457zl7F6XIDpg6Kx+SB8WJnJyIZjUbo9Xqfv6NtiVoiSk1NxYgRI7yODR8+HKWlpT7TK5VKz/bStM006S6NsqVqZqOqmdhEDUTXX389zp8/73WssLAQWVlZIuWIRBK7k8elGhOqm6w+z0crhapZs50WSBObqIHoqaeewnfffYf/+7//w8WLF/Hee+/hz3/+M5YuXSpmtkiEaDTb8cnxCnzyQ4XP81EtjdUWO5WIxCZqIMrPz8dHH32E999/H6NGjcJzzz2Hl19+GYsWLRIzWyRCNLVUuTRK36NU9GoFhqVokZMQHc5sER9En2v205/+FD/96U/FzgaJQM2eQOR7fl6iVol5o1PDmSXSAdGneBDSU9yN0FofgxlJ70KBiEQs96x7jaLjQOTiGUw2J1y8aKNYCCgQkQhmahmo2FEbEQBs/foy3jx4CbUmWkhfTP0rEDEGOO3hfwQwZnT79u2Ij4/3GrgJAAsXLoz4CaqhZmopEUV3EojULT1nZurCF1X/qjy7HMC/Xwz/+077H0Dm36TKu+66C8uXL8enn36Ku+66C4Cw+8auXbs6nVlP2psyMA6NZgfiozv+2bu78JtpUKOo+leJqA9Qq9W47777vFYkeOedd5CZmYkZM2aIl7E+aHCSFhOz46DtZItptVz4W2yhXV9F1b9KRFK5UDoR430D8OijjyI/Px/l5eUYMGAAtm3bhoKCAlrYqwe4u/apaiau/hWIOM7vKpKY8vLyMHbsWGzfvh2zZ8/GmTNnsGvXLrGz1adY7C5UGa2IVsqQqO14uyAaXd079K9A1IcsWbIEL7/8MsrLyzFr1qyIXfKkp1w1WvHxD+VI0CrxwHUdz110V82oRCQuaiPqpe677z5cuXIFb775Jh5++GGxs9PnuANLlNz3qGq3OI0Cw1O1yIpvv14RCR8KRL2UXq/HnXfeiejoaCxcuFDs7PQ57sZnd9WrIyl6FeaOSsWErLhO05GeRYGoFysvL8eiRYugVNKWyIGytJSI1F0EItI7UBtRL9TQ0ID9+/dj//792LRpk9jZ6ZPMLY3PUZ1M73BzunhYnTw0Cin1TIqEAlEvlJeXh4aGBjz//PPIzc0VOzt9kr9VM8YYXtt3EYwBj904sNPpIKTn0E+9F+po3zLiP3djtaqLxmqO46CUSWF1uGB1uCgQiYR+6iQi5WfHwmBxdDqGyE0llwiByMmHIWfEl4gPRCJuUhIR+urPb3CS1u+0arkUjXB4GrhJ+EVsr5lcLkyrMJvNIuekb3P//Nw/z0jkrr5Zab6ZaCK2RCSVShETE4Pq6moAQFRUFPWIBIAxBrPZjOrqasTExEAq7Tvd4DanC+UNFkQrZUjSdb0Xm0ou8byOiCNiAxEApKSkAIAnGJHAxcTEeH6OfUVDswOfHK+AViXDkmkDu0zvLhFZ7NRGJJaIDkQcxyE1NRVJSUlwOGhb4UDJ5fI+VRJyC2QMESCMrh6eqvWrYZv0jIgORG5SqbRP/kKR4Hjmmfk5qnpYig7DUmjXYDGJ2lj9zDPPgOM4r8ewYcPEzBKJAO7BjDS9o+8QvUQ0cuRI/Otf//I8l8lEzxLp4wItEQHCNA+7i/e7OkdCS/Sfukwm87sx1GazeS0qbzQaeypbpA+zeNqI/AtE1UYr3j1U6nfjNgk90ccRXbhwAWlpaRg4cCAWLVqE0tLSDtOuW7cOer3e86DFwogv7qpZV9M73JSeXjPqvhcLx0QcOvv555/DZDIhNzcXlZWVWLt2LcrLy3H69Glote1HxvoqEWVkZMBgMECno8ZGIrhc24z6ZjtyEjSI03S9NLDN6cKmfUUAgGUzB0MuFf3vc8QwGo3Q6/Vd/o6KWjWbN2+e5/sxY8Zg8uTJyMrKwt/+9jc88sgj7dIrlUpam4d0KSdBg5wEjd/pFVIJOE7Yfs7m5CkQiaBX/cRjYmIwdOhQXLx4UeyskH7EPQMfAGw0zUMUvSoQmUwmFBUVITU1VeyskD7K6eJxqcaEKoM1oAm77mkeNANfHKIGopUrV+LAgQMoLi7GN998g9tvvx1SqRT33nuvmNkifViT1YlPjlfg78euBDS3kEpE4hK1jejKlSu49957UVdXh8TERNxwww347rvvkJiYKGa2SB9m9nNlxmtlx0chTqOghdFEIupPfceOHWK+PYlAnkXz/ey6d5s6OKEnskP81KvaiAjpLtq9o2+iQEQiSqAz79tyunjYqbFaFBSISETxq43IVA3UFQGu1v3uvy+ux6tfXcT+87R2lRioZY5ElC6rZpf2AyXfCt9rEoCx9wLKaCik7lUaqUQkBioRkYgyKk2PG4cmIj1W3f7k1bOtQUgqA5prgR93AYzRutUio0BEIkpmfBQmZMUiSXvNWtW8SygNAUDWVGDCQ4BECtRfAhqKoZRRiUhMFIhI/3D1DGA1AAqNEIg0CUDqOOFc+VEo5RSIxESBiEQMnmcoqjGh0mABz18zvaPqpPA1fSIgbdkaacAE4WtdEVRMWNWBqmbioEBEIobV6cKnxyuw43AZvMKQpQFoLAM4Dkge1XpcEw9EJwGMh8p4CQBgd/LtgxjpcRSISMRou9+9VNJmnllNofA1JhNQXbMmTqKwRrqy4QIGJmowPFUHVx/d3bYvo+57EjEsHa1VXS8seoaEoe1fFD8YuHwQEkMZFtyQIjRgk7CjEhGJGD5373BYhWoZAMT5WI86OgmQqwGXA2iqDEMuiS8UiEjE8Ll7R0MxwHggKh6Iimv/Io4DYrM8aZ0uHi5qIwo7CkQkYrjnmXnNvG9s2YwhLqfjF+ozAQCHT5zGq19dxKUaU09lkXSAAhGJGD6ndxhaqmX6TnZ80Qkrgmrs1QBjsDpoLFG4UWM1iRi5KVrEahRI0bWMqnZYgOYa4Xt9escv1CQBEimUvAlKpxE2Jy3MF24UiEjESI+NQnpsVOsBQ7mwNUdUHKCM7viFUhkQnQSp1Ihoew1szk6qcaRHUNWMRC5DS/tQZ9UyN20aZBIO0fZq2Jw0ujrcKBCRiHG5thmVBktrr5exQvjaWbXMTZsCmYSDxl5HbUQioEBEIoKLZ/j4h3LsOFwmrLLI80BTlXBS68f2VNFJkEo4RDnqYXM4u05PQoraiEhEcHfdSzhO2KPMXCcMUpTKhDFEXYmKh0ohQ6LKDqWGSkTh1mtKROvXrwfHcXjyySfFzgrpg1q77iXCfmbuUdLRKYDEj//mUjl0sckYlqLDpAQqEYVbrwhE33//Pd544w2MGTNG7KyQPsrsCUQthfxAqmVu0S3d9s21IcwZ8YfogchkMmHRokV48803ERsbK3Z2SB/lDkQa92BGd4lIm+L/RTRCIHI1XQ1ou2rSfaIHoqVLl2L+/PmYNWtWl2ltNhuMRqPXgxAAsDjc2whJWxqqrwonAigR2VUJOHS5DvuOnYWT5puFVVCB6NKlSyF58x07duDYsWNYt26dX+nXrVsHvV7veWRk+DE+hPQLXlUzcy3AOwGZwvdE1w7IdUlgDFA7GmC1UztROAUViAYPHoybbroJ77zzDqxWa1BvXFZWhl/+8pd49913oVKpun4BgFWrVsFgMHgeZWVlQb03iTyDk6IxPTcRAxM0re1D0SnC7Ho/caoYSGQySJgL9ubGnsko8SmoQHTs2DGMGTMGK1asQEpKCh5//HEcPnw4oGscPXoU1dXVGD9+PGQyGWQyGQ4cOIBXXnkFMpkMLlf70a1KpRI6nc7rQQgApOrVGJ8Zi4y4KKC5ZZPE6OTALiKRwKXUAwAcproQ55B0JqhANG7cOGzcuBEVFRXYsmULKisrccMNN2DUqFF46aWXUFNT0+U1br75Zpw6dQrHjx/3PCZOnIhFixbh+PHjkEpppTwSJFPL/z9NQsAv5ZVCVc7Z1PX/YRI63WqslslkuOOOO/DBBx/g+eefx8WLF7Fy5UpkZGTgwQcfRGVlxyveabVajBo1yuuh0WgQHx+PUaNGdfg6Qnwprm1GRaMFThffpkSUFPB1+JY2Jb65PpTZI13oViA6cuQInnjiCaSmpuKll17CypUrUVRUhD179qCiogILFiwIVT4J6RBjDP84UYG/fl+GZpMBsJuFtqGowEtEULsDEY0lCqegpni89NJL2Lp1K86fP49bbrkF27dvxy233AJJywjWnJwcbNu2DdnZ2QFdd//+/cFkh/RzNifv6W6PcjQIB1UxQq9ZgPTxyVCUy6HhaWhIOAUViDZv3oyHH34YBQUFSE31PU4jKSkJb731VrcyR4g/3NM7FDIJ5JaWkkx0cIubjRkyEKjWAbABTntQwYwELqhAtGfPHmRmZnpKQG6MMZSVlSEzMxMKhQKLFy8OSSYJ6YzZ0WbRfPeKjJrA24cAAIooQK4Sdv+wNADaAHveSFCCaiMaNGgQamvb16Hr6+uRk0Or25HwstjbjKo2Bd9Q7REVL0zxMFMXfrgEFYg6modjMpn8HpxISKh4RlXLJa0TVjXBVc0u1Ziw64IVZyuNgLUxRDkkXQmoarZixQoAAMdxWL16NaKiWtcHdrlcOHToEMaNGxfSDBLSFXcg0rMmYWqHVCY0VgdBJpGgWaqF086EqhkJi4AC0Q8//ABAKBGdOnUKCkVrQ55CocDYsWOxcuXK0OaQkC7kJGiglEmQbCsGGiGUhvxZg8gHpVwCm0wHp4sBlsYQ5pJ0JqBAtG/fPgDAQw89hI0bN9IUC9IrJOtUSNapgMunhANBVssAQCWTwirTwcXzVDULo6B6zbZu3RrqfBDSfe6G6mB7zCCUiKwyHVwMcFmMkLpaqnqkR/n9E77jjjuwbds26HQ63HHHHZ2m3blzZ7czRoi/SuvMkEk5JJtqIAWCHkMEAEqZBC6pCi5ODhfPQ2o1ABo/1rwm3eJ3INLr9cJawC3fE9Jb7DpVCbvNikdRiyiFrFtVM47joJBLW9qJnFBYGykQhYHfgahtdYyqZqS3cLp4WB0uRDvqIVdJAIVGeHRDVpwGOlMiJFwV9ZyFSVCVX4vFAsaYp/u+pKQEH330EUaMGIHZs2eHNIOEdKa5pes+2tUImZTr3kDGFvPHpAJROUBZDfWchUlQfZwLFizA9u3bAQCNjY2YNGkSXnzxRSxYsACbN28OaQYJ6Yx7P7NYvgEcuKDWIPJJ3bKRA/WchUXQKzROmzYNAPDhhx8iJSUFJSUl2L59O1555ZWQZpCQzjTbhEAUwxqFA93oMfOijgEDDWoMl6CqZmazGVqtFgDw5Zdf4o477oBEIsF1112HkpKSkGaQkM4021wAY9C5GgHIu9VQ7XagsAbnL9XjFrMF6ZJGgLGA1r4mgQt68fyPP/4YZWVl2L17t6ddqLq6mgY5krBqtjsh5y1QwSYEixBUzTgAZokGTsYJU0bspu5nlHQqqEC0evVqrFy5EtnZ2Zg8eTKmTJkCQCgd5eXlhTSDhHRmUGI0bkoH4jVKoV1HKu/2NVVyKRgnhVXa0vtG1bMeF1TV7L/+679www03oLKyEmPHjvUcv/nmm3H77beHLHOEdCVZp0Ky3g6oQ1MtA4RBjQBglmoBGISes5jMkFyb+Bb02PWUlBSkpHhv5ztp0qRuZ4iQgHmmdoQmEKnkwg4yZok7EFGJqKcFFYiam5uxfv167N27F9XV1eB53ut8qHaCJaQrl2pMiKutgI4xSEJcIjJJhA4ZWA0huS7pWFCBaMmSJThw4AAeeOABpKameqZ+EBJOPM/w6fEryC+7hPxMLRQhGMwItJaITGhpI6KxRD0uqED0+eefY9euXbj++uu79eabN2/G5s2bUVxcDAAYOXIkVq9ejXnz5nXruqR/sDhcUDqMkDInZHJF0IuhXUutkGJAjBqxLAloBo2uDoOges1iY2MRFxfX7TdPT0/H+vXrcfToURw5cgQzZ87EggULcObMmW5fm0S+ZpsTUY4GyKUcJJqEoBdDu5ZeLcfP8jNwc16ucMDeDLgcIbk28S2oT+65557D6tWrYTabu/Xmt956K2655RYMGTIEQ4cOxe9//3tER0fju+++69Z1Sf/QbHchylEPuVQSsoZqL3KV8ACoVNTDgqqavfjiiygqKkJycjKys7Mhl3uP3Th27FjA13S5XPjggw/Q3NzsGZd0LZvNBpvN5nluNNImeP2ZUCKqh0LWQ4EIAFPFgHNUCe1E3VjniHQuqEC0cOHCkGXg1KlTmDJlCqxWK6Kjoz2z+H1Zt24d1q5dG7L3Jn2byeZElL0ecrUk5EHi/cOlqG2y4QFtFGIAKhH1sKAC0Zo1a0KWgdzcXBw/fhwGgwEffvghFi9ejAMHDvgMRqtWrfLsJAIIJaKMjIyQ5YX0Lc1mC6KcRihkqpCXiJw8g5NnsMndXfiNIb0+8Rb0gMbGxkZ8+OGHKCoqwq9+9SvExcXh2LFjSE5OxoABA/y+jkKhwODBgwEAEyZMwPfff4+NGzfijTfeaJdWqVRCqVQGm2USYUbGOsHi1YiO1gGK6JBeW9UylsgibZk7SSWiHhVUIDp58iRmzZoFvV6P4uJiPProo4iLi8POnTtRWlrqWasoGDzPe7UDEdKRFIkB0KuB2LSQz45XtowlssqoRBQOQfWarVixAgUFBbhw4YLXzq633HILDh486Pd1Vq1ahYMHD6K4uBinTp3CqlWrsH//fixatCiYbJH+pumq8DVEAxnbcpeImiVtAlEHOxyT7guqRPT999/7rDoNGDAAVVVVfl+nuroaDz74ICorK6HX6zFmzBjs3r0bP/nJT4LJFulHnC4etRXF0NgciI5ORqjH9rtLRGYuSihtuZzCeCJlaKuARBBUIFIqlT67zgsLC5GY6H+j4VtvvRXM2xMCo8WOi5cuQ8E5MWlaasiv7y4RWV0coNQJ882sjRSIekhQVbPbbrsNzz77LBwOYbQpx3EoLS3Fr3/9a9x5550hzSAhvjQ3VkPKHJDLFYC6+6P8rxUTpcCAGDViouSAOkY4SA3WPSaoQPTiiy/CZDIhMTERFosF06dPx+DBg6HVavH73/8+1HkkpB1rg9AEwKKD3+e+M7kpWvwsPwP52XGtc9iowbrHBFU10+v12LNnD77++mucOHECJpMJ48ePx6xZs0KdP0J8choqAABcdEoXKUOASkQ9LuBAxPM8tm3bhp07d6K4uBgcxyEnJwcpKSlgjNGSIBHiQGENaptsmDo4Hql6tdjZacdlFEpEUl0YAhGViHpcQGVaxhhuu+02LFmyBOXl5Rg9ejRGjhyJkpISFBQU0DKxESQ3WYvyRgt2HitHo9kudna8MQaYhK57RUzoG6oBwGh14M2Dl/CnA0VUIgqDgEpE27Ztw8GDB7F3717cdNNNXue++uorLFy4ENu3b8eDDz4Y0kyS8EvSKpERp0ZxrRn/vlCLW8emiZ2lVjYjnFYzGDhExfZMIFJIJTC17JnmlGuFXxS7SejGlwY9IYF0IKAS0fvvv4/f/OY37YIQAMycORNPP/003n333ZBljoSfzSls4SyRcJg2RBiKUVRjgsHci9bjMVYiKz4K6RmZSNBH9chbKGUSz2BtK6cSdgdhjJaN7SEBBaKTJ09i7ty5HZ6fN28eTpw40e1MEXEYrQ78+cAlfHG6CowxJEQrkRUfBcaAMxW96BfQWI7YKAUys4ciStEzpROO46BuGdRocfCt1TNqJ+oRAQWi+vp6JCcnd3g+OTkZDQ2040FfVVpnhpNnMFjsnk6H4anCpM8L1Saw3jLFwSj0mEHXs9VFtaJlvpnD1dpgTe1EPSKgQORyuSCTdfwXSCqVwul0djtTRBxl9cKKmxlxrdWdgYkaSCUc6pvtqGvuBY3WvAvNdVdQa7KhQdazC5WpZG0CEZWIelRA5VrGGAoKCjpcioNmzfddjDGUugNRbGsgUsqkmJQTB6VM4qmqiMpUjfomM4oNPKxXgZ90f4fpDqkU7qqZC1DFCgcpEPWIgALR4sWLu0xDPWZ9k9HihNnuglTCIVWv8jp33cB4kXLlg7ECNgcPkyIR8VGKHn2rhGgFrHa1sL2QLEY4SFWzHhFQINq6dWtP5YOIrLrJCgCIj1ZAJg39lImQMV6BzemCSZmMgeru73PfmamDEoBBLU/c1VL3ciA0cDekevH/OBJONU1CtTpJq/J53mh14HS5wROwRMEY0FgKm5OHUZkCnTqM43lUeuGr0w44LOF7336CAhEBAOjUcmTGRWFAjO/pHN8V1WHP2as4X9UU5py1YWkAb22C1QWYFMnQ93CJyItUBihptcaeQkNECQBg1AA9Rg3Qd3g+PTYKZyqMqGgUsTTQWAKrw4UmRRLkSkWPN56X1Zvxxekqz4aLUMcAtiahnaiHhw70N1QiIu0ZK4HyY4CldUyYu6R01WiDw8WLk6/GUlgcLhiVaYiLUvT4BGuphIPJ5vRM9fBUz6hEFHJUIiKwO3nwjAm9Q2XfAxf/JZyQyoFRdwJxOdCpZYhWymCyOXHVaEV6bM9MrehQS/uQTiXHpOHjwOt7vievdWS1MO2FBjX2HCoRERRebcLm/UXYc/gEUPSVcFCpFfZ7P/sJYG8Gx3FIaykVVTSK0GBtugrYTJArlMjKHoqcBE2Pv6WqJRDZnTxcPKNBjT2IAhGBwSJMaE1tOAIwHkjMBSb/XNg91WEBSr8FAKTFCD1qorQT1V0UvsZmh232u9fEV5rm0aMoEBE0mh1QOE2Is5QKB3KmC7/sg2YKz8t/AOxmT4mo0mAN/7yzuotgYDhjT8KFq01whqGdSiLhPKUiS9tpHjYjwLt6/P37E1ED0bp165Cfnw+tVoukpCQsXLgQ58+fFzNL/ZLB4kCCuUjYuUKfDmha2l9icwBtCsA7gapTSIxW4rZxaSiYmh3elThtJsBYCbuTx75qLT47VRW29/e0E9ldwm6yEhktB9IDRA1EBw4cwNKlS/Hdd99hz549cDgcmD17Npqbm8XMVr/TaLEj3lwElVwCJI9sPcFxQNo44fvK45BwwKDEaM+s9LCp+REA0KRIgkOmQUyUHFJJeAJRsk6JAbFq4f04rk07EQWiUBK11+yLL77wer5t2zYkJSXh6NGjuPHGG0XKVf9idbjgsjZDY6+DUh4LJAzxTpA0Eri4FzDXC8tv6AeEP5NXzwhf1IMAu7B6ZLjMHXXNCpCqGKC5lhqsQ6xXtREZDMJfmbg43/tU2Ww2GI1GrwfpnkazA3prORRSDlJtUuvoYTeZojU41ZyD1eHCNxdr8dmpyvBk0B0AOQ5lsiwAQGIYA1E7tH51j+g1gYjneTz55JO4/vrrMWrUKJ9p1q1bB71e73lkZGSEOZeRRyGTYJymDvHRCiBuoO9EicOErzXnwYHhcHE9zlc1ockahuVjK34QvsbmoMoiVAlFDUS0o0eP6DWBaOnSpTh9+jR27NjRYZpVq1bBYDB4HmVlZWHMYWSK0ygwJroJ2fEaoWvcZ6KBwuBGqxFK81VPIOjx8UROO1ApLD1sTxmHxpZhBuEMRGcrjHjzoLB8LgAqEfWQXhGIli1bhn/+85/Yt28f0tPTO0ynVCqh0+m8HqSbHJbWhldtB/OnpHIgfrDwfc2PbQY29vB4oopjgNMGqGNRo8gAY0C0UtZj61T7wnG4ZppHjPCVSkQhJWogYoxh2bJl+Oijj/DVV18hJydHzOz0S4bqMjh5HlDHAnLfS4AAaK2e1RZiQMvCaRWGHgxEDqtnICWypiItRo2CqdmYMzIMGyq2oWkJemb7NfPNHFbhQUJC1EC0dOlSvPPOO3jvvfeg1WpRVVWFqqoqWCy03ku47D96Et8XN8Ag891B4BE3UBhDY2lEmkzoJKhpsnm2Hwq5or3CL7omAUgeBY7jEKtRIDM+vHPcopRCu5TZ3nKfMgWgaMkDlYpCRtRAtHnzZhgMBsyYMQOpqamex1//+lcxs9Vv2J08pM3VAABVXBfd8jIFECeUWKObLkOnloMxoMrQA6WCqtNA5UmhXjR0DiAR779plKJ1QKOLbxlNrm5Zv9pcL1KuIo+o44h6zfY0/ZTB4oDGXguZhIMyxo/1dRKGArUXgNrzGBAzGDanC822EJaIGBPahS60zP7PvA6IyUSj2Y6vL9YhI06NMekxoXs/P6jlUkg4DjxjMNud0KrkQFQ8YCgHzHVhzUsko2VA+jFjkxEqpxEqpUyYytGVhCEAJwFMNZiRK8eckYNCN9WiuQ4o/BxobOkJTR4hzHkDUFxnRuHVJpjtzrAHIo7joFZI0GxzwWJ3tQYiALBQiShUKBD1Y+Z6YaNCaVQsIPe9RKwXuRqIyQQaiqFqLAJ0IdjLh3cBZYeA4q+FOW1SOTBwBpA23rNAfUmdMOUnOwxLf/iSqle3rkkEAOqW9jQqEYUMBaJ+zNbQEoj0AfREJQwFGoqB2vNA5mQAQhU7qJJRcx1w7hOg6arwPG6g0CbkHqsDwOnicaVB6LzICnNDtdutY6+ptrpLROZ62tEjRCgQ9WPORiEQKWJSu0jZRsIQ4MKXgKEcP1wsw7EqJ/IyYzA+MzawN6+/BJzeKSy+JlcBg38iTLi95pe6pN4Mu5NHtFKGxGgRR1S3pY4Rqqguh7CGtYrGs3VXrxjQSMSRqWhCslYJXVKm/y9S6TwLxysaimC0OAIf2Fh/CTj1ofCLHJMJ5C8BUkb5LFn8WCnsGjI0RRvepUc6I5G2GWFN7UShQIGov3LakKawYGBiNOKTAwhEgFA9A5DmFBZSK6u3gOf97AFtrgPOfCy0DSUOBcbe036ibQurw4VLNSYAwLAU32nC4WyFEX8+WIQvTreZ6EvtRCFFgai/amqZO6XStQ7Q81diLgBAb61AlMQOq8OF6pYNGjvlcgCn/y5M29APAIYvEEoXHbA6XEiLUSNBqwzr0h/X4jig2eaCqe1QhSh3IGrw/SISEApE/ZSlvgLNdiecUcmBvzgqDtAkQAKGYTKhodnds9WpyweEEoQyGhh5R5drT8dEKXDnhHT8bGK6qNWydtM8gDaBiEpEoUCBqJ+qKr+Mk1cM+K42yN1Sk0YAAAY6hUXtS+rNnadvLAOuHBG+HzpPCEZ+UsrCvCLkNaJVQiDyTHwFaCxRiFEg6qecBqG9Q6EPcsfS1DEAJ0GiqwZqRwMqG60dzztzOYAfdwld3SmjgYTBnV66otGCXScrhZ0zegFNy3wzm4NvvUd3G5HVALicHbyS+IsCUX/ktMHVLFQp1F3NMeuIUgvED4JKLsUYySWMzdDD6eqgwfrSAWHXWKUWGDyr08uW1pnx8fFyFF5twqHLvaO0oZRJoZQLvyqeKS0KDSBTCsHVQu1E3UWBqD8yXYXN4YJdqoFO3/F+910aMB4AMDXqCmZkR0Gj9NHm01gKlLdUyXLndbrUyKkrBnz0QzlsDh5pMSpcN7CLFQHCSNtybyZrS+mH49oMbKR2ou6iQNQPsaYqWB0umBSJ0KuDbCMChO2GdKlC1aT0u/bnnbbWKlnqWCB+kM/L8DzDgcIa/OvcVfCMYXiqFneOTxe9baitFL0a6bFq74UANC1TXJprRMlTJKGR1f2Qtb4CPAMsygToVN0IRBwHZE8DTv4N/JUjqFDkQBmXLizlyvPAuX8IS6qqdMDgm31ewuHi8fnpKhRVC+OFpgyKx+ScuN4zeLHFT0b46F3UJApfKRB1G5WI+iFrfTkAgNOmQNLd/cHiBwFJw1BaZ0LRvu04XXhRWNDs3CfCkiESGTDydqE9xQez3YUqgwUyCYdbRqfiuoHxvS4IdchTIqoVNx8RgEpE/Y3TDrWzEZlxUVAN7Lz3ym9D5yKmtgKVhsuQntgKhyEecg7CfKzht3qmhPiiV8uxMG8AbA4eGXHiTGoNhNcEX3eJyFIvVE+7GBdFOkYlov6muRoqmQQDkhORN7jjjQoCIldDf92DcMQOgosBdU0tS7yOuxdIGtYuOWMM9c12z/MkrarXB6GKRgv+fLAI7x4qbT2oiBaWRmEMMFOpqDsohPc37qkd0aFdhJ5TaqGdeA++PluCy2oJ7pkwChKp779zBy/U4tSVRtwyOhUDE/0f2CgmhUxYHM3FtznIcUKpqLFUaCfyZ3E54hOViPqbpioYLA4YZPGtazCHyIg0HaQqLa7aVfjxqslnmh9KG3CspAEOF4Pd67e6d4tu6b63OlxwtM03NViHBAWifoY1VaLwahM+uuBEg9ne9QsCoJRJMTFbWJfo64u17UZGX7jahAOFwi/sDUMSMCyl76zjo5RJoJAJvy5N1jYjqanBOiQoEPUnTjscTTVw8gwmZVL3xhB1YFxGDOI0CujUMticQsmBMYaTVxrx2akqMAaMHqDHxKwAF1ITGcdx0LX8vIyWNlttU4koJEQNRAcPHsStt96KtLQ0cByHjz/+WMzsRD5TFWx2J+xSDVQaHeQdtOF0h1wqwR3jB+CuCRmeQPfl2avYe64aPGMYkabDzGFJfaeLvg33/TT6CkRWo7BrLgmKqIGoubkZY8eOxeuvvy5mNvoPYyUsLSOqY6MUPfY2WpXca3ySi2eQSThMG5KA2SOSuz92SSTuQGRoG4jkqtbVGt0dASRgovaazZs3D/PmzRMzC/1LUyWsDh7NikTER4W+WtaR/Ow4zByWBJW890zZCEaSVon0WHX7Kq02RRhB3lTl2YSSBKZPdd/bbDbYbK0rARqNRhFz0we555hFJWJQD5aIrpUo4uqKoTQ8VYfhqT4a2LWpQPWPQFNl+3PEL32qsXrdunXQ6/WeR0ZGhthZ6jscFsDS0DLZNQExYSwRRTz3+CHTVXHz0Yf1qUC0atUqGAwGz6OsrEzsLPUdBmF+WWraAFyXmy7qGtB9ncPFe28W4B4cammkBusg9amqmVKphFJJv0BBMQhTExIHDEZiTu9Z56ev2XG4FJUGK+7Oz0BaTMvuuHIVoI4VFkhrqhQ2iiQB6VMlItINhivC1xiqznaHe8hD27lyAFqrZ01UPQuGqIHIZDLh+PHjOH78OADg8uXLOH78OEpLSzt/IQmMywEYK9Fsd+KyIxZGq6Pr1xCf4qKFRv6OAxE1WAdD1EB05MgR5OXlIS8vDwCwYsUK5OXlYfXq1WJmK/IYywHGo8oqx8fnmnGshNZYDla8poNA5F7qxFguzMYnARG1jWjGjBlg9KH1vIYSAECNNAngOMRrqJ0tWPHRws+url2JKFXYLNJmEtqKoqgdLhDURtQf1BcBAMq5VABAfHT4xhBFmriW8VdGiwN2Z5tZ+FJ5a/XM3R5H/EaBKNLZTEDTVTh5HpVSIRDFaSgQBUutkHr2OWu3eoG+ZaE5CkQB61Pd9yQI9ZcAAE2KRDj4KGhVsj4/1UJsgxKjwTO0nzSszwRwiAJRECgQRbqa8wCAemUGYKVqWSjcPNzHjh4AoG/ZrNJcB9ibhU0YiV+oahbJHBZPiahSkQ0A1FDdk+RqILplWZCWDgLiHyoRRbKaHwHGA9FJGJ07GAkpFk9jK+keF89Qa7IhIVoJadtlTeIGAqYa4Q9A8gjxMtjHUIkoUjEGVPwgfJ88Enq1HMNSdEjSdbzlM/EPYwxb/nMZ7x0qRa3J5n3SPb2j/hKNJwoABaJIZSwXphtIZEDKGLFzE1E4jvO0tVUZrN4n9RlCV769mWbjB4ACUaRy70WfPAJlJuBoST2qm6ydv4b4LVUvTHgtb7xmtr1ECsRmC9+3tM+RrlEgikSGK8J2zxwHZFyH81VNOFhYi8Iq31v8kMBlxgsbQpbVm9vPDnBXz2oLw5yrvosCUaTheeDCHuH7lDGAJh4VBuGvdoqe2odCJUWnglzKwWx3odZ0zcDGhKHCHwFjpTDdg3SJAlGkKf1WWDtZpgRypqHZ5kSdyQ6OA9Jj1WLnLmJIJRwGtPw8S+vN3ieV0UBMlvB99Y9hzlnfRIEoktQVAcX/Fr4fPAtQanGlQSgNJUQraUR1iGXGCQMWL9X4qPImDRe+Vp+h3jM/0DiiSGGsAM5+LPynTx0LpIwGILRhAFQa6glDkqNhc7owNFnb/mRirlBFNtUIn4171DXxiUpEkcBYCZzYATjtQEwmMGQ2wHFgjOFSrfDXOieBphuEmk4lx9RBCUiI9jFaXa5uLRVVHAtvxvogCkR9XWMZcHIH4LQJs79H3wVIhYJuo9kBh4tBKZcgPTZK5Iz2QwMmCF+rzwmrIJAOUSDqy2oKhZKQwyoEoTE/A2StUzhiNQo8fuNA/NeEdO9pCCSkyurN+OR4OSquHVOkSxWqZLyrdVwX8YkCUV9Vfgw4sxPgnUDCEGDsPUJP2TVkUgmStNRt35N+rGrCpZpmHPW1BG/2NOFrxTFhuyHiEwWivoYx4NJ+oHC38H3aOGDkHcK0gjYamu3ee2+RHpOXGQOOAy5Wm9qXiuJygNgsoVR08V/Ug9YBCkR9icMKnPoQKPlWeJ59PTB0LiDx/hidLh47fyjH9m+LUXftpEwScgnRSoxM0wMA9p2vhuvaPwCDZwlTP2ovAFUnRchh70eBqK8wVgDH3gbqLgoTWYffCuTcKIzgvcbh4noYLUJDtU5NW0uHw9RB8VDJpag22nCwsMb7ZHQSkH2D8H3hl0AjbZd1rV4RiF5//XVkZ2dDpVJh8uTJOHz4sNhZ6j2cNqBoH3DsL4C5HlBqgbz7gZRRPpMX1Zhw+HI9AODGoYntlzMlPUKjlGHOSGHlxuNljfjmYq33HLSM64S2PN4JnPpAGHxKPET/X/rXv/4VK1aswJo1a3Ds2DGMHTsWc+bMQXV1tdhZE5e9GSj5Bjj0J6HHhfHCQlv5jwi9Mdcmd/I4dKkO/zxRCcaAUQP0yE3xMdCO9JiBidG4caiwQqPF4QLXtrQqkQAjFgjtRU67EIzOfw5YjSLltnfhmMgbi02ePBn5+fl47bXXAAA8zyMjIwP//d//jaeffrrT1xqNRuj1ehgMBuh0unBkN/QYA5xWwG4W1jpurgYaigGDsCkiAGGPrIE3gSUMgd3Fw+bkwXhAHyVUu6wOF7Z8fRk2h5B+eKoOs0ckQ0Jd9qI4X9WEgYkaT2n0Uo0JjRYHkrRKxKqkUJfsg6SyZdE6TtLSoJ0tVOHUsYAiWmhTigD+/o6KOsXDbrfj6NGjWLVqleeYRCLBrFmz8O2337ZLb7PZYLO1Nr4ajX7+Nam/DBTt9d1j4XWMtRxiOFJc1+aId9oYtRxDk6M9h48U14FnwmnOfY2Wf3UqGYan6DxHjpbUw+lyX5VBytvBMR4MQvFeSCv4rlaBUvUI1GMQ+DMS2J0XPNlN1qlw3+RMAIBKLoVGIYNKxjBlUDyGpWi9/xqTsLq2JHqstNEz1QYAOG4g4p0aZDR+jxh7FcYx3lNVu1RjQpPNCZ6TgpcowHNSAJwwUp6TYFJ2vKddsLDahIZrN3qEkBwAJmbFQtqStqimGbXN3h0XqTo1MuO6MfVnzN2AKjQFAFEDUW1tLVwuF5KTvXdFSE5Oxo8/tp+1vG7dOqxduzbwN3LahDk/fmNwNne8fAMPOWBt3VzPZTGio55yHjLA3vrXjbeZvbrV3VdxSeTg5fFASq6wfXHcIJw+Uo8mqxNwtk0Jn4MTF+YNgFYpo1JQL8MYQ05CFORSDjVNNphsTjAG1EqTURv/U+h4A8YN4gFDGWCug5VvgtnuAuAC4B1kOABoM9GfmZrA+wpEbs0uT9BiJhP4a3pQmVQFKLsx9YfxXafxU5+a9Lpq1SqsWLHC89xoNCIjI6PrF8ZkCAP+gDa9TG1+Yb1KDxwAhswcKwCuzanWNCqFDNAoPMeyhlg91+DaXI8DB7lMAngWrOeQM8ImlGpaLsfJ1GAyFSCRQS7l2qQF7hwfBZ4xcBwHDoBcJoFSJoFMwrUr8eipd6xX4jgOE7LiMKFlVRCeZ7A4XLA6XHAxJhSUdSoAkwEACcPN0FktgMsOzmUHeBc4xgNgwi++XtXyPUNCsx3Rjo6DAadXef6fxZntUNu900YrpUB3/t+EcLskUQNRQkICpFIprl71Xtv36tWrSElJaZdeqVRCqQxiOxyFRqiH+4kDMCDG/8unBFA6TYjuOo1bLO3IGnEkEg4apQwape9fvSRdFKDzb15gfJz/7xsXQFoxiNprplAoMGHCBOzdu9dzjOd57N27F1OmTBExZ4SQcBK9arZixQosXrwYEydOxKRJk/Dyyy+jubkZDz30kNhZI4SEieiB6O6770ZNTQ1Wr16NqqoqjBs3Dl988UW7BmxCSOQSfRxRd0TEOCJCIpi/v6Oij6wmhBAKRIQQ0VEgIoSITvTG6u5wN2/5PdWDEBJW7t/Nrpqi+3QgampqAgD/RlcTQkTT1NQEvV7f4fk+3WvG8zwqKiqg1fbOSZ7uKShlZWUR2asXyfcXyfcGhO/+GGNoampCWloaJJKOW4L6dIlIIpEgPT1d7Gx0SafTReR/ZrdIvr9IvjcgPPfXWUnIjRqrCSGio0BECBEdBaIepFQqsWbNmuBWDOgDIvn+IvnegN53f326sZoQEhmoREQIER0FIkKI6CgQEUJER4GIECI6CkQBCnRX2g8++ADDhg2DSqXC6NGj8dlnn3mdZ4xh9erVSE1NhVqtxqxZs3DhwoWevIUOBXJvb775JqZNm4bY2FjExsZi1qxZ7dIXFBQIC/+3ecydO7enb6NDgdzftm3b2uVdpVJ5pemrn92MGTPa3RvHcZg/f74nTdg/O0b8tmPHDqZQKNiWLVvYmTNn2KOPPspiYmLY1atXfab/+uuvmVQqZX/4wx/Y2bNn2f/7f/+PyeVydurUKU+a9evXM71ezz7++GN24sQJdtttt7GcnBxmsVjCdVuMscDv7b777mOvv/46++GHH9i5c+dYQUEB0+v17MqVK540ixcvZnPnzmWVlZWeR319fbhuyUug97d161am0+m88l5VVeWVpq9+dnV1dV73dfr0aSaVStnWrVs9acL92VEgCsCkSZPY0qVLPc9dLhdLS0tj69at85n+Zz/7GZs/f77XscmTJ7PHH3+cMcYYz/MsJSWF/fGPf/Scb2xsZEqlkr3//vs9cAcdC/TeruV0OplWq2Vvv/2259jixYvZggULQp3VoAR6f1u3bmV6vb7D60XSZ7dhwwam1WqZyWTyHAv3Z0dVMz+5d6WdNWuW51hnu9ICwLfffuuVHgDmzJnjSX/58mVUVVV5pdHr9Zg8eXKH1+wJwdzbtcxmMxwOB+Ku2bdm//79SEpKQm5uLn7xi1+grq4upHn3R7D3ZzKZkJWVhYyMDCxYsABnzpzxnIukz+6tt97CPffcA43Ge5+ycH52FIj81NmutFVVVT5fU1VV1Wl699dArtkTgrm3a/36179GWlqa1y/E3LlzsX37duzduxfPP/88Dhw4gHnz5sHlcoU0/10J5v5yc3OxZcsWfPLJJ3jnnXfA8zymTp2KK1euAIicz+7w4cM4ffo0lixZ4nU83J9dn559T3qH9evXY8eOHdi/f79Xg+4999zj+X706NEYM2YMBg0ahP379+Pmm28WI6t+mzJlitfeelOnTsXw4cPxxhtv4LnnnhMxZ6H11ltvYfTo0Zg0aZLX8XB/dlQi8lOgu9ICQEpKSqfp3V8DuWZPCObe3F544QWsX78eX375JcaMGdNp2oEDByIhIQEXL17sdp4D0Z37c5PL5cjLy/PkPRI+u+bmZuzYsQOPPPJIl+/T058dBSI/BbMr7ZQpU7zSA8CePXs86XNycpCSkuKVxmg04tChQ2Hd6TbYHXf/8Ic/4LnnnsMXX3yBiRMndvk+V65cQV1dHVJTU0OSb3+FYkdhl8uFU6dOefLe1z87QBhaYrPZcP/993f5Pj3+2YWtWTwC7NixgymVSrZt2zZ29uxZ9thjj7GYmBhPt+4DDzzAnn76aU/6r7/+mslkMvbCCy+wc+fOsTVr1vjsvo+JiWGffPIJO3nyJFuwYIFoXcCB3Nv69euZQqFgH374oVcXb1NTE2OMsaamJrZy5Ur27bffssuXL7N//etfbPz48WzIkCHMarWG9d6Cub+1a9ey3bt3s6KiInb06FF2zz33MJVKxc6cOeNJ01c/O7cbbriB3X333e2Oi/HZUSAK0KuvvsoyMzOZQqFgkyZNYt99953n3PTp09nixYu90v/tb39jQ4cOZQqFgo0cOZLt2rXL6zzP8+y3v/0tS05OZkqlkt18883s/Pnz4biVdgK5t6ysLAag3WPNmjWMMcbMZjObPXs2S0xMZHK5nGVlZbFHH3203ViccArk/p588klP2uTkZHbLLbewY8eOeV2vr352jDH2448/MgDsyy+/bHctMT47WgaEECI6aiMihIiOAhEhRHQUiAghoqNARAgRHQUiQojoKBARQkRHgYgQIjoKRIQQ0VEgIn3Ktm3bEBMT43n+zDPPYNy4cZ7nBQUFWLhwYdjzRbqHAhHxyb1m8c9//vN255YuXQqO41BQUOCVPtQBIDs7Gy+//LLXsbvvvhuFhYUdvmbjxo3Ytm2b5/mMGTPw5JNPhjRfJPQoEJEOZWRkYMeOHbBYLJ5jVqsV7733HjIzM0XJk1qtRlJSUofn9Xq9V4mJ9A0UiEiHxo8fj4yMDOzcudNzbOfOncjMzEReXl63ru2rpLJw4UJPKWvGjBkoKSnBU0895dlFAmhfNbtW25JZQUEBDhw4gI0bN3qucfnyZQwePBgvvPCC1+uOHz8OjuPCvlYSEVAgIp16+OGHsXXrVs/zLVu24KGHHurx9925cyfS09Px7LPPorKyEpWVlQFfY+PGjZgyZQoeffRRzzUyMzPb3RMAbN26FTfeeCMGDx4cqlsgAaBARDp1//334z//+Q9KSkpQUlKCr7/+2q+FtLorLi4OUqkUWq0WKSkpQa16qNfroVAoEBUV5bmGVCpFQUEBzp8/79n7y+Fw4L333sPDDz8c6tsgfqI1q0mnEhMTMX/+fGzbtg2MMcyfPx8JCQliZ6tb0tLSMH/+fGzZsgWTJk3CP/7xD9hsNtx1111iZ63fohIR6dLDDz+Mbdu24e233w5ZqUEikeDapbAcDkdIru2PJUuWeBrit27dirvvvhtRUVFhe3/ijQIR6dLcuXNht9vhcDgwZ86ckFwzMTHRq93H5XLh9OnTXmkUCkW3t6/p6Bq33HILNBoNNm/ejC+++IKqZSKjqhnpklQqxblz5zzfd8RgMOD48eNex+Lj45GRkdEu7cyZM7FixQrs2rULgwYNwksvvYTGxkavNNnZ2Th48CDuueceKJXKoKqE2dnZOHToEIqLixEdHY24uDhIJBJPW9GqVaswZMiQsC54T9qjEhHxi06ng06n6zTN/v37kZeX5/VYu3atz7QPP/wwFi9ejAcffBDTp0/HwIEDcdNNN3mlefbZZ1FcXIxBgwYhMTExqHyvXLkSUqkUI0aMQGJiIkpLSz3nHnnkEdjt9rD0ApLO0ZrVpN/697//jZtvvhllZWXtdkol4UWBiPQ7NpsNNTU1WLx4MVJSUvDuu++KnaV+j6pmpN95//33kZWVhcbGRvzhD38QOzsEVCIihPQCVCIihIiOAhEhRHQUiAghoqNARAgRHQUiQojoKBARQkRHgYgQIjoKRIQQ0f1/5ApWzBZiCcwAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:20:39.133445Z", + "iopub.status.busy": "2024-03-23T06:20:39.132486Z", + "iopub.status.idle": "2024-03-23T06:20:39.510502Z", + "shell.execute_reply": "2024-03-23T06:20:39.509528Z" + }, + "papermill": { + "duration": 0.399071, + "end_time": "2024-03-23T06:20:39.512831", + "exception": false, + "start_time": "2024-03-23T06:20:39.113760", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABG7UlEQVR4nO29eXxU9fX//7qzz2SW7BvZWSL7Iou4olIVrGL1I9DSShBptdhW+eFCreDysaBVkbZ+aWuFlLrg8kH9tLaiIks/yKYssgdCNkhC9lmS2e/798fN3GSSSTIzmczNcp6Px2Rm7n3PvedmZl5z3uf9PufNMcYYCIIgJEQmtQEEQRAkRARBSA4JEUEQkkNCRBCE5JAQEQQhOSREBEFIDgkRQRCSQ0JEEITkKKQ2oDfwPI/KykoYDAZwHCe1OQRBdIAxBqvVivT0dMhkXfs9A1qIKisrkZmZKbUZBEH0QEVFBTIyMrrcP6CFyGAwABAu0mg0SmwNQRAdsVgsyMzMFL+rXTGghcjXHTMajSREBNGP6Sl0QsFqgiAkh4SIIAjJISEiCEJyBnSMaLDCGIPH44HX65XaFILoFrlcDoVC0evpMyRE/QyXy4Wqqiq0tLRIbQpBBIVOp0NaWhpUKlXYxyAh6kfwPI+SkhLI5XKkp6dDpVLRRE2i38IYg8vlQm1tLUpKSjBy5MhuJy12BwlRP8LlcoHneWRmZkKn00ltzqDDy/OwOjyQcRz0GgVkJPK9RqvVQqlUoqysDC6XCxqNJqzjULC6HxLurwrRPRaHB04PD5eXh5enUu2RIhKfV/KIiCGBx8vD5eEBAHE6JeQk9v0KejeIIYHLK4iQUi4jEeqH0DtCSEZBQQHuuuuuiB5z1qxZeOSRRzpt93lDKoUMOTk5WL9+PXjqnvUbSIiIiNCVAPQHGGNwewXRUck5MAA2pwdWh1taw/o5zzzzDCZNmhSVc5EQEYMengF86zqiCnnbR95NHlG/gYRogODy8F3ePK3xj2DauoNoGyoFBQXYvXs3NmzYAI7jwHEciouLsXTpUuTm5kKr1SI/Px8bNmwI+Ppnn30WSUlJMBqNePDBB+FyuYI6b3NzM+677z7o9XqkpaXhlVde6dSmpqYG8+68EzkpcZgxYTTefecd+AbtvTwDYwwcx2Hjxo2YM2cOtFot8vLy8OGHH4rHKC0tBcdxeP/993HddddBq9Vi2rRpKCoqwqFDhzB16lTo9XrMmTMHtbW1Qf/fNm3ahLFjx0KtViMtLQ0PP/ywuK+8vBzz5s2DXq+H0WjE/PnzcfnyZXG/z1v5+9//jpycHJhMJixcuBBWq1Vsw/M8XnrpJYwYMQJqtRpZWVl44YUXxP1PPPEERo0aBZ1Oh7y8PDz99NNwuwUvsbCwEM8++yyOHTsmvqeFhYVBX1uo0KjZAOH1nee73JebGIO7Jg8Tn/9lT7HYFelIRpwW905tKya3aW8J7C7/VJJHvzcqJNs2bNiAoqIijBs3Ds899xwAIC4uDhkZGfjggw+QkJCAr7/+Gj/96U+RlpaG+fPni6/dsWMHNBoNdu3ahdLSUixZsgQJCQl+X5iueOyxx7B792588sknSE5Oxq9//WscPnzYrztRUFCAyspK7Ny5E0qlEr/85S9RU1Mj7vcN4z/99NNYt24dNmzYgL///e9YuHAhjh8/jtGjR4tt16xZg9deew1ZWVm4//778aMf/QgGgwEbNmyATqfD/PnzsXr1amzcuLFH2zdu3IgVK1Zg3bp1mDNnDsxmM/bu3QtAEBCfCO3evRsejwfLly/HggULsGvXLvEYxcXF+Pjjj/HPf/4TjY2NmD9/PtatWyf+71atWoU33ngD69evx7XXXouqqiqcOXNGfL3BYEBhYSHS09Nx/PhxLFu2DAaDAY8//jgWLFiAEydO4LPPPsOXX34JADCZTD1eV9iwAYzZbGYAmNlsltqUiGC329mpU6eY3W7vtO/Vz892efvo8EW/tn/YUdRl2/cPlfu13bjrfKc24XDDDTewX/3qV922Wb58ObvnnnvE54sXL2bx8fGsubm5zZ6NG5ler2der7fbY1mtVqZSqdj7778vbquvr2darVa04+zZswwAO3jwoNjm9OnTDAD773W/Y9VmO7O7PAwAe/DBB/2OP2PGDPbQQw8xxhgrKSlhANhf//pXcf+7777LALAdO3aI29auXcvy8/O7tdtHeno6e+qppwLu+/zzz5lcLmfl5W3v1cmTJ/2uZc2aNUyn0zGLxSK2eeyxx9iMGTMYY4xZLBamVqvZG2+8EZQ9jDH2u9/9jl155ZXi8zVr1rCJEyf2+LruPrfBfkfJIxogLL9xRJf7ZB0mCP/0+uFdtu04mfj+a3J7Y1a3vP7669i0aRPKy8tht9vhcrk6BT8nTpzoN4t85syZsNlsqKioQHZ2dpfHLi4uhsvlwowZM8Rt8fHxyM/PF5+fPn0aCoUC4ydOgtvLQyHjcMUVVyA2Nlb8n/k8opkzZ/odf+bMmTh69KjftgkTJoiPU1JSAADjx4/329be2+qKmpoaVFZW4uabbw64//Tp08jMzPQrgzxmzBjExsbi9OnTmDZtGgAgJyfHr/JhWlqaeP7Tp0/D6XR2eQ4AeO+99/D73/8excXFsNls8Hg8khUYpBjRAEGlkHV5ax+A7amtMoi2kWDr1q1YuXIlli5dis8//xxHjx7FkiVLgo7/RBKz3Y2GZhc87YLTvvSOUGZYK5VK8bEvB7DjNp7vOcam1WqDPmew9nQ8f0/n2LdvHxYtWoS5c+fin//8J44cOYKnnnpKkvcHICEiIoRKpfIrW7J3715cffXV+PnPf47JkydjxIgRKC4u7vS6Y8eOwW63i8/3798PvV7f46IIw4cPh1KpxIEDB8RtjY2NKCoqEp9fccUV8Hg8OHL4MABAznE4e/YsmpqaIJdx0ChkUMo58bzt2b9/v198KJIYDAbk5ORgx44dAfePHj0aFRUVqKioELedOnUKTU1NGDNmTFDnGDlyJLRabZfn+Prrr5GdnY2nnnoKU6dOxciRI1FWVubXpuN72pdQ14yICDk5OThw4ABKS0uh1+sxcuRIbNmyBdu3b0dubi7+/ve/49ChQ8jN9e8KulwuLF26FL/5zW9QWlqKNWvW4OGHH+4xf0mv12Pp0qV47LHHkJCQgOTkZDz11FN+r8vPz8ett96Kxx/5BV5cvwFJRh0effRRaLVaKOQymHRtZSs++OADTJ06Fddeey3efvttHDx4EG+++WZk/0nteOaZZ/Dggw8iOTkZc+bMgdVqxd69e/GLX/wCs2fPxvjx47Fo0SK89tpr8Hg8+PnPf44bbrgBU6dODer4Go0GTzzxBB5//HGoVCpcc801qK2txcmTJ7F06VKMHDkS5eXl2Lp1K6ZNm4ZPP/0UH330kd8xcnJyUFJSgqNHjyIjIwMGgwFqtbov/h3SB6svXrzIFi1axOLj45lGo2Hjxo1jhw4dCuq1QylY3d85e/Ysu+qqq5hWq2UA2JkzZ1hBQQEzmUwsNjaWPfTQQ+zJJ5/0C34uXryYzZs3j61evZolJCQwvV7Pli1bxhwOR1DntFqt7Mc//jHT6XQsJSWFvfTSS52C5qXlF9nsW+cwtVrNsrKy2JYtW1h2djZbv3692AYAe/3119n3vvc9plarWU5ODnvvvffE/b5g9ZEjR8RtO3fuZABYY2OjuG3z5s3MZDIF/T/705/+xPLz85lSqWRpaWnsF7/4hbivrKyM3XnnnSwmJoYZDAZ27733surqanF/oEDy+vXrWXZ2tvjc6/Wy//7v/2bZ2dlMqVSyrKws9tvf/lbc/9hjj4n/9wULFrD169f72e9wONg999zDYmNjGQC2efPmgNcRiWA1xxiTbFZXY2MjJk+ejBtvvBEPPfQQkpKScO7cOQwfPhzDh3cdcPVhsVhgMplgNpsHxSoeDocDJSUlyM3NDbucAuFPi8sDq8MDtUKGWJ1/4S7GGHjGoJDL8dFHH0U83WSo0N3nNtjvqKRdsxdffBGZmZnYvHmzuK2j604QvcEXjJZ3HFoEUGtzQrqfYaI9kgar//d//xdTp07Fvffei+TkZEyePBlvvPFGl+2dTicsFovfjRiclJeXQ6/Xd3krLy8P6jiiEAUoghZoW6Tozvb//Oc/fXbegYqkHtGFCxfEGaa//vWvcejQIfzyl7+ESqXC4sWLO7Vfu3Ytnn32WQksJaJNenp6p3k8HfcHg1Yph1IugzLAtASZjAN4hhanG1pVZL8K3dk+bNiwLvcNVSSNEalUKkydOhVff/21uO2Xv/wlDh06hH379nVq73Q64XQ6xee+5WwpRkSEg8Xuht3tRYxaAb2aBpDDJRIxIkm7ZmlpaZ3mRYwePbpLt1utVovLS9My00Rv8YWNqC6R9EgqRNdccw3Onj3rt62oqKjbqf0EESw8Y3C6vZ0qDviQtSoRTxFryZFUiB599FHs378fv/3tb3H+/Hm88847+Mtf/oLly5dLaRYxSPDyDE12N5paAhdA86V5kBBJj6RCNG3aNHz00Ud49913MW7cODz//PN47bXXsGjRIinNIgYJvhGzACP3ACCmeagV8ihaRQRC8gjd97//fXz/+9+X2gxiEOLzdALNIQKEQvomXfirkxKRg5JeiUELL3pEtJBif4eEiIgI/a14/qxZs/D4yv8PQFtQOhCMMbFkLCEdJEREvyHStXDErlk3DlGdzYU6m9OvVhERfUiI+juMAR5X9G8heAjhFs/3rWv2wgsvID09Xayu+PXXX2PSpEnQaDSYOnUqPv74Y3Ac5zdb+cSJE5gzZw70ej1SUlLwk5/8BHV1dX72/Pn//RGpJi10aiVKS0sD2k5zifoHkgeriR7wuoH/dF6dos+57v8DFMEFcntbPN9oNOKLL74AIMzEveOOOzB37ly88847KCsr69Tla2pqwk033YQHHngA69evh91uxxNPPIH58+fjq6++Eu0ZPWYsnnp6DdRKGVJbS7t2xJfmQUP40kJCRPQak8kElUoFnU6H1NRUcXv7vMDc3Fzs27cP77//vp8QxcTE4K9//StUKkH0/vSnP4HjOLzxxhvQaDQYM2YMLl26hGXLlomv+eMf/4jJkyfjt7/9rbht06ZNyMzMRFFREUaNGgWVSgWDPgZ52Rnd2t42l6h3/wOid5AQ9XfkSsE7keK8vSSY4vnjx48XRQgAzp49iwkTJvjlLE2fPt3vNceOHcPOnTuh1+s7nbO4uBijRgW/HJLYNSOPSFJIiPo7HBd0F6k/4Sue/8orr2DmzJkwGAz43e9+51djGhA8olCx2Wy444478OKLL3bal5aWJj728gxuL99pwYD20Ozq/gEJERERuiue7yNQ8fyO5Ofn46233oLT6RTrIx86dMivzZQpU/A///M/yMnJgUIR+COsUCrR4nTBYncjQd91nWVRiEJf4JaIIDRqRkSE9sXz6+rqMHLkSHzzzTfYvn07ioqK8PTTT3cSlED86Ec/As/z+OlPf4rTp09j+/btePnllwG0LeGzfPlyNDQ04Ic//CEOHTqE4uJibN++HUuWLBHFMCsrB4e/OYSL5WWoq6vrcpkfhYyDRhm5ZZSI8KD/PhERVq5cCblcjjFjxiApKQm33nor7r77bixYsAAzZsxAfX29n3fUFUajEf/4xz9w9OhRTJo0CU899RRWr14NAGLcKD09HXv37oXX68Utt9yC8ePH45FHHhEWTmxdxeOXjz4KuVyOmVMnISkpqcvSMkqFDCatCjFUj0hSJC2M1luoeP7Q4O2338aSJUtgNpuDXpzQ6nCjxeWFTiWHQdP7wDvRNQO+eD5BBGLLli3Iy8vDsGHDcOzYMXGOUCgrpPqG44PJMxNW8xBG0DjKS5MEEiKi31FdXY3Vq1ejuroaaWlpuPfee/HCCy+EdAyfo99NmpnYrsYqlB9O1Ku7TQch+g4SIqLf8fjjj+Pxxx/v1TG8rULUk4cjpKQIGS2CeJESSQEJETEoiVEp4OUZFEG4ODKOg7e1e0ZIAwlRP2QAjx/0GzTK4KsuyjjAC5rUGC6R+LzS8H0/QqkURndaWloktmRo4eu+0Q9AePg+r77PbziQR9SPkMvliI2NRU1NDQBAp9PRKE4Y8IzB4+HBybhu0zt8eFwuuD0MdnjA8fSVCBbGGFpaWlBTU4PY2FjI5eHX/qb/ej/Dl73uEyMidLw8Q7PTA47jYND0/BF3uL1weXioFLKQunSEQGxsrF/VhXAgIepncByHtLQ0JCcnw+0OvAwO0T0VDc34v9M1SNSr8f3RPS9NfaHWhrrGFmTExSA3KfQk3KGMUqnslSfkQ1IheuaZZzqtZZ+fn48zZ85IZFH/QS6XR+QNHoo4mRN2Xg6FSh3UDPUxmRqMyYyCYUSXSO4RjR07Fl9++aX4vKtsaoIIFrtbSHzVqkjIBwqSf+sVCkWv+5cE0Z4WlyBEuhCEyOPl4fLy0Kkk/0oMSSQfvj937hzS09ORl5eHRYsWdZklDQBOpxMWi8XvRhAdsbs8AIIXohqLA3/46jzeOdD1Z4/oWyQVohkzZqCwsBCfffYZNm7ciJKSElx33XWwWq0B269duxYmk0m8ZWZSx57ojK9rFuwImLq1nd3l7aEl0Vf0qzIgTU1NyM7OxquvvoqlS5d22u90OuF0OsXnFosFmZmZg6YMCBEZSuqa0dDsQm5iDOJjei6z6/R48f92CtUjH75pRFBzj4jgGJBlQGJjYzFq1CicP38+4H61Wi2WDyWIrshNjEFuYvDD8Cq5TEx8dXq6r3FN9A396j9us9lQXFzsVwCdIPoajuOgVgjdM6ebumdSIKkQrVy5Ert370ZpaSm+/vpr/OAHP4BcLscPf/hDKc0iBjAeL48LtTZUmx0h5Y5plMJXweGhKvpSIGnX7OLFi/jhD3+I+vp6JCUl4dprr8X+/fuRlJQkpVnEAMbq8OCTo5VQKWRYfuOIoF8neERu8ogkQlIh2rp1q5SnJwYhLe7Q5xABQE6CDvExVERfKui/TgwqfEPw2hCTV68ekdgX5hBB0q+C1QTRW0QhovSOAQUJETGoaBFnVYfu7Hu8PFwUrJYEEiJiUBFUjMhWA9QXA16PuOlQaQP+8NV57DpLdaCkgGJExKCix67ZhV1A2T7hcUwiMPGHgFoPVeskRid5RJJAHhExqBiXbsL1o5KQERdgMcbLp9pESK4AmuuAM58CjIl5aQ4avpcEEiJiUJGVoMOV2XFINnQoiMZ7BW8IALKvBq5cAsjkQMMFoLEUagV5RFJCQkQMDS6fBBxmQBUjCFFMIpA2Sdh36VuolSREUkJCRAwaeJ6huNaGKrMdfMfVEqu/E+4zpgLy1mVvhl0p3NcXQ8OEqg7UNZMGEiJi0ODwePG/Ryux9WAF/GTI3gg0VQAcB6SMa9sekwDokwHGQ2O5AABwefjOIkb0OSRExKDBVyJWo5RDLmu3HlxtkXAfmwVoOtTESboCAKBuPIe8pBiMTjPC239KdA0ZaPieGDTYu6pV3SAUPUPiqM4vShgBlOyBzFyBedemCgFsIuqQR0QMGgKu3uF2CN0yAIjP6/wifTKg1AJeN2CtioKVRCBIiIhBQ8DVOxpLAcYDugRAF9/5RRwHxGWLbT1eHl6KEUUdEiJi0ODLM/PLvG9qXZkjPrfrF5qyAAAHj53AH746jwu1tr4ykegCEiJi0BAwvcPc2i0zdbPii1EoTRzjqgEYg8NNc4miDQWriUFDfqoBcTEqpBpbZ1W77UBzrfDYlNH1C2OSAZkcat4GtccCp4cqhEYbEiJi0JARp0NGnK5tg/mSsDSHLh5Q67t+oVwB6JMhl1ugd9XC6emmG0f0CdQ1IwYv5tb4UHfdMh+GdChkHPSuGjg9NLs62pAQEYOGkrpmVJntbaNelkrhvrtumQ9DKhQyDjGueooRSQAJETEo8PIMHx+5hK0HK4QqizwPWKuFnYYg1snTJ0Mu46BzN8Dp9vTcnogo/UaI1q1bB47j8Mgjj0htCjEA8Q3dyzhOWKPM3iBMUpQrhDlEPaFLgEalQJKGx7AY8oiiTb8IVh86dAh//vOfMWHCBKlNIQYobUP3MnAc1zZLWp8KyIL4vZUrYYxLgVFdDySSRxRtJPeIbDYbFi1ahDfeeANxcXFSm0MMUFpEIWr9bQ2lW+ZD3zps31wXQcuIYJBciJYvX47bb78ds2fPltoUYgDjE6IY32RGn0dkSA3+IDGCEHmtl0NarproPZKv9Hr48GEcOnQoqPZOpxNOp1N8brFY+so0YoBhd/uWEZK3BqovCztC8IhcmkQcKamH9eIp3HDF96GUcz2/iIgIYXlEFy5c6PWJKyoq8Ktf/Qpvv/02NBpNzy8AsHbtWphMJvGWmRnE/BBiSODXNWupA3gPoFAFTnTtAqUxGYwBWncjHC6KE0WTsIRoxIgRuPHGG/HWW2/B4XCEdeJvv/0WNTU1mDJlChQKBRQKBXbv3o3f//73UCgU8Ho7TypbtWoVzGazeKuoqAjr3MTgY0SyHjfkJyEvMaYtPqRPFbLrg4TTxEKmUEDGvHA1N/WNoURAwhKiw4cPY8KECVixYgVSU1Pxs5/9DAcPHgzpGDfffDOOHz+Oo0ePirepU6di0aJFOHr0KOTyzgWq1Go1jEaj340gACDNpMWUrDhkxuuA5tZFEvUpoR1EJoNXbQIAuG31EbaQ6I6whGjSpEnYsGEDKisrsWnTJlRVVeHaa6/FuHHj8Oqrr6K2trbHYxgMBowbN87vFhMTg4SEBIwbN67H1xNEl9haP38xiSG/lFcLXTmPtefPMBE5ejVqplAocPfdd+ODDz7Aiy++iPPnz2PlypXIzMzEfffdh6oqqnhHRIfSumZUNtnh8fLtPKLkkI/Dt8aU+OaGSJpH9ECvhOibb77Bz3/+c6SlpeHVV1/FypUrUVxcjC+++AKVlZWYN29eSMfbtWsXXnvttd6YRAxBGGP4x7FKvHeoAs02M+BqEWJDutA9Imh9QkRziaJJWMP3r776KjZv3oyzZ89i7ty52LJlC+bOnQtZ6wzW3NxcFBYWIicnJ5K2EkRAnB4entZEV527UdioiRVGzULElJAC1SUlYniaGhJNwhKijRs34v7770dBQQHS0gLP00hOTsabb77ZK+MIIhh86R0qhQxKe6snow+vuNmEkXlAjRGAE/C4whIzInTCEqIvvvgCWVlZogfkgzGGiooKZGVlQaVSYfHixRExkiC6o8Xdrmi+ryJjTOjxIQCASgcoNcLqH/ZGwBDiyBsRFmHFiIYPH466us596IaGBuTmUnU7IrrYXe1mVdvCD1SL6BKEFI8WGsKPFmEJUVd5ODabLehZ0gQRKcRZ1UpZW8JqTHhdswu1Nnx6zoFTVRbA0RQhC4meCKlrtmLFCgAAx3FYvXo1dLq2+sBerxcHDhzApEmTImogQfSET4hMzCqkdsgVQrA6DBQyGZrlBnhcTOiaEVEhJCE6cuQIAMEjOn78OFSqtkCeSqXCxIkTsXLlyshaSBA9kJsYA7VChhRnKdAEwRsKpgZRANRKGZwKIzxeBtibImgl0R0hCdHOnTsBAEuWLMGGDRsoxYLoF6QYNUgxaoCS48KGMLtlAKBRyOFQGOHleeqaRZGwRs02b94caTsIovf4AtXhjphB8IgcCiO8DPDaLZB7W7t6RJ8S9H/47rvvRmFhIYxGI+6+++5u227btq3XhhFEsJTXt0Ah55Biq4UcCHsOEQCoFTJ45Rp4OSW8PA+5wwzEBFHzmugVQQuRyWQSagG3PiaI/sKnx6vgcjqwDHXQqRS96ppxHAeVUt4aJ/JA5WgiIYoCQQtR++4Ydc2I/oLHy8Ph9kLvboBSIwNUMcKtF2THx8BoS4KMq6aRsygRVufXbreDMSYO35eVleGjjz7CmDFjcMstt0TUQILojubWoXu9twkKOde7iYyt3D4hDdDlAhW1NHIWJcIa45w3bx62bNkCAGhqasL06dPxyiuvYN68edi4cWNEDSSI7vCtZxbHN4IDF1YNooBoW1eUoZGzqBB2hcbrrrsOAPDhhx8iNTUVZWVl2LJlC37/+99H1ECC6I5mpyBEsaxJ2NCLETM/tLFgoEmN0SKsrllLSwsMBgMA4PPPP8fdd98NmUyGq666CmVlZRE1kCC6o9npBRiD0dsEQNmrQLWP3UW1OHuhAXNb7MiQNQGMhVT7mgidsIvnf/zxx6ioqMD27dvFuFBNTQ1NciSiSrPLAyVvhwZOQSwi0DXjALTIYuBhnJAy4rL13lCiW8ISotWrV2PlypXIycnBjBkzMHPmTACCdzR58uSIGkgQ3TE8SY8bM4CEGLUQ15Ere31MjVIOxsnhkLeOvlH3rM8Jq2v2X//1X7j22mtRVVWFiRMnittvvvlm/OAHP4iYcQTREylGDVJMLkAbmW4ZIExqBIAWuQGAWRg5i82KyLGJwIQ9dz01NRWpqf7L+U6fPr3XBhFEyIipHZERIo1SWMqqReYTIvKI+pqwhKi5uRnr1q3Djh07UFNTA57n/fZHYiVYggiGC7U2xNdVwsgYZBH2iGwyYUAGDnNEjkt0TVhC9MADD2D37t34yU9+grS0NDH1gyCiCc8z/O/Ri5hWcQHTsgxQRWAyI9DmEdnQGiOiuUR9TlhC9O9//xuffvoprrnmml6dfOPGjdi4cSNKS0sBAGPHjsXq1asxZ86cXh2XGBrY3V6o3RbImQcKpSrsYmgd0arkGBarRRxLBppBs6ujQFijZnFxcYiPj+/1yTMyMrBu3Tp8++23+Oabb3DTTTdh3rx5OHnyZK+PTQx+mp0e6NyNUMo5yGISwy6G1hGTVon50zJx8+R8YYOrGfC6I3JsIjBhvXPPP/88Vq9ejZaWll6d/I477sDcuXMxcuRIjBo1Ci+88AL0ej3279/fq+MSQ4Nmlxc6dwOUclnEAtV+KDXCDSCvqI8Jq2v2yiuvoLi4GCkpKcjJyYFS6T934/DhwyEf0+v14oMPPkBzc7M4L6kjTqcTTqdTfG6x0CJ4QxnBI2qAStFHQgSAaWLBuauFOFEv6hwR3ROWEN11110RM+D48eOYOXMmHA4H9Hq9mMUfiLVr1+LZZ5+N2LmJgY3N6YHO1QClVhZxkXj3YDnqrE78xKBDLEAeUR8TlhCtWbMmYgbk5+fj6NGjMJvN+PDDD7F48WLs3r07oBitWrVKXEkEEDyizMzMiNlCDCyaW+zQeSxQKTQR94g8PIOHZ3AqfUP4TRE9PuFP2BMam5qa8OGHH6K4uBiPPfYY4uPjcfjwYaSkpGDYsGFBH0elUmHEiBEAgCuvvBKHDh3Chg0b8Oc//7lTW7VaDbVaHa7JxCBjbJwHLEELvd4IqPQRPbamdS6RXd6aO0keUZ8SlhB99913mD17NkwmE0pLS7Fs2TLEx8dj27ZtKC8vF2sVhQPP835xIILoilSZGTBpgbj0iGfHq1vnEjkU5BFFg7BGzVasWIGCggKcO3fOb2XXuXPnYs+ePUEfZ9WqVdizZw9KS0tx/PhxrFq1Crt27cKiRYvCMYsYalgvC/cRmsjYHp9H1CxrJ0RdrHBM9J6wPKJDhw4F7DoNGzYM1dXVQR+npqYG9913H6qqqmAymTBhwgRs374d3/ve98IxixhCeLw86ipLEeN0Q69PQaTn9vs8ohZOJ3hbXo8wn0gd2S4gIRCWEKnV6oBD50VFRUhKCj5o+Oabb4ZzeoKAxe7C+QslUHEeTL8uLeLH93lEDi8HqI1CvpmjiYSojwira3bnnXfiueeeg9stzDblOA7l5eV44okncM8990TUQIIIRHNTDeTMDaVSBWh7P8u/I7E6FYbFahGrUwLaWGEjBaz7jLCE6JVXXoHNZkNSUhLsdjtuuOEGjBgxAgaDAS+88EKkbSSITjgahRAA04e/zn135KcaMH9aJqblxLflsFHAus8Iq2tmMpnwxRdfYO/evTh27BhsNhumTJmC2bNnR9o+ggiIx1wJAOD0qT20jADkEfU5IQsRz/MoLCzEtm3bUFpaCo7jkJubi9TUVDDGqCTIIGF3US3qrE5cPSIBaSat1OZ0wmsRPCK5MQpCRB5RnxOST8sYw5133okHHngAly5dwvjx4zF27FiUlZWhoKCAysQOIvJTDLjUZMe2w5fQ1OKS2hx/GANswtC9KjbygWoAsDjceGPPBfxpdzF5RFEgJI+osLAQe/bswY4dO3DjjTf67fvqq69w1113YcuWLbjvvvsiaiQRfZINamTGa1Fa14L/nKvDHRPTpTapDacFHkcLGDjo4vpGiFRyGWyta6Z5lAbhi+KyCcP48rATEoguCMkjevfdd/HrX/+6kwgBwE033YQnn3wSb7/9dsSMI6KP0yMs4SyTcbhupDAVo7jWBnNLP6rHY6lCdoIOGZlZSDTp+uQUaoVMnKzt4DTC6iCMUdnYPiIkIfruu+9w2223dbl/zpw5OHbsWK+NIqTB4nDjL7sv4LMT1WCMIVGvRnaCDowBJyv70RfQcglxOhWyckZBp+ob74TjOGhbJzXa3Xxb94ziRH1CSELU0NCAlJSULvenpKSgsZFWPBiolNe3wMMzmO0ucdBhdJqQ9HmuxgbWX1IcLMKIGYx9213UqlrzzdzetoA1xYn6hJCEyOv1QqHo+hdILpfD4/H02ihCGioahIqbmfFt3Z28pBjIZRwaml2ob+4HQWvei+b6i6izOdGo6NtCZRpFOyEij6hPCcmvZYyhoKCgy1IclDU/cGGModwnRHFtQqRWyDE9Nx5qhUzsqkiKrQYN1haUmnk4LgPf6/0K012iUfm6Zl5AEydsJCHqE0ISosWLF/fYhkbMBiYWuwctLi/kMg5pJo3fvqvyEiSyKgCWSjjdPGyqJCToVH16qkS9Cg6XVlheSBErbKSuWZ8QkhBt3ry5r+wgJKbG6gAAJOhVUMgjnzIRMSwX4fR4YVOnIE/b+3Xuu+Pq4YnA8NYnvm6prxwITdyNKP34E0dEk1qr0K1ONmgC7rc43DhxySwKliQwBjSVw+nhYVGnwqiN4nwejUm497gAtz165x0ikBARAACjVomseB2GxQZO59hfXI8vTl3G2WprlC1rh70RvMMKhxewqVJg6mOPyA+5AlBTtca+gqaIEgCAccNMGDfM1OX+jDgdTlZaUNkkoTfQVAaH2wurKhlKtarPg+cVDS347ES1uOAitLGA0yrEifp46sBQgzwiojOWKuDSYcDeNifM5yldtjjh9vLS2NVUDrvbC4s6HfE6VZ8nWMtlHGxOj5jqIXbPyCOKOOQREXB5ePCMCaNDFYeA818KO+RKYNw9QHwujFoF9GoFbE4PLlscyIjrm9SKLmmNDxk1SkwfPQm8qe9H8tpmVgtpLzSpse8gj4hA0WUrNu4qxhcHjwHFXwkb1QZhvfdTnwCuZnAch/RWr6iySYKAte0y4LRBqVIjO2cUchNj+vyUmlYhcnl4eHlGkxr7EBIiAma7kNCa1vgNwHggKR+Y8aCweqrbDpTvAwCkxwojapLEierPC/dxOVHLfvdLfKU0jz6FhIhAU4sbKo8N8fZyYUPuDcKXffhNwvNLRwBXi+gRVZkd0c87qz8PBoaTrmScu2yFJwpxKpmME70ie/s0D6cF4L19fv6hhKRCtHbtWkybNg0GgwHJycm46667cPbsWSlNGpKY7W4kthQLK1eYMoCY1vhLXC5gSAV4D1B9HEl6Ne6clI6Cq3OiW4nTaQMsVXB5eOysMeBfx6ujdn4xTuTyCqvJyhRUDqQPkFSIdu/ejeXLl2P//v344osv4Ha7ccstt6C5uVlKs4YcTXYXElqKoVHKgJSxbTs4DkifJDyuOgoZBwxP0otZ6VGj9gwAwKpKhlsRg1idEnJZdIQoxajGsDitcD6OaxcnIiGKJJKOmn322Wd+zwsLC5GcnIxvv/0W119/vURWDS0cbi+8jmbEuOqhVsYBiSP9GySPBc7vAFoahPIbpmHRN/LySeFOOxxwCdUjo8Vt4zpUgNTEAs11FLCOMP0qRmQ2C78y8fGB16lyOp2wWCx+N6J3NLW4YXJcgkrOQW5Ibps97EOhahOn2tNwuL34+nwd/nW8KjoG+gSQ41ChyAYAJEVRiDpB9av7hH4jRDzP45FHHsE111yDcePGBWyzdu1amEwm8ZaZmRllKwcfKoUMk2LqkaBXAfF5gRslXSHc154FB4aDpQ04W22F1RGF8rGVR4T7uFxU24UuoaRCRCt69An9RoiWL1+OEydOYOvWrV22WbVqFcxms3irqKiIooWDk/gYFSborchJiBGGxgM2yhMmNzosULdcFoWgz+cTeVxAlVB62JU6CU2t0wyiKUSnKi14Y49QPhcAeUR9RL8Qoocffhj//Oc/sXPnTmRkZHTZTq1Ww2g0+t2IXuK2twVeDV3kT8mVQMII4XHtmXYTG/t4PlHlYcDjBLRxqFVlgjFAr1b0WZ3qQHAcOqR5xAr35BFFFEmFiDGGhx9+GB999BG++uor5ObmSmnOkMRcUwEPzwPaOEAZuAQIgLbuWV0RhrUWTqs096EQuR3iREpkX430WC0Krs7BrWOjsKBiO2JaRa/F1SHfzO0QbkREkFSIli9fjrfeegvvvPMODAYDqqurUV1dDbud6r1Ei13ffodDpY0wKwIPEIjE5wlzaOxNSFcIgwS1Vqe4/FDEKd4hfNFjEoGUceA4DnExKmQlRDfHTacW4lItrtbrVKgAVasN5BVFDEmFaOPGjTCbzZg1axbS0tLE23vvvSelWUMGl4eHvLkGAKCJ72FYXqEC4gWPVW8tgVGrBGNAtbkPvILqE0DVd0K/aNStgEy6j6lO1Tah0cu3zibXttavbmmQyKrBh6TziPrN8jRDFLPdjRhXHRQyDurYIOrrJI4C6s4BdWcxLHYEnB4vmp0R9IgYE+JC51qz/7OuAmKz0NTiwt7z9ciM12JCRmzkzhcEWqUcMo4DzxhaXB4YNEpAlwCYLwEt9VG1ZTBDZUCGMBarBRqPBRq1Qkjl6InEkQAnA2y1mJWvxK1jh0cu1aK5Hij6N9DUOhKaMkbIeQNQWt+CostWtLg8URcijuOgVcnQ7PTC7vK2CREA2MkjihQkREOYlgZhoUK5Lg5QBi4R64dSC8RmAY2l0DQVA8YIrOXDe4GKA0DpXiGnTa4E8mYB6VPEAvVl9ULKT04USn8EIs2kbatJBADa1ngaeUQRg4RoCONsbBUiUwgjUYmjgMZSoO4skDUDgNDFDsszaq4HTn8CWC8Lz+PzhJiQb64OAI+Xx8VGYfAiO8qBah93TOzQbfV5RC0NtKJHhCAhGsJ4mgQhUsWm9dCyHYkjgXOfA+ZLOHK+AoerPZicFYspWXGhnbzhAnBim1B8TakBRnxPSLjt8KUua2iBy8NDr1YgSS/hjOr2aGOFLqrXLdSw1tB8tt7SLyY0EtKQpbIixaCGMTkr+BdpjGLheFVjMSx2d+gTGxsuAMc/FL7IsVnAtAeA1HEBPYszVcKqIaNSDdEtPdIdMnm7GdYUJ4oEJERDFY8T6So78pL0SEgJQYgAoXsGIN0jFFKraLCD54McAW2uB05+LMSGkkYBExd2TrRtxeH24kKtDQBwRWrgNtHgVKUFf9lTjM9OtEv0pThRRCEhGqpYW3OnNMa2CXrBkpQPADA5KqGTueBwe1HTukBjt3jdwIn/EdI2TMOA0fME76ILHG4v0mO1SDSoo1r6oyMcBzQ7vbC1n6qg8wlRY+AXESFBQjREsTdUotnlgUeXEvqLdfFATCJkYLhCIQSafSNb3VKyW/Ag1Hpg7N091p6O1alwz5UZmD81Q9JuWac0D6CdEJFHFAlIiIYo1ZdK8N1FM/bXhblaavIYAECeRyhqX9bQ0n37pgrg4jfC41FzBDEKErUiyhUhO6DXCEIkJr4CNJcowpAQDVE8ZiHeoTKFuWJp2gSAkyHJWwutuxFVTY6u8868buDMp8JQd+p4IHFEt4eubLLj0++qhJUz+gExrflmTjffdo2+GJHDDHg9XbySCBYSoqGIxwlvs9Cl0PaUY9YVagOQMBwapRwTZBcwMdMEj7eLgPWF3cKqsWoDMGJ2t4ctr2/Bx0cvoeiyFQdK+oe3oVbIoVYKXxUxpUUVAyjUgrjaKU7UW0iIhiK2y3C6vXDJY2A0db3efY8MmwIAuFp3EbNydIhRB4j5NJUDl1q7ZPlzui01cvyiGR8duQSnm0d6rAZX5fVQESCKGFqvzeZo9X44rt3ERooT9RYSoiEIs1bD4fbCpkqCSRtmjAgQlhsypgldk/L9nfd7nG1dsrSJQMLwgIfheYbdRbX48vRl8IxhdJoB90zJkDw21J5UkxYZcVr/QgAxrSkuzbWS2DSYoJnVQxBHQyV4BtjViTBqeiFEHAfkXAd89z74i9+gUpULdXyGUMqV54HT/xBKqmqMwIibAx7C7eXx7xPVKK4R5gvNHJ6AGbnx/WfyYivfGxNgdDEmSbgnIeo15BENQRwNlwAAnCEVst6uD5YwHEi+AuX1NhTv3IITReeFgmanPxFKhsgUwNgfCPGUALS4vKg226GQcZg7Pg1X5SX0OxHqEtEjqpPWjkEAeURDDY8LWk8TsuJ10OR1P3oVNKNuQ2xdJarMJZAf2wy3OQFKDkI+1ug7xJSQQJi0Stw1eRicbh6Z8dIktYaCX4KvzyOyNwjd0x7mRRFdQx7RUKO5BhqFDMNSkjB5RNcLFYSEUgvTVffBHTccXgbUW1tLvE76IZB8RafmjDE0NLvE58kGTb8XocomO/6ypxhvHyhv26jSC6VRGANayCvqDSThQw1faoc+skXoObUBhqkLsfdUGUq0Miy8chxk8sC/c3vO1eH4xSbMHZ+GvKTgJzZKiUohFEfz8u02cpzgFTWVC3GiYIrLEQEhj2ioYa2G2e6GWZHQVoM5QoxJN0KuMeCyS4Mzl20B2xwpb8Thska4vQwuv291/0bfOnzvcHvhbm83BawjAgnREINZq1B02YqPznnQ2OLq+QUhoFbIMTVHqEu093xdp5nR5y5bsbtI+MJeOzIRV6QOnDo+aoUMKoXwdbE62s2kpoB1RCAhGkp4XHBba+HhGWzq5N7NIeqCSZmxiI9RwahVwOkRPAfGGL672IR/Ha8GY8D4YSZMzQ6xkJrEcBwHY+v/y2Jvt9Q2eUQRQVIh2rNnD+644w6kp6eD4zh8/PHHUpoz+LFVw+nywCWPgSbGCGUXMZzeoJTLcPeUYbj3ykxR6D4/dRk7TteAZwxj0o246YrkgTNE3w7f9TQFEiKHRVg1lwgLSYWoubkZEydOxOuvvy6lGUMHSxXsrTOq43SqPjuNQaP0m5/k5RkUMg7XjUzELWNSej93SSJ8QmRuL0RKTVu1Rt9AABEyko6azZkzB3PmzJHShKGFtQoON49mVRISdJHvlnXFtJx43HRFMjTK/pOyEQ7JBjUy4rSdu7SGVGEGubVaXISSCI0BNXzvdDrhdLZVArRYLBJaMwDx5ZjpkjC8Dz2ijiRJWF0xkoxOM2J0WoAAuyENqDkDWKs67yOCYkAFq9euXQuTySTeMjMzpTZp4OC2A/bG1mTXRMRG0SMa9PjmD9kuS2vHAGZACdGqVatgNpvFW0VFhdQmDRzMQn5ZWvowXJWfIWkN6IGO28v7Lxbgmxxqb6KAdZgMqK6ZWq2GWk1foLAwC6kJScNGICm3/9T5GWhsPViOKrMDC6ZlIj22dXVcpQbQxgkF0qxVwkKRREgMKI+I6AXmi8J9LHVne4NvykP7XDkAbd0zK3XPwkFSIbLZbDh69CiOHj0KACgpKcHRo0dRXl7e/QuJ0PC6AUsVml0elLjjYHG4e34NEZB4vRDk71qIKGAdDpIK0TfffIPJkydj8uTJAIAVK1Zg8uTJWL16tZRmDT4slwDGo9qhxMenm3G4jGosh0tCTBdC5Ct1YrkkZOMTISFpjGjWrFlg9Kb1PY1lAIBaeTLAcUiIoThbuCTohf9dfSePKE1YLNJpE2JFOorDhQLFiIYCDcUAgEtcGgAgQR+9OUSDjfjW+VcWuxsuT7ssfLmyrXvmi8cRQUNCNNhx2gDrZXh4HlVyQYjiY0iIwkWrkovrnHWqXmBqLTRHQhQyA2r4ngiDhgsAAKsqCW5eB4NGMeBTLaRmeJIePEPnpGFTFoADJERhQEI02Kk9CwBoUGcCDuqWRYKbRwdY0QMATK2LVbbUA65mYRFGIiioazaYcdtFj6hKlQMAFKjuS5RaQN9aFqR1gIAIDvKIBjO1ZwDGA/pkjM8fgcRUuxhsJXqHl2eoszmRqFdD3r6sSXweYKsVfgBSxkhn4ACDPKLBCmNA5RHhccpYmLRKXJFqRLKx6yWfieBgjGHT/5XgnQPlqLM5/Xf60jsaLtB8ohAgIRqsWC4J6QYyBZA6QWprBhUcx4mxtmqzw3+nKVMYync1UzZ+CJAQDVZ8a9GnjEGFDfi2rAE1Vkf3ryGCJs0kJLxeauqQbS+TA3E5wuPW+BzRMyREgxHzRWG5Z44DMq/C2Wor9hTVoag68BI/ROhkJQgLQlY0tHTODvB1z+qKomzVwIWEaLDB88C5L4THqROAmARUmoVf7VQTxYciRapRA6WcQ4vLizpbh4mNiaOEHwFLlZDuQfQICdFgo3yfUDtZoQZyr0Oz04N6mwscB2TEaaW2btAgl3EY1vr/LG9o8d+p1gOx2cLjmjNRtmxgQkI0mKgvBkr/IzweMRtQG3CxUfCGEvVqmlEdYbLihQmLF2oDdHmTRwv3NSdp9CwIaB7RYMFSCZz6WPjQp00EUscDEGIYAHlDfcHIFD2cHi9GpRg670zKF7rItlrhvfHNuiYCQh7RYMBSBRzbCnhcQGwWMPIWgOPAGMOFOuHXOjeR0g0ijVGjxNXDE5GoDzBbXalt84oqD0fXsAEICdFAp6kC+G4r4HEK2d/j7wXkgqPb1OKG28ugVsqQEaeT2NAhyLArhfua00IVBKJLSIgGMrVFgifkdggiNGE+oGhL4YiLUeFn1+fhv67M8E9DICJKRUMLPjl6CZUd5xQZ04QuGe9tm9dFBISEaKBy6TBwchvAe4DEkcDEhcJIWQcUchmSDTRs35ecqbbiQm0zvg1UgjfnOuG+8rCw3BAREBKigQZjwIVdQNF24XH6JGDs3UJaQTsam13+a28RfcbkrFhwHHC+xtbZK4rPBeKyBa/o/Jc0gtYFJEQDCbcDOP4hULZPeJ5zDTDqNkDm/zZ6vDy2HbmELftKUd8xKZOIOIl6NcammwAAO8/WwNvxB2DEbCH1o+4cUP2dBBb2f0iIBgqWSuDw34D680Ii6+g7gNzrhRm8HThY2gCLXQhUG7W0tHQ0uHp4AjRKOWosTuwpqvXfqU8Gcq4VHhd9DjTRclkd6RdC9PrrryMnJwcajQYzZszAwYMHpTap/+BxAsU7gcN/B1oaALUBmPxjIHVcwObFtTYcLGkAAFw/KqlzOVOiT4hRK3DrWKFy49GKJnx9vs4/By3zKiGWx3uA4x8Ik08JEck/pe+99x5WrFiBNWvW4PDhw5g4cSJuvfVW1NTUSG2atLiagbKvgQN/EkZcGC8U2pq2VBiN6djcw+PAhXr881gVGAPGDTMhPzXARDuiz8hL0uP6UUKFRrvbC669tyqTAWPmCfEij0sQo7P/BhwWiaztX3BM4oXFZsyYgWnTpuGPf/wjAIDneWRmZuIXv/gFnnzyyW5fa7FYYDKZYDabYTQao2Fu5GEM8DgAV4tQ67i5BmgsBczCoogAhDWy8m4ESxwJl5eH08OD8YBJJ3S7HG4vNu0tgdMttB+dZsQtY1IgoyF7SThbbUVeUozojV6otaHJ7kayQY04jRzasp2QVbUWreNkrQHtHKELp40DVHohpjQICPY7KmmKh8vlwrfffotVq1aJ22QyGWbPno19+/Z1au90OuF0tgVfLZYgf00aSoDiHYFHLPy2sdZNDN+U1rfb4t82VqvEqBS9uPmb0nrwTNjN+Y7R+teoUWB0qlHc8m1ZAzxe31EZ5LwLHOPBILj3QluB/XUqlGvHoAHDwZ+UweU5J5qbYtTgRzOyAAAapRwxKgU0CoaZwxNwRarB/9eYiCodPdHD5U1iqg0AcFweEjwxyGw6hFhXNSYxXuyqXai1wer0gOfk4GUq8JwcACfMlOdkmJ6TIMYFi2psaOy40COE5gAwNTsO8ta2xbXNqGv2H7hIM2qRFd+L1J8JCwBNZBwASYWorq4OXq8XKSn+qyKkpKTgzJnOWctr167Fs88+G/qJPE4h5ydoGDzNXZdv4KEEHG2L63ntFnQ1Us5DAbjaft14Z4vfsLrvKF6ZErwyAUjNF5Yvjh+OE980wOrwAJ72LRFwcuJdk4fBoFaQF9TPYIwhN1EHpZxDrdUJm9MDxoA6eQrqEr4PI2/GpOE8YK4AWurh4K1ocXkBeAH4iwwHAO0S/ZnNCj6QEPlo9oqixWw28B1GUJlcA6h7kfrD+J7bBMmASnpdtWoVVqxYIT63WCzIzMzs+YWxmcKEP6DdKFO7L6yf98ABYMjKdQDg2u1qa6NRKYAYlbgte6RDPAbX7ngcOCgVMkAsWM8hd4xT8GpaD8cptGAKDSBTQCnn2rUF7pmiA88YOI4DB0CpkEGtkEEh4zp5PCYaHeuXcByHK7PjcWVrVRCeZ7C7vXC4vfAyJjjKRg2AGQCAxNEtMDrsgNcFzusCeC84xgNgwhffpGl9zJDY7ILe3bUYcCaN+DmLb3FB6/Jvq1fLgd58biK4XJKkQpSYmAi5XI7Ll/1r+16+fBmpqamd2qvVaqjVYSyHo4oR+uFBwgEYFhv84VND8E4T9T238RFHK7IOOmQyDjFqBWLUgb96yUYdYAwuLzAhPvjzxofQVgokHTVTqVS48sorsWPHDnEbz/PYsWMHZs6cKaFlBEFEE8m7ZitWrMDixYsxdepUTJ8+Ha+99hqam5uxZMkSqU0jCCJKSC5ECxYsQG1tLVavXo3q6mpMmjQJn332WacANkEQgxfJ5xH1hkExj4ggBjHBfkcln1lNEARBQkQQhOSQEBEEITmSB6t7gy+8FXSqB0EQUcX33ewpFD2ghchqtQJAcLOrCYKQDKvVCpPJ1OX+AT1qxvM8KisrYTD0zyRPXwpKRUXFoBzVG8zXN5ivDYje9THGYLVakZ6eDpms60jQgPaIZDIZMjIypDajR4xG46D8MPsYzNc3mK8NiM71decJ+aBgNUEQkkNCRBCE5JAQ9SFqtRpr1qwJr2LAAGAwX99gvjag/13fgA5WEwQxOCCPiCAIySEhIghCckiICIKQHBIigiAkh4QoREJdlfaDDz7AFVdcAY1Gg/Hjx+Nf//qX337GGFavXo20tDRotVrMnj0b586d68tL6JJQru2NN97Addddh7i4OMTFxWH27Nmd2hcUFAiF/9vdbrvttr6+jC4J5foKCws72a7RaPzaDNT3btasWZ2ujeM43H777WKbqL93jAiarVu3MpVKxTZt2sROnjzJli1bxmJjY9nly5cDtt+7dy+Ty+XspZdeYqdOnWK/+c1vmFKpZMePHxfbrFu3jplMJvbxxx+zY8eOsTvvvJPl5uYyu90erctijIV+bT/60Y/Y66+/zo4cOcJOnz7NCgoKmMlkYhcvXhTbLF68mN12222sqqpKvDU0NETrkvwI9fo2b97MjEajn+3V1dV+bQbqe1dfX+93XSdOnGByuZxt3rxZbBPt946EKASmT5/Oli9fLj73er0sPT2drV27NmD7+fPns9tvv91v24wZM9jPfvYzxhhjPM+z1NRU9rvf/U7c39TUxNRqNXv33Xf74Aq6JtRr64jH42EGg4H97W9/E7ctXryYzZs3L9KmhkWo17d582ZmMpm6PN5geu/Wr1/PDAYDs9ls4rZov3fUNQsS36q0s2fPFrd1tyotAOzbt8+vPQDceuutYvuSkhJUV1f7tTGZTJgxY0aXx+wLwrm2jrS0tMDtdiO+w7o1u3btQnJyMvLz8/HQQw+hvr4+orYHQ7jXZ7PZkJ2djczMTMybNw8nT54U9w2m9+7NN9/EwoULERPjv05ZNN87EqIg6W5V2urq6oCvqa6u7ra97z6UY/YF4VxbR5544gmkp6f7fSFuu+02bNmyBTt27MCLL76I3bt3Y86cOfB6vRG1vyfCub78/Hxs2rQJn3zyCd566y3wPI+rr74aFy9eBDB43ruDBw/ixIkTeOCBB/y2R/u9G9DZ90T/YN26ddi6dSt27drlF9BduHCh+Hj8+PGYMGEChg8fjl27duHmm2+WwtSgmTlzpt/aeldffTVGjx6NP//5z3j++ecltCyyvPnmmxg/fjymT5/utz3a7x15REES6qq0AJCamtpte999KMfsC8K5Nh8vv/wy1q1bh88//xwTJkzotm1eXh4SExNx/vz5XtscCr25Ph9KpRKTJ08WbR8M711zczO2bt2KpUuX9nievn7vSIiCJJxVaWfOnOnXHgC++OILsX1ubi5SU1P92lgsFhw4cCCqK92Gu+LuSy+9hOeffx6fffYZpk6d2uN5Ll68iPr6eqSlpUXE7mCJxIrCXq8Xx48fF20f6O8dIEwtcTqd+PGPf9zjefr8vYtaWHwQsHXrVqZWq1lhYSE7deoU++lPf8piY2PFYd2f/OQn7MknnxTb7927lykUCvbyyy+z06dPszVr1gQcvo+NjWWffPIJ++6779i8efMkGwIO5drWrVvHVCoV+/DDD/2GeK1WK2OMMavVylauXMn27dvHSkpK2JdffsmmTJnCRo4cyRwOR1SvLZzre/bZZ9n27dtZcXEx+/bbb9nChQuZRqNhJ0+eFNsM1PfOx7XXXssWLFjQabsU7x0JUYj84Q9/YFlZWUylUrHp06ez/fv3i/tuuOEGtnjxYr/277//Phs1ahRTqVRs7Nix7NNPP/Xbz/M8e/rpp1lKSgpTq9Xs5ptvZmfPno3GpXQilGvLzs5mADrd1qxZwxhjrKWlhd1yyy0sKSmJKZVKlp2dzZYtW9ZpLk40CeX6HnnkEbFtSkoKmzt3Ljt8+LDf8Qbqe8cYY2fOnGEA2Oeff97pWFK8d1QGhCAIyaEYEUEQkkNCRBCE5JAQEQQhOSREBEFIDgkRQRCSQ0JEEITkkBARBCE5JETEgKKwsBCxsbHi82eeeQaTJk0SnxcUFOCuu+6Kul1E7yAhIgLiKxX64IMPdtq3fPlycByHgoICv/aRFoCcnBy89tprftsWLFiAoqKiLl+zYcMGFBYWis9nzZqFRx55JKJ2EZGHhIjokszMTGzduhV2u13c5nA48M477yArK0sSm7RaLZKTk7vcbzKZ/DwmYmBAQkR0yZQpU5CZmYlt27aJ27Zt24asrCxMnjy5V8cO5Kncddddopc1a9YslJWV4dFHHxWLtwOdu2Ydae+ZFRQUYPfu3diwYYN4jJKSEowYMQIvv/yy3+uOHj0KjuOiXqKEECAhIrrl/vvvx+bNm8XnmzZtwpIlS/r8vNu2bUNGRgaee+45VFVVoaqqKuRjbNiwATNnzsSyZcvEY2RlZXW6JgDYvHkzrr/+eowYMSJSl0CEAAkR0S0//vGP8X//938oKytDWVkZ9u7dG1T9mt4SHx8PuVwOg8GA1NTUsIqNmUwmqFQq6HQ68RhyuRwFBQU4e/asuOSO2+3GO++8g/vvvz/Sl0EECZWKJbolKSkJt99+OwoLC8EYw+23347ExESpzeoV6enpuP3227Fp0yZMnz4d//jHP+B0OnHvvfdKbdqQhTwiokfuv/9+FBYW4m9/+1vEvAaZTIaOFWjcbndEjh0MDzzwgBiI37x5MxYsWACdThe18xP+kBARPXLbbbfB5XLB7Xbj1ltvjcgxk5KS/OI+Xq8XJ06c8GujUql6vWpEV8eYO3cuYmJisHHjRnz22WfULZMY6poRPSKXy3H69GnxcVeYzWYcPXrUb1tCQgIyMzM7tb3pppuwYsUKfPrppxg+fDheffVVNDU1+bXJycnBnj17sHDhQqjV6rC6hDk5OThw4ABKS0uh1+sRHx8PmUwmxopWrVqFkSNHRrXONNEZ8oiIoDAajTAajd222bVrFyZPnux3e/bZZwO2vf/++7F48WLcd999uOGGG5CXl4cbb7zRr81zzz2H0tJSDB8+HElJSWHZvXLlSsjlcowZMwZJSUkoLy8X9y1duhQulysqo4BE91CpWGLI8p///Ac333wzKioqOi1QSEQXEiJiyOF0OlFbW4vFixcjNTUVb7/9ttQmDXmoa0YMOd59911kZ2ejqakJL730ktTmECCPiCCIfgB5RARBSA4JEUEQkkNCRBCE5JAQEQQhOSREBEFIDgkRQRCSQ0JEEITkkBARBCE5JEQEQUjO/w9k9XAI0VpSiwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:20:39.550838Z", + "iopub.status.busy": "2024-03-23T06:20:39.550491Z", + "iopub.status.idle": "2024-03-23T06:20:39.714961Z", + "shell.execute_reply": "2024-03-23T06:20:39.713679Z" + }, + "papermill": { + "duration": 0.187903, + "end_time": "2024-03-23T06:20:39.719018", + "exception": false, + "start_time": "2024-03-23T06:20:39.531115", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEpCAYAAADCh6TFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAq7UlEQVR4nO3df1xUZaI/8A8/B8YBFFFAAkYhRS1AIRE3r7qJZOXa2iZbFkiuu6nstWZNl22DWO9GP/yBdknutdC0H1Kt/bhlKqG2aSgKaaiIK6KQAorKbx2Gmef7h19OOzLAoMBwmM/79ZoXnuc855zncIaP5zxz5jk2QggBIiKZsbV0A4iIbgfDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkyeLhlZ6eDrVaDScnJ0RERCAvL6/D+jU1NViyZAm8vb2hUCgwcuRI7Nixo5daS0R9hb0lN56VlQWNRoOMjAxEREQgLS0N0dHRKC4uxtChQ9vUb25uRlRUFIYOHYpPPvkEPj4+OH/+PAYOHGj2Ng0GAy5evAgXFxfY2Nh0494QUXcQQqC+vh7Dhg2DrW0H51fCgiZMmCCWLFkiTev1ejFs2DCRmppqsv6GDRvEiBEjRHNz821vs7y8XADgiy+++virvLy8w79lGyEsM6pEc3MzlEolPvnkEzz66KNSeVxcHGpqavD555+3Weahhx6Cu7s7lEolPv/8cwwZMgRPPvkkVqxYATs7O5Pb0Wq10Gq10nRtbS38/PxQWloKFxeXbt+vvkKn02Hv3r2YNm0aHBwcLN0c6gbWckzr6+sxfPhw1NTUwM3Nrd16FrtsrK6uhl6vh6enp1G5p6cnTp06ZXKZs2fPYs+ePZg3bx527NiBM2fOYPHixdDpdEhOTja5TGpqKlJSUtqU5+bmQqlU3vmO9GFKpRKHDh2ydDOoG1nDMW1qagKATrt1LNrn1VUGgwFDhw7F//7v/8LOzg5hYWG4cOEC3njjjXbDKzExERqNRpquq6uDr68vZsyYAVdX195qeq/T6XTIzs5GVFRUv/5f2ppYyzGtq6szq57FwsvDwwN2dnaoqqoyKq+qqoKXl5fJZby9veHg4GB0iTh69GhUVlaiubkZjo6ObZZRKBRQKBRtyh0cHPr1G6CVteynNenvx9TcfbPYrRKOjo4ICwtDTk6OVGYwGJCTk4PIyEiTy/ziF7/AmTNnYDAYpLLTp0/D29vbZHARUf9l0fu8NBoNNm7ciHfffRdFRUVYtGgRGhsbER8fDwCIjY1FYmKiVH/RokW4evUqli5ditOnT+Orr77CK6+8giVLllhqF4jIQiza5xUTE4PLly8jKSkJlZWVCA0Nxc6dO6VO/LKyMqP7PHx9fbFr1y48//zzCA4Oho+PD5YuXYoVK1ZYaheIyEIs3mGfkJCAhIQEk/P27dvXpiwyMhIHDx7s4VYRUV9n8fAioraampra3DLUcF2L7wtLMMjjCFTOP38IFRQU1O9v+zGF4UXUB506dQphYWEm571+y3R+fj7Gjx/f843qYxheRH1QUFAQ8vPzjcqKK2qg+bgQax6/F6O8BxrVtUYML6I+SKlUtjmbsj1/BYrvrmP0PSEI9R9soZb1HRYfEoeI6HYwvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiV+MbufuHX8p/bGfgKsd/wn6l8YXv1Ee+M/3Tr2E2C94z9R/8Lw6iduHf+pvbGfWusSyR3Dq5+4dfwnjv1E/R077IlIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSpT4RXeno61Go1nJycEBERgby8vHbrbt68GTY2NkYvJyenXmwtEfUFFg+vrKwsaDQaJCcno6CgACEhIYiOjsalS5faXcbV1RUVFRXS6/z5873YYiLqCyweXmvWrMHChQsRHx+PMWPGICMjA0qlEpmZme0uY2NjAy8vL+nl6enZiy0mor7AooMRNjc3Iz8/H4mJiVKZra0tpk+fjtzc3HaXa2hogL+/PwwGA8aPH49XXnkFY8eONVlXq9VCq9VK03V1dQAAnU4HnU7XTXvS97S0tEg/+/N+WhNrOabm7ptFw6u6uhp6vb7NmZOnp6fRwyT+3ahRo5CZmYng4GDU1tZi1apVmDRpEk6cOIG77rqrTf3U1FSkpKS0Kd+9e3e/fghFeQMA2OPgwYO4cNzSraHuYC3HtKmpyax6shsGOjIyEpGRkdL0pEmTMHr0aPzP//wPVq5c2aZ+YmIiNBqNNF1XVwdfX1/MmDEDrq6uvdJmSzhWdhUoPIKJEycixM/d0s2hbmAtx7T16qgzFg0vDw8P2NnZoaqqyqi8qqoKXl5eZq3DwcEB48aNw5kzZ0zOVygUUCgUbcodHBzg4ODQ9UbLhL29vfSzP++nNbGWY2ruvlm0w97R0RFhYWHIycmRygwGA3JycozOrjqi1+tRWFgIb2/vnmomEfVBFr9s1Gg0iIuLQ3h4OCZMmIC0tDQ0NjYiPj4eABAbGwsfHx+kpqYCAP72t79h4sSJCAwMRE1NDd544w2cP38ev/vd7yy5G0TUyyweXjExMbh8+TKSkpJQWVmJ0NBQ7Ny5U+rELysrg63tzyeI165dw8KFC1FZWYlBgwYhLCwM33//PcaMGWOpXSAiC7B4eAFAQkICEhISTM7bt2+f0fTatWuxdu3aXmgVEfVlFr9JlYjodjC8iEiWGF5EJEsMLyKSpT7RYU9EQGl1Ixq1Le3OL7ncKP1svWH1VgMU9hjuMaBH2tfXMLyI+oDS6kZMW7XPrLp/+qSww/l7l021igBjeBH1Aa1nXGkxoQgcqjJd57oWX+7LxSNTIzHAue1X3s5casBzWUc7PHvrTxheRH1I4FAV7vFxMzlPp9Ohcggw3n9Qv/5uo7nYYU9EssQzLxnrqIPXnM5dwLo6eKl/YXjJlLkdvJ117gLW08FL/QvDS6Y66+DtrHMXsL4OXupfGF4y114HLzt3qb9jhz0RyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE7zbKlFZ/A7ZOF1BaVwxbp7ZfzG5pacHFlosoulrU7pA4pXUNsHW6AK3+BgDTA+AR9VUML5m62HgeA4a/ib/kdVzvrZ1vdTh/wHDgYmMowuDZja0j6nkML5kaNsAfjaV/xLqYUASYGBKnpaUFB/YfwC/u/0W7Z14llxqwNOsohk3z7+nmEnU7hpdMKeycYLjhg+GuozBmsOkhcUrtSzHafXS7Q+IYbtTCcOMyFHZOPd1com7H8CLqAzrrwwQ678e0tj5MhhdRH2BuHybQcT+mNfVhMryI+oDO+jCBzvsxra0Pk+FF1Ad01ocJdN6PaW19mF2+SfXs2bM90Q4ioi7pcngFBgZi2rRpeO+993Djxo1uaUR6ejrUajWcnJwQERGBvDwzLvwBbNu2DTY2Nnj00Ue7pR1EJB9dDq+CggIEBwdDo9HAy8sLf/jDH8wOG1OysrKg0WiQnJyMgoIChISEIDo6GpcuXepwuXPnzmHZsmWYPHnybW+biOSry+EVGhqKdevW4eLFi8jMzERFRQXuv/9+3HPPPVizZg0uX77cpfWtWbMGCxcuRHx8PMaMGYOMjAwolUpkZma2u4xer8e8efOQkpKCESNGdHUXiKgfuO0Oe3t7e8yZMwcPP/ww3nrrLSQmJmLZsmX4y1/+grlz5+K1116Dt7d3h+tobm5Gfn4+EhMTpTJbW1tMnz4dubm57S73t7/9DUOHDsWCBQvw3XffdbgNrVYLrVYrTdfV1QG42fmp0+nM2dU+qaWlRfppaj9ayzrax87WQb3HnGPR2THtL8fT3LbfdngdOXIEmZmZ2LZtGwYMGIBly5ZhwYIF+Omnn5CSkoLZs2d3ejlZXV0NvV4PT0/je1I8PT1x6tQpk8vs378f77zzDo4ePWpWO1NTU5GSktKmfPfu3VAqlWatoy8qbwAAe+zfvx/nTX+yDgDIzs6+43VQz+vKsWjvmPaX49nU1GRWvS6H15o1a7Bp0yYUFxfjoYcewpYtW/DQQw/B1vbmFejw4cOxefNmqNXqrq66U/X19Xj66aexceNGeHh4mLVMYmIiNBqNNF1XVwdfX1/MmDEDrq6u3d7G3nLiYh1WFR7E/fffj7HD2u6HTqdDdnY2oqKi2v16UGfroN5jzrHo7Jj2l+PZenXUmS6H14YNG/DMM89g/vz57V4WDh06FO+8806n6/Lw8ICdnR2qqqqMyquqquDl5dWmfklJCc6dO4dZs2ZJZQaDAcDNy9ji4mIEBAQYLaNQKKBQKNqsy8HBod0/ajlovUnR3t6+w/3oaD/NXQf1vK4ci/aOaX85nua2vcvhlZ2dDT8/P+lMq5UQAuXl5fDz84OjoyPi4uI6XZejoyPCwsKQk5Mj3e5gMBiQk5ODhISENvWDgoJQWFhoVPbXv/4V9fX1WLduHXx9fbu6O0QkU10Or4CAAFRUVGDo0KFG5VevXsXw4cOh1+u7tD6NRoO4uDiEh4djwoQJSEtLQ2NjI+Lj4wEAsbGx8PHxQWpqKpycnHDPPfcYLT9w4EAAaFNORP1bl8NLCGGyvKGhAU5OXf9aQkxMDC5fvoykpCRUVlYiNDQUO3fulDrxy8rK2pzlERGZHV6tnd42NjZISkoy+qROr9fj0KFDCA0Nva1GJCQkmLxMBIB9+/Z1uOzmzZtva5tEJG9mh9cPP/wA4OaZV2FhIRwdHaV5jo6OCAkJwbJly7q/hUREJpgdXnv37gUAxMfHY926dbK+zYCI5K/LfV6bNm3qiXYQEXWJWeE1Z84cbN68Ga6urpgzZ06Hdbdv394tDSMi6ohZ4eXm5gYbGxvp30RElmZWeP37pSIvG4moL+ANVEQkS2adeY0bN066bOxMQUHBHTWIiMgcZoUXh1kmor7GrPBKTk7u6XYQEXUJ+7yISJbMOvNyd3fH6dOn4eHhgUGDBnXY/3X16tVuaxwRUXvMCq+1a9fCxcVF+re5nfdERD3FrPD694EF58+f31NtISIyW5f7vOzs7Ew+U/HKlSuws7PrlkYREXWmy+HV3mCEWq3WaJgcIqKeZPaoEuvXrwdwczDCt99+GyrVz89W0uv1+Oc//4mgoKDubyERkQlmh9fatWsB3DzzysjIMLpEdHR0hFqtRkZGRve3kIjIBLPDq7S0FAAwbdo0bN++HYMGDeqxRhERdabLgxG2jqhKRGRJXQ6vZ555psP5mZmZt90YIiJzdTm8rl27ZjSt0+lw/Phx1NTU4Je//GW3NYyIqCNdDq9PP/20TZnBYMCiRYsQEBDQLY0iIupMt3wx29bWFhqNRvpEkoiop3XbqBIlJSVoaWnprtUREXWoy5eNrU/ObiWEQEVFBb766iuj70ASEfWkLodX65OzW9na2mLIkCFYvXp1p59EEhF1F97nRUSyxJFUiUiWGF5EJEsMLyKSJYYXEclSt4XXTz/9hN///vfdtToiog51W3hduXIF77zzzm0tm56eDrVaDScnJ0RERCAvL6/dutu3b0d4eDgGDhyIAQMGIDQ0FFu3br3dZhORTFn8sjErKwsajQbJyckoKChASEgIoqOjTY6TD9x8DNuLL76I3Nxc/Pjjj4iPj0d8fDx27drVyy0nIkuyeHitWbMGCxcuRHx8PMaMGYOMjAwolcp2h9aZOnUqfv3rX2P06NEICAjA0qVLERwcjP379/dyy4nIkrp8k2p3am5uRn5+PhITE6UyW1tbTJ8+Hbm5uZ0uL4TAnj17UFxcjNdee81kHa1WC61WK03X1dUBuDmUj06nu8M9sJzW75G2tLSY3I/Wso72sbN1UO8x51h0dkz7y/E0t+1mh9ecOXM6nF9TU2PuqiTV1dXQ6/Xw9PQ0Kvf09MSpU6faXa62thY+Pj7QarWws7PDW2+9haioKJN1U1NTkZKS0qZ89+7dUCqVXW5zX1HeAAD22L9/P86r2q+XnZ19x+ugnldSBwD22LbrAO4aYPoJXToDcFULnP1HNhxMXDNVXbcBYCf749nU1GRWPbPDy83NrdP5sbGx5q7ujri4uODo0aNoaGhATk4ONBoNRowYgalTp7apm5iYaPRl8rq6Ovj6+mLGjBlwdXXtlfb2hBMX67Cq8CDuv/9+jB3Wdj90Oh2ys7MRFRUFBweH21oH9Z6PjvwEnDiJbWfv/Nmn0Q9MgXrwgG5olWW0Xh11xuzw2rRp0203pj0eHh6ws7NDVVWVUXlVVRW8vLzaXc7W1haBgYEAgNDQUBQVFSE1NdVkeCkUCigUijblDg4O7f5Ry4G9vb30s6P96Gg/zV0H9byZwT6ws7NDwFAVnB1MB1hxRS3+9EkhVv/mXozyNn0yMUBhj+Ee8g0uAGa/Fy3a5+Xo6IiwsDDk5OTg0UcfBXBzVNacnBwkJCSYvR6DwWDUr2UNruv0AIDjF2pNzm+8rsWRy4DX+WsY4Nw2vAHgzKWGHmsfdY37AEf8doJfh3Va+7QChgzAPT4dXwlZA7PDy9zhbrr6AA6NRoO4uDiEh4djwoQJSEtLQ2NjI+Lj4wEAsbGx8PHxQWpqKoCbfVjh4eEICAiAVqvFjh07sHXrVmzYsKFL25W7kv8fPH/eXthBLXtsPXO403UNUFj0/zCi22L2u3bz5s3w9/fHuHHjIITpDsXbERMTg8uXLyMpKQmVlZUIDQ3Fzp07pU78srIy2Nr+3DvZ2NiIxYsX46effoKzszOCgoLw3nvvISYmptvaJAczxt68rG7vMsOcSwygf1xmkHWyEWYm0ZIlS/Dhhx/C398f8fHxeOqpp+Du7t7T7et2dXV1cHNzQ21traw77Dtz9PwVPLrhID5bNBGh/oMt3RzqBtZyTM39GzX7JtX09HRUVFRg+fLl+L//+z/4+vpi7ty52LVrV7eeiRERmaNLd9grFAo88cQTyM7OxsmTJzF27FgsXrwYarUaDQ3s/CWi3nPbXw+ytbWFjY0NhBDQ6/Xd2SYiok51Kby0Wi0+/PBDREVFYeTIkSgsLMR///d/o6ysDCqVjG/pJSLZMfvTxsWLF2Pbtm3w9fXFM888gw8//BAeHh492TYionaZHV4ZGRnw8/PDiBEj8O233+Lbb781WW/79u3d1jgiovaYHV6xsbGwsbHpybYQEZmtSzepEhH1FRYfjJCI6HYwvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSz1ifBKT0+HWq2Gk5MTIiIikJeX127djRs3YvLkyRg0aBAGDRqE6dOnd1ifiPoni4dXVlYWNBoNkpOTUVBQgJCQEERHR+PSpUsm6+/btw9PPPEE9u7di9zcXPj6+mLGjBm4cOFCL7eciCzJ4uG1Zs0aLFy4EPHx8RgzZgwyMjKgVCqRmZlpsv7777+PxYsXIzQ0FEFBQXj77bdhMBiQk5PTyy0nIkuyt+TGm5ubkZ+fj8TERKnM1tYW06dPR25urlnraGpqgk6ng7u7u8n5Wq0WWq1Wmq6rqwMA6HQ66HS6O2h939bS0iL97M/7aU2s5Ziau28WDa/q6mro9Xp4enoalXt6euLUqVNmrWPFihUYNmwYpk+fbnJ+amoqUlJS2pTv3r0bSqWy642WifIGALDHwYMHceG4pVtD3cFajmlTU5NZ9SwaXnfq1VdfxbZt27Bv3z44OTmZrJOYmAiNRiNN19XVSf1krq6uvdXUXnes7CpQeAQTJ05EiJ/ps1KSF2s5pq1XR52xaHh5eHjAzs4OVVVVRuVVVVXw8vLqcNlVq1bh1VdfxTfffIPg4OB26ykUCigUijblDg4OcHBwuL2Gy4C9vb30sz/vpzWxlmNq7r5ZtMPe0dERYWFhRp3trZ3vkZGR7S73+uuvY+XKldi5cyfCw8N7o6lE1MdY/LJRo9EgLi4O4eHhmDBhAtLS0tDY2Ij4+HgAQGxsLHx8fJCamgoAeO2115CUlIQPPvgAarUalZWVAACVSgWVSmWx/SCi3mXx8IqJicHly5eRlJSEyspKhIaGYufOnVInfllZGWxtfz5B3LBhA5qbm/Gb3/zGaD3Jycl4+eWXe7PpRGRBFg8vAEhISEBCQoLJefv27TOaPnfuXM83iIj6PIvfpEpEdDsYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJb6xBOz6c41NTXh1KlT0nRxRQ20lWdQdNwZhisDjeoGBQVBqVT2cguJuhfDq584deoUwsLC2pQ/+W7buvn5+Rg/fnwvtIqo5zC8+omgoCDk5+dL0w3Xtfhqby4enhYJlbOiTV0iuWN49RNKpdLobEqn0+Fa9SVETgiHg4ODBVtG1DPYYU9EssTwIiJZYngRkSwxvIhIliweXunp6VCr1XByckJERATy8vLarXvixAk89thjUKvVsLGxQVpaWu81lIj6FIt+2piVlQWNRoOMjAxEREQgLS0N0dHRKC4uxtChQ9vUb2pqwogRI/D444/j+eeft0CLiXrHrTcdA+3feGy1Nx0LC5owYYJYsmSJNK3X68WwYcNEampqp8v6+/uLtWvXdnmbtbW1AoCora3t8rJy0tzcLD777DPR3Nxs6abQbcjPzxcAzHrl5+dburndyty/UYudeTU3NyM/Px+JiYlSma2tLaZPn47c3Nxu245Wq4VWq5Wm6+rqANy8D0qn03Xbdvqa1n3rz/vYnwUEBODQoUNGZQ3Xtdj13WFET77P6MbjgICAfnWczd0Xi4VXdXU19Ho9PD09jco9PT3bnC7fidTUVKSkpLQp3717t1WcamdnZ1u6CdSNJt0bgPqaq6iv+bmsoqLCYu3pCU1NTWbV6/d32CcmJkKj0UjTdXV18PX1xYwZM+Dq6mrBlvUsnU6H7OxsREVF8Q77fsJajmnr1VFnLBZeHh4esLOzQ1VVlVF5VVUVvLy8um07CoUCCoWiTbmDg0O/fgO0spb9tCb9/Ziau28Wu1XC0dERYWFhyMnJkcoMBgNycnIQGRlpqWYRkUxY9LJRo9EgLi4O4eHhmDBhAtLS0tDY2Ij4+HgAQGxsLHx8fJCamgrgZif/yZMnpX9fuHABR48ehUqlQmBgoMX2g4h6n0XDKyYmBpcvX0ZSUhIqKysRGhqKnTt3Sp34ZWVlsLX9+eTw4sWLGDdunDS9atUqrFq1ClOmTMG+fft6u/lEZEEW77BPSEhAQkKCyXm3BpJarYYQohdaRUR9ncW/HkREdDssfubV21rP3Mz9OFaudDodmpqaUFdX168/mbIm1nJMW/82O7vKsrrwqq+vBwD4+vpauCVE1JH6+nq4ubm1O99GWFknksFgwMWLF+Hi4gIbGxtLN6fHtN6MW15e3q9vxrUm1nJMhRCor6/HsGHDjD6wu5XVnXnZ2trirrvusnQzeo2rq2u/fqNbI2s4ph2dcbVihz0RyRLDi4hkieHVTykUCiQnJ5v8XifJE4+pMavrsCei/oFnXkQkSwwvIpIlhhcRyRLDqxPz58/Ho48+2q3rnDp1Kp577rkO66jVaj7ajagDVhVe5oQGycvLL7+M0NBQSzejXX3tPdfX2nMnrCq8iOSoubnZ0k3ok6wmvObPn49vv/0W69atg42NDWxsbFBSUoIFCxZg+PDhcHZ2xqhRo7Bu3TqTy6ekpGDIkCFwdXXFs88+a/YbqrGxEbGxsVCpVPD29sbq1avb1Ll06RJmzZoFZ2dnDB8+HO+//36bOjY2NtiwYQNmzpwJZ2dnjBgxAp988ok0/9y5c7CxscFHH32EyZMnw9nZGffddx9Onz6Nw4cPIzw8HCqVCjNnzsTly5fN/K0BmZmZGDt2LBQKBby9vY3GXisrK8Ps2bOhUqng6uqKuXPnGj2ToPWsaOvWrVCr1XBzc8Nvf/tb6cvxwM3vmr7++usIDAyEQqGAn58f/v73v0vzV6xYgZEjR0KpVGLEiBF46aWXpEdjbd68GSkpKTh27Jh0TDdv3mz2vvW0233PtXZV/P3vf8ewYcMwatQoAMD333+P0NBQODk5ITw8HJ999hlsbGxw9OhRadnjx49j5syZUKlU8PT0xNNPP43q6up223Pu3Lne+nV0vx5+fmSfUVNTIyIjI8XChQtFRUWFqKioEDdu3BBJSUni8OHD4uzZs+K9994TSqVSZGVlScvFxcUJlUolYmJixPHjx8WXX34phgwZIv7yl7+Ytd1FixYJPz8/8c0334gff/xRPPLII8LFxUUsXbpUqjNz5kwREhIicnNzxZEjR8SkSZOEs7Oz0UN1AYjBgweLjRs3iuLiYvHXv/5V2NnZiZMnTwohhCgtLRUARFBQkNi5c6c4efKkmDhxoggLCxNTp04V+/fvFwUFBSIwMFA8++yzZrX9rbfeEk5OTiItLU0UFxeLvLw8qU16vV6EhoaK+++/Xxw5ckQcPHhQhIWFiSlTpkjLJycnC5VKJebMmSMKCwvFP//5T+Hl5WX0u1u+fLkYNGiQ2Lx5szhz5oz47rvvxMaNG6X5K1euFAcOHBClpaXiiy++EJ6enuK1114TQgjR1NQk/vSnP4mxY8dKx7SpqcmsfesNd/qee/rpp8Xx48fF8ePHRW1trXB3dxdPPfWUOHHihNixY4cYOXKkACB++OEHIYQQ165dE0OGDBGJiYmiqKhIFBQUiKioKDFt2rR229PS0mKJX023sJrwEkKIKVOmGIWGKUuWLBGPPfaYNB0XFyfc3d1FY2OjVLZhwwahUqmEXq/vcF319fXC0dFRfPTRR1LZlStXhLOzs9SO4uJiAUDk5eVJdYqKigSANuF1a+hERESIRYsWCSF+Dq+3335bmv/hhx8KACInJ0cqS01NFaNGjeqw3a2GDRsmXnzxRZPzdu/eLezs7ERZWZlUduLECaN9SU5OFkqlUtTV1Ul1XnjhBRERESGEEKKurk4oFAqjsOrMG2+8IcLCwqTp5ORkERISYvbyve1233Oenp5Cq9VKZRs2bBCDBw8W169fl8o2btxoFF4rV64UM2bMMFp3eXm5ACCKi4vNbo9cWN2oErdKT09HZmYmysrKcP36dTQ3N7fpAA4JCTF6QG1kZCQaGhpQXl4Of3//dtddUlKC5uZmRERESGXu7u7SZQAAFBUVwd7eHmFhYVJZUFAQBg4c2GZ9tz5VKTIy0uiSAQCCg4Olf7c+C+Dee+81Krt06VK7bW516dIlXLx4EQ888IDJ+UVFRfD19TUaF23MmDEYOHAgioqKcN999wG4+ampi4uLVMfb21vaflFREbRabbvbAICsrCysX78eJSUlaGhoQEtLi+xHVDDnPXfvvffC0dFRmi4uLkZwcDCcnJyksgkTJhgtc+zYMezduxcqlarNNktKSjBy5Mju3RELs5o+L1O2bduGZcuWYcGCBdi9ezeOHj2K+Ph4WXeQ/vsIm63jld1aZjAYOl2Ps7Nzt7fn1u13to3c3FzMmzcPDz30EL788kv88MMPePHFF2V9fMx9zw0YMKDL625oaMCsWbNw9OhRo9e//vUv/Md//Ed37UKfYVXh5ejoCL1eL00fOHAAkyZNwuLFizFu3DgEBgaipKSkzXLHjh3D9evXpemDBw9CpVJ1OhprQEAAHBwccOjQIans2rVrOH36tDQdFBSElpYW5OfnS2XFxcWoqalps76DBw+2mR49enSHbbhdLi4uUKvVRs/V/HejR49GeXk5ysvLpbKTJ0+ipqYGY8aMMWsbd999N5ydndvdxvfffw9/f3+8+OKLCA8Px913343z588b1bn1mPY1t/ueu9WoUaNQWFgIrVYrlR0+fNiozvjx43HixAmo1WoEBgYavVrDsK//vrrCqsJLrVbj0KFDOHfuHKqrq3H33XfjyJEj2LVrF06fPo2XXnqpzRsCuPlR9YIFC3Dy5Ens2LEDycnJSEhI6HCURwBQqVRYsGABXnjhBezZswfHjx/H/PnzjZYbNWoUHnzwQfzhD3/AoUOHkJ+fj9/97ncmz0o+/vhjZGZm4vTp00hOTkZeXl67T17qDi+//DJWr16N9evX41//+hcKCgrw5ptvAgCmT5+Oe++9F/PmzUNBQQHy8vIQGxuLKVOmIDw83Kz1Ozk5YcWKFVi+fDm2bNmCkpISHDx4EO+88w6Am+FWVlaGbdu2oaSkBOvXr8enn35qtA61Wo3S0lIcPXoU1dXVRn/cfcHtvudu9eSTT8JgMOD3v/89ioqKsGvXLqxatQrAz2fYS5YswdWrV/HEE0/g8OHDKCkpwa5duxAfHy8F1q3tMecsvM+ydKdbbyouLhYTJ04Uzs7OAoA4deqUmD9/vnBzcxMDBw4UixYtEn/+85+NOoDj4uLE7NmzRVJSkhg8eLBQqVRi4cKF4saNG2Zts76+Xjz11FNCqVQKT09P8frrr7fpNK2oqBAPP/ywUCgUws/PT2zZskX4+/u36bBPT08XUVFRQqFQCLVabfQJVWuHfWvnrRBC7N27VwAQ165dk8o2bdok3NzczP6dZWRkiFGjRgkHBwfh7e0t/vjHP0rzzp8/L371q1+JAQMGCBcXF/H444+LyspKab6pzvS1a9cKf39/aVqv14v/+q//Ev7+/sLBwUH4+fmJV155RZr/wgsvSL/3mJgYsXbtWqP237hxQzz22GNi4MCBAoDYtGmT2fvWG+7kPXerAwcOiODgYOHo6CjCwsLEBx98IK2z1enTp8Wvf/1rMXDgQOHs7CyCgoLEc889JwwGg8n2lJaW9vBvoOdwSByZsLGxwaefftrtX1Ui+Xr//fcRHx+P2trabuujlBOr/7SRSC62bNmCESNGwMfHB8eOHcOKFSswd+5cqwwugOF1R8rKyjrsnD558iT8/Px6sUVdY+oj9VZff/01Jk+e3Iutoc5UVlYiKSkJlZWV8Pb2xuOPP270bQRrw8vGO9DS0tLh1yvUajXs7fvu/w9nzpxpd56Pj4/V/o9O8sDwIiJZsqpbJYio/2B4EZEsMbyISJYYXkQkSwwvIpIlhhf1qPnz50ujdjo4OMDT0xNRUVHIzMzs0vfqNm/ebHKYoJ7WEw9goe7B8KIe9+CDD6KiogLnzp3D119/jWnTpmHp0qV45JFH0NLSYunmkVxZ8ouV1P+19yXjnJwcAUAaRXX16tXinnvuEUqlUtx1111i0aJFor6+Xgjx8xfM//2VnJwshBBiy5YtIiwsTKhUKuHp6SmeeOIJUVVVJW3n6tWr4sknnxQeHh7CyclJBAYGiszMTGl+WVmZePzxx4Wbm5sYNGiQ+NWvfiV9WTk5ObnNdvfu3dsjvyfqOp55kUX88pe/REhICLZv3w4AsLW1xfr163HixAm8++672LNnD5YvXw4AmDRpEtLS0uDq6oqKigpUVFRg2bJlAACdToeVK1fi2LFj+Oyzz3Du3DnMnz9f2s5LL72EkydP4uuvv0ZRURE2bNgADw8Padno6Gi4uLjgu+++w4EDB6BSqfDggw+iubkZy5Ytw9y5c6Uzx4qKCkyaNKl3f1HUPkunJ/Vv7Z15CSFETEyMGD16tMl5H3/8sRg8eLA0be5QPocPHxYApLO2WbNmifj4eJN1t27dKkaNGiUNFyOEEFqtVjg7O4tdu3Z12n6yLJ55kcUIIaSB9L755hs88MAD8PHxgYuLC55++mlcuXIFTU1NHa4jPz8fs2bNgp+fH1xcXDBlyhQAN780DwCLFi3Ctm3bEBoaiuXLl+P777+Xlj127BjOnDkDFxcXqFQqqFQquLu748aNG2aNbkqWxfAiiykqKsLw4cNx7tw5PPLIIwgODsY//vEP5OfnIz09HUDHD1xtbGxEdHQ0XF1d8f777+Pw4cPSSKuty82cORPnz5/H888/Lz1QpPWSs6GhAWFhYW3GfD99+jSefPLJHt57ulN9d8gD6tf27NmDwsJCPP/888jPz4fBYMDq1aulIbI/+ugjo/qmxl4/deoUrly5gldffVV6nsCRI0fabGvIkCGIi4tDXFwcJk+ejBdeeAGrVq3C+PHjkZWVhaFDh7b7RKL+NOZ7f8MzL+pxWq0WlZWVuHDhAgoKCvDKK69g9uzZeOSRRxAbG4vAwEDodDq8+eabOHv2LLZu3YqMjAyjdajVajQ0NCAnJwfV1dVoamqCn58fHB0dpeW++OILrFy50mi5pKQkfP755zhz5gxOnDiBL7/8Unpoybx58+Dh4YHZs2fju+++Q2lpKfbt24f//M//xE8//SRt98cff0RxcTGqq6ulp3VTH2DpTjfq3+Li4qTbDOzt7cWQIUPE9OnTRWZmptFDe9esWSO8vb2Fs7OziI6OFlu2bGkz/v6zzz4rBg8ebHSrxAcffCDUarVQKBQiMjJSfPHFF20exDp69Gjh7Ows3N3dxezZs8XZs2eldVZUVIjY2Fjh4eEhFAqFGDFihFi4cKGora0VQghx6dIlERUVJVQqFW+V6GM4nhcRyRIvG4lIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikqX/B6Jtz74vW0mjAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T06:20:39.767779Z", + "iopub.status.busy": "2024-03-23T06:20:39.767403Z", + "iopub.status.idle": "2024-03-23T06:20:40.001958Z", + "shell.execute_reply": "2024-03-23T06:20:40.000950Z" + }, + "papermill": { + "duration": 0.256086, + "end_time": "2024-03-23T06:20:40.003981", + "exception": false, + "start_time": "2024-03-23T06:20:39.747895", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHMElEQVR4nO3deVzU1f4/8NfMwAz7ouzIIooIiqAghmlo4UXTtOX+IvOmkplmlknmrmjeQtOMMpebXdMWE+uq9a20q9w0F9xQFAUxkE3ZRfZltvP7Y2SYgQFmcBYG3s/HYx4wnzmf+Zz5DJ8355zPWTiMMQZCCDEyXENngBBCuoKCFyHEKFHwIoQYJQpehBCjRMGLEGKUKHgRQowSBS9CiFGi4EUIMUomhs6AvkmlUhQWFsLa2hocDsfQ2SGEtMIYQ01NDdzc3MDltl++6nXBq7CwEB4eHobOBiGkEwUFBejXr1+7r/e64GVtbQ1AdmJsbGwMnBtCSGvV1dXw8PCQX6vt6RbBa/v27di8eTOKi4sRFBSEbdu2ISwsTGXavXv3IiYmRmmbQCBAY2OjWsdqrira2NhQ8CKkG+usWcfgDfaJiYmIjY1FXFwcrly5gqCgIERFRaG0tLTdfWxsbFBUVCR/5OXl6THHhJDuwODBa+vWrZg7dy5iYmIQEBCAXbt2wcLCAnv27Gl3Hw6HAxcXF/nD2dlZjzkmhHQHBg1eQqEQKSkpiIyMlG/jcrmIjIxEcnJyu/vV1tbCy8sLHh4emDZtGm7evNlu2qamJlRXVys9CCHGz6DBq7y8HBKJpE3JydnZGcXFxSr38fPzw549e/DTTz/h22+/hVQqxejRo3H37l2V6ePj42Frayt/0J1GQnoGg1cbNRUeHo6ZM2ciODgYEREROHToEBwdHfGvf/1LZfoVK1agqqpK/igoKNBzjgkhumDQu40ODg7g8XgoKSlR2l5SUgIXFxe13sPU1BTDhw9HVlaWytcFAgEEAsEj55UQ0r0YtOTF5/MREhKCpKQk+TapVIqkpCSEh4er9R4SiQRpaWlwdXXVVTYfyb9OZWNxYiqkUpptmxBtMng/r9jYWMyaNQuhoaEICwtDQkIC6urq5H25Zs6cCXd3d8THxwMA3n//fTz22GMYOHAgKisrsXnzZuTl5eG1114z5MdoV/zRWwCAF0b0wxhfBwPnhpCew+DBKzo6GmVlZVi7di2Ki4sRHByMY8eOyRvx8/PzlcY3PXjwAHPnzkVxcTHs7e0REhKCc+fOISAgwFAfQS31QrGhs0BIj8LpbasHVVdXw9bWFlVVVXrpYe+9/FcAwO6ZoZgQQP3RCOmMuteo0d1tNFY0fwUh2kXBixBilCh46QlNHUaIdlHw0hMKXoRoFwUvQohRouClJxxqsidEqyh4EUKMEgUvfaGCFyFaRcFLTyh2EaJdFLwIIUaJghchxChR8NITWuCWEO2i4KUnFLoI0S4KXnpCBS9CtIuCl55QJ1VCtIuCFyHEKFHwIoQYJQpeekJtXoRoFwUvPaHYRYh2UfAihBglCl76QkUvQrSKgpcOKS7MRF0lCNEuCl461LsWlSNEvyh46QndbSREuyh46RAVvAjRHQpeOqTc5kUI0SYKXjqkWPKiKXEI0S4KXjqk2GBPsYsQ7aLgRQgxShS8dIhRkz0hOkPBS4eUqo2GywYhPRIFLz2hNi9CtKtbBK/t27fD29sbZmZmGDVqFC5evKjWfgcOHACHw8Gzzz6r2wxqBUUvQrTJ4MErMTERsbGxiIuLw5UrVxAUFISoqCiUlpZ2uF9ubi6WLFmCsWPH6imnmqPhQYTojsGD19atWzF37lzExMQgICAAu3btgoWFBfbs2dPuPhKJBDNmzMD69evh4+Ojx9xqRrHBnqqNhGiXQYOXUChESkoKIiMj5du4XC4iIyORnJzc7n7vv/8+nJycMGfOnE6P0dTUhOrqaqWHvlDJixDdMWjwKi8vh0QigbOzs9J2Z2dnFBcXq9znzJkz+Pe//43du3erdYz4+HjY2trKHx4eHo+cb3Up9bDX21EJ6R0MXm3URE1NDV555RXs3r0bDg4Oau2zYsUKVFVVyR8FBQU6ziUhRB9MDHlwBwcH8Hg8lJSUKG0vKSmBi4tLm/TZ2dnIzc3FM888I98mlUoBACYmJsjMzMSAAQOU9hEIBBAIBDrIfeeUBmZToxchWmXQkhefz0dISAiSkpLk26RSKZKSkhAeHt4m/eDBg5GWlobU1FT5Y+rUqRg/fjxSU1P1WiVUB1UbCdEdg5a8ACA2NhazZs1CaGgowsLCkJCQgLq6OsTExAAAZs6cCXd3d8THx8PMzAxDhw5V2t/Ozg4A2mzvDmhgNiG6Y/DgFR0djbKyMqxduxbFxcUIDg7GsWPH5I34+fn54HKNqmmuBd1tJERnOIz1rhv61dXVsLW1RVVVFWxsbHR6rKp6EYLe/y8A4OeFj2NYPzudHo+QnkDda9RIizTGQamTKrV6EaJVFLx0iNq8CNEdCl461Kvq44ToGQUvQohRouClQ73sXgghekXBS4eUVw8yWDYI6ZEoeOkQFbwI0R0KXjpEC3AQojsUvAghRomCly4pFLyoCkmIdj3S2Mba2lr5lDTNdD3kxphQvCJEdzQueeXk5GDy5MmwtLSEra0t7O3tYW9vDzs7O9jb2+sij0aLSluE6I7GJa9//OMfYIxhz549cHZ2pkn2CCEGoXHwunbtGlJSUuDn56eL/PQodLeREN3RuNo4cuRImgdeTVRtJER3NC55ffnll5g/fz7u3buHoUOHwtTUVOn1YcOGaS1zxo5iFyG6o3HwKisrQ3Z2tnyaZkC2uARjDBwOBxKJRKsZNGY0tpEQ3dE4eL366qsYPnw4vv/+e2qwJ4QYjMbBKy8vDz///DMGDhyoi/z0KFTwIkR3NG6wf/LJJ3Ht2jVd5IUQQtSmccnrmWeeweLFi5GWlobAwMA2DfZTp07VWuaMHZW8CNEdjYPX/PnzAQDvv/9+m9eowV6ZYj8vCmSEaJfGwav1WEZCCDEEjdq8RCIRTExMcOPGDV3lp0eh0hYhuqNR8DI1NYWnpydVDdVEsYsQ3dH4buOqVauwcuVKVFRU6CI/PQp1UiVEdzRu8/r888+RlZUFNzc3eHl5wdLSUun1K1euaC1zhBDSHo2D17PPPquDbPRMVO4iRHc0Dl5xcXG6yEePpFhrpOlxCNGuLk8DnZKSgoyMDADAkCFDMHz4cK1lqueggEWIrmgcvEpLS/HSSy/h5MmTsLOzAwBUVlZi/PjxOHDgABwdHbWdR6NF7fWE6I7Gdxvfeust1NTU4ObNm6ioqEBFRQVu3LiB6upqvP3227rIIyGEtKFx8Dp27Bh27NgBf39/+baAgABs374dR48e7VImtm/fDm9vb5iZmWHUqFG4ePFiu2kPHTqE0NBQ2NnZwdLSEsHBwfjmm2+6dFxdo4IXIbqjcfCSSqVtBmMDsg6sXRk6lJiYiNjYWMTFxeHKlSsICgpCVFQUSktLVabv06cPVq1aheTkZFy/fh0xMTGIiYnB77//rvGxdY2qjYToTpemxFm0aBEKCwvl2+7du4fFixfjqaee0jgDW7duxdy5cxETE4OAgADs2rULFhYW2LNnj8r048aNw3PPPQd/f38MGDAAixYtwrBhw3DmzBmNj61rNDCbEN3pUifVqVOnwtvbGx4eHgCAgoICDB06FN9++61G7yUUCpGSkoIVK1bIt3G5XERGRiI5ObnT/Rlj+N///ofMzExs2rRJZZqmpiY0NTXJn1dXV2uUx9YkEglEIpFaaaUiIdytebK8ioVobGx8pGMT0hOYmpqCx+M98vtoHLw8PDxw5coVnDhxArdu3QIA+Pv7IzIyUuODl5eXQyKRwNnZWWm7s7Oz/L1Vqaqqgru7O5qamsDj8bBjxw5MmDBBZdr4+HisX79e47y1xhhDcXExKisr1d5HLJFi3XgnAACntgw5OfcfOR+E9AR2dnZwcXF5pGnku9TPi8PhYMKECe0GDF2ztrZGamoqamtrkZSUhNjYWPj4+GDcuHFt0q5YsQKxsbHy59XV1fISoyaaA5eTkxMsLCzUOumNQglYRR0AwLOPBcz5Xe5WR0iPwBhDfX29vE3b1dW1y+/VpaspKSkJSUlJKC0tbdNI315blSoODg7g8XgoKSlR2l5SUgIXF5d29+NyufI59IODg5GRkYH4+HiVwUsgEEAgEKidJ1UkEok8cPXt21ft/RhXDI6JrIopMDODGQUvQmBubg5A1mfUycmpy1VIjRvs169fj7/97W9ISkpCeXk5Hjx4oPTQBJ/PR0hICJKSkuTbpFIpkpKSEB4ervb7SKVSpXYtbWtu47KwsNDZMQjpTZqvJXXbj1XRuCiwa9cu7N27F6+88kqXD6ooNjYWs2bNQmhoKMLCwpCQkIC6ujr5upAzZ86Eu7s74uPjAcjasEJDQzFgwAA0NTXht99+wzfffIOdO3dqJT8doWXeCNEObVxLGgcvoVCI0aNHP/KBm0VHR6OsrAxr165FcXExgoODcezYMXkjfn5+PrjclgJiXV0dFixYgLt378Lc3ByDBw/Gt99+i+joaK3liRDS/XGYhjPmLVu2DFZWVlizZo2u8qRT1dXVsLW1RVVVFWxsbNTap7GxETk5Oejfvz/MzMzUPla9UIys0loAwEAnK1gYSZvX7NmzUVlZiSNHjmjtPceNG4fg4GAkJCS0m8bb2xvvvPMO3nnnHa0dlxhGZ993R9eUuteoxldTY2MjvvjiC5w4cQLDhg1r09t+69atmr4l0SF1ggYxLuvWrcORI0eQmppq6KwYlMbB6/r16wgODgaANgtxUJsQIT2HSCRSORSwu9D4buMff/zR7uN///ufLvJIumj27Nk4deoUPv30U3A4HHA4HGRnZ2POnDno378/zM3N4efnh08//VTl/uvXr4ejoyNsbGwwf/58CIVCtY5bV1eHmTNnwsrKCq6urvj444/bpCktLcUzzzwDc3Nz9O/fH999912bNBwOBzt37sSkSZNgbm4OHx8f/Pjjj/LXc3NzweFwcPDgQYwdOxbm5uYYOXIkbt++jUuXLiE0NBRWVlaYNGkSysrK1Dxrsu4+Q4YMgUAggKurKxYuXCh/LT8/H9OmTYOVlRVsbGzw4osvKnX1WbdunXyyAG9vb9ja2uKll15CTU2NPI1UKsVHH32EgQMHQiAQwNPTEx988IH89WXLlmHQoEGwsLCAj48P1qxZI78rt3fvXqxfvx7Xrl2Tf6d79+7t9DPdunULY8aMgZmZGQICAnDixAlwOBx500DzuUxMTERERATMzMzw3Xff4f79+5g+fTrc3d1hYWGBwMBAfP/990rvrc73rROsl6mqqmIAWFVVldr7NDQ0sPT0dNbQ0CDfJpVKWV2TqMNHWU0Du3CnnF24U87Kqhs6Ta/OQyqVqp3vyspKFh4ezubOncuKiopYUVERa2xsZGvXrmWXLl1id+7cYd9++y2zsLBgiYmJ8v1mzZrFrKysWHR0NLtx4wb75ZdfmKOjI1u5cqVax33jjTeYp6cnO3HiBLt+/TqbMmUKs7a2ZosWLZKnmTRpEgsKCmLJycns8uXLbPTo0czc3Jx98skn8jQAWN++fdnu3btZZmYmW716NePxeCw9PZ0xxlhOTg4DwAYPHsyOHTvG0tPT2WOPPcZCQkLYuHHj2JkzZ9iVK1fYwIED2fz589XK+44dO5iZmRlLSEhgmZmZ7OLFi/I8SSQSFhwczMaMGcMuX77Mzp8/z0JCQlhERIR8/7i4OGZlZcWef/55lpaWxv7880/m4uKidO6WLl3K7O3t2d69e1lWVhY7ffo02717t/z1DRs2sLNnz7KcnBz2888/M2dnZ7Zp0ybGGGP19fXs3XffZUOGDJF/p/X19R1+JrFYzPz8/NiECRNYamoqO336NAsLC2MA2OHDh5XOpbe3N/vPf/7D7ty5wwoLC9ndu3fZ5s2b2dWrV1l2djb77LPPGI/HYxcuXNDo+25N1TXVTN1rlIKXGlSd6LomEfNa9oveH3VNIo0+b0RERId/RIwx9uabb7IXXnhB/nzWrFmsT58+rK6uTr5t586dzMrKikkkkg7fq6amhvH5fHbw4EH5tvv37zNzc3N5PjIzMxkAdvHiRXmajIwMBqBN8GoddEaNGsXeeOMNxljLBffll1/KX//+++8ZAJaUlCTfFh8fz/z8/DrMdzM3Nze2atUqla/997//ZTwej+Xn58u33bx5U+mzxMXFMQsLC1ZdXS1P895777FRo0Yxxhirrq5mAoFAKVh1ZvPmzSwkJET+PC4ujgUFBam9/9GjR5mJiQkrKiqSbzt+/LjK4JWQkNDp+02ePJm9++67jDH1vm9VtBG8jOP2F9Gq7du3Y8+ePcjPz0dDQwOEQqG8HbNZUFCQUqfc8PBw1NbWoqCgAF5eXu2+d3Z2NoRCIUaNGiXf1qdPH/j5+cmfZ2RkwMTEBCEhIfJtgwcPls/Mq6h1Z+Xw8PA2DdXDhg2T/97cxSYwMFBpW3tTLCkqLS1FYWFhu7OjZGRkwMPDQ2l4WUBAAOzs7JCRkYGRI0cCkN01tba2lqdxdXWVHz8jIwNNTU0dzsCSmJiIzz77DNnZ2aitrYVYLFb7zrgqmZmZ8PDwUBq1EhYWpjJtaGio0nOJRIIPP/wQBw8exL179yAUCtHU1CT/21Dn+9YVCl5dZG7KQ/r7UR2mqWsSI6dcNrbRx8ESFoJHP93mpo82Gv/AgQNYsmQJPv74Y4SHh8Pa2hqbN2/GhQsXHjlvhqLYqNx806j1NnXmmmsetqLN/LQ+fmfHSE5OxowZM7B+/XpERUXB1tYWBw4c0Fs7UuulDDdv3oxPP/0UCQkJCAwMhKWlJd555x212z91SeMG+z///BNisbjNdrFYjD///FMrmTIGHA4HFnyTTh9mpjyYmfLUSqvOQ9M7unw+X2mF87Nnz2L06NFYsGABhg8fjoEDByI7O7vNfteuXUNDQ4P8+fnz52FlZdXpoPYBAwbA1NRUKRg+ePAAt2/flj8fPHgwxGIxUlJS5NsyMzNVzthx/vz5Ns8VZ/HVJmtra3h7eysNV1Pk7++PgoICFBQUyLelp6ejsrISAQEBah3D19cX5ubm7R7j3Llz8PLywqpVqxAaGgpfX1/k5eUppWn9nXbGz88PBQUFSjcWLl26pNa+Z8+exbRp0/CPf/wDQUFB8PHxUfou1fm+dUXjosD48eNRVFQEJycnpe1VVVUYP368RieV6J63tzcuXLiA3NxcWFlZwdfXF19//TV+//139O/fH9988w0uXbqE/v37K+0nFAoxZ84crF69Grm5uYiLi8PChQuVRjuoYmVlhTlz5uC9995D37594eTkhFWrVint5+fnh4kTJ2LevHnYuXMnTExM8M4776gslfzwww8IDQ3FmDFj8N133+HixYv497//rZ2To8K6deswf/58ODk5YdKkSaipqcHZs2fx1ltvITIyEoGBgZgxYwYSEhIgFouxYMECREREtKlutcfMzAzLli3D0qVLwefz8fjjj6OsrAw3b97EnDlz4Ovri/z8fBw4cAAjR47Er7/+isOHDyu9h7e3N3JycpCamop+/frB2tq6w8kHJkyYgAEDBmDWrFn46KOPUFNTg9WrVwPovHuTr68vfvzxR5w7dw729vbYunUrSkpK5MFane9bVzQ+AmNM5Qe+f/9+myInaWGoiVSXLFkCHo+HgIAAODo6IioqCs8//zyio6MxatQo3L9/HwsWLGiz31NPPQVfX1888cQTiI6OxtSpU7Fu3Tq1jrl582aMHTsWzzzzDCIjIzFmzBil9i0A+Oqrr+Dm5oaIiAg8//zzeP3119v8QwRk3TUOHDiAYcOG4euvv8b333+vdimnK2bNmoWEhATs2LEDQ4YMwZQpU/DXX38BkF3oP/30E+zt7fHEE08gMjISPj4+SExM1OgYa9aswbvvvou1a9fC398f0dHR8jaxqVOnYvHixVi4cCGCg4Nx7ty5NqNZXnjhBUycOBHjx4+Ho6Njm64LrfF4PBw5cgS1tbUYOXIkXnvtNaxatQoAOh0xsnr1aowYMQJRUVEYN24cXFxc2iw8rc73rQtqDw96/vnnAQA//fQTJk6cqBTpJRIJrl+/Dj8/Pxw7dkw3OdUSfQ4PqmsSI7tMNjxogKMVLLXQ5tWbcDgcHD58mFZp14GzZ89izJgxyMrKwoABA/R+fL0OD7K1tQUgK3lZW1srFfH5fD4ee+wxzJ07V9PPQAjRg8OHD8ubDbKysrBo0SI8/vjjBglc2qJ28Prqq68AyOrbS5YsoSpiL5Wfn99htS09PR2enp56zJFmrKys2n3t6NGjGDt2rB5zox3fffcd5s2bp/I1Ly8v3Lx5EzU1NVi2bBny8/Ph4OCAyMhI/fWE1xGNZ5UwdvqsNtY2iXGnh1UbxWIxcnNz233d29sbJibd93NmZWW1+5q7u7vWukvoU01NTZvZiJuZmpp22C/PUAwyq0RJSQmWLFkinwa6deyju409m4mJiXwKbmNkzHlvj7W1tVKn2N5C4+A1e/Zs5OfnY82aNXB1daWZJAghBqFx8Dpz5gxOnz7dZjgJUaFXVcgJ0S+N+3l5eHi0qSqS9tB5IkRXNA5eCQkJWL58eYeNtoQQomsaVxujo6NRX1+PAQMGwMLCos0g1IqKCq1ljhBC2qNx8KK50HuP1gtiUI930p1oHLxmzZqli3z0SD2txauoqAj29vaGzgYhALrQ5gXIJiBbvXo1pk+fLh9QevToUdy8eVOrmSPdi4uLS4ezF+gDY0zllEyk99E4eJ06dQqBgYG4cOECDh06hNpaWQ/ya9euIS4uTusZJI+mpqYGM2bMgKWlJVxdXfHJJ59g3LhxXVobUdWCDYcOHcL48eNhYWGBoKAgJCcnK+1z5swZ+eIYHh4eePvtt1FXVyd//ZtvvkFoaCisra3h4uKCl19+WWnW05MnT4LD4eDo0aMICQmBQCDAmTNnunQuSM+icfBavnw5/vnPf+L48ePg8/ny7U8++WSbieN6NMYAYV2nD46oHhxRvVpp1Xpo2E0lNjYWZ8+exc8//4zjx4/j9OnTuHLlitZOw6pVq7BkyRKkpqZi0KBBmD59urxklJ2djYkTJ+KFF17A9evXkZiYiDNnziitxiMSibBhwwZcu3YNR44cQW5uLmbPnt3mOMuXL8fGjRuRkZGhNO0z6b00bvNKS0vD/v3722x3cnJCeXm5VjJlFET1wIduHSaxBhDYYYouWFkI8NUbFF9TU4N9+/Zh//798jnTm+fR0pYlS5Zg8uTJAGRzbw0ZMgRZWVkYPHgw4uPjMWPGDHkpz9fXF5999hkiIiKwc+dOmJmZ4dVXX5W/l4+PDz777DOMHDkStbW1SoOo33//fUyYMEFr+SbGT+OSl52dHYqKitpsv3r1Ktzd3bWSKaIdd+7cgUgkUlpswdbWVquLIyiWglxdXQFAXu27du0a9u7dCysrK/kjKioKUqkUOTk5AICUlBQ888wz8PT0hLW1NSIiIgDIZq9QpO5MpaT30Ljk9dJLL2HZsmX44Ycf5AsLnD17FkuWLMHMmTN1kcfuydRCVgrqQHWjCHn36wHIFuDQyqwSphadp9EjVYtfNC82UVtbi3nz5uHtt99us5+npyfq6uoQFRWFqKgofPfdd3B0dER+fj6ioqLaLPBAUzCR1jS+mj788EO8+eab8PDwgEQiQUBAACQSCV5++WX5vNi9AofTefVNIgJrvrb5lgBfv1PF+Pj4wNTUFJcuXZLPsVVVVYXbt2/jiSee0PnxR4wYgfT09HZnckhLS8P9+/exceNG+cIely9f1nm+SM+g8dXE5/Oxe/durFmzBjdu3EBtbS2GDx8OX19fXeSPPAJra2vMmjUL7733Hvr06QMnJyfExcWBy+XqZTaQZcuW4bHHHsPChQvx2muvwdLSEunp6Th+/Dg+//xzeHp6gs/nY9u2bZg/fz5u3LiBDRs26DxfpGfoclHA09OzW8+YSWS2bt2K+fPnY8qUKbCxscHSpUtRUFCg0aSKXTVs2DCcOnUKq1atwtixY8EYw4ABAxAdHQ0AcHR0xN69e7Fy5Up89tlnGDFiBLZs2YKpU6fqPG/E+Kk1k2psbCw2bNgAS0tLxMbGdph269atGmdi+/bt2Lx5M4qLixEUFIRt27a1u6Lv7t278fXXX+PGjRsAgJCQEHz44Yftpm9NnzOpVjeIkHtf1qepu8ykWldXB3d3d3z88ceYM2eOobNDeim9zaR69epViEQi+e/t6UpVJDExEbGxsdi1axdGjRqFhIQEREVFITMzU+VSWCdPnsT06dMxevRomJmZYdOmTfjb3/6Gmzdv0t1OFa5evYpbt24hLCwMVVVVeP/99wEA06ZNM3DOCHlEzMDCwsLYm2++KX8ukUiYm5sbi4+PV2t/sVjMrK2t2b59+9RKX1VVxQCwqqoqtfPY0NDA0tPTWUNDg9r7MMZYVb2QXSt4wK4VPGC1jSKN9tWWK1eusBEjRjBLS0tmb2/PIiMj2fXr19mff/7JLC0t230QoksdXVPqXqMGrccIhUKkpKRgxYoV8m1cLheRkZFthpm0p76+HiKRCH369NFVNo3a8OHDkZKS0mZ7Q0MDUlNT9Z8hQrREreDVvOCsOg4dOqR22vLyckgkEjg7Oyttd3Z2xq1bt9R6j2XLlsHNzQ2RkZEqX29qakJTU5P8eXV1tdr5e1TdeVYJc3PzHrkYBek91Ophb2trK3/Y2NggKSlJqT9OSkoKkpKS5AvT6svGjRtx4MABHD58uN2G9Pj4eKX8N/cnIoQYN7VKXs0LzgKyks6LL76IXbt2gcfjAZAtd7ZgwQK17941c3BwAI/Ha7PmXElJCVxcXDrcd8uWLdi4cSNOnDjR4UDdFStWKN0hra6u7nIAa+45Tgh5NNq4ljRu89qzZw/OnDkjD1wAwOPxEBsbi9GjR2Pz5s1qvxefz0dISAiSkpLks3NKpVIkJSUpzTzQ2kcffYQPPvgAv//+e6dj3gQCwSPPQcXn88HlclFYWAhHR0fw+Xy17qwKm0RgYtkwl6amRvAM28RIiMExxiAUClFWVgYul6s0M42mNL6axGIxbt261WZw761bt7oUTWNjYzFr1iyEhoYiLCwMCQkJqKurQ0xMDABg5syZcHd3R3x8PABg06ZNWLt2Lfbv3w9vb28UFxcDgHzgry5wuVz0798fRUVFKCzseDyjogahBPfrZMGL1QggMOnS3I+E9DgWFhbw9PQEl9v1a0Lj4BUTE4M5c+YgOztb3jH0woUL2LhxozzgaCI6OhplZWVYu3YtiouLERwcjGPHjskb8fPz85U+4M6dOyEUCvH3v/9d6X3i4uKwbt06jY+vLj6fD09PT4jFYrVXBf/zdinW/5EOAPgkOhiD+9npLH+EGAsejwcTE5NHHqKmVg97RVKpFFu2bMGnn34qnxrH1dUVixYtwrvvvqtUneyOutLDvqt+SyvCgu9kE/8dnBeOsP7UnYOQzmi1h70iLpeLpUuXYunSpfJuB7oOAsaK1uYlRHceqQWZghYhxFC6FLx+/PFHHDx4EPn5+W0mjdPm/OjGjumhm2qTWIKdJ7Mxzs8JwR52Oj8eId2Fxk39n332GWJiYuDs7IyrV68iLCwMffv2xZ07dzBp0iRd5NFo6aPa+NXZXCSc+AvPbj+r+4MR0o1oHLx27NiBL774Atu2bQOfz8fSpUtx/PhxvP3226iqqtJFHo2WYuzS8L6I2v4qqdXJ+xLS3WkcvPLz8zF69GgAsvFxNTU1AIBXXnkF33//vXZzRzrFo65jpJfS+E/fxcUFFRUVAGSzqTav1ZiTk6Oz0oWx0sf54OphOmdCuiONg9eTTz6Jn3/+GYCsw+rixYsxYcIEREdH47nnntN6BknHuFwKXqR30vhu4xdffCEfBvTmm2+ib9++OHfuHKZOnYp58+ZpPYPGTB8FUR6VvEgvpVHwEovF+PDDD/Hqq6+iX79+AGTrOL700ks6yZyx00dXCR6VvEgvpVG10cTEBB999BHEYrGu8tNj6SqMUZsX6a00bvN66qmncOrUKV3kpcfRS7WR7jaSXkrjNq9JkyZh+fLlSEtLQ0hISJtl2GnNvRb6CF7UYE96K42D14IFCwCoXp+Rw+GoPV1Mb6CPjiPUYE96K42DF02FrD7q50WI7lCLiZGjaiPprdQueTU0NCApKQlTpkwBIFvYQnFJMR6Phw0bNrS7ik9vpDy2UTfHoGoj6a3UDl779u3Dr7/+Kg9en3/+OYYMGQJzc3MAsjns3dzcsHjxYt3k1BjR3UZCdEbtP/3vvvsOr7/+utK2/fv3448//sAff/yBzZs34+DBg1rPoDHTRydVqjaS3krt4JWVlYXAwED5czMzM6WFMcLCwpCenq7d3JFOUbWR9FZqVxsrKyuV2rjKysqUXpdKpUqvE311UqXgRXontUte/fr1w40bN9p9/fr16/LxjkRGH/28FLtK0JREpDdRO3g9/fTTWLt2LRobG9u81tDQgPXr12Py5MlazZyx00sPe4WCl0RKwYv0HmpXG1euXImDBw/Cz88PCxcuxKBBgwAAmZmZ+PzzzyEWi7Fy5UqdZdQYKTbY66rxXrHaKGHs0ZaDIsSIqP237uzsjHPnzuGNN97A8uXL5VUUDoeDCRMmYMeOHfJVron+KN5tpJIX6U00+kfdv39/HDt2DBUVFcjKygIADBw4EH360ErQquin2kjBi/ROXapl9OnTB2FhYdrOS4+j74HZNOyU9CbUP1uX9DEwW6HaKKboRXoRCl49iIS6SpBehIKXDrF2n+gGtXmR3oSClw7poyCk2DGVghfpTSh46ZC+e7xTkxfpTQwevLZv3w5vb2+YmZlh1KhRuHjxYrtpb968iRdeeAHe3t7gcDhISEjQX0a7QB+hS/EY1GBPehODBq/ExETExsYiLi4OV65cQVBQEKKiolBaWqoyfX19PXx8fLBx40a4uLjoObf6c6+yAfO+uYyLORUa7SelBnvSixg0eG3duhVz585FTEwMAgICsGvXLlhYWGDPnj0q048cORKbN2/GSy+9BIFAoOfcaq6rsWTxgVT8frMEL/4rWY2DtPwqoYIX6UUMFryEQiFSUlIQGRnZkhkuF5GRkUhOVuOiVVNTUxOqq6uVHvrC2vm9M9lltV06HlUbSW9isOBVXl4OiUTSZjyks7MziouLtXac+Ph42Nrayh8eHh5ae+/OdLXBXihWPwgpDvim2EV6E4M32OvaihUrUFVVJX8UFBQYOkudaupi/Y86qZLexGAzqDg4OIDH46GkpERpe0lJiVYb4wUCgVG0jynSqOSl1OZFRS/Sexis5MXn8xESEoKkpCT5NqlUiqSkJISHhxsqW1ql74IQNdiT3sSgc9fFxsZi1qxZCA0NRVhYGBISElBXV4eYmBgAwMyZM+Hu7o74+HgAskb+5kU+hEIh7t27h9TUVFhZWWHgwIEG+xzt0cfqQdTPi/RWBg1e0dHRKCsrw9q1a1FcXIzg4GAcO3ZM3oifn5+vtEJRYWEhhg8fLn++ZcsWbNmyBRERETh58qS+s98pxZKXPkphFLtIb2LwWYMXLlyIhQsXqnytdUDy9vamRSZaUWrzonNDepEef7fRkPQdSqjBnvQmFLx0SC+zSkBxVgndH0/fcsvr8NIXyfjzdlnniUmvQsFLhx61wZ7P0+zr6YlT4ixKTMX5OxWYuaf9Afukd6LgpUOPWvISmHT+9Sj38+p5wau8hlZhJ6pR8NKTrpTC+GoEL0XUYE96Ewpe3ZhaJS+F36WPUPKqaRR1eV9dUlgciRAlFLx06FG7dWha8hJ3MXj961Q2Atf9Fz9c7n7jPg0VvBhjSMooQUFFvWEyQDpFwUuHHrUWp1bwYoqzSnTtgPFHbwEA3vvxepf274lOZpZhzr7LGPvRH4bOCmkHBa9uTGDC0yg9tXl1nVgixbWCSvlNj5S8BwbOEemMwXvY92RdCSUihc5a6pS8lMc29rzgxYF+6o0DVx0FALw2pj9WTwlQq72RGBZ9QzrU0djGX64X4ulPT+PCnfsKaRgiFKopml5Aj9Jgr657lQ16HaKl7zavL8/kAAAEpnRpaJ1UotWe2/QN6VBH3SO+Sc5DelE15uy7jBv3qgAADSIJCqsa5WnUKnkpHELXJa+vk3Px+Mb/Yf3/pev0ON2BplV20kpTLZB/Abj0JfB/i4DdTwEfugP3s7V2CKo26lB7/2QYY7hVXAMAqG0SY/ZXF/HD/NFoFEmU0plwNSt26Lrk9cGvGQCAvedysW7qEJ0eq5k2C17VjSLkltch0N0WnE6KdIqlXrFEChMNRzv0GowB1YVAcZrsUfLwZ0UOVDacFF8HHLQzfRUFLwMorm5EVYMIPC4Hfs7WSC+qxiv/voC7Dxo0fi+lFbOpwb5DEz/5E4VVjfhq9kiMH+wk366qGqxYbWwQSWBNwQsQC4HyTKD4hnKgamjn5oa1K+ASCDgPlf10CQT6+GgtOxS8dKi9UHKrSFbqGuBoiX2vhuH/7TqH3PuP3p+oJw4P0qbmKvlvaUVKwUskaXveTLitgpeZqe4z2J3UVwAlN1oCVXEaUHYLkKrozMzhAY5+LQGqOVhZOug0ixS8dKmdklBGsWz5tcEuNnC0FuCbOaPw913nUFKtPI5PnYKUYpLWwauuSYx3ElMxaagLnh/RT6Osd3YsfemseqcNqmagVVzAt1HYA6fraCaVAg9yHgaqtJZgVX1XdXqB7cMgNbQlUDkOBkzN9JtvUPDSqfYu9oyHJa/BrtYAAI8+Flg1OQBvf3/1kY7XOnjtPn0Hx9NLcDy9RCvByxAeNXRV1YtgY27SYRBUVfJSDF71IvEj5qKbENYDpRkt1b3iNKDkJiBsZ51QO6+W0lTzw9aj24zZouDVDomU4d6DBlQ1iBDYz1a+nTGGrcdv43ZJDT59aTjMTNW7K6V4edwqkpW8/F1s5NvsLdpWS9Qp6SiWzpLv3Mer9SLYPnyvsp4wI8MjXCd/ZJYi5qtLmBnuhfenDW03narqtmJhrEEoafN6t1dT8jBAXW8pVd3PApiKUiRPADj5PwxQw2SlKuchgJlt27TdCAWvdpzLLscr/76IAY6WSHp3nHx74qUCbPtfFgBZ20lHJRpV1b5GkQR3yusAtJS8AMBS0Par0LQ/1cWcCkz5/DROL30SQOdtYEt+uNajx+59dCwTAPB1cp5S8Gp9VsQqZnFUvPnRIOrGwUsiBu7/9bC6pxCo6tqZvNHSUaFd6mGg6usL8IwvFBhfjvVkoKMl+qIKefc5EEmkMOVxcbOwCmt/vilP81NqYcfBS0XZKau0FhIpg52FKVxsWtoJLPld+ypaH6GgouWOZWf9vn5MaaddQ92D6YE6Ba+lP16DUCzFJ9HBStXD9vZtjks/pd7D/10rxHtRg1Wkafmwydn3cTn3ARaMG2DYLhONVbJqXnOgKk6TVQMlKkrYHC7Qd6BCI/rDn9bObdMaKQpe7XBBOVLM3kAZs4Vw73BI3QJxKNUEXhJX2Hj4I6WgBmeyylFe2wQHK9WL2qoqODX37xrsYq10oVnw21Y/HzVWqCpRdObTE39h8jBXDHSyesSj60eDUIKDl2VB+L2Jg+FuZy5/jdtOnGkUSxD9r2RcyKkAoLrUq3jqmkvalgITzBnTH1IpA1fDPngaYQyozFdoRH/4qMxTnZ5v9bAkNbQlUDn5A3wL3eWxG6Dg1Q7O/WxIwYEjpwooOAkUnMQaABAArEKAbCsPpDT2Q/b/XYVDeITsj8fcTuk9VAWf5vauwQrtXYDqC0gdqqqWjDFwOJwu9bj/5MRtfHLiNnI3TpZv++jYLeRX1OtlHcrWNLnb2LqTb3vjIn+9XqT0XFXboFTFeb1xrwpJGSV46/uriH8+ENOC3dXKV3L2faQWVGJ+hE/bzyNukpWeitOU7/g1Val+M5t+ynf7XAIBO+/2I3UPRsGrPQPGY4Xfb7h9/SJGmhfCQ5iNIdw8BPHvgSeux0BkYaBJFpB5Esj8SLaPrefDxk7Zf0G7RktwIAVTGIXVXPLyV2jvAtopeXUxVtQ0iWFjZqrU5nWtoBKB7rZdKjHsOKm9IR3aphhQhWLlkqa6H1VVfFT1T6FeKMacfZcBAIsTU9UKXmU1TXht3yXUCSUY6wYM5eUrB6ry24BUxd1MringNLilutf8d2XRR70P1QtQ8OqAh7MjEpkvrtb7AohA3DMBGBHuBTzIQVXOVew78gv8OXkYb1cCk5p7QFW+7JH5GwBgHoAZAjPcYp5wvRgKVI8Er6gB5nBpU/LqyiwGD+qE+KVVKQIAKmqFsDEzVSp5Tdt+FiufHozXnxig8XEMoblq1jquSKQMc/ZdQn8HS8Q9M0QpwDcpBK+S6kZcu9tO6aUVVSU0VTc76hXuOnI7KhFKJUDFHaA4Del//g/bWBoCBHlw2d9OT3Rze+V2KZehgIMfYMJXK/+9FQWvDgxwbGn3GdW/D2aP9pb9m+47ALZ9B+Biqge2ZpVjyfBBWBjuoNCYKhs6IS7OgBUaEcq5DWTdBrL2Yx8AqYADHB6gVPTnOA+FrKLZclFkFtfgxV3JKK1pxKJIXzw3XPnmwPTd5+UlOUX365rg7WDZ5gLccybXKILXl6fv4NMTfyFxXnibUtHl3AqczCzDycwyWfBSeE1xOqHVR24o7ZfWQSBrXUVkjEFVjft2Scu5NuFxZNVzYR1Qmv6wAf3hd1+aDohkd3EjAECxUN1ngHLblEsgYOPWbfpOGRMKXh1QbLR+JsitTXvF1GA3nMkqx3+u3MMr4d6w9R4DeI+Rv77l1zQknTkHf04elgxrglXlLUiK0mTtaBVZskf6EXn6qwIrZEg9kc68kCH1QkaNJ65W94MIJnjvh+sI6mcHn4cB9ZfrhSoDFwDcrxUCkFVzFJk9HK+XWVyD+d+mqHUOjqeXqJVOm/75cAD4qiNpbV5r3aFUsXqnWG0sqlIeJ/rM52faPV7rNq9vz+epaPNi4FQXYjw3HwGcPPgjD6UfLoGT6B44qtoCTcxxh+eN83WuSGdeSJd64Z0Zz+KJof3bzQfRDAWvDnj1tZT//pS/U5vXJw51wYb/S0dOeR0mJfyJj18MRviAvvLXpRwT/MX64S/WD88PH4nbJTX4MPcWov352PQ4V6Ht4wZQfhv2qMVoXjpGo2XKGRHjIYu5IZ15IeXASfg8PQlwCcTC/e33xv81rQj1QgnO36lQ2t48zcsb36Ug52Ffs440iiRYnJjaaTp9an3TQDFgNYnVrNa1UqQwDREArP/pGiY4VuJ5bjr8m4MVNw99OK16ojcP87NyUeiFLus/dbnaDn//4iK4HKCPJR/ltUKIeKrv/kmlDEKJFHweV7d3MXsYCl4d4Jtw8dvbYyGSSOFqa97mdRszU3zz2igsOnAVeffr8fKX5/H6WB/E/m2Qyvmgmgdku/fzBgb6AgOfanlR1IjJa79EADfv4cWSD39OHmw59fDnFMAfBcD9M8A3/wIAXBDYIV3qhQzmJS+t5TBXSMHFT6mF+Cm1sM3x71XKSiNFlY1tXlPl5d3nUdtkuKExjLVtj1Kszl2/W4lMhdLnq3svY8xAByyeMEjtu5Q2qEWAKB8BvDz4c/IQwM3DQM49CGrEQKsmJzHjIou5o4/PCFwTeWBfjjWkTkPw7dtTlIIOYwwfHjwHAIge6YHM4hqU1wpRXtuEg5cLcOxGMa7frUKTSIImsRTCh9VdJ2sB5kUMQFA/W7jZmcPJWkBT8XSAglcnAtxsOnw92MMOv709Fht+SceBSwX415938Odf5fj0peA2d6wy5HcaVbynqRlusv64KVGsVjC4oxybxnCRduUMvEQ5iLQvBb86F86cSjjzKjEe1+SpG5kpbjEPWZWTeSJD6oVbzBM1kP3Hr20S43ZJjdo9xq/kV6qVTptErfqmKcag/6TcRV+rlogy9fOzbfY/k1WOM1nlbbZzIIUHp0weoJr/QfTjtE0LANXMHBkPq3vN1b4s5g4zc0tcfiUSIY1iLP7oD9QWi/FLWhGmBrnJ9/39ZjGu5FfC3JSHdyIH4a2HpeRl/2lbDVZUWtOEDb+0lLq5HMDZxgyutmZwtTOHm60ZXG3N4WbX/NMcfS35vba0RsFLCywFJtj4wjA8OdgJyw+lIaOoGlO2nYFXn5ZqgkjCkFXa0kFVPRzcgyPsho/Btzf64V59A3yYJUoay+HHKUAANw/+nHz4c/MwmFMAC04Tgjl3EMy9o/Qu+VJHWQmNeeLjTy+hH8cLd5kjOuq/Pu+by2rlUCJl4HXx4lE1yV9Eq9V6FNv13v3hGlZP9u/0fQUQKpwfWbAazCmANUf1fGnN5yf9YdBPZ964yxyg6vxMHuwEUx4XfSz5eP0JH2w9fhsf/zcTE4e4gG/ChUgixaaHw5Lmju0PZxszOFq3dGL2d7XBpKEueGKQI2zNTSEw4UJgwoUJl4ufrt3DL9eLUFjZgJLqRogkDEVVjbJqbTv/SPg8LlxsZQHOzc68TaBztzPvdGC6saLgpUV/G+KCYE87LPvxOv7ILMNfpS1tJHfKaiGSMFgJTNDPvm0VFAC8+1q0mdeLx+VgoJOVvCvFnbI6AOa4wgbhimSQPB0XUnhxSuCvUOX05+bDnXMfntwyeKIMUWgJSNXMHLeY58MqpzcypJ7IZB5oelhX+v1mxw31Xyfn4k5ZHfZfyMdXMSPx+EDZ3E1NYlmP9whfR3j2VW7juZhTgX3ncrFmSgAu51Vg0YFUfD59OCYFugKQVbcKqzqu0n74W4bSc0dUwl+hJBXAyYMPpxA8TttG9CZmikzWDxkPS1MZUk/cYp6ohmWbtM1mhXthX3JLz/bnRrT07Zozpj++Ts5D3v16/PtMDt4YNwDfX8xHTnkd+lry8XqE7M7uqsn+eHKwE0K97ZXaUVubGe6NmeHeAGTtYOW1TSisakRRZQPuVTY8DGQNKKyU/SytaYJQIkV+RT3yOxijasHnKQc3pdKb7GdXO0kbEofpczWFbqC6uhq2traoqqqCjU3HVcKuYozh2wv5+ODXdDSKZNWg54a74/DVewj1ssePb4xWud/eszlY12p++IFOVjgRG4GJCX+2e3exI7aobSmhPbzAfTl3IeC0bcuSMA5ymKus9CH1fvjTC6WwQ2ejDK+umQB7Sz4STtxGwom/AADZHz6tVCrzXv4rAGCsrwNO/9VSXWvuzS+WSOWr+ACyNqDSh3cCeZCgP6cIAZw8BHDzEcDJhT83X3bnVoVyZiOv8jUH6DvMFRK0bYtsz+zR3lg3dQj+vF2GjUdv4d2/DcJT/spjA/dfyMfKw7Lq4OrJ/th5Mhv364TYMG0IXnkYiHRFJJGipFpWMiusbAlqzT+LqhpRUSdU671szU2VApxioLM2M4GZKQ/mfB7MTXmwMTPRaVucutdotwhe27dvx+bNm1FcXIygoCBs27YNYWFh7ab/4YcfsGbNGuTm5sLX1xebNm3C008/rdax9BG8mmWX1eKpj08BkFUVbxXX4B+PeeKfzwaqTC+VMlwtqET8bxm4/HDdwMmBrtg+YwT++Us6frxyFy42Zl0KYopMIMYATqG8yunPyUcANw8OnGqV6e8za6USWgbzQhZzg7hVwf3iqqcQ9kGS/Pncsf2xanKA/Hlz8OrvYKl0t3OwizXWTAnAzpPZOJNVDivUY/DDPDXnzY9TADNO21k8pYyDO60C7k2pF8rUCLiqfDZ9uHxetdv/nNTpIiiMMfzz1wz8++GqQ82f77+Ln4BpN2hsbxRJZCW2ygYUPgxySgGushE1XbgpY21mAnsLPuwtTGGn8NPOwhT2Cj/lv1vyYcnnqVV9VfcaNXhZMTExEbGxsdi1axdGjRqFhIQEREVFITMzE05ObbsnnDt3DtOnT0d8fDymTJmC/fv349lnn8WVK1cwdGj7czYZwgBHKwx1t8GNe9UKA7Lb/zK4XA5CvOxhoVCEH/Cwr9nqKQFY8bQ/Pv5v5iMHLzFMkMk8kck8cUTa3C+NwRGVCFCocjZXwfpyajCGdxNj0DKjhpDxkMX6KZRsvPC3D2oAtLTn7T6dg7D+fVEvFCt1+G1plJfdkPAovYwLX+3DP7h5+ICfBy9uqcp81zGBctuU1AuZzAONUD0wvjPPBrvhiMJd2XkRPogY5Ig+lnyM8LRTa/UmDoeD1ZP90SSW4Nvz+QCA+RE+3SJwAYCZKQ/9HSzR36H96mpNo0heeiuSV1Nlwa24uhH1TRI0iGSP5q4pNY1i1DSKkV/R7tu2YcrjYM/skRjr6/ioHwtANyh5jRo1CiNHjsTnn38OAJBKpfDw8MBbb72F5cuXt0kfHR2Nuro6/PLLL/Jtjz32GIKDg7Fr165Oj6fPkhcATNl2GjfutZRo/vNGOEK8Oh6fNvuriziZKZuP6bPpw5XuZH11NkfrS4+FeffBxVzVf4UCCDGIc1ephObPyYNNO43fRayPPLg03/XMZS4wgQS+nHsI4OYqvY8tR3VbzT3WV6nDbjrzQj5zUhon2hVThrni85dHgDFZ3yq/1ccAyNYT+GnhGFgJTNAkloDP42rUyC2SSPHGtykoqmpE4rxwWBlhG5I6xBIpqhpEeFAvQmW9EA/qRXhQL0RlvRCV9Yrbm5/L0jQHvUMLRmOEp32HxzCKkpdQKERKSgpWrFgh38blchEZGYnk5GSV+yQnJyM2NlZpW1RUFI4cOaIyfVNTE5qaWnpQV1errhrpioXCPF0cDjDIufM7jQGuNvLgFeiuPJulq23LHGCvjekvXyS1q/a9GoaIQY74JjkXu07dkfcFa9YEPtKYD9Ikiqu+MPTjlMtKaAqByItbCldOBVx5FXgKLZ1oGxgfJpDAlNO2i0ZzCa65JNVckqtE5+dpiJsNapvEyOtg8ZKR3va4lPsA08M8kV5UjdUPq7EcDgcCEx7G+zmiuLoJP735uLyk1ZU1G015XHw5a6TG+xkbEx4Xfa0E6NvONFCqMMbQKJLiQb0QfSy1N17ToMGrvLwcEokEzs7KjaDOzs64deuWyn2Ki4tVpi8uLlaZPj4+HuvXr9dOhrtg7ZQAfH8xHyXVTRjr66DWKjRvPemLQ1fuIWKQY5vi/vjBTpg92hvjBzth7EAH8Lgc2FvyMSvcG2amXNy4Vy0fCrNzxgiM8umLERuOAwCsBCb46O/DkFFUjQGOVnC2McNjPrJS4Cvh3nh5lBcGrPxNjU/FgdjaA8erHXEcocDDmGSFevhxCh5WOXMRwM2HH6cA5hxZo/EDZtWmNJXF3CHq4M/QSmACM1MubM1NMWu0N/4e0g91TRJ5IzIA3H1QjzGb/lC5//dzH0Ndk0Q+NXZre2bLAk5P7ErQXXA4HFljP1/1XfYuv68hq42FhYVwd3fHuXPnEB4eLt++dOlSnDp1ChcuXGizD5/Px759+zB9+nT5th07dmD9+vUoKWl7e19VycvDw0Nv1UZj1fxnUdskhpXABGIpgymPK59VFpA1BpfXNsGEy4WzjQCpBZXw6muJ4qpGNIoluHGvCndKqhBmV4OjtyogtXJHnUiCOWP6w93OHMXVjfCwt4C1mQlszU3BGB65w2V1owgmXI48j92l7YmozyiqjQ4ODuDxeG2CTklJCVxcXFTu4+LiolF6gUAAgaBrDbq9WXNJpLmkaMrjPPzZEgzMTHnoZ9/Sl2v4w7aM5qqBYtvG0+PaHsNHoRFfdsxHz7dNb1tfsRcz6L8lPp+PkJAQJCW13F6XSqVISkpSKokpCg8PV0oPAMePH283PSGkZzL4LZHY2FjMmjULoaGhCAsLQ0JCAurq6hATEwMAmDlzJtzd3REfHw8AWLRoESIiIvDxxx9j8uTJOHDgAC5fvowvvvjCkB+DEKJnBg9e0dHRKCsrw9q1a1FcXIzg4GAcO3ZM3iifn58PrsL83KNHj8b+/fuxevVqrFy5Er6+vjhy5Ei36+NFCNEtg/fz0jd99/MihGhG3WuUbsUQQowSBS9CiFGi4EUIMUoGb7DXt+YmPn0PEyKEqKf52uysOb7XBa+aGtmMDB4eHgbOCSGkIzU1NbC1tW339V53t1EqlSIzMxMBAQEoKCigO4560Dwki8637vWEc80YQ01NDdzc3JS6SbXW60peXC4X7u6yqXxtbGyM9gs2RnS+9cfYz3VHJa5m1GBPCDFKFLwIIUapVwYvgUCAuLg4mm1CT+h8609vOte9rsGeENIz9MqSFyHE+FHwIoQYJQpehBCjRMGLEGKUekXwqqiowIwZM2BjYwM7OzvMmTMHtbW1HaZ/66234OfnB3Nzc3h6euLtt99GVZXqpeWJbNVzb29vmJmZYdSoUbh48WKH6X/44QcMHjwYZmZmCAwMxG+/qbNqEQE0O9e7d+/G2LFjYW9vD3t7e0RGRnb63RgN1gtMnDiRBQUFsfPnz7PTp0+zgQMHsunTp7ebPi0tjT3//PPs559/ZllZWSwpKYn5+vqyF154QY+5Nh4HDhxgfD6f7dmzh928eZPNnTuX2dnZsZKSEpXpz549y3g8Hvvoo49Yeno6W716NTM1NWVpaWl6zrnx0fRcv/zyy2z79u3s6tWrLCMjg82ePZvZ2tqyu3fv6jnn2tfjg1d6ejoDwC5duiTfdvToUcbhcNi9e/fUfp+DBw8yPp/PRCKRLrJp1MLCwtibb74pfy6RSJibmxuLj49Xmf7FF19kkydPVto2atQoNm/ePJ3msyfQ9Fy3JhaLmbW1Ndu3b5+usqg3Pb7amJycDDs7O4SGhsq3RUZGgsvlqlwXsj3NU9KamPS64aAdal71PDIyUr5NnVXPFdMDslXP20tPZLpyrlurr6+HSCRCnz59dJVNvenxwau4uBhOTk5K20xMTNCnT592V9lurby8HBs2bMDrr7+uiywatY5WPW/v/Gq66jmR6cq5bm3ZsmVwc3Nr88/DGBlt8Fq+fDk4HE6Hj1u3bj3ycaqrqzF58mQEBARg3bp1j55xQgxk48aNOHDgAA4fPgwzMzNDZ+eRGW0d6N1338Xs2bM7TOPj4wMXFxeUlpYqbReLxaioqGh3le1mNTU1mDhxIqytrXH48GGYmtJqzK3pY9VzItOVc91sy5Yt2LhxI06cOIFhw4bpMpv6Y+hGN11rbrC/fPmyfNvvv//eaYN9VVUVe+yxx1hERASrq6vTR1aNVlhYGFu4cKH8uUQiYe7u7h022E+ZMkVpW3h4ODXYq0HTc80YY5s2bWI2NjYsOTlZH1nUmx4fvBiTdZUYPnw4u3DhAjtz5gzz9fVV6ipx9+5d5ufnxy5cuMAYkwWuUaNGscDAQJaVlcWKiorkD7FYbKiP0W0dOHCACQQCtnfvXpaens5ef/11Zmdnx4qLixljjL3yyits+fLl8vRnz55lJiYmbMuWLSwjI4PFxcVRVwk1aXquN27cyPh8Pvvxxx+V/o5ramoM9RG0plcEr/v377Pp06czKysrZmNjw2JiYpS+vJycHAaA/fHHH4wxxv744w8GQOUjJyfHMB+im9u2bRvz9PRkfD6fhYWFsfPnz8tfi4iIYLNmzVJKf/DgQTZo0CDG5/PZkCFD2K+//qrnHBsvTc61l5eXyr/juLg4/Wdcy2hKHEKIUTLau42EkN6NghchxChR8CKEGCUKXoQQo0TBixBilCh4EUKMEgUvQohRouBFCDFKFLxItzB79myVM4NMnDjR0Fkj3ZTRzipBep6JEyfiq6++UtrW3srPIpGozSwfQqEQfD5f4+N2dT9iWFTyIt2GQCCAi4uL0sPe3h4AwOFwsHPnTkydOhWWlpb44IMPsG7dOgQHB+PLL79E//795XNU5efnY9q0abCysoKNjQ1efPFFpWlk2tuPGBcKXsRorFu3Ds899xzS0tLw6quvAgCysrLwn//8B4cOHUJqaiqkUimmTZuGiooKnDp1CsePH8edO3cQHR2t9F6t9yPGh6qNpNv45ZdfYGVlpbRt5cqVWLlyJQDg5ZdfRkxMjNLrQqEQX3/9NRwdHQEAx48fR1paGnJycuDh4QEA+PrrrzFkyBBcunQJI0eOVLkfMT4UvEi3MX78eOzcuVNpm+JCEYqLqDTz8vJSCkAZGRnw8PCQBy4ACAgIgJ2dHTIyMuTBq/V+xPhQ8CLdhqWlJQYOHNjh6+psU/dYxLhRmxfpUfz9/VFQUICCggL5tvT0dFRWViIgIMCAOSPaRiUv0m00NTW1WcLLxMQEDg4Oar9HZGQkAgMDMWPGDCQkJEAsFmPBggWIiIhQWe0kxotKXqTbOHbsGFxdXZUeY8aM0eg9OBwOfvrpJ9jb2+OJJ55AZGQkfHx8kJiYqKNcE0OhaaAJIUaJSl6EEKNEwYsQYpQoeBFCjBIFL0KIUaLgRQgxShS8CCFGiYIXIcQoUfAihBglCl6EEKNEwYsQYpQoeBFCjBIFL0KIUfr/hEPT76migF0AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019542, + "end_time": "2024-03-23T06:20:40.042905", + "exception": false, + "start_time": "2024-03-23T06:20:40.023363", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4582.628319, + "end_time": "2024-03-23T06:20:42.786532", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tab_ddpm_concat/3/mlu-eval.ipynb", + "output_path": "eval/contraceptive/tab_ddpm_concat/3/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/tab_ddpm_concat/3", + "path_prefix": "../../../../", + "random_seed": 3, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-03-23T05:04:20.158213", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/tab_ddpm_concat/model.pt b/contraceptive/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..f373c2a4ef4c731defe1610ebabd4a425450336f --- /dev/null +++ b/contraceptive/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd9a78df91e76837706914b0b3878144b373cc365fa95fcfaac8d1f26709426c +size 47482955 diff --git a/contraceptive/tab_ddpm_concat/params.json b/contraceptive/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..195ba29f2aec3d623bba742fdc47ae196ca6958b --- /dev/null +++ b/contraceptive/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.73, "gradient_penalty_mode": "NONE", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "head_activation": "softsign", "loss_balancer_beta": 0.67, "loss_balancer_r": 0.943, "tf_activation": "tanh", "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.09, "n_warmup_steps": 100, "Optim": "amsgradw", "fixed_role_model": "tab_ddpm_concat", "mse_mag": false, "mse_mag_target": 0.65, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation_final": "leakyhardtanh", "tf_num_inds": 128, "ada_d_hid": 1024, "ada_n_layers": 9, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation_final": "leakyhardsigmoid", "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/tvae/eval.csv b/contraceptive/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..9a34b06d6f0b2d5437320f48caec513087e39fc0 --- /dev/null +++ b/contraceptive/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.030277189869137557,0.04563582819014448,0.0027671899940053535,12.488234519958496,0.021082285791635513,0.4825451970100403,0.04161680117249489,2.434007910778746e-05,4.115196704864502,0.03820033371448517,0.1258891224861145,0.05260408669710159,0.10057253390550613,0.008725106716156006,16.603431224822998 diff --git a/contraceptive/tvae/history.csv b/contraceptive/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..bc98cd2b91e6d3c444d30709c0f5502546416fac --- /dev/null +++ b/contraceptive/tvae/history.csv @@ -0,0 +1,12 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.020646917774962883,0.7029616862738138,0.0014198488189189598,0.0,0.0,0.0,0.0,0.0,0.020646917774962883,900,225,211.21505284309387,0.9387335681915283,0.23468339204788208,0.12668553197549448,0.00800067097414285,0.8323721770502058,0.00025833536566152146,0.0,0.0,0.0,0.0,0.0,0.00800067097414285,450,113,91.81465721130371,0.8125190903655196,0.20403257158067492,0.07484065717399384 +1,0.007335189075060447,0.5605360431658958,0.00016509135985034204,0.0,0.0,0.0,0.0,0.0,0.007335189075060447,900,225,211.18131518363953,0.9385836230383979,0.23464590575959948,0.09665914196934965,0.005770373049502572,1.328719814272311,0.0001455103024895009,0.0,0.0,0.0,0.0,0.0,0.005770373049502572,450,113,92.31423711776733,0.8169401514846667,0.20514274915059408,0.06159803802889269 +2,0.004101434572932905,0.40740934907574267,4.4437185304309564e-05,0.0,0.0,0.0,0.0,0.0,0.004101434572932905,900,225,211.43363165855408,0.9397050295935737,0.23492625739839343,0.10036693361898263,0.005793615489771279,2.93090469723272,7.996645985781352e-05,0.0,0.0,0.0,0.0,0.0,0.005793615489771279,450,113,92.21822166442871,0.8160904572073338,0.20492938147650824,0.0686845871407654 +3,0.0032525908350200753,0.45932424604433697,1.5064653623929553e-05,0.0,0.0,0.0,0.0,0.0,0.0032525908350200753,900,225,210.6352882385254,0.9361568366156684,0.2340392091539171,0.09941185830367937,0.0025840073977209006,2.9610207560058215,8.98458365437745e-06,0.0,0.0,0.0,0.0,0.0,0.0025840073977209006,450,113,91.6226761341095,0.81082014277973,0.20360594696468778,0.04679510168797147 +4,0.0028994390259807308,0.27792873476118807,1.2238399814356409e-05,0.0,0.0,0.0,0.0,0.0,0.0028994390259807308,900,225,210.90299940109253,0.9373466640048557,0.23433666600121392,0.10629830273903078,0.004226378290137897,3.139442252464839,2.8896792241464515e-05,0.0,0.0,0.0,0.0,0.0,0.004226378290137897,450,113,92.30420923233032,0.8168514091356666,0.20512046496073405,0.04041025609363167 +5,0.0029357373788823477,0.35013624048834924,1.5252040759503315e-05,0.0,0.0,0.0,0.0,0.0,0.0029357373788823477,900,225,210.8098328113556,0.9369325902726915,0.23423314756817287,0.10065761231092943,0.003366937771077371,2.7330855187934775,2.1526505657937356e-05,0.0,0.0,0.0,0.0,0.0,0.003366937771077371,450,113,92.45483732223511,0.8181844010817266,0.20545519404941134,0.048643345435137604 +6,0.0025804581021010463,0.33193543293685124,1.1130432750345e-05,0.0,0.0,0.0,0.0,0.0,0.0025804581021010463,900,225,210.82577991485596,0.9370034662882487,0.23425086657206218,0.10346181529677577,0.0029045950072920986,4.791847205054414,1.7926797626652248e-05,0.0,0.0,0.0,0.0,0.0,0.0029045950072920986,450,113,92.71966814994812,0.8205280367252046,0.20604370699988472,0.04583418705378066 +7,0.0024444010488999385,0.27924930442016427,1.0806890312169106e-05,0.0,0.0,0.0,0.0,0.0,0.0024444010488999385,900,225,211.59207558631897,0.9404092248280843,0.23510230620702108,0.10225464255238573,0.002720622533104486,3.2791891266000865,1.838483538128186e-05,0.0,0.0,0.0,0.0,0.0,0.002720622533104486,450,113,96.57148313522339,0.8546148950019768,0.21460329585605198,0.04418732333965435 +8,0.00242780985414154,0.25769682647126535,1.037415603605397e-05,0.0,0.0,0.0,0.0,0.0,0.00242780985414154,900,225,213.52410340309143,0.9489960151248508,0.2372490037812127,0.10494643683855732,0.002887863010659607,1.8791022815315657,2.555802672021092e-05,0.0,0.0,0.0,0.0,0.0,0.002887863010659607,450,113,92.99426054954529,0.8229580579605777,0.20665391233232286,0.059518284711418096 +9,0.002208985082106665,0.42546326303130916,7.684016633753033e-06,0.0,0.0,0.0,0.0,0.0,0.002208985082106665,900,225,213.6311333179474,0.9494717036353217,0.23736792590883043,0.10293637524772849,0.0024638745025731624,3.782287786237916,1.655159604749657e-05,0.0,0.0,0.0,0.0,0.0,0.0024638745025731624,450,113,97.28867506980896,0.8609617262814953,0.21619705571068656,0.04808065608046965 +10,0.0020764596655175813,0.3896139937922279,7.334516867581215e-06,0.0,0.0,0.0,0.0,0.0,0.0020764596655175813,900,225,221.03631401062012,0.9823836178249783,0.24559590445624457,0.10016421435814765,0.0025203956082178692,2.2354346593865277,2.0540576969837897e-05,0.0,0.0,0.0,0.0,0.0,0.0025203956082178692,450,113,97.72019529342651,0.8647804893223585,0.2171559895409478,0.057214381701758014 diff --git a/contraceptive/tvae/mlu-eval.ipynb b/contraceptive/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..5216be57e4a3d955c463ed2f66c4d0a5444cbf41 --- /dev/null +++ b/contraceptive/tvae/mlu-eval.ipynb @@ -0,0 +1,2330 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:38.420949Z", + "iopub.status.busy": "2024-03-22T16:54:38.420627Z", + "iopub.status.idle": "2024-03-22T16:54:38.453783Z", + "shell.execute_reply": "2024-03-22T16:54:38.452888Z" + }, + "papermill": { + "duration": 0.04789, + "end_time": "2024-03-22T16:54:38.455806", + "exception": false, + "start_time": "2024-03-22T16:54:38.407916", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:38.481275Z", + "iopub.status.busy": "2024-03-22T16:54:38.480905Z", + "iopub.status.idle": "2024-03-22T16:54:38.487727Z", + "shell.execute_reply": "2024-03-22T16:54:38.486854Z" + }, + "papermill": { + "duration": 0.021698, + "end_time": "2024-03-22T16:54:38.489673", + "exception": false, + "start_time": "2024-03-22T16:54:38.467975", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:38.512890Z", + "iopub.status.busy": "2024-03-22T16:54:38.512651Z", + "iopub.status.idle": "2024-03-22T16:54:38.516571Z", + "shell.execute_reply": "2024-03-22T16:54:38.515766Z" + }, + "papermill": { + "duration": 0.017861, + "end_time": "2024-03-22T16:54:38.518464", + "exception": false, + "start_time": "2024-03-22T16:54:38.500603", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:38.541803Z", + "iopub.status.busy": "2024-03-22T16:54:38.541370Z", + "iopub.status.idle": "2024-03-22T16:54:38.545217Z", + "shell.execute_reply": "2024-03-22T16:54:38.544371Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017736, + "end_time": "2024-03-22T16:54:38.547270", + "exception": false, + "start_time": "2024-03-22T16:54:38.529534", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:38.571177Z", + "iopub.status.busy": "2024-03-22T16:54:38.570870Z", + "iopub.status.idle": "2024-03-22T16:54:38.576291Z", + "shell.execute_reply": "2024-03-22T16:54:38.575410Z" + }, + "papermill": { + "duration": 0.019685, + "end_time": "2024-03-22T16:54:38.578251", + "exception": false, + "start_time": "2024-03-22T16:54:38.558566", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e2d3d897", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:38.603452Z", + "iopub.status.busy": "2024-03-22T16:54:38.603196Z", + "iopub.status.idle": "2024-03-22T16:54:38.608013Z", + "shell.execute_reply": "2024-03-22T16:54:38.607193Z" + }, + "papermill": { + "duration": 0.019844, + "end_time": "2024-03-22T16:54:38.609908", + "exception": false, + "start_time": "2024-03-22T16:54:38.590064", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tvae\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tvae/42\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011211, + "end_time": "2024-03-22T16:54:38.632439", + "exception": false, + "start_time": "2024-03-22T16:54:38.621228", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:38.655861Z", + "iopub.status.busy": "2024-03-22T16:54:38.655594Z", + "iopub.status.idle": "2024-03-22T16:54:38.664480Z", + "shell.execute_reply": "2024-03-22T16:54:38.663684Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022742, + "end_time": "2024-03-22T16:54:38.666331", + "exception": false, + "start_time": "2024-03-22T16:54:38.643589", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tvae/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:38.690470Z", + "iopub.status.busy": "2024-03-22T16:54:38.690185Z", + "iopub.status.idle": "2024-03-22T16:54:40.726590Z", + "shell.execute_reply": "2024-03-22T16:54:40.725650Z" + }, + "papermill": { + "duration": 2.050871, + "end_time": "2024-03-22T16:54:40.728604", + "exception": false, + "start_time": "2024-03-22T16:54:38.677733", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:40.760710Z", + "iopub.status.busy": "2024-03-22T16:54:40.760306Z", + "iopub.status.idle": "2024-03-22T16:54:40.772850Z", + "shell.execute_reply": "2024-03-22T16:54:40.771812Z" + }, + "papermill": { + "duration": 0.03277, + "end_time": "2024-03-22T16:54:40.774761", + "exception": false, + "start_time": "2024-03-22T16:54:40.741991", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:40.799242Z", + "iopub.status.busy": "2024-03-22T16:54:40.798929Z", + "iopub.status.idle": "2024-03-22T16:54:40.806026Z", + "shell.execute_reply": "2024-03-22T16:54:40.805321Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021718, + "end_time": "2024-03-22T16:54:40.808141", + "exception": false, + "start_time": "2024-03-22T16:54:40.786423", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:40.833629Z", + "iopub.status.busy": "2024-03-22T16:54:40.833384Z", + "iopub.status.idle": "2024-03-22T16:54:40.928573Z", + "shell.execute_reply": "2024-03-22T16:54:40.927760Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.110635, + "end_time": "2024-03-22T16:54:40.930702", + "exception": false, + "start_time": "2024-03-22T16:54:40.820067", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:40.958708Z", + "iopub.status.busy": "2024-03-22T16:54:40.958059Z", + "iopub.status.idle": "2024-03-22T16:54:45.689430Z", + "shell.execute_reply": "2024-03-22T16:54:45.688570Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.747577, + "end_time": "2024-03-22T16:54:45.691899", + "exception": false, + "start_time": "2024-03-22T16:54:40.944322", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 16:54:43.265606: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 16:54:43.265666: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 16:54:43.267291: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:45.718090Z", + "iopub.status.busy": "2024-03-22T16:54:45.716978Z", + "iopub.status.idle": "2024-03-22T16:54:45.722995Z", + "shell.execute_reply": "2024-03-22T16:54:45.722101Z" + }, + "papermill": { + "duration": 0.020744, + "end_time": "2024-03-22T16:54:45.725071", + "exception": false, + "start_time": "2024-03-22T16:54:45.704327", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:45.751786Z", + "iopub.status.busy": "2024-03-22T16:54:45.751472Z", + "iopub.status.idle": "2024-03-22T16:54:54.134274Z", + "shell.execute_reply": "2024-03-22T16:54:54.133246Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.399093, + "end_time": "2024-03-22T16:54:54.136682", + "exception": false, + "start_time": "2024-03-22T16:54:45.737589", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T16:54:54.164061Z", + "iopub.status.busy": "2024-03-22T16:54:54.163711Z", + "iopub.status.idle": "2024-03-22T16:54:54.171531Z", + "shell.execute_reply": "2024-03-22T16:54:54.170687Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023874, + "end_time": "2024-03-22T16:54:54.173518", + "exception": false, + "start_time": "2024-03-22T16:54:54.149644", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:54.198629Z", + "iopub.status.busy": "2024-03-22T16:54:54.198356Z", + "iopub.status.idle": "2024-03-22T16:54:54.203955Z", + "shell.execute_reply": "2024-03-22T16:54:54.203117Z" + }, + "papermill": { + "duration": 0.020638, + "end_time": "2024-03-22T16:54:54.205880", + "exception": false, + "start_time": "2024-03-22T16:54:54.185242", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T16:54:54.230490Z", + "iopub.status.busy": "2024-03-22T16:54:54.230224Z", + "iopub.status.idle": "2024-03-22T17:02:21.663470Z", + "shell.execute_reply": "2024-03-22T17:02:21.662503Z" + }, + "papermill": { + "duration": 447.460222, + "end_time": "2024-03-22T17:02:21.677822", + "exception": false, + "start_time": "2024-03-22T16:54:54.217600", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/bs_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:02:21.706111Z", + "iopub.status.busy": "2024-03-22T17:02:21.705309Z", + "iopub.status.idle": "2024-03-22T17:02:22.031100Z", + "shell.execute_reply": "2024-03-22T17:02:22.030076Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.342422, + "end_time": "2024-03-22T17:02:22.033338", + "exception": false, + "start_time": "2024-03-22T17:02:21.690916", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.73,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.Softsign,\n", + " 'loss_balancer_beta': 0.67,\n", + " 'loss_balancer_r': 0.943,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.65, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:02:22.061219Z", + "iopub.status.busy": "2024-03-22T17:02:22.060881Z", + "iopub.status.idle": "2024-03-22T17:12:00.661298Z", + "shell.execute_reply": "2024-03-22T17:12:00.660201Z" + }, + "papermill": { + "duration": 578.632009, + "end_time": "2024-03-22T17:12:00.678673", + "exception": false, + "start_time": "2024-03-22T17:02:22.046664", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_train/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/contraceptive [400, 0]\n", + "Caching in ../../../../contraceptive/_cache_aug_val/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/contraceptive [0, 200]\n", + "Caching in ../../../../contraceptive/_cache_bs_train/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/contraceptive [100, 0]\n", + "Caching in ../../../../contraceptive/_cache_bs_val/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/contraceptive [0, 50]\n", + "Caching in ../../../../contraceptive/_cache_synth/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/contraceptive [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T17:12:00.708438Z", + "iopub.status.busy": "2024-03-22T17:12:00.708106Z", + "iopub.status.idle": "2024-03-22T17:12:01.197733Z", + "shell.execute_reply": "2024-03-22T17:12:01.196772Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.506346, + "end_time": "2024-03-22T17:12:01.199915", + "exception": false, + "start_time": "2024-03-22T17:12:00.693569", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:12:01.229980Z", + "iopub.status.busy": "2024-03-22T17:12:01.229573Z", + "iopub.status.idle": "2024-03-22T17:12:01.234682Z", + "shell.execute_reply": "2024-03-22T17:12:01.233844Z" + }, + "papermill": { + "duration": 0.022181, + "end_time": "2024-03-22T17:12:01.236594", + "exception": false, + "start_time": "2024-03-22T17:12:01.214413", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:12:01.264684Z", + "iopub.status.busy": "2024-03-22T17:12:01.264409Z", + "iopub.status.idle": "2024-03-22T17:12:01.271364Z", + "shell.execute_reply": "2024-03-22T17:12:01.270425Z" + }, + "papermill": { + "duration": 0.023683, + "end_time": "2024-03-22T17:12:01.273479", + "exception": false, + "start_time": "2024-03-22T17:12:01.249796", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "11895304" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:12:01.303569Z", + "iopub.status.busy": "2024-03-22T17:12:01.302995Z", + "iopub.status.idle": "2024-03-22T17:12:01.383209Z", + "shell.execute_reply": "2024-03-22T17:12:01.382160Z" + }, + "papermill": { + "duration": 0.096944, + "end_time": "2024-03-22T17:12:01.385367", + "exception": false, + "start_time": "2024-03-22T17:12:01.288423", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 46] --\n", + "├─Adapter: 1-1 [2, 1179, 46] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 48,128\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 46] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 128, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 16, 256] 1\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 128, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 1,048,832\n", + "│ │ │ └─Softsign: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 256] --\n", + "│ │ │ └─Linear: 4-55 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-56 [2, 256] --\n", + "│ │ └─FeedForward: 3-28 [2, 256] --\n", + "│ │ │ └─Linear: 4-57 [2, 256] 65,792\n", + "│ │ │ └─Softsign: 4-58 [2, 256] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 11,895,304\n", + "Trainable params: 11,895,304\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 44.15\n", + "========================================================================================================================\n", + "Input size (MB): 0.54\n", + "Forward/backward pass size (MB): 375.40\n", + "Params size (MB): 47.58\n", + "Estimated Total Size (MB): 423.53\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T17:12:01.417191Z", + "iopub.status.busy": "2024-03-22T17:12:01.416837Z", + "iopub.status.idle": "2024-03-22T18:16:21.301787Z", + "shell.execute_reply": "2024-03-22T18:16:21.300655Z" + }, + "papermill": { + "duration": 3859.922538, + "end_time": "2024-03-22T18:16:21.323259", + "exception": false, + "start_time": "2024-03-22T17:12:01.400721", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.020646917774962883, 'avg_role_model_std_loss': 0.7029616862738138, 'avg_role_model_mean_pred_loss': 0.0014198488189189598, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.020646917774962883, 'n_size': 900, 'n_batch': 225, 'duration': 211.21505284309387, 'duration_batch': 0.9387335681915283, 'duration_size': 0.23468339204788208, 'avg_pred_std': 0.12668553197549448}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00800067097414285, 'avg_role_model_std_loss': 0.8323721770502058, 'avg_role_model_mean_pred_loss': 0.00025833536566152146, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00800067097414285, 'n_size': 450, 'n_batch': 113, 'duration': 91.81465721130371, 'duration_batch': 0.8125190903655196, 'duration_size': 0.20403257158067492, 'avg_pred_std': 0.07484065717399384}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007335189075060447, 'avg_role_model_std_loss': 0.5605360431658958, 'avg_role_model_mean_pred_loss': 0.00016509135985034204, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007335189075060447, 'n_size': 900, 'n_batch': 225, 'duration': 211.18131518363953, 'duration_batch': 0.9385836230383979, 'duration_size': 0.23464590575959948, 'avg_pred_std': 0.09665914196934965}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005770373049502572, 'avg_role_model_std_loss': 1.328719814272311, 'avg_role_model_mean_pred_loss': 0.0001455103024895009, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005770373049502572, 'n_size': 450, 'n_batch': 113, 'duration': 92.31423711776733, 'duration_batch': 0.8169401514846667, 'duration_size': 0.20514274915059408, 'avg_pred_std': 0.06159803802889269}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004101434572932905, 'avg_role_model_std_loss': 0.40740934907574267, 'avg_role_model_mean_pred_loss': 4.4437185304309564e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004101434572932905, 'n_size': 900, 'n_batch': 225, 'duration': 211.43363165855408, 'duration_batch': 0.9397050295935737, 'duration_size': 0.23492625739839343, 'avg_pred_std': 0.10036693361898263}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005793615489771279, 'avg_role_model_std_loss': 2.93090469723272, 'avg_role_model_mean_pred_loss': 7.996645985781352e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005793615489771279, 'n_size': 450, 'n_batch': 113, 'duration': 92.21822166442871, 'duration_batch': 0.8160904572073338, 'duration_size': 0.20492938147650824, 'avg_pred_std': 0.0686845871407654}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0032525908350200753, 'avg_role_model_std_loss': 0.45932424604433697, 'avg_role_model_mean_pred_loss': 1.5064653623929553e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032525908350200753, 'n_size': 900, 'n_batch': 225, 'duration': 210.6352882385254, 'duration_batch': 0.9361568366156684, 'duration_size': 0.2340392091539171, 'avg_pred_std': 0.09941185830367937}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025840073977209006, 'avg_role_model_std_loss': 2.9610207560058215, 'avg_role_model_mean_pred_loss': 8.98458365437745e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025840073977209006, 'n_size': 450, 'n_batch': 113, 'duration': 91.6226761341095, 'duration_batch': 0.81082014277973, 'duration_size': 0.20360594696468778, 'avg_pred_std': 0.04679510168797147}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0028994390259807308, 'avg_role_model_std_loss': 0.27792873476118807, 'avg_role_model_mean_pred_loss': 1.2238399814356409e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0028994390259807308, 'n_size': 900, 'n_batch': 225, 'duration': 210.90299940109253, 'duration_batch': 0.9373466640048557, 'duration_size': 0.23433666600121392, 'avg_pred_std': 0.10629830273903078}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004226378290137897, 'avg_role_model_std_loss': 3.139442252464839, 'avg_role_model_mean_pred_loss': 2.8896792241464515e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004226378290137897, 'n_size': 450, 'n_batch': 113, 'duration': 92.30420923233032, 'duration_batch': 0.8168514091356666, 'duration_size': 0.20512046496073405, 'avg_pred_std': 0.04041025609363167}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0029357373788823477, 'avg_role_model_std_loss': 0.35013624048834924, 'avg_role_model_mean_pred_loss': 1.5252040759503315e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029357373788823477, 'n_size': 900, 'n_batch': 225, 'duration': 210.8098328113556, 'duration_batch': 0.9369325902726915, 'duration_size': 0.23423314756817287, 'avg_pred_std': 0.10065761231092943}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003366937771077371, 'avg_role_model_std_loss': 2.7330855187934775, 'avg_role_model_mean_pred_loss': 2.1526505657937356e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003366937771077371, 'n_size': 450, 'n_batch': 113, 'duration': 92.45483732223511, 'duration_batch': 0.8181844010817266, 'duration_size': 0.20545519404941134, 'avg_pred_std': 0.048643345435137604}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0025804581021010463, 'avg_role_model_std_loss': 0.33193543293685124, 'avg_role_model_mean_pred_loss': 1.1130432750345e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025804581021010463, 'n_size': 900, 'n_batch': 225, 'duration': 210.82577991485596, 'duration_batch': 0.9370034662882487, 'duration_size': 0.23425086657206218, 'avg_pred_std': 0.10346181529677577}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0029045950072920986, 'avg_role_model_std_loss': 4.791847205054414, 'avg_role_model_mean_pred_loss': 1.7926797626652248e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029045950072920986, 'n_size': 450, 'n_batch': 113, 'duration': 92.71966814994812, 'duration_batch': 0.8205280367252046, 'duration_size': 0.20604370699988472, 'avg_pred_std': 0.04583418705378066}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0024444010488999385, 'avg_role_model_std_loss': 0.27924930442016427, 'avg_role_model_mean_pred_loss': 1.0806890312169106e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024444010488999385, 'n_size': 900, 'n_batch': 225, 'duration': 211.59207558631897, 'duration_batch': 0.9404092248280843, 'duration_size': 0.23510230620702108, 'avg_pred_std': 0.10225464255238573}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002720622533104486, 'avg_role_model_std_loss': 3.2791891266000865, 'avg_role_model_mean_pred_loss': 1.838483538128186e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002720622533104486, 'n_size': 450, 'n_batch': 113, 'duration': 96.57148313522339, 'duration_batch': 0.8546148950019768, 'duration_size': 0.21460329585605198, 'avg_pred_std': 0.04418732333965435}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00242780985414154, 'avg_role_model_std_loss': 0.25769682647126535, 'avg_role_model_mean_pred_loss': 1.037415603605397e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00242780985414154, 'n_size': 900, 'n_batch': 225, 'duration': 213.52410340309143, 'duration_batch': 0.9489960151248508, 'duration_size': 0.2372490037812127, 'avg_pred_std': 0.10494643683855732}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002887863010659607, 'avg_role_model_std_loss': 1.8791022815315657, 'avg_role_model_mean_pred_loss': 2.555802672021092e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002887863010659607, 'n_size': 450, 'n_batch': 113, 'duration': 92.99426054954529, 'duration_batch': 0.8229580579605777, 'duration_size': 0.20665391233232286, 'avg_pred_std': 0.059518284711418096}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002208985082106665, 'avg_role_model_std_loss': 0.42546326303130916, 'avg_role_model_mean_pred_loss': 7.684016633753033e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002208985082106665, 'n_size': 900, 'n_batch': 225, 'duration': 213.6311333179474, 'duration_batch': 0.9494717036353217, 'duration_size': 0.23736792590883043, 'avg_pred_std': 0.10293637524772849}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0024638745025731624, 'avg_role_model_std_loss': 3.782287786237916, 'avg_role_model_mean_pred_loss': 1.655159604749657e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024638745025731624, 'n_size': 450, 'n_batch': 113, 'duration': 97.28867506980896, 'duration_batch': 0.8609617262814953, 'duration_size': 0.21619705571068656, 'avg_pred_std': 0.04808065608046965}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0020764596655175813, 'avg_role_model_std_loss': 0.3896139937922279, 'avg_role_model_mean_pred_loss': 7.334516867581215e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0020764596655175813, 'n_size': 900, 'n_batch': 225, 'duration': 221.03631401062012, 'duration_batch': 0.9823836178249783, 'duration_size': 0.24559590445624457, 'avg_pred_std': 0.10016421435814765}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025203956082178692, 'avg_role_model_std_loss': 2.2354346593865277, 'avg_role_model_mean_pred_loss': 2.0540576969837897e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025203956082178692, 'n_size': 450, 'n_batch': 113, 'duration': 97.72019529342651, 'duration_batch': 0.8647804893223585, 'duration_size': 0.2171559895409478, 'avg_pred_std': 0.057214381701758014}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0019838308382329867, 'avg_role_model_std_loss': 0.26038020301704434, 'avg_role_model_mean_pred_loss': 7.2093371565769966e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019838308382329867, 'n_size': 900, 'n_batch': 225, 'duration': 221.59594750404358, 'duration_batch': 0.9848708777957492, 'duration_size': 0.2462177194489373, 'avg_pred_std': 0.1044646823985709}\n", + "Time out: 3600.622545480728/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.0027671898508283663, 'avg_g_mag_loss': 0.06750814222799723, 'avg_g_cos_loss': 0.031209569611071075, 'pred_duration': 4.135732650756836, 'grad_duration': 12.466698169708252, 'total_duration': 16.602430820465088, 'pred_std': 0.10057252645492554, 'std_loss': 0.008725129999220371, 'mean_pred_loss': 2.4340070012840442e-05, 'pred_rmse': 0.05260408669710159, 'pred_mae': 0.03820032626390457, 'pred_mape': 0.1258891075849533, 'grad_rmse': 0.04161679372191429, 'grad_mae': 0.021082276478409767, 'grad_mape': 0.4825449585914612}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0027671898508283663, 'avg_g_mag_loss': 0.06750814222799723, 'avg_g_cos_loss': 0.031209569611071075, 'avg_pred_duration': 4.135732650756836, 'avg_grad_duration': 12.466698169708252, 'avg_total_duration': 16.602430820465088, 'avg_pred_std': 0.10057252645492554, 'avg_std_loss': 0.008725129999220371, 'avg_mean_pred_loss': 2.4340070012840442e-05}, 'min_metrics': {'avg_loss': 0.0027671898508283663, 'avg_g_mag_loss': 0.06750814222799723, 'avg_g_cos_loss': 0.031209569611071075, 'pred_duration': 4.135732650756836, 'grad_duration': 12.466698169708252, 'total_duration': 16.602430820465088, 'pred_std': 0.10057252645492554, 'std_loss': 0.008725129999220371, 'mean_pred_loss': 2.4340070012840442e-05, 'pred_rmse': 0.05260408669710159, 'pred_mae': 0.03820032626390457, 'pred_mape': 0.1258891075849533, 'grad_rmse': 0.04161679372191429, 'grad_mae': 0.021082276478409767, 'grad_mape': 0.4825449585914612}, 'model_metrics': {'tvae': {'avg_loss': 0.0027671898508283663, 'avg_g_mag_loss': 0.06750814222799723, 'avg_g_cos_loss': 0.031209569611071075, 'pred_duration': 4.135732650756836, 'grad_duration': 12.466698169708252, 'total_duration': 16.602430820465088, 'pred_std': 0.10057252645492554, 'std_loss': 0.008725129999220371, 'mean_pred_loss': 2.4340070012840442e-05, 'pred_rmse': 0.05260408669710159, 'pred_mae': 0.03820032626390457, 'pred_mape': 0.1258891075849533, 'grad_rmse': 0.04161679372191429, 'grad_mae': 0.021082276478409767, 'grad_mape': 0.4825449585914612}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:16:21.365486Z", + "iopub.status.busy": "2024-03-22T18:16:21.365041Z", + "iopub.status.idle": "2024-03-22T18:16:21.370168Z", + "shell.execute_reply": "2024-03-22T18:16:21.369073Z" + }, + "papermill": { + "duration": 0.028958, + "end_time": "2024-03-22T18:16:21.372457", + "exception": false, + "start_time": "2024-03-22T18:16:21.343499", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:16:21.409276Z", + "iopub.status.busy": "2024-03-22T18:16:21.408968Z", + "iopub.status.idle": "2024-03-22T18:16:21.506071Z", + "shell.execute_reply": "2024-03-22T18:16:21.505164Z" + }, + "papermill": { + "duration": 0.117463, + "end_time": "2024-03-22T18:16:21.508351", + "exception": false, + "start_time": "2024-03-22T18:16:21.390888", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:16:21.544277Z", + "iopub.status.busy": "2024-03-22T18:16:21.543923Z", + "iopub.status.idle": "2024-03-22T18:16:21.818454Z", + "shell.execute_reply": "2024-03-22T18:16:21.817370Z" + }, + "papermill": { + "duration": 0.295571, + "end_time": "2024-03-22T18:16:21.820671", + "exception": false, + "start_time": "2024-03-22T18:16:21.525100", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATIAAAESCAYAAAB+alnzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA4wElEQVR4nO3de1xUZf4H8M+ZYS7cZkZALqNcvBGGeFldWNS0kqSkVspNM9Zb5qXFsuhibiXV7kbXrc381eoW7m7eS2tXzVLUXAVFAfOCkhqCCgMiMtyZYeb5/XFgZGSAGZgrfN+v17yYOfOcM88wzIfnnPOc5+EYYwyEEOLCBI6uACGE9BQFGSHE5VGQEUJcHgUZIcTlUZARQlweBRkhxOVRkBFCXJ6boytgL3q9HiUlJfD29gbHcY6uDiHkNowx1NTUQKlUQiCwrI3VZ4KspKQEwcHBjq4GIaQLV65cwcCBAy1ap88Embe3NwD+lySTyRxcG0LI7aqrqxEcHGz4rlqizwRZ6+6kTCajICPEiXXn0A8d7CeEuDwKMkKIy6MgI4S4vD5zjIz0jE6ng1ardXQ1iAsTiUQQCoU22TYFGekUYwwqlQpVVVWOrgrpBRQKBQIDA63el5OCjHSqNcT8/f3h4eFBnYlJtzDGUF9fj/LycgBAUFCQVbdPQXabmkYtzqtq0KxjiB3i6+jqOJROpzOEmK9v3/5dkJ5zd3cHAJSXl8Pf39+qu5l0sP82R3+pxKOfZeEvu/MdXRWHaz0m5uHh4eCakN6i9W/J2sdbKchuE+bL/6KLKupB0xnwaHeSWIut/pYoyG4T7OMBjgNqmppRWadxdHUIIWagILuNVCREkEwKALh8o97BtSGEmKNbQbZmzRqEhYVBKpUiJiYG2dnZnZbftm0bIiIiIJVKERUVhd27dxue02q1WLFiBaKiouDp6QmlUom5c+eipKTEaBuVlZVISkqCTCaDQqHAwoULUVtb253qdynU1xMAUHSjzibbJ70fx3H45ptvHF0Nq3r99dcxevRoR1fDJIuDbMuWLUhJSUFqaipyc3MxatQoxMfHG06r3i4zMxOzZ8/GwoULkZeXh8TERCQmJuLMmTMAgPr6euTm5uK1115Dbm4utm/fjoKCAvz2t7812k5SUhLOnj2LvXv3YufOnTh06BAWL17cjbfctTA//jgZtciIK1u/fj0UCoXVtvfCCy8gIyPDatuzKmah6OholpycbHis0+mYUqlkaWlpJsvPnDmTJSQkGC2LiYlhS5Ys6fA1srOzGQBWVFTEGGMsPz+fAWDHjx83lPnuu+8Yx3Hs2rVrZtVbrVYzAEytVndZ9tODF1noip3smU25Zm27t2poaGD5+fmsoaHB0VVxOQDYjh07HFqH9PR0JpfLuyzX1NRk+8q06OxvypLv6O0sapFpNBrk5OQgLi7OsEwgECAuLg5ZWVkm18nKyjIqDwDx8fEdlgcAtVoNjuMM/02ysrKgUCgwbtw4Q5m4uDgIBAIcO3bM5DaamppQXV1tdDOX4cwltciMMMZQr2l2yI1ZeAZ5z549mDhxIhQKBXx9ffHggw/i0qVLAIDx48djxYoVRuWvX78OkUiEQ4cOAQBKS0uRkJAAd3d3DBo0CBs3bkRYWBg++uijbv3uTp8+jXvvvRfu7u7w9fXF4sWLjQ6NHDx4ENHR0fD09IRCocCECRNQVFQEAPjpp59wzz33wNvbGzKZDGPHjsWJEyc6fb2DBw9iwYIFhu8Sx3F4/fXXAQBhYWH405/+hLlz50Imkxn2bFasWIHw8HB4eHhg8ODBeO2114y6Sdy+azl//nwkJibi/fffR1BQEHx9fZGcnOyQS9ks6hBbUVEBnU6HgIAAo+UBAQE4f/68yXVUKpXJ8iqVymT5xsZGrFixArNnzzaMG6ZSqeDv729ccTc3+Pj4dLidtLQ0vPHGG2a9r9vRMTLTGrQ63Lnqe4e8dv6b8fAQm//nWldXh5SUFIwcORK1tbVYtWoVHn74YZw8eRJJSUl499138fbbbxu6A2zZsgVKpRJ33XUXAGDu3LmoqKjAwYMHIRKJkJKS0uHhE3PqEh8fj9jYWBw/fhzl5eV48sknsWzZMqxfvx7Nzc1ITEzEokWLsGnTJmg0GmRnZxvqlpSUhDFjxuDTTz+FUCjEyZMnIRKJOn3N8ePH46OPPsKqVatQUFAAAPDy8jI8//7772PVqlVITU01LPP29sb69euhVCpx+vRpLFq0CN7e3njppZc6fJ0DBw4gKCgIBw4cwMWLFzFr1iyMHj0aixYt6tbvqrucqme/VqvFzJkzwRjDp59+2qNtrVy5EikpKYbHraNPmiO0pUV2s14Ldb0Wco/O/2iI85kxY4bR4y+++AL9+/dHfn4+Zs6ciWeffRaHDx82BNfGjRsxe/ZscByH8+fPY9++fTh+/LhhL+Af//gHhg0b1q26bNy4EY2NjfjXv/4FT0/+n+Qnn3yChx56CO+88w5EIhHUajUefPBBDBkyBAAwfPhww/rFxcV48cUXERERAQBm1UMsFkMul4PjOAQGBrZ7/t5778Xzzz9vtOzVV1813A8LC8MLL7yAzZs3dxpk/fr1wyeffAKhUIiIiAgkJCQgIyPDuYPMz88PQqEQZWVlRsvLyspM/rIAIDAw0KzyrSFWVFSE/fv3G43iGhgY2O6/YXNzMyorKzt8XYlEAolEYvZ7a8tD7AZ/bwnKa5pQVFmHkR6Kbm2nt3EXCZH/ZrzDXtsSFy5cwKpVq3Ds2DFUVFRAr9cD4ENhxIgRmDp1KjZs2IC77roLhYWFyMrKwt///ncAQEFBAdzc3PCrX/3KsL2hQ4eiX79+3ar7uXPnMGrUKEOIAcCECROg1+tRUFCASZMmYf78+YiPj8d9992HuLg4zJw503A9YkpKCp588kn8+9//RlxcHB599FFD4HVX28M0rbZs2YKPP/4Yly5dQm1tLZqbm7scTTkyMtLoUqOgoCCcPn26R3XrDouOkYnFYowdO9bozIVer0dGRgZiY2NNrhMbG9vuTMfevXuNyreG2IULF7Bv37521/XFxsaiqqoKOTk5hmX79++HXq9HTEyMJW/BbGEtu5d05vIWjuPgIXZzyM3SHuEPPfQQKisrsW7dOhw7dsxwLFWj4Ts5JyUl4auvvoJWq8XGjRsRFRWFqKgoq//OzJWeno6srCyMHz8eW7ZsQXh4OI4ePQqAPzZ19uxZJCQkYP/+/bjzzjuxY8eOHr1e21AF+OPQSUlJmDZtGnbu3Im8vDy88sorht9XR27fxeU4zvBPw54s7n6RkpKCdevW4Z///CfOnTuHp556CnV1dViwYAEA/tjCypUrDeWXL1+OPXv24IMPPsD58+fx+uuv48SJE1i2bBkAPsR+97vf4cSJE9iwYQN0Oh1UKhVUKpXhlzh8+HDcf//9WLRoEbKzs3HkyBEsW7YMjz32GJRKpTV+D+2EGi5VouNkrubGjRsoKCjAq6++iilTpmD48OG4efOmUZnp06ejsbERe/bswcaNG5GUlGR47o477kBzczPy8vIMyy5evNhuG+YaPnw4fvrpJ9TV3fpbOnLkCAQCAe644w7DsjFjxmDlypXIzMzEiBEjsHHjRsNz4eHheO655/DDDz/gkUceQXp6epevKxaLodPpzKpjZmYmQkND8corr2DcuHEYNmyY4WSDK7A4yGbNmmU4UDh69GicPHkSe/bsMRzQLy4uRmlpqaH8+PHjsXHjRqxduxajRo3CV199hW+++QYjRowAAFy7dg3/+c9/cPXqVYwePRpBQUGGW2ZmpmE7GzZsQEREBKZMmYJp06Zh4sSJWLt2bU/ff4fC/KhF5qr69esHX19frF27FhcvXsT+/fuNjpcCfIskMTERr732Gs6dO4fZs2cbnouIiEBcXBwWL16M7Oxs5OXlYfHixXB3d+/WtYJJSUmQSqWYN28ezpw5gwMHDuDpp5/GnDlzEBAQgMLCQqxcuRJZWVkoKirCDz/8gAsXLmD48OFoaGjAsmXLcPDgQRQVFeHIkSM4fvy40TG0joSFhaG2thYZGRmoqKhAfX3Hf8vDhg1DcXExNm/ejEuXLuHjjz/ucavPrnrcMcRFWNpH5b8/XWOhK3ayGf93xMY1c16u3I9s7969bPjw4UwikbCRI0eygwcPtuvbtXv3bgaATZo0qd36JSUl7IEHHmASiYSFhoayjRs3Mn9/f/bZZ5+Z9fq3v9apU6fYPffcw6RSKfPx8WGLFi1iNTU1jDHGVCoVS0xMZEFBQUwsFrPQ0FC2atUqptPpWFNTE3vsscdYcHAwE4vFTKlUsmXLlpn9mSxdupT5+voyACw1NZUxxlhoaCj78MMP25V98cUXma+vL/Py8mKzZs1iH374oVE/tNTUVDZq1CjD43nz5rHp06cbbWP58uVs8uTJHdbHVv3IOMb6xhAP1dXVkMvlUKvVZk0Hd+aaGg+uPgw/LwlOvBrXZfneqLGxEYWFhRg0aBCkUqmjq+NQV69eRXBwMPbt24cpU6Y4ujouq7O/KUu/o205VfcLZxLScoysorYJtU3N8JLQr6ov2b9/P2praxEVFYXS0lK89NJLCAsLw6RJkxxdNWICjX7RAZlUBB9PMQDqGNsXabVa/PGPf0RkZCQefvhh9O/f39A5dsOGDfDy8jJ5i4yMtFsdH3jggQ7r8dZbb9mtHs6AmhmdCPX1QGWdBkU36hGplDu6OsSO4uPjER9vus/cb3/72w67/XTV496a/vGPf6ChocHkcz4+PnarhzOgIOtEmK8n8oqrcJlaZKQNb29veHt7O7oaGDBggKOr4DRo17IToW2GvSaEOC8Ksk7c6t1PLTJCnBkFWSdCaTgfQlwCBVknWltkqupGNGjMu9SDEGJ/FGSdUHiIIJPy50OKK6lVRoizoiDrBMdxhmsuqS8ZsURvnHzEmVGQdeHWaLHUIiOuxdqTjwD8ENocx6Gqqsqq2+0pCrIutI7fT2cuCXFeFGRdoBZZG4wBmjrH3GjyEatNPtLU1IQXXngBAwYMgKenJ2JiYnDw4EHDukVFRXjooYfQr18/eHp6IjIyErt378bly5dxzz33AOCHSuI4DvPnz+/W78PaqGd/F6hF1oa2HnjLNgNZdumPJYDYs+tyLWjykY4nH1m2bBny8/OxefNmKJVK7NixA/fffz9Onz6NYcOGITk5GRqNBocOHYKnpyfy8/Ph5eWF4OBgfP3115gxYwYKCgogk8ng7u7erd+JtVGQdaG1RVZS1YCmZh0kbpaNHU8cgyYfMT35SHFxMdLT01FcXGwYXfmFF17Anj17kJ6ejrfeegvFxcWYMWOGYejvwYMHG9ZvvYbT39/f6sffeoKCrAt+XmJ4ioWo0+hw9WYDhvT36nql3krkwbeMHPXaFqDJR0w7ffo0dDodwsPDjZY3NTUZ5sp45pln8NRTT+GHH35AXFwcZsyYgZEjR3br9eyFjpF1geM4mueyFcfxu3eOuNHkI1aZfKS2thZCoRA5OTk4efKk4Xbu3Dn87W9/AwA8+eST+OWXXzBnzhycPn0a48aNw+rVq632Xm2BgswMYX4tx8no4nGXQJOP8ExNPjJmzBjodDqUl5dj6NChRre2u6DBwcFYunQptm/fjueffx7r1q0zbBOA2ZOa2AsFmRlCfKhF5kpo8hGeqclHwsPDkZSUhLlz52L79u0oLCxEdnY20tLSsGvXLgDAs88+i++//x6FhYXIzc3FgQMHDK8XGhoKjuOwc+dOXL9+3ejMq0NZPMq/i+rJxAabjhWx0BU72dzPj9mgZs6LJh/pnZOPaDQatmrVKhYWFsZEIhELCgpiDz/8MDt16hRjjLFly5axIUOGMIlEwvr378/mzJnDKioqDNt88803WWBgIOM4js2bN8+serSiyUd6qCcTG2RduoHZ644izNcDB1+8x0Y1dD40+cgtNPmIddDkIw7Ueozs6s0GaHV6iIS0R97b0eQjroW+kWYI8JZC4iZAs56hpMr0GOmkd6HJR1wLtcjMIBBwCPX1wM9ltbh8o97QHYP0XjT5iGuhIDNTqK8nfi6rbTlz2d/R1SEORJOPOB/atTST4ZrLPtiXrI+cDyJ2YKu/JQoyM/XF3v2tu0n19X0vvIlttP4tWXsXnHYtzdQXZ1QSCoVQKBSGUR88PDy61SGUEMYY6uvrUV5eDoVCAaHQuoMvUJCZqXVGpSuVDdDpGYSCvvGFbr1spbtD2BDSlkKhMLoUylooyMykVLhDJOSg0emhqm7EAIVzjMNkaxzHISgoCP7+/tBqtY6uDnFhIpHI6i2xVhRkZhIKOAT7eOCX63UoqqjrM0HWSigU2uyPkJCeooP9Frh1nIwOfhPiTCjILHBr5vG+c8CfEFdAQWaBvnjmkhBXQEFmgVstMtq1JMSZUJBZoG2LjHq7E+I8KMgsMKCfO4QCDo1aPcprmhxdHUJICwoyC4iEAkO3i8sVdJyMEGdBQWYhOk5GiPOhILMQnbkkxPlQkFmIWmSEOB8KMgtRi4wQ50NBZqHWiUiKbtRTFwxCnAQFmYUG9vMAxwG1Tc24UadxdHUIIaAgs5hUJIRSznfBoGsuCXEO3QqyNWvWICwsDFKpFDExMcjOzu60/LZt2xAREQGpVIqoqCjs3r3b6Pnt27dj6tSp8PX1BcdxOHnyZLtt3H333eA4zui2dOnS7lS/x0L78Pj9hDgji4Nsy5YtSElJQWpqKnJzczFq1CjEx8d3OIJoZmYmZs+ejYULFyIvLw+JiYlITEzEmTNnDGXq6uowceJEvPPOO52+9qJFi1BaWmq4vfvuu5ZW3yr64vj9hDg1ZqHo6GiWnJxseKzT6ZhSqWRpaWkmy8+cOZMlJCQYLYuJiWFLlixpV7awsJABYHl5ee2emzx5Mlu+fLml1TVQq9UMAFOr1d3eRqvPDl5koSt2smc25fZ4W4QQXk++oxa1yDQaDXJychAXF2dYJhAIEBcXh6ysLJPrZGVlGZUH+MlPOyrfmQ0bNsDPzw8jRozAypUrO53dp6mpCdXV1UY3awmlARYJcSoWDXVdUVEBnU6HgIAAo+UBAQE4f/68yXVUKpXJ8iqVyqKKPv744wgNDYVSqcSpU6ewYsUKFBQUYPv27SbLp6Wl4Y033rDoNcx1qwsG7VoS4gxcZsz+xYsXG+5HRUUhKCgIU6ZMwaVLlzBkyJB25VeuXImUlBTD4+rqagQHB1ulLiE+fJBV1WtRVa+BwkNsle0SQrrHol1LPz8/CIVClJWVGS0vKyvrcIqnwMBAi8qbKyYmBgBw8eJFk89LJBLIZDKjm7V4iN0QIJMAoEuVCHEGFgWZWCzG2LFjkZGRYVim1+uRkZGB2NhYk+vExsYalQeAvXv3dljeXK1dNIKCgnq0ne4KpUuVCHEaFu9apqSkYN68eRg3bhyio6Px0Ucfoa6uDgsWLAAAzJ07FwMGDEBaWhoAYPny5Zg8eTI++OADJCQkYPPmzThx4gTWrl1r2GZlZSWKi4tRUlICACgoKADAt+YCAwNx6dIlbNy4EdOmTYOvry9OnTqF5557DpMmTcLIkSN7/EvojjBfD2QXVlKLjBBn0J3TpKtXr2YhISFMLBaz6OhodvToUcNzkydPZvPmzTMqv3XrVhYeHs7EYjGLjIxku3btMno+PT2dAWh3S01NZYwxVlxczCZNmsR8fHyYRCJhQ4cOZS+++KJFp2mt2f2CMcY+2X+Bha7YyZ7bkmeV7RHS1/XkO8ox1jeufK6uroZcLodarbbK8bKdp0qwbGMexob2w9dPjbdCDQnp23ryHaVrLbspjHr3E+I0KMi6KaTlesuKWg1qGrUOrg0hfRsFWTfJpCL4evL9x+iAPyGORUHWAzTsNSHOgYKsB2jYa0KcAwVZD9BwPoQ4BwqyHmi9eJxGwSDEsSjIeoBaZIQ4BwqyHghrOdhfVt2EBo3OwbUhpO+iIOsBhYcYcncRAKC4knYvCXEUCrIeam2V0ZlLQhyHgqyH6DgZIY5HQdZDt1pktGtJiKNQkPUQtcgIcTwKsh4y9CWjyXoJcRgKsh5qbZGVqBvQ1ExdMAhxBAqyHvL1FMNL4gbGgCuVDY6uDiF9EgVZD3EcZ5gejo6TEeIYFGRWQNdcEuJYFGRWQGcuCXEsCjIroL5khDgWBZkVUIuMEMeiILOC1pFir95sgFand3BtCOl7KMiswN9bAqlIAJ2e4dpN6oJBiL1RkFmBQMAh1IfG7yfEUSjIrIRmVCLEcSjIrCTMj1pkhDgKBZmVUIuMEMehILOSMOqCQYjDUJBZSWuL7EplA3R65uDaENK3UJBZSZDcHWKhABqdHqVq6oJBiD1RkFmJUMAh2McdAB0nI8TeKMisqPU4GZ25JMS+KMis6NY1l9QiI8SeKMis6Nb4/dQiI8SeKMis6NZIsdQiI8SeKMisyNCXrLIOeuqCQYjdUJBZ0YB+7hAKODRq9SivaXJ0dQjpMyjIrEgkFGBgP74LBp25JMR+KMisjEaLJcT+KMisjMbvJ8T+KMisjFpkhNgfBZmVGVpkFdQiI8ReKMisrG2LjDHqgkGIPVCQWVmwjzs4DqjT6FBRq3F0dQjpE7oVZGvWrEFYWBikUiliYmKQnZ3daflt27YhIiICUqkUUVFR2L17t9Hz27dvx9SpU+Hr6wuO43Dy5Ml222hsbERycjJ8fX3h5eWFGTNmoKysrDvVtymJmxBKeesoGHScjBB7sDjItmzZgpSUFKSmpiI3NxejRo1CfHw8ysvLTZbPzMzE7NmzsXDhQuTl5SExMRGJiYk4c+aMoUxdXR0mTpyId955p8PXfe655/Df//4X27Ztw48//oiSkhI88sgjllbfLgzXXNKZS0Lsg1koOjqaJScnGx7rdDqmVCpZWlqayfIzZ85kCQkJRstiYmLYkiVL2pUtLCxkAFheXp7R8qqqKiYSidi2bdsMy86dO8cAsKysLLPqrVarGQCmVqvNKt8TK7efYqErdrIPvj9v89cipLfoyXfUohaZRqNBTk4O4uLiDMsEAgHi4uKQlZVlcp2srCyj8gAQHx/fYXlTcnJyoNVqjbYTERGBkJCQDrfT1NSE6upqo5u9UF8yQuzLoiCrqKiATqdDQECA0fKAgACoVCqT66hUKovKd7QNsVgMhUJh9nbS0tIgl8sNt+DgYLNfr6eoLxkh9tVrz1quXLkSarXacLty5YrdXvvWSLHUIiPEHtwsKezn5wehUNjubGFZWRkCAwNNrhMYGGhR+Y62odFoUFVVZdQq62w7EokEEonE7NewptZxydQNWlTVa6DwEDukHoT0FRa1yMRiMcaOHYuMjAzDMr1ej4yMDMTGxppcJzY21qg8AOzdu7fD8qaMHTsWIpHIaDsFBQUoLi62aDv24i4WIlAmBUCtMkLswaIWGQCkpKRg3rx5GDduHKKjo/HRRx+hrq4OCxYsAADMnTsXAwYMQFpaGgBg+fLlmDx5Mj744AMkJCRg8+bNOHHiBNauXWvYZmVlJYqLi1FSUgKADymAb4kFBgZCLpdj4cKFSElJgY+PD2QyGZ5++mnExsbiN7/5TY9/CbYQ4usBVXUjim7UYXSwwtHVIaR3685p0tWrV7OQkBAmFotZdHQ0O3r0qOG5yZMns3nz5hmV37p1KwsPD2disZhFRkayXbt2GT2fnp7OALS7paamGso0NDSwP/zhD6xfv37Mw8ODPfzww6y0tNTsOtuz+wVjjL247SQLXbGTfbT3Z7u8HiGuriffUY6xvnFBYHV1NeRyOdRqNWQymc1fb82Bi3jv+wI8MmYA/jprtM1fjxBX15PvaK89a+loNMclIfZDQWYjob40oxIh9kJBZiOtQXajToPqRq2Da0NI70ZBZiPeUhH8vPj+Y8XUKiPEpijIbCiUjpMRYhcUZDZEx8kIsQ8KMhsynLmsoBYZIbZEQWZD1CIjxD4oyGyI+pIRYh8UZDbUGmTlNU2o1zQ7uDaE9F4UZDYk9xBB4SECABRX0u4lIbZCQWZjhi4YNGEvITZDQWZjYYYD/nScjBBboSCzsVAa9poQm6MgM+XsDuDGJatsilpkhNgeBdnt8r4Ets0HNs4EGm72eHPUl4wQ26Mgu93Q+wDZQODGRWDrPEDXs5ErWnctS9QNaNTqrFFDQshtKMhu5x0APL4ZEHkChT8C360AejCIrq+nGF4SNzAGXL1JrTJCbIGCzJTAKGDGPwBwwInPgex13d4Ux3GG3UvqgkGIbVCQdSRiGnDfG/z9PSuAi/u6vSm6VIkQ26Ig68z4Z4DRvweYHti2ALhe0K3N0AF/QmyLgqwzHAc8+FcgZDzQVM2fyay7YfFmqEVGiG1RkHXFTQLM+hJQhAI3LwNb5wDNGos2YThGRkFGiE1QkJnD0xd4fCsgkQFFR4Bdz1l0JnOovxcEHHClsgHHfrG8RUcI6RwFmbn8I4DfpQOcgO80m7na7FV9vSR4LDoEAPDGf/Oh0/eJOZEJsRsKMksMiwPi0/j7e1cBBd+Zverz94XDW+qG/NJqbD1xxUYVJKRvoiCzVMwSYNwTABjw9ZOA6oxZq/l6SfBsXDgA4P3vC6BuoLkuCbEWCjJLcRzwwLvAoMmAphbY9BhQW27WqnNjQzGkvydu1GmwOuOCjStKSN9BQdYdQhEw85+A71BAfQXYnARoG7tcTSQU4LUH7wQArM+8jEvXa21dU0L6BAqy7nLvB8zeAkgVwNVs4D9Pm3Um8+47/HFvhD+a9Qx/2XXO9vUkpA+gIOsJv6HAzH8BAjfg9Fbgf++btdqrCcPhJuCw/3w5DhSYt1tKCOkYBVlPDZ4MTHuPv7//z0D+t12v0t8LCyaEAQD+tDMfWp3ehhUkpPejILOGcU8AMU/x97cvAUryulzl6SnD4Ospxi/X6/CvrCIbV5CQ3o2CzFqm/hkYGgc0NwCbZgPVpZ0Wl0lFeDH+DgDAR/t+xo3aJnvUkpBeiYLMWoRuwO++APpHADWlfLcMTeejXTw6LhiRShlqGpvxwd6f7VRRQnofCjJrksqB2ZsBD1+g9CTwzVJA3/HxL6GAQ+pDkQCATdnFOFuitlNFCeldKMiszWcQP1qGQMQf+D+Y1mnx6EE+eHBkEBgD3vxvPlgPhtUmpK+iILOF0PHAQ3/j7x96Fzi1rdPiK6cNh8RNgGOFlfjujMoOFSSkd6Egs5UxScCE5fz9b5OBK8c7LDpA4Y6lk4cAAP6y6xzNtkSIhdwcXYFebUoqUHEBKNgNbH4ceGIP4OkH6HX8jd36+dRIITKzK1GpbsDX3zUhKXpgmzL6lp/N7Zf5DAb8hjn6nRLiUBzrIwdlqqurIZfLoVarIZPJ7PfCTbXAF/cDZadts32hBHjqCIUZcXk9+Y5Si8zWJF7A7E3AlzOAitsmL+EEACcEBEKAE4IJhKjV6NGkF0Dk5ga5h7TlOQH/U+DWprwAqLvOd/XYsxL4/VeOeX+EOAEKMntQBAN/OAo0NxpCi//JGRXjAFy+qsZv1xwGawK+nhuLsaE+HW+34iLwf78BLu4Ffv4eCI+37fsgxEnRwX57EQgAsQc/mYnQrV2ItYoaKMfMscEA+GGx9Z0Ni+03FPhNy6VRe1ZaPCkKIb0FBZkTeiH+DnhJ3HDqqhpf517tvPCkFwFPf6DyEnDsU/tUkBAnQ0HmhPp7S/DMlKEAgHf2FKCmsZNhsaUyIO51/v6P7wE1ZbavICFOpltBtmbNGoSFhUEqlSImJgbZ2dmdlt+2bRsiIiIglUoRFRWF3bt3Gz3PGMOqVasQFBQEd3d3xMXF4cIF46Ggw8LCwHGc0e3tt9/uTvVdwvzxgzDIzxMVtU1Yc+BS54VHzQYGjAU0NUDGG/apICFOxOIg27JlC1JSUpCamorc3FyMGjUK8fHxKC83PUBgZmYmZs+ejYULFyIvLw+JiYlITEzEmTO3Ju1499138fHHH+Ozzz7DsWPH4Onpifj4eDQ2Gg8f/eabb6K0tNRwe/rppy2tvssQuwnwasJwAMAXhwtxuaKTyX0FAn4eAQA4uQG4mmOHGhLiRJiFoqOjWXJysuGxTqdjSqWSpaWlmSw/c+ZMlpCQYLQsJiaGLVmyhDHGmF6vZ4GBgey9994zPF9VVcUkEgnbtGmTYVloaCj78MMPLa2ugVqtZgCYWq3u9jbsTa/XszmfH2OhK3ayJ/95vOsVti9hLFXG2Np7GdPpbF9BQqyoJ99Ri1pkGo0GOTk5iIuLMywTCASIi4tDVlaWyXWysrKMygNAfHy8oXxhYSFUKpVRGblcjpiYmHbbfPvtt+Hr64sxY8bgvffeQ3Nzc4d1bWpqQnV1tdHN1XAch1UPDodQwGFvfhkOX6jofIW41wGxF3DtBHBqi13qSIgzsCjIKioqoNPpEBAQYLQ8ICAAKpXpi51VKlWn5Vt/drXNZ555Bps3b8aBAwewZMkSvPXWW3jppZc6rGtaWhrkcrnhFhwcbP4bdSJD/b0xNzYUAPDmzrNo7mxYbO9AYNIL/P19qUBTjR1qSIjjucxZy5SUFNx9990YOXIkli5dig8++ACrV69GU5PpkVVXrlwJtVptuF254rqzez87JRz9PET4uawWG7OLOy/8mz/w11/WlgGHzJsMhRBXZ1GQ+fn5QSgUoqzM+BR/WVkZAgMDTa4TGBjYafnWn5ZsEwBiYmLQ3NyMy5cvm3xeIpFAJpMZ3VyV3EOE56fyw2J/8MPPuFnXScdXNwkQ/xZ//+j/ATe6OONJSC9gUZCJxWKMHTsWGRkZhmV6vR4ZGRmIjY01uU5sbKxReQDYu3evofygQYMQGBhoVKa6uhrHjh3rcJsAcPLkSQgEAvj7+1vyFlzW7OgQRAR6Q92gxUf7uhgWO/x+YMgUQKcBvv+jfSpIiCNZenZg8+bNTCKRsPXr17P8/Hy2ePFiplAomEqlYowxNmfOHPbyyy8byh85coS5ubmx999/n507d46lpqYykUjETp8+bSjz9ttvM4VCwb799lt26tQpNn36dDZo0CDW0NDAGGMsMzOTffjhh+zkyZPs0qVL7Msvv2T9+/dnc+fONbvernjW8nZHLl5noSt2ssErd7HzpdWdFy4vYOwNH/4s5s977VNBQnqgJ99Ri4OMMcZWr17NQkJCmFgsZtHR0ezo0aOG5yZPnszmzZtnVH7r1q0sPDycicViFhkZyXbt2mX0vF6vZ6+99hoLCAhgEomETZkyhRUUFBiez8nJYTExMUwulzOpVMqGDx/O3nrrLdbY2Gh2nXtDkDHG2NJ/n2ChK3ayx9dlMb1e33nhPX/kg+zjsYxpm+xTQUK6qSffURqPzMVcqazHlL/+CE2zHmvnjMXUyI6PI6JRDaweyw/3M/UvwPhl9qsoIRbqyXfUZc5aEl6wjwcW3zUYAPDnXefQ1NzJsNhSOTBlFX//x3eAWtNXX9hcSR5w4C1+tFxCbICCzAU9dfcQBMgkKK6sxxeHL3deePTvgaDRQFM1kPGmPapnLP8//Ai5P74DfPJrYNPjQPEx+9eD9GoUZC7IU+KGlx+IAAD8dW8BXvrqJ1wo66Dza9vrMPO+5FtH9nL0U2DrXH5ASZ/BABhQsAv4YirweTxwflen834SYi4KMhc1fdQATIsKhFbHsPXEVdz34SE8sf44si7daD83ZkgMEDUTAAO+WwHY+rCoXg/s+SOw52X+NcctBJKP87dfzQWEYuDKUX5CljXRQO6/gGbTHZsJMQcd7HdxOUWVWHeoEN/nqwz5FDVAjkWTBmPaiEC4CVv+V1WXAKvHAdo64JF1wMiZtqmQthHYsZifnBjgr/+c8KzxiLg1KuDYZ8DxL4CmltnVvQKAmKXAuCcAd4Vt6kacWk++oxRkvcTlijp8frgQ23KuoFHL764NULhj4cRBmPXrYHhK3PhLlvb/CfAOApad4CdGsab6SmDTbL61JRABiZ8CIx/tuHxjNZD7T34XtPoav0zsBYydz19qJR9g3foRp0ZBZobeHmStKus0+HdWEf6VdRk3Wi5lkknd8PvfhGJ+dCD8/z0ZuHkZuOv5W2c0rfLChcCGR4EbFwCJHHhsAzDoLvPWbdYAZ74GMj8GyvP5ZQI3IOpRYPzTQECk9epJnBYFmRn6SpC1atTqsD33Gv7xv1/wS8ugjCIhh1cG/4L5V17h58NMPgb4DOr5i13LATbO4vuryQbyU9P5D7d8O4wBF/cBR/4GXP7freVD7+NnbQ+b2OGkLcT1UZCZoa8FWSu9nmHfuTKs+98vOH75JgCGf4nexiThadwIvg8+T2wD15NwKNgDfLUA0NYDgVHA49sAWVDPK341B8j8G3DuvwBrObOp/BUw4Rlg+G/56fRIr0JBZoa+GmRt5RbfxLpDv+Bi/gl8J3oZbpwer3j/GdFTHsG0qCCIhBaexD7xBbDreT5ohkwBZv4TkHhbt9I3LgFZa/ghvJtbhj7vN4i/SmF0EiByt+7rEYehIDMDBdktRTfqULJ5OWKvb8PP+gGYpklDgMIbCyaE4bHoEHhJupi3Wa8H9r8JHP6Qfzzm98CDHwFCke0qXXsdyF4LHF8HNNzkl7n7AMPuA0In8LudPoNp19OFUZCZgYLsNg1V0H/8KwgabuA9wQKsqb8PAOAtdcPvxg7Er0L64U6lDIN8PSEQtAmH5ibg22Tg9Db+8d1/BCa/ZL8A0dTxHXszPwHUtw0y6R3UEmoTgLC7AN+hFGwuhILMDBRkJpxIB3Y+CyaVY8eE/+CT7Cr8ct14tiYPsRARgd64UynD6P4c7j/zPLxKj/JnFR/6GBiT5Ji665qBosPA5cPA5SP8PAW62wac9PRvCbWJQOhEoP8dFGxOjILMDBRkJuh1wNq7AdUpYOwC6BM+xP7z5dhfUI78kmqcV1Ub+qQpUYF08bu4Q3AVNcwdb3m+jIaQyYhUynGnUoY7g2To5yl23HvRNgBXj/Ohdvkwf19329UCHn5A6Hi+tRY2Aeg/nL+EizgFCjIzUJB1oCgTSH8AAAcsOQQEjTQ8pdMzFFbU4kr+MYw7shTe2usohw/mN72AfBbWblNKudQQancq5YhUyjCwn3vPzop2l7aR7xZy+TDfcruSfetkQSt3n5Zgm8jf/CMp2ByIgswMFGSd+OoJvkNqyHhgwW7j3a+LGfyF35paoP9wsKRtKOP6I79UjbPXqpFfWo2zJdUorqw3uWlvqRvuDJJhcH/+KoJmnR7NegatTo9mHUOzXg9t25+G55lRWV3rMj2/nlanB8cBQ/p74c4gGSKVMkQOkCMi0BveUhMnHZqbgGu5LbujR4Arx/guI21JFUBILOA7hL9kyjuQv3kFAt4BgETmmF1TxvgZsequ80Mx1V3nbx6+QHCMdbq7OAEKMjNQkHVCfZW/DrO5AZjxORD1O3553gbgv88A+mZ+d2zWlx1eB1ndqMX50hqcLVEjv4QPtwvlNdDq7P/nFebrcWuXVylDZJAM/jKpcaFmDVB6suUY22E+2DS1nW/YzZ0PNO+gW0HXLvACAfd+XQceY/zZV0M4lfNnZuvKWx5XGC+7vTXZljwECI7mQy04GggYAQi7OPPshCjIzEBB1oUf3wUO/AWQDQCWHQcyVwMH0/jnomYC0z/hZ2iygKZZj4vltThbosaVmw1wE3BwE3IQCQRwE3ItjwVwE3AQCVuXCSAS8stFrc+3WUfUUsZNyEGrYyhQ1SC/RG1oGZaqTX/h/bwkiGwNNqUMkUo5Qn08bp2R1TUDpT/xgVZ9jZ9Or0bF32rL+PHczCUU8wHXNuy0DW1CqqVFpe94gmmTRJ6AV3/+JIanH1B1BSg/e6vDcNtyA8e2BFsMMHAcH65OjoLMDBRkXdA28EPqVBXz3RZuXOSXT0zhr8l0kbN9N2qbcK6lZXi2hN/1/eV6LfQm/so9xUIMb9ktvbMl3IYFeEHiZuKqAU09UKsCasqAmtJbQVfb8rimjH++tY+buaRywLMlnFpDysufX+bl32Z5f0Ds2X79xmr+WOCVbD6Erx43Hbr9I9q02mKcsmsKBZkZKMjMkP8fYOsc/j4nABI+4IfVcXH1mmacV9UYdnnzS9Q4r6pBU3P7QR2FAg79PMTo5yFCPw8x5B4i9PMQQeEhhqJlmcKdf9zPUwSFO79cKmoJv+am9q252jL+CoR2IdXf4lZul/R64Pp5PtRaw63SxNym7j4twdYSbspfAWIPy16LMb7LS3NTy62R/6lrc7+5kd+NDx0PSDv/3lGQmYGCzAyM8Qf+C38Epv8fcMf9jq6RzTTr9Pilos7omN7ZkmqoG7Td2p5UJOBDriXo+nmKbt33EEPsJoCAAwQCDgKOg4ADOI6DkOMgEAACjrv1uPU5AX+ffw4tj/n7Ao6DwkMEpcIdMlMnN9qqq7gValeO8Sc9bu+aInADAkcCipCWcLotiNo+1rUJLnMt+Z/RGXFTKMjMQEFmJsb4Yy598KJsxhjKa5pQUduEqnotquq1uFmvQVW9puW+lr/fwC9X12tR1aCFztR+qx15S9ygVLhDqZC2/HTHgJafSoUUATKp8XW0zRq+72BrsBUf43eLe0ooAdykfCvTcGt5PH1NlyOiUJCZgYKM2IJez1DT1Ax1S+jdrNdA3aDFzToNbtZr+fv1GjTrGHR6Bj1j0DO0/OTvM2b83K3HLfcZg17Pr8MYWh4zVLYEbFcEHBAguxVySoWUDzp5S+jJpZBpSsFdPQ7U37gtjFruG4WUFHATGz8Wint8zK0n31HXO0dLiBMRCDjI3UWQu4sQ4mvhMSYrqGtqRqm6AdeqGlFS1YCSqgZca/lZUtWIUnUDtDqGUnUjStWNyCkyfTLCUyyEUtEf/rKBhl1ihQd/DFDuIWrZXRZDIRJBLuHfr8mTIg5CQUaIC/OUuGGovzeG+psePkmvZ6iobUKJ2nTQlVQ14EadBnUaHS6U1+JCeRd96drwEAuhcBdB3nJyROEhgrzl5EdrEMrd+efuVMpMd1S2EgoyQnoxgYCDv0wKf5kUo4MVJss0aHQtrboG3KjVGI4DVrU5JljVsptc1bLrrGdAvUaHeo0OJR303Wtrxx/GY0yI7fqyUZAR0se5i4UY3N/LcBlZV/R6hprGZlQ18MfoqtoE3M06Laoabp0IaQ1CHxsPKEBBRgixiEDAQe4hgtxDhFBfR9eGR5f6E0JcHgUZIcTlUZARQlweBRkhxOVRkBFCXB4FGSHE5VGQEUJcXp/pR9Z6bXx1tQUjfRJC7Kb1u9mdcSz6TJDV1NQAAIKDgx1cE0JIZ2pqaiCXyy1ap88M46PX61FSUgJvb+8upyerrq5GcHAwrly50muG/Olt76m3vR+A3hNjDDU1NVAqlRBYOC1fn2mRCQQCDBw40KJ1ZDJZr/mDatXb3lNvez9A335PlrbEWtHBfkKIy6MgI4S4PAoyEyQSCVJTUyGRWHmGGwfqbe+pt70fgN5TT/SZg/2EkN6LWmSEEJdHQUYIcXkUZIQQl0dBRghxeRRkhBCX12eDbM2aNQgLC4NUKkVMTAyys7M7Lb9t2zZERERAKpUiKioKu3fvtlNNu5aWloZf//rX8Pb2hr+/PxITE1FQUNDpOuvXrwfHcUY3qVRqpxp37fXXX29Xv4iIiE7XcebPCADCwsLavSeO45CcnGyyvLN9RocOHcJDDz0EpVIJjuPwzTffGD3PGMOqVasQFBQEd3d3xMXF4cKFC11u19Lvoil9Msi2bNmClJQUpKamIjc3F6NGjUJ8fDzKy8tNls/MzMTs2bOxcOFC5OXlITExEYmJiThz5oyda27ajz/+iOTkZBw9ehR79+6FVqvF1KlTUVdX1+l6MpkMpaWlhltRUZGdamyeyMhIo/odPny4w7LO/hkBwPHjx43ez969ewEAjz76aIfrONNnVFdXh1GjRmHNmjUmn3/33Xfx8ccf47PPPsOxY8fg6emJ+Ph4NDZ2PO+lpd/FDrE+KDo6miUnJxse63Q6plQqWVpamsnyM2fOZAkJCUbLYmJi2JIlS2xaz+4qLy9nANiPP/7YYZn09HQml8vtVykLpaamslGjRpld3tU+I8YYW758ORsyZAjT6/Umn3fmzwgA27Fjh+GxXq9ngYGB7L333jMsq6qqYhKJhG3atKnD7Vj6XexIn2uRaTQa5OTkIC4uzrBMIBAgLi4OWVlZJtfJysoyKg8A8fHxHZZ3NLVaDQDw8fHptFxtbS1CQ0MRHByM6dOn4+zZs/aontkuXLgApVKJwYMHIykpCcXFxR2WdbXPSKPR4Msvv8QTTzzR6Wgszv4ZtSosLIRKpTL6DORyOWJiYjr8DLrzXexInwuyiooK6HQ6BAQEGC0PCAiASqUyuY5KpbKovCPp9Xo8++yzmDBhAkaMGNFhuTvuuANffPEFvv32W3z55ZfQ6/UYP348rl69asfadiwmJgbr16/Hnj178Omnn6KwsBB33XWXYVy527nSZwQA33zzDaqqqjB//vwOyzj7Z9RW6+/Zks+gO9/FjvSZYXz6iuTkZJw5c6bT40kAEBsbi9jYWMPj8ePHY/jw4fj73/+OP/3pT7auZpceeOABw/2RI0ciJiYGoaGh2Lp1KxYuXOjAmlnH559/jgceeABKpbLDMs7+GTmTPtci8/Pzg1AoRFlZmdHysrIyBAYGmlwnMDDQovKOsmzZMuzcuRMHDhyweOw1kUiEMWPG4OLFizaqXc8oFAqEh4d3WD9X+YwAoKioCPv27cOTTz5p0XrO/Bm1/p4t+Qy6813sSJ8LMrFYjLFjxyIjI8OwTK/XIyMjw+i/X1uxsbFG5QFg7969HZa3N8YYli1bhh07dmD//v0YNGiQxdvQ6XQ4ffo0goKCbFDDnqutrcWlS5c6rJ+zf0Ztpaenw9/fHwkJCRat58yf0aBBgxAYGGj0GVRXV+PYsWMdfgbd+S52yKJTA73E5s2bmUQiYevXr2f5+fls8eLFTKFQMJVKxRhjbM6cOezll182lD9y5Ahzc3Nj77//Pjt37hxLTU1lIpGInT592lFvwchTTz3F5HI5O3jwICstLTXc6uvrDWVuf09vvPEG+/7779mlS5dYTk4Oe+yxx5hUKmVnz551xFto5/nnn2cHDx5khYWF7MiRIywuLo75+fmx8vJyxpjrfUatdDodCwkJYStWrGj3nLN/RjU1NSwvL4/l5eUxAOyvf/0ry8vLY0VFRYwxxt5++22mUCjYt99+y06dOsWmT5/OBg0axBoaGgzbuPfee9nq1asNj7v6LpqrTwYZY4ytXr2ahYSEMLFYzKKjo9nRo0cNz02ePJnNmzfPqPzWrVtZeHg4E4vFLDIyku3atcvONe4YAJO39PR0Q5nb39Ozzz5reP8BAQFs2rRpLDc31/6V78CsWbNYUFAQE4vFbMCAAWzWrFns4sWLhudd7TNq9f333zMArKCgoN1zzv4ZHThwwOTfWWud9Xo9e+2111hAQACTSCRsypQp7d5naGgoS01NNVrW2XfRXDQeGSHE5fW5Y2SEkN6HgowQ4vIoyAghLo+CjBDi8ijICCEuj4KMEOLyKMgIIS6PgowQ4vIoyAghLo+CjBDi8ijICCEu7/8BlajdizYVXO4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:16:21.858143Z", + "iopub.status.busy": "2024-03-22T18:16:21.857786Z", + "iopub.status.idle": "2024-03-22T18:20:40.954647Z", + "shell.execute_reply": "2024-03-22T18:20:40.953627Z" + }, + "papermill": { + "duration": 259.118235, + "end_time": "2024-03-22T18:20:40.957108", + "exception": false, + "start_time": "2024-03-22T18:16:21.838873", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:20:40.994176Z", + "iopub.status.busy": "2024-03-22T18:20:40.993823Z", + "iopub.status.idle": "2024-03-22T18:20:41.015411Z", + "shell.execute_reply": "2024-03-22T18:20:41.014472Z" + }, + "papermill": { + "duration": 0.04219, + "end_time": "2024-03-22T18:20:41.017464", + "exception": false, + "start_time": "2024-03-22T18:20:40.975274", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.0302770.0456360.00276712.4882350.0210820.4825450.0416170.0000244.1151970.03820.1258890.0526040.1005730.00872516.603431
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.030277 0.045636 0.002767 12.488235 0.021082 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 0.482545 0.041617 0.000024 4.115197 0.0382 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.125889 0.052604 0.100573 0.008725 16.603431 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:20:41.051948Z", + "iopub.status.busy": "2024-03-22T18:20:41.051665Z", + "iopub.status.idle": "2024-03-22T18:20:41.424283Z", + "shell.execute_reply": "2024-03-22T18:20:41.423284Z" + }, + "papermill": { + "duration": 0.392558, + "end_time": "2024-03-22T18:20:41.426797", + "exception": false, + "start_time": "2024-03-22T18:20:41.034239", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:20:41.464627Z", + "iopub.status.busy": "2024-03-22T18:20:41.464010Z", + "iopub.status.idle": "2024-03-22T18:25:10.980556Z", + "shell.execute_reply": "2024-03-22T18:25:10.979659Z" + }, + "papermill": { + "duration": 269.538205, + "end_time": "2024-03-22T18:25:10.983174", + "exception": false, + "start_time": "2024-03-22T18:20:41.444969", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tvae/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tvae/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:11.021429Z", + "iopub.status.busy": "2024-03-22T18:25:11.021082Z", + "iopub.status.idle": "2024-03-22T18:25:11.048484Z", + "shell.execute_reply": "2024-03-22T18:25:11.047685Z" + }, + "papermill": { + "duration": 0.049077, + "end_time": "2024-03-22T18:25:11.050646", + "exception": false, + "start_time": "2024-03-22T18:25:11.001569", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:11.085900Z", + "iopub.status.busy": "2024-03-22T18:25:11.085593Z", + "iopub.status.idle": "2024-03-22T18:25:11.091285Z", + "shell.execute_reply": "2024-03-22T18:25:11.090350Z" + }, + "papermill": { + "duration": 0.025923, + "end_time": "2024-03-22T18:25:11.093469", + "exception": false, + "start_time": "2024-03-22T18:25:11.067546", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.3965440729686192}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:11.130756Z", + "iopub.status.busy": "2024-03-22T18:25:11.130471Z", + "iopub.status.idle": "2024-03-22T18:25:11.518772Z", + "shell.execute_reply": "2024-03-22T18:25:11.517773Z" + }, + "papermill": { + "duration": 0.409639, + "end_time": "2024-03-22T18:25:11.520956", + "exception": false, + "start_time": "2024-03-22T18:25:11.111317", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCs0lEQVR4nO3deXxU5b0/8M+ZfcnMZN8TEiBssq8FREBRQapQW8VqFbSot4VrLZf7sunrVxCtBVsXrNdy215J5CJitYi9ruACKMousilLSELIQhKyzGQms57n98fJnGTINjOZmZPMfN+v17wmc+bMmedkku8851m+D8cYYyCEEAnJpC4AIYRQICKESI4CESFEchSICCGSo0BECJEcBSJCiOQoEBFCJEeBiBAiOQpEhBDJUSAihEiOAhEJqa+++gpPPvkkmpqapC4KGUAoEJGQ+uqrr7Bu3ToKRCQgFIgIIZKjQERC5sknn8R//ud/AgDy8/PBcRw4jkNcXBzmzp3baX+e55GVlYWf/OQn4rbnnnsOM2bMQFJSErRaLSZNmoS33367y/fbunUrJk2aBK1Wi8TERNxzzz2oqKgIz8mRsOIoDQgJlRMnTmDDhg1444038OKLLyI5ORkAUFJSgqeeegqVlZVIT08X99+3bx9mz56Nt956SwxGOTk5uOOOOzBq1Cg4nU5s374dhw4dwnvvvYeFCxeKr33mmWfwu9/9DnfffTdmz56Nuro6vPzyy4iLi8M333yD+Pj4iJ476SNGSAj96U9/YgBYaWmpuO3s2bMMAHv55Zd99v3lL3/J4uLimM1mE7d1/JkxxpxOJxs9ejS78cYbxW1lZWVMLpezZ555xmffkydPMoVC0Wk76f/o0oyE3bBhwzB+/Hi8+eab4jaPx4O3334bt99+O7Rarbi948+NjY1obm7GrFmzcOzYMXH7jh07wPM87r77btTX14u39PR0FBQU4PPPP4/MiZGQUUhdABIblixZgt/+9reorKxEVlYW9uzZg9raWixZssRnv/feew+///3vcfz4cTgcDnE7x3Hiz+fPnwdjDAUFBV2+l1KpDM9JkLChQEQiYsmSJSgsLMRbb72Fxx9/HP/4xz9gMpkwf/58cZ8vvvgCd9xxB2644Qb85S9/QUZGBpRKJYqKirBt2zZxP57nwXEcPvzwQ8jl8k7vFRcXF5FzIqFDgYiEVMeaS0f5+fmYOnUq3nzzTaxcuRI7duzA4sWLoVarxX3++c9/QqPR4OOPP/bZXlRU5HOsIUOGgDGG/Px8DBs2LDwnQiKK2ohISOn1egDockDjkiVLcODAAWzevBn19fWdLsvkcjk4joPH4xG3lZWVYefOnT773XnnnZDL5Vi3bh3YNZ2+jDFcvXo1NCdDIoa670lIHT58GFOnTsVtt92Ge+65B0qlErfffjv0ej0uX76M3NxcxMXFQalUoqamxqc957PPPsNNN92EWbNm4d5770VtbS1eeeUVpKen48SJEz5BZ8OGDSgsLMSMGTOwePFiGAwGlJaW4p133sEjjzyC1atXS3H6JFgS9tiRKPX000+zrKwsJpPJOnXlz5w5kwFgy5cv7/K1r776KisoKGBqtZqNGDGCFRUVsbVr17Ku/lT/+c9/suuvv57p9Xqm1+vZiBEj2IoVK9jZs2fDdWokTKhGRAiRHLUREUIkR4GIECI5CkSEEMlRICKESI4CESFEchSICCGSG9BTPHieR1VVFQwGQ7dTCwgh0mGMwWKxIDMzEzJZ9/WeAR2IqqqqkJOTI3UxCCG9qKioQHZ2drfPD+hAZDAYAAgnaTQaJS4NIeRaZrMZOTk54v9qdwZ0IPJejhmNRgpEhPRjvTWdUGM1IURyFIgIIZKTNBDl5eWJS850vK1YsULKYhFCIkzSNqLDhw/7JME6deoUbr75Ztx1110hew/GGNxut8/7EP/I5XIoFAoaGkHCTtJAlJKS4vN4w4YNGDJkCGbPnh2S4zudTlRXV8Nms4XkeLFIp9MhIyMDKpVK6qKQKNZves2cTie2bt2KVatWdfsN7HA4fFZ2MJvN3R6P53mUlpZCLpcjMzMTKpWKvtkDwBiD0+lEXV0dSktLUVBQ0OOANEL6ot8Eop07d6KpqQnLli3rdp/169dj3bp1fh3P6XSC53nk5ORAp9OFqJSxRavVQqlUory8HE6nExqNRuoikSjVb77iXn31VSxYsACZmZnd7lNYWIjm5mbx5s865/Qt3jfR/Puj5KT9R7+oEZWXl+OTTz7Bjh07etxPrVb7LDNDSF/Utzjx3okqFKQaMDkvAc2tLqQZqdYnhX4RiIqKipCamoqFCxdKXRQSQ8qvWtFkc+FEZROOljfCpFVg2cx8qYsVkySvd/M8j6KiIixduhQKRb+IizElLy8PGzdulLoYkrhiFjo+RmeawDOGRpsLDjcN85CC5IHok08+waVLl/DQQw9JXRQSY2otdgDAoCQdDBrhS7DO4ujpJSRMJK+C3HLLLdRo2EdOp5PG+QTI6ebRZHMBAFIMaiTqVbDY3WiyuZCdIHHhYpDkNSIpON18tze3h/d7X5cf+wZjzpw5WLlyJVauXAmTyYTk5GT87ne/EwN2Xl4enn76aTzwwAMwGo145JFHAABffvklZs2aBa1Wi5ycHDz22GOwWq3icWtra3H77bdDq9UiPz8fr7/+elDliwZNNicAQKuSQ6dSIEGnatvukrJYMUvyGpEUXvn8QrfP5SfrsXhClvj4b/tK4PJ0XWPLTtDirsntidk27y9Fq9O3jeHXNw8LqoyvvfYafv7zn+PQoUM4cuQIHnnkEeTm5uLhhx8GADz33HNYs2YN1q5dCwAoKSnB/Pnz8fvf/x6bN29GXV2dGMyKiooAAMuWLUNVVRU+//xzKJVKPPbYY6itrQ2qfANdc6sQcOK1wpLXJp1w39TqlKxMsSwmA9FAkJOTgxdffBEcx2H48OE4efIkXnzxRTEQ3XjjjfiP//gPcf/ly5fjvvvuw+OPPw4AKCgowJ///GfMnj0bmzZtwqVLl/Dhhx/i0KFDmDJlCgBh7NbIkSMjfm79RaJehUS9UBPyBiSqEUkjJgPRirlDu31Ods0skEduGNLtvtfOGHkohF2/P/jBD3ympEyfPh3PP/+8OHl38uTJPvt/++23OHHihM/lFmNMnOpy7tw5KBQKTJo0SXx+xIgRiI+PD1mZB5KCNAMK0tqzBibFqTFxUAIS2mpGJLJiMhCpFP43jYVr377S6/U+j1taWvDoo4/iscce67Rvbm4uzp07F6miDUgmrRKzh6X0viMJi5gMRAPBwYMHfR4fOHAABQUFkMvlXe4/ceJEnDlzBkOHdl3bGzFiBNxuN44ePSpemp09exZNTU0hLTchwYjJXrOB4NKlS1i1ahXOnj2LN954Ay+//DJ+9atfdbv/E088ga+++gorV67E8ePHcf78ebz77rtYuXIlAGD48OGYP38+Hn30URw8eBBHjx7F8uXLodVqI3VK/QbPM/x1bwleP1gOu6u9c8HmdOOK2d6pw4GEHwWifuqBBx5Aa2srpk6dihUrVuBXv/qV2E3flbFjx2Lv3r04d+4cZs2ahQkTJmDNmjU+k4iLioqQmZmJ2bNn484778QjjzyC1NTUSJxOv9Lq8sDm9KDO4oBK3v4v8N6Jamw7eAmXGih/VaTRpVk/pVQqsXHjRmzatKnTc2VlZV2+ZsqUKdi1a1e3x0xPT8d7773ns+3+++/vUzkHIqvDDQDQqeSQdeidiFML/w4tbc+TyKEaEYk53kCjV/t+D1Mgkg4FIhJzrA6hDSjumkCkVwsdAa1OCkSRRpdm/dCePXukLkJUE2tEKt8/f42yLRC5qLE60qhGRGKOzdn1pZm2LRDZqNcs4igQkZijVcqRFKeCSes7ilrXVkOi7vvIo0szEnNmDE3GjKHJnbYbtQpMzkvoVFMi4Ue/cULa6FQKzCqgaR5SoEszQojkKBCRmGJ1uPHfbdM7usoMarG7cMVs95n6QcKPAhGJKVaHG61OD6wOd5cr/757vArbDl5CTbNdgtLFLskDUWVlJX72s58hKSkJWq0WY8aMwZEjR6QuFolS3jFCWlXXzaM6FXXhS0HSxurGxkbMnDkTc+fOxYcffoiUlBScP38eCQmUvZyEhxiIlF2nU9HSoEZJSBqInn32WeTk5Ig5lQEgPz+MC9wxBngkSAUqV3ZO59iNLVu24Ne//jWqqqp8VrVdvHgxDAYD/vd//zdcpYwJ3ppOd4HIO7raQYEooiQNRP/6179w66234q677sLevXuRlZWFX/7yl2Je5ms5HA44HO3rTpnN5sDe0OMCvni+L0UOzqz/ABT+Lfdz11134bHHHsO//vUv3HXXXQCE1Tfef//9HmfWE//Y2wKR9xLsWmql0Fphp4UWI0rSNqKLFy9i06ZNKCgowMcff4xf/OIXeOyxx/Daa691uf/69ethMpnEW05OTpf7DWRarRb33nuvTy1x69atyM3NxZw5c6QrWJTwXnJpuqkRqRXeGlFwS0GR4EhaI+J5HpMnT8Yf/vAHAMCECRNw6tQp/Pd//zeWLl3aaf/CwkKsWrVKfGw2mwMLRnKlUDuJNHlgCdkffvhhTJkyBZWVlcjKykJxcTGWLVvWZS8PCYxWJUdynApGbdd/+hqqEUlC0kCUkZGBUaNG+WwbOXIk/vnPf3a5v1qt9mk3CRjH+X2JJKUJEyZg3Lhx2LJlC2655RacPn0a77//vtTFigozhiRjxpDO0zu8UgxqTMlLFJcZIpEhaSCaOXMmzp4967Pt3LlzGDRokEQl6j+WL1+OjRs3orKyEvPmzYvKy9D+KNWgQapBI3UxYo6kbUS//vWvceDAAfzhD3/AhQsXsG3bNvztb3/DihUrpCxWv3Dvvffi8uXL+Pvf/46HHnpI6uIQElaSBqIpU6bgnXfewRtvvIHRo0fj6aefxsaNG3HfffdJWax+wWQy4cc//jHi4uKwePFiqYsTFRhj+Nu+Emz5uqzbKRyMMTTbXKg127ucAkLCQ/LZ9z/84Q/xwx/+UOpi9EuVlZW47777+tYuRkR2Fw+rwwOrwwOlvOvvYJeHYfP+UgDCisAqBXUQRILkgYh01tjYiD179mDPnj34y1/+InVxooa3616tlEF+7dribZRyDjKOA88Y7G5PRFfvjWUUiPqhCRMmoLGxEc8++yyGDx8udXGihjdFrK6bMUQAwHEcNEoZbE6PMJaI2q0jggJRP9TdumWkb+zihNfuAxEAqBVtgYjGEkUM1TtJzPDOM+tuVLWX93k7ja6OmKgPRNTz0TfR9Ptr7WXCq5d3vhnViCInagORUilMq7DZaB3zvvD+/ry/z4FMrRSmd8Treh417Z1vRjWiyInaNiK5XI74+HjU1tYCAHQ6Hc3VCgBjDDabDbW1tYiPj4dc3nMtYiAYnxOP8Tnxve43OEUPo0aJNCMNm4iUqA1EAJCeng4AYjAigYuPjxd/j7FiRLoRiK1TllxUByKO45CRkYHU1FS4XBIkRBvglEplVNSESP8X1YHISy6X0z8UwesHy+HhGRaOyUBSXPeXXS4PD5vDA3DotBosCY+obawmpCPGGBpanLja4oRC1vOffUldCzbvL8UnZ65EqHSEAhGJCS4Pg5sXhiL0NqBR1TYPzemhXrNIoUBEYoJ3nplCxkEp77n31Du/jBLoRw4FIhITxMGMKnmvwzi8gYhqRJFDgYjEhFY/55kBgLqtY8PppkAUKRSISEzwd3oH0D7Fw+Vh8PDRM8WlP6NARGKCQs4h2aBGQi/TOwD4JE2jWlFkxMQ4IkKGpRkwLM3g175yGYfxOfFQyDl/F+glfUSBiJAuzB2RKnURYoqkl2ZPPvkkOI7zuY0YMULKIhFCJCB5jei6667DJ598Ij5WKCQvEolC7x6vRHOrC3OHpyInUdfr/naXkCpWo5KJaUFI+Ej+X69QKGJudjeJvEarE402/yc+v3+iGpcabJg/Oh0jM4xhLBkB+kGv2fnz55GZmYnBgwfjvvvuw6VLl7rd1+FwwGw2+9wI8UdrW5IznR/jiID2LnzqNYsMSQPRtGnTUFxcjI8++gibNm1CaWkpZs2aBYvF0uX+69evh8lkEm+0DDPxB88zvxPne3nnmzkoEEWEpIFowYIFuOuuuzB27Fjceuut+OCDD9DU1IR//OMfXe5fWFiI5uZm8VZRURHhEpOByDuqmuMAjZ/tPeI0DwpEESF5G1FH8fHxGDZsGC5cuNDl82q1mlY9JQHzBiKNUg5ZNwsrXqt9vhlNfI0EyduIOmppaUFJSQkyMjKkLgqJIoFM7/BSU40ooiQNRKtXr8bevXtRVlaGr776Cj/60Y8gl8vx05/+VMpikSiUbFAjUd/79A4vb5c9tRFFhqSXZpcvX8ZPf/pTXL16FSkpKbj++utx4MABpKSkSFksEmVyEnW4/weDAnpNUpwKY7NNSDFQU0AkcGwAr6BnNpthMpnQ3NwMo5HGehDS3/j7P9qv2ogIIbGpX/WaERIOn5y5gsqmVkwfkuT3DHzGGBxuHk4PD6OGVvIIN6oRkajX1OpCg9XZnuTM1QpY64EeWiVaHG5s2lOCoi/LMIBbLwYMqhGRqNfqdANom95RcRi4+DnAe4D4HGD0TwClptNrvOOIeCas/tFbwn3SN1QjIlHPO6AxrqUMuPCJEIQAoKkC+P69LmtGKrlMTIpGY4nCjwIRiWqMMbQ6eXDMDX15W7qZ7MnA5AcBmRyoPw80lnZ6HcdxYspYCkThR4GIRDWHmwfPGFKsF6ByWwG1ARg8BzCkA5kThJ0qDnX5WjUtKxQxFIhIVGt1egDGkG09BRnHAdlTAHlbL1j2FOG+oRSwN3d6LU3ziBwKRCSqeRhDjtqKJM4sXIpljGt/UhsvNFgDQO33nV4rrvhKgSjsKBCRqJYcp8ZPcswYnWkCEgd37iFLHSnc13UORINT4jAmywSDhjqXw41+wyS6MdZe20kd1fn5pAIAuwBLtTC+SKkVn5qSlxiZMhKqEZEo19oo3GRyIGlo5+c1RkCfLASsxvLIl48AoEBEotyJk9/geEUTLnkSAUU3aUAS84X7a7rxvSlmvWlmSfhQICJRjV29iFaXBzZ9D/nNTbnCffNln80HSxuwaU8JviqpD2MJCUCBiEQz3gOFRchrziXld7+fMVO4t10FXHZxs9hr5qJes3CjQESil7kKHpcDbpkGSmMP6YfVcUJXPmOApap9Mw1ojJigAtHFixdDXQ5CQs9cCbeHwaxOh07dSyoPY5Zw31wpbqJxRJETVCAaOnQo5s6di61bt8Jut/f+AkIkwJoq4PLwsKjTek+c7w1Elmpxk4rmmkVMUIHo2LFjGDt2LFatWoX09HQ8+uijOHSo6/k6hEiCMXiaLoNnEAJRbwsrGtKE+5Yr4iZa2yxyggpE48ePx0svvYSqqips3rwZ1dXVuP766zF69Gi88MILqKurC3U5CQlMayN4pw1ajRoKY0bv+YT0qcIKjI4WwGkF0HFtMwpE4danxmqFQoE777wTb731Fp599llcuHABq1evRk5ODh544AFUV1f3fpA2GzZsAMdxePzxx/tSJEIEzZehksswfuQIPHhDATiul0CkUAHaBOHntlqRTiXH8HQDhqXFhbmwpE+B6MiRI/jlL3+JjIwMvPDCC1i9ejVKSkqwe/duVFVVYdGiRX4d5/Dhw/jrX/+KsWPH9qU4hLQztzU6m7L8f01cqnDfUgsA0KkUuG1MBm4ckRbiwpFrBRWIXnjhBYwZMwYzZsxAVVUVtmzZgvLycvz+979Hfn4+Zs2aheLiYhw7dqzXY7W0tOC+++7D3//+dyQkJPS4r8PhgNls9rkR0iVvo7MxkEDUuZ2IREZQgWjTpk249957UV5ejp07d+KHP/whZDLfQ6WmpuLVV1/t9VgrVqzAwoULMW/evF73Xb9+PUwmk3jLyelhtCyJXbwHsNajqrkV207bcbiswb/XeQORpT0Qead58Dwl0A+noGbf7969G7m5uZ2CD2MMFRUVyM3NhUqlwtKlS3s8zvbt23Hs2DEcPnzYr/ctLCzEqlWrxMdms5mCEenMWg/wHrTySlxxapDnb6+Xvm2F4dZGIZjJ5Ni8vxQWuxv3TctFqrFzkn0SGkEFoiFDhqC6uhqpqak+2xsaGpCfnw+Pp/dJghUVFfjVr36F3bt3Q6Px7wNWq9VQq2kJYNKLtksrizIR4Ljeu+691Aah0drtBGwNQFyKmLeaBjWGV1CXZt2t89TS0uJ3UDl69Chqa2sxceJEKBQKKBQK7N27F3/+85+hUCj8CmaEdKmtsdksE/IJ6dV+ft9yHKBLFn62CRNdaZpHZARUI/JeFnEchzVr1kCn04nPeTweHDx4EOPHj/frWDfddBNOnjzps+3BBx/EiBEj8MQTT0Au9/NbjJBrtdQAAJrk8QDa1jPzly4JMFcJE2BBE18jJaBA9M033wAQakQnT56EStWe30WlUmHcuHFYvXq1X8cyGAwYPXq0zza9Xo+kpKRO2wnxG2PipVk911YjUgXwZ65vqxFZhRoRDWqMjIAC0eeffw5AqLm89NJLMBqNYSkUIUGzNwFuJzyQwcwJf59+txEBnS7NaL5ZZATVWF1UVBTqcgAA9uzZE5bjkhjS1j7k1iYhRaeDw8WL7Tx+0bXlqbY1AjxP880ixO9AdOedd6K4uBhGoxF33nlnj/vu2LGjzwUjJCgWoX1IHZ+B+0YMCvz1mnhApgB4N2BvQqpBg+HpBiQbukkzS0LC70BkMpnE+TomkylsBSKkT9pqRIhLD+71MplQK2qpBWxXMSqzAKMyqQki3PwORB0vx8J1aUZIn3mnZ8Sl9rxfT3RJYiACCkJSLNKzoMYRtba2wmaziY/Ly8uxceNG7Nq1K2QFIyRgTivgsAAAjl5VoHh/KY74O72jI287UWsjAGGaB7URhVdQgWjRokXYsmULAKCpqQlTp07F888/j0WLFmHTpk0hLSAhfvPWhrQJaHbJ0GhzBdft7k0HYmtAWb0VL316Hm8drQhdOUknQWdonDVrFgDg7bffRnp6OsrLy7Flyxb8+c9/DmkBCfGbt33IkAarQxiZrwtkDJGXtr1GpKRes4gIKhDZbDYYDAYAwK5du3DnnXdCJpPhBz/4AcrLabVMIhGxfSgNVodb+FEdxAh9b43IYYEKQkCjQBReQSfP37lzJyoqKvDxxx/jlltuAQDU1tbSIEciHbHHLA0tYiDqZfWOrqh0gFKYM6l2NwOgQBRuQQWiNWvWYPXq1cjLy8O0adMwffp0AELtaMKECSEtICF+8bjE+WG8LkW8NIvTBDVmV6wVqV1CIHLzDG6a5hE2QX1KP/nJT3D99dejuroa48aNE7ffdNNN+NGPfhSywhHit5ZaYZ6ZSg+bTAeeMcg4DrrelhHqjjYBMFdD6WgCIEz7cHkYFDQXOyyC/LoA0tPTkZ7uO2hs6tSpfS4QIUHp0D7k9vBIM2rAwCCT9ZI0vzttDdYyexNUilQ43Twcbk9g89aI34IKRFarFRs2bMCnn36K2tpa8LxvlZVWgiURJ7YPpSJep8K903L7djxvg3VrAwYn68EzBB/USK+CCkTLly/H3r17cf/99yMjI6P3pVoICbe2HERi3um+6jCoccGMjNAck3QrqED04Ycf4v3338fMmTNDXR5CAsfzQEvbop6GIOeYXUvswm8RUscqaNJrOAXVa5aQkIDExMRQl4WQ4LQ2CLPl5UpAm4BPzlxB0f5SfF/Th+WmlFrhBgCtjWCM0UoeYRRUIHr66aexZs0an/lmhEim40RXjkNTqwtNNhe6Sa3uv7Za0RcnzmLjJ+dxsrK5jwck3Qnq0uz5559HSUkJ0tLSkJeXB6XSd9CYPwsrEhIyFt/2ofZR1UF3Cgt0iYC5CiqXGUA6pYsNo6A+qcWLF4e4GIT0QYcR1YyxDqOq+xiI2mpEOo9wiUejq8MnqE9q7dq1oS4HIcHpkCwfcWlwuHkxYPi9jFB32gKR1m0GZBSIwimoNiJASP/xP//zPygsLERDg5Dz5dixY6isrPT7GJs2bcLYsWNhNBphNBoxffp0fPjhh8EWicQihxlwtQKcDNCniJdlaqVMzDcdtGumedAii+ET1FfGiRMnMG/ePJhMJpSVleHhhx9GYmIiduzYgUuXLom5inqTnZ2NDRs2oKCgAIwxvPbaa1i0aBG++eYbXHfddcEUjcQa72WZPgmQK2C2OwAABk0Qk12v1RaIVB4bZHyQuY2IX4L6yli1ahWWLVuG8+fP+6zsetttt2Hfvn1+H+f222/HbbfdhoKCAgwbNgzPPPMM4uLicODAgWCKRWKR2FAtjB+ScUC6SYNUQwiWJldqAaUGchkHjdtMl2ZhFFSN6PDhw/jrX//aaXtWVhZqamqCKojH48Fbb70Fq9Uqzua/lsPhgMPhEB+bzX0YJ0KiQ4f2IQAYlKTHoCR96I6vTYDaYsFgnRNao3/LqZPABRWI1Gp1l0Hg3LlzSElJCehYJ0+exPTp02G32xEXF4d33nkHo0aN6nLf9evXY926dcEUmUQrb43IEKKpHdfSJsCgqcbN+WogNzk870GCuzS744478NRTT8HlcgEAOI7DpUuX8MQTT+DHP/5xQMcaPnw4jh8/joMHD+IXv/gFli5dijNnznS5b2FhIZqbm8VbRQXlEY5pHZLle2tErM+jGK8hTn5tDO1xiY+gAtHzzz+PlpYWpKSkoLW1FbNnz8bQoUNhMBjwzDPPBHQslUqFoUOHYtKkSVi/fj3GjRuHl156qct91Wq12MPmvZEY5r0s0yUCCqFNaPP+MhTtL0Wj1Rma9+gQiEIe5IgoqEszk8mE3bt3Y//+/fj222/R0tKCiRMnYt68eX0uEM/zPu1AhHTL4ts+5PbwMLcKtXRNsAnRrqVNgJvnceDkeRypP49/v7EAckoHEnIBByKe51FcXIwdO3agrKwMHMchPz8f6enpYIwFlBKksLAQCxYsQG5uLiwWC7Zt24Y9e/bg448/DrRYJBZd01BtsQtjiFQKGTTKPo4h8tImQMZxkDst4DxuON08JUcLg4ACEWMMd9xxBz744AOMGzcOY8aMAWMM3333HZYtW4YdO3Zg586dfh+vtrYWDzzwAKqrq2EymTB27Fh8/PHHuPnmmwM9DxKLvIGoraHabBdqQ0aNInQ5spQ6yJRqyDlA7bFQIAqTgAJRcXEx9u3bh08//RRz5871ee6zzz7D4sWLsWXLFjzwwAN+He/VV18N5O0Jaed2ALa2VVzbxhCZW4UakVEbgsGMXhwHaBMgl9VA42qGw+MBEMLjEwABNla/8cYb+O1vf9spCAHAjTfeiN/85jd4/fXXQ1Y4QrplqRbuNUZh+R+014gMwa7c0R1tAg1qDLOAAtGJEycwf/78bp9fsGABvv322z4XipBemauEe2OmuKnRJvSUmbQhzqaoTYBcJqNAFEYBBaKGhgakpXU/cCwtLQ2NjTTegkSANxAZ2gNRgk6FNKMGyXHhCERtNSKabxYWAdVhPR4PFIruXyKXy+F2u/tcKEJ6xFiXNaKZQ5Mxc2gY3k+bgDi1AhmsFdpQDQsgPgLuNVu2bBnU6q4nFNL4HxIRDrMwqpqThS5Zfk+0CchN1CGX44AEmm8WDgEFoqVLl/a6j789ZoQEzVsbiksREuYD8PAMHMK09pgqDpArAI8bsDe3LzVEQiagQFRUVBSuchDiP/GyLEvc9F21GZ99X4uRGUbcPCrEE2DbuvDRUifMOaNAFHIhGn5KSASJDdXtCx822pzw8AyKME2/KLepcbisAQdPXwjL8WMdBSIysHhc7WOITNni5iabMIYoXheewYYeTQLcPKNZ+GFCgYgMLOZKgPcA6rj2mfEAmtrGEMXrwrMiK6cT3ouzN4Xl+LGOAhEZWJouCffxuULbDQCeZ2hsqxElhikQydvahWR2qhGFAwUiMrA0tSXDi89t39TqgodnUMo5GLUhnt7RRq4XApHc0QzwNKgx1CgQkYHD425vqDa1B6KrLcL4taQ4dehm3V9DpTOB5+Tw8B7AQUtPhxoFIjJwmCsB3g2o9D5d6BqlHMPSDBiUpAvbW2tUctgVRrg9DMxGl2ehFp56LBnYGBP+4eX9LN1Fw0XhPiFPbB8CgJxEHXISwxeEACHYaYzJMNla4LE1QJE0OKzvF2soEBFflivAmZ1Crp/EfGDk7UINpD9oKBHuk4ZE/K2Vchlmjh4KVDQAjqaIv3+0o0sz0s5pA068KSYca6k+j8v7tsDc2g/mENqbhZHNHAck5IubPTxDo9UZmcT2YiL9pvC/V4yhQETalX8lTCbVJ+O79NvxTbUdFeUl2LXrA1Q1tUpbNu9lmTFTTIQGAPUtDhR/VYbir8rCXwZaWihsKBARgasVqP4GAGDJmY1PqtS4FD8VSjmHtIYj+OjbS9ImBas7K9wn+eb5uGK2AwBMoUwP242PLjpwqKwBV+trqAs/xCQNROvXr8eUKVNgMBiQmpqKxYsX4+zZs1IWKXZdOSN0j8el4lCjEW6eQZU9HuOH5SNO5sJgdwnsbo80ZXNagcZy4eeUET5PXTELl41pEVgO2iHXw804eNxuwGkJ+/vFEkkD0d69e7FixQocOHAAu3fvhsvlwi233AKr1SplsWJT3fcAAD51NM7VCr//HwxJgSJ3GsZkmzDHUAmjWqK+jbrvAcYLuYeumfnurRGlGbvOkRVKaqVS7MKny7PQkrTX7KOPPvJ5XFxcjNTUVBw9ehQ33HCDRKWKQY4WoFkYsSxLG4H70/UoqW1BdoIWMFwHRclngLVemGzaISNixFw5LdynXeez2e3hcbVFmGOWGoEakUYpg01hhJtvSweSkBf294wV/aqNqLlZGLGamNh1vheHwwGz2exzIyFw9YIwdsiYAWhMiFMrMC4nXhilrNQAKcMAALZL3+BMVeh/51fMduw5W4v6li565yxXgOZKIRtj6kifp+paHOAZg1YlhyECtTW1om1QI8+3L2VEQqLfBCKe5/H4449j5syZGD16dJf7rF+/HiaTSbzl5OREuJRRqqmt/SWxm0F66WPg8vA4duQr7D5dCUvbsj2h8F21GdsPVeCbS01wdZWYvuqYcJ8yDFAbfJ6qbBR68jJMmrBN7ehIo5TBrjDRpVkY9JtAtGLFCpw6dQrbt2/vdp/CwkI0NzeLt4qKigiWMEoxJjYEN6sz8e7xSnxb0eS7T3welFoD4pUeGFqrUFIXmja8Wosdu89cAc8YBqfoO/d82ZuBmpPCz1mTOr0+J1GHqfmJGJlhDEl5eqNRemtEFIhCrV+MrF65ciXee+897Nu3D9nZ2d3up1aru03cT4JkaxB6pWQKlLnicbGuAS4Pw7ic+PZ9ZDIgZTgS6+qRZLuI81eGYXzH54PAGMOes3Xw8EIQumNcplirudxoQ0VDK6a7Dwm5h+JzAVPn2m+aUROR3jKvOLUCpsRUaB1ywN4kBPEI1MRigaQ1IsYYVq5ciXfeeQefffYZ8vPze38RCa2mMuHelIUqs3DJlZ2g7bxfynAk6VVIbC1DVWMLrI6+LRtVftWGysZWKGQc5o5IFYNQc6sLbx+9jFPfnYG59Iiwc/4N/eIfPidRhx/PGIX8FIMw1MFBXfihImkgWrFiBbZu3Ypt27bBYDCgpqYGNTU1aG2VeBRvLOmQ38fbWJxq6KLWacqFWue9PKtGSV1Ln972ZKXQMTE62wSjpv2SzKRVYmyaBkOv7kF5vRUsbRQQ37k2VFLXgot1LXBHesFDmRzQmISfW6nBOlQkDUSbNm1Cc3Mz5syZg4yMDPH25ptvSlms2NKW/9ljyEKDVagRJXcViGQyIHkYkvQqJLVexPkrwQcih9uDSw02AMCYLJPvk24HZjr2QM+b0eBW4ZzxB10e4+uSq3j3eBXOXpGgVqJLEu5tVyP/3lFK0jaiiExUJN1zWsUJnA2yRPDsCtRKWfdd4SkjkKg/isSmMpxotsHp5qFSBP5dplbI8dDMfJRdtSI5rkPQaywHzu+C2lqPjEQjPlDchPOlVgzJ4KGQt79Pk82JOosDHAcMTo4L+P2DxfMMRV+VIfmKA7eZeCitFIhCpV80VhOJWGqEe10S6u1CG0yyvocsh/GDoNHFYXSqB1PHKIIKQl5alby9t6u1ESj5vH0+mUqP9BvuhPyUExa7G8crmjA5r31s2alKYSzToCQdtKrILQEtk3GwuzywyExCalpbfcTeO9r1m+57IgFxfbB0tLo8kMs4JBt6SD7fdnmWoFNB3RDcnEAP36EW7HEDpV8Ah/4uBCGOA7ImAlN+DmVCNmYMSQYAHCxtQKtTmOfmcHtwqkpoX+p0WRcBaoUMNmUCXDxPl2YhRDWiWOZdH8yYiYnZCRifHQ9nb42/aaOA6m+F+V8Ft4BxsoAGE350qgZWhxs35CqRfuk9IccQIEyXGDpPWEa6zcgMAy7UtaAgNQ4apfCdeeCiEJQSdErkR/CyzEunUqBeGQ+3mwlTY1ytgLKLXkYSEApEsYqx9kDUtmKqTMZBI+vlUseUC6jjUFtfjwOff4GhIydiVKZ/AwptTjdK6lqgcjQgzvIlwDmF3EIFtwiz6q8JaBzH4Y5x7XPbaprtOFYuDCScNSwF8jCt6toTrUoGj0wFu0wPwCXUikzdj30j/qFLs1hlbxYyMnIyIC6AteJlwpwvh5uHrPY7XAigG/9MlRlypwVTLbsRxzmF2s+kZcIcMj9qVYl6FWQch5lDkzEkJfK1IQDQKoXv7lZVvLDBSu1EoUCBKFZ5a0Nxqai0uLD1QDm+OF/n32tTr0OiXoWE1nJU1DbB4UeeIsYYTldcxfD6XcjWuoQu8HH3to/J8YNKIcOymXmYmt/1pOhI0LU1jlvl8cIGarAOCQpEsUpsqM5ArdmOOosDDVanf681pENnSoFewcPUUoLS+t7nnl1ubIWh+isY3VeRGG8Cxt7tk/LVX5HIxNiTBJ0K6SYNFIa2tiyahR8SFIhilbfr3pgh5vTxGdPTE44DlzEWSXo10lrO+JUa5ML33yLDchLJcSooRt0OaOODLLi0xmSb8NOpuRg5pG06El2ahQQFoljk01CdKU7t8DsQAUDGOKSYdIhz1qG+qgzNtu5Tg7RYmqA49wEAIGn4TCC5IOii9xs6YWgB7M2A28+aJOkWBaJYZGsAPC5ArgDTJeKq1Vsj6mEM0bVUemiyRiNeq0Sm5aQ4tqcTxqC7uAujkuVITMuC6bpbQnAC/YBK135pSeOJ+oy672OR2FCdBrOdh9PNQy7jkKALIBABQPZUpJUcBddSgSS9ret9qr6BrKEESQYdkibdB8gH9p9ci8ONNw9XwO3h8YgxCZzTBljrhOyWJGhUI4pF3vYhQwbq2i7LEvUqyAIdl2NIQ0L+eIxIMyDlypfCJV9H5ip4zn8i/Dx4DhCX2rdy9wMquQzmVhdsTg/c2rYGa2uttIWKAhSIYpHYPpQOxhgS9aquU3/4gcu/QajlNJYDl75un8JhrUfrsTfxTVkdyrkMsKzJISq8tJRyDkq5ELDtqrZZ+C0UiPpqYNeTSeB4HmhprxEV6A0oSDMEnwlBlwgMuQnu7z/EpUPv4/Lxs/jBqDyg8gjOVdSjWZaAy/obkNsPEpuFAsdx0CjlcHncaFUnwwAIgYiyNfYJBaJYY7sqTDaVKwFt+8DAPiWfz5oIu6UZDRUfQGk+jW+/PAMAaFSmoSJ7Pu4akxOR5PaRolMpYLG70aKIRyonE+abOSyAJjK5s6MRBaJY0/GyjOMAxkISJOJGzEWeIRffHv4CVrsDzZpssNSRWDw2yycDYzTwjq62uTmhRmitF2pFFIiCRoEo1rRcEe4N6ai1OPD20cvITtBi0fisPh86OWsI5mQMxhWzHRwHpBk0gTeADwDiNA+HsEQ3rPVCg3XyUIlLNnBRY3Ws6TDjvr7FAaebh8sTukyZchmHzHgtMkzaqAxCAJAUp0a6SSMkZfNOGPYGeBIUqhHFEp7vUCPKQP3lIAYyEkwalIBJgxKEB1fbuvBb/JwwTLokaY1o3759uP3225GZKaxptXPnTimLE/2stUJDtUIFaBNQbwliagfx5a0RtTbQVI8+kDQQWa1WjBs3Dq+88oqUxYgdzZXCvTEb4Ljg5pgREWMMUMcJS2F3nL9HAibppdmCBQuwYMECKYsQW8yXhXtTFqwON2xODzgOSKJLs4DYXR5sO3gJrS4PfjF7CGTGDKDOIgSihEFSF29AGlBtRA6HAw6HQ3xsNveefoJ0INaIssTUHyatEko59VkEQiWXwWJ3g2cMVqcbBmMWUHeuPccTCdiA+gtcv349TCaTeMvJ6bwCKOmG3SykrOA4wJgFuZzD4BQ9BiUFnpws1slkHPRqbxe+R8z5TZdmwRtQgaiwsBDNzc3iraKiQuoiDRzmttpQXCqgUCErXhg7dOOIAPJVE5G+bRHKFodbCEQcJwR7hwQrz0aBAXVpplaroVZTw2pQmtvah4y04kQo+AQiRRygTxa68M3VQIpB4tINPAOqRkT6oLFMuI/PBc8zYVQwCZpRIwQic2tbZkpD27JHFmonCoakNaKWlhZcuHBBfFxaWorjx48jMTERubm5EpYsytjNwjQEjgMSBuGq1YmtB8qRoFNi2cx8qUs3IHmT+Dd7A5ExQ1h40tshQAIiaSA6cuQI5s6dKz5etWoVAGDp0qUoLi6WqFRRqLFUuDdkAEot6uqE3kadekBdmfcrSXphmoeY1dLU9sVprmrLbkC/20BI+tuaM2dO8HlwiP+8l2UJeQAgZmVMCTIZGgFyk3TITepQa9clAio94LQKHQM0nigg1EYU7XgP0HBR+DlRuAyra5vakUIjqkOn7bIXANB0SdqyDEAUiKJdUzngsgsrThizwRgTA1Gw6WFJO55n7elx49tqSBSIAkaBKNrVnRXuk4cBMhksDjfsLg9kHIdEPU3t6IsPTlbjvz6/gJK6FmFDfFuNyFwpLNdE/EaBKJp53O2BKGU4AKCm2Q4ASDaooKCpHX0i4zh4eIYm7+KS2gRhAizvAZppsG0g6C8xmtWfE/Ipq+OA+DwAQLxWiQm58RiRTmlN+8pbo2ywts1/5DggcbDw89USiUo1MFEfYzSrPi7cZ4wDZMJ3TqpRg1SjRroyRRFv1oL6lg55iJILhPFE9eeBofNoZQ8/UY0oWpmrhLXGOA5IHyt1aaJSUluNqNHqBO9tsE7IA2QKYYKxlbI2+osCUbQq2y/cp10HaOMBABa7CxUNNjjdvHTliiJCChUObp61j7CWK8XxWrh6odvXEl8UiKJRw0Xhn4DjgEEzxc1nayx4++hlfHiK0lWEAsdxSNQLQyCuWtvzZImredR+J0GpBiYKRNHG7QTO7RJ+zpokjPhtc6nBBgDITaQcRKGSl6TD8HQDtKoOza3JwwGZXFjrjJLq+4Uaq6MJY8DZD4DWRqEbOf8G8Sm3h0dlYysACkShNGNocueNKp3Qe1Z/HrhyEoi7MfIFG2CoRhRNyr4ULgc4GTBqEaBoHzld1WSHm2eIUytoIGMkpI8R7q+cFsYVkR5RIIoGjAHlXwmBCAAK5gHxvml0vaN/ByXpomod+v6AMYb6FodvjqekocIkWEcLUPe9dIUbICgQDXQ8D1z4BLi4V3g8eLbQNuSzC8P5WiGFaUEaZQ8MtY9PX8H/fl2OM9UdFnOQyds/h4qDwpcF6RYFooHM4wLOvANcPiI8HnIjMGhGp92Eb2sP1EoZtQ+FQYZJGCBaWm/1fSJzgpCXyHKlPQMC6RIFooHK0QJ8+4awjI1MDly3GMid1uWuqUYNHpg+CHOHp0IepevRSykvSQ8AqG6yo9XZoT1IpROCEQCUfCbUXkmXKBANROYq4GixkJZUqQHG3QOkjuzxJUlxaozMoPll4WDSKZFiUINnDN/XXLPW3qCZwmdkrQcqj0pTwAGAAtFAwpgwj+mbrcKyNfpkYOLS9jw4XfD5hiZhMzrLBAA4Vdnsm3VUqQXyZws/X9wjjC0inVAgGihcrcCZncD3HwjdwckFwMQHfAYsXqvJ5sSrX17Ex6dr4PbQZUE4jUg3QCnnUN/ixIXaFt8nMycIvWi8Gzj5trCYAfHRLwLRK6+8gry8PGg0GkybNg2HDh2Sukj9B2NA9Qng8P8Atd8LY4QGzwZG/9hnnNC1XB4eH56qgcvD0GJ3U9tQmGmUckwclAAAuNzU6vskxwEjbhPyFdmbgePbhEs1IpI8EL355ptYtWoV1q5di2PHjmHcuHG49dZbUVsb41VYj0sIQEdeBb5/X2ic1iUBE+8XesZ6GAvU6vTgX8erUNNsh1opw83XpdHYoQiYNCgBN45IxZxhKZ2fVOmFtjyNSRj5frRImJjsdnTeNwZxTOJlNKZNm4YpU6bgv/7rvwAAPM8jJycH//7v/47f/OY3Pb7WbDbDZDKhubkZRuMAbYhlDPA4hdUfWhuF1BHNl4UlgDxtA+QUKqHRM2tyj8vU1FkcOH/FghOVzWh1eqCUc1g8IQvZCdRlL4VmmwsX6izITdQjUa8SaqVOK3DmX+0rq8gVQFKB0M6nTxEyJSj1Yv6ogc7f/1FJ55o5nU4cPXoUhYWF4jaZTIZ58+bh66+/7rS/w+GAw9H+DWI2+3mt3VAKlHza9aAyn22s+20h3LeswYoGiwMMgIx3gWOe9r2Y0PCplMsAjQknMRhHHDlwl2mBsktgYOKhGYCfTs0VF/s7cbkJJy43AxCSds0fnY5UAyVBk8oXF+pw/koLAOEyTKWQQa2QQc5NQ4IjCQtNF6G0NwK136Hi7FE0WNsTrPEyBXiZEoxTgIHDuJx4qBQKABzKG2yosTgACLXcBL0Kw1LjIn+CY+4GNKGpAEgaiOrr6+HxeJCWluazPS0tDd9/33lY/Pr167Fu3brA38jt6FezoD12Gxx23yo5zyngUMTBpkrE8LyxUKYVAHGpqD9bh6aKJgBdLxHdsUKbHKdGQVochqUZMDQlDjJqF5IMYwy5iTrYXTyumO1wunnxBgBN8jzwk28EbDVAQymabKfRaKmB2m2F8BXjAdDhb6QVQFuOcd5qBW+xt78XrwJ017RLRQILXQfIgJp9X1hYKK4GCwg1opycnB5e0SY+R7g+Bzq0rXT4J/VpP+G63xaifROsDmicHoDjwGRKQKkFp1CJe6mNGqAtiEzKS8B1mUbxEFzbcThO2NegUYrvMC4nHuNy4judPok8juMwNjseY7PjwRhDq8sDh4uHw82DZwwMgFIuB0zZgCkbmSlTYXJ4AN4DzuMAPE5wHmdbbZlBZlC1/U0wJNicUNnbVwnRKGWAFBOZVfqQHUrSQJScnAy5XI4rV674bL9y5QrS09M77a9Wq6FWB7EWl0ovLi7YHyQG8PkZNUoYOwQbMvBwHAedSgFdD7EiKU6NJPHqquf5gPHxQHyIytZfSNoiplKpMGnSJHz66afiNp7n8emnn2L69OkSlowQEkmSX5qtWrUKS5cuxeTJkzF16lRs3LgRVqsVDz74oNRFI4REiOSBaMmSJairq8OaNWtQU1OD8ePH46OPPurUgE0IiV6SjyPqi6gYR0RIFPP3fzQ6Rk0RQgY0CkSEEMlRICKESE7yxuq+8DZv+T3VgxASUd7/zd6aogd0ILJYhITwfo2uJoRIxmKxwGQydfv8gO4143keVVVVMBgM/TLNhXcKSkVFRVT26kXz+UXzuQGROz/GGCwWCzIzMyHrIaPAgK4RyWQyZGdnS12MXhmNxqj8Y/aK5vOL5nMDInN+PdWEvKixmhAiOQpEhBDJUSAKI7VajbVr1waXMWAAiObzi+ZzA/rf+Q3oxmpCSHSgGhEhRHIUiAghkqNARAiRHAUiQojkKBAFKNBVad966y2MGDECGo0GY8aMwQcffODzPGMMa9asQUZGBrRaLebNm4fz58+H8xS6Fci5/f3vf8esWbOQkJCAhIQEzJs3r9P+y5YtA8dxPrf58+eH+zS6Fcj5FRcXdyq7RuO7NNNA/ezmzJnT6dw4jsPChQvFfSL+2THit+3btzOVSsU2b97MTp8+zR5++GEWHx/Prly50uX++/fvZ3K5nP3xj39kZ86cYf/v//0/plQq2cmTJ8V9NmzYwEwmE9u5cyf79ttv2R133MHy8/NZa2trpE6LMRb4ud17773slVdeYd988w377rvv2LJly5jJZGKXL18W91m6dCmbP38+q66uFm8NDQ2ROiUfgZ5fUVERMxqNPmWvqanx2WegfnZXr171Oa9Tp04xuVzOioqKxH0i/dlRIArA1KlT2YoVK8THHo+HZWZmsvXr13e5/913380WLlzos23atGns0UcfZYwxxvM8S09PZ3/605/E55uampharWZvvPFGGM6ge4Ge27XcbjczGAzstddeE7ctXbqULVq0KNRFDUqg51dUVMRMJlO3x4umz+7FF19kBoOBtbS0iNsi/dnRpZmfvKvSzps3T9zW06q0APD111/77A8At956q7h/aWkpampqfPYxmUyYNm1at8cMh2DO7Vo2mw0ulwuJiYk+2/fs2YPU1FQMHz4cv/jFL3D16tWQlt0fwZ5fS0sLBg0ahJycHCxatAinT58Wn4umz+7VV1/FPffcA73ed52rSH52FIj81NOqtDU1NV2+pqampsf9vfeBHDMcgjm3az3xxBPIzMz0+YeYP38+tmzZgk8//RTPPvss9u7diwULFsDj8YS0/L0J5vyGDx+OzZs3491338XWrVvB8zxmzJiBy5cvA4iez+7QoUM4deoUli9f7rM90p/dgJ59T/qHDRs2YPv27dizZ49Pg+4999wj/jxmzBiMHTsWQ4YMwZ49e3DTTTdJUVS/TZ8+3WdtvRkzZmDkyJH461//iqefflrCkoXWq6++ijFjxmDq1Kk+2yP92VGNyE+BrkoLAOnp6T3u770P5JjhEMy5eT333HPYsGEDdu3ahbFjx/a47+DBg5GcnIwLFy70ucyB6Mv5eSmVSkyYMEEsezR8dlarFdu3b8fPf/7zXt8n3J8dBSI/BbMq7fTp0332B4Ddu3eL++fn5yM9Pd1nH7PZjIMHD0Z0pdtgV9z94x//iKeffhofffQRJk+e3Ov7XL58GVevXkVGRkZIyu2vUKwo7PF4cPLkSbHsA/2zA4ShJQ6HAz/72c96fZ+wf3YRaxaPAtu3b2dqtZoVFxezM2fOsEceeYTFx8eL3br3338/+81vfiPuv3//fqZQKNhzzz3HvvvuO7Z27douu+/j4+PZu+++y06cOMEWLVokWRdwIOe2YcMGplKp2Ntvv+3TxWuxWBhjjFksFrZ69Wr29ddfs9LSUvbJJ5+wiRMnsoKCAma32yN6bsGc37p169jHH3/MSkpK2NGjR9k999zDNBoNO336tLjPQP3svK6//nq2ZMmSTtul+OwoEAXo5ZdfZrm5uUylUrGpU6eyAwcOiM/Nnj2bLV261Gf/f/zjH2zYsGFMpVKx6667jr3//vs+z/M8z373u9+xtLQ0plar2U033cTOnj0biVPpJJBzGzRoEAPQ6bZ27VrGGGM2m43dcsstLCUlhSmVSjZo0CD28MMPdxqLE0mBnN/jjz8u7puWlsZuu+02duzYMZ/jDdTPjjHGvv/+ewaA7dq1q9OxpPjsKA0IIURy1EZECJEcBSJCiOQoEBFCJEeBiBAiOQpEhBDJUSAihEiOAhEhRHIUiAghkqNARAaU4uJixMfHi4+ffPJJjB8/Xny8bNkyLF68OOLlIn1DgYh0yZuz+N/+7d86PbdixQpwHIdly5b57B/qAJCXl4eNGzf6bFuyZAnOnTvX7WteeuklFBcXi4/nzJmDxx9/PKTlIqFHgYh0KycnB9u3b0dra6u4zW63Y9u2bcjNzZWkTFqtFqmpqd0+bzKZfGpMZGCgQES6NXHiROTk5GDHjh3ith07diA3NxcTJkzo07G7qqksXrxYrGXNmTMH5eXl+PWvfy2uIgF0vjS7Vsea2bJly7B371689NJL4jFKS0sxdOhQPPfccz6vO378ODiOi3iuJCKgQER69NBDD6GoqEh8vHnzZjz44INhf98dO3YgOzsbTz31FKqrq1FdXR3wMV566SVMnz4dDz/8sHiM3NzcTucEAEVFRbjhhhswdOjQUJ0CCQAFItKjn/3sZ/jyyy9RXl6O8vJy7N+/369EWn2VmJgIuVwOg8GA9PT0oLIemkwmqFQq6HQ68RhyuRzLli3D2bNnxbW/XC4Xtm3bhoceeijUp0H8RDmrSY9SUlKwcOFCFBcXgzGGhQsXIjk5Wepi9UlmZiYWLlyIzZs3Y+rUqfi///s/OBwO3HXXXVIXLWZRjYj06qGHHkJxcTFee+21kNUaZDIZrk2F5XK5QnJsfyxfvlxsiC8qKsKSJUug0+ki9v7EFwUi0qv58+fD6XTC5XLh1ltvDckxU1JSfNp9PB4PTp065bOPSqXq8/I13R3jtttug16vx6ZNm/DRRx/RZZnE6NKM9Eoul+O7774Tf+5Oc3Mzjh8/7rMtKSkJOTk5nfa98cYbsWrVKrz//vsYMmQIXnjhBTQ1Nfnsk5eXh3379uGee+6BWq0O6pIwLy8PBw8eRFlZGeLi4pCYmAiZTCa2FRUWFqKgoCCiCe9JZ1QjIn4xGo0wGo097rNnzx5MmDDB57Zu3bou933ooYewdOlSPPDAA5g9ezYGDx6MuXPn+uzz1FNPoaysDEOGDEFKSkpQ5V69ejXkcjlGjRqFlJQUXLp0SXzu5z//OZxOZ0R6AUnPKGc1iVlffPEFbrrpJlRUVHRaKZVEFgUiEnMcDgfq6uqwdOlSpKen4/XXX5e6SDGPLs1IzHnjjTcwaNAgNDU14Y9//KPUxSGgGhEhpB+gGhEhRHIUiAghkqNARAiRHAUiQojkKBARQiRHgYgQIjkKRIQQyVEgIoRI7v8DlGOIOBpiV+QAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:11.558170Z", + "iopub.status.busy": "2024-03-22T18:25:11.557829Z", + "iopub.status.idle": "2024-03-22T18:25:11.910037Z", + "shell.execute_reply": "2024-03-22T18:25:11.908917Z" + }, + "papermill": { + "duration": 0.37333, + "end_time": "2024-03-22T18:25:11.912313", + "exception": false, + "start_time": "2024-03-22T18:25:11.538983", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBQUlEQVR4nO3deXxU5b0/8M+ZfSaZmewbSUiAsMkuQgFRQFSQKtTWlSq49ireXsqPV5V7W6y2NtiKlVYvta2QchGpWqpttSKoYAVFFtn3kIRAdrLMvp7n98eZOcmQbWYyMyeZ+b5fr3lN5syZM8/JJN95zrN8H44xxkAIIRKSSV0AQgihQEQIkRwFIkKI5CgQEUIkR4GIECI5CkSEEMlRICKESI4CESFEcgqpC9AXPM+jpqYGer0eHMdJXRxCyFUYYzCbzcjLy4NM1n29Z0AHopqaGhQUFEhdDEJIL6qrq5Gfn9/t8wM6EOn1egDCSRoMBolLQwi5mslkQkFBgfi/2p0BHYj8l2MGg4ECESH9WG9NJ9RYTQiRHAUiQojkJA1ERUVF4Diu023ZsmVSFosQEmOSthHt378fXq9XfHz8+HHcfPPNuOuuuyL2HowxeDyegPchkSeXy6FQKGgYBQmLpIEoMzMz4PGaNWswdOhQ3HjjjRE5vsvlQm1tLWw2W0SOR3qm0+mQm5sLlUoldVHIANNves1cLhc2b96MFStWdPut6nQ64XQ6xccmk6nb4/E8j4qKCsjlcuTl5UGlUtG3dZQwxuByudDY2IiKigqUlJT0OHiNkKv1m0D03nvvobW1FUuXLu12n9LSUjz33HNBHc/lcoHneRQUFECn00WolKQ7Wq0WSqUSVVVVcLlc0Gg0UheJDCD95mvrjTfewPz585GXl9ftPqtWrUJbW5t4q66u7vW49M0cOwPtd03p2vuPflEjqqqqws6dO7Ft27Ye91Or1VCr1TEqFYl3TRYX/nm0BiVZekwuSkWb3Y1sA9XkpNAvAtHGjRuRlZWFBQsWSF0UkkCqrljRanPj6OVWHKxqgVGrwNIZxVIXKyFJXpfmeR4bN27EkiVLoFD0i7hIEkS9Sej4GJNnBM8YWmxuOD00zEMKkgeinTt34uLFi3j44YelLkq/MWvWLCxfvlzqYsS9BrMDADA4XQe9RvgSbDQ7e3oJiRLJqyC33HILNRqSmHN5eLTa3ACATL0aaUkqmB0etNrcyE+VuHAJSPIakRRcHr7bm8fLB72vO4h9Q7V06VLs3r0b69atE6e85OfnY/369QH7ffPNN5DJZKiqqgIAvPzyyxg7diySkpJQUFCAJ598EhaLJeA1X3zxBWbOnAmtVouCggL88Ic/hNVqDbmM8aDV5gIAaFVy6FQKpOpUvu1uKYuVsCSvEUnhtc/Od/tccUYSFk0cJD7+w+flcHu7rrHlp2px1+T2xGwb9lTA7gpsY/jRzcNDKtu6detw9uxZjBkzBs8//zwAYO3atdiyZQueeOIJcb8333wTM2bMwODBgwEIXee//e1vUVxcjAsXLuDJJ5/Ej3/8Y/zv//4vAKC8vBzz5s3DL37xC2zYsAGNjY146qmn8NRTT2Hjxo0hlTEetNmFgJOiVQIAjDrhvtXukqxMiSwha0T9mdFohEqlgk6nQ05ODnJycrB48WLs2bMHFy9eBCA08G/duhWLFy8WX7d8+XLMnj0bRUVFmDNnDn7xi1/g7bffFp8vLS3F4sWLsXz5cpSUlGD69On47W9/i02bNsHhcMT8PPuDtCQV0pKEmpA/IFGNSBoJWSNaNntYt8/JrpoF8vgNQ7vd9+oZIw9Hqet3woQJGDVqFLZs2YJnnnkGu3fvRkNDQ8Dk4J07d6K0tBSnT5+GyWSCx+OBw+GAzWaDTqfDkSNHcPToUbz55pviaxhj4lSYUaNGRaXs/VVJth4l2e1ZA9OT1Zg0OBWpvpoRia2ErBGpFLJubwq5LOh9lUHsGymLFy/Gli1bAABbtmzBvHnzkJ6eDgCorKzEt7/9bYwbNw5//etfcfDgQbz22msAhKkuAGCxWPCDH/wAhw8fFm9HjhzBuXPnMHRo98E2URi1Stw4PBPj8lOkLkpCSsgaUX+nUqk6pS25//778ZOf/AQHDx7Eu+++i9///vficwcPHgTP81i7dq04zaLjZRkATJo0CSdPnsSwYd3XBgmRSkLWiPq7oqIi7Nu3D5WVlWhqagLP8ygqKsL06dPxyCOPwOv14o477hD3HzZsGNxuN373u9/hwoUL+L//+7+AQAUATz/9NPbu3YunnnoKhw8fxrlz5/D+++/jqaeeivXpSY7nGV7fXY4391XB4W4P+DaXB/UmR6cOBxJ9FIj6oZUrV0Iul2P06NHIzMwUG6kXL16MI0eO4Dvf+Q60Wq24//jx4/Hyyy/jxRdfxJgxY/Dmm2+itLQ04Jjjxo3D7t27cfbsWcycORMTJ07E6tWre5xkHK/sbi9sLi8azU6oOlxe//NoLbbsu4iLzZS/KtY4NoBHE5pMJhiNRrS1tXVaxcPhcKCiogLFxcWUkiJGBsrvvMHkwJv7LiJJLQ/ojPjwWC3O1Jlxw/BMXDuYRjVGQk//ox1RjYgkHIvTAwBIUgc2kSb7HvufJ7FDgYgkHKtTaANKvioQJanlAAC7iwJRrFEgIglHrBGpAgORRukLRG5qrI41CkQk4dhcXV+aaX2ByEa9ZjFHgYgkHK1SjvRkFYzawFHUOl8NibrvY48GNJKEM31YBqYPy+i03aBVYHJRaqeaEok++o0T4qNTKTCzJLP3HUnE0aUZIURyFIhIQrE6Pfi9b3pHV2N5zQ436k2OgKkfJPooEPVD/S1ndX8rT19YnR7YXV5YnZ4uV/59/3ANtuy7iLq2xMzRJBXJA9Hly5fx/e9/H+np6dBqtRg7diwOHDggdbEGPH/6DxLIP0ZIq+q6eVSnoi58KUgaiFpaWjBjxgwolUr861//wsmTJ7F27VqkpibuPJ+uclaXl5fjkUceQXFxMbRaLUaMGIF169Z1et2iRYvwwgsvIC8vDyNGjAAA7N27FxMmTIBGo8HkyZPx3nvvgeM4HD58WHzt8ePHMX/+fCQnJyM7OxsPPPAAmpqaui1PZWVlrH4dEScGIt+YoatpaVCjJCTtNXvxxRdRUFAQkDO5uDiKC9wxBnglSAUqV3ZO59iNrnJWp6amIj8/H++88w7S09Oxd+9ePP7448jNzcXdd98tvvaTTz6BwWDAjh07AAgTDm+//Xbcdttt2LJlC6qqqjpdYrW2tmLOnDl49NFH8Zvf/AZ2ux1PP/007r77bnz66addliczc+D2LPlrOt0FIv/oaicFopiSNBD9/e9/x6233oq77roLu3fvxqBBg/Dkk0/iscce63J/p9MJp7N93SmTyRTaG3rdwL/X9qXI4Zn5/wCFKqhdr85Z7ffcc8+JPxcXF+PLL7/E22+/HRCIkpKS8Kc//QkqlfBev//978FxHP74xz9Co9Fg9OjRuHz5csDv99VXX8XEiRPxy1/+Uty2YcMGFBQU4OzZsxg+fHiX5RmoHL5A5L8Eu5paKVwkOGihxZiS9NLswoULWL9+PUpKSrB9+3Y88cQT+OEPf4g///nPXe5fWloKo9Eo3goKCrrcLx699tpruPbaa5GZmYnk5GT84Q9/EPMU+Y0dO1YMQgBw5swZjBs3LiAlx5QpUwJec+TIEXz22WdITk4WbyNHjgQgrPwRb/yXXJpuakRqhb9GFPpSUCR8ktaIeJ7H5MmTxW/jiRMn4vjx4/j973+PJUuWdNp/1apVWLFihfjYZDKFFozkSqF2EmvyviVk37p1K1auXIm1a9di2rRp0Ov1+PWvf419+/YF7JeUlBTysS0WC26//Xa8+OKLnZ7Lzc0Nu8z9lVYlR0ayCgZt13/6GqoRSULSQJSbm4vRo0cHbBs1ahT++te/drm/Wq2GWq0O/w05LuhLJCldnbN6z549mD59Op588klxWzC1lREjRmDz5s1wOp3i723//v0B+0yaNAl//etfUVRUBIWi6z+HrnJoD1TTh2Zg+tDO0zv8MvVqXFeUJi4zRGJD0kuzGTNm4MyZMwHbzp49Ky4amKiuzlldUlKCAwcOYPv27Th79ix++tOfdgooXbn//vvB8zwef/xxnDp1Ctu3b8dLL70EAOIYmmXLlqG5uRn33Xcf9u/fj/Lycmzfvh0PPfSQGHy6yqEdr7L0GlxfkoHRed1nEySRJ2kg+tGPfoSvvvoKv/zlL3H+/Hls2bIFf/jDH7Bs2TIpiyW5q3NW33rrrbjzzjtxzz33YOrUqbhy5UpA7ag7BoMB//jHP3D48GFMmDAB//M//4PVq1cDgNhulJeXhz179sDr9eKWW27B2LFjsXz5cqSkpIgrgnSXQ5uQiGES+8c//sHGjBnD1Go1GzlyJPvDH/4Q9Gvb2toYANbW1tbpObvdzk6ePMnsdnskizvgbd68mSmVSmaz2SJ+7P7+O+d5nr2++zz7894KZnd5ut2n1epi9W12xvN8jEsYf3r6H+1I8tn33/72t/Htb39b6mLErU2bNmHIkCEYNGgQjhw5Io4R6rgKSKJwuHlYnV5Ynd5Oi2P6ub0MG/ZUABBWBFYpghv/RfpG8kBEoquurg6rV69GXV0dcnNzcdddd+GFF16QuliS8Hfdq5UyyK9eW9xHKecg4zjwjMHh8UZ0tV7SPQpEce7HP/4xfvzjH0tdjH7BnyJW180YIkBoxNcoZbC5vMJYov67KlJcoXBPEoZDnPDafSACALWvFuSksUQxQ4GIJAz/PLPuRlX7+Z930OjqmIn7QMQG7kK2A05//13be5nw6uefb0Y1otiJ20CkVArTKmw2Wsc8Vvy/a//vvr9RK4XpHSm6nkdN++ebUY0oduK2sVoulyMlJQUNDQ0AAJ1O12VGPtJ3jDHYbDY0NDQgJSUFcnnPNQ6pTChIwYSClF73G5KZBINGiWxDH6YTkZDEbSACIKat8AcjEl0pKSlxkSpkZI4BGPinMaDEdSDiOA65ubnIysqC2y1BQrQEolQq+21NiPR/cR2I/ORyOf2TELy5rwpenmHB2FykJ3d/2eX28rA5vQCHTqvBkuiI28ZqQjpijKHZ4sIViwsKWc9/9uWNFmzYU4GdJ+tjVDpCgYgkBLeXwcMLwwt6G9Co8s1Dc3mp1yxWKBCRhOCfZ6aQcVDKe+499c8vowT6sUOBiCQEcTCjSt7rMA5/IKIaUexQICIJwR7kPDMAUPs6NlweCkSxQoGIJIRgp3cA7VM83F4GL9+/p63ECwpEJCEo5Bwy9Gqk9jK9A0BA0jSqFcVGQowjImR4th7Ds/VB7SuXcZhQkAKFnAt2gV7SRxSICOnC7JFZUhchoUh6afazn/0MHMcF3PyrjBJCEofkNaJrrrkGO3fuFB93t8gfIX3x/uHLaLO7MXtEFgrSdL3u73ALqWI1KpmYFoREj+T/9QqFIi5mbJP+rcXqQost+InPHxytxcVmG+aNycGoXFpsMdok7zU7d+4c8vLyMGTIECxevLjHxfucTidMJlPAjZBg2H1JznRBjCMC2rvwqdcsNiQNRFOnTkVZWRk++ugjrF+/HhUVFZg5cybMZnOX+5eWlsJoNIq3goKCGJeYDEQ8z4JOnO/nn2/mpEAUE5IGovnz5+Ouu+7CuHHjcOutt+LDDz9Ea2sr3n777S73X7VqFdra2sRbdXV1jEtMBiL/qGqOAzRBtveI0zwoEMWE5G1EHaWkpGD48OE4f/58l8+r1Wqo1ZS+k4TGH4g0Sjlk3SyseLX2+WY08TUWJG8j6shisaC8vBy5ublSF4XEkVCmd/ipqUYUU5IGopUrV2L37t2orKzE3r178Z3vfAdyuRz33XeflMUicShDr0ZaUu/TO/z8XfbURhQbkl6aXbp0Cffddx+uXLmCzMxMXH/99fjqq6+QmZkpZbFInClI0+GBbw0O6TXpySqMyzciU09NAbHAsf6+Kl4PTCYTjEYj2traYDDQWA9C+ptg/0f7VRsRISQx9ateM0KiYefJelxutWPa0PSgZ+AzxuD08HB5eRg0tJJHtFGNiMS9VrsbzVZXe5Iztx2wNgE9tEpYnB6s31WOjV9UYgC3XgwYVCMicc/u8gDwTe+o3g9c+AzgvUBKATDme4BS0+k1/nFEPBNW/+gt4T7pG6oRkbjnH9CYbKkEzu8UghAAtFYDp//ZZc1IJZeJSdFoLFH0USAicY0xBruLB8c8SKrypZvJnwxMfgiQyYGmc0BLRafXcRwnpoylQBR9FIhIXHN6ePCMIdN6HiqPFVDrgSGzAH0OkDdR2Kn66y5fq6ZlhWKGAhGJa3aXF2AM+dbjkHEckH8dIPf1guVfJ9w3VwCOtk6vpWkesUOBiMQ1L2MoUFuRzpmES7Hc8e1PalOEBmsAaDjd6bXiiq8UiKKOAhGJaxnJanyvwIQxeUYgbUjnHrKsUcJ9Y+dANCQzGWMHGaHXUOdytNFvmMQ3xtprO1mjOz+fXgLgY8BcK4wvUmrFp64rSotNGQnViEics7cIN5kcSB/W+XmNAUjKEAJWS1Xsy0cAUCAice7osW9wuLoVF71pgKKbNCBpxcL9Vd34/hSz/jSzJHooEJG4xq5cgN3thS2ph/zmxkLhvu1SwOZ9Fc1Yv6sce8ubolhCAlAgIvGM90JhFvKac+nF3e9nyBPubVcAt0PcLPaauanXLNooEJH4ZaqB1+2ER6aB0tBD+mF1stCVzxhgrmnfTAMaYyasQHThwoVIl4OQyDNdhsfLYFLnQKfuJZWHYZBw33ZZ3ETjiGInrEA0bNgwzJ49G5s3b4bD4ej9BYRIgLVWw+3lYVZn95443x+IzLXiJhXNNYuZsALRoUOHMG7cOKxYsQI5OTn4wQ9+gK+/7nq+DiGSYAze1kvgGYRA1NvCivps4d5SL26itc1iJ6xANGHCBKxbtw41NTXYsGEDamtrcf3112PMmDF4+eWX0djYGPIx16xZA47jsHz58nCKREggewt4lw1ajRoKQ27v+YSSsoQVGJ0WwGUF0HFtMwpE0danxmqFQoE777wT77zzDl588UWcP38eK1euREFBAR588EHU1tb2fhAA+/fvx+uvv45x48b1pTiEtGu7BJVchgmjRuKhG0rAcb0EIoUK0KYKP/tqRTqVHCNy9BienRzlwpI+BaIDBw7gySefRG5uLl5++WWsXLkS5eXl2LFjB2pqarBw4cJej2GxWLB48WL88Y9/RGpqal+KQ0g7k6/R2Tgo+NckZwn3lgYAgE6lwG1jczFnZHaEC0euFlYgevnllzF27FhMnz4dNTU12LRpE6qqqvCLX/wCxcXFmDlzJsrKynDo0KFej7Vs2TIsWLAAc+fO7XVfp9MJk8kUcCOkS/5GZ0MogahzOxGJjbAmva5fvx4PP/wwli5d2u3y0FlZWXjjjTd6PM7WrVtx6NAh7N+/P6j3LS0txXPPPRdyeUmC4b2AtQk1bXbsOuFASVFzcBNY/YHI3B6IeJ7B5eWhkssgk1He6mgJKxDt2LEDhYWFkMkCK1SMMVRXV6OwsBAqlQpLlizp9hjV1dX4r//6L+zYsQMaTefk5V1ZtWoVVqxYIT42mUwoKOhh6D5JTNYmgPfCzitR79KgKNheryTfCsP2FiGYyeTYsKcCZocHi6cWIssQ3N8pCV1YgWjo0KGora1FVlZWwPbm5mYUFxfD6+19kuDBgwfR0NCASZMmidu8Xi8+//xzvPrqq3A6nZDLA7tc1Wo11GpaApj0wndpZVamARzXe9e9n1ovNFp7XICtGUjOFPNW06DG6AorEHW3zpPFYgm6dnPTTTfh2LFjAdseeughjBw5Ek8//XSnIERI0HyNzSaZcDmWpA7yz5zjAF0GYKoBbE1AciZN84iRkAKR/7KI4zisXr0aOp1OfM7r9WLfvn2YMGFCUMfS6/UYM2ZMwLakpCSkp6d32k5ISCx1AIBWeQoA33pmwdKl+wLRFQA08TVWQgpE33zzDQChRnTs2DGoVO35XVQqFcaPH4+VK1dGtoSEhIIx8dKsifPViFQh/JknZQj3ViH1Bw1qjI2QAtFnn30GQLiEWrduHQwGQ0QLs2vXrogejyQgRyvgccELGUyc8PcZdBsRIFyaAcKlGWi+WayE1Ua0cePGSJeDkMjwtQ95tOnI1OngdPNiO09QdL5uflsLwPM03yxGgg5Ed955J8rKymAwGHDnnXf2uO+2bdv6XDBCwmIW2ofUKblYPHJw6K/XpAAyBcB7AEcrsvQajMjRI0PfTZpZEhFBByKj0SjO1zEajVErECF94qsRITknvNfLZEKtyNIA2K5gdF4JRudFtgmCdBZ0IOp4OUaXZqTf8k/PSM7qeb+e6NLFQASURKRYpGdhzTWz2+2w2Wzi46qqKrzyyiv4+OOPI1YwQkLmsgJOMwDg4BUFyvZU4EBlc+jH8bcT2VsA+KZ5UBtRVIUViBYuXIhNmzYBAFpbWzFlyhSsXbsWCxcuxPr16yNaQEKC5q8NaVPR5pahxeYOr9vdnw7E1ozKJivWfXIO7xysjlw5SSdhZ2icOXMmAODdd99FTk4OqqqqsGnTJvz2t7+NaAEJCZq/fUifDatTmGakC2UMkZ+2vUakpF6zmAgrENlsNuj1egDAxx9/jDvvvBMymQzf+ta3UFVFq2USiYjtQ9mwOj3Cj+owpgr5a0ROM1QQAhoFougKO3n+e++9h+rqamzfvh233HILAKChoSHigxwJCZrYY5YNixiIelm9oysqHaAU5kyqPW0AKBBFW1iBaPXq1Vi5ciWKioowdepUTJs2DYBQO5o4cWJEC0hIULxucX4Yr8sUL82SNWGN2RVrRWq3EIg8PIOHpnlETVif0ve+9z1cf/31qK2txfjx48XtN910E77zne9ErHCEBM3SIMwzUyXBJtOBZwwyjoOut2WEuqNNBUy1UDpbAQjTPtxeBgUlhYiKML8ugJycHOTkBA4amzJlSp8LREhYOrQPebw8sg0aMLDwsyr6GqxljlaoFFlweXg4Pd7Q5q2RoIUViKxWK9asWYNPPvkEDQ0N4PnAKiutBEtiTmwfykKKToX7pxb27Xj+Bmt7M4ZkJIFnoFSxURRWIHr00Uexe/duPPDAA8jNze19qRZCos2Xg0jMO91XHQY1zp/edV52EjlhBaJ//etf+OCDDzBjxoxIl4eQ0PE8YPEt6qkPc47Z1cQufIuQOlZBk16jKaxes9TUVKSlBbEqAiGxYG8WZsvLlYA2FTtP1mPjngqcruvDclNKrXADAHsLGGPg+a5TJJO+CysQ/fznP8fq1asD5psRIpmOE105Dq12N1ptbnSTWj14vlrRv4+ewSs7z+HY5bY+HpB0J6xLs7Vr16K8vBzZ2dkoKiqCUhk4aCyYhRUJiRhzYPtQ+6jqsDuFBbo0wFQDldsEIIfSxUZRWJ/UokWLIlwMQvqgw4hqxliHUdV9DES+GpHOK1zi0ejq6Anrk3r22Wcj8ubr16/H+vXrUVlZCQC45pprsHr1asyfPz8ixycJoEOyfCRnw+nhxYAR9DJC3fEFIq3HBMgoEEVTWG1EgJD+409/+hNWrVqF5mYh58uhQ4dw+fLloI+Rn5+PNWvW4ODBgzhw4ADmzJmDhQsX4sSJE+EWiyQapwlw2wFOBiRlipdlaqVMzDcdtqumedAii9ET1lfG0aNHMXfuXBiNRlRWVuKxxx5DWloatm3bhosXL4q5inpz++23Bzx+4YUXsH79enz11Ve45pprwikaSTT+y7KkdECugMnhBADoNWFMdr2aLxCpvDbI+DBzG5GghPWVsWLFCixduhTnzp0LWNn1tttuw+effx5WQbxeL7Zu3Qqr1SpOoiWkV2JDtTB+SMYBOUYNsvQRWJpcqQWUGshlHDQeE12aRVFYNaL9+/fj9ddf77R90KBBqKurC+lYx44dw7Rp0+BwOJCcnIy//e1vGD16dJf7Op1OOJ1O8bHJ1IdxIiQ+dGgfAoDB6UkYnJ4UueNrU6E2mzFE54LWENxy6iR0YdWI1Gp1l0Hg7NmzyMzMDOlYI0aMwOHDh7Fv3z488cQTWLJkCU6ePNnlvqWlpTAajeKtoKAgnOKTeOKvEekjNLXjatpU6DVK3FysxvUlGdF5DxJeILrjjjvw/PPPw+12AwA4jsPFixfx9NNP47vf/W5Ix1KpVBg2bBiuvfZalJaWYvz48Vi3bl2X+65atQptbW3irbqa8ggntA7J8v01ItbnUYxXESe/tkT2uCRAWIFo7dq1sFgsyMzMhN1ux4033ohhw4ZBr9fjhRde6FOBeJ4PuPzqSK1Ww2AwBNxIAvNflunSAIXQJrRhTyU27qlAi9UVmffoEIgiHuSIKKw2IqPRiB07dmDPnj04cuQILBYLJk2ahLlz54Z0nFWrVmH+/PkoLCyE2WzGli1bsGvXLmzfvj2cYpFEYw5sH/J4eZjsQi1dE25CtKtpU+HheXx17BwONJ3Df84pgZzSgURcyIGI53mUlZVh27ZtqKysBMdxKC4uRk5ODhhjIaUEaWhowIMPPoja2loYjUaMGzcO27dvx8033xxqsUgiuqqh2uwQxhCpFDJolH0cQ+SnTYWM4yB3mcF5PXB5eEqOFgUhBSLGGO644w58+OGHGD9+PMaOHQvGGE6dOoWlS5di27ZteO+994I+3htvvBFqeQlp5w9EvoZqk0OoDRk0isjlyFLqIFOqIecAtddMgShKQgpEZWVl+Pzzz/HJJ59g9uzZAc99+umnWLRoETZt2oQHH3wwooUkpBOPE7D5VnH1jSEy2YUakUEbgcGMfhwHaFMhl9VB426D0+sFEMHjEwAhNla/9dZb+O///u9OQQgA5syZg2eeeQZvvvlmxApHSLfMtcK9xiAs/4P2GpE+3JU7uqNNpUGNURZSIDp69CjmzZvX7fPz58/HkSNH+lwoQnplqhHuDXniphab0FNm1EY4m6I2FXKZjAJRFIUUiJqbm5Gd3f3AsezsbLS00HgLEgP+QKRvD0SpOhWyDRpkJEcjEPlqRDTfLCpCqsN6vV4oFN2/RC6Xw+Px9LlQhPSIsS5rRDOGZWDGsCi8nzYVyWoFcpkd2kgNCyABQu41W7p0KdTqricUdjcQkZCIcpqEUdWcLHLJ8nuiTUVhmg6FHAek0nyzaAgpEC1ZsqTXfajHjESdvzaUnCkkzAfg5Rk4RGntMVUyIFcAXg/gaGtfaohETEiBaOPGjdEqByHBEy/LBombTtWa8OnpBozKNeDm0RGeAOvrwoelUZhzRoEo4iI0/JSQGBIbqtsXPmyxueDlGRRRmn5RZVNjf2Uz9p04H5XjJzoKRGRg8brbxxAZ88XNrTZhDFGKLjqDDb2aVHh4RrPwo4QCERlYTJcB3guok9tnxgNo9Y0hStFFZ0VWTie8F+dojcrxEx0FIjKwtF4U7lMKhbYbADzP0OKrEaVFKRDJfe1CMgfViKKBAhEZWFp9yfBSCts32d3w8gxKOQeDNsLTO3zkSUIgkjvbAJ4GNUYaBSIycHg97Q3VxvZAdMUijF9LT1ZHbtb9VVQ6I3hODi/vBZy09HSkUSAiA4fpMsB7AFVSQBe6RinH8Gw9BqfrovbWGpUcDoUBHi8Ds9HlWaRFpx5LBjbGhH94eT9Ld9F8QbhPLRLbhwCgIE2HgrToBSFACHYaQwaMNgu8tmYo0odE9f0SDQUiEshcD5x8T8j1k1YMjLpdqIH0B83lwn360Ji/tVIuw4wxw4DqZsDZGvP3j3d0aUbauWzA0b+ICccstedw6fNNMNn7wRxCR5swspnjgNRicbOXZ2ixumKT2F5MpN8a/fdKMBSISLuqvcJk0qQMnMq5Hd/UOlBdVY6PP/4QNa12acvmvywz5ImJ0ACgyeJE2d5KlO2tjH4ZaGmhqKFARARuO1D7DQDAXHAjdtaocTFlCpRyDtnNB/DRkYvSJgVrPCPcpwfm+ag3OQAAxkimh+3GRxec+LqyGVea6qgLP8IkDUSlpaW47rrroNfrkZWVhUWLFuHMmTNSFilx1Z8UuseTs/B1iwEenkGVPwEThhcjWebGEE85HB6vNGVzWYGWKuHnzJEBT9WbhMvG7BgsB+2UJ8HDOHg9HsBljvr7JRJJA9Hu3buxbNkyfPXVV9ixYwfcbjduueUWWK1WKYuVmBpPAwD4rDE42yD8/r81NBOKwqkYm2/ELP1lGNQS9W00ngYYL+Qeumrmu79GlG3oOkdWJKmVSrELny7PIkvSXrOPPvoo4HFZWRmysrJw8OBB3HDDDRKVKgE5LUCbMGJZlj0SD+QkobzBgvxULaC/BoryTwFrkzDZtENGxJipPyHcZ18TsNnj5XHFIswxy4pBjUijlMGmMMDD+9KBpBZF/T0TRb9qI2prE0aspqV1ne/F6XTCZDIF3EgEXDkvjB0y5AIaI5LVCowvSBFGKSs1QOZwAIDt4jc4WRP533m9yYFdZxrQZOmid85cD7RdFrIxZo0KeKrR4gTPGLQqOfQxqK2pFb5BjTzfvpQRiYh+E4h4nsfy5csxY8YMjBkzpst9SktLYTQaxVtBQUGMSxmnWn3tL2ndDNLLGQu3l8ehA3ux48RlmH3L9kTCqVoTtn5djW8utsLdVWL6mkPCfeZwQK0PeOpyi9CTl2vURG1qR0capQwOhZEuzaKg3wSiZcuW4fjx49i6dWu3+6xatQptbW3irbq6OoYljFOMiQ3Bbeo8vH/4Mo5Utwbuk1IEpVaPFKUXensNyhsj04bXYHZgx8l68IxhSGZS554vRxtQd0z4edC1nV5fkKbDlOI0jMo1RKQ8vdEo/TUiCkSR1i9GVj/11FP45z//ic8//xz5+fnd7qdWq7tN3E/CZGsWeqVkClS6U3ChsRluL8P4gpT2fWQyIHME0hqbkG67gHP1wzGh4/NhYIxh15lGeHkhCN0xPk+s1VxqsaG62Y5pnq+F3EMphYCxc+0326CJSW+ZX7JaAWNaFrROOeBoFYJ4DGpiiUDSGhFjDE899RT+9re/4dNPP0VxcXHvLyKR1Vop3BsHocYkXHLlp2o775c5AulJKqTZK1HTYoHV2bdlo6qu2HC5xQ6FjMPskVliEGqzu/HuwUs4fuokTBUHhJ2Lb+gX//AFaTp8d/poFGfqhaEOTurCjxRJA9GyZcuwefNmbNmyBXq9HnV1dairq4PdLvEo3kTSIb+Pv7E4S99FrdNYCLXOf3lWi/JGS5/e9thloWNiTL4RBk37JZlRq8S4bA2GXdmFqiYrWPZoIKVzbai80YILjRZ4Yr3goUwOaIzCz3ZqsI4USQPR+vXr0dbWhlmzZiE3N1e8/eUvf5GyWInFl//Zqx+EZqtQI8roKhDJZEDGcKQnqZBuv4Bz9eEHIqfHi4vNNgDA2EHGwCc9Tsxw7kISb0KzR4Wzhm91eYwvy6/g/cM1OFMvQa1Ely7c267E/r3jlKRtRDGZqEi657KKEzibZWngWT3USln3XeGZI5GWdBBprZU42maDy8NDpQj9u0ytkOPhGcWovGJFRnKHoNdSBZz7GGprE3LTDPhQcRPOVVgxNJeHQt7+Pq02FxrNTnAcMCQjOeT3DxfPM2zcW4mMeiduM/JQWikQRUq/aKwmEjHXCfe6dDQ5hDaYjKQeshymDIZGl4wxWV5MGasIKwj5aVXy9t4uewtQ/ln7fDJVEnJuuBPy4y6YHR4crm7F5KL2sWXHLwtjmQan66BVxW4JaJmMg8PthVlmFFLT2ppi9t7xrt903xMJiOuD5cDu9kIu45Ch7yH5vO/yLFWngro5vDmBXr5DLdjrASr+DXz9RyEIcRwwaBJw3SNQpuZj+tAMAMC+imbYXcI8N6fHi+M1QvtSp8u6GFArZLApU+Hmebo0iyCqESUy//pghjxMyk/FhPwUuHpr/M0eDdQeEeZ/ldwCxslCGkz40fE6WJ0e3FCoRM7Ffwo5hgBhusSwucIy0j6jcvU432hBSVYyNErhO/OrC0JQStUpURzDyzI/nUqBJmUKPB4mTI1x2wFlF72MJCQUiBIVY+2ByLdiqkzGQSPr5VLHWAiok9HQ1ISvPvs3ho2ahNF5wQ0otLk8KG+0QOVsRrL5C4BzCbmFSm4RZtVfFdA4jsMd49vnttW1OXCoShhIOHN4JuRRWtW1J1qVDF6ZCg5ZEgC3UCsydj/2jQSHLs0SlaNNyMjIyYDkENaKlwlzvpweHrKGUzgfQjf+yRoT5C4zpph3IJlzCbWfa5cKc8iCqFWlJakg4zjMGJaBoZmxrw0BgFYpfHfbVSnCBiu1E0UCBaJE5a8NJWfhstmNzV9V4d/nGoN7bdY1SEtSIdVeheqGVjiDyFPEGMOJ6isY0fQx8rVuoQt8/P3tY3KCoFLIsHRGEaYUdz0pOhZ0vsZxqzxF2EAN1hFBgShRiQ3VuWgwOdBodqLZ6grutfoc6IyZSFLwMFrKUdHU+9yzSy126Gv3wuC5grQUIzDu7oCUr8GKRSbGnqTqVMgxaqDQ+9qyaBZ+RFAgSlT+rntDrpjTJ2BMT084DlzuOKQnqZFtORlUapDzp48g13wMGckqKEbfDmhTwiy4tMbmG3HflEKMGuqbjkSXZhFBgSgRBTRU54lTO4IORACQOx6ZRh2SXY1oqqlEm6371CAWcysUZz8EAKSPmAFklIRd9H5DJwwtgKMN8ARZkyTdokCUiGzNgNcNyBVgujRcsfprRD2MIbqaKgmaQWOQolUiz3xMHNvTCWPQXfgYozPkSMseBOM1t0TgBPoBla790pLGE/UZdd8nIrGhOhsmBw+Xh4dcxiFVF0IgAoD8KcguPwjOUo30JFvX+9R8A1lzOdL1OqRfuxiQD+w/OYvTg7/sr4bHy+NxQzo4lw2wNgrZLUnYqEaUiPztQ/pcNPouy9KSVJCFOi5Hn43U4gkYma1HZv0XwiVfR6YaeM/tFH4eMgtIzupbufsBlVwGk90Nm8sLj9bXYG1tkLZQcYACUSIS24dywBhDWpKq69QfQeCKbxBqOS1VwMUv26dwWJtgP/QXfFPZiCouF2zQ5AgVXlpKOQelXAjYDpVvFr6FAlFfDex6MgkdzwOW9hpRSZIeJdn68DMh6NKAoTfBc/pfuPj1B7h0+Ay+NboIuHwAZ6ub0CZLxaWkG1DYDxKbRQLHcdAo5XB7PbCrM6AHhEBE2Rr7hAJRorFdESabypWAtn1gYJ+Szw+aBIe5Dc3VH0JpOoEjX5wEALQos1GdPw93jS2ISXL7WNGpFDA7PLAoUpDFyYT5Zk4zoIlN7ux4RIEo0XS8LOM4gLGIBInkkbNRpC/Ekf3/htXhRJsmHyxrFBaNGxSQgTEe+EdX2zycUCO0Ngm1IgpEYaNAlGgs9cK9PgcNZifePXgJ+alaLJwwqM+Hzhg0FLNyh6De5ADHAdl6TegN4AOAOM3DKSzRDWuT0GCdMUzikg1c1FidaDrMuG+yOOHy8HB7I5cpUy7jkJeiRa5RG5dBCADSk9XIMWqEpGz+CcP+AE/CQjWiRMLzHWpEuWi6FMZARoJrB6fi2sGpwoMrvi58S5AThkmXJK0Rff7557j99tuRlyesafXee+9JWZz4Z20QGqoVKkCbiiZzGFM7SCB/jcjeTFM9+kDSQGS1WjF+/Hi89tprUhYjcbRdFu4N+QDHhTfHjIgYY4A6WVgKu+P8PRIySS/N5s+fj/nz50tZhMRiuiTcGwfB6vTA5vKC44B0ujQLicPtxZZ9F2F3e/HEjUMhM+QCjWYhEKUOlrp4A9KAaiNyOp1wOp3iY5Op9/QTpAOxRjRITP1h1CqhlFOfRShUchnMDg94xmB1eaA3DAIaz7bneCIhG1B/gaWlpTAajeKtoKDzCqCkGw6TkLKC4wDDIMjlHIZkJmFweujJyRKdTMYhSe3vwveKOb/p0ix8AyoQrVq1Cm1tbeKturpa6iINHCZfbSg5C1CoMChFGDs0Z2QI+aqJKMm3CKXF6RECEccJwd4pwcqzcWBAXZqp1Wqo1dSwGpY2X/uQgVaciISAQKRIBpIyhC58Uy2QqZe4dAPPgKoRkT5oqRTuUwrB80wYFUzCZtAIgchk92Wm1PuWPTJTO1E4JK0RWSwWnD9/XnxcUVGBw4cPIy0tDYWFhRKWLM44TMI0BI4DUgfjitWFzV9VIVWnxNIZxVKXbkDyJ/Fv8wciQ66w8KS/Q4CERNJAdODAAcyePVt8vGLFCgDAkiVLUFZWJlGp4lBLhXCvzwWUWjQ2Cr2NOvWAujLvV9KThGkeYlZLo++L01Tjy25Av9tQSPrbmjVrVvh5cEjw/JdlqUUAIGZlzAwzGRoBCtN1KEzvUGvXpQGqJMBlFToGaDxRSKiNKN7xXqD5gvBzmnAZ1uib2pFJI6ojx3fZCwBovShtWQYgCkTxrrUKcDuEFScM+WCMiYEo3PSwpB3Ps/b0uCm+GhIFopBRIIp3jWeE+4zhgEwGs9MDh9sLGcchLYmmdvTFh8dq8epn51HeaBE2pPhqRKbLwnJNJGgUiOKZ19MeiDJHAADq2hwAgAy9Cgqa2tEnMo6Dl2do9S8uqU0VJsDyXqCNBtuGgv4S41nTWSGfsjoZSCkCAKRolZhYmIKROZTWtK/8Ncpmq2/+I8cBaUOEn6+US1SqgYn6GONZ7WHhPnc8IBO+c7IMGmQZNNKVKY74sxY0WTrkIcooEcYTNZ0Dhs2llT2CRDWieGWqEdYa4zggZ5zUpYlL6b4aUYvVBd7fYJ1aBMgUwgRjK2VtDBYFonhVuUe4z74G0KYAAMwON6qbbXB5eOnKFUeEFCocPDxrH2EtV4rjtXDlfLevJYEoEMWj5gvCPwHHAYNniJvP1Jnx7sFL+NdxSlcRCRzHIS1JGAJxxdqeJ0tczaPhlASlGpgoEMUbjws4+7Hw86BrhRG/PhebbQCAwjTKQRQpRek6jMjRQ6vq0NyaMQKQyYW1ziipflCosTqeMAac+RCwtwjdyMU3iE95vDwut9gBUCCKpOnDMjpvVOmE3rOmc0D9MSB5TuwLNsBQjSieVH4hXA5wMmD0QkDRPnK6ptUBD8+QrFbQQMZYyBkr3NefEMYVkR5RIIoHjAFVe4VABAAlc4GUwDS6/tG/g9N1cbUOfX/AGEOTxRmY4yl9mDAJ1mkBGk9LV7gBggLRQMfzwPmdwIXdwuMhNwptQwG7MJxrEFKYlmRT9sBI236iHv/3ZRVO1nZYzEEmb/8cqvcJXxakWxSIBjKvGzj5N+DSAeHx0DnA4OmddhO+rb1QK2XUPhQFuUZhgGhFkzXwibyJQl4ic317BgTSJQpEA5XTAhx5S1jGRiYHrlkEFE7tctcsgwYPThuM2SOyII/T9eilVJSeBACobXXA7urQHqTSCcEIAMo/FWqvpEsUiAYiUw1wsExIS6rUAOPvBbJG9fiS9GQ1RuXS/LJoMOqUyNSrwTOG03VXrbU3eIbwGVmbgMsHpSngAECBaCBhTJjH9M1mYdmapAxg0pL2PDhdCPiGJlEzZpARAHD8cltg1lGlFii+Ufj5wi5hbBHphALRQOG2AyffA05/KHQHZ5QAkx4MGLB4tVabC298cQHbT9TB46XLgmgamaOHUs6hyeLC+QZL4JN5E4VeNN4DHHtXWMyABOgXgei1115DUVERNBoNpk6diq+//lrqIvUfjAG1R4H9fwIaTgtjhIbcCIz5bsA4oau5vTz+dbwObi+DxeGhtqEo0yjlmDQ4FQBwqdUe+CTHASNvE/IVOdqAw1uESzUikjwQ/eUvf8GKFSvw7LPP4tChQxg/fjxuvfVWNDQkeBXW6xYC0IE3gNMfCI3TunRg0gNCz1gPY4HsLi/+frgGdW0OqJUy3HxNNo0dioFrB6dizsgszBqe2flJVZLQlqcxCiPfD24UJiZ7nJ33TUAck3gZjalTp+K6667Dq6++CgDgeR4FBQX4z//8TzzzzDM9vtZkMsFoNKKtrQ0GwwBtiGUM8LqE1R/sLULqiLZLwhJAXt8AOYVKaPQcNLnHZWoazU6cqzfj6OU22F1eKOUcFk0chPxU6rKXQpvNjfONZhSmJSEtSSXUSl1W4OTf21dWkSuA9BKhnS8pU8iUoEwS80cNdMH+j0o618zlcuHgwYNYtWqVuE0mk2Hu3Ln48ssvO+3vdDrhdLZ/g5hMQV5rN1cA5Z90PagsYBvrflsE961stqLZ7AQDIOPd4Ji3fS8mNHwq5TJAY8QxDMEBZwE8lVqg8iIYmHhoBuC+KYXiYn9HL7Xi6KU2AELSrnljcpClpyRoUvn3+Uacq7cAEC7DVAoZ1AoZ5NxUpDrTscB4AUpHC9BwCtVnDqLZ2p5gjZcpwMuUYJwCDBzGF6RApVAA4FDVbEOd2QlAqOWmJqkwPCs59ic49m5AE5kKgKSBqKmpCV6vF9nZ2QHbs7Ozcfp052HxpaWleO6550J/I4+zX82C9jpscDoCq+Q8p4BTkQybKg0jisZBmV0CJGeh6UwjWqtbAXS9RHTHCm1Gshol2ckYnq3HsMxkyKhdSDKMMRSm6eBw86g3OeDy8OINAFrlReAnzwFsdUBzBVptJ9BiroPaY4XwFeMF0OFvxA7Al2Oct1rBmx3t78WrAN1V7VKxwCLXATKgZt+vWrVKXA0WEGpEBQUFPbzCJ6VAuD4HOrStdPgnDWg/4brfFqF9U61OaFxegOPAZEpAqQWnUIl7qQ0awBdEri1KxTV5BvEQnO84HCfsq9coxXcYX5CC8QUpnU6fxB7HcRiXn4Jx+SlgjMHu9sLp5uH08OAZAwOglMsBYz5gzEde5hQYnV6A94LzOgGvC5zX5astM8j0Kt/fBEOqzQWVo32VEI1SBkgxkVmVFLFDSRqIMjIyIJfLUV9fH7C9vr4eOTk5nfZXq9VQq8NYi0uVJC4u2B+khfD5GTRKGDoEGzLwcBwHnUoBXQ+xIj1ZjXTx6qrn+YApKUBKhMrWX0jaIqZSqXDttdfik08+EbfxPI9PPvkE06ZNk7BkhJBYkvzSbMWKFViyZAkmT56MKVOm4JVXXoHVasVDDz0kddEIITEieSC655570NjYiNWrV6Ourg4TJkzARx991KkBmxASvyQfR9QXcTGOiJA4Fuz/aHyMmiKEDGgUiAghkqNARAiRnOSN1X3hb94KeqoHISSm/P+bvTVFD+hAZDYLCeGDGl1NCJGM2WyG0Wjs9vkB3WvG8zxqamqg1+v7ZZoL/xSU6urquOzVi+fzi+dzA2J3fowxmM1m5OXlQdZDRoEBXSOSyWTIz8+Xuhi9MhgMcfnH7BfP5xfP5wbE5vx6qgn5UWM1IURyFIgIIZKjQBRFarUazz77bHgZAwaAeD6/eD43oP+d34BurCaExAeqERFCJEeBiBAiOQpEhBDJUSAihEiOAlGIQl2V9p133sHIkSOh0WgwduxYfPjhhwHPM8awevVq5ObmQqvVYu7cuTh37lw0T6FboZzbH//4R8ycOROpqalITU3F3LlzO+2/dOlScBwXcJs3b160T6NboZxfWVlZp7JrNIFLMw3Uz27WrFmdzo3jOCxYsEDcJ+afHSNB27p1K1OpVGzDhg3sxIkT7LHHHmMpKSmsvr6+y/337NnD5HI5+9WvfsVOnjzJfvKTnzClUsmOHTsm7rNmzRpmNBrZe++9x44cOcLuuOMOVlxczOx2e6xOizEW+rndf//97LXXXmPffPMNO3XqFFu6dCkzGo3s0qVL4j5Llixh8+bNY7W1teKtubk5VqcUINTz27hxIzMYDAFlr6urC9hnoH52V65cCTiv48ePM7lczjZu3CjuE+vPjgJRCKZMmcKWLVsmPvZ6vSwvL4+VlpZ2uf/dd9/NFixYELBt6tSp7Ac/+AFjjDGe51lOTg779a9/LT7f2trK1Go1e+utt6JwBt0L9dyu5vF4mF6vZ3/+85/FbUuWLGELFy6MdFHDEur5bdy4kRmNxm6PF0+f3W9+8xum1+uZxWIRt8X6s6NLsyD5V6WdO3euuK2nVWkB4MsvvwzYHwBuvfVWcf+KigrU1dUF7GM0GjF16tRujxkN4Zzb1Ww2G9xuN9LS0gK279q1C1lZWRgxYgSeeOIJXLlyJaJlD0a452exWDB48GAUFBRg4cKFOHHihPhcPH12b7zxBu69914kJQWucxXLz44CUZB6WpW2rq6uy9fU1dX1uL//PpRjRkM453a1p59+Gnl5eQH/EPPmzcOmTZvwySef4MUXX8Tu3bsxf/58eL3eiJa/N+Gc34gRI7Bhwwa8//772Lx5M3iex/Tp03Hp0iUA8fPZff311zh+/DgeffTRgO2x/uwG9Ox70j+sWbMGW7duxa5duwIadO+9917x57Fjx2LcuHEYOnQodu3ahZtuukmKogZt2rRpAWvrTZ8+HaNGjcLrr7+On//85xKWLLLeeOMNjB07FlOmTAnYHuvPjmpEQQp1VVoAyMnJ6XF//30ox4yGcM7N76WXXsKaNWvw8ccfY9y4cT3uO2TIEGRkZOD8+fN9LnMo+nJ+fkqlEhMnThTLHg+fndVqxdatW/HII4/0+j7R/uwoEAUpnFVpp02bFrA/AOzYsUPcv7i4GDk5OQH7mEwm7Nu3L6Yr3Ya74u6vfvUr/PznP8dHH32EyZMn9/o+ly5dwpUrV5CbmxuRcgcrEisKe71eHDt2TCz7QP/sAGFoidPpxPe///1e3yfqn13MmsXjwNatW5larWZlZWXs5MmT7PHHH2cpKSlit+4DDzzAnnnmGXH/PXv2MIVCwV566SV26tQp9uyzz3bZfZ+SksLef/99dvToUbZw4ULJuoBDObc1a9YwlUrF3n333YAuXrPZzBhjzGw2s5UrV7Ivv/ySVVRUsJ07d7JJkyaxkpIS5nA4Ynpu4Zzfc889x7Zv387Ky8vZwYMH2b333ss0Gg07ceKEuM9A/ez8rr/+enbPPfd02i7FZ0eBKES/+93vWGFhIVOpVGzKlCnsq6++Ep+78cYb2ZIlSwL2f/vtt9nw4cOZSqVi11xzDfvggw8Cnud5nv30pz9l2dnZTK1Ws5tuuomdOXMmFqfSSSjnNnjwYAag0+3ZZ59ljDFms9nYLbfcwjIzM5lSqWSDBw9mjz32WKexOLEUyvktX75c3Dc7O5vddttt7NChQwHHG6ifHWOMnT59mgFgH3/8cadjSfHZURoQQojkqI2IECI5CkSEEMlRICKESI4CESFEchSICCGSo0BECJEcBSJCiOQoEJEBpaysDCkpKeLjn/3sZ5gwYYL4eOnSpVi0aFHMy0X6hgIR6ZI/Veh//Md/dHpu2bJl4DgOS5cuDdg/0gGgqKgIr7zySsC2e+65B2fPnu32NevWrUNZWZn4eNasWVi+fHlEy0UijwIR6VZBQQG2bt0Ku90ubnM4HNiyZQsKCwslKZNWq0VWVla3zxuNxoAaExkYKBCRbk2aNAkFBQXYtm2buG3btm0oLCzExIkT+3TsrmoqixYtEmtZs2bNQlVVFX70ox+JyduBzpdmV+tYM1u6dCl2796NdevWiceoqKjAsGHD8NJLLwW87vDhw+A4LuYpSoiAAhHp0cMPP4yNGzeKjzds2ICHHnoo6u+7bds25Ofn4/nnn0dtbS1qa2tDPsa6deswbdo0PPbYY+IxCgsLO50TAGzcuBE33HADhg0bFqlTICGgQER69P3vfx9ffPEFqqqqUFVVhT179gSVv6av0tLSIJfLodfrkZOTE1ayMaPRCJVKBZ1OJx5DLpdj6dKlOHPmjLjkjtvtxpYtW/Dwww9H+jRIkChVLOlRZmYmFixYgLKyMjDGsGDBAmRkZEhdrD7Jy8vDggULsGHDBkyZMgX/+Mc/4HQ6cdddd0ldtIRFNSLSq4cffhhlZWX485//HLFag0wmw9UZaNxud0SOHYxHH31UbIjfuHEj7rnnHuh0upi9PwlEgYj0at68eXC5XHC73bj11lsjcszMzMyAdh+v14vjx48H7KNSqfq8akR3x7jtttuQlJSE9evX46OPPqLLMonRpRnplVwux6lTp8Sfu9PW1obDhw8HbEtPT0dBQUGnfefMmYMVK1bggw8+wNChQ/Hyyy+jtbU1YJ+ioiJ8/vnnuPfee6FWq8O6JCwqKsK+fftQWVmJ5ORkpKWlQSaTiW1Fq1atQklJSUzzTJPOqEZEgmIwGGAwGHrcZ9euXZg4cWLA7bnnnuty34cffhhLlizBgw8+iBtvvBFDhgzB7NmzA/Z5/vnnUVlZiaFDhyIzMzOscq9cuRJyuRyjR49GZmYmLl68KD73yCOPwOVyxaQXkPSMUsWShPXvf/8bN910E6qrqzstUEhiiwIRSThOpxONjY1YsmQJcnJy8Oabb0pdpIRHl2Yk4bz11lsYPHgwWltb8atf/Urq4hBQjYgQ0g9QjYgQIjkKRIQQyVEgIoRIjgIRIURyFIgIIZKjQEQIkRwFIkKI5CgQEUIkR4GIECK5/w+sLjKcd4hwngAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:11.954190Z", + "iopub.status.busy": "2024-03-22T18:25:11.953310Z", + "iopub.status.idle": "2024-03-22T18:25:12.111585Z", + "shell.execute_reply": "2024-03-22T18:25:12.110496Z" + }, + "papermill": { + "duration": 0.182951, + "end_time": "2024-03-22T18:25:12.114578", + "exception": false, + "start_time": "2024-03-22T18:25:11.931627", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEpCAYAAADCh6TFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAl10lEQVR4nO3de1wU1x028Iflsty9IRcRXcVGNCmg8KJY08RGxKZJtWkNb0wCbqxpVVqTLY2lTaHGNCStUdIWpbVBExMjbWKatkm9FCVVg1EhWryAQVG8AILKPS7L7nn/8GXSdXdhUWB2dp/v58MH58yZ2d84+DhzmIubEEKAiEhhVHIXQER0OxheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIsodXXl4eNBoNvL29MW3aNBw6dKjH/k1NTVi+fDnCwsKgVqtx11134aOPPhqkaonIUXjI+eGFhYXQ6XTIz8/HtGnTkJubi+TkZFRWViI4ONiif2dnJ5KSkhAcHIx3330X4eHhOH/+PIYOHWr3Z5pMJly+fBkBAQFwc3Prx60hov4ghEBraytGjRoFlaqH4ysho4SEBLF8+XJp2mg0ilGjRomcnByr/Tds2CDGjx8vOjs7b/szL1y4IADwi1/8cvCvCxcu9Phv2U0IeZ4q0dnZCV9fX7z77ruYP3++1J6WloampiZ88MEHFss8+OCDGD58OHx9ffHBBx9g5MiRWLhwIVauXAl3d3ern6PX66HX66Xp5uZmjBkzBtXV1QgICOj37XIUBoMBe/fuxaxZs+Dp6Sl3OdQPXGWftra2Yty4cWhqasKQIUNs9pPttLGxsRFGoxEhISFm7SEhIaioqLC6zNmzZ7Fnzx48/vjj+Oijj1BVVYVly5bBYDAgOzvb6jI5OTlYtWqVRXtJSQl8fX3vfEMcmK+vLz799FO5y6B+5Ar7tKOjAwB6HdaRdcyrr0wmE4KDg/GnP/0J7u7uiIuLw6VLl/Db3/7WZnhlZmZCp9NJ0y0tLYiIiMCcOXMQGBg4WKUPOoPBgN27dyMpKcmp/5d2Ja6yT1taWuzqJ1t4BQUFwd3dHfX19Wbt9fX1CA0NtbpMWFgYPD09zU4RJ02ahLq6OnR2dsLLy8tiGbVaDbVabdHu6enp1D8A3VxlO12Js+9Te7dNtkslvLy8EBcXh6KiIqnNZDKhqKgIiYmJVpf52te+hqqqKphMJqnt9OnTCAsLsxpcROS8ZL3OS6fTYePGjXjjjTdw6tQpLF26FO3t7dBqtQCA1NRUZGZmSv2XLl2Ka9euYcWKFTh9+jQ+/PBDvPTSS1i+fLlcm0BEMpF1zCslJQUNDQ3IyspCXV0dYmNjsWPHDmkQv6amxuw6j4iICOzcuRPPPvssoqOjER4ejhUrVmDlypVybQIRyUT2Afv09HSkp6dbnVdcXGzRlpiYiIMHDw5wVUTk6GQPLyKy1NHRYXHJUNsXenxSfgbDgo7A3+fLX0JFRUU5/WU/1jC8iBxQRUUF4uLirM77zS3TpaWlmDp16sAX5WAYXkQOKCoqCqWlpWZtlbVN0P21HGsXfBUTw4aa9XVFDC8iB+Tr62txNKU6fxXqfV9g0j0xiB07QqbKHIfsj8QhIrodDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisS3BzmJW19SausFpYDrvqSUnAvDy0nYeknprS8oBVz3JaXkXBheTuLWl5TaekFpd18ipWN4OYlbX1LKF5SSs+OAPREpEsOLiBSJ4UVEisTwIiJFcojwysvLg0ajgbe3N6ZNm4ZDhw7Z7Lt582a4ubmZfXl7ew9itUTkCGQPr8LCQuh0OmRnZ6OsrAwxMTFITk7GlStXbC4TGBiI2tpa6ev8+fODWDEROQLZw2vt2rVYsmQJtFotJk+ejPz8fPj6+qKgoMDmMm5ubggNDZW+QkJCBrFiInIEsl7n1dnZidLSUmRmZkptKpUKs2fPRklJic3l2traMHbsWJhMJkydOhUvvfQS7r77bqt99Xo99Hq9NN3S0gIAMBgMMBgM/bQljqerq0v67szb6UpcZZ/au22yhldjYyOMRqPFkVNISIjZfXr/a+LEiSgoKEB0dDSam5uxZs0azJgxAydOnMDo0aMt+ufk5GDVqlUW7bt27XLq+/sutAGABw4ePIhLx+WuhvqDq+zTjo4Ou/op7gr7xMREJCYmStMzZszApEmT8Mc//hGrV6+26J+ZmQmdTidNt7S0ICIiAnPmzEFgYOCg1CyHYzXXgPIjmD59OmLGDJe7HOoHrrJPu8+OeiNreAUFBcHd3R319fVm7fX19QgNDbVrHZ6enpgyZQqqqqqszler1VCr1Rbtnp6e8PT07HvRCuHh4SF9d+btdCWusk/t3TZZB+y9vLwQFxeHoqIiqc1kMqGoqMjs6KonRqMR5eXlCAsLG6gyicgByX7aqNPpkJaWhvj4eCQkJCA3Nxft7e3QarUAgNTUVISHhyMnJwcA8MILL2D69OmYMGECmpqa8Nvf/hbnz5/H97//fTk3g4gGmezhlZKSgoaGBmRlZaGurg6xsbHYsWOHNIhfU1MDlerLA8Tr169jyZIlqKurw7BhwxAXF4dPPvkEkydPlmsTiEgGsocXAKSnpyM9Pd3qvOLiYrPpdevWYd26dYNQFRE5MtkvUiUiuh0MLyJSJIc4baTbU93YjnZ9l9V5Zxrape/dv2K3xk/tgXFBfgNSH9FAYngpVHVjO2atKe6130/eLe+1z96M+xlgpDgML4XqPuLKTYnFhGB/y/lf6PHP4hI8dH8i/HwsL9IFgKorbXim8KjNozciR8bwUrgJwf64J3yIRbvBYEDdSGDq2GFOfTW2M+lpGACwbyjAlYYBGF5EDsDeYQCg96EAVxkGYHgROYDehgGA3ocCXG0YgOFF5EBsDQMAHAq4Fa/zIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisSLVBVKb7wBlfclVLdUQuVteUV2V1cXLnddxqlrp2zeB1fd0gaV9yXojTcAWL8wkshRMbwU6nL7efiN+z1+fqjnfut3rO9xvt844HJ7LOIQ0mM/IkfD8FKoUX5j0V79I7yWEotIK/fCdXV14cD+A/jazK/ZPPI6c6UNKwqPYtSssQNdLlG/Y3gplNrdG6Yb4RgXOBGTR1h/JE61RzUmDZ9k8z44041mmG40QO3uPdDlEvU7DtgTkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTrvIgcQG+3ewG93/Llard7MbyIHIC9t3sBPd/y5Uq3ezG8iBxAb7d7Ab3f8uVqt3sxvIgcQG+3ewG93/Llard79XnA/uzZswNRBxFRn/Q5vCZMmIBZs2bhrbfewo0bN/qliLy8PGg0Gnh7e2PatGk4dMiOE38A27Ztg5ubG+bPn98vdRCRcvQ5vMrKyhAdHQ2dTofQ0FD84Ac/sDtsrCksLIROp0N2djbKysoQExOD5ORkXLlypcflzp07h4yMDNx77723/dlEpFx9Dq/Y2Fi89tpruHz5MgoKClBbW4uZM2finnvuwdq1a9HQ0NCn9a1duxZLliyBVqvF5MmTkZ+fD19fXxQUFNhcxmg04vHHH8eqVaswfvz4vm4CETmB2x6w9/DwwCOPPIJvfetbWL9+PTIzM5GRkYGf//znePTRR/HKK68gLCysx3V0dnaitLQUmZmZUptKpcLs2bNRUlJic7kXXngBwcHBWLx4Mfbt29fjZ+j1euj1emm6paUFwM3BT4PBYM+mOqSuri7pu7Xt6G7raRt7WwcNHnv2RW/71Fn2p72133Z4HTlyBAUFBdi2bRv8/PyQkZGBxYsX4+LFi1i1ahXmzZvX6+lkY2MjjEYjQkLMr0kJCQlBRUWF1WX279+P119/HUePHrWrzpycHKxatcqifdeuXfD19bVrHY7oQhsAeGD//v04b/036wCA3bt33/E6aOD1ZV/Y2qfOsj87Ojrs6tfn8Fq7di02bdqEyspKPPjgg3jzzTfx4IMPQqW6eQY6btw4bN68GRqNpq+r7lVrayuefPJJbNy4EUFBQXYtk5mZCZ1OJ023tLQgIiICc+bMQWBgYL/XOFhOXG7BmvKDmDlzJu4eZbkdBoMBu3fvRlJSks0nqfa2Dho89uyL3vaps+zP7rOj3vQ5vDZs2ICnnnoKixYtsnlaGBwcjNdff73XdQUFBcHd3R319fVm7fX19QgNDbXof+bMGZw7dw4PP/yw1GYymQDcPI2trKxEZGSk2TJqtRpqtdpiXZ6enjb/UStB90WKHh4ePW5HT9tp7zpo4PVlX9jap86yP+2tvc/htXv3bowZM0Y60uomhMCFCxcwZswYeHl5IS0trdd1eXl5IS4uDkVFRdLlDiaTCUVFRUhPT7foHxUVhfLycrO2559/Hq2trXjttdcQERHR180hIoXqc3hFRkaitrYWwcHBZu3Xrl3DuHHjYDQa+7Q+nU6HtLQ0xMfHIyEhAbm5uWhvb4dWqwUApKamIjw8HDk5OfD29sY999xjtvzQoUMBwKKdiJxbn8NLCGG1va2tDd7efb8tISUlBQ0NDcjKykJdXR1iY2OxY8cOaRC/pqbG4iiPiMju8Ooe9HZzc0NWVpbZb+qMRiM+/fRTxMbG3lYR6enpVk8TAaC4uLjHZTdv3nxbn0lEymZ3eH322WcAbh55lZeXw8vLS5rn5eWFmJgYZGRk9H+FRERW2B1ee/fuBQBotVq89tprir7MgIiUr89jXps2bRqIOoiI+sSu8HrkkUewefNmBAYG4pFHHumx7/bt2/ulMCKintgVXkOGDIGbm5v0ZyIiudkVXv97qsjTRiJyBLyAiogUya4jrylTpkinjb0pKyu7o4KIiOxhV3jxMctE5GjsCq/s7OyBroOIqE845kVEimTXkdfw4cNx+vRpBAUFYdiwYT2Of127dq3fiiMissWu8Fq3bh0CAgKkP9s7eE9ENFDsCq//fbDgokWLBqoWIiK79XnMy93d3eo7Fa9evQp3d/d+KYqIqDd9Di9bDyPU6/Vmj8khIhpIdj9V4ne/+x2Amw8j/POf/wx//y/frWQ0GvGf//wHUVFR/V8hEZEVdofXunXrANw88srPzzc7RfTy8oJGo0F+fn7/V0hEZIXd4VVdXQ0AmDVrFrZv345hw4YNWFFERL3p88MIu5+oSkQkpz6H11NPPdXj/IKCgtsuhojIXn0Or+vXr5tNGwwGHD9+HE1NTfjGN77Rb4UREfWkz+H1/vvvW7SZTCYsXboUkZGR/VIUEVFv+uXGbJVKBZ1OJ/1GkohooPXbUyXOnDmDrq6u/lodEVGP+nza2P3m7G5CCNTW1uLDDz80uweSBtYXBiMA4PilZqvz27/Q40gDEHr+Ovx81Fb7VF1pG7D6iAZan8Or+83Z3VQqFUaOHIlXX321199EUv858/+D52fby3vo5YEtVYd7XZefus8/BkSy43VeCjXn7lAAQGSwP3w8LW+Ir6xtxk/eLcer3/sqJobZfl2dn9oD44L8BqxOooHC/3IVarifF/5vwhib87vHHyNH+uGecL5rk5wPHwNNRIrE8CIiRWJ4EZEi9Vt4Xbx4EU8//XR/rY6IqEf9Fl5Xr17F66+/flvL5uXlQaPRwNvbG9OmTcOhQ4ds9t2+fTvi4+MxdOhQ+Pn5ITY2Flu2bLndsolIoWQ/bSwsLIROp0N2djbKysoQExOD5ORkq8/JB26+hu0Xv/gFSkpK8N///hdarRZarRY7d+4c5MqJSE6yh9fatWuxZMkSaLVaTJ48Gfn5+fD19bX5aJ37778f3/nOdzBp0iRERkZixYoViI6Oxv79+we5ciKSk6zXeXV2dqK0tBSZmZlSm0qlwuzZs1FSUtLr8kII7NmzB5WVlXjllVes9tHr9dDr9dJ0S0sLgJuP8jEYDHe4BY6r+zqvrq4up95OZ2HP/uputzXfWfa5vbXbHV6PPPJIj/ObmprsXZWksbERRqMRISEhZu0hISGoqKiwuVxzczPCw8Oh1+vh7u6O9evXIykpyWrfnJwcrFq1yqJ9165d8PX17XPNSnGhDQA8cPDgQVw6Lnc11JszLQDggW07D2C0n/U3dBlMwDU9cPa93fC0cs5U/4UbAHfs378f5/0t5ytFR0eHXf3sDq8hQ3q+SnvIkCFITU21d3V3JCAgAEePHkVbWxuKioqg0+kwfvx43H///RZ9MzMzzW4mb2lpQUREBObMmYPAwMBBqVcOx2quAeVHMH36dMSMGS53OdSLvxy5CJw4iW1n7/zdp8kP3AfNCOXe8tV9dtQbu8Nr06ZNt12MLUFBQXB3d0d9fb1Ze319PUJDQ20up1KpMGHCBABAbGwsTp06hZycHKvhpVaroVZbPlXB09MTnp6ed7YBDszDw0P67szb6Sy+GR0Od3d3m/eqAvbdr+oM96ra+/Mq65iXl5cX4uLiUFRUhPnz5wO4+VTWoqIipKen270ek8lkNq5FpDS93asK8H7VW9kdXvY+7qavL+DQ6XRIS0tDfHw8EhISkJubi/b2dmi1WgBAamoqwsPDkZOTA+DmGFZ8fDwiIyOh1+vx0UcfYcuWLdiwYUOfPpeIlM3u8Nq8eTPGjh2LKVOmQAjrA4q3IyUlBQ0NDcjKykJdXR1iY2OxY8cOaRC/pqYGKtWXo5Pt7e1YtmwZLl68CB8fH0RFReGtt95CSkpKv9VERI7P7vBaunQp3nnnHVRXV0Or1eKJJ57A8OH9MxCcnp5u8zSxuLjYbPrFF1/Eiy++2C+fS0TKZfdFqnl5eaitrcVzzz2Hf/zjH4iIiMCjjz6KnTt39uuRGBGRPfp0hb1arcZjjz2G3bt34+TJk7j77ruxbNkyaDQatLXxeehENHhu+/YglUoFNzc3CCFgNBr7syYiol71Kbz0ej3eeecdJCUl4a677kJ5eTn+8Ic/oKamBv7+Cr6kl4gUx+4B+2XLlmHbtm2IiIjAU089hXfeeQdBQUEDWRsRkU12h1d+fj7GjBmD8ePH4+OPP8bHH39std/27dv7rTgiIlvsDq/U1FS4ubkNZC1ERHbr00WqRESOQvaHERIR3Q6GFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSHCK+8vDxoNBp4e3tj2rRpOHTokM2+GzduxL333othw4Zh2LBhmD17do/9icg5yR5ehYWF0Ol0yM7ORllZGWJiYpCcnIwrV65Y7V9cXIzHHnsMe/fuRUlJCSIiIjBnzhxcunRpkCsnIjnJHl5r167FkiVLoNVqMXnyZOTn58PX1xcFBQVW+7/99ttYtmwZYmNjERUVhT//+c8wmUwoKioa5MqJSE4ecn54Z2cnSktLkZmZKbWpVCrMnj0bJSUldq2jo6MDBoMBw4cPtzpfr9dDr9dL0y0tLQAAg8EAg8FwB9U7tq6uLum7M2+nK3GVfWrvtskaXo2NjTAajQgJCTFrDwkJQUVFhV3rWLlyJUaNGoXZs2dbnZ+Tk4NVq1ZZtO/atQu+vr59L1ohLrQBgAcOHjyIS8flrob6g6vs046ODrv6yRped+rll1/Gtm3bUFxcDG9vb6t9MjMzodPppOmWlhZpnCwwMHCwSh10x2quAeVHMH36dMSMsX5USsriKvu0++yoN7KGV1BQENzd3VFfX2/WXl9fj9DQ0B6XXbNmDV5++WX8+9//RnR0tM1+arUaarXaot3T0xOenp63V7gCeHh4SN+deTtdiavsU3u3TdYBey8vL8TFxZkNtncPvicmJtpc7je/+Q1Wr16NHTt2ID4+fjBKJSIHI/tpo06nQ1paGuLj45GQkIDc3Fy0t7dDq9UCAFJTUxEeHo6cnBwAwCuvvIKsrCxs3boVGo0GdXV1AAB/f3/4+/vLth1y6+joMBsnrKxtgr6uCqeO+8B0dahZ36ioKKce7yPXIHt4paSkoKGhAVlZWairq0NsbCx27NghDeLX1NRApfryAHHDhg3o7OzE9773PbP1ZGdn41e/+tVglu5QKioqEBcXZ9G+8A3LvqWlpZg6deogVEU0cGQPLwBIT09Henq61XnFxcVm0+fOnRv4ghQoKioKpaWl0nTbF3p8uLcE35qVCH8ftUVfIqVziPCiO+fr62t2NGUwGHC98QoSE+KdenCXXJfsV9gTEd0OhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSLKHV15eHjQaDby9vTFt2jQcOnTIZt8TJ07gu9/9LjQaDdzc3JCbmzt4hRKRQ/GQ88MLCwuh0+mQn5+PadOmITc3F8nJyaisrERwcLBF/46ODowfPx4LFizAs88+K0PFRIOjo6MDFRUVZm2VtU3Q11Xh1HEfmK4OldqjoqLg6+s7yBU6ACGjhIQEsXz5cmnaaDSKUaNGiZycnF6XHTt2rFi3bl2fP7O5uVkAEM3NzX1eVkk6OzvF3/72N9HZ2Sl3KXQbSktLBQC7vkpLS+Uut1/Z+29UtiOvzs5OlJaWIjMzU2pTqVSYPXs2SkpK+u1z9Ho99Hq9NN3S0gIAMBgMMBgM/fY5jqZ725x5G51ZZGQkPv30U7O2ti/02LnvMJLv/T/w91Gb9XWm/WzvtsgWXo2NjTAajQgJCTFrDwkJsThcvhM5OTlYtWqVRfuuXbtc4lB79+7dcpdA/WjGVyPR2nQNrU1fttXW1spWz0Do6Oiwq5+sY16DITMzEzqdTppuaWlBREQE5syZg8DAQBkrG1gGgwG7d+9GUlISPD095S6H+oGr7NPus6PeyBZeQUFBcHd3R319vVl7fX09QkND++1z1Go11Gq1Rbunp6dT/wB0c5XtdCXOvk/t3TbZLpXw8vJCXFwcioqKpDaTyYSioiIkJibKVRYRKYSsp406nQ5paWmIj49HQkICcnNz0d7eDq1WCwBITU1FeHg4cnJyANwc5D958qT050uXLuHo0aPw9/fHhAkTZNsOIhp8soZXSkoKGhoakJWVhbq6OsTGxmLHjh3SIH5NTQ1Uqi8PDi9fvowpU6ZI02vWrMGaNWtw3333obi4eLDLJyIZyT5gn56ejvT0dKvzbg0kjUYDIcQgVEVEjk7224OIiG6H7Edeg637yM3eX8cqlcFgQEdHB1paWpz6N1OuxFX2afe/zd7OslwuvFpbWwEAERERMldCRD1pbW3FkCFDbM53Ey42iGQymXD58mUEBATAzc1N7nIGTPfFuBcuXHDqi3FdiavsUyEEWltbMWrUKLNf2N3K5Y68VCoVRo8eLXcZgyYwMNCpf9BdkSvs056OuLpxwJ6IFInhRUSKxPByUmq1GtnZ2Vbv6yRl4j4153ID9kTkHHjkRUSKxPAiIkVieBGRIjG8iEiRGF4KcP/99+OZZ56Ruwy6DY627xytnjvB8CJycJ2dnXKX4JgG+BVsdIfS0tIs3tMXHh4u1q9fb9avrKxMuLm5iXPnzgkhhHj11VfFPffcI3x9fcXo0aPF0qVLRWtrq9ky+/btEzNnzhTe3t5i9OjR4kc/+pFoa2sbtG1zdtb2XVVVlXjqqaeERqMR3t7e4q677hK5ubkWy82bN0+8+OKLIiwsTGg0GiGEEAcOHBAxMTFCrVaLuLg48f777wsA4rPPPpOWLS8vF3PnzhV+fn4iODhYPPHEE6KhocFmPdXV1YP119HvGF4OrqmpSSQmJoolS5aI2tpaUVtbKzIyMsTMmTPN+v3kJz8xa1u3bp3Ys2ePqK6uFkVFRWLixIli6dKl0vyqqirh5+cn1q1bJ06fPi0OHDggpkyZIhYtWjRo2+bsrO27GzduiKysLHH48GFx9uxZ8dZbbwlfX19RWFgoLZeWlib8/f3Fk08+KY4fPy6OHz8umpubxfDhw8UTTzwhTpw4IT766CNx1113mYXX9evXxciRI0VmZqY4deqUKCsrE0lJSWLWrFk26+nq6pLjr6ZfMLwU4L777hMrVqyQpj/77DPh5uYmzp8/L4S4+abx8PBwsWHDBpvr+Otf/ypGjBghTS9evFg8/fTTZn327dsnVCqV+OKLL/p3A1zYrfvOmuXLl4vvfve70nRaWpoICQkRer1eatuwYYMYMWKE2b7ZuHGjWXitXr1azJkzx2zdFy5cEABEZWWl3fUoBce8FCg2NhaTJk3C1q1bAQAff/wxrly5ggULFkh9/v3vf+OBBx5AeHg4AgIC8OSTT+Lq1avSCz2PHTuGzZs3w9/fX/pKTk6GyWRCdXW1LNvlKvLy8hAXF4eRI0fC398ff/rTn1BTU2PW56tf/Sq8vLyk6crKSkRHR8Pb21tqS0hIMFvm2LFj2Lt3r9k+jYqKAgCcOXNmALdIHgwvhXr88cel8Nq6dSvmzp2LESNGAADOnTuHhx56CNHR0XjvvfdQWlqKvLw8AF8O/ra1teEHP/gBjh49Kn0dO3YMn3/+OSIjI+XZKBewbds2ZGRkYPHixdi1axeOHj0KrVZrMSjv5+fX53W3tbXh4YcfNtunR48exeeff46vf/3r/bUJDsPlnuelRF5eXjAajWZtCxcuxPPPP4/S0lK8++67yM/Pl+aVlpbCZDLh1VdflR7m9pe//MVs+alTp+LkyZN8ZdwAu3XfHThwADNmzMCyZcukNnuOiiZOnIi33noLer1eujH78OHDZn2mTp2K9957DxqNBh4e1v9pW/tZUioeeSmARqPBp59+inPnzqGxsREmkwkajQYzZszA4sWLYTQa8e1vf1vqP2HCBBgMBvz+97/H2bNnsWXLFrNwA4CVK1fik08+QXp6uvS/8wcffGDzTU50e27dd1/5yldw5MgR7Ny5E6dPn8Yvf/lLixCyZuHChTCZTHj66adx6tQp7Ny5E2vWrAEA6YnAy5cvx7Vr1/DYY4/h8OHDOHPmDHbu3AmtVisFlrWfJcWSe9CNeldZWSmmT58ufHx8zH69vX79egFApKamWiyzdu1aERYWJnx8fERycrJ48803BQBx/fp1qc+hQ4dEUlKS8Pf3F35+fiI6Olr8+te/HqStcg237ruKigqxaNEiMWTIEDF06FCxdOlS8bOf/UzExMRIy3RfKnGrAwcOiOjoaOHl5SXi4uLE1q1bpXV2O336tPjOd74jhg4dKnx8fERUVJR45plnhMlkslqPki+V4CNxiBTq7bffhlarRXNzM3x8fOQuZ9BxzItIId58802MHz8e4eHhOHbsGFauXIlHH33UJYMLYHgRKUZdXR2ysrJQV1eHsLAwLFiwAL/+9a/lLks2PG0kIkXibxuJSJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFA2rRokVwc3ODm5sbPD09ERISgqSkJBQUFPTp1pTNmzdj6NChA1eoDYsWLcL8+fMH/XOpdwwvGnBz585FbW0tzp07h3/961+YNWsWVqxYgYceeghdXV1yl0dKJe/dSeTsbN2nV1RUJACIjRs3CiF6fmz13r17LR5fnJ2dLYQQ4s033xRxcXHC399fhISEiMcee0zU19dLn3Pt2jWxcOFCERQUJLy9vcWECRNEQUGBNL+mpkYsWLBADBkyRAwbNkx8+9vflu73y87OtvjcvXv3DsjfE/Udj7xIFt/4xjcQExOD7du3AwBUKhV+97vf4cSJE3jjjTewZ88ePPfccwCAGTNmIDc3F4GBgaitrUVtbS0yMjIAAAaDAatXr8axY8fwt7/9DefOncOiRYukz/nlL3+JkydP4l//+hdOnTqFDRs2ICgoSFo2OTkZAQEB2LdvHw4cOAB/f3/MnTsXnZ2dyMjIwKOPPiodOdbW1mLGjBmD+xdFtsmdnuTcbB15CSFESkqKmDRpktV5tz62etOmTWLIkCG9ft7hw4cFAOmo7eGHHxZardZq3y1btoiJEydKT1wQQgi9Xi98fHzEzp07e62f5MUjL5KNEEJ6FlVvj622pbS0FA8//DDGjBmDgIAA3HfffQAgPVZ56dKl2LZtG2JjY/Hcc8/hk08+kZY9duwYqqqqEBAQID02efjw4bhx44ZTPjbZ2TC8SDanTp3CuHHj7HpstTXt7e1ITk5GYGAg3n77bRw+fBjvv/++2XLf/OY3cf78eTz77LO4fPkyHnjgAemUs62tDXFxcRaPTT59+jQWLlw4wFtPd4pPlSBZ7NmzB+Xl5Xj22Wftemy1tccXV1RU4OrVq3j55ZcREREBADhy5IjFZ40cORJpaWlIS0vDvffei5/+9KdYs2YNpk6disLCQgQHByMwMNBqnc702GRnwyMvGnB6vR51dXW4dOkSysrK8NJLL2HevHl46KGHkJqaatdjqzUaDdra2lBUVITGxkZ0dHRgzJgx8PLykpb7+9//jtWrV5stl5WVhQ8++ABVVVU4ceIE/vnPf2LSpEkAbr7EJCgoCPPmzcO+fftQXV2N4uJi/PjHP8bFixelz/3vf/+LyspKNDY2wmAwDM5fGvVO7kE3cm7/+5ZmDw8PMXLkSDF79mxRUFAgjEaj1M+ex1b/8Ic/FCNGjDC7VGLr1q1Co9EItVotEhMTxd///neLdxlOmjRJ+Pj4iOHDh4t58+aJs2fPSuusra0VqampIigoSKjVajF+/HixZMkS0dzcLIQQ4sqVK9KjssFLJRwKn+dFRIrE00YiUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEivT/AKMmqVTNH2pFAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:25:12.154928Z", + "iopub.status.busy": "2024-03-22T18:25:12.154594Z", + "iopub.status.idle": "2024-03-22T18:25:12.439272Z", + "shell.execute_reply": "2024-03-22T18:25:12.438355Z" + }, + "papermill": { + "duration": 0.306768, + "end_time": "2024-03-22T18:25:12.441359", + "exception": false, + "start_time": "2024-03-22T18:25:12.134591", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATAAAAEmCAYAAADldMx1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUz0lEQVR4nO2dd3hU1daH3zMtDZJQEwKB0KQ3wSB4EZEgKorYQEBKRCyIormIYAEBFa6UGwuCggh6QYpX0WtBJZ8gKIqE3qUGSQEE0pNp5/tjMpOpSWYyk8kk+32eeWDOnHNmzZmc36y99tprSbIsywgEAkEAovC3AQKBQOApQsAEAkHAIgRMIBAELELABAJBwCIETCAQBCxCwAQCQcAiBEwgEAQsQsAEAkHAovK3AVWN0WgkPT2dunXrIkmSv80RCAR2yLJMbm4uMTExKBRl+1i1TsDS09OJjY31txkCgaAczp8/T7Nmzcrcp9YJWN26dQHTxQkPD/ezNQKBwJ6cnBxiY2Mt92pZ1DoBMw8bw8PDhYAJBNWYioR4RBBfIBAELELABAJBwCIETCAQBCy1LgZWEWRZRq/XYzAY/G2KwEsolUpUKpVInalhCAGzQ6vVkpGRQUFBgb9NEXiZ0NBQmjRpgkaj8bcpAi8hBMwKo9HImTNnUCqVxMTEoNFoxC92DUCWZbRaLZcuXeLMmTO0bdu23ARJQWAgBMwKrVaL0WgkNjaW0NBQf5sj8AC9wchfVwuJDFUTGVrqaYWEhKBWqzl37hxarZbg4GA/WinwFuJnyAni1zlwydcayCnS8Xee1uE18b3WPMQ3KqhZlPSokRG9amoDQsAENRLRa6t2IARMUGMYP348o0Y8ACD8r1pCtRCwJUuWEBcXR3BwML1792bXrl0u9121ahWSJNk8REAWbrnlFp599ll/m1FtEB5Y7cDvArZ+/XqSkpKYNWsWe/bsoVu3bgwePJiLFy+6PCY8PJyMjAzL49y5c1VoscCX6HQ6r5xHxMBqB34XsMWLFzNx4kQSExPp2LEjy5YtIzQ0lJUrV7o8RpIkoqOjLY+oqCif2SfLMgVavV8eFW2aPn78eLZt28Zbb71l8UqbNWvG0qVLbfbbu3cvCoXCIviLFy+mS5cuhIWFERsby6RJk8jLy7M5ZseOHfTr14+QkBBiY2N55plnyM/Pr5BdGRkZDBkyhJCQEFq2bMnatWuJi4sjOTnZso8kSSxdupShQ4cSFhbG66+/jsFgYMKECbRs2ZKQkBDatWvHW2+9ZXNug8FAUlISkZGRNGjQgGnTptleL6FftQK/5oFptVpSU1OZMWOGZZtCoSAhIYGdO3e6PC4vL48WLVpgNBq5/vrreeONN+jUqZPTfYuLiykuLrY8z8nJccvGQp2BjjO/d+sYb3FkzmBCNeV/RW+99RYnTpygc+fOzJkzB4BFixaxdu1annzySct+a9as4aabbqJFixaA6Vq//fbbtGzZktOnTzNp0iSmTZvGe++9B8CpU6e4/fbbee2111i5ciWXLl1i8uTJTJ48mY8++qhcu8aOHcvly5fZunUrarWapKQkp571q6++yvz580lOTkalUmE0GmnWrBkbN26kQYMG/Prrrzz22GM0adKE4cOHWz7fqlWrWLlyJR06dGDRokV88cUX9Ot/CyD0q7bgVwG7fPkyBoPBwYOKiori2LFjTo9p164dK1eupGvXrmRnZ7Nw4UL69u3L4cOHnVZvnDdvHrNnz/aJ/dWFiIgINBoNoaGhREdHAzB69GgWLVpEWloazZs3x2g0sm7dOl5++WXLcdYxs7i4OF577TWeeOIJi4DNmzeP0aNHW/Zr27Ytb7/9Nv3792fp0qVlxh6PHTvGli1b+OOPP+jVqxcAK1asoG3btg77jho1isTERJtt1t9Zy5Yt2blzJxs2bLAIWHJyMjNmzOC+++4DYNmyZXz/fekPjYiB1Q4CLhO/T58+9OnTx/K8b9++dOjQgffff5+5c+c67D9jxgySkpIsz83VHitKiFrJkTmDK2e0h4SolR4f2717dzp06MDatWuZPn0627Zt4+LFizz44IOWfbZs2cK8efM4duwYOTk56PV6ioqKKCgoIDQ0lP3793PgwAHWrFljOUaWZcuSqw4dOrh8/+PHj6NSqbj++ust29q0aUO9evUc9jULnDVLlixh5cqVpKWlUVhYiFarpXv37gBkZ2eTkZFB7969LfurVCp69eqFVm9agC9iYLUDvwpYw4YNUSqVZGVl2WzPysqyeBLloVar6dGjBydPnnT6elBQEEFBQR7bKElShYZx1ZHRo0dbBGzt2rXcfvvtNGjQAICzZ89y11138eSTT/L6669Tv359duzYwYQJE9BqtYSGhpKXl8fjjz/OM88843Du5s2be83OsLAwm+fr1q1j6tSpLFq0iD59+lC3bl0WLFjA77//XvGTCv2qFfg1iK/RaOjZsycpKSmWbUajkZSUFBsvqywMBgMHDx6kSZMmvjIzINBoNA7lf0aNGsWhQ4dITU3ls88+Y/To0ZbXUlNTMRqNLFq0iBtvvJHrrruO9PR0m+Ovv/56jhw5Qps2bRwe5VV0aNeuHXq9nr1791q2nTx5kqtXr5b7WX755Rf69u3LpEmT6NGjB23atOHUqVOW1yMiImjSpImNoOn1elJTUy3PhX7VDvw+C5mUlMTy5ctZvXo1R48e5cknnyQ/P98SExk7dqxNkH/OnDn88MMPnD59mj179vDwww9z7tw5Hn30UX99hGpBXFwcv//+O2fPnuXy5csYjUbi4uLo27cvEyZMwGAwMHToUMv+bdq0QafT8c4773D69Gk++eQTli1bZnPOF154gV9//ZXJkyezb98+/vzzT7788ksmT55crj3t27cnISGBxx57jF27drF3714ee+wxQkJCyq3w0bZtW3bv3s3333/PiRMneOWVV/jjjz9s9pkyZQrz589n06ZNHDt2jEmTJnHt2jXL6yIGVkuQqwHvvPOO3Lx5c1mj0cjx8fHyb7/9Znmtf//+8rhx4yzPn332Wcu+UVFR8p133inv2bOnwu+VnZ0tA3J2drbDa4WFhfKRI0fkwsLCSn0ef3D8+HH5xhtvlENCQmRAPnPmjCzLsvzee+/JgDx27FiHYxYvXiw3adJEDgkJkQcPHix//PHHMiBfvXrVss+uXbvkQYMGyXXq1JHDwsLkrl27yq+//nqFbEpPT5fvuOMOOSgoSG7RooW8du1auXHjxvKyZcss+wDyF198YXNcUVGRPH78eDkiIkKOjIyUn3zySXn69Olyt27dLPvodDp5ypQpcnh4uBwZGSknJSXJY8eOle+86255//mr8v7zV2Wj0Whz3kD+fmsTZd2j9kiyXLt+q3JycoiIiCA7O9uhK1FRURFnzpyhZcuWIrvfB/z111/ExsayZcsWBg4c6JP3uFqg5fwVUzHKLk0jbLw98f0GBmXdo/YEZnRaEBD83//9H3l5eXTp0oWMjAymTZtGXFwcN998s+/e1DqXVQZRj7JmIwRM4BHbt2/njjvucPl6Xl4eOp2OF198kdOnT1O3bl369u3LmjVrUKvVVWJjrRpa1FKEgAk8olevXuzbt6/MfQYPHszgwf7JoQNKlhYJF6wmIwRM4BEhISG0adPG32Y4ILv4v6Bm4vc0CoHAZwgFq/EIARPUWMRyopqPEDBBjaV2JQjVToSACWosQr9qPkLABDUWIWA1HyFgAqdVUjdt2uQ3eyqHXSaroEYj0igEDmRkZDit2xVoCPmq+QgBEzhQ0VpsvkSWZQwGAyqV53+iwgGr+YghZHnIMmjz/fNw4w7Mzc1l9OjRhIWF0aRJE/7973973GrNegh59uxZJEni888/Z8CAAYSGhtKtWzeHngXlNf/45JNP6NWrF3Xr1iU6OppRo0bZ1MffunUrkiTx3Xff0bNnT4KCgtixY4fbtotE1tqF8MDKQ1cAb8T4571fTAdNWPn7Yaqr9ssvv/DVV18RFRXFzJkz2bNnj6UMc2V56aWXWLhwIW3btuWll15i5MiRnDx5EpVKVaHmHzqdjrlz59KuXTsuXrxIUlIS48eP59tvv7V5n+nTp7Nw4UJatWpV+WGscMFqPELAagC5ubmsXr2atWvXWsrUfPTRR8TEeE94p06dypAhQwBTw41OnTpx8uRJ2rdvX6HmH4888ojlXK1ateLtt9/mhhtuIC8vjzp16lhemzNnDoMGDfLcUNFZrVYhBKw81KEmT8hf710BTp8+jU6nIz4+3rItIiKCdu3aec2Url27Wv5vLt998eJF2rdvX6HmH6mpqbz66qvs37+fq1evYjQaAUhLS6Njx46W45w1+PAU4YDVfISAlYckVXgYV5OxLoFjLhJoFqHymn/k5+dbKlOsWbOGRo0akZaWxuDBg9FqtTb72zf4qAxCv2o+QsBqAK1atUKtVvPHH39YugVlZ2dz4sQJ3xYPLMG6+YczDh48yN9//838+fMtLe12797tc7tqWbHhWomYhawB1K1bl3HjxvH888/z008/cfjwYSZMmIBCoSi3gYY3KK/5R/PmzdFoNJYGIl999ZXTHp4CgbsIAashLF68mD59+nDXXXeRkJDATTfdRIcOHaqk9nvXrl3Ztm0bJ06coF+/fvTo0YOZM2daJhEaNWrEqlWr2LhxIx07dmT+/PksXLjQ53YJB6zmI5p6WFGTmj7k5+fTtGlTFi1axIQJE/xtTpXxd14xF64VAtCsXgj1w0qbGtek77cmI5p61EL27t3LsWPHiI+PJzs7mzlz5gBwzz33+Nky/1G7fpprJ0LAahALFy7k+PHjlo7n27dv5+jRo+U236hJiEz82oUQsBpCjx49SE1NddheWFhYbvONmorwwGo+QsBqONW1+UbVIBSspiNmIZ1Qy+Y1aiz2X6P4XmseQsCsMGebFxQU+NkSgTewlyvz93oxX8fxzNyqN0jgdcQQ0gqlUklkZKSlzEtoaGiVJIIKvIeuWIusNy1P0hZLFBWZPK+CggIuXrxIZGQkD674g4u5xaS+nEDd4KrpEi7wDULA7DAX87OuVSUIHPKK9Fwr1AFQGKwiL6RUoCIjI4mOjiYjey8Go8y1Ap0QsACnWgjYkiVLWLBgAZmZmXTr1o133nnHprKCK9atW8fIkSO55557vFbDXZIkmjRpQuPGjdHpdF45p6Dq+Cz1L5ZuvQDA8F6xPN6/JWAKDyiVSlOlV6NpcClCYoGP3wVs/fr1JCUlsWzZMnr37k1ycjKDBw/m+PHjNG7c2OVxZ8+eZerUqfTr188ndimVSpRKpU/OLfAdxbKSC7kGAK5pcci41xlKVcsgFCzg8XsQf/HixUycOJHExEQ6duzIsmXLCA0NZeXKlS6PMRgMjB49mtmzZ9OqVasqtFZQ3bGeaTR7WtboS0oAuXpdEFj4VcC0Wi2pqakkJCRYtikUChISEhxqrlszZ84cGjduXKE1fsXFxeTk5Ng8BLUDa2/L2TaRVhH4+FXALl++jMFgICoqymZ7VFQUmZmZTo/ZsWMHH374IcuXL6/Qe8ybN4+IiAjLw1yPSlDzMVh5W2b0BisPTAhYwOP3IaQ75ObmMmbMGJYvX07Dhg0rdMyMGTPIzs62PM6fP+9jKwX+xFqT9E48ML3VsNGJvgkCDL8G8Rs2bIhSqSQrK8tme1ZWltPehKdOneLs2bPcfffdlm3mssYqlYrjx4/TunVrm2OCgoIICgpCUDuQrdJX9U5iXDorD8woPLCAx68emLlqQkpKimWb0WgkJSWFPn36OOzfvn17Dh48yL59+yyPoUOHMmDAAPbt2yeGhwIb9E6HkGUH+QWBhd/TKJKSkhg3bhy9evUiPj6e5ORk8vPzSUxMBGDs2LE0bdqUefPmERwcTOfOnW2Oj4yMBHDYLqhdbDmSxcbU88Q1KG0K4nwIKTywmoTfBWzEiBFcunSJmTNnkpmZSffu3dm8ebMlsJ+WloZCEVChOoEfePRjU5MQlaJ06ZfzIaRVDEwIWMDjdwEDLF2cnbF169Yyj121apX3DRIEFK6Ggs4ETG8jYD4zSVBFCNdGEPCkXSmtHhJbv7QZsHXKhBmdSGStUVTKA8vLy7PMApoprwi/QOBtjmU4T04u3wMTAhbouO2BnTlzhiFDhhAWFkZERAT16tWjXr16REZGUq9ePV/YKBCUyZ8XS+v6W3tVzjwwmyC+yAMLeNz2wB5++GFkWWblypVERUWJelkCv1OsN1j+by1gTtdCisXcNQq3BWz//v2kpqbSrl07X9gjEFQK62Ghs7WQIo2iZuH2EPKGG24Qy3EE1RZjOdUobNIoRBA/4HHbA1uxYgVPPPEEFy5coHPnzpY68ma6du3qNeMEAnexDnvpysnEF/oV+LgtYJcuXeLUqVOWTHkwVTGVZRlJkjAYDGUcLRD4FlEPrHbhtoA98sgj9OjRg08//VQE8QXVDuvAvLOlRKIeWM3CbQE7d+4cX331VS1uliqoztikUYh6YDUet4P4t956K/v37/eFLQJBpTEay/HAykmzEAQWbntgd999N8899xwHDx6kS5cuDkH8oUOHes04gcBdDDZpFGV7YMIBC3zcFrAnnngCMNWlt0cE8QX+xnrUWG4iq/DAAh63Bcx+7aNAUJ2wSWR1lgcmEllrFG7FwHQ6HSqVikOHDvnKHoGgUhjKS6MQi7lrFG4JmFqtpnnz5mKYKKi2WGuSwSg7pErYzEKKwUTA4/Ys5EsvvcSLL77IlStXfGGPQOA2ZTlS9iV1rIeVwgMLfNyOgb377rucPHmSmJgYWrRoQVhYmM3re/bs8ZpxAkFl0Rtk1Err5yIGVpNwW8CGDRvmAzMEAt9gSmYtVTCxmLtm4baAzZo1yxd2CAQ27PjzMq0ahRETGVKp89gns9qshRT6VXXoiyHzEDTr6dXTelxSOjU1laNHjwLQqVMnevTo4TWjBLWbX05e5uEPfwfg7PwhlTqXfQxML9ZCVj0FV2D9w3BhDyR+C02v99qp3Rawixcv8tBDD7F161ZLT8Zr164xYMAA1q1bR6NGjbxmnKB28vsZ700Q2a+H1IlE1qrl71Owdjj8fRI0daEo26und3sW8umnnyY3N5fDhw9z5coVrly5wqFDh8jJyeGZZ57xqnGC2ok365uUPYQUAuZT0n6DFQkm8YqIhQk/QOsBXn0Ltz2wzZs3s2XLFjp06GDZ1rFjR5YsWcJtt93mVeMEtRNvVmgqewjpvfcR2HHwM9j0JBi0ENMDRq6HulFefxuPlhLZL+AGU5KrWGYk8AaSF30wg8MQUhQ09CmyDD8vhJ9eMz1vfxfctxw0oWUf5yEeldOZMmUK6enplm0XLlzgueeeY+DAgV41LhDJK9ZzrUDrbzMEJdg39tCLRFbfodfCpkml4tVnMgz/2GfiBR4I2LvvvktOTg5xcXG0bt2a1q1b07JlS3JycnjnnXd8YWPAIMsyd729nVsXbbNp9SVwD28OIe29LBsBEx6Y9yi8Cv+5D/avBUkJQxbD4NdBoSz/2Erg9hAyNjaWPXv2sGXLFo4dOwZAhw4dSEhI8LpxgUax3sjZv01t7nMK9TSq69svr6bizSC+fU0w20x8L75RbebKGVjzIPz9p2mm8cFV0LZq9MCjPDBJkhg0aBCDBg3ytj0BTYG21OuSEXeHp/jUAxONbb1L2u+wbiQU/A3hzWD0BojqVGVv75GApaSkkJKSwsWLFx0C9ytXrvSKYYFIfrG+9Im4NzzGm41i7GNgNvXAhAtWOQ79F754EgzF0KQ7jFoPdaOr1AS3Y2CzZ8/mtttuIyUlhcuXL3P16lWbhycsWbKEuLg4goOD6d27N7t27XK57+eff06vXr2IjIwkLCyM7t2788knn3j0vt7G1gMTVBVlXeuyPDARxPcQ80zjZ4+YxKvdEFOGfRWLF3jggS1btoxVq1YxZswYrxiwfv16kpKSWLZsGb179yY5OZnBgwdz/PhxGjdu7LB//fr1eemll2jfvj0ajYavv/6axMREGjduzODBg71ik6fka0s9MHFveI43h5D2zW11oh5Y5dBr4evnYN9/TM9vfApum+vzYL0r3PbAtFotffv29ZoBixcvZuLEiSQmJtKxY0eWLVtGaGioy6HoLbfcwr333kuHDh1o3bo1U6ZMoWvXruzYscNrNnlKQbGIgXmD8vLAzlzOZ8bnB0krmTApC4NIo/Aehddgzf0m8ZIUcOdCuP0Nv4kXeCBgjz76KGvXrvXKm2u1WlJTU21mMBUKBQkJCezcubPc42VZJiUlhePHj3PzzTc73ae4uJicnBybh68QHph3KM8DW/dHGp/uSmPD7vPlnst+LaSoB+YhV8/Ch7fBmZ9BU8eUWR8/0d9WuT+ELCoq4oMPPmDLli107drVISt/8eLFFT7X5cuXMRgMREXZLjGIioqypGg4Izs7m6ZNm1JcXIxSqeS9995zOSM6b948Zs+eXWGbKkOhiIFVCUUl19n6B8MVDhVZxWJu9zn/B3z6EBRchroxppnG6C7+tgrwQMAOHDhA9+7dARyae3hz9qgs6taty759+8jLyyMlJYWkpCRatWrFLbfc4rDvjBkzSEpKsjzPyckhNjbWJ3ZZ31BihstzyvsrMpeF1urLD2KVtZhbfEUV4PAm+OJx0BdBdFfTTGN4jL+tsuC2gP30009ee/OGDRuiVCrJysqy2Z6VlUV0tOsZDYVCQZs2bQDo3r07R48eZd68eU4FLCgoiKCgIK/ZXBbWMTCB55T3O2iOazlrXGtPWYu5xY9MGcgy/JIMW141Pb/udrj/Qwiq40+rHHA7BuZNNBoNPXv2JCUlxbLNaDSSkpJCnz59Knweo9FIcXGxL0x0CxED8w7l6ZJ5ZrFiHpjrWUgRA3OBQQf/e6ZUvHo/AQ+trXbiBZWoyOotkpKSGDduHL169SI+Pp7k5GTy8/NJTEwEYOzYsTRt2pR58+YBpphWr169aN26NcXFxXz77bd88sknLF261J8fAxCZ+O4y77ujIMOMO0tLM319IJ1/bXYd/4TS2JXWEw/M6rnIxHdCUTZsGAunt5pmGm+fD70f97dVLvG7gI0YMYJLly4xc+ZMMjMz6d69O5s3b7YE9tPS0lAoSh3F/Px8Jk2axF9//UVISAjt27fnP//5DyNGjPDXRyi1rVh4YBUlu0DH+9tOA/DkLa2JDNUAMHnt3nKPNQ8DPfHARD2wMrh6zlQ99dIxUIfBAyuh3e3+tqpM/C5gAJMnT2by5MlOX9u6davN89dee43XXnutCqxyH5GJX3Gsg+n2y30qemxxRQTMoS+kqAfmlL9S4dMRkH8J6jYxBeubdPO3VeXidgzs559/Rq93nL7W6/X8/PPPXjEqULH1wMTNURaVmbHWexjEN3XqLn1NDCFLOPIlrLrTJF5RXeDRlIAQL/BAwAYMGOC0K3d2djYDBni33nWgITwwz3A3Xqh3I43C2suyF7xa/yMjy/DLW7BhnClNou1geOQ7iGjqb8sqjNtDSFmWnf56/v333w5dumsbBWIWssLY/AW5ea3MQ8iKBPGtRct+OFmrh5AGHXw7FVJXmZ7HPwaD54GyWkSVKkyFrb3vvvsAk+s/fvx4m9wqg8HAgQMHvLpGMhCx8cCEgpWJ9W+gu1fKPohf1qW2Fin7gH6t1a+ibNg4Hk79HyCZZhpvfMLfVnlEhQUsIiICMN2YdevWJSSktGOyRqPhxhtvZOJE/6+N8ic2eWB+tCMQqEzjDneGkNYTBPaTBbUykfXaedNM48UjoA41Jae2v9PfVnlMhQXso48+AiAuLo6pU6fW+uGiM2yqUdTCe8NT3L1WZk+qIrOX1l2J7Bd217pE1gt7TGsa87KgTjSMWmdqeRbAuD3gnTVrli/sqBHYemC17OZwF5shpGdB/IqkUViLnP26SDezNwKbo/+D/04EfSFEdTalSUQ087dVlcbtWcisrCzGjBlDTEwMKpUKpVJp86itGIwyRbrSG6q2/bi7S2XW/ZfGwMpfe1rWLGStGELKMvz6LqwfYxKvNoPgkc01QrzAAw9s/PjxpKWl8corr9CkSZMqq0BR3SmwK+0iBKziuD2ErMAspEohoTfKNsNG+1nHGj+ENOjhu2mw+0PT814T4I43A26msSzc/iQ7duxg+/btlpI6AhPWM5AghpDlYf2z5/YspNGcyOr6SJWyRMDKCOLX6DSKohz4LBFObgEkU4/GGyd5t153NcCjvpAiRcARm45ECA+sPLyRiW8wyi5FSK1QUITRJvfLPohfY7+j7L9gzXC4eBhUIXD/Cuhwl7+t8glux8CSk5OZPn06Z8+e9YE5gYu9B2bN33nF5BWXXz20tuLuD6K1ELlKpVAppZJ9y/DAaqKCpe+F5QNN4lUnytQtqIaKF3jggY0YMYKCggJat25NaGioQ0lpZ8uMagP2AmaOrxRqDdyycCtR4cFsServD9OqHXnFerJyiizP3dURa6/LtYCZfputk1ftE1lr3BDy2Lfw3wmgK4DGHWHUBoj0TfXh6oLbApacnOwDMwIf+/rs5pvy7/xicov05BbluVyGVdvoMecHtytQWGN9bLHBueerMQuYzRDS9j1rTChEluH3ZbB5BiBD61vhwdUQHO5vy3yO2wI2btw4X9gR8NiXkzbfGtZhF63BSJCq9qaamKmMeIF9aoTzc1mGkFZel30aRY0YQhr08P0M2PWB6XnPRLhzASjVZR9XQ/CopPSpU6d4+eWXGTlyJBcvXgTgu+++4/Dhw141LpBw9MBMN4f1VL11npigFHd1xFqIXA4hFY4xMPtEVmOgfx3FubBuZIl4STBoLtz171ojXuCBgG3bto0uXbrw+++/8/nnn5OXlwfA/v37a3WWfoH9LGTJv9YCVlyBxMvaiLspJxWJgaktMTDXs5ABnQeWfQFW3gF//mCaaRz+Mdz0TI1LkygPtwVs+vTpvPbaa/z4449oNBrL9ltvvZXffvvNq8YFEvn2eWAl94Z12KVYeGCVRpZlG69Kqzc6FUDzELKs4WbACljGflgxELIOQlhjSPwGOg71t1V+wW0BO3jwIPfee6/D9saNG3P58mWvGBWI2Gfim30w4YGVjyw7u37OsQ/Ea10E8VUlfRR0ThZzmwP8ATkLeXyzyfPKzYBGHWBiCjTt6W+r/IbbAhYZGUlGRobD9r1799K0aeBUcvQ2+fZBfIsHJmJg5fHKl4foOPN7dp8tPwXHXnS0euci5EykzB5YkMr0WsDp1+/vm2JeunxoNQAmfA+Rzf1tlV9xW8AeeughXnjhBTIzM5EkCaPRyC+//MLUqVMZO3asL2wMCBzWQpb8ax12ER6Yc7b/afLcF/94wuE1+1QH+5lEV+shzUNIZ9UogtRmAQsQBTMa4LsXTOsaZSNcPxZGb4TgCH9b5nfcFrA33niD9u3bExsbS15eHh07duTmm2+mb9++vPzyy76wMSCwj4GZKx0ID6xy2GuMowdWdiKrs3pgATWELM6DdaNMeV4ACbPh7rdr1UxjWbidB6bRaFi+fDmvvPIKhw4dIi8vjx49etC2bVtf2BcwFDos5jYhYmAVx9kEmr3E2AfiXc5CmtMonCzmDlKbcvGqvQOWkw5rR0DmAVAFw73vQ6dh/raqWuFxXY3mzZvTvHntHn9b42oxt/WPvPDA3EeWZcat/INgtYL3x/Ry8JpctVZzthbSnNQaEB5Y5kHTguzcdAhrBCPXQbNe/raq2lEhAUtKSmLu3LmEhYWRlJRU5r6LFy/2imGBhqtyOsIDqzjO6uSnXyti24lLgCnO6BADc2ctpDFAYmAnfjCVwtHmQcN2MHoD1Ivzt1XVkgoJ2N69e9HpdJb/u6I2r/Ozz8Q3j32sq36KPLCycfbnY7/cx95rKnbhgamdZOLr7Dywailgu5aXButb3gzDP4GQSH9bVW2pkID99NNPTv8vKMXlWkibIaTwwCqDLDtm05frgTlZSmT2wKrVENJogB9eht/eMz3v8TAM+TeoNGUfV8vxaC2kwBGzB6YpyTFylgdWkSYUNRFZljl7Od+j6g/2xzgkspa7lMhqMXeJ+JkX1Fcb/dLmw/qHS8Vr4CwY+q4QrwpQIQ/M3NS2Inz++eceGxOoyLJsiYHVCVJxRa91GgOrrUH8RT+c4N2fTvL0rW34523tXO7nLARhrzH2C7JdBfHVToP4pv9bhpDVQcFyMuDTEablQcoguHcZdK74/VbbqZAHFhERYXmEh4eTkpLC7t27La+npqaSkpJiaX7rLkuWLCEuLo7g4GB69+7Nrl27XO67fPly+vXrR7169ahXrx4JCQll7l8VFOuNluFIWJDtFL1IZIV3fzoJwDv/d9LtY60dsNwiN4L4CmdDyJIYmKqaxMAyD8GKBJN4hTaA8V8L8XKTCnlg5qa2AC+88ALDhw9n2bJlljZqBoOBSZMmER7ufgG19evXk5SUxLJly+jduzfJyckMHjyY48eP07hxY4f9t27dysiRI+nbty/BwcH861//4rbbbuPw4cN+W8pkPQMZpjFdUvPNITywylJ6/W6cl0KjukE2r2oNRqfBf7WzemAlYmYWML/WA/tzC2wcD9pcaHidqXpq/Zb+sydAcTsGtnLlSqZOnWrTA1KpVJKUlMTKlSvdNmDx4sVMnDiRxMREOnbsyLJlywgNDXV5rjVr1jBp0iS6d+9O+/btWbFiBUajkZSUFLff21uYlxEFqxUoS2a/RCKr+/yZlVvuPpdyi22ea/VGp22NzHlgRrl0qGgWM8taSH/9nvyxAtYON4lXXD+Y8IMQLw9xW8D0ej3Hjh1z2H7s2DGMbv5FaLVaUlNTSUhIKDVIoSAhIYGdO3dW6BwFBQXodDrq16/v9PXi4mJycnJsHt7G7IGFaVSl3oBYzO02GdlFDtvKc5JcTYyYh5BQOoy05IFZgvhV7IEZDfD9S/DNP0E2QPfR8PDnEFKvau2oQbidiZ+YmMiECRM4deoU8fHxAPz+++/Mnz+fxMREt851+fJlDAYDUVFRNtujoqKciqQzXnjhBWJiYmxE0Jp58+Yxe/Zst+xyF3MWfmiQ0pKMaQniixhYpShPYlwF8c3DRChNl7AE8f0xhNTmw+ePwbGvTc9vfQX6/bPWFSD0Nm4L2MKFC4mOjmbRokWWsjpNmjTh+eef55///KfXDSyL+fPns27dOrZu3UpwcLDTfWbMmGGzeiAnJ4fYWO92arH2wEpGkGWW05FlmUMXcmjZKIw6QTWnS7I/KK+kNJjSJ0JQWnLIgqxSXaqk0UpulmmmMX2vaaZx2HvQ5QHfvmctwe27R6FQMG3aNKZNm2YZjnkSvAdo2LAhSqWSrKwsm+1ZWVlER0eXeezChQuZP38+W7ZsoWvXri73CwoKIigoyOXr3sDigWmUmGf5nVZkLfHAfjySxWOfpNKiQSjbnh/gU9sCnfKcpPISWQEMJV+Kzs4DA9P3o/SlfmUdMcW7ss9DSH0Y+Sk0v9GHb1i7qFQia3h4uMfiBabKFj179rQJwJsD8n369HF53JtvvsncuXPZvHkzvXr5f4Gr2QML1aiwC4HZBvFLPLCv9qcDcO7vgqoyscbiqh6YUiodnZkTWO2D+ODjONjJFFg52CReDdrAo1uEeHkZj8Yvn332GRs2bCAtLQ2tVmvz2p49e9w6V1JSEuPGjaNXr17Ex8eTnJxMfn6+JZ42duxYmjZtyrx58wD417/+xcyZM1m7di1xcXFkZmYCUKdOHerUqePJx6k05iz8UI3S8n9nXYnMHlhZt8zZy/lERwQTrBbt16D8hh+uYmCSJKFWKNAaSnP0SoP4tvExn1zq3R+VButb/ANGfAKhzieaBJ7jtgf29ttvk5iYSFRUFHv37iU+Pp4GDRpw+vRp7rjjDrcNGDFiBAsXLmTmzJl0796dffv2sXnzZktgPy0tzaaE9dKlS9FqtTzwwAM0adLE8li4cKHb7+0tzOsgw4KceWCl+1lmIV3ck6nnrnDLwq3c+fZ2n9gZiHg6CwlYUlr0liGk7VIi8IEHZjTCD6/A18+axKvbSBjzhRAvH+G2B/bee+/xwQcfMHLkSFatWsW0adNo1aoVM2fO5MqV8muaO2Py5MlMnjzZ6Wtbt261eX727FmP3sOXWHtg5oBwaSZ+xfPANu01DS1PX8r3gZU1E1cxMEkqyQXTWaVR2C3mBi+vh9QWwBePw9GvTM8HvAQ3Py9mGn2I2x5YWloaffv2BSAkJITcXFPy4ZgxY/j000+9a12AYJmFtPbAnA4hy84Ds6+0UBNxd0G3q91DNSYvypWAgVVz2xLPy5KJr3RMsag0eRdh9V0m8VJq4L7l0H+aEC8f47aARUdHWzyt5s2bW3pBnjlzxqNqAzWBAhsPzLStrHI6ruI69uWSayLu/om4GuJZBMxVDAzHkjqWIL6VB+aVv9mLx2D5QLiQapppHPsldB1e+fMKysVtAbv11lv56iuTi5yYmMhzzz3HoEGDGDFihNN+kbUBSwxMo3IcQjrxwFzdM3oXN2NtxrUHZop+uAriI0lWHphdIqvVMrhKe2CnfoIPB0F2GtRvbZppbNG3cucUVBi3Y2AffPCBZcnQU089RYMGDfj1118ZOnQojz/+uNcNDAQsMbAgpVUQ35yJb53IaijzF19XHcq7+Bh3P2G5HlhZQ0hLSR3zENL0r9oq8atSlzx1NXyTBEY9NO8LD60Rwfoqxi0B0+v1vPHGGzzyyCM0a9YMMPWJfOihh3xiXKDgbC2ks0RWo2wazrjSMEOtGEK69xldCVhIOQIm4VhSx+yBqZSmRfcGo+zZLKTRCP83B3b82/S8y3C4511Q+TZhWuCIW0NIlUrFm2++iV5fsTbwtQVzJn6IxnotpAn7G6RIZ3AZA6sVQXwv7W8uW6TVG53uI0k4GUKWemDKkl8at4eQukJTww2zePWfDvd9IMTLT7gdAxs4cCDbtm3zhS0Bi3MPTLb510yx3ujSA6sNQfzsQp1b+7vy2MwemKumHmCVB2a0nYVUKRSW78ktDyzvEqy+G45sAoXa1KdxwAwx0+hH3I6B3XHHHUyfPp2DBw/Ss2dPwsLCbF4fOnSo14wLFGxiYHZ/y/a/8GU19qgNHthjH+8ufycrXDlIYSUCpjMYnYqchFRaF9/oxAMrEbcKX/JLx2HNg3DtHARHmuJdcf+o+AcR+AS3BWzSpEmA8/6PkiRhMNS+kjE2s5DYz0La7lvsYsgDNdMDO3nRtkjhnrRrbh3vykEKKRlCyrJjow8wOUX2mfjWMTCFZC54WIFrfnorrB8LxdlQvxWM2ggN27j1OQS+wW0Bc7doYW0g30kemLOS0mCeiXR+npqWRpF67gr3L61YYUpXlDcLCWV1JjLHuWxnIVUKyVL2qNyaYHs+MS0LMuoh9kZ4aC2ENXDjEwh8iShGVUkMRtmyxjHMqraXszwwMOeCOb9p3PVOqjvfHsys9DnKy8QH5wImUeqB6ew8MLVSgcIyhHTxBkYj/PQabF9ket75AbhnCaid150T+IcKC1hhYSEpKSncddddgKlQYHFxaX1ypVLJ3LlzXRYWrKkUWsW0QjVKy9DEWSY+1K7mtt5IcncVxNdYpUI4S2aVJKvekEZTnMw81FRZzUI61S9dEWx6Eg6XtAi8eRoMeFEE66shFRaw1atX880331gE7N1336VTp06EhIQAppr4MTExPPfcc76xtJpSUJJCoVRIBKkUDrOQzjywWrriyiNcOUgqpQKNUkGh0eC6JphVDMw6TqZWlHpgDmkU+Zdh3Sg4/7tppnHo29B9VOU/iMAnVDiNYs2aNTz22GM229auXctPP/3ETz/9xIIFC9iwYYPXDazu5FuKGZoqUdiX07EXq2Kdwe1cqNqMq5w5lUKyxLicDyElm0RW64a4SmVpDMzmB+bSCVgx0CRewRGmMjhCvKo1FRawkydP0qVLF8vz4OBgFFadX+Lj4zly5Ih3rQsAzEms5sRKyW41t/0vvMkDK92W6aQTT02hvGKEFcG1ByahKanr5bTKh3Uiq1G2BPDBtF1pPwt5Zjt8mABXz0K9OJiwBVr2q7T9At9SYQG7du2aTczr0qVLxMXFWZ4bjUab12sLBVYeGOC4FtJ+CGnXWm359tO+NTDAcTULqVIqLJVVXdfFLy2nY+2BqZUKyw+NUQb2rYVP7oWibGgWD4+mQKPrvPgpBL6iwgLWrFkzDh065PL1AwcOWNZH1iask1ihNM676IcT7D9/zTGIb1fU0O/t7X2IVz6aKw9MIVmaczgN4lPqgRmMsiVJ2JwfZoqPyTTevcgUsDfqoNN9MO5/ENbQC4YLqoIKC9idd97JzJkzKSpyHPIUFhYye/ZshgwZ4lXjAgFzEqu5vIvZB7uYW8w9S35xmKYv1tkmstZg/fIKLj0w6xiYiyC+uR6YzlAaA1OXhD2CJR3J6iXE7H/btHO/f8L9H4o0iQCjwrOQL774Ihs2bKBdu3ZMnjyZ664zudjHjx/n3XffRa/X8+KLL/rM0OqK2QMzL22xn2kvL5G1otUZzPv5vIdhNaPsGJjrIaRkVQ/MYDRaZeFLkP83i4pm0kV5FKOkQjH0LejxsG8+gMCnVFjAoqKi+PXXX3nyySeZPn26zQ01aNAg3nvvPYcO27WBAktX7pIgvt3rzpYSlfW6mclr9zCgXWPu79kMo1Fm2Hu/EKxWsv6xGwNGxLxR7dSVB6ZUKCyloV0lsppjYDpDaRC/tSIDVgyki+EMOXIoaQnv07lH7Vu/W1NwKxO/ZcuWbN68mStXrnDy5EkA2rRpQ/36tbeIW4HOvA7S5IEp7MTF/iYuskujMN+g9vt9fSCDrw9kcH/PZly4VsiBv7IBkwAGSss1r4TAyhhCmj0wV70GzGkUhpI0injpKMv5N1zNI1NqzOji55kTLaqnBjIeLSWqX78+8fHx3rYlILGPgZVXjcI+jcL8sjvOyqtfHUaWZWbf09l9g31EodbAnK8PM7hTNLe0a+y187q6LgqptNqEq0x88xBSZzQSdmwj/9G8gQYDNLuBZ3OncKpQ4b2mHgK/UKnO3AKrGFiQqxiY7XP7pUSWumEVfL+fT1xi1a9nWb3zHFfyteUf4AMu5RZzMdd2MmfptlN8uus84z/6A6NR5ulP9/LxznOVfi9X+mJe+QDOPTBJMiWsgsxNactptvU5NJKBFEVfGPc/cpWRJecXAhbIiMXclcTBA7OLgplvkBC1kkKdweFmc/f+eeyTVMv//eE96AxGbnh9CwDLx/ZiUEdT3DP9WqFln9/O/M3/9qd75f1cJcMqFZQZxAcIRs9i9VJuztgBwBL9UD4LH89AdUhpPTAhYAGN8MAqiXUpHcAhim++Qcyv289CuoqBVVfMgg0w8ePdbP/zksM+o5b/7rX3c6XRCkkqDeI7GUJqtNk8ePRp7lPuwICSUzfOY4H+IZQqleV4gBpWwajWIQSsLNL3mWqgl4F1OWlwNgtZ4oFpSpe9WHsVlhiYi/PnF+t5au0ep6/5YzLS3iMa8+Eu8op91yPBlbBLkoSyJEhvv0uclEH/n0fSLGcvOXIIH7VcSHrrB4HSuJjTtZCCgEMImCsuHjXVP191l6nrsgvyi+0z8e2HkKZ/XXlgZkFwdR8t23bKMgNZHXBm54rtpx2E25fvZ0bp5K/3BukYX2hmUSf/HLnBTbhfO5tjoT1taoGZji2nHpggIBAC5oqibFAo4cJuU4WCi8ec7laeByZbxcDAMeBcngOw7/w19+z2Mc7MzS3ynQdWlodkFiEzQxW/8B/NG9ST8rharwtf9fqEP+Vm6A1Gy0ylOTdMKqsemCBgEALmiuY3mhb11m8F19JM3ZdP/eSwm30MTOEijSLYSsCcxsBcDCK3/3nZpYl6g8yvJy9XaZHEqo7VuRIYWZatcu5knlZ+ztuaJQRJer41xPPrP1ajCzGtadQbS+uBmZcSWdqqiSFkQCMErCwatDaJWPO+UJwDax4wdWO2wtLQI8iunE4J9kPIYru+kFcLdMiy62a3ZfHm5mOMWvE7U9btdf/gAMGVYMqYPDANOhapl/FP9WcALNPfzVO6ZzCoQlCaK7IaZAcPzOy9BcrkicA5fhewJUuWEBcXR3BwML1792bXrl0u9z18+DD3338/cXFxSJJEcnKy7w0MrQ9jN0HXEabGDv97Bn6caenHVWDngbkK4pvTLOyHkD+fuETLGd9yOc/9UkSf770AwPeHs9w+1lOc3e6y7LsJBZf6IkOYIZePNfO5X7kdvaxghm4C8/UjkVEgAWqremDWHYmg1F6RyBrY+FXA1q9fT1JSErNmzWLPnj1069aNwYMHc/Gi86B5QUEBrVq1Yv78+URHR1edoaogUxPTW2aYnv/yFmwch6zNL42BmRt62KdRGG1nIV0N9/7945/et9sHOBOUdX+k+ez9XMXAgnPP8ejxx7hRcZRcOYRE3TQ+NQy02ce6sa25nI5Z1JSuSkoLAgq/CtjixYuZOHEiiYmJdOzYkWXLlhEaGsrKlSud7n/DDTewYMECHnroIYKCqriVuyTBLdPhvuWg1MDRr5BX3UU941XA2gMrZwjpoia+2ZOr7jgbchVoDWzY/ZdP3s+ZvvSUjtNzy4M0KE7jL7kh92tfZbuxq80+1k09TI0/rKpRUJoHJkaQgY3fBEyr1ZKamkpCQkKpMQoFCQkJ7NxZuV6C1hQXF5OTk2PzqBRdh8PYLyGkPor0PXwRNJPrpPMu10La54G56gtZE5vaegN7D+xuxa+s1byBpvgqGWEduLd4DifkWIfjJCSrtmpGS89N8xBSIYL4NQK/Cdjly5cxGAwOJXiioqLIzKx8P0Ez8+bNIyIiwvKIjXX8Y3ebFn3h0S3oIlvRTLrMfzWvojz9f4CzNArTv+Y0Cr1RZtfZKw6nNFSyYfBHv5zh3N/5lTpHRajq2122+t9Tyk28o3mXIEnHxaaDWNdpGZeIdHlsaWNb61lIkchak/B7EN/XzJgxg+zsbMvj/Pnz3jlxg9acu/dLfje2p65UCGsehN0rXVajsG7E6gx9JWMxs/93hAELt1bqHBWhqu93WZZRo2eB6n2eV5u6Xn2gH8KBPm8jq0JcHmcqHV1akVVnF8QXiaw1A78JWMOGDVEqlWRl2c6gZWVleTVAHxQURHh4uM3DW+RKdRmjncFm5S0gG+Dr57gr4z0kSr0po10iqyu8EUyuinuxqj0WtTaH1er5PKj6Gb2s4CXdI7yhH42sUFp6OzrDuqChqR5YyRDS7IGJIH6NwG8CptFo6NmzJykpKZZtRqORlJQU+vTp4y+z3KJAa0CLmn+HJcGAlwC4+e91LFW/RTCmtAjz/WFuxOqKynpgVUVVWhkrZXHHrrH0VR4hVw5hgu551hisYqbl5G6orGJgK3acMW2zC+IHyGUXuMCv5XSSkpIYN24cvXr1Ij4+nuTkZPLz80lMTARg7NixNG3alHnz5gGmwL+596RWq+XChQvs27ePOnXq0KZNmyq337IOMlgF/adB/VboP3+C25V/sF6ay6PafyLLpuJ+CgmCVAqXDSj0AVIWoaoSP6+XTvCBZjGR+TlckBvwiPZ5jsvNbeywX0pkjamgoekH41hmrmW7WbiUARQDyy7UcTQjh6ycIqLCg4mJCCE6IthSTsif6A1GjDJ+s8WvAjZixAguXbrEzJkzyczMpHv37mzevNkS2E9LS7Npnpuenk6PHj0szxcuXMjChQvp378/W7durWrzHdZB0uUBVuwvZvif0+imOM2moJksLZ4H1EGSJILUCnJd5KsGylCmKu73IYrfWKxeSpCk44SyDaPzn+MS9WztoDwPTLJ4W9aY65aVemDV57rLskxGdhFH0nM4nJ7DkYxsjmTkcP6KY0UUSYKGdYKIiQwhJiKYmMgQmkQE0zQyhCaRIcREBtMwLKjMYbanGIwyv5021Xz77lAmaqXEl5P/QdNI1zFJX+H3goaTJ09m8uTJTl+zF6W4uLhqtfTDvqktwPmwrtyrncNK9QJaKzJ4KfNZziueRil1I0jlOg4WKENI3yIzSfkV09TrAfjB0JMpRU9RiGOrM1l2Xo3CGpWTm/fUJdNMbWkMrJIme4jeYOTUpXyTSFkEK4drBTqn+zeNDKFpvRAu5RaTfq2QYr2RS7nFXMotZr+LeSmNUkF0RLBF2GIiQ2gSGVwieiaRqxusrpC9siyzJ+0q/9tv6tVgv3Lk9W+O8N7onm5dA2/gdwELZAos5aRLL6MkwTk5mvu0s1mmTqaP8ggfqhdw6JyGIHUvl+fSeykPbMyHvzNlYFt6xfmm0Yqvfj9U6HldtZIRqq0ArNDfwRv60RjLCNOW5YFZDyGtadUwDChdzF0VHlh+sZ5jmTk2QnUsM9dpJVmlQqJt4zp0bBJOx5iSR5NwIkM1ln1kWeZKvpaM7CIuXCsk41oh6dlFpF8rJP1aIRnZRWTlFKE1GEm7UkDalQKXttUNUtkJWzBNIkxiFxMZTG6Rnq8PZPC//elcsKq6Gxmq5o7OTejZoh4v/PcA3x7MZMefl/lH26ptCiwErBLkFzt6YOZM/GzqMFY3neVhH3NL0Ra6H5jL5KBhTOUBpzelvpJ5YGa2/3mZ7X9e5ux83zQZdlU1ozKEk8976mT+oTyMQZaYpR/PfwyDyrWkzBgYOAwh4+PqM2eYqRGKWdu8nUZxMbfIJFIlQnUkPYezf+c7Ff4wjdIiUKZ/I2gbVafcrlOSJNGgThAN6gTRuWmE0330BiNZJd6a6VFERnbp/9OzC7lWoCO3WM/xrFyOZ+U6PY+9vYM7RXN3txhuatPQEvc6nJ7NR7+cZeZXh9g85eYqjYcJAasE9gu5wTYTX4eK5DrP8UdufZ5Xb+C+4k3UUZ9nis5xWFRbY2DNpIt8pF5AW8UF8uRgJuueZquxR7nHmYaQtgKlVko2Kxqsh5AapYI1E3tblhdVNhPfYJQ5+3e+jVd1JD3H5aL8qPAgOjYJp1NMhEW0mtcP9UmMCkyz3k0jQ8qMSxVo9Y7CVuLBpV8rJD27EFmGW9s3Zmi3GAa0b+xUXJ8bdB3/25/B6Uv5fLjjDE/e0tonn8kZQsAqQWktMKshpN0+ellmiWEYA2+6kc67XuA2ZWrJDOVULloFpqtzDOyPs1dYsf00M+/u5FX/q7t0kuWahTSScsiQ6/OI9nmOyi0qfLz9EDJErURnMH0nkiRZklYBrouuYxEv62MrctmLdAaOZ+aWBtbTTUNAcwzUGkkyDVOthapjTDgN61Tx2t0KEKpR0aZxHdo0ruP0dVmWMTr5obAnPFjNjDva88+N+3nn//5kWI8YmkRUTUBfCFglKK0FZu2B2X7Z5thWVuydfH9eyWPpL9NVcYZNQa8wweqGrc4e2IPLTGtTvz+cxf/9s79XznmH4nf+rX6PYEnHIWMcE7RTyaLicTtzPTBrQjRKckqqw0rYemAdm9gmMLvKxL+Sry0Z/mVbhoKnLuU5FbpgtYL20SaB6lQiVu2i69r8oAUykiThZCLXKfdd35RPd6Wx+9xVXvvmKEtGXe9b40qoGVfaTzj1wOy+cHMhPUmSSA/vxrCzc/hIvYA2inQ2amZbhkyBspj78z0XKnkGmceVXzND/SkAWww9eEb3NAVOZhrLPItcGog3Y/oeSodw1jGwTjG2sSLzoUczclj0w3FLzCoj27bfpZkGYRqboHqnmHBaNqxTrndSW5AkiTn3dOaud7bzzYEMRsVf5qY2vg/oCwGrBKW1wByD+GbMQ0OlQiJUreS8HMV92ldZqn6Lm5SH+VC9kFf14/ja6Juge2WQZdnBo3z3p5Men0+FnjmqjxilMpXm/kg/mLn6MWXONJaFffzIOj5jWgtp5YHF2HlgJZ8r5dhFUo7Z1p+LaxBa4lVFWIaAjesGOVwLgS0dY8IZc2MLVu88x6yvDvPtM/18HtAXAlYJLJn4Vh6Y/SydeQipkEo7F+VQh/G6F3hNNqUNzFWv4jrDRWYxyuOb2du8v+0U7/98mkEdosrfuQLUpYAl6re4WXkQgywxRz+W1YbBHp9PRnbIA7NfMB+iViJJpnhX++i6Nq/d2qEx3x7MoH4dDZ2aRFi8qw5NwqkTJG4LT0m6rR1fH8jg5MU8Vv16hsdu9m1AX3xTlcAhEx/HfC7zEFIhSTY3mA4VL+gnck6OYpp6PWOkb4lWZzJFN9nt4ZQvmPedqQvT+t2Vr97RlEus1CygneIv8uUgntY9zf8ZKxcjkWXHIL79bHDdYDXz7+tCiEblkLDZt3VDfp1hW8FVUHkiQtRMv6M9z392gLe2/Mk93ZsSFe67v+fq8XMfoFgy8a2GkPb5XKUxMJwEdyXeM9zDU9pnKJbVDFLuYYNmDlE41gsLVLpJJ9kUNJN2ir/IlOsxXDur0uIF0Kd1AwcBsxlClgzlR9zQnKHdYir9foKKc//1zbi+eST5WgOvf3PUp+8lBKwSWDLxy/DArGNgrkrqfGO8kYe0L3NZDqez4iybgmbSUTrrG6OrkMGKXazTvEYjKZsjxhYMK57DYTnOK+duWCfIIYBeXs01QdWgUJgC+goJvtqfzs5Tf/vuvXx25lqAs0x8+3yu0hiYVOYNtlduyzDtHP40NqWJdIWNmtkMUHjeLu1fm48RN/0bNvzhpQKObiEzUfk1S9VvESJpSTH04EHtTDJpUOkzd2sWwY4XBgDO88BKLQiMWd2aSuemEYzubUoRmvXVIctIxNsIAfMQg1GmUFcBATNaDSHLCQ7/JTfmfu2r7DB0IkwqZoV6IeOU33tk39KtpwCY9t8DHh3vKUoMvK5ayUvqtSgkmVX623hMl0Q+3klsfGZgW5rVCzW9l5M8MDNFusAoT1STmXpbO+qHaTiRlcfqX8/65D2EgHlIoVV7NOvF3Pa17c35XUrJlEZRHjmEMV73Auv0t6CUZGarVzNLtRoFVXdDHs3wrPFJHQpYqV7AaFUKRlniVd1YXtWPx4D3hnbW0/L2s5DW1T4KnWTJC6qWiFA1L9zeDoDkLX9yMcd5jl1lEALmIQUlKRTmQoVmXFWVUCjKHkJao0fFdP1E5useAiBR9T0fqBcRivf/AJxxx1vb3T4mhsts1Mymv/IABXIQj+mSWGW43eu2dbDKqLcfQlo/LXTRf1NQtTzYM5busZHkFestM9veRAiYh+RbpVBYJzi6WtOokGyHOOUjscwwlCe1UyiS1SQo97JRM5to3A+I/je1Yj0bP9l5lqkb97t9/i6SqXhjB8V5suRIhmtfYYvRs9pQoRold3R23RPBek1hWVnwrhoIC6oWU0C/E5IEX+y9wK4z3p1hFwLmIZYk1iBbUXIVrJQkyaM1ct8Ze/OQ9hUuyeF0UpxjU9BMOkln3DrHPzfu588KlEt55cvDfFZBsTNzm+IPNmjm0Fi6xlFjc4YVz+WQ3KrCx/9ml4v1bEJblj7ck1vaNbJsCw92ft3slxJZIwSs+tC1WSQj45vTtrH3l14JAfMQZ0ms4HpRtrKcWciy2Ce34V7tXE4YmxItXWWjZg4DFalunSPT6/EHmQnKb1mmTiZE0vKToRsPaGeR4eZMY3SEfVkh0783WBVkjAh1XjXUfilRXlFpd3Ox7Kd68dKdHfh2Sj96tqhX/s5uIATMQyy1wOw8MJcxsEoIGMBfciMe0L7KdkNnQqVilqsXk6j8jor2CfJmtQslBuaoVvGK+j8oJJlP9Ak8qpvqlZlGc4XUf1gtBK4T5FzA7H/NT13K4+UhHegeG8mYPhUvyyPwPWFBKptyRt5CCJiHlNbDt/XAXFVWlSQIqsAsZFnkEEaibhpr9QNQSDKz1J8wW7UKJeUPl7xVOjmMQlaoFzJW9SNGWWKu7mFe0Sd6babRLLTWlUab1XMujMF2PQb+vJjHo/1asempmwivYK13QWAjBMxDzDGwMDuvylUQX6mwr1PhGXpUvKh/lDd0IwEYp/qRFeqFhOHYucYab1SsbsLffKaZzQDlfgplDU/onuVDw504lnH0HLNXpVRI7HppINunDaCuixhYp5hwm+qfPZt7d3giqP6IxdweUroO0s4DK2MI6T0kPjDcTZocRbJ6CQOU+/lMms0j2uddxqAKKhnU7iSdYaVmAVHSNS7KkUzQTuWgG8H6imI99Gtc1xQfcxWsVygkXri9PYl949iw+zwjbmjudD9BzUV4YB5iKWZoNyx0FWvyRd27zcZ4Rmhf4ZIcQQdFGpuCXqGzdNrpvubMfFf8ftp1esZARSobNXOIkq5xzBjLsOI5PhEvwOnQr3n90DKPaRwezORb29KobvUr2yzwLULAPKS0nLStB6ZzGQNzVLARvWIrbcd+uQ3Diudw3NiMKOkaGzRzGaTY7bDfsUzX2fWf7kpjxAe/OX1tvHIzy9WLCZWK+dnQhQe0s0inaltnTby5FQ/f2JzVj8RX6fsKqj9CwDwk30lHIigjjcLOBZt7TyeG3+AoYLe2b+y2LRcwzVD+bOhCqFTM++p/M0H5LdYzlK5i+DtP/c2Mzw862ouBV1WreFX9MQpJZo1+II/oniePsr0hT3j9XlOrs/fHOE9+DVYreW1YF/pf18jp64LaixAwD3HlgVW0Qa2rUruuAtbWTB7QxqHLTS6hPKJ7njX6gSgkmVfU/2Gu6iObGcpZXx4iu9C283PqOcfM6DAK+UC9mPGqHzDKEq/rRvGS/hH0PgqZju7dgrPzhzC4k+sMfIHAGSKIXwZPrdlD26g6XN+8Hq0b1+FqvpbUc1dJPXeVbScuAY4emKs0CnvB0qgUli7RZuIauPZuIkPVtGwYxtnL+TzevxW7zlxx6EGoR8VL+kc4I0fzomotY1RbiJUuMVn3NHmEsnrnOQq0BhY82M1yzLFM2wz9aP5mpWYhHRXnKJQ1PKubxPfGyg3dOjQJ52hGDh+O68WE1Y7DW4HAU4SAueDAX9f45mAGOI6uLISolQ6Zxc48sIUPdqNpZIgl9QJAo1RSL0zDtudvIVit5OzlfNo3CWfml4ecvpfBILPh8T4YZZkglRKty/pKEisMQ0iTG/OWegm3KPezUZrNBO3zpNOQjal/kXhTSzrGmETl6wMZliM7SWf5ULOAaOkql+QIJminckB2v6Z560ZhnLqUb3n+1eSbyCnU0aBOEGfnD+HHI1ls3H2eO7oIj0tQOcQQ0gUKSWJgSTyqTeM6qBQSdYNU3HxdI5IGXceaR3uz++UEujaLtDlu9I22GeAz7+rIAz2bOZzf7JG1aBBGVHgwvVs1ICJEbZNR9fPzAyz/1xtl1EqFpWSMVl92YtcPxhsYrp3JRTmSDorzbAqaSZeSGco7396OLMvs+POyZf9bFXvYoJlNtHSV48ZmDCue45F4jevTwknHbAUNrIa8gzpG8cHYXtzbw/G6CATuUC08sCVLlrBgwQIyMzPp1q0b77zzDvHxroctGzdu5JVXXuHs2bO0bduWf/3rX9x5551etalz0wg+HH+D5bneYEQhSeW2gn/m1jb0adWAsCAl+85fs1SlBFMwOjo8mJwiXYXWhDW3GlIa7KLwj93cimfX7yvz+INyK4YVz+FDzQI6KM6zQTOHZ3VP8b3xBrYev8SSraYWaWOV3zNL9TFKSeZnQxee0k0h18Ng/aCO0ewsIyVDIPAmfvfA1q9fT1JSErNmzWLPnj1069aNwYMHc/HiRaf7//rrr4wcOZIJEyawd+9ehg0bxrBhwzh0yPnQy1uolIpyxcu8X5/WDejaLJKxfeJsvBGlQuL7527mtxcHUj9MU6H31ZSsH2sXZdsWbFiPpnw3pV+5x6fTkAe1s9hq6EaIpGWpOpmJyq9J3nKCnIJiZqlWM0e9GqUk86l+AI/onncqXiPjY+nWLIJ/j+jGl0/d5PB66ssJ/PfJPvyjbdWmWAhqN5Ise2mRnIf07t2bG264gXfffRcAo9FIbGwsTz/9NNOnT3fYf8SIEeTn5/P1119btt144410796dZcuWlft+OTk5REREkJ2dTXh4eLn7VzVf7U/nmU/3UjdIxcHZgzmemcvy7aeZMrAtsU4SOv+6WsA//mVqFHtvj6YsHt6Nw+k53PXODpv9TGkRqxmj2gLAWv2tNJKuMUi5B4B5upG8b7gLV8uCtk8bYPP+W49fZPxHf1ien51f2ph3wffHWPKTKXF2eK9mvPlA6aSBQFAe7tyjfhUwrVZLaGgon332GcOGDbNsHzduHNeuXePLL790OKZ58+YkJSXx7LPPWrbNmjWLTZs2sX+/YzG+4uJiiotLZ+tycnKIjY2ttgImyzI7Tl6mfXR4hTPLi3QGDl7I5vrm9Swen9Eo8+upv3lk1R9WAX+ZCcrveEm1BoVk+tqLZDXP6SbxnbG303O3bVyHDY/3oZ4Tj7FIZ2D2/w6T0CGKgVYNcC/mFDF25S7aR9cl+aEebnx6gcA9AfNrDOzy5csYDAaiomy7P0dFRXHsmPPys5mZmU73z8zMdLr/vHnzmD17tncMrgIkSaJfW/cSNoPVSpv6WWBaJ/iPtg05PGcwSkkiu1DHM+v28uGfd1pmKPMJYqJ2KvvkNpbjxveN457uMdQNVhMVHkSoRuWyCF2wWsm8+7o6bG8cHszmZ2926zMIBJ5QLYL4vmTGjBkkJSVZnps9sNqCuQZTvTANn0wweVlFutv549goGkeG80WzxqL4nyBg8auANWzYEKVSSVZWls32rKwsoqOd5whFR0e7tX9QUBBBQWKRrzXBaiX9urT1txkCQaXx6yykRqOhZ8+epKSkWLYZjUZSUlLo06eP02P69Oljsz/Ajz/+6HJ/gUBQc/H7EDIpKYlx48bRq1cv4uPjSU5OJj8/n8TERADGjh1L06ZNmTdvHgBTpkyhf//+LFq0iCFDhrBu3Tp2797NBx984M+PIRAI/IDfBWzEiBFcunSJmTNnkpmZSffu3dm8ebMlUJ+WloZCUeoo9u3bl7Vr1/Lyyy/z4osv0rZtWzZt2kTnzp399REEAoGf8HseWFVT3fPABILajjv3qN8z8QUCgcBThIAJBIKARQiYQCAIWPwexK9qzCG/nBzXNeIFAoH/MN+bFQnP1zoBy801VSCtTdn4AkEgkpubS0RERJn71LpZSKPRSHp6OnXr1q3VS2jMS6rOnz8vZmPdQFw3z6notZNlmdzcXGJiYmxSqJxR6zwwhUJBs2aiEqiZ8PBwcSN6gLhunlORa1ee52VGBPEFAkHAIgRMIBAELELAailBQUHMmjVLVOpwE3HdPMcX167WBfEFAkHNQXhgAoEgYBECJhAIAhYhYAKBIGARAiYQCAIWIWA1mCVLlhAXF0dwcDC9e/dm165dZe6/ceNG2rdvT3BwMF26dOHbb7+tIkurF+5ct+XLl9OvXz/q1atHvXr1SEhIKPc611Tc/Xszs27dOiRJsmmtWGFkQY1k3bp1skajkVeuXCkfPnxYnjhxohwZGSlnZWU53f+XX36RlUql/Oabb8pHjhyRX375ZVmtVssHDx6sYsv9i7vXbdSoUfKSJUvkvXv3ykePHpXHjx8vR0REyH/99VcVW+5f3L1uZs6cOSM3bdpU7tevn3zPPfe4/b5CwGoo8fHx8lNPPWV5bjAY5JiYGHnevHlO9x8+fLg8ZMgQm229e/eWH3/8cZ/aWd1w97rZo9fr5bp168qrV6/2lYnVEk+um16vl/v27SuvWLFCHjdunEcCJoaQNRCtVktqaioJCQmWbQqFgoSEBHbu3On0mJ07d9rsDzB48GCX+9dEPLlu9hQUFKDT6ahfv375O9cQPL1uc+bMoXHjxkyYMMHj9651i7lrA1XR8bwm4sl1s+eFF14gJibG4cegJuPJdduxYwcffvgh+/btq9R7CwETCLzE/PnzWbduHVu3biU4ONjf5lRbcnNzGTNmDMuXL6dhw4aVOpcQsBpIVXQ8r4l4ct3MLFy4kPnz57Nlyxa6du3qSzOrHe5et1OnTnH27Fnuvvtuyzaj0QiASqXi+PHjtG7dukLvLWJgNRDR8dwzPLluAG+++SZz585l8+bN9OrVqypMrVa4e93at2/PwYMH2bdvn+UxdOhQBgwYwL59+9yrluzJjIOg+rNu3To5KChIXrVqlXzkyBH5sccekyMjI+XMzExZlmV5zJgx8vTp0y37//LLL7JKpZIXLlwoHz16VJ41a1atTaNw57rNnz9f1mg08meffSZnZGRYHrm5uf76CH7B3etmj6ezkELAajDvvPOO3Lx5c1mj0cjx8fHyb7/9Znmtf//+8rhx42z237Bhg3zdddfJGo1G7tSpk/zNN99UscXVA3euW4sWLWTA4TFr1qyqN9zPuPv3Zo2nAibK6QgEgoBFxMAEAkHAIgRMIBAELELABAJBwCIETCAQBCxCwAQCQcAiBEwgEAQsQsAEAkHAIgRMIBAELELABNWC8ePHI0mSw+P222/3t2mCaoyoRiGoNtx+++189NFHNttcdXHW6XSo1WqbbVqtFo1G4/b7enqcwP8ID0xQbQgKCiI6OtrmUa9ePQAkSWLp0qUMHTqUsLAwXn/9dV599VW6d+/OihUraNmypaUGV1paGvfccw916tQhPDyc4cOH25R6cXWcIPAQAiYIGF599VXuvfdeDh48yCOPPALAyZMn+e9//8vnn3/Ovn37MBqN3HPPPVy5coVt27bx448/cvr0aUaMGGFzLvvjBIGJGEIKqg1ff/01derUsdn24osv8uKLLwIwatQoEhMTbV7XarV8/PHHNGrUCDDVMDt48CBnzpyx1JX6+OOP6dSpE3/88Qc33HCD0+MEgYkQMEG1YcCAASxdutRmm3VzDGfFAlu0aGEjQkePHiU2NtamKF7Hjh2JjIzk6NGjFgGzP04QmAgBE1QbwsLCaNOmTZmvV2RbRd9LEPiIGJigRtGhQwfOnz/P+fPnLduOHDnCtWvX6Nixox8tE/gC4YEJqg3FxcUObdxUKpVbnWsSEhLo0qULo0ePJjk5Gb1ez6RJk+jfv3+trFdf0xEemKDasHnzZpo0aWLz+Mc//uHWOSRJ4ssvv6RevXrcfPPNJCQk0KpVK9avX+8jqwX+RJSUFggEAYvwwAQCQcAiBEwgEAQsQsAEAkHAIgRMIBAELELABAJBwCIETCAQBCxCwAQCQcAiBEwgEAQsQsAEAkHAIgRMIBAELELABAJBwCIETCAQBCz/D3jK4Mjig5fLAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019727, + "end_time": "2024-03-22T18:25:12.481176", + "exception": false, + "start_time": "2024-03-22T18:25:12.461449", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 5438.254231, + "end_time": "2024-03-22T18:25:15.223561", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tvae/42/mlu-eval.ipynb", + "output_path": "eval/contraceptive/tvae/42/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/tvae/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "tvae" + }, + "start_time": "2024-03-22T16:54:36.969330", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/tvae/model.pt b/contraceptive/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..41fe7c1acba033c130e13335d5fc2b930f046c9a --- /dev/null +++ b/contraceptive/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdb996dcc92c0d5229ad2651364afc4a47042431d9167333d68206233446423e +size 47629899 diff --git a/contraceptive/tvae/params.json b/contraceptive/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..9d2148e27d49c137a0277e6c769ca4462447d066 --- /dev/null +++ b/contraceptive/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.73, "gradient_penalty_mode": "NONE", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "head_activation": "softsign", "loss_balancer_beta": 0.67, "loss_balancer_r": 0.943, "tf_activation": "tanh", "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.09, "n_warmup_steps": 100, "Optim": "amsgradw", "fixed_role_model": "tvae", "mse_mag": false, "mse_mag_target": 0.65, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "prelu", "tf_d_inner": 512, "tf_n_layers_enc": 3, "tf_n_head": 32, "tf_activation_final": "leakyhardtanh", "tf_num_inds": 128, "ada_d_hid": 1024, "ada_n_layers": 9, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 256, "head_n_layers": 9, "head_n_head": 32, "head_activation_final": "leakyhardsigmoid", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/lct_gan/eval.csv b/insurance/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..2ae9fc2576f1d64941f4e035394dad996aefc99e --- /dev/null +++ b/insurance/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.0014364243935034166,0.007374788956466156,0.0008153657148691958,6.472357988357544,0.0165903028100729,0.5433363318443298,0.05834271013736725,1.2863068832302815e-06,2.436617374420166,0.017312856391072273,0.7524448037147522,0.028554610908031464,0.1444527804851532,0.00022781474399380386,8.90897536277771 diff --git a/insurance/lct_gan/history.csv b/insurance/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..3dfdc91f284a1b50fd4fb18344dc4f115f695a6e --- /dev/null +++ b/insurance/lct_gan/history.csv @@ -0,0 +1,21 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.03844140047931837,0.7188729810216106,0.013317291441673443,0.0,0.0,0.0,0.0,0.0,0.03844140047931837,900,113,110.54995560646057,0.9783181912076157,0.12283328400717841,0.11123340914800631,0.008303070723971663,2.3807693450168474,9.48858276775326e-05,0.0,0.0,0.0,0.0,0.0,0.008303070723971663,450,57,45.44606399536133,0.7972993683396724,0.10099125332302518,0.034816826579340716 +1,0.00653058495786455,0.9053095802807317,0.00012669656981902897,0.0,0.0,0.0,0.0,0.0,0.00653058495786455,900,113,110.65184760093689,0.9792198902737778,0.12294649733437432,0.08656557510087712,0.006352552234764315,0.8825915555632886,2.5333444609595783e-05,0.0,0.0,0.0,0.0,0.0,0.006352552234764315,450,57,45.84219408035278,0.8042490189535576,0.10187154240078396,0.04755824741400909 +2,0.005102309041556307,0.5227623938307345,0.0001850789159238102,0.0,0.0,0.0,0.0,0.0,0.005102309041556307,900,113,111.50907039642334,0.986805932711711,0.12389896710713705,0.09074666479651906,0.004312427961267531,0.17915711068516546,0.0001109809016019134,0.0,0.0,0.0,0.0,0.0,0.004312427961267531,450,57,45.79855394363403,0.8034834025198954,0.10177456431918674,0.07046655134150856 +3,0.004384625833740251,0.5408107707275669,0.0001490451962195263,0.0,0.0,0.0,0.0,0.0,0.004384625833740251,900,113,111.38601303100586,0.985716929477928,0.12376223670111762,0.08961118024205213,0.004137574709020555,0.30141546075263476,7.981840502843144e-05,0.0,0.0,0.0,0.0,0.0,0.004137574709020555,450,57,45.67794322967529,0.8013674250820226,0.10150654051038954,0.06743310128845144 +4,0.00368658812250942,0.46921445248119603,2.151566522561455e-05,0.0,0.0,0.0,0.0,0.0,0.00368658812250942,900,113,111.5419237613678,0.9870966704545823,0.12393547084596422,0.09260421899040189,0.0039208269826809155,0.39560824827086744,0.00012955872286843473,0.0,0.0,0.0,0.0,0.0,0.0039208269826809155,450,57,46.108590602874756,0.8089226421556974,0.10246353467305501,0.06722364254937131 +5,0.0037500500336237665,0.38660638477140624,8.546291791805232e-05,0.0,0.0,0.0,0.0,0.0,0.0037500500336237665,900,113,110.84228444099426,0.980905172044197,0.12315809382332696,0.0917343535847896,0.0035700482581887626,0.11842851034616386,0.0002132376919723574,0.0,0.0,0.0,0.0,0.0,0.0035700482581887626,450,57,48.897982120513916,0.8578593354476126,0.10866218249003093,0.0700296791874918 +6,0.0031003811301585906,0.2770809364794333,5.226455323332238e-05,0.0,0.0,0.0,0.0,0.0,0.0031003811301585906,900,113,113.76307344436646,1.006752862339526,0.12640341493818494,0.09047676906385253,0.0031285733293690203,0.25899296287125934,6.688645929435053e-05,0.0,0.0,0.0,0.0,0.0,0.0031285733293690203,450,57,45.767940282821655,0.8029463207512572,0.1017065339618259,0.07119475702117932 +7,0.002322382250438548,0.34646086611721244,1.220678437067165e-05,0.0,0.0,0.0,0.0,0.0,0.002322382250438548,900,113,109.93844747543335,0.9729066148268438,0.12215383052825927,0.09019952334991072,0.002956038533043789,0.4516964087274689,0.00018173267813042553,0.0,0.0,0.0,0.0,0.0,0.002956038533043789,450,57,46.75201344490051,0.8202107621912371,0.10389336321089003,0.07300392768637833 +8,0.0017050553576781467,0.2169923465751567,1.014655068901434e-05,0.0,0.0,0.0,0.0,0.0,0.0017050553576781467,900,113,114.35391974449158,1.0119815906592176,0.12705991082721285,0.09638774892793292,0.0030709211031595867,0.11738263850605092,0.0002604710703875041,0.0,0.0,0.0,0.0,0.0,0.0030709211031595867,450,57,45.301653146743774,0.7947658446797153,0.10067034032609727,0.07127821202831049 +9,0.0015025745798872474,0.20518340586134937,6.9952671160537674e-06,0.0,0.0,0.0,0.0,0.0,0.0015025745798872474,900,113,111.49351572990417,0.9866682807956122,0.12388168414433798,0.09498348522239027,0.0029102627569550857,0.35421860132333804,0.00010688639051289546,0.0,0.0,0.0,0.0,0.0,0.0029102627569550857,450,57,44.809088468551636,0.7861243590973971,0.09957575215233697,0.06238056484021639 +10,0.0015078903491828695,0.20325854836928797,1.2002418886963234e-05,0.0,0.0,0.0,0.0,0.0,0.0015078903491828695,900,113,110.71620535850525,0.9797894279513739,0.12301800595389473,0.09301089175638899,0.0021557584258051874,0.1434766953186761,6.070327296319708e-05,0.0,0.0,0.0,0.0,0.0,0.0021557584258051874,450,57,45.4062922000885,0.7966016175454123,0.10090287155575223,0.07540236165126164 +11,0.0009480594642486217,0.0989064481881752,1.0000245586616449e-06,0.0,0.0,0.0,0.0,0.0,0.0009480594642486217,900,113,110.84421229362488,0.9809222326869458,0.12316023588180541,0.09785350819274916,0.001960534858582024,0.5104609808862529,4.5300757508451246e-05,0.0,0.0,0.0,0.0,0.0,0.001960534858582024,450,57,45.625545501708984,0.8004481666966489,0.10139010111490886,0.06512660700255972 +12,0.0007876609438487018,0.10056789984725945,2.650213134493559e-06,0.0,0.0,0.0,0.0,0.0,0.0007876609438487018,900,113,111.02137565612793,0.9824900500542295,0.12335708406236437,0.09717012973156124,0.00240250650444068,0.20876344482377662,0.00015140544695302596,0.0,0.0,0.0,0.0,0.0,0.00240250650444068,450,57,45.895328760147095,0.8051812063183701,0.10198961946699354,0.06988455496499674 +13,0.001071412844466977,0.08979928175546043,2.8548677461531438e-05,0.0,0.0,0.0,0.0,0.0,0.001071412844466977,900,113,110.40404558181763,0.9770269520514834,0.12267116175757514,0.09673032805785141,0.002080558299649415,0.33820552927165826,1.7519274976611518e-05,0.0,0.0,0.0,0.0,0.0,0.002080558299649415,450,57,45.13530921936035,0.7918475301642167,0.10030068715413411,0.06275787556609302 +14,0.0009464439855623318,0.11049716685496896,1.5092843134298528e-06,0.0,0.0,0.0,0.0,0.0,0.0009464439855623318,900,113,110.10486841201782,0.9743793664780338,0.1223387426800198,0.0963555359306325,0.0019128535713470126,0.2881092154617132,9.992767801322522e-05,0.0,0.0,0.0,0.0,0.0,0.0019128535713470126,450,57,45.23128914833069,0.7935313885672051,0.10051397588517931,0.07075215538293776 +15,0.0007565623041591607,0.10640527475413109,6.627467923142204e-07,0.0,0.0,0.0,0.0,0.0,0.0007565623041591607,900,113,110.25560998916626,0.9757133627359846,0.12250623332129584,0.09287066381853239,0.0023708047673830558,0.21841555112681696,0.00019829534488743176,0.0,0.0,0.0,0.0,0.0,0.0023708047673830558,450,57,45.1703155040741,0.7924616755100719,0.10037847889794244,0.0748492696673789 +16,0.0013394151283298722,0.16507210116501242,1.1373421445702734e-05,0.0,0.0,0.0,0.0,0.0,0.0013394151283298722,900,113,110.16077661514282,0.9748741293375471,0.12240086290571425,0.0946754277148078,0.00335533164971922,0.19024144669157844,1.7741167042891626e-05,0.0,0.0,0.0,0.0,0.0,0.00335533164971922,450,57,45.20561957359314,0.7930810451507568,0.10045693238576253,0.0715425031161622 +17,0.0012095504338503816,0.17023994228466374,2.78441173262379e-06,0.0,0.0,0.0,0.0,0.0,0.0012095504338503816,900,113,110.36352491378784,0.9766683620689189,0.1226261387930976,0.09928367804505128,0.0016752594984912625,0.32005773657970704,1.843289076943494e-05,0.0,0.0,0.0,0.0,0.0,0.0016752594984912625,450,57,45.38660478591919,0.7962562243143717,0.10085912174648709,0.06659886517088141 +18,0.0006933588838890298,0.12216246798768343,7.663908191116571e-07,0.0,0.0,0.0,0.0,0.0,0.0006933588838890298,900,113,110.25478553771973,0.9757060667054843,0.12250531726413302,0.09808128885100637,0.0015123529919138592,0.2243189437115992,5.470234152319095e-05,0.0,0.0,0.0,0.0,0.0,0.0015123529919138592,450,57,45.31874370574951,0.795065679048237,0.10070831934611002,0.07162666100224382 +19,0.0004732113509251374,0.04918161084480092,2.0673360931155154e-07,0.0,0.0,0.0,0.0,0.0,0.0004732113509251374,900,113,110.03912854194641,0.9737975977163399,0.12226569837994046,0.09703757702908684,0.0011618334987562977,0.31888309370694506,6.430942659246202e-07,0.0,0.0,0.0,0.0,0.0,0.0011618334987562977,450,57,45.194119930267334,0.7928792970222339,0.1004313776228163,0.07293911375464839 diff --git a/insurance/lct_gan/mlu-eval.ipynb b/insurance/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..c202d2f43fff41f078112fa906c56462e00e4a1c --- /dev/null +++ b/insurance/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2463 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:36:52.907524Z", + "iopub.status.busy": "2024-03-22T18:36:52.907234Z", + "iopub.status.idle": "2024-03-22T18:36:52.939564Z", + "shell.execute_reply": "2024-03-22T18:36:52.938763Z" + }, + "papermill": { + "duration": 0.047297, + "end_time": "2024-03-22T18:36:52.941606", + "exception": false, + "start_time": "2024-03-22T18:36:52.894309", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:36:52.966797Z", + "iopub.status.busy": "2024-03-22T18:36:52.966437Z", + "iopub.status.idle": "2024-03-22T18:36:52.973144Z", + "shell.execute_reply": "2024-03-22T18:36:52.972285Z" + }, + "papermill": { + "duration": 0.021482, + "end_time": "2024-03-22T18:36:52.975165", + "exception": false, + "start_time": "2024-03-22T18:36:52.953683", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:36:52.998914Z", + "iopub.status.busy": "2024-03-22T18:36:52.998296Z", + "iopub.status.idle": "2024-03-22T18:36:53.002142Z", + "shell.execute_reply": "2024-03-22T18:36:53.001384Z" + }, + "papermill": { + "duration": 0.017741, + "end_time": "2024-03-22T18:36:53.003966", + "exception": false, + "start_time": "2024-03-22T18:36:52.986225", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:36:53.027151Z", + "iopub.status.busy": "2024-03-22T18:36:53.026914Z", + "iopub.status.idle": "2024-03-22T18:36:53.030713Z", + "shell.execute_reply": "2024-03-22T18:36:53.029920Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017624, + "end_time": "2024-03-22T18:36:53.032641", + "exception": false, + "start_time": "2024-03-22T18:36:53.015017", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:36:53.056192Z", + "iopub.status.busy": "2024-03-22T18:36:53.055936Z", + "iopub.status.idle": "2024-03-22T18:36:53.061288Z", + "shell.execute_reply": "2024-03-22T18:36:53.060441Z" + }, + "papermill": { + "duration": 0.019219, + "end_time": "2024-03-22T18:36:53.063225", + "exception": false, + "start_time": "2024-03-22T18:36:53.044006", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e6b4735d", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:36:53.087773Z", + "iopub.status.busy": "2024-03-22T18:36:53.087488Z", + "iopub.status.idle": "2024-03-22T18:36:53.092294Z", + "shell.execute_reply": "2024-03-22T18:36:53.091497Z" + }, + "papermill": { + "duration": 0.019155, + "end_time": "2024-03-22T18:36:53.094189", + "exception": false, + "start_time": "2024-03-22T18:36:53.075034", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"lct_gan\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/lct_gan/4\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.010842, + "end_time": "2024-03-22T18:36:53.116164", + "exception": false, + "start_time": "2024-03-22T18:36:53.105322", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:36:53.139671Z", + "iopub.status.busy": "2024-03-22T18:36:53.139083Z", + "iopub.status.idle": "2024-03-22T18:36:53.147875Z", + "shell.execute_reply": "2024-03-22T18:36:53.147082Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022429, + "end_time": "2024-03-22T18:36:53.149597", + "exception": false, + "start_time": "2024-03-22T18:36:53.127168", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/lct_gan/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:36:53.173516Z", + "iopub.status.busy": "2024-03-22T18:36:53.173040Z", + "iopub.status.idle": "2024-03-22T18:36:55.117898Z", + "shell.execute_reply": "2024-03-22T18:36:55.116995Z" + }, + "papermill": { + "duration": 1.959182, + "end_time": "2024-03-22T18:36:55.120033", + "exception": false, + "start_time": "2024-03-22T18:36:53.160851", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:36:55.146938Z", + "iopub.status.busy": "2024-03-22T18:36:55.146520Z", + "iopub.status.idle": "2024-03-22T18:36:55.158726Z", + "shell.execute_reply": "2024-03-22T18:36:55.158048Z" + }, + "papermill": { + "duration": 0.027548, + "end_time": "2024-03-22T18:36:55.160611", + "exception": false, + "start_time": "2024-03-22T18:36:55.133063", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:36:55.184682Z", + "iopub.status.busy": "2024-03-22T18:36:55.184050Z", + "iopub.status.idle": "2024-03-22T18:36:55.191148Z", + "shell.execute_reply": "2024-03-22T18:36:55.190272Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021188, + "end_time": "2024-03-22T18:36:55.193053", + "exception": false, + "start_time": "2024-03-22T18:36:55.171865", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:36:55.216998Z", + "iopub.status.busy": "2024-03-22T18:36:55.216727Z", + "iopub.status.idle": "2024-03-22T18:36:55.310039Z", + "shell.execute_reply": "2024-03-22T18:36:55.309268Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.107581, + "end_time": "2024-03-22T18:36:55.312316", + "exception": false, + "start_time": "2024-03-22T18:36:55.204735", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:36:55.338323Z", + "iopub.status.busy": "2024-03-22T18:36:55.338043Z", + "iopub.status.idle": "2024-03-22T18:36:59.977327Z", + "shell.execute_reply": "2024-03-22T18:36:59.976546Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.654906, + "end_time": "2024-03-22T18:36:59.979634", + "exception": false, + "start_time": "2024-03-22T18:36:55.324728", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 18:36:57.561168: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 18:36:57.561223: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 18:36:57.563027: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:37:00.004540Z", + "iopub.status.busy": "2024-03-22T18:37:00.003975Z", + "iopub.status.idle": "2024-03-22T18:37:00.010627Z", + "shell.execute_reply": "2024-03-22T18:37:00.009881Z" + }, + "papermill": { + "duration": 0.020921, + "end_time": "2024-03-22T18:37:00.012495", + "exception": false, + "start_time": "2024-03-22T18:36:59.991574", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:37:00.038010Z", + "iopub.status.busy": "2024-03-22T18:37:00.037715Z", + "iopub.status.idle": "2024-03-22T18:37:08.830928Z", + "shell.execute_reply": "2024-03-22T18:37:08.829815Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.809036, + "end_time": "2024-03-22T18:37:08.833695", + "exception": false, + "start_time": "2024-03-22T18:37:00.024659", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T18:37:08.865520Z", + "iopub.status.busy": "2024-03-22T18:37:08.864634Z", + "iopub.status.idle": "2024-03-22T18:37:08.871364Z", + "shell.execute_reply": "2024-03-22T18:37:08.870512Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023155, + "end_time": "2024-03-22T18:37:08.873196", + "exception": false, + "start_time": "2024-03-22T18:37:08.850041", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:37:08.898455Z", + "iopub.status.busy": "2024-03-22T18:37:08.897996Z", + "iopub.status.idle": "2024-03-22T18:37:08.902612Z", + "shell.execute_reply": "2024-03-22T18:37:08.901785Z" + }, + "papermill": { + "duration": 0.019259, + "end_time": "2024-03-22T18:37:08.904512", + "exception": false, + "start_time": "2024-03-22T18:37:08.885253", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:37:08.929317Z", + "iopub.status.busy": "2024-03-22T18:37:08.929024Z", + "iopub.status.idle": "2024-03-22T18:37:09.420366Z", + "shell.execute_reply": "2024-03-22T18:37:09.419399Z" + }, + "papermill": { + "duration": 0.506293, + "end_time": "2024-03-22T18:37:09.422588", + "exception": false, + "start_time": "2024-03-22T18:37:08.916295", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/aug_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_bs_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/bs_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_synth_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:37:09.450666Z", + "iopub.status.busy": "2024-03-22T18:37:09.450367Z", + "iopub.status.idle": "2024-03-22T18:37:09.767900Z", + "shell.execute_reply": "2024-03-22T18:37:09.767111Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.334231, + "end_time": "2024-03-22T18:37:09.770078", + "exception": false, + "start_time": "2024-03-22T18:37:09.435847", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:37:09.798160Z", + "iopub.status.busy": "2024-03-22T18:37:09.797530Z", + "iopub.status.idle": "2024-03-22T18:37:09.920683Z", + "shell.execute_reply": "2024-03-22T18:37:09.919387Z" + }, + "papermill": { + "duration": 0.139768, + "end_time": "2024-03-22T18:37:09.923139", + "exception": false, + "start_time": "2024-03-22T18:37:09.783371", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T18:37:09.991993Z", + "iopub.status.busy": "2024-03-22T18:37:09.990769Z", + "iopub.status.idle": "2024-03-22T18:37:10.417308Z", + "shell.execute_reply": "2024-03-22T18:37:10.416327Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.464328, + "end_time": "2024-03-22T18:37:10.419513", + "exception": false, + "start_time": "2024-03-22T18:37:09.955185", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n", + "[*] Embedding False True\n", + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:37:10.448636Z", + "iopub.status.busy": "2024-03-22T18:37:10.448259Z", + "iopub.status.idle": "2024-03-22T18:37:10.452896Z", + "shell.execute_reply": "2024-03-22T18:37:10.452039Z" + }, + "papermill": { + "duration": 0.021659, + "end_time": "2024-03-22T18:37:10.454762", + "exception": false, + "start_time": "2024-03-22T18:37:10.433103", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:37:10.480899Z", + "iopub.status.busy": "2024-03-22T18:37:10.480604Z", + "iopub.status.idle": "2024-03-22T18:37:10.487805Z", + "shell.execute_reply": "2024-03-22T18:37:10.486990Z" + }, + "papermill": { + "duration": 0.022355, + "end_time": "2024-03-22T18:37:10.489606", + "exception": false, + "start_time": "2024-03-22T18:37:10.467251", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9631369" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:37:10.515549Z", + "iopub.status.busy": "2024-03-22T18:37:10.515272Z", + "iopub.status.idle": "2024-03-22T18:37:10.605095Z", + "shell.execute_reply": "2024-03-22T18:37:10.604243Z" + }, + "papermill": { + "duration": 0.105258, + "end_time": "2024-03-22T18:37:10.607124", + "exception": false, + "start_time": "2024-03-22T18:37:10.501866", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 29] --\n", + "├─Adapter: 1-1 [2, 1071, 29] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 30,720\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 29] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,631,369\n", + "Trainable params: 9,631,369\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.15\n", + "========================================================================================================================\n", + "Input size (MB): 0.31\n", + "Forward/backward pass size (MB): 307.49\n", + "Params size (MB): 38.53\n", + "Estimated Total Size (MB): 346.32\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T18:37:10.636459Z", + "iopub.status.busy": "2024-03-22T18:37:10.636186Z", + "iopub.status.idle": "2024-03-22T19:31:45.427475Z", + "shell.execute_reply": "2024-03-22T19:31:45.426409Z" + }, + "papermill": { + "duration": 3274.82697, + "end_time": "2024-03-22T19:31:45.448255", + "exception": false, + "start_time": "2024-03-22T18:37:10.621285", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.03844140047931837, 'avg_role_model_std_loss': 0.7188729810216106, 'avg_role_model_mean_pred_loss': 0.013317291441673443, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.03844140047931837, 'n_size': 900, 'n_batch': 113, 'duration': 110.54995560646057, 'duration_batch': 0.9783181912076157, 'duration_size': 0.12283328400717841, 'avg_pred_std': 0.11123340914800631}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008303070723971663, 'avg_role_model_std_loss': 2.3807693450168474, 'avg_role_model_mean_pred_loss': 9.48858276775326e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008303070723971663, 'n_size': 450, 'n_batch': 57, 'duration': 45.44606399536133, 'duration_batch': 0.7972993683396724, 'duration_size': 0.10099125332302518, 'avg_pred_std': 0.034816826579340716}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00653058495786455, 'avg_role_model_std_loss': 0.9053095802807317, 'avg_role_model_mean_pred_loss': 0.00012669656981902897, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00653058495786455, 'n_size': 900, 'n_batch': 113, 'duration': 110.65184760093689, 'duration_batch': 0.9792198902737778, 'duration_size': 0.12294649733437432, 'avg_pred_std': 0.08656557510087712}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006352552234764315, 'avg_role_model_std_loss': 0.8825915555632886, 'avg_role_model_mean_pred_loss': 2.5333444609595783e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006352552234764315, 'n_size': 450, 'n_batch': 57, 'duration': 45.84219408035278, 'duration_batch': 0.8042490189535576, 'duration_size': 0.10187154240078396, 'avg_pred_std': 0.04755824741400909}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005102309041556307, 'avg_role_model_std_loss': 0.5227623938307345, 'avg_role_model_mean_pred_loss': 0.0001850789159238102, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005102309041556307, 'n_size': 900, 'n_batch': 113, 'duration': 111.50907039642334, 'duration_batch': 0.986805932711711, 'duration_size': 0.12389896710713705, 'avg_pred_std': 0.09074666479651906}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004312427961267531, 'avg_role_model_std_loss': 0.17915711068516546, 'avg_role_model_mean_pred_loss': 0.0001109809016019134, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004312427961267531, 'n_size': 450, 'n_batch': 57, 'duration': 45.79855394363403, 'duration_batch': 0.8034834025198954, 'duration_size': 0.10177456431918674, 'avg_pred_std': 0.07046655134150856}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004384625833740251, 'avg_role_model_std_loss': 0.5408107707275669, 'avg_role_model_mean_pred_loss': 0.0001490451962195263, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004384625833740251, 'n_size': 900, 'n_batch': 113, 'duration': 111.38601303100586, 'duration_batch': 0.985716929477928, 'duration_size': 0.12376223670111762, 'avg_pred_std': 0.08961118024205213}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004137574709020555, 'avg_role_model_std_loss': 0.30141546075263476, 'avg_role_model_mean_pred_loss': 7.981840502843144e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004137574709020555, 'n_size': 450, 'n_batch': 57, 'duration': 45.67794322967529, 'duration_batch': 0.8013674250820226, 'duration_size': 0.10150654051038954, 'avg_pred_std': 0.06743310128845144}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00368658812250942, 'avg_role_model_std_loss': 0.46921445248119603, 'avg_role_model_mean_pred_loss': 2.151566522561455e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00368658812250942, 'n_size': 900, 'n_batch': 113, 'duration': 111.5419237613678, 'duration_batch': 0.9870966704545823, 'duration_size': 0.12393547084596422, 'avg_pred_std': 0.09260421899040189}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0039208269826809155, 'avg_role_model_std_loss': 0.39560824827086744, 'avg_role_model_mean_pred_loss': 0.00012955872286843473, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0039208269826809155, 'n_size': 450, 'n_batch': 57, 'duration': 46.108590602874756, 'duration_batch': 0.8089226421556974, 'duration_size': 0.10246353467305501, 'avg_pred_std': 0.06722364254937131}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0037500500336237665, 'avg_role_model_std_loss': 0.38660638477140624, 'avg_role_model_mean_pred_loss': 8.546291791805232e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0037500500336237665, 'n_size': 900, 'n_batch': 113, 'duration': 110.84228444099426, 'duration_batch': 0.980905172044197, 'duration_size': 0.12315809382332696, 'avg_pred_std': 0.0917343535847896}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0035700482581887626, 'avg_role_model_std_loss': 0.11842851034616386, 'avg_role_model_mean_pred_loss': 0.0002132376919723574, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0035700482581887626, 'n_size': 450, 'n_batch': 57, 'duration': 48.897982120513916, 'duration_batch': 0.8578593354476126, 'duration_size': 0.10866218249003093, 'avg_pred_std': 0.0700296791874918}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0031003811301585906, 'avg_role_model_std_loss': 0.2770809364794333, 'avg_role_model_mean_pred_loss': 5.226455323332238e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0031003811301585906, 'n_size': 900, 'n_batch': 113, 'duration': 113.76307344436646, 'duration_batch': 1.006752862339526, 'duration_size': 0.12640341493818494, 'avg_pred_std': 0.09047676906385253}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0031285733293690203, 'avg_role_model_std_loss': 0.25899296287125934, 'avg_role_model_mean_pred_loss': 6.688645929435053e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0031285733293690203, 'n_size': 450, 'n_batch': 57, 'duration': 45.767940282821655, 'duration_batch': 0.8029463207512572, 'duration_size': 0.1017065339618259, 'avg_pred_std': 0.07119475702117932}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002322382250438548, 'avg_role_model_std_loss': 0.34646086611721244, 'avg_role_model_mean_pred_loss': 1.220678437067165e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002322382250438548, 'n_size': 900, 'n_batch': 113, 'duration': 109.93844747543335, 'duration_batch': 0.9729066148268438, 'duration_size': 0.12215383052825927, 'avg_pred_std': 0.09019952334991072}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002956038533043789, 'avg_role_model_std_loss': 0.4516964087274689, 'avg_role_model_mean_pred_loss': 0.00018173267813042553, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002956038533043789, 'n_size': 450, 'n_batch': 57, 'duration': 46.75201344490051, 'duration_batch': 0.8202107621912371, 'duration_size': 0.10389336321089003, 'avg_pred_std': 0.07300392768637833}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017050553576781467, 'avg_role_model_std_loss': 0.2169923465751567, 'avg_role_model_mean_pred_loss': 1.014655068901434e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017050553576781467, 'n_size': 900, 'n_batch': 113, 'duration': 114.35391974449158, 'duration_batch': 1.0119815906592176, 'duration_size': 0.12705991082721285, 'avg_pred_std': 0.09638774892793292}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0030709211031595867, 'avg_role_model_std_loss': 0.11738263850605092, 'avg_role_model_mean_pred_loss': 0.0002604710703875041, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0030709211031595867, 'n_size': 450, 'n_batch': 57, 'duration': 45.301653146743774, 'duration_batch': 0.7947658446797153, 'duration_size': 0.10067034032609727, 'avg_pred_std': 0.07127821202831049}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015025745798872474, 'avg_role_model_std_loss': 0.20518340586134937, 'avg_role_model_mean_pred_loss': 6.9952671160537674e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015025745798872474, 'n_size': 900, 'n_batch': 113, 'duration': 111.49351572990417, 'duration_batch': 0.9866682807956122, 'duration_size': 0.12388168414433798, 'avg_pred_std': 0.09498348522239027}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0029102627569550857, 'avg_role_model_std_loss': 0.35421860132333804, 'avg_role_model_mean_pred_loss': 0.00010688639051289546, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029102627569550857, 'n_size': 450, 'n_batch': 57, 'duration': 44.809088468551636, 'duration_batch': 0.7861243590973971, 'duration_size': 0.09957575215233697, 'avg_pred_std': 0.06238056484021639}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015078903491828695, 'avg_role_model_std_loss': 0.20325854836928797, 'avg_role_model_mean_pred_loss': 1.2002418886963234e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015078903491828695, 'n_size': 900, 'n_batch': 113, 'duration': 110.71620535850525, 'duration_batch': 0.9797894279513739, 'duration_size': 0.12301800595389473, 'avg_pred_std': 0.09301089175638899}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021557584258051874, 'avg_role_model_std_loss': 0.1434766953186761, 'avg_role_model_mean_pred_loss': 6.070327296319708e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021557584258051874, 'n_size': 450, 'n_batch': 57, 'duration': 45.4062922000885, 'duration_batch': 0.7966016175454123, 'duration_size': 0.10090287155575223, 'avg_pred_std': 0.07540236165126164}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009480594642486217, 'avg_role_model_std_loss': 0.0989064481881752, 'avg_role_model_mean_pred_loss': 1.0000245586616449e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009480594642486217, 'n_size': 900, 'n_batch': 113, 'duration': 110.84421229362488, 'duration_batch': 0.9809222326869458, 'duration_size': 0.12316023588180541, 'avg_pred_std': 0.09785350819274916}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001960534858582024, 'avg_role_model_std_loss': 0.5104609808862529, 'avg_role_model_mean_pred_loss': 4.5300757508451246e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001960534858582024, 'n_size': 450, 'n_batch': 57, 'duration': 45.625545501708984, 'duration_batch': 0.8004481666966489, 'duration_size': 0.10139010111490886, 'avg_pred_std': 0.06512660700255972}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007876609438487018, 'avg_role_model_std_loss': 0.10056789984725945, 'avg_role_model_mean_pred_loss': 2.650213134493559e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007876609438487018, 'n_size': 900, 'n_batch': 113, 'duration': 111.02137565612793, 'duration_batch': 0.9824900500542295, 'duration_size': 0.12335708406236437, 'avg_pred_std': 0.09717012973156124}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00240250650444068, 'avg_role_model_std_loss': 0.20876344482377662, 'avg_role_model_mean_pred_loss': 0.00015140544695302596, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00240250650444068, 'n_size': 450, 'n_batch': 57, 'duration': 45.895328760147095, 'duration_batch': 0.8051812063183701, 'duration_size': 0.10198961946699354, 'avg_pred_std': 0.06988455496499674}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001071412844466977, 'avg_role_model_std_loss': 0.08979928175546043, 'avg_role_model_mean_pred_loss': 2.8548677461531438e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001071412844466977, 'n_size': 900, 'n_batch': 113, 'duration': 110.40404558181763, 'duration_batch': 0.9770269520514834, 'duration_size': 0.12267116175757514, 'avg_pred_std': 0.09673032805785141}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002080558299649415, 'avg_role_model_std_loss': 0.33820552927165826, 'avg_role_model_mean_pred_loss': 1.7519274976611518e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002080558299649415, 'n_size': 450, 'n_batch': 57, 'duration': 45.13530921936035, 'duration_batch': 0.7918475301642167, 'duration_size': 0.10030068715413411, 'avg_pred_std': 0.06275787556609302}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009464439855623318, 'avg_role_model_std_loss': 0.11049716685496896, 'avg_role_model_mean_pred_loss': 1.5092843134298528e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009464439855623318, 'n_size': 900, 'n_batch': 113, 'duration': 110.10486841201782, 'duration_batch': 0.9743793664780338, 'duration_size': 0.1223387426800198, 'avg_pred_std': 0.0963555359306325}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019128535713470126, 'avg_role_model_std_loss': 0.2881092154617132, 'avg_role_model_mean_pred_loss': 9.992767801322522e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019128535713470126, 'n_size': 450, 'n_batch': 57, 'duration': 45.23128914833069, 'duration_batch': 0.7935313885672051, 'duration_size': 0.10051397588517931, 'avg_pred_std': 0.07075215538293776}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007565623041591607, 'avg_role_model_std_loss': 0.10640527475413109, 'avg_role_model_mean_pred_loss': 6.627467923142204e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007565623041591607, 'n_size': 900, 'n_batch': 113, 'duration': 110.25560998916626, 'duration_batch': 0.9757133627359846, 'duration_size': 0.12250623332129584, 'avg_pred_std': 0.09287066381853239}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023708047673830558, 'avg_role_model_std_loss': 0.21841555112681696, 'avg_role_model_mean_pred_loss': 0.00019829534488743176, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023708047673830558, 'n_size': 450, 'n_batch': 57, 'duration': 45.1703155040741, 'duration_batch': 0.7924616755100719, 'duration_size': 0.10037847889794244, 'avg_pred_std': 0.0748492696673789}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013394151283298722, 'avg_role_model_std_loss': 0.16507210116501242, 'avg_role_model_mean_pred_loss': 1.1373421445702734e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013394151283298722, 'n_size': 900, 'n_batch': 113, 'duration': 110.16077661514282, 'duration_batch': 0.9748741293375471, 'duration_size': 0.12240086290571425, 'avg_pred_std': 0.0946754277148078}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00335533164971922, 'avg_role_model_std_loss': 0.19024144669157844, 'avg_role_model_mean_pred_loss': 1.7741167042891626e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00335533164971922, 'n_size': 450, 'n_batch': 57, 'duration': 45.20561957359314, 'duration_batch': 0.7930810451507568, 'duration_size': 0.10045693238576253, 'avg_pred_std': 0.0715425031161622}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012095504338503816, 'avg_role_model_std_loss': 0.17023994228466374, 'avg_role_model_mean_pred_loss': 2.78441173262379e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012095504338503816, 'n_size': 900, 'n_batch': 113, 'duration': 110.36352491378784, 'duration_batch': 0.9766683620689189, 'duration_size': 0.1226261387930976, 'avg_pred_std': 0.09928367804505128}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0016752594984912625, 'avg_role_model_std_loss': 0.32005773657970704, 'avg_role_model_mean_pred_loss': 1.843289076943494e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016752594984912625, 'n_size': 450, 'n_batch': 57, 'duration': 45.38660478591919, 'duration_batch': 0.7962562243143717, 'duration_size': 0.10085912174648709, 'avg_pred_std': 0.06659886517088141}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006933588838890298, 'avg_role_model_std_loss': 0.12216246798768343, 'avg_role_model_mean_pred_loss': 7.663908191116571e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006933588838890298, 'n_size': 900, 'n_batch': 113, 'duration': 110.25478553771973, 'duration_batch': 0.9757060667054843, 'duration_size': 0.12250531726413302, 'avg_pred_std': 0.09808128885100637}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0015123529919138592, 'avg_role_model_std_loss': 0.2243189437115992, 'avg_role_model_mean_pred_loss': 5.470234152319095e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015123529919138592, 'n_size': 450, 'n_batch': 57, 'duration': 45.31874370574951, 'duration_batch': 0.795065679048237, 'duration_size': 0.10070831934611002, 'avg_pred_std': 0.07162666100224382}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0004732113509251374, 'avg_role_model_std_loss': 0.04918161084480092, 'avg_role_model_mean_pred_loss': 2.0673360931155154e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0004732113509251374, 'n_size': 900, 'n_batch': 113, 'duration': 110.03912854194641, 'duration_batch': 0.9737975977163399, 'duration_size': 0.12226569837994046, 'avg_pred_std': 0.09703757702908684}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0011618334987562977, 'avg_role_model_std_loss': 0.31888309370694506, 'avg_role_model_mean_pred_loss': 6.430942659246202e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011618334987562977, 'n_size': 450, 'n_batch': 57, 'duration': 45.194119930267334, 'duration_batch': 0.7928792970222339, 'duration_size': 0.1004313776228163, 'avg_pred_std': 0.07293911375464839}\n", + "Stopped False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 1050, 'n_batch': 132, 'role_model_metrics': {'avg_loss': 0.0008153651371720202, 'avg_g_mag_loss': 0.009589122017596744, 'avg_g_cos_loss': 0.0019650756456844865, 'pred_duration': 2.387282371520996, 'grad_duration': 6.4947497844696045, 'total_duration': 8.8820321559906, 'pred_std': 0.1444527804851532, 'std_loss': 0.00022781474399380386, 'mean_pred_loss': 1.286301767322584e-06, 'pred_rmse': 0.028554599732160568, 'pred_mae': 0.01731286011636257, 'pred_mape': 0.752444863319397, 'grad_rmse': 0.05834271386265755, 'grad_mae': 0.01659030094742775, 'grad_mape': 0.5433363318443298}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0008153651371720202, 'avg_g_mag_loss': 0.009589122017596744, 'avg_g_cos_loss': 0.0019650756456844865, 'avg_pred_duration': 2.387282371520996, 'avg_grad_duration': 6.4947497844696045, 'avg_total_duration': 8.8820321559906, 'avg_pred_std': 0.1444527804851532, 'avg_std_loss': 0.00022781474399380386, 'avg_mean_pred_loss': 1.286301767322584e-06}, 'min_metrics': {'avg_loss': 0.0008153651371720202, 'avg_g_mag_loss': 0.009589122017596744, 'avg_g_cos_loss': 0.0019650756456844865, 'pred_duration': 2.387282371520996, 'grad_duration': 6.4947497844696045, 'total_duration': 8.8820321559906, 'pred_std': 0.1444527804851532, 'std_loss': 0.00022781474399380386, 'mean_pred_loss': 1.286301767322584e-06, 'pred_rmse': 0.028554599732160568, 'pred_mae': 0.01731286011636257, 'pred_mape': 0.752444863319397, 'grad_rmse': 0.05834271386265755, 'grad_mae': 0.01659030094742775, 'grad_mape': 0.5433363318443298}, 'model_metrics': {'lct_gan': {'avg_loss': 0.0008153651371720202, 'avg_g_mag_loss': 0.009589122017596744, 'avg_g_cos_loss': 0.0019650756456844865, 'pred_duration': 2.387282371520996, 'grad_duration': 6.4947497844696045, 'total_duration': 8.8820321559906, 'pred_std': 0.1444527804851532, 'std_loss': 0.00022781474399380386, 'mean_pred_loss': 1.286301767322584e-06, 'pred_rmse': 0.028554599732160568, 'pred_mae': 0.01731286011636257, 'pred_mape': 0.752444863319397, 'grad_rmse': 0.05834271386265755, 'grad_mae': 0.01659030094742775, 'grad_mape': 0.5433363318443298}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:31:45.492346Z", + "iopub.status.busy": "2024-03-22T19:31:45.491532Z", + "iopub.status.idle": "2024-03-22T19:31:45.496507Z", + "shell.execute_reply": "2024-03-22T19:31:45.495454Z" + }, + "papermill": { + "duration": 0.030233, + "end_time": "2024-03-22T19:31:45.498703", + "exception": false, + "start_time": "2024-03-22T19:31:45.468470", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:31:45.540591Z", + "iopub.status.busy": "2024-03-22T19:31:45.540254Z", + "iopub.status.idle": "2024-03-22T19:31:45.629096Z", + "shell.execute_reply": "2024-03-22T19:31:45.628176Z" + }, + "papermill": { + "duration": 0.11286, + "end_time": "2024-03-22T19:31:45.631629", + "exception": false, + "start_time": "2024-03-22T19:31:45.518769", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:31:45.671950Z", + "iopub.status.busy": "2024-03-22T19:31:45.671594Z", + "iopub.status.idle": "2024-03-22T19:31:45.965550Z", + "shell.execute_reply": "2024-03-22T19:31:45.964479Z" + }, + "papermill": { + "duration": 0.316792, + "end_time": "2024-03-22T19:31:45.967793", + "exception": false, + "start_time": "2024-03-22T19:31:45.651001", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAEWCAYAAADPS+pKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyg0lEQVR4nO3de3xTVb738U+SNum9pfeLpQVsAaGCA7ZWRhi0Y1VA66ggwxHkqDjnBd46KOIB6syZmYrKgVF55GEuMj5jLTJHHA84KFYQlCpyB4EKCLRc0tIC6f2WrOePtOmFtjSlbdLm93699ivNztrJ2qX5svZae6+tUUophBDCCWgdXQEhhGgkgSSEcBoSSEIIpyGBJIRwGhJIQginIYEkhHAaEkhCCKchgSSEcBoSSEIIpyGBJIRwGl0KpJUrVxIbG4uHhwdJSUns3Lmzw/Lr1q1j2LBheHh4kJCQwCeffNJu2V/96ldoNBpWrFjRlaoJIfowN3s3WLt2Lenp6axatYqkpCRWrFhBamoqeXl5hIaGXlF+x44dTJ8+nczMTCZPnkxWVhZpaWns2bOHkSNHtii7fv16vvnmGyIjI+2qk8Vi4dy5c/j6+qLRaOzdJSFED1NKUVZWRmRkJFptB+0gZafExEQ1d+5c23Oz2awiIyNVZmZmm+WnTp2qJk2a1GJdUlKSevLJJ1usO3PmjIqKilKHDh1SMTExavny5Z2uU0FBgQJkkUUWJ18KCgo6/C7b1UKqra1l9+7dLFy40LZOq9WSkpJCbm5um9vk5uaSnp7eYl1qaiofffSR7bnFYuGRRx7h+eefZ8SIEVetR01NDTU1NbbnqmHCgoKCAvz8/OzZJSFELygtLSU6OhpfX98Oy9kVSMXFxZjNZsLCwlqsDwsL4+jRo21uYzQa2yxvNBptz5cuXYqbmxtPP/10p+qRmZnJb37zmyvW+/n5SSAJ4cSu1qXi8FG23bt388c//pE1a9Z0uv9n4cKFmEwm21JQUNDDtRRC9Aa7Aik4OBidTkdhYWGL9YWFhYSHh7e5TXh4eIflt2/fTlFREQMHDsTNzQ03NzdOnz7Nr3/9a2JjY9t8T4PBYGsNSatIiP7DrkDS6/WMGTOGnJwc2zqLxUJOTg7JycltbpOcnNyiPMDmzZtt5R955BEOHDjAvn37bEtkZCTPP/88n376qb37I4Tow+we9k9PT2fWrFmMHTuWxMREVqxYQUVFBbNnzwZg5syZREVFkZmZCcAzzzzDhAkTWLZsGZMmTSI7O5tdu3axevVqAIKCgggKCmrxGe7u7oSHhzN06NBr3T/RCWazmbq6OkdXQ/Rh7u7u6HS6a34fuwNp2rRpXLhwgSVLlmA0Ghk9ejSbNm2ydVzn5+e3OM/g1ltvJSsri0WLFvHSSy8RFxfHRx99dMU5SKL3KaUwGo1cvnzZ0VUR/UBAQADh4eHXdC6gRqm+P8l/aWkp/v7+mEwm6U+yw/nz57l8+TKhoaF4eXnJSaWiS5RSVFZWUlRUREBAABEREVeU6ex31O4WUl9Wb7awt+AypVV1/GxoKDqt634BzWazLYxaHzILYS9PT08AioqKCA0N7fLhm8OH/XuTWSkeWpXLY3/bRXl1vaOr41CNfUZeXl4OronoLxr/lq6lP9KlAsngpsPD3brLpdXSiQtXP1FNiM7qjr8llwokAD8PdwBMVRJIQjgb1wskT2sgSQtJdJVGo2lxLWZ/8PLLLzN69GhHV8MFA8nD2o9fWuXafUiib1uzZg0BAQHd9n7z58+/4gRmR3CpUTZo1kKSQzbhAmpra9Hr9Vct5+Pjg4+PTy/UqGMu2EKSQ7a+btOmTfz0pz8lICCAoKAgJk+ezIkTJwDribgLFixoUf7ChQu4u7uzbds2wHr+1aRJk/D09GTQoEFkZWURGxvb5VlKDx48yO23346npydBQUHMmTOH8vJy2+tbt24lMTERb29vAgICGDduHKdPnwZg//79TJw4EV9fX/z8/BgzZgy7du3q8PO2bt3K7NmzMZlMaDQaNBoNL7/8MgCxsbH813/9FzNnzsTPz485c+YAsGDBAuLj4/Hy8mLw4MEsXry4xWhY60O2Rx99lLS0NF5//XUiIiIICgpi7ty5PX5Gv8u1kPylhdQupRRVdeZe/1xPd51dIzQVFRWkp6dz4403Ul5ezpIlS7j//vvZt28fM2bM4NVXX+WVV16xvefatWuJjIzktttuA6yXNxUXF7N161bc3d1JT0+nqKioS3WvqKggNTWV5ORkvvvuO4qKinj88ceZN28ea9asob6+nrS0NJ544gnef/99amtr2blzp61uM2bM4KabbuLtt99Gp9Oxb98+3N3dO/zMW2+9lRUrVrBkyRLy8vIAWrRuXn/9dZYsWUJGRoZtna+vL2vWrCEyMpKDBw/yxBNP4OvrywsvvNDu52zZsoWIiAi2bNnC8ePHmTZtGqNHj+aJJ57o0u+qM1wukPw8G/qQXPw8pLZU1Zm5YUnvX9B8+LepeOk7/6f4wAMPtHj+17/+lZCQEA4fPszUqVN59tln+eqrr2wBlJWVxfTp09FoNBw9epTPP/+c7777jrFjxwLw5z//mbi4uC7VPSsri+rqat599128vb0BeOutt5gyZQpLly7F3d0dk8nE5MmTGTJkCADDhw+3bZ+fn8/zzz/PsGHDADpVD71ej7+/PxqNps1ZNm6//XZ+/etft1i3aNEi28+xsbHMnz+f7OzsDgNpwIABvPXWW+h0OoYNG8akSZPIycnp0UBy3UM2aSH1WceOHWP69OkMHjwYPz8/2zQ1+fn5hISEcOedd/Lee+8BcPLkSXJzc5kxYwYAeXl5uLm58ZOf/MT2ftdffz0DBgzoUl2OHDnCqFGjbGEEMG7cOCwWC3l5eQQGBvLoo4+SmprKlClT+OMf/8j58+dtZdPT03n88cdJSUnhlVdesR16XovGoG1u7dq1jBs3jvDwcHx8fFi0aBH5+fkdvs+IESNanHEdERHR5ZZkZ7lgC0n6kNrj6a7j8G9THfK59pgyZQoxMTH86U9/IjIyEovFwsiRI6mtrQWsh0FPP/00b775JllZWSQkJJCQkNATVe+Ud955h6effppNmzaxdu1aFi1axObNm7nlllt4+eWX+eUvf8nGjRv517/+RUZGBtnZ2dx///1d/rzm4QjYAvk3v/kNqamp+Pv7k52dzbJlyzp8n9aHjhqNBovF0uV6dYYLt5DkkK01jUaDl96t1xd7+o9KSkrIy8tj0aJF3HHHHQwfPpxLly61KHPfffdRXV3Npk2byMrKsrWOAIYOHUp9fT179+61rTt+/PgV79FZw4cPZ//+/VRUVNjWff3112i12hbT59x0000sXLiQHTt2MHLkSLKysmyvxcfH89xzz/HZZ5/xi1/8gnfeeeeqn6vX6zGbO9fft2PHDmJiYvjP//xPxo4dS1xcnK1T3dm4XiA19CHJmdp904ABAwgKCmL16tUcP36cL7744oqbSHh7e5OWlsbixYs5cuQI06dPt702bNgwUlJSmDNnDjt37mTv3r3MmTMHT0/PLl36MGPGDDw8PJg1axaHDh1iy5YtPPXUUzzyyCOEhYVx8uRJFi5cSG5uLqdPn+azzz7j2LFjDB8+nKqqKubNm8fWrVs5ffo0X3/9Nd99912LPqb2xMbGUl5eTk5ODsXFxVRWVrZbNi4ujvz8fLKzszlx4gRvvPEG69evt3tfe4PrBZIM+/dpWq2W7Oxsdu/ezciRI3nuued47bXXrig3Y8YM9u/fz2233cbAgQNbvPbuu+8SFhbG+PHjuf/++20jTh4eHnbXx8vLi08//ZSLFy9y88038+CDD3LHHXfw1ltv2V4/evQoDzzwAPHx8cyZM4e5c+fy5JNPotPpKCkpYebMmcTHxzN16lTuvvvuNm9g0dqtt97Kr371K6ZNm0ZISAivvvpqu2XvvfdennvuOebNm8fo0aPZsWMHixcvtntfe4PLzYd0qriCn72+FW+9ju9/e1cv1dD5VFdXc/LkSQYNGtSlL2J/cubMGaKjo/n888+54447HF2dPqujvymZD6kdjZ3aFbVm6s0W3HQu10h0eV988QXl5eUkJCRw/vx5XnjhBWJjYxk/fryjq+byXO7b6OvRlMFlci6SS6qrq+Oll15ixIgR3H///YSEhNhOknzvvfdsl1G0XjpzE9Pucvfdd7dbjz/84Q+9Vo/e5nItJHedFi+9jspaM6XVdQzwvvp1PqJ/SU1NJTW17dMb7r33XpKSktp87WpnUHenP//5z1RVVbX5WmBgYK/Vo7e5XCCBtWO7stYsQ//iCr6+vle93XNviIqKcnQVHMLlDtmg+eUjMtImhDNxzUCSWSOFcEouGUhyxb8QzsklA0muZxPCOblmIMk0tkI4JdcMJGkhiWvQHyf5dxauGUgyJ5Lo47p7kn+wTo2r0Wi4fPlyt76vPVwzkGTWSCGckmsGkrSQ+jSZ5L/9Sf5ramqYP38+UVFReHt7k5SUxNatW23bnj59milTpjBgwAC8vb0ZMWIEn3zyCadOnWLixImAdYoXjUbDo48+2qXfx7VwyTO1G4f95TykVpSCuvbn1ekx7l4gk/x3yyT/8+bN4/Dhw2RnZxMZGcn69eu56667OHjwIHFxccydO5fa2lq2bduGt7c3hw8fxsfHh+joaP7nf/6HBx54gLy8PPz8/PD09OzS7+RauGQgSad2O+oq4Q+Rvf+5L50DvffVyzWQSf7bnuQ/Pz+fd955h/z8fCIjrf+O8+fPZ9OmTbzzzjv84Q9/ID8/nwceeMA2pe/gwYNt2zdeIxcaGtrt/VOd5eKHbNKH1BfJJP9tO3jwIGazmfj4+BazA3z55Ze293366af53e9+x7hx48jIyODAgQNd/rye4KItJOtuV9WZqa23oHdzyVy+kruXtbXiiM+1g0zy37by8nJ0Oh27d+9ucbcQaDqke/zxx0lNTWXjxo189tlnZGZmsmzZMp566qlu2ddrpvoBk8mkAGUymTpVvq7erGIWbFAxCzao4rLqHq6dc6qqqlKHDx9WVVVVjq6KXYqLixWgtm3bZlu3fft2Baj169crpZQqLy9X3t7e6uOPP1Y33HCDeuWVV2xljxw5ogC1a9cu27pjx44pQC1fvrxTdWj+WatXr1YDBgxQ5eXlttc3btyotFqtMhqNbW5/yy23qKeeeqrN1x5++GE1ZcqUq9bhvffeUz4+Pi3W5eXlXfG7uZoXX3xRJSQkKKWU+vrrrxWgiouLO719cx39TXX2O+qSTQM3nRYfgwz990Uyyb9VW5P8x8fHM2PGDGbOnMmHH37IyZMn2blzJ5mZmWzcuBGAZ599lk8//ZSTJ0+yZ88etmzZYvu8mJgYNBoNGzZs4MKFCy1GCntNl6LQydjbQlJKqeQ/fK5iFmxQ+/Iv9VzFnFhfbSEppdTmzZvV8OHDlcFgUDfeeKPaunVri1aLUkp98sknClDjx4+/Yvtz586pu+++WxkMBhUTE6OysrJUaGioWrVqVac+v/VnHThwQE2cOFF5eHiowMBA9cQTT6iysjKllFJGo1GlpaWpiIgIpdfrVUxMjFqyZIkym82qpqZGPfzwwyo6Olrp9XoVGRmp5s2b1+l/k1/96lcqKChIASojI0MppVRtba1asmSJio2NVe7u7ioiIkLdf//96sCBA0oppebNm6eGDBmiDAaDCgkJUY888kiLFtFvf/tbFR4erjQajZo1a1an6tGoO1pILjfJf6O7VmzjqLGMd/89kfHxIT1cQ+cjk/w3kUn+u4dM8n8NZOjfdckk/87LJfuQQIb+XZlM8u+8XLiFJNPYuiqZ5N95uW4gyfVsog0yyb9jue4hm/QhCeF0XDeQZNZIAPrBIKtwEt3xt+S6geTiLaTG/pDKSgdc3S/6pca/pWvpa+tSH9LKlSt57bXXMBqNjBo1ijfffJPExMR2y69bt47Fixdz6tQp4uLiWLp0Kffcc4/t9Zdffpns7GwKCgrQ6/WMGTOG3//+9+12LnYHV5+CRKfTERAQYJt2w8vLq0tnKguhlKKyspKioiICAgKuuI7OHnYH0tq1a0lPT2fVqlUkJSWxYsUKUlNTycvLIzQ09IryO3bsYPr06WRmZjJ58mSysrJIS0tjz549jBw5EoD4+HjeeustBg8eTFVVFcuXL+fOO+/k+PHjhIT0zEmL0qmNbeqKrs4FJERzAQEBLaZD6Qq7z9ROSkri5ptv5q233gLAYrEQHR3NU089xYsvvnhF+WnTplFRUcGGDRts62655RZGjx7NqlWr2vyMxrM6O3vmbFfO1P7+nIlJb3xFiK+B7/4zpVPb9Fdms5m6OtcNZnHt3N3dO2wZ9ciZ2rW1tezevZuFCxfa1mm1WlJSUsjNzW1zm9zc3CsufkxNTW33rg21tbWsXr0af39/Ro0a1WaZmpoaampqbM9LS0vt2Q1AWkjN6XS6a2pmC9Fd7OrULi4uxmw2ExYW1mJ9WFgYRqOxzW2MRmOnym/YsAEfHx88PDxYvnw5mzdvJjg4uM33zMzMxN/f37ZER0fbsxtAU6d2Tb2F6jqz3dsLIbqf04yyTZw4kX379rFjxw7uuusupk6d2m7fxsKFCzGZTLaloKDA7s/zNbjZpnEukylIhHAKdgVScHAwOp2OwsLCFusLCwvb7cwKDw/vVHlvb2+uv/56brnlFv7yl7/g5ubGX/7ylzbf02Aw4Ofn12Kxl1araTYnkhy2CeEM7AqkxiH5nJwc2zqLxUJOTg7JycltbpOcnNyiPMDmzZvbLd/8fZv3E/UEVx/6F8LZ2D3sn56ezqxZsxg7diyJiYmsWLGCiooKZs+eDVhvMRMVFUVmZiYAzzzzDBMmTGDZsmVMmjSJ7Oxsdu3axerVqwHrbWR+//vfc++99xIREUFxcTErV67k7NmzPPTQQ924q1eydmxXSce2EE7C7kCaNm0aFy5cYMmSJRiNRkaPHs2mTZtsHdf5+flotU0Nr1tvvZWsrCwWLVrESy+9RFxcHB999JHtHCSdTsfRo0f529/+RnFxMUFBQdx8881s3769x6d7kDvYCuFcXHbGSIA57+7is8OF/C5tJP92S0wP1lAI19bZ76jTjLI5gqtfzyaEs3HtQJJZI4VwKq4dSDJrpBBOxbUDSS4fEcKpuHQgyXlIQjgXlw6kpk5t6UMSwhm4diA1TGNbJi0kIZyCaweSDPsL4VQkkLAO+/eD80OF6PNcO5AaDtlqzRZq6i0Oro0QwqUDyVvvhrZhTiQZ+hfC8Vw6kLRajfQjCeFEXDqQoOnkSDkXSQjHk0DylDvYCuEsJJA85JBNCGchgSTXswnhNCSQZNZIIZyGBJK0kIRwGi4fSP4y7C+E03D5QPKTKUiEcBoSSDLsL4TTkECSYX8hnIYEkqd0agvhLCSQPGTWSCGchQSSrQ+pTuZEEsLBXD6QGof96y2Kqjqzg2sjhGtz+UDydNfh1jApkoy0CeFYLh9IGo1GzkUSwkm4fCBB01S2MvQvhGNJICFD/0I4Cwkk5ORIIZyFBBJy+YgQzkICCZmCRAhnIYGETEEihLOQQEKmIBHCWUgg0WzYX/qQhHAoCSSQm0UK4SQkkJBhfyGchQQSMuwvhLOQQEJaSEI4Cwkkmg37y5xIQjiUBBJNndoWBRW1MieSEI4igQQY3LToddZfhZyLJITjSCDROCdS01S2QgjHkEBqINezCeF4XQqklStXEhsbi4eHB0lJSezcubPD8uvWrWPYsGF4eHiQkJDAJ598Ynutrq6OBQsWkJCQgLe3N5GRkcycOZNz5851pWpd5uspdx8RwtHsDqS1a9eSnp5ORkYGe/bsYdSoUaSmplJUVNRm+R07djB9+nQee+wx9u7dS1paGmlpaRw6dAiAyspK9uzZw+LFi9mzZw8ffvgheXl53Hvvvde2Z3ZqunxEWkhCOIyyU2Jiopo7d67tudlsVpGRkSozM7PN8lOnTlWTJk1qsS4pKUk9+eST7X7Gzp07FaBOnz7dqTqZTCYFKJPJ1KnybZn73m4Vs2CD+utXP3b5PYQQbevsd9SuFlJtbS27d+8mJSXFtk6r1ZKSkkJubm6b2+Tm5rYoD5CamtpueQCTyYRGoyEgIKDN12tqaigtLW2xXKumaWzlkE0IR7ErkIqLizGbzYSFhbVYHxYWhtFobHMbo9FoV/nq6moWLFjA9OnT8fPza7NMZmYm/v7+tiU6Otqe3WiTnK0thOM51ShbXV0dU6dORSnF22+/3W65hQsXYjKZbEtBQcE1f3bjsL+chySE47jZUzg4OBidTkdhYWGL9YWFhYSHh7e5TXh4eKfKN4bR6dOn+eKLL9ptHQEYDAYMBoM9Vb8qGfYXwvHsaiHp9XrGjBlDTk6ObZ3FYiEnJ4fk5OQ2t0lOTm5RHmDz5s0tyjeG0bFjx/j8888JCgqyp1rdQuZEEsLx7GohAaSnpzNr1izGjh1LYmIiK1asoKKigtmzZwMwc+ZMoqKiyMzMBOCZZ55hwoQJLFu2jEmTJpGdnc2uXbtYvXo1YA2jBx98kD179rBhwwbMZrOtfykwMBC9Xt9d+9ohmTVSCMezO5CmTZvGhQsXWLJkCUajkdGjR7Np0yZbx3V+fj5abVPD69ZbbyUrK4tFixbx0ksvERcXx0cffcTIkSMBOHv2LB9//DEAo0ePbvFZW7Zs4Wc/+1kXd80+MtG/EI6nUarvz7dRWlqKv78/JpOpw76njpy4UM4dy77Ez8ONAy+ndnMNhXBtnf2OOtUomyM1dmqX1dRjsfT5jBaiT5JAauDb0IeklDWUhBC9TwKpgYe7DoOb9dchQ/9COIYEUjMy9C+EY0kgNSND/0I4lgRSM9JCEsKxJJCaaX73ESFE75NAaqbpin85ZBPCESSQmpGJ/oVwLAmkZhpbSDIFiRCOIYHUjHRqC+FYEkjNNM2JJH1IQjiCBFIztj4kaSEJ4RASSM3IsL8QjiWB1Iztin8Z9hfCISSQmvGTFpIQDiWB1EzjtWxlNfWYZU4kIXqdBFIzvg2HbABl0rEtRK+TQGpG76bF010HyNC/EI4ggdSKDP0L4TgSSK3I0L8QjiOB1ErTFf8SSEL0NgmkVpqG/qUPSYjeJoHUim0aW2khCdHrJJBaaWwhyRQkQvQ+CaRWmq74l0ASordJILXSNOwvfUhC9DYJpFZk2F8Ix5FAakWG/YVwHAmkVmTYXwjHkUBqRVpIQjiOBFIrciskIRxHAqmVxhZSRa2ZOrPFwbURwrVIILXi23CmNshUtkL0NgmkVtx0Wrz1jXMiyWGbEL1JAqkN/nLDSCEcQgKpDTL0L4RjSCC1QYb+hXAMCaQ2yNC/EI4hgdSGxhaSTEEiRO+SQGqDn3RqC+EQEkhtsM0aKZ3aQvQqCaQ2SAtJCMeQQGqDn8yJJIRDdCmQVq5cSWxsLB4eHiQlJbFz584Oy69bt45hw4bh4eFBQkICn3zySYvXP/zwQ+68806CgoLQaDTs27evK9XqNk3D/nLIJkRvsjuQ1q5dS3p6OhkZGezZs4dRo0aRmppKUVFRm+V37NjB9OnTeeyxx9i7dy9paWmkpaVx6NAhW5mKigp++tOfsnTp0q7vSTeSYX8hHETZKTExUc2dO9f23Gw2q8jISJWZmdlm+alTp6pJkya1WJeUlKSefPLJK8qePHlSAWrv3r121clkMilAmUwmu7Zrz8Ezl1XMgg0q8febu+X9hHB1nf2O2tVCqq2tZffu3aSkpNjWabVaUlJSyM3NbXOb3NzcFuUBUlNT2y3fGTU1NZSWlrZYupO/3ApJCIewK5CKi4sxm82EhYW1WB8WFobRaGxzG6PRaFf5zsjMzMTf39+2REdHd/m92tLYh1RdZ6Gm3tyt7y2EaF+fHGVbuHAhJpPJthQUFHTr+/t6uKHRWH+WOZGE6D1uVy/SJDg4GJ1OR2FhYYv1hYWFhIeHt7lNeHi4XeU7w2AwYDAYurz91Wi1GnwMbpRV11NaVUewT899lhCiiV0tJL1ez5gxY8jJybGts1gs5OTkkJyc3OY2ycnJLcoDbN68ud3yzkKG/oXofXa1kADS09OZNWsWY8eOJTExkRUrVlBRUcHs2bMBmDlzJlFRUWRmZgLwzDPPMGHCBJYtW8akSZPIzs5m165drF692vaeFy9eJD8/n3PnzgGQl5cHWFtX19KSuhZ+nu6cvVwlQ/9C9KauDOG9+eabauDAgUqv16vExET1zTff2F6bMGGCmjVrVovyH3zwgYqPj1d6vV6NGDFCbdy4scXr77zzjgKuWDIyMjpVn+4e9ldKqamrdqiYBRvU/+4/223vKYSr6ux3VKOUUg7Mw25RWlqKv78/JpMJPz+/bnnPJ97dxebDhfz+/pHMSIrplvcUwlV19jvaJ0fZeoOtD0mu+Bei17heIG1fBpuXwFUahjLRvxC9z+5O7T7t/H7I+a315/pauCsT2wlHrcj1bEL0PtdqIUWMgsnLrT9/+zZsTAdL23enlWF/IXqfawUSwNh/h/tWAhrY9Vf4+CmwXHl5iMyJJETvc71AArjp3+AXfwKNDvb9HdY/CeaWLSHbNLbShyREr3HNQAK48SF48K+gdYOD6+Afs639Sg2khSRE73PdQAIYkQbT/g46PRz5GD6YCfU1QPNbIUkfkhC9xbUDCWDo3fDw++DmAT/8C96fDnVVTaNscsgmRK+RQAKIS4FffgDuXnAiB957CH+d9fCttt5CdZ3MiSREb5BAajR4Avzbh6D3hVPb8Vn3MH6aSkBaSUL0Fgmk5mKSYeZHYPBHU5DLe4ZX8KOcs5eqHF0zIVyCBFJr142FWR+D5wASOE6W/g/M++sXfLz/nKNrJkS/J4HUlsjR8OhGzJ5BjNSeYpXltyx6fzvpa/dRJodvQvQYCaT2hI1AN3sjyiuYBO0p/q5/hc/3/sA9b2xn9+mLjq6dEP2SBFJHQoejmfUxeAZyo/ZH3vd8jUsXS3hoVS7/vfkH6s1tXwcnhOgaCaSrCRth61MaoX7gfwcsx1NV8UbOMR76v7mcLqlwdA2F6DckkDojPAFm/hM8/BlU9T3bov4PoR717M2/zD1/3M66XQX0g4k3hXA4CaTOihhlDSWDP0Elu9ke9X+4LcaLilozz//jAPOy9mKqlA5vIa6FBJI9Im+CR9aDwQ/D2W9413MZL/18IG5aDRsPnufny79k+eYfOFksh3FCdIVM8t8VBd/B/7sfastg8M84OP7/8vQ/jrYIolHX+ZN2UxSTb4wkxFduNClcW2e/oxJIXZX/Lfz9F1BbDkNup/rBv7Pp6GXW7z3LV8eLMVusv1adVsO464O5/6ZI7rwhHG+Da80aLARIIPWO0zvg7w9CXQVc/3N4+D1wM3ChrIaNB86xft859hdcthX3dNfx8xvCSLspktviQnDXyRGzcA0SSL3l1FfWUKqvgsET4Yb7wDukYQnmVI0PH31/mX/uP9/ikC7QW889CeHcOyqKsTED0GrbvtmAEP2BBFJv+vFLyJoK9dVtv+7mgfIOpso9kDO1PhwtM3CmzpdCNQCjCqTeO5xRI25g4k9GMjJ6AJp27oQiRF8lgdTbzuyGPX+D8iKouNCwFFsP5zqpXmm5qB1AvU8kfiHR+ITEgF+kdQkZCsHx4CYd5KLv6ex3VHpYu8t1Y6xLa7UVTeHUGFTlRdal7DwW01lqL51BX3UBN42FUFUCZSVQdhB+bPlWSqOjPmAw1YHDqAwYSpl/HCbfOEo9oqgxK2rqLdSZFQlR/sSH+UhLS/Q50kJyFuZ6Ki6d47v9hzh09AgXz58mlIuEa0q4TlNMvOaMbcK41iqUgWPqOvIs0eSpaA5YBlEeOJKJCTHcPTKchCh/CSfhUHLI1sddqqjlX4eMfLz/LN+evIhSikjNRRLczzJMd4ZhmgKGUECspQA9V54hXqd0HFYx7LVcz2nPGwgcOo5bxoxhTEygdKD3B5UNM054BTq2Hp0kgdSP1NSb0Wk0uLV1moC5Hi7+CEWHrYvxEJYzu9BWFF5RtET5clgbT3XYTwgfcRvDxvwMdy//XtgD0W1MZ+CrFbDnXdC5W+8vOOweR9fqqiSQXJlS1j/cMzupz99J+fFv8Ln0PW6qZUvKgoYytyDc/ULxHBCOxivYdrpC06kLzZ5rNFBdCjVlUGOyPlaXQk1pq59LAY11O68g8Gp49A5qeq73cszvpq+6dAq+Wg573wNL839HDaS8DOOesf77OCkJJNFSfQ11Z/dzet8WKn78hhDTQSK54Lj6uHk2BFYgDBgECQ9B3J3gpndcnZxRyQnY/t+w/31QDXe/ib0NbkuHI/9rvR08wOgZMHm5047CSiCJDtWbLew58gPbdu3n6Ikf8TNfJkhTSqi2jISAWuJ8qgnEhKZxdNBc07ClBgy+YPADD7/2f7ZYoLKkYSm2PlY0/GyubbtSXkGQMBVG/xIibuy134VTupAH216HQ/8A1TAR4JDbYfwL1ptRgLUlvPNPsGmBtczAZOuNT72DHVfvdkggiU4zVdbx8f6zrN1VwKGzpbb1UQGePDDmOh76SRTR3g3/O+t9QHsNl7woZb3+r3lAnf4a9mdDebN+r7AEazAlPAQ+IV3/vL6m8HvY9hp8/xHQ8NWMS4UJL1hvQNGW4zmwbrb1MDpgIExfC2E39FaNO0UCSXTJ9+dMfPBdAev3nqW02nobcY0Gxg0J5hc/ieL2YaEEePXAYZW5Hn7cAnv/DnmfNLWitG7WL+ToX3bvIV1FCZz5rmm5cBR8I5pOQA0ZCsFDIXCQtfO4p9RVwdk9UPAtnNoOJ75oem3YZBg/3zrtzdVc+MF6tcClk9b/NB78K8Sn9ly97SSBJK5JdZ2ZT7838sGuAr4+XmJbr9NquDl2ACnDw7jzhnAGBvVA53TlRTj0P7AvC87taVrvFQTD74UBseAbDj5hTY+eA9rv1DXXWVsezQPo4o9tl21N6waBQyAk3hpUwUOtP/tFgWcg6Ow8t7j0vDV8Gpfz+8FS36yABkakwW3zIXykfe9deRE+mGkNNjRw5+8gea5TdHZLIIluU3Cxkn/sPsOn3xs5aixr8Vp8mA8/vyGMlOFhjLouoPvPcSo6Cvuzrjyka02ntwZT85By84Bze61LfRs3+wyOh+tuti5hI6HcaO27Kf6h4fHY1S/98RxgHYH0Cm4YjQxu9jzI+nrJCWv45H8Lpvwr38MnHAYmQXSStRUYHGff76i5+lr4ZL71MiaAm/4NJi13+GCBBJLoEfkllXx+pJDNhwvZeeqibd4ngFBfA3cMD+PnN4SSNCioe+d+MtdbD2dObbcGU5nR+lheCFWXrr69hz9EjbWGT/TNEDXGGhYdsVig9CwU51kPiRofS45ZLwWiC18djdZ644joW6wBFJ1o7ffpzlaMUvDtKvj0JWtnd8w4mPr/rAHpIBJIoseZKuvYklfE5sOFfPnDBcpr6lu87uvhRpifB2F+BkJ9PQj1MxDm63HFOg933bVVpL6mIaQKra2cxrCqKbce9lx3MwTFXVtnfGsWs/UQqbK46TrFypJmPxc3ddr7RVoDaGCSNQgNvt1Xj44c2wz/+HfreWEGf2vweQc1tea8Gs4L8w5udq5YsDWotdf4b9KKBJLoVTX1Zr758SKfHy7k8yOFnDe1MxVLGwxu1qCw/SEqUA3PlLKub/wz9da7kTQ4iAnxwUyID+2ZPqw+wGJR1FksGNyuEhxFR+H9adYTKztLp7cezoYOh9AbrC260OHgH93llpwEknAYpRTlNfUUltZQVFpNYVl1w881FJZVW9eV1lBYWk1N/bXdbDM2yIvx8SGMjwsheUg3HyY6mZLyGr784QJfHC1i2w8XKK2ux+Cmxd/THT9Pd+ujh5vtuZ+HdV2A3sIQlc8N/rV41F22tuIaW3bNW3mVxVBtar8Cel9rMIXdAKENIRU2olPX00kgCaenlKK0up7Sqjo0GmwzEmho+o9YQ8M6jXW9sbSa7ceK+fKHC+w5fYn6Zn1Y7joNY2MCGR8fwoT4EIZH+HbLLAd1ZgtnL1VxqqSC0yWVnLlUibtOa/vS+3q4NfzsZlvn5+l29dbLVVgsioNnTWzJK2JL3gUOnLnMtXxb3bQaRkUHkDw4iFuHBPGTmAFXHi6b66z9ZkVHrCOTRUes10gW/9BqNLCZCQtg4ksdfrYEkuj3yqrr2HGihG0/XGDbsQsUXGw5khborSfcz4NAb32LZYC3niBvPQO8mtb5GNw4Z6ridEkFp4orrY8l1sczl6paBF9nGdysoRXkrSfE10CIj8H62GwJ9TUQ4uuBn4cbGo0GU1Ud249dYMvRC3z5QxHF5S3Pah8Z5cfEoaH8bGgo14f4UFpdh6mqjtLqOkqr6iitqrc9N1VZ15mq6jh+ofyK34/eTctPBgZw65Bgbh0SxI3XBaB3a6efrb4WSo5bw8kWVN/D5XxIWwWjp3f4u5BAEi5FKcWpkkq+zCti27Fick+UUFVn7rb3N7hpiQ3yJibIi+hALyxKUVpV3xQEDS290uo6yqrbaUl0QO+mJdhbT2FZTYuRSx+DG7fFBTNxaCgThoYQ5ufR5X0ouFhJ7okScn8sYceJYgpLa1q87umu4+ZBgdwcM4DoQC/C/T2I8LcOQrQ78FBTZh051Ht3+Nk9GkgrV67ktddew2g0MmrUKN58800SExPbLb9u3ToWL17MqVOniIuLY+nSpdxzT9OUCUopMjIy+NOf/sTly5cZN24cb7/9NnFxnTsfQwJJtFZTb+YHYznFFTVcqqjlYsNyqbKWkvKGx4paLlXUcrmqDqXAS68jJsib2CAvYoK8GRTs1fDcm1BfQ6fPsTJbrH1oZQ2tlJLyWi6U1XChvMb6WFZDUVm17efSVgEWF+rDxGGhTBwaypiYAe23Wq6BUoofiyvYcaKEbxpC6mJFO9cY0tTajPD3sAVVuL8nEf4exIX6EHqVoOyxQFq7di0zZ85k1apVJCUlsWLFCtatW0deXh6hoaFXlN+xYwfjx48nMzOTyZMnk5WVxdKlS9mzZw8jR1rPRF26dCmZmZn87W9/Y9CgQSxevJiDBw9y+PBhPDyu/j+CBJK4FmaLoqK2Hl+Dm0Nm1qyuM1PcEFYhvgauG9D7I4cWiyKvsIwdJ0r4/pwJo6kao6mac6Yqqus6HniYf2c8827vuPHQY4GUlJTEzTffzFtvvdWwIxaio6N56qmnePHFF68oP23aNCoqKtiwYYNt3S233MLo0aNZtWqVdSbEyEh+/etfM3/+fABMJhNhYWGsWbOGhx9++Kp1kkASomeohkPT86VVnG8IKetj0/On7ojj3lGRHb5Pj0zyX1tby+7du1m4cKFtnVarJSUlhdzc3Da3yc3NJT09vcW61NRUPvroIwBOnjyJ0WgkJSXF9rq/vz9JSUnk5ua2GUg1NTXU1DQd/5aWll5RRghx7TQaDf5e7vh7uTMsvOf/s7fr4LS4uBiz2UxYWFiL9WFhYRiNxja3MRqNHZZvfLTnPTMzM/H397ct0dHR9uyGEMJJ9cl7OS9cuBCTyWRbCgoKHF0lIUQ3sCuQgoOD0el0FBa2vOq6sLCQ8PDwNrcJDw/vsHzjoz3vaTAY8PPza7EIIfo+uwJJr9czZswYcnJybOssFgs5OTkkJye3uU1ycnKL8gCbN2+2lR80aBDh4eEtypSWlvLtt9+2+55CiH5K2Sk7O1sZDAa1Zs0adfjwYTVnzhwVEBCgjEajUkqpRx55RL344ou28l9//bVyc3NTr7/+ujpy5IjKyMhQ7u7u6uDBg7Yyr7zyigoICFD//Oc/1YEDB9R9992nBg0apKqqqjpVJ5PJpABlMpns3R0hRC/o7HfU7kBSSqk333xTDRw4UOn1epWYmKi++eYb22sTJkxQs2bNalH+gw8+UPHx8Uqv16sRI0aojRs3tnjdYrGoxYsXq7CwMGUwGNQdd9yh8vLyOl0fCSQhnFtnv6Ny6YgQosf1yHlIzqoxU+V8JCGcU+N382rtn34RSGVl1nme5XwkIZxbWVkZ/v7t3769XxyyWSwWzp07h6/v1ee/KS0tJTo6moKCgn55eNef968/7xv07/1TSlFWVkZkZCTaDqYS7hctJK1Wy3XXXWfXNv39/KX+vH/9ed+g/+5fRy2jRn3yTG0hRP8kgSSEcBouF0gGg4GMjAwMBoOjq9Ij+vP+9ed9g/6/f53RLzq1hRD9g8u1kIQQzksCSQjhNCSQhBBOQwJJCOE0XC6QVq5cSWxsLB4eHiQlJbFz505HV+mavfzyy2g0mhbLsGHDHF2tLtu2bRtTpkwhMjISjUZjm3+9kVKKJUuWEBERgaenJykpKRw7dswxle2Cq+3fo48+esW/51133eWYyvYylwqktWvXkp6eTkZGBnv27GHUqFGkpqZSVFTk6KpdsxEjRnD+/Hnb8tVXXzm6Sl1WUVHBqFGjWLlyZZuvv/rqq7zxxhusWrWKb7/9Fm9vb1JTU6muru7lmnbN1fYP4K677mrx7/n+++/3Yg0dqEcnQXEyiYmJau7cubbnZrNZRUZGqszMTAfW6tplZGSoUaNGOboaPQJQ69evtz23WCwqPDxcvfbaa7Z1ly9fVgaDQb3//vsOqOG1ab1/Sik1a9Ysdd999zmkPo7mMi2kxls4Nb/d0tVu4dSXHDt2jMjISAYPHsyMGTPIz893dJV6xNVum9VfbN26ldDQUIYOHcp//Md/UFJS4ugq9QqXCaSu3MKpr0hKSmLNmjVs2rSJt99+m5MnT3LbbbfZpmXpT7py26y+5q677uLdd98lJyeHpUuX8uWXX3L33XdjNpsdXbUe1y+u9nd1d999t+3nG2+8kaSkJGJiYvjggw947LHHHFgz0RXNb46akJDAjTfeyJAhQ9i6dSt33HGHA2vW81ymhdSVWzj1VQEBAcTHx3P8+HFHV6XbdeW2WX3d4MGDCQ4O7pf/nq25TCB15RZOfVV5eTknTpwgIiLC0VXpdq5426wzZ85QUlLSL/89W3OpQ7b09HRmzZrF2LFjSUxMZMWKFVRUVDB79mxHV+2azJ8/nylTphATE8O5c+fIyMhAp9Mxffp0R1etS8rLy1u0Bk6ePMm+ffsIDAxk4MCBPPvss/zud78jLi6OQYMGsXjxYiIjI0lLS3Ncpe3Q0f4FBgbym9/8hgceeIDw8HBOnDjBCy+8wPXXX09qaqoDa91LHD3M19s6uoVTXzVt2jQVERGh9Hq9ioqKUtOmTVPHjx93dLW6bMuWLQq4Ymm8vda13jbL0Trav8rKSnXnnXeqkJAQ5e7urmJiYtQTTzxhu+9hfyfTjwghnIbL9CEJIZyfBJIQwmlIIAkhnIYEkhDCaUggCSGchgSSEMJpSCAJIZyGBJIQwmlIIAkhnIYEkhDCaUggCSGchgSSEMJp/H8ffe5qlh0BVAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:31:46.011391Z", + "iopub.status.busy": "2024-03-22T19:31:46.011028Z", + "iopub.status.idle": "2024-03-22T19:33:49.003359Z", + "shell.execute_reply": "2024-03-22T19:33:49.002205Z" + }, + "papermill": { + "duration": 123.017721, + "end_time": "2024-03-22T19:33:49.006185", + "exception": false, + "start_time": "2024-03-22T19:31:45.988464", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:33:49.047547Z", + "iopub.status.busy": "2024-03-22T19:33:49.047141Z", + "iopub.status.idle": "2024-03-22T19:33:49.069641Z", + "shell.execute_reply": "2024-03-22T19:33:49.068700Z" + }, + "papermill": { + "duration": 0.044832, + "end_time": "2024-03-22T19:33:49.071669", + "exception": false, + "start_time": "2024-03-22T19:33:49.026837", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.0014360.0073750.0008156.4723580.016590.5433360.0583430.0000012.4366170.0173130.7524450.0285550.1444530.0002288.908975
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.001436 0.007375 0.000815 6.472358 0.01659 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.543336 0.058343 0.000001 2.436617 0.017313 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.752445 0.028555 0.144453 0.000228 8.908975 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:33:49.108480Z", + "iopub.status.busy": "2024-03-22T19:33:49.108179Z", + "iopub.status.idle": "2024-03-22T19:33:49.476230Z", + "shell.execute_reply": "2024-03-22T19:33:49.475315Z" + }, + "papermill": { + "duration": 0.38857, + "end_time": "2024-03-22T19:33:49.478228", + "exception": false, + "start_time": "2024-03-22T19:33:49.089658", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:33:49.517759Z", + "iopub.status.busy": "2024-03-22T19:33:49.517069Z", + "iopub.status.idle": "2024-03-22T19:35:59.371411Z", + "shell.execute_reply": "2024-03-22T19:35:59.370335Z" + }, + "papermill": { + "duration": 129.877089, + "end_time": "2024-03-22T19:35:59.374327", + "exception": false, + "start_time": "2024-03-22T19:33:49.497238", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/lct_gan/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/lct_gan/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:35:59.414640Z", + "iopub.status.busy": "2024-03-22T19:35:59.414332Z", + "iopub.status.idle": "2024-03-22T19:35:59.441125Z", + "shell.execute_reply": "2024-03-22T19:35:59.440223Z" + }, + "papermill": { + "duration": 0.049839, + "end_time": "2024-03-22T19:35:59.443129", + "exception": false, + "start_time": "2024-03-22T19:35:59.393290", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:35:59.480940Z", + "iopub.status.busy": "2024-03-22T19:35:59.480068Z", + "iopub.status.idle": "2024-03-22T19:35:59.485927Z", + "shell.execute_reply": "2024-03-22T19:35:59.485028Z" + }, + "papermill": { + "duration": 0.027001, + "end_time": "2024-03-22T19:35:59.487878", + "exception": false, + "start_time": "2024-03-22T19:35:59.460877", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.04632651686116116}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:35:59.526529Z", + "iopub.status.busy": "2024-03-22T19:35:59.526257Z", + "iopub.status.idle": "2024-03-22T19:35:59.972113Z", + "shell.execute_reply": "2024-03-22T19:35:59.971228Z" + }, + "papermill": { + "duration": 0.468036, + "end_time": "2024-03-22T19:35:59.974193", + "exception": false, + "start_time": "2024-03-22T19:35:59.506157", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA840lEQVR4nO3deXhU1f0/8Pe9d/ZJZib7AtlYBGUHgSLVgFBBXKC21ioVEFHbQtUifZRvf0WxtVjrAm39UtunJvIIaPWLy7d8USkK1AUQlF2BQAgBErLPvt57fn/MQiaZJJPJJHcmfF7PM08yd87c+7lJ5pNzzr3nHI4xxkAIITLi5Q6AEEIoERFCZEeJiBAiO0pEhBDZUSIihMiOEhEhRHaUiAghsqNERAiRHSUiQojsKBGRTpWXl4PjOJw9e1buUEg/RomIxN2mTZuwdu1aucMgSYQSEYk7SkSkuygREUJkR4mIdNu2bdtQWlqK1NRUGAwGTJw4EZs2bQIATJs2DVu3bkVVVRU4jgPHcSguLo5635Ik4amnnkJ+fj50Oh2mT5+O48ePo7i4GIsWLQqVa2pqwooVKzBq1CikpKTAYDDg5ptvxqFDh8L2t3PnTnAch3/+85945plnMHDgQGg0GsyYMQMVFRXx+HGQOFDIHQBJLuXl5Vi8eDFGjBiBlStXwmQy4euvv8YHH3yAe+65B7/+9a9hNptx/vx5vPTSSwCAlJSUqPe/cuVKPPfcc7jtttswa9YsHDp0CLNmzYLL5Qord+bMGbz77ru48847UVJSgkuXLuGVV15BaWkpjh8/jvz8/LDyzz77LHiex4oVK2A2m/Hcc89h/vz52Lt3b89/KKTnGCGdKCsrYwBYZWUla2lpYampqWzy5MnM6XSGlZMkKfT9LbfcwoqKirp9rNraWqZQKNi8efPCtj/11FMMAFu4cGFom8vlYqIohpWrrKxkarWaPf3006Ftn3zyCQPArr76auZ2u0Pb161bxwCwI0eOdDtOEn/UNCNR2759O6xWK5544gloNJqw1ziO6/H+d+zYAZ/Ph5///Odh23/xi1+0K6tWq8Hz/j9fURTR2NiIlJQUDBs2DF999VW78vfddx9UKlXo+fXXXw/AX7Mi8qNERKJ2+vRpAMDIkSN7Zf9VVVUAgCFDhoRtT09PR1paWtg2SZLw0ksvYejQoVCr1cjMzERWVhYOHz4Ms9ncbt+FhYVhz4P7a25ujucpkBhRIiJJ6fe//z2WL1+OG264Aa+//jo+/PBDbN++HSNGjIAkSe3KC4IQcT+MZkpOCNRZTaI2ePBgAMDRo0fb1Vpai7WZVlRUBACoqKhASUlJaHtjY2O7msvbb7+N6dOn4x//+EfY9paWFmRmZsZ0fCIfqhGRqN10001ITU3FmjVr2l3Fal2z0Ov1EZtHXZkxYwYUCgXWr18ftv0vf/lLu7KCILSrzbz11lu4cOFCt49L5Ec1IhI1g8GAl156CUuWLMHEiRNxzz33IC0tDYcOHYLD4cBrr70GAJgwYQLefPNNLF++HBMnTkRKSgpuu+22Lvefk5ODRx55BC+88AJuv/12zJ49G4cOHcK2bduQmZkZVtO69dZb8fTTT+O+++7DddddhyNHjmDjxo0YNGhQr50/6UUyX7UjCa715fug999/n1133XVMq9Uyg8HAJk2axDZv3hx63WazsXvuuYeZTCYGoFuX8n0+H/vNb37DcnNzmVarZTfeeCP75ptvWEZGBvvpT38aKudyudhjjz3G8vLymFarZVOnTmVffPEFKy0tZaWlpaFywcv3b731VthxKisrGQBWVlbW3R8J6QUcY9RbRxJbS0sL0tLS8Lvf/Q6//vWv5Q6H9ALqIyIJxel0ttsWHEA7bdq0vg2G9BnqIyJ9or6+HqIodvi6SqVCeno63nzzTZSXl2POnDlISUnBp59+is2bN+Omm27C1KlT+zBi0pcoEZE+MXHixNANi5GUlpZi586dGD16NBQKBZ577jlYLJZQB/bvfve7PoyW9DXqIyJ94rPPPovY7ApKS0vDhAkT+jAikkgoERFCZEed1YQQ2SV1H5EkSbh48SJSU1PjMvqbEBJfjDFYrVbk5+eHZkuIJKkT0cWLF1FQUCB3GISQLlRXV2PgwIEdvp7UiSg1NRWA/yQNBoPM0RBC2rJYLCgoKAh9VjuS1Iko2BwzGAyUiAhJYF11nVBnNSFEdpSICCGyo0RECJFdUvcRRYMxBp/P1+k4JxKZIAhQKBR0awTpdf06EXk8HtTU1MDhcMgdStLS6XTIy8sLWwGDkHjrt4lIkiRUVlZCEATk5+dDpVLRf/ZuYIzB4/Ggvr4elZWVGDp0aKc3pBHSE/02EXk8HkiShIKCAuh0OrnDSUparRZKpRJVVVXweDzt1jIj7bl9IgSOg0KgpN0d/f6nRf/Fe4Z+ft1z9IIZ//PVedjdPrlDSSr0V0ZIHF1sceFiiwvf1lrkDiWpUCIiJI4abW4AQFYKNWO7gxLRFa64uDg0JzTpGUliaHF6AQD/d7QGWw/XyBxR8qBEREicOLwigtMMOj0iGu1ueQNKIpSI+gGPxyN3CARo10Ftow7rqMmeiC5cuICf/OQnyMjIgFarxahRo7B///5ePabHJ3X48IlS1GW9UZSNxbRp07Bs2TIsW7YMRqMRmZmZ+M1vfhNaYrm4uBi//e1vsWDBAhgMBjz44IMAgE8//RTXX389tFotCgoK8PDDD8Nut4f2W1dXh9tuuw1arRYlJSXYuHFjTPGRyKwuf+JJ1/tv/nR72/+NkMhkvY+oubkZU6dOxfTp07Ft2zZkZWXh1KlTSEtL69XjvvxJRYevlWTqMW/cgNDzv+0+Da8YeVrvgWla3Hnt5YnZXv2sEk5P+FCSX37vqphifO2113D//fdj37592L9/Px588EEUFhbigQceAAA8//zzWLVqFZ588kkAwOnTpzF79mz87ne/w6uvvor6+vpQMisrKwMALFq0CBcvXsQnn3wCpVKJhx9+GHV1dTHFR9rz+CSkiGZc03wI30hFaNQUwekVoaR7irokayL6wx/+gIKCgtAHBQBKSko6LO92u+F2X253Wyz99xJpQUEBXnrpJXAch2HDhuHIkSN46aWXQonoxhtvxGOPPRYqv2TJEsyfPx+PPvooAGDo0KH405/+hNLSUqxfvx7nzp3Dtm3bsG/fPkycOBEA8I9//ANXX311n59bf3VNvgFXN56DVF8HnD+Nz3IXwOUVYdAo5Q4t4cmaiN5//33MmjULd955J3bt2oUBAwbg5z//eejD1taaNWuwevXqHh936fQhHb7GtxkF8uANgzss23bEyOKpHSfR7vrOd74TNiRlypQpeOGFF0KDd6+99tqw8ocOHcLhw4fDmluMsdBQl5MnT0KhUIQt2TN8+HCYTKa4xXzFYwyc+RwEjoOKk6D1NsPtLZI7qqQga53xzJkzWL9+PYYOHYoPP/wQP/vZz/Dwww/jtddei1h+5cqVMJvNoUd1dXVMx1Up+A4fbW/N76xs2yp3pDK9Ra/Xhz232Wx46KGHcPDgwdDj0KFDOHXqFAYP7jiZkjhymQGvCwCgUQrIV/TfGnu8yVojkiQJ1157LX7/+98DAMaNG4ejR4/ir3/9KxYuXNiuvFqthlqt7uswZbF3796w53v27MHQoUMhCELE8uPHj8fx48cxZEjk2t7w4cPh8/lw4MCBUNPsxIkTaGlpiWvcV7I9355F2iUr8o0aDMtJxbBCNZBO4xyjIWuNKC8vD9dcc03Ytquvvhrnzp2TKaLEce7cOSxfvhwnTpzA5s2b8ec//xmPPPJIh+Uff/xxfP7551i2bBkOHjyIU6dO4b333sOyZcsAAMOGDcPs2bPx0EMPYe/evThw4ACWLFkCrVbbV6fU7zU21KHJ7oFPClzc8NjkDSiJyJqIpk6dihMnToRtO3nyJIqKqF29YMECOJ1OTJo0CUuXLsUjjzwSukwfyejRo7Fr1y6cPHkS119/PcaNG4dVq1YhPz8/VKasrAz5+fkoLS3FHXfcgQcffBDZ2dl9cTpXBpcZAKBQBjqn3VYZg0kusjbNfvnLX+K6667D73//e/zoRz/Cvn378Le//Q1/+9vf5AwrISiVSqxduxbr169v99rZs2cjvmfixIn46KOPOtxnbm4u/vWvf4Vtu/fee3sUJ7mMc/sTEW8ciPpLZ1BZVwW3vh7XD82SObLEJ2uNaOLEiXjnnXewefNmjBw5Er/97W+xdu1azJ8/X86wCOk2SWLg3f6mmGAaAJExeBxWmANjz0jnZJ8Y7dZbb8Wtt94qdxiE9IjbJ0Eh+a+YqVKzIHAcBMkDj5eGeURD9kRE2tu5c6fcIZBucnlFKEQXBJ4Dr89A8M4OyUPzpUeD7j0nJA48ogQVc0HBc4BKD07pn49ICtxXRDpHNSJC4iBHLyC72OCfBkSpBa/y3z/EPE55A0sSlIgIiQevAxw4cIIACCrwKv/9WZKXElE0qGlGSDwEE45SC3AcFGodVAKPVIGumkWDakSExEFlTT08l6zQp6dgIACtLgUTitKAIb07pU1/QYmIkDgwWyxw2j0QjYGPlCIwdIaaZlGhphkhcSC6/TNhcsrArAiKwOBsH81bHQ1KRITEQfB+IV4dGG0vqPBtrQU7v7mAZjvNKd4VapoREg+By/S8MtAkU6jg8IiwMztcPrGTNxLgSqsRMQb4PH3/YJHnvI5kw4YNyMjICJsSFwDmzZtHA1QTmOTz37goqAILKwpqCDwHgXnh9UX/+79SXVk1ItEL/OeFvj/u9Y8BClVURe+88048/PDDeP/993HnnXcC8K++sXXr1k5H1hN5Be+gVqqDNaJAIpI88IhUI+rKlVUjSgJarRb33HNP2IICr7/+OgoLCzFt2jT5AiOdC3RKK4JNM0HlH/jKvPBQjahLV1aNSFD6aydyHLcbHnjgAUycOBEXLlzAgAEDUF5ejkWLFoVNpk8Sy3cKUyDZ0oF0o3+DoALPcxAkL61tFoUrKxFxXNRNJDmNGzcOY8aMwYYNG3DTTTfh2LFj2Lp1q9xhkU5wohsCxwHBPiKFOlAj8lAiisKVlYiSyJIlS7B27VpcuHABM2fOREFBQddvIvIJ3i8kBO4fElRQCBw0vNhuiSrSHvURJah77rkH58+fx9///ncsXrxY7nBIJ3w+Ed9eaERFnQ0+PtAMV6hRnKHHtYVpGD8gRd4AkwAlogRlNBrxgx/8ACkpKZg3b57c4ZBOeDwuNNvdqLe5wSsCTTNeAXCBj5dId1d3hRJRArtw4QLmz59/xazllqy8bv+le14QwCsCNSKOu3yRwkd3VneF+ogSUHNzM3bu3ImdO3fiv//7v+UOh3TBExhnBkEVtg55g5PhUr0ZjtO1mDg6Q6bokgMlogQ0btw4NDc34w9/+AOGDRsmdzikC75AjQjBZlmAhylgcflgs9O81V2hRJSAOlq3jCQmnydyIuICzTTJS02zrsjaR/TUU0+B47iwx/Dhw+UMiZBuCyYiThHelxfsL2I+mqWxK7LXiEaMGIF///vfoecKRXxDYt0YcErao59f18RgIlKG14h4wX/zrChSjagrsicihUKB3NzcuO9XGVh/3OFwQKvVxn3/VwqHw9+/Efx5kvYGpytQUpIOMTs7bDuv9CciqhF1TfZEdOrUKeTn50Oj0WDKlClYs2YNCgsLI5Z1u91h02NYLJYO9ysIAkwmE+rq6gAAOp2Oxmp1A2MMDocDdXV1MJlMEARB7pASFufzD+8QgpOiBQiB4UQSzdLYJVkT0eTJk1FeXo5hw4ahpqYGq1evxvXXX4+jR48iNTW1Xfk1a9Zg9erVUe8/WNMKJiPSfSaTqVdqrP1KMNG0GcfIK5QQeA4qjsaadYVjCdQJ0NLSgqKiIrz44ou4//77270eqUZUUFAAs9kMg8HQ4X5FUYTXS9Xj7lIqlVQTisLZ3Zvgqz0G/dU3IW9U6eUXKv4NVH8JFH4HGDxdvgBlZLFYYDQau/yMyt40a81kMuGqq65CRUVFxNfVanVMdxkLgkAfKNJrmi02uKxu+Lw88lq/EBx3JvnkCCupJNQQD5vNhtOnTyMvL6/rwoQkCNZ2mtig4BAPumrWJVkT0YoVK7Br1y6cPXsWn3/+Ob7//e9DEATcfffdcoZFSLcwr7+7QNkmETFeiW9qLfj8ZC1cXpoutjOyNs3Onz+Pu+++G42NjcjKysJ3v/td7NmzB1lZWXKGRUi3sMDoeoU6/DYRTlDC4vSihTngFSVolNQ90BFZE9Ebb7wh5+EJiY/gfNURmmZ8YN5qn5gw14QSUkL1ERGSdBgDCyQiZZsaEXh/IuIlH7wSXcLvDCUiQnpC9EAKJJl2iUhQgucBnvmoRtSFhLp8T0jS8blxbVE6fOCg1HTQNJOoadYVSkSE9ITo8S+kqNQAQpsGBq+EwHHgGTXNukJNM0J6ou3qHa0JSvA8ByVEmsWgC1QjIqQHrHYbLtXbwKXoMbjti4ISV+elguN4IItW8ugM1YgI6QGH04F6qxsXbRFuWOSV4MABjAES3dDYGUpEhPRAcL5qXhm5aRYi0aDrzlAiIqQHRG9wChBN+xd5AZdsXnxba8G3Fxr7NrAkQ31EhPSA6HECgP+qWQR2H4dmhxeclVby6AzViAjpgeDE+bwi8vQ0XKB5JtEI/E5RIiKkB6RA0yxiHxHgX3QRtKRQVygREdIDojcwF1EHTbPgkkKiSJ3VnaE+IkJ64KoMFSQuDVJhdsTX+VDTjBJRZygREdIDCubxD+3QRF6yihP8HzFaUqhz1DQjpCd8gb4fQRXxZV5Ba5tFg2pEhPTAufpmiC47DC4OGRFez01LRV5JOrir0vs8tmRCiYiQHmgyW+B0ucGLQsRExAtKgOMA6iPqFDXNCImVJIEFEoxSHfmqWWiYBw3x6BQlIkJiJbohSf7pPZSqyJ3Vdh+HU3VW7K+s78vIkg41zQiJlc8NUWKQOAEqpTJyEQhosHnQpLD1cXDJhWpEhMRI8rohMkDkVVApIn+UBGXgahoN8ehUwiSiZ599FhzH4dFHH5U7FEKi4gkMeBU5FZQCF7GMEOgjYqKPZmnsREIkoi+//BKvvPIKRo8eLXcohETNF0hEkqCCou181QFCoMnGMx98EiWijsieiGw2G+bPn4+///3vSEtLkzscQqKm40RMKErDd4bmdVhGCNzQyDMfREpEHZI9ES1duhS33HILZs6c2WVZt9sNi8US9iBELrzkgUrgodfrOywjKFTg4E9EXpFW8uiI7EtOf/XVV/jyyy+jKr9mzRqsXr26l6MiJEqdreARxCvA8xwEiWpEnZGtRlRdXY1HHnkEGzduhKbtwnQdWLlyJcxmc+hRXV3dy1ES0rFmqxVnG+04a/Z1XEhQYnyhCdOvSoNRG/kSP5GxRnTgwAHU1dVh/PjxoW2iKGL37t34y1/+ArfbDUEQwt6jVquhVnfy34eQPmS22lBjdsFnEFHcUSFeCQXPA0z0D/UgEcmWiGbMmIEjR46EbbvvvvswfPhwPP744+2SECGJhgWumvEqXceFgkM8aKxZp2RLRKmpqRg5cmTYNr1ej4yMjHbbCUlEUjARdTA7o/9FBc43O+AQXTBZXMg2RNcNcaWR/aoZIclK9PoTkULdeY2oxelFo8UOi5Puru5ITDWiM2fOYNCgQfGOBTt37oz7PgnpLVJgBQ9FBwNeAfivmgX6hrxeap51JKYa0ZAhQzB9+nS8/vrrcLlc8Y6JkKTAfMEaUWeJSAk+0EdN81Z3LKZE9NVXX2H06NFYvnw5cnNz8dBDD2Hfvn3xjo2QxMUYWGAFD6W64xsawfPgeH/DwxdcFZa0E1MiGjt2LNatW4eLFy/i1VdfRU1NDb773e9i5MiRePHFF1FfT3OvkH5O9GBolh5jBhpRnNv5NLBcYEkhieat7lCPOqsVCgXuuOMOvPXWW/jDH/6AiooKrFixAgUFBViwYAFqamriFSchicXrhFLgodNourwhN1gjEqmPqEM9SkT79+/Hz3/+c+Tl5eHFF1/EihUrcPr0aWzfvh0XL17E3Llz4xUnIYklOLyjg6WmW+MCA19FmpOoQzFdNXvxxRdRVlaGEydOYM6cOdiwYQPmzJkDnvfntZKSEpSXl6O4uDiesRKSMJjXgapGOySdBgN9UocTowFAUbYRRVo3uIGpfRhhcokpEa1fvx6LFy/GokWLkJcXeQqE7Oxs/OMf/+hRcIQkKp/HhRqzC1aXDwPQ+WBWpULlX4QRYt8El4RiSkTbt29HYWFhqAYUxBhDdXU1CgsLoVKpsHDhwrgESUii8bkdAACRV0PVwaRoITTMo0sx9RENHjwYDQ0N7bY3NTWhpKSkx0ERkui8bv89RJxSA66LwazNLglnGmz49mJjX4SWlGJKRB3NvWuz2aKe0oOQZOYN1Ih4ZSc3MwbYvMAlixu1zdbeDitpdatptnz5cgAAx3FYtWoVdLrLY2xEUcTevXsxduzYuAZISCLyufyJiOtswGtAcLpYke4j6lC3EtHXX38NwF8jOnLkCFQqVeg1lUqFMWPGYMWKFfGNkJAEJAZrRJ1NARIQvKERYicTqF3hupWIPvnkEwD+eYPWrVsHg8HQK0ERkuhEjx0AwGs6Gd4RIAj+f9jMR/cRdSSmq2ZlZWXxjoOQpJKrk5Ax0Ajf4Pwuy/KBRRYliWpEHYk6Ed1xxx0oLy+HwWDAHXfc0WnZLVu29DgwQhKZUnRBqVIABlOXZUOLLFIfUYeiTkRGozF0mdJoNPZaQIQkPMaAwKRoiOKqWbBGxCRKRB2JOhG1bo5R04xc0bxOXDI74RElaN0CMrsYbpai1WJcoQlcJi0g2pGY7iNyOp1wOByh51VVVVi7di0++uijuAVGSMLyOlBvc6PKLKLF1fWwDUGhhEYhQM3TAosdiSkRzZ07Fxs2bAAAtLS0YNKkSXjhhRcwd+5crF+/Pq4BEpJwvA74RAleXguNMorVZgLTgNAQj47FPEPj9ddfDwB4++23kZubi6qqKmzYsAF/+tOf4hogIQnH64RXZPAKmqgSEeMVqGq049uLTXD7aOBrJDElIofDgdRU/5QGH330Ee644w7wPI/vfOc7qKqqimuAhCQayW2HKDH4eA20USQiTlCi1uLCxSYr3D5qnkUS8+T57777Lqqrq/Hhhx/ipptuAgDU1dXRTY6k3/M6rWAAvEKUTTNBCZ7jwEs+iGLnU4ZcqWJKRKtWrcKKFStQXFyMyZMnY8qUKQD8taNx48bFNUBCEo3HYQEAMFUKBD6KZaQDK3kIzAevRDWiSGJKRD/84Q9x7tw57N+/Hx988EFo+4wZM/DSSy9FvZ/169dj9OjRMBgMMBgMmDJlCrZt2xZLSIT0Ga/TP4qeU0c546LgX9uMYyJ8VCOKKOYlp3Nzc5Gbmxu2bdKkSd3ax8CBA/Hss89i6NChYIzhtddew9y5c/H1119jxIgRsYZGSK8yCi6MGWiEc3hxdG/gleB5DoLPB1GkGlEkMSUiu92OZ599Fjt27EBdXR2kNtXNM2fORLWf2267Lez5M888g/Xr12PPnj2UiEjCErx26FQK6NIzonyDMrDaK4OXhnlEFFMiWrJkCXbt2oV7770XeXl5Xc5QFw1RFPHWW2/BbreH+pzacrvdcLsvL1JnsVh6fFxCukWSgMDIe6i6HnkPILDsdODtNAI/opgS0bZt27B161ZMnTq1xwEcOXIEU6ZMgcvlQkpKCt555x1cc801EcuuWbMGq1ev7vExCYmZ145GmwsOjwSFk0de16sJAbyAwTkGQJIgpKm6Ln8FiqmzOi0tDenpna9uGa1hw4bh4MGD2Lt3L372s59h4cKFOH78eMSyK1euhNlsDj2qq6vjEgMhUXPb0Gjz4IwFqLFEX7vRqNXQKAUoQX1EkcSUiH77299i1apVYePNYqVSqTBkyBBMmDABa9aswZgxY7Bu3bqIZdVqdegKW/BBSJ9yW+EVJXgFHVLU3WhQ8LSSR2diapq98MILOH36NHJyclBcXAylUhn2+ldffRVzQJIkhfUDEZJQXGZ4RAluZSp0qihuZgxocEpwmB0QGi0YkJrTiwEmp5gS0bx58+Jy8JUrV+Lmm29GYWEhrFYrNm3ahJ07d+LDDz+My/4JiTfmaoHXJ8GlNXSrRtTolNDU4oSqxYoBvRhfsoopET355JNxOXhdXR0WLFiAmpoaGI1GjB49Gh9++CG+973vxWX/hMSbz94MkQFuIQU6VfQfHy4wS6Pko+liI4n5hsaWlha8/fbbOH36NH71q18hPT0dX331FXJycjBgQHQ5n5akJsnGY2sGAEhqY6fr3bcVXMlDoj6iiGJKRIcPH8bMmTNhNBpx9uxZPPDAA0hPT8eWLVtw7ty50FxFhPQrjMFr9yciQWfq1lv5wEoedB9RZDFdNVu+fDkWLVqEU6dOha3sOmfOHOzevTtuwRGSULxOGJQSxhaYMGPc0G699XLTjGpEkcRUI/ryyy/xyiuvtNs+YMAA1NbW9jgoQhKSywye46DVG6A1RTngNSC4kgc1zSKLqUakVqsjDq84efIksrKyehwUIQnJ1eL/qjV1+63BPiJaZDGymBLR7bffjqeffhperz+7cxyHc+fO4fHHH8cPfvCDuAZISMJwNOKi2YlvzUo027uXUHLTUjB6oBETCrtXk7pSxJSIXnjhBdhsNmRlZcHpdKK0tBRDhgxBamoqnnnmmXjHSEhisDeg3urGoSYBFlf3mlgqlRp6lQJ6geYjiiSmPiKj0Yjt27fjs88+w6FDh2Cz2TB+/HjMnDkz3vERkjCYoxFurwinwgSDRtn1G1qjIR6d6nYikiQJ5eXl2LJlC86ePQuO41BSUoLc3FwwxuIyJQghCUeS4LXWQ2SAQ5mGVE33PjoOkUdTswNuvgWDh/dSjEmsW00zxhhuv/12LFmyBBcuXMCoUaMwYsQIVFVVYdGiRfj+97/fW3ESIi+3GR6PFxInQKlPg0LoXq+GS+JQ3exEZV1L78SX5LqV1svLy7F7927s2LED06dPD3vt448/xrx587BhwwYsWLAgrkESIjt7I9w+ES6FEQZd9+cUUij872EiDfGIpFtpffPmzfiv//qvdkkIAG688UY88cQT2LhxY9yCIyRhOBrh9IhwKk0warufiHhl4D3URxRRtxLR4cOHMXv27A5fv/nmm3Ho0KEeB0VIwnE0wOWV4FSmISMlhhpR4IZGTvJCkujKWVvdapo1NTUhJ6fjuVRycnLQ3Nzc46AISTiORgzK0iNr0DUQ8ro/IZ8QqBHxzAefxKCKZj20K0i3EpEoilAoOn6LIAjw0TQHpL9hDLA3gOc4GDPzgO7MzBigCEwe6E9EElSx3cLXb3XrJ8oYw6JFi6BWR54xnGZWJP2Sxw743ADHAdrY5mrnBBV4DuAlEV5aZLGdbiWihQsXdlmGrpiRfsfZBKvLi/MuNcRaO0YOMHZ/H7wC1+QZAIUK2m5MMXul6FYiKisr6604CElcjkZY3T6cdRggNsaYiAQlUjVKf62K+ofaoYYqIV1xNIUu3afrY1yXLDjEg0n+BwlDiYiQrjia4PT6b2bM0EezomIEghINNjcutDjRYrPHN75+gBIRIV1grW5mTNN3c7BrEMfjosWNc00ONFl7vh5gf0OJiJDOSCK89mb4JAa30oi0GIZ3AAA4Dhzv75Kleavbo0RESGeczXC6vRA5JTR6E5TdHOzaWnDeap+XElFbsiaiNWvWYOLEiUhNTUV2djbmzZuHEydOyBkSIeEcTfD4JLiVRmSkxtg/FEQT6HdI1kS0a9cuLF26FHv27MH27dvh9Xpx0003wW6nzjySIByNyEpVY9q44Zh5dc+Wig7WiERKRO3EvMBiPHzwwQdhz8vLy5GdnY0DBw7ghhtukCkqQlpxNgEABH0G9DEM7Wjt8pJCNAKhLVkTUVtmsxkAkJ4e+TZ6t9sdNowk0koihMSVo9H/VZfR412FVnul8ZjtJExntSRJePTRRzF16lSMHDkyYpk1a9bAaDSGHgUFBX0cJbnSeG2NOHrRjF3nxR5P31GQYcQ1eQYMzdJ0XfgKkzCJaOnSpTh69CjeeOONDsusXLkSZrM59Kiuru7DCMkVx+OAy2GF1eVDhVUFvodDM1J0Ghi1SqQoaNBrWwnRNFu2bBn+9a9/Yffu3Rg4cGCH5dRqdYcj/wmJO6d/aIdH0MOUqu/5/oLDPCRqmrUla42IMYZly5bhnXfewccff4ySkhI5wyEknKMRTm8Px5i1YvEw1FpcuNBEfZttyVojWrp0KTZt2oT33nsPqampqK2tBeBfN02r1coZGiGXB7sqTMiLQyJqcjHUNNgh6VowIA7h9Sey1ojWr18Ps9mMadOmIS8vL/R488035QyLEL/AGDOX0hiXGlHw8j2j+4jakbVGxBh12pHEJdob4fZJcCri0zQTApfvGa3k0U7CXDUjJKFIEry2JqgUPKBLhy4OsyoKobXNKBG1lRBXzQhJOK4WaARgfEkWRl83Ki5LqfOBRCTRIovtUI2IkEgc/qEd0KZBoYjPHNNUI+oYJSJCIgmMMYvH0I4gPtBHRKu9tkdNM0IicTTieI0FdW4RI4vcyEzp+Y20KToNhuWmgjPE4ebIfoZqRIREINkbYHV5Ue3S9mgytNZUKjXSdSqkqWkVj7YoERESgcvcAIkBPrUJBk2cGg5C4BYAkWZobIuaZoS05XXBZfcPw9AYs+NyxQwAIKjRYHPDazcjzStCo6SFFoMoERHSltO/fJBX0CLNkBK//SpUqGywwyfZoXJ5KRG1Qk0zQtoKLh+kMMW+akckghoCzwFg8HhplsbWKBER0pajCQ6PDy6lERlxuFoWIighBDq+vW5X/PbbD1AiIqQN5miESsGD6dKRmRLHGhHHhTqsfR5KRK1RHxEhbXDOJgzPNWD4qNFAPJtmADilGoADXkpEYahGREhrkgQ4mv3f6yIv4tATnOBv6oke6iNqjRIRIa25WiCJXoBXABpT3HfPK/2JiJpm4ahpRkhrjkacqrOhnhmR2+DAkOw4Xr4HMDAzDSLXBKWJLt23RomIkNbsDXB4fGhWGlGkiH+DwZiqB5wqQEmTArZGTTNCWvFa6+D2SnAqTciI5xWzoEAfEWi11zCUiAhpxdpUCwaAT8mEThX/BoNdFFBvc6O2yRz3fSczapoREsQYHC11AIDUjLxeOUSDC7hYZ4MkNCG3V46QnKhGREiQywyHwwnG8cjIyO6VQwgqf9OM0RCPMJSICAmyXYLN7YNTkYYcU+9MXqZQ+te9Z9RHFIaaZoQEiJYaZOhVsKrykGPQ9MoxFCr/fiVKRGFkrRHt3r0bt912G/Lz88FxHN599105wyFXOMFeh6IMPUrHj/IvI9QLlCqqEUUiayKy2+0YM2YMXn75ZTnDIARgDLDW+L9P7b1uZLUmsJS6zwVRonuJgmRtmt188824+eab5QyBEACAz94ES3MLjHoNhJTe6agGAJXG3/ekkDxw+8ReuUUgGSXVT8HtdsPtvlyltVgsMkZD+pOacydx/pIVktGIqYE16nsDr9JiSHYKBJ6Dki4VhSTVj2LNmjUwGo2hR0FBgdwhkX6i5uwJAIA+e1DvHkihRVaKfzUPpUT9REFJlYhWrlwJs9kcelRXV8sdEukHmmwuuOpOAwCKB1/duwfjeUARGObhpRH4QUnVNFOr1VCr4zh1J7niMcaw7+AhGEUnjIZUmPJ6uUYEwOxTwGOzQGkxw6SP30qyySypakSExNu+yiZ4zh8CzwFFV40B+N6fnqPKIqGizoaahuZeP1aykLVGZLPZUFFREXpeWVmJgwcPIj09HYWFhTJGRq4EFXU27D9RhXGOChRn6KEvntgnx+VVOgCA1+3ok+MlA1kT0f79+zF9+vTQ8+XLlwMAFi5ciPLycpmiIleKIpMCk9xfINuoQk7hVYBxQJ8cl1f57yXyuex9crxkIGsimjZtGhijm7pIH5NEoOYQlFWfYYLBCk4wAkNv6rPDK9T+e4lEj7PPjpnokqqzmpCeOvXtYQgVH6FE7wEHDrzWBAy7GUjJ6rMYFIGmmeihplkQJSJyZZAknN33LzQe2w0GwDAwB5kjbgTyxgBC334MlFp/IvK5qWkWRImI9HtejxvffLwR9gvHAQCa4knIuOF2QNk7I+y7otb6J+QXXVQjCqJERPq1huYWnPx3GTjLBTCOh270XIyZcB04jpMtJqMxDYOz9FCk0EoeQZSISL914sxZ1H6+ESp3M3ilBgU33Iv84mFyhwW1zoDsVA2g8MgdSsKgRET6H9EHnPsC2Sd2odndAp3BhCEz74curXfmoe42dWCtNJ/H/1D0wmohSYYSEelfWqqBkx8A9gakaQQMGzEWprG3gdMY5Y7sMoUaZi8Hj8sFjbUZqWk5ckckO0pEpH/weYAzn6Dp5BdI0Sih0qYCQ7+HtKzhgIz9QR053QK4rTbkNjRSIgIlItIf2OqBo/8DW0s9TtXZ0Gy8BtdN/iEMqQa5I+sQp0kFrA1wO6xyh5IQKBGR5NZ8Fji6BaLHhWNNDMey5iC7aDhSU1LljqxTSm0qXADcdlpoEaBERJJZcxVw+C1A8uGk24S96d+FTp+CmVfnyHp5Phoqvb/Pyu2gRARQIiLJyloLHH0bkHyo1xTgQ+94ME6Bm67JhUaZ+PfnaFNNAACvjaYCAWg+IpKMHE3A4TcBnwfe1IF43zMJjFNgbIEJhRk6uaOLis7gn6BfcjTKHElioEREkovbChx6A/A4gNQcfK6bBosHMOmUmDokU+7ooqZP8w+y5V0t8PpEmaORHzXNSPLwuvw1IZcZ0KUDo+/CFF4LF6vDuAJTry2K2BvUKWkYlJ0KjcCB91gBhUnukGRFiYgkB9EHHP0fwFYPptJDGvkjCCo9VABmjei9BRF7C8cLyMnO9Tcz3S2AziR3SLJKnn8h5MolicDxd4GWc3BKArbieuyq7gfjtLTp/q/UT0SJiCQ40Qccewf2i9/gVIMTb7om4ZRDj+MXLTA7vHJH1yNOdTrqrC5UVZ2ROxTZUdOMJCyn3YqGvW/CUlOBFjfDicxZMKvyUZKpR+lVWTDqem9F1r7QImTgbL0dzHEGRdfJHY28KBER2TDG0Gj3oNbsgtXlg9snoiRTj6IMPdBcBfO+LTh//gJETolT2bOQUzgcswpNGGDSyh16XKTlFuEsANgb4HDYodPpZY5IPpSISJ9jjOHEJSu+PNuMBuvlZZcFyQOj3YGiC1VAYwUyFRIupqaDH/UD/LBkEFI1yV0DakujN0GhT4PP3ozaqhMYdPV4uUOSDSWifk6UGCxOL9L0iTHnjdMjYtuRC2i5WIFU9yVcJVqRo3LCCBvUkgtGUQlolQDHQTFwAsaXTr+8RHN/w3HQ5g6D9fQeNJ87DlAiktfLL7+MP/7xj6itrcWYMWPw5z//GZMmTZI7rMRlb/CPs3I2AxwAXQaQVgJoTQAASWKobLTj8PkWVDc5IfAclk4fEnp7RZ0NPAcMTNP13b03jAHmaqgvfYOSk3vgcVoxwKRFbpoGCp6H/7qJDtCmARlDgAHj/fcK9XPZg0bBenoPPBePweW0Q6O9MptnsieiN998E8uXL8df//pXTJ48GWvXrsWsWbNw4sQJZGdnyx1e4vC5gbrjQM1hwHIxYhG3LhcVKMB+RzaafJcnhte2GXu179RFWFsaoBXtyNd6kZGqgV6XAlVqGtTGbOSlX55ErMXhgUYpxDZ+S/SBWc6j4ewxpDvOQPDYwAMYls6DU+VCmzsM0GX6E6g2DdCYZJvQXi5ZA4eiMiULkq0eJ/d9hNGl35c7JFlwTOYVDidPnoyJEyfiL3/5CwBAkiQUFBTgF7/4BZ544olO32uxWGA0GmE2m2EwJO7cMzERfYDbAlguAE1ngPqTgOTzv8bxQFoxmD4LHCTAWouLVadwrtEO/y+Tg0ebiaycASjIMiFFCXBeJ+A2Q3KaUXWpCS0OL1w+qd1hUzUKjBxUCOizAH0m/udbF1okDQSVDgadGmkaHulaDmkqEWlKEam8B/A6AK8z8NUBn9sOp8MGq82ORpsHNrcPA01aFGSbgMxhQPbVQFpxn6wznwyqTx7E+f+87v8Zjb/JXxtU9Y+aUbSfUVlrRB6PBwcOHMDKlStD23iex8yZM/HFF1+0K+92u+F2X+7ctFgs0R2oqRI4vcPfPOhI2GsMdVYXzjbYwYEh7F2BJ0OyU5AeuHzcYHfjTJ0tvGSrbwdn6ZCZ4u/naLJ7cOpS68mwWKsvDMWZeuSkqgCvC2anFxV11tCuXEoTGlKGoyHlKniatbghMwsjB/hrL1JePSo//w8GoxolymZk6EXwXDXQWB12mjyAkswUQKmFg09Bg1cNq9sHt8MGztUMLe/2D6FwmYHGCgxqbIbLG56wHIGHWa3AqAGXa0/7q5ogSgxSq3P38lpYUwZDUTgKBeOv7fM1xJJBwVVjoXfWIL3pIHD2U+DspzhW74HVJwAcD47nA7NM+qc2USsEjC24/HM/csECh8cXcd9Kgcf4QlPo+fEaC6yuyGUFjsO1xWmh59/WWmF2eqFVChg9MMJUu6N+BGjiUwGQ9a+ioaEBoigiJyd8qsycnBx8++237cqvWbMGq1ev7v6BfG7/LH7dwOwuiPZOFsBzegE+0InqdEN02jrel1sClIGb79weSO6Oy8LDAYEPvsQr0CRkwaLJQ6N2EOyqTP8fpARAkmBzX/6DGpCTidmzbvV3SrssgLXGP3xA8gKcAKh0gNoAaIz+rwoVdAAK2x3fAdjrAUcDYG/AOFM9RLcdLqcdTrcHdi8HuxewiApwKQYgNxdQagGlHqetDXAyNbyCBkqNHmkmEwqy0jAxzwitimo/nUkfPRuoKwSq9wLWS3A57GCi/++g7b9PpuAB2+U7y5mtBZI78sBZJnCA7fKNn8xqhtRBIuJ4DrD5WpW1QHJ6wVRC2PEuF2hfo45VUv17WrlyJZYvXx56brFYUFBQ0PUbTQXAmB/7vw+bMKvN5FmtXtO7RRQ6vYHNbSfZ4qDVKwGlAuA46L0+FDt9oddCpTn/93qtAlD6a086n4ghbf8QWu0/Ra0CVAKg1EHHqTDWJbYLmwvElKJWtNoFd/nKmMYQ+38qlQ5QFQFpRaFNAgB94NHZ+PZZWR7wPAeVwCfFnEAJheOAnBH+h9eFomsaIHrdkEQRjDEw6fKHXiFwQOrlK4l5g9zwiJFr+zwPwHC53y1nsBsmX+SyHAfAeLls1mA3DD7W7nghcWw+ypqIMjMzIQgCLl26FLb90qVLyM1tP5BRrVZDrY7hUq5KD6SXdOstKYFHNLSBRzQ0gUc01ACylMnzv8KkS4xbBJKeUoPMnIFRF8/qxsXFzF4q21OyjjVTqVSYMGECduzYEdomSRJ27NiBKVOmyBgZIaQvyf7vdvny5Vi4cCGuvfZaTJo0CWvXroXdbsd9990nd2iEkD4ieyK66667UF9fj1WrVqG2thZjx47FBx980K4DmxDSf8l+H1FP9Ov7iAjpB6L9jNJ8RIQQ2VEiIoTIjhIRIUR2sndW90SweyvqoR6EkD4V/Gx21RWd1InIavWP2Yrq7mpCiGysViuMxgjj1QKS+qqZJEm4ePEiUlNTE36t86DgsJTq6up+daWvP55XfzwnoG/PizEGq9WK/Px88HzHPUFJXSPieR4DB0Z/K3wiMRgM/eqPO6g/nld/PCeg786rs5pQEHVWE0JkR4mIECI7SkR9TK1W48knn4xtFoEE1h/Pqz+eE5CY55XUndWEkP6BakSEENlRIiKEyI4SESFEdpSICCGyo0TUy5555hlcd9110Ol0MJlMUb2HMYZVq1YhLy8PWq0WM2fOxKlTp3o30G5qamrC/PnzYTAYYDKZcP/998Nm62R1EgDTpk0Dx3Fhj5/+9Kd9FHFkL7/8MoqLi6HRaDB58mTs27ev0/JvvfUWhg8fDo1Gg1GjRuH//u//+ijS7unOeZWXl7f7vWg0fbzQJSO9atWqVezFF19ky5cvZ0ajMar3PPvss8xoNLJ3332XHTp0iN1+++2spKSEOZ3O3g22G2bPns3GjBnD9uzZw/7zn/+wIUOGsLvvvrvT95SWlrIHHniA1dTUhB5ms7mPIm7vjTfeYCqVir366qvs2LFj7IEHHmAmk4ldunQpYvnPPvuMCYLAnnvuOXb8+HH2//7f/2NKpZIdOXKkjyPvXHfPq6ysjBkMhrDfS21tbZ/GTImoj5SVlUWViCRJYrm5ueyPf/xjaFtLSwtTq9Vs8+bNvRhh9I4fP84AsC+//DK0bdu2bYzjOHbhwoUO31daWsoeeeSRPogwOpMmTWJLly4NPRdFkeXn57M1a9ZELP+jH/2I3XLLLWHbJk+ezB566KFejbO7unte0f5t9iZqmiWYyspK1NbWYubMmaFtRqMRkydPjrj6rRy++OILmEwmXHvttaFtM2fOBM/z2Lt3b6fv3bhxIzIzMzFy5EisXLkSDoejt8ONKLjKcOufc2erDAP+825dHgBmzZqVML8XILbzAgCbzYaioiIUFBRg7ty5OHbsWF+EG5LUg177o9raWgCIuPpt8DW51dbWIjs7O2ybQqFAenp6pzHec889KCoqQn5+Pg4fPozHH38cJ06cwJYtW3o75Ha6u8ow4D/vRP69ALGd17Bhw/Dqq69i9OjRMJvNeP7553Hdddfh2LFjfTaonGpEMXjiiSfade61fXT0S09kvX1eDz74IGbNmoVRo0Zh/vz52LBhA9555x2cPn06jmdBumvKlClYsGABxo4di9LSUmzZsgVZWVl45ZVX+iwGqhHF4LHHHsOiRYs6LTNo0KCY9h1c4fbSpUvIy8sLbb906RLGjh0b0z6jFe155ebmoq6uLmy7z+dDU1NTxBV6OzJ58mQAQEVFBQYPHtzteHuiu6sMA/7fTXfKyyGW82pLqVRi3LhxqKio6I0QI6JEFIOsrCxkZWX1yr5LSkqQm5uLHTt2hBKPxWLB3r178bOf/axXjhkU7XlNmTIFLS0tOHDgACZMmAAA+PjjjyFJUii5ROPgwYMAEJZw+0rrVYbnzZsH4PIqw8uWLYv4nilTpmDHjh149NFHQ9u2b9+eUKsSx3JebYmiiCNHjmDOnDm9GGkbsnaVXwGqqqrY119/zVavXs1SUlLY119/zb7++mtmtVpDZYYNG8a2bNkSev7ss88yk8nE3nvvPXb48GE2d+7chLx8P27cOLZ371726aefsqFDh4Zdvj9//jwbNmwY27t3L2OMsYqKCvb000+z/fv3s8rKSvbee++xQYMGsRtuuEGuU2BvvPEGU6vVrLy8nB0/fpw9+OCDzGQyhS5d33vvveyJJ54Ilf/ss8+YQqFgzz//PPvmm2/Yk08+mbCX77tzXqtXr2YffvghO336NDtw4AD78Y9/zDQaDTt27FifxUyJqJctXLiQAWj3+OSTT0JlALCysrLQc0mS2G9+8xuWk5PD1Go1mzFjBjtx4kTfB9+JxsZGdvfdd7OUlBRmMBjYfffdF5ZcKysrw87z3Llz7IYbbmDp6elMrVazIUOGsF/96ley3kfEGGN//vOfWWFhIVOpVGzSpElsz549oddKS0vZwoULw8r/85//ZFdddRVTqVRsxIgRbOvWrX0ccXS6c16PPvpoqGxOTg6bM2cO++qrr/o0XpoGhBAiO7pqRgiRHSUiQojsKBERQmRHiYgQIjtKRIQQ2VEiIoTIjhIRIUR2lIgIIbKjRESSSnl5ediUu0899VTYYOBFixaFxliR5EGJiES0aNGiDueUXrp0KTiOCxup3xsJoLi4GGvXrg3bdtddd+HkyZMdvmfdunUoLy8PPZ82bVrYIFWSmCgRkQ4VFBTgjTfegNPpDG1zuVzYtGkTCgsLZYlJq9W2m5StNaPRGPUiBSRxUCIiHRo/fjwKCgrCZlDcsmULCgsLMW7cuB7tO1JNZd68eaFa1rRp01BVVYVf/vKXoUnZgPZNs7Za18wWLVqEXbt2Yd26daF9VFZWYsiQIXj++efD3nfw4EFwHNenc/CQyygRkU4tXrwYZWVloeevvvoq7rvvvl4/7pYtWzBw4EA8/fTTqKmpQU1NTbf3sW7dOkyZMgUPPPBAaB+FhYXtzgkAysrKcMMNN2DIkCHxOgXSDZSISKd+8pOf4NNPP0VVVRWqqqrw2Wef4Sc/+UmvHzc9PR2CICA1NRW5ubkxzYJoNBqhUqmg0+lC+xAEAYsWLcKJEydCa315vV5s2rQJixcvjvdpkCjRDI2kU1lZWbjllltQXl4OxhhuueUWZGZmyh1Wj+Tn5+OWW27Bq6++ikmTJuF///d/4Xa7ceedd8od2hWLakSkS4sXL0Z5eTlee+21uNUaeJ5H26mwvF5vXPYdjSVLloQ64svKynDXXXdBp9P12fFJOEpEpEuzZ8+Gx+OB1+vFrFmz4rLPrKyssH4fURRx9OjRsDIqlQqiKPboOB3tY86cOdDr9Vi/fj0++OADapbJjJpmpEuCIOCbb74Jfd8Rs9kcmhA/KCMjAwUFBe3K3njjjVi+fDm2bt2KwYMH48UXX0RLS0tYmeLiYuzevRs//vGPoVarY2oSFhcXY+/evTh79ixSUlKQnp4OnudDfUUrV67E0KFDE2oC/CsR1YhIVAwGAwwGQ6dldu7ciXHjxoU9Vq9eHbHs4sWLsXDhQixYsAClpaUYNGgQpk+fHlbm6aefxtmzZzF48OCYV01ZsWIFBEHANddcg6ysLJw7dy702v333w+Px9MnVwFJ52jOanLF+s9//oMZM2agurq63cqopG9RIiJXHLfbjfr6eixcuBC5ubnYuHGj3CFd8ahpRq44mzdvRlFREVpaWvDcc8/JHQ4B1YgIIQmAakSEENlRIiKEyI4SESFEdpSICCGyo0RECJEdJSJCiOwoERFCZEeJiBAiu/8PO5X9/iClFbcAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:00.015266Z", + "iopub.status.busy": "2024-03-22T19:36:00.014972Z", + "iopub.status.idle": "2024-03-22T19:36:00.404547Z", + "shell.execute_reply": "2024-03-22T19:36:00.403586Z" + }, + "papermill": { + "duration": 0.41266, + "end_time": "2024-03-22T19:36:00.406536", + "exception": false, + "start_time": "2024-03-22T19:35:59.993876", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7FElEQVR4nO3deXxU5b0/8M85Z+bMllmyTDayQlhEdhGKVgGlorhAbdVaK0FQ24r31ubSKu2vtNiXhbZCpa0v2ttbSL2FUq1orVYpFxUsCrIICkiAkIQACdlnX885vz9mIZPMJJNkkjOTfN+v17zIzJw58x2SfPM8z3me78NIkiSBEEJkxModACGEUCIihMiOEhEhRHaUiAghsqNERAiRHSUiQojsKBERQmRHiYgQIjuF3AEMhCiKuHz5MvR6PRiGkTscQkgXkiTBZrMhPz8fLBu73ZPSiejy5csoLCyUOwxCSC/q6+tRUFAQ8/mUTkR6vR5A4EMaDAaZoyGEdGW1WlFYWBj+XY0lpRNRqDtmMBgoERGSxHobOqHBakKI7CgREUJkR4mIECK7lB4jiockSfD7/RAEQe5QRjSlUgmO4+QOgySpYZ2IvF4vGhoa4HQ65Q5lxGMYBgUFBUhLS5M7FJKEhm0iEkURNTU14DgO+fn54HmeJj3KRJIkNDc34+LFixg7diy1jEg3wzYReb1eiKKIwsJCaLVaucMZ8cxmM2pra+Hz+YZ1IvL4BXAMAwVHw699Mez/t3qaVk6GzkhpjZ64ZMGrRy/C4fHLHUpKod9SQhLococblzvcON1olTuUlEKJiJAEarV7AADmNLXMkaQWSkRJaN68eXjqqafkDoP0kShK6HD5AAD/PNGAtz5tkDmi1EGJKIW9//77YBgGHR0dcodCADh9AkK7BLq8AlodHnkDSiGyJ6JLly7hG9/4BjIzM6HRaDB58mQcPnxY7rAI6bOuA9R2GrCOm6yJqL29HTfeeCOUSiXefvttnDp1Chs2bEB6evqgvq/XL8a8+QUx7mN9cRw7UB6PB08//TQKCwuhUqlQVlaGP/7xj6itrcX8+fMBAOnp6WAYBsuWLev1fDabDQ899BB0Oh3y8vLwq1/9qltX8H//938xc+ZM6PV65Obm4utf/zqamprCz4daYnv27MHMmTOh1Wpxww03oKqqasCfN5XZ3IHEk6HjAQAeX/efERKdrPOIfv7zn6OwsBBbt24NP1ZaWjro7/vie+diPleapcOS6aPC9/97XzV8QvRduQvSNbhv5tXCbFv218DljVxK8t0vjRtQrEuXLsVHH32EX//615g6dSpqamrQ0tKCwsJCvPrqq/jKV76CqqoqGAwGaDSaXs9XUVGB/fv344033kBOTg7WrFmDo0ePYtq0aeFjfD4ffvrTn2L8+PFoampCRUUFli1bhn/+858R5/rhD3+IDRs2wGw241vf+haWL1+O/fv3D+jzpjKvX0SaYMHE9uP4XCxGq7oYLp8AJc0p6pWsieiNN97AwoULcd9992Hv3r0YNWoUnnjiCTz22GNRj/d4PPB4rva7rdbhfYn0zJkzePnll7F7924sWLAAADB69Ojw8xkZGQCA7OxsmEymXs9ns9nwpz/9Cdu3b8ett94KANi6dSvy8/Mjjlu+fHn469GjR+PXv/41rr/+etjt9oglGs899xzmzp0LAHjmmWdw5513wu12Q60emVeMJuYbcE3rBYjNTcDFauzPXQq3T4BBrZQ7tKQnayI6f/48Nm/ejIqKCvzgBz/AoUOH8J//+Z/geR7l5eXdjl+3bh3Wrl074PddOb8s5nNsl3l3j988JuaxXefoLb8xsa25Y8eOgeO48C/7QJ0/fx4+nw+zZs0KP2Y0GjF+/PiI444cOYKf/OQnOH78ONrb2yGKge7FhQsXMHHixPBxU6ZMCX+dl5cHAGhqakJRUVFC4k05kgTGcgEcw4BnRGh87fD4iuWOKiXImohEUcTMmTPxs5/9DAAwffp0nDhxAr/73e+iJqLVq1ejoqIifD9UhrKveEX8TeXBOjYe8XS1Es3hcGDhwoVYuHAhtm3bBrPZjAsXLmDhwoXwer0RxyqVV//Sh2ZOh5LWiOS2AD43AECt5JCvGN4t9kSStfOal5cX8RcWAK655hpcuHAh6vEqlSpcFnYklIedPHkyRFHE3r17oz7P84FB0XhLnIwePRpKpRKHDh0KP2axWHDmzJnw/dOnT6O1tRXr16/HTTfdhAkTJkQMVJPYDpyuRdUVG2xuH8bn6HF3mQqFGbTOMR6yJqIbb7yx25WWM2fOoLiYmrMAUFJSgvLycixfvhyvv/46ampq8P777+Pll18GABQXF4NhGLz55ptobm6G3W7v8Xx6vR7l5eX43ve+h/feew8nT57EihUrwLJsuEVTVFQEnufxm9/8BufPn8cbb7yBn/70p4P+WYeD1pYmtDm88IvBixvenr8f5CpZE9F3v/tdHDhwAD/72c9w7tw5bN++Hf/93/+NlStXyhlWUtm8eTO++tWv4oknnsCECRPw2GOPweFwAABGjRqFtWvX4plnnkFOTg6efPLJXs+3ceNGzJkzB3fddRcWLFiAG2+8Eddcc014gNlsNqOyshKvvPIKJk6ciPXr1+P5558f1M84bLgtAABFqMvqsckYTGphJEmKfm16iLz55ptYvXo1zp49i9LSUlRUVMS8ataV1WqF0WiExWLp1k1zu92oqalBaWnpiL2KEw+Hw4FRo0Zhw4YNWLFixaC9z0j4fvzzb/+DdMtpTJk0Bc4r51HjUsEz41HcNNYsd2iy6el3tDPZ6xHddddduOuuu+QOY8T45JNPcPr0acyaNQsWiwXPPvssAGDx4sUyR5baRFEC6wl0xTjTKAiN1fA6bbAE156RnsmeiEjidL283tWpU6cAAM8//zyqqqrA8zyuu+46fPDBB8jKyhqqMIclj1+EQgxcMeP1ZnAMA070wuujZR7xoEQ0jOTn5+PYsWM9Pl9UVIQjR44MXVAjhNsnQCG4wbEMWF0mQpOpRS/VS48HJaJhRKFQoKws9mRNMni8ggheckPBMgCvA6NUA7BDDM4rIj2jRERIAuToOGSXGAJlQJQasHxg/pDkdckbWIqgRERIIvicYMCA4TiA48HygVnxoo8SUTxoWTAhiRBKOEoNwDBQqLTgORZ6jq6axYNaRIQkQE1DM7xXbNBlpKEAgEabhuuK04Gywa2tNVxQIiIkASxWK1wOLwRj8FdKEVywTF2zuFDXjJAEEDyBZTeMUhd4QKEK/OunutXxoESUhJJtF49kiycZheYLsarganuOx+lGK97//BLaHd4eXkkASkTDVtfaQWSQBS/Ts8pgl0zBw+kV4HA44PbHV6ZlJBtZiUiSAL936G99WFe8bNky7N27F5s2bQLDMGAYBtXV1VixYgVKS0uh0Wgwfvx4bNq0qdvrlixZgueeew75+fnhqosffvghpk2bBrVajZkzZ+L1118HwzARM7BPnDiBO+64A2lpacjJycHDDz+MlpaWmPHU1tYO+Fsx3Ij+wMRFjg8u6OVU4FgGnOSDzy/ruvKUMLIGqwUf8MGGoX/fm/4LUPBxHbpp0yacOXMGkyZNCi9ITU9PR0FBAV555RVkZmbiww8/xOOPP468vDzcf//94dfu2bMHBoMBu3fvBhBY+Xz33Xdj0aJF2L59O+rq6rp1sTo6OnDLLbfg0Ucfxa9+9Su4XC48/fTTuP/++/Huu+9GjcdsHrmryWMJzaBWqkItomAiEr3wxlm4biQbWYkoBRiNRvA8D61Wi9zc3PDjnWt1l5aW4qOPPsLLL78ckYh0Oh3+53/+J1y58Xe/+x0YhsEf/vAHqNVqTJw4EZcuXYoos/Lb3/4W06dPD5frBYAtW7agsLAQZ86cwbhx46LGQ7oIDkorQl0zjg8sfJV88FKLqFcjKxFxykDrRI73HaAXX3wRW7ZswYULF+ByueD1eiO2AAICpWVDSQgAqqqqMGXKlIj6P50L5wPA8ePH8d5770XszhFSXV2NceMGth3SSPGFojSI9gwgwxh4gOPBsgw40Ud7m8VhZCUihom7i5RMduzYgVWrVmHDhg2YM2cO9Ho9fvnLX+LgwYMRx+l0uj6f22634+6778bPf/7zbs+FduYgvWMEDziGAUJjRApVsEXkpUQUh5GViFIEz/MRBfH379+PG264AU888UT4serq6l7PM378ePz5z3+Gx+OBShWY19K5cD4AzJgxA6+++ipKSkqgUET/cegaD4kiNF+IC84f4ngoOAZqVui2RRXpbmRdNUsRJSUlOHjwIGpra9HS0oKxY8fi8OHD2LVrF86cOYMf/ehH3RJKNF//+tchiiIef/xxfP7559i1a1e4/nSoWP7KlSvR1taGBx98EIcOHUJ1dTV27dqFRx55JJx8usYzorcMisLvF3D6UivONdnhZ4PdcIUKJZk6zCxKx4xR3bu9JBIloiS0atUqcByHiRMnwmw2Y+HChbj33nvxwAMPYPbs2WhtbY1oHcViMBjwj3/8A8eOHcO0adPwwx/+EGvWrAGA8LhRfn4+9u/fD0EQcNttt2Hy5Ml46qmnYDKZwLJs1Hhibfc0Unm9brQ7PGi2e8Aqgl0zVgEwwV8vgWZX90b24vkDQcXz+27btm145JFHYLFYhnQDx+H8/bC0t+LUznVgOQ6zy9df3QL4g42BLtusxwFdprxByiRliueTwfXSSy9h9OjRGDVqFI4fPx6eIyTHLrLDlTe4zgwcH7EPeYtLwpVmC5zVjbh+yshMRPGiRDTMNTY2Ys2aNWhsbEReXh7uu+8+PPfcc3KHNaz4PcFysIrIlp5XUsDq9sPuoLrVvaFENMx9//vfx/e//325wxjW/N7oiYhRBAauRR+t++uNrIPVP/nJT8Lrl0K3CRMmyBkSIX0WSkRMqPRHEBtMRJKfqjT2RvYW0bXXXov/+7//C9+PNZelv1J4LH5YGc7fByGUiJSRLSKWC0yeFQRqEfVG9kSkUCgGZQ2TMrj/uNPppIHZJBAqS8JxnMyRJN6YDAVKSzMgZGdHPM4qA4mIWkS9kz0RnT17Fvn5+VCr1ZgzZw7WrVuHoqKiqMd6PB54PFfnZFit1pjn5TgOJpMJTU1NAACtVhuexEeGliiKaG5uhlarTXiLNxkw/sDyDi5UFC2ICy4nEqlKY69k/amYPXs2KisrMX78eDQ0NGDt2rW46aabcOLECej1+m7Hr1u3LmIVem9CLa1QMiLyYVkWRUVFw/OPQSjRdFnHyCqU4FgGPEMz0XuTVBMaOzo6UFxcjI0bN2LFihXdno/WIiosLOx1spQgCPD5qHksJ57nwzO1h5vafdvhbzwJ3TW3IW/y3KtPnPs/oP4QUPQFYMx8+QKUUUpOaDSZTBg3bhzOnTsX9XmVShVevNkXHMcNy7EJkhzarXa4bR74fSwi6hWE1p2JfjnCSilJ9SfKbrejurqayk+QlCJ1LRMbEqpDRVfNeiVrIlq1ahX27t2L2tpafPjhh/jyl78MjuPw4IMPyhkWIX0i+QLDBcouiUhilfi80YoPzzTC7aMyKj2RtWt28eJFPPjgg2htbYXZbMYXv/hFHDhwgGoik5QiBVfXK1SR00QYTgmry4cOyQmfIEKtpOGBWGRNRDt27JDz7QlJjFC96ihdMzZYt9ovJM01oaSUVGNEhKQcSYIUTETKLi0isIFExIp++KiYXI8oEREyEII3XLGyWyLilGBZgJX81CLqRVJdvick5fg9mFmcAT8YKLsWfAt1zUTqmvWGEhEhAyF4AxspKtUA16WDwSrBMQxYibpmvaGuGSED0XX3js44JViWgRLCsK4+kAjUIiJkAGwOO64028Gk6TCm65OcEtfk6cEwLGCmnTx6Qi0iQgbA6XKi2ebBZXuUCYusEgwYQJIAkSY09oQSESEDEKpXzSqjd83CRFp03RNKRIQMgOALlQCJskUSy+GK3YfTjVacvtQ6tIGlGBojImQABK8LAAJXzaJw+Bm0O31gbLSTR0+oRUTIAIQK57OK6OVpmGD3TKQV+D2iRETIAIjBrlnUMSIgsOkiaEuh3lAiImQABF+wFlGMrlloSyFBoMHqntAYESEDMC6Th8ikQyzKjvo8G+6aUSLqCSUiQgZAIXkDSzvU0besYrjArxhtKdQz6poRMhD+4NgPx0d9mlXQ3mbxoBYRIQNwobkdgtsBg5tBZpTnc9P1yCvNADMuY8hjSyWUiAgZgDaLFS63B6zARU1ELKcEGAagMaIeUdeMkP4SRUjBBKNURb9qFl7mQUs8ekSJiJD+EjwQxUB5DyUffbDa4WdwtsmGwzXNQxlZyqGuGSH95fdAECWIDAdeqYx+CDi02L1oU9iHOLjUQi0iQvpJ9HkgSIDA8uAV0X+VOGXwahot8ehR0iSi9evXg2EYPPXUU3KHQkhcvMEFrwLDQ8kxUY/hgmNEkuCnKo09SIpEdOjQIfz+97/HlClT5A6FkLj5g4lI5HgoutarDuKCXTZW8sMvUiKKRfZEZLfb8dBDD+EPf/gD0tPT5Q6HkLhpGQHXFafjC2PzYh7DBSc0spIfAiWimGRPRCtXrsSdd96JBQsW9Hqsx+OB1WqNuBEiF1b0gudY6HS6mMdwCh4MAonIJ9BOHrHIvuX00aNHcejQobiOX7duHdauXTvIURESp5528AhhFWBZBpxILaKeyNYiqq+vx3e+8x1s27YN6q4b08WwevVqWCyW8K2+vn6QoyQktnabDbWtDtRa/LEP4pSYUWTC/HHpMGqiX+InMraIjhw5gqamJsyYMSP8mCAI2LdvH37729/C4/GA47iI16hUKqhUPfz1IWQIWWx2NFjc8BsElMQ6iFVCwbKAJASWepCoZEtEt956Kz777LOIxx555BFMmDABTz/9dLckREiykYJXzVheG/ug0BIPWmvWI9kSkV6vx6RJkyIe0+l0yMzM7PY4IclIDCWiGNUZA08qcLHdCafghsnqRrYhvmGIkUb2q2aEpCrBF0hEClXPLaIOlw+tVgesLppdHUu/WkTnz5/H6NGjEx0L3n///YSfk5DBIgZ38FDEWPAKIHDVLDg25PNR9yyWfrWIysrKMH/+fPz5z3+G2+1OdEyEpATJH2oR9ZSIlGCDY9RUtzq2fiWio0ePYsqUKaioqEBubi6++c1v4uOPP050bIQkL0mCFNzBQ6mKPaERLAuGDXQ8/KFdYUk3/UpE06ZNw6ZNm3D58mVs2bIFDQ0N+OIXv4hJkyZh48aNaG6m2itkmBO8GGvWYWqBESW5PZeBZYJbColUtzqmAQ1WKxQK3HvvvXjllVfw85//HOfOncOqVatQWFiIpUuXoqGhIVFxEpJcfC4oORZatbrXCbmhFpFAY0QxDSgRHT58GE888QTy8vKwceNGrFq1CtXV1di9ezcuX76MxYsXJypOQpJLaHlHjK2mO2OCC18FqkkUU7+umm3cuBFbt25FVVUVFi1ahJdeegmLFi0CywbyWmlpKSorK1FSUpLIWAlJGpLPibpWB0StGgV+MWZhNAAozjaiWOMBU6AfwghTS78S0ebNm7F8+XIsW7YMeXnRSyBkZ2fjj3/844CCIyRZ+b1uNFjcsLn9GIWeF7MqFXxgE0YIQxNcCupXItq9ezeKiorCLaAQSZJQX1+PoqIi8DyP8vLyhARJSLLxe5wAAIFVgY9RFC2Mlnn0ql9jRGPGjEFLS0u3x9va2lBaWjrgoAhJdj5PYA4Ro1SD6WUxa7tbxPkWO05fbh2K0FJSvxJRrNq7drs97pIehKQyX7BFxCp7mMwYZPcBV6weNLbbBjuslNWnrllFRQUAgGEYrFmzBlrt1TU2giDg4MGDmDZtWkIDJCQZ+d2BRMT0tOA1KFQuVqB5RDH1KRF98sknAAItos8++ww8z4ef43keU6dOxapVqxIbISFJSAi1iHoqARIUmtAIoYcCaiNcnxLRe++9ByBQN2jTpk0wGAyDEhQhyU7wOgAArLqH5R1BHBf4gy35aR5RLP26arZ169ZEx0FISsnVisgsMMI/Jr/XY9ngJouiSC2iWOJORPfeey8qKythMBhw77339njszp07BxwYIclMKbih5BWAwdTrseFNFmmMKKa4E5HRaAxfpjQajYMWECFJT5KAYFE0xHHVLNQikkRKRLHEnYg6d8eoa0ZGNJ8LVywueAURGg+HrF6Wm6VpNJheZAKTRRuIxtKveUQulwtOpzN8v66uDi+88AL+9a9/JSwwQpKWz4lmuwd1FgEd7t6XbXAKJdQKDiqWNliMpV+JaPHixXjppZcAAB0dHZg1axY2bNiAxYsXY/PmzQkNkJCk43PCL4jwsRqolXHsNhMsA0JLPGLrd4XGm266CQDwt7/9Dbm5uairq8NLL72EX//61wkNkJCk43PBJ0jwceq4EpHEKlDX6sDpy23w+GnhazT9SkROpxN6faCkwb/+9S/ce++9YFkWX/jCF1BXV5fQAAlJNqLHAUGU4GfV0MSRiBhOiUarG5fbbPD4qXsWTb+L57/++uuor6/Hrl27cNtttwEAmpqaaJIjGfZ8LhskAD4uzq4ZpwTLMGBFPwSh55IhI1W/EtGaNWuwatUqlJSUYPbs2ZgzZw6AQOto+vTpCQ2QkGTjdVoBABKfBo6NYxvp4E4enOSHT6QWUTT9SkRf/epXceHCBRw+fBjvvPNO+PFbb70Vv/rVr+I+z+bNmzFlyhQYDAYYDAbMmTMHb7/9dn9CImTI+FyBVfSMKs6Ki1xgbzNGEuCnFlFU/d5yOjc3F7m5uRGPzZo1q0/nKCgowPr16zF27FhIkoQ//elPWLx4MT755BNce+21/Q2NkEFl5NyYWmCEa0JJfC9glWBZBpzfD0GgFlE0/UpEDocD69evx549e9DU1ASxS3Pz/PnzcZ3n7rvvjrj/3HPPYfPmzThw4AAlIpK0OJ8DWl4BbUZmnC9QBnd7leCjZR5R9SsRPfroo9i7dy8efvhh5OXl9VqhLh6CIOCVV16Bw+EIjzl15fF44PFc3aTOarUO+H0J6RNRBIIr78H3vvIeQHDb6eDLaQV+VP1KRG+//Tbeeust3HjjjQMO4LPPPsOcOXPgdruRlpaG1157DRMnTox67Lp167B27doBvych/eZzoNXuhtMrQuFikdf7bkIAy2FMjgEQRXDpfO/Hj0D9GqxOT09HRkbPu1vGa/z48Th27BgOHjyIb3/72ygvL8epU6eiHrt69WpYLJbwrb6+PiExEBI3jx2tdi/OW4EGa/ytG7VKBbWSgxI0RhRNvxLRT3/6U6xZsyZivVl/8TyPsrIyXHfddVi3bh2mTp2KTZs2RT1WpVKFr7CFboQMKY8NPkGEj9MiTdWHDgVLO3n0pF9dsw0bNqC6uho5OTkoKSmBUqmMeP7o0aP9DkgUxYhxIEKSitsCryDCo9RDy8cxmTGoxSXCaXGCa7VilD5nEANMTf1KREuWLEnIm69evRp33HEHioqKYLPZsH37drz//vvYtWtXQs5PSKJJ7g74/CLcGkOfWkStLhFtHS7wHTaMGsT4UlW/EtGPf/zjhLx5U1MTli5dioaGBhiNRkyZMgW7du3Cl770pYScn5BE8zvaIUiAh0uDlo//14cJVmkU/VQuNpp+T2js6OjA3/72N1RXV+N73/seMjIycPToUeTk5GDUqPhyPm1JTVKN194OABBVxh73u+8qtJOHSGNEUfUrEX366adYsGABjEYjamtr8dhjjyEjIwM7d+7EhQsXwrWKCBlWJAk+RyARcVpTn17KBnfyoHlE0fXrqllFRQWWLVuGs2fPRuzsumjRIuzbty9hwRGSVHwuGJQiphWacOv0sX166dWuGbWIoulXi+jQoUP4/e9/3+3xUaNGobGxccBBEZKU3BawDAONzgCNKc4Fr0GhnTyoaxZdv1pEKpUq6vKKM2fOwGw2DzgoQpKSuyPwr8bU55eGxohok8Xo+pWI7rnnHjz77LPw+QLZnWEYXLhwAU8//TS+8pWvJDRAQpKGsxWXLS6ctijR7uhbQslNT8OUAiOuK+pbS2qk6Fci2rBhA+x2O8xmM1wuF+bOnYuysjLo9Xo899xziY6RkOTgaEGzzYPjbRys7r51sXheBR2vgI6jekTR9GuMyGg0Yvfu3di/fz+OHz8Ou92OGTNmYMGCBYmOj5CkITlb4fEJcClMMKiVvb+gM1ri0aM+JyJRFFFZWYmdO3eitrYWDMOgtLQUubm5kCQpISVBCEk6ogifrRmCBDiV6dCr+/ar4xRYtLU74WE7MGbCIMWYwvrUNZMkCffccw8effRRXLp0CZMnT8a1116Luro6LFu2DF/+8pcHK05C5OWxwOv1QWQ4KHXpUHB9G9Vwiwzq212oaeoYnPhSXJ/SemVlJfbt24c9e/Zg/vz5Ec+9++67WLJkCV566SUsXbo0oUESIjtHKzx+AW6FEQZt32sKKRSB10gCLfGIpk9p/S9/+Qt+8IMfdEtCAHDLLbfgmWeewbZt2xIWHCFJw9kKl1eAS2mCUdP3RMQqg6+hMaKo+pSIPv30U9x+++0xn7/jjjtw/PjxAQdFSNJxtsDtE+FSpiMzrR8touCERkb0QRTpyllXfeqatbW1IScndi2VnJwctLe3DzgoQpKOsxWjzTqYR08El9f3gnxcsEXESn74RQl8PPuhjSB9SkSCIEChiP0SjuPgpzIHZLiRJMDRApZhYMzKA/pSmTFIESweGEhEIvj+TeEbtvr0PypJEpYtWwaVKnrFcKqsSIYlrwPwewCGATT9q9XOcDxYBmBFAT7aZLGbPiWi8vLyXo+hK2Zk2HG1web24aJbBaHRgUmjjH0/B6vAxDwDoOCh6UOJ2ZGiT4lo69atgxUHIcnL2Qqbx49apwFCaz8TEaeEXq0MtKpofKgb6qgS0htnW/jSfYaun/uShZZ4SGLgRiJQIiKkN842uHyByYyZunh2VIyCU6LF7sGlDhc67I7ExjcMUCIipBdSp8mM6bo+LnYNYVhctnpwoc2JNtvA9wMcbigREdITUYDP0Q6/KMGjNCK9H8s7AAAMA4YNDMlS3eruKBER0hNXO1weHwRGCbXOBGUfF7t2Fqpb7fdRIupK1kS0bt06XH/99dDr9cjOzsaSJUtQVVUlZ0iERHK2wesX4VEakanv5/hQCBXQj0nWRLR3716sXLkSBw4cwO7du+Hz+XDbbbfB4aDBPJIknK0w61WYN30CFlwzsK2iQy0igRJRN/3eYDER3nnnnYj7lZWVyM7OxpEjR3DzzTfLFBUhnbjaAACcLhO6fizt6OzqlkK0AqErWRNRVxaLBQCQkRF9Gr3H44lYRhJtJxFCEsrZGvhXmzngU4V3e6X1mN0kzWC1KIp46qmncOONN2LSpElRj1m3bh2MRmP4VlhYOMRRkpHGZ2/FicsW7L0oDLh8R2GmERPzDBhrVvd+8AiTNIlo5cqVOHHiBHbs2BHzmNWrV8NisYRv9fX1QxghGXG8TridNtjcfpyz8WAHuDQjTauGUaNEmoIWvXaVFF2zJ598Em+++Sb27duHgoKCmMepVKqYK/8JSThXYGmHl9PBpNcN/HyhZR4idc26krVFJEkSnnzySbz22mt49913UVpaKmc4hERytsLlG+Aas06sXgmNVjcutdHYZleytohWrlyJ7du34+9//zv0ej0aGxsBBPZN02g0coZGyNXFrgoT8hKQiNrcEhpaHBC1HRiVgPCGE1lbRJs3b4bFYsG8efOQl5cXvv31r3+VMyxCAoJrzNxKY0JaRKHL9xLNI+pG1haRJNGgHUlegqMVHr8IlyIxXTMuePleop08ukmaq2aEJBVRhM/eBl7BAtoMaBNQVZEL721GiairpLhqRkjScXdAzQEzSs2YcsPkhGylzgYTkUibLHZDLSJConEGlnZAkw6FIjE1pqlFFBslIkKiCa4xS8TSjhA2OEZEu712R10zQqJxtuJUgxVNHgGTij3IShv4RNo0rRrjc/VgDAmYHDnMUIuIkChERwtsbh/q3ZoBFUPrjOdVyNDySFfRLh5dUSIiJAq3pQWiBPhVJhjUCeo4cMEpAAJVaOyKumaEdOVzw+0ILMNQG7MTcsUMAMCp0GL3wOewIN0nQK2kjRZDKBER0pUrsH2Qj9Mg3ZCWuPMqeNS0OOAXHeDdPkpEnVDXjJCuQtsHKUz937UjGk4FjmUASPD6qEpjZ5SICOnK2Qan1w+30ojMBFwtC+OU4IID3z6PO3HnHQYoERHSheRsBa9gIWkzkJWWwBYRw4QHrP1eSkSd0RgRIV0wrjZMyDVgwuQpQCK7ZgAYpQqAEz5KRBGoRURIZ6IIONsDX2ujb+IwEAwX6OoJXhoj6owSESGduTsgCj6AVQBqU8JPzyoDiYi6ZpGoa0ZIZ85WnG2yo1kyIrfFibLsBF6+B1CQlQ6BaYPSRJfuO6NEREhnjhY4vX60K40oViS+w2DU6wAXDyipKGBn1DUjpBOfrQkenwiX0oTMRF4xCwmOEYF2e41AiYiQTmxtjZAAsGlZ0PKJ7zA4BA7Ndg8a2ywJP3cqo64ZISGSBGdHEwBAn5k3KG/R4gYuN9khcm3IHZR3SE3UIiIkxG2B0+mCxLDIzMwelLfg+EDXTKIlHhEoERESYr8Cu8cPlyIdOabBKV6mUAb2vZdojCgCdc0ICRKsDcjU8bDxecgxqAflPRR84LwiJaIIsraI9u3bh7vvvhv5+flgGAavv/66nOGQEY5zNKE4U4e5MyYHthEaBEqeWkTRyJqIHA4Hpk6dihdffFHOMAgBJAmwNQS+1g/eMLJKHdxK3e+GINJcohBZu2Z33HEH7rjjDjlDIAQA4He0wdreAaNODS5tcAaqAYBXB8aeFKIXHr8wKFMEUlFK/S94PB54PFebtFarVcZoyHDScOEMLl6xQTQacWNwj/rBwPIalGWngWMZKOlSUVhK/VesW7cORqMxfCssLJQ7JDJMNNRWAQB02aMH940UGpjTArt5KEUaJwpJqUS0evVqWCyW8K2+vl7ukMgw0GZ3w91UDQAoGXPN4L4ZywKK4DIPH63AD0mprplKpYJKlcDSnWTEkyQJHx87DqPggtGghylvkFtEACx+Bbx2K5RWC0y6xO0km8pSqkVESKJ9XNMG78XjYBmgeNxUgB388hx1VhHnmuxoaGkf9PdKFbK2iOx2O86dOxe+X1NTg2PHjiEjIwNFRUUyRkZGgnNNdhyuqsN05zmUZOqgK7l+SN6X5bUAAJ/HOSTvlwpkTUSHDx/G/Pnzw/crKioAAOXl5aisrJQpKjJSFJsUmOX5CNlGHjlF4wDjqCF5X5YPzCXyux1D8n6pQNZENG/ePEgSTeoiQ0wUgIbjUNbtx3UGGxjOCIy9bcjeXqEKzCUSvK4he89kl1KD1YQM1NnTn4I79y+U6rxgwIDVmIDxdwBp5iGLQRHsmgle6pqFUCIiI4MoovbjN9F6ch8kAIaCHGRdewuQNxXghvbXQKkJJCK/h7pmIZSIyLDn83rw+bvb4Lh0CgCgLpmFzJvvAZSDs8K+NypNoCC/4KYWUQglIjKstbR34Mz/bQVjvQSJYaGdshhTr7sBDMPIFpPRmI4xZh0UabSTRwglIjJsVZ2vReOH28B72sEq1Si8+WHkl4yXOyyotAZk69WAwit3KEmDEhEZfgQ/cOEjZFftRbunA1qDCWULVkCbPjh1qPtMFdwrze8N3BSDsFtIiqFERIaXjnrgzDuAowXpag7jr50G07S7waiNckd2lUIFi4+B1+2G2tYOfXqO3BHJjhIRGR78XuD8e2g78xHS1ErwGj0w9ktIN08AZBwPiqW6A/DY7MhtaaVEBEpEZDiwNwMnXoW9oxlnm+xoN07EDbO/CoPeIHdkMTFqPWBrgcdpkzuUpECJiKS29lrgxE4IXjdOtkk4aV6E7OIJ0Kfp5Y6sR0qNHm4AHgdttAhQIiKprL0O+PQVQPTjjMeEgxlfhFaXhgXX5Mh6eT4evC4wZuVxUiICKBGRVGVrBE78DRD9aFYXYpdvBiRGgdsm5kKtTP75ORq9CQDgs1MpEIDqEZFU5GwDPv0r4PfCpy/AG95ZkBgFphWaUJSplTu6uGgNgQL9orNV5kiSAyUiklo8NuD4DsDrBPQ5+FA7D1YvYNIqcWNZltzRxU2XHlhky7o74PMLMkcjP+qakdThcwdaQm4LoM0ApjyAOawGbqkJ0wtNg7Yp4mBQpaVjdLYeao4B67UBCpPcIcmKEhFJDYIfOPEqYG+GxOsgTrofHK8DD2DhtYO3IeJgYVgOOdm5gW6mpwPQmuQOSVap8yeEjFyiAJx6Hei4AJfI4S3chL31w2CdliYj8C+NE1EiIklO8AMnX4Pj8uc42+LCX92zcNapw6nLVlicPrmjGxCXKgNNNjfq6s7LHYrsqGtGkpbLYUPLwb/C2nAOHR4JVVkLYeHzUZqlw9xxZhi1g7cj61Do4DJR2+yA5DyP4hvkjkZelIiIbCRJQqvDi0aLGza3Hx6/gNIsHYozdUB7HSwf78TFi5cgMEqczV6InKIJWFhkwiiTRu7QEyI9txi1AOBogdPpgFarkzki+VAiIkNOkiRUXbHhUG07WmxXt13mRC+MDieKL9UBreeQpRBxWZ8BdvJX8NXS0dCrU7sF1JVaZ4JClw6/ox2NdVUYfc0MuUOSDSWiYU4QJVhdPqTrkqPmjcsr4O3PLqHj8jnoPVcwTrAhh3fBCDtUohtGQQlolADDQFFwHWbMnX91i+bhhmGgyR0PW/UBtF84BVAikteLL76IX/7yl2hsbMTUqVPxm9/8BrNmzZI7rOTlaAmss3K1AwwAbSaQXgpoTAAAUZRQ0+rApxc7UN/mAscyWDm/LPzyc012sAxQkK4durk3kgRY6qG68jlKzxyA12XDKJMGuelqKFgWgesmWkCTDmSWAaNmBOYKDXPZoyfDVn0A3ssn4XY5oNaMzO6Z7Inor3/9KyoqKvC73/0Os2fPxgsvvICFCxeiqqoK2dnZcoeXPPweoOkU0PApYL0c9RCPNhfnUIjDzmy0+a8Whtd0WXv18dnLsHW0QCM4kK/xIVOvhk6bBl6fDpUxG3kZV4uIdTi9UCu5/q3fEvyQrBfRUnsSGc7z4Lx2sADGZ7Bg+FxocscD2qxAAtWkA2qTbAXt5WIuGIuaNDNEezPOfPwvTJn7ZblDkgUjybzD4ezZs3H99dfjt7/9LQBAFEUUFhbiP/7jP/DMM8/0+Fqr1Qqj0QiLxQKDIXlrz/SL4Ac8VsB6CWg7DzSfAUR/4DmGBdJLIOnMYCACtkZcrjuLC60OBL6ZDLyaLJhzRqHQbEKaEmB8LsBjgeiyoO5KGzqcPrj9Yre31asVmDS6CNCZAV0WXj3tRoeoBsdrYdCqkK5mkaFhkM4LSFcK0LNewOcEfK7gv074PQ64nHbY7A602r2we/woMGlQmG0CssYD2dcA6SVDss98Kqg/cwwXP/hz4P9oxm2B1iA/PFpG8f6Oytoi8nq9OHLkCFavXh1+jGVZLFiwAB999FG34z0eDzyeq4ObVqs1vjdqqwGq9wS6B7FEPCehyeZGbYsDDCREvCp4pyw7DRnBy8ctDg/ON9kjj+z05RizFllpgXGONocXZ690LoYldfpHQkmWDjl6HvC5YXH5cK7JFj6VW2lCS9oEtKSNg7ddg5uzzJg0KtB6EfOaUfPhBxiDepQq25GpE8Ay9UBrfcTHZAGUZqUBSg2cbBpafCrYPH54nHYw7nZoWE9gCYXbArSew+jWdrh9kQnLGbxZVApMHnW19XS4rg2CKEHs9Nl9rAa2tDFQFE1G4YyZQ76HWCooHDcNOlcDMtqOAbX/Bmr/jZPNXtj8HMCwYFg2WGUyUNpEpeAwrfDq//tnl6xwev1Rz63kWMwoMoXvn2qwwuaOfizHMJhZkh6+f7rRBovLB42Sw5SCKKV2J98PqBPTAJD1p6KlpQWCICAnJ7JUZk5ODk6fPt3t+HXr1mHt2rV9fyO/J1DFrw8khxuCo4cN8Fw+gA0Ooro8EFz22OfyiIAyOPnO44XoiX0svAwQ/MUXWQXaODOs6jy0akbDwWcFfiBFAKIIu+fqD9SonCzcvvCuwKC02wrYGgLLB0QfwHAArwVUBkBtDPyr4KEFUNTt/Z2AoxlwtgCOFkw3NUPwOOB2OeDyeOHwMXD4AKugAJNmAHJzAaUGUOpQbWuBS1LBx6mhVOuQbjKh0JyO6/OM0PDU+ulJxpTbgaYioP4gYLsCt9MBSQj8HHT98ykpWMB+dWa5ZO+A6Im+cFbiGMB+deKnZLNAjJGIGJYB7P5Ox1ohunyQeC7i/a4e0L1F3V8p9edp9erVqKioCN+3Wq0oLCzs/YWmQmDq1wJfRxTM6lI8q9NzOo+AIpcv+HDXIlsMNDoloFQADAOdz48Slz/8XPhoJvC1TqMAlIHWk9YvoKzrD0Kn86epeIDnAKUWWobHNLfQLWwmGFOaStHpFMzVK2NqQ///UvFagC8G0ovDD3EAdMFbT+vbF5q9YFkGPMemRE2gpMIwQM61gZvPjeKJLRB8HoiCAEmSIIlXf+kVHAPor15JzBvtgVeI3tpnWQCGq+NuOWM8MPmjH8swAIxXjzWP8cDgl7q9X1gCu4+yJqKsrCxwHIcrV65EPH7lyhXk5nZfyKhSqaBS9eNSLq8DMkr79JK04C0emuAtHurgLR4qAGZl6vytMGmTY4pAylOqkZVTEPfh5j5cXMwapGMHSta1ZjzP47rrrsOePXvCj4miiD179mDOnDkyRkYIGUqy/7mtqKhAeXk5Zs6ciVmzZuGFF16Aw+HAI488IndohJAhInsieuCBB9Dc3Iw1a9agsbER06ZNwzvvvNNtAJsQMnzJPo9oIIb1PCJChoF4f0epHhEhRHaUiAghsqNERAiRneyD1QMRGt6Ke6kHIWRIhX43exuKTulEZLMF1mzFNbuaECIbm80GozHKerWglL5qJooiLl++DL1en/R7nYeElqXU19cPqyt9w/FzDcfPBAzt55IkCTabDfn5+WDZ2CNBKd0iYlkWBQXxT4VPJgaDYVj9cIcMx881HD8TMHSfq6eWUAgNVhNCZEeJiBAiO0pEQ0ylUuHHP/5x/6oIJLHh+LmG42cCkvNzpfRgNSFkeKAWESFEdpSICCGyo0RECJEdJSJCiOwoEQ2y5557DjfccAO0Wi1MJlNcr5EkCWvWrEFeXh40Gg0WLFiAs2fPDm6gfdTW1oaHHnoIBoMBJpMJK1asgN3ew+4kAObNmweGYSJu3/rWt4Yo4uhefPFFlJSUQK1WY/bs2fj44497PP6VV17BhAkToFarMXnyZPzzn/8cokj7pi+fq7Kystv3Ra0e4o0uJTKo1qxZI23cuFGqqKiQjEZjXK9Zv369ZDQapddff106fvy4dM8990ilpaWSy+Ua3GD74Pbbb5emTp0qHThwQPrggw+ksrIy6cEHH+zxNXPnzpUee+wxqaGhIXyzWCxDFHF3O3bskHiel7Zs2SKdPHlSeuyxxySTySRduXIl6vH79++XOI6TfvGLX0inTp2S/t//+3+SUqmUPvvssyGOvGd9/Vxbt26VDAZDxPelsbFxSGOmRDREtm7dGlciEkVRys3NlX75y1+GH+vo6JBUKpX0l7/8ZRAjjN+pU6ckANKhQ4fCj7399tsSwzDSpUuXYr5u7ty50ne+850hiDA+s2bNklauXBm+LwiClJ+fL61bty7q8ffff7905513Rjw2e/Zs6Zvf/OagxtlXff1c8f5sDibqmiWZmpoaNDY2YsGCBeHHjEYjZs+eHXX3Wzl89NFHMJlMmDlzZvixBQsWgGVZHDx4sMfXbtu2DVlZWZg0aRJWr14Np9M52OFGFdpluPP/c0+7DAOBz935eABYuHBh0nxfgP59LgCw2+0oLi5GYWEhFi9ejJMnTw5FuGEpveh1OGpsbASAqLvfhp6TW2NjI7KzsyMeUygUyMjI6DHGr3/96yguLkZ+fj4+/fRTPP3006iqqsLOnTsHO+Ru+rrLMBD43Mn8fQH697nGjx+PLVu2YMqUKbBYLHj++edxww034OTJk0O2qJxaRP3wzDPPdBvc63qL9U1PZoP9uR5//HEsXLgQkydPxkMPPYSXXnoJr732GqqrqxP4KUhfzZkzB0uXLsW0adMwd+5c7Ny5E2azGb///e+HLAZqEfXDf/3Xf2HZsmU9HjN69Oh+nTu0w+2VK1eQl5cXfvzKlSuYNm1av84Zr3g/V25uLpqamiIe9/v9aGtri7pDbyyzZ88GAJw7dw5jxozpc7wD0dddhoHA96Yvx8uhP5+rK6VSienTp+PcuXODEWJUlIj6wWw2w2w2D8q5S0tLkZubiz179oQTj9VqxcGDB/Htb397UN4zJN7PNWfOHHR0dODIkSO47rrrAADvvvsuRFEMJ5d4HDt2DAAiEu5Q6bzL8JIlSwBc3WX4ySefjPqaOXPmYM+ePXjqqafCj+3evTupdiXuz+fqShAEfPbZZ1i0aNEgRtqFrEPlI0BdXZ30ySefSGvXrpXS0tKkTz75RPrkk08km80WPmb8+PHSzp07w/fXr18vmUwm6e9//7v06aefSosXL07Ky/fTp0+XDh48KP373/+Wxo4dG3H5/uLFi9L48eOlgwcPSpIkSefOnZOeffZZ6fDhw1JNTY3097//XRo9erR08803y/URpB07dkgqlUqqrKyUTp06JT3++OOSyWQKX7p++OGHpWeeeSZ8/P79+yWFQiE9//zz0ueffy79+Mc/TtrL9335XGvXrpV27dolVVdXS0eOHJG+9rWvSWq1Wjp58uSQxUyJaJCVl5dLALrd3nvvvfAxAKStW7eG74uiKP3oRz+ScnJyJJVKJd16661SVVXV0Affg9bWVunBBx+U0tLSJIPBID3yyCMRybWmpibic164cEG6+eabpYyMDEmlUkllZWXS9773PVnnEUmSJP3mN7+RioqKJJ7npVmzZkkHDhwIPzd37lypvLw84viXX35ZGjdunMTzvHTttddKb7311hBHHJ++fK6nnnoqfGxOTo60aNEi6ejRo0MaL5UBIYTIjq6aEUJkR4mIECI7SkSEENlRIiKEyI4SESFEdpSICCGyo0RECJEdJSKSUiorKyMqXf7kJz+JWIO3bNmy8NIGkjooEZGoli1bFrOU68qVK8EwTMQC2cFIACUlJXjhhRciHnvggQdw5syZmK/ZtGkTKisrw/fnzZsXsTaMJCdKRCSmwsJC7NixAy6XK/yY2+3G9u3bUVRUJEtMGo2mWy2kzoxGY9y1wUnyoEREYpoxYwYKCwsjCpft3LkTRUVFmD59+oDOHa2lsmTJknAra968eairq8N3v/vdcC0koHvXrKvOLbNly5Zh79692LRpU/gcNTU1KCsrw/PPPx/xumPHjoFhmCEtfUGuokREerR8+XJs3bo1fH/Lli145JFHBv19d+7ciYKCAjz77LNoaGhAQ0NDn8+xadMmzJkzB4899lj4HEVFRd0+EwBs3boVN998M8rKyhL1EUgfUCIiPfrGN76Bf//736irq0NdXR3279+Pb3zjG4P+vhkZGeA4Dnq9Hrm5uf0qPmY0GsHzPLRabfgcHMdh2bJlqKqqCm+x4/P5sH37dixfvjzRH4PEiQqjkR6ZzWbceeedqKyshCRJuPPOO5GVlSV3WAOSn5+PO++8E1u2bMGsWbPwj3/8Ax6PB/fdd5/coY1Y1CIivVq+fDkqKyvxpz/9KWGtBpZl0bUCjc/nS8i54/Hoo4+GB+K3bt2KBx54AFqtdsjen0SiRER6dfvtt8Pr9cLn82HhwoUJOafZbI4Y9xEEASdOnIg4hud5CIIwoPeJdY5FixZBp9Nh8+bNeOedd6hbJjPqmpFecRyHzz//PPx1LBaLJVyHOiQzMxOFhYXdjr3llltQUVGBt956C2PGjMHGjRvR0dERcUxJSQn27duHr33ta1CpVP3qEpaUlODgwYOora1FWloaMjIywLJseKxo9erVGDt2bFLVnR6JqEVE4mIwGGAwGHo85v3338f06dMjbmvXro167PLly1FeXo6lS5di7ty5GD16NObPnx9xzLPPPova2lqMGTOm35sVrFq1ChzHYeLEiTCbzbhw4UL4uRUrVsDr9Q7JVUDSMyoVS0asDz74ALfeeivq6+u7bUhIhhYlIjLieDweNDc3o7y8HLm5udi2bZvcIY141DUjI85f/vIXFBcXo6OjA7/4xS/kDoeAWkSEkCRALSJCiOwoERFCZEeJiBAiO0pEhBDZUSIihMiOEhEhRHaUiAghsqNERAiRHSUiQojs/j+/g4OfF949GAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:00.446197Z", + "iopub.status.busy": "2024-03-22T19:36:00.445889Z", + "iopub.status.idle": "2024-03-22T19:36:00.663629Z", + "shell.execute_reply": "2024-03-22T19:36:00.662779Z" + }, + "papermill": { + "duration": 0.239802, + "end_time": "2024-03-22T19:36:00.665617", + "exception": false, + "start_time": "2024-03-22T19:36:00.425815", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEqCAYAAABqVvf5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5DElEQVR4nO3deVgT194H8C9hR0BEAgiC0eJ6XVBACq1bQUCKBcXlqq1Lqd4udBGshb63Wm+fFhew1urFt4taW5dWRbRIVVRQqoiItlcsoiJURRARIWyGkJz3D9/MJRIgVCDL/D7P4yOZOZOcMMxvzsw58zsGjDEGQgjRcwJNV4AQQroDBTtCCC9QsCOE8AIFO0IIL1CwI4TwAgU7QggvULAjhPACBTtCCC9QsCOE8AIFO0IIL+hcsNu8eTNEIhHMzMzg7e2N8+fPt1r266+/xrhx49CrVy/06tUL/v7+bZYnhOgvnQp2P/74I6KiorBy5UpcvHgRo0aNQmBgIMrLy1WWz8jIwJw5c5Ceno6srCy4uLggICAAJSUl3VxzQoimGehSIgBvb294eXlh06ZNAAC5XA4XFxe8/fbbiImJaXd7mUyGXr16YdOmTZg/f75anymXy3H37l1YWVnBwMDgqepPCOlcjDHU1NTAyckJAkHbbTejbqrTU2tsbERubi5iY2O5ZQKBAP7+/sjKylLrPerr6yGVSmFra9tqGYlEAolEwr0uKSnBsGHD/nrFCSFd7vbt2+jbt2+bZXQm2FVUVEAmk8HBwUFpuYODA65evarWe3zwwQdwcnKCv79/q2Xi4uKwatWqFsu/+eYbWFhYdKzShJAuVV9fj9deew1WVlbtltWZYPe0Vq9ejT179iAjIwNmZmatlouNjUVUVBT3WiwWw8XFBWFhYbC2tu6OqmqEVCpFWloaJk+eDGNjY01XhzwlvuxPsViM1157Ta1bTDoT7Ozs7GBoaIh79+4pLb937x4cHR3b3DY+Ph6rV6/G8ePHMXLkyDbLmpqawtTUtMVyY2Njvf6jUeDL9+QLfd+fHfluOtMba2JiAg8PD5w4cYJbJpfLceLECfj4+LS63dq1a/HJJ5/gyJEj8PT07I6qEkK0kM607AAgKioKCxYsgKenJ8aOHYsNGzagrq4OixYtAgDMnz8fzs7OiIuLAwCsWbMGK1aswK5duyASiVBWVgYAsLS0hKWlpca+ByGk++lUsJs9ezbu37+PFStWoKysDO7u7jhy5AjXaXHr1i2l7ufExEQ0NjZixowZSu+zcuVKfPzxx91ZdUKIhulUsAOAyMhIREZGqlyXkZGh9Lq4uLjrK0QI0Qk6c8+OEEKehs617AghLdXX1yuNN61tkODs5UL0srsAS3Pl0QVDhgzh5ZhRCnY8pe7BwdcDQ9dcvXoVHh4eLZavVVE2NzcXY8aM6fpKaRkKdjyl7sHB1wND1wwZMgS5ubnc64LSKkTtvYz1M0dgcB+bFmX5iIIdT6l7cPD1wNA1FhYWSiclwZ8PYJrZgKHDR8G9X28N1kx7ULDjKTo4CN9QsOOJooo61EmaWl1feL+O+9/IqPU/ix6mRuhv16PT60c6rq19SvuzJZ3KZ6cJYrEYPXv2RHV1tc4mAiiqqMOk+IxOe7/0ZRN5c4Boq87cp7q8PztyfFLLjgcUZ/8Ns93hZq/6Mbm6BglSMrIQMtEHPcxbJkIAgBvltXjvx9/abCGS7tHePqX92RIFOx5xs7fEcOeeKtdJpVKUCYEx/XrpdZYMfdPaPqX92RI9QUEI4QVq2fGARPYIArMSFIkLIDBTfRnb1NSEu013kV+Z3+oN7SJxLQRmJZDIHgFQ3UIkRFtRsOOBu3V/okf/L/GhGrNI/vvIv9tc36M/cLfOHR5waLMc6VrtncDo5NUSBTsecOrRD3VFb+OL2e54ppUOiqamJpz59Qyee/65Vg+OwvJavPvjb3Ca1K8rq0vUoO4JjE5e/0XBjgdMDc0gf+SM/taDMax36x0URUZFGGo7tNUb2vJH1ZA/ug9Tw9bn8CDdo70TGJ28WqJgR4gOau8ERievlqg3lhDCCxTsCCG8QMGOEMILFOwIIbxAwY4QwgsU7AghvEDBjhDCCxTsCCG8QMGOEMILFOwIIbxAwY4QwgsU7AghvEDBjhDCCxTsCCG8QMGOEMILlM+OBxqkMgBAXkl1q2XqGiS4cB9w/PNhm1PvEaKrdC7Ybd68GevWrUNZWRlGjRqFL7/8EmPHjm21/N69e/HRRx+huLgYAwcOxJo1axAcHNyNNda8wv8PUjFJl9spaYTvb+S0+349THXuz0bvtHcCo5NXSzr1V/vjjz8iKioKW7Zsgbe3NzZs2IDAwEAUFBTA3t6+RfmzZ89izpw5iIuLQ0hICHbt2oWwsDBcvHgRw4cP18A30IyAvzkCAJ6xt4S5saHKMgWl1YjedxkJM0ZgcJ/WJ1/pYWqks7PH6xP1TmB08mrOgDHGNF0JdXl7e8PLywubNm0CAMjlcri4uODtt99GTExMi/KzZ89GXV0dUlJSuGXPPvss3N3dsWXLFpWfIZFIIJFIuNdisRguLi6oqKiAtbV1J38j7fH7rUrM+PoC9i32xChXW01Xh7Sjsq4Rx/PLMUDYQ+UJ7FpZNZYfyMfaaUMxyLGtk5chRL119+QlFothZ2eH6urqdo9PnQnpjY2NyM3NRWxsLLdMIBDA398fWVlZKrfJyspCVFSU0rLAwEAkJye3+jlxcXFYtWpVi+XHjh2DhYXFX6u8DrhdCwBGOHfuHEryNF0bog5LAOXlqteV///+LL9xGaZlbb/PH51cr+5UX1+vdlmdCXYVFRWQyWRwcFCe8s3BwQFXr15VuU1ZWZnK8mVlre/92NhYpQCpaNkFBATofcsOly/g2WefpZadHuDL/hSLxWqX1Zlg111MTU1hatryhq6xsXGrszTpA8V0e0ZGRnr9PfmCL/uzI99NZ8bZ2dnZwdDQEPfu3VNafu/ePTg6OqrcxtHRsUPlCSH6S2eCnYmJCTw8PHDixAlumVwux4kTJ+Dj46NyGx8fH6XyAJCWltZqeUKI/tKpy9ioqCgsWLAAnp6eGDt2LDZs2IC6ujosWrQIADB//nw4OzsjLi4OAPDuu+9iwoQJSEhIwIsvvog9e/bgwoUL+OqrrzT5NQghGqBTwW727Nm4f/8+VqxYgbKyMri7u+PIkSNcJ8StW7cgEPy3serr64tdu3bhn//8Jz788EMMHDgQycnJvBpjRwh5TKeCHQBERkYiMjJS5bqMjIwWy2bOnImZM2d2ca0IIdpOZ+7ZEULI06BgRwjhBQp2hBBeoGBHCOEFCnaEEF6gYEcI4QUKdoQQXqBgRwjhBQp2hBBeoGBHCOEFCnaEEF6gYEcI4QUKdoQQXqBgRwjhBQp2hBBeoGBHCOEFCnaEEF6gYEcI4QUKdoQQXqBgRwjhBQp2hBBeoGBHCOEFCnYEMpkMF7J+Rd0fp3Ah61fIZDJNV4mQTkfBjueSkpIgEomweE4YKn5eh8VzwiASiZCUlKTpqhHSqSjY8VhSUhLCw8Nx584dpeV37txBeHg4BTwdVVtbi6WLX8Hdb9/C0sWvoLa2VtNV0goGjDGm6UpoM7FYjJ49e6K6uhrW1taark6nkclkMDExgVwub7WMQCBAY2MjDA0Nu7Fm5GmMHTsWOTk5LZZ7eXnh/PnzGqhR1+rI8UktO57at29fm4EOAORyOfbt29dNNSJPq7VABwA5OTkYO3ZsN9dIu1DLrh362rITCoWoqKhot5ydnR3u37/fDTUiT6O8vBwODg7tljt16hQ8PT1hYWHRDbXqetSyI+2qqqrq1HJEs2bOnKlWuQkTJuDq1atdXBvtRMGOp9S9D0f363RDcXGxWuX69OmDIUOGdG1ltBQFO54yMzPr1HJEs+rr69UqJ5VK9eYStqMo2PGUugOHaYCxbnBycurUcvpIZ4JdZWUl5s2bB2tra9jY2CAiIqLN8UOVlZV4++23MXjwYJibm8PV1RXvvPMOqquru7HW2ksikXRqOaJZjY2NnVpOH+lMsJs3bx6uXLmCtLQ0pKSk4PTp01iyZEmr5e/evYu7d+8iPj4eeXl52L59O44cOYKIiIhurLX2EgjU2/XqliOaJRaLO7WcPurw0JObN29iwIABXVUflfLz8zFs2DDk5OTA09MTAHDkyBEEBwfjzp07ajfN9+7di5dffhl1dXUwMjJSWUYikSi1ZsRiMVxcXFBRUaFXQ0/69u2L8vLydsvZ29u3eMKCaB97e3u1es5tbGzU2u+6QiwWw87OTq2hJ6qP+Da4ublhwoQJiIiIwIwZM7rlBnZWVhZsbGy4QAcA/v7+EAgEyM7OxrRp09R6H8UvpLVABwBxcXFYtWpVi+XHjh3Tqxu7w4YNU+uPftiwYUhNTe2GGpGnoe4jYbW1tXq1P9XtmAH+QrC7ePEitm3bhqioKERGRmL27NmIiIjo0tHZZWVlsLe3V1pmZGQEW1tblJWVqfUeFRUV+OSTT9q89AWA2NhYREVFca8VLbuAgAC9atnt2rVLrXL29vYIDg7u4tqQp9XU1KT02traGlKpFMbGxkqXrk1NTXq1PztyWd7hYOfu7o4vvvgCCQkJOHToELZv347nn38egwYNwquvvopXXnkFQqFQrfeKiYnBmjVr2iyTn5/f0Sq2IBaL8eKLL2LYsGH4+OOP2yxramoKU1PTFsuNjY1hbGz81HXRFiUlJWqX06fvzReKINDQ0NBinT7tz458l79899nIyAjTp0/H3r17sWbNGty4cQPLli2Di4sL5s+fj9LS0nbfIzo6Gvn5+W3+GzBgABwdHVtccjU1NaGyshKOjo5tfkZNTQ2CgoJgZWWFAwcO6NWOfhqurq6dWo4Qbdfhlp3ChQsXsHXrVuzZswc9evTAsmXLEBERgTt37mDVqlUIDQ1tN8uCUChUqxXo4+ODqqoq5ObmwsPDAwBw8uRJyOVyeHt7t7qdWCxGYGAgTE1NcejQIRog28zQoUM7tRzRrJCQEKSkpKhVjrdYByUkJLDhw4czY2NjFhoayn7++Wcmk8mUyty+fZsZGhp29K3bFBQUxEaPHs2ys7PZr7/+ygYOHMjmzJnDrb9z5w4bPHgwy87OZowxVl1dzby9vdmIESPYjRs3WGlpKfevqalJ7c+trq5mAFh1dXWnfh9NGz16NAPQ7r/Ro0druqpEDSEhIWrtz5CQEE1XtVN15PjscMsuMTERr776KhYuXIg+ffqoLGNvb49vv/22o2/dpp07dyIyMhJ+fn4QCAQIDw/Hxo0bufVSqRQFBQVc78zFixeRnZ0N4HEPcnNFRUUQiUSdWj9do85tho6UI5r16NGjTi2njzoc7NLS0uDq6tpisCljDLdv34arqytMTEywYMGCTqskANja2rbZgygSicCaDRmcOHGi0muirEePHtzPffv2VRpL5+Ligtu3b7coR7SXubl5p5bTRx0Ods888wxKS0tbDAWprKxE//796VlKHWFpacn9PGzYMHh6eqKwsBDPPPMM6uvruWDXvBzRXhTs2tfhYNdaa6m2tpY6AHRI84HVx44d436+fPlyq+WI9lI1xORpyukjtf+SFQNtDQwMsGLFCqWnCWQyGbKzs+Hu7t7pFSRdY+zYscjNzVWrHNF+6maT5nPWabWD3aVLlwA8btldvnwZJiYm3DoTExOMGjUKy5Yt6/waki4RFxeHxMREtcoR7dc8iBkYGChdgTV/TcFODenp6QCARYsW4YsvvtCrR6f46JtvvlG7XHR0dBfXhjytyspK7ucnbzU1f928HN90+AmKbdu2UaDTA6dPn+7UckSzVD3i+DTl9JFaLbvp06dj+/btsLa2xvTp09ssSxMr64bmQ03MzMyUxl81f03pnXSDg4ODWkkx1JmBTF+pFex69uwJAwMD7mei+5rfc30yG3Hz183LEe2lOD47q5w+UivYbdu2TeXPRHc1HyzMGIO/vz8cHBxw7949HD9+XGU5or0o83T7aBAVT9nZ2Sm9bh7g2ipHtJOLiwsuXryoVjm+UivYjR49Wu3mrzq/cKJ5Dx486NRyRLNefPFFHDx4UK1yfKVWsAsLC+viapDupm6KeX1KRa/P1E21npqaisWLF3dxbbSTWsFu5cqVXV0P0s3U7ZXjc++dLlHMQWFsbAypVNpivWK5unNV6CO6Z8dTzf/o2xpxz+eDQ5coEjZIpVIEBwfDxMSES+zQ2NjItfz4nNhBrWBna2uLa9euwc7ODr169Wrz/h2fR2jrklu3bnE/txXsmpcj2issLAzJyckwMjLC5cuXuaw1ly9fhqurK4yMjNDU1MTrW1JqBbvPP/8cVlZW3M98HqujLxQZanr27ImamhqldQYGBujZsyeqq6spk42O6NevH4DHc7MoAp1C8xOWohwfqRXsmifiXLhwYVfVhXQjLy8vnDhxAtXV1QgODsaAAQNw7do1DBo0CDdv3uQue7y8vDRcU6KOcePGQSgUtvmgv729PcaNG9eNtdIuHb5nZ2hoqDJ554MHD2Bvb0/JO3WEv78/Vq9eDQA4cuQI5HI5gMe57QwNDZXKEd2gePLF1tYWpqamEIvFsLa2hkQiQWVlJa9TsgN/IRFAa8k7JRIJPVqkQyZOnMidsBSBTkFxwrK3t8fEiRO7u2rkL8jIyIBYLIalpSUqKytRWlqKuro6lJaWorKyEpaWlhCLxcjIyNB0VTVG7ZadYnIbAwMDfPPNN0q9OjKZDKdPn8aQIUM6v4akSxgaGiIxMRHh4eGtlklMTFRq5RHtpQhitbW1MDExwfTp02Fubo6GhgYkJSVxveoZGRnw8/PTYE01R+1g9/nnnwN43LLbsmWL0kFgYmICkUiELVu2dH4NSZfZsWNHu+vby3JDtENjYyOAx+PpampqYGBggNTUVAQHB+O7776DpaUlpFIpV46P1A52RUVFAIBJkyYhKSkJvXr16rJKka7X0NDAPV4kEAiULmUVrw8ePIiGhgZeT9KiK/Lz8wE8vvVgZGSkdO/cyMgIQqEQd+/e5crxUYfv2aWnp1Og0wPvvfce93NQUBA2btyIyMhIbNy4EUFBQSrLEe2l6HwoKSlBaGgozp07h4aGBpw7dw6hoaG4e/euUjk+6nBv7Kuvvtrm+q1bt/7lypDuo7jH4+TkhLy8PKVnK11dXeHk5IS7d+/y+oa2Lhk0aBDS0tIAPM5gk5KSwq1r3jIfNGhQt9dNW3Q42D18+FDptVQqRV5eHqqqqvDCCy90WsVI11I8P6k44zfXfBCqqucsifZZt24dNm/eDENDwxb35SQSCQwNDSGTybBu3ToN1VDzOhzsDhw40GKZXC7HG2+8gWeeeaZTKkW63tixY7n7sO2VI9rP3NwcXl5eyMnJAQCMGTOG641VpF3z8vLi9f3XTklbKhAIEBUVxfXYEu03YsQIpddz587F+vXrMXfu3DbLEe0kk8lw//59btqEixcv4syZM1yg69mzJyoqKng96L/TcjQXFhaiqamps96OdDHF1JgKu3btQlRUFHbt2tVmOaKdMjMzUVxcDLFYjKCgIAwfPhy9e/fG8OHDERQUBLFYjKKiImRmZmq6qhrT4cvYqKgopdeMMZSWluLw4cNKz9AS7fbkw+JPW45oVklJCQBAJBLh2LFj3FCiBw8e4I8//oBIJEJRURFXjo86HOwuXbqk9FogEEAoFCIhIaHdnlqiPVxcXHDt2jW1yhHtp0gAoOo+rFwu55a3lShA33U42NFljX547733cOLECQCPR90///zzkMlkMDQ0xK+//sr1wtI4O93QfOxrW/kJ+TxGljIV81Tzx/2kUmmrJzF6NlY3ZGVlcT+3FeyysrJ4e7tJZyaRrKysxLx582BtbQ0bGxtERESonTKcMYYpU6bAwMAAycnJXVtRHfFkR8TTliOa9Z///If7+cnsQ6ampirL8Y3OtOzmzZuH0tJSpKWlQSqVYtGiRViyZIlaB+OGDRsou/ITmmcnfnKSFhMTE25g6pNZjIl2ar6f/Pz8EBAQgOvXr2PgwIE4duwYDh8+3KIc3+hEsMvPz8eRI0eQk5MDT09PAMCXX36J4OBgxMfHw8nJqdVtf/vtNyQkJODChQvo06dPu58lkUi4JIgAIBaLATy+1NOnpwkUuewsLCxw7949/Prrr0hLS8PkyZPx/PPPw8HBAfX19bC3t9er762vhEIhgMcnqry8PC64AY97aBUnMKFQqFf7syPfpdOC3Z07d/Cvf/0LX331VWe9JScrKws2NjZcoAMeZ9AVCATIzs7GtGnTVG5XX1+PuXPnYvPmzXB0dFTrs+Li4rBq1aoWy48dO6ZXc6gqJkaqr6/HxIkTMXPmTHh5eeH333/HP//5T9TX13Pl1J2TlGiOouOhsbERDx8+xEsvvQRHR0eUlZUhIyODa6n36tVLr/an4u9UHZ0W7B48eIBvv/22S4JdWVlZizTwRkZGsLW1RVlZWavbLV26FL6+vggNDVX7s2JjY5XGEorFYri4uCAgIADW1tYdr7yWys7O5n6+dOkScnNzudfNOyUGDRqE4ODgbq0b6TgzMzMkJSUBePw3e+jQIZXllixZolfPsCuuvNSh0cvYmJgYrFmzps0yfzX/1qFDh3Dy5MkW4wLbY2pqqnRDV8HY2BjGxsZ/qS7ayM/PD3FxcXB2dsa9e/eU8tkZGBjA2dkZJSUl8PPz06vvra/8/f3VmnDH399fr3rYO/K3qdFgFx0d3e5sZQMGDICjoyPKy8uVljc1NaGysrLVy9OTJ0+isLAQNjY2SsvDw8Mxbtw43qcumjhxIoRCIUpKSlqdXYzmoNAdhoaG2LJlC6XZb4NGg51QKORurLbFx8cHVVVVyM3NhYeHB4DHwUwul8Pb21vlNjExMXjttdeUlo0YMQKff/45pk6d+vSV13HND4709HTuPk7ze5N8Pzh0lZmZmVKSTkX2E74zYK1NF/aE9uYiqKqqwqlTp7osq8KUKVNw7949bNmyhRt64unpyQ09UVxy7dixo9W0RAYGBjhw4ECHZkUXi8XchNH6dM9OISkpCVFRUfjzzz+5ZSKRCAkJCTT/hA6RyWRwc3PDiBEjsH//fpw6dQq//PILpkyZggkTJiA8PBx5eXm4fv26Xp3AOnJ8qt2yU6SOaWv9/Pnz1X27Dtu5cyciIyPh5+cHgUCA8PBwbsYz4HEXdEFBQYd6Z8jjk1hoaCjS09O5g2PSpEl6dUDwgSLrye7du2FsbIwJEyagrq4OEyZMgLGxMWJjY+Hr64vMzEze3ppQO9ht27atK+vRLltb2zYHEItEolbntFVQsxHLO4aGhkoHBwU63VNaWgoAGD58uMr1iuWKcnykM4+LEUJapxgwn5eXp3K9Yrk6A+v1ldotO3XTN9GEO4R0v3HjxkEkEuGzzz5r8fy3XC5HXFwc+vfvj3HjxmmmglpA7WC3fft29OvXD6NHj6bLQUK0jKGhIRISEjBjxgyEhYXh/fff56ZSXLduHVJSUrBv3z5e36JQO9i98cYb2L17N4qKirBo0SK8/PLLsLW17cq6EUI6YPr06di3bx+io6Mxfvx4bnn//v2xb98+3veuq33PbvPmzSgtLcXy5cvx888/w8XFBbNmzcLRo0eppUeIlpg+fTpu3LiBtLQ0REVFIS0tDdevX+d9oAM62EFhamqKOXPmIC0tDX/88Qf+9re/4c0334RIJFI7txwhpGspetfHjx9PvevN/OXeWIFAwGVA5fP0bIQQ3dChYCeRSLB7925MnjwZgwYNwuXLl7Fp0ybcunULlpaWXVVHQkgHyGQynDp1CqdPn+7Sp5p0jdodFG+++Sb27NkDFxcXvPrqq9i9ezfs7Oy6sm6EkA5KSkpCdHQ0iouLAQDr16+nx//+n9rBbsuWLXB1dcWAAQNw6tQpnDp1SmU5RU4tQkj3SkpKwowZMxASEoLvv/8ed+7cQd++fbF27VrMmDGD9z2yage7+fPn0zwOhGgpmUyG6OhohISEIDk5GTKZDA8ePIC3tzeSk5MRFhaGZcuWITQ0lLcdFh0aVEwI0U7NEwEIBAKl+3QCgYASAYCejSVELzRPBKCqg4ISAejI7GKEkLYpHvDftGkT/vd//7dFB8WSJUuUyvERtewI0QPjxo2DUChEbGwshg8fjszMTOzevRuZmZkYPnw4PvzwQ9jb2/M6EQAFO0L0RPMORMUjnPQo539RsCNED2RmZqK8vBxxcXHIy8vD+PHjMWfOHIwfPx5XrlzBZ599hvLycmRmZmq6qhpDwY4QPaDoeIiMjFSZCCAyMlKpHB9RsCNEDzTPVKwqEQBlKqZgR4heaJ6puPmE5wBlKlagYEeIHlBkKk5JSUFYWBjOnTvHZSoOCwtDSkoK4uPjefv0BEDj7AjRG5SpuG0U7AjRIzQPcOso2BGiZ2geYNXonh0hhBco2BFCeIGCHaE03oQXKNjxXFJSEtzc3DB58mSsX78ekydPhpubG2WcJnqHOih4TJHG+8UXX8TSpUtx/fp1DBw4EGlpaZTGm+gdA0ZpEdokFovRs2dPVFdXw9raWtPV6TQymQxubm6ws7NDeXk5bt26xa1zdXWFvb09Hjx4gOvXr1Nvng6SSqVITU1FcHAwjI2NNV2dLtOR45NadjylSONdXFwMc3NzpXX379/ngh+f03gT/aIz9+wqKysxb948WFtbw8bGBhEREaitrW13u6ysLLzwwgvo0aMHrK2tMX78eDQ0NHRDjbVbSUkJ97Ofn59Sskc/Pz+V5QjRZToT7ObNm4crV64gLS0NKSkpOH36NJdqujVZWVkICgpCQEAAzp8/j5ycHERGRkIg0Jmv3WXu3bsHABg1ahQOHjwIb29vmJubw9vbGwcPHsTIkSOVyhGi63TiMjY/Px9HjhxBTk4OPD09AQBffvklgoODER8fDycnJ5XbLV26FO+88w5iYmK4ZYMHD+6WOmu7Bw8eAECLS1gFCwsLpXKE6DqdCHZZWVmwsbHhAh0A+Pv7QyAQIDs7G9OmTWuxTXl5ObKzszFv3jz4+vqisLAQQ4YMwaeffornn3++1c+SSCSQSCTca7FYDODxDV+pVNqJ30qzFP1S586dw0svvYTo6Gg0NDTg119/RUJCAs6dO8eV06fvzQcymQwZGRk4ffo0TE1NMXHiRL3tZOrI36ZOBLuysjLY29srLTMyMoKtrS3KyspUbnPz5k0AwMcff4z4+Hi4u7tjx44d8PPzQ15eHgYOHKhyu7i4OKxatarF8mPHjnGtHX1gZmYGAHB2dkZOTg5eeOEFbp2DgwOcnZ1RUlICMzMzpKamaqqapIOysrKwbds2lJeXA3g8u5i9vT0WLVoEHx8fDdeu89XX16tdVqPBLiYmBmvWrGmzTH5+/l96b0UCw3/84x9YtGgRAGD06NE4ceIEtm7diri4OJXbxcbGIioqinstFovh4uKCgIAAvRp6EhgYiE2bNqGkpARBQUEYMGAAbty4ATc3N9y8eRNHjhyBvb09li9frretAn1z4MABrF27FsHBwVi2bBnKysrg6OiI+Ph4rF27Fnv27FF5FaTLFFde6tBosIuOjsbChQvbLDNgwAA4OjpyZyqFpqYmVFZWwtHRUeV2ivTTw4YNU1o+dOhQpTFlTzI1NYWpqWmL5cbGxno1XsnY2BhbtmxBeHg4jh49yl3WHjt2jJulKjExkWsBEu0mk8nwwQcfICQkBMnJyZDJZEhNTcVzzz2H8ePHIywsDDExMQgPD9erk1dHjkmNBjuhUAihUNhuOR8fH1RVVSE3NxceHh4AgJMnT0Iul8Pb21vlNiKRCE5OTigoKFBafu3aNUyZMuXpK69HzMzMlIbjmJubd+jygGieYtzk7t27IRAIlJ5vFggEiI2Nha+vL6/HTerEGIyhQ4ciKCgIixcvxvnz53HmzBlERkbi73//O9cTW1JSgiFDhuD8+fMAHs+h+f7772Pjxo3Yt28fbty4gY8++ghXr15FRESEJr+OVpDJZIiOjsbUqVNRXV2tNBtVVVUVpk6dimXLllFSAB2hmDVs+PDhKtcrlvN5djEwHfHgwQM2Z84cZmlpyaytrdmiRYtYTU0Nt76oqIgBYOnp6UrbxcXFsb59+zILCwvm4+PDMjMzO/S51dXVDACrrq7ujK+hNdLT0xkAlpWVxRhjrLGxkSUnJ7PGxkbGGGNnz55V+fsk2omv+7Mjx6dO9MYCgK2tLXbt2tXqepFIpHL285iYGKVxduQxagnol+aziyUnJyuto9nFHtOZYEc6V/N5Rr28vLh8dj169MCkSZNonlEdo5hdbMaMGQgNDcXkyZNx/fp1/Pnnn0hLS8Phw4exb98+veqc6CjKetIOPmQ9uX//Pv78809uXb9+/SAUCinriQ5avnw51q9fr3Sv1dDQEFFRUVi7dq0Ga9Y1OnJ86kQHBel8hoaGmDlzJi5cuIBHjx4hMTERW7duRWJiIh49eoQLFy5gxowZFOh0SFJSEtatWwcTExOl5SYmJli3bh3vE7JSy64dfGjZVVRUoLi4mFvXv39/9O7dm1p2OkQmk6FPnz64f/8+zM3NWwwlamhogL29Pe7evatX+5NadqRdinFZX375JW7cuKE09OT69evYuHEjioqKkJmZqemqEjVkZGTg/v37AFpP2VVeXo6MjAwN1lKzKNjxVPPeWMU8o+PHj+fmGaXeWN1y8uRJAMCzzz6LpKQkPHr0CDk5OXj06BGSkpLw7LPPKpXjIwp2PNW8N1bV7GLUG6tbbt++DeDxyWvQoEFKEygNGjSIe2xSUY6PaOgJTynGZb399ttK9+zWr18PkUgEOzs73o/L0iUuLi4AgG+++QYhISH4/vvvcefOHfTt2xdr1qzB1q1blcrxEbXseKp5b2xDQwMSExOxbds2JCYmoqGhgXpjdcyECRO4nxlj3AD75j8/WY5vqDe2HXzojX1ynJ2iZUe9sbrjxIkT8Pf3B4BWe2MB4Pjx40pzjOg66o0l7VL0xoaHh7d4zE4ul2P69OnUG6tDnkyB1pwiZVd75fQdBTueUvSyxsbGckMWFO7fv48PP/xQqRzRboqOpLi4uBZZve3t7fHZZ58pleMjCnY81fyAaGsqxScPHKKdFB1OZ8+exfXr15XGTV67dg1ZWVm873CiYMdTimcnbW1tceDAAaWpFA8cOABbW1ulckS7KRIBpKSkIDw8HKampvDy8oKpqSnCw8ORkpKC+Ph4Xt9/paEnPKW4F/fw4UOEhYWhf//+uHbtGo4fP46ioiI8fPiQKxcQEKDJqhI1TZ8+Hfv27UNUVBTGjx/PLReJRNi3bx+mT5+uwdppHgU7nvPx8cHhw4e518eOHQMA+Pr64uzZs5qqFnkKzTskyH/RZSxPKeYhOHv2LAQC5T8DgUDABTq+zlegi5KSkjBjxgyMGDFC6R7siBEjMGPGDMp6QuPs2qav4+waGhq4eXANDAyUhp80f11fXw9zc3ON1JGoTzFucsSIEUqziwUHB8PQ0BBhYWHIy8vTu3GTNM6OtCsxMbHVdc0vg9oqR7SHYtzkhx9+qLKlHhsby/txkxTseOr06dMAgGeeeQaurq5K61xdXfHMM88olSPajeYUaR8FO55SzAs7bNgwlU9QDB48WKkc0W7Ns9ioQllsKNjxlqenJwDg559/xq1bt5TW3bp1C6mpqUrliHZrPruYXC5XWkeziz1GwY6nnuxlDQwMRFxcHAIDA9ssR7RT80HFYWFhOHfuHBoaGnDu3DmEhYXRoGLQODveampqUnp99OhRHD16tN1yRHspBhVHR0crDSru378/DSoGDT1pl74OPQkICEBaWhqAtlMCTZ48mRtoTHSDTCZDeno6fvnlF0yZMgWTJk3S2xYdDT0h7VI8DhYdHd3iYX8HBwcsXbpUqRzRHarmFCF0GctbXl5euHDhAg4ePIjr16/j1KlTXEtgwoQJGDp0KFeOEH1ALTueSkhIAADcuHED06ZNU8qSMW3aNBQWFiqVI0TXUcuOp8zNzREaGoqDBw/i8OHDSskAFEJDQ+lRMaI3qGXHY8nJyQgNDVW5LjQ0FMnJyd1bIUK6EAU7nktOTkZ9fT1ef/11uLu74/XXX0d9fT0FOqJ36DKWwNzcHBs3buSyZBgbG2u6SoR0Op1p2VVWVmLevHmwtraGjY0NIiIiUFtb2+Y2ZWVleOWVV+Do6IgePXpgzJgx2L9/fzfVmBCiTXQm2M2bNw9XrlxBWloaUlJScPr0aSxZsqTNbebPn4+CggIcOnQIly9fxvTp0zFr1ixcunSpm2pNCNEaTAf88ccfDADLycnhlv3yyy/MwMCAlZSUtLpdjx492I4dO5SW2drasq+//lrtz66urmYAWHV1dccrrkMaGxtZcnIya2xs1HRVyFOSSCQsPj6eBQcHs/j4eCaRSDRdpS7TkeNTJ+7ZZWVlwcbGRikDh7+/PwQCAbKzszFt2jSV2/n6+uLHH3/Eiy++CBsbG/z000949OhRmw+3SyQSSCQS7rVYLAYASKVSSKXSzvlCWkjx3fT5O/JBTEwMNmzYwGU+SU1NxfLly/Hee+9h9erVGq5d5+vI36tOBLuysrIWjzQZGRnB1tYWZWVlrW73008/Yfbs2ejduzeMjIxgYWGBAwcOwM3NrdVt4uLisGrVqhbLjx07xqUx12eK52WJ7tm+fbvKXnS5XI7169fj5s2bWLhwYbfXqyt1JN+iRoNdTEwM1qxZ02aZ/Pz8v/z+H330EaqqqnD8+HHY2dkhOTkZs2bN4iYhUSU2NhZRUVHca7FYDBcXFwQEBOhVIoAnSaVSpKWlYfLkydQbq4MaGxtbvcJROHjwIH744QeYmJh0U626nuLKSx0aDXbR0dHtnmkGDBgAR0dHlJeXKy1vampCZWUlHB0dVW5XWFiITZs2IS8vD3/7298AAKNGjUJmZiY2b96MLVu2qNzO1NQUpqamLZYbGxvzIgjw5Xvqmw0bNihlnH755Zfh4eGB3Nxc/PDDDwAAxhj+/e9/4/3339dUNTtdR/5WNRrshEIhhEJhu+V8fHxQVVWF3NxceHh4AABOnjwJuVwOb29vldsomrdPTj5iaGjYIpMrIbqu+TSJDQ0NMDQ0RGpqKt566y18/fXX3GN/SUlJehXsOkInhp4MHToUQUFBWLx4Mc6fP48zZ84gMjISf//73+Hk5AQAKCkpwZAhQ3D+/HkAwJAhQ+Dm5oZ//OMfOH/+PAoLC5GQkIC0tDSEhYVp8NsQ0vkKCgoAAGPGjIGZmZnSOjMzM7i7uyuV4yOdCHYAsHPnTgwZMgR+fn4IDg7G888/j6+++opbL5VKUVBQwLXojI2NkZqaCqFQiKlTp2LkyJHYsWMHvvvuOwQHB2vqaxDSJRT34a5evdoiu3RTUxOuXbumVI6PdKI3FgBsbW2xa9euVteLRKIWs2QNHDiQnpggvODl5YWUlBTU19fD2dkZH3/8MczMzPDNN9/g448/5hoBfM5PqDPBjhDSut27d8PKygoAUF5ejjfffLPVcnylM5exhJDWWVpatttq8/LygqWlZTfVSPtQsCNET5w/f77VgOfl5cV13vEVBTtC9Mj58+dRU1ODqVOnol+/fpg6dSpqamp4H+gAumdHiN6xtLTE/v37KT/hE6hlRwjhBQp2hBBeoGBHCOEFumfXDsVA5Y5kV9BFUqkU9fX1EIvFdI9HD/BlfyqOyycfKFCFgl07ampqAAAuLi4argkhpDU1NTXo2bNnm2UMmDohkcfkcjnu3r0LKysrGBgYaLo6XUaRt+/27dt6nbePL/iyPxljqKmpgZOTU4sMR0+ill07BAIB+vbtq+lqdBtra2u9Pjj4hg/7s70WnQJ1UBBCeIGCHSGEFyjYEQCP09GvXLlSZUp6ontof7ZEHRSEEF6glh0hhBco2BFCeIGCHSGEFyjY6ZCJEyfivffe03Q1CNFJFOz0UEZGBgwMDFBVVaXpqpAO0LaTmbbV52lRsCNEjzQ2Nmq6ClqLgp2Okkgk+OCDD+Di4gJTU1O4ubnh22+/RXFxMSZNmgQA6NWrFwwMDLBw4cJ236+mpgbz5s1Djx490KdPH3z++ectzuzff/89PD09YWVlBUdHR8ydOxfl5eXcekWL8sSJE/D09ISFhQV8fX15PTGzuhYuXIhTp07hiy++gIGBAQwMDFBYWIiIiAj0798f5ubmGDx4ML744osW24WFheHTTz+Fk5MTBg8eDAA4e/Ys3N3dYWZmBk9PTyQnJ8PAwAC//fYbt21eXh6mTJkCS0tLODg44JVXXkFFRUWr9SkuLu6uX0fXYERnTJgwgb377ruMMcZmzZrFXFxcWFJSEissLGTHjx9ne/bsYU1NTWz//v0MACsoKGClpaWsqqqq3fd+7bXXWL9+/djx48fZ5cuX2bRp05iVlRX3eYwx9u2337LU1FRWWFjIsrKymI+PD5syZQq3Pj09nQFg3t7eLCMjg125coWNGzeO+fr6dvavQu9UVVUxHx8ftnjxYlZaWspKS0vZo0eP2IoVK1hOTg67efMm++GHH5iFhQX78ccfue0WLFjALC0t2SuvvMLy8vJYXl4eq66uZra2tuzll19mV65cYampqWzQoEEMALt06RJjjLGHDx8yoVDIYmNjWX5+Prt48SKbPHkymzRpUqv1aWpq0sSvptNQsNMhimBXUFDAALC0tDSV5RRB5+HDh2q9r1gsZsbGxmzv3r3csqqqKmZhYaEU7J6Uk5PDALCamhqlzz1+/DhX5vDhwwwAa2hoUKsufNb8ZNaat956i4WHh3OvFyxYwBwcHJhEIuGWJSYmst69eyv9zr/++mulYPfJJ5+wgIAApfe+ffs2d5JUtz66hC5jddBvv/0GQ0NDTJgwoVPe7+bNm5BKpRg7diy3rGfPntwlkUJubi6mTp0KV1dXWFlZcZ9/69YtpXIjR47kfu7Tpw8AKF3uEvVt3rwZHh4eEAqFsLS0xFdffdXi9z1ixAiYmJhwrwsKCjBy5EiYmZlxy5rvWwD4/fffkZ6eDktLS+7fkCFDAACFhYVd+I00h1I86SBzc/Nu/8y6ujoEBgYiMDAQO3fuhFAoxK1btxAYGNjipnjzzLiKHIByubxb66sP9uzZg2XLliEhIQE+Pj6wsrLCunXrkJ2drVSuR48eHX7v2tpaTJ06FWvWrGmxTnGC0jcU7HTQiBEjIJfLcerUKfj7+7dYrzjLy2Qytd5vwIABMDY2Rk5ODlxdXQEA1dXVuHbtGsaPHw8AuHr1Kh48eIDVq1dzWZsvXLjQGV+H/D8TExOlfXbmzBn4+vrizTff5Jap0+oaPHgwfvjhB0gkEi4RQE5OjlKZMWPGYP/+/RCJRDAyUh0GnqyPrqPLWB0kEomwYMECvPrqq0hOTkZRUREyMjLw008/AQD69esHAwMDpKSk4P79+6itrW3z/aysrLBgwQK8//77SE9Px5UrVxAREQGBQMC1zFxdXWFiYoIvv/wSN2/exKFDh/DJJ590+XflE5FIhOzsbBQXF6OiogIDBw7EhQsXcPToUVy7dg0fffRRi6Clyty5cyGXy7FkyRLk5+fj6NGjiI+PB/DflvZbb72FyspKzJkzBzk5OSgsLMTRo0exaNEiLsA9WR9db51TsNNRiYmJmDFjBt58800MGTIEixcvRl1dHQDA2dkZq1atQkxMDBwcHBAZGdnu+61fvx4+Pj4ICQmBv78/nnvuOQwdOpS77yMUCrF9+3bs3bsXw4YNw+rVq7kDiHSOZcuWwdDQEMOGDYNQKERgYCCmT5+O2bNnw9vbGw8ePFBq5bXG2toaP//8M3777Te4u7vjf/7nf7BixQoA4Pank5MTzpw5A5lMhoCAAIwYMQLvvfcebGxsuPTmT9bnyXuFuoZSPBGV6urq4OzsjISEBERERGi6OuQp7dy5E4sWLUJ1dbVG7vlqA7pnRwAAly5dwtWrVzF27FhUV1fjX//6FwAgNDRUwzUjf8WOHTswYMAAODs74/fff8cHH3yAWbNm8TbQARTseOHWrVsYNmxYq+v/+OMPAEB8fDwKCgpgYmICDw8PZGZmws7OrruqSTpRWVkZVqxYgbKyMvTp0wczZ87Ep59+qulqaRRdxvJAU1NTm4/6tNUjR4i+oGBHCOEF6o0lhPACBTtCCC9QsCOE8AIFO0IIL1CwI4TwAgU7ojUWLlzIZcU1NjaGg4MDJk+ejK1bt3bouczt27fDxsam6yraCkXWYKKdKNgRrRIUFITS0lIUFxfjl19+waRJk/Duu+8iJCQETU1Nmq4e0WWazBxKSHMLFixgoaGhLZafOHGCAWBff/01Y4yxhIQENnz4cGZhYcH69u3L3njjjRbZkpv/W7lyJWOMsR07djAPDw9maWnJHBwc2Jw5c9i9e/e4z6msrGRz585ldnZ2zMzMjLm5ubGtW7dy62/dusVmzpzJevbsyXr16sVeeuklVlRUxBhjbOXKlS0+Nz09vUt+T+SvoZYd0XovvPACRo0ahaSkJACAQCDAxo0bceXKFXz33Xc4efIkli9fDgDw9fXFhg0bYG1tjdLSUpSWlmLZsmUAAKlUik8++QS///47kpOTUVxcrDQZ0UcffYQ//vgDv/zyC/Lz85GYmMg9LieVShEYGAgrKytkZmbizJkzsLS0RFBQEBobG7Fs2TLMmjWLa5mWlpbC19e3e39RpG2ajraEKLTWsmOMsdmzZ7OhQ4eqXLd3717Wu3dv7vW2bdtYz5492/28J+fQmDp1Klu0aJHKst9//z0bPHgwk8vl3DKJRMLMzc3Z0aNH260/0Txq2RGdwBjjEk8eP34cfn5+cHZ2hpWVFV555RU8ePAA9fX1bb5He3NovPHGG9izZw/c3d2xfPlynD17ltv2999/x40bN2BlZcXN2WBra4tHjx7p7ZwN+oaCHdEJ+fn56N+/P4qLixESEoKRI0di//79yM3NxebNmwG0PUG0Yg4Na2tr7Ny5Ezk5OThw4IDSdlOmTMGff/6JpUuX4u7du/Dz8+MugWtra+Hh4YHffvtN6d+1a9cwd+7cLv72pDNQqgui9U6ePInLly9j6dKlyM3NhVwuR0JCApdRV5GOXkHV3AnqzqEhFAqxYMECLFiwAOPGjcP777+P+Ph4jBkzBj/++CPs7e1hbW2tsp76NmeDvqGWHdEqEokEZWVlKCkpwcWLF/HZZ58hNDQUISEhmD9/Ptzc3CCVSrm5ML7//nts2bJF6T1EIhFqa2tx4sQJVFRUoL6+Xq05NFasWIGDBw/ixo0buHLlClJSUjB06FAAwLx582BnZ4fQ0FBkZmZy83688847uHPnDve5//nPf1BQUICKigpIpdLu+aUR9Wj6piEhCgsWLOCGbRgZGTGhUMj8/f3Z1q1bmUwm48qtX7+e9enTh5mbm7PAwEC2Y8eOFpOCv/7666x3795KQ0927drFRCIRMzU1ZT4+PuzQoUMtJo4eOnQoMzc3Z7a2tiw0NJTdvHmTe8/S0lI2f/58Zmdnx0xNTdmAAQPY4sWLWXV1NWOMsfLycjZ58mRmaWlJQ0+0EOWzI4TwAl3GEkJ4gYIdIYQXKNgRQniBgh0hhBco2BFCeIGCHSGEFyjYEUJ4gYIdIYQXKNgRQniBgh0hhBco2BFCeOH/ACGo87Gne4s8AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:00.706190Z", + "iopub.status.busy": "2024-03-22T19:36:00.705898Z", + "iopub.status.idle": "2024-03-22T19:36:00.984100Z", + "shell.execute_reply": "2024-03-22T19:36:00.983182Z" + }, + "papermill": { + "duration": 0.301051, + "end_time": "2024-03-22T19:36:00.986068", + "exception": false, + "start_time": "2024-03-22T19:36:00.685017", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABWTUlEQVR4nO2dd3iTVfvHP5ndg9UBFMoSKHsIggPQMnwRxYmILAUHor5WlCGCDAUVkFdEUBRFZbpxodAfoAyZZe9ZVhfQTZv1/P54mtC0SZu0SZO253Ndua7kmffTJt9zzn3uc98KSZIkBAKBoJKh9LQBAoFAUBaEeAkEgkqJEC+BQFApEeIlEAgqJUK8BAJBpUSIl0AgqJQI8RIIBJUSIV4CgaBSova0ARWNyWTi8uXLBAUFoVAoPG2OQCAogiRJZGVlUbduXZRK+/2raidely9fJioqytNmCASCUrhw4QL169e3u7/aiVdQUBAg/2GCg4M9bI1AIChKZmYmUVFRlt+qPaqdeJmHisHBwUK8BAIvpjS3jnDYCwSCSokQL4FAUCkR4iUQCCol1c7nVRWQJAmDwYDRaPS0KQIvRKVSoVarq3wokBCvSoZOp+PKlSvk5uZ62hSBF+Pv709kZCRardbTprgNIV6lYDCaUKu8Y3RtMpk4e/YsKpWKunXrotVqq3zrKnAOSZLQ6XSkpqZy9uxZmjVrVmKgZ2VGiJcdEhKv8/ZvRwnx0/D5iFs9bQ4g97pMJhNRUVH4+/t72hyBl+Ln54dGo+H8+fPodDp8fX09bZJbEOJlh0AfNbvPX0erUpKdbyDQx3v+VFW1JRW4jurwHan6T1hGmoYF0qh2ADqjic3HUz1tjkBQqVm7/zJxq/eRp3fdJJPHxWvhwoVER0fj6+tL165d2blzZ4nHz58/n+bNm+Pn50dUVBSvvPIKeXl5LrdLoVDQp1U4AH8dSXL59QWC6sLBixm89u1+fki4xKqdiS67rkfFa/Xq1cTFxTF16lT27t1Lu3bt6Nu3LykpKTaPX7FiBRMmTGDq1KkcPXqUzz//nNWrVzNp0iS32NcnJgKA/zuWgs5gcss9qgs9e/bkv//9r6fNqDJs2rQJhUJBenq6p00pkZSsPEZ/tZt8g4m7W4QxtFu0y67tUfGaN28eo0ePZuTIkcTExLB48WL8/f1ZunSpzeO3bdvG7bffzhNPPEF0dDR9+vRh8ODBpfbWykqHqFBqB/qQlWfg3zNX3XIPgTWV5UcpKJ18g5Hnvt5DUmYeTeoEMP/x9qiUrpsd95h46XQ69uzZQ2xs7E1jlEpiY2PZvn27zXO6d+/Onj17LGJ15swZfv/9d/7zn/+4xUalUkHvGDF0FLgHnU7naRPchiRJvPHjIfYmphPsq+az4bcS7Ktx6T08Jl5paWkYjUbCw8OttoeHh5OUZFsonnjiCaZPn84dd9yBRqOhSZMm9OzZs8RhY35+PpmZmVYvZzD7vdYfScZk8q7i4pIkkaszeORVnkLr+fn5jB8/nqioKHx8fGjatCmff/45586do1evXgDUqFEDhULBiBEjSr1eVlYWQ4YMISAggMjISD744INiw9Svv/6azp07ExQUREREBE888YSVe8Lc44uPj6dz5874+/vTvXt3jh8/7vBzzZw5k7CwMIKCghg1ahQTJkygffv2lv0jRoxg4MCBvP3229StW5fmzZs7ZBvA77//zi233IKfnx+9evXi3LlzDtvlCZZuPcd3ey6iVMDCIR1pVDvA5ffwnvl/B9i0aRPvvPMOH3/8MV27duXUqVO8/PLLzJgxgzfffNPmObNmzWLatGllvmf3JrUI0KpIzsznwKUM2keFlvlaruaG3kjMlD89cu8j0/viry3b12fYsGFs376dDz/8kHbt2nH27FnS0tKIiori+++/5+GHH+b48eMEBwfj5+dX6vXi4uLYunUra9euJTw8nClTprB3714r4dDr9cyYMYPmzZuTkpJCXFwcI0aM4Pfff7e61htvvMHcuXOpU6cOzz33HE899RRbt24t1Ybly5fz9ttv8/HHH3P77bezatUq5s6dS6NGjayOi4+PJzg4mPXr1zts24ULF3jooYd44YUXeOaZZ9i9ezevvvpqqTZ5ir9PpPL2b0cAmNw/hjub1XHLfTwmXrVr10alUpGcnGy1PTk5mYiICJvnvPnmmwwdOpRRo0YB0KZNG3JycnjmmWd44403bMa2TJw4kbi4OMtnc6IzR/FRq+jZIozfDlzhr8NJXiVelZETJ06wZs0a1q9fb3EZNG7c2LK/Zs2aAISFhREaGlrq9bKysli2bBkrVqzgnnvuAeCLL76gbt26Vsc99dRTlveNGzfmww8/5NZbbyU7O5vAwEDLvrfffpsePXoAMGHCBPr3709eXl6pgZ4LFizg6aefZuTIkQBMmTKFv/76i+zsbKvjAgIC+Oyzz6yW7ZRm26JFi2jSpAlz584FoHnz5hw8eJB333231L9PRXMmNZuxK/ZikuCxzvUZeXu02+7lMfHSarV06tSJ+Ph4Bg4cCMjLX+Lj4xk7dqzNc3Jzc4sJlEqlArA7jPHx8cHHx6dctvaJCZfF60gyr/drUa5ruRI/jYoj0/t67N5lYd++fahUKotAlJczZ86g1+vp0qWLZVtISIhlSGZmz549vPXWW+zfv5/r169jMsmzx4mJicTExFiOa9u2reV9ZGQkACkpKTRo0KBEO44fP86YMWOstnXp0oX/+7//s9rWpk2bYusNS7Pt6NGjdO3a1eqcbt26lWiPJ8jM0zPqq91k5hno1LAGMwa2duvyNY8OG+Pi4hg+fDidO3emS5cuzJ8/n5ycHEvrNWzYMOrVq8esWbMAGDBgAPPmzaNDhw6WYeObb77JgAEDLCLmDnq1CEOjUnAqJZvTqdk0qRNY+kkVgEKhKPPQzVM4Mgx0NTk5OfTt25e+ffuyfPly6tSpQ2JiIn379i3mNNdobjqVzT88s5i4goAAa9+PM7Z5M0aTxEsrEziTmkNkiC+Ln+yEj9p9v0nwsHgNGjSI1NRUpkyZQlJSEu3bt2fdunUWJ35iYqJVT2vy5MkoFAomT57MpUuXqFOnDgMGDODtt992q53Bvhpua1yLf06msf5IMk16eId4VUbatGmDyWRi8+bNVjPNZsy9EkfT/TRu3BiNRsOuXbssvaOMjAxOnDjBXXfdBcCxY8e4evUqs2fPtrgMdu/e7YrHsdC8eXN27drFsGHDLNt27dpV6nmO2NayZUvWrl1rte3ff/91gdWu4711x9h0PBVfjZIlwzpTJ6h8ox1H8HiE/dixYzl//jz5+fns2LHDqnu8adMmvvzyS8tntVrN1KlTOXXqFDdu3CAxMZGFCxc65BspL31ayX64vw6LkInyEB0dzfDhw3nqqaf46aefOHv2LJs2bWLNmjUANGzYEIVCwa+//kpqamoxn1FRgoKCGD58OK+99hobN27k8OHDPP300yiVSkvPqUGDBmi1WhYsWMCZM2dYu3YtM2bMcOlzvfjii3z++ecsW7aMkydPMnPmTA4cOFDqsMkR25577jlOnjzJa6+9xvHjx1mxYoXV78LT/LD3Ip/8fQaA9x9pR+t6IRVyX4+LV2Whd0u5N5hwIZ2UTNcvR6pOLFq0iEceeYQxY8bQokULRo8eTU5ODgD16tVj2rRpTJgwgfDwcLv+z8LMmzePbt26cd999xEbG8vtt99Oy5YtLU72OnXq8OWXX/Ltt98SExPD7NmzmTNnjkufaciQIUycOJFx48bRsWNHzp49y4gRI0p19DtiW4MGDfj+++/56aefaNeuHYsXL+add95xqf1lJSHxOhN+OAjA2F5NGdCubilnuA6FVJ6AnUpIZmYmISEhZGRkOF096IGFW9l/IZ13HmzDE11LduC6g7y8PM6ePUujRo2qbJoTV5CTk0O9evWYO3cuTz/9tMfs6N27NxEREXz99dcVfu+K+K4kZ+YxYMEWUrLy6R0TzidPdkLpggh6R3+jlcvb62H6tgpn/4V0/jqS5BHxEtgmISGBY8eO0aVLFzIyMpg+fToADzzwQIXZkJuby+LFi+nbty8qlYqVK1eyYcMGq3iuqkSe3sgzX+0mJSufW8ID+WBQe5cIlzOIYaMTmBdqbzt1law8vYetqR4kJiYSGBho95WYKGcpmDNnDu3atSM2NpacnBz++ecfateu7TI7WrVqZdeG5cuXo1Ao+P3337nrrrvo1KkTv/zyC99//73NSYnKjiRJTPzhIPsvZhDqr+GzYbd6JN+d6Hk5QdOwQBrXCeBMag6bT6RyX9uKG99XV+rWrcu+fftK3N+gQQP27NnjVjt+//139HrbDVZ4eDh+fn5s2LDBrTZ4C5/+fYYfEy6hUir4eEhHGtTyTFZfIV5O0icmgsWbT/Pn4WQhXhWAWq2madOmnjaDhg0betoEr2DjsRRmrzsGwFsDYujexHW9W2cRw0YnMS/U3ngshXyDKD0mqD6cSsnipZUJSBI80bUBT97mWUEX4uUk7euHUifIh+x8A/+eueZpcwSCCiEjV8+oZbvJyjfQJbombw1o5fHKVUK8nMQqx5cIWBVUAwxGE2NX7uXc1Vzqhfqx6MmOaNWelw7PW1AJ6RPjvTm+BAJX887vx/jnZBp+GhVLhnWmVqD7l/44ghCvMtCtSS0CfdSkZOWz/2K6p80RCNzGmt0XWLr1LADzHmtHTF3nArvdiRCvMuCjVtGzuZxg7a8jyaUcLSgv0dHRzJ8/3/JZoVDw008/ecye6sKe89eY/OMhAP4b24x720R62CJrhHiVEbFQ23NcuXKFe++919NmVGkup9/g2a/3ojOauLd1BC/d3czTJhVDiFcZ6dm8DhqVgtOpOZxKKTnzgcC1RERElDvBZHmRJAmDweBRG9zFDZ2RZ77eTVp2Pi0jg5n7WLsKX/rjCEK8ykiwr4ZuBQF66z01dJQk0OV45uXken5HimQ4SuFh47lz51AoFPzwww/06tULf39/2rVrV6wC1ZYtW7jzzjstxYpfeuklSyYLcLxAxx9//EGnTp3w8fFhy5YtTtvu7UiSxGvf7efQpUxqBWhZMqyT1ya89E6rKgl9W4Xz94lU/jqSxPM9m1S8AfpceMdDUf6TLoPW8YowjhTJKA9vvPEGc+bMoVmzZrzxxhsMHjyYU6dOoVarOX36NP369WPmzJksXbqU1NRUxo4dy9ixY/niiy8Axwt0TJgwgTlz5tC4cWNq1KjhEtu9iY83nebXA1dQKxUserIT9Wt4ZumPIwjxKge9W4bzxo+HSEiUc3yFBYs0NbZwtEhGeRg3bhz9+/cHYNq0abRq1YpTp07RokULZs2axZAhQyy9vGbNmvHhhx/So0cPFi1ahK+vr8MFOqZPn07v3r1dZrc38dfhJN7/Uy71Nv2B1nRpVNPDFpWMEK9yEBbsS4cGoSQkprP+aDJDulbwcgmNv9wD8gQax1tkR4tklAd7hTNatGjB/v37OXDgAMuXL7ccI0kSJpOJs2fP0rJlS4cLdHTu3NllNnsTx5OyeGX1PgCGdWtYKVI+CfEqJ31iIkhITOfPwx4QL4XCqaFbVaakwhnZ2dk8++yzvPTSS8XOa9CggVNFMIoW0KgKXM/RMeqrXeTojHRrXIs374sp/SQvQIhXOenTKpx31x1j++k0MvP0Li9pXhVwpEiGO+nYsSNHjhyxm53i4MGDbi/Q4a3ojSbGLN/LhWs3iKrpx8dDOqJRVY55vMphpRfTpE4gTeoEoDdKbDqe6mlzvBJHimS4k/Hjx7Nt2zbGjh3Lvn37OHnyJD///LMlP35FFOjwVmb8eoTtZ64SoFXx2bBbqRGgLf0kL0GIlwsQAaulU1qRDHfStm1bNm/ezIkTJ7jzzjvp0KEDU6ZMsUwYVESBDm9kxY5Evtp+HoUC5j/egeYRQZ42ySlEAQ4XkJB4nQc/3kagj5o9b8a6rdhmVSrA4S1FMqoqpX1Xdpy5ypDPdmAwSbzWtzkv9PJ8wkczjv5GRc/LBbSrH0pYQY6v7aevetocryQhIYGVK1dy+vRp9u7dy5AhQ4CKLZIhkLl4PZfnl+/FYJK4r20kYzwRo+gChHi5AKscX2Khtl1sFck4evRoiQU2BK4lJ9/AqGW7uZajo3W9YN5/pJ3HkwqWFTHb6CL6tIpg+Y5E1h9JZuYDrb1yLZgn6dChg80iGTdu3CixwIbAdUiSxLhv93MsKYvagVo+HdoZP617XBwVgRAvF9GtcS2CfNSkZuWz72I6HRtUvaUj7sDPz88rCmxUB9YfSeaPQ0loVAo+GdqJuqF+njapXIhho4vQqpX0bBEGwF+H3Tt0rGZzLIIyYOs78tX28wA8dUcjOjX07qU/jiDEy4X0bWX2e7knZMIcRZ6bm+uW6wuqDubviPk7cyoliy2n0lAqYKiHq/64CjFsdCE9bqmDVqXkTEGOr6ZhrnU4q1QqQkNDLala/P39K62zVeAeJEkiNzeXlJQUQkNDUalkn9bXBb2u2JbhXp0pwhmEeLmQIF8N3ZvWYtNxOU1O0zDX+3IiIuSA2MK5pgSCooSGhlq+K1l5er7bcxGAYd2iPWiVaxHi5WL6xETI4nU4mTE9XS9eCoWCyMhIwsLC7JafF1RvNBqNpccF8GPCJXJ0RhrXCeD2prU8aJlrEeLlYmJjwnjjJ9h3IZ2kjDwiQtwTCa9Sqay+oAKBLSRJYtm2cwAM7xZdpdwMwmHvYsKCfOkQFQrA+qMiYFXgWbadvsrp1BwCtCoe6ljP0+a4FCFebkAs1BZ4C+Ze18Od6hNUxdI1CfFyA+aK2ttPXyXjhvBLCTzDpfQbbCjo/Q/rVjXCIwojxMsNNK4TSNOwQAwmiU3HxaygwDMs//c8Jglub1qLpmGVK92NIwjxchN9xEJtgQfJ0xtZtesCAENvi/asMW5CiJebMPu9Nh1LId9g9LA1gurGbweucC1HR90QX2JbhnnaHLcgxMtNtK0XQniwDzk6I9tEji9BBfPV9nMADLmtIepKkpPeWTz+VAsXLiQ6OhpfX1+6du3Kzp07Szw+PT2dF154gcjISHx8fLjllluKFQb1BqxyfLl5obZAUJh9F9LZfzEDrUrJ47dGedoct+FR8Vq9ejVxcXFMnTqVvXv30q5dO/r27Wt36YtOp6N3796cO3eO7777juPHj7NkyRLq1fPO+JW+BUPH9UeSMZlEJghBxfBVQXjEfe0iqRXo41lj3IhHI+znzZvH6NGjGTlyJACLFy/mt99+Y+nSpUyYMKHY8UuXLuXatWts27bNslo+Ojq6Ik12iq6NahHkqyYtO5+EC+l0aihyfAncS1p2Pr8euALIEfVVmXL1vLKzs8nMzLR6OYpOp2PPnj3ExsbeNEapJDY2lu3bt9s8Z+3atXTr1o0XXniB8PBwWrduzTvvvIPRaN8hnp+fX2Yby4tWreRuc44vN6XJEQgKs3rXBXRGE+2iQmlXsNKjquK0eJ09e5b+/fsTEBBASEgINWrUoEaNGoSGhlKjhuM9i7S0NIxGI+Hh4Vbbw8PDSUqy/UM/c+YM3333HUajkd9//50333yTuXPnMnPmTLv3mTVrFiEhIZaXuahoRdEnxhxtnyySCArcisFoYvm/cuqbYVUkZ1dJOD1sfPLJJ5EkiaVLlxIeHl6hCz1NJhNhYWF8+umnqFQqOnXqxKVLl3j//feZOnWqzXMmTpxIXFyc5XNmZmaFCliP5nKOr7Npco6vZuFVL1hQ4B1sOJrC5Yw8agZo6d820tPmuB2nxWv//v3s2bOH5s2bl+vGtWvXRqVSkZxsPROXnJxsyUNUlMjIyGLpPlq2bElSUhI6nQ6ttni1Xx8fH3x8POe0DPRRc3vTWmw8nsrqXRd4o3/LKrWyX+A9mMMjHr81Cl9N1c844vSw8dZbb+XChQvlvrFWq6VTp07Ex8dbtplMJuLj4+nWrZvNc26//XZOnTqFyWSybDtx4gSRkZE2hctbGNBOrsz82ZazjF2RQHquzsMWCaoaJ5Oz2Hb6KkqFHNtVHXC65/XZZ5/x3HPPcenSJVq3bm2Z9TPTtm1bh68VFxfH8OHD6dy5M126dGH+/Pnk5ORYZh+HDRtGvXr1mDVrFgDPP/88H330ES+//DIvvvgiJ0+e5J133uGll15y9jEqlAc71CMpM495f53gt4NX2Jt4nbmPtaN7k9qeNk1QRTAX1+gdE069Sl4VyFGcFq/U1FROnz5tERiQs3tKkoRCoShx5q8ogwYNIjU1lSlTppCUlET79u1Zt26dxYmfmJiIUnmzcxgVFcWff/7JK6+8Qtu2balXrx4vv/wy48ePd/YxKhSFQsGYnk25o2ltXl61j7NpOQz5bAfP3tWEuN63oFV7PFZYUInJzNPz/V45zXNVD48ojEJycgosJiaGli1b8vrrr9t02Dds6N1d1szMTEJCQsjIyCA4OLjC75+rMzDj1yOs3CkPvVvXC+Z/j3egSR1RHVpQNr7cepa3fjlC07BA1r9yV6X3qTr6G3W653X+/HnWrl0rCoWWEX+tmlkPtaXHLWFM+OEAhy5l0v/Df5hyXysGd4kq8YunM5h4YcVeJEliybDOlf5LKig/kiTxlTk8olvDavWdcHq8cvfdd7N//3532FKt6Nc6gj//exd3NK1Nnt7EpB8P8szXe7iWY9+ZP3f9cdYfSWbD0RSSMvMq0FqBt7L11FXOpOYQ6KPmoY71PW1OheJ0z2vAgAG88sorHDx4kDZt2hRz2N9///0uM66qEx7sy1dPdWHp1rO8t04Wpn0X/mbuo+2465Y6VsduP32VT/8+Y/mccUNPZEj1cMwK7LOsIDzi4Y71CPSpXvV0nPZ5FXagF7uYkw57T+Bpn5c9Dl/O4OVV+ziVkg3A03c04rW+zfHVqMjI1XPv//7mcsbN3taqZ27jtsZVp4yVwHkuXMulx/sbMUmwIa6Hy4scewpHf6NODxtNJpPdl7cLlzfTqm4Iv4y9w1KK/fMtZxm4cCsnkrOY/PMhLmfkEV3LnxYRcoS+yI0vWL4jEZMEdzStXWWEyxmcEi+9Xo9arebQoUPusqda46dVMWNgaz4f3plaAVqOJWXxn//9wy/7L6NSKvhgUHtLHciMXCFe1Z1/TqYC8FgVztlVEk6Jl0ajoUGDBqKH5WbuaRnOH/+9kx631MFQkAfs5Xua0aFBDUL9ZB9j+g0RpV/dMfe+o2pUT9+n08PGN954g0mTJnHt2jV32CMoICzIly9H3sr7j7Tltb7NGdOzCQCh/vIyKDFsFJi/A8F+Vaseo6M4PT3x0UcfcerUKerWrUvDhg0JCAiw2r93716XGVfdUSgUPNrZekgQYu55iWFjtcZkksjONwA3vxNezeEf4VQ89J8LatckSnBavAYOHOiSGwvKhkW8RM+rWpOVb8AcJxDk6+UhElcOwE9jQJ8L4a3htudcclmnn9pe3ixBxRDqL4tXphCvao35/++rUeKj9uL0NzlpsGqILFyNe8Gto1x26TJL9p49ezh69CgArVq1okOHDi4zSmAfs3iJYWP1xuLv8vXiIaNRD2uGQ0Yi1GwMj34BKtf1Ep2+UkpKCo8//jibNm0iNDQUkMuR9erVi1WrVlGnTp2SLyAoFyFitlGAnEkCvNzftW4CnN8C2iB4fCX4ubYAjdOzjS+++CJZWVkcPnyYa9euce3aNQ4dOkRmZqbX59WqCoT4ybONoudVvcm8ITvrvXamcfcXsOszQAEPL4GwFi6/hdM9r3Xr1rFhwwZatmxp2RYTE8PChQvp06ePS40TFMc8bMzKM2A0SaiU1SeLgOAm5p5XsDc6689vh99fk9/f/QY0v9cttynT8qCii7FBDmAtnJ5Z4B4KDxOE0776kumtMV7pF2D1k2DSQ8xAuHOc225VppQ4L7/8MpcvX7Zsu3TpEq+88gr33HOPS40TFEejUhKglWeXSgqXWLv/MvsvpFeQVYKKxixeXuXz0uXC6iGQmwbhbWDgx+DG/GJOi9dHH31EZmYm0dHRNGnShCZNmtCoUSMyMzNZsGCBO2wUFKG0KPtzaTm8tDKBV1bvq0CrBBVJZl6Bz8tbZhslCdaOhSv7wb8WDF4B2oDSzysHTg+Yo6Ki2Lt3Lxs2bODYsWOAXH6scOVrgXsJ8dNwKf2G3SpEyQWJCq+JKkVVlpvDRi/xeW2dD4e+B6UaHvsKQhu4/ZZlenKFQkHv3r3p3bu3q+0ROIB5qGCv52VeNqI3CB9kVcWr4rxO/Akbpsnv730Xou+okNuWSbzi4+OJj48nJSWlmJN+6dKlLjFMYB/zjKM98coqGFLojEK8qipeE+eVegK+HwVI0GkEdH66wm7ttHhNmzaN6dOn07lzZyIjI6t2wn+jHhRKUHrX8ovSouyzzD0vo4TJJKEU4RRVDq+I87qRDqsGQ34mNOgG977vVgd9UZwWr8WLF/Pll18ydOhQd9jjPeSkwbcjoF5H6D3d09ZYUVqgalbeze16kwkfLxNfQfnx+LDRZITvn4arpyC4Pjz2Nagrtmq907ONOp2O7t27u8MW7yJxO5z7B7b+Dw5862lrrCjV51UwbAS5XJqg6mEJUvWUwz5+GpzaAGo/eHw5BFb8skCnxWvUqFGsWLHCHbZ4Fy0HwB1x8vu1Y+GS9+Qpu+nzsj2bmCXEq0qTkpVHrs6IQgF1glyTG8spDqyRG3WABz6Cuu0r3gbKMGzMy8vj008/ZcOGDbRt27ZYtP28efNcZpzHuXsyJB+Gk3/KUcOjN0JQuKetupkK2s6w0TzbCMJpXxU5dCkDgCZ1AvHXVnDP69JeWPui/P6OV6DNIxV7/0I4/eQHDhygffv2AMUKcVQ5571SJS8q/SwW0k7AmqEw/BeXZYIsK6UlJLTyeRmcqmwnqAQcupQJQJt6IRV746xkuRE35EGzvnD3mxV7/yI4LV4bN250hx3ei2+InM5jyd1wYQf89ircv6BCZ1WKEuJgqASAThRLqXIcLOh5tapbgXVHDfly4515CWrfIjfqHp4IctrnVS2p3RQeXSqHTSR8XZDqw3NYlgfl6rFVM7iweOULn1eV43CBeFVYz0uS5Eb7wg7wKWjMfSu412cDIV6O0jQWYguiiP8YD2f/9pgpZp+Xzmjihr54z6qwz0tvlNh/IZ1tp9IqzD6B+7ianW+pnB5TUT2vnUvkRluhhEeWyo25FyDEyxm6vwhtB4FklNPbXj/nETP8tSrUBYGntoaOhX1eOoOJBxZu5YnPdnAl40aF2ShwD4cuy/6uxrUDCKqIGK+zf8sZUQFi34Jm3rOGWYiXMygUMOB/ULcD3LgGK5+A/GwPmKGwG2UvSZL1bGOhYePZtJyKMVDgNswzja0qYsh4/ZzcSEtGaPMYdPeuTMlOi9fff/+NwWAott1gMPD3354bSlUYGj8YtBwCwiDlMPz0PHggCaO9+o35BhN6400/mL5QqEROvnDeV0a2nkqj+6x4Nh5L4eBFs7/LzUPG/Gy5cb5xTW6s7//Qo5NUtnBavHr16mWzWnZGRga9evVyiVFeT0g9OapYpYWja+GfORVvgp0o+8LOerB22OfkF290BN7PTwmXuJyRx8bjKRy6LItXa3f2vEwmuVFOOSw30oOWy422l+G0eEmSZDOe6+rVq8WqZ1dporpA/4KA3I1vw9FfK/T2NxMSWkfZF/Z3gXWQapYQr0rJ8eQsANKy87l4XfZbtqrrRvH6Z47cKCs1MOgbubH2QhyO83rooYcA2d8yYsQIfHxuBmoajUYOHDhQPdY8FqbjUEg6CDs/gR+fhZrrITymQm5tL8o+u4hA5RWajRQ9r8qH0SRxokC89iWmA9Cwlr/7UuEc/VVujAHumwcNurrnPi7AYfEKCZGVXpIkgoKC8PO72Y3UarXcdtttjB492vUWejt934bUo/KszKrB8hIi/5puv22wnSh7MWysWiReyyVPL/8PzSESrd3V60o+IjfCAF2ehY7D3HMfF+GweH3xxRcAREdHM27cuOo1RCwJlQYeXQaf9pRnZ74dAU/+4NLKwLawl5CwqHgVnm3cciqNhrUCeKRTfafudSolm4vXc+nZPKyM1grKyvGkzGLbWkYGuf5GudfkxleXDdF3yo2yl+O0z2vq1KlCuIriXxMGrwRNAJzdDOvdv+bLPGzMyC0qXtafCw8bExLTGfftfnacuerUvWLnbWbEF7s4cDG9bMYKysyxpKxi2wJ9XNwwGg1yo3v9HIQ2lHPQq7wgvXQpOC1eycnJDB06lLp166JWq1GpVFavakt4K3hwsfz+348hYblbb2d22KcXcdgX9XnZWh5kdgA7i3lBsKDiOHal+P9KrXJxeOb6N+VGVxMgN8IV4PZwBU7/FUaMGMHevXt58803+e677/jhhx+sXmVh4cKFREdH4+vrS9euXdm5c6dD561atQqFQsHAgQPLdF+XE3M/9CiIRv71v3Bhl9tu5WiohK18XgZj2TJNGERR4QrHVkOjdaV4JSyXG1uQG9/wVq67tptxuv+5ZcsW/vnnH0tanPKyevVq4uLiWLx4MV27dmX+/Pn07duX48ePExZm38dy7tw5xo0bx5133ukSO1xGj/GQfAiO/SoX4HxmEwTXdfltQuxE2Jc022jGZGMxtyPoyyh6grJxQ2fk3NXiqyLUKhcFi17YJTeyIDe6Mfe75roVhNMSHhUVZTOTQVmZN28eo0ePZuTIkcTExLB48WL8/f1LrEJkNBoZMmQI06ZNo3Hjxi6zxSUolXILFhYD2cmwagjo81x+G0d9XraGjZl5ZZt11IvEhhXKyZQsbP3UXDJszLwsN65GHbS4T250KxlO/xXmz5/PhAkTOHfuXLlvrtPp2LNnj1XBWqVSSWxsLNu3b7d73vTp0wkLC+Ppp0svs5Sfn09mZqbVy+34BMHjK8CvBlzeC7+8jM1vYTkwDxuz8g0YCgeiFghTkK/cqc43FO95Xc8pWzFagxCvCsXsrC8aE64tb89Lnyc3qtnJciP74GK50a1kOG3xoEGD2LRpE02aNCEoKIiaNWtavZwhLS0No9FIeLh1auXw8HCSkpJsnrNlyxY+//xzlixZ4tA9Zs2aRUhIiOUVFRXllI1lpmYjePRLUKjgwCrYvtClly8cpFi4J2UWr1oBskPfVs/rehkraYthY8VidtY3qm09u68uj9BIktyYXt4rN66Pr5Ab20qI0z6v+fPnu8EMx8jKymLo0KEsWbKE2rVrO3TOxIkTiYuLs3zOzMysOAFr3BP6vgPrxsszOmEtoek9Lrm0WqUkyEdNVr6B9FwdNQvEyuzzqhmg5dzVXPL1xcXLXu770hAO+4rleLI8SmhdN4QzqTd9X+XyeW1fKDemCpXcuNZsVE4rPYfT4jV8+HCX3bx27dqoVCqSk5OtticnJxMREVHs+NOnT3Pu3DkGDBhg2Wau2K1Wqzl+/DhNmjSxOsfHx8dqKVOF0/VZSD4ICd/AdyPlCPxaTUo/zwFC/DVk5RusZhzNPq+aAfIz2xw2lrHnVdZZSkHZOF4wbGxTL4S1+y9btpd5tvFU/M0YxL7vyI1rJaZMf4XTp08zefJkBg8eTEpKCgB//PEHhw8fduo6Wq2WTp06ER8fb9lmMpmIj4+nW7duxY5v0aIFBw8eZN++fZbX/fffT69evdi3b1/F9aicQaGQF3DXvxXyMmDlYMhzjd/NViEOc83G2oFyT+zoleL3KqvPSwwbK4607HzSsnUoFNAy0jr9TZkc9ldPy42nZIL2T8qNaiXH6b/C5s2badOmDTt27OCHH34gO1tOxrd//36mTp3qtAFxcXEsWbKEZcuWcfToUZ5//nlycnIYOXIkAMOGDWPixIkA+Pr60rp1a6tXaGgoQUFBtG7dGq22Yiv2OozaR16dH1QX0o7DD8+4JAeYZYlQbuGe181hI0BadnGhui6GjV6PudfVoGbxRdhODxvzMgsazQy5Eb1vntfl5ioLTovXhAkTmDlzJuvXr7cSi7vvvpt///3XaQMGDRrEnDlzmDJlCu3bt2ffvn2sW7fO4sRPTEzkypUrTl/X6wiKgMe/AZUPnPjj5sr9chDqVxBlXzAMNJkksnXW4mWLG3qjzfiv0tAbTSQkXmdv4vUyWCtwBvNMY4uIoGJi5dSw0WSSG8u04xAUKTeiHi7d5yqc9nkdPHjQZsXssLAw0tLKVuRh7NixjB071ua+TZs2lXjul19+WaZ7eoR6neSyaT8+I+dMCm8FrR8q8+WCLVH2smDl6AyWiIxagSX3Qq/n6ogMcS7BXHa+kQc/3gbA0en98NNW4+VgbuZYwXC/eUQwmiLi5VTPa+PbcmOp8pETaAYV9yVXVpzueYWGhtrsCSUkJFCvnncmLfMq2g2SC3kA/PwCXDlQ5ktZ8tgXrG80zzSqlQqCfEpeWHs9x/mhY3ohR7+tqkUC12FeFtQiIghNkZ6Ww6ESh364meX3/g/lxrMK4bR4Pf7444wfP56kpCQUCgUmk4mtW7cybtw4hg3z7vw/XkPsNGhyN+hzYdUTkFO2HmvRKPvCAapadcn/2vQyzDje0N0ULJWy8vtMvJXCCQibRwQVc9A7NGy8ckBuHAG6jYV2j7vaTI/jtHi98847tGjRgqioKLKzs4mJieGuu+6ie/fuTJ482R02Vj2UKrn+Xc0mkHEB1gwDo/M9oaI5vcziFWhHvP78713cGl0DgGtlEK+8QmEXVcDf67WYExD6qJVE1wpAo3Ry2JiTVrAsLVduJHtPd6O1nsNp8dJqtSxZsoTTp0/z66+/8s0333Ds2DG+/vrr6p0Sx1n8asjpR7RBcH6rXMjWSYqGSphjvIJ8NDbFq3lEEDUKUumUZcYxr1DAqyQmHt2GOQFhs/BAVEpF8WFjSeJl1MuNYUYi1GwsN5LKqvm7LHNWswYNGtCgQQNX2lL9qNMcHv4MVj4Ouz+HiNbQ+SmHTw8pMtto9nkF+qrtDi3M4pVehlivwsNGCRHz5S5uzjTK8V1OzTb+MV5uDLVBMHiV3EhWURwSr7i4OGbMmEFAQIDVUhtbzJs3zyWGVRua94N73oT46fD7a1CnBTR0rJDJzWGjLFrmYWNwCT6v0AD5nLIMGwuvk3TxOnNBIcxrGltEyGsOi/e87IjX7qVyI4gCHl4iN45VGIfEKyEhAb1eb3lvD1sl0QQOcEccJB2Cwz/A6qFyDrDQ0lcL3ExIqJMrZZt9Xj72e141zT2vMgwb8/WFe14Cd3G8kLMebIiXrcmS89vkxg/g7snQ/F632ugNOCReGzdutPle4CIUCnjgI7h6Ui6ltuoJeOpP0PqXeJq556U3SuTqjDd9Xr62fV5AIZ9XGYaNhcVLdL3cQuEEhGbxKjqzW1TMSL8gN3omA7R6CO58tUJs9TSVL4lPVUUbAI+vBP/akFQwzV2KQPhpVJYeVsYNvaWobKCvuvgXvACz4JXFYW8w3bTHJLTLLZgTENYM0FIn0HYkvJWY6QrCbXLTIKKN3AhWkxGQQz0vc8FZRyhrHnsB8lBx0NewbIA8hIxoA3fa9zEqFApC/DWkZuWTnqt3KM7LvGyorIuzzQiHvXswO+ubhweV7oaRJLmRSzogN3qPr5AbwWqCQz2vwsn8goODiY+PZ/fu3Zb9e/bsIT4+3lKYVlAOGnaH/7wvv4+fDsfXlXj4zXAJncXnFeSjxsfesLFAvK6VU7yEdrkH84LsFo7UZtzygdzIKdVyubLQ6jX771DPy1xwFmD8+PE89thjLF682BLXZTQaGTNmDMHBwfYuIXCGzk/Jvq/dS+H7UTA63u7MUeEo+6z8mz4ve8PGOkHyUCQ738ANnbHM6xOFdrmHYwUxXuaZRruc+FNu3ADufQ+ib3ezZd6H0z6vpUuXMm7cOKuAVJVKRVxcXIlFMwRO0u9daNAddFlyOpMb6TYPKxxlX3i2UaVU2FzCU7hXlpqVX2bzhL/ePZh7Xs0jSugIpB6XGzUkuaG7tfRaDlURp8XLYDBw7NixYtuPHTtmyWoqcAFqrTwUCImCa6fh+6fBVHwxdHChKPuixTdUNnwmCoXC0vtKzS57VSPh83I9hRMQ3hIeaPOYYHLkxiw/U27c+r1bwVZ6D05H2I8cOZKnn36a06dP06VLFwB27NjB7NmzLQkEBS4isI7shP28D5zaABvegj4zrA65mdPLerYR7NdnDAvy4eL1G+XqeYnZRtdTOAGhv7b4T1OJiQ81H8mNWUiU3LipvTQBZwXgtHjNmTOHiIgI5s6da0mNExkZyWuvvcarr1aP+JIKJbItDPxYTuG77UMIby2n1Smg8LDRHOcV7Ctvs6cv5p5XSrmGjUK9XE3hmUZbvK5eRU/VflD7ybm5AutUpHleh9PipVQqef3113n99dctNRCFo97NtH5IrsL9z1xY+yLUbgb1OgI3ZxvTsvMtC6cDfUrueVmGjcLn5VWYF2S3iCz+e3pAuYXn1L/KHwYuhMh2FWmaV1KuINXg4GAhXBVFr8lwSz8w5svpTrLkikvmntel6zcsh5qHjfYEJizIFyifeAlcT+HUz1Zc2su7GrlO6UeGB6D1wxVtmldSpqwS3333HWvWrCExMRGdzjpeaO/evS4xTFAEpRIeWgKfxcr5yFc/CSN+tfS8Ll7PBcBXo7QbJmHGNcPGMp8qsEHRBIQWspJh1RB8FXo2GDsw1/AothOmVz+c7nl9+OGHjBw5kvDwcBISEujSpQu1atXizJkz3Htv1V8M6lF8g+UcYL4hcHEn/PaqJc4r0zLTWHL6Z8Cy7KR8DnuhXq6kaAJCAAz5sGYoZF3mlKkur+hfQBIr+iw4/Zf4+OOP+fTTT1mwYAFarZbXX3+d9evX89JLL5GRkeEOGwWFqdVETjCnUELC10Sd/Npqd5BP6Z3psGAX+LzKfKbAFkUTECJJ8NurcGEH+IYwSv8qWZS8UL+64bR4JSYm0r27nG/Kz8+PrCy5qzt06FBWrlzpWusEtmkaa0ntW3PLW3RT3iz2a47xKgnzsDEtOx9TGWMexGyja7k501jgQ965BBK+lhupR5ZyTor0oHXeidPiFRERwbVr1wA5m6q5VuPZs2fFF7oi6TYW2g5CIRn5WPM/6ivkyuWBDohXrQBZvAwmqUypcUD0vFyNOcarZWQQnNkM6ybIO3pPlxsrQTGcFq+7776btWvXAnLA6iuvvELv3r0ZNGgQDz74oMsNFNhBoYAB/4O6HamhyGaJZi7+5JVa8gxAq1ZaskukZpdt6CjaKddi7nm1DUyHb4eDZIS2g+RGSmATp2cbP/30U8syoBdeeIFatWqxbds27r//fp599lmXGygoAY0crHh1XndaKi8wT7OIeJ/3HTq1TqAP13J0pGTm06IMdUhFL9t1mBMQ+pNHx61j4MZ1qNtBbpyqSW6usuBUz8tgMDBz5kySkpIs2x5//HE+/PBDXnzxRbTa6rtUwWME12VW8BvkS2r6qXbxn+vfOHRaeQNVhXS5jpMpWSCZWOD7Ceq0oxAYLi8L0zhX0by64ZR4qdVq3nvvPQwGg7vsEZSB5JC2TDbIVYd6XfkMjv5S6jlhlsXZrhk2SpKE3igW5peFY0lZvKj6iXvYASotDPoGgut62iyvx2mf1z333MPmzZvdYYugjAT7afjW2JMvDH3lDT88C8mHSzzHEqiaWbp42ar3UDSrxOivdtN+2l+W6t3Vies5OpbvOG8p/us0R38lTvOd/P6+DyCqi+uMq8I47fO69957mTBhAgcPHqRTp04EBFinnb3//vtdZpzAMcyBqjMNT3JveDoRV3fAysGEMol0bC/yreNEz0utUqIzWPeqiva8NhyVZzvXHb7CoFurV0bPUV/tZs/56/zf0RQ+H3GrcycnH2HAmWkAnIgewi0dnnSDhVUTp8VrzJgxgO36jAqFAqOxeM4pgXsxr280omL/bR8Qse0JuH6Or4I+5sGscRgpni31ps+r9JxeGqWCogEV9vz1Cqqfg3nP+esAxB9Lce7E3Guw8nH8pBtsNbYioNc0u4f2uKUOm0+k0q1xrfKYWqVwethoMpnsvoRweQbz+kYAn+A6chUiTQBt9ft5Q73c5jnOrG+0lZFVLA8qJ0YDfDsC0s+TaKrDWMNL3BJpv7r1h493YObA1nw8pGPF2ejliIVSVQBzQkIoWNsYHgMPfQLAU+p1PKraVOwcZzJL2BIvQTn5azKc3YxR7c9o/asE1wy3mYDQTIi/hidva2gpoCJwYth448YN4uPjue+++wCYOHEi+fk3v/gqlYoZM2bg6+vreisFJRLif7PnZVke1HIA9JwIm2YxU72U06a6QH/LceaeV1aegTy9EV+Nc4U4RMerHCR8AzsWAbAxZibHd0bQx04CQoF9HO55LVu2jE8++cTy+aOPPmLbtm0kJCSQkJDAN998w6JFi9xipKBkQv1siBfAXa9zKLgHPgoDi7XzIfOyZVdwodqOZYn1Ejnsy8iFnfDrK/L7nhP5y9QZsJ2AUFAyDovX8uXLeeaZZ6y2rVixgo0bN7Jx40bef/991qxZ43IDBaVTuOcVWDirhFLJmvqTOGqKIkyRLldW1stJCxUKhSXWqyx5vUTPqwxkXpbzsBl10OI+uOt1+wkIBaXisHidOnWKNm3aWD77+vqiVN48vUuXLhw5csS11gkcwrzQWqtSElDEb5KvlH0q16VAuJwAv7xsUR5Ho+xt6ZQ9h73okdlBf0POgJudDGEx8OAnGFHYTkAocAiHfV7p6elWPq7U1FSr/SaTyWq/oOKoE+TD5P4tCfHToLThXL8ohTFG/zIrfWbDgdUQ0Qa6v1goKaHzJdCERDmBJMmNxuW94FdDXvrjE0hiWk7xBIQCh3G451W/fn0OHTpkd/+BAweoX7++S4wSOM+oOxvzaOcou/u3m1pBv1nyh/VT4NSGciUltDdsrOrDSUmSyMxzMpJ++0K50VCo4NFlULMRYCMBocApHBav//znP0yZMoW8vOKt9I0bN5g2bRr9+/e3cabAk1gN47o8Ax2GgmSC756imUoOqizb+kZ7w8aqzczfjtL2rb/YcjLNsRNObYD1b8rv+82Cxj0su4olIBQ4hcPiNWnSJK5du0bz5s15//33+fnnn/n555957733aN68OdevX2fSpEnutFVQXhQK6D8XorpCXgYDj71KILkOrW8sSnXteX2+5SwA764rXjW+GFdPw3dPyY1FhyflxqMQVgkIBU7jsHiFh4ezbds2WrZsyYQJE3jwwQd58MEHmThxIjExMWzZsoXw8PAyGbFw4UKio6Px9fWla9eu7Ny50+6xS5Ys4c4776RGjRrUqFGD2NjYEo8XFEHtA499DUF1Cck5y3zNQtKybpR+XhHsZY+uLg57jaqUYV5eJqwcDHkZUL8L9J9XLDeXpeclnPVlwqkI+0aNGrFu3TpSU1P5999/+ffff0lNTWXdunU0bty4TAasXr2auLg4pk6dyt69e2nXrh19+/YlJcX2OrFNmzYxePBgNm7cyPbt24mKiqJPnz5cunSpTPevlgSFw+PLMal8iFUl8GD6l8UWXhfGVm/KXjLCqt7zMqMuqbycyQQ/jJZL1AXVlVPcqH2sDjEnIAQhXmWlTMuDatasSZcuXejSpQs1a9YslwHz5s1j9OjRjBw5kpiYGBYvXoy/vz9Lly61efzy5csZM2YM7du3p0WLFnz22WeYTCbi4+PLZUdV5aV7mhHip+H5nk2sd9TrSEZveXH9COP3vPbWVDY6sbC4mmiUXbQ2xMvSsdr4NpxYB2pfeHy53FgU4WRKFpIENQO0lllfgXN4dG2jTqdjz549xMbeLDCgVCqJjY1l+/btDl0jNzcXvV5vV0Tz8/PJzMy0elUn6tfwJ+HN3ozv16LYvoDOT7DYIC/3mq1azLIf1jp8Xbs+rzJZWflQKhVcTr/BjF9vxjYqAA79AP/MkTcM+BDq2V5IfdNZH4RCpHouEx4Vr7S0NIxGYzFfWXh4uFWq6ZIYP348devWtRLAwsyaNYuQkBDLKyrKfjhBVcVW7BfIhTiWaJ5kk7Edfgod7xtmQ3Yql9NvcORyySJv17dVTcaNkiQxatluiwMfIEZ5Hn6SU0bR/UVoN8ju+ceFv6vcVOqsErNnz2bVqlX8+OOPdheET5w4kYyMDMvrwoULFWyld1MzyJ+X9GM5Y4qgjikV1gyjx+y/+M+H/3A53b4jv7r3vACOXLkp8DXJ5BP1XDDckEuVxdrPzQViptEVeFS8ateujUqlIjk52Wp7cnIyEREll7SZM2cOs2fP5q+//qJt27Z2j/Px8SE4ONjqJbiJSZLIJIDR+lfJVwVA4jamqpcBcDo12+551TVUwkzh5VEaDCzSzqeeIg1qNoGHPwNlyVk6jhUEqDaPEN/HsuJR8dJqtXTq1MnK2W52vnfr1s3uee+99x4zZsxg3bp1dO7cuSJMrbLk6uQEkqelevzebAYSCp5UxzNEtQFVgS/G1syi0e5sY9VRL0mSyNPbTrBZ+DGnqL+iq/IY2ZIfDF4pLwEqgbTsfNKydSgUcEt4oCtNrlZ4fNgYFxfHkiVLWLZsGUePHuX5558nJyeHkSNHAjBs2DAmTpxoOf7dd9/lzTffZOnSpURHR5OUlERSUhLZ2fZ7CQL75OTfrAR1tuYdXOgwDoC31MtQXbA/aZKdV/UrSL26Zj9t3vqTKxnFh89m8XpCFc9Q9QZMkoI441io07zU65qHjA1q+peYgFBQMh4Xr0GDBjFnzhymTJlC+/bt2bdvH+vWrbM48RMTE7ly5Yrl+EWLFqHT6XjkkUeIjIy0vObMmeOpR6jUvPvwzSG3BCQ0GMEvxtvQKIw02zwG0hNtnmdvfV/V6XfBDwmX0Bslxq5IIDvfWqwlJG5VHGOa+ksA5hgeYxOdHLpu4ZlGQdnxCtkfO3YsY8faLmu+adMmq8/nzp1zv0HViHvbRNK/bSS/HbiCJMGVzHzm65+lkSKJ1spz6JcPxpfxZBYp4pFpp8xXFRo1Wthz/jqzfj/K9AdaW7YlnjnBWp/5aBRGfjHexsfG+/Fx8NdkXpAtcniVD4/3vASexxwkKSGRlJFHHj48o4sjTQpGk3qIaSyiaJ8qqxoMGwtzMiXbUlTXl3w+1c6jtiKTQ6ZoXtM/C05UTTIPG0X21PIhxEtgQZIgKUPOGnKZ2jyv+y961NzLNsaorANYq8OwsTCpWfkYTBIg8b7mE1orz5EmBfOMLo48ZPF3JNbUaJI4LhIQugQhXgKrH92VzJspj3ZJLZiiHw7AOPUa7lHuseyzN2ysqqRm5WMwmhijWssA1b/oJRXP6/7LZWpbjnGkZmXitVyRgNBFCPESWH50EpBUZGZtpfEevjbEolRIzNd8TBOFvAA+086wsaRQiex8A30/+Jv3/3QgnYyHWbTptNXn7HwD+qN/ME4t12mYahjBLqn4kqvSEAkIXYcQL4Gl52UwmmxmVZ1uGMYOUwuCFDdYoplLMNlWPa/CgjXzt6Nk2OmVrdqZyPHkLBZuPG1zv7dw9EpmsXxdTRSXqLluDEqFxNeGWFYY7yl2niPDRpGA0HUI8RJYBjs/7btcLE9XkzoB6FHzvO6/XJRq01iZxALNR2TfuDm8LNrZWhB/0uZ99MbK4RErWk0pmGyWaOai0mezw9SC6YZhNs9zpB91XFQLchlCvAQWzL2uEd2jCfXX8MxdjWlXPxSAa8jO6VzJhx6qAzyZ/aXlvKKSdC1XZ/P6lSVRYeHcZkpMLNB8RGNlElk+EYzRvYzeToSRI9khbs40CvEqL0K8BFbDnfBgHybc24I9k3sz6T8tLYVpAY5I0UwwPQ/Ak6afYf8qoLifS6OsHF+rrDw93++5WGyYW1i8XlevoofqADckLcsazuIqIXavV5p03dAZOSsSELqMyvEtE7iVG4XW7714dzN8NSqLM7mweAHs9r+LBYaBAEhrX4JLe4r1p9Q2UiRfuJbL93suutTu8vLatwd49dv9vLgywWq7ecnUQOUWnlP/CsA4/XPs1ZWSTqkU9RIJCF2LEC8Bv+y/ufzqsSLl0zRFMobWCNAyz/AI640dURjzYdUQtiQcKvEcgDvf28jp1BwXWl02ft53iVHLdpOdb2DdYTln3N8nrGuQZubpaaM4w7uaJQAsMAzkN9NtJaYIcgSRgNC1CPESWA2biva0in72USvx12p4RT8GXY1mkHWFWr+NQsvNa5RanMKDvLxqHxuOJrNw4ym7xxgzrvCpdh4+Cj3rjR2ZZ3gEoFTxKu2pRQJC1yLES4C6hHijornatWolwX4asvHnTOwS8A2hrXScmeqlmF33JRan8BLS7BXaNeTzn6OvE6m4xklTPV7Rj0Eq+JnYi20zU1pvSsw0uhbv/5YJ3M7yUV1pXCeA5aO6FttXuOcVXcufobdFW4aFuUHR8MgXGCUFj6k3M0L1JwCaShB8aTMfmSTBb68SlXOIDMmf0fo4svF3+JqljQTNCQjFmkbXIMRLQNfGtfi/V3tye9PaxfYV7nl9ObIL/dtGWj5LEtD0Ht4zPQnAZPU3dFceqhQ9r39PXy2+ceenkPA1JpSM1b/EOSmy+DElUJJ2iQSErsf7v2UCj1LU5wWFexgSRpPEJ/p+fG+8A7XCxELNh9TUeX8NzcsZeVafdSc3wjo56eXXQU/zj8l+avGyIBIQuh4hXoISKTxzaBYts3Z9GH+KTjPXAwom6Uexz9SYGops/nPoVcjPqnBby0qUIpnc5U+CZIS2j7NSOaDYMXWCSg9tKMnnJRIQuh4hXoISsdXzMrP5RCrpufIsYz5antXFkSyFUjPnFPz4nFw52svxJ48lmnmEkg11O8KA+TYd82GOiFcJ+86myWnKm4kho8sQ4iUoEdvDRts/02Rq8pzuFYwKDRz7FTa/W+K195y/TvSE3xi1bJdLbHUWBSbmaRbRQnmBFClUrm6t8Su7eNlRrz3nr/HrATmWrn4NxycABCUjxEtQIlobMVvmLbZSuiRIzdjT5k35w+bZcMR+Fe6HF20DYMPRFKtCIBXFS6of6afaRb6k5lndKxBcF73RVCxfPUBYkO26oNYU/3usP5LME0t2kJ6rp31UKAPa1XWB5QIQ4iUohZKGjWYWDO5Aj1vqWD6fb/AQdJXXQPLjc5B82F3mOcW1nJsLxvsqd/GK5nsAJhueIkFqBhTPKGHGEZ9XUVbsSOTZr3eTbzBxT4swVozuSqCjie4FpSLES1AiWpWN4qkFHQzzgmyFwroIqwTQZyaGhneBPofrnz9CDTKLX6cCMJok9l9Ix2A0Me0XWUSbKxKZp/kYgKWGfnxr7CnbLUnFkjGaCQt2fNgoSRIfrD/BpB8PYpJgUOcoPhnaScwyuhghXoISKbzUx5xx1bylcJinsXAiMAlQqfk2ejrnTWHU0F1moeZD1FT80HDN7gs8sHArTyzZwc/7LhNKFks0cwlQ5LPF2Iq3DUMsxz6yeDvn0nJtXseRhdQK5ISOk348yP8Kcpq9dHdTZj/cplLEvlU2xF9UUCIlDRsLB6kbTIV7XvL7XE0oo/TjyJZ86a46whvq5W6z0x7Ltp0DYOe5a6gwslDzIQ2UqZw3hTFW/xLGQiXd9py/zsebbK95DPHTlHqvPL2R577Zw8qdF1AqYObA1sT1aS4WYbsJIV6CEnF0trFwz8ssamqlgpNSfeL0sv9rpPpPHlNtdI+hdiic1nqy+htuVx0mW/JltP5V0ikec3X+qu2el4/GxvC5CJl5BjYcTcFHrWTRk5148raGZTdcUCpCvAQlUnRhtj0MRfNHF9r2l+lW5unlzAwz1UvpqDhhN8+9q1EWzIg+qtrESLW89jJO/zwnJNu5uWw9B4CfA+IFcg9t+aiu9G0V4byxAqcQ4iUoEZs9LxvHGQsFpJp//lmFajsuMA7kd2MXtAojn2g/YOHPm63O/7JgeOdq1EoFHRUnCrJewDz9I/xlurXU84rOCvpqHPupfPtcNzpH13TeUIHTCPESlEjJaxtvYjAWHzYWrqotoWSc/jmOmhpQR5HBwyfG48PN0IX3/zxOcmYev+y/jMHousj8MK6xWDsfH4WB341dWGAc6NB5UTWtg0l9Heh5LXuqC7eI5T8VhhAvQYkUXttYUgENWw77rCJVtXPxZbQ+jmtSIM1Np5itWULhOcvYeZt5cWWCy3ph/x67yPQb7xCmSOeoKYpx+ucsublK49FO9a0+OyJe7QuKlQgqBiFeghIp3PMyly5T2yiwYcthn2Vjmc1FKYwX9C9jkJQ8qNrKaNVvln3m4+OPppTfcEni8vJnaac8wzUpkNH6V8nFkSh5eeXA8O7RtK53M++WIz6vYD8Rx1WRCPESlEhhh725qk6XRsV9OgabPi/bcV3bTa2YbhgKwAT1Snoo91vtzy1UEKQsSJJE4m/v8ZBqCwZJyQv6l7kohTl8fniQDyqlgj4xN53uPg6sNBAhERWLEC9BiViJV4EvamCHesWOMxYuKFvQ9crMsz+j+JWxDysNvVApJBZoFtBIcbMIyP4L6eWyed1Py6m3axYA0w1D2W5q5dT5DWsFAHB3C1nwQvw0lllLgfcg+rmCEin8o9UXiFe7+iE0qh3A2bSb1YD0NkIMDlzMKOHKCqYaRtBMeYnOyhMs0czlQd10spxIu1wYg9HEqK92c3uNdAbtex2VQmKVoSdfGfs4fa12UaEAtK4XQvyrPQgPLn24+fGQjk7fR1A+RM9L4DD1Qv0AeXg0sL1176uwz+uG3sjyHedLvZ4ODc/pXuGyVJOmysvM1yxESdlmGuOPpbD7+Hl67X2ZYEUOu023MMUwktJr+hSnW5NalvdN6gSWupi6fVQo/2njXMpoQfkR4iUolfWv3MW3z3WjboF4ATzQ3jq1S+HwhtSsfGb9fsyha6cRwrO6OPIkDfeoEnhVvaZMNubpdMzXLKSp8jKXpZo8r/svOkpf0mOLO2zk8rfHK7G38PnwzmW6j6B8CPESlEqz8CBuLRJ4GV07wOK4rx3oY9XzOpaUZTMnlj0OSo15Xf8MAC+o13Kfcjtz/zrulI0tjiwgVpVAnqThWV0cqYQ6db4ZH7XSZp4ye7wc24xaovq1RxDiJSgzHw/pyOfDO9O1UU2rOK9/TqY5fa21pttZbJBzx7+v+YT/27ie5My8Us6Sh6tjJ02m+clPARivH81BqbHT9zdTVKQLE1XTz+rzY53r2zlSUBEIh72gzNQO9OGeluEAPNq5Pt/8mwjIM5S6MkTJv2cYRHNFIr1U+/lUO49vN3dk7IDudo9fs+sCy35Yy3faTwBYbLiPn013lOFJblJSMOqPY25n97lrdGtcm+1nrlolYBRUPApJslV9s+qSmZlJSEgIGRkZBAeL4p+uIk9vZNPxFG5vWpsgXw1p2fl0nrmBRrUDeLRzfd5b59gwMJgcftROoYnyCjtMLZgT/i7fvtCz2HG/7L/MWys38bPPm9RXpLHJ2I6n9K9hKudgYsv4XiLPvIdx9Dcqho0Cl+CrUdGvdSRBvrKTvHagD/um9ObP/97FmJ5NHb5OJgE8o48jU/Kjq/IYA698aPO4uJW7+Fj7P+or0jhtiuQl/dgyC9eCwR1oHxXKHy/fKYSrEiHES+A2Qv21luVF8x5r5/B5p6V6shhJCoao47mx7VOr/SlZeUxVL6Or8hiZkp8sdgSU2c5uTWrx0wu30zJS9MQrE14hXgsXLiQ6OhpfX1+6du3Kzp07Szz+22+/pUWLFvj6+tKmTRt+//33CrJUUFYe7FCPT4d24p/XezGka4NSj99k6sB7hkEAqP+cwMkdf2AqmBT43+yJPKmOxyQpeFk/ltNS8Yj/kpj+QCtUSgVNwwIZ0T2a2mK2sFLicZ/X6tWrGTZsGIsXL6Zr167Mnz+fb7/9luPHjxMWVnw92rZt27jrrruYNWsW9913HytWrODdd99l7969tG7dutT7CZ+XdzDks3/ZeupqKUdJ/E+zkAdU27gqBfF9p294tJlE4KqH0SiMvKt/nEXG+5267/fPd6NTw5oYjCaRV95LcfQ36nHx6tq1K7feeisfffQRACaTiaioKF588UUmTJhQ7PhBgwaRk5PDr7/+atl222230b59exYvXlzq/YR4eQd5eiMdZ6ynTb0QruXoyNUZuZRevHKPL/l8q51GG+U50qUAtBjwV+Sz1tiNl/RjcSSC/o6mtflmVFc3PIXAHTj6G/VoqIROp2PPnj1MnDjRsk2pVBIbG8v27dttnrN9+3bi4uKstvXt25effvrJ5vH5+fnk59/MY56Z6ZkSXAJrfDUqjkzvZ7Vt84lUhi+1dhnk4cOzujh+9plMHYX8vztoii4IarUtXCqlgkVDOtJHpGKu0nhUvNLS0jAajYSHh1ttDw8P59gx28tLkpKSbB6flJRk8/hZs2Yxbdo01xgscCs9bqnDudn9kSSJr7afZ+pauc7iZWrzvO6/LNfOIoMAeTkRN/1U97ery9zH2mEwSmidjJAXVF6qfJDqxIkTrXpqmZmZREXZLr4g8A4UCjkZ4PDu0ZZtN3T92JRwD6HBIfx9S0Ob/ioHa2QIqggeFa/atWujUqlITk622p6cnExEhO0uf0REhFPH+/j44OMjZpMqO35aFX27tve0GQIvwqPTLVqtlk6dOhEfH2/ZZjKZiI+Pp1u3bjbP6datm9XxAOvXr7d7vEAgqJp4fNgYFxfH8OHD6dy5M126dGH+/Pnk5OQwcuRIAIYNG0a9evWYNUvOjPnyyy/To0cP5s6dS//+/Vm1ahW7d+/m008/Lek2AoGgiuFx8Ro0aBCpqalMmTKFpKQk2rdvz7p16yxO+cTERJSFCj50796dFStWMHnyZCZNmkSzZs346aefHIrxEggEVQePx3lVNCLOSyDwbsTCbIFAUKUR4iUQCColQrwEAkGlxOMO+4rG7OITy4QEAu/E/NsszR1f7cQrKysLQETZCwReTlZWFiEhIXb3V7vZRpPJxOXLlwkKCrJbnt28hOjChQuVfkZSPIt3Ip7FPpIkkZWVRd26da3CpIpS7XpeSqWS+vUdq/oSHBxc6b9YZsSzeCfiWWxTUo/LjHDYCwSCSokQL4FAUCkR4mUDHx8fpk6dWiWyUYhn8U7Es5SfauewFwgEVQPR8xIIBJUSIV4CgaBSIsRLIBBUSoR4CQSCSokQrwKuXbvGkCFDCA4OJjQ0lKeffprs7OwSj3/xxRdp3rw5fn5+NGjQgJdeeomMjIwKtNq+bc48C8Cnn35Kz549CQ4ORqFQkJ6eXjHGFqEqVU935lkOHz7Mww8/THR0NAqFgvnz51ecoaXgzHMsWbKEO++8kxo1alCjRg1iY2NL/R+WGUkgSZIk9evXT2rXrp3077//Sv/884/UtGlTafDgwXaPP3jwoPTQQw9Ja9eulU6dOiXFx8dLzZo1kx5++OEKtNo2zj6LJEnSBx98IM2aNUuaNWuWBEjXr1+vGGMLsWrVKkmr1UpLly6VDh8+LI0ePVoKDQ2VkpOTbR6/detWSaVSSe+995505MgRafLkyZJGo5EOHjxYwZYXx9ln2blzpzRu3Dhp5cqVUkREhPTBBx9UrMF2cPY5nnjiCWnhwoVSQkKCdPToUWnEiBFSSEiIdPHiRZfbJsRLkqQjR45IgLRr1y7Ltj/++ENSKBTSpUuXHL7OmjVrJK1WK+n1eneY6RDlfZaNGzd6TLy6dOkivfDCC5bPRqNRqlu3rjRr1iybxz/22GNS//79rbZ17dpVevbZZ91qpyM4+yyFadiwodeIV3meQ5IkyWAwSEFBQdKyZctcbpsYNiJX4Q4NDaVz586WbbGxsSiVSnbs2OHwdcxpa9Vqzy0ZddWzVDTm6umxsbGWbY5UTy98PMjV0+0dX1GU5Vm8EVc8R25uLnq9npo1a7rcPiFeyFW4w8LCrLap1Wpq1qxptxJ3UdLS0pgxYwbPPPOMO0x0GFc8iycoqXq6PbudrZ5eUZTlWbwRVzzH+PHjqVu3brFGxhVUafGaMGECCoWixNexY8fKfZ/MzEz69+9PTEwMb731VvkNt0FFPYtA4Cpmz57NqlWr+PHHH/H19XX59at0SpxXX32VESNGlHhM48aNiYiIICUlxWq7wWDg2rVrditxm8nKyqJfv34EBQXx448/otFoymu2TSriWTxJRVRPryjK8izeSHmeY86cOcyePZsNGzbQtm1b9xjoci9aJcTs5N69e7dl259//lmqkzsjI0O67bbbpB49ekg5OTkVYWqplPVZzHjaYT927FjLZ6PRKNWrV69Eh/19991nta1bt25e47B35lkK420Oe2ef491335WCg4Ol7du3u9U2IV4F9OvXT+rQoYO0Y8cOacuWLVKzZs2swgsuXrwoNW/eXNqxY4ckSbJwde3aVWrTpo106tQp6cqVK5aXwWDw1GNIkuT8s0iSJF25ckVKSEiQlixZIgHS33//LSUkJEhXr16tMLtXrVol+fj4SF9++aV05MgR6ZlnnpFCQ0OlpKQkSZIkaejQodKECRMsx2/dulVSq9XSnDlzpKNHj0pTp071qlAJZ54lPz9fSkhIkBISEqTIyEhp3LhxUkJCgnTy5ElPPYIkSc4/x+zZsyWtVit99913Vr+JrKwsl9smxKuAq1evSoMHD5YCAwOl4OBgaeTIkVZ/8LNnz0qAtHHjRkmSbvZQbL3Onj3rmYcowNlnkSRJmjp1qs1n+eKLLyrU9gULFkgNGjSQtFqt1KVLF+nff/+17OvRo4c0fPhwq+PXrFkj3XLLLZJWq5VatWol/fbbbxVqb0k48yzm/0nRV48ePSre8CI48xwNGza0+RxTp051uV0iJY5AIKiUVOnZRoFAUHUR4iUQCColQrwEAkGlRIiXQCColAjxEggElRIhXgKBoFIixEsgEFRKhHgJBIJKiRAvgVcwYsQIm5ky+vXr52nTBF5Klc4qIahc9OvXjy+++MJqm70qzHq9vlgGD51Oh1ardfq+ZT1P4FlEz0vgNfj4+BAREWH1qlGjBgAKhYJFixZx//33ExAQwNtvv81bb71F+/bt+eyzz2jUqJElZ1RiYiIPPPAAgYGBBAcH89hjj1mldbF3nqByIcRLUGl46623ePDBBzl48CBPPfUUAKdOneL777/nhx9+YN++fZhMJh544AGuXbvG5s2bWb9+PWfOnGHQoEFW1yp6nqDyIYaNAq/h119/JTAw0GrbpEmTmDRpEgBPPPEEI0eOtNqv0+n46quvqFOnDgDr16/n4MGDnD17lqioKAC++uorWrVqxa5du7j11lttnieofAjxEngNvXr1YtGiRVbbChduKFxUxEzDhg2tBOjo0aNERUVZhAsgJiaG0NBQjh49ahGvoucJKh9CvAReQ0BAAE2bNi1xvyPbHL2XoHIjfF6CKkXLli25cOECFy5csGw7cuQI6enpxMTEeNAygasRPS+B15Cfn1+spJZaraZ27doOXyM2NpY2bdowZMgQ5s+fj8FgYMyYMfTo0cPmsFNQeRE9L4HXsG7dOiIjI61ed9xxh1PXUCgU/Pzzz9SoUYO77rqL2NhYGjduzOrVq91ktcBTiDTQAoGgUiJ6XgKBoFIixEsgEFRKhHgJBIJKiRAvgUBQKRHiJRAIKiVCvAQCQaVEiJdAIKiUCPESCASVEiFeAoGgUiLESyAQVEqEeAkEgkqJEC+BQFAp+X+lnewd+yFioAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019611, + "end_time": "2024-03-22T19:36:01.025689", + "exception": false, + "start_time": "2024-03-22T19:36:01.006078", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 3552.247584, + "end_time": "2024-03-22T19:36:03.766922", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/lct_gan/4/mlu-eval.ipynb", + "output_path": "eval/insurance/lct_gan/4/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/lct_gan/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "lct_gan" + }, + "start_time": "2024-03-22T18:36:51.519338", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/lct_gan/model.pt b/insurance/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..505b85820cfc344306c09acc43645bd233b68d38 --- /dev/null +++ b/insurance/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56f2e5d2620ac45edefaad1f1b2207c7976d90edb48ff23aaf22597d30cb38e0 +size 38583573 diff --git a/insurance/lct_gan/params.json b/insurance/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..b7fdce5ae4d3f18c2a088d25ad95646e47901fee --- /dev/null +++ b/insurance/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "NONE", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "fixed_role_model": "lct_gan", "mse_mag": false, "mse_mag_target": 0.1, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/realtabformer/eval.csv b/insurance/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..cd4b02b3b55a34191aa486a65cbdc896f0ccb7f8 --- /dev/null +++ b/insurance/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.004295671779534695,0.005862241469609823,0.0007627403576736382,4.4447386264801025,0.20265215635299683,9.470953941345215,0.4992390275001526,8.549841936655866e-07,5.64751410484314,0.01658162847161293,0.6326260566711426,0.02761775441467762,0.13868294656276703,0.002371334470808506,10.092252731323242 diff --git a/insurance/realtabformer/history.csv b/insurance/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..7f388860adf8af9325229db3b2bc51e24c15d239 --- /dev/null +++ b/insurance/realtabformer/history.csv @@ -0,0 +1,17 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.03506428348624872,0.4538130360197343,0.00792232936242202,0.0,0.0,0.0,0.0,0.0,0.03506428348624872,900,113,125.6448929309845,1.111901707353845,0.13960543658998278,0.13160241616115106,0.01397167039879908,0.13293859043912098,0.002445040742022052,0.0,0.0,0.0,0.0,0.0,0.01397167039879908,450,57,51.65371060371399,0.9062054491879648,0.11478602356380886,0.0867383274241563 +1,0.00870141801217364,0.7070900697936303,0.0004758712758562291,0.0,0.0,0.0,0.0,0.0,0.00870141801217364,900,113,124.63241243362427,1.102941702952427,0.13848045825958252,0.09212413534060516,0.008885490597587907,0.16697826483946382,0.001734984528900567,0.0,0.0,0.0,0.0,0.0,0.008885490597587907,450,57,51.52860236167908,0.9040105677487558,0.11450800524817573,0.07432921211186208 +2,0.005305013003655606,0.5324694752190876,0.00016715914267360208,0.0,0.0,0.0,0.0,0.0,0.005305013003655606,900,113,124.62815642356873,1.1029040391466258,0.1384757293595208,0.09308656097381515,0.0029972506677990573,0.6092274434202413,5.252186918157018e-05,0.0,0.0,0.0,0.0,0.0,0.0029972506677990573,450,57,51.712756872177124,0.9072413486346864,0.11491723749372694,0.05699627291362144 +3,0.0040883604651834405,0.7858639722572959,3.0227560953076958e-05,0.0,0.0,0.0,0.0,0.0,0.0040883604651834405,900,113,124.75911545753479,1.1040629686507504,0.13862123939726087,0.08776520042622511,0.006447157069859612,0.8308172208510114,0.0002398531965743563,0.0,0.0,0.0,0.0,0.0,0.006447157069859612,450,57,51.98431992530823,0.9120056127247057,0.11552071094512939,0.05349372850175489 +4,0.002022715635701186,0.3040784713077333,9.828579557799773e-06,0.0,0.0,0.0,0.0,0.0,0.002022715635701186,900,113,125.08361649513245,1.1069346592489597,0.13898179610570272,0.09222022145656886,0.00254811349499505,0.07651601909258302,2.4787554458056445e-05,0.0,0.0,0.0,0.0,0.0,0.00254811349499505,450,57,51.83201622962952,0.9093336180636757,0.1151822582880656,0.0869178914775451 +5,0.001798461573101425,0.2503146630212105,1.3495003242444538e-05,0.0,0.0,0.0,0.0,0.0,0.001798461573101425,900,113,125.20910906791687,1.1080452129904148,0.13912123229768542,0.09624636015005872,0.00245836095231223,0.28769085691075147,1.3764144745344905e-05,0.0,0.0,0.0,0.0,0.0,0.00245836095231223,450,57,51.629480838775635,0.905780365592555,0.11473217964172364,0.07527359396529694 +6,0.0013259276232889129,0.23208431061799636,4.7836219041035795e-06,0.0,0.0,0.0,0.0,0.0,0.0013259276232889129,900,113,124.9141948223114,1.1054353524098355,0.13879354980256822,0.09697889212716733,0.0033668742331469225,0.8360707927999607,0.00017527166296598044,0.0,0.0,0.0,0.0,0.0,0.0033668742331469225,450,57,52.27232575416565,0.9170583465643096,0.11616072389814588,0.05769461206683334 +7,0.001323487046950807,0.2029421608147736,4.776099965856173e-06,0.0,0.0,0.0,0.0,0.0,0.001323487046950807,900,113,127.35284996032715,1.1270163713303287,0.14150316662258572,0.09650758934100118,0.0012376802931086989,0.1487765556306695,3.4437281075641283e-06,0.0,0.0,0.0,0.0,0.0,0.0012376802931086989,450,57,53.573572397232056,0.9398872350391588,0.11905238310496012,0.07214564077654168 +8,0.0009430775794155327,0.13147762805823807,2.152724269898497e-06,0.0,0.0,0.0,0.0,0.0,0.0009430775794155327,900,113,127.76970887184143,1.1307053882463844,0.14196634319093493,0.09628061390589032,0.0007872949669318688,0.18410880861784518,3.336931345282087e-06,0.0,0.0,0.0,0.0,0.0,0.0007872949669318688,450,57,53.49133658409119,0.938444501475284,0.11886963685353596,0.07019469502643404 +9,0.0009089046284659869,0.097098820553298,9.63799082052168e-06,0.0,0.0,0.0,0.0,0.0,0.0009089046284659869,900,113,127.74243187904358,1.1304639989295893,0.14193603542115954,0.09850719976609787,0.0009410815907176584,0.7538224018950778,2.0871374143320728e-06,0.0,0.0,0.0,0.0,0.0,0.0009410815907176584,450,57,53.4216194152832,0.9372213932505825,0.11871470981174045,0.06436136332616259 +10,0.0005724812648905855,0.12220282757398657,1.0156194488626337e-06,0.0,0.0,0.0,0.0,0.0,0.0005724812648905855,900,113,127.79642581939697,1.1309418214105926,0.14199602868821887,0.09549391159243815,0.0011512360008944396,0.48300685461102216,1.8063996093559748e-06,0.0,0.0,0.0,0.0,0.0,0.0011512360008944396,450,57,53.28672909736633,0.9348548964450234,0.11841495354970297,0.07293740793931902 +11,0.000575461219123099,0.07051163187836926,1.427012848514419e-06,0.0,0.0,0.0,0.0,0.0,0.000575461219123099,900,113,127.76422357559204,1.1306568458016995,0.1419602484173245,0.09717198909647697,0.0008690591479858591,0.8044250670493349,2.2709774584184216e-07,0.0,0.0,0.0,0.0,0.0,0.0008690591479858591,450,57,53.31593942642212,0.9353673583582828,0.11847986539204915,0.06398673398264994 +12,0.0005083383906619727,0.0760834188396614,3.377694875707717e-07,0.0,0.0,0.0,0.0,0.0,0.0005083383906619727,900,113,127.2828004360199,1.1263964640355744,0.1414253338177999,0.09947530912087027,0.0012337127990516214,0.6099953325571875,5.033288178477591e-06,0.0,0.0,0.0,0.0,0.0,0.0012337127990516214,450,57,53.24035835266113,0.93404137460809,0.11831190745035808,0.0727369972022675 +13,0.0008676275550452474,0.0859259017737159,6.595443268466302e-07,0.0,0.0,0.0,0.0,0.0,0.0008676275550452474,900,113,127.27700209617615,1.1263451512935942,0.1414188912179735,0.09762696997649901,0.0010123544972803857,0.5137522601167133,2.4046812298386368e-06,0.0,0.0,0.0,0.0,0.0,0.0010123544972803857,450,57,53.15213441848755,0.9324935862892553,0.11811585426330566,0.07203809639618716 +14,0.00046521230586222374,0.06125880159901201,3.3935850183466135e-07,0.0,0.0,0.0,0.0,0.0,0.00046521230586222374,900,113,127.70906496047974,1.1301687164644225,0.1418989610671997,0.09863226325045117,0.0008455607482270958,0.9153862733114074,1.1769104552178559e-05,0.0,0.0,0.0,0.0,0.0,0.0008455607482270958,450,57,53.55443787574768,0.9395515416797838,0.11900986194610595,0.07057310182503179 +15,0.0004456719228215257,0.04856209839710041,4.839545050809473e-07,0.0,0.0,0.0,0.0,0.0,0.0004456719228215257,900,113,127.83944082260132,1.131322485155764,0.14204382313622368,0.09787282459767518,0.000815138778561959,0.3977327043629637,1.0837310133676681e-05,0.0,0.0,0.0,0.0,0.0,0.000815138778561959,450,57,53.49911832809448,0.9385810232999032,0.11888692961798773,0.06857214915860248 diff --git a/insurance/realtabformer/mlu-eval.ipynb b/insurance/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..6596af54c0945bb7a3192fbf5c1c7b0f4f056683 --- /dev/null +++ b/insurance/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2458 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:47.349540Z", + "iopub.status.busy": "2024-03-22T12:35:47.349160Z", + "iopub.status.idle": "2024-03-22T12:35:47.384279Z", + "shell.execute_reply": "2024-03-22T12:35:47.383378Z" + }, + "papermill": { + "duration": 0.049819, + "end_time": "2024-03-22T12:35:47.386356", + "exception": false, + "start_time": "2024-03-22T12:35:47.336537", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:47.412800Z", + "iopub.status.busy": "2024-03-22T12:35:47.412396Z", + "iopub.status.idle": "2024-03-22T12:35:47.419312Z", + "shell.execute_reply": "2024-03-22T12:35:47.418540Z" + }, + "papermill": { + "duration": 0.022638, + "end_time": "2024-03-22T12:35:47.421235", + "exception": false, + "start_time": "2024-03-22T12:35:47.398597", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:47.444989Z", + "iopub.status.busy": "2024-03-22T12:35:47.444683Z", + "iopub.status.idle": "2024-03-22T12:35:47.448862Z", + "shell.execute_reply": "2024-03-22T12:35:47.447990Z" + }, + "papermill": { + "duration": 0.018321, + "end_time": "2024-03-22T12:35:47.450838", + "exception": false, + "start_time": "2024-03-22T12:35:47.432517", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:47.475174Z", + "iopub.status.busy": "2024-03-22T12:35:47.474895Z", + "iopub.status.idle": "2024-03-22T12:35:47.478902Z", + "shell.execute_reply": "2024-03-22T12:35:47.478157Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018498, + "end_time": "2024-03-22T12:35:47.480833", + "exception": false, + "start_time": "2024-03-22T12:35:47.462335", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:47.504509Z", + "iopub.status.busy": "2024-03-22T12:35:47.503928Z", + "iopub.status.idle": "2024-03-22T12:35:47.509824Z", + "shell.execute_reply": "2024-03-22T12:35:47.508938Z" + }, + "papermill": { + "duration": 0.019779, + "end_time": "2024-03-22T12:35:47.511720", + "exception": false, + "start_time": "2024-03-22T12:35:47.491941", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "0bbc779e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:47.538346Z", + "iopub.status.busy": "2024-03-22T12:35:47.537666Z", + "iopub.status.idle": "2024-03-22T12:35:47.542776Z", + "shell.execute_reply": "2024-03-22T12:35:47.541930Z" + }, + "papermill": { + "duration": 0.021299, + "end_time": "2024-03-22T12:35:47.544697", + "exception": false, + "start_time": "2024-03-22T12:35:47.523398", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"realtabformer\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/realtabformer/42\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011192, + "end_time": "2024-03-22T12:35:47.567165", + "exception": false, + "start_time": "2024-03-22T12:35:47.555973", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:47.592215Z", + "iopub.status.busy": "2024-03-22T12:35:47.591643Z", + "iopub.status.idle": "2024-03-22T12:35:47.601859Z", + "shell.execute_reply": "2024-03-22T12:35:47.600997Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.02545, + "end_time": "2024-03-22T12:35:47.603702", + "exception": false, + "start_time": "2024-03-22T12:35:47.578252", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/realtabformer/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:47.627355Z", + "iopub.status.busy": "2024-03-22T12:35:47.627082Z", + "iopub.status.idle": "2024-03-22T12:35:49.653993Z", + "shell.execute_reply": "2024-03-22T12:35:49.653041Z" + }, + "papermill": { + "duration": 2.041195, + "end_time": "2024-03-22T12:35:49.656109", + "exception": false, + "start_time": "2024-03-22T12:35:47.614914", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:49.682589Z", + "iopub.status.busy": "2024-03-22T12:35:49.682162Z", + "iopub.status.idle": "2024-03-22T12:35:49.697952Z", + "shell.execute_reply": "2024-03-22T12:35:49.697189Z" + }, + "papermill": { + "duration": 0.031104, + "end_time": "2024-03-22T12:35:49.699859", + "exception": false, + "start_time": "2024-03-22T12:35:49.668755", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:49.724468Z", + "iopub.status.busy": "2024-03-22T12:35:49.724192Z", + "iopub.status.idle": "2024-03-22T12:35:49.733927Z", + "shell.execute_reply": "2024-03-22T12:35:49.733133Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.024166, + "end_time": "2024-03-22T12:35:49.736049", + "exception": false, + "start_time": "2024-03-22T12:35:49.711883", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:49.761891Z", + "iopub.status.busy": "2024-03-22T12:35:49.761579Z", + "iopub.status.idle": "2024-03-22T12:35:50.229322Z", + "shell.execute_reply": "2024-03-22T12:35:50.228449Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.48255, + "end_time": "2024-03-22T12:35:50.231538", + "exception": false, + "start_time": "2024-03-22T12:35:49.748988", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:35:50.257526Z", + "iopub.status.busy": "2024-03-22T12:35:50.257199Z", + "iopub.status.idle": "2024-03-22T12:36:03.484359Z", + "shell.execute_reply": "2024-03-22T12:36:03.483512Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 13.243065, + "end_time": "2024-03-22T12:36:03.486883", + "exception": false, + "start_time": "2024-03-22T12:35:50.243818", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 12:35:54.847396: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 12:35:54.847529: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 12:35:54.998663: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:36:03.514835Z", + "iopub.status.busy": "2024-03-22T12:36:03.513621Z", + "iopub.status.idle": "2024-03-22T12:36:03.520436Z", + "shell.execute_reply": "2024-03-22T12:36:03.519529Z" + }, + "papermill": { + "duration": 0.022792, + "end_time": "2024-03-22T12:36:03.522496", + "exception": false, + "start_time": "2024-03-22T12:36:03.499704", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:36:03.547255Z", + "iopub.status.busy": "2024-03-22T12:36:03.546651Z", + "iopub.status.idle": "2024-03-22T12:36:12.307056Z", + "shell.execute_reply": "2024-03-22T12:36:12.305918Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.775305, + "end_time": "2024-03-22T12:36:12.309458", + "exception": false, + "start_time": "2024-03-22T12:36:03.534153", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T12:36:12.337511Z", + "iopub.status.busy": "2024-03-22T12:36:12.336643Z", + "iopub.status.idle": "2024-03-22T12:36:12.343460Z", + "shell.execute_reply": "2024-03-22T12:36:12.342628Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022975, + "end_time": "2024-03-22T12:36:12.345591", + "exception": false, + "start_time": "2024-03-22T12:36:12.322616", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:36:12.370339Z", + "iopub.status.busy": "2024-03-22T12:36:12.370061Z", + "iopub.status.idle": "2024-03-22T12:36:12.374901Z", + "shell.execute_reply": "2024-03-22T12:36:12.374127Z" + }, + "papermill": { + "duration": 0.019421, + "end_time": "2024-03-22T12:36:12.376704", + "exception": false, + "start_time": "2024-03-22T12:36:12.357283", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:36:12.401546Z", + "iopub.status.busy": "2024-03-22T12:36:12.401274Z", + "iopub.status.idle": "2024-03-22T12:42:06.227311Z", + "shell.execute_reply": "2024-03-22T12:42:06.226366Z" + }, + "papermill": { + "duration": 353.85408, + "end_time": "2024-03-22T12:42:06.242495", + "exception": false, + "start_time": "2024-03-22T12:36:12.388415", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_bs_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/bs_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_synth_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:42:06.270659Z", + "iopub.status.busy": "2024-03-22T12:42:06.270345Z", + "iopub.status.idle": "2024-03-22T12:42:06.858633Z", + "shell.execute_reply": "2024-03-22T12:42:06.857143Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.605842, + "end_time": "2024-03-22T12:42:06.861524", + "exception": false, + "start_time": "2024-03-22T12:42:06.255682", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:42:06.896865Z", + "iopub.status.busy": "2024-03-22T12:42:06.896457Z", + "iopub.status.idle": "2024-03-22T12:49:39.565230Z", + "shell.execute_reply": "2024-03-22T12:49:39.564269Z" + }, + "papermill": { + "duration": 452.702282, + "end_time": "2024-03-22T12:49:39.580103", + "exception": false, + "start_time": "2024-03-22T12:42:06.877821", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T12:49:39.611247Z", + "iopub.status.busy": "2024-03-22T12:49:39.610933Z", + "iopub.status.idle": "2024-03-22T12:49:40.028883Z", + "shell.execute_reply": "2024-03-22T12:49:40.027913Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.437046, + "end_time": "2024-03-22T12:49:40.031135", + "exception": false, + "start_time": "2024-03-22T12:49:39.594089", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n", + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:49:40.060914Z", + "iopub.status.busy": "2024-03-22T12:49:40.060576Z", + "iopub.status.idle": "2024-03-22T12:49:40.065041Z", + "shell.execute_reply": "2024-03-22T12:49:40.064138Z" + }, + "papermill": { + "duration": 0.02147, + "end_time": "2024-03-22T12:49:40.067072", + "exception": false, + "start_time": "2024-03-22T12:49:40.045602", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:49:40.094511Z", + "iopub.status.busy": "2024-03-22T12:49:40.094266Z", + "iopub.status.idle": "2024-03-22T12:49:40.101252Z", + "shell.execute_reply": "2024-03-22T12:49:40.100375Z" + }, + "papermill": { + "duration": 0.023185, + "end_time": "2024-03-22T12:49:40.103291", + "exception": false, + "start_time": "2024-03-22T12:49:40.080106", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10420892" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:49:40.130650Z", + "iopub.status.busy": "2024-03-22T12:49:40.130382Z", + "iopub.status.idle": "2024-03-22T12:49:40.225756Z", + "shell.execute_reply": "2024-03-22T12:49:40.224893Z" + }, + "papermill": { + "duration": 0.111822, + "end_time": "2024-03-22T12:49:40.228204", + "exception": false, + "start_time": "2024-03-22T12:49:40.116382", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 15200] --\n", + "├─Adapter: 1-1 [2, 1071, 15200] --\n", + "│ └─Embedding: 2-1 [2, 1071, 19, 800] (440,800)\n", + "│ └─TensorInductionPoint: 2-2 [19, 1] 19\n", + "│ └─Sequential: 2-3 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 820,224\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 15200] (recursive)\n", + "│ └─Embedding: 2-4 [2, 267, 19, 800] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [19, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-7 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-8 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,861,692\n", + "Trainable params: 10,420,892\n", + "Non-trainable params: 440,800\n", + "Total mult-adds (M): 43.07\n", + "========================================================================================================================\n", + "Input size (MB): 0.20\n", + "Forward/backward pass size (MB): 632.89\n", + "Params size (MB): 43.45\n", + "Estimated Total Size (MB): 676.54\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T12:49:40.260781Z", + "iopub.status.busy": "2024-03-22T12:49:40.260467Z", + "iopub.status.idle": "2024-03-22T13:39:59.912945Z", + "shell.execute_reply": "2024-03-22T13:39:59.911946Z" + }, + "papermill": { + "duration": 3019.68848, + "end_time": "2024-03-22T13:39:59.931703", + "exception": false, + "start_time": "2024-03-22T12:49:40.243223", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding True True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.03506428348624872, 'avg_role_model_std_loss': 0.4538130360197343, 'avg_role_model_mean_pred_loss': 0.00792232936242202, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.03506428348624872, 'n_size': 900, 'n_batch': 113, 'duration': 125.6448929309845, 'duration_batch': 1.111901707353845, 'duration_size': 0.13960543658998278, 'avg_pred_std': 0.13160241616115106}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01397167039879908, 'avg_role_model_std_loss': 0.13293859043912098, 'avg_role_model_mean_pred_loss': 0.002445040742022052, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01397167039879908, 'n_size': 450, 'n_batch': 57, 'duration': 51.65371060371399, 'duration_batch': 0.9062054491879648, 'duration_size': 0.11478602356380886, 'avg_pred_std': 0.0867383274241563}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00870141801217364, 'avg_role_model_std_loss': 0.7070900697936303, 'avg_role_model_mean_pred_loss': 0.0004758712758562291, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00870141801217364, 'n_size': 900, 'n_batch': 113, 'duration': 124.63241243362427, 'duration_batch': 1.102941702952427, 'duration_size': 0.13848045825958252, 'avg_pred_std': 0.09212413534060516}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008885490597587907, 'avg_role_model_std_loss': 0.16697826483946382, 'avg_role_model_mean_pred_loss': 0.001734984528900567, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008885490597587907, 'n_size': 450, 'n_batch': 57, 'duration': 51.52860236167908, 'duration_batch': 0.9040105677487558, 'duration_size': 0.11450800524817573, 'avg_pred_std': 0.07432921211186208}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005305013003655606, 'avg_role_model_std_loss': 0.5324694752190876, 'avg_role_model_mean_pred_loss': 0.00016715914267360208, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005305013003655606, 'n_size': 900, 'n_batch': 113, 'duration': 124.62815642356873, 'duration_batch': 1.1029040391466258, 'duration_size': 0.1384757293595208, 'avg_pred_std': 0.09308656097381515}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0029972506677990573, 'avg_role_model_std_loss': 0.6092274434202413, 'avg_role_model_mean_pred_loss': 5.252186918157018e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029972506677990573, 'n_size': 450, 'n_batch': 57, 'duration': 51.712756872177124, 'duration_batch': 0.9072413486346864, 'duration_size': 0.11491723749372694, 'avg_pred_std': 0.05699627291362144}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0040883604651834405, 'avg_role_model_std_loss': 0.7858639722572959, 'avg_role_model_mean_pred_loss': 3.0227560953076958e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0040883604651834405, 'n_size': 900, 'n_batch': 113, 'duration': 124.75911545753479, 'duration_batch': 1.1040629686507504, 'duration_size': 0.13862123939726087, 'avg_pred_std': 0.08776520042622511}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006447157069859612, 'avg_role_model_std_loss': 0.8308172208510114, 'avg_role_model_mean_pred_loss': 0.0002398531965743563, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006447157069859612, 'n_size': 450, 'n_batch': 57, 'duration': 51.98431992530823, 'duration_batch': 0.9120056127247057, 'duration_size': 0.11552071094512939, 'avg_pred_std': 0.05349372850175489}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002022715635701186, 'avg_role_model_std_loss': 0.3040784713077333, 'avg_role_model_mean_pred_loss': 9.828579557799773e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002022715635701186, 'n_size': 900, 'n_batch': 113, 'duration': 125.08361649513245, 'duration_batch': 1.1069346592489597, 'duration_size': 0.13898179610570272, 'avg_pred_std': 0.09222022145656886}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00254811349499505, 'avg_role_model_std_loss': 0.07651601909258302, 'avg_role_model_mean_pred_loss': 2.4787554458056445e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00254811349499505, 'n_size': 450, 'n_batch': 57, 'duration': 51.83201622962952, 'duration_batch': 0.9093336180636757, 'duration_size': 0.1151822582880656, 'avg_pred_std': 0.0869178914775451}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001798461573101425, 'avg_role_model_std_loss': 0.2503146630212105, 'avg_role_model_mean_pred_loss': 1.3495003242444538e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001798461573101425, 'n_size': 900, 'n_batch': 113, 'duration': 125.20910906791687, 'duration_batch': 1.1080452129904148, 'duration_size': 0.13912123229768542, 'avg_pred_std': 0.09624636015005872}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00245836095231223, 'avg_role_model_std_loss': 0.28769085691075147, 'avg_role_model_mean_pred_loss': 1.3764144745344905e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00245836095231223, 'n_size': 450, 'n_batch': 57, 'duration': 51.629480838775635, 'duration_batch': 0.905780365592555, 'duration_size': 0.11473217964172364, 'avg_pred_std': 0.07527359396529694}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013259276232889129, 'avg_role_model_std_loss': 0.23208431061799636, 'avg_role_model_mean_pred_loss': 4.7836219041035795e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013259276232889129, 'n_size': 900, 'n_batch': 113, 'duration': 124.9141948223114, 'duration_batch': 1.1054353524098355, 'duration_size': 0.13879354980256822, 'avg_pred_std': 0.09697889212716733}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0033668742331469225, 'avg_role_model_std_loss': 0.8360707927999607, 'avg_role_model_mean_pred_loss': 0.00017527166296598044, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033668742331469225, 'n_size': 450, 'n_batch': 57, 'duration': 52.27232575416565, 'duration_batch': 0.9170583465643096, 'duration_size': 0.11616072389814588, 'avg_pred_std': 0.05769461206683334}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001323487046950807, 'avg_role_model_std_loss': 0.2029421608147736, 'avg_role_model_mean_pred_loss': 4.776099965856173e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001323487046950807, 'n_size': 900, 'n_batch': 113, 'duration': 127.35284996032715, 'duration_batch': 1.1270163713303287, 'duration_size': 0.14150316662258572, 'avg_pred_std': 0.09650758934100118}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0012376802931086989, 'avg_role_model_std_loss': 0.1487765556306695, 'avg_role_model_mean_pred_loss': 3.4437281075641283e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012376802931086989, 'n_size': 450, 'n_batch': 57, 'duration': 53.573572397232056, 'duration_batch': 0.9398872350391588, 'duration_size': 0.11905238310496012, 'avg_pred_std': 0.07214564077654168}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009430775794155327, 'avg_role_model_std_loss': 0.13147762805823807, 'avg_role_model_mean_pred_loss': 2.152724269898497e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009430775794155327, 'n_size': 900, 'n_batch': 113, 'duration': 127.76970887184143, 'duration_batch': 1.1307053882463844, 'duration_size': 0.14196634319093493, 'avg_pred_std': 0.09628061390589032}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0007872949669318688, 'avg_role_model_std_loss': 0.18410880861784518, 'avg_role_model_mean_pred_loss': 3.336931345282087e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007872949669318688, 'n_size': 450, 'n_batch': 57, 'duration': 53.49133658409119, 'duration_batch': 0.938444501475284, 'duration_size': 0.11886963685353596, 'avg_pred_std': 0.07019469502643404}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009089046284659869, 'avg_role_model_std_loss': 0.097098820553298, 'avg_role_model_mean_pred_loss': 9.63799082052168e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009089046284659869, 'n_size': 900, 'n_batch': 113, 'duration': 127.74243187904358, 'duration_batch': 1.1304639989295893, 'duration_size': 0.14193603542115954, 'avg_pred_std': 0.09850719976609787}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0009410815907176584, 'avg_role_model_std_loss': 0.7538224018950778, 'avg_role_model_mean_pred_loss': 2.0871374143320728e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009410815907176584, 'n_size': 450, 'n_batch': 57, 'duration': 53.4216194152832, 'duration_batch': 0.9372213932505825, 'duration_size': 0.11871470981174045, 'avg_pred_std': 0.06436136332616259}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005724812648905855, 'avg_role_model_std_loss': 0.12220282757398657, 'avg_role_model_mean_pred_loss': 1.0156194488626337e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005724812648905855, 'n_size': 900, 'n_batch': 113, 'duration': 127.79642581939697, 'duration_batch': 1.1309418214105926, 'duration_size': 0.14199602868821887, 'avg_pred_std': 0.09549391159243815}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0011512360008944396, 'avg_role_model_std_loss': 0.48300685461102216, 'avg_role_model_mean_pred_loss': 1.8063996093559748e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011512360008944396, 'n_size': 450, 'n_batch': 57, 'duration': 53.28672909736633, 'duration_batch': 0.9348548964450234, 'duration_size': 0.11841495354970297, 'avg_pred_std': 0.07293740793931902}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.000575461219123099, 'avg_role_model_std_loss': 0.07051163187836926, 'avg_role_model_mean_pred_loss': 1.427012848514419e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000575461219123099, 'n_size': 900, 'n_batch': 113, 'duration': 127.76422357559204, 'duration_batch': 1.1306568458016995, 'duration_size': 0.1419602484173245, 'avg_pred_std': 0.09717198909647697}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008690591479858591, 'avg_role_model_std_loss': 0.8044250670493349, 'avg_role_model_mean_pred_loss': 2.2709774584184216e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008690591479858591, 'n_size': 450, 'n_batch': 57, 'duration': 53.31593942642212, 'duration_batch': 0.9353673583582828, 'duration_size': 0.11847986539204915, 'avg_pred_std': 0.06398673398264994}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005083383906619727, 'avg_role_model_std_loss': 0.0760834188396614, 'avg_role_model_mean_pred_loss': 3.377694875707717e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005083383906619727, 'n_size': 900, 'n_batch': 113, 'duration': 127.2828004360199, 'duration_batch': 1.1263964640355744, 'duration_size': 0.1414253338177999, 'avg_pred_std': 0.09947530912087027}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0012337127990516214, 'avg_role_model_std_loss': 0.6099953325571875, 'avg_role_model_mean_pred_loss': 5.033288178477591e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012337127990516214, 'n_size': 450, 'n_batch': 57, 'duration': 53.24035835266113, 'duration_batch': 0.93404137460809, 'duration_size': 0.11831190745035808, 'avg_pred_std': 0.0727369972022675}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008676275550452474, 'avg_role_model_std_loss': 0.0859259017737159, 'avg_role_model_mean_pred_loss': 6.595443268466302e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008676275550452474, 'n_size': 900, 'n_batch': 113, 'duration': 127.27700209617615, 'duration_batch': 1.1263451512935942, 'duration_size': 0.1414188912179735, 'avg_pred_std': 0.09762696997649901}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0010123544972803857, 'avg_role_model_std_loss': 0.5137522601167133, 'avg_role_model_mean_pred_loss': 2.4046812298386368e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010123544972803857, 'n_size': 450, 'n_batch': 57, 'duration': 53.15213441848755, 'duration_batch': 0.9324935862892553, 'duration_size': 0.11811585426330566, 'avg_pred_std': 0.07203809639618716}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00046521230586222374, 'avg_role_model_std_loss': 0.06125880159901201, 'avg_role_model_mean_pred_loss': 3.3935850183466135e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00046521230586222374, 'n_size': 900, 'n_batch': 113, 'duration': 127.70906496047974, 'duration_batch': 1.1301687164644225, 'duration_size': 0.1418989610671997, 'avg_pred_std': 0.09863226325045117}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008455607482270958, 'avg_role_model_std_loss': 0.9153862733114074, 'avg_role_model_mean_pred_loss': 1.1769104552178559e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008455607482270958, 'n_size': 450, 'n_batch': 57, 'duration': 53.55443787574768, 'duration_batch': 0.9395515416797838, 'duration_size': 0.11900986194610595, 'avg_pred_std': 0.07057310182503179}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0004456719228215257, 'avg_role_model_std_loss': 0.04856209839710041, 'avg_role_model_mean_pred_loss': 4.839545050809473e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0004456719228215257, 'n_size': 900, 'n_batch': 113, 'duration': 127.83944082260132, 'duration_batch': 1.131322485155764, 'duration_size': 0.14204382313622368, 'avg_pred_std': 0.09787282459767518}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.000815138778561959, 'avg_role_model_std_loss': 0.3977327043629637, 'avg_role_model_mean_pred_loss': 1.0837310133676681e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000815138778561959, 'n_size': 450, 'n_batch': 57, 'duration': 53.49911832809448, 'duration_batch': 0.9385810232999032, 'duration_size': 0.11888692961798773, 'avg_pred_std': 0.06857214915860248}\n", + "Stopped False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 1050, 'n_batch': 132, 'role_model_metrics': {'avg_loss': 0.0007627404085499057, 'avg_g_mag_loss': 0.005190866909429349, 'avg_g_cos_loss': 0.0021792732999338545, 'pred_duration': 5.654877424240112, 'grad_duration': 4.461353063583374, 'total_duration': 10.116230487823486, 'pred_std': 0.13868294656276703, 'std_loss': 0.002371334470808506, 'mean_pred_loss': 8.549841936655866e-07, 'pred_rmse': 0.02761775441467762, 'pred_mae': 0.01658163033425808, 'pred_mape': 0.6326261162757874, 'grad_rmse': 0.49923819303512573, 'grad_mae': 0.2026481330394745, 'grad_mape': 9.470596313476562}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0007627404085499057, 'avg_g_mag_loss': 0.005190866909429349, 'avg_g_cos_loss': 0.0021792732999338545, 'avg_pred_duration': 5.654877424240112, 'avg_grad_duration': 4.461353063583374, 'avg_total_duration': 10.116230487823486, 'avg_pred_std': 0.13868294656276703, 'avg_std_loss': 0.002371334470808506, 'avg_mean_pred_loss': 8.549841936655866e-07}, 'min_metrics': {'avg_loss': 0.0007627404085499057, 'avg_g_mag_loss': 0.005190866909429349, 'avg_g_cos_loss': 0.0021792732999338545, 'pred_duration': 5.654877424240112, 'grad_duration': 4.461353063583374, 'total_duration': 10.116230487823486, 'pred_std': 0.13868294656276703, 'std_loss': 0.002371334470808506, 'mean_pred_loss': 8.549841936655866e-07, 'pred_rmse': 0.02761775441467762, 'pred_mae': 0.01658163033425808, 'pred_mape': 0.6326261162757874, 'grad_rmse': 0.49923819303512573, 'grad_mae': 0.2026481330394745, 'grad_mape': 9.470596313476562}, 'model_metrics': {'realtabformer': {'avg_loss': 0.0007627404085499057, 'avg_g_mag_loss': 0.005190866909429349, 'avg_g_cos_loss': 0.0021792732999338545, 'pred_duration': 5.654877424240112, 'grad_duration': 4.461353063583374, 'total_duration': 10.116230487823486, 'pred_std': 0.13868294656276703, 'std_loss': 0.002371334470808506, 'mean_pred_loss': 8.549841936655866e-07, 'pred_rmse': 0.02761775441467762, 'pred_mae': 0.01658163033425808, 'pred_mape': 0.6326261162757874, 'grad_rmse': 0.49923819303512573, 'grad_mae': 0.2026481330394745, 'grad_mape': 9.470596313476562}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:39:59.969766Z", + "iopub.status.busy": "2024-03-22T13:39:59.968855Z", + "iopub.status.idle": "2024-03-22T13:39:59.973787Z", + "shell.execute_reply": "2024-03-22T13:39:59.972895Z" + }, + "papermill": { + "duration": 0.026082, + "end_time": "2024-03-22T13:39:59.975670", + "exception": false, + "start_time": "2024-03-22T13:39:59.949588", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:40:00.010145Z", + "iopub.status.busy": "2024-03-22T13:40:00.009867Z", + "iopub.status.idle": "2024-03-22T13:40:00.100752Z", + "shell.execute_reply": "2024-03-22T13:40:00.099927Z" + }, + "papermill": { + "duration": 0.111008, + "end_time": "2024-03-22T13:40:00.103178", + "exception": false, + "start_time": "2024-03-22T13:39:59.992170", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:40:00.140607Z", + "iopub.status.busy": "2024-03-22T13:40:00.140288Z", + "iopub.status.idle": "2024-03-22T13:40:00.451199Z", + "shell.execute_reply": "2024-03-22T13:40:00.450153Z" + }, + "papermill": { + "duration": 0.332177, + "end_time": "2024-03-22T13:40:00.453299", + "exception": false, + "start_time": "2024-03-22T13:40:00.121122", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDhUlEQVR4nO3de1xUdf4/8NfMwMxwHeQ6YAheQAyRTGPCSnPlKxpdcC2KdZVc89JqmWybUinm97dhWaub+dW1bbN2I9Rdazc1C1GzBFG5eAklNVQUBkFkuDMw8/79MXBwZMAZbgPyfj4e58HMOZ/zOZ8Z57z9nM/5nM9HREQExhjrJ8TWLgBjjFmCgxZjrF/hoMUY61c4aDHG+hUOWoyxfoWDFmOsX+GgxRjrV2ysXYDeotfrUVRUBCcnJ4hEImsXhzF2GyJCVVUVfHx8IBa3X58aMEGrqKgIvr6+1i4GY+wOCgsLcc8997S7fcAELScnJwCGL8TZ2dnKpWGM3a6yshK+vr7CudqeARO0Wi4JnZ2dOWgx1ofdqfmGG+IZY/0KBy3GWL/CQYsx1q8MmDYt1jU6nQ6NjY3WLgbrx2xtbSGRSLqcT6eC1qZNm7Bu3Tqo1WqEhoZi48aNCAsLazf9zp07sXLlSly6dAkBAQF455138NhjjwnbV69ejZSUFBQWFkIqlWLcuHH405/+BJVKJaTx9/fH5cuXjfJNSkrCihUrOvMRmJmICGq1GhUVFdYuCrsLuLi4QKlUdqmvpMVBa/v27YiPj8eWLVugUqmwYcMGREZGIj8/H56enm3Sp6enIzY2FklJSXj88ceRnJyM6OhoZGdnY/To0QCAwMBAfPjhhxg2bBjq6uqwfv16TJ06FRcuXICHh4eQ15o1azB//nzh/Z1ujbKuawlYnp6esLe35465rFOICLW1tbh+/ToAwNvbu0uZWSQsLIwWL14svNfpdOTj40NJSUkm08fExFBUVJTROpVKRQsXLmz3GBqNhgDQ/v37hXV+fn60fv16S4vbJk+NRtNhuopaLWVcLKOjF8s6fay7RVNTE+Xl5VFZGX8XrHuUlZVRXl4eNTU1tdlm7jlqUUO8VqtFVlYWIiIihHVisRgRERHIyMgwuU9GRoZRegCIjIxsN71Wq8XWrVuhUCgQGhpqtG3t2rVwc3PD2LFjsW7dOjQ1NbVb1oaGBlRWVhot5si+chPPbT2K1V/nmZX+btbShmVvb2/lkrC7RctvqSvtoxZdHpaVlUGn08HLy8tovZeXF86dO2dyH7VabTK9Wq02Wrd7924899xzqK2thbe3N1JTU+Hu7i5sf/nll3H//ffD1dUV6enpSEhIQHFxMf785z+bPG5SUhLeeustSz4eAMDDUWb4rNUNFu97t+JLQtZduuO31GfuHk6ePBm5ubkoKyvDRx99hJiYGGRmZgrtZPHx8ULaMWPGQCqVYuHChUhKSoJMJmuTX0JCgtE+LY8I3ImboxQAUF6jhV5PEIv5hGWsL7Ho8tDd3R0SiQQlJSVG60tKSqBUKk3uo1QqzUrv4OCAESNG4MEHH8THH38MGxsbfPzxx+2WRaVSoampCZcuXTK5XSaTCY/sWPLojpuDIQDq9ISKOr7Fz1hfY1HQaumOkJaWJqzT6/VIS0tDeHi4yX3Cw8ON0gNAampqu+lvzbehof1LtNzcXIjFYpN3LLtCaiOGws4WAHCDLxFZJ4lEInz11VfWLka3Wr16Ne677z5rF8PyHvHx8fH46KOP8Omnn+Ls2bN48cUXUVNTg7lz5wIA5syZg4SEBCH90qVLsW/fPrz//vs4d+4cVq9ejRMnTmDJkiUAgJqaGrz++us4evQoLl++jKysLPzud7/DtWvX8MwzzwAwNOZv2LABJ0+exC+//ILPP/8cy5Ytw29/+1sMGjSoO74HIy2XiKUctFg/tm3bNri4uHRbfq+++mqbCog1WNym9eyzz6K0tBSrVq2CWq3Gfffdh3379gmN7VeuXDEawGvChAlITk7Gm2++iddffx0BAQH46quvhD5aEokE586dw6effoqysjK4ubnhgQcewA8//IDg4GAAhku9lJQUrF69Gg0NDRg6dCiWLVtm1GbVndwdZfiltAY3qrU9kj9jfYlWq4VUKr1jOkdHRzg6OvZCie6gp/pj9DXm9gEhInrxnyfIb/lu+vuPv/RCyfquuro6ysvLo7q6OiIi0uv1VNPQaJVFr9dbVPZvvvmGHnroIVIoFOTq6kpRUVF04cIFIiIKDw+n1157zSj99evXycbGhr7//nsiIioqKqLHHnuM5HI5+fv70+eff25RX0EA9OWXXwrvT506RZMnTya5XE6urq40f/58qqqqErYfPHiQHnjgAbK3tyeFQkETJkygS5cuERFRbm4uPfroo+To6EhOTk50//330/Hjxzs8/sGDBwmA0ZKYmEhEhj6Pa9asodmzZ5OTkxPFxcUREdFrr71GAQEBZGdnR0OHDqU333yTtFqtkGdiYiKFhoYK7+Pi4uipp56idevWkVKpJFdXV/r9739vtM/tbv9N3crcc7TP3D3sS9ybuz1wTctYXaMO96761irHzlsTCXup+T/XmpoaxMfHY8yYMaiursaqVaswY8YM5ObmYtasWXj33Xexdu1a4Rb89u3b4ePjg0ceeQSAoZmjrKwMhw4dgq2tLeLj44Xe3JaqqalBZGQkwsPDcfz4cVy/fh0vvPAClixZgm3btqGpqQnR0dGYP38+vvjiC2i1Whw7dkwo26xZszB27Fhs3rwZEokEubm5sLW17fCYEyZMwIYNG7Bq1Srk5+cDgFEt6b333sOqVauQmJgorHNycsK2bdvg4+OD06dPY/78+XBycsJrr73W7nEOHjwIb29vHDx4EBcuXMCzzz6L++67z+jJle7GQcuEljuI3Fer/5o5c6bR+7///e/w8PBAXl4eYmJi8Morr+DHH38UglRycjJiY2MhEolw7tw57N+/H8ePH8f48eMBAH/7298QEBDQqbIkJyejvr4en332GRwcHAAAH374IZ544gm88847sLW1hUajweOPP47hw4cDAEaNGiXsf+XKFfzxj39EUFAQAJhVDqlUCoVCAZFIZPLO/q9+9Sv84Q9/MFr35ptvCq/9/f3x6quvIiUlpcOgNWjQIHz44YeQSCQICgpCVFQU0tLSOGj1Nncnw/U9By1jdrYS5K2JtNqxLXH+/HmsWrUKmZmZKCsrg16vB2AIAKNHj8bUqVPx+eef45FHHkFBQQEyMjLw17/+FQCQn58PGxsb3H///UJ+I0aM6PRNn7NnzyI0NFQIWADw0EMPQa/XIz8/HxMnTsTzzz+PyMhI/M///A8iIiIQExMjPJ8XHx+PF154Af/4xz8QERGBZ555RghundUSjG+1fft2fPDBB7h48SKqq6vR1NR0x65CwcHBRiM3eHt74/Tp010q253weFomtNa0+PLwViKRCPZSG6sslvakfuKJJ1BeXo6PPvoImZmZyMzMBGBodAYMl1z/+te/0NjYiOTkZISEhCAkJKTbvzNzffLJJ8jIyMCECROwfft2BAYG4ujRowAMXQ1++uknREVF4cCBA7j33nvx5Zdfdul4twZQwHCHftasWXjsscewe/du5OTk4I033hC+r/bcfpkqEomE/yB6CgctEzy4ptWv3bhxA/n5+XjzzTcxZcoUjBo1Cjdv3jRK89RTT6G+vh779u1DcnIyZs2aJWwbOXIkmpqakJOTI6y7cOFCmzzMNWrUKJw8eRI1NTXCuiNHjkAsFmPkyJHCurFjxyIhIQHp6ekYPXo0kpOThW2BgYFYtmwZvvvuO/z617/GJ598csfjSqVS6HQ6s8qYnp4OPz8/vPHGGxg/fjwCAgLaDAXVV3DQMoEb4vu3QYMGwc3NDVu3bsWFCxdw4MCBNt1jHBwcEB0djZUrV+Ls2bOIjY0VtgUFBSEiIgILFizAsWPHkJOTgwULFsDOzq5Tz87NmjULcrkccXFxOHPmDA4ePIiXXnoJs2fPhpeXFwoKCpCQkICMjAxcvnwZ3333Hc6fP49Ro0ahrq4OS5YswaFDh3D58mUcOXIEx48fN2rzao+/vz+qq6uRlpaGsrIy1NbWtps2ICAAV65cQUpKCi5evIgPPvigy7W5nsJBywS35qBV16hDTUP7I0mwvkksFiMlJQVZWVkYPXo0li1bhnXr1rVJN2vWLJw8eRKPPPIIhgwZYrTts88+g5eXFyZOnIgZM2YId9LkcrnF5bG3t8e3336L8vJyPPDAA3j66acxZcoUfPjhh8L2c+fOYebMmQgMDMSCBQuwePFiLFy4EBKJBDdu3MCcOXMQGBiImJgYTJ8+3azBACZMmIBFixbh2WefhYeHB95999120z755JNYtmwZlixZgvvuuw/p6elYuXKlxZ+1N4iIiKxdiN5QWVkJhUIBjUZzx8ZFIsKoVftQ36jH4T9OxhC3gTk0S319PQoKCjB06NBOnax3k6tXr8LX1xf79+/HlClTrF2cfquj35S55yjfPTRBJBLBzUGGaxV1KK1uGLBBayA7cOAAqqurERISguLiYrz22mvw9/fHxIkTrV20AY8vD9vh7tTSrsWN8QNRY2MjXn/9dQQHB2PGjBnw8PAQOpp+/vnnwiMtty8tj571hunTp7dbjrfffrvXytHbuKbVDneHljuI3Bg/EEVGRiIy0nSftCeffNJo0pVb3amnenf629/+hrq6OpPbXF1de60cvY2DVjta7yByTYsZc3Jy6hOTqgwePNjaRbAKvjxsR8vwNNxXi7G+hYNWO9wduVc8Y30RB612tDTEc02Lsb6Fg1Y7WhviOWgx1pdw0GqH0OWhhi8PGetLOGi1w625plVR24hGXc8+tc7uPnfjxBZ9BQetdgyyl6JlysNyrm2xfqi7J7YAgEOHDkEkEqGioqJb87UEB612iMUiuDaPq1Vaxe1ajPUVnQpamzZtgr+/P+RyOVQqFY4dO9Zh+p07dyIoKAhyuRwhISHYu3ev0fbVq1cjKCgIDg4OGDRoECIiIoRB21qUl5dj1qxZcHZ2houLC+bNm4fq6urOFN9s7s19tbhdqxkRoK2xzmLhc/379u3Dww8/DBcXF7i5ueHxxx/HxYsXARhGP1i+fLlR+tLSUtja2uLw4cMAgOLiYkRFRcHOzg5Dhw5FcnIy/P39sWHDhk59dadPn8avfvUr2NnZwc3NDQsWLDD6/R46dAhhYWFwcHCAi4sLHnroIWE8q5MnT2Ly5MlwcnKCs7Mzxo0bhxMnTnR4vEOHDmHu3LnQaDQQiUQQiURYvXo1AKChoQGvvvoqBg8eDAcHB6hUKhw6dEjY9/Lly3jiiScwaNAgODg4IDg4GHv37sWlS5cwefJkAIbhf0QiEZ5//vlOfR9dYXGP+O3btyM+Ph5btmyBSqXChg0bEBkZifz8fJMTp6anpyM2NhZJSUl4/PHHkZycjOjoaGRnZwvTiAUGBuLDDz/EsGHDUFdXh/Xr12Pq1Km4cOECPDw8ABiGESkuLkZqaioaGxsxd+5cLFiwwGigtO5m6KtVhTKuaRk01gJv+1jn2K8XAVKHO6drxhNbtD+xxZIlS5CXl4eUlBT4+Pjgyy+/xLRp03D69GkEBARg8eLF0Gq1OHz4MBwcHJCXlwdHR0f4+vri3//+N2bOnIn8/Hw4OzvDzs6uU99Jl3Q4V48JYWFhtHjxYuG9TqcjHx8fSkpKMpk+JiaGoqKijNapVCpauHBhu8domUpo//79RESUl5dHAIymTfrmm29IJBLRtWvXTOZRX19PGo1GWAoLC82eQqzF0i+yyW/5btpy6ILZ+9xN2kz31FBNlOhsnaWhukufpbS0lADQ6dOnhenCDh8+LGwPDw+n5cuXExHR2bNn2/zezp8/TwA6NYXY1q1badCgQVRd3foZ9uzZQ2KxmNRqNd24cYMA0KFDh0zm5eTkRNu2bbPwExN98sknpFAojNZdvnyZJBJJm/NmypQplJCQQEREISEhtHr1apN5tkxNdvPmTYvLQ2SFKcS0Wi2ysrKMZpAWi8WIiIhARkaGyX0yMjLajBoZGRnZ7p0VrVaLrVu3QqFQIDQ0VMjDxcXFaDD+iIgIiMViZGZmYsaMGW3ySUpKMmugtI4Izx/y5aGBrb2hxmOtY1uAJ7Yw7fTp09DpdAgMDDRa39DQADc3NwDAyy+/jBdffBHfffcdIiIiMHPmTIwZM6ZTx+sJFrVplZWVQafTCbNJt/Dy8oJarTa5j1qtNiv97t274ejoCLlcjvXr1yM1NRXu7u5CHrdfetrY2MDV1bXd4yYkJECj0QhLYWGhJR8VQOsIpnx52EwkMlyiWWPhiS26ZWKL6upqSCQSZGVlITc3V1jOnj2Lv/zlLwCAF154Ab/88gtmz56N06dPY/z48di4cWO3fdau6jN3DydPnozc3Fykp6dj2rRpiImJ6XQbAgDIZDI4OzsbLZZqaYgv45pWv8ITWxiYmthi7Nix0Ol0uH79OkaMGGG03Do/oq+vLxYtWoRdu3bhD3/4Az766CMhTwBmT5jREywKWu7u7pBIJCgpKTFaX1JSYnJCSABQKpVmpXdwcMCIESPw4IMP4uOPP4aNjQ0+/vhjIY/bA1hTUxPKy8vbPW53cOeaVr/EE1sYmJrYIjAwELNmzcKcOXOwa9cuFBQU4NixY0hKSsKePXsAAK+88gq+/fZbFBQUIDs7GwcPHhSO5+fnB5FIhN27d6O0tLTH7+CbZGlDWlhYGC1ZskR4r9PpaPDgwR02xD/++ONG68LDwztsiCciGjZsGCUmJhJRa0P8iRMnhO3ffvtthw3xtzO3ke9WpworyG/5bgr7U6rZ+9xNOmo07etSU1Np1KhRJJPJaMyYMXTo0CGjxnEior179xIAmjhxYpv9i4qKaPr06SSTycjPz4+Sk5PJ09OTtmzZYtbxbz/WqVOnaPLkySSXy8nV1ZXmz59PVVVVRESkVqspOjqavL29SSqVkp+fH61atYp0Oh01NDTQc889R76+viSVSsnHx4eWLFli9r/JokWLyM3NjQAI55NWq6VVq1aRv78/2drakre3N82YMYNOnTpFRERLliyh4cOHk0wmIw8PD5o9ezaVlZUJea5Zs4aUSiWJRCKKi4szqxwtuqMh3uKglZKSQjKZjLZt20Z5eXm0YMECcnFxIbVaTUREs2fPphUrVgjpjxw5QjY2NvTee+/R2bNnKTExkWxtben06dNERFRdXU0JCQmUkZFBly5dohMnTtDcuXNJJpPRmTNnhHymTZtGY8eOpczMTPrxxx8pICCAYmNjzS53Z4LWtZu15Ld8Nw1P2EM6nd7s/e4W/TlodbeWu88td7RZ5/T63UMAePbZZ1FaWopVq1ZBrVbjvvvuw759+4TG9itXrkAsbr3qnDBhApKTk/Hmm2/i9ddfR0BAAL766iuhj5ZEIsG5c+fw6aefoqysDG5ubnjggQfwww8/GI23/fnnn2PJkiWYMmUKxGIxZs6ciQ8++KBz1UsztQwE2KQnVNY3wsVe2qPHY30HT2zRh/VURO1rOlPTIiIKSdxHfst30/mSyh4qWd81kGta+/bto+DgYLKzsyNPT0+Kjo6mS5cuERHRP//5T3JwcDC53Hvvvb1WxmnTprVbjj/96U+9Vg5LWKWmNdC4O8pQWd+E0iotRrTt8M/uUjyxRd/FQesO3B1l+KWsBjdq+A4iM+CJLayrz/TT6quECS4GcLcHGhiTkLNe0B2/JQ5adzCQH+VpudSpra21cknY3aLlt9SVy2i+PLyDgTyVmEQigYuLi9Cx197evlOdKxkjItTW1uL69etwcXGBRCLpdF4ctO5goE8l1vLEQVceqWKshYuLS5efYuGgdQfuA7imBRjGOvf29oanpycaGxutXRzWj9na2naphtWCg9YdCG1aA7Sm1UIikXTLD46xruKG+DtovTwcmDUtxvoaDlp30NIQX6vVoVbbZOXSMMY4aN2Bo8wGMhvD1zTQLxEZ6ws4aN2BSCQSLhFL+RKRMavjoGUGYSoxrmkxZnUctMzgxo3xjPUZHLTM0FrT4qDFmLVx0DKD2wDvFc9YX8JBywzcV4uxvoODlhkG+qM8jPUlHLTMMNAfmmasL+lU0Nq0aRP8/f0hl8uhUqlw7NixDtPv3LkTQUFBkMvlCAkJwd69e4VtjY2NWL58OUJCQuDg4AAfHx/MmTMHRUXG06/7+/tDJBIZLWvXru1M8S3W+vwh17QYszaLg9b27dsRHx+PxMREZGdnIzQ0FJGRke0OXZKeno7Y2FjMmzcPOTk5iI6ORnR0NM6cOQPAMChYdnY2Vq5ciezsbOzatQv5+fl48skn2+S1Zs0aFBcXC8tLL71kafE7peVRnpu1jWjU6XvlmIyxdlg6m0ZYWBgtXrxYeK/T6cjHx6fDyVqjoqKM1qlUqg4naz127BgBoMuXLwvr/Pz8aP369ZYWV9DZ2XiIiJp0ehq6Yjf5Ld9NJZqBNzMNY73B3HPUopqWVqtFVlYWIiIihHVisRgRERHIyMgwuU9GRoZResAw00l76QFAo9FAJBLBxcXFaP3atWvh5uaGsWPHYt26dWhqav8B5oaGBlRWVhotnSURi+DqYKht8aM8jFmXReNplZWVQafTCROztvDy8sK5c+dM7qNWq02mV6vVJtPX19dj+fLliI2NhbOzs7D+5Zdfxv333w9XV1ekp6cjISEBxcXF+POf/2wyn6SkJLz11luWfLwOuTvKUFat5Ud5GLOyPjUIYGNjI2JiYkBE2Lx5s9G2+Ph44fWYMWMglUqxcOFCJCUlQSaTtckrISHBaJ/Kykr4+vp2umwDeax4xvoSi4KWu7s7JBIJSkpKjNaXlJS0O+6zUqk0K31LwLp8+TIOHDhgVMsyRaVSoampCZcuXcLIkSPbbJfJZCaDWWfxCKaM9Q0WtWlJpVKMGzcOaWlpwjq9Xo+0tDSEh4eb3Cc8PNwoPQCkpqYapW8JWOfPn8f+/fvh5uZ2x7Lk5uZCLBbD07N3pn3mXvGM9Q0WXx7Gx8cjLi4O48ePR1hYGDZs2ICamhrMnTsXADBnzhwMHjwYSUlJAIClS5di0qRJeP/99xEVFYWUlBScOHECW7duBWAIWE8//TSys7Oxe/du6HQ6ob3L1dUVUqkUGRkZyMzMxOTJk+Hk5ISMjAwsW7YMv/3tbzFo0KDu+i461Hp5yDUtxqyqM7cmN27cSEOGDCGpVEphYWF09OhRYdukSZMoLi7OKP2OHTsoMDCQpFIpBQcH0549e4RtBQUFBMDkcvDgQSIiysrKIpVKRQqFguRyOY0aNYrefvttqq+vN7vMXenyQES0/fgV8lu+m+Z8nNmp/RljHTP3HBURDYw5zysrK6FQKKDRaO7YXmbKgXMl+N22Ewj2ccaelx/pgRIyNrCZe47ys4dm4oZ4xvoGDlpmahlT60ZNAwZI5ZSxPomDlpncmnvEN+oIlXU8lRhj1sJBy0xyWwmc5IabrfwoD2PWw0HLAjxEDWPWx0HLAu7cV4sxq+OgZQE3h9bGeMaYdXDQsoC7U3NNq4qDFmPWwkHLAi01rVK+PGTMajhoWcDdiRviGbM2DloWcHfgMbUYszYOWhYQalo1fHnImLVw0LKAMKYWN8QzZjUctCzQMqZWjVaHOq3OyqVhbGDioGUBJ5kNpDaGr4zbtRizDg5aFhCJREJjPLdrMWYdHLQs1NIYz+1ajFkHBy0LuXG3B8asioOWhYSRHvjykDGr6FTQ2rRpE/z9/SGXy6FSqXDs2LEO0+/cuRNBQUGQy+UICQnB3r17hW2NjY1Yvnw5QkJC4ODgAB8fH8yZMwdFRUVGeZSXl2PWrFlwdnaGi4sL5s2bh+rq6s4Uv0taRjAt5ctDxqzC4qC1fft2xMfHIzExEdnZ2QgNDUVkZCSuX79uMn16ejpiY2Mxb9485OTkIDo6GtHR0Thz5gwAoLa2FtnZ2Vi5ciWys7Oxa9cu5Ofn48knnzTKZ9asWfjpp5+QmpqK3bt34/Dhw1iwYEEnPnLXtAxPwzUtxqzE0ml+wsLCaPHixcJ7nU5HPj4+lJSUZDJ9TEwMRUVFGa1TqVS0cOHCdo9x7NgxAkCXL18mIqK8vDwCQMePHxfSfPPNNyQSiejatWtmlburU4i1+CrnKvkt303P/TWjS/kwxoyZe45aVNPSarXIyspCRESEsE4sFiMiIgIZGRkm98nIyDBKDwCRkZHtpgcAjUYDkUgEFxcXIQ8XFxeMHz9eSBMREQGxWIzMzEyTeTQ0NKCystJo6Q48phZj1mVR0CorK4NOp4OXl5fRei8vL2FW6Nup1WqL0tfX12P58uWIjY0V5j5Tq9Xw9PQ0SmdjYwNXV9d280lKSoJCoRAWX19fsz7jnQhjavHwNIxZRZ+6e9jY2IiYmBgQETZv3tylvBISEqDRaISlsLCwW8rYUtO6WatFk07fLXkyxsxnY0lid3d3SCQSlJSUGK0vKSmBUqk0uY9SqTQrfUvAunz5Mg4cOGA0w6xSqWzT0N/U1ITy8vJ2jyuTySCTycz+bOZydZBCJAKIgPJaLTyd5N1+DMZY+yyqaUmlUowbNw5paWnCOr1ej7S0NISHh5vcJzw83Cg9AKSmphqlbwlY58+fx/79++Hm5tYmj4qKCmRlZQnrDhw4AL1eD5VKZclH6DKJWARX+5Zhl/kSkbFeZ2kLf0pKCslkMtq2bRvl5eXRggULyMXFhdRqNRERzZ49m1asWCGkP3LkCNnY2NB7771HZ8+epcTERLK1taXTp08TEZFWq6Unn3yS7rnnHsrNzaXi4mJhaWhoEPKZNm0ajR07ljIzM+nHH3+kgIAAio2NNbvc3XX3kIho6p+/J7/lu+nwz9e7nBdjzMDcc9TioEVEtHHjRhoyZAhJpVIKCwujo0ePCtsmTZpEcXFxRul37NhBgYGBJJVKKTg4mPbs2SNsKygoIAAml4MHDwrpbty4QbGxseTo6EjOzs40d+5cqqqqMrvM3Rm0YrdmkN/y3bQru7DLeTHGDMw9R0VERNaq5fWmyspKKBQKaDQao/ayznj5ixz892QR3owahRceGdZNJWRsYDP3HO1Tdw/7i5bnD0v5oWnGeh0HrU5oGcH0BvfVYqzXcdDqBI+WseK5psVYr+Og1Qlc02LMejhodYI717QYsxoOWp1wa01rgNx8ZazP4KDVCS01La1Oj8r6JiuXhrGBhYNWJ8htJXCSGR7b5EtExnoXB61O4sZ4xqyDg1YncWM8Y9bBQauTWmtaHLQY600ctDqp9VEevjxkrDdx0OqklqnEuKbFWO/ioNVJHo480zRj1sBBq5Naa1p8echYb+Kg1Ul895Ax6+Cg1UnujjyVGGPWwEGrk1ouD6sbmlDfqLNyaRgbODhodZKz3AZSieHr40tExnoPB61OEolE/CgPY1bQqaC1adMm+Pv7Qy6XQ6VS4dixYx2m37lzJ4KCgiCXyxESEoK9e/cabd+1axemTp0KNzc3iEQi5Obmtsnj0UcfhUgkMloWLVrUmeJ3G26MZ6z3WRy0tm/fjvj4eCQmJiI7OxuhoaGIjIxsMwN0i/T0dMTGxmLevHnIyclBdHQ0oqOjcebMGSFNTU0NHn74YbzzzjsdHnv+/PkoLi4WlnfffdfS4ncrrmkxZgWWzk0WFhZGixcvFt7rdDry8fGhpKQkk+ljYmIoKirKaJ1KpaKFCxe2SdsyB2JOTk6bbZMmTaKlS5eaXc76+nrSaDTCUlhY2G3zHrb4w45c8lu+mz48cL7b8mRsoDJ33kOLalparRZZWVmIiIgQ1onFYkRERCAjI8PkPhkZGUbpASAyMrLd9B35/PPP4e7ujtGjRyMhIQG1tbXtpk1KSoJCoRAWX19fi493J1zTYqz32ViSuKysDDqdDl5eXkbrvby8cO7cOZP7qNVqk+nVarVFBf3Nb34DPz8/+Pj44NSpU1i+fDny8/Oxa9cuk+kTEhIQHx8vvK+srOz2wMWz8jDW+ywKWta0YMEC4XVISAi8vb0xZcoUXLx4EcOHD2+TXiaTQSaT9WiZuCGesd5n0eWhu7s7JBIJSkpKjNaXlJRAqVSa3EepVFqU3lwqlQoAcOHChS7l0xV8echY77MoaEmlUowbNw5paWnCOr1ej7S0NISHh5vcJzw83Cg9AKSmprab3lwt3SK8vb27lE9XcE2Lsd5n8eVhfHw84uLiMH78eISFhWHDhg2oqanB3LlzAQBz5szB4MGDkZSUBABYunQpJk2ahPfffx9RUVFISUnBiRMnsHXrViHP8vJyXLlyBUVFRQCA/Px8AIZamlKpxMWLF5GcnIzHHnsMbm5uOHXqFJYtW4aJEydizJgxXf4SjBAB51MBsRgYEdFh0paaVnmtFjo9QSIWdW9ZGGNtdebW5MaNG2nIkCEklUopLCyMjh49KmybNGkSxcXFGaXfsWMHBQYGklQqpeDgYNqzZ4/R9k8++YQAtFkSExOJiOjKlSs0ceJEcnV1JZlMRiNGjKA//vGPFnVfMPd2Kh3/mCjRmehDFZFe32HSxiYd+a/YTX7Ld9P1ynqzy8IYa8vcc1RENDBmG62srIRCoYBGo4Gzs3P7CesqgPXBgLYa+M1OIHBqh/ne/7+pKK/RYt8rjyBI2UG+jLEOmXuO8rOHt7NzAcY9b3id/sEdkwtD1FRxYzxjvYGDlikPvgiIbYBLPwDXsjtM6ubQPIJpDTfGM9YbOGiZorgHGD3T8Dp9Y4dJ3Z2aZ+Wp4qDFWG/goNWeCS8Z/uZ9Bdy81G4yHsGUsd7FQas9yhBg+K8A0gMZ/9duMneeSoyxXsVBqyMTXjb8zfkHUFtuMok7TyXGWK/ioNWRYY8aalyNtcDxj00maW2I58tDxnoDB62OiETAhKWG18f+CjTWt0nS0hBfxg3xjPUKDlp3EhwNKHyBmlLg5BdtNrs5NF8e1mgxQPrpMmZVHLTuRGILPPh7w+uMDwG93mizR3NNS9ukR1VDU2+XjrEBh4OWOe6fDcgUwI0LQL7xpBxyWwkcZYbnznmIGsZ6Hgctc8icgAd+Z3ht4tEeN76DyFiv4aBlLtUiQCIFCjOBK5lGm4RxtbgxnrEex0HLXE5KYEyM4fVtta1bG+MZYz2Lg5YlWjqbntsDlLUO88zdHhjrPRy0LOExEgicBoAMdxKbuTfXtHikB8Z6HgctS7XUtnKTgepSALfWtPjykLGexkHLUn4TgMHjAF0DcMwwzj2PqcVY7+GgZSmRqLW2dfwjQFvDw9Mw1os6FbQ2bdoEf39/yOVyqFQqHDt2rMP0O3fuRFBQEORyOUJCQrB3r3EHzV27dmHq1Klwc3ODSCQSpge7VX19PRYvXgw3Nzc4Ojpi5syZbeZT7DWjngAGDQXqbgI5n7deHnI/LcZ6nMVBa/v27YiPj0diYiKys7MRGhqKyMhIXL9+3WT69PR0xMbGYt68ecjJyUF0dDSio6Nx5swZIU1NTQ0efvhhvPPOO+0ed9myZfj666+xc+dOfP/99ygqKsKvf/1rS4vfPcQSIHyx4XXGh3C3kwAAquqbUN+os06ZGBsoLJ3mJywsjBYvXiy81+l05OPjQ0lJSSbTx8TEUFRUlNE6lUpFCxcubJO2oKCAAFBOTo7R+oqKCrK1taWdO3cK686ePUsAKCMjw6xymz2FmLkaaojW+hMlOpP+9L9pxOt7yG/5brp2s7Z78mdsgDH3HLWopqXVapGVlYWIiNZJTMViMSIiIpCRkWFyn4yMDKP0ABAZGdluelOysrLQ2NholE9QUBCGDBnSbj4NDQ2orKw0WrqV1B4Imw8AEKV/ADd7fpSHsd5gUdAqKyuDTqeDl5eX0XovLy+o1WqT+6jVaovSt5eHVCqFi4uL2fkkJSVBoVAIi6+vr9nHM1vYAsBGDhTlYLKdYVZsDlqM9ay79u5hQkICNBqNsBQWFnb/QRzcgft+AwB4VvsVAL6DyFhPsyhoubu7QyKRtLlrV1JSAqVSaXIfpVJpUfr28tBqtaioqDA7H5lMBmdnZ6OlR4QvASDCffXHECC6yjUtxnqYRUFLKpVi3LhxSEtLE9bp9XqkpaUhPDzc5D7h4eFG6QEgNTW13fSmjBs3Dra2tkb55Ofn48qVKxbl0yPchgOjHgcALJDs5jG1GOthNpbuEB8fj7i4OIwfPx5hYWHYsGEDampqMHfuXADAnDlzMHjwYCQlJQEAli5dikmTJuH9999HVFQUUlJScOLECWzdulXIs7y8HFeuXEFRUREAQ0ACDDUspVIJhUKBefPmIT4+Hq6urnB2dsZLL72E8PBwPPjgg13+ErpswlLg7Nd4SnIEp25eA3CvtUvE2N2rM7cmN27cSEOGDCGpVEphYWF09OhRYdukSZMoLi7OKP2OHTsoMDCQpFIpBQcH0549e4y2f/LJJwSgzZKYmCikqauro9///vc0aNAgsre3pxkzZlBxcbHZZe72Lg+3Kf3Lo0SJzvTf9+b3SP6M3e3MPUdFRANjNobKykooFApoNJoead86c+ALjD68CFUiBzi9fhGwtev2YzB2NzP3HL1r7x72NknQdBTqPeBENajM/pe1i8PYXYuDVjcJVCpw0D4SAKA+8FeeToyxHsJBq5tIxCJMePpl6EiEwIbT+Pb7w9YuEmN3JQ5a3WjEiJEodH8EAKA+uBXXKuqsXCLG7j4ctLqZb8SLAIAn8D0SdhyHXs+XiYx1Jw5a3UwSOBVNDkq4iargdCkVn2VcsnaRGLurcNDqbhIb2IybDQB4TnIAa/edwy+l1VYuFGN3Dw5aPWHsbBBEeERyBh5NxYjfcRJNOr21S8XYXYGDVk8Y5AfR8MkAgNmyw8gtrMBfD/9i5UIxdnfgoNVT7o8DAMyW/QgJdNiw/2fkFXXzQISMDUActHrKyMcAe3fYNZTiD/4FaNQR4nfkoqGJx5BnrCs4aPUUG6kwQOAL9j/AzUGKc+oqbNh/3soFY6x/46DVk5ovEaUFafjzNA8AwF+/v4isy+XWLBVj/RoHrZ7kPgLwexggPSbVfIuZ998DPQHxO06iVtvU9fxP/B344X2An3NkAwgHrZ42zlDbQs4/kPj4SPgo5Lh8oxZJe891Ld/zqcDuZUDaGuDMv7teTsb6CQ5aPW3Uk4DcBdAUwvnaj1j3TCgA4B9HL+OH86Wdy7PuJvDfl1rff/cm0FDV9bIy1g9w0OpptnIg9DnD6+xteGiEO+LC/QAAf9x5Cpq6Rsvz3JcAVBUDbiOAQUMNr79/txsLzVjfxUGrNzQ3yCP/G6D6OlZMH4Wh7g5QV9bjrf/+ZFle5/YCJ78ARGIgejMwvTlYHf0/oDS/e8vNWB/EQas3eN0L3PMAoG8Ccj+HnVSC92NCIRYBu3KuYd8ZMyeurS0Hvl5qeD3hJcA3DAicCgRON+S994/cKM/uep0KWps2bYK/vz/kcjlUKhWOHTvWYfqdO3ciKCgIcrkcISEh2Lt3r9F2IsKqVavg7e0NOzs7RERE4Px54/5M/v7+EIlERsvatWs7U3zraKltZX8GEOH+IYOwaNJwAMAbX542b77EvX8Eaq4D7iOBR19vXT8tCZDIgILvgbz/9EDhGes7LA5a27dvR3x8PBITE5GdnY3Q0FBERkbi+vXrJtOnp6cjNjYW8+bNQ05ODqKjoxEdHY0zZ84Iad5991188MEH2LJlCzIzM+Hg4IDIyEjU19cb5bVmzRoUFxcLy0svvXT74fqu0b8GpE5A+S/ApR8AAEsjAhCkdMKNGi1eSclFsaaDQQPz/gOc+RcgkgAzNhvaylq4DgUefsXw+ts3AG1Nz30OxqzN0ml+wsLCaPHixcJ7nU5HPj4+lJSUZDJ9TEwMRUVFGa1TqVS0cOFCIiLS6/WkVCpp3bp1wvaKigqSyWT0xRdfCOv8/Pxo/fr1lhZX0NNTiJnlv0uJEp2Jdv5OWJVXpKGA1/eS3/LdFPDGXvrfr3+isqp64/2qS4neGWbYd/8a03lra4nWj25O81bPfQbGeoi556hFNS2tVousrCxEREQI68RiMSIiIpCRkWFyn4yMDKP0ABAZGSmkLygogFqtNkqjUCigUqna5Ll27Vq4ublh7NixWLduHZqa2u+g2dDQgMrKSqPF6lr6bJ39r6F9CsAob2fsWBSOsKGu0Dbp8bcfCzDx3YP4c+rPqKxvNLRR7V4G1JYBnsHApNdM521rB0QaJshF+kbgxsVe+ECM9T6LglZZWRl0Oh28vLyM1nt5eUGtNt2YrFarO0zf8vdOeb788stISUnBwYMHsXDhQrz99tt47bV2TmAASUlJUCgUwuLr62v+B+0pPmMB5RhApwVOpgir7/N1wfYFD+LT34Vh9GBn1Gh1+CDtPCa+exD7d242BDmxjeGy0EbWfv5BUcCICEP+37zGjfLsrtRv7h7Gx8fj0UcfxZgxY7Bo0SK8//772LhxIxoaTDdgJyQkQKPRCEthYWEvl7gdLbWt7E+NgopIJMKkQA98veRhbJ51P4Z7OMC2thTjfvoTAODksPnQeoR0nLdIBEx7BxDbAhf2A/l7O07PWD9kUdByd3eHRCJBSUmJ0fqSkhIolUqT+yiVyg7Tt/y1JE8AUKlUaGpqwqVLl0xul8lkcHZ2Nlr6hJBnAFt7oPQcUNj2rqtIJML0EG9898pEfOX3LwwSVeMnvR9mngnHlD8fwq7sq9B1NFmG+whDdwgA2LcCaOQZgdjdxaKgJZVKMW7cOKSlpQnr9Ho90tLSEB4ebnKf8PBwo/QAkJqaKqQfOnQolEqlUZrKykpkZma2mycA5ObmQiwWw9PT05KPYH1yBRA8w/A6+9N2k0nO7MDgkgMgsS0uPvw+XBwdUFheh/gdJzFtw2HsO6Nuf0LYia8CzoOBiivAjxu6/zMwZk2WtvCnpKSQTCajbdu2UV5eHi1YsIBcXFxIrVYTEdHs2bNpxYoVQvojR46QjY0Nvffee3T27FlKTEwkW1tbOn36tJBm7dq15OLiQv/5z3/o1KlT9NRTT9HQoUOprq6OiIjS09Np/fr1lJubSxcvXqR//vOf5OHhQXPmzDG73H3i7mGLy0cNd/n+14uorqLtds01oiRfQ5rvDXdVaxoa6f8OXqAxq78lv+W7yW/5bnpy4w905EKp6WOc2WXYf40H0Y1fevDDMNY9zD1HLQ5aREQbN26kIUOGkFQqpbCwMDp69KiwbdKkSRQXF2eUfseOHRQYGEhSqZSCg4Npz549Rtv1ej2tXLmSvLy8SCaT0ZQpUyg/P1/YnpWVRSqVihQKBcnlcho1ahS9/fbbVF9/W9eADvSpoKXXE30YZggqxz5qu+2fTxu2/XUSUVOj0eaKWi299+05GrXyG/Jbvpv8V+ymv/1gIijp9UTbHjfkk/xcz30WxrqJueeoiGhg3GKqrKyEQqGARqPpG+1bGZuAb1833E1c9EPr+ux/AP9dYujhvvAw4Blkcvey6ga888057My6CgCY/8hQJEwfBbFY1Jro+jlgy0OGR3x+s9PwyA9jfZS552i/uXt41xnzHCCRAupTQFGOYV1FoSGQAcCv3mg3YAGAu6MM7z49BsunGdJ89EMBlm6/bQx6zyBAtcjwet9yoMmMR4UY6+M4aFmLgxsw6gnD66zm7g//fQloqDQ8XB2+5I5ZiEQivPjocKx/NhQ2YhG+PlmE5/9+3NAptcWk5YCj0vD4UPrGHvowjPUeDlrW1PIQ9el/GS4XfzkI2MgNQ86IJWZnM2PsPfhk7gNwlNkg45cbiNmSAbWm+blNuTMw9X8Nrw+/Z6jN9ZbGesNn+/QJ4L2RhmF1GOsiDlrW5P+IYRA/bRXw3RuGdVMSAfcAi7N6JMAD2xc+CA8nGc6pq/Dr/zuCn0uaRzMNeQYYMgFoqms9Tk8q+Qn4Zjnw/kjg3/OAgsNAtRrY/lsg5/OePz67q3HQsiaxGLh/Tuv7IRNa26A6IdhHgV0vTsBwDwcUaerx9OZ0ZP5yw9BT/rF1hhEi8v4DXDzYDYW/TUOV4TL3oynA5glA5hagvsLQX2zSciA0FiAd8J/fA0f+0v3HZwMG3z20tqoS4IOxhteLfgDchnc5y4paLeZ9egJZl29CKhFj/bP3IWqMt6H2k7kFcA8EFh0xzM3YFUTA1ROGTrJndgGNzUPiiG2AkdMNl7/Df2W41CUCUle2tqtNeAn4n/81BFTGYP45ykGrLyg7bzixXYd1W5b1jTq8/EUOvssrgUgErIy6F78bNwj4cDxQUwr8zxrgoaWdy7y2HDi13TCg4fW81vVuIww1x9BYwLGdJxWO/AVIXWV4fd8s4IkPAIlN58rB7ioctG7Tp4NWD9HpCav/+xP+cfQyAGDBxGFYocyC+L+LDQkkMsOQNrb2zX9vXexNbLMHblwAzn5tGEkCMNw4CJ5hCFZDws2rOeX803CnlPTAyMeAp/9uyJ8NaBy0bjMQgxZgGMp68/cX8e4+w6QXT45RYkPjGogLDnUtY+UYw4gVo58G7Fws3//cXmDn84CuwdCWF/tF5/Jhdw0OWrcZqEGrxa7sq3jtX6fQpCeED3XF1mf84SRuMowC0Vhr/Lepvu26lr+2doZA5XNf1wt16QjwxXOGvmleIcBv/w04ed15P3ZX4qB1m4EetADgh/OlWPSPLNRodfBWyBGkdIKHkwyeTnJ4Osvg4SiDp7PhvYeTDHJb8/uKdVrxKeCfMw0TdgzyB2Z/2a1te6z/4KB1Gw5aBmeuaTB323GUVt35kR4nuQ08bwtqPi52GD1YgWAfZzjIuqkBvfwX4B8zgJuXAAdPQ43Le0z35M36DQ5at+Gg1aqyvhEnLpWjtKoB1ysbcL2qwfC6qh7XqwzvtU36DvMQiYARHo4Yc48LxtyjQMg9Ctzr7dz52lmV2lDjKjkDyJyB2BTA/6HO5cX6JQ5at+GgZT4iQmV9E0qbg1hLcCutbkBBWQ3OXNOguOUxoVtIxCIEejlhzGBDEBtzjwIjlU6Q2ZgZyOoqgC9igSvphruST38CBD3WvR+O9VkctG7DQat7Xa+qx5lrGpy6qsHpqxqcvKoxOeGsVCJGkLcTAjydILUx1R3CeJ0tNeC5y4m4t/JH6CDB134rcDPwGQR4OiHAyxGeTjKIuEMqoGsCRGLDUxV3CQ5at+Gg1bOICCWVDTh5tQKnr2pw6poGp69W4GZt4513vo0EOqy1+QjP2BwGAFwjN/yk98cZ/VD8YjsM9W6j4ab0R4DSCSM8HRHg5QQfhbx3gpleD9SVA9XXDTcPqksNf2vKDF02FL6AyxDDX0fPrvf4r9cYOh+X/dy8nAdK84GbBYanDBw8DHdcHb0Mx3NUGl7fvk5q3y0fvydx0LoNB63eR0S4erMOp69pUFBWI4xpf/svjoz2aX0x4fL/YdzVTyFG2/a1UnJGnt4fZ8gfP+n9cdFmOGQewzHCy1CrG+bhAHupBBKxCBKRCDYSESRiMWzEIoiF9yLYQA+bpirYaitho9VA2lgJh8abENVcbw5MpbcFqFLDM5TmkMgAxT2Ai69xMGt57zzY8DSAXg9UXmsNSkKA+hmoLrnzccwhdTIEMCclYO8G2LsCdoMAu+a/t7+3G9T1x7wsxEHrNhy0+qn6SkB9Gig+CV3RSTRey4H05gWITQSOSrJHHvnhjN4fP9M9sIUOCtTAWVQDBWqgMPHXEXUQizpxCtgNMtzpdGxe7N2AupuGoX80hUBlEYzDsQkiiSGI1N009INrj5O3YeQP98BblgDD/tUlrUtVidF7qi4BVakhbmrb/mgWqWNrALMbZHgvlhgWkcTwjKlY0nyZanPbenHz6+b3454HnH06PBwHrdtw0LqLNNYBJXlAcS6gPgV90Ungeh7Eus6PzFpLMmjgAA054AY5owwKlFHzAmeUkgtKSYE6qRuc3bxwj7sLhro7wN/NAf7uDvB3s4e9tLULCOkaIKosgkhT2LxchaiyEGLh9VWI9K2XziS2gch1uHFw8ggE3AIMY6J1QFPbiItl1filtAYXS6tx8Xo1fimrweUbNWjU6eGIOniINPBEBTxFNzFIVAUX1GCQqAoKUQ0GoQoeNrVwl9RCQVWQ66ogulPAtdSC7+/YIdncc5SfVGX9j60dcM84w4Lm8ZV0jYa2nuKThiGsy342PCspVwByF0N7Uwd/7W2ksAfg1qSH9GYtGm7UoKysFtVlNVDfqEFBWQ2uVdSBGgAU1SG3yJL5JN2al1BhjQh6eEADH9ENVMIeV8gTogZbyEolkNqIIZWIIbOthFSSA5lt83ub5m3NS2lVA34prTF5A6SFna0NhrgrMdxzBIa5O2CYhwOadISfS6rwfUkV8tVVhjvBtzQ9iqCHM2rhKqrGqEE63KtownDHRnjIGiEGQQw9JNBDRE0QQw8x6SGBDqLm12LoICYdRKRv3q6Dk80gOFrwjXWkUzWtTZs2Yd26dVCr1QgNDcXGjRsRFhbWbvqdO3di5cqVuHTpEgICAvDOO+/gscdab2UTERITE/HRRx+hoqICDz30EDZv3oyAgNbB8MrLy/HSSy/h66+/hlgsxsyZM/GXv/wFjo7mfRVc02JdVd+oQ2F5LQrKanDpRg0Kympxqfm1qS4gvUnpLMdwTwcMc3fEcA8HDPd0xDAPR3g7y40nOzFBU9eIn5sD2M8lVTinNrzW1Fl+E6U9u196GKMHKzpM02M1re3btyM+Ph5btmyBSqXChg0bEBkZifz8fJMTp6anpyM2NhZJSUl4/PHHkZycjOjoaGRnZ2P06NEAgHfffRcffPABPv30UwwdOhQrV65EZGQk8vLyIJfLAQCzZs1CcXExUlNT0djYiLlz52LBggVITk629CMw1ilyWwkCvJwQ4OXUZltDkw5Nutb//2+9aSi6pVuHqZuJTXpCQ6MOWp0e2iY9Gppa/zY06YzWtb7WwcVeiuEejhjq4QDHLjydoLCzxQP+rnjA31VYR0QorWrAuVsCWXmNFnoi6MmwXU8EvR7QGaYihJ4gbNfrySit3LYbu2ZYOjdZWFgYLV68WHiv0+nIx8eHkpKSTKaPiYmhqKgoo3UqlYoWLlxIRIY5D5VKJa1bt07YXlFRQTKZjL744gsiIsrLyyMAdPz4cSHNN998QyKRiK5du2byuPX19aTRaISlsLCw78x7yBhrw9x5Dy0Kf1qtFllZWYiIiBDWicViREREICMjw+Q+GRkZRukBIDIyUkhfUFAAtVptlEahUEClUglpMjIy4OLigvHjxwtpIiIiIBaLkZmZafK4SUlJUCgUwuLr62vJR2WM9VEWBa2ysjLodDp4eRkPH+Ll5QW1Wm1yH7Va3WH6lr93SnP7paeNjQ1cXV3bPW5CQgI0Go2wFBb24iw0jLEec9fePZTJZJDJZNYuBmOsm1lU03J3d4dEIkFJiXEv3ZKSEiiVSpP7KJXKDtO3/L1TmuvXrxttb2pqQnl5ebvHZYzdnSwKWlKpFOPGjUNaWpqwTq/XIy0tDeHh4Sb3CQ8PN0oPAKmpqUL6oUOHQqlUGqWprKxEZmamkCY8PBwVFRXIysoS0hw4cAB6vR4qlcqSj8AY6+8sbeFPSUkhmUxG27Zto7y8PFqwYAG5uLiQWq0mIqLZs2fTihUrhPRHjhwhGxsbeu+99+js2bOUmJhItra2dPr0aSHN2rVrycXFhf7zn//QqVOn6KmnnqKhQ4dSXV2dkGbatGk0duxYyszMpB9//JECAgIoNjbW7HKbe2eCMWYd5p6jFgctIqKNGzfSkCFDSCqVUlhYGB09elTYNmnSJIqLizNKv2PHDgoMDCSpVErBwcG0Z88eo+16vZ5WrlxJXl5eJJPJaMqUKZSfn2+U5saNGxQbG0uOjo7k7OxMc+fOpaqqKrPLzEGLsb7N3HN0wDx7qNFo4OLigsLCQu4Rz1gfVFlZCV9fX1RUVEChaL/3/F179/B2VVVVAMD9tRjr46qqqjoMWgOmpqXX61FUVAQnJ6c7DhbXEvG5VmaMv5f28XdjmiXfCxGhqqoKPj4+EHcwIuuAqWmJxWLcc889Fu3j7OzMP0AT+HtpH383ppn7vXRUw2px9wwwzRgbEDhoMcb6FQ5aJshkMiQmJvJjQLfh76V9/N2Y1hPfy4BpiGeM3R24psUY61c4aDHG+hUOWoyxfoWDFmOsX+GgxRjrVzho3WbTpk3w9/eHXC6HSqXCsWPHrF0kq1u9ejVEIpHREhQUZO1i9brDhw/jiSeegI+PD0QiEb766iuj7USEVatWwdvbG3Z2doiIiMD58+etU9hedqfv5vnnn2/zG5o2bVqnjsVB6xYt06MlJiYiOzsboaGhiIyMbDNq6kAUHByM4uJiYfnxxx+tXaReV1NTg9DQUGzatMnk9pap8LZs2YLMzEw4ODggMjIS9fXWnROxN9zpuwGAadOmGf2Gvvjii84drIeHyOlXLJ0ebaBITEyk0NBQaxejTwFAX375pfDenKnwBorbvxsiori4OHrqqae6JX+uaTXrzPRoA8n58+fh4+ODYcOGYdasWbhy5Yq1i9SnmDMV3kB36NAheHp6YuTIkXjxxRdx48aNTuXDQatZZ6ZHGyhUKhW2bduGffv2YfPmzSgoKMAjjzwijFHGzJsKbyCbNm0aPvvsM6SlpeGdd97B999/j+nTp0On01mc14AZmoZ13vTp04XXY8aMgUqlgp+fH3bs2IF58+ZZsWSsv3juueeE1yEhIRgzZgyGDx+OQ4cOYcqUKRblxTWtZp2ZHm2gcnFxQWBgIC5cuGDtovQZ5kyFx1oNGzYM7u7unfoNcdBq1pnp0Qaq6upqXLx4Ed7e3tYuSp9hzlR4rNXVq1dx48aNTv2G+PLwFvHx8YiLi8P48eMRFhaGDRs2oKamBnPnzrV20azq1VdfxRNPPAE/Pz8UFRUhMTEREokEsbGx1i5ar6qurjaqGRQUFCA3Nxeurq4YMmQIXnnlFfy///f/EBAQgKFDh2LlypXw8fFBdHS09QrdSzr6blxdXfHWW29h5syZUCqVuHjxIl577TWMGDECkZGRlh+sW+5B3kU6mh5toHr22WfJ29ubpFIpDR48mJ599lm6cOGCtYvV6w4ePEgA2iwtU+aZMxXe3aqj76a2tpamTp1KHh4eZGtrS35+fjR//nxhrlRL8XhajLF+hdu0GGP9Cgctxli/wkGLMdavcNBijPUrHLQYY/0KBy3GWL/CQYsx1q9w0GKM9SsctBhj/QoHLcZYv8JBizHWr/x/4/h0vMG//goAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:40:00.492582Z", + "iopub.status.busy": "2024-03-22T13:40:00.492251Z", + "iopub.status.idle": "2024-03-22T13:42:12.226470Z", + "shell.execute_reply": "2024-03-22T13:42:12.225615Z" + }, + "papermill": { + "duration": 131.756801, + "end_time": "2024-03-22T13:42:12.229047", + "exception": false, + "start_time": "2024-03-22T13:40:00.472246", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:42:12.269366Z", + "iopub.status.busy": "2024-03-22T13:42:12.268607Z", + "iopub.status.idle": "2024-03-22T13:42:12.289169Z", + "shell.execute_reply": "2024-03-22T13:42:12.288266Z" + }, + "papermill": { + "duration": 0.042992, + "end_time": "2024-03-22T13:42:12.291262", + "exception": false, + "start_time": "2024-03-22T13:42:12.248270", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.0042960.0058620.0007634.4447390.2026529.4709540.4992398.549842e-075.6475140.0165820.6326260.0276180.1386830.00237110.092253
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.004296 0.005862 0.000763 4.444739 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.202652 9.470954 0.499239 8.549842e-07 5.647514 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.016582 0.632626 0.027618 0.138683 0.002371 \n", + "\n", + " total_duration \n", + "realtabformer 10.092253 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:42:12.327178Z", + "iopub.status.busy": "2024-03-22T13:42:12.326911Z", + "iopub.status.idle": "2024-03-22T13:42:12.830959Z", + "shell.execute_reply": "2024-03-22T13:42:12.829952Z" + }, + "papermill": { + "duration": 0.524447, + "end_time": "2024-03-22T13:42:12.833077", + "exception": false, + "start_time": "2024-03-22T13:42:12.308630", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:42:12.873245Z", + "iopub.status.busy": "2024-03-22T13:42:12.872921Z", + "iopub.status.idle": "2024-03-22T13:44:39.815505Z", + "shell.execute_reply": "2024-03-22T13:44:39.814649Z" + }, + "papermill": { + "duration": 146.966209, + "end_time": "2024-03-22T13:44:39.818151", + "exception": false, + "start_time": "2024-03-22T13:42:12.851942", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/realtabformer/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/realtabformer/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:44:39.858052Z", + "iopub.status.busy": "2024-03-22T13:44:39.857166Z", + "iopub.status.idle": "2024-03-22T13:44:39.884768Z", + "shell.execute_reply": "2024-03-22T13:44:39.883976Z" + }, + "papermill": { + "duration": 0.049532, + "end_time": "2024-03-22T13:44:39.886903", + "exception": false, + "start_time": "2024-03-22T13:44:39.837371", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:44:39.924142Z", + "iopub.status.busy": "2024-03-22T13:44:39.923864Z", + "iopub.status.idle": "2024-03-22T13:44:39.929666Z", + "shell.execute_reply": "2024-03-22T13:44:39.928712Z" + }, + "papermill": { + "duration": 0.02635, + "end_time": "2024-03-22T13:44:39.931591", + "exception": false, + "start_time": "2024-03-22T13:44:39.905241", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.037575025149528214}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:44:39.970436Z", + "iopub.status.busy": "2024-03-22T13:44:39.970136Z", + "iopub.status.idle": "2024-03-22T13:44:40.408869Z", + "shell.execute_reply": "2024-03-22T13:44:40.407985Z" + }, + "papermill": { + "duration": 0.460497, + "end_time": "2024-03-22T13:44:40.410930", + "exception": false, + "start_time": "2024-03-22T13:44:39.950433", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBiElEQVR4nO3deXhU1f348fedO0tmspIQEpYEwiKoCCIIxQ2suKBW0aoVtYCK2BaqFulP6fMtiq1iW0WobWltayItaqtFbWvdFXcWARVF2SQhQCAhezKT2e75/THJkCHbZLLcBD6v55kH5s6dez93kvnknHPPoimlFEIIYSKL2QEIIYQkIiGE6SQRCSFMJ4lICGE6SURCCNNJIhJCmE4SkRDCdJKIhBCmk0QkhDCdJCLRovvvvx9N0zr1mHl5eWiaxieffBLV/r/5zW8YOnQouq5z+umnd2osoueQRCTa5aGHHuLFF1/slnO9/vrr/L//9/84++yzyc3N5aGHHuqW84ruZzU7ANG7PPTQQ1xzzTXMmDGjy8/19ttvY7FY+Otf/4rdbu/y8wnzSInoOFFbW2t2CJ2uuLgYp9PZaUlIKYXH4+mUY3WE2+02O4QeRxJRL9TQdrN9+3ZuuOEG+vTpwznnnAPA3//+d8aPH4/T6SQ1NZXrr7+ewsLCiPe///77XHvttWRnZ+NwOMjKyuInP/lJm19STdOora3lqaeeQtM0NE1jzpw5ABQUFPCjH/2IkSNH4nQ6SUtL49prryU/P7/ZY7ndbm6//XbS0tJISkpi1qxZlJeXR5wrNzeX2tra8Lny8vIACAQC/OIXv2DYsGE4HA6GDBnCz372M7xeb8Q5hgwZwuWXX85rr73GhAkTcDqd/OlPf2LdunVomsY///lPli5dysCBA0lMTOSaa66hsrISr9fLXXfdRb9+/UhISODmm29ucuxoP+upU6cyevRoNm/ezHnnnYfL5eJnP/tZq5/ziUiqZr3Ytddey4gRI3jooYdQSvHggw/y85//nOuuu465c+dSUlLC448/znnnncfWrVtJSUkB4LnnnsPtdvPDH/6QtLQ0Nm7cyOOPP87+/ft57rnnWjzf3/72N+bOncvEiROZN28eAMOGDQNg06ZNfPTRR1x//fUMGjSI/Px8Vq1axdSpU9m+fTsulyviWAsWLCAlJYX777+fHTt2sGrVKgoKCsJJ4m9/+xtPPPEEGzdu5C9/+QsAZ511FgBz587lqaee4pprruHuu+9mw4YNLFu2jK+++ooXXngh4jw7duxg5syZ3H777dx2222MHDky/NqyZctwOp3ce++97N69m8cffxybzYbFYqG8vJz777+f9evXk5eXR05ODkuWLAm/N9rPGqC0tJTp06dz/fXXc9NNN5GRkdHOn/QJQIle57777lOAmjlzZnhbfn6+0nVdPfjggxH7btu2TVmt1ojtbre7yTGXLVumNE1TBQUFTc7TWHx8vJo9e3aT9zd3zI8//lgBavXq1eFtubm5ClDjx49XPp8vvP3Xv/61AtRLL70U3jZ79mwVHx8fccxPP/1UAWru3LkR2xctWqQA9fbbb4e3DR48WAHq1Vdfjdj3nXfeUYAaPXp0RAwzZ85Umqap6dOnR+w/efJkNXjw4PDz9nzWU6ZMUYD64x//2OTzEUdJ1awX+8EPfhD+/9q1azEMg+uuu44jR46EH5mZmYwYMYJ33nknvK/T6Qz/v7a2liNHjnDWWWehlGLr1q0xxdL4mH6/n9LSUoYPH05KSgpbtmxpsv+8efOw2Wzh5z/84Q+xWq3873//a/U8Da8vXLgwYvvdd98NwMsvvxyxPScnh4svvrjZY82aNSsihkmTJqGU4pZbbonYb9KkSRQWFhIIBID2fdYADoeDm2++udXrOtFJ1awXy8nJCf9/165dKKUYMWJEs/s2/sLt27ePJUuW8O9//zuiXQagsrIyplg8Hg/Lli0jNzeXAwcOoBpN/NncMY+NMyEhgf79+7fYptSgoKAAi8XC8OHDI7ZnZmaSkpJCQUFBxPbGn9GxsrOzI54nJycDkJWV1WS7YRhUVlaSlpbWrs8aYODAgXLXrw2SiHqxxqUQwzDQNI1XXnkFXdeb7JuQkABAMBjkwgsvpKysjHvuuYdRo0YRHx/PgQMHmDNnDoZhxBTLj3/8Y3Jzc7nrrruYPHkyycnJaJrG9ddfH/MxWxNtR8vGn9GxmvucWtvekFyj/ayjiUGESCI6TgwbNgylFDk5OZx00kkt7rdt2zZ27tzJU089xaxZs8Lb33jjjajO01ICeP7555k9ezaPPvpoeFtdXR0VFRXN7r9r1y7OP//88POamhqKioq49NJLWz3/4MGDMQyDXbt2cfLJJ4e3Hz58mIqKCgYPHhzVdXREtJ+1iJ60ER0nrr76anRdZ+nSpRHVIgj9JS8tLQWO/rVvvI9SipUrV0Z1nvj4+GaTi67rTc77+OOPEwwGmz3OE088gd/vDz9ftWoVgUCA6dOnt3r+hkS1YsWKiO3Lly8H4LLLLmvrEjos2s9aRE9KRMeJYcOG8ctf/pLFixeTn5/PjBkzSExMZO/evbzwwgvMmzePRYsWMWrUKIYNG8aiRYs4cOAASUlJ/Otf/2rSVtSS8ePH8+abb7J8+XIGDBhATk4OkyZN4vLLL+dvf/sbycnJnHLKKXz88ce8+eabpKWlNXscn8/HBRdcwHXXXceOHTv4wx/+wDnnnMMVV1zR6vnHjh3L7NmzeeKJJ6ioqGDKlCls3LiRp556ihkzZkSUsrpKtJ+1iJ4kouPIvffey0knncRjjz3G0qVLgVDD60UXXRT+gttsNv7zn/9wxx13sGzZMuLi4rjqqqtYsGABY8eObfMcy5cvZ968efzf//0fHo+H2bNnM2nSJFauXImu66xZs4a6ujrOPvts3nzzzRbvWP3ud79jzZo1LFmyBL/fz8yZM/ntb38bVdvPX/7yF4YOHUpeXh4vvPACmZmZLF68mPvuu68dn1bHRPNZi+hp6tiypRBCdDNpIxJCmE4SkRDCdJKIhBCmk0QkhDCdJCIhhOkkEQkhTNer+xEZhsHBgwdJTEzs9EnehRAdp5SiurqaAQMGYLG0XO7p1Yno4MGDTUZKCyF6nsLCQgYNGtTi6706ESUmJgKhi0xKSjI5GiHEsaqqqsjKygp/V1vSqxNRQ3UsKSlJEpEQPVhbTSfSWC2EMJ0kIiGE6SQRCSFM16vbiKKhlCIQCLQ4QZdoma7rWK1W6RohutxxnYh8Ph9FRUWysmYHuFwu+vfvL5O/iy513CYiwzDYu3cvuq4zYMAA7Ha7/GVvB6UUPp+PkpIS9u7dy4gRI1rtkCZERxy3icjn82EYBllZWU1WGRXRcTqd2Gw2CgoK8Pl8xMXFmR1Sj1bp9vP2jsOM6JfI6IHJZofTqxz3f+Lkr3jHyOcXvY35ZeQfcbMpv4ygIROftof8lgnRSQ5VetA0OO+kdHSLNAO0hyQiITpBIGhQWutDKchMkipse0kiOsENGTKkyRphov1qvUGUApuu4bI3v1KsaJkkIiE6QbU3tFikP6hYs2Ef73xdbHJEvYvpiejAgQPcdNNNpKWl4XQ6Oe200/jkk0/MDqtX8fl8ZodwwqvxBsL/L6n2UlorP5P2MDURlZeXc/bZZ2Oz2XjllVfYvn07jz76KH369OnS8/oCRouPQNCIel9/FPvGYurUqSxYsIAFCxaQnJxM3759+fnPfx5e3njIkCH84he/YNasWSQlJTFv3jwAPvjgA84991ycTidZWVnccccd1NbWho9bXFzMd77zHZxOJzk5OaxZsyam+ERT/oDCpmsk2xUZ1dvx15SZHVKvYmo/ol/96ldkZWWRm5sb3paTk9Pl5/39O7tbfC2nbzwzxg0MP3/ivT34g83fih3Ux8m1E45OzPbkh3vx+CKHkvzkwpNiivGpp57i1ltvZePGjXzyySfMmzeP7OxsbrvtNgAeeeQRlixZEl7ddM+ePVxyySX88pe/5Mknn6SkpCSczBo+3zlz5nDw4EHeeecdbDYbd9xxB8XFUoXoDKcNSmb0wCQqt/2Pr3Z/gN/9JajTQDrRRsXURPTvf/+biy++mGuvvZZ3332XgQMH8qMf/Sj8ZTuW1+vF6/WGn1dVVXVXqN0uKyuLxx57DE3TGDlyJNu2beOxxx4Lfzbf/va3ufvuu8P7z507lxtvvJG77roLgBEjRvDb3/6WKVOmsGrVKvbt28crr7zCxo0bOfPMMwH461//ysknn9zt13a80jQNV9VeAGzecgx3OZb4VJOj6h1MTUTffPMNq1atYuHChfzsZz9j06ZN3HHHHdjtdmbPnt1k/2XLloXXGe+I+ecPb/G1Y7t/zDtvWIv7HvvH7pazO680961vfStiSMrkyZN59NFHw4N3J0yYELH/Z599xueffx5R3VJKhYe67Ny5E6vVyvjx48Ovjxo1ipSUlE6L+YTnq8XqO/rH0V9ZhEMSUVRMTUSGYTBhwgQeeughAMaNG8cXX3zBH//4x2YT0eLFi1m4cGH4ecM0lO1lt0bfNNZV+3ZUfHx8xPOamhpuv/127rjjjib7Zmdns3Pnzu4K7YT06heH0KoPco4/iK5BUIG/qgTHALMj6x1MTUT9+/fnlFNOidh28skn869//avZ/R0OBw6HoztCM92GDRsinq9fv54RI0ag6833UTnjjDPYvn07w4c3X9obNWoUgUCAzZs3h6tmO3bsoKKiolPjPlHtL3djLzuMckCcTcdQCuWtNjusXsPUu2Znn302O3bsiNi2c+dOBg8ebFJEPce+fftYuHAhO3bs4JlnnuHxxx/nzjvvbHH/e+65h48++ogFCxbw6aefsmvXLl566SUWLFgAwMiRI7nkkku4/fbb2bBhA5s3b2bu3Lk4nc7uuqTjWp0/iD1Qg9WiMWZQCqdn9SFR85gdVq9haiL6yU9+wvr163nooYfYvXs3Tz/9NE888QTz5883M6weYdasWXg8HiZOnMj8+fO58847w7fpmzNmzBjeffdddu7cybnnnsu4ceNYsmQJAwYcrRvk5uYyYMAApkyZwtVXX828efPo169fd1zOcS0QNPAHFY5ADVZdg8SM0Ave4/dmSmcztWp25pln8sILL7B48WIeeOABcnJyWLFiBTfeeKOZYfUINpuNFStWsGrVqiav5efnN/ueM888k9dff73FY2ZmZvLf//43Ytv3v//9DsUpwOMP3UCIM2pCg12TBkL1YfDWmBxZ72H6fESXX345l19+udlhCBGzOn+o46oTLxoaX1baCe6vICPZS4ZS0pcoCqYP8RCit6urLxE5CfVxq7IkUusL4vUHIeBt7a2inuklItHUunXrzA5BtEPACA3viMMLONGdyfg1K0HDgIAHbDItSFukRCREB+X0jWfB1KGMyQglHKvdRcDiIGgA/jpzg+slJBEJ0Rn8nlBPeE1DtztDiUgp8MsKMtGQRCREZ/DX9xmyxmGzWglYHBhKQUBKRNGQRCREB32+v4LXP9tLSY0XbC5sukZAj8MwlFTNoiSJSIgOKqn2UnSkLHT3zObEYdXRHc5Qn6KA9K6Ohtw1E6KDfAEDW7AulHhsTrLTXGSPHQL7DkuJKEpSIhKig3xBA6tRh9Wiga1+MU+9fnB2UKaMjYYkIiE6yOsPJaKGEhEA1oZEJB0aoyGJSIgO8gYNrIYvlIiscdT5g7z+dTnbDlSipGd1VE6sRKQUBHzd/1DRLz+8evVq0tLSIqbEBZgxY4YMUO2hfAEDPZyI7GgaFNUa1HgDBANSNYvGidVYHfTD+492/3nPvRus9qh2vfbaa7njjjv497//zbXXXguEVt94+eWXWx1ZL8yjlMKqfOhaqERks1gIajYADJ80VkfjxCoR9QJOp5MbbrghYmWTv//972RnZzN16lTzAhMtmnvuUC48KQWnXQfdgcWiodlCf3iCfqmaRePEKhHptlDpxIzztsNtt93GmWeeyYEDBxg4cCB5eXnMmTMnYjJ90bNYgj5ACzdSW6yhcWeG3L6PyomViDQt6iqSmcaNG8fYsWNZvXo1F110EV9++SUvv/yy2WGJ1jQM5ahPQBZbKCEZ0kYUlRMrEfUic+fOZcWKFRw4cIBp06bFtFqJ6HpVdX7e+eowo4vKGNbXGf5DZ6n/1wgGIRgAXb5qrZE2oh7qhhtuYP/+/fz5z3/mlltuMTsc0QK3N0h+cRWV7sgSkSPOiU3XUCjpSxQFSUQ9VHJyMt/97ndJSEhgxowZZocjWuALGOiq/ta9poEeKgldMyGbCUMzSHHapXd1FCQR9WAHDhzgxhtvPGHWcuuNfMHg0T5Euj1yfur6pIS0E7VJKq49UHl5OevWrWPdunX84Q9/MDsc0QpvRGfGY/5g6HagFgy/KbH1JpKIeqBx48ZRXl7Or371K0aOHGl2OKIV/qAKDe/QIhPRln3lqP3VZNnr6BeURNQWU6tm999/P5qmRTxGjRplZkg9Qn5+PpWVlSxatMjsUEQbItqI9KOJqNLtp8yj8PqNUI9+0SrTS0Snnnoqb775Zvi51Wp6SEJELXDMgNcGVl3Da7GF5q2WqlmbTP/WW61WMjMzu+z4qh0DTkVT8vm17qzhfZlk6wd7XBGdZa0WC4ZmDX1+UiJqk+l3zXbt2sWAAQMYOnQoN954I/v27WtxX6/XS1VVVcSjJTZbaFiF2y2rKHREw+fX8HmKpnTj6IDXBlZdw9CsBBVgBMwLrpcwtUQ0adIk8vLyGDlyJEVFRSxdupRzzz2XL774gsTExCb7L1u2jKVLl0Z1bF3XSUlJobi4GACXyyVjtdpBKYXb7aa4uJiUlBR0XTc7pJ6rYc4hvXGJSCOoWUMT6Es/ojaZmoimT58e/v+YMWOYNGkSgwcP5p///Ce33nprk/0XL17MwoULw8+rqqpaHfrQUOVrSEai/VJSUrq06tzbfbTnCHHfHOIkFSChcYnIYsGwWDGCUjWLhultRI2lpKRw0kknsXv37mZfdzgc7ercp2ka/fv3p1+/fvj98svQXjabTUpCbdhX6iaxtJIhSUZEG5HNqqFZbGiGJo3VUehRiaimpoY9e/Z0+kyEuq7LF0p0CV/QCE2Kdsxds1GZSYyaMAT2FkqJKAqmNlYvWrSId999l/z8fD766COuuuoqdF1n5syZZoYlRNQiponVjymtW+ob+CURtcnUEtH+/fuZOXMmpaWlpKenc84557B+/XrS09PNDEuIqPmCDYnI0swQj/pEJFWzNpmaiJ599lkzTy9Ehyil8AcUuuHHojkiElGF28fW3eUMKq5mRKokoraY3o9IiN4qYCgMpbAq79HR9/X8QcW+Cj9VHr9UzaIgiUiIGPmDBpoKoCmj6RAPi0ZQs9V3aJRE1JYedddMiN7EZbey4Nws1Id90IgsETX0rA51aJSe1W2RRCREB1iVH3RLqGHacrSCYdPrx5oBRsArVY82yOcjREc0DN845o6ZbtEwLKG/84a0EbVJEpEQMTpcVce72ws5UOGJaB+Co21EIEsKRUMSkRAxqnD72XuonAq3L6J9CELDiyy6DQ0wggEwDHOC7CWkjUiIGEXMznhsZ0bg9vNHYvkgLfTE8INFFkFoiZSIhIhRqFe1v0kfogYWq+3oqh7STtQqSURCxMgXaH6a2DBNg/oGa+lL1DpJRELEyB9sXDVrWiJ6f1cJXx52U+sNSImoDZKIhIhReOS91szIe2B/uYcSt4E3KCt5tEUSkRAx8jduI2qmsdpq0Y5OoC9Vs1bJXTMhYnThKRkEAynoZaXNNlaHJ9A3/DLMow2SiISIkVW3YCUQGtrRTGO11WIJTaAvA1/bJFUzIToiUBf6t5nG6oaqmSFrm7VJEpEQMXpvZwk7D5ZSFwg221htrR/4ahhK1jZrgyQiIWL09aEqDpdVETRU843Vemjgq5JFFtskbURCxMjnD9bfvnc2m4jOHd4XzcjCcqhMqmZtkEQkRAwMQxEM+NFQza/gQahqFkpQsrZZW6RqJkQMGlbvALBYLEdX7DiWpX49Pbl93ypJRELEIDS8w48GWGyOo4NbG9lX6mbDvioOVnqkRNQGSURCxKDxwopaM+1DAJUeP3vLfFTXyViztvSYRPTwww+jaRp33XWX2aEI0SZ/UDVaarr5RKRbGk2gLyWiVvWIxupNmzbxpz/9iTFjxpgdihBR6ZfoYOYZ/bBsT2q2oRrApjfq0GgEuznC3sX0ElFNTQ033ngjf/7zn+nTp4/Z4QgRFYtFw2kJ4rDqrZeILPVDPKRq1irTE9H8+fO57LLLmDZtWpv7er1eqqqqIh5CmKZhUvxmBrxC47FmUjVri6lVs2effZYtW7awadOmqPZftmwZS5cu7eKohGjb3iO1VOw9RHatl7TmZmek0SKLMtasTaaViAoLC7nzzjtZs2YNcXHN/yCPtXjxYiorK8OPwsLCLo5SiOYVVXrYe6iMKk+g2QGv0DDoVQ8t4CFDPFplWolo8+bNFBcXc8YZZ4S3BYNB3nvvPX73u9/h9XrRdT3iPQ6HA4dDVkIQ5vMGDKyGF93WwnzVQFqCg5vOGo790xQpEbXBtER0wQUXsG3btohtN998M6NGjeKee+5pkoSE6Em8/lA/ImtLE+cTaqx2xsWFOjtKG1GrTEtEiYmJjB49OmJbfHw8aWlpTbYL0dP4ggYOo67lFTwaNKziIUM8WmX6XTMheiOvP3h0KSFb84koEDR4Z1c5u0tqMIyg9CVqRUwlom+++YahQ4d2diysW7eu048pRFdoaCOytjBNLISWnf6sqJa4ai9D0lxYjMDRQbAiQkwlouHDh3P++efz97//nbq6us6OSYgeLzzWTG+5ambRAE0HNOnU2IaYEtGWLVsYM2YMCxcuJDMzk9tvv52NGzd2dmxC9FjXTxjA+EHxuOzWVktENqtefwtfOjW2JqZEdPrpp7Ny5UoOHjzIk08+SVFREeeccw6jR49m+fLllJSUdHacQvQoLksAp00PLa7YSmO1HjGBvjRYt6RDjdVWq5Wrr76a5557jl/96lfs3r2bRYsWkZWVxaxZsygqKuqsOIXoWfyNVu+wtPw1slo0WVIoCh1KRJ988gk/+tGP6N+/P8uXL2fRokXs2bOHN954g4MHD3LllVd2VpxC9BgeX5CPduynsNwNVmer+1rDA19lmEdrYrprtnz5cnJzc9mxYweXXnopq1ev5tJLLw1NmQnk5OSQl5fHkCFDOjNWIXqEWl+AHftLsFXWkZXVek//8JJCyivDPFoRUyJatWoVt9xyC3PmzKF///7N7tOvXz/++te/dig4IXqihjtmVosGttZLRDPGDcRm6Y+t5oAkolbElIjeeOMNsrOzwyWgBkopCgsLyc7Oxm63M3v27E4JUoieJDzOTG95dsYGCQ4rOBxQq0nVrBUxtRENGzaMI0eONNleVlZGTk5Oh4MSoifz+IJYDS82i6XNNiLg6DAPaaxuUUwlIqVUs9tramqintJDiN6qLhBKRNYoSkTbD1ZhHKoly+8nWW7ft6hdiWjhwoVAqKPWkiVLcLlc4deCwSAbNmzg9NNP79QAhehp6vyNElEbbUQFpbX4Sn2kxgVIlhJRi9qViLZu3QqESkTbtm3Dbj86IZTdbmfs2LEsWrSocyMUoocJJaI6rHYL2Fyt7mvVLdQ1rOQhbUQtalcieuedd4DQvEErV64kKSmpS4ISoic7e3hfAmXx2NzVbSei+n5ESjo0tiqmNqLc3NzOjkOIXsNh1XFYfKBbwN5WiUiGeEQj6kR09dVXk5eXR1JSEldffXWr+65du7bDgQnRo/lqQ/+2USKKXGRRElFLok5EycnJaPXreycnJ3dZQEL0dB/tPERWSSUZyXHY22isPrqkEFI1a0XUiahxdUyqZuJE9kXBIbQKD32TnK1PE4tUzaIVU4dGj8eD2+0OPy8oKGDFihW8/vrrnRaYED1RIGiAL/S7b41zhSbGb8Up/ZO44owshqTFS4moFTEloiuvvJLVq1cDUFFRwcSJE3n00Ue58sorWbVqVacGKERPUhcwsAbr0ACrI6HN/eNsOkkuFzbdIrfvWxHzDI3nnnsuAM8//zyZmZkUFBSwevVqfvvb33ZqgEL0JG5vAKtRh03X0Nq4YxamyxCPtsSUiNxuN4mJiQC8/vrrXH311VgsFr71rW9RUFDQqQEK0ZO4fUFshidUwmmjoRqgvNbHpn01HKz0SBtRK2KePP/FF1+ksLCQ1157jYsuugiA4uJi6eQojmu1vgD2oBub1QL2xDb3r64LsPVADSXVXikRtSKmRLRkyRIWLVrEkCFDmDRpEpMnTwZCpaNx48Z1aoBC9CRuXxBb0B0qEUXRRqTrDVPFSj+i1sSUiK655hr27dvHJ598wquvvhrefsEFF/DYY49FfZxVq1YxZswYkpKSSEpKYvLkybzyyiuxhCREtzg9K4ULclwM6uMEe9uJyFrfoVEppGrWipiXnM7MzCQzMzNi28SJE9t1jEGDBvHwww8zYsQIlFI89dRTXHnllWzdupVTTz011tCE6DI23YJNqwOrHlWJqCERBRt6VhtGq5Ptn6hiSkS1tbU8/PDDvPXWWxQXF2MYRsTr33zzTVTH+c53vhPx/MEHH2TVqlWsX7++2UTk9Xrxer3h51VVVTFEL0QHeatD/0ZVIrJgaPrRObyMAFjsrb/pBBRTIpo7dy7vvvsu3//+9+nfv3946EdHBINBnnvuOWpra8NtTsdatmwZS5cu7fC5hIjVx7uKGFRSTr9EB44oEpFe37M6qECh0Aw/IInoWDEloldeeYWXX36Zs88+u8MBbNu2jcmTJ1NXV0dCQgIvvPACp5xySrP7Ll68ODw5G4RKRFlZWR2OQYhobc8/COUe0pLi25ydEUJVMzStvlQEmnRqbFZMiahPnz6kpqZ2SgAjR47k008/pbKykueff57Zs2fz7rvvNpuMHA4HDkfbP3whuoIvYKC8oVH3dldSm8M7AOy6hRu/lU2Cno6m+eTOWQtiajX7xS9+wZIlSyLGm8XKbrczfPhwxo8fz7Jlyxg7diwrV67s8HGF6GzVdX4cgWqsFg2rM7oZKCwWjX6Jcbji4tDQJBG1IKYS0aOPPsqePXvIyMhgyJAh2Gy2iNe3bNkSc0CGYUQ0SAvRU1TVBXAEqrFbLRDXzqlw9PrviFTNmhVTIpoxY0annHzx4sVMnz6d7Oxsqqurefrpp1m3bh2vvfZapxxfiM7UUCJytDMRbdlXTnKplyy7gV16VzcrpkR03333dcrJi4uLmTVrFkVFRSQnJzNmzBhee+01Lrzwwk45vhCdqbougCNYg8OugzMl6vdtKSgnu8xLRl8Du3RqbFbMHRorKip4/vnn2bNnDz/96U9JTU1ly5YtZGRkMHDgwKiOIUtSi96kus5PXKAKh6t9JSLdoqE0vX6Yh5SImhNTIvr888+ZNm0aycnJ5Ofnc9ttt5GamsratWvZt29feK4iIY4nF4xKJ1BkQ9es7UpEVt1ydLyZtBE1K6a7ZgsXLmTOnDns2rUrYmXXSy+9lPfee6/TghOiJ7EFanFaNew2W1Qj7xs0DPMIzVstVbPmxJSINm3axO23395k+8CBAzl06FCHgxKiR6qrDP3rSGrXeDE9PPBVSkQtiSkRORyOZsd57dy5k/T09A4HJURPU1Tp4cNtuyiurmtXQzWATT9m4KtoIqZEdMUVV/DAAw/g94eyu6Zp7Nu3j3vuuYfvfve7nRqgED1BUWUdh4oOUO72gyutXe/V65cUktVeWxZTInr00UepqakhPT0dj8fDlClTGD58OImJiTz44IOdHaMQpiut8eEMVOCy6+Bq3/Cmc4b35fxTBpAab5c5iVoQ012z5ORk3njjDT788EM+++wzampqOOOMM5g2bVpnxydEj1Ba4yXTX44rSQdX33a9NzXeDonxUGKRElEL2p2IDMMgLy+PtWvXkp+fj6Zp5OTkkJmZiVKqU6YEEaInUUpRVuNmcKAGpy2p3VUzACz1XzVprG5Wu6pmSimuuOIK5s6dy4EDBzjttNM49dRTKSgoYM6cOVx11VVdFacQpqmqC6B7yrFoijhnPNjj2/X+AxUevjzspsIjo+9b0q4SUV5eHu+99x5vvfUW559/fsRrb7/9NjNmzGD16tXMmjWrU4MUwkylNV6c/nLibDqWhL5RTf/RWMGRWvbsr8Hl95MiJaJmtatE9Mwzz/Czn/2sSRIC+Pa3v829997LmjVrOi04IXqCWm8QV6CyvqG6/dUyq26p79AoQzxa0q5E9Pnnn3PJJZe0+Pr06dP57LPPOhyUED3JaYOSuWqkI7R+fTsbqiHUoVGWFGpdu6pmZWVlZGRktPh6RkYG5eXlHQ5KiJ5GrytD1y2xlYgaD/GQ2/fNaleJKBgMYrW2nLt0XScQkA9aHGcMA9z1f2DjY6maaVI1a0O7SkRKKebMmdPivNEys6I43lS4fby1dQdnlFSSk54MjnbOzEj9kkIWK4YhY81a0q5ENHv27Db3kTtm4nhypMZLVelhauoCoTFmMSyOGBr0qtePvpdE1Jx2JaLc3NyuikOIHulIjY+4QCXOGO+YAQxMcXLFuMH0+Sw+1EakVLu7ABzvYp6hUYgTQWmND6e/Apej/WPMGjjtOoP6JoG9/utmBI5Opi+AGAe9CnGiKK314gxU4rLFXiICwNIo8Ug7UROSiIRoQSBoUF7rJ85fEaqaOWMrEdX5g3x6oIqial9og/QlakISkRAtKHf70YJeXMoTWsssxqqZ12/wztfF5JdLImqJJCIhWuANBEm31uK062j2eLA5YzqOrocapv1KRyG38JtjaiJatmwZZ555JomJifTr148ZM2awY8cOM0MSImxQHxczRydwav+kmKtlEOpZDWBouszS2AJTE9G7777L/PnzWb9+PW+88QZ+v5+LLrqI2tpaM8MS4ih3aWiOrQ40VDckIllSqGWm3r5/9dVXI57n5eXRr18/Nm/ezHnnnWdSVEI04i4L/Rtj+xCEOjQC9cM8AtJG1Iwe1Y+osjK0XEtqavM/dK/XGzGMpLmVRIToDIahePLDvZxevIexyQa2DpSINE07OvDV8EuJqBk9prHaMAzuuusuzj77bEaPHt3sPsuWLSM5OTn8yMrK6uYoxYmiqs5PtcePv+pIqGrVgTYikDmJ2tJjEtH8+fP54osvePbZZ1vcZ/HixVRWVoYfhYWF3RihOJGU1fqwB2txWQ00i97utcyOddlp/TlzWAYOqy5Vs2b0iKrZggUL+O9//8t7773HoEGDWtzP4XC0OPJfiM5UVtto+SBnClj0Dh0vO80FxfHg0WROomaYmoiUUvz4xz/mhRdeYN26deTk5JgZjhBhZbU+4vyVxHV0aEdjDcM8pGrWhKmJaP78+Tz99NO89NJLJCYmcujQISC0bprTGVvnMSE6Q7hE5NLB2afDxysorSVYVkd/fxCnNFY3YWob0apVq6isrGTq1Kn0798//PjHP/5hZljiBKeUoszduETUsYZqgK37Ktiyv4bqOr+0ETXD9KqZED2NP6jISIwjhWqcnVQ1s+kWfOF5q6VEdKwe0VgtRE9it1r47ukZUG0LTWLWwVv3ALb6eauDhty+b06PuX0vRI/iKQ8lIauj3Su7NsfW0I9I5q1uliQiIY5hGCpyaEcnTOtq0y0ELVaCSoER7PDxjjdSNRPiGC9+egDb/i84R/fRJ7Nzbt03VM0MqZo1S0pEQhyjrNaHcpeGBqt2QvsQgM0aqpoFZfR9s6REJEQjvoBBdV2Awf7KTrtjBjC0bzyppwwkfY9TElEzpEQkRCMVbh8oRaKqwqbHPj3ssVJcdoZk9CHeboWgr1OOeTyRRCREI2VuH1ajjkQ9EGqk7oRe1WG6PfRvUFZEPpYkIiEaKasNrWMWZ9PBkdRp6495fEG+PuKjtNYLgVCpSxwliUiIRspr/TjDK7t2TrUMoNLj59WvyygodYc2SPUsgjRWC9FI3wQ7CXFu4pW180bdE7p9r9AJNPztD3hDnSUFIIlIiAiThqZBrRWO2Drt1j2Ebt+jafhV/VdOSkQRpGomxLHcpaF/O7FqZtdDXzW/ZgsN9g5Ig3VjUiISol6dP4hFGdg9FaENnVo1CyWioGYnqAyscucsgpSIhKi3Kb+M3Dc3U1hWA1Y7OBI77di6RcOiaQQsttAI/IBUzRqTRCREvSM1Xpz+8lA1ytW3Uwa7NmazagQt9vo5iaRE1JhUzYQgNElfSbWXFH8FroROnKe6kQtPziDB0Q+bu1pKRMeQRCQEUO0NUOsNMjBQgctuhfi+nX6OERmJUJECdRYpER1DqmZCAAcrPAD002vQNS1UNesKDT215a5ZBElEQgBFFXWgDPpaakIb4ju/anaoso6CqgAef1D6ER1DEpEQwP4KD3GBKpIcGlis4Eju9HN8UlDGR/k1oRH+UiKKIIlInPCUUpyRncLJiW6S4myQkA6Wzv9qOKx6qB+RoaREdAxprBYnPE3TOHVAMngM2GeBxP5dch6H1ULQYiPol57VxzK1RPTee+/xne98hwEDBqBpGi+++KKZ4YgTXXVopWESMrrk8A6rhaBmJ2AouWt2DFMTUW1tLWPHjuX3v/+9mWGIE1hZrY+P95RS7fEdTUSJmV1yLntDiciQEtGxTK2aTZ8+nenTp5sZgjiBKaX4YPcR9hTXUFV6kIsDXtCtEJ/eJedzWHUCFkd9IqrrknP0Vr2qjcjr9eL1Hv1LUlVVZWI0ojczDMXH35Syp7gGi6YxqU8N1ALJWWDRu+ScDpuFgCUuVDUL+ELrm3XRuXqbXnXXbNmyZSQnJ4cfWVlZZockeqHqOj/Pb9nPxr2hRRTPHp5Girco9GJKdpedt1+ig2+flkVWav3KsX5Pl52rt+lViWjx4sVUVlaGH4WFhWaHJHqZwjI3azbs40C5B7vVwvTTMpmQnQzl+aEdujARJcbZOHVgH1KSEkIbpHoW1quqZg6HA4dDptcUsVFK8e7OEjy+IP2SHFw6uj994u1QuifUeGyPh8QBXR+I1Qn+OvC7u/5cvUSvSkRCdISmaXxn7AA2F5Rx3oh0rPWTlVH0Wejf9JFd0pGxgWEoCsrcxHk0MpTC4pcSUQNTE1FNTQ27d+8OP9+7dy+ffvopqampZGd3XRFZnLiSnTa+PapRP6HaI3BkZ+j/A8Z16bk1DV769AAjD9dxQYbCHpA2ogamthF98sknjBs3jnHjQr8ACxcuZNy4cSxZssTMsMRxRCnFK9uK2H6wKjRXdGO+WvjqP6E1xvqOgIR+XRqLpmnE2XQCuoNA0JDG6kZMLRFNnTq16S+HEJ1o24FKvj5Uze7iGgalOkNjyQJeKNwIhRtC69Db4mD4tG6Jx2nT8Vvi8AeVJKJGpI1IHLfq/EE+3B1akeOs4X1Jcljh0Bew5y3w1TcUJ2bCyOngTOmWmJy2UKfGgGHIXbNGJBGJ49bH35RS5w/SN9HBuIEJoWrY4S9DL7pSIec8SB/V6XNTt8Zp16kOl4jkrlkDSUTiuFRa4+XzwkoApuQkYvniOSgvAM0COedC1iRTejU7bTrlFgd+aSOKIIlIHJfe33UEQylOSoHsfWuhpiS0RNCpV0HqUNPictl1fHo8Ab8B3hrT4uhpJBGJ405xdR17j9SS5C/hfPdmMGpDnRXHfA8Su2aKj2gN65dAqmUw/XbGga8mdMeuG6uGPZUkInHc6efUmNV3J/59n+AyHKH2oDHf67YG6dZkJMWR4RoABdbQHbugD6wyWkASkTi+HNkNu14jra4KkhyQORqGXQB2l9mRHWW1hx4BX6h6JolIEpE4Tvg9+L56BePwduKseqj0c9LFprYHNccfNNhf7sHltZGh+8BX3SUrhvQ2kohE71d1EL74FwcOHuZwlY9+p5zL0AkXh0odPYw/aPDi1gOcUhzgwv4KizRYA71sGhAhmjiyGz5dg7u6ggK3g8/7XYF1xAU9MglB6Pa91aLhs7jwBYzQMBMhJSLRi5XugS/+Bcpgu7cvn2ecy+CMPmSn9aD2oGNomkZinBWf7sIbMIjzVpsdUo8giUj0TpX74cu1oAyKnUP5wD8Gi27lvBFdM990Z0qMs+G1JuINBKGuwuxwegSpmonep6YEPv8nBAME++TwSmA8StMZm5USmuish0ty2qizJoeqZp5ys8PpESQRid7FUwGfPxsaQZ88kC2J51PmMYh36EzKSTU7uqgkxlmpsyXhDRih65EZKCQRiV7E5w6VhLw1EN8XTrsWTbdhtWicOyKdOFvvWBEjMc6KV0+gLggYAairNDsk00kbkegdgn744nlwl4IjMdRT2uZkwhAnJ2UmkujoPb/KA1OcXHhqf7J3DwCjKlQ96wG9vs0kJSLR8wUD8OULUHkArA7UmOuo0+PDLyfF2dB60XitFJed0QOTSexTPyOkp8zcgHoASUSiZwsGQrfoS/eAbsUYfQ3vHdT4+/oCSmt6+bLNDVPTNix1fQLrPeVZceLx1cIXa6FyP4bFysFBl/PhbjhYEbrTdKDCQ1pC7xynVeH2cdgdT6bHT3J1kdnhmE4SkeiZyvaivn6ZyopSSjywwXUOJXt0oA671cK0kzMYmZlodpQx215UxZZCnUuq60h2HQndBTyBB79KIhI9h99DsHQv+uHPoWwvoPiqQuezlAvxWPvgsuuMyEhgwpDU0CT4vdigFBcbrPGUBF0MUwZaeX5oXbUTlCQi0f0MAyryoaIQ5SmnpvIIlaXFVFdX4/YFOCO7D5pFRxt4Bvqg0YwIWDgpI5GBKU4slt7TKN2azOQ4dIvGIetA3L79xJfulkRktt///vf85je/4dChQ4wdO5bHH3+ciRMnmh1Wz1VTElqr3VMemoPZlUqB6oeK60P/lDgc1h7Yn8YwoLIQir+Ckq/xuGsorq6jtMYX6thXz2NLoSxtHGknfQucfThefwvsVgs5feM57B7Kkeo9xBdvD82bZIszOzRTmJ6I/vGPf7Bw4UL++Mc/MmnSJFasWMHFF1/Mjh076Nevaxe861X8dVDyFXWFWykryqfWG2B4vwQ0QiWEusPV7PMnsylhOK5Boxk8IJNh6Qk47U2TkreulqryUuqqy7B4q3DYrbhcCTiTUtHi05u0VQQNhccfxOMLopQi2WWLLtkF/VB1IDRCvuSr8BzNRZUedpcblDuz8ST0IeBIpk9aOpkZ/RmVkdorhml0hlMGJLH7cCb5lS76+3zYCz6E4ReYHZYpNGXyCoeTJk3izDPP5He/+x0AhmGQlZXFj3/8Y+69995W31tVVUVycjKVlZUkJSV1R7jdJxhA1VXiK9+Pu2gn7oNfUVnrocrjx0CjMm4Qk8eeQkqcBaoPsT9/JyVVHurqSxduWxpuex+SEhLITLAyMlUHbyVBdwWbdh+iuR+6RQstyTxqSBbEp6Pi+/LcV3VUqzgCFgdK07GoIBYVINkaYHAiTBzkDC2L43ejfG40vzu0OoXfTTDgw+sPUuMNEGfTSUpIgPSRlCUM429faQxJT+DUAUkMSYs/ug79CUQpxbObCqk7tIMJVW8xekASWs45MHB8aI7t40C031FTS0Q+n4/NmzezePHi8DaLxcK0adP4+OOPm+zv9Xrxeo/2HamqqoruRGV7Yc9b7Ct1U1TZsITLMV9FpThtYDIuuxVQ7C93s7/cg4aK3LM+b586MIlEe+jjO1jpYV+Z+5gxQ0f/f3JmIsnOUOPqoao68o/UHn298XuUYmRmIn1cNvDXcajSQ37p0bWvPLYUilNGYh1wGsMHZRKXmQj1wxoGnVLDwJKvqSncRmXRN5TVVlBbWwq14K+0gkoGQAccNgteLQ6cyQSsifgCQQJeN3H+CqB+yEFdJVrpboYeKcMfVGiAVQ+VvvzBUMzJ8XbwJdZfrWLDN2VYLKEymqEUhgK/xUmFM4s+g0dz7sSJoFtJBW7rF6j/rE9cmqZxyamZ/MPjx5FWi+b7CvI/pOrrdewqCxDUHYCG0iyAVv+AIX3jyUgKlVor3H52HG55KpHsVBf9k0PVveq6ANuLWv7ODEpxMbBPaN9ab5AvDjY/9GRcVgp2qwVOuw7iOqcAYOpvwpEjRwgGg2RkRK6skJGRwddff91k/2XLlrF06dL2nyjghZoSVG0tRk0rq2u6AxAIfSTK7Ua5Pc2WHADwGGDU37mp86A8rSyW59XAUr+vtw7lbWUyLL8V/EEAbHYHNfYEvAkDsWWeQlpmNuMzkkh2NXPHyJGANmgCiYMmkOitZlBVETUVxVRU1xDUdMhIDw2NiEvmtLMTsdqbVr9q6gIYvlqgCmpLoPYIp7gOYzM8WA0vmgqCxYpfWagOWtHs8ZCcDDYXfksceypKQ8sp63H4LU4CehxWexyp8Q4GZiSAfvTX7URPQg36xNu56VuDibflQMkIKNyAv3Yfvjo30MLvVFw8WOrbkjw+jOqWE5Gyu0B3hp7U+TGqW05EyuoEW/1cTr4ARnULY+Bq/aBbQBnNvx6DXvXbsHjxYhYuXBh+XlVVRVZWVttvTMmCsdfTp86PtS704WkaTZZxsSY4wBqqIqR4g9i8AY7u3JiGPd4Gug6aRrIvwDBvMPza0d1DpYM4pxWsoY862R9khC+ACr969Pga4IqzgVUHm4sUi4MpaKG/Pu3hSIT0RBLSTyKhmZeb+6HrFi2U4FwpQAqkZAPQ3BRjNuDYce42pbhqUBBfwMBQoeM5rBYcVkuvGn5hhoSGcXIZp0LGqTjdNWSXlaEFfYARKjU3+tLHO61Qn8jj/AbZtb6Wj+20Qv3x7QGD7JEt75sYZ4W40L7WoEH2iOb3tSQ5QvX4Tqw+mpqI+vbti67rHD58OGL74cOHyczMbLK/w+HA4Yih05c9HlJzSAKiLUgm1j+iEV//iIaz/hGN3vRXQtM0XHYrrhOjnblLuVwJuFzN/QlpKg4YGOVxHcDAKO//OICB3TjHnKkthHa7nfHjx/PWW2+FtxmGwVtvvcXkyZNNjEwI0Z1M/6O7cOFCZs+ezYQJE5g4cSIrVqygtraWm2++2ezQhBDdxPRE9L3vfY+SkhKWLFnCoUOHOP3003n11VebNGALIY5fpvcj6ojjuh+REMeBaL+jJ14vMiFEjyOJSAhhOklEQgjTmd5Y3RENzVtRD/UQQnSrhu9mW03RvToRVdd3bY+qd7UQwjTV1dUkJye3+HqvvmtmGAYHDx4kMTGx1wwjaBiWUlhYeFzd6Tser+t4vCbo3utSSlFdXc2AAQOwWFpuCerVJSKLxcKgQYPMDiMmSUlJx9Uvd4Pj8bqOx2uC7ruu1kpCDaSxWghhOklEQgjTSSLqZg6Hg/vuuy+2WQR6sOPxuo7Ha4KeeV29urFaCHF8kBKREMJ0koiEEKaTRCSEMJ0kIiGE6SQRdbEHH3yQs846C5fLRUpKSlTvUUqxZMkS+vfvj9PpZNq0aezatatrA22nsrIybrzxRpKSkkhJSeHWW2+lpqam1fdMnToVTdMiHj/4wQ+6KeLm/f73v2fIkCHExcUxadIkNm7c2Or+zz33HKNGjSIuLo7TTjuN//3vf90Uafu057ry8vKa/Fzi4rp5xVklutSSJUvU8uXL1cKFC1VycnJU73n44YdVcnKyevHFF9Vnn32mrrjiCpWTk6M8Hk/XBtsOl1xyiRo7dqxav369ev/999Xw4cPVzJkzW33PlClT1G233aaKiorCj8rKym6KuKlnn31W2e129eSTT6ovv/xS3XbbbSolJUUdPny42f0//PBDpeu6+vWvf622b9+u/u///k/ZbDa1bdu2bo68de29rtzcXJWUlBTxczl06FC3xiyJqJvk5uZGlYgMw1CZmZnqN7/5TXhbRUWFcjgc6plnnunCCKO3fft2BahNmzaFt73yyitK0zR14MCBFt83ZcoUdeedd3ZDhNGZOHGimj9/fvh5MBhUAwYMUMuWLWt2/+uuu05ddtllEdsmTZqkbr/99i6Ns73ae13R/m52Jama9TB79+7l0KFDTJs2LbwtOTmZSZMmNbv6rRk+/vhjUlJSmDBhQnjbtGnTsFgsbNiwodX3rlmzhr59+zJ69GgWL16M293KwpRdqGGV4cafc2urDEPouhvvD3DxxRf3mJ8LxHZdADU1NQwePJisrCyuvPJKvvzyy+4IN6xXD3o9Hh06dAig2dVvG14z26FDh+jXL3KBLKvVSmpqaqsx3nDDDQwePJgBAwbw+eefc88997Bjxw7Wrl3b1SE30d5VhiF03T355wKxXdfIkSN58sknGTNmDJWVlTzyyCOcddZZfPnll902qFxKRDG49957mzTuHfto6Yfek3X1dc2bN4+LL76Y0047jRtvvJHVq1fzwgsvsGfPnk68CtFekydPZtasWZx++ulMmTKFtWvXkp6ezp/+9Kdui0FKRDG4++67mTNnTqv7DB06NKZjN6xwe/jwYfr37x/efvjwYU4//fSYjhmtaK8rMzOT4uLiiO2BQICysrJmV+htyaRJkwDYvXs3w4YNa3e8HdHeVYYh9LNpz/5miOW6jmWz2Rg3bhy7d+/uihCbJYkoBunp6aSnd816vDk5OWRmZvLWW2+FE09VVRUbNmzghz/8YZecs0G01zV58mQqKirYvHkz48ePB+Dtt9/GMIxwconGp59+ChCRcLtL41WGZ8yYARxdZXjBggXNvmfy5Mm89dZb3HXXXeFtb7zxRo9alTiW6zpWMBhk27ZtXHrppV0Y6TFMbSo/ARQUFKitW7eqpUuXqoSEBLV161a1detWVV1dHd5n5MiRau3ateHnDz/8sEpJSVEvvfSS+vzzz9WVV17ZI2/fjxs3Tm3YsEF98MEHasSIERG37/fv369GjhypNmzYoJRSavfu3eqBBx5Qn3zyidq7d6966aWX1NChQ9V5551n1iWoZ599VjkcDpWXl6e2b9+u5s2bp1JSUsK3rr///e+re++9N7z/hx9+qKxWq3rkkUfUV199pe67774ee/u+Pde1dOlS9dprr6k9e/aozZs3q+uvv17FxcWpL7/8sttilkTUxWbPnq2AJo933nknvA+gcnNzw88Nw1A///nPVUZGhnI4HOqCCy5QO3bs6P7gW1FaWqpmzpypEhISVFJSkrr55psjkuvevXsjrnPfvn3qvPPOU6mpqcrhcKjhw4ern/70p6b2I1JKqccff1xlZ2cru92uJk6cqNavXx9+bcqUKWr27NkR+//zn/9UJ510krLb7erUU09VL7/8cjdHHJ32XNddd90V3jcjI0NdeumlasuWLd0ar0wDIoQwndw1E0KYThKREMJ0koiEEKaTRCSEMJ0kIiGE6SQRCSFMJ4lICGE6SURCCNNJIhK9Sl5eXsSUu/fff3/EYOA5c+aEx1iJ3kMSkWjWnDlzWpxTev78+WiaFjFSvysSwJAhQ1ixYkXEtu9973vs3LmzxfesXLmSvLy88POpU6dGDFIVPZMkItGirKwsnn32WTweT3hbXV0dTz/9NNnZ2abE5HQ6m0zK1lhycnLUixSInkMSkWjRGWecQVZWVsQMimvXriU7O5tx48Z16NjNlVRmzJgRLmVNnTqVgoICfvKTn4QnZYOmVbNjNS6ZzZkzh3fffZeVK1eGj7F3716GDx/OI488EvG+Tz/9FE3TunUOHnGUJCLRqltuuYXc3Nzw8yeffJKbb765y8+7du1aBg0axAMPPEBRURFFRUXtPsbKlSuZPHkyt912W/gY2dnZTa4JIDc3l/POO4/hw4d31iWIdpBEJFp100038cEHH1BQUEBBQQEffvghN910U5efNzU1FV3XSUxMJDMzM6ZZEJOTk7Hb7bhcrvAxdF1nzpw57NixI7zWl9/v5+mnn+aWW27p7MsQUZIZGkWr0tPTueyyy8jLy0MpxWWXXUbfvn3NDqtDBgwYwGWXXcaTTz7JxIkT+c9//oPX6+Xaa681O7QTlpSIRJtuueUW8vLyeOqppzqt1GCxWDh2Kiy/398px47G3Llzww3xubm5fO9738PlcnXb+UUkSUSiTZdccgk+nw+/38/FF1/cKcdMT0+PaPcJBoN88cUXEfvY7XaCwWCHztPSMS699FLi4+NZtWoVr776qlTLTCZVM9EmXdf56quvwv9vSWVlZXhC/AZpaWlkZWU12ffb3/42Cxcu5OWXX2bYsGEsX76cioqKiH2GDBnCe++9x/XXX4/D4YipSjhkyBA2bNhAfn4+CQkJpKamYrFYwm1FixcvZsSIET1qAvwTkZSIRFSSkpJISkpqdZ9169Yxbty4iMfSpUub3feWW25h9uzZzJo1iylTpjB06FDOP//8iH0eeOAB8vPzGTZsWMyrpixatAhd1znllFNIT09n37594dduvfVWfD5ft9wFFK2TOavFCev999/nggsuoLCwsMnKqKJ7SSISJxyv10tJSQmzZ88mMzOTNWvWmB3SCU+qZuKE88wzzzB48GAqKir49a9/bXY4AikRCSF6ACkRCSFMJ4lICGE6SURCCNNJIhJCmE4SkRDCdJKIhBCmk0QkhDCdJCIhhOn+P6kCrympvhjlAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:44:40.452057Z", + "iopub.status.busy": "2024-03-22T13:44:40.451681Z", + "iopub.status.idle": "2024-03-22T13:44:40.811712Z", + "shell.execute_reply": "2024-03-22T13:44:40.810790Z" + }, + "papermill": { + "duration": 0.38307, + "end_time": "2024-03-22T13:44:40.813861", + "exception": false, + "start_time": "2024-03-22T13:44:40.430791", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+lElEQVR4nO3deXxU9b34/9eZM3symRAIWSCBIIgiq1IobqCiKF4Vbau1LuDaVuyt5tIq/d6i2FpsXa5c9aK1BfRXldYWta1ValGhoiKCqCiLRMKaQMg+mcx6zu+PkwwZMkkmk2Um4f18POaRzJkz57xPknnn8/mcz6Louq4jhBBJZEp2AEIIIYlICJF0koiEEEkniUgIkXSSiIQQSSeJSAiRdJKIhBBJJ4lICJF05mQH0BWapnHo0CFcLheKoiQ7HCHEcXRdp76+nvz8fEymtss9fToRHTp0iIKCgmSHIYTowP79+xk6dGibr/fpRORyuQDjIjMyMpIcjRDieHV1dRQUFEQ+q23p04mouTqWkZEhiUiIFNZR04k0Vgshkk4SkRAi6SQRCSGSrk+3EcVD13VCoRDhcDjZoYg+yGKxoKpqssPo9/p1IgoEApSVleH1epMdiuijFEVh6NChpKenJzuUfq3fJiJN09izZw+qqpKfn4/VapVOj6JTdF2noqKCAwcOMGrUKCkZ9aB+m4gCgQCaplFQUIDT6Ux2OKKPys7OprS0lGAw2GEiqvUGeXvnYUYNdjF2iLuXIuwf+n1jdXvdyoXoSGdK0R+VVlF61Mum0irCmkwF3xnyKRWim5TXNqIocO7J2agmaQboDElEQnSDUFijsiGArkNuhj3Z4fQ5kohOEKWlpSiKwtatW3vlOBs2bGDcuHFYLBbmzJnTpXP2BQ3+MLoOFlXBaZVG7c5KeiI6ePAg119/PQMHDsThcDBu3Dg+/vjjZIfV77377rsoikJNTU2PHL+4uJiJEyeyZ88eVq5c2SPnSCX1/iAAwbDOCxv38c6OI0mOqG9J6l2z6upqzjrrLM477zzeeOMNsrOz+eqrrxgwYEAyw0o5gUAAq9Wa7DA6paSkhB/84AftTv3Qkd68bl3XCYfDmM2JfSQ8/lDk+4p6P3aLlIo6I6klol//+tcUFBSwYsUKpkyZQlFRERdddBEnnXRSj543ENLafITCWtz7BuPYNxEzZszgzjvv5K677mLQoEHMmjWLbdu2cckll5Cenk5OTg433HADR48ejbznzTff5OyzzyYzM5OBAwfyH//xH5SUlMQ8fmlpKeeddx4AAwYMQFEU5s2b16nj7NixgzPPPBO73c7YsWNZt25d5NiKolBZWcnNN9+MoiiREtG6deuYMmUKNpuNvLw87r33XkKhYx/gWNfdXHJbs2YNkyZNwuFwcP7553PkyBHeeOMNTj31VDIyMvje974X1XFV0zSWLFlCUVERDoeDCRMm8Oc//znyevNx33jjDc444wxsNhvvvfdeQr8vgGBIx6IquK06OfVfEvRUJXysE1FSS0R//etfmTVrFt/5zndYt24dQ4YM4Y477uC2227r0fM+9c7uNl8rGpTGnElDIs9/u76EYDj2rdihAxx8Z/KxidmWb9hDYyB6KMndF56cUIzPPfccP/zhD9mwYQM1NTWcf/753HrrrfzP//wPjY2N3HPPPVx99dW8/fbbADQ0NFBcXMz48ePxeDwsWrSIK6+8kq1bt7bqwlBQUMBf/vIXvvWtb7Fz504yMjJwOBydOs5PfvITHn/8ccaMGcNjjz3GZZddxp49eygoKKCsrIzRo0fzwAMPcM011+B2uzl48CCzZ89m3rx5PP/88+zYsYPbbrsNu93O/fffH/O6AcrKygC4//77efLJJ3E6nVx99dVcffXV2Gw2XnzxRTweD1deeSVPPPEE99xzDwBLlizhD3/4A08//TSjRo1i/fr1XH/99WRnZzN9+vTI+e69914eeeQRRowY0aWS+LihbsYOyaD283+wffd7BL1fgD4OpBNtXJKaiL7++muWLVtGcXExP/vZz9i0aRP/+Z//idVqZe7cua329/v9+P3+yPO6urreDLdXjRo1it/85jcA/PKXv2TSpEn86le/iry+fPlyCgoK2LVrFyeffDLf+ta3ot6/fPlysrOz+fLLLxk7dmzUa6qqkpWVBcDgwYPJzMyMvBbvce68887IvsuWLePNN9/k97//PT/96U/Jzc1FURTcbje5ubkA/N///R8FBQU8+eSTKIrCKaecwqFDh7jnnntYtGhRJMm1vG44loh++ctfctZZZwFwyy23sHDhQkpKShgxYgQA3/72t3nnnXe455578Pv9/OpXv+Jf//oX06ZNA2DEiBG89957PPPMM1GJ6IEHHuDCCy/s+BcSB0VRcNbtAcDir0bzVmNKy+qWY/d3SU1EmqYxefLkyAds0qRJbNu2jaeffjpmIlqyZAmLFy/u8nnnnzeyzdeO7/5x+7ltVxOP/2d381lFXQkryhlnnBH5/tNPP+Wdd96JOd6ppKSEk08+ma+++opFixaxceNGjh49iqYZ1cJ9+/a1SkTtifc4zR9wALPZzOTJk9m+fXubx92+fTvTpk2L6iB41lln4fF4OHDgAIWFha2uu6Xx48dHvs/JycHpdEaSUPO2jz76CIDdu3fj9XpbJZhAIMCkSZOitk2ePLnNmDst0IA5cOyfY7C2DJskorgkNRHl5eUxZsyYqG2nnnoqf/nLX2Luv3DhQoqLiyPPm6eh7CyrOf6msZ7atyNpaWmR7z0eD5dddhm//vWvW+2Xl5cHwGWXXcawYcN49tlnyc/PR9M0xo4dSyAQ6NR5u+s4iWp53S1ZLJbI94qiRD1v3tacND0eDwCvv/46Q4YMidrPZrPFdb7OenNbOUr9Ic4OhlEVCOsQrKvAlt8th+/3kpqIzjrrLHbu3Bm1bdeuXQwbNizm/jabrdUf0ong9NNP5y9/+QvDhw+PeVensrKSnTt38uyzz3LOOecAdNjw2nw3quX0KJ05zocffsi5554LQCgUYvPmzdx5551tnq/5H4yu65FS0YYNG3C5XF26sxbLmDFjsNls7Nu3L6oa1pMOVHuxVh1Gt4HdoqLpOrq/vlfO3R8k9a7Z3XffzYcffsivfvUrdu/ezYsvvshvf/tb5s+fn8ywUs78+fOpqqri2muvZdOmTZSUlLBmzRpuuukmwuEwAwYMYODAgfz2t79l9+7dvP3221Elx1iGDRuGoij8/e9/p6KiAo/H06njPPXUU7zyyivs2LGD+fPnU11dzc0339zm+e644w7279/Pj370I3bs2MFrr73GfffdR3FxcbePB3S5XCxYsIC7776b5557jpKSErZs2cITTzzBc889163nauYLhrGGPJhNCuOHZjKxYAAupbFHztUfJTURfeMb3+CVV17hpZdeYuzYsfziF7/g8ccf57rrrktmWCknPz+fDRs2EA6Hueiiixg3bhx33XUXmZmZmEwmTCYTq1atYvPmzYwdO5a7776bhx9+uN1jDhkyhMWLF3PvvfeSk5PDnXfe2anjPPTQQzz00ENMmDCB9957j7/+9a8MGjSo3fP94x//4KOPPmLChAn84Ac/4JZbbuG///u/u/SzacsvfvELfv7zn7NkyRJOPfVULr74Yl5//XWKirqvHa9ZKKwRDOvYQh7MqgKuHOMFf/+9mdLdFF3X++ww4bq6OtxuN7W1ta1W8fD5fOzZs4eioiLsdhn7IxITz99RvS/I7/69h1OOruHi3HqUIWfAwS1gccDZd/VuwCmmvc9oS/12PiIheosvaDSSO/CjoPBFrZXwgRpy3H5ydF36EsUh6WPNhOjrfEGjwd+B0cetzuSiIRDGHwxDyN/eW0UTSURCdFFIM4Z32JsSkepwoylmwpoGIWmwjockIiG6qGhQGnfOGMH4HKMNyWx1EjLZCGtA0Jfc4PoISURCdIdgo9E/SlFQrQ4jEek6BGUFmXhIIhKiOwSbqmBmOxazmZDJhqbrEJISUTwkEQnRRZ8dqOGfn+6hwuMHixOLqhBS7WiaLlWzOEkiEqKLKur9lB2tMu6eWRzYzCqqzWFMoC+N1XGRfkRCdFEgpGEJ+4zEY3FQONBJ4YThsO+wlIjiJCWiFDRjxgzuuuuuZIcRkWrxpJpAWMOs+TCbFLA0LeapNg3ODvfOrAV9nSSifqq3pu0Q4A8aiai5RASAuTkRSYfGeEgiSjHz5s1j3bp1LF26FEVRUBSFkpISbrnllsj8y6NHj2bp0qWt3jdnzhwefPBB8vPzGT16NADvv/8+EydOxG63M3nyZF599dVWywG1Nx92rHhKS0t768fRJ/jDGmYtYCQisx1fMMw/d1Tz+cFadOlZHZcTq41I1yEc7P3zqpa4xxstXbqUXbt2MXbsWB544AHAmOB+6NChvPzyywwcOJD333+f22+/nby8PK6++urIe9euXUtGRgZvvfUWYAw4vOyyy5g9ezYvvvgie/fubVXF6mg+7FjxZGdnd8MPpf8IhDTUSCKyoihQ1qDh8ocIhwIn2IcsMSfWzygchH8/2vvnPee/wBzfsjhutxur1YrT6YzM9wxETZFbVFTEBx98wJ/+9KeoRJSWlsbvfve7yKRnTz/9NIqi8Oyzz2K32xkzZgwHDx6MWpzgySef7HA+7FjxiGN0XcesB1AVo0RkMZkIK8YMklpAGqvjcWIloj7sqaeeYvny5ezbt4/GxkYCgQATJ06M2mfcuHFR64Dt3LmT8ePHR01fMWXKlKj3xDMftmjfreeMQNuUieLxgWrDZFJQLE0zYAalahaPEysRqRajdJKM83bBqlWrWLBgAY8++ijTpk3D5XLx8MMPs3Hjxqj9Epl/OZ75sEXHTOEAoEQaqU1mI/lrcvs+LidWIlKUuKtIyWS1WqPmkt6wYQNnnnkmd9xxR2RbW4sntjR69Gj+8Ic/4Pf7I3N9b9q0KWqfjubDjhWPiKF5KEdTAjJZjJ+3FpK7l/GQu2YpaPjw4WzcuJHS0lKOHj3KqFGj+Pjjj1mzZg27du3i5z//eauEEsv3vvc9NE3j9ttvZ/v27axZs4ZHHnkEIDKBfUfzYceKp3m1DAF1viCvfXKAkrKmlV2b/tGZmr5q4TCEQ229XTSRRJSCFixYgKqqjBkzhuzsbGbNmsVVV13FNddcw9SpU6msrIwqHbUlIyODv/3tb2zdupWJEyfy//7f/2PRokUAkXajjubDjhXPvn37eu7i+xivP0zpkTpqvdElIpvdgUVV0NGlL1EcZM7qE8wLL7zATTfdRG1tbWSZadG2jv6O9lV6+eumXZxT8RITCgbA9HuOddVY/4hxp/abPwBH4stZ92UyZ7UA4Pnnn2fEiBEMGTKETz/9NNJHSJJQ9wiEw8f6EKnW6P5iqtVIRNJO1CFJRP1ceXk5ixYtory8nLy8PL7zne/w4IMPJjusfsMf1ZnxuMU/VSvQAFoSOtH2MZKI+rmf/vSn/PSnP012GP1WMKwbwzuU6ES0ZV81+oF6Cqw+BiejN38fk9TG6vvvvz8yfqn5ccoppyQzJCE6JRDSUPXmqtmxRFTrDVLVqOMPaskZVtTHJL1EdNppp/Gvf/0r8rytvixCpKLQcQNem5lVBb/JYsxbLVWzDiX9U282m3t0DFMfvikoUkBHfz9njhzEVMtgKHFGdZY1m0xoitl4v5SIOpT0fkRfffUV+fn5jBgxguuuu67dPip+v5+6urqoR1ssFmNYhdcrqyiIxDXP66Sqapv7qNqxAa/NzKpirG2mA5p0aOxIUktEU6dOZeXKlYwePZqysjIWL17MOeecw7Zt23C5XK32X7JkSdQo9PaoqkpmZiZHjhwBwOl0RnoTCxEPTdOoqKjA6XS232TQPOeQ2rJEpBBWzMYE+jJLY4eSmoguueSSyPfjx49n6tSpDBs2jD/96U/ccsstrfZfuHAhxcXFked1dXUUFBS0efzmKl9zMhKis0wmE4WFhW3+E3u/5Cj2r8s5WQ+R3rJEZDKhmcxoYamaxSPpbUQtZWZmcvLJJ7N79+6Yr9tstsjgzXgoikJeXh6DBw8mGJQ/BtF5Vqs1MtQlln2VXlyVtQzP0KLaiCxmBcVkQdEUaayOQ0olIo/HQ0lJCTfccEO3HldV1Xbr+EIkKhDWjEnRjrtrdkpuBqdMHg579kuJKA5JbaxesGAB69ato7S0lPfff58rr7wSVVW59tprkxmWEHGLmiZWPa60bmqah0oSUYeSWiI6cOAA1157LZWVlWRnZ3P22Wfz4YcfypzIos8IhJsTkSnGEI+mRCRVsw4lNRGtWrUqmacXokt0XScY0lG1ICbFFpWIarwBPtldzdAj9YzKkkTUkaT3IxKirwppOpquY9b9x0bfNwmGdfbVBKlrDErVLA6SiIRIUDCsoeghFF1rPcTDpBBWLE0dGiURdSSl7poJ0Zc4rWbuPKcAfcMAFKJLRM09q40OjdKzuiOSiIToArMeBNVkNEy36G9kUZvGmgFayC9Vjw7Iz0eIrmgevnHcHTPVpKCZjP/zmrQRdUgSkRAJOlznY92X+zlY0xjVPgTH2ohAlhSKhyQiIRJU4w2yp7yaGm8gqn0IjOFFJtWCAmjhEMgSTO2SNiIhEhQ1O+PxnRmB7583GtN7A40nWhBM8Y+TPNFIiUiIBBm9qoOt+hA1M5ktx1b1kHaidkkiEiJBgVDsaWIjFAWaGqylL1H7JBEJkaBguGXVrHWJ6N9fVfDFYS8N/pCUiDogiUiIBEVG3isxRt4DB6obqfBq+MOykkdHJBEJkaBgyzaiGI3VZpNybAJ9qZq1S+6aCZGgC8fkEA5lolZVxmysjkygrwVlmEcHJBEJkSCzasJMyBjaEaOx2mwyGRPoy8DXDknVTIiuCPmMrzEaq5urZpqsbdYhSURCJGj9rgp2HarEFwrHbKw2Nw181TRd1jbrgCQiIRK0o7yOw1V1hDU9dmO1agx81WWRxQ5JG5EQCQoEw0237x0xE9E5IwehaAWYyqukatYBSURCJEDTdMKhIAp67BU8MKpmRoKStc06IlUzIRLQvHoHGKvBRlbsOJ6paT09uX3fLklEQiTAGN4RRAFMFtuxwa0t7Kv0snFfHYdqG6VE1AFJREIkoOXCikqM9iGA2sYge6oC1PtkrFlHUiYRPfTQQyiKwl133ZXsUIToUDCst1hqOnYiUk0tJtCXElG7UqKxetOmTTzzzDOMHz8+2aEIEZfBLhvXnj4Y05cZMRuqASxqiw6NWriXI+xbkl4i8ng8XHfddTz77LMMGDAg2eEIEReTScFhCmMzq+2XiExNQzykataupCei+fPnc+mllzJz5swO9/X7/dTV1UU9hEia5knxYwx4hZZjzaRq1pGkVs1WrVrFli1b2LRpU1z7L1myhMWLF/dwVEJ0bM/RBmr2lFPY4GdgrNkZabHIoow161DSSkT79+/nxz/+MS+88AJ2e+xf5PEWLlxIbW1t5LF///4ejlKI2MpqG9lTXkVdYyjmgFdoHvSqGgt4yBCPdiWtRLR582aOHDnC6aefHtkWDodZv349Tz75JH6/H1VVo95js9mw2WQlBJF8/pCGWfOjWtqYrxoYmG7j+jNHYt2aKSWiDiQtEV1wwQV8/vnnUdtuuukmTjnlFO65555WSUiIVOIPGv2IzG1NnI/RWO2w243OjtJG1K6kJSKXy8XYsWOjtqWlpTFw4MBW24VINYGwhk3ztb2CR7PmVTxkiEe7kn7XTIi+yB8MH1tKyBI7EYXCGu98Vc3uCg+aFpa+RO1IqET09ddfM2LEiO6OhXfffbfbjylET2huIzK3MU0sGMtOf1rWgL3ez/CBTkxa6NggWBEloRLRyJEjOe+88/jDH/6Az+fr7piESHmRsWZq21UzkwIoKqBIp8YOJJSItmzZwvjx4ykuLiY3N5fvf//7fPTRR90dmxAp67uT8zljaBpOq7ndEpHFrDbdwpdOje1JKBFNnDiRpUuXcujQIZYvX05ZWRlnn302Y8eO5bHHHqOioqK74xQipThNIRwW1VhcsZ3GajVqAn1psG5LlxqrzWYzV111FS+//DK//vWv2b17NwsWLKCgoIAbb7yRsrKy7opTiNQSbLF6h6ntj5HZpMiSQnHoUiL6+OOPueOOO8jLy+Oxxx5jwYIFlJSU8NZbb3Ho0CGuuOKK7opTiJTRGAjz/s4D7K/2gtnR7r7myMBXGebRnoTumj322GOsWLGCnTt3Mnv2bJ5//nlmz55tTJkJFBUVsXLlSoYPH96dsQqREhoCIXYeqMBS66OgoP2e/pElhXS/DPNoR0KJaNmyZdx8883MmzePvLy8mPsMHjyY3//+910KTohU1HzHzGxSwNJ+iWjOpCFYTHlYPAclEbUjoUT01ltvUVhYGCkBNdN1nf3791NYWIjVamXu3LndEqQQqSQyzkxte3bGZuk2M9hs0KBI1awdCbURnXTSSRw9erTV9qqqKoqKiroclBCprDEQxqz5sZhMHbYRAceGeUhjdZsSKhHpuh5zu8fjiXtKDyH6Kl/ISETmOEpEXx6qQytvoCAYxC2379vUqURUXFwMGB21Fi1ahNPpjLwWDofZuHEjEydO7NYAhUg1vmCLRNRBG9HeygYClQGy7CHcUiJqU6cS0SeffAIYJaLPP/8cq/XYhFBWq5UJEyawYMGC7o1QiBRjJCIfZqsJLM529zWrJnzNK3lIG1GbOpWI3nnnHcCYN2jp0qVkZGT0SFBCpLKzRg4iVJWGxVvfcSJq6kekS4fGdiXURrRixYrujkOIPsNmVrGZAqCawNpRiUiGeMQj7kR01VVXsXLlSjIyMrjqqqva3Xf16tVdDkyIlBZoML52UCKKXmRRElFb4k5EbrcbpWl9b7fb3WMBCZHq3t9VTkFFLTluO9YOGquPLSmEVM3aEXcialkdk6qZOJFt21uOUtPIoAxH+9PEIlWzeCXUobGxsRGv1xt5vnfvXh5//HH++c9/dltgQqSiUFiDgPG3b7Y7jYnx2zEmL4PLTy9g+MA0KRG1I6FEdMUVV/D8888DUFNTw5QpU3j00Ue54oorWLZsWbcGKEQq8YU0zGEfCmC2pXe4v92ikuF0YlFNcvu+HQnP0HjOOecA8Oc//5nc3Fz27t3L888/z//+7/92a4BCpBKvP4RZ82FRFZQO7phFqDLEoyMJJSKv14vL5QLgn//8J1dddRUmk4lvfvOb7N27t1sDFCKVeANhLFqjUcLpoKEaoLohwKZ9Hg7VNkobUTsSnjz/1VdfZf/+/axZs4aLLroIgCNHjkgnR9GvNQRCWMNeLGYTWF0d7l/vC/HJQQ8V9X4pEbUjoUS0aNEiFixYwPDhw5k6dSrTpk0DjNLRpEmT4j7OsmXLGD9+PBkZGWRkZDBt2jTeeOONREISold4A2EsYa9RIoqjjUhVm6eKlX5E7UmoZ/W3v/1tzj77bMrKypgwYUJk+wUXXMCVV14Z93GGDh3KQw89xKhRo9B1neeee44rrriCTz75hNNOOy2R0IToURMLMhlz2Ila5wBrx4nI3NShUdeRqlk7El5yOjc3l9zc3KhtU6ZM6dQxLrvssqjnDz74IMuWLePDDz+URCRSkkU1YVF8YFbjKhE1J6Jwc89qTWt3sv0TVUKJqKGhgYceeoi1a9dy5MgRNE2Lev3rr7/u9DHD4TAvv/wyDQ0Nkare8fx+P36/P/K8rq6u0+cRosv89cbXuEpEJjRFPTaHlxYCk7X9N52AEkpEt956K+vWreOGG24gLy8vMvQjEZ9//jnTpk3D5/ORnp7OK6+8wpgxY2Luu2TJEhYvXpzwuYToqg++KmNoRTWDXTZscSQitalndVgHHR1FCwKSiI6XUCJ64403eP311znrrLO6HMDo0aPZunUrtbW1/PnPf2bu3LmsW7cuZjJauHBhZHI2MEpEBQUFXY5BiHh9WXoIqhsZmJHW4eyMYFTNUJSmUhEo0qkxpoQS0YABA8jKyuqWAKxWKyNHjgTgjDPOYNOmTSxdupRnnnmm1b42mw2breNfvhA9IRDS0P3GqHurM6PD4R0AVtXEdd8sJF3NRlECcuesDQm1mv3iF79g0aJFUePNuoumaVHtQEKkinpfEFuoHrNJweyIbwYKk0lhsMuO025HQZFE1IaESkSPPvooJSUl5OTkMHz4cCwWS9TrW7Zsies4Cxcu5JJLLqGwsJD6+npefPFF3n33XdasWZNIWEL0qDpfCFuoHqvZBPZOToWjNn1GpGoWU0KJaM6cOd1y8iNHjnDjjTdSVlaG2+1m/PjxrFmzhgsvvLBbji9Ed2ouEdk6mYi27KvGXemnwKphld7VMSWUiO67775uObmsBCv6knpfCFvYg82qgiMz7vdt2VtNYZWfnEEaVunUGFPCPatqamr43e9+x8KFC6mqqgKMKtnBgwe7LTghUkm9L4g9VNfpEpFqUtAVtWmYh5SIYkmoRPTZZ58xc+ZM3G43paWl3HbbbWRlZbF69Wr27dsXmatIiP7kglOyCZVZUBVzpxKRWTUdG28mbUQxJVQiKi4uZt68eXz11VdRK7vOnj2b9evXd1twQqQSS6gBh1nBarHENfK+WfMwD2PeaqmaxZJQItq0aRPf//73W20fMmQI5eXlXQ5KiJTkqzW+2jI6NV5MjQx8lRJRWxJKRDabLeY4r127dpGdnd3loIRINWW1jWz4/CuO1Ps61VANYFGPG/gqWkkoEV1++eU88MADBINGdlcUhX379nHPPffwrW99q1sDFCIVlNX6KC87SLU3CM6BnXqv2rSkkKz22raEEtGjjz6Kx+MhOzubxsZGpk+fzsiRI3G5XDz44IPdHaMQSVfpCeAI1eC0quDs3PCms0cO4rwx+WSlWWVOojYkdNfM7Xbz1ltvsWHDBj799FM8Hg+nn346M2fO7O74hEgJlR4/ucFqnBkqOAd16r1ZaVZwpUGFSUpEbeh0ItI0jZUrV7J69WpKS0tRFIWioiJyc3PRdb1LU4IIkYp0XafK42VYyIPDktHpqhkApqaPmjRWx9Spqpmu61x++eXceuutHDx4kHHjxnHaaaexd+9e5s2b16lpYoXoK+p8IdTGakyKjt2RBta0Tr3/YE0jXxz2UtMoo+/b0qkS0cqVK1m/fj1r167lvPPOi3rt7bffZs6cOTz//PPceOON3RqkEMlU6fHjCFZjt6iY0gfFNf1HS3uPNlBywIMzGCRTSkQxdapE9NJLL/Gzn/2sVRICOP/887n33nt54YUXui04IVJBgz+MM1Tb1FDd+WqZWTU1dWiUIR5t6VQi+uyzz7j44ovbfP2SSy7h008/7XJQQqSScUPdXDnaZqxf38mGajA6NMqSQu3rVNWsqqqKnJycNl/Pycmhurq6y0EJkWpUXxWqakqsRNRyiIfcvo+pUyWicDiM2dx27lJVlVBIftCin9E08Db9g01LpGqmSNWsA50qEem6zrx589qcN1qmeBX9TY03wNpPdnJ6RS1F2W6wdXJmRpqWFDKZ0TQZa9aWTiWiuXPndriP3DET/clRj5+6ysN4fCFjjFkCiyMag17VptH3kohi6VQiWrFiRU/FIURKOuoJYA/V4kjwjhnAkEwHl08axoBP04w2Il3vdBeA/i7hJaeFOBFUegI4gjU4bZ0fY9bMYVUZOigDrE0fNy10bDJ9AXRhqlghTgSVDX4coVqclsRLRACYWiQeaSdqRRKREG0IhTWqG4LYgzVG1cyRWInIFwyz9WAdZfUBY4P0JWpFEpEQbaj2BlHCfpx6o7GWWYJVM39Q450dRyitlkTUFklEQrTBHwqTbW7AYVVRrGlgcSR0HFU1GqaDuoqO3MKPJamJaMmSJXzjG9/A5XIxePBg5syZw86dO5MZkhARQwc4uXZsOqflZSRcLQOjZzWApqgyS2MbkpqI1q1bx/z58/nwww956623CAaDXHTRRTQ0NCQzLCGO8VYac2x1oaG6ORHJkkJtS+rt+zfffDPq+cqVKxk8eDCbN2/m3HPPTVJUQrTgNRYPTbR9CIwOjUDTMI+QtBHFkFL9iGprjeVasrJi/9L9fn/UMJJYK4kI0R00TWf5hj1MPFLCBLeGpQslIkVRjg181YJSIoohZRqrNU3jrrvu4qyzzmLs2LEx91myZAlutzvyKCgo6OUoxYmizhekvjFIsO6oUbXqQhsRyJxEHUmZRDR//ny2bdvGqlWr2txn4cKF1NbWRh779+/vxQjFiaSqIYA13IDTrKGY1E6vZXa8S8fl8Y2TcrCZVamaxZASVbM777yTv//976xfv56hQ4e2uZ/NZmtz5L8Q3amqocXyQY5MMKldOl7hQCccSYNGReYkiiGpiUjXdX70ox/xyiuv8O6771JUVJTMcISIqGoIYA/WYu/q0I6Wmod5SNWslaQmovnz5/Piiy/y2muv4XK5KC8vB4x10xyOxDqPCdEdIiUipwqOAV0+3t7KBsJVPvKCYRzSWN1KUtuIli1bRm1tLTNmzCAvLy/y+OMf/5jMsMQJTtd1qrwtS0Rda6gG+GRfDVsOeKj3BaWNKIakV82ESDXBsE6Oy04m9Ti6qWpmUU0EIvNWS4noeCnRWC1EKrGaTXxrYg7UW4xJzLp46x7A0jRvdViT2/expMzteyFSSmO1kYTMtk6v7BqLpbkfkcxbHZMkIiGOo2l69NCObpjW1aKaCJvMhHUdtHCXj9ffSNVMiOO8uvUglgPbOFsNMCC3e27dN1fNNKmaxSQlIiGOU9UQQPdWGoNVu6F9CMBiNqpmYRl9H5OUiIRoIRDSqPeFGBas7bY7ZgAjBqWRNWYI2SUOSUQxSIlIiBZqvAHQdVx6HRY18elhj5fptDI8ZwBpVjOEA91yzP5EEpEQLVR5A5g1Hy41ZDRSd0Ov6gjVanwNy4rIx5NEJEQLVQ3GOmZ2iwq2jG5bf6wxEGbH0QCVDX4IGaUucYwkIiFaqG4I4ois7No91TKA2sYgb+6oYm+l19gg1bMo0lgtRAuD0q2k272k6ebuG3WPcfteRyXU/L8/5Dc6SwpAEpEQUaaOGAgNZjhq6bZb92DcvkdRCOpNHzkpEUWRqpkQx/NWGl+7sWpmVY2PWlCxGIO9Q9Jg3ZKUiIRo4guGMeka1sYaY0O3Vs2MRBRWrIR1DbPcOYsiJSIhmmwqrWLFvzazv8oDZivYXN12bNWkYFIUQiaLMQI/JFWzliQRCdHkqMePI1htVKOcg7plsGtLFrNC2GRtmpNISkQtSdVMCIxJ+irq/WQGa3Cmd+M81S1ceGoO6bbBWLz1UiI6jiQiIYB6f4gGf5ghoRqcVjOkDer2c4zKcUFNJvhMUiI6jlTNhAAO1TQCMFj1oCqKUTXrCc09teWuWRRJREIAZTU+0DUGmTzGhrTur5qV1/rYWxeiMRiWfkTHkUQkBHCgphF7qI4MmwImM9jc3X6Oj/dW8X6pxxjhLyWiKJKIxAlP13VOL8zkVJeXDLsF0rPB1P0fDZtZNfoRabqUiI6T1ES0fv16LrvsMvLz81EUhVdffTWZ4YgTlKIonJbvZka+ZnQ8dOX1yHlsZhPhSD8iKRG1lNRE1NDQwIQJE3jqqaeSGYYQhnpjpWHSc3rk8DazibBiJaTpctfsOEm9fX/JJZdwySWXJDMEcYKragiws7yesfkuXM2JyJXbI+eySomoTdKPSJywdF3nvd1HKTnioa7yELNCflDNkJbdI+ezmVVCJltTIvL1yDn6qj6ViPx+P37/sf8kdXV1SYxG9GWapvPB15WUHPFgUhSmDvBAA+AuAJPaI+e0WUyETHajahYKGOub9dC5+po+dddsyZIluN3uyKOgoCDZIYk+qN4X5M9bDvDRHmMRxbNGDiTTX2a8mFnYY+cd7LJx/rgCCrKaVo4NNvbYufqaPpWIFi5cSG1tbeSxf//+ZIck+pj9VV5e2LiPg9WNWM0mLhmXy+RCN1SXGjv0YCJy2S2cNmQAmRnpxgapnkX0qaqZzWbDZpPpNUVidF1n3a4KGgNhBmfYmD02jwFpVqgsMRqPrWngyu/5QMwOCPog6O35c/URSU1EHo+H3bt3R57v2bOHrVu3kpWVRWFhz/1nEicmRVG4bEI+m/dWce6obMxNk5VR9qnxNXt0j3RkbKZpOnurvNgbFXJ0HVNQSkTNklo1+/jjj5k0aRKTJk0CoLi4mEmTJrFo0aJkhiX6MbfDwvmn5BxLQg1H4egu4/v8ST16bkWB17YeZOshX1ODtbQRNUtqiWjGjBnG/L1C9BBd13lzWznDBqZxap4LpeVkZ4EG2P43Y42xQaMgfXCPxqIoCnaLSki1EQrXYZXG6og+1UYkRGd9frCWHeX17D7iYWiWwxhLFvLD/o9g/0ZjHXqLHUbO7JV4HBaVoMlOMKzLXbMWJBGJfssXDLNht7Eix5kjB5FhM0P5NihZC4GmhmJXLoy+BByZvRKTw2J0agxpmtw1a0ESkei3Pvi6El8wzCCXjUlD0o1q2OEvjBedWVB0LmSf0u1zU7fHYVWpj5SI5K5ZM0lEol+q9Pj5bH8tANOLXJi2vQzVe0ExQdE5UDA1Kb2aHRaVapONYFiTqlkLkohEv/Tvr46i6TonZ0LhvtXgqTCWCDrtSsgakbS4nFaVgJpGKKiB35O0OFKNJCLR7xyp97HnaAMZwQrO824GrcHorDj+GnD1zBQf8TppcDpZpmEM3mWHgMe4Y9eLVcNUJYlI9DuDHQo3DtpFcN/HODWb0R40/ppea5BuT06GnRxnPuw1G3fswgEwy2gBSUSifzm6G75aw0BfHWTYIHcsnHQBWJ3JjuwYs9V4hAJG9UwSkSQi0U8EGwlsfwPt8JfYzapR+jl5VlLbg2IJhjUOVDfi9FvIUQMQqO+RFUP6GklEou+rOwTb/sLBQ4c5XBdg8JhzGDF5llHqSDHBsMarnxxkzJEQF+bpmKTBGuhj04AI0crR3bD1Bbz1Nez12vhs8OWYR12QkkkIjNv3ZpNCwOQkENKMYSZCSkSiD6ssgW1/AV3jS/8gPss5h2E5AygcmELtQcdRFAWX3UxAdeIPadj99ckOKSVIIhJ9U+0B+GI16BpHHCN4Lzgek2rm3FE9M990d3LZLfjNLvyhMPhqkh1OSpCqmeh7PBXw2Z8gHCI8oIg3QmegKyoTCjKNic5SXIbDgs/sNqpmjdXJDiclSCISfUtjDXy2yhhB7x7CFtd5VDVqpNlUphZlJTu6uLjsZnyWDPwhzbgemQpHEpHoQwJeoyTk90DaIBj3HRTVgtmkcM6obOyWvrEihstuxq+m4wsDWgh8tckOKemkjUj0DeEgbPszeCvB5jJ6SlscTB7u4ORcFy5b3/lTHpLp4MLT8ijcnQ9anVE9S4Fe38kkJSKR+sIh+OIVqD0IZhv6+KvxqWmRlzPsluiZF1NcptPK2CFuXAOaZoRsrEpuQClAEpFIbeGQcYu+sgRUM9rYb7P+kMIfPtxLpaePL9vcPDVt81LXJ7C+U54VJ55AA2xbDbUH0ExmDg39DzbshkM1xp2mgzWNDEzvm+O0arwBDnvTyG0M4q4vS3Y4SSeJSKSmqj3oO16ntqaSikbY6DybihIV8GE1m5h5ag6jc13JjjJhX5bVsWW/ysX1PtzOo8ZdwBN48KskIpE6go2EK/egHv4MqvYAOttrVD7NvJBG8wCcVpVROelMHp5lTILfhw3NdLLRnEZF2MlJuoZSXWqsq3aCkkQkep+mQU0p1OxHb6zGU3uU2soj1NfX4w2EOL1wAIpJRRlyOurQsYwKmTg5x8WQTAcmU99plG5PrtuOalIoNw/BGzhAWuVuSUTJ9tRTT/Hwww9TXl7OhAkTeOKJJ5gyZUqyw0pdngpjrfbGamMOZmcWe/XB6PYB5GXasZlTsD+NpkHtfjiyHSp20Oj1cKTeR6UnYHTsa9JoyaRq4CQGnvxNcAygv/4VWM0migalcdg7gqP1JaQd+dKYN8liT3ZoSZH0RPTHP/6R4uJinn76aaZOncrjjz/OrFmz2LlzJ4MH9+yCd31K0AcV2/Ht/4SqslIa/CFGDk5HwSgh+A7Xsy/oZlP6SJxDxzIsP5eTstNxWFsnJb+vgbrqSnz1VZj8ddisZpzOdBwZWShp2a3aKsKaTmMwTGMgjK7ruJ2W+JJdOAh1B40R8hXbI3M0l9U2srtao9pRSGP6AEI2NwMGZpObk8cpOVl9YphGdxiTn8Huw7mU1jrJCwSw7t0AIy9IdlhJoehJXmp16tSpfOMb3+DJJ58EQNM0CgoK+NGPfsS9997b7nvr6upwu93U1taSkZHRG+H2nnAI3VdLoPoA3rJdeA9tp7ahkbrGIBoKtfahTJswhky7CerLOVC6i4q6RnxNpQuvZSBe6wAy0tPJTTczOksFfy1hbw2bdpcT65duUowlmU8ZXgBp2ehpg3h5u4963U7IZENXVEx6GJMewm0OMcwFU4Y6jGVxgl70gBcl6DVWpwh6CYcC+INhPP4QdotKRno6ZI+mKv0k/r/tCsOz0zktP4PhA9OOLQF9AtF1nVWb9uMr38nkurWMzc9AKTobhpxhzLHdD8T7GU1qiSgQCLB582YWLlwY2WYymZg5cyYffPBBq/39fj9+/7G+I3V1dfGdqGoPlKxlX6WXstrmJVyO+yjqOuOGuHFazYDOgWovB6obUdCj92zK26cNycBlNX58h2ob2VflPW7M0LHvT8114XYYjavldT5KjzYce73le3Sd0bkuBjgtEPRRXttIaeWxta8aLZkcyRyNOX8cI4fmYs91QdOwhqFjPAyp2IFn/+fUln1NVUMNDQ2V0ADBWjPobgBUwGYx4Vfs4HATMrsIhMKE/F7swRqgaciBrxalcjcjjlYRDOsogFk1Sl/BsBGzO80KAVfT1eps/LoKk8koo2m6jqZD0OSgxlHAgGFjOWfKFFDNZAG3DQ41/axPXIqicPFpufyxMYhtYANKYDuUbqBux7t8VRUirNoABV0xAUrTA4YPSiMnwyi11niD7Dzc9lQihVlO8txGda/eF+LLsrY/M0MznQwZYOzb4A+z7VDsoSeTCjKxmk0w7mqwd08BIKl/CUePHiUcDpOTE72yQk5ODjt27Gi1/5IlS1i8eHHnTxTyg6cCvaEBzdPO6preEISMH4nu9aJ7G2OWHABo1EBrunPja0RvbGexPL8CpqZ9/T50fzuTYQXNEAwDYLHa8FjT8acPwZI7hoG5hZyRk4HbGeOOkS0dZehkXEMn4/LXM7SuDE/NEWrqPYQVFXKyjaERdjfjznJhtraufnl8IbRAA1AHDRXQcJQxzsNYtEbMmh9FD4PJTFA3UR82o1jTwO0Gi5OgyU5JTaWxnLJqJ2hyEFLtmK12stJsDMlJB/XYn9uJnoSaDUizcv03h5FmKYKKUbB/I8GGfQR8XqCNvyl7Gpia2pIaA2j1bSci3eoE1WE88QXR6ttORLrZAZamuZwCIbT6NsbANQRBNYGuxX49AX3qr2HhwoUUFxdHntfV1VFQUNDxGzMLYMJ3GeALYvYZPzxFodUyLuZ0G5iNKkKmP4zFH+LYzi0pWNMsoKqgKLgDIU7yhyOvHdvdKB3YHWYwGz9qdzDMqEAIPfLqseMrgNNuAbMKFieZJhvTUYz/Pp1hc0G2i/Tsk0mP8XKsX7pqUowE58wEMiGzEIBYU4xZgOPHuVt0nSuHhgmENDTdOJ7NbMJmNvWp4RfJkN48Ti7nNMg5DYfXQ2FVFUo4AGhGqbnFhz7NYYamRG4PahQ2BNo+tsMMTce3hjQKR7e9r8tuBruxrzmsUTgq9r6mDJtRj+/G6mNSE9GgQYNQVZXDhw9HbT98+DC5ubmt9rfZbNhsCXT6sqZBVhEZQLwFSVfTIx5pTY94OJoe8ehL/yUURcFpNeM8MdqZe5TTmY7TGetfSGt2YEicx7UBQ+K8/2MDhvTiHHNJbSG0Wq2cccYZrF27NrJN0zTWrl3LtGnTkhiZEKI3Jf2fbnFxMXPnzmXy5MlMmTKFxx9/nIaGBm666aZkhyaE6CVJT0TXXHMNFRUVLFq0iPLyciZOnMibb77ZqgFbCNF/Jb0fUVf0635EQvQD8X5GT7xeZEKIlCOJSAiRdJKIhBBJl/TG6q5obt6Ke6iHEKJXNX82O2qK7tOJqL6pa3tcvauFEElTX1+P2+1u8/U+fddM0zQOHTqEy+XqM8MImoel7N+/v1/d6euP19Ufrwl697p0Xae+vp78/HxMprZbgvp0ichkMjF06NBkh5GQjIyMfvXH3aw/Xld/vCbovetqryTUTBqrhRBJJ4lICJF0koh6mc1m47777ktsFoEU1h+vqz9eE6TmdfXpxmohRP8gJSIhRNJJIhJCJJ0kIiFE0kkiEkIknSSiHvbggw9y5pln4nQ6yczMjOs9uq6zaNEi8vLycDgczJw5k6+++qpnA+2kqqoqrrvuOjIyMsjMzOSWW27B4/G0+54ZM2agKErU4wc/+EEvRRzbU089xfDhw7Hb7UydOpWPPvqo3f1ffvllTjnlFOx2O+PGjeMf//hHL0XaOZ25rpUrV7b6vdjtvbzirC561KJFi/THHntMLy4u1t1ud1zveeihh3S3262/+uqr+qeffqpffvnlelFRkd7Y2NizwXbCxRdfrE+YMEH/8MMP9X//+9/6yJEj9Wuvvbbd90yfPl2/7bbb9LKyssijtra2lyJubdWqVbrVatWXL1+uf/HFF/ptt92mZ2Zm6ocPH465/4YNG3RVVfXf/OY3+pdffqn/93//t26xWPTPP/+8lyNvX2eva8WKFXpGRkbU76W8vLxXY5ZE1EtWrFgRVyLSNE3Pzc3VH3744ci2mpoa3Waz6S+99FIPRhi/L7/8Ugf0TZs2Rba98cYbuqIo+sGDB9t83/Tp0/Uf//jHvRBhfKZMmaLPnz8/8jwcDuv5+fn6kiVLYu5/9dVX65deemnUtqlTp+rf//73ezTOzursdcX7t9mTpGqWYvbs2UN5eTkzZ86MbHO73UydOjXm6rfJ8MEHH5CZmcnkyZMj22bOnInJZGLjxo3tvveFF15g0KBBjB07loULF+L1trMwZQ9qXmW45c+5vVWGwbjulvsDzJo1K2V+L5DYdQF4PB6GDRtGQUEBV1xxBV988UVvhBvRpwe99kfl5eUAMVe/bX4t2crLyxk8OHqBLLPZTFZWVrsxfu9732PYsGHk5+fz2Wefcc8997Bz505Wr17d0yG30tlVhsG47lT+vUBi1zV69GiWL1/O+PHjqa2t5ZFHHuHMM8/kiy++6LVB5VIiSsC9997bqnHv+Edbv/RU1tPXdfvttzNr1izGjRvHddddx/PPP88rr7xCSUlJN16F6Kxp06Zx4403MnHiRKZPn87q1avJzs7mmWee6bUYpESUgP/6r/9i3rx57e4zYsSIhI7dvMLt4cOHycvLi2w/fPgwEydOTOiY8Yr3unJzczly5EjU9lAoRFVVVcwVetsydepUAHbv3s1JJ53U6Xi7orOrDIPxu+nM/smQyHUdz2KxMGnSJHbv3t0TIcYkiSgB2dnZZGf3zHq8RUVF5Obmsnbt2kjiqaurY+PGjfzwhz/skXM2i/e6pk2bRk1NDZs3b+aMM84A4O2330bTtEhyicfWrVsBohJub2m5yvCcOXOAY6sM33nnnTHfM23aNNauXctdd90V2fbWW2+l1KrEiVzX8cLhMJ9//jmzZ8/uwUiPk9Sm8hPA3r179U8++URfvHixnp6ern/yySf6J598otfX10f2GT16tL569erI84ceekjPzMzUX3vtNf2zzz7Tr7jiipS8fT9p0iR948aN+nvvvaePGjUq6vb9gQMH9NGjR+sbN27UdV3Xd+/erT/wwAP6xx9/rO/Zs0d/7bXX9BEjRujnnntusi5BX7VqlW6z2fSVK1fqX375pX777bfrmZmZkVvXN9xwg37vvfdG9t+wYYNuNpv1Rx55RN++fbt+3333pezt+85c1+LFi/U1a9boJSUl+ubNm/Xvfve7ut1u17/44otei1kSUQ+bO3euDrR6vPPOO5F9AH3FihWR55qm6T//+c/1nJwc3Waz6RdccIG+c+fO3g++HZWVlfq1116rp6en6xkZGfpNN90UlVz37NkTdZ379u3Tzz33XD0rK0u32Wz6yJEj9Z/85CdJ7Uek67r+xBNP6IWFhbrVatWnTJmif/jhh5HXpk+frs+dOzdq/z/96U/6ySefrFutVv20007TX3/99V6OOD6dua677rorsm9OTo4+e/ZsfcuWLb0ar0wDIoRIOrlrJoRIOklEQoikk0QkhEg6SURCiKSTRCSESDpJREKIpJNEJIRIOklEok9ZuXJl1EyX999/f9QYvHnz5kWGNoi+QxKRiGnevHltTuU6f/58FEWJGiDbEwlg+PDhPP7441HbrrnmGnbt2tXme5YuXcrKlSsjz2fMmBE1NkykJklEok0FBQWsWrWKxsbGyDafz8eLL75IYWFhUmJyOByt5kJqye12xz03uEgdkohEm04//XQKCgqiJi5bvXo1hYWFTJo0qUvHjlVSmTNnTqSUNWPGDPbu3cvdd98dmQsJWlfNjteyZDZv3jzWrVvH0qVLI8fYs2cPI0eO5JFHHol639atW1EUpVenvhDHSCIS7br55ptZsWJF5Pny5cu56aabevy8q1evZujQoTzwwAOUlZVRVlbW6WMsXbqUadOmcdttt0WOUVhY2OqaAFasWMG5557LyJEju+sSRCdIIhLtuv7663nvvffYu3cve/fuZcOGDVx//fU9ft6srCxUVcXlcpGbm5vQ5GNutxur1YrT6YwcQ1VV5s2bx86dOyNL7ASDQV588UVuvvnm7r4MESeZGE20Kzs7m0svvZSVK1ei6zqXXnopgwYNSnZYXZKfn8+ll17K8uXLmTJlCn/729/w+/185zvfSXZoJywpEYkO3XzzzaxcuZLnnnuu20oNJpOJ42egCQaD3XLseNx6662RhvgVK1ZwzTXX4HQ6e+38IpokItGhiy++mEAgQDAYZNasWd1yzOzs7Kh2n3A4zLZt26L2sVqthMPhLp2nrWPMnj2btLQ0li1bxptvvinVsiSTqpnokKqqbN++PfJ9W2prayPzUDcbOHAgBQUFrfY9//zzKS4u5vXXX+ekk07iscceo6amJmqf4cOHs379er773e9is9kSqhIOHz6cjRs3UlpaSnp6OllZWZhMpkhb0cKFCxk1alRKzTt9IpISkYhLRkYGGRkZ7e7z7rvvMmnSpKjH4sWLY+578803M3fuXG688UamT5/OiBEjOO+886L2eeCBBygtLeWkk05KeLGCBQsWoKoqY8aMITs7m3379kVeu+WWWwgEAr1yF1C0T6aKFSesf//731xwwQXs37+/1YKEondJIhInHL/fT0VFBXPnziU3N5cXXngh2SGd8KRqJk44L730EsOGDaOmpobf/OY3yQ5HICUiIUQKkBKRECLpJBEJIZJOEpEQIukkEQkhkk4SkRAi6SQRCSGSThKRECLpJBEJIZJOEpEQIun+f2FqK8fBEz4CAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:44:40.855469Z", + "iopub.status.busy": "2024-03-22T13:44:40.855153Z", + "iopub.status.idle": "2024-03-22T13:44:41.083240Z", + "shell.execute_reply": "2024-03-22T13:44:41.082266Z" + }, + "papermill": { + "duration": 0.251736, + "end_time": "2024-03-22T13:44:41.085246", + "exception": false, + "start_time": "2024-03-22T13:44:40.833510", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEqCAYAAABqVvf5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7o0lEQVR4nO3dd1gUV9s/8C9LWepSpFmQIlE0FiIqYlQ0KCJKIJjE2AuJT0zQRGyYJ2p8TYIFjYn609ck1lgSFbEhSlSUWFCxPGqQCGIFRERYKS7L7vn9wbvzsNJ2Fdgy9+e6vGRnzszes7Nz75mZM+cYMMYYCCFEzwk0HQAhhDQHSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXtC5ZLd27Vq4ubnB1NQUvr6+uHDhQp1lf/75Z/Tr1w+2trawtbXFoEGD6i1PCNFfOpXsfv/9d0RFRWHhwoW4fPkyunXrhiFDhiA/P7/W8snJyRg1ahROnjyJc+fOwcXFBYGBgXj06FEzR04I0TQDXeoIwNfXFz179sSaNWsAAHK5HC4uLpg2bRqio6MbXF4mk8HW1hZr1qzB+PHjVXpPuVyOnJwcWFlZwcDA4LXiJ4Q0LsYYnj9/jlatWkEgqL/uZtRMMb22iooKpKWlYd68edw0gUCAQYMG4dy5cyqto6ysDFKpFHZ2dnWWkUgkkEgk3OtHjx6hU6dOrx44IaTJPXjwAG3atKm3jM4ku4KCAshkMjg5OSlNd3Jywq1bt1Rax9y5c9GqVSsMGjSozjIxMTFYtGhRjem//PILzM3N1QuaENKkysrK8PHHH8PKyqrBsjqT7F7XkiVLsGvXLiQnJ8PU1LTOcvPmzUNUVBT3WiwWw8XFBWFhYRCJRM0RqkZIpVIkJSVh8ODBMDY21nQ45DXxZX+KxWJ8/PHHKl1i0plkZ29vD0NDQzx+/Fhp+uPHj+Hs7FzvsrGxsViyZAn+/PNPdO3atd6yQqEQQqGwxnRjY2O9/tIo8GU7+ULf96c626Yzd2NNTEzg4+OD48ePc9PkcjmOHz8OPz+/OpdbtmwZFi9ejMTERPTo0aM5QiWEaCGdqdkBQFRUFCZMmIAePXqgV69eWLVqFUpLSzFp0iQAwPjx49G6dWvExMQAAJYuXYoFCxZgx44dcHNzQ15eHgDA0tISlpaWGtsOQkjz06lkN3LkSDx58gQLFixAXl4evL29kZiYyN20uH//vtLt53Xr1qGiogLvv/++0noWLlyIb775pjlDJ4RomE4lOwCIjIxEZGRkrfOSk5OVXt+9e7fpAyKE6ASduWZHCCGvQ+dqdoSQmsrKypTam5aUS3D2ehZs7S/B0ky5dYGXlxcv24xSsuMpVQ8Ovh4YuubWrVvw8fGpMX1ZLWXT0tLQvXv3pg9Ky1Cy4ylVDw6+Hhi6xsvLC2lpadzrjNwiRO2+jpUfdEGHljY1yvIRJTueUvXg4OuBoWvMzc2VfpQE955CmFKOjp27wdu1hQYj0x6U7HiKDg7CN5TsCNFR2QWlKJVU1jov60kp97+RUd2HuYXQCO72Fk0Sn7ahZEeIDsouKMXA2OQGy83cc73BMidnDeBFwqNkxxP11QIAqgnoGsW+XDXSG56ONR99LC2X4FDyOQwf4AcLs5odWwBAZn4Jvvz9ar3fC31CyY4HVK0FAFQT0DWejpbo3Nq6xnSpVIo8B6C7q61e93qiDkp2PNBQLQCgmgDRf5TseKSuWgBANQGi/yjZEaKDJLIXEJg+QrY4AwLTmrX1yspK5FTmIL0wvc5rsNniEghMH0EiewGg9h9BfULJjhAdlFN6Dxbuq/FVA8Mg/7/E/1fvfAt3IKfUGz5wqrecPqBkR4gOamXhitLsafhxpDfa1XIdtrKyEmf+OoO3+75dZ80uK78EX/x+Fa0GujZ1uFqBkh0PNHTKA9Bpj64RGppC/qI13EUd0KlF7Xdjs42y0dGuY53XYOUviiF/8QRCw7oHoNInlOx4QNVTHoBOe4j+omTHAw2d8gB02kP0HyU7HmjolAeg0x6i/6hbdkIIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL+hcslu7di3c3NxgamoKX19fXLhQfydtu3fvhpeXF0xNTdGlSxckJCQ0U6SENJ1yqQwAcONRca3/Lt97hktPgMv3ntVZJjO/RMNb0bx0qoun33//HVFRUVi/fj18fX2xatUqDBkyBBkZGXB0dKxR/uzZsxg1ahRiYmIwfPhw7NixA2FhYbh8+TI6d+6sgS0gpHFk/V+iio6rb5xfI2zLvNjguiyEOpUGXplObeXKlSvxySefYNKkSQCA9evX4/Dhw9i4cSOio6NrlP/xxx8RFBSE2bNnAwAWL16MpKQkrFmzBuvXr2/W2DWpei2gLqXlElx6Ajjfe1bvuLFEOwS+6QwAaOdoCTNjwxrzM3KLMXPPdax4vws6tKy7C30LoRFvBjzXmWRXUVGBtLQ0zJs3j5smEAgwaNAgnDt3rtZlzp07h6ioKKVpQ4YMQXx8fJ3vI5FIIJFIuNdisRhAVeeWUqn0NbZAc/7JrUpy9dcCAFVrAkJDprOfhb6wMjHAiLda1jn/xYsXAABXWyE6OJrXuy5d3pfqxK4zya6goAAymQxOTspjHzg5OeHWrVu1LpOXl1dr+by8vDrfJyYmBosWLaox/dixYzA3r/9Lo7WkwEceBnA0YzCp4yrt43JgW6YRxnlWwsms7lUJDYG/U0/h76aJlDSSByUAYITz58/j0Q1NR9N0ysrKVC6rM8muucybN0+pNigWi+Hi4oLAwECIRCINRvZ6Pmxg/rX7hdiWeQmhA3ujW1u7ZomJNJ1r9wuB65fQu7d+70/FmZcqdCbZ2dvbw9DQEI8fP1aa/vjxYzg7O9e6jLOzs1rlAUAoFEIorHnNytjYuM6xGfSBYpAdIyMjvd5OvuDL/lRn23Sm6YmJiQl8fHxw/PhxbppcLsfx48fh5+dX6zJ+fn5K5QEgKSmpzvKEEP2lMzU7AIiKisKECRPQo0cP9OrVC6tWrUJpaSl3d3b8+PFo3bo1YmJiAABffPEF/P39sWLFCgwbNgy7du3CpUuXsGHDBk1uBiFEA3Qq2Y0cORJPnjzBggULkJeXB29vbyQmJnI3Ie7fvw+B4L+V1T59+mDHjh34+uuv8dVXX+GNN95AfHw8tbEjhId0KtkBQGRkJCIjI2udl5ycXGPaBx98gA8++KCJoyKEaDuduWZHCCGvg5IdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEaJnysvLETN/Dh7/Ph8x8+egvLxc0yFpBUp2BBUVFdj+6zoUJq3H9l/XoaKiQtMhkVcUFhYGc3Nz/LFtI17cvYI/tm2Eubk5wsLCNB2a5jFSr+LiYgaAFRcXazqUJjF79mxmaGjIAHD/DA0N2ezZszUdGlFTaGio0n58+V9oaKimQ2x06hyfBowxppk0qxvEYjGsra1RXFwMkUik6XAa1Zw5c7B8+fI658+ePRvLli1rxojIqyovL4e5uXmD5crKymBmZtYMETUPdY5POo3lqaKiIsTGxtZbJjY2FkVFRc0TEHkt06dPV6nc+PHjUVZW1sTRaCdKdjz1zTffoKFKPWMM33zzTfMERF5LSkqKSuX27NmDW7duNXE02omSHU+lpqY2ajmiWc+fP1epnIODA7y8vJo4Gu1EyY6nSkpKGrUc0SxTU1OVyllZWal0bU8f6UyyKywsxJgxYyASiWBjY4OIiIh6D8TCwkJMmzYNHTp0gJmZGdq2bYvp06ejuLi4GaPWXnZ2do1ajmjWixcvGrWcPtKZZDdmzBjcvHkTSUlJOHToEE6fPo0pU6bUWT4nJwc5OTmIjY3FjRs3sHnzZiQmJiIiIqIZo9Zejx49atRyRLMo2TVM7aYnd+7cgYeHR1PFU6v09HR06tQJFy9eRI8ePQAAiYmJCA4OxsOHD9GqVSuV1rN7926MHTsWpaWlMDIyqrWMRCKBRCLhXovFYri4uKCgoECvmp44ODioVMu1trbGkydPmiEi8jqcnJzw7NmzBsvZ2tri8ePHzRBR8xCLxbC3t1ep6UntR3w9PD094e/vj4iICLz//vsqXyt4HefOnYONjQ2X6ABg0KBBEAgESE1NxXvvvafSehQfSF2JDgBiYmKwaNGiGtOPHTumV9c6VG1+UFZWhoSEhCaOhrwuvu5PdZrRqF2zu3r1KjZt2oSdO3eioqICI0eOREREBHr16qV2oKr6/vvvsWXLFmRkZChNd3R0xKJFizB16tQG11FQUAAfHx+MHTsW3333XZ3l+FKzs7KyUtrOugiFQpXv9BHNMTExUbmsPj0O2KQ1O29vb/z4449YsWIFDhw4gM2bN6Nv375o3749Jk+ejHHjxsHBwUGldUVHR2Pp0qX1lklPT1c3xBrEYjGGDRuGTp06NdhuTCgUQigU1phubGwMY2Pj145FW5iYmKiU7ExMTPRquwn0an+qsy2vfIPCyMgI4eHh2L17N5YuXYrMzEzMmjULLi4uGD9+PHJzcxtcx8yZM5Genl7vPw8PDzg7OyM/P19p2crKShQWFsLZ2bne93j+/DmCgoJgZWWFffv26dWOfh2q/iCpWo5oloGBQaOW00dq1+wULl26hI0bN2LXrl2wsLDArFmzEBERgYcPH2LRokUIDQ3FhQsX6l2Hg4ODSgeTn58fioqKkJaWBh8fHwDAiRMnIJfL4evrW+dyYrEYQ4YMgVAoxIEDB5rl+qKuePPNN3Hnzh2VyhHt5+XlpdJZEF8bFAOvULNbuXIlunTpgj59+iAnJwdbt27FvXv38O2338Ld3R39+vXD5s2bcfny5UYLsmPHjggKCsInn3yCCxcu4MyZM4iMjMRHH33E3Yl99OgRvLy8uAQrFosRGBiI0tJS/PrrrxCLxcjLy0NeXh5kMlmjxaar+vXr16jliGYJBKodyqqW00dq1+zWrVuHyZMnY+LEiWjZsmWtZRwdHfHrr7++dnDVbd++HZGRkQgICIBAIMCIESPw008/cfOlUikyMjK4uzOXL1/mHnXy9PRUWld2djbc3NwaNT5d07lz50YtRzTL2dkZN2/eVKkcX6md7JKSktC2bdsavxCMMTx48ABt27aFiYkJJkyY0GhBAlUt+Xfs2FHnfDc3N6UH2wcMGNDgg+589tdff6lcbujQoU0cDXld7dq1w/Hjx1Uqx1dq12nbtWuHgoKCGtMLCwvh7u7eKEGRppednd2o5YhmqfrDzucKgNrJrq4Pq6SkhG4A6BDF3W1zc3OIxWJ8+umn8Pb2xqeffgqxWMx18PjyXXCine7evduo5fSRyqexUVFRAKpuXS9YsEDpaQKZTIbU1FR4e3s3eoCkaSiekSwrK4ODgwPX5k7RaFzxms/PUuoSVZtU8bnplcrJ7sqVKwCqanbXr19XarFtYmKCbt26YdasWY0fIWkSbm5uOHPmDADUaFxc/TXfb+ToiocPH3J/m5qaKv1IVX9dvRzfqJzsTp48CQCYNGkSfvzxR716dIqPxo0bh+3btwOoao4gl8u5edVfjxs3TiPxEfVUv7b6cm28+ms+X4NV+27spk2bmiIO0syqt6Svnuhefs3nFvdEv6iU7MLDw7F582aIRCKEh4fXWzYuLq5RAiNNS9UxC1JSUhAYGNjE0ZDX5e7ujv/85z8qleMrlZKdtbU19wtvbW3dpAGR5vFybe51yxHNatGiRaOW00cqJbvqp650GqsfbGxsuL8dHR3RsWNHPHnyBA4ODkhPT+eanFQvR7SXKj3YqFNOH71yRwBEt1VvGJ6fn19ne7raGpAT7aPqwNf6NEC2ulRKdm+99ZbKF6obswMA0nRU3U+0P3VDt27dVHpcrFu3bs0QjXZSKdmFhYU1cRikudX3tIuBgQH3pAw9FaMbbt++3ajl9JFKyW7hwoVNHQfRInx+flJXlZeXN2o5fcTfzq0I0SOWlpbc3y8PK1C9dl69HN+olOzs7Oy4C9W2traws7Or8x/RDTTOqH5RXGoyMjKq0fu3g4MDN6Ieny9JqXQa+8MPP8DKyor7m1rV6z7FAWFgYACBQKDUe7ORkRFkMhkYYzQGhY5wdXUFUDU2y8vPvz548KBGOT5SKdlV74hz4sSJTRULaUaKHyzGGGxtbeHv749nz57B1tYWp06d4mry9MOmG/r16wcHB4d6BzR3dHTkdTf7arezMzQ0RG5uLhwdHZWmP336FI6OjjS+g46o3ptJQUEB9u7d22A5ot0UDYbt7OwgFAohFoshEokgkUhQWFjI+0sSjdZ5p0QiUWugXqJZ77zzDvf3yw1Nq7+uXo5or+TkZIjFYlhaWqKwsBC5ubkoLS1Fbm4uCgsLYWlpCbFYjOTkZE2HqjEq1+wUg9sYGBjgl19+UbqrI5PJcPr0aV4P06ZrBgwYAEdHR+Tn59dojqB47ejoiAEDBmggOqIuRRIrKSmBiYkJwsPDYWZmhvLycsTFxaGkpIQrFxAQoMFINUflZPfDDz8AqKrZrV+/HoaGhtw8ExMTuLm5Yf369Y0fIWkShoaG8PPzw/79++ss4+fnp7SfifaqqKgAUNUT8fPnz2FgYICEhAQEBwdjy5YtsLS0hFQq5crxkcrJTtHp38CBAxEXFwdbW9smC4o0vYqKChw8eBAAuBqAguL1wYMHUVFRQZcndIBigGxHR0fubrqCojlKTk6OSgNp6yu1r9mdPHmSEp0eWLNmDeRyOVxdXWttl+Xq6gq5XI41a9ZoKEKiDsXNh0ePHiE0NBTnz59HeXk5zp8/j9DQUOTk5CiV4yO178ZOnjy53vkbN2585WBI81F03nnv3j2EhIRg+/btePjwIdq0aYNly5Zxtb6UlBRusCWivdq3b4+kpCQAwJ9//olDhw5x86rfcGrfvn2zx6Yt1K7ZPXv2TOlffn4+Tpw4gbi4OBQVFTVBiKQpWFhYAADefPNN7N27Fy9evMDFixfx4sUL7N27F506dVIqR7Tb8uXLAVRdi335upxEIuGuvSrK8ZHaNbt9+/bVmCaXyzF16lRejzaua7p164bt27fjzp07aN++PTee6MqVK+Hm5obHjx9z5Yj2MzMzQ8+ePXHx4kUAQPfu3blrr4puunr27En92b0ugUCAqKgoDBgwAHPmzGmMVZIm1qpVKwBVzUwePXqEkSNHwtzcHGVlZYiLi4NUKlUqR7SbTCbDkydPYG1tjeLi4hr9EFpbW6OgoAAymYy3d9gbrafirKwsVFZWNtbqSBNzdnbm/pZKpfj9998bLEe0V0pKCu7evQsDAwMMHToUz58/x7179+Dq6gorKyskJiaiuLgYKSkpvG07qXaye/liNWMMubm5OHz4sNIztISQ5vPo0SMAgLe3N65fv851BvDgwQO0adMG3t7euHLlCleOj9ROdleuXFF6LRAI4ODggBUrVjR4p5Zoj7y8PO5vY2NjhIeHw8LCAqWlpUqnsdXLEe2l6ADg5eMTAB4+fMglv/o6CtB3aie7kydPNkUcpJkp2l2ZmZnBwcFB6TTW1dWVe4xMUY5oNxpKsWE0uhhPXb16FQDQrl07XL58GadOncKRI0cwdOhQ+Pv7o3v37rhx4wZXjmi36j9Kjo6OGD16NMrKymBubo4dO3Zwo8fx+cdLZ7plLywsxJgxYyASiWBjY4OIiAju4eaGMMYwdOhQGBgYID4+vmkD1RFlZWUAgJs3b2LEiBEQCoXo2bMnhEIhRowYgZs3byqVI9rt2LFjAKoeDSsoKMCqVauwYcMGrFq1Ck+fPuV6KlaU4yOdqdmNGTMGubm5SEpKglQqxaRJkzBlyhTs2LGjwWVXrVpFnVC+pG/fvoiPj0fbtm1x/fp19O/fn5vn7u4OFxcX3L9/H3379tVglERVimtylZWVcHJywujRo1FaWgoLCwvs2LGDazf5ci/GfGLAdGAoqfT0dHTq1AkXL15Ejx49AACJiYkIDg7Gw4cP620LdvXqVQwfPhyXLl1Cy5YtsW/fvnr74ZdIJEqjpovFYri4uKCgoAAikajRtknTKioqIBKJIJfLERQUhMDAQNy5cwceHh44duwYEhMTIRAIIBaLqSMAHTBkyBCcPHkSJiYmaNmyJe7du8fNc3NzQ05ODioqKjBw4EAcPXpUg5E2LrFYDHt7exQXFzd4fDZaze7hw4f4n//5H2zYsKGxVsk5d+4cbGxsuEQHAIMGDYJAIEBqairee++9WpcrKyvD6NGjsXbtWpXbi8XExGDRokU1ph87dgzm5uavtgFa6t1330V8fDyOHj2KxMREbrqiFvzuu+/izz//1FR4RA1t2rQBUPUjZmdnh8DAQAiFQkgkEly6dIl7QqZNmzZISEjQYKSNS53LLI2W7J4+fYpff/21SZJdXl5ejW7gjYyMYGdnV2/TiBkzZqBPnz4IDQ1V+b3mzZun1JZQUbMLDAzUq5odAAQHByM6Oho//vijUpdAhoaGmD59OpYsWaLB6Ig6ioqKsG3bNgBVzU9qa4ICAAEBAQgODm7O0JqUWCxWuaxGr9lFR0dj6dKl9ZZ51f63Dhw4gBMnTtS50+siFAprjLsJVLVFMzY2fqVYtNmKFSsQExOD1atX48SJE3jnnXcwbdo0OnXVMaqOGubq6qpX32N1tkWjyW7mzJkNjlbm4eEBZ2dn7ta5QmVlJQoLC+s8PT1x4gSysrJgY2OjNH3EiBHo168fr/vif5mJiQmmT58OT09PBAcH69XBwBf9+vWDm5sbDA0Ncffu3Ro1dTc3N8jlchpdTFMcHBxUGpfUz88PRUVFSEtLg4+PD4CqZCaXy+Hr61vrMtHR0fj444+VpnXp0gU//PADQkJCXj94QrSIoaEhPvjgAyxfvhyOjo4YM2YMdzd2+/btyMrKwuzZs3nbCQCgxt3Y8PDweucXFRXh1KlTTTaU4tChQ/H48WOsX7+ea3rSo0cPrunJo0ePEBAQgK1bt6JXr161rsPAwKDBu7EvE4vFXE8S+nbNrjqpVMqNWUA1O90jk8ng6ekJe3t7FBQUcDckgKqmRC1atMDTp09x+/ZtvUp46hyfKtfsrK2tG5w/fvx4VVentu3btyMyMhIBAQEQCAQYMWIEN+IZUHWwZmRkUCNYwkuKXk927tyJnj174uTJk9wTMQMHDsSFCxfQp08f6vVEFZs2bWrKOBpkZ2dXbwNiNze3Ose0VdCBJoWEvJLc3FwAQOfOnWFoaAh/f3+UlpbC398fhoaG6Ny5s1I5PtKZx8UIIXVr2bIlAODGjRu1zldMV5TjI5Vrdqp230QD7hDS/BR3Y7///vsaz3/L5XLExMTA3d2d7saqYvPmzXB1dcVbb71Fp4OEaBlDQ0OsWLEC77//PsLCwjB79mxuKMXly5fj0KFD2LNnj17dnFCXyslu6tSp2LlzJ7KzszFp0iSMHTsWdnZ2TRkbIUQN4eHh2LNnD2bOnFmjY4c9e/Y02KJC36l8zW7t2rXIzc3FnDlzcPDgQbi4uODDDz/E0aNHqaZHiJYIDw9HZmYmkpKSEBUVhaSkJNy+fZv3iQ5Q8waFUCjEqFGjkJSUhL///htvvvkmPvvsM7i5uanctxwhpGkp7sb279+fuxtLXuNurEAggIGBARhjTdaQmBBCGotayU4ikWDnzp0YPHgw2rdvj+vXr2PNmjW4f/8+LC0tmypGQogaZDIZTp06hdOnTzfpU026RuUbFJ999hl27doFFxcXTJ48GTt37oS9vX1TxkYIUVNcXBxmzpzJPS62cuVKuLm5YcWKFby/bqdyslu/fj3atm0LDw8PnDp1CqdOnaq1XFxcXKMFRwhRXVxcHN5//30MHz4c27Ztw8OHD9GmTRssW7YM77//Pu/vyKqc7MaPH0/jOBCipWQyGWbOnInhw4cjPj4eMpkMT58+ha+vL+Lj4xEWFoZZs2YhNDSUtzcs1GpUTAjRTtU7AhAIBErX6QQCAebNm8f7jgDo2VhC9ED1jgBqu0FBHQHo0FCKhJC6KR7wX7NmDf73f/+3xg2KKVOmKJXjI6rZEaIH+vXrBwcHB8ybNw+dO3dGSkoKdu7ciZSUFHTu3BlfffUVHB0ded0RACU7QvRE9RuIikc46VHO/6JkR4geSElJQX5+PmJiYnDjxg30798fo0aNQv/+/XHz5k18//33yM/PR0pKiqZD1RhKdoToAcWNh8jIyFo7AoiMjFQqx0eU7AjRA9V7Kq6tIwDqqZiSHSF6oXpPxXK5XGke9VRchZIdIXpA0VPxoUOHEBYWhvPnz3M9FYeFheHQoUOIjY3l7dMTALWzI1DuJcPCwgIDBw7k9UGhq6in4vqpPEg2X+n7INkv95IBgHrJ0HEymazGuLH6+uPVJINkE/2j6CUjODgYISEhyMjIQIcOHXDnzh3qJUOH1TZuLKGaXYP0tWYnk8ng6ekJQ0ND3Lt3D5WVldw8IyMjuLq6Qi6X4/bt23Sw6CCpVIqEhAQEBwfD2NhY0+E0GarZkQYpeskAACcnJ4wePRqlpaWwsLDAjh07kJWVxZXjay8ZRL9QsuOpBw8eAABEIhFMTU3xww8/cPNcXV0hEokgFou5coToOmp6wlOpqakAqk4DunbtqvTgeNeuXSEWi5XKEaLrqGbHU4qGp/b29oiLiwNjjOvZNi4uDi1btkRBQUGNBqqE6Cqq2fGUQFC16wsKChAeHq7UCDU8PBwFBQVK5QjRdVSz4ylfX1+sXbsWIpEI165dU2qEWv2ana+vrwajJKTx6MzPdmFhIcaMGQORSAQbGxtERESgpKSkweXOnTuHd955BxYWFhCJROjfvz/Ky8ubIWLt5uLiAqDqmp1EIsGXX36JKVOm4Msvv8SLFy+4a3aKcoToOp2p2Y0ZMwa5ublISkqCVCrFpEmTMGXKFOzYsaPOZc6dO4egoCDMmzcPq1evhpGREa5du0anZvjvg+OGhoa4e/cuVq1axc0zMjJCu3btIJfLef3gONEvOpHs0tPTkZiYiIsXL6JHjx4AgNWrVyM4OBixsbFo1apVrcvNmDED06dPR3R0NDetQ4cOzRKztlM8OK54gsLd3R3//PMP2rdvj+zsbCQkJGDPnj3UoJjoDZ1IdufOnYONjQ2X6ABg0KBBEAgESE1NxXvvvVdjmfz8fKSmpmLMmDHo06cPsrKy4OXlhe+++w59+/at870kEgkkEgn3WnE6J5VKIZVKG3GrNC8kJAS7du3C3LlzcfjwYQDAsWPH4O7ujl27diEkJETvtpkvFPtN3/efOtunE8kuLy8Pjo6OStOMjIxgZ2eHvLy8Wpe5c+cOAOCbb75BbGwsvL29sXXrVgQEBODGjRt44403al0uJiYGixYtqjH92LFjMDc3f80t0T5CoRArVqzA33//jWfPnsHW1hadOnWCoaEhEhISNB0eeQUymYzbn9evX+f2pz4qKytTuaxGk110dDSWLl1ab5n09PRXWreifdi//vUvTJo0CQDw1ltv4fjx49i4cSNiYmJqXW7evHmIioriXovFYri4uCAwMFCvno19WVBQEJKSkjB48GC9fpZS3+3btw9z586t0YvN0qVLaz0D0nWKMy9VaDTZzZw5ExMnTqy3jIeHB5ydnZGfn680vbKyEoWFhXB2dq51OUX30506dVKa3rFjR9y/f7/O9xMKhRAKhTWmGxsb8yIJ8GU79VFcXBw++ugjDB8+HNu2bcPDhw/Rpk0bLFu2DB999JFe9mKjzndVo8nOwcEBDg4ODZbz8/NDUVER0tLS4OPjAwA4ceIE5HJ5ne3A3Nzc0KpVK2RkZChN/+effzB06NDXD54QLSKTyTBz5kwMHz4c8fHxkMlk3BMx8fHxCAsLw6xZsxAaGqq3p7QN0Yk2GB07dkRQUBA++eQTXLhwAWfOnEFkZCQ++ugj7k7so0eP4OXlhQsXLgCoGkNz9uzZ+Omnn7Bnzx5kZmZi/vz5uHXrFiIiIjS5OYQ0OkUvNl999VWNplUCgQDz5s1DdnY2r4dS1IkbFACwfft2REZGIiAgAAKBACNGjMBPP/3EzZdKpcjIyFC6YKloIDtjxgwUFhaiW7duSEpKQrt27TSxCYQ0GcUQiZ07d651vmI6n4dS1JlkZ2dnV28DYjc3t1pHP4+OjlZqZ0eIPqo+lGLv3r1rzKehFHXkNJYQUr/qQylKpVJuAKVTp05BKpXSUIrQoZodIaRu1Z+IEYlEePHiBQBg5cqVMDU1hUQi4f0TMVSzI0SPMMaUngACqp4KoqFmKNkRKI8be+rUKchkMk2HRNQkk8nw6aefAgBMTU2V5ileT506ldf7lpIdz8XFxcHDwwODBw/GypUrMXjwYHh4eCAuLk7ToRE1JCcn48mTJwCAgIAApW72AwICAFQ9L56cnKzBKDWLkh2PxcXFYcSIETWeKLl//z5GjBhBCU+HnDhxAgDQu3dv7N+/H76+vjAzM4Ovry/279/P3aFVlOMjSnY8JZPJMHbsWACAiYkJ5syZg3Xr1mHOnDkwMTEBAIwdO5bXpz26RDEK3JgxY8AYU7oswRjDqFGjlMrxEd2N5aljx46hvLwcxsbGeP78OQwMDJCQkICIiAgsXrwYlpaWKC8vx7Fjx+jxOh2g6FF69erViI2Nxb179wBU3Y11dXXlnvfmc8/TVLPjqZUrVwIApk2bxtXkFExMTPD5558rlSPa7Z133gFQ9ex3eXk51q1bh40bN2LdunUoLy/HP//8o1SOj6hmx1NFRUUAqjpZqI2igwVFOaLd+vXrB4FAALlcjufPn2Pq1KncPDMzMwBVz8jyuVEx1ex4StHrc3R0dI2xYeVyOb7++mulckS7nT17ts4xfg0MDABU7dezZ882Z1hahZIdTylOT7OyshASEqI0bmxISAiysrKUyhHtpnjA/7fffqvRq7ejoyN+++03pXJ8RKexPGVmZobQ0FDs378fCQkJtXbBHhoayp0CEe2meMC/Xbt2yMrKwsmTJ3HkyBEMHToUAwcO5Lo+43NHAAaMniOpl1gshrW1NYqLi/WyW/awsDDs37+/xvTQ0FDEx8c3f0DklchkMnh6eqJLly5c550JCQkIDg6GoaEhwsLCcOPGDdy+fVuvno9V5/ikmh3PxcfHo7y8HFFRUTh//jx69+6NlStXUo1Ox1TvCCA0NBSDBw/G7du3ce/ePSQlJeHw4cO87wiAanYN0PeanYJUKuVqAjQGhe6aM2cOfvjhB1RWVnLTjIyMMGPGDCxbtkyDkTUNqtkRwkNxcXGIjY3FsGHDuJrdG2+8gaSkJMTGxqJ37956N+COOqhm1wCq2RFdQNfsGj4+qekJIXqABtxpGCU7QvQADbjTMEp2hOiB6gPu1IYG3KFkR4heqD7gTm2P/9GAO5TsCNELinZ2hw4dQlhYmNLjf2FhYTh06BBiY2P16uaEuqjpCSF6Ijw8HHv27MHMmTPRv39/brq7uzv27NnD62YnACU7QvRKeHg4QkNDazwby+canQIlO0L0jKGhIfz9/VFaWgp/f39KdP+HrtkRQniBkh0hhBco2RGUl5dj+vTp+OabbzB9+nSUl5drOiRCGh0lO54LCwuDubk51q9fj6tXr2L9+vUwNzdHWFiYpkMjpFFRsuOxujruBID9+/dTwiN6hZIdT5WXl3OJLjg4GCkpKdi5cydSUlIQHBwMoCrh0Skt0Rc6k+wKCwsxZswYiEQi2NjYICIiAiUlJfUuk5eXh3HjxsHZ2RkWFhbo3r079u7d20wRa7eoqCgAVWMWHDx4EL6+vjAzM4Ovry8OHjyIdu3aKZUjRNfpTLIbM2YMbt68iaSkJBw6dAinT5/GlClT6l1m/PjxyMjIwIEDB3D9+nWEh4fjww8/xJUrV5opau116dIlAMCSJUtq7RLo22+/VSpHiK7TiWSXnp6OxMRE/PLLL/D19UXfvn2xevVq7Nq1Czk5OXUud/bsWUybNg29evWCh4cHvv76a9jY2CAtLa0Zo9dONjY2AIBz587VOj81NVWpHNEdMpkMp06dwunTp3Hq1CnIZDJNh6QVdKKn4o0bN2LmzJl49uwZN62yshKmpqbYvXs33nvvvVqXCwwMhImJCbZu3QobGxv88ccfiIiIwLVr1+Dp6VnrMhKJBBKJhHstFovh4uKCgoICveqp+OjRowgJCYGxsTFycnLw73//GxcuXECvXr3w3XffoVWrVpBKpTh48CCGDBmi6XCJivbt24fZs2fj/v373LS2bdti+fLldR4nukwsFsPe3l5/xqDIy8urMfCvkZER7OzskJeXV+dyf/zxB0aOHIkWLVrAyMgI5ubm2LdvX52JDgBiYmKwaNGiGtOPHTsGc3PzV98ILSOTyWBiYoKKigo4ODhw069evYoNGzYAADe/tjFlifY5d+4cli5dWmP6/fv3MXLkSMydOxd+fn4aiKzplJWVqVxWo8kuOjq61p1TXXp6+iuvf/78+SgqKsKff/4Je3t7xMfH48MPP0RKSgq6dOlS6zLz5s1TuiivqNkFBgbqVc0OALp27VrvNbmuXbsiJCSkGSMir0omk2HUqFH1llm9ejW++eYbvXpWViwWq1xWo8lu5syZmDhxYr1lPDw84OzsjPz8fKXplZWVKCwshLOzc63LZWVlYc2aNbhx4wbefPNNAEC3bt2QkpKCtWvXYv369bUuJxQKIRQKa0w3NjbWq4FoysvLuUSnqMEpKF5funQJlZWVNIasDjh+/LhSM6GxY8fCx8cHaWlp+O233wBU1YKSk5MRFBSkqTAbnTrHpEZvUDg4OMDLy6vefyYmJvDz80NRUZHSjYUTJ05ALpfD19e31nUrqrcv32k0NDSs0ZMrH82cORMA4OnpiZKSEiQlJSEqKgpJSUkoKSnhmp4oyhHtVv0Mqby8HBs3boS7uzs2btyolAQbOpPSa0xHBAUFsbfeeoulpqayv/76i73xxhts1KhR3PyHDx+yDh06sNTUVMYYYxUVFczT05P169ePpaamsszMTBYbG8sMDAzY4cOHVX7f4uJiBoAVFxc3+jZpUo8ePRgAtnv3bsZY1ecVHx/PKioqGGOM7dy5kwFgPXr00GSYREUikYgBYP3792eM1dyfb7/9NgPARCKRJsNsdOocnzrR9AQAtm/fDi8vLwQEBCA4OBh9+/blLqQDVeOeZmRkcDU6Y2NjJCQkwMHBASEhIejatSu2bt2KLVu2cE8I8JmtrS2AhpueKMoR7aa4DpednV3rGBT37t1TKsdLzZB8dZq+1uyOHDnCADBjY2MmkUiUagISiYQZGxszAOzIkSOaDpWoYPjw4QwAA8CGDRvGTp8+zXbu3MlOnz7Nhg0bxs0bPny4pkNtVOocnzrRzk6T1BlxXJfIZDKIRCKUlZXBxMQE06dPh7u7O7Kzs/HTTz+hoqIC5ubmEIvF/K4N6IiSkhJYWVk1WO758+ewtLRshoiahzrHp060syONz9DQENu2bcOIESNQUVGB2NjYGmW2bdtGiU5HWFpaomfPnrh48WKdZXr27KlXiU5dOnPNjjS+8PBw7N27F23btlWa7urqir179/J+NCpdc+HCBfTs2bPWeT179sSFCxeaOSLtQqexDdDX09jqZDIZjUalR0pKSjB69Gj85z//QdeuXbFjxw69rdHRaSxRC41GpV8sLS2xd+9eJCQkIDg4WK8aw78OOo0lhPACJTtCCC9QsiOE8AJds2uA4v6NOr0r6CKpVIqysjKIxWK6xqMH+LI/FcelKvdZKdk14Pnz5wAAFxcXDUdCCKnL8+fPYW1tXW8ZanrSALlcjpycHFhZWcHAwEDT4TQZRb99Dx480NsmNnzCl/3JGMPz58/RqlWrGj0cvYxqdg0QCARo06aNpsNoNiKRSK8PDr7hw/5sqEanQDcoCCG8QMmOEMILlOwIgKru6BcuXFhrl/RE99D+rIluUBBCeIFqdoQQXqBkRwjhBUp2hBBeoGSnA+7evQsDAwNcvXq1WdZz5swZdOnSBcbGxggLC3ut9yREW1Cy00HJyckwMDBAUVFRk6w/KioK3t7eyM7OxubNm5vkPUhNAwYMwJdffqnpMDjaFs/romTXxCoqKjQdgtqysrLwzjvvoE2bNrCxsXmldTTndjPGUFlZ2Wzvp8108fvWbJpohDPe8vf3Z59//jn74osvWIsWLdiAAQPY9evXWVBQELOwsGCOjo5s7Nix7MmTJ9wyR44cYW+//TaztrZmdnZ2bNiwYSwzM5Obn52dzQCwK1eucH9X/zdhwgS11rNz507m5+fHhEIhe/PNN1lycrLS/Or/Nm3axBhjLDk5mfXs2ZOZmJgwZ2dnNnfuXCaVSuvd7pMnTzIALDExkXl7ezNTU1M2cOBA9vjxY5aQkMC8vLyYlZUVGzVqFCstLeXWJZPJ2Pfff8/c3NyYqakp69q1KzeYN2OMW29CQgLr3r07MzY2ZidPnmzM3djsJkyYUOOzz8zMZJMnT+Y+h/bt27NVq1bVWC40NJR9++23rGXLlszNzY0xxtiZM2dYt27dmFAoZD4+Pmzfvn3cd0ihvu9lbfFkZ2c318fRJCjZNTJ/f39maWnJZs+ezW7dusXOnz/PHBwc2Lx581h6ejq7fPkyGzx4MBs4cCC3zJ49e9jevXvZ7du32ZUrV1hISAjr0qULk8lkjDHlZFdZWcn27t3LALCMjAyWm5vLioqK1FpPmzZt2J49e9jff//NPv74Y2ZlZcUKCgpYZWUly83NZSKRiK1atYrl5uaysrIy9vDhQ2Zubs4+++wzlp6ezvbt28fs7e3ZwoUL69zuW7ducUmpd+/e7K+//mKXL19mnp6ezN/fnwUGBrLLly+z06dPsxYtWrAlS5Zw6/r222+Zl5cXS0xMZFlZWWzTpk1MKBRySVmx3q5du7Jjx46xzMxM9vTp06betU2qqKiI+fn5sU8++YTl5uay3Nxc9uLFC7ZgwQJ28eJFdufOHfbbb78xc3Nz9vvvv3PLTZgwgVlaWrJx48axGzdusBs3brDi4mJmZ2fHxo4dy27evMkSEhJY+/btlZLds2fP6v1e1hZPZWWlJj6aRkPJrpH5+/uzt956i3u9ePFiFhgYqFTmwYMHXLKqzZMnTxgAdv36dcaYcrJj7L8H+7Nnz+qNpa71VE8sUqmUtWnThi1dupSbZm1tzdXoGGPsq6++Yh06dGByuZybtnbtWmZpackl0pe3u3qcf/75JzctJiaGAWBZWVnctH/9619syJAhjDHGXrx4wczNzdnZs2eV1hUREcFGjRqltN74+Ph6t1/X+Pv7sy+++KLeMp9//jkbMWIE93rChAnMycmJSSQSbtq6detYixYtWHl5OTft559/VvoOqfK9VCUeXULX7JqAj48P9/e1a9dw8uRJWFpacv+8vLwAVF0bA4Dbt29j1KhR8PDwgEgkgpubGwDg/v37ar2vquvx8/Pj/jYyMkKPHj2Qnp5e53rT09Ph5+en1MXV22+/jZKSEjx8+LDW7a6ua9eu3N9OTk4wNzeHh4eH0rT8/HwAQGZmJsrKyjB48GClz2zr1q3c56XQo0ePOmPWF2vXroWPjw8cHBxgaWmJDRs21NifXbp0gYmJCfc6IyMDXbt2hampKTetV69eSsuo8r3UN9TFUxOwsLDg/i4pKUFISAiWLl1ao1zLli0BACEhIXB1dcXPP/+MVq1aQS6Xo3PnzmpfbG6s9byq6ttdXfWecg0MDGr0nGtgYAC5XA6g6vMCgMOHD6N169ZK5V5+zrOu99MXu3btwqxZs7BixQr4+fnBysoKy5cvR2pqqlK5V/kcVPle6htKdk2se/fu2Lt3L9zc3GBkVPPjfvr0KTIyMvDzzz+jX79+AIC//vqr3nUqfsVlMtkrref8+fPo378/AKCyshJpaWmIjIys8/06duyIvXv3gjHG1e7OnDkDKyurRu/rr1OnThAKhbh//z78/f0bdd3azsTERGmfnjlzBn369MFnn33GTVOl1tWhQwf89ttvkEgk3A/ExYsXlco09L2sLR5dR6exTezzzz9HYWEhRo0ahYsXLyIrKwtHjx7FpEmTIJPJYGtrixYtWmDDhg3IzMzEiRMnEBUVVe86XV1dYWBggEOHDuHJkycoKSlRaz1r167Fvn37cOvWLXz++ed49uwZJk+eXOf7ffbZZ3jw4AGmTZuGW7duYf/+/Vi4cCGioqIa7B1WXVZWVpg1axZmzJiBLVu2ICsrC5cvX8bq1auxZcuWRn0vbePm5obU1FTcvXsXBQUFeOONN3Dp0iUcPXoU//zzD+bPn18jadVm9OjRkMvlmDJlCtLT03H06FHExsYCAPdj1dD3srZ4FLVvXUXJrom1atUKZ86cgUwmQ2BgILp06YIvv/wSNjY2EAgEEAgE2LVrF9LS0tC5c2fMmDEDy5cvr3edrVu3xqJFixAdHQ0nJydERkaqtZ4lS5ZgyZIl6NatG/766y8cOHAA9vb29b5fQkICLly4gG7duuHTTz9FREQEvv7669f6bOqyePFizJ8/HzExMejYsSOCgoJw+PBhuLu7N8n7aYtZs2bB0NAQnTp1goODA4YMGYLw8HCMHDkSvr6+ePr0qVItry4ikQgHDx7E1atX4e3tjX//+99YsGABAHDX8Rr6XtYWj7rXkLUNdfFECA9s374dkyZNQnFxMczMzDQdjkbQNTtC9NDWrVvh4eGB1q1b49q1a5g7dy4+/PBD3iY6gJIdIXopLy8PCxYsQF5eHlq2bIkPPvgA3333nabD0ig6jSWE8ALdoCCE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiNaY+LEiTAwMOB6RnFycsLgwYOxceNGtZ7L3Lx58yt3J/86Jk6cSAMUaTFKdkSrBAUFITc3F3fv3sWRI0cwcOBAfPHFFxg+fDiNM0Fej0a7DiWkGsV4Ci87fvw4A8B+/vlnxhhjK1asYJ07d2bm5uasTZs2bOrUqez58+eMsf/2Ylz9n6L7+K1btzIfHx9maWnJnJyc2KhRo9jjx4+59yksLGSjR49m9vb2zNTUlHl6erKNGzdy8+/fv88++OADZm1tzWxtbdm7777LjcuwcOHCGu+r6+Ni6Buq2RGt984776Bbt26Ii4sDAAgEAvz000+4efMmtmzZghMnTmDOnDkAgD59+mDVqlUQiUTIzc1Fbm4uZs2aBQCQSqVYvHgxrl27hvj4eNy9excTJ07k3mf+/Pn4+++/ceTIEaSnp2PdunVcbzBSqRRDhgyBlZUVUlJScObMGVhaWiIoKAgVFRWYNWsWPvzwQ65mmpubiz59+jTvB0Xqp+lsS4hCXTU7xhgbOXIk69ixY63zdu/ezVq0aMG93rRpE7O2tm7w/S5evMgAcLXCkJAQNmnSpFrLbtu2rcY4HBKJhJmZmbGjR482GD/RPKrZEZ3AqvWS/OeffyIgIACtW7eGlZUVxo0bh6dPn6KsrKzedaSlpSEkJARt27aFlZUV1xOyop+2qVOnYteuXfD29sacOXNw9uxZbtlr164hMzMTVlZW3JgNdnZ2ePHihd6O2aBvKNkRnZCeng53d3fcvXsXw4cPR9euXbF3716kpaVh7dq1AOofILq0tBRDhgyBSCTC9u3bcfHiRezbt09puaFDh+LevXuYMWMGcnJyEBAQwJ0Cl5SUwMfHB1evXlX6988//2D06NFNvPWkMVAXT0TrnThxAtevX8eMGTOQlpYGuVyOFStWcD3q/vHHH0rlaxs74datW3j69CmWLFkCFxcXAMClS5dqvJeDgwMmTJiACRMmoF+/fpg9ezZiY2PRvXt3/P7773B0dIRIJKo1Tn0bs0HfUM2OaBWJRIK8vDw8evQIly9fxvfff4/Q0FAMHz4c48ePh6enJ6RSKVavXo07d+5g27ZtWL9+vdI63NzcUFJSguPHj6OgoABlZWVo27YtTExMuOUOHDiAxYsXKy23YMEC7N+/H5mZmbh58yYOHTqEjh07AgDGjBkDe3t7hIaGIiUlBdnZ2UhOTsb06dO54STd3Nzwn//8BxkZGSgoKIBUKm2eD42oRtMXDQlRmDBhAtdsw8jIiDk4OLBBgwaxjRs3coNxM8bYypUrWcuWLZmZmRkbMmQI27p1a41Bwz/99FPWokULpaYnO3bsYG5ubkwoFDI/Pz924MCBGgNHd+zYkZmZmTE7OzsWGhrK7ty5w60zNzeXjR8/ntnb2zOhUMg8PDzYJ598woqLixljjOXn57PBgwczS0tLanqihajzTkIIL9BpLCGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBf+P4t7FKZ+DSZUAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T13:44:41.126054Z", + "iopub.status.busy": "2024-03-22T13:44:41.125713Z", + "iopub.status.idle": "2024-03-22T13:44:41.399039Z", + "shell.execute_reply": "2024-03-22T13:44:41.398099Z" + }, + "papermill": { + "duration": 0.295965, + "end_time": "2024-03-22T13:44:41.401130", + "exception": false, + "start_time": "2024-03-22T13:44:41.105165", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASsAAAEmCAYAAAA6OrZqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGPklEQVR4nO3dd3hTZfvA8W+SNt0LKB3Qsvde5QVcIIiKLAdTZDkBUXAAvgIypOiLyE9QEBABZYiAigooIEP23pRRChRoaUvpHlnn90ea0NCVtEmTlOdzXb2gJ+fk3C307nOec5/7kUmSJCEIguDg5PYOQBAEwRwiWQmC4BREshIEwSmIZCUIglMQyUoQBKcgkpUgCE5BJCtBEJyCSFaCIDgFF3sHYGs6nY7bt2/j4+ODTCazdziCIOQjSRLp6emEhoYilxc/dqrwyer27duEhYXZOwxBEIoRGxtL9erVi92nwicrHx8fQP/N8PX1tXM0giDkl5aWRlhYmPHntDgVPlkZLv18fX1FshIEB2XOFI2YYBcEwSmIZCUIglMQyUoQBKdQ4eeszCFJEhqNBq1Wa+9QBMFqFAoFLi4uFaZk56FPViqViri4OLKysuwdiiBYnaenJyEhISiVSnuHUmYPdbLS6XTExMSgUCgIDQ1FqVRWmN9CwsNNkiRUKhWJiYnExMRQr169EosurSlXo8XNRWHV93yok5VKpUKn0xEWFoanp6e9wxEEq/Lw8MDV1ZXr16+jUqlwd3cvl/Nm5Gp4/POdPF4/kOl9muLtZp00IybYoVx/4whCebLH/+3fTt7ibqaKkzdT8FJab3Rl15/SPXv20LNnT0JDQ5HJZPz6668mr0uSxJQpUwgJCcHDw4OuXbty+fJl+wQrCEKJJEli1cEbAAyKCLfqtIpdk1VmZiYtWrTg66+/LvT1zz//nK+++opFixZx6NAhvLy86N69Ozk5OeUcqSAI5jh1M5XzcWkoXeS82Kb4Z/0sZddk9cwzzzBz5kz69u1b4DVJkpg3bx4ff/wxvXv3pnnz5qxcuZLbt28XGIEJ1nHt2jVkMhknT54sl/fZt28fzZo1w9XVlT59+pTpnA+rXbt2IZPJSElJsXcoAKw6eB2A55qH4O9p3TuQDjtZExMTQ3x8PF27djVu8/Pzo3379hw4cKDI43Jzc0lLSzP5EErH1j8I48ePp2XLlsTExLB8+XKbnEMoP6nZan4/fRuAwe3Drf7+Dpus4uPjAQgKCjLZHhQUZHytMJGRkfj5+Rk/Hpb2MCqVyt4hWCw6OpouXbpQvXp1/P39S/Ue5fl1G4qHrc0Z/+0K88vxm+SodTQM9qF1eIDV399hk1VpTZo0idTUVONHbGys2cdKkkSWSmOXD0sXxn7iiScYM2YM7777LlWqVKF79+6cPXuWZ555Bm9vb4KCghgyZAhJSUnGY7Zu3cojjzyCv78/lStX5rnnniM6OrrQ97927RqdO3cGICAgAJlMxrBhwyx6n6ioKDp27Ii7uztNmzZl9+7dxveWyWTcvXuXESNGIJPJjCOr3bt3ExERgZubGyEhIUycONEkQRT2dRtGgH/99RetWrXCw8ODLl26kJCQwJYtW2jUqBG+vr4MGjTIpPhXp9MRGRlJrVq18PDwoEWLFqxfv974uuF9t2zZQps2bXBzc2Pv3r0l/tvMnDmTqlWr4uPjw6uvvsrEiRNp2bKl8fVhw4bRp08fPv30U0JDQ2nQoAEAP/zwA23btsXHx4fg4GAGDRpEQkKCyXtv3ryZ+vXr4+HhQefOnbl27VqJ8ZQHSZJYdUg/sT64vXUn1g0cts4qODgYgDt37hASEmLcfufOHZN/+Ae5ubnh5uZWqnNmq7U0nvJXqY4tq/PTu+OptOyfY8WKFbz11lvs27ePlJQUunTpwquvvsqXX35JdnY2EyZMoF+/fvzzzz+A/obG+PHjad68ORkZGUyZMoW+ffty8uTJAre4w8LC2LBhAy+88AIXL17E19cXDw8Pi97ngw8+YN68eTRu3Ji5c+fSs2dPYmJiCAsLIy4ujgYNGjB9+nT69++Pn58ft27d4tlnn2XYsGGsXLmSqKgoXnvtNdzd3fnkk08K/boB4uLiAPjkk09YsGABnp6e9OvXj379+uHm5sbq1avJyMigb9++zJ8/nwkTJgD6UfiPP/7IokWLqFevHnv27OHll18mMDCQxx9/3Hi+iRMnMmfOHGrXrk1AQPEjhlWrVvHpp5/yzTff0KlTJ9auXcsXX3xBrVq1TPbbsWMHvr6+bNu2zbhNrVYzY8YMGjRoQEJCAuPHj2fYsGFs3rwZ0Pdke/755xk9ejSvv/46R48e5b333iv5P0o5OHLtHpcTMvBwVdC7VTWbnMNhk1WtWrUIDg5mx44dxuSUlpbGoUOHeOutt+wbnIOoV68en3/+OaD/bd6qVStmzZplfH3ZsmWEhYVx6dIl6tevzwsvvGBy/LJlywgMDOT8+fM0bdrU5DWFQkGlSpUAqFq1qsllmrnvM2bMGOO+CxcuZOvWrXz33Xd8+OGHBAcHI5PJ8PPzM/5i+uabbwgLC2PBggXIZDIaNmzI7du3mTBhAlOmTDEmwvxfN9xPVjNnzqRTp04AjBw5kkmTJhEdHU3t2rUBePHFF9m5cycTJkwgNzeXWbNmsX37djp06ABA7dq12bt3L99++61Jspo+fTrdunUr+R8EmD9/PiNHjmT48OEATJkyhb///puMjAyT/by8vFi6dKnJYzAjRoww/r127dp89dVXtGvXjoyMDLy9vVm4cCF16tThiy++AKBBgwacOXOGzz77zKzYbGn1If3Eeu+Wofi6u9rkHHZNVhkZGVy5csX4eUxMDCdPnqRSpUqEh4fz7rvvMnPmTOrVq0etWrWYPHkyoaGhNrtz5OGq4Pz07jZ5b3PObak2bdoY/37q1Cl27tyJt7d3gf2io6OpX78+ly9fZsqUKRw6dIikpCR0Oh0AN27cKJCsimPu+xiSAICLiwtt27blwoULRb7vhQsX6NChg8klRKdOncjIyODmzZuEh4cX+Lrza968ufHvQUFBeHp6GhOVYdvhw4cBuHLlCllZWQWSkEqlolWrVibb2rZtW2TMD7p48SKjRo0y2RYREWEc3Ro0a9aswPN6x44d45NPPuHUqVPcu3fP5PvauHFjLly4QPv27U2Oyf89tpfkTBWbz+jnkQe3r2Gz89g1WR09etQ4LwL6u0MAQ4cOZfny5Xz44YdkZmby+uuvk5KSwiOPPMLWrVtt9tiATCaz+FLMnry8vIx/z8jIoGfPnoX+ljVcRvfs2ZMaNWqwZMkSQkND0el0NG3a1OIJXmu9T2nl/7rzc3W9/xtdJpOZfG7YZkgAhpHOn3/+SbVqppctD04jFHW+snjwPTMzM+nevTvdu3dn1apVBAYGcuPGDbp37+7wE/Drj8Wi0upoVs2PZtX9bHYeu/5kPvHEE8VOLMtkMqZPn8706dPLMSrn1Lp1azZs2EDNmjVxcSn4z3r37l0uXrzIkiVLePTRRwFKnCw2/ObP3zrHkvc5ePAgjz32GAAajYZjx44xZsyYIs/XqFEjNmzYgCRJxtHVvn378PHxKXExAUs1btwYNzc3bty4YXLJV1YNGjTgyJEjvPLKK8ZtR44cKfG4qKgo7t69y+zZs413sI8ePWqyT6NGjdi0aZPJtoMHD1oh6tLT6STWHNbfxLJFuUJ+Fe5u4MNq9OjRJCcnM3DgQI4cOUJ0dDR//fUXw4cPR6vVEhAQQOXKlVm8eDFXrlzhn3/+MY5ki1KjRg1kMhl//PEHiYmJZGRkWPQ+X3/9Nb/88gtRUVGMHj2ae/fumczLPGjUqFHExsby9ttvExUVxW+//cbUqVMZP3681Z9x8/Hx4f3332fcuHGsWLGC6Ohojh8/zvz581mxYkWp3/ftt9/mu+++Y8WKFVy+fJmZM2dy+vTpEu+OhYeHo1QqmT9/PlevXmXTpk3MmDHDZJ8333yTy5cv88EHH3Dx4kVWr15t9/q0A1fvEpOUiY+bCz1bhNr0XCJZVRChoaHs27cPrVbLU089RbNmzXj33Xfx9/dHLpcjl8tZu3Ytx44do2nTpowbN47//e9/xb5ntWrVmDZtGhMnTiQoKIgxY8ZY9D6zZ89m9uzZtGjRgr1797Jp0yaqVKlS7Pk2b97M4cOHadGiBW+++SYjR47k448/LtP3pigzZsxg8uTJREZG0qhRI55++mn+/PPPAnfuLDF48GAmTZrE+++/T+vWrYmJiWHYsGElTl0EBgayfPlyfv75Zxo3bszs2bOZM2eOyT7h4eFs2LCBX3/9lRYtWrBo0SKTGyr2sCpvYr1Pq2p4Wam7QlFkkqUFPk4mLS0NPz8/UlNTC6xuk5OTQ0xMDLVq1Sq39hnCw6dbt24EBwfzww8/lPu5bfl/PCE9h46R/6DRSWx551EahVi+elRxP58Pcp7ZZEFwAllZWSxatIju3bujUChYs2YN27dvN6mnqih+PnoTjU6iTY2AUiUqS4nLQEGwQJMmTfD29i70Y9WqVchkMjZv3sxjjz1GmzZt+P3339mwYYPJM64VgVYnsfrQ/VYw5UGMrATBAps3b0atVhf6WlBQEB4eHmzfvr2coyp/ey4ncislGz8PV3o0Dyn5ACsQyUoQLFCjhu2KHp2JocHei22q416KgubSEJeBgiBY5HZKNv9E3QFgYDldAoJIVoIgWOinI7HoJPhP7UrUrVrw8S5bEclKEASzabQ61h4xtIIp30tikawEQTDbjqgE7qTlUtlLSfcmweV6bpGsBEEwm6Fc4aW2YShdyjd9iGT1EKpZsybz5s0zfl7YMmiC8KAbd7PYczkRKL/aqvxE6YJAXFxciR0wBWHNkRtIEjxarwrhlct/BXORrARjp057kiQJrVZbaHsbwf5UGh0/HzW0grFPrZm4DMxPkkCVaZ8PC58nT09PZ/DgwXh5eRESEsKXX37JE088wbvvvmvxl53/MtCwmMPGjRvp3Lkznp6etGjRosDyZ3v37uXRRx/Fw8ODsLAwxo4dS2ZmpvH1khY/KO1iDIJ9/H0+nqQMFUG+bjzZqKpdYhC/xvJTZ8Es2/bkKdJHt0FpfkfK8ePHs2/fPjZt2kRQUBBTpkzh+PHjxS6mYYn//ve/zJkzh3r16vHf//6XgQMHcuXKFVxcXIiOjubpp59m5syZLFu2jMTERMaMGcOYMWP4/vvvgZIXPzCwZDEGwX4MFev924bhqrDPGEckKyeUnp7OihUrWL16NU8++SQA33//PaGh1ku077//Pj169ABg2rRpNGnShCtXrtCwYUMiIyMZPHiwcRRXr149vvrqKx5//HEWLlyIu7t7iYsfGFiyGINgH9GJGRy4ehe5DPrbYWLdQCSr/Fw99SMce53bTFevXkWtVhMREWHc5ufnZ1x/zhryL75g6OGekJBAw4YNOXXqFKdPn2bVqlXGfSRJQqfTERMTQ6NGjUpc/MDAksUYBPtYk1eu0KVhVar5e9gtDpGs8pPJLLoUq8geXHwBMFls4Y033mDs2LEFjgsPD7do8QNbLMYgWE+OWsv64zcBGGTjHuslEcnKCdWuXRtXV1eOHDliXJ4qNTWVS5cuGRdosKXWrVtz/vx56tatW+jrZ86cKXHxA8E5bDkbR0qWmmr+Hjxe3z4T6wbibqAT8vHxYejQoXzwwQfs3LmTc+fOMXLkSORyuU2W7X7QhAkT2L9/P2PGjOHkyZNcvnyZ3377zbhyjTmLHwjOwTCxPjAiDIXc9v+3iiOSlZOaO3cuHTp04LnnnqNr16506tSJRo0alUsv+ebNm7N7924uXbrEo48+SqtWrZgyZYpxgt+cxQ8Ex3cxPp2j1+/hIpfRr22YvcMRC0ZUlAUjMjMzqVatGl988QUjR460dziCgyjL//Gpv51lxYHrPNM0mIUvF74KdlmJBSMeAidOnCAqKoqIiAhSU1ONC8H27t3bzpEJFUGWSsPG47cA+1WsP0gkKyc2Z84cLl68iFKppE2bNvz7779cuHCBZ555pshjDMumC0Jxfj91m/RcDTUqe9KxTmV7hwOIZOW0WrVqxbFjxwpsz87O5uTJk+UfkFCh5F+5Rm7niXUDkawqGA8PjyJLCgTBHGdupnLqZipKhZwX21S3dzhG4m4g+uprQaiISvN/e/Vh/ZLwTzcNprK3m7VDKrWHOlkZqrSzsrLsHIkg2Ibh/3b+JxKKk56j5reT+kfOBtu5Yv1BDn0ZqNVq+eSTT/jxxx+Jj48nNDSUYcOG8fHHH1ul+FGhUODv729sXeLp6VkuRZWCYGuSJJGVlUVCQgL+/v4oFOat7ffrydtkqbTUrepNRK1KNo7SMg6drD777DMWLlzIihUraNKkCUePHmX48OH4+fkV+lxaaRgaz+XvtSQIFYW/v7/ZzRUlSWLVQf0l4KCIcIf7xe3QyWr//v307t3b2KqkZs2arFmzhsOHD1vtHDKZjJCQEKpWrVrksuCC4IxcXV3NHlEBnIhNISo+HTcXOS+0dpyJdQOHTlYdO3Zk8eLFXLp0ifr163Pq1Cn27t3L3LlzizwmNzeX3Nxc4+dpaWlmnUuhUFj0DysIFY3hOcCeLULx8zRvjqs8OXSymjhxImlpaTRs2BCFQoFWq+XTTz9l8ODBRR4TGRnJtGnTyjFKQXB+qVlq/jitn1i3dyuYojj03cB169axatUqVq9ezfHjx1mxYgVz5sxhxYoVRR4zadIkUlNTjR+xsbHlGLEgOKcNx2+Sq9HRKMSXVmH+9g6nUA49svrggw+YOHEiAwYMAKBZs2Zcv36dyMhIhg4dWugxbm5uuLk5Tm2IIDg6SZJYdUg/sT64veNNrBs49MgqKysLudw0RIVCYexYKQhC2R2KSSY6MRNPpYLeLe20YIoZHHpk1bNnTz799FPCw8Np0qQJJ06cYO7cuSaLEQiCUDaG5wB7t6yGj7vjTawbOHSymj9/PpMnT2bUqFEkJCQQGhrKG2+8wZQpU+wdmiBUCEkZuWw5Gwc4XsX6gxw6Wfn4+DBv3jzmzZtn71AEoUJaf+wmaq1Ei+p+NK3mZ+9wiuXQc1aCINiOTiex5rD+EtBRGuwVRyQrQXhI7YtO4vrdLHzcXXiuRYi9wymRSFaC8JAyVKw/36oankqHnhECyjhnlZGRUaCMoKSm74Ig2N+dtBy2XbgDwCAnuASEUoysYmJi6NGjB15eXvj5+REQEEBAQAD+/v4EBATYIkZBEKxs3ZFYtDqJdjUDaBDsY+9wzGLxyOrll19GkiSWLVtGUFCQw1a7CoJQOG2+iXVHfQ6wMBYnq1OnTnHs2DEaNGhgi3gEQbCx3ZcSuJ2ag7+nK880dfyJdQOLLwPbtWsnHg4WBCdmmFh/qU113F2dpy2SxSOrpUuX8uabb3Lr1i2aNm1aoLdz8+bNrRacIAjWdSslm50X9V1xB0Y4zyUglCJZJSYmEh0dzfDhw43bZDIZkiQhk8nQarVWDVAQBOv56fANdBJ0rFOZ2oHe9g7HIhYnqxEjRtCqVSvWrFkjJtgFwYmotTrWHtFP4ThDxfqDLE5W169fZ9OmTWIhTUFwMjsu3CEhPZcq3kq6NQ6ydzgWs3iCvUuXLpw6dcoWsQiCYEOr8lrB9GsbhtLF+R5esXhk1bNnT8aNG8eZM2do1qxZgQn2Xr16WS04QRCs4/rdTP69nIRM5nwT6wYWJ6s333wTgOnTpxd4TUywC4JjWp1XBPpYvUDCKnnaOZrSsThZiZbCguBccjVa1h+9CTh+g73iWHThqlarcXFx4ezZs7aKRxAEK/vr3B3uZqoI9nWnS8Oq9g6n1CxKVq6uroSHh4tLPUFwIoYl4fu3C8NF4XwT6wYWR/7f//6Xjz76iOTkZFvEIwiCFV1JyOBQTDJyGQyICLN3OGVi8ZzVggULuHLlCqGhodSoUQMvLy+T148fP2614ARBKBvDyjVPNgoixM/DztGUjcXJqk+fPjYIQxAEa8tRa1l/TF+x7kytYIpicbKaOnWqLeIQBMHK/jwdR1qOhuoBHjxWL9De4ZRZqdsaHzt2jAsXLgDQpEkTWrVqZbWgBEEoO8OS8AMjwlHInf8ZXouTVUJCAgMGDGDXrl34+/sDkJKSQufOnVm7di2Bgc6fwQXB2V2IS+P4jRRc5DJealvd3uFYhcV3A99++23S09M5d+4cycnJJCcnc/bsWdLS0hg7dqwtYhQEwUKGifXuTYKp6uNu52isw+KR1datW9m+fTuNGjUybmvcuDFff/01Tz31lFWDEwTBcpm5Gn45cQtw7or1B1k8stLpdAUeXgZ9wah4FEcQ7G/Tqdtk5GqoVcWLDnUq2zscqylVi5h33nmH27dvG7fdunWLcePG8eSTT1o1OEEQLGe4BBwUEV6hmmNanKwWLFhAWloaNWvWpE6dOtSpU4datWqRlpbG/PnzbRGjIAhmOn0zhTO3UlG6yHmhTcWYWDeweM4qLCyM48ePs337dqKiogBo1KgRXbt2tXpwgiBYxrByzbNNg6nkpbRzNNZVqjormUxGt27d6Natm7XjKeDWrVtMmDCBLVu2kJWVRd26dfn+++9p27atzc8tCM4kLUfNplP66ZnB/7Fjj3VJgiNLIbAh1HrUam9bqmS1Y8cOduzYQUJCQoFJ9WXLllklMIB79+7RqVMnOnfuzJYtWwgMDOTy5ctimXpBKMSvJ26RrdZSP8ibtjXs9DOiUcGWD+DYcvCoBKMOgo91+r1bnKymTZvG9OnTadu2LSEhITadwPvss88ICwvj+++/N26rVauWzc4nCM5KkiTjJaDdJtYzEmHdK3BjPyCDTu+At/X6Z1mcrBYtWsTy5csZMmSI1YIoyqZNm+jevTsvvfQSu3fvplq1aowaNYrXXnutyGNyc3PJzc01fp6WlmbzOAXB3o7fuMfFO+m4u8rp29oOE+txp2HtIEiNBTdfeGEp1O9u1VNYfDdQpVLRsWNHqwZRlKtXr7Jw4ULq1avHX3/9xVtvvcXYsWNZsWJFkcdERkbi5+dn/AgLc+4ePoJgDsOoqleLUPw8CtZB2tS5X2BZd32iqlQbXt1u9UQFIJMkSbLkgAkTJuDt7c3kyZOtHsyDlEolbdu2Zf/+/cZtY8eO5ciRIxw4cKDQYwobWYWFhZGamoqvr6/NYxaE8nYvU0X7yB2oNDp+Hd2JlmH+5XNinQ52RcKez/Wf1+kCLy4DD/Pny9LS0vDz8zPr59Piy8CcnBwWL17M9u3bad68eYFq9rlz51r6lkUKCQmhcePGJtsaNWrEhg0bijzGzc0NNzc3q8UgCI5uw/GbqDQ6moT60qK6X/mcNDcdfnkTov7Qf95hDHSdBopSN3IpkcXvfPr0aVq2bAlQYOEIa0/qderUiYsXL5psu3TpEjVqON/S14JgC5IkGSvWB7evUT4T68kx+vmphPOgUELP/4OWg2x+WouT1c6dO20RR6HGjRtHx44dmTVrFv369ePw4cMsXryYxYsXl1sMguDIDly9y9WkTLyUCnq1DLX9CWP26O/4Zd8D7yDovwrC2tn+vJRigr08tWvXjl9++YU1a9bQtGlTZsyYwbx58xg8eLC9QxMEh2AYVfVpVQ1vN9tdgiFJcHgJrOyjT1ShreD1XeWWqKAMnULLy3PPPcdzzz1n7zAEweEkpufy17l4QH8JaDP5Cz0BmvWDXl+Ba/kuQOHwyUoQhML9fCwWtVaiZZg/jUNtdKf7wULPrp/oiz3tUHQqkpUgOCGdTmLNYcPEuo0a7JVDoaclLJ6z2rNnDxqNpsB2jUbDnj17rBKUIAjF+/dKErHJ2fi6u/BccxtMrJdToaclLE5WnTt3LnQ15tTUVDp37myVoARBKJ5hSfjnW1fHQ6mw3hvrdPDPp/DzMFBn6Qs9X/sHAhtY7xylZPFloCRJhdZy3L17t8DqzIIgWF98ag47ohIAK18C2qHQ0xJmR/H8888D+sLPYcOGmVSJa7VaTp8+XW7PDArCw+ynI7FodRIRtSpRL8jHOm9qp0JPS5idrPz89GX8kiTh4+ODh8f925ZKpZL//Oc/xXZDEASh7DRaHWuPWHli3Y6FnpYwO1kZekrVrFmT999/X1zyCYId7LyYSFxqDpW8lDzdNLhsb2bo6LllAkhafaHngNXgWw6V8KVg8cXo1KlTbRGHIAhmWJ23JPyLbarj5lKGiXWNCja/D8fz2i3ZqdDTEhbfDbxz5w5DhgwhNDQUFxcXFAqFyYcgCLYRm5zFrkuJAAyMKMMlYEYirOyVl6hk+kn05xc7dKKCUoyshg0bxo0bN5g8ebLN2xoLgnDf2iM3kCR4pG4ValUp5TRMgULP76C+c6ykbnGy2rt3L//++6+xTYwgCLan1ur46chNAAaVdmL93C/w6yh9/VSl2jBwrUPUT5mrVOsGWthcVBCEMtp2/g5JGbkE+rjRrbGFq8VYoaOnI7B4zmrevHlMnDiRa9eu2SAcQRAKsypvYr1/2zBcFRb82Oamw7oh9xNVhzEw6GenS1RQipFV//79ycrKok6dOnh6ehZoa1zYoziCIJReTFIm+67cRSaDAREWLIDiBIWelrA4Wc2bN88GYQiCUBRDd4Un6gdSPcDTvIOcpNDTEhYnq6FDh9oiDkEQCpGj1vLz0VjAzAZ7TlboaYlStTWOjo7m448/ZuDAgSQk6B+o3LJlC+fOnbNqcILwsPvrXDz3stSE+LnzRIPA4nfWqOD3d/TFnpJWX+g5fEuFSFRQimS1e/dumjVrxqFDh9i4cSMZGRkAnDp1SlS3C4KVGRYvHdAuHJfiJtadtNDTEhYnq4kTJzJz5ky2bduGUqk0bu/SpQsHDx60anCC8DC7dCedw9eSUchl9G9XzMR63GlY0hluHNAXeg5aB4+8a5fWw7Zk8ZzVmTNnWL16dYHtVatWJSkpySpBCYJwf+WaJxtWJdjPvfCdTAo96+QVetYvxyjLj8UjK39/f+Li4gpsP3HiBNWqVbNKUILwsMtWadlwXF+xPvg/hUysF9rRc0eFTVRQimQ1YMAAJkyYQHx8PDKZDJ1Ox759+3j//fd55ZVXbBGjIDx0fj99m/QcDWGVPHi0bhXTFytQoaclLL4MnDVrFqNHjyYsLAytVkvjxo3RarUMGjSIjz/+2BYxCsJDx3AJODAiHLk839xTBSv0tITFyUqpVLJkyRImT57M2bNnycjIoFWrVtSrV88W8QnCQ+fsrVROxqbgqpDxUpt8E+sVsNDTEqXuBB8eHk54uI3WKxOEh9jqvIr17k2CCfRxq9CFnpYwK1mNHz+eGTNm4OXlxfjx44vdd+7cuVYJTBAeRhm5Gn47cQvIawXjhB09bcWsZHXixAnUarXx70URjfgEoWx+O3mLTJWW2lW86BCk0xd63jiAvZdudwRmJaudO3cW+ndBEKxHkiRjxfqYxtnIlnRxyo6etuIYqxcKgsCpm6mcj0ujl+th+h7/FjTZFb7Q0xJmJSvDAqfm2LhxY6mDKcns2bOZNGkS77zzjmhVI1Q4qw/EMN5lHWMVv4IGqPMkvPhdha+fMpdZycqwwCnoh6q//PILfn5+tG3bFoBjx46RkpJiUVKz1JEjR/j2229p3ry5zc4hCPaSmpLMU+fep6vLUf0GB1u63RGY9Z0wLHAKMGHCBPr168eiRYuMS29ptVpGjRqFr6+vTYLMyMhg8ODBLFmyhJkzZ9rkHIJgN8kx6Ja9SFfZFVS44Nr7K2StBts7Kodj8eM2y5Yt4/333zdZI1ChUDB+/HiWLVtm1eAMRo8eTY8ePejatWuJ++bm5pKWlmbyIQgOK2YP0pLOBGRcIUHyZ3v770WiKoLFyUqj0RAVFVVge1RUFDqdzipB5bd27VqOHz9OZGSkWftHRkbi5+dn/AgLs6BntSCUF0mCw0tgZR9k2fc4pavNS7pIHun8jL0jc1gWXxAPHz6ckSNHEh0dTUREBACHDh1i9uzZDB8+3KrBxcbG8s4777Bt2zbc3YtokfGASZMmmRSupqWliYQlOJYHCj2P+nZlcMLL9GlbB1931xIOfnjJJAsXAdTpdMyZM4f/+7//M7aKCQkJ4Z133uG9996z6hLyv/76K3379jV5T61Wi0wmQy6Xk5ubW+L50tLS8PPzIzU11WZzaoJgtoxEfceEvELPrMcm03JHI1RaiU1jOtG8ur+9IyxXlvx8WpysHjwRYLMkkJ6ezvXr1022DR8+nIYNGzJhwgSaNm1qVowiWQkOoZCl2xfH12HW5iiaVfPj97cfsXeE5c6Sn88y3Re19Q+/j49PgYTk5eVF5cqVzUpUguAwzv0Cv7xlUuipq1yPNZt2A2VYEv4hUqpktX79etatW8eNGzdQqVQmrx0/ftwqgQlChaDTwa5ZsOd/+s/zFXoeuJJETFIm3m4u9GrxcHVQKA2L7wZ+9dVXDB8+nKCgIE6cOEFERASVK1fm6tWrPPOM7e9k7Nq1S1SvC84hNx1+evl+ouowRr+YQ15FumFJ+L6tquHlJoo/S2Jxsvrmm29YvHgx8+fPR6lU8uGHH7Jt2zbGjh1LamqqLWIUBOeTHANLu8HFP/UdPfsshO6fGivSE9Jz+PvcHUBcAprL4mR148YNOnbsCICHhwfp6ekADBkyhDVr1lg3OkFwRjF79EtjJV7Qd/QctrlA6+Gfj95Eo5NoHe5PoxBx48ccFier4OBgkpOTAX23UMNagTExMZThxqIgOL98hZ5k39N39Hx9V4HWw1qdZOyxbtaS8AJQimTVpUsXNm3aBOjLCMaNG0e3bt3o378/ffv2tXqAguAULFi6fc/lRG6lZOPn4UqP5iF2CNY5WTyrt3jxYuNjNaNHj6Zy5crs37+fXr168cYbb1g9QEFweA8UepbU0dPQYO+F1tVxd7VeEXVFZ1Gy0mg0zJo1ixEjRlC9enVAv47ggAEDbBKcIDi8Qgo9i+voeTslm3+ixMR6aVh0Geji4sLnn3+ORqOxVTyC4DzO/QLfPaVPVJXqwKs7Smw9/NORWHQStK9VibpVvcsp0IrB4jmrJ598kt27d9siFkEwmyRJ/HUunut3M8v/5Dod/DNTv3S7Jltf6GnG0u0arY61R/Im1gtbEl4olsVzVs888wwTJ07kzJkztGnTBi8vL5PXe/XqZbXgBKEoJ2NTeOOHY3i7ubDklbZ0qFO5fE6cmw4b39DXT4FFHT13RCVwJy2Xyl5KujcJsnGgFY/FyWrUqFFA4esDymQytFpt2aMShBLcSM4C9OvsDf3+MF8Pak23xjZOAMkxsGagvn5KoYSeX0HLgWYfbihXeLFtddxcxMS6pSy+DNTpdEV+iEQllJd7mfpnUhVyGSqNjjd/PMaGYzdtd8Kruwsp9DQ/Ud24m8Wey4kADIoQE+ulYXGyEgRHkJylX3S3X9vqvNimOlqdxHs/n2Lpv1eteyJJgkOL4Ye+xRZ6lmTNkRtIEjxarwo1KnuVfIBQgNmXgdnZ2ezYsYPnnnsO0HfkzM3NNb6uUCiYMWOG2R09BaEsDCOrQG833u1aH38PV5bujWHmnxdIyVLz3lP1y75CuJWWbldpdPx8NBaAwaJcodTMTlYrVqzgzz//NCarBQsW0KRJEzw89P9wUVFRhIaGMm7cONtEKgj5JGfpk1WAlxK5XMZ/ezQiwEvJ//66yIKdV7iXpWJ676Yo5KVMWBYWehbn7/PxJGWoqOrjxpONxMR6aZl9Gbhq1Spef/11k22rV69m586d7Ny5k//973+sW7fO6gEKQmFSDMnKUwnob+6M7lyXT/s2RSaDVYdu8M7aE6g0pVjEJO60fn7qxgF9oeegdfDIu6VKVHC/Yn1AuzBcFWLmpbTM/s5duXKFZs2aGT93d3dHLr9/eEREBOfPn7dudIJQhORM/ZxVgJfSZPvg9jWYP7AVrgoZf5yO49WVR8lSWVDEXIpCz+JEJ2Zw4Opd5DLoLybWy8TsZJWSkmIyR5WYmEjNmjWNn+t0OpPXBcGWDHNWlTyVBV57rnkoS4e2w8NVwZ5Liby89JBxJFakUhZ6lmRNXrlC5wZVqeZv2VyXYMrsZFW9enXOnj1b5OunT582Pi8oCLYkSVK+OavCl656vH4gq15rj5+HK8dvpND/24PcScsp/A1L6OhZWjlqLeuP68spBv9HjKrKyuxk9eyzzzJlyhRycgr+g2dnZzNt2jR69Ohh1eCEimXb+Ts8+3//ciHO/FWyczVariVlsu9KEuuOxPLltku89/Mp41xUJa+CIyuD1uEBrHujA1V93Lh4J50XF+0v+HhOgY6ei0w6epbFlrNxpGSpqebvweP1q5b5/R52Zi/FdefOHVq2bIlSqWTMmDHUr68fHl+8eJEFCxag0Wg4ceIEQUGOdbdDLMXlOGpO1D+i0jrcn42jOgGQpdIQm5zNrZQsbt3L5mZKNrfuZXMr78/EjFyK+h8a4ufO/oldSixRiE3O4uXvDnH9bhaBPm6sHBGh7855dTf8PFRfP+UdBP1XWVw/VZwXF+7n6PV7vNetPm8/Wc9q71uR2GQprqCgIPbv389bb73FxIkTjV1BZTIZ3bp145tvvnG4RCU4ppS8gs5j15MZvPQQOeri79i5u8qp5u9BtQBP/Z/+7lQL8KBD7Spm1VKFVfLk5zc7MHTZES7EpdHv2/380f4CNQ7P0DfKC20NA1YV2iivtC7Gp3P0+j0Uchn924kVwa3BorFurVq12Lp1K8nJyVy5cgWAunXrUqlSJZsEJ1Qcau39hOTjoZ9nOnEjhRy1DndXObWreFMtwINq/h5Uz/vT8HklL2WZCzyr+riz9vX/8Oby/fS69SU1Du0EQGrWD1kpCj1Lsjpv5ZpujYKo6isKpa2hVBfmlSpVIiIiwtqxCBWY4cFjAJe8Qk3D5d0zTUP4sn9Lm8fgp03hR9dZKFwOopNkfKYZwJozfWiRcpqWYf60DPOnRZg/VbzdynSeLJWGjcdvAWJi3ZrEYmVCuYhJvD+xfTdDX+KiM04llEMAcadhzUAUaTeR3Hz4IXQKyy+HkZuj5d/LSfx7Ocm4a/UAD1qE+dMqL4E1CfXDQ2l+l4TfT90mPVdDjcqedKpTxRZfzUNJJCuhXFxNyjD+PSlDX3agyxtZyW2drR5Yul02cC1DA+szSKvjYnw6J2NTOBmbwqnYFK4kZnDzXjY372Xz5+k4QN/ZoWGwDy3yklfLMH/qBnojL+JRHkMrmIER4UXuI1hOJCuhXMQk3R9ZZeRqyFFrkdBnK5v9PBezdDuAq0JO02p+NK3mx8t5nTvTctScvZnKibzkdTI2hYT0XM7dTuPc7TRjIvJ2c6F5dT+GdqxJ9ybBxlOeuZnKqZupuCpkvNRG1B1ak0hWBlq1/jdws5fK6bqk4tNodYxadZyW4f5cS8oyeS0x/X5JggwbfL9L2dHT192VjnWr0LGu/vJNkiTiUnOMietkbApnbqWSkathf/RdouLTeapxkPEGwOrD+on1Z5qGULmMc1+CKZGsDLZOhCNL4fLf0Gu+1e8OPYxe+vYAJ26k8Pf5OzQJNa2hScrIRZd3HSi39rO9ZezomZ9MJiPU34NQfw+eaaZf40+j1XE5IYPnv9lPcqaKKwkZ1AvyIT1HzW8nbwNi5RpbEI+AGwQ2BLkLnPkZlj0NqbfsHZFTu5KQzokbKcbPM3NNHyZOylAZ56zK3HcqvzJ29DSHi0JOoxBfWtfwB+BgjH6F8l9P3iZLpaVOoBfta4lyHmsTycog4jUY8it4VIK4k7D4CbhxyM5BOa/LdzJMPs9U6VteB+fVHCVl5BrnrKySqgp09Gxdqo6eloioqV+k4nBMMpIkseqg/hJwcPsa1k3AAuDgySoyMpJ27drh4+ND1apV6dOnDxcvXrTdCWs9Cq/vhKpNIDMBlveA4z/Y7nwV2M172SafZ+WNrGpU9gQgKT3XencDNSr4fSxs+UBfkd68PwzfbNWK9MJE5I2eDl29y/EbKUTFp+PmIueF1mJi3RYcOlnt3r2b0aNHc/DgQbZt24Zareapp54iM9OGa8UF1ISRf0OjnqBTw6YxsGUCaMXCrpa4nWqarAwjK2Oyysg1PrJVpruBGYmwshccXwnIoNt06Pttucw5tgr3R6mQk5Cey+wtFwB9exo/z8I7QQhl49AT7Fu3bjX5fPny5VStWpVjx47x2GOP2e7Ebt7w0kr9Le9ds+DQIkg4Dy+tAE/z5yIS03NRa3WEPkR9jPZHJ3H02j1y1IWvdGRYLCEpQ4W3u/6/X6kvmeJOwZpBkHbTrKXbrc3dVUGLMD+OXLvHkWv3AFGxbksOnawelJqaClDss4i5ubkmTQDT0sxvR2JCLocnJkBQY/0t8Jg9+onbAWv020qg00n0WrCXjBwNRz7uirvrw7FO3KAl+nk+pUvBQbtMpq8OB0jMyDWOskqVq85uhF9HGQs9Gbi2zI3ySqN9rcrGRNUw2IdWYf7lHsPDwqEvA/PT6XS8++67dOrUiaZNmxa5X2RkJH5+fsaPsLAyPvHeqCe8ug38a8C9a/BdN4j6s8TDbqVkE5eaQ3quhrRsddlicEKF9T73UroQ6KOvPUrKKOWclaGj5/rhVu3oWVoR+e76Df6PmFi3JadJVqNHj+bs2bOsXbu22P0mTZpEamqq8SM2NrbsJw9qAq/thJqPgioD1g6C3Z9TZKMl9L23DdQ6s1qGVXhebgoC8wolk9JLMWdlo46eZdGmRgCVvJRU9lLSp6VtJ/Qfdk5xGThmzBj++OMP9uzZU2LrZDc3N9zcSlc5nJGr4ea9LBoGF9IEzKsyDPkF/voIDi+GnZ/CnbPQZyEoCy5aGZ3vwV2NthQrrFRAXkoXY0eDtBwNuXmjL7NGVlYs9LQmLzcXNo99FJkMfNzFxLotOfTISpIkxowZwy+//MI///xDrVq1bHq+D9ef4ul5//Lt7ujCd1C4wrP/g57/B3JXOP8bfNcdUm4U2DX/yEojRlaA/gfbz8PV2CImMT1vbrGkXFUOhZ5lEeznTpDoWWVzDp2sRo8ezY8//sjq1avx8fEhPj6e+Ph4srOzSz64FC7GpwMQuSWKDcduFr1jm2Ew9HfwCoQ7Z/QFpNf2mewSnZAvWWlFsgLwVCqQy2VU9tb3TU9I1/fzL3JkZYdCT8FxOXSyWrhwIampqTzxxBOEhIQYP3766SebnC858/5yTR9uOM3OqISid67RQT+PFdICsu7qa32OfGd8+Wq+LgPqCnwZmJCew4w/znM1MaPEfb3d9LMOhkvBhLyRVaFzVnYq9BQcl0MnK0mSCv0YNmyY1c+l1Umk5N2169KwKlqdxKhVxzl+417RB/mHwfCt0PQF0Gngz/HwxzhSMzLvX+JQsS8D31lzku/2xtB7wb4S9/V8IFklGpPVA9nKjoWeguNy6GRVnlKyVMabe98Mbs0TDQLJVmsZsfwIVxLSiz5Q6akvRnxyCiCDo8uQ/9iHStyv79LqCh9ZLfjnMv2/PUC2qvACSmdgSObpuSVX+Hu76WvNDMkqK+/rNklVcafynsvMt3R7p3dE2x5BJCsDwyWgn4cr7q4KvhncmhZh/qRkqXnlu8PEpRYzTyaTwaPv6QsTlT74xB9mk9vHNJZdA0BdxJzVnL8vcSgmmbVHCk7QOwtLxoyeSv3IylBrZWCsTTq7UX/DIu2mVZZuFyoWkazy3M1LVpXzFs30VLrw/bB21A704nZqDq98d7jkJcgbPA2v7SDZvTrVZUmsV07jWfnBEifYb6fY5oZBubAgW3m5FZ6s5EiwY4bDFHoKjkkkqzz3Mg3Lkd9f4beSl5KVIyII8nXjckIGr644WvIlW2ADpgctYI+2GZ6yXL5RfkX1k3P1lddFuJtZQhJ0YOp8X1dJ6+V65S26UDVfsvIim+eiPoB/5+g3OEChp+CYRLLKY0gYDy5HXj3Ak5Uj2uPr7sLR6/d4e83xEos8z96TM1z9IUs0zwJQ89zX+srr3MLnvu5mOG+yyp+fdlwo5u4p90dWhmQVJrvDRuVU6tzdbfWl24WKRySrPMkPXAbm1yDYh6VD2+HmImf7hQQ++uVMkaMItVbH9buZaFHwvderjFe9iVau1PcCX9qN3IQrvPnDMRbuul94uvtSosn7bT0bz6nYFOt+geVgy9n4Yl/3yptgr+rrTgf5OTYpJ9NAfpNMZWWHK/QUHI9IVnmSC7kMzC+iViXmD2yFXAbrjt5kzt+FNwGMTc5CrZXwcFVQvZInG3WPceCxleAdDIkXkC19krQL2/n8ryiT4w5E3wXg/O003vzxGL2/LrkUwNHkaoq/RPZSuoAkEXpxJT+4RhIgy+CkrjYb2qwShZ5CiUSyytMw2IdujYNoHFLIc4F5nmoSzKy+zQD4emc03++LKbCP4ZnA2oFeKBX6b2+SX3N95XW1NihVKax0nc1Q+Vbyz04bViy+dKeYMgkHV1Lxq7eLDn4fi9u2ibjIdGzUPkJ/1RSy3KuWU4SCMxPJKs+AiHCWvNKWni2Kr5AeEBHOe930d6mm/3GeTadum7xueCawTqA3Lgr9LXm1Vge+ITBsM3s8nsRFpuMT15V85rIEJabtY7KLaFqXmq3mufn/sqio5xbL2f7oJHot2GuyrbC2MAaVSaX5P68YCz0Xug5lvPotclHabt1AoUIRyaoUxnSpyysdaiBJ8N66k+zNt/S44ZnAOoHexgd2P1h/ms+3RqFVuPFW5mvMUA9Gh5z+LrtYrfyUQFKM1fNF3W38bm8MZ2+lMXtLVKGvl7dBSw5x+maqyTZVESOrJrJrbHL7GO87R4yFnjsrD8RQDmrzFZmFCkEkq1KQyWRM7dmEHs1CUGsl3vjhKGfyfnANzwTWqeqFS74F8Rbujmbj8ZtkqnSsVfQiuc8q0iRP2sov8ZvbxzTUXgGKHlllmVEhbm+56oLJqof8IOuVn1BNdhdNwP1Cz0Df++ULomGdYA6RrEpJIZcxt38LOtapTKZKy7DvDxOTlMmVvJFV7SrenIu7P/KQJPjvr2cBaBHmT5WWz5I9bBtXdKGEypJ5bN8QOP2zSe9yXb5nCrX57hamZqtLrGmyh9x8l4EydLznso6vlV/hIVOxW9uczCF/GQs989daiVQlmEMkqzJwc1Hw7ZA2NA7x5W6mikFLDpKarUYmg1pVvIxN/Pw8XPFUKoxzOi3z+nQH1WrK5MD/Y4e2FXJtLmx8Fc89M5Cj3+9gzF3jufInrhbT/mbcTyfL54tE/5C3OQyJ1otsvnX9krddfgVgsaYHI9Qf4OlX2bhvVZ/7/Z/EnJVgDpGsysjH3ZXlI9oRXsmTuFR9f6Zq/h54KBV89Gwjxnapy54POvN2l3rGY1qF36/ODqhUmdfU73Gq5ggA3nL5naWuc/Ahy2Skon1gJPXrSdOJfVsZveo4dT7abNI+pyiXEzIIzyv0fEpxjFzJhfGqN5mlGYzCxRVXxf3/bvlHVnKRrQQziGRlBVV93Fk5IoIqeU3lagd6A/rR1finGuDn6cqIR2rSJNSXSl5KImreX2Qg1M8DHXKWKIcwVjWGHMmVLoqT/KqcjGfa/dIIe3WZ+fNMHABdvthl3JaeU/gCGB3lZ/ktr9AzQfJngGoyG3X6JdMMvawMAsVloGAh8VyDldSs4sWKERF8tvUirz5SsP2ym4uCjaM6IkmYLMtlWFPwj9NxQEeuqoJZrJxLHXkcNf5+AfyXQ72uJpeB9pCSpeavc/F0axRkXG7rPolXFH8zxeUHXGQ6Tupq84ZqPHe4n5Q9laZLkVUVE+yChcTIyoqahPqxckQEj9UPLPR1NxdFgfUDH1wA9axUm165n3JUVx8XdTqsfgn2fYXWAbqNvvHDMdYcucGZW/dvHLiiIdJlKdNdV5gUeuZPVFBwZJV/zsrcOTHh4SaSlZ1VK2S15iT8GKT6L2s1T4Ckg22TeTF2Bm6U7wPPhd1x/Cffw8qVSWWV8lMGuuxEJ8mYpR5oLPR80IMjq4B8S6zfzch9cHdBKEAkKzurFlB4q14VrkzUvMYU9VA0kpz26dv5STmdIJLLLbbCijzv5fX0MhR6RsgvkiZ5MEL9Pou1PSlqBsrrgZFV/ku/RCfuOiGUH5Gs7Cz/CKMgGSu13RminsQ9yZuW8qtscvuYVrLLxj3Sc9Q8PW8Pc/++SKaVC0dzC3l85viNFJNCz2hdCH1V09mla1Xoe7z/lL6uyktZ9PRokhhZCWYQycrOZDIZs59vVuw+B3RN6KWaQZQujCBZCmuVM3hBvocW0/7my22XiYpP56t/rtBk6l+cjE0psfuBuXIeqKYvrNCzr2o60VK1It/D10OfjKv4FLw07FRXX3fVv22YVeIVKjaZ5Iil0FaUlpaGn58fqamp+PoW3VHB3sauOVHgoegHeZHNXNeFdFccBWCp5hkiNYPQoiiw77z+LenTqugkUhy1Vsete9lcScjg1ZVHjef+0vUbnlIcA/SFnrM1A9EV8/uuRZg/P4yM4KfDsfRoHlLgZkKuRsvNe9nUySv1EB4+lvx8imTlIFKz1Gw6fZs7qTks2HmlyP1k6HjXZSPvuGwEYI+2GWPUb5NGwR/4a7N7WBzHtaRMnpizy2RbuOwOS1y/oIH8JrmSC5PUrxrrp/Kb+ExD44PWhz56En9PV9xcCiZSQTCw5OdTXAY6CD9PV4b8pwaPNyi87MFAQs6Xmhd5U/UuWZIbjynO8JtyMnVlBVeQTkjLsTiOd9aeMPm8uELPB+WffwvydReJSrAqkawcjLk1R1t1Ebyg+oSbUhVqye/wi3IqT8qPmezzzS7Le1/dX5xVYqjiL1a6zjZ29OyZO5MTUr0ij/XzKLzLqiBYg0hWDsaSAskLUg165c7koK4RPrJslrjOZZTiVwwdSH84eN3i7gwuCrmx0HNaCYWe+QX5ulG3qph7EmxHJCsHY2k1dzK+vKyaxEpNN+QyiQ9d1zHfdT7u5KLVSeyPvlvym+RThVRWK2eaVeiZ33tPNaBuVW/mD2zFqlfbW3ROQTCHSFYOJqJWJSp5KakfZP4oRYMLUzTDmaQeiVpS0FNxkPXKaYSSxAc/nyrxeK1O4kJcGrpbJ1mU/R7t5JfMKvSc268FdQK9AHgib66tZ4tQOtWtYnbsgmAucTfQAam1OlzkMr7ZFY27q4IZf5w3+9h2sigWKudRRZZGkuTLm6p3adrhaT7p1aTIYz7+9Qz3Dq9jjusiPGQqonUhvK4eX2T91JjOdZHJ9KMplUZHtkqLX7HFrYJQOFG6kI8zJqsHXYhL4/dTt6ni7cZ0MxJXNRJZrJxLE/l1VJKCKZrhTJn6GZ4PVJFLkkRiejY/ff6WsVHebm1z3la/TRpeRb5/aUoiBKEwFa504euvv6ZmzZq4u7vTvn17Dh8+bO+QylWjEF8+fLohQzvW5PMXmrPjvceZ268FUPiD0LcI5EXVVP7Qtkcp0zLbdSk/Tx/AzaT8bZYlBn+9nZP/e86ko+dw9YfFJipBsBeHT1Y//fQT48ePZ+rUqRw/fpwWLVrQvXt3EhKKX6q8IlLIZfRrF0adQG+eb12dfRO7sPuDJ3ikkDmibNwZox7L/9T9ABjqso3Y/3ua7/4+AsDjH33P1IR38zp6uho7ehZXkQ7Qq4SlygTBVhz+MrB9+/a0a9eOBQsWAKDT6QgLC+Ptt99m4sSJJR5fES4DS3InLYf2s3YU+XpX+THmuX6NtyyHWF0gX2t7M8FlLQGyDO5I/ryhGs9Jqa5x/3e71mN4x1qotDombTzNoPbhhFfywsfdhSBf9yLPIwiWqjBzViqVCk9PT9avX0+fPn2M24cOHUpKSgq//fZbgWNyc3PJzb3/FH9aWhphYWEVOlmBfkGJP87EMXbNiUJfrye7yRLXL6gpv2Pc9mBHz3pVvXm2WQjj8hZxFQRbsyRZOXRb46SkJLRaLUFBQSbbg4KCiIoqfLHPyMhIpk2bVh7hORS5XEavFqHGyzRJkpi44Qw/HY0F4LJUnd6qGSxw/YpHFWfZqH2ESepXyUXJ4PbhTOvVBBeFw88KCA8xh05WpTFp0iTGjx9v/NwwsnrYyGQyPnuxOZ+92Nxk+9GYriw+e44B3TrwvLsoNxCch0MnqypVqqBQKLhz547J9jt37hAcHFzoMW5ubri5uRX6mgBta1Wmba3CH0QWBEfm0ON+pVJJmzZt2LHj/uSxTqdjx44ddOjQwY6RCYJQ3hx6ZAUwfvx4hg4dStu2bYmIiGDevHlkZmYyfPhwe4cmCEI5cvhk1b9/fxITE5kyZQrx8fG0bNmSrVu3Fph0FwShYnPo0gVreBjqrATBWVW4x20EQRBEshIEwSmIZCUIglNw+An2sjJMyaWlpdk5EkEQHmT4uTRn6rzCJ6v09HSAh7KKXRCcRXp6On5+fsXuU+HvBup0Om7fvo2Pjw8yWeHteR8mhsePYmNjxd1RKxLf19KRJIn09HRCQ0ORy4uflarwIyu5XE716tXtHYbD8fX1FT9UNiC+r5YraURlICbYBUFwCiJZCYLgFESyesi4ubkxdepU0ZnCysT31fYq/AS7IAgVgxhZCYLgFESyEgTBKYhkJQiCUxDJShAEpyCSVQVk6QrWP//8Mw0bNsTd3Z1mzZqxefPmcorUuVjyfV2yZAmPPvooAQEBBAQE0LVr14duJXFrE8mqgrF0Bev9+/czcOBARo4cyYkTJ+jTpw99+vTh7Nmz5Ry5Y7P0+7pr1y4GDhzIzp07OXDgAGFhYTz11FPcunWrnCOvQCShQomIiJBGjx5t/Fyr1UqhoaFSZGRkofv369dP6tGjh8m29u3bS2+88YZN43Q2ln5fH6TRaCQfHx9pxYoVtgqxwhMjqwpEpVJx7Ngxunbtatwml8vp2rUrBw4cKPSYAwcOmOwP0L179yL3fxiV5vv6oKysLNRqNZUqVbJVmBWeSFYVSHErWMfHxxd6THx8vEX7P4xK83190IQJEwgNDS3wi0EwX4XvuiAI9jZ79mzWrl3Lrl27cHd3t3c4TkskqwqkNCtYBwcHW7T/w6g031eDOXPmMHv2bLZv307z5s1tGWaFJy4DK5DSrGDdoUMHk/0Btm3bJla8zqe0K4N//vnnzJgxg61bt9K2bdvyCLVis/cMv2Bda9euldzc3KTly5dL58+fl15//XXJ399fio+PlyRJkoYMGSJNnDjRuP++ffskFxcXac6cOdKFCxekqVOnSq6urtKZM2fs9SU4JEu/r7Nnz5aUSqW0fv16KS4uzviRnp5ury/B6YlkVQHNnz9fCg8Pl5RKpRQRESEdPHjQ+Nrjjz8uDR061GT/devWSfXr15eUSqXUpEkT6c8//yzniJ2DJd/XGjVqSECBj6lTp5Z/4BWEaBEjCIJTEHNWgiA4BZGsBEFwCiJZCYLgFESyEgTBKYhkJQiCUxDJShAEpyCSlSAITkEkK0EQnIJIVoJdDBs2DJlMVuDj6aeftndogoMSXRcEu3n66af5/vvvTbYVtaKxWq3G1dXVZJtKpUKpVFp83tIeJ9iXGFkJduPm5kZwcLDJR0BAAAAymYyFCxfSq1cvvLy8+PTTT/nkk09o2bIlS5cupVatWsbeUDdu3KB37954e3vj6+tLv379TNq5FHWc4FxEshIc1ieffELfvn05c+YMI0aMAODKlSts2LCBjRs3cvLkSXQ6Hb179yY5OZndu3ezbds2rl69Sv/+/U3e68HjBOcjLgMFu/njjz/w9vY22fbRRx/x0UcfATBo0CCGDx9u8rpKpWLlypUEBgYC+t5bZ86cISYmhrCwMABWrlxJkyZNOHLkCO3atSv0OMH5iGQl2E3nzp1ZuHChybb8CyoU1rCuRo0aJgnnwoULhIWFGRMVQOPGjfH39+fChQvGZPXgcYLzEclKsBsvLy/q1q1b7OvmbDP3XIJzE3NWglNr1KgRsbGxxMbGGredP3+elJQUGjdubMfIBGsTIyvBbnJzcwssZeXi4kKVKlXMfo+uXbvSrFkzBg8ezLx589BoNIwaNYrHH39c9D2vYMTISrCbrVu3EhISYvLxyCOPWPQeMpmM3377jYCAAB577DG6du1K7dq1+emnn2wUtWAvoq2xIAhOQYysBEFwCiJZCYLgFESyEgTBKYhkJQiCUxDJShAEpyCSlSAITkEkK0EQnIJIVoIgOAWRrARBcAoiWQmC4BREshIEwSmIZCUIglP4f0b9ay/1cofbAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019956, + "end_time": "2024-03-22T13:44:41.441616", + "exception": false, + "start_time": "2024-03-22T13:44:41.421660", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4138.383388, + "end_time": "2024-03-22T13:44:44.184516", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/realtabformer/42/mlu-eval.ipynb", + "output_path": "eval/insurance/realtabformer/42/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/realtabformer/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "realtabformer" + }, + "start_time": "2024-03-22T12:35:45.801128", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/realtabformer/model.pt b/insurance/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a6981d90420b11eb54de3bc7ce924778bec719b --- /dev/null +++ b/insurance/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29e1d4cdba4270251db8b7f22a9c1763e7f4475a12e215667377096e4e7e0f64 +size 43505805 diff --git a/insurance/realtabformer/params.json b/insurance/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..e8462b6d0b9ff9f92df45835aa50977cc57fcbb4 --- /dev/null +++ b/insurance/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "NONE", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "fixed_role_model": "realtabformer", "mse_mag": false, "mse_mag_target": 0.1, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/tab_ddpm_concat/eval.csv b/insurance/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..78ce478a4c8eee54dd404a0f3bb518c221e1d8a8 --- /dev/null +++ b/insurance/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,0.010057446495336643,,0.018736911170362008,6.4715576171875,0.08312908560037613,0.9944403171539307,0.13684093952178955,9.326592589786742e-06,2.3459200859069824,0.08326641470193863,3.0670642852783203,0.13688284158706665,0.04763114079833031,0.8643301725387573,8.817477703094482 diff --git a/insurance/tab_ddpm_concat/history.csv b/insurance/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..3064daf6aa990bdd3fe0a05a11283d3701fcc360 --- /dev/null +++ b/insurance/tab_ddpm_concat/history.csv @@ -0,0 +1,23 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.016726151166690722,6.681427996787429,0.0007995499490898637,0.0,0.0,0.0,0.0,0.0,0.016726151166690722,900,113,113.29283952713013,1.0025915002400896,0.12588093280792237,0.04266167516960243,0.013463321674304704,5.035531684387279,0.0003260195470640307,0.0,0.0,0.0,0.0,0.0,0.013463321674304704,450,57,46.18605709075928,0.810281703346654,0.10263568242390951,0.02743101706564949 +1,0.013343319840108355,7.374786443262831,0.00018221230446668457,0.0,0.0,0.0,0.0,0.0,0.013343319840108355,900,113,113.2004017829895,1.0017734671061018,0.12577822420332166,0.033394961601403435,0.013095296140398002,4.660537766719707,0.0003757634961128373,0.0,0.0,0.0,0.0,0.0,0.013095296140398002,450,57,46.44777750968933,0.8148732896436724,0.10321728335486519,0.0280298860416862 +2,0.013104651555832889,5.731545712810247,0.00016133929766812432,0.0,0.0,0.0,0.0,0.0,0.013104651555832889,900,113,113.58225321769714,1.0051526833424527,0.12620250357521906,0.03886226541568748,0.01287688842560682,3.2905811405824417,0.0003377204293245642,0.0,0.0,0.0,0.0,0.0,0.01287688842560682,450,57,47.43314838409424,0.8321604979665655,0.10540699640909831,0.03367136853436629 +3,0.013487551987895535,6.5134642504676314,0.0002293419825039046,0.0,0.0,0.0,0.0,0.0,0.013487551987895535,900,113,112.98276948928833,0.9998475176043216,0.1255364105436537,0.037405478042772916,0.013358833422470424,6.6747851123451865,0.000459600389062934,0.0,0.0,0.0,0.0,0.0,0.013358833422470424,450,57,46.22313857078552,0.8109322556278162,0.10271808571285672,0.020856579136626238 +4,0.013331625766845214,6.895425937681058,0.00015765312878396825,0.0,0.0,0.0,0.0,0.0,0.013331625766845214,900,113,112.3973479270935,0.9946667958149867,0.12488594214121501,0.03705255519104215,0.015463804398766823,5.035077505065904,0.0007592894264012608,0.0,0.0,0.0,0.0,0.0,0.015463804398766823,450,57,46.07612347602844,0.8083530434390955,0.10239138550228542,0.03493314252741504 +5,0.013444565045129921,6.88672233373744,0.0002211289928355282,0.0,0.0,0.0,0.0,0.0,0.013444565045129921,900,113,108.1660463809967,0.9572216493893514,0.12018449597888523,0.03778205175710991,0.013861719023229347,5.471305163596984,0.0005421772549703974,0.0,0.0,0.0,0.0,0.0,0.013861719023229347,450,57,44.389360666275024,0.7787607134434215,0.09864302370283339,0.025694214632701978 +6,0.013114380333055224,5.976081524419583,0.00027622638839541855,0.0,0.0,0.0,0.0,0.0,0.013114380333055224,900,113,111.54308128356934,0.987106914013888,0.12393675698174371,0.038547951167663644,0.012811201657168567,3.3985922191304563,0.00027678651687506,0.0,0.0,0.0,0.0,0.0,0.012811201657168567,450,57,45.941070795059204,0.8059836981589334,0.1020912684334649,0.03597862523441252 +7,0.013140132142644789,6.532687967344641,0.00014520771743971967,0.0,0.0,0.0,0.0,0.0,0.013140132142644789,900,113,108.82160568237305,0.9630230591360447,0.12091289520263672,0.03722985821520596,0.012903139407539533,3.578145878625659,0.0003883562163952724,0.0,0.0,0.0,0.0,0.0,0.012903139407539533,450,57,44.52135992050171,0.7810764898333633,0.09893635537889268,0.03204377959564067 +8,0.012976793174942335,5.416468192214529,0.00026929119924884524,0.0,0.0,0.0,0.0,0.0,0.012976793174942335,900,113,110.61598777770996,0.9789025467053979,0.1229066530863444,0.04178034744134783,0.013890320318751037,3.7336453318763363,0.0005463759208261207,0.0,0.0,0.0,0.0,0.0,0.013890320318751037,450,57,45.11232590675354,0.7914443141535709,0.10024961312611898,0.037292727285571266 +9,0.013011944204982783,6.452372519567247,0.0001708998166064858,0.0,0.0,0.0,0.0,0.0,0.013011944204982783,900,113,108.52205491065979,0.9603721673509716,0.12058006101184421,0.036126165387047604,0.01294027495249692,4.294494187321249,0.0004066406136260209,0.0,0.0,0.0,0.0,0.0,0.01294027495249692,450,57,45.50484013557434,0.7983305286942867,0.10112186696794298,0.029192425953959555 +10,0.01281262576735268,5.773519790146028,0.00017255405067423624,0.0,0.0,0.0,0.0,0.0,0.01281262576735268,900,113,110.79968285560608,0.9805281668637706,0.1231107587284512,0.04245836566721575,0.012781418984652394,2.9514776281277206,0.00034504475288883897,0.0,0.0,0.0,0.0,0.0,0.012781418984652394,450,57,46.79314422607422,0.8209323548434073,0.10398476494683159,0.03811844466907675 +11,0.013128257239651348,5.248821901264445,0.00028772900066602206,0.0,0.0,0.0,0.0,0.0,0.013128257239651348,900,113,112.41662859916687,0.9948374212315653,0.12490736511018541,0.039066291603762494,0.012926328503009345,4.4593022893954055,0.00044354169371757735,0.0,0.0,0.0,0.0,0.0,0.012926328503009345,450,57,46.57635951042175,0.8171291142179254,0.10350302113427057,0.02754177166998648 +12,0.013119876470623746,5.730330355907093,0.00015690460682182876,0.0,0.0,0.0,0.0,0.0,0.013119876470623746,900,113,113.1574444770813,1.0013933139564717,0.12573049386342366,0.04159025917142893,0.013785471472785705,4.6235486922847935,0.0005177017974133631,0.0,0.0,0.0,0.0,0.0,0.013785471472785705,450,57,45.30210542678833,0.7947737794173392,0.10067134539286296,0.029443472623825073 +13,0.012679316052235663,5.353459168564728,0.00018473560166641948,0.0,0.0,0.0,0.0,0.0,0.012679316052235663,900,113,109.81215143203735,0.9717889507259942,0.12201350159115261,0.03975286655358772,0.01452305714185867,6.82052209327945,0.0006187541751139886,0.0,0.0,0.0,0.0,0.0,0.01452305714185867,450,57,45.597567081451416,0.7999573172184459,0.10132792684766981,0.028516328191025217 +14,0.012685477107038929,4.885799459394998,9.410771968618429e-05,0.0,0.0,0.0,0.0,0.0,0.012685477107038929,900,113,112.43323254585266,0.9949843588128554,0.12492581393983629,0.043478165464723,0.013303691690218531,4.612162083723674,0.0004478314968633236,0.0,0.0,0.0,0.0,0.0,0.013303691690218531,450,57,46.16491627693176,0.8099108118759958,0.10258870283762614,0.030379774735162134 +15,0.012951436785774099,5.6180261968808605,0.0001260765955767044,0.0,0.0,0.0,0.0,0.0,0.012951436785774099,900,113,112.08748507499695,0.9919246466813889,0.12454165008332994,0.0389383625469904,0.013147126083624446,5.849595955204062,0.00047660569695545343,0.0,0.0,0.0,0.0,0.0,0.013147126083624446,450,57,46.19347643852234,0.8104118673424971,0.10265216986338298,0.024712568949581237 +16,0.013076940375483698,6.462012599512772,0.0003414776301815831,0.0,0.0,0.0,0.0,0.0,0.013076940375483698,900,113,112.6052417755127,0.9965065643850681,0.12511693530612522,0.03943786635764141,0.013706634990457031,2.6824540562413044,0.00021449473134149837,0.0,0.0,0.0,0.0,0.0,0.013706634990457031,450,57,46.32444953918457,0.8127096410383258,0.10294322119818794,0.04417712587797851 +17,0.013131085654927625,7.07163172544333,0.00013269016459129516,0.0,0.0,0.0,0.0,0.0,0.013131085654927625,900,113,112.05166149139404,0.9916076238176464,0.12450184610154894,0.03639758312332947,0.013306772463385843,3.5713896107044203,0.0004134395122505591,0.0,0.0,0.0,0.0,0.0,0.013306772463385843,450,57,45.87373447418213,0.8048023591961777,0.10194163216484918,0.03344645112622203 +18,0.013018570302778648,5.669214043441515,0.00020193976867753112,0.0,0.0,0.0,0.0,0.0,0.013018570302778648,900,113,111.77040791511536,0.9891186541160651,0.12418934212790596,0.03950189917752173,0.013145547265497347,4.102805095807446,0.00038455364744524763,0.0,0.0,0.0,0.0,0.0,0.013145547265497347,450,57,45.8624849319458,0.8046049988060667,0.10191663318210178,0.034161844544047325 +19,0.012822051873275389,5.499414050482283,0.00015386838247107923,0.0,0.0,0.0,0.0,0.0,0.012822051873275389,900,113,111.74085307121277,0.9888571068248918,0.12415650341245864,0.03887049661767957,0.012638136894804322,4.0768485525334475,0.0003349378748481513,0.0,0.0,0.0,0.0,0.0,0.012638136894804322,450,57,46.025766372680664,0.8074695854856256,0.10227948082817925,0.031167056632081146 +20,0.012691254431588783,5.174604149312652,5.650333046863428e-05,0.0,0.0,0.0,0.0,0.0,0.012691254431588783,900,113,111.79648971557617,0.9893494665095236,0.12421832190619575,0.041397298776866064,0.012594756139959726,3.085094394737736,0.0002528211896517658,0.0,0.0,0.0,0.0,0.0,0.012594756139959726,450,57,46.05099582672119,0.8079122074863367,0.10233554628160264,0.03782708470693283 +21,0.012717143062295185,5.476102162761738,0.00015562177650104382,0.0,0.0,0.0,0.0,0.0,0.012717143062295185,900,113,111.75997877120972,0.9890263608071657,0.12417775419023303,0.04065807265089413,0.012463197727791137,3.047649189417655,0.0003119605865624761,0.0,0.0,0.0,0.0,0.0,0.012463197727791137,450,57,45.863757610321045,0.8046273264968604,0.10191946135626899,0.03551346450848015 diff --git a/insurance/tab_ddpm_concat/mlu-eval.ipynb b/insurance/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..d47e3e018a5c42da87bbcce9938c881b00ad34d9 --- /dev/null +++ b/insurance/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2552 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:22.712382Z", + "iopub.status.busy": "2024-03-22T14:28:22.711885Z", + "iopub.status.idle": "2024-03-22T14:28:22.751017Z", + "shell.execute_reply": "2024-03-22T14:28:22.750283Z" + }, + "papermill": { + "duration": 0.059065, + "end_time": "2024-03-22T14:28:22.753226", + "exception": false, + "start_time": "2024-03-22T14:28:22.694161", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:22.778729Z", + "iopub.status.busy": "2024-03-22T14:28:22.778350Z", + "iopub.status.idle": "2024-03-22T14:28:22.785122Z", + "shell.execute_reply": "2024-03-22T14:28:22.784296Z" + }, + "papermill": { + "duration": 0.021549, + "end_time": "2024-03-22T14:28:22.787100", + "exception": false, + "start_time": "2024-03-22T14:28:22.765551", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:22.810425Z", + "iopub.status.busy": "2024-03-22T14:28:22.810153Z", + "iopub.status.idle": "2024-03-22T14:28:22.814409Z", + "shell.execute_reply": "2024-03-22T14:28:22.813540Z" + }, + "papermill": { + "duration": 0.018306, + "end_time": "2024-03-22T14:28:22.816449", + "exception": false, + "start_time": "2024-03-22T14:28:22.798143", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:22.840425Z", + "iopub.status.busy": "2024-03-22T14:28:22.840157Z", + "iopub.status.idle": "2024-03-22T14:28:22.844167Z", + "shell.execute_reply": "2024-03-22T14:28:22.843327Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018567, + "end_time": "2024-03-22T14:28:22.846201", + "exception": false, + "start_time": "2024-03-22T14:28:22.827634", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:22.869826Z", + "iopub.status.busy": "2024-03-22T14:28:22.869335Z", + "iopub.status.idle": "2024-03-22T14:28:22.874948Z", + "shell.execute_reply": "2024-03-22T14:28:22.874114Z" + }, + "papermill": { + "duration": 0.019401, + "end_time": "2024-03-22T14:28:22.876772", + "exception": false, + "start_time": "2024-03-22T14:28:22.857371", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "181cc350", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:22.902286Z", + "iopub.status.busy": "2024-03-22T14:28:22.902021Z", + "iopub.status.idle": "2024-03-22T14:28:22.906860Z", + "shell.execute_reply": "2024-03-22T14:28:22.906112Z" + }, + "papermill": { + "duration": 0.019919, + "end_time": "2024-03-22T14:28:22.908779", + "exception": false, + "start_time": "2024-03-22T14:28:22.888860", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 0\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/tab_ddpm_concat/0\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011075, + "end_time": "2024-03-22T14:28:22.930960", + "exception": false, + "start_time": "2024-03-22T14:28:22.919885", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:22.954386Z", + "iopub.status.busy": "2024-03-22T14:28:22.954120Z", + "iopub.status.idle": "2024-03-22T14:28:22.962998Z", + "shell.execute_reply": "2024-03-22T14:28:22.962178Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022966, + "end_time": "2024-03-22T14:28:22.964970", + "exception": false, + "start_time": "2024-03-22T14:28:22.942004", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/tab_ddpm_concat/0\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:22.988510Z", + "iopub.status.busy": "2024-03-22T14:28:22.988249Z", + "iopub.status.idle": "2024-03-22T14:28:24.972444Z", + "shell.execute_reply": "2024-03-22T14:28:24.971345Z" + }, + "papermill": { + "duration": 1.99836, + "end_time": "2024-03-22T14:28:24.974620", + "exception": false, + "start_time": "2024-03-22T14:28:22.976260", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:25.002365Z", + "iopub.status.busy": "2024-03-22T14:28:25.001964Z", + "iopub.status.idle": "2024-03-22T14:28:25.014498Z", + "shell.execute_reply": "2024-03-22T14:28:25.013592Z" + }, + "papermill": { + "duration": 0.02854, + "end_time": "2024-03-22T14:28:25.016521", + "exception": false, + "start_time": "2024-03-22T14:28:24.987981", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:25.040635Z", + "iopub.status.busy": "2024-03-22T14:28:25.040392Z", + "iopub.status.idle": "2024-03-22T14:28:25.047363Z", + "shell.execute_reply": "2024-03-22T14:28:25.046631Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021359, + "end_time": "2024-03-22T14:28:25.049522", + "exception": false, + "start_time": "2024-03-22T14:28:25.028163", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:25.073608Z", + "iopub.status.busy": "2024-03-22T14:28:25.073369Z", + "iopub.status.idle": "2024-03-22T14:28:25.166665Z", + "shell.execute_reply": "2024-03-22T14:28:25.165914Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.107895, + "end_time": "2024-03-22T14:28:25.168887", + "exception": false, + "start_time": "2024-03-22T14:28:25.060992", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:25.195471Z", + "iopub.status.busy": "2024-03-22T14:28:25.195191Z", + "iopub.status.idle": "2024-03-22T14:28:29.788144Z", + "shell.execute_reply": "2024-03-22T14:28:29.787359Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.609258, + "end_time": "2024-03-22T14:28:29.790488", + "exception": false, + "start_time": "2024-03-22T14:28:25.181230", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 14:28:27.399095: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 14:28:27.399150: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 14:28:27.400845: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:29.816220Z", + "iopub.status.busy": "2024-03-22T14:28:29.815644Z", + "iopub.status.idle": "2024-03-22T14:28:29.822362Z", + "shell.execute_reply": "2024-03-22T14:28:29.821638Z" + }, + "papermill": { + "duration": 0.021526, + "end_time": "2024-03-22T14:28:29.824288", + "exception": false, + "start_time": "2024-03-22T14:28:29.802762", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:29.849336Z", + "iopub.status.busy": "2024-03-22T14:28:29.849070Z", + "iopub.status.idle": "2024-03-22T14:28:38.728835Z", + "shell.execute_reply": "2024-03-22T14:28:38.727751Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.895421, + "end_time": "2024-03-22T14:28:38.731645", + "exception": false, + "start_time": "2024-03-22T14:28:29.836224", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T14:28:38.759079Z", + "iopub.status.busy": "2024-03-22T14:28:38.758762Z", + "iopub.status.idle": "2024-03-22T14:28:38.765346Z", + "shell.execute_reply": "2024-03-22T14:28:38.764555Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022044, + "end_time": "2024-03-22T14:28:38.767245", + "exception": false, + "start_time": "2024-03-22T14:28:38.745201", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:38.792400Z", + "iopub.status.busy": "2024-03-22T14:28:38.791781Z", + "iopub.status.idle": "2024-03-22T14:28:38.796660Z", + "shell.execute_reply": "2024-03-22T14:28:38.795821Z" + }, + "papermill": { + "duration": 0.019562, + "end_time": "2024-03-22T14:28:38.798603", + "exception": false, + "start_time": "2024-03-22T14:28:38.779041", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:28:38.823546Z", + "iopub.status.busy": "2024-03-22T14:28:38.823305Z", + "iopub.status.idle": "2024-03-22T14:29:19.453844Z", + "shell.execute_reply": "2024-03-22T14:29:19.452858Z" + }, + "papermill": { + "duration": 40.657114, + "end_time": "2024-03-22T14:29:19.467500", + "exception": false, + "start_time": "2024-03-22T14:28:38.810386", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_bs_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/bs_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:29:19.495009Z", + "iopub.status.busy": "2024-03-22T14:29:19.494671Z", + "iopub.status.idle": "2024-03-22T14:29:19.814690Z", + "shell.execute_reply": "2024-03-22T14:29:19.813766Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.336357, + "end_time": "2024-03-22T14:29:19.816750", + "exception": false, + "start_time": "2024-03-22T14:29:19.480393", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:29:19.845842Z", + "iopub.status.busy": "2024-03-22T14:29:19.845524Z", + "iopub.status.idle": "2024-03-22T14:30:11.208799Z", + "shell.execute_reply": "2024-03-22T14:30:11.207769Z" + }, + "papermill": { + "duration": 51.393053, + "end_time": "2024-03-22T14:30:11.223271", + "exception": false, + "start_time": "2024-03-22T14:29:19.830218", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T14:30:11.252134Z", + "iopub.status.busy": "2024-03-22T14:30:11.251841Z", + "iopub.status.idle": "2024-03-22T14:30:11.667944Z", + "shell.execute_reply": "2024-03-22T14:30:11.666922Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.433012, + "end_time": "2024-03-22T14:30:11.670133", + "exception": false, + "start_time": "2024-03-22T14:30:11.237121", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:30:11.700280Z", + "iopub.status.busy": "2024-03-22T14:30:11.699872Z", + "iopub.status.idle": "2024-03-22T14:30:11.704571Z", + "shell.execute_reply": "2024-03-22T14:30:11.703669Z" + }, + "papermill": { + "duration": 0.02186, + "end_time": "2024-03-22T14:30:11.706511", + "exception": false, + "start_time": "2024-03-22T14:30:11.684651", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:30:11.735591Z", + "iopub.status.busy": "2024-03-22T14:30:11.735315Z", + "iopub.status.idle": "2024-03-22T14:30:11.742687Z", + "shell.execute_reply": "2024-03-22T14:30:11.741775Z" + }, + "papermill": { + "duration": 0.024819, + "end_time": "2024-03-22T14:30:11.745149", + "exception": false, + "start_time": "2024-03-22T14:30:11.720330", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9613961" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:30:11.773580Z", + "iopub.status.busy": "2024-03-22T14:30:11.773302Z", + "iopub.status.idle": "2024-03-22T14:30:11.870431Z", + "shell.execute_reply": "2024-03-22T14:30:11.869372Z" + }, + "papermill": { + "duration": 0.114296, + "end_time": "2024-03-22T14:30:11.872927", + "exception": false, + "start_time": "2024-03-22T14:30:11.758631", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 12] --\n", + "├─Adapter: 1-1 [2, 1071, 12] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 13,312\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 12] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,613,961\n", + "Trainable params: 9,613,961\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.08\n", + "========================================================================================================================\n", + "Input size (MB): 0.13\n", + "Forward/backward pass size (MB): 307.49\n", + "Params size (MB): 38.46\n", + "Estimated Total Size (MB): 346.07\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T14:30:11.909262Z", + "iopub.status.busy": "2024-03-22T14:30:11.908515Z", + "iopub.status.idle": "2024-03-22T15:32:14.293056Z", + "shell.execute_reply": "2024-03-22T15:32:14.291944Z" + }, + "papermill": { + "duration": 3722.423919, + "end_time": "2024-03-22T15:32:14.314157", + "exception": false, + "start_time": "2024-03-22T14:30:11.890238", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.016726151166690722, 'avg_role_model_std_loss': 6.681427996787429, 'avg_role_model_mean_pred_loss': 0.0007995499490898637, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.016726151166690722, 'n_size': 900, 'n_batch': 113, 'duration': 113.29283952713013, 'duration_batch': 1.0025915002400896, 'duration_size': 0.12588093280792237, 'avg_pred_std': 0.04266167516960243}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013463321674304704, 'avg_role_model_std_loss': 5.035531684387279, 'avg_role_model_mean_pred_loss': 0.0003260195470640307, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013463321674304704, 'n_size': 450, 'n_batch': 57, 'duration': 46.18605709075928, 'duration_batch': 0.810281703346654, 'duration_size': 0.10263568242390951, 'avg_pred_std': 0.02743101706564949}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013343319840108355, 'avg_role_model_std_loss': 7.374786443262831, 'avg_role_model_mean_pred_loss': 0.00018221230446668457, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013343319840108355, 'n_size': 900, 'n_batch': 113, 'duration': 113.2004017829895, 'duration_batch': 1.0017734671061018, 'duration_size': 0.12577822420332166, 'avg_pred_std': 0.033394961601403435}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013095296140398002, 'avg_role_model_std_loss': 4.660537766719707, 'avg_role_model_mean_pred_loss': 0.0003757634961128373, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013095296140398002, 'n_size': 450, 'n_batch': 57, 'duration': 46.44777750968933, 'duration_batch': 0.8148732896436724, 'duration_size': 0.10321728335486519, 'avg_pred_std': 0.0280298860416862}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013104651555832889, 'avg_role_model_std_loss': 5.731545712810247, 'avg_role_model_mean_pred_loss': 0.00016133929766812432, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013104651555832889, 'n_size': 900, 'n_batch': 113, 'duration': 113.58225321769714, 'duration_batch': 1.0051526833424527, 'duration_size': 0.12620250357521906, 'avg_pred_std': 0.03886226541568748}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01287688842560682, 'avg_role_model_std_loss': 3.2905811405824417, 'avg_role_model_mean_pred_loss': 0.0003377204293245642, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01287688842560682, 'n_size': 450, 'n_batch': 57, 'duration': 47.43314838409424, 'duration_batch': 0.8321604979665655, 'duration_size': 0.10540699640909831, 'avg_pred_std': 0.03367136853436629}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013487551987895535, 'avg_role_model_std_loss': 6.5134642504676314, 'avg_role_model_mean_pred_loss': 0.0002293419825039046, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013487551987895535, 'n_size': 900, 'n_batch': 113, 'duration': 112.98276948928833, 'duration_batch': 0.9998475176043216, 'duration_size': 0.1255364105436537, 'avg_pred_std': 0.037405478042772916}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013358833422470424, 'avg_role_model_std_loss': 6.6747851123451865, 'avg_role_model_mean_pred_loss': 0.000459600389062934, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013358833422470424, 'n_size': 450, 'n_batch': 57, 'duration': 46.22313857078552, 'duration_batch': 0.8109322556278162, 'duration_size': 0.10271808571285672, 'avg_pred_std': 0.020856579136626238}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013331625766845214, 'avg_role_model_std_loss': 6.895425937681058, 'avg_role_model_mean_pred_loss': 0.00015765312878396825, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013331625766845214, 'n_size': 900, 'n_batch': 113, 'duration': 112.3973479270935, 'duration_batch': 0.9946667958149867, 'duration_size': 0.12488594214121501, 'avg_pred_std': 0.03705255519104215}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.015463804398766823, 'avg_role_model_std_loss': 5.035077505065904, 'avg_role_model_mean_pred_loss': 0.0007592894264012608, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.015463804398766823, 'n_size': 450, 'n_batch': 57, 'duration': 46.07612347602844, 'duration_batch': 0.8083530434390955, 'duration_size': 0.10239138550228542, 'avg_pred_std': 0.03493314252741504}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013444565045129921, 'avg_role_model_std_loss': 6.88672233373744, 'avg_role_model_mean_pred_loss': 0.0002211289928355282, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013444565045129921, 'n_size': 900, 'n_batch': 113, 'duration': 108.1660463809967, 'duration_batch': 0.9572216493893514, 'duration_size': 0.12018449597888523, 'avg_pred_std': 0.03778205175710991}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013861719023229347, 'avg_role_model_std_loss': 5.471305163596984, 'avg_role_model_mean_pred_loss': 0.0005421772549703974, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013861719023229347, 'n_size': 450, 'n_batch': 57, 'duration': 44.389360666275024, 'duration_batch': 0.7787607134434215, 'duration_size': 0.09864302370283339, 'avg_pred_std': 0.025694214632701978}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013114380333055224, 'avg_role_model_std_loss': 5.976081524419583, 'avg_role_model_mean_pred_loss': 0.00027622638839541855, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013114380333055224, 'n_size': 900, 'n_batch': 113, 'duration': 111.54308128356934, 'duration_batch': 0.987106914013888, 'duration_size': 0.12393675698174371, 'avg_pred_std': 0.038547951167663644}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012811201657168567, 'avg_role_model_std_loss': 3.3985922191304563, 'avg_role_model_mean_pred_loss': 0.00027678651687506, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012811201657168567, 'n_size': 450, 'n_batch': 57, 'duration': 45.941070795059204, 'duration_batch': 0.8059836981589334, 'duration_size': 0.1020912684334649, 'avg_pred_std': 0.03597862523441252}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013140132142644789, 'avg_role_model_std_loss': 6.532687967344641, 'avg_role_model_mean_pred_loss': 0.00014520771743971967, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013140132142644789, 'n_size': 900, 'n_batch': 113, 'duration': 108.82160568237305, 'duration_batch': 0.9630230591360447, 'duration_size': 0.12091289520263672, 'avg_pred_std': 0.03722985821520596}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012903139407539533, 'avg_role_model_std_loss': 3.578145878625659, 'avg_role_model_mean_pred_loss': 0.0003883562163952724, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012903139407539533, 'n_size': 450, 'n_batch': 57, 'duration': 44.52135992050171, 'duration_batch': 0.7810764898333633, 'duration_size': 0.09893635537889268, 'avg_pred_std': 0.03204377959564067}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012976793174942335, 'avg_role_model_std_loss': 5.416468192214529, 'avg_role_model_mean_pred_loss': 0.00026929119924884524, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012976793174942335, 'n_size': 900, 'n_batch': 113, 'duration': 110.61598777770996, 'duration_batch': 0.9789025467053979, 'duration_size': 0.1229066530863444, 'avg_pred_std': 0.04178034744134783}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013890320318751037, 'avg_role_model_std_loss': 3.7336453318763363, 'avg_role_model_mean_pred_loss': 0.0005463759208261207, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013890320318751037, 'n_size': 450, 'n_batch': 57, 'duration': 45.11232590675354, 'duration_batch': 0.7914443141535709, 'duration_size': 0.10024961312611898, 'avg_pred_std': 0.037292727285571266}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013011944204982783, 'avg_role_model_std_loss': 6.452372519567247, 'avg_role_model_mean_pred_loss': 0.0001708998166064858, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013011944204982783, 'n_size': 900, 'n_batch': 113, 'duration': 108.52205491065979, 'duration_batch': 0.9603721673509716, 'duration_size': 0.12058006101184421, 'avg_pred_std': 0.036126165387047604}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01294027495249692, 'avg_role_model_std_loss': 4.294494187321249, 'avg_role_model_mean_pred_loss': 0.0004066406136260209, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01294027495249692, 'n_size': 450, 'n_batch': 57, 'duration': 45.50484013557434, 'duration_batch': 0.7983305286942867, 'duration_size': 0.10112186696794298, 'avg_pred_std': 0.029192425953959555}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.01281262576735268, 'avg_role_model_std_loss': 5.773519790146028, 'avg_role_model_mean_pred_loss': 0.00017255405067423624, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01281262576735268, 'n_size': 900, 'n_batch': 113, 'duration': 110.79968285560608, 'duration_batch': 0.9805281668637706, 'duration_size': 0.1231107587284512, 'avg_pred_std': 0.04245836566721575}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012781418984652394, 'avg_role_model_std_loss': 2.9514776281277206, 'avg_role_model_mean_pred_loss': 0.00034504475288883897, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012781418984652394, 'n_size': 450, 'n_batch': 57, 'duration': 46.79314422607422, 'duration_batch': 0.8209323548434073, 'duration_size': 0.10398476494683159, 'avg_pred_std': 0.03811844466907675}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013128257239651348, 'avg_role_model_std_loss': 5.248821901264445, 'avg_role_model_mean_pred_loss': 0.00028772900066602206, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013128257239651348, 'n_size': 900, 'n_batch': 113, 'duration': 112.41662859916687, 'duration_batch': 0.9948374212315653, 'duration_size': 0.12490736511018541, 'avg_pred_std': 0.039066291603762494}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012926328503009345, 'avg_role_model_std_loss': 4.4593022893954055, 'avg_role_model_mean_pred_loss': 0.00044354169371757735, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012926328503009345, 'n_size': 450, 'n_batch': 57, 'duration': 46.57635951042175, 'duration_batch': 0.8171291142179254, 'duration_size': 0.10350302113427057, 'avg_pred_std': 0.02754177166998648}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013119876470623746, 'avg_role_model_std_loss': 5.730330355907093, 'avg_role_model_mean_pred_loss': 0.00015690460682182876, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013119876470623746, 'n_size': 900, 'n_batch': 113, 'duration': 113.1574444770813, 'duration_batch': 1.0013933139564717, 'duration_size': 0.12573049386342366, 'avg_pred_std': 0.04159025917142893}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013785471472785705, 'avg_role_model_std_loss': 4.6235486922847935, 'avg_role_model_mean_pred_loss': 0.0005177017974133631, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013785471472785705, 'n_size': 450, 'n_batch': 57, 'duration': 45.30210542678833, 'duration_batch': 0.7947737794173392, 'duration_size': 0.10067134539286296, 'avg_pred_std': 0.029443472623825073}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012679316052235663, 'avg_role_model_std_loss': 5.353459168564728, 'avg_role_model_mean_pred_loss': 0.00018473560166641948, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012679316052235663, 'n_size': 900, 'n_batch': 113, 'duration': 109.81215143203735, 'duration_batch': 0.9717889507259942, 'duration_size': 0.12201350159115261, 'avg_pred_std': 0.03975286655358772}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01452305714185867, 'avg_role_model_std_loss': 6.82052209327945, 'avg_role_model_mean_pred_loss': 0.0006187541751139886, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01452305714185867, 'n_size': 450, 'n_batch': 57, 'duration': 45.597567081451416, 'duration_batch': 0.7999573172184459, 'duration_size': 0.10132792684766981, 'avg_pred_std': 0.028516328191025217}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012685477107038929, 'avg_role_model_std_loss': 4.885799459394998, 'avg_role_model_mean_pred_loss': 9.410771968618429e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012685477107038929, 'n_size': 900, 'n_batch': 113, 'duration': 112.43323254585266, 'duration_batch': 0.9949843588128554, 'duration_size': 0.12492581393983629, 'avg_pred_std': 0.043478165464723}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013303691690218531, 'avg_role_model_std_loss': 4.612162083723674, 'avg_role_model_mean_pred_loss': 0.0004478314968633236, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013303691690218531, 'n_size': 450, 'n_batch': 57, 'duration': 46.16491627693176, 'duration_batch': 0.8099108118759958, 'duration_size': 0.10258870283762614, 'avg_pred_std': 0.030379774735162134}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012951436785774099, 'avg_role_model_std_loss': 5.6180261968808605, 'avg_role_model_mean_pred_loss': 0.0001260765955767044, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012951436785774099, 'n_size': 900, 'n_batch': 113, 'duration': 112.08748507499695, 'duration_batch': 0.9919246466813889, 'duration_size': 0.12454165008332994, 'avg_pred_std': 0.0389383625469904}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013147126083624446, 'avg_role_model_std_loss': 5.849595955204062, 'avg_role_model_mean_pred_loss': 0.00047660569695545343, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013147126083624446, 'n_size': 450, 'n_batch': 57, 'duration': 46.19347643852234, 'duration_batch': 0.8104118673424971, 'duration_size': 0.10265216986338298, 'avg_pred_std': 0.024712568949581237}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013076940375483698, 'avg_role_model_std_loss': 6.462012599512772, 'avg_role_model_mean_pred_loss': 0.0003414776301815831, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013076940375483698, 'n_size': 900, 'n_batch': 113, 'duration': 112.6052417755127, 'duration_batch': 0.9965065643850681, 'duration_size': 0.12511693530612522, 'avg_pred_std': 0.03943786635764141}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013706634990457031, 'avg_role_model_std_loss': 2.6824540562413044, 'avg_role_model_mean_pred_loss': 0.00021449473134149837, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013706634990457031, 'n_size': 450, 'n_batch': 57, 'duration': 46.32444953918457, 'duration_batch': 0.8127096410383258, 'duration_size': 0.10294322119818794, 'avg_pred_std': 0.04417712587797851}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013131085654927625, 'avg_role_model_std_loss': 7.07163172544333, 'avg_role_model_mean_pred_loss': 0.00013269016459129516, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013131085654927625, 'n_size': 900, 'n_batch': 113, 'duration': 112.05166149139404, 'duration_batch': 0.9916076238176464, 'duration_size': 0.12450184610154894, 'avg_pred_std': 0.03639758312332947}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013306772463385843, 'avg_role_model_std_loss': 3.5713896107044203, 'avg_role_model_mean_pred_loss': 0.0004134395122505591, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013306772463385843, 'n_size': 450, 'n_batch': 57, 'duration': 45.87373447418213, 'duration_batch': 0.8048023591961777, 'duration_size': 0.10194163216484918, 'avg_pred_std': 0.03344645112622203}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013018570302778648, 'avg_role_model_std_loss': 5.669214043441515, 'avg_role_model_mean_pred_loss': 0.00020193976867753112, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013018570302778648, 'n_size': 900, 'n_batch': 113, 'duration': 111.77040791511536, 'duration_batch': 0.9891186541160651, 'duration_size': 0.12418934212790596, 'avg_pred_std': 0.03950189917752173}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013145547265497347, 'avg_role_model_std_loss': 4.102805095807446, 'avg_role_model_mean_pred_loss': 0.00038455364744524763, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013145547265497347, 'n_size': 450, 'n_batch': 57, 'duration': 45.8624849319458, 'duration_batch': 0.8046049988060667, 'duration_size': 0.10191663318210178, 'avg_pred_std': 0.034161844544047325}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012822051873275389, 'avg_role_model_std_loss': 5.499414050482283, 'avg_role_model_mean_pred_loss': 0.00015386838247107923, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012822051873275389, 'n_size': 900, 'n_batch': 113, 'duration': 111.74085307121277, 'duration_batch': 0.9888571068248918, 'duration_size': 0.12415650341245864, 'avg_pred_std': 0.03887049661767957}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012638136894804322, 'avg_role_model_std_loss': 4.0768485525334475, 'avg_role_model_mean_pred_loss': 0.0003349378748481513, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012638136894804322, 'n_size': 450, 'n_batch': 57, 'duration': 46.025766372680664, 'duration_batch': 0.8074695854856256, 'duration_size': 0.10227948082817925, 'avg_pred_std': 0.031167056632081146}\n", + "Epoch 20\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012691254431588783, 'avg_role_model_std_loss': 5.174604149312652, 'avg_role_model_mean_pred_loss': 5.650333046863428e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012691254431588783, 'n_size': 900, 'n_batch': 113, 'duration': 111.79648971557617, 'duration_batch': 0.9893494665095236, 'duration_size': 0.12421832190619575, 'avg_pred_std': 0.041397298776866064}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012594756139959726, 'avg_role_model_std_loss': 3.085094394737736, 'avg_role_model_mean_pred_loss': 0.0002528211896517658, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012594756139959726, 'n_size': 450, 'n_batch': 57, 'duration': 46.05099582672119, 'duration_batch': 0.8079122074863367, 'duration_size': 0.10233554628160264, 'avg_pred_std': 0.03782708470693283}\n", + "Epoch 21\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012717143062295185, 'avg_role_model_std_loss': 5.476102162761738, 'avg_role_model_mean_pred_loss': 0.00015562177650104382, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012717143062295185, 'n_size': 900, 'n_batch': 113, 'duration': 111.75997877120972, 'duration_batch': 0.9890263608071657, 'duration_size': 0.12417775419023303, 'avg_pred_std': 0.04065807265089413}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012463197727791137, 'avg_role_model_std_loss': 3.047649189417655, 'avg_role_model_mean_pred_loss': 0.0003119605865624761, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012463197727791137, 'n_size': 450, 'n_batch': 57, 'duration': 45.863757610321045, 'duration_batch': 0.8046273264968604, 'duration_size': 0.10191946135626899, 'avg_pred_std': 0.03551346450848015}\n", + "Epoch 22\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012550297869214168, 'avg_role_model_std_loss': 5.768059328748397, 'avg_role_model_mean_pred_loss': 0.0001121935050624429, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012550297869214168, 'n_size': 900, 'n_batch': 113, 'duration': 111.46515083312988, 'duration_batch': 0.986417264009999, 'duration_size': 0.12385016759236654, 'avg_pred_std': 0.04134711747230813}\n", + "Time out: 3600.0357847213745/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 1050, 'n_batch': 132, 'role_model_metrics': {'avg_loss': 0.018736953249844236, 'avg_g_mag_loss': 0.6755777371781213, 'avg_g_cos_loss': 0.01375962085758006, 'pred_duration': 2.3168554306030273, 'grad_duration': 6.480625629425049, 'total_duration': 8.797481060028076, 'pred_std': 0.04763111099600792, 'std_loss': 0.864331066608429, 'mean_pred_loss': 9.326593499281444e-06, 'pred_rmse': 0.1368829905986786, 'pred_mae': 0.08326660096645355, 'pred_mape': 3.067068099975586, 'grad_rmse': 0.13684110343456268, 'grad_mae': 0.08312927931547165, 'grad_mape': 0.9944403171539307}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.018736953249844236, 'avg_g_mag_loss': 0.6755777371781213, 'avg_g_cos_loss': 0.01375962085758006, 'avg_pred_duration': 2.3168554306030273, 'avg_grad_duration': 6.480625629425049, 'avg_total_duration': 8.797481060028076, 'avg_pred_std': 0.04763111099600792, 'avg_std_loss': 0.864331066608429, 'avg_mean_pred_loss': 9.326593499281444e-06}, 'min_metrics': {'avg_loss': 0.018736953249844236, 'avg_g_mag_loss': 0.6755777371781213, 'avg_g_cos_loss': 0.01375962085758006, 'pred_duration': 2.3168554306030273, 'grad_duration': 6.480625629425049, 'total_duration': 8.797481060028076, 'pred_std': 0.04763111099600792, 'std_loss': 0.864331066608429, 'mean_pred_loss': 9.326593499281444e-06, 'pred_rmse': 0.1368829905986786, 'pred_mae': 0.08326660096645355, 'pred_mape': 3.067068099975586, 'grad_rmse': 0.13684110343456268, 'grad_mae': 0.08312927931547165, 'grad_mape': 0.9944403171539307}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.018736953249844236, 'avg_g_mag_loss': 0.6755777371781213, 'avg_g_cos_loss': 0.01375962085758006, 'pred_duration': 2.3168554306030273, 'grad_duration': 6.480625629425049, 'total_duration': 8.797481060028076, 'pred_std': 0.04763111099600792, 'std_loss': 0.864331066608429, 'mean_pred_loss': 9.326593499281444e-06, 'pred_rmse': 0.1368829905986786, 'pred_mae': 0.08326660096645355, 'pred_mape': 3.067068099975586, 'grad_rmse': 0.13684110343456268, 'grad_mae': 0.08312927931547165, 'grad_mape': 0.9944403171539307}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:32:14.356600Z", + "iopub.status.busy": "2024-03-22T15:32:14.356191Z", + "iopub.status.idle": "2024-03-22T15:32:14.361074Z", + "shell.execute_reply": "2024-03-22T15:32:14.360119Z" + }, + "papermill": { + "duration": 0.028774, + "end_time": "2024-03-22T15:32:14.363233", + "exception": false, + "start_time": "2024-03-22T15:32:14.334459", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:32:14.403512Z", + "iopub.status.busy": "2024-03-22T15:32:14.403213Z", + "iopub.status.idle": "2024-03-22T15:32:14.485670Z", + "shell.execute_reply": "2024-03-22T15:32:14.484875Z" + }, + "papermill": { + "duration": 0.105605, + "end_time": "2024-03-22T15:32:14.488131", + "exception": false, + "start_time": "2024-03-22T15:32:14.382526", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:32:14.528399Z", + "iopub.status.busy": "2024-03-22T15:32:14.527691Z", + "iopub.status.idle": "2024-03-22T15:32:14.808061Z", + "shell.execute_reply": "2024-03-22T15:32:14.807102Z" + }, + "papermill": { + "duration": 0.302777, + "end_time": "2024-03-22T15:32:14.810066", + "exception": false, + "start_time": "2024-03-22T15:32:14.507289", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABKY0lEQVR4nO2dd3xUVfr/3zNJJr1AQhoEQpUemsQgRSVSRCXIShFpi4AuWIhtsYCr35/YRYUVcVfQXREsgKgsCqEoJPREQCBSAgFCEkJJrzP398fJTNpMMjOZzCTkvF+v+5qbuefee2Ym85nnPOd5nqNSFEVBIpFImghqR3dAIpFILEGKlkQiaVJI0ZJIJE0KKVoSiaRJIUVLIpE0KaRoSSSSJoUULYlE0qRwdnQH7IVOpyMtLQ1vb29UKpWjuyORSKqhKAq5ubmEhoaiVpu2p5qNaKWlpREWFubobkgkkjq4cOECbdq0MXm82YiWt7c3IN4QHx8fB/dGIpFUJycnh7CwMMN31RTNRrT0Q0IfHx8pWhJJI6Yu9410xEskkiaFFC2JRNKkkKIlkUiaFM3GpyWpH1qtltLSUkd3Q9KEcXFxwcnJqd7XkaIlqRVFUUhPT+fGjRuO7orkJsDPz4/g4OB6xUpK0ZLUil6wAgMD8fDwkIG5EqtQFIWCggIyMzMBCAkJsfpaUrSqUVii5cjFG5RodQzp3MrR3XEoWq3WIFj+/v6O7o6kiePu7g5AZmYmgYGBVg8VpSO+GpezC5m4ci9/++9hR3fF4eh9WB4eHg7uieRmQf+/VB//qBStavi4uwCQV1KGTifL50PdwX4SibnY4n9JilY1vN3EiFlRhHBJJJLGhRStarg6O6FxFm9LTqGc4pdIGhtStIzg4yaGiLlF0tKSWIdKpWLjxo2O7oZNeeWVV+jTp4+juyFFyxg+5UNEaWlJmjKrV6/Gz8/PZtd75plniIuLs9n1rEWGPBjB211aWpLmQ0lJCRqNps52Xl5eeHl52aFHtSMtLSMYLK0iaWlVR1EUCkrK7L5ZuhD6li1bGDx4MH5+fvj7+3Pvvfdy5swZAAYNGsTzzz9fpf2VK1dwcXHh119/BeDy5cuMGTMGd3d32rdvz5o1awgPD2fp0qVWvW9Hjx7lrrvuwt3dHX9/f+bMmUNeXp7h+M6dOxk4cCCenp74+flx++23c/78eQB+//137rzzTry9vfHx8aF///4cPHiw1vvt3LmTmTNnkp2djUqlQqVS8corrwAQHh7Oa6+9xrRp0/Dx8WHOnDkAPP/883Tp0gUPDw86dOjAyy+/XCU0ofrwcMaMGcTExPDOO+8QEhKCv78/8+bNa/B0L2lpGUH6tExTWKql+6Kf7X7f46+OxENj/r9rfn4+sbGx9O7dm7y8PBYtWsS4ceNISkpiypQpvPXWW7zxxhuGKfh169YRGhrKkCFDAJg2bRpZWVns3LkTFxcXYmNjDdHclpKfn8/IkSOJioriwIEDZGZm8sgjjzB//nxWr15NWVkZMTExzJ49m6+++oqSkhL2799v6NuUKVPo27cvH3/8MU5OTiQlJeHi4lLrPQcNGsTSpUtZtGgRycnJAFWspHfeeYdFixaxePFiw3Pe3t6sXr2a0NBQjh49yuzZs/H29ua5554zeZ8dO3YQEhLCjh07OH36NBMnTqRPnz7Mnj3bqvfKHKRoGcFb+rSaPOPHj6/y92effUarVq04fvw4EyZM4KmnnmL37t0GkVqzZg2TJ09GpVJx8uRJtm3bxoEDBxgwYAAA//rXv+jcubNVfVmzZg1FRUV88cUXeHp6ArBs2TLuu+8+3nzzTVxcXMjOzubee++lY8eOAHTr1s1wfmpqKs8++yxdu3YFMKsfGo0GX19fVCoVwcHBNY7fddddPP3001Wee+mllwz74eHhPPPMM6xdu7ZW0WrRogXLli3DycmJrl27MmbMGOLi4qRo2Rt9gGlusbS0quPu4sTxV0c65L6WcOrUKRYtWsS+ffvIyspCp9MBQgB69uzJiBEj+PLLLxkyZAgpKSkkJCTwySefAJCcnIyzszP9+vUzXK9Tp060aNHCqr6fOHGCiIgIg2AB3H777eh0OpKTkxk6dCgzZsxg5MiR3H333URHRzNhwgRDfl5sbCyPPPII//nPf4iOjubBBx80iJu16MW4MuvWrePDDz/kzJkz5OXlUVZWVmeV3x49elRJxwkJCeHo0aP16ltdSJ+WEbxdpaVlCpVKhYfG2e6bpZHU9913H9euXePTTz9l37597Nu3DxBOZxBDrm+//ZbS0lLWrFlDr1696NWrl83fL3NZtWoVCQkJDBo0iHXr1tGlSxf27t0LCF/SH3/8wZgxY9i+fTvdu3dnw4YN9bpfZQEFSEhIYMqUKdxzzz38+OOPJCYm8uKLLxreL1NUH6aqVCrDD0RDIUXLCD5y9rBJc/XqVZKTk3nppZcYPnw43bp14/r161XajB07lqKiIrZs2cKaNWuYMmWK4dgtt9xCWVkZiYmJhudOnz5d4xrm0q1bN37//Xfy8/MNz+3Zswe1Ws0tt9xieK5v374sXLiQ+Ph4evbsyZo1awzHunTpwoIFC/jll1944IEHWLVqVZ331Wg0aLVas/oYHx9Pu3btePHFFxkwYACdO3c2TAQ0NqRoGcFbzh42aVq0aIG/vz8rV67k9OnTbN++ndjY2CptPD09iYmJ4eWXX+bEiRNMnjzZcKxr165ER0czZ84c9u/fT2JiInPmzMHd3d2q3LkpU6bg5ubG9OnTOXbsGDt27ODxxx9n6tSpBAUFkZKSwsKFC0lISOD8+fP88ssvnDp1im7dulFYWMj8+fPZuXMn58+fZ8+ePRw4cKCKz8sU4eHh5OXlERcXR1ZWFgUFBSbbdu7cmdTUVNauXcuZM2f48MMP623NNRRStIygnz3MkZZWk0StVrN27VoOHTpEz549WbBgAW+//XaNdlOmTOH3339nyJAhtG3btsqxL774gqCgIIYOHcq4ceMMM2lubm4W98fDw4Off/6Za9euceutt/KXv/yF4cOHs2zZMsPxkydPMn78eLp06cKcOXOYN28ec+fOxcnJiatXrzJt2jS6dOnChAkTGD16NP/4xz/qvO+gQYN49NFHmThxIq1ateKtt94y2fb+++9nwYIFzJ8/nz59+hAfH8/LL79s8Wu1ByrF0gCYJkpOTg6+vr5kZ2fX6Vzcd/YqE1fupUOAJ9ufucM+HWyEFBUVkZKSQvv27a36st5MXLx4kbCwMLZt28bw4cMd3Z0mS23/U+Z+R+XsoRG8paXV7Nm+fTt5eXn06tWLy5cv89xzzxEeHs7QoUMd3bVmjxweGsHHXWh5rvRpNVtKS0t54YUX6NGjB+PGjaNVq1aGQNMvv/zSkNJSfevRo4fd+jh69GiT/Xj99dft1g97Iy0tI+gtreIyHcVlWlyd67+CiKRpMXLkSEaONB6Pdv/99xMZGWn0WF2R6rbkX//6F4WFhUaPtWzZ0m79sDdStIzg7eqMSiUKAeYWleHqJUVLUoG3tzfe3t6O7gatW7d2dBccghweGkGtVuGlkQGmEkljRIqWCWSAqUTSOJGiZQIZYCqRNE6kaJlAlqeRSBonUrRMIMvTSCSNEylaJpA+LUl9uBkXtmgsSNEygfRpSZo6tl7YAkQZZ5VKxY0bN2x6XUuQomUC6dOSSBonUrRMIH1aJlAUKMm3/yYXtrDZwhbFxcU888wztG7dGk9PTyIjI9m5c6fh3PPnz3PffffRokULPD096dGjB5s3b+bcuXPceeedgCj/o1KpmDFjhlXvR32QEfEmkEnTJigtgNdD7X/fF9JA41l3u3LkwhamF7aYP38+x48fZ+3atYSGhrJhwwZGjRrF0aNH6dy5M/PmzaOkpIRff/0VT09Pjh8/jpeXF2FhYXz33XeMHz+e5ORkfHx8cHd3t+o9qQ9StEygT5qWPq2miVzYwvjCFqmpqaxatYrU1FRCQ8WPzzPPPMOWLVtYtWoVr7/+OqmpqYwfP95QfrpDhw6G8/U5jYGBgTb3l5mLFC0TeEuflnFcPITV44j7WoBc2MI4R48eRavV0qVLlyrPFxcX4+/vD8ATTzzBY489xi+//EJ0dDTjx4+nd+/eVt2vIZA+LRPoF2yV5WmqoVKJYZq9N7mwhU0WtsjLy8PJyYlDhw6RlJRk2E6cOMEHH3wAwCOPPMLZs2eZOnUqR48eZcCAAXz00Uc2e631xSrRWr58OeHh4bi5uREZGcn+/ftrbf/NN9/QtWtX3Nzc6NWrF5s3b65yfP369YwYMQJ/f39UKhVJSUlGr5OQkMBdd92Fp6cnPj4+DB061GRpjvpi8GlJR3yTQy5sITC2sEXfvn3RarVkZmbSqVOnKlvlYWRYWBiPPvoo69ev5+mnn+bTTz81XBMwe8GMhsBi0Vq3bh2xsbEsXryYw4cPExERwciRI006KePj45k8eTKzZs0iMTGRmJgYYmJiOHbsmKFNfn4+gwcP5s033zR534SEBEaNGsWIESPYv38/Bw4cYP78+ajVDWMs6n1aecVl6HTNoiL1TYNc2EJgbGGLLl26MGXKFKZNm8b69etJSUlh//79LFmyhJ9++gmAp556ip9//pmUlBQOHz7Mjh07DPdr164dKpWKH3/8kStXrlSZAbUbioUMHDhQmTdvnuFvrVarhIaGKkuWLDHafsKECcqYMWOqPBcZGanMnTu3RtuUlBQFUBITE2sci4yMVF566SVLu2sgOztbAZTs7Gyz2heWlCntnv9Raff8j0pOYYnV923KFBYWKsePH1cKCwsd3RWL2bp1q9KtWzfF1dVV6d27t7Jz504FUDZs2GBos3nzZgVQhg4dWuP8tLQ0ZfTo0Yqrq6vSrl07Zc2aNUpgYKCyYsUKs+5f/V5HjhxR7rzzTsXNzU1p2bKlMnv2bCU3N1dRFEVJT09XYmJilJCQEEWj0Sjt2rVTFi1apGi1WqW4uFiZNGmSEhYWpmg0GiU0NFSZP3++2Z/Jo48+qvj7+yuAsnjxYkVRFKWkpERZtGiREh4erri4uCghISHKuHHjlCNHjiiKoijz589XOnbsqLi6uiqtWrVSpk6dqmRlZRmu+eqrryrBwcGKSqVSpk+fblY/9NT2P2Xud9Qi0SouLlacnJyqfBiKoijTpk1T7r//fqPnhIWFKe+//36V5xYtWqT07t27RltTopWRkaEAyocffqhERUUpgYGBytChQ5XffvvNZF+LioqU7Oxsw3bhwgWLREun0ymdX9istHv+R+XS9QKzzrnZaMqiZWv0/z/btm1zdFeaNLYQLYvGVllZWWi1WoKCgqo8HxQURHp6utFz0tPTLWpvjLNnzwLCITl79my2bNlCv379GD58OKdOnTJ6zpIlS/D19TVsYWFhZt8PRO6YTOVpvmzfvp1NmzaRkpJCfHw8kyZNkgtbNBKaxOyhfrp67ty5zJw5k759+/L+++9zyy238Nlnnxk9Z+HChWRnZxu2CxcuWHxfmTTdfJELWzReLIrTCggIwMnJiYyMjCrPZ2RkVJl5qExwcLBF7Y2hj1fp3r17lee7detGamqq0XNcXV1xdXU1+x7GkKk8zRe5sEXjxSLR0mg09O/fn7i4OGJiYgBhBcXFxTF//nyj50RFRREXF8dTTz1leG7r1q1ERUWZfd/w8HBCQ0MN6Qh6/vzzT0aPHm3JS7AImTQtMYZc2MKxWBwRHxsby/Tp0xkwYAADBw5k6dKl5OfnM3PmTEDkbLVu3ZolS5YA8OSTTzJs2DDeffddxowZw9q1azl48CArV640XPPatWukpqaSliYirfXiFBwcTHBwMCqVimeffZbFixcTERFBnz59+Pzzzzl58iTffvttvd8EU0iflkBpHouQS+yATf6XrJkB+Oijj5S2bdsqGo1GGThwoLJ3717DsWHDhtWYBv3666+VLl26KBqNRunRo4fy008/VTm+atUqBaix6ado9SxZskRp06aN4uHhoURFRdU6e1gdS0MeFEVRnvvmd6Xd8z8qy7afMvucm4mysjLl+PHjVaa7JZL6kJWVpRw/flwpKyurcczc76hKUZrHz2hOTg6+vr5kZ2fj4+Nj1jn/9+Nx/rU7hblDO7DwnrqD+W5GLl++zI0bNwgMDMTDw8Oq4EqJRFEUCgoKyMzMxM/Pz+Cnroy531GZMF0LsjwNhgkTa8uySCSV8fPzs2gSzhhStGpBn8rTnJOmVSoVISEhBAYGUlrafN8HSf1xcXHByan+q7VL0aoFaWlV4OTkZJN/OImkvjSJ4FJHIcvTSCSNDylatSDL00gkjQ8pWrVQ4dOSw0OJpLEgRasWfAw+LWlpSSSNBSlataAXraJSHSVlOgf3RiKRgBStWvFyq5hclc54iaRxIEWrFpzUKrxcpV9LImlMSNGqA5k0LZE0LqRo1YEsTyORNC6kaNWBLAQokTQupGjVgbeb9GlJJI0JKVp1oK8TL31aEknjQIpWHVQ44qWlJZE0BqRo1UGFI15aWhJJY0CKVh1UJE03IktLUUArRVTSPJGiVQeNshDg+jnw7i2Qf9XRPZFI7I4UrTrwboxJ06d+gYKrcOmgo3sikdgdKVp14NPYQh5Ki6Dohti/YXyhWonkZkaKVh00Oksrv9ICE9fPOawbEomjkKJVB76NrRBgbkbF/o3zjuuHROIgpGjVgXel3MNGsURkXnrFvhweSpohUrTqQB+npdUpFJRoHdwbILeSaF2Xlpak+SFFqw7cXNQ4q8Wqyo3Cr5VXaXhYdAOKsh3WFYnEEUjRqgOVStW4kqYrW1ogh4iSZocULTMwJE03hvI0edWWp5dDREkzQ4qWGTQqS0vviNd4i0c5gyhpZkjRMoNGtZSYPuShTX/xKIeHkmaGFC0zaDTlaXTaiuDSsEjxKIeHkmaGFC0zaDTlafKzQNEBKmgtLS1J80SKlhk0mvI0en+WZyto2UHs3zgvStVIJM0EKVpm0GjK0+j9WV5B4Bsm9kvyoOCa4/okkdgZKVpmUJE07WhLq1y0vIPAxQ28Q8TfcgZR0oyQomUGFeVpHGxp6YeHXsHi0a+teJSiJWlGSNEygwqfViMZHnoHiUe/duJRziBKmhFStMzAp7GUp6luabUoFy05gyhpRlglWsuXLyc8PBw3NzciIyPZv39/re2/+eYbunbtipubG7169WLz5s1Vjq9fv54RI0bg7++PSqUiKSmpxjXuuOMOVCpVle3RRx+1pvsW02iCS2tYWnJ4KGl+WCxa69atIzY2lsWLF3P48GEiIiIYOXIkmZmZRtvHx8czefJkZs2aRWJiIjExMcTExHDs2DFDm/z8fAYPHsybb75Z671nz57N5cuXDdtbb71lafetwqdSTS2HYrC05PBQ0nyxWLTee+89Zs+ezcyZM+nevTsrVqzAw8ODzz77zGj7Dz74gFGjRvHss8/SrVs3XnvtNfr168eyZcsMbaZOncqiRYuIjo6u9d4eHh4EBwcbNh8fH0u7bxX6iPiCEi2lWp1d7lkDRalIltaLln54mH1BxmpJmg0WiVZJSQmHDh2qIi5qtZro6GgSEhKMnpOQkFBDjEaOHGmyfW18+eWXBAQE0LNnTxYuXEhBQYHJtsXFxeTk5FTZrMWrXLQA8hxlbRVlQ1mR2Pcu92n5tAaVWjxfuc6WRHIT41x3kwqysrLQarUEBQVVeT4oKIiTJ08aPSc9Pd1o+/T0dKPtTfHQQw/Rrl07QkNDOXLkCM8//zzJycmsX7/eaPslS5bwj3/8w6J7mMLFSY2HxomCEi05RaW08NTY5LoWoRclV19wcRf7Ti7g0wayU8UQUS9mEslNjEWi5UjmzJlj2O/VqxchISEMHz6cM2fO0LFjxxrtFy5cSGxsrOHvnJwcwsLCrL6/t5szBSVax/m19MX/vKv+AODXVojWjVRoG2n/fkkkdsai4WFAQABOTk5kZFQdimRkZBAcbPxXPjg42KL25hIZKb6gp0+fNnrc1dUVHx+fKlt98HF0rFZepRSeyhjCHs7ZtTsSiaOwSLQ0Gg39+/cnLi7O8JxOpyMuLo6oqCij50RFRVVpD7B161aT7c1FHxYREhJSr+uYi8PL0xgsrWpiL2cQJc0Mi4eHsbGxTJ8+nQEDBjBw4ECWLl1Kfn4+M2fOBGDatGm0bt2aJUuWAPDkk08ybNgw3n33XcaMGcPatWs5ePAgK1euNFzz2rVrpKamkpaWBkBycjKAYZbwzJkzrFmzhnvuuQd/f3+OHDnCggULGDp0KL179673m2AO+pLLDkvlMWVpGWK1ZICppHlgsWhNnDiRK1eusGjRItLT0+nTpw9btmwxONtTU1NRqysMuEGDBrFmzRpeeuklXnjhBTp37szGjRvp2bOnoc2mTZsMogcwadIkABYvXswrr7yCRqNh27ZtBoEMCwtj/PjxvPTSS1a/cEtxeNJ0ncNDaWlJmgcqpVGsQNrw5OTk4OvrS3Z2tlX+rRc3HOXLfak8Fd2Zp6K7NEAP62D1vXDuN3jgU+g9oeL57EvwfndQO8NLmaB2sn/fJBIbYO53VOYemonDCwGasrS8g0HtAroyyEmzf78kEjsjRctMHF4I0JB3WM0Rr3YCv/JQDjlElDQDpGiZibcjk6ZLC6G4fCXp6pYWyBlESbNCipaZ+Dhy7UN9uIOTK7j51jwuZxAlzQgpWmbi0PI0+kRp7yBQqWoelzOIkmaEFC0zcegq09WL/1VHDg8lzQgpWmaiDy51SBpP9eJ/1fGTFUwlzQcpWmZS2dKye2hbXZaWfniYcwnKSuzTJ4nEQUjRMhO9T6tMp1BYqrXvzeuytDxbgbM7oIiCgBLJTYwULTPx0DjhpBZOcLv7teqytFQqOYMoaTZI0TITlUpVUenB3n4tU9HwlZEziJJmghQtC3BYeZq6hocgLS1Js0GKlgVUrMpjR0tLWwb5V8S+qeEhyLAHSbNBipYFOMTSyr8CKGIBC88A0+3k8FDSTJCiZQEOsbT0TnjPwNrLzsjhoaSZIEXLAhxSnsYcfxZUDA/zMkSCtaT+lBbB5mfhz58d3RNJJaRoWYBDytOYM3MI4N4CXMsLp0lryzYc+xb2r4SfX3R0TySVkKJlAQ4pT2OuaMlYLdtzunxBlmtnhNUlaRRI0bIAh5SnMbUKjzEMM4jnGqw7zQadFs7uEPuKDq6ecmx/JAakaFmAQ9Y+NNfSAjmDaEvSkqDwesXfV5Id1hVJVaRoWYBDytNYZGnJ4aHNOFN1rU4yTzimH5IaSNGyAEN5Gof4tCwZHkpLq97o/VmtuonHKycd1xdJFaRoWYDdLS1FqRCtukIeoJKlJUWrXhRlw8UDYj/qb+JRWlqNBilaFmB3n1bhddCW18fyDKy7vd6nVXgdinIarl83O2d3gaIF/07QeYR47nqKnEFsJEjRsgC9pZVfoqVMq2v4G+qtLDc/cHGru72rN7i3FPvSr2U9en9Wx+FiAsTNT8wgZv3p0G5JBFK0LEAfpwWQV2yHIaIlTng9cohYPxQFTm8X+52Gi/i3QOnXakxI0bIAjbMaNxfxltnFr2VJuIOeFrJefL24egayU8FJA+GDxXOtuopH6ddqFEjRshC7LiVmlaUlZxDrhX5o2PY20HiKfWlpNSqkaFlIRfXSRmppyeFh/ThdyZ+lR1pajQopWhaij9WyS9K0VcPDcPEoh4eWU1YM534T+50qiZbe0rp+TlbQaARI0bKQiqRpezji9TFaVg4P7b3UWVMndS+UFogfiaCeFc97tiqflVXkDGIjQIqWhVQkTdvD0tKvwmPJ8DBMPJbkVs2dk9SNIdThLjFrqKfyDGKm9Gs5GilaFmLXQoDWWFou7hUiJ/1alqEPdajsz9Kj92tdkX4tRyNFy0LsZmmV5AtrCSyztEDOIFpDbgZkHBX7He6oedzgjJeWlqORomUhdkua1oc7uHiISHdLkDOIlqOvnRUSAV6tah4PlJZWY0GKloXYLWk6L1M8egVV9a+YgwwwtRxjoQ6V0Vd7uH4eSgrs0yeJUaRoWYjdgkutccLr0VtacnhoHjodnKmUumMMr1bg4Y+YQZQFAR2JFC0LsZulZe4qPMbwk5aWRaQfgYIs0HhBm4Gm27WSM4iNAatEa/ny5YSHh+Pm5kZkZCT79++vtf0333xD165dcXNzo1evXmzevLnK8fXr1zNixAj8/f1RqVQkJSWZvJaiKIwePRqVSsXGjRut6X69MPi0Gro8jcHSsmDmUE/l4aGM1aobfahD+6HgrDHdTvq1GgUWi9a6deuIjY1l8eLFHD58mIiICEaOHElmZqbR9vHx8UyePJlZs2aRmJhITEwMMTExHDt2zNAmPz+fwYMH8+abb9Z5/6VLl6Ky1MdjQ5qEpeXTBlBBWWGFb0xiGkOow121t5MziI0Ci0XrvffeY/bs2cycOZPu3buzYsUKPDw8+Oyzz4y2/+CDDxg1ahTPPvss3bp147XXXqNfv34sW7bM0Gbq1KksWrSI6OjoWu+dlJTEu+++a/Je9qCyT0tpSCumPpaWswZ8Wot9OUSsneJcuLBX7JvyZ+mRidONAotEq6SkhEOHDlURF7VaTXR0NAkJCUbPSUhIqCFGI0eONNneFAUFBTz00EMsX76c4OC6v8jFxcXk5ORU2WyB3tIq1SoUlzVgIcD6WFogV+Yxl5TfQFcGLdpDyw61t9X7tG6cF3F0EodgkWhlZWWh1WoJCqr6RQoKCiI9Pd3oOenp6Ra1N8WCBQsYNGgQY8eONav9kiVL8PX1NWxhYWEW3c8Unhpn1OWj0wadQbQmWboyhhnEczbpzk2L3p9Vl5UF4Okv8hBBLinmQJrE7OGmTZvYvn07S5cuNfuchQsXkp2dbdguXLhgk76o1Sq8XBu4PI22VMxmgXXDQ5AziOZSV3xWdQzpPHKI6CgsEq2AgACcnJzIyMio8nxGRobJIVtwcLBF7Y2xfft2zpw5g5+fH87Ozjg7C9EYP348d9xxh9FzXF1d8fHxqbLZigYvT6N3nqudy2ODrEAOD+vm2lmxYIXauaJKaV3I2loOxyLR0mg09O/fn7i4ioUsdTodcXFxREVFGT0nKiqqSnuArVu3mmxvjL///e8cOXKEpKQkwwbw/vvvs2rVKktegk1o8PI0eie8ZyCorTSGZYBp3egDSsMiwc3MH7VAaWk5GmdLT4iNjWX69OkMGDCAgQMHsnTpUvLz85k5cyYA06ZNo3Xr1ixZsgSAJ598kmHDhvHuu+8yZswY1q5dy8GDB1m5cqXhmteuXSM1NZW0tDQAkpOFvyA4OLjKVp22bdvSvn17y191PfFu6KTp+jrhoWJ4mH0RdFpQO9W/X+aSmw5HvoZbHwGNh/3uaynmhjpURgaYOhyLRWvixIlcuXKFRYsWkZ6eTp8+fdiyZYvB2Z6amoq6knUwaNAg1qxZw0svvcQLL7xA586d2bhxIz17VhRZ27Rpk0H0ACZNmgTA4sWLeeWVV6x9bQ2GT0OXp6lPuIMen1Ax7NGVQu5l8G1jm76ZQ9xrkPRfKCuCYc/Z776WoC2FlF/FvjlOeD36sIfsVCjOA1cv2/dNUisWixbA/PnzmT9/vtFjO3furPHcgw8+yIMPPmjyejNmzGDGjBkW9aFBY6TqoMHL0+h9WvWxtNROQqiunxObPUXrwj7xmLrXfve0lAv7RekfjwAIjjD/PI+WYtienylmENv0b7g+SozSJGYPGxsNXp4mtx7J0pUJuEU8Zhyv33UsoSgbrp4S+5cONd40IkOV0jst9xvKdB6HIkXLCho8lae+MVp6QsotiPTf63cdS0hLqtgvuiHWEWyMWBrqUBmDX0uKliOQomUFFT6tBra0LCmzbIyQ3uLxsj1F63DVvy8esN+9zSU/q+I9scQJr8dgackAU0cgRcsK7Gdp1Ve0yi2tzBNieSx7cKlctFzKFzq9dNA+97WElF2AAkG9rPMbygBThyJFywoa1Kel01WIVn0c8QC+YeDeQuTWZdrJr5WWKB4jxAwwFxuhaJ2PF4/th1h3vl60si+IhGuJXZGiZQUNamkVXhciA2KWqj6oVBXW1uUj9buWOeRdEV9kVHDrLPFcxrHGt8Dp+fJk/bbmBzhXwaNlhb9RDhHtjhQtK2hQn5Y+Rsu9Ze0F6cwl2I5+Lb0/K6ALBHYXycW6MvsIprkUXq+wOtveZv11Gjqd58+f4eyuhrl2E0eKlhU0qKVlKye8HoOlZQfR0vuzWvcTVl7rAeXPN6IhYuo+QAH/TuBVD0u2IWtrZZ2GNRPhvw/AtRTbX7+JI0XLCvQ+rbySMnQ6G8ch2SrcQU9IH/GYcQy0DVxtVW9phfYTj/rAy8bk10ot92dZOzTU05CW1pG1gCKs1F/ftv31mzhStKxAb2kpCuQW21gIbG1ptewgFmwoK4KsP21zTWMoSlVLCxqnpaX3Z7UbVL/rNJSlpdPB7+sq/v79K2F5SQxI0bICV2cnXJ3FW2fzVB5bW1pqdYVfK70BfUvZF0QNMLUzBJXnlbbuB6hETa+8Kw13b3MpLayY3bSVpZVzSWQB2IrUBJHX6OoDHe4ERQe73rDd9W8CpGhZiXdDJU3b2tIC+wSZ6q2soB7g4ib23XyFUx4ah7V16ZBIIPcOgRbh9buWu5+4Dth2BvH3r8Rj97Fw9z/E/tFvZfR9JaRoWUmDJU1XXlnaVtjDGV/dn6WnTfkQsTH4tSqHOthiRSdb+7VKC+H492I/YpL43LrdByiwc4lt7nETIEXLSrzdG6gQYH1WljZF5VgtXQMtxlHdn6WndbkzvjFYWrZywutpZeN0nuTNUJwDvm2hbbnP7Y4XAJUQs/SjtrlPE0eKlpU0mKVlKABow+FhwC3g7CZKsVxvgCl0na7Ciqthad0qHi8dbjjBNAdtmShHA9DORqJl62oPegd87wkVlSeCukOPcWJ/h7S2QIqW1TRIgGlxLpSWL01lS0vLyVn4mqBhhohXTwsLwdm9wvrQE9gdXDzEcX3JGkeQcRRK8sDVV/TJFtiyimleJpzeJvb1KVB67lgIKjUk/1Rh0TZjpGhZSYMEmOqtLI2X7StiNmRkvN6fFRIhBLIyTs4VsWKO9GsZ/FmRtis93aq8XlluGhTeqN+1jn0HilYMpwM6V7tPF+g1QezveL1+97kJkKJlJQ2RNL3tgBAUbX1zDo3RkM54U/4sPW0agV/L1v4sKJ9BDBX79fVr6WcNIyYbPz7sOVA5wemtFcPcZooUreqUFsGeD2Hnm7U283a1raV1PC2HjbtFDNGRbDeOXrRh7A9UFS1bVxM1NXOop7WDZxAVxXZBpdWxhV8r84T4XNTO0OMB4238O0KfckHb8f+sv9dNgBSt6qTGw9aX4bd3al2d2daW1ts/nySQGwBcKvVh/Mfx/Gfveatr4et0Cj8eSeOfO09TXKYVfhy1MxReEyv02AptacWslklLq1y0Mv6AkgLb3dtcrp4Wga9OrhDa17bXtoVf6/e14rHzSLGKtSmGPgdqFzi7E87tsf5+TRwpWtXpcKfYtCUQ96rJZrb0ae07e5UdyVcIUt8AwK1FKCVaHS9vPMZT65LItzBVaO/Zq8T8cw/z1yTy1pZklu84IwI+9U5yW0bGZx4XKUKuviJlyBg+rUVBQ0Vr3yqqelLLraw2A8DZ1bbXrq+lpdPC0W/EfsTE2tu2aAf9por9Hf+v8dbfb2CkaFVHpYIRrwEq4Rw1MaSx1eyhoii8uUX8Sg8MKAFg+K29efGebjipVXyflMbY5Xs4lVF3sbnTmXk88vlBJq3cy5GL2YZUoxU7z3DmSl7D+LX0/qzQPqYDNlWqCmvLEX4tgxO+HqVoTFFfS+vcbyIVyM0Xuoyqu/2QZ4TFeH6PsLiaIVK0jBHcC/pMEfs/v2j0F81Wlta2E5kcTr2Bm4ua7t6iWJ7KO5jZQzuwds5tBPm4cjozj/uX7WFj4iWj18jKK+aljUcZufRXtp3IwEmtYupt7djz97u445ZWBqtNaYgZxLQ6nPB69EGmjqgZb3DC29ifBRUziHnpolaXpehjs3o8YJ4V6NsaBpSvEdpMrS0pWqa460URd3RhL5z4ocZhHxtExGt1Cm//LH6h/3p7e1wLy5OKy8ss3xrekp+eGMLgTgEUlmp5al0SL244SlGpFoDCEi3Ltp9i2Fs7+O/eVLQ6hbu7B/HzU0N5LaYnAV6uvDa2J67OauLPXOXXvPKZLptaWuUJyKac8HoM6TyHbHdvc8i5LHyTKjWEDbT99d18xPAXLJ9BLMmHE5vEvqlZQ2MMXiCChS8eqIjtakZI0TKFTygMelzsb1sMZSVVDustrfo44tcfvsifGXn4urswd1BoealiKhJxgQAvVz7/60CeGN4ZlQq+3JfKX1bE83n8Oe58Zyfv/PIn+SVaerfxZd2c2/h02gA6BVbEeIW19OCJ4SLu58V4UFCJFaf1OY71oaSgogpoXZZWaF9ABTkXK5LC7YHeygrqKQSmIbA2B/HkTyLgtUV7ywTVOxhufUTsN0NrS4pWbdz+hKjTfu0sHPysyqGWnhpcnFSUlOn4PP6cxZcuKtWydJuIEP/bHR3xPbNJ/AP7tq2ojFCOk1pF7N1d+HzmQFp4uHDsUg6LN/1Bek4Rrf3c+WBSHzb+7XYiOxifeZo9pAOdA724WODEFdcw8aQtSiCnHxXOdc/ACmvDFK7eFTWo7Bn60FChDpWxtraWftYwYpLlCdyDF4gVj9ISRc5iM0KKVm24esOdL4j9XW9UiXr20Diz4G4hLv/44Q+2n8yw6NL/3XueSzcKCfZxY3pUO9j3iThw6yyTEdtDu7TipyeGMLB9S/w8XHjhnq7EPT2MsX1ao1ab/qfXOKv5vxhR4yqhoI148nKSRf01Slq18sp14Yjk6dRKlR0aCr2ldfmI+VZPbjqc3SH2e0+w/J6eARA5V+zHvQoF1yy/RhNFilZd9J0qZogKr8Nv71Y59NiwjkwY0AadAo+vSeR4Wo5Zl8wtKmX5DlGN8qnozrilHxJhCM5u0G9areeG+rnz9dwoDr90N3OGdsTNxbyUlMgO/jzYvw3HdOEA6NJs4Ne6VEdQaXXsXaam8IaIDYOGtbT0YpwaD9/NMm9ZsaPfiAJ/YZGmQ0XqYtDj4OYnLLyVdzSuBUQaECladeHkXB4CAexbAdfPGw6pVCr+L6YXgzr6k1+iZdbnB8jIKarzkp/+epbrBaV0aOXJX/q3gf3lVlavv4jlqcygNsvKFAvv6cZ5TScA8s7ZwCFu7syhHn1kfFqiiE9qaC6UL2LRsmP9FrGoi6DuMPotEbx77Dv49K66/VuVh4bW4tESZvwkChreOA//HlG1VPNNihQtc+gUDR3uMBpwqnFW8/GU/nRs5cnl7CJmfX6AghLTM4pXcov5125RHubZEbfgnJ9RUfht4JyGegWA8MONvnskAD5Fl7h0Oc36ixXeEJHmYL6lFdhN+GFK8uyzXqA9hoZ6IufCjM0iFzHrTyFcR7423jb9qFhoxElTUXbGWoJ7wpyd0OluKCuEDXNg83MiU+EmRYqWOahUcLc+4PTbGtP2vh4urJoxEH9PDccu5fDEV0loTazSs2z7KQpKtESE+TGqZzAcWi1WXQm7rSL4swGJiepBppMIqVjz/U/WX0jvE/NrW3vqSWXUThVpNPbwaxmc8HYQLRAVJB79TWRUlBbA+tnw4wKRz1oZvZXVZZRYAby+uLeAh76GYc+Lv/d/Ap/fV1E15CZDipa5hPSGPg+J/V9qBpy29fdg5bQBaJzVbDuRweubaw4PUq8WsGZ/KgDPj7oFlbYUDq0SByMb1srSo1Kp8GgrLKPiC4n8/IeV4QeW+rP02GtZsdKiiuGrPSwtPZ4B8PB3MOzvgErMOn82ssKtoC2rlLZTj6FhddRqMWk0ea1YFCM1AT4ZWr7O482FFC1LuLM84DQ1AU7+WONw/3YtePdBYS39e3cK/9l7vsrx97YmU6pVGNI5gEEdA0RgYV6GyMvrdr9dXgKAV3shHD3VKbyy6Q+LcxsBy/1ZegyVTBs4yPTSITGc9wqy3tFtLWonuHMhTPlWWEGXk4SAJG+BlJ3iM3dvKYZ0tuaW0WK42KqbiNJfPQb2f3pTxXJJ0bIE39YwaL7Y37qoRsApwH0RoTwzQoRCvLLpD3YmiyDO42k5fP+78CE9P6p8ilwf5jDgr+Dk0rB9r0x5Ub4+zue5nF3E+1utWA/R3Ej46uid8ZnHoTjP8vuaS+X6WbZYxMIaOkfD3N/E7GLRDfhqIvywQBzrOR6cNQ1zX/+O8Mg24S/TlcLmZ2Dj38TCGTcBUrQs5fYnKwJO9UO7asy7sxN/6d8GrU5h/ppETqbn8NbPJ1EUuLd3CD1b+4oZtIv7RamR/jPs+xrKcxDbKWl4UMSq+HP8kVazflepVkfq1QLiT2ex7kAq7/6SzNs/n+TK5VQR2Y5KJEpbgk+ICERVdLaJFauGTqeIcj72CCo1B78wmLkFBpbHVGUL94BFaTu1kF1YSmGJkZlYVy/4yyoY8X8ihen3NcLqKiu2yX0diXPdTSRV0Aec/vgU7HwDek8UFSwroVKpeH1cLy5dLyTh7FUmr9zL9YJSnNUqnhlRnmC7/1Px2CPGkGtoN7yDwCsYVV46f+2Uz7LTbjz37RGGdwvi4vUCLl4v5OK1AtJzijA2n5C2/yjvg4jcd/W2/P6t+4vKBhcPQvjg+r4aA7tPZfHIFwcoLSvjd008XiqYsd2ZtPhduLs44a5xMjx6apx5KLItfdvawBFeF84auOct4ajf9KRIsrZ0WF1OYYmWg+evsft0FntOZ/FHWg7uLk7MHtKBOUM74Ola6SutUolYruDe8PVUMWQ+8YMIrWnCSNGyhr5TRczWlZOw+z24u2bdLY2zmhUP92fcx3s4e0UsVjFpYBjhAZ6QnyUW4ISKX2B7ExIBp9KZ0zmX1ReC+CMthz+MBMe6Oqtp08KdNi08aNPCnQPnrhF+NRmc4axrF6zyFrUZIPx5NpxBzCsu47lvf6eoVEcP1Xm8VIXkKO78mh2ILtv4MHRfyjW2Pz0MZyc7DTh6jocuo0Wog5lDVq1O4eilbPaczmL3qSwOnb9OibbqqkYFJVo+iDvFmv2pLIjuwoQBbaq+pg7DIPJR2PWmmK2WotUMcXIWIRBrHoSE5dDudugyskYzEQpxK+M/jqdUq/DEXeULFhz+ArTFwrekjxK3NyERcOpnfK4f570J41l34AKBPm7lAuVOWEshUgGerlUCWfOLy0j54P+gAFafa0np+iO8cn8PXJ0tWCzCUH7Zds74t7ecJC27iLCW7nw5QAu/grptJN/ePZiiEi0FJVoKS7UUlj8u3fYnqdcK+N+xdO6LCLVZP+pE41FnE51OYWPSJbYcSyfh7NUa5Y9CfN24vVMAgzsFENXRn4PnrvPmlpOkXivghQ1HWbUnhb+P7spdXQNR6cWx3zT49W1RvyvrVM3FM5oQUrSspfPdwi/x+1fw9TQxzW1kqNPO35O42Dso0+nw93IVU94H/i0ORs51nJO4UkHAETHBjOhh3jqLnhonenAGgKNKRxL3X+Bkei4rHu5PkI+befcO7SMWachNg5w0UVGjHhw4d40vymdql4zrjV/iagC8Og+hn4nh37X8Ej6IO8WKXWe4t3dIxZfbwRy7lM1LG4+RdOGG4TlvN2cGdfTn9k4B3N4pgA4BnlX6O6Z3CHd3D+K/e8/z4fZTnMrMY9bnB7mtQ0tevKc7vdr4gm8b6DwC/twirK2RTbfOvFV28fLlywkPD8fNzY3IyEj27699dZBvvvmGrl274ubmRq9evdi8uWpW+vr16xkxYgT+/v6oVCqSkpJqXGPu3Ll07NgRd3d3WrVqxdixYzl50gbrzVmLSgX3fyTM/bIiWDPJ5Jp0vh4uQrAA/vyfcGJ7+JtexMAehJQXBLxywjLn7I1UVAVXQe3MgocfwMfNmcTUG9z70W4OnTczaVfjWbH2YD3jtYpKtTz/3REUBSYMaMPgTv5mOeGnDwrHzUXNH2k57Dl9tV59sAV5xWW8+sNx7l+2m6QLN/BydeaJ4Z3ZOO92El++m0+mDmBaVDgdW3kZFViNs5q/Dm7PrmfvZO6wDmic1ew9e437lu3mybWJXLhWAP3LiwcmfVkz4LUJYbForVu3jtjYWBYvXszhw4eJiIhg5MiRZGYar88UHx/P5MmTmTVrFomJicTExBATE8OxY8cMbfLz8xk8eDBvvml6BZz+/fuzatUqTpw4wc8//4yiKIwYMQKt1g45bKZwcoEHV0P4ELF683/H1112Vx/m0G+6qNvuKHzDRAyRrqyiJpY56OOzgnowtHsYPzw+mFuCvLmSW8yklXv5ct/52s8vR1eeZHzpj98oq+ajsYRl209z9ko+rbxdefGe7mJWNz9T+I1qCcdo6alh0q1tAVix64zV968viqLw05HLDH93J5/tSUGnCMsp7ulhxN7dhT5hfhb53HzdXVg4uhvbnx7GuL6iXND3SWkMf3cXb58NQ/FpI5L/9cUHmyAqxcLlXiIjI7n11ltZtmwZADqdjrCwMB5//HH+/ve/12g/ceJE8vPz+fHHimDM2267jT59+rBixYoqbc+dO0f79u1JTEykT58+tfbjyJEjREREcPr0aTp27FjjeHFxMcXFFRZETk4OYWFhZGdn4+Nj42Jwxbnw+f3iC+0dAn/dIpJYq5N5Av55m5iCfvKImA53JF+MFXXG7/vA/LCLX16G+A/Fr/Z9SwHh53ru2yP8dPQyAJMHhhn8XIqikJ5TRHJ6Ln9m5JKcnsefGbn0urKJ19WfsFfXjRXtP2TFw/3Nrlih54+0bMYu20OZTmHFw/1FWtTh/8Cm+SI+669baj3/wrUC7nhnJ1qdwg/zB4thlB05fzWfl7//g1//FBVr2/l78NrYngzt0spm9zh2KZvXN58g/oywJpe13sq9V1eJ0tN//Z/N7mMLcnJy8PX1rfM7apGlVVJSwqFDh4iOjq64gFpNdHQ0CQkJRs9JSEio0h5g5MiRJtubQ35+PqtWraJ9+/aEhRn/4i9ZsgRfX1/DZqqdTXD1Fj6tVl1FVdAvYoxX59y/Ujx2HeN4wQLrFrpIKw8qrTRl7+nqzLKH+vL30V1Rq+Cr/RcYu2wPf/k4nt7/+IWoJduZseoAr28+yXeHL3L0UjYHSsUPTW/VWX5LTmf2FweNxxuZoEyr4/nvjlCmUxjdM1gIFliUJB3W0oN7e4sqsZ/8aj9rq7hMy4dxp7j7/V/59c8raJzUPDm8Mz8/NdSmggXQs7UvXz4SyQeT+uDipOLVSwPQohbBt/VZ9syBWCRaWVlZaLVagoKqxhUFBQWRnm48hy09Pd2i9rXxz3/+Ey8vL7y8vPjf//7H1q1b0WiMRxUvXLiQ7Oxsw3bhwgWL72cRHi1h6kbwawfXU+A/46oWZiu8UZEo28DVHMzGUtHS6SAtSexXG3qpVCoeHdaR1TMH4uvuwsn0XA6ev05uURlOahWdA70Y0zuE2Lu7sOLh/qxcMBlF442HqpjemjR+O5XFzNX7zU4p+vfuFI5dysHHzZl/jO1RceB8eSS8mUGlc4cK8dx89DLnr+abdU59iD+Txeilv/He1j8pKdMxuFMAW54awoK7u1hsaZqLSqVibJ/WfDK1Pzec/dmmFZ9dmX5CqInRpCLip0yZQmJiIrt27aJLly5MmDCBoiLjDkVXV1d8fHyqbA2OTwhM+17kEmYehy8frEhVSVojMv8DuwsfWGMguFy0Mv4Qs5p1cfWU8N05u1dU66yGqK46mJfGdOODSX3Y8tQQjr86kq2xw1j+UD+eGN6ZUT2DaR/og6q1qPjw4cBsvFyd2Xv2GtM/209uHXX3U7Lyea889ejle7sT6F3uG8xNFz8YqCpyHOuge6gPw7q0QqfAp7+dNesca9l9KouH/7WPs1nCB/fh5L78Z9ZAOrTyqvtkG3BX1yBWzbiVbxE5j0UH15CfZ0bBwkaGRaIVEBCAk5MTGRlVS15kZGQQHGx8yjw4ONii9rXh6+tL586dGTp0KN9++y0nT55kw4YNFl+nQWnZHqZuEE7uSwdh7UMi5+tAeQT8wNmOC3OoTssOoPESs59ZZuQf6mdHQyJErJoJ2rTw4JEhHRjbpzVdg31Mx3B1uBOAsINL+KX/Xnzc1Bw8f52p/95Pton1JHU6hb9/d4TiMh1DOgeIIoog6kfpK8sG9ayRpVAbjw4T1tY3By+SldcwaS6Xswt5Ym2icLT3Eo72+yNC7R5qcXunAGbPfISLSiu8lDz+/cl7Nlsl3V5YJFoajYb+/fsTFxdneE6n0xEXF0dUlHEfQlRUVJX2AFu3bjXZ3lwUReSYVXa2NxqCusOU74QgpOyCT4eLWS1XX+hlRT3whkKtNuQh1jlETEusyLW0MgWlBlHzyicAFEIPv8fusJWEuReTdOEGD/9rHzcKaiakrz1wgX0p13B3ceL1cb3El/7qGVG1U+8zrKNkdXVu69CSiDA/iq1cpKQuSsp0zPvyMNfyS+gR6sO7EyIMi/06goEdAlAPmAHA7dk/MOXTfVzPr/leN1YsHh7Gxsby6aef8vnnn3PixAkee+wx8vPzmTlTxIBMmzaNhQsXGto/+eSTbNmyhXfffZeTJ0/yyiuvcPDgQebPn29oc+3aNZKSkjh+XEy9Jycnk5SUZPB7nT17liVLlnDo0CFSU1OJj4/nwQcfxN3dnXvuuadeb0CD0aY/TP5KrAacWV6nvO/DIpG1MVGbX0unE+VUVo0RNcgvlNdm6hRds601OLuKmcux/wRnN3wubCfOazFRHsJZP/nTfVytZPlczi5kSXmdsmdH3kJYC3eRXbBisJi5dfMTISgW1iZTqVQ8OlQkJH2RcN66Uj218Mb/TnI49Qbebs58PMXyWdKGIPSO2SgqZ/qrT1GSdpTJn+7lSm4jNACMYLFoTZw4kXfeeYdFixbRp08fkpKS2LJli8HZnpqayuXLlw3tBw0axJo1a1i5ciURERF8++23bNy4kZ49exrabNq0ib59+zJmzBgAJk2aRN++fQ0hEW5ubvz222/cc889dOrUiYkTJ+Lt7U18fDyBgQ1Y+7u+tB8qvkQqJ7HdOsvRPaqJMdEqLYSDq2D5QFFO5fxuUf+890RRaqXTcNv2oe8UmLUV/NqhyU3lS9XLzPTYw4nLOYYvk6IovLzxGLnFZfRt68f0Pj6w7mHY9LjwFYYPgcfirS5fPKJHMO0DPMkuLOWr8kKNtuCnI5f5bI8or/3ehD609a87jccueAeh6ip+8Ge57+Rkei4TVyaQnt34g04tjtNqqpgbA9IgXDggCtKF327f+5pDxh/w8SAxlH0iUaQYHfgXFGSJ464+YggX+aioJ9aQFF6H9XPh1M8AbFRH83zBw7Ru1YIpke147cfjaJzUbH9Aoc3OWBFeonaB4S9D1ONiuFsPvtqfysL1RwnxdePX5+7EpZ6J1Geu5HH/R7vJL9Eyd1gHFo7uVq/r2Zwz2+E/49BpvLlbtZIz2QptW3rw5SORhLW0v7ia+x2VotXc0ZbBktbCGa92EUXjQCwae9tj0G+qdeVnrEWnEw71Hf8PUDih6sgjhU9wiVZoKGVth1/ol/alaOvfGcb/y/KaXiYoKtUy5K0dXMkt5t0HIxivd/JbQUFJGeOWx5OckUtk+5Z8+Uik/apJmItOBx/1hevnuBb9HjHxHUi9VkCwjxtv/qU3w2wcM1YXDRJcKrkJcXKG4F5iX1cq4q/+8pmwuqL+Zl/BAmEtDXtWBOu6t6SbcobNbi8yyWk7WzwWVwjWgFkw91ebCRaAm4sTM28PB0Swqc7E4iR1oSgKL204RnJGLq28Xfnoob6NT7BAvNflmRAtT3zJ13Oj6BToRXpOEdM/20/suiSuNUIHvbS0JGL4euw76HafCMpsLCEZN1JFBQ19FD6ARwCMXSZqoTcA2YWl3P7GdvKKy/hsxgDu6mp5gcY1+1J5YcNRnNQqvnwkkts6mLlakSPIuwLvdRM/WHN/I79ld975JZnV8edQFJGjufi+7nYJz5CWlsR8wm6F0W8In1tjESwQy5P99eeK6gSd7hbO9gYSLBAJx1MiyxOpd1oebHr0YjavbBKzxc+OvKVxCxaAVyvodq/YP7QKT1dnFt/Xg/WPDeKWIG+u5Zfw5NokZq4+wMXrBY7tazlStCSNG2dXkZj9/Dl4+Fu7lKb+6+D2uDip2H/uGofOXzf7vOyCUh778hAlWh3R3YKYO9TOqwBZi/5H4cjXIvkf6Nu2BT88PphnRnRB46RmZ/IVRrz/K5/tTjG5pqe9kKIlaRrYYlFTMwnycTOUdfnEzLI1Op1C7NdJXLxeSNuWHrw7IaLRFBask/ZDoWVHsfK3vgw4okbX/Ls6s/nJIQwMb0lBiZZXfzzOAx/HczK9Zmnu3KJSTlzOYevxDD7bncKrPxxnzhcHGf3Bb1WKGtYXWblUIjHCnKEd+ebQRbaeyOB0Zh6dAr1QFIUSrY6SMh3F+q1US4lWx4+/XybuZCYaZzX/nNIPX3fHRbxbjEoFA2bCLy+JrIcBM6sc7hToxdo5t/HVgVTe2HyS3y/c4N4Pd3N/n1AKS7RcKF8M5UaB6XSglKw8+oT52aa70hEvkRhnzhcH+eV4Bq7OYkBSXFZ3scI3HujFpIFtG7prtif/KrzXVcQTzt5hMlUrPbuIl78/xtbjGUaPt/BwMSyCol9nIKyFBz1b+9LK27XWLpj7HZWWlkRigvl3dWL7yUyTYqVxVuPqrMbV2QlXZzUTBoQx8dZGUCfNGjz9oftYOPqNsLZMiFawrxsrp/Yn7kQmB85dI9jXjTYtPAhr6U5rP3e87ZBTKS0tiaQWMnOLyCsqw9XFCY2TGlcXIVQaJ3XT8VmZy7k9sPoecHYTy5217i+2oB52WQFdWloSiQ0I9HYj0M7xtQ6j3SCRi3r5d7H4RVJ5IK+zm3heL2Kt+4ty4g4SbSlaEolEoFLBjM1ibcRLhyq2omxR4UNf5QPAvaVYMm/4YgjoZN9uyuGhRCIxiU4nasFdOiSKWl46BOlHhcMeROmlO/4Ogx6v9xBSJkxXQ4qWRGIjyorh8hHYuQTOlBf4DO4l1gEN7Wv1ZWUaj0QiaRicXUXq18PfwbhPROBv+lH49C6xxFxJw6b7SNGSSCTWoVJBxCSYd0DMNio6sSbmx4Pg7K4Gu60ULYlEUj+8WolyRpPXgXeoWBHpi/vh+/misKONkaIlkUhswy2jYN4+UesMIPE/sDwSjn9v09tI0ZJIJLbDzQfufQ9m/g/8O0FehqiJtnupzW4hRUsikdiedoPg0T0w5BnhqO/1oM0uLUMeJBJJw1KUIyywOpAhDxKJpHFghmBZghQtiUTSpJCiJZFImhRStCQSSZNCipZEImlSSNGSSCRNCilaEomkSdFsigDqw9FycmoufSSRSByP/rtZV+hosxGt3FyxCGVYWBNdeEAiaSbk5ubi6+tr8niziYjX6XSkpaXh7e1d54IEOTk5hIWFceHChSYdPX+zvA64eV6LfB2mURSF3NxcQkNDUatNe66ajaWlVqtp06aNRef4+Pg06X8sPTfL64Cb57XI12Gc2iwsPdIRL5FImhRStCQSSZNCipYRXF1dWbx4Ma6utS/j3di5WV4H3DyvRb6O+tNsHPESieTmQFpaEomkSSFFSyKRNCmkaEkkkiaFFC2JRNKkkKIlkUiaFFK0qrF8+XLCw8Nxc3MjMjKS/fv3O7pLFvPKK6+gUqmqbF27dnV0t+rk119/5b777iM0NBSVSsXGjRurHFcUhUWLFhESEoK7uzvR0dGcOnXKMZ2tg7pey4wZM2p8RqNGjXJMZ02wZMkSbr31Vry9vQkMDCQmJobk5OQqbYqKipg3bx7+/v54eXkxfvx4MjIyGrRfUrQqsW7dOmJjY1m8eDGHDx8mIiKCkSNHkpmZ6eiuWUyPHj24fPmyYdu9e7eju1Qn+fn5REREsHz5cqPH33rrLT788ENWrFjBvn378PT0ZOTIkRQVFdm5p3VT12sBGDVqVJXP6KuvvrJjD+tm165dzJs3j71797J161ZKS0sZMWIE+fn5hjYLFizghx9+4JtvvmHXrl2kpaXxwAMPNGzHFImBgQMHKvPmzTP8rdVqldDQUGXJkiUO7JXlLF68WImIiHB0N+oFoGzYsMHwt06nU4KDg5W3337b8NyNGzcUV1dX5auvvnJAD82n+mtRFEWZPn26MnbsWIf0x1oyMzMVQNm1a5eiKOL9d3FxUb755htDmxMnTiiAkpCQ0GD9kJZWOSUlJRw6dIjo6GjDc2q1mujoaBISEhzYM+s4deoUoaGhdOjQgSlTppCamuroLtWLlJQU0tPTq3w+vr6+REZGNsnPB2Dnzp0EBgZyyy238Nhjj3H16lVHd6lWsrOzAWjZsiUAhw4dorS0tMpn0rVrV9q2bdugn4kUrXKysrLQarUEBQVVeT4oKIj09HQH9co6IiMjWb16NVu2bOHjjz8mJSWFIUOGGGqKNUX0n8HN8PmAGBp+8cUXxMXF8eabb7Jr1y5Gjx6NVqt1dNeMotPpeOqpp7j99tvp2bMnID4TjUaDn59flbYN/Zk0m9I0zYnRo0cb9nv37k1kZCTt2rXj66+/ZtasWQ7smUTPpEmTDPu9evWid+/edOzYkZ07dzJ8+HAH9sw48+bN49ixY43CNyotrXICAgJwcnKqMfORkZFBcHCwg3plG/z8/OjSpQunT592dFesRv8Z3IyfD0CHDh0ICAholJ/R/Pnz+fHHH9mxY0eVmnTBwcGUlJRw48aNKu0b+jORolWORqOhf//+xMXFGZ7T6XTExcURFRXlwJ7Vn7y8PM6cOUNISIiju2I17du3Jzg4uMrnk5OTw759+5r85wNw8eJFrl692qg+I0VRmD9/Phs2bGD79u20b9++yvH+/fvj4uJS5TNJTk4mNTW1YT+TBnPxN0HWrl2ruLq6KqtXr1aOHz+uzJkzR/Hz81PS09Md3TWLePrpp5WdO3cqKSkpyp49e5To6GglICBAyczMdHTXaiU3N1dJTExUEhMTFUB57733lMTEROX8+fOKoijKG2+8ofj5+Snff/+9cuTIEWXs2LFK+/btlcLCQgf3vCa1vZbc3FzlmWeeURISEpSUlBRl27ZtSr9+/ZTOnTsrRUVFju66gccee0zx9fVVdu7cqVy+fNmwFRQUGNo8+uijStu2bZXt27crBw8eVKKiopSoqKgG7ZcUrWp89NFHStu2bRWNRqMMHDhQ2bt3r6O7ZDETJ05UQkJCFI1Go7Ru3VqZOHGicvr0aUd3q0527NihADW26dOnK4oiwh5efvllJSgoSHF1dVWGDx+uJCcnO7bTJqjttRQUFCgjRoxQWrVqpbi4uCjt2rVTZs+e3eh+HI31H1BWrVplaFNYWKj87W9/U1q0aKF4eHgo48aNUy5fvtyg/ZL1tCQSSZNC+rQkEkmTQoqWRCJpUkjRkkgkTQopWhKJpEkhRUsikTQppGhJJJImhRQtiUTSpJCiJZFImhRStCQSSZNCipZEImlSSNGSSCRNiv8P0NDj/guckxoAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:32:14.853193Z", + "iopub.status.busy": "2024-03-22T15:32:14.852886Z", + "iopub.status.idle": "2024-03-22T15:34:18.403745Z", + "shell.execute_reply": "2024-03-22T15:34:18.402880Z" + }, + "papermill": { + "duration": 123.573926, + "end_time": "2024-03-22T15:34:18.406323", + "exception": false, + "start_time": "2024-03-22T15:32:14.832397", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:34:18.447363Z", + "iopub.status.busy": "2024-03-22T15:34:18.446530Z", + "iopub.status.idle": "2024-03-22T15:34:18.468428Z", + "shell.execute_reply": "2024-03-22T15:34:18.467518Z" + }, + "papermill": { + "duration": 0.043772, + "end_time": "2024-03-22T15:34:18.470346", + "exception": false, + "start_time": "2024-03-22T15:34:18.426574", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.010057NaN0.0187376.4715580.0831290.994440.1368410.0000092.345920.0832663.0670640.1368830.0476310.864338.817478
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.010057 NaN 0.018737 6.471558 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.083129 0.99444 0.136841 0.000009 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 2.34592 0.083266 3.067064 0.136883 0.047631 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.86433 8.817478 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:34:18.509088Z", + "iopub.status.busy": "2024-03-22T15:34:18.508430Z", + "iopub.status.idle": "2024-03-22T15:34:18.869146Z", + "shell.execute_reply": "2024-03-22T15:34:18.868200Z" + }, + "papermill": { + "duration": 0.382188, + "end_time": "2024-03-22T15:34:18.871279", + "exception": false, + "start_time": "2024-03-22T15:34:18.489091", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:34:18.912473Z", + "iopub.status.busy": "2024-03-22T15:34:18.912179Z", + "iopub.status.idle": "2024-03-22T15:36:29.398389Z", + "shell.execute_reply": "2024-03-22T15:36:29.397576Z" + }, + "papermill": { + "duration": 130.509753, + "end_time": "2024-03-22T15:36:29.401031", + "exception": false, + "start_time": "2024-03-22T15:34:18.891278", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:36:29.442257Z", + "iopub.status.busy": "2024-03-22T15:36:29.441947Z", + "iopub.status.idle": "2024-03-22T15:36:29.468537Z", + "shell.execute_reply": "2024-03-22T15:36:29.467652Z" + }, + "papermill": { + "duration": 0.049389, + "end_time": "2024-03-22T15:36:29.470551", + "exception": false, + "start_time": "2024-03-22T15:36:29.421162", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:36:29.508490Z", + "iopub.status.busy": "2024-03-22T15:36:29.508204Z", + "iopub.status.idle": "2024-03-22T15:36:29.513557Z", + "shell.execute_reply": "2024-03-22T15:36:29.512690Z" + }, + "papermill": { + "duration": 0.026473, + "end_time": "2024-03-22T15:36:29.515492", + "exception": false, + "start_time": "2024-03-22T15:36:29.489019", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.05068456961917469}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:36:29.555333Z", + "iopub.status.busy": "2024-03-22T15:36:29.555062Z", + "iopub.status.idle": "2024-03-22T15:36:29.936809Z", + "shell.execute_reply": "2024-03-22T15:36:29.935892Z" + }, + "papermill": { + "duration": 0.404495, + "end_time": "2024-03-22T15:36:29.939133", + "exception": false, + "start_time": "2024-03-22T15:36:29.534638", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAE8CAYAAABU7q18AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHf0lEQVR4nO3dd3hUZdo/8O+ZPmmThJRJIAUIJFITQzEoBiQvAVkk6rIaCwkIuCu8qFl3FX+vVPeNjaYiuPsuiayCrruILiJlI2URROlFDSSEJEB6r1Of3x8nc5JJMqlTksn9ua65wpx5zpn7TDI3zzlP4xhjDIQQ4oREjg6AEEJshRIcIcRpUYIjhDgtSnCEEKdFCY4Q4rQowRFCnBYlOEKI06IERwhxWpTgCCFOixIcAQCEhobiV7/6lU3fg+M4rFmzptNya9asAcdxNo2FDAyU4JzEyZMnsWbNGlRWVjo6FNIPfPDBB0hPT3d0GDZHCc5JnDx5EmvXrqUER7qEEhwhhPRzlOCcwJo1a/CHP/wBADB06FBwHAeO43Dz5k2kpaXhgQcegJ+fH+RyOUaNGoVt27ZZPNahQ4cQGRkJhUKBUaNGYc+ePd2OR6PR4MUXX4Svry/c3d3x0EMP4datW+2WPXHiBCZOnAiFQoHhw4fjww8/bLccx3FYvnw5PvnkE4SHh0OhUCA6OhrHjx9v81lwHIdr167hqaeegkqlgq+vL1577TUwxpCfn4958+bBw8MDarUaGzZs6Pb5AcA333yD2NhYuLu7w8PDAxMnTsSuXbvMynz++eeIjo6GUqmEj48PnnrqKdy+fdusTHJyMtzc3HD79m0kJCTAzc0Nvr6+eOmll2AwGMzKGo1GbNmyBWPHjoVCoYCvry9mzZqFM2fOCGW68vsODQ3F1atXcezYMeFvZdq0aT36HPo8Rvq9ixcvssTERAaAbdq0if3tb39jf/vb31htbS2bOHEiS05OZps2bWLvvfcemzlzJgPA3n//fbNjhISEsJEjRzJPT0/2yiuvsI0bN7KxY8cykUjEDh061K14nnrqKQaAPfHEE+z9999njzzyCBs3bhwDwFavXi2Uu3TpElMqlSw4OJilpqay9evXM39/f6FsSwDYmDFjmI+PD1u3bh178803WUhICFMqlezy5ctCudWrVzMALDIykiUmJrIPPviAzZkzhwFgGzduZOHh4ex3v/sd++CDD9i9997LALBjx4516/zS0tIYx3FszJgx7E9/+hPbunUrW7x4MXv66afNygBgEydOZJs2bWKvvPIKUyqVLDQ0lFVUVAjlkpKSmEKhYKNHj2aLFi1i27ZtY48++igDwD744AOz901OTmYA2OzZs9nmzZvZO++8w+bNm8fee+89oUxXft9ffPEFGzJkCIuIiBD+Vrr7O+4vKME5ibfffpsBYDk5OWbb6+vr25SNj49nw4YNM9sWEhLCALB//vOfwraqqioWEBDAoqKiuhzHhQsXGAD23HPPmW1/4okn2iS4hIQEplAoWG5urrDtp59+YmKxuN0EB4CdOXNG2Jabm8sUCgV7+OGHhW2mBLd06VJhm16vZ0OGDGEcx7E33nhD2F5RUcGUSiVLSkrq8vlVVlYyd3d3NnnyZNbQ0GD2mtFoZIwxptVqmZ+fHxszZoxZmX379jEAbNWqVcK2pKQkBoCtW7fO7FhRUVEsOjpaeP7tt98yAGzFihVtYjK9L2Nd/32PHj2axcbGduGM+ze6RHVySqVS+HdVVRVKS0sRGxuLGzduoKqqyqxsYGAgHn74YeG5h4cHFixYgPPnz6OwsLBL77d//34AwIoVK8y2v/DCC2bPDQYDDh48iISEBAQHBwvb77rrLsTHx7d77JiYGERHRwvPg4ODMW/ePBw8eLDN5dzixYuFf4vFYkyYMAGMMTzzzDPCdk9PT4SHh+PGjRtdOjcAOHz4MGpqavDKK69AoVCYvWbq2nLmzBkUFxfjueeeMyszZ84cRERE4Ouvv25z3N/+9rdmz6dOnWoW1z//+U9wHIfVq1e32bdll5ru/L4HAkpwTu67775DXFwcXF1d4enpCV9fX7z66qsA0OYPPiwsrE3/s5EjRwIAbt682aX3y83NhUgkwvDhw822h4eHmz0vKSlBQ0MDRowY0eYYrcuatFd25MiRqK+vR0lJidn2lkkTAFQqFRQKBXx8fNpsr6iosHxCrWRnZwMAxowZY7FMbm4ugPbPIyIiQnjdxHQ/rSUvLy+zuLKzsxEYGAhvb+8O4+vO73sgkDg6AGI72dnZmDFjBiIiIrBx40YEBQVBJpNh//792LRpE4xGo6NDtBmxWNylbQDAHDxrv6W4umsg/74toQTnJNrr+f+vf/0LGo0GX331lVmN5siRI+0eIysrC4wxs2Ndu3YNAN/y1hUhISEwGo3Izs42q8FkZmaalfP19YVSqcT169fbHKN1WZP2yl67dg0uLi5takC2YqqZXrlyBWFhYe2WCQkJAcCfxwMPPGD2WmZmpvB6d9/34MGDKC8vt1iL687ve6CMFKFLVCfh6uoKAGYdfU01g5Y1lKqqKqSlpbV7jDt37uCLL74QnldXV2Pnzp2IjIyEWq3uUhyzZ88GALz77rtm2zdv3mz2XCwWIz4+Hnv37kVeXp6w/eeff8bBgwfbPfapU6dw7tw54Xl+fj6+/PJLzJw502q1oM7MnDkT7u7uSE1NRWNjo9lrps95woQJ8PPzw/bt26HRaITXv/nmG/z888+YM2dOt9/30UcfBWMMa9eubfOa6X278/t2dXUdEJ3CqQbnJEw33//f//t/ePzxxyGVSnH//fdDJpNh7ty5ePbZZ1FbW4u//OUv8PPzQ0FBQZtjjBw5Es888wx+/PFH+Pv7Y8eOHSgqKrKYENsTGRmJxMREfPDBB6iqqsKUKVOQkZGBrKysNmXXrl2LAwcOYOrUqXjuueeg1+vx3nvvYfTo0bh06VKb8mPGjEF8fDxWrFgBuVyODz74QDiOvXh4eGDTpk1YvHgxJk6ciCeeeAJeXl64ePEi6uvr8dFHH0EqleLNN9/EwoULERsbi8TERBQVFWHLli0IDQ3Fiy++2O33nT59Op5++mm8++67uH79OmbNmgWj0Yj//Oc/mD59OpYvX46ZM2d2+fcdHR2Nbdu24fXXX0dYWBj8/Pza1DadguMacIm1rV+/ng0ePJiJRCKhy8hXX33Fxo0bxxQKBQsNDWVvvvkm27FjR5suJSEhIWzOnDns4MGDbNy4cUwul7OIiAj2+eefdzuOhoYGtmLFCjZo0CDm6urK5s6dy/Lz89t0E2GMsWPHjrHo6Ggmk8nYsGHD2Pbt24WuHi0BYMuWLWMff/wxGzFiBJPL5SwqKoodOXLErJxp35KSErPtSUlJzNXVtU2ssbGxbPTo0d0+x6+++opNmTKFKZVK5uHhwSZNmsR2795tVuazzz5jUVFRTC6XM29vb/bkk0+yW7dudSmu9j4DvV7P3n77bRYREcFkMhnz9fVls2fPZmfPnjWLqyu/78LCQjZnzhzm7u7OADhtlxGOMVoXlfR9HMdh2bJleP/99x0dCulH6B4cIcRp0T040mWddfZVKpVQqVR2isb6SkpK2nQYbkkmk3XaD430LZTgSJcFBAR0+HpSUlK/noJn4sSJbTrhthQbG4ujR4/aLyDSa5TgSJcdPny4w9cDAwNt9t72uFX8ySefoKGhweLrXl5eNo+BWBc1MhBCnBY1MhBCnBZdorbDaDTizp07cHd3HzBDWgjpTxhjqKmpQWBgIEQiy/U0SnDtuHPnDoKCghwdBiGkE/n5+RgyZIjF1ynBtcPd3R0A/+F5eHg4OBpCSGvV1dUICgoSvquWUIJrh+my1MPDgxIcIX1YZ7eQqJGBEOK0KMERQpwWJThCiNOie3A9xBiDXq/vcOwiaZ9YLIZEIqEuOMTmKMH1gFarRUFBAerr6x0dSr/l4uKCgIAAyGQyR4dCnBgluG4yGo3IycmBWCxGYGAgZDIZ1US6gTEGrVaLkpIS5OTkYMSIER121CSkNyjBdZNWq4XRaERQUBBcXFwcHU6/pFQqIZVKkZubC61W22Z90YHiVkU9frxZjqkjfOHjJnd0OE6J/uvsIap19A59fsD3N8pxs7Qep2+UOzoUp+XQv7Ljx49j7ty5CAwMBMdx2Lt3r9nrycnJ4DjO7DFr1qxOj7t161aEhoZCoVBg8uTJ+OGHH2x0BoT0XH45fw/3enGNgyNxXg5NcHV1dRg/fjy2bt1qscysWbNQUFAgPHbv3t3hMT/77DOkpKRg9erVOHfuHMaPH4/4+HgUFxdbO3xCekUm4b9+D4233Tx6A51DE9zs2bPx+uuv4+GHH7ZYRi6XQ61WC4/OJh3cuHEjlixZgoULF2LUqFHYvn07XFxcsGPHDmuHTzoQGhraZi1UYk4s4hunVEqpgyNxXn3+RsjRo0fh5+eH8PBw/O53v0NZWZnFslqtFmfPnkVcXJywTSQSIS4uDqdOnbK4n0ajQXV1tdmDEFvTG4wAAIm4z38N+60+/cnOmjULO3fuREZGBt58800cO3YMs2fPtti5trS0FAaDAf7+/mbb/f39O1wwJTU1FSqVSnjQVEk8rVbr6BCcFmMMOgM/mfa5vAoHR+O8+nSCe/zxx/HQQw9h7NixSEhIwL59+/Djjz9afeGPlStXoqqqSnjk5+f36DhavdHiw/S/dVfK6rpQtiemTZuG5cuXY/ny5VCpVPDx8cFrr70mrHcQGhqK9evXY8GCBfDw8MDSpUsBACdOnMDUqVOhVCoRFBSEFStWoK6uTjhucXEx5s6dC6VSiaFDh+KTTz7pUXwDCcdxeDomBABwrZAaGWylX/WDGzZsGHx8fJCVlYUZM2a0ed3HxwdisRhFRUVm24uKiqBWqy0eVy6XQy7vfT+krUeyLL421McVCVGDhed/Pp4t/A/e2hAvJeZPaK5F7vguBw1a81rri/81skcxfvTRR3jmmWfwww8/4MyZM1i6dCmCg4OxZMkSAMA777yDVatWYfXq1QCA7OxszJo1C6+//jp27NiBkpISIUmmpaUB4Fu779y5gyNHjkAqlWLFihXUqNMFkqZ7cHojLYtiK/0qwd26dQtlZWUWl6+TyWSIjo5GRkYGEhISAPAjDzIyMrB8+XI7Rtp3BQUFYdOmTeA4DuHh4bh8+TI2bdokJLgHHngAv//974XyixcvxpNPPokXXngBADBixAi8++67iI2NxbZt25CXl4dvvvkGP/zwAyZOnAgA+Otf/4q77rrL7ufW35gaGfQW/qMjvefQBFdbW4usrOZaT05ODi5cuABvb294e3tj7dq1ePTRR6FWq5GdnY0//vGPCAsLQ3x8vLDPjBkz8PDDDwsJLCUlBUlJSZgwYQImTZqEzZs3o66uDgsXLrT5+SybHmbxNVGr0VxL7x9usWzrkV+L7h3am7DM3HPPPWZDy2JiYrBhwwbhvuaECRPMyl+8eBGXLl0yu+xkjAlD1q5duwaJRILo6Gjh9YiICHh6elotZmdU1aDDVxfvAACMjMFgZELCI9bj0AR35swZTJ8+XXiekpICgF9AeNu2bbh06RI++ugjVFZWIjAwEDNnzsT69evNLiezs7NRWloqPH/sscdQUlKCVatWobCwEJGRkThw4ECbhgdbMPVrcmTZ3nJ1dTV7Xltbi2effRYrVqxoUzY4OBjXrl2zV2hORaMzoLRGIzzXG40Qi8QOjMg5OTTBTZs2rcMFfQ8ePNjpMW7evNlmm+keEWnr9OnTZs+///57jBgxAmJx+1+uu+++Gz/99BPCwtqvnUZERECv1+Ps2bPCJWpmZiYqKyutGrez0bW676Y3MMj71Q2j/qFPt6IS68vLy0NKSgoyMzOxe/duvPfee3j++ectln/55Zdx8uRJLF++HBcuXMD169fx5ZdfCv+BhIeHY9asWXj22Wdx+vRpnD17FosXL4ZSqbTXKfVLrVvVjbT+uk1QghtgFixYgIaGBkyaNAnLli3D888/L3QHac+4ceNw7NgxXLt2DVOnTkVUVBRWrVqFwMDm4UVpaWkIDAxEbGwsHnnkESxduhR+fn72OJ1+y9RyKhZxeOqeELjIqPpmC/SpDjBSqRSbN2/Gtm3b2rzW3uU+AEycOBGHDh2yeEy1Wo19+/aZbXv66ad7FaezM92aUXso4OtOUyXZCtXgCHEA0y04mivVtijBEeIAjPGXp7cqGvD9jTLUavSODskp0SXqAGLtIW6k58LV7ghXu+OvJ3JwKrsMoYNc4UbNqFZHNThCHEjcdIlqoFZUm6AER4gDiZpGLxhpPKpNUJ2YEAfILqnFldtVKKvlp6SifnC2QTU4Qhygsl6HGyXNU04ZqAZnE5TgCHGA1kMUqQZnG5TgCHGA1hU2qsDZBt2DI8QBTDW2wZ5KTB3pA0+lzMEROSdKcIQ4gKnV1NddjgAVTUxgK3SJSogDmC5JRTTJpU1RgrMGxgC91v6PbtyY3rlzJwYNGgSNRmO2PSEhgQbGO4DpEjWvrA5nc8tRVN3o4IicE12iWoNBB/xng/3fd+rvAUnX7t3Mnz8fK1aswFdffYX58+cD4FfD+vrrrzucKYTYxtQRPrgvzAffXCnE8WuluH+kL/w9FI4Oy+lQDW6AUCqVeOKJJ4SVsADg448/RnBwMKZNm+a4wAYojuMgEnGQNI3V6mhma9JzVIOzBrGUr0054n27YcmSJZg4cSJu376NwYMHIz09HcnJyWaL0BD7Ejd99tTR1zYowVkDx3X5UtGRoqKiMH78eOzcuRMzZ87E1atX8fXXXzs6rAHpYn4lblc24E5lAwDqB2crlOAGmMWLF2Pz5s24ffs24uLiEBQU1PlOxOoKqhqR2WJFewbKcLZA9+AGmCeeeAK3bt3CX/7yFyxatMjR4QxYre+50S0426AEN8CoVCo8+uijcHNzQ0JCgqPDGbBM879JTNMlUYazCYcmuOPHj2Pu3LkIDAwEx3HYu3ev8JpOp8PLL7+MsWPHwtXVFYGBgViwYAHu3LnT4THXrFkDjuPMHhERETY+k/7l9u3bePLJJ80W0Cb2ZbrnNi7IE4/ePQRjB6scG5CTcmiCq6urw/jx47F169Y2r9XX1+PcuXN47bXXcO7cOezZsweZmZl46KGHOj3u6NGjUVBQIDxOnDhhi/D7nYqKCnzxxRc4evQoli1b5uhwBjTTJeogVxmCB7nA06XvN1L1Rw5tZJg9ezZmz57d7msqlQqHDx822/b+++9j0qRJyMvLQ3BwsMXjSiQSqNVqq8bqDKKiolBRUYE333wT4eHhjg5nQDNdklIPHdvqV62oVVVV4DgOnp6eHZa7fv06AgMDoVAoEBMTg9TU1A4TokajMRvCVF1dba2Q+xRL654S+zM2LWxfXqfFpVuV8HKRIcjbxbFBOaF+08jQ2NiIl19+GYmJifDw8LBYbvLkyUhPT8eBAwewbds25OTkYOrUqaipqbG4T2pqKlQqlfCgrhPE1h6KDMTvpg2HRCRCxs/F+LnAOf9TdbR+keB0Oh1+85vfgDHW7orsLc2ePRvz58/HuHHjEB8fj/3796OyshJ///vfLe6zcuVKVFVVCY/8/PxOY6KhNb0z0D8/qVgEhVQMqdjUiurggJxUn79ENSW33NxcfPvttx3W3trj6emJkSNHIisry2IZuVze5RZFqZQfHlVfXw+lkubx6qn6+noAzZ/nQGUaJjfQE76t9OkEZ0pu169fx5EjRzBo0KBuH6O2thbZ2dlWmxJILBbD09MTxcXFAAAXFxcay9kNjDHU19ejuLgYnp6eEIvFjg7JIU5ml6JOY4BpOjhKb7bh0ARXW1trVrPKycnBhQsX4O3tjYCAAPz617/GuXPnsG/fPhgMBhQWFgIAvL29IZPxzeozZszAww8/jOXLlwMAXnrpJcydOxchISG4c+cOVq9eDbFYjMTERKvFbWqhNSU50n2enp4DuqU7u6QOpTUahPm5AaCOvrbi0AR35swZTJ8+XXiekpICAEhKSsKaNWvw1VdfAQAiIyPN9jty5IgwxU92djZKS0uF127duoXExESUlZXB19cX9913H77//nv4+vpaLW6O4xAQEAA/Pz/odDqrHXegkEqlA7bmZmK6JKV7cLbl0AQ3bdq0Du89dOW+ROuuD59++mlvw+oysVg84L+opGdM0yOJ6B6cTfXpe3CEOCtTjS3UxxXD/dzgKqOvoi3Qp0qIA5hqbJ5KKfxoqnKb6Rf94AhxNs1DtagF3pYowRHiAKZL1HqtHj8XVCOntM6xATkpukQlxAEWxITAyICCygYcuFKIAJUCQ31cHR2W06EER4gDuDQ1KohF1E3ElugSlRAHErqJ0FgGm6AaHCEO8O0vReA4DkFe/BRJVIOzDUpwhNiZ0chwMb8KABDkxU/YQB19bYMuUQmxs5bjTsUi/itopCqcTVCCI8TOWuYyCTUy2BRdohJiZy1rcF6uMsweq4ZMTHUNW6AER4idtbzd5ioTI0LdvUlcSdfRfxuE2JmhxYpaNFTLtqgGR4idmS5RRRwHrd6IvPI6AJww+SWxHkpwhNiZm0yCRfcOhZEx1Gn0+NfFAsgkIoT5hTk6NKdDCY4QOxOJOKhc+MV2qur5GaGpH5xt0D04QhyIa/oGUn6zDarBEWJntRo9zudVQCEV464AvgWV+sHZBtXgCLGzOo0eZ25W4GJ+pbBsoJExuky1AUpwhNhZy1ZUUYtuIpTfrI8SHCF2ZrocFXF8X7jm7ZThrI3uwRFiZ6aB9SIRB4lIhLi7/MFxMKvNEetwaA3u+PHjmDt3LgIDA8FxHPbu3Wv2OmMMq1atQkBAAJRKJeLi4nD9+vVOj7t161aEhoZCoVBg8uTJ+OGHH2x0BoR0X8sFZ8QiDmOHqDBmsAoiESU4a3Nogqurq8P48eOxdevWdl9/66238O6772L79u04ffo0XF1dER8fj8bGRovH/Oyzz5CSkoLVq1fj3LlzGD9+POLj41FcXGyr0yCkW0yXqGKqsdke6yMAsC+++EJ4bjQamVqtZm+//bawrbKyksnlcrZ7926Lx5k0aRJbtmyZ8NxgMLDAwECWmpra5ViqqqoYAFZVVdW9kyCkC7KKa9jGQ5ls9+lcxhhjN0tr2Y2SWqbTGxwcWf/R1e9on21kyMnJQWFhIeLi4oRtKpUKkydPxqlTp9rdR6vV4uzZs2b7iEQixMXFWdwHADQaDaqrq80ehNjKEC8lno4JQfxoNQBg7/k72Hv+Nhp0BgdH5nz6bIIrLCwEAPj7+5tt9/f3F15rrbS0FAaDoVv7AEBqaipUKpXwCAoK6mX0hFgml4jh4yaHl6sMAIS+cNSGan19NsHZ08qVK1FVVSU88vPzHR0SGUBMjQvM6OBAnFCf7SaiVvPV96KiIgQEBAjbi4qKEBkZ2e4+Pj4+EIvFKCoqMtteVFQkHK89crkccrm890ET0gXF1Y3ILqmDt6sM4Wp3oS8c9YOzvj5bgxs6dCjUajUyMjKEbdXV1Th9+jRiYmLa3UcmkyE6OtpsH6PRiIyMDIv7EGJvxTUafH+jDL8U8vd6Tf3fDJTgrM6hNbja2lpkZWUJz3NycnDhwgV4e3sjODgYL7zwAl5//XWMGDECQ4cOxWuvvYbAwEAkJCQI+8yYMQMPP/wwli9fDgBISUlBUlISJkyYgEmTJmHz5s2oq6vDwoUL7X16hLTLYGweqsX/5LdTfrM+hya4M2fOYPr06cLzlJQUAEBSUhLS09Pxxz/+EXV1dVi6dCkqKytx33334cCBA1AoFMI+2dnZKC0tFZ4/9thjKCkpwapVq1BYWIjIyEgcOHCgTcMDIY7Scixqy5+MMpzVcYw+1Taqq6uhUqlQVVUFDw9aEIRY19ncChy/VoK7Atwxa0wArtyugt7IMNLfDS6yPntbvE/p6neUPk1C7Iy1GKoFAGMGqxwZjlPrs40MhDir5tlEaKiWrfUowd24ccPacRAyYDQ3MvDPi6sbkV9ej0YayWB1PUpwYWFhmD59Oj7++OMOB74TQtoaM9gDj08KwoQQbwDAoZ+K8I+zt1BUTd8la+tRgjt37hzGjRuHlJQUqNVqPPvsszQlESFd5K6QIkClFFbWMl2q0roM1tejBBcZGYktW7bgzp072LFjBwoKCnDfffdhzJgx2LhxI0pKSqwdJyFOS0QjGWymV40MEokEjzzyCD7//HO8+eabyMrKwksvvYSgoCAsWLAABQUF1oqTEKeRV1aPs7nluF3ZAID6wdlSrxLcmTNn8NxzzyEgIAAbN27ESy+9hOzsbBw+fBh37tzBvHnzrBUnIU4jq6QGx6+VIresDgBajEV1YFBOqkf94DZu3Ii0tDRkZmbiwQcfxM6dO/Hggw9CJOLz5dChQ5Geno7Q0FBrxkqIUzA2zRrSeiQDXaJaX48S3LZt27Bo0SIkJyebzfTRkp+fH/7617/2KjhCnFGboVpN11FGmi7J6nqU4A4fPozg4GChxmbCGEN+fj6Cg4Mhk8mQlJRklSAJcSbCmgxNX59RASoM8XKBrztN2WVtPUpww4cPR0FBAfz8/My2l5eXY+jQoTAYqMMiIZYYWw3VCle7OzIcp9ajRgZLrT21tbVmM30QQtpqfYlKbKdbNTjTdEYcx2HVqlVwcXERXjMYDDh9+rTF2XYJIbyWK9sDQFWDDo06A9zkErjKaf4La+rWp3n+/HkAfA3u8uXLkMlkwmsymQzjx4/HSy+9ZN0ICXEyU4YPQlSQJzybRjKcyi7FzwU1uH+kL6JDvBwcnXPpVoI7cuQIAGDhwoXYsmULzZVGSA/4uJk3JnDU0ddmelQfTktLs3YchAxYNBbVdrqc4B555BGkp6fDw8MDjzzySIdl9+zZ0+vACHFWWcU1qNMYEDrIFSoXKY1FtaEuJziVSiVUpVUqmoGUkJ46m1uBO5WNmDs+oCnB0UgGW+lygmt5WUqXqIT0XOsZfTlaVctmetQPrqGhAfX19cLz3NxcbN68GYcOHbJaYIQ4K0uralENzvp6lODmzZuHnTt3AgAqKysxadIkbNiwAfPmzcO2bdusGiAhzqZ1DS7Y2wWTh3kj2Nulg71IT/R4Rt+pU6cCAP7xj39ArVYjNzcXO3fuxLvvvmvVAAlxNs2ravHPQ31cMWW4D0IGuTowKufUowRXX18Pd3d+/NyhQ4fwyCOPQCQS4Z577kFubq5VAyTE2QiLzohoqJat9XjRmb179yI/Px8HDx7EzJkzAQDFxcVW7/wbGhoKjuPaPJYtW9Zu+fT09DZlaXws6UuE2USaqnANWgPKajWoadQ5MCrn1KOOvqtWrcITTzyBF198ETNmzEBMTAwAvjYXFRVl1QB//PFHs9lJrly5gv/6r//C/PnzLe7j4eGBzMxM4TlHg5pJHzJzlD+0BqMwVOvKnSqcuF6K0YEemDla7eDonEuPEtyvf/1r3HfffSgoKMD48eOF7TNmzMDDDz9steAAwNfX1+z5G2+8geHDhyM2NtbiPhzHQa2mPxTSNwW1akwQ0ZTlNtPjqQvUanWbJDJp0qReB9QRrVaLjz/+GCkpKR3WympraxESEgKj0Yi7774b//u//4vRo0dbLK/RaKDRaITn1dXVVo2bkI7QWFTb6VGCq6urwxtvvIGMjAwUFxfD2GquZVutfL93715UVlYiOTnZYpnw8HDs2LED48aNQ1VVFd555x1MmTIFV69exZAhQ9rdJzU1FWvXrrVJzIS09nNBNRgDhvu5Qi4R01hUG+JYD/7bSExMxLFjx/D0008jICCgTW3q+eeft1qALcXHx0Mmk+Ff//pXl/fR6XS46667kJiYiPXr17dbpr0aXFBQEKqqqmjGFGJ17397HToDw6L7hkKllOLSrUpk/FyMMD83zB0f6Ojw+oXq6mqoVKpOv6M9qsF98803+Prrr3Hvvff2OMDuys3Nxb///e9uD+SXSqWIiopCVlaWxTJyuRxyOc2HT+yj9YSXNJLBdnrUTcTLywve3t7WjqVDaWlp8PPzw5w5c7q1n8FgwOXLly2u/kWIvbUeqkVjUW2nRwlu/fr1WLVqldl4VFsyGo1IS0tDUlISJBLzSueCBQuwcuVK4fm6detw6NAh3LhxA+fOncNTTz2F3NxcLF682C6xEtIRo5EJicyU4Hzc5IgO8UKYn5sDI3NOPbpE3bBhA7Kzs+Hv74/Q0FBIpVKz18+dO2eV4Ez+/e9/Iy8vD4sWLWrzWl5entnyhRUVFViyZAkKCwvh5eWF6OhonDx5EqNGjbJqTIT0RMvLUNOfrb+HAv4e1BndFnqU4BISEqwcRsdmzpxpsQn96NGjZs83bdqETZs22SEqQrqvZUspraplez1KcKtXr7Z2HIQMCGY1uKYEpzMY0aAzgAPgrpBa2JP0RI/uwQH8NEn/93//h5UrV6K8vBwAf2l6+/ZtqwVHiLORikX41bgAPDg2QGhFzS+vx1//k4N9lwocG5wT6lEN7tKlS4iLi4NKpcLNmzexZMkSeHt7Y8+ePcjLyxPmiiOEmBOLOIzwN1/JnrqJ2E6PanApKSlITk7G9evXzWbqePDBB3H8+HGrBUfIQEAjGWynRzW4H3/8ER9++GGb7YMHD0ZhYWGvgyLEWWn0BtwsrYdEzGG4L98tpLkfHGU4a+tRDU4ul7c7IP3atWttZv8ghDSr1xiw/3IBDl5trgiYJr40UhXO6nqU4B566CGsW7cOOh0/QR/HccjLy8PLL7+MRx991KoBEuJMDK1GMfD/5n9SfrO+HiW4DRs2oLa2Fr6+vmhoaEBsbCzCwsLg7u6OP/3pT9aOkRCnYWpIEJslOGpksJUe3YNTqVQ4fPgwvvvuO1y8eBG1tbW4++67ERcXZ+34CHEqphzWso+vUibGuCEqKKVixwTlxLqd4IxGI9LT07Fnzx7cvHkTHMdh6NChUKvVYIzR9OCEdKD1QHsA8FBIMeMu/073/f5GGXQGI+4L86HvWRd16xKVMYaHHnoIixcvxu3btzF27FiMHj0aubm5SE5Otvp05YQ4G2FFrW7mp4o6LU5ll+HMzQqU1Gg634EA6GYNLj09HcePH0dGRgamT59u9tq3336LhIQE7Ny5EwsWLLBqkIQ4C2EmkRYZzmhk0OiNYGBwkbX/laysrsKg+mxUygejrE4LPxqc3yXdqsHt3r0br776apvkBgAPPPAAXnnlFXzyySdWC44QZ+PlKsPM0f6YMnyQsK1Go8f2Y9nYcSKn/Z2MRkiv/gMjSjMwungf6jV6O0Xb/3UrwV26dAmzZs2y+Prs2bNx8eLFXgdFiLNyk0swOlCFML/m4VqddhOpvIkAUQU8lVK46MqB8mzbB+okupXgysvL4e9v+Waov78/Kioqeh0UIQNJp91EyrIh5ji4KyTgAEirbtottv6uW/fgDAZDmxl1WxKLxdDrqfpMiCW1Gj2KqxuhlIkRoFICaE5wjKH9ngjl/KVrQPgEDC6+Cs6NlrXsqm4lOMYYkpOTLS7Q0nJlKkJIW7crGrD/cgGCvF3w62h+GcuW+czIAHHL/KbXAPVlKK5pRP6g4YjQXoJrfRlgNAAi6jfXmW4luKSkpE7LUAsqIZY194Nr3tayT5yRMYjR4sXaYgDAnUYpjhdKEWIQw9VoABoqAFcfu8Tcn3UrwaWlpdkqDkIGhPY6+oq4tq8Laov4HxJvgONwqUKKoXWVGFpXQgmuC3o0VIsQ0jPtDdUScRzuCnAHx3Hg0Or+W1OCq5HyyaxWrEJlQwlQX26PcPs9SnCE2FHzSIYWNTgRh1ljLKzbW1cKAKgRewIANBJ3GBsZ0Fhl0zidRY/XZCCEdF97l6gdauC7XdWK+H5zGrEbDAyAhlpSu4ISHCF2ZOrM23osqsHIoNEbzCe91DUAugYwxlDDeQBoqsEZGVhDpX0C7uf6dIJbs2YNf1+ixSMiIqLDfT7//HNERERAoVBg7Nix2L9/v52iJaRzQd5KPBDhh1GBHmbb/3z8Bj44ko3KBl3zxqbam17qCqOIX05QI3EHA2BoqGq+oUcs6tMJDgBGjx6NgoIC4XHixAmLZU+ePInExEQ888wzOH/+PBISEpCQkIArV67YMWJCLPNzV2B8kCdCBrmabRc3fRMNLWtwTQ0JYldvPB0Tgt9MDIJG7AoGDsygB7S19gq73+rzCU4ikUCtVgsPHx/LTeNbtmzBrFmz8Ic//AF33XUX1q9fj7vvvhvvv/++HSMmpPtM9+TMElxTDU7kMgg+bnIM9lRCJBJDK3Hl7+VRQ0On+nyCu379OgIDAzFs2DA8+eSTyMvLs1j21KlTbWYVjo+Px6lTpzp8D41Gg+rqarMHIbZQWa9Ffnk9qup1ZtvFTTflDC0vOxuauoIovYRNz00fjtixwyCXiAFNjc3j7e/6dIKbPHky0tPTceDAAWzbtg05OTmYOnUqamra/8UWFha2mQzA39+/06UMU1NToVKphEdQUJDVzoGQli7frsI/zt7CxVuVZtvF7a2s1VSDq+LccSq7DD/dqYZULAIn45cbhLbOHiH3a306wc2ePRvz58/HuHHjEB8fj/3796OyshJ///vfrfo+K1euRFVVlfDIz8+36vEJMWluRTVvRhVqcKYCjAn34MqNLvj+Rhku367kX5M3TbVE9+A61a86+np6emLkyJHIyspq93W1Wo2ioiKzbUVFRVCr1R0eVy6XW5xAgBBrMlqYsty0ypZwiapr4AfaA2iQqACUQSoW4cT1UojyGxGp08NFQwmuM326BtdabW0tsrOzERDQfq/vmJgYZGRkmG07fPgwYmJi7BEeIZ0SOvq2ynBB3i4Y6e8OV9OU5Y2V/E+5G7SMnzVEJhHhZlkdblQBWoORLlG7oE/X4F566SXMnTsXISEhuHPnDlavXg2xWIzExEQA/MwlgwcPRmpqKgDg+eefR2xsLDZs2IA5c+bg008/xZkzZ/DnP//ZkadBiKC9oVoAcG9Yq94BTfffoPSCzmAEAEjFIkhEHOrFLvylrpYaGTrTpxPcrVu3kJiYiLKyMvj6+uK+++7D999/D19fXwBAXl4eRKLmSuiUKVOwa9cu/M///A9effVVjBgxAnv37sWYMWMcdQqEmDHdYhN3du3UIsFp9XyCk0lEEIs46MQu/KUu1eA61acT3Kefftrh60ePHm2zbf78+Zg/f76NIiKkd1jTJWp765oyxsBY0+WraSiWwhNaTVOCE4sgEXPQil1gNDBAW08TX3aiTyc4QpxNuNodvu58p92WvrlcgF8KazAt3BdRwV7mNbi6ljU4EfQiBYym2+faOkBhPuyLNKMER4gdDfN1wzDfttu51gvPtEhw9wwbhLGDVXBTSFBaowE4DnqJC4Cm4VqU4CyiBEdIH9DcDw6AXtt8f03pCZVUCpVSalZOJ3EBUE334TpBCY4QOyqt1UBvYPB0kUIhbb53ZjbY3tRFRKoApOaXstPC/RAb7gvpTxeBsmrq7NuJftUPjpD+7mhmCXb/kIfcsnqz7WZro7a4PAWAS7cqcTa3ArUaPWQSEeQSMURyGq7VFVSDI8SOLI5kaNqgN7ZIcApPAMAPOeWoadRjsKcSbvKmr6ysabolSnAdogRHiB1ZGslgGqplNLLmLiJNNTidgd9HKuZws7QO14pqMEzDEAZQgusEXaISYkcGC2syeLnKMNTHFd6uMqC+jN/o4g3GmFlH37I6Da7eqUZBY4tuIsQiqsERYkfCSIZWCe6uAA/cFdDU3SOPX0kLLj4wGJlQ65OK+X5wAKARNTU+UILrENXgCLGj5pEMFgpo6/kHALgM4gfVN5E1jUUFAK2Q4KgVtSOU4AixI2GwfetWhiasvqn2pvAAJDLh8lQq5iAScZCI+f0auaYEZ9Dx/eZIu+gSlRA7igr2QoPWAHeF+VfvWlENDl0txGjuBqYDgAs/u4ipBieT8HURUw1OBwkglgCmxWck3nY7h/6EEhwhdhQZ5Nnudg58a6lYWw64AHAdBADwVMrwm4lBwqWt6R6c3sgAmRvf4qqtA1wowbWHEhwhfYC0aSiDuLGUT3BNNTiZRGQ2MF/ScmpzuSuf4HT1rQ9HmlCCI8SOSmv5aci9XGRC514A/L01xiBvKAGgBNz8290/QKXAkvuH8Yku09TZlxoaLKFGBkLs6LMf8/G3U7moaTRfNlAmFkFuqIHI0MjP7+bKTzlSXN2Ic3kVyGsa2iURi+Aml/DjWGl1rU5RgiPEjowWWlElYhFctaX8paerD9+AACC/oh7HMkvwU0E7izzTcK1O0SUqIXZkaSSDRMzBTVvCd+p1b15UqVHHt6LKm2YeadQZcOpGGYxGhhkeLnwhSnAWUQ2OEDsxTUkOtB1sLxOLECwug4dSCqN7oLC9UWcAACgkfIIzMoYLeZW4dKsKTEb34DpDNThC7MTQYtX61jU4BWfARG8NwDwAr2Bhu6kGp5Ca+sE110n0EhdIgeaRD6QNqsERYict8lubBIfq2wAz8iMYmqZJAlrU4JouUSUtqn4GSYt7cKzFwYmAEhwhdmJskYTEra9Ry2/wPz2D0TJVNerNE5xIxAnJUSdSNB1YD+g1Nom5v6NLVELsRMRxmDTUG3ojM78HxxhQeh1XblfhYq0MMwI1UKv45NX6EhXgGyS0egYDJwEksuY1HKQKe55Ov9Cna3CpqamYOHEi3N3d4efnh4SEBGRmZna4T3p6OjiOM3soFPSLJ44nk4hwb5gPYkf6mq+LWlcKNFRAz4lRKhsiXJYCwJyxAZgXGQgvF5mwTdJy9l+hLxw1NLSnTye4Y8eOYdmyZfj+++9x+PBh6HQ6zJw5E3V1HTeLe3h4oKCgQHjk5ubaKWJCeqDkZwCA1i0IRpEUDS0SnFqlwDBft1YL1DQlOAOjvnCd6NOXqAcOHDB7np6eDj8/P5w9exb333+/xf04joNarbZ1eIR0i95gRE2jHhIxB3cFvwwgjEag4BIAoHHQXUADzBJce+ZPCALHAa4yCSW4TvTpGlxrVVV8b25v745nTqitrUVISAiCgoIwb948XL16tcPyGo0G1dXVZg9CrK28Tov0kzfx2Y/5zRsrcgBNDSBVwOg9AgDQqDUI5c/mluNmqXnyUiml8FBI+Zqc6RJVRwmuPf0mwRmNRrzwwgu49957MWbMGIvlwsPDsWPHDnz55Zf4+OOPYTQaMWXKFNy6dcviPqmpqVCpVMIjKCjIFqdABji9sZ1RDAUX+Z/+YyCXywE01+BuVdTj+LVSXMivtHxQqsF1qN8kuGXLluHKlSv49NNPOywXExODBQsWIDIyErGxsdizZw98fX3x4YcfWtxn5cqVqKqqEh75+fkWyxLSU6aOvqZZeaGtB8qy+H8HjBcmwaxuGohfXsfP1OvtKjM7zpXbVTiaWYyi6kZASsO1OtKn78GZLF++HPv27cPx48cxZMiQbu0rlUoRFRWFrKwsi2XkcrnwvychtmJoXYMr/gkwGgB3NeDmB299A9QqBQa58n+LJTV837bWCS6ruBY5pXXwdZfDn1pRO9Sna3CMMSxfvhxffPEFvv32WwwdOrTbxzAYDLh8+TICAgI6L0yIDRmEWXmbElwh37gA9TgAQKCnEomTgnH/SF/oDEYUVjUK21uiVtSu69M1uGXLlmHXrl348ssv4e7ujsLCQgCASqWCUsn/0hcsWIDBgwcjNTUVALBu3Trcc889CAsLQ2VlJd5++23k5uZi8eLFDjsPQoDmqZLEIg6oLQZqivi53/zualP2Qn4l9EYGlVIKLxep2Wvm/eBMCa6e7zBscbmugalPJ7ht27YBAKZNm2a2PS0tDcnJyQCAvLw8iFoMQK6oqMCSJUtQWFgILy8vREdH4+TJkxg1apS9wiakXaZGBjHHNdfeBg0HZC5m5bR6I37IKQcARId4mXcKBj93HNB0yWtKcMwI6BraHGug69MJjnVhAPHRo0fNnm/atAmbNm2yUUSE9JyXiwyRwZ7wUoiBW01dl5ouT1u6VlQDvYEhMtgT44ao2rzeXIMz8jVAqQLQNfKXqZTgzPTpBEeIM1GrFPwY09Lr/CWlzAXwHtam3JjBKowK8LC4dqrZPTiA7wuna2xqaPC1Vfj9Up9uZCDEKZkuT/1H8zWwdlhKbkCrlbUAamjoANXgCLETjd4AXWMdFCVZ/BevncvTrhgf5IlwtTuUsqbkSAnOIkpwhNjJ+bxK5Fw4hsn6agwbNhxw8+vRcVzlErjKW3x1TX3hNDVWiNK50CUqIXZiNDL41l3jW0V7WHtrl9KT/9lYab1jOglKcITYS10JXLWl4MRiwK/n3ZZKajQ4mV2KK7eblhI0TXFOCa4NSnCE2ImyjO8aolUN61V3jsp6LU7fKMdPBU2z3iiaupI0VtHaDK1QgiPEHowGKCt+AQBofHrX6Vza1NFXq+enMxcSnF4L6Bt7dWxnQwmOEHsovQ6Rrh46sRJGz7Z937pDJmmV4MTS5hphY1Wvju1sKMERYg8FF2BgDMWu4ZBIe9d5QUhwBmPzRuE+HCW4lijBEWJrDRVAeQ48lVJ4h000W0CmJ9rU4IDmy9SGyl4d29lQPzhCbC3vewBAwNDRCBgf3uvDyVoMttcbjPzge1NXkYaKXh/fmVANjhBbaqgQFpVByBSrHNKU4IAWl6kug/if9aVWeQ9nQTU4QmyFMeDaIX4qI69QNLgEQqQ3QCYWtZkCqTtEIg6PTwqCTCyCQtI0XMu1aZB9XSnNC9cCJThCbMFoBLK/BcpvACIJMGImdp66iXqtAU/HhMDHrXdT5AeozGf5hcsgPqnpGvgxqXK3Xh3fWVCCI8RatHVAZR5QkcsnNlOL5sh4wHUQtHp+Ekup2AZ3hsRSviW1oYK/TKUEB4ASHCG9U1cGlPwClGbyU5C3JJbyyU09FnqDUZjRVy7pfYL7pbAaFXU6jPB3a64NuvrwCa62GPAK7fV7OANKcIR0l6YGKLkGFF0GqgvMX3PzBTxDAa8QwDMYkPDJp7GpSwfHWSnBFdQgp7QO7gpJc4LzCOQn06y+3evjOwtKcIR0RtfQdOl5k7/8rC9rfo0TAd5DAZ+RgM+I5rnZWtE0LeYsl4h71cBg4tI0F1ydRt+80WMw/7OKEpwJJThC2qPXAkVX+Nl3awrNB7FzHL+Wqd9owH+UxaTWkqkGp5Ba5/6baT64+qbECQBwD+ATrqaGv/+naLuew0BDCY6QlhoqgNtn+b5rek3zdlcfwDOEv7flGQRIlRYP0Z7GpkSkkLY/RXl3mWbzrde0SHASGeDuz182l98AAqOs8l79GSU4QhgDKnKA2+eAsqzm2prSCxgcDfiGAwqPXr2Fu1yC0YEecJNb5yvnKuOPY3aJCvCXytUF/D1CSnCU4MgApmvkL0NvnzO/r+Y9DBgygf9ppQ6zfh4KzByttsqxAAiLQZfXa8EYa76v5xMO3DgGVOY2LSPY+eWzM+sXQ7W2bt2K0NBQKBQKTJ48GT/88EOH5T///HNERERAoVBg7Nix2L9/v50iJX2eXguUZAI/fQWceg+4fphPbmIpn9QmLQXGP8YvyNyHRwN4ucrAcYBGZ4Sm5aB710GARwBgNAD5HX9PBoI+X4P77LPPkJKSgu3bt2Py5MnYvHkz4uPjkZmZCT+/tot2nDx5EomJiUhNTcWvfvUr7Nq1CwkJCTh37hzGjBnjgDMgdmM08DfXNdVAYzWgq+dbQPUa/t/1ZfyjZYOBqw8w+G7Af4zQpcMWSms18FRKhVXpe0sqFuFX4wIw2NOl7X294CnAlX8Ct8/w5+U2cNdK5VhXlo93oMmTJ2PixIl4//33AQBGoxFBQUH47//+b7zyyittyj/22GOoq6vDvn37hG333HMPIiMjsX379i69Z3V1NVQqFaqqquDh0bt7L30SY/zMr9o6PiE0Vjb9bPFgjL+RLnMF5O5ND4/mnxI5P8bSqAcMOsCo438atE0/dfzrYhl/81ss539KFPy+Yjn/b5HIPC5mND+GUWfhuZ5PWprqppir+YWPu/LnrPTku3T4RvBdK2xcU2vUGfDn4zfAGLDovlC4K6RWf4+yWg3cFBLIJWL+M7j0d76hQeYKhMXxNVIbJnB76+p3tE/X4LRaLc6ePYuVK1cK20QiEeLi4nDq1Kl29zl16hRSUlLMtsXHx2Pv3r0W30ej0UCjaW4xq66u7lqABZeAWz90/KUye431jdcMOj4xdUbXANSXd16uN0QSAE2JzRr/14okfPcIhQf/5ZYo+S+2VMk3Grj58UnaRqobddh7/jYMRgYj41fSatQZYDAy+LjLbZLcjEaGz8/eQqPOADe5BDKJCFLDOAwryYVCcxueBbsQMsiV/wwkcr4rCbjmxN7XLsUnLrbaofp0gistLYXBYIC/v7/Zdn9/f/zyyy/t7lNYWNhu+cLCQovvk5qairVr13Y/QF0DUFvS/f36ComcTwQKz6ak4NmcHDgxX0PS1vH9qjQ1fG1JU83/W6/hk4lIDIik/D0ssZSvsYkk/E9O1FTb0vLlDU1rBugb+RoYYDnRcqL2j2naJpLyNUBTvAoV/5C6OPQLyxhQVqtts10uFeGBiJ6tg9qZygYdJCIOjAE1jabPU4xi1RwE1lzCKFEuAMb/veoabBJDX9WnE5y9rFy50qzWV11djaCgoM539A1vXrzX7EvV6gvW114TSfjFgsUO/PUbDXzSM+og1CY4UVNik/GJsx9ylYnx6+ghEIk4iDkOIo6f3shDIRVm4rU2b1cZFk8dhjqNHjWNeugMxhaV4aF8nzm5gf/PyqBpqilbsdbch/XpBOfj4wOxWIyiIvNBzEVFRVCr229yV6vV3SoPAHK5HHJ5D+5PKD2bZ1Il3SMS92rpvL5KIhYhyNsx59VmxfvWnPDz7kyf7iYik8kQHR2NjIwMYZvRaERGRgZiYmLa3ScmJsasPAAcPnzYYnlCiPPq0zU4AEhJSUFSUhImTJiASZMmYfPmzairq8PChQsBAAsWLMDgwYORmpoKAHj++ecRGxuLDRs2YM6cOfj0009x5swZ/PnPf3bkaRBCHKDPJ7jHHnsMJSUlWLVqFQoLCxEZGYkDBw4IDQl5eXkQtehqMGXKFOzatQv/8z//g1dffRUjRozA3r17qQ8cIQNQn+8H5whO3w+OkH6uq9/RPn0PjhBCeoMSHCHEaVGCI4Q4rT7fyOAIptuSXR6yRQixK9N3s7MmBEpw7aipqQGAro1mIIQ4TE1NDVQqy1OzUytqO4xGI+7cuQN3d3erLBBiD6bhZfn5+U7T8uuM5wTQeVkDYww1NTUIDAw06ybWGtXg2iESiTBkyBBHh9EjHh4eTvWlAZzznAA6r97qqOZmQo0MhBCnRQmOEOK0KME5CblcjtWrV/dsVpQ+yhnPCaDzsidqZCCEOC2qwRFCnBYlOEKI06IERwhxWpTgCCFOixJcP/WnP/0JU6ZMgYuLCzw9Pbu0D2MMq1atQkBAAJRKJeLi4nD9+nXbBtpN5eXlePLJJ+Hh4QFPT08888wzqK2t7XCfadOmgeM4s8dvf/tbO0Xcvq1btyI0NBQKhQKTJ0/GDz90vMr8559/joiICCgUCowdOxb79++3U6Td053zSk9Pb/N7USgUdoyWEly/pdVqMX/+fPzud7/r8j5vvfUW3n33XWzfvh2nT5+Gq6sr4uPj0djYaMNIu+fJJ5/E1atXcfjwYezbtw/Hjx/H0qVLO91vyZIlKCgoEB5vvfWWHaJt32effYaUlBSsXr0a586dw/jx4xEfH4/i4uJ2y588eRKJiYl45plncP78eSQkJCAhIQFXrlyxc+Qd6+55Afyohpa/l9zcXDtGDICRfi0tLY2pVKpOyxmNRqZWq9nbb78tbKusrGRyuZzt3r3bhhF23U8//cQAsB9//FHY9s033zCO49jt27ct7hcbG8uef/55O0TYNZMmTWLLli0TnhsMBhYYGMhSU1PbLf+b3/yGzZkzx2zb5MmT2bPPPmvTOLuru+fV1b9NW6Ia3ACRk5ODwsJCxMXFCdtUKhUmT56MU6dOOTCyZqdOnYKnpycmTJggbIuLi4NIJMLp06c73PeTTz6Bj48PxowZg5UrV6K+vt7W4bZLq9Xi7NmzZp+zSCRCXFycxc/51KlTZuUBID4+vs/8XoCenRcA1NbWIiQkBEFBQZg3bx6uXr1qj3AFNNh+gCgsLAQAYbEeE39/f+E1RyssLISfn/nq7xKJBN7e3h3G+MQTTyAkJASBgYG4dOkSXn75ZWRmZmLPnj22DrmN0tJSGAyGdj/nX375pd19CgsL+/TvBejZeYWHh2PHjh0YN24cqqqq8M4772DKlCm4evWq3SazoBpcH/LKK6+0uSnb+mHpj6kvs/V5LV26FPHx8Rg7diyefPJJ7Ny5E1988QWys7OteBaku2JiYrBgwQJERkYiNjYWe/bsga+vLz788EO7xUA1uD7k97//PZKTkzssM2zYsB4dW61WAwCKiooQEBAgbC8qKkJkZGSPjtlVXT0vtVrd5oa1Xq9HeXm5EH9XTJ48GQCQlZWF4cOHdzve3vDx8YFYLEZRUZHZ9qKiIovnoFaru1XeEXpyXq1JpVJERUUhKyvLFiG2ixJcH+Lr6wtfX1+bHHvo0KFQq9XIyMgQElp1dTVOnz7drZbYnujqecXExKCyshJnz55FdHQ0AODbb7+F0WgUklZXXLhwAQDMErm9yGQyREdHIyMjAwkJCQD4CVQzMjKwfPnydveJiYlBRkYGXnjhBWHb4cOHERMTY4eIu6Yn59WawWDA5cuX8eCDD9ow0lYc2sRBeiw3N5edP3+erV27lrm5ubHz58+z8+fPs5qaGqFMeHg427Nnj/D8jTfeYJ6enuzLL79kly5dYvPmzWNDhw5lDQ0NjjiFds2aNYtFRUWx06dPsxMnTrARI0awxMRE4fVbt26x8PBwdvr0acYYY1lZWWzdunXszJkzLCcnh3355Zds2LBh7P7773fUKbBPP/2UyeVylp6ezn766Se2dOlS5unpyQoLCxljjD399NPslVdeEcp/9913TCKRsHfeeYf9/PPPbPXq1UwqlbLLly876hTa1d3zWrt2LTt48CDLzs5mZ8+eZY8//jhTKBTs6tWrdouZElw/lZSUxAC0eRw5ckQoA4ClpaUJz41GI3vttdeYv78/k8vlbMaMGSwzM9P+wXegrKyMJSYmMjc3N+bh4cEWLlxolrRzcnLMzjMvL4/df//9zNvbm8nlchYWFsb+8Ic/sKqqKgedAe+9995jwcHBTCaTsUmTJrHvv/9eeC02NpYlJSWZlf/73//ORo4cyWQyGRs9ejT7+uuv7Rxx13TnvF544QWhrL+/P3vwwQfZuXPn7BovTZdECHFa1IpKCHFalOAIIU6LEhwhxGlRgiOEOC1KcIQQp0UJjhDitCjBEUKcFiU4QojTogRHBrz09HSzad/XrFljNgFBcnKyMP6S9C+U4IjdJCcnW1wvYdmyZeA4zmzWEVskltDQUGzevNls22OPPYZr165Z3GfLli1IT08Xnk+bNs1sYDzpuyjBEbsKCgrCp59+ioaGBmFbY2Mjdu3aheDgYIfEpFQq20y02ZJKperywj6kb6EER+zq7rvvRlBQkNlsu3v27EFwcDCioqJ6dez2alYJCQlCrXDatGnIzc3Fiy++KEy0CbS9RG2tZU0yOTkZx44dw5YtW4Rj5OTkICwsDO+8847ZfhcuXADHcXad/4yYowRH7G7RokVIS0sTnu/YsQMLFy60+fvu2bMHQ4YMwbp164RVnrpry5YtiImJMVvFKzg4uM05AUBaWhruv/9+hIWFWesUSDdRgiN299RTT+HEiRPIzc1Fbm4uvvvuOzz11FM2f19vb2+IxWK4u7tDrVb3aMZclUoFmUwGFxcX4RhisRjJycnIzMwU1gnV6XTYtWsXFi1aZO3TIN1AM/oSu/P19cWcOXOQnp4OxhjmzJkDHx8fR4fVK4GBgZgzZw527NiBSZMm4V//+hc0Gg3mz5/v6NAGNKrBEYdYtGgR0tPT8dFHH1mtliMSidB6ekOdTmeVY3fF4sWLhQaUtLQ0PPbYY3BxcbHb+5O2KMERh5g1axa0Wi10Oh3i4+OtckxfX1+z+2oGg6HN6vAymQwGg6FX72PpGA8++CBcXV2xbds2HDhwgC5P+wC6RCUOIRaL8fPPPwv/tqSqqkpYRMZk0KBBCAoKalP2gQceQEpKCr7++msMHz4cGzduRGVlpVmZ0NBQHD9+HI8//jjkcnmPLo1DQ0Nx+vRp3Lx5E25ubvD29oZIJBLuxa1cuRIjRozoU4vGDFRUgyMO4+HhAQ8Pjw7LHD16FFFRUWaPtWvXtlt20aJFSEpKwoIFCxAbG4thw4Zh+vTpZmXWrVuHmzdvYvjw4T1eweyll16CWCzGqFGj4Ovri7y8POG1Z555Blqt1i6twqRztCYDIVb0n//8BzNmzEB+fn6bVeCJ/VGCI8QKNBoNSkpKkJSUBLVajU8++cTRIRHQJSohVrF7926EhISgsrISb731lqPDIU2oBkcIcVpUgyOEOC1KcIQQp0UJjhDitCjBEUKcFiU4QojTogRHCHFalOAIIU6LEhwhxGn9fxFxZjo3LTDOAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:36:29.981202Z", + "iopub.status.busy": "2024-03-22T15:36:29.980878Z", + "iopub.status.idle": "2024-03-22T15:36:30.334109Z", + "shell.execute_reply": "2024-03-22T15:36:30.333251Z" + }, + "papermill": { + "duration": 0.376796, + "end_time": "2024-03-22T15:36:30.336314", + "exception": false, + "start_time": "2024-03-22T15:36:29.959518", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABEWUlEQVR4nO3dd3hUZdo/8O+ZnjpJSJkEUimRHgTJBpGA5DUUEcQFRFcS6rrC/sQsFnQBwfWNawVXXnQLZFkF1H0p76oLYhRY6b1LiSEJkE4y6VOf3x9n5iSTzKRMpjHcn+uaK85pc5+QuX3OUznGGAMhhHghkbsDIIQQZ6EERwjxWpTgCCFeixIcIcRrUYIjhHgtSnCEEK9FCY4Q4rUowRFCvJbE3QF4IqPRiNu3byMgIAAcx7k7HEJIK4wx1NbWIioqCiKR7XIaJTgrbt++jejoaHeHQQjpQFFREXr16mVzPyU4KwICAgDwv7zAwEA3R0MIaa2mpgbR0dHCd9UWSnBWmB9LAwMDKcER4sE6qkKiRgZCiNeiBEcI8VqU4AghXovq4O4hjDHo9XoYDAZ3h0JIu8RiMSQSSbe7aVGCu0dotVoUFxejoaHB3aEQ0im+vr6IjIyETCaz+xqU4O4BRqMR+fn5EIvFiIqKgkwmow7MxGMxxqDValFeXo78/Hz07du33c687aEEdw/QarUwGo2Ijo6Gr6+vu8MhJlq9AfVaA/zlEkjFVB3eko+PD6RSKQoKCqDVaqFQKOy6Dv1W7yH2/l+QOEedxgCt3oh6jd7doXgkR/y9uvUv/sCBA5gyZQqioqLAcRx27txpsT8zMxMcx1m8JkyY0OF1169fj7i4OCgUCiQnJ+PYsWNOugNC7KczGAEAGr3RzZF4L7cmuPr6egwdOhTr16+3ecyECRNQXFwsvLZu3druNT///HNkZWVh1apVOHXqFIYOHYr09HSUlZU5OnxCusVcCxrkI3VrHN7MrQlu4sSJ+MMf/oDHH3/c5jFyuRwqlUp4BQcHt3vN999/HwsXLsTcuXMxYMAAfPzxx/D19cXGjRsdHT7xUJmZmZg2bZpDrzl27FgsXbq03WPi4uKwdu3azl/UlOHEImrwcRaPr5TZt28fwsPDkZiYiN/85jeorKy0eaxWq8XJkyeRlpYmbBOJREhLS8Phw4dtnqfRaFBTU2PxIp6hM4nlrmVekfgey2+vv/46kpKSXPJZHp3gJkyYgM2bNyM3Nxd//OMfsX//fkycONFmR9WKigoYDAZERERYbI+IiEBJSYnNz8nOzoZSqRReNFUScTbGmJDfGrTU8dpZPDrBPfnkk3jssccwePBgTJs2DV999RWOHz+Offv2OfRzli9fDrVaLbyKioocen1PpdUbbb70BmOnj9V18tiuyszMxP79+7Fu3TqhkSkvLw/z589HfHw8fHx8kJiYiHXr1lk9f/Xq1QgLC0NgYCCeffZZaLXaTn1ufX095syZA39/f0RGRuK9995rc0xZWRmmTJkCHx8fxMfH47PPPmtzDMdx2LBhAyZOnAgfHx8kJCTgn//8p7Cvpvw2VEoffPH553jooYfg4+ODBx54AFevXsXx48cxYsQI+Pv7Y+LEiSgvL+/0723jxo0YOHAg5HI5IiMjsWTJEmFfYWEhpk6dCn9/fwQGBmLmzJkoLS0V9ptLV//4xz8QFxcHpVKJJ598ErW1tcIxRqMRb7/9Nvr06QO5XI6YmBi8+eabwv6XX34Z/fr1g6+vLxISErBixQrodDoAQE5ODlavXo2zZ88K/6Y5OTmdvreuuqv6wSUkJCA0NBTXr1/H+PHj2+wPDQ2FWCy2+AcDgNLSUqhUKpvXlcvlkMvlDo/X063/4brNffGhfpg2rKfw/s8H8qAzMKvH9gr2wYwRzaXejQfz0WilVPLCf/XrUnzr1q3D1atXMWjQIKxZswYAEBwcjF69euHLL79Ejx49cOjQISxatAiRkZGYOXOmcG5ubi4UCgX27duHGzduYO7cuejRo4fFF9GWF198Efv378euXbsQHh6OV199FadOnbJ4rMrMzMTt27fxww8/QCqV4v/9v/9ntSFrxYoVeOutt7Bu3Tr84x//wJNPPonz58+jf//+wpPp2//9B/zpw3WIiYnBvHnz8NRTTyEgIADr1q2Dr68vZs6ciZUrV2LDhg0dxr5hwwZkZWXhrbfewsSJE6FWq3Hw4EEAfGIyJ7f9+/dDr9dj8eLFmDVrlkWhIS8vDzt37sRXX32FqqoqzJw5E2+99Zbwu1u+fDn+8pe/4IMPPsDo0aNRXFyMn376STg/ICAAOTk5iIqKwvnz57Fw4UIEBATgpZdewqxZs3DhwgXs3r0b3333HQBAqVR2eF92Yx4CANuxY0e7xxQVFTGO49iuXbtsHjNy5Ei2ZMkS4b3BYGA9e/Zk2dnZnY5FrVYzAEytVnf6HE/W2NjILl26xBobGy22v//tFZuvHaduWhz7p9yrNo/94nihxbEb9l23epw9UlNT2fPPP9/uMYsXL2ZPPPGE8D4jI4OFhISw+vr65pg2bGD+/v7MYDC0e63a2lomk8nYF198IWyrrKxkPj4+QhxXrlxhANixY8eEYy5fvswAsA8++EDYBoA9++yzFtdPTk5mv/nNbxhjjF3Py2MA2Pt/2iDs37p1KwPAcnNzhW3Z2dksMTGx3bjNoqKi2GuvvWZ137fffsvEYjErLGz+97p48aLFvaxatYr5+vqympoa4ZgXX3yRJScnM8YYq6mpYXK5nP3lL3/pVDyMMfbOO++w4cOHC+9XrVrFhg4d2uF5tv5uGev8d9StJbi6ujpcv95cisjPz8eZM2cQEhKCkJAQrF69Gk888QRUKhXy8vLw0ksvoU+fPkhPTxfOGT9+PB5//HGhGJ6VlYWMjAyMGDECI0eOxNq1a1FfX4+5c+e6/P483eJxfWzua92wt2hMb5vHth71Ne/B+O6E1aH169dj48aNKCwsRGNjI7RabZtK66FDh1qM2khJSUFdXR2KiooQGxtr89p5eXnQarVITk4WtoWEhCAxMVF4f/nyZUgkEgwfPlzYdt999yEoKKjN9VJSUtq8P3PmDPRGI2oa+ce2/gMHgTEGjuOE+uPBgwcL50RERHSqm1NZWRlu375t9enGHHd0dLRFHfOAAQMQFBSEy5cv44EHHgDAtwa3nCk3MjJS+PzLly9Do9HY/AyA76r14YcfIi8vD3V1ddDr9W6bONatCe7EiRMYN26c8D4rKwsAkJGRgQ0bNuDcuXP4+9//jurqakRFReGRRx7BG2+8YfE4mZeXh4qKCuH9rFmzUF5ejpUrV6KkpARJSUnYvXt3m4YHAsgkna+CddaxXbVt2zYsW7YM7733HlJSUhAQEIB33nkHR48eddpnOgNjgLlaUiqVgoFvTDWPEZZKm/vGcRwHo7HjOkwfHx+HxNbys1t/fkefcfjwYTz99NNYvXo10tPToVQqsW3bNqv1mK7g1gQ3duxYMGa9XgcA9uzZ0+E1bty40WbbkiVLLCpWyd1LJpNZtJofPHgQo0aNwnPPPSdsy8vLa3Pe2bNn0djYKHwhjxw5An9//w5byHv37g2pVIqjR48iJiYGAFBVVYWrV68iNTUVAF9a0+v1OHnypFDquXLlCqqrq9tc78iRI5gzZ47F+2HDhqH1nz0zZ7huCAgIQFxcHHJzcy0KDmb9+/dHUVERioqKhN/DpUuXUF1djQEDBnTqM/r27QsfHx/k5uZiwYIFbfYfOnQIsbGxeO2114RtBQUFFse0/jd1pruqkYHce+Li4nD06FHcuHED/v7+6Nu3LzZv3ow9e/YgPj4e//jHP3D8+HHEx1s+Fmu1WsyfPx+///3vcePGDaxatQpLlizpcHyjv78/5s+fjxdffBE9evRAeHg4XnvtNYvzEhMTMWHCBPz617/Ghg0bIJFIsHTpUqulmy+//BIjRozA6NGj8dlnn+HYsWP429/+huZOImYOyHDgW0GfffZZhIeHY+LEiaitrcXBgwfx29/+FmlpaRg8eDCefvpprF27Fnq9Hs899xxSU1MxYsSITl1foVDg5ZdfxksvvQSZTIYHH3wQ5eXluHjxIubPn4++ffuisLAQ27ZtwwMPPICvv/4aO3bssLhGXFycUB3Vq1cvBAQEOK2Rz6O7iRCybNkyiMViDBgwAGFhYUhPT8f06dMxa9YsJCcno7Ky0qI0ZzZ+/Hj07dsXY8aMwaxZs/DYY4/h9ddf79RnvvPOO3jooYcwZcoUpKWlYfTo0Rb1bQCwadMmREVFITU1FdOnT8eiRYsQHh7e5lqrV6/Gtm3bMGTIEGzevBlbt27FgAEDLEpwgT5SiBw0fVVGRgbWrl2L//mf/8HAgQPx6KOP4tq1awD4R81du3YhODgYY8aMQVpaGhISEvD555936TNWrFiB3/3ud1i5ciX69++PWbNmCXV0jz32GF544QUsWbIESUlJOHToEFasWGFx/hNPPIEJEyZg3LhxCAsL63D4ZXdwrL1nxHtUTU0NlEol1Gq1V6yq1dTUhPz8fMTHx9s97QzpOo7jsGPHDqvDxpp0BqgbdZCKRQjxs39CR2/W3t9tZ7+jVIIjhHgtSnDknlJYWAh/f3+br8LCQpfGozMYUafRw2Ds3INUe7H/5z//cXK0dx9qZCD3lKioKJw5c6bd/Y7SXu2PQiqGQipGea0G9Ro95GJRp2YVaS/2nj172tx3r6IER+4pEokEffrY7uDsahwHgKFNm6otnhT73YAeUQnxCNTW5wxUgiPEDZp0BjTpDELdG6U356ASHCFuYDAyy7UYKMM5BSU4QjwA5TfnoARHCPFaVAdHiBuYS2xSsQj+cgkktPCMU1AJjng0T1t0xmHxmPrISUQcZBIRRJTgnIISHPF6nV2LwZWEBbUorzkVJbh7EWOAXuueVxfmdrB30RnzuqhvvvkmoqKihNl4Dx06hKSkJCgUCowYMQI7d+4Ex3EWowMuXLiAiRMnwt/fHxEREXjmmWeECVWtxWNtPsKu0OqNqNfo2yzcQxyD6uDuRQYd8B/3zLCKh34HSDo3e0Z3F50JDAzE3r17AfCzT0yZMgWTJk3Cli1bUFBQ0OZRs7q6Gg8//DAWLFiADz74AI2NjXj55Zcxc+ZMfP/991bjCQsLs+vX4C+XwF8ugbpRhzqNHv6QQCqm8oajUYIjHkupVEImk8HX19diVbTVq1cL/x0fH4/Dhw/jiy++sEhwfn5++Otf/wqZjE+mH3/8MTiOw1/+8hcoFAoMGDAAt27dwsKFC4VzPvroIwwbNgz//d//LWzbuHEjoqOjcfXqVfTr189qPPYwT03O0TOqU1GCuxeJpXxJyl2f3U2dWXRm8ODBQnID+CnFhwwZYjGv2MiRIy3OOXv2LH744Qf4+/u3+cy8vDz069e1ZQ87w5zeqB+cc1CCuxdxXKcfEz1NZxed8fPz6/K16+rqMGXKFPzxj39ssy8yMtLumK1p0Oqh0xuhNa81SxnOKSjBEY9m76IzrSUmJuLTTz+FRqMR5v8/fvy4xTH3338//vd//xdxcXGQSKx/NRy1YIrOwNDUcqgWZTinoFpN4tFaLjpTUVGBvn374sSJE9izZw+uXr2KFStWtElU1jz11FMwGo1YtGgRLl++jD179uDdd98F0FwPtnjxYty5cwezZ8/G8ePHkZeXhz179mDu3LlCUmsdT2eW87OKVgpwCUpwxKPZu+hMa4GBgfjXv/6FM2fOICkpCa+99hpWrlwJAEK9XFRUFA4ePAiDwYBHHnkEgwcPxtKlSxEUFCSsqtU6HntnALa2phZxgnbXvXey/fv3s0cffZRFRkYyAGzHjh3CPq1Wy1566SU2aNAg5uvryyIjI9kzzzzDbt261e41V61axcD/vQivxMTELsWlVqsZAKZWq+25LY/T2NjILl26xBobG90dikf59NNPmVQqZQ0NDS7/7Kp6DStRN7KaRi1r0umZTm9weQyerr2/285+R91aB1dfX4+hQ4di3rx5mD59usW+hoYGnDp1CitWrMDQoUNRVVWF559/Ho899hhOnDjR7nUHDhyI7777Tnhvqz6F3Fs2b96MhIQE9OzZE2fPnhX6uDlqRXh7SEQc5BKx2z7f27n1mz9x4kRMnDjR6j6lUil00jT76KOPMHLkSBQWFgqrjlsjkUi63U+JeJ+SkhKsXLkSJSUliIyMxIwZM/Dmm2+6JRZ6JHWNu6poo1arwXEcgoKC2j3u2rVriIqKgkKhQEpKCrKzs9tNiBqNBhqNRnhfU1PjqJCJB3nppZfw0ksvuTsMninD6Y0MDVq9adA9leQc7a5pZGhqasLLL7+M2bNnt7vQa3JyMnJycrB7925s2LAB+fn5eOihh1BbW2vznOzsbCiVSuEVHR3tjFsgRKD0lSLMXw4OHGqb9GjU0VhUZ7grEpxOp8PMmTPBGMOGDRvaPXbixImYMWMGhgwZgvT0dHzzzTeorq7GF198YfOc5cuXQ61WC6+ioiJH34JHYNQ1wWOIOA4iEUezibTDEX+vHv+Iak5uBQUF+P7779stvVkTFBSEfv364fr16zaPkcvlQudPbySV8sOjGhoa3FqhTtpB//Npo6GhAUDz3689PDrBmZPbtWvX8MMPP6BHjx5dvkZdXR3y8vLwzDPPOCHCu4NYLEZQUBDKysoAAL6+vjTI283qNTohp+n0RogMHJpE9JgK8CW3hoYGlJWVISgoCGKx/XWTbk1wdXV1FiWr/Px8nDlzBiEhIYiMjMQvf/lLnDp1Cl999RUMBgNKSkoAACEhIcJA6vHjx+Pxxx/HkiVLAPAdMadMmYLY2Fjcvn0bq1atglgsxuzZs11/gx7E3KpsTnLEveo0ehiNDBKxCHqDEVKxCD4yamRoKSgoqNu9Idya4E6cOIFx48YJ77OysgAAGRkZeP311/F///d/ANBmpogffvgBY8eOBcCPQzRPSAgAN2/exOzZs1FZWYmwsDCMHj0aR44csXveLm/BcRwiIyMRHh4OnU7n7nDueTtP30J1gxa9w/yQV1WPmB5+eDg+3N1heQypVNqtkpuZWxPc2LFj261I7EwlY+sZVbdt29bdsLyaWCx2yB8O6Z5GowiNRjGYWIZGYxN0EFtM5UQcw6Pr4AjxVqYF7REX6ofe4f7wk9FX0Rnot0qIG5ifToJ8pAgPpJKbs9wV/eAI8TZGU4Kj1mznogRHiBuYH1EbtHpcLq5BfkW9ewPyUvSISogbzEmJhZEBxdWN2H2hBJFKBeJDuz7NOmkfJThC3MDX1KggNq1ob6SBDE5Bj6iEuJHIVAfHaAIlp6ASHCFu8P1PpeA4DtHBvgCoBOcslOAIcTGjkeFskRoAEB3MT35AM704Bz2iEuJixhbJTGxazMZIRTinoARHiIu1zGUSamRwKnpEJcTFWpbggv1kmDhYBZmYyhrOQAmOEBdrWd3mJxPjPlXXJnElnUf/2yDExQzCMC0aquVsVIIjxMXMj6gijoNWb0ThnXoAHPqE+7s3MC9ECY4QF/OXSTDvwXgYGUO9Ro9/nS2GTCJCn/A+7g7N61CCI8TFRCIOSl9+IRV1Az+7MvWDcw6qgyPEjTjTN5Dym3NQCY4QF6vT6HG6sAoKqRj9I/kWVOoH5xxUgiPExeo1epy4UYWzRdUw9fOFkTF6THUCSnCEuFjLVlRRi24ilN8cjxIcIS5mfhwVcXxfuObtlOEcjergCHEx88B6kYiDRCRCWv8IcBwsSnPEMdxagjtw4ACmTJmCqKgocByHnTt3WuxnjGHlypWIjIyEj48P0tLScO3atQ6vu379esTFxUGhUCA5ORnHjh1z0h0Q0nUtF5wRizgM7qXEoJ5KiESU4BzNrQmuvr4eQ4cOxfr1663uf/vtt/Hhhx/i448/xtGjR+Hn54f09HQ0NTXZvObnn3+OrKwsrFq1CqdOncLQoUORnp6OsrIyZ90GIV1ifkQVU4nN+ZiHAMB27NghvDcajUylUrF33nlH2FZdXc3kcjnbunWrzeuMHDmSLV68WHhvMBhYVFQUy87O7nQsarWaAWBqtbprN0FIJ1wvq2Xvf3uFbT1awBhj7EZFHfu5vI7p9AY3R3b36Ox31GMbGfLz81FSUoK0tDRhm1KpRHJyMg4fPmz1HK1Wi5MnT1qcIxKJkJaWZvMcANBoNKipqbF4EeIsvYJ98ExKLNIHqgAAO0/fxs7Tt9CoM7g5Mu/jsQmupKQEABAREWGxPSIiQtjXWkVFBQwGQ5fOAYDs7GwolUrhFR0d3c3oCbFNLhEj1F+OYD8ZAAh94agN1fE8NsG50vLly6FWq4VXUVGRu0Mi9xBz4wIzujkQL+Sx3URUKr74XlpaisjISGF7aWkpkpKSrJ4TGhoKsViM0tJSi+2lpaXC9ayRy+WQy+XdD5qQTiiraUJeeT1C/GRIVAUIfeGoH5zjeWwJLj4+HiqVCrm5ucK2mpoaHD16FCkpKVbPkclkGD58uMU5RqMRubm5Ns8hxNXKajU48nMlfirh63rN/d8MlOAczq0luLq6Oly/fl14n5+fjzNnziAkJAQxMTFYunQp/vCHP6Bv376Ij4/HihUrEBUVhWnTpgnnjB8/Ho8//jiWLFkCAMjKykJGRgZGjBiBkSNHYu3ataivr8fcuXNdfXuEWGUwNg/V4n/y2ym/OZ5bE9yJEycwbtw44X1WVhYAICMjAzk5OXjppZdQX1+PRYsWobq6GqNHj8bu3buhUCiEc/Ly8lBRUSG8nzVrFsrLy7Fy5UqUlJQgKSkJu3fvbtPwQIi7tByL2vInowzncByj32obNTU1UCqVUKvVCAykBUGIY50sqMKBq+XoHxmACYMiceGWGnojQ78If/jKPLZa3KN09jtKv01CXIy1GKoFAIN6Kt0Zjlfz2EYGQrxV82wiNFTL2exKcD///LOj4yDkntHcyMC/L6tpQtGdBjTRSAaHsyvB9enTB+PGjcOnn37a7sB3Qkhbg3oG4smR0RgRGwIA+PZSKf558iZKa+i75Gh2JbhTp05hyJAhyMrKgkqlwq9//WuakoiQTgpQSBGp9BFW1jI/qtK6DI5nV4JLSkrCunXrcPv2bWzcuBHFxcUYPXo0Bg0ahPfffx/l5eWOjpMQryWikQxO061GBolEgunTp+PLL7/EH//4R1y/fh3Lli1DdHQ05syZg+LiYkfFSYjXKKxswMmCO7hV3QiA+sE5U7cS3IkTJ/Dcc88hMjIS77//PpYtW4a8vDzs3bsXt2/fxtSpUx0VJyFe43p5LQ5crUBBZT0AtBiL6sagvJRd/eDef/99bNq0CVeuXMGkSZOwefNmTJo0CSIRny/j4+ORk5ODuLg4R8ZKiFcwmmYNaT2SgR5RHc+uBLdhwwbMmzcPmZmZFjN9tBQeHo6//e1v3QqOEG/UZqiW6TnKSNMlOZxdCW7v3r2IiYkRSmxmjDEUFRUhJiYGMpkMGRkZDgmSEG8irMlg+voMiFSiV7AvwgJoyi5HsyvB9e7dG8XFxQgPD7fYfufOHcTHx8NgoA6LhNhibDVUK1EV4M5wvJpdjQy2Wnvq6uosZvoghLTV+hGVOE+XSnDm6Yw4jsPKlSvh6+sr7DMYDDh69KjN2XYJIbyWK9sDgLpRhyadAf5yCfzkNP+FI3Xpt3n69GkAfAnu/PnzkMlkwj6ZTIahQ4di2bJljo2QEC8zqncPDIsOQpBpJMPhvApcLq7FmH5hGB4b7ObovEuXEtwPP/wAAJg7dy7WrVtHc6URYodQf8vGBI46+jqNXeXhTZs2OToOQu5ZNBbVeTqd4KZPn46cnBwEBgZi+vTp7R67ffv2bgdGiLe6XlaLeo0BcT38oPSV0lhUJ+p0glMqlUJRWqmkGUgJsdfJgircrm7ClKGRpgRHIxmcpdMJruVjKT2iEmK/1jP6crSqltPY1Q+usbERDQ0NwvuCggKsXbsW3377rcMCI8Rb2VpVi0pwjmdXgps6dSo2b94MAKiursbIkSPx3nvvYerUqdiwYYNDAyTE27QuwcWE+CI5IQQxIb7tnEXsYfeMvg899BAA4J///CdUKhUKCgqwefNmfPjhhw4NkBBv07yqFv8+LtQPo3qHIraHnxuj8k52JbiGhgYEBPDj57799ltMnz4dIpEIv/jFL1BQUODQAOPi4sBxXJvX4sWLrR6fk5PT5lgaPkY8ibDojIiGajmbXf3g+vTpg507d+Lxxx/Hnj178MILLwAAysrKHN759/jx4xaD9y9cuID/+q//wowZM2yeExgYiCtXrgjvORrzRzyIMJuI6e+yUWtAg1YPmUSEAIXUjZF5H7sS3MqVK/HUU0/hhRdewPjx45GSkgKAL80NGzbMoQGGhYVZvH/rrbfQu3dvpKam2jyH4zioVCqHxkGIozwyIAJag1EYqnXhtho/XqvAwKhAPDKQ/m4dya5H1F/+8pcoLCzEiRMnsHv3bmH7+PHj8cEHHzgsuNa0Wi0+/fRTzJs3r91SWV1dHWJjYxEdHY2pU6fi4sWLTouJkK6KDvFF7zB/KKRiAC0XnXFjUF7K7qkLVCpVm1LSyJEjux1Qe3bu3Inq6mpkZmbaPCYxMREbN27EkCFDoFar8e6772LUqFG4ePEievXqZfUcjUYDjUYjvK+pqXF06ITYRGNRnceuBFdfX4+33noLubm5KCsrg7HVXMvOWvn+b3/7GyZOnIioqCibx6SkpAiPzAAwatQo9O/fH5988gneeOMNq+dkZ2dj9erVDo+XEGsuF9eAMaB3uB/kEjGNRXUiuxLcggULsH//fjzzzDOIjIx0SSV+QUEBvvvuuy6Pc5VKpRg2bBiuX79u85jly5cLc90BfAkuOjra7lgJaU/u5VLoDAzzguNNCY7fTh19Hc+uBPfvf/8bX3/9NR588EFHx2PTpk2bEB4ejsmTJ3fpPIPBgPPnz2PSpEk2j5HL5ZDLaT584hqtJ7ykkQzOY1cjQ3BwMEJCQhwdi01GoxGbNm1CRkYGJBLLnDxnzhwsX75ceL9mzRp8++23+Pnnn3Hq1Cn86le/QkFBARYsWOCyeAlpT+uhWjQW1XnsSnBvvPEGVq5caTEe1Zm+++47FBYWYt68eW32FRYWori4WHhfVVWFhQsXon///pg0aRJqampw6NAhDBgwwCWxEtIeo5EJicyc4EL95RgeG4w+4f5ujMw7ccyOppthw4YhLy8PjDHExcVBKrXsnHjq1CmHBegONTU1UCqVUKvVNGsxcSi9wYg/fc/XBz83rjfkErGbI7o7dfY7alcd3LRp0+yNi5B7WsuWUlpVy/nsSnCrVq1ydByE3BNaNiSYE5zOYESjzgAOoKFaDmZXHRzAT5P017/+FcuXL8edO3cA8I+mt27dclhwhHgbqViER4dEYtLgSKEVtehOA/72n3x8da64/ZNJl9lVgjt37hzS0tKgVCpx48YNLFy4ECEhIdi+fTsKCwuFueIIIZbEIg59IyxXsqduIs5jVwkuKysLmZmZuHbtmsVURJMmTcKBAwccFhwh9wIayeA8dpXgjh8/jk8++aTN9p49e6KkpKTbQRHirTR6A25UNEAi5tA7jO8W0twPjjKco9lVgpPL5VYHpF+9erXN9EaEkGYNGgO+OV+MPRebCwLmiS+NVIRzOLsS3GOPPYY1a9ZAp9MB4GdDKCwsxMsvv4wnnnjCoQES4k0MrUYx8P/N/6T85nh2Jbj33nsPdXV1CAsLQ2NjI1JTU9GnTx8EBATgzTffdHSMhHgNc0OC2CLBUSODs9hVB6dUKrF3714cPHgQZ8+eRV1dHe6//36kpaU5Oj5CvIo5h7Xs4+sjE2NILyV8pDSqwdG6nOCMRiNycnKwfft23LhxAxzHIT4+HiqVCowxWv+AkHa0HmgPAIEKKcb3j+jw3CM/V0JnMGJ0n1D6nnVSlx5RGWN47LHHsGDBAty6dQuDBw/GwIEDUVBQgMzMTDz++OPOipMQryCsqNXF/FRVr8XhvEqcuFGF8lpNxycQAF0sweXk5ODAgQPIzc3FuHHjLPZ9//33mDZtGjZv3ow5c+Y4NEhCvIUwk0iLDGc0Mmj0RjAw+MqsfyWra9To0ZCHanlPVNZrER5IS2F2RpdKcFu3bsWrr77aJrkBwMMPP4xXXnkFn332mcOCI8TbBPvJ8MjACIzq3UPYVqvR4+P9edj4Y771k4xGSC/+E30rcjGw7Cs0aPQuivbu16UEd+7cOUyYMMHm/okTJ+Ls2bPdDooQb+Uvl2BglBJ9wpuHa3XYTaT6BiJFVQjykcJXdwe4k+f8QL1ElxLcnTt3EBFhuzI0IiICVVVV3Q6KkHtJh91EKvMg5jgEKCTgAEjVN1wW292uS3VwBoOhzZThLYnFYuj1VHwmxJY6jR5lNU3wkYkRqfQB0JzgGIP1ngh3+EfXyMQR6Fl2EZw/LWvZWV1KcIwxZGZm2lygpeXaooSQtm5VNeKb88WIDvHFL4fz6/S2zGdGBohb5je9BmioRFltE4p69MZ92nPwa6gEjAZARP3mOtKlBJeRkdHhMdSCSohtzf3gmre17BNnZAxitNhZVwYAuN0kxYESKWINYvgZDUBjFeAX6pKY72ZdSnCbNm1yVhyE3BOsdfQVcW33C+pK+R+SEIDjcK5Kivj6asTXl1OC6wS7hmoRQuxjbaiWiOPQPzIAHMeBQ6v6N1OCq5XyyaxOrER1YznQcMcV4d71KMER4kLNIxlalOBEHCYMirR+Qn0FAKBWHAQA0EgCYGxiQJPaqXF6C7vXZCCEdJ21R9R2NfLdrupEfL85jdgfBgZAQy2pnUEJjhAXMnfmbT0W1WBk0OgNlpNe6hoBXSMYY6jl+LU/NZIAfvHoxmrXBHyX8+gE9/rrr/P1Ei1e9913X7vnfPnll7jvvvugUCgwePBgfPPNNy6KlpCORYf44OH7wjEgynKx4j8f+Bn/80Meqht1zRtNpTe91A9GEb+coEYSAAbA0KhurtAjNnl0ggOAgQMHori4WHj9+OOPNo89dOgQZs+ejfnz5+P06dOYNm0apk2bhgsXLrgwYkJsCw9QYGh0EGJ7+FlsF5u+iYaWJThTQ4LYLwTPpMRi5gPR0Ij9wMCBGfSAts5VYd+1PD7BSSQSqFQq4RUaartpfN26dZgwYQJefPFF9O/fH2+88Qbuv/9+fPTRRy6MmJCuM9fJWSQ4UwlO5NsDof5y9AzygUgkhlbix9flUUNDhzw+wV27dg1RUVFISEjA008/jcLCQpvHHj58uM2swunp6Th8+HC7n6HRaFBTU2PxIsQZqhu0KLrTAHWDzmK72FQpZ2j52Nlo6griEyxsem5cb6QOToBcIgY0tU6P927n0QkuOTkZOTk52L17NzZs2ID8/Hw89NBDqK21/g9bUlLSZjKAiIiIDpcyzM7OhlKpFF7R0dEOuwdCWjp/S41/nryJszerLbaLra2sZSrBqbkAHM6rxKXbNZCKReBk/HKD0Na7IuS7mkcnuIkTJ2LGjBkYMmQI0tPT8c0336C6uhpffPGFQz9n+fLlUKvVwquoqMih1yfErLkV1bIZVSjBmQ9gTKiDu2P0xZGfK3H+VjW/T26aaonq4Dp0V3X0DQoKQr9+/XD9+nWr+1UqFUpLSy22lZaWQqVStXtduVxucwIBQhzJaGPKcvMqW8Ijqq6RH2gPoFGiBFAJqViEH69VQFTUhCSdHr4aSnAd8egSXGt1dXXIy8tDZKT1Xt8pKSnIzc212LZ3716kpKS4IjxCOiR09G2V4aJDfNEvIgB+5inLm6r5n3J/aBk/a4hMIsKNynr8rAa0BiM9onaCR5fgli1bhilTpiA2Nha3b9/GqlWrIBaLMXv2bAD8zCU9e/ZEdnY2AOD5559Hamoq3nvvPUyePBnbtm3DiRMn8Oc//9mdt0GIwNpQLQB4sE+r3gGm+jf4BENnMAIApGIRJCIODWJf/lFXS40MHfHoBHfz5k3Mnj0blZWVCAsLw+jRo3HkyBGEhYUBAAoLCyESNRdCR40ahS1btuD3v/89Xn31VfTt2xc7d+7EoEGD3HULhFgwV7GJO3p2apHgtHo+wckkIohFHHRiX/5Rl0pwHfLoBLdt27Z29+/bt6/NthkzZmDGjBlOioiQ7mGmR1Rr65oyxsCY6fHVPBRLEQStxpTgxCJIxBy0Yl8YDQzQNtDElx3w6ARHiLdJVAUgLIDvtNvSv88X46eSWoxNDMOwmGDLElx9yxKcCHqRAkZz9bm2HlBYDvsizSjBEeJCCWH+SAhru51rvfBMiwT3i4QeGNxTCX+FBBW1GoDjoJf4AjAN16IEZxMlOEI8QHM/OAB6bXP9mk8QlFIplD5Si+N0El8ANVQP1wFKcIS4UEWdBnoDQ5CvFAppc92ZxWB7cxcRqQKQWj7Kjk0MR2piGKSXzgKVNdTZtwN3VT84Qu52+66UY+uxQhRUNlhst1gbtcXjKQCcu1mNkwVVqNPoIZOIIJeIIZLTcK3OoBIcIS5kcySDaYPe2CLBKYIAAMfy76C2SY+eQT7wl5u+sjLTdEuU4NpFCY4QF7I1ksE8VMtoZM1dREwlOJ2BP0cq5nCjoh5XS2uRoGHoA1CC6wA9ohLiQgYbazIE+8kQH+qHED8Z0FDJb/QNAWPMoqNvZb0GF2/XoLipRTcRYhOV4AhxIWEkQ6sE1z8yEP0jTd09CvmVtOAbCoORCaU+qZjvBwcAGpGp8YESXLuoBEeICzWPZLBxgLaBfwGAbw9+UL2JzDQWFQC0QoKjVtT2UIIjxIWEwfatWxlMWIOp9KYIBCQy4fFUKuYgEnGQiPnzmjhTgjPo+H5zxCp6RCXEhYbFBKNRa0CAwvKrd7W0Ft9eLMFA7meMAwBffnYRcwlOJuHLIuYSnA4SQCwBzIvPSEJcdg93E0pwhLhQUnSQ1e0c+NZSsfYO4AvArwcAIMhHhpkPRAuPtuY6OL2RATJ/vsVVWw/4UoKzhhIcIR5AahrKIG6q4BOcqQQnk4gsBuZLWk5tLvfjE5yuofXliAklOEJcqKKOn4Y82FcmdO4FwNetMQZ5YzkAH8A/wur5kUoFFo5J4BPdFXNnX2posIUaGQhxoc+PF+EfhwtQ22S5bKBMLILcUAuRoYmf382Pn3KkrKYJpwqrUGga2iURi+Avl/DjWGl1rQ5RgiPEhYw2WlElYhH8tBX8o6dfKN+AAKCoqgH7r5TjUrGVRZ5puFaH6BGVEBeyNZJBIubgry3nO/UGNC+q1KTjW1HlpplHmnQGHP65EkYjw/hAX/4gSnA2UQmOEBcxT0kOtB1sLxOLECOuRKCPFMaAKGF7k84AAFBI+ARnZAxnCqtx7qYaTEZ1cB2hEhwhLmJosWp96xKcgjPggRANwAKB4Bhhu7kEp5Ca+8E1l0n0El9IgeaRD6QNKsER4iIt8lubBIeaWwAz8iMYTNMkAS1KcKZHVEmLop9B0qIOjrW4OBFQgiPERYwtkpC49TPqnZ/5n0ExaJmqmvSWCU4k4oTkqBMpTBfWA3qNU2K+29EjKiEuIuI4jIwPgd7ILOvgGAMqruHCLTXO1skwPkoDlZJPXq0fUQG+QUKrZzBwEkAia17DQapw5e3cFTy6BJednY0HHngAAQEBCA8Px7Rp03DlypV2z8nJyQHHcRYvhYL+4Yn7ySQiPNgnFKn9wizXRa2vABqroOfEqJD1Eh5LAWDy4EhMTYpCsK9M2CZpOfuv0BeOGhqs8egEt3//fixevBhHjhzB3r17odPp8Mgjj6C+vv1m8cDAQBQXFwuvgoICF0VMiB3KLwMAtP7RMIqkaGyR4FRKBRLC/FstUGNKcAZGfeE64NGPqLt377Z4n5OTg/DwcJw8eRJjxoyxeR7HcVCpVM4Oj5Au0RuMqG3SQyLmEKDglwGE0QgUnwMANPXoDzTCIsFZM2NENDgO8JNJKMF1wKNLcK2p1Xxv7pCQ9mdOqKurQ2xsLKKjozF16lRcvHix3eM1Gg1qamosXoQ42p16LXIO3cDnx4uaN1blA5paQKqAMaQvAKBJaxCOP1lwBzcqLJOX0keKQIWUL8mZH1F1lOCsuWsSnNFoxNKlS/Hggw9i0KBBNo9LTEzExo0bsWvXLnz66acwGo0YNWoUbt68afOc7OxsKJVK4RUdHe2MWyD3OL3RyiiG4rP8z4hBkMvlAJpLcDerGnDgagXOFFXbviiV4Np11yS4xYsX48KFC9i2bVu7x6WkpGDOnDlISkpCamoqtm/fjrCwMHzyySc2z1m+fDnUarXwKioqsnksIfYyd/Q1z8oLbQNQeZ3/78ihwiSYNaaB+Hfq+Zl6Q/xkFte5cEuNfVfKUFrTBEhpuFZ7PLoOzmzJkiX46quvcODAAfTq1atL50qlUgwbNgzXr1+3eYxcLhf+70mIsxhal+DKLgFGAxCgAvzDEaJvhEqpQA8//m+xvJbv29Y6wV0vq0N+RT3CAuSIoFbUdnl0CY4xhiVLlmDHjh34/vvvER8f3+VrGAwGnD9/HpGRkR0fTIgTGYRZeU0JroRvXIBqCAAgKsgHs0fGYEy/MOgMRpSom4TtLVEraud5dAlu8eLF2LJlC3bt2oWAgACUlJQAAJRKJXx8+H/0OXPmoGfPnsjOzgYArFmzBr/4xS/Qp08fVFdX45133kFBQQEWLFjgtvsgBGieKkks4oC6MqC2lJ/7Lbx/m2PPFFVDb2RQ+kgR7Cu12GfZD86c4Br4DsM2l+u6N3l0gtuwYQMAYOzYsRbbN23ahMzMTABAYWEhRC0GIFdVVWHhwoUoKSlBcHAwhg8fjkOHDmHAgAGuCpsQq8yNDGKOay699egNyHwtjtPqjTiWfwcAMDw22LJTMPi54wDTI685wTEjoGtsc617nUcnONaJAcT79u2zeP/BBx/ggw8+cFJEhNgv2FeGpJggBCvEwE1T1yXT42lLV0troTcwJMUEYUgvZZv9zSU4I18ClCoAXRP/mEoJzoJHJzhCvIlKqeDHmFZc4x8pZb5ASEKb4wb1VGJAZKDNtVMt6uAAvi+crsnU0BDmrPDvSh7dyECIVzI/nkYM5EtgVthKbkCrlbUAamhoB5XgCHERjd4AXVM9FOXX+S+elcfTzhgaHYREVQB8ZKbkSAnOJkpwhLjI6cJq5J/Zj2R9DRISegP+4XZdx08ugZ+8xVfX3BdOU+uAKL0LPaIS4iJGI0NY/VW+VdTO0ptVPkH8z6Zqx13TS1CCI8RV6svhp60AJxYD4fZ3Wyqv1eBQXgUu3DItJWie4pwSXBuU4AhxEZ9KvmuIVpnQre4c1Q1aHP35Di4Vm2a9UZi6kjSpaW2GVijBEeIKRgN8qn4CAGhCu9fpXGrq6KvV89OZCwlOrwX0Td26trehBEeIK1Rcg0jXAJ3YB8agtn3fukImaZXgxNLmEmGTulvX9jaU4AhxheIzMDCGMr9ESKTd67wgJDiDsXmjUA9HCa4lSnCEOFtjFXAnH0E+UoT0ecBiARl7tCnBAc2PqY3V3bq2t6F+cIQ4W+ERAEBk/EBEDk3s9uVkLQbb6w1GfvC9uatIY1W3r+9NqARHiDM1VgmLyiB2lEMuaU5wQIvHVN8e/M+GCod8hregEhwhzsIYcPVbfiqj4Dg0+kZBpDdAJha1mQKpK0QiDk+OjIZMLIJCYhqu5WcaZF9fQfPCtUAJjhBnMBqBvO+BOz8DIgnQ9xFsPnwDDVoDnkmJRah/96bIj1RazvIL3x58UtM18mNS5f7dur63oARHiKNo64HqQqCqgE9s5hbNfumAXw9o9fwkllKxE2qGxFK+JbWxin9MpQQHgBIcId1TXwmU/wRUXOGnIG9JLOWTm2ow9AajMKOvXNL9BPdTSQ2q6nXoG+HfXBr0C+UTXF0ZEBzX7c/wBpTgCOkqTS1QfhUoPQ/UFFvu8w8DguKA4FggKAaQ8MmnydSlg+MclOCKa5FfUY8AhaQ5wQVG8ZNp1tzq9vW9BSU4QjqiazQ9et7gHz8bKpv3cSIgJB4I7QeE9m2em60VjWkxZ7lE3K0GBjNf01xw9Rp988bAnvxPNSU4M0pwhFij1wKlF/jZd2tLLAexcxy/lmn4QCBigM2k1pK5BKeQOqb+zTwfXIMpcQIAAiL5hKup5ev/FG3Xc7jXUIIjpKXGKuDWSb7vml7TvN0vFAiK5eu2gqIBqY/NS1jTZEpECqn1Kcq7yjybb4OmRYKTyICACP6x+c7PQNQwh3zW3YwSHCGMAVX5wK1TQOX15tKaTzDQczgQlggoArv1EQFyCQZGBcJf7pivnJ+Mv47FIyrAPyrXFPN1hJTgKMGRe5iuiX8MvXXKsl4tJAHoNYL/6aAOs+GBCjwyUOWQawEQFoO+06AFY6y5Xi80Efh5P1BdYFpGsOPHZ292VwzVWr9+PeLi4qBQKJCcnIxjx461e/yXX36J++67DwqFAoMHD8Y333zjokiJx9NrgfIrwKX/Aw7/Cbi2l09uYimf1EYuAobO4hdk9uDRAMF+MnAcoNEZoWk56N6vBxAYCRgNQFH735N7gceX4D7//HNkZWXh448/RnJyMtauXYv09HRcuXIF4eFtF+04dOgQZs+ejezsbDz66KPYsmULpk2bhlOnTmHQoEFuuAPiMkYDX7muqQGaagBdA98Cqtfw/91Qyb9aNhj4hQI97wciBgldOpyhok6DIB+psCp9d0nFIjw6JBI9g3zb1uvFjAIu/C9w6wR/X/737lqpHOvM8vFulJycjAceeAAfffQRAMBoNCI6Ohq//e1v8corr7Q5ftasWaivr8dXX30lbPvFL36BpKQkfPzxx536zJqaGiiVSqjVagQGdq/uxSMxxs/8qq3nE0JTtelnixdjfEW6zA+QB5hegc0/JXJ+jKVRDxh0gFHH/zRoTT91/H6xjK/8Fsv5nxIFf65Yzv+3SGQZFzNaXsOos/FezyctTY0p5hp+4ePO/Dn7BPFdOsLu47tWOLmk1qQz4M8HfgZjwLzRcQhQSB3+GZV1GvgrJJBLxPzv4NwXfEODzA/ok8aXSJ2YwF2ts99Rjy7BabVanDx5EsuXLxe2iUQipKWl4fDhw1bPOXz4MLKysiy2paenY+fOnTY/R6PRQKNpbjGrqanpXIDF54Cbx9r/UlnsY56xz6DjE1NHdI1Aw52Oj+sOkQSAKbE54v+1IgnfPUIRyH+5JT78F1vqwzca+IfzSdpJapp02Hn6FgxGBiPjV9Jq0hlgMDKEBsidktyMRoYvT95Ek84Af7kEMokIUsMQJJQXQKG5haDiLYjt4cf/DiRyvisJuObE7mmP4g8scNilPDrBVVRUwGAwICIiwmJ7REQEfvrpJ6vnlJSUWD2+pKTE5udkZ2dj9erVXQ9Q1wjUlXf9PE8hkfOJQBFkSgpBzcmBE/MlJG09369KU8uXljQ1/H/rNXwyEYkBkZSvwxJL+RKbSML/5ESm0paWP95gWjNA38SXwADbiZYTWb+meZtIypcAzfEqlPxL6uvWLyxjQGWdts12uVSEh++zbx3UjlQ36iARcWAMqG0y/z7FKFNORlTtOQwQFQBg/N+rrtEpMXgqj05wrrJ8+XKLUl9NTQ2io6M7PjEssXnxXosvVasvmKftE0n4xYLFbvznNxr4pGfUQShNcCJTYpPxifMu5CcT45fDe0Ek4iDmOIg4fnqjQIVUmInX0UL8ZFjwUALqNXrUNumhMxhbFIbj+T5zcgP/PyuDxlRSdmCp2YN5dIILDQ2FWCxGaanlIObS0lKoVNab3FUqVZeOBwC5XA653I76CZ+g5plUSdeIxN1aOs9TScQiRIe4577arHjfmhf+vjvi0d1EZDIZhg8fjtzcXGGb0WhEbm4uUlJSrJ6TkpJicTwA7N271+bxhBDv5dElOADIyspCRkYGRowYgZEjR2Lt2rWor6/H3LlzAQBz5sxBz549kZ2dDQB4/vnnkZqaivfeew+TJ0/Gtm3bcOLECfz5z392520QQtzA4xPcrFmzUF5ejpUrV6KkpARJSUnYvXu30JBQWFgIUYuuBqNGjcKWLVvw+9//Hq+++ir69u2LnTt3Uh84Qu5BHt8Pzh28vh8cIXe5zn5HPboOjhBCuoMSHCHEa1GCI4R4LY9vZHAHc7Vkp4dsEUJcyvzd7KgJgRKcFbW1tQDQudEMhBC3qa2thVJpe2p2akW1wmg04vbt2wgICHDIAiGuYB5eVlRU5DUtv954TwDdlyMwxlBbW4uoqCiLbmKtUQnOCpFIhF69erk7DLsEBgZ61ZcG8M57Aui+uqu9kpsZNTIQQrwWJThCiNeiBOcl5HI5Vq1aZd+sKB7KG+8JoPtyJWpkIIR4LSrBEUK8FiU4QojXogRHCPFalOAIIV6LEtxd6s0338SoUaPg6+uLoKCgTp3DGMPKlSsRGRkJHx8fpKWl4dq1a84NtIvu3LmDp59+GoGBgQgKCsL8+fNRV1fX7jljx44Fx3EWr2effdZFEVu3fv16xMXFQaFQIDk5GceOtb/K/Jdffon77rsPCoUCgwcPxjfffOOiSLumK/eVk5PT5t9FoVC4MFpKcHctrVaLGTNm4De/+U2nz3n77bfx4Ycf4uOPP8bRo0fh5+eH9PR0NDU1OTHSrnn66adx8eJF7N27F1999RUOHDiARYsWdXjewoULUVxcLLzefvttF0Rr3eeff46srCysWrUKp06dwtChQ5Geno6ysjKrxx86dAizZ8/G/Pnzcfr0aUybNg3Tpk3DhQsXXBx5+7p6XwA/qqHlv0tBQYELIwbAyF1t06ZNTKlUdnic0WhkKpWKvfPOO8K26upqJpfL2datW50YYeddunSJAWDHjx8Xtv373/9mHMexW7du2TwvNTWVPf/88y6IsHNGjhzJFi9eLLw3GAwsKiqKZWdnWz1+5syZbPLkyRbbkpOT2a9//WunxtlVXb2vzv5tOhOV4O4R+fn5KCkpQVpamrBNqVQiOTkZhw8fdmNkzQ4fPoygoCCMGDFC2JaWlgaRSISjR4+2e+5nn32G0NBQDBo0CMuXL0dDQ4Ozw7VKq9Xi5MmTFr9nkUiEtLQ0m7/nw4cPWxwPAOnp6R7z7wLYd18AUFdXh9jYWERHR2Pq1Km4ePGiK8IV0GD7e0RJSQkACIv1mEVERAj73K2kpATh4Zarv0skEoSEhLQb41NPPYXY2FhERUXh3LlzePnll3HlyhVs377d2SG3UVFRAYPBYPX3/NNPP1k9p6SkxKP/XQD77isxMREbN27EkCFDoFar8e6772LUqFG4ePGiyyazoBKcB3nllVfaVMq2ftn6Y/Jkzr6vRYsWIT09HYMHD8bTTz+NzZs3Y8eOHcjLy3PgXZCuSklJwZw5c5CUlITU1FRs374dYWFh+OSTT1wWA5XgPMjvfvc7ZGZmtntMQkKCXddWqVQAgNLSUkRGRgrbS0tLkZSUZNc1O6uz96VSqdpUWOv1ety5c0eIvzOSk5MBANevX0fv3r27HG93hIaGQiwWo7S01GJ7aWmpzXtQqVRdOt4d7Lmv1qRSKYYNG4br1687I0SrKMF5kLCwMISFhTnl2vHx8VCpVMjNzRUSWk1NDY4ePdqlllh7dPa+UlJSUF1djZMnT2L48OEAgO+//x5Go1FIWp1x5swZALBI5K4ik8kwfPhw5ObmYtq0aQD4CVRzc3OxZMkSq+ekpKQgNzcXS5cuFbbt3bsXKSkpLoi4c+y5r9YMBgPOnz+PSZMmOTHSVtzaxEHsVlBQwE6fPs1Wr17N/P392enTp9np06dZbW2tcExiYiLbvn278P6tt95iQUFBbNeuXezcuXNs6tSpLD4+njU2NrrjFqyaMGECGzZsGDt69Cj78ccfWd++fdns2bOF/Tdv3mSJiYns6NGjjDHGrl+/ztasWcNOnDjB8vPz2a5du1hCQgIbM2aMu26Bbdu2jcnlcpaTk8MuXbrEFi1axIKCglhJSQljjLFnnnmGvfLKK8LxBw8eZBKJhL377rvs8uXLbNWqVUwqlbLz58+76xas6up9rV69mu3Zs4fl5eWxkydPsieffJIpFAp28eJFl8VMCe4ulZGRwQC0ef3www/CMQDYpk2bhPdGo5GtWLGCRUREMLlczsaPH8+uXLni+uDbUVlZyWbPns38/f1ZYGAgmzt3rkXSzs/Pt7jPwsJCNmbMGBYSEsLkcjnr06cPe/HFF5larXbTHfD+9Kc/sZiYGCaTydjIkSPZkSNHhH2pqaksIyPD4vgvvviC9evXj8lkMjZw4ED29ddfuzjizunKfS1dulQ4NiIigk2aNImdOnXKpfHSdEmEEK9FraiEEK9FCY4Q4rUowRFCvBYlOEKI16IERwjxWpTgCCFeixIcIcRrUYIj97ycnByLWZFff/11i/G5mZmZwvAkcnehBEdcJjMz0+Z04osXLwbHcRaD8p2RWOLi4rB27VqLbbNmzcLVq1dtnrNu3Trk5OQI78eOHWsxbpR4LkpwxKWio6Oxbds2NDY2CtuampqwZcsWxMTEuCUmHx+fNvPQtaRUKju97gXxLJTgiEvdf//9iI6OtpiMcvv27YiJicGwYcO6dW1rJatp06YJpcKxY8eioKAAL7zwgjAPHdD2EbW1liXJzMxM7N+/H+vWrROukZ+fjz59+uDdd9+1OO/MmTPgOM6l0wMRS5TgiMvNmzcPmzZtEt5v3LgRc+fOdfrnbt++Hb169cKaNWuERVC6at26dUhJSbFY5CYmJqbNPQHApk2bMGbMGPTp08dRt0C6iBIccblf/epX+PHHH1FQUICCggIcPHgQv/rVr5z+uSEhIRCLxQgICIBKpbJrQkmlUgmZTAZfX1/hGmKxGJmZmbhy5YqwjJ5Op8OWLVswb948R98G6QKa8JK4XFhYGCZPnoycnBwwxjB58mSEhoa6O6xuiYqKwuTJk7Fx40aMHDkS//rXv6DRaDBjxgx3h3ZPoxIccYt58+YhJycHf//73x1WyhGJRGg9+5dOp3PItTtjwYIFQgPKpk2bMGvWLPj6+rrs80lblOCIW0yYMAFarRY6nQ7p6ekOuWZYWJhFvZrBYGizeLJMJoPBYOjW59i6xqRJk+Dn54cNGzZg9+7d9HjqAegRlbiFWCzG5cuXhf+2Ra1WC2ssmPXo0QPR0dFtjn344YeRlZWFr7/+Gr1798b777+P6upqi2Pi4uJw4MABPPnkk5DL5XY9GsfFxeHo0aO4ceMG/P39ERISApFIJNTFLV++HH379vWoNRXuVVSCI24TGBiIwMDAdo/Zt28fhg0bZvFavXq11WPnzZuHjIwMzJkzB6mpqUhISMC4ceMsjlmzZg1u3LiB3r17273Az7JlyyAWizFgwACEhYWhsLBQ2Dd//nxotVqXtAqTjtGU5YQ40H/+8x+MHz8eRUVFbRZJJq5HCY4QB9BoNCgvL0dGRgZUKhU+++wzd4dEQI+ohDjE1q1bERsbi+rqarz99tvuDoeYUAmOEOK1qARHCPFalOAIIV6LEhwhxGtRgiOEeC1KcIQQr0UJjhDitSjBEUK8FiU4QojXogRHCPFa/x9oNW2CXUlNgwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:36:30.377865Z", + "iopub.status.busy": "2024-03-22T15:36:30.377550Z", + "iopub.status.idle": "2024-03-22T15:36:30.599022Z", + "shell.execute_reply": "2024-03-22T15:36:30.598077Z" + }, + "papermill": { + "duration": 0.244382, + "end_time": "2024-03-22T15:36:30.601055", + "exception": false, + "start_time": "2024-03-22T15:36:30.356673", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEmCAYAAAAdlDeCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1SElEQVR4nO3deVhT17oG8JcxBMNQFMEBDBYrzigowqlTGRSUI8XWobQO5djWlk44VOxR9OmtVAtqtVZubakH61iL1EsdSB25ioCgHhFERagTEZVCkCGEsO4f3uxjhEBQICT7+z1Pn7LXXjv5Ngmve1zbiDHGQAghBs5Y1wUQQkhHoLAjhPAChR0hhBco7AghvEBhRwjhBQo7QggvUNgRQniBwo4Qwgumui6gs2toaMDdu3dhZWUFIyMjXZdDCHkCYwyVlZXo2bMnjI2b33ajsGvB3bt34eTkpOsyCCHNuHXrFnr37t1sHwq7FlhZWQF4/Mu0trbWcTXtR6FQIDU1FQEBATAzM9N1OeQ58eXzlMlkcHJy4v5Om6N3Ybd582Z8/fXXkEqlGDZsGDZt2oRRo0Y12Xfr1q1ITExEbm4uAMDDwwOrV6/W2L8pql1Xa2trgw87S0tLWFtbG/QfB1/w7fPU5hCTXp2g2LNnDyIjIxEdHY2cnBwMGzYMEydORGlpaZP9T5w4gVmzZuH48eNIT0+Hk5MTAgICcOfOnQ6unBCia3oVduvWrcP8+fMxb948DBw4EPHx8bC0tERCQkKT/Xfs2IH3338f7u7ucHNzww8//ICGhgYcPXq0gysnhOia3uzG1tXVITs7G1FRUVybsbEx/Pz8kJ6ertVrVFdXQ6FQwM7OTmMfuVwOuVzOTctkMgCPdwsUCsUzVt/5qdbNkNeRT/jyebZm/fQm7B48eAClUgkHBwe1dgcHB1y5ckWr1/jss8/Qs2dP+Pn5aewTExODVatWNWpPTU2FpaVl64rWQxKJRNclkDZk6J9ndXW11n31Juye11dffYXdu3fjxIkTsLCw0NgvKioKkZGR3LTqbE9AQIDBn6CQSCTw9/fnxQFtQ8eXz1O156UNvQm7bt26wcTEBPfu3VNrv3fvHhwdHZtdNjY2Fl999RX++OMPDB06tNm+AoEAAoGgUbuZmZlBf2lU+LKefGHon2dr1k1vws7c3BweHh44evQoQkJCAIA72RAREaFxubVr1+LLL7/EkSNH4Onp2UHVdn7V1dVqu/+PauQ4c6kQL3Q7B5HwP2Hv5ubGi913Yvj0JuwAIDIyEnPmzIGnpydGjRqFDRs2oKqqCvPmzQMAzJ49G7169UJMTAwAYM2aNVixYgV27twJsVgMqVQKABCJRBCJRDpbj87gypUr8PDwaNS+9qnp7OxsjBgxomOKIqQd6VXYzZgxA/fv38eKFSsglUrh7u6Ow4cPcyctbt68qXZ/3JYtW1BXV4fXXntN7XWio6OxcuXKjiy903Fzc0N2djY3XVBSjshfLmHd60PQv4etWj9CDIFehR0AREREaNxtPXHihNp0cXFx+xekpywtLdW22Iz/fAhBWg0GDB4G9z5ddVgZIe1Dry4qJoSQZ6V3W3aEkMa0PeEE8PekE4UdIQZA2xNOAH9POlHYEWIAtD3hpOrLRxR2hBgAOuHUMjpBQQjhBdqyI0RPFT2oQpW8vsl5hferuP+bmmr+M+8iMIVLty7tUl9nQ2FHiB4qelCFCbEnWuy3cN+lFvscXzSeF4FHYUeIHlJt0W2Y4Q7X7o1vfayqkSPlRDqmjPdGF2HjgS0A4HrpI3yy54LGrUNDQ2FHiB5z7S7C4F42jdoVCgWk9sCIPi8Y9KgnrUEnKAghvEBhRwjhBdqNJUQPyZW1MLa4gyJZAYwtGh+zq6+vx936u8gvy9d4NrZI9gjGFncgV9YCaLwrbGgo7HiiucsUALpUQd/crfoTXVw2YVlm8/2+O/xds/O7uAB3q9zhAYdm+xkCCjse0PYyBYAuVdAXPbv0QVXRh/hmhjtebOJsbH19PU7/72n87eW/afzHq7D0ET7ecwE9J/Rp73I7BQo7HmjpMgWALlXQNwITCzTU9oKLdX8M7Nr02dgi0yIMsBug8WxsQ20FGmrvQ2Ci+QFUhoTCjkc0XaYA0KUKxPDR2VhCCC9Q2BFCeIHCjhDCCxR2hBBeoLAjhPAChR0hhBco7AghvEBhRwjhBQo7QggvUNgRQniBwo4Qwgt0bywPtDT2GUDjnxHDR2HHA9qOfQbQ+GfEcOld2G3evBlff/01pFIphg0bhk2bNmHUqFEa+//yyy9Yvnw5iouL0a9fP6xZswZBQUEdWLHuvWDeG1VFH+LDCa4ah3iqkdch7dwljPEcAqHAvMk+t8qqESu5ypvxz4hh0auw27NnDyIjIxEfHw8vLy9s2LABEydOREFBAbp3796o/5kzZzBr1izExMRgypQp2LlzJ0JCQpCTk4PBgwfrYA104/bDejTU9sI3h2oA1DTT0xn/c72ihVfrBTvLpgOTdJwahRIAkHun6c+rqkaOc/cBxz//anZ8Qj4xYowxXRehLS8vL4wcORLffvstAKChoQFOTk748MMPsXTp0kb9Z8yYgaqqKqSkpHBto0ePhru7O+Lj47V6T5lMBhsbG1RUVMDa2rptVqSDlVXVIfWyFC92F0FoZtJkn4KSCizcdwlxrw1B/x6aj8fRsOydw+7Mm1ia1PKo0trQ55GnW/P3qTdbdnV1dcjOzkZUVBTXZmxsDD8/P6Snpze5THp6OiIjI9XaJk6ciOTk5PYstdOx62KOmaOc1dqqq6tx5coVbrruXjnk0uuouydEnbEt1+7m5gZLS8uOKpVoKWCQIwBo/AeM/vFqTG/C7sGDB1AqlXBwUD8w7uDgoPZH+ySpVNpkf6lUqvF95HI55HI5Ny2TyQA8HslXoVA8a/mdTm5uLry8vBq1v/Ev9emMjAwMHz68g6oi2rIyN8K04T00zq+trQUA9HlBgP7dm//HSp+/162pXW/CrqPExMRg1apVjdpTU1MNagtHLpcjLi6Om1Y0AGW1gJ0FYPbE1ZfFxcUoKSnRQYXkedx6BACmOHv2LO7k6rqa9lNdXa11X70Ju27dusHExAT37t1Ta7937x4cHR2bXMbR0bFV/QEgKipKbddXJpPByckJAQEBenvMThsKhQISiQT+/v70DAoDcPFmGXDpHEaPHo1hzna6LqfdqPa8tKE3YWdubg4PDw8cPXoUISEhAB6foDh69CgiIiKaXMbb2xtHjx7FJ598wrVJJBJ4e3trfB+BQACBoPHZKzMzM16EAF/W09CpLgw3NTU16M+zNeumN2EHAJGRkZgzZw48PT0xatQobNiwAVVVVZg3bx4AYPbs2ejVqxdiYmIAAB9//DHGjRuHuLg4TJ48Gbt378a5c+fw/fff63I1CCE6oFdhN2PGDNy/fx8rVqyAVCqFu7s7Dh8+zJ2EuHnzJoyN/3PAycfHBzt37sQ///lPLFu2DP369UNycjKvrrEjhDymV2EHABERERp3W0+cONGo7fXXX8frr7/ezlURQjo7GvWEEMILFHaEEF6gsCOE8AKFHSGEFyjsCCG8QGFHCOEFCjtCCC9Q2BFCeIHCjhDCCxR2hBBeoLAjhPAChR0hhBco7AghvEBhRwjhBQo7QggvUNgRQniBwo4QwgsUdoQQXqCwI4TwAoUdIYQXKOwIIbxAYUcI4QUKO0IIL1DYEUJ4gcKOEANTU1ODmOVLcG/PcsQsX4Kamhpdl9QpUNgRYkBCQkJgaWmJvdsTUFt8Hnu3J8DS0hIhISG6Lk3nKOwIMRAhISH47bffmpz322+/8T7wKOwIMQA1NTUag07lt99+4/UuLYUdIQbgo48+0qrf7NmzUV1d3c7VdE4UdoQYgLS0NK367du3D1euXGnnajqnVofdjRs32qMOQshzqKys1Kqfvb093Nzc2rmazqnVYefq6ooJEybg559/Rm1tbXvU1KSysjKEhYXB2toatra2CA8Px6NHj5rt/+GHH6J///4QCoVwdnbGRx99hIqKig6rmZCOYmFhoVU/KysrWFpatnM1nVOrwy4nJwdDhw5FZGQkHB0d8e677yIzM7M9alMTFhaGy5cvQyKRICUlBadOncI777yjsf/du3dx9+5dxMbGIjc3F9u2bcPhw4cRHh7e7rUS0tG03fDoyA2UToc9I4VCwX799VcWHBzMzMzM2KBBg1hcXBwrLS191pfUKC8vjwFgWVlZXNuhQ4eYkZERu3Pnjtavs3fvXmZubs4UCoXWy1RUVDAArKKiolU165u6ujqWnJzM6urqdF0KeQZ2dnYMQIv/2dnZ6brUNtWav0/TZw1JU1NThIaGYvLkyfjuu+8QFRWFRYsWYdmyZZg+fTrWrFmDHj16PG8WAwDS09Nha2sLT09Prs3Pzw/GxsbIyMjAq6++qtXrVFRUwNraGqammldbLpdDLpdz0zKZDACgUCigUCiecQ06P9W6GfI6GjLGmNb9DOkzbs26PHPYnTt3DgkJCdi9eze6dOmCRYsWITw8HLdv38aqVaswderUNtu9lUql6N69u1qbqakp7OzsIJVKtXqNBw8e4Isvvmh21xcAYmJisGrVqkbtqampvDjWIZFIdF0CeQbaXk5SXV2NgwcPtnM1Hac1l9G0OuzWrVuHn376CQUFBQgKCkJiYiKCgoJgbPz48J+Liwu2bdsGsVjc4mstXboUa9asabZPfn5+a0tsRCaTYfLkyRg4cCBWrlzZbN+oqChERkaqLevk5ISAgABYW1s/dy2dlUKhgEQigb+/P8zMzHRdDmmlJ/dGWuoXFBTUztV0HNWelzZaHXZbtmzB22+/jblz52rcTe3evTt+/PHHFl9r4cKFmDt3brN9+vbtC0dHR5SWlqq119fXo6ysDI6Ojs0uX1lZiUmTJsHKygr79+9v8Q9ZIBBAIBA0ajczM+NFCPBlPfnMkD7f1qxLq8NOIpHA2dmZ25JTYYzh1q1bcHZ2hrm5OebMmdPia9nb28Pe3r7Fft7e3igvL0d2djY8PDwAAMeOHUNDQwO8vLw0LieTyTBx4kQIBAIcOHBA69PzhBDD0+pLT1588UU8ePCgUXtZWRlcXFzapKinDRgwAJMmTcL8+fORmZmJ06dPIyIiAjNnzkTPnj0BAHfu3IGbmxt3nFAmkyEgIABVVVX48ccfIZPJIJVKIZVKoVQq26VOQnTF3d29TfsZolZv2Wk66/Po0aN23XLasWMHIiIi4OvrC2NjY0ybNg0bN27k5isUChQUFHAHLHNycpCRkQHg8YXQTyoqKtLqmCIh+kIoFLZpP0OkddipDtobGRlhxYoVamcmlUolMjIy2vVfDTs7O+zcuVPjfLFYrBbE48eP1/p0PCH6jsKuZVqH3fnz5wE83rK7dOkSzM3NuXnm5uYYNmwYFi1a1PYVEkJaRHdQtEzrsDt+/DgAYN68efjmm28M+jIMQvRN165d27SfIWr1MbuffvqpPeoghDwHbS9458OF8ZpoFXahoaHYtm0brK2tERoa2mzfpKSkNimMEKI9bYd40rafIdIq7GxsbGBkZMT9TAjpXC5evKg2bW1tDYVCATMzM7W7DJ7uxydahd2Tu660G0tI5/P0bVOq6aefOdGa26sMDQ3LTogB0PYaVz7fRaTVlt3w4cO53diW5OTkPFdBhJDWc3Fxwf3797npESNGwMLCArW1tWp/k+11l5M+0Crs+P68SUI6u6fvVde00fF0Pz7RKuyio6Pbuw5CyHOoq6tr036GiL8xT4gBsbOza9N+hkirLTs7OztcvXoV3bp1wwsvvNDs8buysrI2K44Qop3x48fjjz/+0KofX2kVduvXr4eVlRX3s7YnKwghHePpEb1HjBgBoVCImpoateN3bTHyt77SKuyeHIizpZGFCSEd7+lnKGs6QdHcs5YNXauP2ZmYmDQaIh0AHj58CBMTkzYpihDSOqpBbB0dHdG7d2+1eU5OTtzjC1T9+KjVYadpjDi5XK427BMhpON4e3sDePwkvqdHEr9//z73FD5VPz7SetQT1ajARkZG+OGHHyASibh5SqUSp06dgpubW9tXSAhpkZOTE/fz02PWPTn9ZD++0Trs1q9fD+Dxll18fLzaLqu5uTnEYjHi4+PbvkJCSIt8fHxgamoKY2PjJq+lMzc3R0NDA3x8fHRQXeegddgVFRUBACZMmICkpCS88MIL7VYUIaR1zpw5g/r6eo3zVQF45swZ3l5+0urBO1UjFhNCOo87d+60aT9D1Oqwe/vtt5udn5CQ8MzFEEKejSrETE1NIZPJcPr0aRw6dAiBgYH429/+BisrKyiVSgq71vjrr7/UphUKBXJzc1FeXo5XXnmlzQojhGhPIpEAeHwCQiAQYNy4caiqqsK4ceNgYmICZ2dnFBUVQSKRYMmSJTquVjdaHXb79+9v1NbQ0IAFCxbgxRdfbJOiCCGtU15eDuDxsfWQkBAsXrwYNTU1OHv2LL7++mvumLuqHx+1OuyaYmxsjMjISIwfP563/2oQokuenp44d+4cHB0dcfHiRYwdO5ab16dPHzg6OkIqlcLT01OHVepWm416UlhY2OzZIEJI+1m3bh2AxxcVP32H071797iLilX9+KjVW3aRkZFq04wxlJSU4Pfff1e7h5YQ0nGEQiFGjhyJrKwsjRcVjxw5EkKhUBfldQqtDrvz58+rTRsbG8Pe3h5xcXEtnqklhLQPpVKJ+/fvw8bGBhUVFY3m29jY4MGDB1Aqlby9h52usyPEAKSlpaG4uBhGRkYIDAxEZWUl/vzzT/Tp0wdWVlY4fPgwKioqkJaWRhcVE0L0l+r6OXd3d1y6dAm3b98GANy6dQu9e/eGu7s7zp8/T9fZEUL0m+rJYk8fZgKA27dvc+H35BPI+IaeQUGIAejatWub9jNEehN2ZWVlCAsLg7W1NWxtbREeHq71qKuMMQQGBsLIyAjJycntWyghOnD37l3u56cfl/jk9JP9+KbNwu727dt455132urlGgkLC8Ply5chkUiQkpKCU6dOaf1+GzZsoOdmEIOWmprK/fz0ILoCgaDJfnzTZsfsHj58iB9//BHff/99W70kJz8/H4cPH0ZWVhZ3BfimTZsQFBSE2NjYZoeavnDhAuLi4nDu3Dn06NGjzWsjpDNQHZMDAF9fXwQEBODatWvo168fUlNT8fvvvzfqxzd6cYIiPT0dtra2are6+Pn5wdjYGBkZGXj11VebXK66uhpvvPEGNm/ezI3B3xK5XA65XM5Ny2QyAI8HPFAoFM+xFp2bat0MeR0NmWrkcFtbW+Tm5nLhBgBisRi2trYoLy+HSCQyqM+4NeuiF2EnlUrRvXt3tTZTU1PY2dlxt8E05dNPP4WPjw+mTp2q9XvFxMRg1apVjdpTU1NhaWmpfdF6SjV6BtEvzs7OyMnJQXl5OVxcXBAQEACBQAC5XI5z586huLiY63fw4EHdFtuGqqurte6r07BbunQp1qxZ02yfZ33O5YEDB3Ds2LEmT8U3JyoqSu2WOJlMBicnJwQEBMDa2vqZatEHCoUCEokE/v7+MDMz03U5pJUsLCy4k2/nz5/X+L1/7733DGooNtWelza0DrvQ0NBm5z/L0DELFy5s8Tm0ffv2haOjY6Obm+vr61FWVqZx9/TYsWMoLCyEra2tWvu0adMwZswYnDhxosnlBAKB2gFdFTMzM16EAF/W09D4+fnB3t6+2evounfvDj8/P4O6Xaw131Wtw87GxqbF+bNnz9b6jQHA3t4e9vb2Lfbz9vZGeXk5srOz4eHhAeBxmDU0NMDLy6vJZZYuXYp//OMfam1DhgzB+vXrERwc3Ko6CensTExMEB8fj2nTpmnss2XLFoMKutbSOux++umn9qyjWQMGDMCkSZMwf/58xMfHQ6FQICIiAjNnzuTOxN65cwe+vr5ITEzEqFGj4Ojo2ORWn7OzM1xcXDp6FQjpMBYWFmojnwiFQtTU1Oiwos5Bby4q3rFjB9zc3ODr64ugoCC8/PLLape5KBQKFBQUtOqAJSGGQqlUYuHChQgODoZMJoNEIkFkZCQkEgkqKioQHByMRYsWQalU6rpUndF6y07b4Zva64E7dnZ22Llzp8b5YrEYjLFmX6Ol+YToK9WoJ7t27YKZmZnaMyjMzMwQFRUFHx8fGvVEG9u2bUOfPn0wfPhwCg1COpmSkhIAwODBg5ucr2pX9eMjrcNuwYIF2LVrF4qKijBv3jy8+eabsLOza8/aCCFaUt0dlJubi9GjRzean5ubq9aPj7Q+Zrd582aUlJRgyZIl+J//+R84OTlh+vTpOHLkCG3pEaJjY8aMgVgsxurVq9HQ0KA2r6GhATExMXBxccGYMWN0VKHuteoEhUAgwKxZsyCRSJCXl4dBgwbh/fffh1gs1noEEkJI2zMxMUFcXBxSUlIQEhKCs2fPco9SDAkJQUpKCmJjY+nSk2dhbGwMIyMjMMZ4fYaHkM4iNDQU+/btw8KFC9Uepeji4oJ9+/a1eGOAoWvVlp1cLseuXbvg7++Pl156CZcuXcK3336LmzdvcjciE0J0JzQ0FNevX1e79OTatWu8DzqgFVt277//Pnbv3g0nJye8/fbb2LVrF7p169aetRFCnoGJiYnapSd83nV9ktZhFx8fD2dnZ/Tt2xcnT57EyZMnm+yXlJTUZsURQlpPqVTi5MmTOHXqFLp06YIJEyZQ4KEVYTd79mwa7ZeQTi4pKQkLFy7khnRat24dxGIx4uLieL8r26qLigkhnVdSUhJee+01TJkyBdu3b8ft27fRu3dvrF27Fq+99hrvT1Lozb2xhBDNVPfGTpkyBcnJyfDy8oJQKISXlxeSk5MxZcoU3t8bS2FHiAFQ3Ru7bNmyJp8uFhUVhaKiIqSlpemoQt2jsCPEADx5b+yTJyhOnjwJpVJJ98ZCT55BQQhpnuqe12+//Rb//d//3egEheqxo3RvLCFEr40ZMwb29vaIiorC4MGDkZaWhl27diEtLQ2DBw/GsmXL0L17d7o3lhCi/568NEw1OAcN0vEfFHaEGIC0tDSUlpYiJiYGubm5GDt2LGbNmoWxY8fi8uXLWL16NUpLS+kEBSFEv6lOPERERDR5b2xERIRaPz6isCPEADw5eKfq3tixY8dy98bS4J0UdoQYBBq8s2UUdoQYABq8s2V0nR0hBoIG72wehR0hBiQ0NBRTp07F8ePHcejQIQQGBtIQT/+Pwo4QA0ODdzaNjtkRQniBwo4QwgsUdoQQXqCwI4TwAoUdIYQXKOwIIbxAYUcI4QW9CbuysjKEhYXB2toatra2CA8Px6NHj1pcLj09Ha+88gq6dOkCa2trjB07FjU1NR1QMSGkM9GbsAsLC8Ply5chkUiQkpKCU6dOcUNNa5Keno5JkyYhICAAmZmZyMrKQkRERKMHkhBCDJ9e3EGRn5+Pw4cPIysrC56engCATZs2ISgoCLGxsejZs2eTy3366af46KOPsHTpUq6tf//+HVIzIaRz0YuwS09Ph62tLRd0AODn5wdjY2NkZGTg1VdfbbRMaWkpMjIyEBYWBh8fHxQWFsLNzQ1ffvklXn75ZY3vJZfLIZfLuWmZTAYAUCgUUCgUbbhWnYtq3Qx5HfmEL59na9ZPL8JOKpWie/fuam2mpqaws7ODVCptcpkbN24AAFauXInY2Fi4u7sjMTERvr6+yM3NRb9+/ZpcLiYmBqtWrWrUnpqaCktLy+dck85PIpHougTynJRKJfLy8vDXX3/h0qVLGDhwoMHeH1tdXa11X52G3dKlS7FmzZpm++Tn5z/Ta6sGMHz33Xcxb948AMDw4cNx9OhRJCQkICYmpsnloqKiEBkZyU3LZDI4OTkhICAA1tbWz1SLPlAoFJBIJPD394eZmZmuyyHPaP/+/fjss8+4RykCgFgsxpo1a5rcA9J3qj0vbeg07BYuXIi5c+c226dv375wdHREaWmpWnt9fT3Kysrg6OjY5HKq4acHDhyo1j5gwADcvHlT4/sJBAIIBIJG7WZmZrwIAb6spyFKSkrCzJkzMWXKFGzfvh23b99G7969sXbtWsycOdMgx7RrzXdVp2Fnb28Pe3v7Fvt5e3ujvLwc2dnZ8PDwAAAcO3YMDQ0N8PLyanIZsViMnj17oqCgQK396tWrCAwMfP7iCelElEolFi5ciClTpiA5ORlKpRIPHz6El5cXkpOTERISgkWLFmHq1KkGu0vbEr24BmPAgAGYNGkS5s+fj8zMTJw+fRoRERGYOXMmdyb2zp07cHNzQ2ZmJoDHz9BcvHgxNm7ciH379uH69etYvnw5rly5gvDwcF2uDiFtLi0tDcXFxVi2bFmjS6uMjY0RFRWFoqIiXj9KUS9OUADAjh07EBERAV9fXxgbG2PatGnYuHEjN1+hUKCgoEDtgOUnn3yC2tpafPrppygrK8OwYcMgkUjw4osv6mIVCGk3qkckDh48uMn5qnY+P0pRb8LOzs4OO3fu1DhfLBY3+fTzpUuXql1nR4ghevJRiqNHj240nx6lqCe7sYSQ5j35KEWFQoGTJ0/i1KlTOHnyJBQKBT1KEXq0ZUcI0Uz1KMXXXnsN1tbWqK2tBQCsW7cOFhYWkMvl2LdvH29PTgC0ZUeIQWGMqd0BBDy+K6ipQzx8Q2FHiAFQKpV47733AAAWFhZq81TTCxYsgFKp7PDaOgsKO0IMwIkTJ3D//n0AgK+vL9LS0rBr1y6kpaXB19cXwOP7xU+cOKHDKnWLwo4QA3Ds2DEAwOjRo5GUlITa2lpkZWWhtrYWSUlJ3BlaVT8+ohMUhBiAW7duAXh8Pd1LL73E3Ru7bt06iMVivPLKKzh79izXj49oy44QA+Dk5AQA+OGHHzB48GC13djBgwcjISFBrR8fUdgRYgDGjRvH/cwY486+Pvnz0/34hnZjCTEAT14/d+zYMfz+++/ctFAobLIf39CWHSEG4Okh0J5kZGSkVT9DR2FHiAFQ3fMaExPTaFTv7t27Y/Xq1Wr9+IjCjhADoLo39syZM7h27RokEgkiIyMhkUhw9epVpKen8/7eWAo7QgyA6t7YlJQUhIaGIi8vD3V1dcjLy0NoaChSUlIQGxvL62N2dIKCEAMRGhqKRYsWYf369UhJSeHaTU1NsWjRIoMbkr21KOwIMRBJSUmIjY3F5MmT4e/vj2vXrqFfv36QSCSIjY3F6NGjeR14RoyGQ2iWTCaDjY0NKioqDP7pYgcPHkRQUBA9cEcPKZVKuLq6YsiQIdwzKFSfp4mJCUJCQpCbm4tr164Z1K5sa/4+6ZgdIQaAnkHRMgo7QgwAPYOiZRR2hBiAJ59B0RR6BgWFHSEG4clnUDQ0NKjNa2hooGdQgMKOEIPw5HV2ISEhOHv2LGpqanD27FmEhITQdXagS08IMRihoaHYt28fFi5ciLFjx3LtLi4u2LdvH68vOwEo7AgxKKGhoZg6dSqOHz+OQ4cOITAwEBMmTOD1Fp0KhR0hBsbExATjxo1DVVUVxo0bR0H3/+iYHSGEFyjsCCG8QGFHCOEFCjtCCC9Q2BFCeIHCjhDCC3oTdmVlZQgLC4O1tTVsbW0RHh6OR48eNbuMVCrFW2+9BUdHR3Tp0gUjRozAr7/+2kEVE0I6E70Ju7CwMFy+fBkSiQQpKSk4deoU3nnnnWaXmT17NgoKCnDgwAFcunQJoaGhmD59Os6fP99BVRNCOg2mB/Ly8hgAlpWVxbUdOnSIGRkZsTt37mhcrkuXLiwxMVGtzc7Ojm3dulXr966oqGAAWEVFResL1yN1dXUsOTmZ1dXV6boU8pzkcjmLjY1lQUFBLDY2lsnlcl2X1G5a8/epF3dQpKenw9bWFp6enlybn58fjI2NkZGRgVdffbXJ5Xx8fLBnzx5MnjwZtra22Lt3L2prazF+/HiN7yWXyyGXy7lpmUwG4PFIvgqFom1WqBNSrZshryMfLF26FBs2bOBGPjl48CCWLFmCTz75BF999ZWOq2t7rfm+6kXYSaXSRs/CNDU1hZ2dHaRSqcbl9u7dixkzZqBr164wNTWFpaUl9u/fD1dXV43LxMTEYNWqVY3aU1NTYWlp+ewroSckEomuSyDPaNu2bUhOTm7U3tDQgHXr1uHGjRuYO3duh9fVnqqrq7Xuq9OwW7p0KdasWdNsn/z8/Gd+/eXLl6O8vBx//PEHunXrhuTkZEyfPh1paWkYMmRIk8tERUUhMjKSm5bJZHByckJAQIDBP4NCIpHA39+fnkGhh+rq6jTu4aj89ttv+Pnnn2Fubt5BVbU/1Z6XNnQadgsXLmzxX5q+ffvC0dERpaWlau319fUoKyuDo6Njk8sVFhbi22+/RW5uLgYNGgQAGDZsGNLS0rB582bEx8c3uZxAIIBAIGjUbmZmxosQ4Mt6GpoNGzaAPfHsrDfffBMeHh7Izs7Gzz//DABgjOG7777D4sWLdVVmm2vNd1WnYWdvbw97e/sW+3l7e6O8vBzZ2dnw8PAAABw7dgwNDQ3w8vJqchnV5u3TDx8xMTFpNJIrIfouKSmJ+7mmpgYmJiY4ePAgPvjgA2zduhVCoZDrZ0hh1xp6cenJgAEDMGnSJMyfPx+ZmZk4ffo0IiIiMHPmTPTs2RMAcOfOHbi5uSEzMxMA4ObmBldXV7z77rvIzMxEYWEh4uLiIJFIEBISosO1IaTtFRQUAABGjBgBCwsLtXkWFhZwd3dX68dHehF2ALBjxw64ubnB19cXQUFBePnll/H9999z8xUKBQoKCrgtOjMzMxw8eBD29vYIDg7G0KFDkZiYiH/9618ICgrS1WoQ0i5Ux+GuXLmC+vp6tXn19fW4evWqWj8+0ouzsQBgZ2eHnTt3apwvFovVjlkAQL9+/eiOCcILI0eOREpKCqqrq9GrVy+sXLkSFhYW+OGHH7By5UpuI2DkyJE6rlR39CbsCCGa7dq1C1ZWVgCA0tJSvP/++xr78ZXe7MYSQjQTiUQtbrWNHDkSIpGogyrqfCjsCDEQmZmZGgNv5MiR3Mk7vqKwI8SAZGZmorKyEsHBwejTpw+Cg4NRWVnJ+6AD6JgdIQZHJBLh119/xcGDBxEUFEQXif8/2rIjhPAChR0hhBco7AghvEDH7FqgulC5NaMr6COFQoHq6mrIZDI6xmMA+PJ5qv4un76hoCkUdi2orKwEADg5Oem4EkKIJpWVlbCxsWm2jxHTJhJ5rKGhAXfv3oWVlRWMjIx0XU67UY3bd+vWLYMet48v+PJ5MsZQWVmJnj17Nhrh6Gm0ZdcCY2Nj9O7dW9dldBhra2uD/uPgGz58ni1t0anQCQpCCC9Q2BFCeIHCjgB4PBx9dHR0k0PSE/1Dn2djdIKCEMILtGVHCOEFCjtCCC9Q2BFCeIHCro3NnTu3zZ9eNn78eHzyySfN9hGLxdiwYUObvi8hhoTCrhnahAzRLytXruQeK9jZdLbvW2er53lR2BFiQOrq6nRdQqdFYafB3LlzcfLkSXzzzTcwMjKCkZERCgsLER4eDhcXFwiFQvTv3x/ffPNNk8uvWrUK9vb2sLa2xnvvvaf1l7CqqgqzZ8+GSCRCjx49EBcX16hPaWkpgoODIRQK4eLigh07djTqY2RkhC1btiAwMBBCoRB9+/bFvn37uPnFxcUwMjLC3r17MWbMGAiFQowcORJXr15FVlYWPD09IRKJEBgYiPv372v5WwMSEhIwaNAgCAQC9OjRAxEREdy8mzdvYurUqRCJRLC2tsb06dNx7949br5qq2v79u0Qi8WwsbHBzJkzucEYgMf3Kq9duxaurq4QCARwdnbGl19+yc3/7LPP8NJLL8HS0hJ9+/bF8uXLoVAoAADbtm3DqlWrcPHiRe4z3bZtm9br1p6e9fumOmzy5ZdfomfPnujfvz8A4MyZM3B3d4eFhQU8PT2RnJwMIyMjXLhwgVs2NzcXgYGBEIlEcHBwwFtvvYUHDx5orKe4uLijfh3tg5EmlZeXM29vbzZ//nxWUlLCSkpKWG1tLVuxYgXLyspiN27cYD///DOztLRke/bs4ZabM2cOE4lEbMaMGSw3N5elpKQwe3t7tmzZMq3ed8GCBczZ2Zn98ccf7N///jebMmUKs7KyYh9//DHXJzAwkA0bNoylp6ezc+fOMR8fHyYUCtn69eu5PgBY165d2datW1lBQQH75z//yUxMTFheXh5jjLGioiIGgLm5ubHDhw+zvLw8Nnr0aObh4cHGjx/P/vd//5fl5OQwV1dX9t5772lV+3fffccsLCzYhg0bWEFBAcvMzORqUiqVzN3dnb388svs3Llz7OzZs8zDw4ONGzeOWz46OpqJRCIWGhrKLl26xE6dOsUcHR3VfndLlixhL7zwAtu2bRu7fv06S0tLY1u3buXmf/HFF+z06dOsqKiIHThwgDk4OLA1a9Ywxhirrq5mCxcuZIMGDeI+0+rqaq3Wrb097/ftrbfeYrm5uSw3N5dVVFQwOzs79uabb7LLly+zgwcPspdeeokBYOfPn2eMMfbXX38xe3t7FhUVxfLz81lOTg7z9/dnEyZM0FhPfX29Ln41bYbCrhnjxo1TC5mmfPDBB2zatGnc9Jw5c5idnR2rqqri2rZs2cJEIhFTKpXNvlZlZSUzNzdne/fu5doePnzIhEIhV0dBQQEDwDIzM7k++fn5DECjsHs6pLy8vNiCBQsYY/8Jux9++IGbv2vXLgaAHT16lGuLiYlh/fv3b7ZulZ49e7LPP/+8yXmpqanMxMSE3bx5k2u7fPmy2rpER0czS0tLJpPJuD6LFy9mXl5ejDHGZDIZEwgEauHWkq+//pp5eHhw09HR0WzYsGFaL9+RnvX75uDgwORyOde2ZcsW1rVrV1ZTU8O1bd26VS3svvjiCxYQEKD22rdu3WIAWEFBgdb16BMa9aSVNm/ejISEBNy8eRM1NTWoq6trdMB72LBhsLS05Ka9vb3x6NEj3Lp1C3369NH42oWFhairq4OXlxfXZmdnx+2aAEB+fj5MTU3h4eHBtbm5ucHW1rbR63l7ezeafnI3BgCGDh3K/ezg4AAAGDJkiFpbaWmpxppVSktLcffuXfj6+jY5Pz8/H05OTmrjAg4cOBC2trbIz8/nHgEoFou5hz0DQI8ePbj3z8/Ph1wu1/geALBnzx5s3LgRhYWFePToEerr6/V61A9tvm9DhgyBubk5N11QUIChQ4fCwsKCaxs1apTaMhcvXsTx48ebfI5sYWEhXnrppbZdkU6Ajtm1wu7du7Fo0SKEh4cjNTUVFy5cwLx58/T6oPCTo9iqxut7uq2hoaHF1xEKhW1ez9Pv39J7pKenIywsDEFBQUhJScH58+fx+eef6+3no+33rUuXLq1+7UePHiE4OBgXLlxQ++/atWsYO3ZsW61Cp0Jh1wxzc3MolUpu+vTp0/Dx8cH777+P4cOHw9XVFYWFhY2Wu3jxImpqarjps2fPQiQStTja8YsvvggzMzNkZGRwbX/99ReuXr3KTbu5uaG+vh7Z2dlcW0FBAcrLyxu93tmzZxtNDxgwoNkanpWVlRXEYjGOHj3a5PwBAwbg1q1buHXrFteWl5eH8vJyDBw4UKv36NevH4RCocb3OHPmDPr06YPPP/8cnp6e6NevH/7880+1Pk9/pp3Js37fnta/f39cunQJcrmca8vKylLrM2LECFy+fBlisRiurq5q/6nCszP/rp4FhV0zxGIxMjIyUFxcjAcPHqBfv344d+4cjhw5gqtXr2L58uWNvkTA49P/4eHhyMvLw8GDBxEdHY2IiIgWR1IViUQIDw/H4sWLcezYMeTm5mLu3Llqy/Xv3x+TJk3Cu+++i4yMDGRnZ+Mf//hHk1s9v/zyCxISEnD16lVER0cjMzNT7exoW1u5ciXi4uKwceNGXLt2DTk5Odi0aRMAwM/PD0OGDEFYWBhycnKQmZmJ2bNnY9y4cfD09NTq9S0sLPDZZ59hyZIlSExMRGFhIc6ePYsff/wRwOMwvHnzJnbv3o3CwkJs3LgR+/fvV3sNsViMoqIiXLhwAQ8ePFALBF171u/b09544w00NDTgnXfeQX5+Po4cOYLY2FgA/9l6/+CDD1BWVoZZs2YhKysLhYWFOHLkCObNm8cF3NP1aLOF36np+qBhZ1ZQUMBGjx7NhEIhA8CuXLnC5s6dy2xsbJitrS1bsGABW7p0qdoB7zlz5rCpU6eyFStWsK5duzKRSMTmz5/PamtrtXrPyspK9uabbzJLS0vm4ODA1q5d2+hAcUlJCZs8eTITCATM2dmZJSYmsj59+jQ6QbF582bm7+/PBAIBE4vFamfxVCcoVAesGWPs+PHjDAD766+/uLaffvqJ2djYaP07i4+PZ/3792dmZmasR48e7MMPP+Tm/fnnn+zvf/8769KlC7OysmKvv/46k0ql3PymTh6sX7+e9enTh5tWKpXsv/7rv1ifPn2YmZkZc3Z2ZqtXr+bmL168mPu9z5gxg61fv16t/traWjZt2jRma2vLALCffvpJ63Vrb8/zfXva6dOn2dChQ5m5uTnz8PBgO3fu5F5T5erVq+zVV19ltra2TCgUMjc3N/bJJ5+whoaGJuspKipq599A+6IhngyUkZER9u/f3+a3rhH9tGPHDsybNw8VFRVtdnxV39DZWEIMUGJiIvr27YtevXrh4sWL+OyzzzB9+nTeBh1AYdehbt682ezB+Ly8PDg7O3dgRa3T1GUKKocOHcKYMWM6sBrSHKlUihUrVkAqlaJHjx54/fXX1e404SPaje1A9fX1zd5yIxaLYWraef/9uX79usZ5vXr14vVWA+n8KOwIIbxAl54QQniBwo4QwgsUdoQQXqCwI4TwAoUd6TTmzp3LDRRpZmYGBwcH+Pv7IyEhoVW3Km3btq3JUWDaW3s8f4S0HQo70qlMmjQJJSUlKC4uxqFDhzBhwgR8/PHHmDJlCurr63VdHtFnurxXjZAnabrP8+jRowwAN2hnXFwcGzx4MLO0tGS9e/dmCxYsYJWVlYyx/9zf++R/0dHRjDHGEhMTmYeHBxOJRMzBwYHNmjWL3bt3j3ufsrIy9sYbb7Bu3boxCwsL5urqyhISErj5N2/eZK+//jqzsbFhL7zwAvv73//O3S8aHR3d6H2PHz/eLr8n8mxoy450eq+88gqGDRuGpKQkAICxsTE2btyIy5cv41//+heOHTuGJUuWAAB8fHywYcMGWFtbo6SkBCUlJVi0aBEAQKFQ4IsvvsDFixeRnJyM4uJizJ07l3uf5cuXIy8vD4cOHUJ+fj62bNmCbt26cctOnDgRVlZWSEtLw+nTpyESiTBp0iTU1dVh0aJFmD59OrdlWlJSAh8fn479RZHm6TptCVHRtGXHGGMzZsxgAwYMaHLeL7/8wrp27cpNaztSS1ZWFgPAbRUGBwezefPmNdl3+/btrH///tyIIIwxJpfLmVAoZEeOHGmxfqJ7tGVH9AJjjBuL7Y8//oCvry969eoFKysrvPXWW3j48CGqq6ubfY3s7GwEBwfD2dkZVlZWGDduHIDH9ywDwIIFC7B79264u7tjyZIlOHPmDLfsxYsXcf36dVhZWUEkEkEkEsHOzg61tbVaDahJdI/CjuiF/Px8uLi4oLi4GFOmTMHQoUPx66+/Ijs7G5s3bwbQ/DNTq6qqMHHiRFhbW2PHjh3IysriBvZULRcYGIg///wTn376Kfc8DdUu8KNHj+Dh4dFoGPOrV6/ijTfeaOe1J22h8951Tsj/O3bsGC5duoRPP/0U2dnZaGhoQFxcHDeC8969e9X6NzWc+JUrV/Dw4UN89dVX3PD4586da/Re9vb2mDNnDubMmYMxY8Zg8eLFiI2NxYgRI7Bnzx50795d4wN8DG0Yc0NDW3akU5HL5ZBKpbhz5w5ycnKwevVqTJ06FVOmTMHs2bPh6uoKhUKBTZs24caNG9i+fTvi4+PVXkMsFuPRo0c4evQoHjx4gOrqajg7O8Pc3Jxb7sCBA/jiiy/UlluxYgV+++03XL9+HZcvX0ZKSgr3zI6wsDB069YNU6dORVpaGoqKinDixAl89NFHuH37Nve+//73v1FQUIAHDx5wD+cmnYSuDxoSojJnzhzusg1TU1Nmb2/P/Pz8WEJCgtozd9etW8d69OjBhEIhmzhxIktMTGw0nPx7773Hunbtqnbpyc6dO5lYLGYCgYB5e3uzAwcONHqW6oABA5hQKGR2dnZs6tSp7MaNG9xrlpSUsNmzZ7Nu3boxgUDA+vbty+bPn88qKioYY4yVlpYyf39/JhKJ6NKTToiGeCKE8ALtxhJCeIHCjhDCCxR2hBBeoLAjhPAChR0hhBco7AghvEBhRwjhBQo7QggvUNgRQniBwo4QwgsUdoQQXqCwI4Twwv8BJUZiialWzQEAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T15:36:30.644445Z", + "iopub.status.busy": "2024-03-22T15:36:30.643625Z", + "iopub.status.idle": "2024-03-22T15:36:30.933373Z", + "shell.execute_reply": "2024-03-22T15:36:30.932477Z" + }, + "papermill": { + "duration": 0.31373, + "end_time": "2024-03-22T15:36:30.935421", + "exception": false, + "start_time": "2024-03-22T15:36:30.621691", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAEmCAYAAAAEH9kkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHBUlEQVR4nO3de1hU1foH8O9wmeE+qCgDyk3DO14OCmEWlaNodIROv+OlUlS8VGoW3tBUvFSQl7yn2QnUk4bQMbU0CzlqJ0FUREVBDERRYVBEGERlYGb9/iB2bhmYGRiYAd7P88wDs9a7116bcV733mvvvQSMMQZCCCF1MjF0BwghxNhRoiSEEA0oURJCiAaUKAkhRANKlIQQogElSkII0YASJSGEaECJkhBCNDAzdAdaM5VKhfz8fNja2kIgEBi6O4SQpzDGUFZWBmdnZ5iY1L/PSImyCeXn58PFxcXQ3SCE1OPWrVvo0qVLvTGUKJuQra0tgOoPws7OzsC9IYQ8TS6Xw8XFhfue1ocSZROqOdy2s7OjREmIkdLmtBgN5hBCiAaUKAkhRANKlIQQogGdozRCjDFUVVVBqVQauiuEtFimpqYwMzPTy6V5lCiNjEKhQEFBAR49emTorhDS4llZWcHJyQlCobBR7VCiNCIqlQq5ubkwNTWFs7MzhEIhXahOSAMwxqBQKHDv3j3k5ubC09NT40Xl9aFEaUQUCgVUKhVcXFxgZWVl6O7US6ViyCt+BDtLc7S3btz/1oQ0BUtLS5ibm+PmzZtQKBSwsLBocFs0mGOEGvM/X3MpKq+A/Eklbj+gUwTEeOnru2T830hilJQqmpOOtB2UKAkhRANKlMRgJk2ahODgYL22+fLLL+PDDz+sN8bd3R0bNmzQ63qJYWjzeesDJUqiF831D5Y0n+XLl2PAgAGG7oZRoERJCDGoyspKQ3dBI0qURo4xhkeKKoO8GNNuwGbSpEk4efIkNm7cCIFAAIFAgJycHISGhsLDwwOWlpbo0aMHNm7cqHb5FStWoGPHjrCzs8O7774LhUKh1XrLy8sxceJE2NjYwMnJCevWrasVc/fuXfz973+HpaUlPDw8sGfPnloxAoEA27Ztw6hRo2BpaYmuXbvi+++/5+pv3LgBgUCAuLg4vPjii7C0tMTgwYNx7do1nD17FoMGDYKNjQ1GjRqFe/fuadV3AIiOjkafPn0gEong5OSEWbNmcXV5eXkICgqCjY0N7OzsMGbMGBQWFnL1NXt7//73v+Hu7g6xWIxx48ahrKyMi1GpVFi9ejWee+45iEQiuLq64tNPP+XqFy5ciO7du8PKygpdu3bF0qVLuaS1c+dOrFixAhcvXuQ+0507d2rcpqtXr2Lo0KGwsLBA7969cezYMQgEAhw4cID3t9y3bx/8/f1hYWGBPXv24P79+xg/fjw6d+4MKysreHl54bvvvuO1rc3n3VToOkoj97hSid7LfjHIujNWBsBKWMc/kady6MaNG3Ht2jX07dsXK1euBAC0a9cOXbp0QXx8PDp06ICkpCRMnz4dTk5OGDNmDLdsYmIiLCwscOLECdy4cQOTJ09Ghw4deF/ousyfPx8nT57EwYMH0alTJyxevBjnz5/nHS5OmjQJ+fn5OH78OMzNzfHBBx/g7t27tdpaunQpoqKisHHjRvz73//GuHHjkJ6ejl69enExERER2LBhA1xdXTFlyhS89dZbsLW1xcaNG2FlZYUxY8Zg2bJl2LZtm8a+b9u2DWFhYYiKisKoUaNQWlqKU6dOAahOcDVJ8uTJk6iqqsLMmTMxduxYnDhxgmsjJycHBw4cwE8//YQHDx5gzJgxiIqK4v52ixYtwtdff43169dj6NChKCgowNWrV7nlbW1tsXPnTjg7OyM9PR3Tpk2Dra0tFixYgLFjx+Ly5cs4evQojh07BgAQi8X1bpNSqURwcDBcXV2RkpKCsrIyzJ07V21seHg41q1bh4EDB8LCwgJPnjyBt7c3Fi5cCDs7Oxw+fBgTJkxAt27d4OPjA0C7z7upUKIkjSYWiyEUCmFlZQWJRMKVr1ixgvvdw8MDycnJiIuL4yVKoVCI6OhoWFlZoU+fPli5ciXmz5+PVatW1XsN3MOHD/HNN9/g22+/xbBhwwAAu3bt4j2p+tq1a/j5559x5swZDB48GADwzTff8JJfjX/+85+YOnUqAGDVqlVISEjA5s2b8eWXX3Ix8+bNQ0BAAABgzpw5GD9+PBITE/HCCy8AAEJDQ7Xa6wKATz75BHPnzsWcOXO4spo+JiYmIj09Hbm5udwT8nfv3o0+ffrg7NmzXJxKpcLOnTu5B89OmDABiYmJ+PTTT1FWVoaNGzdiy5YtCAkJAQB069YNQ4cO5da3ZMkS7nd3d3fMmzcPsbGxWLBgASwtLWFjYwMzMzPeZ1qfhIQE5OTk4MSJE9wyn376KYYPH14r9sMPP8Q//vEPXtm8efO432fPno1ffvkFcXFx8PHx0erzbkqUKI2cpbkpMlYGGGzdjbF161ZER0cjLy8Pjx8/hkKhqPW/f//+/Xl3Ifn5+eHhw4e4desW3Nzc6mw7JycHCoUCvr6+XFn79u3Ro0cP7n1mZibMzMzg7e3NlfXs2RP29va12vPz86v1/sKFC7yyfv36cb87OjoCALy8vHhl6vZWn3X37l3k5+dzX/hnZWZmwsXFhTeNSO/evWFvb4/MzEwuUbq7u/Oezu3k5MStPzMzExUVFXWuAwD27duHTZs2IScnBw8fPkRVVVWjHjCdlZUFFxcXXmKt2Rt81qBBg3jvlUolPvvsM8TFxeHOnTtQKBSoqKjg/m1o83k3JUqURk4gENR9+GtAms5exsbGYt68eVi3bh38/Pxga2uLNWvWICUlpVn61xTMzc2532vuwX+2TKVSaWzH0tJS7/15dv2a1pGcnIy3334bK1asQEBAAMRiMWJjY5vtvJ+1tTXv/Zo1a7Bx40Zs2LABXl5esLa2xocffqj1+eqmRoM5RC+EQiHvsXCnTp3CkCFD8P7772PgwIF47rnnkJOTU2u5ixcv4vHjx9z706dPw8bGRuOkbN26dYO5uTkv8T548ADXrl3j3vfs2RNVVVVITU3lyrKyslBSUlKrvdOnT9d6r+4QXR9sbW3h7u6OxMREtfW9evXCrVu3cOvWLa4sIyMDJSUl6N27t1br8PT0hKWlZZ3rSEpKgpubGz7++GMMGjQInp6euHnzJi/m2c9Ukx49euDWrVu8QaezZ89qteypU6cQFBSEd955B/3790fXrl15n6U2n3dTMnii3Lp1K9zd3WFhYQFfX1+cOXOm3vj4+Hj07NkTFhYW8PLywpEjR3j1+/fvx4gRI9ChQwcIBIJah09A9TV/NSN5Na93332XF5OXl4fAwEBYWVmhU6dOmD9/Pqqqqhq9va2Vu7s7UlJScOPGDRQVFcHT0xPnzp3DL7/8gmvXrmHp0qVqvzQKhQKhoaHIyMjAkSNHEBERgVmzZmm8R9fGxgahoaGYP38+/vvf/+Ly5cuYNGkSb7kePXpg5MiRmDFjBlJSUpCamoqpU6eq3duKj49HdHQ0rl27hoiICJw5c4Y3Cq1vy5cvx7p167Bp0yb88ccfOH/+PDZv3gwAkEql8PLywttvv43z58/jzJkzmDhxIvz9/WsdstbFwsICCxcuxIIFC7B7927k5OTg9OnT+OabbwBUJ9K8vDzExsYiJycHmzZtwg8//MBrw93dHbm5ubhw4QKKiopQUVFR7zqHDx+Obt26ISQkBJcuXcKpU6e486CanoLl6emJhIQEJCUlITMzEzNmzOAlXG0+7ybFDCg2NpYJhUIWHR3Nrly5wqZNm8bs7e1ZYWGh2vhTp04xU1NTtnr1apaRkcGWLFnCzM3NWXp6Oheze/dutmLFCvb1118zACwtLa1WO/7+/mzatGmsoKCAe5WWlnL1VVVVrG/fvkwqlbK0tDR25MgR5uDgwBYtWqTT9pWWljIAvLbr8/jxY5aRkcEeP36s03oM4c6DR+zirQfs4q0HjDHGsrKy2PPPP88sLS0ZAHb16lU2adIkJhaLmb29PXvvvfdYeHg469+/P9dGSEgICwoKYsuWLWMdOnRgNjY2bNq0aezJkyda9aGsrIy98847zMrKijk6OrLVq1czf39/NmfOHC6moKCABQYGMpFIxFxdXdnu3buZm5sbW79+PRcDgG3dupUNHz6ciUQi5u7uzvbt28fV5+bm1vq3dPz4cQaAPXjwgCuLiYlhYrFY67/h9u3bWY8ePZi5uTlzcnJis2fP5upu3rzJRo8ezaytrZmtrS375z//yWQyGVcfERHB+1syxtj69euZm5sb916pVLJPPvmEubm5MXNzc+bq6so+++wzrn7+/Pnc333s2LFs/fr1vP4/efKEvfnmm8ze3p4BYDExMRq3KTMzk73wwgtMKBSynj17sh9//JEBYEePHmWMqf9bMsbY/fv3WVBQELOxsWGdOnViS5YsYRMnTmRBQUFcjDaf97Pq+07p8v00aKL08fFhM2fO5N4rlUrm7OzMIiMj1caPGTOGBQYG8sp8fX3ZjBkzasXW9YEwxjT+cY8cOcJMTEx4/zC3bdvG7OzsWEVFhYat+ktbSpQtGQD2ww8/GLobrdLvv//OALDs7GyDrF9fidJgh94KhQKpqamQSqVcmYmJCaRSKZKTk9Uuk5yczIsHgICAgDrj67Nnzx44ODigb9++WLRoEe+J4snJyfDy8uJGNmvWI5fLceXKlTrbrKiogFwu570IaUt++OEHJCQk4MaNGzh27BimT5+OF154Ad26dTN01xrFYMOpRUVFUCqVvGQEVF9i8fRFsU+TyWRq42UymU7rfuutt+Dm5gZnZ2dcunQJCxcuRFZWFvbv31/vemrq6hIZGcm7dpA0XF5eXr0DFxkZGXB1dW3GHunGxsamzrqff/4ZL774YjP2Rj/27NmDGTNmqK1zc3PDlStXUFZWhoULFyIvLw8ODg6QSqXNegdNUzG+606awfTp07nfvby84OTkhGHDhiEnJ6dR//MtWrQIYWFh3Hu5XK5x9Jao5+zsrHYg7ul6fWFa3qqpi/r63rlzZ72vrzmMHj2adx3j02ouVZo4cSImTpzYnN1qFgZLlA4ODjA1NeWNbAFAYWFhnXcCSCQSneK1VfPhZ2dno1u3bpBIJLVG32vWW9+6RCIRRCJRo/pCqpmZmeG5554zdDcarCX3vS62tra8C9zbEoOdoxQKhfD29uZd56VSqZCYmFjrLokafn5+ta4LS0hIqDNeWzX/+zs5OXHrSU9P591lkZCQADs7O62vYyOEtB4GPfQOCwtDSEgIBg0aBB8fH2zYsAHl5eWYPHkygOrd+M6dOyMyMhJA9f21/v7+WLduHQIDAxEbG4tz585hx44dXJvFxcXIy8tDfn4+gOoLjIHqPUGJRIKcnBzs3bsXr732Gjp06IBLly7ho48+wksvvcTdojZixAj07t0bEyZMwOrVqyGTybBkyRLMnDmT9hgJaYuaYEReJ5s3b2aurq5MKBQyHx8fdvr0aa7O39+fhYSE8OLj4uJY9+7dmVAoZH369GGHDx/m1cfExDBU32HHe0VERDDGGMvLy2MvvfQSa9++PROJROy5555j8+fPr3WJwI0bN9ioUaOYpaUlc3BwYHPnzmWVlZU6bRtdHkSIYenr8iABY01wJpsAqB7MEYvFKC0t1ephA0+ePEFubi48PDwaNbVmc8gveYyih9V3avTrYm/YzhBSh/q+U7p8Pw1+CyMhhBg7SpTE4J6d7OvpJ2ITYgza5HWUpPGa8nxNQUEB2rVr14RrIEQ3lCiJ0WnsdbH6wBiDUqmEmRl9RQgdehs/xgBFuWFeOozzlZWV4e2334a1tTWcnJywfv36Bk9hq24yqv379+OVV16BlZUV+vfvX+v+/t9//52b+MvFxQUffPABysvLufp///vfGDRoEGxtbSGRSPDWW2/xrpM9ceIEBAIBfv75Z3h7e0MkEuH333/Xue+kdaL/Lo1d5SPgM/3drqeTxfmA0FpzHKqviT116hQOHToER0dHLFu2TK8TP3388cdYu3YtPD098fHHH2P8+PHIzs6GmZkZcnJyMHLkSHzyySeIjo7GvXv3MGvWLMyaNQsxMTEAqqdEXbVqFXr06IG7d+8iLCwMkyZNqvU80/DwcKxduxZdu3alw3/CoURJGq2srAy7du3C3r17uTlaYmJi9Ho/9rx58xAYGAigetKyPn36IDs7Gz179kRkZCTefvttbu/V09MTmzZtgr+/P7Zt2wYLCwtMmTKFa6tr167YtGkTBg8ejIcPH/IeYLFy5Uq1k2GRto0SpbEzt6reszPUurVw/fp1VFZW8iaSEovFep346emJvWpuNb179y569uyJixcv4tKlS7w5uxljUKlUyM3NRa9evZCamorly5fj4sWLePDgATe3zLNPKdL2CeKkbaFEaewEAq0Pf1szdRN71SS7hw8fYsaMGfjggw9qLefq6ory8nIEBAQgICAAe/bsQceOHZGXl4eAgIBak1c9O+kVIQAlSqIHXbt2hbm5Oc6ePcs9I7K0tBTXrl3DSy+91OTr/9vf/oaMjIw6n9iTnp6O+/fvIyoqinvs3blz55q8X6T1oFFv0jBPDYjb2toiJCQE8+fPx/Hjx3HlyhWEhobCxMRE46RS+rBw4UIkJSVh1qxZuHDhAv744w8cPHiQmxzM1dUVQqEQmzdvxvXr13Ho0CGsWrWqyftFWg9KlEQvvvjiC/j5+eH111+HVCrFCy+8gF69ejXLPev9+vXDyZMnce3aNbz44osYOHAgli1bxg0mdezYETt37kR8fDx69+6NqKgorF27tsn7RVoPeihGE2rND8W48+Ax7pfX/VCM8vJydO7cGevWrUNoaGgz946Qavp6KAadoyQNxP//NS0tDVevXoWPjw9KS0uxcuVKAEBQUJAhOkeIXlGiJHqzdu1aZGVlcU+v/9///ofMzEyMGjWqzmUePnzYjD0kpGEoURK9GDhwIFJTU2uVP378uN6JtghpCShRkiZlaWnZKifaIm0LjXobIRpfI0Q/9PVdokRpRGruPnn06JGBe6IZpXLSEtR8l56+s6sh6NDbiJiamsLe3p57/JeVlVWzXLDdEFWKCrCqSgDVl2AQYkwYY3j06BHu3r0Le3t7mJqaNqo9SpRGpuahtU8/K9EYPXikQHmFEgAgfGxp4N4Qop69vb1eHgRt8ES5detWrFmzBjKZDP3798fmzZt5T6F5Vnx8PJYuXYobN27A09MTn3/+OV577TWufv/+/di+fTtSU1NRXFyMtLQ03jMRi4uLERERgV9//RV5eXno2LEjgoODsWrVKojFYi5O3Z7cd999h3Hjxulnw+sgEAjg5OSETp06obKysknX1Rg//HoNh9Ork3ni3JcN2xlC1DA3N2/0nmQNgybKffv2ISwsDNu3b4evry82bNiAgIAAZGVloVOnTrXik5KSMH78eERGRuL111/H3r17ERwcjPPnz6Nv374Aqu8IGTp0KMaMGYNp06bVaiM/Px/5+flYu3YtevfujZs3b+Ldd99Ffn4+vv/+e15sTEwMRo4cyb23t7fX7x+gHqampnr7kJtCWZUAd8qq9yiN/S4iQhpNj3ON68zHx4fNnDmTe69UKpmzszOLjIxUGz9mzBgWGBjIK/P19WUzZsyoFZubm8sAsLS0NI39iIuLY0KhkFVWVnJlANgPP/yg3YbUQZcJ1lua8P9cYm4Lf2JuC38ydFcIaRBdvp8GG/VWKBRITU2FVCrlykxMTCCVSmvNh1IjOTmZFw8AAQEBdcZrq+Zez2cnkpo5cyYcHBzg4+OD6OhoumyHh/4WpO0w2KF3UVERlEolHB0deeWOjo64evWq2mVkMpnaeJlM1qh+rFq1CtOnT+eVr1y5Eq+++iqsrKzw66+/4v3338fDhw/VPhy2RkVFBSoqKrj3crm8wf0ihBgPgw/mGJJcLkdgYCB69+6N5cuX8+qWLl3K/T5w4ECUl5djzZo19SbKyMhIrFixoqm6SwgxEIMdejs4OMDU1BSFhYW88sLCwjqH8yUSiU7x9SkrK8PIkSNha2uLH374QeMFqb6+vrh9+zZvj/FZixYtQmlpKfe6deuWzv1qKegsBGlLDJYoa54wk5iYyJWpVCokJibCz89P7TJ+fn68eABISEioM74ucrkcI0aMgFAoxKFDh7Qatb1w4QLatWsHkUhUZ4xIJIKdnR3vRQhp+Qx66B0WFoaQkBAMGjQIPj4+2LBhA8rLyzF58mQAwMSJE9G5c2dERkYCAObMmQN/f3+sW7cOgYGBiI2Nxblz57Bjxw6uzeLiYuTl5SE/v3rmwqysLADVe6MSiYRLko8ePcK3334LuVzOnUvs2LEjTE1N8eOPP6KwsBDPP/88LCwskJCQgM8++wzz5s1rzj8PIcRYNP0gfP02b97MXF1dmVAoZD4+Puz06dNcnb+/PwsJCeHFx8XFse7duzOhUMj69OnDDh8+zKuPiYlhqB6S5b0iIiIYY4wdP35cbT0Alpubyxhj7Oeff2YDBgxgNjY2zNramvXv359t376dKZVKnbatNV8etCD+Il0eRFo0Xb6fNBVEE9J1KoiWZMH3FxF37jYA4EZUoIF7Q4judPl+0tODCCFEg0ado3z48CE3CX2N1rbnRAghOu9R5ubmIjAwENbW1hCLxWjXrh3atWsHe3t7tGvXrin6SIwQnbAhbYnOe5TvvPMOGGOIjo6Go6Oj0T4vkRBC9EXnRHnx4kWkpqaiR48eTdEfQggxOjofeg8ePLhV33FCtENH3qQt0XmP8l//+hfeffdd3LlzB3379q1161+/fv301jlCCDEGOifKe/fuIScnh7t7Bqh+KjdjDAKBAEqlUq8dJIQQQ9M5UU6ZMgUDBw7Ed999R4M5bRiNepO2ROdEefPmTRw6dIgmtSeEtBk6D+a8+uqruHjxYlP0hRBCjJLOe5R///vf8dFHHyE9PR1eXl61BnNGjx6tt84R48Vo3Ju0ITonynfffRdA9VQJz6LBHEJIa6Rzonz23m5CCGntdDpHWVlZCTMzM1y+fLmp+kNaCjryJm2ITonS3Nwcrq6udHhNCGlTdB71/vjjj7F48WIUFxc3RX8IIcTo6HyOcsuWLcjOzoazszPc3NxgbW3Nqz9//rzeOkeMFx15k7ZE50QZHBzcBN0ghBDjpXOijIiIaIp+EEKI0WrwVBCpqanIzMwEAPTp0wcDBw7UW6eI8aM56UhbonOivHv3LsaNG4cTJ07A3t4eAFBSUoJXXnkFsbGx6Nixo777SAghBqXzqPfs2bNRVlaGK1euoLi4GMXFxbh8+TLkcjk++OADnTuwdetWuLu7w8LCAr6+vjhz5ky98fHx8ejZsycsLCzg5eWFI0eO8Or379+PESNGoEOHDhAIBLhw4UKtNp48eYKZM2eiQ4cOsLGxwZtvvonCwkJeTF5eHgIDA2FlZYVOnTph/vz5qKqq0nn7CCEtn86J8ujRo/jyyy/Rq1cvrqx3797YunUrfv75Z53a2rdvH8LCwhAREYHz58+jf//+CAgIwN27d9XGJyUlYfz48QgNDUVaWhqCg4MRHBzMuwC+vLwcQ4cOxeeff17nej/66CP8+OOPiI+Px8mTJ5Gfn49//OMfXL1SqURgYCAUCgWSkpKwa9cu7Ny5E8uWLdNp+1ozOvAmbQrTkY2NDUtLS6tVfv78eWZra6tTWz4+PmzmzJnce6VSyZydnVlkZKTa+DFjxrDAwEBema+vL5sxY0at2NzcXAagVl9LSkqYubk5i4+P58oyMzMZAJacnMwYY+zIkSPMxMSEyWQyLmbbtm3Mzs6OVVRUaL19paWlDAArLS3VepmW4oPvzjO3hT8xt4U/GborhDSILt/PBj1mbc6cOcjPz+fK7ty5g48++gjDhg3Tuh2FQoHU1FRIpVKuzMTEBFKpFMnJyWqXSU5O5sUDQEBAQJ3x6qSmpqKyspLXTs+ePeHq6sq1k5ycDC8vLzg6OvLWI5fLceXKlTrbrqiogFwu570IIS2fzolyy5YtkMvlcHd3R7du3dCtWzd4eHhALpdj8+bNWrdTVFQEpVLJS0YA4OjoCJlMpnYZmUymU3xdbQiFQm4gSl07da2npq4ukZGREIvF3MvFxUXrfrU0NOhN2hKdR71dXFxw/vx5HDt2DFevXgUA9OrVq9aeXlu0aNEihIWFce/lcnmrTpaEtBUNuo5SIBBg+PDhGD58eINX7ODgAFNT01qjzYWFhZBIJGqXkUgkOsXX1YZCoUBJSQlvr/LpdiQSSa3R95r11rcukUgEkUikdV9aMtqhJG2JzofeAJCYmIjFixdj6tSpmDJlCu+lLaFQCG9vbyQmJnJlKpUKiYmJ8PPzU7uMn58fLx4AEhIS6oxXx9vbG+bm5rx2srKykJeXx7Xj5+eH9PR03uh7QkIC7Ozs0Lt3b63XRQhpHXTeo1yxYgVWrlyJQYMGwcnJqVGzMIaFhSEkJASDBg2Cj48PNmzYgPLycm4q3IkTJ6Jz586IjIwEAMyZMwf+/v5Yt24dAgMDERsbi3PnzmHHjh1cm8XFxcjLy+MGm7KysgBU7wlKJBKIxWKEhoYiLCwM7du3h52dHWbPng0/Pz88//zzAIARI0agd+/emDBhAlavXg2ZTIYlS5Zg5syZbWaPkRDyFF2H1CUSCdu9e3eDhuPV2bx5M3N1dWVCoZD5+Piw06dPc3X+/v4sJCSEFx8XF8e6d+/OhEIh69OnDzt8+DCvPiYmhqH6yJD3ioiI4GIeP37M3n//fdauXTtmZWXF3njjDVZQUMBr58aNG2zUqFHM0tKSOTg4sLlz57LKykqdtq01Xx40c08qXR5EWjRdvp8CxnQbv+zQoQPOnDmDbt266T1ptzZyuRxisRilpaWws7MzdHf0atbe8/jpUgEA4EZUoIF7Q4judPl+6nyOcurUqdi7d2+DO0cIIS2Nzuconzx5gh07duDYsWPo169frelqv/jiC711jhgvGvUmbYnOifLSpUsYMGAAANSaZKwxAzuEEGKsdE6Ux48fb4p+EEKI0WrQdZSE0LE3aUsoURJCiAaUKAkhRANKlKRBGB17kzZE50T522+/qZ0SoaqqCr/99pteOkUIIcZE50T5yiuvoLi4uFZ5aWkpXnnlFb10ihBCjInOiZIxpvZ6yfv378Pa2lovnSLGjx7cS9oSra+jrJl8SyAQYNKkSbyn6CiVSly6dAlDhgzRfw8JIcTAtE6UYrEYQPUepa2tLSwtLbk6oVCI559/HtOmTdN/DwkhxMC0TpQxMTEAAHd3d8ybN48Os9s4OvQmbYnOtzBGREQ0RT8IIcRo6TyYU1hYiAkTJsDZ2RlmZmYwNTXlvQghpLXReY9y0qRJyMvLw9KlSxs9FQRpueiCc9KW6Jwof//9d/zvf//jHrVGCCGtnc6H3i4uLtBx9ghCCGnRdE6UGzZsQHh4OG7cuNEE3SEtBf1fSdoSnQ+9x44di0ePHqFbt26wsrKqNRWEutsbCSGkJWvQHuWOHTsQHR2NLVu2YP369bxXQ2zduhXu7u6wsLCAr68vzpw5U298fHw8evbsCQsLC3h5eeHIkSO8esYYli1bBicnJ1haWkIqleKPP/7g6k+cOAGBQKD2dfbsWQDAjRs31NafPn26QdvY2tAOJWlTmnTiXC3ExsYyoVDIoqOj2ZUrV9i0adOYvb09KywsVBt/6tQpZmpqylavXs0yMjLYkiVLmLm5OUtPT+dioqKimFgsZgcOHGAXL15ko0ePZh4eHuzx48eMMcYqKipYQUEB7zV16lTm4eHBVCoVY4yx3NxcBoAdO3aMF6dQKLTettY8r/fUXWdpXm/Souny/WxQoszOzmYff/wxGzduHJfQjhw5wi5fvqxzWz4+PmzmzJnce6VSyZydnVlkZKTa+DFjxrDAwEBema+vL5sxYwZjjDGVSsUkEglbs2YNV19SUsJEIhH77rvv1LapUChYx44d2cqVK7mymkSZlpam8zbVoERJiPHS5fup86H3yZMn4eXlhZSUFOzfvx8PHz4EAFy8eFHnu3YUCgVSU1MhlUq5MhMTE0ilUiQnJ6tdJjk5mRcPAAEBAVx8bm4uZDIZL0YsFsPX17fONg8dOoT79+9j8uTJtepGjx6NTp06YejQoTh06FC921NRUQG5XM57tVY0mEPaEp0TZXh4OD755BMkJCRAKBRy5a+++qrO5++KioqgVCrh6OjIK3d0dIRMJlO7jEwmqze+5qcubX7zzTcICAhAly5duDIbGxusW7cO8fHxOHz4MIYOHYrg4OB6k2VkZCTEYjH3cnFxqTOWENJy6DzqnZ6ejr1799Yq79SpE4qKivTSqeZ0+/Zt/PLLL4iLi+OVOzg4ICwsjHs/ePBg5OfnY82aNRg9erTathYtWsRbRi6XU7IkpBXQeY/S3t4eBQUFtcrT0tLQuXNnndpycHCAqakpCgsLeeWFhYWQSCRql5FIJPXG1/zUts2YmBh06NChzuT3NF9fX2RnZ9dZLxKJYGdnx3u1XnTsTdoOnRPluHHjsHDhQshkMggEAqhUKpw6dQrz5s3DxIkTdWpLKBTC29sbiYmJXJlKpUJiYiL8/PzULuPn58eLB4CEhAQu3sPDAxKJhBcjl8uRkpJSq03GGGJiYjBx4sRa14Oqc+HCBTg5OWm9fYSQVkLXkaKKigo2depUZmZmxgQCATM3N2cmJibsnXfeYVVVVTqPPMXGxjKRSMR27tzJMjIy2PTp05m9vT2TyWSMMcYmTJjAwsPDufhTp04xMzMztnbtWpaZmckiIiLUXh5kb2/PDh48yC5dusSCgoJ4lwfVOHbsGAPAMjMza/Vr586dbO/evSwzM5NlZmayTz/9lJmYmLDo6Gitt601j3qH7jxDo96kRWvyy4MYY+zmzZvs8OHDbN++fezatWsNbYYxxtjmzZuZq6srEwqFzMfHh50+fZqr8/f3ZyEhIbz4uLg41r17dyYUClmfPn3Y4cOHefUqlYotXbqUOTo6MpFIxIYNG8aysrJqrXf8+PFsyJAhavu0c+dO1qtXL2ZlZcXs7OyYj48Pi4+P12m7WkOirFKq2IPyilrlU2IoUZKWTZfvp4AxutCjqcjlcojFYpSWlrbY85X/ty0J524+wLEwfzzXyYYrD915FolX7wIAbkQFGqp7hDSYLt9PrUa9w8LCsGrVKlhbW/NGddX54osvtO8pMXrnbj4AAOw/fxsLRvY0cG8IMQytEmVaWhoqKyu53+tCD/FtO+gwhLQlWiXK48ePq/2dEELaAp0vDyJtE+1BkrZMqz3Kf/zjH1o3uH///gZ3hrQcNAZI2hKt9iifvn/Zzs4OiYmJOHfuHFefmpqKxMREiMXiJusoMS6UJklbotUeZUxMDPf7woULMWbMGGzfvp2bnlapVOL9999vsZfAEEJIfXQ+RxkdHY158+bx5vA2NTVFWFgYoqOj9do5YjzoSJu0ZTonyqqqKly9erVW+dWrV6FSqfTSKWL8KHGStkTnx6xNnjwZoaGhyMnJgY+PDwAgJSUFUVFRah98SwghLZ3OiXLt2rWQSCRYt24d97g1JycnzJ8/H3PnztV7BwkhxNB0TpQmJiZYsGABFixYwE11QIM4rR97ZpybjrxJW6JzonwaJUhCSFvQoET5/fffIy4uDnl5eVAoFLy68+fP66VjhBBiLHQe9d60aRMmT54MR0dHpKWlwcfHBx06dMD169cxatSopugjMUJ0Zw5pS3ROlF9++SV27NiBzZs3QygUYsGCBUhISMAHH3yA0tLSpugjaWa5ReWYE5uGLFnZX4WUF0kbpnOizMvLw5AhQwAAlpaWKCur/jJNmDAB3333nX57RwwiJPoMDl7IxxtfnjJ0VwgxCjonSolEguLiYgCAq6srN5d3bm4uHY61EnnFjwAAjxRKA/eEEOOgc6J89dVXcejQIQDVF59/9NFHGD58OMaOHYs33nhD7x0khBBD03nUe8eOHdytijNnzkSHDh2QlJSE0aNHY8aMGXrvIDEOdKxA2jKdEmVVVRU+++wzTJkyBV26dAFQPc/3uHHjmqRzhBBiDHQ69DYzM8Pq1atRVVWl105s3boV7u7usLCwgK+vL86cOVNvfHx8PHr27AkLCwt4eXnhyJEjvHrGGJYtWwYnJydYWlpCKpXijz/+4MW4u7tDIBDwXlFRUbyYS5cu4cUXX4SFhQVcXFywevVq/WxwK0Cno0lbovM5ymHDhuHkyZN668C+ffsQFhaGiIgInD9/Hv3790dAQADu3r2rNj4pKQnjx49HaGgo0tLSEBwcjODgYFy+fJmLWb16NTZt2oTt27cjJSUF1tbWCAgIwJMnT3htrVy5EgUFBdxr9uzZXJ1cLseIESPg5uaG1NRUrFmzBsuXL8eOHTv0tu2EkJZB53OUo0aNQnh4ONLT0+Ht7Q1ra2te/ejRo3Vq74svvsC0adO4Jw9t374dhw8fRnR0NMLDw2vFb9y4ESNHjsT8+fMBAKtWrUJCQgK2bNmC7du3gzGGDRs2YMmSJQgKCgIA7N69G46Ojjhw4ADvNIGtrS0kEonafu3ZswcKhQLR0dEQCoXo06cPLly4gC+++ALTp0/XaRtbA7qigbRlOu9Rvv/++ygsLMQXX3yBt99+m9ujCw4O1nnUW6FQIDU1FVKp9K8OmZhAKpUiOTlZ7TLJycm8eAAICAjg4nNzcyGTyXgxYrEYvr6+tdqMiopChw4dMHDgQKxZs4Z3SiE5ORkvvfQShEIhbz1ZWVl48OCBTtvZ2twte4Lfs4sM3Q1Cmo3Oe5T6fDhvUVERlEolHB0deeWOjo5qHw4MADKZTG28TCbj6mvK6ooBgA8++AB/+9vf0L59eyQlJWHRokUoKCjAF198wbXj4eFRq42aunbt2tXqW0VFBSoqKrj3NU9Xam3C/5Nu6C4Q0qwa9fSgliwsLIz7vV+/fhAKhZgxYwYiIyMhEoka1GZkZCRWrFihry4arWuFZZqDCGlFtE6Ujx8/RmJiIl5//XUAwKJFi3h7T6ampli1ahUsLCy0XrmDgwNMTU1RWFjIKy8sLKzz3KFEIqk3vuZnYWEhnJyceDEDBgyosy++vr6oqqrCjRs30KNHjzrX8/Q6nrVo0SJeApbL5XBxcalznS3J06co6XQlaWu0Pke5a9cufPXVV9z7LVu2ICkpCWlpaUhLS8O3336Lbdu26bRyoVAIb29vJCYmcmUqlQqJiYnw8/NTu4yfnx8vHgASEhK4eA8PD0gkEl6MXC5HSkpKnW0CwIULF2BiYoJOnTpx6/ntt99QWVnJW0+PHj3UHnYDgEgkgp2dHe/VGilVlClJ26J1otyzZ0+t0d69e/fi+PHjOH78ONasWYO4uDidOxAWFoavv/4au3btQmZmJt577z2Ul5dzo+ATJ07EokWLuPg5c+bg6NGjWLduHa5evYrly5fj3LlzmDVrFgBAIBDgww8/xCeffIJDhw4hPT0dEydOhLOzM4KDgwFUD9Rs2LABFy9exPXr17Fnzx589NFHeOedd7gk+NZbb0EoFCI0NBRXrlzBvn37sHHjRt4eY1ulpF1K0sZofeidnZ0NLy8v7r2FhQVMTP7Ksz4+Ppg5c6bOHRg7dizu3buHZcuWQSaTYcCAATh69Cg3cJKXl8dbz5AhQ7B3714sWbIEixcvhqenJw4cOIC+fftyMQsWLEB5eTmmT5+OkpISDB06FEePHuVOC4hEIsTGxmL58uWoqKiAh4cHPvroI14SFIvF+PXXXzFz5kx4e3vDwcEBy5Yta5OXBj1LRXuUpI0RMC0vkLO0tMSFCxfQo0cPtfVXr17FgAEDal3U3ZbJ5XKIxWKUlpa2qMNw9/DDtcpCh3pg6eu9AQADV/6KB4/+OiVxIyqw2fpGiL7o8v3U+tC7S5cuvLtfnnXp0iXu/m/SutE5StLWaJ0oX3vtNSxbtkztHuPjx4+xYsUKBAbSnkVbQHmStDVan6NcvHgx4uLi0KNHD8yaNQvdu3cHAGRlZWHLli2oqqrC4sWLm6yjxHjQHiVpa7ROlI6OjkhKSsJ7772H8PBw7t5fgUCA4cOH48svv6x1NwxpPZ4+k/3sqDdjDAKBoJl7REjz0enOHA8PDxw9ehTFxcXIzs4GADz33HNo3759k3SOGCd6QAZpaxp0C2P79u3h4+Oj776QFoIOvUlbo/PTgwh5Nk/SDiZp7ShREq0wmjWHtGGUKAkhRANKlKTRaF+TtHaUKIlW6DwkacsoURJCiAaUKEmj0XWVpLWjREkIIRpQoiSEEA0oURJCiAaUKEmj0RlK0tpRoiSEEA0oURKt0Mg2acsoUZJGoxxKWjtKlIQQogElSkII0cAoEuXWrVvh7u4OCwsL+Pr64syZM/XGx8fHo2fPnrCwsICXlxeOHDnCq2eMYdmyZXBycoKlpSWkUin++OMPrv7GjRsIDQ2Fh4cHLC0t0a1bN0REREChUPBiBAJBrdfp06f1u/EtRH1H1/QINtLaGTxR7tu3D2FhYYiIiMD58+fRv39/BAQE4O7du2rjk5KSMH78eISGhiItLQ3BwcEIDg7mTaW7evVqbNq0Cdu3b0dKSgqsra0REBDAzSB59epVqFQqfPXVV7hy5QrWr1+P7du3q50c7dixYygoKOBe3t7eTfOHIIQYLQEz8HCmr68vBg8ejC1btgAAVCoVXFxcMHv2bISHh9eKHzt2LMrLy/HTTz9xZc8//zwGDBiA7du3gzEGZ2dnzJ07F/PmzQMAlJaWwtHRETt37sS4cePU9mPNmjXYtm0brl+/DqB6j9LDwwNpaWkYMGBAg7ZNlwnWjYl7+OFaZRP93LAyqK/a+qxPRkJkZtosfSNEX3T5fhp0j1KhUCA1NRVSqZQrMzExgVQqRXJystplkpOTefEAEBAQwMXn5uZCJpPxYsRiMXx9fetsE6hOpuomSRs9ejQ6deqEoUOH4tChQ/VuT0VFBeRyOe/VFtCoN2ntDJooi4qKoFQqa01z6+joCJlMpnYZmUxWb3zNT13azM7OxubNmzFjxgyuzMbGBuvWrUN8fDwOHz6MoUOHIjg4uN5kGRkZCbFYzL1cXFzqjG1pKBmStqxBszC2Jnfu3MHIkSPxz3/+E9OmTePKHRwcEBYWxr0fPHgw8vPzsWbNGowePVptW4sWLeItI5fLW1SyrKhSIuduuaG7QYjRMWiidHBwgKmpKQoLC3nlhYWFkEgkapeRSCT1xtf8LCwshJOTEy/m2XON+fn5eOWVVzBkyBDs2LFDY399fX2RkJBQZ71IJIJIJNLYjrEKiT6D09eLDd0NQoyOQQ+9hUIhvL29kZiYyJWpVCokJibCz89P7TJ+fn68eABISEjg4j08PCCRSHgxcrkcKSkpvDbv3LmDl19+Gd7e3oiJiYGJieY/xYULF3jJt7WhJEmIegY/9A4LC0NISAgGDRoEHx8fbNiwAeXl5Zg8eTIAYOLEiejcuTMiIyMBAHPmzIG/vz/WrVuHwMBAxMbG4ty5c9weoUAgwIcffohPPvkEnp6e8PDwwNKlS+Hs7Izg4GAAfyVJNzc3rF27Fvfu3eP6U7NHumvXLgiFQgwcOBAAsH//fkRHR+Nf//pXc/1pOL9du4cDaXcQMboPxJbmzb5+gK6VJG2bwRPl2LFjce/ePSxbtgwymQwDBgzA0aNHucGYvLw83t7ekCFDsHfvXixZsgSLFy+Gp6cnDhw4gL59+3IxCxYsQHl5OaZPn46SkhIMHToUR48ehYWFBYDqPdDs7GxkZ2ejS5cuvP48fbXUqlWrcPPmTZiZmaFnz57Yt28f/u///q8p/xxqTYyuvgDf1sIMK4L6aogmhOibwa+jbM30dR1lzXWL0l6d8K+QwfrqXp3rUeed513xSbCX2rjMlSNhKaTrKEnL0mKuoySEkJaAEiXRSs1xh7oDkNwiuqSItG6UKFsQYzhJolTV7sRrm/4H+ZNKA/SGkOZBiZLopEpNogSA5Jz7zdwTQpoPJUqiE1Udu7UmAkEz94SQ5kOJkmilJj3WtUdpQnmStGKUKIlOVHUmSsqUpPWiRNmCGMFYTp17lKA8SVoxSpREJ7RHSdoiSpREKzVjONGnbqitN6VESVoxSpQtSEPuNv2jsAxf/3YdFVVKvfRh+8kcteU0mENaM4M/FIM0reHrfwNQ/VDeWa96NqqtgtLHdVdSoiStGO1RthFrf73WqOUrlSr4Rf63zno6R0laM0qULYghR71LH9d/iyIlStKaUaJs4ZQqhkkxZ7DyxwxUKVVIyi7CI0WV2tjGPFEvIaNQcxAhrRSdo2zhzuQW40TWPZzIuofoU7kAAL+uHfDtVF8sOXCZF+v9yTGEDvXA6P7OcGlvpdd+1HVrIyGtAe1RtnAKpapWWfL1++i2+Ai+O5PHKy8uV2DNL1l4cfVxlD7S79N+Lt8p1Wt7hBgTSpQtyImse3hSyb/Mp6F7cv1X/ooqNUm2oT45nKm3tggxNpQoW5g3tyVh7FfJuCqTA2jcecdbD+q53KcBEjIKMeGbFOSX6LddQgyNEmULcyVfjpTcYozc8D88KFc06mG+r6w9gYMX7uitb9N2n8P//ijC5v9m661NQowBDeYYKcYYFv7nErq0q3vQ5bVN/8PKRs7KOCf2AoIGdG5UG7XRwA7RM8YAVRWgrARUlYBK+dfvysqn6qqqy2wkgFh//66NIlFu3boVa9asgUwmQ//+/bF582b4+PjUGR8fH4+lS5fixo0b8PT0xOeff47XXnuNq2eMISIiAl9//TVKSkrwwgsvYNu2bfD0/OvOlOLiYsyePRs//vgjTExM8Oabb2Ljxo2wsbHhYi5duoSZM2fi7Nmz6NixI2bPno0FCxboffvvlj3BP75MQmA/J9wufoyAvhKc+qMIcedu17tcQekTTNt9rtHr/0/qbbzp3UVzoJZc21tzv+fdf4STf9zDuMEuMDelA5hmoWtSUVbx655eVpu6p9vTap1atqGs+quPKvWXvNXp5cXAywv19ic1eKLct28fwsLCsH37dvj6+mLDhg0ICAhAVlYWOnXqVCs+KSkJ48ePR2RkJF5//XXs3bsXwcHBOH/+PDe39+rVq7Fp0ybs2rULHh4eWLp0KQICApCRkcHN7f3222+joKAACQkJqKysxOTJkzF9+nTs3bsXQPVUliNGjIBUKsX27duRnp6OKVOmwN7eHtOnT9fr3+DL4zm4/eAxvjp5HQBwOL1Ar+1rMjf+IqS9HfXW3tPXno/e+jtKHlWiqKwCHw3vrrd1aEupYjCt70b0Z5PK01/OWl/+qqZJKvWuU6l7+7omlRZLAJiaAybmgKlZ9U8Ts+oykY3mxXVZk6Hn9fb19cXgwYOxZcsWAIBKpYKLiwtmz56N8PDwWvFjx45FeXk5fvrpJ67s+eefx4ABA7B9+3YwxuDs7Iy5c+di3rx5AIDS0lI4Ojpi586dGDduHDIzM9G7d2+cPXsWgwYNAgAcPXoUr732Gm7fvg1nZ2ds27YNH3/8MWQyGYRCIQAgPDwcBw4cwNWrV7XaNm3nDY7ZuBTX78qhhCmqYAIl+/MnTFEFUyhhwv/Jq1cTx2ri1cepIEDDbs5mMIMSZlDCHEqYoQpmUFWXCar+LFNiil9njPN2BpRVGLvtfzATVMFVbI7IoF562ovRLqlUKCpQUFwGsUgAewsBBJRU1CcV7mdN3Z/1z9aZmNZuoyamVht1tG9iVntZrg0t2+XqGjeXvC7zeht0j1KhUCA1NRWLFi3iykxMTCCVSpGcnKx2meTkZISFhfHKAgICcODAAQBAbm4uZDIZpFIpVy8Wi+Hr64vk5GSMGzcOycnJsLe355IkAEilUpiYmCAlJQVvvPEGkpOT8dJLL3FJsmY9n3/+OR48eIB27drV6ltFRQUqKiq493K5XKu/w/8Vfw1b8+YdKa5ktRNwzUsAxiW96qRYnV7NBVo+gej8ny8A+0R/lj0GENsEG1IPEQB3AQDFny8tqSD48+9ihirBnz///I9HKTDj1VX/V2GCqqfL/4zj/gP7s64KZlAKTJ/6j+3pNky535UwRdWfy9ffRvXvVerW+UzfmaCe0x4MgMH/v6jSayfeft4NE55301t7Bk2URUVFUCqVcHTkH/Y5OjrWudcmk8nUxstkMq6+pqy+mGcP683MzNC+fXtejIeHR602aurUJcrIyEisWLGi7g1W40mlEr+oBsMCFX9+XZR/flWq99RMBX/+hPKp+qfiuPpnfyohrCexmQuq9wqBxl14rmQCXpKthNlf71lNmSn3Ja/8c4+4krdMzZfdBFXM7JllnopjZrXLuPX82TZMeH2oZE+1/WdMTd8qYfZn/F/JphKmYK3iYpCa7GfwDGgQRWUVmoN0YPBzlK3JokWLeHu7crkcLi4u9S5jYW6K4Yv3Iym7CKl5D7Ar6SZm+HeFt1s7JN98gJTrxWhvLcTRK9UJfEi3DjA1ESAjX45hvTqhg40IXR2swVj1XTpCUxNcvF0C9w7VAyr5JeUY2MUGRy/eQR+JFcwFKtgIBTCDEtfvlsLZ1gyFJQ9hLzJB/87WeFD2GFcKyuBobwPXjmLckSsgtrFGD+d2OHzlHkxMzfFK786wtrKAiakQv2TchbO9JaxFZuhkK8IjhRKPFX8laAaG+w8VcLAR8bbbBIDwz1dTEpqZwM7SDEVlOuxSkhbPVc+36Bo0UTo4OMDU1BSFhfwHLhQWFkIikahdRiKR1Btf87OwsBBOTk68mAEDBnAxd+/e5bVRVVWF4uJiXjvq1vP0Op4lEokgEonU1tVHbGmOUV5OGOXlhI8De3PlL/eoPZiljTGDayfn0X/zUBOpXkAd5bO71n6e5dQX9XvSvMmo/8gI0YpBjzGEQiG8vb2RmJjIlalUKiQmJsLPz0/tMn5+frx4AEhISODiPTw8IJFIeDFyuRwpKSlcjJ+fH0pKSpCamsrF/Pe//4VKpYKvry8X89tvv6GyspK3nh49eqg97CaEtGLMwGJjY5lIJGI7d+5kGRkZbPr06cze3p7JZDLGGGMTJkxg4eHhXPypU6eYmZkZW7t2LcvMzGQRERHM3NycpaenczFRUVHM3t6eHTx4kF26dIkFBQUxDw8P9vjxYy5m5MiRbODAgSwlJYX9/vvvzNPTk40fP56rLykpYY6OjmzChAns8uXLLDY2lllZWbGvvvpK620rLS1lAFhpaWlj/kSEkCagy/fT4ImSMcY2b97MXF1dmVAoZD4+Puz06dNcnb+/PwsJCeHFx8XFse7duzOhUMj69OnDDh8+zKtXqVRs6dKlzNHRkYlEIjZs2DCWlZXFi7l//z4bP348s7GxYXZ2dmzy5MmsrKyMF3Px4kU2dOhQJhKJWOfOnVlUVJRO20WJkhDjpcv30+DXUbZmulynRQhpXrp8P1vDdRCEENKkKFESQogGlCgJIUQDuuC8CdWc/tX2VkZCSPOp+V5qM0xDibIJlZWVAYDGu3MIIYZTVlYGsVhcbwyNejchlUqF/Px82NraQmDE817X3Gp569atVjU631q3C2i929ac28UYQ1lZGZydnWFiUv9ZSNqjbEImJibo0kV/D8RtanZ2dq3qS1ejtW4X0Hq3rbm2S9OeZA0azCGEEA0oURJCiAaUKAlEIhEiIiIa9OQjY9ZatwtovdtmrNtFgzmEEKIB7VESQogGlCgJIUQDSpSEEKIBJUpCCNGAEmUrtXXrVri7u8PCwgK+vr44c+ZMvfHx8fHo2bMnLCws4OXlhSNHjvDqGWNYtmwZnJycYGlpCalUij/++KMpN0EtXbbr66+/xosvvoh27dqhXbt2kEqlteInTZoEgUDAe40cObKpN6MWXbZr586dtfpsYWHBizGWzwvQbdtefvnlWtsmEAgQGBjIxRjkM2uqpwcTw4mNjWVCoZBFR0ezK1eusGnTpjF7e3tWWFioNv7UqVPM1NSUrV69mmVkZLAlS5aonV5DLBazAwcOsIsXL7LRo0fXml6jqem6XW+99RbbunUrS0tLY5mZmWzSpElMLBaz27dvczEhISFs5MiRrKCggHsVFxc31yYxxnTfrpiYGGZnZ8frc83UKTWM4fNiTPdtu3//Pm+7Ll++zExNTVlMTAwXY4jPjBJlK+Tj48NmzpzJvVcqlczZ2ZlFRkaqjR8zZgwLDAzklfn6+rIZM2Ywxqqn1pBIJGzNmjVcfUlJCROJROy7775rgi1QT9ftelZVVRWztbVlu3bt4spCQkJYUFCQvruqE123KyYmhonF4jrbM5bPi7HGf2br169ntra27OHDh1yZIT4zOvRuZRQKBVJTUyGVSrkyExMTSKVSJCcnq10mOTmZFw8AAQEBXHxubi5kMhkvRiwWw9fXt8429a0h2/WsR48eobKyEu3bt+eVnzhxAp06dUKPHj3w3nvv4f79+3rte30aul0PHz6Em5sbXFxcEBQUhCtXrnB1xvB5Afr5zL755huMGzcO1tbWvPLm/swoUbYyRUVFUCqVcHR05JU7OjpCJpOpXUYmk9UbX/NTlzb1rSHb9ayFCxfC2dmZ98UdOXIkdu/ejcTERHz++ec4efIkRo0aBaVSqdf+16Uh29WjRw9ER0fj4MGD+Pbbb6FSqTBkyBDcvn0bgHF8XkDjP7MzZ87g8uXLmDp1Kq/cEJ8ZPT2ItAlRUVGIjY3FiRMneAMf48aN43738vJCv3790K1bN5w4cQLDhg0zRFc18vPz4817P2TIEPTq1QtfffUVVq1aZcCe6dc333wDLy8v+Pj48MoN8ZnRHmUr4+DgAFNTUxQWFvLKCwsLIZFI1C4jkUjqja/5qUub+taQ7aqxdu1aREVF4ddff0W/fv3qje3atSscHByQnZ3d6D5rozHbVcPc3BwDBw7k+mwMnxfQuG0rLy9HbGwsQkNDNa6nOT4zSpStjFAohLe3NxITE7kylUqFxMRE3l7I0/z8/HjxAJCQkMDFe3h4QCKR8GLkcjlSUlLqbFPfGrJdALB69WqsWrUKR48exaBBgzSu5/bt27h//z6cnJz00m9NGrpdT1MqlUhPT+f6bAyfF9C4bYuPj0dFRQXeeecdjetpls+sWYeOSLOIjY1lIpGI7dy5k2VkZLDp06cze3t77hKSCRMmsPDwcC7+1KlTzMzMjK1du5ZlZmayiIgItZcH2dvbs4MHD7JLly6xoKAgg1wepMt2RUVFMaFQyL7//nvepSRlZWWMMcbKysrYvHnzWHJyMsvNzWXHjh1jf/vb35inpyd78uSJ0W7XihUr2C+//MJycnJYamoqGzduHLOwsGBXrlzhbbuhP6+GbFuNoUOHsrFjx9YqN9RnRomyldq8eTNzdXVlQqGQ+fj4sNOnT3N1/v7+LCQkhBcfFxfHunfvzoRCIevTpw87fPgwr16lUrGlS5cyR0dHJhKJ2LBhw1hWVlZzbAqPLtvl5ubGANR6RUREMMYYe/ToERsxYgTr2LEjMzc3Z25ubmzatGm1rklsDrps14cffsjFOjo6stdee42dP3+e156xfF6M6f5v8erVqwwA+/XXX2u1ZajPjB6zRgghGtA5SkII0YASJSGEaECJkhBCNKBESQghGlCiJIQQDShREkKIBpQoCSFEA0qUhBCiASVK0qaom0bAUNM/kJaDHrNG2pyRI0ciJiaGVyYSidTGVlZWwtzcnFemUCggFAp1Xm9DlyOGR3uUpM0RiUSQSCS8V7t27QAAAoEA27Ztw+jRo2FtbY1PP/0Uy5cvx4ABA/Cvf/0LHh4e3PMs8/LyEBQUBBsbG9jZ2WHMmDG8R4rVtRxpeShREvKM5cuX44033kB6ejqmTJkCAMjOzsZ//vMf7N+/HxcuXIBKpUJQUBCKi4tx8uRJJCQk4Pr16xg7diyvrWeXIy0THXqTNuenn36CjY0Nr2zx4sVYvHgxAOCtt97C5MmTefUKhQK7d+9Gx44dAVQ/rzM9PR25ublwcXEBAOzevRt9+vTB2bNnMXjwYLXLkZaJEiVpc1555RVs27aNV/b0hGPqHvDr5ubGS3aZmZlwcXHhkiQA9O7dG/b29sjMzOQS5bPLkZaJEiVpc6ytrfHcc8/VW69NmbbrIi0fnaMkpAF69eqFW7du4datW1xZRkYGSkpK0Lt3bwP2jDQF2qMkbU5FRUWt6VLNzMzg4OCgdRtSqRReXl54++23sWHDBlRVVeH999+Hv7+/VnPzkJaF9ihJm3P06FE4OTnxXkOHDtWpDYFAgIMHD6Jdu3Z46aWXIJVK0bVrV+zbt6+Jek0MiaaCIIQQDWiPkhBCNKBESQghGlCiJIQQDShREkKIBpQoCSFEA0qUhBCiASVKQgjRgBIlIYRoQImSEEI0oERJCCEaUKIkhBANKFESQogG/w8ziwal50tv9QAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.021519, + "end_time": "2024-03-22T15:36:30.978068", + "exception": false, + "start_time": "2024-03-22T15:36:30.956549", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4092.449375, + "end_time": "2024-03-22T15:36:33.721338", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/tab_ddpm_concat/0/mlu-eval.ipynb", + "output_path": "eval/insurance/tab_ddpm_concat/0/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/tab_ddpm_concat/0", + "path_prefix": "../../../../", + "random_seed": 0, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-03-22T14:28:21.271963", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/tab_ddpm_concat/model.pt b/insurance/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..27af9ed8e42c0648b45c239283ddfd46604d475f --- /dev/null +++ b/insurance/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eee2095aa46c24eef37ab46962474cd61a69f63df282cbec474316fad5cb6434 +size 38514197 diff --git a/insurance/tab_ddpm_concat/params.json b/insurance/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..0fcce4f65df4f88805a080b85bb9aecc9f572a11 --- /dev/null +++ b/insurance/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "NONE", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "fixed_role_model": "tab_ddpm_concat", "mse_mag": false, "mse_mag_target": 0.1, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/tvae/eval.csv b/insurance/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..ae5da4e49cbebc568fced2cf2ce020868f693827 --- /dev/null +++ b/insurance/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.006532281896756364,0.006128870656899121,0.0005013272912696092,6.598551273345947,0.004663586150854826,0.35133224725723267,0.013413011096417904,4.282439931557747e-06,2.3958842754364014,0.013242576271295547,0.5489071607589722,0.02239033952355385,0.14861759543418884,2.2368631107383408e-06,8.994435548782349 diff --git a/insurance/tvae/history.csv b/insurance/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..bc183484bef28ff17ab2867600888ce53d89f5ea --- /dev/null +++ b/insurance/tvae/history.csv @@ -0,0 +1,19 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.043772892344924104,0.9691945454827052,0.020741946386080963,0.0,0.0,0.0,0.0,0.0,0.043772892344924104,900,113,111.33565783500671,0.9852713082743957,0.1237062864833408,0.11609506887276616,0.007104319949220452,0.48655429802302186,0.002825021496607657,0.0,0.0,0.0,0.0,0.0,0.007104319949220452,450,57,46.08293104171753,0.808472474416097,0.10240651342603896,0.062048352895337236 +1,0.005487369931975587,0.7738947147227733,0.0006779940362103653,0.0,0.0,0.0,0.0,0.0,0.005487369931975587,900,113,112.22859740257263,0.9931734283413507,0.12469844155841403,0.08416169975777116,0.006982278515998688,0.4736000236227578,0.0015233677555578614,0.0,0.0,0.0,0.0,0.0,0.006982278515998688,450,57,46.10599446296692,0.8088770958415249,0.10245776547325983,0.05901416842090456 +2,0.0028347509825188254,0.3354986571064682,1.5617633661304407e-05,0.0,0.0,0.0,0.0,0.0,0.0028347509825188254,900,113,111.97222852706909,0.9909046772307,0.124413587252299,0.09286229135221348,0.0037852605881117697,0.25047095601733244,0.0007742868338338995,0.0,0.0,0.0,0.0,0.0,0.0037852605881117697,450,57,46.205010652542114,0.810614221974423,0.10267780145009359,0.07840819853325293 +3,0.0019783554043776045,0.21749820665938444,1.8945457476570026e-05,0.0,0.0,0.0,0.0,0.0,0.0019783554043776045,900,113,111.77877640724182,0.9891927115685117,0.12419864045249092,0.09672643638224201,0.003601218523528789,0.548966435297753,0.0004845707171051153,0.0,0.0,0.0,0.0,0.0,0.003601218523528789,450,57,45.79237937927246,0.8033750768293414,0.10176084306504991,0.060725935640859235 +4,0.0015077938184711254,0.1864952617832508,1.1623385739693803e-05,0.0,0.0,0.0,0.0,0.0,0.0015077938184711254,900,113,111.0480124950409,0.9827257742923973,0.12338668055004544,0.09504026204215742,0.002632722834694303,0.21741927998951283,0.00043822755775758303,0.0,0.0,0.0,0.0,0.0,0.002632722834694303,450,57,45.50333523750305,0.7983041269737378,0.10111852275000678,0.07700717940583433 +5,0.0007533894496090296,0.0954851591936076,1.1907169007617899e-06,0.0,0.0,0.0,0.0,0.0,0.0007533894496090296,900,113,112.60472917556763,0.9965020281023684,0.1251163657506307,0.09669116178972531,0.0019194719764669167,0.1813253381316862,8.795860155298997e-05,0.0,0.0,0.0,0.0,0.0,0.0019194719764669167,450,57,47.84044623374939,0.8393060742763051,0.1063121027416653,0.07234944092322207 +6,0.0008143671194233724,0.07775253331788894,2.0004750451023264e-06,0.0,0.0,0.0,0.0,0.0,0.0008143671194233724,900,113,113.22330784797668,1.0019761756458114,0.12580367538664075,0.09869995380265523,0.0025437849643316843,0.1558871896347034,0.0004921607287944122,0.0,0.0,0.0,0.0,0.0,0.0025437849643316843,450,57,45.85625100135803,0.8044956316027725,0.10190278000301785,0.07156802042951121 +7,0.0006387245669288354,0.05651727137908936,2.687618409259153e-06,0.0,0.0,0.0,0.0,0.0,0.0006387245669288354,900,113,111.7732937335968,0.9891441923327151,0.12419254859288534,0.09920014948707766,0.0021676956941114947,0.39689580948561143,0.00020905914819549176,0.0,0.0,0.0,0.0,0.0,0.0021676956941114947,450,57,45.570988178253174,0.7994910206711083,0.10126886261834038,0.06560296467668786 +8,0.0005393729344682975,0.09357239947843243,5.010212360393345e-07,0.0,0.0,0.0,0.0,0.0,0.0005393729344682975,900,113,112.51995134353638,0.9957517818012068,0.12502216815948486,0.09689272767493287,0.0020265914760804866,0.585885123172634,0.00028776428296029374,0.0,0.0,0.0,0.0,0.0,0.0020265914760804866,450,57,47.63360095024109,0.8356772096533525,0.10585244655609131,0.07264613465757289 +9,0.00030413587897783147,0.03623591635612869,6.036034924474541e-08,0.0,0.0,0.0,0.0,0.0,0.00030413587897783147,900,113,113.07414937019348,1.0006561891167565,0.12563794374465942,0.09631216084271406,0.001967307192001802,0.5783613658454081,0.00021626046908912677,0.0,0.0,0.0,0.0,0.0,0.001967307192001802,450,57,45.948439598083496,0.8061129754049736,0.10210764355129666,0.0713601329470086 +10,0.0002241133607943387,0.02460616329780395,5.1179004124437636e-08,0.0,0.0,0.0,0.0,0.0,0.0002241133607943387,900,113,111.44915223121643,0.9862756834620923,0.12383239136801825,0.09834809370536719,0.001825745276728412,0.44179768500308064,0.00015850803496538857,0.0,0.0,0.0,0.0,0.0,0.001825745276728412,450,57,45.60394787788391,0.8000692610155072,0.10134210639529757,0.06884222721942422 +11,0.00033303717160985496,0.03369398813643545,2.4871881194393203e-07,0.0,0.0,0.0,0.0,0.0,0.00033303717160985496,900,113,111.49134683609009,0.9866490870450451,0.12387927426232231,0.10159270219768571,0.0023319287935414145,0.2600294268881457,0.00038547293633122223,0.0,0.0,0.0,0.0,0.0,0.0023319287935414145,450,57,45.5991792678833,0.7999856011909351,0.10133150948418511,0.07801991980522871 +12,0.0002562747773643221,0.028784445838175392,2.593463381772865e-08,0.0,0.0,0.0,0.0,0.0,0.0002562747773643221,900,113,111.06798338890076,0.9829025078663783,0.12340887043211195,0.09953929795430297,0.001840519470117417,0.3625226045031678,0.00015924821697381686,0.0,0.0,0.0,0.0,0.0,0.001840519470117417,450,57,45.293455839157104,0.7946220322659141,0.10065212408701579,0.07107936053969816 +13,0.0001757131654206508,0.018957749903747066,4.845866322408336e-08,0.0,0.0,0.0,0.0,0.0,0.0001757131654206508,900,113,110.83889031410217,0.9808751355230281,0.12315432257122463,0.09919166324281059,0.0019331971814648974,0.6399211409059746,0.00020584285959550467,0.0,0.0,0.0,0.0,0.0,0.0019331971814648974,450,57,45.47143840789795,0.7977445334718939,0.10104764090643989,0.07058916541363783 +14,0.00021455827955732174,0.031199176489886352,2.883131269934558e-08,0.0,0.0,0.0,0.0,0.0,0.00021455827955732174,900,113,110.77680969238281,0.9803257494901134,0.12308534410264757,0.10197576969466378,0.0018430328381752285,0.4276724044195193,0.00019247611021483608,0.0,0.0,0.0,0.0,0.0,0.0018430328381752285,450,57,45.94011998176575,0.8059670172239605,0.10208915551503499,0.07040622951707949 +15,0.0004361093971238006,0.05909316669737025,2.2801978405284285e-07,0.0,0.0,0.0,0.0,0.0,0.0004361093971238006,900,113,112.09613871574402,0.992001227572956,0.12455126523971558,0.1005768165882446,0.001964426144291388,0.25929170938355406,0.00015272240531687408,0.0,0.0,0.0,0.0,0.0,0.001964426144291388,450,57,45.62297987937927,0.8004031557785837,0.10138439973195394,0.06955495999570478 +16,0.0001958047746868235,0.016338927822493122,4.4443133083278144e-08,0.0,0.0,0.0,0.0,0.0,0.0001958047746868235,900,113,111.31650757789612,0.9851018369725321,0.12368500841988457,0.09745020749030915,0.002082800301011755,0.7195598722440445,0.0001506238370249388,0.0,0.0,0.0,0.0,0.0,0.002082800301011755,450,57,45.92464828491211,0.8056955839458265,0.10205477396647135,0.0661866773580346 +17,0.0003942243970474616,0.05046572757249398,1.0653629163880412e-07,0.0,0.0,0.0,0.0,0.0,0.0003942243970474616,900,113,112.87545800209045,0.9988978584255792,0.1254171755578783,0.09861106915086244,0.0017001370618042225,0.6304228919395235,0.0001222305167977231,0.0,0.0,0.0,0.0,0.0,0.0017001370618042225,450,57,48.029250144958496,0.8426184235957631,0.10673166698879666,0.07135208257424988 diff --git a/insurance/tvae/mlu-eval.ipynb b/insurance/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..7e4cc09285075ecc0a3cd64e9f395d94b873b850 --- /dev/null +++ b/insurance/tvae/mlu-eval.ipynb @@ -0,0 +1,2439 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:25.941055Z", + "iopub.status.busy": "2024-03-22T19:36:25.940459Z", + "iopub.status.idle": "2024-03-22T19:36:25.974075Z", + "shell.execute_reply": "2024-03-22T19:36:25.973199Z" + }, + "papermill": { + "duration": 0.048368, + "end_time": "2024-03-22T19:36:25.976101", + "exception": false, + "start_time": "2024-03-22T19:36:25.927733", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:26.001236Z", + "iopub.status.busy": "2024-03-22T19:36:26.000620Z", + "iopub.status.idle": "2024-03-22T19:36:26.007106Z", + "shell.execute_reply": "2024-03-22T19:36:26.006337Z" + }, + "papermill": { + "duration": 0.021117, + "end_time": "2024-03-22T19:36:26.009100", + "exception": false, + "start_time": "2024-03-22T19:36:25.987983", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:26.032442Z", + "iopub.status.busy": "2024-03-22T19:36:26.032190Z", + "iopub.status.idle": "2024-03-22T19:36:26.036150Z", + "shell.execute_reply": "2024-03-22T19:36:26.035447Z" + }, + "papermill": { + "duration": 0.017829, + "end_time": "2024-03-22T19:36:26.038024", + "exception": false, + "start_time": "2024-03-22T19:36:26.020195", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:26.061356Z", + "iopub.status.busy": "2024-03-22T19:36:26.061092Z", + "iopub.status.idle": "2024-03-22T19:36:26.064917Z", + "shell.execute_reply": "2024-03-22T19:36:26.064114Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017808, + "end_time": "2024-03-22T19:36:26.066934", + "exception": false, + "start_time": "2024-03-22T19:36:26.049126", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:26.090371Z", + "iopub.status.busy": "2024-03-22T19:36:26.089852Z", + "iopub.status.idle": "2024-03-22T19:36:26.095346Z", + "shell.execute_reply": "2024-03-22T19:36:26.094581Z" + }, + "papermill": { + "duration": 0.019246, + "end_time": "2024-03-22T19:36:26.097152", + "exception": false, + "start_time": "2024-03-22T19:36:26.077906", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "9cecfcd4", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:26.122151Z", + "iopub.status.busy": "2024-03-22T19:36:26.121898Z", + "iopub.status.idle": "2024-03-22T19:36:26.126548Z", + "shell.execute_reply": "2024-03-22T19:36:26.125771Z" + }, + "papermill": { + "duration": 0.019312, + "end_time": "2024-03-22T19:36:26.128420", + "exception": false, + "start_time": "2024-03-22T19:36:26.109108", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"tvae\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/tvae/4\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011044, + "end_time": "2024-03-22T19:36:26.150696", + "exception": false, + "start_time": "2024-03-22T19:36:26.139652", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:26.174130Z", + "iopub.status.busy": "2024-03-22T19:36:26.173877Z", + "iopub.status.idle": "2024-03-22T19:36:26.182541Z", + "shell.execute_reply": "2024-03-22T19:36:26.181801Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022594, + "end_time": "2024-03-22T19:36:26.184360", + "exception": false, + "start_time": "2024-03-22T19:36:26.161766", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/tvae/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:26.208093Z", + "iopub.status.busy": "2024-03-22T19:36:26.207835Z", + "iopub.status.idle": "2024-03-22T19:36:28.197587Z", + "shell.execute_reply": "2024-03-22T19:36:28.196656Z" + }, + "papermill": { + "duration": 2.004156, + "end_time": "2024-03-22T19:36:28.199776", + "exception": false, + "start_time": "2024-03-22T19:36:26.195620", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:28.227085Z", + "iopub.status.busy": "2024-03-22T19:36:28.226421Z", + "iopub.status.idle": "2024-03-22T19:36:28.239393Z", + "shell.execute_reply": "2024-03-22T19:36:28.238658Z" + }, + "papermill": { + "duration": 0.028615, + "end_time": "2024-03-22T19:36:28.241339", + "exception": false, + "start_time": "2024-03-22T19:36:28.212724", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:28.266996Z", + "iopub.status.busy": "2024-03-22T19:36:28.266705Z", + "iopub.status.idle": "2024-03-22T19:36:28.273884Z", + "shell.execute_reply": "2024-03-22T19:36:28.273064Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.022113, + "end_time": "2024-03-22T19:36:28.275699", + "exception": false, + "start_time": "2024-03-22T19:36:28.253586", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:28.299266Z", + "iopub.status.busy": "2024-03-22T19:36:28.299010Z", + "iopub.status.idle": "2024-03-22T19:36:28.392106Z", + "shell.execute_reply": "2024-03-22T19:36:28.391185Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.107264, + "end_time": "2024-03-22T19:36:28.394206", + "exception": false, + "start_time": "2024-03-22T19:36:28.286942", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:28.420380Z", + "iopub.status.busy": "2024-03-22T19:36:28.420065Z", + "iopub.status.idle": "2024-03-22T19:36:32.969915Z", + "shell.execute_reply": "2024-03-22T19:36:32.969117Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.565459, + "end_time": "2024-03-22T19:36:32.972258", + "exception": false, + "start_time": "2024-03-22T19:36:28.406799", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 19:36:30.629297: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 19:36:30.629359: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 19:36:30.631138: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:32.997148Z", + "iopub.status.busy": "2024-03-22T19:36:32.996589Z", + "iopub.status.idle": "2024-03-22T19:36:33.003437Z", + "shell.execute_reply": "2024-03-22T19:36:33.002562Z" + }, + "papermill": { + "duration": 0.021183, + "end_time": "2024-03-22T19:36:33.005353", + "exception": false, + "start_time": "2024-03-22T19:36:32.984170", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:33.030823Z", + "iopub.status.busy": "2024-03-22T19:36:33.030522Z", + "iopub.status.idle": "2024-03-22T19:36:41.275443Z", + "shell.execute_reply": "2024-03-22T19:36:41.274377Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.26053, + "end_time": "2024-03-22T19:36:41.277787", + "exception": false, + "start_time": "2024-03-22T19:36:33.017257", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T19:36:41.305399Z", + "iopub.status.busy": "2024-03-22T19:36:41.305077Z", + "iopub.status.idle": "2024-03-22T19:36:41.311782Z", + "shell.execute_reply": "2024-03-22T19:36:41.310866Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022694, + "end_time": "2024-03-22T19:36:41.313761", + "exception": false, + "start_time": "2024-03-22T19:36:41.291067", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:41.338293Z", + "iopub.status.busy": "2024-03-22T19:36:41.338022Z", + "iopub.status.idle": "2024-03-22T19:36:41.342673Z", + "shell.execute_reply": "2024-03-22T19:36:41.341898Z" + }, + "papermill": { + "duration": 0.01919, + "end_time": "2024-03-22T19:36:41.344514", + "exception": false, + "start_time": "2024-03-22T19:36:41.325324", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:41.369046Z", + "iopub.status.busy": "2024-03-22T19:36:41.368744Z", + "iopub.status.idle": "2024-03-22T19:36:41.844096Z", + "shell.execute_reply": "2024-03-22T19:36:41.843218Z" + }, + "papermill": { + "duration": 0.490223, + "end_time": "2024-03-22T19:36:41.846363", + "exception": false, + "start_time": "2024-03-22T19:36:41.356140", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/tvae/all inf False\n", + "../../../../ml-utility-loss/aug_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_bs_test/tvae/all inf False\n", + "../../../../ml-utility-loss/bs_test/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../insurance/_cache_synth_test/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/insurance [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:41.873690Z", + "iopub.status.busy": "2024-03-22T19:36:41.873394Z", + "iopub.status.idle": "2024-03-22T19:36:42.191447Z", + "shell.execute_reply": "2024-03-22T19:36:42.190368Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.333964, + "end_time": "2024-03-22T19:36:42.193502", + "exception": false, + "start_time": "2024-03-22T19:36:41.859538", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:42.221326Z", + "iopub.status.busy": "2024-03-22T19:36:42.221002Z", + "iopub.status.idle": "2024-03-22T19:36:42.326968Z", + "shell.execute_reply": "2024-03-22T19:36:42.325962Z" + }, + "papermill": { + "duration": 0.122171, + "end_time": "2024-03-22T19:36:42.329061", + "exception": false, + "start_time": "2024-03-22T19:36:42.206890", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T19:36:42.356294Z", + "iopub.status.busy": "2024-03-22T19:36:42.355993Z", + "iopub.status.idle": "2024-03-22T19:36:42.770869Z", + "shell.execute_reply": "2024-03-22T19:36:42.769958Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.430746, + "end_time": "2024-03-22T19:36:42.772931", + "exception": false, + "start_time": "2024-03-22T19:36:42.342185", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:42.802094Z", + "iopub.status.busy": "2024-03-22T19:36:42.801206Z", + "iopub.status.idle": "2024-03-22T19:36:42.805978Z", + "shell.execute_reply": "2024-03-22T19:36:42.805186Z" + }, + "papermill": { + "duration": 0.021363, + "end_time": "2024-03-22T19:36:42.807778", + "exception": false, + "start_time": "2024-03-22T19:36:42.786415", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:42.834150Z", + "iopub.status.busy": "2024-03-22T19:36:42.833642Z", + "iopub.status.idle": "2024-03-22T19:36:42.840629Z", + "shell.execute_reply": "2024-03-22T19:36:42.839780Z" + }, + "papermill": { + "duration": 0.022435, + "end_time": "2024-03-22T19:36:42.842600", + "exception": false, + "start_time": "2024-03-22T19:36:42.820165", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9638537" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:42.868649Z", + "iopub.status.busy": "2024-03-22T19:36:42.868387Z", + "iopub.status.idle": "2024-03-22T19:36:42.958407Z", + "shell.execute_reply": "2024-03-22T19:36:42.957528Z" + }, + "papermill": { + "duration": 0.105438, + "end_time": "2024-03-22T19:36:42.960463", + "exception": false, + "start_time": "2024-03-22T19:36:42.855025", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 36] --\n", + "├─Adapter: 1-1 [2, 1071, 36] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 37,888\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 36] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-34 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-40 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-46 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-52 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,638,537\n", + "Trainable params: 9,638,537\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 38.18\n", + "========================================================================================================================\n", + "Input size (MB): 0.39\n", + "Forward/backward pass size (MB): 307.49\n", + "Params size (MB): 38.55\n", + "Estimated Total Size (MB): 346.43\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T19:36:42.992211Z", + "iopub.status.busy": "2024-03-22T19:36:42.991928Z", + "iopub.status.idle": "2024-03-22T20:26:30.394468Z", + "shell.execute_reply": "2024-03-22T20:26:30.393281Z" + }, + "papermill": { + "duration": 2987.441272, + "end_time": "2024-03-22T20:26:30.416656", + "exception": false, + "start_time": "2024-03-22T19:36:42.975384", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.043772892344924104, 'avg_role_model_std_loss': 0.9691945454827052, 'avg_role_model_mean_pred_loss': 0.020741946386080963, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.043772892344924104, 'n_size': 900, 'n_batch': 113, 'duration': 111.33565783500671, 'duration_batch': 0.9852713082743957, 'duration_size': 0.1237062864833408, 'avg_pred_std': 0.11609506887276616}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007104319949220452, 'avg_role_model_std_loss': 0.48655429802302186, 'avg_role_model_mean_pred_loss': 0.002825021496607657, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007104319949220452, 'n_size': 450, 'n_batch': 57, 'duration': 46.08293104171753, 'duration_batch': 0.808472474416097, 'duration_size': 0.10240651342603896, 'avg_pred_std': 0.062048352895337236}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005487369931975587, 'avg_role_model_std_loss': 0.7738947147227733, 'avg_role_model_mean_pred_loss': 0.0006779940362103653, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005487369931975587, 'n_size': 900, 'n_batch': 113, 'duration': 112.22859740257263, 'duration_batch': 0.9931734283413507, 'duration_size': 0.12469844155841403, 'avg_pred_std': 0.08416169975777116}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006982278515998688, 'avg_role_model_std_loss': 0.4736000236227578, 'avg_role_model_mean_pred_loss': 0.0015233677555578614, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006982278515998688, 'n_size': 450, 'n_batch': 57, 'duration': 46.10599446296692, 'duration_batch': 0.8088770958415249, 'duration_size': 0.10245776547325983, 'avg_pred_std': 0.05901416842090456}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0028347509825188254, 'avg_role_model_std_loss': 0.3354986571064682, 'avg_role_model_mean_pred_loss': 1.5617633661304407e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0028347509825188254, 'n_size': 900, 'n_batch': 113, 'duration': 111.97222852706909, 'duration_batch': 0.9909046772307, 'duration_size': 0.124413587252299, 'avg_pred_std': 0.09286229135221348}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0037852605881117697, 'avg_role_model_std_loss': 0.25047095601733244, 'avg_role_model_mean_pred_loss': 0.0007742868338338995, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0037852605881117697, 'n_size': 450, 'n_batch': 57, 'duration': 46.205010652542114, 'duration_batch': 0.810614221974423, 'duration_size': 0.10267780145009359, 'avg_pred_std': 0.07840819853325293}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0019783554043776045, 'avg_role_model_std_loss': 0.21749820665938444, 'avg_role_model_mean_pred_loss': 1.8945457476570026e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019783554043776045, 'n_size': 900, 'n_batch': 113, 'duration': 111.77877640724182, 'duration_batch': 0.9891927115685117, 'duration_size': 0.12419864045249092, 'avg_pred_std': 0.09672643638224201}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003601218523528789, 'avg_role_model_std_loss': 0.548966435297753, 'avg_role_model_mean_pred_loss': 0.0004845707171051153, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003601218523528789, 'n_size': 450, 'n_batch': 57, 'duration': 45.79237937927246, 'duration_batch': 0.8033750768293414, 'duration_size': 0.10176084306504991, 'avg_pred_std': 0.060725935640859235}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0015077938184711254, 'avg_role_model_std_loss': 0.1864952617832508, 'avg_role_model_mean_pred_loss': 1.1623385739693803e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015077938184711254, 'n_size': 900, 'n_batch': 113, 'duration': 111.0480124950409, 'duration_batch': 0.9827257742923973, 'duration_size': 0.12338668055004544, 'avg_pred_std': 0.09504026204215742}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002632722834694303, 'avg_role_model_std_loss': 0.21741927998951283, 'avg_role_model_mean_pred_loss': 0.00043822755775758303, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002632722834694303, 'n_size': 450, 'n_batch': 57, 'duration': 45.50333523750305, 'duration_batch': 0.7983041269737378, 'duration_size': 0.10111852275000678, 'avg_pred_std': 0.07700717940583433}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007533894496090296, 'avg_role_model_std_loss': 0.0954851591936076, 'avg_role_model_mean_pred_loss': 1.1907169007617899e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007533894496090296, 'n_size': 900, 'n_batch': 113, 'duration': 112.60472917556763, 'duration_batch': 0.9965020281023684, 'duration_size': 0.1251163657506307, 'avg_pred_std': 0.09669116178972531}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019194719764669167, 'avg_role_model_std_loss': 0.1813253381316862, 'avg_role_model_mean_pred_loss': 8.795860155298997e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019194719764669167, 'n_size': 450, 'n_batch': 57, 'duration': 47.84044623374939, 'duration_batch': 0.8393060742763051, 'duration_size': 0.1063121027416653, 'avg_pred_std': 0.07234944092322207}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008143671194233724, 'avg_role_model_std_loss': 0.07775253331788894, 'avg_role_model_mean_pred_loss': 2.0004750451023264e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008143671194233724, 'n_size': 900, 'n_batch': 113, 'duration': 113.22330784797668, 'duration_batch': 1.0019761756458114, 'duration_size': 0.12580367538664075, 'avg_pred_std': 0.09869995380265523}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025437849643316843, 'avg_role_model_std_loss': 0.1558871896347034, 'avg_role_model_mean_pred_loss': 0.0004921607287944122, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025437849643316843, 'n_size': 450, 'n_batch': 57, 'duration': 45.85625100135803, 'duration_batch': 0.8044956316027725, 'duration_size': 0.10190278000301785, 'avg_pred_std': 0.07156802042951121}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0006387245669288354, 'avg_role_model_std_loss': 0.05651727137908936, 'avg_role_model_mean_pred_loss': 2.687618409259153e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0006387245669288354, 'n_size': 900, 'n_batch': 113, 'duration': 111.7732937335968, 'duration_batch': 0.9891441923327151, 'duration_size': 0.12419254859288534, 'avg_pred_std': 0.09920014948707766}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021676956941114947, 'avg_role_model_std_loss': 0.39689580948561143, 'avg_role_model_mean_pred_loss': 0.00020905914819549176, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021676956941114947, 'n_size': 450, 'n_batch': 57, 'duration': 45.570988178253174, 'duration_batch': 0.7994910206711083, 'duration_size': 0.10126886261834038, 'avg_pred_std': 0.06560296467668786}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005393729344682975, 'avg_role_model_std_loss': 0.09357239947843243, 'avg_role_model_mean_pred_loss': 5.010212360393345e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005393729344682975, 'n_size': 900, 'n_batch': 113, 'duration': 112.51995134353638, 'duration_batch': 0.9957517818012068, 'duration_size': 0.12502216815948486, 'avg_pred_std': 0.09689272767493287}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0020265914760804866, 'avg_role_model_std_loss': 0.585885123172634, 'avg_role_model_mean_pred_loss': 0.00028776428296029374, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0020265914760804866, 'n_size': 450, 'n_batch': 57, 'duration': 47.63360095024109, 'duration_batch': 0.8356772096533525, 'duration_size': 0.10585244655609131, 'avg_pred_std': 0.07264613465757289}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00030413587897783147, 'avg_role_model_std_loss': 0.03623591635612869, 'avg_role_model_mean_pred_loss': 6.036034924474541e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00030413587897783147, 'n_size': 900, 'n_batch': 113, 'duration': 113.07414937019348, 'duration_batch': 1.0006561891167565, 'duration_size': 0.12563794374465942, 'avg_pred_std': 0.09631216084271406}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001967307192001802, 'avg_role_model_std_loss': 0.5783613658454081, 'avg_role_model_mean_pred_loss': 0.00021626046908912677, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001967307192001802, 'n_size': 450, 'n_batch': 57, 'duration': 45.948439598083496, 'duration_batch': 0.8061129754049736, 'duration_size': 0.10210764355129666, 'avg_pred_std': 0.0713601329470086}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0002241133607943387, 'avg_role_model_std_loss': 0.02460616329780395, 'avg_role_model_mean_pred_loss': 5.1179004124437636e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0002241133607943387, 'n_size': 900, 'n_batch': 113, 'duration': 111.44915223121643, 'duration_batch': 0.9862756834620923, 'duration_size': 0.12383239136801825, 'avg_pred_std': 0.09834809370536719}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001825745276728412, 'avg_role_model_std_loss': 0.44179768500308064, 'avg_role_model_mean_pred_loss': 0.00015850803496538857, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001825745276728412, 'n_size': 450, 'n_batch': 57, 'duration': 45.60394787788391, 'duration_batch': 0.8000692610155072, 'duration_size': 0.10134210639529757, 'avg_pred_std': 0.06884222721942422}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00033303717160985496, 'avg_role_model_std_loss': 0.03369398813643545, 'avg_role_model_mean_pred_loss': 2.4871881194393203e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00033303717160985496, 'n_size': 900, 'n_batch': 113, 'duration': 111.49134683609009, 'duration_batch': 0.9866490870450451, 'duration_size': 0.12387927426232231, 'avg_pred_std': 0.10159270219768571}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023319287935414145, 'avg_role_model_std_loss': 0.2600294268881457, 'avg_role_model_mean_pred_loss': 0.00038547293633122223, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023319287935414145, 'n_size': 450, 'n_batch': 57, 'duration': 45.5991792678833, 'duration_batch': 0.7999856011909351, 'duration_size': 0.10133150948418511, 'avg_pred_std': 0.07801991980522871}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0002562747773643221, 'avg_role_model_std_loss': 0.028784445838175392, 'avg_role_model_mean_pred_loss': 2.593463381772865e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0002562747773643221, 'n_size': 900, 'n_batch': 113, 'duration': 111.06798338890076, 'duration_batch': 0.9829025078663783, 'duration_size': 0.12340887043211195, 'avg_pred_std': 0.09953929795430297}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001840519470117417, 'avg_role_model_std_loss': 0.3625226045031678, 'avg_role_model_mean_pred_loss': 0.00015924821697381686, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001840519470117417, 'n_size': 450, 'n_batch': 57, 'duration': 45.293455839157104, 'duration_batch': 0.7946220322659141, 'duration_size': 0.10065212408701579, 'avg_pred_std': 0.07107936053969816}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0001757131654206508, 'avg_role_model_std_loss': 0.018957749903747066, 'avg_role_model_mean_pred_loss': 4.845866322408336e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0001757131654206508, 'n_size': 900, 'n_batch': 113, 'duration': 110.83889031410217, 'duration_batch': 0.9808751355230281, 'duration_size': 0.12315432257122463, 'avg_pred_std': 0.09919166324281059}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019331971814648974, 'avg_role_model_std_loss': 0.6399211409059746, 'avg_role_model_mean_pred_loss': 0.00020584285959550467, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019331971814648974, 'n_size': 450, 'n_batch': 57, 'duration': 45.47143840789795, 'duration_batch': 0.7977445334718939, 'duration_size': 0.10104764090643989, 'avg_pred_std': 0.07058916541363783}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00021455827955732174, 'avg_role_model_std_loss': 0.031199176489886352, 'avg_role_model_mean_pred_loss': 2.883131269934558e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00021455827955732174, 'n_size': 900, 'n_batch': 113, 'duration': 110.77680969238281, 'duration_batch': 0.9803257494901134, 'duration_size': 0.12308534410264757, 'avg_pred_std': 0.10197576969466378}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0018430328381752285, 'avg_role_model_std_loss': 0.4276724044195193, 'avg_role_model_mean_pred_loss': 0.00019247611021483608, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018430328381752285, 'n_size': 450, 'n_batch': 57, 'duration': 45.94011998176575, 'duration_batch': 0.8059670172239605, 'duration_size': 0.10208915551503499, 'avg_pred_std': 0.07040622951707949}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0004361093971238006, 'avg_role_model_std_loss': 0.05909316669737025, 'avg_role_model_mean_pred_loss': 2.2801978405284285e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0004361093971238006, 'n_size': 900, 'n_batch': 113, 'duration': 112.09613871574402, 'duration_batch': 0.992001227572956, 'duration_size': 0.12455126523971558, 'avg_pred_std': 0.1005768165882446}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001964426144291388, 'avg_role_model_std_loss': 0.25929170938355406, 'avg_role_model_mean_pred_loss': 0.00015272240531687408, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001964426144291388, 'n_size': 450, 'n_batch': 57, 'duration': 45.62297987937927, 'duration_batch': 0.8004031557785837, 'duration_size': 0.10138439973195394, 'avg_pred_std': 0.06955495999570478}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0001958047746868235, 'avg_role_model_std_loss': 0.016338927822493122, 'avg_role_model_mean_pred_loss': 4.4443133083278144e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0001958047746868235, 'n_size': 900, 'n_batch': 113, 'duration': 111.31650757789612, 'duration_batch': 0.9851018369725321, 'duration_size': 0.12368500841988457, 'avg_pred_std': 0.09745020749030915}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002082800301011755, 'avg_role_model_std_loss': 0.7195598722440445, 'avg_role_model_mean_pred_loss': 0.0001506238370249388, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002082800301011755, 'n_size': 450, 'n_batch': 57, 'duration': 45.92464828491211, 'duration_batch': 0.8056955839458265, 'duration_size': 0.10205477396647135, 'avg_pred_std': 0.0661866773580346}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0003942243970474616, 'avg_role_model_std_loss': 0.05046572757249398, 'avg_role_model_mean_pred_loss': 1.0653629163880412e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0003942243970474616, 'n_size': 900, 'n_batch': 113, 'duration': 112.87545800209045, 'duration_batch': 0.9988978584255792, 'duration_size': 0.1254171755578783, 'avg_pred_std': 0.09861106915086244}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0017001370618042225, 'avg_role_model_std_loss': 0.6304228919395235, 'avg_role_model_mean_pred_loss': 0.0001222305167977231, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017001370618042225, 'n_size': 450, 'n_batch': 57, 'duration': 48.029250144958496, 'duration_batch': 0.8426184235957631, 'duration_size': 0.10673166698879666, 'avg_pred_std': 0.07135208257424988}\n", + "Stopped False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 1050, 'n_batch': 132, 'role_model_metrics': {'avg_loss': 0.0005013270681610308, 'avg_g_mag_loss': 0.0048045497529870945, 'avg_g_cos_loss': 0.00603572992904533, 'pred_duration': 2.3833107948303223, 'grad_duration': 6.518470764160156, 'total_duration': 8.901781558990479, 'pred_std': 0.14861759543418884, 'std_loss': 2.2368631107383408e-06, 'mean_pred_loss': 4.282439022063045e-06, 'pred_rmse': 0.0223903339356184, 'pred_mae': 0.01324256882071495, 'pred_mape': 0.5489070415496826, 'grad_rmse': 0.013413011096417904, 'grad_mae': 0.004663583356887102, 'grad_mape': 0.3513322174549103}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0005013270681610308, 'avg_g_mag_loss': 0.0048045497529870945, 'avg_g_cos_loss': 0.00603572992904533, 'avg_pred_duration': 2.3833107948303223, 'avg_grad_duration': 6.518470764160156, 'avg_total_duration': 8.901781558990479, 'avg_pred_std': 0.14861759543418884, 'avg_std_loss': 2.2368631107383408e-06, 'avg_mean_pred_loss': 4.282439022063045e-06}, 'min_metrics': {'avg_loss': 0.0005013270681610308, 'avg_g_mag_loss': 0.0048045497529870945, 'avg_g_cos_loss': 0.00603572992904533, 'pred_duration': 2.3833107948303223, 'grad_duration': 6.518470764160156, 'total_duration': 8.901781558990479, 'pred_std': 0.14861759543418884, 'std_loss': 2.2368631107383408e-06, 'mean_pred_loss': 4.282439022063045e-06, 'pred_rmse': 0.0223903339356184, 'pred_mae': 0.01324256882071495, 'pred_mape': 0.5489070415496826, 'grad_rmse': 0.013413011096417904, 'grad_mae': 0.004663583356887102, 'grad_mape': 0.3513322174549103}, 'model_metrics': {'tvae': {'avg_loss': 0.0005013270681610308, 'avg_g_mag_loss': 0.0048045497529870945, 'avg_g_cos_loss': 0.00603572992904533, 'pred_duration': 2.3833107948303223, 'grad_duration': 6.518470764160156, 'total_duration': 8.901781558990479, 'pred_std': 0.14861759543418884, 'std_loss': 2.2368631107383408e-06, 'mean_pred_loss': 4.282439022063045e-06, 'pred_rmse': 0.0223903339356184, 'pred_mae': 0.01324256882071495, 'pred_mape': 0.5489070415496826, 'grad_rmse': 0.013413011096417904, 'grad_mae': 0.004663583356887102, 'grad_mape': 0.3513322174549103}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:26:30.463309Z", + "iopub.status.busy": "2024-03-22T20:26:30.462344Z", + "iopub.status.idle": "2024-03-22T20:26:30.467584Z", + "shell.execute_reply": "2024-03-22T20:26:30.466548Z" + }, + "papermill": { + "duration": 0.031323, + "end_time": "2024-03-22T20:26:30.469923", + "exception": false, + "start_time": "2024-03-22T20:26:30.438600", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:26:30.511952Z", + "iopub.status.busy": "2024-03-22T20:26:30.511273Z", + "iopub.status.idle": "2024-03-22T20:26:30.604402Z", + "shell.execute_reply": "2024-03-22T20:26:30.603508Z" + }, + "papermill": { + "duration": 0.117577, + "end_time": "2024-03-22T20:26:30.607604", + "exception": false, + "start_time": "2024-03-22T20:26:30.490027", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:26:30.647409Z", + "iopub.status.busy": "2024-03-22T20:26:30.647051Z", + "iopub.status.idle": "2024-03-22T20:26:30.937486Z", + "shell.execute_reply": "2024-03-22T20:26:30.936493Z" + }, + "papermill": { + "duration": 0.313113, + "end_time": "2024-03-22T20:26:30.939684", + "exception": false, + "start_time": "2024-03-22T20:26:30.626571", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAwj0lEQVR4nO3de3wTdb7/8VeSJumFXqClN6wtQqHIpXWB1uKFVXosiJd6WKksC8hB0H2Aq1YUcYHq7tnt4uUsKvxk0VX0HCvIrnB8IAeFAutqK8hNilwWWKRdIS0FSe9Jmszvj7RpU9LSlNKkzef5eMwj6cw3k++kzbvf+c7Md1SKoigIIYQXUHu6AkII0UQCSQjhNSSQhBBeQwJJCOE1JJCEEF5DAkkI4TUkkIQQXsPP0xXoCjabjbNnzxIcHIxKpfJ0dYQQrSiKQlVVFbGxsajVbbeDekUgnT17lri4OE9XQwhxBaWlpVx33XVtLu8VgRQcHAzYNzYkJMTDtRFCtFZZWUlcXJzju9qWXhFITbtpISEhEkhCeLErdalIp7YQwmtIIAkhvIYEkhDCa/SKPiRxdaxWKxaLxdPVED2YVqtFo9Fc9XokkHyYoigYDAYuXbrk6aqIXiAsLIzo6OirOhdQAsmHNYVRZGQkgYGBclKp6BRFUaitraW8vByAmJiYTq/LpwKpwWrjQOkljLUW7kyKRK323S+g1Wp1hFF4eLinqyN6uICAAADKy8uJjIzs9O6bT3VqWxWFB1cX8cj7e6kyNXi6Oh7V1GcUGBjo4ZqI3qLpb+lq+iN9KpD0fhr8tfZNrqyTTly48olqQnRUV/wt+VQgAYQF6AC4VCuBJIS38blACg3QAmCUFpIQXkcCSQg3qVQqNm3a5OlqdKkXXniBlJQUT1fD9wIppDGQLtWZPVwTITpv7dq1hIWFddn6Fi5cSEFBQZetr7N86rA/QFigtJCE7zCbzeh0uiuW69OnD3369OmGGrXP51pIssvWNkVRqDU3dPvk7s2Tt27dyq233kpYWBjh4eHcc889nDp1CoBx48axaNEip/Lnz59Hq9XyxRdfAHDu3DkmT55MQEAAAwcOJD8/n4SEBFasWNGpz624uJg777yTgIAAwsPDmTdvHtXV1Y7lu3btIjU1laCgIMLCwrjllls4c+YMAN9++y133HEHwcHBhISEMHr0aPbu3dvu++3atYvZs2djNBpRqVSoVCpeeOEFABISEvjtb3/LzJkzCQkJYd68eQAsWrSIIUOGEBgYyA033MDSpUudDs+33mV7+OGHycrK4pVXXiEmJobw8HDmz59/zS8x8rkWUlMgyWH/y9VZrNy47LNuf98jv8kkUNfxP8WamhpycnIYNWoU1dXVLFu2jAceeICDBw8yffp0XnrpJf7whz84DkOvX7+e2NhYbrvtNgBmzpxJRUUFu3btQqvVkpOT4zjL2F01NTVkZmaSnp7ON998Q3l5OY888ggLFixg7dq1NDQ0kJWVxdy5c/nwww8xm83s2bPHUbfp06dz00038eabb6LRaDh48CBarbbd9xw3bhwrVqxg2bJlHD9+HMCpdfPKK6+wbNkycnNzHfOCg4NZu3YtsbGxFBcXM3fuXIKDg3n22WfbfJ+dO3cSExPDzp07OXnyJNnZ2aSkpDB37txOfVYd4bOBJIf9e64pU6Y4/fzOO+/Qv39/jhw5wtSpU3nyySf58ssvHQGUn5/PtGnTUKlUHDt2jO3bt/PNN98wZswYAN5++20SExM7VZf8/Hzq6+t5//33CQoKAmDlypXce++9LF++HK1Wi9Fo5J577mHQoEEADBs2zPH6kpISnnnmGZKSkgA6VA+dTkdoaCgqlYro6OjLlt955508/fTTTvOWLFnieJ6QkMDChQtZt25du4HUt29fVq5ciUajISkpicmTJ1NQUCCB1JWkD6ltAVoNR36T6ZH3dceJEydYtmwZu3fvpqKiApvNBti/3CNGjOCuu+7igw8+4LbbbuP06dMUFRXxpz/9CYDjx4/j5+fHT37yE8f6Bg8eTN++fTtV96NHj5KcnOwII4BbbrkFm83G8ePHuf3223n44YfJzMzk3/7t38jIyGDq1KmO671ycnJ45JFH+O///m8yMjJ48MEHHcHVWU1B29L69et5/fXXOXXqFNXV1TQ0NFxxdNXhw4c7XQISExNDcXHxVdXtSnyuDylE+pDapFKpCNT5dfvk7hm+9957LxcvXuStt95i9+7d7N69G7B34IJ9N+gvf/kLFouF/Px8Ro4cyciRI7v88+qod999l6KiIsaNG8f69esZMmQIX3/9NWDvu/nuu++YPHkyO3bs4MYbb2Tjxo1X9X4twxGgqKiI6dOnc/fdd7N582YOHDjAr3/9a8fn1ZbWu44qlcoR/teKzwWSdGr3bBcuXOD48eMsWbKECRMmMGzYMH788UenMvfffz/19fVs3bqV/Px8pk+f7lg2dOhQGhoaOHDggGPeyZMnL1tHRw0bNoxvv/2Wmpoax7yvvvoKtVrN0KFDHfNuuukmFi9eTGFhISNGjCA/P9+xbMiQITz11FN8/vnn/Pu//zvvvvvuFd9Xp9NhtVo7VMfCwkLi4+P59a9/zZgxY0hMTHR0qnsb3w0k6UPqkfr27Ut4eDhr1qzh5MmT7Nixg5ycHKcyQUFBZGVlsXTpUo4ePcq0adMcy5KSksjIyGDevHns2bOHAwcOMG/ePAICAjp1Ldb06dPx9/dn1qxZHD58mJ07d/L4448zY8YMoqKiOH36NIsXL6aoqIgzZ87w+eefc+LECYYNG0ZdXR0LFixg165dnDlzhq+++opvvvnGqY+pLQkJCVRXV1NQUEBFRQW1tbVtlk1MTKSkpIR169Zx6tQpXn/99atuhV0rPhdIYY2BVGVqwGpz73Cz8Dy1Ws26devYt28fI0aM4KmnnuLll1++rNz06dP59ttvue2227j++uudlr3//vtERUVx++2388ADDziOOPn7+7tdn8DAQD777DMuXrzI2LFj+dnPfsaECRNYuXKlY/mxY8eYMmUKQ4YMYd68ecyfP59HH30UjUbDhQsXmDlzJkOGDGHq1KlMmjSJF1988YrvO27cOB577DGys7Pp378/L730Uptl77vvPp566ikWLFhASkoKhYWFLF261O1t7RZKJ6xcuVKJj49X9Hq9kpqaquzevbvd8h999JEydOhQRa/XKyNGjFA+/fTTNss++uijCqD88Y9/7HB9jEajAihGo/GKZc0NViV+0WYlftFm5WK1qcPv0dvU1dUpR44cUerq6jxdFY8rLS1VAGX79u2erkqP1t7fVEe/o263kNavX09OTg65ubns37+f5ORkMjMz2zyPo7CwkGnTpjFnzhwOHDhAVlYWWVlZHD58+LKyGzdu5OuvvyY2NtbdanWYVqMmSGc/ciD9SL5px44dfPLJJ5w+fZrCwkIeeughEhISuP322z1dNZ/ndiD913/9F3PnzmX27NnceOONrF69msDAQN555x2X5V977TUmTpzIM888w7Bhw/jtb3/LT37yE0eTtskPP/zA448/zgcffHDFE8OuluNcJAkkn2SxWHj++ecZPnw4DzzwAP3793ecJPnBBx84LqNoPQ0fPrzb6jhp0qQ26/H73/++2+rR3dw6D8lsNrNv3z4WL17smKdWq8nIyKCoqMjla4qKii7rdMzMzHS6WtpmszFjxgyeeeaZDv3STSYTJpPJ8XNlZaU7m0FooI6zxnppIfmozMxMMjNdn2913333kZaW5nLZtf5H2dLbb79NXV2dy2X9+vXrtnp0N7cCqaKiAqvVSlRUlNP8qKgojh075vI1BoPBZXmDweD4efny5fj5+fGrX/2qQ/XIy8vrUMdfW0ID7JstgSRaCw4OvuL957vDgAEDPF0Fj/D4UbZ9+/bx2muvsXbt2g4fdl28eDFGo9ExlZaWuvWeci6SEN7JrUCKiIhAo9FQVlbmNL+srMzlNTUA0dHR7Zb/+9//Tnl5Oddffz1+fn74+flx5swZnn76aRISElyuU6/XExIS4jS5o2kYW2OtjIkkhDdxK5B0Oh2jR492GsjJZrNRUFBAenq6y9ekp6dfNvDTtm3bHOVnzJjBoUOHOHjwoGOKjY3lmWee4bPPrs2V56FyPZsQXsnti2tzcnKYNWsWY8aMITU1lRUrVlBTU8Ps2bMB+9AOAwYMIC8vD4AnnniC8ePH8+qrrzJ58mTWrVvH3r17WbNmDQDh4eGX3RdMq9USHR3tdOp9V5JdNiG8k9uBlJ2dzfnz51m2bBkGg4GUlBS2bt3q6LguKSlBrW5ueI0bN478/HyWLFnC888/T2JiIps2bWLEiBFdtxVuCpEhSITwSp0afmTBggUsWLDA5bJdu3ZdNu/BBx/kwQcf7PD6v//++85Uq8PCpIUkroJKpWLjxo1kZWV5uiq9jsePsnmC7LKJnq6rB/kHe2NCpVJx6dKlLl2vO3w6kGQYWyG8i08Hklw60oqigLmm+ycZ5L/LBvk3mUwsXLiQAQMGEBQURFpamlM3ypkzZ7j33nvp27cvQUFBDB8+nC1btvD9999zxx13APYhXlQqFQ8//HCnPo+r4XND2ELzMLa1ZisWqw2txidz+XKWWvj9tbuwuU3PnwVd0JXLNZJB/tse5H/BggUcOXKEdevWERsby8aNG5k4cSLFxcUkJiYyf/58zGYzX3zxBUFBQRw5coQ+ffoQFxfHX//6V6ZMmcLx48cJCQkhICCgU5/J1fDJQAr2b/6FG+ssRPTRe7A2wl0yyL/rQf5LSkp49913KSkpcYyYsXDhQrZu3cq7777L73//e0pKSpgyZYpjSN8bbrjB8fqma+QiIyO7vH+qo3wykDRqFcH+flTVN0ggtaQNtLdWPPG+bpBB/l0rLi7GarUyZMgQp/kmk8lxrt+vfvUrfvnLX/L555+TkZHBlClTGDVqVKfe71rw2X0VuR2SCyqVfdepuycZ5L9LBvmvrq5Go9Gwb98+pysfjh49ymuvvQbAI488wj//+U9mzJhBcXExY8aM4Y033uiybb1aPhtITf1IcqStZ5FB/u1cDfJ/0003YbVaKS8vZ/DgwU5Ty127uLg4HnvsMT7++GOefvpp3nrrLcc6gQ7fPOBa8NlAknOReiYZ5N/O1SD/Q4YMYfr06cycOZOPP/6Y06dPs2fPHvLy8vj0008BePLJJ/nss884ffo0+/fvZ+fOnY73i4+PR6VSsXnzZs6fP+90pLDbXKvxdbuTO2NqN/nl/+xV4hdtVtZ+dfraVcyL9eQxtbdt26YMGzZM0ev1yqhRo5Rdu3YpgLJx40ZHmS1btiiAcvvtt1/2+rNnzyqTJk1S9Hq9Eh8fr+Tn5yuRkZHK6tWrO/T+rd/r0KFDyh133KH4+/sr/fr1U+bOnatUVVUpiqIoBoNBycrKUmJiYhSdTqfEx8cry5YtU6xWq2IymZSHHnpIiYuLU3Q6nRIbG6ssWLCgw7+Txx57TAkPD1cAJTc3V1EURTGbzcqyZcuUhIQERavVKjExMcoDDzygHDp0SFEURVmwYIEyaNAgRa/XK/3791dmzJihVFRUONb5m9/8RomOjlZUKpUya9asDtWjSVeMqa1SFDdPAvFClZWVhIaGYjQaOzwUyeKPD/HhnlKeyhjCExmdO8LSk9XX13P69GkGDhzYqbtt9Cb/+te/iIuLY/v27UyYMMHT1emx2vub6uh31CePsgGENo2JJLtsPmfHjh1UV1czcuRIzp07x7PPPiuD/HsJ6UOSQPI5Msi/9/LhFlJTIMmokb5GBvn3XhJI0kISLcgg/57ls7tsYTKMLQC94JiG8BJd8bfks4Hk6y2kpt2P2tpaD9dE9BZNf0tXs2vr87tsvnrpiEajISwszHGVe2BgYKdODBRCURRqa2spLy8nLCwMjUbT6XX5biA17rKZGmzUW6z4azv/IfZUTZcTdHboDSFaCgsLa/N2aB3ls4HUR+eHWgU2xX49my8GkkqlIiYmhsjISCwW32wpiq6h1WqvqmXUxGcDSa1WERKg5VKtBWOdhcgQ3z1bWaPRdMkfkxBXy2c7tUGGshXC2/h0IDluh+SjHdtCeBufDqQQHz/0L4S38elA8vVzkYTwNhJISB+SEN7CpwNJhrEVwrv4dCDJLpsQ3kUCCQkkIbyFBBJwqVbGRBLCG/h4IMkwtkJ4Ex8PpKZdtgYP10QIAb4eSIHNw9jKQGVCeJ5vB1JjC8liVaizeO5unUIIO58OpCCdBj+1fVAy6UcSwvN8OpBUKpUc+hfCi/h0IIEMZSuEN+lUIK1atYqEhAT8/f1JS0tjz5497ZbfsGEDSUlJ+Pv7M3LkSLZs2eK0/IUXXiApKYmgoCD69u1LRkYGu3fv7kzV3CZX/AvhPdwOpPXr15OTk0Nubi779+8nOTmZzMzMNsdlLiwsZNq0acyZM4cDBw6QlZVFVlYWhw8fdpQZMmQIK1eupLi4mC+//JKEhATuuusuzp8/3/kt6yC5HZIQXkRxU2pqqjJ//nzHz1arVYmNjVXy8vJclp86daoyefJkp3lpaWnKo48+2uZ7GI1GBVC2b9/eoTo1lTcajR0q39KvPtyvxC/arLz1xSm3XyuE6JiOfkfdaiGZzWb27dtHRkaGY55arSYjI4OioiKXrykqKnIqD/ZbGbdV3mw2s2bNGkJDQ0lOTnZZxmQyUVlZ6TR1lvQhCeE93AqkiooKrFYrUVFRTvOjoqIwGAwuX2MwGDpUfvPmzfTp0wd/f3/++Mc/sm3bNiIiIlyuMy8vj9DQUMcUFxfnzmY4CZM+JCG8htccZbvjjjs4ePAghYWFTJw4kalTp7bZL7V48WKMRqNjKi0t7fT7Sqe2EN7DrUCKiIhAo9FQVlbmNL+srKzNG8RFR0d3qHxQUBCDBw/m5ptv5s9//jN+fn78+c9/drlOvV5PSEiI09RZch6SEN7DrUDS6XSMHj2agoICxzybzUZBQQHp6ekuX5Oenu5UHmDbtm1tlm+5XpPJ5E71OkWGsRXCe7h9o8icnBxmzZrFmDFjSE1NZcWKFdTU1DB79mwAZs6cyYABA8jLywPgiSeeYPz48bz66qtMnjyZdevWsXfvXtasWQNATU0Nv/vd77jvvvuIiYmhoqKCVatW8cMPP/Dggw924aa6FhZoH4JEhrEVwvPcDqTs7GzOnz/PsmXLMBgMpKSksHXrVkfHdUlJCWp1c8Nr3Lhx5Ofns2TJEp5//nkSExPZtGkTI0aMAOx3TT127BjvvfceFRUVhIeHM3bsWP7+978zfPjwLtrMtskumxDeQ6UoPX/cjcrKSkJDQzEajW73JxmM9dycV4BGreLk7yahUqmuUS2F8F0d/Y56zVE2T2lqIVltCtUmGahNCE/y+UDy16rR+dk/BtltE8KzfD6QZAgSIbyHzwcStOjYlstHhPAoCSTkSJsQ3kICCbmeTQhvIYGEtJCE8BYSSDRfYCuXjwjhWRJISAtJCG8hgYQMYyuEt5BAormFJBfYCuFZEkjIMLZCeAsJJKQPSQhvIYGE9CEJ4S0kkGg+7F9Zb8Fm6/GjsQjRY0kg0bzLpihQVS9DkAjhKRJIgN5PQ4BWA8humxCeJIHUSDq2hfA8CaRGEkhCeJ4EUqPm2yGZPVwTIXyXBFKjUDn0L4THSSA1kl02ITxPAqmRDGMrhOdJIDWSFpIQnieB1EguHxHC8ySQGkkLSQjPk0BqFCJDkAjhcRJIjaSFJITnSSA1CpNRI4XwOAmkRk0tpCpTAw1Wm4drI4RvkkBq1NSHBFApQ5AI4RESSI20GjVBOhmCRAhPkkBqISxQB0ggCeEpEkgthMiRNiE8SgKphdAAPwAu1coQJEJ4ggRSC3LDSCE8SwKphbAA6UMSwpMkkFqQQdqE8KxOBdKqVatISEjA39+ftLQ09uzZ0275DRs2kJSUhL+/PyNHjmTLli2OZRaLhUWLFjFy5EiCgoKIjY1l5syZnD17tjNVuypyS20hPMvtQFq/fj05OTnk5uayf/9+kpOTyczMpLy83GX5wsJCpk2bxpw5czhw4ABZWVlkZWVx+PBhAGpra9m/fz9Lly5l//79fPzxxxw/fpz77rvv6rasE+QomxAeprgpNTVVmT9/vuNnq9WqxMbGKnl5eS7LT506VZk8ebLTvLS0NOXRRx9t8z327NmjAMqZM2dcLq+vr1eMRqNjKi0tVQDFaDS6uzlOPjn4gxK/aLMydXXhVa1HCOHMaDR26DvqVgvJbDazb98+MjIyHPPUajUZGRkUFRW5fE1RUZFTeYDMzMw2ywMYjUZUKhVhYWEul+fl5REaGuqY4uLi3NmMNskV/0J4lluBVFFRgdVqJSoqyml+VFQUBoPB5WsMBoNb5evr61m0aBHTpk0jJCTEZZnFixdjNBodU2lpqTub0SYJJCE8y8/TFWjJYrEwdepUFEXhzTffbLOcXq9Hr9d3+fvLMLZCeJZbLaSIiAg0Gg1lZWVO88vKyoiOjnb5mujo6A6VbwqjM2fOsG3btjZbR9dSUwup1mzF3CBDkAjR3dwKJJ1Ox+jRoykoKHDMs9lsFBQUkJ6e7vI16enpTuUBtm3b5lS+KYxOnDjB9u3bCQ8Pd6daXSbYv3kIEmklCdH93N5ly8nJYdasWYwZM4bU1FRWrFhBTU0Ns2fPBmDmzJkMGDCAvLw8AJ544gnGjx/Pq6++yuTJk1m3bh179+5lzZo1gD2Mfvazn7F//342b96M1Wp19C/169cPnU7XVdt6RRq1imB/P6rqGzDWWegf3PW7hUKItrkdSNnZ2Zw/f55ly5ZhMBhISUlh69atjo7rkpIS1Ormhte4cePIz89nyZIlPP/88yQmJrJp0yZGjBgBwA8//MAnn3wCQEpKitN77dy5k5/+9Ked3LTOCQvUOgJJCNG9VIqiKJ6uxNWqrKwkNDQUo9F41X1P97zxdw7/UMm7D4/ljqTILqqhEL6to99RuZatFcflI3UyBIkQ3U0CqRXHuUhyPZsQ3U4CqZVQxxAkMtC/EN1NAqkVOVtbCM+RQGpF+pCE8BwJpFZkGFshPEcCqRW5nk0Iz5FAakX6kITwHAmkVmQYWyE8RwKpFWkhCeE5EkitNN15xNRgo95i9XBthPAtEkit9NH5oVbZn0srSYjuJYHUilqtkruPCOEhEkguSD+SEJ4hgeRCmFxgK4RHSCC5EOK4fEQCSYjuJIHkguyyCeEZEkguyOUjQniGBJILcoGtEJ4hgeRC8+UjMgSJEN1JAskF6UMSwjMkkFxoHsZWAkmI7iSB5IK0kITwDAkkFySQhPAMCSQXQlsc9u8F99EUoseQQHKh6dIRi1WhToYgEaLbSCC5EKjT4Nc4BonstgnRfSSQXFCpVDKUrRAeIIHUBunYFqL7SSC1IVSuZxOi20kgtSFUxkQSottJILVBdtmE6H4SSG2QQBKi+0kgtSFMAkmIbieB1AYZxlaI7ieB1AbZZROi+0kgtUECSYju16lAWrVqFQkJCfj7+5OWlsaePXvaLb9hwwaSkpLw9/dn5MiRbNmyxWn5xx9/zF133UV4eDgqlYqDBw92plpdKizQPiaSDGMrRPdxO5DWr19PTk4Oubm57N+/n+TkZDIzMykvL3dZvrCwkGnTpjFnzhwOHDhAVlYWWVlZHD582FGmpqaGW2+9leXLl3d+S7qYDGMrRPdTKW6Or5GWlsbYsWNZuXIlADabjbi4OB5//HGee+65y8pnZ2dTU1PD5s2bHfNuvvlmUlJSWL16tVPZ77//noEDB3LgwAFSUlLarIPJZMJkMjl+rqysJC4uDqPRSEhIiDub0yaDsZ6b8wrQqFWc/N0kVCpVl6xXCF9UWVlJaGjoFb+jbrWQzGYz+/btIyMjo3kFajUZGRkUFRW5fE1RUZFTeYDMzMw2y3dEXl4eoaGhjikuLq7T62pL062QrDaFalNDl69fCHE5twKpoqICq9VKVFSU0/yoqCgMBoPL1xgMBrfKd8TixYsxGo2OqbS0tNPraou/VoPOz/7xSMe2EN3Dz9MV6Ay9Xo9er7/m7xMaoOV8lYlLtRau63vN304In+dWCykiIgKNRkNZWZnT/LKyMqKjo12+Jjo62q3y3kRuGClE93IrkHQ6HaNHj6agoMAxz2azUVBQQHp6usvXpKenO5UH2LZtW5vlvYlcPiJE93J7ly0nJ4dZs2YxZswYUlNTWbFiBTU1NcyePRuAmTNnMmDAAPLy8gB44oknGD9+PK+++iqTJ09m3bp17N27lzVr1jjWefHiRUpKSjh79iwAx48fB+ytK0+2pOTkSCG6l9uBlJ2dzfnz51m2bBkGg4GUlBS2bt3q6LguKSlBrW5ueI0bN478/HyWLFnC888/T2JiIps2bWLEiBGOMp988okj0AAeeughAHJzc3nhhRc6u21XLVSuZxOiW7l9HpI36ug5Du564ZPvWFv4Pb/86SAWTUzqsvUK4WuuyXlIviZMhrEVoltJILVDhrEVontJILVDOrWF6F4SSO2QQBKie0kgtUP6kIToXj3y0pGr8tdHQKMDXVDj1KdxavlzEOj7EGGC/vxIXa3N07UWwif4ViBZG6B4Q4eLxwPf+EO9osW291XUY2Zdu7oJIXwskFDg7lfAXA2majDX2J+ba1pM1Y55irkaW301/ioLypYciBgECbd6eiOE6LV8K5A0Wkid2+HiKmDE0v/jD7zO/RTCR7Ng3i4I6/rxl4QQ0ql9RaEBOhZZ5lIXPhxqK2Ddz8Fc6+lqCdErSSBdQWiAlnr0FN/6/yAwHAyH4JPHoedfcSOE15FAuoKmc5HKNZHw4Hug0sDhv0DhGx6umRC9jwTSFYS2PBdp4G0w8Q/2Bdtz4eR2D9ZMiN5HAukKmlpIZZWNdzlJnQs3/QIUG/zlP+DCKQ/WTojeRQLpCoZGBQOw+m+n+OpkBahUMPm/YMAYqDfCuulgqvJwLYXoHSSQruDhWxLIGBaFucHGnPe+Yfc/L4CfHrL/B/pEw/mjsPExsMnZ3EJcLQmkK9Bq1KyafhM/HdqfeouN/1j7DfvO/AghMfZQ0ujg2Gb44mVPV1WIHk8CqQP0fhpW/2I0tw6OoMZs5eF39vBt6SWIG2vffQPY9Xs4tsWj9RSip5NA6iB/rYa3Zo4hdWA/qkwNzPjzbr47a4SfzICxjWd/fzwPzh/3bEWF6MEkkNwQoNPwzsNj+cn1YVTWN/CLt3dz3FAFE/Mg/lYwV8GH06DukqerKkSPJIHkpj56P9b+RyrJ14XyY62F6W9/zckLJpj6HoTGwcVT9iFObFZPV1WIHkfuOtJJxloLP3/7a747W0lksJ71j6Yz0HwC3pkIDXUwOAMih0FA38apX/PzwMbn2kD7aQRC9HId/Y5KIF2FizVmfv7W1xwzVBET6s9Hj6YT98MW+Oucjq1Ao2sOq+BouPE+GP7vEBB2TestRHeTQOomFdUmsv9UxKnzNVzXN4D1j6Yz4Me9ULob6n609yfVXWx83jjVXgRbG8PiavQw7B5I+TnccAeoNd26PUJcCxJI3aissp7sPxXx/YVa4sMD+ejRdKJC/Nt+gaLYB4NrGVKGQ3AwH8qPNJcLjoHkhyD559B/yLXfECGuEQmkbnb2Uh3Za4oovVjHoP5BrJuXTv9gvXsrURQ4d9AeTMUb7EHV5Lqx9laT7NKJHkgCyQNKL9aS/acizhrr6RuoZeKIaO4eGUP6DeH4adw8oNlggn9stYfTiW2gNB618/OHpKZdup/KLp3oESSQPOT7ihpmvrOHkovNo0r2DdSSObwxnAaFo3U3nKrKoPgjOPCB/dq5Jv5h9lAadCcMngCh1139Btis9t3GM4Vw9oC9073fDRA+CMIHQ8h1oJazRYR7JJA8qMFqY/fpi3xafI7PDhu4UGN2LAsL1JJ5YzSTRkZzy+AI98JJUewhcTDfPkhcy106gIih9mAaNAHix4Eu8MrrtFrg7EE485U9hEq+BpOx7fIaPfQbaA+npqDq1xhWwdE99zQGcy2UfWffZS47DPoQiB4FMaPs2yYt0asigeQlGqw29jSF03cGKqqbwyk0QMtdN0Zx96gYbhkUgc7PjXCyNsDZ/XCyAE4VwA/77GM0NdHoIT7dHk6D7oSo4fawMNfCD3vt4XOmEP71DVhajRGuC4br0+z9VqYq+5hPF0/BxdNtHx0E0AZBSCz4h4A+uHFq+dzFPF0f++gJGq39NAiNzv5crW1+3tUhZ6oCQzGc+9Yexue+hYrjzp+f03YF2j+/6FEQPdIeUpHDQdvOgQtvY7W0uMNObfNzS63znXd0fewBHD4YgiK67LOXQPJCVpvC7tMX2FJ8jq2Hy6ioNjmW9dH7cWNMCMNighkWE0JSTAhDo4IJ0HXwP3Pdj/DPv9nD6eQOqPyX8/I+0fZdunPfXh4qAf3sLaqmKWokaFzckMbaAMZSezhd+CdcONn4/BRcOtP2F/pqtQwnjQ60AW0EXRsBqNjsrZ5z39qnC6cAF3/2QZEQmwJRI+xjXRmK7a9rHdhgH8q4/9DmkOp3A1hNYKm3nxhrqbe/rqEeLHXNj03PG+pB7WfvE9QGdOwRLg8Qc409YJ1u49Xqtl6WWrCaL9+GK9GHNu+qhw9u8XyQ/XN1gwSSl7PaFPacvsiW4nP832GDUzg1UasgISKIYdHOQRUb6o+qvf9cigIV/4BTO+wtqO+/tH9JmgTHQPwt9hZU/C32Xb2r7RdqMNtDqcrQeN+7KjBVNj62nlrNt5rt/8GtFvuXujuEXAcxyc5TSMzl5WxWe4AZDtmnc42PtRe6p55dTe3X4m7Ngc53a9YGQv0l+z+aS6W4DO0mfaKaw+nGLHtXQTskkHoQq03huKGKo+cqOWao5Og5+/OWfU8thQZoSYoOJnVgP346NJKUuDA06nYCylIPJUVQUwHXjYG+Cd7b16Mo9hCwmlsEldneqrNa7EcfLXVthF2l6/m2BvtlPE3BE50MffpfXR2rzjWHk+EQGH9obs04Wjb+4BfQal7jo5+/vV6tW1DtPSoK6Ps4h0h7z7VBLco3/uyn69g2Wurhx9P2ML5wsnFqfF5T7lz2rt/BuAXtrk4CqYdTFIXz1SaOnqvi2LlKjp6zB9Wp89U02Jx/Zf2CdIwf0p87kiIZn9jfcWMCIa6JemNjODUG1NBJ9l3ddkgg9VKmBisny6v57odKvjhxni/+cZ7K+gbHcrUKRsf35Y6kSO5MimRoVHD7u3dCdAMJJB/RYLWx78yP7Dhezs5j5fyjrNppeWyoPz9NiuTOoZGMju+LApgbbFisNsxW+6OlQWl+3jiZGxQabDaC9H6EBmgJC9AS2ji5fZKn8HkSSD6q9GItu46Xs+NYOYWnLmBq6PojX30aQ6rlFBZofwz298NfqyFAp8Hfz/4YoNU0z9OqCdA2zmtc5vaJoqLHuaaBtGrVKl5++WUMBgPJycm88cYbpKamtll+w4YNLF26lO+//57ExESWL1/O3Xff7ViuKAq5ubm89dZbXLp0iVtuuYU333yTxMTEDtVHAsm1OrOVr/95gR3H7AH1wyX7kTY/tQqtRo1Wo0Lnp258rnY812nsy9VqFbXmBi7VWjDWWahqsWvYlUL8/YgI1tO/j775sY+O/sF6IvrYp/7BesL76ND7XfsTFK02BRWgbu9AgRdpsNq4WGvmQnXjVGOiotqMsc5CaICW/o2faf9g+xTi79fp3Xhzg41LtWYu1pq5WGPmxxoLN8aGMDAiqN3XXbNAWr9+PTNnzmT16tWkpaWxYsUKNmzYwPHjx4mMjLysfGFhIbfffjt5eXncc8895Ofns3z5cvbv38+IESMAWL58OXl5ebz33nsMHDiQpUuXUlxczJEjR/D3v/LJZxJIV6Yo9t0yrVrd6S+a1aZQWWcPp0tNj7Xm5nm1FqpNDdRbrNRZrNRZbNSbm55bqTNbqbfYp1qLlc60zUP8/QgJ0KL3U6P306DXqtH7qfHXaprn+akb59uf+6lV1Fms1Jrtdag129+/ztxw2fw6sxWz1d6q1GpUjnXo/NQtHtue1/J9W9av9XKdnxpFUbDamqeGFo82x882x/xas5UL1SYqasxcqDZxodpMRbWJS3UWtz5LnZ/aKfybgqp/sB4V8GONmR9rLfzYFDq1jVON/ffbWu69NzL7loHtvuc1C6S0tDTGjh3LypUrAbDZbMTFxfH444/z3HPPXVY+OzubmpoaNm/e7Jh38803k5KSwurVq1EUhdjYWJ5++mkWLlwIgNFoJCoqirVr1/LQQw9dsU4SSD1PU0DWmOxfsvPV9v/q56tMVFSbqKhqmmeiosr+xWt9dFE0U6mgX6CO8D46woPsrcmQAC3GOovjszxfZeqSVq5aBX0DdfQN0tE3UMsvbo7n/pQB7b6mo99RF6fjts1sNrNv3z4WL17cXDm1moyMDIqKily+pqioiJycHKd5mZmZbNq0CYDTp09jMBjIyMhwLA8NDSUtLY2ioiKXgWQymTCZmk+gq6ysdGczhBdQqZpaHxr6BelIjGr/zF+bTbF/uapNVJsaMDXY7JPF2vy8wYrJ0uJ5gw2TxUaDzWbvt9JpCNRpCND5EaBteq4h0MUyRVEwNdgwN67b3LhOc0Or9Tcuq7fYW1au3r9l2Zb1VatAo1ahUavwczyqHT+rnearCNBpHGET3kdPRJD9MbyPjr6BuvbPRWtUb7FyvinsWwRV0wT200iawqZvoI5+QTrCGh/7BeoI9ve7ZruzbgVSRUUFVquVqKgop/lRUVEcO3bM5WsMBoPL8gaDwbG8aV5bZVrLy8vjxRdfdKfqoodTq1X2L0lQB0/sEy75azXE9Qskrl8HLrz2gB55eGPx4sUYjUbHVFpa6ukqCSG6gFuBFBERgUajoayszGl+WVkZ0dHRLl8THR3dbvmmR3fWqdfrCQkJcZqEED2fW4Gk0+kYPXo0BQUFjnk2m42CggLS09NdviY9Pd2pPMC2bdsc5QcOHEh0dLRTmcrKSnbv3t3mOoUQvZTipnXr1il6vV5Zu3atcuTIEWXevHlKWFiYYjAYFEVRlBkzZijPPfeco/xXX32l+Pn5Ka+88opy9OhRJTc3V9FqtUpxcbGjzB/+8AclLCxM+d///V/l0KFDyv33368MHDhQqaur61CdjEajAihGo9HdzRFCdIOOfkfdDiRFUZQ33nhDuf766xWdTqekpqYqX3/9tWPZ+PHjlVmzZjmV/+ijj5QhQ4YoOp1OGT58uPLpp586LbfZbMrSpUuVqKgoRa/XKxMmTFCOHz/e4fpIIAnh3Tr6HZVLR4QQ11xHv6M98iibEKJ3cus8JG/V1MiTEySF8E5N380r7ZD1ikCqqqoCIC4uzsM1EUK0p6qqitDQ0DaX94o+JJvNxtmzZwkOvvJgZJWVlcTFxVFaWupT/U2y3b613eBd264oClVVVcTGxqJuZ/z2XtFCUqvVXHedezdJ9NUTKmW7fY+3bHt7LaMm0qkthPAaEkhCCK/hc4Gk1+vJzc1Fr9d7uirdSrbbt7Ybeua294pObSFE7+BzLSQhhPeSQBJCeA0JJCGE15BAEkJ4DQkkIYTX8LlAWrVqFQkJCfj7+5OWlsaePXs8XaVr6oUXXkClUjlNSUlJnq5Wl/viiy+49957iY2NRaVSOe5q00RRFJYtW0ZMTAwBAQFkZGRw4sQJz1S2C11pux9++OHLfv8TJ070TGU7wKcCaf369eTk5JCbm8v+/ftJTk4mMzOT8vJyT1ftmho+fDjnzp1zTF9++aWnq9TlampqSE5OZtWqVS6Xv/TSS7z++uusXr2a3bt3ExQURGZmJvX19d1c0651pe0GmDhxotPv/8MPP+zGGrrpmg4T52VSU1OV+fPnO362Wq1KbGyskpeX58FaXVu5ublKcnKyp6vRrQBl48aNjp9tNpsSHR2tvPzyy455ly5dUvR6vfLhhx96oIbXRuvtVhRFmTVrlnL//fd7pD6d4TMtpKabXLa8IeWVbnLZW5w4cYLY2FhuuOEGpk+fTklJiaer1K2udDPS3m7Xrl1ERkYydOhQfvnLX3LhwgVPV6lNPhNI7d3ksq0bUvYGaWlprF27lq1bt/Lmm29y+vRpbrvtNscYUr6gMzcj7S0mTpzI+++/T0FBAcuXL+dvf/sbkyZNwmq1erpqLvWK4UdE2yZNmuR4PmrUKNLS0oiPj+ejjz5izpw5HqyZ6A4tb0U/cuRIRo0axaBBg9i1axcTJkzwYM1c85kWUmductkbhYWFMWTIEE6ePOnpqnSbztyMtLe64YYbiIiI8Nrfv88EUmductkbVVdXc+rUKWJiYjxdlW4jNyNt9q9//YsLFy547e/fp3bZcnJymDVrFmPGjCE1NZUVK1ZQU1PD7NmzPV21a2bhwoXce++9xMfHc/bsWXJzc9FoNEybNs3TVetS1dXVTv/1T58+zcGDB+nXrx/XX389Tz75JP/5n/9JYmIiAwcOZOnSpcTGxpKVleW5SneB9ra7X79+vPjii0yZMoXo6GhOnTrFs88+y+DBg8nMzPRgrdvh6cN83a29m1z2RtnZ2UpMTIyi0+mUAQMGKNnZ2crJkyc9Xa0ut3PnTgW4bGq6aenV3ozUW7W33bW1tcpdd92l9O/fX9FqtUp8fLwyd+5cx12mvZGMhySE8Bo+04ckhPB+EkhCCK8hgSSE8BoSSEIIryGBJITwGhJIQgivIYEkhPAaEkhCCK8hgSSE8BoSSEIIryGBJITwGv8fS9t4Px8iaRkAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:26:30.980393Z", + "iopub.status.busy": "2024-03-22T20:26:30.980008Z", + "iopub.status.idle": "2024-03-22T20:28:44.369467Z", + "shell.execute_reply": "2024-03-22T20:28:44.368409Z" + }, + "papermill": { + "duration": 133.413762, + "end_time": "2024-03-22T20:28:44.372215", + "exception": false, + "start_time": "2024-03-22T20:26:30.958453", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:28:44.411780Z", + "iopub.status.busy": "2024-03-22T20:28:44.411403Z", + "iopub.status.idle": "2024-03-22T20:28:44.432887Z", + "shell.execute_reply": "2024-03-22T20:28:44.431938Z" + }, + "papermill": { + "duration": 0.043747, + "end_time": "2024-03-22T20:28:44.435025", + "exception": false, + "start_time": "2024-03-22T20:28:44.391278", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.0065320.0061290.0005016.5985510.0046640.3513320.0134130.0000042.3958840.0132430.5489070.022390.1486180.0000028.994436
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.006532 0.006129 0.000501 6.598551 0.004664 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 0.351332 0.013413 0.000004 2.395884 0.013243 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.548907 0.02239 0.148618 0.000002 8.994436 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:28:44.473539Z", + "iopub.status.busy": "2024-03-22T20:28:44.472687Z", + "iopub.status.idle": "2024-03-22T20:28:44.862917Z", + "shell.execute_reply": "2024-03-22T20:28:44.861927Z" + }, + "papermill": { + "duration": 0.41144, + "end_time": "2024-03-22T20:28:44.864947", + "exception": false, + "start_time": "2024-03-22T20:28:44.453507", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:28:44.905022Z", + "iopub.status.busy": "2024-03-22T20:28:44.904639Z", + "iopub.status.idle": "2024-03-22T20:30:57.227328Z", + "shell.execute_reply": "2024-03-22T20:30:57.226307Z" + }, + "papermill": { + "duration": 132.3464, + "end_time": "2024-03-22T20:30:57.229978", + "exception": false, + "start_time": "2024-03-22T20:28:44.883578", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/tvae/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/tvae/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:30:57.270094Z", + "iopub.status.busy": "2024-03-22T20:30:57.269519Z", + "iopub.status.idle": "2024-03-22T20:30:57.296520Z", + "shell.execute_reply": "2024-03-22T20:30:57.295842Z" + }, + "papermill": { + "duration": 0.049669, + "end_time": "2024-03-22T20:30:57.298595", + "exception": false, + "start_time": "2024-03-22T20:30:57.248926", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:30:57.334434Z", + "iopub.status.busy": "2024-03-22T20:30:57.333849Z", + "iopub.status.idle": "2024-03-22T20:30:57.339056Z", + "shell.execute_reply": "2024-03-22T20:30:57.338231Z" + }, + "papermill": { + "duration": 0.025463, + "end_time": "2024-03-22T20:30:57.341173", + "exception": false, + "start_time": "2024-03-22T20:30:57.315710", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.04507807671516535}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:30:57.378626Z", + "iopub.status.busy": "2024-03-22T20:30:57.378377Z", + "iopub.status.idle": "2024-03-22T20:30:57.773260Z", + "shell.execute_reply": "2024-03-22T20:30:57.772382Z" + }, + "papermill": { + "duration": 0.416013, + "end_time": "2024-03-22T20:30:57.775310", + "exception": false, + "start_time": "2024-03-22T20:30:57.359297", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA850lEQVR4nO3deXhU9b348fc5Z9ZskwSyQgJhd0NQgeIGKq2KVamtxaUV3G+LVUvpVfr8imJrsa1arfXS3t5K5KmK1aL26sWlVsRqQUURxMpmgMgWSMg6+5zv749JhoRksk5yJsnn9TzzQM5858znJJlPvue7akophRBCWEi3OgAhhJBEJISwnCQiIYTlJBEJISwniUgIYTlJREIIy0kiEkJYThKREMJykoiEEJaTRCSEsJwkIpFQ7733Hvfeey/V1dVWhyL6EUlEIqHee+89li5dKolIdIkkIiGE5SQRiYS59957+fGPfwxASUkJmqahaRppaWmcd955rcqbpsmwYcP41re+FTv24IMPcuaZZzJkyBDcbjenn346zz//fJvv9+c//5nTTz8dt9tNdnY2V111FeXl5b1zcaJXabIMiEiUzZs388ADD/DMM8/wm9/8hqFDhwKwa9cu7rvvPvbt20d+fn6s/Lp165gxYwbPPfdcLBkVFRVx2WWXceKJJxIMBlm1ahXvv/8+L7/8Mpdccknstffffz8//elP+fa3v82MGTM4fPgwjz32GGlpaXz88cdkZmb26bWLHlJCJNCvf/1rBaiysrLYsW3btilAPfbYYy3Kfv/731dpaWnK6/XGjjX/v1JKBYNBdfLJJ6vzzz8/dmz37t3KMAx1//33tyi7ZcsWZbPZWh0XyU9uzUSvGzduHJMmTeLZZ5+NHYtEIjz//PNceumluN3u2PHm/z969Cg1NTWcc845fPTRR7Hjq1evxjRNvv3tb3PkyJHYIz8/n7Fjx/LWW2/1zYWJhLFZHYAYHObOnctPfvIT9u3bx7Bhw1i7di0VFRXMnTu3RbmXX36Zn//852zatIlAIBA7rmla7P87duxAKcXYsWPbfC+73d47FyF6jSQi0Sfmzp3L4sWLee6557jzzjv5y1/+gsfj4aKLLoqVeeedd7jssss499xz+a//+i8KCgqw2+2sWLGCp59+OlbONE00TWPNmjUYhtHqvdLS0vrkmkTiSCISCdW85tJcSUkJU6dO5dlnn+W2225j9erVzJkzB6fTGSvz17/+FZfLxWuvvdbi+IoVK1qca/To0SilKCkpYdy4cb1zIaJPSRuRSKjU1FSANgc0zp07l/Xr1/PEE09w5MiRVrdlhmGgaRqRSCR2bPfu3bz44ostyl1xxRUYhsHSpUtRx3X6KqWorKxMzMWIPiPd9yKhPvjgA6ZOncrs2bO56qqrsNvtXHrppaSmpvLll19SXFxMWloadrudgwcPtmjP+cc//sEFF1zAOeecwzXXXENFRQWPP/44+fn5bN68uUXSeeCBB1i8eDFnnnkmc+bMIT09nbKyMl544QVuueUWFi1aZMXli+6ysMdODFA/+9nP1LBhw5Su66268s866ywFqJtuuqnN1/7pT39SY8eOVU6nU02YMEGtWLFC3XPPPaqtX9W//vWv6uyzz1apqakqNTVVTZgwQS1YsEBt27atty5N9BKpEQkhLCdtREIIy0kiEkJYThKREMJykoiEEJaTRCSEsJwkIiGE5fr1FA/TNNm/fz/p6elxpxYIIayjlKKuro7CwkJ0PX69p18nov3791NUVGR1GEKIDpSXlzN8+PC4z/frRJSeng5ELzIjI8PiaIQQx6utraWoqCj2WY2nXyeiptuxjIwMSURCJLGOmk6ksVoIYTlJREIIy0kiEkJYrl+3EXWGUopwONxisS3ROYZhYLPZZGiE6HUDOhEFg0EOHDiA1+u1OpR+KyUlhYKCAhwOh9WhiAHM8kS0b98+7rrrLtasWYPX62XMmDGsWLGCM844o0fnNU2TsrIyDMOgsLAQh8Mhf9m7QClFMBjk8OHDlJWVMXbs2HYHpAnRE5YmoqNHj3LWWWdx3nnnsWbNGnJyctixYwdZWVk9PncwGMQ0TYqKikhJSUlAtIOP2+3GbrezZ88egsEgLpfL6pCSX/1h2PUm5E+EvBOtjqbfsDQR/fKXv6SoqKjFLg0lJSVxywcCgRZ7XdXW1nb4HvJXvGfk+9dFZW9DVVn0MXQcGJbfdPQLlv6W/e1vf+OMM87gyiuvJDc3l8mTJ/PHP/4xbvlly5bh8XhiD5neIZKKUviO7OFgrR9vMAxe2U2ksyxNRF988QXLly9n7NixvPbaa3zve9/j9ttv58knn2yz/OLFi6mpqYk9ysvL+zhiIdoRqKW6poayIw3sqfJCQ4XVEfUbliYi0zQ57bTT+MUvfsHkyZO55ZZbuPnmm/n973/fZnmn0xmbziHTOhJj5MiRPPLII1aHMTD4a/GHosNEqr0hXv/wM0xT9qboDEsTUUFBASee2LJB74QTTmDv3r0WRSREDwRqCYTN2Jf++hq8IRm/1hmWJqKzzjqLbdu2tTi2fft2RowYYVFE/VMwGLQ6BAHgryXYLBE5Il4aAmELA+o/LE1EP/zhD1m/fj2/+MUv2LlzJ08//TT//d//zYIFC3r1fYNhM+4jHDE7XTbUibLdMXPmTG677TZuu+02PB4PQ4cO5ac//Wlsp9ORI0fys5/9jOuuu46MjAxuueUWAP75z39yzjnn4Ha7KSoq4vbbb6ehoSF23oqKCi699FLcbjclJSU89dRT3YpPxNFYI6p35GJoYI80UC+JqFMs7VucMmUKL7zwAosXL+a+++6jpKSERx55hGuvvbZX3/fxt3bGfa5kaCpzJg+Lff3f63YRirR9nz88y82VZxzruXvi3TJ8wZZV8R9+dVy3YnzyySe58cYbef/99/nwww+55ZZbKC4u5uabbwbgwQcfZMmSJdxzzz0A7Nq1i4suuoif//znPPHEExw+fDiWzJqGR8yfP5/9+/fz1ltvYbfbuf3226mokAbVRAl7qwmbinpnDsOMowQCXur9kog6w/JBDl//+tf5+te/bnUYSaeoqIjf/OY3aJrG+PHj2bJlC7/5zW9iiej888/nRz/6Uaz8TTfdxLXXXsudd94JwNixY/ntb3/LjBkzWL58OXv37mXNmjW8//77TJkyBYA//elPnHDCCX1+bQNVwFsPQMg1BJdfx+YL4pUaUadYnoissOC8MXGf04+bBXLLuaPjlj1+xsgNZ8UfjNlVX/nKV1pMSZk+fToPPfRQbPLu8VNgPvnkEzZv3tzidkspFZvqsn37dmw2G6effnrs+QkTJpCZmZmwmAc7J35OyE+nYNxo9M0fAIpQ0Gd1WP3CoExEDlvnm8Z6q2xPpaamtvi6vr6eW2+9ldtvv71V2eLiYrZv395XoQ1atoifzBQHmcMK2PdvOzbdj2kGOn6hGJyJqD/YsGFDi6/Xr1/P2LFjMQyjzfKnnXYan332GWPGtF3bmzBhAuFwmI0bN8ZuzbZt20Z1dXVC4x60zAiEG3sv7W6GDc1mWLodilPbf50AZGG0pLV3714WLlzItm3beOaZZ3jssce444474pa/6667eO+997jtttvYtGkTO3bs4KWXXuK2224DYPz48Vx00UXceuutbNiwgY0bN3LTTTfhdrv76pIGtpCX+kCYQ/VBKryAzRk9HpZbs86QRJSkrrvuOnw+H1OnTmXBggXccccdsW76tkycOJG3336b7du3c8455zB58mSWLFlCYWFhrMyKFSsoLCxkxowZXHHFFdxyyy3k5ub2xeUMfCEflQ0Bth0J8+9D9WBvTPBhuTXrDLk1S1J2u51HHnmE5cuXt3pu9+7dbb5mypQpvP7663HPmZ+fz8svv9zi2He/+90exSkahbyEI4qw4SLVpuNVdnYfqOWAuYcLLpCeyY5IjUiIRAj5CJuKkO7C7TDA5qLGF+JoTW1sIKqITxKREIkQ9hOOmER0J267gd0ZXYxPC/vjDogVx8itWRJau3at1SGIrgoHCZuKiN2O02ZgczjRAEOFCIQjfTq0oz+S744QiRAJEDEVYc2J066j2VwYuoZhhqRG1AmSiIRIhKYakW7HYehg2KOJSIW6Pfl5MJFbMyESIexnbG4a+YUFpDptYDgba0TBVqs0iNYkEQmRCJEAWSkOsnIzwaaD4cCmazi1CKb0mnVIEpEQidA0vcNoHFFtc3BSoQdSPDBEpnl0RNqIhEiAYMBHRZ2f8trGZT+Mxp1xI7J6ZmdIIhIiAbw+L7sON/DensYVMZsSkUzx6BRJREIkgBmKJhzD0bgbrs3JoVo/n31ZyadfVlsXWD8hiUiIBDBDfgAMe2MbkeHAH45Ep3nU1VsYWf8wuBKRUtFGxb5+dKHXZOXKlQwZMqTF1toAc+bMkQmqyco0MRsbq+1NNSLdhq5H146KhOT2rCODq9csEoJ3Hur79z3nR2BzdKrolVdeye23387f/vY3rrzySiC6+8Yrr7zS7sx6YaFIgHDjRoqxWzNNQ7M5gAbCQUlEHRlcNaJ+wO12c80118R23gD485//THFxMTNnzrQuMBFfODq9w9QMnA577LDRuDhaOCw9Zx0ZXDUiwx6tnVjxvl1w8803M2XKFPbt28ewYcMoLS1l/vz5LRbTF0kkEiRiKiK6A0ezpXx1W/TnHglJIurI4EpEmtbpWyQrTZ48mVNPPZWVK1fyta99ja1bt/LKK69YHZaIJ+wnJ92J25OBlpcWO3wsEYWsiqzfsPTW7N5770XTtBaPCRMmWBlS0rjpppsoLS1lxYoVzJo1i6Kioo5fJKwRDpLqsJGXlUFuhit22LA50ABdSSLqiOU1opNOOom///3vsa9tNstDSgrXXHMNixYt4o9//CMrV660OhzRnkhjY3TTgvmNstJTmDYqG+2EoRYE1b9Y/qm32Wzk5+dbHUbS8Xg8fPOb3+SVV15hzpw5Vocj2hMOcNQbJOAwyQiGSXFEP1aa4QA0MKVG1BHLe8127NhBYWEho0aN4tprr2Xv3r1xywYCAWpra1s8BrJ9+/Zx7bXX4nQ6Oy4srBMJsqfSy/vlDVTWN2uYbuqkiMi20x2xNBFNmzaN0tJSXn31VZYvX05ZWRnnnHMOdXV1bZZftmwZHo8n9hio7SZHjx7lhRdeYO3atSxYsMDqcERHGrvvI7oDZ7MlYUMYbDtUx9uf7ydiylIg7bH01uziiy+O/X/ixIlMmzaNESNG8Je//IUbb7yxVfnFixezcOHC2Ne1tbUDMhlNnjyZo0eP8stf/pLx48dbHY7oSDhARCkimqPF2tS6YaeqIchho56waWLobe/SK5Kgjai5zMxMxo0bx86dO9t83ul0DorblHj7lonkZIb9zWpEzccRNfWahQlHFM6k+rQlF8vbiJqrr69n165dFBQUWB2KEJ0WDkYnvEY0O3bj2KBTzbCja8cSkYjP0kS0aNEi3n77bXbv3s17773HN77xDQzD4Oqrr07Ye8jmdj0j37+ONU1qVTYnNqPZR0q3o+saugoTMmXd6vZYWln88ssvufrqq6msrCQnJ4ezzz6b9evXk5OT0+Nz2+3RHguv14vb7e7x+QYrr9cLHPt+itaaakS6/bhmA8OGrmkYUiPqkKWJaNWqVb12bsMwyMzMpKKiAoCUlBSZq9UFSim8Xi8VFRVkZmZiGNLQGo9LCzM2N42ho49rUtDt6JqGboYIS42oXQO6+axpoGRTMhJdl5mZKQNOO2BXQYamORlaOKTlE4YdXQeDsHTfd2BAJyJN0ygoKCA3N5eQTDzsMrvdLjWhzojt4HHchGrdzsmFHk7JGIImO3m0a0AnoiaGYcgHSvQOM4I/EMAbDBP2KnKbN0ca0VszTBlZ3ZGk6r4Xot8JB6j2hdh2qJ4PvvS2fC42xUNq4x0ZFDUiIXpNpGl1Rht223G1bt3OwVo/VZVhHBX1jMlNa/scQhKRED3SOM8srLec3gGAYaMhEKaqPoijQVZpbI/cmgnRE80mvDqM4z5Ojd33hgoTDkesia+fkEQkRE80rVet2duoEUW770ERkqVA2iWJSIieaJp5rzuwx6kRwbGdYEXbJBEJ0RONjdVt1oh0Hb1x2EgkLD1n7ZHGaiF6Ihyk0OMixZODp9nC+U20xkGOEdnbrF2SiIToibCfzBQHmXlDILX1VlVa41giU2pE7ZJEJERPRBprOnH2y8vNTCPXEUI/IbsPg+p/pI1IiJ5o3MHjkFcRjrSeYW/Y7Bi6hibTPNoliUiIHlDhANsO1vH37dX4Qm2MFdIbp3mYMo6oPZKIhOiBSDiAgrZ7zQBvWGPn4Xre33Wo74PrRyQRCdEDZtN61W2NrAZCms7hugBfHhnYe/D1lCQiIXogHPQBoNtdba4AajR235sR6b5vjyQiIXqgacS0dvx61Y10W1Miksbq9kgiEqK7lCISit6aGba2E5FhO7YmkSnLxcYliUiI7jLDmJFob5juaHunmKZEZKgwYUlEccmARiG6K+zH7TAoGZpGVklum0WMxlszTUUImyYO+dvfJklEQnRXOIjLZpCfnUJ+gafNIpphQ9eiNaKQ7G0WlyQiIborHG0fIk77EAC6ncnFWZA3FLtLPm7xJE098YEHHkDTNO68806rQxGicyIBfKEIVQGdGl+cSa2GDYeh49AissFnO5IiEX3wwQf84Q9/YOLEiVaHIkTnhQNU1Pr54MsGPimvbruMTPHoFMsTUX19Pddeey1//OMfycrKsjocITqv+XrVbUzvAEC3sa/ax6fllRypl1Ua47E8ES1YsIBLLrmEWbNmdVg2EAhQW1vb4iGEZdpbJraJYaeqIciBqtr4t2/C2sbqVatW8dFHH/HBBx90qvyyZctYunRpL0clRCdFjtWInO3UiHQNdDNCWHrN4rKsRlReXs4dd9zBU089hcvVeonNtixevJiamprYo7y8vJejFKIdzfY0i1sj0m3ouoauwoTN1usViSjLakQbN26koqKC0047LXYsEomwbt06fve73xEIBFrtV+90OnE62+kqFaIvNbs1i9tGZER38tBVWGpE7bAsEV1wwQVs2bKlxbHrr7+eCRMmcNddd7VKQkIknbAf01RENAd2I07XvG7H0EBXEZni0Q7LElF6ejonn3xyi2OpqakMGTKk1XEhklIkSGGmG1tRLpkpba9ZjW6gxWpEcmsWjwz1FKK7wn5y013kFueBM85HybA3thFFJBG1I6kS0dq1a60OQYjOa9qrzIhTGwLQ7QzPdFPocaEVp/dNXP1QUiUiIfqTSMhHvS9E0A858QrptmM9aprUiOKxfECjEP2SUoQCfj47UMsLm4/EL6froDd2vJgyoDEeSURCdIcZJtK4/KsRZ1G0JrVB2F3ZwGdfVvVFZP2SJCIhuiPsbxwXpGGLs151k4awxoEaP3uP1PRNbP1QtxLRF198keg4hOhfwsHG6R12nPb2x7zpRrQp1gzLrVk83UpEY8aM4bzzzuPPf/4zfr8/0TEJkfzCvsbpHU6c9vY/RlrjutWypVB83UpEH330ERMnTmThwoXk5+dz66238v777yc6NiGSV8hP2DQJ604cHcwC0Bu79yNh2VIonm4lokmTJvHoo4+yf/9+nnjiCQ4cOMDZZ5/NySefzMMPP8zhw4cTHacQySXkJWIqQror/sz7Rk23ZkTk1iyeHjVW22w2rrjiCp577jl++ctfsnPnThYtWkRRURHXXXcdBw4cSFScQiSXsB+P286YYTmMzk1rt2jTJosRaSOKq0eJ6MMPP+T73/8+BQUFPPzwwyxatIhdu3bxxhtvsH//fi6//PJExSlEcgn5SHfZGVuYQ8nQ1HaL6s02WRRt69bI6ocffpgVK1awbds2Zs+ezcqVK5k9eza6Hs1rJSUllJaWMnLkyETGKkTyCEX3vMfe/hgigFSXi0lFmUwcPbSXg+q/upWIli9fzg033MD8+fMpKChos0xubi5/+tOfehScEEkr7KMhGCYQ1EkNR3Da4jdYGzY7brsBhiwDEk+3EtEbb7xBcXFxrAbURClFeXk5xcXFOBwO5s2bl5AghUg6IR9fHG5gk6+aM7N9jM5pp53IaNrJQ27N4ulWG9Ho0aM5cqT1/JqqqipKSkp6HJQQSS/kI2Kaneo1U7qN8iovm/dWEpKlQNrUrUSkVNtVzPr6+k6vPy1Evxb2xwY0xl0mtolusK/axxeHqgmEJRG1pUu3ZgsXLgRA0zSWLFlCSkpK7LlIJMKGDRuYNGlSQgMUIimFvISbRla30z4EoBmO6E4eKkxE1q1uU5cS0ccffwxEa0RbtmzB4Ti2IJTD4eDUU09l0aJFiY1QiGQTCWOGQ5gKwkbHt2bo0VUaDRUmJDt5tKlLieitt94CoovcP/roo2RkZPRKUEIktbCvcSF8DbO9Pc2aGDZ0TUNTESKygH6butVrtmLFikTHIUT/EfIRikTnmbkcNjQtzg4eTRo3WTRUWBqr4+h0IrriiisoLS0lIyODK664ot2yq1ev7nFgQiStkBe7oVOUNwTXiKyOy+uNe5uZsrdZPJ1ORB6PJ5b5PR5PrwUkRNILNuAwdEYW5DJyZHbH5Y3G3V4jsrdZPJ1ORM1vx+TWTAxqQW/0X0f7c8xidBujc9IY6crAnt3xlJDBqFvjiHw+H16vN/b1nj17eOSRR3j99dcTFpgQSStYjz8coTpixxeMdFxej07xSLOpDrv6B6tuJaLLL7+clStXAlBdXc3UqVN56KGHuPzyy1m+fHlCAxQi6QQbOFTj5/Xt9by/uxML4semeMjCaPF0e4XGc845B4Dnn3+e/Px89uzZw8qVK/ntb3/b6fMsX76ciRMnkpGRQUZGBtOnT2fNmjXdCUmIvtM4mDFopEQns3ZENzjqDbL3SC3lVd6Oyw9C3UpEXq+X9PTorpWvv/46V1xxBbqu85WvfIU9e/Z0+jzDhw/ngQceYOPGjXz44Yecf/75XH755WzdurU7YQnRN4L1hCImIcONq4P1qgHQ7Rz1BtlXWcu+o5KI2tLtxfNffPFFysvLee211/ja174GQEVFRZcGOV566aXMnj2bsWPHMm7cOO6//37S0tJYv359d8ISom8EG6LLxHa2RmREu+9BVmmMp1uJaMmSJSxatIiRI0cybdo0pk+fDkRrR5MnT+5WIJFIhFWrVtHQ0BA73/ECgQC1tbUtHkL0KaUgGL01C+kuXJ26NbPFElFYElGbujWy+lvf+hZnn302Bw4c4NRTT40dv+CCC/jGN77RpXNt2bKF6dOn4/f7SUtL44UXXuDEE09ss+yyZctYunRpd0IWIjFCXlAmwYgiZLhxOzrXRtS0dpfsbda2bq9ZnZ+fz+TJk1ssjjZ16lQmTJjQpfOMHz+eTZs2sWHDBr73ve8xb948PvvsszbLLl68mJqamtijvLy8u+EL0T3+Wkyl8OFEaQapjs79LY/tbRYO9GZ0/Va3akQNDQ088MADvPnmm1RUVGAeN6O4KzvBOhwOxowZA8Dpp5/OBx98wKOPPsof/vCHVmWdTidOZ/vb+wrRqwJ1KKC4MB81IqtzjdWA1tiFH4lIF35bupWIbrrpJt5++22++93vUlBQ0PGkvy4wTZNAQP5qiCQVqMXQNEYW5jNyXE6nX6bpsu10e7qViNasWcMrr7zCWWed1aM3X7x4MRdffDHFxcXU1dXx9NNPs3btWl577bUenVeIXhNo7CBxdm2+5ZCMVNLIQI2UpXPa0q1ElJWVRXZ2Jyb7daCioiK2EaPH42HixIm89tprfPWrX+3xuYXoFf5aAuEIDREn7mCkc43VgMPhxOG0g7NHWwkOWN1KRD/72c9YsmQJTz75ZIvlYrtKthsS/U6gjsN1Af6xrZ4C8whfPTGvc6/TZdvp9nQrET300EPs2rWLvLw8Ro4cid1ub/H8Rx99lJDghEg6gVpCEZOAI41UZ+cnsPpNnaoaH7X7q5iQ24vx9VPdSkRz5sxJcBhC9AOREATqCEYUfltGp7vuAQKmzp5KL4fslUyY1Hsh9lfdSkT33HNPouMQIvn5joJS+EwbYd1FqrPzHx89No5Iuu/b0u2Ws+rqav7nf/6HxYsXU1UVXQrho48+Yt++fQkLToik4q0EoFrLAE3D47Z38IJjmhKRkjaiNnWrRrR582ZmzZqFx+Nh9+7d3HzzzWRnZ7N69Wr27t0bW6tIiAHFW0XYNKnVoitPZLg7//ExjOjWWyoSQimV0LF3A0G3akQLFy5k/vz57Nixo8XOrrNnz2bdunUJC06IpOKtJBg28dsycdmNLq22qDd26GhK1q1uS7dqRB988EGbUzCGDRvGwYMHexyUEEnJW4nN0Dlp7Ci8nk7s3tGM0TjFw1BhIqaiM5P2B5NuJSKn09nmEhzbt28nJ6fzw96F6DeUAl8VDkPnpNEjIa1rA3oNmwON6LbToYjZueVDBpFu3Zpddtll3HfffYRC0YY3TdPYu3cvd911F9/85jcTGqAQSSFYD+EgaBq4u1YbAsCwc0JBBmeN8nRuMbVBpluJ6KGHHqK+vp6cnBx8Ph8zZsxgzJgxpKenc//99yc6RiGs5432DFeZKVR4w4S7umOrbuBx2xniNrAZMs3jeN26NfN4PLzxxhu8++67fPLJJ9TX13Paaacxa9asRMcnRHJo7Lr/pFJn0/q9zJ1SRGFmF/Yo0xu7+qX7vk1dTkSmaVJaWsrq1avZvXs3mqZRUlJCfn6+dEuKgaux6/6oinbdZ6c6uvZ6w05lQ4D6SA2ZvlCXxiANBl2qIyqluOyyy7jpppvYt28fp5xyCieddBJ79uxh/vz5XV4mVoh+w1eFLxjBb/OQ7rJ1vbFZt7Gv2seOA9VUNQR7J8Z+rEs1otLSUtatW8ebb77Jeeed1+K5f/zjH8yZM4eVK1dy3XXXJTRIISznrcQbjOCzZzIkrYu1IQDdhqFp6GY32pcGgS7ViJ555hl+8pOftEpCAOeffz533303Tz31VMKCEyIpRMLgr6HOH8JnzyQvw9Xxa47XuKWQboYJRWRA4/G6lIg2b97MRRddFPf5iy++mE8++aTHQQmRVHxHUcrkaFAjpLspyurGGly6HV3X0FWEiIysbqVLiaiqqoq8vPgLQeXl5XH06NEeByVEUvFWEgiZVJOBYejke7pRI9INdK1xQKMpt2bH61IbUSQSwWaL/xLDMAjLMgdioPFV4bDpTBo3ivxhudi7Mw6o8dbMUCHCYUlEx+tSIlJKMX/+/Lhb+sjuG2JA8laiaxo5eYXkDO/aovkxetO204qwjCVqpUuJaN68eR2WkR4zMeA0jqomZUj3z2HYyctwkpViR8tLTUxcA0iXEtGKFSt6Kw4hkpNSBGoPc6CyAc3nZER3z6PppDjtpDgUyB6hrcikFyHaE2ygtr6eAzUB/nWgB207miY7ebRDEpEQ7fFVUecLEbClUpjds80RvRGdijo/e4/UJSi4gUMSkRDt8VZR6w/ht2UyLKsLk1zbUO032XW4gc/3VyUouIHD0kS0bNkypkyZQnp6Orm5ucyZM4dt27ZZGZIQLfjrjuALmfhsHoZ1ZbZ9G7TGdavNsMw1O56liejtt99mwYIFrF+/njfeeINQKMTXvvY1GhoarAxLiJjqI4cAcGYM6fGqirEthSIy1u543VqPKFFeffXVFl+XlpaSm5vLxo0bOffccy2KSohj6qsPA5CZ3fPtWXUj+nEzw9JYfTxLE9HxampqAMjObns94EAg0GLQZFvrZguRMEoRqo8uiJadk9/j0x2rEcmt2fGSprHaNE3uvPNOzjrrLE4++eQ2yyxbtgyPxxN7FBUV9XGUYlAJ1jMh182k4mxGDivs8el0W+PeZlIjaiVpEtGCBQv49NNPWbVqVdwyixcvpqamJvYoLy/vwwjFoOOtQkPDnZ6Fy9mNNYiOc6xGJInoeElxa3bbbbfx8ssvs27dOoYPHx63nNPpjDvPTYiE8zWuJOHu2tZB8aS4nIzLS6NkRM/GIw1EliYipRQ/+MEPeOGFF1i7di0lJSVWhiNEC9t274VDdWSmu4m/+E3n2e1OhqQ6IS0p/v4nFUu/IwsWLODpp5/mpZdeIj09PbZLrMfjwe3u2ZgNIXqq+sghzIYgNj09IYkoNsXDlO7741naRrR8+XJqamqYOXMmBQUFscezzz5rZVhCABBs7DHLyE5IGgLdRmVDgN0VNYRk3eoWLL81EyIZ+QJhdH81AJ4EjCECwLCzq6KeA/VHyJoUweNOmr4iy8l3Qog2VB6tQldhnHYbjrRubDHdlmbrVkuNqCVJREK0obYqOrXDnpoJeoL2qteN6JZCKiyJ6DiSiIRoQ/3R6NQOZ/rQxJ3UiNaIDBUmFJZmieakH1GINuiBapQGbk8CE5Fux9CjNaJgJJK48w4AkoiEaMMZuQpFNmZBz6d2xBj22G6vQakRtSC3ZkK0xXcUTdMwUnuwYP7xdFtjY7W0ER1PakRCHE+pZtM7EtRjBmA4KPC4yDLcOIZ0Y7fYAUwSkRDH+WRXOew5Qm6GmwJXN/cxa4vNSYbLToYNSOn5JNqBRBKREMeprqrAEYzgN9IT13UP0LhULJFQtNalaYk7dz8nbURCHMdXE+26d3sS2D4EYDjwhyJU1vvZXyWL+jUniUiIZpRSBGqPAJCamaCpHU0MO1W+ENsP1fPpnorEnrufk0QkRDO1vjC2QA26BqmenMSeXNPQbdH1tCKhQAeFBxdJREI0c6QhgCtcg9tuoKcmZkG05nSbC4Bw0J/wc/dnkoiEaKayLoArXEuKw4CUXkhEdtnbrC3SayZEM65IHU47uF0OcCaw675R062ZGZZbs+YkEQnRzMQhCoZnQsoQ0BN/w2A4ordmKiS3Zs3JrZkQzfka96VP5IjqZgy71IjaIjUiIRpFTIXurUIDSOmdROR0uRidk8qw4vReOX9/JYlIiEbbD9Wx78OtjLLVM2pCgrvuG9kdbnLTXZCRwBHbA4DcmgnR6Eh9AEegsrFGlMB1iJprmuYhvWYtSCISotHR6mrsER8pDhuk9lIisjmp9YfYX1mNPySLozWRRCREo/qq6L56roxssPXSjsKNO3l8XFbBUa/UippIIhIC8AbD0HAEDUjNyu+9NzKcGIaGoUIEQrI4WhNJREIAh+sCpISqcNp1bOkJnuzanM2JTdcxzCCBsCSiJpYmonXr1nHppZdSWFiIpmm8+OKLVoYjBrEDNX7SgodJc9ogvTdrRA4MvbFGFJY2oiaWJqKGhgZOPfVUHn/8cSvDEIJsl8Ywex0etx0yErhg/vFsTmy6JjWi41g6jujiiy/m4osv7nT5QCBAIHBsRGptrSwuJRJjXKoX8tLAkQrOjN57I5sTQ9ewmUFpI2qmX7URLVu2DI/HE3sUFRVZHZIYKGr2Rf/NKOzdJVxt7mN7m8lSIDH9KhEtXryYmpqa2KO8vNzqkMQAUFkfwHdoR/SLzBG9+2Y2J1mpLkYNTeWEnF4aItAP9aspHk6nE6dTfngisd7dvp+hWzYzaoib/OxRvftmmkZaaippDh1kR6GYflUjEiLRwhGTugM70ZRJakZWryyG1ord3fjmvt5/r35CEpEY1PZWecmq+RyHoZNWdEqfbPET0h3U+ELsO1zV6+/VX1h6a1ZfX8/OnTtjX5eVlbFp0yays7MpLi62MDIxWOwsKyPLt5chHidawal98p4+5eCzA7XsM/fxrbGT+uQ9k52liejDDz/kvPPOi329cOFCAObNm0dpaalFUYnBot7rxdjxKqDIGnFy7010PY7dFW0cigS9KKXQZKNFaxPRzJkzUUpZGYIYjEwTDn3KwQ9fJ8V/iNSUVDwnX9Rnb+9wpgJgRAL4QyZuh6xN1K96zYTosaovYMffMRuO0FBdQ8hwk/WVa8Gd2WchGA43Nl3DZgbwBsOSiJBEJAaTvRtg1z8A0B0pnHTOTLYbYxk+vG9uyWLsbuxGUyKKkOCNrfslSURi4FMqmoDK30ehoHAy2qjzsNldnGhFPHY3NkPHHvHJ4miNJBGJgc2MwLb/g4OfolBscpxOg346Z9mcWNZEbE/BbujYgz68QUlEIIlIDGQhP3z2IlSVgabzSeqZrK3Og91VjMpJpTDTbU1cznRy05140HFky/BqkEQkBqqaL+Hz/wNvJUq38XH6TN6ujO7cet6EXOuSEIAjlayUxkX0XdJ1D5KIxEATDsAXb8P+j0ApwvYU3nHOZFNlNPGcM3Yok4oyrY3R5gLdBmYYgg192mOXrCQRiYGjchdsWwOBOgAOp47jZe8Eqn12NA0umJDHKcMTv599l2kaYZubhpoqag9WUFSSaXVElpNEJPo9FQlRv/U1vGUf4LIbeLJzYdyFKFsB1Rv2kuG287UT8yhKovaYOuXi8wO17DfKKSoZZ3U4lpNEJPqliKkoO1LPjn2VuLa9gKshurCZrXgqU6Z8Aww7ucDXJxYwYkgqDltyze92pkZXgQz56mSaB5KIRD+0+0gDa7dVUFPv44TDr+AKHEIZDqpLLiG/5EQw7LGyY/OSc495lzsDDbCFGvAGI6Q6B/dHcXBfveh3whGTNz+voNYb4OSatYxz1ZCVl0vqtOswMnpx940E013pOGw6jkgDdf6wJCKrAxCiK2yGztdPyefgxv/lpCFHsdky4NSroB8lIQDcWThsOq5wLXX+EPkel9URWUoSkegXTFOh69F2lLyaT8jTdoFuwAmXQmY/XLvKnYnTpuP01VLrD1sdjeWSqwVPiDbUeEOUvrebnRX1cGgr7Hor+sToCyD3BGuD667GGpEj4qWuocHqaCwnNSKR1ExT8erWA9T4Qnz+708Yrf4ZnSNWNCX66K/sbrI8GbhsBka2rMkliUgktfVfVLK/2k+WWcmsyHtomgm5E6K1oX4uIyuXDOMAGF6rQ7Gc3JqJpFVe5eX93VU4QzXMVu/g0sKQNQImXNoni9z3OndW9F9vpbVxJAFJRCIp+YIRXtt6EGewmgsCfyfXGYG0XDjpCjAGSEU+LR9vMMyePV9wtCFodTSWkkQkko5pKl7behBV8yVTqtcwJkNFF7afOBfsA6ibOz2PvVVeysp28sWRwd1gLYlIJB8zQnHDJ5xy+BVOyNYxMvJg0jXgTLM6ssRKyyfd7cAZrqei8ojV0VhqgNRxRb9mmuCvRjUcQas/hH5wC5ND1fiHpeMedhKMnw22AbjVuN1FSnYhVO6g/sBOTHNMbKzUYCOJSPS9hsrobhp1+/EePUBN5SFqvQH8oQgnD/NgaBqaIw33hJmQ3ze7r1rFM2w8tl07San9gn3VvqRaIaAvJUUievzxx/n1r3/NwYMHOfXUU3nssceYOnWq1WElJ6XwHf4Cd91e8FWBplNvZFLtHk7h8NHoRpLebQe90cGIh7bgrdzH4foAVfVB/GETAFOz4bdlstsoYvS4idGBis0mrw5Ueu4JZKe+Qah+D5/v3U9R9hirQ7KE5Yno2WefZeHChfz+979n2rRpPPLII1x44YVs27aN3Nxcq8NLGmFfLQe2fcjRne/jrTnCxOEeUhzRH19tjY/ySi97XJmkF0+k6ISpZA2NM/fKNCFYD/5q8NcQbKim2heiKmiAK4sJo0eBIxU0jXd2HCYQMtF10NAwdI10l40Mt53cdCfprnYShVLR1RKr98Lhz+HwNjDDVNT52XnYS61rGLWp+ficQ/EMyacwv4AxQ1PJTXcO6BpQK+l55AwroeLzz/FtX0fF6OHkpg+gBvlO0pTFW61OmzaNKVOm8Lvf/Q4A0zQpKiriBz/4AXfffXe7r62trcXj8VBTU0NGRkZfhNtnzFCQ+ppKqg+WUbvv33gPbCcSie74ENHtFI45lVElY0CZ7C7bQUXZp0TCodjrHRlDyRxaiCcjnaFuPbokqb+WqqrD+AIhvMEwDYEwvpAZe02608bJwzxgd0NaLmvKIlSSSdBIJaw7MTUDXUXQVZhMe4hvTRwSremEfBw5ehSH8uMw/egRP6a/gWAoSH0gjNtuRNdoTsvFO/QUntzhZljeEE7IT6d4SApO2yDfYPDoHra/9geqfSHGnjGLrHFnDZiG+c5+Ri1NRMFgkJSUFJ5//nnmzJkTOz5v3jyqq6t56aWXWpQPBAIEAoHY17W1tRQVFXWciKrKqN36Gv/eX9PsYPSylQINRVFWSmxB9fpAiE/31bQoF/0n+v9hmW6KsqJlfaEwm8pr0FCtzg1QkOFixJDofb8/FGZTeXWL9z32EkVuhotRQ1NBmfh9Xj5uLNsklJJP+qgzGHnCGWRltPxFjYQC7NuxicO7PiZ4+AtQ0QTjMHROH5EVK/dx+VF8IUXQlkbAlo7fSMft0BniMMkx6il2B6LBAQdqfERMFbv0iFIEwya+UAS33WBc41o/CsWHu48SNlv/KvltGThyx3LWWTMgvQA0jUA4IsnnOP4dawntWke6M1rLDGDno/0+NN0ATUPTdECjaQ+kfI+LEdlNv1cRPvmyJs6ZIS/dxcih0bLBsOLj8qNxyw5NczI6J7oldthUbNzTuqyuaUwZmQWnfBtc7VcAOpuILL01O3LkCJFIhLy8vBbH8/Ly+Pzzz1uVX7ZsGUuXLu36G4UDUF+BWV/b5tMKUE4fOBp3dgiEUd74P1icAXA3DkALRcBfQ7xsrpwhCESTghaOoALHxou0ek3IhFC0jcdh00G3o9LzcOWUkDNqEsOGFcftVTHsTopPnEbxidOor69ld9kujh45QCQSgtH50dstZzoNQ0PUKTdD0t2UZDjJSXfGbvEAiISg4Qg0VFDQcBjqD0OoIbo1jxmOLvpu2FE2V/Scdjch3UV1sIHqkA2vchLQnGh2NzZ3OkM8aQzPckPGsWQoSag119iZuDILYO+/oPYAoYCvxe9Vq98Vo/H7DxCKYNZVxz230lzgaiwbMTHr4icicIK78Y+c2XZZTQPqw7E/dolgaY1o//79DBs2jPfee4/p06fHjv/nf/4nb7/9Nhs2bGhRvts1omADgeqDHKkPAtqxJohmbRHpbjvpjYtTBcMmVd5jtznNy2maRorDFi2raYQiJkcbyzZf7lNr/NPlcuikOaNbx0SUipU9dtpjr3E6bKQ5bNFjjhSU4UTTk7TxOQ6lFEoxaLuhEyIcIOStpqaugUgkgqlMlGnS/JOa7jLIaGyjC5mKitpAnJNBqtMg0x0tGzYVh9opm+LQY1sdmQoO1PhbldE0KPS4wDO8ww6FflEjGjp0KIZhcOjQoRbHDx06RH5+68ZWp9OJ09mN8SSOVJy5oxnWybZvB9DZZbbsQG5Wh8UAMIChXWjK6o8fZU3TBlVbc6+wObFn5HX6d8UODBvayVN3oawODBvSubI9ZemfW4fDwemnn86bb74ZO2aaJm+++WaLGpIQYmCzvPt+4cKFzJs3jzPOOIOpU6fyyCOP0NDQwPXXX291aEKIPmJ5Ipo7dy6HDx9myZIlHDx4kEmTJvHqq6+2asAWQgxclo8j6omBPI5IiIGgs5/R/tUlI4QYkCQRCSEsJ4lICGE5yxure6Kpeau2tu0R00IIazV9Njtqiu7Xiaiurg6AoqIiiyMRQrSnrq4Oj8cT9/l+3Wtmmib79+8nPT29xfSKZNY0LaW8vHxA9fQNxOsaiNcEfXtdSinq6uooLCxEb2e6Ur+uEem6zvDhw60Oo1syMjIG1C93k4F4XQPxmqDvrqu9mlATaawWQlhOEpEQwnKSiPqY0+nknnvu6d4qAklsIF7XQLwmSM7r6teN1UKIgUFqREIIy0kiEkJYThKREMJykoiEEJaTRNTL7r//fs4880xSUlLIzMzs1GuUUixZsoSCggLcbjezZs1ix44dvRtoF1VVVXHttdeSkZFBZmYmN954I/X19e2+ZubMmY1rWh97/Md//EcfRdy2xx9/nJEjR+JyuZg2bRrvv/9+u+Wfe+45JkyYgMvl4pRTTuH//u//+ijSrunKdZWWlrb6ubhcfbzJoxK9asmSJerhhx9WCxcuVB6Pp1OveeCBB5TH41Evvvii+uSTT9Rll12mSkpKlM/n691gu+Ciiy5Sp556qlq/fr1655131JgxY9TVV1/d7mtmzJihbr75ZnXgwIHYo6ampo8ibm3VqlXK4XCoJ554Qm3dulXdfPPNKjMzUx06dKjN8u+++64yDEP96le/Up999pn6f//v/ym73a62bNnSx5G3r6vXtWLFCpWRkdHi53Lw4ME+jVkSUR9ZsWJFpxKRaZoqPz9f/frXv44dq66uVk6nUz3zzDO9GGHnffbZZwpQH3zwQezYmjVrlKZpat++fXFfN2PGDHXHHXf0QYSdM3XqVLVgwYLY15FIRBUWFqply5a1Wf7b3/62uuSSS1ocmzZtmrr11lt7Nc6u6up1dfZ3szfJrVmSKSsr4+DBg8yaNSt2zOPxMG3aNP71r39ZGNkx//rXv8jMzOSMM86IHZs1axa6rrfai+54Tz31FEOHDuXkk09m8eLFeL3e3g63TcFgkI0bN7b4Puu6zqxZs+J+n//1r3+1KA9w4YUXJs3PBbp3XQD19fWMGDGCoqIiLr/8crZu3doX4cb060mvA9HBgwcB2tz9tuk5qx08eJDc3JabxNlsNrKzs9uN8ZprrmHEiBEUFhayefNm7rrrLrZt28bq1at7O+RWurrLMESvO5l/LtC96xo/fjxPPPEEEydOpKamhgcffJAzzzyTrVu39tmkcqkRdcPdd9/dqnHv+Ee8H3oy6+3ruuWWW7jwwgs55ZRTuPbaa1m5ciUvvPACu3btSuBViK6aPn061113HZMmTWLGjBmsXr2anJwc/vCHP/RZDFIj6oYf/ehHzJ8/v90yo0aN6ta5m3a4PXToEAUFBbHjhw4dYtKkSd06Z2d19rry8/OpqKhocTwcDlNVVdXmDr3xTJs2DYCdO3cyevToLsfbE13dZRiiP5uulLdCd67reHa7ncmTJ7Nz587eCLFNkoi6IScnh5ycnF45d0lJCfn5+bz55puxxFNbW8uGDRv43ve+1yvv2aSz1zV9+nSqq6vZuHEjp59+OgD/+Mc/ME0zllw6Y9OmTQAtEm5fab7L8Jw5c4Bjuwzfdtttbb5m+vTpvPnmm9x5552xY2+88UZS7Urcnes6XiQSYcuWLcyePbsXIz2OpU3lg8CePXvUxx9/rJYuXarS0tLUxx9/rD7++GNVV1cXKzN+/Hi1evXq2NcPPPCAyszMVC+99JLavHmzuvzyy5Oy+37y5Mlqw4YN6p///KcaO3Zsi+77L7/8Uo0fP15t2LBBKaXUzp071X333ac+/PBDVVZWpl566SU1atQode6551p1CWrVqlXK6XSq0tJS9dlnn6lbbrlFZWZmxrquv/vd76q77747Vv7dd99VNptNPfjgg+rf//63uueee5K2+74r17V06VL12muvqV27dqmNGzeqq666SrlcLrV169Y+i1kSUS+bN2+eAlo93nrrrVgZQK1YsSL2tWma6qc//anKy8tTTqdTXXDBBWrbtm19H3w7Kisr1dVXX63S0tJURkaGuv7661sk17KyshbXuXfvXnXuueeq7Oxs5XQ61ZgxY9SPf/xjS8cRKaXUY489poqLi5XD4VBTp05V69evjz03Y8YMNW/evBbl//KXv6hx48Yph8OhTjrpJPXKK6/0ccSd05XruvPOO2Nl8/Ly1OzZs9VHH33Up/HKMiBCCMtJr5kQwnKSiIQQlpNEJISwnCQiIYTlJBEJISwniUgIYTlJREIIy0kiEkJYThKR6FdKS0tbLLl77733tpgMPH/+/NgcK9F/SCISbZo/f37cNaUXLFiApmktZur3RgIYOXIkjzzySItjc+fOZfv27XFf8+ijj1JaWhr7eubMmS0mqYrkJIlIxFVUVMSqVavw+XyxY36/n6effpri4mJLYnK73a0WZWvO4/F0epMCkTwkEYm4TjvtNIqKilqsoLh69WqKi4uZPHlyj87dVk1lzpw5sVrWzJkz2bNnDz/84Q9ji7JB61uz4zWvmc2fP5+3336bRx99NHaOsrIyxowZw4MPPtjidZs2bULTtD5dg0ccI4lItOuGG25gxYoVsa+feOIJrr/++l5/39WrVzN8+HDuu+8+Dhw4wIEDB7p8jkcffZTp06dz8803x85RXFzc6poAVqxYwbnnnsuYMWMSdQmiCyQRiXZ95zvf4Z///Cd79uxhz549vPvuu3znO9/p9ffNzs7GMAzS09PJz8/v1iqIHo8Hh8NBSkpK7ByGYTB//ny2bdsW2+srFArx9NNPc8MNNyT6MkQnyQqNol05OTlccskllJaWopTikksuYejQoVaH1SOFhYVccsklPPHEE0ydOpX//d//JRAIcOWVV1od2qAlNSLRoRtuuIHS0lKefPLJhNUadF3n+KWwQqFQQs7dGTfddFOsIX7FihXMnTuXlJSUPnt/0ZIkItGhiy66iGAwSCgU4sILL0zIOXNyclq0+0QiET799NMWZRwOB5FIpEfvE+8cs2fPJjU1leXLl/Pqq6/KbZnF5NZMdMgwDP7973/H/h9PTU1NbEH8JkOGDKGoqKhV2fPPP5+FCxfyyiuvMHr0aB5++GGqq6tblBk5ciTr1q3jqquuwul0duuWcOTIkWzYsIHdu3eTlpZGdnY2uq7H2ooWL17M2LFjk2oB/MFIakSiUzIyMsjIyGi3zNq1a5k8eXKLx9KlS9sse8MNNzBv3jyuu+46ZsyYwahRozjvvPNalLnvvvvYvXs3o0eP7vauKYsWLcIwDE488URycnLYu3dv7Lkbb7yRYDDYJ72Aon2yZrUYtN555x0uuOACysvLW+2MKvqWJCIx6AQCAQ4fPsy8efPIz8/nqaeesjqkQU9uzcSg88wzzzBixAiqq6v51a9+ZXU4AqkRCSGSgNSIhBCWk0QkhLCcJCIhhOUkEQkhLCeJSAhhOUlEQgjLSSISQlhOEpEQwnL/H1gp357NDcN2AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:30:57.814859Z", + "iopub.status.busy": "2024-03-22T20:30:57.814516Z", + "iopub.status.idle": "2024-03-22T20:30:58.164627Z", + "shell.execute_reply": "2024-03-22T20:30:58.163782Z" + }, + "papermill": { + "duration": 0.372765, + "end_time": "2024-03-22T20:30:58.166864", + "exception": false, + "start_time": "2024-03-22T20:30:57.794099", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7gklEQVR4nO3deXxV9Zn48c85565Z7k0CWSGQsKOA4AKDVkVlquJGnbHW0gquHcW2lGFU5tfSYmvRVh1p69BtgDoFqW1Rp+MotahYURFRcGeJIUQIBBKy3Nz9nPP74yaX7GS5yblJnvfrdV+Qc7/33OckuU++57sqpmmaCCGEhVSrAxBCCElEQgjLSSISQlhOEpEQwnKSiIQQlpNEJISwnCQiIYTlJBEJISxnszqA3jAMgyNHjpCeno6iKFaHI4RoxTRN6uvrKSgoQFU7rvcM6ER05MgRCgsLrQ5DCHEa5eXljBw5ssPnB3QiSk9PB2IX6fF4LI5GCNFaXV0dhYWF8c9qRwZ0Imq6HfN4PJKIhEhip2s6kcZqIYTlJBEJISwniUgIYbkB3UbUFaZpEo1G0XXd6lAGNU3TsNlsMoxC9Ijliejw4cPcd999vPDCC/j9fsaNG8e6des499xze33ucDhMRUUFfr8/AZGK00lJSSE/Px+Hw2F1KGKAsTQRnTx5kgsuuIBLLrmEF154gezsbPbv309mZmavz20YBqWlpWiaRkFBAQ6HQ/5a9xHTNAmHwxw/fpzS0lLGjx/f6eA1IVqzNBE9/PDDFBYWsm7duvix4uLihJw7HA5jGAaFhYWkpKQk5JyiY263G7vdTllZGeFwGJfLZXVI1vAdh5KtkDcNcs+wOpoBw9I/W//zP//Dueeeyw033EBOTg4zZszgN7/5TYflQ6EQdXV1LR6nI3+Z+498r4HSbVBdCh8/B3rU6mgGDEt/cz777DPWrFnD+PHj2bJlC3fddRff+ta3+N3vftdu+VWrVuH1euMPmd4hkoppEjhRxtG6IP5wFPxVVkc0YFiaiAzD4Oyzz+bHP/4xM2bM4M477+SOO+7gl7/8Zbvlly9fTm1tbfxRXl7ezxEL0YlQHTW1tZSeaKCs2g8NlVZHNGBYmojy8/M544yW99GTJ0/m0KFD7ZZ3Op3x6RwyrUMknWAdwUhsmEiNP8Jf3/kYw5DdurrC0kR0wQUXsHfv3hbH9u3bx+jRoy2KKDnMmTOHJUuWWB2G6K5QHaGoEf8y6KvFH5Hxa11haSL6zne+w1tvvcWPf/xjDhw4wMaNG/n1r3/N4sWLrQxLiJ4J1hFulogcup+GkDRYd4Wliei8887jmWee4amnnmLKlCn88Ic/5PHHH2fBggV9+r7hqNHhI6obXS4b6ULZ7lq0aBHbtm1j9erVKIqCoiiMHDmSNWvWtCj33nvvoaoqZWVlADz22GNMnTqV1NRUCgsLufvuu/H5fC1e8/rrr3PhhRfidrspLCzkW9/6Fg0NDd2OUXSgsUbkc+SgKWDXG/BJIuoSy0dWX3311Vx99dX9+p5PvHKgw+eKh6cyf8aI+Ne/fq2EiN7+ff7ITDc3nHuq527t9lIC4ZZV8e/844RuxbZ69Wr27dvHlClTeOCBBwB49NFH2bhxI3fddVe83IYNG7jgggvit7GqqvKzn/2M4uJiPvvsM+6++27uvfde/vM//xOAkpISrrjiCn70ox+xdu1ajh8/zj333MM999zTYhyX6Lmov4aoYeJzZjNCO0ko5McXlETUFTLwI8l4vV4cDgcpKSnk5eWRl5fHggUL2L59e7wR3zAMNm3a1KLmuGTJEi655BKKioq49NJL+dGPfsTTTz8df37VqlUsWLCAJUuWMH78eM4//3x+9rOf8eSTTxIMBvv9OgejkD9WA424huGyqdiMMH6pEXWJ5TUiKyy+ZFyHz6mtZoHcedHYDsu2njFy6wWJGRXe2vTp05k8eTIbN27k/vvvZ9u2bVRWVnLDDTfEy/ztb39j1apVfPrpp9TV1RGNRgkGg/j9flJSUtizZw/vv/8+GzZsiL/GNM34VJjJkyf3SexDiZMgk/PSyZ8wFvX9nYBJJBywOqwBYUjWiBw2tcOHTVO7XNbehbKJsmDBAjZu3AjAxo0bueKKKxg2bBgABw8e5Oqrr2batGn8+c9/ZteuXTzxxBNAbKoLgM/n4xvf+Aa7d++OP/bs2cP+/fsZO7bjZCu6zqYHyUhxUDQiH81ux6YqaEbI6rAGhCFZI0p2DoejzbIlX/3qV/nud7/Lrl27+NOf/tRi0OeuXbswDINHH300Ps2i+W0ZwNlnn83HH3/MuHEd1wZFLxg6RGNJH7ubEcOzGJFuh1Gp1sY1QAzJGlGyKyoqYseOHRw8eJATJ05gGAZFRUWcf/753Hbbbei6zrXXXhsvP27cOCKRCD//+c/57LPP+O///u82o9Pvu+8+3njjDe655x52797N/v37ee6557jnnnv6+/IGp4gfXyjKMV+YSj9gc8aOR+XWrCskESWhZcuWoWkaZ5xxBtnZ2fFG6gULFrBnzx6+9KUv4Xa74+XPOussHnvsMR5++GGmTJnChg0bWLVqVYtzTps2jW3btrFv3z4uvPBCZsyYwYoVKygoKOjXaxu0IgGqGkLsPRHlk2M+sDf+fKJya9YVimmaA3YMel1dHV6vl9ra2jbTPYLBIKWlpRQXFw/dJSn62ZD+np88SMlLv6Es6Cb1C3cxteZvHPx0NxW5F3PZZVdYHZ1lOvuMNic1IiESIRIgaphEVBduhwY2F7WBCCdr6xjAf+v7jSQiIRIhGiSqG+iqE7ddw+6MLcanRIMdDogVp0giEiIRomGihomu2nHaNGwOJwqgmRFCUZn4ejqSiIRIBD2EbphEFSdOu4pic6GpCpoRkRpRF0giEiIRmtWIHJoKmj2WiMxIjyY/DzUyoFGIRIgGGZ+TRl5BPqlOG2jOxhpRuM0qDaItSURCJIIeIjPFQWZOBthU0BzYVAWnomNIr9lpSSISIhGapndojSOqbQ7OLPBCiheGyTSP05E2IiESIBwKUFkfpLyucdkPrXG3Wz1sXVADiCSiJJRsa1YnWzzJyB/wU3K8gTfKGle8bEpEMsWjSyQRDVJNy3+I/mFEYglHczRObbE5OVYX5OPPq/jw8xrrAhsgJBElmfbWrC4pKeG2226juLgYt9vNxIkTWb16dZvXzZ8/nwcffJCCggImTpwIwBtvvMH06dNxuVyce+65PPvssyiKwu7du+Ov/fDDD7nyyitJS0sjNzeXr3/965w4caLDeA4ePNhf344Bw4jEVrnU7I1tRJqDYFSPTfOo93XySgFDrbHaNEGP9P/7ava2yzl2oL01qzMzMxk5ciR//OMfGTZsGG+88QZ33nkn+fn5fPnLX46/duvWrXg8Hl566SUgNuHwmmuuYd68eWzcuJGysrI2t1g1NTVceuml3H777fzHf/wHgUCA++67jy9/+cu8/PLL7caTnZ2dgG/KIGIYGI2N1famGpFqQ1U1APSI3J6dztBKRHoE/v5o/7/vhf8KNkeXirZes7rJypUr4/8vLi7mzTff5Omnn26RiFJTU/ntb3+LwxF7r1/+8pcoisJvfvMbXC4XZ5xxBocPH+aOO+6Iv+YXv/gFM2bM4Mc//nH82Nq1ayksLGTfvn1MmDCh3XhEM3qIaONGivFbM0VBsTmABqJhSUSnM7QS0QD2xBNPsHbtWg4dOkQgECAcDjN9+vQWZaZOnRpPQgB79+5l2rRpLZbkmDlzZovX7Nmzh1deeYW0tLQ271lSUsKECd3bhWRIisamdxiKhtNhjx/WGhdHi0alve50hlYi0uyx2okV79sLmzZtYtmyZTz66KPMnj2b9PR0fvrTn7Jjx44W5VJTuz9exefzcc011/Dwww+3eS4/P7/HMQ8pehjdMNFVBw5Nix9WbbGfux6RRHQ6QysRKUqXb5Gs1HrN6u3bt3P++edz9913x4+VlJSc9jwTJ07k97//PaFQCKcz9td5586dLcqcffbZ/PnPf6aoqAibrf1fh/bW0BbNRINkpztxez0ouadqlqcSkQXtkgOMpb1mP/jBD+I9MU2PSZMmWRlSUmi9ZvX48eN555132LJlC/v27eN73/tem4TSnq9+9asYhsGdd97JJ598wpYtW3jkkUcAUBobzxcvXkx1dTU33XQTO3fupKSkhC1btnDLLbfEk097a2iLZqJhUh02cjM95HhO3QZrNgcKoJqSiE7H8u77M888k4qKivjj9ddftzoky7Ves/ryyy/n+uuv58Ybb2TWrFlUVVW1qB11xOPx8Je//IXdu3czffp0/t//+3+sWLECIN5uVFBQwPbt29F1nS9+8YtMnTqVJUuWkJGREd8RpKM1tEUjvbExumnB/EaZ6SnMGpPF1WcOtyCogcXyWzObzSa9Ma1MmDCBN998s8WxdevWtdkauvkC+evXr2/3XOeffz579uyJf71hwwbsdjujRo2KHxs/fjybN2/uVjyimWiIk/4wIYeBJxwlxRH7WCmaA1DAkBrR6VieiPbv309BQQEul4vZs2ezatWqFh+S5kKhEKHQqa7Qurq6/gpzwHryyScZM2YMI0aMYM+ePfExQs13ARG9pIcpq/JzqL6BGZPDpGQ1fqyaOil02Xb6dCy9NZs1axbr16/nxRdfZM2aNZSWlnLhhRdSX1/fbvlVq1bh9Xrjj8LCwn6OeOA5evQoX/va15g8eTLf+c53uOGGG/j1r39tdViDS2P3va46cDbb3TeCxt5j9Wz79Ai6IUuBdMbSGtGVV14Z//+0adOYNWsWo0eP5umnn+a2225rU3758uUsXbo0/nVdXZ0ko9O49957uffee60OY3CLhtBNE11xtNhmXNXsVDeEOa75iBoGmqp1cpKhzfJbs+YyMjKYMGECBw4caPd5p9MZ74YWIlkY0WCzGlHzcURNvWZRorqJM6k+bcnF8l6z5nw+HyUlJTKQTgwo0XBswquu2LFrp+YUKpodVTmViETHLE1Ey5YtY9u2bRw8eJA33niDL33pS2iaxk033ZSw95DN7frPUP1eN01qNW1ObFqzj5RqR1UVVDNKRMZedcrSyuLnn3/OTTfdRFVVFdnZ2XzhC1/grbfeSsjsbrs91mPh9/ulh6if+P1+4NT3fqhoqhGp9lbNBpoNVVHQpEZ0WpYmok2bNvXZuTVNIyMjg8rKSgBSUlLio4lFYpmmid/vp7KykoyMDDRtaDXKupQo43PSGD62VZOCakdVFFQjQlRqRJ0a1M1nTQMlm5KR6FsZGRlDcnCq3QwzPM3J8IJhLZ/Q7KgqaESl+/40BnUiUhSF/Px8cnJyiMjEwz5lt9uHXE0oLr6DR6sJ1aqdKQVepnqGochOHp0a1ImoiaZpQ/dDIvqWoRMMhfCHo0T9JjnNmyO12K0ZhoysPp2k6r4XYsCJhqgJRNh7zMfOz/0tn4tP8ZDa+OkMiRqREH1Gb1qd0Ybd1qrWrdo5WhekuiqKo9LHuJy2q2CKGElEQvRG4zyzqNpyegcAmo2GUJRqXxhHg6zS2Bm5NROiN5pNeHVorT5Ojd33mhklGpUVLjsjiUiI3mhar1qxt1MjinXfg0lElgLplCQiIXqjaea96sDeQY0ITu0EK9oniUiI3mhsrG63RqSqqI3DRvSo9Jx1RhqrheiNaJgCr4sUbzbeZgvnN1EaBznqsrdZpyQRCdEb0SAZKQ4ycodBatutqpTGsUSG1Ig6JYlIiN7QG2s6HeyXl5ORRo4jgjo5qx+DGnikjUiI3mjcweOY3ySqt51hr9nsaKqCItM8OiWJSIheMKMh9h6t52/7aghE2hkrpDZO8zBkHFFnJBEJ0Qt6NIQJ7feaAf6owoHjPt4uOdb/wQ0gkoiE6AWjab3q9kZWAxFF5Xh9iM9PyB58nZFEJEQvRMMBAFS7q90VQLXG7ntDl+77zkgiEqIXmkZMK63Xq26k2poSkTRWd0YSkRA9ZZrokditmWZrPxFptlNrEhmyXGyHJBEJ0VNGFEOP9YapjvZ3imlKRJoZJSqJqEMyoFGInooGcTs0ioenkVmc024RrfHWTDF1ooaBQ/72t0sSkRA9FQ3jsmnkZaWQl+9tt4ii2VCVWI0oInubdUgSkRA9FY21D9FB+xAAqp0ZozIhdzh2l3zcOpI09cSHHnoIRVFYsmSJ1aEI0TV6iEBEpzqkUhvoYFKrZsOhqTgUXTb47ERSJKKdO3fyq1/9imnTplkdihBdFw1RWRdk5+cN7Cmvab+MTPHoEssTkc/nY8GCBfzmN78hMzPT6nCE6Lrm61W3M70DANXG4ZoAH5ZXccInqzR2xPJEtHjxYq666irmzp172rKhUIi6uroWDyEs09kysU00O9UNYSqq6zq+fRPWNlZv2rSJd999l507d3ap/KpVq1i5cmUfRyVEF+mnakTOTmpEqgKqoROVXrMOWVYjKi8v59vf/jYbNmzA5Wq7xGZ7li9fTm1tbfxRXl7ex1EK0Ylme5p1WCNSbaiqgmpGiRpt1ysSMZbViHbt2kVlZSVnn312/Jiu67z22mv84he/IBQKtdmv3ul04nR20lUqRH9qdmvWYRuRFtvJQzWjUiPqhGWJ6LLLLuODDz5oceyWW25h0qRJ3HfffW2SkBBJJxrEMEx0xYFd66BrXrWjKaCaukzx6IRliSg9PZ0pU6a0OJaamsqwYcPaHBciKelhCjLc2ApzyEhpf81qVA0lXiOSW7OOyFBPIXoqGiQn3UXOqFxwdvBR0uyNbUS6JKJOJFUievXVV60OQYiua9qrTOugNgSg2hmZ4abA60IZld4/cQ1ASZWIhBhI9EgAXyBCOAjZHRVSbad61BSpEXXE8gGNQgxIpkkkFOTjijqeef9Ex+VUFdTGjhdDBjR2RBKRED1hRNEbl3/VOlgUrUldGA5WNfDx59X9EdmAJIlIiJ6IBhvHBSnYOlivuklDVKGiNsihE7X9E9sA1KNE9NlnnyU6DiEGlmi4cXqHHae98zFvqhZrijWicmvWkR4lonHjxnHJJZfw+9//nmAwmOiYhEh+0UDj9A4nTnvnHyOlcd1q2VKoYz1KRO+++y7Tpk1j6dKl5OXl8Y1vfIO333470bEJkbwiQaKGQVR14jjNLAC1sXtfj8qWQh3pUSKaPn06q1ev5siRI6xdu5aKigq+8IUvMGXKFB577DGOHz+e6DiFSC4RP7phElFdHc+8b9R0a4Yut2Yd6VVjtc1m4/rrr+ePf/wjDz/8MAcOHGDZsmUUFhZy8803U1FRkag4hUgu0SBet51xI7IZm5PWadGmTRZ1aSPqUK8S0TvvvMPdd99Nfn4+jz32GMuWLaOkpISXXnqJI0eOcN111yUqTiGSSyRAusvO+IJsioendlpUbbbJomhfj0ZWP/bYY6xbt469e/cyb948nnzySebNm4eqxvJacXEx69evp6ioKJGxCpE8IrE977F3PoYIINXlYnphBtPGDu/joAauHiWiNWvWcOutt7Jo0SLy8/PbLZOTk8N//dd/9So4IZJWNEBDOEoorJIa1XHaOm6w1mx23HYNNFkGpCM9SkQvvfQSo0aNiteAmpimSXl5OaNGjcLhcLBw4cKEBClE0okE+Ox4A7sDNZyfFWBsdiftRFrTTh5ya9aRHrURjR07lhMn2s6vqa6upri4uNdBCZH0IgF0w+hSr5mp2iiv9vP+oSoishRIu3qUiEyz/Sqmz+fr8vrTQgxo0WB8QGOHy8Q2UTUO1wT47FgNoagkovZ069Zs6dKlACiKwooVK0hJSYk/p+s6O3bsYPr06QkNUIikFPETbRpZ3Un7EICiOWI7eZhRdFm3ul3dSkTvvfceEKsRffDBBzgcpxaEcjgcnHXWWSxbtiyxEQqRbPQoRjSCYUJUO/2tGWpslUbNjBKRnTza1a1E9MorrwCxRe5Xr16Nx+Ppk6CESGrRQONC+ApGZ3uaNdFsqIqCYurosoB+u3rUa7Zu3bpExyHEwBEJENFj88xcDhuK0sEOHk0aN1nUzKg0Vnegy4no+uuvZ/369Xg8Hq6//vpOy27evLnXgQmRtCJ+7JpKYe4wXKMzT19ebdzbzJC9zTrS5UTk9Xrjmd/r9fZZQEIkvXADDk2lKD+HoqKs05fXGnd71WVvs450ORE1vx2TWzMxpIX9sX8dnc8xi1NtjM1Oo8jlwZ51+ikhQ1GPxhEFAgH8fn/867KyMh5//HH++te/JiwwIZJW2EcwqlOj2wmE9dOXV2NTPNJs5mm7+oeqHiWi6667jieffBKAmpoaZs6cyaOPPsp1113HmjVrEhqgEEkn3MCx2iB/3efj7YNdWBA/PsVDFkbrSI9XaLzwwgsB+NOf/kReXh5lZWU8+eST/OxnP+vyedasWcO0adPweDx4PB5mz57NCy+80JOQhOg/jYMZw1pKbDLr6agaJ/1hDp2oo7zaf/ryQ1CPEpHf7yc9PbZr5V//+leuv/56VFXlH/7hHygrK+vyeUaOHMlDDz3Erl27eOedd7j00ku57rrr+Oijj3oSlhD9I+wjohtENDeu06xXDYBq56Q/zOGqOg6flETUnh4vnv/ss89SXl7Oli1b+OIXvwhAZWVltwY5XnPNNcybN4/x48czYcIEHnzwQdLS0njrrbd6EpYQ/SPcEFsmtqs1Ii3WfQ+ySmNHepSIVqxYwbJlyygqKmLWrFnMnj0biNWOZsyY0aNAdF1n06ZNNDQ0xM/XWigUoq6ursVDiH5lmhCO3ZpFVBeuLt2a2eKJKCqJqF09Gln9z//8z3zhC1+goqKCs846K378sssu40tf+lK3zvXBBx8we/ZsgsEgaWlpPPPMM5xxxhntll21ahUrV67sSchCJEbED6ZBWDeJaG7cjq61ETWt3SV7m7Wvx2tW5+XlMWPGjBaLo82cOZNJkyZ16zwTJ05k9+7d7Nixg7vuuouFCxfy8ccft1t2+fLl1NbWxh/l5eU9DV+IngnWYZgmAZyYikaqo2t/y+N7m0VDfRndgNWjGlFDQwMPPfQQW7dupbKyEqPVjOLu7ATrcDgYN24cAOeccw47d+5k9erV/OpXv2pT1ul04nR2vr2vEH0qVI8JjCrIwxyd2bXGakBp7MLXdenCb0+PEtHtt9/Otm3b+PrXv05+fv7pJ/11g2EYhELyV0MkqVAdmqJQVJBH0YTsLr9MUWXb6c70KBG98MILPP/881xwwQW9evPly5dz5ZVXMmrUKOrr69m4cSOvvvoqW7Zs6dV5hegzocYOEmf35lsO86SShgezSJbOaU+PElFmZiZZWV2Y7HcalZWV8Y0YvV4v06ZNY8uWLfzjP/5jr88tRJ8I1hGK6jToTtxhvWuN1YDD4cThtIOzV1sJDlo9SkQ//OEPWbFiBb/73e9aLBfbXbLdkBhwQvUcrw/x8l4f+cYJ/vGM3K69TpVtpzvTo0T06KOPUlJSQm5uLkVFRdjt9hbPv/vuuwkJToikE6ojohuEHGmkOrs+gTVoqFTXBqg7Us2knD6Mb4DqUSKaP39+gsMQYgDQIxCqJ6ybBG2eLnfdA4QMlbIqP8fsVUya3nchDlQ9SkTf//73Ex2HEMkvcBJMk4BhI6q6SHV2/eOjxscRSfd9e3rcclZTU8Nvf/tbli9fTnV1bCmEd999l8OHDycsOCGSir8KgBrFA4qC120/zQtOaUpEprQRtatHNaL333+fuXPn4vV6OXjwIHfccQdZWVls3ryZQ4cOxdcqEmJQ8VcTNQzqlNjKEx531z8+mhbbesvUI5immdCxd4NBj2pES5cuZdGiRezfv7/Fzq7z5s3jtddeS1hwQiQVfxXhqEHQloHLrnVrtUW1sUNHMWXd6vb0qEa0c+fOdqdgjBgxgqNHj/Y6KCGSkr8Km6Zy5vgx+L1d2L2jGa1xiodmRtENk65M2h9KepSInE5nu0tw7Nu3j+zsrg97F2LAME0IVOPQVM4cWwRp3RvQq9kcKMS2nY7oRteWDxlCenRrdu211/LAAw8QicQa3hRF4dChQ9x333380z/9U0IDFCIphH0QDYOigLt7tSEANDuT8z1cMMbbtcXUhpgeJaJHH30Un89HdnY2gUCAiy++mHHjxpGens6DDz6Y6BiFsJ4/1jNcbaRQ6Y8S7e6OraqG121nmFvDpsk0j9Z6dGvm9Xp56aWX2L59O3v27MHn83H22Wczd+7cRMcnRHJo7LrfU6Wy+61D3HheIQUZ3dijTG3s6pfu+3Z1OxEZhsH69evZvHkzBw8eRFEUiouLycvLk25JMXg1dt2fNGNd91mpju69XrNT1RDCp9eSEYh0awzSUNCtOqJpmlx77bXcfvvtHD58mKlTp3LmmWdSVlbGokWLur1MrBADRqCaQFgnaPOS7rJ1v7FZtXG4JsD+ihqqG8J9E+MA1q0a0fr163nttdfYunUrl1xySYvnXn75ZebPn8+TTz7JzTffnNAghbCcvwp/WCdgz2BYWjdrQwCqDU1RUI0etC8NAd2qET311FP8+7//e5skBHDppZdy//33s2HDhoQFJ0RS0KMQrKU+GCFgzyDX4zr9a1pr3FJINaJEdBnQ2Fq3EtH777/PFVdc0eHzV155JXv27Ol1UEIklcBJTNPgZFghoropzOzBGlyqHVVVUE0dXUZWt9GtRFRdXU1ubscLQeXm5nLy5MleByVEUvFXEYoY1OBB01TyvD2oEakaqtI4oNGQW7PWutVGpOs6NlvHL9E0jagscyAGm0A1DpvK9AljyBuRg70n44Aab800M0I0KomotW4lItM0WbRoUYdb+sjuG2JQ8lehKgrZuQVkj+zeovlxatO20yZRGUvURrcS0cKFC09bRnrMxKDTOKqalGE9P4dmJ9fjJDPFjpKbmpi4BpFuJaJ169b1VRxCJCfTJFR3nIqqBpSAk9E9PY+ikuK0k+IwQfYIbUMmvQjRmXADdT4fFbUh3qzoRduOoshOHp2QRCREZwLV1AcihGypFGT1bnNEv65SWR/k0In6BAU3eFiaiFatWsV5551Heno6OTk5zJ8/n71791oZkhAt+aupC0YI2jIYkdmNSa7tqAkalBxv4NMj1QkKbvCwNBFt27aNxYsX89Zbb/HSSy8RiUT44he/SENDg5VhCREXrD9BIGIQsHkZ0Z3Z9u1QGtetNqIy16y1Hi0Dkigvvvhii6/Xr19PTk4Ou3bt4qKLLrIoKiFOqTlxDACnZ1ivV1WMbymky1i71pKqjai2thaArKzuLcMpRF/x1RwHICOr99uzqlrs774Rlcbq1iytETVnGAZLlizhggsuYMqUKe2WCYVCLQZNtrduthAJY5pEfLEF0bKy83p9ulM1Irk1ay1pEtHixYv58MMPef311zsss2rVKlauXNmPUYkhLexjUo6bYJYbZURBr0+n2hr3NpMaURtJcWt2zz338L//+7+88sorjBw5ssNyy5cvp7a2Nv4oLy/vxyjFkOOvRkHBnZ6Jy9mDNYhaOVUjkkTUmqU1ItM0+eY3v8kzzzzDq6++SnFxcaflnU5nh/PchEi4QONKEu7EtFmmuJxMyE2jeHTvxiMNRpYmosWLF7Nx40aee+450tPT45szer1e3O7edZUK0Vt7Dx6CY/VkpLvpePGbrrPbnQxLdUJa0rSIJA1Lb83WrFlDbW0tc+bMIT8/P/74wx/+YGVYQgCxrvvqhjA+NT0xJ2ya4mFI931rlt+aCZGswr4qbIAnKxH1IUC1UdUQor6ylhFFRs/WNRqk5DshRDsCoShqsAYAbwLGEAGg2Smp9LGn7AT+sJ6Ycw4SkoiEaEfVyWpUM4rTbsOR1oMtptvTbN3qiOzk0YIkIiHaUVcdm9phT80ANUF71atabEshMyqJqBVJREK0w3cyNrXDmT48cSfVYjUizYwSiUr7aHPSjyhEO9RQDaYCbm8CE5FqR1NjNaKwLm1EzUkiEqId5+aYmGRh5Pd+akecZo/v9hqWGlELcmsmRHsCJ1EUBS21Fwvmt6baGhurpY2oNakRCdGaaTab3pGgHjMAzUG+10Wm5sYxrAe7xQ5ikoiEaGVPSTmUnSDH4ybf1cN9zNpjc+Jx2fHYgJTeT6IdTCQRCdFKTXUljrBOUEtPXNc9QONSseiRWK1LURJ37gFO2oiEaCVQG+u6d3sT2D4EoDkIRnSqfEGOVMuifs1JIhKiGdM0CdWdACA1I0FTO5podqoDEfYd8/FhWWVizz3ASSISopm6QBRbqBZVgVRvdmJPriiotth6WnokdJrCQ4skIiGaOdEQwhWtxW3XUFMTv4mDanMBEA0HE37ugUwSkRDNVNWHcEXrSHFokNIHicgue5u1R3rNhGjGpdfjtIPb5QBnArvuGzXdmhlRuTVrThKREM1MG2bCyAxIGQZq4m8YNEfs1syMyK1Zc3JrJkRzgcZ96RM5oroZzS41ovZIjUiIRrphovqrUQBS+iYROV0uxmanMmJUgtbBHiQkEQnRaN+xeg6/8xFjbD7GTEpw130ju8NNTroLPAkcsT0IyK2ZEI1O+EI4QlWNNaIErkPUXNM0D+k1a0ESkRCNTtbUYNcDpDhskNpHicjmpC4Y4UhVDcGILI7WRBKREI181bENPl2eLLD10Y7CjTt5vFdayUm/1IqaSCISAvCHo9BwAgVIzczruzfSnGiagmZGCEVkcbQmliai1157jWuuuYaCggIUReHZZ5+1MhwxhB2vD5ESqcZpV7GlJ3iya3M2JzZVRTPChKKSiJpYmogaGho466yzeOKJJ6wMQwgqaoOkhY+T5rRBel/WiBxoamONKCptRE0s7b6/8sorufLKK60MQQgAslwKXns9XrcdPAlcML81mxObqkiNqJUBNY4oFAoRCp0akVpXJ4tLicSYkOqH3DRwpILT03dvZHOiqQo2IyxtRM0MqMbqVatW4fV644/CwkKrQxKDRe3h2L+egr5dwtXmPrW3mSwFEjegEtHy5cupra2NP8rLy60OSQwCVb4QgWP7Y19kjO7bN7M5yUx1MWZ4KpOz+2iIwAA0oG7NnE4nTqf88ERibd93hOEfvM+YYW7yssb07ZspCmmpqaQ5VJAdheIGVI1IiESL6gb1FQdQTINUT2afLIbWht3d+OaBvn+vAcLSGpHP5+PAgQPxr0tLS9m9ezdZWVmMGjXKwsjEUHGo2k9m7ac4NJW0wqn9ssVPRHXgD0TwHa9mRIb8noPFNaJ33nmHGTNmMGPGDACWLl3KjBkzWLFihZVhiSHkQGkpmYFDDEtzoOSf1S/vGTAdfFxRx5t7D/fL+w0EltaI5syZg2maVoYghjCf34+2/0XAJHP0lL6b6NqK3RVrHNLDfkzTRJGNFgdWY7UQCWEYcOxDjr7zV1KCx0hNScU75Yp+e3uHMxUATQ8RjBi4HbI2kSQiMbRUfwb7/4bRcIKGmloimpvMf1gA7ox+C0FzuLGpCjYjhD8clUSEJCIxlBzaASUvA6A6Ujjzwjns08YzcmT/3JLF2d3YtaZEpJPgja0HJElEYvAzzVgCKn8bExMKZqCMuQSb3cUZVsRjd2PTVOx6QBZHaySJSAxuhg57/w+OfoiJyW7HOTSo53CBzYllTcT2FOyaij0cwB+WRASSiMRgFgnCx89CdSkoKntSz+fVmlw4WM2Y7FQKMtzWxOVMJyfdiRcVR5YMrwZJRGKwqv0cPv0/8FdhqjbeS5/DtqrYzq2XTMqxLgkBOFLJTGlcRN8lXfcgiUgMNtEQfLYNjrwLpknUnsLfnXPYXRVLPBeOH870wgxrY7S5QLWBEYVwQ7/22CUrSURi8Kgqgb0vQKgegOOpE/hf/yRqAnYUBS6blMvUkYnfz77bFIWozU1DbTV1RyspLM6wOiLLSSISA56pR/B9tAV/6U5cdg1vVg5MuBzTlk/NjkN43Ha+eEYuhUnUHlNvuvi0oo4jWjmFxROsDsdykojEgKQbJqUnfOw/XIVr7zO4GmLztmyjZnLeeV8CzU4OcPW0fEYPS8VhS66FJpypsVUgI4F6meaBJCIxAB080cCreyup9QWYfPx5XKFjmJqDmuKryCs+AzR7vOz43OTcY97l9qAAtkgD/rBOqnNofxSH9tWLASeqG2z9tJI6f4gpta8ywVVLZm4OqbNuRvP04e4bCaa60nHYVBx6A/XBqCQiqwMQojtsmsrVU/M4uusvnDnsJDabB876CgygJASAOxOHTcUVraM+GCHP67I6IktJIhIDgmGYqGqsHSW3dg+5SgmoGky+Bgbi4mLuDJw2FWegjrpg1OpoLJdcLXhCtKPWH2H9Gwc5UOmDYx9BySuxJ8ZeBjmTrQ2upxprRA7dT31Dg9XRWE5qRCKpGYbJix9VUBuI8Oknexhrvh6bI1Z4XuwxUNndZHo9uGwaWpYsDiiJSCS1tz6r4khNkEyjirn6GyiKATmTYrWhAc6TmYNHqwDNb3UolpNbM5G0yqv9vH2wGmeklnnm33EpUcgcDZOu6ZdF7vucOzP2r7/K2jiSgCQikZQCYZ0tHx3FGa7hstDfyHHqkJYDZ14P2iCpyKfl4Q9HKSv7jJMNYaujsZQkIpF0DMNky0dHMWs/57yaFxjnMWML20+7EeyDqJs7PZdD1X5KSw/w2Ymh3WAtiUgkH0NnVMMeph5/nslZKponF6Z/FZxpVkeWWGl5pLsdOKM+KqtOWB2NpQZJHVcMaIYBwRrMhhMovmOoRz9gRqSG4Ih03CPOhInzwDYItxq3u0jJKoCq/fgqDmAY4+JjpYYaSUSi/zVUxXbTqD+C/2QFtVXHqPOHCEZ0pozwoikKiiMN96Q5kNc/u69axTtiIraSA6TUfcbhmkBSrRDQn5IiET3xxBP89Kc/5ejRo5x11ln8/Oc/Z+bMmVaHlZxMk8Dxz3DXH4JANSgqPi2DGvdICkaORdWS9G477I8NRjz2Af6qwxz3haj2hQlGDQAMxUbQlsFBrZCxE6bFBio2m7w6WKk5k8lKfYmIr4xPDx2hMGuc1SFZwvJE9Ic//IGlS5fyy1/+klmzZvH4449z+eWXs3fvXnJycqwOL2lEA3VU7H2Hkwfexl97gmkjvaQ4Yj++utoA5VV+ylwZpI+aRuHkmWQO72DulWFA2AfBGgjWEm6ooSYQoTqsgSuTSWPHgCMVFIW/7z9OKGKgqqCgoKkK6S4bHrednHQn6a5OEoVpxlZLrDkExz+F43vBiFJZH+TAcT91rhHUpeYRcA7HOyyPgrx8xg1PJSfdOahrQG2k55I9opjKTz8lsO81KseOJCd9EDXId5FiWrzn86xZszjvvPP4xS9+AYBhGBQWFvLNb36T+++/v9PX1tXV4fV6qa2txePx9Ee4/caIhPHVVlFztJS6w5/gr9iHrsd2fNBVOwXjzmJM8TgwDQ6W7qey9EP0aCT+eodnOBnDC/B60hnuVmNLkgbrqK4+TiAUwR+O0hCKEogY8dekO21MGeEFuxvScnihVKeKDMJaKlHViaFoqKaOakbJsEf452nDYjWdSIATJ0/iMIM4jCCqHsQINhCOhPGForjtWmyN5rQc/MOn8rv9bkbkDmNyXjqjhqXgtA3xDQZPlrFvy6+oCUQYf+5cMidcMGga5rv6GbU0EYXDYVJSUvjTn/7E/Pnz48cXLlxITU0Nzz33XIvyoVCIUCgU/7quro7CwsLTJ6LqUuo+2sInR2qbHYxdtmmCgklhZkp8QXVfKMKHh2tblIv9E/v/iAw3hZmxsoFIlN3ltSiYbc4NkO9xMXpY7L4/GImyu7ymxfueeolJjsfFmOGpYBoEA37eayzbJJKSR/qYcymafC6Znpa/qHokxOH9uzle8h7h45+BGUswDk3lnNGZ8XLvlZ8kEDEJ29II2dIJaum4HSrDHAbZmo9R7lAsOKCiNoBumPFL102TcNQgENFx2zUmNK71Y2LyzsGTRI22v0pBmwdHznguuOBiSM8HRSEU1SX5tBLc/yqRktdId8ZqmSHsvHskgKJqoCgoigooNO2BlOd1MTqr6fdKZ8/ntR2cGXLTXRQNj5UNR03eKz/ZYdnhaU7GZse2xI4aJrvK2pZVFYXzijJh6pfB1XkFoKuJyNJbsxMnTqDrOrm5uS2O5+bm8umnn7Ypv2rVKlauXNn9N4qGwFeJ4atr92kTMJ0BcDTu7BCKYvo7/sHiDIG7cQBaRIdgLR1lc9MZgVAsKShRHTN0arxIm9dEDIjE2ngcNhVUO2Z6Lq7sYrLHTGfEiFEd9qpodiejzpjFqDNm4fPVcbC0hJMnKtD1CIzNi91uOdNpGB6h3nQzLN1NscdJdrozfosHgB6BhhPQUEl+w3HwHYdIQ2xrHiMaW/Rds2PaXLFz2t1EVBc14QZqIjb8ppOQ4kSxu7G50xnmTWNkphs8p5KhJKG2XOPn4MrIh0NvQl0FkVCgxe9Vm98VrfH7DxDRMeprOjy3qbjA1VhWNzDqO05E4AR34x85o/2yigL4ovE/dolgaY3oyJEjjBgxgjfeeIPZs2fHj997771s27aNHTt2tCjf4xpRuIFQzVFO+MKAcqoJollbRLrbTnrj4lThqEG1/9RtTvNyiqKQ4rDFyioKEd3gZGPZ5st9Ko1/ulwOlTRnbOsY3TTjZU+d9tRrnA4baQ5b7JgjBVNzoqhJ2vjcAdM0MU2GbDd0QkRDRPw11NY3oOs6hmlgGgbNP6npLg1PYxtdxDCprAt1cDJIdWpkuGNlo4bJsU7KpjjU+FZHhgkVtcE2ZRQFCrwu8I48bYfCgKgRDR8+HE3TOHbsWIvjx44dIy+vbWOr0+nE6ezBeBJHKs6csYzoYtu3A+jqMlt2ICfztMUA0IDh3WjKGogfZUVRhlRbc5+wObF7crv8u2IHRgzv4qm7UVYFRgzrWtnesvTPrcPh4JxzzmHr1q3xY4ZhsHXr1hY1JCHE4GZ59/3SpUtZuHAh5557LjNnzuTxxx+noaGBW265xerQhBD9xPJEdOONN3L8+HFWrFjB0aNHmT59Oi+++GKbBmwhxOBl+Tii3hjM44iEGAy6+hkdWF0yQohBSRKREMJykoiEEJazvLG6N5qat+rq2h8xLYSwVtNn83RN0QM6EdXX1wNQWFhocSRCiM7U19fj9Xo7fH5A95oZhsGRI0dIT09vMb0imTVNSykvLx9UPX2D8boG4zVB/16XaZrU19dTUFCA2sl0pQFdI1JVlZEjR1odRo94PJ5B9cvdZDBe12C8Jui/6+qsJtREGquFEJaTRCSEsJwkon7mdDr5/ve/37NVBJLYYLyuwXhNkJzXNaAbq4UQg4PUiIQQlpNEJISwnCQiIYTlJBEJISwniaiPPfjgg5x//vmkpKSQkZHRpdeYpsmKFSvIz8/H7XYzd+5c9u/f37eBdlN1dTULFizA4/GQkZHBbbfdhs/n6/Q1c+bMaVzT+tTjX/7lX/op4vY98cQTFBUV4XK5mDVrFm+//Xan5f/4xz8yadIkXC4XU6dO5f/+7//6KdLu6c51rV+/vs3PxeXq500eTdGnVqxYYT722GPm0qVLTa/X26XXPPTQQ6bX6zWfffZZc8+ePea1115rFhcXm4FAoG+D7YYrrrjCPOuss8y33nrL/Pvf/26OGzfOvOmmmzp9zcUXX2zecccdZkVFRfxRW1vbTxG3tWnTJtPhcJhr1641P/roI/OOO+4wMzIyzGPHjrVbfvv27aamaeZPfvIT8+OPPza/+93vmna73fzggw/6OfLOdfe61q1bZ3o8nhY/l6NHj/ZrzJKI+sm6deu6lIgMwzDz8vLMn/70p/FjNTU1ptPpNJ966qk+jLDrPv74YxMwd+7cGT/2wgsvmIqimIcPH+7wdRdffLH57W9/ux8i7JqZM2eaixcvjn+t67pZUFBgrlq1qt3yX/7yl82rrrqqxbFZs2aZ3/jGN/o0zu7q7nV19XezL8mtWZIpLS3l6NGjzJ07N37M6/Uya9Ys3nzzTQsjO+XNN98kIyODc889N35s7ty5qKraZi+61jZs2MDw4cOZMmUKy5cvx+/393W47QqHw+zatavF91lVVebOndvh9/nNN99sUR7g8ssvT5qfC/TsugB8Ph+jR4+msLCQ6667jo8++qg/wo0b0JNeB6OjR48CtLv7bdNzVjt69Cg5OS03ibPZbGRlZXUa41e/+lVGjx5NQUEB77//Pvfddx979+5l8+bNfR1yG93dZRhi153MPxfo2XVNnDiRtWvXMm3aNGpra3nkkUc4//zz+eijj/ptUrnUiHrg/vvvb9O41/rR0Q89mfX1dd15551cfvnlTJ06lQULFvDkk0/yzDPPUFJSksCrEN01e/Zsbr75ZqZPn87FF1/M5s2byc7O5le/+lW/xSA1oh7413/9VxYtWtRpmTFjxvTo3E073B47doz8/Pz48WPHjjF9+vQenbOrunpdeXl5VFZWtjgejUaprq5ud4fejsyaNQuAAwcOMHbs2G7H2xvd3WUYYj+b7pS3Qk+uqzW73c6MGTM4cOBAX4TYLklEPZCdnU12dnafnLu4uJi8vDy2bt0aTzx1dXXs2LGDu+66q0/es0lXr2v27NnU1NSwa9cuzjnnHABefvllDMOIJ5eu2L17N0CLhNtfmu8yPH/+fODULsP33HNPu6+ZPXs2W7duZcmSJfFjL730UlLtStyT62pN13U++OAD5s2b14eRtmJpU/kQUFZWZr733nvmypUrzbS0NPO9994z33vvPbO+vj5eZuLEiebmzZvjXz/00ENmRkaG+dxzz5nvv/++ed111yVl9/2MGTPMHTt2mK+//ro5fvz4Ft33n3/+uTlx4kRzx44dpmma5oEDB8wHHnjAfOedd8zS0lLzueeeM8eMGWNedNFFVl2CuWnTJtPpdJrr1683P/74Y/POO+80MzIy4l3XX//61837778/Xn779u2mzWYzH3nkEfOTTz4xv//97ydt9313rmvlypXmli1bzJKSEnPXrl3mV77yFdPlcpkfffRRv8UsiaiPLVy40ATaPF555ZV4GcBct25d/GvDMMzvfe97Zm5urul0Os3LLrvM3Lt3b/8H34mqqirzpptuMtPS0kyPx2PecsstLZJraWlpi+s8dOiQedFFF5lZWVmm0+k0x40bZ/7bv/2bpeOITNM0f/7zn5ujRo0yHQ6HOXPmTPOtt96KP3fxxRebCxcubFH+6aefNidMmGA6HA7zzDPPNJ9//vl+jrhrunNdS5YsiZfNzc01582bZ7777rv9Gq8sAyKEsJz0mgkhLCeJSAhhOUlEQgjLSSISQlhOEpEQwnKSiIQQlpNEJISwnCQiMaCsX7++xUqXP/jBD1rMwVu0aFF8aoMYOCQRiXYtWrSow6VcFy9ejKIoLSbI9kUCKCoq4vHHH29x7MYbb2Tfvn0dvmb16tWsX78+/vWcOXNazA0TyUkSkehQYWEhmzZtIhAIxI8Fg0E2btzIqFGjLInJ7Xa3WQupOa/X2+W1wUXykEQkOnT22WdTWFjYYuGyzZs3M2rUKGbMmNGrc7dXU5k/f368ljVnzhzKysr4zne+E18LCdremrXWvGa2aNEitm3bxurVq+PnKC0tZdy4cTzyyCMtXrd7924URenXpS/EKZKIRKduvfVW1q1bF/967dq13HLLLX3+vps3b2bkyJE88MADVFRUUFFR0e1zrF69mtmzZ3PHHXfEzzFq1Kg21wSwbt06LrroIsaNG5eoSxDdIIlIdOprX/sar7/+OmVlZZSVlbF9+3a+9rWv9fn7ZmVloWka6enp5OXl9WjxMa/Xi8PhICUlJX4OTdNYtGgRe/fujW+xE4lE2LhxI7feemuiL0N0kSyMJjqVnZ3NVVddxfr16zFNk6uuuorhw4dbHVavFBQUcNVVV7F27VpmzpzJX/7yF0KhEDfccIPVoQ1ZUiMSp3Xrrbeyfv16fve73yWs1qCqKq1XoIlEIgk5d1fcfvvt8Yb4devWceONN5KSktJv7y9akkQkTuuKK64gHA4TiUS4/PLLE3LO7OzsFu0+uq7z4YcftijjcDjQdb1X79PROebNm0dqaipr1qzhxRdflNsyi8mtmTgtTdP45JNP4v/vSG1tbXwd6ibDhg2jsLCwTdlLL72UpUuX8vzzzzN27Fgee+wxampqWpQpKiritdde4ytf+QpOp7NHt4RFRUXs2LGDgwcPkpaWRlZWFqqqxtuKli9fzvjx45Nq3emhSGpEoks8Hg8ej6fTMq+++iozZsxo8Vi5cmW7ZW+99VYWLlzIzTffzMUXX8yYMWO45JJLWpR54IEHOHjwIGPHju3xZgXLli1D0zTOOOMMsrOzOXToUPy52267jXA43C+9gKJzslSsGLL+/ve/c9lll1FeXt5mQ0LRvyQRiSEnFApx/PhxFi5cSF5eHhs2bLA6pCFPbs3EkPPUU08xevRoampq+MlPfmJ1OAKpEQkhkoDUiIQQlpNEJISwnCQiIYTlJBEJISwniUgIYTlJREIIy0kiEkJYThKREMJykoiEEJb7/xokHcxhJSB/AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:30:58.206690Z", + "iopub.status.busy": "2024-03-22T20:30:58.206379Z", + "iopub.status.idle": "2024-03-22T20:30:58.426483Z", + "shell.execute_reply": "2024-03-22T20:30:58.425592Z" + }, + "papermill": { + "duration": 0.242577, + "end_time": "2024-03-22T20:30:58.428444", + "exception": false, + "start_time": "2024-03-22T20:30:58.185867", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEqCAYAAABqVvf5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA4NUlEQVR4nO3deVgT194H8G8CEnYR2VTQgKjgVaEibreuCC6VgntdqlKrrb1oK+BCn1brtS3Vgnpdqm97q9aFalVEX+qGCparKLjUouKGgMgiIkpYA4Tz/uGbuUQCJAqEZH6f5+HRmTmT/IYhv5yZc+YcAWOMgRBCdJxQ0wEQQkhLoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QWtS3Zbt26FWCyGoaEhBgwYgKSkpHrL/vTTTxgyZAjatWuHdu3aYdSoUQ2WJ4ToLq1KdgcOHEBQUBBWrVqFa9euwc3NDaNHj0Z+fr7S8vHx8Zg+fTri4uKQmJgIBwcH+Pj4IDs7u4UjJ4RomkCbBgIYMGAAPD09sWXLFgBATU0NHBwcsGjRIqxYsaLR/WUyGdq1a4ctW7Zg9uzZKr1nTU0NcnJyYGZmBoFA8EbxE0KaFmMMxcXF6NixI4TChutu+i0U0xurrKzE1atXERoayq0TCoUYNWoUEhMTVXqNsrIyVFVVwdLSst4yUqkUUqmUW87OzkbPnj1fP3BCSLPLysqCvb19g2W0JtkVFBRAJpPB1tZWYb2trS3u3Lmj0mssX74cHTt2xKhRo+otExYWhtWrV9dZ/+9//xvGxsbqBU0IaVZlZWX48MMPYWZm1mhZrUl2b+q7777D/v37ER8fD0NDw3rLhYaGIigoiFuWSCRwcHCAv78/zM3NWyJUjaiqqkJsbCy8vb3Rpk0bTYdD3hBfzqdEIsGHH36o0i0mrUl2VlZW0NPTw5MnTxTWP3nyBHZ2dg3uGx4eju+++w5nzpxBnz59GiwrEokgEonqrG/Tpo1O/9HI8eU4+ULXz6c6x6Y1rbEGBgbw8PDA2bNnuXU1NTU4e/YsBg0aVO9+69atw5o1a3Dy5En069evJUIlhLRCWlOzA4CgoCDMmTMH/fr1Q//+/bFx40aUlpYiICAAADB79mx06tQJYWFhAIC1a9di5cqViIyMhFgsRl5eHgDA1NQUpqamGjsOQkjL06pkN23aNDx9+hQrV65EXl4e3N3dcfLkSa7R4tGjRwrNz9u2bUNlZSUmT56s8DqrVq3CV1991ZKhE0I0TKuSHQAEBgYiMDBQ6bb4+HiF5YyMjOYPiBCiFbTmnh0hhLwJravZkaZRVlam0D+xpFyKiylpaGd1BaZG/22NdnFxof6FWkDV8wnw95xSsuOpO3fuwMPDo876da8sX716FX379m2ZoMhrU/V8Avw9p5TseMrFxQVXr17llu/mvkDQwRSsn9IbPTpYKJQjrZ+q51Nelo8o2fGUsbGxwre7MPMZRAnlcO3lBvcu7TUYGXkddD4bR8mOJ9ILSlEqra53e9rTUu5fff36/yxMRPpwtDJp8vgIaW6U7HggvaAUI8LjVSobfCil0TJxIcMp4bUCDX2B0ZdXXZTseED+gdg4zR3ONsqfHCktlyImPhHjhw+CiVHdZ4MB4EF+CT478GeDNUTSMlT9AqMvr/+iZMcjzjam6NWprdJtVVVVyLMG+nZpp9MPjuuKxr7A6MurLkp2hGix+r7A6MurLnqCghDCC1Sz4wGprAJCw2ykS+5CaKj8nl11dTVyqnOQWpha7w3tdEkJhIbZkMoqACi/HCaktaJkxwM5pZkwcdyMz1WYRfKHkz80uN3EEcgpdYcHbBssR5pXY19g9OVVFyU7Huho0gWl6Yvwr2nu6FpPa2x1dTUu/OcC/v723+v9cKTll+DTA3+i44guzRkuUYGqX2D05fVflOx4QKRniJqKTnA074Ge7etvjU3XT4erpWu9N7RrKopQU/EUIr365/AgLaOxLzD68qqLkh0hWqixLzD68qqLWmMJIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC1qX7LZu3QqxWAxDQ0MMGDAASUkNj0t98OBBuLi4wNDQEL1798bx48dbKFJCmk95lQwAcDO7SOnPtcznuPIUuJb5vN4yD/JLNHwULUurRio+cOAAgoKCsH37dgwYMAAbN27E6NGjcffuXdjY2NQpf/HiRUyfPh1hYWEYP348IiMj4e/vj2vXrqFXr14aOALNqP3BqE9puRRXngJ2mc8bnFSZtA5p/38uVkSlNFBKH3seJDf6WiYirUoDr03AGGOaDkJVAwYMgKenJ7Zs2QIAqKmpgYODAxYtWoQVK1bUKT9t2jSUlpYiJiaGWzdw4EC4u7tj+/btKr2nRCJB27ZtUVRUBHNz86Y5kBa2P+lRIx8K9cSFDIejlUmTvR5RX2FpJU7fykNXG1MYtdGrs/1ubhGCD6UgYnJv9OhQ/8xhJiJ9rT6X6nw+tSalV1ZW4urVqwgNDeXWCYVCjBo1ComJiUr3SUxMRFBQkMK60aNHIzo6ut73kUqlkEql3LJEIgHwckz/qqqqNzgCzRnRvT2+8esJJ2sTpR8MALiXV4RlR1KxboIruts19OHQg31bA639XegKMwMBJr3Vod7tFRUVAIAu7UToYWPc4Gtp87lUJ3atSXYFBQWQyWSwtVWc8s3W1hZ37txRuk9eXp7S8nl5efW+T1hYGFavXl1n/enTp2Fs3PAfTWtmCiA/v/7tL6+K9JH/IAWi+n89AIDbTRgXaR5Z/38+L126hOybmo6m+ZSVlalcVmuSXUsJDQ1VqA1KJBI4ODjAx8dHay9jVXHjUSGQcgUDBw6EW2dLTYdD3hBfzqf8yksVWpPsrKysoKenhydPniisf/LkCezs7JTuY2dnp1Z5ABCJRBCJ6t6gb9OmTb1T0ukC+dyi+vr6On2cfMGX86nOsWlN1xMDAwN4eHjg7Nmz3LqamhqcPXsWgwYNUrrPoEGDFMoDQGxsbL3lCSG6S2tqdgAQFBSEOXPmoF+/fujfvz82btyI0tJSBAQEAABmz56NTp06ISwsDADw6aefYtiwYYiIiMA777yD/fv348qVK/jxxx81eRiEEA3QqmQ3bdo0PH36FCtXrkReXh7c3d1x8uRJrhHi0aNHEAr/W1kdPHgwIiMj8cUXX+Dzzz9Ht27dEB0dzas+doSQl7Qq2QFAYGAgAgMDlW6Lj4+vs27KlCmYMmVKM0dFCGnttOaeHSGEvAlKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEeIjikvL0fYl8vw5MCXCPtyGcrLyzUdUqtAyY4QHeLv7w9jY2P8tmcHKjKu47c9O2BsbAx/f39Nh6ZxlOwI0RH+/v44evSo0m1Hjx7lfcKjZEcgk8lwJfE/KL19HlcS/wOZTKbpkIiaysvL6010ckePHuX1JS0lO56LioqCo6Mj5k/3R8H/fo/50/3h6OiIqKgoTYdG1LB48WKVys2ePRtlZWXNHE3rRMmOp8rKyhAeHo5JkyYhKytLYVtWVhYmTZqE8PBw3n4wtE1CQoJK5Q4dOoQ7d+40czStEyU7nrp16xaWLl3aYJmlS5fi1q1bLRQReRPFxcUqlbO2toaLi0szR9M6UbLjqVdrc29ajmiWoaGhSuXMzMxgbGzczNG0TlqT7AoLCzFz5kyYm5vDwsIC8+bNQ0lJSYPlFy1ahB49esDIyAidO3fG4sWLUVRU1IJRt15r1qzh/m9lZYXJkydj5MiRmDx5MqysrJSWI61XRUVFk5bTRfqaDkBVM2fORG5uLmJjY1FVVYWAgAAsWLAAkZGRSsvn5OQgJycH4eHh6NmzJzIzM/Hxxx8jJycHhw4dauHoW5+0tDQAgEAgwPPnzxV+J3p6ehAIBGCMceVI60bJrnECxhhTZ4eHDx/CycmpueJRKjU1FT179kRycjL69esHADh58iTGjRuHx48fo2PHjiq9zsGDBzFr1iyUlpZCX195npdKpZBKpdyyRCKBg4MDCgoKYG5u/uYH00pYWFio1PhgbGyMFy9eNH9A5I3Y2tri+fPnjZZr164dnjx50gIRtQyJRAIrKysUFRU1+vlUu2bn7OyMYcOGYd68eZg8ebLK9wreRGJiIiwsLLhEBwCjRo2CUCjE5cuXMWHCBJVeR/4LqS/RAUBYWBhWr15dZ/3p06d16l6HqsnOwsICx48fb4GIyJtQtdW8rKxMp86nOr0F1E52165dw86dOxEUFITAwEBMmzYN8+bNQ//+/dV9KZXl5eXBxsZGYZ2+vj4sLS2Rl5en0msUFBRgzZo1WLBgQYPlQkNDERQUxC3La3Y+Pj46VbPr3bs3cnJyVCo3bty4FoiIvInaVyONldOl8ymRSFQuq3ayc3d3x7/+9S9ERETg2LFj2LVrF95++210794dH3zwAd5//31YW1ur9ForVqzA2rVrGyyTmpqqboh1SCQSvPPOO+jZsye++uqrBsuKRCKIRKI669u0aYM2bdq8cSythap/JBKJRKeOm0Cnzqc6x/LarbH6+vqYOHEiDh48iLVr1+LBgwcICQmBg4MDZs+ejdzc3EZfIzg4GKmpqQ3+ODk5wc7ODvn5+Qr7VldXo7CwEHZ2dg2+R3FxMcaMGQMzMzMcOXJEp070m1C1X5aq5YhmCQSCJi2ni167NfbKlSvYsWMH9u/fDxMTE4SEhGDevHl4/PgxVq9eDT8/PyQlJTX4GtbW1irVAgcNGoQXL17g6tWr8PDwAACcO3cONTU1GDBgQL37SSQSjB49GiKRCMeOHWuR+4vawtXVFTdv3lSpHGn9XFxcVLoK4muHYuA1anbr169H7969MXjwYOTk5GD37t3IzMzE119/DUdHRwwZMgS7du3CtWvXmixIV1dXjBkzBvPnz0dSUhIuXLiAwMBAvPfee1xLbHZ2NlxcXLgEK5FI4OPjg9LSUvz888+QSCTIy8tDXl4ePegOwNTUtEnLEc0SClX7KKtaTicxNTk7O7Nvv/2W5eTk1FtGKpWyXbt2qfvSDXr27BmbPn06MzU1Zebm5iwgIIAVFxdz29PT0xkAFhcXxxhjLC4ujgFQ+pOenq7y+xYVFTEArKioqEmPR9McHBzq/f3U/nFwcNB0qEQFXl5eKp1PLy8vTYfapNT5fKp9GRsbG4vOnTvX+YZgjCErKwudO3eGgYEB5syZo+5LN8jS0rLeDsQAIBaLwWp1GRw+fLjCMlGkSkusOuWIZnXt2hVnz55VqRxfqV2n7dq1KwoKCuqsLywshKOjY5MERZqfqpfydMmvHVT9YudzBUDtZFffL6ukpIQaALQItd7ployMjCYtp4tUvoyVd7QVCARYuXKlwtMEMpkMly9fhru7e5MHSJqH/NlXVcqR1k/VLlV87nqlcrK7fv06gJc1u5SUFBgYGHDbDAwM4ObmhpCQkKaPkDQLkUik0hDdyjpYk9bn8ePH3P8NDQ0VHvivvVy7HN+onOzi4uIAAAEBAfjXv/6lU49O8VH79u1V+sNv3759C0RD3lR6ejr3/1dHNqm9XLsc36h9z27nzp2U6HRAt27dmrQcIa2dSjW7iRMnYteuXTA3N8fEiRMbLEsTtWiH2gN0NkU5olmOjo7466+/VCrHVyolu7Zt23I3qtu2bdusAZGWcf/+/SYtRzRL1dsNfL4toVKy27lzp9L/E+1Fc1DoFnWGeOIrHj8ox2/Uz063GBkZNWk5XaRSze6tt95S+Y++KQcAIM2nU6dOSp+EUVaOtH5ubm4qPS7m5ubWAtG0TiolO39//2YOg7Q0Ve+90j1a7UD3YBunUrJbtWpVc8dBWpgqtTp1yhHNUqWDuDrldBHds+MpenBct9Qed/DVp15qP7PO5/EJVUp2lpaW3Dd8u3btYGlpWe8P0Q61b1S/etO6oW2kdZLfatLX168z+re1tTU3ox6fb0mpdBm7YcMGmJmZcf+nFjrtZ29vzzUmNfR4kb29fYvGRV5Ply5dALycm+XVxwBrdx+Sl+MjlZJd7YE4586d21yxkBY0dOhQHDt2DEDdS9Xay0OHDm3RuMjrGTJkCKytrfH06dN6y9jY2GDIkCEtGFXrovY9Oz09vTozfQHAs2fPoKen1yRBkea3aNGiRucjEAqFWLRoUQtFRN6UvMOwpaUlOnToABMTE3To0IG7vfRqDZ5vmmzwTqlUqjDsE2ndDAwMEBwcDKDuJCzy5eDgYDqnWiI+Ph4SiQSmpqYoLCxEbm4uSktLkZubi8LCQpiamkIikSA+Pl7ToWqMykM8bdq0CcDLHvX//ve/FVp1ZDIZ/vjjD15P06aN1q1bB+DljHG1CYVCBAcHc9tJ6ydPYiUlJTAwMMDEiRNhZGSE8vJyREVFoaSkhCvn5eWlwUg1R+Vkt2HDBgAva3bbt29XuGQ1MDCAWCzG9u3bmz5C0qzWrVuHr7/+Gps3b8a5c+cwcuRILFq0iGp0WqayshLAy5GIi4uLIRAIcPz4cYwbNw6//PILTE1NUVVVxZXjI5WTnXzQvxEjRiAqKgrt2rVrtqBIy9LT04ObmxtycnLg5uZG9161kHyCbBsbG+jr6ytMlCTvjpKTk6PSRNq6Su17dnFxcZTodEhUVBS6du0Kb29vrF+/Ht7e3ujatSuNS6hl5I0P2dnZ8PPzw6VLl1BeXo5Lly7Bz8+PmxKTz40Uas8b+8EHHzS4fceOHa8dDGlZUVFRmDRpUp2Ow/n5+Zg0aRIOHz7c6GCtpHXo3r07YmNjAQBnzpxBTEwMt632+e3evXuLx9ZaCJiazwNNmDBBYbmqqgo3b97EixcvMHLkSJ2rEUgkErRt2xZFRUU6NRy9TCZDhw4d8PTpU+5Gtpx82cbGBjk5OXRZqwXKy8thbGwMPT09MMZQU1PDbRMKhRAIBJDJZCgrK9Opp2LU+XyqXbM7cuRInXU1NTVYuHAhr2cb1zbx8fFcB9T6OhXn5+fzuvVOmxgZGcHT0xPJyckAgL59+3JfWvInZTw9PXUq0alL7WSnjFAoRFBQEIYPH45ly5Y1xUuSZnbu3Dnu/15eXvDx8cH9+/fRrVs3nD59Gr///jtXjpJd6yeTyfD06VOulvPquJJt27ZFQUEBZDIZb2vqTZLsACAtLQ3V1dVN9XKkmWVmZgIAHBwckJKSwiU3AOjcuTMcHByQlZXFlSOtW0JCAjIyMiAQCDB27FgUFxcjMzMTXbp0gZmZGU6ePImioiIkJCRg+PDhmg5XI9ROdkFBQQrLjDHk5ubi999/V3iGlmiHrKysOkMCPXnyhNdzFWij7OxsAIC7uztSUlK4wQCysrJgb28Pd3d3XL9+nSvHR2onu+vXryssC4VCWFtbIyIiotGWWtJ6ODg4cP9/NbHVXq5djrRe8vuvr34+AeDx48dc8mtooABdp3ayi4uLa444SAtTdexBGqNQO9BUio1rsnt2RLvUHm69vq4nr5YjrZe80zDw8imKGTNmoKysDMbGxoiMjORGKqpdjm+0Zlj2wsJCzJw5E+bm5rCwsMC8efO4h5sbwxjD2LFjIRAIEB0d3byBaonalzsNdbVUdllEWp/Tp08DePloWEFBATZu3Igff/wRGzduxLNnz7iRiuXl+EhranYzZ85Ebm4uYmNjUVVVhYCAACxYsACRkZGN7rtx40YaXfkVxsbGAICuXbtCJpMhIyOD22ZnZwehUIi0tDSuHGnd5PfkqqurYWtrixkzZqC0tBQmJiaIjIzEkydPFMrxkVYku9TUVJw8eRLJycno168fAGDz5s0YN24cwsPD0bFjx3r3/fPPPxEREYErV66gQ4cOjb6XVCpVuEEvkUgAvHxSpKqq6g2PpPUYPHgwjh49irS0NIwdOxaLFy/Gw4cP4eTkhNjYWJw4cYIrp0vHras6deqEe/fuwcDAAIaGhtwoRQAgFothYGCAyspKdOrUSafOpzrH0mTJ7vHjx/jnP/+JH3/8salekpOYmAgLCwsu0QHAqFGjIBQKcfny5TqPsMmVlZVhxowZ2Lp1K+zs7FR6r7CwMKxevbrO+tOnT+tULcfR0RECgQCMMZw5c4ZLbsDLYYKAl2MXOjo64vjx45oKk6hIPldIZWUlLC0t4ePjA5FIBKlUiitXrnA1d3t7e506n2VlZSqXbbJk9+zZM/z888/Nkuzy8vJgY2OjsE5fXx+WlpbIy8urd78lS5Zg8ODB8PPzU/m9QkNDFfoSSiQSODg4wMfHR6eejQVe/n7Wr1+vMBwQAG55yZIlvJ6NSpu8ePECe/bsAfDyPmt991q9vLwwbty4lgytWcmvvFSh0cvYFStWYO3atQ2Wed3xt44dO4Zz586pfYNdJBLV6WQLvKztyGs8uiIiIgJ6eno0UrEOUHXWsC5duujU37E6x6LRZBccHNzobGVOTk6ws7OrM8lPdXU1CgsL6708PXfuHNLS0mBhYaGwftKkSRgyZAivx+KvjUYq1g1DhgyBWCyGnp4eMjIyFGrrenp6EIvFqKmp4fXsYhpNdtbW1nUm9FVm0KBBePHiBa5evQoPDw8AL5NZTU0NBgwYoHSfFStW4MMPP1RY17t3b2zYsAG+vr5vHrwOMTAwwOLFi+Hs7Ixx48bp1Dc/X+jp6WHKlCn4/vvvYWNjg5kzZ3Ktsfv27UNaWhqWLl3K20EAADXGs2tsEMcXL17g/Pnzde7/NJWxY8fiyZMn2L59O9f1pF+/flzXk+zsbHh5eWH37t3o37+/0tcQCAQ4cuSIWvehdHU8u1dVVVVxcxZQstM+MpkMzs7OsLKyQkFBgUJXIkdHR7Rv3x7Pnj3D/fv3dSrhNct4dm3btm10++zZs1V9ObXt27cPgYGB8PLyglAoxKRJk7gZz4CXH9a7d++q1TpDiK6Qj3ry66+/wtPTE3FxcThx4gTGjh2LESNGICkpCYMHD6ZRT1Sxc+fO5oyjUZaWlg12IBaLxQ0+CQA0/KQAIdosNzcXANCrVy/o6elh2LBhKC0txbBhw6Cnp4devXoplOMjrXlcjBBSP3mH+Zs3byrdLl+vSsd6XaVyzU7V4Ztowh1CWp68Nfbbb7+t8/x3TU0NwsLC4OjoSK2xqti1axe6dOmCt956iy4HCWll9PT0EBERgcmTJ8Pf3x9Lly7lplL8/vvvERMTg0OHDulU44S6VE52CxcuxK+//or09HQEBARg1qxZNNYZIa3IxIkTcejQIQQHB2Po0KHcekdHRxw6dIj302KqfM9u69atyM3NxbJly/C///u/cHBwwNSpU3Hq1Cmq6RHSSkycOBEPHjxAbGwsgoKCEBsbi/v37/M+0QFqNlCIRCJMnz4dsbGxuH37Nv72t7/hk08+gVgsVnlsOUJI85K3xg4dOpRrjSVv0Born3iXMdZsHYkJIaSpqJXspFIpfv31V3h7e6N79+5ISUnBli1b8OjRI5iamjZXjIQQNchkMpw/fx5//PFHsz7VpG1UbqD45JNPsH//fjg4OOCDDz7Ar7/+Cisrq+aMjRCipqioKAQHB3OPi61fvx5isRgRERG8v2+ncrLbvn07OnfuDCcnJ5w/fx7nz59XWi4qKqrJgiOEqC4qKgqTJ0/G+PHjsWfPHjx+/Bj29vZYt24dJk+ezPsWWZWT3ezZs2keB0JaKZlMhuDgYIwfPx7R0dGQyWR49uwZBgwYgOjoaPj7+yMkJAR+fn68bbBQq1MxIaR1qj0QgFAoVLhPJxQKERoayvuBAOjZWEJ0QO2BAJQ1UNBAAFoyuxghpGHyB/y3bNmC//mf/6nTQLFgwQKFcnxENTtCdMCQIUNgbW2N0NBQ9OrVCwkJCfj111+RkJCAXr164fPPP4eNjQ2vBwKgZEeIjqjdgCh/hJMe5fwvSnaE6ICEhATk5+cjLCwMN2/exNChQzF9+nQMHToUt27dwrfffov8/HwkJCRoOlSNoWRHiA6QNzwEBgYqHQggMDBQoRwfUbIjRAfUHqlY2UAANFIxJTsCoLKyEps2bcKPP/6ITZs2obKyUtMhETXVHqm4pqZGYRuNVPwSJTueW7ZsGUxMTBASEoLjx48jJCQEJiYmWLZsmaZDI2qQj1QcExMDf39/XLp0iRup2N/fHzExMQgPD+ft0xMAAEYaVFRUxACwoqIiTYfS5JYuXcoAMFtbW7Z9+3a2c+dOtn37dmZra8sAsKVLl2o6RKKmw4cPM7FYzABwP46Ojuzw4cOaDq1ZqPP5VHmSbL7S1UmyKysrYWJigvbt2+Px48dgjHGTZAsEAtjb2+PZs2coLS2FgYGBpsMlapDJZHXmjdXVGp06n0+6jOWpH374AdXV1fj6668hEAgUHi8SCAT45z//ierqavzwww+aDpWoiUYqVo6SHU+lpaUBeNkR1dnZGd7e3li/fj28vb3h7OwMoVCoUI4QbUfJjqe6du0KAPjwww/Ru3dvhceLevfujfnz5yuUI0TbUbLjqY8++ggAYGBggAMHDqCiogLJycmoqKjAgQMHuPt08nKEaDsa9YSnLl++DOBlQ4WpqSnXN2v9+vUQCoXc8uXLl3k7/hnRLVSz46najw292iBfe5nPjxcR3ULJjqdsbGwAAK6urnBwcFDY5uDgABcXF4VyhGg7SnY8l5qaij59+ig0UPTp0wd37tzRdGiENCmtSXaFhYWYOXMmzM3NYWFhgXnz5qGkpKTR/RITEzFy5EiYmJjA3NwcQ4cORXl5eQtE3Lrl5eUpLLN6xj97tRwh2kprkt3MmTNx69YtxMbGIiYmBn/88Qc31HR9EhMTMWbMGPj4+CApKQnJyckIDAzk+pDx2dOnTwEACxcuVDr+2ccff6xQjhBtpxWtsampqTh58iSSk5PRr18/AMDmzZsxbtw4hIeHo2PHjkr3W7JkCRYvXowVK1Zw63r06NEiMbd21tbWAICMjAzcu3cP58+f5x4vGjZsGPz8/BTKEaLttCLZJSYmwsLCgkt0ADBq1CgIhUJcvnwZEyZMqLNPfn4+Ll++jJkzZ2Lw4MFIS0uDi4sLvvnmG7z99tv1vpdUKoVUKuWWJRIJAKCqqgpVVVVNeFSaZWtrCwA4efIkJkyYgODgYHh6ekJPTw8TJkzAyZMnuXK6dNx8IT9nun7u1Dk+rUh2eXl5dVoF9fX1YWlpWe89pYcPHwIAvvrqK4SHh8Pd3R27d++Gl5cXbt68iW7duindLywsDKtXr66z/vTp0zA2Nn7DI2k9ZDIZbGxsYGZmhuTkZIwcOZLbZmtrCycnJ5SUlEAikeD48eMajJSoSyaT4fbt23j+/DlSUlLQs2dPnX0+tqysTOWyGk12K1aswNq1axssk5qa+lqvLe8U+9FHHyEgIAAA8NZbb+Hs2bPYsWMHwsLClO4XGhqKoKAgblkikcDBwQE+Pj46NeoJAFRXV+O9997D2LFjMWrUKDx8+BBOTk44c+YMTpw4gf3798PX11fTYRI1HDlyBMuXL+emUgQAsViMtWvXKr0C0nbyKy9VaDTZBQcHY+7cuQ2WcXJygp2dHfLz8xXWV1dXo7CwEHZ2dkr3kw8/3bNnT4X1rq6uePToUb3vJxKJIBKJ6qxv06YN2rRp02Cs2mbq1KnQ19dHcHCwQu3N0dERhw4dwsSJEzUYHVFXVFQU3nvvPYwfPx579uzB48ePYW9vj3Xr1uG9997TyXOq1meyOQfWayq3b99mANiVK1e4dadOnWICgYBlZ2cr3aempoZ17NiRffHFFwrr3d3dWWhoqMrvrcuDd8pVV1ez2NhYFhQUxGJjY1l1dbWmQyJqqq6uZmKxmPn6+jKZTMYqKytZdHQ0q6ysZDKZjPn6+jJHR0edO7fqfD61ItkxxtiYMWPYW2+9xS5fvsz+85//sG7durHp06dz2x8/fsx69OjBLl++zK3bsGEDMzc3ZwcPHmT3799nX3zxBTM0NGQPHjxQ+X35kOwYYwofDqJ94uLiGACWmJjIGKt7Pi9evMgAsLi4OA1G2fTU+XxqRQMFAOzbtw+BgYHw8vKCUCjEpEmTsGnTJm57VVUV7t69q3DD8rPPPkNFRQWWLFmCwsJCuLm5ITY2loYtekVJSQlmzJiBv/76Cz///DMiIyNhamqq6bCIGuTPMPfq1Uvpdvl6Pj/rrDXJztLSEpGRkfVuF4vFSmc/X7FihUI/O6Kof//+SE5O5pYzMzNhZmYGT09PJCUlaTAyoo7aUykOHDiwznaaSlGLnqAgTU+e6AQCAWbNmoUNGzZg1qxZEAgESE5ORv/+/TUdIlFR7akUq6qqFIbZr6qqoqkUAdCEO43Q1Ql3SkpKYGZmBoFAgLKyMujp6XET7shkMhgbG4MxhuLiYrqk1RJRUVGYPHkyRCIRKioquPWGhoaQSqU62RpLE+6QRr3//vsAgFmzZsHQ0FBhm6GhIWbMmKFQjmgHxpjCE0DAy6eCqE5DyY635BPphISEKN0u71hNE+5oB5lMxg3eoOzLC3g56INMJmvx2FoLSnY8JW+RDg8PV7p9/fr1CuVI6xYfH8+NUOPl5aUwPqGXlxeAl8+Lx8fHazBKzaJkx1N79uwBAOzdu1fh/g4AVFRUcC3f8nKkdTt37hwAYODAgTh69CgGDBgAIyMjDBgwAEePHuVaaOXl+IiSHU+ZmprC09MTjDEYGxtj7ty5SEtLw9y5c7nGCU9PT2qc0BJZWVkAXo77yBhTaI1ljGH69OkK5fhIa/rZkaaXlJTEdT+JjIxU6MdI/ey0i3wekc2bNyM8PByZmZkAXt6O6NKlC/e896vzjfAJ1ex4LikpCcXFxfD19UWXLl3g6+uL4uJiSnRaRj5E171791BeXo5t27Zhx44d2LZtG8rLy3Hv3j2FcnxE/ewaoav97F5VVVXF9bPTtdFd+KCyshJGRkaoqamBkZGRwjwr8mWhUIjy8nJuAnRdQP3sCOGZixcvcmM4vkogEAB4OcbjxYsXWzKsVoWSHSE6QP6A/969e+uM6m1jY4O9e/cqlOMjSnaE6AD5A/5du3ZFWloaYmNjERQUhNjYWDx48ABOTk4K5fiIkh0hOqD2QAACgQDDhg3D0KFDMWzYMAgEAhoIANT1hBCdoKenh4iICEyePBl+fn7w9vbG/fv3kZmZidjYWPz+++84dOiQzk68owpKdoToiIkTJyIkJAQbNmxATEwMt15fXx8hISE6N+KJuijZEaIjoqKiEB4ejnfeeYer2XXr1g2xsbEIDw/HwIEDeZ3wqJ9dI6ifHdEGMpkMzs7O6N27N6KjoyGTybjzqaenB39/f9y8eRP379/XqUtZ6mdHCM8kJCQgIyMDn3/+OYRCxY+1UChEaGgo0tPTkZCQoKEINY+SHSE6gCbcaRwlO0J0QO0Jd5ShCXco2RGiE2r3s3v1sbGamhrqZwdKdoToBHk/u5iYGPj7++PSpUsoLy/HpUuX4O/vj5iYGISHh+tU44S6qOsJITpi4sSJOHToEIKDgzF06FBuvaOjo07OLKYuSnaE6JCJEyfCz88PcXFxOHHiBMaOHYsRI0bwukYnR8mOEB2jp6eHYcOGobS0FMOGDaNE9//onh0hhBco2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQnhBa5JdYWEhZs6cCXNzc1hYWGDevHkoKSlpcJ+8vDy8//77sLOzg4mJCfr27YvDhw+3UMTaQyaTKcwgL5PJNB0SIU1Oa5LdzJkzcevWLcTGxiImJgZ//PEHFixY0OA+s2fPxt27d3Hs2DGkpKRg4sSJmDp1Kq5fv95CUbd+UVFRcHZ2hre3N9avXw9vb284OzsjKipK06ER0rSYFrh9+zYDwJKTk7l1J06cYAKBgGVnZ9e7n4mJCdu9e7fCOktLS/bTTz+p/N5FRUUMACsqKlI/8Fbu8OHDTCAQsHfeeYctWrSI+fj4sEWLFrF33nmHCQQCdvjwYU2HSF5DdXU1i42NZUFBQSw2NpZVV1drOqRmo87nUytGKt6xYweCg4Px/Plzbl11dTUMDQ1x8OBBTJgwQel+Pj4+MDAwwO7du2FhYYHffvsN8+bNw40bN+Ds7Kx0H6lUCqlUyi1LJBI4ODigoKBAp0YqlslkcHV1hUAgwKNHjxQuXfX09NC5c2cAwO3bt6kHvhY5cuQIli5dikePHnHrOnfujO+//77ez4k2k0gksLKyUmmkYq14XCwvL6/OxL/6+vqwtLREXl5evfv99ttvmDZtGtq3bw99fX0YGxvjyJEj9SY6AAgLC8Pq1avrrD99+jSMjY1f/yBamZSUFGRkZCjdJpPJkJ6eDgAIDw9H7969WzAy8roSExOxdu3aOusfPXqEadOmYfny5Rg0aJAGIms+ZWVlKpfVaLJbsWKF0pNTW2pq6mu//pdffokXL17gzJkzsLKyQnR0NKZOnYqEhIR6P8ChoaEICgriluU1Ox8fH52q2RUWFqpUrlOnThg3blwzR0PelEwmw/Tp0xsss3nzZnz11Vc6VVOXSCQql9VosgsODsbcuXMbLOPk5AQ7Ozvk5+crrK+urkZhYSHs7OyU7peWloYtW7bg5s2b+Nvf/gYAcHNzQ0JCArZu3Yrt27cr3U8kEkEkEtVZ36ZNG52aiCYpKYn7v76+PoKCguDo6Ij09HSsX78e1dXVXLmAgABNhUlUdPbsWZSXl3PLs2bNgoeHB65evYq9e/cCeFkLio+Px5gxYzQVZpNT6zPZ7HcQm4C8geLKlSvculOnTjXYQPHXX38xAOz27dsK6318fNj8+fNVfm9dbaAYNGgQA8AAsPLyclZZWcmio6NZZWUlKy8v57YNGjRI06ESFQwfPlyl8zl8+HBNh9qk1Pl8akXXE1dXV4wZMwbz589HUlISLly4gMDAQLz33nvo2LEjACA7OxsuLi5cjcXFxQXOzs746KOPkJSUhLS0NERERCA2Nhb+/v4aPJrWQX5PDgCmTp2qMLLt1KlTlZYjrde1a9cAAEOHDoWhoaHCNkNDQ/z9739XKMdHWtFAAQD79u1DYGAgvLy8IBQKMWnSJGzatInbXlVVhbt373I3LNu0aYPjx49jxYoV8PX1RUlJCZydnfHLL7/QPSgAZmZmyMvLg0gkwo0bNxRGtu3SpQtEIhGkUinMzMw0GCVRlfw+XHp6utI5KDIzMxXK8ZHWJDtLS0tERkbWu10sFoO90oumW7du9MREPUaOHIn79+9DKpWioqICn332GcrKymBsbIzIyEiu+83IkSM1HClRxd///nfExMQgKysL7777LpYvX87V1NeuXYvHjx9z5fhKK/rZaZI6M45rk/LycpW60pSVlcHIyKgFIiJvoqSkRKVaeHFxMUxNTVsgopahzudTK+7ZkaZnZGQEPz+/Bsv4+flRotMSpqam8PT0bLCMp6enTiU6dVGy47Ho6Oh6E56fnx+io6NbNiDyRpKSkupNeJ6engrdjfiIkh3PRUdHo6ysDB9//DHc3d3x8ccfo6ysjBKdlkpKSkJxcTF8fX3RpUsX+Pr6ori4mPeJDqB7do3S1Xt2r6qqqsLx48cxbtw4neo8zVd8OZ90z44QQl5ByY4QwguU7AghvKA1nYo1RX5LU53RFbRRVVUVysrKIJFIdPoeD1/w5XzKP5eqND1QsmtEcXExAMDBwUHDkRBC6lNcXIy2bds2WIZaYxtRU1ODnJwcmJmZQSAQaDqcZiMfty8rK0unW535gi/nkzGG4uJidOzYEUJhw3flqGbXCKFQCHt7e02H0WLMzc11+sPBN3w4n43V6OSogYIQwguU7AghvEDJjgB4ORz9qlWrlA5JT7QPnc+6qIGCEMILVLMjhPACJTtCCC9QsiOE8AIlO0IIL1Cy00HDhw/HZ599pukwiJpa23lrbfG8KUp2hOiQyspKTYfQejXTRN1EQ+bMmcPN/i7/6dSpE/vhhx8Uyl27do0JBAKWkZHBGGMsIiKC9erVixkbGzN7e3u2cOFCVlxcrLBPQkICe/vtt5mhoSGzt7dnixYtYiUlJS12bLpM2Xl78OAB++CDD5hYLGaGhoase/fubOPGjXX28/PzY19//TXr0KEDE4vFjDHGLly4wNzc3JhIJGIeHh7syJEjDAC7fv06t29KSgobM2YMMzExYTY2NmzWrFns6dOn9caTnp7eUr+OZkHJTse8ePGCDRo0iM2fP5/l5uay3NxcFhISwt5++22FcsHBwQrrNmzYwM6dO8fS09PZ2bNnWY8ePdjChQu57Q8ePGAmJiZsw4YN7N69e+zChQvsrbfeYnPnzm2xY9Nlys5bRUUFW7lyJUtOTmYPHz5ke/fuZcbGxuzAgQPcfnPmzGGmpqbs/fffZzdv3mQ3b95kRUVFzNLSks2aNYvdunWLHT9+nHXv3l0h2T1//pxZW1uz0NBQlpqayq5du8a8vb3ZiBEj6o2nurpaE7+aJkPJTgcNGzaMffrpp9zy9evXmUAgYJmZmYwxxmQyGevUqRPbtm1bva9x8OBB1r59e2553rx5bMGCBQplEhISmFAoZOXl5U17ADz16nlT5h//+AebNGkStzxnzhxma2vLpFIpt27btm2sffv2Cuflp59+Ukh2a9asYT4+PgqvnZWVxQCwu3fvqhyPNqF7djzg7u4OV1dXREZGAgDOnz+P/Px8TJkyhStz5swZeHl5oVOnTjAzM8P777+PZ8+eoaysDABw48YN7Nq1C6amptzP6NGjUVNTg/T0dI0cFx9s3boVHh4esLa2hqmpKX788Uc8evRIoUzv3r1hYGDALd+9exd9+vSBoaEht65///4K+9y4cQNxcXEK59PFxQUAkJaW1oxHpDmU7Hhi5syZXLKLjIzEmDFj0L59ewBARkYGxo8fjz59+uDw4cO4evUqtm7dCuC/N7xLSkrw0Ucf4c8//+R+bty4gfv376Nr166aOSgdt3//foSEhGDevHk4ffo0/vzzTwQEBNRphDAxMVH7tUtKSuDr66twPv/880/cv38fQ4cObapDaFVoPDsdZGBgAJlMprBuxowZ+OKLL3D16lUcOnQI27dv57ZdvXoVNTU1iIiI4AZA/O233xT279u3L27fvg1nZ+fmPwCeevW8XbhwAYMHD8Ynn3zCrVOl1tWjRw/s3bsXUqmUGwggOTlZoUzfvn1x+PBhiMVi6OsrTwPK/o60GdXsdJBYLMbly5eRkZGBgoIC1NTUQCwWY/DgwZg3bx5kMhneffddrryzszOqqqqwefNmPHz4EHv27FFIhgCwfPlyXLx4EYGBgVwN4OjRowgMDGzpw9NZr563bt264cqVKzh16hTu3buHL7/8sk7SUmbGjBmoqanBggULkJqailOnTiE8PBwAuNG2//GPf6CwsBDTp09HcnIy0tLScOrUKQQEBHAJTtnfkVbT9E1D0vTu3r3LBg4cyIyMjBS6DPzwww8MAJs9e3adfdavX886dOjAjIyM2OjRo9nu3bsZAPb8+XOuTFJSEvP29mampqbMxMSE9enTh33zzTctdFS679XzdufOHTZ37lzWtm1bZmFhwRYuXMhWrFjB3NzcuH3kXU9edeHCBdanTx9mYGDAPDw8WGRkJPeacvfu3WMTJkxgFhYWzMjIiLm4uLDPPvuM1dTUKI1H27ue0BBPhPDAvn37EBAQgKKiIhgZGWk6HI2ge3aE6KDdu3fDyckJnTp1wo0bN7B8+XJMnTqVt4kOoGRHiE7Ky8vDypUrkZeXhw4dOmDKlCn45ptvNB2WRtFlLCGEF6g1lhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjrQac+fOhUAggEAgQJs2bWBrawtvb2/s2LFDrecyd+3aBQsLi+YLtB5z586Fv79/i78vUQ0lO9KqjBkzBrm5ucjIyMCJEycwYsQIfPrppxg/fjyqq6s1HR7RZpp9NJeQ/6rvofazZ88yAOynn35ijDU8X0ZcXFyduRNWrVrFGGNs9+7dzMPDg5mamjJbW1s2ffp09uTJE+59CgsL2YwZM5iVlRUzNDRkzs7ObMeOHdz2R48esSlTprC2bduydu3asXfffZd7OH7VqlV13jcuLq5Zfk/k9VDNjrR6I0eOhJubG6KiogAAQqEQmzZtwq1bt/DLL7/g3LlzWLZsGQBg8ODB2LhxI8zNzZGbm4vc3FyEhIQAAKqqqrBmzRrcuHED0dHRyMjIwNy5c7n3+fLLL3H79m2cOHECqamp2LZtG6ysrLh9R48eDTMzMyQkJODChQswNTXFmDFjUFlZiZCQEEydOpWrmebm5mLw4MEt+4siDdN0tiVErr6aHWOMTZs2jbm6uird9up8GTt37mRt27Zt9P2Sk5MZAK5W6OvrywICApSW3bNnD+vRowc3/BFjjEmlUmZkZMROnTrVaPxE86hmR7QCY4wbeLKx+TLqc/XqVfj6+qJz584wMzPDsGHDAICb02HhwoXYv38/3N3dsWzZMly8eJHb98aNG3jw4AHMzMy4ORssLS1RUVGhs3M26BpKdkQrpKamwtHRUaX5MpQpLS3F6NGjYW5ujn379iE5ORlHjhxR2G/s2LHIzMzEkiVLkJOTAy8vL+4SuKSkBB4eHnXmbLh37x5mzJjRzEdPmgIN8URavXPnziElJQVLlixRab4MZXMn3LlzB8+ePcN3330HBwcHAMCVK1fqvJe1tTXmzJmDOXPmYMiQIVi6dCnCw8PRt29fHDhwADY2NjA3N1cap67N2aBrqGZHWhWpVIq8vDxkZ2fj2rVr+Pbbb+Hn54fx48dj9uzZKs2XIRaLUVJSgrNnz6KgoABlZWXo3LkzDAwMuP2OHTuGNWvWKOy3cuVKHD16FA8ePMCtW7cQExMDV1dXAC9nZ7OysoKfnx8SEhKQnp6O+Ph4LF68GI8fP+be96+//sLdu3dRUFCAqqqqlvmlEdVo+qYhIXJz5szhum3o6+sza2trNmrUKLZjxw4mk8m4cqrMl/Hxxx+z9u3bK3Q9iYyMZGKxmIlEIjZo0CB27NixOhNHu7q6MiMjI2Zpacn8/PzYw4cPudfMzc1ls2fPZlZWVkwkEjEnJyc2f/58VlRUxBhjLD8/n5ujA9T1pNWhwTsJIbxAl7GEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBe+D/Z7bZreZZ7YAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:30:58.468878Z", + "iopub.status.busy": "2024-03-22T20:30:58.468106Z", + "iopub.status.idle": "2024-03-22T20:30:58.715170Z", + "shell.execute_reply": "2024-03-22T20:30:58.714218Z" + }, + "papermill": { + "duration": 0.269502, + "end_time": "2024-03-22T20:30:58.717276", + "exception": false, + "start_time": "2024-03-22T20:30:58.447774", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABLA0lEQVR4nO3dd3yT1RrA8V9G00UXq2UUyi5lFQpFUECwigoiuFgyCrILaEVkKFtBlsgQBC/gABEFhasIVypFGQKWPZVZBNpSSvfIeu8foYHQlZS0adLz/XzyIX1H8jSlT8973nPOI5MkSUIQBMHOyG0dgCAIQnGI5CUIgl0SyUsQBLskkpcgCHZJJC9BEOySSF6CINglkbwEQbBLInkJgmCXlLYOoLTp9Xpu3ryJh4cHMpnM1uEIgvAQSZJIS0ujevXqyOUFt6/KXfK6efMm/v7+tg5DEIQiXL9+nZo1axa4v9wlLw8PD8DwwXh6eto4GkEQHpaamoq/v7/xd7Ug5S555V4qenp6iuQlCGVYUd06osNeEAS7VCaS14oVKwgICMDFxYW2bdty+PBhs87btGkTMpmMnj17lmyAgiCUOTZPXt9++y2RkZFMnz6do0eP0qJFC7p27UpCQkKh5129epUJEybQoUOHUopUEISyxObJa/HixQwbNozw8HCCgoJYtWoVbm5urF27tsBzdDod/fv3Z+bMmdStW7cUoxUEoaywafJSq9XExMQQFhZm3CaXywkLC+PgwYMFnjdr1iyqVq3K0KFDi3yPnJwcUlNTTR6CINg/myavxMREdDodvr6+Jtt9fX2Ji4vL95x9+/bxn//8hzVr1pj1HnPnzsXLy8v4EGO8BMEGtDmgybLqS9r8stESaWlpDBgwgDVr1lC5cmWzzpk8eTIpKSnGx/Xr10s4SkEQ8tg9A9Z0gYRzVntJm47zqly5MgqFgvj4eJPt8fHx+Pn55Tn+0qVLXL16lRdeeMG4Ta/XA6BUKrlw4QL16tUzOcfZ2RlnZ+cSiF4QBLP8vQv+/NTw/O41qNrYKi9r05aXSqUiJCSEqKgo4za9Xk9UVBTt2rXLc3xgYCCnTp3i+PHjxkePHj3o3Lkzx48fF5eEglDWpN6CH0cZnrcdBY2etdpL23yEfWRkJIMGDaJ169aEhoayZMkSMjIyCA8PB2DgwIHUqFGDuXPn4uLiQtOmTU3O9/b2BsizXRAEG9Pr4IfhkHkH/JrB0zOt+vI2T169e/fm9u3bTJs2jbi4OIKDg9m5c6exEz82NrbQmeWCIJRR+5fAld/ByQ1eWQdK63bfyMpb3cbU1FS8vLxISUkRcxsFoaRcPwxrnwVJBy+ugJavm32qub+jokkjCIJ1ZSXD90MNiavpKxDcn+tJmfx5+Y5V30YkL0EQrEeS4Kc3ISUWvGtD98Vka/WM2hBDvzV/8uOxG1Z7K5G8BEGwnmNfwZkfQK409HO5eDHrp7OcvpGKl6sTbepUtNpbieQlCIJ13L4AOyYannd5H2qGsPXov2w8FItMBkv6tKSGt6vV3k4kL0EQHp0mG74fAtosqNsZ2o/jfFwqU344BcC4Lg3o1LCKVd9SJC9BEB7dr+9D/Glwqwy9PiNNrWP010fJ1ujp0KAy455qYPW3FMlLEIRHc/5nOLza8LzXZ0gVqjJpyykuJ2ZQzcuFT/q0RCG3fqUukbwEQSi+lBuwbYzhebsIaBDGuv1X+fnULZRyGSv6t6Kiu6pE3lokL0EQikevg63DIesuVAuGp6YTc+0uH+4wrBwxtVtjWtXyKbG3F8lLEITi+WMRXNsHqgrwylruZEtEbDyKVi/RrXk1BrcPKNG3F8lLEATLXTsI0XMNz7stRudTl/GbjnMrJZu6Vdz56OXmJV6RXiQvQRAsk3UXtrwBkh6a94EWvfkk6h/2XUzE1UnBqtdDqOBc8ms+iOQlCIL5JAm2j4XUf6FiXei2kOgLCSz77R8A5r7UjIa+hVe6thaRvARBMN9fa+Hcf0HuBK+s5UaWkje/PY4kQf+2tejZskaphSKSlyAI5ok/C7umGJ6HzSCnanNGbzhKcqaG5jW9mPZCUKmGI5KXIAhFU2fem/6TDfXD4LHRfPjzOU5cT8bL1YkV/VrhrFSUakgieQmCULT/TYXb56CCL/RcxfZTcXxx8BoAH/dugX9Ft1IPSSQvQRAKd3aboa8LGfT6jH8yXJi05SQAYzrXo0ugb+HnlxCRvARBKFjydcPdRYDHx5NRswOjNhwlU62jfb1KRD7dyGahieQlCEL+dFrDeK7sFKgRgtR5KpO3nuJiQjq+ns4s7VsyE67NJZKXIAj52/sRXP8TnD3h5f/w9ZGbbD9xE4VcxvJ+rahcwbbFnG1e+qy8yVRruZKYYXjcNvxb0V3FlOcbI7fhXzFBMHHlD/h9geF59485nuHDrJ8OADD5uUDaBFhvOefiEsmrlMzYfoadp+OIS83Od3/PljVoWsOrlKMShHxkJhlWi0CC4Ne5W7cHY5btQ6OTeLaJH0OfqGPrCAGRvEpFSqaG9QeuGr+u6K6iTmV36lR2539n4kjN1pKj1dkuQEHIJUmG9bnSbkKlBuif/Yi3vjnOjeQsAiq5Mf/Vkp9wbS6RvEpB9r3EpJDLiHkvDG+3+4uzdYm9S2q2Fp3eVtEJwgMOr4ELO0ChglfWsnz/LaIv3MZZKefT/iF4ujjZOkIj0WFfCnI0hszkrJSbJC4Axb2/Ylq9yF6CjcWdgv+9Z3j+9Gz2pVfn491/AzCnZ1OCqpetCvMieZUCtc7Q8nJW5v24c281i9wl2JQ6wzD9R5cDDZ/lVuBAxm06hiRB79b+vNra39YR5iGSVynIvtfyUhWSvETLS7CpX96FxL/BoxqaF5YzZuMxkjLUBFXzZOaLTWwdXb5E8ioFal3uZWPeiavKe8lLp5dKNSZBMDq9xVDpGhm8tJq50bc5GpuMh4uSla+3wsWpdCdcm0skr1KQU0jLSy6Sl2BLd6/Cf980PO/wNjvSG7B2/xUAFr3agtqV3G0WWlFE8ioF91teeT9u0fISbEanMUz/yUkF/7ZcbjqWid8bJlyP6FSXZ5r42TjAwonkVQpyNIYO+3xbXvfuNuokkbyEUrbnQ/j3CDh7kd1jNaM2niQ9R0tonYq884ztJlybSySvUlBoy0shWl6CDVyOhn0fAyD1WMqUPclciE+jcgVnlvdtiVJR9lND2Y/QAdzv88rb8WlseYnkJZSWjETYOgKQIGQw36S3YuvRG8hlsKxvS6p6utg6QrOI5FUKzOnz0orkJZQGvR5+HAXpcVAlkNPNJjNj+xkA3ukaSLt6lWwcoPlE8ioFhfV53R+kKpKXUAoOrYJ//gcKZ9K6r2bkt2dR6/SENfZlRMe6to7OIiJ5lYLCWl4K0fISSsvN4/DrNAD0z3zAm3vU/Hs3C/+Krix6tYXdLckkklcpeHBu48OMLS9xt1EoSTnphuk/eg0EdmdV5pNEnU9ApZSzsn8IXm5lZ8K1uUTyKgWFjbBXyA0/Aq1OJC+hBO14B5IugWcNjjSfxcL/GSZcz+zRxG7XkRPJqxTkaAuZ23ivpS5aXkKJObkZTmwEmZy7z37KqK1X0Evwcqua9GlT9iZcm0skr1Kg1hZ22Xiv5SX6vISSkHQZfooEQNdhIiN+dyYxPYdAPw/m9GxaZhYWLA6RvEpB7iqpqnwG/uVuEuO8BKvTqg39XOo0qNWeBVndOXw1iQrOSj7t3wpXVdmccG2uMpG8VqxYQUBAAC4uLrRt25bDhw8XeOzWrVtp3bo13t7euLu7ExwczFdffVWK0Vou97LR2anglpdIXoLV/TYbbh4DF2/2NvuQVX/EAjD/lebUrVLBxsE9Opsnr2+//ZbIyEimT5/O0aNHadGiBV27diUhISHf4ytWrMjUqVM5ePAgJ0+eJDw8nPDwcHbt2lXKkZvP2OclWl5Cabm4Gw4sBSChyyIifjL8Pg19og7PN6tmy8isxubJa/HixQwbNozw8HCCgoJYtWoVbm5urF27Nt/jn3zySXr16kXjxo2pV68e48ePp3nz5uzbt6+UIzefsc8rn3WRlKLlJVhbegL8MBIAbchQBh30JS1HS0htHyY9F2jj4KzHpslLrVYTExNDWFiYcZtcLicsLIyDBw8Web4kSURFRXHhwgU6duyY7zE5OTmkpqaaPEpbYS0vsaqEYFV6PfwwAjJuQ9UmTM/qy7lbqVRyV7GiXyuc7GDCtbls+p0kJiai0+nw9fU12e7r60tcXFyB56WkpFChQgVUKhXdunVj2bJlPP300/keO3fuXLy8vIwPf//SvzWsvtdhn1+fl1hVQrCqg8vh0m+gdGVn4w/ZcDQBmQyW9m2Jn5d9TLg2l12mYQ8PD44fP86RI0f44IMPiIyMJDo6Ot9jJ0+eTEpKivFx/fr10g0WM1teInkJj+pGDETNBOBmuxmMj8oCIDKsIY/Xr2zLyEqETes2Vq5cGYVCQXx8vMn2+Ph4/PwKXsVRLpdTv359AIKDgzl37hxz587lySefzHOss7Mzzs7OVo3bUoX3eYnkJVhBduq96T9aNIEv0i+mITnaLJ5sVIUxnevbOroSYdOWl0qlIiQkhKioKOM2vV5PVFQU7dq1M/t19Ho9OTk5JRGiVRTa8hLJS3hUkgQ/R8Ldq0he/kzIGsLVpCxqeLvy8WvBdjfh2lw2r5gdGRnJoEGDaN26NaGhoSxZsoSMjAzCw8MBGDhwIDVq1GDu3LmAoQ+rdevW1KtXj5ycHHbs2MFXX33FypUrbfltFEpdyDgvsZ6X8MhOfAOnvgOZgu31Z7FtfwZOChmf9m+Fj7uq6PPt1CMlr/T0dPQP1Rv09LSsqm7v3r25ffs206ZNIy4ujuDgYHbu3GnsxI+NjUUuv/9Ln5GRwejRo/n3339xdXUlMDCQr7/+mt69ez/Kt1KiCh9hL9bzEh5B4kX4eQIA14PfJPKgCyAxrXsQLfy9bRpaSbM4eV25coWIiAiio6PJzs42bpckCZlMhu5edWhLREREEBERke++hzvi58yZw5w5cyx+D1vKbXm55DvCXrS8hGLS5sD34aDJQO3/BK+eaotOr+XF4Oq8/lhtW0dX4ixOXq+//jqSJLF27Vp8fX3temJnabnf55XPkjgysZ6XUEy7Z0LcSSTXirypHkVcupYGVSvwYa9m5eL30uLkdeLECWJiYmjUqOyXRiorCuvzEi0voVj+/h/8uQKArbWnsuO4DDeVgpWvt8Ld2eZd2aXC4ruNbdq0sclYKXul00vGxCT6vASrSL0FPxqm/1xvMJC3jxvmKs57uTn1q3rYMrJSZXGK/vzzzxk5ciQ3btygadOmODmZLh/bvHlzqwXnCHJbXVB4AQ7tQzc+BCFfeh38MBwy76Cu3JSe/3QFYFC72vRoUd3GwZUui5PX7du3uXTpknEoA4BMJnukDntHlnunEQovfaYTuUswx/4lcOV3JCc3xmojuJMtI9jfm6ndgmwdWamzOHkNGTKEli1b8s0334gOezPktrzkMvKtQnx/kKrIXkIRrh+B3z4AYIvfeHb944mPmxMr+rfKt1Xv6CxOXteuXWP79u3G6TlC4YwLEeZTfAPEIFXBTFnJsGUISDqu13ieCf80RSaDJX1aUsPb1dbR2YTF6bpLly6cOHGiJGJxSIUV3wBR+kwwgyTBT29Bcixqj1r0in0FkDGuSwM6Naxi6+hsxuKW1wsvvMBbb73FqVOnaNasWZ4O+x49elgtOEeQ2+eVX38XPNBhL0qfCQU59hWc2YokV/KmNoJEjQsdGlRm3FMNbB2ZTVmcvEaONNyinTVrVp59osM+L3VRLS8xSFUozO0LsGMiANt8hrDjRk2qebmwpHew8Q9feWVx8np4LqNQuJxCyp6BGKQqFEKTbVjmRpvFjYpteetGR5RyGcv7taJSBdsu81QWWNTnpdFoUCqVnD59uqTicTj3W175d9iLQapCgX59H+JPo3GpxCtxg5CQM7VbY0Jq+9g6sjLBouTl5ORErVq1xKWhBUTLSyiW8zvg8GoAJupGcUvvTbfm1RjcPsC2cZUhFt9tnDp1KlOmTCEpKakk4nE4RfZ5icUIhYel3IBtowHY4fEKP6QFUbeKOx+93FyMq3yAxX1ey5cv5+LFi1SvXp3atWvj7u5usv/o0aNWC84RmHu3USQvATBM/9k6HLLuEufemPG3e+DqpGBl/xAqlJMJ1+ay+NPo2bNnCYThuNRFXTaK0mfCg/5YBNf2oVW60/fuG2hQMv+lpjTyKz8Trs1lcfKaPn16ScThsIocYS9Knwm5rh2EaMNy59N14VzRV6N/21r0alnTxoGVTcVuh8bExHDu3DkAmjRpQsuWLa0WlCMpqs9LlD4TAMi6C1veAEnPHucubEhpT7MaXrzfvfxNuDaXxckrISGBPn36EB0djbe3NwDJycl07tyZTZs2UaVK+Z2ukJ+i+ryU99bnF8mrHJMk2D4WUv/ljnNNIlL64+XqxKf9W+GST7k8wcDiu41jx44lLS2NM2fOkJSURFJSEqdPnyY1NZVx48aVRIx2rciW173NInmVYzHr4Nx/0cucGJQ6kgxc+bh3C/wrutk6sjLN4pbXzp072b17N40bNzZuCwoKYsWKFTzzzDNWDc4RFDXOS7S8yrn4s7BzMgALdH04LdVlTOd6dAn0tXFgZZ/FLS+9Xp9nMjYYBrCKqUN5Fb2qhOFfcbexHNJk3Zv+k81hRStWqbvSrm4l3gpraOvI7EKxlsQZP348N2/eNG67ceMGb731Fk899ZRVg3MERd1tVOS2vMSqEuXPrilw+xypCh9GZQyjiocrS/u2zHfRSiEviz+l5cuXk5qaSkBAAPXq1aNevXrUqVOH1NRUli1bVhIx2jVzV5UQLa9y5uw2+GstAKOzRpIs92Z5v1ZU8RATrs1lcZ+Xv78/R48eZffu3Zw/fx6Axo0bExYWZvXgHEGRI+wVYm5juZN83XB3EfhM14N9+mZMfT6Q0DoVbRyYfSnWOC+ZTMbTTz/N008/be14HI7Z63mJ5FU+6LSG8VzZKZyRNWCB5hW6NvHljQ51bB2Z3SlW8oqKiiIqKoqEhIQ8nfRr1661SmCOoug+L9HyKld+nw/X/yRT5sbI7NHUrOTJgldbiAnXxWBx8po5cyazZs2idevWVKtWTXzoRTB3VQkwtL7k5Xx1TId2dR/8vgCASTlDSFBU44f+IXi65L17LxTN4uS1atUq1q9fz4ABA0oiHodj7qoSYGh9qUTyckyZSbBlGEh6Nus6sV3fnvkvNSWouqetI7NbFt9tVKvVtG/fviRicUhqnQUtL3HH0TFJEmwbA2k3uUJ1pmsG0bu1P6+19rd1ZHbN4uT1xhtvsHHjxpKIxSHlaIoaYW/a8hIc0OE1cGEHGpSMyYmgTrWqzHyxia2jsnsWXzZmZ2ezevVqdu/eTfPmzfOMtl+8eLHVgnMEuS2vgpKX/IE+QzFFyAHFnYL/vQfAB5p+XHeuz0+viwnX1mBx8jp58iTBwcEAeQpxiM77vO63vAqvmA0ieTkcdYZh+o8uh926lqzXdWV1vxbUruRe9LlCkSxOXnv27CmJOBxWUX1ecpG8HNfOSZD4N/GSD+9oRjCiYz2eaeJn66gchphEVcJyNIXfbYT7rS+RvBzI6S1w9Ev0yHhTM5oGdQJ4p2sjW0flUMSK/iWsqJYX3Gt96SUxv9FR3L2K9N83kQErtC/yj1srdogJ11YnklcJ0uslNPdWiyiozwsMLS81YmUJh6DTwJY3kOWkEqNvwFLdy3zZtyVVPV1sHZnDEX8KSlBuqwsKb3nlzm/UivXQ7N+eD+HfI6RKbozXRPBW1yDa1atk66gcksXJ6/fff0er1ebZrtVq+f33360SlKPIvdMIhfd55a4sIQap2rnL0Uj7PgZgkuYNAgObMLJjPRsH5bgsTl6dO3fOt1p2SkoKnTt3tkpQjiJHZ+isl8lMh0Q87H7LSyQvu5WRiLR1BDIkNmo7c8q7M4teDRZzVUuQxX1ekiTlO57rzp07eapnl3cPjq4vbAycqJpt5yQJfhyFLD2Ov/U1mMdgNvYPwctNTLguSWYnr5deegkwDEQdPHgwzs73V3zU6XScPHlSzHl8iPFOYxF3mUTysnN/roR//keO5MRYzVgm92pF0xpeto7K4Zl92ejl5YWXlxeSJOHh4WH82svLCz8/P4YPH87XX39drCBWrFhBQEAALi4utG3blsOHDxd47Jo1a+jQoQM+Pj74+PgQFhZW6PG2lNvyUhVypxFE8rJrN48j/ToNgNna12nSsh192ogJ16XB7JbXunXrAAgICGDChAlWu0T89ttviYyMZNWqVbRt25YlS5bQtWtXLly4QNWqVfMcHx0dTd++fWnfvj0uLi589NFHPPPMM5w5c4YaNWpYJSZrKWpeYy6RvOxUTjrS90OQ6TXs0rXmr8q9+KFnMzFNrpRY3GE/ffp0q/ZtLV68mGHDhhEeHk5QUBCrVq3Czc2twBVZN2zYwOjRowkODiYwMJDPP/8cvV5PVFSU1WKyFnNG14NIXnZrxzvIki5xU6rITPloPn09BFeVmHBdWixOXvHx8QwYMIDq1aujVCpRKBQmD0uo1WpiYmJMinfI5XLCwsI4ePCgWa+RmZmJRqOhYsX8ixfk5OSQmppq8igt5oyuhwcqCInkZT9OboYTG9FJMt5Uj+G9V9pTt0oFW0dVrlh8t3Hw4MHExsby/vvvP/Iy0ImJieh0Onx9TasD+/r6GisTFeXdd9+levXqBVYvmjt3LjNnzix2jI+iqLW8chlbXmKcl31Iuoz+v28hB5bpetG0/fM836yaraMqdyxOXvv27eOPP/4wLotjS/PmzWPTpk1ER0fj4pL/9IvJkycTGRlp/Do1NRV//9LpUL3f52Veh70Y52UHtGr03w1FrknnkD6Q/dWHsPH5QFtHVS4Vq26jZKUWQuXKlVEoFMTHx5tsj4+Px8+v8KVDFi5cyLx584yLIhbE2dnZZFhHacpdv76oy8bcAayi/Jkd+G028ltHSZbcmal8k7X92+AkJlzbhMWf+pIlS5g0aRJXr1595DdXqVSEhISYdLbndr63a9euwPPmz5/P7Nmz2blzJ61bt37kOEqKWmveZaNctLzsw8UoOLAUgIna4Uzp+zR+XmLCta1Y3PLq3bs3mZmZ1KtXDzc3tzzLQOc3dagwkZGRDBo0iNatWxMaGsqSJUvIyMggPDwcgIEDB1KjRg3mzp0LwEcffcS0adPYuHEjAQEBxMXFAVChQgUqVChbHaY5RZQ9yyVaXnYgPQHtlhEogS+1T9OsS3+eaFDZ1lGVaxYnryVLllg1gN69e3P79m2mTZtGXFwcwcHB7Ny509iJHxsbi1x+/5d/5cqVqNVqXnnlFZPXmT59OjNmzLBqbI/K7JaXmNtYtun16LYMR5l1m3N6f/6oM57POte3dVTlnsXJa9CgQVYPIiIigoiIiHz3RUdHm3xtjcvV0mJ2y0usKlG2HVyO4soesiQVs5wn8GmftmLCdRlQrJ7GS5cu8d5779G3b18SEhIA+OWXXzhz5oxVg7N3OVrz7jYaW15iMcKy50YM+t2GoTaztAOI6N0dH3eVjYMSoBjJa+/evTRr1oxDhw6xdetW0tPTAThx4gTTp0+3eoD2zNK7jWKcVxmTnYruuyHIJS0/6dri2nYoj9cX/VxlhcXJa9KkScyZM4dff/0Vler+X6AuXbrw559/WjU4e2dun5fiXp+eGGFfhkgS0s9vo0i+yr9SZT73Hs/E58R4rrLE4uR16tQpevXqlWd71apVSUxMtEpQjsLcPq/cYUIieZUhJzYhO7UZrSQnUjeWOX2eEIViyxiLk5e3tze3bt3Ks/3YsWNlblUHW1Ob2eelFC2vsiXxIvqfDbMyPta+wpNh3cX6XGWQxcmrT58+vPvuu8TFxSGTydDr9ezfv58JEyYwcODAkojRbpnb8pKLVSXKDm0O0vfhyDWZHNQFcaTGQEaIdejLJIuT14cffkhgYCD+/v6kp6cTFBREx44dad++Pe+9915JxGi31FrzlsQRRWfLkN0zkcWdJEmqwBTZWBb1CTHOPRXKFovHealUKtasWcP777/P6dOnSU9Pp2XLljRo0KAk4rNrZre8ZOJuY5nw9//gzxUAvKMZwaiXOuBf0c3GQQkFKXbR2Vq1alGrVi1rxuJwzL3bKFpeZUBaHNKPI5EB67RdUQQ+z6shNW0dlVAIs5JXZGQks2fPxt3d3WR5mfwsXrzYKoE5ghwLJ2aL5GUjej1sHY4s8w5n9LVZ7TyYn14SyzmXdWYlr2PHjqHRaIzPCyJ+2KbMv9so5jba1P4lcGUvmZIzYzVj+aBfCJUq2GYZJcF8ZiWvPXv25PtcKJy5I+wVYlUJ27l+BOm3OciA6dpBPBb6GF0CfYs8TbC9Yvd5CUUzf4S9aHnZRFYybBmCTNKxXdeOI17P8fPzjW0dlWAms5JXbsFZc2zdurXYwTga80fYi1UlSp0kwU9vQXIssfoqvK8dyro+LXF3Fn/P7YVZ47weLDDr6elJVFQUf/31l3F/TEwMUVFReHmJUcgPMrfPy9jyEqtKlJ5jX8GZrWhQME4zlkGdm9Oqlo+toxIsYNafmdyCs2Co1vPaa6+xatUqY6kznU7H6NGj8fT0LJko7ZTZLS9j6TN9icckALcvIP3yLjJgkeZV9DVCGPuUGKdobyweYb927VomTJhgUqNRoVAQGRlZYKHY8srSPi8xSLUUaLLh+yHINJn8oWvKF/IefNw7WBTRsEMW/8S0Wm2+NRXPnz+P3tFaDtocUGcU61RJkswvOivGeZWeX6dB/GnuSJ5EakYx+fkm1BPFYu2Sxb2T4eHhDB06lEuXLhEaGgrAoUOHmDdvnrFohkNI/Ae+Dwe/5tDzU4tPz71kBAtaXiJ5lazzO+DwZwC8rRlJ44YNGfBYbRsHJRSXxclr4cKF+Pn5sWjRIuPSONWqVeOdd97h7bfftnqANpMeD/FnIO4U1O0MzV+16PTcVheY3/ISQyVKUMoN2DYagDXa5znu0oZdrzQXA6vtmMWXjXK5nIkTJ3Ljxg2Sk5NJTk7mxo0bTJw40aQfzO4FPAEd3zE8/+ktSLps0ek5mgeSVxH9KaL0WQnT62DrcMi6yyl9HeZr+/Bhr2b4eoqai/bskXopPT09HfsOY8eJUKs9qNPg+6GgVZt96oP9XUX9dRelz0rYH4vg2j4ycWGsJoIXWtbm+WbVbB2V8IiKNSLv+++/Z/PmzcTGxqJWm/5CHz161CqBlQkKJby0GlY9ATePwp458PQss07N0Zi3lheI0mclKvZPiDYULJ6qDkfjVZcZLzaxcVCCNVjc8lq6dCnh4eH4+vpy7NgxQkNDqVSpEpcvX+a5554riRhty9sfXlxueL7/E0PJdzPktrzMSV6i9FkJyboLW94ASc8W3RP8KHVg4ast8HRxKvpcocyzOHl9+umnrF69mmXLlqFSqZg4cSK//vor48aNIyUlpSRitL3GL0DroYbnP4yEdEOtypRMDdO2nebwlaQ8p+T2eRU1uh4e6PMSLS/rkSTYPg5SrhOLH9M04QzrUJd29SrZOjLBSixOXrGxsbRv3x4AV1dX0tLSABgwYADffPONdaMrS7p+AFWDICPBkMD0ej7e/TdfHrxG+LrDnI9LNTnc3DFecH89L9HnZUUx6+DcdrQoGZ0Tgb9fVd5+pqGtoxKsyOLk5efnR1KSoaVRq1YtY63GK1euIDlyy8HJFV5ZC0oXuBRF+t5P+OZwLAAZah1D1/9FYnqO8fDclldRdxpBrKRqdfFnYedkAOZpevO3vD4f9w42qxUs2A+Lk1eXLl3Yvn07YBiw+tZbb/H000/Tu3fvfOs5OpSqjeFZQ+ev6+9zaKi7SJPqngRUcuNGchYjv4oxruGl1t3rsHcq+iMWg1StSJMF3w8BbTZ/SMH8R/ccbz/TkMbVHPiueDll8d3G1atXG6cBjRkzhkqVKnHgwAF69OjBiBEjrB5gmRMSjvrvKFR//8Qyp2VcffIXavr50uvT/fx17S6Tt55i0astLGp5ieRlRbumwO1z3JX78FbmCELrVOaNDnVtHZVQAixqeWm1WubMmUNcXJxxW58+fVi6dCljx45FpVJZPcAyRyZjjc9b/CtVJkAeT6eL86hftQKf9m+FQi5j69EbfPb75ft3G81peclE8rKKs9vhL8PiAGOzR5LtXIlFr7UQpcsclEXJS6lUMn/+fLRabUnFU+alZmv47NAdxqvHoJcpkJ3cDMe/oUODKkx/IQiAj3aeZ8cpw9Qpi1pejtxnWNKSr8P2CAA+0/Vgn74ZM3s0oaaPKF3mqCzu83rqqafYu3dvScRiF746eI3UbC0pVUKg0yTDxp/fhsSLDGwXwOuP1UKSYNeZeMC8oRLisvER6bSwdRhkp3BO3pAFmld4rqkfL7WqYevIhBJkcZ/Xc889x6RJkzh16hQhISG4u7ub7O/Ro4fVgitrMtVaPv/DMMcxonN95C2egKu/w9U/YMsQGPor019owpXEDPZfvAOYN1RCJK9H9Pt8iD1IjtyN4Vmj8PFw54NeonSZo7M4eY0ebZiZn199RplMhu7eXTZHtPFQLHczNdSu5Eb35tVALjdMH1rZHm6dgKhZOHX9gE/7hdDr0/1cTswwa010kbwewdV98PsCAN7JHsJ1yZf1rzSnons56H8t5yy+bNTr9QU+HDlxZWt0fPa7odU1+sl6KHP7sjyrw4v31vs6uBz++RUvNye+GBLK64/VIvzxgCJfWySvYspMgi3DQNLzX3kXtuvbM+Cx2jzZqKqtIxNKgVj71kzf/XWd22k5VPdyoVfLh8rABz4PofeGifwwEtLi8K/oxpyezWjo61Hka4u7jcUgSbBtDKTdJF7lz7uZr1O3sjuTnw+0dWRCKTH7sjErK4uoqCi6d+8OwOTJk8nJuT+iXKFQMHv2bFxcHG+NJLVWz6q9hlbXyCfr5d+P9fQsuHYA4k/BDyPg9R8Ml5VmyF1VQtxttMCRz+HCDnQyJ8LTRpEjd2Vx72DcVKJ0WXlhdsvriy++4LPPPjN+vXz5cg4cOMCxY8c4duwYX3/9NStXriyRIG3tx2M3uJGcRRUPZ15r7Z//QU4uhulDTm5wOdpQQt5MYlUJC8WdQto1FYDZ6r6clQIY/1QDgv29bRuXUKrMTl4bNmxg+PDhJts2btzInj172LNnDwsWLGDz5s1WD9DWtDo9n0ZfBGB4h7q4OBUy9KFKQ3huvuH5b3Pg+hGz3kN5r4UmVpUwgzoD7eZwZLocduta8oW+K5OeC2Rsl/q2jkwoZWYnr4sXL9KsWTPj1y4uLsgfuCwKDQ3l7Nmz1o2uDPjp5C2u3snEx82J/o/VKvqElq9Dk5dA0hmGT2QXvUxQ7sdo6aoS3xyOpfuyP0hIzbboPHt2d+vbKJP+IV7yZqZ8DP8Z1IaRneqJYRHlkNnJKzk52aSP6/bt2wQEBBi/1uv1JvsdgV4vsXyPodX1Roe65vWnyGTwwhLwrgXJsfDfNw2dy4UwtrwsTF6Tt57i9I1U5v2StxSdIzr2y1p8zn+DXpIx1/Vt1o15li6BvrYOS7ARs5NXzZo1OX36dIH7T548Sc2aNQvcb492nYnjYkI6Hi5KBrSzoESWixe8vBZkCjizFY59XejhimK2vHKl5zj2dC29XmLtT9HU+9PQz/Vfrz7MHDeK+lWLvpMrOC6zk9fzzz/PtGnTyM7Oe4mSlZXFzJkz6datm8UBrFixgoCAAFxcXGjbti2HDx8u8NgzZ87w8ssvExAQgEwmY8mSJRa/n7kkSWLZb4ZWV3j7AMuXDvZvA13eMzz/ZSLc/rvAQxXFbHnlcuS+sowcLWM3HCb48Nt4yjK57t6UbhGf4OUmlnIu78xOXlOmTCEpKYlGjRqxYMECtm3bxrZt25g/fz6NGjXi7t27TJkyxaI3//bbb4mMjGT69OkcPXqUFi1a0LVrVxISEvI9PjMzk7p16zJv3jz8/Pwsei9L7bmQwNlbqbipFIQ/Xqd4L/L4m1D3SdBkGtaY0uTfN6V4xOpBjjo+LClDzcsrD9D4wgpayS+iVnrg/8ZGlCpnW4cmlAFmJy9fX18OHDhA48aNmTRpEr169aJXr15MnjyZoKAg9u3bh6+vZf0PixcvZtiwYYSHhxMUFMSqVatwc3Nj7dq1+R7fpk0bFixYQJ8+fXB2Ltn/wM1qeDOiU12Gd6yLT3Gnmsjl0OszcKtsGP/167R8D1M84jgvRx1h8cWBq/gk/MlopWHxS1WvZeAjKlwLBhaN6KtTpw47d+4kKSmJixcNl1T169enYsWKFr+xWq0mJiaGyZMnG7fJ5XLCwsI4ePCgxa9XkJycHJMbCampqYUcfV8VD2cmP9f40QPw8IOeK2Hjq4ZS8/U6QyPTKkuPOsLeUYvVnvnnEkucViBHglaDoImDr9QrWKRY04MqVqxIaGgooaGhxUpcAImJieh0ujytNV9fX5PFDh/V3Llz8fLyMj78/QsYZFqSGj4Dj40xPP9xNKTeNNn94NzG4tQBcMTLxhyNlv5xH+ErS0bt0wCenWfrkIQyxuHnNk6ePJmUlBTj4/r167YJJGw6VGsBWUmG0vP6+5PYH1zp878nb1ncknLEaUVx//uEzvJj5OCEU+/1oBKLCgqmbJa8KleujEKhID4+3mR7fHy8VTvjnZ2d8fT0NHnYhNLZMHzCyd2w/tcf95cUejB5jfvmGGv3X7HopR2uatOtE9T8y1Do5Icqo5H5NbVxQEJZZLPkpVKpCAkJISrqfgVqvV5PVFQU7dq1s1VYJatyfei2yPA8eq6hFD3kWWN9wa4LXEnMMPtlHeqyMScdvgtHIWnYpWtNTnC4rSMSyiibXjZGRkayZs0avvjiC86dO8eoUaPIyMggPNzwH3bgwIEmHfpqtZrjx49z/Phx1Go1N27c4Pjx48abB3YhuC80731v+tAbkHUXd5WCtnUq0qyGF6F1KpKj1fPulpNmXz461N3GXyZC0iVuSZWYqBlOW1HhWiiATdcP6d27N7dv32batGnExcURHBzMzp07jZ34sbGxJvMnb968ScuWLY1fL1y4kIULF9KpUyeio6NLO/zi67YIrh+Gu1dg+zhkr33JpuGPAfDv3Sy6Lvmdw1eS2HDoGgPaBRT5cg5zt/Hkd3B8A5JMzvic0cjcfGgoRtELBbB5h31ERATXrl0jJyeHQ4cO0bZtW+O+6Oho1q9fb/w6ICAASZLyPOwqcQE4exiWz5Er4dx2iFmPTCZDJpPhX9GNd581LKg395fzXE/KLPLlHGKEfdJl+OktAGJqv8FhqTFtAioiF2XLhALYPHmVWzVawVPTDc93ToKEc8ZdAx6rTZsAHzLVOqb+WPB80lx23+elVcP3Q0GdBrXas0p6GYC2dYo3DEcoH0TysqV2EVDvKdBm35s+lAWAXC5j3svNAfj979ukZWsKfRm7b3ntmQM3j4KLN7peqzl0zbCMUNs6or9LKJhYM9eW5HLotQpWPg4JZ2HXVOhuGEIRUOl+STlNET3ydt3yuhgF+z8xPH9xBecyPUnL1lLBWUnjaob+Lp1Oh0ZTeAIX7IeTkxMKRdH1TIsikpetVahqSGBfvwR//ccwfajxCzzY1aPR6UnKUBdYzstuc1d6gqFgCUCbN6Bxdw7vM4xxax3gg0Iu49atWyQnJ9suRqFEeHt74+fn90iLSIrkVRbUfwoeH29ogWyLgGrByLz9Uchl6PQSHefvQaeX+N9bHalbpUKe0+2y5aXXGxJXRgJUbQLPzAHg0BVDsd7QOhWJi4sjOTmZqlWr4ubmJlZLdQCSJJGZmWlcOaZatWrFfi2RvMqKzu/BlT8MfT9bh8Ggn4xJKUerB+Dy7QzHSV4Hl8OlKFC63itc4ookSRy+kgRA29o+JCcnULVqVSpVEn1fjsTV1RWAhATDz7e4l5Ciw76sUKrglf+AygNiDxqrQD9Io9Pne+qD04MkSSJHW3aL/0qSxN49O5F2zzRseHYuVDUMDfknIZ27mRpcnOQ0rGr4D+7mJuY0OqLcn+uj9GWK5FWWVKwL3T82PP99Pm1l50x2Z6p1+c5jfHBi9lvfHqfx+zu5lZJVoqEW195Tl6m9ZxwySQtBL0LIYOO+Q/daXSG1fXC6tza2uFR0TNb4uYrkVdY0fxWC+4Ok52PVCrxJM+56+7sTjN5wNM8pDzbIfjx+E70EGw/FWj20bI2O387Hk6ku3pr5kiTBz28TII/nX6kyac8sMhQsuefQ5Xv9XQHiMlEomkheZdFz86FSfarLkpjvtBq437L65XTetc5Ka5zXjO1nGLL+L8Z9c6xY51/8dQ1P5uxBK8kZp47gavr9u6cm/V11xeBUezd48GB69uxZou8hkldZ5FwBXllLjqTkGUUMryt2m+zedNi0VVUaHfZ6vcSmI4a10Hafy7/GQKESL+J/8H0APta+wlGpIZcT0427r97JJCEtB5VCbveVr5988knefPNNW4fh8ETyKquqteAjXV8A3ld+zYzQ+9eGk7aeMjk0v4nZ1u4p2nD4ES5DtTlkfjMQFymbP/VBnAwwrBry4LI/uZeMwf7ehVclF0pNWR8YLJJXGbZJ3o0oXUucZRr6Xp+JK/erDz2YsEpjJdXNRx5hBdrdM3G7c4YkqQK/Bs6mfQPDqiEPJq/cS8ZQO5/POHjwYPbu3csnn3xinGxfs2ZNVq5caXLcsWPHkMvlXLt2DTAUo2nWrBnu7u74+/szevRo0tPTTc7Zt28fHTp0wNXVFX9/f8aNG0dGhnnrvt26dYtu3brh6upKnTp12LhxIwEBASblA2UyGStXrqRHjx64u7vzwQcfoNPpGDp0KHXq1MHV1ZVGjRrxySefmLy2TqcjMjISb29vKlWqxMSJE0tlgUyRvMowFycl72hGEC9543z3H95XfmXcl5hxv6hIafR5Ffvm0N//gz9XADBRO4L+Tz9GncqG2+RXH2x5FdHfJUkSmWqtTR6W/CJ+8skntGvXjmHDhnHr1i1u3bpF37592bhxo8lxGzZs4PHHH6d2bUM1JLlcztKlSzlz5gxffPEFv/32GxMnTjQef+nSJZ599llefvllTp48ybfffsu+ffuIiIgwK66BAwdy8+ZNoqOj2bJlC6tXr863xOCMGTPo1asXp06dYsiQIej1emrWrMl3333H2bNnmTZtGlOmTGHz5s3GcxYtWsT69etZu3Yt+/btIykpiR9++MHsz6y4xCDVMsxFKecmnrypGcNG1Yf0U+5hn74ZO/SPEfrBgyvQ5j33aGwyH//6N2M610elfPS/UcXKXWlx8OMoANZpu+LSpDt1q1QwztW8nJiBJEncSM7iRnIWCrmMVrV88n2pLI2OoGm7ihn9ozk7qytuKvN+Vby8vFCpVLi5uRmXM+/fvz+LFi0iNjaWWrVqodfr2bRpE++9957xvAf7yAICApgzZw4jR47k008/BQyFZPr37288rkGDBixdupROnTqxcuVKXFxcCozp/Pnz7N69myNHjtC6dWsAPv/8cxo0aJDn2H79+hkXA801c+ZM4/M6depw8OBBNm/ezGuvvQbAkiVLmDx5Mi+99BIAq1atYteukv9ZiZZXGZbb93NQ34SkVoa/sPOcPqem7LbJcWqdnowc0xbCvouJfBL1D18evIpeL3HgYiJ3M9TFjqWwcTnpOfkMndDrDYVGMhM5q6/NPG1fRj9ZH4DaldyQySAtW8udDDWHLhtaXc1qeOHu7Hh/T4ODg2ncuLGx9bV3714SEhJ49dVXjcfs3r2bp556iho1auDh4cGAAQO4c+cOmZmG9dxOnDjB+vXrqVChgvHRtWtX9Ho9V64UXvPgwoULKJVKWrVqZdxWv359fHzy/qHITW4PWrFiBSEhIVSpUoUKFSqwevVqYmMNfaApKSncunXLZB0+pVKZ7+tYm+P9T3Egzg90XCe1juTaX7/QSn6RT5yW01v9PtoHfnyRm4/Tr23egqyXbqez7cQN3vr2BNW9XDgw+alixVJQ7vr55C3GbDzKO10bMaZz/fs79i+BK3vRyF2IyBlL+0Y1CKpuKH7i4qSgupcrN5KzuJqYcX+IRCH9Xa5OCs7O6lqs2B+VqxVuIPTv35+NGzcyadIkNm7cyLPPPmuc9nT16lW6d+/OqFGj+OCDD6hYsSL79u1j6NChqNVq3NzcSE9PZ8SIEYwbNy7Pa9eqVeuR48vl7u5u8vWmTZuYMGECixYtol27dnh4eLBgwQIOHTpktfcsLtHyKsNcnB748SicGKcZS6rkRoj8H8Yrt5ocu+tMPH/HpZGXjF/PGio03UzJzme/eQpqd73z/QnAUDTE6PoR+M0w0XpDpbFclqrToUEVk/PqVDb8klxOzDBOxi5sfJdMJsNNpbTJw9LR4CqVCp3OdIpWv379OH36NDExMXz//ff079/fuC8mJga9Xs+iRYt47LHHaNiwITdvmtb2bNWqFWfPnqV+/fp5HipV4RXdGzVqhFar5dix++PzLl68yN27d4v8Xvbv30/79u0ZPXo0LVu2pH79+ly6dMm438vLi2rVqpkkM61WS0xMTJGv/ahE8irDXJT3/+LLZPCvVIVJmjcAGKPYRjv5GZPjP9p5Ps9raHR641SbR1HQL/CD8y03H7nO+av/krVpsKHASNOXWZveHoDAaqZr0ecmr0OXk7h6JxOZDEJq2/edxlwBAQEcOnSIq1evkpiYiF6vJyAggPbt2zN06FB0Oh09evQwHl+/fn00Gg3Lli3j8uXLfPXVV6xatcrkNd99910OHDhAREQEx48f559//mHbtm1mddgHBgYSFhbG8OHDOXz4MMeOHWP48OG4uroWmZgbNGjAX3/9xa5du/j77795//33OXLkiMkx48ePZ968efz444+cP3+e0aNHl8oyRiJ5lWEmLa97bZ8d+sf4RtsZuUxiidMKKpJqPEKbz3iv72P+5fJt88uoFaSg/+IPLpQ4ccsJ/vnPUFwz/kXt4U/60wuJvWuYYxnoZ1ovMzd5bTn6LwBB1TzxcnV65DjLggkTJqBQKAgKCqJKlSrG/qH+/ftz4sQJevXqZVxZAaBFixYsXryYjz76iKZNm7Jhwwbmzp1r8prNmzdn7969/P3333To0IGWLVsybdo0qlevblZMX375Jb6+vnTs2JFevXoxbNgwPDw8Cu3oBxgxYgQvvfQSvXv3pm3btty5c4fRo0ebHPP2228zYMAABg0aZLy07NWrl1lxPQqZ5HAVSwuXmpqKl5cXKSkptitAa6YVey4aL8d2R3YibPFeAFzI4b+q92ggv0GUriVDNRMw937g1XndihXLa6sOcvhqkvHrUzOeYevRG0zffr/196oimgVOq9FICqIf/5KKjR7n5ZUH8PV05tCUMJPX23M+gfD19/+Chz8ewPQXmhi/zs7O5sqVK9SpU6fIXzDBcv/++y/+/v7GGwWlrbCfr7m/o6LDvgwb1qEuKoWcDg0rm3SYZ+PMWM1Ytqne5ynFMcL1O1mne65kg3koN0794TTbT9zvl6knu8FM5RcALNK+ip97ExLiDK3Ch1tdcL/llSussa+VAxYe9Ntvv5Genk6zZs24desWEydOJCAggI4dO9o6tGITl41lmEopZ1jHugT6eZrkjp1vdmDIy9051eQdACYpv6GJ7GqJxvJwu+7BxOWMmmVOy3GT5fCHrimf6bqTpdFzKcFwudrQN+8CijV97l82VfVwpr0oLltsf/zxh8kQiocfYJjqM2XKFJo0aUKvXr2oUqUK0dHRODnZ76W6aHnZocoVnHmttT/Zzd/mxLW9tMjYz1KnZbyg/oBMSuYSq7B+3UnKbwiSXyNR8iRSMwoJOVkaHclZhnFllSo45zlHqZATVM2Ts7dSea97kFi36xG0bt2a48ePF3pM165d6drVNkNNSopIXnYidylouD941UWlpMWYr7n5UQj15LeYofyCidoRJfL+sgL61MLkMYQrDaOpJ2hGchvDwMdsjY7ULMPgVQ+X/P+brXy9FdfuZNKxYZV89wvmcXV1pX79+kUf6GDEZaOdyNbcHzfk8uB0H7eKvKUeg06S8ZpyLz3kB0rk/fNrGPmSxAKnzwBYo32eaH2wcV96jtZYb9LTJf9Lk9qV3EXiEopNJC87ka253/JSPjRu65DUmOU6w63pD5z+Qy1ZfIGvc/l2eoH7CnPg0h2Tr+Xo+US1Ah9ZOif1dZiv7WOyPyVTQ2q2oeXl6SBDIISyRSQvO9GyljdVPZxpXTv/ictLtb04rG+EhyyLpU7LcCL/pZq7LNrLnfScfPdZYrRiG4/Jz5EuuTBOE4HmoR6I5Cy1seVV0GWjIDwKkbzshIuTgv2TuvDdyHZ59v0wuj3vPNeEN9VjSJbcCZZf5m3ldwW+1sTvTz5SLCGyC7yp3ALA+5pwrkp5a+8lZ2pIzSr8slEQHoVIXnbESSHP965cy1o+jOxUj5tU5l3NMABGKv9LB3n+SSrqfDGWcb7Hk3Q+Ua1AKdOzVfcEP+g75Hvc3Qy1cbUJT9HyEkqASF4OZpc+lK+1hhHTi51WUpkUK766dG9JnkSu6H15XxNe4JE3U7LJna0k+ryEkiCSlwOarR3Aeb0/VWQpLHRahYz8i9Waq//nfwLQT/EbzysOo5YUjNWMJQPXIs4EJ4UMZysshlhe5Lc0848//mizeMoy8b/KAeWgYqxmLNmSE08qTjBU8UueY64mZuRbuONht9Ny2H/xDg1k/zJN+SUA87V9OC3Vzff47RGPm3zt6eIkBqA+glu3bvHccyU89ctOieTloP6RajJLOxCAicpNNJNdNtn/5MJo2s2LIjmz4NVVE9KyOR+XijNqljstxUWmIVrXgv8UMo+yorvKZHUIcafx0fj5+eHsnHeGQmmSJAmttniFhkuSSF4ObKOuCzt0oahkOpY5LcOdLJP98ak5xlqMD1u3/wqhH0Qx9Iu/eE/5NY3k/3Jb8mKCZiRSIf9tvFydaOR3f+0uq/V3SRKoM2zzsHDhlbS0NPr374+7uzvVqlXj448/LnYtxwcvG69evYpMJmPr1q107twZNzc3WrRowcGDB03OKarK0FdffUXr1q3x8PDAz8+Pfv36mRTjiI6ORiaT8csvvxASEoKzszP79u2zOPaSJv4sOpAhj9dh7f4H1zOXMUnzBs3llwmQxzPbaR2RGtO1mB7+vdxw6BpXbmfw+T7D63TW/8kAlaHo7Vua0STiVWgM7iolLf29jUs7W63lpcmED81bu8rqptwElXvRx90TGRnJ/v372b59O76+vkybNo2jR48SHBxslXCmTp3KwoULadCgAVOnTqVv375cvHgRpVJprDI0Z84c1q5dy+3bt4mIiCAiIoJ169YBhknas2fPplGjRiQkJBAZGcngwYPZsWOHyftMmjSJhQsXUrdu3XzXu7c1kbwcyJTnAx9KXpBKBcarx7BZNYuXFPv4Q9fMZHjDyX+TTY6f+sNp4/PqJDLfaTUAq7QvsE/frMD3nvFCED7uKuRyGS0eqHhd3sZ4paWl8cUXX7Bx40bjOlnr1q0ze9FAc0yYMIFu3Qzrss2cOZMmTZpw8eJFAgMDzaoyNGTIEONr1a1bl6VLl9KmTRvS09ONq1AAzJo1i6efftpqcVubSF4ORKmQ0yWwKr89NI4rRmrEEu3LvO30PXOc1nJMXd84sPSX03H5vpYCHUtUK/CSZXJcX4+F2lfzPS7X4MfrGJ8HP5C8rLEEteGF3AwtIFtwcjP70MuXL6PRaAgNDTVu8/LyolGjRlYLp3nz5sbn1aoZfo4JCQkEBgZy4sQJTp48yYYNG4zHSJJkrDLUuHFjYmJimDFjBidOnODu3bvo79XOi42NJSgoyHheaVQAehQieTmYT/oE88upOCZuMR2gukLXk/bys7RTnGWZ0zJeUs8yTumRJCnPHcFxyh8IlV8gTXJlrCbCpFJRUap53V+WJzYp8xG+mwfIZBZdujmyB9fgyv255SagoqoMZWRkGJfH2bBhg3GZ6q5du6JWm968ebiSUFkjOuwdjIeLE6+18c+zXY+cNzWjuStVoJn8KhOVm4z7dp0xTOS+mGCoPtRWdo4IhaHi8VTNUK5Llq1y+mAirFyh8Mo2jqZu3bo4OTmZFKlISUnh77//LpX3L6rK0Pnz57lz5w7z5s2jQ4cOBAYG5ls52x6I5OWgOjfKu9RMPBV5R2NY72uYcgdPyo8D8N+ThsuxHsv3400aS1QrUMgkNms7sV3fvsj36tCgcp5tW0a155kgX97rFpTPGY7Lw8ODQYMG8c4777Bnzx7OnDnD0KFDkcvzn9plbUVVGapVqxYqlcpYqWj79u3Mnj27xOMqCSJ5OajnmuadLA2wWx/COq1hRc2FTquowl2OxyYDkKnWMt9pNdVkSVzSV2OGdlCh7/H6Y7WY/0pzlvdtlWdfSG0fVg9sTUDlsn3pURIWL15Mu3bt6N69O2FhYTz++OM0bty4VAqJFFVlqEqVKqxfv57vvvuOoKAg5s2bx8KFC0s8rpJQJqoHrVixggULFhAXF0eLFi1YtmyZSYfnw7777jvef/99rl69SoMGDfjoo494/vnnzXove6oe9Cj0eom6U3bku88ZNT+ophMkv8YfuqYM1Ezi7Kzn+XBmJLOd1pMjKXlJPYszUkCeczcOa0s1L1eOXEnipVY18qwtZi2OVD0oIyODGjVqsGjRIoYOHWrrcMoEa1QPsnnL69tvvyUyMpLp06dz9OhRWrRoQdeuXQu8Dj9w4AB9+/Zl6NChHDt2jJ49e9KzZ09Onz6d7/HllVwu44+JnRnwWO08+wzThyLIlJzpoDjNCMVP9Jq+mveUhjtU87R9801cAO3rVaZOZXdea+NfYonL3h07doxvvvmGS5cucfToUWN17BdffNHGkTkWm7e82rZtS5s2bVi+fDlguGvi7+/P2LFjmTRpUp7je/fuTUZGBj/99JNx22OPPUZwcHCeKsP5KS8tr4fdSM7i8Xm/mWx7TbGH+U5rTLbt1rXkjXzqQI7sVI+xXerj7lw6N6jtueV17Ngx3njjDS5cuIBKpSIkJITFixeTnJxc6DzF9PTirXJrj+y+bqNarSYmJobJkycbt8nlcsLCwvJMech18OBBIiMjTbZ17dq1wJn3OTk55OTcXzk0NTU13+McXQ1vVy5+8By3UrLpuuR3MtU6NuuepIP8FC8oDKtGxEve9zr07yeuAY/VpmPDKjwdJOoqmqtly5bExMTk2Z6VlVVklR/BfDZNXomJieh0Onx9TX8xfH19OX/+fL7nxMXF5Xt8XFz+gy3nzp3LzJkzrROwnVMq5PhXdOPsrGdRa/V0WrCHKSlv0Ex2hZqy27ypGcNdPFk3uA1NanhS1cO+WjxlXXmt8lNSHH6Q6uTJk01aaqmpqfj75x0HVd6olHIOTjZMX7kZ9wTnEuL4pnlLG0eVVxm4nySUAGv8XG2avCpXroxCoSA+3rTaTXx8PH5+fvme4+fnZ9Hxzs7ONl9SpKyr7leN6n75D62wldxR5JmZmbi6Fr3ooWBfMjMNMy8epWK3TZNXbmdmVFQUPXv2BAwd9lFRUcZBdQ9r164dUVFRJsuL/Prrr7Rrl7cwhWC/FAoF3t7exrvObm5uYlFDByBJEpmZmSQkJODt7Y1CoSj2a9n8sjEyMpJBgwbRunVrQkNDWbJkCRkZGYSHG9ZHHzhwIDVq1GDu3LkAjB8/nk6dOrFo0SK6devGpk2b+Ouvv1i9erUtvw2hBOS2pu11+opQMG9v7wKvlsxl8+TVu3dvbt++zbRp04iLiyM4OJidO3caO+VjY2ORy++PJ2rfvj0bN27kvffeY8qUKTRo0IAff/yRpk2b2upbEEqITCajWrVqVK1aFY1GY+twBCtxcnJ6pBZXLpuP8ypt5XWclyDYC7sZYS8IglAcInkJgmCXRPISBMEu2bzDvrTldvGV12lCglDW5f5uFtUdX+6SV1qaYbVQMcpeEMq2tLQ0vLwKrlZV7u426vV6bt68iYeHhxj0eE/ulKnr16+LO7BWJD7X4pEkibS0NKpXr24yTOph5a7lJZfLqVmzpq3DKJM8PT3FL1kJEJ+r5QprceUSHfaCINglkbwEQbBLInkJODs7M336dLH6hpWJz7VklbsOe0EQHINoeQmCYJdE8hIEwS6J5CUIgl0SyUsQBLskklc5sWLFCgICAnBxcaFt27YcPny40OO/++47AgMDcXFxoVmzZuzYkX/17fLOks91zZo1dOjQAR8fH3x8fAgLCyvy5yAUTCSvckBUJS8Zln6u0dHR9O3blz179nDw4EH8/f155plnuHHjRilH7iAkweGFhoZKY8aMMX6t0+mk6tWrS3Pnzs33+Ndee03q1q2byba2bdtKI0aMKNE47Y2ln+vDtFqt5OHhIX3xxRclFaJDEy0vB5dblTwsLMy4zZyq5A8eD4aq5AUdXx4V53N9WGZmJhqNhooVK5ZUmA5NJC8HV1hV8oKqjFtalbw8Ks7n+rB3332X6tWr5/lDIZin3K0qIQhlwbx589i0aRPR0dG4uLjYOhy7JJKXgyuNquTlUXE+11wLFy5k3rx57N69m+bNm5dkmA5NXDY6uAerkufKrUpeUJXx3KrkDxJVyU0V53MFmD9/PrNnz2bnzp20bt26NEJ1XLa+YyCUvE2bNknOzs7S+vXrpbNnz0rDhw+XvL29pbi4OEmSJGnAgAHSpEmTjMfv379fUiqV0sKFC6Vz585J06dPl5ycnKRTp07Z6lsokyz9XOfNmyepVCrp+++/l27dumV8pKWl2epbsGsieZUTy5Ytk2rVqiWpVCopNDRU+vPPP437OnXqJA0aNMjk+M2bN0sNGzaUVCqV1KRJE+nnn38u5YjtgyWfa+3atSUgz2P69OmlH7gDEEviCIJgl0SflyAIdkkkL0EQ7JJIXoIg2CWRvARBsEsieQmCYJdE8hIEwS6J5CUIgl0SyUsQBLskkpdQJgwePBiZTJbn8eyzz9o6NKGMEqtKCGXGs88+y7p160y2FVRtWqPR4OTkZLJNrVajUqksft/inifYlmh5CWWGs7Mzfn5+Jg8fHx8AZDIZK1eupEePHri7u/PBBx8wY8YMgoOD+fzzz6lTp45xXazY2FhefPFFKlSogKenJ6+99prJ0jUFnSfYF5G8BLsxY8YMevXqxalTpxgyZAgAFy9eZMuWLWzdupXjx4+j1+t58cUXSUpKYu/evfz6669cvnyZ3r17m7zWw+cJ9kdcNgplxk8//USFChVMtk2ZMoUpU6YA0K9fP8LDw032q9VqvvzyS6pUqQIY1h07deoUV65cwd/fH4Avv/ySJk2acOTIEdq0aZPveYL9EclLKDM6d+7MypUrTbY9WJwiv8X7ateubZKAzp07h7+/vzFxAQQFBeHt7c25c+eMyevh8wT7I5KXUGa4u7tTv379Qvebs83c9xLsm+jzEhxK48aNuX79OtevXzduO3v2LMnJyQQFBdkwMsHaRMtLKDNycnLylA1TKpVUrlzZ7NcICwujWbNm9O/fnyVLlqDVahk9ejSdOnUSa8Y7GNHyEsqMnTt3Uq1aNZPHE088YdFryGQytm3bho+PDx07diQsLIy6devy7bffllDUgq2IZaAFQbBLouUlCIJdEslLEAS7JJKXIAh2SSQvQRDskkhegiDYJZG8BEGwSyJ5CYJgl0TyEgTBLonkJQiCXRLJSxAEuySSlyAIdkkkL0EQ7NL/AVJtlIKwCBx5AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019459, + "end_time": "2024-03-22T20:30:58.756492", + "exception": false, + "start_time": "2024-03-22T20:30:58.737033", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 3276.970132, + "end_time": "2024-03-22T20:31:01.498010", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/tvae/4/mlu-eval.ipynb", + "output_path": "eval/insurance/tvae/4/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/tvae/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "tvae" + }, + "start_time": "2024-03-22T19:36:24.527878", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/tvae/model.pt b/insurance/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6725d5dbda61778aef5288212f9fad1be32f042 --- /dev/null +++ b/insurance/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63ada6dad85f692025cc8aa68c63fcd8c153ce0ce7eeb74552993acf519f6e26 +size 38612117 diff --git a/insurance/tvae/params.json b/insurance/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..39cbe83c085641c1cf8bec94de5f8ae997b0f624 --- /dev/null +++ b/insurance/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "NONE", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "fixed_role_model": "tvae", "mse_mag": false, "mse_mag_target": 0.1, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/lct_gan/eval.csv b/treatment/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..3dfd5df287f6fcecd70851ba2c742ac879293e42 --- /dev/null +++ b/treatment/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.0,0.013995862244785677,0.004479979896409315,11.700660228729248,0.10576117038726807,1.955125331878662,0.1900063008069992,5.562088335864246e-05,6.2541420459747314,0.04323010519146919,2659710.75,0.06693265587091446,0.24518649280071259,1.0283116580467322e-07,17.95480227470398 diff --git a/treatment/lct_gan/history.csv b/treatment/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..3753d50d6a52923ffe6d4e5e6bbf111997990cc0 --- /dev/null +++ b/treatment/lct_gan/history.csv @@ -0,0 +1,11 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.1568385149637389,35.06780496466763,0.06735080731399752,0.0,0.0,0.0,0.0,0.0,0.1568385149637389,900,225,251.36616969108582,1.1171829764048258,0.27929574410120644,0.17218083781136656,0.016657082374949443,0.6645107248224963,0.0009922695764642195,0.0,0.0,0.0,0.0,0.0,0.016657082374949443,450,113,88.97782921791077,0.7874144178576175,0.19772850937313505,0.12430944651547114 +1,0.013602297218005535,0.7166610674290971,0.000675762991130306,0.0,0.0,0.0,0.0,0.0,0.013602297218005535,900,225,252.238872051239,1.1210616535610622,0.28026541339026556,0.2211370967532922,0.010051749147562027,1.011745478109109,0.0004482628718359983,0.0,0.0,0.0,0.0,0.0,0.010051749147562027,450,113,89.63172817230225,0.7932011342681615,0.19918161816067165,0.13382232088393575 +2,0.009725581167071545,0.12862642994658147,0.00025997880217867837,0.0,0.0,0.0,0.0,0.0,0.009725581167071545,900,225,252.7475070953369,1.1233222537570529,0.2808305634392632,0.23383142546362554,0.009780957170489981,2.8031414232013194,0.0003623511504917325,0.0,0.0,0.0,0.0,0.0,0.009780957170489981,450,113,89.94277811050415,0.7959537885885323,0.1998728402455648,0.12263423703156696 +3,0.006879980324560569,0.1529390263434276,0.0001266732238959002,0.0,0.0,0.0,0.0,0.0,0.006879980324560569,900,225,253.08241868019104,1.124810749689738,0.2812026874224345,0.23675976120452913,0.009132593497263567,2.349483191307895,0.00031874165080905583,0.0,0.0,0.0,0.0,0.0,0.009132593497263567,450,113,89.89717721939087,0.7955502408795652,0.1997715049319797,0.12599473668269962 +4,0.005557837531780226,0.10897728779356435,0.00010824206081475525,0.0,0.0,0.0,0.0,0.0,0.005557837531780226,900,225,252.57941794395447,1.1225751908620198,0.28064379771550496,0.24095872966145787,0.007977272890635857,1.5098599609827625,0.00018786837228384554,0.0,0.0,0.0,0.0,0.0,0.007977272890635857,450,113,89.83993029594421,0.7950436309375594,0.1996442895465427,0.13417804435596584 +5,0.004936696493952898,0.09284011989876738,9.488825836738386e-05,0.0,0.0,0.0,0.0,0.0,0.004936696493952898,900,225,252.75963163375854,1.1233761405944824,0.2808440351486206,0.2406902328133583,0.009863585224850592,3.727565739812436,0.0004114330789984428,0.0,0.0,0.0,0.0,0.0,0.009863585224850592,450,113,89.957528591156,0.7960843238155398,0.1999056190914578,0.1339049061824502 +6,0.003923159539046234,0.060394366573882884,7.543663868940605e-05,0.0,0.0,0.0,0.0,0.0,0.003923159539046234,900,225,253.02440571784973,1.1245529143015545,0.2811382285753886,0.23062875824508308,0.006808710179325948,1.3504862622616833,0.00019859803310097395,0.0,0.0,0.0,0.0,0.0,0.006808710179325948,450,113,89.9857177734375,0.7963337856056416,0.19996826171875,0.1247912253736337 +7,0.0027492816115166838,0.13162874239411848,4.767503536376824e-05,0.0,0.0,0.0,0.0,0.0,0.0027492816115166838,900,225,253.01451897621155,1.1245089732276068,0.2811272433069017,0.23854128369026714,0.01000592881146531,3.7195748066041157,0.0004304353354714678,0.0,0.0,0.0,0.0,0.0,0.01000592881146531,450,113,90.18618249893188,0.7981078097250609,0.20041373888651529,0.1279042437419486 +8,0.0024273904216483466,0.11162616240413387,4.6507011236350724e-05,0.0,0.0,0.0,0.0,0.0,0.0024273904216483466,900,225,252.93700242042542,1.1241644552018908,0.2810411138004727,0.24135627153213135,0.006604909333373143,1.494413609870378,0.00017382205726341554,0.0,0.0,0.0,0.0,0.0,0.006604909333373143,450,113,90.14172339439392,0.79771436632207,0.20031494087643092,0.11940927416499832 +9,0.001791998714582203,0.02997249101643904,3.2543868669724134e-05,0.0,0.0,0.0,0.0,0.0,0.001791998714582203,900,225,253.0452380180359,1.1246455023023818,0.28116137557559545,0.2516257931456332,0.006831648460120555,2.186309788208537,0.0002196008244519274,0.0,0.0,0.0,0.0,0.0,0.006831648460120555,450,113,89.89717698097229,0.7955502387696662,0.19977150440216065,0.13539934268081083 diff --git a/treatment/lct_gan/mlu-eval.ipynb b/treatment/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..9642fa8c79bd7670f4a7e6e640e57f5daa6b6793 --- /dev/null +++ b/treatment/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2382 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:32.666736Z", + "iopub.status.busy": "2024-03-22T22:04:32.666363Z", + "iopub.status.idle": "2024-03-22T22:04:32.707476Z", + "shell.execute_reply": "2024-03-22T22:04:32.706500Z" + }, + "papermill": { + "duration": 0.059615, + "end_time": "2024-03-22T22:04:32.709595", + "exception": false, + "start_time": "2024-03-22T22:04:32.649980", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:32.739286Z", + "iopub.status.busy": "2024-03-22T22:04:32.738933Z", + "iopub.status.idle": "2024-03-22T22:04:32.745847Z", + "shell.execute_reply": "2024-03-22T22:04:32.744850Z" + }, + "papermill": { + "duration": 0.022795, + "end_time": "2024-03-22T22:04:32.748048", + "exception": false, + "start_time": "2024-03-22T22:04:32.725253", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:32.776539Z", + "iopub.status.busy": "2024-03-22T22:04:32.776283Z", + "iopub.status.idle": "2024-03-22T22:04:32.780308Z", + "shell.execute_reply": "2024-03-22T22:04:32.779414Z" + }, + "papermill": { + "duration": 0.018736, + "end_time": "2024-03-22T22:04:32.782214", + "exception": false, + "start_time": "2024-03-22T22:04:32.763478", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:32.817646Z", + "iopub.status.busy": "2024-03-22T22:04:32.817217Z", + "iopub.status.idle": "2024-03-22T22:04:32.823097Z", + "shell.execute_reply": "2024-03-22T22:04:32.822285Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.031631, + "end_time": "2024-03-22T22:04:32.825888", + "exception": false, + "start_time": "2024-03-22T22:04:32.794257", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:32.878345Z", + "iopub.status.busy": "2024-03-22T22:04:32.877855Z", + "iopub.status.idle": "2024-03-22T22:04:32.886298Z", + "shell.execute_reply": "2024-03-22T22:04:32.885464Z" + }, + "papermill": { + "duration": 0.034853, + "end_time": "2024-03-22T22:04:32.889026", + "exception": false, + "start_time": "2024-03-22T22:04:32.854173", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "aed790a6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:32.946365Z", + "iopub.status.busy": "2024-03-22T22:04:32.945921Z", + "iopub.status.idle": "2024-03-22T22:04:32.952776Z", + "shell.execute_reply": "2024-03-22T22:04:32.951906Z" + }, + "papermill": { + "duration": 0.033854, + "end_time": "2024-03-22T22:04:32.955634", + "exception": false, + "start_time": "2024-03-22T22:04:32.921780", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"lct_gan\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 1\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/lct_gan/1\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.034696, + "end_time": "2024-03-22T22:04:33.014287", + "exception": false, + "start_time": "2024-03-22T22:04:32.979591", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:33.103505Z", + "iopub.status.busy": "2024-03-22T22:04:33.103118Z", + "iopub.status.idle": "2024-03-22T22:04:33.115601Z", + "shell.execute_reply": "2024-03-22T22:04:33.114596Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.063463, + "end_time": "2024-03-22T22:04:33.118498", + "exception": false, + "start_time": "2024-03-22T22:04:33.055035", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/lct_gan/1\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:33.177495Z", + "iopub.status.busy": "2024-03-22T22:04:33.175657Z", + "iopub.status.idle": "2024-03-22T22:04:35.220478Z", + "shell.execute_reply": "2024-03-22T22:04:35.219540Z" + }, + "papermill": { + "duration": 2.085719, + "end_time": "2024-03-22T22:04:35.222560", + "exception": false, + "start_time": "2024-03-22T22:04:33.136841", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:35.248789Z", + "iopub.status.busy": "2024-03-22T22:04:35.247844Z", + "iopub.status.idle": "2024-03-22T22:04:35.263957Z", + "shell.execute_reply": "2024-03-22T22:04:35.263105Z" + }, + "papermill": { + "duration": 0.031312, + "end_time": "2024-03-22T22:04:35.266009", + "exception": false, + "start_time": "2024-03-22T22:04:35.234697", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:35.293107Z", + "iopub.status.busy": "2024-03-22T22:04:35.292509Z", + "iopub.status.idle": "2024-03-22T22:04:35.300278Z", + "shell.execute_reply": "2024-03-22T22:04:35.299435Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.023466, + "end_time": "2024-03-22T22:04:35.302250", + "exception": false, + "start_time": "2024-03-22T22:04:35.278784", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:35.326001Z", + "iopub.status.busy": "2024-03-22T22:04:35.325752Z", + "iopub.status.idle": "2024-03-22T22:04:35.431604Z", + "shell.execute_reply": "2024-03-22T22:04:35.430860Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.120816, + "end_time": "2024-03-22T22:04:35.434221", + "exception": false, + "start_time": "2024-03-22T22:04:35.313405", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:35.463858Z", + "iopub.status.busy": "2024-03-22T22:04:35.463547Z", + "iopub.status.idle": "2024-03-22T22:04:40.581090Z", + "shell.execute_reply": "2024-03-22T22:04:40.580169Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 5.135802, + "end_time": "2024-03-22T22:04:40.583383", + "exception": false, + "start_time": "2024-03-22T22:04:35.447581", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 22:04:37.993512: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 22:04:37.993566: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 22:04:37.995301: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:40.609194Z", + "iopub.status.busy": "2024-03-22T22:04:40.608233Z", + "iopub.status.idle": "2024-03-22T22:04:40.614272Z", + "shell.execute_reply": "2024-03-22T22:04:40.613444Z" + }, + "papermill": { + "duration": 0.020861, + "end_time": "2024-03-22T22:04:40.616323", + "exception": false, + "start_time": "2024-03-22T22:04:40.595462", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:40.642141Z", + "iopub.status.busy": "2024-03-22T22:04:40.641612Z", + "iopub.status.idle": "2024-03-22T22:05:00.816417Z", + "shell.execute_reply": "2024-03-22T22:05:00.815288Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 20.190347, + "end_time": "2024-03-22T22:05:00.818954", + "exception": false, + "start_time": "2024-03-22T22:04:40.628607", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T22:05:00.846415Z", + "iopub.status.busy": "2024-03-22T22:05:00.846103Z", + "iopub.status.idle": "2024-03-22T22:05:00.852448Z", + "shell.execute_reply": "2024-03-22T22:05:00.851637Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022124, + "end_time": "2024-03-22T22:05:00.854393", + "exception": false, + "start_time": "2024-03-22T22:05:00.832269", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:05:00.880136Z", + "iopub.status.busy": "2024-03-22T22:05:00.879865Z", + "iopub.status.idle": "2024-03-22T22:05:00.884303Z", + "shell.execute_reply": "2024-03-22T22:05:00.883542Z" + }, + "papermill": { + "duration": 0.019689, + "end_time": "2024-03-22T22:05:00.886245", + "exception": false, + "start_time": "2024-03-22T22:05:00.866556", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:05:00.911515Z", + "iopub.status.busy": "2024-03-22T22:05:00.911252Z", + "iopub.status.idle": "2024-03-22T22:05:01.537675Z", + "shell.execute_reply": "2024-03-22T22:05:01.536777Z" + }, + "papermill": { + "duration": 0.641421, + "end_time": "2024-03-22T22:05:01.539739", + "exception": false, + "start_time": "2024-03-22T22:05:00.898318", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/aug_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_bs_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/bs_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_synth_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:05:01.568125Z", + "iopub.status.busy": "2024-03-22T22:05:01.567837Z", + "iopub.status.idle": "2024-03-22T22:05:01.880824Z", + "shell.execute_reply": "2024-03-22T22:05:01.879995Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.32949, + "end_time": "2024-03-22T22:05:01.882875", + "exception": false, + "start_time": "2024-03-22T22:05:01.553385", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.1,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': False, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:05:01.911992Z", + "iopub.status.busy": "2024-03-22T22:05:01.911108Z", + "iopub.status.idle": "2024-03-22T22:05:02.008490Z", + "shell.execute_reply": "2024-03-22T22:05:02.007684Z" + }, + "papermill": { + "duration": 0.11385, + "end_time": "2024-03-22T22:05:02.010325", + "exception": false, + "start_time": "2024-03-22T22:05:01.896475", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/treatment [400, 0]\n", + "Caching in ../../../../treatment/_cache_aug_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/treatment [0, 200]\n", + "Caching in ../../../../treatment/_cache_bs_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/treatment [100, 0]\n", + "Caching in ../../../../treatment/_cache_bs_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/treatment [0, 50]\n", + "Caching in ../../../../treatment/_cache_synth/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/treatment [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T22:05:02.038171Z", + "iopub.status.busy": "2024-03-22T22:05:02.037909Z", + "iopub.status.idle": "2024-03-22T22:05:02.540917Z", + "shell.execute_reply": "2024-03-22T22:05:02.540009Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.519371, + "end_time": "2024-03-22T22:05:02.542986", + "exception": false, + "start_time": "2024-03-22T22:05:02.023615", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:05:02.573084Z", + "iopub.status.busy": "2024-03-22T22:05:02.572346Z", + "iopub.status.idle": "2024-03-22T22:05:02.576938Z", + "shell.execute_reply": "2024-03-22T22:05:02.576092Z" + }, + "papermill": { + "duration": 0.021575, + "end_time": "2024-03-22T22:05:02.578891", + "exception": false, + "start_time": "2024-03-22T22:05:02.557316", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:05:02.605685Z", + "iopub.status.busy": "2024-03-22T22:05:02.605412Z", + "iopub.status.idle": "2024-03-22T22:05:02.611953Z", + "shell.execute_reply": "2024-03-22T22:05:02.611135Z" + }, + "papermill": { + "duration": 0.022263, + "end_time": "2024-03-22T22:05:02.613896", + "exception": false, + "start_time": "2024-03-22T22:05:02.591633", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18680833" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:05:02.640882Z", + "iopub.status.busy": "2024-03-22T22:05:02.640608Z", + "iopub.status.idle": "2024-03-22T22:05:02.734262Z", + "shell.execute_reply": "2024-03-22T22:05:02.733432Z" + }, + "papermill": { + "duration": 0.109367, + "end_time": "2024-03-22T22:05:02.736263", + "exception": false, + "start_time": "2024-03-22T22:05:02.626896", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 75] --\n", + "├─Adapter: 1-1 [2, 2648, 75] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 77,824\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 75] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,680,833\n", + "Trainable params: 18,680,833\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 73.97\n", + "========================================================================================================================\n", + "Input size (MB): 1.99\n", + "Forward/backward pass size (MB): 1079.48\n", + "Params size (MB): 74.72\n", + "Estimated Total Size (MB): 1156.19\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:05:02.766385Z", + "iopub.status.busy": "2024-03-22T22:05:02.766124Z", + "iopub.status.idle": "2024-03-22T23:10:50.885027Z", + "shell.execute_reply": "2024-03-22T23:10:50.884010Z" + }, + "papermill": { + "duration": 3948.151267, + "end_time": "2024-03-22T23:10:50.902045", + "exception": false, + "start_time": "2024-03-22T22:05:02.750778", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.1568385149637389, 'avg_role_model_std_loss': 35.06780496466763, 'avg_role_model_mean_pred_loss': 0.06735080731399752, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.1568385149637389, 'n_size': 900, 'n_batch': 225, 'duration': 251.36616969108582, 'duration_batch': 1.1171829764048258, 'duration_size': 0.27929574410120644, 'avg_pred_std': 0.17218083781136656}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.016657082374949443, 'avg_role_model_std_loss': 0.6645107248224963, 'avg_role_model_mean_pred_loss': 0.0009922695764642195, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.016657082374949443, 'n_size': 450, 'n_batch': 113, 'duration': 88.97782921791077, 'duration_batch': 0.7874144178576175, 'duration_size': 0.19772850937313505, 'avg_pred_std': 0.12430944651547114}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013602297218005535, 'avg_role_model_std_loss': 0.7166610674290971, 'avg_role_model_mean_pred_loss': 0.000675762991130306, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013602297218005535, 'n_size': 900, 'n_batch': 225, 'duration': 252.238872051239, 'duration_batch': 1.1210616535610622, 'duration_size': 0.28026541339026556, 'avg_pred_std': 0.2211370967532922}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010051749147562027, 'avg_role_model_std_loss': 1.011745478109109, 'avg_role_model_mean_pred_loss': 0.0004482628718359983, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010051749147562027, 'n_size': 450, 'n_batch': 113, 'duration': 89.63172817230225, 'duration_batch': 0.7932011342681615, 'duration_size': 0.19918161816067165, 'avg_pred_std': 0.13382232088393575}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.009725581167071545, 'avg_role_model_std_loss': 0.12862642994658147, 'avg_role_model_mean_pred_loss': 0.00025997880217867837, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009725581167071545, 'n_size': 900, 'n_batch': 225, 'duration': 252.7475070953369, 'duration_batch': 1.1233222537570529, 'duration_size': 0.2808305634392632, 'avg_pred_std': 0.23383142546362554}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009780957170489981, 'avg_role_model_std_loss': 2.8031414232013194, 'avg_role_model_mean_pred_loss': 0.0003623511504917325, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009780957170489981, 'n_size': 450, 'n_batch': 113, 'duration': 89.94277811050415, 'duration_batch': 0.7959537885885323, 'duration_size': 0.1998728402455648, 'avg_pred_std': 0.12263423703156696}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006879980324560569, 'avg_role_model_std_loss': 0.1529390263434276, 'avg_role_model_mean_pred_loss': 0.0001266732238959002, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006879980324560569, 'n_size': 900, 'n_batch': 225, 'duration': 253.08241868019104, 'duration_batch': 1.124810749689738, 'duration_size': 0.2812026874224345, 'avg_pred_std': 0.23675976120452913}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009132593497263567, 'avg_role_model_std_loss': 2.349483191307895, 'avg_role_model_mean_pred_loss': 0.00031874165080905583, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009132593497263567, 'n_size': 450, 'n_batch': 113, 'duration': 89.89717721939087, 'duration_batch': 0.7955502408795652, 'duration_size': 0.1997715049319797, 'avg_pred_std': 0.12599473668269962}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005557837531780226, 'avg_role_model_std_loss': 0.10897728779356435, 'avg_role_model_mean_pred_loss': 0.00010824206081475525, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005557837531780226, 'n_size': 900, 'n_batch': 225, 'duration': 252.57941794395447, 'duration_batch': 1.1225751908620198, 'duration_size': 0.28064379771550496, 'avg_pred_std': 0.24095872966145787}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007977272890635857, 'avg_role_model_std_loss': 1.5098599609827625, 'avg_role_model_mean_pred_loss': 0.00018786837228384554, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007977272890635857, 'n_size': 450, 'n_batch': 113, 'duration': 89.83993029594421, 'duration_batch': 0.7950436309375594, 'duration_size': 0.1996442895465427, 'avg_pred_std': 0.13417804435596584}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004936696493952898, 'avg_role_model_std_loss': 0.09284011989876738, 'avg_role_model_mean_pred_loss': 9.488825836738386e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004936696493952898, 'n_size': 900, 'n_batch': 225, 'duration': 252.75963163375854, 'duration_batch': 1.1233761405944824, 'duration_size': 0.2808440351486206, 'avg_pred_std': 0.2406902328133583}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009863585224850592, 'avg_role_model_std_loss': 3.727565739812436, 'avg_role_model_mean_pred_loss': 0.0004114330789984428, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009863585224850592, 'n_size': 450, 'n_batch': 113, 'duration': 89.957528591156, 'duration_batch': 0.7960843238155398, 'duration_size': 0.1999056190914578, 'avg_pred_std': 0.1339049061824502}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003923159539046234, 'avg_role_model_std_loss': 0.060394366573882884, 'avg_role_model_mean_pred_loss': 7.543663868940605e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003923159539046234, 'n_size': 900, 'n_batch': 225, 'duration': 253.02440571784973, 'duration_batch': 1.1245529143015545, 'duration_size': 0.2811382285753886, 'avg_pred_std': 0.23062875824508308}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006808710179325948, 'avg_role_model_std_loss': 1.3504862622616833, 'avg_role_model_mean_pred_loss': 0.00019859803310097395, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006808710179325948, 'n_size': 450, 'n_batch': 113, 'duration': 89.9857177734375, 'duration_batch': 0.7963337856056416, 'duration_size': 0.19996826171875, 'avg_pred_std': 0.1247912253736337}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0027492816115166838, 'avg_role_model_std_loss': 0.13162874239411848, 'avg_role_model_mean_pred_loss': 4.767503536376824e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027492816115166838, 'n_size': 900, 'n_batch': 225, 'duration': 253.01451897621155, 'duration_batch': 1.1245089732276068, 'duration_size': 0.2811272433069017, 'avg_pred_std': 0.23854128369026714}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01000592881146531, 'avg_role_model_std_loss': 3.7195748066041157, 'avg_role_model_mean_pred_loss': 0.0004304353354714678, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01000592881146531, 'n_size': 450, 'n_batch': 113, 'duration': 90.18618249893188, 'duration_batch': 0.7981078097250609, 'duration_size': 0.20041373888651529, 'avg_pred_std': 0.1279042437419486}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0024273904216483466, 'avg_role_model_std_loss': 0.11162616240413387, 'avg_role_model_mean_pred_loss': 4.6507011236350724e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024273904216483466, 'n_size': 900, 'n_batch': 225, 'duration': 252.93700242042542, 'duration_batch': 1.1241644552018908, 'duration_size': 0.2810411138004727, 'avg_pred_std': 0.24135627153213135}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006604909333373143, 'avg_role_model_std_loss': 1.494413609870378, 'avg_role_model_mean_pred_loss': 0.00017382205726341554, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006604909333373143, 'n_size': 450, 'n_batch': 113, 'duration': 90.14172339439392, 'duration_batch': 0.79771436632207, 'duration_size': 0.20031494087643092, 'avg_pred_std': 0.11940927416499832}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001791998714582203, 'avg_role_model_std_loss': 0.02997249101643904, 'avg_role_model_mean_pred_loss': 3.2543868669724134e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001791998714582203, 'n_size': 900, 'n_batch': 225, 'duration': 253.0452380180359, 'duration_batch': 1.1246455023023818, 'duration_size': 0.28116137557559545, 'avg_pred_std': 0.2516257931456332}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006831648460120555, 'avg_role_model_std_loss': 2.186309788208537, 'avg_role_model_mean_pred_loss': 0.0002196008244519274, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006831648460120555, 'n_size': 450, 'n_batch': 113, 'duration': 89.89717698097229, 'duration_batch': 0.7955502387696662, 'duration_size': 0.19977150440216065, 'avg_pred_std': 0.13539934268081083}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0018094152492651322, 'avg_role_model_std_loss': 0.05782403560635162, 'avg_role_model_mean_pred_loss': 7.047225610889758e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018094152492651322, 'n_size': 900, 'n_batch': 225, 'duration': 253.36524987220764, 'duration_batch': 1.1260677772098118, 'duration_size': 0.28151694430245294, 'avg_pred_std': 0.25753333795194827}\n", + "Time out: 3688.4081094264984/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.004479979920447693, 'avg_g_mag_loss': 0.0043779365174823545, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.250043630599976, 'grad_duration': 11.716935396194458, 'total_duration': 17.966979026794434, 'pred_std': 0.24518649280071259, 'std_loss': 1.0283116580467322e-07, 'mean_pred_loss': 5.562089427257888e-05, 'pred_rmse': 0.06693265587091446, 'pred_mae': 0.043230101466178894, 'pred_mape': 2659710.75, 'grad_rmse': 0.1900063008069992, 'grad_mae': 0.10576116293668747, 'grad_mape': 1.955125331878662}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.004479979920447693, 'avg_g_mag_loss': 0.0043779365174823545, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 6.250043630599976, 'avg_grad_duration': 11.716935396194458, 'avg_total_duration': 17.966979026794434, 'avg_pred_std': 0.24518649280071259, 'avg_std_loss': 1.0283116580467322e-07, 'avg_mean_pred_loss': 5.562089427257888e-05}, 'min_metrics': {'avg_loss': 0.004479979920447693, 'avg_g_mag_loss': 0.0043779365174823545, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.250043630599976, 'grad_duration': 11.716935396194458, 'total_duration': 17.966979026794434, 'pred_std': 0.24518649280071259, 'std_loss': 1.0283116580467322e-07, 'mean_pred_loss': 5.562089427257888e-05, 'pred_rmse': 0.06693265587091446, 'pred_mae': 0.043230101466178894, 'pred_mape': 2659710.75, 'grad_rmse': 0.1900063008069992, 'grad_mae': 0.10576116293668747, 'grad_mape': 1.955125331878662}, 'model_metrics': {'lct_gan': {'avg_loss': 0.004479979920447693, 'avg_g_mag_loss': 0.0043779365174823545, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.250043630599976, 'grad_duration': 11.716935396194458, 'total_duration': 17.966979026794434, 'pred_std': 0.24518649280071259, 'std_loss': 1.0283116580467322e-07, 'mean_pred_loss': 5.562089427257888e-05, 'pred_rmse': 0.06693265587091446, 'pred_mae': 0.043230101466178894, 'pred_mape': 2659710.75, 'grad_rmse': 0.1900063008069992, 'grad_mae': 0.10576116293668747, 'grad_mape': 1.955125331878662}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:10:50.937418Z", + "iopub.status.busy": "2024-03-22T23:10:50.937124Z", + "iopub.status.idle": "2024-03-22T23:10:50.941346Z", + "shell.execute_reply": "2024-03-22T23:10:50.940559Z" + }, + "papermill": { + "duration": 0.024303, + "end_time": "2024-03-22T23:10:50.943305", + "exception": false, + "start_time": "2024-03-22T23:10:50.919002", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:10:50.977407Z", + "iopub.status.busy": "2024-03-22T23:10:50.977137Z", + "iopub.status.idle": "2024-03-22T23:10:51.106610Z", + "shell.execute_reply": "2024-03-22T23:10:51.105677Z" + }, + "papermill": { + "duration": 0.149117, + "end_time": "2024-03-22T23:10:51.108957", + "exception": false, + "start_time": "2024-03-22T23:10:50.959840", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:10:51.144137Z", + "iopub.status.busy": "2024-03-22T23:10:51.143846Z", + "iopub.status.idle": "2024-03-22T23:10:51.413057Z", + "shell.execute_reply": "2024-03-22T23:10:51.412163Z" + }, + "papermill": { + "duration": 0.288689, + "end_time": "2024-03-22T23:10:51.414911", + "exception": false, + "start_time": "2024-03-22T23:10:51.126222", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5rElEQVR4nO3de1xUdf4/8NeZYS7ch4sMF7logZoiXhAiN9mSb2h2wdg0vpTk19RatJTclErp8tuwu5v6za3dtL4bq/ndbL+hWxFeWnMUAykVJTVUVC6hMsN1rp/fH8McGRhgZphhQN7Px+M8Zjjnc875DDIvP+dzPuccjjHGQAghQ4TA1RUghBBbUGgRQoYUCi1CyJBCoUUIGVIotAghQwqFFiFkSKHQIoQMKW6ursBAMRgMuHLlCry9vcFxnKurQwjpgjGGpqYmhIaGQiDouT01bELrypUrCA8Pd3U1CCF9qK6uxsiRI3tcPmxCy9vbG4DxF+Lj4+Pi2hBCulKpVAgPD+e/qz0ZNqFlOiT08fGh0CJkEOur+4Y64gkhQwqFFiFkSKHQIoQMKcOmT4v0j16vh1ardXU1yBAmEokgFAr7vR0KLdIrxhhqa2vR2Njo6qqQm4BMJkNwcHC/xkpSaJFemQIrKCgIHh4eNDCX2IUxhtbWVtTX1wMAQkJC7N4WhVYXqnYtTl1RwcCApFsCXF0dl9Lr9XxgBQQM798F6T93d3cAQH19PYKCguw+VKSO+C5+OH8N8z84jP+3u8LVVXE5Ux+Wh4eHi2tCbhamv6X+9I9SaHURJjP+Ui83trm4JoMHHRISR3HE3xKFVhdhfsYmbGOrFi1qnYtrQwjpikKrCy+JG3zdRQCotUXIYEShZUGYzNjaunydQovYh+M4fPHFF66uhkO99NJLmDRpkqurQaFliekQ8RK1tMgQtm3bNshkModtb9WqVSguLnbY9uxlV2ht3rwZUVFRkEqlSExMRElJSY9lT548ifT0dERFRYHjOGzYsKFbmZdeegkcx5lNY8eONSvT3t6O7OxsBAQEwMvLC+np6airq7On+n2ilhYZTjQajVXlvLy8BsXQF5tDa8eOHcjJyUFeXh7KysoQFxeH1NRUftBYV62trRg9ejTWr1+P4ODgHrc7fvx41NTU8NPBgwfNlq9cuRJffvkldu7ciQMHDuDKlSt46KGHbK2+VUZ2tLSoT8scYwytGp1LJlsfhP7VV1/hN7/5DWQyGQICAnDffffh3LlzAIA77rgDq1evNiv/66+/QiQS4bvvvgMA1NTUYM6cOXB3d8eoUaNQUFCAqKgoi//pWuP48eO4++674e7ujoCAACxZsgTNzc388v379yMhIQGenp6QyWSYPn06Lly4AAD48ccfcdddd8Hb2xs+Pj6YOnUqfvjhh173t3//fixcuBBKpZJvCLz00ksAgKioKLz66qtYsGABfHx8sGTJEgDA6tWrERMTAw8PD4wePRpr1641G5rQ9fDw8ccfR1paGt566y2EhIQgICAA2dnZTr/cy+bBpe+88w4WL16MhQsXAgC2bNmC3bt346OPPsKaNWu6lZ82bRqmTZsGABaX8xVxc+sx1JRKJf7617+ioKAAd999NwBg69atGDduHA4fPozbb7/d1o/RqxstrVaHbneoa9Pqcdu6r12y74pXUuEhtv7PtaWlBTk5OZg4cSKam5uxbt06zJ07F+Xl5cjMzMQbb7yB9evX86fgd+zYgdDQUNx5550AgAULFqChoQH79++HSCRCTk5Oj/8xW1OX1NRUJCUl4ejRo6ivr8cTTzyBZcuWYdu2bdDpdEhLS8PixYvx97//HRqNBiUlJXzdMjMzMXnyZLz//vsQCoUoLy+HSCTqdZ933HEHNmzYgHXr1qGyshKAsaVk8tZbb2HdunXIy8vj53l7e2Pbtm0IDQ3F8ePHsXjxYnh7e+O5557rcT/79u1DSEgI9u3bh7Nnz2L+/PmYNGkSFi9ebNfvyho2hZZGo0FpaSlyc3P5eQKBACkpKVAoFP2qyJkzZxAaGgqpVIqkpCTk5+cjIiICAFBaWgqtVouUlBS+/NixYxEREQGFQmExtNRqNdRqNf+zSqWyui5h1NIa8tLT081+/uijjzBixAhUVFRg3rx5WLFiBQ4ePMiHVEFBATIyMsBxHE6fPo1vv/0WR48eRXx8PADgL3/5C6Kjo+2qS0FBAdrb2/HJJ5/A09MTALBp0ybcf//9eP311yESiaBUKnHffffhlltuAQCMGzeOX//ixYv4wx/+wHeZWFMPsVgMX19fcBxnsTFw991349lnnzWb9+KLL/Lvo6KisGrVKmzfvr3X0PLz88OmTZsgFAoxduxYzJkzB8XFxYMntBoaGqDX6yGXy83my+VynD592u5KJCYmYtu2bRgzZgxqamrw8ssv484778SJEyfg7e2N2tpaiMXibp2KcrkctbW1FreZn5+Pl19+2a76mFpa9U1qaHQGiN3ofAUAuIuEqHgl1WX7tsWZM2ewbt06HDlyBA0NDTAYDACMATBhwgTcc889+PTTT3HnnXeiqqoKCoUCf/7znwEAlZWVcHNzw5QpU/jt3XrrrfDz87Or7qdOnUJcXBwfWAAwffp0GAwGVFZWYsaMGXj88ceRmpqK//iP/0BKSgrmzZvHX5+Xk5ODJ554Av/zP/+DlJQUPPzww3y42csUxp3t2LED7733Hs6dO4fm5mbodLo+7/I7fvx4s8txQkJCcPz48X7VrS+D4ts4e/ZsPPzww5g4cSJSU1OxZ88eNDY24rPPPrN7m7m5uVAqlfxUXV1t9br+nmJIRQIwBtQoqbVlwnEcPMRuLplsHUl9//3349q1a/jwww9x5MgRHDlyBMCNTufMzEz87//+L7RaLQoKChAbG4vY2FiH/86stXXrVigUCtxxxx3YsWMHYmJicPjwYQDGvqSTJ09izpw52Lt3L2677Tbs2rWrX/vrHKAAoFAokJmZiXvvvReFhYU4duwYXnjhhT476bsepnIcx/8H4Sw2hVZgYCCEQmG3s3Z1dXW9drLbSiaTISYmBmfPngUABAcHQ6PRdLs9Sm/7lUgk/P3gbb0vPMdxCKUziEPW1atXUVlZiRdffBEzZ87EuHHjcP36dbMyDz74INrb2/HVV1+hoKAAmZmZ/LIxY8ZAp9Ph2LFj/LyzZ89224a1xo0bhx9//BEtLS38vO+//x4CgQBjxozh502ePBm5ubk4dOgQJkyYgIKCAn5ZTEwMVq5ciW+++QYPPfQQtm7d2ud+xWIx9Hq9VXU8dOgQIiMj8cILLyA+Ph7R0dH8iYDBxqbQEovFmDp1qtlYDYPBgOLiYiQlJTmsUs3NzTh37hzfPJ46dSpEIpHZfisrK3Hx4kWH7rcz0yEijdUaevz8/BAQEIAPPvgAZ8+exd69e5GTk2NWxtPTE2lpaVi7di1OnTqFjIwMftnYsWORkpKCJUuWoKSkBMeOHcOSJUvg7u5u17VzmZmZkEqlyMrKwokTJ7Bv3z4sX74cjz32GORyOaqqqpCbmwuFQoELFy7gm2++wZkzZzBu3Di0tbVh2bJl2L9/Py5cuIDvv/8eR48eNevz6klUVBSam5tRXFyMhoYGtLb2fGIpOjoaFy9exPbt23Hu3Dm89957/W7NOYvNh4c5OTn48MMP8fHHH+PUqVN46qmn0NLSwp9NXLBggVlHvUajQXl5OcrLy6HRaHD58mWUl5fzrSjAOGjtwIEDOH/+PA4dOoS5c+dCKBTyf0i+vr5YtGgRcnJysG/fPpSWlmLhwoVISkpy+JlDE37YA7W0hhyBQIDt27ejtLQUEyZMwMqVK/Hmm292K5eZmYkff/wRd955J3/Sx+STTz6BXC7HjBkzMHfuXP5MmlQqtbk+Hh4e+Prrr3Ht2jVMmzYNv/vd7zBz5kxs2rSJX3769Gmkp6cjJiYGS5YsQXZ2NpYuXQqhUIirV69iwYIFiImJwbx58zB79myr+mvvuOMOPPnkk5g/fz5GjBiBN954o8eyDzzwAFauXIlly5Zh0qRJOHToENauXWvzZx0QzA4bN25kERERTCwWs4SEBHb48GF+WXJyMsvKyuJ/rqqqYgC6TcnJyXyZ+fPns5CQECYWi1lYWBibP38+O3v2rNk+29ra2O9//3vm5+fHPDw82Ny5c1lNTY3VdVYqlQwAUyqV1n3G4p9Z5OpC9uxn5Vbv42bT1tbGKioqWFtbm6ur4nLV1dUMAPv2229dXZUhrbe/KWu/oxxjNo7aG6JUKhV8fX2hVCqt6t/adewSVu74EUmjA/D3Jc5pzQ127e3tqKqqwqhRo+xqYQxle/fuRXNzM2JjY1FTU4PnnnsOly9fxs8//9znGCnSs97+pqz9jg6Ks4eDEd1Xa3jTarV4/vnnMX78eMydOxcjRozgB5p++umn8PLysjiNHz9+wOo4e/bsHuvx2muvDVg9BhrdbrkHpgGmNco2GAwMAgHdCG84SU1NRWqq5TFpDzzwABITEy0uG8hW2F/+8he0tVn+T9Xf33/A6jHQKLR6IPeWQCjgoNUz1DepEew7vA6PSM+8vb3h7e3t6mogLCzM1VVwCTo87IGbUIBgH2NQXW6kaxAJGSwotHrB31eLhj0QMmhQaPVipIwunCZksKHQ6kUYDTAlZNCh0OpFGLW0CBl0KLR6QS0tYq+b8cEWgwWFVi86t7SGyYUD5Cbi6AdbAMbbOHMc1+2OKwOJQqsXptvTtGr0aGx17n2vCSHWodDqhVQkRKCXBAD1awEAGAM0La6Z6MEWDnuwhVqtxqpVqxAWFgZPT08kJiZi//79/LoXLlzA/fffDz8/P3h6emL8+PHYs2cPzp8/j7vuuguA8fY/HMfh8ccft+v30R80Ir4PYX7uaGhW49L1NkwI83V1dVxL2wq8FuqafT9/BRB79l2uAz3YoucHWyxbtgwVFRXYvn07QkNDsWvXLsyaNQvHjx9HdHQ0srOzodFo8N1338HT0xMVFRXw8vJCeHg4/vGPfyA9PR2VlZXw8fGBu7u7Xb+T/qDQ6sNImTt+rG6kltYQQw+2sPxgi4sXL2Lr1q24ePEiQkON/wGtWrUKX331FbZu3YrXXnsNFy9eRHp6On/76dGjR/Prm65pDAoKcnh/mbUotPpAZxA7EXkYWzyu2rcN6MEWlh0/fhx6vR4xMTFm89VqNf8g1qeffhpPPfUUvvnmG6SkpCA9PR0TJ060a3/OQH1afbhxBpGuPwTHGQ/RXDHRgy0c8mCL5uZmCIVClJaW8ncULi8vx6lTp/CnP/0JAPDEE0/gl19+wWOPPYbjx48jPj4eGzdudNhn7S8KrT7QANOhhx5sYWTpwRaTJ0+GXq9HfX09br31VrOp82FkeHg4nnzySXz++ed49tln8eGHH/LbBGD1AzOcgUKrD3R4OPTQgy2MLD3YIiYmBpmZmViwYAE+//xzVFVVoaSkBPn5+di9ezcAYMWKFfj6669RVVWFsrIy7Nu3j99fZGQkOI5DYWEhfv31V7MzoAPGKTeCHoRsvUc8v16bhkWuLmSRqwtZi1rrpNoNTkP5HvFFRUVs3LhxTCKRsIkTJ7L9+/czAGzXrl18mT179jAAbMaMGd3Wv3LlCps9ezaTSCQsMjKSFRQUsKCgILZlyxar9t91Xz/99BO76667mFQqZf7+/mzx4sWsqamJMcZYbW0tS0tL45+TEBkZydatW8f0ej1Tq9XskUceYeHh4UwsFrPQ0FC2bNkyq/9NnnzySRYQEMAAsLy8PMYYYxqNhq1bt45FRUUxkUjEQkJC2Ny5c9lPP/3EGGNs2bJl7JZbbmESiYSNGDGCPfbYY6yhoYHf5iuvvMKCg4MZx3Fmz4OwBt0j3ga23iO+s9iXvkZTuw5FK2cgWu76m78NlOF8j/iuLl26hPDwcHz77beYOXOmq6szZDniHvF09tAKYTJ3nK5twqXGtmEVWsOZpQdbREVFYcaMGa6u2rBHfVpWoGcgDj/0YIvBi1paVqAziMMPPdhi8LKrpbV582ZERUVBKpUiMTERJSUlPZY9efIk0tPTERUVBY7jLF67lZ+fj2nTpsHb2xtBQUFIS0vjLz0w+e1vf8tfQ2WannzySXuqbzM6g0g68/b27jZcwDRFRkYOWD3CwsJ6rAeFVic7duxATk4O8vLyUFZWhri4OKSmpvZ4XVZraytGjx6N9evXm40D6ezAgQPIzs7G4cOHUVRUBK1Wi3vuucdsXAsALF68GDU1NfzU22O+HWm4PwNxmJyrIQPAEX9LNh8evvPOO1i8eDEWLlwIANiyZQt2796Njz76CGvWrOlWftq0aZg2bRoAWFwOGK/I72zbtm0ICgpCaWmpWcenh4dHj8HnTMO1pWU61GltbXXJhbHk5tPaaryypD+H0TaFlkajQWlpKXJzc/l5AoEAKSkpUCgUdleiK6VSCaD7cfmnn36Kv/3tbwgODsb999+PtWvXwsPD8jVparUaarWa/1mlUtldH1OfVl1TOzQ6A8Ruw+P8hVAohEwm41vRHh4edg2uJIQxhtbWVtTX10Mmk0EoFNq9LZtCq6GhAXq9HnK53Gy+XC7H6dOn7a5EZwaDAStWrMD06dMxYcIEfv5//ud/IjIyEqGhofjpp5+wevVqVFZW4vPPP7e4nfz8fLz88ssOqVOglxgSNwHUOgNqle2ICLDt4t2hzNSytfe2LIR0JpPJ+n20NOjOHmZnZ+PEiRM4ePCg2fwlS5bw72NjYxESEoKZM2fi3LlzFq94z83NNbt0Q6VSITw83K46cRyHMJk7fmlowaXG1mEVWhzHISQkBEFBQdBq6e6txH4ikahfLSwTm0IrMDAQQqEQdXV1ZvPr6uoc0te0bNkyFBYW4rvvvsPIkSN7LWs65Xz27FmLoSWRSCCRSPpdJ5MwP2NoDbd+LROhUOiQPzhC+sumzhmxWIypU6eiuLiYn2cwGFBcXIykpCS7K8EYw7Jly7Br1y7s3bsXo0aN6nOd8vJyAODvOeRsNFaLkMHB5sPDnJwcZGVlIT4+HgkJCdiwYQNaWlr4s4kLFixAWFgY8vPzARg77ysqKvj3ly9fRnl5Oby8vHDrrbcCMB4SFhQU4J///Ce8vb1RW1sLAPD19YW7uzvOnTuHgoIC3HvvvQgICMBPP/2ElStXYsaMGQN2czI+tIZpS4uQQcOmS7Q7bNy4kUVERDCxWMwSEhLY4cOH+WXJyclmV35XVVUxAN2m5ORkvoyl5QDY1q1bGWOMXbx4kc2YMYP5+/sziUTCbr31VvaHP/zBpjs22HuXB5N/lFazyNWFLOMDhV3rE0J6R3d56KI/d3kAgCO/XMX8Dw4jMsADB/5wlxNqSMjwZu13dHgMOHIA0wDTmsZ2GAzDIucJGZQotKwU7COFUMBBozfg12Z13ysQQpyCQstKbkIBgn2MNy27RJ3xhLgMhZYNaNgDIa5HoWWD4XrhNCGDCYWWDegZiIS4HoWWDailRYjrUWjZgPq0CHE9Ci0bdG5pDZMxuYQMOhRaNjC1tFo0eijb6DYthLgChZYNpCIhAr3EAGisFiGuQqFlI+rXIsS1KLRsRGcQCXEtCi0bUUuLENei0LIR3QyQENei0LJRmN/wfnArIa5GoWUjOjwkxLUotGxk6oi/1qJBq0bn4toQMvxQaNnI110Eb4nxeSBXqLVFyICj0LKDqbVFA0wJGXgUWnagfi1CXIdCyw40wJQQ16HQsgO1tAhxHbtCa/PmzYiKioJUKkViYiJKSkp6LHvy5Emkp6cjKioKHMdhw4YNdm2zvb0d2dnZCAgIgJeXF9LT01FXV2dP9fuNWlqEuI7NobVjxw7k5OQgLy8PZWVliIuLQ2pqKurr6y2Wb21txejRo7F+/XoEBwfbvc2VK1fiyy+/xM6dO3HgwAFcuXIFDz30kK3VdwhqaRHiQrY+ujohIYFlZ2fzP+v1ehYaGsry8/P7XDcyMpK9++67Nm+zsbGRiUQitnPnTr7MqVOnGACmUFj3mHprH7ltjTpVG4tcXchGrSlkGp2+39sjhFj/HbWppaXRaFBaWoqUlBR+nkAgQEpKChQKhV2hac02S0tLodVqzcqMHTsWERERPe5XrVZDpVKZTY4S6CmB2E0AAwNqle0O2y4hpG82hVZDQwP0ej3kcrnZfLlcjtraWrsqYM02a2trIRaLIZPJrN5vfn4+fH19+Sk8PNyu+lkiEHD8ISKN1SJkYN20Zw9zc3OhVCr5qbq62qHbp34tQlzDzZbCgYGBEAqF3c7a1dXV9djJ7ohtBgcHQ6PRoLGx0ay11dt+JRIJJBKJXXWyBt2ihhDXsKmlJRaLMXXqVBQXF/PzDAYDiouLkZSUZFcFrNnm1KlTIRKJzMpUVlbi4sWLdu+3v/hhD/TgVkIGlE0tLQDIyclBVlYW4uPjkZCQgA0bNqClpQULFy4EACxYsABhYWHIz88HYOxor6io4N9fvnwZ5eXl8PLywq233mrVNn19fbFo0SLk5OTA398fPj4+WL58OZKSknD77bc75BdhKzo8JMRF7Dk1uXHjRhYREcHEYjFLSEhghw8f5pclJyezrKws/ueqqioGoNuUnJxs9TYZY6ytrY39/ve/Z35+fszDw4PNnTuX1dTUWF1nRw55YIwxxbkGFrm6kCW/sdch2yNkuLP2O8oxNjyeOqpSqeDr6wulUgkfH59+b6/6WivufGMfxEIBTr86CwIB54BaEjJ8WfsdvWnPHjpbsK8UAg7Q6A1oaFa7ujqEDBsUWnYSCQUI9pECAC5RvxYhA4ZCqx/owmlCBh6FVj/QGURCBh6FVj9QS4uQgUeh1Q9hMnoGIiEDjUKrH6ilRcjAo9Dqh859WsNkuBshLkeh1Q+m0GpW66Bqowe3EjIQKLT6wV0sRICnGABwiS6cJmRAUGj1E/VrETKwKLT6icZqETKwKLT6iW4GSMjAotDqpxs3A6TQImQgUGj1UygdHhIyoCi0+okODwkZWBRa/TSy4/DwaosGbRq9i2tDyM2PQquffN1F8BQLAdAhIiEDgUKrnziOo854QgYQhZYDUL8WIQOHQssB6BmIhAwcCi0H4O+rRS0tQpyOQssBTC2tK43tLq4JITc/u0Jr8+bNiIqKglQqRWJiIkpKSnotv3PnTowdOxZSqRSxsbHYs2eP2XKO4yxOb775Jl8mKiqq2/L169fbU32Ho+sPCRk4NofWjh07kJOTg7y8PJSVlSEuLg6pqamor6+3WP7QoUPIyMjAokWLcOzYMaSlpSEtLQ0nTpzgy9TU1JhNH330ETiOQ3p6utm2XnnlFbNyy5cvt7X6TmEaq1WraodOb3BxbQi5ydn66OqEhASWnZ3N/6zX61loaCjLz8+3WH7evHlszpw5ZvMSExPZ0qVLe9zHgw8+yO6++26zeZGRkezdd9+1tbo8ax+5bQ+93sCin9/DIlcXsuprLQ7fPiHDgbXfUZtaWhqNBqWlpUhJSeHnCQQCpKSkQKFQWFxHoVCYlQeA1NTUHsvX1dVh9+7dWLRoUbdl69evR0BAACZPnow333wTOl3PdwtVq9VQqVRmk7MIBBxCZMYHt1JnPCHO5WZL4YaGBuj1esjlcrP5crkcp0+ftrhObW2txfK1tbUWy3/88cfw9vbGQw89ZDb/6aefxpQpU+Dv749Dhw4hNzcXNTU1eOeddyxuJz8/Hy+//LK1H63fwmTuuHC1lfq1CHEym0JrIHz00UfIzMyEVCo1m5+Tk8O/nzhxIsRiMZYuXYr8/HxIJJJu28nNzTVbR6VSITw83Gn1pgGmhAwMm0IrMDAQQqEQdXV1ZvPr6uoQHBxscZ3g4GCry//73/9GZWUlduzY0WddEhMTodPpcP78eYwZM6bbcolEYjHMnIUu5SFkYNjUpyUWizF16lQUFxfz8wwGA4qLi5GUlGRxnaSkJLPyAFBUVGSx/F//+ldMnToVcXFxfdalvLwcAoEAQUFBtnwEp6FhD4QMDJsPD3NycpCVlYX4+HgkJCRgw4YNaGlpwcKFCwEACxYsQFhYGPLz8wEAzzzzDJKTk/H2229jzpw52L59O3744Qd88MEHZttVqVTYuXMn3n777W77VCgUOHLkCO666y54e3tDoVBg5cqVePTRR+Hn52fP53Y4esAFIQPD5tCaP38+fv31V6xbtw61tbWYNGkSvvrqK76z/eLFixAIbjTg7rjjDhQUFODFF1/E888/j+joaHzxxReYMGGC2Xa3b98OxhgyMjK67VMikWD79u146aWXoFarMWrUKKxcudKsz8rVRpou5el4cCvHcS6uESE3J46x4fFoZJVKBV9fXyiVSvj4+Dh8+xqdAWPW/guMAUdfSMEI74HrTyPkZmDtd5SuPXQQsZsAcu+OsVrUr0WI01BoORD1axHifBRaDnTjDCLdV4sQZ6HQciBqaRHifBRaDkRjtQhxPgotBzK1tC5RS4sQp6HQcqCR1NIixOkotBzI1NJqatdB1a51cW0IuTlRaDmQh9gNfh4iANQZT4izUGg5GJ1BJMS5KLQcjM4gEuJcFFoOFtbpwmlCiONRaDkYHR4S4lwUWg5mOjy8RC0tQpyCQsvBRlJLixCnotByMFNLq6FZjXat3sW1IeTmQ6HlYDIPETzEQgDAFTpEJMThKLQcjOM4GvZAiBNRaDkBnUEkxHkotJyAWlqEOA+FlhNQS4sQ56HQcgIaq0WI81BoOQGN1SLEeewKrc2bNyMqKgpSqRSJiYkoKSnptfzOnTsxduxYSKVSxMbGYs+ePWbLH3/8cXAcZzbNmjXLrMy1a9eQmZkJHx8fyGQyLFq0CM3NzfZU3+lM1x/Wqtqh0xtcXBtCbi42h9aOHTuQk5ODvLw8lJWVIS4uDqmpqaivr7dY/tChQ8jIyMCiRYtw7NgxpKWlIS0tDSdOnDArN2vWLNTU1PDT3//+d7PlmZmZOHnyJIqKilBYWIjvvvsOS5YssbX6AyLIWwKRkIPewFDXpHZ1dQi5uTAbJSQksOzsbP5nvV7PQkNDWX5+vsXy8+bNY3PmzDGbl5iYyJYuXcr/nJWVxR588MEe91lRUcEAsKNHj/Lz/vWvfzGO49jly5ctrtPe3s6USiU/VVdXMwBMqVRa8zH77c7X97LI1YXsyC9XB2R/hAx1SqXSqu+oTS0tjUaD0tJSpKSk8PMEAgFSUlKgUCgsrqNQKMzKA0Bqamq38vv370dQUBDGjBmDp556ClevXjXbhkwmQ3x8PD8vJSUFAoEAR44csbjf/Px8+Pr68lN4eLgtH7Xf6BmIhDiHTaHV0NAAvV4PuVxuNl8ul6O2ttbiOrW1tX2WnzVrFj755BMUFxfj9ddfx4EDBzB79mzo9Xp+G0FBQWbbcHNzg7+/f4/7zc3NhVKp5Kfq6mpbPmq/0bAHQpzDzdUVAIBHHnmEfx8bG4uJEyfilltuwf79+zFz5ky7timRSCCRSBxVRZvRAFNCnMOmllZgYCCEQiHq6urM5tfV1SE4ONjiOsHBwTaVB4DRo0cjMDAQZ8+e5bfRtaNfp9Ph2rVrvW7HlegZiIQ4h02hJRaLMXXqVBQXF/PzDAYDiouLkZSUZHGdpKQks/IAUFRU1GN5ALh06RKuXr2KkJAQfhuNjY0oLS3ly+zduxcGgwGJiYm2fIQBQ89AJMRJbO3h3759O5NIJGzbtm2soqKCLVmyhMlkMlZbW8sYY+yxxx5ja9as4ct///33zM3Njb311lvs1KlTLC8vj4lEInb8+HHGGGNNTU1s1apVTKFQsKqqKvbtt9+yKVOmsOjoaNbe3s5vZ9asWWzy5MnsyJEj7ODBgyw6OpplZGRYXW9rz0w4yvmGZha5upCNeXEPMxgMA7JPQoYya7+jNocWY4xt3LiRRUREMLFYzBISEtjhw4f5ZcnJySwrK8us/GeffcZiYmKYWCxm48ePZ7t37+aXtba2snvuuYeNGDGCiUQiFhkZyRYvXsyHoMnVq1dZRkYG8/LyYj4+PmzhwoWsqanJ6joPdGiptXoWtaaQRa4uZL82tfe9AiHDnLXfUY4xxlzb1hsYKpUKvr6+UCqV8PHxGZB9Jr72LepUavwzezriwmUDsk9Chiprv6N07aET0RlEQhyPQsuJwvw6noFIZxAJcRgKLSeilhYhjkeh5UQ0VosQx6PQciIaq0WI41FoOdGN6w/pomlCHIVCy4lMfVqqdh2a2rUurg0hNwcKLSfylLhB5iECQIeIhDgKhZaT8WcQqTOeEIeg0HIyGvZAiGNRaDkZ3QyQEMei0HIyegYiIY5FoeVk9AxEQhyLQsvJTM9ApD4tQhyDQsvJTH1avzap0a7Vu7g2hAx9FFpO5uchgrtICACoUba7uDaEDH0UWk7GcRydQSTEgSi0BgA9uJUQx6HQGgDU0iLEcSi0BgCN1SLEcSi0BgCN1SLEcSi0BgBdf0iI41BoDQBTn1atsh16w7B4YhshTmNXaG3evBlRUVGQSqVITExESUlJr+V37tyJsWPHQiqVIjY2Fnv27OGXabVarF69GrGxsfD09ERoaCgWLFiAK1eumG0jKioKHMeZTevXr7en+gMuyFsKNwEHnYGhTkVjtQjpD5tDa8eOHcjJyUFeXh7KysoQFxeH1NRU1NfXWyx/6NAhZGRkYNGiRTh27BjS0tKQlpaGEydOAABaW1tRVlaGtWvXoqysDJ9//jkqKyvxwAMPdNvWK6+8gpqaGn5avny5rdV3CaGAQ4hMCoAOEQnpN1sfXZ2QkMCys7P5n/V6PQsNDWX5+fkWy8+bN4/NmTPHbF5iYiJbunRpj/soKSlhANiFCxf4eZGRkezdd9+1up7t7e1MqVTyU3V1tVWP3HaW+X8+xCJXF7JdZZdcsn9CBjulUmnVd9SmlpZGo0FpaSlSUlL4eQKBACkpKVAoFBbXUSgUZuUBIDU1tcfyAKBUKsFxHGQymdn89evXIyAgAJMnT8abb74JnU7X4zby8/Ph6+vLT+Hh4VZ8QuehC6cJcQw3Wwo3NDRAr9dDLpebzZfL5Th9+rTFdWpray2Wr62ttVi+vb0dq1evRkZGBnx8fPj5Tz/9NKZMmQJ/f38cOnQIubm5qKmpwTvvvGNxO7m5ucjJyeF/VqlULg0uegYiIY5hU2g5m1arxbx588AYw/vvv2+2rHMATZw4EWKxGEuXLkV+fj4kEkm3bUkkEovzXYWegUiIY9h0eBgYGAihUIi6ujqz+XV1dQgODra4TnBwsFXlTYF14cIFFBUVmbWyLElMTIROp8P58+dt+QguQ89AJMQxbAotsViMqVOnori4mJ9nMBhQXFyMpKQki+skJSWZlQeAoqIis/KmwDpz5gy+/fZbBAQE9FmX8vJyCAQCBAUF2fIRXKbzAFPGaKwWIfay+fAwJycHWVlZiI+PR0JCAjZs2ICWlhYsXLgQALBgwQKEhYUhPz8fAPDMM88gOTkZb7/9NubMmYPt27fjhx9+wAcffADAGFi/+93vUFZWhsLCQuj1er6/y9/fH2KxGAqFAkeOHMFdd90Fb29vKBQKrFy5Eo8++ij8/Pwc9btwKtOQh3atAddaNAjwGjyHroQMKfacmty4cSOLiIhgYrGYJSQksMOHD/PLkpOTWVZWlln5zz77jMXExDCxWMzGjx/Pdu/ezS+rqqpiACxO+/btY4wxVlpayhITE5mvry+TSqVs3Lhx7LXXXmPt7e1W19na06nONO3/FbHI1YXsx+rrLqsDIYOVtd9RjrHhcayiUqng6+sLpVLZZ3+Zs8z97+9x7GIj3s+cgtmxIS6pAyGDlbXfUbr2cACF0hlEQvqNQmsAmYY90FgtQuxHodWVQQ8U5QGXywAHHznzwx6opUWI3Si0uvplP/D9BuDDu4D3pwOKzUBLg0M2zQ97oJYWIXaj0OrKKwiY8DtAKAHqTwJfPw+8PQbYnglUfgXoe77esS/U0iKk/wbVZTyDQnAs8Lu/Am3XgRP/AI59ClwpA04XGicvORD3CDDpUWBEjE2bNrW0lG1aNKt18JLQr58QW1FLqyfufsC0J4Al+4CnFEDSMsAjEGiuA77/E7B5GvCX/wBKtwHtKqs26S0VwUdqDKqLV+lyHkLsQeO0bKHTAGe+AY79zfjKOh5z7+YO3PYgMPlRIHI6IOj5/4J7//RvVNSo4CbgcFuoD6ZE+GFKpB+mRMgQJnMHx3H21Y2QIc7a7yiFlr2a6oCfthsPHxsqb8yXRRrDKy4DkHW/Fc4/Si/h9a9Oo75J3W1ZkLcEUyL8MDXSD1MiZRgf6gupSNj/uhIyBFBodeG0EfGMAZd+AMr/Bhz/B6Bp6ljAAaN/awywsXMAkXunVRguN7ah7GIjyi5cR9nF66i4ooKuy0MvxEIBbgv1MYZYhDHIQnzdQcjNiEKriwG5jEfTCpz6Ejj2P8D5f9+YL/U1npGc/CgQOhmwcAjYptHj+GUlSjtC7NjF62ho1nQrF+Ir7TicNB5Sjg/1hdiNuibJ0Eeh1cWAX3t4/TxQXmCclNU35gfdBtyWBkh9AIEbIBB2vLqZ/cw4AX5tMeBMQyt+/rUdlfWt+OWaGlqDADoIoYfxVSB0wy1yGW4b6YfbwvwRGx6AQJkPIPIAhCKLATns6NRAUw2g1wJ+owDhEDlry5jxxE/tcUDdBMgnAAG39tpnOpRRaHXhsgumDQag6gBQ/ilQ8X+AvntfltN2DQH0QimYyB1CsQeEYg9AJDUGmsjdeAJB1GXi53Uq6ya9MU/sAUhlgLsMEHu7/gukaQFUNYDqMqC6cuO1qdO8ll9vlBeKgcAxgPw2438g8vHGV59Q1wa8Xgs0/GwMqNrjQN0JoPYE0NplYLPYyzgsJyTuxhQ4ZugEcS8otLoYDHd5QFujcezXpaPGP1KDrmPSd3pvaeq63PgzM+hg0Otg0Bm3xTE93KAfuM/DCYyHvqYQs+VV4tN74DEGqFXmQWT2viOU2hutq6tQbGzJansYaiKV3Qgw+W1A0HggaJyxRexordduhFLtcaDuOPBrJaDv3h0ATmBsXUm8gboKQGdhYLKb1NgKM4VY6CRgxDjATez4ujsRhVYXgyK0BkBLuxbnaq+hqqYBF+uv4kr9NdRcvQ6lSgUJNJBCDSm0cIca7pwGUmggE2kR4skh2N2AAIkB/mI9fN10kHJqcNp2QNtmnHRtgLoZaFda/vLYghMYg8sszHyNwW5qKWmarduWyBPwDTO2lrxDja8+oYBP2I1XD39jEDZeAOorjAFQf9L4evXsjeErXflGGEOMD7TxxhARivqul8EAXPvFGEp8QJ0whq0lEh/j9uUTjK2p4AnG8BEbn+QEvQ64egao+RG4Um58rf3J8u9JIDLWOyQOCJlknOS3mZ0QGmwotLoYLqHVkzaNHud+bcbZ+macqW/CmTrj+/NXW2Do4S/AUyzErXJvRAd5GSe5F6ICPBHgKYG3mw4CtdIYMu2N1r22XTe+19nwlG2prFP4mIIoxDyUJD79O7TTthsPzeorgLqTN0Kt6Yrl8kIxEBhj3iobEWMcBtM5oOorem7ZySI7gim2I6QmGOfZ+jlMwVhT3jH9aJzald3LckJgxNgbrbGQOOO+JV627dMSxm4cBTB9p/cG43uPgD6DnkKri+EeWj1p1+px/moLztQ140x9M852BFpVQ0u3IRidCQUcZO4i+HmK4echgp+H2DiZfvYUw99DDD/PG8t83UUQCDhjSFgKtnalsbXFt5hCALHnwPwiLGm9BtSf6giyjlZZ/alOw1qs4OZuPMwMngAETzSGhHy8cw47TUwtSlOAXSk3BlrrVQuFOWPLUezZKXA6QofpjaHIv++8vEs4oY8YWfqdMSR7QaHVBYWWbbR6Ay50CrMz9c04U9eEi9da0aqxr99MwAG+fNCZJhH8PcWQdbz3lLhBKhJCKhIYX906ve80XyR00QkAxoDGi91bZVfPAJ4jzA/t5LFAwC3GM8KuxpjxsNsUZKZWWVON8/fNCYDFe43DfXpBodUFhZbjqHV6NLZqca1Fg+utGlxv0Xa8anC9teN9x8/XWjVobNGiSW3/3TEsEQo4SN0EcBcLIbEUbG5CSMXmoechFsJb6gYviQjeUreOyXg9qLfUOM9DLLTvUirGhubwkqY6YytSrzOeGBG4GQ8jTUNxOGEP8wVdynRM3dYVWv17sfY7OvTPk5IBJ3ETQu4jhNxHavU6Gp0BjW09BJwp3Fq1aFHr0K4zQK3Vo12rR7vWgHbdjfcmegNDi0aPFjtbfT0RCjh4SW4EmrfUzSzUvM3emy/3dTdOQ+rSK2+5cRpCKLTIgBC7CRDkLUWQt/VB1xVjDGqd4UaYafVo1+nRprkRburOy7R6tHUp19SuQ1O7Fqp2Hf/e9GpgxjBUtmmhbNMCsO8MqcRNwAeYzMP46uMugsxd3DHfDbKOPj6fTmV83UWuO+wdQii0yJDBcRx/COhojDG0dgm1ZrV5qDV1BJ3KwjzTOnqDMVjrm9QWL4rvi6dY2C3MZO5ieErcIBJycBNycBMI4Cbg4CYUGOcJOAiFAog65hmXGcuJhByEAg4i4Y11TMtFQoFxmUAAqUgAScehtVgoGNR3G7ErtDZv3ow333wTtbW1iIuLw8aNG5GQkNBj+Z07d2Lt2rU4f/48oqOj8frrr+Pee+/llzPGkJeXhw8//BCNjY2YPn063n//fURHR/Nlrl27huXLl+PLL7+EQCBAeno6/vSnP8HLywGna8mwx3EcPCVu8JS4IdjXvtYgYwzNah0aW40tNVWbFo0drTZlm7bLfI3ZvKZ2Y5+f6ZD3itKGYSEOJuAAqUgI947/ICQiAf9e2vFewi839h+6izvKdvQz3phnXB470hfeUivGtlnB5tDasWMHcnJysGXLFiQmJmLDhg1ITU1FZWWlxUfUHzp0CBkZGcjPz8d9992HgoICpKWloaysDBMmTAAAvPHGG3jvvffw8ccfY9SoUVi7di1SU1NRUVEBqdT4B5SZmYmamhoUFRVBq9Vi4cKFWLJkCQoKCvr5KyDEMTiO6+jnEqH7TYl6pzcwqDoHXKf3ylYNmtV66A0GaPUMOoMBegMzvtcboDUw6Dvmm5br9Aw6g3G58ZVB2zHfuK6BX67VM6h1en68noEBrRq93WeJLfln9nTEhcscsi2bzx4mJiZi2rRp2LRpEwDAYDAgPDwcy5cvx5o1a7qVnz9/PlpaWlBYWMjPu/322zFp0iRs2bIFjDGEhobi2WefxapVqwAASqUScrkc27ZtwyOPPIJTp07htttuw9GjRxEfHw8A+Oqrr3Dvvffi0qVLCA0N7bPedPaQkJ4xZgzBNu2NfsE2vl9Qb96P2LW/sNPyzuuoO/oZ2zR6/PmxqRg9ovejIqecPdRoNCgtLUVubi4/TyAQICUlBQqFwuI6CoUCOTk5ZvNSU1PxxRdfAACqqqpQW1uLlJQUfrmvry8SExOhUCjwyCOPQKFQQCaT8YEFACkpKRAIBDhy5Ajmzp3bbb9qtRpq9Y0+BZXKulsiEzIccRwHsRtnvM2Ru2MO45zFplMVDQ0N0Ov1kMvNT5HK5XLU1tZaXKe2trbX8qbXvsp0PfR0c3ODv79/j/vNz8+Hr68vP4WH29pgJ4QMRjft+dXc3FwolUp+qq6u7nslQsigZ1NoBQYGQigUoq6uzmx+XV0dgoODLa4THBzca3nTa19l6uvrzZbrdDpcu3atx/1KJBL4+PiYTYSQoc+m0BKLxZg6dSqKi4v5eQaDAcXFxUhKSrK4TlJSkll5ACgqKuLLjxo1CsHBwWZlVCoVjhw5wpdJSkpCY2MjSktL+TJ79+6FwWBAYmKiLR+BEDLUMRtt376dSSQStm3bNlZRUcGWLFnCZDIZq62tZYwx9thjj7E1a9bw5b///nvm5ubG3nrrLXbq1CmWl5fHRCIRO378OF9m/fr1TCaTsX/+85/sp59+Yg8++CAbNWoUa2tr48vMmjWLTZ48mR05coQdPHiQRUdHs4yMDKvrrVQqGQCmVCpt/ciEkAFg7XfU5tBijLGNGzeyiIgIJhaLWUJCAjt8+DC/LDk5mWVlZZmV/+yzz1hMTAwTi8Vs/PjxbPfu3WbLDQYDW7t2LZPL5UwikbCZM2eyyspKszJXr15lGRkZzMvLi/n4+LCFCxeypqYmq+tMoUXI4Gbtd5Tu8kAIGRToLg9dmLKZxmsRMjiZvpt9taOGTWg1NRnvNknjtQgZ3JqamuDr69vj8mFzeGgwGHDlyhV4e3v3eQW7SqVCeHg4qqurh9yhJNXdNaju/ccYQ1NTE0JDQyHo5UlNw6alJRAIMHLkSJvWGcrju6jurkF175/eWlgmN+2IeELIzYlCixAypFBoWSCRSJCXlweJROLqqtiM6u4aVPeBM2w64gkhNwdqaRFChhQKLULIkEKhRQgZUii0CCFDCoUWIWRIodDqYvPmzYiKioJUKkViYiJKSkpcXaU+5efnY9q0afD29kZQUBDS0tJQWVnp6mrZZf369eA4DitWrHB1Vaxy+fJlPProowgICIC7uztiY2Pxww8/uLpafdLr9Vi7di1GjRoFd3d33HLLLXj11Vf7vFh5MKDQ6sT0TMe8vDyUlZUhLi4Oqamp3W71PNgcOHAA2dnZOHz4MP9cyHvuuQctLS2urppNjh49ij//+c+YOHGiq6tilevXr2P69OkQiUT417/+hYqKCrz99tvw8/NzddX69Prrr+P999/Hpk2bcOrUKbz++ut44403sHHjRldXrW9OvavXEJOQkMCys7P5n/V6PQsNDWX5+fkurJXt6uvrGQB24MABV1fFak1NTSw6OpoVFRWx5ORk9swzz7i6Sn1avXo1+81vfuPqathlzpw57L/+67/M5j300EMsMzPTRTWyHrW0Opie6dj5+Yt9PdNxsFIqlQAAf39/F9fEetnZ2ZgzZ47Z73+w+7//+z/Ex8fj4YcfRlBQECZPnowPP/zQ1dWyyh133IHi4mL8/PPPAIAff/wRBw8exOzZs11cs74Nm7s89KW3ZzqePn3aRbWyncFgwIoVKzB9+nRMmDDB1dWxyvbt21FWVoajR4+6uio2+eWXX/D+++8jJycHzz//PI4ePYqnn34aYrEYWVlZrq5er9asWQOVSoWxY8dCKBRCr9fjj3/8IzIzM11dtT5RaN1ksrOzceLECRw8eNDVVbFKdXU1nnnmGRQVFUEqlbq6OjYxGAyIj4/Ha6+9BgCYPHkyTpw4gS1btgz60Prss8/w6aefoqCgAOPHj0d5eTlWrFiB0NDQQV936tPqoFarmVAoZLt27TKbv2DBAvbAAw+4plI2ys7OZiNHjmS//PKLq6titV27djEATCgU8hMAxnEcEwqFTKfTubqKPYqIiGCLFi0ym/ff//3fLDQ01EU1st7IkSPZpk2bzOa9+uqrbMyYMS6qkfWoT6uDPc90HCwYY1i2bBl27dqFvXv3YtSoUa6uktVmzpyJ48ePo7y8nJ/i4+ORmZmJ8vJyCIVCV1exR9OnT+82tOTnn39GZGSki2pkvdbW1m53BxUKhTAYDC6qkQ1cnZqDSV/PdBysnnrqKebr68v279/Pampq+Km1tdXVVbPLUDl7WFJSwtzc3Ngf//hHdubMGfbpp58yDw8P9re//c3VVetTVlYWCwsLY4WFhayqqop9/vnnLDAwkD333HOurlqfKLS66O2ZjoMVAIvT1q1bXV01uwyV0GKMsS+//JJNmDCBSSQSNnbsWPbBBx+4ukpWUalU7JlnnmERERFMKpWy0aNHsxdeeIGp1WpXV61PdD8tQsiQQn1ahJAhhUKLEDKkUGgRQoYUCi1CyJBCoUUIGVIotAghQwqFFiFkSKHQIoQMKRRahJAhhUKLEDKkUGgRQoaU/w88s50UwxpUUwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:10:51.450573Z", + "iopub.status.busy": "2024-03-22T23:10:51.449898Z", + "iopub.status.idle": "2024-03-22T23:15:14.123422Z", + "shell.execute_reply": "2024-03-22T23:15:14.122618Z" + }, + "papermill": { + "duration": 262.694118, + "end_time": "2024-03-22T23:15:14.125906", + "exception": false, + "start_time": "2024-03-22T23:10:51.431788", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:15:14.164495Z", + "iopub.status.busy": "2024-03-22T23:15:14.163786Z", + "iopub.status.idle": "2024-03-22T23:15:14.184334Z", + "shell.execute_reply": "2024-03-22T23:15:14.183490Z" + }, + "papermill": { + "duration": 0.042063, + "end_time": "2024-03-22T23:15:14.186171", + "exception": false, + "start_time": "2024-03-22T23:15:14.144108", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.00.0139960.0044811.700660.1057611.9551250.1900060.0000566.2541420.043232659710.750.0669330.2451861.028312e-0717.954802
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.0 0.013996 0.00448 11.70066 0.105761 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 1.955125 0.190006 0.000056 6.254142 0.04323 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 2659710.75 0.066933 0.245186 1.028312e-07 17.954802 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:15:14.220065Z", + "iopub.status.busy": "2024-03-22T23:15:14.219810Z", + "iopub.status.idle": "2024-03-22T23:15:14.706701Z", + "shell.execute_reply": "2024-03-22T23:15:14.705713Z" + }, + "papermill": { + "duration": 0.506484, + "end_time": "2024-03-22T23:15:14.708914", + "exception": false, + "start_time": "2024-03-22T23:15:14.202430", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:15:14.746946Z", + "iopub.status.busy": "2024-03-22T23:15:14.746089Z", + "iopub.status.idle": "2024-03-22T23:20:13.576108Z", + "shell.execute_reply": "2024-03-22T23:20:13.575282Z" + }, + "papermill": { + "duration": 298.852041, + "end_time": "2024-03-22T23:20:13.579170", + "exception": false, + "start_time": "2024-03-22T23:15:14.727129", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/lct_gan/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/lct_gan/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:20:13.624273Z", + "iopub.status.busy": "2024-03-22T23:20:13.623956Z", + "iopub.status.idle": "2024-03-22T23:20:13.649471Z", + "shell.execute_reply": "2024-03-22T23:20:13.648761Z" + }, + "papermill": { + "duration": 0.046948, + "end_time": "2024-03-22T23:20:13.651446", + "exception": false, + "start_time": "2024-03-22T23:20:13.604498", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:20:13.686236Z", + "iopub.status.busy": "2024-03-22T23:20:13.685955Z", + "iopub.status.idle": "2024-03-22T23:20:13.691091Z", + "shell.execute_reply": "2024-03-22T23:20:13.690214Z" + }, + "papermill": { + "duration": 0.024673, + "end_time": "2024-03-22T23:20:13.693021", + "exception": false, + "start_time": "2024-03-22T23:20:13.668348", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.43215544331545797}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:20:13.734346Z", + "iopub.status.busy": "2024-03-22T23:20:13.734065Z", + "iopub.status.idle": "2024-03-22T23:20:14.164429Z", + "shell.execute_reply": "2024-03-22T23:20:14.163560Z" + }, + "papermill": { + "duration": 0.456307, + "end_time": "2024-03-22T23:20:14.166501", + "exception": false, + "start_time": "2024-03-22T23:20:13.710194", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAE8CAYAAACRnDftAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABKtklEQVR4nO3deZxU1Znw8d+tvapr6X3vptlEAVlEkMYYQI2IjIGY1ziaBFBxmcFEQ5yMZPJqJMl0xgSRmThoRqHjq2iiQUzckBDRUXEBAQERZOtuoPel9r3u+0d1lzR0N73U3uf7+dSnqepTt54qup8+99znnCPJsiwjCIKQYhSJDkAQBGEwRPISBCElieQlCEJKEslLEISUJJKXIAgpSSQvQRBSkkhegiCkJJG8BEFISSJ5CYKQkkTyEqKuuroaSZI4ceJEokMR0phIXkJS2LhxI4899liiwxBSiEheQlIQyUsYKJG8BEFISSJ5CXHxxhtvMHv2bEwmE2azmenTp7Nx40YA5syZw2uvvUZNTQ2SJCFJEhUVFf0+digU4uc//znFxcUYDAbmzp3L559/TkVFBUuXLo20a2tr4/777+fiiy/GaDRiNpuZP38+e/fu7Xa87du3I0kSf/rTn/jVr35FaWkpOp2Oq666iiNHjkTj4xCiQJXoAIT0V11dzW233caECRNYuXIlmZmZ7N69mzfffJNbbrmFf/u3f8NqtXLy5EnWrFkDgNFo7PfxV65cySOPPML111/PvHnz2Lt3L/PmzcPj8XRrd+zYMTZv3syNN97IyJEjaWxs5Mknn2T27Nl8/vnnFBcXd2v/61//GoVCwf3334/VauWRRx7hu9/9Lh999NHQPxRh6GRBiLINGzbIgHz8+HG5o6NDNplM8mWXXSa73e5u7UKhUOTfCxYskEeMGDHg12poaJBVKpW8aNGibo///Oc/lwF5yZIlkcc8Ho8cDAa7tTt+/Lis1WrlVatWRR57++23ZUC+6KKLZK/XG3l87dq1MiDv27dvwHEK0SdOG4WY2rp1K3a7nQceeACdTtfte5IkDfn427ZtIxAI8M///M/dHv/BD35wTlutVotCEf6RDwaDtLa2YjQaGTduHJ9++uk57W+99VY0Gk3k/hVXXAGEe3BC4onkJcTU0aNHAZg4cWJMjl9TUwPAmDFjuj2enZ1NVlZWt8dCoRBr1qxh7NixaLVacnNzycvL47PPPsNqtZ5z7PLy8m73u47X3t4ezbcgDJJIXsKw8e///u+sWLGCr3/96zz77LNs2bKFrVu3MmHCBEKh0DntlUplj8eRxcrpSUEM2AsxNXr0aAD2799/Tu/oTIM9hRwxYgQAR44cYeTIkZHHW1tbz+khvfTSS8ydO5enn3662+MdHR3k5uYO6vWFxBE9LyGmrrnmGkwmE1VVVedc/TuzB5ORkdHjqdv5XHXVVahUKtatW9ft8d/97nfntFUqlef0ml588UVOnTo14NcVEk/0vISYMpvNrFmzhmXLljF9+nRuueUWsrKy2Lt3Ly6Xiz/84Q8ATJs2jT/+8Y+sWLGC6dOnYzQauf766897/IKCAu69915Wr17NN7/5Ta699lr27t3LG2+8QW5ubrce3T/8wz+watUqbr31VmbNmsW+fft47rnnGDVqVMzevxBDCb7aKaShM0sluvzlL3+RZ82aJev1etlsNsszZsyQn3/++cj3HQ6HfMstt8iZmZkyMKCyiUAgIP/f//t/5cLCQlmv18tXXnmlfPDgQTknJ0e+++67I+08Ho/84x//WC4qKpL1er18+eWXyzt27JBnz54tz549O9Kuq1TixRdf7PY6x48flwF5w4YNA/1IhBiQZFmMPgrpp6Ojg6ysLH75y1/yb//2b4kOR4gBMeYlpDy3233OY12TvOfMmRPfYIS4EWNeQtJqbm4mGAz2+n2NRkN2djZ//OMfqa6u5rrrrsNoNPLee+/x/PPPc80113D55ZfHMWIhnkTyEpLW9OnTI0WoPZk9ezbbt29n0qRJqFQqHnnkEWw2W2QQ/5e//GUcoxXiTYx5CUnr/fff7/GUsEtWVhbTpk2LY0RCMhHJSxCElCQG7AVBSEnDbswrFApx+vRpTCZTVFY1EAQhumRZxm63U1xcHFkFpCfDLnmdPn2asrKyRIchCMJ51NXVUVpa2uv3h13yMplMQPiDMZvNCY5GEISz2Ww2ysrKIr+rvRl2yavrVNFsNovkJQhJ7HzDOmLAXhCElCSSlyAIKUkkL0EQUlJCx7zWrVvHunXrOHHiBAATJkzgwQcfZP78+T22r66u5tZbb+32mFarPWeRu6GSZZlAINDnvDqhZ0qlEpVKJcpQhJhLaPIqLS3l17/+NWPHjkWWZf7whz+wcOFCdu/ezYQJE3p8jtls5tChQ5H70f4l8fl81NfX43K5onrc4cRgMFBUVNRt5x1BiLaEJq+zV8r81a9+xbp16/jwww97TV6SJFFYWBiTeEKhEMePH0epVFJcXIxGoxE9iAGQZRmfz0dzczPHjx9n7NixfRYZCsJQJE2pRDAY5MUXX8TpdFJZWdlrO4fDwYgRIwiFQlxyySX8+7//e6+JDsDr9eL1eiP3bTZbr219Ph+hUIiysjIMBsPg3sgwp9frUavV1NTU4PP5ztmrMS2d+hQaD0DOGCifCeIPXlwk/M/ivn37MBqNaLVa7r77bl5++WXGjx/fY9tx48axfv16XnnlFZ599llCoRCzZs3i5MmTvR6/qqoKi8USufWnul70FoZmWH1+LUfg8BawnoRj26Hhs0RHNGwkfFUJn89HbW0tVquVl156iaeeeop33nmn1wR2Jr/fz0UXXcTNN9/ML37xix7b9NTzKisrw2q1nlOk6vF4OH78OCNHjhwePYYYGTafoyzDzqfB0QwqLQS8oLPAZXfDcErgUWaz2bBYLD3+jp4p4Z+wRqNhzJgxTJs2jaqqKiZPnszatWv79Vy1Ws3UqVM5cuRIr220Wm2kml5U1QtRZa8PJy6FCmbcAWodeKxgrUt0ZMNCwpPX2UKhULeeUl+CwSD79u2jqKgoxlEJZ6qoqIisET+stXwZ/po7BrQmQjljkZGh+VDfzxOiIqED9itXrmT+/PmUl5djt9vZuHEj27dvZ8uWLQAsXryYkpISqqqqAFi1ahUzZ85kzJgxdHR08Jvf/IaamhqWLVuWyLchDFOh1mMgyyiyR+PwBni/0cQlDh95Hb0vXS1ET0KTV1NTE4sXL6a+vh6LxcKkSZPYsmUL3/jGNwCora3tNvjb3t7OHXfcQUNDQ2QJ4A8++KBf42NCdz6fT9RhDYXPRXN9DXVtTlQjsvB7bRz2ZFLU4SbP2AJ+N6j1iY4yrSU0eT399NN9fn/79u3d7q9Zs4Y1a9bEMKLe+QKhXr+nkEClVPSrrSSBuh9tNaqBndHPmTOHiRMnAvD//t//Q61W80//9E+sWrUKSZKoqKjg9ttv58svv2Tz5s3ccMMNVFdX895777Fy5Up27txJbm4u3/rWt6iqqiIjIwMI/4G5/fbb+dvf/kZhYaHY1KKLvZ4WuwebwoJJmcHFJRZ2HG2lRTbh9gfRW0+FTyeFmEmaOq9k9/jbvV8UGJmbwaKpJZH7v3/3KP5gzxdxS7P03HjpV+Ua698/jtt37jSkH33jggHH+Ic//IHbb7+djz/+mJ07d3LnnXdSXl7OHXfcAcBvf/tbHnzwQR566CEAjh49yrXXXssvf/lL1q9fT3NzM/fccw/33HMPGzZsAGDp0qWcPn2at99+G7VazQ9/+EOampoGHFu68bWfxO4J4MjIY1q+CZ1aSVm2HntzAW3OOkpsInnFmkheaaSsrIw1a9YgSRLjxo1j3759rFmzJpK8rrzySn784x9H2i9btozvfve73HfffQCMHTuW//zP/2T27NmsW7eO2tpa3njjDT7++GOmT58OhHvLF110UdzfW7Jpa6hBBhTmIiwGNQAjc418fjwXm/sYJc7mxAY4DIjk1U/L5/b+V1RxVkH1nV8f3Wvbs4uvb7t85FDC6mbmzJndpjNVVlayevXqyATzSy+9tFv7vXv38tlnn/Hcc89FHpNlOTJN6vDhw6hUqm7bi1144YVkZmZGLeaUJMs4msPlENmFIyIPl2Tq2aXOwm4NELI3Jt+l/DQjklc/DWQMKlZth6prHKuLw+Hgrrvu4oc//OE5bcvLyzl8+HC8QkstXjtupx0ZidzCr4YAcjI0BAy5BEMyblsbGQFvuHhViAmRvNLIRx991O3+hx9+yNixY1EqlT22v+SSS/j8888ZM6bnXuWFF15IIBBg165dkdPGQ4cO0dHREdW4U03A3oTLF8CjzqQg86t11hUKicKcLNRNFvyhEDibwdL7BhLC0IiebRqpra1lxYoVHDp0iOeff57/+q//4t577+21/b/+67/ywQcfcM8997Bnzx6+/PJLXnnlFe655x4gPJf02muv5a677uKjjz5i165dLFu2DL1+eJcABJ0t5Jt1GLMLMOu7//3/5uRiLh0/lky9BlytCYpweBDJK40sXrwYt9vNjBkzWL58Offeey933nlnr+0nTZrEO++8w+HDh7niiiuYOnUqDz74IMXFxZE2GzZsoLi4mNmzZ3PDDTdw5513kp+fH4+3k7S03nZG5mQwY/wF5yyZJEkS6LPCd9ztCYhu+BCnjWlErVbz2GOPsW7dunO+17Va7dmmT5/OW2+91esxCwsLefXVV7s99v3vf39IcaY8Z0v4a0Zuz9/XZyPLMrjaEIvjxI7oeQnCQMgyzvYGAqEQGHpOXm8c9fDxiTZc1pY4Bze8iOQlCAMg+5wcqG3ikxMdtIYyemzjUpkJyeC2toSXzRFiQpw2pomzp1IJseHqaCIYkvGqTFiMPa+2m2HJASS8Xg/4HKDte+dnYXBEz0sQBsDWFp4aJWXkdJvPeqbMDD1elRG3PygG7WNIJC9BGACnLTyOpTFm99omy6DBqzLh8QfDixMKMSGSlyAMgNsWrt3Sm3N6bZNlUONVGjuTV+8bvghDI5KXIAyAz94GQEZmL2USQGZnz8sflPE5O+IU2fAjBuwFYQACzvAYljkzr9c2GpWCzKwcsgMagu6OOEU2/IjkJQj9JPvdlJnA69diyeq95wXwjaljYe8uCImd12NFJC9B6CfJY6PYogeNAfTn2ZRYZwl/9VjDtV5iI9qoE2NegtBfno7wV13m+dtqTcjIBAM+CHhiGdWwldDktW7dOiZNmhTZT7GyspI33nijz+e8+OKLXHjhheh0Oi6++GJef/31OEUrDHeOjhacvgB+zfn3/jzS6mFHnYfDDXZxxTFGEpq8SktL+fWvf82uXbvYuXMnV155JQsXLuTAgQM9tv/ggw+4+eabuf3229m9ezeLFi1i0aJF7N+/P3ZByjIEfIm59XNqyTPPPENOTs45+10uWrRITKKOouMnT/HZSSuft/e8PtqZtColLsmANxAEr0hesZDQMa/rr7++2/1f/epXrFu3jg8//JAJEyac037t2rVce+21/Mu//AsAv/jFL9i6dSu/+93veOKJJ2ITZNAP/7s6Nsc+nyt+DKrzb09244038sMf/pC//OUv3HjjjUB415/XXnutzxUjhIHxO8JlEjpT7wWqXcy6cK2X19OC7LGK1SViIGnGvILBIC+88AJOp5PKysoe2+zYsYOrr76622Pz5s1jx44dvR7X6/Vis9m63dKNXq/nlltuiez4A/Dss89SXl7OnDlzEhdYmgm6wmUSfRWodjHqVPhVGYRk8Dg6YhzZ8JTwq4379u2jsrISj8eD0Wjk5Zdf7nUT2YaGBgoKCro9VlBQQENDQ6/Hr6qq4uGHHx58gEp1uAeUCEp1v5vecccdTJ8+nVOnTlFSUkJ1dTVLly49Z7E8YXDkUIigOzzVx2g5f89LqZBQ6kxgB7fTxvBeezY2Ep68xo0bx549e7Barbz00kssWbKEd955J2q7YK9cuZIVK1ZE7ttsNsrKyvp4xlkkqV+nbok2depUJk+ezDPPPMM111zDgQMHeO211xIdVtrwuJ3IwQAAJktWv56jzbBAM3idYn5jLCQ8eWk0msgGENOmTeOTTz5h7dq1PPnkk+e0LSwspLGxsdtjjY2NFBYW9np8rVaLVjs8dnBZtmwZjz32GKdOneLqq68eWJIW+mTvCI93KTQZqNX9+2OmzQjXenld6TdUkQySZsyrSygUOueqWZfKykq2bdvW7bGtW7f2OkY23Nxyyy2cPHmS//mf/+G2225LdDhpxWkPj3cpDOcvk+iSm5NNlkGNQXbHKqxhLaE9r5UrVzJ//nzKy8ux2+1s3LiR7du3s2XLFiC8oURJSQlVVVUA3HvvvcyePZvVq1ezYMECXnjhBXbu3Mnvf//7RL6NpGGxWPj2t7/Na6+9xqJFixIdTlrJUnrQZhsI5vR/85EJFSVw0gzIEAyAMuEnOmkloZ9mU1MTixcvpr6+HovFwqRJk9iyZQvf+MY3gPBWXgrFV53DWbNmsXHjRn72s5/x05/+lLFjx7J582YmTpyYqLeQdE6dOsV3v/vdYXOqHC+ZSg+ZmXooLDh/4y4qHShUEAqEV1TVZ8YsvuEoocnr6aef7vP7PS1tfOONN0ZqmYSvtLe3s337drZv385///d/Jzqc9OO1h78OZElnSULWGAi6OlCJ5BV1oh+bJqZOnUp7ezv/8R//wbhx4xIdTtppb29F4wugUWfQ3wIWjz/I+8dcGDztzJhoR2mJaYjDjkheaaK3fRmF6PjixGlwWxl9kZb+jnppVQp8ygwMgNthxTiAM07h/JLuaqMgJBs5FELuXIteZ+pfjReEd89W6sKnmR6nKJeINpG8eiCLvfaGJN0+P5fLgSQHkYCMASQvAJU+nLxEoWr0ieR1BrU6PJrhconVL4ei6/Pr+jxTndveAYCkyUCpGth7UnfWhflEoWrUiTGvMyiVSjIzM2lqCu/NZzAYxNzAAZBlGZfLRVNTE5mZmSiV5186JhW4HOECVUk38M1jtQYzQSDgFskr2kTyOkvXVKOuBCYMXGZmZp9TtlKNxx4+5VPqB365UJthxgUEPM4oRyWI5HUWSZIoKioiPz8fv9+f6HBSjlqtTpseVxefqwP46hRwILItWWgNaswqP4RCoBAjNdEiklcvlEpl2v0SCoNTrAvgyTagyO19u7PelBXkQJElvCqu3wVaYwwiHJ5E8hKE88hWeSBTD/kDT14oFKDWg88lkleUiT6sIJxPZGrQwE8bAWS1gUAoRMgrxr2iSSQvQeiLLNPW3orTGyCkGfjVRoB3Tzj55EQ7Hdb2KAc3vInTRkHog9fj5NCp8EKE01QZDGpNXU34VNHrskcvMEH0vAShL05buLckq/VoBrnMkEqXAYDP5YhaXIJIXoLQJ3fnzj+KQRSodlF1LqPjc4ueVzSJ5CUIffBEktfgBusB1IZwz8vvEckrmkTyEoQ++DqTl2oQ1fVdNPpw4guKKvuoEslLEPrgc4WnBqkzMgd9DL0hfNoYFKUSUSWuNgpCH4KdE6p1Q0heZrOFQIYGvdofrrQXk/2jIqE9r6qqKqZPn47JZCI/P59FixZx6NChPp9TXV2NJEndbjqdLk4RC8PNCGOQEdkGcnJyBn2MvJxsxhWYKM/UQqDnbf2EgUto8nrnnXdYvnw5H374IVu3bsXv93PNNdfgdPbdvTabzdTX10duNTU1cYpYGFZkmRyVl+JMPdnZuYM/jlL91a7rfrFWXLQk9LTxzTff7Ha/urqa/Px8du3axde//vVenydJUlotuSIkqYAXgp0riwxk16AeyGoDQZ8HyetAaciOQnBCUg3YW63hwdHs7L7/cx0OByNGjKCsrIyFCxdy4MCBXtt6vV5sNlu3myD0h8fZQbvLhyOkCveehuCDWjefnGinta0tStEJSZO8QqEQ9913H5dffnmfm8iOGzeO9evX88orr/Dss88SCoWYNWsWJ0+e7LF9VVUVFoslcisrK4vVWxDSTEdHG1802PmsOTTkY0laMUUo2pImeS1fvpz9+/fzwgsv9NmusrKSxYsXM2XKFGbPns2mTZvIy8vjySef7LH9ypUrsVqtkVtdXV0swhfSkMcenhqk0A19w0WlNlyo6nWLKULRkhSlEvfccw+vvvoq7777LqWlpQN6rlqtZurUqRw5cqTH72u1WrSDnJMmDG9eZwcAqkGsoHo2td6IF/CLnlfUJLTnJcsy99xzDy+//DJ///vfGTly5ICPEQwG2bdvH0VFRTGIUBjO/F0Fqoah97zUnVug+T2i5xUtCe15LV++nI0bN/LKK69gMploaGgAwGKxoNfrAVi8eDElJSVUVVUBsGrVKmbOnMmYMWPo6OjgN7/5DTU1NSxbtixh70NIT4HO7cqikbx0eiMORJV9NCU0ea1btw6AOXPmdHt8w4YNLF26FIDa2loUZ2xa0N7ezh133EFDQwNZWVlMmzaNDz74gPHjx8crbGGYCHVW1+tNmUM+llZMEYq6hCav/uysvH379m7316xZw5o1a2IUkSB8JejpTF7GzCEfy2gyk52hQasNDvlYQlhSDNgLQtIJeBmVpcLrN2DJHPzUoC7ZmVlkF3QWuoaCoBA7Uw2VSF6C0BOPjZwMLah10Dn+OiRqPUgKkEOduwgNrWJfSKI6L0FIKt7OmRjRSjKShKzWEwiFCIgrjlEhkpcg9MBua6fN5cOBIWrH/OSUh09OtNPYKqYIRYM4bRSEHjQ1N9PUYEdShJgZpWMqtBkE7eBxip5XNIielyD0wOcMF6hqMoZe49Wla4qQ2IgjOkTyEoQe+N2dycuQGbVjqnThydk+sZZ9VIjkJQg96CpQ1Rqj1/PS6MM9r4CYnB0VInkJQg9CkQLVrKgdU9M5vzHgFckrGkTyEoSzBbwEfB4ADOboJS+tXkwRiiZxtVEQzuJ1thMMyQQVGjIMUShQ7WQ0WcjO0KDWiSlC0SCSlyCcReFzMDbfiEuTjVYVvWk8ZrMZc4EJlGLrs2gQyUsQzqL2O8g1aiEnypu8aMID9gT94dsQ18Uf7sSYlyCcLTI1aOgrqHaj1CArlGKKUJSI5CUIZ+lobw1PDZKiN94FgCSxp97LJyfaOdXUEt1jD0ODSl7Hjh2LdhyCkDRONzZxqMHOkY7oj01JkY04xBXHoRpU8hozZgxz587l2WefxePxRDsmQUioQGd1vTYKK6ieTakTW6BFy6CS16effsqkSZNYsWIFhYWF3HXXXXz88cfRjk0Q4k+WCXnCiUUXhRVUz6bu7Hn5RZX9kA0qeU2ZMoW1a9dy+vRp1q9fT319PV/72teYOHEijz76KM3NzdGOUxDiI+Al2FmgmhHF6vouKn3X/EaRvIZqSAP2KpWKG264gRdffJH/+I//4MiRI9x///2UlZWxePFi6uvr+3x+VVUV06dPx2QykZ+fz6JFizh06NB5X/fFF1/kwgsvRKfTcfHFF/P6668P5W0IQkTQbcUflAkodBgzojxgz1dThIIieQ3ZkJLXzp07+ed//meKiop49NFHuf/++zl69Chbt27l9OnTLFy4sM/nv/POOyxfvpwPP/yQrVu34vf7ueaaa3A6ex/M/OCDD7j55pu5/fbb2b17N4sWLWLRokXs379/KG9FEABw2duRAb86A706+uvMaw3hnldATBEaOnkQVq9eLU+cOFFWq9XywoUL5b/+9a9yMBjs1qaurk5WKpUDOm5TU5MMyO+8806vbb7zne/ICxYs6PbYZZddJt911139eg2r1SoDstVqHVBswvDQ+MUO+YOnfiy//eLvYnJ8+6mD8qHnV8qHX10bk+Ong/7+jg6qwn7dunXcdtttLF26tNedqvPz83n66acHdFyrNXyVJzs7u9c2O3bsYMWKFd0emzdvHps3b+6xvdfrxev1Ru7bbLYBxSQML2bJxdh8I578kpgc32i0cEGBCbSixHKoBpW8tm7dSnl5ebfNYCG8D2NdXR3l5eVoNBqWLFnS72OGQiHuu+8+Lr/8ciZOnNhru4aGBgoKCro9VlBQENlt+2xVVVU8/PDD/Y5DGN50QRc6oxYK82PzAprONfH9LpBlkMQ8x8EaVPofPXo0LS3nVgi3tbUxcuTIQQWyfPly9u/fzwsvvDCo5/dm5cqVWK3WyK2uri6qxxfSTKymBnVRG5CR8fv9+H3u2LzGMDGonpfcy07XDocDnU434OPdc889vPrqq7z77ruUlpb22bawsJDGxsZujzU2NlJY2PMkWq1Wi1arHXBMwvDU2tqC5PGhkwxR3DfoDEo1+xvcOFxuCi5oYVR5eSxeZVgYUPLqGmuSJIkHH3wQg+Gr/95gMMhHH33ElClT+n08WZb5wQ9+wMsvv8z27dv71WurrKxk27Zt3HfffZHHtm7dSmVlZb9fVxB6JMucamzC6fZQ4FUxKkYvI2kywOUWuwgN0YCS1+7du4Fw0tm3bx8ajSbyPY1Gw+TJk7n//vv7fbzly5ezceNGXnnlFUwmU2TcymKxoO/cpXjx4sWUlJRQVVUFwL333svs2bNZvXo1CxYs4IUXXmDnzp38/ve/H8hbEYRz+d0E/D4ADKbeLxoNlVKXAbTgFbsIDcmAktfbb78NwK233sratWsxm4c2LrBu3ToA5syZ0+3xDRs2sHTpUgBqa2u7XRiYNWsWGzdu5Gc/+xk//elPGTt2LJs3b+5zkF8Q+iPkseELhPAr9BgNAx/+6K/ILkIu0fMaikGNeW3YsCEqL97b2NmZtm/ffs5jN954IzfeeGNUYhCELi57W2eBqhFDDApUu6g7k5dfVNkPSb+T1w033EB1dTVms5kbbrihz7abNm0acmCCEG8ee3v4HzoLCkXsShjUnfMbxYKEQ9Pv5GWxWJA6a1IslujtZScIycJrbwNAGcWNZnuiNXRugSaS15D0O3mdeaoYrdNGQUgmPme456UyRH81iTMZMszkZGjIMsb0ZdLeoMa83G43sixHSiVqamp4+eWXGT9+PNdcc01UAxSEeCnUeDHlG/GX9TzlLVoMGcbwFKEMMUVoKAb16S1cuJBnnnkGgI6ODmbMmMHq1atZuHBh5AqiIKQafchJrlFLUUGUdw06m7pzFyG/qLAfikGvpHrFFVcA8NJLL1FYWEhNTQ3PPPMM//mf/xnVAAUhLgK+r5KJLsZjuprOKUIeBz6/2IB2sAaVvFwuFyZTeNDxrbfe4oYbbkChUDBz5kxqamqiGqAgxIXHSrPdS6tPgY8Y76eoNnCw3s7OE22caGiN7WulsUFvwLF582bq6urYsmVLZJyrqalpyIWrgpAIsqeDYy0OdjfJeAIx7g0plCg04RkkHpdYommwBpW8HnzwQe6//34qKiq47LLLIvMK33rrLaZOnRrVAAUhHjz2NkIy+FRGMjSx30he2bUFmqiyH7RB/S/9n//zf/ja175GfX09kydPjjx+1VVX8a1vfStqwQlCvHgcHQBIOgvKGBaodumaIiTmNw7eoP/EFBYWnrMMzYwZM4YckCAkgqezQFUV4wLVLipduOcV8IjkNViDSl5Op5Nf//rXbNu2jaamJkKhULfvix21hVTjc4QLVDXG2K0mcaauXYT8YufsQRtU8lq2bBnvvPMO3//+9ykqKopMGxKEVOV3hZNXLHbJ7omms+cV9Ioxr8EaVPJ64403eO2117j88sujHY8gxF/QT9AT7gEZTDlxeUm90UROhgazSfzhH6xBJa+srKw+d/gRhJTisVKapSfbbEKdH9t5jV10BnN4ilBm7K9spqtBlUr84he/4MEHH8TlckU7HkGIP3cHGRoVeXl5ZGbEab8DdecS6j7xOzRYg0r7q1ev5ujRoxQUFFBRUYFa3b0i+dNPP41KcIIQF56O8Fd9fHpdAGgyOqcI2SEQQqMSk7QHalDJa9GiRVEOQxASx21rocPqRpGpp+D8zaNDbeCLBjsdrjYso62ML41j4kwTg0peDz30ULTjEISEsbc1caLVRYdBYn68XlStR6VUAn48bjsgktdADbqv2tHRwVNPPcXKlStpawsX+H366aecOnWq38d49913uf766ykuLkaSJDZv3txn++3btyNJ0jm33nbLFoT+8DnDP79qY3yuNAIgSZEpQh4xRWhQBtXz+uyzz7j66quxWCycOHGCO+64g+zsbDZt2kRtbW1kra/zcTqdTJ48mdtuu+286+Kf6dChQ90mgOfnx2hrdiH9hUIEOgtUdTHc7qwn4S3QWvG7RJX9YAwqea1YsYKlS5fyyCOPRJbGAbjuuuu45ZZb+n2c+fPnM3/+wDvq+fn5ZGZmDvh5gnAOrw2f348sKTCY43vqpu4sVPWJtewHZVCnjZ988gl33XXXOY+XlJTE5RRuypQpFBUV8Y1vfIP333+/z7ZerxebzdbtJggR7na8gRAelRmzXnP+9lGk0XVOERLJa1AGlby0Wm2PSeDw4cPk5eUNOajeFBUV8cQTT/DnP/+ZP//5z5SVlTFnzpw+SzOqqqqwWCyRW1lZWcziE1KQpwNfIIhXacKsi/EihGf5ags0Mb9xMAaVvL75zW+yatUq/H4/AJIkUVtby7/+67/y7W9/O6oBnmncuHHcddddTJs2jVmzZrF+/XpmzZrFmjVren3OypUrsVqtkVtdXV3M4hNST8DZhi8o41GbMcU5eekywlOEKsxSvzZgFrobVPJavXo1DoeDvLw83G43s2fPZsyYMZhMJn71q19FO8Y+zZgxgyNHjvT6fa1Wi9ls7nYbtoIBOLkTTrwHYkIwAApPBxOLzVwybhQ6dXwLRbX68C5Ck/LVYnGDQRjUgL3FYmHr1q28//777N27F4fDwSWXXMLVV18d7fjOa8+ePRQVxXarqrQQCsLejWDtLGU5vRsuWQK6YZzMCScvk06NqbQU4p1AunYR8onTxsEYcPIKhUJUV1ezadMmTpw4gSRJjBw5ksLCQmRZHtBfEIfD0a3XdPz4cfbs2UN2djbl5eWsXLmSU6dORUovHnvsMUaOHMmECRPweDw89dRT/P3vf+ett94a6NsYfuo+CiculQZUOvDY4PAWmHRjoiNLHFkGd7hMAl1m/F9fY0CWZfxuB3IgiFaljH8MKWxAyUuWZb75zW/y+uuvM3nyZC6++GJkWebgwYMsXbqUTZs2nbfQ9Ew7d+5k7ty5kfsrVqwAYMmSJVRXV1NfX09tbW3k+z6fjx//+MecOnUKg8HApEmT+Nvf/tbtGEIPAj6o+zj877HXgKkYPnkKWo+A7TSYixMbX6J47bTZHHhDEtqQgbivk6I2cKjRTovbTuZIBxNLYrzlWpoZUPKqrq7m3XffZdu2beckjL///e8sWrSIZ555hsWLF/freHPmzOlzoLK6urrb/Z/85Cf85Cc/GUjIAkDzwfCehPosnJnjsHtD5OVdiLLpczi9Z/gmL1crTXYv9T4DuVYf2SZ9fF9fk4FaqUApe3G63YBIXgMxoBHK559/np/+9Kc99nSuvPJKHnjgAZ577rmoBSdESdNBAI6rR/H0+zU8/3EtrzTm4guGwokt6E9wgAniasPjD+JSZ5FpiO+VRgCUGtTqcP/B4xRV9gM1oOT12Wefce211/b6/fnz57N3794hByVEkc8J7TU4vAG2NGUTDIV7ujWBXA60gRzwQvuJxMaYICFnS7hAVW3BkojkJUmoOwtVPWKK0IANKHm1tbVRUND7oiEFBQW0t7cPOSghitqOgRzikDMDtyq8euetl1egUSuplYpxeALQejTRUSaEx9oc3qtRk4VJm5gVTbsKVcXk7IEb0P9YMBhEper9KUqlkkAgMOSghChqP4EvGOKkVIhCkrjiglzMOjXXTyom21uJ8VBDeOBeluNfKpBgXlsTAGpzXsLqrDSdycvvFslroAZ8tXHp0qVotT0vlev1eqMSlBAlsgztJ9AoFVx7xUxOkR+ZAlOeY4DgaDiiAq8dnC1gjN3UrqQT8OJzWgHQmnMTFobWED5tDHqdBIIhVEqxomp/DSh5LVmy5Lxt+nulUYgDV2u4kl6hQptdzijlWf/dSjWYSwi2nkBprRteyatzsN6v1GMyms7fPkY0eiO5Rg2GHAVBWR78LtDD0IA+qw0bNsQqDiEWrCeRkZHMxXB24gLsHj+7GrSYGtqZWliLouSSBASZIK4WSrL0mIuKkUszExaGpMlgbL4JCjQgilQHRPRR05ntNLWtLradVnGk6dwxFYNGxUk5B19QxtF0Iv7xJZKrFZVCQVZuEdkZ8V0Kpxuxi9CgieSVzmyn6HD7qQtkEeqhGFipkMgqqkBGwtreCh5rAoJMEFdr+Kshjks/96RzFyGv247LJy52DYRIXukq4MVra8LlC+LU5lGebeix2Yi8LJyaXDpcfrCejHOQieO2NnGi1clhZwJ7XQBqPbVtLj78oo6Pj7clNpYUI5JXurLXY3P58SqNZGfnoFP3PJ5SkZuBQ5OP0xvA297/zVNSWtCPu6OZequHT1sSnbwMaJQK1CE3To/oeQ2ESF7pylaP3evHqc2jJLP3OXtGrQpVZhEyYG0aJgs1Oltw+wIEFDos5gTPJ9Sa0KgUSHIIt1tU2Q+ESF7pyl6P3RPAocmnyKLrs6klvxwAZ+vJcG1YunM24fYHcWmyyTL2XLMYNwplpMre6xhGY45RIJJXmvLbG3H7gjjVORT10fMCKC4uJctkwKyWwTUMxl0czXh8QZzq7MReaeykMYQXhAy47WI56AEQySsdBf0EnW1kZ2iw5BVhPM+8vTEFZi4cM5acDC3Y6+MUZOLIziZc/iAudU6SJC8LEqAKOHH5gokOJ2WI5JWOnC3olAouKM3n2zMv7N9zTJ1LadvTfPdxWcZnbSAYknFrssnUJ2A1ibModGbUSgXqoBOHVwza95dIXunI2Rz+mpHX78nWsqkAjz+IvTnNB+19TjxOByChNuclx1xCjZE8k5aLskGrSoJ4UoSYSpWGZEcTXn8QbUYe/V0r4aA9A2tdB+YMLxNmhkCRpr9EziYsejXTLhzJ6EnliY4mTGsM1+HlqMCQ+NPYVJGmP6HDm8fayO66Dv582BtZfPB8cvMLCUkqXG4PsjuNB+3tjQBoMovIN/V9FTZuNJ0Tw32iVGIgEpq83n33Xa6//nqKi4uRJKlfm3ds376dSy65BK1Wy5gxY85Z514AZ1t40F3KyEep6F/fK8ekx6PNIRCSsTWncaV91wUJUxJtl6c1ISPjc1rpcPkSHU3KSGjycjqdTJ48mccff7xf7Y8fP86CBQuYO3cue/bs4b777mPZsmVs2bIlxpGmEJ8LT2e9kDG7/7+gSoWE2hJu35HGxaohWz1fNNj4pFWLPxhKdDhhWiPtTj+7jpzizX3DZJZDFCR0zGv+/PnMnz+/3+2feOIJRo4cyerVqwG46KKLeO+991izZg3z5s2LVZipxdmM0xfEqzKRmzmwdaoyckvwNe7B2Xo6RsElmM+F295Gu8vP3jYNl/azVxpzagMaTfiqp1sUqvZbSo157dix45xduefNm8eOHTt6fY7X68Vms3W7pTVnM05vAJc6m3zTwKrHLXllAPg6Tqdnpb2jAZcviEdlIdtsTtjSz+eQJLQZXxWqJk2PMMmlVPJqaGg4ZwOQgoICbDYbbre7x+dUVVVhsVgit7KysniEmjA+ayPeQAiXOou8ASavvIISZCS8bicBV0dsAkwkewMuXwCHJm/An02sqXQmlAoJTdCJXUzQ7peUSl6DsXLlSqxWa+RWV5e+4zkAjs5TPoWpoNeVJHpjztBRWFzG2HwjOBpjEV5i2etxeYM4NbnkJnpO41kkrQmtSoEm6MLmHqb7aA5QStV5FRYW0tjY/ZeqsbERs9mMXt/z/D2tVtvrhiFpR5bR+dsoydSjLht4DZMkSYwaORrqneBqAvpZnZ8iZNtpHN4ADlMe+eYk+5mIJC8nNo9IXv2RUsmrsrKS119/vdtjW7dupbKyMkERJRlPBwZFiPJcE+UXjRncMYydp+X2NOt5eax4HFb8IfDq8pKu5xVOXspw8nKL08b+SOhpo8PhYM+ePezZswcIl0Ls2bOH2tpaIHzKd+ZuRHfffTfHjh3jJz/5CV988QX//d//zZ/+9Cd+9KMfJSL85ONsCX81ZINicJs5BAx5dLh8nDx5InpxJQPrKXyBED59HrmZxn7Xv8WNxojFoGZsJpRm9b0KiBCW0J7Xzp07mTt3buT+ihUrgPAWa9XV1dTX10cSGcDIkSN57bXX+NGPfsTatWspLS3lqaeeEmUSnQK2BhxuH/qcHAbbrwjo8zjY4ADs5Dhs6I3maIaYONaTWPRqZo+ZgqeiJNHRnEtnJtugIVsvQ25GoqNJCQlNXnPmzOlz/aKequfnzJnD7t27YxhV6nK0NXCw3k6jP8jCi+VBlQLo9AYUGVmEnG20NNZRZpwQg0gTwBaeNSBllqHXJOEWY7rOFV299mG5e/lgpP3VxuHE1R5ezkafWTSkGiZNZjGQRpX2AS84msL/NidhrwtAY0KWJHx+Pw3Nzf2ekzqcieSVLkJBvLbwL2hGztDm7Zlywr/grtY0mapiO43d7ePD+gB/P9FzPWDCKRSgNbGntp2/fvyFKJfoB5G80oWrFZfHT1ChISsre0iHyizoqrSvT49lia0nsXkCNEt5OLzJu1KppLOgVSvRBhx0iOR1XiJ5pQnZ0YjLF8SlzibPPLSlXrIKylBIoPJ20G47d6ftlNN+ApvHj01XRFkyX8nTWdCplWiDDtrF6hLnJZJXmnB1NBIIyXg02WQPcUE7pTYDbUYmAK0NKT7uFfARsp7C7glg1ZVQmtXz5rtJQWdBr1aiDdixukTP63xSqkhV6J2zJTwtSGXOj8rSxiMqRqHpOIpe7xzysRLKWofT48OlyEBhyCLXmMQrlWrN6NQKtB47HW7R8zof0fNKE+ZQB6PzMhg7clRUjpeVX0aGRoXCmeKV9u3HsXkCWLUllGTqk2cliZ7oLGhV4dPGDtHzOi/R80oHfg+6oBOdSUf+qJHROWbXNCFHiu8m1F6D1e3Haijh4mQe74LOMS8F2oADm8tPMCQn30yAJCKSVzro2i1IawJ1lH5BTQU0O7x0NJ/AMtZBfqYxOseNJ58THE2YtCpUORWMyEnyynWtGY1KQYlZhXGMiZAso+z3FirDj0heaSBgb6TV5kFTUE6mPLjK+nNozTR7FFjtHtz1J8nPTMEVJlqPAFA2YiS3XDo+wcH0g1KFpDVRkSNBDpAM27IlMfHppAF7y2mOtTh5v16K3piOJKHNChe7WptSdEOOzuRFztjExjEQXdOEPGI56PMRySsNdO0WpM2M7o44ptxSAFxtp1KvWDUYQG47TofLRyArOhcx4kKXSTAk09rSSL01SWcDJAmRvFKdLOPrCA+qG4c4LehsWfmlKCRQOBpTb42pjhocLhd7m4Os3+tOneRryKbV6eW9zw7z/pHWREeT1ETySnXudtweNyFJiSWnMKqHVluKMWpVGPxt1LWlWKV96xFaHT469OWUZWckd4nEmfRZGDQqdAErbU5voqNJaiJ5pTjZ0YTLF8StziLPHOVSAEM2xgwDCjlAY2N9dI8dS6EQoaYvaHX4aNNXcEHhwLaASyh9Nnq1En3AitMbxO1L3rmYiSaSV4pzttV3TgvKITsjytXjkoQxuwgJUKRSvVfHCew2Ky5Zg8dURkWyl0icSZ+FUiFhlLwoQz5aRe+rV6JUIsXZO3cLUlsKYlLQmFlQzqX+JlQlKTJmBNB0kFaHl1bDKMYUZKZWoadaBxoDBo0NXcBGm9OX3PMxE0gkrxSXJ9kwFJlxXDDIDTfOQ2kuDK81lSpboQUDBBq/oMXpozV3NNNS6ZSxiz4bvboZnb+DVoeY49gbcdqYygI+VN4OLHo1JaUjYvMakd2EGggEUmD8pfUINrsDt2RAlT0iNTez0Geh1yjRBWy0OkXy6k1SJK/HH3+ciooKdDodl112GR9//HGvbaurq5EkqdtNpxva+lUpy9m5tLHWCJoYjetk5OGVJQ7UNvHc9t2Ekn154vo9ZGdomDrjCq68qCB1rjKeyZCNWadmWr7MzFFDW1gynSU8ef3xj39kxYoVPPTQQ3z66adMnjyZefPm0dTU1OtzzGYz9fX1kVtNTU0cI04eztaTnGh1cjqYGbsXUShRZ5bg8gVR2U/RYPPE7rWGytUGbcdBkjCPnEZZdoqOFRly0KmVjDZ4xHhXHxKevB599FHuuOMObr31VsaPH88TTzyBwWBg/fr1vT5HkiQKCwsjt4KCgjhGnDysjTXUWz0csMf2B1yRNYJMgxqzp54TLUm8vlf9XvzBEGSNBH1WoqMZvIy88FdXC4RCiY0liSU0efl8Pnbt2sXVV18deUyhUHD11VezY8eOXp/ncDgYMWIEZWVlLFy4kAMHDvTa1uv1YrPZut3ShbtzgwxDTmlsX8hSRqZeg9lbz/GWJC1WDfhwnNjFrpp23nOVpk5FfU90maBU4fV6OVxbR22rK9ERJaWEJq+WlhaCweA5PaeCggIaGnquKxo3bhzr16/nlVde4dlnnyUUCjFr1ixOnux58nBVVRUWiyVyKysri/r7SIiAF689vEO2JT/WyasUi1GHNujE2taMNRk3h2j4jNPNbXhUJmwZFak51tVFoQBDLq1OHzv2fsFnpzoSHVFSSvhp40BVVlayePFipkyZwuzZs9m0aRN5eXk8+eSTPbZfuXIlVqs1cqurS/E12TuFbPU4PH68SiP5OTmxfTGlGk1mMWa9Gou3nsON9ti+3kCFgti+fI9Wp4/T5slMH5mb6IiGLiOPDE14alajTRSq9iShySs3NxelUkljY/caosbGRgoL+zdPT61WM3XqVI4cOdLj97VaLWazudstHVib6giGZLz6fHKN2ti/YFYFORkaMj11HGpIruQlN+yj7nQDfqWevDGXkmeKw+cRa8Z8MrRKDP42bG6/mCbUg4QmL41Gw7Rp09i2bVvksVAoxLZt26isrOzXMYLBIPv27aOoKLorKiQ7W3MtANqsEhTxqCDPGUOOUUOFoomppabkGVMK+mnevw2bJ0CTZTIzx6bJxZuMXFQKBTmE1/VqsifxVd4ESXiF/YoVK1iyZAmXXnopM2bM4LHHHsPpdHLrrbcCsHjxYkpKSqiqqgJg1apVzJw5kzFjxtDR0cFvfvMbampqWLZsWSLfRtz52sJjfKb8OI3hmYpQ6YxMyHdBhg2k5Kg/8p74iLrTDXhVRkomXoFJp050SNHRecUxR+FEEQpwusOT/MtYx1nCk9dNN91Ec3MzDz74IA0NDUyZMoU333wzMohfW1uLQvFVB7G9vZ077riDhoYGsrKymDZtGh988AHjx6fAMr/R4rEy1hyiPCMb/wVxWp5ZkiB7NDTsC69QmlURn9fti7sd79H/RQZsRbO4blReoiOKHo0RNAZMOg8Gfwv11vQY7ogmSU6a/n982Gw2LBYLVqs1dce/Gg/A538BUyFcemv8XrfpCzjwMj6Nhc+Kb8Js0HBBQYLmDsoy7H0B2k/gNZVgH/cdck1pNtPisxdx1n/BX5zjacuazD/NHh2fIYIE6+/vaMJ7XsIgWDvLQjLjXPaRPQqUalqbG9jd/jmKzFJG5xkTsmqDXPcxUvsJUKjQjv8HtIY0S1wA5mIMLV8yp9CPaVrFsEhcA5FypRICHPnyc75osFFPnEsCVBrIvYB8s45S3zFsbj8HTsd/owhPSw0H3nslvNbVmCvBkBzjb1FnLkKSJIoU7Ri1op9xNpG8Uo3fjaO1nnaXH7ehOP6vXzABpSQxWVuPJAfYcbQ1rpfxg7ZGvvzbeuxuH596ivDkTY7ba8edqfP/190R3oNS6EYkrxTjaDyK2x/Crc6kOC/Gxak9yRoJWhOF+hBj5BpcviDvHG6Oy0sHbQ188db/4HA6cesLmXzVzeg0adwjUevAkENIltl38HNe3n2SYLKv6hFHInmlmNbagwAockahUyvjH4BCAaWXopAkrtAfR0LmYL2N/adie/roazzMoTfXYbN24NHkMPrKpeRnpegFl4HILEOSoO7oQU60uDjdIbZD6yKSVyqRZRz1XwJgLr4gcXEUTQGlGkvQypycDgB21bTHZq2vgBfnvlc5+NbTWO1OHPoSRl1zJyMK02AKUH9kjkBCYoQyPI+1RkzSjhDJK4UE7M04be2EJCXFI8YlLhC1DkqnAzDZu4uppUYWTYlypX8wACd3wUdPYD/2CS5fkI7sSUy49g5GFCTgdDlRssIr5OYrbKiC7uRd1SMB0njAIP00ndhPMCTjN5VQkGVMbDDlldDwGZLHyhzNITjj4sGumnbKsw2Dm2PobofTe/Cf3IM6FJ4SU1BQRE3ZdC4fPwlzulTQ95cmA4x5ZIUayXKcptmhp9XhJSce81mTnEheKUTX8SUWvRpn2cTEL/mi0sDYa2D/Jqj9MFwwm38RTTYP73YO4JdlG7iw0ER5jqHvpONuh+ZDuE8fwNZYS5Pdiy8QZPLYEagqKpGKpnCZIgHje8kiayQqRzMXqBpoZjSHGx1UiuQlklfKcLeTHWwju9iCfMmMREcTljcOSi+FkzvDFf8BD1rLBC4oMPFlk526Nhd1beExGqNWRXaGhsvH5FJoVEJHLW0nv6Cl9hAhRzNufxCPPwRIWHUlNOdMIGvsZYzKHwaD8ueTNw7qPqZCqucDOcDhRjszR2Un/g9YgonklSqavgh/zSxH0ib4lPFMo68Cvzs8ZenQm1jMn7GgeAr2kly+aAlS22Kno6MNjdOK1NCC0uEDOiAUJGD30NrsBCRsuiJaTaPQFFxIRXE+Xy82Y0jnMoiBMJeA1kh2yEaup54c03gCIRm1UiQvIdnJMs1ffoSFEJr8JJuArlDARdeDMR9O/C/YToPtNCZgeuctoAvhVgZxK4KYfWpQKUFnRp05Hk1xPuqckZSYzeSbtYkp/0h2kgS541Cd2sVNI5yoJw6v5Z96I5JXCnA0HuFozUmCCg2TKseSdNuoShKUz4SCCVC/F1qPgrsNAl6QFKh0Zky6TEzGfDAXg6kI9FlkSRLTEh17qiiYAKd2oW49FO7pqlNwP8ooE8krBdR99h4yEMofjykjidd00pqg4mvhmxBd5uJw79bRBA37aMmaTJPNy/ji4TsmKJJXkvO0n8Z+6nNAouziKxIdjpAokgTFU+HwFhzHPuI5nwmFQkVptn74lY90EkWqSe7YJ28SDMn4ci5gRFl5osMREqlgImgMZITsjJdOEAjJvP9lS6KjShiRvJJYW91B7CfDva5R064Z9pfGhz2VBkZcjoTEZdJBVLKXLxrsHGlKrg1R4kUkryQV9Do59v6fCckgl15KWdmIRIckJIOiKaDPwqzwcrXqMwC2ft5Eu9OX2LgSQCSvZBQMoPh8M4UaLyGthclXXJ/oiIRkoVTBRf8AksQ4apggH8bjD/Ly7lNYXUm4GXAMJUXyevzxx6moqECn03HZZZfx8ccf99n+xRdf5MILL0Sn03HxxRfz+uuvxynSOPB7YP9LSB21lOZamHrd7WQYDImOSkgmllIY+XUUksRcxV7GeD/H6vLx5TA7fUx48vrjH//IihUreOihh/j000+ZPHky8+bNo6mpqcf2H3zwATfffDO33347u3fvZtGiRSxatIj9+/fHOfLokkNBag/uZM/Lqwm0HAWlGiZ+G0N2AlZLFZJfeSWUTketVHCNdj/XK95jWs5XPS+nN5A8e2vGSMJ3D7rsssuYPn06v/vd74DwprNlZWX84Ac/4IEHHjin/U033YTT6eTVV1+NPDZz5kymTJnCE088cd7XS4bdg+RQELfLgcthxdZSj6O5BvfpLwh5bADk5BVwwZzvgVlUUgt9kGU4tQuO/h1CnUtxm4vwmSt48bAfhSGbovx8ci0mso1aMrQqMjRKVMqE91n6lBK7B/l8Pnbt2sXKlSsjjykUCq6++mp27NjR43N27NjBihUruj02b948Nm/e3GN7r9eL1+uN3LfZbP2Oz+ry85e9p5CBgradWBzHkOUQEjLIUGTRhjcClWW8gSCf1rSFnyjLSMjIyOEfMKDApGVUrgHkED6Pi721Hee8nqzSYRo9k/JLrwKdqKAWzkOSwhPjs0fB8Xeg+TDY6nHU1zCmyU5IBo6BHYl2hYaQpERGwag8I+U5RpAU2L0B9p/u/jshIUUOX5KppzQr/LPo8gfZ18uKuafyvs6IURcybUQWAHaPn817Tndrc+2EwsEtk9SLhCavlpYWgsFgZIPZLgUFBXzxxRc9PqehoaHH9g0NDT22r6qq4uGHHx5UfEFZpsURvopjdlgxO8Knsl1dVVmtA31nYgwEwfPVf+w53VlfAPzhH4rIXz61HpUxF01mIZkl4ygeeRFqjVjqRBggQzZM+BZ47dB6lOyOWi7Jb6ajtQmX04nTF8Tj9+MPegnJoPKHwN15iun1g7N78pLP+Cqr9aDpHHP1BQjZe05eVp0Tly8QuR8KQYvd261NtNffT/sK+5UrV3brqdlsNsrK+rffoVGr4tuXlCJJoHB9A8k/C0mSIje9RgU6NUgSKhnGjPcB4e/RdQMkSUKnVoFWDUgo1Toum61DoRSTkIUo0pqgeAoUT0EN5MkyhAIQ8EDAixwK4PUHUCB3jnbLaPwBRl3k+yphyd2/GnUq0IZ/TlVBmfKx3RNSl1JDHkbTV6d4Bq2Sb19S2q1NpiG6MwESmrxyc3NRKpU0NjZ2e7yxsZHCwsIen1NYWDig9lqtFq12cL0ZjUpBeU7nX53svqvblUBeZv+PndyjDkJakKTwhR+lGrQmJODsrXm1QEEPT+2JBijJ619btfKM350YSejvkEajYdq0aWzbti3yWCgUYtu2bVRWVvb4nMrKym7tAbZu3dpre0EQ0lPCTxtXrFjBkiVLuPTSS5kxYwaPPfYYTqeTW2+9FYDFixdTUlJCVVUVAPfeey+zZ89m9erVLFiwgBdeeIGdO3fy+9//PpFvQxCEOEt48rrppptobm7mwQcfpKGhgSlTpvDmm29GBuVra2tRKL7qIM6aNYuNGzfys5/9jJ/+9KeMHTuWzZs3M3HixES9BUEQEiDhdV7xlgx1XoIg9K6/v6Ni3FgQhJQkkpcgCClJJC9BEFJSwgfs461riG8g04QEQYifrt/N8w3HD7vkZbeHlw3pb5W9IAiJYbfbsVgsvX5/2F1tDIVCnD59GpPJNCyXVe6aHlVXVyeutvZCfEZ9i/XnI8sydrud4uLibmVSZxt2PS+FQkFpaen5G6Y5s9ksfjHPQ3xGfYvl59NXj6uLGLAXBCElieQlCEJKEslrmNFqtTz00EODXmljOBCfUd+S5fMZdgP2giCkB9HzEgQhJYnkJQhCShLJSxCElCSSlyAIKUkkrzQkdiDv20A+n+rq6m6brkiShE539krw6eXdd9/l+uuvp7i4GEmSet1W8Ezbt2/nkksuQavVMmbMGKqrq2Mep0heaUbsQN63gX4+EK4kr6+vj9xqamriGHH8OZ1OJk+ezOOPP96v9sePH2fBggXMnTuXPXv2cN9997Fs2TK2bNkS20BlIa3MmDFDXr58eeR+MBiUi4uL5aqqqh7bf+c735EXLFjQ7bHLLrtMvuuuu2IaZ6IM9PPZsGGDbLFY4hRd8gHkl19+uc82P/nJT+QJEyZ0e+ymm26S582bF8PIZFn0vNJI1w7kV199deSx/uxAfmZ7CO9A3lv7VDaYzwfA4XAwYsQIysrKWLhwIQcOHIhHuCkjUT9DInmlkb52IO9tR/GB7kCeygbz+YwbN47169fzyiuv8OyzzxIKhZg1axYnT56MR8gpobefIZvNhtvtjtnrDrtVJQRhICorK7vtCTpr1iwuuuginnzySX7xi18kMDJB9LzSSDx2IE9lg/l8zqZWq5k6dSpHjhyJRYgpqbefIbPZjF6vj9nriuSVRsQO5H0bzOdztmAwyL59+ygqKopVmCknYT9DMb0cIMTdCy+8IGu1Wrm6ulr+/PPP5TvvvFPOzMyUGxoaZFmW5e9///vyAw88EGn//vvvyyqVSv7tb38rHzx4UH7ooYdktVot79u3L1FvIaYG+vk8/PDD8pYtW+SjR4/Ku3btkv/xH/9R1ul08oEDBxL1FmLObrfLu3fvlnfv3i0D8qOPPirv3r1brqmpkWVZlh944AH5+9//fqT9sWPHZIPBIP/Lv/yLfPDgQfnxxx+XlUql/Oabb8Y0TpG80tB//dd/yeXl5bJGo5FnzJghf/jhh5HvzZ49W16yZEm39n/605/kCy64QNZoNPKECRPk1157Lc4Rx9dAPp/77rsv0ragoEC+7rrr5E8//TQBUcfP22+/LQPn3Lo+lyVLlsizZ88+5zlTpkyRNRqNPGrUKHnDhg0xj1MsiSMIQkoSY16CIKQkkbwEQUhJInkJgpCSRPISBCElieQlCEJKEslLEISUJJKXIAgpSSQvQRBSkkheQtqrrq4mMzMzcv/nP/85U6ZMidxfunQpixYtintcwtCI5CVEzdKlS5Ekibvvvvuc7y1fvhxJkli6dGm39tFOGhUVFTz22GPdHrvppps4fPhwr89Zu3ZttzXX58yZw3333RfVuIToE8lLiKqysjJeeOGFbovQeTweNm7cSHl5eUJi0uv15Ofn9/p9i8XSrWcmpAaRvISouuSSSygrK2PTpk2RxzZt2kR5eTlTp04d0rF76hEtWrQo0pubM2cONTU1/OhHP4rs9APnnjae7cwe4NKlS3nnnXdYu3Zt5BjHjx9nzJgx/Pa3v+32vD179iBJkljbK0FE8hKi7rbbbmPDhg2R++vXr+fWW2+N+etu2rSJ0tJSVq1aFdnpZ6DWrl1LZWUld9xxR+QY5eXl57wngA0bNvD1r3+dMWPGROstCAMgkpcQdd/73vd47733qKmpoaamhvfff5/vfe97MX/d7OxslEolJpOJwsLCQa0Ga7FY0Gg0GAyGyDGUSiVLly7l0KFDkT0e/X4/Gzdu5Lbbbov22xD6SaxhL0RdXl4eCxYsoLq6GlmWWbBgAbm5uYkOa0iKi4tZsGAB69evZ8aMGfz1r3/F6/Vy4403Jjq0YUv0vISYuO2226iuruYPf/hD1HonCoWCs5ef8/v9UTl2fyxbtixyMWLDhg3cdNNNGAyGuL2+0J1IXkJMXHvttfh8Pvx+P/PmzYvKMfPy8rqNYwWDwXN29tZoNASDwSG9Tm/HuO6668jIyGDdunW8+eab4pQxwcRpoxATSqWSgwcPRv7dG6vVyp49e7o9lpOTQ1lZ2Tltr7zySlasWMFrr73G6NGjefTRR+no6OjWpqKignfffZd//Md/RKvVDup0taKigo8++ogTJ05gNBrJzs5GoVBExr5WrlzJ2LFj03KTklQiel5CzJjNZsxmc59ttm/fztSpU7vdHn744R7b3nbbbSxZsoTFixcze/ZsRo0axdy5c7u1WbVqFSdOnGD06NHk5eUNKu77778fpVLJ+PHjycvLo7a2NvK922+/HZ/PF5erp0LfxBr2gjAA//u//8tVV11FXV3dObtEC/Elkpcg9IPX66W5uZklS5ZQWFjIc889l+iQhj1x2igI/fD8888zYsQIOjo6eOSRRxIdjoDoeQmCkKJEz0sQhJQkkpcgCClJJC9BEFKSSF6CIKQkkbwEQUhJInkJgpCSRPISBCElieQlCEJK+v8Z5FRJbFrplQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:20:14.204550Z", + "iopub.status.busy": "2024-03-22T23:20:14.203877Z", + "iopub.status.idle": "2024-03-22T23:20:14.533084Z", + "shell.execute_reply": "2024-03-22T23:20:14.532207Z" + }, + "papermill": { + "duration": 0.35042, + "end_time": "2024-03-22T23:20:14.535123", + "exception": false, + "start_time": "2024-03-22T23:20:14.184703", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABI+klEQVR4nO3deXzU1b34/9dn9plkZrLvCWFXQAFBMFgLKpUq18K1V712AUSp/krvo5Rrrem9X6z6sNgFLb31YnstpL3WeqUitmpFigIVcGEVkKUsWYDsy+z7fH5/TDISSEKWmXxmkvN8POYBmTnzmfdMknfO53ze5xxJlmUZQRCEJKNSOgBBEIT+EMlLEISkJJKXIAhJSSQvQRCSkkhegiAkJZG8BEFISiJ5CYKQlETyEgQhKWmUDmCwhcNhLly4gNlsRpIkpcMRBOESsizjcDgoKChApeq+fzXskteFCxcoLi5WOgxBEK6gpqaGoqKibh8fdsnLbDYDkQ/GYrEoHI0gCJey2+0UFxdHf1e7M+ySV8eposViEclLEBLYlYZ1xIC9IAhJSSQvQRCSkkhegiAkJUXHvNatW8e6deuorKwEYOLEiaxatYrbb7+9y/YVFRXcf//9ne7T6/V4vd6YxiXLMsFgkFAoFNPjCn2j1WpRq9VKhyEkKEWTV1FREc888wxjx45FlmV+97vfsWDBAg4cOMDEiRO7fI7FYuHEiRPRr2Ndq+X3+6mtrcXtdsf0uELfSZJEUVERqampSociJCBFk9edd97Z6eunn36adevW8eGHH3abvCRJIi8vLy7xhMNhzp49i1qtpqCgAJ1OJwpZFSLLMo2NjZw7d46xY8eKHphwmYQplQiFQmzcuBGXy0VZWVm37ZxOJyNGjCAcDnPdddfx4x//uNtEB+Dz+fD5fNGv7XZ7t239fj/hcJji4mJMJlP/3ogQM9nZ2VRWVhIIBBI7eZ3fD/VHIXMMlNwA4g/eoFB8wP7w4cOkpqai1+t5+OGHef3115kwYUKXbcePH8/69et54403eOmllwiHw8yaNYtz5851e/zVq1djtVqjt95U1/c0JUEYPEnR6206BSe3gO0cnNkOdZ8qHdGwISm9AYff76e6uhqbzcaf/vQnXnzxRXbs2NFtArtYIBDg6quv5r777uOpp57qsk1XPa/i4mJsNttlRaper5ezZ88ycuRIDAbDwN6YMGAJ//2QZdj7W3A2gkYPQR8YrDDzYRB/APvNbrdjtVq7/B29mOKfsE6nY8yYMUybNo3Vq1czefJk1q5d26vnarVapk6dyqlTp7pto9fro9X0oqpeiClHbSRxqTQwYxloDeC1ga1G6ciGBcWT16XC4XCnnlJPQqEQhw8fJj8/P85RJb45c+awYsUKpcMYXpr+Efk3awzozYQzxyIjQ+OJnp8nxISiyau8vJydO3dSWVnJ4cOHKS8vZ/v27Xz9618HYNGiRZSXl0fbP/nkk7z77rucOXOG/fv3841vfIOqqioefPBBpd5C0tm+fTuSJNHW1qZ0KEkv3HyGsCxDxmicviBb6800Of3QVqV0aMOColcbGxoaWLRoEbW1tVitVq699lq2bNnCl770JQCqq6s7DZ63traybNky6urqSE9PZ9q0aezevbtX42OCEFN+N421VdS0uNCMSCfgs3PSm0Z+m4fs1CYIeEBrVDrKIU3Rntdvf/tbKisr8fl8NDQ08Le//S2auCDSS6ioqIh+/dxzz1FVVYXP56Ouro633nqLqVOnDkqs/mC421swFO5120Av2w6Ez+fjBz/4AcXFxej1esaMGRP9rG+++WYA0tPTkSSJJUuWXPF4DoeDr3/966SkpJCfn89zzz132Wnq//7v/zJ9+nTMZjN5eXl87Wtfo6GhIfp4R49v27ZtTJ8+HZPJxKxZszoVHCcVRy1NDi92lZWAOoVrCq3IWhNNshlPIAS280pHOOQlTJ1Xonv+/e4vCozMSmHh1MLo17/ZeZpAqOuLuEXpRu6e/nm5xvpdZ/H4L5+G9L0vjet3rIsWLWLPnj388pe/ZPLkyZw9e5ampiaKi4t57bXX+OpXv8qJEyewWCwYjVfuHaxcuZJdu3bx5z//mdzcXFatWsX+/fuZMmVKtE0gEOCpp55i/PjxNDQ0sHLlSpYsWcLbb7/d6Vj/8R//wZo1a8jOzubhhx9m6dKl7Nq1q9/vVSn+1nM4vEGcKdlMyzFj0KopzjDiaMylxVVDof18ZCxMiBuRvIaYkydP8uqrr7J161bmzp0LwKhRo6KPZ2RkAJCTk0NaWtoVj+dwOPjd737Hyy+/zK233grAhg0bKCgo6NRu6dKl0f+PGjWKX/7yl1x//fU4nc5O03uefvppZs+eDcBjjz3G/Pnz8Xq9iVkK0YOWuipkQGXJx2rSAjAyK5XPzmZh95yh0NWobIDDgEhevbT85u7/iqouqaX81hdHd9v20rrLpTeOHEhYlzl48CBqtTqaIAbqzJkzBAIBZsyYEb3ParUyfvz4Tu327dvHj370Iw4dOkRrayvhcOTUt7q6utOY5LXXXhv9f8dV4oaGBkpKSmIS76CQZZyNkXKIjLwR0bsL04zs06bjsAUJO+oT71L+ECOSVy/pNL3/UYxX297ozWlgrLlcLubNm8e8efP4wx/+QHZ2NtXV1cybNw+/39+prVarjf6/o4K+I9ElDZ8Dj8uBjERW3udDAJkpOoKmLEJhGY+9hZSgL1K8KsSF+OMwxFxzzTWEw2F27NjR5eM6nQ6g18v9jBo1Cq1WyyeffBK9z2azcfLkyejXx48fp7m5mWeeeYabbrqJq666qtNg/VATdDTg9gfxaq3kpn2+zrpKJZGXmY7WZCUQDoM4dYwrkbyGmNLSUhYvXszSpUvZvHkzZ8+eZfv27bz66qsAjBgxAkmSePPNN2lsbMTpdPZ4PLPZzOLFi/n+97/P+++/z9GjR3nggQdQqVTRnlNJSQk6nY7/+q//4syZM/z5z3/udrrWUBByNZFjMZCakYvF2Pnk5SuTC5g+YSxpRh24mxWKcHgQyWsIWrduHf/yL//Ct7/9ba666iqWLVuGy+UCoLCwkCeeeILHHnuM3NxcvvOd71zxeM8++yxlZWX80z/9E3PnzuXGG2/k6quvjg6yZ2dnU1FRwcaNG5kwYQLPPPMMP//5z+P6HpWk97UyMjOFGRPGXTZ5XJIkMKZHvvC0KhDd8KH4xOzB1tOkz4SfCJwgXC4XhYWFrFmzhgceeCBur5Ow34/9/xtZRWLCVyC3i+WYaj5B/sdWyLkKadJdgx9fkuvtxGwxYC9c0YEDBzh+/DgzZszAZrPx5JNPArBgwQKFI1OALONqrUNPGI0pq8smfz3tJaOyhUn6JlIGObzhRCSvYe7SUoZLffbZZwD8/Oc/58SJE+h0OqZNm8bf//53srK6/uUdymS/i6PVDYTCMG56CpldtHFrLKTJ4LE1kSLLYnHCOBHJa5grKCjg4MGDPT5eUlLCvn37Bi+oBOZuayAUlvFpzFhTu15tN8WaCUj4fF7wO0Hf887PQv+I5DXMaTQaxowR01h6y94SKQGRUjLRqLu+3pWWYsSrScUT8EcG7UXyigtxtVEQ+sBlbwJAl5rRbZt0kw6fxow3EIosTijEhUhegtAHHnukdsto6Wq0KyLdpMWnTm1PXt1v+CIMjEhegtAHfkcLAClp3V+sSGvveQVCMn5X2yBFNvyIMS9B6IOgK1J4aknL7raNTqMiLT2TjKCOkKdtkCIbfkTyEoRekgMeis3gC+ixpvdcJvKlqWPh0D4Ii53X40UkL0HoJclrp8BqBJ0JjFfYlNhgjfzrtUW2SBO1XjEnxryGiETbPSjR4okJb1vkX0PaldvqzcjIhIJ+CHrjGdWwpWjyWrduHddee210P8WysjL++te/9vicjRs3ctVVV2EwGLjmmmsuW2ZY6L9L194SOnO2NeHyBwnorrz356lmL3tqvJysc4grjnGiaPIqKirimWeeYd++fezdu5dbbrmFBQsWcPTo0S7b7969m/vuu48HHniAAwcOsHDhQhYuXMiRI0fiF6QsQ9CvzK2Xc+aXLFnCjh07WLt2LZIkIUkSp0+f5oEHHmDkyJEYjUbGjx9/2Wa+S5YsYeHChTz99NMUFBREV0fdvXs3U6ZMwWAwMH36dDZv3owkSZ0q8Y8cOcLtt99Oamoqubm5fPOb36SpqanbeCorK2Py7VDS2XPn+fScjc9a1Vdsq9eocUsmfMEQ+ETyigdFx7zuvPPOTl8//fTTrFu3jg8//JCJEy+frb927Vq+/OUv8/3vfx+Ap556iq1bt/KrX/2KF154IT5BhgLw9zXxOfaV3PTvoNFdsdnatWs5efIkkyZNik6aTk9Pp6ioiI0bN5KZmcnu3bv51re+RX5+Pvfcc0/0udu2bcNisbB161YgMqP/zjvv5I477uDll1+mqqrqstO/trY2brnlFh588EGee+45PB4PP/jBD7jnnnt47733uownO7v7q3PJIuCMlEkYzN0XqHawGCK1Xj5vE7LXhhjxir2EGbAPhUJs3LgRl8tFWVlZl2327NnDypUrO903b948Nm/e3O1xfT5fpx247fah91fQarWi0+kwmUzk5eVF73/iiSei/x85ciR79uzh1Vdf7ZS8UlJSePHFF6MrrL7wwgtIksT//M//YDAYmDBhAufPn2fZsmXR5/zqV79i6tSp/PjHP47et379eoqLizl58iTjxo3rMp5kF3JHyiR6KlDtkGrQENCkEJbB62xD7OAYe4onr8OHD1NWVobX6yU1NZXXX3+921UO6urqyM3N7XRfbm4udXV13R5/9erVnX6J+0ytjfSAlKDWXrlND55//nnWr19PdXU1Ho8Hv9/fabsyiCwb3ZG4AE6cOMG1117baf2sizffADh06BDvv/9+p12BOpw+fZpx4/q/bVuiksNhQp7IVJ9U65V7XmqVhNpgBgd4XHaRvOJA8eQ1fvx4Dh48iM1m409/+hOLFy9mx44dMdsFu7y8vFNvzW63U1xc3MMzLiFJvTp1SzSvvPIKjzzyCGvWrKGsrAyz2czPfvYzPvroo07tUlL6vuKU0+nkzjvv5Cc/+cllj3XsCDTUeD0u5FAQALM1vVfP0adYoRF8LjG/MR4UT146nS66qsG0adP45JNPWLt2Lb/+9a8va5uXl0d9fX2n++rr63s8NdHr9ej1Q38HF51O12lTjV27djFr1iy+/e1vR+87ffr0FY8zfvx4XnrpJXw+X/Rzu3jzDYDrrruO1157jdLSUjSarn+ELo0n2TnaIuNdKl0KWm3v/pjpUyK1Xj730BuqSAQJV+cVDoc7jVFdrKysjG3btnW6b+vWrd2OkQ0npaWlfPTRR1RWVtLU1MTYsWPZu3cvW7Zs4eTJk/y///f/LktCXfna175GOBzmW9/6FseOHWPLli3R9eg71mtfvnw5LS0t3HfffXzyySecPn2aLVu2cP/990cT1qXxJN32ZpdwOSLjXSrTlcskOmRlZpBu0mKSPfEKa1hTNHmVl5ezc+dOKisrOXz4MOXl5Wzfvp2vf/3rQGTb+vLy8mj77373u7zzzjusWbOG48eP86Mf/Yi9e/f2ahOJoe6RRx5BrVYzYcIEsrOzmTdvHnfddRf33nsvM2fOpLm5uVMvrDsWi4W//OUvHDx4kClTpvAf//EfrFq1CiA6DlZQUMCuXbsIhULcdtttXHPNNaxYsYK0tDRUKlWX8VRXV8fvzQ+CdLWXERkm8rJzev2ciaWFXJVnIccgQ/sppxBDsoKWLl0qjxgxQtbpdHJ2drZ86623yu+++2708dmzZ8uLFy/u9JxXX31VHjdunKzT6eSJEyfKb731Vp9e02azyYBss9kue8zj8cifffaZ7PF4+vV+hqqXXnpJ1mq1stvtHtTXTajvx+n3Zfm9H8vyiS29f044LMvbfxp5nrs1XpENOT39jl5M0TGv3/72tz0+vn379svuu/vuu7n77rvjFJEA8Pvf/55Ro0ZRWFjIoUOHojVcSuzGnTB8jsi/fVkVVZKQdSZC7jY0ficY0+IS2nCl+IC9kHjq6upYtWoVdXV15Ofnc/fdd/P0008rHZaiWlub0fmD6LQp9LaAxRsIseuMG5O3lRmTHKitcQ1x2BHJS7jMo48+yqOPPqp0GAnleOUF8NgYfbWe3o566TUq/OoUTIDHaSM194pPEfog4a42CkKikcNh5Pa16A3m3tV4QeTqrNoQOc30ukS5RKyJ5NUFeXhtIp6wEuX74HY7keQQEpDSh+QFoDFGkpcoVI09kbwuotVGRjPcbrH6ZSLoWKJHrb7yKg7x5HG0ASDpUlBr+jZlS9teF+YXhaoxJ8a8LqJWq0lLS6OhIbI3n8lkihZmCoMrHA7T2NiIyWTqtop/sLidkQJVydD3/Rf1JgshIOgRySvWRPK6RMdUo44EJihHpVJRUlKi+B8QryNyyqc29v1yoT7FghsIel0xjkoQyesSkiSRn59PTk4OgUBA6XCGNZ1OF63YV5Lf3QZ8fgrYFxnWdPQmLRZNAMJhSID3M1SI5NUNtVqt+FiLkBgKDEG8GSZUWX1fULE4NxPyrZFVcQNu0F++jJDQPyJ5CcIVZGi8kGaEnH6sBqtSgdYIfrdIXjEm+rCCcCXRqUF9P20EkLUmguEwYZ8Y94olkbwEoSeyTEtrMy5fkLCu71cbAXZWuvikspU2W2uMgxvexGmjIPTA53Vx4nxkIcJpmhT6taauLnKq6HM7YheYIHpegtATlz3SW5K1RnT9XJFXY4gste13O2MWlyCSlyD0yONsA0DVjwLVDpr2ZXT8HtHziiWRvAShB95o8urfYD2A1hTpeQW8InnFkkhegtADf3vy0vSjur6DzhhJfCFRZR9TInkJQg/87sjUIG1KWr+PYTRFThtDolQipsTVRkHoQah9QrVhAMnLYrESTNFh1AYilfZisn9MKNrzWr16Nddffz1ms5mcnBwWLlzIiRMnenxORUUFkiR1ul28u7MgxNKI1BAjMkxkZmb2+xjZmRmMzzVTkqaHYNfb+gl9p2jy2rFjB8uXL+fDDz9k69atBAIBbrvtNlyunrvXFouF2tra6K2qqmqQIhaGFVkmU+OjIM1IRkZW/4+j1n6+63pArBUXK4qeNr7zzjudvq6oqCAnJ4d9+/bxxS9+sdvnSZLU4y7ZghATQR+E2lcW6cuuQV2QtSZCfi+Sz4nalBGD4ISEGrC32SKDoxkZPX9znU4nI0aMoLi4mAULFnD06NFu2/p8Pux2e6ebIPSG19VGq9uPM6yJ9J4GYHe1h08qW2luaYlRdELCJK9wOMyKFSu48cYbmTRpUrftxo8fz/r163njjTd46aWXCIfDzJo1i3PnznXZfvXq1Vit1uituLg4Xm9BGGLa2lo4Xufg08bwgI8l6cUUoVhLmOS1fPlyjhw5wiuvvNJju7KyMhYtWsSUKVOYPXs2mzZtIjs7m1//+tddti8vL8dms0VvNTU18QhfGIK8jsjUIJVh4BsuqvWRQlWfR0wRipWEKJX4zne+w5tvvsnOnTspKirq03O1Wi1Tp07l1KlTXT6u1+vR93NOmjC8+VxtAGj6sYLqpbTGVHxAQPS8YkbRnpcsy3znO9/h9ddf57333mPkyJF9PkYoFOLw4cPk5+fHIUJhOAt0FKiaBt7z0rZvgRbwip5XrCja81q+fDkvv/wyb7zxBmazmbq6OgCsVitGoxGARYsWUVhYyOrVqwF48sknueGGGxgzZgxtbW387Gc/o6qqigcffFCx9yEMTcH27cpikbwMxlSciCr7WFI0ea1btw6AOXPmdLp/w4YNLFmyBIDq6upOmzC0traybNky6urqSE9PZ9q0aezevZsJEyYMVtjCMBFur643mtMGfCy9mCIUc4omr97siLx9+/ZOXz/33HM899xzcYpIED4X8rYnr9S0AR8r1WwhI0WHXh8a8LGEiIQYsBeEhBP0MSpdgy9gwprW/6lBHTLS0snIbS90DYdAJXamGiiRvAShK147mSl60Bqgffx1QLRGkFQgh9t3ERpYxb6QQHVegpBQfO0zMWKVZCQJWWskGA4TFFccY0IkL0HogsPeSovbjxNTzI75yXkvn1S2Ut8spgjFgjhtFIQuNDQ20lDnQFKFuSFGx1TpUwg5wOsSPa9YED0vQeiC3xUpUNWlDLzGq0PHFCGxEUdsiOQlCF0IeNqTlyktZsfUGCKTs/1iLfuYEMlLELrQUaCqT41dz0tnjPS8gmJydkyI5CUIXQhHC1TTY3ZMXfv8xqBPJK9YEMlLEC4V9BH0ewEwWWKXvPRGMUUolsTVRkG4hM/VSigsE1LpSDHFoEC1XarZSkaKDq1BTBGKBZG8BOESKr+TsTmpuHUZ6DWxm8ZjsViw5JpBLbY+iwWRvAThEtqAk6xUPWTGeJMXXWTAnlAgchvguvjDnRjzEoRLRacGDXwF1U7UOmSVWkwRihGRvAThEm2tzZGpQVLsxrsAkCQO1vr4pLKV8w1NsT32MNSv5HXmzJlYxyEICeNCfQMn6hycaov92JQU3YhDXHEcqH4lrzFjxnDzzTfz0ksv4fV6Yx2TICgq2F5dr4/BCqqXUhvEFmix0q/ktX//fq699lpWrlxJXl4eDz30EB9//HGsYxOEwSfLhL2RxGKIwQqql9K297wCosp+wPqVvKZMmcLatWu5cOEC69evp7a2li984QtMmjSJZ599lsbGxljHKQiDI+gj1F6gmhLD6voOGmPH/EaRvAZqQAP2Go2Gu+66i40bN/KTn/yEU6dO8cgjj1BcXMyiRYuora3t8fmrV6/m+uuvx2w2k5OTw8KFCzlx4sQVX3fjxo1cddVVGAwGrrnmGt5+++2BvA1BiAp5bARCMkGVgdSUGA/Y8/kUoZBIXgM2oOS1d+9evv3tb5Ofn8+zzz7LI488wunTp9m6dSsXLlxgwYIFPT5/x44dLF++nA8//JCtW7cSCAS47bbbcLm6H8zcvXs39913Hw888AAHDhxg4cKFLFy4kCNHjgzkrQgCAG5HKzIQ0KZg1MZ+nXm9KdLzCoopQgMn98OaNWvkSZMmyVqtVl6wYIH8l7/8RQ6FQp3a1NTUyGq1uk/HbWhokAF5x44d3ba555575Pnz53e6b+bMmfJDDz3Uq9ew2WwyINtstj7FJgwP9cf3yLtf/Hf5/Y2/isvxHeePySf+WC6ffHNtXI4/FPT2d7RfFfbr1q1j6dKlLFmypNudqnNycvjtb3/bp+PabJGrPBkZGd222bNnDytXrux037x589i8eXOX7X0+Hz6fL/q13W7vU0zC8GKR3IzNScWbUxiX46emWhmXawa9KLEcqH4lr61bt1JSUtJpM1iI7MNYU1NDSUkJOp2OxYsX9/qY4XCYFStWcOONNzJp0qRu29XV1ZGbm9vpvtzc3Ohu25davXo1TzzxRK/jEIY3Q8iNIVUPeTnxeQFd+5r4ATfIMkhinmN/9Sv9jx49mqamyyuEW1paGDlyZL8CWb58OUeOHOGVV17p1/O7U15ejs1mi95qampienxhiInX1KAOWhMyMoFAgIDfE5/XGCb61fOSu9np2ul0YjAY+ny873znO7z55pvs3LmToqKiHtvm5eVRX1/f6b76+nry8rqeRKvX69Hr9X2OSRiempubkLx+DJIphvsGXUSt5UidB6fbQ+64JkaVlMTjVYaFPiWvjrEmSZJYtWoVJtPn395QKMRHH33ElClTen08WZb5t3/7N15//XW2b9/eq15bWVkZ27ZtY8WKFdH7tm7dSllZWa9fVxC6JMucr2/A5fGS69MwKk4vI+lSwO0RuwgNUJ+S14EDB4BI0jl8+DA6nS76mE6nY/LkyTzyyCO9Pt7y5ct5+eWXeeONNzCbzdFxK6vVirF9l+JFixZRWFjI6tWrAfjud7/L7NmzWbNmDfPnz+eVV15h7969/OY3v+nLWxGEywU8BAN+AEzm7i8aDZTakAI04RO7CA1In5LX+++/D8D999/P2rVrsVgGNi6wbt06AObMmdPp/g0bNrBkyRIAqqurO10YmDVrFi+//DL/+Z//yQ9/+EPGjh3L5s2bexzkF4TeCHvt+INhAiojqaa+D3/0VnQXIbfoeQ1Ev8a8NmzYEJMX727s7GLbt2+/7L67776bu+++OyYxCEIHt6OlvUA1FVMcClQ7aNuTV0BU2Q9Ir5PXXXfdRUVFBRaLhbvuuqvHtps2bRpwYIIw2LyO1sh/DFZUqviVMGjb5zeKBQkHptfJy2q1IrXXpFitsdvLThAShc/RAoA6hhvNdkVvat8CTSSvAel18rr4VDFWp42CkEj8rkjPS2OK/WoSFzOlWMhM0ZGeGteXGfL6Nebl8XiQZTlaKlFVVcXrr7/OhAkTuO2222IaoCAMljydD3NOKoHirqe8xYopJTUyRShFTBEaiH59egsWLOD3v/89AG1tbcyYMYM1a9awYMGC6BVEQUg2xrCLrFQ9+bkx3jXoUtr2XYQCosJ+IPq9kupNN90EwJ/+9Cfy8vKoqqri97//Pb/85S9jGqAgDIqg//NkYojzmK6ufYqQ14k/IDag7a9+JS+3243ZHBl0fPfdd7nrrrtQqVTccMMNVFVVxTRAQRgUXhuNDh/NfhV+4ryfotbEsVoHeytbqKxrju9rDWH93oBj8+bN1NTUsGXLlug4V0NDw4ALVwVBCbK3jTNNTg40yHiDce4NqdSodJEZJF63WKKpv/qVvFatWsUjjzxCaWkpM2fOjM4rfPfdd5k6dWpMAxSEweB1tBCWwa9JJUUX/43k1R1boIkq+37r13fpX/7lX/jCF75AbW0tkydPjt5/66238s///M8xC04QBovX2QaAZLCijmOBaoeOKUJifmP/9ftPTF5e3mXL0MyYMWPAAQmCErztBaqaOBeodtAYIj2voFckr/7qV/JyuVw888wzbNu2jYaGBsLhcKfHxY7aQrLxOyMFqrrU+K0mcbGOXYQCYufsfutX8nrwwQfZsWMH3/zmN8nPz49OGxKEZBVwR5JXPHbJ7oquvecV8okxr/7qV/L661//yltvvcWNN94Y63gEYfCFAoS8kR6QyZw5KC9pTDWTmaLDYhZ/+PurX8krPT29xx1+BCGpeG0UpRvJsJjR5sR3XmMHg8kSmSKUFv8rm0NVv0olnnrqKVatWoXb7Y51PIIw+DxtpOg0ZGdnk5YySPsdaNuXUPeL36H+6lfaX7NmDadPnyY3N5fS0lK02s4Vyfv3749JcIIwKLxtkX+Ng9PrAkCX0j5FyAHBMDqNmKTdV/1KXgsXLoxxGIKgHI+9iTabB1WakdwrN48NrYnjdQ7a3C1YR9uYUDSIiXOI6Ffyevzxx2MdhyAoxtHSQGWzmzaTxO2D9aJaIxq1Ggjg9TgAkbz6qt991ba2Nl588UXKy8tpaYkU+O3fv5/z58/3+hg7d+7kzjvvpKCgAEmS2Lx5c4/tt2/fjiRJl9262y1bEHrD74r8/GpTB+dKIwCSFJ0i5BVThPqlXz2vTz/9lLlz52K1WqmsrGTZsmVkZGSwadMmqquro2t9XYnL5WLy5MksXbr0iuviX+zEiROdJoDn5MRpa3Zh6AuHCbYXqBriuN1ZVyJboDUTcIsq+/7oV/JauXIlS5Ys4ac//Wl0aRyAO+64g6997Wu9Ps7tt9/O7bf3vaOek5NDWlpan58nCJfx2fEHAsiSCpNlcE/dtO2Fqn6xln2/9Ou08ZNPPuGhhx667P7CwsJBOYWbMmUK+fn5fOlLX2LXrl09tvX5fNjt9k43QYjytOILhvFqLFiMuiu3jyGdoX2KkEhe/dKv5KXX67tMAidPniQ7O3vAQXUnPz+fF154gddee43XXnuN4uJi5syZ02NpxurVq7FardFbcXFx3OITkpC3DX8whE9txmKI8yKEl/h8CzQxv7E/+pW8vvKVr/Dkk08SCAQAkCSJ6upqfvCDH/DVr341pgFebPz48Tz00ENMmzaNWbNmsX79embNmsVzzz3X7XPKy8ux2WzRW01NTdziE5JP0NWCPyTj1VowD3LyMqREpgiVWqRebcAsdNav5LVmzRqcTifZ2dl4PB5mz57NmDFjMJvNPP3007GOsUczZszg1KlT3T6u1+uxWCydbsNWKAjn9kLlByAmBAOg8rYxqcDCdeNHYdAObqGo3hjZRejaHK1Y3KAf+jVgb7Va2bp1K7t27eLQoUM4nU6uu+465s6dG+v4rujgwYPk58d3q6ohIRyCQy+Drb2U5cIBuG4xGIZxMieSvMwGLeaiIhjsBNKxi5BfnDb2R5+TVzgcpqKigk2bNlFZWYkkSYwcOZK8vDxkWe7TXxCn09mp13T27FkOHjxIRkYGJSUllJeXc/78+WjpxS9+8QtGjhzJxIkT8Xq9vPjii7z33nu8++67fX0bw0/NR5HEpdGBxgBeO5zcAtferXRkypFl8ETKJDCkDf7r60zIskzA40QOhtBr1IMfQxLrU/KSZZmvfOUrvP3220yePJlrrrkGWZY5duwYS5YsYdOmTVcsNL3Y3r17ufnmm6Nfr1y5EoDFixdTUVFBbW0t1dXV0cf9fj///u//zvnz5zGZTFx77bX87W9/63QMoQtBP9R8HPn/2NvAXACfvAjNp8B+ASwFysanFJ+DFrsTX1hCHzYx6OukaE2cqHfQ5HGQNtLJpMI4b7k2xPQpeVVUVLBz5062bdt2WcJ47733WLhwIb///e9ZtGhRr443Z86cHgcqKyoqOn396KOP8uijj/YlZAGg8VhkT0JjOq608Th8YbKzr0Ld8BlcODh8k5e7mQaHj1q/iSybnwyzcXBfX5eCVq1CLftweTyASF590acRyj/+8Y/88Ic/7LKnc8stt/DYY4/xhz/8IWbBCTHScAyAs9pR/HZXFX/8uJo36rPwh8KRxBYKKBygQtwteAMh3Np00kyDe6URALUOrTbSf/C6RJV9X/UpeX366ad8+ctf7vbx22+/nUOHDg04KCGG/C5orcLpC7KlIYNQONLTrQpmcbQF5KAPWiuVjVEhYVdTpEBVa8WqRPKSJLTthapeMUWoz/qUvFpaWsjN7X7RkNzcXFpbWwcclBBDLWdADnPClYJHE1m98/4bS9Fp1VRLBTi9QWg+rXSUivDaGiN7NerSMeuVWdG0o1BVTM7uuz59x0KhEBpN909Rq9UEg8EBByXEUGsl/lCYc1IeKknipnFZWAxa7ry2gAxfGakn6iID97I8+KUCCvPZGwDQWrIVq7PStSevgEckr77q89XGJUuWoNd3vVSuz+eLSVBCjMgytFaiU6v48k03cJ6c6BSYkkwThEbDKQ34HOBqgtT4Te1KOEEffpcNAL0lS7Ew9KbIaWPI5yIYCqNRixVVe6tPyWvx4sVXbNPbK43CIHA3RyrpVRr0GSWMUl/y7VZrwVJIqLkSta1meCWv9sH6gNqIOdV85fZxojOmkpWqw5SpIiTL/d8Fehjq02e1YcOGeMUhxIPtHDIykqUALk1cgMMbYF+dHnNdK1PzqlEVXqdAkApxN1GYbsSSX4BclKZYGJIuhbE5ZsjVgShS7RPRRx3K7Beobnaz7YKGUw2Xj6mYdBrOyZn4QzLOhsrBj09J7mY0KhXpWflkpAzuUjidiF2E+k0kr6HMfp42T4CaYDrhLoqB1SqJ9PxSZCRsrc3gtSkQpELczZF/TYO49HNX2ncR8nkcuP3iYldfiOQ1VAV9+OwNuP0hXPpsSjJMXTYbkZ2OS5dFmzsAtnODHKRyPLYGKptdnHQp2OsC0BqpbnHz4fEaPj7bomwsSUYkr6HKUYvdHcCnTiUjIxODtuvxlNKsFJy6HFy+IL7W3m+ektRCATxtjdTavOxvUjp5mdCpVWjDHlxe0fPqC5G8hip7LQ5fAJc+m8K07ufspeo1aNLykQFbwzBZqNHVhMcfJKgyYLUoPJ9Qb0anUSHJYTweUWXfFyJ5DVWOWhzeIE5dDvlWQ49NrTklALiaz0Vqw4Y6VwOeQAi3LoP01K5rFgeNSh2tsvc5h9GYYwyI5DVEBRz1ePwhXNpM8nvoeQEUFBSRbjZh0crgHgbjLs5GvP4QLm2Gslca2+lMkQUhgx6HWA66D0TyGopCAUKuFjJSdFiz80m9wry9MbkWrhozlswUPThqBylI5ciuBtyBEG5tZoIkLysSoAm6cPtDSoeTNETyGopcTRjUKsYV5fDVG67q3XPM7UtpO4b47uOyjN9WRygs49FlkGZUYDWJS6gMFrRqFdqQC6dPDNr3lkheQ5GrMfJvSnavJ1vL5ly8gRCOxiE+aO934XU5AQmtJTsx5hLqUsk267k6A/SaBIgnSYipVEOQ7GzAFwihT8mmt2slHHOkYKtpw5LiY+INYVAN0V8iVwNWo5ZpV41k9LUlSkcToU+N1OFlasCk/GlsshiiP6HDm9dWz4GaNl476YsuPnglWTl5hCUNbo8X2TOEB+0d9QDo0vLJMfd8FXbQ6NonhvtFqURfKJq8du7cyZ133klBQQGSJPVq847t27dz3XXXodfrGTNmzGXr3AvgaokMukspOahVvet7ZZqNePWZBMMy9sYhXGnfcUHCnEDb5enNyMj4XTba3H6lo0kaiiYvl8vF5MmTef7553vV/uzZs8yfP5+bb76ZgwcPsmLFCh588EG2bNkS50iTiN+Nt71eKDWj97+gapWE1hpp3zaEi1XD9lqO19n5pFlPIBRWOpwIfSqtrgD7Tp3nncPDZJZDDCg65nX77bdz++2397r9Cy+8wMiRI1mzZg0AV199NR988AHPPfcc8+bNi1eYycXViMsfwqcxk5XWt3WqUrIK8dcfxNV8IU7BKczvxuNoodUd4FCLjum97JXGndaEThe56ukRhaq9llRjXnv27LlsV+558+axZ8+ebp/j8/mw2+2dbkOaqxGXL4hbm0GOuW/V49bsYgD8bReGZqW9sw63P4RXYyXDYlFs6efLSBL6lM8LVROmR5jgkip51dXVXbYBSG5uLna7HY/H0+VzVq9ejdVqjd6Ki4sHI1TF+G31+IJh3Np0svuYvLJzC5GR8HlcBN1t8QlQSY463P4gTl12nz+beNMYzKhVErqQC4eYoN0rSZW8+qO8vBybzRa91dQM3fEcAGf7KZ/KnNvtShLdsaQYyCsoZmxOKjjr4xGeshy1uH0hXLosspSe03gJSW9Gr1GhC7mxe4bpPpp9lFR1Xnl5edTXd/6lqq+vx2KxYDR2PX9Pr9d3u2HIkCPLGAItFKYZ0Rb3vYZJkiRGjRwNtS5wNwC9rM5PErL9Ak5fEKc5mxxLgv1MRJOXC7tXJK/eSKrkVVZWxttvv93pvq1bt1JWVqZQRAnG24ZJFaYky0zJ1WP6d4zU9tNyxxDreXlteJ02AmHwGbITrucVSV7qSPLyiNPG3lD0tNHpdHLw4EEOHjwIREohDh48SHV1NRA55bt4N6KHH36YM2fO8Oijj3L8+HH++7//m1dffZXvfe97SoSfeFxNkX9NGaDq32YOQVM2bW4/585Vxi6uRGA7jz8Yxm/MJisttdf1b4NGl4rVpGVsGhSl97wKiBChaM9r79693HzzzdGvV65cCUS2WKuoqKC2tjaayABGjhzJW2+9xfe+9z3Wrl1LUVERL774oiiTaBe01+H0+DFmZtLffkXQmM2xOifgINNpx5hqiWWIyrGdw2rUMnvMFLylhUpHczmDhQyTjgyjDFkpSkeTFBRNXnPmzOlx/aKuqufnzJnDgQMH4hhV8nK21HGs1kF9IMSCa+R+lQIYjCZUKemEXS001ddQnDoxDpEqwB6ZNSClFWPUJeAWY4b2FV19jmG5e3l/DPmrjcOJuzWynI0xLX9ANUy6tAJgCFXaB33gbIj835KAvS4AnRlZkvAHAtQ1NvZ6TupwJpLXUBEO4bNHfkFTMgc2b8+cGfkFdzcPkakq9gs4PH4+rA3yXmXX9YCKU6lAb+ZgdSt/+fi4KJfoBZG8hgp3M25vgJBKR3p6xoAOlZbbUWlfOzSWJbadw+4N0ihl4/Ql7kqlksGKXqtGH3TSJpLXFYnkNUTIznrc/hBubQbZloEt9ZKeW4xKAo2vjVb75TttJ53WSuzeAHZDPsWJfCXPYMWgVaMPOWkVq0tckUheQ4S7rZ5gWMaryyBjgAvaqfUp6FPSAGiuS/Jxr6CfsO08Dm8Qm6GQovSuN99NCAYrRq0afdCBzS16XleSVEWqQvdcTZFpQRpLTkyWNh5ROgpd22mMRteAj6UoWw0urx+3KgWVKZ2s1AReqVRvwaBVofc6aPOInteViJ7XEGEJtzE6O4WxI0fF5HjpOcWk6DSoXElead96Frs3iE1fSGGaMXFWkuiKwYpeEzltbBM9rysSPa+hIODFEHJhMBvIGTUyNsfsmCbkTPLdhFqrsHkC2EyFXJPI413QPualQh90YncHCIXlxJsJkEBE8hoKOnYL0ptBG6NfUHMujU4fbY2VWMc6yUlLjc1xB5PfBc4GzHoNmsxSRmQmeOW63oJOo6LQoiF1jJmwLKPu9RYqw49IXkNA0FFPs92LLreENLl/lfWX0Vto9KqwObx4as+Rk5aEK0w0nwKgeMRIvjZ9gsLB9IJag6Q3U5opQSaQCNuyJTDx6QwBjqYLnGlysatWit2YjiShT48Uu9oaknRDjvbkReZYZePoi45pQl6xHPSViOQ1BHTsFqRPi+2OOOasIgDcLeeTr1g1FERuOUub208wPTYXMQaFIY1QWKa5qZ5aW4LOBkgQInklO1nG3xYZVE8d4LSgS6XnFKGSQOWsT741ptqqcLrdHGoMsf6QJ3mSrymDZpePDz49ya5TzUpHk9BE8kp2nlY8Xg9hSY01My+mh9ZaC0jVazAFWqhpSbJK++ZTNDv9tBlLKM5ISewSiYsZ0zHpNBiCNlpcPqWjSWgieSU52dmA2x/Co00n2xLjUgBTBqkpJlRykPr62tgeO57CYcINx2l2+mkxljIur29bwCnKmIFRq8YYtOHyhfD4E3cuptJE8kpyrpba9mlBmWSkxLh6XJJIzchHAlTJVO/VVonDbsMt6/CaiylN9BKJixnTUaskUiUf6rCfZtH76pYolUhyjvbdgrTW3LgUNKblljA90ICmMEnGjAAajtHs9NFsGsWY3LTkKvTUGkBnwqSzYwjaaXH5E3s+poJE8kpy2ZIdU74F57h+brhxBWpLXmStqWTZCi0UJFh/nCaXn+as0UxLplPGDsYMjNpGDIE2mp1ijmN3xGljMgv60fjasBq1FBaNiM9rRHcTqiMYTILxl+ZT2B1OPJIJTcaI5NzMwpiOUafGELTT7BLJqzsJkbyef/55SktLMRgMzJw5k48//rjbthUVFUiS1OlmMAxs/aqk5Wpf2lifCro4jeukZOOTJY5WN/CH7QcIJ/ryxLUHyUjRMXXGTdxydW7yXGW8mCkDi0HLtByZG0YNbGHJoUzx5PV///d/rFy5kscff5z9+/czefJk5s2bR0NDQ7fPsVgs1NbWRm9VVVWDGHHicDWfo7LZxYVQWvxeRKVGm1aI2x9C4zhPnd0bv9caKHcLtJwFScIychrFGUk6VmTKxKBVM9rkFeNdPVA8eT377LMsW7aM+++/nwkTJvDCCy9gMplYv359t8+RJIm8vLzoLTc3dxAjThy2+ipqbV6OOuL7A65KH0GaSYvFW0tlUwKv71V7iEAoDOkjwZiudDT9l5Id+dfdBOGwsrEkMEWTl9/vZ9++fcydOzd6n0qlYu7cuezZs6fb5zmdTkaMGEFxcTELFizg6NGj3bb1+XzY7fZOt6HC075BhimzKL4vZC0mzajD4qvlbFOCFqsG/Tgr97GvqpUP3EXJU1HfFUMaqDX4fD5OVtdQ3exWOqKEpGjyampqIhQKXdZzys3Npa6u67qi8ePHs379et544w1eeuklwuEws2bN4ty5ricPr169GqvVGr0VFxfH/H0oIujD54jskG3NiXfyKsKaakAfcmFracSWiJtD1H3KhcYWvBoz9pTS5Bzr6qBSgSmLZpefPYeO8+n5NqUjSkiKnzb2VVlZGYsWLWLKlCnMnj2bTZs2kZ2dza9//esu25eXl2Oz2aK3mpokX5O9Xdhei9MbwKdOJSczM74vptaiSyvAYtRi9dVyst4R39frq3AI+z8+oNnl54JlMtePzFI6ooFLySZFF5maVW8XhapdUTR5ZWVloVarqa/vXENUX19PXl7v5ulptVqmTp3KqVOnunxcr9djsVg63YYCW0MNobCMz5hDVqo+/i+YXkpmio40bw0n6hIrecl1h6m5UEdAbSR7zHSyzYPwecRbag4pejWmQAt2T0BME+qCoslLp9Mxbdo0tm3bFr0vHA6zbds2ysrKenWMUCjE4cOHyc+P7YoKic7eWA2APr0Q1WBUkGeOITNVR6mqgalF5sQZUwoFaDyyDbs3SIN1MjeMHSIXb1Ky0KhUZBJZ16vBkcBXeRWieIX9ypUrWbx4MdOnT2fGjBn84he/wOVycf/99wOwaNEiCgsLWb16NQBPPvkkN9xwA2PGjKGtrY2f/exnVFVV8eCDDyr5NgadvyUyxmfOGaQxPHM+GkMqE3PckGIHKTHqj3yVH1FzoQ6fJpXCSTdhNmiVDik22q84ZqpcqMJBLrR5E38Z60GmePK69957aWxsZNWqVdTV1TFlyhTeeeed6CB+dXU1KtXnHcTW1laWLVtGXV0d6enpTJs2jd27dzNhQhIs8xsrXhtjLWFKUjIIjBuk5ZklCTJGQ93hyAql6aWD87o98bTiO/13ZMCeP4s7RmUrHVHs6FJBZ8Js8GIKNFFrGxrDHbEkyQnT/x8cdrsdq9WKzWZL3vGv+qPw2Z/BnAfT7x+81204Dkdfx6+z8mnBvVhMOsblKjR3UJbh0CvQWonPXIhj/D1kmYfYTItPN+KqPc6fXRNoSZ/M/zd79OAMESist7+jive8hH6wtZeFpA1y2UfGKFBraW6s40DrZ6jSihidnarIqg1yzcdIrZWg0qCf8E/oTUMscQFYCjA1/YM5eQHM00qHReLqi6QrlRDg1D8+43idnVoGuSRAo4OsceRYDBT5z2D3BDh6YfA3ivA2VXH0gzcia12NuQVMiTH+FnOWfCRJIl/VSqpe9DMuJZJXsgl4cDbX0uoO4DEVDP7r505ELUlM1tciyUH2nG4e1Mv4IXs9//jbehweP/u9+XizJw/aaw86c/v319MW2YNS6EQkryTjrD+NJxDGo02jIDvOxaldSR8JejN5xjBj5Crc/hA7TjYOykuH7HUcf/d/cLpceIx5TL71Pgy6Idwj0RrAlElYljl87DNeP3COUKKv6jGIRPJKMs3VxwBQZY7CoFUPfgAqFRRNRyVJ3GQ8i4TMsVo7R87H9/TRX3+SE++sw25rw6vLZPQtS8hJT9ILLn2RVowkQc3pY1Q2ubnQJrZD6yCSVzKRZZy1/wDAUjBOuTjyp4BaizVkY05mGwD7qlrjs9ZX0Ifr8Jsce/e32BwunMZCRt32LUbkDYEpQL2RNgIJiRHqyDzWKjFJO0okryQSdDTisrcSltQUjBivXCBaAxRdD8Bk3z6mFqWycEqMK/1DQTi3Dz56AceZT3D7Q7RlXMvELy9jRK4Cp8tKSY+skJujsqMJeRJ3VQ8FDOEBg6GnofIIobBMwFxIbnqqssGUlEHdp0heG3N0J+Ciiwf7qlopyTD1b46hpxUuHCRw7iDacGRKTG5uPlXF13PjhGuxDJUK+t7SpUBqNunhetKdF2h0Gml2+sgcjPmsCU4kryRiaPsHVqMWV/Ek5Zd80ehg7G1wZBNUfxgpmM25mga7l53tA/jFGSauyjNTkmnqOel4WqHxBJ4LR7HXV9Pg8OEPhpg8dgSa0jKk/CnMVCkwvpco0keicTYyTlNHI6M5We+kTCQvkbyShqeVjFALGQVW5OtmKB1NRPZ4KJoO5/ZGKv6DXvTWiYzLNfOPBgc1LW5qWiJjNKl6DRkpOm4ck0Veqhraqmk5d5ym6hOEnY14AiG8gTAgYTMU0pg5kfSxMxmVMwwG5a8kezzUfEypVMtuOcjJegc3jMpQ/g+YwkTyShYNxyP/ppUg6RU+ZbzY6Fsh4IlMWTrxDlbLp8wvmIKjMIvjTSGqmxy0tbWgc9mQ6ppQO/1AG4RDBB1emhtdgITdkE+zeRS63KsoLcjhiwUWTEO5DKIvLIWgTyUjbCfLW0umeQLBsIxWLZKXkOhkmcZ/fISVMLqcBJuArlLB1XdCag5U/h3sF8B+ATNwffstaAjjUYfwqEJY/FrQqMFgQZs2AV1BDtrMkRRaLORY9MqUfyQ6SYKs8WjO7+PeES60k4bX8k/dEckrCTjrT3G66hwhlY5ry8aScNuoShKU3AC5E6H2EDSfBk8LBH0gqdAYLJgNaZhTc8BSAOZ8MKaTLklMUzr2ZJE7Ec7vQ9t8ItLT1SbhfpQxJpJXEqj59ANkIJwzAXNKAq/ppDdD6RciNyG2LAWR3q2zAeoO05Q+mQa7jwkFw3dMUCSvBOdtvYDj/GeARPE1NykdjqAUSYKCqXByC84zH/EHvxmVSkNRhnH4lY+0E0WqCe7MJ+8QCsv4M8cxorhE6XAEJeVOAp2JlLCDCVIlwbDMrn80KR2VYkTySmAtNcdwnIv0ukZNu23YXxof9jQ6GHEjEhIzpWNoZB/H6xycakisDVEGi0heCSrkc3Fm12uEZZCLplNcPELpkIREkD8FjOlYVD7maj4FYOtnDbS6/MrGpQCRvBJRKIjqs83k6XyE9VYm33Sn0hEJiUKtgav/CSSJ8VQxUT6JNxDi9QPnsbkTcDPgOEqI5PX8889TWlqKwWBg5syZfPzxxz2237hxI1dddRUGg4FrrrmGt99+e5AiHQQBLxz5E1JbNUVZVqbe8QApJpPSUQmJxFoEI7+ISpK4WXWIMb7PsLn9/GOYnT4qnrz+7//+j5UrV/L444+zf/9+Jk+ezLx582hoaOiy/e7du7nvvvt44IEHOHDgAAsXLmThwoUcOXJkkCOPLTkcovrYXg6+voZg02lQa2HSVzFlKLBaqpD4Ssqg6Hq0ahW36Y9wp+oDpmV+3vNy+YKJs7dmnCi+e9DMmTO5/vrr+dWvfgVENp0tLi7m3/7t33jssccua3/vvfficrl48803o/fdcMMNTJkyhRdeeOGKr5cIuwfJ4RAetxO304a9qRZnYxWeC8cJe+0AZGbnMm7ON8AiKqmFHsgynN8Hp9+DcPtS3JZ8/JZSNp4MoDJlkJ+TQ5bVTEaqnhS9hhSdGo1a8T5Lj5Ji9yC/38++ffsoLy+P3qdSqZg7dy579uzp8jl79uxh5cqVne6bN28emzdv7rK9z+fD5/NFv7bb7b2Oz+YO8OdD55GB3Ja9WJ1nkOUwEjLIkG/VRzYClWV8wRD7q1oiT5RlJGRk5MgPGJBr1jMqywRyGL/XzaHqtsteT9YYMI++gZLpt4JBVFALVyBJkYnxGaPg7A5oPAn2Wpy1VYxpcBCWgTPgQKJVpSMsqZFRMSo7lZLMVJBUOHxBjlzo/DshIUUPX5hmpCg98rPoDoQ43M2Kueezv8iIUVcxbUQ6AA5vgM0HL3Rq8+WJef1bJqkbiiavpqYmQqFQdIPZDrm5uRw/frzL59TV1XXZvq6ursv2q1ev5oknnuhXfCFZpskZuYpjcdqwOCOnsh1dVVlrAGN7YgyGwPv5N/ay7qw/CIHID0X0L5/WiCY1C11aHmmF4ykYeTVanVjqROgjUwZM/GfwOaD5NBlt1VyX00hbcwNulwuXP4Q3ECAQ8hGWQRMIg6f9FNMXAFfn5CVf9K+sNYKufczVHyTs6Dp52Qwu3P5g9OtwGJocvk5tYr3+/pCvsC8vL+/UU7Pb7RQX926/w1S9hq9eV4Qkgcr9JaTALCRJit6MOg0YtCBJaGQYM8EPRB6j4wZIkoRBqwG9FpBQaw3MnG1ApRaTkIUY0puhYAoUTEELZMsyhIMQ9ELQhxwO4gsEUSG3j3bL6AJBRl3t/zxhyZ3/TTVoQB/5OdWEZErGdk5IHYpM2aSaPz/FM+nVfPW6ok5t0kyxnQmgaPLKyspCrVZTX1/f6f76+nry8vK6fE5eXl6f2uv1evT6/vVmdBoVJZntf3Uyeq5uVwPZab0/dmKPOghDgiRFLvyotaA3IwGXbs2rB3K7eGpXdEBhdu/aatUX/e7EiaK/QzqdjmnTprFt27bofeFwmG3btlFWVtblc8rKyjq1B9i6dWu37QVBGJoUP21cuXIlixcvZvr06cyYMYNf/OIXuFwu7r//fgAWLVpEYWEhq1evBuC73/0us2fPZs2aNcyfP59XXnmFvXv38pvf/EbJtyEIwiBTPHnde++9NDY2smrVKurq6pgyZQrvvPNOdFC+uroalerzDuKsWbN4+eWX+c///E9++MMfMnbsWDZv3sykSZOUeguCIChA8TqvwZYIdV6CIHSvt7+jYtxYEISkJJKXIAhJSSQvQRCSkuID9oOtY4ivL9OEBEEYPB2/m1cajh92ycvhiCwb0tsqe0EQlOFwOLBard0+PuyuNobDYS5cuIDZbB6Wyyp3TI+qqakRV1u7IT6jnsX785FlGYfDQUFBQacyqUsNu56XSqWiqKjoyg2HOIvFIn4xr0B8Rj2L5+fTU4+rgxiwFwQhKYnkJQhCUhLJa5jR6/U8/vjj/V5pYzgQn1HPEuXzGXYD9oIgDA2i5yUIQlISyUsQhKQkkpcgCElJJC9BEJKSSF5DkNiBvGd9+XwqKio6bboiSRIGw6UrwQ8tO3fu5M4776SgoABJkrrdVvBi27dv57rrrkOv1zNmzBgqKiriHqdIXkOM2IG8Z339fCBSSV5bWxu9VVVVDWLEg8/lcjF58mSef/75XrU/e/Ys8+fP5+abb+bgwYOsWLGCBx98kC1btsQ3UFkYUmbMmCEvX748+nUoFJILCgrk1atXd9n+nnvukefPn9/pvpkzZ8oPPfRQXONUSl8/nw0bNshWq3WQoks8gPz666/32ObRRx+VJ06c2Om+e++9V543b14cI5Nl0fMaQjp2IJ87d270vt7sQH5xe4jsQN5d+2TWn88HwOl0MmLECIqLi1mwYAFHjx4djHCThlI/QyJ5DSE97UDe3Y7ifd2BPJn15/MZP34869ev54033uCll14iHA4za9Yszp07NxghJ4XufobsdjsejydurzvsVpUQhL4oKyvrtCforFmzuPrqq/n1r3/NU089pWBkguh5DSGDsQN5MuvP53MprVbL1KlTOXXqVDxCTErd/QxZLBaMRmPcXlckryFE7EDes/58PpcKhUIcPnyY/Pz8eIWZdBT7GYrr5QBh0L3yyiuyXq+XKyoq5M8++0z+1re+Jaelpcl1dXWyLMvyN7/5Tfmxxx6Ltt+1a5es0Wjkn//85/KxY8fkxx9/XNZqtfLhw4eVegtx1dfP54knnpC3bNkinz59Wt63b5/8r//6r7LBYJCPHj2q1FuIO4fDIR84cEA+cOCADMjPPvusfODAAbmqqkqWZVl+7LHH5G9+85vR9mfOnJFNJpP8/e9/Xz527Jj8/PPPy2q1Wn7nnXfiGqdIXkPQf/3Xf8klJSWyTqeTZ8yYIX/44YfRx2bPni0vXry4U/tXX31VHjdunKzT6eSJEyfKb7311iBHPLj68vmsWLEi2jY3N1e+44475P379ysQ9eB5//33ZeCyW8fnsnjxYnn27NmXPWfKlCmyTqeTR40aJW/YsCHucYolcQRBSEpizEsQhKQkkpcgCElJJC9BEJKSSF6CICQlkbwEQUhKInkJgpCURPISBCEpieQlDHkVFRWkpaVFv/7Rj37ElClTol8vWbKEhQsXDnpcwsCI5CXEzJIlS5AkiYcffviyx5YvX44kSSxZsqRT+1gnjdLSUn7xi190uu/ee+/l5MmT3T5n7dq1nZYtnjNnDitWrIhpXELsieQlxFRxcTGvvPJKp3WcvF4vL7/8MiUlJYrEZDQaycnJ6fZxq9XaqWcmJAeRvISYuu666yguLmbTpk3R+zZt2kRJSQlTp04d0LG76hEtXLgw2pubM2cOVVVVfO9734tulgGXnzZe6uIe4JIlS9ixYwdr166NHuPs2bOMGTOGn//8552ed/DgQSRJEsvjKEQkLyHmli5dyoYNG6Jfr1+/nvvvvz/ur7tp0yaKiop48skno5tl9NXatWspKytj2bJl0WOUlJRc9p4ANmzYwBe/+EXGjBkTq7cg9IFIXkLMfeMb3+CDDz6gqqqKqqoqdu3axTe+8Y24v25GRgZqtRqz2UxeXl6/FlS0Wq3odDpMJlP0GGq1miVLlnDixInoNmmBQICXX36ZpUuXxvptCL0kloEWYi47O5v58+dTUVGBLMvMnz+frKwspcMakIKCAubPn8/69euZMWMGf/nLX/D5fNx9991KhzZsiZ6XEBdLly6loqKC3/3udzHrnahUKi5dwSkQCMTk2L3x4IMPRi9GbNiwgXvvvReTyTRory90JpKXEBdf/vKX8fv9BAIB5s2bF5NjZmdndxrHCoVCl22Oq9PpCIVCA3qd7o5xxx13kJKSwrp163jnnXfEKaPCxGmjEBdqtZpjx45F/98dm83GwYMHO92XmZlJcXHxZW1vueUWVq5cyVtvvcXo0aN59tlnaWtr69SmtLSUnTt38q//+q/o9fp+na6Wlpby0UcfUVlZSWpqKhkZGahUqujYV3l5OWPHjh2S6/wnE9HzEuLGYrFgsVh6bLN9+3amTp3a6fbEE0902Xbp0qUsXryYRYsWMXv2bEaNGsXNN9/cqc2TTz5JZWUlo0ePJjs7u19xP/LII6jVaiZMmEB2djbV1dXRxx544AH8fv+gXD0VeiaWgRaEPvj73//OrbfeSk1NzWUbrQqDSyQvQegFn89HY2MjixcvJi8vjz/84Q9KhzTsidNGQeiFP/7xj4wYMYK2tjZ++tOfKh2OgOh5CYKQpETPSxCEpCSSlyAISUkkL0EQkpJIXoIgJCWRvARBSEoieQmCkJRE8hIEISmJ5CUIQlISyUsQhKT0/wO9swY0Ouru+AAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:20:14.573027Z", + "iopub.status.busy": "2024-03-22T23:20:14.572750Z", + "iopub.status.idle": "2024-03-22T23:20:14.769345Z", + "shell.execute_reply": "2024-03-22T23:20:14.768511Z" + }, + "papermill": { + "duration": 0.217868, + "end_time": "2024-03-22T23:20:14.771212", + "exception": false, + "start_time": "2024-03-22T23:20:14.553344", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAh3ElEQVR4nO3de1RTV/428IdwCSIXL1yl0XhpReuFiq8MVqdji2A7OjpOq69aQXTsjJd3bDO0lpkWxrpa7VSodupPVq1otVodp9ZetCpScazaqngZtYpLlGLloki51xDIfv/wR6YxCQlKCDt5PmuxNCf7nHwPO3k4Zydnx00IIUBEJBmFowsgIroXDC8ikhLDi4ikxPAiIikxvIhISgwvIpISw4uIpMTwIiIpeTi6gPam1+tRXFwMPz8/uLm5ObocIrqLEAI1NTXo0aMHFArLx1cuF17FxcVQqVSOLoOIrLh27RoeeOABi/e7XHj5+fkBuPOL8ff3d3A19qPT6bBv3z7ExcXB09PT0eVQG3CVPq2uroZKpTK8Vi1xufBqPlX09/d3+vDy8fGBv7+/Uz/RXYmr9am1YR0O2BORlBheRCQlhhcRSYnhRURSYngRkZQYXkQkJZf7qISzqq+vx8WLFw23a3/S4sjZAnQNPAHfTkqjthEREfDx8WnvEonaFMPLSVy8eBFRUVEmy/9upm1eXh6GDRtm/6KI7Ijh5SQiIiKQl5dnuJ1fUgnN9rPIeGYw+od1MWlLJDuGl5Pw8fExOppSfH8LykM/YcCgoYjs1d2BlRHZBwfsiUhKDC8ikhLDi4ikxPAiIikxvIhISgwvIpISw4uIpMTwIiIpMbyISEr8hD1RB3T3hfaA5YvtXfVCe4YXUQdk6UJ7wPRie1e90J7hRdQB3X2hPWD5YntXvdCe4UXUAd19oT3Ai+3vxgF7IpISw4uIpMTwIiIpMbyISEoMLyKSEsOLiKTE8CIiKTG8iEhKDC8ikhLDi4ikxPAiIikxvIhISgwvIpISw4uIpMTwIiIpMbyISEoOD6/Vq1dDrVbD29sb0dHROHbsWIvtKysrsWDBAoSFhUGpVOKhhx7C7t2726laIuooHDqT6rZt26DRaJCZmYno6GisXLkS8fHxyM/PR3BwsEn7hoYGjB07FsHBwfjXv/6F8PBwfP/99+jSpUv7F09EDuXQ8MrIyMDcuXORlJQEAMjMzMSuXbuQlZWFl19+2aR9VlYWKioqcOTIEXh6egIA1Gp1e5ZMRB2Ew8KroaEBeXl5SElJMSxTKBSIjY3F0aNHza7z2WefISYmBgsWLMCnn36KoKAgTJ8+HYsXL4a7u3t7ld5hXC2vQ5220ex9BTfrDP96eFju5s5KD/QO7GyX+qh1WupPwLY+daX+dFh4lZeXo6mpCSEhIUbLQ0JCTL6vrtmVK1fw1VdfYcaMGdi9ezcuX76M+fPnQ6fTIS0tzew6Wq0WWq3WcLu6uhoAoNPpoNPp2mhv2l/hrTqMXXnYars//+us1TbZzz8KdXfXeMJ3VLb2J2C9T2XvT1tfl1J9e5Ber0dwcDDee+89uLu7IyoqCtevX8dbb71lMbyWLVuGJUuWmCzft2+f1F/Uea0WADwws18TQjoJk/t1eqBCC3RTAp4W3pYp+8kNmy67Y2/OQah87VouWWGtPwHrfeos/VlfX29TO4eFV2BgINzd3VFWVma0vKysDKGhoWbXCQsLg6enp9Ep4oABA1BaWoqGhgZ4eXmZrJOSkgKNRmO4XV1dDZVKhbi4OPj7+7fR3rS/88XVWHH2G/xu7KN4uIfpfuh0OmRnZ2Ps2LGG8UFz29h0+RuMGjXK7Dao/VjrT8B6nzpLfzafHVnjsPDy8vJCVFQUcnJyMGnSJAB3jqxycnKwcOFCs+s8+uij2LJlC/R6PRSKO396Ll26hLCwMLPBBQBKpRJKpdJkuaenp8UXtQyaxzw8PDxa3I+W9tPWbZD9taYvLPWps/SnrbU79HNeGo0Ga9euxQcffIALFy5g3rx5qKurM7z7mJCQYDSgP2/ePFRUVGDRokW4dOkSdu3ahTfeeAMLFixw1C4QkYM4dMxr6tSpuHnzJlJTU1FaWorIyEjs2bPHMIhfVFRkOMICAJVKhb179+KFF17AkCFDEB4ejkWLFmHx4sWO2gUichCHD9gvXLjQ4mlibm6uybKYmBh88803dq6KiDo6h18eRER0LxheRCQlhhcRSYnhRURSYngRkZQc/m4j3Rtt020ovK/janU+FN6m14I0NjaiuLEYFyouWLyI92p1LRTe16Ftug0gwM4VU0us9SdgvU9drT8ZXpIqrvsenXv/A39pee5G/M+e/2nx/s69geK6SEQhpMV2ZF+29ifQcp+6Un8yvCTVo3Mv1F39f1g1NRJ9g80feR3++jAeHfWoxSOvghu1WLTtNHqM6WXvcskKa/0JWO9TV+tPhpeklO7e0N8OR2///hjY3fQUQafT4arHVQzoNsDitWL621XQ374Jpbu3vcslK6z1J2C9T12tPzlgT0RSYngRkZQYXkQkJYYXEUmJ4UVEUmJ4EZGUGF5EJCWGFxFJieFFRFJieBGRlBheRCQlhhcRSYnhRURSYngRkZQYXkQkJYYXEUmJ4UVEUmJ4EZGUGF5EJCWGFxFJieFFRFJieBGRlFodXleuXLFHHURErdLq8OrXrx/GjBmDDz/8ELdv37ZHTUREVrU6vE6ePIkhQ4ZAo9EgNDQUf/jDH3DsmA3fUU5E1IZaHV6RkZFYtWoViouLkZWVhZKSEowaNQqDBg1CRkYGbt68aY86iYiM3POAvYeHByZPnozt27fjzTffxOXLl5GcnAyVSoWEhASUlJS0ZZ1EREbuObxOnDiB+fPnIywsDBkZGUhOTkZBQQGys7NRXFyMiRMntmWdRERGPFq7QkZGBtavX4/8/Hw89dRT2LhxI5566ikoFHdysHfv3tiwYQPUanVb10pEZNDq8FqzZg1mz56NWbNmISwszGyb4OBgrFu37r6LIyKypNXhlZ2djZ49exqOtJoJIXDt2jX07NkTXl5eSExMbLMiiYju1uoxr759+6K8vNxkeUVFBXr37t0mRRERWdPq8BJCmF1eW1sLb2/v+y6IiMgWNp82ajQaAICbmxtSU1Ph4+NjuK+pqQnffvstIiMj27xAIiJzbA6vU6dOAbhz5HX27Fl4eXkZ7vPy8sLQoUORnJzc9hUSEZlhc3gdOHAAAJCUlIRVq1bB39/fbkUREVnT6ncb169fb486iIhaxabwmjx5MjZs2AB/f39Mnjy5xbY7duxok8KIiFpi07uNAQEBcHNzM/y/pZ97sXr1aqjVanh7eyM6OtrmWSq2bt0KNzc3TJo06Z4el4jkZdOR189PFdv6tHHbtm3QaDTIzMxEdHQ0Vq5cifj4eOTn5yM4ONjieoWFhUhOTsbo0aPbtB4ikoPDp4HOyMjA3LlzkZSUhIEDByIzMxM+Pj7IysqyuE5TUxNmzJiBJUuWoE+fPu1YLRF1FDYdeT3yyCOG00ZrTp48afODNzQ0IC8vDykpKYZlCoUCsbGxOHr0qMX1XnvtNQQHB2POnDk4dOhQi4+h1Wqh1WoNt6urqwEAOp0OOp3O5lo7msbGRsO/5vajeVlL+2htG9R+bOkLa33qLP1pa+02hZe9xpTKy8vR1NSEkJAQo+UhISG4ePGi2XW+/vprrFu3DqdPn7bpMZYtW4YlS5aYLN+3b5/RB21lc60WADzw9ddf43tfy+2ys7Pvextkf63pC0t96iz9WV9fb1M7m8IrLS3tvoppKzU1NZg5cybWrl2LwMBAm9ZJSUkxXB0A3DnyUqlUiIuLk/qzaueLq7Hi7DcYNWoUHu5huh86nQ7Z2dkYO3YsPD0972kb1H5s6Qtrfeos/dl8dmRNqz/n1ZYCAwPh7u6OsrIyo+VlZWUIDQ01aV9QUIDCwkJMmDDBsEyv1wO4M7Nrfn4++vbta7SOUqmEUqk02Zanp6fFF7UMPDw8DP+2tB8t7aet2yD7a01fWOpTZ+lPW2u3Kby6deuGS5cuITAwEF27dm1x/KuiosK2CnHnsqKoqCjk5OQYTk31ej1ycnKwcOFCk/YRERE4e/as0bJXXnkFNTU1WLVqFVQqlc2PTURysym83n77bfj5+Rn+b+vgvS00Gg0SExMxfPhwjBgxAitXrkRdXR2SkpIAAAkJCQgPD8eyZcvg7e2NQYMGGa3fpUsXADBZTkTOzabw+vnEgrNmzWrTAqZOnYqbN28iNTUVpaWliIyMxJ49ewyD+EVFRSYTHxIRtXrMy93dHSUlJSYfIL116xaCg4PR1NTU6iIWLlxo9jQRAHJzc1tcd8OGDa1+PCKSX5tNRqjVao2mySEisiebj7zeeecdAHcmI3z//ffh6/vfD5I0NTXh3//+NyIiItq+QiIiM2wOr7fffhvAnSOvzMxMuLu7G+7z8vKCWq1GZmZm21dIRGSGzeF19epVAMCYMWOwY8cOdO3a1W5FERFZ0+oB++YZVYmIHKnV4TV79uwW729pNggiorbS6vD68ccfjW7rdDqcO3cOlZWVePzxx9usMCKilrQ6vD755BOTZXq9HvPmzTO5rpCIyF7a5KPrCoUCGo3G8I4kEZG9tdl1NwUFBYbJ0IiI7K3Vp40/nxsLuPO5r5KSEuzatcvoGkgiIntqdXg1f3N2M4VCgaCgIKSnp1t9J5KIqK3wc15EJCXONUNEUmJ4EZGUGF5EJCWGFxFJqc3C64cffsBzzz3XVpsjImpRm4XXrVu3sG7durbaHBFRi3jaSERSYngRkZQYXkQkJZs/YT958uQW76+srLzfWoiIbGZzeAUEBFi9PyEh4b4LIiKyhc3htX79envWQUTUKhzzIiIp2XzkZet0N/wCDiJqDzaH14YNG9CrVy888sgjEELYsyYiIqtsDq958+bho48+wtWrV5GUlIRnn30W3bp1s2dtREQW2TzmtXr1apSUlOCll17C559/DpVKhSlTpmDv3r08EiOidteqAXulUolp06YhOzsb3333HR5++GHMnz8farUatbW19qqRiMhEq6eBbqZQKODm5gYhBJqamtqyJiKX85Puzmvo3PUqi23qftLixE0g9Psf0bmT0uT+yzdc6wCiVeGl1WqxY8cOZGVl4euvv8b48ePx7rvvYty4cVAo+KmL9mTtyW7tiQ643pO9Iyv43754ecdZKy09sOny8RZbdFbe8zGJVGzey/nz52Pr1q1QqVSYPXs2PvroIwQGBtqzNmqBbU926090wHWe7B1Z3MOhAIC+wb7o5Olutk1+SRX+/K+zSH96MPqHmb/ipbPSA70DO9utzo7E5mdtZmYmevbsiT59+uDgwYM4ePCg2XY7duxos+LIMmtPdlue6IBrPdk7sm6dvfB/R/RssU3zlzr3DeqMQeEtX67nCmwOr4SEBLi5udmzFmoFa092PtHJ2bXqQ6pERB0FR9mJSEoMLyKSEsOLiKTE8CIiKTG8iEhKDC8ikhLDi4ikxPAiIikxvIhISgwvIpJShwiv1atXQ61Ww9vbG9HR0Th27JjFtmvXrsXo0aPRtWtXdO3aFbGxsS22JyLn5PDw2rZtGzQaDdLS0nDy5EkMHToU8fHxuHHjhtn2ubm5mDZtGg4cOICjR49CpVIhLi4O169fb+fKiciRHB5eGRkZmDt3LpKSkjBw4EBkZmbCx8fH4leobd68GfPnz0dkZCQiIiLw/vvvQ6/XIycnp50rJyJHcugsdA0NDcjLy0NKSophmUKhQGxsLI4ePWrTNurr66HT6Sx+k5FWq4VWqzXcrq6uBgDodDrodLr7qL5ja54Sp7Gx0an305W4Sp/aum8ODa/y8nI0NTUhJCTEaHlISAguXrxo0zYWL16MHj16IDY21uz9y5Ytw5IlS0yW79u3Dz4+Pq0vWhLXagHAA9988w2un3N0NdQWXKVP6+vrbWon9fy/y5cvx9atW5Gbmwtvb2+zbVJSUqDRaAy3q6urDeNk/v7+7VVquztTVAGcPYFf/OIXGNqT36/pDFylT5vPjqxxaHgFBgbC3d0dZWVlRsvLysoQGhra4rorVqzA8uXLsX//fgwZMsRiO6VSCaXS9AsoPD094enpeW+FS8DDw8PwrzPvpytxlT61dd8cOmDv5eWFqKgoo8H25sH3mJgYi+v9/e9/x9KlS7Fnzx4MHz68PUolog7G4aeNGo0GiYmJGD58OEaMGIGVK1eirq4OSUlJAO7MnR8eHo5ly5YBAN58802kpqZiy5YtUKvVKC0tBQD4+vrC19fXYftBRO3L4eE1depU3Lx5E6mpqSgtLUVkZCT27NljGMQvKioy+k7INWvWoKGhAU8//bTRdtLS0vC3v/2tPUsnIgdyeHgBwMKFC7Fw4UKz9+Xm5hrdLiwstH9BRNThOfxDqkRE94LhRURSYngRkZQYXkQkJYYXEUmJ4UVEUmJ4EZGUGF5EJCWGFxFJieFFRFJieBGRlBheRCQlhhcRSYnhRURSYngRkZQYXkQkJYYXEUmJ4UVEUmJ4EZGUGF5EJCWGFxFJieFFRFJieBGRlBheRCQlhhcRSYnhRURSYngRkZQYXkQkJYYXEUmJ4UVEUmJ4EZGUGF5EJCWGFxFJieFFRFJieBGRlBheRCQlhhcRSYnhRURSYngRkZQYXkQkJYYXEUmJ4UVEUmJ4EZGUGF5EJCWGFxFJieFFRFJieBGRlDpEeK1evRpqtRre3t6Ijo7GsWPHWmy/fft2REREwNvbG4MHD8bu3bvbqVIi6igcHl7btm2DRqNBWloaTp48iaFDhyI+Ph43btww2/7IkSOYNm0a5syZg1OnTmHSpEmYNGkSzp07186VE5EjOTy8MjIyMHfuXCQlJWHgwIHIzMyEj48PsrKyzLZftWoVxo0bhxdffBEDBgzA0qVLMWzYMLz77rvtXDkROZKHIx+8oaEBeXl5SElJMSxTKBSIjY3F0aNHza5z9OhRaDQao2Xx8fHYuXOn2fZarRZardZwu7q6GgCg0+mg0+nucw86jvr6euTn5xtuXyqpgrb0Ms6d9kJDWYBR2/79+8PHx6e9S6RWuLs/Act96mz9aevr0qHhVV5ejqamJoSEhBgtDwkJwcWLF82uU1paarZ9aWmp2fbLli3DkiVLTJbv27fPqTq8oKAAf/7zn02Wz/zAtG16ejr69u3bDlXRvbLUn4Bpnzpbf9bX19vUzqHh1R5SUlKMjtSqq6uhUqkQFxcHf39/B1bWturr6zFq1CjD7dqftNh76DjiR/8f+HZSGrV1tr/Uzuju/gQs96mz9Wfz2ZE1Dg2vwMBAuLu7o6yszGh5WVkZQkNDza4TGhraqvZKpRJKpdJkuaenJzw9Pe+x8o4nICAAI0aMMNzW6XSoqazA6JG/cKr9dBV39yfgOn1q6745dMDey8sLUVFRyMnJMSzT6/XIyclBTEyM2XViYmKM2gNAdna2xfZE5Jwcftqo0WiQmJiI4cOHY8SIEVi5ciXq6uqQlJQEAEhISEB4eDiWLVsGAFi0aBEee+wxpKen49e//jW2bt2KEydO4L333nPkbhBRO3N4eE2dOhU3b95EamoqSktLERkZiT179hgG5YuKiqBQ/PcAceTIkdiyZQteeeUV/OUvf8GDDz6InTt3YtCgQY7aBSJyADchhHB0Ee2puroaAQEBqKqqcqoB+7vpdDrs3r0bTz31lFOPj7gSV+lTW1+jDv+QKhHRvWB4EZGUGF5EJCWHD9i3t+YhPls/CCcrnU6H+vp6VFdXO/X4iCtxlT5tfm1aG453ufCqqakBAKhUKgdXQkQtqampQUBAgMX7Xe7dRr1ej+LiYvj5+cHNzc3R5dhN82VQ165dc+p3VV2Jq/SpEAI1NTXo0aOH0cek7uZyR14KhQIPPPCAo8toN/7+/k79RHdFrtCnLR1xNeOAPRFJieFFRFJieDkppVKJtLQ0szNqkJzYp8ZcbsCeiJwDj7yISEoMLyKSEsOLiKTE8HKwX/3qV3j++ecdXQaRdBheksjNzYWbmxsqKysdXQq1Qkf749TR6rkfDC+iDq6hocHRJXRIDK8ORKvVYvHixVCpVFAqlejXrx/WrVuHwsJCjBkzBgDQtWtXuLm5YdasWVa3V1NTgxkzZqBz584ICwvD22+/bfKXd9OmTRg+fDj8/PwQGhqK6dOn48aNG4b7m4/4cnJyMHz4cPj4+GDkyJEmX4hKpmbNmoWDBw9i1apVcHNzg5ubGwoKCjBnzhz07t0bnTp1Qv/+/bFq1SqT9SZNmoTXX38dPXr0QP/+/QEAR44cQWRkJLy9vTF8+HDs3LkTbm5uOH36tGHdc+fO4cknn4Svry9CQkIwc+ZMlJeXW6ynsLCwvX4dbU+QQz322GNi0aJFQgghpkyZIlQqldixY4coKCgQ+/fvF1u3bhWNjY3i448/FgBEfn6+KCkpEZWVlVa3/fvf/1706tVL7N+/X5w9e1b89re/FX5+fobHE0KIdevWid27d4uCggJx9OhRERMTI5588knD/QcOHBAARHR0tMjNzRXnz58Xo0ePFiNHjmzrX4XTqaysFDExMWLu3LmipKRElJSUiNu3b4vU1FRx/PhxceXKFfHhhx8KHx8fsW3bNsN6iYmJwtfXV8ycOVOcO3dOnDt3TlRVVYlu3bqJZ599Vpw/f17s3r1bPPTQQwKAOHXqlBBCiB9//FEEBQWJlJQUceHCBXHy5EkxduxYMWbMGIv1NDY2OuJX0yYYXg7WHF75+fkCgMjOzjbbrjlEfvzxR5u2W11dLTw9PcX27dsNyyorK4WPj49ReN3t+PHjAoCoqakxetz9+/cb2uzatUsAED/99JNNtbiyn/9xsmTBggXid7/7neF2YmKiCAkJEVqt1rBszZo1onv37ka/87Vr1xqF19KlS0VcXJzRtq9du2b4o2drPbLgaWMHcfr0abi7u+Oxxx5rk+1duXIFOp3O6ItLAwICDKcgzfLy8jBhwgT07NkTfn5+hscvKioyajdkyBDD/8PCwgDA6PSSbLd69WpERUUhKCgIvr6+eO+990x+34MHD4aXl5fhdn5+PoYMGQJvb2/Dsru/lPbMmTM4cOAAfH19DT8REREAgIKCAjvukWO43JQ4HVWnTp3a/THr6uoQHx+P+Ph4bN68GUFBQSgqKkJ8fLzJIPHPZ+5sngdNr9e3a73OYOvWrUhOTkZ6ejpiYmLg5+eHt956C99++61Ru86dO7d627W1tZgwYQLefPNNk/ua/+A4E4ZXBzF48GDo9XocPHgQsbGxJvc3/xVuamqyaXt9+vSBp6cnjh8/jp49ewIAqqqqcOnSJfzyl78EAFy8eBG3bt3C8uXLDTPLnjhxoi12h/6Xl5eXUZ8dPnwYI0eOxPz58w3LbDkq6t+/Pz788ENotVrDhdnHjx83ajNs2DB8/PHHUKvV8PAw/9K+ux6Z8bSxg1Cr1UhMTMTs2bOxc+dOXL16Fbm5ufjnP/8JAOjVqxfc3NzwxRdf4ObNm6itrW1xe35+fkhMTMSLL76IAwcO4Pz585gzZw4UCoXhyKlnz57w8vLCP/7xD1y5cgWfffYZli5davd9dSVqtRrffvstCgsLUV5ejgcffBAnTpzA3r17cenSJbz66qsmIWTO9OnTodfr8dxzz+HChQvYu3cvVqxYAeC/R8ILFixARUUFpk2bhuPHj6OgoAB79+5FUlKSIbDurkfmo2eGVweyZs0aPP3005g/fz4iIiIwd+5c1NXVAQDCw8OxZMkSvPzyywgJCcHChQutbi8jIwMxMTEYP348YmNj8eijj2LAgAGGcZOgoCBs2LAB27dvx8CBA7F8+XLDC4LaRnJyMtzd3TFw4EAEBQUhPj4ekydPxtSpUxEdHY1bt24ZHYVZ4u/vj88//xynT59GZGQk/vrXvyI1NRUADP3Zo0cPHD58GE1NTYiLi8PgwYPx/PPPo0uXLobplO+u5+6xNplwShwXUldXh/DwcKSnp2POnDmOLofu0+bNm5GUlISqqiqHjJk6Gse8nNipU6dw8eJFjBgxAlVVVXjttdcAABMnTnRwZXQvNm7ciD59+iA8PBxnzpzB4sWLMWXKFJcMLoDhJa2ioiIMHDjQ4v3fffcdAGDFihXIz8+Hl5cXoqKicOjQIQQGBrZXmdSGSktLkZqaitLSUoSFheGZZ57B66+/7uiyHIanjZJqbGxs8dKOlt5xInIGDC8ikhLfbSQiKTG8iEhKDC8ikhLDi4ikxPAiu5o1a5Zh4jtPT0+EhIRg7NixyMrKatWlKRs2bECXLl3sV6gFzRMDUsfD8CK7GzduHEpKSlBYWIgvv/wSY8aMwaJFizB+/Hg0NjY6ujySlSMnEyPnl5iYKCZOnGiyPCcnRwAQa9euFUIIkZ6eLgYNGiR8fHzEAw88IObNm2cyIeLPf9LS0oQQQmzcuFFERUUJX19fERISIqZNmybKysoMj1NRUSGmT58uAgMDhbe3t+jXr5/Iysoy3F9UVCSeeeYZERAQILp27Sp+85vfiKtXrwohhEhLSzN53AMHDtjl90StxyMvcojHH38cQ4cOxY4dOwAACoUC77zzDs6fP48PPvgAX331FV566SUAwMiRI7Fy5Ur4+/ujpKQEJSUlSE5OBgDodDosXboUZ86cwc6dO1FYWGg0v/+rr76K7777Dl9++SUuXLiANWvWGK4w0Ol0iI+Ph5+fHw4dOoTDhw/D19cX48aNQ0NDA5KTkzFlyhTDkWNJSQlGjhzZvr8osszR6UnOzdKRlxBCTJ06VQwYMMDsfdu3bxfdu3c33F6/fr0ICAiw+nh3T2M9YcIEkZSUZLbtpk2bRP/+/YVerzcs02q1olOnTmLv3r1W6yfH4pEXOYwQwjAX1f79+/HEE08gPDwcfn5+mDlzJm7duoX6+voWt2FtGut58+Zh69atiIyMxEsvvYQjR44Y1j1z5gwuX74MPz8/w7TJ3bp1w+3bt51y2mRnw/Aih7lw4QJ69+6NwsJCjB8/HkOGDMHHH3+MvLw8rF69GkDL31nYPI21v78/Nm/ejOPHj+OTTz4xWu/JJ5/E999/jxdeeAHFxcV44oknDKectbW1iIqKwunTp41+Ll26hOnTp9t57+l+8cpdcoivvvoKZ8+exQsvvIC8vDzo9Xqkp6cbJs1rnkG2mbnpi22dxjooKAiJiYlITEzE6NGj8eKLL2LFihUYNmwYtm3bhuDgYPj7+5ut05mmTXY2PPIiu9NqtSgtLcX169dx8uRJvPHGG5g4cSLGjx+PhIQE9OvXDzqdzjAd9aZNm5CZmWm0DbVajdraWuTk5KC8vBz19fU2TWOdmpqKTz/9FJcvX8b58+fxxRdfYMCAAQCAGTNmIDAwEBMnTsShQ4cMU2//6U9/wg8//GB43P/85z/Iz89HeXk5dDpd+/zSyDpHD7qRc0tMTDR8zMDDw0MEBQWJ2NhYkZWVJZqamgztMjIyRFhYmOjUqZOIj48XGzduNPmeyj/+8Y+ie/fuRh+V2LJli1Cr1UKpVIqYmBjx2WefmXyX4YABA0SnTp1Et27dxMSJE8WVK1cM2ywpKREJCQkiMDBQKJVK0adPHzF37lxRVVUlhBDixo0bYuzYscLX15cflehgOCUOEUmJp41EJCWGFxFJieFFRFJieBGRlBheRCQlhhcRSYnhRURSYngRkZQYXkQkJYYXEUmJ4UVEUmJ4EZGU/j8xp2xEn/CuKAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T23:20:14.810763Z", + "iopub.status.busy": "2024-03-22T23:20:14.809983Z", + "iopub.status.idle": "2024-03-22T23:20:15.081413Z", + "shell.execute_reply": "2024-03-22T23:20:15.080580Z" + }, + "papermill": { + "duration": 0.293759, + "end_time": "2024-03-22T23:20:15.083400", + "exception": false, + "start_time": "2024-03-22T23:20:14.789641", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABnaklEQVR4nO2dd3hTVRvAfzej6R5QuqBsZAllWQREQIsFUcCJuFiCCxcqS2UICiIiDpTPgYCCiIKIyC5LsSJ7yIayShejk46M+/2RJk3apE3adHJ+z3MfknvPPXkTmjfvOe+SZFmWEQgEghqIorIFEAgEgvJCKDiBQFBjEQpOIBDUWISCEwgENRah4AQCQY1FKDiBQFBjEQpOIBDUWISCEwgENRZVZQtQFTEYDFy+fBkfHx8kSapscQQCQSFkWSYjI4OwsDAUCvt2mlBwNrh8+TLh4eGVLYZAICiBixcvUq9ePbvXhYKzgY+PD2D88Hx9fStZGoFAUJj09HTCw8PN31V7CAVnA9Oy1NfXVyg4gaAKU9IWknAyCASCGotQcAKBoMYiFJxAIKixVOoe3I4dO/jwww/Zu3cvCQkJ/PrrrwwcONDu+G3bttGrV68i5xMSEggJCTE/nzdvHh9++CGJiYlERETw2WefERkZWR5vocqi1+vRarWVLYagiqJUKlGpVDU+DKpSFVxWVhYREREMHz6cBx980OH7Tpw4YbX5HxQUZH78008/MWbMGObPn0/nzp2ZO3cu0dHRnDhxwmpcTSYzM5NLly4hapkKisPT05PQ0FDc3NwqW5Ryo1IVXN++fenbt6/T9wUFBeHv72/z2pw5cxg5ciTDhg0DYP78+fzxxx8sWLCA8ePHl0XcaoFer+fSpUt4enpSp06dGv8LLXAeWZbJy8sjJSWFuLg4mjVrVmywbHWmWoaJtGvXjtzcXG699VamTJlCt27dAMjLy2Pv3r1MmDDBPFahUBAVFUVsbKzd+XJzc8nNzTU/T09Pd0qerFwd41cepkWIDy/2aurku3EtWq0WWZapU6cOHh4elSqLoOri4eGBWq3m/Pnz5OXl4e7uXtkilQvVSm2HhoYyf/58VqxYwYoVKwgPD6dnz57s27cPgCtXrqDX6wkODra6Lzg4mMTERLvzzpgxAz8/P/PhTBaDLMuMW3GI3w9e5qONJ0jJyC35pgpAWG6CkqipVpsl1eodNm/enGeffZaOHTvStWtXFixYQNeuXfn444/LNO+ECRNIS0szHxcvXnT43oV/n2PNoQQADDKsPZxQJlkEAoHrqFYKzhaRkZGcPn0agMDAQJRKJUlJSVZjkpKSrLyshdFoNOasBWeyF/aev8Z7fxwDoG09PwB+P3i5NG9DIBCUA9VewR04cIDQ0FAA3Nzc6NixIzExMebrBoOBmJgYunTp4vLXXns4EZ1Bpl/bUL56qhOSBHvOXyc+Ndvlr1XT6dmzJ6+++mpli1Fj2LZtG5IkkZqaWtmiVCqV6mTIzMw0W18AcXFxHDhwgFq1alG/fn0mTJhAfHw8ixcvBmDu3Lk0atSI1q1bk5OTwzfffMOWLVvYuHGjeY4xY8YwZMgQOnXqRGRkJHPnziUrK8vsVXUlb/drSctQX/rcGoK3RkVkw1rsirvGmoOXebZHE5e/nqAgFvL69et2PekCgYlKVXB79uyxCtwdM2YMAEOGDGHhwoUkJCRw4cIF8/W8vDxef/114uPj8fT0pG3btmzevNlqjkGDBpGSksKkSZNITEykXbt2rF+/vojjwRVIksTDHQtKtdwfEcauuGv8fkgoOEHZycvLq9ExahWCLChCWlqaDMhpaWlO3Xc1M1duPOEPucG4NfKZ5Ixykq54srOz5aNHj8rZ2dmyLMuywWCQs3K1lXIYDAaH5e7Ro4f8yiuvyLIsyzk5OfLYsWPlevXqyW5ubnKTJk3kb775Ro6Li5MBq2PIkCElzp2eni4//vjjsqenpxwSEiLPmTPH6vVkWZYXL14sd+zYUfb29paDg4PlwYMHy0lJSebrW7dulQF58+bNcseOHWUPDw+5S5cu8vHjxx1+j9OmTZPr1Kkje3t7yyNGjJDHjRsnR0REmK8PGTJEHjBggDx9+nQ5NDRUbtiwoUOyybIs//HHH3KzZs1kd3d3uWfPnvJ3330nA/L169ftylP4b6U64eh3tFrGwVVVanm5cUfTQLafTOH3gwm8EtWsskUiW6un1aQNlfLaR9+NxtPN+T+xp59+mtjYWD799FMiIiKIi4vjypUrhIeHs2LFCh566CFzNosjsX5jxoxh586drF69muDgYCZNmsS+ffto166deYxWq2XatGk0b96c5ORkxowZw9ChQ1m7dq3VXG+99RYfffQRderU4bnnnmP48OHs3LmzRBmWLFnCe++9xxdffEG3bt1YtmwZH330EY0aNbIaFxMTg6+vL5s2bXJYtosXL/Lggw/y4osvMmrUKPbs2cPrr79eokylJStXR+yZq9zRLBB3tbLcXscVCAXnYu6PCGP7yRRWH4zn5bubing0Jzl58iTLly9n06ZNREVFAdC4cWPz9Vq1agHFZ7NYkpGRwaJFi1i6dCl33303AN999x1hYWFW44YPH25+3LhxYz799FNuu+02MjMz8fb2Nl9777336NGjBwDjx4+nX79+5OTklBgo+9lnnzFixAjzXvCkSZPYuHEjmZmZVuO8vLz45ptvrJamJcn25Zdf0qRJEz766CPAGE51+PBhPvjggxI/n9Iwf/sZPttymqn9WzOka8NyeQ1XIRSci7mndTBuvyo4k5LF8cQMWoZWbsFMD7WSo+9GV9prO8uBAwdQKpVmJVJWzp49i1artSq24OfnR/Pmza3G7d27lylTpnDw4EGuX7+OwWAA4MKFC7Rq1co8rm3btubHJu99cnIy9evXL1aOEydO8MILL1idi4yMZMuWLVbn2rRpU2TfrSTZjh07RufOna3uKY+oAROJaTkAXLp+o9xew1UIBedifN3V9Gpehw3/JbH64OVKV3CSJJVqmVhZVEZ6WVZWFtHR0URHR7NkyRLq1KnDhQsXiI6OJi8vz2qsWq02PzZZ5yaF4wq8vLxKLVtFkaszvt+MHF2lvL4zVPs4uKpI/4i6gDHoVxYVPZyiTZs2GAwGtm/fbvO6ybrR6/UOzde4cWPUajW7d+82n0tLS+PkyZPm58ePH+fq1avMnDmT7t2706JFC5KTk8vwLorSvHlzKxmAIs9t4YhsLVu25N9//7U6988//5RdaDvk6oyffUauUHA3JXe1CMLTTcml69kcuJha2eJUKxo2bMiQIUMYPnw4q1atIi4ujm3btrF8+XIAGjRogCRJrFmzhpSUlCJ7WIXx8fFhyJAhvPnmm2zdupX//vuPESNGoFAozBZY/fr1cXNz47PPPuPs2bOsXr2aadOmufR9vfTSS3z77bcsWrSIU6dOMX36dA4dOlTiHq0jsj333HOcOnWKN998kxMnTrB06VIWLlzoUvktMVlwmcKCuznxcFPSu5Ux7m61SN1ymi+//JKHH36YF154gRYtWjBy5EiysrIAqFu3LlOnTmX8+PEEBwczevToEuebM2cOXbp04b777iMqKopu3brRsmVLs2OgTp06LFy4kJ9//plWrVoxc+ZMZs+e7dL39MQTTzBhwgTeeOMNOnToQFxcHEOHDi3ROeGIbPXr12fFihWsWrWKiIgI5s+fz/vvv+9S+S3J1ZqWqFW/oKokizVUEdLT0/Hz8yMtLa3UXbVijiUxYtEegnw0xE64G6WiYrypOTk5xMXF0ahRoxpbAqesZGVlUbduXT766CNGjBhRaXL07t2bkJAQvv/++0p5/dL+rTz4xU72XUjllmBvNr7mGmeQszj6Ha0+u8/VjO7N6uDrriI5I5ddcVfp2iSwskW6adm/fz/Hjx8nMjKStLQ03n33XQAGDBhQYTLcuHGD+fPnEx0djVKp5Mcff2Tz5s1W8W7VBeFkEOCmUtD3VmMYwe8HRQml8uLChQt4e3vbPUypfrNnzyYiIoKoqCiysrL4888/CQx03Y9O69at7cqwZMkSJEli7dq13HnnnXTs2JHff/+dFStWmGP9qhM5WqOToTrswQkLrhzp3y6Mn/ZcZN2RBN4d0Bq1UvyeuJqwsDAOHDhQ7PX69euzd+/ecpVj7dq1dpv8BAcH4+HhwebNm8tVhorC7GTI02EwyCgqaPulNAgFV47c3rg2gd4armTm8tepK/RqcXM0valIVCoVTZtWbpl4MHp3bxZMCk6WIStPh4+7uoQ7Kg9hUpQjSoVEvzbGQpuiEKagppCrLYhBrOr7cELBlTP92xlzHjceTTLvXQgE1RmTBQeQWcWDfYWCK2fahwdQ19+DzFwdW4+7NjpeIKhoZFm2UnBVPRZOKLhyRqGQuK9tvjf1kFimCqo3eXrrvFuxRBVwf4RxmRpzLLnKm/QCQXFYWm/gWgVXHt8NoeAqgNZhvjQO9CJXZ2DTUfv9WQWlp2HDhsydO9f8XJIkVq1aVWny1FRMaVomXKWUtHoDHadtovusLVzJdF1vYaHgKgBJksxWnAj6rRgSEhLo27dvZYtR4yjsKHPVHtyJxAxydQbSbmip7eW6PhRCwVUQ90cY9+F2nEwh9Ubl1PG6mQgJCUGj0VSqDLIso9PVrC2JwktUV2UzHI5PA6BtPX+XVsEWCq6CaBrkQ8tQX3QGmXVHKnCZKsuQl1U5h5N1HDIyMnjiiSfw8vIiNDSUjz/+uNT9Ui2XqOfOnUOSJFauXEmvXr3w9PQkIiKC2NhYq3v++usvunfvjoeHB+Hh4bz88svmKiYA33//PZ06dcLHx4eQkBAef/xxq9pspl6k69ato2PHjmg0Gv766y+nZa/KmGrBmUh3kYI7dMmo4NrkN1B3FSKToQLpHxHGsYR0fj94mcGRxZe4dhnaG/B+WMnjyoOJl8HNq+Rx+TjSHKYsvPXWW8yePZtmzZrx1ltvMXjwYE6fPo1KpeLMmTP06dOH6dOns2DBAlJSUhg9ejSjR4/mu+++AxxvTDN+/Hhmz55N48aNCQgIcInsVYUiFpyL9uAOx6cySbWYTtruIDcHF1lxQsFVIPe1DeWD9ceJPXuV5PQcgnxFOSMTjjaHKQtvvPEG/fr1A2Dq1Km0bt2a06dP06JFC2bMmMETTzxhthabNWvGp59+So8ePfjyyy9xd3d3uDHNu+++S+/evV0md1WisJPBFXtwOVo9gYk7Ga5ej7x3A0TeBcGtSr7RAYSCq0DCa3nSvr4/+y+k8sfhBIZ1a1TyTWVF7Wm0pCoDtafDQx1tDlMW7DWMadGiBQcPHuTQoUMsWbLEPEaWZQwGA3FxcbRs2dLhxjSdOnVymcxVjcJLVFdYcCcvJjFN+Y3xSednXabcoJL34Hbs2MH9999PWFiYQ279lStX0rt3b+rUqYOvry9dunRhwwbrnp9TpkxBkiSro0WLFuX4Lpyjv9mbWkFKR5KMy8TKOKpYy8TiGsZkZmby7LPPcuDAAfNx8OBBTp06RZMmTczNX3x9fVmyZAm7d+/m119/BSjS/KVw45iqyNWsXK5nOe/sKo84ONX29wlXpHBFGYR01ztlns+SSlVwWVlZREREMG/ePIfG79ixg969e7N27Vr27t1Lr169uP/++9m/f7/VuNatW5OQkGA+qtJGb782oUgS7LuQysVrVb/tWkXhSHOY8qRDhw4cPXqUpk2bFjnc3NwqpDFNRaE3yFy+nk1KRo7TTZFMCk6VXyKpzF7U+L20OG+0mnfcMhE03iXc4ByVukTt27evU7FKloGcAO+//z6//fYbv//+O+3btzefV6lUhISEuEpMlxLk687tjWoTe/Yqaw4l8HzPJpUtUpXAsjlMrVq1CAoKYvLkyVbNYcqTcePGcfvttzN69GieeeYZvLy8OHr0KJs2beLzzz+3av7y3HPPceTIEZc3pqkodHoDMpgPZzBVEgn01pCYnlM2L6peC6tfRoGBX/Xd8G1zb+nnskO1DhMxGAxkZGSYu52bOHXqFGFhYTRu3JgnnnjCXNXVHrm5uaSnp1sd5YmpwogooWRNSc1hypO2bduyfft2Tp48Sffu3Wnfvj2TJk0yOzkqojFNRaE1WKg1JzVcTr4FF+hjDMbNzC2Dk2HnJ5B0hGuyN9O0T7k8RASquZNh9uzZZGZm8uijj5rPde7cmYULF9K8eXMSEhKYOnUq3bt358iRI/j4+NicZ8aMGUydOrWixKZP6xDeWXWEownpnE7OpGmQa83y6oqPj4/VJn9WVhZTp05l1KhRJd577tw5q+eWS6+GDRsWWYr5+/sXOXfbbbexceNGu68xePBgBg8ebPd1evbsWS364Or0pW9UbbLgansZg6hztAa0eoPz1aqvnILtswCYqn0atW8dgsshqqDaWnBLly5l6tSpLF++nKCggkq5ffv25ZFHHqFt27ZER0ezdu1aUlNTzX01bTFhwgTS0tLMx8WLF8tV9gAvN+68pQ4grDhL9u/fz48//siZM2fYt28fTzzxBFCxzWFuBnT6AiXs9BI134Kr7V2QTuX0PpzBAL+/AvpcLtbuxm+GbrSt5++kJI5RLRXcsmXLeOaZZ1i+fHmJTTv8/f255ZZbOH36tN0xGo0GX19fq6O8MaVu/X7ocrX41a8obDWHOXbsWLGNZQTOoTWUwYLLV3Bebio81EqgFJ7UfYvg/E5Qe7HA/yVAom1d1y9PoRouUX/88UeGDx/OsmXLzEGbxZGZmcmZM2d46qmnKkA6x+ndKgSN6jBnU7L473I6t5bTf3B1on379jabw2RnZxfbWEbgHJYWnLOY4uA0KgU+7iqytXoynNmHS0+ATZOMj+9+h+1/eQBZ5bL/BpWs4DIzM60sq7i4OA4cOECtWrWoX78+EyZMID4+nsWLFwPGZemQIUP45JNP6Ny5M4mJxpxODw8P/PyMH9Abb7zB/fffT4MGDbh8+TKTJ09GqVQW2TupbLw1Ku5uGcTaw4n8fuiyUHDF4OHhUSUay9QUtGXagzPeq1Er8M7v++uUBbf2DchNh7odSW87jLOrYgBq5hJ1z549tG/f3hziMWbMGLP3Cowlbyw9oF999RU6nY4XX3yR0NBQ8/HKK6+Yx1y6dInBgwfTvHlzHn30UWrXrs0///xDnTp1KvbNOcD9bY0eujUHE1y+TBXLXoE9dCYvqiwjO7kLZ1qialRKczcth/fgjq6G42tAoYL+n3EkIROAegEe1HJhiSRLKtWCK8nrtHDhQqvn27ZtK3HOZcuWlVGqiqNXiyC8NSriU7PZd+E6HRvU4kaejn/jrnF749q45+9xOINSabwnLy8PDw8PV4ssqAGYlqiyLg9Zts7wKAmrJarGqD4cWqJmpxqtN4A7XoPg1hw6fgaAtuW0PIVquAdXk3BXK7mnVTAr98ez7N+L7Iq7xrd/xnE1K4/nejRhfF/nU8xUKhWenp6kpKSgVqtRKKqlH0lQThhkGW1uNrIuj+vXrhAcWMv8o+gIJgvOXa3EO1/BOWTBbXoHMpOgdjPoblR0hy8V1IArL4SCq2Tujwhj5f54ft57yep8cnpOqeaTJInQ0FDi4uI4f/68K0QU1CD0BpnktGy0epmYs5m80Ne5vU3zHly+kwEcqAkXtwP2GffR6f8ZqI3xbofiUwHKzYMKQsFVOnc0CyTIR0NyRi5N6njRKNCbzceSyC3DRrCbmxvNmjUrkgQuEJxITGfSr/u4nmMgRyfzfF/n0uDMS9R8JwOUUFFEm22MeQPoNAIadAHgWlYeF69lA9BaKLiai1qp4Kdnu5CQmk3nxrVZvueiUcFpS6/gABQKRYWkOAmqFyk30kjILCh55HSyvbaok6HYmnDbP4BrZ8EnDKImm0+bSpQ3DvTCz8PxPUBnEQquCtAo0ItGgcYSO275KS+F+08KBK4gpVDHKr3T1USKOhns7sElHIKdnxof9/sI3AsstcOXUgHXlygvjNiBrmK4qfIVXKHCggKBK0hOt1ZwziY1WIeJFLNE1etg9Usg66HVQGhhXSnkoKkHQznHfwoFV8XQmBWcsOAEricl09p5ZShlPTjLPTibToZdX0LCAaPV1ndWkcsmD2pEuL9Tr+8sQsFVMUwWXOHKqQKBKyhiwTm9B2eZqmUn0PdaHGx5z/j4nvfAJ7iQDDkkpuegkKBVaPnmfYs9uCqGm7DgBOVIckZhBefc/dZxcMabrQJ9ZRnWvAq6bGh0J7R/ssgcphaBTYO88dKUrwoSCq6KYV6iCieDoBxIKaLgSpuqpUByt+FkOLAUzm4DlTvc/4nNvhyH4k37b/5OvXZpEAquiqFRGaPKyxomIhAURpZls4JTKyW0erkUPRlMS1SlebWRkaNDlmWkrBTYMNE4sNdEqNXY5hwmD2pEePkXmBB7cFUMN2HBCcqJ9Gyd+e8qyMcYI+nMn5neIKPNz2O13IPTGWSjZbduHOSkQkhbuP1Fm3PIslzQxb4CKugIBVfFMMfBiT04gYtJzjB6UP081LirjX9nzixRLXuiatQKPNVK8wo057818N9KkJTGdCyl7cXh5bQcrmbloVJItCxnBwMIBVfl0KhNXlQRBydwLSYHQx0fDcr8tn9OKTiLbRM3pQKFQsJbo8KbG3hvHme80HU0hLWzO4dpedo8xKdU1XKcpUx7cJmZmebGuSYqotx3TcZkwWn1MgaDjEJRtZonC6ovpv23IB8N1/KbPlt+fS9eu0FattZu8VXLnqiq/L9TH42KZ3U/ocpMgIBG0GN8sTIcNFcQqZgCr05bcHFxcfTr1w8vLy/8/PwICAggICAAf39/AgICykPGmwrTHhyIfTiBazEtUev4aMy9Zi0tuCe/3cUDX+ws0vHeYDA6IyzTtEx0Vp3iKeVm45P7PwE3z2JlOHyp4jyoUAoL7sknn0SWZRYsWEBwcHCFNOW9mTB5UcGo4CrCjBfcHFhacKeTjdV0TQpOlmUuXLuBLBvzVQPyK+xq9Qbu/eRPgnw1TL6/NUDB36QulzE5n6OQZC41fIh6jXsU+/pGB0MqUHEWnNMK7uDBg+zdu5fmzZuXhzzVj7wsUHvajPcpDWplwTy5WgOIgiACF5FsVnDu5j04kwF3I09vfpyjLdj/TUrP4VRyJqeSM83nzRbcnx8Rrr9IiuzHnlvGUK+E1z9/9QbpOTrcVApuCbbdo9jVOL1Eve2228q9b2i14eQG+Pw2OPyzy6aUJEmEigjKhRQLJ4Np5aXPT2WwTJjPzitQcDkWjoUbeaZacEpIOgp/zgFgknYo1wxeJb6+KcC3Zaiv1VZMeeK0BffNN9/w3HPPER8fz6233lqknnvbtm1dJlyVJ/EwpMcb43+a3AVegS6ZVqNUkKcziFARgUtJtliimnxXpiWqZWesHIu/O0tr7kaecYyHUjZWCjFo+c+3O+uSI2lZXNHLfMwBvhW0PIVSKLiUlBTOnDnDsGHDzOckSTJGMksSev1NFN7Q7RU4shKS/4P1E+Chr10yrUatICNXhIoIXIupDH4dHw1Ks5PBeM3SgrNUapaPM3ONjwfq1kH8HtD4EtP4TUjOLL7oZT4VGeBrwmkFN3z4cNq3b8+PP/4onAxKtTGo8dsoOLwc2j4KzXqXeVoR7CtwNTlavbmsUZCPOwrJtAeXv0TNsa3gsi0eZ+XqCOMKT99YZDwRNQXSw4CTxZctx7gUPhJf/k1mCuO0gjt//jyrV68WjXhN1OsInZ+Hf+bBmtfghX9A412mKUVFEYGzHLiYSr0ADwK9NTavm/bf3FQKfD1UZp+YqaJvpkVFEGsLruBvMCtHy3T1AtzlHKjfBToOwyfW2NiopMYzcVcyycrT46FW0jSobN8PZ3B6p++uu+7i4MGD5SFL9eWut8C/PqRdhC3TyzydOeFeKDiBAxxPTGfgvJ28sGSf3TGmUuV1vI0OBkWhJarlHpy1k6Hgcd34tdylPIAWNdz/KSgUDrcONC1Pb63ra/bgVgROK7j777+f1157jSlTprBixQpWr15tdTjDjh07uP/++wkLC0OSJFatWlXiPdu2baNDhw5oNBqaNm1apDk0wLx582jYsCHu7u507tyZf//91ym5nMbNC+772Ph413y4tKds0wkLTuAExxLSAaMVp7dT4M1U6DLI12jhFYSJFPWiWjoZTEvUANLpcWY2AOtrPwV1bgFwrPEMlvtv/g6+K9fg9BL1ueeeA+Ddd98tcs1ZJ0NWVhYREREMHz6cBx98sMTxpiyK5557jiVLlhATE8MzzzxDaGgo0dHRAPz000+MGTOG+fPn07lzZ+bOnUt0dDQnTpwgKCjIYdmcpmkUtH0MDi0zephGbQeVW6mmElV9Bc4Qf93Yfi9PZ+DCtRvmBkaWpJiyGPKXsOYlqqH4PThTBd+31T/gqUvlmCGcHUGPc3/+9WL7MlhQ0QG+Jpy24AwGg93DWQ9q3759mT59Og888IBD4+fPn0+jRo346KOPaNmyJaNHj+bhhx/m448/No+ZM2cOI0eOZNiwYbRq1Yr58+fj6enJggULnJKtVPSZAZ6BkHwUdn5S6mlEZy2BM1zKV3AAJ5MybI4xZzHkW3CFl6hWcXCF9uDuVBzkIeVfGJAYrx2JSl0QfW5ScBnFLFF1egP/XTZamVVawWm1WlQqFUeOHCkveYolNjaWqKgoq3PR0dHExsYCkJeXx969e63GKBQKoqKizGNskZubS3p6utVRKjxrQd8PjI93zIKUk6WaxlRR5GqhFm8CgS0sFdwpOwrOMosBKBoHZ6HgLKuG6HIyeF/9LQAxvg9yUG5qlYvqyB7cyaRMcnUGfDQqGtYuOSDYlTil4NRqNfXr16+0WLfExESCg60bWAQHB5Oenk52djZXrlxBr9fbHJOYmGh33hkzZuDn52c+wsPDSy/krQ9Bs3tAnwe/v+x8XzagdZixIsvsDSfMpr1AYI/4VEsLLtPmGMssBrCxB2fHydDp7JfUk65wSQ5kqddTQMEPMBTswWXm6TDY2f87HJ8KwK11/Sq8Oo7TS9S33nqLiRMncu3atfKQp1KYMGECaWlp5qNMqWiSBP3mgNoLLsTCXueXxi/f3YxuTWuTladn6He7OZNi+49WIDAYZPMeHNhfolpmMQAWqVrG69ZOhnwFd2kvHRN/AuAt7QiuaY3KzLIghGmJKsuQlWfbijM5GNpWQInywjit4D7//HN27NhBWFgYzZs3p0OHDlZHeRISEkJSUpLVuaSkJHx9ffHw8CAwMBClUmlzTEhIiN15NRoNvr6+VkeZ8A+HqMnGx5umQFq8U7drVEr+91Qn2tbz41pWHk9/+y8Jadkl3yi46biSmWu1V3v2ShY6G3u3hS24wkvUIk4GvRZWv4QCAyv1d7DdEGFWgpZLVI1KYS4QYW8fzqzgKtiDCqXwog4cOLAcxHCMLl26sHbtWqtzmzZtokuXLgC4ubnRsWNHYmJizHIaDAZiYmIYPXp0xQp72zPGJPxLu+GP12Hwj05VHPHWqPhu6G08Mj+Ws1eyePrbf/n5uS74e5bOMyuomVzMt97C/Ny5diOPHK3Rk9q4TkEwrcEgcyWz8B6c9RI1w8rJYDA6yZL/I1Ppx7QcY+s/c7K9hYKTJGNV3+s3tDY9qbk6PccTK8fBAKVQcJMnT3bZi2dmZnL69Gnz87i4OA4cOECtWrWoX78+EyZMID4+nsWLFwPGEJXPP/+csWPHMnz4cLZs2cLy5cv5448/zHOMGTOGIUOG0KlTJyIjI5k7dy5ZWVlWubMVgiK/Nv387nByHRxdBa0d8xabqO2tYfGISB768m9OJWcyfOFufnimM55uohmawIhp/61eLU9qebtxJD6dk0mZVgru2o08dAYZSYLa3sYfSIWisBe1II4tIPscbDd2o18e+CLXs3zzx+RbcIVqFPq4q7l+Q2szFu5EYgZavUyAp5p6AR4ueMfOUeqaJXv37uWHH37ghx9+YP/+/aWaY8+ePbRv35727dsDRuXUvn17Jk2aBEBCQgIXLlwwj2/UqBF//PEHmzZtIiIigo8++ohvvvnGHAMHMGjQIGbPns2kSZNo164dBw4cYP369UUcDxVCUEvoPsb4eO2bcMP5fct6AZ58P6Izfh5q9l1I5YUl+9CK8BFBPpeu3wCgXoAHtwQZa6wV9qSalqe1PN1Q54cgKQqXS8pfXkoYGHV9LuhzoWlv/tT0NM9jsuDcC5U6MnlSbS1RzQG+9fwrJW/daVMgOTmZxx57jG3btuHv7w9AamoqvXr1YtmyZdSpU8fhuXr27FlsX0ZbWQo9e/YsUaGOHj264pek9uj+Ovy3Cq6cgE3vwIB5Tk9xS7APC4bexhPf/MO2Eym8+fNB5jzaTvRrEJgdDPX8PfDIt+xPJls7pZIL7b+B9R6cLMtm62ywciutdf8ZnWT3zSFn+WXzPSZlWNSCs6/g4q5kAdA8uOLyTy1x2oJ76aWXyMjI4L///uPatWtcu3aNI0eOkJ6ezssvv1weMlZvVBrjUhUJ9v9g7PpdCjo2CODLJzuiUkisOnCZaX8cdbppr6DmYYqBqxfgyS35SsSeBWep4JTmPThjxoxWLxPMNcarlhoH3D0J/OtbBf2a0BSy4IrLZkhMM2ZQhPpV/PIUSqHg1q9fzxdffEHLli3N51q1asW8efNYt26dS4WrMdTvbHQ6APz+KuTdKNU0vZoHMfuRCAC+23mOL7adcZGAgqrMuStZDPvuX3afK7rFYVqi1g3wMJcBP5ti7Uk1NZsxORjAIkzEbL3JTFN/h6+UzWGpGUSOBKzTtkwUVXD281Ev53v/w/wrp/Z+qVK1ClfxBWMQcOEWggIL7p4EvnXhehxsm1HqaQa2r8uk+1oB8OGGEyzddaGEOwTVnZd+3M/WEyk8Mt86G0eW5QInQ4AHdf098FArydMbOH+t4EfUlgVnuUTNzNHRR7Gbe5R70cpKJsnPGp1k2M6HtoyDg+KzGUwWXEh1seDuuusuXnnlFS5fLlibx8fH89prr3H33Xe7VLgahbuvMQAYIPZzuHyg1FMNv6MRL/ZqAsDbqw6z7nCCCwQUVFXsxUBezTKGhUiScQmoUEjmWmuWy9TCQb5gGSYCN9Kv8K56IQBf6Pvzn7agfYxlVoMJy0wGsNiDK7RE1ekN5tcO9asmFtznn39Oeno6DRs2pEmTJjRp0oRGjRqRnp7OZ599Vh4y1hya94HWD4JsgNWjjcGUpeSNe5ozODIcgwyvLDvA36evuFBQQVVCYcf7aHIwBPu4myvQNMvfh7NM2UopVCoJLMJEDDK1/36PICmVOOoyTzeQPL3B7FDIsVE2v/AS1duOkyElMxe9QUalkOwW4ixvnPaihoeHs2/fPjZv3szx48cBaNmyZZEkeIEd+n4AZ7YYG9bEzoM7Xi3VNJIkMX1gG65naVn/XyIjF+9h2agutKmEYEpB+aKy4y03ORjqWsSXmfbhTDXiwLrYpQnTlKHXdxN82piO9bn3S+TlGLefcrR6vDQq2xacqmgcHBRdoibkL0+Dfd0rtMilJaWKg5Mkid69e/PSSy/x0ksvCeXmDN5BEP2+8fG2GXC19I4CpUJi7mPt6NLYlLf6r3nPQ1BzsBcOFJ9aEANn4raGtQD489QVs4PA1GwmyLdgmaiQJDTkcfdp49/i97ooEv3bm69na/X53eyL7sG5F16imuLgcq1XJAX7b5XX3LdUIfExMTHExMSQnJxcxLFQIXXXqjvtHjc2qTm7DX5/BYb8XurG0e5qJV893ZFH//cPxxLSmbPpBLMejnCtvIJKxZ71Y7bg/AsUXPtwf0L93ElIy2HHyRS6NQ0kK98KK+xkeFW1goCci2Rpgvgg5zG6u6vRqBTk6gzkaPV2C6466mS4nO8Aqaz9NyiFBTd16lTuueceYmJiuHLlCtevX7c6BA4gSXDfXFB5wLk/jfFxZcDHXc17D9wKwC97L9mtCSaonthTcPEWMXAmFAqJvreGArD2cILZg+rppjQrIoCwnJOMVBpTHLc3G08mnnhrVLjnB/HmaA02l6dQjJOhkIIriIGrRhbc/PnzWbhwIU899VR5yHPzUKuRsVnNxrdh41vGGnI+pU8n61A/gOjWwWz4L4lZG07w9dOdAONeSlq2lmDfyvsjE5QNpR3r3tYeHEC/tiEs2BnH5mPJPNTR6BG19KCi19H//ExUkoFjtaI44t0NOIO3uwp3tYK0bOPfjS0HAxTjZCjkRU1Ir9wQESiFBZeXl0fXrl3LQ5abj87PQ2g7yEmDdW+Webo3o1ugkGDT0ST25AeFjv3lEN1mbuHgxdQyzy+oHGxZcIVj4CxpHx5AqJ87mbk6Vuy9BFgvT/nnC0JvnCBV9mJjg9fMGQg+GhUeZgtOb9Uy0BJTSX0TvnYCfRNSCyqdVBZOK7hnnnmGpUuXlocsNx9KlTGNS1LC0d/g2JoyTdc0yJtBtxmrEc9cd5w8nYHVBy+jM8h8uOGEKyQWVBDZeXrzEtGWgkvLLihPZLkHB9bL1D/yYyTNWQzXzsJWo2PhPd0TZKpqmffOjBZc8UtUjUpRJGnetPTN0RqsCkFUSydDTk4OX331FZs3b6Zt27ZFshrmzJnjMuFuCkLbQreX4a+PYe0b0Kg7uJc+1OOVu29h5b549py/ztzNBT0hzl/LcoW0ggpAb5BpPXk9BhlOvdfXpoIzLU8DvTVmpWRJv7ahLNgZh1ZvjGer46MxRvX+/irosjnn24mfk3sw3FCwtPTWqM2J9Nl2lqiFl6dQsEQFo6MhwMsNvUEmKX//L8y/Gi1RDx06RLt27VAoFBw5coT9+/ebjwMHDpSDiDcBPcZBrcaQkQCbp5RpqhA/d4bf0QjAKlc1KS1XJOdXE9KyteY6bddv5BWr4OzVWDN5U03U8dHAgaUQtx1U7mxqMgGQzKlaYFRUHvkOhBytnhybWQxFlalaqTCHjpisypQMY5CvshKDfKEUFtzWrVvLQ46bG7UH3P8JLLof9iyANo9Ag9Lvcz7XowlLd10gLbtgTyRPbyA9W4efZ9E8YkHVwmDxQyQh2XQyWCbZ28K0TF2wMw6AcLcM2DDReLHXRNKzwoHTVqWSfCy8qPYsuMIxcCZ83NXkaHNJz9+HM6WXBftoKi3IF8pQ8FLgYhrdCR2eNj5e/TJoSx+w6+ehZnSvpkXOp4g2hNUCS0NbIdneg7PnYLCkX9tQ8+PbT8yCnFQIjYDbX7Tqi2pScEYLLj/J3o6ToXAMnAmfQrFw5hCRSlyeglBwVYve08A7GK6egj9nl2mqp7o0ICLcn04NAmhY2xgndUUouGqBjIUFJ0molMUsUYtRIO3D/Wkc6EW0ci9BF9YanVn9PwOlqqCiryyb49cciYOztQcHRWPhLlcBBwMIBVe18PCHe/MV218fQ9J/pZ7KXa3ktxe78cvzXc0pOqagT0HVRas38MnmU+bnEraT7W0F+RZGoZBY+nQrPvdbYjzR9SWjBUdBLqpxiWpcVhoVnFElOONkgAJHg8kaTMxfooZWcvylUHBVjVb9ocV9YNDB6pfAUPYm26Yka2HBVX2W7rrAEosafzLWyfam5sol7cGZCNn9AeobiRDQCHqON5835bca07KMS1EfqzARZ5eo1rFwCdV1ibpjxw50uqKF7XQ6HTt27HCJUDc9984GjS/E74Vd/yvzdKYgT2HBVX2OJ1qn2cmybLUHpzUYSM/Rkp5jOwbOigv/wO5vjI/7f2p0ZuVjsgot06u8CjsZbFXzteNkKJzNkFAF0rSgFAquV69eNrvap6Wl0atXL5cIddPjGwq93zU+3jINrp8v03SB+a3ihAVX/ZCxXqJq9QWd7AM81Xhp7ARC6HKNKwCA9k8ZnVgWmHSmydPurlagViosMhkMZgVnuUJ2dA+uKgT5QikUnCzLNtt/Xb16FS8vL5cIJQA6DIEG3UB7A9a8Zu1acxKTBXclM89V0gkqCENhC05ncGj/jT8/gisnwSsI7plW5LJJaaZnm/bfjEtM0x5crlZPXn5WgrvFstQRL6reIJOUn4caVol5qOBEHNyDDz4IGL06Q4cORaMpCN7T6/UcOnRI5Ki6EoXCGBv3ZTc4EwOHlkPEoFJNZQq03HI8mUm/HWFsnxZWlSUEVQm52KdavaFg/83e8jTpKPyZn1F074fgEVBkiMlGMVlcJgvMcolqem21UsIUUllcHJxxPi1XMnPR5Qf5WuXAVgIOW3B+fn74+fkhyzI+Pj7m535+foSEhDBq1Ch++KFsZX8EhQhsBj3GGh+vHw9ZpStLbhlJvjj2PG/+fNAV0gkqABnrwN88vaH4LAaDPt85pTU6q1oNsDmvySpMyy7woAJWTgbTq7pZLEvtWXCWXlTT/ltQJQf5ghMW3HfffQdAw4YNeeONN1y6HJ03bx4ffvghiYmJRERE8NlnnxEZGWlzbM+ePdm+fXuR8/feey9//GGsbzV06FAWLVpkdT06Opr169e7TOYKo9sr8N+vkHQE1k+Ah752eorCv6LrjiSi1RvMXc4FVYfCOxEGWcYifx2tXubaDeNWQ6At6+jfryF+j9FJde+Hdgupmpao5iBfTVELzpTaZ/l3UtIeXHqOriBEpJL336AUe3CTJ092qXL76aefGDNmDJMnT2bfvn1EREQQHR1NcnKyzfErV64kISHBfBw5cgSlUskjjzxiNa5Pnz5W43788UeXyVyhKNVGD5ikMFYBPrXJ6Slq5zsZLNl+IsUV0glcTGEFJ8vWFpxWX7D57+lWyJpKvQAx+c6p3lPBN8zu6xQ2rEwWmKWTwWBeolooOHteVIs9uMupldvs2RKnFVxSUhJPPfUUYWFhqFQqlEql1eEsc+bMYeTIkQwbNoxWrVoxf/58PD097ZY+r1WrFiEhIeZj06ZNeHp6FlFwGo3GalxAQNF9iGpD3Y5w+wvGx2teg9zM4scXQqNSmj2pJkypPoKqTZElqq4gw8Cqiogsw5oxoM2C+l2hw9Bi5y3c58HHbMEVJNubXtYyk8Kuk8G0B5erJTG9anhQoRTJ9kOHDuXChQu88847hIaG2vSoOkpeXh579+5lwoQJ5nMKhYKoqChiY2OLubOAb7/9lscee6yIVblt2zaCgoIICAjgrrvuYvr06dSuXdvmHLm5ueTmFoRQpKen2xxXqfSaCMdWG3+lt0wzdudygroBnlZe1IwcLTq9AZVYplYpDIVMOFmWzS38wGjBZedbcB6WCu7wL3B6EyjdjBa/ovj/18LZEd7utvbgjK/r5sQSNTNHV2Vi4KAUCu6vv/7izz//pF27dmV+8StXrqDX6wkOti7VHRwcbG5JWBz//vsvR44c4dtvv7U636dPHx588EEaNWrEmTNnmDhxIn379iU2NtamlTljxgymTp1atjdT3rh5Gfs4/PCgMfj31och/DaHb6/tZW3Bzd54ki+3neH3l+6gcR1vFwsrKC2FG70UXaLKZOdnGJgVXNZVWD/O+LjHWKNzqgSKLFE1RZeosq0lqgNxcAnmZjPVcIkaHh5eZeqKffvtt7Rp06aIQ+Kxxx6jf//+tGnThoEDB7JmzRp2797Ntm3bbM4zYcIE0tLSzMfFixcrQPpS0PRuiBgMyEZPmc7xuDZ/G2WSsvL0fBpzysZoQWWRWyj/U5bBYOVkMJjrtHmY9uA2TIQbVyGoNXR9xaHXKVKV173QElVn6WSwWKLaqAcHBQpSZ5A5d9VYXLUqLFGdVnBz585l/PjxnDt3rswvHhgYiFKpJCkpyep8UlISISEhxd6blZXFsmXLGDFiRImv07hxYwIDAzl9+rTN6xqNBl9fX6ujyhL9PngGQsox2DnX4dsCPIs6GsB+z01B5VA4/1NGRl8oTMS0RHVXK+HUZji0zOiE6v8ZqGz/PxemcI050x6caY8tO68gTETlgAXn5aYyO2xNWyFh/tVQwQ0aNIht27bRpEkTfHx8qFWrltXhDG5ubnTs2JGYmBjzOYPBQExMDF26dCn23p9//pnc3FyefPLJEl/n0qVLXL16ldDQ0BLHVnk8axXsv+34EFIc67UQEe5v87xCkoo0CxFUHoXzP2UZqxWTVleg4DzJMTqdwNjAqF5Hh1+n8Bad2YuabxXm6gzmpbHlHpyt8ujG+SS83Qp2vBRSQZGHysTpPbi5c+e6VIAxY8YwZMgQOnXqRGRkJHPnziUrK4thw4YB8PTTT1O3bl1mzJhhdd+3337LwIEDizgOMjMzmTp1Kg899BAhISGcOXOGsWPH0rRpU6Kjo10qe6Vx60Nw6Cc4tdFYHHPYuhI3le9rE8qFq1lcTsthqUW1il/2XmLNoctseb1npdbOv1n473IaDWp72c0kySm0B2co5GTIs1iihu37CNIugH99YwtKJyjiZDCnahUosOw8oyxWS1Q7FhwYlaQp2T7Y171KOLCcVnBDhgxxqQCDBg0iJSWFSZMmkZiYSLt27Vi/fr3Z8XDhwgUUhb68J06c4K+//mLjxo1F5lMqlRw6dIhFixaRmppKWFgY99xzD9OmTbNKL6vWSBL0mwNf3A4X/4G9C+C2Z4q9RaGQGH1XM/48lWKl4MC4LNp97hoD2tUtT6lveo5eTqffp39R19+DnePvsjkmt7AFB+gttrxNXtR20ml8D+ZXCrnvY6MTygnsdcZyt1BgJmvS2slgPxTMx11FQprxcVXYf4NSKDiAM2fO8N1333HmzBk++eQTgoKCWLduHfXr16d169ZOzzd69GhGjx5t85otx0Dz5s3tOjo8PDzYsGGD0zJUO/zD4e7Jxn6qm6bALX3Br2QFZc9yOJmUYfO8wHXEHDPuNRcXg1jSEvVGnh7JoGWG29dIyND2MWga5bQsRfbg8peoKqUCtVJCq5e5kWe0xtSWqVp2An3B+m+rKoSIQCn24LZv306bNm3YtWsXK1euJDPTGHR68OBBJk+e7HIBBcVw2wioFwl5GfDH6w5VHPFxt63gTiU5FzwscJ48i5wrk7IrTNEik9ZL1PRsHaOUa2ipuIjsWdvodCoF9sJEoKB6iGmvz5E4OCgI9oWqESICpVBw48ePZ/r06WzatAk3twKPzV133cU///zjUuEEJaBQ5gd1quHkOmPOaglY/hFacipZKLjyxlJRjVi0x+aYwmXCDTJY3MbxI3t4WbXS+KTPTPCyHbxeEvbCRADc3Qo8qWBdUbi4JarlHNXWgjt8+DAPPPBAkfNBQUFcuVK6aheCMhDUErq/bny8bizcKFqM1BJ7S9TzV7NsVnAVuA69AxZ2buEwEbmgTLmEgceT56CRdOyQ2yG1ecTWFA5RuMqHlQWXvwy9ka/grJaoxVhwvhYKrqrswTmt4Pz9/UlISChyfv/+/dStKzapK4XuYyCwOWSlwMZ3ih1aJEE7H4MMZ1OyykM6QT4l6TdZlotYcJZxcI8rt9BZcZwsWcMs1bN2K4U4gqV+UyslK8XloS5miVrT9+Aee+wxxo0bR2JiIpIkYTAY2LlzJ2+88QZPP/10ecgoKAmVxhjkiQQHfoAz9ptzF5c7fCpZOBrKE4OheA2XpzfYrSYSzDXGq4wVcWbpHiPVrfhA+JKwDBPx1qis/i7MJZNMFpxFmIh7cUtUTcH2R5BPNVVw77//Pi1atCA8PJzMzExatWrFnXfeSdeuXXn77bfLQ0aBI9TvXBAqsuZVyLvh9BTC0VC+lKDfbHax+uGf86Tf0DJd/R0+Ujb7DE35Xt/bOtG+FFj+znkXcjyZlJhpiapy0IKznDOwCgT5QinCRNzc3Pj666955513OHLkCJmZmbRv355mzUpO8BWUM1GT4cRauH4Ots2wWYu/OESoSPlSuFJIYQrHwAEs2XWBvopd9HbbS56sZJx2FAYUBXmopcRyD87S8gILJ0O+PH4ealqE+Fg1pbFFtoX8ZZXPVZS6MH/9+vWpX7++K2URlBWNjzEA+MdBEPu5MeMhrF2Jt7mpFOTpDJwWntRypSQFZ8uC8yWTd9ULAfhSP4BTcj3AfsqUo1guUX00hS04aytNKUn88XJ3oPgtjpKW4JWBQwpuzJgxTJs2DS8vL8aMGVPs2Dlz5rhEMEEpad7HqNiOrIDVo2HkVmNVYAseuy2cZbsLKqY0rePN0YR0zuV7Usv65RHYRl+CArDVSX6iail1pDROG8KYpyvor1CeS9TC1pckFfW62mJYt0as2HeJhzrWK5NsrsQhBbd//360Wq35sT3KUvxS4EL6fABntkDiYaMld8drVpdnPNiGsX1a0GGasfx5kK+G+FQ1adlazqZk0SqsCldTqcaUvAdnreC6KP7jMdU2AMZpR5JHwQ9VWRVcYSeDJcU5EoojxM+d3W9FVSk94JCC27p1q83HgiqKdx1jhPuq52HbTGjZH2o3MV+WJIlaFgUwlZJEsyBv9py/zqnkDKHgXESuTo+bUmH+wpe0hLMsdqkhjxkqY67pYl1v9srNCfTWmJt3u3IPrnDz6MKtAQsn5hdHVVJuUAovqqCaEDEYGvcEXQ78/kqxQViSBM2CfYACT6pObzB3XBI4T1q2ljZTNjL464LsnpL34AosuNdUK2ioSOKyXAtNn6ksHh5Jw9oFjZ7LvgdX8Lhw+p67jSVqdcUhC87U9NkRVq5cWWphBC5Ekowlzr/oAuf+hP3fQwd7cYpGCw7g862n+e9yGieTMrmalcvut6LspncJ7LPhSCJ5OgP/nC3ILCkpk8HkZHgg5ArPpBpbYL6tHc6bTcJpGerL/O1nzGPLvgfn+BK1Gus3xyw4yybPvr6+xMTEsGdPQS7d3r17iYmJwc/Pr9wEFZSCWo0K6oRtfBsyEm0OU0jQJKigL8PWEynEp2aTozVw6FJaRUha47BURiZKWqLmaPUo0fNy5ieoMPC7/na2GDrg52H8gbEsW+ThVrbFl7I4BVdIeVaVtKvS4JAFZ2r6DDBu3DgeffRR5s+fb27gotfreeGFF6p2qe+blc7PGz2ql/cbc1UfXWy+dEfTQP46fYUnb29QxJNmwl6pc0HxnL1SNO3NESfDCOVaGunOkIY3U7XG2ou+thScK50Mhb2oFntwDWp7EtXSuilUdcLpn4EFCxbwxhtvWHWnUiqVjBkzxm4vU0ElolQZ07gkJRz9DY6tMV9aMPQ2tr3RkztvqWM3idq0b3T+ahbXsxxvciMoSnFLVINBxi39HGNUvwDwgeEprmBcEXnl74lZ/h+VdQ/Ocl+tcBycpSJ+6a5mVaIyb2lxWnKdTmezpd/x48cxGIoGKgqqACFtoFt+t6W1b0COcdnpplLQMNBYCdZeGRyDLHM1M5feH++w2jAXOI+9Iq2/HYin7dQNNN/9Du6SlmMeHViad4f5umm/zDIntKxe1OIsOMtE+YHtwsr0OpWN05kMw4YNY8SIEZw5c8bcrm/Xrl3MnDnT3EdBUAXpMdZowV07A5smw/1zrS7bs+D0BpkTSRnk6QxcLqYSraBkCv/+/7r/Eg+0r8f2Eyn01cXQIns/2bIbPwS+BteLbu27colqnaplrQZ6twrm7X4tuatFULW23qAUCm727NmEhITw0UcfmcsmhYaG8uabb/L666+7XECBi1B7GItjLuwHe7+DNo9Aw27my/aWPAZZJv66UbFVkXa4VRJZlkuMASu8RH3tp4M80L4e2vREJqmWADBH9zBX3MKAohV/3WyUNCotxYWJqJQKnuneuEzzVxWcVs8KhYKxY8cSHx9PamoqqampxMfHM3bsWJtd4wVViIZ3QIf8pkG/vwzaHPMlyyoRgyPDzY8NckEPgZLiuG5WEtKyuX1GDLM3FN/C0V6q1qPJn+EvZXFEbswCfV+ra5aWlqUFVzhWzVmsw0RqbhhQmezPKt8kWVCU3u+CdzBcPW3sq5qP5RK1e7M6NK5j3JvTG2Tz0rQK5lJXCd79/ShJ6bl8vtV2Y3ETWr2NPerjf9Bd+xc6WcFbulHoUSJZRJ5ZWlflZcHZ86DXBEr1zn755ReWL1/OhQsXyMuz9qzt27fPJYIJygkPf7h3Nix/CnbOhdYPQMitVlVbG9T2NG9Cy8KCK5FjCekOjdPprT8/H24g//EOEvCV/j4OGYpW57FScOWwBydJ4FmDiys4bcF9+umnDBs2jODgYPbv309kZCS1a9fm7Nmz9O3bt+QJBJVPq/7Q4j4w6GD1S2DQI0kSb93bkud7NqF1mJ/5F14We3AlkqtzLHqgsAU3VrUMKSOBOEMwn+geNH++llt5vhZZJNaBvmVTSrW83HBXK2gc6IXCgUoh1RWnLbgvvviCr776isGDB7Nw4ULGjh1L48aNmTRpEteuFd/wRFCFuHc2xP0Jl/fBrv9BlxcYeWfBxrJpmaQzyFxONe7VCQvONiqlYwoiK68g17STdJynVJsBmKh7hlxsB1RbWnBqlUWYSBmtLh93NZte61Ek0b6m4bQFd+HCBbp27QoYmyxnZBirwD711FP8+OOPpRJi3rx5NGzYEHd3dzp37sy///5rd+zChQuRJMnqcHe3TiWRZZlJkyYRGhqKh4cHUVFRnDp1qlSy1Vh8Q+Ged42Pt0yD6+etLpusiJSMXHM/T51B5plFezh62bEl2c2CSuHY1ygrv3iBhjxmqo2VQuIbPUKswbpZulUQroUFZ7lEdUXNvvBanlZVZWoiTiu4kJAQs6VWv359cy/UuLg4u4GMxfHTTz8xZswYJk+ezL59+4iIiCA6Oprk5GS79/j6+pKQkGA+zp+3/nLOmjWLTz/9lPnz57Nr1y68vLyIjo4mJyfHzow3Ke2fhgZ3gPaGsY+Dxf+faQ/u4nXr3g6bjyUxcrHtnp43KyoHl3imTvEvqH6jqeIyybI/2xu8VOw9lhacpQVdVUqCV3WcVnB33XUXq1evBoxBv6+99hq9e/dm0KBBNvullsScOXMYOXIkw4YNo1WrVsyfPx9PT89i074kSSIkJMR8BAcX5MrJsszcuXN5++23GTBgAG3btmXx4sVcvnyZVatWOS1fjUahgPs/AaXGWCDz0E9WlwAuXisa3JuULn4oLLEM5SjuRz4rV09z6QIvKI3fn0naoZzNLBqiISFRO9+y6tcm1HxeZ+HGLlxWXGAbpxfgX331lTkl68UXX6R27dr8/fff9O/fn2effdapufLy8ti7dy8TJkwwn1MoFERFRREbG2v3vszMTBo0aIDBYKBDhw68//77tG5tNPPj4uJITEwkKirKPN7Pz4/OnTsTGxvLY489VmS+3NxccnNzzc/T02+iJVhgU+g5DmLehfXjocnd4F3HrgUH1bu6RHlgufmfpzfYLBCpN8jkarXMdPsGtaRng74T6w23EW3j8wXY+NqdnE7OpHPjgs71ltVIqnuGQUXh1Kek0+mYPn06iYkFZXcee+wxPv30U1566SXc3Jxbz1+5cgW9Xm9lgQEEBwdbvYYlzZs3Z8GCBfz222/88MMPGAwGunbtyqVLlwDM9zkz54wZM6xKQoWHh9scV2Pp+jIE3wrZ12GD8cfG9BU1eVAtCfG1VnDnrmQxb+tpMnK05S1plcTSghv3yyHzXhsUdHu/kadjiHID7RWnSZc9eEc7DJC4ZOPzRYLa3hor5QbWFpzAMZxScCqVilmzZqHTVV6l1y5duvD000/Trl07evTowcqVK6lTpw7/+9//Sj3nhAkTSEtLMx8XL14s+aaahFJtTOOSFHD4Zzi50RzpHm8j/9TU81KWZY5eTueej3fw4YYTTP39aIWKXVWw3INbdeAy+y+kmp+bgnNzUuJ4Q7UcgBm6x0kmAMC2grNDSU1rBEVx2s69++672b59u0tePDAwEKVSSVKSdd5dUlISISGOde5Wq9W0b9+e06eNUeSm+5yZU6PRmLMybtrsjLod4fYXjI/XvIanbL9xtEIBszecoNGEtdz76Z9mL+s/Z69WhKSVzrkrWWTkaNl0NIk9564VCZ+5VrislCzjtelNvKRcdhlasEzfy3wpLbuo1WvPZSEUnPM4vQfXt29fxo8fz+HDh+nYsSNeXl5W1/v37+/wXG5ubnTs2JGYmBgGDhwIgMFgICYmhtGjRzs0h16v5/Dhw9x7770ANGrUiJCQEGJiYmjXrh1g3FPbtWsXzz//vMOy3ZT0mgjHfofU8zzh8T1/M8jmMFnGZlpSuo0va03jdHImUXOsf+Ajwv2tnt8o3MD58M94XthGrqzmQ7cXkPNKt3/WPMSnVPfdzDit4F54wfgrb6v/qSRJ6PVFezsWx5gxYxgyZAidOnUiMjKSuXPnkpWVZS699PTTT1O3bl1mzJgBwLvvvsvtt99O06ZNSU1N5cMPP+T8+fM888wzZhleffVVpk+fTrNmzWjUqBHvvPMOYWFhZiUqsIObl7GM0vcP0Dd7Ne2lDuyXmxUZtu6I7b3M9Jya36Rm+Z6i2xf6QnWQsvMKPgdD5hXkdRORgE90DxDvXg8o3gttryrJ/W3DuJ6VR8cGtZyW+2bFaQXn6qKWgwYNIiUlhUmTJpGYmEi7du1Yv3692Ulw4cIFFBaBlNevX2fkyJEkJiYSEBBAx44d+fvvv2nVqpV5zNixY8nKymLUqFGkpqZyxx13sH79+iIBwQIbNLkLIh5HcXApM9Vfc1/e+7i5aayi8G9mvtpxtsg5rc566XjD4rN6W/0DUvY1jhnC+Up/H7q0kkNs7MXVKRQSQ7s1clLimxtJLk10bg0nPT0dPz8/0tLSbs79uBvXSJvdDj9DGnO0D7OxzhCOJ2Y4dOu5mf3KWbjKpeH4P4qca1Dbk/NXC/Ysn72zMf/bcZYeioMscvsAvSzxYN5UDspNHXqN6QNv5cnbG7hM5pqIo99Rhy247OxsYmJiuO+++wCj59EydkypVDJt2jRhJdUEPGux2O8FXro+g9GqX7nu1ZfjeFS2VFWWwh3ps7V6PMnhPfW3AHyn7+OwcgO4u2WQS+W7mXF4t3PRokVWoRiff/45f//9N/v372f//v388MMPfPnll+UipKDi+cezBzH69rhJekamzkVC9Nuwh6mfqYkbeXreUC2nnnSFi4Y6fKR7xOG5bq3rS6if+DFxFQ4ruCVLljBq1Circ0uXLmXr1q1s3bqVDz/8kOXLl7tcQEHloFAoeEc7jEzZnfpZh3lCGVPZIlVZCltwaadiGarcAMBE3QiyKX5VY+p7CnB3i+rboq8q4rCCO336NG3atDE/d3d3t9r8j4yM5OjRmzPQsyYiSRKXCWSWzhgqMk61jFBujjg3Z7GsB6dGx+s5n6OQZFbou/OnoW2J91tW9OjdSig4V+KwgktNTbXac0tJSaFhw4bm5waDweq6oHpjcuR9r+9NVlBHfKRspqkXAMInVRzPKn+nheIiV2RfpmmfdOiedhZxdK3DbkKnVjnisIKrV68eR44csXv90KFD1KtXzyVCCSofU/VZGQXafp+QJyuJUu6nn2JXsfddz8rj05hTXLKTRF6TaSLF85LqVwDe1T5NKo4F5rYI8WHdK935Z8LdJXbmEjiHwwru3nvvZdKkSTZrqmVnZzN16lT69avZIQI3EwkW8Vp+9W9lnm4gAFPUC/HHfsjIp1tOMWfTSaI/3lHeIlYpJAzMUH+DRtKxRd+O1YYuDt9b21tDy1BfUaWlHHBYwU2cOJFr167RvHlzPvzwQ3777Td+++03Zs2aRfPmzbl+/ToTJ04sT1kFFUiuhWdQkiS+1PfnpKEudaR03srv4WmLMylZgLE8d0aOlvSbpMLI48otRCpOkCVreFs7HPsZpUUJ9K7ZVXUrE4fj4IKDg/n77795/vnnGT9+vLmwnyRJ9O7dmy+++KJIiSJBzSEPNeO1I/nFbSqPqHawytCNnYY2RcZZJty3mbIRgBPT+6BR1dwKtMFcY7zKWK5/lu4xLhPo1P2m6iwC1+NUqlajRo1Yv349165dM1fvaNq0KbVqidy4moatNgP75FtYrO/NUNVGZqi+4Z68WeRg/eXMs9FhKiUjl3oBnuUlaiUjM139HT5SNnsNzfhe39vpGWoLC67cKFVZg1q1ahEZGUlkZKRQbjUUyc4S60PdIC7LtaivSOE11S8OzVWTy/zcq9hFb+Ve8mQl47UjMdj4Sk0beKv5sS0fQk1v/FKZiLrHApvYc+Zl4ZG/xwTPKNdyq1Q0+bwwE1Ye5sDFVBdKVzXwI5Op6oUAfKEfwCnZdhTBY7cVVIhWFvpgAzzVNXr5XtkIBSewSXFb5FsMHVit74JSkvlA/TUqii+T9PeZqwyct9O1ApYjn285xcs/7rfqgWCLiaql1JHSOWWoyxe6AXbHWSo1yybLKoXEpPtb2bpF4CKEghPYpKR4rKnap7kue9NacZ5nlGsdmjPmWFLJg6oAszeeZPXBy8VWKO6qOMIg1TYMssQ47UjyKNody4SlUrNUdouGR/JAexE7Wp4IBSewSUlBDlfxM0fqv6paQUMpocQ5x/5yyAWSVRy5NhwmAO7k8r7KWCnke30U++RbSpxrYLswAF7s1YT1r3bns8Ht6dbUOW+rwHmEghPYxoEwrpWG7uzQt8Fd0jJD9S0lpXEV7l1Q1ZBlmZ8tK/ba+QxeVa2koSKJy3ItPtTZLutemA8fiWDNS3fwQs+mtAjx5f6IMBdILCgJoeAENrHV27MoEhN1I7gha+iiPMog5bbiR1fxNKTfDyXwZjFW5j9nr9JaiuMZpbHo5dva4WTiWPiLWqng1rp+VstVQfnjdMlywc2Bo1/DS3IQH+ke5h31Et5SLWGLvh0p+S3xSjtnZTFr/XGr5yYl/+KSfSSkZaPCYHSqSAZW67uwxdDBPNZNqTB3F/N1V90U/SmqA8KCE9jEMQvOyEJ9Hw4aGuMr3WCqepHdcVXZgLuWlVekR6lJ3D8OJ7DvQiqdEpZyq+Ic12VvpmqfthqrURd8laJaBvPlEx0QVD5CwQls4owy0mMMctXJCu5V/ku0Yre9WV0iW3nQYdomm+dNQcoNpEReVvwMwHTtk1zFz2qcu9oilk0CfRXfb7xZEApO4BKOyQ34n97Yr+Nd9Xf4klVkTFW24GwhSaAzGACZGapvcJe0/Km/lRWG7kXGalQFXyUJiQ71bS/TBRWLUHACm5TGIfCp7kHOGEIJllIZp1pWdE5XCFaBSEjM23KaR5Tb6ao8SrbsxkTdCGy9E0sLTpIgzN+DP8f24uCkeypQYkFhhIIT2KQ0yigXNyZqjQ24n1DFECkdc61QFcx/l9P4ccse3lb9AMBHuke4KNuumONusQfn6WZUduG1PPHztB8ALCh/hIIT2KS0y8ldckuW6u4CMBaAJK/Mc1YWM9YdZ7J6EX7SDQ4ZGvGdvo/dscE+BcUqX767WUWIJ3CAKqHg5s2bR8OGDXF3d6dz5878+++/dsd+/fXXdO/enYCAAAICAoiKiioyfujQoUiSZHX06WP/j1NQlLIoo5m6wSTJ/jRRJJhLeIP9CiVVld6KPdyn3IVOVjBeOxI99pPih9/RiAfa12VC3xaivlsVotIV3E8//cSYMWOYPHky+/btIyIigujoaJKTk22O37ZtG4MHD2br1q3ExsYSHh7OPffcQ3x8vNW4Pn36kJCQYD5+/PHHing7NQZnwkQKk44Xk7TDAHhWuYaW0nnAWmlm5eo4Ep9mLpxa1fDhBtPU3wHwP/19HJUbFjs+1M+djwe149keTSpAOoGjVLqCmzNnDiNHjmTYsGG0atWK+fPn4+npyYIFC2yOX7JkCS+88ALt2rWjRYsWfPPNNxgMBmJirPt2ajQaQkJCzEdAgPBqOUNx6k2lkOhfQqrRBsNtrNPfhlrSM1P9NQoMyLKxmY3eINP/87+477O/2PBf1UzAH6f6kRDpOmcNIXyqe7DE8UG+op9CVaRSFVxeXh579+4lKirKfE6hUBAVFUVsbKxDc9y4cQOtVluk8Oa2bdsICgqiefPmPP/881y9ar8yRG5uLunp6VbHTU8xFtzf4+9i1sMl9/ucrB1KuuxJhOIsw5TrSUzPodlb6+g1e5u5d8Pqg/ElzFLx3CYd50mV8QdzgnYkuZRckNJbI5KCqiKVquCuXLmCXq8v0sshODiYxMREh+YYN24cYWFhVkqyT58+LF68mJiYGD744AO2b99O37590ev1NueYMWMGfn5+5iM8PNzmuJsJe+rNy01JkK+7dWCrHZIJ4H3d4wC8rvqZepJx2+HCtYKWgmsPJ1Z6xd+k9IIOYhrymKn+GoClul7skltWllgCF1DpS9SyMHPmTJYtW8avv/6Ku3vBEuGxxx6jf//+tGnThoEDB7JmzRp2797Ntm3bbM4zYcIE0tLSzMfFixdtjruZsGfA2VNF9QI8bJ5fpu9FrL4VnpKpxFDRGX7aXbmf99urCvr9vqhaRRNFAsmyPzPzlbM9GtT2ZHBkfb4bdlt5iygoJZWq4AIDA1EqlSQlWe/DJCUlERISUuy9s2fPZubMmWzcuJG2bYtfLjVu3JjAwEBzo5zCaDQafH19rY6bHXtOBns+gU4N7O1xSkzQjSBXVnOn8jAPKP4qMuLHfy+UUkrXsOW40bJsLl3geeXvALyjHUo6XsXe98tzXZnxYBt6NQ8qdxkFpaNSFZybmxsdO3a0chCYHAZduthvnDtr1iymTZvG+vXr6dSpU4mvc+nSJa5evUpoaKhL5L4ZsLdEle3YcE/e3oA+rW3/KJ2TQ5mrewiAServqU2a1fXD8WmsOXS51LI6y4x1x+gzdwc38owVP5SShCK/Uoha0rNefxsbDJElzlNbNIup8lT6EnXMmDF8/fXXLFq0iGPHjvH888+TlZXFsGHGMIOnn36aCRMmmMd/8MEHvPPOOyxYsICGDRuSmJhIYmIimZmZAGRmZvLmm2/yzz//cO7cOWJiYhgwYABNmzYlOjq6Ut5jdcTuEtVCv/0wojN1/T1YPDySTg1rMf+pjmwec6fN+77W38tRQwMCpEwmqb8vcn3hznMukNox/rf9LMcTM/hl7yXA2CJxiHID7RRnSJc9maQdWuz9j90Wzp9je4nabtWASnf9DBo0iJSUFCZNmkRiYiLt2rVj/fr1ZsfDhQsXUFg06fzyyy/Jy8vj4Ycftppn8uTJTJkyBaVSyaFDh1i0aBGpqamEhYVxzz33MG3aNDQaEYDpKI4E5d7RLJCd4++yOmevQ5QOFeO0I1nl9g4DlH+zSt+NrYb25utlibtzFJ3ewP92nLV4btTWgbok3nRbDsD7usdJtlPPDmDja3dyS7BP+QoqcBmVruAARo8ezejRo21eK+wYOHfuXLFzeXh4sGHDBhdJdhPjpJPBRHgtT4Z0aUB8ajabj1kHax+WG/Ot/l5Gqf5gunoB9+TOIgujc8JWo2lXs/TfC3y44YT5+b9x17i9US3eU32Lp5TLP4aW/KTvaff+NnX9hHKrZlT6ElVQNbG3+nIk82DqgFsZ07u5zWsf6x7igqEOdaWrvKFabvF65W/BnUjMsHq+/r9Efv/hY3ooD5Erq5mgfQa5mK/Ery90LW8RBS5GKDiBTewtUR3NrDJV1ChMNu5M0BkrjgxRbqSDdBIw9k79oFDJcFdTWIfWIp2RN4wxb5/oHiROtu+E+nZIJ1RK8XWpboj/MYFNnI2DK0zDQPshFjsNbfhZdycKSWam+mvU+Y2jv9x2hhyt7WBsVxB7xjqb5R3199SSMjlmqM9X+n7F3nt3S9tlkgRVG6HgBDax70V1POvg6Lv2vdbv6Z4gRfblFkU8zytXOyue0xgMsjk9DKCn4gAPKHeiz2/crLOxHd06zJeIcH8WikDeaotQcAKb2F2iOjGHp5uKM+/fS9Mg7yLXUvFhqnYIAKNVv9JUMoZslNdW3KP/K8ht9iSH6WpjMYcF+r4ckm1XABkcWZ/fXuxGTxHIW20RCk5gE0fi4BxBqZBY8bztzfk1htvZrG+Pm6TnA/XXSNjuJO8K9py/bn78hmo59aQrXDDUYY7uYbv3WPZZEFRPxP+goNzx81Az6yFb6XQS72iHkym701FxiieVm82J97Isl0utuPbSKYYqjWFEE3XPkI3tMkf1a3mK7vM1AKHgBDZxddjGo7eF06B20S7wCdTmA91jAIxTLSP3ygVkWebh+bE8PD/WJUruTIoxy0WNzlibTpL5RX8nfxna2Bx/6r2+7Bjby6GKKYKqjVBwApuUx17Ytjd62jz/gz6KvYZmeEs57P1yOOsOJ7D3/HX2nr9OckZumV/37o+2A/CccjXNFZe4IvsyXfuEzbFv92uJWoSD1BjE/6TAJuUReCtJEudmFg3HkFEwTjuSPFlJlHI/a5d9Yb6Wp3PNvlwTKZ7RqlUATNU+TSq2MxKqaAV1QSkRCk5gk/LMKzj1Xl8Cva0rcZyW6zFPNxCAKepF+GPMOtDqy67gJAzMVH+NRtIRo2/P7wb7lWpseXwF1Reh4AQVjlqpYM/bvYuc/0I/gBOGegRK6bylWgLAsYSMIuOcQZZlnlDGcJviJJmyO+9oh1FYfR+cfA+rXuzG+w+0oWfzOmV6PUHVQig4gU1K09neWQovV7WomKB9BoMs8YhqB90Uh3lx6b5S14p7cek++r+/nHGqZQDM0g3iMoFFxvl5qGkX7s/jnetXyPsWVBxCwQlsUlHf88JNkvfJt7BIfw8AM1Tf4EEOo5fud3peg0Hmj0OXeTlnPj5SNnsNzfhBX9Rq/H5EyYUtBdUXoeAENqkoO8bLRlL+bN2jxMu1qa9I4TXVCqfnzNXpaTxxLf0Uu+it3EeerGS8diQGiz/38X1bsO6V7nRvJpakNRmh4AQ2qSgLztbrZOHB29rhAIxQrqWNdJYFf8U5NJ/eIBP98Q78yGSKeiEA83QDOSXXsxoX3TqElqGi90ZNRyg4gU0qoj5bYb4bWpDUvtXQnt/0XVFKMh+ov+b9NYc4nlhyv9rb3tvMuas3eEu1hDpSOicNdflS399qzIwH29ComGongpqDUHACm1SYBWexGO7VIoihXRuan7+rfYrrsjetFOcZqVxLn7l/kmzRw7QwsixzLSuPborDPKrajkGWGK8dSR5qAO5oGsjZ9+9lcGT9cns/gqqFUHACmzjSk8Elr1PoZab0b21uXHMVP6ZpnwTgVdUKGkoJ7D53vfAUZj6JOYU7pv6rsFjfm33yLebrPzzTWTSKuckQCk5gm0rUA02DfBjZvREAKw3d2aFvg0bSMlP9DaOX7kFnJ/h37uZTvKpaQQNFMpflWnyoG2S+ZiuDQlDzEQpOYJPKtnPe6tcq/5HERN0Ibsgablcc41Hldt74+WCR8Ufi02gtxTFS+QcAb2uHmxva/PHyHRUltqCKIRScwCYV5WQoLrDWZHVdkoP4SPcIAG+plvD3gf+KjD2VcI1Z6q9QSjKr9V3YYugAwNn376V1mF85SC6oDggFJ7BJxTkZHOM7fR8OGBrjK91gqnoh204UtCTM0xk48etMWivOc132Zqr2aQAOTrpH7Lnd5AgFJ7BJRakFRxWpAQUTtCPRykr6Knfz46J55lpx97yzgFfzA4KnaZ/kKn6cnN4XP091eYktqCZUCQU3b948GjZsiLu7O507d+bff/8tdvzPP/9MixYtcHd3p02bNqxdu9bquizLTJo0idDQUDw8PIiKiuLUqVPl+RZqHFUtJ/PN6OYckxvwP/19ALyrXsiw+ZtJSc9hhupb3CUtO/RtWGnozp9je+Emyo0LqAIK7qeffmLMmDFMnjyZffv2ERERQXR0NMnJyTbH//333wwePJgRI0awf/9+Bg4cyMCBAzly5Ih5zKxZs/j000+ZP38+u3btwsvLi+joaHJy7MdQCaypMAvOwXEKSeLE9D58pnuAM4ZQgqVU7on/gg8/eJsuyqPckDVM1I1g/at3El6raOVgwc1JpSu4OXPmMHLkSIYNG0arVq2YP38+np6eLFiwwOb4Tz75hD59+vDmm2/SsmVLpk2bRocOHfj8888Bo/U2d+5c3n77bQYMGEDbtm1ZvHgxly9fZtWqVRX4zqo3FWXBOfo6kgQalZJc3JigNTaOfly1hVlqY+Pmj3QPM++FB2gRItKvBAVUqoLLy8tj7969REVFmc8pFAqioqKIjY21eU9sbKzVeIDo6Gjz+Li4OBITE63G+Pn50blzZ7tz5ubmkp6ebnXc7ESEW3seA/L3s/w8XLuv5agetRz2r9ySJbq7zc8PGhrT6N4xRIT7u1Q2QfWnUhXclStX0Ov1BAdbdw0PDg4mMTHR5j2JiYnFjjf968ycM2bMwM/Pz3yEh4eX6v3UJB6PrM/U/q1Z/2p3AH56tgt9Woew/Fn71XBLQ782oUgS9LjFdlWPO5oa67f1b2fd4WqmbjDxcm1yZTW7bp3Ck92aulQuQc2gaDvvm5AJEyYwZswY8/P09PSbXsmplAqGWOSF3hLsw/ynOrr8dWp7azg+rQ9udhq9LB4eSY5Oj6eb8U/15PS+/HP2Kt2bBbJlX2tuCVAyqnFzl8slqBlUqoILDAxEqVSSlJRkdT4pKYmQkBCb94SEhBQ73vRvUlISoaGhVmPatWtnc06NRoNGoynt2xCUEY3Kfns+hUIyKzcAN5WCO/Otvbs7trJ3m0AAVPIS1c3NjY4dOxITE2M+ZzAYiImJoUsX20uhLl26WI0H2LRpk3l8o0aNCAkJsRqTnp7Orl277M4pEAhqKHIls2zZMlmj0cgLFy6Ujx49Ko8aNUr29/eXExMTZVmW5aeeekoeP368efzOnTtllUolz549Wz527Jg8efJkWa1Wy4cPHzaPmTlzpuzv7y//9ttv8qFDh+QBAwbIjRo1krOzsx2SKS0tTQbktLQ0175ZgUDgEhz9jlb6HtygQYNISUlh0qRJJCYm0q5dO9avX292Ely4cAGFosDQ7Nq1K0uXLuXtt99m4sSJNGvWjFWrVnHrrbeax4wdO5asrCxGjRpFamoqd9xxB+vXr8fd3b3C359AIKg8JFkWrW4Lk56ejp+fH2lpafj6irgqgaCq4eh3tNIDfQUCgaC8EApOIBDUWISCEwgENZZKdzJURUzbkiJlSyCompi+myW5EISCs0FGRgbATZ/NIBBUdTIyMvDzs1+xWXhRbWAwGLh8+TI+Pj5Vri6aCVM62cWLF29aT6/4DG7ez0CWZTIyMggLC7MKIyuMsOBsoFAoqFevXskDqwC+vr431R+2LcRncHN+BsVZbiaEk0EgENRYhIITCAQ1FqHgqikajYbJkyff1FVQxGcgPoOSEE4GgUBQYxEWnEAgqLEIBScQCGosQsEJBIIai1BwAoGgxiIUXDXh2rVrPPHEE/j6+uLv78+IESPIzMwsdvxLL71E8+bN8fDwoH79+rz88sukpaVVoNRlZ968eTRs2BB3d3c6d+7Mv//+W+z4n3/+mRYtWuDu7k6bNm1Yu3ZtBUlafjjzGXz99dd0796dgIAAAgICiIqKKvEzq9GUb2Fhgavo06ePHBERIf/zzz/yn3/+KTdt2lQePHiw3fGHDx+WH3zwQXn16tXy6dOn5ZiYGLlZs2byQw89VIFSl41ly5bJbm5u8oIFC+T//vtPHjlypOzv7y8nJSXZHL9z505ZqVTKs2bNko8ePSq//fbbRcrZVzec/Qwef/xxed68efL+/fvlY8eOyUOHDpX9/PzkS5cuVbDkVQOh4KoBR48elQF59+7d5nPr1q2TJUmS4+PjHZ5n+fLlspubm6zVastDTJcTGRkpv/jii+bner1eDgsLk2fMmGFz/KOPPir369fP6lznzp3lZ599tlzlLE+c/QwKo9PpZB8fH3nRokXlJWKVRixRqwGxsbH4+/vTqVMn87moqCgUCgW7du1yeB5TeWeVquqnIOfl5bF3716ioqLM5xQKBVFRUcTGxtq8JzY21mo8QHR0tN3xVZ3SfAaFuXHjBlqtllq1apWXmFUaoeCqAYmJiQQFBVmdU6lU1KpVi8TERIfmuHLlCtOmTWPUqFHlIaLLuXLlCnq93tx8yERwcLDd95yYmOjU+KpOaT6DwowbN46wsLAiiv9mQSi4SmT8+PFIklTscfz48TK/Tnp6Ov369aNVq1ZMmTKl7IILqgUzZ85k2bJl/PrrrzdtR7mqv1apwbz++usMHTq02DGNGzcmJCSE5ORkq/M6nY5r164REhJS7P0ZGRn06dMHHx8ffv31V9RqdVnFrhACAwNRKpUkJSVZnU9KSrL7nkNCQpwaX9UpzWdgYvbs2cycOZPNmzfTtm3b8hSzalPZm4CCkjE5Gfbs2WM+t2HDhhKdDGlpafLtt98u9+jRQ87KyqoIUV1KZGSkPHr0aPNzvV4v161bt1gnw3333Wd1rkuXLtXeyeDMZyDLsvzBBx/Ivr6+cmxsbEWIWKURCq6a0KdPH7l9+/byrl275L/++ktu1qyZVZjIpUuX5ObNm8u7du2SZdmo3Dp37iy3adNGPn36tJyQkGA+dDpdZb0Np1i2bJms0WjkhQsXykePHpVHjRol+/v7y4mJibIsy/JTTz0ljx8/3jx+586dskqlkmfPni0fO3ZMnjx5co0IE3HmM5g5c6bs5uYm//LLL1b/5xkZGZX1FioVoeCqCVevXpUHDx4se3t7y76+vvKwYcOs/mjj4uJkQN66dassy7K8detWGbB5xMXFVc6bKAWfffaZXL9+fdnNzU2OjIyU//nnH/O1Hj16yEOGDLEav3z5cvmWW26R3dzc5NatW8t//PFHBUvsepz5DBo0aGDz/3zy5MkVL3gVQJRLEggENRbhRRUIBDUWoeAEAkGNRSg4gUBQYxEKTiAQ1FiEghMIBDUWoeAEAkGNRSg4gUBQYxEKTiAQ1FiEghNUG4YOHWqz4kqfPn0qWzRBFUVUExFUK/r06cN3331ndc5eV3etVlukekpeXh5ubm5Ov25p7xNULsKCE1QrNBoNISEhVkdAQAAAkiTx5Zdf0r9/f7y8vHjvvfeYMmUK7dq145tvvqFRo0bmumgXLlxgwIABeHt74+vry6OPPmpVlsjefYLqhVBwghrFlClTeOCBBzh8+DDDhw8H4PTp06xYsYKVK1dy4MABDAYDAwYM4Nq1a2zfvp1NmzZx9uxZBg0aZDVX4fsE1Q+xRBVUK9asWYO3t7fVuYkTJzJx4kQAHn/8cYYNG2Z1PS8vj8WLF1OnTh0ANm3axOHDh4mLiyM8PByAxYsX07p1a3bv3s1tt91m8z5B9UMoOEG1olevXnz55ZdW5ywbqlg25jHRoEEDKyV17NgxwsPDzcoNoFWrVvj7+3Ps2DGzgit8n6D6IRScoFrh5eVF06ZNi73uyDlHX0tQvRF7cIKbjpYtW3Lx4kUuXrxoPnf06FFSU1Np1apVJUomcDXCghNUK3Jzc4u0zFOpVAQGBjo8R1RUFG3atOGJJ55g7ty56HQ6XnjhBXr06GFziSuovggLTlCtWL9+PaGhoVbHHXfc4dQckiTx22+/ERAQwJ133klUVBSNGzfmp59+KiepBZWFKFkuEAhqLMKCEwgENRah4AQCQY1FKDiBQFBjEQpOIBDUWISCEwgENRah4AQCQY1FKDiBQFBjEQpOIBDUWISCEwgENRah4AQCQY1FKDiBQFBjEQpOIBDUWP4PyeZpmgC58pMAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019523, + "end_time": "2024-03-22T23:20:15.121881", + "exception": false, + "start_time": "2024-03-22T23:20:15.102358", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4546.574128, + "end_time": "2024-03-22T23:20:17.861529", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/lct_gan/1/mlu-eval.ipynb", + "output_path": "eval/treatment/lct_gan/1/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/lct_gan/1", + "path_prefix": "../../../../", + "random_seed": 1, + "single_model": "lct_gan" + }, + "start_time": "2024-03-22T22:04:31.287401", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/lct_gan/model.pt b/treatment/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ea2430dcb202442326bcf9ef955e30a8d6489a4 --- /dev/null +++ b/treatment/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ecae4001da9abe16fe2126bd7970b74a51e8b787afd26d4400d57c64385c710 +size 74778241 diff --git a/treatment/lct_gan/params.json b/treatment/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..d502505c98db57c00f21d785a1b528b273ca7e09 --- /dev/null +++ b/treatment/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "NONE", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.1, "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "fixed_role_model": "lct_gan", "mse_mag": false, "mse_mag_target": 0.2, "mse_mag_multiply": false, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/realtabformer/eval.csv b/treatment/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..76fd486adb90bd424420e17856c29b9e662d411b --- /dev/null +++ b/treatment/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,1.3982927692787988e-05,0.396190471649171,0.004820992158222723,6.060643196105957,0.4483191967010498,7.328680992126465,1.377678632736206,6.148087413748726e-05,27.904374599456787,0.04260999336838722,5612931.0,0.06943336874246597,0.23635642230510712,0.0008501424454152584,33.965017795562744 diff --git a/treatment/realtabformer/history.csv b/treatment/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..8221c44bc9aef36aec769349ec670b07e7e5f8be --- /dev/null +++ b/treatment/realtabformer/history.csv @@ -0,0 +1,6 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.07077005374701326,7.7690568529201345,0.028255754237470072,0.0,0.0,0.0,0.0,0.0,0.07077005374701326,900,450,490.3524549007416,1.0896721220016479,0.5448360610008239,0.15626739564745068,0.007018332518638089,5.231984866959922,0.00025027377600039385,0.0,0.0,0.0,0.0,0.0,0.007018332518638089,450,225,197.8156943321228,0.8791808636983236,0.4395904318491618,0.09444081523999052 +1,0.009379093581180642,2.6030105899689175,0.0004631821013883831,0.0,0.0,0.0,0.0,0.0,0.009379093581180642,900,450,485.68966460227966,1.0793103657828438,0.5396551828914219,0.2023497386896573,0.00616615310408106,7.329303962527855,0.00015382666007410638,0.0,0.0,0.0,0.0,0.0,0.00616615310408106,450,225,198.96484756469727,0.8842882113986545,0.44214410569932727,0.07584491885941583 +2,0.007078851991594096,2.0421946790576384,0.00021883253530301686,0.0,0.0,0.0,0.0,0.0,0.007078851991594096,900,450,487.61613154411316,1.0835914034313625,0.5417957017156813,0.1953078424528414,0.0064364255767268555,6.630073926702688,0.00046532210426971943,0.0,0.0,0.0,0.0,0.0,0.0064364255767268555,450,225,197.56161880493164,0.8780516391330295,0.43902581956651476,0.0723328961941095 +3,0.005932264231048244,2.1949213939262298,0.0001685581630674178,0.0,0.0,0.0,0.0,0.0,0.005932264231048244,900,450,485.7743980884552,1.0794986624187892,0.5397493312093946,0.1975122331425367,0.005354185296421948,6.1442645163203204,0.0003165367537867994,0.0,0.0,0.0,0.0,0.0,0.005354185296421948,450,225,197.5341498851776,0.8779295550452338,0.4389647775226169,0.07811297937988507 +4,0.005074872308783256,2.5619912489473453,9.861685678809986e-05,0.0,0.0,0.0,0.0,0.0,0.005074872308783256,900,450,481.7331359386444,1.0705180798636542,0.5352590399318271,0.18441032754652825,0.006421878464142566,6.356629352012609,0.0003719029411907014,0.0,0.0,0.0,0.0,0.0,0.006421878464142566,450,225,195.86327409744263,0.8705034404330784,0.4352517202165392,0.0808127932534593 diff --git a/treatment/realtabformer/mlu-eval.ipynb b/treatment/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..203f69263bdd970100251d077639e04e5c9711dd --- /dev/null +++ b/treatment/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2326 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:03.572011Z", + "iopub.status.busy": "2024-03-23T10:38:03.571077Z", + "iopub.status.idle": "2024-03-23T10:38:03.626504Z", + "shell.execute_reply": "2024-03-23T10:38:03.625457Z" + }, + "papermill": { + "duration": 0.078686, + "end_time": "2024-03-23T10:38:03.630542", + "exception": false, + "start_time": "2024-03-23T10:38:03.551856", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:03.672739Z", + "iopub.status.busy": "2024-03-23T10:38:03.671524Z", + "iopub.status.idle": "2024-03-23T10:38:03.681666Z", + "shell.execute_reply": "2024-03-23T10:38:03.680561Z" + }, + "papermill": { + "duration": 0.033236, + "end_time": "2024-03-23T10:38:03.684738", + "exception": false, + "start_time": "2024-03-23T10:38:03.651502", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:03.735841Z", + "iopub.status.busy": "2024-03-23T10:38:03.734770Z", + "iopub.status.idle": "2024-03-23T10:38:03.739730Z", + "shell.execute_reply": "2024-03-23T10:38:03.738873Z" + }, + "papermill": { + "duration": 0.035632, + "end_time": "2024-03-23T10:38:03.742594", + "exception": false, + "start_time": "2024-03-23T10:38:03.706962", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:03.790002Z", + "iopub.status.busy": "2024-03-23T10:38:03.789170Z", + "iopub.status.idle": "2024-03-23T10:38:03.795174Z", + "shell.execute_reply": "2024-03-23T10:38:03.793819Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.026974, + "end_time": "2024-03-23T10:38:03.797432", + "exception": false, + "start_time": "2024-03-23T10:38:03.770458", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:03.871354Z", + "iopub.status.busy": "2024-03-23T10:38:03.870425Z", + "iopub.status.idle": "2024-03-23T10:38:03.879465Z", + "shell.execute_reply": "2024-03-23T10:38:03.878452Z" + }, + "papermill": { + "duration": 0.053073, + "end_time": "2024-03-23T10:38:03.881627", + "exception": false, + "start_time": "2024-03-23T10:38:03.828554", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a8023c07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:03.946219Z", + "iopub.status.busy": "2024-03-23T10:38:03.945419Z", + "iopub.status.idle": "2024-03-23T10:38:03.951282Z", + "shell.execute_reply": "2024-03-23T10:38:03.950242Z" + }, + "papermill": { + "duration": 0.054273, + "end_time": "2024-03-23T10:38:03.953279", + "exception": false, + "start_time": "2024-03-23T10:38:03.899006", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"realtabformer\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/realtabformer/2\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.010993, + "end_time": "2024-03-23T10:38:03.975663", + "exception": false, + "start_time": "2024-03-23T10:38:03.964670", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:03.999307Z", + "iopub.status.busy": "2024-03-23T10:38:03.998970Z", + "iopub.status.idle": "2024-03-23T10:38:04.008921Z", + "shell.execute_reply": "2024-03-23T10:38:04.008148Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024186, + "end_time": "2024-03-23T10:38:04.010757", + "exception": false, + "start_time": "2024-03-23T10:38:03.986571", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/realtabformer/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:04.035818Z", + "iopub.status.busy": "2024-03-23T10:38:04.035531Z", + "iopub.status.idle": "2024-03-23T10:38:06.019519Z", + "shell.execute_reply": "2024-03-23T10:38:06.018574Z" + }, + "papermill": { + "duration": 1.998838, + "end_time": "2024-03-23T10:38:06.021693", + "exception": false, + "start_time": "2024-03-23T10:38:04.022855", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:06.047206Z", + "iopub.status.busy": "2024-03-23T10:38:06.046730Z", + "iopub.status.idle": "2024-03-23T10:38:06.062936Z", + "shell.execute_reply": "2024-03-23T10:38:06.062096Z" + }, + "papermill": { + "duration": 0.031321, + "end_time": "2024-03-23T10:38:06.065085", + "exception": false, + "start_time": "2024-03-23T10:38:06.033764", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:06.093606Z", + "iopub.status.busy": "2024-03-23T10:38:06.093294Z", + "iopub.status.idle": "2024-03-23T10:38:06.101200Z", + "shell.execute_reply": "2024-03-23T10:38:06.100464Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.024071, + "end_time": "2024-03-23T10:38:06.103161", + "exception": false, + "start_time": "2024-03-23T10:38:06.079090", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:06.127831Z", + "iopub.status.busy": "2024-03-23T10:38:06.127536Z", + "iopub.status.idle": "2024-03-23T10:38:06.228272Z", + "shell.execute_reply": "2024-03-23T10:38:06.227437Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.115865, + "end_time": "2024-03-23T10:38:06.230662", + "exception": false, + "start_time": "2024-03-23T10:38:06.114797", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:06.257767Z", + "iopub.status.busy": "2024-03-23T10:38:06.256956Z", + "iopub.status.idle": "2024-03-23T10:38:11.035854Z", + "shell.execute_reply": "2024-03-23T10:38:11.035079Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.794805, + "end_time": "2024-03-23T10:38:11.038303", + "exception": false, + "start_time": "2024-03-23T10:38:06.243498", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-23 10:38:08.478755: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-23 10:38:08.478812: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-23 10:38:08.480457: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:11.064392Z", + "iopub.status.busy": "2024-03-23T10:38:11.063707Z", + "iopub.status.idle": "2024-03-23T10:38:11.070397Z", + "shell.execute_reply": "2024-03-23T10:38:11.069656Z" + }, + "papermill": { + "duration": 0.021717, + "end_time": "2024-03-23T10:38:11.072370", + "exception": false, + "start_time": "2024-03-23T10:38:11.050653", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:11.098335Z", + "iopub.status.busy": "2024-03-23T10:38:11.098010Z", + "iopub.status.idle": "2024-03-23T10:38:32.650405Z", + "shell.execute_reply": "2024-03-23T10:38:32.649219Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 21.568357, + "end_time": "2024-03-23T10:38:32.652850", + "exception": false, + "start_time": "2024-03-23T10:38:11.084493", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-23T10:38:32.681206Z", + "iopub.status.busy": "2024-03-23T10:38:32.680360Z", + "iopub.status.idle": "2024-03-23T10:38:32.687924Z", + "shell.execute_reply": "2024-03-23T10:38:32.687067Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023864, + "end_time": "2024-03-23T10:38:32.689992", + "exception": false, + "start_time": "2024-03-23T10:38:32.666128", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:32.715763Z", + "iopub.status.busy": "2024-03-23T10:38:32.715477Z", + "iopub.status.idle": "2024-03-23T10:38:32.720172Z", + "shell.execute_reply": "2024-03-23T10:38:32.719329Z" + }, + "papermill": { + "duration": 0.01973, + "end_time": "2024-03-23T10:38:32.721959", + "exception": false, + "start_time": "2024-03-23T10:38:32.702229", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:32.747534Z", + "iopub.status.busy": "2024-03-23T10:38:32.747250Z", + "iopub.status.idle": "2024-03-23T10:38:40.234969Z", + "shell.execute_reply": "2024-03-23T10:38:40.234090Z" + }, + "papermill": { + "duration": 7.503307, + "end_time": "2024-03-23T10:38:40.237349", + "exception": false, + "start_time": "2024-03-23T10:38:32.734042", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_bs_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/bs_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_synth_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:40.268854Z", + "iopub.status.busy": "2024-03-23T10:38:40.267929Z", + "iopub.status.idle": "2024-03-23T10:38:40.586798Z", + "shell.execute_reply": "2024-03-23T10:38:40.585836Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.336872, + "end_time": "2024-03-23T10:38:40.588802", + "exception": false, + "start_time": "2024-03-23T10:38:40.251930", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.1,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': False, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:40.617517Z", + "iopub.status.busy": "2024-03-23T10:38:40.617188Z", + "iopub.status.idle": "2024-03-23T10:38:40.719176Z", + "shell.execute_reply": "2024-03-23T10:38:40.718233Z" + }, + "papermill": { + "duration": 0.11873, + "end_time": "2024-03-23T10:38:40.721284", + "exception": false, + "start_time": "2024-03-23T10:38:40.602554", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_train/realtabformer/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/treatment [400, 0]\n", + "Caching in ../../../../treatment/_cache_aug_val/realtabformer/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/treatment [0, 200]\n", + "Caching in ../../../../treatment/_cache_bs_train/realtabformer/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/treatment [100, 0]\n", + "Caching in ../../../../treatment/_cache_bs_val/realtabformer/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/treatment [0, 50]\n", + "Caching in ../../../../treatment/_cache_synth/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/treatment [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-23T10:38:40.751828Z", + "iopub.status.busy": "2024-03-23T10:38:40.751488Z", + "iopub.status.idle": "2024-03-23T10:38:41.286598Z", + "shell.execute_reply": "2024-03-23T10:38:41.285732Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.552546, + "end_time": "2024-03-23T10:38:41.288638", + "exception": false, + "start_time": "2024-03-23T10:38:40.736092", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:41.318298Z", + "iopub.status.busy": "2024-03-23T10:38:41.317963Z", + "iopub.status.idle": "2024-03-23T10:38:41.322429Z", + "shell.execute_reply": "2024-03-23T10:38:41.321675Z" + }, + "papermill": { + "duration": 0.021488, + "end_time": "2024-03-23T10:38:41.324378", + "exception": false, + "start_time": "2024-03-23T10:38:41.302890", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:41.351819Z", + "iopub.status.busy": "2024-03-23T10:38:41.351531Z", + "iopub.status.idle": "2024-03-23T10:38:41.358317Z", + "shell.execute_reply": "2024-03-23T10:38:41.357473Z" + }, + "papermill": { + "duration": 0.022918, + "end_time": "2024-03-23T10:38:41.360378", + "exception": false, + "start_time": "2024-03-23T10:38:41.337460", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "19390534" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:41.388072Z", + "iopub.status.busy": "2024-03-23T10:38:41.387769Z", + "iopub.status.idle": "2024-03-23T10:38:41.547899Z", + "shell.execute_reply": "2024-03-23T10:38:41.546929Z" + }, + "papermill": { + "duration": 0.176787, + "end_time": "2024-03-23T10:38:41.550337", + "exception": false, + "start_time": "2024-03-23T10:38:41.373550", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 52992] --\n", + "├─Adapter: 1-1 [2, 2648, 52992] --\n", + "│ └─Embedding: 2-1 [2, 2648, 69, 768] (215,808)\n", + "│ └─TensorInductionPoint: 2-2 [69, 1] 69\n", + "│ └─Sequential: 2-3 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 787,456\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 52992] (recursive)\n", + "│ └─Embedding: 2-4 [2, 661, 69, 768] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [69, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-7 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-8 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 19,606,342\n", + "Trainable params: 19,390,534\n", + "Non-trainable params: 215,808\n", + "Total mult-adds (M): 77.67\n", + "========================================================================================================================\n", + "Input size (MB): 1.83\n", + "Forward/backward pass size (MB): 3885.09\n", + "Params size (MB): 78.43\n", + "Estimated Total Size (MB): 3965.34\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T10:38:41.583803Z", + "iopub.status.busy": "2024-03-23T10:38:41.583444Z", + "iopub.status.idle": "2024-03-23T11:52:09.886092Z", + "shell.execute_reply": "2024-03-23T11:52:09.884985Z" + }, + "papermill": { + "duration": 4408.324194, + "end_time": "2024-03-23T11:52:09.889820", + "exception": false, + "start_time": "2024-03-23T10:38:41.565626", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding True True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.07077005374701326, 'avg_role_model_std_loss': 7.7690568529201345, 'avg_role_model_mean_pred_loss': 0.028255754237470072, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.07077005374701326, 'n_size': 900, 'n_batch': 450, 'duration': 490.3524549007416, 'duration_batch': 1.0896721220016479, 'duration_size': 0.5448360610008239, 'avg_pred_std': 0.15626739564745068}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007018332518638089, 'avg_role_model_std_loss': 5.231984866959922, 'avg_role_model_mean_pred_loss': 0.00025027377600039385, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007018332518638089, 'n_size': 450, 'n_batch': 225, 'duration': 197.8156943321228, 'duration_batch': 0.8791808636983236, 'duration_size': 0.4395904318491618, 'avg_pred_std': 0.09444081523999052}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.009379093581180642, 'avg_role_model_std_loss': 2.6030105899689175, 'avg_role_model_mean_pred_loss': 0.0004631821013883831, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009379093581180642, 'n_size': 900, 'n_batch': 450, 'duration': 485.68966460227966, 'duration_batch': 1.0793103657828438, 'duration_size': 0.5396551828914219, 'avg_pred_std': 0.2023497386896573}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00616615310408106, 'avg_role_model_std_loss': 7.329303962527855, 'avg_role_model_mean_pred_loss': 0.00015382666007410638, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00616615310408106, 'n_size': 450, 'n_batch': 225, 'duration': 198.96484756469727, 'duration_batch': 0.8842882113986545, 'duration_size': 0.44214410569932727, 'avg_pred_std': 0.07584491885941583}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007078851991594096, 'avg_role_model_std_loss': 2.0421946790576384, 'avg_role_model_mean_pred_loss': 0.00021883253530301686, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007078851991594096, 'n_size': 900, 'n_batch': 450, 'duration': 487.61613154411316, 'duration_batch': 1.0835914034313625, 'duration_size': 0.5417957017156813, 'avg_pred_std': 0.1953078424528414}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0064364255767268555, 'avg_role_model_std_loss': 6.630073926702688, 'avg_role_model_mean_pred_loss': 0.00046532210426971943, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0064364255767268555, 'n_size': 450, 'n_batch': 225, 'duration': 197.56161880493164, 'duration_batch': 0.8780516391330295, 'duration_size': 0.43902581956651476, 'avg_pred_std': 0.0723328961941095}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005932264231048244, 'avg_role_model_std_loss': 2.1949213939262298, 'avg_role_model_mean_pred_loss': 0.0001685581630674178, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005932264231048244, 'n_size': 900, 'n_batch': 450, 'duration': 485.7743980884552, 'duration_batch': 1.0794986624187892, 'duration_size': 0.5397493312093946, 'avg_pred_std': 0.1975122331425367}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005354185296421948, 'avg_role_model_std_loss': 6.1442645163203204, 'avg_role_model_mean_pred_loss': 0.0003165367537867994, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005354185296421948, 'n_size': 450, 'n_batch': 225, 'duration': 197.5341498851776, 'duration_batch': 0.8779295550452338, 'duration_size': 0.4389647775226169, 'avg_pred_std': 0.07811297937988507}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005074872308783256, 'avg_role_model_std_loss': 2.5619912489473453, 'avg_role_model_mean_pred_loss': 9.861685678809986e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005074872308783256, 'n_size': 900, 'n_batch': 450, 'duration': 481.7331359386444, 'duration_batch': 1.0705180798636542, 'duration_size': 0.5352590399318271, 'avg_pred_std': 0.18441032754652825}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006421878464142566, 'avg_role_model_std_loss': 6.356629352012609, 'avg_role_model_mean_pred_loss': 0.0003719029411907014, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006421878464142566, 'n_size': 450, 'n_batch': 225, 'duration': 195.86327409744263, 'duration_batch': 0.8705034404330784, 'duration_size': 0.4352517202165392, 'avg_pred_std': 0.0808127932534593}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004811109941874126, 'avg_role_model_std_loss': 2.220166819560064, 'avg_role_model_mean_pred_loss': 7.900620778736926e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004811109941874126, 'n_size': 900, 'n_batch': 450, 'duration': 482.6724216938019, 'duration_batch': 1.072605381541782, 'duration_size': 0.536302690770891, 'avg_pred_std': 0.18854611320442777}\n", + "Time out: 3907.791595220566/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 1050, 'n_batch': 525, 'role_model_metrics': {'avg_loss': 0.004820992151532578, 'avg_g_mag_loss': 0.4799999954586944, 'avg_g_cos_loss': 4.7382691742054054e-05, 'pred_duration': 27.937996864318848, 'grad_duration': 6.047521114349365, 'total_duration': 33.98551797866821, 'pred_std': 0.23635642230510712, 'std_loss': 0.0008501424454152584, 'mean_pred_loss': 6.148086686152965e-05, 'pred_rmse': 0.06943336874246597, 'pred_mae': 0.04260999336838722, 'pred_mape': 5612931.0, 'grad_rmse': 1.3776787519454956, 'grad_mae': 0.4483191668987274, 'grad_mape': 7.328680515289307}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.004820992151532578, 'avg_g_mag_loss': 0.4799999954586944, 'avg_g_cos_loss': 4.7382691742054054e-05, 'avg_pred_duration': 27.937996864318848, 'avg_grad_duration': 6.047521114349365, 'avg_total_duration': 33.98551797866821, 'avg_pred_std': 0.23635642230510712, 'avg_std_loss': 0.0008501424454152584, 'avg_mean_pred_loss': 6.148086686152965e-05}, 'min_metrics': {'avg_loss': 0.004820992151532578, 'avg_g_mag_loss': 0.4799999954586944, 'avg_g_cos_loss': 4.7382691742054054e-05, 'pred_duration': 27.937996864318848, 'grad_duration': 6.047521114349365, 'total_duration': 33.98551797866821, 'pred_std': 0.23635642230510712, 'std_loss': 0.0008501424454152584, 'mean_pred_loss': 6.148086686152965e-05, 'pred_rmse': 0.06943336874246597, 'pred_mae': 0.04260999336838722, 'pred_mape': 5612931.0, 'grad_rmse': 1.3776787519454956, 'grad_mae': 0.4483191668987274, 'grad_mape': 7.328680515289307}, 'model_metrics': {'realtabformer': {'avg_loss': 0.004820992151532578, 'avg_g_mag_loss': 0.4799999954586944, 'avg_g_cos_loss': 4.7382691742054054e-05, 'pred_duration': 27.937996864318848, 'grad_duration': 6.047521114349365, 'total_duration': 33.98551797866821, 'pred_std': 0.23635642230510712, 'std_loss': 0.0008501424454152584, 'mean_pred_loss': 6.148086686152965e-05, 'pred_rmse': 0.06943336874246597, 'pred_mae': 0.04260999336838722, 'pred_mape': 5612931.0, 'grad_rmse': 1.3776787519454956, 'grad_mae': 0.4483191668987274, 'grad_mape': 7.328680515289307}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T11:52:09.924722Z", + "iopub.status.busy": "2024-03-23T11:52:09.924381Z", + "iopub.status.idle": "2024-03-23T11:52:09.928866Z", + "shell.execute_reply": "2024-03-23T11:52:09.928005Z" + }, + "papermill": { + "duration": 0.024701, + "end_time": "2024-03-23T11:52:09.930906", + "exception": false, + "start_time": "2024-03-23T11:52:09.906205", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T11:52:09.963385Z", + "iopub.status.busy": "2024-03-23T11:52:09.963093Z", + "iopub.status.idle": "2024-03-23T11:52:10.102393Z", + "shell.execute_reply": "2024-03-23T11:52:10.101373Z" + }, + "papermill": { + "duration": 0.158663, + "end_time": "2024-03-23T11:52:10.104969", + "exception": false, + "start_time": "2024-03-23T11:52:09.946306", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T11:52:10.139338Z", + "iopub.status.busy": "2024-03-23T11:52:10.138998Z", + "iopub.status.idle": "2024-03-23T11:52:10.433865Z", + "shell.execute_reply": "2024-03-23T11:52:10.432944Z" + }, + "papermill": { + "duration": 0.314338, + "end_time": "2024-03-23T11:52:10.435934", + "exception": false, + "start_time": "2024-03-23T11:52:10.121596", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA0JklEQVR4nO3de1iT5/0/8HfOgQABQUEsglYQFYotCIVqnZWJjtqm64pyuXqYVdsf7KtldQ5nwa2/32i7udpVv23twdrvVQZ13+o6dLRIxRMocqpaD1OrQKvh4CGBAAkk9++PkEg0QBIiCcnndV3PpXlyP0/uB8nb57nz5P5wGGMMhBDiBLiO7gAhhBhQIBFCnAYFEiHEaVAgEUKcBgUSIcRpUCARQpwGBRIhxGnwHd0Be9DpdLh27Rq8vb3B4XAc3R1CyF0YY2hra0NwcDC43P7Pg1wikK5du4aQkBBHd4MQMojGxkY88MAD/T7vEoHk7e0NQH+wPj4+Du4NIeRuSqUSISEhxvdqf1wikAyXaT4+PhRIhDixwYZUaFCbEOI0KJAIIU6DAokQ4jRcYgyJDI1Wq0V3d7eju0FGMIFAAB6PN+T9UCC5McYY5HI5bt++7eiuEBfg6+uLoKCgId0LSIHkxgxhNGbMGHh6etJNpcQmjDF0dHSgubkZADB27Ngh7cxq27ZtY6GhoUwkErH4+Hh24sSJAdt//vnnbPLkyUwkErGoqCi2b98+k+cBmF3efPNNi/qjUCgYAKZQKAZsp9Pp2MUmJftn3Y8W7deV9fT0sLNnz7LW1lZHd4W4iNbWVnb27FnW09Nzz3OWvketHtQuLCxEVlYWcnNzUVNTg5iYGKSkpBjT8W7l5eVIT0/HypUrUVtbC5lMBplMhjNnzhjbXL9+3WT5+OOPweFw8Oyzz1qfsAO4ruhC8l8PY11BLRSd7j1mYhgz8vT0dHBPiKsw/C4NaTzS2hSMj49nGRkZxsdarZYFBwezvLw8s+3T0tJYamqqybqEhAS2Zs2afl/j6aefZk888YTFfbI0fRljbM6fD7LQDUXs36evW7x/V9TZ2cnOnj3LOjs7Hd0V4iIG+p26L2dIGo0G1dXVSE5ONq7jcrlITk5GRUWF2W0qKipM2gNASkpKv+2bmpqwb98+rFy5st9+qNVqKJVKk8VSM8MDAABHL7VYvA0hZHhYFUitra3QarUIDAw0WR8YGAi5XG52G7lcblX7Xbt2wdvbGz//+c/77UdeXh6kUqlxseaLtTMn9QbSxVaLtyGEDA+nuzHy448/xpIlSyAWi/ttk52dDYVCYVwaGxst3v+jD/qDx+Xg6o0ONN7ssEeXiZvhcDjYu3evo7thV5s3b8b06dMd3Q3rAikgIAA8Hg9NTU0m65uamhAUFGR2m6CgIIvbHzlyBBcuXMALL7wwYD9EIpHxi7TWfqHWRyzA9BBfAMDRS3SWREamTz75BL6+vnbb3yuvvILS0lK77c9WVgWSUChEbGysScd1Oh1KS0uRmJhodpvExMR7DrSkpMRs+48++gixsbGIiYmxpltWM162USARF6fRaCxq5+XlBX9///vcm8FZfcmWlZWFDz74ALt27cK5c+fw0ksvQaVSYcWKFQCApUuXIjs729h+7dq1KC4uxpYtW3D+/Hls3rwZVVVVyMzMNNmvUqnE7t27Bz07sodZvQPb5ZdaodNR4V4Dxhg6ND3DvjAriycXFxdj5syZ8PX1hb+/P5588klcvnwZAJCUlIQNGzaYtG9paYFAIMDhw4cB6G8zSU1NhYeHByZMmID8/HyEhYVh69atNv3cTp8+jSeeeAIeHh7w9/fH6tWr0d7ebny+rKwM8fHxkEgk8PX1xWOPPYb6+noAwLfffos5c+bA29sbPj4+iI2NRVVV1YCvV1ZWhhUrVkChUIDD4YDD4WDz5s0AgLCwMLz22mtYunQpfHx8sHr1agDAhg0bEBERAU9PT0ycOBGvvvqqycfzd1+yLV++HDKZDH/5y18wduxY+Pv7IyMj475/xcjqO7UXLVqElpYW5OTkQC6XY/r06SguLjYOXDc0NJhMUZmUlIT8/Hxs2rQJGzduRHh4OPbu3YuoqCiT/RYUFIAxhvT09CEe0uBiQnzhJeLjVkc3vrumRPQD0vv+miNBZ7cWU3O+GvbXPfvHFHgKLf9VVKlUyMrKwkMPPYT29nbk5OTgmWeeQV1dHZYsWYI333wTr7/+uvHO88LCQgQHB2PWrFkA9P9ptra2oqysDAKBAFlZWf3eR2dJX1JSUpCYmIiTJ0+iubkZL7zwAjIzM/HJJ5+gp6cHMpkMq1atwt///ndoNBpUVlYa+7ZkyRI8/PDDePfdd8Hj8VBXVweBQDDgayYlJWHr1q3IycnBhQsXAOjPcAz+8pe/ICcnB7m5ucZ13t7e+OSTTxAcHIzTp09j1apV8Pb2xm9/+9t+X+fgwYMYO3YsDh48iEuXLmHRokWYPn06Vq1aZdPPyhI2fXUkMzPznjMcg7KysnvWPffcc3juuecG3Ofq1auNaX6/CXhcPDrRHwfONeHIpRYKpBHm7htmP/74Y4wePRpnz55FWloa1q1bh6NHjxoDKD8/H+np6eBwODh//jwOHDiAkydPIi4uDgDw4YcfIjw83Ka+5Ofno6urC59++ikkEgkAYNu2bVi4cCHeeOMNCAQCKBQKPPnkk3jwwQcBAFOmTDFu39DQgPXr1yMyMhIALOqHUCiEVCoFh8MxOxb7xBNP4De/+Y3Juk2bNhn/HhYWhldeeQUFBQUDBpKfnx+2bdsGHo+HyMhIpKamorS01PkCyRXMCg/AgXNNOHqxFf/nJ5Mc3R2n4CHg4ewfUxzyuta4ePEicnJycOLECbS2tkKn0wHQv7mjoqIwb948fPbZZ5g1axauXLmCiooKvP/++wCACxcugM/n45FHHjHub9KkSfDz87Op7+fOnUNMTIwxjADgscceg06nw4ULF/D4449j+fLlSElJwU9/+lMkJycjLS3N+H2vrKwsvPDCC/if//kfJCcn47nnnjMGl60MQdtXYWEh/va3v+Hy5ctob29HT0/PoB8GTZs2zeQb/GPHjsXp06eH1LfBON3H/sPFcINk1dVb6NRoHdwb58DhcOAp5A/7Yu2XehcuXIibN2/igw8+wIkTJ3DixAkAdwZwlyxZgn/84x/o7u5Gfn4+oqOjER0dbfefl6V27tyJiooKJCUlobCwEBERETh+/DgA/djNd999h9TUVHzzzTeYOnUq9uzZM6TX6xuOgP7m5CVLluBnP/sZioqKUFtbi9///veDDnjffenI4XCM4X+/uG0gTQyQIFgqhkarQ+XVm47uDrHQjRs3cOHCBWzatAlz587FlClTcOvWLZM2Tz/9NLq6ulBcXIz8/HwsWbLE+NzkyZPR09OD2tpa47pLly7dsw9LTZkyBd9++y1UKpVx3bFjx8DlcjF58mTjuocffhjZ2dkoLy9HVFQU8vPzjc9FRETg5Zdfxtdff42f//zn2Llz56CvKxQKodVa9h9peXk5QkND8fvf/x5xcXEIDw83Dqo7G7cNJA6Hc+drJBfpayQjhZ+fH/z9/bFjxw5cunQJ33zzDbKyskzaSCQSyGQyvPrqqzh37pzJByWRkZFITk7G6tWrUVlZidraWqxevRoeHh42Tb9iuIl32bJlOHPmDA4ePIhf//rXeP755xEYGIgrV64gOzsbFRUVqK+vx9dff42LFy9iypQp6OzsRGZmJsrKylBfX49jx47h5MmTJmNM/QkLC0N7eztKS0vR2tqKjo7+b/INDw9HQ0MDCgoKcPnyZfztb38b8lnY/eK2gQQAM8NHAwCO0NdIRgwul4uCggJUV1cjKioKL7/8Mv785z/f027JkiX49ttvMWvWLIwfP97kuU8//RSBgYF4/PHH8cwzzxg/cRro2wH98fT0xFdffYWbN29ixowZ+MUvfoG5c+di27ZtxufPnz+PZ599FhEREVi9ejUyMjKwZs0a8Hg83LhxA0uXLkVERATS0tKwYMEC/OEPfxj0dZOSkvDiiy9i0aJFGD16NN58881+2z711FN4+eWXkZmZienTp6O8vByvvvqq1cc6HDjM2ptAnJBSqYRUKoVCobDqru3WdjXi/u8BAMDJ3ydjtLfofnXR6XR1deHKlSuYMGGCTW9EV/LDDz8gJCQEBw4cwNy5cx3dnRFroN8pS9+jbn2GFOAlwtSx+h/OMbpr22188803+PLLL3HlyhWUl5dj8eLFCAsLw+OPP+7orrk9tw4k4M5d23TZ5j66u7uxceNGTJs2Dc888wxGjx5tvEnys88+g5eXl9ll2rRpw9bHBQsW9NuPP/3pT8PWj+HmtvchGcwMD8D7h7/H0UstYIzRvNJuICUlBSkp5u+3euqpp5CQkGD2ucHuoLanDz/8EJ2dnWafGzVq1LD1Y7i5fSDNCBsFIZ+LJqUal5rbER44cO1x4tq8vb0HrT8/HMaNG+foLjiE21+yiQU8xIfp/8ehb/8T4lhuH0hAn2ltaRyJEIeiQMKd+ZGOf38D3dr7e2s8IaR/FEgApo71gb9ECJVGi9qG247uDiFuiwIJAJfLQdIk+hoJIY5GgdRrVm8gHaGBbTIIV5zk31lQIPUyDGx/23jb7avaEudn70n+Af3kihwOB7dv37brfq1BgdQr2NcDE0dLoGNAxeUbju4OIW6JAqmPWZPcvKotY4BGNfwLTfJvt0n+1Wo1XnnlFYwbNw4SiQQJCQkm00rX19dj4cKF8PPzg0QiwbRp07B//35cvXoVc+bMAaCf4oXD4WD58uU2/TyGwu3v1O7rsUkB2FVR7773I3V3AH8KHv7X3XgNEEoGb9eLJvnvf5L/zMxMnD17FgUFBQgODsaePXswf/58nD59GuHh4cjIyIBGo8Hhw4chkUhw9uxZeHl5ISQkBP/7v/+LZ599FhcuXICPjw88PDxs+pkMBQVSH3dXtQ0Z5enoLhEzaJJ/85P8NzQ0YOfOnWhoaEBwsP4/lldeeQXFxcXYuXMn/vSnP6GhoQHPPvuscUrfiRMnGrc3fEduzJgxdh+fshizwbZt21hoaCgTiUQsPj6enThxYsD2n3/+OZs8eTITiUQsKiqK7du37542Z8+eZQsXLmQ+Pj7M09OTxcXFsfr6eov6o1AoGACmUChsORwTP//vYyx0QxHLP2HZa49UnZ2d7OzZs6yzs/POSp2OMXX78C86nVV9/89//sMWL17MJkyYwLy9vZlEImEAjL9XP/vZz9iaNWsYY4x9//33DAA7deoUY4yxvXv3Mj6fz7Rarck+/fz82FtvvWXR6wNge/bsYYwx9vLLL7Of/OQnJs/fvn2bAWCHDh1ijDG2fPlyJhKJ2JNPPsm2bt3Krl27Zmybm5vL+Hw+mzt3LsvLy2OXLl2yqA87d+5kUqnUZF1RUREDwCQSicnC5/NZWloaY4yxDz74gPH5fJaUlMRycnLYt99+a9z+4MGDDAC7deuWRX24m9nfqV6WvketHkMqLCxEVlYWcnNzUVNTg5iYGKSkpPR7ylteXo709HSsXLkStbW1kMlkkMlkOHPmjLHN5cuXMXPmTERGRqKsrAynTp3Cq6++6pCJw4xVbd3xso3D0V86DfdCk/zbZZL/9vZ28Hg8VFdXo66uzricO3cOb7/9NgDghRdewPfff4/nn38ep0+fRlxcHN555x27HeuQWZuC8fHxLCMjw/hYq9Wy4OBglpeXZ7Z9WloaS01NNVmXkJBg/B+MMcYWLVrEfvnLX1rbFSN7niGdvHKDhW4oYjF/+Ir1aK37n3skGeh/M2fW2trKALDDhw8b1x05csTkrKW9vZ1JJBL25ZdfsqlTp7LXX3/d2PbcuXMMAKuqqjKuu3jxIgNg0xnSjh07mJ+fH2tvbzc+v2/fPsblcplcLje7/aOPPsp+/etfm31u8eLFbOHChYP24bPPPmNeXl4m6y5cuHDPz2Ywv/vd71h0dDRjjLFjx44xAKy1tdXi7fsa9jMkjUaD6upqJCcnG9dxuVwkJyejoqLC7DYVFRUm7QH9fDSG9jqdDvv27UNERARSUlIwZswYJCQkDHjjmVqthlKpNFnsxVDV9nZHN85es99+iX3QJP965ib5j4iIwJIlS7B06VJ88cUXuHLlCiorK5GXl4d9+/YBANatW4evvvoKV65cQU1NDQ4ePGh8vdDQUHA4HBQVFaGlpcXkk8JhY00C/vjjjwwAKy8vN1m/fv16Fh8fb3YbgUDA8vPzTdZt376djRkzhjHG2PXr1xkA5unpyf7617+y2tpalpeXxzgcDisrKzO7z9zcXAbgnsUeZ0iMMbbyk5MsdEMR237wol3254xG6hkSY4yVlJSwKVOmMJFIxB566CFWVlZmctbCGGP79+9nANjjjz9+z/bXrl1jCxYsYCKRiIWGhrL8/Hw2ZswY9t5771n0+ne/1qlTp9icOXOYWCxmo0aNYqtWrWJtbW2MMcbkcjmTyWRs7NixTCgUstDQUJaTk8O0Wi1Tq9Vs8eLFLCQkhAmFQhYcHMwyMzMt/jd58cUXmb+/PwPAcnNzGWOMaTQalpOTw8LCwphAIGBjx45lzzzzjHEMLTMzkz344INMJBKx0aNHs+eff97kjOiPf/wjCwoKYhwOhy1btsyifhjY4wzJ4YFk2Gd6erpJm4ULF7LFixeb3WdXVxdTKBTGpbGx0a6B9MmxKyx0QxFL31Fhl/05o5EcSPZm+P05cOCAo7syotkjkKz62D8gIAA8Hg9NTU0m65uamszWGAeAoKCgAdsHBASAz+dj6tSpJm2mTJmCo0ePmt2nSCSCSHT/KoTcXdXWQ2hdqWfi3L755hu0t7cjOjoa169fx29/+1ua5N9JWDWGJBQKERsbi9LSUuM6nU6H0tJSJCYmmt0mMTHRpD0AlJSUGNsLhULMmDHDeIOXwX/+8x+EhoZa0z27oaq2ro0m+XdeVt8YmZWVhWXLliEuLg7x8fHYunUrVCoVVqxYAUB/F+y4ceOQl5cHAFi7di1mz56NLVu2IDU1FQUFBaiqqsKOHTuM+1y/fj0WLVqExx9/HHPmzEFxcTH+9a9/mdzyPpwMVW0/r/oBRy+2YHbEaIf0g9wfNMm/E7PlWvGdd95h48ePZ0KhkMXHx7Pjx48bn5s9e/Y9g2Gff/45i4iIYEKhkE2bNs3sjZEfffQRmzRpEhOLxSwmJobt3bvX4v7Y82N/g3/W/chCNxSxlLcO2W2fzoTGkIi92WMMya0r1w7kRrsasS5c1dZQZTQsLMwh31kirqezsxNXr16lyrX3g7+XCNOCXbeqreHyo6Ojw8E9Ia7C8Ls0lEtb+nLtAGZOCsB315Q4crEVsoddq04Wj8eDr6+v8Ss/np6eVCST2IQxho6ODjQ3N8PX1xc8nu2fSlMgDcDVq9oabr2wdeoNQvry9fXt9/YfS1EgDcDVq9pyOByMHTsWY8aMQXc3TdtLbCcQCIZ0ZmRAgTQAQ1Xbo5daceRiq8sFkgGPx7PLLxMhQ0WD2oMwVrV1wYFtQpwNBdIgqKotIcOHAmkQhqq2HVTVlpD7jgJpEFTVlpDhQ4FkAapqS8jwoECyAFW1JWR4UCBZgKraEjI8KJAs5PZVbQkZBhRIFpoZrp8TyS3LIxEyTCiQLPToxFEmVW0JIfZHgWQhb7EA00N8AdBd24TcLxRIVnDrqraEDAMKJCvM6v34/9jlVmh1I36iTUKcDgWSFfpWtf3umsLR3SHE5VAgWUHA4+LRif4AaByJkPuBAslKhss2GkcixP5sCqTt27cjLCwMYrEYCQkJqKysHLD97t27ERkZCbFYjOjoaOzfv9/k+eXLl4PD4Zgs8+fPt6Vr993dVW0JIfZjdSAVFhYiKysLubm5qKmpQUxMDFJSUvqdl7m8vBzp6elYuXIlamtrIZPJIJPJcObMGZN28+fPx/Xr143L3//+d9uO6D6jqraE3D9WB9Jf//pXrFq1CitWrMDUqVPx3nvvwdPTEx9//LHZ9m+//Tbmz5+P9evXY8qUKXjttdfwyCOPYNu2bSbtRCIRgoKCjIufn59tR3SfGaraAjQdCSH2ZlUgaTQaVFdXIzk5+c4OuFwkJyejoqLC7DYVFRUm7QF9KeO725eVlWHMmDGYPHkyXnrpJdy40f+XWNVqNZRKpckynAxfIzlC40iE2JVVgdTa2gqtVovAwECT9YGBgZDL5Wa3kcvlg7afP38+Pv30U5SWluKNN97AoUOHsGDBAmi15sdo8vLyIJVKjUtISIg1hzFkjz2o/6TtvLwNLW3qYX1tQlyZU3zKtnjxYjz11FOIjo6GTCZDUVERTp48ibKyMrPts7OzoVAojEtjY+Ow9tfVq9oS4ihWBVJAQAB4PB6amppM1jc1NfVbIC4oKMiq9gAwceJEBAQE4NKlS2afF4lE8PHxMVmGm2EciS7bCLEfqwJJKBQiNjYWpaWlxnU6nQ6lpaVITEw0u01iYqJJewAoKSnptz0A/PDDD7hx4wbGjh1rTfeG1axJvdOR9Fa1JYQMndWXbFlZWfjggw+wa9cunDt3Di+99BJUKhVWrFgBAFi6dCmys7ON7deuXYvi4mJs2bIF58+fx+bNm1FVVYXMzEwAQHt7O9avX4/jx4/j6tWrKC0txdNPP41JkyYhJSXFTodpf3FhfhD1qWpLCBk6qyvXLlq0CC0tLcjJyYFcLsf06dNRXFxsHLhuaGgAl3sn55KSkpCfn49NmzZh48aNCA8Px969exEVFQVAXzX11KlT2LVrF27fvo3g4GDMmzcPr732GkQikZ0O0/7EAh5muEFVW0KGE4e5wPWGUqmEVCqFQqEY1vGk9w5dxuv/Po8nIsfg4+Uzhu11CRlpLH2POsWnbCNV36q2mh6qakvIUFEgDUHfqrZ1jbcd3R1CRjwKpCGgqraE2BcF0hBRVVtC7IcCaYioqi0h9kOBNERU1ZYQ+6FAsgOqakuIfVAg2QFVtSXEPiiQ7ICq2hJiHxRIduAtFuBhqmpLyJBRINnJTKpGQsiQUSDZCVW1JWToKJDs5KEHqKotIUNFgWQnfava0iyShNiGAsmOqKotIUNDgWRHhoHt6nqqakuILSiQ7Iiq2hIyNBRIdkRVbQkZGgokO6OqtoTYjgLJzqiqLSG2o0CyM6pqS4jtbAqk7du3IywsDGKxGAkJCaisrByw/e7duxEZGQmxWIzo6Gjs37+/37YvvvgiOBwOtm7dakvXnAJVtSXENlYHUmFhIbKyspCbm4uamhrExMQgJSUFzc3NZtuXl5cjPT0dK1euRG1tLWQyGWQyGc6cOXNP2z179uD48eMIDg62/kicCFW1JcRGzErx8fEsIyPD+Fir1bLg4GCWl5dntn1aWhpLTU01WZeQkMDWrFljsu6HH35g48aNY2fOnGGhoaHsrbfesrhPCoWCAWAKhcLyA7mPOjU9LOL3+1nohiL2H7nS0d0hxOEsfY9adYak0WhQXV2N5ORk4zoul4vk5GRUVFSY3aaiosKkPQCkpKSYtNfpdHj++eexfv16TJs2bdB+qNVqKJVKk8WZiAU8xE8YBYAu2wixhlWB1NraCq1WayybbRAYGAi5XG52G7lcPmj7N954A3w+H//1X/9lUT/y8vIglUqNS0hIiDWHMSxmGqe1pUAixFIO/5Sturoab7/9Nj755BNwOByLtsnOzoZCoTAujY2N97mX1nuMqtoSYjWrAikgIAA8Hg9NTU0m65uamhAUFGR2m6CgoAHbHzlyBM3NzRg/fjz4fD74fD7q6+vxm9/8BmFhYWb3KRKJ4OPjY7I4m75VbWsbbjm6O4SMCFYFklAoRGxsLEpLS43rdDodSktLkZiYaHabxMREk/YAUFJSYmz//PPP49SpU6irqzMuwcHBWL9+Pb766itrj8dpmFS1pcs2QizCt3aDrKwsLFu2DHFxcYiPj8fWrVuhUqmwYsUKAMDSpUsxbtw45OXlAQDWrl2L2bNnY8uWLUhNTUVBQQGqqqqwY8cOAIC/vz/8/f1NXkMgECAoKAiTJ08e6vE51KxJAfjXt9dw9FIrfjNvZB8LIcPB6kBatGgRWlpakJOTA7lcjunTp6O4uNg4cN3Q0AAu986JV1JSEvLz87Fp0yZs3LgR4eHh2Lt3L6Kioux3FE7q7qq2Ug+Bg3tEiHPjMDby79xTKpWQSqVQKBRON570xJYyfN+iwnu/jMX8KPPjbIS4Okvfow7/lM3VUVVbQixHgXSfUVVbQixHgXSfUVVbQixHgXSfUVVbQixHgTQMqKotIZahQBoGVNWWEMtQIA2DmAd84U1VbQkZFAXSMODzuHj0QapqS8hgKJCGiXE6EgokQvpFgTRMqKotIYOjQBomfavanrhyw9HdIcQpUSANk75Vbak8EiHmUSANI6pqS8jAKJCGEVW1JWRgFEjDiKraEjIwCqRhRlVtCekfBdIwo6q2hPSPAmmYxYX5QcTnokmpxqXmdkd3hxCnQoE0zKiqLSH9o0ByAKpqS4h5FEgOYBjYpqq2hJiyKZC2b9+OsLAwiMViJCQkoLKycsD2u3fvRmRkJMRiMaKjo7F//36T5zdv3ozIyEhIJBL4+fkhOTkZJ06csKVrI8KUIKpqS4g5VgdSYWEhsrKykJubi5qaGsTExCAlJQXNzc1m25eXlyM9PR0rV65EbW0tZDIZZDIZzpw5Y2wTERGBbdu24fTp0zh69CjCwsIwb948tLS4ZqUOqmpLSD+YleLj41lGRobxsVarZcHBwSwvL89s+7S0NJaammqyLiEhga1Zs6bf11AoFAwAO3DggEV9MrRXKBQWtXcGhZUNLHRDEXt621FHd4WQ+87S96hVZ0gajQbV1dVITk42ruNyuUhOTkZFRYXZbSoqKkzaA0BKSkq/7TUaDXbs2AGpVIqYmBizbdRqNZRKpcky0hjGkU79cBuKjm4H94YQ52BVILW2tkKr1RrLZhsEBgZCLpeb3UYul1vUvqioCF5eXhCLxXjrrbdQUlKCgIAAs/vMy8uDVCo1LiEhIdYchlMI9vXAxNES6BhQ8T1NR0II4ESfss2ZMwd1dXUoLy/H/PnzkZaW1u+4VHZ2NhQKhXFpbGwc5t7aB1W1JcSUVYEUEBAAHo+HpqYmk/VNTU0ICjJftz4oKMii9hKJBJMmTcKjjz6Kjz76CHw+Hx999JHZfYpEIvj4+JgsIxFVtSXElFWBJBQKERsbi9LSUuM6nU6H0tJSJCYmmt0mMTHRpD0AlJSU9Nu+737VateeooOq2hJiyupLtqysLHzwwQfYtWsXzp07h5deegkqlQorVqwAACxduhTZ2dnG9mvXrkVxcTG2bNmC8+fPY/PmzaiqqkJmZiYAQKVSYePGjTh+/Djq6+tRXV2NX/3qV/jxxx/x3HPP2ekwnRNVtSXEFN/aDRYtWoSWlhbk5ORALpdj+vTpKC4uNg5cNzQ0gMu9k3NJSUnIz8/Hpk2bsHHjRoSHh2Pv3r2IiooCAPB4PJw/fx67du1Ca2sr/P39MWPGDBw5cgTTpk2z02E6r5nhAaiqv4WjF1uRHj/e0d0hxKE4jI38OTCUSiWkUikUCsWIG0+qrr+JZ9+tgK+nANWbfgoel+PoLhFid5a+R53mUzZ3RVVtCbmDAsnBqKotIXdQIDmBWeFU1ZYQgALJKRjmR6KqtsTdUSA5gQlU1ZYQABRITqFvVVu6bCPujALJSRi/RkI3SBI3RoHkJPpWtW1u63JwbwhxDAokJ9G3qm35JRpHIu6JAsmJUFVb4u4okJwIVbUl7o4CyYlQVVvi7iiQnAhVtSXujgLJyVBVW+LOKJCcDFW1Je6MAsnJUFVb4s4okJwMl8vBY3TZRtwUBZITovuRiLuiQHJChoFtqmpL3A0FkhMyrWpLZ0nEfVAgOSlDVVu6bCPuxKZA2r59O8LCwiAWi5GQkIDKysoB2+/evRuRkZEQi8WIjo7G/v37jc91d3djw4YNiI6OhkQiQXBwMJYuXYpr167Z0jWXYZiO5BgNbBM3YnUgFRYWIisrC7m5uaipqUFMTAxSUlLQ3Nxstn15eTnS09OxcuVK1NbWQiaTQSaT4cyZMwCAjo4O1NTU4NVXX0VNTQ2++OILXLhwAU899dTQjmyEo6q2xC0xK8XHx7OMjAzjY61Wy4KDg1leXp7Z9mlpaSw1NdVkXUJCAluzZk2/r1FZWckAsPr6eov6pFAoGACmUCgsaj9SPPvfx1johiKWf8KynwMhzsrS96hVZ0gajQbV1dVITk42ruNyuUhOTkZFRYXZbSoqKkzaA0BKSkq/7QFAoVCAw+HA19fX7PNqtRpKpdJkcUU0rS1xN1YFUmtrK7RarbFstkFgYCDkcrnZbeRyuVXtu7q6sGHDBqSnp/db4TIvLw9SqdS4hISEWHMYI4ahPNKxy63Q6mg6EuL6nOpTtu7ubqSlpYExhnfffbffdtnZ2VAoFMalsbFxGHs5fKiqLXE3VgVSQEAAeDwempqaTNY3NTUhKCjI7DZBQUEWtTeEUX19PUpKSgas/y0SieDj42OyuCKqakvcjVWBJBQKERsbi9LSUuM6nU6H0tJSJCYmmt0mMTHRpD0AlJSUmLQ3hNHFixdx4MAB+Pv7W9Mtl0ZVbYk74Vu7QVZWFpYtW4a4uDjEx8dj69atUKlUWLFiBQBg6dKlGDduHPLy8gAAa9euxezZs7FlyxakpqaioKAAVVVV2LFjBwB9GP3iF79ATU0NioqKoNVqjeNLo0aNglAotNexjkh3V7X1EPIc3CNC7iNbPsJ755132Pjx45lQKGTx8fHs+PHjxudmz57Nli1bZtL+888/ZxEREUwoFLJp06axffv2GZ+7cuUKA2B2OXjwoEX9cdWP/RljTKfTsaS8Uha6oYgdPN/k6O4QYhNL36Mcxkb+bPJKpRJSqRQKhcIlx5M2/OMUCqsa8cLMCdj05FRHd4cQq1n6HnWqT9mIeY+F0/xIxD1QII0AVNWWuAsKpBGgb1Vb+rItcWUUSCPEna+RUJlt4rookEYIqmpL3AEF0ghBVW2JO6BAGiGoqi1xBxRIIwhVtSWujgJpBKGqtsTVUSCNIFTVlrg6CqQRhKraEldHgTTCUFVb4sookEYYw/xIVNWWuCIKpBFmrNQDD1JVW+KiKJBGoJlU1Za4KAqkEchQ1ZYGtomroUAagQxVbeupqi1xMRRII5C3WICHQ3wB0FkScS0USCMUVbUlrogCaYSiqrbEFVEgjVBU1Za4IpsCafv27QgLC4NYLEZCQgIqKysHbL97925ERkZCLBYjOjoa+/fvN3n+iy++wLx58+Dv7w8Oh4O6ujpbuuVWqKotcUVWB1JhYSGysrKQm5uLmpoaxMTEICUlBc3NzWbbl5eXIz09HStXrkRtbS1kMhlkMhnOnDljbKNSqTBz5ky88cYbth+JG6KqtsTlWFvwLT4+nmVkZBgfa7VaFhwczPLy8sy2T0tLY6mpqSbrEhIS2Jo1a+5paygaWVtba1WfXLlQ5EAuN7ex0A1FLHzjftah7nF0dwjpl6XvUavOkDQaDaqrq5GcnGxcx+VykZycjIqKCrPbVFRUmLQHgJSUlH7bW0KtVkOpVJos7mhCgATjfD2g0epw4gpN/k9GPqsCqbW1FVqtFoGBgSbrAwMDIZfLzW4jl8utam+JvLw8SKVS4xISEmLzvkYyDodzZxZJumwjLoDv6A7YIjs7G1lZWcbHSqXSbUNpZngACqsa8WlFPcov30CovydC/SUI8/fEeH9PhPlLEOQjBpfLcXRXCRmUVYEUEBAAHo+HpqYmk/VNTU0ICgoyu01QUJBV7S0hEokgEols3t6VPB4xGkE+YsiVXTh7XYmz1++9fBXyuRg/ylMfUqMkCAvw7H0swTg/Dwh4dPcHcQ5WBZJQKERsbCxKS0shk8kAADqdDqWlpcjMzDS7TWJiIkpLS7Fu3TrjupKSEiQmJtrcaXKH1EOAw7+dg4abHWi4qcLV1g7U31Ch/maH8btumh4dLjW3my2fxONyMM7XA6G9Z1N9z7BCRnlCLOA54KiIu7L6ki0rKwvLli1DXFwc4uPjsXXrVqhUKqxYsQIAsHTpUowbNw55eXkAgLVr12L27NnYsmULUlNTUVBQgKqqKuzYscO4z5s3b6KhoQHXrl0DAFy4cAGA/uxqKGdS7kLI52LSGC9MGuN1z3M9Wh2uK7pw9YYKV290oMH4Zwfqb6rQ1a3rDbMOs/czjZWKjWdToQGeCB1lCC1PeIsFw3F4xI1YHUiLFi1CS0sLcnJyIJfLMX36dBQXFxsHrhsaGsDl3rkESEpKQn5+PjZt2oSNGzciPDwce/fuRVRUlLHNl19+aQw0AFi8eDEAIDc3F5s3b7b12Aj0N1CGjNKf7cwKN31Op2NoblPrz6hudOCq8cxKhfrWDrSpe3Bd0YXrii6cuHLznn37S4TGMyrDGZZh3MrPUwAOh8atiHU4jI38usxKpRJSqRQKhQI+Pj6O7o5LYIzhVkc3rt5QocEQVjc6jOF1Q6UZcHtvMf9OWI0yvRwc4y2iQXY3Y+l7lAKJ2KStq7s3oPSXfvWt+tBquNmB64quAbcVC7gIHWU4m/LE+N4xqzB/CcZKxeDTILvLoUDqz6E3Ab4IEHoBIh9A5NX7dy9A6K3/U+QNCDwBuuSwSVe3Fg29g+oml4M3OvDj7c4BZyfgczkIGaX/FHC0twi+HgL4egog9RBA6ik0eezrIYS3mE9nWyOApe/REXkfks10OuDg/7OsLYerDypDWIm8e//uPfC6/oKO5z4/arGAh4hAb0QEet/zXLdWhx9vdRrPpvp+KthwowMarQ5XWlW40qqy6LU4HP0njfqAuhNa0r7B5Sk0Pta30a8X8ekTRGfjPu8SAND1ADNWAZp2QN2mXzTtgLrddB0YwHSAWqlf2uzw2nwPM2dj3n3WDRJ0Iu87Z3B8sf3O3ljvsep67izaHtPHuh5ApwV03Xc97gG03aaP+120gLYbAl0PwnQ9CDO09+wBxN3AWC2YthsdXWq0dXaho1ONdiaEknnhls4DN7QeaOnxRFO3GHK1CD92idDU7YEOJsLtjm7c7uhGvZWH7iHg3Tn7MgaWEL6eAvjc9bhvGy8R3z0G7HW974Gu20CXAujs/fPux9NkQNhMu7ykewUSXwik/mXgNowB3R36kFK3AZq2PoHVrv8HMhdi/a3T9g7+9nTqF1XL0I+Dw7s31Di8QcPgnjDR9ehDxklwAEh6l0Hx9Avj8qEV+qBH6AM13wedPG+oOBK0cbyghAQ3dRLc1HqgpVuMpm4xrqnFuNYlxG0mQVu3J64rtIOOed3z0lxOnzMyQZ8zMuFdZ2YCSO8KtGG/CbW7y3yIdN3u/fvt/gOnSwnAghEdv1AKpPuGwwGEEv3iHTh4+8H0aPoJrrY+oddu+lx/67p7L2OYtvcX5j5PzMYVAFx+78IDeHc95vL7tOH1eY6vv0Tt+5jLu3d/xraCe7fncPX/MRjeNObeSLoecHQ94HfdBL/rJsQApJYcV+9N/gwcaIXe6BH4oIvvrQ8zrhfaIIGCSXBT54kbWk+0dIsg13jgmlqEG1pPKHQSKFWeuKmy/u0jFnDhJeJDIuKb/Hnn7zzTdUIufHld8IEK3kwFia4dnqwd4h4lBN1t/Z+1GB73WBe2ZvE9ALEU8PDV/yn27fPYF3hgxtBfw/BSdtsTMY8vBPijAM9RQ9+XTgtoVHcFV2+wMd0QwuDuQBEAXCf/pMtwJmsSWLf7vClv93M20Pt8Tyc4YOBrlOBrlBAD8B3sNfkwecdo+Z7QCHyg5nujk+uFdq4X2uCF20yCWzoPtPZ4oKXHA9c1YjSpRWiHJzx7uuCjVcGnowNSjgo+MPypMvvYG53gcob2uRMDBxqBPni1IimYyAfw8AXXwxdcTz8IJX7gS/zA8fAzEzhS/YdAw4QCaSTh8gCxj35xd33PZKXjrN++Rz1wYA20TqMfVOT1dMCjpwMekA8eZkN8T6shhBISKJgnbjMJlMwTCvT9U2L8UwlPKJikt70E7RCDdQ38HwyXg94zMy68RB2QiDTwEt2698xNbDhzu/P3iaMlGCv1GNoB9qJAIu6JL9JfkttyWa7t0Y8ldt6y7qxM3aYP0IEuf+55LAXEUogEYowGMBr6rwOpNFqo1D1o711UvUtbV+/fNVr9c72P29U9UGn0j/XttcZ1jAE6BrR16be31u8WROLF2Q9a/3M0gwKJEGvx+PpLcHtchtuAz+NC6sGF1GPo3yXU6Rg6u/Xh1qbuE15qLdrV3WhXa82EXZ+/q7UI8hHb4aj0KJAIcWNcLgeS3suyMY7uDKgMEiHEiVAgEUKcBgUSIcRpUCARQpwGBRIhxGlQIBFCnAYFEiHEabjEfUiGOebctYItIc7O8N4cbD5Ilwiktjb9d4vctVgkISNFW1sbpNL+52RwiSlsdTodrl27Bm9v70EnzjJUuW1sbHSZ+bfpmEYGVzsma46HMYa2tjYEBwebVCW6m0ucIXG5XDzwwANWbePj4+MSvxR90TGNDK52TJYez0BnRgY0qE0IcRoUSIQQp+F2gSQSiZCbmwuRaPhmwbvf6JhGBlc7pvtxPC4xqE0IcQ1ud4ZECHFeFEiEEKdBgUQIcRoUSIQQp0GBRAhxGm4XSNu3b0dYWBjEYjESEhJQWVnp6C7Z7PDhw1i4cCGCg4PB4XCwd+9eR3dpSPLy8jBjxgx4e3tjzJgxkMlkuHDhgqO7NSTvvvsuHnroIePdzImJifj3v//t6G7Z1euvvw4Oh4N169YNeV9uFUiFhYXIyspCbm4uampqEBMTg5SUFDQ3Nzu6azZRqVSIiYnB9u3bHd0Vuzh06BAyMjJw/PhxlJSUoLu7G/PmzYNKpXJ012z2wAMP4PXXX0d1dTWqqqrwxBNP4Omnn8Z3333n6K7ZxcmTJ/H+++/joYcess8OmRuJj49nGRkZxsdarZYFBwezvLw8B/bKPgCwPXv2OLobdtXc3MwAsEOHDjm6K3bl5+fHPvzwQ0d3Y8ja2tpYeHg4KykpYbNnz2Zr164d8j7d5gxJo9GguroaycnJxnVcLhfJycmoqKhwYM9IfxQKBQBg1CjHFGS0N61Wi4KCAqhUKiQmJjq6O0OWkZGB1NRUk/fUULnEt/0t0draCq1Wi8BA09LJgYGBOH/+vIN6Rfqj0+mwbt06PPbYY4iKinJ0d4bk9OnTSExMRFdXF7y8vLBnzx5MnTrV0d0akoKCAtTU1ODkyZN23a/bBBIZWTIyMnDmzBkcPXrU0V0ZssmTJ6Ourg4KhQL/+Mc/sGzZMhw6dGjEhlJjYyPWrl2LkpISiMX2K6MNuFEgBQQEgMfjoampyWR9U1MTgoKCHNQrYk5mZiaKiopw+PBhq+e5ckZCoRCTJk0CAMTGxuLkyZN4++238f777zu4Z7aprq5Gc3MzHnnkEeM6rVaLw4cPY9u2bVCr1eDxeDbt223GkIRCIWJjY1FaWmpcp9PpUFpa6hLX866AMYbMzEzs2bMH33zzDSZMmODoLt0XOp0OarXa0d2w2dy5c3H69GnU1dUZl7i4OCxZsgR1dXU2hxHgRmdIAJCVlYVly5YhLi4O8fHx2Lp1K1QqFVasWOHortmkvb0dly5dMj6+cuUK6urqMGrUKIwfP96BPbNNRkYG8vPz8c9//hPe3t6Qy+UA9DMNenh4OLh3tsnOzsaCBQswfvx4tLW1IT8/H2VlZfjqq68c3TWbeXt73zOuJ5FI4O/vP/TxviF/TjfCvPPOO2z8+PFMKBSy+Ph4dvz4cUd3yWYHDx5kAO5Zli1b5uiu2cTcsQBgO3fudHTXbParX/2KhYaGMqFQyEaPHs3mzp3Lvv76a0d3y+7s9bE/zYdECHEabjOGRAhxfhRIhBCnQYFECHEaFEiEEKdBgUQIcRoUSIQQp0GBRAhxGhRIhBCnQYFECHEaFEiEEKdBgUQIcRr/H2GebDJiUUVGAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T11:52:10.471502Z", + "iopub.status.busy": "2024-03-23T11:52:10.471183Z", + "iopub.status.idle": "2024-03-23T12:00:38.076334Z", + "shell.execute_reply": "2024-03-23T12:00:38.075258Z" + }, + "papermill": { + "duration": 507.625668, + "end_time": "2024-03-23T12:00:38.079120", + "exception": false, + "start_time": "2024-03-23T11:52:10.453452", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T12:00:38.115207Z", + "iopub.status.busy": "2024-03-23T12:00:38.114850Z", + "iopub.status.idle": "2024-03-23T12:00:38.136810Z", + "shell.execute_reply": "2024-03-23T12:00:38.135908Z" + }, + "papermill": { + "duration": 0.042317, + "end_time": "2024-03-23T12:00:38.139059", + "exception": false, + "start_time": "2024-03-23T12:00:38.096742", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.0000140.396190.0048216.0606430.4483197.3286811.3776790.00006127.9043750.042615612931.00.0694330.2363560.0008533.965018
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.000014 0.39619 0.004821 6.060643 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.448319 7.328681 1.377679 0.000061 27.904375 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.04261 5612931.0 0.069433 0.236356 0.00085 \n", + "\n", + " total_duration \n", + "realtabformer 33.965018 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T12:00:38.174253Z", + "iopub.status.busy": "2024-03-23T12:00:38.173901Z", + "iopub.status.idle": "2024-03-23T12:00:38.635597Z", + "shell.execute_reply": "2024-03-23T12:00:38.634624Z" + }, + "papermill": { + "duration": 0.480868, + "end_time": "2024-03-23T12:00:38.637502", + "exception": false, + "start_time": "2024-03-23T12:00:38.156634", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T12:00:38.673292Z", + "iopub.status.busy": "2024-03-23T12:00:38.672910Z", + "iopub.status.idle": "2024-03-23T12:10:05.038843Z", + "shell.execute_reply": "2024-03-23T12:10:05.037806Z" + }, + "papermill": { + "duration": 566.386877, + "end_time": "2024-03-23T12:10:05.041527", + "exception": false, + "start_time": "2024-03-23T12:00:38.654650", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/realtabformer/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/realtabformer/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T12:10:05.078722Z", + "iopub.status.busy": "2024-03-23T12:10:05.078375Z", + "iopub.status.idle": "2024-03-23T12:10:05.105204Z", + "shell.execute_reply": "2024-03-23T12:10:05.104430Z" + }, + "papermill": { + "duration": 0.048226, + "end_time": "2024-03-23T12:10:05.107148", + "exception": false, + "start_time": "2024-03-23T12:10:05.058922", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T12:10:05.151899Z", + "iopub.status.busy": "2024-03-23T12:10:05.151285Z", + "iopub.status.idle": "2024-03-23T12:10:05.157550Z", + "shell.execute_reply": "2024-03-23T12:10:05.156521Z" + }, + "papermill": { + "duration": 0.032918, + "end_time": "2024-03-23T12:10:05.160436", + "exception": false, + "start_time": "2024-03-23T12:10:05.127518", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.4394384708673377}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T12:10:05.201905Z", + "iopub.status.busy": "2024-03-23T12:10:05.201102Z", + "iopub.status.idle": "2024-03-23T12:10:05.602702Z", + "shell.execute_reply": "2024-03-23T12:10:05.601801Z" + }, + "papermill": { + "duration": 0.422745, + "end_time": "2024-03-23T12:10:05.604793", + "exception": false, + "start_time": "2024-03-23T12:10:05.182048", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFDklEQVR4nO3deZhT5dk/8O/JyZ7JMvu+McOm7GsRFFSqolWpdV+YgbK0lapFWsG+RdEqta9SqW157eIMKNZftWhbd6sgroAsiiCrzMLsayb7dp7fHycJhFnIZJI5yXB/rmuuTE5OTu5kZu559odjjDEQQoiEZFIHQAghlIgIIZKjREQIkRwlIkKI5CgREUIkR4mIECI5SkSEEMlRIiKESI4SESFEcpSISK8efvhhcBwX1WtWVlaC4zh88cUXYZ3/v//7vxg2bBh4nseECROiGguJH5SISL88/vjjeO211wbltd5991384he/wMyZM1FRUYHHH398UF6XDD651AGQxPL444/jxhtvxPz582P+Wh988AFkMhn+9re/QalUxvz1iHSoRDRE2Gw2qUOIuubmZmg0mqglIcYYHA5HVK41EHa7XeoQ4g4logQUaLs5dOgQbr/9diQnJ2PWrFkAgBdeeAGTJ0+GRqNBSkoKbr31VtTW1oY8/6OPPsJNN92EgoICqFQq5Ofn42c/+9k5/0g5joPNZsOmTZvAcRw4jkN5eTkAoLq6Gj/5yU8wcuRIaDQapKam4qabbkJVVVWP17Lb7Vi2bBlSU1NhMBiwYMECdHR0hLxWRUUFbDZb8LUqKysBAF6vF48++ihKSkqgUqlQVFSEBx98EC6XK+Q1ioqK8L3vfQ/vvPMOpkyZAo1Gg2effRbbt28Hx3H4xz/+gbVr1yI3Nxd6vR433ngjzGYzXC4X7rvvPmRkZCApKQkLFy7sdu1wP+s5c+ZgzJgx2LNnDy655BJotVo8+OCDfX7O5yOqmiWwm266CcOHD8fjjz8Oxhgee+wx/OpXv8LNN9+MxYsXo6WlBc888wwuueQS7Nu3DyaTCQDw8ssvw26348c//jFSU1Oxa9cuPPPMMzh16hRefvnlXl/v+eefx+LFizFt2jQsXboUAFBSUgIA2L17Nz799FPceuutyMvLQ1VVFTZu3Ig5c+bg0KFD0Gq1Iddavnw5TCYTHn74YRw5cgQbN25EdXV1MEk8//zz+POf/4xdu3bhr3/9KwDgoosuAgAsXrwYmzZtwo033oj7778fO3fuxLp16/DNN9/g1VdfDXmdI0eO4LbbbsOyZcuwZMkSjBw5MvjYunXroNFosGrVKhw/fhzPPPMMFAoFZDIZOjo68PDDD+Pzzz9HZWUliouLsWbNmuBzw/2sAaCtrQ3z5s3DrbfeijvvvBOZmZn9/EmfBxhJOA899BADwG677bbgsaqqKsbzPHvsscdCzj1w4ACTy+Uhx+12e7drrlu3jnEcx6qrq7u9zpl0Oh0rKyvr9vyervnZZ58xAGzz5s3BYxUVFQwAmzx5MnO73cHjv/3tbxkA9q9//St4rKysjOl0upBr7t+/nwFgixcvDjm+cuVKBoB98MEHwWOFhYUMAHv77bdDzt22bRsDwMaMGRMSw2233cY4jmPz5s0LOX/GjBmssLAweL8/n/Xs2bMZAPZ///d/3T4fchpVzRLYj370o+D3W7duhSAIuPnmm9Ha2hr8ysrKwvDhw7Ft27bguRqNJvi9zWZDa2srLrroIjDGsG/fvohiOfOaHo8HbW1tKC0thclkwt69e7udv3TpUigUiuD9H//4x5DL5XjzzTf7fJ3A4ytWrAg5fv/99wMA3njjjZDjxcXFuPLKK3u81oIFC0JimD59OhhjWLRoUch506dPR21tLbxeL4D+fdYAoFKpsHDhwj7f1/mOqmYJrLi4OPj9sWPHwBjD8OHDezz3zD+4mpoarFmzBv/+979D2mUAwGw2RxSLw+HAunXrUFFRgbq6OrAzFv7s6Zpnx5mUlITs7Oxe25QCqqurIZPJUFpaGnI8KysLJpMJ1dXVIcfP/IzOVlBQEHLfaDQCAPLz87sdFwQBZrMZqamp/fqsASA3N5d6/c6BElECO7MUIggCOI7DW2+9BZ7nu52blJQEAPD5fPjud7+L9vZ2PPDAAxg1ahR0Oh3q6upQXl4OQRAiiuWnP/0pKioqcN9992HGjBkwGo3gOA633nprxNfsS7gDLc/8jM7W0+fU1/FAcg33sw4nBiKiRDRElJSUgDGG4uJijBgxotfzDhw4gKNHj2LTpk1YsGBB8Ph7770X1uv0lgBeeeUVlJWV4amnngoeczqd6Ozs7PH8Y8eO4dJLLw3et1qtaGhowNVXX93n6xcWFkIQBBw7dgyjR48OHm9qakJnZycKCwvDeh8DEe5nTcJHbURDxA033ACe57F27dqQahEg/idva2sDcPq//ZnnMMawYcOGsF5Hp9P1mFx4nu/2us888wx8Pl+P1/nzn/8Mj8cTvL9x40Z4vV7Mmzevz9cPJKqnn3465Pj69esBANdcc8253sKAhftZk/BRiWiIKCkpwa9//WusXr0aVVVVmD9/PvR6PU6ePIlXX30VS5cuxcqVKzFq1CiUlJRg5cqVqKurg8FgwD//+c9ubUW9mTx5Mv773/9i/fr1yMnJQXFxMaZPn47vfe97eP7552E0GnHBBRfgs88+w3//+1+kpqb2eB23243LL78cN998M44cOYI//elPmDVrFq677ro+X3/8+PEoKyvDn//8Z3R2dmL27NnYtWsXNm3ahPnz54eUsmIl3M+ahI8S0RCyatUqjBgxAr/73e+wdu1aAGLD6xVXXBH8A1coFPjPf/6De+65B+vWrYNarcb3v/99LF++HOPHjz/na6xfvx5Lly7F//zP/8DhcKCsrAzTp0/Hhg0bwPM8tmzZAqfTiZkzZ+K///1vrz1Wf/jDH7BlyxasWbMGHo8Ht912G37/+9+H1fbz17/+FcOGDUNlZSVeffVVZGVlYfXq1XjooYf68WkNTDifNQkfx84uWxJCyCCjNiJCiOQoERFCJEeJiBAiOUpEhBDJUSIihEiOEhEhRHIJPY5IEATU19dDr9dHfZF3QsjAMcZgsViQk5MDmaz3ck9CJ6L6+vpuM6UJIfGntrYWeXl5vT6e0IlIr9cDEN+kwWCQOBpCyNm6urqQn58f/FvtTUInokB1zGAwUCIiJI6dq+mEGqsJIZKjREQIkRwlIkKI5BK6jYiQaGCMwev19rqIG+kdz/OQy+UDHj5DiYic19xuNxoaGmj31QHQarXIzs4e0AYBlIjIeUsQBJw8eRI8zyMnJwdKpZIGxvYDYwxutxstLS04efIkhg8f3uegxb5QIiJDzrEmC/bXdmJiQTJKM5J6Pc/tdkMQBOTn53fbiZaER6PRQKFQoLq6Gm63G2q1OqLrUGM1GVJsLi/e/roRpzocePvrBtjd3nM+J9L/4kQUjc+PfgJkSDnWbIVXEFc/5jgOrRa3xBGRcFDVjAwpte1io/PYXCPmjEyHnKf/tYmAfkpkSGnqcgIARmXrKQlFSVFRUbd95KKNflJkyLC5vLA4veA4IF2vkjoc0g9UNSNDBi/jcPnoDLi8AlRyHh8fa8XJVisuHZWBvOTzu1fM7XYPaJxPrFGJiAwZagWPcXkmTC1KAQC0WJ1otbrRafec45ndub1Cr19enxD2uZ4wzo3EnDlzsHz5cixfvhxGoxFpaWn41a9+FdwCu6ioCI8++igWLFgAg8GApUuXAgA+/vhjXHzxxdBoNMjPz8c999wDm80WvG5zczOuvfZaaDQaFBcXY8uWLRHF119UIiJDlkmjBGCPKBH9cdvxXh8rTtNh/sTc4P0/7zgBj6/nfUrzkjW4acrpxfue++QkHO7QqSQ/++6IfscHAJs2bcIPf/hD7Nq1C1988QWWLl2KgoICLFmyBADw5JNPYs2aNcEdcE+cOIGrrroKv/71r/Hcc8+hpaUlmMwqKioAAOXl5aivr8e2bdugUChwzz33oLm5OaL4+oMSERkyatvtwfYhlZyHUasAAHTYh2YXfn5+Pn73u9+B4ziMHDkSBw4cwO9+97tgIrrssstw//33B89fvHgx7rjjDtx3330AgOHDh+P3v/89Zs+ejY0bN6KmpgZvvfUWdu3ahalTpwIA/va3v2H06NExfy+UiMiQ8eHRFrRYXJg/MRfFaTqYNGIi6nT0v0R096WlvT4mO2sWyNJLSno99+wZI4tmFvc7lt585zvfCZmSMmPGDDz11FPBybtTpkwJOf/LL7/EV199FVLdYowFp7ocPXoUcrkckydPDj4+atQomEymqMXcG0pEZEhgjMHsTzhGfwIy+G8tzv4nIqU8/ObTWJ07UDqdLuS+1WrFsmXLcM8993Q7t6CgAEePHh2s0LqhRESGBKfndMOvQS3+WiepxFuXR2w0VgyxcUU7d+4Muf/5559j+PDh4Hm+x/MnTZqEQ4cOobS059LeqFGj4PV6sWfPnmDV7MiRI+js7Ixq3D0ZWj8Zct6yuMRSj1bJBwcyquQyaJU8krUKOD1Db62hmpoarFixAkeOHMHf//53PPPMM7j33nt7Pf+BBx7Ap59+iuXLl2P//v04duwY/vWvf2H58uUAgJEjR+Kqq67CsmXLsHPnTuzZsweLFy+GRqOJ+XuhEhEZEuwuMdFoVad/pTmOw7LZvbffJLoFCxbA4XBg2rRp4Hke9957b7Cbvifjxo3Dhx9+iF/+8pe4+OKLwRhDSUkJbrnlluA5FRUVWLx4MWbPno3MzEz8+te/xq9+9auYvxdKRGRIsLrEWfZJqp6rJUORQqHA008/jY0bN3Z7rKqqqsfnTJ06Fe+++26v18zKysLrr78ecuyuu+4aUJzhoKoZGRLs/rE5WiX9b01E9FMjQ0Jxmg5qhcw/iPG0g/Vm7K/tRHGaDheVpEkUHTkXSkRkSEjXq3qc6OrxMTR3uWBQKySIKna2b98udQhRRVUzMqTplGKb0dnTKkh8oUREhoTjzVbUttu7TTJVK8REFM6SsUQ6lIhIwmOM4a0DDXhlz6lgN36ANlAi8kQ2y50MjrhJRL/5zW/AcVxwQh4h4XJ5heA61dqzuu8DvWhOjw8+oecZ8kR6cZGIdu/ejWeffRbjxo2TOhSSgAJd90q5rNs0DrVCFpx46hiCo6uHCskTkdVqxR133IG//OUvSE5OljockoACCSZQDTsTx3EwaRQwaRXdFjQj8UPyRHT33Xfjmmuuwdy5c895rsvlQldXV8gXIYEeMY2i51HV5TOLsXBmMUza+F0q9Xwn6Tiil156CXv37sXu3bvDOn/dunVYu3ZtjKMiiSYwoVXTQ4mIJAbJSkS1tbW49957sWXLlrC3qV29ejXMZnPwq7a2NsZRkkQQSEQqOSWiRCVZiWjPnj1obm7GpEmTgsd8Ph927NiBP/zhD3C5XN3WVVGpVFCpaJsYEqowVScuDavpefT0/tpOHKw3Y1SWHpMLU/q+GGOAr/8LqQ0Yr+i+nGMvNm/ejJ/97Geor68P+XuYP38+9Ho9nn/++VhFGTOSJaLLL78cBw4cCDm2cOFCjBo1Cg888ECvizsRcrbepncEONw+NHe5kKkPo+Tt8wAfPRXF6MJ08f2APLw2rJtuugn33HMP/v3vf+Omm24CIO6+8cYbb/Q5sz6eSZaI9Ho9xowZE3JMp9MhNTW123FCBiLQm2YfIt33Go0Gt99+OyoqKoKJ6IUXXkBBQQHmzJkjbXARokmvJOHVtJ3evUPdQ89ZoBHbGc58M14hlk4GG9+/SblLlizB1KlTUVdXh9zcXFRWVqK8vDxkMf1EEleJaKjNKCaD44PDTeiwe3DTlLwed3QNdOs7vWEkIo4Lu4okpYkTJ2L8+PHYvHkzrrjiChw8eBBvvPGG1GFFLK4SESGRCMwj620ckUohdg4PtXWrFy9ejKeffhp1dXWYO3cu8vPzz/2kOCX5gEZCBkIQGFz+kk5P1bIzjzs9QnBL5qHg9ttvx6lTp/CXv/wFixYtkjqcAaFERBKayysgkFt6TURyHmoFD71aPqQmvhqNRvzgBz9AUlIS5s+fL3U4A0JVM5LQAvPMVAoZ+LO3YPVTymX48ZyhuZtHXV0d7rjjjoQfX0eJiCS0QCLqrX1oqOro6MD27duxfft2/OlPf5I6nAGjREQS2rkmvA5VEydOREdHB5544gmMHDlS6nAGjBIRSWjpehXmjs4M9oz1ZvuRZtR1OjCzJA1Fabo+z00Eve1blqgoEZGEZtQoMDbPeM7zzA4PmrtcwY0YSXyhXjNyXgjMzO9pLNFQ6tKXQjQ+P0pEJKE1W5yoabPD4ux7xrw6OKjx9CqNCoU4rcJut8cuwPNA4PMLfJ6RoKoZSWh7qjpwuNGCS0akY3Jh70sNnx7UeLpExPM8TCYTmpubAQBarTZh52pJgTEGu92O5uZmmEymAa2YQYmIJDSXVyzhqOR9F+4DiShwfkBWVhYABJMR6T+TyRT8HCNFiYgktNPTO/pORIFEdXYbEcdxyM7ORkZGBjweCRZES3AKhSIqa4dRIiIJ7XSJqO8/Bo2Ch0bJQ9FLyYnneVqMT0KUiEhCc/kbn881jqgoTYcfzR6a0zyGAkpEJKEFqmYhJSLGgOPvA41fAemjgBFXATLqII5n9NMhCcvrE+DxiWNYQhqrm74GTu0GvC6g4UugYZ9EEZJwUYmIJLS5ozPh9PpOJyLGgJrPxe+VWsBtB2p3gWVPxD/31cPl9eEHk/J6XTKESINKRCRhyXkZxuYZMbUo5fT4H2sTYGsFeDkwZZG47KujE5ylAU1dTjR3uYbcSo1DASUiMrS0HRdvk4sBlV68BYCOk8FSk/ussUREepSISMKyubyoabOjzeo6fbD9W/E2tVS8TfEnovbTiejsQY1EepSISMI61eHAP/eewgeH/aOifV7A0iR+byoQb43+W2sjVLxYfXOFs5sHGVSUiEjC6rZovrUJELyAQgNo/PPOtCliO5HPiyTBDCB04iuJD5SISMLqNs+sq068Nead3kee44CkTACAwdMa8jwSPygRkYTlDC6c7y8RWRrFW3126Il6cUKm3ttG3fZxisYRkYQVnN4RKBHZ/G1FSRmhJ/pLROOTPRg/gaZ5xCMqEZGEFVI1E3yAvV18QJcWeqI2Vby1tw1idKQ/KBGRhBUyz8zeLiYjXgGoTaEnalL8T7ACHufgBknCQomIJKxxeSbMGp6GTIMKsLWIB3XppxuqAxRqQJWELqcHr+88iP8eahr8YEmfqI2IJKzSjKTTd5r9iejs9qEAbSqEjnZ0tDaAk6f1fA6RDJWIyNDg8LcPBdqDzqZJAc9z0HjM1H0fhygRkYTEGENNmx1NXU4IAgMcHeIDml4W0NeYIJfJoPJZaGR1HKJERBKSV2D4595TeHFnDbw+4XSPWW+JSG0EL+Og8lrg9gq0l1mcoUREElJgMKOM46AQ7IDPIzZSn91jFqA2BRMRY4DbR9WzeEKJiCSk4BgihQyco1M8qDKI6xD1RG2EjAPUgh0c81I7UZyRNBFt3LgR48aNg8FggMFgwIwZM/DWW29JGRJJECGDGc/VPgQACg04Xgm5TAYj7PBQIoorknbf5+Xl4Te/+Q2GDx8Oxhg2bdqE66+/Hvv27cOFF14oZWgkzrk8ZwxmdJyjfQgQq20aEyYVujF5XDKQpBqEKEm4JE1E1157bcj9xx57DBs3bsTnn39OiYj0qccSkTal7yepTeCsLYDTHOPoSH/FzYBGn8+Hl19+GTabDTNmzOjxHJfLBZfr9Gp8XV1dgxUeiTOBRKRW8OFVzQBAbRRvKRHFHckbqw8cOICkpCSoVCr86Ec/wquvvooLLrigx3PXrVsHo9EY/MrPzx/kaEm8yDGqMWt4GkZm6sJPRCo96s0OfPpNNY42WWIfJAmb5Ilo5MiR2L9/P3bu3Ikf//jHKCsrw6FDh3o8d/Xq1TCbzcGv2traQY6WxIsMgxpTi1JQmiwHvG7xYG9d9wHKJDjcPljM7eiwuWMeIwmf5FUzpVKJ0lJxofPJkydj9+7d2LBhA5599tlu56pUKqhU1MhIzhCoZil1vXfdB6j0kMs4KL026r6PM5InorMJghDSDkRIT1osLgiMwWTrgAo43f7TF5UevIyDwmeHg/Y2iyuSJqLVq1dj3rx5KCgogMViwYsvvojt27fjnXfekTIskgA+PNqC2nY7vp/RjCIAUBvO/SSVHnJeBp554HE7Yhwh6Q9JE1FzczMWLFiAhoYGGI1GjBs3Du+88w6++93vShkWSQDBHTy8VvGAKoxExCsgU2oA2MCc1OMaTyRNRH/729+kfHmSwALrVSu9/t6vczVU+3FqPYBWCE7qNYsnkveaERKJ4IBGn008EE7VDIBMbQAHQOG1xSgyEom4a6wm5FwYY8GqmcLjr2KFUzUDkJKcgunDUsAV62IVHokAlYhIwnH7BDAGyAQP5D7/Yvjh9JoB4FQGcOAAF1XN4gklIpJwAltGa5kdMo4Tt5SWhzm+TKUXbykRxZWIEtG3334b7TgICZtKLsOs4WmYmuX/9VUZuu/c0euT9TjaZMHeY7VwuGksUbyIKBGVlpbi0ksvxQsvvACnk/aJIoNLreAxtSgFEzL8v75hVssAAMokmB0eWC2dsLu9sQmQ9FtEiWjv3r0YN24cVqxYgaysLCxbtgy7du2KdmyE9C0wFqhfiUjnH13thNtLiSheRJSIJkyYgA0bNqC+vh7PPfccGhoaMGvWLIwZMwbr169HS0tLtOMkJMjq8qK5ywmH1b8gWpg9ZgAAhRY8LwPA4LZTF368GFBjtVwuxw033ICXX34ZTzzxBI4fP46VK1ciPz8/OGKakGg72mTBlp01OFZdJx4IcwwRAEAmAxRaAICHBjXGjQEloi+++AI/+clPkJ2djfXr12PlypU4ceIE3nvvPdTX1+P666+PVpyEBAVGVauFwGDGflTNAEAl7hDrdVAiihcRDWhcv349KioqcOTIEVx99dXYvHkzrr76ashkYl4rLi5GZWUlioqKohkrIQD888yYALXPDkDdv6oZAE4pDmb0Oq0xiI5EIqJEtHHjRixatAjl5eXIzs7u8ZyMjAyaS0ZiwukRoPTZwXMM4GSAMqlfz5epksABgJsSUbyIKBG99957KCgoCJaAAhhjqK2tRUFBAZRKJcrKyqISJCFncnl9UPmskMs4sX1I1r8WhtGFOZDJqsGl0QyneBFRG1FJSQlaW1u7HW9vb0dxcfGAgyKkLy6vAJXXAjnP9btaBgC8Kkmc5kElorgRUSLqbd9wq9UKtVo9oIAIORcxEVkhl8n612MW4G8jgpu67+NFv8qmK1asAABwHIc1a9ZAq9UGH/P5fNi5cycmTJgQ1QAJOdv4PCN4BweVV9b/HjMArW452pos4KyNGDExBgGSfutXItq3bx8AsUR04MABKJXK4GNKpRLjx4/HypUroxshIWcZl2cC2hjQzkdUNXPJNGi3uaHydUY9NhKZfiWibdu2AQAWLlyIDRs2wGCIoFhMSDS4AtM7+v87qNCIM/CZxwX4PACviGZkJAIRdRtUVFREOw5CwuL1CWizumC0dEAtQ9hLxJ5JpdJC4Hj4BJ/YTqTp/zVIdIWdiG644QZUVlbCYDDghhtu6PPcrVu3DjgwQnpicXrxj8+OYXpDC6YVpZxeX6gfVEoeHl4DmdcKr9MCOSUiyYWdiIxGIzj/mi9GY/8bCAmJBpdXOD2GSKmNqFql5GXwyLRQwQqPwwL5OXaqJrEXdiI6szpGVTMiFZfXB6XXCl7GRdRjBgAyGQdBqQPcgMdhhSbKMZL+i2gckcPhgN1uD96vrq7G008/jXfffTdqgRHSE7FEZBFLRBH0mAUpdeAAeGkGflyIKBFdf/312Lx5MwCgs7MT06ZNw1NPPYXrr78eGzdujGqAhJzJ6fGJgxn5CAcz+l02dhimD0tBitwdxehIpCJeofHiiy8GALzyyivIyspCdXU1Nm/ejN///vdRDZCQMwXaiMSqmSni6yg0Ov80DxpdHQ8iSkR2ux16vdhb8e677+KGG26ATCbDd77zHVRXV0c1QELO5PL455kNuGrmn7FPiSguRLx4/muvvYba2lq88847uOKKKwCIe9nTIEcSS3nJGpQYBBg1igFVzY60CzjSZEFzW3sUoyORiigRrVmzBitXrkRRURGmT5+OGTNmABBLRxMn0uQdEjtFyUoUGzgka5UR95oBQLNbjnabG06bGehlEjcZPBGNrL7xxhsxa9YsNDQ0YPz48cHjl19+Ob7//e9HLThCugns3MErAHnkKz3IVeIMfJ/XC/jc4W/QSGIi4pWhsrKykJWVFXJs2rRpAw6IkL50drRC7fFBoU0FH+6mij1QqjRwcAr4BEFsJ6JEJKmIEpHNZsNvfvMbvP/++2huboYgCCGP006wJFY+OnAcafWdGKEvROoArqOSy9DFa+AVHOICadqUqMVI+i+iRLR48WJ8+OGHuOuuu5CdnR2c+kFIzPln3Q90fphKLoOH18In2AG3/dxPIDEVUSJ666238MYbb2DmzJnRjoeQXjHGwPlHQit0pgFdSyXn4ZGp4fUx6sKPAxH1miUnJyMlhYqyZHB5fAwKr1giGnAiUgRKRIzWro4DESWiRx99FGvWrAmZb0ZIrAV27+AAyLWmAV0rPUmFq6eUYmKBiUpEcSCiqtlTTz2FEydOIDMzE0VFRVAoQpdi2Lt3b1SCI+RMLo8XSq8Ncp4DN4AxRIA4A1+p0QPgAA/9Q5VaRIlo/vz5UXnxdevWYevWrTh8+DA0Gg0uuugiPPHEExg5cmRUrk+GFpfNDA4MMl4R0YJo3SgCu3lQ1UxqESWihx56KCov/uGHH+Luu+/G1KlT4fV68eCDD+KKK67AoUOHoNPpovIaZOhIgh25Jg2Y2gREoaf201oH0posKOB1oE2wpBXxgMbOzk688sorOHHiBH7+858jJSUFe/fuRWZmJnJzc8O6xttvvx1yv7KyEhkZGdizZw8uueSSSEMjQ5QRdhhTtIAp69wnh+F4J8DZ3MhxWsVpHjQMRTIRJaKvvvoKc+fOhdFoRFVVFZYsWYKUlBRs3boVNTU1wbWK+stsNgNArz1yLpcLLpcreL+rqyui1yEJKrhzR3SWKpapxH35fF4v4HUBCioXSSWiXrMVK1agvLwcx44dC9nZ9eqrr8aOHTsiCkQQBNx3332YOXMmxowZ0+M569atg9FoDH7l5+dH9FokMTks7XB6fPAootA+BECpVMMnO2OaB5FMRIlo9+7dWLZsWbfjubm5aGxsjCiQu+++G19//TVeeumlXs9ZvXo1zGZz8Ku2tjai1yKJ6eSpBuyr7cTBKK3coVKIi+h7aSyR5CKqmqlUqh6rRUePHkV6enq/r7d8+XK8/vrr2LFjB/Ly8vp8XZWKJieet5xi1Z3XRqdqppLL4OY18ApdVCKSWEQlouuuuw6PPPIIPB4PAIDjONTU1OCBBx7AD37wg7CvwxjD8uXL8eqrr+KDDz5AcXFxJOGQ8wFjgLMTACDXRWf/H5WcPz26msYSSSqiRPTUU0/BarUiPT0dDocDs2fPRmlpKfR6PR577LGwr3P33XfjhRdewIsvvgi9Xo/GxkY0NjbC4XBEEhYZytxW+LxeMHBQDnBUdYBKLoNHpqGqWRyIqGpmNBrx3nvv4ZNPPsGXX34Jq9WKSZMmYe7cuf26TmDHjzlz5oQcr6ioQHl5eSShkaHK0QmvT4BbngS1ShmVS04uSgbHlUJR00BVM4n1OxEJgoDKykps3boVVVVV4DgOxcXFyMrKEmdH92MsBqMlOkm4nJ3wCgxOpQEaBR+VS6rkPBCY5kGJSFL9qpoxxnDddddh8eLFqKurw9ixY3HhhReiuroa5eXltEwsiRnB3gGvwOCS66FWRNSi0DPazSMu9KtEVFlZiR07duD999/HpZdeGvLYBx98gPnz52Pz5s1YsGBBVIMkhDk6kWvSQJ6WDbU8OiUis92DL6tsyG+zoVhFiUhK/frX8ve//x0PPvhgtyQEAJdddhlWrVqFLVu2RC04QgJ4lxkFKVpMHT0MMll0pmK4fQIOtPjQbnWJvWbUVCCZfiWir776CldddVWvj8+bNw9ffvnlgIMipBt/1/1Adnc9m1ohg4dXw+NjYIIX8Dqjdm3SP/1KRO3t7cjMzOz18czMTHR0dAw4KEJC+DzwOrrg9PjgitL0DgBQK3gwTg6PTOVfqZGqZ1LpVyLy+XyQy3tvVuJ5Hl6vd8BBERLC0Yl2mxtf1Nnx5uHOqF1WLuMgl3Hw8DSWSGr9aqxmjKG8vLzXaRZnzownJGoCXfdyQ9QaqgFxRoBawcMj08Dns1OJSEL9SkRlZWXnPId6zEjUOTrh9Yld91pl9BIREGgn0sIr2GhbIQn1KxFVVFTEKg5CeufshFcQ4JQbkBLFEhEAqBS8f+KrQFUzCUW8QiMhg8bRIZaIlFEezAjge+OyIU8phaK6nqpmEqJEROKfvR1egcGhMEIT5aqZVimnaR5xILr/XgiJNsEHOM3w+gQ45KaoNlYHBaZ5eCgRSYUSEYlvjg6ACUgxJmFUQTZMWsW5n9MPte12fFhlR2OXk0pEEqKqGYlvdnFd2LycXORdGJ3dO87UYXfj62YPshweZLnttJuHRKhEROKbw79AtTY1JpdXK3hxQKNPAJhAKzVKhBIRiW/2NgiMwSozwOX1Rf3yGgUPxvFwcv5BujSWSBKUiEh8s7fD5vZi62EHXvi8JuqXV/mHAzgCe73SWCJJUBtRorI0AlUfi98XzwaS+r97SkJwtMPjY3AqTNBHueseEKtmAGCHGgwOcNRgLQlKRInIaQb2vyjuTgoA5lPAlEWA2iBtXNHmcQBuu9h1rzIiMxaJyD8cwMVpIAgO8JSIJEFVs0R0YpuYhJLSAV2a+AcbKB0NJf4eM6dMC0GmiNpa1WdS8Bz44Ax8gcYSSYQSUaJxdgEth8XvR10LjJwnft94AHBZpIsrFuxtAAAbL26oqFVGvwDPcRzKZxbhe1NLoZTLaCyRRCgRJZrGr8SxLqZ8QJ8JGPMAY67Y9dx0SOroosvWAgCwysUNFaM9vSPAoFZAqdaDo2kekqFElGiavxFvs8efPpY5Rrxt+nrw44klfyLqkpkAIOpLgIRQ6sRbSkSSoESUSJxmwNYqjvxNLT19PH2UeMzaHNwffkjwJ6LsnHyMzTUiVRedjRXPdqTRgu1VdrTb3ZSIJEKJKJG0HQcAeJKy4cTpP8pjHT5Uuf09Zu3fShFZ9HkcgEsc03PB8BLMvSATGQZ1TF6q3uzA1y0+WJ1ecWS1IMTkdUjvKBElkvaTAIBvXBl44fNq1Lbb0Wxx4vWvGrDHkgKn1zd0EpG/NAS1EZD3vDRxtGgUPDwyNTwCE9vfaJrHoKNElCgYA8yn4BMY9lkMsDi9cHkFpCepkJ+iRZsqD41mJ9BRPTT25/InIkGbhg6bG05P9Kd3BGiVPMDJaHS1hCgRJQp7O+BxoNXuQ4csBSatAsPSdOA4DpMLk2FTpqHJJsDncZ4uTSQyWysAwCo3ofLTKlR8UhWzlwo0gjtkWvHAUBsGkQAoESUKcy0AoF4wgXE8RmcbgjueFqZoYdCq0CFPR6fdHTw3ofmTqV2RAiC2PWaBaR42BBJRV8xei/SMElGi6KqDVxBQ5RHH1JRmJAUfksk4DM9IgkWVhXabW5zykcgYA6xNAACLfwyRThW72UiBgZJWaMQDVCIadJSIEoX5FMwODzqVmUjWKrp1ZZdmJMGiykSn3QOhM8FLRI4OwOsGeDm6ZOKo6iRV7EpEgdKWDVoIjFEikgBNek0EHgdgb4fZ4YFVn4mx/rahM2Ub1WD6XCjNPNzWTqhdFkAVve2ZB5WlQbzVZcDiErvSY1kiUsllWDSrGNoOF2RHjlMikgAlokTgr6Ykp6RjbF4WStKTup3CcRwWXDwcij0lYkOvpTGBE1GjeKvPgs0p9pbFMhFxHAejRgG4xNIXJaLBR1WzRGBtBgAkZ+ZjzsgM5KdoezxNwcsAQ454p6t+sKKLvjMTkcsLAEiKYSIKCiRuV9fQGAKRQCgRJQJ/iQhJmec+V58FH2PwmRM0ETEGWP2JKCkLI7L0GJtrREqMpncEfF1nxnvHbTA7PIDPK1aHyaCRNBHt2LED1157LXJycsBxHF577TUpw4lflkZ02N1o8BnERd77sKNBji+q2tHeVJOY/9UDDdUyOaBLw4R8E+ZekIm0pNiOrq5pt+PrRjssgj/hUfVsUEmaiGw2G8aPH48//vGPUoYR33wewN6O2g47th714mRr35MymS4dPshgsXQl5gTYQEN1Ujogi+Fs+7PQoEZpSdpYPW/ePMybN0/KEOKfrQWC4EOXVwE3rzvnxM+8NANOKlLQ5egQ/6g1psGJM1rMdeKtIRdurwCrywudiocqFju8niHQBmWHBoCNBjUOsoRqI3K5XOjq6gr5GvKsTbC7fbAq0qBRyWFQ9/2/I9ekgU2VDofHB0d7Ag5s7PLHbMxDfacDmz6twj++iP37CPTKWUAlIikkVCJat24djEZj8Cs/P1/qkGLP0gSrywubMhWZBlW38UNnUyt4KExiz1lnU4INbPS6gz2EMOTC4hR7zM6VfKMhUCKi0dXSSKhEtHr1apjN5uBXbW2C/aFFwtoEm8sLmyIVmfrw1uMxZRYCAGyttYnVYN1VJ8arNgBqA7qcHgDiUq6xFmgjMgv+z5gS0aBKqAGNKpUKKlVse0/iiiAAtmaxRJSUhkxjeIkoMysXjRwPq80u9kJpU2IcaJR0+duHjHniXYeYiPSDUCIKVM2sTAsfY+ATsaE/gSVUIjrvONrh83hg9crglBuRGeYKhbkpSehKyUUGWsG66sAlSiIKNlT7E1GgRKSJfYlIJZdh0cxiaFk6+F2fiY3VggDIEqrSkLAkTURWqxXHjx8P3j958iT279+PlJQUFBQUSBhZnLA0QsYBY0aUwliUFfboYrWCx8QLRwOnvgAsTUDW2BgHGgWC7/TyJf4S0ek2otgnIo7jYNQqAMEoDhsQfIDbIq4QSWJO0kT0xRdf4NJLLw3eX7FiBQCgrKwMlZWVEkUVR6xN4h9Ieh6Muf38g9BnibeBcTnxrqtOHDOl1AJJGfD6xK57ADBoBvHXVCYDVAaxSuvopEQ0SCRNRHPmzAFLpMbUwRboQQpnasfZ9Nlwen2wNtYgLRGqGB1V4q2pEOA4CIxhxrBUWF3emOzw2pMjjRZUtdkwwaNCJpCYA0ITFLURxSv/4mC17XZYLRoUe3zBlQTD4VQYseeUDTLBA01nE3Qp2TEMNgoCiSi5CACglMswfVjqoIbQYHbgUH0X8uQafyLqHNTXP5/F+b/J85irC16XDafMTrxT7RMX7OoHtVIBzl89a22ojkWE0eN1AV3+KqQ/EUkh0AZn4fzLrFCJaNBQIopX1mbYXD7Y5clI0qgj2vddlyY2+nY2VUU5uCjrqBa3zNaYglNSWq0utFld8AmDV3VP8g8TMAv+0dWOzkF77fMdJaJ4ZW2CzR0YUR3ZxoLJ/oGN9ta6aEYWfW3HxNszdq/95HgrNn9Wja/rBq9Uovf3znUEEhFVzQYNJaJ45R9RbVdEnogycovAARCszbA5XNGNL1oEIbiD7ZmJqN3mBoCYr0N0psBUkjafVuxEcVnFtYlIzFEiilfWZlidYokoK8JEpNanQqXRQsZ8aGyI0+kwXXWA2y7u5moSx455fYK4QBmA5EFMRDqlHLyMg5tTwcX5xy45Ogbt9c9nlIjikccJr60dTq8AmyIVGYYIp7VwHNQpuQCA9sY4bbAOVstKgusPmR0eMCb2nOliuJ/Z2WQyTmyw5jg4FSbxoKN90F7/fEbd9/HI2gSbyweXPAl6vb5f3fZny8orQqa3ATqjM4oBRgljQMtR8fu0EcHDHXaxWpasVZ5ztYFou3lqPtRyGeRHjwCNrYC9bVBf/3xFiSgeWZth1Chw0fgSXFiSM6BLJWcWAS17AXtjdGKLpq56serDK4CUkuDhNmugfSj2UzvOFpxGo/WPYaJENCioahaP/IvlK43ZSNcPcLWBwK4ethbAE2eloqaD4m36SEB+ui2o2SI2rA/4vQ8EJaJBRSWieBTcxSKCqR1nU+lh5/Vob21C1+FDuGDspIFfMxoEH9B8SPw+88KQhybkm5CWpOp126RYarY4sa+mEwYfwwxATESMAYNcRTzfUIko3njdcJmbcKihCztbFFGZi2dWZaO2w4G66qNRCDBK2o6LW/YodYCpKOSh/BQtZpSkIiPMheCiye0VcKi+C9908mLy8boBt3XQ4zjfUCKKN9ZG2JwetLiVONYli0pjbUpOCTgArLMWFv8aP5Kr2yPeZo2Nqwm5gXFLXW4Gn8o/856qZzEXP78BRNTVAKvLB6syPeKBjGdTpRUiSS1HkrsFJ5o6o3LNAbG1itM6OA7IDa0q1rbbcbzZCqfHJ0loGgUPlUIGxgC73J+IbJSIYo0SUbyx1MPq8sKqyoh4IGM3ahOMplRwTEBd9fFznx9rgdJQ2vBu6/3sqe7Af76sxzcN0uzQwnEckrViqcgsSxYP2polieV8QokozghmfyJSpiPTGKVeI45DSq44fcLVdBx2t4TTFlwWoOEr8fvcKSEPMcZQbxa3es4xaQY7sqBAIurg/InIEodDH4YYSkTxxGWFw9IOnwC4tZlI00Wv+1qXPQJJKjmMjhocbZKw8bXmc0DwisvBmkKXA262uODyCFDKZUiP8RbTfUnWiuOXWjiTeMDWKvbykZihRBRPuurR5fDAoTAiK8UAmSyKXcYpw5CmV0Hv6wSTankLRydQv1/8vmhWty7xb1vE7bQLUrTRfe/9FJjfZpfpxfFNghew01SPWKJxRPHEXAPGAKc2B7mmKI+hUWiQnleCjKRT4HXtAIqie/1wnHhf/KNOLuxxAbSqNjERFafpBjmwUMVpOvzk0hJxm+u9GYD5lDi2Kyld0riGMioRxZPOGuSYNJg3axomFpiifnl5Wil4GQe0Hov6tc+p7YQ4r4yTAaXf7VYaMts9aDQ7wXHSJyIFLxOTEHDGJgRN0gV0HqBEFC88zuBi+VxyIRR8DH406aMAAKz9W9Q2tAT3DYs5tx04/Ib4fe7kHksWNe12AGK1TBfmtkmDQu9f67vrlLRxDHFx9BM/z5lPwevzgdelglPpY/Ma2hTAkIPqk8fw2Wc7kDF6Fi4fHYVpJH0RBODw64DbBujSgGGzezxtbJ4ReckaeAQhtvGE6UijBV/WdmKYXocpgFgi8rpD5sSR6KESUbzorMaJFhveq1fi25YY9mpljkGyVokM6xF8fcqMFksMV25kDDj+nlgtk8mB0deJM+17kaxTSjKtoycurw91nQ7U2uWASi+uqW2plzqsIYsSUZwQ2k7A7PCgUZYV0UL5Ycu8AMYkDfKVVuidp/DB4SYIsVigXhCAo+8AdXvF9qDR1wL67qUvu9uLpq44WxUACCbEJosbzL/zLMxUPYsVSkTxwNEBS1sjvALgNhQiM9IVGcOh0ABZ41GYqkW+7QDqO5349ESUpzC4rMCBl4H6fWISGnElkDGq22ken4DXv2rA33fVYH9tZ3RjGKB0vQoKnoPD7YNZ6W+w7qyRNqghjNqI4kHbCXQ63LCospCXkRr7VQnzp0JVvw8TdR2odZzC7ipxK50J+aaIL+nxCbA43BDq9kFR+zF4rxO8XA712GvBnbXMhyAwnOpwYMexFrRYXFDKZchLlm4kdU94GYdsowY17XacQiZMgFgi8rrE9bVJVFEiigOs9SjabW506AowcTC6rjXJQO5kpJ3ajTm2fXhdyMDHx1pQkq4LbqkTrm1HmlHd0AJN2yFkWg5C5RPbt+yKFBxLvRRL0i9AYKHbHUdbcLLVBqvLC7dXbJTWKHlcOz4HaRKOpO5NbrKYiKodaozRpoiDGttP9li6IwNDiUhqLgtsTd/C6RFgSSoevDE0RbOAlsMoRhfmCV9AMeaGYBJyuH1weHwwahTgZRw8PgEddjeau1xoNDtx+egMcK4uoPU4Uo/thLrpJAAmjlFSadGcMgVNSaMhg0w85tducwe3CVIpZBidZcC04pT46q4/Q6CUdqrDDpZTAs7eLq6jRIko6uLzN+B80nQIrRYnLKpM5ObkQikfpGY7hRq48Pvg9m/BcDQA9W8A+nmAxoQjTRZsO9wMjgN4joPXJ0Dps0HvboLBWQ9bqxNJPnF2/DC5F75sPTSpuVDkTxarYb30jM0sTcPkwmToVHKYNApJp3GEI9uogV4tR45JA7dpGFSndou7jgi+4I4jJDooEUmJMaDpADL0KjRnTUB6tmFwX9+YC4z5AXBwK9BRBez8P8BUAKVZiSKzAzKvEyqfDVp3GzRwQqvkYVAroHCpAIUcMOQgKW0kkD5CrO6dg6RrUEeAl3FYNLNYTJiCAKiSxIb41mNUKooySkRS6qwBrC3QqtWYNn2W2KM12FJLgMkLgWPvicmooxoXABidyuDxMTDGIOPUkMt14HTpgKlQnCtmzBdLVUNcsNQmkwGZY8TVAxq/okQUZZSIpFS7S7zNGitNEgrQpQETbhNXIjTXAI5OcIIPSoUaUCYBSRmALr3PwYhDXXOXE1zSSKTjc6D9W3FpEF2a1GENGZSIpGI+hbaag2i1eZAxdgLOXbEZBLpU8YuE2F/biW2Hm1GQosUP0keIk3erPwUuuE7q0IYMGtAoBUGA98h7qGmz4zArwmHz+VvSSATFaTpwnDgxt944UTzYdJBGWkcRJSIJsKodqK46DpvAozPzO5hUaJI6JNIHo0aBsbni2trvn5LBmzlWfODwm/G3aWWCiotE9Mc//hFFRUVQq9WYPn06du3aJXVIMcPq9qH+yw/QbHHh2+SLcdn4ktNr35C4dVFJGrRKHq1WN95zXgBBmSRuM3ToNcAXJ1s0JTDJE9H/+3//DytWrMBDDz2EvXv3Yvz48bjyyivR3DzEdk7wOOE69Da+/eQVsYivH4cLJ86QZDdT0n8aJY+rx2ZDxnE43ObFW5gJpyATR1rv3UwL7A8Qx6KxlegATJ8+HVOnTsUf/vAHAIAgCMjPz8dPf/pTrFq1qs/ndnV1wWg0wmw2w2AY5DE45+J1w23vAmdrgqKrBmj+BqdaOlDb4UC9cQIKJ8/DlGJqGE40J1qseOtAA7wCw52jeaSd/A/gcaDL6YHHUAAubThYUjagMUGm1EHGi6PLk84YPe7xCeAAyDgOHIfYzy2UULh/o5L2mrndbuzZswerV68OHpPJZJg7dy4+++yzbue7XC64XKfXz+nqCn/vq7pOBz74pglnZt1ACmaMYVpxKi7IET+o9q/eweGD+4Azz2YCODCAAQUpGuSa1ABjsLo8+LquU7yY/4Iy5gUEL3wCQ2GKNrg1TmpGLr4wXohpEydTSShBlaQn4fbphTjVYUdarglIWwyceB9Vez+Drf5rAF+fcTYHn0wOuVKN75RkAOAAjsNX1R0wO71gEBMQx8nAQVxFVyHjMLP09LCAg3VdMDs94uP+hHUi9zr4eDV4GYc7phcGz/3gcBPqOhy9xn7ndwqD19hxtAXV/jXCe3LL1ILgKP9PT7TiREvouTdOyoNGGb0mBUkTUWtrK3w+HzIzQ9epyczMxOHDh7udv27dOqxduzai1/J4BbRa3b0+7vKe3i6GOc3g7K29X8zpFr/EJ4K5Qn9IgSv5OAWsqnQgdwSQNgIaUyGu57gh/R/wfJCiUwa3poYqCbjgetTbR0JoOgStrQ4qdzvkXhsYY+CZBwqfT9zBxE/pNkPj6XlvOQXPAdYz/gHazOCc4rmBo+1WF7y8DPKzpshYnN4+f8fPZHX1fS4745+w1elF61kL6DFEtyKVUOOIVq9ejRUrVgTvd3V1IT8/P6znZhrU+MGkvLPXbA/eN2lPLwGaNOJilGSOE/+BQQbGcQA4cDLxVqeSA0oFwHFQCQzDHV4gmGA4QMaDV+mg0Wig5GUhC8VTChqarpoyCsAZo619XsDrBHxu8UvwARBLzaPG+SD4S9CCIEBggAAGFlglV336zzJnuBvpPgYBCC5gl6fP63Gu20UlaZhUEN6ItGnFKcGewJ4oZKebjycXJmNUVmi1KtodLJImorS0NPA8j6am0B0SmpqakJWV1e18lUoFlSqy+UoaJY+C1PCqQypTNjJM2WGdqwCQ1vvPk5yveDnAJ/X4UH9GjSWnhH9uf+by9WfZldQkFVJ7fitRI2mvmVKpxOTJk/H+++8HjwmCgPfffx8zZsyQMDJCyGCSvGq2YsUKlJWVYcqUKZg2bRqefvpp2Gw2LFy4UOrQCCGDRPJEdMstt6ClpQVr1qxBY2MjJkyYgLfffrtbAzYhZOiSfBzRQMT1OCJCSNh/o5KPrCaEEEpEhBDJUSIihEhO8sbqgQg0b/VnqgchZPAE/jbP1RSd0InIYrEAQNijqwkh0rBYLDAaex/5m9C9ZoIgoL6+Hnq9HhaLBfn5+aitrT1vetACU1zoPQ9dif5+GWOwWCzIycmBTNZ7S1BCl4hkMhny8vIAnJ6ZbDAYEvIHNhD0noe+RH6/fZWEAqixmhAiOUpEhBDJDZlEpFKp8NBDD0U8Oz8R0Xse+s6X95vQjdWEkKFhyJSICCGJixIRIURylIgIIZKjREQIkVxCJ6L29nbccccdMBgMMJlM+OEPfwir1drnc+bMmQPOv9B94OtHP/rRIEXcf/3dBffll1/GqFGjoFarMXbsWLz55puDFGn09Oc9V1ZWdvt5qtXqQYx2YHbs2IFrr70WOTk54DgOr7322jmfs337dkyaNAkqlQqlpaWorKyMeZwxxxLYVVddxcaPH88+//xz9tFHH7HS0lJ222239fmc2bNnsyVLlrCGhobgl9lsHqSI++ell15iSqWSPffcc+zgwYNsyZIlzGQysaamph7P/+STTxjP8+y3v/0tO3ToEPuf//kfplAo2IEDBwY58sj19z1XVFQwg8EQ8vNsbGwc5Kgj9+abb7Jf/vKXbOvWrQwAe/XVV/s8/9tvv2VarZatWLGCHTp0iD3zzDOM53n29ttvD07AMZKwiejQoUMMANu9e3fw2FtvvcU4jmN1dXW9Pm/27Nns3nvvHYQIB27atGns7rvvDt73+XwsJyeHrVu3rsfzb775ZnbNNdeEHJs+fTpbtmxZTOOMpv6+54qKCmY0GgcputgKJxH94he/YBdeeGHIsVtuuYVdeeWVMYws9hK2avbZZ5/BZDJhypQpwWNz586FTCbDzp07+3zuli1bkJaWhjFjxmD16tWw2+2xDrffArvgzp07N3isr11wAfEzOfN8ALjyyit7PT/eRPKeAcBqtaKwsBD5+fm4/vrrcfDgwcEIVxKJ/jPuTcJOem1sbERGRkbIMblcjpSUFDQ2Nvb6vNtvvx2FhYXIycnBV199hQceeABHjhzB1q1bYx1yv/R3F1xA/Ex6Or+vzyOeRPKeR44cieeeew7jxo2D2WzGk08+iYsuuggHDx4MTogeSnr7GXd1dcHhcECj0UgU2cDEXSJatWoVnnjiiT7P+eabbyK+/tKlS4Pfjx07FtnZ2bj88stx4sQJlJSURHxdIo0ZM2aE7IF30UUXYfTo0Xj22Wfx6KOPShgZ6Y+4S0T3338/ysvL+zxn2LBhyMrKQnNzc8hxr9eL9vb2HneJ7c306dMBAMePH4+rRNTfXXABICsrq1/nx5tI3vPZFAoFJk6ciOPHj8ciRMn19jM2GAwJWxoC4rD7Pj09HaNGjerzS6lUYsaMGejs7MSePXuCz/3ggw8gCEIwuYRj//79AIDs7PC2mB4skeyCO2PGjJDzAeC9995LmF1zo7Hzr8/nw4EDB+Lu5xktif4z7pXUreUDcdVVV7GJEyeynTt3so8//pgNHz48pPv+1KlTbOTIkWznzp2MMcaOHz/OHnnkEfbFF1+wkydPsn/9619s2LBh7JJLLpHqLfTppZdeYiqVilVWVrJDhw6xpUuXMpPJFOyevuuuu9iqVauC53/yySdMLpezJ598kn3zzTfsoYceSsju+/6857Vr17J33nmHnThxgu3Zs4fdeuutTK1Ws4MHD0r1FvrFYrGwffv2sX379jEAbP369Wzfvn2surqaMcbYqlWr2F133RU8P9B9//Of/5x988037I9//CN130utra2N3XbbbSwpKYkZDAa2cOFCZrFYgo+fPHmSAWDbtm1jjDFWU1PDLrnkEpaSksJUKhUrLS1lP//5z+N2HBFjjD3zzDOsoKCAKZVKNm3aNPb5558HH5s9ezYrKysLOf8f//gHGzFiBFMqlezCCy9kb7zxxiBHPHD9ec/33Xdf8NzMzEx29dVXs71790oQdWS2bdvGAHT7CrzHsrIyNnv27G7PmTBhAlMqlWzYsGGsoqJi0OOONloGhBAiubhrIyKEnH8oERFCJEeJiBAiOUpEhBDJUSIihEiOEhEhRHKUiAghkqNERAiRHCUiklAqKythMpmC9x9++GFMmDAheL+8vBzz588f9LjIwFAiIj0qLy/vdT3vu+++GxzHhaySEIsEUFRUhKeffjrk2C233IKjR4/2+pwNGzaErOE8Z84c3HfffVGNi0QfJSLSq/z8fLz00ktwOBzBY06nEy+++CIKCgokiUmj0XRbEO9MRqMxpMREEgMlItKrSZMmIT8/P2T1yq1bt6KgoAATJ04c0LV7KqnMnz8/WMqaM2cOqqur8bOf/Sy4OwfQvWp2tjNLZuXl5fjwww+xYcOG4DVOnjyJ0tJSPPnkkyHP279/PziOG7LrGMU7SkSkT4sWLUJFRUXw/nPPPYeFCxfG/HW3bt2KvLw8PPLII2hoaEBDQ0O/r7FhwwbMmDEDS5YsCV6joKCg23sCgIqKClxyySUoLS2N1lsg/UCJiPTpzjvvxMcff4zq6mpUV1fjk08+wZ133hnz101JSQHP89Dr9cjKyopolUmj0QilUgmtVhu8Bs/zKC8vx5EjR4L7pXk8Hrz44otYtGhRtN8GCVPcLRVL4kt6ejquueYaVFZWgjGGa665BmlpaVKHNSA5OTm45ppr8Nxzz2HatGn4z3/+A5fLhZtuuknq0M5bVCIi57Ro0SJUVlZi06ZNUSs1yGQynL0Ulsfjicq1w7F48eJgQ3xFRQVuueUWaLXaQXt9EooSETmnq666Cm63Gx6PB1deeWVUrpmenh7S7uPz+fD111+HnKNUKuHz+Qb0Or1d4+qrr4ZOp8PGjRvx9ttvU7VMYlQ1I+fE83xwCyee53s9z2w2BzcjCEhNTUV+fn63cy+77DKsWLECb7zxBkpKSrB+/Xp0dnaGnFNUVIQdO3bg1ltvhUqliqhKWFRUhJ07d6KqqgpJSUlISUmBTCYLthWtXr0aw4cPT/zF5xMclYhIWAwGAwwGQ5/nbN++HRMnTgz5Wrt2bY/nLlq0CGVlZViwYAFmz56NYcOG4dJLLw0555FHHkFVVRVKSkqQnp4eUdwrV64Ez/O44IILkJ6ejpqamuBjP/zhD+F2uwelF5D0jdasJuetjz76CJdffjlqa2u77Z5KBhclInLecblcaGlpQVlZGbKysrBlyxapQzrvUdWMnHf+/ve/o7CwEJ2dnfjtb38rdTgEVCIihMQBKhERQiRHiYgQIjlKRIQQyVEiIoRIjhIRIURylIgIIZKjREQIkRwlIkKI5P4/UdQRknsizpkAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T12:10:05.641834Z", + "iopub.status.busy": "2024-03-23T12:10:05.641223Z", + "iopub.status.idle": "2024-03-23T12:10:05.974121Z", + "shell.execute_reply": "2024-03-23T12:10:05.973046Z" + }, + "papermill": { + "duration": 0.353771, + "end_time": "2024-03-23T12:10:05.976155", + "exception": false, + "start_time": "2024-03-23T12:10:05.622384", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBn0lEQVR4nO3deXxU9b34/9eZfSbJzGTfIeyLrIJQ1AourYqXurXa2iqIS22xj1ouVeiCldairfXKVb/WawvoT61dRHtbF+pVlIqKyqIgm0RIAtlJMpnJ7HPO74+TGQhJIJnM5MyEz/PxmMdkzpw5854s73z2j6QoioIgCIKGdFoHIAiCIBKRIAiaE4lIEATNiUQkCILmRCISBEFzIhEJgqA5kYgEQdCcSESCIGjOoHUAAyHLMrW1tWRlZSFJktbhCIJwEkVRcLvdlJSUoNP1Xu5J60RUW1tLeXm51mEIgnAaNTU1lJWV9fp8WieirKwsQP2Qdrtd42gEQThZe3s75eXlsb/V3qR1IopWx+x2u0hEgpDCTtd0IhqrBUHQnEhEgiBoTiQiQRA0l9ZtRMLAKYpCOBwmEoloHYqQhvR6PQaDYcDDZ0QiOoMFg0Hq6urwer1ahyKkMZvNRnFxMSaTKe5riER0hpJlmUOHDqHX6ykpKcFkMolBoUK/KIpCMBikqamJQ4cOMWbMmFMOWjwVkYjOUMFgEFmWKS8vx2azaR1OQvlDEbzBCDaTHotRr3U4Q5rVasVoNFJVVUUwGMRiscR1HZGIznDx/gdLVRFZweULAdDukzHqdeh1oqSXTIn4HRpav4XCGS8Q7troHpZljSIR+kOUiIQhJRhWE4/VqCfLMvDeHGFwiBKRMKSEIuruWBajrl9J6PDhw0iSxM6dOwf0/n29zpYtW5g8eTJGo5GrrrpqQO85FIhEJAwZEVlB7tymz6Af2K/222+/jSRJtLW1JSCy7pYuXcq0adM4dOgQ69evT8p7pBORiIS0FwwGAZCALIuBTLMBnSTh9oc45gkQDKfeYM3KykouuugiysrKcDqdcV0j+rkHQ3Tga7KIRCR0EwzLvd7CEbnP54b6eG5/zZs3jzvvvJO77rqLvLw8Lr30Unbv3s0VV8ynIMfJyGGl3HjjjTQ0NhGWFSKywuuvv87555+P0+kkNzeX//iP/6CysrLH6x8+fJgLL7wQgOzsbCRJYtGiRQB9vs6+ffs499xzsVgsTJo0iXfeeSd2bUmSOHbsGIsXL0aSpFiJ6J133mHWrFmYzWaKi4tZvnx5lz/+nj53tOS2ceNGpk+fjtVq5aKLLqKxsZHXXnuNCRMmYLfbueGGG7oMXJVlmdWrVzNixAisVitTp07lb3/7W+z56HVfe+01ZsyYgdls5t133+33z6qvRGO10M3jmw72+tyIvAyuml4ae/w/mytj7TInK8u28o2ZxxeuW7vlEL5g99LJj74ytt8xPv3003zve99jy5YttLW1cdFFF3HrrbfyX//1X/h8Pu655x4W3/Rt/vq/rxGWFTo6Oli6dClTpkzB4/GwcuVKrr76anbu3Nmt+7m8vJwXX3yRa6+9lv3792O327FarQB9vs6Pf/xjHnnkESZOnMjDDz/MggULOHToEOXl5dTV1TFu3DhWrVrF9ddfj8Ph4OjRo8yfP59FixbxzDPPsG/fPm677TYsFgu/+MUvevzcAHV1dQD84he/4LHHHsNms3Hddddx3XXXYTabef755/F4PFx99dU8+uij3HPPPQCsXr2aZ599lt///veMGTOGzZs3853vfIf8/Hzmzp0be7/ly5fz0EMPMXLkSLKzs/v9c+orkYiEtDRmzBh+85vfAPCrX/2K6dOn84tVvwTU9qG1a9dSXl5O5cHPmTh+HNdee22X169du5b8/Hz27NnDpEmTujyn1+vJyckBoKCgoEvVqa/XufPOO2PnPvHEE7z++uv88Y9/5O6776aoqAhJknA4HBQVFQHw//7f/6O8vJzHHnsMSZIYP348tbW13HPPPaxcuTKW5E783HA8Ef3qV7/ivPPOA+CWW25hxYoVVFZWMnLkSAC+/vWvs2nTJu655x4CgQC//vWv+b//+z/mzJkDwMiRI3n33Xd58sknuySiVatW8ZWvfKVvP5QBEIlI6GbJhaN7fe7ksYG3XzCq13NP7rRafN6IgYTVxYwZM2Jff/LJJ2zatIkcpwMFta0o6vChLxg3diyff/45K1euZOvWrTQ3NyN3ji+qrq7ulohOpa/Xif6BAxgMBmbOnMnevXt7ve7evXuZM2dOl56+8847D4/Hw5EjRxg2bFi3z32iKVOmxL4uLCzEZrPFklD02IcffgjAwYMH8Xq93RJMMBhk+vTpXY7NnDmz15gTSSQioRuToe9Nh8k693QyMjJiX3s8HhYsWMDdP1uFAjhtRgx6HaGIjNmeS0RRWLBgAcOHD+epp56ipKQEWZaZNGlSvxt8E3WdeJ34uU9kNBpjX0uS1OVx9Fg0aXo8HgBeeeUVSktLu5xnNpv79H6JJhKRkPbOPvtsXnzxRcqGD8dgMFCQZe78w1No8gQ4duwY+/fv56mnnuLLX/4ywGkbXqMzyU9cHqU/1/nggw+44IILAAiHw2zbto0777yz1/ebMGECL774IoqixEpFW7ZsISsr65SLzsdj4sSJmM1mqquru1TDtCR6zYS0t2TJElpaWrhj8U18sv1jvvjiCzZu3MgttyxGkSPk5uSQm5vL//zP/3Dw4EHeeustli5desprDh8+HEmS+Oc//0lTUxMej4fs7Ow+X+fxxx/npZdeYt++fSxZsoTW1lYWL17c6/t9//vfp6amhh/84Afs27ePv//979x7770sXbo04fMBs7KyWLZsGT/60Y94+umnqaysZPv27Tz66KM8/fTTCX2vvhKJSEh7JSUlbHp7M3IkwnVXL2Dy5MncddddOJ1OCh02CuxWXnjhBbZt28akSZP40Y9+xG9/+9tTXrO0tJT77ruP5cuXU1hYyJ133olOp+vzdR544AEeeOABpk6dyrvvvsv//u//kpeXd8r3e/XVV/nwww+ZOnUqd9xxB7fccgs/+9nPBvS96c0vf/lLfv7zn7N69WomTJjAZZddxiuvvMKIEYlrx+sPSVGUnvte00B7ezsOhwOXyyV28egnv9/PoUOHGDFiRNxLN6QSXzBMuz+MyaAj2xb/Al1C/53qd6mvf6OiRCQMCXLnv1OdmOSalkRjtTAkmAw6siRDt7WHfMEw3mAEs0FHpsXYy6sFrYlEJAwJRr0OYw8TXRUgLCvo5bRtgTgjiKqZMKRFq2oiD6U2USIShgR/KIJOUqd3nNhOFK2pyenbJ3NGECUiIe0pirpOdas3hHxS0ed4iUgkolSWMonogQceQJIk7rrrLq1DEdLMiTlGd1JjdTQRKYqasITUlBKJ6KOPPuLJJ5/sMnFPEPoqWtqR6N59f+JD0U6UujRPRB6Ph29/+9s89dRTSV3vRBi6YmOIetg2SJIk9Dr1JkpEqUvzRLRkyRKuuOIKLrnkktOeGwgEaG9v73ITzjzz5s3rUoWPloh6274sL9NMXqZ5wOtY9zUeof807TV74YUX2L59Ox999FGfzl+9ejX33XdfkqMS0o0SS0R9H1UdDAYHtFe7kFialYhqamr44Q9/yHPPPdfnuU4rVqzA5XLFbjU1NUmOUkg1ixYt4p133mHNmjVIkoQkSRw8WMmPltzB1InjsFqtjBs3jjVr1nR73VVXXcX9999PSUkJ48aNA+C9995j2rRpWCwWZs6cycsvv9xtO6Ddu3dz+eWXk5mZSWFhITfeeCPNzc29xnP48OHB+nYMGZqViLZt20ZjYyNnn3127FgkEmHz5s089thjBAIB9Pqu+5abzeZuCzcJCaQoEAlp8956Y/clHXuwZs0aDhw4wKRJk1i1ahUAmXYHFcPLuXPJCxQXFvDee+9x++23U1xczHXXXYc3GCYQivDmm29it9t54403AHVC5oIFC5g/fz7PP/88VVVV3apYva2Hfd111/HWW2/1GE9+fn5ivzdnAM0S0cUXX8yuXbu6HLv55psZP34899xzT7ckJAyCSAj+/Ttt3vvL/wmG01eVHA4HJpMJm80WW+8Z4Ne/+mXs6xEjRvD+++/zl7/8heuuuw5ZBhmw2TL4wx/+EKuS/f73v0eSJJ566iksFgsTJ07k6NGj3HbbbbFrPfbYY0yfPp1f//rXsWPR9bAPHDjA2LFje4xH6B/NElFWVla3tYIzMjLIzc3t1xrCggDqQmRr166luroan89HMBhk2rRpAETXFZt41lld2oX279/PlClTujQNzJo1q8t1o+thZ2ZmdnvPyspKxo7t/w4kQndiiodwnN6olky0eu84/X/PPc+yZct46KGHOPfcc8nKyuK3v/0tW7duBY43YlvjWH85uh72gw8+2O254uLiuGMWukqpRPT2229rHcKZTZL6VD3Smslk6rKW9Dv/fpeZs77Ebd+9A5NBrdKfuOlhrFv/pGFE48aN49lnnyUQCMTaHk/uwY2uh11RUYHB0POfy8nxCP2n+TgiQeiviooKtm7dyuHDh2lubmbEyFF8snM7//evf3HgwAF+/vOfd0ko0cXolZMy0Q033IAsy9x+++3s3buXjRs38tBDD3V5TXQ97G9961t89NFHVFZWsnHjRm6++eZY8jk5nuhuGULfiUQkpJ1ly5ah1+uZOHEi+fn5zLv4K8xfcCU33PAtZs+ezbFjx/j+978fO//EgY4njq622+384x//YOfOnUybNo2f/vSnrFy5EiDWblRSUsKWLVuIRCJ89atf7bIednRR+5Pjqa6uHoTvwtAi1qw+Qw2VNaujWwYBsW2Eup2jKDR7AugkidwMU4/nRD333HPcfPPNuFyu2DbTwqklYs3qlGojEoT+ik14leg1wegkiYKsnpPtM888w8iRIyktLeWTTz6JjRESSWhwiUQkpLWBLppfX1/PypUrqa+vp7i4mG984xvcf//9CYxQ6AuRiIS0droJr6dz9913c/fddycwIiEeIhEJac2gl8iyGE5bInL7QwTDMplmA2ajGLWfakQiEtKaQafDYDp9529EVgjLilgyNkWJ7vszXBp3mvaLJHbzSJpE/A6JRHSGMhrVKRVer1fjSAYmFJEJhCNETjOIUOzmkTzR36Ho71Q8RNXsDKXX63E6nTQ2NgJgs9lOOb4mVbl9QQIRBZtJj83U+69zMBgmFIwQkHUYEdMxEkFRFLxeL42NjTidzgGtmCES0RksumxFNBmlI28wTDiiYDHqMRl6L+AHwzL+UASjXofVJBqrE8npdA54CRSRiM5gkiRRXFxMQUEBoZBGC6IN0Ku7amlsD3Dh+AKG5/Y+u76y0cOOz5socVr56jixblCiGI3GhKwdJhKRgF6vT9uF6DxhHT5Zj81qPeVUlQxbRF1ZwGBK6yktQ5VIREJaC4TURmqz8dT9LhV5Gdwxd9RghCTEQSQiIa0FwmrDs9lwQolOUeDgm1D/KeSPh7GXHV+mUUhJ4qcjpK1wRCYUUbvjzSc2VDfshiMfQTgAdZ9A3Q6NIhT6SpSIhLR2yYRC/OHI8USkKFD9gfq1yQZBL9R8iFI8nRd31BIIR7j27DIsYppHShElIiFtGfQ6Jpc5OKci5/gYKE8DdDSD3gAzF6sN1L42JHcdDe1+GtsD+ENiHFGqEYlIGFqOHVTvs0eAOUu9B2g9FCs1BcNiKddUIxKRkLY6AmGqj3k51rlCIwAtX6j3uaPV+5zORNRyPBEFRCJKOSIRCWnrSKuPF7cf4a19nSPDI2FwN6hfO4ep947Oe089Zr1afYv2tAmpQyQiIW1FE0qs4dnTAHIYjFawZqvHbDlqO1EkTKbsAsAfEiWiVCMSkZC2olWsWI9Z+1H13lGmLmIN6n1mIQD2UHOX1wmpQyQiIW1Fe79iKy6669X7rJN2YM1S55ZlhY+JbvsUJcYRCWkrNr0jWiLq6GwryizoemJniWhqdoip08Q0j1QkSkRC2upSNZMj4G1Rn8jI63qiLVe99x4bxOiE/hCJSEhbXeaZeVvUZKQ3gsXZ9URrTucLPBDyD26QQp+IRCSkrSllTs4fk0eh3QwdTerBjPzjDdVRRguYM2n3h/jn1s/4vz0Ngx+scEqijUhIW6MLMo8/aOxMRCe3D0XZcpFbW2htrkMy5PV8jqAZUSIShgZfZ/tQtD3oZNYc9HoJa8gluu9TkEhEQlpSFIXqY14a2v3IsgK+VvWJ6EDGk1mdGHQ6zBG3GFmdgkQiEtJSWFZ4cfsRnt9aTTgiH+8x6y0RWRzodRLmsJtgWD5j9nNLFyIRCWkpOphRJ0kYZS9EQmoj9ck9ZlEWZywRKQoEI6J6lkpEIhLSUmwMkVGH5GtTD5rt6jpEPbE40Elgkb1ISli0E6UYTRPRE088wZQpU7Db7djtdubMmcNrr72mZUhCmugymPF07UMARiuS3oRBp8OBl5BIRClF0+77srIyHnjgAcaMGYOiKDz99NNceeWV7Nixg7POOkvL0IQUFwidMJjRd5r2IVCrbVYnZw8PMmNKNmSaByFKoa80TUQLFizo8vj+++/niSee4IMPPhCJSDilHktEtpxTv8jiRPI0gd+V5OiE/kqZAY2RSIS//vWvdHR0MGfOnB7PCQQCBALHV+Nrb28frPCEFBNNRBajvm9VMwCLQ70XiSjlaN5YvWvXLjIzMzGbzdxxxx289NJLTJw4scdzV69ejcPhiN3Ky8sHOVohVZQ4LJw/Jo9xhRl9T0TmLGpdPt7bW8WBBnfygxT6TPNENG7cOHbu3MnWrVv53ve+x8KFC9mzZ0+P565YsQKXyxW71dTUDHK0QqoosFs4pyKH0dkGCAfVg7113UeZMvEFI7hdLbR2BJMeo9B3mlfNTCYTo0erC53PmDGDjz76iDVr1vDkk092O9dsNmM2i0ZG4QTRapYpo/eu+yhzFgadhCncIbrvU4zmiehksix3aQcShJ40uQPIioKzoxUzHG//ORVzFnqdhDHixSf2NkspmiaiFStWcPnllzNs2DDcbjfPP/88b7/9Nhs3btQyLCENvHOgiZoWL1cXNFIBYLGf/kXmLAx6HXolRCjoS3KEQn9omogaGxu56aabqKurw+FwMGXKFDZu3MhXvvIVLcMS0kBsB4+wRz1g7kMi0hvRmaxAB4pf9LimEk0T0R//+Ect315IY9H1qk3hzt6v0zVUd5IsWUAzsl/0mqUSzXvNBCEesQGNkQ71QF+qZoDOYkcCjOGOJEUmxCPlGqsF4XQURYlVzYyhzipWX6pmQE52DrNH5iCNyEhWeEIcRIlISDvBiIyigE4OYYh0Lobfl14zQDLbkZAgIKpmqUQkIiHtRLeMtiledJKkbilt6OP4MnOWei8SUUqJKxF98cUXiY5DEPrMbNBx/pg8zinq/PU127vv3NHri7M40OBm++c1+IJiLFGqiCsRjR49mgsvvJBnn30Wv1/sEyUMLotRzzkVOUwr6Pz17WO1DABTJi5fCI+7DW8wnJwAhX6LKxFt376dKVOmsHTpUoqKivjud7/Lhx9+mOjYBOHUomOB+pWIMjpHV/sJhkUiShVxJaJp06axZs0aamtrWbt2LXV1dZx//vlMmjSJhx9+mKampkTHKQgxnkCYxnY/Pk/ngmh97DEDwGhDr9cBCkGv6MJPFQNqrDYYDFxzzTX89a9/5cEHH+TgwYMsW7aM8vLy2IhpQUi0Aw1unttazedVR9UDfRxDBIBOB0YbACExqDFlDCgRffzxx3z/+9+nuLiYhx9+mGXLllFZWckbb7xBbW0tV155ZaLiFISY6KhqixwdzNiPqhmAWd0hNuwTiShVxDWg8eGHH2bdunXs37+f+fPn88wzzzB//nx0OjWvjRgxgvXr11NRUZHIWAUB6JxnpshYIl7A0r+qGSCZ1MGMYb8nCdEJ8YgrET3xxBMsXryYRYsWUVxc3OM5BQUFYi6ZkBT+kIwp4kUvKSDpwJTZr9frzJlIAEGRiFJFXInojTfeYNiwYbESUJSiKNTU1DBs2DBMJhMLFy5MSJCCcKJAOII54sGgk9T2IV3/WhgmDC9Bp6tCyhMznFJFXG1Eo0aNorm5udvxlpYWRowYMeCgBOFUAmEZc9iNQS/1u1oGoDdnqtM8RIkoZcSViHrbN9zj8WCxWAYUkCCcjpqIPBh0uv71mEV1thERFN33qaJfZdOlS5cCIEkSK1euxGazxZ6LRCJs3bqVadOmJTRAQTjZ1DIHep+EOazrf48Z0Bw0cKzBjeSpZ+z0JAQo9Fu/EtGOHTsAtUS0a9cuTCZT7DmTycTUqVNZtmxZYiMUhJNMKXPCMQVa9HFVzQI6Ky0dQcyRtoTHJsSnX4lo06ZNANx8882sWbMGuz2OYrEgJEIgOr2j/7+DRqs6A18JBSASAr0xkZEJcYir22DdunWJjkMQ+iQckTnmCeBwt2LR0eclYk9kNtuQJT0ROaK2E1n7fw0hsfqciK655hrWr1+P3W7nmmuuOeW5GzZsGHBggtATtz/MX97/nNl1TcyqyDm+vlA/mE16QnorurCHsN+NQSQizfU5ETkcDqTONV8cjv43EApCIgTC8vExRCZbXNUqk15HSGfDjIeQz43hNDtVC8nX50R0YnVMVM0ErQTCEUxhD3qdFFePGYBOJyGbMiAIIZ8Ha4JjFPovrnFEPp8Pr9cbe1xVVcUjjzzCv/71r4QFJgg9UUtEbrVEFEePWYwpAwkIixn4KSGuRHTllVfyzDPPANDW1sasWbP43e9+x5VXXskTTzyR0AAF4UT+UEQdzKiPczBjp4smj2T2yBxyDMEERifEK+4VGr/85S8D8Le//Y2ioiKqqqp45pln+O///u+EBigIJ4q2EalVM2fc1zFaMzqneYjR1akgrkTk9XrJylJ7K/71r39xzTXXoNPp+NKXvkRVVVVCAxSEEwVCnfPMBlw165yxLxJRSoh78fyXX36ZmpoaNm7cyFe/+lVA3cteDHIUkqks28oou4zDahxQ1Wx/i8z+BjeNx1oSGJ0Qr7gS0cqVK1m2bBkVFRXMnj2bOXPmAGrpaPp0MXlHSJ6KbBMj7BLZNlPcvWYAjUEDLR1B/B0u6GUStzB44hpZ/fWvf53zzz+furo6pk6dGjt+8cUXc/XVVycsOEHoJrpzh94IhvhXejCY1Rn4kXAYIsG+b9AoJEXcK0MVFRVRVFTU5disWbMGHJAgnEpbazOWUASjLRd9XzdV7IHJbMUnGYnIstpOJBKRpuJKRB0dHTzwwAO8+eabNDY2Istyl+fFTrBCsvx710HyatsYmzWc3AFcx2zQ0a63EpZ96gJptpyExSj0X1yJ6NZbb+Wdd97hxhtvpLi4ODb1QxCSrnPW/UDnh5kNOkJ6GxHZC0Hv6V8gJFVciei1117jlVde4bzzzkt0PILQK0VRkDpHQhsznAO6ltmgJ6SzEI4oogs/BcTVa5adnU1OjijKCoMrFFEwhtUS0YATkTFaIlLE2tUpIK5E9Mtf/pKVK1d2mW8mCMkW3b1DAgw254CulZ9pZv7M0Uwf5hQlohQQV9Xsd7/7HZWVlRQWFlJRUYHR2HUphu3bt/fpOqtXr2bDhg3s27cPq9XKueeey4MPPsi4cePiCUsY4gKhMKZwBwa9hDSAMUSgzsA3WbMACULiH6rW4kpEV111VULe/J133mHJkiWcc845hMNhfvKTn/DVr36VPXv2kJGRkZD3EIaOQIcLCQWd3hjXgmjdGKO7eYiqmdbiSkT33ntvQt789ddf7/J4/fr1FBQUsG3bNi644IKEvIcwdGTipdRpRbE4IQE9te/V+MhrcDNMn4HYBEtbcbURgbr8xx/+8AdWrFhBS4s6X2f79u0cPXo07mBcLheAaAgXeuTAy7AcG8NLik5/ch8cbINjHUHCfo+Y5qGxuEpEn376KZdccgkOh4PDhw9z2223kZOTw4YNG6iuro6tVdQfsixz1113cd555zFp0qQezwkEAgQCgdjj9vb2eMIX0lVs547ELFWsM6v78kXCYQgHwCjKRVqJq0S0dOlSFi1axOeff95lZ9f58+ezefPmuAJZsmQJu3fv5oUXXuj1nNWrV+NwOGK38vLyuN5LSE8+dwv+UISQMQHtQ4DJZCGiO2Gah6CZuBLRRx99xHe/+91ux0tLS6mvr+/39e68807++c9/smnTJsrKyno9b8WKFbhcrtitpqam3+8lpK9DR+rYUdPGZwlaucNsVBfRD4uxRJqLq2pmNpt7rBYdOHCA/Pz8Pl9HURR+8IMf8NJLL/H2228zYsSI076v2SwmJ56x/Gobot6WmKqZ2aAjqLcSlttFiUhjcZWIvva1r7Fq1SpCoRAAkiRRXV3NPffcw7XXXtvn6yxZsoRnn32W559/nqysLOrr66mvr8fn88UTljCUKQr42wAwZCRm/x+zQX98dLUYS6SpuBLR7373OzweD/n5+fh8PubOncvo0aPJysri/vvv7/N1nnjiCVwuF/PmzaO4uDh2+/Of/xxPWMJQFvQQCYdRkDANcFR1lNmgI6SziqpZCoirauZwOHjjjTfYsmULn3zyCR6Ph7PPPptLLrmkX9dRRJep0Fe+NsIRmaAhE4vZlJBLzqjIRpJGY6yuE1UzjfU7EcmyzPr169mwYQOHDx9GkiRGjBhBUVGROjtaLAkiJIO/jbCs4DfZsRr1Cbmk2aCH6DQPkYg01a+qmaIofO1rX+PWW2/l6NGjTJ48mbPOOouqqioWLVoklokVkkb2thKWFQKGLCzGuMfhdid280gJ/SoRrV+/ns2bN/Pmm29y4YUXdnnurbfe4qqrruKZZ57hpptuSmiQgqD42ih1WjHkFWMxJKZE5PKG+ORwB+XHOhhhFolIS/361/KnP/2Jn/zkJ92SEMBFF13E8uXLee655xIWnCBE6QMuhuXYOGfCSHS6xFT/gxGZXU0RWjwBtddMtFlqpl+J6NNPP+Wyyy7r9fnLL7+cTz75ZMBBCUI3nV33A9nd9WQWo46Q3kIooqDIYQj7E3ZtoX/6lYhaWlooLCzs9fnCwkJaW1sHHJQgdBEJEfa14w9FCCRoegeAxahHkQyEdObOlRpF9Uwr/UpEkUgEg6H3ZiW9Xk84HB5wUILQha+Nlo4gHx/18uq+toRd1qCTMOgkQnoxlkhr/WqsVhSFRYsW9TrN4sSZ8YKQMNGue4M9YQ3VoM4IsBj1hHRWIhGvKBFpqF+JaOHChac9R/SYCQnnayMcUbvubabEJSKIthPZCMsdYlshDfUrEa1bty5ZcQhC7/xthGUZv8FOTgJLRABmo75z4qssqmYainvLaUEYNL5WtURkSvBgRuA/phRjyBmNsapWVM00JBKRkPq8LYRlBZ/RgTXBVTObySCmeaSAxP57EYREkyPgdxGOyPgMzoQ2VsdEp3mERCLSikhEQmrztYIik+PIZPywYpw24+lf0w81LV7eOeylvt0vSkQaElUzIbV51XVhy0pKKTsrMbt3nKjVG2R3Y4giX4iiYOc0D7GCxKATJSIhtfk6F6i25Sbl8hajXh3QGJFBkcVKjRoRiUhIbd5jyIqCR2cnEI4k/PJWox5F0uOXOgfpirFEmhCJSEht3hY6gmE27PPx7AfVCb+8uXM4gC+616sYS6QJ0UaUrtz1cPhd9esRcyGz77unpBVfC6GIgt/oJCvBXfegVs0AvFhQ8CGJBmtNiESUjvwu2Pm8ujspgOsIzFwMFru2cSVayAdBr9p1b3ZQmIxE1DkcICBZkWUfepGINCGqZumocpOahDLzISNP/YONlo6Gks4eM7/OhqwzJmyt6hMZ9RL62Ax8WYwl0ohIROnG3w5N+9Svxy+AcZerX9fvgoBbu7iSwXsMgA69uqGizZT4ArwkSSw6r4L/OGc0JoNOjCXSiEhE6ab+U3Wsi7McsgrBUQaOUrXruWGP1tElVkcTAB6DuqFioqd3RNktRkyWLCQxzUMzIhGlm8a96n3x1OPHCiep9w27Bz+eZOpMRO06J0DClwDpwpSh3otEpAmRiNKJ3wUdzerI39zRx4/nj1ePeRpj+8MPCZ2JqLiknMmlDnIzErOx4sn217t5+7CXFm9QJCKNiESUTo4dBCCUWYyf43+Un7dGOBzs7DFr+UKLyBIv5IOAOqZn4phRXDKxkAK7JSlvVevysbspgscfVkdWy3JS3kfonUhE6aTlEAB7AwU8+0EVNS1eGt1+/vlpHdvcOfjDkaGTiDpLQ1gcYOh5aeJEsRr1hHQWQrKitr+JaR6DTiSidKEo4DpCRFbY4bbj9ocJhGXyM82U59g4Zi6j3uWH1qqhsT9XZyKSbXm0dgTxhxI/vSPKZtKDpBOjqzUkElG68LZAyEezN0KrLgenzcjIvAwkSWLG8Gw6THk0dMhEQv7jpYl01tEMgMfgZP17h1m35XDS3iraCO7T2dQDQ20YRBoQiShduGoAqJWdKJKeCcX22I6nw3Ns2G1mWg35tHmDsXPTWmcy9RpzgOT2mEWneXQQTUTtSXsvoWciEaWL9qOEZZnDIXVMzeiCzNhTOp3EmIJM3OYiWjqC6pSPdKYo4GkAwN05hijDnLzZSNGBkh6s6gFRIhp0IhGlC9cRXL4QbaZCsm3Gbl3ZowsycZsLafOGkNvSvETka4VwEPQG2nXqqOpMc/JKRNHSVgc2ZEURiUgDYtJrOgj5wNuCyxfCk1XI5M62oRMVOywoWaWYXHqCnjYsATeYE7c986By16n3GQW4A2pXejJLRGaDjsXnj8DWGkC3/6BIRBoQiSgddFZTsnPymVxWxKj8zG6nSJLETV8eg3HbKLWh112fxomoXr3PKqLDr/aWJTMRSZKEw2qEgFr6Eolo8ImqWTrwNAKQXVjOvHEFlOfYejzNqNeBvUR90F47WNEl3omJKBAGIDOJiSgmmrgD7UNjCEQaEYkoHXSWiMgsPP25WUVEFIWIK00TkaKApzMRZRYxtiiLyaUOcpI0vSNq91EXbxzswOULQSSsVoeFQaNpItq8eTMLFiygpKQESZJ4+eWXtQwndbnrafUGqYvY1UXeT2FznYGPD7fQ0lCdnv/Vow3VOgNk5DGt3MklEwvJy0zu6OrqFi+767245c6EJ6png0rTRNTR0cHUqVN5/PHHtQwjtUVC4G2hptXLhgNhDjWfelKmkpFPBB1ud3t6ToCNNlRn5oMuibPtTyIGNWpL08bqyy+/nMsvv1zLEFJfRxOyHKE9bCSozzjtxM+yPDuHjDm0+1rVP2qrc3DiTBTXUfXeXkowLOMJhMkw6zEnY4fXE0TboLxYgQ4xqHGQpVUbUSAQoL29vcttyPM04A1G8BjzsJoN2C2n/t9R6rTSYc7HF4rga0nDgY3tnTE7yqht8/H0e4f5y8fJ/xzRXjk3okSkhbRKRKtXr8bhcMRu5eXlWoeUfO4GPIEwHaZcCu3mbuOHTmYx6jE61Z6ztoY0G9gYDsZ6CLGX4varPWanS76JEC0RidHV2kirRLRixQpcLlfsVlOTZn9o8fA00BEI02HMpTCrb+vxOAuHA9DRXJNeDdbtR9V4LXaw2Gn3hwB1Kddki7YRueTO77FIRIMqrQY0ms1mzObk9p6kFFmGjka1RJSZR6Gjb4mosKiUekmPp8Or9kLZcpIcaIK0d7YPOcrUhz41EWUNQokoWjXzKDYiioI+HRv601haJaIzjq+FSCiEJ6zDb3BQ2McVCktzMmnPKaWAZpT2o0jpkohiDdWdiShaIrImv0RkNuhYfN4IbEo++g/fVxurZRl0aVVpSFuaJiKPx8PBgwdjjw8dOsTOnTvJyclh2LBhGkaWItz16CSYNHY0joqiPo8uthj1TD9rAhz5GNwNUDQ5yYEmgBw5vnxJZ4noeBtR8hORJEk4bEaQHeqwATkCQbe6QqSQdJomoo8//pgLL7ww9njp0qUALFy4kPXr12sUVQrxNKh/IPllOEr7+QeRVaTeR8flpLr2o+qYKZMNMgsIR9SuewC7dRB/TXU6MNvVKq2vTSSiQaJpIpo3bx5KOjWmDrZoD1JfpnacLKsYfziCp76avHSoYrQeVu+dw0GSkBWFOSNz8QTCSdnhtSf7690cPtbBtJCZQkjPAaFpSrQRparOxcFqWrx43FZGhCKxlQT7wm90sO1IBzo5hLWtgYyc4iQGmwDRRJRdAYDJoGP2yNxBDaHO5WNPbTtlBmtnImob1Pc/k6X4v8kzWKCdcKCDIy4/G6si6oJd/WAxGZE6q2fNdVXJiDBxwgFo76xCdiYiLUTb4NxS5zIrokQ0aEQiSlWeRjoCEbyGbDKtlrj2fc/IUxt92xoOJzi4BGutUrfMtjpjU1KaPQGOeQJE5MGrumd2DhNwyZ2jq31tg/beZzqRiFKVp4GOYHREdXwbC2Z3Dmz0Nh9NZGSJd+xz9f6E3Wu3HGzmmfer2H108EolWZ29c63RRCSqZoNGJKJU1Tmi2muMPxEVlFYgAbKnkQ5fILHxJYosx3awPTERtXQEAZK+DtGJolNJjkVsaidKwKOuTSQknUhEqcrTiMevloiK4kxElqxczFYbOiVCfV2KTodpPwpBr7qbq1MdOxaOyOoCZUD2ICaiDJMBvU4iKJkJSJ1jl3ytg/b+ZzKRiFJRyE+4owV/WKbDmEuBPc5pLZKEJacUgJb6FG2wjlXLRsXWH3L5QiiK2nOWkcT9zE6m00lqg7Uk4Tc61YO+lkF7/zOZ6L5PRZ4GOgIRAoZMsrKy+tVtf7KisgoKw3VkOPwJDDBBFAWaDqhf542NHW71qtWybJvptKsNJNp155RjMegwHNgP9c3gPTao73+mEokoFXkacViNnDt1FGeNKhnQpbILK6BpO3jrExNbIrXXqlUfvRFyRsUOH/NE24eSP7XjZLFpNLbOMUwiEQ0KUTVLRZ2L5ZscxeRnDXC1geiuHh1NEEqxUlHDZ+p9/jgwHG8LanSrDesD/uwDIRLRoBIlolQU28UijqkdJzNn4dVn0dLcQPu+PUycfPbAr5kIcgQa96hfF57V5alp5U7yMs29bpuUTI1uPzuq27BHFOaAmogUBQa5inimESWiVBMOEnA1sKeuna1NxoTMxXOZi6lp9XG06kACAkyQYwfVLXtMGeCs6PJUeY6NOaNyKejjQnCJFAzL7KltZ2+bXk0+4SAEPYMex5lGJKJU46mnwx+iKWji83ZdQhprc0pGIQFKWw3uzjV+NHd0m3pfNDmlJuRGxy21BxUi5s6Z96J6lnSp8xsgqNrr8AQieEz5cQ9kPJk5bziZFgOZwSYqG9oScs0B6WhWp3VIEpR2rSrWtHg52OjBH4poEprVqMds1KEo4DV0JqIOkYiSTSSiVOOuxRMI4zEXxD2QsRuLE4czF0mROVp18PTnJ1u0NJQ3ptt6P9uqWvnHJ7XsrdNmhxZJksi2qaUily5bPdjRqEksZxKRiFKM7OpMRKZ8Ch0J6jWSJHJK1ekTgYaDeIMaTlsIuKHuU/Xr0pldnlIUhVqXutVzidM62JHFRBNRq9SZiNwpOPRhiBGJKJUEPPjcLURkCNoKyctIXPd1RvFYMs0GHL5qDjRo2Pha/QHIYXU5WGfX5YAb3QECIRmTQUd+kreYPpVsmzp+qUlyqgc6mtVePiFpRCJKJe21tPtC+IwOinLs6HQJ7DLOGUlelpmsSBuKVstb+Nqgdqf6dcX53brEv2hSt9MelmNL7Gfvp+j8Nq8uSx3fJIfBK6Z6JJMYR5RKXNUoCvhtJZQ6EzyGxmglv2wUBZlH0Ge0ABWJvX5fVL6p/lFnD+9xAbTDx9RENCIvY5AD62pEXgbfv3CUus319gJwHVHHdmXmaxrXUCZKRKmkrZoSp5XLz5/F9GHOhF/ekDcavU6C5s8Tfu3TOlapziuTdDD6K91KQy5viHqXH0nSPhEZ9To1CcEJmxA0aBfQGUAkolQR8scWy5eyh2PUJ+FHkz8eAKXlC2rqmmL7hiVd0Av7XlG/Lp3RY8miusULqNWyjD5umzQosjrX+m4/om0cQ1wK/cTPcK4jhCMR9Bm5SOas5LyHLQfsJVQd+pz3399MwYTzuXhCAqaRnIosw75/QrADMvJg5NweT5tc5qAs20pIlpMbTx/tr3fzSU0bI7MymAlqiSgc7DInTkgcUSJKFW1VVDZ18EatiS+aktirVTiJbJuJAs9+dh9x0eRO4sqNigIH31CrZToDTPiaOtO+F9kZJk2mdfQkEI5wtM1HjdcA5ix1TW13rdZhDVkiEaUI+VglLl+Iel1RXAvl91nhRByZVspNHrL8R3hrXwNyMhaol2U4sBGOblfbgyYsgKzupS9vMExDe4qtCgCxhNjgDqJ07jyLS1TPkkUkolTga8V9rJ6wDEH7cArjXZGxL4xWKJrK8Fwb5R27qG3z815lgqcwBDyw669Qu0NNQmMvhYLx3U4LRWT++Wkdf/qwmp01bYmNYYDys8wY9RK+YASXqbPBuq1a26CGMNFGlAqOVdLmC+I2F1FWkJv8VQnLz8Fcu4PpGa3U+I7w0WF1K51p5c64LxmKyLh9QeSjOzDWvIs+7EdvMGCZvADppGU+ZFnhSKuPzZ830eQOYDLoKMvWbiR1T/Q6iWKHleoWL0coxAlqiSgcUNfXFhJKJKIUoDQfoKUjSGvGMKYPRte1NRtKZ5B35CPmdezgn3IB737exKj8jNiWOn21aX8jVXVNWI/todD9GeaI2r7lNebwee6F3JY/kehCt5sPNHGouQNPIEwwrDZKW016FkwtIU/DkdS9Kc1WE1GVz8IkW446qLHlUI+lO2FgRCLSWsBNR8MX+EMy7swRgzeGpuJ8aNrHCNq5XP4Y46RrYknIF4zgC0VwWI3odRKhiEyrN0hje4B6l5+LJxQgBdqh+SC5n2/F0nAIUNQxSmYbjTkzacicgA6deqxTS0cwtk2Q2ahjQpGdWSNyUqu7/gTRUtqRVi9KySgkb4u6jpJIRAmXmr8BZ5KGPTS7/bjNhZSWlGIyDFKzndECZ12NtPM5xlAHta9A1uVgdbK/wc2mfY1IEugliXBExhTpICvYgN1fS0ezn8yIOjt+pCFMpDgLa24pxvIZajWsl56x80bnMWN4NhlmA06rUdNpHH1R7LCSZTFQ4rQSdI7EfOQjddcRORLbcURIDJGItKQo0LCLgiwzjUXTyC+2D+77O0ph0rXw2QZoPQxbfw/OYZhcJipcPnRhP+ZIB7bgMaz4sZn02C1GjAEzGA1gLyEzbxzkj1Wre6eh6RrUcdDrJBafN0JNmLIM5ky1Ib75c1EqSjCRiLTUVg2eJmwWC7Nmn6/2aA223FEw42b4/A01GbVWMRGYkKsQiigoioJOsmAwZCBl5INzuDpXzFGulqqGuFipTaeDwknq6gH1n4pElGAiEWmp5kP1vmiyNkkoKiMPpn1LXYnQVQ2+NiQ5gsloAVMmZBZARv4pByMOdY3tfqTMceTzAbR8oS4NkpGndVhDhkhEWnEd4Vj1ZzR3hCiYPI3TV2wGQUauehO62FnTxqZ9jQzLsXFt/lh18m7VezDxa1qHNmSIAY1akGXC+9+g+piXfUoF+1xnbkkjHYzIy0CS1Im5tY7p6sGGz8RI6wQSiUgDyuHNVB0+SIesp63wS5w93Kl1SMIpOKxGJpeqa2u/eURHuHCy+sS+V1Nv08o0lRKJ6PHHH6eiogKLxcLs2bP58MMPtQ4paZSjO6j95C0a3QG+yP4yF00ddXztGyFlnTsqD5tJT7MnyBv+icimTHWboT0vQyRFtmhKY5onoj//+c8sXbqUe++9l+3btzN16lQuvfRSGhuH2M4JIT+BPa/zxZa/qUX8rCmcNX2OJruZCv1nNemZP7kYnSSx71iY1zgPv6xTR1pvf0YssD9AkpKIrUQHYPbs2Zxzzjk89thjAMiyTHl5OT/4wQ9Yvnz5KV/b3t6Ow+HA5XJhtw/yGJzTCQcJetuROhowtldD416ONLVS0+qj1jGN4TMuZ+YI0TCcbiqbPLy2q46wrPCdCXryDv0DQj7a/SFC9mFIeWNQMovB6kRnykCnV0eXZ54wejwUkZEAnSQhSSR/bqGG+vo3qmmvWTAYZNu2baxYsSJ2TKfTcckll/D+++93Oz8QCBAIHF8/p72973tfHW3z8dbeBk7MutEUrCgKs0bkMrFE/Ua1fLqRfZ/tgBPPVmQkFFBgWI6VUqcFFAVPIMTuo23qxTovqFPCIIeJyArDc2yxrXFyC0r52HEWs6bPECWhNDUqP5MbZg/nSKuXvFIn5N0KlW9yePv7dNTuBnafcLZERGfAYLLwpVEFgASSxKdVrbj8YRTUBCRJOiTUVXSNOonzRh8fFvDZ0XZc/pD6fGfCqiz9GhG9Bb1O4tuzh8fOfWtfA0dbfb3G/p0vDY9dY/OBJqo61wjvyfXnDIuN8n+vspnKpq7nfv3sMqymxDUpaJqImpubiUQiFBZ2XaemsLCQffv2dTt/9erV3HfffXG9Vygs0+wJ9vp8IHx8uxjF70LyNvd+MX9QvakvRAl0/SFFrxSRjHjM+VA6FvLGYnUO50pJGtL/Ac8EORmm2NbUmDNh4pXUeschN+zB1nEUc7AFQ7gDRVHQKyGMkYi6g0knU9CFNdTz3nJGvQSeE/4BdriQ/Oq50aMtngBhvQ7DSVNk3P7wKX/HT+QJnPpc5YR/wh5/mOaTFtBTSGxFKq3GEa1YsYKlS5fGHre3t1NeXt6n1xbaLVx7dtnJa7bHHjttx5cAzRz7ZUYVTlH/gaFDkSRAQtKp9xlmA5iMIEmYZYUxvjDEEowEOj16cwZWqxWTXtdloXiRgoamy2aOB04YbR0JQ9gPkaB6kyOAWmoePyWC3FmClmUZWQEZBSW6Sq7l+J9lyZgg+REFGWIL2JVllfU41+3cUXmcPaxvI9JmjciJ9QT2xKg73nw8Y3g244u6VqsS3cGiaSLKy8tDr9fT0NB1h4SGhgaKioq6nW82mzGb45uvZDXpGZbbt+qQ2VlMgbO4T+cagbzef57CmUpvAH1mj0/1Z9RYdk7fz+3PXL7+LLuSm2kmt+ePkjCa9pqZTCZmzJjBm2++GTsmyzJvvvkmc+bM0TAyQRAGk+ZVs6VLl7Jw4UJmzpzJrFmzeOSRR+jo6ODmm2/WOjRBEAaJ5ono+uuvp6mpiZUrV1JfX8+0adN4/fXXuzVgC4IwdGk+jmggUnockSAIff4b1XxktSAIgkhEgiBoTiQiQRA0p3lj9UBEm7f6M9VDEITBE/3bPF1TdFonIrfbDdDn0dWCIGjD7XbjcPQ+8jete81kWaa2tpasrCzcbjfl5eXU1NScMT1o0Sku4jMPXen+eRVFwe12U1JSgk7Xe0tQWpeIdDodZWVlwPGZyXa7PS1/YAMhPvPQl86f91QloSjRWC0IguZEIhIEQXNDJhGZzWbuvffeuGfnpyPxmYe+M+XzpnVjtSAIQ8OQKREJgpC+RCISBEFzIhEJgqA5kYgEQdBcWieilpYWvv3tb2O323E6ndxyyy14PJ5TvmbevHlInQvdR2933HHHIEXcf/3dBfevf/0r48ePx2KxMHnyZF599dVBijRx+vOZ169f3+3nabFYBjHagdm8eTMLFiygpKQESZJ4+eWXT/uat99+m7PPPhuz2czo0aNZv3590uNMOiWNXXbZZcrUqVOVDz74QPn3v/+tjB49WvnWt751ytfMnTtXue2225S6urrYzeVyDVLE/fPCCy8oJpNJWbt2rfLZZ58pt912m+J0OpWGhoYez9+yZYui1+uV3/zmN8qePXuUn/3sZ4rRaFR27do1yJHHr7+fed26dYrdbu/y86yvrx/kqOP36quvKj/96U+VDRs2KIDy0ksvnfL8L774QrHZbMrSpUuVPXv2KI8++qii1+uV119/fXACTpK0TUR79uxRAOWjjz6KHXvttdcUSZKUo0eP9vq6uXPnKj/84Q8HIcKBmzVrlrJkyZLY40gkopSUlCirV6/u8fzrrrtOueKKK7ocmz17tvLd7343qXEmUn8/87p16xSHwzFI0SVXXxLR3XffrZx11lldjl1//fXKpZdemsTIki9tq2bvv/8+TqeTmTNnxo5dcskl6HQ6tm7desrXPvfcc+Tl5TFp0iRWrFiB1+tNdrj9Ft0F95JLLokdO9UuuKB+T048H+DSSy/t9fxUE89nBvB4PAwfPpzy8nKuvPJKPvvss8EIVxPp/jPuTdpOeq2vr6egoKDLMYPBQE5ODvX19b2+7oYbbmD48OGUlJTw6aefcs8997B//342bNiQ7JD7pb+74IL6Penp/FN9P1JJPJ953LhxrF27lilTpuByuXjooYc499xz+eyzz2ITooeS3n7G7e3t+Hw+rFarRpENTMolouXLl/Pggw+e8py9e/fGff3bb7899vXkyZMpLi7m4osvprKyklGjRsV9XUEbc+bM6bIH3rnnnsuECRN48skn+eUvf6lhZEJ/pFwi+s///E8WLVp0ynNGjhxJUVERjY2NXY6Hw2FaWlp63CW2N7Nnzwbg4MGDKZWI+rsLLkBRUVG/zk818XzmkxmNRqZPn87BgweTEaLmevsZ2+32tC0NQQp23+fn5zN+/PhT3kwmE3PmzKGtrY1t27bFXvvWW28hy3IsufTFzp07ASgu7tsW04Mlnl1w58yZ0+V8gDfeeCNtds1NxM6/kUiEXbt2pdzPM1HS/WfcK61bywfisssuU6ZPn65s3bpVeffdd5UxY8Z06b4/cuSIMm7cOGXr1q2KoijKwYMHlVWrVikff/yxcujQIeXvf/+7MnLkSOWCCy7Q6iOc0gsvvKCYzWZl/fr1yp49e5Tbb79dcTqdse7pG2+8UVm+fHns/C1btigGg0F56KGHlL179yr33ntvWnbf9+cz33fffcrGjRuVyspKZdu2bco3v/lNxWKxKJ999plWH6Ff3G63smPHDmXHjh0KoDz88MPKjh07lKqqKkVRFGX58uXKjTfeGDs/2n3/4x//WNm7d6/y+OOPi+57rR07dkz51re+pWRmZip2u125+eabFbfbHXv+0KFDCqBs2rRJURRFqa6uVi644AIlJydHMZvNyujRo5Uf//jHKTuOSFEU5dFHH1WGDRummEwmZdasWcoHH3wQe27u3LnKwoULu5z/l7/8RRk7dqxiMpmUs846S3nllVcGOeKB689nvuuuu2LnFhYWKvPnz1e2b9+uQdTx2bRpkwJ0u0U/48KFC5W5c+d2e820adMUk8mkjBw5Ulm3bt2gx51oYhkQQRA0l3JtRIIgnHlEIhIEQXMiEQmCoDmRiARB0JxIRIIgaE4kIkEQNCcSkSAImhOJSEgr69evx+l0xh7/4he/YNq0abHHixYt4qqrrhr0uISBEYlI6NGiRYt6XUZ3yZIlSJLUZXJyMhJARUUFjzzySJdj119/PQcOHOj1NWvWrOmydOq8efO46667EhqXkHgiEQm9Ki8v54UXXsDn88WO+f1+nn/+eYYNG6ZJTFartds6VCdyOBxdSkxCehCJSOjV2WefTXl5eZdF4zZs2MCwYcOYPn36gK7dU0nlqquuipWy5s2bR1VVFT/60Y9ii+JD96rZyU4smS1atIh33nmHNWvWxK5x6NAhRo8ezUMPPdTldTt37kSSpCG7fEiqE4lIOKXFixezbt262OO1a9dy8803J/19N2zYQFlZGatWraKuro66urp+X2PNmjXMmTOH2267LXaNYcOGdftMAOvWreOCCy5g9OjRifoIQj+IRCSc0ne+8x3effddqqqqqKqqYsuWLXznO99J+vvm5OSg1+vJysqiqKgorsXdHA4HJpMJm80Wu4Zer2fRokXs378/tk1RKBTi+eefZ/HixYn+GEIfpdwKjUJqyc/P54orrmD9+vUoisIVV1xBXl6e1mENSElJCVdccQVr165l1qxZ/OMf/yAQCPCNb3xD69DOWKJEJJzW4sWLWb9+PU8//XTCSg06nY6TV6AJhUIJuXZf3HrrrbGG+HXr1nH99ddjs9kG7f2FrkQiEk7rsssuIxgMEgqFuPTSSxNyzfz8/C7tPpFIhN27d3c5x2QyEYlEBvQ+vV1j/vz5ZGRk8MQTT/D666+LapnGRNVMOC29Xh/bOUWv1/d6nsvliq0BHpWbm0t5eXm3cy+66CKWLl3KK6+8wqhRo3j44Ydpa2vrck5FRQWbN2/mm9/8JmazOa4qYUVFBVu3buXw4cNkZmaSk5ODTqeLtRWtWLGCMWPGpP+az2lOlIiEPrHb7djt9lOe8/bbbzN9+vQut/vuu6/HcxcvXszChQu56aabmDt3LiNHjuTCCy/scs6qVas4fPgwo0aNIj8/P664ly1bhl6vZ+LEieTn51NdXR177pZbbiEYDA5KL6BwamKpWOGM9e9//5uLL76YmpqabpsWCoNLJCLhjBMIBGhqamLhwoUUFRXx3HPPaR3SGU9UzYQzzp/+9CeGDx9OW1sbv/nNb7QOR0CUiARBSAGiRCQIguZEIhIEQXMiEQmCoDmRiARB0JxIRIIgaE4kIkEQNCcSkSAImhOJSBAEzYlEJAiC5v5/DTyJfk28PiEAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T12:10:06.015203Z", + "iopub.status.busy": "2024-03-23T12:10:06.014310Z", + "iopub.status.idle": "2024-03-23T12:10:06.162523Z", + "shell.execute_reply": "2024-03-23T12:10:06.161534Z" + }, + "papermill": { + "duration": 0.1726, + "end_time": "2024-03-23T12:10:06.167192", + "exception": false, + "start_time": "2024-03-23T12:10:05.994592", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnUklEQVR4nO3de1gU590+8JvlsIgcFEFERNdDFY0KilXxUGvKwdeG6mWT+MNEEK1tVPoatyRKGuG1tsE0kZCmVC5t8JCT1niIrUQlRFKPUSH6YgJYUMQoIAcVAV0W9vn9kZeJKwssBBhmuT/X5aX77DOz39lZbmcedp6xEkIIEBEpjEruAoiI2oPhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESmSjdwFdDWDwYBbt27ByckJVlZWcpdDRI8RQuD+/fsYOHAgVKrmj696XHjdunUL3t7ecpdBRK24ceMGBg0a1OzzPS68nJycAHz3xjg7O8tcTefR6/U4duwYgoODYWtrK3c51AF6yj6tqqqCt7e39LPanB4XXo2nis7OzhYfXg4ODnB2drboD3pP0tP2aWvDOhywJyJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpUo/7qoSlqq2tRW5urvS4+oEOp7ML0NftAhx7qY36+vj4wMHBoatLJOpQDC8LkZubC39//ybtfzbRNzMzExMnTuz8oog6EcPLQvj4+CAzM1N6nFd8F9q92Uh4ZhxGefZp0pdI6RheFsLBwcHoaEp1vQLqEw8weqwv/Ib0k7Eyos7BAXsiUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiTZwyspKQkajQb29vaYMmUKzp0712L/xMREjBo1Cr169YK3tzfWrFmDhw8fdlG1RNRdyBpee/bsgVarRVxcHLKysuDr64uQkBDcvn3bZP8PP/wQ69atQ1xcHHJycvDuu+9iz549eOWVV7q4ciKSm6zhlZCQgOXLlyMyMhJjxoxBcnIyHBwckJKSYrL/6dOnMX36dCxatAgajQbBwcEICwtr9WiNiCyPbBdm19XVITMzEzExMVKbSqVCYGAgzpw5Y3KZadOm4f3338e5c+cwefJkXL16FampqVi8eHGzr6PT6aDT6aTHVVVVAL67jZRer++grel+6uvrpb8teTt7ksb9aOn709ztky28ysvL0dDQAA8PD6N2Dw8Po0n1HrVo0SKUl5djxowZEEKgvr4eL7zwQounjfHx8diwYUOT9mPHjln0hHw3qgHABmfPnsXNy3JXQx0pLS1N7hI6VW1trVn9FDUlTkZGBl577TX87W9/w5QpU5Cfn4/Vq1dj48aNWL9+vcllYmJioNVqpceNd+MNDg626JvOXiqqBLIvYOrUqfAd7Cp3OdQB9Ho90tLSEBQUZNE3nW08O2qNbOHl5uYGa2trlJaWGrWXlpZiwIABJpdZv349Fi9ejF/96lcAgHHjxqGmpga//vWv8fvf/x4qVdMhPLVaDbVa3aTd1tbWoj8ANjY20t+WvJ09kaV/ds3dNtkG7O3s7ODv74/09HSpzWAwID09HQEBASaXqa2tbRJQ1tbWAAAhROcVS0TdjqynjVqtFhEREZg0aRImT56MxMRE1NTUIDIyEgAQHh4OLy8vxMfHAwBCQ0ORkJCACRMmSKeN69evR2hoqBRiRNQzyBpeCxcuRFlZGWJjY1FSUgI/Pz8cOXJEGsQvKioyOtJ69dVXYWVlhVdffRU3b96Eu7s7QkND8ac//UmuTSAimViJHna+VVVVBRcXF9y7d8+iB+wvXq/A/C1ncXDFVM5hbyH0ej1SU1Mxd+5cix7zMvdnVPbLg4iI2oPhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRZL31GRGZVltbi9zcXKO26gc6nM4uQF+3C3Ds9f1d4H18fODg4NDVJcqO4UXUDeXm5sLf39/kc39+7HFmZiYmTpzY+UV1Mwwvom7Ix8cHmZmZRm15xXeh3ZuNhGfGYZRnH6O+PRHDi6gbcnBwaHI0pbpeAfWJBxg91pc3EgYH7IlIoRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBRJ9vBKSkqCRqOBvb09pkyZgnPnzrXY/+7du1i1ahU8PT2hVqsxcuRIpKamdlG1RNRdyDolzp49e6DVapGcnIwpU6YgMTERISEhyMvLQ//+/Zv0r6urQ1BQEPr374+PP/4YXl5euH79Ovr06dP1xRORrGQNr4SEBCxfvhyRkZEAgOTkZBw+fBgpKSlYt25dk/4pKSmorKzE6dOnYWtrCwDQaDRdWTJRp7lWXoMaXX2zzxeU1Uh/29iY/tHtrbbBULfenVJfdyNbeNXV1SEzMxMxMTFSm0qlQmBgIM6cOWNymUOHDiEgIACrVq3CJ598And3dyxatAhr166FtbW1yWV0Oh10Op30uKqqCgCg1+uh1+s7cIu6l/r6eulvS95OS1FYUYOgxFNm9f3dx9ktPp/24nRo+ik3wMz9vMoWXuXl5WhoaICHh4dRu4eHR5MbDzS6evUqPv/8czz33HNITU1Ffn4+Vq5cCb1ej7i4OJPLxMfHY8OGDU3ajx07ZtE3LbhRDQA2OHv2LG5elrsaak3j/lo8ogEevYTJPnoDUKkDXNWArYnR6tIHVngv3xpH07+At2OnltupamtrzeqnqGmgDQYD+vfvj61bt8La2hr+/v64efMm3njjjWbDKyYmBlqtVnpcVVUFb29vBAcHw9nZuatK73KXiiqB7AuYOnUqfAe7yl0OteLrW1V4M/ssfhk0HU8MNP251Ov1SEtLQ1BQkDRs8vg63ss/ixkzZjS7DiVoPDtqjWzh5ebmBmtra5SWlhq1l5aWYsCAASaX8fT0hK2trdEp4ujRo1FSUoK6ujrY2dk1WUatVkOtVjdpt7W1NfkBsBSNYyI2NjYWvZ2Woi37q7nPrqXsc3Nrl+2rEnZ2dvD390d6errUZjAYkJ6ejoCAAJPLTJ8+Hfn5+TAYDFLblStX4OnpaTK4iMhyyfo9L61Wi23btmHnzp3IycnBihUrUFNTI/32MTw83GhAf8WKFaisrMTq1atx5coVHD58GK+99hpWrVol1yYQkUxkHfNauHAhysrKEBsbi5KSEvj5+eHIkSPSIH5RURFUqu/z1dvbG0ePHsWaNWswfvx4eHl5YfXq1Vi7dq1cmyCbB/UPcOr6N3hQ12Dy+cLyaqjsb+JE0SXcqG1+9LaXnTWmDxmDXja9OqtUok4h+4B9VFQUoqKiTD6XkZHRpC0gIABnz57t5Kq6v1PXv8Gak0ta7NN7KLD1KoCrLa/rLexA4HDTd2cm6q5kDy9qn762g1Bz7beIDhoJb9emX/l4oKvDiQvZmDlpHHqpTY8H3qisxZtpV9B39qDOLpeowzG8FEptbQ/DQy/8RDMBY71cmjyv1+uhvl6BuU9Mafa3N5dv3sOfH9ZAbW3f2eUSdbg2h9fVq1cxbNiwzqiFqMfSNTyEyv4mrlXlQWVveoyyvr4et+pvIacyx+TlQdeqvhvn1DU8BND0PzRL0+bwGjFiBGbNmoVly5bh6aefhr09/9cm+qFu1VxH76Hv4JWWJ1UBAPztyN+afa73UOBWjR/84dFsH0vR5vDKysrC9u3bodVqERUVhYULF2LZsmWYPHlyZ9RH1CMM7D0ENdd+i7cX+mF4/+aPvE6dPIXpM6abPPIquF2N1XsuYuDsIZ1dbrfQ5vDy8/PD22+/jc2bN+PQoUPYsWMHZsyYgZEjR2Lp0qVYvHgx3N3dO6NWIovVOIY51HkUxvQzfcqn1+txzeYaRruONjmOaXh4D4aHZT1mDLPdX1K1sbHBggULsHfvXrz++uvIz89HdHQ0vL29ER4ejuLi4o6sk4jISLvD68KFC1i5ciU8PT2RkJCA6OhoFBQUIC0tDbdu3cK8efM6sk4iIiNtPm1MSEjA9u3bkZeXh7lz52LXrl2YO3eu9E34oUOHYseOHZwkkIg6VZvDa8uWLVi6dCmWLFkCT09Pk3369++Pd9999wcXR0TUnDaHV1paGgYPHmx0zSEACCFw48YNDB48GHZ2doiIiOiwIomIHtfmMa/hw4ejvLy8SXtlZSWGDh3aIUUREbWmzeElhOkpaqurq/mFVSLqMmafNjZOpWxlZYXY2Fij+d8bGhrw5Zdfws/Pr8MLJCIyxezw+uqrrwB8d+SVnZ1tNHOpnZ0dfH19ER0d3fEVEhGZYHZ4HT9+HAAQGRmJt99+26JvXkFE3V+bf9u4ffv2zqiDiKhNzAqvBQsWYMeOHXB2dsaCBQta7Lt///4OKYyIqCVmhZeLiwusrKykfxMRyc2s8Hr0VJGnjUTUHch66zMiovYy68hrwoQJ0mlja7Kysn5QQURE5jArvObPn9/JZRARtY1Z4RUXF9fZdRARtQnHvIhIkcw68nJ1dcWVK1fg5uaGvn37tjj+VVlZ2WHFERE1x6zweuutt+Dk5CT929zBeyKizmJWeD06seCSJUs6qxYiIrO1eczL2toat2/fbtJeUVEBa2vrDimKiKg1HTYZoU6nM5omh4ioM5k9q8Rf/vIXAN9NRvj3v/8djo7f39W3oaEB//73v+Hj49PxFRIRmWB2eL311lsAvjvySk5ONjpFtLOzg0ajQXJycsdXSERkgtnhde3aNQDA7NmzsX//fvTt27fTiiIiak2bJyNsnFGViEhObQ6vpUuXtvh8SkpKu4shIjJXm8Przp07Ro/1ej0uX76Mu3fv4sknn+ywwoiIWtLm8Dpw4ECTNoPBgBUrVmD48OEdUhQRUWs65MJslUoFrVYr/UayrZKSkqDRaGBvb48pU6bg3LlzZi23e/duWFlZccoeoh6ow2aVKCgoQH19fZuX27NnD7RaLeLi4pCVlQVfX1+EhISY/Bb/owoLCxEdHY2ZM2e2t2QiUrA2nzY23jm7kRACxcXFOHz4sNE1kOZKSEjA8uXLERkZCQBITk7G4cOHkZKSgnXr1plcpqGhAc899xw2bNiAEydO4O7du21+XSJStjaHV+OdsxupVCq4u7tj8+bNrf4m8nF1dXXIzMxETEyM0foCAwNx5syZZpf7wx/+gP79+2PZsmU4ceJEi6+h0+mg0+mkx1VVVQC++0WDXq9vU73dSeNRbn19vcntaGxraRtbWwd1HXP2RWv71FL2p7m1y/o9r/LycjQ0NMDDw8Oo3cPDA7m5uSaXOXnyJN59911cvHjRrNeIj4/Hhg0bmrQfO3YMDg4Oba65u7hRDQA2OHnyJK47Nt8vLS3tB6+DOl9b9kVz+9RS9mdtba1Z/docXnK6f/8+Fi9ejG3btsHNzc2sZWJiYoxOdauqquDt7Y3g4GA4Ozt3Vqmd7utbVXgz+yxmzJiBJwY23Q69Xo+0tDQEBQXB1ta2XeugrmPOvmhtn1rK/mw8O2qNrOHl5uYGa2trlJaWGrWXlpZiwIABTfoXFBSgsLAQoaGhUpvBYAAA2NjYIC8vr8nXNdRqNdRqdZN12draNvtDrQQ2NjbS3y1tR0vbae46qPO1ZV80t08tZX+aW7usc9jb2dnB398f6enpUpvBYEB6ejoCAgKa9Pfx8UF2djYuXrwo/fnFL36B2bNn4+LFi/D29u7K8olIRrKfNmq1WkRERGDSpEmYPHkyEhMTUVNTI/32MTw8HF5eXoiPj4e9vT3Gjh1rtHyfPn0AoEk7EVm2Dguvb7/9Fn/4wx+wdevWNi23cOFClJWVITY2FiUlJfDz88ORI0ekQfyioiKoVLzJEREZ67DwqqiowLvvvtvm8AKAqKgoREVFmXwuIyOjxWV37NjR5tcjIuXjIQ0RKRLDi4gUieFFRIpk9pjXggULWnye1xcSUVcyO7xcXFxafT48PPwHF0REZA6zw2v79u2dWQcRUZtwzIuIFMnsIy9zp7vhDTiIqCuYHV47duzAkCFDMGHCBAghOrMmIqJWmR1eK1aswEcffYRr164hMjISzz//PFxdXTuzNiKiZpk95pWUlITi4mK8/PLL+Oc//wlvb288++yzOHr0KI/EiKjLtWnAXq1WIywsDGlpafjmm2/wxBNPYOXKldBoNKiuru6sGomImmj3bxtVKhWsrKwghEBDQ0NH1kRE1Ko2hZdOp8NHH32EoKAgjBw5EtnZ2fjrX/+KoqIiODoqeNJsIlIcswfsV65cid27d8Pb2xtLly7FRx99ZPY88tTxHui/O9q9fPOeyedrHuhwoQwYcP0OevdqOg02AOTf5qk+KZfZ4ZWcnIzBgwdj2LBh+OKLL/DFF1+Y7Ld///4OK46aV/B/wbNuf3YLvWzwXv75VtfVWy37hLpEbWb2pzY8PBxWVladWQu1QfAT392gZHh/R/SytW7yfF7xPfzu42xsfnocRnk2f11qb7UNhrr17rQ6iTpLm76kSt2Ha287/L/Jg5t9vvEGpMPde2OsV8sX1RMpEa9tJCJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJX60m6gZau9wLaP2Sr552uRfDi6gbMO9yL8CcS756yuVePWMribq51i73Asy75KsnXe7F8CLqBlq73AvgJV+P44A9ESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSpG4RXklJSdBoNLC3t8eUKVNw7ty5Zvtu27YNM2fORN++fdG3b18EBga22J+ILJPs4bVnzx5otVrExcUhKysLvr6+CAkJwe3bt032z8jIQFhYGI4fP44zZ87A29sbwcHBuHnzZhdXTkRykj28EhISsHz5ckRGRmLMmDFITk6Gg4MDUlJSTPb/4IMPsHLlSvj5+cHHxwd///vfYTAYkJ6e3sWVE5GcZL22sa6uDpmZmYiJiZHaVCoVAgMDcebMGbPWUVtbC71eD1dXV5PP63Q66HQ66XFVVRUAQK/XQ6/X/4Dqu7fG6+Dq6+stejt7kp6yT83dNlnDq7y8HA0NDfDw8DBq9/DwQG5urlnrWLt2LQYOHIjAwECTz8fHx2PDhg1N2o8dOwYHB4e2F60QN6oBwAZnz57FzctyV0Mdoafs09raWrP6KXpWiU2bNmH37t3IyMiAvb29yT4xMTHQarXS46qqKmmczNnZuatK7XKXiiqB7AuYOnUqfAebPiolZekp+7Tx7Kg1soaXm5sbrK2tUVpaatReWlqKAQMGtLjsm2++iU2bNuGzzz7D+PHjm+2nVquhVjedddLW1ha2trbtK1wBbGxspL8teTt7kp6yT83dNlkH7O3s7ODv72802N44+B4QENDscn/+85+xceNGHDlyBJMmTeqKUomom5H9tFGr1SIiIgKTJk3C5MmTkZiYiJqaGkRGRgIAwsPD4eXlhfj4eADA66+/jtjYWHz44YfQaDQoKSkBADg6OsLR0VG27SCiriV7eC1cuBBlZWWIjY1FSUkJ/Pz8cOTIEWkQv6ioCCrV9weIW7ZsQV1dHZ5++mmj9cTFxeF//ud/urJ0IpKR7OEFAFFRUYiKijL5XEZGhtHjwsLCzi+IiLo92b+kSkTUHgwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiReoW4ZWUlASNRgN7e3tMmTIF586da7H/3r174ePjA3t7e4wbNw6pqaldVCkRdReyh9eePXug1WoRFxeHrKws+Pr6IiQkBLdv3zbZ//Tp0wgLC8OyZcvw1VdfYf78+Zg/fz4uX77cxZUTkZxkD6+EhAQsX74ckZGRGDNmDJKTk+Hg4ICUlBST/d9++23MmTMHL730EkaPHo2NGzdi4sSJ+Otf/9rFlRORnGzkfPG6ujpkZmYiJiZGalOpVAgMDMSZM2dMLnPmzBlotVqjtpCQEBw8eNBkf51OB51OJz2uqqoCAOj1euj1+h+4Bd1HbW0t8vLypMdXiu9BV5KPyxftUFfqYtR31KhRcHBw6OoSqQ0e359A8/vU0vanuT+XsoZXeXk5Ghoa4OHhYdTu4eGB3Nxck8uUlJSY7F9SUmKyf3x8PDZs2NCk/dixYxa1wwsKCvC73/2uSfvinU37bt68GcOHD++Cqqi9mtufQNN9amn7s7a21qx+soZXV4iJiTE6UquqqoK3tzeCg4Ph7OwsY2Udq7a2FjNmzJAeVz/Q4eiJ8wiZ+WM49lIb9bW0/6kt0eP7E2h+n1ra/mw8O2qNrOHl5uYGa2trlJaWGrWXlpZiwIABJpcZMGBAm/qr1Wqo1eom7ba2trC1tW1n5d2Pi4sLJk+eLD3W6/W4f7cSM6dNtajt7Cke359Az9mn5m6brAP2dnZ28Pf3R3p6utRmMBiQnp6OgIAAk8sEBAQY9QeAtLS0ZvsTkWWS/bRRq9UiIiICkyZNwuTJk5GYmIiamhpERkYCAMLDw+Hl5YX4+HgAwOrVqzFr1ixs3rwZP//5z7F7925cuHABW7dulXMziKiLyR5eCxcuRFlZGWJjY1FSUgI/Pz8cOXJEGpQvKiqCSvX9AeK0adPw4Ycf4tVXX8Urr7yCH/3oRzh48CDGjh0r1yYQkQyshBBC7iK6UlVVFVxcXHDv3j2LGrB/nF6vR2pqKubOnWvR4yM9SU/Zp+b+jMr+JVUiovZgeBGRIjG8iEiRZB+w72qNQ3zmfhFOqfR6PWpra1FVVWXR4yM9SU/Zp40/m60Nx/e48Lp//z4AwNvbW+ZKiKgl9+/fh4uLS7PP97jfNhoMBty6dQtOTk6wsrKSu5xO03gZ1I0bNyz6t6o9SU/Zp0II3L9/HwMHDjT6mtTjetyRl0qlwqBBg+Quo8s4Oztb9Ae9J+oJ+7SlI65GHLAnIkVieBGRIjG8LJRarUZcXJzJGTVImbhPjfW4AXsisgw88iIiRWJ4EZEiMbyISJEYXjIpLCyElZUVLl682CXrOXXqFMaNGwdbW1vMnz//B70mUXfA8OomMjIyYGVlhbt373bK+rVaLfz8/HDt2jXs2LGjU16DmvrpT3+KF198Ue4yJN2tnh+C4dUOdXV1cpfQZgUFBXjyyScxaNAg9OnTp13r6MrtFkKgvr6+y16vO1Pi561LCGrVrFmzxKpVq8Tq1atFv379xE9/+lORnZ0t5syZI3r37i369+8vnn/+eVFWViYt8+mnn4rp06cLFxcX4erqKn7+85+L/Px86flr164JAOKrr76S/v3on4iIiDat56OPPhIBAQFCrVaLJ554QmRkZBg9/+if7du3CyGEyMjIED/+8Y+FnZ2dGDBggFi7dq3Q6/Utbvfx48cFAHHkyBHh5+cn7O3txezZs0VpaalITU0VPj4+wsnJSYSFhYmamhppXQ0NDeK1114TGo1G2Nvbi/Hjx4u9e/dKzzeuNzU1VUycOFHY2tqK48ePd+Ru7HIRERFN3vv8/HyxdOlS6X0YOXKkSExMbLLcvHnzxB//+Efh6ekpNBqNEEKIU6dOCV9fX6FWq4W/v784cOCA9Blq1NLn0lQ9165d66q3o8MxvMwwa9Ys4ejoKF566SWRm5srzp49K9zd3UVMTIzIyckRWVlZIigoSMyePVta5uOPPxb79u0T//nPf8RXX30lQkNDxbhx40RDQ4MQwji86uvrxb59+wQAkZeXJ4qLi8Xdu3fbtJ5BgwaJjz/+WHzzzTfiV7/6lXBychLl5eWivr5eFBcXC2dnZ5GYmCiKi4tFbW2t+Pbbb4WDg4NYuXKlyMnJEQcOHBBubm4iLi6u2e3Ozc2VQmbq1Kni5MmTIisrS4wYMULMmjVLBAcHi6ysLPHvf/9b9OvXT2zatEla1x//+Efh4+Mjjhw5IgoKCsT27duFWq2WQrZxvePHjxfHjh0T+fn5oqKiorN3bae6e/euCAgIEMuXLxfFxcWiuLhYPHz4UMTGxorz58+Lq1evivfff184ODiIPXv2SMtFREQIR0dHsXjxYnH58mVx+fJlce/ePeHq6iqef/558fXXX4vU1FQxcuRIo/C6c+dOi59LU/XU19fL8dZ0CIaXGWbNmiUmTJggPd64caMIDg426nPjxg0pfEwpKysTAER2drYQwji8hPj+h/fOnTst1tLceh4NCr1eLwYNGiRef/11qc3FxUU64hJCiFdeeUWMGjVKGAwGqS0pKUk4OjpKwfj4dj9a52effSa1xcfHCwCioKBAavvNb34jQkJChBBCPHz4UDg4OIjTp08brWvZsmUiLCzMaL0HDx5scfuVZtasWWL16tUt9lm1apX45S9/KT2OiIgQHh4eQqfTSW1btmwR/fr1Ew8ePJDatm3bZvQZMudzaU49SsExLzP5+/tL/7506RKOHz8OR0dH6Y+Pjw+A78aWAOA///kPwsLCMGzYMDg7O0Oj0QD47m5IbWHueh69b6WNjQ0mTZqEnJycZtebk5ODgIAAo2mBpk+fjurqanz77bcmt/tR48ePl/7t4eEBBwcHDBs2zKjt9u3bAID8/HzU1tYiKCjI6D3btWuX9H41mjRpUrM1W4qkpCT4+/vD3d0djo6O2Lp1a5P9OW7cONjZ2UmP8/LyMH78eNjb20ttj9+U1pzPpSXpcVPitFfv3r2lf1dXVyM0NBSvv/56k36enp4AgNDQUAwZMgTbtm3DwIEDYTAYMHbs2DYPvnbUetrr0e1+1KMzeVpZWTWZ2dPKygoGgwHAd+8XABw+fBheXl5G/R6/Tq+517MUu3fvRnR0NDZv3oyAgAA4OTnhjTfewJdffmnUrz3vgzmfS0vC8GqHiRMnYt++fdBoNLCxafoWVlRUIC8vD9u2bcPMmTMBACdPnmxxnY3/yzY0NLRrPWfPnsVPfvITAEB9fT0yMzMRFRXV7OuNHj0a+/btgxBCOvo6deoUnJycOny+szFjxkCtVqOoqAizZs3q0HV3d3Z2dkb79NSpU5g2bRpWrlwptZlzVDRq1Ci8//770Ol0UuCfP3/eqE9rn0tT9SgZTxvbYdWqVaisrERYWBjOnz+PgoICHD16FJGRkWhoaEDfvn3Rr18/bN26Ffn5+fj888+h1WpbXOeQIUNgZWWFf/3rXygrK0N1dXWb1pOUlIQDBw4gNzcXq1atwp07d7B06dJmX2/lypW4ceMGfvvb3yI3NxeffPIJ4uLioNVqW5y9sj2cnJwQHR2NNWvWYOfOnSgoKEBWVhbeeecd7Ny5s0Nfq7vRaDT48ssvUVhYiPLycvzoRz/ChQsXcPToUVy5cgXr169vEkKmLFq0CAaDAb/+9a+Rk5ODo0eP4s033wQA6T+f1j6XpuppPDpWIoZXOwwcOBCnTp1CQ0MDgoODMW7cOLz44ovo06cPVCoVVCoVdu/ejczMTIwdOxZr1qzBG2+80eI6vby8sGHDBqxbtw4eHh6Iiopq03o2bdqETZs2wdfXFydPnsShQ4fg5ubW4uulpqbi3Llz8PX1xQsvvIBly5bh1Vdf/UHvTXM2btyI9evXIz4+HqNHj8acOXNw+PBhDB06tFNer7uIjo6GtbU1xowZA3d3d4SEhGDBggVYuHAhpkyZgoqKCqOjsOY4Ozvjn//8Jy5evAg/Pz/8/ve/R2xsLABI42CtfS5N1dPWMdjuhFPiECnUBx98gMjISNy7dw+9evWSu5wuxzEvIoXYtWsXhg0bBi8vL1y6dAlr167Fs88+2yODC2B4ESlGSUkJYmNjUVJSAk9PTzzzzDP405/+JHdZsuFpIxEpEgfsiUiRGF5EpEgMLyJSJIYXESkSw4uIFInhRZ1qyZIlsLKyki7e9vDwQFBQEFJSUtp0acqOHTvaPQPsD7FkyRLO+d9NMbyo082ZMwfFxcUoLCzEp59+itmzZ2P16tV46qmnONUztZ+ss4mRxWuc0vhx6enpAoDYtm2bEEKIzZs3i7FjxwoHBwcxaNAgsWLFCnH//n0hxPcTFT76p3HG1127dgl/f3/h6OgoPDw8RFhYmCgtLZVep7KyUixatEi4ubkJe3t7MWLECJGSkiI9X1RUJJ555hnh4uIi+vbtK37xi19IUyPHxcU1eV2lT01tSXjkRbJ48skn4evri/379wMAVCoV/vKXv+Drr7/Gzp078fnnn+Pll18GAEybNg2JiYlwdnZGcXExiouLER0dDQDQ6/XYuHEjLl26hIMHD6KwsBBLliyRXmf9+vX45ptv8OmnnyInJwdbtmyRLljX6/UICQmBk5MTTpw4gVOnTsHR0RFz5sxBXV0doqOj8eyzz0pHjsXFxZg2bVrXvlHUPLnTkyxbc0deQgixcOFCMXr0aJPP7d27V/Tr1096vH37duHi4tLq650/f14AkI7aQkNDRWRkpMm+7733XpOpsHU6nejVq5c4evRoq/WTvHjkRbIRj0yE+Nlnn+FnP/sZvLy84OTkhMWLF6OiogK1tbUtriMzMxOhoaEYPHgwnJycpMkOG6d6WbFiBXbv3g0/Pz+8/PLLOH36tLTspUuXkJ+fDycnJ2naZFdXVzx8+NAip022NAwvkk1OTg6GDh2KwsJCPPXUUxg/fjz27duHzMxMJCUlAWj5noU1NTUICQmBs7MzPvjgA5w/fx4HDhwwWu6//uu/cP36daxZswa3bt3Cz372M+mUs7q6Gv7+/rh48aLRnytXrmDRokWdvPX0Q3FWCZLF559/juzsbKxZswaZmZkwGAzYvHmzNGneP/7xD6P+pqYvzs3NRUVFBTZt2gRvb28AwIULF5q8lru7OyIiIhAREYGZM2fipZdewptvvomJEydiz5496N+/P5ydnU3WaUnTJlsaHnlRp9PpdCgpKcHNmzeRlZWF1157DfPmzcNTTz2F8PBwjBgxAnq9Hu+88w6uXr2K9957D8nJyUbr0Gg0qK6uRnp6OsrLy1FbW4vBgwfDzs5OWu7QoUPYuHGj0XKxsbH45JNPkJ+fj6+//hr/+te/MHr0aADAc889Bzc3N8ybNw8nTpzAtWvXkJGRgf/+7/+W7qCk0Wjwv//7v8jLy0N5eTn0en3XvGnUOrkH3ciyPXqXZhsbG+Hu7i4CAwNFSkqKdH9IIYRISEgQnp6eolevXiIkJETs2rWryX0sX3jhBdGvXz+jr0p8+OGHQqPRCLVaLQICAsShQ4ea3Mtw9OjRolevXsLV1VXMmzdPXL16VVpncXGxCA8PF25ubkKtVothw4aJ5cuXi3v37gkhhLh9+7YICgoSjo6O/KpEN8P5vIhIkXjaSESKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiR/j/hUgrb3e7pOwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T12:10:06.222690Z", + "iopub.status.busy": "2024-03-23T12:10:06.222340Z", + "iopub.status.idle": "2024-03-23T12:10:06.484004Z", + "shell.execute_reply": "2024-03-23T12:10:06.483113Z" + }, + "papermill": { + "duration": 0.283076, + "end_time": "2024-03-23T12:10:06.486189", + "exception": false, + "start_time": "2024-03-23T12:10:06.203113", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASsAAAEmCAYAAAA6OrZqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUUElEQVR4nO2dd3iTZffHP09m94BCB7RA2RtkCSoIgoCCCCiCvr6AKA7An6CyVEDxFVRUXhDxdYA4EFEcKIgCMmTvPaSl7BZoS3ebNsnz+yNNmrRJm5S2Scr9ua5ctE+ePDmhyTfnnPvc50iyLMsIBAKBh6NwtwECgUDgDEKsBAKBVyDESiAQeAVCrAQCgVcgxEogEHgFQqwEAoFXIMRKIBB4BUKsBAKBV6BytwGVjdFo5MqVKwQGBiJJkrvNEQgEVsiyTGZmJlFRUSgUpftO1V6srly5QnR0tLvNEAgEpXDx4kXq1q1b6jnVXqwCAwMB039GUFCQm60RCATWZGRkEB0dbfmclka1Fytz6BcUFCTESiDwUJxJ0YgEu0Ag8AqEWAkEAq9AiJVAIPAKqn3OyhlkWUav12MwGNxtikBQYSiVSlQqVbUp2bnlxSo/P5/ExERycnLcbYpAUOH4+fkRGRmJRqNxtyk3zS0tVkajkYSEBJRKJVFRUWg0mmrzLSS4tZFlmfz8fK5fv05CQgKNGzcus+jS07mlxSo/Px+j0Uh0dDR+fn7uNkcgqFB8fX1Rq9WcP3+e/Px8fHx8KvX5Pt16lnMp2bwxqBVKRcV/6btVaufMmUOnTp0IDAykdu3aPPjgg5w+fdrmnLy8PMaNG0fNmjUJCAhg6NChXL16tULt8PZvHIHAEVX13i4wGHl73Sm+2X2B/edvVMpzuPVTumXLFsaNG8euXbtYv349BQUF3HvvvWRnZ1vOmThxIr/++ivff/89W7Zs4cqVKwwZMsSNVgsEguKcT8lBbzTNntl7LrVSnsOtYeC6detsfv/iiy+oXbs2+/fvp3v37qSnp/P555+zfPlyevXqBcDSpUtp3rw5u3bt4vbbb3eH2QKBoBhx17IsP++rJLHyqPgnPT0dgBo1agCwf/9+CgoK6N27t+WcZs2aERMTw86dO+1eQ6fTkZGRYXMTOMe5c+eQJIlDhw5VyXW2b99O69atUavVPPjggzf1nLcqmzdvRpIk0tLS3GpH/HUrsTp/A4Ox4if8eYxYGY1GXnjhBe644w5atWoFQFJSEhqNhpCQEJtzw8PDSUpKsnudOXPmEBwcbLmJjgvlp7I/CJMmTaJdu3YkJCTwxRdfVMpzCKqGeCvPKjNPzz9XMyv8OTxGrMaNG8exY8dYsWLFTV1n2rRppKenW24XL16sIAs9m/z8fHeb4DLx8fH06tWLunXrlvhCcpaqfN3m4uGKxhv/dsWJK/SsNEqTpFRGKOgRYjV+/Hh+++03Nm3aZNPTJiIigvz8/BLf7FevXiUiIsLutbRaraXDgqudFmRZJidf75abq4Ox7777bsaPH88LL7xAWFgYffv25dixY/Tv35+AgADCw8N5/PHHSU5Otjxm3bp13HnnnYSEhFCzZk0GDBhAfHy83eufO3eOnj17AhAaGookSYwaNcql65w6dYpu3brh4+NDq1at2LJli+XakiSRkpLCE088gSRJFs9qy5YtdO7cGa1WS2RkJFOnTrURCHuv2+wB/vHHH7Rv3x5fX1969erFtWvX+P3332nevDlBQUE8+uijNsW/RqOROXPm0KBBA3x9fWnbti0//PCD5X7zdX///Xc6dOiAVqtl27ZtZf5t3nzzTWrXrk1gYCBPPvkkU6dOpV27dpb7R40axYMPPsh//vMfoqKiaNq0KQBfffUVHTt2JDAwkIiICB599FGuXbtmc+21a9fSpEkTfH196dmzJ+fOnSvTnspGlmWLZ9W3lelzuedcxa8IujXBLssyEyZM4KeffmLz5s00aNDA5v4OHTqgVqvZuHEjQ4cOBeD06dNcuHCBrl27Vrg9uQUGWsz4o8Kv6wwn3uiLn8a1P8eyZct49tln2b59O2lpafTq1Ysnn3ySDz74gNzcXKZMmcKwYcP466+/AMjOzmbSpEm0adOGrKwsZsyYweDBgzl06FCJJe7o6GhWrVrF0KFDOX36NEFBQfj6+rp0nZdffpn58+fTokUL3n//fQYOHEhCQgLR0dEkJibStGlT3njjDR555BGCg4O5fPky9913H6NGjeLLL7/k1KlTPPXUU/j4+DBr1iy7rxsgMTERgFmzZvHhhx/i5+fHsGHDGDZsGFqtluXLl5OVlcXgwYNZuHAhU6ZMAUwpg6+//pqPP/6Yxo0bs3XrVv71r39Rq1YtevToYXm+qVOnMm/ePGJjYwkNDS31b/LNN9/wn//8h48++og77riDFStW8N5775V4b2/cuJGgoCDWr19vOVZQUMDs2bNp2rQp165dY9KkSYwaNYq1a9cCpp5sQ4YMYdy4cYwdO5Z9+/bx4osvlv1GqWSSMvLIzjegVEg83KEuvx6+wt6EVGRZrtAia7eK1bhx41i+fDm//PILgYGBljxUcHAwvr6+BAcHM2bMGCZNmkSNGjUICgpiwoQJdO3aVawEAo0bN+add94BTN/m7du356233rLcv2TJEqKjo/nnn39o0qSJRfCt769VqxYnTpyw5AnNKJVKy0JH7dq1bcI0Z68zfvx4y7mLFy9m3bp1fP7550yePJmIiAgkSSI4ONjiJX/00UdER0fz4YcfIkkSzZo148qVK0yZMoUZM2ZYhND6dUORWL355pvccccdAIwZM4Zp06YRHx9PbGwsAA899BCbNm1iypQp6HQ63nrrLTZs2GD54ouNjWXbtm3873//sxGrN954gz59+pT9BwEWLlzImDFjGD16NAAzZszgzz//JCsry+Y8f39/PvvsM5ttME888YTl59jYWBYsWECnTp3IysoiICCAxYsX07BhQ9577z0AmjZtytGjR3n77bedsq2yMK8E1qvpR6f6NVApJJIy8riclkvd0IortnarWC1evBgwufbWLF261BJyfPDBBygUCoYOHYpOp6Nv37589NFHlWKPr1rJiTf6Vsq1nXluV+nQoYPl58OHD7Np0yYCAgJKnBcfH0+TJk04c+YMM2bMYPfu3SQnJ2M0GgG4cOFCCbEqDWevY+39qlQqOnbsyMmTJx1e9+TJk3Tt2tXm2/iOO+4gKyuLS5cuERMTU+J1W9OmTRvLz+Hh4fj5+VmEynxsz549AMTFxZGTk1NChPLz82nfvr3NsY4dOzq0uTinT5/mueeesznWuXNni3drpnXr1iX26+3fv59Zs2Zx+PBhbty4YfP/2qJFC06ePEmXLl1sHlMZEYarmMWqYa0AfDVKWtUJ5tDFNPaeS60+YuVMnsbHx4dFixaxaNGiSrdHkiSXQzF34u/vb/k5KyuLgQMH2v2WjYyMBGDgwIHUq1ePTz/9lKioKIxGI61atXI5wVtR1ykv1q/bGrVabflZkiSb383HzAJg9nTWrFlDnTp1bM7TarVOPd/NUPya2dnZ9O3bl759+/LNN99Qq1YtLly4QN++fT0+AW8uW2hU2/RF2al+aKFY3WBw+9L7qruCRyTYBTfPbbfdxvHjx6lfvz6NGjWyufn7+5OSksLp06d59dVXueeee2jevDk3bpSeBDV/81u3znHlOrt27bL8rNfr2b9/P82bN3f4fM2bN2fnzp02X2Lbt28nMDCwzGECrtKiRQu0Wi0XLlwo8f91M+UuTZs2Ze/evTbHiv9uj1OnTpGSksLcuXO56667aNasWYnkevPmzS2eoRnr/2N3YfasGtUyiVXH+qb0QUWvCAqxqiaMGzeO1NRURowYwd69e4mPj+ePP/5g9OjRGAwGQkNDqVmzJp988glxcXH89ddfTJo0qdRr1qtXD0mS+O2337h+/TpZWVkuXWfRokX89NNPnDp1inHjxnHjxg2bvExxnnvuOS5evMiECRM4deoUv/zyCzNnzmTSpEkVvsctMDCQl156iYkTJ7Js2TLi4+M5cOAACxcuZNmyZeW+7oQJE/j8889ZtmwZZ86c4c033+TIkSNlJppjYmLQaDQsXLiQs2fPsnr1ambPnm1zzjPPPMOZM2d4+eWXOX36NMuXL/eI+rS4a6btcQ0LPauO9UIJ1KqIDPalwGCssOcRYlVNiIqKYvv27RgMBu69915at27NCy+8QEhICAqFAoVCwYoVK9i/fz+tWrVi4sSJvPvuu6Ves06dOrz++utMnTqV8PBwxo8f79J15s6dy9y5c2nbti3btm1j9erVhIWFlfp8a9euZc+ePbRt25ZnnnmGMWPG8Oqrr97U/40jZs+ezWuvvcacOXNo3rw5/fr1Y82aNSVW7lzhscceY9q0abz00kvcdtttJCQkMGrUqDI7HtSqVYsvvviC77//nhYtWjB37lzmzZtnc05MTAyrVq3i559/pm3btnz88cc2CyruID2ngOQsHQANa5lC25oBWg7NvJdlT3RGraw4iZFkVwt8vIyMjAyCg4NJT08vUXOVl5dHQkICDRo0qPT2GYJblz59+hAREcFXX31V5c9d2e/x/edvMHTxDiKCfNg1/R6XH1/a57M43pNNFgi8gJycHD7++GP69u2LUqnk22+/ZcOGDTb1VNUJczFow9oVvwhRHBEGCgQu0LJlSwICAuzevvnmGyRJYu3atXTv3p0OHTrw66+/smrVKpvN+NUJy0pgrZIlMxWN8KwEAhdYu3YtBQUFdu8LDw/H19eXDRs2VLFV7sOyElhbiJVA4FHUq1fP3SZ4FOYNzA2rwLMSYaBAICgXeQUGLqaaNoZXhWclxEogEJSLcynZGGUI9FFRK1Bb9gNuEiFWAoGgXFjvCayKEXZCrAQCQbmIL6xcr4oQEIRYCQSCchJ3vepWAkGI1S1J/fr1mT9/vuV3SZL4+eef3WaPwDuxDgOrAlG6ICAxMbHMDpgCgTVGo8zZKvashFgJHPazr0pkWcZgMKBSibekN3A5LRed3ohGqSA61LdKnlOEgdbIMuRnu+fm4n7yzMxMHnvsMfz9/YmMjOSDDz7g7rvv5oUXXnD5ZVuHgeZhDj/++CM9e/bEz8+Ptm3blpjTuG3bNu666y58fX2Jjo7m+eeft5mkXdbwg/IOYxB4BuYQsH6YH6oK7KxQGuJrzJqCHHgryj3PPf0KaJzfDDpp0iS2b9/O6tWrCQ8PZ8aMGRw4cMBmisrN8MorrzBv3jwaN27MK6+8wogRI4iLi0OlUhEfH0+/fv148803WbJkCdevX2f8+PGMHz+epUuXAmUPPzDjyjAGgedQvDtoVSDEygvJzMxk2bJlLF++nHvuMbXlWLp0KVFRFSe0L730Evfffz8Ar7/+Oi1btiQuLo5mzZoxZ84cHnvsMYsX17hxYxYsWECPHj1YvHgxPj4+ZQ4/MOPKMAaB51C8O2hVIMTKGrWfycNx13M7ydmzZykoKKBz586WY8HBwZb5cxWB9fAFcw/3a9eu0axZMw4fPsyRI0f45ptvLOfIsozRaCQhIYHmzZuXOfzAjCvDGASeg2UlUHhWbkKSXArFqjPFhy8ANsMWnn76aZ5//vkSj4uJiXFp+EFlDGMQVD7xVbiB2YwQKy8kNjYWtVrN3r17LeOp0tPT+eeff+jevXulP/9tt93GiRMnaNSokd37jx49ahl+YB6+sG/fvkq3S1A1pGTpuJFjapNTlWIlVgO9kMDAQEaOHMnLL7/Mpk2bOH78OGPGjEGhUFTJHq0pU6awY8cOxo8fz6FDhzhz5gy//PIL48ePB5wbfiDwXswhYJ0QX3w1rs+7LC9CrLyU999/n65duzJgwAB69+7NHXfcQfPmzaukl3ybNm3YsmUL//zzD3fddRft27dnxowZlgS/M8MPBN5L/PWq3RNoRgyMqCYDI7Kzs6lTpw7vvfceY8aMcbc5Ag+hMt7jb/x6giXbExhzZwNeG9Ci7AeUghgYcQtw8OBBTp06RefOnUlPT+eNN94AYNCgQW62TFDdqcruoNYIsfJi5s2bx+nTp9FoNHTo0IG///6bkydP0r9/f4ePMY9NFwjKS3wV9l23RoiVl9K+fXv2799f4nhubi6HDh2qeoMEtwQ5+Xoup+UCQqwEN4mvr6/DkgKB4GY5W5hcD/VTU8NfU6XPLVYDMVVfCwTVkYp+b7tjT6CZW1qszFXaOTk5brZEIKgczO9t6x0JN0NVzgkszi0dBiqVSkJCQiytS/z8/KqkqFIgqGxkWSYnJ4dr164REhKCUlkxxZtV3R3UmltarKCo8Zx1ryWBoLoQEhJSoc0VLXsChWdV9UiSRGRkJLVr13Y4Flwg8EbUanWFeVQAeoORhOTC6nXhWbkPpVJZoX9YgaC6cSE1hwKDjI9aQZ2QqmllbM0tnWAXCATOY94TGBsWgEJR9bldIVYCgcAp3LkSCEKsBAKBk7hzJRCEWAkEAidxZ0EoCLESCAROIMuy2zYwmxFiJRAIyuRapo5MnR6FZJoV6A7cKlZbt25l4MCBREVF2QzaNDNq1CgkSbK59evXzz3GCgS3MGavKqaGH1qVe0p83CpW2dnZtG3blkWLFjk8p1+/fiQmJlpu3377bRVaKBAIoKjhnrtCQHBzUWj//v1LbRQHoNVqK3S7gEAgcB13rwSCF+SsNm/eTO3atWnatCnPPvssKSkppZ6v0+nIyMiwuQkEgpvDnXsCzdyUZ5WVlWUZfGmmrKbvrtCvXz+GDBlCgwYNiI+PZ/r06fTv35+dO3c63BozZ84cXn/99QqzQSAQuL8gFMohVgkJCYwfP57NmzeTl5dnOS7LMpIkYTAYKsy44cOHW35u3bo1bdq0oWHDhmzevJl77rnH7mOmTZvGpEmTLL9nZGRYBm0KBALXycwr4GqGDnBvGOiyWP3rX/9ClmWWLFlCeHh4lfZ/io2NJSwsjLi4OIdipdVq0Wq1VWaTQFDdMe8JrBWoJdi3Ypr4lQeXxerw4cPs37+fpk2bVoY9pXLp0iVSUlKIjIys8ucWCG5VLCGgG70qKEeCvVOnTly8eLFCnjwrK4tDhw5ZprEkJCRw6NAhLly4QFZWFi+//DK7du3i3LlzbNy4kUGDBtGoUSP69u1bIc8vEAjKpii57u9WO1z2rD777DOeeeYZLl++TKtWrUr0dm7Tpo3T19q3bx89e/a0/G7ONY0cOZLFixdz5MgRli1bRlpaGlFRUdx7773Mnj1bhHkCQRXiKZ6Vy2J1/fp14uPjGT16tOWYJEnlSrDffffdpU7f+OOPP1w1TyAQVDBFewID3WqHy2L1xBNP0L59e7799tsqT7ALBIKqJV9v5HyqaUKO14WB58+fZ/Xq1WKQpkBwC3A+JRuDUSZAqyIiyMettricYO/VqxeHDx+uDFsEAoGHUbTNxt/tUZTLntXAgQOZOHEiR48epXXr1iUS7A888ECFGScQCNyLZSXQzcl1KIdYPfPMMwC88cYbJe6r6Ap2gUDgXiyelRu32ZhxWayK7wUUCATVF09oDWPGpZxVQUEBKpWKY8eOVZY9AoHAQzAaZeKvmbbaeEIY6JJYqdVqYmJiRKgnENwCJGbkkVtgQKWQqFfTPa2MrXF5NfCVV15h+vTppKamVoY9AoHAQzDnq+qH+aNWur/1ncs5qw8//JC4uDiioqKoV68e/v62hWIHDhyoMOMEAoH7iLcqW/AEXBarBx98sBLMEAgEnoYnJdehHGI1c+bMyrBDIBB4GJ7QHdSacrc13r9/PydPngSgZcuWtG/fvsKMEggE7uesBxWEQjnE6tq1awwfPpzNmzcTEhICQFpaGj179mTFihXUqlWrom0UCARVTFpOPslZ+YDniJXLKf4JEyaQmZnJ8ePHSU1NJTU1lWPHjpGRkcHzzz9fGTYKBIIqxhwCRgX74K9168Q+Cy5bsW7dOjZs2EDz5s0tx1q0aMGiRYu49957K9Q4gUDgHjxh9FZxXPasjEZjic3LYCoYFVtxBILqgScMNS1OuVrE/N///R9XrlyxHLt8+TITJ050OHFGIBB4F562EgjlEKsPP/yQjIwM6tevT8OGDWnYsCENGjQgIyODhQsXVoaNAoGgijGP3/Ikz8rlnFV0dDQHDhxgw4YNnDp1CoDmzZvTu3fvCjdOIBBUPXkFBi7eMLUy9iTPqlxpfkmS6NOnD3369KloewQCgZs5ez0bWYZgXzVhARp3m2OhXGK1ceNGNm7cyLVr10ok1ZcsWVIhhgkEAvdQ1B3U/a2MrXFZrF5//XXeeOMNOnbsSGRkpEe9GIFAcPN4YnIdyiFWH3/8MV988QWPP/54ZdgjEAjcjKdtYDbj8mpgfn4+3bp1qwxbBAKBBxDvgTVWUA6xevLJJ1m+fHll2CIQCNyMwShzNtlUtuBpnpXLYWBeXh6ffPIJGzZsoE2bNiWq2d9///0KM04gEFQtl27kkK83olEpqBvq/lbG1rgsVkeOHKFdu3YAJQZHiGS7QODdmFcCY8P8USo86/Psslht2rSpMuwQCAQegCfNCSyO+7vACwQCj8FStuBhyXUQYiUQCKyw7AkUnpVAIPBUZFkWnpVAIPB8krPySc8tQJIg1kPGb1njslht3boVvV5f4rher2fr1q0VYpRAIKh6zCuBdUN98VEr3WxNSVwWq549e9qdxpyenk7Pnj0rxCiBQFD1eHIICOUQK1mW7dZTpaSklJjOLBAIvAdP3cBsxuk6qyFDhgCmws9Ro0ah1Wot9xkMBo4cOSL2DAoEXky8h80JLI7TYhUcHAyYPKvAwEB8fX0t92k0Gm6//XaeeuqpirdQIBBUCfHVxbNaunQpAPXr1+ell14SIZ9AUI3I1um5kp4HVAPPyszMmTMrww6BQOBGzhYWg9b01xDq7zmtjK1xOcF+9epVHn/8caKiolCpVCiVSpubK2zdupWBAwcSFRWFJEn8/PPPNvfLssyMGTOIjIzE19eX3r17c+bMGVdNFggEZRB3PRPwzMp1My57VqNGjeLChQu89tprN93WODs7m7Zt2/LEE09YEvjWvPPOOyxYsIBly5bRoEEDXnvtNfr27cuJEyfw8fEp9/MKBAJbPH0lEMohVtu2bePvv/+2tIm5Gfr370///v3t3ifLMvPnz+fVV19l0KBBAHz55ZeEh4fz888/M3z48Jt+foFAYCL+mufNCSyOy2FgdHQ0sixXhi02JCQkkJSUZDOPMDg4mC5durBz506Hj9PpdGRkZNjcBAJB6Xhq33VrXBar+fPnM3XqVM6dO1cJ5hSRlJQEQHh4uM3x8PBwy332mDNnDsHBwZZbdHR0pdopEHg7BQYj5zy0lbE1LoeBjzzyCDk5OTRs2BA/P78SbY3tbcWpSqZNm8akSZMsv2dkZAjBEghK4UJqDnqjjK9aSWSQ5+aCXRar+fPnV4IZJYmIiABMq4+RkZGW41evXi01X6bVam2q6wUCQekUdQf1R+FhrYytcVmsRo4cWRl2lKBBgwZERESwceNGizhlZGSwe/dunn322SqxQSC4FYjz0NFbxSnX+Pj4+HiWLl1KfHw8//3vf6lduza///47MTExtGzZ0unrZGVlERcXZ/k9ISGBQ4cOUaNGDWJiYnjhhRd48803ady4saV0ISoqigcffLA8ZgsEAjuY9wR6arcFMy4n2Lds2ULr1q3ZvXs3P/74I1lZphd6+PBhl6vb9+3bR/v27Wnfvj0AkyZNon379syYMQOAyZMnM2HCBMaOHUunTp3Iyspi3bp1osZKIKhAPH1PoBlJdrEOoWvXrjz88MNMmjSJwMBADh8+TGxsLHv27GHIkCFcunSpsmwtFxkZGQQHB5Oenk5QUJC7zREIPApZlmk960+ydHr+nNidJuGBVfr8rnw+Xfasjh49yuDBg0scr127NsnJya5eTiAQuJGrGTqydHqUCon6NT27OYHLYhUSEkJiYmKJ4wcPHqROnToVYpRAIKgazMn1ejX80Kg8eySDy9YNHz6cKVOmkJSUhCRJGI1Gtm/fzksvvcS///3vyrBRIBBUEnHXTBuYYz08uQ7lEKu33nqLZs2aER0dTVZWFi1atKB79+5069aNV199tTJsrDKMRrlKthIJBJ6CeU6gpyfXoRylCxqNhk8//ZTXXnuNY8eOkZWVRfv27WncuHFl2FdlfLvnAh9viec/D7bmzsZh7jZHIKgSvKHbgply1VkBxMTEEBMTU5G2uJVTiRmcT8lh6fYEIVaCW4Y4S991z06ug5NiNWnSJGbPno2/v7/Nvjt7vP/++xViWFUzslt9lu08z1+nr3EuOZv6YZ7/xxPcWuw+m0JksC8xNf0q5HrpuQVcz9QBnt10z4xTYnXw4EEKCgosPzviZhrxuZvYWgH0bFqLTaev88WOc8x6wPlKfIGgsjmVlMEjn+wC4Nzc+yvkmubK9fAgLUE+6jLOdj9OidWmTZvs/lzdGH1HAzadvs4P+y/x4r1NCPSCP6Dg1uDIxfQKv2a8l+wJNOPZhRVVzF2Nw2hUO4AsnZ7v93lWJb7gFscqaKmoFWtvaLhnjVOelb3+6I748ccfy22Mu5EkiVHd6vPqz8dYtvMcI7vVR+nBLTMEtyY6vREftWvDWezhLXsCzTjlWVl33gwKCmLjxo3s27fPcv/+/fvZuHGjZRCqNzPktjoE+ag4n5LDplPX3G2OQADYelNZOn2FXNNcY+UtYaBTnpV5wCnAlClTGDZsGB9//LFl9JbBYOC5556rFhuF/TQqRnSO4X9bz7J0RwK9W4SX/SCBoJLJKzBafs7RGeAm9UWnN3A+xXsKQqEcOaslS5bw0ksv2cwIVCqVTJo0iSVLllSoce7i8a71UEiwPS6F00mZ7jZHICA7v8ibqgjP6lxyDkYZArUqagd6R2ddl8VKr9dz6tSpEsdPnTqF0Wi08wjvo26oH31bmtoqf7Ejwc3WCASF3pT55/ybFytz2UJs7QCvKTlyuYJ99OjRjBkzhvj4eDp37gzA7t27mTt3LqNHj65wA93F6Dsa8PuxJH48cJnJfZt57Ehtwa1BRXtWlm02XpKvgnKI1bx584iIiOC9996ztIqJjIzk5Zdf5sUXX6xwA91Fp/qhtIwK4viVDL7de4Hn7m7kbpMEtzC2npWhlDOdw5v2BJpxOQxUKBRMnjyZy5cvk5aWRlpaGpcvX2by5Mk2eSxvR5IkRt/RAICvdp6nwFA9QlyBd1LRnlW8F+0JNHNTRaFBQUHVYgXQEQPbRhIWoCExPY8/jjserCoQVDbW3lTOTYqV0SgXDYnwIs+qXF0XfvjhB1auXMmFCxfIz8+3ue/AgQMVYpgnoFUpebRLPRZsPMPS7ecY0CbK3SYJblGyrQQq+ybDwMtpueQVGFErJWJqVMym6KrAZc9qwYIFjB49mvDwcA4ePEjnzp2pWbMmZ8+epX///pVho1v51+0xqJUS+8/f4MilNHebI7hFsfassm/SszJ7VfVr+qNSes+OO5ct/eijj/jkk09YuHAhGo2GyZMns379ep5//nnS0yt+s6W7qR3oY/Golm4/515jBLcs1jmrmxUrb0yuQznE6sKFC3Tr1g0AX19fMjNNRZOPP/443377bcVa5yGMvqM+AL8ducK1jDz3GiO4JbFeDczS3VwYWOn5KlmGf/4w3SoQl8UqIiKC1NRUwNQtdNcuU4+dhISEatu/vE3dEDrUC6XAIPP17gvuNkdwC2LtWa06cImOb24gMT23XNeKv1aJewIv7YMv7oflw2DNS1BQcV/uLotVr169WL16NWAqEJ04cSJ9+vThkUcesTtPsLpg9q6W7z6PTn/zdS4CgbPIslyitio5S8f6E1fLdb1KaQ2TEg8r/w2f3QPnt4NSC60Gg7FiNl1DOVYDP/nkE8u2mnHjxlGzZk127NjBAw88wNNPP11hhnkafVtGEBnsQ2J6Hr8eTuShDnXdbZLgFkGnN2Iwloxa7B0ri9TsfFKzTSv4sRVRY5V1DTbPhQPLCoVJgnaPQs/pEFyxnxGXxEqv1/PWW2/xxBNPULeuyZDhw4czfPjwCjXKE1ErFTzetR7vrDvN0u0JDL2tjtfsqRJ4N44q1m9k59s9XhrmfFWdEF/8NOWeFwO6TNjxIexYCAWmsJLG90LvWRBeOS3BXQoDVSoV77zzDnp9xbl23sSITjH4qBUcv5LB3nM3SM8tYPZvJ3j156NcTM1xt3mCaoqj1b/UHNfFyrwSWO4BEYYC2PMpLGgPW+aahCrqNhj5Gzz2faUJFZQjDLznnnvYsmUL9evXrwRzPJtQfw2D29fh2z0Xmf3bCZIy8izTQb7be5FHO8cwrlcjagf6uNlSQXUiM88kVmEBWqb0a8qO+BR+OniZG9kFLl+r3BuYZRlO/AIb34DUeNOxGrFwzwxo8SBUQZThslj179+fqVOncvToUTp06IC/v23c+8ADD1SYcZ7IqG4N+HbPRY5eNtWUxYb5Exniw/a4FJbtPM/KfZcYdUd9nunekGA/MXCiOnH0UjrpuQVVPlfSvBcw0EfFwx2j0agU/HTwsiX35AqWPYG1XchXndsG62fA5f2m3/3C4O6p0GEUKKvuPe6yWD333HOA/fmAkiRhMFTvlbKmEYEMbBvFumOJPNOjIeN6NsJHrWRHXDLv/nmagxfSWLw5nq93neeZHg0Z1a0+/tqbyA0IPAKjUebxJbvJzNNz4NU+VfpFlKUzeVABhe+jGoXtim7cRBjolGd19QRsmAVnCuul1P7QbQJ0Gw/aQJef+2Zx+VNUXRrs3QzzH2mH4eG2aFRFKb9ujcL4sWFNNp68xrw/T3MqKZN3/zAl48f1bMSjXWLQqqpPV4pbjYs3ckjLMYlGRl5BFYuVyQEwi1Won0msUlz0rHLzDVxOM9VmlVq2kH4ZNr0Fh5eDbARJafKiekyBQPe1+RZf+eVAqZDsTr2RJIneLcLp1aw2vx65wvvr/+F8Sg6v/3qCz/5O4P/uacyQ2+p41X6sWx2jUSY5W8eZq1lFx6q4+DmrMGflX9yzys5HlmWnV6XPJmchyxDip7Zcw4bcNNj2Puz+H+gLizmbPwD3zIQw9/dzc1qscnNz2bhxIwMGDABg2rRp6HQ6y/1KpZLZs2fj4yOSywqFxKB2dbivdSTf77vEgo1nuJyWy+RVR/h4azwv9mlK/1YRKMSYL49n8qoj/LD/Et0a1rQcK099081gDgMDfWw9K71RJlOnd3qasnUIaCNwBXmw91PYOg/y0kzHYrpBnzcgulPFvIgKwGmxWrZsGWvWrLGI1YcffkjLli3x9fUFTD3Yo6KimDhxYuVY6oWolQoe7RLDkNvq8PWu8yzaFMfZ69mMW36AllFBTO7XjB5NarnbTEEp/LDfNOx2R3yK5VjVi5VtGOirUeKrVpJbYOBGdr7TYlViTqDRCEdXwl9vQvpF07FazU21Uk36VskKnys4HY988803jB071ubY8uXL2bRpE5s2beLdd99l5cqVFW5gdcBHreTJu2LZOrknE3s3IUCr4viVDEYu2cP+8zfcbZ7ARQwuhoG/H03kw7/O8PL3h1lejr2lxcNAKAoFXVkRtMwJDPOHuA3wv+7w09MmoQqMggc+hGe3Q9N+HidU4IJnFRcXR+vWrS2/+/j4oFAUaV3nzp0ZN25cxVpXzQj0UfN/vRvz7671GPrxDs5ezxZdHDwUo1Fm+Ce77N7nimclyzKTVx2x1Er9fSaZR7vEuGRL8TAQTGJ1OS3XpRXBuGtZtJLO8tDxhbBpp+mgNhjufAG6PAMaz27E57RYpaWl2eSorl+/bnO/0Wi0uV/gmFB/DWH+Ws4WftMJPI9LN3LZcy7V7n2uLIin5RRYhArK7kX1vy3xXM/U8cr9zS15pexiYSBgmbaU6mRhqD75LBNuzGGAdgdcA5Qa6DwW7noR/Go4/4LciNNiVbduXY4dO0bTpk3t3n/kyBHLfkGBa2z55zpNwwOJCBaLE55CQorjLxK9C2p16YZtG5fcgtLrEOf9eZoCg8zjXetRr6apcDNTZycMLCydSM0uw0HIToat76Lc+zkDFAUYZQnaPIyi16sQWs/p1+EJOJ2zuu+++5gxYwZ5eSXDltzcXF5//XXuv//+CjXuVuDQpTRGLtnD2K/2uduUWxp9selFCdezHJzpWunCpRu2e0b1RtnhpCSDUabAYLq2uR4KICvPtigUnPCs8rNh67vw33aw+2MkYwFbDa2ZEPRfFEM/9TqhAhc8q+nTp7Ny5UqaNm3K+PHjadKkCQCnT5/mww8/RK/XM3369EoztLpi/uY9cimduGuZNKpd9ZXBtzr7z6fy+Od7mNKvGSO71QcgIdmxZ+XKVLbinhWYuigE+5b0E/L1RRe+klbkFJjDQJuclV9RrZWtcXo4+JWpbUtW4USmiDb8Gv4sE3YHMzDKe4eeOO1ZhYeHs2PHDpo3b87UqVMZPHgwgwcPZtq0abRo0YJt27YRHl6x1a2zZs1CkiSbW7NmzSr0OdxG4WJLrlX7j18PJ7rJmFubKauOkpNvYObq45ZjCSmOu2i4kmAv7lkB5DkIBa3FKtHas7ITBlo8K3OCXZbh5G+wuCv89oJJqELqwdDPYewWtupN3RC8aU5gcVyqYG/QoAHr1q0jNTWVuLg4ABo1akSNGpWXoGvZsiUbNmyw/K5SVa+i+xyrdrW/HbnCC70biz5ZVYxWVfI7OyG5ZBioUSnI1xtdDANLela5DvpT5Vu5bFfSizyrTDthYE2rKnYu7DJtNL6423Snbw3oMRk6PgEqLVBJ3UGrmHJ98mvUqEHnzp0r2ha7qFQqIiIiquS53IH1Gzf+ejYnEzNpEVV9B8d6In4a2z2bOr2hhMgEalXUreHHycQM9C55VnbEypFnZS1WhZ6VLMuWOYHWYWCov4aG0mVeSPkvLCkUKZUvdH0O7vg/8Am2nCvLstdOtLHG4zepnTlzhqioKGJjY3nssce4cKH0ojqdTkdGRobNzZMp3gXytyNX3GRJ9edGdj493t3Ee3+etjnuo7YVqwspORR3nv47oh2qwu1RRifFSpZlu2Ggo86fNmFg4TCIvIKilsaWMDAjkWZ7X+VPzWTu1O8GSQG3/RueP2DqL2UlVADXs3Rk5ulRSKZZgd6KR4tVly5d+OKLL1i3bh2LFy8mISGBu+66yzL+yx5z5swhODjYcouOjq5Ci53HHOiZ37h1Q03bln49cqXaTglyN59vS+B8Sg4L/4qzOe5bTKzsJdclJMteTmdzVjdyCuxOT3YuZ2UKAzMLC0IlCfyM2bBxNixoT8jJ5SglmT8NHdA/vQMeWAhB9pPnZq8quoZfCWH2JjxarPr378/DDz9MmzZt6Nu3L2vXriUtLa3UbT3Tpk0jPT3dcrt48WIVWuw65pzV/a0j8dMouZiay5FL1W9YrCdQ4KA+ylfjhFhJoCz8hnF2u409rwoc56ysSxoydXoy8grI1hlQo2es5k8UC9vD3/NAn4uxbice0s1gbMGLpAfElmpHfHm7g3oYHi1WxQkJCaFJkyaW5L49tFotQUFBNjdPxuxZ1fDXcE9z02rqr4dFKFhejl9J570/T9uvFHegMdY5qwKD0a5YKSQJVeH2Mmc9K3v5KnCcs9LpbcU08UYOyuOr2Kh5kWnSF5CTAjUbwyNfoxiznn+0rYCym/BZ9gR6cb4KvEyssrKyiI+PJzIy0t2mVBjmN6ifRsnANqbX9duRRKfzIgJb7l+wjYV/xfHen/+UuM/RKp51aJSVp+esA7Eyb4V1Xqxc86ysw8BuimNEfn8fMZsmEKO4TooUCgPmw3O7oPlAkCRqBphW+sraclPuvusehkeL1UsvvcSWLVs4d+4cO3bsYPDgwSiVSkaMGOFu0yocX42KHk1rEahVkZSRx/4LohuDqzz6adHG4xOJzofSEkWlIlk6PecchYHmBLvTYaBrnlW+wUhz6TzL1HNZrnmLoNRj6FX+zCt4mOdqfg4dR4PSakXQsuWmdM/qpifaeAgeLVaXLl1ixIgRNG3alGHDhlGzZk127dpFrVre3wOqeCmVn0aJVqXk3pamMg0RCrqOdc8pawEy40hjDFa5rKSMPK4VTizytwoPJcnkXZnOrwSxSrtAk+2TWKOZTg/lEfJlJQcihvFH73V8aBiMxrek0DjTiz1LpyepsLOHt3tWHl1huWLFCnebUGWYk7wD2kay6sAl1h5NZMaAFqIFshPk5hsY9r+dNsfs1dU60hjruqmjhYsbYQEaNEqFZTXPlLOSSpxfGk6FgTmp8Pd7sOcTIg35IMGvhtt5V/8IHUNvo71syrn62xlIau4YWppnZU6uhwVovX7akkeLVZVi0EPWVQiu45an9yvMm9zZKIwQPzXJWfnsTkjljkZVO/bJG/n18BXLaDQz9sRKdpBht/aUzNdpEOZPolUVuUIq6rvvTD7RVGNVimdVkAu7P4a/PwCd6Tmvh3XhicsDOSqbVvei0nNprDPtFQ3wKflRdaYBX1ExqPfWV5kRX9tmDn0NC28zbVvIrfp8kXmUt1qpoH8rUygoCkTLj7NhoMEos2JvUXnLsUKxqlfT32YoiE0Y6ETO6kZOgWWlN6wwEQ6gwEizxNWwsINpzJUuHcJbwWOr2NTlM47KsZZtNYnpeSXGcFkT6u9gM7MVljmBXh4CghCrIs5uMU302P5fU1uN7QtMjfQrieIfJutan4FtTMV9vx9LKtFO5EJKDmuOJIrCUSs0dvb2Obu9clVhj3UzyVmmfFWAVoXS6iIK6wS7E56VOQSsHagt3Hso00txgN81Uxly8S3IuAzB0TD4f/D0Vmjcm/zC9jD1apo6diam5Vka99kTK3PnhdLGyFeHbTZmhFiZeWgJjPjO1DA/Lw3Wv2b69ju0HIyVP7jVutanS2xNwgK0pOUUsC0u2ea8aT8dYdzyA+wTvdstqJ3M69kT+ONXbMNH666eChvPqqiC3ZmclTkErBvqS0v5H77TzGaJZh5NFZfIVgTCvW/C+H3QdjgoTH97c+lCdKgfkmRaHTxf2P2htDCwNM+qOmxgNiPEyowkmRrlP7sdBn0EQXUg4xL8/Cx8fBf886fj5aQKwFqslAqJ+1vbXxU019ScL6WFya2Gfc+qpGvlTF7cLESminUrsQJLgt2Z1cBLN3KoLyXyWs7bfKKbQhfFKXSymo/1A5lS50vTZGO1bWdY80ZmP62S2oGm0PHMVdPWstLCQEeeVYHByIXC94kIA6sjCiW0fwwm7DfNTfMJhmvHYfnD8MUAuLS/Up62+JaPAW1NoeCfx6/a7CUzL7OnZIl+92bUypLCZC8KtJdgdyQ71nsBoTDB7mzpQtY12h2ZzXrNZNpnbcGIxEp9D+7Wvc9c/QhSDPYHMxQUelZalYLIYNNeUXOrGLthoFmssuyL1fmUbPRGGX+Nkshq0DJbiJUj1L6mVhvPH4Juz4NSC+e3wWe9YOW/ISX+pi5v/cWvVEhoioUyHWJCiQz2IUunZ8s/RcM5zB+UZCFWFpzNWbniGJuKQIt+V0gSYYXeTpKjiUS6TNg0B/7bjs7JP6GWDFyu1Z0nfOczWf80iZgGpZbVIkatVFAnxNfmvtJyVtn5Brubo62LQatDjzQhVmXhVwPunW3ytNo9Bkhw4hf4sBP8Ngkyr978U6iVJd5MCoXE/a1N229+PXyF2b+dYM7vJ63EyvkRTNWd4kIPJnG5nqnji+0JpOeaQmdXg3ibMNCqvUqJvYOGAtjzKSxoD1vmQkE2p5RNeET3GvF9lpCgsO13XlbXBY1SUcITspezCvRRWZL+aTklt9xY9gRWgxAQRJ2V84REw4MfQddxsOF1OPMH7PscDq+AbuNNOQht+fqnFw8BzdzZOIzPtiWwOyGV65m2npTwrIqw5zRIwJhlezlyKZ3t8Sl8+u+Odj0rR96WRPEEu6n2CijajiPLpi+ujW9AaqGnXSMWudcMhqz0IUc2MifUt0RI6rCfVaFnpVEpLPv+zNjzrBQKidDCmrzU7PwS05Gq00ogCM/KdcJbwmMrYdQaqNMBCrJhy9uFU0Q+Ab3rHk/xTpVmzG9Qs2dgjfCsirAnOJKEpdXO+hNXC89z3rcqnmBXSBKxhf3LL97IRX/2b/jsHvh+pEmo/GvBffNg3B5uNLifnHyT8ESF+Jbwmh2GgfoisaoTUsyzsiNWUFTFbm/LjSUMrCaelRCr8lL/TnhyIwz7Emo0hJxk+P1lWNQZjq0qcxKm9fvX185WCgCtynZJ2xrhWZXOhpPXShyz61mVEhwWT7DXDtTSRnOZT5TvoPpyAFzeD2p/6DEVnj8InZ8CpdqmxspHrSzhWeWV0XVBrSxKsJuxFwaC4yp2WZYtBaHVoXodRBh4c0gStBgETe+DA1+axh/dSIAfnoDIBdDndYi9u8zLOPKstGrH3yWp2fkYjbLNB+pWxVl/yZ4wfb3LfptsyWr1D0CTfQXplwX8pFiOEiNGSYmiwyjoMQUCbac6Tf7hCIDD+YBlJdi1KgVRTiTYwfFm5sT0PHLyDagUkmVYqrcjPKuKQKmGTmNM3649XwFNACQegi8HwVdDIPFIqQ93KFZ2VrnMGIxymU3XBLa4tBqIaZU2iGymqr6l3vK74NDXKDGyxtCZH25fBQPeLyFUAKeSTLVRN8xJ72LfJ3qjbNdbLrDOWflrLAsHKoXk8L1grrVKKZYWMIeA9Wr6OV006+lUj1fhKWgDTCOQnj8EnZ8GhRriN8L/usOPY+HGebsPK94D3HI5Ven9slPK6GN0q+CsCLmyGqiUddyX9QNbtS/wjOpXFAYd1LuDr1stYVzBCxzOsb/B3N5WHHu+rz3vyjoMVCgkS8I8wEflsPSghoOcVXXaE2hGiFVlEFAL7nsHxu+BVkMBGY58Bx92hHXTITvF5vTyeFYAyZnVO2+VkqWj8382UH/qGrYX23Z0KimD5bsvuNRRtXjTPHvbVCSMDFb8zdOHH+HRtE8IkbI5baxL0v3LYNQafBp0ARxPbE60U4Nlz0J75Qs6q9IFgKjCJLujEBCsx8jb96yqy0ogiJxV5VIj1rTnsNsEWD8TErbArkVw8Cse0A5lP93JQ+s4wV5KzgpMI5aqM7N+PWFphPfYZ7s5N/d+y3395v8NmATdPBmoTKxUIyk9j7vnbbK5s7viCFNVK2ihOA/5kKoMY07eEFYZurMpthdIUsnyhWLYO643FD2xJJk8QXutja1XAwGiCpPspYlVDX9Tj6rinpUQK0H5iGoP//4F4v+CDTMh6SiP6JbSQ/szH+gfwl/9b7sPs1fsaE3xPEV1w/yBK43Dl9KcFitrD2fW6uPkFZjEoZV0lmmqb7lDaRofnyH7cSBmNN8r72fNqTSgqEtGbKFYXUnPIzff4NRkHL1Voj1QqyIjT2+31so6ZwUQ6YxnZWnAZ1veUt0KQkGEgVWHJEGje2DsVhjyKdeU4URIN3hb/SnPnvg3nFpTIvmiUiosm2ftUd3LF5ypiyowlFZ84Ph6644nES1dZYF6Ib9pX+UO5XF0sopP9ffRXfcBe+r8G6OqqNbJnDIK9dcQ7GvyZs6nlhQme55VvpVnFVT4WLs5K4NtGBgdatpDGFJKh8+a/qbiUeuQNj2nwPLe8Pa+69YIz6qqUSigzTAm744k9tx3jFf9RM3cBFjxKER3MW2ejrndcrpWpUDvoC6n+otV2ecYyqhns7le4b81yGCC6iceU25AIxkwyhI/G+/gff3DXJKL+vvb1FlZ/dwgzJ9DF9NIuJ5NswjbUW/2PKt8fdHfz9ye2F7OqngYeF+bSE4lZTKonf3hpQCh/kVDI2RZRpIk4q6bViMjg31K9cq8jerzSrwMg0LDEkN/vjf04OvmO2l7cTlc3A1L+kLT+01jwGs3Q6tW2p3qC9W/it0Zn0lvlJ1eDVQbchmn/JlnVL8SKJn6TW0xtOFt/XBOyPVtzrXXIsaMWazsjexKSCl5rMDKszKHjfbDQNN5ZrEK8lEz64GWpb4mc51VvsFIdr6BAK2K+GvVLwQEEQa6nUz8ON1ioqlG67aRICng9BpY3BV+GU9dpeMme+VtE/PGrycY9r+dDosWPQWn+k8Zyj5JiQHj3qW8dvZfvKxeSaCUy1FjfR7Nn87IgqklhApMOSplsQp2M46S7HqDkYupJfuMWf8/m8tUyipdcBZftdKyamwOBatTwz1rhFh5AL4aJQRFwgML4Lnd0GwAyEY4+BUr88fxsmoFQWSXKGUor2e1ZHsCexJS2Xz6etknuxFnclZ6o7EUD0zmXsVe/tBMQbHmBYINKVww1uL5/PE8kP8mO4ytbM5uFx1i87vCZm9g0fH6Yfa7L1xJy7PxoopsLOlZ2dtyU7x0wRkkSSqx5aa6zAksjhArD8CmzqpWExj+DTzxJ0Tfjg/5jFOtZot2IhN816GhaNXnepbOqQ+0LMt8tfMc+8+n2hz3dM/KmejOnjgAdJBO84PmdT7RfEAjxRWMvjX4PmwcvfPnsdrYDdnOWz+6RlFTvOL9rKyLMs0rgueKhXz2QsDilO5ZmY7Z689VGqHFerEXFYRWj202ZoRYeQB2W8TEdIEn1vFm4GucMdYhVMpifMEX/KV9kcGKv5Ewkq83kqXTl3xsMTaevMZrvxxn6GLb2XrODuusTGRZZtTSPUz49qCd+8p+fFaenj+OJVl+byhd5hP1e6zSvk5HxT/kyhoW6h/kyr93sil0KPk4XlkL0FoNNYUS023MmD2r5Kx8MvKKvjwc1V5Z40zOqqxi4OJY92LPKzBYQlERBgoqHD8HRaFIEof8utEvfy6TC54iU12LulIyH2gW87v2FXooDluq2F/7+RgTvj1o19P61cFIL2fHoFcmF1Jz2Hz6Or8evlJiv5wzXuPOsyks23me2tzgLdWn/KmZzL3K/RhkieX6nvTQfcB7+mFkS/5lip/NIFFJsvGmrEPCAK2KWoVdQ60FylFVuzWlelYG13NWYNt54VxKNkYZgnxU1CrWE8vbEWLlATjabgOmKnYDSlYaevJ1p5+Q75mJrA2imXSeZZq3qbnqIeTLB/hq13l+PXzFEgJY88sh+2LlAVplY0Nx8VQ42A9nTQA5vKhayRbtRB5VbUIpyfxp6MC9+e8wXf8U1wgFTOJQljj7FVvmt3724uVuDezkrYqHhfaw5KyKiZXBKFs8XVfDQGuxqm6tjK0RYuUBONrIDLabmZU+fkh3TUL6v8Os9h2MTlYRlLQT6dOefKheQD0pyVKVDbDtTDL3/fdvh9d2Vxi45Z/rPLlsL9cy8mzCq+Jioio2CKL+1DWs2HOBQxfTUKNnlHKdKZen+hlfKZ99xiYM1c1kbMGLxMu2k7Vz8vVlinPxMNDatuIf/AaFbVfOXrcSKyc8Kx+zZ1UsDLT2Ksubs7qRUyRWjapZ2QKIOiu3Yf3mL9WzsnrjWlaJ/GrwW+R43j5xN1/Grif2yhoGKHfRV7GXG9sOwf0zIKAWn207y4nEDIfXdlcYOHLJHgD0xiPMGNDCcry4eCoVJT+00348zOcdLrBR8wExCtNqZrwxkrf1w/nT2BH7PQ5M4lCWNvtbeVaSZDuItriT0qCWbZK9wGDkooNx8db4OchZ5Vstdtib1lMaNawKQ7N0putWt3wVCM/KI3CYs8JWrLRWHlhYoJbL1GJ1g5kYntrCJkNb1JKB2ie/hAXtYPNc1IbSPzwXb+Sy+bRtR01ZlsnJLztpXxFsPn2dXu9tsfxuMMpMWnmIOWtP2j2/m+IYv2heo9fx6cQornNNDmFawRjuzX+HP42dcCRU4LjvuTXW1d4Skk3oVzwkLT484tKNXAxGGV+1ktBStseYvejiYaCNZ+VizqpojHxBtWtlbI3wrNyMJIFPKd0VrMNAa+EKK0yeJmfpkCNaM7pgCl0Nx/k0YjUBqUdh8xzeVYQwTzmYFYae6O38qRdsPAPA9PuaMbZ7QwBe/P4wPx64zB8vdKdpRPkGYDgir8DAtjPJDu8/fiWDHw9cBmDafc0tCfbm0nmmqr6lh9LUxDBT9uV/+gF8buhPLs7NwzMltIuFmQrJpgaq+JeG5KDOCrD0Y09IzkaWZUsIWK+mn8P6N7VScphgt94X6GquydzTKjlLx+U00xeU8KwEFY6vnTFc1li3ibEWrrCAog6R5nBup7El+/v+CA8thdAGhBjTeFO9lPWal7lPsQuw36HyrbWnSCocpmkWi0+2nr3p11acub+f4skv9zm832wDwNFL6WQmneV99Ues0Uynh/II+bKSpfq+9NB9wIeGwU4LFdgPAwOL9TX3t85ZScVyVsW8tpgaphHvmXl6UrLzLVtvGoT548gxUisV+Gjs56wKiu0LdAWzZ5WQko1Ob0SjVNjUjFUXhGflZkrLV0GxMNCRZ2X1IVx7NIl3E2vx/sMb+fHT/zDG+D0NFFf5SLOAQ8ZYFn9xDSjZijf+epbNKCfnexk4zze77XdKNfPi94cBCCGTI0vG8Zf2d7SSKSRdbejKPP0wLsglbXeGnHxDiVIIf62qqPUwxcNA29Cv+PeJj1pJVLAvl9NyOZecbfGs6of5W6bqFEetVOBX6FnlOPCsXM1XAdQsFCvzyzMJZvVaCQQhVm7D/FZyNDPQjLU3pXEgVtZ8t+8iAKuPJvOprjdfGbvxlGoNTynX0E5xlnaXJtJG3Za39SM4JcdYHncyMYN6NYu+jYvPKawIQvw0pV5XSz6jlet4TrWaIGMOSLDd0JK5+hEclWNv6rlz8/Ul5Lf4KmyJMNDqZ3tlFLG1/LmclsvZ5GxLor1BTX/srAsAJrFytN2meMcFVwgpDAPNVMcQEIRYuR0/del/AkeeVc0Ac54i3+6S/KGLaRiMMtn4Ml//EF/r+/C86kdGKP+ip/IwPRRH+Ml4J+8VPMwVwnhzzUneXFOU2P77TDKLN8fTtWHNEnvmXGXGL8eo6a91mJtTYGSocisTVT8QJZm2BJ00xjBXP4ItxjaUljh3loMX00oIjrVXWzfU19azKhYG2nNU6tf05+8zyZxLzrYk2uuH+TusD9MopaLShWKele4mxEqjUhCoVZFZuJuhuu0JNCPEys2U6VmpHawGFnpWWTo9Z5NLFoIevGDbrSGZYGboR7PE0I+XVN8zQLmLocq/GaDYxTLDvSzSDyId2zf52+tOAdi0E3aFpPQ83l9/mpX7LgH2vvFleikOMkW1gqYK0zmX5DDeK3iYn4132N2/V17+PpNM9ya1bI5Z/98ve6KzTc4qX28sFgaWFCBzYejppEyuFCa264f52bSWsUatUjhMsBeUs3rdTKi/pkisqtmeQDNCrNxM2Tkr+6uBQT4qNEoF+QYjG06UHOjpqAfWOTmS8QXP84n+fqapvqWr8gRjVWsYrtzER/oHWGrohw6N3ce6yvBPdnIupahlivVwh3ZSHNPUy+miMAlimuzPh/oH+crQp8KevzjFQy/rsK9+sdl62fkGm6JUe56VWax2nk3BKIO/RkmtAK3DWY5qpcLy93ZUFOpq2YKZUH8NF6rpnkAzQqzchPnL15UEu3WIIEkSNQM0JKbnseHkVZef/4jckBEFr3C34TBTVN/SXHGRqeoVjFT9yfv6h1hl6I7RyrNJTM8lM09Pk3DnyxmshQrgbHI29aVEXlZ9x/1KU2GoTlaz1NCPj/QDyaByP2TpubZ9yq1zVgrJ1nvKydcT7FskmqV5VuYarvph/kiS5LAVtXXOKrfAYOnsCUVi5eomZjM1Cmu7JAliw4RYCSoBR5NtzNiWLti+kcMCtCSm53H0sv3Vp7KR2Gxsx9b8NgxSbOdF9ffUlZJ5V/0JTynX8LZ+OBuNtwHQdc5fAOyefg/hQaWXDNhrPRNGOv+nWsVw5SbUha2EfzB05wP9QyRSs5z2u0YJsdJYlyrYCky2zkBIGav/dUN9bWq1zN0YujUM41RSZom/l3XOymCUKTDIaFSm5y0+LMJVzOULdUJ8y0wteCtCrNyMXyn7AqF4GGh7rrnW6mYxouAn412s1XXhceV6xqt+poniMp9r3mOPsSlDpp0BmgBw5mpWqWJ1MTWHu97ZRHiQKafmT65lNdJfMq0EbjS05239cP6RoyvEfmcpzbMqTk6+3m7oZ41KqSCmhl9RjVVhKPly36bUCfWlT3PbMgu1UmHznLkFBos4lbfjghlz+UJ1DQFBFIW6nbJLF6wT7CU9K3uUd/VOh4bPDPfTXTefj/QPkCer6aw4zY/aWXys/oBY6Qq/HrbfwcHMXe+YZvGlZGTzL+V6Nmsn8oLqR/wlHYeMDXlE9xpjCl6ucqGCkknt0v7vs3UGp7o+mENBKPKsfDVKxtzZgJiatq6ZWqlAoyqaWPTbkSuW3NXNrAYCtK4bAkC3hlXjpboDr/CsFi1axLvvvktSUhJt27Zl4cKFdO7c2d1m3RTmj0F5E+wANR2IVd+WERy6mFZu2zLw5x39cL7U9+EF1SoeVm6hn3IvvRX7WXnobu45NoxGDRvzxB0N6BJr+nAYjTLnU3MAmf6KPbys+o5YhakpXoIxnHf1j7DW2IWKKEOoKPxLCcFz8vVOTYapbyVWDcJKjxvNQhRT04+z17N55adjzF17ioHtoiyJ9fIm2B9oG8XtDWpY+mxVRzzes/ruu++YNGkSM2fO5MCBA7Rt25a+ffty7VrJFTBvpEyxsvKmir+R7YWBWpWC7k3CKsS2JGoyVT+Wfvlvs97QAZVk5FHVX/wmP0+r0wt58pO/SEw3Ldnft+Bvpr63mJ81M1is+S+xiiSuy0G8WjCaPvnvstZ4O54kVGC7vaY42fmGUuf1mbHxrGqWXjJgrk5fMfZ2JvVpQt1QXzJ1epbvvsAXO86ZzimnZwVQO8in2vWwssbjxer999/nqaeeYvTo0bRo0YKPP/4YPz8/lixZ4m7TKoQyE+yFb16NquQGV+tv0VZ1TPPrYmsF0LBWQIntITfDGbkuTxW8yMO6Gew3NsZXymeC6me2aF/gk3de5tPvfuLllBl8p51NO0U82bKW+foh3K37gK8NfexuovYEQv0c5/zy9Uab1UBHmMUq0EdlaYLniDaFoVrtQB+ev6cxW1/uyfKnuvBguyjL37l2NfaMbhbPfBcVkp+fz/79+5k2bZrlmEKhoHfv3uzcudPuY3Q6HTpd0ZaOjAzH/ZzciVl4nA0DtXbCA/M0XqVC4raYUI5dzqBR7QB81EpiavhxPiUHtVJyOFTBVfbKzRiaP4t7FfuYolpBQ0UiM9VfwcmvQAl6WcG3hl4s0A/hOiEV8pzlxV+jRKc32nRVsKZuqC8D2kby86HLtIwKthx/Z2gbZq85wfvD2hJb2GYlyMfxx6Rj/VB6Nw+nc4NQh17NbxPuZNOpa4ztYbtlSKGQ6NYwjG4Nw3g9t4C9Cal0rcY5p5vFo8UqOTkZg8FAeLjtqkp4eDinTp2y+5g5c+bw+uuvV4V5N8WgdlEkZ+m4q3HpIVtsLX861Q+1+UCZaREVRIifms71a9CrWW1+PniZ/q0iAJjUpwkbTl5jSPs6fLQ5jr3nHM8fdA2JP42d2Jh/Gw8rtzBR9QPhUhprDJ2Zp3+EBDmygp7Hlnua1WbjKdvQf0q/ZuyIT+afq5l0ql+D22JCWbQpjtwCAwdm9CFHZ+Bapo4GYf4cuZRmeVxUiC81/DVoVUq+GtPF5prDOkXzUIe6lsLOfa/2LrOT62cjO5Zqe6s6wbSqU/LvZ02wr5reLcq3SftWQZKd6crvJq5cuUKdOnXYsWMHXbt2tRyfPHkyW7ZsYffu3SUeY8+zio6OJj09naCgoBLnezs6vcHSA8m6yLA8yLJMUkYeQT6mXE16bgFGWSYjV0+WTs+F1Bx0egMXUnM4eD6NNnWD2X82kcdb+7E5yYdmkYHUr+lPbr6BBrX8USsURAT7oFZKKBQSaoUCH7UCWTbvvau++RWBc2RkZBAcHOzU59OjPauwsDCUSiVXr9pWaF+9epWIiAi7j9FqtWi1t07cb71aeLMffkmSiAz2tfxuafNrmrlA5wY17DzK1JZ4iEvPUz77BLc2Hp1g12g0dOjQgY0bN1qOGY1GNm7caONpCQSC6o9He1YAkyZNYuTIkXTs2JHOnTszf/58srOzGT16tLtNEwgEVYjHi9UjjzzC9evXmTFjBklJSbRr145169aVSLoLBILqjUcn2CsCVxJ4AoGganHl8+nROSuBQCAwI8RKIBB4BUKsBAKBV+DxCfabxZyS89RtNwLBrYz5c+lM6rzai1VmZiYA0dFV3z9JIBA4R2ZmJsHBpW9JqvargUajkStXrhAYGFjl2zvMW30uXrxYbVcib4XXCLfG63THa5RlmczMTKKiolA4GrhYSLX3rBQKBXXr1nWrDUFBQdX2DW7mVniNcGu8zqp+jWV5VGZEgl0gEHgFQqwEAoFXIMSqEtFqtcycObNad4G4FV4j3Bqv09NfY7VPsAsEguqB8KwEAoFXIMRKIBB4BUKsBAKBVyDESiAQeAVCrCqQ1NRUHnvsMYKCgggJCWHMmDFkZWWVev6ECRNo2rQpvr6+xMTE8Pzzz5Oenl6FVpfNokWLqF+/Pj4+PnTp0oU9e/aUev73339Ps2bN8PHxoXXr1qxdu7aKLL05XHmdn376KXfddRehoaGEhobSu3fvMv9fPAFX/5ZmVqxYgSRJPPjgg5VrYGnIggqjX79+ctu2beVdu3bJf//9t9yoUSN5xIgRDs8/evSoPGTIEHn16tVyXFycvHHjRrlx48by0KFDq9Dq0lmxYoWs0WjkJUuWyMePH5efeuopOSQkRL569ard87dv3y4rlUr5nXfekU+cOCG/+uqrslqtlo8ePVrFlruGq6/z0UcflRctWiQfPHhQPnnypDxq1Cg5ODhYvnTpUhVb7jyuvkYzCQkJcp06deS77rpLHjRoUNUYawchVhXEiRMnZEDeu3ev5djvv/8uS5IkX7582enrrFy5UtZoNHJBQUFlmOkynTt3lseNG2f53WAwyFFRUfKcOXPsnj9s2DD5/vvvtznWpUsX+emnn65UO28WV19ncfR6vRwYGCgvW7assky8acrzGvV6vdytWzf5s88+k0eOHOlWsRJhYAWxc+dOQkJC6NixaOBl7969USgUducbOsLc3lWlcv+2TfNE7N69e1uOlTURe+fOnTbnA/Tt29fh+Z5AeV5ncXJycigoKKBGDXvjytxPeV/jG2+8Qe3atRkzZkxVmFkq7v9EVBOSkpKoXbu2zTGVSkWNGjVISkpy6hrJycnMnj2bsWPHVoaJLlOeidhJSUl2z3f2/8AdlOd1FmfKlClERUWVEGpPoTyvcdu2bXz++eccOnSoCiwsG+FZlcHUqVORJKnUm7Nv6NLIyMjg/vvvp0WLFsyaNevmDRdUGXPnzmXFihX89NNP+Pj4uNucCiEzM5PHH3+cTz/9lLCwMHebAwjPqkxefPFFRo0aVeo5sbGxREREcO3aNZvjer2e1NRUh9OjzWRmZtKvXz8CAwP56aefUKvVN2t2hVCeidgREREune8JlOd1mpk3bx5z585lw4YNtGnTpjLNvClcfY3x8fGcO3eOgQMHWo4ZjUbAFDGcPn2ahg0bVq7RxXFbtqyaYU6w79u3z3Lsjz/+KDPBnp6eLt9+++1yjx495Ozs7Kow1SU6d+4sjx8/3vK7wWCQ69SpU2qCfcCAATbHunbt6hUJdldepyzL8ttvvy0HBQXJO3furAoTbxpXXmNubq589OhRm9ugQYPkXr16yUePHpV1Ol1Vmi7LslgNrFD69esnt2/fXt69e7e8bds2uXHjxjalC5cuXZKbNm0q7969W5Zlk1B16dJFbt26tRwXFycnJiZabnq93l0vw4YVK1bIWq1W/uKLL+QTJ07IY8eOlUNCQuSkpCRZlmX58ccfl6dOnWo5f/v27bJKpZLnzZsnnzx5Up45c6bXlC648jrnzp0razQa+YcffrD5u2VmZrrrJZSJq6+xOO5eDRRiVYGkpKTII0aMkAMCAuSgoCB59OjRNm/ehIQEGZA3bdoky7Isb9q0SQbs3hISEtzzIuywcOFCOSYmRtZoNHLnzp3lXbt2We7r0aOHPHLkSJvzV65cKTdp0kTWaDRyy5Yt5TVr1lSxxeXDlddZr149u3+3mTNnVr3hLuDq39Iad4uVaBEjEAi8ArEaKBAIvAIhVgKBwCsQYiUQCLwCIVYCgcArEGIlEAi8AiFWAoHAKxBiJRAIvAIhVgKBwCsQYiVwC6NGjbLbwaJfv37uNk3goYiuCwK30a9fP5YuXWpzzNE04IKCghLdKPLz89FoNC4/b3kfJ3AvwrMSuA2tVktERITNLTQ0FABJkli8eDEPPPAA/v7+/Oc//2HWrFm0a9eOzz77jAYNGlh6R124cIFBgwYREBBAUFAQw4YNs2mF4uhxAu9CiJXAY5k1axaDBw/m6NGjPPHEEwDExcWxatUqfvzxRw4dOoTRaGTQoEGkpqayZcsW1q9fz9mzZ3nkkUdsrlX8cQLvQ4SBArfx22+/ERAQYHNs+vTpTJ8+HYBHH32U0aNH29yfn5/Pl19+Sa1atQBYv349R48eJSEhgejoaAC+/PJLWrZsyd69e+nUqZPdxwm8DyFWArfRs2dPFi9ebHPMeuCC9fANM/Xq1bMRnJMnTxIdHW0RKoAWLVoQEhLCyZMnLWJV/HEC70OIlcBt+Pv706hRo1Lvd+aYs88l8G5Ezkrg1TRv3pyLFy9y8eJFy7ETJ06QlpZGixYt3GiZoKIRnpXAbeh0uhIjulQqlUvTVHr37k3r1q157LHHmD9/Pnq9nueee44ePXrYDSMF3ovwrARuY926dURGRtrc7rzzTpeuIUkSv/zyC6GhoXTv3p3evXsTGxvLd999V0lWC9yFaGssEAi8AuFZCQQCr0CIlUAg8AqEWAkEAq9AiJVAIPAKhFgJBAKvQIiVQCDwCoRYCQQCr0CIlUAg8AqEWAkEAq9AiJVAIPAKhFgJBAKvQIiVQCDwCv4fPout5kb/r84AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018681, + "end_time": "2024-03-23T12:10:06.523501", + "exception": false, + "start_time": "2024-03-23T12:10:06.504820", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 5527.109229, + "end_time": "2024-03-23T12:10:09.264788", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/realtabformer/2/mlu-eval.ipynb", + "output_path": "eval/treatment/realtabformer/2/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/realtabformer/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "realtabformer" + }, + "start_time": "2024-03-23T10:38:02.155559", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/realtabformer/model.pt b/treatment/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..4867f29d7b738b22ab1bb4f16e8354235ca2db0c --- /dev/null +++ b/treatment/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0acff5eb9c30e75b1ef15ed8122fe200e94020ff0583fdfc871bca2015d3ce43 +size 78481207 diff --git a/treatment/realtabformer/params.json b/treatment/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..21dd1fb24b6a3a973ea2117d8477393e2a5d679b --- /dev/null +++ b/treatment/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "NONE", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.1, "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "dataset_size": 2048, "batch_size": 2, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "fixed_role_model": "realtabformer", "mse_mag": false, "mse_mag_target": 0.2, "mse_mag_multiply": false, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/tab_ddpm_concat/eval.csv b/treatment/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..d2aadcd5c159b36f57c749353bedf0f79b625e9a --- /dev/null +++ b/treatment/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,0.0,0.00933234132444691,0.0034467438412026413,11.47805142402649,0.012975177727639675,0.434597909450531,0.030787667259573936,5.471021358971484e-05,6.269394874572754,0.03615765646100044,3378291.5,0.058708976954221725,0.24351681768894196,2.0739604224218056e-05,17.747446298599243 diff --git a/treatment/tab_ddpm_concat/history.csv b/treatment/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..84e7fa09ef7849d033d5dc5ef8f46379df012da9 --- /dev/null +++ b/treatment/tab_ddpm_concat/history.csv @@ -0,0 +1,9 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.1593249796038597,38.32478675916096,0.08493479980118587,0.0,0.0,0.0,0.0,0.0,0.1593249796038597,900,225,296.148889541626,1.316217286851671,0.32905432171291776,0.18687162672061783,0.012289107807673581,1.519535903538306,0.0025395565246152793,0.0,0.0,0.0,0.0,0.0,0.012289107807673581,450,113,110.64389157295227,0.9791494829464803,0.2458753146065606,0.11459510927767219 +1,0.009436246975670252,0.16965037501825211,0.00024518938042836187,0.0,0.0,0.0,0.0,0.0,0.009436246975670252,900,225,292.4781291484833,1.2999027962154812,0.3249756990538703,0.23130405430164602,0.01923421653492369,7.067227336121174,0.0030869791105636382,0.0,0.0,0.0,0.0,0.0,0.01923421653492369,450,113,113.68414735794067,1.006054401397705,0.25263143857320147,0.09829258351315585 +2,0.0072775980280271645,0.20101061383553606,0.00024995846221528785,0.0,0.0,0.0,0.0,0.0,0.0072775980280271645,900,225,295.3896453380585,1.3128428681691489,0.3282107170422872,0.22988744627076407,0.0073902537548176245,1.8595952647372243,0.0004061482044698016,0.0,0.0,0.0,0.0,0.0,0.0073902537548176245,450,113,112.0707745552063,0.991776765975277,0.24904616567823623,0.11153677370851504 +3,0.006081614995972207,0.28913738945182105,0.00010754254828174891,0.0,0.0,0.0,0.0,0.0,0.006081614995972207,900,225,295.19276666641235,1.3119678518507216,0.3279919629626804,0.22885953691581057,0.008240144269055791,3.4400916960033965,0.0006446722911166196,0.0,0.0,0.0,0.0,0.0,0.008240144269055791,450,113,112.21956896781921,0.9930935306886656,0.24937681992848715,0.09865027220083766 +4,0.005871379718669737,0.1360336290616753,0.0001008218179352885,0.0,0.0,0.0,0.0,0.0,0.005871379718669737,900,225,295.90435433387756,1.3151304637061225,0.32878261592653063,0.23736182113048723,0.007101165456341442,2.645643536452133,0.000494350846501724,0.0,0.0,0.0,0.0,0.0,0.007101165456341442,450,113,112.5444118976593,0.9959682468819407,0.25009869310590954,0.10926411079190156 +5,0.0059188109788475335,0.13654618756688353,9.134781137381922e-05,0.0,0.0,0.0,0.0,0.0,0.0059188109788475335,900,225,294.9814076423645,1.3110284784105088,0.3277571196026272,0.24004095225698419,0.009655498555774026,4.006451106402225,0.0008652036921123643,0.0,0.0,0.0,0.0,0.0,0.009655498555774026,450,113,112.00565361976624,0.9912004745112056,0.24890145248836942,0.11541654484422117 +6,0.00488773422920935,0.12989713646643514,7.097226211497443e-05,0.0,0.0,0.0,0.0,0.0,0.00488773422920935,900,225,295.9172704219818,1.3151878685421414,0.32879696713553536,0.23536086357302136,0.0074696344484894124,1.8159953127400679,0.0006341720502412485,0.0,0.0,0.0,0.0,0.0,0.0074696344484894124,450,113,113.0309898853302,1.0002742467728336,0.251179977522956,0.10902682123553786 +7,0.004355493340135581,0.2025226039046048,3.489317478565809e-05,0.0,0.0,0.0,0.0,0.0,0.004355493340135581,900,225,295.52129220962524,1.3134279653761123,0.32835699134402807,0.24828977358152365,0.006594920561917976,4.189112022148085,0.00016057833332648575,0.0,0.0,0.0,0.0,0.0,0.006594920561917976,450,113,112.15122652053833,0.9924887302702508,0.2492249478234185,0.1086470079413002 diff --git a/treatment/tab_ddpm_concat/mlu-eval.ipynb b/treatment/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..5b02b3460a1c244ecf293826530afe3fc4e36314 --- /dev/null +++ b/treatment/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2374 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:26.011670Z", + "iopub.status.busy": "2024-03-23T03:48:26.010897Z", + "iopub.status.idle": "2024-03-23T03:48:26.044452Z", + "shell.execute_reply": "2024-03-23T03:48:26.043556Z" + }, + "papermill": { + "duration": 0.049178, + "end_time": "2024-03-23T03:48:26.046652", + "exception": false, + "start_time": "2024-03-23T03:48:25.997474", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:26.072921Z", + "iopub.status.busy": "2024-03-23T03:48:26.072532Z", + "iopub.status.idle": "2024-03-23T03:48:26.079456Z", + "shell.execute_reply": "2024-03-23T03:48:26.078611Z" + }, + "papermill": { + "duration": 0.022547, + "end_time": "2024-03-23T03:48:26.081414", + "exception": false, + "start_time": "2024-03-23T03:48:26.058867", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:26.105980Z", + "iopub.status.busy": "2024-03-23T03:48:26.105684Z", + "iopub.status.idle": "2024-03-23T03:48:26.109984Z", + "shell.execute_reply": "2024-03-23T03:48:26.109163Z" + }, + "papermill": { + "duration": 0.01928, + "end_time": "2024-03-23T03:48:26.111928", + "exception": false, + "start_time": "2024-03-23T03:48:26.092648", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:26.135389Z", + "iopub.status.busy": "2024-03-23T03:48:26.135095Z", + "iopub.status.idle": "2024-03-23T03:48:26.140123Z", + "shell.execute_reply": "2024-03-23T03:48:26.139360Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.019132, + "end_time": "2024-03-23T03:48:26.142080", + "exception": false, + "start_time": "2024-03-23T03:48:26.122948", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:26.166659Z", + "iopub.status.busy": "2024-03-23T03:48:26.165895Z", + "iopub.status.idle": "2024-03-23T03:48:26.171930Z", + "shell.execute_reply": "2024-03-23T03:48:26.171072Z" + }, + "papermill": { + "duration": 0.02035, + "end_time": "2024-03-23T03:48:26.173917", + "exception": false, + "start_time": "2024-03-23T03:48:26.153567", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a9fa00a8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:26.201509Z", + "iopub.status.busy": "2024-03-23T03:48:26.201168Z", + "iopub.status.idle": "2024-03-23T03:48:26.206592Z", + "shell.execute_reply": "2024-03-23T03:48:26.205701Z" + }, + "papermill": { + "duration": 0.02254, + "end_time": "2024-03-23T03:48:26.208631", + "exception": false, + "start_time": "2024-03-23T03:48:26.186091", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 1\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/tab_ddpm_concat/1\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011253, + "end_time": "2024-03-23T03:48:26.231262", + "exception": false, + "start_time": "2024-03-23T03:48:26.220009", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:26.255876Z", + "iopub.status.busy": "2024-03-23T03:48:26.255113Z", + "iopub.status.idle": "2024-03-23T03:48:26.264492Z", + "shell.execute_reply": "2024-03-23T03:48:26.263692Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023952, + "end_time": "2024-03-23T03:48:26.266446", + "exception": false, + "start_time": "2024-03-23T03:48:26.242494", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/tab_ddpm_concat/1\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:26.290413Z", + "iopub.status.busy": "2024-03-23T03:48:26.290131Z", + "iopub.status.idle": "2024-03-23T03:48:28.341963Z", + "shell.execute_reply": "2024-03-23T03:48:28.341055Z" + }, + "papermill": { + "duration": 2.066361, + "end_time": "2024-03-23T03:48:28.344234", + "exception": false, + "start_time": "2024-03-23T03:48:26.277873", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:28.371532Z", + "iopub.status.busy": "2024-03-23T03:48:28.370503Z", + "iopub.status.idle": "2024-03-23T03:48:28.395734Z", + "shell.execute_reply": "2024-03-23T03:48:28.394923Z" + }, + "papermill": { + "duration": 0.041126, + "end_time": "2024-03-23T03:48:28.397918", + "exception": false, + "start_time": "2024-03-23T03:48:28.356792", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:28.422616Z", + "iopub.status.busy": "2024-03-23T03:48:28.422312Z", + "iopub.status.idle": "2024-03-23T03:48:28.432214Z", + "shell.execute_reply": "2024-03-23T03:48:28.431381Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.024416, + "end_time": "2024-03-23T03:48:28.434140", + "exception": false, + "start_time": "2024-03-23T03:48:28.409724", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:28.459598Z", + "iopub.status.busy": "2024-03-23T03:48:28.458874Z", + "iopub.status.idle": "2024-03-23T03:48:28.956836Z", + "shell.execute_reply": "2024-03-23T03:48:28.955870Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.513483, + "end_time": "2024-03-23T03:48:28.959376", + "exception": false, + "start_time": "2024-03-23T03:48:28.445893", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:28.985907Z", + "iopub.status.busy": "2024-03-23T03:48:28.985588Z", + "iopub.status.idle": "2024-03-23T03:48:42.322700Z", + "shell.execute_reply": "2024-03-23T03:48:42.321716Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 13.353564, + "end_time": "2024-03-23T03:48:42.325461", + "exception": false, + "start_time": "2024-03-23T03:48:28.971897", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-23 03:48:33.604220: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-23 03:48:33.604369: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-23 03:48:33.748949: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:42.353271Z", + "iopub.status.busy": "2024-03-23T03:48:42.352575Z", + "iopub.status.idle": "2024-03-23T03:48:42.372549Z", + "shell.execute_reply": "2024-03-23T03:48:42.371806Z" + }, + "papermill": { + "duration": 0.036516, + "end_time": "2024-03-23T03:48:42.374647", + "exception": false, + "start_time": "2024-03-23T03:48:42.338131", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:48:42.399509Z", + "iopub.status.busy": "2024-03-23T03:48:42.398692Z", + "iopub.status.idle": "2024-03-23T03:49:05.197343Z", + "shell.execute_reply": "2024-03-23T03:49:05.195939Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.814101, + "end_time": "2024-03-23T03:49:05.200350", + "exception": false, + "start_time": "2024-03-23T03:48:42.386249", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-23T03:49:05.233285Z", + "iopub.status.busy": "2024-03-23T03:49:05.232931Z", + "iopub.status.idle": "2024-03-23T03:49:05.239603Z", + "shell.execute_reply": "2024-03-23T03:49:05.238714Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023496, + "end_time": "2024-03-23T03:49:05.241796", + "exception": false, + "start_time": "2024-03-23T03:49:05.218300", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:49:05.268458Z", + "iopub.status.busy": "2024-03-23T03:49:05.267811Z", + "iopub.status.idle": "2024-03-23T03:49:05.272739Z", + "shell.execute_reply": "2024-03-23T03:49:05.271857Z" + }, + "papermill": { + "duration": 0.020567, + "end_time": "2024-03-23T03:49:05.274746", + "exception": false, + "start_time": "2024-03-23T03:49:05.254179", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:49:05.301892Z", + "iopub.status.busy": "2024-03-23T03:49:05.301281Z", + "iopub.status.idle": "2024-03-23T03:49:08.746158Z", + "shell.execute_reply": "2024-03-23T03:49:08.745213Z" + }, + "papermill": { + "duration": 3.461094, + "end_time": "2024-03-23T03:49:08.748304", + "exception": false, + "start_time": "2024-03-23T03:49:05.287210", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/aug_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/bs_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:49:08.777473Z", + "iopub.status.busy": "2024-03-23T03:49:08.777127Z", + "iopub.status.idle": "2024-03-23T03:49:09.403611Z", + "shell.execute_reply": "2024-03-23T03:49:09.402724Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.643908, + "end_time": "2024-03-23T03:49:09.405775", + "exception": false, + "start_time": "2024-03-23T03:49:08.761867", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.1,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': False, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:49:09.435335Z", + "iopub.status.busy": "2024-03-23T03:49:09.434976Z", + "iopub.status.idle": "2024-03-23T03:49:09.548111Z", + "shell.execute_reply": "2024-03-23T03:49:09.547108Z" + }, + "papermill": { + "duration": 0.130763, + "end_time": "2024-03-23T03:49:09.550439", + "exception": false, + "start_time": "2024-03-23T03:49:09.419676", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_train/tab_ddpm_concat/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/treatment [400, 0]\n", + "Caching in ../../../../treatment/_cache_aug_val/tab_ddpm_concat/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/treatment [0, 200]\n", + "Caching in ../../../../treatment/_cache_bs_train/tab_ddpm_concat/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/treatment [100, 0]\n", + "Caching in ../../../../treatment/_cache_bs_val/tab_ddpm_concat/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/treatment [0, 50]\n", + "Caching in ../../../../treatment/_cache_synth/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/treatment [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-23T03:49:09.580896Z", + "iopub.status.busy": "2024-03-23T03:49:09.580526Z", + "iopub.status.idle": "2024-03-23T03:49:10.137596Z", + "shell.execute_reply": "2024-03-23T03:49:10.136594Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.574903, + "end_time": "2024-03-23T03:49:10.139995", + "exception": false, + "start_time": "2024-03-23T03:49:09.565092", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:49:10.170796Z", + "iopub.status.busy": "2024-03-23T03:49:10.170465Z", + "iopub.status.idle": "2024-03-23T03:49:10.174781Z", + "shell.execute_reply": "2024-03-23T03:49:10.173950Z" + }, + "papermill": { + "duration": 0.021987, + "end_time": "2024-03-23T03:49:10.176746", + "exception": false, + "start_time": "2024-03-23T03:49:10.154759", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:49:10.204311Z", + "iopub.status.busy": "2024-03-23T03:49:10.203989Z", + "iopub.status.idle": "2024-03-23T03:49:10.210790Z", + "shell.execute_reply": "2024-03-23T03:49:10.209954Z" + }, + "papermill": { + "duration": 0.022951, + "end_time": "2024-03-23T03:49:10.212821", + "exception": false, + "start_time": "2024-03-23T03:49:10.189870", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18616321" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:49:10.244517Z", + "iopub.status.busy": "2024-03-23T03:49:10.244188Z", + "iopub.status.idle": "2024-03-23T03:49:10.340514Z", + "shell.execute_reply": "2024-03-23T03:49:10.339565Z" + }, + "papermill": { + "duration": 0.115891, + "end_time": "2024-03-23T03:49:10.343081", + "exception": false, + "start_time": "2024-03-23T03:49:10.227190", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 12] --\n", + "├─Adapter: 1-1 [2, 2648, 12] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 13,312\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 12] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,616,321\n", + "Trainable params: 18,616,321\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 73.71\n", + "========================================================================================================================\n", + "Input size (MB): 0.32\n", + "Forward/backward pass size (MB): 1079.48\n", + "Params size (MB): 74.47\n", + "Estimated Total Size (MB): 1154.27\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T03:49:10.376482Z", + "iopub.status.busy": "2024-03-23T03:49:10.375616Z", + "iopub.status.idle": "2024-03-23T04:55:35.670410Z", + "shell.execute_reply": "2024-03-23T04:55:35.669385Z" + }, + "papermill": { + "duration": 3985.314347, + "end_time": "2024-03-23T04:55:35.672881", + "exception": false, + "start_time": "2024-03-23T03:49:10.358534", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.1593249796038597, 'avg_role_model_std_loss': 38.32478675916096, 'avg_role_model_mean_pred_loss': 0.08493479980118587, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.1593249796038597, 'n_size': 900, 'n_batch': 225, 'duration': 296.148889541626, 'duration_batch': 1.316217286851671, 'duration_size': 0.32905432171291776, 'avg_pred_std': 0.18687162672061783}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012289107807673581, 'avg_role_model_std_loss': 1.519535903538306, 'avg_role_model_mean_pred_loss': 0.0025395565246152793, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012289107807673581, 'n_size': 450, 'n_batch': 113, 'duration': 110.64389157295227, 'duration_batch': 0.9791494829464803, 'duration_size': 0.2458753146065606, 'avg_pred_std': 0.11459510927767219}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.009436246975670252, 'avg_role_model_std_loss': 0.16965037501825211, 'avg_role_model_mean_pred_loss': 0.00024518938042836187, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009436246975670252, 'n_size': 900, 'n_batch': 225, 'duration': 292.4781291484833, 'duration_batch': 1.2999027962154812, 'duration_size': 0.3249756990538703, 'avg_pred_std': 0.23130405430164602}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01923421653492369, 'avg_role_model_std_loss': 7.067227336121174, 'avg_role_model_mean_pred_loss': 0.0030869791105636382, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01923421653492369, 'n_size': 450, 'n_batch': 113, 'duration': 113.68414735794067, 'duration_batch': 1.006054401397705, 'duration_size': 0.25263143857320147, 'avg_pred_std': 0.09829258351315585}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0072775980280271645, 'avg_role_model_std_loss': 0.20101061383553606, 'avg_role_model_mean_pred_loss': 0.00024995846221528785, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0072775980280271645, 'n_size': 900, 'n_batch': 225, 'duration': 295.3896453380585, 'duration_batch': 1.3128428681691489, 'duration_size': 0.3282107170422872, 'avg_pred_std': 0.22988744627076407}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0073902537548176245, 'avg_role_model_std_loss': 1.8595952647372243, 'avg_role_model_mean_pred_loss': 0.0004061482044698016, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0073902537548176245, 'n_size': 450, 'n_batch': 113, 'duration': 112.0707745552063, 'duration_batch': 0.991776765975277, 'duration_size': 0.24904616567823623, 'avg_pred_std': 0.11153677370851504}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006081614995972207, 'avg_role_model_std_loss': 0.28913738945182105, 'avg_role_model_mean_pred_loss': 0.00010754254828174891, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006081614995972207, 'n_size': 900, 'n_batch': 225, 'duration': 295.19276666641235, 'duration_batch': 1.3119678518507216, 'duration_size': 0.3279919629626804, 'avg_pred_std': 0.22885953691581057}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008240144269055791, 'avg_role_model_std_loss': 3.4400916960033965, 'avg_role_model_mean_pred_loss': 0.0006446722911166196, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008240144269055791, 'n_size': 450, 'n_batch': 113, 'duration': 112.21956896781921, 'duration_batch': 0.9930935306886656, 'duration_size': 0.24937681992848715, 'avg_pred_std': 0.09865027220083766}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005871379718669737, 'avg_role_model_std_loss': 0.1360336290616753, 'avg_role_model_mean_pred_loss': 0.0001008218179352885, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005871379718669737, 'n_size': 900, 'n_batch': 225, 'duration': 295.90435433387756, 'duration_batch': 1.3151304637061225, 'duration_size': 0.32878261592653063, 'avg_pred_std': 0.23736182113048723}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007101165456341442, 'avg_role_model_std_loss': 2.645643536452133, 'avg_role_model_mean_pred_loss': 0.000494350846501724, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007101165456341442, 'n_size': 450, 'n_batch': 113, 'duration': 112.5444118976593, 'duration_batch': 0.9959682468819407, 'duration_size': 0.25009869310590954, 'avg_pred_std': 0.10926411079190156}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0059188109788475335, 'avg_role_model_std_loss': 0.13654618756688353, 'avg_role_model_mean_pred_loss': 9.134781137381922e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0059188109788475335, 'n_size': 900, 'n_batch': 225, 'duration': 294.9814076423645, 'duration_batch': 1.3110284784105088, 'duration_size': 0.3277571196026272, 'avg_pred_std': 0.24004095225698419}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009655498555774026, 'avg_role_model_std_loss': 4.006451106402225, 'avg_role_model_mean_pred_loss': 0.0008652036921123643, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009655498555774026, 'n_size': 450, 'n_batch': 113, 'duration': 112.00565361976624, 'duration_batch': 0.9912004745112056, 'duration_size': 0.24890145248836942, 'avg_pred_std': 0.11541654484422117}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00488773422920935, 'avg_role_model_std_loss': 0.12989713646643514, 'avg_role_model_mean_pred_loss': 7.097226211497443e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00488773422920935, 'n_size': 900, 'n_batch': 225, 'duration': 295.9172704219818, 'duration_batch': 1.3151878685421414, 'duration_size': 0.32879696713553536, 'avg_pred_std': 0.23536086357302136}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0074696344484894124, 'avg_role_model_std_loss': 1.8159953127400679, 'avg_role_model_mean_pred_loss': 0.0006341720502412485, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0074696344484894124, 'n_size': 450, 'n_batch': 113, 'duration': 113.0309898853302, 'duration_batch': 1.0002742467728336, 'duration_size': 0.251179977522956, 'avg_pred_std': 0.10902682123553786}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004355493340135581, 'avg_role_model_std_loss': 0.2025226039046048, 'avg_role_model_mean_pred_loss': 3.489317478565809e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004355493340135581, 'n_size': 900, 'n_batch': 225, 'duration': 295.52129220962524, 'duration_batch': 1.3134279653761123, 'duration_size': 0.32835699134402807, 'avg_pred_std': 0.24828977358152365}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006594920561917976, 'avg_role_model_std_loss': 4.189112022148085, 'avg_role_model_mean_pred_loss': 0.00016057833332648575, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006594920561917976, 'n_size': 450, 'n_batch': 113, 'duration': 112.15122652053833, 'duration_batch': 0.9924887302702508, 'duration_size': 0.2492249478234185, 'avg_pred_std': 0.1086470079413002}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004756967619367869, 'avg_role_model_std_loss': 0.121430992607959, 'avg_role_model_mean_pred_loss': 4.847650491266009e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004756967619367869, 'n_size': 900, 'n_batch': 225, 'duration': 294.4956142902374, 'duration_batch': 1.3088693968454996, 'duration_size': 0.3272173492113749, 'avg_pred_std': 0.2423650716410743}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006066898118242913, 'avg_role_model_std_loss': 3.690666341061687, 'avg_role_model_mean_pred_loss': 0.0002841883362242727, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006066898118242913, 'n_size': 450, 'n_batch': 113, 'duration': 111.66992807388306, 'duration_batch': 0.988229451981266, 'duration_size': 0.24815539571974013, 'avg_pred_std': 0.112550658222006}\n", + "Time out: 3676.9595005512238/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.0034467438291015976, 'avg_g_mag_loss': 0.02844205359155494, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.189708232879639, 'grad_duration': 11.53635287284851, 'total_duration': 17.72606110572815, 'pred_std': 0.24351681768894196, 'std_loss': 2.0739604224218056e-05, 'mean_pred_loss': 5.471021722769365e-05, 'pred_rmse': 0.058708976954221725, 'pred_mae': 0.03615765646100044, 'pred_mape': 3378291.5, 'grad_rmse': 0.030787667259573936, 'grad_mae': 0.01297517865896225, 'grad_mape': 0.434597909450531}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0034467438291015976, 'avg_g_mag_loss': 0.02844205359155494, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 6.189708232879639, 'avg_grad_duration': 11.53635287284851, 'avg_total_duration': 17.72606110572815, 'avg_pred_std': 0.24351681768894196, 'avg_std_loss': 2.0739604224218056e-05, 'avg_mean_pred_loss': 5.471021722769365e-05}, 'min_metrics': {'avg_loss': 0.0034467438291015976, 'avg_g_mag_loss': 0.02844205359155494, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.189708232879639, 'grad_duration': 11.53635287284851, 'total_duration': 17.72606110572815, 'pred_std': 0.24351681768894196, 'std_loss': 2.0739604224218056e-05, 'mean_pred_loss': 5.471021722769365e-05, 'pred_rmse': 0.058708976954221725, 'pred_mae': 0.03615765646100044, 'pred_mape': 3378291.5, 'grad_rmse': 0.030787667259573936, 'grad_mae': 0.01297517865896225, 'grad_mape': 0.434597909450531}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.0034467438291015976, 'avg_g_mag_loss': 0.02844205359155494, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.189708232879639, 'grad_duration': 11.53635287284851, 'total_duration': 17.72606110572815, 'pred_std': 0.24351681768894196, 'std_loss': 2.0739604224218056e-05, 'mean_pred_loss': 5.471021722769365e-05, 'pred_rmse': 0.058708976954221725, 'pred_mae': 0.03615765646100044, 'pred_mape': 3378291.5, 'grad_rmse': 0.030787667259573936, 'grad_mae': 0.01297517865896225, 'grad_mape': 0.434597909450531}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T04:55:35.708507Z", + "iopub.status.busy": "2024-03-23T04:55:35.707628Z", + "iopub.status.idle": "2024-03-23T04:55:35.712310Z", + "shell.execute_reply": "2024-03-23T04:55:35.711536Z" + }, + "papermill": { + "duration": 0.024401, + "end_time": "2024-03-23T04:55:35.714238", + "exception": false, + "start_time": "2024-03-23T04:55:35.689837", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T04:55:35.746993Z", + "iopub.status.busy": "2024-03-23T04:55:35.746439Z", + "iopub.status.idle": "2024-03-23T04:55:35.875046Z", + "shell.execute_reply": "2024-03-23T04:55:35.874201Z" + }, + "papermill": { + "duration": 0.147662, + "end_time": "2024-03-23T04:55:35.877455", + "exception": false, + "start_time": "2024-03-23T04:55:35.729793", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T04:55:35.912864Z", + "iopub.status.busy": "2024-03-23T04:55:35.912532Z", + "iopub.status.idle": "2024-03-23T04:55:36.222177Z", + "shell.execute_reply": "2024-03-23T04:55:36.221171Z" + }, + "papermill": { + "duration": 0.330069, + "end_time": "2024-03-23T04:55:36.224245", + "exception": false, + "start_time": "2024-03-23T04:55:35.894176", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5mklEQVR4nO3deVxTV/4//lf2sIZ9U2SpWK3iBsKgrYyVT7FjF1qndvhQpf6s2g60VbpYOi20nd8UW231U+undhl1+hkpjjO1M0VrS3HpWHADrQvKuKDgEhAUAgkkJDnfPwIXgkGSkIXl/Xw87iPJvSf3ngB5ce+5557LY4wxEELIIMF3dgUIIcQSFFqEkEGFQosQMqhQaBFCBhUKLULIoEKhRQgZVCi0CCGDitDZFXAUvV6Pa9euwcPDAzwez9nVIYT0wBhDc3MzQkJCwOf3vj81bELr2rVrCA0NdXY1CCF9qKmpwciRI3tdPmxCy8PDA4DhB+Lp6enk2hBCelIoFAgNDeW+q70ZNqHVeUjo6elJoUXIANZX8w01xBNCBhUKLULIoEKhRQgZVIZNmxbpH51Oh/b2dmdXgwxiIpEIAoGg3+uh0CJ3xBiDXC5HY2Ojs6tChgAvLy8EBQX1q68khRa5o87ACggIgKurK3XMJVZhjEGlUqGurg4AEBwcbPW6KLR6aFFrcea6Au06Pabf5efs6jiVTqfjAsvX19fZ1SGDnIuLCwCgrq4OAQEBVh8qUkN8DyXn6/HExlK8u+uMs6vidJ1tWK6urk6uCRkqOv+W+tM+SqHVQ6S/GwCg6oYSNHy+AR0SEluxxd8ShVYPoT6u4PMApUaHG81qZ1eHENIDhVYPEqEAI70Nu7AX65VOrg0hpCcKLRMi/DoOESm0iJV4PB6++eYbZ1fDpt566y1MnjzZ2dWg0DKFQosMBVu2bIGXl5fN1vfyyy+juLjYZuuzFnV5MOGujsb4izcotMjQp9FoIBaL+yzn7u4Od3d3B9TozmhPy4QIP8Mvpqq+xck1GVgYY1BptE6ZLD2Tu3v3btx7773w8vKCr68vHnroIVy4cAEAMH36dKxcudKo/I0bNyASifDTTz8BAK5fv465c+fCxcUFERERyM/PR3h4ONatW2fVz+7kyZO4//774eLiAl9fXyxduhQtLV1/X/v27UNcXBzc3Nzg5eWFGTNm4PLlywCAX375BbNmzYKHhwc8PT0RExODo0eP3nF7+/btw6JFi9DU1AQejwcej4e33noLABAeHo4//vGPWLhwITw9PbF06VIAwMqVKzFmzBi4uroiMjISb775plHXhJ6Hh08//TRSUlKwZs0aBAcHw9fXFxkZGXa/3Iv2tEyI6NjTqr6pglanh1BA2Q4Are063JPzvVO2XfFOMlzF5v+5KpVKZGVlYeLEiWhpaUFOTg4ee+wxHD9+HGlpaXj//fexatUq7hT8tm3bEBISgvvuuw8AsHDhQtTX12Pfvn0QiUTIysrienNbSqlUIjk5GQkJCThy5Ajq6urwzDPPIDMzE1u2bIFWq0VKSgqWLFmCr776ChqNBocPH+bqlpaWhilTpuCTTz6BQCDA8ePHIRKJ7rjN6dOnY926dcjJyUFlZSUAGO0lrVmzBjk5OcjNzeXmeXh4YMuWLQgJCcHJkyexZMkSeHh44NVXX+11O3v37kVwcDD27t2L8+fP48knn8TkyZOxZMkSq35W5qDQMiHYUwqJkA+1Vo8rt1oR3tHGRQaPefPmGb3etGkT/P39UVFRgfnz52P58uU4cOAAF1L5+flITU0Fj8fD2bNn8eOPP+LIkSOIjY0FAHzxxReIioqyqi75+floa2vDl19+CTc3w9/Sxx9/jIcffhjvvfceRCIRmpqa8NBDD+Guu+4CAIwbN457f3V1NV555RWMHTsWAMyqh1gshkwmA4/HQ1BQ0G3L77//frz00ktG89544w3ueXh4OF5++WUUFBTcMbS8vb3x8ccfQyAQYOzYsZg7dy6Ki4sHXmht2LABq1evhlwux6RJk7B+/XrExcWZLHv69Gnk5OSgrKwMly9fxtq1a7F8+XKjMm+99Rbefvtto3l33303zp49y71ua2vDSy+9hIKCAqjVaiQnJ+N///d/ERgYaM1HuCM+n4cIPzeclTejql5JodXBRSRAxTvJTtu2Jc6dO4ecnBwcOnQI9fX10Ov1AAwBMGHCBDzwwAPYunUr7rvvPlRVVaG0tBSffvopAKCyshJCoRBTp07l1jd69Gh4e3tbVfczZ85g0qRJXGABwIwZM6DX61FZWYmZM2fi6aefRnJyMv7rv/4LSUlJmD9/Pnd9XlZWFp555hn83//9H5KSkvDEE09w4WatzjDubtu2bfjoo49w4cIFtLS0QKvV9jnK7/jx440uxwkODsbJkyf7Vbe+WHzcs23bNmRlZSE3Nxfl5eWYNGkSkpOTe911VqlUiIyMxKpVq0wmfqfx48fj+vXr3HTgwAGj5StWrMC3336L7du3Y//+/bh27Roef/xxS6tvts4ziNRXqwuPx4OrWOiUydKe1A8//DBu3ryJzz//HIcOHcKhQ4cAGBqdAcMh19///ne0t7cjPz8f0dHRiI6OtvnPzFybN29GaWkppk+fjm3btmHMmDE4ePAgAMM/9dOnT2Pu3LnYs2cP7rnnHuzYsaNf2+seoABQWlqKtLQ0/OY3v0FhYSGOHTuGP/zhD9zPqzc9D1N5PB73D8JeLA6tDz/8EEuWLMGiRYtwzz33YOPGjXB1dcWmTZtMlp82bRpWr16N3/3ud5BIJL2uVygUIigoiJv8/LouVm5qasKf//xnfPjhh7j//vsRExODzZs3o6SkhPvF9qRWq6FQKIwmS3R1e6DG+MGmoaEBlZWVeOONNzB79myMGzcOt27dMirz6KOPoq2tDbt370Z+fj7S0tK4ZXfffTe0Wi2OHTvGzTt//vxt6zDXuHHj8Msvv0Cp7PoH+PPPP4PP5+Puu+/m5k2ZMgXZ2dkoKSnBhAkTkJ+fzy0bM2YMVqxYgR9++AGPP/44Nm/e3Od2xWIxdDqdWXUsKSlBWFgY/vCHPyA2NhZRUVHciYCBxqLQ0mg0KCsrQ1JSUtcK+HwkJSWhtLS0XxU5d+4cQkJCEBkZibS0NFRXV3PLysrK0N7ebrTdsWPHYtSoUb1uNy8vDzKZjJssvX0Y9dUavLy9veHr64vPPvsM58+fx549e5CVlWVUxs3NDSkpKXjzzTdx5swZpKamcsvGjh2LpKQkLF26FIcPH8axY8ewdOlSuLi4WHXtXFpaGqRSKdLT03Hq1Cns3bsXzz//PBYsWIDAwEBUVVUhOzsbpaWluHz5Mn744QecO3cO48aNQ2trKzIzM7Fv3z5cvnwZP//8M44cOWLU5tWb8PBwtLS0oLi4GPX19VCpVL2WjYqKQnV1NQoKCnDhwgV89NFH/d6bsxeLQqu+vh46ne62dqTAwEDI5XKrKxEfH48tW7Zg9+7d+OSTT1BVVYX77rsPzc3NAAxjOonF4ts6yt1pu9nZ2WhqauKmmpoai+rU/cJpMrjw+XwUFBSgrKwMEyZMwIoVK7B69erbyqWlpeGXX37Bfffdh1GjRhkt+/LLLxEYGIiZM2fiscce486kSaVSi+vj6uqK77//Hjdv3sS0adPw29/+FrNnz8bHH3/MLT979izmzZuHMWPGYOnSpcjIyMCyZcsgEAjQ0NCAhQsXYsyYMZg/fz4efPDB29qATZk+fTqeffZZPPnkk/D398f777/fa9lHHnkEK1asQGZmJiZPnoySkhK8+eabFn9Wh2AWuHr1KgPASkpKjOa/8sorLC4urs/3h4WFsbVr1/ZZ7tatW8zT05N98cUXjDHGtm7dysRi8W3lpk2bxl599VWz6t7U1MQAsKamJrPKN7SoWdjKQha2spCp1Fqz3jPUtLa2soqKCtba2ursqjhdTU0NA8B+/PFHZ1dlULvT35S531GLzh76+flBIBCgtrbWaH5tbe0dG9kt5eXlhTFjxuD8+fMAgKCgIGg0GjQ2Nhrtbdl6u915u4ogcxGhqbUdlxqUGBdM90ocTvbs2YOWlhZER0fj+vXrePXVVxEeHo6ZM2c6u2rDnkWHh2KxGDExMUbXH+n1ehQXFyMhIcFmlWppacGFCxe4U74xMTEQiURG262srER1dbVNt9sdj8ejdq1hrL29Ha+//jrGjx+Pxx57DP7+/lxH061bt3KXtPScxo8f77A6Pvjgg73W491333VYPRzN4n5aWVlZSE9PR2xsLOLi4rBu3ToolUosWrQIgKEn8YgRI5CXlwfA0HhfUVHBPb969SqOHz8Od3d3jB49GoDhQsyHH34YYWFhuHbtGnJzcyEQCLjGUZlMhsWLFyMrKws+Pj7w9PTE888/j4SEBPzqV7+yyQ/ClEg/NxyvaaTQGoaSk5ORnGy6T9ojjzyC+Ph4k8v66qluS1988QVaW1tNLvPx8XFYPRzN4tB68skncePGDeTk5EAul2Py5MnYvXs31zhfXV0NPr9rB+7atWuYMmUK93rNmjVYs2YNEhMTsW/fPgDAlStXkJqaioaGBvj7++Pee+/FwYMH4e/vz71v7dq14PP5mDdvnlHnUnvi+mpRYzzpxsPDAx4eHs6uBkaMGOHsKjgFj7HhMaawQqGATCZDU1NTn718OxWeuIbM/GOYOsoLX/9+hp1rOPC0tbWhqqoKERERVp01I6SnO/1NmfsdpSuB74DatAgZeCi07qAztG6p2nFLeefLGQghjkGhdQeuYiGCZYZd2KoG2tsiZCCg0OoDd4hIjfGEDAgUWn2gdi1ijaF4Y4uBgkKrDxRaZLCy9Y0tAMMwzjweD42NjTZdryUotPrQeeE0jatFyMBAodWH7je50OuHRZe23jEGaJTOmejGFja7sYVarcbLL7+MESNGwM3NDfHx8VxHbwC4fPkyHn74YXh7e8PNzQ3jx4/Hrl27cOnSJcyaNQuAYfgfHo+Hp59+2qqfR3/QGPF9GOntAiGfh7Z2PeSKNoR4uTi7Ss7TrgLeDXHOtl+/BojNH/aabmzR+40tMjMzUVFRgYKCAoSEhGDHjh2YM2cOTp48iaioKGRkZECj0eCnn36Cm5sbKioq4O7ujtDQUPzjH//AvHnzUFlZCU9PT7i4OP77QKHVB5GAj1E+rrhYr0RVvXJ4h9YgQje2MH1ji+rqamzevBnV1dUICTH8A3r55Zexe/dubN68Ge+++y6qq6sxb948bvjpyMhI7v2d1zQGBATYvL3MXBRaZojwc8PFeiUu1isxY7Rf328YqkSuhj0eZ23bAnRjC9NOnjwJnU6HMWPGGM1Xq9Xw9fUFALzwwgt47rnn8MMPPyApKQnz5s3DxIkTrdqePVCblhmor1YHHs9wiOaMiW5sYZMbW7S0tEAgEKCsrAzHjx/npjNnzuB//ud/AADPPPMMLl68iAULFuDkyZOIjY3F+vXrbfZZ+4tCywydN2+lm1wMDnRjCwNTN7aYMmUKdDod6urqMHr0aKOp+2FkaGgonn32WXz99dd46aWX8Pnnn3PrBGD2DTPsgULLDNRXa3ChG1sYmLqxxZgxY5CWloaFCxfi66+/RlVVFQ4fPoy8vDzs3LkTALB8+XJ8//33qKqqQnl5Ofbu3cttLywsDDweD4WFhbhx44bRGVCHsc9I0AOPpWPEd3e9sZWFrSxkkdk7mbpdZ4faDUyDeYz4oqIiNm7cOCaRSNjEiRPZvn37GAC2Y8cOrsyuXbsYADZz5szb3n/t2jX24IMPMolEwsLCwlh+fj4LCAhgGzduNGv7Pbd14sQJNmvWLCaVSpmPjw9bsmQJa25uZowxJpfLWUpKCgsODmZisZiFhYWxnJwcptPpmFqtZr/73e9YaGgoE4vFLCQkhGVmZpr9O3n22WeZr68vA8Byc3MZY4xpNBqWk5PDwsPDmUgkYsHBweyxxx5jJ06cYIwxlpmZye666y4mkUiYv78/W7BgAauvr+fW+c4777CgoCDG4/FYenq6WfXoZIsx4mk8LTMwxnBPzvdobdeh+KVE3OXvbqdaDiw0nlaXK1euIDQ0FD/++CNmz57t7OoMWrYYT4vOHpqhc7z4iusKVN1QDpvQGs7oxhYDF7VpmamrMZ7atYYDurHFwEV7WmaK7BwvnkJrWKAbWwxcFFpm6jqDSN0ehju6sYVz0eGhmSL9Oy+cHn57WsPkXA1xAFv8LVFomSnC17CnVatQQ6nWOrk2jtF5qKNSqZxcEzJUdP4t9ecwmg4PzSRzFcHXTYwGpQZV9UpMGCFzdpXsTiAQwMvLixvdwNXV1arOlYQwxqBSqVBXVwcvLy8IBAKr12VVaG3YsAGrV6+GXC7HpEmTsH79esTFxZkse/r0aeTk5KCsrAyXL1/G2rVrsXz5cqMyeXl5+Prrr3H27Fm4uLhg+vTpeO+994wucfj1r3+N/fv3G71v2bJl2LhxozUfwSoRfm7DKrQAcJd2WDssCyHdeXl5GV0uZA2LQ2vbtm3IysrCxo0bER8fj3Xr1iE5ORmVlZUICAi4rbxKpUJkZCSeeOIJrFixwuQ69+/fj4yMDEybNg1arRavv/46HnjgAVRUVBhdGb9kyRK888473GtXV8uu/O+vCD83HL18a1i1a/F4PAQHByMgIADt7e3Org4ZxEQiUb/2sDpZHFoffvghlixZgkWLFgEANm7ciJ07d2LTpk147bXXbis/bdo0TJs2DQBMLgcMo0x2t2XLFgQEBKCsrMyoM5+rq6vZKa1Wq6FWq7nXCoXCrPfdSWdfrYs3ht8ZRIFAYJM/OEL6y6KGeI1Gg7KyMiQlJXWtgM9HUlISSktLbVappqYmALf3Ndm6dSv8/PwwYcIEZGdn37GBOC8vDzKZjJtCQ0P7Xa9IunCaEKezaE+rvr4eOp0OgYGBRvMDAwNx9uxZm1RIr9dj+fLlmDFjBiZMmMDN/+///m+EhYUhJCQEJ06cwMqVK1FZWYmvv/7a5Hqys7ONruxXKBT9Dq7O8eIv1ivBGKNGaUKcYMCdPczIyMCpU6dw4MABo/lLly7lnkdHRyM4OBizZ8/GhQsXTI7iKJFIIJFIbFq3MF9X8HhAc5sWDUoN/Nxtu35CSN8sOjz08/ODQCBAbW2t0fza2tp+nxEADAPuFxYWYu/evRg5cuQdy3ZeRnH+/Pl+b9dcUpEAITLDGPF0iEiIc1gUWmKxGDExMSguLubm6fV6FBcXIyEhwepKMMaQmZmJHTt2YM+ePYiIiOjzPcePHwcAbhxtR+m8D+KwH3qZECex+PAwKysL6enpiI2NRVxcHNatWwelUsmdTVy4cCFGjBiBvLw8AIbG+4qKCu751atXcfz4cbi7u2P06NEADIeE+fn5+Oc//wkPDw/I5XIAgEwmg4uLCy5cuID8/Hz85je/ga+vL06cOIEVK1Zg5syZDh9wP8LPDf8+V08XThPiLBYNO9hh/fr1bNSoUUwsFrO4uDh28OBBblliYqLRaIZVVVUMwG1TYmIiV8bUcgBs8+bNjDHGqqur2cyZM5mPjw+TSCRs9OjR7JVXXrFoFNL+jFza3aYDF1nYykK29Msj/VoPIcQYjVzaQ39GLu1uX2Udnt58BGMC3fHDikQb1pCQ4c3c7yhdMG2hyI5uD5caVNDph0XeEzKgUGhZaIS3C0QCHjRaPa41mh6AjRBiPxRaFhLweQjzpZ7xhDgLhZYV6D6IhDgPhZYVIukmF4Q4DYWWFegmF4Q4D4WWFTovnKabXBDieBRaVuhs07pyqxVqrc7JtSFkeKHQsoKfuxgeEiEYA6ob6KYPhDgShZYVeDweN4rpBbpwmhCHotCyEnV7IMQ5KLSsRHecJsQ5KLSsRHtahDgHhZaVIrluDxRahDgShZaVwv0M91ysb9GgqZXuB0iIo1BoWclDKoK/h+HGFpdob4sQh6HQ6gdq1yLE8Si0+oGuQSTE8Si0+oH2tAhxPAqtfqC+WoQ4HoVWP3S/B+IwuT8IIU5HodUPo3zcwOcBSo0ON5rVzq4OIcOCVaG1YcMGhIeHQyqVIj4+HocPH+617OnTpzFv3jyEh4eDx+Nh3bp1Vq2zra0NGRkZ8PX1hbu7O+bNm4fa2lprqm8zYiEfoT6G/lrUGE+IY1gcWtu2bUNWVhZyc3NRXl6OSZMmITk5GXV1dSbLq1QqREZGYtWqVQgKCrJ6nStWrMC3336L7du3Y//+/bh27Roef/xxS6tvc9QYT4iDWXoX2Li4OJaRkcG91ul0LCQkhOXl5fX53rCwMLZ27VqL19nY2MhEIhHbvn07V+bMmTMMACstLTWr3ra6w3RPb/3rFAtbWcj+tLPCpuslZLgx9ztq0Z6WRqNBWVkZkpKSuHl8Ph9JSUkoLS21KjTNWWdZWRna29uNyowdOxajRo3qdbtqtRoKhcJosgeurxaNq0WIQ1gUWvX19dDpdAgMDDSaHxgYCLlcblUFzFmnXC6HWCyGl5eX2dvNy8uDTCbjptDQUKvq1xcaL54QxxqyZw+zs7PR1NTETTU1NXbZTucIptU3VdDq9HbZBiGki9CSwn5+fhAIBLedtautre21kd0W6wwKCoJGo0FjY6PR3tadtiuRSCCRSKyqkyWCPaWQCPlQa/W4cqsV4R2Hi4QQ+7BoT0ssFiMmJgbFxcXcPL1ej+LiYiQkJFhVAXPWGRMTA5FIZFSmsrIS1dXVVm/XVvh8Hp1BJMSBLNrTAoCsrCykp6cjNjYWcXFxWLduHZRKJRYtWgQAWLhwIUaMGIG8vDwAhob2iooK7vnVq1dx/PhxuLu7Y/To0WatUyaTYfHixcjKyoKPjw88PT3x/PPPIyEhAb/61a9s8oPojwg/N5yVN+NivRKznF0ZQoY6a05Nrl+/no0aNYqJxWIWFxfHDh48yC1LTExk6enp3OuqqioG4LYpMTHR7HUyxlhrayv7/e9/z7y9vZmrqyt77LHH2PXr182us726PDDG2HvfnWFhKwvZH3acsPm6CRkuzP2O8hgbHhfNKRQKyGQyNDU1wdPT06br3n60Bq/8/QRmjPbF1mecv+dHyGBk7nd0yJ49dKTuF04TQuyLQssGOvtqXWtqQ6tG5+TaEDK0UWjZgLerCDIXEQDgUgPtbRFiTxRaNsDjUbcHQhyFQstGIim0CHEICi0biaALpwlxCAotG4n0pwunCXEECi0boTYtQhyDQstGwv0Mwy7fUrXjllLj5NoQMnRRaNmIq1iIYJkUAFBF3R4IsRsKLRviDhGpMZ4Qu6HQsiFq1yLE/ii0bIhCixD7o9Cyoc4Lpy/coG4PhNgLhZYNdV44falBCb1+WIz4Q4jDUWjZ0EhvFwj5PLS16yFXtDm7OoQMSRRaNiQS8DHKx9Bfi9q1CLEPCi0b465BpNAixC4otGyM+moRYl8UWjbWefNWunCaEPug0LIx6qtFiH1RaNlYZEe3h5pbrdBo9U6uDSFDD4WWjQV6SuAiEkCnZ6i5pXJ2dQgZcqwKrQ0bNiA8PBxSqRTx8fE4fPjwHctv374dY8eOhVQqRXR0NHbt2mW0nMfjmZxWr17NlQkPD79t+apVq6ypvl0ZjRdPjfGE2JzFobVt2zZkZWUhNzcX5eXlmDRpEpKTk1FXV2eyfElJCVJTU7F48WIcO3YMKSkpSElJwalTp7gy169fN5o2bdoEHo+HefPmGa3rnXfeMSr3/PPPW1p9h+Dug0jtWoTYnqW3ro6Li2MZGRnca51Ox0JCQlheXp7J8vPnz2dz5841mhcfH8+WLVvW6zYeffRRdv/99xvNCwsLY2vXrrW0uhxzb7ltCx98f5aFrSxkr/3jhN23RchQYe531KI9LY1Gg7KyMiQlJXHz+Hw+kpKSUFpaavI9paWlRuUBIDk5udfytbW12LlzJxYvXnzbslWrVsHX1xdTpkzB6tWrodVqe62rWq2GQqEwmhyFuj0QYj9CSwrX19dDp9MhMDDQaH5gYCDOnj1r8j1yudxkeblcbrL8X/7yF3h4eODxxx83mv/CCy9g6tSp8PHxQUlJCbKzs3H9+nV8+OGHJteTl5eHt99+29yPZlOdF07T4SEhtmdRaDnCpk2bkJaWBqlUajQ/KyuLez5x4kSIxWIsW7YMeXl5kEgkt60nOzvb6D0KhQKhoaH2q3g3Eb6GPa1ahRpKtRZukgH3YyZk0LLo8NDPzw8CgQC1tbVG82traxEUFGTyPUFBQWaX//e//43Kyko888wzfdYlPj4eWq0Wly5dMrlcIpHA09PTaHIUmasIvm5iALS3RYitWRRaYrEYMTExKC4u5ubp9XoUFxcjISHB5HsSEhKMygNAUVGRyfJ//vOfERMTg0mTJvVZl+PHj4PP5yMgIMCSj+Aw1DOeEPuw+LglKysL6enpiI2NRVxcHNatWwelUolFixYBABYuXIgRI0YgLy8PAPDiiy8iMTERH3zwAebOnYuCggIcPXoUn332mdF6FQoFtm/fjg8++OC2bZaWluLQoUOYNWsWPDw8UFpaihUrVuCpp56Ct7e3NZ/b7iL83HD08i264zQhNmZxaD355JO4ceMGcnJyIJfLMXnyZOzevZtrbK+urgaf37UDN336dOTn5+ONN97A66+/jqioKHzzzTeYMGGC0XoLCgrAGENqaupt25RIJCgoKMBbb70FtVqNiIgIrFixwqjNaqChM4iE2AePMTYsxgVWKBSQyWRoampySPvW7lPX8exfyzFppAz/zLzX7tsjZLAz9ztK1x7aSWe3h4v1SgyT/wuEOASFlp2E+bqCxwOa27RoUGqcXR1ChgwKLTuRigQIkbkAoDOIhNgShZYdcRdO0xlEQmyGQsuO6CYXhNgehZYddXUwpW4PhNgKhZYdUa94QmyPQsuOOseLv9Sggk5P3R4IsQUKLTsa4e0CsYAPjVaPa42tzq4OIUMChZYdCfg8hPm6AqBDREJshULLzqhdixDbotCyswi6yQUhNkWhZWeR1FeLEJui0LKzrvHiqa8WIbZAoWVnnW1aV261Qq3VObk2hAx+FFp25ucuhodECMaA6gaVs6tDyKBHoWVnPB6Pa4y/QBdOE9JvFFoOQN0eCLEdCi0HoAunCbEdCi0HoD0tQmyHQssBIrluDxRahPQXhZYDhPsZrj+sb9GgqbXdybUhZHCj0HIAD6kI/h4SAMAl2tsipF+sCq0NGzYgPDwcUqkU8fHxOHz48B3Lb9++HWPHjoVUKkV0dDR27dpltPzpp58Gj8czmubMmWNU5ubNm0hLS4Onpye8vLywePFitLQMnoZtatcixDYsDq1t27YhKysLubm5KC8vx6RJk5CcnIy6ujqT5UtKSpCamorFixfj2LFjSElJQUpKCk6dOmVUbs6cObh+/To3ffXVV0bL09LScPr0aRQVFaGwsBA//fQTli5damn1nYauQSTERpiF4uLiWEZGBvdap9OxkJAQlpeXZ7L8/Pnz2dy5c43mxcfHs2XLlnGv09PT2aOPPtrrNisqKhgAduTIEW7ed999x3g8Hrt69apZ9W5qamIAWFNTk1nlbe3T/edZ2MpClplf7pTtEzLQmfsdtWhPS6PRoKysDElJSdw8Pp+PpKQklJaWmnxPaWmpUXkASE5Ovq38vn37EBAQgLvvvhvPPfccGhoajNbh5eWF2NhYbl5SUhL4fD4OHTpkcrtqtRoKhcJocia6cJoQ27AotOrr66HT6RAYGGg0PzAwEHK53OR75HJ5n+XnzJmDL7/8EsXFxXjvvfewf/9+PPjgg9DpdNw6AgICjNYhFArh4+PT63bz8vIgk8m4KTQ01JKPanNcm9YNJRij8eIJsZbQ2RUAgN/97nfc8+joaEycOBF33XUX9u3bh9mzZ1u1zuzsbGRlZXGvFQqFU4NrlI8r+DxAqdHhRrMaAZ5Sp9WFkMHMoj0tPz8/CAQC1NbWGs2vra1FUFCQyfcEBQVZVB4AIiMj4efnh/Pnz3Pr6NnQr9VqcfPmzV7XI5FI4OnpaTQ5k1jIR6iPob8WNcYTYj2LQkssFiMmJgbFxcXcPL1ej+LiYiQkJJh8T0JCglF5ACgqKuq1PABcuXIFDQ0NCA4O5tbR2NiIsrIyrsyePXug1+sRHx9vyUdwKur2QIgNWNrCX1BQwCQSCduyZQurqKhgS5cuZV5eXkwulzPGGFuwYAF77bXXuPI///wzEwqFbM2aNezMmTMsNzeXiUQidvLkScYYY83Nzezll19mpaWlrKqqiv34449s6tSpLCoqirW1tXHrmTNnDpsyZQo7dOgQO3DgAIuKimKpqalm19vZZw8ZY+ytf51iYSsL2Z92VjitDoQMVOZ+Ry0OLcYYW79+PRs1ahQTi8UsLi6OHTx4kFuWmJjI0tPTjcr/7W9/Y2PGjGFisZiNHz+e7dy5k1umUqnYAw88wPz9/ZlIJGJhYWFsyZIlXAh2amhoYKmpqczd3Z15enqyRYsWsebmZrPrPBBC68uSKha2spAt3nKk78KEDDPmfkd5jA2PU1kKhQIymQxNTU1Oa986cK4eT/35EO7yd0PxS792Sh0IGajM/Y7StYcO1DmCafVNFbQ6vZNrQ8jgRKHlQMGeUkiEfLTrGK7canV2dQgZlCi0HIjP59EZREL6iULLwSLowmlC+oVCy8FovHhC+odCy8Ho8JCQ/qHQcrBI/64LpwkhlqPQcrDOIWquNbWhVaNzcm0IGXwotBzM21UEmYsIAHCpgfa2CLEUhZaD8Xi8rkNEatcixGIUWk5AjfGEWI9Cywm4m1xQYzwhFqPQcgIaL54Q61FoOQEdHhJiPQotJwj3Mwy7fEvVjltKjZNrQ8jgQqHlBK5iIYJlhhtbVFG3B0IsQqHlJN1vKUYIMR+FlpNQuxYh1qHQcpKuIWroDCIhlqDQcpLOXvHUV4sQy1BoOUlnX61LDUro9cPi3iKE2ASFlpOM9HaBkM9DW7seckWbs6tDyKBBoeUkIgEfo3wM/bWoMZ4Q81kVWhs2bEB4eDikUini4+Nx+PDhO5bfvn07xo4dC6lUiujoaOzatYtb1t7ejpUrVyI6Ohpubm4ICQnBwoULce3aNaN1hIeHg8fjGU2rVq2ypvoDBo0XT4jlLA6tbdu2ISsrC7m5uSgvL8ekSZOQnJyMuro6k+VLSkqQmpqKxYsX49ixY0hJSUFKSgpOnToFAFCpVCgvL8ebb76J8vJyfP3116isrMQjjzxy27reeecdXL9+nZuef/55S6s/oFBfLUKsYOmtq+Pi4lhGRgb3WqfTsZCQEJaXl2ey/Pz589ncuXON5sXHx7Nly5b1uo3Dhw8zAOzy5cvcvLCwMLZ27VpLq8sx95bbjvTXg5dY2MpC9vSmQ86uCiFOZ+531KI9LY1Gg7KyMiQlJXHz+Hw+kpKSUFpaavI9paWlRuUBIDk5udfyANDU1AQejwcvLy+j+atWrYKvry+mTJmC1atXQ6vV9roOtVoNhUJhNA001MGUEMsJLSlcX18PnU6HwMBAo/mBgYE4e/asyffI5XKT5eVyucnybW1tWLlyJVJTU+Hp6cnNf+GFFzB16lT4+PigpKQE2dnZuH79Oj788EOT68nLy8Pbb79tycdzuLv8Dd0eam61QqPVQyyk8yKE9MWi0LK39vZ2zJ8/H4wxfPLJJ0bLsrKyuOcTJ06EWCzGsmXLkJeXB4lEctu6srOzjd6jUCgQGhpqv8pbIcBDAlexACqNDjW3VFyIEUJ6Z9G/dj8/PwgEAtTW1hrNr62tRVBQkMn3BAUFmVW+M7AuX76MoqIio70sU+Lj46HVanHp0iWTyyUSCTw9PY2mgYbH41FjPCEWsii0xGIxYmJiUFxczM3T6/UoLi5GQkKCyfckJCQYlQeAoqIio/KdgXXu3Dn8+OOP8PX17bMux48fB5/PR0BAgCUfYcChdi1CLGPx4WFWVhbS09MRGxuLuLg4rFu3DkqlEosWLQIALFy4ECNGjEBeXh4A4MUXX0RiYiI++OADzJ07FwUFBTh69Cg+++wzAIbA+u1vf4vy8nIUFhZCp9Nx7V0+Pj4Qi8UoLS3FoUOHMGvWLHh4eKC0tBQrVqzAU089BW9vb1v9LJwikvpqEWIZa05Nrl+/no0aNYqJxWIWFxfHDh48yC1LTExk6enpRuX/9re/sTFjxjCxWMzGjx/Pdu7cyS2rqqpiAExOe/fuZYwxVlZWxuLj45lMJmNSqZSNGzeOvfvuu6ytrc3sOg/ELg+MMfZ1eQ0LW1nInvy0xNlVIcSpzP2O8hhjw+JqXYVCAZlMhqampgHVvnW8phEpG35GuIce++YJAKkMCJsO8HjOrhohDmXud3RAnT0cdlrqMOZKIf4s+hL3ak4BBe2G+aOmA0m5wKhfObd+hAxAFFqO1nABOFsInN0J1ByGKxhmCwyLNB6jIG6tA6pLgE3JwJgHgdlvAoHjnVtnQgYQCi170+uBa8cMQVW5C7jRoxNuyFTkK6KxuWE8Xkh5CA9H8oD97wHl/wf85zvgP7uBiU8Cs7IB73CnfARCBhIKLXvQaoBL/zbsTVXuApqvdy3jC4GImcDdvzFMshE4tv0XnKu/gov1KmByFPDw/wAJzwN7/3/g9A7gRAFw6h9A7P8HzHwZcB/c3TwI6Q8KLVtpUwDniwxBda4IUHe71lHsDkT9FzD2IWB0EuDiZfTWCP/Ovlrdxov3Gw08sQWY/gJQ/A5wcS9w+FPg2F+BhAxgeqah0Z6QYYZCqz+a5YY9qbM7gYv7AX171zL3QMOe1Ni5hj0r4e2XGnWKvFMH0xFTgYXfABf3AT++DVwrB356HzjyBXDfS8C0ZwCR1Lafi5ABjELLUjf+09WQfvWo8TLfKENIjX0IGBED8M274KBzvPiL9UowxsAz1d0h8tfAkkTgzLeGPa+Gc8APfwAOfgL8+jVgUiogoF8nGfror7wver0hnDqDquG88fKR0wxBdfdcwH+MVZsI83UFjwc0t2nRoNTAz72XvTIeD7jnEcMe3C9fAfvyAMUV4F+ZQMlHwP1vAuMepj5eZEij0DKlvQ2o+gmo3Amc3QUou43KKhADEYnA2I6GdA/TF4pbQioSIETmgquNraiqV/YeWlwdhMDUBUD0E4bDxH+vAer/A/xtgWEPb3YuEJnY73oRMhBRaPV05ltgx7OAplujuMQTiHrAsEc1OgmQ2r5HfaS/myG0bigxLdzHvDeJpIYG+akLgJL1QOkG4GoZ8OUjwF33A7NzgJApNq8rIc5EodWT392GwPII7mifmguE3QsIxXbdbISfG/59rt66C6elMuD+N4C4pcBPq4Gjm4ELewzT+MeAWW8YzkYSMgRQaPXkPwZY9hMQGG12Q7otdJ1BbOmj5B24BwC/WQ386veG9q4TfzP086r4l2FvLHEl4BlioxoT4hwUWqYET3L4JiM6Ri29cEOJdp0eIkE/AtMnAnj8M0Mfrz1/NPSqL9sC/FIAxC8DZiwHXM08BCX9wxigagCargCKq4DiGiCUGtpC3QMNj65+Dv0HOdjRKA8DRM1NFe57fy8Aw8k/H1cx/D0kCPCUwt9dggBPCQI8JAjwkCLAU8LNcxWb8X/ncinw41tAzUHDa4kMuPdFIP5ZQOxmvw81HKibgaarhrO4TVc6nl8Fmmq6nmv7uIM4Xwi4BQAegYB7kInHjsktYEh3azH3O0qhNUDo9QxLvjyKff+5AZ3e/F+Ju0SIAA8J/DyMQy3AQ2IIPQ8pAjwk8HIRgne+yNBBte50x5sDgcRXganpgEDU3w9g+HK2twLtKsOjttX4tdGkMpTXaQCRKyDxMFw5IHHvePTs9twdEHs4/gurVRv2jBRXOwLpSrfnHUHV1mTeutwCANlIw+G5tg1orgVa5IDyhgUV4gFufrcHm0dw116be6BhGoQdjim0ehjoodVJr2e4qdKgTqFGXXMbbjSrUdes7nhsQ51CjRstatQp1Ght15m9XrGAbwgxdyHm8krweNMW+GgM10Sq3Efh5j0LIRGLIGUaSKCGSK8Gzyh02nqET0focAHVx96ELQilxiHWPdQkHrfP6+21xMOwrpa62/eKOp83XTHu6nInEpkhkGQjAM8RHc9Hdj33DOn9ighdu6EeLfKuIDP5WAsw83/fkHp1OwQNNoSbi7fhH4TIpduji4l5roZJKHFonz8KrR4GS2iZizGGFrW2W6CpUacwEXLNajSq2m97vwhapAqK8bxwB/x5tr0npF4gARMavhA8kQv44h5fEKG06wvCFwLtSkDdYjhryz02dz3qNDatn0UEkm6BNNL0c4mH/euh1xnaxprlhgBrlt8ebJ3zbPbz4vURcr0FnolloXG3XXPbE4VWD0MttCyh1upQ36JBncIQYnXNatxQtOFGixpNjY2Ir/87ItsqoNSL0KwTQcXEaIMYbZCglYnRCgnaIEZrx/zWjvmdz9s6yrRCDDXE0Pe4X4pIwIO7RAh3qRDuEhHcJYKO1yK4iQXg83ng8wABjwcejwdBx2s+jwc+nwcha4eLXgWJvhVS1gqJTgmxXgWpvhVinRISvQpirQpivRJinQoinQpinRIirQoirRJCXcdjx8SD4U+egY82lwC0uQaj1SUYrS5BaHMNhsolqON1MDRib7COvY3uX5XOp6xzXaz7PBiV597V8UQi4sNVLISbWAAXsQBuEiFcxQK4iYVwlQggFvBNX8plDsaA1lvdgq3bY1tTt73lHnvN3R/t8U9iyV7DdbR3QCOXEo5EKMAILxeM8HLppcRM7hljDGqtHs1tWrSotWjpfFRr0aJu73it4543q7VQqrvKNnc8KtVaKDWGw5l2HcMtVTtuqdoBtNrgE7l0TNZgcIUaLlCjEe7QtQmAW6bKaQBc7pgcS8jnwVUsgGtHiLmJO0KtW7gZwk7AhZ+rRMiVM5QNgatnKNz8hdy6BHwzg1CnNdEe2SPkNKo7B1/PZTYckYRCixjh8XiQigSQigTw9+jjcqI+6PQMSk1XiDX3DME2LVQaLXR6QM8YGGPQMQY9M7Tt6Tue6/TGyxhj0OmNy+lYt3Xob1+H4dF4WefODA/ck85nty3j8UzP6/mzu/39xssYDHu+Ko0OKrUOSo0WKo0OSrUWaq0eAKDVMyjatFC0afv18+9JLOBDJOBBJORDJOB3vRYYXouEfEgEfIiEXfO6ygghEsogFnhDLDR+n1jAh8iFB5GH4bWkY/2ijveKBXxEu8lgq4NoCi1iNwI+D55SETyl/TwzOUxodXqo2nVo7QgxVfdHjdYo5FQaLZTqjkeNDqqOPdtWE2U7z0ZrdHpodAA0FjTo28i3mfcieqRt9rYotAgZIIQCPjwFfJuGfOfhvkqjg1qrQ7uWQaPTo12nh0bb8ajTo13H0N7zdc8yWsO8rjJ6bn2G5R3zdF3b6CzjKhHY7DNRaBEyhHU/3B8qrLp2YMOGDQgPD4dUKkV8fDwOHz58x/Lbt2/H2LFjIZVKER0djV27dhktZ4whJycHwcHBcHFxQVJSEs6dO2dU5ubNm0hLS4Onpye8vLywePFitLT04zo9QsigZHFobdu2DVlZWcjNzUV5eTkmTZqE5ORk1NWZ7ohXUlKC1NRULF68GMeOHUNKSgpSUlJw6tQprsz777+Pjz76CBs3bsShQ4fg5uaG5ORktLV1dVhMS0vD6dOnUVRUhMLCQvz0009YunSpFR+ZEDKoWXrr6ri4OJaRkcG91ul0LCQkhOXl5ZksP3/+fDZ37lyjefHx8WzZsmWMMcb0ej0LCgpiq1ev5pY3NjYyiUTCvvrqK8YYYxUVFQwAO3LkCFfmu+++Yzwej129etWsept7y21CiHOY+x21aE9Lo9GgrKwMSUlJ3Dw+n4+kpCSUlpaafE9paalReQBITk7myldVVUEulxuVkclkiI+P58qUlpbCy8sLsbGxXJmkpCTw+XwcOnTI5HbVajUUCoXRRAgZ/CwKrfr6euh0OgQGBhrNDwwMhFwuN/keuVx+x/Kdj32VCQgwvtefUCiEj49Pr9vNy8uDTCbjptDQUDM/JSFkIBuyg/hkZ2ejqamJm2pqapxdJUKIDVgUWn5+fhAIBKitrTWaX1tbi6Ag0zd4CAoKumP5zse+yvRs6Ndqtbh582av25VIJPD09DSaCCGDn0X9tMRiMWJiYlBcXIyUlBQAgF6vR3FxMTIzM02+JyEhAcXFxVi+fDk3r6ioCAkJCQCAiIgIBAUFobi4GJMnTwZguHDy0KFDeO6557h1NDY2oqysDDExMQCAPXv2QK/XIz4+3qy6s46LV6lti5CBqfO7yfoaw8HSFv6CggImkUjYli1bWEVFBVu6dCnz8vJicrmcMcbYggUL2GuvvcaV//nnn5lQKGRr1qxhZ86cYbm5uUwkErGTJ09yZVatWsW8vLzYP//5T3bixAn26KOPsoiICNba2sqVmTNnDpsyZQo7dOgQO3DgAIuKimKpqalm17umpobBcOkXTTTRNICnmpqaO36XLe4R/+STT+LGjRvIycmBXC7H5MmTsXv3bq4hvbq6Gvxu411Pnz4d+fn5eOONN/D6668jKioK33zzDSZMmMCVefXVV6FUKrF06VI0Njbi3nvvxe7duyGVdo2+uHXrVmRmZmL27Nng8/mYN28ePvroI7PrHRISgpqaGnh4ePQ57IdCoUBoaChqamqGxWHlcPu8wPD7zIPh8zLG0NzcjJCQO998ZdiMp2WJ4Tb21nD7vMDw+8xD6fMO2bOHhJChiUKLEDKoUGiZIJFIkJubC4mkf4PgDRbD7fMCw+8zD6XPS21ahJBBhfa0CCGDCoUWIWRQodAihAwqFFqEkEGFQosQMqhQaPVg6fj3g1leXh6mTZsGDw8PBAQEICUlBZWVlc6ulsOsWrUKPB7P6GL+oejq1at46qmn4OvrCxcXF0RHR+Po0aPOrpbVKLS6sXT8+8Fu//79yMjIwMGDB1FUVIT29nY88MADUCqVzq6a3R05cgSffvopJk6c6Oyq2NWtW7cwY8YMiEQifPfdd6ioqMAHH3wAb29vZ1fNemYPkzAMWDr+/VBTV1fHALD9+/c7uyp21dzczKKiolhRURFLTExkL774orOrZDcrV65k9957r7OrYVO0p9XBmvHvh5qmpiYAgI+Pj5NrYl8ZGRmYO3fubfcuGIr+9a9/ITY2Fk888QQCAgIwZcoUfP75586uVr9QaHWwZvz7oUSv12P58uWYMWOG0bBBQ01BQQHKy8uRl5fn7Ko4xMWLF/HJJ58gKioK33//PZ577jm88MIL+Mtf/uLsqlmN7jBNABj2Pk6dOoUDBw44uyp2U1NTgxdffBFFRUVGY7UNZXq9HrGxsXj33XcBAFOmTMGpU6ewceNGpKenO7l21qE9rQ7WjH8/VGRmZqKwsBB79+7FyJEjnV0duykrK0NdXR2mTp0KoVAIoVCI/fv346OPPoJQKIROp3N2FW0uODgY99xzj9G8cePGobq62kk16j8KrQ7dx7/v1Dn+fed49kMNYwyZmZnYsWMH9uzZg4iICGdXya5mz56NkydP4vjx49wUGxuLtLQ0HD9+HAKBwNlVtLkZM2bc1o3lP//5D8LCwpxUIxtw9pmAgaSv8e+Hmueee47JZDK2b98+dv36dW5SqVTOrprDDPWzh4cPH2ZCoZD96U9/YufOnWNbt25lrq6u7K9//auzq2Y1Cq0e1q9fz0aNGsXEYjGLi4tjBw8edHaV7Aa93Fhg8+bNzq6awwz10GKMsW+//ZZNmDCBSSQSNnbsWPbZZ585u0r9QuNpEUIGFWrTIoQMKhRahJBBhUKLEDKoUGgRQgYVCi1CyKBCoUUIGVQotAghgwqFFiFkUKHQIoQMKhRahJBBhUKLEDKo/D+U6lr9Xig+3wAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T04:55:36.261381Z", + "iopub.status.busy": "2024-03-23T04:55:36.260453Z", + "iopub.status.idle": "2024-03-23T05:00:45.484914Z", + "shell.execute_reply": "2024-03-23T05:00:45.483897Z" + }, + "papermill": { + "duration": 309.246008, + "end_time": "2024-03-23T05:00:45.487804", + "exception": false, + "start_time": "2024-03-23T04:55:36.241796", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:00:45.526622Z", + "iopub.status.busy": "2024-03-23T05:00:45.526264Z", + "iopub.status.idle": "2024-03-23T05:00:45.550055Z", + "shell.execute_reply": "2024-03-23T05:00:45.549092Z" + }, + "papermill": { + "duration": 0.045793, + "end_time": "2024-03-23T05:00:45.552371", + "exception": false, + "start_time": "2024-03-23T05:00:45.506578", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.00.0093320.00344711.4780510.0129750.4345980.0307880.0000556.2693950.0361583378291.50.0587090.2435170.00002117.747446
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.0 0.009332 0.003447 11.478051 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.012975 0.434598 0.030788 0.000055 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 6.269395 0.036158 3378291.5 0.058709 0.243517 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.000021 17.747446 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:00:45.590436Z", + "iopub.status.busy": "2024-03-23T05:00:45.589543Z", + "iopub.status.idle": "2024-03-23T05:00:46.129420Z", + "shell.execute_reply": "2024-03-23T05:00:46.128427Z" + }, + "papermill": { + "duration": 0.561157, + "end_time": "2024-03-23T05:00:46.131553", + "exception": false, + "start_time": "2024-03-23T05:00:45.570396", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:00:46.169864Z", + "iopub.status.busy": "2024-03-23T05:00:46.169547Z", + "iopub.status.idle": "2024-03-23T05:06:30.883988Z", + "shell.execute_reply": "2024-03-23T05:06:30.882972Z" + }, + "papermill": { + "duration": 344.73738, + "end_time": "2024-03-23T05:06:30.887174", + "exception": false, + "start_time": "2024-03-23T05:00:46.149794", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:06:30.930207Z", + "iopub.status.busy": "2024-03-23T05:06:30.929230Z", + "iopub.status.idle": "2024-03-23T05:06:30.957184Z", + "shell.execute_reply": "2024-03-23T05:06:30.956273Z" + }, + "papermill": { + "duration": 0.050061, + "end_time": "2024-03-23T05:06:30.959477", + "exception": false, + "start_time": "2024-03-23T05:06:30.909416", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:06:30.994693Z", + "iopub.status.busy": "2024-03-23T05:06:30.993949Z", + "iopub.status.idle": "2024-03-23T05:06:31.000929Z", + "shell.execute_reply": "2024-03-23T05:06:30.999307Z" + }, + "papermill": { + "duration": 0.027723, + "end_time": "2024-03-23T05:06:31.003832", + "exception": false, + "start_time": "2024-03-23T05:06:30.976109", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.4281892660688131}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:06:31.048480Z", + "iopub.status.busy": "2024-03-23T05:06:31.048023Z", + "iopub.status.idle": "2024-03-23T05:06:31.542381Z", + "shell.execute_reply": "2024-03-23T05:06:31.541252Z" + }, + "papermill": { + "duration": 0.518801, + "end_time": "2024-03-23T05:06:31.544950", + "exception": false, + "start_time": "2024-03-23T05:06:31.026149", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGGklEQVR4nO3deXxU1f34/9edfcnMJCE7JIQdZRFkUWwVUKsotVL99GNdQUVtxaqltoV+Hx8VbR/WVsGd9tdWqK3afmzd6oZ8lKVuKAiCSNkJISxJyDqZfe75/TGTkSEJZCYzmTvkPB+PPJK5c+be981M3jn33LMoQgiBJElSBukyHYAkSZJMRJIkZZxMRJIkZZxMRJIkZZxMRJIkZZxMRJIkZZxMRJIkZZxMRJIkZZxMRJIkZZxMRFmmsrKSb3/722k9hqIo3H///Sctd//996MoSlpjkfoGmYh62UcffcT9999PU1NTpkORssAzzzzD8uXLMx1G2slE1Ms++ugjFi1aJBOR1C0yEUmSJPUSmYh60f33389Pf/pTAAYNGoSiKCiKwr59+1i2bBnnn38+RUVFmM1mTj/9dJYuXdrlvt59913GjRuHxWLh9NNP5+WXX044Hr/fz49//GMKCwtxOBx85zvf4cCBA52W/eCDD5g0aRIWi4UhQ4bw+9//vtNyiqJwxx138PzzzzNixAgsFgsTJkxg7dq1HX4XiqKwY8cOrrvuOlwuF4WFhfzP//wPQgiqq6u5/PLLcTqdlJSU8OijjyZ8fgBvv/02U6dOxeFw4HQ6mTRpEi+88EJcmZdeeokJEyZgtVopKCjguuuuo6amJq7MnDlzyMnJoaamhlmzZpGTk0NhYSH33HMP4XA4rqyqqjz++OOMGTMGi8VCYWEhM2bMYP369bEy3Xm/Kysr2bp1K2vWrIl9VqZNm5bU70HzhNRrvvjiC3H11VcLQCxZskT85S9/EX/5y1+E2+0WkyZNEnPmzBFLliwRTz75pLjooosEIJ566qm4fQwcOFAMHz5c5ObmigULFojFixeLMWPGCJ1OJ959992E4rnuuusEIK655hrx1FNPiSuuuEKMHTtWAOK+++6Lldu8ebOwWq2ioqJCPPTQQ+LBBx8UxcXFsbLHAsTo0aNFQUGBeOCBB8TDDz8sBg4cKKxWq9iyZUus3H333ScAMW7cOHH11VeLZ555RsycOVMAYvHixWLEiBHihz/8oXjmmWfEN77xDQGINWvWJHR+y5YtE4qiiNGjR4tf/epX4umnnxZz584V119/fVwZQEyaNEksWbJELFiwQFitVlFZWSkaGxtj5WbPni0sFosYNWqUuOmmm8TSpUvFlVdeKQDxzDPPxB13zpw5AhCXXHKJeOyxx8QjjzwiLr/8cvHkk0/GynTn/X7llVfEgAEDxMiRI2OflUTf42whE1Ev++1vfysAsXfv3rjtHo+nQ9mLL75YDB48OG7bwIEDBSD++c9/xrY1NzeL0tJSMX78+G7HsWnTJgGI22+/PW77Nddc0yERzZo1S1gsFlFVVRXb9tVXXwm9Xt9pIgLE+vXrY9uqqqqExWIR3/3ud2Pb2hPRrbfeGtsWCoXEgAEDhKIo4te//nVse2Njo7BarWL27NndPr+mpibhcDjEWWedJbxeb9xzqqoKIYQIBAKiqKhIjB49Oq7MG2+8IQBx7733xrbNnj1bAOKBBx6I29f48ePFhAkTYo/ff/99AYg777yzQ0ztxxWi++/3qFGjxNSpU7txxtlNXppphNVqjf3c3NxMfX09U6dOZc+ePTQ3N8eVLSsr47vf/W7ssdPp5IYbbmDjxo0cPny4W8d76623ALjzzjvjtt99991xj8PhMCtWrGDWrFlUVFTEtp922mlcfPHFne57ypQpTJgwIfa4oqKCyy+/nBUrVnS4jJk7d27sZ71ez8SJExFCcPPNN8e25+bmMmLECPbs2dOtcwNYuXIlra2tLFiwAIvFEvdce5eD9evXU1tby+233x5XZubMmYwcOZI333yzw35/8IMfxD0+99xz4+L65z//iaIo3HfffR1ee2xXh0Te775AJiKN+PDDD7nwwgux2+3k5uZSWFjIL37xC4AOH8yhQ4d26L8zfPhwAPbt29et41VVVaHT6RgyZEjc9hEjRsQ9rqurw+v1MmzYsA77OL5su87KDh8+HI/HQ11dXdz2Y5MbgMvlwmKxUFBQ0GF7Y2Nj1yd0nN27dwMwevToLstUVVUBnZ/HyJEjY8+3a2/vOVZeXl5cXLt376asrIz8/PwTxpfI+90XGDIdgBT58F5wwQWMHDmSxYsXU15ejslk4q233mLJkiWoqprpENNGr9d3axuAyPCsxl3Flai+/H53RSaiXtZZT+R//etf+P1+Xn/99bgawqpVqzrdx65duxBCxO1rx44dQOROS3cMHDgQVVXZvXt3XI1g+/btceUKCwuxWq3s3Lmzwz6OL9uus7I7duzAZrN1qFGkS3tN78svv2To0KGdlhk4cCAQOY/zzz8/7rnt27fHnk/0uCtWrKChoaHLWlEi73df6bkuL816md1uB4jr0Nj+n/bY//jNzc0sW7as030cPHiQV155Jfa4paWF5557jnHjxlFSUtKtOC655BIAnnjiibjtjz32WNxjvV7PxRdfzKuvvsr+/ftj27dt28aKFSs63ffHH3/M559/HntcXV3Na6+9xkUXXZSyWsXJXHTRRTgcDh566CF8Pl/cc+2/54kTJ1JUVMTvfvc7/H5/7Pm3336bbdu2MXPmzISPe+WVVyKEYNGiRR2eaz9uIu+33W7vE51fZY2ol7U34v6///f/+P73v4/RaOS8887DZDJx2WWXcdttt+F2u/nDH/5AUVERhw4d6rCP4cOHc/PNN/PZZ59RXFzMs88+y5EjR7pMXJ0ZN24cV199Nc888wzNzc2cc845vPfee+zatatD2UWLFvHOO+9w7rnncvvttxMKhXjyyScZNWoUmzdv7lB+9OjRXHzxxdx5552YzWaeeeaZ2H56i9PpZMmSJcydO5dJkyZxzTXXkJeXxxdffIHH4+HPf/4zRqORhx9+mBtvvJGpU6dy9dVXc+TIER5//HEqKyv58Y9/nPBxp0+fzvXXX88TTzzBzp07mTFjBqqq8u9//5vp06dzxx13cNFFF3X7/Z4wYQJLly7ll7/8JUOHDqWoqKhD7e2UkLkbdn3Xgw8+KPr37y90Ol3sVv7rr78uxo4dKywWi6isrBQPP/ywePbZZzvc6h84cKCYOXOmWLFihRg7dqwwm81i5MiR4qWXXko4Dq/XK+68807Rr18/YbfbxWWXXSaqq6s73L4XQog1a9aICRMmCJPJJAYPHix+97vfxW7BHwsQ8+bNE3/961/FsGHDhNlsFuPHjxerVq2KK9f+2rq6urjts2fPFna7vUOsU6dOFaNGjUr4HF9//XVxzjnnCKvVKpxOp5g8ebJ48cUX48r8/e9/F+PHjxdms1nk5+eLa6+9Vhw4cKBbcXX2OwiFQuK3v/2tGDlypDCZTKKwsFBccsklYsOGDXFxdef9Pnz4sJg5c6ZwOBwCOGVv5StCyHXNpNRRFIV58+bx1FNPZToUKYvINiJJkjJOthGdgk7WqdFqteJyuXopmtSrq6vr0DHyWCaT6aT9eCRtkYnoFFRaWnrC52fPnp3VU0tMmjSpQ2fDY02dOpXVq1f3XkBSj8lEdApauXLlCZ8vKytL27F7o8nx+eefx+v1dvl8Xl5e2mOQUks2VkuSlHGysVqSpIzL6kszVVU5ePAgDoejz3SFl6RsIoSgtbWVsrIydLqu6z1ZnYgOHjxIeXl5psOQJOkkqqurGTBgQJfPZ3UicjgcQOQknU5nhqORJOl4LS0tlJeXx/5Wu5LViaj9cszpdMpEJEkadrKmE9lYLUlSxslEJElSxslEJElSxmV1G5EkpYIQglAodMLxa1Ln9Ho9BoOhx91nZCKS+rRAIMChQ4fweDyZDiVr2Ww2SktLMZlMSe9DJiKpz1JVlb1796LX6ykrK8NkMsmOsQkQQhAIBKirq2Pv3r0MGzbshJ0WT0QmIimrtPlDrNlRh1GvY/qIQgz65Js5A4EAqqpSXl6OzWZLYZR9h9VqxWg0UlVVRSAQ6LCGXHfJxmopq6zZUcf2w600tgVoC6SmTSfZ/+JSRCp+f/IdkLKGPxRmV60bgOkji3BZjRmOSEoVmYikrHG42UdYFbisRgod5kyHI6WQTERS1jjYFFmfrCw30g7hD4XxBeUt93SrrKzssN5dqslEJGWNIy2RRFTqsvL+f47wzKrdbD3YkuGopFSQiUjKGg1tAQDy7SZspsgN37pW/4leIkUFAoFMh3BCMhFJWUEIQUW+jf55VvLtJgpyIp3n2pNTqgVCapdfobDa7bLBbpRNxrRp07jjjju44447cLlcFBQU8D//8z+xOcMrKyt58MEHueGGG3A6ndx6660AfPDBB5x77rlYrVbKy8u58847aWtri+23traWyy67DKvVyqBBg3j++eeTii9Rsh+RlBUUReHC04sh4IEDH5KvywfMtPiCaTne06s6Lr3dblCBnVnj+8ce/39rdxMMdz71+4A8K9+b+PXkfc9+uBfvcd0Ofvyt4UnF+Oc//5mbb76ZTz/9lPXr13PrrbdSUVHBLbfcAsAjjzzCvffey3333QfA7t27mTFjBr/85S959tlnqauriyWz9uXK58yZw8GDB1m1ahVGo5E777yT2trapOJLhExEUnb56jVo3IdTVXH6zqGF/vhDYcwGfaYj63Xl5eUsWbIERVEYMWIEW7ZsYcmSJbFEdP755/OTn/wkVn7u3Llce+213H333QAMGzaMJ554gqlTp7J06VL279/P22+/zaeffsqkSZMA+NOf/sRpp52W9nORiUjKCr5gGIO3HkPjPgAMOh0Vvv/wpaU/Ld4QhY7UJqJ504d2+ZzuuFEgt543pMuyx48Yuekbg3oSVpyzzz47bkjKlClTePTRR2ODdydOnBhX/osvvmDz5s1xl1tCiNhQlx07dmAwGJgwYULs+ZEjR5Kbm5uymLsiE5GUFT7aXU/91jV8U+elrLgQ/G6KAjUgVJq9wZT3KzIZut98mq6yPWW32+Meu91ubrvtNu68884OZSsqKtixY0dvhdaBTERSVmj1hXD4DqLPUaD8LNj3AQU2L2fm+7GZ+t5lGcC6deviHn/yyScMGzYMvb7z38eZZ57JV199xdChndf2Ro4cSSgUYsOGDbFLs+3bt9PU1JTSuDsj75pJWcHtC+IIHInUKHIHgrM//XOtTC0JUpZrzXR4GbF//37mz5/P9u3befHFF3nyySe56667uiz/85//nI8++og77riDTZs2sXPnTl577TXuuOMOAEaMGMGMGTO47bbbWLduHRs2bGDu3LlYren//cpEJGWFgLsJvRrEaDSCvQAcJZEn2uoyG1gG3XDDDXi9XiZPnsy8efO46667YrfpOzN27FjWrFnDjh07OPfccxk/fjz33ntv3BLky5Yto6ysjKlTp3LFFVdw6623UlRUlPZzkZdmkuaFwip4IgnH5CgEnT6SjICQuw6fP0SOue99lI1GI4899hhLly7t8Ny+ffs6fc2kSZN49913u9xnSUkJb7zxRty266+/vkdxdoesEUma1+YPYw80oFPA6CyMbLQV0OwN8vHm7byyoTqzAUo9JhORpHmt/iDWUCMmgw4lJ3qZYMvHqFcwqH583rYT70DSvL5Xn5WyjtmgZ1BOkByfCax5kY16I0arA2hGeJsJhdUezdaYbVavXp3pEFJKJiJJ8wodZgrzBATsYMmNbTfY89ApBzCHW2kLhHFZ+04iOtXId07SvnAQAtHLL2tubLNicWHU6zCH3LT5Q5mJTUoJmYgkzfO1HkUVAgxmMBwzOXt7Igq34pUTpGU1mYgkzfvgy92s29tAjd8cP3jL4sKoVzCH3B1GtEvZRbYRSZqnehoB0B9zWQaAxYXLamSQLkyOnEg/q8lEJGme6m0GwGjPi3/CZKfUZaXUpIN8uS5ZNpOXZpLmCV8rACa7M/4JU07ke9ADanIzHUraIBORpGmBkIouGLljZra74p802hBAOKziaWvu/eCklJGJSNI0byCMKdwWGd5hPa5GpNNxNGjk030NrNy0OzMBSikhE5Gkad5ACFPYg1GvQzE7Ojyvt0Quz8Jed2oOKASEAr3/JTqf87ozzz33HP369cPvj1/BZNasWb0yQDUdZGO1pGlGAhTn6NHrDNBJIjJEa0lhf2tqDhgOwr8fTc2+EnHuT8Bg6lbR733ve9x55528/vrrfO973wMiq2+8+eabJxxZr2WyRiRpWj9DgKGFOQwq6Qf6jrfoTbZIIhJ+N2G1+7WKbGa1WrnmmmtiK28A/PWvf6WiooJp06ZlLrAekDUiSdsC0ZpO+x2y4xgtDhTAFPbgC4ax93ReIr0xUjvpbZ0k2RO55ZZbmDRpEjU1NfTv35/ly5czZ86cuMn0s4lMRJKm+dtaMAqBrpPLMgCdJQe9TsEY9uJNRSJSlG5fImXS+PHjOeOMM3juuee46KKL2Lp1K2+++Wamw0qaTESSpm3aVY26t4Ei42A6XbTHaMOo12FQfX1umMfcuXN57LHHqKmp4cILL6S8vPzkL9KojLYRLV26lLFjx+J0OnE6nUyZMoW33347kyFJGhP2tQCgtzg7L2CMLEFd4VCwGPvWah7XXHMNBw4c4A9/+AM33XRTpsPpkYwmogEDBvDrX/+aDRs2sH79es4//3wuv/xytm7dmsmwJA0R/shtecPxfYjaGW1U5Ns4rcCQ8rXNtM7lcnHllVeSk5PDrFmzMh1Oj2T00uyyyy6Le/yrX/2KpUuX8sknnzBq1KgMRSVpSXsiMtq6rhEBEPRG+uJkaWNtsmpqarj22msxm7M7CWumjSgcDvPSSy/R1tbGlClTOi3j9/vjOnG1tLT0VnhSpgQ9AJgsnd81iwzzEITDKmGfG7O180btU01jYyOrV69m9erVPPPMM5kOp8cynoi2bNnClClT8Pl85OTk8Morr3D66ad3Wvahhx5i0aJFvRyhlClCCJT2RGTrIsHo9OxvUTlY30So4ADnnnFaL0aYOePHj6exsZGHH36YESNGZDqcHst4IhoxYgSbNm2iubmZf/zjH8yePZs1a9Z0mowWLlzI/PnzY49bWlqy+k6BdGJ+vx+dGgTAbO2iRgQoJhvQRMjn6aXIMq+rdcuyVcYTkclkiq3FPWHCBD777DMef/xxfv/733coazabs/5aWOo+EfBQ6DATFgoGk6XLcnpTZC6ikF8uK5StMp6IjqeqaofBfFLfZMXP0MKcyBizEzRC682RRKT6k6sRiQQGnEodpeL3l9FEtHDhQi655BIqKipobW3lhRdeYPXq1axYsSKTYUlaEW0fwnTi2RcNZjsAaiCxRGQ0RoZVeDwerFZr4vFJQOT3B1//PpOR0URUW1vLDTfcwKFDh3C5XIwdO5YVK1bwrW99K5NhSRoR8rnRCYHOeOJE1D4ViJrgpZleryc3N5fa2loAbDZb1o7VygQhBB6Ph9raWnJzc9Hrk+9QmtFE9Kc//SmTh5c0bu+hOur3NmAK9GfCGV2XM1kjNSIR9EbutCWQTEpKSgBiyUhKXG5ubuz3mCzNtRFJUruQL9KZUTHZT1jOZMkh324ix6mgCtAnUKlRFIXS0lKKiooIBoM9CbdPMhqNPaoJtZOJSNKscLTxWW8+cSIyWuyMKHaA0wi65C6t9Hp9Sv6gpOTIidEkzVLbx5lZTpyIYqu/Bn1pjkhKF5mIJM1qvwtmMHfdmTFSwIJAEAp4CYXlskLZSCYiSbNENBG1N0Z3yWhha00Ln+08SNVR2akxG8lEJGlXIJJUTCcY3gGAwYJepwCCgM+b/riklJON1ZI2hQLkWRR8eiO241d4PZ7eiM5gBIIEkuxdLWWWTESSNgU9VPazg84AzpPUiACd0QJ4CMoaUVaSl2aSNh07vKMbHRSV6DCQYEC2EWUjmYgkTVL9bYSF+HoGxpPQR0fnhwPyFn42kolI0qSmlmY+3dvA6r3da/PRRRNW2C8vzbKRbCOSNCkYXctedLNGZLfn0M9uQrXKfkTZSCYiSZPax5npTjLOrF1BrouCYgcUaH9xRKkjeWkmaVL7tK8684mnAIkxWtpfmKaIpHSSiUjSpFB0nJn+ZMM72hmskWEesh9RVpKJSNKk9knODF0tI3ScNlXPuj0NrPpyfzrDktJEJiJJk9oTkbGbl2YGsw0BKGG/HPiahWRjtaQ9QuA0BBE2IyaXq1svMVlsKIBB9REIqxj08n9sNpGJSNKecJDSHAOlOU4oK+7WSxSDFZ1OwaAG8AdVbPLmWVaR/zYk7QlGh2noDKDv5soQRgsGnYJeDRAIhdIXm5QWMhFJ2hP0ogrR7XFmwDFTgUDAK++cZRuZiCTtCXhYv6+B93e7afZ2c0J7nR7FELkeCya4vpmUebKNSNKckM9NWIBfsWA2dP9/pSPHgVkJY9XJS7NsIxORpDmB6PCOkD6xRDSoJB/cYehmZ2xJO+SlmaQ5sfXMzPbEVl41RAfIhvxpiEpKJ5mIJM0J+iJ3zXQnWfO+A4MZgSAckFOBZBuZiCTNiY287+44s6hdjSHW7Wlgw+6D6QhLSiOZiCTNCUeHd5xshdfjKUYLAlDlQotZRzZWS5pjEX7ybEZMebkJvU4fnURNlbM0Zh2ZiCRtEYJ8Y4j8EicM6Z/QSw3maCKScxJlHXlpJmlLOABqtB+QMbHGaoMpkoiEvDTLOjIRSdoS9ESGd+gNYEhs5KrREklcIigvzbKNTESStgQ8bD/cytp9HnYcaU3opbG5i2Q/oqwj24gkbQl6CKkCvxIZTZ8Ik8WKy2oEo0AIkVhnSCmjZCKStCXoIRRWCRksmI36hF5qsdo5vdQJiqzoZxv5jknaEojUiIJ6K9YEExGG6EoeQo00ektZQyYiSVNUf1ssEVmMCX48dQbQ6REI2WCdZWQikjQl6Is0UAd1FiyGBGtEisLGQz7W7Wng4NHmNEQnpUtSiWjPnj2pjkOSgK8HvCpmO7oEG6sBVJ0JAQR9cnK0bJJUIho6dCjTp0/nr3/9Kz6f7DwmpY4u6CHPZqQoPy+p1yvRFV/lLI3ZJalE9PnnnzN27Fjmz59PSUkJt912G59++mmqY5P6IJsSYGSJk+mjK5N6vS463iwsx5tllaQS0bhx43j88cc5ePAgzz77LIcOHeKb3/wmo0ePZvHixdTV1aU6TqkvEAKC0ZpMonMRRSnRYR4hOSdRVulRY7XBYOCKK67gpZde4uGHH2bXrl3cc889lJeXc8MNN3Do0KFUxSn1BSE/IhxCIMCY2BQg7fTRSzM5OVp26VEiWr9+PbfffjulpaUsXryYe+65h927d7Ny5UoOHjzI5Zdfnqo4pb4g6OFAk5ePq9x8sKcpqV3oTJFEpAZk22U2Sapn9eLFi1m2bBnbt2/n0ksv5bnnnuPSSy9Fp4vktUGDBrF8+XIqKytTGat0qgu0EQqrkdU7kvwXabc7wGrEbJAreWSTpBLR0qVLuemmm5gzZw6lpaWdlikqKuJPf/pTj4KT+pjoOLOQzoIr0V7VUQOL8qDRCXndXCFW0oSkEtHKlSupqKiI1YDaCSGorq6moqICk8nE7NmzUxKk1EcEPYTC0V7ViXZmbNc+zENOjpZVkqoADxkyhPr6+g7bGxoaGDRoULf389BDDzFp0iQcDgdFRUXMmjWL7du3JxOSdCpoH2emsyQ+vKOdUSaibJTUuy2E6HS72+3GYrF0ez9r1qxh3rx5fPLJJ6xcuZJgMMhFF11EW1tbMmFJ2S468j4yziy5GtFRv47P9jWwbqdcySObJHRpNn/+fAAUReHee+/FZvu6r0c4HGbdunWMGzeu2/t755134h4vX76coqIiNmzYwHnnnZdIaNKpIHDsgNfkEpHBZCGkChQ5XWxWSSgRbdy4EYjUiLZs2YLJ9PVUniaTiTPOOIN77rkn6WCamyMDFfPz8zt93u/34/d/PfteS0tL0seStEcE2nBajeS7chOfAiTKFJ2lUYSDqKEgOoNstM4GCSWiVatWAXDjjTfy+OOP43Q6UxaIqqrcfffdfOMb32D06NGdlnnooYdYtGhRyo4paYsS9DCi2MGIMwaBKclEZLEBCiDw+zxYc1wpjVFKj6TaiJYtW5bSJAQwb948vvzyS/72t791WWbhwoU0NzfHvqqrq1Mag5Rh7XMIJdmrGkCv1yH0kZp60Cd7V2eLbteIrrjiCpYvX47T6eSKK644YdmXX345oSDuuOMO3njjDdauXcuAAQO6LGc2mzGbzQntW8oSQiACbYBASXKcWTvFaIGwn4BfjsDPFt1ORC6XKzYZucuVmuquEIIf/ehHvPLKK6xevTqhW//SKSbopckTYMeRVg66GvjumY7k92W0gK+ZoExEWaPbiWjZsmWd/twT8+bN44UXXuC1117D4XBw+PBhIJLorFZrSo4hZYloZ8aAYkaQZGfGqBx7DvpQAwYh563OFkm1EXm9Xjyer//bVFVV8dhjj/Huu+8mtJ+lS5fS3NzMtGnTKC0tjX39/e9/TyYsKZsF2gipPetD1G7MwCJOL3VSaOm8v5ukPUkN8bj88su54oor+MEPfkBTUxOTJ0/GZDJRX1/P4sWL+eEPf9it/XTVMVLqg44ZZ5Z0r+p2hmhtWvYlyhpJz9B47rnnAvCPf/yDkpISqqqqeO6553jiiSdSGqDURwSi48ySmTT/eIboDQ05zCNrJFUj8ng8OByRxsR3332XK664Ap1Ox9lnn01VVVVKA5T6iICbkKoS0NsTXljxeNvqA7RVNWClllFDUxSflFZJT57/6quvUl1dzYoVK7jooosAqK2tTXn/IqmPCPRgPbPjqHoLwbAgJOetzhpJveP33nsv99xzD5WVlZx11llMmTIFiNSOxo8fn9IApT4i0IbNqCc3Nw+npWfDMgzReavlLI3ZI6lLs//6r//im9/8JocOHeKMM86Ibb/gggv47ne/m7LgpD4k0MrAfnYGjhkC+T3r0Kg3RxKRkI3VWSOpRARQUlJCSUlJ3LbJkyf3OCCpjwpEp34xJT+8o50xOm+1kI3VWSOpRNTW1savf/1r3nvvPWpra1FVNe55uRKslJC44R09T0Qma2QfIijbiLJFUolo7ty5rFmzhuuvv57S0tLY0A9JSkrQgz8YYlN1M/8x1jLnmz274WGMXpoR8kfWSpOfT81LKhG9/fbbvPnmm3zjG99IdTxSXxRoiw7vsBASPezMCFisduxmPUadLpKMjN2fNVTKjKQSUV5eXpeTl0lSwgJugtEpYq1JzkN0rByrhbEVBRAOQcgrE1EWSOrfz4MPPsi9994bN95MkpIWaIsmIlvSMzN2EFvNw3/icpImJFUjevTRR9m9ezfFxcVUVlZiNMb3+/j8889TEpzUR0Q7Mwb0VmwpqBEBkUTkd8thHlkiqUQ0a9asFIch9WmxSzMbzhTViD474EFpaWDgwGYK81KySymNkkpE9913X6rjkPqyaGN1UG9N2aWZHyP6sCDok80H2SDpWxRNTU388Y9/ZOHChTQ0NACRS7KampqUBSf1EYE2LEY9eXl55NlNJy/fDUq0gTooh3lkhaRqRJs3b+bCCy/E5XKxb98+brnlFvLz83n55ZfZv38/zz33XKrjlE5lfjf9c630HzcE8nowRewxdNHxZmE5XWxWSKpGNH/+fObMmcPOnTvjVna99NJLWbt2bcqCk/qIgDvy3ZSTsl3qjdFEFJC9q7NBUonos88+47bbbuuwvX///rF5pyWpW0IBCPkRCDCnpjYEXw98DcuBr1khqURkNps7XWV1x44dFBYW9jgoqQ8JuFGF4OMqN3/4qAZ/KJyS3RqibUSqrBFlhaQS0Xe+8x0eeOABgsEgAIqisH//fn7+859z5ZVXpjRA6RTnbyEYVvHpbHgCYUz6ng/xALDac7Cb9ViVYEr2J6VXUu/6o48+itvtprCwEK/Xy9SpUxk6dCgOh4Nf/epXqY5ROpX5W6OdGe1YTbqUDaAeXlbA2P65DMpNeqYbqRcl9S65XC5WrlzJhx9+yBdffIHb7ebMM8/kwgsvTHV80qnO30owpEYTUQqTRmyIh2wjygYJv/OqqrJ8+XJefvll9u3bh6IoDBo0iJKSEoQQckoQKTF+N4FwJBHZUzW8A74e6BryyalAskBCl2ZCCL7zne8wd+5campqGDNmDKNGjaKqqoo5c+bIaWJ7m9+d/Wt3+VsIhgUBvQ27OXU1onqfwsbqRjbvPwpqKGX7ldIjoXd++fLlrF27lvfee4/p06fHPff+++8za9YsnnvuOW644YaUBil1omYD7FwJeiOM+W/ILc90RMnxt0ZqREY79hRemukMZrxBgUEnIrUifc8m5JfSK6Ea0YsvvsgvfvGLDkkI4Pzzz2fBggU8//zzKQtO6oKvBXa9H7nkCAVg+1tw3HS9WcPfitmgI79fAfkpGt4BYDbqCevMqKqQU8ZmgYQS0ebNm5kxY0aXz19yySV88cUXPQ5KOolDX0QuN3KKIm0hngZo3JvpqBKnhiHoocxl5TuTR3B6WerWxDMbdIR0JgTg98lEpHUJJaKGhgaKi4u7fL64uJjGxsYeByWdRP32yPcBk6BoVOTn2m2ZiydZ/tZIrU6nB2PPlhA6nkGvQ+gjS08H5XgzzUvoojwcDmMwdP0SvV5PKCQbBtPK2wTuOnwhweojDqw+P+eEVYyN+7Lv7lB0jJkw2dNzt9VoAT9yKpAskFAiEkIwZ84czGZzp8/7/XJazrRrPoAQgs+bLGwjiE61k9/gZXSJguJtBFsWzSXubyWkqqza7WVfcA9zvlGJMUU9q+GYqUDk0tOal1Aimj179knLyDtmadZSQ4MnwGHRHwBVZ8BvLSYUbsHYVJVdicgXuXXv0+cQCKspTUIAdlsOFo8enSr/QWpdQolo2bJl6YpD6q7mA9S2+nHbijhrUD5nDszDsn807P8EWrNs5gNfM8Gwit+Qk9rOjFGTh5WCeT9YRMr3LaVWav8FSekVDhJqraXFG6TVVMzIUicWox4c0aW/szARBUIqfoMDWwo7M8YYogstZnunzz5AJqJs4jmK2xcgoJixOXLJs0U76TlKCIZVAs1HIrfEs4WvKVIj0jtS2pkxxhBty5TjzTRPDk3OJm115FpNnDV6ECNH9o/dafrwQAhxwEOlU8eAtrqva0haJkSkRhRW8ZsiU3ak2s6GEO7qRmz+ekaMSfnupRSSNaJs0lYHgMlVQqHj6zuXNrOBNkM+bn8oVkbzgl4IB6OXZjk4LKkfghHUmfEFVQLy9r3myUSUTdrqI9/tBXGby3KteI15tPpCiLajGQgsCb5mAEw2F2X5jpQO72hnMstZGrOFvDTLIu6GQ+w/1EKo0MjY/l9vL8gxEzTnEXIL2hqPkLop6NMomogG9i9l4JnpGbBrMkd6a6uysVrzZI0oW4SDeFsaafYG2dlmiXtKr1Ow5hUB4G7Mkjtn0USExZW2Q5isdgBEyJ+9g4L7CJmIsoW3CW8wTEhnJs/VcXCoIz/SQO1vqYdwFgyz8TUjhCBsSl8islgit+9DqpB3zjROJqJs4WvCEwjhNzjoZ+84xCYvN5+wzognEAJvFgw89jXT6g/x9y+b+ftn+9NyCIvZRFgxElaFXFZI42QiyhbeJjyBMD6Dk345HRt2S3Nt5BeW0c9uAk99BgJMkK+JQEjFp3OkbXphk16H0WzFbtYT9LWl5RhSasjG6iwRbDuKP6Tit3VeIyp0mCkcMggOuyPzE2mZEOCNJiKLkzJLej6GOp3CWcPLwF0HclkhTZM1oizhborUcnTWPKxdjcuy5kW+a/3SzNcMaghfGPz69PQhiomt5iEHvmqZrBFliVBbY+TumKtfl2WCZhcBfwia67D3YmwJ80ZqbK2KHRQdLmsvJKKg7NSoZTIRZQMhKDT4KKjMIzBueJfFNtUphGqaKfDpGdaL4SUsWmNrFJG17tOZiDbXBQkfaKSf6ygV/U9eXsoMeWmWDfytoIZQFD1me16XxXJyIz2uA20tkUn1tcrTiBCCRhHpeumypS8R+URkmIff407bMaSey2giWrt2LZdddhllZWUoisKrr76ayXC0y9cU+W5xgq7rtyzP5SKks+ALhrXdTuRtJCwExcWl9M+z4kjHFCBRenPkIlX1y7tmWpbRRNTW1sYZZ5zB008/nckwNE94G9lS08zntQJvoOtpPvLsRnwGJ4GwwN+q4Vv43gYMOh3njBnGf08sT+vqwHpLpNYVDsgakZZltI3okksu4ZJLLslkCFnB19qA2x+i1mfmDEPX/zvMBj2KLQ8CtbQ21WMu68Ugu0tVIwsAwNd3+dLIYGmvEcnGai3LqsZqv98fN0F/S0tLBqPpPZ6WyIh6nS0Pve7EtQdjTj9ogramOgpOWDJDfE0gVILo0BkdpH4WonjGaI1IBGQi0rKsaqx+6KGHcLlcsa/y8ixdZjlBvmgiMuecfGJ8qyuSfnytGp0OJNrZ8qsmA0+t2s3Wg81pPZzFFrkzpwY8kY6UkiZlVSJauHAhzc3Nsa/q6upMh9QrgtGkYnV23Yeo3YDSMgbm2yg1aXQOnujEbfWqE1UInOnszAhY7TmYDTrMeiKTsUmalFWXZmazucs11U5ZoQBBX6Sh1e46+cVWWWkZ5FqByAyI6NP7h56wtjrCquAokRkEOhs3l0pFLjtFg0siPauDXjCldkVZKTWyqkbUJ/ma8AVVQjozTofj5OWN1q8njfel97InKW11eINhPIY8bCY9tnRMmn88U7SfeVDewteqjNaI3G43u3btij3eu3cvmzZtIj8/n4qKigxGpiHeJnQKhEwucrvT8U9RaNM78TUfwNpci82uoSZrVQVPA95AGI8xj8I0TA/bKWN0WSHZYK1ZGU1E69evZ/r06bHH8+fPByIryi5fvjxDUWmMr4lRZS5OLxwG3fzD3VgH+no3/euOUFF2epoDTICvCdQQbWEFv6Xz6UzS4fNDAZT6RsrLGiko6pVDSgnKaCKaNm0aQt7JOLFonxvFmgvd7PhncfYjWA+eRo2t6BFtqG7EAYrS6XQmaTmsMGIIqfg8rb1yPClxWdVY3Se1D++w5nb7JVZXIUHAp7Xe1dFElFfYn+FOB6W5lpO8IDX05khfIjk5mnbJRKRxVTU1NNU3YSzWc3o3e0rn5BbSAoTaNDZBmrsWgMqKgVRWlPbaYQ2WHMIQu/soaY+8a6ZlqkqgtQFPIIxX33HC/K7k5hcDEPY0oQY1NArffSTyPae4Vw9rjA7zCMsakWbJRKRlgVb8wSACBYer++OyHA4nQm9GFdCilXaioBe8TbQFQjTq8nu1bdBojQ58lSPwNUsmIi3zNuELhvEbHOTaut+wq9Pr0Nkjw0FaGmvTFV1iWiPrre1o1rP8syNsPdh74wTN9khtUvhlY7VWyTYiDfO7GwiGBX6LM+HJwwYOGID5aCs2g0b6zriPoArBYTVSsyvLtfbaoa12F2aDDguByJpvevmx1xr5jmiYpzly10tYXJgNiY1TLy0pBd8eCGlkhoLWQ7j9IVqM/bCb9eSlcVbG4w0ozGdAZSGoIQi4E7oDKfUOeWmmYd7oqHtTN0bdd2CNvkYrSwu1HqbZG6TNVMCAPFtaJ0PrQFEgegsfOUGaJslEpGF6f1NkPFZu4sM0wpZcGjwBqg/WZL7TaKANvE20eIO4jQUMyOu9y7IYUzQR+WUi0iJ5aaZhxUYfxQNy4fShib/YmseOw60IWnF5vDjtGRx13lxDSFWpVZ2E9RYG5vf+YkfrDwXQ1TdSWXaUfDnMQ3NkjUirgr6v589Jok1Db7ZjtESST9PRDN85a66m0ROk2VRMQY4prat2dMUtLPhDKr42Dc5IIMlEpFm+JgQiMn+OIbkxWcZo21Jr45FURpa45gPk201MGDuGKUNOPrlbOhiskVv4/jaNNN5LcWQi0qig+yif7Wvk44NhgmE1qX1YnIUAtDZksFNjOAjuI+gVhfLK4Qwt6sacSmlgik4ZG/LKRKRFMhFplLupnrAqaCYHoz65tyknL9IY4m3O4KVZSw2o4chdK4srY2GYbJFjh7yyU6MWyUSkUd6WSB8iYzK37qNchZFRsmF3bdK1qh5r3MeuOjf/8eXTdoI12dLNbI8kItUna0RaJBORRvlbI/1/zDnJt6nYcosx6hWsgSaOtvpP/oI08BzZRX2rn89aXITCmetGYHXkAhAK+CLzV0uaIm/fa1T7yh0WV/KJSLHmM7QkF6OiYjX6gF7uvxPwUHuwCgE4S4dl5G5ZO4fdTp3ZhlkfjMzlnSPv4WuJTERaFA4RjDaq2l2Fye9HpyO3X0lkHiBvPdjTv7LqsQL1u6lt8eEx5jN2yIBePfbx+uWY6TeyElqPyESkQfLSTIt8zfiDIcKKEZezhw289mgia+v9O2fVu74krArU3Eoq+2lgGZ/2xnLZTqQ5MhFpkNpWj91sQG/PJz+nZ/M6C3sBh1t8fL5tB75g7zUW+wMB6vZtBaBixLjeHVvWlVgiaspoGFJH8tJMg3TeBkYUO6B4OBh7tjq8Yi/iULOXRg5yuNlHZUHvDK/4z7YvEUEfBksOw4aO7JVjnsymOlD3N1KgP0RFEqNmpPSRNSIt8kTXrbeloBdyThEOsxFrsJFDDb13STJUOUCRw0zp0HHokuwHlWp+vR1/SMXvbsx0KNJxtPEJkeKE3PWR4R22FCyOaHZgd7hQEDTUHuj5/rojHMLespshhTkMGHZG7xyzGyyOSJ+ssKcps4FIHchEpDVCsGPvPj7b18iuthSs+6UoOIoiq+a21VUTCKW3Y2ObP4So3x4ZtGtxQu7AtB4vEXZnJBGFfG0Q0tCiApJMRJoTcOP3eQirYOpBZ8Zj2fv1x2zQYfXVUt2YvqljfcEwL366n42friGkqlAyFnTa+Yg5HQ5COjP+UBi88vJMS7TzKZEACLXW4w+q+AwO8p2pueWtOMvIsxnJCdSxrz49K1kIIVj1n1qCrfUoDfsisyKWjEnLsZLltBrxGnMJhAVBt8YWn+zjZCLSmNbGIwggaMnHburZHbMYRym5NhPWUHNktsQ0WF/VyH8Ot9K/dTNDinIwFA7X3NzQZoOOsDkXgDatrG4iAfL2vea01tcAYHIWpa7vjcmGs6CM8aYjmEtTP85q68FmPtxVjynkZpK1BqfFBBVnp/w4PaUoCjn5xTgDu7++MylpgqwRaYy34SAA1n79U7pfff6gyEogjVUp26cQgs/3N/Lu1iMIAVONX1HqMEFuBbgyO6SjK+eOHcmoMhe5yOlAtEQmIi1RVYItkdkUXQWpTUTkRe9eNVVx1O3H7Q/1eJfeYJhP90ZmCTi7wMso/f7IE0Om93jfaWNrX93kKGR6UQEpRiYiLfE2kGfR4cqxUVBYktp9u8pB0VFzsIaX/v0Fn+xO7tKk1ReMzW1kMxmYMaqE84bmcnbgExQUKBkNzrJURp5a1jxQdKihAPjlmDOtkG1EWuI+Qv9cK/1d/cFhSe2+jRbIq8Th3U5+216+PJjLiBIH5fknvzPX4guyp66NPXVuqhu8TB9ZyNgBuQBU9rNRWfteZP00cw4MvTC1cadYW1DwxSGBwdvAxNFH0GVw1kjpazIRaUl0fXhyUlwbalc4AmfDHsYED1EjxvPG5kNcPq6sw/LPqiqoafKyv8HDvqNt1LbEN3AfaX8sBOxZBUe2gqKD0y4DYwbWLEuA1ainUZdPvqjH01BDTtHwTIckIRORprTW7ccSVjE60pSICobDzncZbvew3djCXr+T/11fTXmejYp+NiZVRodACMGrG2sIqZE2FEWBMpeVwYV2BhfmkG83RdaQ3/kuHPoisu8RMyCvMj1xp5BOp2DMLQX3Dlrqa8jJdEASIBORdoRD7N2zkzZfgOLhuQxOxzFMNigcif7IVmbm1/AeA9h2qIX9DR7aAqFYIjLqdQwtykFRoDzfxqACOzbTMR+V5gOw/W1oq49kqWEXQal2xpSdTE5BfzgA3oaaTIciRclEpBGBpho8vgBBnTX1DdXHGjARjmzFWL+NGRPP4axBlexv8MRqP+0uGVPa8bWth2HfB1C/M/LYZIeRM6HfkPTFmwa5RQNoBnytjZEOnqbeX3lWiicTkUY0HtqLAEI5pTgsaZzb2VkGhcOhbgdsf4u8cdeSV57bdflwEOp3QM0GaI7WINqHbwyeHqllZZmiPBeHjHnoA02ojdXoirUxX1JfJhORRjQd3AWAragy/bMZDrkg0rGx5SB89SqM/Hbkrlq7UACaq6H2q0gSah+pruigaCQM/AbYUzBFSYbk20x4bf2xNTfSemQPLpmIMk4mIi0IB/HW7QWgX8Vp6T+eNRdOvxy+fDlymfXJ05F+RjpDZGL5trrIoojtLM5IG1DpuMgt+iyn0ynk9x9GsdiJoaU60+FIyESkCe4ju/D6/AQMOQwoK++dg/YbAuOu/rrR+eju+OctTug3DIpOiwzX0MKc0yl0zoRx4FsLwQbwu0+JBJvNZCLSgNq9X0V+yB+C1dyLb4lrAEyaG7lEa6uNLg3tiCy1Y8k95ZJPHJMdHCWRBvijO6FsfKYj6tNkIso0NUxFuApHYQ5tI8f1/vEVBVz9I199jCgcQWtdNVR/iVMmooySY80y7ehuDGE/hQX9qBwyKtPR9CmfuYvYerCFw1XbI5dnUsbIRJRh4sCnkR+KR2tqWtW+oGLAANymIhrdPnz712c6nD5NfvIzqK2uis1fbuFQix+17MxMh9PnFDvNBEvPRBVweNtHEEr9pHFS98hElCmqyt6PX8MTCLNLV4licWY6oj5HURSGjpqIz+Ck9mgjnl3/znRIfZYmEtHTTz9NZWUlFouFs846i08//TTTIaVd1YZ3aK3bj6ozMuzsb2tjSeY+aFiJC2/FeYRVwb6N76Me3ZvpkPqkjCeiv//978yfP5/77ruPzz//nDPOOIOLL76Y2tpTc3JzEQ6xZ90bHNr8PgD20d+mpDB7eylnO0VRmDzxbBodw2n2BKha81xk+IvUqxQhMjtf5llnncWkSZN46qmnAFBVlfLycn70ox+xYMGCE762paUFl8tFc3MzTqeGL21CfvA20lCzkz2bPyTYGlnKRjf4PCZNvQydTtaGMm3XoUYO/PuvnJ3bjMWoh7xKWpzDMOWXY87JQzGYEtqfEAIhQER/Nhyz7LYvGEYVAp2ioNcp6BXllP0MdPdvNKP9iAKBABs2bGDhwoWxbTqdjgsvvJCPP/64Q3m/34/f/3WDYktL96f6rGny8v62yFI97dpTsBCCyYP6cXpZ5Bd1uNnHW1sOxZU1BN0MPfwmAAPzrQzItQICty/I5gONKEJFCFAiH73IlxAMcJkod0Z+zQZfkGBrC6reTN4Zl3LauHPkJZlGDC3No/K7t2HY/wFUfwaN+9i7ZROeQBgFwGBB6PQIRY+q6LGZjZxZkRd7/bq9DXgDIQQKAhE3HbbNpGfKkGit97Tv8I8vvdS1xjeMKwroFQW72cBN3xwU2/7Ol4c40uLvtG+pUa/j6skVscf/99URDjV7Oz0/RVG47uyvV91dvb2W6sbOywJcM7kCfTQ5frirnj3HrYf3X2cOwJqq5a7IcCKqr68nHA5TXFwct724uJj//Oc/Hco/9NBDLFq0KKljBUMq9e6ulxn2h74eWxVSVZq9wbjnTaEgOk+kJqNYbGCytr8Qxdcc2d7ZjkMABjBayMkbhKugiMpRZ2GzyakntMZgNMGQ86HsTMShL2ip3YASrEMvghDyAZH3WA/oVT20ff2ZMfubUIPhTvdrETpwRzOTGkLXSVYRAkJCED5uOpZmb5CGts4/tyZDfMtKiy/Y5Wf8+GO2+kLUt3Z9lzByoaREywY7lBWk9kIqo5dmBw8epH///nz00UdMmTIltv1nP/sZa9asYd26dXHlO6sRlZeXd+vSzBsIU9fa8T9L++Ncm4mc6PAKfygce/OV9vSihjC0HkAoCnaTEZvZACgEVUGLLwwokZ3plOirIo8tJiO2nFxIsGovaUMoFMbn9xLwtCLUEIoaBjWEXgGX9evpWtz+IO1/ujpFQVFAEZGPhKIomPTRz5GjFGEwAxBWBWEhUNXIrJhq9HLu2P3WtvrwB9VOY1MUGJBn61ZZIG5+8tpWH77AicpaY7X1ercfjz8+yfbPs8ZqTCeSFZdmBQUF6PV6jhw5Erf9yJEjlJR0nBzMbDZjNpuTOpbVpKeiX/fmzjEb9JS6Opl7Oa/jdBFGIDUr1EtaZDDoyTHkgP3Eg2ITGTLb/udr0Csn/QMsSmARhXSVLcgxJ3aCScjoXTOTycSECRN47733YttUVeW9996LqyFJknRqy/ig1/nz5zN79mwmTpzI5MmTeeyxx2hra+PGG2/MdGiSJPWSjCeiq666irq6Ou69914OHz7MuHHjeOeddzo0YEuSdOrKeD+insiafkSS1Ed192804z2rJUmSZCKSJCnjZCKSJCnjMt5Y3RPtzVuJDPWQJKn3tP9tnqwpOqsTUWtrKwDl5b208oUkSUlpbW3F5XJ1+XxW3zVTVZWDBw/icDhOOni0fThIdXX1KXOH7VQ7p1PtfODUO6dEz0cIQWtrK2VlZehOMBVyVteIdDodAwYMSOg1TqfzlPhAHOtUO6dT7Xzg1DunRM7nRDWhdrKxWpKkjJOJSJKkjOszichsNnPfffclPXpfi061czrVzgdOvXNK1/lkdWO1JEmnhj5TI5IkSbtkIpIkKeNkIpIkKeNkIpIkKeNO6UTU0NDAtddei9PpJDc3l5tvvhm3233C10ybNg1FUeK+fvCDH/RSxB0lugruSy+9xMiRI7FYLIwZM4a33nqrlyLtnkTOZ/ny5R3eC4ul+3Mtp9vatWu57LLLKCsrQ1EUXn311ZO+ZvXq1Zx55pmYzWaGDh3K8uXL0x5nIhI9p9WrV3d4jxRF4fDhwwkd95RORNdeey1bt25l5cqVvPHGG6xdu5Zbb731pK+75ZZbOHToUOzrN7/5TS9E21Giq+B+9NFHXH311dx8881s3LiRWbNmMWvWLL788stejrxzyazq63Q6496LqqqqXoz4xNra2jjjjDN4+umnu1V+7969zJw5k+nTp7Np0ybuvvtu5s6dy4oVK9Icafclek7ttm/fHvc+FRUVJXZgcYr66quvBCA+++yz2La3335bKIoiampqunzd1KlTxV133dULEZ7c5MmTxbx582KPw+GwKCsrEw899FCn5f/7v/9bzJw5M27bWWedJW677ba0xtldiZ7PsmXLhMvl6qXoegYQr7zyygnL/OxnPxOjRo2K23bVVVeJiy++OI2RJa8757Rq1SoBiMbGxh4d65StEX388cfk5uYyceLE2LYLL7wQnU7XYb204z3//PMUFBQwevRoFi5ciMfjSXe4HbSvgnvhhRfGtp1oFVyInPOx5QEuvvjiLsv3pmTOB8DtdjNw4EDKy8u5/PLL2bp1a2+EmxZafn96aty4cZSWlvKtb32LDz/8MOHXZ/Wg1xM5fPhwh+qhwWAgPz//hNev11xzDQMHDqSsrIzNmzfz85//nO3bt/Pyyy+nO+Q4ia6CC5Fz7qx8otfr6ZDM+YwYMYJnn32WsWPH0tzczCOPPMI555zD1q1bEx7srAVdvT8tLS14vV6s1k7W0tO40tJSfve73zFx4kT8fj9//OMfmTZtGuvWrePMM8/s9n6yLhEtWLCAhx9++IRltm3blvT+j21DGjNmDKWlpVxwwQXs3r2bIUOGJL1fKXFTpkyJW9/unHPO4bTTTuP3v/89Dz74YAYjk9qNGDGCESNGxB6fc8457N69myVLlvCXv/yl2/vJukT0k5/8hDlz5pywzODBgykpKenQCBoKhWhoaOh0FdmunHXWWQDs2rWrVxNRoqvgApSUlCRUvjclcz7HMxqNjB8/nl27dqUjxLTr6v1xOp1ZWRvqyuTJk/nggw8Sek3WtREVFhYycuTIE36ZTCamTJlCU1MTGzZsiL32/fffR1XVWHLpjk2bNgGRKmhvSmYV3ClTpsSVB1i5cqUmVs1Nxaq+4XCYLVu29Pp7kSpafn9SadOmTYm/Rz1q6ta4GTNmiPHjx4t169aJDz74QAwbNkxcffXVsecPHDggRowYIdatWyeEEGLXrl3igQceEOvXrxd79+4Vr732mhg8eLA477zzMhL/3/72N2E2m8Xy5cvFV199JW699VaRm5srDh8+LIQQ4vrrrxcLFiyIlf/www+FwWAQjzzyiNi2bZu47777hNFoFFu2bMlI/MdL9HwWLVokVqxYIXbv3i02bNggvv/97wuLxSK2bt2aqVOI09raKjZu3Cg2btwoALF48WKxceNGUVVVJYQQYsGCBeL666+Pld+zZ4+w2Wzipz/9qdi2bZt4+umnhV6vF++8806mTqGDRM9pyZIl4tVXXxU7d+4UW7ZsEXfddZfQ6XTi//7v/xI67imdiI4ePSquvvpqkZOTI5xOp7jxxhtFa2tr7Pm9e/cKQKxatUoIIcT+/fvFeeedJ/Lz84XZbBZDhw4VP/3pT0Vzc3OGzkCIJ598UlRUVAiTySQmT54sPvnkk9hzU6dOFbNnz44r/7//+79i+PDhwmQyiVGjRok333yzlyM+sUTO5+67746VLS4uFpdeeqn4/PPPMxB159pvXR//1X4Os2fPFlOnTu3wmnHjxgmTySQGDx4sli1b1utxn0ii5/Twww+LIUOGCIvFIvLz88W0adPE+++/n/Bx5TQgkiRlXNa1EUmSdOqRiUiSpIyTiUiSpIyTiUiSpIyTiUiSpIyTiUiSpIyTiUiSpIyTiUiSpIyTiUjKKsuXLyc3Nzf2+P7772fcuHGxx3PmzGHWrFm9HpfUMzIRSZ2aM2dOl/N1z5s3D0VR4mZBSEcCqKys5LHHHovbdtVVV7Fjx44uX/P444/HzQM9bdo07r777pTGJaWeTERSl8rLy/nb3/6G1+uNbfP5fLzwwgtUVFRkJCar1XrC+ZBdLldcjUnKDjIRSV0688wzKS8vj5ud8uWXX6aiooLx48f3aN+d1VRmzZoVq2VNmzaNqqoqfvzjH8dWhoCOl2bHO7ZmNmfOHNasWcPjjz8e28fevXsZOnQojzzySNzrNm3ahKIoWTvXUbaTiUg6oZtuuolly5bFHj/77LPceOONaT/uyy+/zIABA3jggQdiK0Mk6vHHH2fKlClxq7JUVFR0OCeAZcuWcd555zF06NBUnYKUAJmIpBO67rrr+OCDD6iqqqKqqooPP/yQ6667Lu3Hzc/PR6/X43A4KCkpSWqWSZfLhclkwmazxfah1+uZM2cO27dvj62pFgwGeeGFF7jppptSfRpSN2XdVLFS7yosLGTmzJksX74cIQQzZ86koKAg02H1SFlZGTNnzuTZZ59l8uTJ/Otf/8Lv9/O9730v06H1WbJGJJ3UTTfdxPLly/nzn/+cslqDTqfj+KmwgsFgSvbdHXPnzo01xC9btoyrrroKm83Wa8eX4slEJJ3UjBkzCAQCBINBLr744pTss7CwMK7dJxwOd1iR1mQyEQ6He3ScrvZx6aWXYrfbWbp0Ke+88468LMsweWkmnZRer48t0aTX67ss19zcHFtsoF2/fv0oLy/vUPb8889n/vz5vPnmmwwZMoTFixfT1NQUV6ayspK1a9fy/e9/H7PZnNQlYWVlJevWrWPfvn3k5OSQn5+PTqeLtRUtXLiQYcOGnXIT2GcbWSOSusXpdOJ0Ok9YZvXq1YwfPz7ua9GiRZ2Wvemmm5g9ezY33HADU6dOZfDgwUyfPj2uzAMPPMC+ffsYMmQIhYWFScV9zz33oNfrOf300yksLGT//v2x526++WYCgUCv3AWUTkzOWS31Wf/+97+54IILqK6u7rACq9S7ZCKS+hy/309dXR2zZ8+mpKSE559/PtMh9Xny0kzqc1588UUGDhxIU1MTv/nNbzIdjoSsEUmSpAGyRiRJUsbJRCRJUsbJRCRJUsbJRCRJUsbJRCRJUsbJRCRJUsbJRCRJUsbJRCRJUsb9/4AEvLu+O0+wAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:06:31.585766Z", + "iopub.status.busy": "2024-03-23T05:06:31.584899Z", + "iopub.status.idle": "2024-03-23T05:06:32.057116Z", + "shell.execute_reply": "2024-03-23T05:06:32.056151Z" + }, + "papermill": { + "duration": 0.496285, + "end_time": "2024-03-23T05:06:32.059417", + "exception": false, + "start_time": "2024-03-23T05:06:31.563132", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCc0lEQVR4nO3deXxU5b0/8M85s+/JZA9JSCBhFQiLILiAlRZRqVzaSq1VUNy1V3DneoVir5dqXaC3ll9vW6C2KmqLeFuraFGhoiIIKFtBQjZIIAlJZl/PeX5/nMlAyMLMZCbnDPm+X6+8dOZsz2Em3zzPc57n+3CMMQZCCJERL3cBCCGEAhEhRHYUiAghsqNARAiRHQUiQojsKBARQmRHgYgQIjsKRIQQ2anlLkBfiKKIhoYGWCwWcBwnd3EIIedgjMHlcqGwsBA833O9J60DUUNDA4qLi+UuBiHkPOrr61FUVNTj9rQORBaLBYB0k1arVebSEELO5XQ6UVxcHP1d7UlaB6KO5pjVaqVARIiCna/rhDqrCSGyo0BECJEdBSJCiOwoEBFCZEeBiBAiOwpEhBDZpfXjezLweAJhbD3SDI2Kx5XDc6BW0d/SCwF9iiStbD3SjMMnXWjzBOEJCnIXhyQJBSKSNgJhAUeb3ACAK0fkwmbQyFwikiwUiEjaOOnwQxAZbAYNciw6uYtDkogCEUkbDe1+AEBhhh6AVEPyh6h5diGgQETSximnFIgKbAZ8+K9T+PVHVTjQ4JS5VCQZ6KkZSRutniAAwG7SwhepCTW7AnIWiSQJBSKSFhhjKLEb0eoNwm7SIhCWAlFHcCLpjQIRSQscx2HmqDwg6AWOb4edtwPQwekPyV00kgQUiEh6Ofg20FYDqyjC6p8GJwYhEBagU6vkLhnpA+qsJmnBHxIQdp4C2moAAGqeR4n/XwAApy8sY8lIMlAgImnh06oWbPxgKxrafYDODADIDZ4AmAiHj5pn6Y4CEUkLLn8YFn8DVDwHFE8B1DpkG3lMsAdg1FKzLN1RHxFJC25/CIODp6BV64GMwYC1GoPCAQzKDwEZBrmLR/qIakQkLQTd7VCJIWg0GsCUDVjypQ2eZnkLRpKCAhFRvLAgAl4p4GgtOQCvkoIRgLC7Ge4AdVanOwpERPE8AQGmYCt4DtBYc6Q3jdlw+EL47OvDeOvLenkLSPqMAhFRPFcgBEO4DVo1D86cK71ptEOj4qAWA/D7PPIWkPQZBSKieDq1CmXmEDKNWsCQKb2p0kBjkFYPZT6H1HwjaYuemhHFy7HokJPJgKAJ0GdE31ebMsFzx6ETXPAEBdgM9Hc1XdEnR5RPCAHBSPPLkBF9m9PboFHx0IXd8FCHdVqjQEQUz+86DZExQK0D1PozGzoCkeCKpgUh6YkCEVG8T/ZXYUd1K04EdADHndmgt0Gj4qALu+GjRPppjfqIiOKJ3jYAgOqsZhkAQG+DzaBBGS/ATIn00xoFIqJ4os8BANCYMjtv0JpQYDOgQMsDdqMMJSPJQk0zonjM7wIAaE3Wzhu00ix8hLyASI/v0xkFIqJowbAIPiQ9MdOZbJ03aoxgAARBhNfj6P/CkaShQEQUzRcUoBU80vQOwzk1Ip7H6ZAGX9S04oO9VfIUkCQFBSKiaL5gGFrBC42KB6ezdNmu0kvNM8Hn7u+ikSSizmqiaBoEkWdWQcWrgW4CkTpSSxICrv4uGkkiCkRE0bLUQWTlmAGNHlB1fUSvNUqBiAXcEEQmZXAkaYeaZkTZgpGaTscTsnNo9BZwALSCl5afTmMUiIiiBTxOaXpHN80yAOD1Zqh4DhrBR9M80hg1zYii7T1aD7G6FbmaIRja3Q4aIzQqHmrRT9M80pisNaI1a9Zg7NixsFqtsFqtmDp1Kt599105i0QURvA7AQAqvbX7HTQG2E1alFg46DW0mke6krVGVFRUhJ///OeoqKgAYwx/+MMfcP3112PPnj0YPXq0nEUjCsECbnA483SsC40RJXYjoFUDFl2/lo0kj6yBaM6cOZ1eP/3001izZg0+//xzCkQEgBSIAEBj7LlGBAAI+QDGOs/OJ2lDMX1EgiDgzTffhMfjwdSpU7vdJxAIIBAIRF87nc7+Kh6RS8gLANDqu39qJk3zYBAEEYLfDZ2h+05tomyyPzXbt28fzGYzdDod7r77brz11lsYNWpUt/uuXLkSNpst+lNcXNzPpSX9iTEGriMQGXsIMLwKdU4RO2va8MWR4/1YOpJMsgei4cOHY+/evdixYwfuueceLFiwAAcPHux236VLl8LhcER/6utpGZkLWSAQAC9K69rrDD3UiABwWikFSNjv7ZdykeSTvWmm1WpRXl4OAJg4cSJ27tyJ1atX4ze/+U2XfXU6HXQ66pAcKFjQixyLDgLjoNbqe9xP1RGIArSsULqSPRCdSxTFTv1AZOAyIIDyHLM0mLGXTmiVTgpEYoBqROlK1kC0dOlSzJ49GyUlJXC5XHj11Vfx8ccfY/PmzXIWiyhFpH8I2t6zL6p1JgCAGKRAlK5kDURNTU245ZZb0NjYCJvNhrFjx2Lz5s349re/LWexiEKE/W7wjIHX9B6IOlKBiNQ0S1uyBqLf//73cl6eKFx1YzNaqluhDQ7CxHE976c1SDUiFvJJT9poLFHaUVwfESEdwn5pMCOnNfW6n1Zvht2khdnKQWSAiuJQ2qFARBRLiHQ+q3S9ByKN3oTheRbAqgEoH1Fakn0cESE9ESPTO9T63gNRdPXXkD/FJSKpQoGIKFbHUzC1rufBjNIOejAwhIM+hAVaVigdUSAiisUigaijM7pHGj0OnHBi5zcNqD1NT87SEQUiolxBKahoe5neAQBQ6yO5qhmCfl/qy0WSjjqriTKFg8jUc/CrNDCeu8LruVQa8GoNgBCCNLo6LVEgIsoU8qI0ywTwasB6nhoRAF6jB+BFiGpEaYmaZkSZzp7eEcMAxY4Z+KEg9RGlIwpERJHEgAcCY2cyMJ6HKjI7XwjSI/x0RIGIKFK704EvqlvxcXVsfT58JGAJAWqapSPqIyKKFIqsZc9irBGZTGZkmbQQDTSOKB1RICKK1DHPjD/PPLMO2Rk2ZOdZgGxtKotFUoSaZkSROtK+8rreU4BEafQdB6aoRCSVKBARRQpH5pmpzje9o4PaIE3zoHFEaYkCEVGkjiRn6p6WETqHR1Rhx7FWfLS/LpXFIilCgYgoUkcg0sTYNFPrjGAAOCFAE1/TEHVWE+VhDFZ1CMyogdZmi+kQrd4oLU0t+hEURKhV9Dc2nVAgIsojhFBgVqPAbAUK82I6hFMbwPMc1GIQgZAIIz08Syv0Z4MoTygyTYNXAypNbMdo9FDzHFRiEMFwOHVlIylBgYgoT8gHkbGY55kBOCsVCBD00ZOzdEOBiChP0ItdNa34sMoNhy8U2zG8Cpxaao+FaH2ztEN9RERxwn43BAYEOD106tj/VlrMFug4AQaemmbphgIRUZxgZHpHWBVfICrLtwNuAYhxMDZRDmqaEcWJrmemM8W3WKI6MkE2HEhBqUgqUSAiihPyS0/N+POsed+FWgcGBiFIqUDSDQUiojjRmfexzjOLONoWxo5jrfiyqiEVxSIpRIGIKI4Qmd5xvhVez8Vp9GAARFpoMe1QZzVRHD0LINOogTYzI67jVJEkaiJlaUw7FIiIsjAGuyYMe74VGDoorkPVukggopxEaYeaZkRZhCAgRsYBaeLrrFZrpUDEqGmWdigQEWUJeaXpHSo1oI5v5qpGLwUuFqKmWbqhQESUJejF4ZMubKvx4sgpV1yHRnMX0TiitEN9RERZQl6ERYYAJ82mj4dWb4DNoAE0DIyx+AZDEllRICLKEvIiLIgIq/XQaVRxHao3mDCqwApwVNFPN/SJEWUJSjWikMoAQ5yBCOrISh5MlDq9SdqgQEQURQx4ooFIr4nz68mrAV4FBkYd1mmGmmYKJQgCQqEYc/FcQII+H6CzSD/hEPxifCk9DraoEPT5UHDqNPLy9SkqJemg0WigUsVZc+1GQoHo2LFjGDJkSJ8vTrpijOHkyZNob2+XuyiyEFk+DMO/jcEqPWpra+I+XjX4UuiZCKfTA6+vOvkFJF1kZGQgPz+/Tw8HEgpE5eXlmD59OhYtWoTvf//70OvpL0+ydASh3NxcGI3GAffkR/C0IhQMIqy1wGyKb64ZAHgdLWBCCGqjDTo9JSZKJcYYvF4vmpqaAAAFBQUJnyuhQLR7926sW7cODz74IO6//37Mnz8fixYtwuTJkxMuCJGaYx1BKCsrS+7iyCOsBjQcYLIAqviX4hD9OoghBpVWQ38g+4HBII1mb2pqQm5ubsLNtIQ6qysrK7F69Wo0NDRg7dq1aGxsxGWXXYaLLroIL7zwApqbmxMqzEDX0SdkNA7Qv+SMSU+8gMQfwXfUIEWWnDKR8+r4vvalT7NPT83UajXmzZuHN998E8888wyOHj2Khx9+GMXFxbjlllvQ2NjYl9MPWAOtORbFpIGIDAC4xP6ycpEAxhit9tpfkvF97VMg2rVrF+69914UFBTghRdewMMPP4yqqip88MEHaGhowPXXX9/nApIBhIkICSI8QQGuQIIJ8Hk+ei6SPhIKRC+88ALGjBmDadOmoaGhAS+//DJqa2vxX//1XygrK8Pll1+O9evXY/fu3ckuL7mALFy4EHPnzj3zBhPBGCCCR6J/Y2ddMwePP7kCPNdz06y0tBSrVq1K8AokFRIKRGvWrMGPfvQj1NbWYtOmTbjuuuvA851PlZubi9///vdJKSRRvhkzZmDx4sV9OwkTwcAAjk+4us9zHNQ8B71qgDZvk+inP/0pKisr++VaCT01++CDD1BSUtIl+DDGUF9fj5KSEmi1WixYsCAphSQDRKRGxMCB72u/A6PO6nSSUI1o6NChaGlp6fJ+a2srysrKYj7PypUrcfHFF8NisSA3Nxdz587F4cOHEykSkdHChQuxdetWrF69GhzHgeM4VFVVYdGiRSgrK4PBYMDw4cOxevXqbo9fsWIFcnJyYM3Ox08eWopAKIxYJt57PB7ccsstMJvNKCgowPPPP3/WVqmPqKmpCXPmzIHBYEBZWRleeeWVLufhOA5r1qzB7NmzYTAYMGTIEPz5z3+Obq+pqQHHcXjjjTdw+eWXw2Aw4OKLL8aRI0ewc+dOTJo0CWazGbNnz47rifHatWsxevRo6HQ6FBQU4P77749uq6urw/XXXw+z2Qyr1YobbrgBp06dim7vqK388Y9/RGlpKWw2G374wx/C5TqTOkUURTz77LMoLy+HTqdDSUkJnn766ej2xx57DMOGDYPRaMSQIUPw5JNPRp98rV+/HitWrMBXX30V/UzXr18f873FjSWA4zh26tSpLu/X1NQwo9EY83lmzZrF1q1bx/bv38/27t3LrrnmGlZSUsLcbndMxzscDgaAORyOmK+pZD6fjx08eJD5fL4u2wIhocefUFiIed9gDPvGq729nU2dOpXdcccdrLGxkTU2NjK/38+WLVvGdu7cyY4dO8b+9Kc/MaPRyF5//fXocQsWLGBms5nNnz+f7d+/n/1t4xssOzuLLVmyJKZy3HPPPaykpIT94x//YF9//TW77rrrmMViYffeeRtztZxgjDE2e/ZsNm7cOPbZZ5+xXbt2sWnTpjGDwcBefPHF6HkAsKysLPbb3/6WHT58mP3nf/4nU6lU7ODBg4wxxqqrqxkANmLECPbee++xgwcPsksuuYRNnDiRzZgxg33yySds9+7drLy8nN19990x/Zv9+te/Znq9nq1atYodPnyYffHFF9EyCYLAKisr2WWXXcZ27drFPv/8czZx4kQ2ffr06PHLly9nZrOZzZs3j+3bt49t27aN5efns//4j/+I7vPoo4+yzMxMtn79enb06FH2z3/+k/32t7+Nbv/Zz37Gtm/fzqqrq9n//d//sby8PPbMM88wxhjzer3soYceYqNHj45+pl6vt9t76e17G+vvaFyBaMmSJWzJkiWM53l21113RV8vWbKE/fu//zubMmUKmzZtWjyn7KSpqYkBYFu3bo1p/4EUiF54/3CPP2/tPt5p3//ZcqTHfd/YWddp3zUfH+2yTyKmT5/OHnjggV73ue+++9j3vve96OsFCxYwu93OPB6P9Ia3la36xdPMZDazQDDU67lcLhfTarXsjTfeiL53+vRpZjAYpEDUXM8OHz7MALAvvvgius+hQ4cYgC6B6NwAMmXKFHbPPfcwxs4Eot/97nfR7a+99hoDwLZs2RJ9b+XKlWz48OG9lrtDYWEhe+KJJ7rd9v777zOVSsXq6s58VgcOHOh0L8uXL2dGo5E5nc7oPo888gibMmUKY4wxp9PJdDpdp8BzPr/4xS/YxIkTo6+XL1/Oxo0bd97jkhGI4uoj2rNnT0ctCvv27YNWe2bkq1arxbhx4/Dwww8nXDtzOBwAALvd3u32QCCAQOBM9j2n05nwtUjqvfTSS1i7di3q6urg8/kQDAa7dH6OGzcuOiCOMRFTJ0+Cx+3GiePHUVZW2uO5q6qqEAwGMWXKlOh7drsdw4cPR+RkOHjwANRqNSZOnBjdZ8SIEcjIyOhyvqlTp3Z5vXfv3k7vjR07Nvr/eXl5AIAxY8Z0eq9jukNvmpqa0NDQgKuuuqrb7YcOHUJxcTGKi4uj740aNQoZGRk4dOgQLr74YgDS0z+LxRLdp6CgIHr9Q4cOIRAI9HgNAHj99dfxy1/+ElVVVXC73QiHw7BarectfyrEFYg++ugjAMCtt96K1atXJ7XQoihi8eLFuPTSS3HRRRd1u8/KlSuxYsWKpF0zndx3ZXmP287tT7nziqE97ntuH/Btl8bepxePDRs24OGHH8bzzz+PqVOnwmKx4Be/+AV27NjRc9mYCJ1K6rbk48zO2B2W5A5rjUYT/f+Op3rnvieK5x+/1DEtIpnlOff657vGZ599hptuugkrVqzArFmzYLPZsGHDhnP62fpPQp3V69atS3rkvO+++7B//35s2LChx32WLl0Kh8MR/amvr09qGZRMq+Z7/FGr+Jj31cSwb0Ll02ohCEL09fbt2zFt2jTce++9GD9+PMrLy1FVVdXluK+++go+XyR3EBPx+c7dMJvNnWoD3Rk6dCg0Gk2nwNbW1oYjR44AkVFIwyuGIRwO48svv4zuc/jw4W4zG3z++eddXo8cOfJ8t50Qi8WC0tJSbNmypdvtI0eORH19fafv98GDB9He3o5Ro0bFdI2KigoYDIYer/Hpp59i8ODBeOKJJzBp0iRUVFSgtra20z7nfqapFHONaN68eVi/fj2sVivmzZvX674bN26MqxD3338//va3v2Hbtm0oKirqcT+dTgedThfXuUn/KC0txY4dO1BTUwOz2YyKigq8/PLL2Lx5M8rKyvDHP/4RO3fu7PJUNRgMYtGiRfjPJ55A9aE9WL7yOdx/371dhoacy2w2Y9GiRXjkkUeQlZWF3NxcPPHEE9JxkcpUxbAKXH311bjrrruwZs0aqNVqLF68uNvawptvvolJkybhsssuwyuvvIIvvvgipePgfvrTn+Luu+9Gbm4uZs+eDZfLhe3bt+MnP/kJZs6ciTFjxuCmm27CqlWrEA6Hce+992L69OmYNGlSTOfX6/V47LHH8Oijj0Kr1eLSSy9Fc3MzDhw4gEWLFqGiogJ1dXXYsGEDLr74Yrzzzjt46623Op2jtLQU1dXV2Lt3L4qKimCxWFL2+xfznz+bzRatjtpstl5/YsUYw/3334+33noLH374YVyP/omyPPzww1CpVBg1ahRycnIwa9YszJs3D/Pnz8eUKVNw+vRp3HvvvV2Ou+qqq1BRUYErpk/HD2+9G7NnzcQDjz4R0zV/8Ytf4PLLL8ecOXMwc+ZMXHbZZZH+IOl7ypiIdevWobCwENOnT8e8efNw5513Ijc3t8u5VqxYgQ0bNmDs2LF4+eWX8dprr8Vc+0jEggULsGrVKvz617/G6NGjcd111+Gbb74BIDWx3n77bWRmZuKKK67AzJkzMWTIELz++utxXePJJ5/EQw89hGXLlmHkyJGYP39+tA/pu9/9LpYsWYL7778flZWV+PTTT/Hkk092Ov573/serr76alx55ZXIycnBa6+9lpyb7wbHkt2QjsO9996LV199FW+//faZTkZIgS6WdrTT6YTNZoPD4ZCtky2Z/H4/qqurUVZWNvBSWAghhFzN8IcZwoYcZJriTwHSwdveBE4IQm3KhEZ//pxGHMfhrbfe6jzdhMSst+9trL+jCXUI+Hw+eL3e6Ova2lqsWrUK77//flznWbNmDRwOB2bMmIGCgoLoT7yRn1wAmBiZec/FNJixN0adBgaNCvGmvCbySWiKx/XXX4958+bh7rvvRnt7OyZPngytVouWlha88MILuOeee2I6j4yVMaI0kdnyLDLPrK6urtem0cGDB1FSUtLD1o4Z+PJ9v8xmc4/b3n33XVx++eX9WBrlSzhD44svvggA+POf/4z8/Hzs2bMHf/nLX7Bs2bKYAxEhUdF5ZjxUHFBYWNhlHM/ZCgsLez5XxxiFGFOBpOIPYm9lHzRoUNKvl+4SCkRerzc6kOr999/HvHnzwPM8Lrnkki6PAAmJSWTmfUeNSK1Wo7y857FTvfGFGcRgGDxCMMjU1ZZo2QeqhFrR5eXl2LRpE+rr67F582Z85zvfASCNGL0QOo2JDMQzNaI+j2XkOOlc1PRPGwkFomXLluHhhx9GaWkppkyZEh0e//7772P8+PFJLSAZIJgAnuOgUqug6mMk6kgXS1ka00dCTbPvf//7uOyyy9DY2Ihx48ZF37/qqqvwb//2b0krHBlAmAidmofOoAPUfVywj/JWp52EV3rNz89Hfn5+p/doOSGSMBaZSnCeEdWx4Hj5n5qR+CQUiDweD37+859jy5YtaGpq6jLR79ixY0kpHBkgGIvWXrgEV+84GzXN0k9Cgej222/H1q1bcfPNN6OgoGDgLn9DkiMymNEbFOD3hJBt6Vsw6lQjYqxrygGiOAkFonfffRfvvPMOLr300mSXh6SpGTNmoLKyMrHVMc4aQ8QSXr/jDJ7ncc3cGzB2zGj86tf/S4EoDSTUIM/MzOwxeRkhcYsEokAwtlzV56PiefC8tJpHR+5qomwJBaKf/exnWLZsWaf5ZmTgSjR5fse6Zk//90qUjJyAiVMvB89x+PTTT1FZWQm9Xo9JkyZh06ZN4Diu02jl/fv3Y/bs2TCbzcjLy8PNN98cXdBh4cKF2PrJZ1i95nfgVBpwHIeampp+/Bch8Uqoafb888+jqqoKeXl5KC0t7ZIpjhZWTCLGACHxNcUTptLE3KRZvXo1jhw5gosuughPPfUUAKnWXFRUhDfffBNZWVn49NNPceedd6KgoAA33HBD9NgtW7bAajLi7395FT5o4HY7MWfOHFxzzTV49dVXUVtb22W9tPb2dnzrW9/C7bffjhdffBE+nw+PPfYYbrjhBnz44YdSef51ABeNGI6n/utpQK1DTk5O0v5pSPIlFIgoXUI/EkLAP2VI33n5Q4A6tlQcNpsNWq0WRqOx05COs9P6lpWV4bPPPsMbb7zRKRCZTCb8bs0vwYQAvEyPP7yyARzH4be//S30ej1GjRqFEydO4I477oge86tf/Qrjx4/Hf//3f0ffW7t2LYqLi3HkyBEMGzYMKrUGGp0OWTk50OiMffmXIP0goUC0fPnyZJeDXIBiSZ4/ZswYaDVq+MMBMI7D0W+OYOzYsZ3y2pw7Pu2rr77CRx991O0M96qqKgwbNgzR5Ggx5JAm8kt4QGN7ezv+/Oc/o6qqCo888gjsdjt2796NvLw8ml2cTCqNVDuR47p9EGvyfJPJFJneAahU6phag263G3PmzMEzzzzTZVtBQUGn1zS6Oj0kFIi+/vprzJw5EzabDTU1Nbjjjjtgt9uxceNG1NXV4eWXX052OQcujou5iSSn3pLnd+gueT4AQBShVfHQGnUYPXIkNrz6KgKBQDQ/8s6dOzvtPmHCBPzlL39BaWkp1Oruv8JSeUSAakRpIaGnZg8++CAWLlyIb775plMV+pprrsG2bduSVjiSPs5Ont/S0oKKigrs2rULmzdvxpEjR/Dkk092CShRHdM7OB4/+tGPIIoi7rzzThw6dAibN2/Gc889J22OVJfuu+8+tLa24sYbb8TOnTtRVVWFzZs349Zbb40Gw5KSEuzavSdanliW+SHySSgQ7dy5E3fddVeX9wcNGoSTJ0/2uVAk/SSaPB+QRj8zAOBVsFqt+Otf/4q9e/eisrISTzzxBJYtWwYA0T96hYWF2L59OwRBwHe+8x2MGTMGixcvRkZGRnT1jwce+HfwvArjJ09DTk4O6urq+ucfgiQmxtVoO8nJyWG7d+9mjDFmNptZVVUVY0xaKreoqCiRUyZkIC05fcEKh5joaGCu5nrW5PQzQRS77PKnP/2JaTSaHtde747H7WSupjrmbW9KZmlJN5Kx5HRCNaLvfve7eOqppxAKSeNbOnIMP/bYY/je976XxDBJLnhMAAODCB4ikyZ4vPzyy/jkk09QXV2NTZs2RccIxbNCKh8ZXc2DZuCng4QC0fPPPw+3242cnBz4fD5Mnz4d5eXlsFgsePrpp5NdRnIhY4I0z4xTgeekP2onT57Ej3/8Y4wcORJLlizBD37wA/zv//5vXKfVa9QwalTQqWieWTpI6KmZzWbDBx98gO3bt+Orr76C2+3GhAkTMHPmzGSXj1zoOlLEcjz4SGf0o48+ikcffbRv5+1IBUJzzdJC3IFIFEWsX78eGzduRE1NDTiOQ1lZGfLz88EYo5QgJD5nNc34ZH53oit5UCqQdBBX04wxhu9+97u4/fbbceLECYwZMwajR49GbW0tFi5cSGli+5sopP84GVE4UyNKxtT7iJDIwRsU4A2GAeonUry4akTr16/Htm3bsGXLFlx55ZWdtn344YeYO3cuXn75Zdxyyy1JLeRAE9OYl6AHCDgBcIDBnhaDHrvVkYuIT8LqHWfhOC7a+S3ViJJ3btJZMsZoxRWIXnvtNfzHf/xHlyAEAN/61rfw+OOP45VXXqFAlCCtVgue59HQ0ICcnBxotdrum7qiAPhaz+RkDrYAxsz0bH74/QiGBYhqAUIwCD+E8x8TA0FkCIRFgIngfV5wfZyyQrpijCEYDKK5uRk8z0OrTfyPYVyB6Ouvv8azzz7b4/bZs2fjl7/8ZcKFGeh4nkdZWRkaGxvR0NDQ845hPxAOAJwqkpeZAZqWPs8P63eMRWp1AHTuszqYk3FqhoDHAQ4M2lY3OFXC0yrJeRiNRpSUlEQHkyYirk+ntbUVeXl5PW7Py8tDW1tbwoUhUq2opKQE4XC409ytTva9CXhbgbIZgKcZaDoAZA8DyrrWVBXN7wS++kAKqCMWJb1G9/n//Ra6YCuKL/8+THllST03kahUKqjV6j4/pIorEAmC0OMkw45ChcPhPhWISP0bGo2mS8I5AICvHXAfhz/M8HGrFQZ/ENNCbmgcVYDu6vRqngVaANELpreCi2OwYqxEJgIBF7iwv9OcSKI8cQUixhgWLlwYnRV9rkAgkJRCkV44joMxht3tehxCCLxogr3Vh4vyOXC+NsCYRrnEAy6ERREfVflQEzqGhZeWQqNKXvOM00jBJxTwJe2cJDXiCkQLFiw47z7UUZ1izhNo9QZxkkk5n0RejYAhD2HBCU17bXoFIr8TIYHBrzIjKIhJDUIAYDKaofeqwIv0B1Lp4gpE69atS1U5SKwcx9HkCsBtzMWUMjsmDM6Evu4ioO5zwJVmmQ/8DoQEEQG1GSZt3xdWPNfkigJAVwfoaRyR0iX3TxBJLSGEsKsJTl8ILm0eRhRYodeoAEskT3QaBqJgWERAbYFRl4KnWupIv1PIn/xzk6SiQJROvKfh9gcR5HQwWjKQaYx0ZlvyERJEBB2npDFG6cLfLtWIVBaYtKkIRJG+zDAFIqWjwRXpxNOMDIMWUy4qw4gRg6KPTLcfD4Md96LUyqPI03ymhqRkjEk1IkFEQGuGSZf8ptk3rWG469tgDLRg+Jikn54kEdWI0omnGQCgteUjx3LmyaVRp4ZHbYc7EI7uo3ghHyCEIk0zMyz65A/GDPE6+EMign5aCFTpKBClE4+0kilM2Z3eLswwwKfJhMsfBvOclqFgCfA7AABaow2FdgvspuTPldPqpMf3YpAe3ysdNc3SiLu1EXWNToRzNBh71opN2WYdQrpMhN0MnrZT6LralwJFAtHgQQUYPKE4JZfQRhZWFKmzWvGoRpQuhBB8zjY4fCF84+k8SljFczBk5gIA3G1p8uQsEoigt6XsElqDCQDAwoH0T5dygaNAlC587fCFBIR5HTJt1i6bLXapgzrgbAGENJhm43eAMQZBm7pApNdLj+/DIqMnZwpHgShd+NvhDYYRUFuQZeo6xSYzww6B10iJwHxpMPHY74ArEMbr+x14fWdqlvrR67QQOA0EkUGg5pmiUSBKF752eIMC/GorssxdO3YLMoyw5xQiy6QFvC0yFDBO/nYEwyL8vCVl6YW1Kh4anQEmnQohvycl1yDJQZ3VaSLkOY1AWETA2H2NKMeiQ87QMuCkW0oRomSMAb5IINJbUahPzdeQ5zlMGVYIuJsBLpSSa5DkoBpRmnC3S7Uc3pAJQ0/zsgyZ0n+V3jTzOwAxDL8ABFSpGUMUpY507Idp4quSUY0oTYQ9bdLTMVtWj/uEdDYEA2HA0QxTP5Ytbj6pxubiTADHw2boh0AUokGNSkaBKB0whhy1H9mlmQhWDutxt73NHMInHMj2q1DRj8WLW6TG1sYsAJDSQPR1cwjC8TZk2U6jZND59yfykLVptm3bNsyZMweFhYXgOA6bNm2SszjKFXABYhgcp4LOlNnjbuYMacR10OMEwsH+Kl38vG1gjKGNSUMvbcbUBSI/k6Z5BLzulF2D9J2sgcjj8WDcuHF46aWX5CyG8vnbpf/qrUAvCcozbTaEeT38IUHZ/US+NgiMIS+vAIMyDbCkIgVIhEonNVLFAD01UzJZm2azZ8/G7Nmz5SxCWmC+Nuw/4UDIasPIoNBjZ3WmSQO/2gp1sAkBVwt0lp4XOpCVrxVqnse0MRVAZmqmd3RQ6aValxCkGpGS0VOzNOB3tcIdCKPer4NW3fNHplOrwBmlppurXaFjiURRWgAAOPOUL4XU+o4aEXVWK1ladVYHAoFOCfqdTqeMpek/Xqc0o543ZkJ1nuVQNeYsoB3wtDcju9c9ZeJvB5iIEHjwGguSn4WoM02kRsSCFIiULK1qRCtXroTNZov+FBentlqvFP5IINKZz58Y32CTwo/fpdB0IJHBlgfb1fjVR1U40OBI6eX0RunJnBj0nlkZlyhOWgWipUuXwuFwRH/q6+vlLlK/CEWCisHa8xiiDkUFhRhsN6JAq9AcPJHEbS2iFSJjsKZyMCMAg8kMnZqHTgUpGRtRpLRqmul0uh7XVLtghYMI+aWOVpPt/I2twoJCIMMAQMqAqLhlqD3NEESG05AyCHQ3by6Zcm0m5A7Jl0ZWh3yA1pjS65HEyBqI3G43jh49Gn1dXV2NvXv3wm63o6SkRMaSKYi/Hf6QiDCvQ47Fcv79NQYpaXw4IE2lMCmsp8jTDF9IgFedCaNWBWMqkuafS2uKBCIPgPPXKkn/k7VptmvXLowfPx7jx48HADz44IMYP348li1bJmexlMXXDp4DwlobMmIZ+Mdx8KisOO0JwOtoSn354iGKgLcVvqAAryYzJelhu6WJLCtEHdaKJWuNaMaMGWDUgdg7fztGF9owKqcCiPEXd08zoGpxY1DzKZQUjkpxAePgbwfEMDwCh4C++3QmqbC7MQiupQ3FhW3Izu2XS5I4pVVn9YAUGXPDGTJiztujj3Rqe9sUtqJHpKO6DRaA47pNZ5KSyzIN/GERfq+rX65H4pdWndUDUsf0DkNGzIcYbDkIAfC7FDaoMRKIMnMGYZjVgoIM/XkOSA6VThpLRMnRlIsCkcLVnjiB9pZ2aPJUGFUY2zHmjBw4AYQ9CkuQ5pb6rEpLBqO0pKDfLqvWmyEA0aePRHmoaaZkooigqxXeoACfqmvC/J5k2KU5ZoK3HWJIQbPw3aek/5r7dw6cJjLNQ6AakWJRIFKyoAuBUAgMHCy22OdlWSxWMJUOIgOcSuknCvkAXzs8wTDaeHu/PqTQGCITX2kGvmJRIFIyXzv8IQEBtQUZxtg7dnkVD94kTQdxtinkEb5LWm/tiEOF9TtP4UBD/80T1Jmk2iQLUGe1UlEfkYIF3K0ICQwBvTXu5GGDi4qgO+2CUa2QsTPuUxAZw0lRqtkVZhj67dIGkw06NQ89gtKabyr62isNfSIK5nVIT72Y3gadOr556gX5BYD/GBBWSIYCVyPcgTCcmiyYdCpkpjAr47mKcuwoKs0BxDAQdMf1BJL0D2qaKZgvMuteG8Os+y4MkWOUsrSQ6yQcvhA82mwUZRpTtpZZtzgOiDzCByVIUyQKRAqmCrRL87Ey4p8vJugz0OoNor7hhPyj14MewNcOpy8EtyYbRZn91yyL0kYCUYACkRJR00zB8jR+5BVlAKPK4z/YkIkjJ11gcMHm9cFqknHWueMEwqKIJtEKQaXHYHv/L3a0qzEIvqUNpYWnYadpHopDNSKlCvnP5M9JoE9DpTNBo5eCT/tpmZ+cOerR5g3Boc1Dtlmb0lU7euJmegTCIvye1CZiI4mhQKRU/nYwMCl/jjqxOVmaSN+Sq+1UMksWP8dx2E1aTBw7BlOHypOGQ22QHuEHPArpvCedUCBSqJD7NHbWtOGzBgEhQUzoHHprDgDA1SrjoEYhBLhPQcVxKC4dhvLcGHIqpYA2kjI27KNApEQUiBTK3d4CQWRwwAyNKrGPyZwpdYb45MxL5DwBiIL01Epvk60YWqN07bCPBjUqEQUihfI5pTFEmkQe3UfYcqRZsoK7KeFaVZ+11eBosxv/8tvhCQrylAGAziQFItFPNSIlokCkUAGXNP5HZ068T8WYkQeNioMh2I7TrsD5D0gB76mjaHEFsNNpQ1iQbxiBwZIBAAgH/VLaWKIo9PheoTpW7tDbEg9EnMGO8vwMaDgRBo0fQD+P3wl60dRQCwbAWlAhy9OyDhaTCc06I3SqkJTL20zP8JWEApESCWGEIp2qJltO4ufheWRk5Ut5gHwtgCn1K6ueLdhShSanH16NHWOHFvXrtc+VZdYha0Qp4DpFgUiBqGmmRH4HAqEwBE4Dm7WPHbymSCDz9P+Ts/qj+yGIDGJGKUqzFLCMT0dnOfUTKQ4FIgUSPS0w6dRQmeywm/uW15mZsnHS6cfuQ0fgD/VfZ3EgGERzzQEAQMnwyv6dW9aTaCBql7UYpCtqmikQ72vF8DwLkDcM0PRtdXjOlItGhw9taMBJhx+l2f0zveJfh/aDhfxQ682oKB/RL9c8n73NgFjXhmxVI0oSmDVDUodqRErkjaxbb0zCKGRzLiw6DQyhNjS29l+TpJw7jlyLDgXlleATHAeVbAGVCYGwiIC7Te6ikHMo4xtCOgm7W6TpHcYkrNKqs8BksYEDQ2vT8b6fLxZCGCZnFYbmmFFUMa5/rhkDvUUakyV42+UtCOmCApHSMIYj1TXYWdOGo54krPvFcbDkSst3e5rrEQyndmCjJxAGazksTdrVW4GMwSm9XjxMVikQhf0eIKygRQUIBSLFCboR8HshiIC2D4MZz2bKGgSdmofB34T6ttSljvWHBLz2RR32fLEVYVEE8scCvHK+YlaLBWFeh0BYAHzUPFMS5XxLCAAg7GpBICTCr7bAbk3OI2/OWohMowbmYDNqWlKzkgVjDB/9qwkhVwu41hopK2L+mJRcK1FWgwY+TQaCAkPIrbDFJwc4CkQK42o7BQYgpLfDpO3bE7MoSwEyjFoYwg4pW2IK7Kptw79OujDI9TWG5pqhzhmmuNzQOjUPQZcBAPAoZXUTAoAe3yuOq+UEAEBrzU3e2ButEdbsQozXnoKuIPnzrA40OLD9aAu0YTcuNpyAVa8FSi5J+nX6iuM4mO15sAarzjyZJIpANSKF8bU2AAAMWYOSel6VvUxaCaStNmnnZIxhd10b3j9wCowB0zUHUWDRAhklgE3eKR09uXzsCIwutCEDlA5ESSgQKYkoIuSUsinaspMbiJAZeXrVXovT7gDcgXCfT+kLCfiiWsoScEm2D6NVddKGoVf2+dwpY+xY3eQ0IPeiAiSKApGS+FqRqedhMxuRnZOf3HPbigGOx4mGE3jzn1/h86rEmiYufyia28ioVePq0fm4ojwDlwQ/BwcOyL8IsBYms+TJZcgEOB5iOAgEaM6ZUlAfkZK4T2FQhgGDbIMAiz6559bogcxSWHyHYfdUY39DBobnW1BsP/+TOac/hGPNHhxrdqO+1YcrR+RgbFEGAKA0y4jSpi3S+mk6M1A+M7nlTjJPiOGrRga1rxWTLjoFXsaskeQMCkRKElkfHuYk14Y65AyHtfUYxoQacYKNx9++bsT1lYVdln8WRYYT7T7UtXpRc9qDJmfnDu5THa8ZA459BJw6AHA8MHIOoJFhzbI4GDQqtPF22FkLvK0nYM4dJneRCCgQKYqruQ56QYTGkqJAlD0M+OZ9DDN5cVjjRHXAijd21aM404iSLCMuLo1MgWAMm/acQFiU+lA4Dii0GTAkx4QhOWbYTVppDflv3gcav5LOPfxqILM0NeVOIp7noMkoANxH4Gw5AbPcBSIAKBAphxBG9bFv4PEHkTcsA0NScQ2tEcgZAdWpA7jWfgJbUIRDjU7UtXrhCYajgUij4lGeawbHAcV2I8qyTTBqz/qqOI4Dh98FPC1SlKr4DlCgnDll52POHgQcB3ytJ+QuComgQKQQwfYT8PqDCPGG5HdUn61oEnDqADQth3D1pGmYUlaKulZvtPbTYfaYgq7Huk4CNZ8ALd9Ir7UmYMS1QNbQ1JU3BTJyi+AA4He1SQM8tf2/8izpjAKRQrQ1VoMBCJsLYNGnMLeztRDIGQY0HwEO/x2ZlTchszij5/2FENByBDjxJeCI1CA6pm8MuVKqZaWZ3EwbGjWZUAXbIbbVg89TRr6kgYwCkUK0NxwFABhzS1OfzXDoVdLARmcDcHATMOI66alah3AQcNQDTQelINQxU53jgdwRwOBLAVMSUpTIxG7UwmccBKOjDa5Tx2CjQCQ7CkRKIITga64GAGSVjEz99QwZwKjrgf0bpWbW5y9J44x4tZRY3tMsLYrYQW+V+oAKKqVH9GmO5znYB1Ugj30DtbNe7uIQUCBSBPepo/D5AwiqzSgqLO6fi2YNBSpvPNPpfLqq83a9FciqAHJHStM1lJBzOommTawE/NuAUCsQcF8QATadUSBSgKbqg9L/2IfCoOvHj8RWBFx8u9RE8zRFloa2SEvt6DMuuODTidYEWPKlDvjT3wCF4+Uu0YBGgUhuooASoRaWHDM8Iyr7//ocB9gGST8DDMsZDldzPVC/H1YKRLKiuWZyO10FtRBATnYWSoeOlrs0A8pOdy4ONDhxsvaw1DwjsqFAJDN2/Avpf/IuUlRa1YGgpKgIbm0u2tx++Ot2yV2cAY2++TLyNNfi6/370OgMQCycIHdxBpw8qw6hggkQGXDy0KdAOPlJ40hsKBDJRRRR/dnb8AYFHOVLwemtcpdowOE4DuWjJ8GvtqLpdBu8R/8pd5EGLEUEopdeegmlpaXQ6/WYMmUKvvjiC7mLlHK1X74HV3MdRF6DikuuU8aSzANQRb4NvpIrIIgMNXs+hHi6Wu4iDUiyB6LXX38dDz74IJYvX47du3dj3LhxmDVrFpqaLszk5kwI49iOv6Hx6w8BAKaLrkN+TvqOUk53HMdh8qRL0GYZBoc3iNqtL0vTX0i/4hiTN1/mlClTcPHFF+NXv/oVAEAURRQXF+MnP/kJHn/88V6PdTqdsNlscDgcsFoV3LQJBwBfG1pPfINjX29HyCUtZcMPuQIXT58DnqfakNyONrbh+D//hEsyHNBrVEBmKZzWCmjtxdCZM8GptXGdjzEGxgAW+X/1Wctu+0MCRMbAcxxUPAcVx12w34FYf0dlHUcUDAbx5ZdfYunSpdH3eJ7HzJkz8dlnn3XZPxAIIBA406HodMae6vNEuw8fHpKW6unQEYIZY5hcloVRhdI/1EmHH3/f19hpX3XIjfKT7wAABtsNKMowAGBw+0P4+ngbOCaCMYCTvnrSD2MosmlRbJX+mdX+EEIuJ0SVDpnjrsHIymnUJFOI8oJMlP7bXVDXfQLU7wTaalC9by+8QQEcAKj1YLwKjFNB5FQw6jSYUJIZPX5HdSt8wTAYODCwTumwjVoVpg6N1HpHfhd/3u9Ds6tzxzjHASqOg0mnxm2XlUXff29/I045A92OLdWoeNw4uST6+h8HT6HR4ev2/jiOw48vObPq7seHm1Df1v2+APCjySVQRYLj9qMtOHbOenjfn1AEQ7KWu4LMgailpQWCICAvL6/T+3l5efjXv/7VZf+VK1dixYoVCV0rFBbR4u55meFA+MzcqrAowuELddquDYfAe6WaDKc3AlpDx4Hg/A7p/e5OHAYANaDRw5xZBlt2LkpHT4HRSKknlEat0QJDvwUUTgBr/ArOpi/BhZqhYiEg7AcgfcYqACpRBXjOfGd0gXaIIaHb8+oZD7gjkUkMg+8mqjAGhBmDcE46FocvhFZP999brbpzz4rTH+rxO37uNV3+MFpcPT8llBpKXGTfUJd9GZLbkJK1adbQ0IBBgwbh008/xdSpU6PvP/roo9i6dSt27NjRaf/uakTFxcUxNc18QQHNrq5/WTpeZxi1MEemVwTCQvTD5zrCixiG2nUcjONg0mpg1KkBcAiJDE6/AICTTsZzkaOk13qtBkZzBhBn1Z4oQzgswB/wIeh1gYlhcKIAiGGoOMBmOJOuxR0IoeNXl+c4cBzAMekrwXEctKrI98hSAKbWAQAEkUFgDKIoZcUUI825s8/b5PIjEBK7LRvHAUWZxpj2BdApP3mTyw9/sLd9DdHaeos7AG+gc5AdlGmI1ph6kxZNs+zsbKhUKpw6darT+6dOnUJ+ftfkYDqdDjqdLqFrGbQqlGTFljtHp1ahwNZN7uXMrukiNACSs0I9USK1WgWz2gyYep8UG8+U2Y5fX7WKO+8vYG4ciyikat9ssy6+G0yArE/NtFotJk6ciC1btkTfE0URW7Zs6VRDIoRc2GSf9Prggw9iwYIFmDRpEiZPnoxVq1bB4/Hg1ltvlbtohJB+Insgmj9/Ppqbm7Fs2TKcPHkSlZWVeO+997p0YBNCLlyyjyPqi7QZR0TIABXr76jsI6sJIYQCESFEdhSICCGyk72zui86urfimepBCOk/Hb+b5+uKTutA5HK5AADFxf208gUhJCEulws2m63H7Wn91EwURTQ0NMBisZx38mjHdJD6+voL5gnbhXZPF9r9ABfePcV7P4wxuFwuFBYWgu8lFXJa14h4nkdRUVFcx1it1gviC3G2C+2eLrT7AS68e4rnfnqrCXWgzmpCiOwoEBFCZDdgApFOp8Py5csTnr2vRBfaPV1o9wNcePeUqvtJ685qQsiFYcDUiAghykWBiBAiOwpEhBDZUSAihMjugg5Era2tuOmmm2C1WpGRkYFFixbB7Xb3esyMGTPAcVynn7vvvrufStxVvKvgvvnmmxgxYgT0ej3GjBmDv//97/1U0tjEcz/r16/v8lno9bHnWk61bdu2Yc6cOSgsLATHcdi0adN5j/n4448xYcIE6HQ6lJeXY/369SkvZzzivaePP/64y2fEcRxOnjwZ13Uv6EB000034cCBA/jggw/wt7/9Ddu2bcOdd9553uPuuOMONDY2Rn+effbZfihtV/Gugvvpp5/ixhtvxKJFi7Bnzx7MnTsXc+fOxf79+/u55N1LZFVfq9Xa6bOora3txxL3zuPxYNy4cXjppZdi2r+6uhrXXnstrrzySuzduxeLFy/G7bffjs2bN6e4pLGL9546HD58uNPnlJubG9+F2QXq4MGDDADbuXNn9L13332XcRzHTpw40eNx06dPZw888EA/lPD8Jk+ezO67777oa0EQWGFhIVu5cmW3+99www3s2muv7fTelClT2F133ZXScsYq3vtZt24ds9ls/VS6vgHA3nrrrV73efTRR9no0aM7vTd//nw2a9asFJYscbHc00cffcQAsLa2tj5d64KtEX322WfIyMjApEmTou/NnDkTPM93WS/tXK+88gqys7Nx0UUXYenSpfB6vakubhcdq+DOnDkz+l5vq+AC0j2fvT8AzJo1q8f9+1Mi9wMAbrcbgwcPRnFxMa6//nocOHCgP4qbEkr+fPqqsrISBQUF+Pa3v43t27fHfXxaT3rtzcmTJ7tUD9VqNex2e6/t1x/96EcYPHgwCgsL8fXXX+Oxxx7D4cOHsXHjxlQXuZN4V8EFpHvubv942+upkMj9DB8+HGvXrsXYsWPhcDjw3HPPYdq0aThw4EDck52VoKfPx+l0wufzwWDoZi09hSsoKMD/+3//D5MmTUIgEMDvfvc7zJgxAzt27MCECRNiPk/aBaLHH38czzzzTK/7HDp0KOHzn92HNGbMGBQUFOCqq65CVVUVhg4dmvB5SfymTp3aaX27adOmYeTIkfjNb36Dn/3sZzKWjHQYPnw4hg8fHn09bdo0VFVV4cUXX8Qf//jHmM+TdoHooYcewsKFC3vdZ8iQIcjPz+/SCRoOh9Ha2trtKrI9mTJlCgDg6NGj/RqI4l0FFwDy8/Pj2r8/JXI/59JoNBg/fjyOHj2aiiKmXE+fj9VqTcvaUE8mT56MTz75JK5j0q6PKCcnByNGjOj1R6vVYurUqWhvb8eXX34ZPfbDDz+EKIrR4BKLvXv3ApCqoP0pkVVwp06d2ml/APjggw8UsWpuMlb1FQQB+/bt6/fPIlmU/Pkk0969e+P/jPrU1a1wV199NRs/fjzbsWMH++STT1hFRQW78cYbo9uPHz/Ohg8fznbs2MEYY+zo0aPsqaeeYrt27WLV1dXs7bffZkOGDGFXXHGFLOXfsGED0+l0bP369ezgwYPszjvvZBkZGezkyZOMMcZuvvlm9vjjj0f33759O1Or1ey5555jhw4dYsuXL2cajYbt27dPlvKfK977WbFiBdu8eTOrqqpiX375JfvhD3/I9Ho9O3DggFy30InL5WJ79uxhe/bsYQDYCy+8wPbs2cNqa2sZY4w9/vjj7Oabb47uf+zYMWY0GtkjjzzCDh06xF566SWmUqnYe++9J9ctdBHvPb344ots06ZN7JtvvmH79u1jDzzwAON5nv3jH/+I67oXdCA6ffo0u/HGG5nZbGZWq5XdeuutzOVyRbdXV1czAOyjjz5ijDFWV1fHrrjiCma325lOp2Pl5eXskUceYQ6HQ6Y7YOx//ud/WElJCdNqtWzy5Mns888/j26bPn06W7BgQaf933jjDTZs2DCm1WrZ6NGj2TvvvNPPJe5dPPezePHi6L55eXnsmmuuYbt375ah1N3reHR97k/HPSxYsIBNnz69yzGVlZVMq9WyIUOGsHXr1vV7uXsT7z0988wzbOjQoUyv1zO73c5mzJjBPvzww7ivS2lACCGyS7s+IkLIhYcCESFEdhSICCGyo0BECJEdBSJCiOwoEBFCZEeBiBAiOwpEJK2sX78eGRkZ0dc//elPUVlZGX29cOFCzJ07t9/LRfqGAhHp1sKFC3tMk3vfffeB47hOk49TEQBKS0uxatWqTu/Nnz8fR44c6fGY1atXd0q/OmPGDCxevDip5SLJR4GI9Ki4uBgbNmyAz+eLvuf3+/Hqq6+ipKREljIZDIZe05DabLZONSaSHigQkR5NmDABxcXFnZLCbdy4ESUlJRg/fnyfzt1dTWXu3LnRWtaMGTNQW1uLJUuWRBOyA12bZuc6u2a2cOFCbN26FatXr46eo7q6GuXl5Xjuuec6Hbd3715wHJe2KUbSHQUi0qvbbrsN69ati75eu3Ytbr311pRfd+PGjSgqKsJTTz0VTcger9WrV2Pq1KmdFkMoKSnpck8AsG7dOlxxxRUoLy9P1i2QOFAgIr368Y9/jE8++QS1tbWora3F9u3b8eMf/zjl17Xb7VCpVLBYLMjPz08ouZvNZoNWq4XRaIyeQ6VSYeHChTh8+HB0KaNQKIRXX30Vt912W7Jvg8Qo7TI0kv6Vk5ODa6+9FuvXrwdjDNdeey2ys7PlLlafFBYW4tprr8XatWsxefJk/PWvf0UgEMAPfvADuYs2YFGNiJzXbbfdhvXr1+MPf/hD0moNPM/j3Aw0oVAoKeeOxe233x7tiF+3bh3mz58Po9HYb9cnnVEgIud19dVXIxgMIhQKYdasWUk5Z05OTqd+H0EQuiwEqdVqIQhCn67T0zmuueYamEwmrFmzBu+99x41y2RGTTNyXiqVKroyikql6nE/h8MRzfHdISsrC8XFxV32/da3voUHH3wQ77zzDoYOHYoXXngB7e3tnfYpLS3Ftm3b8MMf/hA6nS6hJmFpaSl27NiBmpoamM1m2O128Dwf7StaunQpKioqLri80emGakQkJlarFVartdd9Pv74Y4wfP77Tz4oVK7rd97bbbsOCBQtwyy23YPr06RgyZAiuvPLKTvs89dRTqKmpwdChQ5GTk5NQuR9++GGoVCqMGjUKOTk5qKuri25btGgRgsFgvzwFJL2jVLFkwPrnP/+Jq666CvX19V0WPiT9iwIRGXACgQCam5uxYMEC5Ofn45VXXpG7SAMeNc3IgPPaa69h8ODBaG9vx7PPPit3cQioRkQIUQCqERFCZEeBiBAiOwpEhBDZUSAihMiOAhEhRHYUiAghsqNARAiRHQUiQojsKBARQmT3/wG47942+O1Q6gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:06:32.097598Z", + "iopub.status.busy": "2024-03-23T05:06:32.097271Z", + "iopub.status.idle": "2024-03-23T05:06:32.314064Z", + "shell.execute_reply": "2024-03-23T05:06:32.313082Z" + }, + "papermill": { + "duration": 0.238252, + "end_time": "2024-03-23T05:06:32.316107", + "exception": false, + "start_time": "2024-03-23T05:06:32.077855", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAqiUlEQVR4nO3de1xUdf4/8BfXARxAER3RwPGSoKmgmIjp19xFycrVr7vlqgWSayvqfk3yEqWQ6xbdwEtL8lgTycqk+pb1LVOJtEzxBuKKclkRxFVA0XS46DDA5/eHP047zgADAsOZeT0fDx84n/mcM+/DZ3jNOZ85c8ZGCCFARCQztuYugIioLRheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSzZm7uAztbQ0IArV67A1dUVNjY25i6HiO4hhEBlZSX69u0LW9um96+sLryuXLkCb29vc5dBRC24dOkSHnjggSbvt7rwcnV1BXD3F+Pm5mbmajqOTqfD/v37MXXqVDg4OJi7HGoH1jKmGo0G3t7e0t9qU6wuvBoPFd3c3Cw+vFxcXODm5mbRT3RrYm1j2tK0DifsiUiWGF5EJEsMLyKSJYYXEcmSWcPrp59+wvTp09G3b1/Y2Nhg9+7dLS5z8OBBjB49GgqFAoMHD0ZKSkqH10lEXY9Zw6u6uhr+/v5ITEw0qX9RURGeeOIJTJ48GdnZ2XjhhRfwpz/9Cfv27evgSomoqzHrqRLTpk3DtGnTTO6flJSEAQMGID4+HgAwdOhQ/Pzzz9iwYQNCQ0M7qkxZqKmpQV5ennS76rYWR84UoofnSSidFXp9/fz84OLi0tklErUrWZ3nlZGRgZCQEL220NBQvPDCC00uo9VqodVqpdsajQbA3XNmdDpdh9RpDjk5OQgKCjJof8tI32PHjmHUqFEdXxS1q8bnqyU9b40xdftkFV5lZWVQqVR6bSqVChqNBrdv34azs7PBMnFxcVi3bp1B+/79+y1q70Or1Up7pABQXgPsOG+HsMH1UN2zmcXFxSgtLe3kCqm9pKWlmbuEDlVTU2NSP1mFV1tER0cjKipKut340YOpU6da9Bn2p0tuIHXrSTw1Zwz8fTzMXQ61A51Oh7S0NEyZMsWiz7BvPDpqiazCq0+fPigvL9drKy8vh5ubm9G9LgBQKBRQKBQG7Q4ODhb9BLC3t5d+WvJ2WiNLf+6aum2yOs8rODgY6enpem1paWkIDg42U0VEZC5mDa+qqipkZ2cjOzsbwN1TIbKzs1FSUgLg7iFfWFiY1H/RokW4cOECVq1ahby8PLz33nv49NNPsXz5cnOUT0RmZNbwOnnyJEaNGiW98xUVFYVRo0YhJiYGAFBaWioFGQAMGDAA3377LdLS0uDv74/4+Hi8//77Vn+aBJE1Muuc16OPPgohRJP3Gzt7/tFHH8WpU6c6sCoikgNZzXkRETVieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEcmS2cMrMTERarUaTk5OCAoKwvHjx5vtv3HjRvj6+sLZ2Rne3t5Yvnw57ty500nVElFXYdbwSk1NRVRUFGJjY5GVlQV/f3+Ehobi6tWrRvvv3LkTL730EmJjY5Gbm4tt27YhNTUVL7/8cidXTkTmZtbwSkhIwMKFCxEREYFhw4YhKSkJLi4uSE5ONtr/yJEjeOSRRzB37lyo1WpMnToVc+bMaXFvjYgsj9nCq7a2FpmZmQgJCfm1GFtbhISEICMjw+gy48ePR2ZmphRWFy5cwJ49e/D44493Ss1E1HXYm+uBKyoqUF9fD5VKpdeuUqmQl5dndJm5c+eioqICEyZMgBACdXV1WLRoUbOHjVqtFlqtVrqt0WgAADqdDjqdrh22pGuqq6uTflrydlqTxnG09PE0dfvMFl5tcfDgQbz++ut47733EBQUhPPnz2PZsmVYv3491q5da3SZuLg4rFu3zqB9//79cHFx6eiSzeZSFQDY4+jRo7icY+5qqD2lpaWZu4QOVVNTY1I/GyGE6OBajKqtrYWLiws+//xzzJw5U2oPDw/HzZs38dVXXxksM3HiRIwbNw5vv/221PbRRx/h+eefR1VVFWxtDY+Cje15eXt7o6KiAm5ubu27UV3I6ZIb+MPWk/h84Rj4+3iYuxxqBzqdDmlpaZgyZQocHBzMXU6H0Wg08PT0xK1bt5r9GzXbnpejoyMCAwORnp4uhVdDQwPS09OxdOlSo8vU1NQYBJSdnR0AoKkMVigUUCgUBu0ODg4W/QSwt7eXflrydlojS3/umrptZj1sjIqKQnh4OMaMGYOxY8di48aNqK6uRkREBAAgLCwM/fr1Q1xcHABg+vTpSEhIwKhRo6TDxrVr12L69OlSiBGRdTBreM2ePRvXrl1DTEwMysrKEBAQgL1790qT+CUlJXp7WmvWrIGNjQ3WrFmDy5cvo1evXpg+fTpee+01c20CEZmJ2ea8zEWj0cDd3b3F42m5y754HTO3HMXuyHEI6N/T3OVQO9DpdNKpQZZ82Gjq36jZPx5ERNQWDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLsvrqMyJrUVNTY/D9pVW3tThyphA9PE9C6fzrl8r4+flZ9Nf4NYXhRdQF5eXlITAw0Oh9b91zOzMzE6NHj+74oroYhhdRF+Tn54fMzEy9tvzSm4j67AwSnhoBX6/uen2tEcOLqAtycXEx2JuyvXgdikO3MXS4P79UBZywJyKZYngRkSwxvIhIlhheRCRLrQ6vCxcudEQdRESt0urwGjx4MCZPnoyPPvoId+7c6YiaiIha1OrwysrKwsiRIxEVFYU+ffrgz3/+M44fP94RtRERNanV4RUQEIBNmzbhypUrSE5ORmlpKSZMmIDhw4cjISEB165d64g6iYj0tHnC3t7eHrNmzcJnn32GN998E+fPn8eKFSvg7e2NsLAwlJaWtmedRER62hxeJ0+exOLFi+Hl5YWEhASsWLEChYWFSEtLw5UrVzBjxoz2rJOISE+rwyshIQEjRozA+PHjceXKFezYsQMXL17E3/72NwwYMAATJ05ESkoKsrKyTFpfYmIi1Go1nJycEBQU1OL82c2bN7FkyRJ4eXlBoVBgyJAh2LNnT2s3g4hkrtWfbdyyZQuee+45zJ8/H15eXkb79O7dG9u2bWtxXampqYiKikJSUhKCgoKwceNGhIaGIj8/H7179zboX1tbiylTpqB37974/PPP0a9fP1y8eBHdu3dv7WYQkcy1OrzS0tLg4+MDW1v9nTYhBC5dugQfHx84OjoiPDy8xXUlJCRg4cKFiIiIAAAkJSXh22+/RXJyMl566SWD/snJybhx4waOHDkCBwcHAIBarW7tJhCRBWh1eA0aNAilpaUGe0Y3btzAgAEDUF9fb9J6amtrkZmZiejoaKnN1tYWISEhyMjIMLrM119/jeDgYCxZsgRfffUVevXqhblz52L16tWws7MzuoxWq4VWq5VuazQaAIBOp4NOpzOp1q6q+Ho1qrXGf98FZbf0fjalm8IO6p7d2r02ar3mxhMwbUwtYTxN/btsdXgJIYy2V1VVwcnJyeT1VFRUoL6+HiqVSq9dpVIZXEGy0YULF/DDDz9g3rx52LNnD86fP4/FixdDp9MhNjbW6DJxcXFYt26dQfv+/ftlffXJq7eB17JbHr5VX+a22OeVgDr0dm6PqqitTB1PoOUxlft41tTUmNTP5PCKiooCANjY2CAmJkbvD7++vh7Hjh1DQEBA66pspYaGBvTu3Rv/+Mc/YGdnh8DAQFy+fBlvv/12k+EVHR0t1Q7c3fPy9vbG1KlT4ebm1qH1dqSzVzRA9lG884cRGNzL8JW2+o4Wew+dwGMTH0Y3J4WRNQDnr1Vjxedn8HDwBDzUV76/C0vQ0ngCLY+ppYxn49FRS0wOr1OnTgG4u+d15swZODo6Svc5OjrC398fK1asMLlAT09P2NnZoby8XK+9vLwcffr0MbqMl5cXHBwc9A4Rhw4dirKyMtTW1urV1EihUEChMBxoBwcHad5Mjuzt7w6dn5c7hvdzN7hfp9OhIg8YO7BXk9vZuA57e3tZ/y4sQUvjCbQ8ppYynqbWbnJ4HThwAAAQERGBTZs23fdei6OjIwIDA5Geno6ZM2cCuLtnlZ6ejqVLlxpd5pFHHsHOnTvR0NAgvWFQUFAALy8vo8FFRJar1ed5bd++vd0Ot6KiorB161Z88MEHyM3NRWRkJKqrq6V3H8PCwvQm9CMjI3Hjxg0sW7YMBQUF+Pbbb/H6669jyZIl7VIPEcmHSXtes2bNQkpKCtzc3DBr1qxm+37xxRcmP/js2bNx7do1xMTEoKysDAEBAdi7d680iV9SUqJ3Soa3tzf27duH5cuXY+TIkejXrx+WLVuG1atXm/yYRGQZTAovd3d32NjYSP9vT0uXLm3yMPHgwYMGbcHBwTh69Gi71kBE8mNSeG3fvt3o/8l8tPV3YOt0GUWafNg6KQ3ur6urw5W6K8i9kStN5N6rSFMFW6fL0NbfAdC+L0pEHY1ffSZTV6ovotuAd/FyC5dSe2/ve83e320AcKU6AIFQNduPOlZLL0ZAyy9I1vZiZFJ4jRo1SjpsbImpH8im+9O3W39UF/0Fm2YHYFBv43teh38+jEcmPNLknlfh1SosS81G38n9O7pcaoGpL0ZA8y9I1vRiZFJ4NZ7KQF2Hws4JDXf6YYCbL4b1NH6eV5F9EYZ6DG3yvJmGO7fQcOcaFHamfzKCOkZLL0ZAyy9I1vZiZFJ4NXX2OhG1j5ZejICWX5Cs7cWIX31GRLJk0p6Xh4cHCgoK4OnpiR49ejQ7/3Xjxo12K46IqCkmhdeGDRvg6uoq/d/UyXsioo5iUnj954UF58+f31G1EBGZrNVzXnZ2drh69apB+/Xr15u8ICARUXtrdXg1dTFCrVbLKzsQUacx+Qz7zZs3A7h7McL3338fSuWv56LU19fjp59+gp+fX/tXSERkhMnhtWHDBgB397ySkpL0DhEdHR2hVquRlJTU/hUSERlhcngVFRUBACZPnowvvvgCPXr06LCiiIha0uoPZjdeUZWIyJxaHV7PPfdcs/cnJye3uRgiIlO1Orx++eUXvds6nQ45OTm4efMmfvOb37RbYUREzWl1eH355ZcGbQ0NDYiMjMSgQYPapSgiopa0ywezbW1tERUVJb0jSUTU0drtqhKFhYWoq6trr9URETWr1YeN//nt08Dd875KS0vx7bff6n0GkoioI7U6vBq/ObuRra0tevXqhfj4+BbfiSQiai88z4uIZIlXUiUiWWJ4EZEsMbyISJYYXkQkS+0WXv/+97/x/PPPt9fqiIia1W7hdf36dWzbtq29VkdE1CweNhKRLDG8iEiWukR4JSYmQq1Ww8nJCUFBQTh+/LhJy+3atQs2NjaYOXNmxxZIRF2OyWfYz5o1q9n7b9682aYCUlNTERUVhaSkJAQFBWHjxo0IDQ1Ffn4+evfu3eRyxcXFWLFiBSZOnNimxyUieTN5z8vd3b3Zf/3790dYWFirC0hISMDChQsRERGBYcOGISkpCS4uLs1ekbW+vh7z5s3DunXrMHDgwFY/JhHJn8l7Xtu3b2/3B6+trUVmZiaio6OlNltbW4SEhCAjI6PJ5f7617+id+/eWLBgAQ4dOtTsY2i1Wmi1Wum2RqMBcPcKsDqd7j63wHwaLz9UV1dndDsa25rbxpbWQZ3HlLFoaUwtZTxNrb3VH8xuTxUVFaivr4dKpdJrV6lUyMvLM7rMzz//jG3btiE7O9ukx4iLi8O6desM2vfv3w8XF5dW19xVXKoCAHv8/PPPuKhsul9aWtp9r4M6XmvGoqkxtZTxrKmpMamfyeFl6uVuOvILOCorK/Hss89i69at8PT0NGmZ6OhovWuQaTQaeHt7Y+rUqXBzc+uoUjvc2SsavHPmKCZMmICH+hpuh06nQ1paGqZMmQIHB4c2rYM6jylj0dKYWsp4Nh4dtcTk8EpJSUH//v0xatQoCCHaXNh/8vT0hJ2dHcrLy/Xay8vL0adPH4P+hYWFKC4uxvTp06W2hoYGAIC9vT3y8/MNrqOvUCigUCgM1uXg4NDkH7Uc2NvbSz+b247mttPUdVDHa81YNDWmljKeptZucnhFRkbik08+QVFRESIiIvDMM8/Aw8OjzQUCd79pOzAwEOnp6dLpDg0NDUhPT8fSpUsN+vv5+eHMmTN6bWvWrEFlZSU2bdoEb2/v+6qHiOTD5HcbExMTUVpailWrVuH//u//4O3tjaeffhr79u27rz2xqKgobN26FR988AFyc3MRGRmJ6upqREREAADCwsKkCX0nJycMHz5c71/37t3h6uqK4cOHw9HRsc11EJG8tGrCXqFQYM6cOZgzZw4uXryIlJQULF68GHV1dTh79iyUytbPEs6ePRvXrl1DTEwMysrKEBAQgL1790qT+CUlJbC17RLn0hJRF9LmdxttbW1hY2MDIQTq6+vvq4ilS5caPUwEgIMHDza7bEpKyn09NhHJU6t2abRaLT755BNMmTIFQ4YMwZkzZ/D3v/8dJSUlbdrrIiJqK5P3vBYvXoxdu3bB29sbzz33HD755BOTT1cgImpvJodXUlISfHx8MHDgQPz444/48ccfjfb74osv2q04IqKmmBxeYWFhsLGx6chaiIhM1qqTVImIugqeg0BEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJUpu/gIOI2s9t3d0vscm5fKvJPtW3tTh5Dehz8Rd0czb8IuXzV6s6rL6uiOFF1AUU/v/geemLMy30tMeH508026Obwjr+rK1jK4m6uKkP9QEADOqthLODndE++aW38OLnZxD/hxHw9XI32qebwh4DPLt1WJ1dCcOLqAvw6OaIP471abZPXV0dAGBQr24Y3s94eFkThpdMtTRH0tL8CGB9cyRkWRheMmXaHEnL8yOA9cyRkGXhs1amWpojMWV+BLCuORKyLAwvmWppjoTzI2TpeJIqEckSw4uIZInhRUSy1CXCKzExEWq1Gk5OTggKCsLx48eb7Lt161ZMnDgRPXr0QI8ePRASEtJsfyKyTGYPr9TUVERFRSE2NhZZWVnw9/dHaGgorl69arT/wYMHMWfOHBw4cAAZGRnw9vbG1KlTcfny5U6unIjMyezhlZCQgIULFyIiIgLDhg1DUlISXFxckJycbLT/xx9/jMWLFyMgIAB+fn54//330dDQgPT09E6unIjMyaynStTW1iIzMxPR0dFSm62tLUJCQpCRkWHSOmpqaqDT6eDh4WH0fq1WC61WK93WaDQAAJ1OB51Odx/Vd22Np0rU1dVZ9HZaE2sZU1O3zazhVVFRgfr6eqhUKr12lUqFvLw8k9axevVq9O3bFyEhIUbvj4uLw7p16wza9+/fDxcXl9YXLROXqgDAHkePHsXlHHNXQ+3BWsa0pqbGpH6yPkn1jTfewK5du3Dw4EE4OTkZ7RMdHY2oqCjptkajkebJ3NzcOqvUTne65AZw5iTGjRsHfx/je6UkL9Yypo1HRy0xa3h5enrCzs4O5eXleu3l5eXo06dPs8u+8847eOONN/D9999j5MiRTfZTKBRQKAw/mOzg4AAHB4e2FS4D9vb20k9L3k5rYi1jauq2mXXC3tHREYGBgXqT7Y2T78HBwU0u99Zbb2H9+vXYu3cvxowZ0xmlElEXY/bDxqioKISHh2PMmDEYO3YsNm7ciOrqakRERAAAwsLC0K9fP8TFxQEA3nzzTcTExGDnzp1Qq9UoKysDACiVSiiVSrNtBxF1LrOH1+zZs3Ht2jXExMSgrKwMAQEB2Lt3rzSJX1JSAlvbX3cQt2zZgtraWvzhD3/QW09sbCxeffXVziydiMzI7OEFAEuXLsXSpUuN3nfw4EG928XFxR1fEBF1eWY/SZWIqC0YXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWeoS4ZWYmAi1Wg0nJycEBQXh+PHjzfb/7LPP4OfnBycnJ4wYMQJ79uzppEqJqKswe3ilpqYiKioKsbGxyMrKgr+/P0JDQ3H16lWj/Y8cOYI5c+ZgwYIFOHXqFGbOnImZM2ciJyenkysnInMye3glJCRg4cKFiIiIwLBhw5CUlAQXFxckJycb7b9p0yY89thjWLlyJYYOHYr169dj9OjR+Pvf/97JlROROdmb88Fra2uRmZmJ6Ohoqc3W1hYhISHIyMgwukxGRgaioqL02kJDQ7F7926j/bVaLbRarXRbo9EAAHQ6HXQ63X1uQddRU1OD/Px86XZB6S1oy84jJ9sRteXuen19fX3h4uLS2SVSK9w7nkDTY2pp42nq36VZw6uiogL19fVQqVR67SqVCnl5eUaXKSsrM9q/rKzMaP+4uDisW7fOoH3//v0WNeCFhYV48cUXDdqf/cCwb3x8PAYNGtQJVVFbNTWegOGYWtp41tTUmNTPrOHVGaKjo/X21DQaDby9vTF16lS4ubmZsbL2VVNTgwkTJki3q25rse/QCYROfBhKZ4VeX0t7pbZE944n0PSYWtp4Nh4dtcSs4eXp6Qk7OzuUl5frtZeXl6NPnz5Gl+nTp0+r+isUCigUCoN2BwcHODg4tLHyrsfd3R1jx46Vbut0OlTevIGJ48dZ1HZai3vHE7CeMTV128w6Ye/o6IjAwECkp6dLbQ0NDUhPT0dwcLDRZYKDg/X6A0BaWlqT/YnIMpn9sDEqKgrh4eEYM2YMxo4di40bN6K6uhoREREAgLCwMPTr1w9xcXEAgGXLlmHSpEmIj4/HE088gV27duHkyZP4xz/+Yc7NIKJOZvbwmj17Nq5du4aYmBiUlZUhICAAe/fulSblS0pKYGv76w7i+PHjsXPnTqxZswYvv/wyHnzwQezevRvDhw831yYQkRnYCCGEuYvoTBqNBu7u7rh165ZFTdjfS6fTYc+ePXj88ccten7EmljLmJr6N2r2k1SJiNqC4UVEssTwIiJZMvuEfWdrnOIz9UQ4udLpdKipqYFGo7Ho+RFrYi1j2vi32dJ0vNWFV2VlJQDA29vbzJUQUXMqKyvh7u7e5P1W925jQ0MDrly5AldXV9jY2Ji7nA7T+DGoS5cuWfS7qtbEWsZUCIHKykr07dtX7zSpe1ndnpetrS0eeOABc5fRadzc3Cz6iW6NrGFMm9vjasQJeyKSJYYXEckSw8tCKRQKxMbGGr2iBskTx1Sf1U3YE5Fl4J4XEckSw4uIZInhRUSyxPBqwfz58zFz5sx2Xeejjz6KF154odk+arUaGzdubNfHJbIkVhVepoQGycurr76KgIAAc5fRpK72nOtq9dwPqwovIjmqra01dwldktWE1/z58/Hjjz9i06ZNsLGxgY2NDQoLC7FgwQIMGDAAzs7O8PX1xaZNm4wuv27dOvTq1Qtubm5YtGiRyU+o6upqhIWFQalUwsvLC/Hx8QZ9rl69iunTp8PZ2RkDBgzAxx9/bNDHxsYGW7ZswbRp0+Ds7IyBAwfi888/l+4vLi6GjY0NPv30U0ycOBHOzs54+OGHUVBQgBMnTmDMmDFQKpWYNm0arl27ZuJvDUhOTsZDDz0EhUIBLy8vLF26VLqvpKQEM2bMgFKphJubG55++mm9b3Zq3Cv68MMPoVar4e7ujj/+8Y/Sh+OBu581feuttzB48GAoFAr4+Pjgtddek+5fvXo1hgwZAhcXFwwcOBBr166VvpQ0JSUF69atw+nTp6UxTUlJMXnbOlpbn3ONUxWvvfYa+vbtC19fXwDAkSNHEBAQACcnJ4wZMwa7d++GjY0NsrOzpWVzcnIwbdo0KJVKqFQqPPvss6ioqGiynuLi4s76dbQ/YSVu3rwpgoODxcKFC0VpaakoLS0Vd+7cETExMeLEiRPiwoUL4qOPPhIuLi4iNTVVWi48PFwolUoxe/ZskZOTI7755hvRq1cv8fLLL5v0uJGRkcLHx0d8//334p///Kd48sknhaurq1i2bJnUZ9q0acLf319kZGSIkydPivHjxwtnZ2exYcMGqQ8A0bNnT7F161aRn58v1qxZI+zs7MS5c+eEEEIUFRUJAMLPz0/s3btXnDt3TowbN04EBgaKRx99VPz8888iKytLDB48WCxatMik2t977z3h5OQkNm7cKPLz88Xx48elmurr60VAQICYMGGCOHnypDh69KgIDAwUkyZNkpaPjY0VSqVSzJo1S5w5c0b89NNPok+fPnq/u1WrVokePXqIlJQUcf78eXHo0CGxdetW6f7169eLw4cPi6KiIvH1118LlUol3nzzTSGEEDU1NeLFF18UDz30kDSmNTU1Jm1bZ7jf59yzzz4rcnJyRE5Ojrh165bw8PAQzzzzjDh79qzYs2ePGDJkiAAgTp06JYQQ4pdffhG9evUS0dHRIjc3V2RlZYkpU6aIyZMnN1lPXV2dOX417cJqwksIISZNmqQXGsYsWbJE/P73v5duh4eHCw8PD1FdXS21bdmyRSiVSlFfX9/suiorK4Wjo6P49NNPpbbr168LZ2dnqY78/HwBQBw/flzqk5ubKwAYhNe9oRMUFCQiIyOFEL+G1/vvvy/d/8knnwgAIj09XWqLi4sTvr6+zdbdqG/fvuKVV14xet/+/fuFnZ2dKCkpkdrOnj2rty2xsbHCxcVFaDQaqc/KlStFUFCQEEIIjUYjFAqFXli15O233xaBgYHS7djYWOHv72/y8p2trc85lUoltFqt1LZlyxbRs2dPcfv2balt69ateuG1fv16MXXqVL11X7p0SQAQ+fn5JtcjF1Z3VYl7JSYmIjk5GSUlJbh9+zZqa2sNJoD9/f31vpE4ODgYVVVVuHTpEvr379/kugsLC1FbW4ugoCCpzcPDQzoMAIDc3FzY29sjMDBQavPz80P37t0N1nfvd1MGBwfrHTIAwMiRI6X/N34D04gRI/Tarl692mTNja5evYorV67gt7/9rdH7c3Nz4e3trXddtGHDhqF79+7Izc3Fww8/DODuu6aurq5SHy8vL+nxc3NzodVqm3wMAEhNTcXmzZtRWFiIqqoq1NXVyf6KCqY850aMGAFHR0fpdn5+PkaOHAknJyep7d4vpT19+jQOHDgApVJp8JiFhYUYMmRI+26ImVnNnJcxu3btwooVK7BgwQLs378f2dnZiIiIkPUE6X9eYbPxemX3tjU0NLS4Hmdn53av597Hb+kxMjIyMG/ePDz++OP45ptvcOrUKbzyyiuyHh9Tn3PdunVr9bqrqqowffp0ZGdn6/3717/+hf/6r/9qr03oMqwqvBwdHVFfXy/dPnz4MMaPH4/Fixdj1KhRGDx4MAoLCw2WO336NG7fvi3dPnr0KJRKZYtXYx00aBAcHBxw7Ngxqe2XX35BQUGBdNvPzw91dXXIzMyU2vLz83Hz5k2D9R09etTg9tChQ5utoa1cXV2hVqsNvp280dChQ3Hp0iVcunRJajt37hxu3ryJYcOGmfQYDz74IJydnZt8jCNHjqB///545ZVXMGbMGDz44IO4ePGiXp97x7Sraetz7l6+vr44c+YMtFqt1HbixAm9PqNHj8bZs2ehVqsxePBgvX+NYdjVf1+tYVXhpVarcezYMRQXF6OiogIPPvggTp48iX379qGgoABr1641eEIAd9+qXrBgAc6dO4c9e/YgNjYWS5cubfYqjwCgVCqxYMECrFy5Ej/88ANycnIwf/58veV8fX3x2GOP4c9//jOOHTuGzMxM/OlPfzK6V/LZZ58hOTkZBQUFiI2NxfHjx/Xe/Wtvr776KuLj47F582b861//QlZWFt59910AQEhICEaMGIF58+YhKysLx48fR1hYGCZNmoQxY8aYtH4nJyesXr0aq1atwo4dO1BYWIijR49i27ZtAO6GW0lJCXbt2oXCwkJs3rwZX375pd461Go1ioqKkJ2djYqKCr0/7q6grc+5e82dOxcNDQ14/vnnkZubi3379uGdd94B8Ose9pIlS3Djxg3MmTMHJ06cQGFhIfbt24eIiAgpsO6tx5S98C7L3JNunSk/P1+MGzdOODs7CwAiLy9PzJ8/X7i7u4vu3buLyMhI8dJLL+lNAIeHh4sZM2aImJgY0bNnT6FUKsXChQvFnTt3THrMyspK8cwzzwgXFxehUqnEW2+9ZTBpWlpaKp544gmhUCiEj4+P2LFjh+jfv7/BhH1iYqKYMmWKUCgUQq1W671D1Thh3zh5K4QQBw4cEADEL7/8IrVt375duLu7m/w7S0pKEr6+vsLBwUF4eXmJv/zlL9J9Fy9eFL/73e9Et27dhKurq3jqqadEWVmZdL+xyfQNGzaI/v37S7fr6+vF3/72N9G/f3/h4OAgfHx8xOuvvy7dv3LlSun3Pnv2bLFhwwa9+u/cuSN+//vfi+7duwsAYvv27SZvW2e4n+fcvQ4fPixGjhwpHB0dRWBgoNi5c6e0zkYFBQXiv//7v0X37t2Fs7Oz8PPzEy+88IJoaGgwWk9RUVEH/wY6Di+JIxM2Njb48ssv2/2jSiRfH3/8MSIiInDr1q12m6OUE6t/t5FILnbs2IGBAweiX79+OH36NFavXo2nn37aKoMLYHjdl5KSkmYnp8+dOwcfH59OrKh1jL2l3ui7777DxIkTO7EaaklZWRliYmJQVlYGLy8vPPXUU3qfRrA2PGy8D3V1dc1+vEKtVsPevuu+Ppw/f77J+/r162e1r+gkDwwvIpIlqzpVgogsB8OLiGSJ4UVEssTwIiJZYnhRh5o/f7504TsHBweoVCpMmTIFycnJrfpoSkpKitErbXS0jvgOA2ofDC/qcI899hhKS0tRXFyM7777DpMnT8ayZcvw5JNPoq6uztzlkVyZ87NJZPma+pxeenq6ACBdiDA+Pl4MHz5cuLi4iAceeEBERkaKyspKIcSvn9H8z3+xsbFCCCF27NghAgMDhVKpFCqVSsyZM0eUl5dLj3Pjxg0xd+5c4enpKZycnMTgwYNFcnKydH9JSYl46qmnhLu7u+jRo4f43e9+J33eLzY21uBxDxw40CG/J2o97nmRWfzmN7+Bv78/vvjiCwCAra0tNm/ejLNnz+KDDz7ADz/8gFWrVgEAxo8fj40bN8LNzQ2lpaUoLS3FihUrAAA6nQ7r16/H6dOnsXv3bhQXF2P+/PnS46xduxbnzp3Dd999h9zcXGzZsgWenp7SsqGhoXB1dcWhQ4dw+PBhKJVKPPbYY6itrcWKFSvw9NNPS3uOpaWlGD9+fOf+oqhp5k5PsmxN7XkJIcTs2bPF0KFDjd732WefiZ49e0q3Tb0axokTJwQAaa9t+vTpIiIiwmjfDz/8UPj6+kpXXBBCCK1WK5ydncW+fftarJ/Mi3teZDZCCOlaVN9//z1++9vfol+/fnB1dcWzzz6L69evo6amptl1ZGZmYvr06fDx8YGrqysmTZoE4O7nTgEgMjISu3btQkBAAFatWoUjR45Iy54+fRrnz5+Hq6srlEollEolPDw8cOfOHZMuEEjmxfAis8nNzcWAAQNQXFyMJ598EiNHjsT//u//IjMzE4mJiQCa/87C6upqhIaGws3NDR9//DFOnDghXaywcblp06bh4sWLWL58uXRN/sZDzqqqKgQGBhpcNrmgoABz587t4K2n+9V1PzVMFu2HH37AmTNnsHz5cmRmZqKhoQHx8fHSVWY//fRTvf7GLl+cl5eH69ev44033pAuyX3y5EmDx+rVqxfCw8MRHh6OiRMnYuXKlXjnnXcwevRopKamonfv3k1+qYclXTbZ0nDPizqcVqtFWVkZLl++jKysLLz++uuYMWMGnnzySYSFhWHw4MHQ6XR49913ceHCBXz44YdISkrSW4darUZVVRXS09NRUVGBmpoa+Pj4wNHRUVru66+/xvr16/WWi4mJwVdffYXz58/j7Nmz+Oabb6Tr/s+bNw+enp6YMWMGDh06hKKiIhw8eBD/8z//g3//+9/S4/7zn/9Efn4+KioqpC+8pS7A3JNuZNnCw8Ol0wzs7e1Fr169REhIiEhOTtb73suEhATh5eUlnJ2dRWhoqNixY4fBJawXLVokevbsqXeqxM6dO4VarRYKhUIEBweLr7/+2uC7DIcOHSqcnZ2Fh4eHmDFjhrhw4YK0ztLSUhEWFiY8PT2FQqEQAwcOFAsXLhS3bt0SQghx9epVMWXKFKFUKnmqRBfDS+IQkSzxsJGIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLP0/HtlqTEKTFncAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-23T05:06:32.355452Z", + "iopub.status.busy": "2024-03-23T05:06:32.355125Z", + "iopub.status.idle": "2024-03-23T05:06:32.594209Z", + "shell.execute_reply": "2024-03-23T05:06:32.593215Z" + }, + "papermill": { + "duration": 0.261282, + "end_time": "2024-03-23T05:06:32.596245", + "exception": false, + "start_time": "2024-03-23T05:06:32.334963", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABqKklEQVR4nO2dd3gU5dbAf7M1vdBCC4QmEKRJE6SIRoGLCsq9YLlSRKyIGkFAEAQLKIhRRFAUKwoWLJ8Fr0ZAqSpNulJDSwPSk63z/bHZyfbsht1kN5nf8+yT7NR3dnfOnHPeUwRRFEVkZGRkQgxFTQ9ARkZGpirIwktGRiYkkYWXjIxMSCILLxkZmZBEFl4yMjIhiSy8ZGRkQhJZeMnIyIQksvCSkZEJSVQ1PYDqxmw2c+7cOaKjoxEEoaaHIyMj44AoihQWFtK0aVMUCvf6VZ0TXufOnSMxMbGmhyEjI1MJp0+fpnnz5m7X1znhFR0dDVg+mJiYmBoejYyMjCMFBQUkJiZK96o76pzwspqKMTExsvCSkQliKnPryA57GRmZkEQWXjIyMiGJLLxkZGRCkjrn8/IGURQxGo2YTKaaHoqMTMiiVCpRqVQBC0kKCuG1bNkyFi1aRGZmJl27dmXp0qX07t3b5bbvvfceEyZMsFum1WopKyvzy1j0ej3nz5+npKTEL8eTkanLRERE0KRJEzQajd+PXePCa+3ataSmprJixQr69OlDWloaQ4YM4ciRIzRq1MjlPjExMRw5ckR67y/JbjabOXHiBEqlkqZNm6LRaORAVhmZKiCKInq9npycHE6cOEG7du08BpxWhRoXXkuWLGHSpEmSNrVixQq+++47Vq1axYwZM1zuIwgCjRs39vtY9Ho9ZrOZxMREIiIivNrHLIqcvlhCpEZFg2it38ckIxOqhIeHo1arOXXqFHq9nrCwML8ev0Yd9nq9np07d5KSkiItUygUpKSksG3bNrf7FRUV0bJlSxITExkxYgQHDhzw67h8eUKU6IzklxrILdL5dQwyMrUBf2tbdscO2JG9IDc3F5PJREJCgt3yhIQEMjMzXe7Tvn17Vq1axddff81HH32E2WymX79+nDlzxuX2Op2OgoICu5c/0RnNAMhdTGRkqpeQC5Xo27cvY8eOpVu3bgwaNIh169bRsGFD3nzzTZfbL1iwgNjYWOnl77xGSXjJ0ktGplqpUeHVoEEDlEolWVlZdsuzsrK89mmp1Wq6d+/O0aNHXa6fOXMm+fn50uv06dOXPW5brMIrFBk/fjwjR4706zGvvfZaHnvsMY/bJCUlkZaW5tfzytQM3nzfgaJGhZdGo6FHjx6kp6dLy8xmM+np6fTt29erY5hMJvbt20eTJk1crtdqtVIeYyDyGXVGSyyYWMOGY03+iGQCwzPPPEO3bt1qehhBS43PNqampjJu3Dh69uxJ7969SUtLo7i4WJp9HDt2LM2aNWPBggUAzJ8/n6uvvpq2bduSl5fHokWLOHXqFPfee2+1j91sFtGHsOYlIwNgMBhQq9U1PQyfqXGf15gxY1i8eDFz5syhW7du7Nmzh/Xr10tO/IyMDM6fPy9tf+nSJSZNmkTHjh3517/+RUFBAVu3biU5OTkg4xNFkRK90eUrr1RPmcFkeelNbrer6svbZubjx49n06ZNvPrqqwiCgCAIHDt2jIkTJ9KqVSvCw8Np3749r776qsv9582bR8OGDYmJieGBBx5Ar9d7dd7i4mLGjh1LVFQUTZo04eWXX3baJjs7m5tvvpnw8HBatWrF6tWrnbYRBIHly5czbNgwwsPDad26NZ9//rm0/uTJkwiCwKeffsqAAQMIDw+nV69e/P333/zxxx/07NmTqKgohg0bRk5OjldjB1i1ahWdOnVCq9XSpEkTJk+eLK3LyMhgxIgRREVFERMTw+jRo+3cG1at6MMPPyQpKYnY2Fhuv/12CgsLpW3MZjMvvfQSbdu2RavV0qJFC55//nlp/fTp07niiiuIiIigdevWPP300xgMBsASjD1v3jz27t0rfafvvfdepdd0+PBh+vfvT1hYGMnJyfz8888IgsBXX31l91muXbuWQYMGERYWxurVq7lw4QJ33HEHzZo1IyIigs6dO/PJJ5/YHdub77s6qXHNC2Dy5Ml2PxxbNm7caPf+lVde4ZVXXqmGUVkoNZhInvNjtZ3PloPzhxChqfwrevXVV/n777+58sormT9/PgDx8fE0b96czz77jPr167N161buu+8+mjRpwujRo6V909PTCQsLY+PGjZw8eZIJEyZQv359u5vMHdOmTWPTpk18/fXXNGrUiKeeeopdu3bZmTrjx4/n3LlzbNiwAbVazZQpU8jOznY61tNPP83ChQt59dVX+fDDD7n99tvZt28fHTt2lLaZO3cuaWlptGjRgnvuuYc777yT6OhoXn31VSIiIhg9ejRz5sxh+fLllY59+fLlpKamsnDhQoYNG0Z+fj5btmwBLELHKrg2bdqE0Wjk4YcfZsyYMXa/x2PHjvHVV1/x7bffcunSJUaPHs3ChQulz27mzJmsXLmSV155hf79+3P+/HkOHz4s7R8dHc17771H06ZN2bdvH5MmTSI6Oponn3ySMWPGsH//ftavX8/PP/8MQGxsrMdrMplMjBw5khYtWrBjxw4KCwt54oknXG47Y8YMXn75Zbp3705YWBhlZWX06NGD6dOnExMTw3fffcfdd99NmzZtpGwXb77vakWsY+Tn54uAmJ+f77SutLRUPHjwoFhaWiotK9YZxJbTv62RV7HO4PV1DRo0SHz00Uc9bvPwww+Lo0aNkt6PGzdOrFevnlhcXCwtW758uRgVFSWaTCaPxyosLBQ1Go346aefSssuXLgghoeHS+M4cuSICIi///67tM2hQ4dEQHzllVekZYD4wAMP2B2/T58+4oMPPiiKoiieOHFCBMS3335bWv/JJ5+IgJieni4tW7Bggdi+fXuP47bStGlTcdasWS7X/e9//xOVSqWYkZEhLTtw4IDdtcydO1eMiIgQCwoKpG2mTZsm9unTRxRFUSwoKBC1Wq24cuVKr8YjiqK4aNEisUePHtL7uXPnil27dvV6/x9++EFUqVTi+fPnpWU//fSTCIhffvmlKIoVn2VaWlqlxxs+fLj4xBNPiKLo3fftClf3VGV4ukdtCQrNK5gJVys5OH+Iy3VnLpaSV2oxsQRBoFNT/04GhKuVl7X/smXLWLVqFRkZGZSWlqLX652ekl27drXLJujbty9FRUWcPn2ali1buj32sWPH0Ov19OnTR1pWr1492rdvL70/dOgQKpWKHj16SMs6dOhAXFyc0/EcJ2j69u3Lnj177JZ16dJF+t/qVujcubPdMldanSPZ2dmcO3eO66+/3uX6Q4cOkZiYaBdWk5ycTFxcHIcOHaJXr16AZdbUttpnkyZNpPMfOnQInU7n9hxgSY177bXXOHbsGEVFRRiNxsuaUDpy5AiJiYl2M/XucoR79uxp995kMvHCCy/w6aefcvbsWfR6PTqdTvptePN9Vzey8KoEQRDcmm4KhUBYuYARcL9dTbBmzRqmTp3Kyy+/TN++fYmOjmbRokXs2LGjpodWZWydytacU8dlZnPlEyjh4eF+H4/j+Ss7x7Zt27jrrruYN28eQ4YMITY2ljVr1lSbHykyMtLu/aJFi3j11VdJS0ujc+fOREZG8thjj3nt/6wJatxhH6qIoiiFSUDNh0poNBq7Ej5btmyhX79+PPTQQ3Tv3p22bdty7Ngxp/327t1LaWmp9H779u1ERUVVGszbpk0b1Gq1nTC8dOkSf//9t/S+Q4cOGI1Gdu7cKS07cuQIeXl5Tsfbvn2703tbf5c/iY6OJikpyS5Ex5aOHTty+vRpu5jAgwcPkpeX5/XEULt27QgPD3d7jq1bt9KyZUtmzZpFz549adeuHadOnbLbxvE7rYz27dtz+vRpu4mFP/74w6t9t2zZwogRI/jvf/9L165dad26td136c33bSWnsIxD5wvs7o9AEDyqQohhNIuYzPYCSxTFGqtCkZSUxI4dOzh58iRRUVG0a9eODz74gB9//JFWrVrx4Ycf8scff9CqVSu7/fR6PRMnTmT27NmcPHmSuXPnMnny5Epz0qKiopg4cSLTpk2jfv36NGrUiFmzZtnt1759e4YOHcr999/P8uXLUalUPPbYYy61ks8++4yePXvSv39/Vq9eze+//84777zjnw/HBc888wwPPPAAjRo1YtiwYRQWFrJlyxYeeeQRUlJS6Ny5M3fddRdpaWkYjUYeeughBg0a5GRuuSMsLIzp06fz5JNPotFouOaaa8jJyeHAgQNMnDiRdu3akZGRwZo1a+jVqxffffcdX375pd0xkpKSOHHiBHv27KF58+ZER0ej1bpP/r/hhhto06YN48aN46WXXqKwsJDZs2cDlVdeadeuHZ9//jlbt24lPj6eJUuWkJWVJQlrb75vAKPJTFaBDrMoUqwzEXF5ng+PyJpXFbHGdykVwVEyZ+rUqSiVSpKTk2nYsCFDhgzhtttuY8yYMfTp04cLFy7w0EMPOe13/fXX065dOwYOHMiYMWO45ZZbeOaZZ7w656JFixgwYAA333wzKSkp9O/f386/BfDuu+/StGlTBg0axG233cZ9993nstTRvHnzWLNmDV26dOGDDz7gk08+CVj4C8C4ceNIS0vjjTfeoFOnTtx00038888/gOVG//rrr4mPj2fgwIGkpKTQunVr1q5d69M5nn76aZ544gnmzJlDx44dGTNmjOQTu+WWW3j88ceZPHky3bp1Y+vWrTz99NN2+48aNYqhQ4cyePBgGjZs6BS64IhSqeSrr76iqKiIXr16ce+99zJr1iyASis6zJ49m6uuuoohQ4Zw7bXX0rhxY6fsC2++70slBszlIT5igHPmBDHQZwgyCgoKiI2NJT8/38k5WlZWxokTJ2jVqlWlX/bFYh1nLpUSoVFRojcCcGWzWBRy/S+fEQSBL7/80u+pSjIWc7B///4cPXqUNm3aBPRcoihyJKtQerA3jQsnSiV6fU9Z8XSP2iKbjVXEmtMYplJQYvVpioAsu2RqkC+//FJyGxw9epRHH32Ua665JuCCC6CwzGiXcRJotUg2G6uIzmD5krTqio+wNqmwGRkZREVFuX1lZGTU9BA94mnsv/32W00Pr0qsXr3a7TV16tQJgMLCQh5++GE6dOjA+PHj6dWrF19//XW1jM+xpl2gjTpZ86oiVs1Lq7L1SNYe1atp06ZOcVaO6/1FIH7knsberFkzv5+vOrjlllvs4qxssYZtjB07lrFjx1bnsAAoM5go0lncJ9FhagrLDAF/mMvCqwqYxYqE7DBbzasWqV4qlYq2bdvW9DCqTCiP3R3R0dF2QbHBxIVii+8kJkyNRqWgEOtDKXAPc9lsrAJ6oxkREYUgoFbKH6FM3cZkNnOpXHjVj9JgnbOSfV5BiF4yGe0/vlqkeMnIeI01PEKrUhKlVSGUa1uBLhYlC68qYI0c1qqUlnIlNTweGZmaQhRFLhTZal2CjeZl8zg3GcDo31QjWXhVAeeZRuu3VTPjkZGpKYp0RnRGE0pBID7C0ljWpdm4401YdSNccE5Rqyqy8KoCOkezUZZdMnUUq9YVH6mRsk2stogkvIxlsOt9OLcbsvb77dyy8KoCjsKrwmwMTfHl2BDDtvKmjIw7dEYTBWWWyq/1IzXScoX0MBct5mLJBUCEHuMheYTfzi+HSviI0WTGWF72RKOyzzoNTdHlzPnz54mPj6/pYcgEOVatKzpMjdam9pyd2VhwHswmiGsFQxb49fyy8PIRq9alVips1ORyaon08rbtXCARRRGTyYRKJf9EgxGTWeRSeV6crdYFFWZjtOkSGIoAAYY8D5oIx8NcFrLZWBmiCPpi6aUrKUQwlBAmlknLBGMJgqEE0WY7v7x8CJQpLCzkrrvuIjIykiZNmvDKK69UuR2aq4YN69atY/DgwURERNC1a1e2bdtmt8/mzZul5hiJiYlMmTKF4uJiaf2HH35Iz549iY6OpnHjxtx55512VU83btyIIAj88MMP9OjRA61Wy+bNm30eu0z1kFeix2QW0aoURIfZP2AEAcLQE2/KtSwIj4OGV/h9DPJjrTIMJfBCRSpMvfKXLQEr3PLUOdBEVr4dlhZyW7Zs4ZtvviEhIYE5c+b4tTnCrFmzWLx4Me3atWPWrFnccccdHD16FJVKxbFjxxg6dCjPPfccq1atIicnR2qq8u677wKW9lrPPvss7du3Jzs7m9TUVMaPH8/3339vd54ZM2awePFiWrduLZuuQYooilJEfb1IrVOtMAUiLYRsFIigjgJtYIoSysKrFlBYWMj777/Pxx9/LNVMt9bR8hdTp05l+PDhgKX2VqdOnTh69CgdOnRgwYIF3HXXXZKW165dO1577TUGDRrE8uXLCQsL45577pGO1bp1a1577TV69epFUVERUVFR0rr58+dzww03+G3cMv6nWGekzGBCIQjUi3Tu9xhWch6NYMCICmKaQp5/u9RbkYVXZagjLBpQOf9kF1FmMJFUP4LoMMsXdzizEIPJTNuGUYRr/Fg6Uu2dj+D48eMYDAa7ZguxsbF+bY5g2/zC2p08OzubDh06sHfvXv766y+7noyiKGI2mzlx4gQdO3Zk586dPPPMM+zdu5dLly5Jtd4zMjLsig56W6lUpubItYZHRKhROlbcLb2ERn8JUYRMZQINlIETMbLwqgxBkEw3URQpE4yIahFtRDSUzzaKahOiwmxxSAZREw5/4qr5hVUAFRUVcf/99zNlyhSn/Vq0aEFxcTFDhgxhyJAhrF69moYNG5KRkcGQIUOcGjw4NoaQCS70RhOF1vCIKIeS1EadpGXlEEcx4TQI4Fhq550WIPQms1Sn3jYh22rx19RkY+vWrVGr1fzxxx+0aNECgPz8fP7++28GDhwY8PNfddVVHDx40G0lh3379nHhwgUWLlwoNfb4888/Az4uGf9zoViPCERpVVLnLMAyuXTpFIgmTKoIsvTxqAOcmS3PNvqAbXCqy4YGNSS9oqOjGTduHNOmTWPDhg1SkweFws04/cz06dPZunUrkydPZs+ePfzzzz98/fXXUhf0Fi1aoNFoWLp0KcePH+ebb77h2WefDfi4ZPyL2SxyUaoe4aB1FWaCoRgEJYboREQCfzvIwssHpJxGh2oSwZAetGTJEvr27ctNN91ESkoK11xzDR07dvS6bvjl0KVLFzZt2sTff//NgAED6N69O3PmzJEmDBo2bMh7773HZ599RnJyMgsXLmTx4sUBH5eMf8krtYRHaJQKYmzDI3SFUJRp+T+2Oagsgi3QJXHkBhw2VNaA4+ylEi4U62kUraVxbEX7riOZheiMJto0jCJSGxyWeHFxMc2aNePll19m4sSJNT0cmRBHFEVpsqpJbBgNo8vvD5MRcg6D2QDh9SC+JTqDiSNZhSgFgTb1tXIDjmDAdennCmryKbB7924OHz5M7969yc/PZ/78+QCMGOG/XDKZukuJ3iSFR1irRyCKkJ9hEVxKrUXrwmZCJ8BjkoWXDzhVkyhHcivVsBK7ePFijhw5gkajoUePHvz2228cOnSIYcOGud2nqKioGkcoE6pYm2vERahRWSerSnKhLB8QID4JFJaHum09r0AadrLw8hKT2YzBVJ6QrXbtKqxJ0dW9e3d27tzptLy0tNRjMwoZmcrQG80UlFqaa9SPLHfUG0oh/6zl/5imdnmLtlNEgbwngsJhv2zZMpKSkggLC6NPnz78/vvvXu23Zs0aBEGolmalVq1LpVCgcgjMC+ZKquHh4bRt29btqy5gMouczy+ltLw5sIxvXCzWIyISqVVZgrDNZrh0EhBBGw2RDe22t53hDqQxUuPCa+3ataSmpjJ37lx27dpF165dGTJkiF3SritOnjzJ1KlTGTBggN/H5ErVdWcy2u3n95HI+IOsgjJyCnX8kx38JnJmQRlHMgsxmgLtMfIO2/CIBtbqEQVnLQUGFSqIa2njN7Fg+9YayBwIalx4LVmyhEmTJjFhwgSSk5NZsWIFERERrFq1yu0+JpOJu+66i3nz5tG6dWu/jcUaRV5SUuK0zlWTWStCTUepynik1BCYxOBAkF9iQGc0UaIPjjHnlxowms2olQpiwtVQmmfxdYFFcCmdcxttRVlpqeVess3Q8Bc16vPS6/Xs3LmTmTNnSssUCgUpKSlOJVdsmT9/Po0aNWLixImVdj/W6XTodBWdfAsKCtxuq1QqiYuLk7S+iIgISQUuLilBNBpRmAXKysrs9jMZ9IhGE3qdkjIhOH50MhWY9DrE8qYpjt9dsGHQ6xDNZkpKFWhq+LckiiJZeSWIRhMxYVp0xYVw8RQgQnh9QANuPk/RqEc06MjOL6R+vXiUSj/m/JZTo8IrNzcXk8lEQkKC3fKEhAQOHz7scp/NmzfzzjvveO2EXrBgAfPmzfN6TNZCfI5ma3ZBGXqTiDlKQ77a/ovIKdShM5ox5Wv8m5gt4xes3w+ApjS8kq1rlsy8UkQRdJfUXAqr2fk0vdFMdqEOQQBljJaCklyLuajUQFQY5J5wu29OXil6o5mkpg0DVtwypGYbCwsLufvuu1m5ciUNGniX8jlz5kxSU1Ol9wUFBVJ+nSsEQaBJkyY0atQIg8GSgGo2izy09Df0RjPvTehFYj375OFX1+xm39l85t6czMBWjapwZTKB5JVPdnPgXD4A6U9cW7OD8YDOYOLedRZL4s4+LZjYv1WNjuf57w7yy+FsbkxuzHTDRvh9uaXSyeiPIL6Fx31T39jCiYs6vnmkfcBS1GpUeDVo0AClUklWVpbd8qysLJfS+tixY5w8eZKbb75ZWmZ1CKpUKo4cOUKbNm3s9tFqtWi1DnlYXqBUKiVV92xeKScuGVApBJIS4p26ZF/SwdlCE3pU1ZKOI+MbuaVmzhZaTLBg/n4KDGXSOM8Wmmp0rNmFZazZlYnBJPLfxFzCvnsaRBPc+iY0qbwq6sUykTKjKDVoDgQ16rC3BlOmp6dLy8xmM+np6fTt29dp+w4dOrBv3z727NkjvW655RYGDx7Mnj17PGpUl8PxHMssVYv6EU6CC5Bq2ZvMssc+GAmVb8UaSwWWMss1ycc7MjCYRPo3V9Hmt8csgqvzaOh6u1f7W+8TQwBnTWvcbExNTWXcuHH07NmT3r17k5aWRnFxMRMmTABg7NixNGvWjAULFhAWFsaVV15pt39cXByA03J/cjzHUou9dYMol+sVgkOfOpmgIlSeKdY6WQCXig0etgwseqOZ1TsyAJEXNe9A7mlLBP3wl70+hqY8pCiQmleNC68xY8aQk5PDnDlzyMzMpFu3bqxfv15y4mdkZKBwrNZYzVg1rzYNXRfKU8iaV3ATIk+VgrIKzetSDWpeP+w/T06hjomRm2l27kdLPNeoVRDmPknaEY2kedXy9CBrswZXbNy40eO+7733nv8H5MDx3HLNy53wKvdHmkPkJqlrhMozxVbzyi+tOc3r/a0naSOcZYZoaZ7CdbOheQ+fjlEnzMZg55fDWWw5agnKS24S63IbpTWLXhZeQYkYIl4vW59XTWle+87kcyAjm6+0r6M2l0Hra6Hfoz4fx2o26mqrwz7YOZxZwCMf78Yswu29ErmymWu12Wo2hsoTvq4RKs+UAhvNq8xgpqwGMgPe23qS6ao1dBROQUR9y+xiFdw2aqXlngik5iULLzfkFOqY+N6fFOtN9GtTn2dHXuk2XsVqNso+r+AkVISXrdkI1a99XSjSUbjvW+5RrbcsGLkcoqsWYFodZqMsvNyw53QeWQVltG4QyfK7ergMkbCiVMhmYzATKt+KrdkI1T/j+M3mXSxQLAdA7PMAXDGkysfS1oXZxmDlhuQEPpjYmyax4cRGeE4qrWgFFiq3Sd0iVCqdFzhoXtUZ62U0Grlyx5PUFwrJi+lA3A3zL+t4ssO+hunXxrsUJKvDPoCzwjKXQYjILgrLHDSvkurTvI5+9QK9xL8oRUvEne9JTTSqilV46YM1VKKoqMipXo+ngvm1FYVN2VuZ4CNUzPmC8vAIhWCZ/MkrrSbN68xO2u5PA2BT66kMbdzxsg9ZHUGqPvu8Tpw4wfDhw4mMjCQ2Npb4+Hji4+OJi4sjPj4+EGMMeuQg1eAmVL4Vq+bVpLwzVV51aF5lBeg/HY8KE9+ZrqbrLa7jLX0lKM3G//73v4iiyKpVq0hISKiWpqbBjkKQQyWCmVDRiK0+r5b1IzibV8ql4gBrXqII36WiKcjgjNiADe1mMTwuovL9vEBTDaESPguvvXv3snPnTtq3bx+I8YQkcpBqcBMqX4vVbGxRL4Ktxy4E3ue1dw3s+wyjqGCKfjIzBvgvPzgozcZevXpx+vTpQIwlZJGCVMtVL7NZZMK7vzP7q301OSyZckJBdhlNZorLSz8n1rNoPwGdbbxwDL6fCkCacRSljXvSK8l/bp8Kh30QaV5vv/02DzzwAGfPnuXKK690qk3dpUsXvw0uVJCCVMsf8fvP5bPhSA4Az45wH9wqUz2EgtlYpKuYaWxRLrwCFqRq1MPn94C+iN2KTrxhGsGCfi39+jsNSp9XTk4Ox44dk0rWgCXOSRRFBEHAZKp7NdyVDpqX0cb5pTeZ3XbYlqkeqlt0fb/vPG/+epylt3enRX3vfEjWANVwtZKG0ZYwhYA57H+ZD+f3oNfE8WDBA8REaBnRrZlfTxGUJXHuueceunfvzieffCI77MtxdNjbfiJlell41TTVrXh9vecse0/nseFINuP6JXm1j9VZHxOuIj7C0mIsLxCVJY6mw9alALwe/SiZBfW5v1ciYWr//kaDsiTOqVOn+Oabb+pMw1JvUEhBquU+L5u7pcRgJBb/t32S8Z7qnkixWkoFPggfq/CKDlMTX57RkVeix2wWJZ/qZVOUDV8+YDl2p3G8trM9CgHuvrqlf45vgzUxO5A+L58d9tdddx179+4NxFhCFsd6XmWGii+sNEj679Vlqt/lZTmhY7qPJ6wxXjFhKikdzSw6R91XGbMZvnoQirOhUTJpyrEApHRMoHm8f8IjbNGUWxtBZTbefPPNPP744+zbt4/OnTs7OexvueUWvw0uVHD0eemMFQIrlBqe1lYC7bAvM5hQCILk57GezjHR2hNWLS0mXI1WpSRCo6REb+JSib7S3Fqv2LEcjv4MqjCKbn6TT1eeA2C8l2atr1RHSRyfhdcDD1jUzvnznRM366rDXnDwecmaV3ARSNFlMosMf+03SvQmNk67Fq1KKWnghTpfzEaLoIsOswiq+AgNJfpSLpXoScJ1BV+vObcHfppr+X/IC6w9FU2J3sQVCVH0bVP/8o7tBqsgD6qSOGaz2e2rLgouAGu1HGt6kG0ROVnzqnkCqXjtOHGBYznFnM8vI7vA0pndejpfNC9rLa+Y8kazcZLf6zKd9roiS1iE2QAdbsJ81QQ+3HYSgLF9kwI24WZ12AdNkKrBYEClUrF///5AjSckcYywty19K2teNU8gHfY/7MuU/re6CySz0Qefl1XQ2Wpe4IdYrx+ehIvHIKYZ3LKUTf/kcvJCCdFhKm7t7t/wCFuqo6qET8JLrVbTokWLOqthuUNwEF6y5hVcBOr2MZlF1h+oEF6lestDy/o7qMpsY0y4HzWvfZ/DntUgKOC2tyCiHu9tPQnA6J6JRGoDVxFLbTUbg0XzApg1axZPPfUUFy9eDMR4QpKKprOW97LmFVwESvH68+RFcgp10vsSvb2ZWODDTGGF2WiveVU5RejiCfj2ccv/A6dBUn+O5xSx6e8cBAHG9vV/eIQtmmBMD3r99dc5evQoTZs2pWXLlkRG2jsTd+3a5bfBhQqO9bwCoXlZjy0HBVeFwEivr/actXtv/a4rZhsNUuZJZVSYjZZb0hrrVaXkbJMBvrgXdAWQeDUMfBKAD7adAmBw+0a0rH+ZkwCVoFEF4WzjyJEjAzCM0MaxnpfdbKMfhJcoivxnxTYUgsDa+6+WBZiPBKJU0ec7z/DJ7/YFCqwPLavZaDSLlBnMhGsqj163zkzGhFuEVuzl+Lw2LoCzf4I2FkatBKWKIp2Rz3eeAfA66v9ykHIbgynOa+7cuYEYR0jjmB5kG+dV5gezsUhn5M9TlwAo1puICqCvojbi7ziv8/mlzPjiLwAeGNSGA+fy+e2fXCfNCyy+LG+El1XzinHQvHz2eR3fBL8tsfx/y6sQ1wKAdbvOUKQz0rpBJAPaelfe/HKQchuDSfOysnPnTg4dOgRAp06d6N69u98GFWo4zjbaal4lfhBetpW2DUYzXF558TqHvxWvE7nFGM0iifXCeXJIe+7/aCdQ4bC3bXJbUGogISas0mMWuPF5+aR5FV+AdfcBIlw1FjrdahmPKPJ+uaN+bN+W/ks38oC6GkIlfBZe2dnZ3H777WzcuJG4uDgA8vLyGDx4MGvWrKFhw4b+HmPQ42g2+jvC3mTzKA/kk6y24veuTuWHC1crUSgEwsuTmksls7FiU2/CJURRrEgPKjcbfZ5tFEX4+mEoyoQGV8DQhdKqzUdzOZZTTKRGyagezb073mVSHYnZPs82PvLIIxQWFnLgwAEuXrzIxYsX2b9/PwUFBUyZMiUQYwx6HHMbdX72ednGKQXySVZb8fftYz2eUF4/xCq8pIkaW+HlRaBqid4kPfgqHPY+zjb+vhL+/gGUGvj3KtBUOOStWte/ezSX4sgCTVCajevXr+fnn3+mY8eKDiPJycksW7aMG2+80a+DCxUcm87azjb6o2W7reYQyNmb2oq/QyXM0syv5b3Vp2UNi7EzG73QvKxal8pGi7MKr2K9Cb3RLAkDl2Tuh//Ntvx/w7PQuLO0KuNCCemHswEYWw2OeitWs9FkFjGZReke8SdVSg9yTMYGSwCrYxu0ukJF01nLe9s4L3/4vGSz8fLwt8Peejjr9x7m0WysXPOqCFBVS8eMDlNJGr1H7UtfYkn/Memg3RDoc7/d6g+3n0QUYUC7BrRpGFXpWPyFNTEbAvfArVJJnEcffZRz585Jy86ePcvjjz/O9ddf79fBhQpKh3pednFe/hBetpqXMfhLGgcb/v7ErJqXVbg4+rxshaU3UfbWbawmI1j8qLHhXsR6/TgTco9AVGMY+UaFOoglaHbtH5ZwjkBVj3CHraYYqAeuz8Lr9ddfp6CggKSkJNq0aUObNm1o1aoVBQUFLF26tEqDWLZsGUlJSYSFhdGnTx9+//13t9uuW7eOnj17EhcXR2RkJN26dePDDz+s0nn9hWOQqq3m5Q+z0VZx0MupWT7jb7NR8nlJZqPlNiqTzMYKfDEbYxz8UZXOOB74Cna+Bwhw25sQaR8C8dXucxSUGWlRL4Jr2zeqdBz+RK2oEC2BivXy2eeVmJjIrl27+Pnnnzl8+DAAHTt2JCUlpUoDWLt2LampqaxYsYI+ffqQlpbGkCFDOHLkCI0aOX/g9erVY9asWXTo0AGNRsO3337LhAkTaNSoEUOGDKnSGC4X5yBVP8822tbElzUvnxH9rHtJ2Q4ODnuXZqMXDvuKKqr2t2OcTUVVJ/JOw/+VT5D1fwxaX+s0RtvwiED4nDyhUAioFAJGsxiwGccqxXkJgsANN9zADTfccNkDWLJkCZMmTZIaeqxYsYLvvvuOVatWMWPGDKftr732Wrv3jz76KO+//z6bN2+uOeHlWM/LJlTC3z4v2WHvO36PlCg/nlUeOPq8bFU9bzQvqRChW83L4RgmI6ybBGX50KwHDJ7ldMztxy9yJKuQcLWS//RMrHQMgUCjUmAsn3AIBFUSXunp6aSnp5Odne3kpF+1apXXx9Hr9ezcuZOZM2dKyxQKBSkpKWzbtq3S/UVR5JdffuHIkSO8+OKLLrfR6XTodBXJswUFBV6Pz1us9bxchUr4e7ZRDpWoAgESXla7MUJjuY1KXZmN3vi8pBgvR83LGi7hcIxfF0HGNtBEw6h3QOk8gWbVum69qpnkO6tuLDOOpuDxec2bN48bb7yR9PR0cnNzuXTpkt3LF3JzczGZTCQkJNgtT0hIIDMz081ekJ+fT1RUFBqNhuHDh7N06VK3WuCCBQuIjY2VXomJ/n8KSQ04XJmN/oiwt7kbZM3Ld6paz6tIZ+TRNbv56WAWoihKwcdODvtyn1epQ24jeFeD3rb5hi3xrszGU1vh15cs/9/0CtRr5XS8s3ml/O+g5f4Z1zep0vMHikD3bvRZ81qxYgXvvfced999dyDG4xXR0dHs2bOHoqIi0tPTSU1NpXXr1k4mJcDMmTNJTU2V3hcUFPhdgCk8FCMsMZi8rizgDpNDH0gZ36iq4rVsw1G+3nOOr/ec4989mvPjgUzSnxhkE6RqQTIbrZqXjxH2bh32kQ4O+5KL8MUkEM3Q9U7o8h+Xx/to+ynMIvRtXZ/2jaO9uNLAoA1w70afhZder6dfv35+OXmDBg1QKpVkZWXZLc/KyqJx48Zu91MoFFLrtW7dunHo0CEWLFjgUnhptVq02sAmA1Y04KD8CV3xZYmiRZhdTl88OcL+8qhqnFdWfpn0/66MSxSWGfk7s0g6nvWh5RwqUXEMrxz2pfaFCK3YhUqIosVBX3AG6rWBf73k8lhlBhNrfs8Aqqd6hCcC3YTDZ7Px3nvv5eOPP/bLyTUaDT169CA9PV1aZjabSU9Pp2/fvl4fx2w22/m1qhvb9CCdC+FyuX4vuzivAOaK1Vb86bAvM5hsglQtf60R9o4lccBLh71D8w0rdilCO9+FQ/8HCjX8+x3Qutaovtl7jkslBprFhZPSsXrDIxxRB7ggoc+aV1lZGW+99RY///wzXbp0cYq2X7JkiU/HS01NZdy4cfTs2ZPevXuTlpZGcXGxNPs4duxYmjVrxoIFCwCLD6tnz560adMGnU7H999/z4cffsjy5ct9vRS/Ydt0NrfIIkSVCgFRFDGLlidy3GUc317zkuO8qgtXMq/UYJI0badQCRf+Tb3RTJnB5FHzdmy+YcXq84opPAbryye1UuZCU9cVXGzDI/57dUtUSp91E7+iCTaz8a+//qJbt24ATo04quLXGTNmDDk5OcyZM4fMzEy6devG+vXrJSd+RkYGCpuAt+LiYh566CHOnDlDeHg4HTp04KOPPmLMmDE+n9tf2IZK/HUmH4AOjaM5fbGEgjLjZYdLmEVZ86pxpLZ2JknTkjQvG7NRFEWnoNiCMoNH4VURYW+vCMRFaNCiZ0bxS0AZtLkern7Y7XF2nrrEgXMFaFUKbu9VM+ERtqgDXFnCZ+G1YcMGvw9i8uTJTJ482eW6jRs32r1/7rnneO655/w+hsvBtuns3jN5AHRNjCO3SEdBmZELRXraXEalIFutW3bY1yxlNv5Lq/AKKxdmZtHy/TjObhaWGWnkwW/uLlQiPlLNU6rVtCMDMbIhwq0rQOFem7I21xjRrank7K9JNAGebaxZvbKWYBthv/d0HgDdmsfRqWksAE99uY98HzrJOGKS47yqnZxCnUtHv85gsgmVsDcbAcr0Zidzs7JYL8fmG1bqn0lnnOonAEqHvw5R7n1YWQVlrN9fHh5Rw456K4E2G2Xh5QesDnuj2cy+crOxa2IcC27rTOOYMI5mFzH5410Yq/gEEu3MRll4BZof9p2n1/M/89Wec07rbCdfrJqXWqlAVf4jsJqOtniqLGHxiVm+UzvhVXAOzbcWa+Qt43AuNB7occyrt5/CaBbplRQvPTRrGutsY9AEqco4Y60qcTK3hGK9iQiNkraNokiICePtcT0JVyv57Z9cnvm/A1WatpfTg6qXw5mFTsus30CZocIsFKjw8dr7vSzLrO4ET5pXoc1sZJTVYW82Wco5l17ikNCGRcYxHstB64wmPg6S8AhbAh2kKgsvP2CdqLA+YTo3i5V+uFc2iyXt9m4IAny0PUPyS/iCbDZWL0YPdelchUpAhd+rVG+SBJ01TstTuIQUJqFVVSRPb14CJ38DdSSLop7EgMpjWZzv/jpPbpGexjFhDOnkPj6yugk6s/HXX3/FaHRWg41GI7/++qtfBhVqOGbsd0uMs3s/pFNjZgztAMCz3x5kQ3llS2+xC5WQZxsDjqfZsTKjyakYITjPOEKF8PKUIuRUy+v077DBEhbE8MWURicBngsSWsMj7urTQtJ2goGgc9gPHjzYZbfs/Px8Bg8e7JdBhRqO1Ua6NI9z2ua+ga0Z3bM5ZhEe+WQ3hzO9TxC3VQRkzSvwuPqMKxoKm51yG8G+jr1VUbY20/BsNto03ijNg88ngmiCzv+BrncQH1keZV/sWnjtzrjE3jP5aJQK7ujTwvuLrAaCLlTCXZ7ehQsXnLpn1xUcW0l1TXR2mAqCwHMjO3PqQgk7Tlxk4nt/MrhDQwrLjOUvA6UGE2OvTmK0Q4yO7POqXjx9xmUGk1NuI9jXsbfWD/PObCzXvLRK+PZxyM+AuJYwfAkIQkVlCTcC0Kp13dSlCQ2igqsnntVsdJV14g+8Fl633XYbYLkJx48fb5cvaDKZ+Ouvv/yW8xhqKGyEeYMoDc3iwl1up1EpWPHfHtz6xhZOXijho+0ZTtus2nLCSXjJDTiqF8/Cy+yU2wgVmleJjU9MEl4e8hutWtkw4y9wYB0oVJbuP2ExgOfms9mFZXy37zwQXI56K0FTVSI21qJNiKJIdHQ04eEVN6hGo+Hqq69m0qRJ/h9hCKC0+RF3bR7nMdMgPlLDJ/ddzdo/TmMWLSkhUVoVZ/NKWfrLUZcli21z82SzMfAYPZg5OqNrh72U36i3FV6W28uT5lVYZqS1cI67Lr5uWTB4FjTvKa33VAr6kx2nMZhEuiXG0dXBzxoMqFXlidk1rXm9++67ACQlJTF16tQ6ayK6wjbo2ZsfUZPYcB5LucJu2dajuSz95ajL7eXuQdWLq8+4IlTCZBOE6tlhHxduETyefF7FJcUsVS9FK5ZBq4FwzWN269014dAbzazecQqo/uYa3qINtsTsuXPnBmIcIY2t+XC5T0BX9dZls7F6qcxsdOWwty0F7Rgq4Wm2sc/xpXRSnKJEFUvErW85pf+4az67/kAm2YU6GkRp+VfnJl5dV3UTdHFeWVlZ3H333TRt2hSVSoVSqbR71UVsQyW6Nvd/dLMc5xUYCsoMUnVUWzyGSriJ85Kqqeor0odiKjMb//4ffbPXAvBr8jyIcRZC0myjg/CyOurv7NPCc0PaGkQtxXkFyWzj+PHjycjI4Omnn6ZJkyaXVSE06Dm+CTRR0LyHx82axoUTH6HmioRoaXbIZzx8jHJVCf9TpDPSf+EvtGoQydeT+9ut86h5GU0eHfZldg57q9noQvMqzISvHgTgXeMQwpq77nkqzTYWVwjA/Wfz2XnqEiqFwF1BFh5hiybYzMbNmzfz22+/SWVxai3711k6Ece1gAd+gzD3GlWUVsXWGdf75Qno2mEva17+5uylUgrKjOw9k4/BZLYL7nQd52X5W2YwO/VtBAefV/kyq9lYarB00JF+H2YzfPkAlORyUtWKhWV3sCTMdZMMq9lYqDNK47RmaQzr3ISEmDDfL76asGpegXLY+3y3JSYm+r19elDS5jqIS4S8U/BtaqWdS8M1yoD1xrN9cMk+L/9Qoq/QhqwFJK0YPZReLTOYJB+krdVhlx7kYDaCfQ4j25bC8Q2gCuf58Gno0Dj1bLQSG66WhGR+qYELRTq+2WtJGB/fr2Vll1mjaIKtDHRaWhozZszg5MmTARhOEBEeZ2krJShh/+ew95OAns6a5OvqtpFnG/2PbdXT7AJ74VWlIFUXidlqpYIordXvVS4sz+6E9PmW/4e9yEGDxc8V46Y9mVIhSNUm8kr0rPnjNHqjmSubxXBVi/jKLrNGkXIbg0V4jRkzho0bN9KmTRuio6OpV6+e3atWkdgbBj9l+f+7qZDrOpQhENjOMIqy2eh3bKvbZhfaCy+XZmO5yDKYRGkCxVVuo6Nws2pUhWUG0BVa0n/MRkgeAVeNlZz5jiWgbbF2zs4p1LN6uyU8YlzfpKD3N0s17Gs6zstKWlpaAIYRxPR/HI5vtGT5f3EPTPwJVP5Pw7D9HeaV6BmS9iuD2zdi4aguDg04ZOHlD0oMtsKrzG5dZZ+xVWtTuAhSLbUpVigIlhpd5/PLLE7776bDpRMQmwg3v4pZtEwcgHMJaFviIjSculDCZztPcy6/jHqRGm7u2tT7i60hgibC3sq4ceMCMY7gRaGE296C5f3g/F6Lyj/k+YCdThRFVm05SVaBjjV/nHYhvOqAv7EaKLXxeTmbjZ4/Y6vgs9V7bHs32ladsPq9oo58Dn+tAUEBo96G8HiKygzStu58XlCRIvTV7rMA3N4r8bJa6VUXQWc2Ahw7dozZs2dzxx13kJ1tKe/yww8/cODAAb8OLmiIaQoj3rD8v+11+OfngJ7ufF6p3Xtbs0Y2G/2Dvdlor3lVVvG2QvNyFWFfkfsoYNG8WgqZdNo9z7LhoBnQ4mqgIvJeq1J4FEbWGUezaPGB/ffq4HbUW5FK4gQozstn4bVp0yY6d+7Mjh07WLduHUVFRQDs3bu3dkffd/gX9CrP3fzqASjyrSZXZdg+xc/bNDs9kVvMW78el97rTWavZnvrxIzwZVDiwWHvqmaa7ccplYJ2ldtosNe84sPgVfXrqE0l0PIaGDhV2sca/+XJZIQKnxfAjckJNHWT+B9sBF2E/YwZM3juuef46aef0GgqAjKvu+46tm/f7tfBBR03PguNOkFxjiVOx0PFzaoiAufyKzSv3/7JcdqmMrPmhe8PcfWCdC4U1Vwj3mDBUYhP/Wwvq3ecsp9tLPR+thEqBJ+rMtAleqPksFcIMPLiu3RTHKdUGWNxPygqNCyp8Ua4Z+9NvE3gczBWj3BHoEvi+Cy89u3bx6233uq0vFGjRuTm5vplUEGLOtxSrkQVDsfSYfuygJzmfF6F5rU7I89pfWU311u/HierQMc7m0/4e2ghh2PI1uc7zzDry/0U2/q8fHXYGzw47G3ivMIzfqV/9moAvmoxA2Kb2x3HXadsRxpFWyaIOjSOpk+r0JnRVwdbnFdcXBznz593Wr57926aNWvml0EFNY06wNAXLP//PA/O7fbLYW2nvUttZsJ2Z1xy2tbbH4PJn33uQxR3n4Gt5pVbpPdpRte6r6sIe0viNtQnn/o/TQHgI+P1/B52jdNx3HXKduSWbk15YFAbXruje9CHR9gSdGWgb7/9dqZPn05mZiaCIGA2m9myZQtTp05l7NixgRhj8NFjAnS8GcwGSwqRzrnbTFVxrCV18kKJ0zbeOu0dm5/WRdx9BrY+L5NZ5EKxxXQURdGlWW57mArNy9ls1JvMmM0mFqtXoCzJJi+qLc8a73ZZFse6zF2AqpUIjYoZwzpwRYKHzrVBSNA14HjhhRfo0KEDiYmJFBUVkZyczMCBA+nXrx+zZ88OxBiDD0GAm1+DmOZw8Th8/6TfDp2ZX1bpNt5OPcshYe6zumyFF1Q47T2lBjnu66oYIcBd/MBg5V5EpZZ9V7+MDo3LyhJSp+xKNK9QJdA17H0WXhqNhpUrV3Ls2DG+/fZbPvroIw4fPsyHH35Yt0riRNSDUSstcTt7P4a/Pr2sw1lvBG8Ek7c/Blnzcv8ZlBrsKz3klDvtvTFxrLONtiactlzL6CScZLrKkkqWP/AZhIQrAdeVJdx1yq4tqG2qSgRi9rvKIr9Fixa0aBG85TiqhZb9YOCTsGmhJXm7eS+o1yrgp/VWDZd9XpWbjYJg0c6sTntvYpIkn5fNMkEQqK828JqwFK1g5H+mHnTuOp7oQksdLpeaV6lN56BaiG2VFYNJRKPyr7/OK+GVmprKs88+S2RkJKmpqR63XbJkiV8GFjIMnAYnNkHGNvhiItzzIygD+2P02mEva15uo1msAqhpbDhn80ols9EbzddakcLRd/606n3aiOc5L9bjScN9/KhQSILJVTVVqXNQLTUbNUpb4WX2e9FErz613bt3YzAYpP/dEUozIX5DqYLbVsKKaywVAzY8DynP+HwYXz45b+NmzLLmVanm1apBpEV4+WQ2Wraxddiz/wtGir9gFgUeNzxEHtHlEfaWW6xIZ8RoMqOyuaGlno211mys+HwCMePolfDasGGDy/9lyolLhFuWwqdjYXMatL7W8vIDkRolxQ7OZW9/CLLPy5PwsgiOlvUj2HwUsgrKzUYvPlurdibdmpdOwv89BsDrphFsNydj3cA2hqtIZ7SrtFvbNS+VUoFCsMTaBWLGMSiKXy9btoykpCTCwsLo06cPv//+u9ttV65cyYABA4iPjyc+Pp6UlBSP21cbySOgx3hAhHX3Q7F/AnZb1nfu0uR9nJdfhhDSuFM+rZpXUvnnW6F5ud7BlcNZEAQwGeCLe0FXwCFVB141jpLWKwQBjUohhVE4Ou3tumXXUtQBLAXtlci3Npz1hnXr1vk0gLVr15KamsqKFSvo06cPaWlpDBkyhCNHjtCoUSOn7Tdu3Mgdd9xBv379CAsL48UXX+TGG2/kwIEDNR8kO2QBnNoGuUfgq4fgzrXOjhE3uNusRb0IDp4vsFsmx3l5xmwWpS7m7ma5rLFaLetHAL7NNloRBGDjQjjzB2hjWRo9A1NRxYy79SuNCVdRajA5Oe2lOK9aajaCxe+lM5oDEi7hleYVGxsrvWJiYkhPT+fPP/+U1u/cuZP09HSpMa0vLFmyhEmTJjFhwgSSk5NZsWIFERERrFq1yuX2q1ev5qGHHqJbt2506NCBt99+G7PZTHp6us/n9juaCEv6kFIL//wIO9687ENaby5b5Ah793yw7SRd5v2PvafzAPeTFtbFSQ0smldOoa48QNV74dWqcDf89rLlzc2vUBhuX2PL6gO2mo62gaqiKNZ6sxECG6jqlfB69913pVdCQgKjR4/mxIkTrFu3jnXr1nH8+HFuv/12GjRo4NPJ9Xo9O3fuJCUlpWJACgUpKSls27bNq2OUlJRgMBjcVnHV6XQUFBTYvQJK4yvhxucs///0NGTu83JH16pXYj1n4eWq6oEr6uJs45yvD1CkMzL1s72Ae7PRSovyz1dvMpNXYvBaeMVTwMgTcwERut8NV46SzEMr1txHq9O+wGbG0VYbqQtmYyAc9j77vFatWsXUqVPtAlKVSiWpqalutSV35ObmYjKZSEhIsFuekJBAZmamV8eYPn06TZs2tROAtixYsMBOc0xMTPRpjFWi9yS4YhiY9Jb0IX1xlQ/lSvPy2mysg5qXFeuVe/oMrHW0rMX+sgrL3PYYFB3evaReSYwhF+q3g2EvAhChsRde1qoTVuFkazZatTCFYJmUqa2oy2O7AuHz8ll4GY1GDh8+7LT88OHDmANQIsYTCxcuZM2aNXz55ZeEhbluATVz5kzy8/Ol1+nTpwM/MEGAEcsgugnk/g3rZ1T5UK5aW8mzjd7j6SOILG+O0Sja8hlnF+gwevEb/q/yZ25Q7sQoqOHf74DGYnqGOwohSfNyNhsrTEZ1rQ4x0gSwjr3PxvaECROYOHEix44do3fv3gDs2LGDhQsXMmHCBJ+O1aBBA5RKJVlZWXbLs7KyaNy4scd9Fy9ezMKFC/n555/p0qWL2+20Wi1arf9rzldKZH1L/ab3b4FdH1haqXVyLiVkxfH327FJDCvH9pBiimzxPsLepxGHPK4+F08C3GrmNYrRciSrkOxCHfUiPZtw7YUMnlZ9BMCGFpO5oUlXaZ1jNVTJbAx3NhulvMZKanmFOoE0G33+5BYvXkzjxo15+eWXpdI4TZo0Ydq0aTzxxBM+HUuj0dCjRw/S09MZOXIkgOR8nzx5stv9XnrpJZ5//nl+/PFHevbs6eslVB+tBsKAVItT95tHoVkPSxNbL2jTMJLm8REcyylyWufuh/DHyYu0tPGR1TXN66eDFQ9B6yyjJ7+f1cxrWF4vK7uwTGpV5ogoQhg6lqqXohUM/GLqxq7GY7jBZhtHn5dVo3KpeZX/H62tvf4uqHDYB4XwUigUPPnkkzz55JOS8zsmJqbKA0hNTWXcuHH07NmT3r17k5aWRnFxsaTFjR07lmbNmrFgwQIAXnzxRebMmcPHH39MUlKS5BuLiooiKiqqyuMIGNfOhBO/WqbTv7gXxn9vicqvBOsPX+HCpHDlP/jj5EX+s8J+kqOuzTZmFlRU5LBeuqeEYKvwsjUbE+OdfYxgaX02W/URVyjOki3GMc1wP2Mcvhsn4VX+V5pttPF5FdYRzSuQZuNlBanGxMRcluACSx/IxYsXM2fOHLp168aePXtYv3695MTPyMiwK364fPly9Ho9//73v2nSpIn0Wrx48WWNI2Ao1ZZuMdoYOL0DNr3ocjNHESW4WQ6ufwi/n7jotKyuaV4am3QUq9DyJL+tPqqEGIvmlVOoc6shDDJt57+q9PL0nwe5QKyTqe/o87I+eKwCqtDObKz9MV5gG6QaJFUlPv/8cz799FMyMjLQ6/V263bt2uXz8SZPnuzWTNy4caPd+5Ds1B2fBDe9Yknc/m0xtB4ESf097mK9MRQOZVcsU+zON5grc6euCS9bQSXNNnrUvOwd9lkFZS4/2yZcYJZpOQBvmW5ii7kz4KwVO/q8BE8Oey+bb4Q6aqvZGAya12uvvcaECRNISEhg9+7d9O7dm/r163P8+HGGDRvm9wHWGjr/G7r9F0QzrLsPSpw1JVskzcvm/rDeHK6ilV0Jr7pmNtoKKuu/nj4Dq6bUKMbq89JJn63V2a7ATJpmGbEUscfcmpeN/5H2d9SKHc1GKxWhEhWal7fNN0IdTQDTg3wWXm+88QZvvfUWS5cuRaPR8OSTT/LTTz8xZcoU8vPz/T7AWsWwF6F+Wyg4C988YjeP7zhdbn1vL7zc+w8iXWledWy20V7zsrzxpHxGWGcbbRz21s/W+jCYrPyKPorDFBHOFMMjGGyMFcfvzK3ZWB6kmplfytNf7Wdp+j+cuWTpEFXbNS9rDa+gCFLNyMigX79+AISHh1NYaKnffvfdd/PJJ5/4d3S1DW0UjHoHFGo4/C386T6ot0LzqrhBrJrXe1tPsqq8M5DVtxOpdX7qu5ppW77xGOv3excAHGqILjQvz2ajvcO+zGDmUonFDRKlVdFTOMyjqi8AeEGYRIZoH0zt5PNyYzZaBdSlEgMfbj/Fyz/9zTd7zwG1twS0FXUwOewbN27MxYsWk6dFixZSr8YTJ07IjU69oWk3uKG8e/KPT0HWQdfbCXZ/APubY/63B9l6LJdu83/im73nUCqcXfuOJtPOUxd5cf1hHvho52VcQPDiykT07LBXlf9VSvmFZ8s1ogRNGa9qlqEURL4w9ec7BjjtX6nPq/yvo2nYPL6iaWyDqBqIQaxGgspsvO666/jmm28AS8Dq448/zg033MCYMWNc9nOUcUGfB6FtChjLLOlDhlIXs43OoRJah5vjzpU7yC81MOUT1wUiHbUOx87QtQ07s9EHzQsqTMczeaWAyDTdMpoJFzhhTmCOYYLrkjgO792bjRWmYdtGUfw6bTCrxvdk6o1XcGMne22utlHhsA+C2ca33npLSgN6+OGHqV+/Plu3buWWW27h/vvv9/sAayUKBYxcDsuvgZxD8OMs6DrHbhNrcJ+dz8vHMrp1b7bRheblQfWyF15hHMsp5uylUm5XbqCffgsGUckUwyMUE060i8N4azaGqZVolAr0JjOzh3dEoRC4rkMC13Wo3YILAtu70SfhZTQaeeGFF7jnnnto3tzS/ff222/n9ttv9/vAaj1RjeDWFfDRbfDnO8TE9gEqgmytybquZhu9xfH3UttFma2g8iXOCypmHKMKjjJX/QEAi4yj2Se2Lj+Om2KEtsdzE2EP8Ml9V5Nfqufa9s416mozUkmcmjYbVSoVL730EkajczMBmSrQ9nroZ+mqnLh5Gk24IK2yagUCtg5711+XYzUDK3WtqoSrCQpP2mekpuLZ3ShaixY9aaqlhAt6jkT2ZKVpuLTe1VEqC1K1pUfL+DqhaTlirWMfFA7766+/nk2bNvl9IHWW656Gpt1R6fJ5RfMGCixfckT5VL3CC80rzk09KG8qJNQmfA1SDXcwG2eoPqGjIoNcMYZvWs1BtLk9XB3H0WFvezwX8yd1kqBKzB42bBgzZsxg37599OjRg8hI+xrrt9xyi98GVydQaWDUO5iWD+Bq4yEeUn7N66ZbKzQvF41NHXFXzM6x+3Ntd4G5CpXwGOdlI2w6F2/latWPAEw1PEByVAJQkRTvSol1lE+2PsnaXObGFwJZSdVn4fXQQw8BrvszCoKAyWRyWi5TCfXbcLz3PNptncpjqi/YZk4mQmMptWL7BNeq3GheEW6EV4DarAcrdhH25bqXpwh7SXgVnKfHntkAvG0cxkZzN3o5Bv2WH0apEKRjOmpeKqVCcszLostCIB32PpuNZrPZ7UsWXFXnTIsRfGm6BpVg5lXNMmKxVF+19Xm51bzC1C6bmgbiB1PTiKLIzlMXKdI5X6/t5XoTKhGuVoHZBOsmodZdYr85iZeMlsknRz+iVRjaa1fOx7T6JV1VA6mLVJiNNdSAQybw6IxmnjZM4JS5Ec2FXLrumQuiiGDzDbnrOPy/g1nc/6Fz4Kmj2Vgb+OzPM4xavo1/L9/qtM5VLFalZuOWNDj5G6I6gimGyeixaLEalcLu87Z+lJXN+Ep+L1l2ARW/WW8bJfuC12ZjaWkp6enp3HTTTYClvLJOVxH0qFQqefbZZ92WY5bxjM5oogjLDfS5Zh6NTv8Auz9ESL5D2sbXdum1UfP6YtcZAA5nFjqtszURvXHYx+Tuhl+et7wZ9hLnv2wI5S3R1AoFWpVC8tVYj2P7HbjSrqzhErLsshAUDTjef/993nyzopXX66+/ztatW9m9eze7d+/mo48+Yvny5X4fYF3B+iXvFdtWVC74YTqKi/9I2/ga5+Xo8xJrQaSXp3gh1xH2rreNpoS4Hx4C0QRXjkLo/l+prhdYGkfY+hitx7N+T+DObLTsI5uNFqyhEjUqvFavXs19991nt+zjjz9mw4YNbNiwgUWLFvHpp5/6fYB1hRuSK2KA3jTdRGniADCUEP7VJLRYkoV9jbCvjZrX7ow8t+vstSxPDnuR59XvoMg/ZSnLfdMrIAhSgjZYhJQrH2OEXTiEC83LRXBxXUZb030bAY4ePUrnzp2l92FhYSgUFbv37t2bgwfdJBnLVIpaqeCR69oCIKKgaNgyiKiPIns/01VrgCpoXmaxTiXLuzIRXV3/v5W/cotyGwhKGLUKwizNkhvaal5uhFf9KI30vysBJZuN9gSF2ZiXl2fn48rJySEpKUl6bzab7dbLXB5h9ZrCyBUA3KNaz3WKXWjdRNh7ojY67R0pM5jYeeqSnZZldmM2thLOM0/1nuXN4KcgsZe0zpqcDZYpflc+xvgIG+HlYizhstloRyDLQHt9NzRv3pz9+/e7Xf/XX39J+Y4yVcNW0ERoVHDFjRh7W5LdF6nfJMZQkT5kW1bF4zFrSazX8ZwiVv563OW6+z7cyajlW1m9I0NaVpHbWHH9GgwsVS8lUtCxU7gS+j9ud5zKzEaFALE2AcGuAlHD5NlGOyqCVP0fRuW18PrXv/7FnDlzKCsrc1pXWlrKvHnzGD58uIs9ZbzFaKNaW+tzidfP44C5JfWFQrrvnI5Qnj6UGB/Bwts6uzyOLQabFKFQtiCve3kTz39/yOW6X//OcbufrfCaplrLlYqTXBSjeCkiFRT2Zrit5qVSCk5mepRWZaeNyWZj5QRFnNdTTz3FxYsXad++PYsWLeLrr7/m66+/5qWXXqJ9+/ZcunSJp556yu8DrEu4+oIFlZYphsmUiFrqZW3jfuW35duavXIK1xbNy1ccQyUGKfYySfU9AE8a7qc03DlJ2rY7uVqpYNKA1nbro8PUUsQ4eA6VUMjJjUCFw75GcxsTEhLYunUrDz74IDNmzJDUckEQuOGGG3jjjTekdmUyVcPVzJhCEDgmNuMZ41heUq/kCdVnbDMnYzD39Cp/LtRnHI/lFPFgFSq/SqESZmhIHovVljCe94w38rO5B31cTH40irH3eaUkJxCmVkhdy500LxfnragGIgNBVAa6VatWrF+/npycHLZv38727dvJyclh/fr1tG7duvIDyHjElXPdKp8+NV1LXqubUAsmXlO/jspQ5NUN4k543bT0N0r0wV/aaMYXf/F3lnPX8MqQfF5mEy+rl9NQKOCQOZEFxjsB12WEbM1GdXnjCNv0rKgwlV2clyvNy2pqyonZFqSSOMGQ2whQr149evfuTe/evalXr56/x1RnaVnfuVtzxU0gcG7AAs6IDWipyOahomVefXlGk0h2YRnvbz1plw+4/2wBa/847Z+BB5AS/eU5etsd/4CByn2UihoeMTyCDstsYYTG2eiIDVdLmpXVPLS1/hw1L1dPD2ucl2w1WtAEg9koE3jG90siu0BHSkfX1TYjY+szRT+ZTzXzud64iT/PfgMkeTym0Wzm7rd3ciTLOZ0mEPlm/sZVYxFvEAHO7qLz4TQA5hvv5qhYMRvuqnCgIAhMua4tR7KKSKofKS2z4qh5eQqVkA1HC5oAmo2y8AoiwtRK5tyc7LT8+VuvJK/EQMv6kewSryDNOIqp6s/o+tdzJAnPclJs4vaYBpPoUnBBaNS4r6r5FSGWwhcTUYpGfjD14hPTdfbr3VQ9nXxdO/vz2/wfrVWhUVYs8ZjbKMsuoMLnZRYtPt2qPoxcIVeVCAHu6tOShwe3ld6/YRrBLqETalMJr6lfR41735UndT0EZBfKKv7WZ7IKLh6nOKwxMwyTcNSEPJVstsOD2egyt1E2G+2w/bz8bTrKwisEMaNgvuYx9OpYuihOMFW1VlrneEN5iq8JhdShqkSqj1BsZqSwCQQFv3VZQL5NYxMrEWrvjA6FB7PRc1UJWXqBfSK7v90UsvAKUXKEBuztYSnncr/qOwYq9gLQrlEUf85OkZz/xhDUvM7llbLvTD7ge7xUCyGL59TvWt4Mmk5mbHeX27kzGx0RfNS8ZLPRHrWN6ixrXjIS2U1T+MB4AwAvq1fQAMsN3yBKS1h5ORdPuY3BmvbYb+Ev3Pz6Zk7mFqP0QQqoMPKa+nWihVJ2ih1gwFTcKZ7emo12Pi9Hh73LqhIKp/3qMoIgBKwUdI0Lr2XLlpGUlERYWBh9+vTh999/d7vtgQMHGDVqFElJSQiCQFpaWvUNNAhRCPC88S4OmxNpKOSzWL0CQbT8QFRe1FEKdof9gXMFKHz4haaqPqeb4hj5YgTTzI+AUuXWNPZW87IzG7X2EfauBNQVCdE0iwtnUPuG3g+8lhOo9mc1KrzWrl1Lamoqc+fOZdeuXXTt2pUhQ4aQnZ3tcvuSkhJat27NwoULady4cTWPNvgQBAEdlvilMlHNtcq93Kb/BrA0gwDP6UHB7vMSEb32eV2j2McDyv8DYLrhPs5RH6gQ0DcmJ/Bz6iBp+yqZjWGVm43RYWo2Tx/Mgtu6eHX8uoA6QLFeNSq8lixZwqRJk5gwYQLJycmsWLGCiIgIVq1a5XL7Xr16sWjRIm6//Xa0Wq3LbeoS1pvnH7E5zxrvBuCesg/g3B7U5b6iF35wncwModFB2xvhVY8CXlEvRyGIfGy8jvXm3k6VVGPC1XaVUsNdBKm6xlbzqtxhD3J0vSMVsV7+/cXVmPDS6/Xs3LmTlJSUisEoFKSkpLBt2za/nUen01FQUGD3qg2Ior1Wstp0PetNvSxhE5/fQ5Rgqa12PKfY7TGC3WwUECqPCxJFFqnfpJGQxz/mZswvF+JWrNfoeJhIr83Giv+jHTUvr44gU1HTq5ZoXrm5uZhMJqdk7oSEBDIzM/12ngULFhAbGyu9EhMT/XbsmkZweDfdMAldRBO4eIyJhW9Uuv+e03lsPZYbqOH5hUonG3e8yfXK3ehEtcV8xqJdiVh8LAfPFZQfR7DTiLx22DvMNtrOnskalncEKkWoxh32gWbmzJnk5+dLr9Ongz+fzxsUCsHJmZ1PFNrRb4OgYEDx/7hF4dwezJYtRy9w58odZBU412gLFjyajef/gp+eBuB5450cFltIq0xmkdFvbuPbv84DkNw0xk7Yu8ptdIVjYramkgYcMs5Is421xWHfoEEDlEolWVlZdsuzsrL86ozXarXExMTYvWoDSoXgFAgZrVVBUn8YOA2A59TvkChkudrdjnN5pQEZoz9wJ7zCKYMvJoJJz0+mq/jAdKPdepNZZM/pPACW3tGdu69uabfe+9nGiv8jNbLZWBWsFTp0tUXz0mg09OjRg/T0dGmZ2WwmPT2dvn371tSwQgaLGeSw0Pp+4JNkRHYhRijlNfUyVB7ShwBufWMr68r7IQYTIu5z4eaoPoTcvyG6CU8a7sOdKFErBW7u2hRBsPefeT/baNknUqNEqRC8ctjL2KOubZoXQGpqKitXruT999/n0KFDPPjggxQXFzNhwgQAxo4dy8yZM6Xt9Xo9e/bsYc+ePej1es6ePcuePXs4evRoTV1CjaEQPNw8ShXr2z9LgRhBd8VRHld9XunxUj/d6+cRes/WY7kMXrzRpf/NVYT9vxTbuUO1ARDgtre4hHtt2tbMC1MrmTy4LZMHtyU6TO12H1dEhVnMzMpCJWSc0QSoFHSNVpUYM2YMOTk5zJkzh8zMTLp168b69eslJ35GRoZde7Vz587RvXtFusfixYtZvHgxgwYNYuPGjdU9/BpFqXDWvGzfmmMTmWG4lzc0r/Gg8v/YbO7MNnOnah2jt9y5cof09+RC+z4IjrKrGTksVL9teTMgFVoNBL5ze+wB7eyDRacOae/T2KyfcZTWcqvYal51oTOTP5CacJj824SjxkviTJ48mcmTJ7tc5yiQkpKSgj6wsrpQCIJHsyVKq+J789V8bNzHnaoNpKmXMVS30KOWEmwI2F+jEhOvapYRI5Swy9yWq66d6WFvC6N7XV5HK+v5o8o1NVtNrq72B/CVCrOxlsR5yVweCsFz3QKrpvCs8W6OmpuSIOTxkvotQiM0tQJb4TVFtY6eir8pEMOZYpgMSs+mX6NoLQPbXV6ajvX00Vpns9FoDv5ijsGAprbFeclcHhaz0SHSy+Z9ZPnNVkqYpfyxqOIG5S7GKv9XreO8HC4W6zh1wRJk20c4xCPKrwCYZZjIGdFSbVbnoR/gqB7NpTSpqmL9RK0PA1unfyDaedVG1FLvRll4yWDxBXlyGFtvNoBDYkteMN4FwCzVx3QUTgV6eH7h6a8P8OepS8RRyCuaZSgEkU+Ng/g/cz8A/ncgk/az17vd/z89Lr8JcoXZ6Oxh8VRuSKYCtRdFAqqCLLxCFIXC2eflGA1uy/umG/nZ1B2tYOkaHYbO5XENJjOf/J7ByVz3aUXVi8iL6pU0FS5yzNyEZ4zjpDX3fei5JVrrhs5FCH3GwWFvi0F22HtFoHo3ysIrRFEKgtNM3LArK2rZR2od45gEnjTcT5YYR1vFOUuclAve23KSmev2ce3ijf4dcBX5r/Jnhij/RC8qLc13Cat8J+CzB/wTK2j9iKNlzavKBKp3oyy8QhTHINWrWsQx56aK5h2uzJyLxPC44SHMosCdql8YptjhtM3z31dUoVi+8Riv//KPfwfuA1cIp5mt+giAF413cEBs5dV+aWO60SvJPy35JLPRheblqkmwjDMVidnybKMMFhPR1kH/n56JdsnGrm42gK3mK1lhuhmAheqVNCPH7TleXH+Yxf/7mwtFrk3MQKJFbzFvBQMbTF1ZZRrq9b7xkRq/jUOK83LxMJAd9t4hJ2bL2OEYKuGYRhPuop29lSXGf7PH3IZYoYQ0zTKUeA4eDES348qYrfqI9ooz5IixTDU8gFjJT/Xe/q1YNb4nj6dcwcB2Dfw2DuunbPswsN6M17St77fz1GZks1HGDqWDw96x1rsgCHRs4jog1YiKRwyTKRTD6aX4m0dUX3o8V3XHBd+o+IO7VT8DkGp4kAvEVrrPdR0acV2HBB5NaefXUjWOEfYAO2Zez/dTBtCleZzfzlOb0cizjTK2KBzSg1wlMD+W0s5pmZXTYgKzDBMBeET5Jb2FQ+SV6P0+Tl8xXzpdHkwLK4w38ZvZu3LKPf3k43Lk5q5NaZ8QTY+W8dKy+EgNyU1DJ1OhpqlID/Kv8Krx9CCZquGYmO0qgblbYpzHY3xj7sdA01/8W/kraZplDJqfCC56HFYXCszsfHU0vYRi9ppb87JxtNttlQqB3XNu4OUfj3BVy3i7yHd/8vDgtnYNf2V8RzYbZexQOsw2umoRFh9RueN6rmEcx82NaSpcZKF6Ja7Shxb/7wiFZYbLGa7El7vPsOCHQy5zVB9Sfk0vDlIkhvGo4WEMHp6th58dSkyYmnkjrmREt2Z+GZtMYFDX1tZnMlVD4VCM0JXZqFEpWDK6q8fjFGPJE9SLSoYp/+BO5S9O26zbdZbnv3PfyMMXHl+7lzc3HWfbsQt2y3sIR3hM9QUATxsmcFJs4mp3ABrHhNlVd5AJbipmG+VQCRnKzUabb89d0b7brqo8RWa/2JqXjLcDMEf1Ae0E58KE1qqk/uKijX8thmJe1SxDJZhZZ+rPl+YBHvftJ8/yhRQa2WyUscWxDPTlKiLvmIaxydSFMMHAa+qlaKkm570o8oL6bZoLuZwyN2KOYXylu8z6V8fAj0vGbwSqe5DssA9RFA7pQd6UJL66dT22H7/ocp2IgicMD/KDYjodFad5SrWaucYJ0vrLbZN2Nq+U297YIr3fk5HH/w5kMT9xFzcpd2AoT/8pIsLl/rufvgGFIBAVpqq8HZpMUCEHqcrY4Zge5M0N7bkCGOQSyxOGBwEYp/qJGxR/Susut2roNQt/IaugIlL/7c0nOPDXH4SlWwoKLjaOZq/oPKsnCHBy4XDiIzXERqhlwRWCWKtKyGajDOBcz8ubm1r0ohDhr+auvGW0lGJ+Sf0WCVg0teM5xX6NA7Ok/7xOmKjjN9OVvGUa7nK7Q/O9TwuSCU408myjjC2OZaBdhUp44osH+7HszqtcrltkHMNf5lbEC0Wkqd9AgeVHt3pHRpXGumLTMadl01VrSFacIo8YUg0Pukz/EQRL0wyZ0EaebZSxQyHgMbfRFbZmo1opMLxLE1b811mAGVAxxTCZYlFLX+VBHlR+A8CiH49UqYfAwh8O272/TrGLe1SWIoILtFPIId7VbrLWVUuQg1Rl7HDUvFxF2Fe2P8DQK5swtJNzk9+TYhPmGCwO+8dVn3OV8DcArWZ+T5mh6l1gGnKJReo3AVhlHMra/GSnbRpEaTj6/DBZ66olBGq2URZeIYpCQaUR9p6w3XzF3T34c3aK0zZfmAfwlakfKsHMa5rXicFSXfX7fee9Pk92QVnFOTHzivoN6guFHDC3ZKHxDqftr2lbnz9mpVx27XmZ4EGebZSxoyqzjY7729IgSsu/nWq+C8w23EOGuSHNhVyeV78DiF5XmTiWU0TvFyo6ot+v/Jb+ygOUiFpLVD/23X9W/PcqVt97tV+rQsjUPHKQqowdVZlttMVVXNji/3Tlp8cH2i0rIoIphkcwiEpuVm7nP8pNmLyUXgu+r/B1dRWO8oTqMwCeMY7lmGifj/jMzcncmOxsvsqEPmqVXBJHxgbHIFVfhZc7q6xFfecg0T1iW5YY/wPAPNX7rPhivceWY1ZU5WOKooTX1K+jFkx8a7qaT03X2m23beZ1jL+mlc9+O5nQoCJUQvRr02hZeIUoTg57n31erre3nZFsUa9CkK0w3cRmUyciBB1L1UvZdSyz0nNo1QpA5Dn1KloqsjkjNuApw0Rs50nv7d+KJrHhPo1dJrRQ25Qr8qfTXhZeIYpS4XuohC3uhJ3t4rTbu0nOVhEFqYaHuChG0UlxioMfPuHx+FuP5vLbP7ncpviNkcqtGEUFU/STKSASgJb1Izi5cDizb3KebZSpXWhs1Hx/xnrJwitEsTjsKySNymfh5Xq57WKNUsHfzw2jd3mV0mzimWp4AICJqh/gb9fdt3dlXOLOt3cQU3KK+er3AEgzjmKXeAUAPVrGs2naYJ/GKxO62JYvMvjRaS8LrxDFsQy0N/4iu+3dal7Oyz+16YH4i/kq3jUOsbz56kEuZWVQrDPabT9+1e+oMfKa+nWihDK2mzvyhmmEtP6LB/tVOlaZ2oNSIUiWgWw2ypQ3nbX1eVW+j62v1J2LzN1hTi4cTqsGFpNvofEODppbQkku+1+/gyvn/mDXYbugzMhU1Vq6KE5wSYziMf1DmOWfWp0mEMnZ8i8qRHFMD/LVYe/r9gDzbukEgA4NjxgmUypqGKDcz33K73h0zW5pu4GKvdyv+g6AJw33kYlcPLCuE4jk7KAQXsuWLSMpKYmwsDD69OnD77//7nH7zz77jA4dOhAWFkbnzp35/vvvq2mkwYOj2eiNKLKtyeWNw97TumNiM54xjgNgqupTxLO7AOg542NeVq8A4APjDfxk7knamG58OLE3AI2itV6MVKa2EYgOQjUuvNauXUtqaipz585l165ddO3alSFDhpCdne1y+61bt3LHHXcwceJEdu/ezciRIxk5ciT79++v5pHXLEpBcDADvSiJY7O9W4e9lxrZ9pnXs9Z0Ld+a+qAWTLymfp38vAssVq+goZDPYXMizxvv4rGUdozs3owB7Rry7SP9+fmJQV4dX6Z2ITXhMNai2cYlS5YwadIkJkyYQHJyMitWrCAiIoJVq1a53P7VV19l6NChTJs2jY4dO/Lss89y1VVX8frrr1fzyGsWhUJw0KQq38c2Mr4qKTi2MWCNY8NYc19fnjLcyxmxAUmKLGLTWnOtci9loppHDI/w7r0DeCzlCmmfK5vFEhOmdnVomVpOrdO89Ho9O3fuJCWlIilYoVCQkpLCtm3bXO6zbds2u+0BhgwZ4nZ7nU5HQUGB3as24FTPywvpZSvsqlKR1FHeRWpUFBDJo/qH7ZY/a7ybc+qW9GvbwOdzyNROAlEWp0aFV25uLiaTiYSEBLvlCQkJZGa6juDOzMz0afsFCxYQGxsrvRITE/0z+BpifL8kmsaGcWfvFsRHarijdyJ39mlBnIcejY+nXEGkRsmLo7oQG66WXu5oHh+OVqWgbSP7BrRXNo0FILy8VE1SA0sE/k6xPYsMlgax35t688i0F9j3zJDLuk6Z2kVKxwRGXdWc+lGV9xL1GrEGOXv2rAiIW7dutVs+bdo0sXfv3i73UavV4scff2y3bNmyZWKjRo1cbl9WVibm5+dLr9OnT4uAmJ+f75+LqAHMZrPP+xhNln3KDEZRZzB53NZgNImleqPLdReLdGKJrmJdxoVi8fD5AvFEdqE47fWPRKNB7/PYZGRsyc/P9+oerdHuQQ0aNECpVJKVlWW3PCsri8aNXVcYaNy4sU/ba7VatNraNcNVFX+V1UzUqiov8KdSKnC3WXyk/ZMz0Sb/8aWH7/J5XDIyVaVGzUaNRkOPHj1IT6+o+WQ2m0lPT6dv374u9+nbt6/d9gA//fST2+1lZGRqJzXetzE1NZVx48bRs2dPevfuTVpaGsXFxUyYYClBPHbsWJo1a8aCBQsAePTRRxk0aBAvv/wyw4cPZ82aNfz555+89dZbNXkZMjIy1UyNC68xY8aQk5PDnDlzyMzMpFu3bqxfv15yymdkZKCw6Wvfr18/Pv74Y2bPns1TTz1Fu3bt+Oqrr7jyyitr6hJkZGRqAEEU/VgdLAQoKCggNjaW/Px8YmJiano4MjIyDnh7j9Z4kKqMjIxMVZCFl4yMTEgiCy8ZGZmQpMYd9tWN1cVXW9KEZGRqG9Z7szJ3fJ0TXoWFhQAhnyYkI1PbKSwsJDY21u36OjfbaDabOXfuHNHR0QFrblpQUEBiYiKnT5+uUzOadfG66+I1Q2CvWxRFCgsLadq0qV2YlCN1TvNSKBQ0b+7YGTowxMTE1KkftJW6eN118ZohcNftSeOyIjvsZWRkQhJZeMnIyIQksvAKAFqtlrlz59a6ahaVURevuy5eMwTHddc5h72MjEztQNa8ZGRkQhJZeMnIyIQksvCSkZEJSWThJSMjE5LIwstPXLx4kbvuuouYmBji4uKYOHEiRUVFHrd/5JFHaN++PeHh4bRo0YIpU6aQn59fjaP2nbrY3dyXa165ciUDBgwgPj6e+Ph4UlJSKv2MghVfv2sra9asQRAERo4cGdgBBrYPSN1h6NChYteuXcXt27eLv/32m9i2bVvxjjvucLv9vn37xNtuu0385ptvxKNHj4rp6eliu3btxFGjRlXjqH1jzZo1okajEVetWiUeOHBAnDRpkhgXFydmZWW53H7Lli2iUqkUX3rpJfHgwYPi7NmzRbVaLe7bt6+aR151fL3mO++8U1y2bJm4e/du8dChQ+L48ePF2NhY8cyZM9U88svD1+u2cuLECbFZs2bigAEDxBEjRgR0jLLw8gMHDx4UAfGPP/6Qlv3www+iIAji2bNnvT7Op59+Kmo0GtFgMARimJdN7969xYcfflh6bzKZxKZNm4oLFixwuf3o0aPF4cOH2y3r06ePeP/99wd0nP7E12t2xGg0itHR0eL7778fqCEGhKpct9FoFPv16ye+/fbb4rhx4wIuvGSz0Q9s27aNuLg4evbsKS1LSUlBoVCwY8cOr49jLXurUgVfyml1dDcPNqpyzY6UlJRgMBioV69eoIbpd6p63fPnz6dRo0ZMnDixOoZZ9xKzA0FmZiaNGjWyW6ZSqahXr57bTt6O5Obm8uyzz3LfffcFYoiXjafu5ocPH3a5j6/dzYONqlyzI9OnT6dp06ZOQjyYqcp1b968mXfeeYc9e/ZUwwgtyJqXB2bMmIEgCB5f3v6IPVFQUMDw4cNJTk7mmWeeufyBywQFCxcuZM2aNXz55ZeEhYXV9HACRmFhIXfffTcrV66kQYMG1XZeWfPywBNPPMH48eM9btO6dWsaN25Mdna23XKj0cjFixfddvK2UlhYyNChQ4mOjubLL79ErVZf7rADQnV0Nw82qnLNVhYvXszChQv5+eef6dKlSyCH6Xd8ve5jx45x8uRJbr75ZmmZ2WwGLBbIkSNHaNOmjf8HGlCPWh3B6rD/888/pWU//vhjpQ77/Px88eqrrxYHDRokFhcXV8dQL4vevXuLkydPlt6bTCaxWbNmHh32N910k92yvn37hpzD3pdrFkVRfPHFF8WYmBhx27Zt1THEgODLdZeWlor79u2ze40YMUK87rrrxH379ok6nS4gY5SFl58YOnSo2L17d3HHjh3i5s2bxXbt2tmFSpw5c0Zs3769uGPHDlEULYKrT58+YufOncWjR4+K58+fl15Go7GmLsMja9asEbVarfjee++JBw8eFO+77z4xLi5OzMzMFEVRFO+++25xxowZ0vZbtmwRVSqVuHjxYvHQoUPi3LlzQzJUwpdrXrhwoajRaMTPP//c7jstLCysqUuoEr5etyPVMdsoCy8/ceHCBfGOO+4Qo6KixJiYGHHChAl2P9gTJ06IgLhhwwZRFEVxw4YNIuDydeLEiZq5CC9YunSp2KJFC1Gj0Yi9e/cWt2/fLq0bNGiQOG7cOLvtP/30U/GKK64QNRqN2KlTJ/G7776r5hFfPr5cc8uWLV1+p3Pnzq3+gV8mvn7XtlSH8JJL4sjIyIQk8myjjIxMSCILLxkZmZBEFl4yMjIhiSy8ZGRkQhJZeMnIyIQksvCSkZEJSWThJSMjE5LIwktGRiYkkYWXTFAwfvx4l1U7hg4dWtNDkwlS5KoSMkHD0KFDeffdd+2WuevIbDAYnCpw6PV6NBqNz+et6n4yNYuseckEDVqtlsaNG9u94uPjARAEgeXLl3PLLbcQGRnJ888/zzPPPEO3bt14++23adWqlVQzKyMjgxEjRhAVFUVMTAyjR4+2K+/ibj+Z0EIWXjIhwzPPPMOtt97Kvn37uOeeewA4evQoX3zxBevWrWPPnj2YzWZGjBjBxYsX2bRpEz/99BPHjx9nzJgxdsdy3E8m9JDNRpmg4dtvvyUqKspu2VNPPcVTTz0FwJ133smECRPs1uv1ej744AMaNmwIwE8//cS+ffs4ceIEiYmJAHzwwQd06tSJP/74g169erncTyb0kIWXTNAwePBgli9fbrfMtnGFbYMTKy1btrQTQIcOHSIxMVESXADJycnExcVx6NAhSXg57icTesjCSyZoiIyMpG3bth7Xe7PM23PJhDayz0umVtGxY0dOnz7N6dOnpWUHDx4kLy+P5OTkGhyZjL+RNS+ZoEGn0zm1RVOpVD51pElJSaFz587cddddpKWlYTQaeeihhxg0aJBLs1MmdJE1L5mgYf369TRp0sTu1b9/f5+OIQgCX3/9NfHx8QwcOJCUlBRat27N2rVrAzRqmZpCLgMtIyMTksial4yMTEgiCy8ZGZmQRBZeMjIyIYksvGRkZEISWXjJyMiEJLLwkpGRCUlk4SUjIxOSyMJLRkYmJJGFl4yMTEgiCy8ZGZmQRBZeMjIyIYksvGRkZEKS/wecPDaMHyd/cAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019186, + "end_time": "2024-03-23T05:06:32.634514", + "exception": false, + "start_time": "2024-03-23T05:06:32.615328", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4690.86179, + "end_time": "2024-03-23T05:06:35.379854", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/tab_ddpm_concat/1/mlu-eval.ipynb", + "output_path": "eval/treatment/tab_ddpm_concat/1/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/tab_ddpm_concat/1", + "path_prefix": "../../../../", + "random_seed": 1, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-03-23T03:48:24.518064", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/tab_ddpm_concat/model.pt b/treatment/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..56ccb27c70b21e6943c6f2649482967b36699048 --- /dev/null +++ b/treatment/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c347640e4c54f8a1ba1618251d95f4d082ce48c81dd0ef4cd266def230b4e13f +size 74520513 diff --git a/treatment/tab_ddpm_concat/params.json b/treatment/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..c0fa7cd6b50ed0a3945a48e1c343a629a41f89c5 --- /dev/null +++ b/treatment/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "NONE", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.1, "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "fixed_role_model": "tab_ddpm_concat", "mse_mag": false, "mse_mag_target": 0.2, "mse_mag_multiply": false, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/tvae/eval.csv b/treatment/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..8077b9d82f9acddb23601a70297fd1014023a709 --- /dev/null +++ b/treatment/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.0,0.02144074160234721,0.004939371281186704,11.521236896514893,0.10742823779582977,2.0802133083343506,0.20339180529117584,0.00015690832515247166,6.522213459014893,0.04292616993188858,5856186.5,0.07028066366910934,0.24378906190395355,1.3198671695136e-05,18.043450355529785 diff --git a/treatment/tvae/history.csv b/treatment/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..8bc40bcf4c9dc44533cd28b17d62527ca16fadbc --- /dev/null +++ b/treatment/tvae/history.csv @@ -0,0 +1,11 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.17688229696199617,51.26139500584347,0.09516493396723655,0.0,0.0,0.0,0.0,0.0,0.17688229696199617,900,225,251.06022119522095,1.1158232053120931,0.2789558013280233,0.16631094145205882,0.012841981946456195,1.4655955983946216,0.0012319917510180377,0.0,0.0,0.0,0.0,0.0,0.012841981946456195,450,113,89.19078063964844,0.789298943713703,0.19820173475477432,0.1320871310418037 +1,0.013371216690704589,0.16934259043439145,0.0010018549940221091,0.0,0.0,0.0,0.0,0.0,0.013371216690704589,900,225,252.06693148612976,1.1202974732716877,0.28007436831792193,0.23207386730445756,0.011202888993763559,1.445202684893326,0.0005740046501933723,0.0,0.0,0.0,0.0,0.0,0.011202888993763559,450,113,89.31532549858093,0.7904011106069109,0.19847850110795762,0.14023152483578008 +2,0.0079494156414229,0.14582720703121596,0.0002692205208845359,0.0,0.0,0.0,0.0,0.0,0.0079494156414229,900,225,252.16153764724731,1.120717945098877,0.2801794862747192,0.24328775689005852,0.010035193292916978,2.6130222165015553,0.0003902935866260455,0.0,0.0,0.0,0.0,0.0,0.010035193292916978,450,113,89.53161573410034,0.7923151834876137,0.19895914607577853,0.1205827880154392 +3,0.006967855532416757,0.1364346780172354,0.0001765495462835635,0.0,0.0,0.0,0.0,0.0,0.006967855532416757,900,225,252.14747762680054,1.1206554561191135,0.2801638640297784,0.24151555318799284,0.009119572052470567,2.3887041220100858,0.0003625035497088226,0.0,0.0,0.0,0.0,0.0,0.009119572052470567,450,113,90.05608463287354,0.796956501175872,0.20012463251749674,0.12774084246738496 +4,0.005291444591320114,0.1013827408555537,0.00012751025595556618,0.0,0.0,0.0,0.0,0.0,0.005291444591320114,900,225,253.6334581375122,1.1272598139444987,0.2818149534861247,0.2422306108309163,0.007817782134829308,1.5934409776440495,0.0002491298151480138,0.0,0.0,0.0,0.0,0.0,0.007817782134829308,450,113,90.28413987159729,0.7989746891291796,0.20063142193688285,0.11994293265180421 +5,0.004564365030828034,0.06386706282769958,0.00010843671611407867,0.0,0.0,0.0,0.0,0.0,0.004564365030828034,900,225,253.86279034614563,1.1282790682050916,0.2820697670512729,0.24002165024809075,0.007527416996059091,1.8274651395106845,0.0001829654243329953,0.0,0.0,0.0,0.0,0.0,0.007527416996059091,450,113,90.3235170841217,0.7993231600364753,0.2007189268536038,0.12028665929384397 +6,0.003292381468013951,0.1837900654759681,2.077242192302188e-05,0.0,0.0,0.0,0.0,0.0,0.003292381468013951,900,225,253.96865129470825,1.1287495613098144,0.2821873903274536,0.24063070929259994,0.008300325454455838,1.7176267483548238,0.0002900127046175885,0.0,0.0,0.0,0.0,0.0,0.008300325454455838,450,113,90.87350010871887,0.8041902664488396,0.2019411113527086,0.11718585676895053 +7,0.0027719743528211893,0.03998318220309714,0.00011375563237977545,0.0,0.0,0.0,0.0,0.0,0.0027719743528211893,900,225,254.43462657928467,1.1308205625745986,0.28270514064364966,0.24318249386896948,0.00691453033982422,1.6120667248579639,0.00015303980521086687,0.0,0.0,0.0,0.0,0.0,0.00691453033982422,450,113,90.7755823135376,0.8033237372879434,0.20172351625230578,0.11331881761575822 +8,0.002773414360858118,0.0599636348949313,0.00013436852597274414,0.0,0.0,0.0,0.0,0.0,0.002773414360858118,900,225,254.54391837120056,1.1313063038720026,0.28282657596800065,0.24226343726946248,0.008801623073795528,2.225178384829181,0.00045555476667698215,0.0,0.0,0.0,0.0,0.0,0.008801623073795528,450,113,90.71591401100159,0.8027956992124035,0.20159092002444798,0.11741957670739665 +9,0.002050752183333265,0.37704243565958634,3.9393338507444054e-05,0.0,0.0,0.0,0.0,0.0,0.002050752183333265,900,225,254.6133759021759,1.1316150040096706,0.28290375100241766,0.25117826619909867,0.0073609895478978,2.00508483488406,0.00018564131555239355,0.0,0.0,0.0,0.0,0.0,0.0073609895478978,450,113,90.77795052528381,0.803344694914016,0.20172877894507515,0.11334262730599173 diff --git a/treatment/tvae/mlu-eval.ipynb b/treatment/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..af9888c9c38fe44d6751437859140c72b292cfa4 --- /dev/null +++ b/treatment/tvae/mlu-eval.ipynb @@ -0,0 +1,2382 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:11.412788Z", + "iopub.status.busy": "2024-03-22T20:48:11.412009Z", + "iopub.status.idle": "2024-03-22T20:48:11.444858Z", + "shell.execute_reply": "2024-03-22T20:48:11.444130Z" + }, + "papermill": { + "duration": 0.047702, + "end_time": "2024-03-22T20:48:11.446787", + "exception": false, + "start_time": "2024-03-22T20:48:11.399085", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:11.471408Z", + "iopub.status.busy": "2024-03-22T20:48:11.471082Z", + "iopub.status.idle": "2024-03-22T20:48:11.477858Z", + "shell.execute_reply": "2024-03-22T20:48:11.476964Z" + }, + "papermill": { + "duration": 0.021267, + "end_time": "2024-03-22T20:48:11.479671", + "exception": false, + "start_time": "2024-03-22T20:48:11.458404", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:11.502999Z", + "iopub.status.busy": "2024-03-22T20:48:11.502754Z", + "iopub.status.idle": "2024-03-22T20:48:11.506604Z", + "shell.execute_reply": "2024-03-22T20:48:11.505801Z" + }, + "papermill": { + "duration": 0.017546, + "end_time": "2024-03-22T20:48:11.508436", + "exception": false, + "start_time": "2024-03-22T20:48:11.490890", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:11.532213Z", + "iopub.status.busy": "2024-03-22T20:48:11.531838Z", + "iopub.status.idle": "2024-03-22T20:48:11.535701Z", + "shell.execute_reply": "2024-03-22T20:48:11.534900Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018189, + "end_time": "2024-03-22T20:48:11.537502", + "exception": false, + "start_time": "2024-03-22T20:48:11.519313", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:11.561296Z", + "iopub.status.busy": "2024-03-22T20:48:11.560761Z", + "iopub.status.idle": "2024-03-22T20:48:11.566109Z", + "shell.execute_reply": "2024-03-22T20:48:11.565357Z" + }, + "papermill": { + "duration": 0.019151, + "end_time": "2024-03-22T20:48:11.567884", + "exception": false, + "start_time": "2024-03-22T20:48:11.548733", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "79120980", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:11.593283Z", + "iopub.status.busy": "2024-03-22T20:48:11.593028Z", + "iopub.status.idle": "2024-03-22T20:48:11.597565Z", + "shell.execute_reply": "2024-03-22T20:48:11.596781Z" + }, + "papermill": { + "duration": 0.019708, + "end_time": "2024-03-22T20:48:11.599379", + "exception": false, + "start_time": "2024-03-22T20:48:11.579671", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"tvae\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 1\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/tvae/1\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.01107, + "end_time": "2024-03-22T20:48:11.621366", + "exception": false, + "start_time": "2024-03-22T20:48:11.610296", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:11.644652Z", + "iopub.status.busy": "2024-03-22T20:48:11.644387Z", + "iopub.status.idle": "2024-03-22T20:48:11.653587Z", + "shell.execute_reply": "2024-03-22T20:48:11.652854Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.02314, + "end_time": "2024-03-22T20:48:11.655471", + "exception": false, + "start_time": "2024-03-22T20:48:11.632331", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/tvae/1\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:11.678643Z", + "iopub.status.busy": "2024-03-22T20:48:11.678377Z", + "iopub.status.idle": "2024-03-22T20:48:13.706685Z", + "shell.execute_reply": "2024-03-22T20:48:13.705776Z" + }, + "papermill": { + "duration": 2.042228, + "end_time": "2024-03-22T20:48:13.708752", + "exception": false, + "start_time": "2024-03-22T20:48:11.666524", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:13.743756Z", + "iopub.status.busy": "2024-03-22T20:48:13.742659Z", + "iopub.status.idle": "2024-03-22T20:48:13.764130Z", + "shell.execute_reply": "2024-03-22T20:48:13.763046Z" + }, + "papermill": { + "duration": 0.043168, + "end_time": "2024-03-22T20:48:13.766975", + "exception": false, + "start_time": "2024-03-22T20:48:13.723807", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:13.796992Z", + "iopub.status.busy": "2024-03-22T20:48:13.796677Z", + "iopub.status.idle": "2024-03-22T20:48:13.804198Z", + "shell.execute_reply": "2024-03-22T20:48:13.803383Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.024704, + "end_time": "2024-03-22T20:48:13.806041", + "exception": false, + "start_time": "2024-03-22T20:48:13.781337", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:13.830398Z", + "iopub.status.busy": "2024-03-22T20:48:13.830120Z", + "iopub.status.idle": "2024-03-22T20:48:13.923361Z", + "shell.execute_reply": "2024-03-22T20:48:13.922602Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.1086, + "end_time": "2024-03-22T20:48:13.925766", + "exception": false, + "start_time": "2024-03-22T20:48:13.817166", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:13.953141Z", + "iopub.status.busy": "2024-03-22T20:48:13.952873Z", + "iopub.status.idle": "2024-03-22T20:48:18.454062Z", + "shell.execute_reply": "2024-03-22T20:48:18.453109Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.51653, + "end_time": "2024-03-22T20:48:18.456595", + "exception": false, + "start_time": "2024-03-22T20:48:13.940065", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-22 20:48:16.147842: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-22 20:48:16.147902: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-22 20:48:16.150193: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:18.481371Z", + "iopub.status.busy": "2024-03-22T20:48:18.480792Z", + "iopub.status.idle": "2024-03-22T20:48:18.487093Z", + "shell.execute_reply": "2024-03-22T20:48:18.486354Z" + }, + "papermill": { + "duration": 0.020697, + "end_time": "2024-03-22T20:48:18.488947", + "exception": false, + "start_time": "2024-03-22T20:48:18.468250", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:18.514488Z", + "iopub.status.busy": "2024-03-22T20:48:18.514225Z", + "iopub.status.idle": "2024-03-22T20:48:39.812076Z", + "shell.execute_reply": "2024-03-22T20:48:39.810964Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 21.313251, + "end_time": "2024-03-22T20:48:39.814379", + "exception": false, + "start_time": "2024-03-22T20:48:18.501128", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-22T20:48:39.842878Z", + "iopub.status.busy": "2024-03-22T20:48:39.842000Z", + "iopub.status.idle": "2024-03-22T20:48:39.848590Z", + "shell.execute_reply": "2024-03-22T20:48:39.847872Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022871, + "end_time": "2024-03-22T20:48:39.850500", + "exception": false, + "start_time": "2024-03-22T20:48:39.827629", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:39.875777Z", + "iopub.status.busy": "2024-03-22T20:48:39.875505Z", + "iopub.status.idle": "2024-03-22T20:48:39.879959Z", + "shell.execute_reply": "2024-03-22T20:48:39.879172Z" + }, + "papermill": { + "duration": 0.019262, + "end_time": "2024-03-22T20:48:39.881864", + "exception": false, + "start_time": "2024-03-22T20:48:39.862602", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:39.907398Z", + "iopub.status.busy": "2024-03-22T20:48:39.906902Z", + "iopub.status.idle": "2024-03-22T20:48:40.590441Z", + "shell.execute_reply": "2024-03-22T20:48:40.589566Z" + }, + "papermill": { + "duration": 0.698689, + "end_time": "2024-03-22T20:48:40.592550", + "exception": false, + "start_time": "2024-03-22T20:48:39.893861", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tvae/all inf False\n", + "../../../../ml-utility-loss/aug_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_bs_test/tvae/all inf False\n", + "../../../../ml-utility-loss/bs_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_synth_test/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:40.621097Z", + "iopub.status.busy": "2024-03-22T20:48:40.620618Z", + "iopub.status.idle": "2024-03-22T20:48:40.928630Z", + "shell.execute_reply": "2024-03-22T20:48:40.927771Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.324286, + "end_time": "2024-03-22T20:48:40.930496", + "exception": false, + "start_time": "2024-03-22T20:48:40.606210", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.8,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.1,\n", + " 'loss_balancer_beta': 0.73,\n", + " 'loss_balancer_r': 0.94,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 220,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': False, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:40.959223Z", + "iopub.status.busy": "2024-03-22T20:48:40.958924Z", + "iopub.status.idle": "2024-03-22T20:48:41.057038Z", + "shell.execute_reply": "2024-03-22T20:48:41.056322Z" + }, + "papermill": { + "duration": 0.11493, + "end_time": "2024-03-22T20:48:41.058832", + "exception": false, + "start_time": "2024-03-22T20:48:40.943902", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/treatment [400, 0]\n", + "Caching in ../../../../treatment/_cache_aug_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/treatment [0, 200]\n", + "Caching in ../../../../treatment/_cache_bs_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/treatment [100, 0]\n", + "Caching in ../../../../treatment/_cache_bs_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/treatment [0, 50]\n", + "Caching in ../../../../treatment/_cache_synth/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/treatment [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-22T20:48:41.086784Z", + "iopub.status.busy": "2024-03-22T20:48:41.086494Z", + "iopub.status.idle": "2024-03-22T20:48:41.578432Z", + "shell.execute_reply": "2024-03-22T20:48:41.577566Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.508296, + "end_time": "2024-03-22T20:48:41.580414", + "exception": false, + "start_time": "2024-03-22T20:48:41.072118", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:41.609381Z", + "iopub.status.busy": "2024-03-22T20:48:41.609004Z", + "iopub.status.idle": "2024-03-22T20:48:41.613503Z", + "shell.execute_reply": "2024-03-22T20:48:41.612756Z" + }, + "papermill": { + "duration": 0.020833, + "end_time": "2024-03-22T20:48:41.615307", + "exception": false, + "start_time": "2024-03-22T20:48:41.594474", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:41.642323Z", + "iopub.status.busy": "2024-03-22T20:48:41.642055Z", + "iopub.status.idle": "2024-03-22T20:48:41.648937Z", + "shell.execute_reply": "2024-03-22T20:48:41.648136Z" + }, + "papermill": { + "duration": 0.022939, + "end_time": "2024-03-22T20:48:41.650955", + "exception": false, + "start_time": "2024-03-22T20:48:41.628016", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18701313" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:41.678226Z", + "iopub.status.busy": "2024-03-22T20:48:41.677597Z", + "iopub.status.idle": "2024-03-22T20:48:41.773743Z", + "shell.execute_reply": "2024-03-22T20:48:41.772930Z" + }, + "papermill": { + "duration": 0.111782, + "end_time": "2024-03-22T20:48:41.775573", + "exception": false, + "start_time": "2024-03-22T20:48:41.663791", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 95] --\n", + "├─Adapter: 1-1 [2, 2648, 95] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 98,304\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 95] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-54 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 16, 512] (recursive)\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-43 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-45 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-46 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-48 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-49 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-18 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-51 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-52 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-53 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-19 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-54 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-55 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardsigmoid: 7-108 [2, 16, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-20 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-57 [2, 16, 512] --\n", + "│ │ │ │ │ └─Linear: 6-58 [2, 16, 512] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,701,313\n", + "Trainable params: 18,701,313\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 74.05\n", + "========================================================================================================================\n", + "Input size (MB): 2.51\n", + "Forward/backward pass size (MB): 1079.48\n", + "Params size (MB): 74.81\n", + "Estimated Total Size (MB): 1156.80\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T20:48:41.806222Z", + "iopub.status.busy": "2024-03-22T20:48:41.805957Z", + "iopub.status.idle": "2024-03-22T21:54:41.702583Z", + "shell.execute_reply": "2024-03-22T21:54:41.701031Z" + }, + "papermill": { + "duration": 3959.91573, + "end_time": "2024-03-22T21:54:41.705856", + "exception": false, + "start_time": "2024-03-22T20:48:41.790126", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.17688229696199617, 'avg_role_model_std_loss': 51.26139500584347, 'avg_role_model_mean_pred_loss': 0.09516493396723655, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.17688229696199617, 'n_size': 900, 'n_batch': 225, 'duration': 251.06022119522095, 'duration_batch': 1.1158232053120931, 'duration_size': 0.2789558013280233, 'avg_pred_std': 0.16631094145205882}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012841981946456195, 'avg_role_model_std_loss': 1.4655955983946216, 'avg_role_model_mean_pred_loss': 0.0012319917510180377, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012841981946456195, 'n_size': 450, 'n_batch': 113, 'duration': 89.19078063964844, 'duration_batch': 0.789298943713703, 'duration_size': 0.19820173475477432, 'avg_pred_std': 0.1320871310418037}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013371216690704589, 'avg_role_model_std_loss': 0.16934259043439145, 'avg_role_model_mean_pred_loss': 0.0010018549940221091, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013371216690704589, 'n_size': 900, 'n_batch': 225, 'duration': 252.06693148612976, 'duration_batch': 1.1202974732716877, 'duration_size': 0.28007436831792193, 'avg_pred_std': 0.23207386730445756}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011202888993763559, 'avg_role_model_std_loss': 1.445202684893326, 'avg_role_model_mean_pred_loss': 0.0005740046501933723, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011202888993763559, 'n_size': 450, 'n_batch': 113, 'duration': 89.31532549858093, 'duration_batch': 0.7904011106069109, 'duration_size': 0.19847850110795762, 'avg_pred_std': 0.14023152483578008}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0079494156414229, 'avg_role_model_std_loss': 0.14582720703121596, 'avg_role_model_mean_pred_loss': 0.0002692205208845359, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0079494156414229, 'n_size': 900, 'n_batch': 225, 'duration': 252.16153764724731, 'duration_batch': 1.120717945098877, 'duration_size': 0.2801794862747192, 'avg_pred_std': 0.24328775689005852}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010035193292916978, 'avg_role_model_std_loss': 2.6130222165015553, 'avg_role_model_mean_pred_loss': 0.0003902935866260455, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010035193292916978, 'n_size': 450, 'n_batch': 113, 'duration': 89.53161573410034, 'duration_batch': 0.7923151834876137, 'duration_size': 0.19895914607577853, 'avg_pred_std': 0.1205827880154392}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006967855532416757, 'avg_role_model_std_loss': 0.1364346780172354, 'avg_role_model_mean_pred_loss': 0.0001765495462835635, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006967855532416757, 'n_size': 900, 'n_batch': 225, 'duration': 252.14747762680054, 'duration_batch': 1.1206554561191135, 'duration_size': 0.2801638640297784, 'avg_pred_std': 0.24151555318799284}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009119572052470567, 'avg_role_model_std_loss': 2.3887041220100858, 'avg_role_model_mean_pred_loss': 0.0003625035497088226, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009119572052470567, 'n_size': 450, 'n_batch': 113, 'duration': 90.05608463287354, 'duration_batch': 0.796956501175872, 'duration_size': 0.20012463251749674, 'avg_pred_std': 0.12774084246738496}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005291444591320114, 'avg_role_model_std_loss': 0.1013827408555537, 'avg_role_model_mean_pred_loss': 0.00012751025595556618, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005291444591320114, 'n_size': 900, 'n_batch': 225, 'duration': 253.6334581375122, 'duration_batch': 1.1272598139444987, 'duration_size': 0.2818149534861247, 'avg_pred_std': 0.2422306108309163}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007817782134829308, 'avg_role_model_std_loss': 1.5934409776440495, 'avg_role_model_mean_pred_loss': 0.0002491298151480138, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007817782134829308, 'n_size': 450, 'n_batch': 113, 'duration': 90.28413987159729, 'duration_batch': 0.7989746891291796, 'duration_size': 0.20063142193688285, 'avg_pred_std': 0.11994293265180421}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004564365030828034, 'avg_role_model_std_loss': 0.06386706282769958, 'avg_role_model_mean_pred_loss': 0.00010843671611407867, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004564365030828034, 'n_size': 900, 'n_batch': 225, 'duration': 253.86279034614563, 'duration_batch': 1.1282790682050916, 'duration_size': 0.2820697670512729, 'avg_pred_std': 0.24002165024809075}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007527416996059091, 'avg_role_model_std_loss': 1.8274651395106845, 'avg_role_model_mean_pred_loss': 0.0001829654243329953, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007527416996059091, 'n_size': 450, 'n_batch': 113, 'duration': 90.3235170841217, 'duration_batch': 0.7993231600364753, 'duration_size': 0.2007189268536038, 'avg_pred_std': 0.12028665929384397}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003292381468013951, 'avg_role_model_std_loss': 0.1837900654759681, 'avg_role_model_mean_pred_loss': 2.077242192302188e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003292381468013951, 'n_size': 900, 'n_batch': 225, 'duration': 253.96865129470825, 'duration_batch': 1.1287495613098144, 'duration_size': 0.2821873903274536, 'avg_pred_std': 0.24063070929259994}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008300325454455838, 'avg_role_model_std_loss': 1.7176267483548238, 'avg_role_model_mean_pred_loss': 0.0002900127046175885, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008300325454455838, 'n_size': 450, 'n_batch': 113, 'duration': 90.87350010871887, 'duration_batch': 0.8041902664488396, 'duration_size': 0.2019411113527086, 'avg_pred_std': 0.11718585676895053}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0027719743528211893, 'avg_role_model_std_loss': 0.03998318220309714, 'avg_role_model_mean_pred_loss': 0.00011375563237977545, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027719743528211893, 'n_size': 900, 'n_batch': 225, 'duration': 254.43462657928467, 'duration_batch': 1.1308205625745986, 'duration_size': 0.28270514064364966, 'avg_pred_std': 0.24318249386896948}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00691453033982422, 'avg_role_model_std_loss': 1.6120667248579639, 'avg_role_model_mean_pred_loss': 0.00015303980521086687, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00691453033982422, 'n_size': 450, 'n_batch': 113, 'duration': 90.7755823135376, 'duration_batch': 0.8033237372879434, 'duration_size': 0.20172351625230578, 'avg_pred_std': 0.11331881761575822}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002773414360858118, 'avg_role_model_std_loss': 0.0599636348949313, 'avg_role_model_mean_pred_loss': 0.00013436852597274414, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002773414360858118, 'n_size': 900, 'n_batch': 225, 'duration': 254.54391837120056, 'duration_batch': 1.1313063038720026, 'duration_size': 0.28282657596800065, 'avg_pred_std': 0.24226343726946248}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008801623073795528, 'avg_role_model_std_loss': 2.225178384829181, 'avg_role_model_mean_pred_loss': 0.00045555476667698215, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008801623073795528, 'n_size': 450, 'n_batch': 113, 'duration': 90.71591401100159, 'duration_batch': 0.8027956992124035, 'duration_size': 0.20159092002444798, 'avg_pred_std': 0.11741957670739665}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002050752183333265, 'avg_role_model_std_loss': 0.37704243565958634, 'avg_role_model_mean_pred_loss': 3.9393338507444054e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002050752183333265, 'n_size': 900, 'n_batch': 225, 'duration': 254.6133759021759, 'duration_batch': 1.1316150040096706, 'duration_size': 0.28290375100241766, 'avg_pred_std': 0.25117826619909867}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0073609895478978, 'avg_role_model_std_loss': 2.00508483488406, 'avg_role_model_mean_pred_loss': 0.00018564131555239355, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0073609895478978, 'n_size': 450, 'n_batch': 113, 'duration': 90.77795052528381, 'duration_batch': 0.803344694914016, 'duration_size': 0.20172877894507515, 'avg_pred_std': 0.11334262730599173}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014729692572422613, 'avg_role_model_std_loss': 0.03864952014059409, 'avg_role_model_mean_pred_loss': 8.544724943504433e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014729692572422613, 'n_size': 900, 'n_batch': 225, 'duration': 254.37208485603333, 'duration_batch': 1.1305425993601481, 'duration_size': 0.28263564984003703, 'avg_pred_std': 0.24201426328884232}\n", + "Time out: 3698.5479187965393/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.004939371294403069, 'avg_g_mag_loss': 0.012724290520978721, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.449154615402222, 'grad_duration': 11.607890367507935, 'total_duration': 18.057044982910156, 'pred_std': 0.24378906190395355, 'std_loss': 1.3198671695136e-05, 'mean_pred_loss': 0.00015690828149672598, 'pred_rmse': 0.07028066366910934, 'pred_mae': 0.04292616993188858, 'pred_mape': 5856186.5, 'grad_rmse': 0.20339177548885345, 'grad_mae': 0.10742825269699097, 'grad_mape': 2.0802133083343506}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.004939371294403069, 'avg_g_mag_loss': 0.012724290520978721, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 6.449154615402222, 'avg_grad_duration': 11.607890367507935, 'avg_total_duration': 18.057044982910156, 'avg_pred_std': 0.24378906190395355, 'avg_std_loss': 1.3198671695136e-05, 'avg_mean_pred_loss': 0.00015690828149672598}, 'min_metrics': {'avg_loss': 0.004939371294403069, 'avg_g_mag_loss': 0.012724290520978721, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.449154615402222, 'grad_duration': 11.607890367507935, 'total_duration': 18.057044982910156, 'pred_std': 0.24378906190395355, 'std_loss': 1.3198671695136e-05, 'mean_pred_loss': 0.00015690828149672598, 'pred_rmse': 0.07028066366910934, 'pred_mae': 0.04292616993188858, 'pred_mape': 5856186.5, 'grad_rmse': 0.20339177548885345, 'grad_mae': 0.10742825269699097, 'grad_mape': 2.0802133083343506}, 'model_metrics': {'tvae': {'avg_loss': 0.004939371294403069, 'avg_g_mag_loss': 0.012724290520978721, 'avg_g_cos_loss': 0.0, 'pred_duration': 6.449154615402222, 'grad_duration': 11.607890367507935, 'total_duration': 18.057044982910156, 'pred_std': 0.24378906190395355, 'std_loss': 1.3198671695136e-05, 'mean_pred_loss': 0.00015690828149672598, 'pred_rmse': 0.07028066366910934, 'pred_mae': 0.04292616993188858, 'pred_mape': 5856186.5, 'grad_rmse': 0.20339177548885345, 'grad_mae': 0.10742825269699097, 'grad_mape': 2.0802133083343506}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:54:41.751282Z", + "iopub.status.busy": "2024-03-22T21:54:41.750377Z", + "iopub.status.idle": "2024-03-22T21:54:41.762747Z", + "shell.execute_reply": "2024-03-22T21:54:41.761795Z" + }, + "papermill": { + "duration": 0.032728, + "end_time": "2024-03-22T21:54:41.765118", + "exception": false, + "start_time": "2024-03-22T21:54:41.732390", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:54:41.799855Z", + "iopub.status.busy": "2024-03-22T21:54:41.799162Z", + "iopub.status.idle": "2024-03-22T21:54:41.946126Z", + "shell.execute_reply": "2024-03-22T21:54:41.945202Z" + }, + "papermill": { + "duration": 0.166809, + "end_time": "2024-03-22T21:54:41.948609", + "exception": false, + "start_time": "2024-03-22T21:54:41.781800", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:54:41.985411Z", + "iopub.status.busy": "2024-03-22T21:54:41.985028Z", + "iopub.status.idle": "2024-03-22T21:54:42.285291Z", + "shell.execute_reply": "2024-03-22T21:54:42.284359Z" + }, + "papermill": { + "duration": 0.322212, + "end_time": "2024-03-22T21:54:42.287884", + "exception": false, + "start_time": "2024-03-22T21:54:41.965672", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA78ElEQVR4nO3de1xUdf4/8Nfch+twk8sgF+8aXjIRIjf5lnxDswvqZrkUZKZWYOnkpuyu0uVXY2lKqZvVlta3WMn9arth66aY+jXHSyCloaSGd0BBmeE6w8x8fn8MHBgZYGaYC8j7+XicBzOf8znnvAeZt+d8zud8PjzGGAMhhPQRfHcHQAghtqCkRQjpUyhpEUL6FEpahJA+hZIWIaRPoaRFCOlTKGkRQvoUobsDcBWj0YirV6/Cx8cHPB7P3eEQQm7BGENtbS3kcjn4/M7Pp/pN0rp69SoiIiLcHQYhpBuXLl3CwIEDO13fb5KWj48PANMvxNfX183REEJupdFoEBERwX1XO9NvklbrJaGvry8lLUJ6se6ab6ghnhDSp1DSIoT0KZS0CCF9Sr9p0yI9YzAY0Nzc7O4wSB8mEokgEAh6vB9KWqRLjDFUVFSgpqbG3aGQ24Cfnx9CQ0N71FeSkhbpUmvCCg4OhqenJ3XMJXZhjKGhoQHXrl0DAISFhfVoZzbbsGEDi4qKYhKJhMXFxbEjR450WvfkyZNs5syZLCoqigFg69at61Cndd2tywsvvMDVSUxM7LB+4cKFVsesVqsZAKZWq7uu16hjR36rZqpzVVbv+3al1+tZSUkJq6qi3wVxjKqqKlZSUsL0en2HddZ+R21uiM/Ly4NCoUB2djaKioowbtw4JCcncxn0Vg0NDRg8eDBWrVqF0NBQi3WOHTuG8vJybtm9ezcA4LHHHjOrN3/+fLN677zzjq3hd+vH8zcw+0MV3sgvcfi++5rWNixPT083R0JuF61/Sz1pH7U5aa1duxbz58/H3Llzcccdd2DTpk3w9PTEp59+arH+xIkTsXr1ajzxxBOQSCQW6wwYMAChoaHckp+fjyFDhiAxMdGsnqenp1k9Z3QSlft5AACu1jQ6fN99FV0SEkdxxN+STUlLp9OhsLAQSUlJbTvg85GUlASVStXjYFqP8cUXX+CZZ57p8AG//PJLBAUFYfTo0cjKykJDQ0On+9FqtdBoNGaLNcJbktbNhmY06PT2fxBCiFPY1BBfVVUFg8GAkJAQs/KQkBCcPn3aIQF9/fXXqKmpwdNPP21W/oc//AFRUVGQy+X4+eefsWzZMpSWlmL79u0W96NUKvHaa6/ZfHwfqQg+UiFqm/S4crMRw0K6fg6KEOJava5z6SeffIJp06ZBLpeblS9YsADJyckYM2YMUlNT8fnnn2PHjh04d+6cxf1kZWVBrVZzy6VLl6yOofVs6wpdIhI78Xg8fP311+4Ow6FeffVV3Hnnne4Ow7akFRQUBIFAgMrKSrPyysrKThvZbXHhwgXs2bMHzz77bLd14+PjAQBnz561uF4ikXAPR9v6kPRAf0papO/bsmUL/Pz8HLa/pUuXoqCgwGH7s5dNSUssFmPChAlmgRuNRhQUFCAhIaHHwWzevBnBwcGYPn16t3WLi4sB9LC/RyeoMZ70Jzqdzqp63t7eCAwMdHI03bP58lChUODjjz/GZ599hlOnTuH5559HfX095s6dCwBIS0tDVlYWV1+n06G4uBjFxcXQ6XS4cuUKiouLO5whGY1GbN68Genp6RAKzZvazp07hzfeeAOFhYU4f/48/vWvfyEtLQ2TJ0/G2LFj7fncXeIuD29S0mqPMYYGnd4tC7NxIvRdu3bhd7/7Hfz8/BAYGIiHHnqIa0q45557sGzZMrP6169fh0gkwoEDBwAA5eXlmD59Ojw8PDBo0CDk5uYiOjoaOTk5dv3uTpw4gfvvvx8eHh4IDAzEggULUFdXx63ft28f4uLi4OXlBT8/P0yaNAkXLlwAAPz000+477774OPjA19fX0yYMAE//vhjl8fbt28f5s6dC7VaDR6PBx6Ph1dffRUAEB0djTfeeANpaWnw9fXFggULAADLli3D8OHD4enpicGDB2PFihVmXRNuvTx8+umnkZKSgjVr1iAsLAyBgYHIyMhw+uNeNveIf/zxx3H9+nWsXLkSFRUVuPPOO7Fr1y6ucf7ixYtmQ6VevXoV48eP596vWbMGa9asQWJiIvbt28eV79mzBxcvXsQzzzzT4ZhisRh79uxBTk4O6uvrERERgVmzZuEvf/mLreFbRU5tWhY1Nhtwx8r/uOXYJa8nw1Ns/Z9rfX09FAoFxo4di7q6OqxcuRIzZsxAcXExUlNT8c4772DVqlXcHeq8vDzI5XLce++9AEz/+VZVVWHfvn0QiURQKBSd9kW0Jpbk5GQkJCTg2LFjuHbtGp599llkZmZiy5Yt0Ov1SElJwfz58/H3v/8dOp0OR48e5WJLTU3F+PHj8cEHH0AgEKC4uBgikajLY95zzz3IycnBypUrUVpaCsB0ptRqzZo1WLlyJbKzs7kyHx8fbNmyBXK5HCdOnMD8+fPh4+ODV155pdPjfP/99wgLC8P333+Ps2fP4vHHH8edd96J+fPn2/W7soZdj/FkZmYiMzPT4rr2iQgwZXVr/pd84IEHOq0XERGB/fv32xynvcL9Wy8Pm1x2TOJYs2bNMnv/6aefYsCAASgpKcHs2bOxePFiHDx4kEtSubm5mDNnDng8Hk6fPo09e/bg2LFjiI2NBQD87W9/w7Bhw+yKJTc3F01NTfj888/h5eUFANiwYQMefvhhvP322xCJRFCr1XjooYcwZMgQAMCoUaO47S9evIg//vGPGDlyJABYFYdYLIZMJgOPx7PY3nz//ffj5ZdfNitrfxIQHR2NpUuXYuvWrV0mLX9/f2zYsAECgQAjR47E9OnTUVBQ0PuS1u1uYMuZVoWmCXqDEUJBr7vJ6hYeIgFKXk9227FtcebMGaxcuRJHjhxBVVUVjEYjAFMCGD16NB544AF8+eWXuPfee1FWVgaVSoUPP/wQAFBaWgqhUIi77rqL29/QoUPh7+9vV+ynTp3CuHHjuIQFAJMmTYLRaERpaSkmT56Mp59+GsnJyfjv//5vJCUlYfbs2Vx7rUKhwLPPPov/+Z//QVJSEh577DEuudmrNRm3l5eXh/fffx/nzp1DXV0d9Hp9tzewYmJizEZuCAsLw4kTJ3oUW3fo22hBkLcEYgEfBiNDZa3W3eH0GjweD55ioVsWW3tSP/zww7hx4wY+/vhjHDlyBEeOHAHQ1uicmpqKf/zjH2hubkZubi7GjBmDMWPGOPx3Zq3NmzdDpVLhnnvuQV5eHoYPH47Dhw8DMLUl/fLLL5g+fTr27t2LO+64Azt27OjR8donUABQqVRITU3Fgw8+iPz8fBw/fhx//vOfu22kv/Uylcfjcf9BOAslLQv4fB7C/KQAqDG+L6qurkZpaSn+8pe/YMqUKRg1ahRu3rxpVufRRx9FU1MTdu3ahdzcXKSmpnLrRowYAb1ej+PHj3NlZ8+e7bAPa40aNQo//fQT6uvrubIffvgBfD4fI0aM4MrGjx+PrKwsHDp0CKNHj0Zubi63bvjw4ViyZAm+++47zJw5E5s3b+72uGKxGAaDwaoYDx06hKioKPz5z39GbGwshg0bxt0I6G0oaXWirYNp548Kkd7J398fgYGB+Oijj3D27Fns3bsXCoXCrI6XlxdSUlKwYsUKnDp1CnPmzOHWjRw5EklJSViwYAGOHj2K48ePY8GCBfDw8LDr2bnU1FRIpVKkp6fj5MmT+P7777Fo0SI89dRTCAkJQVlZGbKysqBSqXDhwgV89913OHPmDEaNGoXGxkZkZmZi3759uHDhAn744QccO3bMrM2rM9HR0airq0NBQQGqqqq6fOxt2LBhuHjxIrZu3Ypz587h/fff7/HZnLNQ0upEW18taozva/h8PrZu3YrCwkKMHj0aS5YswerVqzvUS01NxU8//YR7770XkZGRZus+//xzhISEYPLkyZgxYwZ3J00qldocj6enJ/7zn//gxo0bmDhxIn7/+99jypQp2LBhA7f+9OnTmDVrFoYPH44FCxYgIyMDCxcuhEAgQHV1NdLS0jB8+HDMnj0b06ZNs+oRtXvuuQfPPfccHn/8cQwYMKDLUVEeeeQRLFmyBJmZmbjzzjtx6NAhrFixwubP6go8ZmsHmD5Ko9FAJpNBrVZb1Tt+3e5f8V7BGcyJi4RypvvaOtypqakJZWVlGDRokF1f1tvJ5cuXERERgT179mDKlCnuDqfP6upvytrvKN097AQ9f9i/7d27F3V1dRgzZgzKy8vxyiuvIDo6GpMnT3Z3aP0eXR52oq2vFiWt/qi5uRl/+tOfEBMTgxkzZmDAgAFcR9Mvv/wS3t7eFpeYmBiXxTht2rRO43jrrbdcFoer0ZlWJ9o/ysMYo4Hw+pnk5GQkJ1vuk/bII49wD+zfqrue6o70t7/9DY2Nlv9TDQgIcFkcrkZJqxOhMtP1dmOzATUNzfD3Ers5ItJb+Pj4wMfH/eOshYeHuzsEt6DLw05IRQIM8DEND03tWoT0HpS0utDa7eEydTAlpNegpNWFgTSuFiG9DiWtLoTTCKaE9DqUtLogb2mMpzMtQnoPSlpdCPc3TSxJZ1rEVrfjxBa9BSWtLshppAfShzl6YgvANMgnj8dDTU2NQ/drC0paXRjoZzrTqq7XoanZuiE+CCHORUmrC74eQnhLTP1v6RIRAGOArt49C01s4bCJLbRaLZYuXYrw8HB4eXkhPj7ebJj0Cxcu4OGHH4a/vz+8vLwQExODb7/9FufPn8d9990HwDT8D4/H6zCpsivY1SN+48aNWL16NSoqKjBu3DisX78ecXFxFuv+8ssvWLlyJQoLC3HhwgWsW7cOixcvNqvz6quvdhhqY8SIEWazVjc1NeHll1/G1q1bodVqkZycjL/+9a8dZrt2JB6PB7mfFL9W1uHKzUYMGeDd/Ua3s+YG4C159/Wc4U9XAbFX9/Va0MQWnU9skZmZiZKSEmzduhVyuRw7duzA1KlTceLECQwbNgwZGRnQ6XQ4cOAAvLy8UFJSAm9vb0REROB///d/MWvWLJSWlsLX1xceHh52/U56wuaklZeXB4VCgU2bNiE+Ph45OTlITk5GaWkpgoODO9RvaGjA4MGD8dhjj2HJkiWd7jcmJgZ79uxpC+yWacSWLFmCnTt3Ytu2bZDJZMjMzMTMmTPxww8/2PoRbBLu54FfK+voDmIfQxNbWJ7Y4uLFi9i8eTMuXrzIzeK+dOlS7Nq1C5s3b8Zbb72FixcvYtasWdzw04MHD+a2b32mMTg42OHtZdayOWmtXbsW8+fP5+Y53LRpE3bu3IlPP/0Uy5cv71B/4sSJmDhxIgBYXM8FIhR2Oku1Wq3GJ598gtzcXNx///0ATGNqjxo1CocPH8bdd99t68ewGvXVakfkaTrjcdexbUATW1h24sQJGAwGDB8+3Kxcq9VyE7G++OKLeP755/Hdd98hKSkJs2bNcsr8ovayqU1Lp9OhsLAQSUlJbTvg85GUlASVStWjQM6cOQO5XI7BgwcjNTUVFy9e5NYVFhaiubnZ7LgjR45EZGRkp8fVarXQaDRmiz1oDsR2eDzTJZo7FprYwiETW9TV1UEgEKCwsJCbRLm4uBinTp3Ce++9BwB49tln8dtvv+Gpp57CiRMnEBsbi/Xr1zvss/aUTUmrqqoKBoOhQztSSEgIKioq7A4iPj4eW7Zswa5du/DBBx+grKwM9957L2prawEAFRUVEIvFHU5HuzquUqmETCbjloiICLtio9mm+x6a2MLE0sQW48ePh8FgwLVr1zB06FCzpf2VTkREBJ577jls374dL7/8Mj7++GNunwCsnjDDGXrF3cNp06bhsccew9ixY5GcnIxvv/0WNTU1+Oqrr+zeZ1ZWFtRqNbdcunTJrv3QCKZ9D01sYWJpYovhw4cjNTUVaWlp2L59O8rKynD06FEolUrs3LkTALB48WL85z//QVlZGYqKivD9999zx4uKigKPx0N+fj6uX79udgfUZZgNtFotEwgEbMeOHWblaWlp7JFHHul2+6ioKLZu3TqrjhUbG8uWL1/OGGOsoKCAAWA3b940qxMZGcnWrl1r1f7UajUDwNRqtVX1W12taWBRy/LZkKydTG8w2rRtX9fY2MhKSkpYY2Oju0Ox2e7du9moUaOYRCJhY8eOZfv27WMAzP52v/32WwaATZ48ucP2V69eZdOmTWMSiYRFRUWx3NxcFhwczDZt2mTV8W891s8//8zuu+8+JpVKWUBAAJs/fz6rra1ljDFWUVHBUlJSWFhYGBOLxSwqKoqtXLmSGQwGptVq2RNPPMEiIiKYWCxmcrmcZWZmWv1v8txzz7HAwEAGgGVnZzPGGNPpdGzlypUsOjqaiUQiFhYWxmbMmMF+/vlnxhhjmZmZbMiQIUwikbABAwawp556ilVVVXH7fP3111loaCjj8XgsPT3dqjhadfU3Ze131KakxRhjcXFxLDMzk3tvMBhYeHg4UyqV3W5rbdKqra1l/v7+7L333mOMMVZTU8NEIhH7xz/+wdU5ffo0A8BUKpVVcdubtPQGIxuStZNFLctnV2sabNq2r+vLScvRLl26xACwPXv2uDuUPs0RScvmu4cKhQLp6emIjY1FXFwccnJyUF9fz91NTEtLQ3h4OJRKJQBTw2dJSQn3+sqVKyguLoa3tzeGDh0KwHTL9eGHH0ZUVBSuXr2K7OxsCAQC7pRdJpNh3rx5UCgUCAgIgK+vLxYtWoSEhASn3jkEAAGfh1CZFJdvNuJqTSPCZK7vl0Jcjya26MXsyZbr169nkZGRTCwWs7i4OHb48GFuXWJiotkpY1lZGQPQYUlMTOTqPP7449ypcXh4OHv88cfZ2bNnO2ToF154gfn7+zNPT082Y8YMVl5ebnXM9p5pMcbY7E2HWNSyfPb18cs2b9uX9eczrV27drGYmBjm4eHBgoODWUpKCjt//jxjjLEvvviCeXl5WVzuuOMOl8U4derUTuN48803XRaHLRxxpkXzHlpBkVeM7cev4JWpI/DCfw11UoS9D817aFltbS0qKystrhOJRIiKinJJHFeuXOlyYoveOLkFzXvoIjSdGGmPJrZwr17R5aG36+99tfrJyThxAUf8LVHSsoKcGyu+yc2RuFbrQ7kNDQ1ujoTcLlr/lnoyPyRdHlqh/fOHrB9N3CoQCODn58eNbuDp6dlvPjtxLMYYGhoacO3aNfj5+UEgENi9L0paVmi9PKzT6qFp1EPm6bpZhN2t9dEOe4dlIaQ9Pz+/TgdGsBYlLStIRQIEeolRXa/DlZrGfpW0eDwewsLCEBwcjObmZneHQ/owkUjUozOsVpS0rBTu78ElrTvktnWZuB0IBAKH/MER0lPUEG8luaz1DiI1ShPiTpS0rMT11VL3rzuIhPQ2lLSs1N/7ahHSW1DSshKNYEpI70BJy0oDaax4QnoFSlpWaj3Tul6rpYlbCXEjSlpW8vcUwUNkuuVfQY3xhLgNJS0r8Xg8mk6MkF6AkpYNqDGeEPejpGUD6vZAiPtR0rJBuJ9ppEU60yLEfShp2YBGMCXE/exKWhs3bkR0dDSkUini4+Nx9OjRTuv+8ssvmDVrFqKjo8Hj8ZCTk9OhjlKpxMSJE+Hj44Pg4GCkpKSgtLTUrM5//dd/gcfjmS3PPfecPeHbLdzPEwCdaRHiTjYnrby8PCgUCmRnZ6OoqAjjxo1DcnJyp+MtNTQ0YPDgwVi1alWn4+js378fGRkZOHz4MHbv3o3m5mY88MADZtOIA8D8+fNRXl7OLe+8846t4feIvOXysLymCUYjDUFMiDvYPDTN2rVrMX/+fG6ew02bNmHnzp349NNPsXz58g71J06ciIkTJwKAxfUAsGvXLrP3W7ZsQXBwMAoLC83mmfP09OzxAGI9EeorhYDPg85gRFWdFsG+NEMNIa5m05mWTqdDYWEhkpKS2nbA5yMpKQkqlcphQanVagDoMAXSl19+iaCgIIwePRpZWVldjl2u1Wqh0WjMlp4SCvgIbUlUl+kSkRC3sOlMq6qqCgaDASEhIWblISEhOH36tEMCMhqNWLx4MSZNmoTRo0dz5X/4wx8QFRUFuVyOn3/+GcuWLUNpaSm2b99ucT9KpRKvvfaaQ2JqT+4nxZUa02zTd0X6O3z/hJCu9bqRSzMyMnDy5EkcPHjQrHzBggXc6zFjxiAsLAxTpkzBuXPnMGTIkA77ycrKgkKh4N5rNBpERET0OL5wPw8cw03qq0WIm9iUtIKCgiAQCDrMrltZWemQtqbMzEzk5+fjwIEDGDhwYJd14+PjAQBnz561mLQkEgkkEkmPY7oV9YonxL1satMSi8WYMGECCgoKuDKj0YiCggIkJCTYHQRjDJmZmdixYwf27t2LQYMGdbtNcXExACAsLMzu49qD+moR4l42Xx4qFAqkp6cjNjYWcXFxyMnJQX19PXc3MS0tDeHh4VAqlQBMjfclJSXc6ytXrqC4uBje3t4YOnQoANMlYW5uLv75z3/Cx8cHFRUVAACZTAYPDw+cO3cOubm5ePDBBxEYGIiff/4ZS5YsweTJkzF27FiH/CKs1fooz2W6PCTEPZgd1q9fzyIjI5lYLGZxcXHs8OHD3LrExESWnp7OvS8rK2MAOiyJiYlcHUvrAbDNmzczxhi7ePEimzx5MgsICGASiYQNHTqU/fGPf2RqtdrqmNVqNQNg0zaW/FqhYVHL8tmY7F092g8hxJy131EeY6xf9JLUaDSQyWRQq9Xw9bV/CrB6rR4x2f8BAJx49QH4SPvPHIiEOJO131F69tBGXhIh/Foma6XGeEJcj5KWHVrbtagxnhDXo6RlBxpXixD3oaRlh9a+WvQoDyGuR0nLDgO5vlo0wQUhrkZJyw5cr/ibnT+wTQhxDkpadmhriKczLUJcjZKWHVof5amsbYJOb3RzNIT0L5S07BDoJYZEyAdjNHErIa5GScsOPB6vrdsD3UEkxKUoadmJZpsmxD0oadlJLqNe8YS4AyUtO3FnWtQrnhCXoqRlJxrBlBD3oKRlJ3pomhD3oKRlp4HtGuL7yZBkhPQKlLTsFOIrBY8HaPVGVNfr3B0OIf0GJS07iYV8hPiYJm6lxnhCXIeSVg/I/VqSFrVrEeIydiWtjRs3Ijo6GlKpFPHx8Th69GindX/55RfMmjUL0dHR4PF4yMnJsWufTU1NyMjIQGBgILy9vTFr1qwO8y+6Wri/JwBqjCfElWxOWnl5eVAoFMjOzkZRURHGjRuH5ORkXLt2zWL9hoYGDB48GKtWrep0Qldr9rlkyRJ888032LZtG/bv34+rV69i5syZtobvUDSdGCFuYOs0P3FxcSwjI4N7bzAYmFwuZ0qlsttto6Ki2Lp162zeZ01NDROJRGzbtm1cnVOnTjEATKVSWRW3o6YQa+/zQ2Usalk+e/azYw7bJyH9lbXfUZvOtHQ6HQoLC5GUlMSV8fl8JCUlQaVS2ZU0rdlnYWEhmpubzeqMHDkSkZGRnR5Xq9VCo9GYLY5Gs00T4no2Ja2qqioYDAaEhISYlYeEhHCzQtvKmn1WVFRALBbDz8/P6uMqlUrIZDJuiYiIsCu+roT7mdq0qCGeENe5be8eZmVlQa1Wc8ulS5ccfozWu4c1Dc2o1+odvn9CSEc2Ja2goCAIBIIOd+0qKys7bWR3xD5DQ0Oh0+lQU1Nj9XElEgl8fX3NFkfzkYrgKxUCoEtEQlzFpqQlFosxYcIEFBQUcGVGoxEFBQVISEiwKwBr9jlhwgSIRCKzOqWlpbh48aLdx3UUmk6MENcS2rqBQqFAeno6YmNjERcXh5ycHNTX12Pu3LkAgLS0NISHh0OpVAIwNbSXlJRwr69cuYLi4mJ4e3tj6NChVu1TJpNh3rx5UCgUCAgIgK+vLxYtWoSEhATcfffdDvlF2GugvwdOV9TSmRYhrmLPrcn169ezyMhIJhaLWVxcHDt8+DC3LjExkaWnp3Pvy8rKGIAOS2JiotX7ZIyxxsZG9sILLzB/f3/m6enJZsyYwcrLy62O2RldHhhjbOXXJ1jUsnz29r9POXS/hPQ31n5HeYz1jyEKNBoNZDIZ1Gq1Q9u3Ptx/Dsp/n0bKnXLkPDHeYfslpL+x9jt62949dBUaK54Q16Kk1UNts01T0iLEFShp9dDAlqRVoWmC3kATtxLibJS0eijIWwKxgA8jMyUuQohzUdLqIT6fhzA/GgyQEFehpOUA3CQXakpahDgbJS0HoMZ4QlyHkpYDhHNzIFKbFiHORknLAaivFiGuQ0nLAbgzrZsNbo6EkNsfJS0HaJttuokmbiXEyShpOUBrl4fGZgNuNjS7ORpCbm+UtBxAIhRggI8EAA0GSIizUdJyEJpOjBDXoKTlIG3dHihpEeJMlLQchKYTI8Q1KGk5SDj1iifEJShpOYicLg8JcQlKWg7S1leLkhYhzmRX0tq4cSOio6MhlUoRHx+Po0ePdll/27ZtGDlyJKRSKcaMGYNvv/3WbD2Px7O4rF69mqsTHR3dYf2qVavsCd8pWpNWdb0OjTqDm6Mh5PZlc9LKy8uDQqFAdnY2ioqKMG7cOCQnJ+PatWsW6x86dAhz5szBvHnzcPz4caSkpCAlJQUnT57k6pSXl5stn376KXg8HmbNmmW2r9dff92s3qJFi2wN32l8PYTwlrRM3EpD1BDiPLZO8xMXF8cyMjK49waDgcnlcqZUKi3Wnz17Nps+fbpZWXx8PFu4cGGnx3j00UfZ/fffb1YWFRXF1q1bZ2u4HGdNIdbeA2v3s6hl+Wx/6TWnHYOQ25W131GbzrR0Oh0KCwuRlJTElfH5fCQlJUGlUlncRqVSmdUHgOTk5E7rV1ZWYufOnZg3b16HdatWrUJgYCDGjx+P1atXQ6/XdxqrVquFRqMxW5xN3jqCKbVrEeI0Ns0wXVVVBYPBgJCQELPykJAQnD592uI2FRUVFutXVFRYrP/ZZ5/Bx8cHM2fONCt/8cUXcddddyEgIACHDh1CVlYWysvLsXbtWov7USqVeO2116z9aA5BfbUIcT6bkpYrfPrpp0hNTYVUKjUrVygU3OuxY8dCLBZj4cKFUCqVkEgkHfaTlZVlto1Go0FERITzAgeNYEqIK9iUtIKCgiAQCFBZWWlWXllZidDQUIvbhIaGWl3///7v/1BaWoq8vLxuY4mPj4der8f58+cxYsSIDuslEonFZOZM9CgPIc5nU5uWWCzGhAkTUFBQwJUZjUYUFBQgISHB4jYJCQlm9QFg9+7dFut/8sknmDBhAsaNG9dtLMXFxeDz+QgODrblIzjVQBrBlBCns/nyUKFQID09HbGxsYiLi0NOTg7q6+sxd+5cAEBaWhrCw8OhVCoBAC+99BISExPx7rvvYvr06di6dSt+/PFHfPTRR2b71Wg02LZtG959990Ox1SpVDhy5Ajuu+8++Pj4QKVSYcmSJXjyySfh7+9vz+d2itbLwwp1EwxGBgGf5+aICLkN2XNrcv369SwyMpKJxWIWFxfHDh8+zK1LTExk6enpZvW/+uorNnz4cCYWi1lMTAzbuXNnh31++OGHzMPDg9XU1HRYV1hYyOLj45lMJmNSqZSNGjWKvfXWW6ypqcnqmF3R5UFvMLIhWTtZ1LJ8drWmwWnHIeR2ZO13lMdY/xgfWKPRQCaTQa1Ww9fX12nHufedvbh0oxH/eC4BsdEBTjsOIbcba7+j9Oyhg8ll1K5FiDNR0nIwmk6MEOeipOVgNK4WIc5FScvBaIgaQpyLkpaD0eUhIc5FScvB2j/K009uzBLiUpS0HKz18rBeZ4CmsfNRKAgh9qGk5WBSkQCBXmIAwOWaBjdHQ8jth5KWE7QNUdPk5kgIuf1Q0nKCtm4PdKZFiKNR0nICmk6MEOehpOUEbX216PKQEEejpOUErW1al+lMixCHo6TlBNQrnhDnoaTlBK1J63qtFk3NNHErIY5EScsJ/DxF8BAJAADlamrXIsSRKGk5AY/Ho+nECHESSlpOQkPUEOIclLScpLWvFt1BJMSx7EpaGzduRHR0NKRSKeLj43H06NEu62/btg0jR46EVCrFmDFj8O2335qtf/rpp8Hj8cyWqVOnmtW5ceMGUlNT4evrCz8/P8ybNw91dXX2hO8SA+nykBCnsDlp5eXlQaFQIDs7G0VFRRg3bhySk5Nx7do1i/UPHTqEOXPmYN68eTh+/DhSUlKQkpKCkydPmtWbOnUqysvLueXvf/+72frU1FT88ssv2L17N/Lz83HgwAEsWLDA1vBdRu5nmiGbLg8JcTBbp/mJi4tjGRkZ3HuDwcDkcjlTKpUW68+ePZtNnz7drCw+Pp4tXLiQe5+ens4effTRTo9ZUlLCALBjx45xZf/+978Zj8djV65csSpuV0wh1t6R36pZ1LJ8NvmdvS45HiF9nbXfUZvOtHQ6HQoLC5GUlMSV8fl8JCUlQaVSWdxGpVKZ1QeA5OTkDvX37duH4OBgjBgxAs8//zyqq6vN9uHn54fY2FiuLCkpCXw+H0eOHLF4XK1WC41GY7a4Uuvdw/KaJhiNNBggIY5iU9KqqqqCwWBASEiIWXlISAgqKiosblNRUdFt/alTp+Lzzz9HQUEB3n77bezfvx/Tpk2DwWDg9hEcHGy2D6FQiICAgE6Pq1QqIZPJuCUiIsKWj9pjIT4SCPg86AxGXK/TuvTYhNzOhO4OAACeeOIJ7vWYMWMwduxYDBkyBPv27cOUKVPs2mdWVhYUCgX3XqPRuDRxCQV8hPpKcaWmEVdqGhHiK3XZsQm5ndl0phUUFASBQIDKykqz8srKSoSGhlrcJjQ01Kb6ADB48GAEBQXh7Nmz3D5ubejX6/W4ceNGp/uRSCTw9fU1W1yNGuMJcTybkpZYLMaECRNQUFDAlRmNRhQUFCAhIcHiNgkJCWb1AWD37t2d1geAy5cvo7q6GmFhYdw+ampqUFhYyNXZu3cvjEYj4uPjbfkILkUPThPiBLa28G/dupVJJBK2ZcsWVlJSwhYsWMD8/PxYRUUFY4yxp556ii1fvpyr/8MPPzChUMjWrFnDTp06xbKzs5lIJGInTpxgjDFWW1vLli5dylQqFSsrK2N79uxhd911Fxs2bBhramri9jN16lQ2fvx4duTIEXbw4EE2bNgwNmfOHKvjdvXdQ8YYe2fXKRa1LJ+t+PqEy45JSF9l7XfU5qTFGGPr169nkZGRTCwWs7i4OHb48GFuXWJiIktPTzer/9VXX7Hhw4czsVjMYmJi2M6dO7l1DQ0N7IEHHmADBgxgIpGIRUVFsfnz53NJsFV1dTWbM2cO8/b2Zr6+vmzu3LmstrbW6pjdkbS+OHyeRS3LZ89sPuqyYxLSV1n7HeUx1j8m59NoNJDJZFCr1S5r39pXeg1Pbz6GkaE+2LV4skuOSUhfZe13lJ49dKKBNNs0IQ5HScuJWh+arm3SQ9PU7OZoCLk9UNJyIk+xEP6eIgB0B5EQR6Gk5WRyGleLEIeipOVk1FeLEMeipOVkNJ0YIY5FScvJaNhlQhyLkpaT0eUhIY5FScvJuIZ4SlqEOAQlLSdrbdO6VquFTm90czSE9H2UtJws0EsMiZAPxoAKmriVkB6jpOVkPB6Pa9e6XNPg5mgI6fsoablA22zTdKZFSE9R0nIBuYy6PRDiKJS0XCCcJm4lxGEoablAOHV7IMRhKGm5APXVIsRxKGm5QPvBAPvJQLGEOA0lLRcIlUnB4wE6vRFVdTp3h0NIn2ZX0tq4cSOio6MhlUoRHx+Po0ePdll/27ZtGDlyJKRSKcaMGYNvv/2WW9fc3Ixly5ZhzJgx8PLyglwuR1paGq5evWq2j+joaPB4PLNl1apV9oTvciIBHyE+LXMg0iUiIT1ic9LKy8uDQqFAdnY2ioqKMG7cOCQnJ3eYTLXVoUOHMGfOHMybNw/Hjx9HSkoKUlJScPLkSQBAQ0MDioqKsGLFChQVFWH79u0oLS3FI4880mFfr7/+OsrLy7ll0aJFtobvNnQHkRAHsXWan7i4OJaRkcG9NxgMTC6XM6VSabH+7Nmz2fTp083K4uPj2cKFCzs9xtGjRxkAduHCBa4sKiqKrVu3ztZwOe6YQqy9zNwiFrUsn320/5xbjk9Ib2ftd9SmMy2dTofCwkIkJSVxZXw+H0lJSVCpVBa3UalUZvUBIDk5udP6AKBWq8Hj8eDn52dWvmrVKgQGBmL8+PFYvXo19Hp9p/vQarXQaDRmiztRtwdCHENoS+WqqioYDAaEhISYlYeEhOD06dMWt6moqLBYv6KiwmL9pqYmLFu2DHPmzDGb++zFF1/EXXfdhYCAABw6dAhZWVkoLy/H2rVrLe5HqVTitddes+XjOVU4TSdGiEPYlLScrbm5GbNnzwZjDB988IHZOoVCwb0eO3YsxGIxFi5cCKVSCYlE0mFfWVlZZttoNBpEREQ4L/huhPu1NMTTozyE9IhNSSsoKAgCgQCVlZVm5ZWVlQgNDbW4TWhoqFX1WxPWhQsXsHfv3m5ngY6Pj4der8f58+cxYsSIDuslEonFZOYu4X6eAICrakpahPSETW1aYrEYEyZMQEFBAVdmNBpRUFCAhIQEi9skJCSY1QeA3bt3m9VvTVhnzpzBnj17EBgY2G0sxcXF4PP5CA4OtuUjuI285UyrpqEZ9drO2+IIIV2z+fJQoVAgPT0dsbGxiIuLQ05ODurr6zF37lwAQFpaGsLDw6FUKgEAL730EhITE/Huu+9i+vTp2Lp1K3788Ud89NFHAEwJ6/e//z2KioqQn58Pg8HAtXcFBARALBZDpVLhyJEjuO++++Dj4wOVSoUlS5bgySefhL+/v6N+F07lIxXBVyqEpkmPqzWNGBbi4+6QCOmb7Lk1uX79ehYZGcnEYjGLi4tjhw8f5tYlJiay9PR0s/pfffUVGz58OBOLxSwmJobt3LmTW1dWVsYAWFy+//57xhhjhYWFLD4+nslkMiaVStmoUaPYW2+9xZqamqyO2d1dHhhjbGrOARa1LJ/tPV3pthgI6a2s/Y7yGOsfD8NpNBrIZDKo1epu28uc5dnPjmHPqWv4fymj8eTdUW6JgZDeytrvKD176EI0nRghPUdJy4VoiBpCeo6SlgtxHUyprxYhdqOk5UJ0eUhIz1HScqHWpFWhaUKzgSZuJcQelLRcKMhbArGADyMDKjU0nRgh9qCkdasmDfDbfqCyBKi7Bhgc13udz+dxPePXfvcrvi+9hkadwWH7J6Q/6FUPTPcK10qAz9sPQMgDPPwBrwGAV5Bp8Qxqe+8Z2G7dAFNdvqDT3Y8d6Ifz1Q3YfvwKth+/ArGQj7joAEweHoR7hw3AyFAf8Hg8539OQvooSlq3YkYgaATQUAU03ADAgMYbpqWqtPvteXzAI6AtiZkltSCsiQlAaggfR8oNOHBBhzMaPg6d1ePg2SoApxHsI8G9wwZwSSzAS+zsT0xIn0I94rtiNJgSV/11UxKrvw7UV3d83/q68abd8TXyPKA2ekDNPFELT9QyD9TCEyIvfwQEBEEeGgJ5SAgEHn6A1BeQygCJr+m1xBeQ+AB0hkb6MGu/o3Sm1RW+APAeYFqsYWg2JTkuoVWZlg7vqwGtxtR+pjd1f/BgjfDgNSKUd8N8n00ArrYsXeHxTYlLIjP9FHu1W7wBsWe7117WvRaIKRGSXoeSliMJRIBPiGmxll7XksDUbYms5XWtuhoXr5TjWtV1qG9WQ2Kogy8a4MNrgA8a4Cdogg/qIWR602Vtk9q0OApfaDmhSWWAh5+p/U7qZ3rN/fRvey+VAQL6E+t1GDNdFdRWAHWVpv9IhRLAM8DUtNH6U9g7myboL8rdhGJA2NLAfwsfADEti8HIcPKKGgd+vY4DZ66j6GINDDrTgBgSNCNA0Ih75ELcEyFGhJcRXrwmePK08GRNkLJGSIyNEBsbwG9uAHT1LUud5df6lu4YRn3PE6HY55akJrvlvZ958hNKTMmSLzSd6fIE7d63lPHblfEEAN/Om+AGPdDcADQ3tvvZCDTX31LWfl3La129eZm+CRB5mj7LrYtngPl7qZ9zkrmh2XTHu67C9LM1KdVWtJXXVprKjM3d70/sbUpeHn4dE9qtP1vrSGT2/3tYidq0+ihNUzMOna3GgTPXceDX67hs5aNBEiEfXhIhvCQCeImF8JYI4SVp/SmAp1gIHzHgJ2yGTKCFr0AHH74W3jwtvNAELzTCB3Xw0NeC11QDNNUAjTWmxNZYY/ofvKnGlARdhce/JbkJLP9kzDzRWPPFdRaJr+UEZynJefibPl9dZdvSmpDqKlsSUYWp2cEWHv6Ad6jpP0y91nSzqeGG6d+P2dn5mcdvifmWxHbvy0DgkC43tfY7SknrNsAYw/nqBhz49Tp+OFuFqjot6rUG1Gn1qNfp0aA1QOfgHvhCPg8BXmIEeksQ5C1GYMvrQG8xgrwkCPLkYYBIiyBBA/z5DZDqa1sS3M12ia7GPOE11QAGnekMz2hoWfRtC5z1p8oznSWJPExtf62vuZ+trz0tlLX+lAK6hpY7zTfNl4bWshpA68DLd0v4QsAr2NRE4R0KeAcDPqGAd0jbT+8QU7mwk+HIjca2f6uGG22fqfX1rT8ba0yvm+s7j2vhASBsXJehU9K6xe2ctKyh0xtRr9Vziaxeq0e91tBWptWjXmdoKdejrmVdvc60vqElCWoam1Frx3DRnmIBAr3FCPSStCS4liTnJUZQS7IL8BLDRyKCVMyHVCSAh0gAkaDdpYbRCLBbEpml5Nb6nqvb8hO4JQG1vBZKXHfDwdByyW0puZkluPbLDdNn8A42JSKfkLbk4xParjzUdGbj5MuzTjU3tcV7a7Kb8LTpjKsLdPeQmBEL+RALxfB3QL8vrd6AG/U6VNfpUFWnbXtdr0V1nQ7VdVpUt5Rdr9NCpzeiQWdAw41GXLph28PiAj4PHiIBpCJTImtNZq3vPW4tEwsgFQrgIZZAKuTDQyzgtvMUCyDji+ArEkEmFMFXKoKA7+K7owIh4BVoWm43IikgCgN8w5x6GEpaxGYSoQBhMg+EyTy6rcsYQ73OgOo6LarMElrL+5bX1XU6VNebLmub9Aa0nv8bjAx1Wj3qtM75LN4SIWQeIvh6mMbwl3mIuPdtr9uVS9vWS0WdP/lAnIeSFnEqHo8H75aG/qhAL6u2YYxBqzdC22xEY7MBTc2GDj+bmo1o1JkSXKPOAK2+5T1Xx9hSr22beq0BmqZmqBub0dDyzGddy+WxPQMzioX8lkRmSmpSkcB0Rivgt5zZtnvdrkwk4ENy6/qWcrGQD4mFMpGTL/nEQj539ioR8nv1o2R2Ja2NGzdi9erVqKiowLhx47B+/XrExcV1Wn/btm1YsWIFzp8/j2HDhuHtt9/Ggw8+yK1njCE7Oxsff/wxampqMGnSJHzwwQcYNmwYV+fGjRtYtGgRvvnmG/D5fMyaNQvvvfcevL297fkIpBfj8XjcJZ0MIqcco9lghKbRlMA0TXqoW1+3+9ma4Ezv9S11TeuMzNROeL1Wi+u1TjoNdBMeD5AKTZfbrZffUguX5K1l3CW62JTwzC7bxXxIhQKMGSiDj9Qx/5Y2J628vDwoFAps2rQJ8fHxyMnJQXJyMkpLSy3OQXjo0CHMmTMHSqUSDz30EHJzc5GSkoKioiKMHj0aAPDOO+/g/fffx2effYZBgwZhxYoVSE5ORklJCaRS06gIqampKC8vx+7du9Hc3Iy5c+diwYIFyM3N7eGvgPRHIgG/5W6n7RP6Go0MdTo9l+Bak5pWb4BOb4TOYDT9bFmaDUZoLZS11tO2lhtaytvVa62jNzrvfhljgM5ghKHlGIwBjS1nqDfhmG4h/8yYhHERfg7Zl813D+Pj4zFx4kRs2LABgGmy1oiICCxatAjLly/vUP/xxx9HfX098vPzubK7774bd955JzZt2gTGGORyOV5++WUsXboUAKBWqxESEoItW7bgiSeewKlTp3DHHXfg2LFjiI2NBQDs2rULDz74IC5fvgy5XN5t3P397iEh3Wk2tF2ON+mMaNK3XF7rDGhqufzWtlyONzW3lTXpDWjStVyS69vqN7W7fP/wqQkYPKDrqyKn3D3U6XQoLCxEVlYWV8bn85GUlASVSmVxG5VKBYVCYVaWnJyMr7/+GgBQVlaGiooKJCUlcetlMhni4+OhUqnwxBNPQKVSwc/Pj0tYAJCUlAQ+n48jR45gxowZHY6r1Wqh1badtms0Gls+KiH9jkhgakPzddBlnLPY1LpXVVUFg8GAkBDzZ+tCQkK4WaFvVVFR0WX91p/d1bn10lMoFCIgIKDT4yqVSshkMm6JiIiw8lMSQnqz23bk0qysLKjVam65dOmSu0MihDiATUkrKCgIAoEAlZWVZuWVlZUIDQ21uE1oaGiX9Vt/dlfn2rVrZuv1ej1u3LjR6XElEgl8fX3NFkJI32dT0hKLxZgwYQIKCgq4MqPRiIKCAiQkJFjcJiEhwaw+AOzevZurP2jQIISGhprV0Wg0OHLkCFcnISEBNTU1KCws5Ors3bsXRqMR8fHxtnwEQkhfx2y0detWJpFI2JYtW1hJSQlbsGAB8/PzYxUVFYwxxp566im2fPlyrv4PP/zAhEIhW7NmDTt16hTLzs5mIpGInThxgquzatUq5ufnx/75z3+yn3/+mT366KNs0KBBrLGxkaszdepUNn78eHbkyBF28OBBNmzYMDZnzhyr41ar1QwAU6vVtn5kQogLWPsdtTlpMcbY+vXrWWRkJBOLxSwuLo4dPnyYW5eYmMjS09PN6n/11Vds+PDhTCwWs5iYGLZz506z9Uajka1YsYKFhIQwiUTCpkyZwkpLS83qVFdXszlz5jBvb2/m6+vL5s6dy2pra62OmZIWIb2btd9RGuWBENIr0CgPt2jNzdRfi5DeqfW72d15VL9JWrW1tQBA/bUI6eVqa2shk8k6Xd9vLg+NRiOuXr0KH5/uJ0PVaDSIiIjApUuX+tylJMXuHhR7zzHGUFtbC7lcDn4Xo1r0mzMtPp+PgQMH2rRNX+7fRbG7B8XeM12dYbW6bXvEE0JuT5S0CCF9CiUtCyQSCbKzsyGR2D7WkrtR7O5BsbtOv2mIJ4TcHuhMixDSp1DSIoT0KZS0CCF9CiUtQkifQkmLENKnUNK6xcaNGxEdHQ2pVIr4+HgcPXrU3SF1S6lUYuLEifDx8UFwcDBSUlJQWlrq7rDssmrVKvB4PCxevNjdoVjlypUrePLJJxEYGAgPDw+MGTMGP/74o7vD6pbBYMCKFSswaNAgeHh4YMiQIXjjjTe6fVi5N6Ck1U7rnI7Z2dkoKirCuHHjkJyc3GGo595m//79yMjIwOHDh7l5IR944AHU19e7OzSbHDt2DB9++CHGjh3r7lCscvPmTUyaNAkikQj//ve/UVJSgnfffRf+/v7uDq1bb7/9Nj744ANs2LABp06dwttvv4133nkH69evd3do3XPqqF59TFxcHMvIyODeGwwGJpfLmVKpdGNUtrt27RoDwPbv3+/uUKxWW1vLhg0bxnbv3s0SExPZSy+95O6QurVs2TL2u9/9zt1h2GX69OnsmWeeMSubOXMmS01NdVNE1qMzrRatczq2n3+xuzkdeyu1Wg0ACAgIcHMk1svIyMD06dPNfv+93b/+9S/ExsbiscceQ3BwMMaPH4+PP/7Y3WFZ5Z577kFBQQF+/fVXAMBPP/2EgwcPYtq0aW6OrHv9ZpSH7nQ1p+Pp06fdFJXtjEYjFi9ejEmTJmH06NHuDscqW7duRVFREY4dO+buUGzy22+/4YMPPoBCocCf/vQnHDt2DC+++CLEYjHS09PdHV6Xli9fDo1Gg5EjR0IgEMBgMODNN99Eamqqu0PrFiWt20xGRgZOnjyJgwcPujsUq1y6dAkvvfQSdu/eDalU6u5wbGI0GhEbG4u33noLADB+/HicPHkSmzZt6vVJ66uvvsKXX36J3NxcxMTEoLi4GIsXL4ZcLu/1sVObVgutVssEAgHbsWOHWXlaWhp75JFH3BOUjTIyMtjAgQPZb7/95u5QrLZjxw4GgAkEAm4BwHg8HhMIBEyv17s7xE5FRkayefPmmZX99a9/ZXK53E0RWW/gwIFsw4YNZmVvvPEGGzFihJsish61abWwZ07H3oIxhszMTOzYsQN79+7FoEGD3B2S1aZMmYITJ06guLiYW2JjY5Gamori4mIIBAJ3h9ipSZMmdeha8uuvvyIqKspNEVmvoaGhw+igAoEARqPRTRHZwN1Zszfpbk7H3ur5559nMpmM7du3j5WXl3NLQ0ODu0OzS1+5e3j06FEmFArZm2++yc6cOcO+/PJL5unpyb744gt3h9at9PR0Fh4ezvLz81lZWRnbvn07CwoKYq+88oq7Q+sWJa1bdDWnY28FwOKyefNmd4dml76StBhj7JtvvmGjR49mEomEjRw5kn300UfuDskqGo2GvfTSSywyMpJJpVI2ePBg9uc//5lptVp3h9YtGk+LENKnUJsWIaRPoaRFCOlTKGkRQvoUSlqEkD6FkhYhpE+hpEUI6VMoaRFC+hRKWoSQPoWSFiGkT6GkRQjpUyhpEUL6lP8P0QipOFM1UFwAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:54:42.330432Z", + "iopub.status.busy": "2024-03-22T21:54:42.329654Z", + "iopub.status.idle": "2024-03-22T21:59:05.411353Z", + "shell.execute_reply": "2024-03-22T21:59:05.410296Z" + }, + "papermill": { + "duration": 263.104523, + "end_time": "2024-03-22T21:59:05.413963", + "exception": false, + "start_time": "2024-03-22T21:54:42.309440", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:59:05.450505Z", + "iopub.status.busy": "2024-03-22T21:59:05.450208Z", + "iopub.status.idle": "2024-03-22T21:59:05.471590Z", + "shell.execute_reply": "2024-03-22T21:59:05.470883Z" + }, + "papermill": { + "duration": 0.041585, + "end_time": "2024-03-22T21:59:05.473550", + "exception": false, + "start_time": "2024-03-22T21:59:05.431965", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.00.0214410.00493911.5212370.1074282.0802130.2033920.0001576.5222130.0429265856186.50.0702810.2437890.00001318.04345
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.0 0.021441 0.004939 11.521237 0.107428 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 2.080213 0.203392 0.000157 6.522213 0.042926 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 5856186.5 0.070281 0.243789 0.000013 18.04345 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:59:05.507357Z", + "iopub.status.busy": "2024-03-22T21:59:05.507085Z", + "iopub.status.idle": "2024-03-22T21:59:05.984100Z", + "shell.execute_reply": "2024-03-22T21:59:05.983283Z" + }, + "papermill": { + "duration": 0.496252, + "end_time": "2024-03-22T21:59:05.986124", + "exception": false, + "start_time": "2024-03-22T21:59:05.489872", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T21:59:06.022869Z", + "iopub.status.busy": "2024-03-22T21:59:06.022081Z", + "iopub.status.idle": "2024-03-22T22:04:06.983559Z", + "shell.execute_reply": "2024-03-22T22:04:06.982770Z" + }, + "papermill": { + "duration": 300.982358, + "end_time": "2024-03-22T22:04:06.986018", + "exception": false, + "start_time": "2024-03-22T21:59:06.003660", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tvae/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/tvae/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:07.023587Z", + "iopub.status.busy": "2024-03-22T22:04:07.022779Z", + "iopub.status.idle": "2024-03-22T22:04:07.049088Z", + "shell.execute_reply": "2024-03-22T22:04:07.048370Z" + }, + "papermill": { + "duration": 0.0469, + "end_time": "2024-03-22T22:04:07.050967", + "exception": false, + "start_time": "2024-03-22T22:04:07.004067", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:07.085294Z", + "iopub.status.busy": "2024-03-22T22:04:07.084641Z", + "iopub.status.idle": "2024-03-22T22:04:07.089638Z", + "shell.execute_reply": "2024-03-22T22:04:07.088768Z" + }, + "papermill": { + "duration": 0.024367, + "end_time": "2024-03-22T22:04:07.091706", + "exception": false, + "start_time": "2024-03-22T22:04:07.067339", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.43801916990182255}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:07.128840Z", + "iopub.status.busy": "2024-03-22T22:04:07.128544Z", + "iopub.status.idle": "2024-03-22T22:04:07.487397Z", + "shell.execute_reply": "2024-03-22T22:04:07.486534Z" + }, + "papermill": { + "duration": 0.379816, + "end_time": "2024-03-22T22:04:07.489221", + "exception": false, + "start_time": "2024-03-22T22:04:07.109405", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAE8CAYAAACRnDftAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABMo0lEQVR4nO3deXxU1d348c+dubNmMpOE7JCwBhBQNkGDlcUNlZ+Fn62l1hZR0fZ5oJVSfSp9PY9Wrb/oUxVpa1FrIVpLcUN8HjekKFJlUTZB1MgaAiQkkGSS2Zd7f3/MZCSQhCwzmYXzfr3mFebOuXfOZTLfnHvu95wjqaqqIgiCkGQ08a6AIAhCd4jgJQhCUhLBSxCEpCSClyAISUkEL0EQkpIIXoIgJCURvARBSEoieAmCkJRE8BIEISmJ4CUIQlISwUuIu02bNvHb3/6WxsbGeFdFSCIieAlxt2nTJh588EERvIQuEcFLEISkJIKXEFe//e1vuffeewEYOHAgkiQhSRIWi4Vp06adVV5RFPr27cv3v//9yLbHH3+cSZMm0adPH0wmE+PHj+e1115r8/1eeuklxo8fj8lkIisrix/+8IdUVVXF5uSEmJLElDhCPO3evZtHH32Uf/zjHyxZsoTs7GwADhw4wEMPPcSxY8fIz8+PlN+4cSNTpkzh1VdfjQSwoqIivvvd7zJixAh8Ph+rVq3i008/5a233mLGjBmRfR955BH+67/+ix/84AdMmTKFuro6/vjHP2KxWNi5cycZGRm9eu5CD6mCEGe///3vVUA9dOhQZFtFRYUKqH/84x9blf33f/931WKxqC6XK7Lt9H+rqqr6fD511KhR6hVXXBHZdvjwYVWr1aqPPPJIq7J79uxRZVk+a7uQ+MRlo5CQhg4dypgxY3j55Zcj24LBIK+99ho33HADJpMpsv30fzc0NGC327n88svZsWNHZPvq1atRFIUf/OAHnDx5MvLIz8+npKSEDz/8sHdOTIgaOd4VEIT2zJ49m9/85jccO3aMvn37smHDBmpra5k9e3arcm+99Ra/+93v2LVrF16vN7JdkqTIv/ft24eqqpSUlLT5XjqdLjYnIcSMCF5Cwpo9ezaLFy/m1VdfZeHChbzyyivYbDauvfbaSJl//etffPe732Xy5Mn8+c9/pqCgAJ1Ox4oVK1i5cmWknKIoSJLEu+++i1arPeu9LBZLr5yTED0ieAlxd3oL6XQDBw5k4sSJvPzyyyxYsIDVq1cza9YsDAZDpMzrr7+O0Whk7dq1rbavWLGi1bEGDx6MqqoMHDiQoUOHxuZEhF4l+ryEuEtLSwNoM0l19uzZbNmyheXLl3Py5MmzLhm1Wi2SJBEMBiPbDh8+zJo1a1qVu/HGG9FqtTz44IOoZ9xgV1WVU6dORedkhF4jUiWEuPvss8+YOHEi119/PT/84Q/R6XTccMMNpKWlcfToUYqLi7FYLOh0Ompqalr1T33wwQdceeWVXH755fzoRz+itraWp59+mvz8fHbv3t0qUD366KMsXryYSZMmMWvWLNLT0zl06BBvvPEGd911F/fcc088Tl/orjje6RSEiIcffljt27evqtFozkqbuOyyy1RAnTdvXpv7/vWvf1VLSkpUg8GgDh8+XF2xYoX6wAMPqG39er/++uvqd77zHTUtLU1NS0tThw8frs6fP1+tqKiI1akJMSJaXoIgJCXR5yUIQlISwUsQhKQkgpcgCElJBC9BEJKSCF6CICQlEbwEQUhK593wIEVROH78OOnp6e0OSxEEIX5UVaW5uZnCwkI0mvbbV+dd8Dp+/DhFRUXxroYgCOdQVVVFv3792n39vAte6enpQOg/xmq1xrk2giCcqampiaKiosh3tT3nXfBquVS0Wq0ieAlCAjtXt47osBcEISmJ4CUIQlISwUsQhKR03vV5CUI0qKpKIBBoNQmi0DlarRZZlnucqhTX4LVs2TKWLVvG4cOHARg5ciT3338/1113XZvly8vLue2221ptMxgMeDyeWFdVECJ8Ph/V1dW4XK54VyVpmc1mCgoK0Ov13T5GXINXv379ePTRRykpKUFVVV544QVmzpzJzp07GTlyZJv7WK1WKioqIs9FoqnQmxRF4dChQ2i1WgoLC9Hr9eJ3sAtUVcXn81FXV8ehQ4coKSnpMBG1I3ENXjfccEOr54888gjLli1jy5Yt7QYvSZJaraAsCJ1Rb29mw/4Gcq1mLhvSp9sBx+fzoSgKRUVFmM3mKNfy/GAymdDpdFRWVuLz+TAajd06TsJ02AeDQVatWoXT6aS0tLTdcg6Hg/79+1NUVMTMmTPZu3dvh8f1er00NTW1egjnF/XUAY689Sh99jzP7v2H2Vfr6PExu9taEEKi8f8X909gz549WCwWDAYDP/vZz3jjjTcYMWJEm2WHDRvG8uXLefPNN3nppZdQFIVJkyZx9OjRdo9fVlaGzWaLPMTQoPOMquL44l3cHi/6oIu+9p18ccwe71oJURD3Oex9Ph9HjhzBbrfz2muv8fzzz/PRRx+1G8BO5/f7ueCCC7j55pt5+OGH2yzj9XpbraLcMvTAbreLDPvzQVM1x9cvo7LehVmvxZaejjLpFwzK7d5n7/F4OHToEAMHDuz25Y7Q8f9jU1MTNpvtnN/RuLe89Ho9Q4YMYfz48ZSVlTF69GiWLl3aqX11Oh1jx45l//797ZYxGAyRoUBiSNB5qLGSJo+fBlMx2RnpDLBpGWRyx7tWKW/AgAE89dRTMX2PuAevMymK0qql1JFgMMiePXsoKCiIca2EpNVYhaKC01SIObs4tK3pWHzrJERFXO82Ll68mOuuu47i4mKam5tZuXIlGzZsYO3atQDMmTOHvn37UlZWBsBDDz3EpZdeypAhQ2hsbOT3v/89lZWVzJs3L56nISQyxwlGFFgZPmYiUsMh/AerqD92CJ9pOEVZ4m5hR3w+X4/ysGItri2v2tpa5syZw7Bhw7jyyiv57LPPWLt2LVdffTUAR44cobq6OlK+oaGBO++8kwsuuIDrr7+epqYmNm3a1Kn+MeE85PeAtxkAjSUXyZJLo8vP598cZOuh+qi/nS+gtPsIBJVOl/V3omx3TJ06lQULFrBgwQJsNhvZ2dn813/9V2RV8QEDBvDwww8zZ84crFYrd911FwAff/wxl19+OSaTiaKiIn7xi1/gdDojx62treWGG27AZDIxcOBA/v73v3erfl0V15bXX//61w5f37BhQ6vnS5YsYcmSJTGskZBSnHWhn4Z00BnB3AejXoPJ30ilo3NdE13x9Ift970OzE5j1ti+kefPbTyAP9j2vbJ+mSZuuvjbu+LLPzmE29d6GNIvrx7arTq+8MIL3HHHHXz66ads27aNu+66i+LiYu68804AHn/8ce6//34eeOABAA4cOMC1117L7373O5YvX05dXV0kAK5YsQKAuXPncvz4cT788EN0Oh2/+MUvqK2t7Vb9ukKMbRRSl+skVQ0ujmAl47idkXmZGHUystKEx+3AF1DQywnX7RtTRUVFLFmyBEmSGDZsGHv27GHJkiWR4HXFFVfwq1/9KlJ+3rx53HLLLSxcuBCAkpIS/vCHPzBlyhSWLVvGkSNHePfdd/n000+ZMGECEGqUXHDBBTE/FxG8hNTlOoXTG+CENo20oApaHTqzDZ22AWPATqPbR2569NId5k8b0u5rmjMS+u+aPLjdsmcm/99+2cCeVKuVSy+9tNXogtLSUp544onIAPOLL764VfnPP/+c3bt3t7oUVFU1Mkzqm2++QZZlxo8fH3l9+PDhZGRkRK3O7RHBS0hdHjvegILXYCXTrAttM9rQyxoMAQcOT4Dcjmca7pKutOJiVban0tLSWj13OBz89Kc/5Re/+MVZZYuLi/nmm296q2pnEcFLSF0eO76AgkdOJ90YDl4GK3qtBkPQQbMnEN/6xcHWrVtbPd+yZQslJSVotdo2y48bN44vv/ySIUPablUOHz6cQCDA9u3bI5eNFRUVNDY2RrXebTm/LviF80rA2UBAUfFq07EYwn+njVYMsjbU8vKef8HryJEjLFq0iIqKCv7xj3/wxz/+kbvvvrvd8r/+9a/ZtGkTCxYsYNeuXezbt48333yTBQsWAKEhe9deey0//elP2bp1K9u3b2fevHmYTKaYn4sIXkJq8nvwecLzbZls3156GW1kp+uZUKBleH4UrxmTxJw5c3C73UycOJH58+dz9913R1Ii2nLRRRfx0Ucf8c0333D55ZczduxY7r//fgoLCyNlVqxYQWFhIVOmTOHGG2/krrvuIjc3N+bnIi4bhdTkseMLKvg1JtJObwUYrKQbdKQbfWAxxK9+caLT6XjqqadYtmzZWa+1TAp6pgkTJvD++++3e8z8/HzeeuutVtt+8pOf9KienSFaXkJq8jSiqiCnZZCZdlqWuCE8tjWcvCokL9HyElKTt5lMs55J/QfCqG8vcTBYUFQVe1MzR4+cYlRRlpgJNUmJ4CWkppaWleGMfi3ZiCpp+bqmnh2aSkoKMjDq2r7TlmrOHLGS7MRlo5CafOGxd3pL6+2ShNZoQauR0AeduHxi9Z9kJYKXkJp8DvYet/NWRTO1zWesLqVPQ6+V0AXduHznX7pEqhDBS0hN3mbc/iC1HhnNmX1aegs6rQZ90HXWgGcheYjgJaQkxevAH1Txac2Y9Wf0aenTkLUadIpbXDYmMRG8hNQTDBDwhhJU/bIZo3x28NJpJXRBF05x2Zi0RPASUo/PgT+ooEhaDEYzmjOndNCnodNq0AXd4rIxiYlUCSH1hIOXX2vGZNCd/breQlaannGZOuR+Gb1ePSE6RPASUo/XgT+g4tOmYW4rh0tnIk0vk2YG0s+/IUKpQlw2CqnH50CSQG+2kmFuo+WlCy+84Xf1br2EqIpr8Fq2bBkXXXRRZD3F0tJS3n333Q73efXVVxk+fDhGo5ELL7yQd955p5dqKyQNbzPZFgNTLxzElRfknf26bERRVertzXxxtKHn76eqEPD1/qML60W/+OKL9OnT56xlBWfNmtUrg6hjIa6Xjf369ePRRx+lpKQEVVV54YUXmDlzJjt37mTkyJFnld+0aRM333wzZWVl/J//839YuXIls2bNYseOHYwaNSoOZyAkpPay61voTCiqSkVNE9u0RxheYEPW9uDveNAP/3qi+/t31+W/ArlzS5PddNNN/OIXv+B//ud/uOmmm4DQqj9vv/12hzNGJLK4trxuuOEGrr/+ekpKShg6dCiPPPIIFouFLVu2tFl+6dKlXHvttdx7771ccMEFPPzww4wbN44//elPvVxzIaFFglc76zJqtGj1JiRAp3hw+1P/jqPJZOJHP/pRZMUfgJdeeoni4mKmTp0av4r1QMJ02AeDQV599VWcTielpaVtltm8eTOLFi1qtW369OmsWbOm3eN6vd5WTeWmpqao1FdIYH4XX1U3UaE0conZTYHt7Fk9JZ0ZWSshK148foUercOh1YVaQb1N20Z/XgfuvPNOJkyYwLFjx+jbty/l5eXMnTs3aWfViHvw2rNnD6WlpXg8HiwWC2+88Ua7i8jW1NSQl9e6DyMvL4+ampp2j19WVsaDDz4Y1ToLCc7vxu0PUufWnD00qIXOhKzRIAc9eHra8pKkTl++xdPYsWMZPXo0L774Itdccw179+7l7bffjne1ui3udxuHDRvGrl272Lp1K//2b//Grbfeypdffhm14y9evBi73R55VFVVRe3YQmJSfc5QnpfGiOnMoUEtwi0vnRKF4JVE5s2bR3l5OStWrOCqq66iqKjo3DslqLgHL71ez5AhQxg/fjxlZWWMHj2apUuXtlk2Pz+fEydOtNp24sQJ8vPz2z2+wWCI3M1seQgpLOgnGPChqBDQmtrO84Jwy0tCVjx4/Erv1jGOfvSjH3H06FH+8pe/cPvtt8e7Oj0S9+B1JkVRzrqd26K0tJT169e32rZu3bp2+8iE85DfhT+ookoatDpD+3cRdUZkbfiyMXD+tLxsNhvf+973sFgszJo1K97V6ZG49nktXryY6667juLiYpqbm1m5ciUbNmxg7dq1QGilk759+1JWVgbA3XffzZQpU3jiiSeYMWMGq1atYtu2bTz33HPxPA0hkfhcBCKXjB38euvM5KYbuDTDjCmnnZSKFHXs2DFuueUWDIbkHl0Q1+BVW1vLnDlzqK6uxmazcdFFF7F27VquvvpqILTGnEbz7V/OSZMmsXLlSv7zP/+T3/zmN5SUlLBmzRqR4yV8y+/Cr6gENKaOp3fWmbAadVgtQFrid7ZHQ0NDAxs2bGDDhg38+c9/jnd1eiyuweuvf/1rh6+3Nef2TTfdFEmyE4Sz+N1IgMFsIbOtoUEtzsMhQmPHjqWhoYHHHnuMYcOGxbs6PRb3VAlBiCq/i0yznu8MKIKRBe2X05nwBxUa6xtprnMw6Dy4dGxvXcZklXAd9oLQIy0tKX1ax+VkEy5fkK+ravlk/8nY10uIOhG8hNTid4d+6s7Oqm9FZ0TWSGgVL55uTEiodmFQtHC2aPz/ieAlpBafk4N1Dt75upH9tR2sii0b0WolJFR8Pk+nv0w6XagfzeU6f/rKYqHl/6/l/7M7RJ+XkFpahgZ5ZQYpHQQkjYwsh784AS/+oIpePvcYP61WS0ZGBrW1tQCYzeakHRsYD6qq4nK5qK2tJSMjA622+wv+iuAlpBa/m4CiEtAYMXWUKiFJaPUmNBKhLPtAEL3cuQuRlhEdLQFM6LqMjIwOR8Z0hgheQmrxOwkEVfwaY8d5XoAkG8NDhHx4/EGsxs5dwkiSREFBAbm5ufj9/mjU+ryi0+l61OJqIYKXkDoUBdXvIaAoBLTnDl7IoeFDsuLF243xjVqtNipfQqF7RPASUkfAjaKoKCrhltc5LgN1JooyTVgHpJN1nmTZpxIRvITU4XOFWl0aA1qtFv25pnaWDWSlGciyyWAQX4VkI1IlhNThd6GoYDClkZmmP/ddQDk8fWpLbpiQVMSfGyF1+N2YdFomDu3HxHH9z11eNuL2B2k41Yic7SHX2pO5oIXeJoKXkDr84YU3zpVd30I2crLZy676aizpTSJ4JRlx2SikjsjQoHZWDTqTbECrlZCDXrzn0YSEqUK0vITU4XdTbXfzhbOZTGM9Fw/I6rh8eCporerDHTh/poJOFSJ4CanD58QbUDgV1GLqTN6WbAgnqXYvz0uIL3HZKKQOvzuUXa/tRI4XhAZna8JJquKyMemI4CWkDn8oz8t/rimgW8gt0+L48IrLxqQjgpeQOvwuAsHQoOzOBi+tRkKjBvH72l6xSkhccQ1eZWVlTJgwgfT0dHJzc5k1axYVFRUd7lNeXo4kSa0eRqO4xX3eU1Xwu/GHxzW2u9js6WQDOlnLwOw0riyxiQkGk0xcg9dHH33E/Pnz2bJlC+vWrcPv93PNNdfgdDo73M9qtVJdXR15VFZW9lKNhYQV9IESDM8oYcLYmeltJAmt3ki+1cjQPp3IyBcSSlzvNr733nutnpeXl5Obm8v27duZPHlyu/tJktTjuYCEFONzoqIi6/SYTIbOtbwgNETI7xFDhJJQQvV52e12ALKyOs7PcTgc9O/fn6KiImbOnMnevXvbLev1emlqamr1EFKQ342ExNjBhdw1eTDmjhacPZ1spNnrp7K2HpcvENs6ClGVMMFLURQWLlzIZZdd1uEissOGDWP58uW8+eabvPTSSyiKwqRJkzh69Gib5cvKyrDZbJFHUVFRrE5BiKeuZte3kI0crHOy8csqTjb7ol8vIWYSJnjNnz+fL774glWrVnVYrrS0lDlz5jBmzBimTJnC6tWrycnJ4dlnn22z/OLFi7Hb7ZFHVVVVLKovxFtkXGNXg5chnC7hF7leSSYhMuwXLFjAW2+9xcaNG+nXr1+X9tXpdIwdO5b9+/e3+brBYMBgMESjmkIi87tpdPvYdaiZoP4EV16Q17n9wukSsuLFI7Lsk0pcW16qqrJgwQLeeOMNPvjgAwYOHNjlYwSDQfbs2UNBQQerIwupz+/CF1Co9+lo9nSh70o2IGs04URV0fJKJnFtec2fP5+VK1fy5ptvkp6eTk1NDQA2mw2TKTStyZw5c+jbty9lZWUAPPTQQ1x66aUMGTKExsZGfv/731NZWcm8efPidh5CAvC5IqsGWTszNKhFeP1GOeAVWfZJJq7Ba9myZQBMnTq11fYVK1Ywd+5cAI4cOYJG8+0vY0NDA3feeSc1NTVkZmYyfvx4Nm3axIgRI3qr2kIiOm1co6Ez2fUtWlYQUkXLK9nENXh1JqN5w4YNrZ4vWbKEJUuWxKhGQtLyuwgEFfyyqeP1Gs8kG9C2jG8UfV5JJSE67AWhx/zhy8bOLHl2OtmIzaRjZJoB+tpiVz8h6kTwElJDOHh1asmz08kG0vQyaSYNZHUxzUKIq4TJ8xKEbgsGIOBDq5HQm9Iw67rwN7llBaGAJzZ1E2JGtLyE5Od3ATAs38awKSOgKwOsZQNBVcXR7KC+zsHAHEuMKilEm2h5CckvMjTI2LXABSAb8QcUvjzeyLufH4l+3YSYEcFLSH7hlleXhwYBaHVo5VAHv+r3EFTEnF7JQgQvIfn5XfiCCp8ccfH69rYH6LdLktCG13kUWfbJRQQvIfn53fiDCo1+mVPOrk/nrNGFxzeqItcrmYjgJSQ/n/PbGVS7kuPVIjKzhBgilExE8BKSn98dXjXIiFHuTvBqmVlCXDYmExG8hOTnD7e8tCaMnZ3++XSRlpdYAi2ZiDwvIfn53eEZJQydW3jjTLKRfJsRQ66Z7HQx91uyEMFLSH6+8KBsQ/f7vPqkGeiTJYNZH/36CTEhLhuF5Od3oZFCQ4PSDN3r8wIgIBaeTSbdCl4HDx6Mdj0EoXsUBQIeirLM/OTyEYzv3/HKU22SjfiCCicbm6htFmMck0W3gteQIUOYNm0aL730Eh6P+LCFOAq4Q6tlA4STTbtMNnDS4WXbgWq2H26IXt2EmOpW8NqxYwcXXXQRixYtIj8/n5/+9Kd8+umn0a6bIJybr2VokBE03bhkhG9nU1W8+ILibmOy6FbwGjNmDEuXLuX48eMsX76c6upqvvOd7zBq1CiefPJJ6urqol1PQWhbeFzjjmofr2yrwu72d/0Yp6dKiAz7pNGjDntZlrnxxht59dVXeeyxx9i/fz/33HMPRUVFzJkzh+rq6mjVUxDa5nehonLKq+VYgxutpouzSkA4SVUjklSTTI+C17Zt2/j3f/93CgoKePLJJ7nnnns4cOAA69at4/jx48ycObPD/cvKypgwYQLp6enk5uYya9YsKioqzvm+r776KsOHD8doNHLhhRfyzjvv9OQ0hGTmdxFUVHya0B3D7uV5nTY8yC+CV7LoVvB68sknufDCC5k0aRLHjx/nxRdfpLKykt/97ncMHDiQyy+/nPLycnbs2NHhcT766CPmz5/Pli1bWLduHX6/n2uuuQan09nuPps2beLmm2/mjjvuYOfOncyaNYtZs2bxxRdfdOdUhGTncxEIhuau12klZG33klS1GgkJFb/fF/06CjHRrSTVZcuWcfvttzN37tx2F3vNzc3lr3/9a4fHee+991o9Ly8vJzc3l+3btzN58uQ291m6dCnXXnst9957LwAPP/ww69at409/+hPPPPNMN85GSGrh7PpuD8qG0Jxe2tC+is9NUFG7d/kp9KpuBa9169ZRXFzcaj1FCC1lVlVVRXFxMXq9nltvvbVLx7Xb7QBkZbWfq7N582YWLVrUatv06dNZs2ZNm+W9Xi9e77fJh01NTV2qk5Dg/M5Qdn1XVw06nSSh1ZsozjJjHWyNbv2EmOnWZePgwYM5efLkWdvr6+sZOHBgtyqiKAoLFy7ksssuY9SoUe2Wq6mpIS8vr9W2vLy8yGrbZyorK8Nms0UeRUVF3aqfkKCi0fIiNKdX3wwTI3MNotWVJLoVvNpbLNbhcGA0GrtVkfnz5/PFF1+watWqbu3fnsWLF2O32yOPqqqqqB5fiDOfExXQGc1YujM0qIUcHpAthggljS5dNrZcrkmSxP3334/Z/O2c4cFgkK1btzJmzJguV2LBggW89dZbbNy4kX79+nVYNj8/nxMnTrTaduLECfLz89ssbzAYMBjETAEpy+8mx2Lg5osvgPS8c5dvj2zE7Q9ir2/Eag1g1os5CxJdlz6hnTt3AqGW1549e9Drvx2Br9frGT16NPfcc0+nj6eqKj//+c9544032LBhQ6cuOUtLS1m/fj0LFy6MbFu3bh2lpaWdPxEhNajqaSsHdXNoUAvZwME6B587qphgG0ZJXnrP6yfEVJeC14cffgjAbbfdxtKlS7Fae9a5OX/+fFauXMmbb75Jenp6pN/KZrNhMoV+GefMmUPfvn0pKysD4O6772bKlCk88cQTzJgxg1WrVrFt2zaee+65HtVFSEIBD6jhjPjurBx0ushsqmIq6GTRrT6vFStW9DhwQSjlwm63M3XqVAoKCiKPl19+OVLmyJEjrTL1J02axMqVK3nuuecYPXo0r732GmvWrOmwk19IUeFxjfvrfbyyo5qqelf3j9VqNlWRqJoMOt3yuvHGGykvL8dqtXLjjTd2WHb16tWdOmZ7Hf+n27Bhw1nbbrrpJm666aZOvYeQwsLjGhsDeo41uns2qLpliFDQK8Y3JolOBy+bzYYUXo3YZrPFrEKC0Gnh4OUm1Pdq7s789S104ZklAmIe+2TR6eC1YsWKNv8tCHETDl4uNRS8TD3I80I2otWKRTiSSbf6vNxuNy7Xt/0LlZWVPPXUU7z//vtRq5ggnJPPSVBV8UihmzumnrS8TpvTS/R5JYduBa+ZM2fy4osvAtDY2MjEiRN54oknmDlzJsuWLYtqBQWhXT4n/qCCX2NCq5HQd2dQdgvZQJpBpqSPjpGFolskGXR7JtXLL78cgNdee438/HwqKyt58cUX+cMf/hDVCgpCu3wOAkEVn9aMWa+N9Ml2i2wkTS/T36plSK4lenUUYqZbwcvlcpGeHkrie//997nxxhvRaDRceumlVFZWRrWCgtAunwtFVdGZLKQZepgRf/rwoE7cBRfir9sLcKxZs4aqqirWrl3LNddcA0BtbW1U8r8EoVN8TqxGHTeVDufmicU9O5ZsRFFVnF4fR0/Zo1M/Iaa6Fbzuv/9+7rnnHgYMGMAll1wSGZrz/vvvM3bs2KhWUBDa5XOEfurTen4sjUwQDbuP2nnzs4Moimh9JbputbW///3v853vfIfq6mpGjx4d2X7llVfyf//v/41a5QShXQEfBMOLbUQjeIXn9ALQKj58QQVjd1cjEnpFtzsK8vPzz5rJYeLEiT2ukCB0SrjVVWn3sXXHCcb2z+zxYGqNzohWIpQu4Vd6ND+YEHvdCl5Op5NHH32U9evXU1tbi6K0TuoTK2oLMecLrXPQFDRwzO7hgmgM6ZEN4VWEWnK9dD0/phAz3Qpe8+bN46OPPuInP/kJBQUFPbtFLQjdERkaFJr80qTv0UJYIbIRWSuhVUWWfTLoVvB69913efvtt7nsssuiXR9B6JzwZaMrEryiMHmgbAhPiyNmlkgG3fpzlZmZ2eEiGYIQc+HLRmc0xjW2CA8R0ipePGJmiYTXreD18MMPc//997ca3ygIvcrnRFFVXITuEPZoRokWsoFsi4HR+UZyrWLq8ETXrbb2E088wYEDB8jLy2PAgAHodK07Ns+12Kwg9JjPGVk1SCNJGLqzUvaZZCPZFgPZ2XpI795CMkLv6VbwmjVrVpSrIQhd5HMSVFRkUzpphh6Oa2wRGSLk6fmxhJjrVvB64IEHol0PQeganxOTTssPLhmOai2MzjFlIwFFweVwEHD6yErTn3sfIW663dZubGzk+eefZ/HixdTX1wOhy8Vjx45FrXKC0CZVjXTYozdHL1VHNnLK4ePTfdX8a19ddI4pxEy3gtfu3bsZOnQojz32GI8//jiNjY1AaO76xYsXd/o4Gzdu5IYbbqCwsBBJklizZk2H5Tds2IAkSWc92lstW0hRfjcogdC/9VFcokw2IGvFCkLJolvBa9GiRcydO5d9+/a1WiH7+uuvZ+PGjZ0+jtPpZPTo0Tz99NNdev+Kigqqq6sjj9zc3C7tLyS5cI5XlUPllR3VfHm8KTrHDS9/JqaCTg7d6vP67LPPePbZZ8/a3rdv3y61gq677jquu+66Lr9/bm4uGRkZnSrr9Xrxer9dwr2pKUq/6EL8eJsBaAoaOdboZnC0Jg+UDcgaTShJ1ReIzjGFmOlWy8tgMLQZBL755htycnJ6XKlzGTNmDAUFBVx99dV88sknHZYtKyvDZrNFHkVFRTGvnxBj4eDlkEILzaYZojSAOtzyAhW/33vO4kJ8dSt4ffe73+Whhx7C7w9NSSJJEkeOHOHXv/413/ve96JawdMVFBTwzDPP8Prrr/P6669TVFTE1KlTO8wrW7x4MXa7PfKoqqqKWf2EXhIOXs7w0KC0aAwNAtDIaOXQsRSfR8zpleC6naT6/e9/n5ycHNxuN1OmTKGmpobS0lIeeeSRaNcxYtiwYQwbNizyfNKkSRw4cIAlS5bwt7/9rc19DAYDBoPIlo5w1IFGC+YkHt4V7vNqVqKYXQ+hOb10oYAoK14xp1eC61bwstlsrFu3jk8++YTPP/8ch8PBuHHjuOqqq6Jdv3OaOHEiH3/8ca+/b1I6shUOfACSBCNmQu4F8a5R93ibUVQVR0vLq6fz159GqzfRN8OEqTgNMVlKYuvyp64oCuXl5axevZrDhw8jSRIDBw4kPz8fVVV7fXqcXbt2UVBQ0KvvmZS8zXAofCdYVWHf+9BnCGiTcM4qbzP+oIJPm4ZWE6WhQS1kA8VZZooLTCCLVlci61LwUlWV7373u7zzzjuMHj2aCy+8EFVV+eqrr5g7dy6rV68+Z67W6RwOB/v37488P3ToELt27SIrK4vi4mIWL17MsWPHImtEPvXUUwwcOJCRI0fi8Xh4/vnn+eCDD8Rit51xYm8oNyo9P5Qn5bHDyX2QNyLeNes6bzNBRUVvtpJulKP7B1MOp/6IIUIJr0vBq7y8nI0bN7J+/XqmTZvW6rUPPviAWbNm8eKLLzJnzpxOHW/btm2tjrNo0SIAbr31VsrLy6murubIkSOR130+H7/61a84duwYZrOZiy66iH/+859n1UVow8lvQj8LLgq1wio3Q91XyRe8ggHwuzHrZW65bBSqzhTd48sGfEEFR1MTpqxgz1bhFmJKUtXOL1J3zTXXcMUVV3Dfffe1+fr/+3//j48++oi1a9dGrYLR1tTUhM1mw263nz/LtPlcsOkPocvF0vmhDu/tL4DOCJctJKk6d9wNsOUZ0Mgw+Z7o173iXb7esZHduosYUXo9o/qK1bN7W2e/o13qLNi9ezfXXnttu69fd911fP755105pNAbmo6HApe5DxitYMkHWQ9+Dzhq4127rvGGlzszWGITdFvNpiqy7BNZl4JXfX09eXl57b6el5dHQ0NDjyslRFnz8dDPltkXNBqwhZN1G4+0vU+iCud47WuElz87wr4TzdE9vmxE1mhCQ4T8YiroRNal4BUMBpHl9rvJtFotgYAYVpFwmlqCVwGKouLwBlDTw8vWOZJsUHs4eDUEDRxv9ES/dXTa4Gy3CF4Jrct3G+fOndtu0ufpYwiFBKGqkeDlNuXz2qdHONnsZaRB4ipVReM4EecKdpE3NCzNroaGBqUbo5fjBUTmsZdVn5jHPsF16ZO/9dZbz1mms3cahV7ibYaAFyQNm6vhZHPoD8w+Zxoj3G76aU6F7uBpoxwEYsVjB8Aezq63RDFBFQgvfxa6bHSIlldC69Inv2LFiljVQ4gV10kAfHorX9SEFky5qJ+NihqJoN8EqgLOOrAmSaKvp5GgotJMGgCWqLe8DKGWl+LFI5Y/S2hJ8udW6DbnKQBqAhaCikqu1cC0YblMHpqDbs8AaKhMsuDVhC+o4JXT0csaDNHOgpeNmHRaim2QWSjSJBKZCF6pzhmazrjKG2qpDM6xoNFIaJBCqRMNleA6Fc8adp7fAwEvvoCCV2shM9qtLgDZgFGnZVCGBP1E8EpkIniluvBl45jhQ7CQS3GWOfKSasrC5Qtgdp0iKdJUw/1dAa2RNLMJmykG4zJbhgepKgR9364oJCQcEbxSnTMUvNIy8xltyYhsVhSV175yU3TUzkXm2nAPUoIL32ns0yeHeRcPoguDQzpPI4NGi9fno6nRTmZmNrI2igO/hagRn0oq87tDdxoBTJmtXtJoJHSWbACcjXWgJEHndLjlhTE0ZCQmM5hIEsgGdh+1s3rrAexuf/TfQ4gKEbxSWfjLfsytYecxB82e1l/EPn2yCUo6HB4fuBvjUMEuigSvGPdFyUZkbWghDpGomrhE8Epl4YB0qFlmQ0UdDc7Wwasgw4RHZ8PhCSRHp304eH142MvLnx2hrjlGSdGyAZ1GE0qXEImqCUsEr1TmsRNQFBqUUI9WTnrrzud8mxG3nIHLFyTgSIJFVj12VFSOufUcb/Sg08boNkNLy0v14REtr4Qlglcq8zTi9gfxyhYsBvmsuanSjTokcyYqYG84GZ86doWnEX9QxakJTdGcbozRLLCRRFUfXpGomrBE8EplHjsubxCvbKWPRd9mkTRbHwCcjQkevPxu8Hvw+IN4ZBvpRl14mbIYkI1otRq0ihe3T1w2JiqRKpHK3I24/EG8BgtFlrbzlfoV9sV4ykSGxtHLlesid2iqJZdkQtHoYpPj1UJnQqeR0AU84rIxgYnglapUNdzyCuBJa7/lVdK/HxwzA+5QukSiLvXlqgfAIaUDkBHT4GUm3ahjmF5Gn20+d3khLkTwSlU+BygB3P7QKjvZ7bS80FtCM0oEA6G7eYm6nmO45WUPBy+bObYtL5tJhy1Thtz02L2P0CNx7fPauHEjN9xwA4WFhUiS1KmVhzZs2MC4ceMwGAwMGTKE8vLymNczKYXTJEaXFHPzpQPpk9Z2ywtJwiNbaXT58DQlcLpEOHhJ5kzSjXLMW14A+F2xew+hx+IavJxOJ6NHj+bpp5/uVPlDhw4xY8YMpk2bxq5du1i4cCHz5s1L6AU/4iacE6UzZ5JnNXY4xGVbjcJXNc3U1SXwxITh4DVu6CDmXT6IkrwYtoh0ZlRVxeN2xC6XTOixuF42XnfddVx33XWdLv/MM88wcOBAnnjiCQAuuOACPv74Y5YsWcL06dPb3Mfr9baa4bWpqalnlU4WnsbQT1PGOYsarX3wNxzA0ZjAuV7h4HXmMKeY0JnwBhW2Hz7Odl8lP7+ypNcXUxbOLalSJTZv3sxVV13Vatv06dPZvHlzu/uUlZVhs9kij6KiolhXMzF47Jx0eNlZq3DkVMeXP2kZoTGO7qYETZfwu8HvRkXtpeBlRqfVoFGDEBSrCCWqpApeNTU1Z61elJeXR1NTE263u819Fi9ejN1ujzyqqqp6o6rx526k0eXni3oNNU0dr/6cnpEDgM+RoCs/hVtdNR4df9l0lH/ti3ELUdajlXXhJdA8uH0iXSIRpfzdRoPB0O6CISnNY8fjD+JNSyfjHHfmMrJyqAIUVwOBQBA52rOT9lR43GWTlI7DGyCgxGAqnDPpzOg0EnLQg8sfpBfae0IXJVXLKz8/nxMnWncqnzhxAqvViskU5WXfk5kSBG8TnkAQj/bcwSvN1getVoOk+GmwN/ZOHbsiPBtsvRqaCqfdO6fRpDOh02rQKW7R8kpQSRW8SktLWb9+fatt69ato7S0NE41SlDeJgLBIF5Fg19rJsPU8Zdd0urQm0LTzNgbErDTPjwPf50SusOYae6N4GVG1kroguKyMVHFNXg5HA527drFrl27gFAqxK5duzhyJLSK8+LFi1stpfazn/2MgwcP8h//8R98/fXX/PnPf+aVV17hl7/8ZTyqn7jcjXj8oUUqLEYdevncH3N+fgGDc9LI17XddxhXzjoCihIJXu0m3EZTuNNeVjy4fGIh5UQU1z6vbdu2MW3atMjzRYsWAaH1IcvLy6muro4EMoCBAwfy9ttv88tf/pKlS5fSr18/nn/++XbTJM5bLf1d2oxOZ6Ln5+WDcgKU5hhXrosCPvDYcXqDuHRZpBvPnh0jJnRmMsw6RlplbDbRJZGI4hq8pk6d2uE85G1lz0+dOpWdO3fGsFYpwNOIN6Dgla2dz0RvSUFItBlVwwuINKt6AlojuVZj77yvzkSfNAN9cvXQR4xvTEQpf7fxvOSx0zfDRHb/Yfj7ZndqF9WUQZPbj/34MfKHKp261OwV4QVENJYc+qWZ6JvRS8FL3zJEKAEvowVABK/UFG49GSx9MBg69xFLpiz21zbjUn3onF4KEuVSKXynsbCgiJuG9mKCccsQIVcTToe3d/rZhC5JkD+vQlR1Z6EKUwYmvYyseKhvTKAhVC1z66d1rgUZNToTLn+QLV8f5fXtR3v3vYVOEcEr1QT9BDzNfF3TxMYqf+fXNpQNGMwWIIHSJVQVmmvwBxU8xt4OXuZv87z8wdisESn0iAheqcZjx+NXOOmBipP+Lg0oNqaHAoQjUaaE9jnA56Smycuz2xyxHxZ0On0askZCq/iRggGxilACEsEr1bgbw2kSVmxdzEQ3hQdoe5oSpOXVHBpNcVJNR9HIWGO14EZbZCMarQ5ZI6FTXDhFrlfCEcEr1XjsuP1BPHJ6lyfss4QHaKuuxsTIKnfUoKgq1Uqo764woxdvIkgS6NPQyxp0QRdOrwheiUYEr1TjacDrD+KV08nsYstLn5aFQdZgDDRx0pEAk/A11+DwBrBr+2DSa3tnTOPpDBb0Wg36oAuHCF4JR6RKpJrw0CCPuQsJqi1MmQzMTmOwSYOpt5JBO9JcQ6PLj1OfTf8sM5pYLXXWHn0aOlmDLujG6U2AlqjQigheqSay0Gx61xepMGWGBz17QAoS14a5txm8zTS6/bgsfejfJ63366BPJ9Os40KbhgxbAgRzoRURvFKJqhJwNQLgka3nnE3iLDoTyAYIeEOJrpacqFex0+zH8AUV6hQbQY2e/vEYoqNPCw0RypEhSwwRSjSizyuV+N3Iip+LB2Qy94rRXR/iI0kEjBnUNnvYVnEwvrlNTUfRShIlQ4czvn8maZ0cKRBVhlDeGz5n77+3cE4ieKWS8KIbksGKydi94SySMYODdU72H66Kbye1/ShajUTxgBImD41TC1BvQVVV3M5Gas8xlbbQ+0TwSiUtM0J0ZVjQGbRpWZj02vAdR1906tVVQX8kxwtb3/jUAUCfhi+o8FlFFf/4tEpk2ScYEbxSiaeRylNOPq1RqKrv5oKp5j6Y9VpMgTi2NpqrOWF3ccSpwaWxxKcOAHpLeIiQB0UJ4kqE3DchQgSvVOJuxO7xU+nSd3+5LnM2FoOMyddwzlWHYkU5dYhjjW52N1uprI/jlDQ6MxqNFr0G9CJRNeGI4JVCVHdDKMdLDt3i7xZzHywGGZ3i5mRDQ1wuleqqvsEbUPCkF1GSG8eWl0YDhnR0sgZ90CESVROMCF4pxOdoIKio+GQrtq4mqLaQ9ZhtfZAA1XGK5t7+wvo91B0/BEDfgSORtXH+FTVa0Ws1GAIOkaiaYBIieD399NMMGDAAo9HIJZdcwqefftpu2fLyciRJavUwGkUCIUoQT3jRWJ0lq0dfeq0lB7NBi9nfQF1z7w4TOlH1Dc1uH16djVGD+/Xqe7fJYMUgazAEnTR5/PGujXCauCepvvzyyyxatIhnnnmGSy65hKeeeorp06dTUVFBbm5um/tYrVYqKioiz7sy7UvK8tjx+gMokhZLevfvNgJg7sPgHAtDi40Yc3r3su1AxR4kwFZYQnpvziLRHqMVg6xF73PQ5BbBK5HEveX15JNPcuedd3LbbbcxYsQInnnmGcxmM8uXL293H0mSyM/Pjzzy8vJ6scYJylUf7u+ykZnWwymL07JJ08sYvQ3RqVsnHWtw4T9RgQQMGT6mV9+7XUYb6SaZCzIVSvLi2P8mnCWuwcvn87F9+3auuuqqyDaNRsNVV13F5s2b293P4XDQv39/ioqKmDlzJnv37m23rNfrpampqdUjJbnrUVSVgCHjnCtkn5M5PGups3fn9bL66+hrCpCdkU56wZBefe92GaykG3QMz1QZkpse79oIp4lr8Dp58iTBYPCsllNeXh41NTVt7jNs2DCWL1/Om2++yUsvvYSiKEyaNImjR9ueZ7ysrAybzRZ5FBX14iIOvclVz4A+aVwzYQQX9cvo2bHSckCSOHHyFGu2VnDoZO8Mj0lvPsCQ3HQGDx8N2rj3aIS0JPx6UvSPXhKL+2VjV5WWljJnzhzGjBnDlClTWL16NTk5OTz77LNtll+8eDF2uz3yqKqq6uUa95LwQhWSuQ/ank4dI+vB3AenL0DDiSNUnopt8FIUNTRffd03AEg5w2L6fl1isALg87o5drIBj1/ccUwUcf3zlp2djVar5cSJE622nzhxgvz8/E4dQ6fTMXbsWPbv39/m6waDAYPhPFi2yl0f+mnuE53jpedjMx3D4jnZ/Wz9TvpoXx3B+komO0+h1xsga3BM369LZD3oTHx1tJot3q+ZfslFDMiOw/Q8wlni2vLS6/WMHz+e9evXR7YpisL69espLS3t1DGCwSB79uyhoKAgVtVMfAEvp+pP8fnRRjZXRympNL0Am0mHxX+Skw4fDc7YjHOstrv5vKoRV+VOXL4A5I4IBYxEYrSF0iUCzSJdIoHE/bJx0aJF/OUvf+GFF17gq6++4t/+7d9wOp3cdtttAMyZM4fFixdHyj/00EO8//77HDx4kB07dvDjH/+YyspK5s2bF69TiD9XPS5fEHtQjyMYpca0JQ9Zo6GvNrQG5IE6R3SOexpfQOH9vSfQBH0M1x4LzT9WcFHU36fHzFkYdFpMATuNLhG8EkXce0Vnz55NXV0d999/PzU1NYwZM4b33nsv0ol/5MgRNJpvY2xDQwN33nknNTU1ZGZmMn78eDZt2sSIESPidQrx5zqFyxfELWeRHa153i15IEnkGfzoAw721Rq5eEBWdI4dtqGilnqnj8G+bxiYqQ8tLGuN4ywS7TFlYdJpMbobaXDFaaYN4SxxD14ACxYsYMGCBW2+tmHDhlbPlyxZwpIlS3qhVknEWYfbF8RtyCTbEqXgJeshPZ/MwDEynDXU2C3UNXvJSY9O/+FX1U3sPd6EhiCTTYfRaTVQNDG0ak+iMYeDV7Od4zG6fBa6Lu6XjULPBZtP4PEHcemy6GOJ4s2JjGL0Wg3DjA0MjuIA6eONbv75ZegmzTRrNRlab2jW0tyRUXuPqDKF5jgzBezY3X4CQbEAbSIQwSsFOOurUYFgWi5pem30DmwrBmC8rYnvji6MWqtLL2swG2SGZMmM8u0Obew/KXFyu85kykSnlTCpHjRBH41imFBCEMEr2flcuJsbAbD2KYjuOE9bP5A0SO5GcEdvqFC2xcDNE4u41vw1moA71NdVMDZqx486nRFJb6FvpomrBsiYo/kHQug2EbySneMEkgRyWiY5mdboHltnDAUwgJP7cPkCvL27mtrmrk9SeNLh5eBpdyzNTYfR1XweelJydWjurESWlk2hzcQIqxezPkFbiOcZ8SkkO2cduelGcgcNhcHZ0T9+zjBoPAJ1FXzcVMw3J5o55fTyg4uLMOo61wI5WOfg3S9qCARVZo4pZIDRCV+9GXqx3wTIHBD9ekebJRcaKsFRG++aCGEJ/udOOKfm6tBPS4xm1sguCf1sOsZ3ivRYDDKnHD7e2HnsnDOLevxB3t9bw5u7juMLKBRkGMnTNMLnqyDgg4xiGDwtNvWONkseqqpirzvK3uP2eNdGQASvpBdoPIqiqmAtjM0bGG2Q2R9UFfPJPcwa2xejTkuN3cNLWyrZXtlw1tzutc0eNn5Tx18/PsTe401IEowtsvG9wnpMX/wjtA6iJRdG3QiaJOk/suQRVFX2VHzD+1/UhEYDCHElglcy8zo4ceIEnx1uYHNdDMdvFo4L/Ty+kxwTzJ5QRHa6AbcvyMZv6nhu40Hcp62s88FXtWyvbMAXUMhJN/CDC4xM9X2EtuKd0LJmmf1hzC2hFbqThbkPsqzDog1iCDZT29S7M8wKZxN9Xsms6TjNHj8OOYNMUwyXo88eCuYscNVD5SayBk/jRxOL+fJ4E18ct9Pg8mHUfft3MN9mxGLQMsrqpL9rJ9K+r0OzRkgaGHg5FF2a+B30Z9JoIS2HNEMjFm8tJ5oGigHacSaCVxJTGo/S7AngMOcxKiOGrRiNBgZNgy9eh6qtkNkfbdYgLuxn48J+NnwB5dsUDb+bqenHwPE5HK7+9hg5w2DQ1FAQTFa2IiyGQ1i9NVTbxQra8SaCVxJz1OwjoKi4zX3JiWZmfVtyhkL+hVCzB/a8DgO+A7nDQTai9zSB/Sg0HIL6g6CELyE1MuSNgL4XQ3oKTNWdUYTVpCO9qZqvG90EFbXnc6cJ3SaCV7LyOrDXhWaPtRaWoOmNL9HQ6RDwwMl9cHBD6NEWSy7kjYT8i0Afw8vZ3mbrh1mvxaY0oHhd1DR56BvLFq/QIRG8klXDIRpcPpz6bPoX5PTOe2p1MOp7ULMbju+E5hOgKqFkVkt+qCO+TwlYeqk+vU2fhpSWg9XowOY9RlV9XxG84kgEryTlrv4apzeI3daPSb3ZcSxJUDA69FDV0CVioo5JjIU+gymsr+H6zGYy+mfGuzbntSS75SMAEPCiazzIwOw0cgeNid9wFUk6vwIXQPYwLAaZbF8VMmJ2iXgSwSsZ1X2NjEJ+QT8uH5ug08ikKmthaPqegA/qD6CqUZp2W+gyEbySjarC0c9C/84blZiT96UySYK8UQQUhYodH/G3LZUEFRHA4kEErySjnjrA/oOHOOEMEshLwPnezweFY9FoNDiq9+M5dZQvj4s1HeNBBK9kEgxwYuc71Dm8bPH0x02CrbJzvjBloMkfRWGGkf6NW9h8oE6s5xgHCRG8nn76aQYMGIDRaOSSSy7h008/7bD8q6++yvDhwzEajVx44YW88847vVTTOFJVmr9cS9XRKvwaEwWjryTdqIt3rc5fAyeTl5lOnlJLRt023v/yRGjxXKHXxD14vfzyyyxatIgHHniAHTt2MHr0aKZPn05tbdvzJm3atImbb76ZO+64g507dzJr1ixmzZrFF1980cs170XBAKd2vc032z8ioKg4Bk5n/ODzeJ3KRGC0oSm5miG56fRv2oZn30be3n1MtMB6kaTG+XbJJZdcwoQJE/jTn/4EhBadLSoq4uc//zn33XffWeVnz56N0+nkrbfeimy79NJLGTNmDM8888w536+pqQmbzYbdbsdqjfLMo9GiqhDwgrsBZ+0BDn3+MY7Gk6hAY+FkJl9xPWmG8yxFIRGpKhz6iPovP2JfbTNOOQO531gmXzwOTJkoWgMqnDWESFVVFBUkiIyM8AcVnN4Aihp6XQUUVUUjScgaCZNei0HWRvYHojvldwLp7Hc0rt8An8/H9u3bWy0qq9FouOqqq9i8eXOb+2zevJlFixa12jZ9+nTWrFnTZnmv14vX++30JU1Nne9ctbv8/M/nxwBoifAtoV5VVUb2tTEhvJahwxvglc+qIvuqgBxwMaT6fwHITzcwOCcNUPEHAmw7XB/KTo8cNPSQVJWcNC2Ds0JjFeVAkObGRvwaE9Kwa5k24dJOz2AqxJgkwcApZJkyueCLtRysPkU/52ewPbSoSJMXdle7QKsFSQuSBlVVCYZ/h4bnWykMZ+g3OX3sqmpsOXCrtzlqG8+oMRMY3z/0u1bT5OHlz6rQShIajRRKt5MkNOHn4/tnMqYoA4B6p4+3dh9v9xRG9bUxrjiUbGt3+/mfXcfaLTu8wBr5fXd6A6ze2X7ZITkWSgf3AcAbCPLKtqNcOzI/aou4QJyD18mTJwkGg5EFZlvk5eXx9ddft7lPTU1Nm+VramraLF9WVsaDDz7YrfoFVZWTjvbX6XOdNoeVoqrYz1hVRh/wonGdBECSDeAKz0QQVMDT9mycKqAG9IABdCYMffqSZsumcNjFZNvSu3UeQgyFRxxYs4dxUfUeqD8Ajmrwe/D7vegUN+3lsmrc/tDQKkB2+0kPNiPxbfaLROj34aTkRz5tCqGgooYa56oKbfSz+QLfvmFAUTjVwe/w6fOwKUrHv++nTzqpqConm9uf06zAaoz8W1XhZLM36iklKX/tsXjx4lYttaamJoqKijq1r8Ug871x/dpMpZIkSDd822Fu1mmZPaGoVVlJCaJtmgcaCYNOBqMOkNCqUDLCH1qZBw2q1HIJICFJEjqdDqyZobGEgEiISAI6I5riCVA8IfQ84CPb58Di9REI+FADAVRVRdKEWkmyRkLWAOHLRpuqMrGdQ19s7gPGby+fCmwm7pw8CEVVUZTQJejp/04zfNsyzzDp+f74fu1W23raTR+LUW5V9swOJYvx23Bh0mn53rj2j3t6HXRaDd8b148Mc3RvMMU1eGVnZ6PVajlx4kSr7SdOnCA/P7/NffLz87tU3mAwYDB0r6mqlzUU9+ncrAiyVhO5BGglc/hZmzRAthgWl9pkPZKcRSzmiNRqJCyd7PPUyxqKsjpXCZ2282Vlbee/G1qN1OmyXRHXu416vZ7x48ezfv36yDZFUVi/fj2lpaVt7lNaWtqqPMC6devaLS8IQmqK+2XjokWLuPXWW7n44ouZOHEiTz31FE6nk9tuuw2AOXPm0LdvX8rKygC4++67mTJlCk888QQzZsxg1apVbNu2jeeeey6epyEIQi+Le/CaPXs2dXV13H///dTU1DBmzBjee++9SKf8kSNH0JzWWTlp0iRWrlzJf/7nf/Kb3/yGkpIS1qxZw6hRo+J1CoIgxEHc87x6W1LkeQnCeayz39G4Z9gLgiB0hwhegiAkJRG8BEFISnHvsO9tLV18XRkmJAhC72n5bp6rO/68C17Nzc0Anc6yFwQhPpqbm7HZbO2+ft7dbVQUhePHj5Oenn7OUfktQ4mqqqpS5s5kqp1Tqp0PiHNSVZXm5mYKCwtbpUmd6bxreWk0Gvr1a39MVlusVmvK/BK1SLVzSrXzgfP7nDpqcbUQHfaCICQlEbwEQUhKInh1wGAw8MADD3R7VopElGrnlGrnA+KcOuu867AXBCE1iJaXIAhJSQQvQRCSkghegiAkJRG8BEFISiJ4naG+vp5bbrkFq9VKRkYGd9xxBw6Ho8N9pk6diiRJrR4/+9nPeqnGZ0u1Fci7cj7l5eVnfRZGo7Hd8vGwceNGbrjhBgoLC5Ekqd1l+063YcMGxo0bh8FgYMiQIZSXl8e8np3V1fPZsGHDWZ+RJEntrgDWHhG8znDLLbewd+9e1q1bx1tvvcXGjRu56667zrnfnXfeSXV1deTx3//9371Q27Ol2grkXT0fCGVxn/5ZVFZW9mKNz83pdDJ69GiefvrpTpU/dOgQM2bMYNq0aezatYuFCxcyb9481q5dG+Oadk5Xz6dFRUVFq88pNze3a2+sChFffvmlCqifffZZZNu7776rSpKkHjt2rN39pkyZot599929UMNzmzhxojp//vzI82AwqBYWFqplZWVtlv/BD36gzpgxo9W2Sy65RP3pT38a03p2VlfPZ8WKFarNZuul2vUcoL7xxhsdlvmP//gPdeTIka22zZ49W50+fXoMa9Y9nTmfDz/8UAXUhoaGHr2XaHmdZvPmzWRkZHDxxRdHtl111VVoNBq2bt3a4b5///vfyc7OZtSoUSxevBiXyxXr6p6lZQXyq666KrKtMyuQn14eQiuQt1e+N3XnfAAcDgf9+/enqKiImTNnsnfv3t6obswk8mfUE2PGjKGgoICrr76aTz75pMv7n3cDsztSU1NzVtNVlmWysrI6vB7/0Y9+RP/+/SksLGT37t38+te/pqKigtWrV8e6yq30xgrkvak75zNs2DCWL1/ORRddhN1u5/HHH2fSpEns3bu3ywPyE0V7n1FTUxNutxuTqY31QhNYQUEBzzzzDBdffDFer5fnn3+eqVOnsnXrVsaNG9fp45wXweu+++7jscce67DMV1991e3jn94nduGFF1JQUMCVV17JgQMHGDx4cLePK3RdaWlpqzU8J02axAUXXMCzzz7Lww8/HMeaCS2GDRvGsGHDIs8nTZrEgQMHWLJkCX/72986fZzzInj96le/Yu7cuR2WGTRoEPn5+Wd1BAcCAerr69tdkbstl1xyCQD79+/v1eDVGyuQ96bunM+ZdDodY8eOZf/+/bGoYq9o7zOyWq1J1+pqz8SJE/n444+7tM950eeVk5PD8OHDO3zo9XpKS0tpbGxk+/btkX0/+OADFEWJBKTO2LVrFxBqHvemVFuBvDvnc6ZgMMiePXt6/bOIpkT+jKJl165dXf+MetTdn4KuvfZadezYserWrVvVjz/+WC0pKVFvvvnmyOtHjx5Vhw0bpm7dulVVVVXdv3+/+tBDD6nbtm1TDx06pL755pvqoEGD1MmTJ8el/qtWrVINBoNaXl6ufvnll+pdd92lZmRkqDU1NaqqqupPfvIT9b777ouU/+STT1RZltXHH39c/eqrr9QHHnhA1el06p49e+JS/zN19XwefPBBde3ateqBAwfU7du3qz/84Q9Vo9Go7t27N16ncJbm5mZ1586d6s6dO1VAffLJJ9WdO3eqlZWVqqqq6n333af+5Cc/iZQ/ePCgajab1XvvvVf96quv1KefflrVarXqe++9F69TaKWr57NkyRJ1zZo16r59+9Q9e/aod999t6rRaNR//vOfXXpfEbzOcOrUKfXmm29WLRaLarVa1dtuu01tbm6OvH7o0CEVUD/88ENVVVX1yJEj6uTJk9WsrCzVYDCoQ4YMUe+9917VbrfH6QxU9Y9//KNaXFys6vV6deLEieqWLVsir02ZMkW99dZbW5V/5ZVX1KFDh6p6vV4dOXKk+vbbb/dyjTvWlfNZuHBhpGxeXp56/fXXqzt27IhDrdvXkipw5qPlPG699VZ1ypQpZ+0zZswYVa/Xq4MGDVJXrFjR6/VuT1fP57HHHlMHDx6sGo1GNSsrS506dar6wQcfdPl9xZQ4giAkpfOiz0sQhNQjgpcgCElJBC9BEJKSCF6CICQlEbwEQUhKIngJgpCURPASBCEpieAlCEJSEsFLSHnl5eVkZGREnv/2t79lzJgxkedz585l1qxZvV4voWdE8BKiZu7cue3O3z9//nwkSWo1u0csgsaAAQN46qmnWm2bPXs233zzTbv7LF26tNWc8FOnTmXhwoVRrZcQfSJ4CVFVVFTEqlWrcLvdkW0ej4eVK1dSXFwclzqZTKYO50e32WytWmZCchDBS4iqcePGUVRU1GoW2dWrV1NcXMzYsWN7dOy2WkSzZs2KtOamTp1KZWUlv/zlLyMr0sDZl41nOr0FOHfuXD766COWLl0aOcahQ4cYMmQIjz/+eKv9du3ahSRJST1XWDITwUuIuttvv50VK1ZEni9fvpzbbrst5u+7evVq+vXrx0MPPRRZkaarli5dSmlpaavVoIqLi886J4AVK1YwefJkhgwZEq1TELpABC8h6n784x/z8ccfU1lZSWVlJZ988gk//vGPY/6+WVlZaLVa0tPTyc/P79ZssDabDb1ej9lsjhxDq9Uyd+5cKioqImtG+v1+Vq5cye233x7t0xA66byYBlroXTk5OcyYMYPy8nJUVWXGjBlkZ2fHu1o9UlhYyIwZM1i+fDkTJ07kf//3f/F6vdx0003xrtp5S7S8hJi4/fbbKS8v54UXXoha60Sj0XDm9HN+vz8qx+6MefPmRW5GrFixgtmzZ2M2m3vt/YXWRPASYuLaa6/F5/Ph9/uZPn16VI6Zk5PTqh8rGAyetbK3Xq8nGAz26H3aO8b1119PWloay5Yt47333hOXjHEmLhuFmNBqtZHl5LRabbvl7HZ7ZMGSFn369KGoqOissldccQWLFi3i7bffZvDgwTz55JM0Nja2KjNgwAA2btzID3/4QwwGQ7cuVwcMGMDWrVs5fPgwFouFrKwsNBpNpO9r8eLFlJSUpNQCGMlItLyEmLFarVit1g7LbNiwgbFjx7Z6PPjgg22Wvf3227n11luZM2cOU6ZMYdCgQUybNq1VmYceeojDhw8zePBgcnJyulXve+65B61Wy4gRI8jJyeHIkSOR1+644w58Pl+v3D0VOibmsBeELvjXv/7FlVdeSVVV1VmrWAu9SwQvQegEr9dLXV0dt956K/n5+fz973+Pd5XOe+KyURA64R//+Af9+/ensbGR//7v/453dQREy0sQhCQlWl6CICQlEbwEQUhKIngJgpCURPASBCEpieAlCEJSEsFLEISkJIKXIAhJSQQvQRCS0v8H6dP0LhnO38IAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:07.526254Z", + "iopub.status.busy": "2024-03-22T22:04:07.525983Z", + "iopub.status.idle": "2024-03-22T22:04:07.908204Z", + "shell.execute_reply": "2024-03-22T22:04:07.907337Z" + }, + "papermill": { + "duration": 0.403204, + "end_time": "2024-03-22T22:04:07.910150", + "exception": false, + "start_time": "2024-03-22T22:04:07.506946", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABK5ElEQVR4nO3de3xU1bnw8d+e2XNNMpOE3Em4x3AngIKhVcBLEamFY4961Ap4obbFc6TUVvH00Ld6fFPbonJOPVDrAWqVUq2Ib9WKFAWrIuUuoIZ7EiAJgSSTzP2y9/vHTEYCSchlJnNhfT+f+UD27L1nbUKerL32s54lqaqqIgiCkGA0sW6AIAhCT4jgJQhCQhLBSxCEhCSClyAICUkEL0EQEpIIXoIgJCQRvARBSEgieAmCkJDkWDegrymKwunTp0lLS0OSpFg3RxCEC6iqSktLCwUFBWg0HfevLrvgdfr0aYqKimLdDEEQLqG6uprCwsIO37/sgldaWhoQ/IexWCwxbo0gCBdqbm6mqKgo/LPakcsueLXeKlosFhG8BCGOXWpYRwzYC4KQkETwEgQhIYngJQhCQrrsxrwEoSOqquL3+wkEArFuSlLTarXIstzrVKWYBq8VK1awYsUKTpw4AcCoUaNYunQpM2fObHf/NWvWcO+997bZZjAYcLvd0W6qkOS8Xi81NTU4nc5YN+WyYDabyc/PR6/X9/gcMQ1ehYWF/OIXv6C4uBhVVfn973/P7Nmz2bNnD6NGjWr3GIvFQkVFRfhrkWgq9JaiKBw/fhytVktBQQF6vV78v4oSVVXxer3U19dz/PhxiouLO01E7UxMg9ctt9zS5uunnnqKFStW8Omnn3YYvCRJIi8vry+aJySRBlsLW440kmMx87Vh/doEJ6/Xi6IoFBUVYTabY9jKy4PJZEKn01FZWYnX68VoNPboPHEzYB8IBFi3bh0Oh4OysrIO97Pb7QwcOJCioiJmz57NwYMHOz2vx+Ohubm5zUu4vKjnjlL11i/ot/9FPjtygsNn7O3u19MegNB9kfi3jvl3a//+/aSmpmIwGPje977HG2+8wciRI9vdt6SkhFWrVvHmm2/y8ssvoygKU6ZM4eTJkx2ev7y8HKvVGn6JqUGXGVXFfuCvuNwe9AEn/W17OHDKFutWCREgxXr1IK/XS1VVFTabjT//+c+8+OKLbN26tcMAdj6fz8eIESO48847efLJJ9vdx+Px4PF4wl+3Tj2w2Wwiw/5y0FzD6c0rqGxwYtZrsaaloUz5N4bkfPW9d7vdHD9+nMGDB/f4Fkbons7+zZubm7FarZf8GY15qoRer2fYsGEATJw4kR07drB8+XJ++9vfXvJYnU7H+PHjOXLkSIf7GAwGDAZDxNorJJimSprdPhpNAxiQ1kz/VC2YXID4xZXoYn7beCFFUdr0lDoTCATYv38/+fn5UW6VkLCaqlFUcJgKMGcNCG5rPhXbNkXQtGnTWLRoUaybERMx7XktWbKEmTNnMmDAAFpaWli7di1btmxh48aNAMydO5f+/ftTXl4OwBNPPMHVV1/NsGHDaGpq4le/+hWVlZU88MADsbwMIZ7Z6xiZb2F46SSkxuP4jlXTcOo4XtNwijLFk8VEFtOe15kzZ5g7dy4lJSVcf/317Nixg40bN3LjjTcCUFVVRU1NTXj/xsZGFixYwIgRI7j55ptpbm7mk08+6dL4mHAZ8rnB0wKAJjUHKTWHJqePfYeOsf14Q5dO4fUrHb78AaXL+/q6uG93zJ8/n61bt7J8+XIkSUKSJAoLC1mxYkWb/fbs2YNGo6GyshKAZ555hjFjxpCSkkJRURE/+MEPsNvbPoH96KOPuOaaazCZTBQVFfFv//ZvOByObrUv2mI+YN/XujoYKCSBpmrY8zIY0mDKQ2A/Q8tHK9lb4+HLIfexYOpQoPPB42c3Herw9IOzUpgzvn/469+8fxhfoP0fp8IME7dd+dWT7pVbj+LyXjwN6Yc3XtHly7PZbMycOZPRo0fzxBNPALBs2TI+/fRT/v73v4f3e+SRR9i+fXt423PPPce4ceMYPHgwx44d4wc/+AHXXXcd//M//wPA0aNHGTduHP/5n//JrFmzqK+v56GHHmLcuHGsXr26y+3rTCQG7ONuzEsQIsZ5lupGJx/XShw8bQNTBkadjKx4cLvs3e7pxBur1Yper8dsNpOXl0deXh533303H3/8MVVVVUBwDHndunXcfffd4eMWLVrE9OnTGTRoENdddx3/+Z//yauvvhp+v7y8nLvvvptFixZRXFzMlClT+K//+i9eeumluJqKF/OnjYIQNc5zODx+6rQppARU0OrQma3otI0Y/TaaXF5y0jpPjVg4fViH72kumEH03WuHdrjvhbON7vva4Es2vydKS0sZMWIEa9eu5bHHHmPr1q2cOXOG2267LbzP3/72N8rLy/nyyy9pbm7G7/fjdrtxOp2YzWb27dvHZ599xiuvvBI+RlXV8DSqESNGRKXt3SV6XkLyctvw+BU8soUMsy64zWhFL2sw+O3Y3f5LnkIvazp8yVpNl/fVdXHfSLj77rtZu3YtAGvXruWmm26iX79+AJw4cYJvfvObjB07ltdff51du3bx/PPPA8GcSwjOYnnwwQfZu3dv+LVv3z4OHz7M0KEdB+i+JnpeQvJy2/D6FdxyGmnGUPAyWNBrNRgCdlq6ELzinV6vv6iEz1133cVPf/pTdu3axZ///GdWrlwZfm/Xrl0oisKyZcvCU3TOv2UEmDBhAp9//nk4/zJeiZ6XkLT8jkb8iopHm0aqIfR72mjBIGuDPS9P4gevQYMGsX37dk6cOMHZs2dRFIVBgwYxZcoU7r//fgKBAN/61rfC+w8bNgyfz8d///d/c+zYMf7whz+0CW4Ajz76KJ988gkPPfQQe/fu5fDhw7z55ps89NBDfX15nRLBS0hOPjded6g2l8n61S2Z0UpWmp6r8rUMz+t8dZpE8Mgjj6DVahk5ciTZ2dnhgfq7776bffv28U//9E+YTKbw/uPGjeOZZ57h6aefZvTo0bzyyivhPMpWY8eOZevWrRw6dIhrrrmG8ePHs3TpUgoKCvr02i5FpEoIyamljqa/r+SzOj+Vw+/nnrJBwe3njsJnr0JKFkxaAIi5jbEgUiUEoSPuJlQV5JR0MlLOq9ZpCP0whJJXhcQlBuyF5ORpIcOsZ8rAwTD6vNsdQyqKqmJrbuFk1TlGF2XGro1Cr4iel5CcWntWhgvGtWQjqqTly9oWPjxYiSfBE1UvZyJ4CcnJG5qHp09tu12S0BpT0Wok9AEHznam6AiJQQQvITl57Rw8beOtihbOtFwwpUWfgl4roQu4cHoTP13iciWCl5CcPC24fAHOuGU0F87N0aei02rQB5ztTo4WEoMIXkJSUjx2fAEVr9aMWa9t+6Y+BVmrQae4xG1jAhPBS0g+AT9+TzBB1SebMcoXBy+dVkIXcOIQt40JSwQvIfl47fgCCoqkxWA0o7mw/IM+BZ1Wgy7gEreNCUwELyH5hIKXT2vGZNBd/L4+lcwUPRPydYwtTO/z5kVSvNWw78v2iOAlJB+PHZ8/NN6l0178vs5Eil4m3wzZaWJlqdZSOIlGBC8h+XjtSBLozRbSze30vHShhTd8zr5tV4S1V8P+6NGj3H///QwePBiTyURJSQnLly+/6Lg5c+bw1FNPUVBQQElJCQCffPIJpaWlGI1GrrzySjZs2IAkSezduzd87IEDB5g5cyapqank5uZyzz33cPbs2Q7bc+LEiahdf0yD14oVKxg7diwWiwWLxUJZWRl//etfOz3mtddeY/jw4RiNRsaMGcM777zTR60VEoanhaxUA9PGDOH6EbkXvy8bUVSVBlsLB042tn8OVQW/NzavLtZKWL58OWVlZSxYsICamhpqamooLCyksLCQ1157jc8//5ylS5fy+OOPX1Sza/PmzVRUVLBp0ybeeustmpubueWWWxgzZgy7d+/mySef5NFHH21zTFNTE9dddx3jx49n586dvPvuu9TV1XH77bd32J5orlAf07mNhYWF/OIXv6C4uBhVVfn973/P7Nmz2bNnD6NGjbpo/08++YQ777yT8vJyvvnNb7J27VrmzJnD7t27GT16dAyuQIhLHWXXt9KZUFSVitpmdmqrGHRN8cX7BHzw92XRa2NnrvkRyPpL7nZhDftWP//5z8N/Hzx4MNu2bePVV18NBxmAlJQUXnzxRfT64OesXLkSSZL43e9+h9FoZOTIkZw6dYoFCxaEj/nNb37D+PHj+b//9/+Gt61atYqioiIOHTrEFVdc0W57oiWmPa9bbrmFm2++meLiYq644gqeeuopUlNT+fTTT9vdf/ny5dx00038+Mc/ZsSIETz55JNMmDCB3/zmN33cciGuhYNXB+syarRo9SYkQKe48fiT64nj888/z8SJE8nOziY1NZUXXnghXOer1ZgxY8KBC6CiooKxY8e2KU8zadKkNsfs27ePDz74gNTU1PBr+PDhQHDFob4WN1UlAoEAr732Gg6Hg7Kysnb32bZtG4sXL26zbcaMGWzYsKHD83o8njYrcDc3N0ekvUIc8zn5oqaZCqWJyWYX+VbTRbtIOjOyVkJWPHj87dymaXXBHlAsaNsZp+uidevW8cgjj7Bs2TLKyspIS0vjV7/6Fdu3b2+zX0pKSrfPbbfbueWWW3j66acvei8Wq9bHPHjt37+fsrIy3G43qampvPHGGx0uIltbW0tubtsxjNzcXGprazs8f3l5eZtutHAZ8Llw+QLUuzQXTw1qpTMhazTIATceXzs9L0nq0q1brF1Yw/7jjz9mypQp/OAHPwhv60qvqKSkhJdffhmPx4PBEHwCu2PHjjb7TJgwgddff51BgwYhy+2HjvZq6kdLzJ82lpSUsHfvXrZv3873v/995s2bx+effx6x8y9ZsgSbzRZ+VVdXR+zcQnxSvY5gnpfGiOnCqUGtQj2v4G1j4pbFubCGfXFxMTt37mTjxo0cOnSI//iP/7goCLXnrrvuQlEUvvvd7/LFF1+wceNGfv3rXwMghX4BLFy4kIaGBu6880527NjB0aNH2bhxI/fee284YLVXUz9aYh689Ho9w4YNY+LEiZSXlzNu3LiLHu22ysvLo66urs22urq6TgcHDQZD+Glm60tIYgEfAb8XRQW/1tR+nheEel4SsuJO6MVnL6xhP2PGDG699VbuuOMOJk+ezLlz59r0wjpisVj4y1/+wt69eyktLeXf//3fWbp0KUB4HKygoICPP/6YQCDAN77xDcaMGcOiRYtIT08Pr0TUUU39aIj5beOFFEVpM0Z1vrKyMjZv3twmg3fTpk0djpEJlyGfE19ARZU0aHWGi9ZWDNMZkbWh28ZAgJ6PMsXWFVdcwbZt29psW716NatXr26z7fxFNtasWdPuuaZMmcK+ffvCX7/yyivodDoGDBgQ3lZcXMz69eu71Z5oiWnwWrJkCTNnzmTAgAG0tLSwdu1atmzZwsaNGwGYO3cu/fv3D//DP/zww0ydOpVly5Yxa9Ys1q1bx86dO3nhhRdieRlCPPE68YdvGTv5760zk5Nm4Op0M9oMM411HeR7XUZeeuklhgwZQv/+/dm3bx+PPvoot99+e5vVh+JJTIPXmTNnmDt3LjU1NVitVsaOHcvGjRu58cYbAaiqqgp3RyH4m2Ht2rX89Kc/5fHHH6e4uJgNGzaIHC/hKz4nPkXFrzFh7OiWEUBnwmLUYUkFt1mPCF3BB2JLly6ltraW/Px8brvtNp566qlYN6tDYukzIbnUHqBx1+scdGXSUvLPzBzTwSP8+go4sB6s/XGPvF0sfdbHIrH0WdyNeQlCr/icZJj1fH1QEYzqJPdIZ8IXUGhqaKKpwdF37RMiJuZPGwUholonW+svkYQpm3B6A3xZfYbdlU1Rb5YQeSJ4CcnF5wr+qbvEILPOiKyR0CqecJLqZTaCElOR+LcWwUtILl4Hx+rtvPNlE0fOdLIqtmxEq5WQUHF6gvWsnM7ELpGTSFr/rXW6niepiDEvIbm0Tg3yyAxROvntrpGR5eAPTsDvJc2Sx5kzZwAwm83hrHIhslRVxel0cubMGdLT09FqO3kifAkieAnJxefCr6j4NUZMnaVKSBJavQmNBLLiJi0zC61GCgcwIbrS09N7XTZHBC8hufgc+AMqPo2x8zwvQJKNoSlCXjx+hfz8fHJycvD5fH3U2MuTTqfrVY+rlQheQvJQFFSfG7+i4NdeOnghB6cPyYoHjy84v1Gr1UbkB0uIPhG8hOThd6EoKopKqOd1iedROhNFGSYsg9LITIn/8jdCWyJ4CcnD6wz2ujQGtFot+o4mZbeSDWSmGMi0ymAQPwqJRqRKCMnD50RRwWBKISNFf+knhnJoWkprbpiQUMSvGyF5+FyYdFomXVHIpAkDL72/bMTlC9B4rgk5y02ORcxrTCQieAnJwxeao3ip7PpWspGzLR72NtSQmtYsgleCEbeNQvIITw3qYNWgC8kGtFoJOeBJuhWELgei5yUkD5+LGpuLA44WMowNXDkos/P9Q6WgtaoXVwKXgr5cieAlJA+vA49f4VxAi8nXhWAkG0JJql/leQmJQ9w2CsnD5wpm12u7kOMFwcnZmlCSqrhtTDgieAnJwxfM8/JdqgR0K7m1LI43oZc/u1zFNHiVl5dz1VVXkZaWRk5ODnPmzKGioqLTY9asWYMkSW1eonSvAASDVyA4KburwUurkdCoAXze9lesEuJXTIPX1q1bWbhwIZ9++imbNm3C5/PxjW98A4ej87K8FouFmpqa8KuysrKPWizELVUFnwtfaF5jh4vNnk82oJO1DM5K4fpiqyhGmGBiOmD/7rvvtvl6zZo15OTksGvXLq699toOj5MkqdflNIQkE/CCEghVlDBhlLvwe1mS0OqN5Fkgr58eRA2vhBJXY142mw2AzMzOH3Hb7XYGDhxIUVERs2fP5uDBg33RPCGeeR2oqMg6PSaToWs9LxBThBJY3AQvRVFYtGgRX/va1zpdh7GkpIRVq1bx5ptv8vLLL6MoClOmTOHkyZPt7u/xeGhubm7zEpKQz4WExPihBXz32qGYO1tw9nyykRaPj8ozDTi9/ui2UYiouAleCxcu5MCBA6xbt67T/crKypg7dy6lpaVMnTqV9evXk52dzW9/+9t29y8vL8dqtYZfRUVF0Wi+EGvdza5vJRs5Vu/gw8+rOdvijXy7hKiJi+D10EMP8dZbb/HBBx9QWFjYrWN1Oh3jx4/nyJEj7b6/ZMkSbDZb+FVdXR2JJgvxJjyvsbvByxBKl/CJXK8EE9PgpaoqDz30EG+88Qbvv/8+gwcP7vY5AoEA+/fvJz+//QVGDQYDFoulzUtIQj4XTS4vW463sPmLuq4fF0qXkBUPbpFln1Bi+rRx4cKFrF27ljfffJO0tDRqa2sBsFqtmEzBygBz586lf//+lJeXA/DEE09w9dVXM2zYMJqamvjVr35FZWUlDzzwQMyuQ4gDPidev0KDV4fG3Y2xK9mArNGEElVFzyuRxDR4rVixAoBp06a12b569Wrmz58PQFVVFRrNVx3ExsZGFixYQG1tLRkZGUycOJFPPvmEkSNH9lWzhXjkdYZXDbJ0ZWpQq9D6jbLfI7LsE0xMg1dXkgK3bNnS5utnn32WZ599NkotEhLWefMaDV3Jrm/VuoKQKnpeiSYuBuwFodd8TvyB4LzGTtdrvJBsQNs6v1GMeSUUURJHSA6+0G1jV5Y8O59sxGrSMSrFAP2t0WufEHEieAnJIRS8urTk2flkAyl6mRSTBjK7mWYhxJS4bRQSX8APfi9ajYTelIJZ143fya3Tg/zu6LRNiBrR8xISn88JQEmelZKpI7s3wVo2EFBV7C12GurtDM5OjVIjhUgTPS8h8YWnBhm7XxlCNuLzK3x+uom/7quKfNuEqBHBS0h8oZ5Xt6cGAWh1aOXgAL/qcxNQRE2vRCGCl5D4fE68AYWPq5y8vqv96iIdkiS0oXUeRZZ9YhHBS0h8Phe+gEKTT+aco/vlnDW60PxGVeR6JRIRvITE53V8VUG1OzlercKVJcQUoUQigpeQ+Hyu0KpBRoxyT4JXa2UJcduYSETwEhKfL9Tz0powdrX88/nCPS+xBFoiEXleQuLzuUIVJQxdW3jjQrKRPKsRQ46ZrDRD5NsnRIUIXkLi84YmZRt6PubVL8VAv0wZzPrIt0+ICnHbKCQ+nxONFJwalGLo2ZgXAH6x8Gwi6VHwOnbsWKTbIQg9oyjgd1OUaeaea0YycWDny+a1SzbiDSicbWrmTIuY45goehS8hg0bxvTp03n55Zdxu8U3W4ghvyu4WjZAKNm022QDZ+0edh6tYdeJxsi1TYiqHgWv3bt3M3bsWBYvXkxeXh4PPvgg//jHPyLdNkG4NG/r1CAjaHpwywhfVVNVPHgD4mljouhR8CotLWX58uWcPn2aVatWUVNTw9e//nVGjx7NM888Q319faTbKQjtC81r3F3j5dWd1dhcvu6f4/xUCZFhnzB6NWAvyzK33norr732Gk8//TRHjhzhkUceoaioiLlz51JTU9Pp8eXl5Vx11VWkpaWRk5PDnDlzqKiouOTnvvbaawwfPhyj0ciYMWN45513enMZQiLzOVFROefRcqrRhVbTzaoSEEpS1Ygk1QTTq+C1c+dOfvCDH5Cfn88zzzzDI488wtGjR9m0aROnT59m9uzZnR6/detWFi5cyKeffsqmTZvw+Xx84xvfwOFwdHjMJ598wp133sn999/Pnj17mDNnDnPmzOHAgQO9uRQhUfmcBBQVryb4xLBneV7nTQ/yieCVKCS1K0v4XOCZZ55h9erVVFRUcPPNN/PAAw9w8803t1mi7OTJkwwaNAi/v+tr6NXX15OTk8PWrVu59tpr293njjvuwOFw8NZbb4W3XX311ZSWlrJy5cpLfkZzczNWqxWbzSYWoE0Gx/+O+/BWNjbmUZ09lYeuK+7+Ofxe3O//kj3VTewddD/fv35E5NspdFlXf0Z71PNasWIFd911F5WVlWzYsIFvfvObbQIXQE5ODv/7v//brfPabDYAMjM7fty9bds2brjhhjbbZsyYwbZt27r1WUKSCGXX93hSNgRremmDxypel6jplSB6lGG/adMmBgwYcFHAUlWV6upqBgwYgF6vZ968eV0+p6IoLFq0iK997WuMHj26w/1qa2vJzc1tsy03Nze82vaFPB4PHs9XyYfNzc1dbpOQAHyOYHZ9d1cNOp8kodWbGJBpxjJU9MYTRY96XkOHDuXs2bMXbW9oaGDw4ME9asjChQs5cOAA69at69HxHSkvL8dqtYZfRUVFET2/EGOR6HkRrOnVP93EqBxDzwb9hT7Xo+DV0TCZ3W7HaDR2+3wPPfQQb731Fh988AGFhYWd7puXl0ddXV2bbXV1deTl5bW7/5IlS7DZbOFXdXV1t9snxDGvAxXQGc2k9mRqUCs5NCFbTBFKGN26bVy8eDEAkiSxdOlSzOavaoYHAgG2b99OaWlpl8+nqir/+q//yhtvvMGWLVu61GsrKytj8+bNLFq0KLxt06ZNlJWVtbu/wWDAYBCVApKWz0V2qoE7rxwBabmX3r8jshGXL4CtoQmLxY9ZL2oWxLtufYf27NkDBIPO/v370eu/moGv1+sZN24cjzzySJfPt3DhQtauXcubb75JWlpaeNzKarViMgWnesydO5f+/ftTXl4OwMMPP8zUqVNZtmwZs2bNYt26dezcuZMXXnihO5ciJANVPW/loB5ODWolGzhWb2efvZqrrCUU56b1vn1CVHUreH3wwQcA3HvvvSxfvrzXqQYrVqwAYNq0aW22r169mvnz5wNQVVXV5sHAlClTWLt2LT/96U95/PHHKS4uZsOGDZ0O8gtJyu8GNZQR35OVg84XrqYqSkEnih71jVevXh2RD+9KitmWLVsu2nbbbbdx2223RaQNQgILzWs80uBl9+4ayob2oyizh0GsTTVVkaiaCLocvG699VbWrFmDxWLh1ltv7XTf9evX97phgnBJoXmNTX49p5pcvZtU3TpFKOAR8xsTRJeDl9VqRQqtRmy1WqPWIEHoslDwchEcezX3pH59K12osoRf1LFPFF0OXuffKkbqtlEQeiUUvJxqMHiZepHnhWxEqxWLcCSSHuV5uVwunE5n+OvKykqee+453nvvvYg1TBAuyesgoKq4peCTRlNvel7n1fQSY16JoUfBa/bs2bz00ksANDU1MWnSJJYtW8bs2bPDTxAFIeq8DnwBBZ/GhFYjodf2okiKbCDFIFPcT8eoAjEskgh6XEn1mmuuAeDPf/4zeXl5VFZW8tJLL/Ff//VfEW2gIHTIa8cfUPFqzZj12vCYbI/IRlL0MgMtWoblpEaujULU9Ch4OZ1O0tKCSXzvvfcet956KxqNhquvvprKysqINlAQOuR1oqgqOlMqKYZeZsSfPz2o+1WihBjo8QIcGzZsoLq6mo0bN/KNb3wDgDNnzogaWULf8TqwGHXcVjacOycN6N25ZCOKquLweDl5zhaZ9glR1aPgtXTpUh555BEGDRrE5MmTw/MK33vvPcaPHx/RBgpCh7z24J/6lN6fSyMTQMNnJ228ueMYiqjpFfd61Nf+53/+Z77+9a9TU1PDuHHjwtuvv/56/umf/ilijROEDvm9EAgtthGJ4BWq6QWgVbx4AwrGnq5GJPSJHg8U5OXlXVSGZtKkSb1ukCB0SajXVWnzsn13HeMHZvR6MrVGZ0QrEUyX8Cm9qg8mRF+PgpfD4eAXv/gFmzdv5syZMyhK26Q+saK2EHXe4CItzQEDp2xuRkRiSo9sCK0i1Jrrpev9OYWo6VHweuCBB9i6dSv33HMP+fn5vXtELQg9EZ4aFCx+adL3aiGsINmIrJXQqiLLPhH0KHj99a9/5e233+ZrX/tapNsjCF0Tum10hoNXBIoHyoZQWRxRWSIR9OjXVUZGRqcr/AhC1IVuGx2RmNfYKjRFSKt4cIvKEnGvR8HrySefZOnSpW3mNwpCn/I6UFQVJ8EnhL2qKNFKNpCVamBcnpEciygdHu961NdetmwZR48eJTc3l0GDBqHTtR3Y3L17d0QaJwgd8jrCqwZpJAlDT1bKvpBsJCvVQFaWHtK6v5CM0Ld6FLzmzJkT4WYIQjd5HQQUFdmURoqhl/MaW4WnCLl7fy4h6noUvH72s59Fuh2C0D1eByadltsnD0e1FETmnLIRv6LgtNvxO7xkpugvfYwQMz3uazc1NfHiiy+yZMkSGhoagODt4qlTp7p8jg8//JBbbrmFgoICJEliw4YNne6/ZcsWJEm66NXRatlCklLV8IA9enPkUnVkI+fsXv5xuIa/H66PzDmFqOlRz+uzzz7jhhtuwGq1cuLECRYsWEBmZibr16+nqqoqXOvrUhwOB+PGjeO+++67ZF3881VUVLSZAJ6Tk9PtaxASmM8Fij/4d30ElyiTDchasYJQouhR8Fq8eDHz58/nl7/8Zbg0DsDNN9/MXXfd1eXzzJw5k5kzZ3b783NyckhPT+/2cUKSCOV4VdtVtu2uYXR/KyMLIlDNJLT8mSgFnRh6dNu4Y8cOHnzwwYu29+/fv09u4UpLS8nPz+fGG2/k448/7nRfj8dDc3Nzm5eQ4DwtADQHjJxqcuHyRSihVDYgazTBJFWvPzLnFKKmR8HLYDC0GwQOHTpEdnZ2rxvVkfz8fFauXMnrr7/O66+/TlFREdOmTes0NaO8vByr1Rp+FRUVRa19Qh8JBS+7FFyjMcUQoQnUoZ4XqPh8nsicU4iaHgWvb33rWzzxxBP4fMGSJJIkUVVVxaOPPsq3v/3tiDbwfCUlJTz44INMnDiRKVOmsGrVKqZMmcKzzz7b4TFLlizBZrOFX9XV1VFrn9BHQsHLEZoalBKJqUEAGhmtHDyX4nWLml5xrkfBa9myZdjtdrKzs3G5XEydOpVhw4aRlpbGU089Fek2dmrSpEkcOXKkw/cNBgMWi6XN67JmrwdnQ6xb0TuhMa8WJYLZ9RCs6aULBkRZ8fRuEVsh6nr0K8tqtbJp0yY+/vhj9u3bh91uZ8KECdxwww2Rbt8l7d27l/z8/D7/3IRUtR2Ovg+SBCNnQ86IWLeoZzwtKKqKvbXn1dv69efR6k30TzdhGpCCKJYS37r9XVcUhTVr1rB+/XpOnDiBJEkMHjyYvLw8VFXtVs6N3W5v02s6fvw4e/fuJTMzkwEDBrBkyRJOnToVTr147rnnGDx4MKNGjcLtdvPiiy/y/vvvi/Uiu8LTAsc/DP5dVeHwe9BvGGgTsGaVpwVfQMGrTUGridDUoFaygQGZZgbkm0AWxQjjWbeCl6qqfOtb3+Kdd95h3LhxjBkzBlVV+eKLL5g/fz7r16+/ZKLp+Xbu3Mn06dPDXy9evBiAefPmsWbNGmpqaqiqqgq/7/V6+dGPfsSpU6cwm82MHTuWv/3tb23OIXSg7mAwNyotL5gn5bbB2cOQOzLWLes+TwsBRUVvtpBmlCNbT04OzWkUU4TiXreC15o1a/jwww/ZvHnzRQHj/fffZ86cObz00kvMnTu3S+ebNm0aaifLTK1Zs6bN1z/5yU/4yU9+0p0mC63OHgr+mT822Aur3Ab1XyRe8Ar4wefCrJe5+2ujUXWmyJ5fNuANKNibmzFlBnq3CrcQVd3qb//xj3/k8ccfb7enc9111/HYY4/xyiuvRKxxQoR4ndB8Ovj3fsWQdUXw701VibdGoTf4pBGNDDpT5Kv4ykaO1dv522dVHK23R/bcQkR1K3h99tln3HTTTR2+P3PmTPbt29frRgkR1nw6GKTM/cBogdQ8kPXgc4P9TKxb1z2eUEAxpBKVEfU21VTF08Z41q3g1dDQQG5ubofv5+bm0tjY2OtGCRHWEup1tVZf0GjAGkrWbapq/5h4FcrxOtwEf9pRxeG6lsieXzYiazTBKUKRytwXoqJbwSsQCCDLHQ+TabVa/H4xrSLutN4yWvJRFBW7x4+aFlq2zp5gFTlCwasxYOB0kzvyvaPzJmdHbNqREBXdfto4f/58DIb2S+R6PGJKRdxR1XDwcpny+PM/qjjb4mGUQeIGVUVjr4txA7vJE5yWZlODU4PSjJHL8QLCdexl1Svq2Me5bn3n582bd8l9uvqkUegjnhbwe0DSsK0GzrYEf8EcdqQw0uWiUHMu+ARPG+EgEC1uGwC2UHZ9agQTVIHQ8mfB20a76HnFtW5951evXh2tdgjR4jwLgFdv4UBtcMGUsYVWKmolAj4TqAo46sGSILMU3E0EFJUWUgBIjXjPyxDseSke3GL5s7iWIL9uhR5znAOg1p9KQFHJsRiYXpLDtVdko9s/CBorEyx4NeMNKHjkNPSyBkOks+BlIyadlgFWyCiwRvbcQkSJ4JXsHMFyxtWeYE9laHYqGo2EBimYOtFYCc5zsWxh1/nc4Pfg9St4tKlkRLrXBSAbMOq0DEmXoFAEr3gmgleyC902lg4fRio5DMg0h99STZk4vX7MznMkxBzk0HiXX2skxWzCaorCvMzW6UGqCgHvVysKCXFHBK9k5wgGr5SMPMalpoc3K4rKn79wUXTSxljzmdAIUpwLPWns1y+bB64c0unUsh7TyKDR4vF6aW6ykZGRhayN4MRvIWLEdyWZ+VzBJ40Apow2b2k0ErrULAAcTfWgJMDgdKjnhTFYky3iU4OCJwXZwGcnbazffhSbyxf5zxAiQgSvZBb6YT/l0rDnlJ0Wd9sfxH79sghIOuxuL7iaYtDAbgoHryiPRclGZG1wIQ6RqBq/RPBKZqGAdLxFZktFPY2OtsErP92EW2fF7vYnxqB9KHh9cMLDn3ZUUd8SpaRo2YBOowmmS4hE1bglglcyc9vwKwqNSnBEKzut7eBzntWIS07H6Q3gtyfAIqtuGyoqp1x6Tje50Wmj9JihteelenGLnlfcEsErmbmbcPkCeORUUg3yRbWp0ow6JHMGKmBrPBubNnaHuwlfQMWhCZZoTjNGqQpsOFHVi0ckqsYtEbySmduG0xPAI1vol6pvd5cUaz8AHE1xHrx8LvC5cfsCuGUraUZdaJmyKJCNaLUatIoHl1fcNsYrkSqRzFxNOH0BPIZUilLbz1cqLOiP8ZyJdE2cF95zBUstOSUTikYXnRyvVjoTOo2Ezu8Wt41xTASvZKWqoZ6XH3dKxz2v4oGFcMoMuILpEpo4LXscWq7NLqUBkB7V4GUmzaijRC+jzzJfen8hJmJ62/jhhx9yyy23UFBQgCRJXVq8Y8uWLUyYMAGDwcCwYcMuqnMvhHjtoPhx+YKr7GR10PNCnxqsKBEKdnEr1POyhYKX1RzdnpfVpGNklsywnLTofY7QKzENXg6Hg3HjxvH88893af/jx48za9Yspk+fzt69e1m0aBEPPPAAGzdujHJLE1AoTWJc8QDuvHow/VLa73khSbhlC01OL+7mOE6XCAUvyZxBmlGOes8LAJ8zep8h9FpMbxtnzpzJzJkzu7z/ypUrGTx4MMuWLQNgxIgRfPTRRzz77LPMmDEjWs1MTKFelM6cQa7F2OmuO2sVtI0tFNbXUZRX3Bet675Q8JpwxRAm5AyJ7mfpzKiqisdlp6XFc1GKiRAfEupp47Zt2y5alXvGjBls27atw2M8Hg/Nzc1tXpcFd1PwT1P6JXc1WoJPHO1NcZzrFQpeF05zigqdCU9AYdeR0/xxe2V05lAKvZZQwau2tvaiBUByc3Npbm7G5XK1e0x5eTlWqzX8Kioq6oumxp7bxlm7hz1nFKrOdX77k5IenOPoao7TdAmfC3wuVNQ+Cl5mdFoNGjUAAbGKULxKqODVE0uWLMFms4Vf1dXVsW5S33A10eT0caBBQ21z56s/p6VnA+C1x+nKT6FeV61bx+8+OcnfD0e5hyjr0cq60BJoblxekS4RjxIqVSIvL4+6urYLRtTV1WGxWDCZ2l852WAwdLhgSFJz23D7AnhS0ki/xJO59MxsqgHF2YjfH0COdHXS3grNu2yW0rB7/PiVPriN05nRaSTkgBunL0Af9PeEbkqonldZWRmbN29us23Tpk2UlZXFqEVxSgmApxm3P4Bbe+nglWLth1arQVJ8NNqa+qaN3RGqBtugBkvhdPjkNJJ0JnRaDTrFJXpecSqmwctut7N371727t0LBFMh9u7dS1VVcCHUJUuWtFmN6Hvf+x7Hjh3jJz/5CV9++SX/8z//w6uvvsoPf/jDWDQ/fnma8QcCeBQNPq2ZdFPnP+ySVofeFCwzY2uMw0H7UB3+eiWYc5Vh7ovgZUbWSugC4rYxXsX0tnHnzp1Mnz49/PXixYuB4BJra9asoaamJhzIAAYPHszbb7/ND3/4Q5YvX05hYSEvvviiSJO4kKsJty+4SEWqUYdevvTvqLy8fDQ2L2Zd+w8+YspRj19RwsGrw4TbSAoN2suKG6dXLKQcj2IavKZNm9bpY+j2suenTZvGnj17otiqJNA63qVN73Imel5uHih1oLREuXHd5PeC24bDE8CpyyTNeHF1jKjQmUk36xhlkbFa2x9PFWIroQbshS5yN+HxK3hkS9cz0VtTEOKtompoAZEWVY9fayTnEgm3EaMz0S/FQL8cPfQT8xvjkQheychto3+6iayBJfj6Z3XpENWUTrPLh+30KfKuULp0q9knQguIaFKzKUwx0T+9j4KXvnWKUBzeRguACF7JKdR7MqT2w2Do2rdYMmVy5EwLTtWLzuEhP15ulUJPGgvyi7jtij5MMA5NEXI7m3HYPX0zziZ0S5z8ehUiqicLVZjSMellZMVNQ1McTaFqra2f0rUeZMToTDh9AT798iSv7zrZt58tdIkIXskm4MPvbuHL2mY+rPZ1fV6ebMBgTgXiKF1CVaGlFl9AwW3s6+Bl/irPyxcQ8xvjkAheycZtw+1TOOuGirO+bq1taEwLBgh7vJSE9trB66C22cNvd9qjPy3ofPoUZI2EVvEhBfxiFaE4JIJXsnE1hdIkLFi7mYluCk3QdjfHSc+rJTgV7KyahqKRsURrwY32yEY0Wh2yRkKnOHGIXK+4I4JXsnHbcPkCuOW0bhfsSw1N0FadTfGRVW6vRVFVapTg2F1Beh8+RJAk0KeglzXoAk4cHhG84o0IXsnG3YjHF8Ajp5HRzZ6XPiUTg6zB6G/mrD1KC7p2R0stdo8fm7YfJr22b+Y0ns+Qil6rQR9wYhfBK+6IVIlkE5oa5DZ3I0G1lSmDwVkpDDVpMPVVMmhnWmppcvpw6LMYmGlGE62lzjqiT0Ena9AFXDg8cdATFdoQwSvZhBeaTev+IhWmjNCkZzdIAWLaMfe0gKeFJpcPZ2o/BvZL6fs26NPIMOsYY9WQbo2DYC60IYJXMlFV/M4mANyy5ZLVJC6iM4FsAL8nmOiamh3xJnaZ7RTegEK9YiWg0TMwFlN09CnBKULZMmSKKULxRox5JROfC1nxceWgDOZfN677U3wkCb8xnTMtbnZWHIttblPzSbSSRPEVw5k4MIOULs4UiChDMO8Nr6PvP1u4JBG8kklo0Q3JYMFk7Nl0FsmYzrF6B0dOVMd2kNp2Eq1GYsCgYq69IkY9QH0qqqricjRx5hKltIW+J4JXMmmtCNGdaUEX0KZkYtJrQ08cvZFpV3cFfOEcL6z9Y9MGAH0K3oDCjopq/viPapFlH2dE8Eom7iYqzzn4R61CdUMPF0w198Os12Lyx7C30VJDnc1JlUODU5MamzYA6FNDU4TcKEoAZzzkvglhInglE1cTNrePSqe+58t1mbNINciYvI2XXHUoWpRzxznV5OKzFguVDTEsSaMzo9Fo0WtALxJV444IXklEdTUGc7zk4CP+HjH3I9Ugo1NcnG1sjMmtUn31ITx+BXdaEcU5Mex5aTRgSEMna9AH7CJRNc6I4JVEvPZGAoqKV7Zg7W6CaitZj9naDwlQ7edo6esfWJ+b+tPHAeg/eBSyNsb/RY0W9FoNBr9dJKrGmbgIXs8//zyDBg3CaDQyefJk/vGPf3S475o1a5Akqc3LaBQJhCgB3KFFY3Wpmb36odemZmM2aDH7Gqlv6dtpQnXVh2hxefHorIweWtinn90ugwWDrMEQcNDs9sW6NcJ5Yp6k+qc//YnFixezcuVKJk+ezHPPPceMGTOoqKggJyen3WMsFgsVFRXhr7tT9iVpuW14fH4USUtqWs+fNgJg7sfQ7FSuGGDEmN23t21HK/YjAdaCYtL6sopER4wWDLIWvddOs0sEr3gS857XM888w4IFC7j33nsZOXIkK1euxGw2s2rVqg6PkSSJvLy88Cs3N7cPWxynnA2h8S4rGSm9LFmckkWKXsboaYxM27roVKMTX10FEjBseGmffnaHjFbSTDIjMhSKc2M4/iZcJKbBy+v1smvXLm644YbwNo1Gww033MC2bds6PM5utzNw4ECKioqYPXs2Bw8e7HBfj8dDc3Nzm1dScjWgqCp+Q/olV8i+JHOoaqmjb+t6WXz19Df5yUpPIy1/WJ9+docMFtIMOoZnqAzLSYt1a4TzxDR4nT17lkAgcFHPKTc3l9ra2naPKSkpYdWqVbz55pu8/PLLKIrClClTOHmy/Trj5eXlWK3W8KuoqA8XcehLzgYG9UvhG1eNZGxheu/OlZINkkTd2XNs2F7B8bN9Mz0mreUow3LSGDp8HGhjPqIR1Jrw607SX3oJLOa3jd1VVlbG3LlzKS0tZerUqaxfv57s7Gx++9vftrv/kiVLsNls4Vd1dXUft7iPhBaqkMz90Pa2dIysB3M/HF4/jXVVVJ6LbvBSFDVYr77+EABSdklUP69bDBYAvB4Xp8424vaJJ47xIqa/3rKystBqtdTV1bXZXldXR15eXpfOodPpGD9+PEeOHGn3fYPBgMFwGSxb5WoI/mnuF5nzpeVhNZ0i1X2259n6XbT1cD2BhkqudZxDrzdA5tCofl63yHrQmfjiZA2fer5kxuSxDMqKQXke4SIx7Xnp9XomTpzI5s2bw9sURWHz5s2UlZV16RyBQID9+/eTn58frWbGP7+Hcw3n2HeyiW01EUoqTcvHatKR6jvLWbuXRkd05jnW2Fzsq27CWbkHp9cPOSODASOeGK3BdAl/i0iXiCMxv21cvHgxv/vd7/j973/PF198wfe//30cDgf33nsvAHPnzmXJkiXh/Z944gnee+89jh07xu7du/nOd75DZWUlDzzwQKwuIfacDTi9AWwBPfZAhDrTqbnIGg39tcE1II/W2yNz3vN4/QrvHaxDE/AyXHsqWH8sf2zEP6fXzJkYdFpMfhtNThG84kXMR0XvuOMO6uvrWbp0KbW1tZSWlvLuu++GB/GrqqrQaL6KsY2NjSxYsIDa2loyMjKYOHEin3zyCSNHjozVJcSe8xxObwCXnElWpOq8p+aCJJFr8KH32zl8xsiVgzIjc+6QLRVnaHB4Geo9xOAMfXBhWUsMq0h0xJSJSafF6Gqi0RmjShvCRWIevAAeeughHnrooXbf27JlS5uvn332WZ599tk+aFUCcdTj8gZwGTLISo1Q8JL1kJZHhv8U6Y5aam2p1Ld4yE6LzPjhFzXNHDzdjIYA15pOoNNqoGhScNWeeGMOBa8WG6ejdPssdF/MbxuF3gu01OH2BXDqMumXGsGHE+kD0Gs1lBgbGRrBCdKnm1z87fPgQ5rplhrStZ5g1dKcURH7jIgyBWucmfw2bC4f/oBYgDYeiOCVBBwNNahAICWHFL02cie2DgBgorWZb40riFivSy9rMBtkhmXKjPZ+Ftw4cEr85HZdyJSBTithUt1oAl6axDShuCCCV6LzOnG1NAFg6Zcf2Xme1kKQNEiuJnBFbqpQVqqBOycVcZP5SzR+V3CsK398xM4fcTojkj6V/hkmbhgkY47kLwihx0TwSnT2OiQJ5JQMsjMskT23zhgMYABnD+P0+nn7sxrOtHS/SOFZu4dj5z2xNDefQFe7L/hF8Y3B2lnxLCWLAquJkRYPZn2c9hAvM+K7kOgc9eSkGckZcgUMzYr8+bNLoKkK6iv4qHkAh+paOOfwcPuVRRh1XeuBHKu389cDtfgDKrNLCxhkdMAXbwbfLLwKMgZFvt2RlpoDjZVgPxPrlgghcf7rTriklprgn6lRqqyRVRz8s/kUXy/Sk2qQOWf38saeU5esLOr2BXjvYC1v7j2N16+Qn24kV9ME+9aB3wvpA2Do9Oi0O9JSc1FVFVv9SQ6etsW6NQIieCU8f9NJFFUFS0F0PsBohYyBoKqYz+5nzvj+GHVaam1uXv60kl2VjRfVdj/T4ubDQ/X870fHOXi6GUmC8UVWvl3QgOnAH4PrIKbmwOhbQZMg40epuQRUlf0Vh3jvQG1wNoAQUyJ4JTKPnbq6OnacaGRbfRTnbxZMCP55eg/ZJrjjqiKy0gy4vAE+PFTPCx8ew3Xeyjrvf3GGXZWNeP0K2WkGbh9hZJp3K9qKd4LLmmUMhNK7gyt0JwpzP2RZR6o2gCHQwpnmvq0wK1xMjHklsubTtLh92OV0MkxRXI4+6wowZ4KzASo/IXPodO6aNIDPTzdz4LSNRqcXo+6r34N5ViOpBi2jLQ4GOvcgHf4yWDVC0sDga6Do6vgfoL+QRgsp2aQYmkj1nKGuebCYoB1jInglMKXpJC1uP3ZzLqPTo9iL0WhgyHQ48DpUb4eMgWgzhzCm0MqYQitev/JViobPxbS0U2DfBydqvjpHdgkMmRYMgonKWkSq4TgWTy01NrGCdqyJ4JXA7LWH8SsqLnN/siOZWd+e7CsgbwzU7of9r8Ogr0POcJCN6N3NYDsJjceh4RgooVtIjQy5I6H/lZCWBKW604uwmHSkNdfwZZOLgKL2vnaa0GMieCUqjx1bfbB6rKWgGE1f/BBdMQP8bjh7GI5tCb7ak5oDuaMgbyzoo3g729eshZj1WqxKI4rHSW2zm/7R7PEKnRLBK1E1HqfR6cWhz2JgfnbffKZWB6O/DbWfwek90FIHqhJMZk3NCw7E9yuG1D5qT1/TpyClZGMx2rF6TlHd0F8ErxgSwStBuWq+xOEJYLMWMqUvB44lCfLHBV+qGrxFjNc5idHQbygFDbXcnNFC+sCMWLfmspZgj3wEAPwedE3HGJyVQs6Q0thNV5GkyytwAWSVkGqQyfJWIyOqS8SSCF6JqP5LZBTy8gu5ZnyclpFJVpaCYPkevxcajqKqESq7LXSbCF6JRlXh5I7g33NHx2fxvmQmSZA7Gr+iULF7K3/4tJKAIgJYLIjglWDUc0c5cuw4dY4A/tw4rPd+OSgYj0ajwV5zBPe5k3x+WqzpGAsieCWSgJ+6Pe9Qb/fwqXsgLuJslZ3LhSkdTd5oCtKNDGz6lG1H68V6jjEQF8Hr+eefZ9CgQRiNRiZPnsw//vGPTvd/7bXXGD58OEajkTFjxvDOO+/0UUtjSFVp+Xwj1Ser8WlM5I+7njSjLtatunwNvpbcjDRylTOk1+/kvc/rgovnCn0m5sHrT3/6E4sXL+ZnP/sZu3fvZty4ccyYMYMzZ9qvm/TJJ59w5513cv/997Nnzx7mzJnDnDlzOHDgQB+3vA8F/Jzb+zaHdm3Fr6jYB89g4tDLeJ3KeGC0oim+kWE5aQxs3on78Ie8/dkp0QPrQ5Ia48clkydP5qqrruI3v/kNEFx0tqioiH/913/lscceu2j/O+64A4fDwVtvvRXedvXVV1NaWsrKlSsv+XnNzc1YrVZsNhsWS4Qrj0aKqoLfA65GHGeOcnzfR9ibzqICTQXXcu11N5NiuMxSFOKRqsLxrTR8vpXDZ1pwyOnIheO59soJYMpA0RpQ4aIpRKqqoqggQXhmhC+g4PD4UdTg+yqgqCoaSULWSJj0WgyyNnw8ENmS33Gkqz+jMf0J8Hq97Nq1q82ishqNhhtuuIFt27a1e8y2bdtYvHhxm20zZsxgw4YN7e7v8XjweL4qX9Lc3PXBVZvTx//bdwqA1gjfGupVVWVUfytXhdYytHv8vLqjOnysCsh+J8Nq/gJAXpqBodkpgIrP72fniYZgdnr4pMGXpKpkp2gZmhmcqyj7A7Q0NeHTmJBKbmL6VVd3uYKpEGWSBIOnkmnKYMSBjRyrOUehYwfsCi4q0uyBz2qcoNWCpAVJg6qqBEL/h4bnWSgIZeg3O7zsrW5qPXGbjzlpncjo0quYODD4f6222c2fdlSjlSQ0GimYbidJaEJfTxyYQWlROgANDi9vfXa6w0sY3d/KhAHBZFuby8f/23uqw32H51vC/98dHj/r93S877DsVMqG9gPA4w/w6s6T3DQqL2KLuECMg9fZs2cJBALhBWZb5ebm8uWXX7Z7TG1tbbv719bWtrt/eXk5P//5z3vUvoCqctbe8Tp9zvNqWCmqiu2CVWX0fg8a51kAJNkAzlAlgoAC7varcaqA6tcDBtCZMPTrT4o1i4KSK8mypvXoOoQoCs04sGSVMLZmPzQcBXsN+Nz4fB50iouOclk1Ll9wahUgu3ykBVqQ+Cr7RSL4/+Gs5EM+r4RQQFGDnXNVhXbG2bz+rz7Qryic6+T/8Pl12BSl8//v5xedVFSVsy0d1zTLtxjDf1dVONviiXhKSdLfeyxZsqRNT625uZmioqIuHZtqkPn2hMJ2U6kkCdIMXw2Ym3Va7riqqM2+khJA2/wAaCQMOhmMOkBCq0LxSF9wZR40qFLrLYCEJEnodDqwZATnEgIiISIB6IxoBlwFA64Kfu33kuW1k+rx4vd7Uf1+VFVF0gR7SbJGQtYAodtGq6oyqYNTX2nuB8avbp/yrSYWXDsERVVRlOAt6Pl/TzF81TNPN+n554mFHTbbct5Dn1Sj3GbfCweUUo1fhQuTTsu3J3R83vPboNNq+PaEQtLNkX3AFNPglZWVhVarpa6urs32uro68vLy2j0mLy+vW/sbDAYMhp51VfWyhgH9ulYVQdZqwrcAbWQMv2iTBsgS0+KSm6xHkjOJRo1IrUYitYtjnnpZQ1Fm1xqh03Z9X1nb9Z8NrUbq8r7dEdOnjXq9nokTJ7J58+bwNkVR2Lx5M2VlZe0eU1ZW1mZ/gE2bNnW4vyAIySnmt42LFy9m3rx5XHnllUyaNInnnnsOh8PBvffeC8DcuXPp378/5eXlADz88MNMnTqVZcuWMWvWLNatW8fOnTt54YUXYnkZgiD0sZgHrzvuuIP6+nqWLl1KbW0tpaWlvPvuu+FB+aqqKjTnDVZOmTKFtWvX8tOf/pTHH3+c4uJiNmzYwOjRo2N1CYIgxEDM87z6WkLkeQnCZayrP6Mxz7AXBEHoCRG8BEFISCJ4CYKQkGI+YN/XWof4ujNNSBCEvtP6s3mp4fjLLni1tLQAdDnLXhCE2GhpacFqtXb4/mX3tFFRFE6fPk1aWtolZ+W3TiWqrq5OmieTyXZNyXY9IK5JVVVaWlooKChokyZ1ocuu56XRaCgs7HhOVnssFkvS/CdqlWzXlGzXA5f3NXXW42olBuwFQUhIIngJgpCQRPDqhMFg4Gc/+1mPq1LEo2S7pmS7HhDX1FWX3YC9IAjJQfS8BEFISCJ4CYKQkETwEgQhIYngJQhCQhLB6wINDQ3cfffdWCwW0tPTuf/++7Hb7Z0eM23aNCRJavP63ve+10ctvliyrUDenetZs2bNRd8Lo9HY4f6x8OGHH3LLLbdQUFCAJEkdLtt3vi1btjBhwgQMBgPDhg1jzZo1UW9nV3X3erZs2XLR90iSpA5XAOuICF4XuPvuuzl48CCbNm3irbfe4sMPP+S73/3uJY9bsGABNTU14dcvf/nLPmjtxZJtBfLuXg8Es7jP/15UVlb2YYsvzeFwMG7cOJ5//vku7X/8+HFmzZrF9OnT2bt3L4sWLeKBBx5g48aNUW5p13T3elpVVFS0+T7l5OR074NVIezzzz9XAXXHjh3hbX/9619VSZLUU6dOdXjc1KlT1YcffrgPWnhpkyZNUhcuXBj+OhAIqAUFBWp5eXm7+99+++3qrFmz2mybPHmy+uCDD0a1nV3V3etZvXq1arVa+6h1vQeob7zxRqf7/OQnP1FHjRrVZtsdd9yhzpgxI4ot65muXM8HH3ygAmpjY2OvPkv0vM6zbds20tPTufLKK8PbbrjhBjQaDdu3b+/02FdeeYWsrCxGjx7NkiVLcDqd0W7uRVpXIL/hhhvC27qyAvn5+0NwBfKO9u9LPbkeALvdzsCBAykqKmL27NkcPHiwL5obNfH8PeqN0tJS8vPzufHGG/n444+7ffxlNzG7M7W1tRd1XWVZJjMzs9P78bvuuouBAwdSUFDAZ599xqOPPkpFRQXr16+PdpPb6IsVyPtST66npKSEVatWMXbsWGw2G7/+9a+ZMmUKBw8e7PaE/HjR0feoubkZl8uFydTOeqFxLD8/n5UrV3LllVfi8Xh48cUXmTZtGtu3b2fChAldPs9lEbwee+wxnn766U73+eKLL3p8/vPHxMaMGUN+fj7XX389R48eZejQoT0+r9B9ZWVlbdbwnDJlCiNGjOC3v/0tTz75ZAxbJrQqKSmhpKQk/PWUKVM4evQozz77LH/4wx+6fJ7LInj96Ec/Yv78+Z3uM2TIEPLy8i4aCPb7/TQ0NHS4Ind7Jk+eDMCRI0f6NHj1xQrkfakn13MhnU7H+PHjOXLkSDSa2Cc6+h5ZLJaE63V1ZNKkSXz00UfdOuayGPPKzs5m+PDhnb70ej1lZWU0NTWxa9eu8LHvv/8+iqKEA1JX7N27Fwh2j/tSsq1A3pPruVAgEGD//v19/r2IpHj+HkXK3r17u/896tVwfxK66aab1PHjx6vbt29XP/roI7W4uFi98847w++fPHlSLSkpUbdv366qqqoeOXJEfeKJJ9SdO3eqx48fV9988011yJAh6rXXXhuT9q9bt041GAzqmjVr1M8//1z97ne/q6anp6u1tbWqqqrqPffcoz722GPh/T/++GNVlmX117/+tfrFF1+oP/vZz1SdTqfu378/Ju2/UHev5+c//7m6ceNG9ejRo+quXbvUf/mXf1GNRqN68ODBWF3CRVpaWtQ9e/aoe/bsUQH1mWeeUffs2aNWVlaqqqqqjz32mHrPPfeE9z927JhqNpvVH//4x+oXX3yhPv/886pWq1XffffdWF1CG929nmeffVbdsGGDevjwYXX//v3qww8/rGo0GvVvf/tbtz5XBK8LnDt3Tr3zzjvV1NRU1WKxqPfee6/a0tISfv/48eMqoH7wwQeqqqpqVVWVeu2116qZmZmqwWBQhw0bpv74xz9WbTZbjK5AVf/7v/9bHTBggKrX69VJkyapn376afi9qVOnqvPmzWuz/6uvvqpeccUVql6vV0eNGqW+/fbbfdziznXnehYtWhTeNzc3V7355pvV3bt3x6DVHWtNFbjw1Xod8+bNU6dOnXrRMaWlpaper1eHDBmirl69us/b3ZHuXs/TTz+tDh06VDUajWpmZqY6bdo09f333+/254qSOIIgJKTLYsxLEITkI4KXIAgJSQQvQRASkghegiAkJBG8BEFISCJ4CYKQkETwEgQhIYngJSS9NWvWkJ6eHv76//yf/0NpaWn46/nz5zNnzpw+b5fQOyJ4CREzf/78DktgL1y4EEmS2kyQj0bQGDRoEM8991ybbXfccQeHDh3q8Jjly5e3Kas8bdo0Fi1aFNF2CZEngpcQUUVFRaxbtw6XyxXe5na7Wbt2LQMGDIhJm0wmU6clhq1Wa5uemZAYRPASImrChAkUFRW1KcS4fv16BgwYwPjx43t17vZ6RHPmzAn35qZNm0ZlZSU//OEPw4s6wMW3jRc6vwc4f/58tm7dyvLly8PnOH78OMOGDePXv/51m+P27t2LJEkJXW4nkYngJUTcfffdx+rVq8Nfr1q1invvvTfqn7t+/XoKCwt54oknwos6dNfy5cspKytrs6DKgAEDLromgNWrV3PttdcybNiwSF2C0A0ieAkR953vfIePPvqIyspKKisr+fjjj/nOd74T9c/NzMxEq9WSlpZGXl5ejwoqWq1W9Ho9ZrM5fA6tVsv8+fOpqKgIL7vm8/lYu3Yt9913X6QvQ+iiy6KSqtC3srOzmTVrFmvWrEFVVWbNmkVWVlasm9UrBQUFzJo1i1WrVjFp0iT+8pe/4PF4uO2222LdtMuW6HkJUXHfffexZs0afv/730esd6LRaLiwgpPP54vIubvigQceCD+MWL16NXfccQdms7nPPl9oSwQvISpuuukmvF4vPp+PGTNmROSc2dnZbcaxAoHARYvj6vV6AoFArz6no3PcfPPNpKSksGLFCt59911xyxhj4rZRiAqtVhtekUmr1Xa4n81mC9f8b9WvXz+Kioou2ve6665j8eLFvP322wwdOpRnnnmGpqamNvsMGjSIDz/8kH/5l3/BYDD06HZ10KBBbN++nRMnTpCamkpmZiYajSY89rVkyRKKi4uTqoZ8IhI9LyFqLBYLFoul0322bNnC+PHj27x+/vOft7vvfffdx7x585g7dy5Tp05lyJAhTJ8+vc0+TzzxBCdOnGDo0KFkZ2f3qN2PPPIIWq2WkSNHkp2dTVVVVfi9+++/H6/X2ydPT4XOiTLQgtANf//737n++uuprq6+aCFYoW+J4CUIXeDxeKivr2fevHnk5eXxyiuvxLpJlz1x2ygIXfDHP/6RgQMH0tTUxC9/+ctYN0dA9LwEQUhQouclCEJCEsFLEISEJIKXIAgJSQQvQRASkghegiAkJBG8BEFISCJ4CYKQkETwEgQhIYngJQhCQvr/sLxPB6j/+p4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:07.948391Z", + "iopub.status.busy": "2024-03-22T22:04:07.947583Z", + "iopub.status.idle": "2024-03-22T22:04:08.157180Z", + "shell.execute_reply": "2024-03-22T22:04:08.156346Z" + }, + "papermill": { + "duration": 0.230634, + "end_time": "2024-03-22T22:04:08.159027", + "exception": false, + "start_time": "2024-03-22T22:04:07.928393", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAmC0lEQVR4nO3de1hU170+8JfhMtzxwkUkGLw0oscIipVg9CS2XEwbq02b+NMkILGmQckxTqiG/AzU2oS0UdQ2KE9MjDGJkSY1JqexKiWSeo2K4lErWBTEykXQyAjoMDDr/OFh0nEGGAjDZs28n+fxwdmz9p7vZs28s/Zizx4nIYQAEZFkVEoXQETUEwwvIpISw4uIpMTwIiIpMbyISEoMLyKSEsOLiKTE8CIiKbkoXUBfMxgMqKqqgo+PD5ycnJQuh4juIoTAzZs3MXToUKhUHY+vHC68qqqqEBoaqnQZRNSFy5cv45577unwfocLLx8fHwB3fjG+vr4KV2M7er0ee/fuRXx8PFxdXZUuh3qBo/SpVqtFaGio8bXaEYcLr/ZDRV9fX7sPL09PT/j6+tr1E92ROFqfdjWtwwl7IpISw4uIpMTwIiIpMbyISEqKhtff//53zJw5E0OHDoWTkxN27tzZ5TqFhYWYOHEi1Go1Ro0ahS1btti8TiLqfxQNr6amJkRERCAnJ8eq9uXl5fjxj3+M6dOno7i4GC+88AJ+8YtfYM+ePTaulIj6G0VPlXjkkUfwyCOPWN0+NzcXw4cPx5o1awAAY8aMwYEDB7B27VokJCTYqkwpNDc3o6SkxHi78ZYOh05fwED/4/D2UJu0DQ8Ph6enZ1+XSNSrpDrP6/Dhw4iNjTVZlpCQgBdeeKHDdXQ6HXQ6nfG2VqsFcOecGb1eb5M6lXDmzBlER0ebLf+9hbZff/01JkyYYPuiqFe1P1/t6XlribX7J1V41dTUICgoyGRZUFAQtFotbt26BQ8PD7N1srKysHLlSrPle/futavRh06nM45IAaC2Gdha5ozEUW0Iums3KyoqUF1d3ccVUm/Jz89XugSbam5utqqdVOHVE+np6dBoNMbb7R89iI+Pt+sz7E9VXkfepuN4fO4kRAwbpHQ51Av0ej3y8/MRFxdn12fYtx8ddUWq8BoyZAhqa2tNltXW1sLX19fiqAsA1Go11Gq12XJXV1e7fgK4uLgYf9rzfjoie3/uWrtvUp3nFRMTg4KCApNl+fn5iImJUagiIlKKouHV2NiI4uJiFBcXA7hzKkRxcTEqKysB3DnkS0xMNLZ/7rnncPHiRSxbtgwlJSXYsGED/vSnP2Hp0qVKlE9EClI0vI4fP44JEyYY//Kl0WgwYcIEZGRkAACqq6uNQQYAw4cPxxdffIH8/HxERERgzZo1ePvttx3+NAkiR6TonNfDDz8MIUSH91s6e/7hhx/GyZMnbVgVEclAqjkvIqJ2DC8ikhLDi4ikxPAiIikxvIhISgwvIpISw4uIpMTwIiIpMbyISEoMLyKSEsOLiKTE8CIiKTG8iEhKDC8ikhLDi4ikxPAiIikxvIhISgwvIpISw4uIpMTwIiIpMbyISEoMLyKSEsOLiKTE8CIiKTG8iEhKDC8ikhLDi4ikxPAiIikxvIhISgwvIpKS4uGVk5ODsLAwuLu7Izo6GkePHu20/bp16zB69Gh4eHggNDQUS5cuxe3bt/uoWiLqLxQNr7y8PGg0GmRmZuLEiROIiIhAQkICrl69arH9tm3b8NJLLyEzMxPnzp3DO++8g7y8PLz88st9XDkRKU3R8MrOzsbChQuRnJyMsWPHIjc3F56enti8ebPF9ocOHcKDDz6IefPmISwsDPHx8Zg7d26XozUisj+KhVdLSwuKiooQGxv7bTEqFWJjY3H48GGL60yZMgVFRUXGsLp48SJ27dqFH/3oR31SMxH1Hy5KPXB9fT3a2toQFBRksjwoKAglJSUW15k3bx7q6+sxdepUCCHQ2tqK5557rtPDRp1OB51OZ7yt1WoBAHq9Hnq9vhf2pH9qbW01/rTn/XQk7f1o7/1p7f4pFl49UVhYiNdeew0bNmxAdHQ0ysrKsGTJEqxatQqvvPKKxXWysrKwcuVKs+V79+6Fp6enrUtWzOVGAHDBkSNHcOWM0tVQb8rPz1e6BJtqbm62qp2TEELYuBaLWlpa4OnpiU8++QSzZ882Lk9KSsKNGzfw2Wefma0zbdo0PPDAA3jjjTeMyz744AM8++yzaGxshEplfhRsaeQVGhqK+vp6+Pr69u5O9SOnKq/j55uO45OFkxAxbJDS5VAv0Ov1yM/PR1xcHFxdXZUux2a0Wi38/f3R0NDQ6WtUsZGXm5sboqKiUFBQYAwvg8GAgoICpKamWlynubnZLKCcnZ0BAB1lsFqthlqtNlvu6upq108AFxcX40973k9HZO/PXWv3TdHDRo1Gg6SkJEyaNAmTJ0/GunXr0NTUhOTkZABAYmIiQkJCkJWVBQCYOXMmsrOzMWHCBONh4yuvvIKZM2caQ4yIHIOi4TVnzhzU1dUhIyMDNTU1iIyMxO7du42T+JWVlSYjrRUrVsDJyQkrVqzAlStXEBAQgJkzZ+LVV19VaheISCGKzXkpRavVws/Pr8vjadkVX7qG2RuPYGfKA4i8d7DS5VAv0Ov1xlOD7Pmw0drXqOIfDyIi6gmGFxFJieFFRFJieBGRlBheRCQlhhcRSYnhRURSYngRkZQYXkQkJYYXEUmJ4UVEUmJ4EZGUGF5EJCWGFxFJieFFRFJieBGRlBheRCQlhhcRSUmq720kchTNzc1mX77ceEuHQ6cvYKD/cXh7fPuNWOHh4Xb9HaQdYXgR9UMlJSWIioqyeN/v77pdVFSEiRMn2r6ofobhRdQPhYeHo6ioyGRZafUNaD4+jezH78fo4AEmbR0Rw4uoH/L09DQbTakuXYN6/y2MGRfBb4QCJ+yJSFIMLyKSEsOLiKTU7fC6ePGiLeogIuqWbofXqFGjMH36dHzwwQe4ffu2LWoiIupSt8PrxIkTGD9+PDQaDYYMGYJf/vKXOHr0qC1qIyLqULfDKzIyEuvXr0dVVRU2b96M6upqTJ06FePGjUN2djbq6upsUScRkYkeT9i7uLjgsccew8cff4zf/e53KCsrQ1paGkJDQ5GYmIjq6urerJOIyESPw+v48eNYtGgRgoODkZ2djbS0NFy4cAH5+fmoqqrCrFmzerNOIiIT3Q6v7Oxs3H///ZgyZQqqqqqwdetWXLp0Cb/97W8xfPhwTJs2DVu2bMGJEyes2l5OTg7CwsLg7u6O6OjoLufPbty4gcWLFyM4OBhqtRr33Xcfdu3a1d3dICLJdfvjQRs3bsQzzzyD+fPnIzg42GKbwMBAvPPOO11uKy8vDxqNBrm5uYiOjsa6deuQkJCA0tJSBAYGmrVvaWlBXFwcAgMD8cknnyAkJASXLl3CgAEDursbdqG8vglNulaL912oazL+dHHpuJu91C4Y7u9lk/qIbMlJCCG6s0JFRQWGDRsGlcp00CaEwOXLlzFs2DCrtxUdHY3vf//7ePPNNwEABoMBoaGheP755/HSSy+Ztc/NzcUbb7yBkpISuLq6dqdsI61WCz8/PzQ0NMDX17dH2+gPyuubMH11Ya9sa1/awwwwCRRfuobZG49gZ8oDdv3ZRmtfo90eeY0cORLV1dVmI6Pr169j+PDhaGtrs2o7LS0tKCoqQnp6unGZSqVCbGwsDh8+bHGdzz//HDExMVi8eDE+++wzBAQEYN68eVi+fDmcnZ0trqPT6aDT6Yy3tVotAECv10Ov11tVa3/U0HTnHLvVP78fowLMg6fptg679x/DjGnfh5e72ux+ACira0LaJ6fR0HQbej83m9ZLXau41oQmXcevn/M1DSY/LfFSOyNssNxvRNa+LrsdXh0N1BobG+Hu7m71durr69HW1oagoCCT5UFBQWYXYWt38eJFfPnll3jyySexa9culJWVYdGiRdDr9cjMzLS4TlZWFlauXGm2fO/evVJfwO1yIwC4oKb0JFyvWG4zKQCoLzmG+g62UfN/2zhw4AAuedukTLLS1VvAq8XWvRyXfXqu0/v/f2QrAj16oyplNDc3W9XO6vDSaDQAACcnJ2RkZJi88Nva2vD1118jMjKye1V2k8FgQGBgIN566y04OzsjKioKV65cwRtvvNFheKWnpxtrB+6MvEJDQxEfHy/1YePZKi1Wnz6CqVOn4j+Gmu+HXq9Hfn4+4uLiOjzE7mob1HfOVmmB4iMdjqSBrkfT7SPp78fI3Z/tR0ddsTq8Tp48CeDOyOv06dNwc/v2MMPNzQ0RERFIS0uzukB/f384OzujtrbWZHltbS2GDBlicZ3g4GC4urqaHCKOGTMGNTU1aGlpMampnVqthlpt3tGurq49njfrD9on4V1cXDrdj87209ptkO2190V4sB/GhfhZbKPX61FfAkweEWCxv+ylP62t3erw2rdvHwAgOTkZ69ev/86jFjc3N0RFRaGgoACzZ88GcGdkVVBQgNTUVIvrPPjgg9i2bRsMBoPxDwbnz59HcHCwxeAiIvvV7fO83n333V473NJoNNi0aRPee+89nDt3DikpKWhqakJycjIAIDEx0WRCPyUlBdevX8eSJUtw/vx5fPHFF3jttdewePHiXqmHiORh1cjrsccew5YtW+Dr64vHHnus07Y7duyw+sHnzJmDuro6ZGRkoKamBpGRkdi9e7dxEr+ystLklIzQ0FDs2bMHS5cuxfjx4xESEoIlS5Zg+fLlVj8mEdkHq8LLz88PTk5Oxv/3ptTU1A4PEwsLC82WxcTE4MiRI71aAxHJx6rwevfddy3+n5Sja7sNlfsVlGtLoXI3P8+htbUVVa1VOHf9XIdn2JdrG6FyvwJd220AvfumRGRr/PYgSVU1XYLX8D/i5S4upbZh94ZO7/caDlQ1RSIKQZ22I9vq6s0I6PoNydHejKwKrwkTJhgPG7ti7Qey6bsZ6nUvmsqfx/o5kRgZaHnkdfDAQTw49cEOR14XrjZiSV4xhk6/19blUhesfTMCOn9DcqQ3I6vCq/1UBuo/1M7uMNwOwXDf0Rg72PxdVq/Xo9ylHGMGjenwvBnD7QYYbtdB7Wz9JyPINrp6MwK6fkNytDcjq8Kro7PXiah3dPVmBHT9huRob0b86jMikpJVI69Bgwbh/Pnz8Pf3x8CBAzud/7p+/XqvFUdE1BGrwmvt2rXw8fEx/t/ayXsiIluxKrySkpKM/58/f76taiEislq357ycnZ1x9epVs+XXrl3r8IKARES9rdvh1dHFCHU6Ha/sQER9xuoz7P/whz8AuHMxwrfffhve3t+ei9LW1oa///3vCA8P7/0KiYgssDq81q5dC+DOyCs3N9fkENHNzQ1hYWHIzc3t/QqJiCywOrzKy8sBANOnT8eOHTswcOBAmxVFRNSVbn8wu/2KqkRESup2eD3zzDOd3r958+YeF0NEZK1uh9c333xjcluv1+PMmTO4ceMGfvCDH/RaYUREnel2eH366admywwGA1JSUjBy5MheKYqIqCu98sFslUoFjUZj/IskEZGt9dpVJS5cuIDW1tbe2hwRUae6fdj4798+Ddw576u6uhpffPGFyWcgiYhsqdvh1f7N2e1UKhUCAgKwZs2aLv8SSUTUW3ieFxFJiVdSJSIpMbyISEoMLyKSEsOLiKTUa+H1r3/9C88++2xvbY6IqFO9Fl7Xrl3DO++801ubIyLqFA8biUhKDC8iklK/CK+cnByEhYXB3d0d0dHROHr0qFXrbd++HU5OTpg9e7ZtCySifsfqM+wfe+yxTu+/ceNGjwrIy8uDRqNBbm4uoqOjsW7dOiQkJKC0tBSBgYEdrldRUYG0tDRMmzatR49LRHKzeuTl5+fX6b97770XiYmJ3S4gOzsbCxcuRHJyMsaOHYvc3Fx4enp2ekXWtrY2PPnkk1i5ciVGjBjR7cckIvlZPfJ69913e/3BW1paUFRUhPT0dOMylUqF2NhYHD58uMP1fvOb3yAwMBALFizA/v37O30MnU4HnU5nvK3VagHcuQKsXq//jnugnPbLD7W2tlrcj/Zlne1jV9ugvmNNX3TVp/bSn9bW3u0PZvem+vp6tLW1ISgoyGR5UFAQSkpKLK5z4MABvPPOOyguLrbqMbKysrBy5Uqz5Xv37oWnp2e3a+4vLjcCgAsOHDiAS94dt8vPz//O2yDb605fdNSn9tKfzc3NVrWzOrysvdyNLb+A4+bNm3j66aexadMm+Pv7W7VOenq6yTXItFotQkNDER8fD19fX1uVanNnq7RYffoIpk6div8Yar4fer0e+fn5iIuLg6ura4+2QX3Hmr7oqk/tpT/bj466YnV4bdmyBffeey8mTJgAIUSPC/t3/v7+cHZ2Rm1trcny2tpaDBkyxKz9hQsXUFFRgZkzZxqXGQwGAICLiwtKS0vNrqOvVquhVqvNtuXq6trhi1oGLi4uxp+d7Udn+2ntNsj2utMXHfWpvfSntbVbHV4pKSn46KOPUF5ejuTkZDz11FMYNGhQjwsE7nzTdlRUFAoKCoynOxgMBhQUFCA1NdWsfXh4OE6fPm2ybMWKFbh58ybWr1+P0NDQ71QPEcnD6r825uTkoLq6GsuWLcN///d/IzQ0FE888QT27NnznUZiGo0GmzZtwnvvvYdz584hJSUFTU1NSE5OBgAkJiYaJ/Td3d0xbtw4k38DBgyAj48Pxo0bBzc3tx7XQURy6daEvVqtxty5czF37lxcunQJW7ZswaJFi9Da2oqzZ8/C27v7s4Rz5sxBXV0dMjIyUFNTg8jISOzevds4iV9ZWQmVql+cS0tE/UiP/9qoUqng5OQEIQTa2tq+UxGpqakWDxMBoLCwsNN1t2zZ8p0em4jk1K0hjU6nw0cffYS4uDjcd999OH36NN58801UVlb2aNRFRNRTVo+8Fi1ahO3btyM0NBTPPPMMPvroI6tPVyAi6m1Wh1dubi6GDRuGESNG4KuvvsJXX31lsd2OHTt6rTgioo5YHV6JiYlwcnKyZS1ERFbr1kmqRET9Bc9BICIpMbyISEoMLyKSEsOLiKTE8CIiKTG8iEhKDC8ikhLDi4ikxPAiIikxvIhISgwvIpISw4uIpKTo9zZSz93S37l67ZkrDRbvb7qlw/E6YMilb+DlYf7tSQBQdrXRZvUR2RrDS1IX/i94XtpxupNWLni/7FiX2/JS82lA8uGzVlLx/3Hney1HBnrDw9XZ7P7S6ga8+MlprPn5/Rgd7NfhdrzULhju72WzOolsheElqUFebvh/k4d1eH9raysAYGSAF8aFdBxeRLJieBH1A13NYQJdz2M62hwmw4uoH7BuDhOwZh7TUeYwHWMvifq5ruYwAevmMR1pDpPhRdQPdDWHCXAe8248SZWIpMTwIiIpMbyISEoMLyKSEsOLiKTE8CIiKfWL8MrJyUFYWBjc3d0RHR2No0ePdth206ZNmDZtGgYOHIiBAwciNja20/ZEZJ8UD6+8vDxoNBpkZmbixIkTiIiIQEJCAq5evWqxfWFhIebOnYt9+/bh8OHDCA0NRXx8PK5cudLHlRORkhQPr+zsbCxcuBDJyckYO3YscnNz4enpic2bN1ts/+GHH2LRokWIjIxEeHg43n77bRgMBhQUFPRx5USkJEXPsG9paUFRURHS09ONy1QqFWJjY3H48GGrttHc3Ay9Xo9BgwZZvF+n00Gn0xlva7VaAIBer4der/8O1fdv7Wdjt7a22vV+OhJH6VNr903R8Kqvr0dbWxuCgoJMlgcFBaGkpMSqbSxfvhxDhw5FbGysxfuzsrKwcuVKs+V79+6Fp6dn94uWxOVGAHDBkSNHcOWM0tVQb3CUPm1ubraqndSfbXz99dexfft2FBYWwt3d3WKb9PR0aDQa422tVmucJ/P19e2rUvvcqcrrwOnjeOCBBxAxzPKolOTiKH3afnTUFUXDy9/fH87OzqitrTVZXltbiyFDhnS67urVq/H666/jb3/7G8aPH99hO7VaDbXa/NpHrq6ucHV17VnhEnBxcTH+tOf9dCSO0qfW7puiE/Zubm6IiooymWxvn3yPiYnpcL3f//73WLVqFXbv3o1Jkyb1RalE1M8oftio0WiQlJSESZMmYfLkyVi3bh2ampqQnJwMAEhMTERISAiysrIAAL/73e+QkZGBbdu2ISwsDDU1NQAAb29veHt7K7YfRNS3FA+vOXPmoK6uDhkZGaipqUFkZCR2795tnMSvrKyESvXtAHHjxo1oaWnBz3/+c5PtZGZm4te//nVflk5EClI8vAAgNTUVqampFu8rLCw0uV1RUWH7goio31P8JFUiop5geBGRlBheRCQlhhcRSYnhRURSYngRkZQYXkQkJYYXEUmJ4UVEUmJ4EZGUGF5EJCWGFxFJieFFRFJieBGRlBheRCQlhhcRSYnhRURSYngRkZQYXkQkJYYXEUmJ4UVEUmJ4EZGUGF5EJCWGFxFJieFFRFJieBGRlBheRCQlhhcRSYnhRURSYngRkZQYXkQkpX4RXjk5OQgLC4O7uzuio6Nx9OjRTtt//PHHCA8Ph7u7O+6//37s2rWrjyolov5C8fDKy8uDRqNBZmYmTpw4gYiICCQkJODq1asW2x86dAhz587FggULcPLkScyePRuzZ8/GmTNn+rhyIlKS4uGVnZ2NhQsXIjk5GWPHjkVubi48PT2xefNmi+3Xr1+PGTNm4Fe/+hXGjBmDVatWYeLEiXjzzTf7uHIiUpKLkg/e0tKCoqIipKenG5epVCrExsbi8OHDFtc5fPgwNBqNybKEhATs3LnTYnudTgedTme8rdVqAQB6vR56vf477kH/0dzcjNLSUuPt89UN0NWU4UyxG1pq/Uzajh49Gp6enn1dInXD3f0JdNyn9taf1r4uFQ2v+vp6tLW1ISgoyGR5UFAQSkpKLK5TU1NjsX1NTY3F9llZWVi5cqXZ8r1799pVh1+4cAEvvvii2fKn3zNvu2bNGowcObIPqqKe6qg/AfM+tbf+bG5utqqdouHVF9LT001GalqtFqGhoYiPj4evr6+ClfWu5uZmTJ061Xi78ZYOe/YfQ8K078PbQ23S1t7eqe3R3f0JdNyn9taf7UdHXVE0vPz9/eHs7Iza2lqT5bW1tRgyZIjFdYYMGdKt9mq1Gmq12my5q6srXF1de1h5/+Pn54fJkycbb+v1ety8cR3TpjxgV/vpKO7uT8Bx+tTafVN0wt7NzQ1RUVEoKCgwLjMYDCgoKEBMTIzFdWJiYkzaA0B+fn6H7YnIPil+2KjRaJCUlIRJkyZh8uTJWLduHZqampCcnAwASExMREhICLKysgAAS5YswUMPPYQ1a9bgxz/+MbZv347jx4/jrbfeUnI3iKiPKR5ec+bMQV1dHTIyMlBTU4PIyEjs3r3bOClfWVkJlerbAeKUKVOwbds2rFixAi+//DK+973vYefOnRg3bpxSu0BECnASQgili+hLWq0Wfn5+aGhosKsJ+7vp9Xrs2rULP/rRj+x6fsSROEqfWvsaVfwkVSKinmB4EZGUGF5EJCXFJ+z7WvsUn7UnwslKr9ejubkZWq3WrudHHImj9Gn7a7Or6XiHC6+bN28CAEJDQxWuhIg6c/PmTfj5+XV4v8P9tdFgMKCqqgo+Pj5wcnJSuhybaf8Y1OXLl+36r6qOxFH6VAiBmzdvYujQoSanSd3N4UZeKpUK99xzj9Jl9BlfX1+7fqI7Ikfo085GXO04YU9EUmJ4EZGUGF52Sq1WIzMz0+IVNUhO7FNTDjdhT0T2gSMvIpISw4uIpMTwIiIpMbyISEoMLwk8/PDDeOGFF5Qug3qgv/Vdf6vnu2B4EfVzLS0tSpfQPwnq15KSkgQAk38hISFiw4YNJu1OnDghnJycREVFhRBCiDVr1ohx48YJT09Pcc8994iUlBRx8+ZNk3X2798vpk6dKtzd3cU999wjnn/+edHY2Nhn+2bvLPVdWVmZeOaZZ0RYWJhwd3cX9913n1i3bp3ZerNmzRK//e1vRXBwsAgLCxNCCHHw4EEREREh1Gq1iIqKEp9++qkAIE6ePGlc9/Tp02LGjBnCy8tLBAYGiqeeekrU1dV1WE95eXlf/Tp6HcOrn7tx44aIiYkRCxcuFNXV1aK6ulqkpaWJqVOnmrR78cUXTZatXbtWfPnll6K8vFwUFBSI0aNHi5SUFOP9ZWVlwsvLS6xdu1acP39eHDx4UEyYMEHMnz+/z/bN3lnqu9u3b4uMjAxx7NgxcfHiRfHBBx8IT09PkZeXZ1wvKSlJeHt7i6efflqcOXNGnDlzRjQ0NIhBgwaJp556Spw9e1bs2rVL3HfffSbh9c0334iAgACRnp4uzp07J06cOCHi4uLE9OnTO6yntbVViV9Nr2B4SeChhx4SS5YsMd4+efKkcHJyEpcuXRJCCNHW1iZCQkLExo0bO9zGxx9/LAYPHmy8vWDBAvHss8+atNm/f79QqVTi1q1bvbsDDuzuvrNk8eLF4mc/+5nxdlJSkggKChI6nc64bOPGjWLw4MEmfbNp0yaT8Fq1apWIj4832fbly5cFAFFaWmp1PbLgnJeEIiMjMWbMGGzbtg0A8NVXX+Hq1at4/PHHjW3+9re/4Yc//CFCQkLg4+ODp59+GteuXTN+lfqpU6ewZcsWeHt7G/8lJCTAYDCgvLxckf1yFDk5OYiKikJAQAC8vb3x1ltvobKy0qTN/fffDzc3N+Pt0tJSjB8/Hu7u7sZld38p7alTp7Bv3z6TPg0PDwcAXLhwwYZ7pAyGl6SefPJJY3ht27YNM2bMwODBgwEAFRUVePTRRzF+/Hj8+c9/RlFREXJycgB8O/nb2NiIX/7ylyguLjb+O3XqFP75z39i5MiRyuyUA9i+fTvS0tKwYMEC7N27F8XFxUhOTjablPfy8ur2thsbGzFz5kyTPi0uLsY///lP/Od//mdv7UK/4XDX85KRm5sb2traTJbNmzcPK1asQFFRET755BPk5uYa7ysqKoLBYMCaNWuMF3P705/+ZLL+xIkT8Y9//AOjRo2y/Q44sLv77uDBg5gyZQoWLVpkXGbNqGj06NH44IMPoNPpjB/MPnbsmEmbiRMn4s9//jPCwsLg4mL5pW3puSQrjrwkEBYWhq+//hoVFRWor6+HwWBAWFgYpkyZggULFqCtrQ0/+clPjO1HjRoFvV6PP/7xj7h48SLef/99k3ADgOXLl+PQoUNITU01vjt/9tlnSE1N7evds2t39933vvc9HD9+HHv27MH58+fxyiuvmIWQJfPmzYPBYMCzzz6Lc+fOYc+ePVi9ejUAGK8IvHjxYly/fh1z587FsWPHcOHCBezZswfJycnGwLL0XJKW0pNu1LXS0lLxwAMPCA8PD5M/b2/YsEEAEImJiWbrZGdni+DgYOHh4SESEhLE1q1bBQDxzTffGNscPXpUxMXFCW9vb+Hl5SXGjx8vXn311T7aK8dwd9+VlJSI+fPnCz8/PzFgwACRkpIiXnrpJREREWFcp/1UibsdPHhQjB8/Xri5uYmoqCixbds24zbbnT9/Xvz0pz8VAwYMEB4eHiI8PFy88MILwmAwWKxH5lMleEkcIkl9+OGHSE5ORkNDAzw8PJQup89xzotIElu3bsWIESMQEhKCU6dOYfny5XjiiSccMrgAhheRNGpqapCRkYGamhoEBwfj8ccfx6uvvqp0WYrhYSMRSYl/bSQiKTG8iEhKDC8ikhLDi4ikxPAiIikxvMim5s+fDycnJzg5OcHV1RVBQUGIi4vD5s2bu/XRlC1btmDAgAG2K7QD8+fPx+zZs/v8calrDC+yuRkzZqC6uhoVFRX461//iunTp2PJkiV49NFH0draqnR5JCtlP51E9q6jz+kVFBQIAGLTpk1CiM4vW71v3z6zyxdnZmYKIYTYunWriIqKEt7e3iIoKEjMnTtX1NbWGh/n+vXrYt68ecLf31+4u7uLUaNGic2bNxvvr6ysFI8//rjw8/MTAwcOFD/5yU+Mn/fLzMw0e9x9+/bZ5PdE3ceRFyniBz/4ASIiIrBjxw4AgEqlwh/+8AecPXsW7733Hr788kssW7YMADBlyhSsW7cOvr6+qK6uRnV1NdLS0gAAer0eq1atwqlTp7Bz505UVFRg/vz5xsd55ZVX8I9//AN//etfce7cOWzcuBH+/v7GdRMSEuDj44P9+/fj4MGD8Pb2xowZM9DS0oK0tDQ88cQTxpFjdXU1pkyZ0re/KOqY0ulJ9q2jkZcQQsyZM0eMGTPG4n13X7b63XffFX5+fl0+3rFjxwQA46ht5syZIjk52WLb999/X4wePdp4xQUhhNDpdMLDw0Ps2bOny/pJWRx5kWKEEMZrUXV12eqOFBUVYebMmRg2bBh8fHzw0EMPAYDxssopKSnYvn07IiMjsWzZMhw6dMi47qlTp1BWVgYfHx/jZZMHDRqE27dv2+Vlk+0Nw4sUc+7cOQwfPtyqy1Zb0tTUhISEBPj6+uLDDz/EsWPH8Omnn5qs98gjj+DSpUtYunQpqqqq8MMf/tB4yNnY2IioqCizyyafP38e8+bNs/He03fFq0qQIr788kucPn0aS5cuteqy1ZYuX1xSUoJr167h9ddfR2hoKADg+PHjZo8VEBCApKQkJCUlYdq0afjVr36F1atXY+LEicjLy0NgYCB8fX0t1mlPl022Nxx5kc3pdDrU1NTgypUrOHHiBF577TXMmjULjz76KBITE626bHVYWBgaGxtRUFCA+vp6NDc3Y9iwYXBzczOu9/nnn2PVqlUm62VkZOCzzz5DWVkZzp49i7/85S8YM2YMgDtfYuLv749Zs2Zh//79KC8vR2FhIf7rv/4L//rXv4yP+z//8z8oLS1FfX099Hp93/zSqGtKT7qRffv3b2l2cXERAQEBIjY2VmzevFm0tbUZ21lz2ernnntODB482ORUiW3btomwsDChVqtFTEyM+Pzzz82+y3DMmDHCw8NDDBo0SMyaNUtcvHjRuM3q6mqRmJgo/P39hVqtFiNGjBALFy4UDQ0NQgghrl69arxUNniqRL/C63kRkZR42EhEUmJ4EZGUGF5EJCWGFxFJieFFRFJieBGRlBheRCQlhhcRSYnhRURSYngRkZQYXkQkJYYXEUnpfwH4TJ/ulZLqiQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-22T22:04:08.196738Z", + "iopub.status.busy": "2024-03-22T22:04:08.196454Z", + "iopub.status.idle": "2024-03-22T22:04:08.463317Z", + "shell.execute_reply": "2024-03-22T22:04:08.462466Z" + }, + "papermill": { + "duration": 0.288043, + "end_time": "2024-03-22T22:04:08.465249", + "exception": false, + "start_time": "2024-03-22T22:04:08.177206", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABWfUlEQVR4nO2dd3xT1fvH30mbpHuw2kJbKFDK3oLFAUjZsvQnU5aIyhABEQEZCmpRAUEREJElVESU8VUBARkiQ0aBUoaUVUYn0E3bjPv7IyQ0dCUlbdpy3q/XfZHce+69zyXNJ+c85znPI5MkSUIgEAjKGHJbGyAQCARFQYiXQCAokwjxEggEZRIhXgKBoEwixEsgEJRJhHgJBIIyiRAvgUBQJhHiJRAIyiT2tjagpNHpdNy+fRtXV1dkMpmtzREIBI8gSRKpqalUrVoVuTz//tUTJ163b9/Gz8/P1mYIBIJCuHHjBr6+vvkef+LEy9XVFdD/x7i5udnYGoFA8CgpKSn4+fkZv6v58cSJl2Go6ObmJsRLICjFFObWsanDfunSpTRu3NgoJMHBwWzfvj3f9qtXr0Ymk5lsDg4OJWixQCAoLdi05+Xr68vcuXMJDAxEkiTWrFlDr169CA8Pp0GDBnme4+bmxsWLF43vhdNdIHgysal49ejRw+T9J598wtKlSzly5Ei+4iWTyfD29i4J8wQCQSmm1Pi8tFotP//8M+np6QQHB+fbLi0tjerVq6PT6WjevDmffvppvkIHkJWVRVZWlvF9SkpKobZIkoRGo0Gr1Vr2EIJSi52dHfb29qKnXo6wuXhFREQQHBxMZmYmLi4ubN68mfr16+fZNigoiJUrV9K4cWOSk5OZN28ebdq0ITIyMt8p1dDQUD766COz7cnOziYmJoaMjIwiPY+g9OLk5ISPjw9KpdLWpgisgMzWmVSzs7OJjo4mOTmZTZs2sWLFCvbv35+vgOVErVZTr149BgwYwJw5c/Jsk1fPy8/Pj+Tk5FyzjTqdjkuXLmFnZ0flypVRKpXil7ocIEkS2dnZJCQkoNVqCQwMLDD4UWB91FodCjvz/s9TUlJwd3fP8zuaE5v3vJRKJbVr1wagRYsWHDt2jEWLFvHtt98Weq5CoaBZs2ZERUXl20alUqFSqcyyJTs7G51Oh5+fH05OTuY9gKBM4OjoiEKh4Pr162RnZ4tZ6hJm0s+nuXYng6ld6/J0zYpWuWap+/nR6XQmPaWC0Gq1RERE4OPjY1UbxK9y+UR8rrYhI1vDn5FxnL6RhMreep+BTXteU6dOpWvXrvj7+5OamkpYWBj79u1j586dAAwZMoRq1aoRGhoKwOzZs3n66aepXbs2SUlJfPHFF1y/fp3XX3/dlo8hEAgKYPf5eO6rtfhXcKKpn4fVrmtT8YqPj2fIkCHExMTg7u5O48aN2blzJx07dgQgOjra5Nfy3r17jBw5ktjYWDw9PWnRogWHDh0yyz8mEAhsw7ZTtwDo2aSqVX3INnfYlzQFOQMzMzO5evUqAQEBwidSBhk2bBhJSUls2bIlz+Pi8y15kjKyeeqT3ai1ErsmPE+gV8HrFcF8h71wApQT2rVrx/jx421thkBgwvazsai1EvV83MwSLksQ4iUoVajValubILAiW3MMGa2NEK9CkCSJjGyNTTZzR/TDhg1j//79LFq0yLhg3dfXl6VLl5q0Cw8PRy6Xc/36dQAWLFhAo0aNcHZ2xs/Pj9GjR5OWlmZyzsGDB3nuuedwdHTEz8+PcePGkZ6ebpZdMTExdO/eHUdHRwICAggLC6NGjRosXLjQ2EYmk7F06VJ69uyJs7Mzn3zyCVqtlhEjRhAQEICjoyNBQUEsWrTI5NparZaJEyfi4eFBxYoVmTx5stn/X4KSISb5Pkev3gWgRxPrRgRAKYjzKu3cV2upP3OnTe59bnZnnJSFf0SLFi3iv//+o2HDhsyePRuA+fPnExYWxqhRo4zt1q9fzzPPPEP16tUBfejAV199RUBAAFeuXGH06NFMnjyZJUuWAHD58mW6dOnCxx9/zMqVK0lISGDs2LGMHTuWVatWFWrXkCFDSExMZN++fSgUCiZOnEh8fHyudh9++CFz585l4cKF2Nvbo9Pp8PX15eeff6ZixYocOnSIN954Ax8fH/r27Wt8vtWrV7Ny5Urq1avH/Pnz2bx5My+88ELh/7GCEuG30zFIEjxVwxNfT+vHTQrxKge4u7ujVCpxcnIyLlofNGgQ8+fPJzo6Gn9/f3Q6HRs2bGD69OnG83L6yGrUqMHHH3/MW2+9ZRSv0NBQBg0aZGwXGBjIV199Rdu2bVm6dGmBTu8LFy6we/dujh07RsuWLQFYsWIFgYGBudoOHDiQ4cOHm+zLuaQrICCAw4cPs3HjRqN4LVy4kKlTp/LSSy8BsGzZMmOIjaB0sO30baB4howgxKtQHBV2nJvd2Wb3LipNmzalXr16hIWFMWXKFPbv3098fDyvvPKKsc3u3bsJDQ3lwoULpKSkoNFoyMzMJCMjAycnJ06fPs2ZM2dYv3698RxJktDpdFy9epV69erle/+LFy9ib29P8+bNjftq166Np6dnrrYGccvJN998w8qVK4mOjub+/ftkZ2fTtGlTAJKTk4mJiaF169bG9vb29rRs2VIMHUsJlxPSiLiVjJ1cRrdG1h8yghCvQpHJZGYN3UojgwYNMopXWFgYXbp0oWJF/dKMa9eu8eKLLzJq1Cg++eQTKlSowMGDBxkxYgTZ2dk4OTmRlpbGm2++ybhx43Jd29/f32p2Ojs7m7zfsGEDkyZNYv78+QQHB+Pq6soXX3zB0aNHrXZPQfGy7ZS+1/VcYCUqupi3PM9Syua3UpALpVKZK4XPwIEDmT59OidOnGDTpk0sW7bMeOzEiRPodDrmz59vDATeuHGjyfnNmzfn3LlzxrWnlhAUFIRGoyE8PJwWLVoAEBUVxb179wo9959//qFNmzaMHj3auO/y5cvG1+7u7vj4+HD06FGef/55ADQaDSdOnDDp6QlsgyRJ/K+Yh4wgZhvLDTVq1ODo0aNcu3aNxMREdDodNWrUoE2bNowYMQKtVkvPnj2N7WvXro1arebrr7/mypUr/PDDDybiBvD+++9z6NAhxo4dy6lTp7h06RJbt25l7NixhdpTt25dQkJCeOONN/j3338JDw/njTfewNHRsdAo68DAQI4fP87OnTv577//mDFjBseOHTNp88477zB37ly2bNnChQsXGD16NElJSeb/hwmKjbO3UriSmI7KXk6nBsWXOFSIVzlh0qRJ2NnZUb9+fSpXrkx0dDSgHzqePn2aPn364OjoaGzfpEkTFixYwGeffUbDhg1Zv369cQ2pgcaNG7N//37+++8/nnvuOZo1a8bMmTOpWtW8X9O1a9fi5eXF888/T58+fRg5ciSurq6FRre/+eabvPTSS/Tr14/WrVtz584dk14YwLvvvsvgwYMZOnSocWjZp08fs+wSFC+G2K6Q+l64qIpvcCeWB+VALB8pXm7evImfnx+7d++mQ4cOJX5/8fkWP1qdRJu5e4hLyWL54BZF6nmVmXxegvLLX3/9RVpaGo0aNSImJobJkydTo0YNo59KUP749+pd4lKycHOwp21Q5WK9lxAvQZH4+++/6dq1a77H09LSUKvVTJs2jStXruDq6kqbNm1Yv349CoWiBC0VlCTbTuuHjF0b+qCyL3qojzkI8RIUiZYtW3Lq1KkC23Tu3JnOnW0TIycoebI1Ov6IiAWgV9Pim2U0IMRLUCQcHR2LFEIhKL8c+C+B5PtqqriqaG2lVM8FIWYbBQKBVdj6ILbrxcZVsZMXf+EaIV4CgeCxSc/SsOtcyQ0ZQYiXQCCwArvPx5Gp1lGjohONfd1L5J5CvAQCwWOz9dTD5UAlVevUpuK1dOlSGjdujJubG25ubgQHB7N9+/YCz/n555+pW7cuDg4ONGrUiD/++KOErBUIBHlxNz2bA/8lANCzhIaMYGPx8vX1Ze7cuZw4cYLjx4/zwgsv0KtXLyIjI/Nsf+jQIQYMGMCIESMIDw+nd+/e9O7dm7Nnz5aw5eWLvLKb5lfEQiB4lD8iYtDoJBpUdaN2FevmqS8Im4pXjx496NatG4GBgdSpU4dPPvkEFxcXjhw5kmf7RYsW0aVLF9577z3q1avHnDlzaN68OYsXLy5hy8s3MTExBQagCgQ5Ke6kg/lRanxeWq2WDRs2kJ6eTnBwcJ5tDh8+TEhIiMm+zp07c/jw4Xyvm5WVRUpKiskmKBhvb29UquLJwWQukiSh0WhsaoOgcG4n3edfY576J0y8IiIicHFxQaVS8dZbb7F58+Z8i8jGxsbi5eVlss/Ly4vY2Nh8rx8aGoq7u7tx8/Pzs8xASYLsdNtsFqyZT01NZdCgQTg7O+Pj48OXX35Z5HJoOYeN165dQyaT8euvv9K+fXucnJxo0qRJrh+Mwgp1/PDDD7Rs2RJXV1e8vb0ZOHCgST77ffv2IZPJ2L59Oy1atEClUnHw4EGLbReULIa8Xa0CKlDVw7GQ1tbF5hH2QUFBnDp1iuTkZDZt2sTQoUPZv3+/1apgT506lYkTJxrfp6SkWCZg6gz4tGR/UYxMuw1K58LbARMnTuSff/5h27ZteHl5MXPmTE6ePGlMnfy4fPDBB8ybN4/AwEA++OADBgwYQFRUFPb29mYV6lCr1cyZM4egoCDi4+OZOHEiw4YNyzXhMmXKFObNm0fNmjXzTBktKF0YhowlFduVE5uLl1KpNC4zadGiBceOHWPRokV8++23udp6e3sTFxdnsi8uLs5YdCIvVCqVzYdAxU1qaipr1qwhLCzMmGpm1apVZufdModJkybRvXt3QF8co0GDBkRFRVG3bl2zCnW89tprxmvVrFmTr776iqeeeoq0tDRcXFyMx2bPnk3Hjh2tZreg+IiKTyXydgr2chndGhZPnvqCsLl4PYpOpyMrKyvPY8HBwezZs8dkKLRr1658fWRWQeGk7wHZAoV55aKuXLmCWq2mVatWxn3u7u4EBQVZzZTGjRsbX/v46P9Q4+PjqVu3rlmFOk6cOMGHH37I6dOnuXfvHjqdDoDo6GiTXnZexTgEpRNDnvrn61TG01lZ4ve3qXhNnTqVrl274u/vT2pqKmFhYezbt89YwmrIkCFUq1bNmOHznXfeoW3btsyfP5/u3buzYcMGjh8/zvLly4vPSJnM7KFbeSZnGhtDEKJBgAor1JGenm7MMLF+/XpjptfOnTuTnZ1t0v7RYhyC0okkSca1jLYYMoKNxSs+Pp4hQ4YQExODu7s7jRs3ZufOncZhQ3R0tLE4BECbNm0ICwtj+vTpTJs2jcDAQLZs2ULDhg1t9Qilgpo1a6JQKDh27Jixqk9ycjL//fdfiST+K6xQR0REBHfu3GHu3LlGf+Px48eL3S5B8XHmZjLX72TgoJATUs+r8BOKAZuK1/fff1/g8X379uXa98orr5jUHhSAq6srQ4cO5b333qNChQpUqVKFWbNmIZfLS2Spxvvvv8/TTz/N2LFjef3113F2dubcuXPs2rWLxYsX4+/vj1Kp5Ouvv+att97i7NmzzJkzp9jtEhQfhuVAHet741yMeeoLwuahEgLrsGDBAoKDg3nxxRcJCQnhmWeeoV69eiWSq72wQh2VK1dm9erV/Pzzz9SvX5+5c+cyb968YrdLUDxodRL/O/NgyFjCsV05EQU4clCeCjSkp6dTrVo15s+fz4gRI2xtTqmgPH2+tuRQVCIDVxzF3VHBsQ9CUNpbtw8kCnA8YYSHh3PhwgVatWpFcnIys2fPBqBXr142tkxQ3jAMGbs18ra6cFmCEK9yxLx587h48SJKpZIWLVrw999/c/78+UILZQgE5pKl0fLH2RgAejapZlNbhHiVE5o1a8aJEydy7b9//36hhTIEAnPZfzGB1EwN3m4OtAqoYFNbhHiVc0ShDIE1eZin3qdE8tQXhJhtzIMnbA7jiUF8ro9HWpaG3ef0y/N6NbXtkBGEeJlgiCLPyMiwsSWC4sDwuYqit0Xjz8hYsjQ6alZypmG1/GcBSwoxbMyBnZ0dHh4exlQtTk5OJZaPW1B8SJJERkYG8fHxeHh4YGdXvJWcyyuGDBI9SjBPfUEI8XoEQ4aKnLmmBOUDDw+PAjOQCPLnTloWf19KBEo2T31BCPF6BJlMho+PD1WqVEGtVtvaHIGVUCgUosf1GPwREYNWJ9Gomju1KrsUfkIJIMQrH+zs7MQfu0DwAFsmHcwP4bAXCAQFcvNeBseu3UMmgxcbC/ESCARlhP+d1kfUtw6ogLd76VkTKsRLIBAUyNZTt4DSEduVEyFeAoEgX/6LS+VCbCoKOxldG5aumVohXgKBIF8Meerb1qmMh1PJ56kvCCFeAoEgTyRJelgNu5QNGUGIl0AgyIdTN5KIvpuBk9KOkHpVbG1OLoR4CQSCPHmYp94LJ2XpCwm1qXiFhoby1FNP4erqSpUqVejduzcXL14s8JzVq1cjk8lMNpHSVyCwLhqtjt/O6EMkSlNgak5sKl779+9nzJgxHDlyhF27dqFWq+nUqRPp6ekFnufm5kZMTIxxu379eglZLBA8GRy5cpfEtCw8nRQ8F1j58S946yTs/wKsmJbosfqCaWlpxsKjBgpKmP8oO3bsMHm/evVqqlSpwokTJwqsNyiTycQCW4GgGDHEdnVt5IPC7jH7OPEXYN3LcP8uOHpAq5GPbyBF6HldvXqV7t274+zsjLu7O56ennh6euLh4YGnp+djGZOcnAxAhQoFp5dNS0ujevXq+Pn50atXLyIjI/Ntm5WVRUpKiskmEAjyJ1OtZcfZWMAKpc3uXoW1vfTCVbU5NOlvBQv1WNzzevXVV5EkiZUrV+Ll5WW1vD46nY7x48fzzDPPFFgBOygoiJUrV9K4cWOSk5OZN28ebdq0ITIyEl9f31ztQ0ND+eijj6xio0DwJLDvYjypWRp83B14qsZj5KlPuQ1re0JaLFSpD6/+AipXq9lpcd1GFxcXTpw4QVBQkNWMABg1ahTbt2/n4MGDeYpQfqjVaurVq8eAAQPyrMKclZVFVlaW8X1KSgp+fn6F1oQTCJ5URq8/wR8RsbzxfE2mdatXtIukJ8KqbpB4ETwD4LUd4Gqeq6fY6jY+9dRT3Lhxw6riNXbsWH777TcOHDhgkXCBPk9Ts2bNiIqKyvO4SqVCpVJZw0yBoNyTmqlm93l9Is6eRR0yZibDupf0wuVaFYZsNVu4LMFi8VqxYgVvvfUWt27domHDhrnygTdu3Njsa0mSxNtvv83mzZvZt28fAQEBlpqDVqslIiKCbt26WXyuQCAwZWdkHNkaHbUqO9OgahFGJtkZENYPYk6DU0W9cHlWt76hFEG8EhISuHz5MsOHDzfuk8lkSJKETCZDq9Wafa0xY8YQFhbG1q1bcXV1JTZW7yR0d3fH0dERgCFDhlCtWjVCQ0MBmD17Nk8//TS1a9cmKSmJL774guvXr/P6669b+igCgeARHiYdrGa5P1uTBT+9CtGHQeUOgzdD5TrFYKUei8Xrtddeo1mzZvz444+P7bBfunQpAO3atTPZv2rVKoYNGwZAdHQ0cvnDSdF79+4xcuRIYmNj8fT0pEWLFhw6dIj69esX2Q6BQAAJqVn8E/UgT72lQ0atBn55HS7vAYUTDNoIPk2KwcqHWOywd3Z25vTp02W2kKm5zkCB4EljzaFrzNoWSRNfd7aOfdb8E3U62DYWTq0HOyUM2AC1OxTZDnO/oxbHeb3wwgucPn26yIYJBILSSZEySEgS7JyqFy6ZHF7+/rGEyxIsHjb26NGDCRMmEBERQaNGjXI57Hv27Gk14wQCQclw424GJ64b8tT7mH/i3k/h6DL9615LoH7Jff8tFq+33noL0DvOH8VSh71AICgdGHpdwTUr4uVmZqKDQ1/Dgc/1r7vNg6YDism6vLFYvB5dyygQCMo+hoypZmeQOL4K/pyuf/3CDKutV7QEi3xearUae3t7zp49W1z2CASCEuZCbAoX41JR2snp0sCMIWPEJvhtgv71M+/Ac+8Wr4H5YJF4KRQK/P39xdBQIChHGPPUB1XG3UlRcOOLO2Dzm4AELV+DkI/ASuubLcXi2cYPPviAadOmcffu3eKwRyAQlCA589QXOmS8egA2DgGdBhq9At3m20y4oAg+r8WLFxMVFUXVqlWpXr06zs7OJsdPnjxpNeMEAkHxcjI6iZv37uOstKNDXa/8G948Dj8OAG0WBHWD3ktBbtss8haLV+/evYvBDIFAYAu2PUg62KmBN45Ku7wbxUXqkwlmp0FAW/i/VWBXyPCyBLBYvGbNmlUcdggEghImZ576nvkNGe9chrW9ITMJfJ+C/mGgKB01I4qcBvrEiROcP38egAYNGtCsWTOrGSUQCIqffy7f4U56NhWclTxbu1LuBsk39VlQ0+PBqyEM+hlULiVvaD5YLF7x8fH079+fffv24eHhAUBSUhLt27dnw4YNVK5shWT9AoGg2DHMMnbPK099WoJeuJJvQIVa+gwRjo+X5t3aWOxxe/vtt0lNTSUyMpK7d+9y9+5dzp49S0pKCuPGjSsOGwUCgZXJVGvZGalPQZVryHg/Cdb1gTtR4Oarz8nlUvqKzlrc89qxYwe7d++mXr2H6WHr16/PN998Q6dOnaxqnEAgKB7+uhBPWpaGah6OtPDP0aPKToewvhAbAc6V9cLl4Wc7QwvA4p6XTqfLtRgb9AGsYumQQFA2MAwZezSpilz+IFZLkwUbBsGNo+DgDoO3QKXSm/qqSClx3nnnHW7fvm3cd+vWLSZMmECHDiWTCkMgEBSd5Ptq/rr4SJ56rQY2vQZX9oLCGQb9At75V/EqDVgsXosXLyYlJYUaNWpQq1YtatWqRUBAACkpKXz99dfFYaNAILAiOyNjydboCKziQj0fV30ywa1j4MJvD5IJhoHfU7Y2s1As9nn5+flx8uRJdu/ezYULFwCoV68eISEhVjdOIBBYn//lWA4kA9j+HpzZADI7eGUN1GxnS/PMpkhxXjKZjI4dO9KxY0dr2yMQCIqR+NRMY576Hk2qwp7ZcGwFIIM+30LdslOFq0jitWfPHvbs2UN8fHwuJ/3KlSvNvk5oaCi//vorFy5cwNHRkTZt2vDZZ58VWhPy559/ZsaMGVy7do3AwEA+++wzUfpMIDCD38/EoJOgqZ8H1c8vh4ML9AdeXACNX7GtcRZisc/ro48+olOnTuzZs4fExETu3btnslnC/v37GTNmDEeOHGHXrl2o1Wo6depEenp6vuccOnSIAQMGMGLECMLDw+nduze9e/cWOcYEAjPY+mCW8f2Kf8PuD/U7Qz7Sp7cpY1hcPcjHx4fPP/+cwYMHW92YhIQEqlSpwv79+3n++efzbNOvXz/S09P57bffjPuefvppmjZtyrJlywq9h6geJHhSib6TwfNf7KWP3UG+VCzR73zuXegw07aGPUKxVQ/Kzs6mTZs2j2VcfiQnJwNQoUKFfNscPnw41+RA586dOXz4cJ7ts7KySElJMdkEgieRbadv0Ul+jHmKBz/yrd7Qp3Auo1gsXq+//jphYWFWN0Sn0zF+/HieeeYZGjbMP74kNjYWLy/TvENeXl7GatuPEhoairu7u3Hz8yud0cICQXEiSRLRx//ga8XX2KGDJgOgy2c2TSb4uFjssM/MzGT58uXs3r2bxo0b54q2X7BgQZEMGTNmDGfPnuXgwYNFOj8/pk6dysSJE43vU1JShIAJnjiun9rHh+mfoJJpUNd5EUXPxTZPJvi4WCxeZ86coWnTpgC5nOSyIqr42LFj+e233zhw4AC+vr4FtvX29iYuLs5kX1xcHN7e3nm2V6lUqFSqItklEJQLYiPw/m0wDrIsIh1b0qDvSrArcjasUoPFT7B3716r3VySJN5++202b97Mvn37CAgIKPSc4OBg9uzZw/jx4437du3aRXBwsNXsEgjKDYmXkH7og4M2lWO6OtzttJwG9uXjx9ym8jtmzBjCwsLYunUrrq6uRr+Vu7s7jo6OAAwZMoRq1aoRGhoKwDvvvEPbtm2ZP38+3bt3Z8OGDRw/fpzly5fb7DkEglJJUjSs7YUsPYFIXXXGyaayt2ENW1tlNWw66F26dCnJycm0a9cOHx8f4/bTTz8Z20RHRxMTE2N836ZNG8LCwli+fDlNmjRh06ZNbNmypUAnv0DwxJEap08mmHKLBFV1BmdPpU2DWjgo8slTXwaxOM6rrCPivATlnoy7sPpFiI9Ecveja+p0LmS4sua1VrStU/ozHRdbnJdAICjFZKXC+lcgPhJcvDj63CouZLhS0VnJM7Uq2to6q2KxeB04cACNRpNrv0aj4cCBA1YxSiAQFAF1JmwYCLeO6/PND97Cxsv6UKYXG/tg/2ie+jKOxU/Tvn37PKtlJycn0759e6sYJRAILESrhp+H6ataK13g1V+47xmUf576coDF4iVJUp7xXHfu3MlVPVsgEJQAOi1sfgv+2w72DjBgA1RrwZ4LcaRna/H1dKS5f+mq/GMNzA6VeOmllwB9IOqwYcNMAj+1Wi1nzpwptjWPAoEgHyQJfp8IZzeB3B76roWA54CHeep7Nqla5ADy0ozZ4uXu7g7oe16urq7GOCwApVLJ008/zciRI61voUAgyBtJgl0z4cRqQAYvLYc6nQFIzlCz72ICAL2aVrOdjcWI2eK1atUqAGrUqMGkSZPEEFEgsDV/z4NDX+lf91gEDV82HtoRGUO2VkeQlytB3q42MrB4sTjCftasWcVhh0AgsISj38JfH+tfd/oEWgw1OWxIOlgeHfUGLHbYx8XFMXjwYKpWrYq9vT12dnYmm0AgKGZOhcH2yfrXbd+HNmNNDsenZHL4yh0gR2mzcojFPa9hw4YRHR3NjBkz8PHxKZeOQIGg1HJuq75MGcDTo6Hd1FxN/ncmBkmC5v4e+FVwKmEDSw6LxevgwYP8/fffxrQ4AoGghIjaDZtGgKSDZq9C50/zTCa47dQtoPw66g1YPGz08/PjCVsOKRDYnuuHYcOroFND/d7Q46s8hetaYjqnbyZjJ5fRrZFPydtZglgsXgsXLmTKlClcu3atGMwRCAS5uH0KwvqC5j7U7ggvfQfyvP3L2x4UlG1TqyKVXctH3q78sHjY2K9fPzIyMqhVqxZOTk650kDntXRIIBAUkYSLsO4lyEoB/zb6IFR7ZZ5NJUliyxMyZIQiiNfChQuLwQyBQJCLe9f0Obky7kDVZjDwJ1Dm74CPvJ3ClYR0lPZyOjfwyrddecFi8Ro6dGjhjQQCweOREqMXrtQYqFwXXv0VHArOP/e/B0PGkHpVcHVQFNi2PFCkHBmXL19m+vTpDBgwgPj4eAC2b99OZGSkVY0TCJ5IMu7CD330PS/PGjB4CzjlX8sUQKeTjP6u8hzblROLxWv//v00atSIo0eP8uuvv5KWlgbA6dOnRfS9QPC4ZKbofVwJ58HVB4ZsBbfCZw2PXbtLTHImrip72gVVKQFDbY/F4jVlyhQ+/vhjdu3ahVL50HH4wgsvcOTIEasaJxA8Uajvw4/94XY4OFbQ97g8a5h1qqHX1aWhd7nKU18QFotXREQEffr0ybW/SpUqJCYmWsUogeCJQ5MNG4fA9X9A5QaDf4Uqdc06NVuj4/cIfZGa8ryW8VEsFi8PDw+Taj4GwsPDqVbNsunZAwcO0KNHD6pW1ecb2rJlS4Ht9+3bh0wmy7UZSqYJBGUSnRZ+HQmX/gR7R/2sYtVmZp9+MCqBpAw1lVxUBNcsX3nqC8Ji8erfvz/vv/8+sbGxyGQydDod//zzD5MmTWLIkCEWXSs9PZ0mTZrwzTffWHTexYsXiYmJMW5VqjwZY3xBOUSS4H/vwLktIFdA/3VQ3bKknoakg+UxT31BWBwq8emnnzJmzBj8/PzQarXUr18frVbLwIEDmT59ukXX6tq1K127drXUBKpUqYKHh4fF5wkEpQpJgp0fQPgPIJPD/30PtUMsukRGtoY/z8UB0OsJGjJCEcRLqVTy3XffMWPGDM6ePUtaWhrNmjUjMDCwOOzLk6ZNm5KVlUXDhg358MMPeeaZZ/Jtm5WVRVZWlvF9SkpKSZgoEBTO/s/gyINRR8/FUL+XxZfYfT6ejGwt/hWcaOrnYV37SjkWi5cBf39//P39rWlLofj4+LBs2TJatmxJVlYWK1asoF27dhw9epTmzZvneU5oaCgfffRRidopEBTK4SWwL1T/ustn0GxQkS5jyCBRXvPUF4RZFbMnTpzInDlzcHZ2ZuLEiQW2XbBgQdEMkcnYvHkzvXv3tui8tm3b4u/vzw8//JDn8bx6Xn5+fqJitsB2nFwL297Wv24/Hdq+V6TLpGdpaDr7T9RaiV0TnifQq3ykeza3YrZZPa/w8HDUarXxdX7YQvlbtWrFwYMH8z2uUqlMKh0JBDbl7K+wbZz+dZu34flJRb7U5YQ01FqJSi7KciNclmCWeO3duzfP16WBU6dO4eNTvvMWCcoJ//2pD4lAghbDoOOcPHNymcuVhHQAalZysY59ZYwi+7ysQVpaGlFRUcb3V69e5dSpU1SoUAF/f3+mTp3KrVu3WLt2LaDPaBEQEECDBg3IzMxkxYoV/PXXX/z555+2egSBwDyuHYSNg0Gn0Vf56b7gsYQL4EqCfmlezcpPZiUvs8TLUHDWHH799Vez2x4/fpz27dsb3xv8aUOHDmX16tXExMQQHR1tPJ6dnc27777LrVu3cHJyonHjxuzevdvkGgKBNbkUl0rErWR6NKmKoqgxVLdOQlh/0GRCnS7Q59t8kwlawuXEBz0vIV75Yyg4C/qEZ5s3b8bd3Z2WLVsCcOLECZKSkiwSOYB27doVmFJ69erVJu8nT57M5MmTLbqHQFAUMrI1LNp9iRUHr6LVSbg7KuhQz/IcWedOH8V3y8u4SalQ4zl4ZTXYWSddjRg2moGh4CzA+++/T9++fVm2bJmx1JlWq2X06NFi9k5QLjh1I4kx609yK+m+cV/yfbXF14m6cIZKm/vhRir/KYKoM+BHUDgWfqIZ6HQSVxOf7GGjxf3glStXMmnSJJMajXZ2dkycOJGVK1da1TiBoKRJSM3ijbXHuZV0H19PR6pX1Gcu1eosKzpz49olHDe8TBXucV7nxzj5B6Cy3oxgTEommWodCjtZuS5vVhAWi5dGo+HChQu59l+4cAGdTmcVowQCW6DVSUz46RTxqVkEVnFhx/jnqV1ZPyTTWVAxKy7mJto1vahGPDdlPgzJnsqlVIXFAlgQBme9fwWnovviyjgWzzYOHz6cESNGcPnyZVq1agXA0aNHmTt3LsOHD7e6gQJBSfH1X5c4GJWIo8KOJYOa46Kyx06unxHUmvm7fO9uIinf9SBQukW8rCKqEf/j3pJLaHUSCalZeLs7WMVWo7+r8pPp74IiiNe8efPw9vZm/vz5xtQ4Pj4+vPfee7z77rtWN1AgKAn+iUpk0Z5LAHzSp6Ex6POheBWuXmlpKdxe0pMGuivcxQ3tq1vw8Q3Ey+0Gt5LucyvpvhXF68n2d0ERxEsulxtn/QyLnIWjXlCWiU/J5J0N4UgS9H/Kj5ea+xqPyY3iVfCQL/N+Bpe/7k0TTSSpOJH6ys9Ur9UYgGoejtxKus/tpPu0qO5pFZuvPAiTqPWEzjTCYwapCtESlHW0Oom3fwwnMS2but6ufNizgclxuweBpNoCtEujziby6760yDpBhqQitscPBDZ42ni8qoe+t3U7x+zl4/Jw2Ch6XhaxadMmNm7cSHR0NNnZ2SbHTp48aRXDBIKS4PDlOxy9ehdnpd7P9Wj+d8OwUZdPz0un1XJy8WBaZfxNtmTPtY7fUb+laU4uHw99eERMcqZVbM7I1hjDOJ5kn5fF0xRfffUVw4cPx8vLi/DwcFq1akXFihW5cuVKkRILCgS2JOJWMgDt61bJUwjkxp5XbvGSdDqOLXuDVsk70Ehyzj+7iPrP5s7JVfWBeN2yUs/r6oMho4eTggrOeVfPfhKwWLyWLFnC8uXL+frrr1EqlUyePJldu3Yxbtw4kpOTi8NGgaDYiLyt/5utXzVvF4h9AT6voyvfpXXCJgBOtfiEJh1fzfMa1aw8bHwYWf/kDhmhCOIVHR1Nmzb6HNuOjo6kpqYCMHjwYH788UfrWicQFDPnYvSTTg2quud5PD+H/dH1H/L0TX1Q9tF602jZc3S+9/Bx1/e8rC5eT/CQEYogXt7e3ty9exfQZ1M11Gq8evVqgesUyyTl7XkEJmRka4xDsPo+efe8DPGfOcXr+C8LaH3pSwAO1RhL637vF3gfw7DxXoaa+9naxzWbK0/4siADFovXCy+8wLZt2wB9wOqECRPo2LEj/fr1y7OeY5lFq4ZfRsCJNba2RGBlfjtzm9fXHOPYtXtIElR2VVHZNe+ElYbZRkOE/ek/VtD8zGwA/vEeTPDQjwu9n5uDPS4q/dzY7eTH73096QuyDVg827h8+XLjMqAxY8ZQsWJFDh06RM+ePXnzzTetbqDNOLMRzv6iz3ypctHnYDIDSZL4IyKW+X9epKm/Bwv6Ni1eOwUW8f3Bq8z57RwAf12IB/LvdYHpsPHc3p9ocPQ95DKJQ569afPGV2ZlD5bJZFT1cOC/uDRuJ92n1mMM9yRJMgao1nrCe14WiZdGo+HTTz/ltddew9dXH8jXv39/+vfvXyzG2ZSmA+HWcTi+En59A5QuUKdzgadExafy4bZzHIzSVw6/cS+Def/XxPgFENgWSZKMwgVgGAnm56yHhz2v7Kh91EqYjr1MxxGXEFqN+R6Z3PyBi4+7I//FpRGT9HjhEvGpWaRna5HLwL/ik7kg24BFw0Z7e3s+//xzNBpNcdlTepDJoNt8aPSKPvvlxiFw9e88m6ZlaQj94zxdFv7NwahElPb6/1a1VuJeRnae5whKnpv38h6yFdTzspPLaCqLYkLCTFQyNccd2tBsXBj29pYNWqwVLnH5Qa/Lr4ITKvvHT2hYlrHY59WhQwf2799fHLaUPuRy6L0Ugrrps2D+2B9unjAeliSJbadv02H+Pr49cAWNTiKkXhV2T2hLJRd9/E1sinUCEwVFJyNbQ3j0Pe6m5/1DUlDPy/t+FKuVn+Esy+JvbUN0L3+PSml5QRdrhUuIMImHWOzz6tq1K1OmTCEiIoIWLVrg7Gz6n9izZ0+rGVcqsFPA/62CsFfg6gFY9xIM385/+DFraySHr9wB9KlJPuxZnxfq6rNterk5kJiWTVxKZr7T8IKS4dUVRzkZncTLOdYsGnBU2FGjYj5CcOcyfS+Mw1mWzgldIG+oJ7K3SoUi2WDoeT2uw16ESTzEYvEaPVofz5JXfUaZTIZW+/hTwaUOhQP0/xHW9oJbx0lb8SKj0qdzWeeFyl7O6Ha1ebNtTZOlJd5uDkTeTiE2OauACwtKgpPRSQD8cvJmrmN1fVyNS4BMSLoBa3vhrL7LOV11hmdPRmfvRJV8ZiULwxDr9bg+LxEm8RCLh406nS7fzVLhOnDgAD169KBqVX213y1bthR6zr59+2jevDkqlYratWvnynNfXEhKZ/5o8hWXqI6L+g5rFJ/Qt46c3RPb8k5IYK41cV4PUp8Yho2X4lIZG3aS8w+CIgWlgzz9XWnx8ENvSL7BPUd/BmdPIQVn/Cs4FXnypVoOn9fjxEOKMImH2DQFY3p6Ok2aNOGbb74xq/3Vq1fp3r077du359SpU4wfP57XX3+dnTt3FqudF2NT6b/8CKN/vcaAzCnckFXFV5bI5+kz8FOm53mOl6tevOIeLMb94ch1fjsTwxs/HCc10/J86ILiIdeQ/v49+OEluBMF7n5sa7yUO+jbVH+M2T0vdxUyGWRpdPn63gojS6Pl5r0MQIRJgAXDxvv377Nnzx5efPFFAKZOnUpW1sMhkZ2dHXPmzMHBwfxka127drVoMfeyZcsICAhg/vz5ANSrV4+DBw/y5Zdf0rlzwWEMRSElU83CXZdYc/gaWp2Eg0LOsPZPUbnxDljbDe5cgnV9YOhv4Ohhcq63u354EZeqFy/DTNeNu/eZtTWSBf2aWt1egeWYOOuz0mB9X4iLAOcqMGQrGREA+vWP/hWKLhgqezsquahISM3idlImFV0sH35ev5OBTgIXlX2+QbVPEmb3vNasWcO3335rfL948WIOHTpEeHg44eHhrFu3jqVLlxaLkQYOHz5MSIhpupHOnTtz+PDhfM/JysoiJSXFZDOHQ5cTeWHeflb+oy991aWBN7sntmXsC4E4VKoOQ7aCc2WIjYCwvpBt2gPzcnswbHzQ88o5y/Rr+C22nrpllh2C4qWez4OiGOpM2DAQbv4LDh4wZAtUrGVcmA2P1/OCx3fa58yeak5wbHnHbPFav349b7zxhsm+sLAw9u7dy969e/niiy/YuHGj1Q3MSWxsLF5eprXzvLy8SElJ4f79vP8gQkNDcXd3N25+fn5m3cvXw4mUTDUBlZxZ81orlg1uga9njj/eSrVh8GZwcIcbR2HDINA87Ika0v3GPfB5GeJ7XmzsA8D0zWe5cTfDvAcXFAvLXm2uj5XSqmHTa3B1Pyic4dVfwEuflDCnj+txg0IfN1zisgiTMMFs8YqKiqJRo0bG9w4ODshzRBi3atWKc+fO5XWqTZk6dSrJycnG7caNG2ad51/RiXUjWrNj/HO0rVM570bejWDQJv0f/JW9+i+AVh/A6/2g53UvQ01iWhapmfr9n77UiOb+HqRmaZi48RQacys7CKyOTCYDnQ62joGLv4OdCgZuAN+WxjZ2OTo41R+zxFjVx8wuIcIkTDFbvJKSkkx8XAkJCdSoUcP4XqfTmRwvDry9vYmLizPZFxcXh5ubG46OeRfzVKlUuLm5mWzm0iqgQuFRzH6tYEAY2Cnhwm+wbSzodLg7KoyR9uEPpuo9nBS4OShY2K8ZLip7jl27x9J9l822R2BlJAn+mARnfgK5PfRdAwHPmzS5l/FwcqWa5+MVjDVkVL1dxHAJESZhitni5evry9mzZ/M9fubMGeN6x+IiODiYPXv2mOzbtWsXwcHBxXrfQqnZTl/GXWYHp3+E7ZOR8bD3dTL6HvDwl9e/ohOze+mHJQv3XDIeFxSdm/cyWH7gskUzuYFnF8Dx7wEZ9PkWgnJPHuVcUvS4y3GMw8Yi+Lz0C7JFmEROzBavbt26MXPmTDIzc/9q3L9/n48++oju3btbdPO0tDROnTrFqVOnAH0oxKlTp4iOjgb0Q74hQ4YY27/11ltcuXKFyZMnc+HCBZYsWcLGjRuZMGGCRfctFup2hz7LABkc+w7+mmMUr3CDeHk8/OXu06waPZtURauTGL/hFGlZT8B60WJkwZ//8ekfF9gSbt5EyCi7bdS8sFz/5sUvodH/5dnORWW99YNGh30Rho1307NJvq8X5gDh8wIsCJWYNm0aGzduJCgoiLFjx1KnTh0ALl68yOLFi9FoNEybNs2imx8/fpz27dsb30+cOBGAoUOHsnr1amJiYoxCBhAQEMDvv//OhAkTWLRoEb6+vqxYsaJYwiSKROO+kJUKv0+Ev+czsFIa/9KW0zf0U+2GX17Q+1vm9G7Iiev3iL6bwaytkczv28RWlpdZbifd55cTN/nnsj6TR2Ja4TFUr9rt4n3FBv2bjnOgZf7Fkt/uEEhiejYDnvJ/bFsNUfbxqVlka3RGt4I5GEqdVfNwxFH5ZC/INmC2eHl5eXHo0CFGjRrFlClTjFHCMpmMjh07smTJklwzgYXRrl27AqON84qeb9euHeHh4Rbdp0R5agRkpcDuD+md+C3H7NSsV+vDOx71mbg7KviyX1P6Lz/MLydv0i6oMj2aVLWF1WUSnU6izdy/TPZlZBfcg+0j/5uPFasAiKo7itrPjCuwfSUXFd8MbP54hj6gorMSpb2cbI2OuJRM/CyYABBFZnNjUYR9QEAAO3bsICEhgSNHjnDkyBESEhLYsWMHNWvWLC4byx7PToBn9b3IOfar6CU/CJgOGw20CqjAmPa1AZi2OcJqFWaeBGb/lnt2O72ANMud5cf4QqGPVVyl6czZOmOLzba8kMtlVHUvWriEwd/1OIkMyxtFWh5UoUIFWrVqRatWrahQoWir7Ms9HWZyteZA5DKJ+YpldJQfz1O8AMZ1CKSJrzupmRq+O3ClhA0tm2Sqtaw+dC3X/ox8fIfPyiP4SvE19jIdP2ueZ7ZmMPc1JR+mUtRA1cuiyGwubLq2sVwjk5H43Bx+0T6HvUzHYsVXBKQcy7Opwk7O4OAaAOw+H0emuhxm5rAyx6/lPUOblpX7/6657D+WKxagkmn4Q9uKKZqRSMjJsEIxDEt5WEnIsnAJY5iEmGk0IsSrGPF2d2Ky+g12aJ9CJdPgsXUY3Pg3z7adG3hRxVXFzXv3+Xa/6H0Vxo17ea9OeNTnpbl1mtXKz3GSZbFf25jx6jFo0Tu87xfiHysOihJlr9bqiL6jf17R83qIEK9ipIqbCi12jFOP5V95E2TqdFj/f/r1kI/g6qBgxov1AfhmXxRHr9wx/sEK9Oh0EhuP3WDvxXhjdoVHMfF5JV5Cvv4l3GQZ/KsL4k31BLJRGA/boudVlHCJG3cz0OgkHBV2xvAbgRCvYkVlb4enk4JsFCyu/BH4tYbMZPihDyRG5Wr/YmMfngusRLZGR7/lR3hh/j7iU0UaaQP/O3Obyb+cYdS6E/nmozf6vJKiYW0v5BmJROhqMCL7PTIxzcRgW/Ey/3M1OOsDKjmLYi45EOJVzBiyS1Sq6AkDN+rXQ6Yn6LOyJpmus5TJZMzu1RDHB4kNNTqJdYevl7jNpZVzt/UZQTLVunzFKz1LA6lx+v/flFtketRmaPYUUtGHJahyxFY9WgW7JKhahGGjWBaUN0K8ihlDdolqHo76nF+vboaKgZByU/8FS4s3aR9QyZk/JzzPpE76IOBl+68QFZ9a0maXShJSH66dzTcjR2YSurW94O4V8KjO2Q5ruMvD9axOSjve6xxENQ9HRrevVdwm58LgsE/N0pBi5lKmy/FiQXZeCPEqZp6tXQl7uYzgmhX1O1wq63OBufvD3cv6IeR905kzvwpOjGlfmw51q5Ct1THllwh0NugllDYS0h6KV3xq7iQAVVRqlkifIE84j87FC4ZsJVFWyaSNk9KeMe1r88+UF4xCUpI4q+zxcNL73czNZ2/oeYnsqaYI8SpmXn+uJmc/6kyb2jm+RO7V9MnuXLwg7iysf0WfxTMHhuVDzko7jl+/R9i/0TzpJOQhWAb6Nq3ELu+lNJVf5q7kwijZDGLtfHIt1Ha24lrFouJjYWocsSA7b4R4lQCPFucAoGKtB8kMPeDmMdgwQJ/NMwdVPRx5r3MQAJ9tv2BcIvKkIEmSSejDo+JlKGphj4axiR/jHncErcKF8fYz2JlQgZeXHiL8RpLJOU5KiwtmWR1LskskZ6i58yDnfYDoeZkgxMuWeDWAV38FpYu+JuSm4fqsnjkYHFyDFtU9Sc3SMOC7I1xNzLvgR3lk0s9nqD9zJ+dupyBJkvFLbKBWFRfk6FigWIp/4gGwd8Bu0EY+GTOEmpWcuZV0n7Cjpj1Wp1KwqNmScInLD4aMXm4qXFS2F97ShBAvW+PbAgZsAHsHuPgHbBmlz+75ADu5jOWDWxDk5UpcShYDlh8hMS3/4VN5IT4101hn8dsDl43pYHISUMGRj+2/p6fdYbQye+i3Dmo8g18FJzaNakNTP49c55SGnpcl4RJiyJg/QrxKAwHPQd+1+myeET/DH+/qs3w+oKKLivUjW1O9ohOxKZnsjIy1obHFT1xKJs/O3Wt8L0kP618+ROKlu8sZaL8XrSRje505ENjReLSCs5Kwka3pULcKgNFJXjp8XvphozmL8EU2ifwR4lVaqNMZXloOyOD4Stg9y0TAKrmoaB+k/yJGl/PCHbO2RpKdI7e/xMMqTAbG2m2hSfRaAKZoRnLO84Vc13FS2rNiaEsOT32Bfk/5GffZGoOvLsYMn5fIW58/QrxKEw1fhh4L9a//WQR/zzc57P8g/1N5rTqUpdEyJuwkOx7pWUqSRHzKw6HyMLsdTFL8DMBs9WB+1rZDk08oiUwmw8fdkdYBFXBU2PF0TdtnQTEMG2OTMwsNlBUBqvlj+58hgSkthumzsf45Hf6aAyo3aK0vOWcYbsQmZyJJEmqtZFE2ztJMplpL96/+NqZ+yYnEw2HjK3b7+FCh73HdbzOZlX81BfSLlwvihbpeRHzYCXs72/9/VXFVYSeXodZKJKZlGVdhPIpWJ3HtwfrWWsLnlQvbf5KC3LR5G56frH+9/T04FQboU+eA/o961LqTPPvZX/kuUC5rzN1+IU/hAkDS+8G6yo8y1/47AFZL3XEImWpsUph4AaVCuEBvh9eDitcF+b1u3btvTBf9uJWLyiOl49MU5Kb9NGg9Sv966xg4tw27B0UEz9xKZkdkLPGpWTz72d4yn301PjUzz8SCBiQkKsUeYJFiMXYyiR817fnR/U1kOeqGarRlawWCYehYUJS9IUyiRkUn7MSC7FwI8SqtyGTQ+VNo+ipIOtj0GpXi9OmkH037/8zcv4pcyLQ08MuJgiv+1Mo4w6i4j1DKtPxP+zQfaEbg+0j+d+cyFgNlTqyXCJMomFIhXt988w01atTAwcGB1q1b8++/eSfsA31RDplMZrI5OJTTHEdyOfT8Cur3Ap2a+vtH0UJ2Mc+mn/5xvoSNsx6yAjoVDWRXGR3zAQ5k8Ze2KRPVo9Ehx/fBMGreK00IrlmRsQ/qAJQVfDwKD5cQYRIFY3Px+umnn5g4cSKzZs3i5MmTNGnShM6dOxMfH5/vOW5ubsTExBi369fLcdoYuR28tAJqh2CnzWSV8nMayK7larYzMrbMBq/mp121ZTf5QRmKoy6dI7p6jFKPR/1gjsnXU9/z+r8Wvvz4xtN4OitLyFrrYE64hAiTKBibi9eCBQsYOXIkw4cPp379+ixbtgwnJydWrlyZ7zkymQxvb2/jZmnJtTKHvRL6/kCq11O4ye6zVhlKLZnpUEutlfjlxE2m/HKmzGRg3Xb6Np/8nrsCEICvLJ51ylAqyNL4z74Or2e/i1auws3BIF5l24Fd1Yxc9iJMomBsKl7Z2dmcOHGCkJAQ4z65XE5ISAiHDx/O97y0tDSqV6+On58fvXr1IjIyMt+2WVlZpKSkmGxlEqUTVzutJEJXg4qyVNYpQ/GVJZg0Cd1+gQ3HbvDBltxppksj434M57u/r/LXBdNedhXusV7xKd6ye1zU+TKaaaThRBVXlTHI1L+i+TUPSyOF+bzSsjTEPYhtE2ESeWNT8UpMTESr1ebqOXl5eREbm/cSmKCgIFauXMnWrVtZt24dOp2ONm3acPPmzTzbh4aG4u7ubtz8/Pys/hwlhdzBnSHZU/hPVw0f2V3WKT6lMrmr6Px79a4NrCs6ORdce5LCOuWnVJfHc03nxavZU4lK0w8Jq3o4MrVbXca2r019H7f8LlcmMGRUvZOenWe1qKsPhowVnZW4OylyHReUgmGjpQQHBzNkyBCaNm1K27Zt+fXXX6lcuTLffvttnu2nTp1KcnKycbtx40ae7coCcpmMe7jxavY0onWVqSGPY50yFA9MM61maXS5simUJtYfvc5LS/4xvjdEAbiQwRrlZ9SR3yJGqsCr6mkk4GlsV8fblV5NqzGpcxCygrz8ZQB3R4Uxw0VMcu6hoxgyFo5NxatSpUrY2dkRFxdnsj8uLg5vb2+zrqFQKGjWrBlRUbkLWgCoVCrc3NxMtrKK/YM4r3g8GaSeRqzkSZD8JquVn+GM6fBj2uYIxm8It4WZhfLB5rOcjE4yvv8vLg0HsvheOY/G8qvckVx5NXsqN6XKJufV9XYtYUuLD5lMVuDQ8bIIkygUm4qXUqmkRYsW7Nmzx7hPp9OxZ88egoODzbqGVqslIiICHx+f4jKz1JAzUPGGpB9S3ZVcaCq/wvfKeagwzXe15dRtsm1QFdpSFGhYqlhIa/kFUiRHhmRP4bJULVe7IK/yI15QcHYJESZRODYfNk6cOJHvvvuONWvWcP78eUaNGkV6ejrDhw8HYMiQIUyd+nAZyOzZs/nzzz+5cuUKJ0+e5NVXX+X69eu8/vrrtnqEEsP+kSjrKMmXIdlTyLJz5mn5eZYoFqHAtJBqnenbGbn2eEmaaRFydHyp+Ib2dqfJkFQMz55MpBSQZ9u63mW315wX1QroeRnCJGqJMIl8sXlYcr9+/UhISGDmzJnExsbStGlTduzYYXTiR0dHI8+xDOTevXuMHDmS2NhYPD09adGiBYcOHaJ+/fq2eoQSQ56Hn+esVJPNdefTO3IcHezC+ZIljFOPRZfjd2nXubhc55UGZOgItV/Bi3ZHyZLseVM9gROSPu11XW9XLiekoX6w7MfbzaHcOa7zWyKk00nGjLmi55U/NhcvgLFjxzJ27Ng8j+3bt8/k/ZdffsmXX35ZAlaVPgw+r0e57tqMD1TvE5r5KS/aHSFVcmSq5nVyhn+mZKpxc7Ddl//j385xIvoea15r9WCPxHT79fSz34dWkjFOPZa/dY2N7X8Y0Zq3fzzJkSv6mdO6PuVryAg5wiUeCVSNTcnkvlqLvVyGX4WyHRJSnNh82Cgwn/wW52p1EhEOT/GOeixaScYA+718YL8efTIZPe2+2GeTIquSJPH+pjOsOHiV8Ogk/ozU9wLH2//CCPvtAExWv8lOXSvjOf2f8qOyqwrXHGI7qm3J11gsbqrm4/MyDBn9KzoZM4kIciP+Z8oQ9vK8Py6tTp/Xa7uuNVM0IwEYaf8H4+w2G9vcTc+m1rQ/kB5d1V3MDFn5Lz8dfxieotHqGGH3B+PtfwVgpnoov+ieNznnnZBA/bnB1Qmp58Vvbz9La0Pdy3JEztnGnJ+LMUxCzDQWiBCvMoRdPrFNWp3E/Wx9oOPP2nZ8pB4MwETFJl6z227S9vuDV4vXyEf4+1Kiyfsbu5cxQ7EOgC/UfVmr7Wxy/MjUDsa6hs8FVmbF0JY0rOZeMsaWMIZq6plqHUkZDwuMPHTWC39XQQjxKkPY5ePz0uokUjMfzjKu0nZlgfr/AJip+IFX7PYZj338+3mbDB8BXpQf5t2sJQAs0/TgG20v4zGZDBb2a2r8Qj8JOCjsqOSSOynhZREmYRalwmEvMI/88tFpdBIpj1SG/krbB1dZBiPt/2Cu/XekSw78oXsa0KdcLu78V6PXnyAx7WHcWXt5OF8qliCXSazTdGCupj85JxSufNqtzEfNF4WqHg4kpmVxO+m+sYcpskmYh+h5lSFU9g/Ldh2e+rBajk4nkal+NBhVxieaQfyoaY+dTGKh4hvayU8BerE7euWOcahpbTaduMkfEbHGNZatZedZqliIQqZli7YNMzTDySlc4TM6PpHCBQ+zSxiWCN3P1hpnH2tWEj2vghA9rzKEnVzG6Zmd0EmSSf4qbR5O+IrOSu6kZ/OBZgQusvv0sDvCMsWXDMmewpK9Nfn2wBWeC6zEDyNaW93OeTsfJkxsLLvM98ovcJCp2aVtwST1W0g5fjMvf9rtiU5x/OgSoauJ6UiSfu1jhTKWo6ykET2vMoa7kyJX4j2tTuK5wEom+4Y/UwNPJwU65ExQj2aPthkOMjXfK+dx6O/dQG5n+uOSfF9NnenbjZV+6shusEb5GS6yTP7RNmCs+m00OX4v2wVVfqKFCx5mlzD4vHIuyH5Se6PmIsSrHKDVSSzq34w5vRsa97k7KTk5Q19BWoM9o9XvcFhbH1fZfdYo5xIo06cQunE3g7O3kq1ix0f/izSupawui2WdMhRPWRondbUZqX6XLB6KbrdG3nzZt6lV7luWebTnJfLWm48Qr3KAVidRwVnJ4KerM/+VJrzY2Ie+LX2RyWRcm9sdgCyUvK5+l1O6mlSQpbFO+Sl+sjie+3wvL359kOt38ik7ZgGX4/W9Bm/usF75KVVkSZzX+TMsezIZPJxFPD+7C0sGtShzqZuLA+MSoQc+L7Eg23yEeJUDNLqHzvqXW/iyeGBzE+e+gXQcGZb9Phd0fnjJkghTfIoXeqf6mZuP1/vacTaW0zeTqUDKgyyviVzReTM4eyopPOxFvNMhEEdlbtueVAzDxriUTNRaHVcSRYyXuQjxKgcUFrcVNvKhUz4JVwZnT+Gazgs/eYI+TzwpZhVtzY895+N4a90J3EhnrXIuteW3uSVV5NXsaSSin/5vU6siv497lgkd6xT5PuWRSs4qFHYydJK+EroIkzAfIV7lgMLEq00tU2d+Ap68qp7GbakCgfJbrFHO5VDkFV5eeohLcan5XCVvvt1/mRFrjuNIJt8rv6Ch/BoJkj7b62309w0b2ZqwkU/ToGr5jJR/HORymXFFwZmbyaRlaZDLoHoZz9FfEgjxKgdYWiz6nQ6B3JQqMzh7KomSG43k1+h7aRKR12Pp+OUBs68jSRKh2y+gRM23ii95Sv4fyZITQ7KnclXSJ4e8+HGXXOIpMMUwdDwYpS+o4uvplOewX2CKEK9ygFZn/pBvSte6TOhYh1MzO3JZqsbQ7CmkSE60kl/kW8WXKFFz+PIds6617fRt7NDylWIxz9tFkC6pGJb9Puel6oA+kFZ8CQvHEKhqCF0RznrzEOJVDrBkraIhntXDSUmtys5ESjUYnv0eGZKKtnZnWKj4hle/+4eLsQUPH6Pi0xi/4SSfK5bTxe4YWZI9I9XvEi7pM0Jcm9vdOBwSFIxhxvHmPUNkvfB3mYMQr3KAReKVI8fXnnfbAXBCCuJN9QSyJHu62f3LXPvvmLn5TP7XkCRCFuxjlv1aXrb7G40kZ6x6HId0DaniqjKGZwjMwyBeBkTPyzyEeJUDitLzepS/dY0Zp34bjSTnFfsDdLm1iLRHFnsbGLfhFO/a/8ww+z/RSTLeVb/FLl1Lujf24dCUF/I8R5A/Bp+XASFe5iHEqxxgicO+oGSEO3VPMVn9BgDD7Xfy/ccjc7VZf/Q6Pme/5W37LQDM0Axnq+5Zzs/uwjcDm2MvMn9azKM9L1F0wzxKxV/aN998Q40aNXBwcKB169b8+++/Bbb/+eefqVu3Lg4ODjRq1Ig//vijhCwtnVjisM9Pu7zcVFyY04Vfdc8zQz0MgHfsN/PJB6OMFZ0lSSJy2yKmKX4EIFQ9gPXaEP6Z8oIIPH0MfHLkMHNW2lHFVWVDa8oONhevn376iYkTJzJr1ixOnjxJkyZN6Ny5M/Hx8Xm2P3ToEAMGDGDEiBGEh4fTu3dvevfuzdmzZ0vY8tKDJfGlBY0wHRR27JvUjh+0nfhc3ReADxRhfPThewB89OlHfGy/EoDFml58q+3BtbndjSW8BEXD1UGBm4N+wXrNyi5iQbaZ2Fy8FixYwMiRIxk+fDj169dn2bJlODk5sXLlyjzbL1q0iC5duvDee+9Rr1495syZQ/PmzVm8eHEJW156sKjnRd7qZeiR1ajkzIU5XVii7cUyTQ8APrFfydYZXZmevQi5TGK1phPzNH2FY96KGIaOwt9lPjYVr+zsbE6cOEFISIhxn1wuJyQkhMOHD+d5zuHDh03aA3Tu3Dnf9llZWaSkpJhs5Q1LHPb5Nc2520Fhx7W5L1KlTyjrNB2QyyR62R3CXqbjF+1zuPaez7W5Lz6e0QITDL1XESZhPjYVr8TERLRarbHArAEvLy9iY2PzPCc2Ntai9qGhobi7uxs3Pz8/6xhfCujb0heA8SGFrxc0ZOXs1sjbZP+rT/sD8F7noFzndGzgzQzNcH7VPgvADu1TPD3hR15u6f9YdgtyM7C1Py2re9KzaVVbm1JmKPeZVKdOncrEiRON71NSUsqNgH32cmPe7RSEl1vhRSu2j3+Oe+nqXAUu5vRqyJj2tfMMKHV1UHB0WkeUdp24Gh1JSGBj7O3L/Z+MTehQz4sO9bwKbygwYtO/xEqVKmFnZ0dcnGk5+ri4OLy9vfM8x9vb26L2KpUKlap8zt7IZDKzhAv0+e+93XPPCMpksgIj4as8uL5HveZFM1IgKCZsOmxUKpW0aNGCPXv2GPfpdDr27NlDcHBwnucEBwebtAfYtWtXvu0FAkH5xOZjgIkTJzJ06FBatmxJq1atWLhwIenp6QwfPhyAIUOGUK1aNUJDQwF45513aNu2LfPnz6d79+5s2LCB48ePs3z5cls+hkAgKGFsLl79+vUjISGBmTNnEhsbS9OmTdmxY4fRKR8dHY08R5n7Nm3aEBYWxvTp05k2bRqBgYFs2bKFhg0b5ncLgUBQDpFJBa0XKYekpKTg7u5OcnIybm5utjZHIBA8grnfUZsHqQoEAkFREOIlEAjKJEK8BAJBmcTmDvuSxuDiK4/LhASC8oDhu1mYO/6JE6/UVH164/ISZS8QlFdSU1Nxd8+/4tQTN9uo0+m4ffs2rq6uxZZ6xLAE6caNG+VmRrO8PVN5ex4oP88kSRKpqalUrVrVJEzqUZ64npdcLsfX17dE7uXm5lam/4jyorw9U3l7Higfz1RQj8uAcNgLBIIyiRAvgUBQJhHiVQyoVCpmzZpVrrJZlLdnKm/PA+XzmQriiXPYCwSC8oHoeQkEgjKJEC+BQFAmEeIlEAjKJEK8BAJBmUSIVxG4e/cugwYNws3NDQ8PD0aMGEFaWlqB7d9++22CgoJwdHTE39+fcePGkZycbNJOJpPl2jZs2FAsz2DtKuWSJDFz5kx8fHxwdHQkJCSES5cuFYvt+WHJM3333Xc899xzeHp64unpSUhISK72w4YNy/V5dOnSpbgfwwRLnmn16tW57HVwMK1xUBo+J6shCSymS5cuUpMmTaQjR45If//9t1S7dm1pwIAB+baPiIiQXnrpJWnbtm1SVFSUtGfPHikwMFB6+eWXTdoB0qpVq6SYmBjjdv/+favbv2HDBkmpVEorV66UIiMjpZEjR0oeHh5SXFxcnu3/+ecfyc7OTvr888+lc+fOSdOnT5cUCoUUERFhbDN37lzJ3d1d2rJli3T69GmpZ8+eUkBAQLHYb41nGjhwoPTNN99I4eHh0vnz56Vhw4ZJ7u7u0s2bN41thg4dKnXp0sXk87h7926JPI8kWf5Mq1atktzc3EzsjY2NNWlj68/JmgjxspBz585JgHTs2DHjvu3bt0symUy6deuW2dfZuHGjpFQqJbVabdwHSJs3b7amuXnSqlUracyYMcb3Wq1Wqlq1qhQaGppn+759+0rdu3c32de6dWvpzTfflCRJknQ6neTt7S198cUXxuNJSUmSSqWSfvzxx2J4gtxY+kyPotFoJFdXV2nNmjXGfUOHDpV69eplbVPNxtJnWrVqleTu7p7v9UrD52RNxLDRQg4fPoyHhwctW7Y07gsJCUEul3P06FGzr2NIcftoHcQxY8ZQqVIlWrVqxcqVKwtNC2IpxVGl/OrVq8TGxpq0cXd3p3Xr1vle05oU5ZkeJSMjA7VaTYUKFUz279u3jypVqhAUFMSoUaO4c+eOVW3Pj6I+U1paGtWrV8fPz49evXoRGRlpPGbrz8naCPGykNjYWKpUqWKyz97engoVKuRbtftREhMTmTNnDm+88YbJ/tmzZ7Nx40Z27drFyy+/zOjRo/n666+tZrvh3tauUm7415JrWpOiPNOjvP/++1StWtXki92lSxfWrl3Lnj17+Oyzz9i/fz9du3ZFq9Va1f68KMozBQUFsXLlSrZu3cq6devQ6XS0adOGmzdvArb/nKzNE5dVIj+mTJnCZ599VmCb8+fPP/Z9UlJS6N69O/Xr1+fDDz80OTZjxgzj62bNmpGens4XX3zBuHHjHvu+gvyZO3cuGzZsYN++fSYO7v79+xtfN2rUiMaNG1OrVi327dtHhw4dbGFqgQQHB5vUL23Tpg316tXj22+/Zc6cOTa0rHgQPa8HvPvuu5w/f77ArWbNmnh7exMfH29yrkaj4e7du/lW7TaQmppKly5dcHV1ZfPmzSgUigLbt27dmps3b5KVlfXYz2egOKqUG/615JrWpCjPZGDevHnMnTuXP//8k8aNGxfYtmbNmlSqVImoqKjHtrkwHueZDCgUCpo1a2a019afk7UR4vWAypUrU7du3QI3pVJJcHAwSUlJnDhxwnjuX3/9hU6no3Xr1vlePyUlhU6dOqFUKtm2bVuuKey8OHXqFJ6enlZdaFscVcoDAgLw9vY2aZOSksLRo0dLpJJ5UZ4J4PPPP2fOnDns2LHDxIeZHzdv3uTOnTv4+PhYxe6CKOoz5USr1RIREWG019afk9Wx9YxBWaRLly5Ss2bNpKNHj0oHDx6UAgMDTUIlbt68KQUFBUlHjx6VJEmSkpOTpdatW0uNGjWSoqKiTKayNRqNJEmStG3bNum7776TIiIipEuXLklLliyRnJycpJkzZ1rd/g0bNkgqlUpavXq1dO7cOemNN96QPDw8jNPqgwcPlqZMmWJs/88//0j29vbSvHnzpPPnz0uzZs3KM1TCw8ND2rp1q3TmzBmpV69eJR4qYckzzZ07V1IqldKmTZtMPo/U1FRJkiQpNTVVmjRpknT48GHp6tWr0u7du6XmzZtLgYGBUmZmZql8po8++kjauXOndPnyZenEiRNS//79JQcHBykyMtLkuW35OVkTIV5F4M6dO9KAAQMkFxcXyc3NTRo+fLjxj16SJOnq1asSIO3du1eSJEnau3evBOS5Xb16VZIkfbhF06ZNJRcXF8nZ2Vlq0qSJtGzZMkmr1RbLM3z99deSv7+/pFQqpVatWklHjhwxHmvbtq00dOhQk/YbN26U6tSpIymVSqlBgwbS77//bnJcp9NJM2bMkLy8vCSVSiV16NBBunjxYrHYnh+WPFP16tXz/DxmzZolSZIkZWRkSJ06dZIqV64sKRQKqXr16tLIkSNzxU2VpmcaP368sa2Xl5fUrVs36eTJkybXKw2fk7UQKXEEAkGZRPi8BAJBmUSIl0AgKJMI8RIIBGUSIV4CgaBMIsRLIBCUSYR4CQSCMokQL4FAUCYR4iUQCMokQrwEpYK8Ui7bIu2yoOwgUuIISg1dunRh1apVJvvyW5SuVqtzZeXIzs5GqVRafN+iniewLaLnJSg1qFQqvL29TTZPT09AX5xk6dKl9OzZE2dnZz755BM+/PBDmjZtyooVKwgICDBm6oiOjqZXr164uLjg5uZG3759TdLA5HeeoGwhxEtQZvjwww/p06cPERERvPbaawBERUXxyy+/8Ouvv3Lq1Cl0Oh29evXi7t277N+/n127dnHlyhX69etncq1HzxOUPcSwUVBq+O2333BxcTHZN23aNKZNmwbAwIEDGT58uMnx7Oxs1q5dS+XKlQF9nrGIiAiuXr2Kn58fAGvXrqVBgwYcO3aMp556Ks/zBGUPIV6CUkP79u1ZunSpyb6cBTHyShhYvXp1EwE6f/48fn5+RuECqF+/Ph4eHpw/f94oXo+eJyh7CPESlBqcnZ2pXbt2gcfN2WfuvQRlG+HzEpQr6tWrx40bN7hx44Zx37lz50hKSqJ+/fo2tExgbUTPS1BqyMrKylWCy97enkqVKpl9jZCQEBo1asSgQYNYuHAhGo2G0aNH07ZtW7Py1AvKDqLnJSg17NixAx8fH5Pt2WeftegaMpmMrVu34unpyfPPP09ISAg1a9bkp59+KiarBbZCpIEWCARlEtHzEggEZRIhXgKBoEwixEsgEJRJhHgJBIIyiRAvgUBQJhHiJRAIyiRCvAQCQZlEiJdAICiTCPESCARlEiFeAoGgTCLESyAQlEmEeAkEgjLJ/wNof6Vm71NJ9gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018703, + "end_time": "2024-03-22T22:04:08.502709", + "exception": false, + "start_time": "2024-03-22T22:04:08.484006", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4561.24704, + "end_time": "2024-03-22T22:04:11.242936", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/tvae/1/mlu-eval.ipynb", + "output_path": "eval/treatment/tvae/1/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/tvae/1", + "path_prefix": "../../../../", + "random_seed": 1, + "single_model": "tvae" + }, + "start_time": "2024-03-22T20:48:09.995896", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/tvae/model.pt b/treatment/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..2be747e8c0bed9f32e10e12fb96e0e7b0074f230 --- /dev/null +++ b/treatment/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04d6c66a58a73055b4a0300eec7e1c49f3ceab2928b62346543d551b698c360d +size 74860097 diff --git a/treatment/tvae/params.json b/treatment/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..32c61a74f7ac873e0f31157b3d4a5515e0a76ec2 --- /dev/null +++ b/treatment/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "shared", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.8, "gradient_penalty_mode": "NONE", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.1, "loss_balancer_beta": 0.73, "loss_balancer_r": 0.94, "dataset_size": 2048, "batch_size": 4, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 220, "Optim": "diffgrad", "fixed_role_model": "tvae", "mse_mag": false, "mse_mag_target": 0.2, "mse_mag_multiply": false, "d_model": 512, "attn_activation": "leakyhardsigmoid", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds": 64, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid": 128, "head_n_layers": 8, "head_n_head": 64, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file